mirrors/flatcar-scripts
1
0
Fork 0
mirror of https://github.com/flatcar/scripts.git synced 2025-08-09 05:56:58 +02:00
Code Issues Packages Projects Releases Wiki Activity

build_jobs: import current jenkins build scripts

Browse Source
This commit is contained in:
Michael Marineau 2016-10-21 13:44:31 -07:00
parent 61b6ac2d89
commit 7a2fbfe67d
6 changed files with 565 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

114
build_jobs/01_manifest.sh Normal file
View File

@ -0,0 +1,114 @@
#!/bin/bash
#
# Jenkins job for creating build manifests.
#
# Input Parameters:
#
# MANIFEST_REF=master
# Git branch or tag in github.com/coreos/manifest to build
#
# LOCAL_MANIFEST=
# Repo local manifest to amend the branch's default manifest with.
# https://wiki.cyanogenmod.org/w/Doc:_Using_manifests#The_local_manifest
#
# Input Artifacts:
#
# $WORKSPACE/bin/cork from a recent mantle build.
#
# Git:
#
# github.com/coreos/manifest checked out to $WORKSPACE/manifest
# SSH push access to github.com/coreos/manifest-builds
#
# Output:
#
# Pushes build tag to manifest-builds.
# Writes manifest.properties w/ parameters for sdk and toolchain jobs.
set -ex
export GIT_SSH_COMMAND="ssh -o StrictHostKeyChecking=no"
finish() {
local tag="$1"
git -C "${WORKSPACE}/manifest" push \
"ssh://git@github.com/coreos/manifest-builds.git" \
"refs/tags/${tag}:refs/tags/${tag}"
tee "${WORKSPACE}/manifest.properties" <<EOF
MANIFEST_URL = https://github.com/coreos/manifest-builds.git
MANIFEST_REF = refs/tags/${tag}
MANIFEST_NAME = release.xml
COREOS_OFFICIAL = ${COREOS_OFFICIAL:-0}
EOF
}
# Branches are of the form remote-name/branch-name. Tags are just tag-name.
# If we have a release tag use it, for branches we need to make a tag.
COREOS_OFFICIAL=0
if [[ "${GIT_BRANCH}" != */* ]]; then
COREOS_OFFICIAL=1
finish "${GIT_BRANCH}"
exit
fi
MANIFEST_BRANCH="${GIT_BRANCH##*/}"
MANIFEST_NAME="${MANIFEST_BRANCH}.xml"
[[ -f "manifest/${MANIFEST_NAME}" ]]
source manifest/version.txt
export COREOS_BUILD_ID="${MANIFEST_BRANCH}-${BUILD_NUMBER}"
# hack to get repo to set things up using the manifest repo we already have
# (amazing that it tolerates this considering it usually is so intolerant)
mkdir -p .repo
ln -sfT ../manifest .repo/manifests
ln -sfT ../manifest/.git .repo/manifests.git
# Cleanup/setup local manifests
rm -rf .repo/local_manifests
if [[ -n "${LOCAL_MANIFEST}" ]]; then
mkdir -p .repo/local_manifests
cat >.repo/local_manifests/local.xml <<<"${LOCAL_MANIFEST}"
fi
./bin/cork update --create --downgrade-replace --verbose \
--manifest-url "${GIT_URL}" \
--manifest-branch "${GIT_COMMIT}" \
--manifest-name "${MANIFEST_NAME}" \
--new-version "${COREOS_VERSION}" \
--sdk-version "${COREOS_SDK_VERSION}"
./bin/cork enter --experimental -- sh -c \
"pwd; repo manifest -r > '/mnt/host/source/manifest/${COREOS_BUILD_ID}.xml'"
cd manifest
git add "${COREOS_BUILD_ID}.xml"
ln -sf "${COREOS_BUILD_ID}.xml" default.xml
ln -sf "${COREOS_BUILD_ID}.xml" release.xml
git add default.xml release.xml
tee version.txt <<EOF
COREOS_VERSION=${COREOS_VERSION_ID}+${COREOS_BUILD_ID}
COREOS_VERSION_ID=${COREOS_VERSION_ID}
COREOS_BUILD_ID=${COREOS_BUILD_ID}
COREOS_SDK_VERSION=${COREOS_SDK_VERSION}
EOF
git add version.txt
EMAIL="jenkins@jenkins.coreos.systems"
GIT_AUTHOR_NAME="CoreOS Jenkins"
GIT_COMMITTER_NAME="${GIT_AUTHOR_NAME}"
export EMAIL GIT_AUTHOR_NAME GIT_COMMITTER_NAME
git commit \
-m "${COREOS_BUILD_ID}: add build manifest" \
-m "Based on ${GIT_URL} branch ${MANIFEST_BRANCH}" \
-m "${BUILD_URL}"
git tag -m "${COREOS_BUILD_ID}" "${COREOS_BUILD_ID}" HEAD
# assert that what we just did will work, update symlink because verify doesn't have a --manifest-name option yet
cd "${WORKSPACE}"
ln -sf "manifests/${COREOS_BUILD_ID}.xml" .repo/manifest.xml
./bin/cork verify
finish "${COREOS_BUILD_ID}"

71
build_jobs/02_sdk.sh Normal file
View File

@ -0,0 +1,71 @@
#!/bin/bash
#
# Jenkins job for building the SDK tarball.
#
# Input Parameters:
#
# USE_CACHE=false
# Enable use of any binary packages cached locally from previous builds.
# Currently not safe to enable, particularly bad with multiple branches.
#
# MANIFEST_URL=https://github.com/coreos/manifest-builds.git
# MANIFEST_REF=refs/tags/
# MANIFEST_NAME=release.xml
# Git URL, tag, and manifest file for this build.
#
# COREOS_OFFICIAL=0
# Set to 1 when building official releases.
#
# Input Artifacts:
#
# $WORKSPACE/bin/cork from a recent mantle build.
#
# Secrets:
#
# GPG_SECRET_KEY_FILE=
# Exported GPG public/private key used to sign uploaded files.
#
# GOOGLE_APPLICATION_CREDENTIALS=
# JSON file defining a Google service account for uploading files.
#
# Output:
#
# Uploads binary packages and tarball to gs://builds.developer.core-os.net
set -ex
# build may not be started without a ref value
[[ -n "${MANIFEST_REF#refs/tags/}" ]]
enter() {
./bin/cork enter --experimental -- "$@"
}
# hack because catalyst leaves things chowned as root
[[ -d .cache/sdks ]] && sudo chown -R $USER .cache/sdks
./bin/cork update --create --downgrade-replace --verify --verbose \
--manifest-url "${MANIFEST_URL}" \
--manifest-branch "${MANIFEST_REF}" \
--manifest-name "${MANIFEST_NAME}"
source .repo/manifests/version.txt
export COREOS_BUILD_ID
# Set up GPG for signing images
export GNUPGHOME="${PWD}/.gnupg"
sudo rm -rf "${GNUPGHOME}"
trap "sudo rm -rf '${GNUPGHOME}'" EXIT
mkdir --mode=0700 "${GNUPGHOME}"
gpg --import "${GPG_SECRET_KEY_FILE}"
# Wipe all of catalyst or just clear out old tarballs taking up space
if [[ "${COREOS_OFFICIAL:-0}" -eq 1 || "$USE_CACHE" == false ]]; then
sudo rm -rf src/build/catalyst
fi
sudo rm -rf src/build/catalyst/builds
enter sudo emerge -uv --jobs=2 catalyst
enter sudo /mnt/host/source/src/scripts/bootstrap_sdk \
--sign buildbot@coreos.com --sign_digests buildbot@coreos.com \
--upload --upload_root gs://builds.developer.core-os.net

71
build_jobs/02_toolchains.sh Normal file
View File

@ -0,0 +1,71 @@
#!/bin/bash
#
# Jenkins job for building the SDK's cross toolchains.
#
# Input Parameters:
#
# USE_CACHE=false
# Enable use of any binary packages cached locally from previous builds.
# Currently not safe to enable, particularly bad with multiple branches.
#
# MANIFEST_URL=https://github.com/coreos/manifest-builds.git
# MANIFEST_REF=refs/tags/
# MANIFEST_NAME=release.xml
# Git URL, tag, and manifest file for this build.
#
# COREOS_OFFICIAL=0
# Set to 1 when building official releases.
#
# Input Artifacts:
#
# $WORKSPACE/bin/cork from a recent mantle build.
#
# Secrets:
#
# GPG_SECRET_KEY_FILE=
# Exported GPG public/private key used to sign uploaded files.
#
# GOOGLE_APPLICATION_CREDENTIALS=
# JSON file defining a Google service account for uploading files.
#
# Output:
#
# Uploads binary packages to gs://builds.developer.core-os.net
set -ex
# build may not be started without a ref value
[[ -n "${MANIFEST_REF#refs/tags/}" ]]
enter() {
./bin/cork enter --experimental -- "$@"
}
# hack because catalyst leaves things chowned as root
[[ -d .cache/sdks ]] && sudo chown -R $USER .cache/sdks
./bin/cork update --create --downgrade-replace --verify --verbose \
--manifest-url "${MANIFEST_URL}" \
--manifest-branch "${MANIFEST_REF}" \
--manifest-name "${MANIFEST_NAME}"
source .repo/manifests/version.txt
export COREOS_BUILD_ID
# Set up GPG for signing images
export GNUPGHOME="${PWD}/.gnupg"
sudo rm -rf "${GNUPGHOME}"
trap "sudo rm -rf '${GNUPGHOME}'" EXIT
mkdir --mode=0700 "${GNUPGHOME}"
gpg --import "${GPG_SECRET_KEY_FILE}"
# Wipe all of catalyst or just clear out old tarballs taking up space
if [[ "${COREOS_OFFICIAL:-0}" -eq 1 || "$USE_CACHE" == false ]]; then
sudo rm -rf src/build/catalyst
fi
sudo rm -rf src/build/catalyst/builds
enter sudo emerge -uv --jobs=2 catalyst
enter sudo /mnt/host/source/src/scripts/build_toolchains \
--sign buildbot@coreos.com --sign_digests buildbot@coreos.com \
--upload --upload_root gs://builds.developer.core-os.net

81
build_jobs/03_packages.sh Normal file
View File

@ -0,0 +1,81 @@
#!/bin/bash
#
# Jenkins job for building board binary packages.
#
# Input Parameters:
#
# MANIFEST_URL=https://github.com/coreos/manifest-builds.git
# MANIFEST_REF=refs/tags/
# MANIFEST_NAME=release.xml
# Git URL, tag, and manifest file for this build.
#
# COREOS_OFFICIAL=0
# Set to 1 when building official releases.
#
# BOARD=amd64-usr
# Target board to build.
#
# Input Artifacts:
#
# $WORKSPACE/bin/cork from a recent mantle build.
#
# Secrets:
#
# GOOGLE_APPLICATION_CREDENTIALS=
# JSON file defining a Google service account for uploading files.
#
# Output:
#
# Uploads binary packages to gs://builds.developer.core-os.net
set -ex
# build may not be started without a ref value
[[ -n "${MANIFEST_REF#refs/tags/}" ]]
# use a ccache dir that persists across sdk recreations
# XXX: alternatively use a ccache dir that is usable by all jobs on a given node.
mkdir -p .cache/ccache
enter() {
./bin/cork enter --experimental -- env \
CCACHE_DIR="/mnt/host/source/.cache/ccache" \
CCACHE_MAXSIZE="5G" "$@"
}
script() {
local script="/mnt/host/source/src/scripts/${1}"; shift
enter "${script}" "$@"
}
./bin/cork update --create --downgrade-replace --verify --verbose \
--manifest-url "${MANIFEST_URL}" \
--manifest-branch "${MANIFEST_REF}" \
--manifest-name "${MANIFEST_NAME}" \
-- --toolchain_boards=${BOARD}
source .repo/manifests/version.txt
export COREOS_BUILD_ID
if [[ "${COREOS_VERSION}" == 1010.* && "${BOARD}" == arm64-usr ]]; then
echo "SKIPPING ARM"
exit 0
fi
# figure out if ccache is doing us any good in this scheme
enter ccache --zero-stats
#if [[ "${COREOS_OFFICIAL:-0}" -eq 1 ]]; then
script setup_board --board=${BOARD} \
--skip_chroot_upgrade \
--getbinpkgver=${COREOS_VERSION} \
--toolchainpkgonly \
--force
#fi
script build_packages --board=${BOARD} \
--skip_chroot_upgrade \
--getbinpkgver=${COREOS_VERSION} \
--toolchainpkgonly \
--upload --upload_root gs://builds.developer.core-os.net
enter ccache --show-stats

109
build_jobs/04_images.sh Normal file
View File

@ -0,0 +1,109 @@
#!/bin/bash
#
# Jenkins job for building the base production image and dev container.
#
# Input Parameters:
#
# USE_CACHE=false
# Enable use of any binary packages cached locally from previous builds.
# Currently not safe to enable, particularly bad with multiple branches.
#
# MANIFEST_URL=https://github.com/coreos/manifest-builds.git
# MANIFEST_REF=refs/tags/
# MANIFEST_NAME=release.xml
# Git URL, tag, and manifest file for this build.
#
# COREOS_OFFICIAL=0
# Set to 1 when building official releases.
#
# BOARD=amd64-usr
# Target board to build.
#
# Input Artifacts:
#
# $WORKSPACE/bin/cork from a recent mantle build.
#
# Secrets:
#
# GPG_SECRET_KEY_FILE=
# Exported GPG public/private key used to sign uploaded files.
#
# GOOGLE_APPLICATION_CREDENTIALS=
# JSON file defining a Google service account for uploading files.
#
# Output:
#
# Uploads test branch images to gs://builds.developer.core-os.net and
# official images to gs://builds.release.core-os.net
set -ex
# build may not be started without a ref value
[[ -n "${MANIFEST_REF#refs/tags/}" ]]
# first thing, clear out old images
sudo rm -rf src/build
script() {
local script="/mnt/host/source/src/scripts/${1}"; shift
./bin/cork enter --experimental -- "${script}" "$@"
}
./bin/cork update --create --downgrade-replace --verify --verbose \
--manifest-url "${MANIFEST_URL}" \
--manifest-branch "${MANIFEST_REF}" \
--manifest-name "${MANIFEST_NAME}"
source .repo/manifests/version.txt
export COREOS_BUILD_ID
if [[ "${COREOS_VERSION}" == 1010.* && "${BOARD}" == arm64-usr ]]; then
echo "SKIPPING ARM"
exit 0
fi
# Set up GPG for signing images
export GNUPGHOME="${PWD}/.gnupg"
rm -rf "${GNUPGHOME}"
trap "rm -rf '${GNUPGHOME}'" EXIT
mkdir --mode=0700 "${GNUPGHOME}"
gpg --import "${GPG_SECRET_KEY_FILE}"
sudo rm -rf chroot/build
script setup_board --board=${BOARD} \
--getbinpkgver="${COREOS_VERSION}" \
--regen_configs_only
if [[ "${COREOS_OFFICIAL}" -eq 1 ]]; then
GROUP=stable
UPLOAD=gs://builds.release.core-os.net/stable
script set_official --board=${BOARD} --official
else
GROUP=developer
UPLOAD=gs://builds.developer.core-os.net
script set_official --board=${BOARD} --noofficial
fi
script build_image --board=${BOARD} \
--group=${GROUP} \
--getbinpkg \
--getbinpkgver="${COREOS_VERSION}" \
--sign=buildbot@coreos.com \
--sign_digests=buildbot@coreos.com \
--upload_root=${UPLOAD} \
--upload prod container
if [[ "${COREOS_OFFICIAL}" -eq 1 ]]; then
script image_set_group --board=${BOARD} \
--group=alpha \
--sign=buildbot@coreos.com \
--sign_digests=buildbot@coreos.com \
--upload_root=gs://builds.release.core-os.net/alpha \
--upload
script image_set_group --board=${BOARD} \
--group=beta \
--sign=buildbot@coreos.com \
--sign_digests=buildbot@coreos.com \
--upload_root=gs://builds.release.core-os.net/beta \
--upload
fi

119
build_jobs/05_vm.sh Normal file
View File

@ -0,0 +1,119 @@
#!/bin/bash
#
# Jenkins job for building final VM and OEM target images.
#
# Input Parameters:
#
# USE_CACHE=false
# Enable use of any binary packages cached locally from previous builds.
# Currently not safe to enable, particularly bad with multiple branches.
#
# MANIFEST_URL=https://github.com/coreos/manifest-builds.git
# MANIFEST_REF=refs/tags/
# MANIFEST_NAME=release.xml
# Git URL, tag, and manifest file for this build.
#
# COREOS_OFFICIAL=0
# Set to 1 when building official releases.
#
# BOARD=amd64-usr
# Target board to build.
#
# FORMAT=qemu
# Target VM or OEM.
#
# GROUP=developer
# Target update group.
#
# Input Artifacts:
#
# $WORKSPACE/bin/cork from a recent mantle build.
#
# Secrets:
#
# GPG_SECRET_KEY_FILE=
# Exported GPG public/private key used to sign uploaded files.
#
# GOOGLE_APPLICATION_CREDENTIALS=
# JSON file defining a Google service account for uploading files.
#
# Output:
#
# Uploads test branch images to gs://builds.developer.core-os.net and
# official images to gs://builds.release.core-os.net
# Writes gce.properties for triggering a GCE test job if applicable.
set -ex
rm -f gce.properties
sudo rm -rf tmp
# build may not be started without a ref value
[[ -n "${MANIFEST_REF#refs/tags/}" ]]
# check that the matrix didn't go bananas
if [[ "${COREOS_OFFICIAL}" -eq 1 ]]; then
[[ "${GROUP}" != developer ]]
else
[[ "${GROUP}" == developer ]]
fi
script() {
local script="/mnt/host/source/src/scripts/${1}"; shift
./bin/cork enter --experimental -- "${script}" "$@"
}
enter() {
./bin/cork enter --experimental -- "$@"
}
./bin/cork update --create --downgrade-replace --verify --verbose \
--manifest-url "${MANIFEST_URL}" \
--manifest-branch "${MANIFEST_REF}" \
--manifest-name "${MANIFEST_NAME}"
source .repo/manifests/version.txt
export COREOS_BUILD_ID
if [[ "${COREOS_VERSION}" == 1010.* && "${BOARD}" == arm64-usr ]]; then
echo "SKIPPING ARM"
exit 0
fi
# Set up GPG for signing images
export GNUPGHOME="${PWD}/.gnupg"
rm -rf "${GNUPGHOME}"
trap "rm -rf '${GNUPGHOME}'" EXIT
mkdir --mode=0700 "${GNUPGHOME}"
gpg --import "${GPG_SECRET_KEY_FILE}"
if [[ "${GROUP}" == developer ]]; then
root="gs://builds.developer.core-os.net"
dlroot=""
else
root="gs://builds.release.core-os.net/${GROUP}"
dlroot="--download_root https://${GROUP}.release.core-os.net"
fi
mkdir -p src tmp
./bin/cork download-image --root="${root}/boards/${BOARD}/${COREOS_VERSION}" \
--json-key="${GOOGLE_APPLICATION_CREDENTIALS}" \
--cache-dir=./src \
--platform=qemu
img=src/coreos_production_image.bin
if [[ "${img}.bz2" -nt "${img}" ]]; then
enter lbunzip2 -k -f "/mnt/host/source/${img}.bz2"
fi
sudo rm -rf chroot/build
script image_to_vm.sh --board=${BOARD} \
--format=${FORMAT} \
--prod_image \
--getbinpkg \
--getbinpkgver=${COREOS_VERSION} \
--from=/mnt/host/source/src/ \
--to=/mnt/host/source/tmp/ \
--sign=buildbot@coreos.com \
--sign_digests=buildbot@coreos.com \
--upload_root="${root}" \
--upload ${dlroot}