diff --git a/jenkins/README.md b/jenkins/README.md new file mode 100644 index 0000000000..aed78ce79a --- /dev/null +++ b/jenkins/README.md @@ -0,0 +1,11 @@ +The scripts in this directory are run from [OS Jenkins jobs][jenkins-os]. By +storing the Jenkins scripts in this repository, they are more tightly coupled +to the release branch of the SDK scripts that they require. The Jenkins jobs +are responsible for setting up the environment and securely initializing an SDK +in the workspace before running these scripts. + +The special files named `formats-${BOARD}.txt` are space-separated lists of VM +image formats that should be built for releases on this branch; i.e. the script +`vm.sh` is run for each item in the list. + +[jenkins-os]: https://github.com/coreos/jenkins-os diff --git a/jenkins/formats-amd64-usr.txt b/jenkins/formats-amd64-usr.txt new file mode 100644 index 0000000000..9f98a63209 --- /dev/null +++ b/jenkins/formats-amd64-usr.txt @@ -0,0 +1,33 @@ +ami +ami_vmdk +azure +gce +iso +pxe +qemu +qemu_uefi +brightbox +cloudsigma +cloudstack +cloudstack_vhd +digitalocean +exoscale +hyperv +niftycloud +openstack +openstack_mini +packet +parallels +rackspace +rackspace_onmetal +rackspace_vhd +vagrant +vagrant_parallels +vagrant_virtualbox +vagrant_vmware_fusion +virtualbox +vmware +vmware_insecure +vmware_ova +vmware_raw +xen diff --git a/jenkins/formats-arm64-usr.txt b/jenkins/formats-arm64-usr.txt new file mode 100644 index 0000000000..fe6f2e2598 --- /dev/null +++ b/jenkins/formats-arm64-usr.txt @@ -0,0 +1,5 @@ +openstack +openstack_mini +packet +pxe +qemu_uefi diff --git a/jenkins/images.sh b/jenkins/images.sh new file mode 100644 index 0000000000..a2b41eb05c --- /dev/null +++ b/jenkins/images.sh @@ -0,0 +1,78 @@ +#!/bin/bash -ex + +# Clear out old images. +sudo rm -rf chroot/build src/build torcx + +enter() { + local verify_key= + trap 'sudo rm -f chroot/etc/portage/gangue.*' RETURN + [ -s verify.asc ] && + sudo ln -f verify.asc chroot/etc/portage/gangue.asc && + verify_key=--verify-key=/etc/portage/gangue.asc + sudo ln -f "${GS_DEVEL_CREDS}" chroot/etc/portage/gangue.json + bin/cork enter --experimental -- env \ + COREOS_DEV_BUILDS="${DOWNLOAD_ROOT}" \ + PORTAGE_SSH_OPTS= \ + {FETCH,RESUME}COMMAND_GS="/usr/bin/gangue get \ +--json-key=/etc/portage/gangue.json $verify_key \ +"'"${URI}" "${DISTDIR}/${FILE}"' \ + "$@" +} + +script() { + enter "/mnt/host/source/src/scripts/$@" +} + +sudo cp bin/gangue chroot/usr/bin/gangue # XXX: until SDK mantle has it + +source .repo/manifests/version.txt +export COREOS_BUILD_ID + +# Set up GPG for signing uploads. +gpg --import "${GPG_SECRET_KEY_FILE}" + +script setup_board \ + --board="${BOARD}" \ + --getbinpkgver="${COREOS_VERSION}" \ + --regen_configs_only + +if [ "x${COREOS_OFFICIAL}" == x1 ] +then + script set_official --board="${BOARD}" --official +else + script set_official --board="${BOARD}" --noofficial +fi + +# Try to find the version's torcx store, but don't require it. +torcx_store= +enter gsutil cp -r \ + "${DOWNLOAD_ROOT}/boards/${BOARD}/${COREOS_VERSION}/torcx" \ + /mnt/host/source/ && +torcx_store=/mnt/host/source/torcx && +for image in torcx/*.torcx.tgz +do + gpg --verify "${image}.sig" +done + +# Work around the lack of symlink support in GCS. +shopt -s nullglob +for default in torcx/*:com.coreos.cl.torcx.tgz +do + for image in torcx/*.torcx.tgz + do + [ "x${default}" != "x${image}" ] && + cmp --silent -- "${default}" "${image}" && + ln -fns "${image##*/}" "${default}" + done +done + +script build_image \ + --board="${BOARD}" \ + --group="${GROUP}" \ + --getbinpkg \ + --getbinpkgver="${COREOS_VERSION}" \ + --sign="${SIGNING_USER}" \ + --sign_digests="${SIGNING_USER}" \ + ${torcx_store:+--torcx_store="${torcx_store}"} \ + --upload_root="${UPLOAD_ROOT}" \ + --upload prod container diff --git a/jenkins/packages.sh b/jenkins/packages.sh new file mode 100644 index 0000000000..9702ce045c --- /dev/null +++ b/jenkins/packages.sh @@ -0,0 +1,66 @@ +#!/bin/bash -ex + +# Use a ccache dir that persists across SDK recreations. +# XXX: alternatively use a ccache dir that is usable by all jobs on a given node. +mkdir -p .cache/ccache + +enter() { + local verify_key= + trap 'sudo rm -f chroot/etc/portage/gangue.*' RETURN + [ -s verify.asc ] && + sudo ln -f verify.asc chroot/etc/portage/gangue.asc && + verify_key=--verify-key=/etc/portage/gangue.asc + sudo ln -f "${GOOGLE_APPLICATION_CREDENTIALS}" \ + chroot/etc/portage/gangue.json + bin/cork enter --experimental -- env \ + CCACHE_DIR=/mnt/host/source/.cache/ccache \ + CCACHE_MAXSIZE=5G \ + COREOS_DEV_BUILDS="${DOWNLOAD_ROOT}" \ + PORTAGE_SSH_OPTS= \ + {FETCH,RESUME}COMMAND_GS="/usr/bin/gangue get \ +--json-key=/etc/portage/gangue.json $verify_key \ +"'"${URI}" "${DISTDIR}/${FILE}"' \ + "$@" +} + +script() { + enter "/mnt/host/source/src/scripts/$@" +} + +sudo cp bin/gangue chroot/usr/bin/gangue # XXX: until SDK mantle has it + +source .repo/manifests/version.txt +export COREOS_BUILD_ID + +# Set up GPG for signing uploads. +gpg --import "${GPG_SECRET_KEY_FILE}" + +# Figure out if ccache is doing us any good in this scheme. +enter ccache --zero-stats + +script setup_board \ + --board="${BOARD}" \ + --getbinpkgver=${RELEASE_BASE:-"${COREOS_VERSION}" --toolchainpkgonly} \ + --skip_chroot_upgrade \ + --force + +script build_packages \ + --board="${BOARD}" \ + --getbinpkgver=${RELEASE_BASE:-"${COREOS_VERSION}" --toolchainpkgonly} \ + --skip_chroot_upgrade \ + $([ -x src/scripts/build_torcx_store ] && echo --skip_torcx_store) \ + --sign="${SIGNING_USER}" \ + --sign_digests="${SIGNING_USER}" \ + --upload_root="${UPLOAD_ROOT}" \ + --upload + +# Build and upload torcx images if this version supports it. +[ -x src/scripts/build_torcx_store ] && +script build_torcx_store \ + --board="${BOARD}" \ + --sign="${SIGNING_USER}" \ + --sign_digests="${SIGNING_USER}" \ + --upload_root="${UPLOAD_ROOT}" \ + --upload + +enter ccache --show-stats diff --git a/jenkins/sdk.sh b/jenkins/sdk.sh new file mode 100644 index 0000000000..bdf5e60918 --- /dev/null +++ b/jenkins/sdk.sh @@ -0,0 +1,26 @@ +#!/bin/bash -ex + +enter() { + bin/cork enter --experimental -- "$@" +} + +source .repo/manifests/version.txt +export COREOS_BUILD_ID + +# Set up GPG for signing uploads. +gpg --import "${GPG_SECRET_KEY_FILE}" + +# Wipe all of catalyst. +sudo rm -rf src/build + +S=/mnt/host/source/src/scripts +enter ${S}/update_chroot +enter sudo emerge -uv --jobs=2 catalyst +enter sudo ${S}/bootstrap_sdk \ + --sign="${SIGNING_USER}" \ + --sign_digests="${SIGNING_USER}" \ + --upload_root="${UPLOAD_ROOT}" \ + --upload + +# Free some disk space only on success to allow debugging failures. +sudo rm -rf src/build/catalyst/builds diff --git a/jenkins/toolchains.sh b/jenkins/toolchains.sh new file mode 100644 index 0000000000..a56ec9e867 --- /dev/null +++ b/jenkins/toolchains.sh @@ -0,0 +1,25 @@ +#!/bin/bash -ex + +enter() { + bin/cork enter --experimental -- "$@" +} + +source .repo/manifests/version.txt +export COREOS_BUILD_ID + +# Set up GPG for signing uploads. +gpg --import "${GPG_SECRET_KEY_FILE}" + +# Wipe all of catalyst. +sudo rm -rf src/build + +S=/mnt/host/source/src/scripts +enter sudo emerge -uv --jobs=2 catalyst +enter sudo ${S}/build_toolchains \ + --sign="${SIGNING_USER}" \ + --sign_digests="${SIGNING_USER}" \ + --upload_root="${UPLOAD_ROOT}" \ + --upload + +# Free some disk space only on success to allow debugging failures. +sudo rm -rf src/build/catalyst/builds diff --git a/jenkins/vm.sh b/jenkins/vm.sh new file mode 100644 index 0000000000..a4c79a5308 --- /dev/null +++ b/jenkins/vm.sh @@ -0,0 +1,60 @@ +#!/bin/bash -ex + +# Clear out old images. +sudo rm -rf chroot/build tmp + +enter() { + local verify_key= + trap 'sudo rm -f chroot/etc/portage/gangue.*' RETURN + [ -s verify.asc ] && + sudo ln -f verify.asc chroot/etc/portage/gangue.asc && + verify_key=--verify-key=/etc/portage/gangue.asc + sudo ln -f "${GS_DEVEL_CREDS}" chroot/etc/portage/gangue.json + bin/cork enter --experimental -- env \ + COREOS_DEV_BUILDS="${GS_DEVEL_ROOT}" \ + PORTAGE_SSH_OPTS= \ + {FETCH,RESUME}COMMAND_GS="/usr/bin/gangue get \ +--json-key=/etc/portage/gangue.json $verify_key \ +"'"${URI}" "${DISTDIR}/${FILE}"' \ + "$@" +} + +script() { + enter "/mnt/host/source/src/scripts/$@" +} + +sudo cp bin/gangue chroot/usr/bin/gangue # XXX: until SDK mantle has it + +source .repo/manifests/version.txt +export COREOS_BUILD_ID + +# Set up GPG for signing uploads. +gpg --import "${GPG_SECRET_KEY_FILE}" + +[ -s verify.asc ] && verify_key=--verify-key=verify.asc || verify_key= + +mkdir -p src tmp +bin/cork download-image \ + --root="${UPLOAD_ROOT}/boards/${BOARD}/${COREOS_VERSION}" \ + --json-key="${GOOGLE_APPLICATION_CREDENTIALS}" \ + --cache-dir=./src \ + --platform=qemu \ + --verify=true $verify_key + +img=src/coreos_production_image.bin +[[ "${img}.bz2" -nt "${img}" ]] && +enter lbunzip2 -k -f "/mnt/host/source/${img}.bz2" + +script image_to_vm.sh \ + --board="${BOARD}" \ + --format="${FORMAT}" \ + --prod_image \ + --getbinpkg \ + --getbinpkgver="${COREOS_VERSION}" \ + --from=/mnt/host/source/src \ + --to=/mnt/host/source/tmp \ + --sign="${SIGNING_USER}" \ + --sign_digests="${SIGNING_USER}" \ + --download_root="${DOWNLOAD_ROOT}" \ + --upload_root="${UPLOAD_ROOT}" \ + --upload