Merge pull request #707 from dm0-/build-1451

Add Jenkins scripts for alpha

jenkins/README.md

@@ -0,0 +1,11 @@
+The scripts in this directory are run from [OS Jenkins jobs][jenkins-os].  By
+storing the Jenkins scripts in this repository, they are more tightly coupled
+to the release branch of the SDK scripts that they require.  The Jenkins jobs
+are responsible for setting up the environment and securely initializing an SDK
+in the workspace before running these scripts.
+
+The special files named `formats-${BOARD}.txt` are space-separated lists of VM
+image formats that should be built for releases on this branch; i.e. the script
+`vm.sh` is run for each item in the list.
+
+[jenkins-os]: https://github.com/coreos/jenkins-os

jenkins/formats-amd64-usr.txt

@@ -0,0 +1,33 @@
+ami
+ami_vmdk
+azure
+gce
+iso
+pxe
+qemu
+qemu_uefi
+brightbox
+cloudsigma
+cloudstack
+cloudstack_vhd
+digitalocean
+exoscale
+hyperv
+niftycloud
+openstack
+openstack_mini
+packet
+parallels
+rackspace
+rackspace_onmetal
+rackspace_vhd
+vagrant
+vagrant_parallels
+vagrant_virtualbox
+vagrant_vmware_fusion
+virtualbox
+vmware
+vmware_insecure
+vmware_ova
+vmware_raw
+xen

jenkins/formats-arm64-usr.txt

@@ -0,0 +1,5 @@
+openstack
+openstack_mini
+packet
+pxe
+qemu_uefi

jenkins/images.sh

@@ -0,0 +1,73 @@
+#!/bin/bash -ex
+
+# Clear out old images.
+sudo rm -rf chroot/build src/build torcx
+
+enter() {
+        local verify_key=
+        trap 'sudo rm -f chroot/etc/portage/gangue.*' RETURN
+        [ -s verify.asc ] &&
+        sudo ln -f verify.asc chroot/etc/portage/gangue.asc &&
+        verify_key=--verify-key=/etc/portage/gangue.asc
+        sudo ln -f "${GS_DEVEL_CREDS}" chroot/etc/portage/gangue.json
+        bin/cork enter --experimental -- env \
+            COREOS_DEV_BUILDS="${DOWNLOAD_ROOT}" \
+            {FETCH,RESUME}COMMAND_GS="/usr/bin/gangue get \
+--json-key=/etc/portage/gangue.json $verify_key \
+"'"${URI}" "${DISTDIR}/${FILE}"' \
+            "$@"
+}
+
+script() {
+        enter "/mnt/host/source/src/scripts/$@"
+}
+
+source .repo/manifests/version.txt
+export COREOS_BUILD_ID
+
+# Set up GPG for signing uploads.
+gpg --import "${GPG_SECRET_KEY_FILE}"
+
+script setup_board \
+    --board="${BOARD}" \
+    --getbinpkgver="${COREOS_VERSION}" \
+    --regen_configs_only
+
+if [ "x${COREOS_OFFICIAL}" == x1 ]
+then
+        script set_official --board="${BOARD}" --official
+else
+        script set_official --board="${BOARD}" --noofficial
+fi
+
+# Retrieve this version's torcx vendor store.
+enter gsutil cp -r \
+    "${DOWNLOAD_ROOT}/boards/${BOARD}/${COREOS_VERSION}/torcx" \
+    /mnt/host/source/
+for image in torcx/*.torcx.tgz
+do
+        gpg --verify "${image}.sig"
+done
+
+# Work around the lack of symlink support in GCS.
+shopt -s nullglob
+for default in torcx/*:com.coreos.cl.torcx.tgz
+do
+        for image in torcx/*.torcx.tgz
+        do
+                [ "x${default}" != "x${image}" ] &&
+                cmp --silent -- "${default}" "${image}" &&
+                ln -fns "${image##*/}" "${default}"
+        done
+done
+
+script build_image \
+    --board="${BOARD}" \
+    --group="${GROUP}" \
+    --getbinpkg \
+    --getbinpkgver="${COREOS_VERSION}" \
+    --sign="${SIGNING_USER}" \
+    --sign_digests="${SIGNING_USER}" \
+    --torcx_store=/mnt/host/source/torcx \
+    --upload_root="${UPLOAD_ROOT}" \
+    --upload prod container

jenkins/packages.sh

@@ -0,0 +1,61 @@
+#!/bin/bash -ex
+
+# Use a ccache dir that persists across SDK recreations.
+# XXX: alternatively use a ccache dir that is usable by all jobs on a given node.
+mkdir -p .cache/ccache
+
+enter() {
+        local verify_key=
+        trap 'sudo rm -f chroot/etc/portage/gangue.*' RETURN
+        [ -s verify.asc ] &&
+        sudo ln -f verify.asc chroot/etc/portage/gangue.asc &&
+        verify_key=--verify-key=/etc/portage/gangue.asc
+        sudo ln -f "${GOOGLE_APPLICATION_CREDENTIALS}" \
+            chroot/etc/portage/gangue.json
+        bin/cork enter --experimental -- env \
+            CCACHE_DIR=/mnt/host/source/.cache/ccache \
+            CCACHE_MAXSIZE=5G \
+            COREOS_DEV_BUILDS="${DOWNLOAD_ROOT}" \
+            {FETCH,RESUME}COMMAND_GS="/usr/bin/gangue get \
+--json-key=/etc/portage/gangue.json $verify_key \
+"'"${URI}" "${DISTDIR}/${FILE}"' \
+            "$@"
+}
+
+script() {
+        enter "/mnt/host/source/src/scripts/$@"
+}
+
+source .repo/manifests/version.txt
+export COREOS_BUILD_ID
+
+# Set up GPG for signing uploads.
+gpg --import "${GPG_SECRET_KEY_FILE}"
+
+# Figure out if ccache is doing us any good in this scheme.
+enter ccache --zero-stats
+
+script setup_board \
+    --board="${BOARD}" \
+    --getbinpkgver=${RELEASE_BASE:-"${COREOS_VERSION}" --toolchainpkgonly} \
+    --skip_chroot_upgrade \
+    --force
+
+script build_packages \
+    --board="${BOARD}" \
+    --getbinpkgver=${RELEASE_BASE:-"${COREOS_VERSION}" --toolchainpkgonly} \
+    --skip_chroot_upgrade \
+    --skip_torcx_store \
+    --sign="${SIGNING_USER}" \
+    --sign_digests="${SIGNING_USER}" \
+    --upload_root="${UPLOAD_ROOT}" \
+    --upload
+
+script build_torcx_store \
+    --board="${BOARD}" \
+    --sign="${SIGNING_USER}" \
+    --sign_digests="${SIGNING_USER}" \
+    --upload_root="${UPLOAD_ROOT}" \
+    --upload
+
+enter ccache --show-stats

jenkins/sdk.sh

@@ -0,0 +1,26 @@
+#!/bin/bash -ex
+
+enter() {
+        bin/cork enter --experimental -- "$@"
+}
+
+source .repo/manifests/version.txt
+export COREOS_BUILD_ID
+
+# Set up GPG for signing uploads.
+gpg --import "${GPG_SECRET_KEY_FILE}"
+
+# Wipe all of catalyst.
+sudo rm -rf src/build
+
+S=/mnt/host/source/src/scripts
+enter ${S}/update_chroot
+enter sudo emerge -uv --jobs=2 catalyst
+enter sudo ${S}/bootstrap_sdk \
+    --sign="${SIGNING_USER}" \
+    --sign_digests="${SIGNING_USER}" \
+    --upload_root="${UPLOAD_ROOT}" \
+    --upload
+
+# Free some disk space only on success to allow debugging failures.
+sudo rm -rf src/build/catalyst/builds

jenkins/toolchains.sh

@@ -0,0 +1,25 @@
+#!/bin/bash -ex
+
+enter() {
+        bin/cork enter --experimental -- "$@"
+}
+
+source .repo/manifests/version.txt
+export COREOS_BUILD_ID
+
+# Set up GPG for signing uploads.
+gpg --import "${GPG_SECRET_KEY_FILE}"
+
+# Wipe all of catalyst.
+sudo rm -rf src/build
+
+S=/mnt/host/source/src/scripts
+enter sudo emerge -uv --jobs=2 catalyst
+enter sudo ${S}/build_toolchains \
+    --sign="${SIGNING_USER}" \
+    --sign_digests="${SIGNING_USER}" \
+    --upload_root="${UPLOAD_ROOT}" \
+    --upload
+
+# Free some disk space only on success to allow debugging failures.
+sudo rm -rf src/build/catalyst/builds

jenkins/vm.sh

@@ -0,0 +1,57 @@
+#!/bin/bash -ex
+
+# Clear out old images.
+sudo rm -rf chroot/build tmp
+
+enter() {
+        local verify_key=
+        trap 'sudo rm -f chroot/etc/portage/gangue.*' RETURN
+        [ -s verify.asc ] &&
+        sudo ln -f verify.asc chroot/etc/portage/gangue.asc &&
+        verify_key=--verify-key=/etc/portage/gangue.asc
+        sudo ln -f "${GS_DEVEL_CREDS}" chroot/etc/portage/gangue.json
+        bin/cork enter --experimental -- env \
+            COREOS_DEV_BUILDS="${GS_DEVEL_ROOT}" \
+            {FETCH,RESUME}COMMAND_GS="/usr/bin/gangue get \
+--json-key=/etc/portage/gangue.json $verify_key \
+"'"${URI}" "${DISTDIR}/${FILE}"' \
+            "$@"
+}
+
+script() {
+        enter "/mnt/host/source/src/scripts/$@"
+}
+
+source .repo/manifests/version.txt
+export COREOS_BUILD_ID
+
+# Set up GPG for signing uploads.
+gpg --import "${GPG_SECRET_KEY_FILE}"
+
+[ -s verify.asc ] && verify_key=--verify-key=verify.asc || verify_key=
+
+mkdir -p src tmp
+bin/cork download-image \
+    --root="${UPLOAD_ROOT}/boards/${BOARD}/${COREOS_VERSION}" \
+    --json-key="${GOOGLE_APPLICATION_CREDENTIALS}" \
+    --cache-dir=./src \
+    --platform=qemu \
+    --verify=true $verify_key
+
+img=src/coreos_production_image.bin
+[[ "${img}.bz2" -nt "${img}" ]] &&
+enter lbunzip2 -k -f "/mnt/host/source/${img}.bz2"
+
+script image_to_vm.sh \
+    --board="${BOARD}" \
+    --format="${FORMAT}" \
+    --prod_image \
+    --getbinpkg \
+    --getbinpkgver="${COREOS_VERSION}" \
+    --from=/mnt/host/source/src \
+    --to=/mnt/host/source/tmp \
+    --sign="${SIGNING_USER}" \
+    --sign_digests="${SIGNING_USER}" \
+    --download_root="${DOWNLOAD_ROOT}" \
+    --upload_root="${UPLOAD_ROOT}" \
+    --upload