jenkins: Re-add Jenkins scripts to this repository

We are going to restore the split-script setup from the old Jenkins
server.  This ensures that the each version's release process is
actually running with scripts in the correct release branch.  It
also allows branching the VM format lists.

Note that the scripts added here only cover the currently active
jobs in the main build pipeline.  There is no reason to add other
jobs, since they are mostly just running a single command using a
mantle binary from its master branch.

The scripts in this repository pick up after Jenkins has set up an
environment with all parameters and credentials defined, and an SDK
was prepared and validated.

jenkins/README.md

@@ -0,0 +1,11 @@
+The scripts in this directory are run from [OS Jenkins jobs][jenkins-os].  By
+storing the Jenkins scripts in this repository, they are more tightly coupled
+to the release branch of the SDK scripts that they require.  The Jenkins jobs
+are responsible for setting up the environment and securely initializing an SDK
+in the workspace before running these scripts.
+
+The special files named `formats-${BOARD}.txt` are space-separated lists of VM
+image formats that should be built for releases on this branch; i.e. the script
+`vm.sh` is run for each item in the list.
+
+[jenkins-os]: https://github.com/coreos/jenkins-os

jenkins/formats-amd64-usr.txt

@@ -0,0 +1,33 @@
+ami
+ami_vmdk
+azure
+gce
+iso
+pxe
+qemu
+qemu_uefi
+brightbox
+cloudsigma
+cloudstack
+cloudstack_vhd
+digitalocean
+exoscale
+hyperv
+niftycloud
+openstack
+openstack_mini
+packet
+parallels
+rackspace
+rackspace_onmetal
+rackspace_vhd
+vagrant
+vagrant_parallels
+vagrant_virtualbox
+vagrant_vmware_fusion
+virtualbox
+vmware
+vmware_insecure
+vmware_ova
+vmware_raw
+xen

jenkins/formats-arm64-usr.txt

@@ -0,0 +1,5 @@
+openstack
+openstack_mini
+packet
+pxe
+qemu_uefi

jenkins/images.sh

@@ -0,0 +1,78 @@
+#!/bin/bash -ex
+
+# Clear out old images.
+sudo rm -rf chroot/build src/build torcx
+
+enter() {
+        local verify_key=
+        trap 'sudo rm -f chroot/etc/portage/gangue.*' RETURN
+        [ -s verify.asc ] &&
+        sudo ln -f verify.asc chroot/etc/portage/gangue.asc &&
+        verify_key=--verify-key=/etc/portage/gangue.asc
+        sudo ln -f "${GS_DEVEL_CREDS}" chroot/etc/portage/gangue.json
+        bin/cork enter --experimental -- env \
+            COREOS_DEV_BUILDS="${DOWNLOAD_ROOT}" \
+            PORTAGE_SSH_OPTS= \
+            {FETCH,RESUME}COMMAND_GS="/usr/bin/gangue get \
+--json-key=/etc/portage/gangue.json $verify_key \
+"'"${URI}" "${DISTDIR}/${FILE}"' \
+            "$@"
+}
+
+script() {
+        enter "/mnt/host/source/src/scripts/$@"
+}
+
+sudo cp bin/gangue chroot/usr/bin/gangue  # XXX: until SDK mantle has it
+
+source .repo/manifests/version.txt
+export COREOS_BUILD_ID
+
+# Set up GPG for signing uploads.
+gpg --import "${GPG_SECRET_KEY_FILE}"
+
+script setup_board \
+    --board="${BOARD}" \
+    --getbinpkgver="${COREOS_VERSION}" \
+    --regen_configs_only
+
+if [ "x${COREOS_OFFICIAL}" == x1 ]
+then
+        script set_official --board="${BOARD}" --official
+else
+        script set_official --board="${BOARD}" --noofficial
+fi
+
+# Try to find the version's torcx store, but don't require it.
+torcx_store=
+enter gsutil cp -r \
+    "${DOWNLOAD_ROOT}/boards/${BOARD}/${COREOS_VERSION}/torcx" \
+    /mnt/host/source/ &&
+torcx_store=/mnt/host/source/torcx &&
+for image in torcx/*.torcx.tgz
+do
+        gpg --verify "${image}.sig"
+done
+
+# Work around the lack of symlink support in GCS.
+shopt -s nullglob
+for default in torcx/*:com.coreos.cl.torcx.tgz
+do
+        for image in torcx/*.torcx.tgz
+        do
+                [ "x${default}" != "x${image}" ] &&
+                cmp --silent -- "${default}" "${image}" &&
+                ln -fns "${image##*/}" "${default}"
+        done
+done
+
+script build_image \
+    --board="${BOARD}" \
+    --group="${GROUP}" \
+    --getbinpkg \
+    --getbinpkgver="${COREOS_VERSION}" \
+    --sign="${SIGNING_USER}" \
+    --sign_digests="${SIGNING_USER}" \
+    ${torcx_store:+--torcx_store="${torcx_store}"} \
+    --upload_root="${UPLOAD_ROOT}" \
+    --upload prod container

jenkins/packages.sh

@@ -0,0 +1,66 @@
+#!/bin/bash -ex
+
+# Use a ccache dir that persists across SDK recreations.
+# XXX: alternatively use a ccache dir that is usable by all jobs on a given node.
+mkdir -p .cache/ccache
+
+enter() {
+        local verify_key=
+        trap 'sudo rm -f chroot/etc/portage/gangue.*' RETURN
+        [ -s verify.asc ] &&
+        sudo ln -f verify.asc chroot/etc/portage/gangue.asc &&
+        verify_key=--verify-key=/etc/portage/gangue.asc
+        sudo ln -f "${GOOGLE_APPLICATION_CREDENTIALS}" \
+            chroot/etc/portage/gangue.json
+        bin/cork enter --experimental -- env \
+            CCACHE_DIR=/mnt/host/source/.cache/ccache \
+            CCACHE_MAXSIZE=5G \
+            COREOS_DEV_BUILDS="${DOWNLOAD_ROOT}" \
+            PORTAGE_SSH_OPTS= \
+            {FETCH,RESUME}COMMAND_GS="/usr/bin/gangue get \
+--json-key=/etc/portage/gangue.json $verify_key \
+"'"${URI}" "${DISTDIR}/${FILE}"' \
+            "$@"
+}
+
+script() {
+        enter "/mnt/host/source/src/scripts/$@"
+}
+
+sudo cp bin/gangue chroot/usr/bin/gangue  # XXX: until SDK mantle has it
+
+source .repo/manifests/version.txt
+export COREOS_BUILD_ID
+
+# Set up GPG for signing uploads.
+gpg --import "${GPG_SECRET_KEY_FILE}"
+
+# Figure out if ccache is doing us any good in this scheme.
+enter ccache --zero-stats
+
+script setup_board \
+    --board="${BOARD}" \
+    --getbinpkgver=${RELEASE_BASE:-"${COREOS_VERSION}" --toolchainpkgonly} \
+    --skip_chroot_upgrade \
+    --force
+
+script build_packages \
+    --board="${BOARD}" \
+    --getbinpkgver=${RELEASE_BASE:-"${COREOS_VERSION}" --toolchainpkgonly} \
+    --skip_chroot_upgrade \
+    $([ -x src/scripts/build_torcx_store ] && echo --skip_torcx_store) \
+    --sign="${SIGNING_USER}" \
+    --sign_digests="${SIGNING_USER}" \
+    --upload_root="${UPLOAD_ROOT}" \
+    --upload
+
+# Build and upload torcx images if this version supports it.
+[ -x src/scripts/build_torcx_store ] &&
+script build_torcx_store \
+    --board="${BOARD}" \
+    --sign="${SIGNING_USER}" \
+    --sign_digests="${SIGNING_USER}" \
+    --upload_root="${UPLOAD_ROOT}" \
+    --upload
+
+enter ccache --show-stats

jenkins/sdk.sh

@@ -0,0 +1,26 @@
+#!/bin/bash -ex
+
+enter() {
+        bin/cork enter --experimental -- "$@"
+}
+
+source .repo/manifests/version.txt
+export COREOS_BUILD_ID
+
+# Set up GPG for signing uploads.
+gpg --import "${GPG_SECRET_KEY_FILE}"
+
+# Wipe all of catalyst.
+sudo rm -rf src/build
+
+S=/mnt/host/source/src/scripts
+enter ${S}/update_chroot
+enter sudo emerge -uv --jobs=2 catalyst
+enter sudo ${S}/bootstrap_sdk \
+    --sign="${SIGNING_USER}" \
+    --sign_digests="${SIGNING_USER}" \
+    --upload_root="${UPLOAD_ROOT}" \
+    --upload
+
+# Free some disk space only on success to allow debugging failures.
+sudo rm -rf src/build/catalyst/builds

jenkins/toolchains.sh

@@ -0,0 +1,25 @@
+#!/bin/bash -ex
+
+enter() {
+        bin/cork enter --experimental -- "$@"
+}
+
+source .repo/manifests/version.txt
+export COREOS_BUILD_ID
+
+# Set up GPG for signing uploads.
+gpg --import "${GPG_SECRET_KEY_FILE}"
+
+# Wipe all of catalyst.
+sudo rm -rf src/build
+
+S=/mnt/host/source/src/scripts
+enter sudo emerge -uv --jobs=2 catalyst
+enter sudo ${S}/build_toolchains \
+    --sign="${SIGNING_USER}" \
+    --sign_digests="${SIGNING_USER}" \
+    --upload_root="${UPLOAD_ROOT}" \
+    --upload
+
+# Free some disk space only on success to allow debugging failures.
+sudo rm -rf src/build/catalyst/builds

jenkins/vm.sh

@@ -0,0 +1,60 @@
+#!/bin/bash -ex
+
+# Clear out old images.
+sudo rm -rf chroot/build tmp
+
+enter() {
+        local verify_key=
+        trap 'sudo rm -f chroot/etc/portage/gangue.*' RETURN
+        [ -s verify.asc ] &&
+        sudo ln -f verify.asc chroot/etc/portage/gangue.asc &&
+        verify_key=--verify-key=/etc/portage/gangue.asc
+        sudo ln -f "${GS_DEVEL_CREDS}" chroot/etc/portage/gangue.json
+        bin/cork enter --experimental -- env \
+            COREOS_DEV_BUILDS="${GS_DEVEL_ROOT}" \
+            PORTAGE_SSH_OPTS= \
+            {FETCH,RESUME}COMMAND_GS="/usr/bin/gangue get \
+--json-key=/etc/portage/gangue.json $verify_key \
+"'"${URI}" "${DISTDIR}/${FILE}"' \
+            "$@"
+}
+
+script() {
+        enter "/mnt/host/source/src/scripts/$@"
+}
+
+sudo cp bin/gangue chroot/usr/bin/gangue  # XXX: until SDK mantle has it
+
+source .repo/manifests/version.txt
+export COREOS_BUILD_ID
+
+# Set up GPG for signing uploads.
+gpg --import "${GPG_SECRET_KEY_FILE}"
+
+[ -s verify.asc ] && verify_key=--verify-key=verify.asc || verify_key=
+
+mkdir -p src tmp
+bin/cork download-image \
+    --root="${UPLOAD_ROOT}/boards/${BOARD}/${COREOS_VERSION}" \
+    --json-key="${GOOGLE_APPLICATION_CREDENTIALS}" \
+    --cache-dir=./src \
+    --platform=qemu \
+    --verify=true $verify_key
+
+img=src/coreos_production_image.bin
+[[ "${img}.bz2" -nt "${img}" ]] &&
+enter lbunzip2 -k -f "/mnt/host/source/${img}.bz2"
+
+script image_to_vm.sh \
+    --board="${BOARD}" \
+    --format="${FORMAT}" \
+    --prod_image \
+    --getbinpkg \
+    --getbinpkgver="${COREOS_VERSION}" \
+    --from=/mnt/host/source/src \
+    --to=/mnt/host/source/tmp \
+    --sign="${SIGNING_USER}" \
+    --sign_digests="${SIGNING_USER}" \
+    --download_root="${DOWNLOAD_ROOT}" \
+    --upload_root="${UPLOAD_ROOT}" \
+    --upload