From 9cfed47be03eeea7ac3eb6f1a1ad205f4d6ca2ce Mon Sep 17 00:00:00 2001
From: David Michael <david.michael@coreos.com>
Date: Thu, 29 Jun 2017 15:16:54 -0700
Subject: [PATCH] jenkins: Re-add Jenkins scripts to this repository

We are going to restore the split-script setup from the old Jenkins
server.  This ensures that the each version's release process is
actually running with scripts in the correct release branch.  It
also allows branching the VM format lists.

Note that the scripts added here only cover the currently active
jobs in the main build pipeline.  There is no reason to add other
jobs, since they are mostly just running a single command using a
mantle binary from its master branch.

The scripts in this repository pick up after Jenkins has set up an
environment with all parameters and credentials defined, and an SDK
was prepared and validated.
---
 jenkins/README.md             | 11 +++++
 jenkins/formats-amd64-usr.txt | 33 +++++++++++++++
 jenkins/formats-arm64-usr.txt |  5 +++
 jenkins/images.sh             | 78 +++++++++++++++++++++++++++++++++++
 jenkins/packages.sh           | 66 +++++++++++++++++++++++++++++
 jenkins/sdk.sh                | 26 ++++++++++++
 jenkins/toolchains.sh         | 25 +++++++++++
 jenkins/vm.sh                 | 60 +++++++++++++++++++++++++++
 8 files changed, 304 insertions(+)
 create mode 100644 jenkins/README.md
 create mode 100644 jenkins/formats-amd64-usr.txt
 create mode 100644 jenkins/formats-arm64-usr.txt
 create mode 100644 jenkins/images.sh
 create mode 100644 jenkins/packages.sh
 create mode 100644 jenkins/sdk.sh
 create mode 100644 jenkins/toolchains.sh
 create mode 100644 jenkins/vm.sh

diff --git a/jenkins/README.md b/jenkins/README.md
new file mode 100644
index 0000000000..aed78ce79a
--- /dev/null
+++ b/jenkins/README.md
@@ -0,0 +1,11 @@
+The scripts in this directory are run from [OS Jenkins jobs][jenkins-os].  By
+storing the Jenkins scripts in this repository, they are more tightly coupled
+to the release branch of the SDK scripts that they require.  The Jenkins jobs
+are responsible for setting up the environment and securely initializing an SDK
+in the workspace before running these scripts.
+
+The special files named `formats-${BOARD}.txt` are space-separated lists of VM
+image formats that should be built for releases on this branch; i.e. the script
+`vm.sh` is run for each item in the list.
+
+[jenkins-os]: https://github.com/coreos/jenkins-os
diff --git a/jenkins/formats-amd64-usr.txt b/jenkins/formats-amd64-usr.txt
new file mode 100644
index 0000000000..9f98a63209
--- /dev/null
+++ b/jenkins/formats-amd64-usr.txt
@@ -0,0 +1,33 @@
+ami
+ami_vmdk
+azure
+gce
+iso
+pxe
+qemu
+qemu_uefi
+brightbox
+cloudsigma
+cloudstack
+cloudstack_vhd
+digitalocean
+exoscale
+hyperv
+niftycloud
+openstack
+openstack_mini
+packet
+parallels
+rackspace
+rackspace_onmetal
+rackspace_vhd
+vagrant
+vagrant_parallels
+vagrant_virtualbox
+vagrant_vmware_fusion
+virtualbox
+vmware
+vmware_insecure
+vmware_ova
+vmware_raw
+xen
diff --git a/jenkins/formats-arm64-usr.txt b/jenkins/formats-arm64-usr.txt
new file mode 100644
index 0000000000..fe6f2e2598
--- /dev/null
+++ b/jenkins/formats-arm64-usr.txt
@@ -0,0 +1,5 @@
+openstack
+openstack_mini
+packet
+pxe
+qemu_uefi
diff --git a/jenkins/images.sh b/jenkins/images.sh
new file mode 100644
index 0000000000..a2b41eb05c
--- /dev/null
+++ b/jenkins/images.sh
@@ -0,0 +1,78 @@
+#!/bin/bash -ex
+
+# Clear out old images.
+sudo rm -rf chroot/build src/build torcx
+
+enter() {
+        local verify_key=
+        trap 'sudo rm -f chroot/etc/portage/gangue.*' RETURN
+        [ -s verify.asc ] &&
+        sudo ln -f verify.asc chroot/etc/portage/gangue.asc &&
+        verify_key=--verify-key=/etc/portage/gangue.asc
+        sudo ln -f "${GS_DEVEL_CREDS}" chroot/etc/portage/gangue.json
+        bin/cork enter --experimental -- env \
+            COREOS_DEV_BUILDS="${DOWNLOAD_ROOT}" \
+            PORTAGE_SSH_OPTS= \
+            {FETCH,RESUME}COMMAND_GS="/usr/bin/gangue get \
+--json-key=/etc/portage/gangue.json $verify_key \
+"'"${URI}" "${DISTDIR}/${FILE}"' \
+            "$@"
+}
+
+script() {
+        enter "/mnt/host/source/src/scripts/$@"
+}
+
+sudo cp bin/gangue chroot/usr/bin/gangue  # XXX: until SDK mantle has it
+
+source .repo/manifests/version.txt
+export COREOS_BUILD_ID
+
+# Set up GPG for signing uploads.
+gpg --import "${GPG_SECRET_KEY_FILE}"
+
+script setup_board \
+    --board="${BOARD}" \
+    --getbinpkgver="${COREOS_VERSION}" \
+    --regen_configs_only
+
+if [ "x${COREOS_OFFICIAL}" == x1 ]
+then
+        script set_official --board="${BOARD}" --official
+else
+        script set_official --board="${BOARD}" --noofficial
+fi
+
+# Try to find the version's torcx store, but don't require it.
+torcx_store=
+enter gsutil cp -r \
+    "${DOWNLOAD_ROOT}/boards/${BOARD}/${COREOS_VERSION}/torcx" \
+    /mnt/host/source/ &&
+torcx_store=/mnt/host/source/torcx &&
+for image in torcx/*.torcx.tgz
+do
+        gpg --verify "${image}.sig"
+done
+
+# Work around the lack of symlink support in GCS.
+shopt -s nullglob
+for default in torcx/*:com.coreos.cl.torcx.tgz
+do
+        for image in torcx/*.torcx.tgz
+        do
+                [ "x${default}" != "x${image}" ] &&
+                cmp --silent -- "${default}" "${image}" &&
+                ln -fns "${image##*/}" "${default}"
+        done
+done
+
+script build_image \
+    --board="${BOARD}" \
+    --group="${GROUP}" \
+    --getbinpkg \
+    --getbinpkgver="${COREOS_VERSION}" \
+    --sign="${SIGNING_USER}" \
+    --sign_digests="${SIGNING_USER}" \
+    ${torcx_store:+--torcx_store="${torcx_store}"} \
+    --upload_root="${UPLOAD_ROOT}" \
+    --upload prod container
diff --git a/jenkins/packages.sh b/jenkins/packages.sh
new file mode 100644
index 0000000000..9702ce045c
--- /dev/null
+++ b/jenkins/packages.sh
@@ -0,0 +1,66 @@
+#!/bin/bash -ex
+
+# Use a ccache dir that persists across SDK recreations.
+# XXX: alternatively use a ccache dir that is usable by all jobs on a given node.
+mkdir -p .cache/ccache
+
+enter() {
+        local verify_key=
+        trap 'sudo rm -f chroot/etc/portage/gangue.*' RETURN
+        [ -s verify.asc ] &&
+        sudo ln -f verify.asc chroot/etc/portage/gangue.asc &&
+        verify_key=--verify-key=/etc/portage/gangue.asc
+        sudo ln -f "${GOOGLE_APPLICATION_CREDENTIALS}" \
+            chroot/etc/portage/gangue.json
+        bin/cork enter --experimental -- env \
+            CCACHE_DIR=/mnt/host/source/.cache/ccache \
+            CCACHE_MAXSIZE=5G \
+            COREOS_DEV_BUILDS="${DOWNLOAD_ROOT}" \
+            PORTAGE_SSH_OPTS= \
+            {FETCH,RESUME}COMMAND_GS="/usr/bin/gangue get \
+--json-key=/etc/portage/gangue.json $verify_key \
+"'"${URI}" "${DISTDIR}/${FILE}"' \
+            "$@"
+}
+
+script() {
+        enter "/mnt/host/source/src/scripts/$@"
+}
+
+sudo cp bin/gangue chroot/usr/bin/gangue  # XXX: until SDK mantle has it
+
+source .repo/manifests/version.txt
+export COREOS_BUILD_ID
+
+# Set up GPG for signing uploads.
+gpg --import "${GPG_SECRET_KEY_FILE}"
+
+# Figure out if ccache is doing us any good in this scheme.
+enter ccache --zero-stats
+
+script setup_board \
+    --board="${BOARD}" \
+    --getbinpkgver=${RELEASE_BASE:-"${COREOS_VERSION}" --toolchainpkgonly} \
+    --skip_chroot_upgrade \
+    --force
+
+script build_packages \
+    --board="${BOARD}" \
+    --getbinpkgver=${RELEASE_BASE:-"${COREOS_VERSION}" --toolchainpkgonly} \
+    --skip_chroot_upgrade \
+    $([ -x src/scripts/build_torcx_store ] && echo --skip_torcx_store) \
+    --sign="${SIGNING_USER}" \
+    --sign_digests="${SIGNING_USER}" \
+    --upload_root="${UPLOAD_ROOT}" \
+    --upload
+
+# Build and upload torcx images if this version supports it.
+[ -x src/scripts/build_torcx_store ] &&
+script build_torcx_store \
+    --board="${BOARD}" \
+    --sign="${SIGNING_USER}" \
+    --sign_digests="${SIGNING_USER}" \
+    --upload_root="${UPLOAD_ROOT}" \
+    --upload
+
+enter ccache --show-stats
diff --git a/jenkins/sdk.sh b/jenkins/sdk.sh
new file mode 100644
index 0000000000..bdf5e60918
--- /dev/null
+++ b/jenkins/sdk.sh
@@ -0,0 +1,26 @@
+#!/bin/bash -ex
+
+enter() {
+        bin/cork enter --experimental -- "$@"
+}
+
+source .repo/manifests/version.txt
+export COREOS_BUILD_ID
+
+# Set up GPG for signing uploads.
+gpg --import "${GPG_SECRET_KEY_FILE}"
+
+# Wipe all of catalyst.
+sudo rm -rf src/build
+
+S=/mnt/host/source/src/scripts
+enter ${S}/update_chroot
+enter sudo emerge -uv --jobs=2 catalyst
+enter sudo ${S}/bootstrap_sdk \
+    --sign="${SIGNING_USER}" \
+    --sign_digests="${SIGNING_USER}" \
+    --upload_root="${UPLOAD_ROOT}" \
+    --upload
+
+# Free some disk space only on success to allow debugging failures.
+sudo rm -rf src/build/catalyst/builds
diff --git a/jenkins/toolchains.sh b/jenkins/toolchains.sh
new file mode 100644
index 0000000000..a56ec9e867
--- /dev/null
+++ b/jenkins/toolchains.sh
@@ -0,0 +1,25 @@
+#!/bin/bash -ex
+
+enter() {
+        bin/cork enter --experimental -- "$@"
+}
+
+source .repo/manifests/version.txt
+export COREOS_BUILD_ID
+
+# Set up GPG for signing uploads.
+gpg --import "${GPG_SECRET_KEY_FILE}"
+
+# Wipe all of catalyst.
+sudo rm -rf src/build
+
+S=/mnt/host/source/src/scripts
+enter sudo emerge -uv --jobs=2 catalyst
+enter sudo ${S}/build_toolchains \
+    --sign="${SIGNING_USER}" \
+    --sign_digests="${SIGNING_USER}" \
+    --upload_root="${UPLOAD_ROOT}" \
+    --upload
+
+# Free some disk space only on success to allow debugging failures.
+sudo rm -rf src/build/catalyst/builds
diff --git a/jenkins/vm.sh b/jenkins/vm.sh
new file mode 100644
index 0000000000..a4c79a5308
--- /dev/null
+++ b/jenkins/vm.sh
@@ -0,0 +1,60 @@
+#!/bin/bash -ex
+
+# Clear out old images.
+sudo rm -rf chroot/build tmp
+
+enter() {
+        local verify_key=
+        trap 'sudo rm -f chroot/etc/portage/gangue.*' RETURN
+        [ -s verify.asc ] &&
+        sudo ln -f verify.asc chroot/etc/portage/gangue.asc &&
+        verify_key=--verify-key=/etc/portage/gangue.asc
+        sudo ln -f "${GS_DEVEL_CREDS}" chroot/etc/portage/gangue.json
+        bin/cork enter --experimental -- env \
+            COREOS_DEV_BUILDS="${GS_DEVEL_ROOT}" \
+            PORTAGE_SSH_OPTS= \
+            {FETCH,RESUME}COMMAND_GS="/usr/bin/gangue get \
+--json-key=/etc/portage/gangue.json $verify_key \
+"'"${URI}" "${DISTDIR}/${FILE}"' \
+            "$@"
+}
+
+script() {
+        enter "/mnt/host/source/src/scripts/$@"
+}
+
+sudo cp bin/gangue chroot/usr/bin/gangue  # XXX: until SDK mantle has it
+
+source .repo/manifests/version.txt
+export COREOS_BUILD_ID
+
+# Set up GPG for signing uploads.
+gpg --import "${GPG_SECRET_KEY_FILE}"
+
+[ -s verify.asc ] && verify_key=--verify-key=verify.asc || verify_key=
+
+mkdir -p src tmp
+bin/cork download-image \
+    --root="${UPLOAD_ROOT}/boards/${BOARD}/${COREOS_VERSION}" \
+    --json-key="${GOOGLE_APPLICATION_CREDENTIALS}" \
+    --cache-dir=./src \
+    --platform=qemu \
+    --verify=true $verify_key
+
+img=src/coreos_production_image.bin
+[[ "${img}.bz2" -nt "${img}" ]] &&
+enter lbunzip2 -k -f "/mnt/host/source/${img}.bz2"
+
+script image_to_vm.sh \
+    --board="${BOARD}" \
+    --format="${FORMAT}" \
+    --prod_image \
+    --getbinpkg \
+    --getbinpkgver="${COREOS_VERSION}" \
+    --from=/mnt/host/source/src \
+    --to=/mnt/host/source/tmp \
+    --sign="${SIGNING_USER}" \
+    --sign_digests="${SIGNING_USER}" \
+    --download_root="${DOWNLOAD_ROOT}" \
+    --upload_root="${UPLOAD_ROOT}" \
+    --upload