From 7a2fbfe67dffb75b080a11ca1d033d663952b636 Mon Sep 17 00:00:00 2001
From: Michael Marineau <michael.marineau@coreos.com>
Date: Fri, 21 Oct 2016 13:44:31 -0700
Subject: [PATCH] build_jobs: import current jenkins build scripts

---
 build_jobs/01_manifest.sh   | 114 ++++++++++++++++++++++++++++++++++
 build_jobs/02_sdk.sh        |  71 +++++++++++++++++++++
 build_jobs/02_toolchains.sh |  71 +++++++++++++++++++++
 build_jobs/03_packages.sh   |  81 ++++++++++++++++++++++++
 build_jobs/04_images.sh     | 109 +++++++++++++++++++++++++++++++++
 build_jobs/05_vm.sh         | 119 ++++++++++++++++++++++++++++++++++++
 6 files changed, 565 insertions(+)
 create mode 100644 build_jobs/01_manifest.sh
 create mode 100644 build_jobs/02_sdk.sh
 create mode 100644 build_jobs/02_toolchains.sh
 create mode 100644 build_jobs/03_packages.sh
 create mode 100644 build_jobs/04_images.sh
 create mode 100644 build_jobs/05_vm.sh

diff --git a/build_jobs/01_manifest.sh b/build_jobs/01_manifest.sh
new file mode 100644
index 0000000000..4ad75e4d1c
--- /dev/null
+++ b/build_jobs/01_manifest.sh
@@ -0,0 +1,114 @@
+#!/bin/bash
+#
+# Jenkins job for creating build manifests.
+#
+# Input Parameters:
+#
+#   MANIFEST_REF=master
+#     Git branch or tag in github.com/coreos/manifest to build
+#
+#   LOCAL_MANIFEST=
+#     Repo local manifest to amend the branch's default manifest with.
+#     https://wiki.cyanogenmod.org/w/Doc:_Using_manifests#The_local_manifest
+#
+# Input Artifacts:
+#
+#   $WORKSPACE/bin/cork from a recent mantle build.
+#
+# Git:
+#
+#   github.com/coreos/manifest checked out to $WORKSPACE/manifest
+#   SSH push access to github.com/coreos/manifest-builds
+#
+# Output:
+#
+#   Pushes build tag to manifest-builds.
+#   Writes manifest.properties w/ parameters for sdk and toolchain jobs.
+
+set -ex
+
+export GIT_SSH_COMMAND="ssh -o StrictHostKeyChecking=no"
+
+finish() {
+  local tag="$1"
+  git -C "${WORKSPACE}/manifest" push \
+    "ssh://git@github.com/coreos/manifest-builds.git" \
+    "refs/tags/${tag}:refs/tags/${tag}"
+  tee "${WORKSPACE}/manifest.properties" <<EOF
+MANIFEST_URL = https://github.com/coreos/manifest-builds.git
+MANIFEST_REF = refs/tags/${tag}
+MANIFEST_NAME = release.xml
+COREOS_OFFICIAL = ${COREOS_OFFICIAL:-0}
+EOF
+}
+
+# Branches are of the form remote-name/branch-name. Tags are just tag-name.
+# If we have a release tag use it, for branches we need to make a tag.
+COREOS_OFFICIAL=0
+if [[ "${GIT_BRANCH}" != */* ]]; then
+  COREOS_OFFICIAL=1
+  finish "${GIT_BRANCH}"
+  exit
+fi
+
+MANIFEST_BRANCH="${GIT_BRANCH##*/}"
+MANIFEST_NAME="${MANIFEST_BRANCH}.xml"
+[[ -f "manifest/${MANIFEST_NAME}" ]]
+
+source manifest/version.txt
+export COREOS_BUILD_ID="${MANIFEST_BRANCH}-${BUILD_NUMBER}"
+
+# hack to get repo to set things up using the manifest repo we already have
+# (amazing that it tolerates this considering it usually is so intolerant)
+mkdir -p .repo
+ln -sfT ../manifest .repo/manifests
+ln -sfT ../manifest/.git .repo/manifests.git
+
+# Cleanup/setup local manifests
+rm -rf .repo/local_manifests
+if [[ -n "${LOCAL_MANIFEST}" ]]; then
+  mkdir -p .repo/local_manifests
+  cat >.repo/local_manifests/local.xml <<<"${LOCAL_MANIFEST}"
+fi
+
+./bin/cork update --create --downgrade-replace --verbose \
+                  --manifest-url "${GIT_URL}" \
+                  --manifest-branch "${GIT_COMMIT}" \
+                  --manifest-name "${MANIFEST_NAME}" \
+                  --new-version "${COREOS_VERSION}" \
+                  --sdk-version "${COREOS_SDK_VERSION}"
+
+./bin/cork enter --experimental -- sh -c \
+  "pwd; repo manifest -r > '/mnt/host/source/manifest/${COREOS_BUILD_ID}.xml'"
+
+cd manifest
+git add "${COREOS_BUILD_ID}.xml" 
+
+ln -sf "${COREOS_BUILD_ID}.xml" default.xml
+ln -sf "${COREOS_BUILD_ID}.xml" release.xml
+git add default.xml release.xml
+
+tee version.txt <<EOF
+COREOS_VERSION=${COREOS_VERSION_ID}+${COREOS_BUILD_ID}
+COREOS_VERSION_ID=${COREOS_VERSION_ID}
+COREOS_BUILD_ID=${COREOS_BUILD_ID}
+COREOS_SDK_VERSION=${COREOS_SDK_VERSION}
+EOF
+git add version.txt
+
+EMAIL="jenkins@jenkins.coreos.systems"
+GIT_AUTHOR_NAME="CoreOS Jenkins"
+GIT_COMMITTER_NAME="${GIT_AUTHOR_NAME}"
+export EMAIL GIT_AUTHOR_NAME GIT_COMMITTER_NAME
+git commit \
+  -m "${COREOS_BUILD_ID}: add build manifest" \
+  -m "Based on ${GIT_URL} branch ${MANIFEST_BRANCH}" \
+  -m "${BUILD_URL}"
+git tag -m "${COREOS_BUILD_ID}" "${COREOS_BUILD_ID}" HEAD
+
+# assert that what we just did will work, update symlink because verify doesn't have a --manifest-name option yet
+cd "${WORKSPACE}"
+ln -sf "manifests/${COREOS_BUILD_ID}.xml" .repo/manifest.xml
+./bin/cork verify
+
+finish "${COREOS_BUILD_ID}"
diff --git a/build_jobs/02_sdk.sh b/build_jobs/02_sdk.sh
new file mode 100644
index 0000000000..a91a44c59b
--- /dev/null
+++ b/build_jobs/02_sdk.sh
@@ -0,0 +1,71 @@
+#!/bin/bash
+#
+# Jenkins job for building the SDK tarball.
+#
+# Input Parameters:
+#
+#   USE_CACHE=false
+#     Enable use of any binary packages cached locally from previous builds.
+#     Currently not safe to enable, particularly bad with multiple branches.
+#
+#   MANIFEST_URL=https://github.com/coreos/manifest-builds.git
+#   MANIFEST_REF=refs/tags/
+#   MANIFEST_NAME=release.xml
+#     Git URL, tag, and manifest file for this build.
+#
+#   COREOS_OFFICIAL=0
+#     Set to 1 when building official releases.
+#
+# Input Artifacts:
+#
+#   $WORKSPACE/bin/cork from a recent mantle build.
+#
+# Secrets:
+#
+#   GPG_SECRET_KEY_FILE=
+#     Exported GPG public/private key used to sign uploaded files.
+#
+#   GOOGLE_APPLICATION_CREDENTIALS=
+#     JSON file defining a Google service account for uploading files.
+#
+# Output:
+#
+#   Uploads binary packages and tarball to gs://builds.developer.core-os.net
+
+set -ex
+
+# build may not be started without a ref value
+[[ -n "${MANIFEST_REF#refs/tags/}" ]]
+
+enter() {
+  ./bin/cork enter --experimental -- "$@"
+}
+
+# hack because catalyst leaves things chowned as root
+[[ -d .cache/sdks ]] && sudo chown -R $USER .cache/sdks
+
+./bin/cork update --create --downgrade-replace --verify --verbose \
+                  --manifest-url "${MANIFEST_URL}" \
+                  --manifest-branch "${MANIFEST_REF}" \
+                  --manifest-name "${MANIFEST_NAME}"
+
+source .repo/manifests/version.txt
+export COREOS_BUILD_ID
+
+# Set up GPG for signing images
+export GNUPGHOME="${PWD}/.gnupg"
+sudo rm -rf "${GNUPGHOME}"
+trap "sudo rm -rf '${GNUPGHOME}'" EXIT
+mkdir --mode=0700 "${GNUPGHOME}"
+gpg --import "${GPG_SECRET_KEY_FILE}"
+
+# Wipe all of catalyst or just clear out old tarballs taking up space
+if [[ "${COREOS_OFFICIAL:-0}" -eq 1 || "$USE_CACHE" == false ]]; then
+  sudo rm -rf src/build/catalyst
+fi
+sudo rm -rf src/build/catalyst/builds
+
+enter sudo emerge -uv --jobs=2 catalyst
+enter sudo /mnt/host/source/src/scripts/bootstrap_sdk \
+  --sign buildbot@coreos.com --sign_digests buildbot@coreos.com \
+  --upload --upload_root gs://builds.developer.core-os.net
diff --git a/build_jobs/02_toolchains.sh b/build_jobs/02_toolchains.sh
new file mode 100644
index 0000000000..a27876cf94
--- /dev/null
+++ b/build_jobs/02_toolchains.sh
@@ -0,0 +1,71 @@
+#!/bin/bash
+#
+# Jenkins job for building the SDK's cross toolchains.
+#
+# Input Parameters:
+#
+#   USE_CACHE=false
+#     Enable use of any binary packages cached locally from previous builds.
+#     Currently not safe to enable, particularly bad with multiple branches.
+#
+#   MANIFEST_URL=https://github.com/coreos/manifest-builds.git
+#   MANIFEST_REF=refs/tags/
+#   MANIFEST_NAME=release.xml
+#     Git URL, tag, and manifest file for this build.
+#
+#   COREOS_OFFICIAL=0
+#     Set to 1 when building official releases.
+#
+# Input Artifacts:
+#
+#   $WORKSPACE/bin/cork from a recent mantle build.
+#
+# Secrets:
+#
+#   GPG_SECRET_KEY_FILE=
+#     Exported GPG public/private key used to sign uploaded files.
+#
+#   GOOGLE_APPLICATION_CREDENTIALS=
+#     JSON file defining a Google service account for uploading files.
+#
+# Output:
+#
+#   Uploads binary packages to gs://builds.developer.core-os.net
+
+set -ex
+
+# build may not be started without a ref value
+[[ -n "${MANIFEST_REF#refs/tags/}" ]]
+
+enter() {
+  ./bin/cork enter --experimental -- "$@"
+}
+
+# hack because catalyst leaves things chowned as root
+[[ -d .cache/sdks ]] && sudo chown -R $USER .cache/sdks
+
+./bin/cork update --create --downgrade-replace --verify --verbose \
+                  --manifest-url "${MANIFEST_URL}" \
+                  --manifest-branch "${MANIFEST_REF}" \
+                  --manifest-name "${MANIFEST_NAME}"
+
+source .repo/manifests/version.txt
+export COREOS_BUILD_ID
+
+# Set up GPG for signing images
+export GNUPGHOME="${PWD}/.gnupg"
+sudo rm -rf "${GNUPGHOME}"
+trap "sudo rm -rf '${GNUPGHOME}'" EXIT
+mkdir --mode=0700 "${GNUPGHOME}"
+gpg --import "${GPG_SECRET_KEY_FILE}"
+
+# Wipe all of catalyst or just clear out old tarballs taking up space
+if [[ "${COREOS_OFFICIAL:-0}" -eq 1 || "$USE_CACHE" == false ]]; then
+  sudo rm -rf src/build/catalyst
+fi
+sudo rm -rf src/build/catalyst/builds
+
+enter sudo emerge -uv --jobs=2 catalyst
+enter sudo /mnt/host/source/src/scripts/build_toolchains \
+  --sign buildbot@coreos.com --sign_digests buildbot@coreos.com \
+  --upload --upload_root gs://builds.developer.core-os.net
diff --git a/build_jobs/03_packages.sh b/build_jobs/03_packages.sh
new file mode 100644
index 0000000000..ca87b9c1cc
--- /dev/null
+++ b/build_jobs/03_packages.sh
@@ -0,0 +1,81 @@
+#!/bin/bash
+#
+# Jenkins job for building board binary packages.
+#
+# Input Parameters:
+#
+#   MANIFEST_URL=https://github.com/coreos/manifest-builds.git
+#   MANIFEST_REF=refs/tags/
+#   MANIFEST_NAME=release.xml
+#     Git URL, tag, and manifest file for this build.
+#
+#   COREOS_OFFICIAL=0
+#     Set to 1 when building official releases.
+#
+#   BOARD=amd64-usr
+#     Target board to build.
+#
+# Input Artifacts:
+#
+#   $WORKSPACE/bin/cork from a recent mantle build.
+#
+# Secrets:
+#
+#   GOOGLE_APPLICATION_CREDENTIALS=
+#     JSON file defining a Google service account for uploading files.
+#
+# Output:
+#
+#   Uploads binary packages to gs://builds.developer.core-os.net
+
+set -ex
+
+# build may not be started without a ref value
+[[ -n "${MANIFEST_REF#refs/tags/}" ]]
+
+# use a ccache dir that persists across sdk recreations
+# XXX: alternatively use a ccache dir that is usable by all jobs on a given node.
+mkdir -p .cache/ccache
+
+enter() {
+  ./bin/cork enter --experimental -- env \
+    CCACHE_DIR="/mnt/host/source/.cache/ccache" \
+    CCACHE_MAXSIZE="5G" "$@"
+}
+
+script() {
+  local script="/mnt/host/source/src/scripts/${1}"; shift
+  enter "${script}" "$@"
+}
+
+./bin/cork update --create --downgrade-replace --verify --verbose \
+                  --manifest-url "${MANIFEST_URL}" \
+                  --manifest-branch "${MANIFEST_REF}" \
+                  --manifest-name "${MANIFEST_NAME}" \
+                  -- --toolchain_boards=${BOARD}
+
+source .repo/manifests/version.txt
+export COREOS_BUILD_ID
+
+if [[ "${COREOS_VERSION}" == 1010.* && "${BOARD}" == arm64-usr ]]; then
+  echo "SKIPPING ARM"
+  exit 0
+fi
+
+# figure out if ccache is doing us any good in this scheme
+enter ccache --zero-stats
+
+#if [[ "${COREOS_OFFICIAL:-0}" -eq 1 ]]; then
+  script setup_board --board=${BOARD} \
+                     --skip_chroot_upgrade \
+                     --getbinpkgver=${COREOS_VERSION} \
+                     --toolchainpkgonly \
+                     --force
+#fi
+script build_packages --board=${BOARD} \
+                      --skip_chroot_upgrade \
+                      --getbinpkgver=${COREOS_VERSION} \
+                      --toolchainpkgonly \
+                      --upload --upload_root gs://builds.developer.core-os.net
+
+enter ccache --show-stats
diff --git a/build_jobs/04_images.sh b/build_jobs/04_images.sh
new file mode 100644
index 0000000000..34c4da57ea
--- /dev/null
+++ b/build_jobs/04_images.sh
@@ -0,0 +1,109 @@
+#!/bin/bash
+#
+# Jenkins job for building the base production image and dev container.
+#
+# Input Parameters:
+#
+#   USE_CACHE=false
+#     Enable use of any binary packages cached locally from previous builds.
+#     Currently not safe to enable, particularly bad with multiple branches.
+#
+#   MANIFEST_URL=https://github.com/coreos/manifest-builds.git
+#   MANIFEST_REF=refs/tags/
+#   MANIFEST_NAME=release.xml
+#     Git URL, tag, and manifest file for this build.
+#
+#   COREOS_OFFICIAL=0
+#     Set to 1 when building official releases.
+#
+#   BOARD=amd64-usr
+#     Target board to build.
+#
+# Input Artifacts:
+#
+#   $WORKSPACE/bin/cork from a recent mantle build.
+#
+# Secrets:
+#
+#   GPG_SECRET_KEY_FILE=
+#     Exported GPG public/private key used to sign uploaded files.
+#
+#   GOOGLE_APPLICATION_CREDENTIALS=
+#     JSON file defining a Google service account for uploading files.
+#
+# Output:
+#
+#   Uploads test branch images to gs://builds.developer.core-os.net and
+#   official images to gs://builds.release.core-os.net
+
+set -ex
+
+# build may not be started without a ref value
+[[ -n "${MANIFEST_REF#refs/tags/}" ]]
+
+# first thing, clear out old images
+sudo rm -rf src/build
+
+script() {
+  local script="/mnt/host/source/src/scripts/${1}"; shift
+  ./bin/cork enter --experimental -- "${script}" "$@"
+}
+
+./bin/cork update --create --downgrade-replace --verify --verbose \
+                  --manifest-url "${MANIFEST_URL}" \
+                  --manifest-branch "${MANIFEST_REF}" \
+                  --manifest-name "${MANIFEST_NAME}"
+
+source .repo/manifests/version.txt
+export COREOS_BUILD_ID
+
+if [[ "${COREOS_VERSION}" == 1010.* && "${BOARD}" == arm64-usr ]]; then
+  echo "SKIPPING ARM"
+  exit 0
+fi
+
+# Set up GPG for signing images
+export GNUPGHOME="${PWD}/.gnupg"
+rm -rf "${GNUPGHOME}"
+trap "rm -rf '${GNUPGHOME}'" EXIT
+mkdir --mode=0700 "${GNUPGHOME}"
+gpg --import "${GPG_SECRET_KEY_FILE}"
+
+sudo rm -rf chroot/build
+script setup_board --board=${BOARD} \
+                   --getbinpkgver="${COREOS_VERSION}" \
+                   --regen_configs_only
+
+if [[ "${COREOS_OFFICIAL}" -eq 1 ]]; then
+  GROUP=stable
+  UPLOAD=gs://builds.release.core-os.net/stable
+  script set_official --board=${BOARD} --official
+else
+  GROUP=developer
+  UPLOAD=gs://builds.developer.core-os.net
+  script set_official --board=${BOARD} --noofficial
+fi
+
+script build_image --board=${BOARD} \
+                   --group=${GROUP} \
+                   --getbinpkg \
+                   --getbinpkgver="${COREOS_VERSION}" \
+                   --sign=buildbot@coreos.com \
+                   --sign_digests=buildbot@coreos.com \
+                   --upload_root=${UPLOAD} \
+                   --upload prod container
+
+if [[ "${COREOS_OFFICIAL}" -eq 1 ]]; then
+  script image_set_group --board=${BOARD} \
+                         --group=alpha \
+                         --sign=buildbot@coreos.com \
+                         --sign_digests=buildbot@coreos.com \
+                         --upload_root=gs://builds.release.core-os.net/alpha \
+                         --upload
+  script image_set_group --board=${BOARD} \
+                         --group=beta \
+                         --sign=buildbot@coreos.com \
+                         --sign_digests=buildbot@coreos.com \
+                         --upload_root=gs://builds.release.core-os.net/beta \
+                         --upload
+fi
diff --git a/build_jobs/05_vm.sh b/build_jobs/05_vm.sh
new file mode 100644
index 0000000000..5e9aaefec4
--- /dev/null
+++ b/build_jobs/05_vm.sh
@@ -0,0 +1,119 @@
+#!/bin/bash
+#
+# Jenkins job for building final VM and OEM target images.
+#
+# Input Parameters:
+#
+#   USE_CACHE=false
+#     Enable use of any binary packages cached locally from previous builds.
+#     Currently not safe to enable, particularly bad with multiple branches.
+#
+#   MANIFEST_URL=https://github.com/coreos/manifest-builds.git
+#   MANIFEST_REF=refs/tags/
+#   MANIFEST_NAME=release.xml
+#     Git URL, tag, and manifest file for this build.
+#
+#   COREOS_OFFICIAL=0
+#     Set to 1 when building official releases.
+#
+#   BOARD=amd64-usr
+#     Target board to build.
+#
+#   FORMAT=qemu
+#     Target VM or OEM.
+#
+#   GROUP=developer
+#     Target update group.
+#
+# Input Artifacts:
+#
+#   $WORKSPACE/bin/cork from a recent mantle build.
+#
+# Secrets:
+#
+#   GPG_SECRET_KEY_FILE=
+#     Exported GPG public/private key used to sign uploaded files.
+#
+#   GOOGLE_APPLICATION_CREDENTIALS=
+#     JSON file defining a Google service account for uploading files.
+#
+# Output:
+#
+#   Uploads test branch images to gs://builds.developer.core-os.net and
+#   official images to gs://builds.release.core-os.net
+#   Writes gce.properties for triggering a GCE test job if applicable.
+
+set -ex
+
+rm -f gce.properties
+sudo rm -rf tmp
+
+# build may not be started without a ref value
+[[ -n "${MANIFEST_REF#refs/tags/}" ]]
+
+# check that the matrix didn't go bananas
+if [[ "${COREOS_OFFICIAL}" -eq 1 ]]; then
+  [[ "${GROUP}" != developer ]]
+else
+  [[ "${GROUP}" == developer ]]
+fi
+
+script() {
+  local script="/mnt/host/source/src/scripts/${1}"; shift
+  ./bin/cork enter --experimental -- "${script}" "$@"
+}
+
+enter() {
+  ./bin/cork enter --experimental -- "$@"
+}
+
+./bin/cork update --create --downgrade-replace --verify --verbose \
+                  --manifest-url "${MANIFEST_URL}" \
+                  --manifest-branch "${MANIFEST_REF}" \
+                  --manifest-name "${MANIFEST_NAME}"
+
+source .repo/manifests/version.txt
+export COREOS_BUILD_ID
+
+if [[ "${COREOS_VERSION}" == 1010.* && "${BOARD}" == arm64-usr ]]; then
+  echo "SKIPPING ARM"
+  exit 0
+fi
+
+# Set up GPG for signing images
+export GNUPGHOME="${PWD}/.gnupg"
+rm -rf "${GNUPGHOME}"
+trap "rm -rf '${GNUPGHOME}'" EXIT
+mkdir --mode=0700 "${GNUPGHOME}"
+gpg --import "${GPG_SECRET_KEY_FILE}"
+
+if [[ "${GROUP}" == developer ]]; then
+  root="gs://builds.developer.core-os.net"
+  dlroot=""
+else
+  root="gs://builds.release.core-os.net/${GROUP}"
+  dlroot="--download_root https://${GROUP}.release.core-os.net"
+fi
+
+mkdir -p src tmp
+./bin/cork download-image --root="${root}/boards/${BOARD}/${COREOS_VERSION}" \
+                          --json-key="${GOOGLE_APPLICATION_CREDENTIALS}" \
+                          --cache-dir=./src \
+                          --platform=qemu
+img=src/coreos_production_image.bin
+if [[ "${img}.bz2" -nt "${img}" ]]; then
+  enter lbunzip2 -k -f "/mnt/host/source/${img}.bz2"
+fi
+
+sudo rm -rf chroot/build
+script image_to_vm.sh --board=${BOARD} \
+                      --format=${FORMAT} \
+                      --prod_image \
+                      --getbinpkg \
+                      --getbinpkgver=${COREOS_VERSION} \
+                      --from=/mnt/host/source/src/ \
+                      --to=/mnt/host/source/tmp/ \
+                      --sign=buildbot@coreos.com \
+                      --sign_digests=buildbot@coreos.com \
+                      --upload_root="${root}" \
+                      --upload ${dlroot}