bootstrap_sdk: build and upload SDK toolchain pkgs

Before, we were relying on the toolchains job to build and upload
packages that were part of the SDK. With this change, all packages that
should be part of the SDK are built and uploaded by the SDK job. The
toolchains job only builds toolchain packages specific for the release.

This change includes several adjustments done to both the SDK and the
toolchains jobs to make this work:
 * Make the SDK job build all cross toolchains, including Rust
 * Stop building Rust in the toolchains job and use the one in the SDK
   instead.
 * In toolchain_util.sh: detect when the symlink folder for crossdev
   packages is missing and run crossdev to create it during
   update_chroot setup.
 * Make it possible to build the SDK starting from stage 4 instead of
   stage 1, to make the SDK building faster for PR branches / nightlies
   (full build should still be done for releases / weeklies).

bootstrap_sdk

@@ -40,14 +40,16 @@ TYPE="flatcar-sdk"
 # include upload options
 . "${BUILD_LIBRARY_DIR}/release_util.sh" || exit 1
 
+ROOT_OVERLAY=${TEMPDIR}/stage4_overlay
+
 ## Define the stage4 config template
 catalyst_stage4() {
 cat <<EOF
 target: stage4
 pkgcache_path: $BINPKGS
 stage4/packages: coreos-devel/sdk-depends
-stage4/fsscript: ${BUILD_LIBRARY_DIR}/catalyst_default_stage4.sh
-stage4/root_overlay: ${TEMPDIR}/stage4_overlay
+stage4/fsscript: ${BUILD_LIBRARY_DIR}/catalyst_sdk.sh
+stage4/root_overlay: ${ROOT_OVERLAY}
 stage4/empty: /etc/portage/repos.conf /root /usr/portage /var/cache/edb
 stage4/rm: /etc/machine-id /etc/resolv.conf
 EOF
@@ -64,17 +66,24 @@ check_gsutil_opts
 
 if [[ "$STAGES" =~ stage4 ]]; then
     info "Setting release to ${FLATCAR_VERSION}"
-    rm -rf "${TEMPDIR}/stage4_overlay"
+    rm -rf "${ROOT_OVERLAY}"
     # need to setup the lib->lib64 symlink correctly
     libdir=$(get_sdk_libdir)
-    mkdir -p "${TEMPDIR}/stage4_overlay/usr/${libdir}"
+    mkdir -p "${ROOT_OVERLAY}/usr/${libdir}"
     if [[ "${libdir}" != lib ]]; then
-        ln -s "${libdir}" "${TEMPDIR}/stage4_overlay/usr/lib"
+        ln -s "${libdir}" "${ROOT_OVERLAY}/usr/lib"
     fi
     "${BUILD_LIBRARY_DIR}/set_lsb_release" \
-        --root "${TEMPDIR}/stage4_overlay"
+        --root "${ROOT_OVERLAY}"
 fi
 
+# toolchain_util.sh is required by catalyst_sdk.sh
+# To copy it, we need to create /tmp with the right permissions as it will be
+# used in the exported chroot.
+mkdir -p "${ROOT_OVERLAY}/tmp"
+chmod 1777 "${ROOT_OVERLAY}/tmp"
+cp "${BUILD_LIBRARY_DIR}/toolchain_util.sh" "${ROOT_OVERLAY}/tmp"
+
 catalyst_build
 
 if [[ "$STAGES" =~ stage4 ]]; then
@@ -97,6 +106,10 @@ if [[ "$STAGES" =~ stage4 ]]; then
         "$BUILDS/${release_name}.CONTENTS" "$BUILDS/${release_name}.DIGESTS"
     sign_and_upload_files "packages" "${def_upload_path}" "pkgs/" \
         "${BINPKGS}"/*
+
+    # Upload the SDK toolchain packages
+    sign_and_upload_files "cross toolchain packages" "${def_upload_path}" \
+        "toolchain/" "${BINPKGS}/crossdev"/*
 fi
 
 command_completed

build_library/catalyst_default_stage4.sh

@@ -1,9 +0,0 @@
-#!/bin/bash
-
-source /tmp/chroot-functions.sh
-
-echo "Double checking everything is fresh and happy."
-run_merge -uDN --with-bdeps=y world
-
-echo "Setting the default Python interpreter to Python 2."
-eselect python set python2.7

build_library/catalyst_sdk.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+set -e
+
+source /tmp/chroot-functions.sh
+source /tmp/toolchain_util.sh
+
+echo "Double checking everything is fresh and happy."
+run_merge -uDN --with-bdeps=y world
+
+echo "Setting the default Python interpreter to Python 2."
+eselect python set python2.7
+
+echo "Building cross toolchain for the SDK."
+configure_crossdev_overlay / /tmp/crossdev
+
+for cross_chost in $(get_chost_list); do
+    echo "Building cross toolchain for ${cross_chost}"
+    PKGDIR="$(portageq envvar PKGDIR)/crossdev" \
+        install_cross_toolchain "${cross_chost}" ${clst_myemergeopts}
+    PKGDIR="$(portageq envvar PKGDIR)/crossdev" \
+        install_cross_rust "${cross_chost}" ${clst_myemergeopts}
+done

build_library/catalyst_toolchains.sh

@@ -35,15 +35,11 @@ build_target_toolchain() {
     # --root is required because run_merge overrides ROOT=
     PORTAGE_CONFIGROOT="$ROOT" \
         run_merge -u --root="$ROOT" --sysroot="$ROOT" "${TOOLCHAIN_PKGS[@]}"
-
-    export clst_myemergeopts="$( echo "$clst_myemergeopts" | sed -e 's/--newuse//' )"
-
-    PORTAGE_CONFIGROOT="$ROOT" \
-        run_merge --root="$ROOT" --sysroot="$ROOT" virtual/rust
 }
 
 configure_crossdev_overlay / /tmp/crossdev
 
+# TODO: this is building the SDK packages and shouldn't actually be needed
 for cross_chost in $(get_chost_list); do
     echo "Building cross toolchain for ${cross_chost}"
     PKGDIR="$(portageq envvar PKGDIR)/crossdev" \

build_library/toolchain_util.sh

@@ -314,7 +314,7 @@ install_cross_toolchain() {
     fi
 
     # Only call crossdev to regenerate configs if something has changed
-    if ! cmp --quiet - "${cross_cfg}" <<<"${cross_cfg_data}"
+    if [[ ! -d "${cross_overlay}/cross-${cross_chost}" ]] || ! cmp --quiet - "${cross_cfg}" <<<"${cross_cfg_data}"
     then
         $sudo crossdev "${cross_flags[@]}" --init-target
         $sudo tee "${cross_cfg}" <<<"${cross_cfg_data}" >/dev/null
@@ -326,8 +326,10 @@ install_cross_toolchain() {
     if emerge "${emerge_flags[@]}" \
         --pretend "${cross_pkgs[@]}" | grep -q '^\[ebuild'
     then
+        echo "Doing a full bootstrap via crossdev"
         $sudo crossdev "${cross_flags[@]}" --stage4
     else
+        echo "Installing existing binaries"
         $sudo emerge "${emerge_flags[@]}" \
             "cross-${cross_chost}/gdb" "${cross_pkgs[@]}"
         if [ "${cross_chost}" = aarch64-cros-linux-gnu ]; then
@@ -392,6 +394,25 @@ install_cross_libs() {
     PORTAGE_CONFIGROOT="$ROOT" $sudo emerge --root="$ROOT" --sysroot="$ROOT" "$@" -u $cross_deps
 }
 
+install_cross_rust() {
+    local cross_chost="$1"; shift
+    local emerge_flags=( "$@" --binpkg-respect-use=y --update )
+
+    # may be called from either catalyst (root) or upgrade_chroot (user)
+    local sudo="env"
+    if [[ $(id -u) -ne 0 ]]; then
+        sudo="sudo -E"
+    fi
+
+    if [ "${cross_chost}" = "aarch64-cros-linux-gnu" ]; then
+        echo "Building Rust for arm64"
+        # If no aarch64 folder exists, try to remove any existing Rust packages.
+        [ ! -d /usr/lib/rust-*/rustlib/aarch64-unknown-linux-gnu ] && ($sudo emerge -C dev-lang/rust || true)
+        $sudo emerge "${emerge_flags[@]}" dev-lang/rust
+    fi
+}
+
+
 # Get the latest GCC profile for a given CHOST
 # The extra flag can be blank, hardenednopie, and so on. See gcc-config -l
 # Usage: gcc_get_latest_profile chost [extra]

build_toolchains

@@ -38,17 +38,6 @@ cp "${BUILD_LIBRARY_DIR}/toolchain_util.sh" "${ROOT_OVERLAY}/tmp"
 
 catalyst_build
 
-def_upload_path="${UPLOAD_ROOT}/sdk/${ARCH}/${FLAGS_version}"
-sign_and_upload_files "cross toolchain packages" "${def_upload_path}" \
-    "toolchain/" "${BINPKGS}/crossdev"/*
-
-for board in $(get_board_list); do
-    if [ "$board" = arm64-usr ]; then
-        sign_and_upload_files "Rust aarch64 crossdev toolchain packages" "${def_upload_path}" \
-            "toolchain-arm64/" "${BINPKGS}/target/${board}"/Packages "${BINPKGS}/target/${board}"/dev-lang
-    fi
-done
-
 # TODO: Actually just TOOLCHAIN_PKGS and the exact dependencies should be uploaded
 for board in $(get_board_list); do
     board_packages="${BINPKGS}/target/${board}"

common.sh

@@ -236,8 +236,9 @@ get_gclient_root() {
 # Populate the ENVIRONMENT_WHITELIST array.
 load_environment_whitelist() {
   ENVIRONMENT_WHITELIST=(
-    FLATCAR_BUILD_ID
     COREOS_OFFICIAL
+    FLATCAR_BUILD_ID
+    FORCE_STAGES
     GIT_AUTHOR_EMAIL
     GIT_AUTHOR_NAME
     GIT_COMMITTER_EMAIL

jenkins/sdk.sh

@@ -13,8 +13,11 @@ gpg --import "${GPG_SECRET_KEY_FILE}"
 # Wipe all of catalyst.
 sudo rm -rf src/build
 
-enter sudo FLATCAR_DEV_BUILDS_SDK="${DOWNLOAD_ROOT_SDK}" /mnt/host/source/src/scripts/bootstrap_sdk \
-    --sign="${SIGNING_USER}" \
-    --sign_digests="${SIGNING_USER}" \
-    --upload_root="${UPLOAD_ROOT}" \
-    --upload
+enter sudo \
+    FLATCAR_DEV_BUILDS_SDK="${DOWNLOAD_ROOT_SDK}" \
+    FORCE_STAGES="${FORCE_STAGES}" \
+    /mnt/host/source/src/scripts/bootstrap_sdk \
+        --sign="${SIGNING_USER}" \
+        --sign_digests="${SIGNING_USER}" \
+        --upload_root="${UPLOAD_ROOT}" \
+        --upload