From e04156a0f1fcd08042b9a80a6723e4b04504c863 Mon Sep 17 00:00:00 2001 From: Marga Manterola Date: Tue, 12 May 2020 11:46:17 +0200 Subject: [PATCH] bootstrap_sdk: build and upload SDK toolchain pkgs Before, we were relying on the toolchains job to build and upload packages that were part of the SDK. With this change, all packages that should be part of the SDK are built and uploaded by the SDK job. The toolchains job only builds toolchain packages specific for the release. This change includes several adjustments done to both the SDK and the toolchains jobs to make this work: * Make the SDK job build all cross toolchains, including Rust * Stop building Rust in the toolchains job and use the one in the SDK instead. * In toolchain_util.sh: detect when the symlink folder for crossdev packages is missing and run crossdev to create it during update_chroot setup. * Make it possible to build the SDK starting from stage 4 instead of stage 1, to make the SDK building faster for PR branches / nightlies (full build should still be done for releases / weeklies). --- bootstrap_sdk | 25 ++++++++++++++++++------ build_library/catalyst_default_stage4.sh | 9 --------- build_library/catalyst_sdk.sh | 22 +++++++++++++++++++++ build_library/catalyst_toolchains.sh | 6 +----- build_library/toolchain_util.sh | 23 +++++++++++++++++++++- build_toolchains | 11 ----------- common.sh | 3 ++- jenkins/sdk.sh | 13 +++++++----- 8 files changed, 74 insertions(+), 38 deletions(-) delete mode 100644 build_library/catalyst_default_stage4.sh create mode 100644 build_library/catalyst_sdk.sh diff --git a/bootstrap_sdk b/bootstrap_sdk index 2193b8f357..9bfbe92c77 100755 --- a/bootstrap_sdk +++ b/bootstrap_sdk @@ -40,14 +40,16 @@ TYPE="flatcar-sdk" # include upload options . "${BUILD_LIBRARY_DIR}/release_util.sh" || exit 1 +ROOT_OVERLAY=${TEMPDIR}/stage4_overlay + ## Define the stage4 config template catalyst_stage4() { cat <lib64 symlink correctly libdir=$(get_sdk_libdir) - mkdir -p "${TEMPDIR}/stage4_overlay/usr/${libdir}" + mkdir -p "${ROOT_OVERLAY}/usr/${libdir}" if [[ "${libdir}" != lib ]]; then - ln -s "${libdir}" "${TEMPDIR}/stage4_overlay/usr/lib" + ln -s "${libdir}" "${ROOT_OVERLAY}/usr/lib" fi "${BUILD_LIBRARY_DIR}/set_lsb_release" \ - --root "${TEMPDIR}/stage4_overlay" + --root "${ROOT_OVERLAY}" fi +# toolchain_util.sh is required by catalyst_sdk.sh +# To copy it, we need to create /tmp with the right permissions as it will be +# used in the exported chroot. +mkdir -p "${ROOT_OVERLAY}/tmp" +chmod 1777 "${ROOT_OVERLAY}/tmp" +cp "${BUILD_LIBRARY_DIR}/toolchain_util.sh" "${ROOT_OVERLAY}/tmp" + catalyst_build if [[ "$STAGES" =~ stage4 ]]; then @@ -97,6 +106,10 @@ if [[ "$STAGES" =~ stage4 ]]; then "$BUILDS/${release_name}.CONTENTS" "$BUILDS/${release_name}.DIGESTS" sign_and_upload_files "packages" "${def_upload_path}" "pkgs/" \ "${BINPKGS}"/* + + # Upload the SDK toolchain packages + sign_and_upload_files "cross toolchain packages" "${def_upload_path}" \ + "toolchain/" "${BINPKGS}/crossdev"/* fi command_completed diff --git a/build_library/catalyst_default_stage4.sh b/build_library/catalyst_default_stage4.sh deleted file mode 100644 index a1a3be9f13..0000000000 --- a/build_library/catalyst_default_stage4.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -source /tmp/chroot-functions.sh - -echo "Double checking everything is fresh and happy." -run_merge -uDN --with-bdeps=y world - -echo "Setting the default Python interpreter to Python 2." -eselect python set python2.7 diff --git a/build_library/catalyst_sdk.sh b/build_library/catalyst_sdk.sh new file mode 100644 index 0000000000..edc739ab98 --- /dev/null +++ b/build_library/catalyst_sdk.sh @@ -0,0 +1,22 @@ +#!/bin/bash +set -e + +source /tmp/chroot-functions.sh +source /tmp/toolchain_util.sh + +echo "Double checking everything is fresh and happy." +run_merge -uDN --with-bdeps=y world + +echo "Setting the default Python interpreter to Python 2." +eselect python set python2.7 + +echo "Building cross toolchain for the SDK." +configure_crossdev_overlay / /tmp/crossdev + +for cross_chost in $(get_chost_list); do + echo "Building cross toolchain for ${cross_chost}" + PKGDIR="$(portageq envvar PKGDIR)/crossdev" \ + install_cross_toolchain "${cross_chost}" ${clst_myemergeopts} + PKGDIR="$(portageq envvar PKGDIR)/crossdev" \ + install_cross_rust "${cross_chost}" ${clst_myemergeopts} +done diff --git a/build_library/catalyst_toolchains.sh b/build_library/catalyst_toolchains.sh index ded91c49c7..939fe3d4e3 100644 --- a/build_library/catalyst_toolchains.sh +++ b/build_library/catalyst_toolchains.sh @@ -35,15 +35,11 @@ build_target_toolchain() { # --root is required because run_merge overrides ROOT= PORTAGE_CONFIGROOT="$ROOT" \ run_merge -u --root="$ROOT" --sysroot="$ROOT" "${TOOLCHAIN_PKGS[@]}" - - export clst_myemergeopts="$( echo "$clst_myemergeopts" | sed -e 's/--newuse//' )" - - PORTAGE_CONFIGROOT="$ROOT" \ - run_merge --root="$ROOT" --sysroot="$ROOT" virtual/rust } configure_crossdev_overlay / /tmp/crossdev +# TODO: this is building the SDK packages and shouldn't actually be needed for cross_chost in $(get_chost_list); do echo "Building cross toolchain for ${cross_chost}" PKGDIR="$(portageq envvar PKGDIR)/crossdev" \ diff --git a/build_library/toolchain_util.sh b/build_library/toolchain_util.sh index 1352257f26..0e8889cace 100644 --- a/build_library/toolchain_util.sh +++ b/build_library/toolchain_util.sh @@ -314,7 +314,7 @@ install_cross_toolchain() { fi # Only call crossdev to regenerate configs if something has changed - if ! cmp --quiet - "${cross_cfg}" <<<"${cross_cfg_data}" + if [[ ! -d "${cross_overlay}/cross-${cross_chost}" ]] || ! cmp --quiet - "${cross_cfg}" <<<"${cross_cfg_data}" then $sudo crossdev "${cross_flags[@]}" --init-target $sudo tee "${cross_cfg}" <<<"${cross_cfg_data}" >/dev/null @@ -326,8 +326,10 @@ install_cross_toolchain() { if emerge "${emerge_flags[@]}" \ --pretend "${cross_pkgs[@]}" | grep -q '^\[ebuild' then + echo "Doing a full bootstrap via crossdev" $sudo crossdev "${cross_flags[@]}" --stage4 else + echo "Installing existing binaries" $sudo emerge "${emerge_flags[@]}" \ "cross-${cross_chost}/gdb" "${cross_pkgs[@]}" if [ "${cross_chost}" = aarch64-cros-linux-gnu ]; then @@ -392,6 +394,25 @@ install_cross_libs() { PORTAGE_CONFIGROOT="$ROOT" $sudo emerge --root="$ROOT" --sysroot="$ROOT" "$@" -u $cross_deps } +install_cross_rust() { + local cross_chost="$1"; shift + local emerge_flags=( "$@" --binpkg-respect-use=y --update ) + + # may be called from either catalyst (root) or upgrade_chroot (user) + local sudo="env" + if [[ $(id -u) -ne 0 ]]; then + sudo="sudo -E" + fi + + if [ "${cross_chost}" = "aarch64-cros-linux-gnu" ]; then + echo "Building Rust for arm64" + # If no aarch64 folder exists, try to remove any existing Rust packages. + [ ! -d /usr/lib/rust-*/rustlib/aarch64-unknown-linux-gnu ] && ($sudo emerge -C dev-lang/rust || true) + $sudo emerge "${emerge_flags[@]}" dev-lang/rust + fi +} + + # Get the latest GCC profile for a given CHOST # The extra flag can be blank, hardenednopie, and so on. See gcc-config -l # Usage: gcc_get_latest_profile chost [extra] diff --git a/build_toolchains b/build_toolchains index 76d1cbb0cf..85c09001a9 100755 --- a/build_toolchains +++ b/build_toolchains @@ -38,17 +38,6 @@ cp "${BUILD_LIBRARY_DIR}/toolchain_util.sh" "${ROOT_OVERLAY}/tmp" catalyst_build -def_upload_path="${UPLOAD_ROOT}/sdk/${ARCH}/${FLAGS_version}" -sign_and_upload_files "cross toolchain packages" "${def_upload_path}" \ - "toolchain/" "${BINPKGS}/crossdev"/* - -for board in $(get_board_list); do - if [ "$board" = arm64-usr ]; then - sign_and_upload_files "Rust aarch64 crossdev toolchain packages" "${def_upload_path}" \ - "toolchain-arm64/" "${BINPKGS}/target/${board}"/Packages "${BINPKGS}/target/${board}"/dev-lang - fi -done - # TODO: Actually just TOOLCHAIN_PKGS and the exact dependencies should be uploaded for board in $(get_board_list); do board_packages="${BINPKGS}/target/${board}" diff --git a/common.sh b/common.sh index 37cee5e809..7928df1ef5 100644 --- a/common.sh +++ b/common.sh @@ -236,8 +236,9 @@ get_gclient_root() { # Populate the ENVIRONMENT_WHITELIST array. load_environment_whitelist() { ENVIRONMENT_WHITELIST=( - FLATCAR_BUILD_ID COREOS_OFFICIAL + FLATCAR_BUILD_ID + FORCE_STAGES GIT_AUTHOR_EMAIL GIT_AUTHOR_NAME GIT_COMMITTER_EMAIL diff --git a/jenkins/sdk.sh b/jenkins/sdk.sh index 526c1d9f21..c072941ccd 100644 --- a/jenkins/sdk.sh +++ b/jenkins/sdk.sh @@ -13,8 +13,11 @@ gpg --import "${GPG_SECRET_KEY_FILE}" # Wipe all of catalyst. sudo rm -rf src/build -enter sudo FLATCAR_DEV_BUILDS_SDK="${DOWNLOAD_ROOT_SDK}" /mnt/host/source/src/scripts/bootstrap_sdk \ - --sign="${SIGNING_USER}" \ - --sign_digests="${SIGNING_USER}" \ - --upload_root="${UPLOAD_ROOT}" \ - --upload +enter sudo \ + FLATCAR_DEV_BUILDS_SDK="${DOWNLOAD_ROOT_SDK}" \ + FORCE_STAGES="${FORCE_STAGES}" \ + /mnt/host/source/src/scripts/bootstrap_sdk \ + --sign="${SIGNING_USER}" \ + --sign_digests="${SIGNING_USER}" \ + --upload_root="${UPLOAD_ROOT}" \ + --upload