From 14d245c19ac2bbf5df49a816241959edc21a07c7 Mon Sep 17 00:00:00 2001 From: Marga Manterola Date: Tue, 12 May 2020 11:46:17 +0200 Subject: [PATCH] bootstrap_sdk: build and upload SDK toolchain pkgs Before, we were relying on the toolchains job to build and upload packages that were part of the SDK. With these change, all packages that should be part of the SDK are built and uploaded by the SDK job. --- bootstrap_sdk | 25 ++++++++++++++++++------ build_library/catalyst_default_stage4.sh | 9 --------- build_library/catalyst_sdk.sh | 22 +++++++++++++++++++++ build_library/catalyst_toolchains.sh | 1 + build_library/toolchain_util.sh | 21 ++++++++++++++++++++ build_toolchains | 11 ----------- common.sh | 3 ++- jenkins/sdk.sh | 13 +++++++----- 8 files changed, 73 insertions(+), 32 deletions(-) delete mode 100644 build_library/catalyst_default_stage4.sh create mode 100644 build_library/catalyst_sdk.sh diff --git a/bootstrap_sdk b/bootstrap_sdk index 2193b8f357..9bfbe92c77 100755 --- a/bootstrap_sdk +++ b/bootstrap_sdk @@ -40,14 +40,16 @@ TYPE="flatcar-sdk" # include upload options . "${BUILD_LIBRARY_DIR}/release_util.sh" || exit 1 +ROOT_OVERLAY=${TEMPDIR}/stage4_overlay + ## Define the stage4 config template catalyst_stage4() { cat <lib64 symlink correctly libdir=$(get_sdk_libdir) - mkdir -p "${TEMPDIR}/stage4_overlay/usr/${libdir}" + mkdir -p "${ROOT_OVERLAY}/usr/${libdir}" if [[ "${libdir}" != lib ]]; then - ln -s "${libdir}" "${TEMPDIR}/stage4_overlay/usr/lib" + ln -s "${libdir}" "${ROOT_OVERLAY}/usr/lib" fi "${BUILD_LIBRARY_DIR}/set_lsb_release" \ - --root "${TEMPDIR}/stage4_overlay" + --root "${ROOT_OVERLAY}" fi +# toolchain_util.sh is required by catalyst_sdk.sh +# To copy it, we need to create /tmp with the right permissions as it will be +# used in the exported chroot. +mkdir -p "${ROOT_OVERLAY}/tmp" +chmod 1777 "${ROOT_OVERLAY}/tmp" +cp "${BUILD_LIBRARY_DIR}/toolchain_util.sh" "${ROOT_OVERLAY}/tmp" + catalyst_build if [[ "$STAGES" =~ stage4 ]]; then @@ -97,6 +106,10 @@ if [[ "$STAGES" =~ stage4 ]]; then "$BUILDS/${release_name}.CONTENTS" "$BUILDS/${release_name}.DIGESTS" sign_and_upload_files "packages" "${def_upload_path}" "pkgs/" \ "${BINPKGS}"/* + + # Upload the SDK toolchain packages + sign_and_upload_files "cross toolchain packages" "${def_upload_path}" \ + "toolchain/" "${BINPKGS}/crossdev"/* fi command_completed diff --git a/build_library/catalyst_default_stage4.sh b/build_library/catalyst_default_stage4.sh deleted file mode 100644 index a1a3be9f13..0000000000 --- a/build_library/catalyst_default_stage4.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -source /tmp/chroot-functions.sh - -echo "Double checking everything is fresh and happy." -run_merge -uDN --with-bdeps=y world - -echo "Setting the default Python interpreter to Python 2." -eselect python set python2.7 diff --git a/build_library/catalyst_sdk.sh b/build_library/catalyst_sdk.sh new file mode 100644 index 0000000000..edc739ab98 --- /dev/null +++ b/build_library/catalyst_sdk.sh @@ -0,0 +1,22 @@ +#!/bin/bash +set -e + +source /tmp/chroot-functions.sh +source /tmp/toolchain_util.sh + +echo "Double checking everything is fresh and happy." +run_merge -uDN --with-bdeps=y world + +echo "Setting the default Python interpreter to Python 2." +eselect python set python2.7 + +echo "Building cross toolchain for the SDK." +configure_crossdev_overlay / /tmp/crossdev + +for cross_chost in $(get_chost_list); do + echo "Building cross toolchain for ${cross_chost}" + PKGDIR="$(portageq envvar PKGDIR)/crossdev" \ + install_cross_toolchain "${cross_chost}" ${clst_myemergeopts} + PKGDIR="$(portageq envvar PKGDIR)/crossdev" \ + install_cross_rust "${cross_chost}" ${clst_myemergeopts} +done diff --git a/build_library/catalyst_toolchains.sh b/build_library/catalyst_toolchains.sh index f5ef591e5f..5700662f46 100644 --- a/build_library/catalyst_toolchains.sh +++ b/build_library/catalyst_toolchains.sh @@ -44,6 +44,7 @@ build_target_toolchain() { configure_crossdev_overlay / /tmp/crossdev +# TODO: this is building the SDK packages and shouldn't actually be needed for cross_chost in $(get_chost_list); do echo "Building cross toolchain for ${cross_chost}" PKGDIR="$(portageq envvar PKGDIR)/crossdev" \ diff --git a/build_library/toolchain_util.sh b/build_library/toolchain_util.sh index 1024a4e2fb..1e853d3cc4 100644 --- a/build_library/toolchain_util.sh +++ b/build_library/toolchain_util.sh @@ -326,8 +326,10 @@ install_cross_toolchain() { if emerge "${emerge_flags[@]}" \ --pretend "${cross_pkgs[@]}" | grep -q '^\[ebuild' then + echo "Doing a full bootstrap via crossdev" $sudo crossdev "${cross_flags[@]}" --stage4 else + echo "Installing existing binaries" $sudo emerge "${emerge_flags[@]}" \ "cross-${cross_chost}/gdb" "${cross_pkgs[@]}" if [ "${cross_chost}" = aarch64-cros-linux-gnu ]; then @@ -392,6 +394,25 @@ install_cross_libs() { PORTAGE_CONFIGROOT="$ROOT" $sudo emerge --root="$ROOT" --sysroot="$ROOT" "$@" -u $cross_deps } +install_cross_rust() { + local cross_chost="$1"; shift + local emerge_flags=( "$@" --binpkg-respect-use=y --update ) + + # may be called from either catalyst (root) or upgrade_chroot (user) + local sudo="env" + if [[ $(id -u) -ne 0 ]]; then + sudo="sudo -E" + fi + + if [ "${cross_chost}" = "aarch64-cros-linux-gnu" ]; then + echo "Building Rust for arm64" + # If no aarch64 folder exists, try to remove any existing Rust packages. + [ ! -d /usr/lib/rust-*/rustlib/aarch64-unknown-linux-gnu ] && ($sudo emerge -C dev-lang/rust || true) + $sudo emerge "${emerge_flags[@]}" dev-lang/rust + fi +} + + # Get the latest GCC profile for a given CHOST # The extra flag can be blank, hardenednopie, and so on. See gcc-config -l # Usage: gcc_get_latest_profile chost [extra] diff --git a/build_toolchains b/build_toolchains index 76d1cbb0cf..85c09001a9 100755 --- a/build_toolchains +++ b/build_toolchains @@ -38,17 +38,6 @@ cp "${BUILD_LIBRARY_DIR}/toolchain_util.sh" "${ROOT_OVERLAY}/tmp" catalyst_build -def_upload_path="${UPLOAD_ROOT}/sdk/${ARCH}/${FLAGS_version}" -sign_and_upload_files "cross toolchain packages" "${def_upload_path}" \ - "toolchain/" "${BINPKGS}/crossdev"/* - -for board in $(get_board_list); do - if [ "$board" = arm64-usr ]; then - sign_and_upload_files "Rust aarch64 crossdev toolchain packages" "${def_upload_path}" \ - "toolchain-arm64/" "${BINPKGS}/target/${board}"/Packages "${BINPKGS}/target/${board}"/dev-lang - fi -done - # TODO: Actually just TOOLCHAIN_PKGS and the exact dependencies should be uploaded for board in $(get_board_list); do board_packages="${BINPKGS}/target/${board}" diff --git a/common.sh b/common.sh index 37cee5e809..7928df1ef5 100644 --- a/common.sh +++ b/common.sh @@ -236,8 +236,9 @@ get_gclient_root() { # Populate the ENVIRONMENT_WHITELIST array. load_environment_whitelist() { ENVIRONMENT_WHITELIST=( - FLATCAR_BUILD_ID COREOS_OFFICIAL + FLATCAR_BUILD_ID + FORCE_STAGES GIT_AUTHOR_EMAIL GIT_AUTHOR_NAME GIT_COMMITTER_EMAIL diff --git a/jenkins/sdk.sh b/jenkins/sdk.sh index 526c1d9f21..c072941ccd 100644 --- a/jenkins/sdk.sh +++ b/jenkins/sdk.sh @@ -13,8 +13,11 @@ gpg --import "${GPG_SECRET_KEY_FILE}" # Wipe all of catalyst. sudo rm -rf src/build -enter sudo FLATCAR_DEV_BUILDS_SDK="${DOWNLOAD_ROOT_SDK}" /mnt/host/source/src/scripts/bootstrap_sdk \ - --sign="${SIGNING_USER}" \ - --sign_digests="${SIGNING_USER}" \ - --upload_root="${UPLOAD_ROOT}" \ - --upload +enter sudo \ + FLATCAR_DEV_BUILDS_SDK="${DOWNLOAD_ROOT_SDK}" \ + FORCE_STAGES="${FORCE_STAGES}" \ + /mnt/host/source/src/scripts/bootstrap_sdk \ + --sign="${SIGNING_USER}" \ + --sign_digests="${SIGNING_USER}" \ + --upload_root="${UPLOAD_ROOT}" \ + --upload