From 6778eeda8332a692d91e2b2bf677436bfaa2a815 Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Wed, 23 Oct 2013 12:53:15 -0700 Subject: [PATCH 01/13] fix(bootstrap_sdk): Remove toolchain build step, replaced by build_toolchains --- bootstrap_sdk | 2 +- build_container | 2 +- ...t_stage4.sh => catalyst_default_stage4.sh} | 0 build_library/catalyst_sdk_stage4.sh | 36 ------------------- 4 files changed, 2 insertions(+), 38 deletions(-) rename build_library/{catalyst_buildbot_stage4.sh => catalyst_default_stage4.sh} (100%) delete mode 100644 build_library/catalyst_sdk_stage4.sh diff --git a/bootstrap_sdk b/bootstrap_sdk index 59a1634109..747ddeb1c3 100755 --- a/bootstrap_sdk +++ b/bootstrap_sdk @@ -48,7 +48,7 @@ cat </dev/null - then - run_merge -u "${cross_pkgs[@]}" - else - crossdev --ov-output "/usr/local/portage/crossdev" \ - --portage "${clst_myemergeopts}" \ - --env 'FEATURES=splitdebug' \ - --stable --ex-gdb --stage4 \ - --target "${cross_chost}" || exit 1 - fi - - # There is no point to including the built packages in the final tarball - # because the packages will have to be downloaded anyway due to how the - # cross toolchains are managed in board sysroots. - crossdev --force -C "${cross_chost}" -done - -echo "Double checking everything is fresh and happy." -run_merge -uDN --with-bdeps=y world From 0ba3d9ea04459805f0c471bd75daa2a886dbed67 Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Wed, 23 Oct 2013 13:42:14 -0700 Subject: [PATCH 02/13] fix(get_board_chost): Accept a list of boards to get_board_* funcs --- build_library/toolchain_util.sh | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/build_library/toolchain_util.sh b/build_library/toolchain_util.sh index 716207a6fa..c8156433ff 100644 --- a/build_library/toolchain_util.sh +++ b/build_library/toolchain_util.sh @@ -42,17 +42,23 @@ get_profile_list() { } get_board_chost() { - if [[ ${#BOARD_CHOST["$1"]} -ne 0 ]]; then - echo "${BOARD_CHOST["$1"]}" - else - die "Unknown board '$1'" - fi + local board + for board in "$@"; do + if [[ ${#BOARD_CHOST["$board"]} -ne 0 ]]; then + echo "${BOARD_CHOST["$board"]}" + else + die "Unknown board '$board'" + fi + done } get_board_profile() { - if [[ ${#BOARD_PROFILE["$1"]} -ne 0 ]]; then - echo "${BOARD_PROFILE["$1"]}" - else - die "Unknown board '$1'" - fi + local board + for board in "$@"; do + if [[ ${#BOARD_PROFILE["$board"]} -ne 0 ]]; then + echo "${BOARD_PROFILE["$board"]}" + else + die "Unknown board '$board'" + fi + done } From 986548722d7a144d83bbd7d5125a2d9bc376721b Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Wed, 23 Oct 2013 13:56:35 -0700 Subject: [PATCH 03/13] fix(toolchain_util.sh): Move native->cross pkg translation to function Will be useful in other places and seems better to hide the bash magic. --- build_library/catalyst_toolchains.sh | 2 +- build_library/toolchain_util.sh | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/build_library/catalyst_toolchains.sh b/build_library/catalyst_toolchains.sh index 066b599cd1..1e0ea642f5 100644 --- a/build_library/catalyst_toolchains.sh +++ b/build_library/catalyst_toolchains.sh @@ -47,7 +47,7 @@ EOF build_cross_toolchain() { local cross_chost="$1" - local cross_pkgs=( "${TOOLCHAIN_PKGS[@]/*\//cross-${cross_chost}/}" ) + local cross_pkgs=( $(get_cross_pkgs $cross_chost) ) local PORTDIR="$(portageq envvar PORTDIR)" local PKGDIR="${PORTDIR}/packages/crossdev" diff --git a/build_library/toolchain_util.sh b/build_library/toolchain_util.sh index c8156433ff..f51e8d16ac 100644 --- a/build_library/toolchain_util.sh +++ b/build_library/toolchain_util.sh @@ -62,3 +62,12 @@ get_board_profile() { fi done } + +get_cross_pkgs() { + local cross_chost native_pkg + for cross_chost in "$@"; do + for native_pkg in "${TOOLCHAIN_PKGS[@]}"; do + echo "${native_pkg/*\//cross-${cross_chost}/}" + done + done +} From 55a4517d65a04eec5a5167ebac214336d262d63f Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Wed, 23 Oct 2013 15:08:49 -0700 Subject: [PATCH 04/13] fix(common.sh): Use $SCRIPT_NAME as the default log prefix. Seems useful with all the scripts-calling-scripts-turtles but very few scripts explicitly set a prefix. --- common.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/common.sh b/common.sh index da89510005..b1cba1c95f 100644 --- a/common.sh +++ b/common.sh @@ -76,10 +76,10 @@ _dump_trace() { # Declare these asap so that code below can safely assume they exist. _message() { - local prefix=$1 + local prefix="$1${CROS_LOG_PREFIX:-${SCRIPT_NAME}}" shift if [[ $# -eq 0 ]]; then - echo -e "${prefix}${CROS_LOG_PREFIX:-""}:${V_VIDOFF}" >&2 + echo -e "${prefix}:${V_VIDOFF}" >&2 return fi ( @@ -95,7 +95,7 @@ _message() { set -- '' fi for line in "$@"; do - echo -e "${prefix}${CROS_LOG_PREFIX:-}: ${line}${V_VIDOFF}" >&2 + echo -e "${prefix}: ${line}${V_VIDOFF}" >&2 done ) } From e145b916f2d1db9979c45ac1340f135317ee4678 Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Wed, 6 Nov 2013 15:09:41 -0800 Subject: [PATCH 05/13] fix(catalyst.sh): Update catalyst build more aggressively. Use 2*CPUs for the target load average but add load average throttling to emerge in addition to make. Also work around how catalyst sets FEATURES so we can disable extra locking for hopefully faster builds. --- build_library/catalyst.sh | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/build_library/catalyst.sh b/build_library/catalyst.sh index e5a2d642d6..8d53a23863 100644 --- a/build_library/catalyst.sh +++ b/build_library/catalyst.sh @@ -77,9 +77,14 @@ EOF } catalystrc() { -echo "export MAKEOPTS='--jobs=${NUM_JOBS} --load-average=${NUM_JOBS}'" -echo "export EMERGE_DEFAULT_OPTS=--jobs=${NUM_JOBS}" -echo "export FEATURES='parallel-install -ebuild-locks'" +local load=$((NUM_JOBS * 2)) +cat < Date: Wed, 6 Nov 2013 15:28:55 -0800 Subject: [PATCH 06/13] fix(toolchain_util.sh): Move cross-toolchain install code to common lib. This will eventually be used by setup_board instead of the even more complicated toolchain code provided by chromite. --- build_library/catalyst_toolchains.sh | 104 ++++++--------------------- build_library/toolchain_util.sh | 84 ++++++++++++++++++++++ 2 files changed, 105 insertions(+), 83 deletions(-) diff --git a/build_library/catalyst_toolchains.sh b/build_library/catalyst_toolchains.sh index 1e0ea642f5..c09ebbfd9a 100644 --- a/build_library/catalyst_toolchains.sh +++ b/build_library/catalyst_toolchains.sh @@ -11,83 +11,18 @@ source /tmp/toolchain_util.sh # build deps in crossdev's sysroot use ${PKGDIR}/cross/${CHOST} (no upload) # native toolchains use ${PKGDIR}/target/${BOARD} (uploaded to board location) -get_dependency_list() { - local ROOT="$1" - local IFS=$'| \t\n' - shift - - PORTAGE_CONFIGROOT="$ROOT" SYSROOT="$ROOT" \ - ROOT="$ROOT" emerge ${clst_myemergeopts} \ - --pretend --with-bdeps=y --onlydeps --quiet \ - "$@" | sed -e 's/.*\] \([^ :]*\).*/=\1/' | - egrep -v "(=$(echo "$*")-[0-9])" -} - -configure_portage() { - local pkg_path="$1" - local profile="$2" - - mkdir -p "${ROOT}/etc/portage" - echo "eselect will report '!!! Warning: Strange path.' but that's OK" - eselect profile set --force "$profile" - - cat >"${ROOT}/etc/portage/make.conf" </dev/null - then - PKGDIR="${PKGDIR}" run_merge -u "${cross_pkgs[@]}" - else - PKGDIR="${PKGDIR}" crossdev \ - --ov-output "/tmp/crossdev" --stable \ - --env PKGDIR="${PORTDIR}/packages/crossdev" \ - --portage "${clst_myemergeopts}" \ - --stage4 --target "${cross_chost}" - fi - - # Setup ccache for our shiny new toolchain - ccache-config --install-links "${cross_chost}" -} - -configure_cross_root() { +build_cross_libs() { local cross_chost="$1" local ROOT="/usr/${cross_chost}" CHOST="${cross_chost}" ROOT="$ROOT" SYSROOT="$ROOT" \ - configure_portage "cross/${cross_chost}" \ - "${CROSS_PROFILE[${cross_chost}]}" + _configure_sysroot "${CROSS_PROFILE[${cross_chost}]}" # In order to get a dependency list we must calculate it before # updating package.provided. Otherwise portage will no-op. - get_dependency_list "$ROOT" "${TOOLCHAIN_PKGS[@]}" \ - > "$ROOT/etc/portage/cross-${cross_chost}-depends" + ROOT="$ROOT" _get_dependency_list \ + ${clst_myemergeopts} "${TOOLCHAIN_PKGS[@]}" > \ + "$ROOT/etc/portage/cross-${cross_chost}-depends" # Add toolchain to packages.provided since they are on the host system mkdir -p "$ROOT/etc/portage/profile/package.provided" @@ -97,11 +32,6 @@ configure_cross_root() { cross_pkg_version=$(portageq match / "${cross_pkg}") echo "${native_pkg%/*}/${cross_pkg_version#*/}" done > "$ROOT/etc/portage/profile/package.provided/cross-${cross_chost}" -} - -build_cross_libs() { - local cross_chost="$1" - local ROOT="/usr/${cross_chost}" local cross_deps=$(<"$ROOT/etc/portage/cross-${cross_chost}-depends") # --root is required because run_merge overrides ROOT= @@ -111,10 +41,12 @@ build_cross_libs() { configure_target_root() { local board="$1" local cross_chost=$(get_board_chost "$1") + local profile=$(get_board_profile "${board}") - CHOST="${cross_chost}" ROOT="/build/${board}" \ - SYSROOT="/usr/${cross_chost}" configure_portage \ - "target/${board}" "$(get_board_profile "${board}")" + CHOST="${cross_chost}" \ + ROOT="/build/${board}" \ + SYSROOT="/usr/${cross_chost}" \ + _configure_sysroot "${profile}" } build_target_toolchain() { @@ -126,15 +58,21 @@ build_target_toolchain() { run_merge -u --root="$ROOT" "${TOOLCHAIN_PKGS[@]}" } +mkdir -p "/tmp/crossdev" +export PORTDIR_OVERLAY="/tmp/crossdev $(portageq envvar PORTDIR_OVERLAY)" + for cross_chost in $(get_chost_list); do echo "Building cross toolchain for ${cross_chost}" - build_cross_toolchain "${cross_chost}" - configure_cross_root "${cross_chost}" - build_cross_libs "${cross_chost}" + PKGDIR="$(portageq envvar PKGDIR)/crossdev" \ + install_cross_toolchain "${cross_chost}" ${clst_myemergeopts} + + sysroot_pkgdir="$(portageq envvar PKGDIR)/cross/${cross_chost}" + PKGDIR="${sysroot_pkgdir}" build_cross_libs "${cross_chost}" done for board in $(get_board_list); do echo "Building native toolchain for ${board}" - configure_target_root "${board}" - build_target_toolchain "${board}" + target_pkgdir="$(portageq envvar PKGDIR)/target/${board}" + PKGDIR="${target_pkgdir}" configure_target_root "${board}" + PKGDIR="${target_pkgdir}" build_target_toolchain "${board}" done diff --git a/build_library/toolchain_util.sh b/build_library/toolchain_util.sh index f51e8d16ac..231f9b3ea8 100644 --- a/build_library/toolchain_util.sh +++ b/build_library/toolchain_util.sh @@ -71,3 +71,87 @@ get_cross_pkgs() { done done } + +### Toolchain building utilities ### + +# Ugly hack to get a dependency list of a set of packages. +# This is required to figure out what to install in the crossdev sysroot. +# Usage: ROOT=/foo/bar _get_dependency_list pkgs... [--portage-opts...] +_get_dependency_list() { + local pkgs=( ${*/#-*/} ) + local IFS=$'| \t\n' + + PORTAGE_CONFIGROOT="$ROOT" emerge "$@" --pretend \ + --emptytree --root-deps=rdeps --onlydeps --quiet | \ + sed -e 's/.*\] \([^ :]*\).*/=\1/' | + egrep -v "(=$(echo "${pkgs[*]}")-[0-9])" +} + +# Configure a new ROOT +# Values are copied from the environment or the current host configuration. +# Usage: ROOT=/foo/bar SYSROOT=/foo/bar configure_portage coreos:some/profile +_configure_sysroot() { + local profile="$1" + + mkdir -p "${ROOT}/etc/portage" + echo "eselect will report '!!! Warning: Strange path.' but that's OK" + eselect profile set --force "$profile" + + cat >"${ROOT}/etc/portage/make.conf" </dev/null + then + $sudo emerge "$@" --binpkg-respect-use=y -u "${cross_pkgs[@]}" + else + $sudo crossdev --stable --portage "$*" \ + --stage4 --target "${cross_chost}" + fi + + # Setup wrappers for our shiny new toolchain + if [[ ! -e "/usr/lib/ccache/bin/${cross_chost}-gcc" ]]; then + $sudo ccache-config --install-links "${cross_chost}" + fi + if [[ ! -e "/usr/lib/sysroot-wrappers/bin/${cross_chost}-gcc" ]]; then + $sudo sysroot-config --install-links "${cross_chost}" + fi +} From b5816f7a8d3f69cb8d769a7d6e784c23bbe9796c Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Wed, 6 Nov 2013 15:44:12 -0800 Subject: [PATCH 07/13] fix(disk_layout_util.sh): Fix changing build_image's disk layout. Previously the code in base_image_util.sh properly handled the disk layout command line flag but the spaghetti code later on calls a function from disk_layout_util.sh which only returned 'base' resulting in a bit of a mess if something other than 'base' is used. Sync up the two code paths to avoid that... --- build_library/base_image_util.sh | 6 ++---- build_library/disk_layout_util.sh | 4 ++++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/build_library/base_image_util.sh b/build_library/base_image_util.sh index 56ff4eef61..9006021be6 100755 --- a/build_library/base_image_util.sh +++ b/build_library/base_image_util.sh @@ -39,11 +39,9 @@ cleanup_mounts() { create_base_image() { local image_name=$1 local rootfs_verification_enabled=$2 - local image_type="base" - if [[ "${FLAGS_disk_layout}" != "default" ]]; then - image_type="${FLAGS_disk_layout}" - fi + get_disk_layout_type + local image_type="${DISK_LAYOUT_TYPE}" check_valid_layout "base" check_valid_layout ${image_type} diff --git a/build_library/disk_layout_util.sh b/build_library/disk_layout_util.sh index 56d8cb2a4c..e337111e70 100644 --- a/build_library/disk_layout_util.sh +++ b/build_library/disk_layout_util.sh @@ -112,6 +112,10 @@ check_valid_layout() { get_disk_layout_type() { DISK_LAYOUT_TYPE="base" + if [[ -n "${FLAGS_disk_layout}" && \ + "${FLAGS_disk_layout}" != "default" ]]; then + DISK_LAYOUT_TYPE="${FLAGS_disk_layout}" + fi } emit_gpt_scripts() { From 41099eef6ffb39ef2027a7a690e5ab2fb1b80b82 Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Wed, 6 Nov 2013 17:24:46 -0800 Subject: [PATCH 08/13] fix(board_options.sh): Move CHOST->ARCH mapping code. Use the complete chost to portage arch mapping list from crossdev and move the logic to toolchain_util.sh where it can be used by other scripts. --- bin/cros_make_image_bootable | 1 + build_image | 1 + build_library/board_options.sh | 19 +---------------- build_library/toolchain_util.sh | 38 ++++++++++++++++++++++++++++++++- build_packages | 1 + image_to_vm.sh | 1 + 6 files changed, 42 insertions(+), 19 deletions(-) diff --git a/bin/cros_make_image_bootable b/bin/cros_make_image_bootable index 6fb60d1ac0..6d33cf1cfd 100755 --- a/bin/cros_make_image_bootable +++ b/bin/cros_make_image_bootable @@ -33,6 +33,7 @@ assert_inside_chroot # Load functions and constants for chromeos-install . /usr/lib/installer/chromeos-common.sh || exit 1 +. "${BUILD_LIBRARY_DIR}/toolchain_util.sh" || exit 1 . "${BUILD_LIBRARY_DIR}/build_image_util.sh" || exit 1 . "${BUILD_LIBRARY_DIR}/disk_layout_util.sh" || exit 1 diff --git a/build_image b/build_image index 9800b25388..86f9b0a08c 100755 --- a/build_image +++ b/build_image @@ -71,6 +71,7 @@ check_gsutil_opts # N.B. Ordering matters for some of the libraries below, because # some of the files contain initialization used by later files. +. "${BUILD_LIBRARY_DIR}/toolchain_util.sh" || exit 1 . "${BUILD_LIBRARY_DIR}/board_options.sh" || exit 1 . "${BUILD_LIBRARY_DIR}/disk_layout_util.sh" || exit 1 . "${BUILD_LIBRARY_DIR}/mount_gpt_util.sh" || exit 1 diff --git a/build_library/board_options.sh b/build_library/board_options.sh index 6281018824..5a75022048 100644 --- a/build_library/board_options.sh +++ b/build_library/board_options.sh @@ -9,24 +9,7 @@ fi BOARD="${FLAGS_board}" BOARD_ROOT="/build/${BOARD}" +ARCH=$(get_board_arch ${BOARD}) # What cross-build are we targeting? . "${BOARD_ROOT}/etc/make.conf.board_setup" - -# Figure out ARCH from the given toolchain. -# TODO(jrbarnette): There's a copy of this code in setup_board; -# it should be shared. -case "$(echo "${CHOST}" | awk -F'-' '{ print $1 }')" in - arm*) - ARCH="arm" - ;; - *86) - ARCH="x86" - ;; - *x86_64) - ARCH="amd64" - ;; - *) - error "Unable to determine ARCH from toolchain: ${CHOST}" - exit 1 -esac diff --git a/build_library/toolchain_util.sh b/build_library/toolchain_util.sh index 231f9b3ea8..2548330440 100644 --- a/build_library/toolchain_util.sh +++ b/build_library/toolchain_util.sh @@ -11,7 +11,8 @@ TOOLCHAIN_PKGS=( ) # Portage arguments to enforce the toolchain to only use binpkgs. -TOOLCHAIN_BINONLY=( "${TOOLCHAIN_PKGS[@]/#/--useoldpkg-atoms=}" ) +TOOLCHAIN_BINONLY=( "${TOOLCHAIN_PKGS[@]/#/--useoldpkg-atoms=}" + "${TOOLCHAIN_PKGS[@]/#/--rebuild-exclude=}" ) # Portage profile to use for building out the cross compiler's SYSROOT. # This is only used as an intermediate step to be able to use the cross @@ -26,6 +27,30 @@ BOARD_CHOST["amd64-generic"]="x86_64-cros-linux-gnu" BOARD_PROFILE["amd64-generic"]="coreos:coreos/amd64/generic" BOARD_NAMES=( "${!BOARD_CHOST[@]}" ) +### Generic metadata fetching functions ### + +# map CHOST to portage ARCH, list came from crossdev +# Usage: get_portage_arch chost +get_portage_arch() { + case "$1" in + aarch64*) echo arm;; + alpha*) echo alpha;; + arm*) echo arm;; + hppa*) echo hppa;; + ia64*) echo ia64;; + i?86*) echo x86;; + m68*) echo m68k;; + mips*) echo mips;; + powerpc64*) echo ppc64;; + powerpc*) echo ppc;; + sparc*) echo sparc;; + s390*) echo s390;; + sh*) echo sh;; + x86_64*) echo amd64;; + *) die "Unknown CHOST '$1'";; + esac +} + get_board_list() { local IFS=$'\n\t ' sort <<<"${BOARD_NAMES[*]}" @@ -41,6 +66,15 @@ get_profile_list() { sort -u <<<"${BOARD_PROFILE[*]}" } +# Usage: get_board_arch board [board...] +get_board_arch() { + local board + for board in "$@"; do + get_portage_arch $(get_board_chost "${board}") + done +} + +# Usage: get_board_chost board [board...] get_board_chost() { local board for board in "$@"; do @@ -52,6 +86,7 @@ get_board_chost() { done } +# Usage: get_board_profile board [board...] get_board_profile() { local board for board in "$@"; do @@ -63,6 +98,7 @@ get_board_profile() { done } +# Usage: get_cross_pkgs chost [chost2...] get_cross_pkgs() { local cross_chost native_pkg for cross_chost in "$@"; do diff --git a/build_packages b/build_packages index 4084ee3a4c..21234f69b3 100755 --- a/build_packages +++ b/build_packages @@ -126,6 +126,7 @@ fi "${SCRIPTS_DIR}"/setup_board --quiet --board=${FLAGS_board} "${UPDATE_ARGS[@]}" # set BOARD and BOARD_ROOT +. "${BUILD_LIBRARY_DIR}/toolchain_util.sh" || exit 1 . "${BUILD_LIBRARY_DIR}/board_options.sh" || exit 1 # Setup all the emerge command/flags. diff --git a/image_to_vm.sh b/image_to_vm.sh index a617ea41e0..c6727987b1 100755 --- a/image_to_vm.sh +++ b/image_to_vm.sh @@ -12,6 +12,7 @@ # Helper scripts should be run from the same location as this script. SCRIPT_ROOT=$(dirname "$(readlink -f "$0")") . "${SCRIPT_ROOT}/common.sh" || exit 1 +. "${BUILD_LIBRARY_DIR}/toolchain_util.sh" || exit 1 . "${BUILD_LIBRARY_DIR}/disk_layout_util.sh" || exit 1 . "${BUILD_LIBRARY_DIR}/build_common.sh" || exit 1 . "${BUILD_LIBRARY_DIR}/build_image_util.sh" || exit 1 From 500fe74856fe99af08c60ae25ba3564b4f01a136 Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Wed, 6 Nov 2013 18:04:26 -0800 Subject: [PATCH 09/13] fix(setup_board): Move to new toolchain_util.sh and eselect The new toolchain utils define chost, portage profiles, and portage arch per board. Replace the tricker logic from the old platform/dev repo and switch to setting the profile with the standard eselect tool. A few cleanups here and there, replacing echo with info and renames. --- setup_board | 88 ++++++++++++----------------------------------------- 1 file changed, 19 insertions(+), 69 deletions(-) diff --git a/setup_board b/setup_board index 99cf25fbfe..2badc3fff5 100755 --- a/setup_board +++ b/setup_board @@ -5,7 +5,7 @@ # found in the LICENSE file. . "$(dirname "$0")/common.sh" || exit 1 -. "${SRC_ROOT}/platform/dev/toolchain_utils.sh" +. "${BUILD_LIBRARY_DIR}/toolchain_util.sh" || exit 1 # Script must run inside the chroot restart_in_chroot_if_needed "$@" @@ -47,16 +47,12 @@ DEFINE_boolean latest_toolchain $FLAGS_FALSE \ kernel). This overrides the other toolchain version options." DEFINE_string libc_version "[stable]" \ "Version of libc to use." -DEFINE_string profile "" \ - "The portage configuration profile to use. Profile must be located in overlay-board/profiles" DEFINE_boolean quiet $FLAGS_FALSE \ "Don't print warnings when board already exists." DEFINE_boolean skip_toolchain_update $FLAGS_FALSE \ "Don't update toolchain automatically." DEFINE_boolean skip_chroot_upgrade $FLAGS_FALSE \ "Don't run the chroot upgrade automatically; use with care." -DEFINE_string toolchain "" \ - "Toolchain. For example: i686-pc-linux-gnu, armv7a-softfloat-linux-gnueabi" DEFINE_string variant "" \ "Board variant." DEFINE_boolean regen_configs ${FLAGS_FALSE} \ @@ -77,10 +73,8 @@ _generate_wrapper() { sudo_clobber "${target}" < Date: Wed, 6 Nov 2013 18:18:46 -0800 Subject: [PATCH 10/13] fix(build_packages): Don't redirect emerge output. The stats upload has been removed so there is no longer a need to capture the emerge output to parse the logged output. Remove a bit of dead chromeos logic too. --- build_packages | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/build_packages b/build_packages index 21234f69b3..785b5306fd 100755 --- a/build_packages +++ b/build_packages @@ -199,11 +199,6 @@ if [ "${FLAGS_noworkon}" -eq "${FLAGS_FALSE}" ]; then CROS_WORKON_PKGS+=( $("${LIST_MODIFIED_PACKAGES}" --board=${FLAGS_board}) ) fi -# TODO(anush): Make chrome a fake cros-workon package. -if [[ -n "${CHROME_ORIGIN}" ]]; then - CROS_WORKON_PKGS+=( chromeos-base/chromeos-chrome ) -fi - if [[ ${#CROS_WORKON_PKGS[@]} -gt 0 ]]; then EMERGE_FLAGS+=( --reinstall-atoms="${CROS_WORKON_PKGS[*]}" @@ -211,25 +206,12 @@ if [[ ${#CROS_WORKON_PKGS[@]} -gt 0 ]]; then ) fi -# Prepare tmp file to capture emerge output from tee. -tmpfile=$(mktemp -t tmp.build_packages-emerge.XXXXXX) -trap "rm -f '${tmpfile}'" EXIT - info "Merging board packages now" -( - set -o pipefail - sudo -E "${EMERGE_CMD[@]}" "${EMERGE_FLAGS[@]}" "${PACKAGES[@]}" | \ - tee "${tmpfile}" -) +sudo -E "${EMERGE_CMD[@]}" "${EMERGE_FLAGS[@]}" "${PACKAGES[@]}" # upload packages if enabled upload_packages -# Extract total package count from emerge output. -package_count=$(awk '$0 ~ /^Total: [0-9]+ packages/ { print $2 }' "${tmpfile}") -rm "${tmpfile}" -trap - EXIT - echo "Builds complete" command_completed echo "Done" From 84fadc8236e5369fb47fdc194acb97175b3d61bb Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Wed, 6 Nov 2013 18:28:02 -0800 Subject: [PATCH 11/13] fix(check_deps): Fix support for includes in ld.so.conf The path passed to _ReadLdSoConf must start with a '/' which wasn't happening since relpath returns a relative path. --- build_library/check_deps | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build_library/check_deps b/build_library/check_deps index 9907b9093e..47257855f5 100755 --- a/build_library/check_deps +++ b/build_library/check_deps @@ -47,7 +47,8 @@ class CheckDependencies(object): # Includes are absolute or relative to the file itself. line = os.path.join(os.path.dirname(path), line[8:]) for p in glob.glob(self._root + line): - libdirs.extend(self._ReadLdSoConf(os.path.relpath(p, self._root))) + rel_p = "/%s" % os.path.relpath(p, self._root) + libdirs.extend(self._ReadLdSoConf(rel_p)) f.close() From b5e608c5512e7e48ce604392ee142ac59a4b10f1 Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Thu, 7 Nov 2013 16:07:17 -0800 Subject: [PATCH 12/13] fix(toolchain_util.sh): Switch from sudo_clobber to sudo tee sudo_clobber is provided by common.sh which isn't included in the catalyst environment so call sudo tee (what sudo_clobber does) directly instead. --- build_library/toolchain_util.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_library/toolchain_util.sh b/build_library/toolchain_util.sh index 2548330440..b70ee9cf77 100644 --- a/build_library/toolchain_util.sh +++ b/build_library/toolchain_util.sh @@ -168,7 +168,7 @@ install_cross_toolchain() { then $sudo crossdev --stable --portage "$*" \ --init-target --target "${cross_chost}" - sudo_clobber "${cross_cfg}" <<<"${cross_cfg_data}" + $sudo tee "${cross_cfg}" <<<"${cross_cfg_data}" >/dev/null fi # If binary packages are enabled try to just emerge them instead of From 1d8d99241a425eaaeecd22bcd15b97811f949db0 Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Thu, 7 Nov 2013 16:20:13 -0800 Subject: [PATCH 13/13] fix(build_toolchains): Upload packages to a new toolchain directory. Currently we don't have a good way to upload packages from different jobs to the same location. The 'Packages' index file is only generated locally so the second upload would always win. --- build_toolchains | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/build_toolchains b/build_toolchains index c22ed0977f..0b283b1b06 100755 --- a/build_toolchains +++ b/build_toolchains @@ -43,12 +43,14 @@ cp "${BUILD_LIBRARY_DIR}/toolchain_util.sh" "${ROOT_OVERLAY}/tmp" catalyst_build def_upload_path="${UPLOAD_ROOT}/sdk/${ARCH}/${FLAGS_version}" -upload_files "packages" "${def_upload_path}" "pkgs/" "${BINPKGS}/crossdev"/* +upload_files "cross toolchain packages" "${def_upload_path}" \ + "toolchain/" "${BINPKGS}/crossdev"/* for board in $(get_board_list); do board_packages="${BINPKGS}/target/${board}" def_upload_path="${UPLOAD_ROOT}/${board}/${FLAGS_version}" - upload_files packages "${def_upload_path}" "pkgs/" "${board_packages}"/* + upload_files "board toolchain packages" "${def_upload_path}" \ + "toolchain/" "${board_packages}"/* done command_completed