diff --git a/bin/cros_make_image_bootable b/bin/cros_make_image_bootable index 6fb60d1ac0..6d33cf1cfd 100755 --- a/bin/cros_make_image_bootable +++ b/bin/cros_make_image_bootable @@ -33,6 +33,7 @@ assert_inside_chroot # Load functions and constants for chromeos-install . /usr/lib/installer/chromeos-common.sh || exit 1 +. "${BUILD_LIBRARY_DIR}/toolchain_util.sh" || exit 1 . "${BUILD_LIBRARY_DIR}/build_image_util.sh" || exit 1 . "${BUILD_LIBRARY_DIR}/disk_layout_util.sh" || exit 1 diff --git a/bootstrap_sdk b/bootstrap_sdk index 59a1634109..747ddeb1c3 100755 --- a/bootstrap_sdk +++ b/bootstrap_sdk @@ -48,7 +48,7 @@ cat </dev/null - then - run_merge -u "${cross_pkgs[@]}" - else - crossdev --ov-output "/usr/local/portage/crossdev" \ - --portage "${clst_myemergeopts}" \ - --env 'FEATURES=splitdebug' \ - --stable --ex-gdb --stage4 \ - --target "${cross_chost}" || exit 1 - fi - - # There is no point to including the built packages in the final tarball - # because the packages will have to be downloaded anyway due to how the - # cross toolchains are managed in board sysroots. - crossdev --force -C "${cross_chost}" -done - -echo "Double checking everything is fresh and happy." -run_merge -uDN --with-bdeps=y world diff --git a/build_library/catalyst_toolchains.sh b/build_library/catalyst_toolchains.sh index 066b599cd1..c09ebbfd9a 100644 --- a/build_library/catalyst_toolchains.sh +++ b/build_library/catalyst_toolchains.sh @@ -11,83 +11,18 @@ source /tmp/toolchain_util.sh # build deps in crossdev's sysroot use ${PKGDIR}/cross/${CHOST} (no upload) # native toolchains use ${PKGDIR}/target/${BOARD} (uploaded to board location) -get_dependency_list() { - local ROOT="$1" - local IFS=$'| \t\n' - shift - - PORTAGE_CONFIGROOT="$ROOT" SYSROOT="$ROOT" \ - ROOT="$ROOT" emerge ${clst_myemergeopts} \ - --pretend --with-bdeps=y --onlydeps --quiet \ - "$@" | sed -e 's/.*\] \([^ :]*\).*/=\1/' | - egrep -v "(=$(echo "$*")-[0-9])" -} - -configure_portage() { - local pkg_path="$1" - local profile="$2" - - mkdir -p "${ROOT}/etc/portage" - echo "eselect will report '!!! Warning: Strange path.' but that's OK" - eselect profile set --force "$profile" - - cat >"${ROOT}/etc/portage/make.conf" </dev/null - then - PKGDIR="${PKGDIR}" run_merge -u "${cross_pkgs[@]}" - else - PKGDIR="${PKGDIR}" crossdev \ - --ov-output "/tmp/crossdev" --stable \ - --env PKGDIR="${PORTDIR}/packages/crossdev" \ - --portage "${clst_myemergeopts}" \ - --stage4 --target "${cross_chost}" - fi - - # Setup ccache for our shiny new toolchain - ccache-config --install-links "${cross_chost}" -} - -configure_cross_root() { +build_cross_libs() { local cross_chost="$1" local ROOT="/usr/${cross_chost}" CHOST="${cross_chost}" ROOT="$ROOT" SYSROOT="$ROOT" \ - configure_portage "cross/${cross_chost}" \ - "${CROSS_PROFILE[${cross_chost}]}" + _configure_sysroot "${CROSS_PROFILE[${cross_chost}]}" # In order to get a dependency list we must calculate it before # updating package.provided. Otherwise portage will no-op. - get_dependency_list "$ROOT" "${TOOLCHAIN_PKGS[@]}" \ - > "$ROOT/etc/portage/cross-${cross_chost}-depends" + ROOT="$ROOT" _get_dependency_list \ + ${clst_myemergeopts} "${TOOLCHAIN_PKGS[@]}" > \ + "$ROOT/etc/portage/cross-${cross_chost}-depends" # Add toolchain to packages.provided since they are on the host system mkdir -p "$ROOT/etc/portage/profile/package.provided" @@ -97,11 +32,6 @@ configure_cross_root() { cross_pkg_version=$(portageq match / "${cross_pkg}") echo "${native_pkg%/*}/${cross_pkg_version#*/}" done > "$ROOT/etc/portage/profile/package.provided/cross-${cross_chost}" -} - -build_cross_libs() { - local cross_chost="$1" - local ROOT="/usr/${cross_chost}" local cross_deps=$(<"$ROOT/etc/portage/cross-${cross_chost}-depends") # --root is required because run_merge overrides ROOT= @@ -111,10 +41,12 @@ build_cross_libs() { configure_target_root() { local board="$1" local cross_chost=$(get_board_chost "$1") + local profile=$(get_board_profile "${board}") - CHOST="${cross_chost}" ROOT="/build/${board}" \ - SYSROOT="/usr/${cross_chost}" configure_portage \ - "target/${board}" "$(get_board_profile "${board}")" + CHOST="${cross_chost}" \ + ROOT="/build/${board}" \ + SYSROOT="/usr/${cross_chost}" \ + _configure_sysroot "${profile}" } build_target_toolchain() { @@ -126,15 +58,21 @@ build_target_toolchain() { run_merge -u --root="$ROOT" "${TOOLCHAIN_PKGS[@]}" } +mkdir -p "/tmp/crossdev" +export PORTDIR_OVERLAY="/tmp/crossdev $(portageq envvar PORTDIR_OVERLAY)" + for cross_chost in $(get_chost_list); do echo "Building cross toolchain for ${cross_chost}" - build_cross_toolchain "${cross_chost}" - configure_cross_root "${cross_chost}" - build_cross_libs "${cross_chost}" + PKGDIR="$(portageq envvar PKGDIR)/crossdev" \ + install_cross_toolchain "${cross_chost}" ${clst_myemergeopts} + + sysroot_pkgdir="$(portageq envvar PKGDIR)/cross/${cross_chost}" + PKGDIR="${sysroot_pkgdir}" build_cross_libs "${cross_chost}" done for board in $(get_board_list); do echo "Building native toolchain for ${board}" - configure_target_root "${board}" - build_target_toolchain "${board}" + target_pkgdir="$(portageq envvar PKGDIR)/target/${board}" + PKGDIR="${target_pkgdir}" configure_target_root "${board}" + PKGDIR="${target_pkgdir}" build_target_toolchain "${board}" done diff --git a/build_library/check_deps b/build_library/check_deps index 9907b9093e..47257855f5 100755 --- a/build_library/check_deps +++ b/build_library/check_deps @@ -47,7 +47,8 @@ class CheckDependencies(object): # Includes are absolute or relative to the file itself. line = os.path.join(os.path.dirname(path), line[8:]) for p in glob.glob(self._root + line): - libdirs.extend(self._ReadLdSoConf(os.path.relpath(p, self._root))) + rel_p = "/%s" % os.path.relpath(p, self._root) + libdirs.extend(self._ReadLdSoConf(rel_p)) f.close() diff --git a/build_library/disk_layout_util.sh b/build_library/disk_layout_util.sh index 56d8cb2a4c..e337111e70 100644 --- a/build_library/disk_layout_util.sh +++ b/build_library/disk_layout_util.sh @@ -112,6 +112,10 @@ check_valid_layout() { get_disk_layout_type() { DISK_LAYOUT_TYPE="base" + if [[ -n "${FLAGS_disk_layout}" && \ + "${FLAGS_disk_layout}" != "default" ]]; then + DISK_LAYOUT_TYPE="${FLAGS_disk_layout}" + fi } emit_gpt_scripts() { diff --git a/build_library/toolchain_util.sh b/build_library/toolchain_util.sh index 716207a6fa..b70ee9cf77 100644 --- a/build_library/toolchain_util.sh +++ b/build_library/toolchain_util.sh @@ -11,7 +11,8 @@ TOOLCHAIN_PKGS=( ) # Portage arguments to enforce the toolchain to only use binpkgs. -TOOLCHAIN_BINONLY=( "${TOOLCHAIN_PKGS[@]/#/--useoldpkg-atoms=}" ) +TOOLCHAIN_BINONLY=( "${TOOLCHAIN_PKGS[@]/#/--useoldpkg-atoms=}" + "${TOOLCHAIN_PKGS[@]/#/--rebuild-exclude=}" ) # Portage profile to use for building out the cross compiler's SYSROOT. # This is only used as an intermediate step to be able to use the cross @@ -26,6 +27,30 @@ BOARD_CHOST["amd64-generic"]="x86_64-cros-linux-gnu" BOARD_PROFILE["amd64-generic"]="coreos:coreos/amd64/generic" BOARD_NAMES=( "${!BOARD_CHOST[@]}" ) +### Generic metadata fetching functions ### + +# map CHOST to portage ARCH, list came from crossdev +# Usage: get_portage_arch chost +get_portage_arch() { + case "$1" in + aarch64*) echo arm;; + alpha*) echo alpha;; + arm*) echo arm;; + hppa*) echo hppa;; + ia64*) echo ia64;; + i?86*) echo x86;; + m68*) echo m68k;; + mips*) echo mips;; + powerpc64*) echo ppc64;; + powerpc*) echo ppc;; + sparc*) echo sparc;; + s390*) echo s390;; + sh*) echo sh;; + x86_64*) echo amd64;; + *) die "Unknown CHOST '$1'";; + esac +} + get_board_list() { local IFS=$'\n\t ' sort <<<"${BOARD_NAMES[*]}" @@ -41,18 +66,128 @@ get_profile_list() { sort -u <<<"${BOARD_PROFILE[*]}" } -get_board_chost() { - if [[ ${#BOARD_CHOST["$1"]} -ne 0 ]]; then - echo "${BOARD_CHOST["$1"]}" - else - die "Unknown board '$1'" - fi +# Usage: get_board_arch board [board...] +get_board_arch() { + local board + for board in "$@"; do + get_portage_arch $(get_board_chost "${board}") + done } +# Usage: get_board_chost board [board...] +get_board_chost() { + local board + for board in "$@"; do + if [[ ${#BOARD_CHOST["$board"]} -ne 0 ]]; then + echo "${BOARD_CHOST["$board"]}" + else + die "Unknown board '$board'" + fi + done +} + +# Usage: get_board_profile board [board...] get_board_profile() { - if [[ ${#BOARD_PROFILE["$1"]} -ne 0 ]]; then - echo "${BOARD_PROFILE["$1"]}" + local board + for board in "$@"; do + if [[ ${#BOARD_PROFILE["$board"]} -ne 0 ]]; then + echo "${BOARD_PROFILE["$board"]}" + else + die "Unknown board '$board'" + fi + done +} + +# Usage: get_cross_pkgs chost [chost2...] +get_cross_pkgs() { + local cross_chost native_pkg + for cross_chost in "$@"; do + for native_pkg in "${TOOLCHAIN_PKGS[@]}"; do + echo "${native_pkg/*\//cross-${cross_chost}/}" + done + done +} + +### Toolchain building utilities ### + +# Ugly hack to get a dependency list of a set of packages. +# This is required to figure out what to install in the crossdev sysroot. +# Usage: ROOT=/foo/bar _get_dependency_list pkgs... [--portage-opts...] +_get_dependency_list() { + local pkgs=( ${*/#-*/} ) + local IFS=$'| \t\n' + + PORTAGE_CONFIGROOT="$ROOT" emerge "$@" --pretend \ + --emptytree --root-deps=rdeps --onlydeps --quiet | \ + sed -e 's/.*\] \([^ :]*\).*/=\1/' | + egrep -v "(=$(echo "${pkgs[*]}")-[0-9])" +} + +# Configure a new ROOT +# Values are copied from the environment or the current host configuration. +# Usage: ROOT=/foo/bar SYSROOT=/foo/bar configure_portage coreos:some/profile +_configure_sysroot() { + local profile="$1" + + mkdir -p "${ROOT}/etc/portage" + echo "eselect will report '!!! Warning: Strange path.' but that's OK" + eselect profile set --force "$profile" + + cat >"${ROOT}/etc/portage/make.conf" </dev/null + fi + + # If binary packages are enabled try to just emerge them instead of + # doing a full bootstrap which speeds things up greatly. :) + if [[ "$*" == *--usepkg* ]] && \ + emerge "$@" --usepkgonly --binpkg-respect-use=y \ + --pretend "${cross_pkgs[@]}" &>/dev/null + then + $sudo emerge "$@" --binpkg-respect-use=y -u "${cross_pkgs[@]}" else - die "Unknown board '$1'" + $sudo crossdev --stable --portage "$*" \ + --stage4 --target "${cross_chost}" + fi + + # Setup wrappers for our shiny new toolchain + if [[ ! -e "/usr/lib/ccache/bin/${cross_chost}-gcc" ]]; then + $sudo ccache-config --install-links "${cross_chost}" + fi + if [[ ! -e "/usr/lib/sysroot-wrappers/bin/${cross_chost}-gcc" ]]; then + $sudo sysroot-config --install-links "${cross_chost}" fi } diff --git a/build_packages b/build_packages index 4084ee3a4c..785b5306fd 100755 --- a/build_packages +++ b/build_packages @@ -126,6 +126,7 @@ fi "${SCRIPTS_DIR}"/setup_board --quiet --board=${FLAGS_board} "${UPDATE_ARGS[@]}" # set BOARD and BOARD_ROOT +. "${BUILD_LIBRARY_DIR}/toolchain_util.sh" || exit 1 . "${BUILD_LIBRARY_DIR}/board_options.sh" || exit 1 # Setup all the emerge command/flags. @@ -198,11 +199,6 @@ if [ "${FLAGS_noworkon}" -eq "${FLAGS_FALSE}" ]; then CROS_WORKON_PKGS+=( $("${LIST_MODIFIED_PACKAGES}" --board=${FLAGS_board}) ) fi -# TODO(anush): Make chrome a fake cros-workon package. -if [[ -n "${CHROME_ORIGIN}" ]]; then - CROS_WORKON_PKGS+=( chromeos-base/chromeos-chrome ) -fi - if [[ ${#CROS_WORKON_PKGS[@]} -gt 0 ]]; then EMERGE_FLAGS+=( --reinstall-atoms="${CROS_WORKON_PKGS[*]}" @@ -210,25 +206,12 @@ if [[ ${#CROS_WORKON_PKGS[@]} -gt 0 ]]; then ) fi -# Prepare tmp file to capture emerge output from tee. -tmpfile=$(mktemp -t tmp.build_packages-emerge.XXXXXX) -trap "rm -f '${tmpfile}'" EXIT - info "Merging board packages now" -( - set -o pipefail - sudo -E "${EMERGE_CMD[@]}" "${EMERGE_FLAGS[@]}" "${PACKAGES[@]}" | \ - tee "${tmpfile}" -) +sudo -E "${EMERGE_CMD[@]}" "${EMERGE_FLAGS[@]}" "${PACKAGES[@]}" # upload packages if enabled upload_packages -# Extract total package count from emerge output. -package_count=$(awk '$0 ~ /^Total: [0-9]+ packages/ { print $2 }' "${tmpfile}") -rm "${tmpfile}" -trap - EXIT - echo "Builds complete" command_completed echo "Done" diff --git a/build_toolchains b/build_toolchains index c22ed0977f..0b283b1b06 100755 --- a/build_toolchains +++ b/build_toolchains @@ -43,12 +43,14 @@ cp "${BUILD_LIBRARY_DIR}/toolchain_util.sh" "${ROOT_OVERLAY}/tmp" catalyst_build def_upload_path="${UPLOAD_ROOT}/sdk/${ARCH}/${FLAGS_version}" -upload_files "packages" "${def_upload_path}" "pkgs/" "${BINPKGS}/crossdev"/* +upload_files "cross toolchain packages" "${def_upload_path}" \ + "toolchain/" "${BINPKGS}/crossdev"/* for board in $(get_board_list); do board_packages="${BINPKGS}/target/${board}" def_upload_path="${UPLOAD_ROOT}/${board}/${FLAGS_version}" - upload_files packages "${def_upload_path}" "pkgs/" "${board_packages}"/* + upload_files "board toolchain packages" "${def_upload_path}" \ + "toolchain/" "${board_packages}"/* done command_completed diff --git a/common.sh b/common.sh index da89510005..b1cba1c95f 100644 --- a/common.sh +++ b/common.sh @@ -76,10 +76,10 @@ _dump_trace() { # Declare these asap so that code below can safely assume they exist. _message() { - local prefix=$1 + local prefix="$1${CROS_LOG_PREFIX:-${SCRIPT_NAME}}" shift if [[ $# -eq 0 ]]; then - echo -e "${prefix}${CROS_LOG_PREFIX:-""}:${V_VIDOFF}" >&2 + echo -e "${prefix}:${V_VIDOFF}" >&2 return fi ( @@ -95,7 +95,7 @@ _message() { set -- '' fi for line in "$@"; do - echo -e "${prefix}${CROS_LOG_PREFIX:-}: ${line}${V_VIDOFF}" >&2 + echo -e "${prefix}: ${line}${V_VIDOFF}" >&2 done ) } diff --git a/image_to_vm.sh b/image_to_vm.sh index a617ea41e0..c6727987b1 100755 --- a/image_to_vm.sh +++ b/image_to_vm.sh @@ -12,6 +12,7 @@ # Helper scripts should be run from the same location as this script. SCRIPT_ROOT=$(dirname "$(readlink -f "$0")") . "${SCRIPT_ROOT}/common.sh" || exit 1 +. "${BUILD_LIBRARY_DIR}/toolchain_util.sh" || exit 1 . "${BUILD_LIBRARY_DIR}/disk_layout_util.sh" || exit 1 . "${BUILD_LIBRARY_DIR}/build_common.sh" || exit 1 . "${BUILD_LIBRARY_DIR}/build_image_util.sh" || exit 1 diff --git a/setup_board b/setup_board index 99cf25fbfe..2badc3fff5 100755 --- a/setup_board +++ b/setup_board @@ -5,7 +5,7 @@ # found in the LICENSE file. . "$(dirname "$0")/common.sh" || exit 1 -. "${SRC_ROOT}/platform/dev/toolchain_utils.sh" +. "${BUILD_LIBRARY_DIR}/toolchain_util.sh" || exit 1 # Script must run inside the chroot restart_in_chroot_if_needed "$@" @@ -47,16 +47,12 @@ DEFINE_boolean latest_toolchain $FLAGS_FALSE \ kernel). This overrides the other toolchain version options." DEFINE_string libc_version "[stable]" \ "Version of libc to use." -DEFINE_string profile "" \ - "The portage configuration profile to use. Profile must be located in overlay-board/profiles" DEFINE_boolean quiet $FLAGS_FALSE \ "Don't print warnings when board already exists." DEFINE_boolean skip_toolchain_update $FLAGS_FALSE \ "Don't update toolchain automatically." DEFINE_boolean skip_chroot_upgrade $FLAGS_FALSE \ "Don't run the chroot upgrade automatically; use with care." -DEFINE_string toolchain "" \ - "Toolchain. For example: i686-pc-linux-gnu, armv7a-softfloat-linux-gnueabi" DEFINE_string variant "" \ "Board variant." DEFINE_boolean regen_configs ${FLAGS_FALSE} \ @@ -77,10 +73,8 @@ _generate_wrapper() { sudo_clobber "${target}" <