From 3d8948d91f60bdd338b538ade896f0511b3d1f52 Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Sat, 19 Jul 2014 16:10:34 -0700 Subject: [PATCH 1/5] setup_board: copy portage's repos.conf files from the chroot --- setup_board | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup_board b/setup_board index 98805277d1..6ce046518b 100755 --- a/setup_board +++ b/setup_board @@ -174,7 +174,6 @@ esac COREOS_CONFIG="/usr/local/portage/coreos/coreos/config" BOARD_ROOT="/build/${BOARD_VARIANT}" BOARD_ETC="${BOARD_ROOT}/etc" -BOARD_PROFILE="${BOARD_ETC}/portage/profile" BOARD_ARCH=$(get_board_arch "$BOARD") BOARD_CHOST=$(get_board_chost ${BOARD}) PORTAGE_PROFILE=$(get_board_profile "$BOARD") @@ -218,7 +217,7 @@ if [[ ! -L "${BOARD_ROOT}/packages" ]]; then fi info "Configuring portage in ${BOARD_ROOT}" -sudo mkdir -p "${BOARD_PROFILE}" +sudo mkdir -p "${BOARD_ETC}/portage/"{profile,repos.conf} sudo ROOT="${BOARD_ROOT}" eselect profile set --force "${PORTAGE_PROFILE}" # Cleanup/migrate from older make.conf files @@ -258,6 +257,8 @@ EMERGE_DEFAULT_OPTS="--oneshot" source "${BOARD_ETC}/portage/make.conf.user" EOF +sudo cp /etc/portage/repos.conf/* "${BOARD_ETC}"/portage/repos.conf/ + generate_all_wrappers if [[ ${FLAGS_regen_configs} -eq ${FLAGS_FALSE} ]]; then From 4d3c198161f9617d7de0b9e9845b43a77547bd59 Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Sat, 19 Jul 2014 16:32:26 -0700 Subject: [PATCH 2/5] tools: remove support for parallel_emerge Using parallel_emerge has been disabled by default for all commands except build_image for quite a while now, build_image kept it just because it was still a bit faster than normal emerge. Keeping parallel_emerge complicates future changes to build_image so it needs to drop it entirely. Since that means nothing uses it by default we might as well just rip out support for it entirely. --- build_image | 2 -- build_library/build_image_util.sh | 6 +----- build_packages | 28 +--------------------------- common.sh | 3 --- sdk_lib/make_chroot.sh | 7 ------- setup_board | 7 ------- update_chroot | 7 +------ 7 files changed, 3 insertions(+), 57 deletions(-) diff --git a/build_image b/build_image index a67bc3163d..e5891f9ed2 100755 --- a/build_image +++ b/build_image @@ -62,8 +62,6 @@ show_help_if_requested "$@" # not needed for the typical developer workflow. DEFINE_integer build_attempt 1 \ "The build attempt for this image build." -DEFINE_boolean fast ${FLAGS_TRUE} \ - "Use the parallel_emerge wrapper script." DEFINE_integer jobs "${NUM_JOBS}" \ "How many packages to build in parallel at maximum." DEFINE_boolean replace ${FLAGS_FALSE} \ diff --git a/build_library/build_image_util.sh b/build_library/build_image_util.sh index 6f9fe65474..3e6b418e05 100755 --- a/build_library/build_image_util.sh +++ b/build_library/build_image_util.sh @@ -102,11 +102,7 @@ emerge_to_image() { test -n "$mask" || die "PROD_INSTALL_MASK not defined" local emerge_cmd - if [[ "${FLAGS_fast}" -eq "${FLAGS_TRUE}" ]]; then - emerge_cmd="$GCLIENT_ROOT/chromite/bin/parallel_emerge --board=$BOARD" - else - emerge_cmd="emerge-$BOARD" - fi + emerge_cmd="emerge-$BOARD" emerge_cmd+=" --root-deps=rdeps --usepkgonly -v" if [[ $FLAGS_jobs -ne -1 ]]; then diff --git a/build_packages b/build_packages index 59969e5f84..728c500c64 100755 --- a/build_packages +++ b/build_packages @@ -20,10 +20,6 @@ DEFINE_boolean getbinpkg "${FLAGS_TRUE}" \ "Download binary packages from remote repository." DEFINE_boolean noworkon "${FLAGS_FALSE}" \ "Don't force-build workon packages." -DEFINE_boolean showoutput "${FLAGS_FALSE}" \ - "Show all output from parallel_emerge." -DEFINE_boolean withautotest "${FLAGS_TRUE}" \ - "Build autotest client code." DEFINE_boolean fetchonly "${FLAGS_FALSE}" \ "Don't build anything, instead only fetch what is needed." @@ -39,8 +35,6 @@ up by the build_image script to put together a bootable Chrome OS image. If [packages] are specified, only build those specific packages (and any dependencies they might need). - -For the fastest builds, use --nowithautotest --noworkon. " show_help_if_requested "$@" @@ -53,8 +47,6 @@ DEFINE_boolean toolchainpkgonly $FLAGS_FALSE \ "Use binary packages only for the board toolchain." DEFINE_string accept_licenses "" \ "Licenses to append to the accept list." -DEFINE_boolean fast "${DEFAULT_FAST}" \ - "Use the parallel_emerge wrapper script." DEFINE_integer jobs "${NUM_JOBS}" \ "How many packages to build in parallel at maximum." DEFINE_boolean norebuild "${FLAGS_FALSE}" \ @@ -93,11 +85,6 @@ UPDATE_ARGS=( --regen_configs ) if [[ -n ${FLAGS_accept_licenses} ]]; then UPDATE_ARGS+=( --accept_licenses "${FLAGS_accept_licenses}" ) fi -if [ "${FLAGS_fast}" -eq "${FLAGS_TRUE}" ]; then - UPDATE_ARGS+=( --fast ) -else - UPDATE_ARGS+=( --nofast ) -fi if [ "${FLAGS_usepkg}" -eq "${FLAGS_TRUE}" ]; then UPDATE_ARGS+=( --usepkg ) if [[ "${FLAGS_getbinpkg}" -eq "${FLAGS_TRUE}" ]]; then @@ -138,15 +125,7 @@ fi # Setup all the emerge command/flags. EMERGE_FLAGS=( -uDNv --backtrack=30 --select ) - -if [[ "${FLAGS_fast}" -eq "${FLAGS_TRUE}" ]]; then - EMERGE_CMD=( - "${CHROMITE_BIN}/parallel_emerge" - --board=${FLAGS_board} - ) -else - EMERGE_CMD=( "emerge-${FLAGS_board}" ) -fi +EMERGE_CMD=( "emerge-${FLAGS_board}" ) if [[ "${FLAGS_fetchonly}" -eq "${FLAGS_TRUE}" ]]; then EMERGE_CMD+=( --fetchonly ) fi @@ -174,11 +153,6 @@ fi if [[ "${FLAGS_norebuild}" -eq "${FLAGS_FALSE}" ]]; then EMERGE_FLAGS+=( --rebuild-if-unbuilt ) fi -if [[ "${FLAGS_showoutput}" -eq "${FLAGS_TRUE}" && \ - "${FLAGS_fast}" -eq "${FLAGS_TRUE}" ]]; then - # Only parallel_emerge supports --show-output. - EMERGE_FLAGS+=( --show-output ) -fi # Build cros_workon packages when they are changed. CROS_WORKON_PKGS=() diff --git a/common.sh b/common.sh index b2f055276d..84b5bbee95 100644 --- a/common.sh +++ b/common.sh @@ -390,9 +390,6 @@ if [[ -f ${GCLIENT_ROOT}/src/scripts/.default_board ]]; then fi fi -# Disable --fast in most commands -DEFAULT_FAST=${FLAGS_FALSE} - # Directory to store built images. Should be set by sourcing script when used. BUILD_DIR= diff --git a/sdk_lib/make_chroot.sh b/sdk_lib/make_chroot.sh index 7cfb4d590f..9d7ce0f39a 100755 --- a/sdk_lib/make_chroot.sh +++ b/sdk_lib/make_chroot.sh @@ -40,8 +40,6 @@ DEFINE_boolean getbinpkg $FLAGS_TRUE \ "Download binary packages from remote repository." DEFINE_boolean delete $FLAGS_FALSE "Delete an existing chroot." DEFINE_boolean replace $FLAGS_FALSE "Overwrite existing chroot, if any." -DEFINE_boolean fast "${DEFAULT_FAST}" \ - "Use the parallel_emerge wrapper script." DEFINE_integer jobs "${NUM_JOBS}" \ "How many packages to build in parallel at maximum." DEFINE_string stage3_path "" \ @@ -358,11 +356,6 @@ if [[ ${FLAGS_usepkg} -eq ${FLAGS_TRUE} ]]; then else UPDATE_ARGS+=( --nousepkg ) fi -if [[ ${FLAGS_fast} -eq ${FLAGS_TRUE} ]]; then - UPDATE_ARGS+=( --fast ) -else - UPDATE_ARGS+=( --nofast ) -fi if [[ "${FLAGS_jobs}" -ne -1 ]]; then UPDATE_ARGS+=( --jobs=${FLAGS_jobs} ) fi diff --git a/setup_board b/setup_board index 6ce046518b..a88cff2489 100755 --- a/setup_board +++ b/setup_board @@ -37,8 +37,6 @@ DEFINE_string getbinpkgver "" \ "Use binary packages from a specific version." DEFINE_boolean toolchainpkgonly $FLAGS_FALSE \ "Use binary packages only for the board toolchain." -DEFINE_boolean fast "${DEFAULT_FAST}" \ - "Use the parallel_emerge wrapper script." DEFINE_integer jobs "${NUM_JOBS}" \ "How many packages to build in parallel at maximum." DEFINE_boolean latest_toolchain $FLAGS_FALSE \ @@ -140,11 +138,6 @@ get_board_and_variant $FLAGS_board $FLAGS_variant # Before we can run any tools, we need to update chroot UPDATE_ARGS="--toolchain_boards=${BOARD}" -if [ "${FLAGS_fast}" -eq "${FLAGS_TRUE}" ]; then - UPDATE_ARGS+=" --fast" -else - UPDATE_ARGS+=" --nofast" -fi if [ "${FLAGS_usepkg}" -eq "${FLAGS_TRUE}" ]; then UPDATE_ARGS+=" --usepkg" if [[ "${FLAGS_getbinpkg}" -eq "${FLAGS_TRUE}" ]]; then diff --git a/update_chroot b/update_chroot index add4b3dbd8..7a2b419150 100755 --- a/update_chroot +++ b/update_chroot @@ -28,8 +28,6 @@ show_help_if_requested "$@" # The following options are advanced options, only available to those willing # to read the source code. They are not shown in help output, since they are # not needed for the typical developer workflow. -DEFINE_boolean fast ${DEFAULT_FAST} \ - "Use the parallel_emerge wrapper script." DEFINE_integer jobs "${NUM_JOBS}" \ "How many packages to build in parallel at maximum." DEFINE_boolean skip_toolchain_update $FLAGS_FALSE \ @@ -148,10 +146,6 @@ fi # Perform an update of coreos-devel/sdk-depends and world in the chroot. EMERGE_CMD="emerge" -CHROMITE_BIN="${GCLIENT_ROOT}/chromite/bin" -if [ "${FLAGS_fast}" -eq "${FLAGS_TRUE}" ]; then - EMERGE_CMD="${CHROMITE_BIN}/parallel_emerge" -fi # In first pass, update portage and toolchains. Lagged updates of both # can cause serious issues later. @@ -175,6 +169,7 @@ if [[ "${FLAGS_skip_toolchain_update}" -eq "${FLAGS_FALSE}" && \ fi # Build cros_workon packages when they are changed. +CHROMITE_BIN="${GCLIENT_ROOT}/chromite/bin" for pkg in $("${CHROMITE_BIN}/cros_list_modified_packages" --host); do EMERGE_FLAGS+=" --reinstall-atoms=${pkg} --usepkg-exclude=${pkg}" done From a28a8966e49e0c7975d4cbb58ffc3c606f3bfec9 Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Sat, 19 Jul 2014 17:17:28 -0700 Subject: [PATCH 3/5] build_image: use dev and prod profile variants Instead of gluing in a special PROD_INSTALL_MASK for all images use profiles to configure the differences between the base build root, production images, and developer images. This offers much more flexibility and is needed for providing a full dev environment in developer images. --- build_library/build_image_util.sh | 33 +++++++++++++++++++------------ build_library/dev_image_util.sh | 1 + build_library/prod_image_util.sh | 5 +++-- 3 files changed, 24 insertions(+), 15 deletions(-) diff --git a/build_library/build_image_util.sh b/build_library/build_image_util.sh index 3e6b418e05..6d4f09376f 100755 --- a/build_library/build_image_util.sh +++ b/build_library/build_image_util.sh @@ -98,23 +98,26 @@ generate_update() { # to the basic emerge command. emerge_to_image() { local root_fs_dir="$1"; shift - local mask="${INSTALL_MASK:-$(portageq-$BOARD envvar PROD_INSTALL_MASK)}" - test -n "$mask" || die "PROD_INSTALL_MASK not defined" - local emerge_cmd - emerge_cmd="emerge-$BOARD" - emerge_cmd+=" --root-deps=rdeps --usepkgonly -v" - - if [[ $FLAGS_jobs -ne -1 ]]; then - emerge_cmd+=" --jobs=$FLAGS_jobs" - fi - - sudo -E INSTALL_MASK="$mask" ${emerge_cmd} --root="${root_fs_dir}" "$@" + sudo -E ROOT="${root_fs_dir}" \ + PORTAGE_CONFIGROOT="${BUILD_DIR}"/configroot \ + emerge --root-deps=rdeps --usepkgonly --jobs=$FLAGS_jobs -v "$@" # Make sure profile.env and ld.so.cache has been generated sudo -E ROOT="${root_fs_dir}" env-update } +# Switch to the dev or prod sub-profile +set_image_profile() { + local suffix="$1" + local profile="${BUILD_DIR}/configroot/etc/portage/make.profile" + if [[ ! -d "${profile}/${suffix}" ]]; then + die "Not a valid profile: ${profile}/${suffix}" + fi + local realpath=$(readlink -f "${profile}/${suffix}") + ln -snf "${realpath}" "${profile}" +} + # Usage: systemd_enable /root default.target something.service # Or: systemd_enable /root default.target some@.service some@thing.service systemd_enable() { @@ -156,6 +159,10 @@ start_image() { local disk_img="${BUILD_DIR}/${image_name}" + mkdir -p "${BUILD_DIR}"/configroot/etc/portage + ln -s "${BOARD_ROOT}"/etc/portage/* \ + "${BUILD_DIR}"/configroot/etc/portage/ + info "Using image type ${disk_layout}" "${BUILD_LIBRARY_DIR}/disk_util" --disk_layout="${disk_layout}" \ format "${disk_img}" @@ -165,8 +172,7 @@ start_image() { trap "cleanup_mounts '${root_fs_dir}' && delete_prompt" EXIT # First thing first, install baselayout to create a working filesystem. - emerge-${BOARD} --root="${root_fs_dir}" \ - --usepkgonly --oneshot --quiet --nodeps sys-apps/baselayout + emerge_to_image "${root_fs_dir}" --nodeps --oneshot sys-apps/baselayout # FIXME(marineam): Work around glibc setting EROOT=$ROOT # https://bugs.gentoo.org/show_bug.cgi?id=473728#c12 @@ -222,6 +228,7 @@ finish_image() { sudo fstrim "${root_fs_dir}/usr" || true fi + rm -rf "${BUILD_DIR}"/configroot cleanup_mounts "${root_fs_dir}" trap - EXIT } diff --git a/build_library/dev_image_util.sh b/build_library/dev_image_util.sh index 93c128177a..47f8506695 100755 --- a/build_library/dev_image_util.sh +++ b/build_library/dev_image_util.sh @@ -71,6 +71,7 @@ create_dev_image() { start_image "${image_name}" "${disk_layout}" "${root_fs_dir}" "${update_group}" + set_image_profile dev emerge_to_image "${root_fs_dir}" coreos-base/coreos-dev write_packages "${root_fs_dir}" "${BUILD_DIR}/${image_packages}" diff --git a/build_library/prod_image_util.sh b/build_library/prod_image_util.sh index 29b03cf8a2..a4f11bfcc2 100755 --- a/build_library/prod_image_util.sh +++ b/build_library/prod_image_util.sh @@ -7,8 +7,8 @@ # In prod images we only need the shared libraries. emerge_prod_gcc() { local root_fs_dir="$1"; shift - local mask="${INSTALL_MASK:-$(portageq-$BOARD envvar PROD_INSTALL_MASK)}" - test -n "$mask" || die "PROD_INSTALL_MASK not defined" + local mask="${INSTALL_MASK:-$(portageq-$BOARD envvar INSTALL_MASK)}" + test -n "$mask" || die "INSTALL_MASK not defined" mask="${mask} /usr/bin @@ -38,6 +38,7 @@ create_prod_image() { start_image "${image_name}" "${disk_layout}" "${root_fs_dir}" "${update_group}" # Install minimal GCC (libs only) and then everything else + set_image_profile prod emerge_prod_gcc "${root_fs_dir}" emerge_to_image "${root_fs_dir}" coreos-base/coreos write_packages "${root_fs_dir}" "${BUILD_DIR}/${image_packages}" From 07cb281d6a96d155d662160ad6222f69a4cd84f2 Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Sat, 19 Jul 2014 17:25:56 -0700 Subject: [PATCH 4/5] build_image: install the complete system package set in dev images dev images, now with fancy tools like `make`! --- build_library/dev_image_util.sh | 2 +- build_packages | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/build_library/dev_image_util.sh b/build_library/dev_image_util.sh index 47f8506695..e552b71641 100755 --- a/build_library/dev_image_util.sh +++ b/build_library/dev_image_util.sh @@ -72,7 +72,7 @@ create_dev_image() { start_image "${image_name}" "${disk_layout}" "${root_fs_dir}" "${update_group}" set_image_profile dev - emerge_to_image "${root_fs_dir}" coreos-base/coreos-dev + emerge_to_image "${root_fs_dir}" @system coreos-base/coreos-dev write_packages "${root_fs_dir}" "${BUILD_DIR}/${image_packages}" # Setup portage for emerge and gmerge diff --git a/build_packages b/build_packages index 728c500c64..42b19d1ceb 100755 --- a/build_packages +++ b/build_packages @@ -212,7 +212,8 @@ break_dep_loop sys-apps/util-linux udev break_dep_loop sys-apps/systemd cryptsetup info "Merging board packages now" -sudo -E "${EMERGE_CMD[@]}" "${EMERGE_FLAGS[@]}" coreos-devel/board-packages +sudo -E "${EMERGE_CMD[@]}" "${EMERGE_FLAGS[@]}" \ + @system coreos-devel/board-packages info "Checking build root" test_image_content "${BOARD_ROOT}" From f9e03a8e6f5b13d68af45837970cc71fbbc0701c Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Sat, 19 Jul 2014 17:52:31 -0700 Subject: [PATCH 5/5] build_image: add repos.conf and binpkgs url to dev images Needed for portage 2.2. Sync URIs are included but not very useful yet because portage only can do `git pull` but not `git clone`. An extra helper script will be required to do the initial clone it seems. Binary packages may be useful for re-installing a package with a different INSTALL_MASK. Can be used to install debug symbols. --- build_library/dev_image_util.sh | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/build_library/dev_image_util.sh b/build_library/dev_image_util.sh index e552b71641..84fd5e6b0a 100755 --- a/build_library/dev_image_util.sh +++ b/build_library/dev_image_util.sh @@ -2,10 +2,6 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -# Shell function library for functions specific to creating dev -# images from base images. The main function for export in this -# library is 'install_dev_packages'. - configure_dev_portage() { # Need profiles at the bare minimum local repo @@ -17,7 +13,7 @@ configure_dev_portage() { "$1/var/lib/portage/${repo}" done - sudo mkdir -p "$1/etc/portage" + sudo mkdir -p "$1/etc/portage/repos.conf" sudo_clobber "$1/etc/portage/make.conf" <