diff --git a/build_packages b/build_packages index 23722d68dd..39f8b8a914 100755 --- a/build_packages +++ b/build_packages @@ -15,13 +15,27 @@ assert_not_root_user DEFINE_string board "${DEFAULT_BOARD}" \ "The board to build packages for." DEFINE_boolean usepkg "${FLAGS_TRUE}" \ - "Use binary packages to bootstrap when possible." + "Use binary packages when possible." +DEFINE_boolean usepkgonly "${FLAGS_FALSE}" \ + "Only use/download binary packages. Implies --noworkon" DEFINE_boolean getbinpkg "${FLAGS_TRUE}" \ "Download binary packages from remote repository." -DEFINE_boolean noworkon "${FLAGS_FALSE}" \ - "Don't force-build workon packages." +DEFINE_string getbinpkgver "" \ + "Use binary packages from a specific version." +DEFINE_boolean toolchainpkgonly "${FLAGS_FALSE}" \ + "Use binary packages only for the board toolchain." +DEFINE_boolean workon "${FLAGS_TRUE}" \ + "Automatically rebuild updated cros-workon packages." DEFINE_boolean fetchonly "${FLAGS_FALSE}" \ "Don't build anything, instead only fetch what is needed." +DEFINE_integer jobs "${NUM_JOBS}" \ + "How many packages to build in parallel at maximum." +DEFINE_boolean rebuild "${FLAGS_FALSE}" \ + "Automatically rebuild dependencies." +DEFINE_boolean skip_toolchain_update "${FLAGS_FALSE}" \ + "Don't update toolchain automatically." +DEFINE_boolean skip_chroot_upgrade "${FLAGS_FALSE}" \ + "Don't run the chroot upgrade automatically; use with care." # include upload options . "${BUILD_LIBRARY_DIR}/release_util.sh" || exit 1 @@ -36,33 +50,6 @@ up by the build_image script to put together a bootable CoreOS image. If [packages] are specified, only build those specific packages (and any dependencies they might need). " -show_help_if_requested "$@" - -# The following options are advanced options, only available to those willing -# to read the source code. They are not shown in help output, since they are -# not needed for the typical developer workflow. -DEFINE_string getbinpkgver "" \ - "Use binary packages from a specific version." -DEFINE_boolean toolchainpkgonly $FLAGS_FALSE \ - "Use binary packages only for the board toolchain." -DEFINE_string accept_licenses "" \ - "Licenses to append to the accept list." -DEFINE_integer jobs "${NUM_JOBS}" \ - "How many packages to build in parallel at maximum." -DEFINE_boolean rebuild "${FLAGS_FALSE}" \ - "Don't automatically rebuild dependencies." -DEFINE_boolean skip_chroot_upgrade "${FLAGS_FALSE}" \ - "Don't run the chroot upgrade automatically; use with care." -DEFINE_boolean skip_toolchain_update "${FLAGS_FALSE}" \ - "Don't update toolchain automatically." - -# The --reuse_pkgs_from_local_boards flag tells Portage to share binary -# packages between boards that are built locally, so that the total time -# required to build several boards is reduced. This flag is only useful -# when you are not able to use remote binary packages, since remote binary -# packages are usually more up to date than anything you have locally. -DEFINE_boolean reuse_pkgs_from_local_boards $FLAGS_FALSE \ - "Bootstrap from local packages instead of remote packages." # Parse command line FLAGS "$@" || exit 1 @@ -71,22 +58,43 @@ eval set -- "${FLAGS_ARGV}" # Die on any errors. switch_to_strict_mode +# TODO(marineam): specify the default top-level ebuild in the portage profile. +# I could have sworn we did that or similar but maybe got lost at some point. +if [[ $# -eq 0 ]]; then + set -- @system coreos-devel/board-packages +fi + if [[ -z "${FLAGS_board}" ]]; then echo "Error: --board is required." exit 1 fi +if [[ "${FLAGS_usepkgonly}" -eq "${FLAGS_TRUE}" ]]; then + for flag in usepkg getbinpkg; do + fvar="FLAGS_${flag}" + if [[ "${!fvar}" -ne "${FLAGS_TRUE}" ]]; then + die_notrace "--usepkgonly is incompatible with --no${flag}" + fi + done + if [[ "${FLAGS_rebuild}" -eq "${FLAGS_TRUE}" ]]; then + die_notrace "--usepkgonly is incompatible with --rebuild" + fi + FLAGS_workon="${FLAGS_FALSE}" +fi + check_gsutil_opts CHROMITE_BIN="${GCLIENT_ROOT}/chromite/bin" # Before we can run any tools, we need to update chroot or setup_board. UPDATE_ARGS=( --regen_configs ) -if [[ -n ${FLAGS_accept_licenses} ]]; then - UPDATE_ARGS+=( --accept_licenses "${FLAGS_accept_licenses}" ) -fi if [ "${FLAGS_usepkg}" -eq "${FLAGS_TRUE}" ]; then UPDATE_ARGS+=( --usepkg ) + if [[ "${FLAGS_usepkgonly}" -eq "${FLAGS_TRUE}" ]]; then + UPDATE_ARGS+=( --usepkgonly ) + else + UPDATE_ARGS+=( --nousepkgonly ) + fi if [[ "${FLAGS_getbinpkg}" -eq "${FLAGS_TRUE}" ]]; then UPDATE_ARGS+=( --getbinpkg ) else @@ -106,9 +114,6 @@ fi if [[ "${FLAGS_jobs}" -ne -1 ]]; then UPDATE_ARGS+=( --jobs=${FLAGS_jobs} ) fi -if [ "${FLAGS_reuse_pkgs_from_local_boards}" -eq "${FLAGS_TRUE}" ]; then - UPDATE_ARGS+=( --reuse_pkgs_from_local_boards ) -fi if [ "${FLAGS_skip_toolchain_update}" -eq "${FLAGS_TRUE}" ]; then UPDATE_ARGS+=( --skip_toolchain_update ) fi @@ -133,19 +138,22 @@ fi EMERGE_CMD+=( ${EXTRA_BOARD_FLAGS} ) -if [[ "${FLAGS_usepkg}" -eq "${FLAGS_TRUE}" || - "${FLAGS_reuse_pkgs_from_local_boards}" -eq "${FLAGS_TRUE}" ]]; then +if [[ "${FLAGS_usepkg}" -eq "${FLAGS_TRUE}" ]]; then # Use binary packages. Include all build-time dependencies, # so as to avoid unnecessary differences between source - # and binary builds. - EMERGE_FLAGS+=( --usepkg --with-bdeps y ) + # and binary builds except for --usepkgonly for speed. + if [[ "${FLAGS_usepkgonly}" -eq "${FLAGS_TRUE}" ]]; then + EMERGE_FLAGS+=( --usepkgonly ) + else + EMERGE_FLAGS+=( --usepkg --with-bdeps y ) + + # Only update toolchain when binpkgs are available. + EMERGE_FLAGS+=( $(get_binonly_args) ) + REBUILD_FLAGS+=( $(get_binonly_args) ) + fi if [[ "${FLAGS_getbinpkg}" -eq "${FLAGS_TRUE}" ]]; then EMERGE_FLAGS+=( --getbinpkg ) fi - - # Only update toolchain when binpkgs are available. - EMERGE_FLAGS+=( $(get_binonly_args) ) - REBUILD_FLAGS+=( $(get_binonly_args) ) fi if [[ "${FLAGS_jobs}" -ne -1 ]]; then @@ -159,7 +167,7 @@ fi # Build cros_workon packages when they are changed. CROS_WORKON_PKGS=() -if [ "${FLAGS_noworkon}" -eq "${FLAGS_FALSE}" ]; then +if [ "${FLAGS_workon}" -eq "${FLAGS_TRUE}" ]; then LIST_MODIFIED_PACKAGES="${CHROMITE_BIN}/cros_list_modified_packages" CROS_WORKON_PKGS+=( $("${LIST_MODIFIED_PACKAGES}" --board=${FLAGS_board}) ) fi @@ -215,21 +223,24 @@ break_dep_loop() { sudo rm -f "${flag_file}" } -# util-linux[udev] -> virtual->udev -> systemd -> util-linux -break_dep_loop sys-apps/util-linux udev,systemd sys-apps/systemd cryptsetup +if [[ "${FLAGS_usepkgonly}" -eq "${FLAGS_FALSE}" ]]; then + # util-linux[udev] -> virtual->udev -> systemd -> util-linux + break_dep_loop sys-apps/util-linux udev,systemd sys-apps/systemd cryptsetup -# systemd[cryptsetup] -> cryptsetup -> lvm2 -> virtual/udev -> systemd -break_dep_loop sys-apps/systemd cryptsetup + # systemd[cryptsetup] -> cryptsetup -> lvm2 -> virtual/udev -> systemd + break_dep_loop sys-apps/systemd cryptsetup +fi info "Merging board packages now" -sudo -E "${EMERGE_CMD[@]}" "${EMERGE_FLAGS[@]}" \ - @system coreos-devel/board-packages +sudo -E "${EMERGE_CMD[@]}" "${EMERGE_FLAGS[@]}" "$@" info "Removing obsolete packages" sudo -E "${EMERGE_CMD[@]}" --quiet --depclean @unavailable -if "portageq-${BOARD}" list_preserved_libs "${BOARD_ROOT}" >/dev/null; then - sudo -E "${EMERGE_CMD[@]}" "${REBUILD_FLAGS[@]}" @preserved-rebuild +if [[ "${FLAGS_usepkgonly}" -eq "${FLAGS_TRUE}" ]]; then + if "portageq-${BOARD}" list_preserved_libs "${BOARD_ROOT}" >/dev/null; then + sudo -E "${EMERGE_CMD[@]}" "${REBUILD_FLAGS[@]}" @preserved-rebuild + fi fi info "Checking build root" diff --git a/setup_board b/setup_board index 725901954f..c10ca694e6 100755 --- a/setup_board +++ b/setup_board @@ -13,14 +13,28 @@ restart_in_chroot_if_needed "$@" assert_not_root_user # Developer-visible flags. -DEFINE_string board "$DEFAULT_BOARD" \ +DEFINE_string board "${DEFAULT_BOARD}" \ "The name of the board to set up." -DEFINE_boolean default $FLAGS_FALSE \ +DEFINE_boolean default "${FLAGS_FALSE}" \ "Set board to the default board in your chroot" -DEFINE_boolean force $FLAGS_FALSE \ +DEFINE_boolean force "${FLAGS_FALSE}" \ "Force re-creating board root." -DEFINE_boolean usepkg $FLAGS_TRUE \ - "Use binary packages to bootstrap." +DEFINE_boolean usepkg "${FLAGS_TRUE}" \ + "Use binary packages when possible." +DEFINE_boolean usepkgonly "${FLAGS_FALSE}" \ + "Only use/download binary packages." +DEFINE_boolean getbinpkg "${FLAGS_TRUE}" \ + "Download binary packages from remote repository." +DEFINE_string getbinpkgver "" \ + "Use binary packages from a specific version." +DEFINE_boolean toolchainpkgonly "${FLAGS_FALSE}" \ + "Use binary packages only for the board toolchain." +DEFINE_integer jobs "${NUM_JOBS}" \ + "How many packages to build in parallel at maximum." +DEFINE_boolean skip_toolchain_update "${FLAGS_FALSE}" \ + "Don't update toolchain automatically." +DEFINE_boolean skip_chroot_upgrade "${FLAGS_FALSE}" \ + "Don't run the chroot upgrade automatically; use with care." FLAGS_HELP="usage: $(basename $0) [flags] @@ -33,26 +47,14 @@ show_help_if_requested "$@" # The following options are advanced options, only available to those willing # to read the source code. They are not shown in help output, since they are # not needed for the typical developer workflow. -DEFINE_string getbinpkgver "" \ - "Use binary packages from a specific version." -DEFINE_boolean toolchainpkgonly $FLAGS_FALSE \ - "Use binary packages only for the board toolchain." -DEFINE_integer jobs "${NUM_JOBS}" \ - "How many packages to build in parallel at maximum." DEFINE_string libc_version "[stable]" \ "Version of libc to use." DEFINE_boolean quiet $FLAGS_FALSE \ "Don't print warnings when board already exists." -DEFINE_boolean skip_toolchain_update $FLAGS_FALSE \ - "Don't update toolchain automatically." -DEFINE_boolean skip_chroot_upgrade $FLAGS_FALSE \ - "Don't run the chroot upgrade automatically; use with care." DEFINE_string variant "" \ "Board variant." DEFINE_boolean regen_configs ${FLAGS_FALSE} \ "Regenerate all config files (useful for modifying profiles w/out rebuild)." -DEFINE_boolean getbinpkg $FLAGS_TRUE \ - "Passed to update_chroot, ignored by setup_board itself." # builds wrappers like equery-arm-generic. @@ -135,12 +137,26 @@ if [ -z "$FLAGS_board" ] ; then exit 1 fi +if [[ "${FLAGS_usepkgonly}" -eq "${FLAGS_TRUE}" ]]; then + for flag in usepkg getbinpkg; do + fvar="FLAGS_${flag}" + if [[ "${!fvar}" -ne "${FLAGS_TRUE}" ]]; then + die_notrace "--usepkgonly is incompatible with --no${flag}" + fi + done +fi + get_board_and_variant $FLAGS_board $FLAGS_variant # Before we can run any tools, we need to update chroot UPDATE_ARGS="--toolchain_boards=${BOARD}" if [ "${FLAGS_usepkg}" -eq "${FLAGS_TRUE}" ]; then UPDATE_ARGS+=" --usepkg" + if [[ "${FLAGS_usepkgonly}" -eq "${FLAGS_TRUE}" ]]; then + UPDATE_ARGS+=" --usepkgonly" + else + UPDATE_ARGS+=" --usepkgonly" + fi if [[ "${FLAGS_getbinpkg}" -eq "${FLAGS_TRUE}" ]]; then UPDATE_ARGS+=" --getbinpkg " else @@ -284,7 +300,12 @@ if [[ ${FLAGS_regen_configs} -eq ${FLAGS_FALSE} ]]; then if [[ "${FLAGS_usepkg}" -eq "${FLAGS_TRUE}" && \ "${FLAGS_getbinpkg}" -eq "${FLAGS_TRUE}" ]] then - EMERGE_FLAGS+=" --usepkg --getbinpkg" + if [[ "${FLAGS_usepkgonly}" -eq "${FLAGS_TRUE}" ]]; then + EMERGE_FLAGS+=" --usepkgonly" + else + EMERGE_FLAGS+=" --usepkg" + fi + EMERGE_FLAGS+=" --getbinpkg" EMERGE_TOOLCHAIN_FLAGS+=" --usepkgonly --getbinpkg" else # When binary packages are disabled we need to make sure the cross diff --git a/update_chroot b/update_chroot index ca3bfa4163..d0a9b35af2 100755 --- a/update_chroot +++ b/update_chroot @@ -13,27 +13,25 @@ assert_inside_chroot "$@" # Do not run as root assert_not_root_user -# Developer-visible flags. -DEFINE_boolean usepkg $FLAGS_TRUE \ - "Use binary packages to bootstrap." -DEFINE_boolean getbinpkg $FLAGS_TRUE \ +DEFINE_boolean usepkg "${FLAGS_TRUE}" \ + "Use binary packages when possible." +DEFINE_boolean getbinpkg "${FLAGS_TRUE}" \ "Download binary packages from remote repository." +DEFINE_boolean usepkgonly "${FLAGS_FALSE}" \ + "Only use/download binary packages. Implies --noworkon" +DEFINE_boolean workon "${FLAGS_TRUE}" \ + "Automatically rebuild updated cros-workon packages." +DEFINE_integer jobs "${NUM_JOBS}" \ + "How many packages to build in parallel at maximum." +DEFINE_boolean skip_toolchain_update "${FLAGS_FALSE}" \ + "Don't update the toolchains." +DEFINE_string toolchain_boards "" \ + "Extra toolchains to setup for the specified boards." FLAGS_HELP="usage: $(basename $0) [flags] Performs an update of the chroot. This script is called as part of build_packages, so there is typically no need to call this script directly. " -show_help_if_requested "$@" - -# The following options are advanced options, only available to those willing -# to read the source code. They are not shown in help output, since they are -# not needed for the typical developer workflow. -DEFINE_integer jobs "${NUM_JOBS}" \ - "How many packages to build in parallel at maximum." -DEFINE_boolean skip_toolchain_update $FLAGS_FALSE \ - "Don't update the toolchains." -DEFINE_string toolchain_boards "" \ - "Extra toolchains to setup for the specified boards." # Parse command line flags FLAGS "$@" || exit 1 @@ -43,6 +41,16 @@ eval set -- "${FLAGS_ARGV}" # so will die prematurely if 'switch_to_strict_mode' is specified before now. switch_to_strict_mode +if [[ "${FLAGS_usepkgonly}" -eq "${FLAGS_TRUE}" ]]; then + for flag in usepkg getbinpkg; do + fvar="FLAGS_${flag}" + if [[ "${!fvar}" -ne "${FLAGS_TRUE}" ]]; then + die_notrace "--usepkgonly is incompatible with --no${flag}" + fi + done + FLAGS_workon="${FLAGS_FALSE}" +fi + . "${BUILD_LIBRARY_DIR}/toolchain_util.sh" PORTAGE_STABLE_OVERLAY="${REPO_ROOT}/src/third_party/portage-stable" @@ -172,6 +180,9 @@ EMERGE_FLAGS="-uNv --with-bdeps=y --select" REBUILD_FLAGS="" if [ "${FLAGS_usepkg}" -eq "${FLAGS_TRUE}" ]; then EMERGE_FLAGS="${EMERGE_FLAGS} --usepkg" + if [[ "${FLAGS_usepkgonly}" -eq "${FLAGS_TRUE}" ]]; then + EMERGE_FLAGS+=" --usepkgonly" + fi if [ "${FLAGS_getbinpkg}" -eq "${FLAGS_TRUE}" ]; then EMERGE_FLAGS="${EMERGE_FLAGS} --getbinpkg" fi @@ -216,9 +227,11 @@ fi # Build cros_workon packages when they are changed. CHROMITE_BIN="${GCLIENT_ROOT}/chromite/bin" -for pkg in $("${CHROMITE_BIN}/cros_list_modified_packages" --host); do - EMERGE_FLAGS+=" --reinstall-atoms=${pkg} --usepkg-exclude=${pkg}" -done +if [ "${FLAGS_workon}" -eq "${FLAGS_TRUE}" ]; then + for pkg in $("${CHROMITE_BIN}/cros_list_modified_packages" --host); do + EMERGE_FLAGS+=" --reinstall-atoms=${pkg} --usepkg-exclude=${pkg}" + done +fi # Second pass, update everything else. EMERGE_FLAGS+=" --deep"