From 89e31f1e579dfff4fe18c30056abec5093abdbf2 Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Fri, 19 Feb 2016 13:08:57 -0800 Subject: [PATCH 1/8] setup_board: add --regen_configs_only flag --- setup_board | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/setup_board b/setup_board index 0e005a9c34..697beab6b2 100755 --- a/setup_board +++ b/setup_board @@ -35,6 +35,10 @@ DEFINE_boolean skip_toolchain_update "${FLAGS_FALSE}" \ "Don't update toolchain automatically." DEFINE_boolean skip_chroot_upgrade "${FLAGS_FALSE}" \ "Don't run the chroot upgrade automatically; use with care." +DEFINE_boolean regen_configs "${FLAGS_FALSE}" \ + "Regenerate all config files (useful for modifying profiles w/out rebuild)." +DEFINE_boolean regen_configs_only "${FLAGS_FALSE}" \ + "Regenerate all config files and nothing else, even if nothing else is installed." FLAGS_HELP="usage: $(basename $0) [flags] @@ -53,8 +57,6 @@ DEFINE_boolean quiet $FLAGS_FALSE \ "Don't print warnings when board already exists." DEFINE_string variant "" \ "Board variant." -DEFINE_boolean regen_configs ${FLAGS_FALSE} \ - "Regenerate all config files (useful for modifying profiles w/out rebuild)." # builds wrappers like equery-arm-generic. @@ -197,7 +199,9 @@ BOARD_CHOST=$(get_board_chost ${BOARD}) PORTAGE_PROFILE=$(get_board_profile "$BOARD") BOARD_BINHOST=$(generate_binhost_list) -if [ -d "${BOARD_ROOT}" ]; then +if [[ ${FLAGS_regen_configs_only} -eq ${FLAGS_TRUE} ]]; then + FLAGS_regen_configs=${FLAGS_TRUE} +elif [[ -d "${BOARD_ROOT}" ]]; then if [[ ${FLAGS_force} -eq ${FLAGS_TRUE} ]]; then info "--force set. Re-creating ${BOARD_ROOT}..." # Removal takes long. Make it asynchronous. @@ -213,7 +217,7 @@ if [ -d "${BOARD_ROOT}" ]; then exit 0 fi else - # Regenerating configs w/out a board root doesn't make sense. + # Missing board root and --regen_configs_only wasn't used. FLAGS_regen_configs=${FLAGS_FALSE} fi From 7979650cdecd50b91dba8e8bfd57cc37b26cf5d3 Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Fri, 19 Feb 2016 13:21:16 -0800 Subject: [PATCH 2/8] setup_board: move arm64 grub recompile to update_chroot --- setup_board | 21 +++------------------ update_chroot | 11 +++++++++++ 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/setup_board b/setup_board index 697beab6b2..285b457154 100755 --- a/setup_board +++ b/setup_board @@ -177,18 +177,6 @@ if [ "${FLAGS_skip_chroot_upgrade}" -eq "${FLAGS_FALSE}" ] ; then "${SRC_ROOT}/scripts"/update_chroot ${UPDATE_ARGS} fi -case "$BOARD" in - *-host) - die_notrace "host boards not supported by setup_board" - ;; - arm64-usr) - # Set grub use flags to build arm64. - [[ ! -d /etc/portage/package.use ]] && sudo mkdir /etc/portage/package.use - sudo_clobber "/etc/portage/package.use/grub" < "$GCLIENT_ROOT/src/scripts/.default_board" fi @@ -342,9 +333,3 @@ if [ -n "${WORKING_ON}" ]; then info "Currently working on the following ebuilds for this board:" info "${WORKING_ON}" fi - -# Setup BOARD_ROOT for QEMU user emulation. -setup_qemu_static "${BOARD_ROOT}" - -# Build grub platform modules if needed. -sudo -E emerge --changed-use sys-boot/grub diff --git a/update_chroot b/update_chroot index d8106697a1..6440b1e77e 100755 --- a/update_chroot +++ b/update_chroot @@ -222,6 +222,17 @@ if [[ "${FLAGS_skip_toolchain_update}" -eq "${FLAGS_FALSE}" && \ for cross_chost in "${CROSS_CHOSTS[@]}"; do info "Updating cross ${cross_chost} toolchain" install_cross_toolchain "${cross_chost}" --quiet ${EMERGE_FLAGS} + + # Set grub for arm64, not enabled by default since it requires the above + # cross toolchain which isn't in the SDK by default. + if [[ "${cross_chost}" == aarch64-* ]]; then + if [[ ! -d /etc/portage/package.use ]]; then + sudo mkdir /etc/portage/package.use + fi + sudo_clobber "/etc/portage/package.use/grub" \ + <<<"sys-boot/grub grub_platforms_arm64" + sudo -E emerge --changed-use sys-boot/grub + fi done fi From a714804ca6f44d9f7dd67c3edcb0e7f3212ad8dc Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Fri, 19 Feb 2016 13:24:44 -0800 Subject: [PATCH 3/8] setup_board: check regen flags before calling update_chroot Abort early if applicable, skip update_chroot if regen only is enabled. --- setup_board | 65 +++++++++++++++++++++++++++-------------------------- 1 file changed, 33 insertions(+), 32 deletions(-) diff --git a/setup_board b/setup_board index 285b457154..1868b8e1f8 100755 --- a/setup_board +++ b/setup_board @@ -150,6 +150,39 @@ fi get_board_and_variant $FLAGS_board $FLAGS_variant +# Locations we will need +COREOS_OVERLAY="${REPO_ROOT}/src/third_party/coreos-overlay" +COREOS_CONFIG="${COREOS_OVERLAY}/coreos/config" +BOARD_ROOT="/build/${BOARD_VARIANT}" +BOARD_ETC="${BOARD_ROOT}/etc" +BOARD_ARCH=$(get_board_arch "$BOARD") +BOARD_CHOST=$(get_board_chost ${BOARD}) +PORTAGE_PROFILE=$(get_board_profile "$BOARD") +BOARD_BINHOST=$(generate_binhost_list) + +if [[ ${FLAGS_regen_configs_only} -eq ${FLAGS_TRUE} ]]; then + FLAGS_regen_configs=${FLAGS_TRUE} + FLAGS_skip_chroot_upgrade=${FLAGS_TRUE} +elif [[ -d "${BOARD_ROOT}" ]]; then + if [[ ${FLAGS_force} -eq ${FLAGS_TRUE} ]]; then + info "--force set. Re-creating ${BOARD_ROOT}..." + # Removal takes long. Make it asynchronous. + TEMP_DIR=`mktemp -d` + sudo mv "${BOARD_ROOT}" "${TEMP_DIR}" + sudo rm -rf "${TEMP_DIR}" & + elif [[ ${FLAGS_regen_configs} -eq ${FLAGS_FALSE} ]]; then + if [[ ${FLAGS_quiet} -eq ${FLAGS_FALSE} ]]; then + warn "Board output directory '$BOARD_ROOT' already exists." + warn "Not setting up board root. " + warn "Use --force to clobber the board root and start again." + fi + exit 0 + fi +else + # Missing board root and --regen_configs_only wasn't used. + FLAGS_regen_configs=${FLAGS_FALSE} +fi + # Before we can run any tools, we need to update chroot UPDATE_ARGS="--toolchain_boards=${BOARD}" if [ "${FLAGS_usepkg}" -eq "${FLAGS_TRUE}" ]; then @@ -177,38 +210,6 @@ if [ "${FLAGS_skip_chroot_upgrade}" -eq "${FLAGS_FALSE}" ] ; then "${SRC_ROOT}/scripts"/update_chroot ${UPDATE_ARGS} fi -# Locations we will need -COREOS_OVERLAY="${REPO_ROOT}/src/third_party/coreos-overlay" -COREOS_CONFIG="${COREOS_OVERLAY}/coreos/config" -BOARD_ROOT="/build/${BOARD_VARIANT}" -BOARD_ETC="${BOARD_ROOT}/etc" -BOARD_ARCH=$(get_board_arch "$BOARD") -BOARD_CHOST=$(get_board_chost ${BOARD}) -PORTAGE_PROFILE=$(get_board_profile "$BOARD") -BOARD_BINHOST=$(generate_binhost_list) - -if [[ ${FLAGS_regen_configs_only} -eq ${FLAGS_TRUE} ]]; then - FLAGS_regen_configs=${FLAGS_TRUE} -elif [[ -d "${BOARD_ROOT}" ]]; then - if [[ ${FLAGS_force} -eq ${FLAGS_TRUE} ]]; then - info "--force set. Re-creating ${BOARD_ROOT}..." - # Removal takes long. Make it asynchronous. - TEMP_DIR=`mktemp -d` - sudo mv "${BOARD_ROOT}" "${TEMP_DIR}" - sudo rm -rf "${TEMP_DIR}" & - elif [[ ${FLAGS_regen_configs} -eq ${FLAGS_FALSE} ]]; then - if [[ ${FLAGS_quiet} -eq ${FLAGS_FALSE} ]]; then - warn "Board output directory '$BOARD_ROOT' already exists." - warn "Not setting up board root. " - warn "Use --force to clobber the board root and start again." - fi - exit 0 - fi -else - # Missing board root and --regen_configs_only wasn't used. - FLAGS_regen_configs=${FLAGS_FALSE} -fi - # Migrate board roots that were created before the package location # was standardized to /var/lib/portage/pkgs, build_image will fail if we # simply forget about the old location and start writing to the new. From 61672f7f75e1011028ecc753b495dd171949ca1e Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Sat, 20 Feb 2016 14:05:50 -0800 Subject: [PATCH 4/8] setup_board: always create tmp directories --- setup_board | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup_board b/setup_board index 1868b8e1f8..163d0dd673 100755 --- a/setup_board +++ b/setup_board @@ -271,6 +271,9 @@ EOF sudo cp /etc/portage/repos.conf/* "${BOARD_ETC}"/portage/repos.conf/ +# required when using --regen_configs_only +sudo mkdir -p --mode=01777 "${BOARD_ROOT}"{/tmp,/var/tmp} + # make it easy to find debug symbols sudo mkdir -p /usr/lib/debug/build sudo ln -sfT ${BOARD_ROOT}/usr/lib/debug /usr/lib/debug/${BOARD_ROOT} From 9ab853a668d6a62fc5f466efd5abadb9f3d96138 Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Sat, 20 Feb 2016 14:11:41 -0800 Subject: [PATCH 5/8] build_image: remove eclean, run in build_packages instead Allows the binary package cache to be preserved when using build_image without a fully populated board root. --- build_image | 2 -- build_packages | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build_image b/build_image index fc3d9debdc..52bf3a8f0c 100755 --- a/build_image +++ b/build_image @@ -105,8 +105,6 @@ for arg in "$@"; do esac done -eclean-$BOARD -d packages - # Check that the build root is sane. if [[ ${skip_test_build_root} -ne 1 ]]; then info "Checking build root" diff --git a/build_packages b/build_packages index 518f97fc84..c40c4ef6e7 100755 --- a/build_packages +++ b/build_packages @@ -243,6 +243,8 @@ if [[ "${FLAGS_usepkgonly}" -eq "${FLAGS_TRUE}" ]]; then fi fi +eclean-$BOARD -d packages + info "Checking build root" test_image_content "${BOARD_ROOT}" From ab3f1ee0b8a4912a20249b6fbeedcf909dd1ca02 Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Sat, 20 Feb 2016 14:14:16 -0800 Subject: [PATCH 6/8] build_image: remove hack to work around an old bug --- build_image | 5 ----- 1 file changed, 5 deletions(-) diff --git a/build_image b/build_image index 52bf3a8f0c..bd87b69ca5 100755 --- a/build_image +++ b/build_image @@ -111,11 +111,6 @@ if [[ ${skip_test_build_root} -ne 1 ]]; then test_image_content "${BOARD_ROOT}" fi -# Hack to fix bug where x86_64 CHOST line gets incorrectly added. -# ToDo(msb): remove this hack. -PACKAGES_FILE="${BOARD_ROOT}/packages/Packages" -sudo sed -e "s/CHOST: x86_64-pc-linux-gnu//" -i "${PACKAGES_FILE}" - # Handle existing directory. if [[ -e "${BUILD_DIR}" ]]; then if [[ ${FLAGS_replace} -eq ${FLAGS_TRUE} ]]; then From bc5de30442acedef156dba69e1deb271e48a6637 Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Sat, 20 Feb 2016 14:17:45 -0800 Subject: [PATCH 7/8] board_options: new portageq wrappers - May be sourced early, so explicitly die if source fails. - Add a function for getting the latest version of a package. - Read PROVIDES metadata using portageq, enabling data to be read from binary packages in addition to installed packages. The performance issue is not an issue here and needed to support empty build roots. --- build_library/board_options.sh | 18 ++++++++++++------ build_library/build_image_util.sh | 2 +- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/build_library/board_options.sh b/build_library/board_options.sh index cd3fb6c4bd..e2280ccad1 100644 --- a/build_library/board_options.sh +++ b/build_library/board_options.sh @@ -12,7 +12,7 @@ BOARD_ROOT="/build/${BOARD}" ARCH=$(get_board_arch ${BOARD}) # What cross-build are we targeting? -. "${BOARD_ROOT}/etc/portage/make.conf" +. "${BOARD_ROOT}/etc/portage/make.conf" || die # check if any of the given use flags are enabled for a pkg pkg_use_enabled() { @@ -25,12 +25,18 @@ pkg_use_enabled() { return $? } -# get a package's SONAMEs in soname.provided format -pkg_soname_provides() { +# Usage: pkg_version [installed|binary|ebuild] some-pkg/name +# Prints: some-pkg/name-1.2.3 +# Note: returns 0 even if the package was not found. +pkg_version() { + portageq-"${BOARD}" best_visible "${BOARD_ROOT}" "$1" "$2" +} + +# Usage: pkg_provides [installed|binary] some-pkg/name-1.2.3 +# Prints: x86_32: libfoo.so.2 x86_64: libfoo.so.2 +pkg_provides() { local provides p - # We could run this command but it ugly and silly slow: - # portageq-"${BOARD}" metadata "${BOARD_ROOT}" installed "$1" PROVIDES - provides=$(<"${BOARD_ROOT}/var/db/pkg/$1/PROVIDES") + provides=$(portageq-"${BOARD}" metadata "${BOARD_ROOT}" "$1" "$2" PROVIDES) if [[ -z "$provides" ]]; then return diff --git a/build_library/build_image_util.sh b/build_library/build_image_util.sh index a600fb3755..65b241a4e5 100755 --- a/build_library/build_image_util.sh +++ b/build_library/build_image_util.sh @@ -211,7 +211,7 @@ package_provided() { for p in "$@"; do info "Writing $p to package.provided and soname.provided" echo "$p" >> "${profile}/package.provided" - pkg_soname_provides "$p" >> "${profile}/soname.provided" + pkg_provides binary "$p" >> "${profile}/soname.provided" done } From 743d4bce378aeeeb3686d25ea9cf5fb1dc8e89b1 Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Sat, 20 Feb 2016 14:26:50 -0800 Subject: [PATCH 8/8] build_image: Add support for --getbinpkg Allows build_image to be used without first running build_packages. Note: setup_board --force is required before build_packages will work properly after doing this since baselayout won't be installed otherwise. --- build_image | 10 ++++++++ build_library/build_image_util.sh | 4 ++++ build_library/prod_image_util.sh | 38 ++++++++++++++++++++++++++----- 3 files changed, 46 insertions(+), 6 deletions(-) diff --git a/build_image b/build_image index bd87b69ca5..9a8cc163fc 100755 --- a/build_image +++ b/build_image @@ -22,6 +22,10 @@ DEFAULT_GROUP=developer # Developer-visible flags. DEFINE_string board "${DEFAULT_BOARD}" \ "The board to build an image for." +DEFINE_boolean getbinpkg "${FLAGS_FALSE}" \ + "Download binary packages from remote repository." +DEFINE_string getbinpkgver "" \ + "Use binary packages from a specific version." DEFINE_boolean enable_rootfs_verification ${FLAGS_TRUE} \ "Default all bootloaders to use kernel-based root fs integrity checking." DEFINE_boolean enable_verity ${FLAGS_FALSE} \ @@ -84,6 +88,12 @@ switch_to_strict_mode check_gsutil_opts +# If downloading packages is enabled ensure the board is configured properly. +if [[ ${FLAGS_getbinpkg} -eq ${FLAGS_TRUE} ]]; then + "${SRC_ROOT}/scripts/setup_board" --board="${FLAGS_board}" \ + --getbinpkgver="${FLAGS_getbinpkgver}" --regen_configs_only +fi + # N.B. Ordering matters for some of the libraries below, because # some of the files contain initialization used by later files. . "${BUILD_LIBRARY_DIR}/toolchain_util.sh" || exit 1 diff --git a/build_library/build_image_util.sh b/build_library/build_image_util.sh index 65b241a4e5..3dc817d8e1 100755 --- a/build_library/build_image_util.sh +++ b/build_library/build_image_util.sh @@ -114,6 +114,10 @@ run_ldconfig() { emerge_to_image() { local root_fs_dir="$1"; shift + if [[ ${FLAGS_getbinpkg} -eq ${FLAGS_TRUE} ]]; then + set -- --getbinpkg "$@" + fi + sudo -E ROOT="${root_fs_dir}" \ PORTAGE_CONFIGROOT="${BUILD_DIR}"/configroot \ emerge --root-deps=rdeps --usepkgonly --jobs=$FLAGS_jobs -v "$@" diff --git a/build_library/prod_image_util.sh b/build_library/prod_image_util.sh index 9193e62355..7eb519efc6 100755 --- a/build_library/prod_image_util.sh +++ b/build_library/prod_image_util.sh @@ -3,16 +3,42 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +# Lookup the current version of a binary package, downloading it if needed. +# Usage: get_binary_pkg some-pkg/name +# Prints: some-pkg/name-1.2.3 +get_binary_pkg() { + local name="$1" version + + # If possible use the version installed in $BOARD_ROOT, + # fall back to any binary package that is available. + version=$(pkg_version installed "${name}") + if [[ -z "${version}" ]]; then + version=$(pkg_version binary "${name}") + fi + + # Nothing? Maybe we can fetch it. + if [[ -z "${version}" && ${FLAGS_getbinpkg} -eq ${FLAGS_TRUE} ]]; then + emerge-${BOARD} --getbinpkg --usepkgonly --fetchonly --nodeps "${name}" >&2 + version=$(pkg_version binary "${name}") + fi + + # Cry + if [[ -z "${version}" ]]; then + die "Binary package missing for ${name}" + fi + + echo "${version}" +} + # The GCC package includes both its libraries and the compiler. # In prod images we only need the shared libraries. extract_prod_gcc() { - local root_fs_dir="$1"; shift - local gcc=$(portageq-${BOARD} best_version "${BOARD_ROOT}" sys-devel/gcc) - local pkg="$(portageq-${BOARD} pkgdir)/${gcc}.tbz2" + local root_fs_dir="$1" gcc pkg + gcc=$(get_binary_pkg sys-devel/gcc) - if [[ ! -f "${pkg}" ]]; then - die "Binary package missing: $pkg" - fi + # FIXME(marineam): Incompatible with FEATURES=binpkg-multi-instance + pkg="$(portageq-${BOARD} pkgdir)/${gcc}.tbz2" + [[ -f "${pkg}" ]] || die "${pkg} is missing" # Normally GCC's shared libraries are installed to: # /usr/lib/gcc/x86_64-cros-linux-gnu/$version/*