Merge pull request #500 from marineam/setup

Support using build_image without running build_packages first

build_image

@@ -22,6 +22,10 @@ DEFAULT_GROUP=developer
 # Developer-visible flags.
 DEFINE_string board "${DEFAULT_BOARD}" \
   "The board to build an image for."
+DEFINE_boolean getbinpkg "${FLAGS_FALSE}" \
+  "Download binary packages from remote repository."
+DEFINE_string getbinpkgver "" \
+  "Use binary packages from a specific version."
 DEFINE_boolean enable_rootfs_verification ${FLAGS_TRUE} \
   "Default all bootloaders to use kernel-based root fs integrity checking."
 DEFINE_boolean enable_verity ${FLAGS_FALSE} \
@@ -84,6 +88,12 @@ switch_to_strict_mode
 
 check_gsutil_opts
 
+# If downloading packages is enabled ensure the board is configured properly.
+if [[ ${FLAGS_getbinpkg} -eq ${FLAGS_TRUE} ]]; then
+  "${SRC_ROOT}/scripts/setup_board" --board="${FLAGS_board}" \
+      --getbinpkgver="${FLAGS_getbinpkgver}" --regen_configs_only
+fi
+
 # N.B.  Ordering matters for some of the libraries below, because
 # some of the files contain initialization used by later files.
 . "${BUILD_LIBRARY_DIR}/toolchain_util.sh" || exit 1
@@ -105,19 +115,12 @@ for arg in "$@"; do
   esac
 done
 
-eclean-$BOARD -d packages
-
 # Check that the build root is sane.
 if [[ ${skip_test_build_root} -ne 1 ]]; then
   info "Checking build root"
   test_image_content "${BOARD_ROOT}"
 fi
 
-# Hack to fix bug where x86_64 CHOST line gets incorrectly added.
-# ToDo(msb): remove this hack.
-PACKAGES_FILE="${BOARD_ROOT}/packages/Packages"
-sudo sed -e "s/CHOST: x86_64-pc-linux-gnu//" -i "${PACKAGES_FILE}"
-
 # Handle existing directory.
 if [[ -e "${BUILD_DIR}" ]]; then
   if [[ ${FLAGS_replace} -eq ${FLAGS_TRUE} ]]; then

build_library/board_options.sh

@@ -12,7 +12,7 @@ BOARD_ROOT="/build/${BOARD}"
 ARCH=$(get_board_arch ${BOARD})
 
 # What cross-build are we targeting?
-. "${BOARD_ROOT}/etc/portage/make.conf"
+. "${BOARD_ROOT}/etc/portage/make.conf" || die
 
 # check if any of the given use flags are enabled for a pkg
 pkg_use_enabled() {
@@ -25,12 +25,18 @@ pkg_use_enabled() {
   return $?
 }
 
-# get a package's SONAMEs in soname.provided format
-pkg_soname_provides() {
+# Usage: pkg_version [installed|binary|ebuild] some-pkg/name
+# Prints: some-pkg/name-1.2.3
+# Note: returns 0 even if the package was not found.
+pkg_version() {
+  portageq-"${BOARD}" best_visible "${BOARD_ROOT}" "$1" "$2"
+}
+
+# Usage: pkg_provides [installed|binary] some-pkg/name-1.2.3
+# Prints: x86_32: libfoo.so.2 x86_64: libfoo.so.2
+pkg_provides() {
   local provides p
-  # We could run this command but it ugly and silly slow:
-  # portageq-"${BOARD}" metadata "${BOARD_ROOT}" installed "$1" PROVIDES
-  provides=$(<"${BOARD_ROOT}/var/db/pkg/$1/PROVIDES")
+  provides=$(portageq-"${BOARD}" metadata "${BOARD_ROOT}" "$1" "$2" PROVIDES)
 
   if [[ -z "$provides" ]]; then
     return

build_library/build_image_util.sh

@@ -114,6 +114,10 @@ run_ldconfig() {
 emerge_to_image() {
   local root_fs_dir="$1"; shift
 
+  if [[ ${FLAGS_getbinpkg} -eq ${FLAGS_TRUE} ]]; then
+    set -- --getbinpkg "$@"
+  fi
+
   sudo -E ROOT="${root_fs_dir}" \
       PORTAGE_CONFIGROOT="${BUILD_DIR}"/configroot \
       emerge --root-deps=rdeps --usepkgonly --jobs=$FLAGS_jobs -v "$@"
@@ -211,7 +215,7 @@ package_provided() {
     for p in "$@"; do
         info "Writing $p to package.provided and soname.provided"
         echo "$p" >> "${profile}/package.provided"
-	pkg_soname_provides "$p" >> "${profile}/soname.provided"
+	pkg_provides binary "$p" >> "${profile}/soname.provided"
     done
 }
 

build_library/prod_image_util.sh

@@ -3,16 +3,42 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+# Lookup the current version of a binary package, downloading it if needed.
+# Usage: get_binary_pkg some-pkg/name
+# Prints: some-pkg/name-1.2.3
+get_binary_pkg() {
+    local name="$1" version
+
+    # If possible use the version installed in $BOARD_ROOT,
+    # fall back to any binary package that is available.
+    version=$(pkg_version installed "${name}")
+    if [[ -z "${version}" ]]; then
+        version=$(pkg_version binary "${name}")
+    fi
+
+    # Nothing? Maybe we can fetch it.
+    if [[ -z "${version}" && ${FLAGS_getbinpkg} -eq ${FLAGS_TRUE} ]]; then
+        emerge-${BOARD} --getbinpkg --usepkgonly --fetchonly --nodeps "${name}" >&2
+        version=$(pkg_version binary "${name}")
+    fi
+
+    # Cry
+    if [[ -z "${version}" ]]; then
+        die "Binary package missing for ${name}"
+    fi
+
+    echo "${version}"
+}
+
 # The GCC package includes both its libraries and the compiler.
 # In prod images we only need the shared libraries.
 extract_prod_gcc() {
-    local root_fs_dir="$1"; shift
-    local gcc=$(portageq-${BOARD} best_version "${BOARD_ROOT}" sys-devel/gcc)
-    local pkg="$(portageq-${BOARD} pkgdir)/${gcc}.tbz2"
+    local root_fs_dir="$1" gcc pkg
+    gcc=$(get_binary_pkg sys-devel/gcc)
 
-    if [[ ! -f "${pkg}" ]]; then
-        die "Binary package missing: $pkg"
-    fi
+    # FIXME(marineam): Incompatible with FEATURES=binpkg-multi-instance
+    pkg="$(portageq-${BOARD} pkgdir)/${gcc}.tbz2"
+    [[ -f "${pkg}" ]] || die "${pkg} is missing"
 
     # Normally GCC's shared libraries are installed to:
     #  /usr/lib/gcc/x86_64-cros-linux-gnu/$version/*

build_packages

@@ -243,6 +243,8 @@ if [[ "${FLAGS_usepkgonly}" -eq "${FLAGS_TRUE}" ]]; then
   fi
 fi
 
+eclean-$BOARD -d packages
+
 info "Checking build root"
 test_image_content "${BOARD_ROOT}"
 

setup_board

@@ -35,6 +35,10 @@ DEFINE_boolean skip_toolchain_update "${FLAGS_FALSE}" \
   "Don't update toolchain automatically."
 DEFINE_boolean skip_chroot_upgrade "${FLAGS_FALSE}" \
   "Don't run the chroot upgrade automatically; use with care."
+DEFINE_boolean regen_configs "${FLAGS_FALSE}" \
+  "Regenerate all config files (useful for modifying profiles w/out rebuild)."
+DEFINE_boolean regen_configs_only "${FLAGS_FALSE}" \
+  "Regenerate all config files and nothing else, even if nothing else is installed."
 
 FLAGS_HELP="usage: $(basename $0) [flags]
 
@@ -53,8 +57,6 @@ DEFINE_boolean quiet $FLAGS_FALSE \
   "Don't print warnings when board already exists."
 DEFINE_string variant "" \
   "Board variant."
-DEFINE_boolean regen_configs ${FLAGS_FALSE} \
-  "Regenerate all config files (useful for modifying profiles w/out rebuild)."
 
 
 # builds wrappers like equery-arm-generic.
@@ -148,6 +150,39 @@ fi
 
 get_board_and_variant $FLAGS_board $FLAGS_variant
 
+# Locations we will need
+COREOS_OVERLAY="${REPO_ROOT}/src/third_party/coreos-overlay"
+COREOS_CONFIG="${COREOS_OVERLAY}/coreos/config"
+BOARD_ROOT="/build/${BOARD_VARIANT}"
+BOARD_ETC="${BOARD_ROOT}/etc"
+BOARD_ARCH=$(get_board_arch "$BOARD")
+BOARD_CHOST=$(get_board_chost ${BOARD})
+PORTAGE_PROFILE=$(get_board_profile "$BOARD")
+BOARD_BINHOST=$(generate_binhost_list)
+
+if [[ ${FLAGS_regen_configs_only} -eq ${FLAGS_TRUE} ]]; then
+  FLAGS_regen_configs=${FLAGS_TRUE}
+  FLAGS_skip_chroot_upgrade=${FLAGS_TRUE}
+elif [[ -d "${BOARD_ROOT}" ]]; then
+  if [[ ${FLAGS_force} -eq ${FLAGS_TRUE} ]]; then
+    info "--force set.  Re-creating ${BOARD_ROOT}..."
+    # Removal takes long. Make it asynchronous.
+    TEMP_DIR=`mktemp -d`
+    sudo mv "${BOARD_ROOT}" "${TEMP_DIR}"
+    sudo rm -rf "${TEMP_DIR}" &
+  elif [[ ${FLAGS_regen_configs} -eq ${FLAGS_FALSE} ]]; then
+    if [[ ${FLAGS_quiet} -eq ${FLAGS_FALSE} ]]; then
+      warn "Board output directory '$BOARD_ROOT' already exists."
+      warn "Not setting up board root. "
+      warn "Use --force to clobber the board root and start again."
+    fi
+    exit 0
+  fi
+else
+  # Missing board root and --regen_configs_only wasn't used.
+  FLAGS_regen_configs=${FLAGS_FALSE}
+fi
+
 # Before we can run any tools, we need to update chroot
 UPDATE_ARGS="--toolchain_boards=${BOARD}"
 if [ "${FLAGS_usepkg}" -eq "${FLAGS_TRUE}" ]; then
@@ -175,48 +210,6 @@ if [ "${FLAGS_skip_chroot_upgrade}" -eq "${FLAGS_FALSE}" ] ; then
   "${SRC_ROOT}/scripts"/update_chroot ${UPDATE_ARGS}
 fi
 
-case "$BOARD" in
-  *-host)
-    die_notrace "host boards not supported by setup_board"
-    ;;
-  arm64-usr)
-    # Set grub use flags to build arm64.
-    [[ ! -d /etc/portage/package.use ]] && sudo mkdir /etc/portage/package.use
-    sudo_clobber "/etc/portage/package.use/grub" <<EOF
-sys-boot/grub grub_platforms_arm64
-EOF
-    ;;
-esac
-# Locations we will need
-COREOS_OVERLAY="${REPO_ROOT}/src/third_party/coreos-overlay"
-COREOS_CONFIG="${COREOS_OVERLAY}/coreos/config"
-BOARD_ROOT="/build/${BOARD_VARIANT}"
-BOARD_ETC="${BOARD_ROOT}/etc"
-BOARD_ARCH=$(get_board_arch "$BOARD")
-BOARD_CHOST=$(get_board_chost ${BOARD})
-PORTAGE_PROFILE=$(get_board_profile "$BOARD")
-BOARD_BINHOST=$(generate_binhost_list)
-
-if [ -d "${BOARD_ROOT}" ]; then
-  if [[ ${FLAGS_force} -eq ${FLAGS_TRUE} ]]; then
-    info "--force set.  Re-creating ${BOARD_ROOT}..."
-    # Removal takes long. Make it asynchronous.
-    TEMP_DIR=`mktemp -d`
-    sudo mv "${BOARD_ROOT}" "${TEMP_DIR}"
-    sudo rm -rf "${TEMP_DIR}" &
-  elif [[ ${FLAGS_regen_configs} -eq ${FLAGS_FALSE} ]]; then
-    if [[ ${FLAGS_quiet} -eq ${FLAGS_FALSE} ]]; then
-      warn "Board output directory '$BOARD_ROOT' already exists."
-      warn "Not setting up board root. "
-      warn "Use --force to clobber the board root and start again."
-    fi
-    exit 0
-  fi
-else
-  # Regenerating configs w/out a board root doesn't make sense.
-  FLAGS_regen_configs=${FLAGS_FALSE}
-fi
-
 # Migrate board roots that were created before the package location
 # was standardized to /var/lib/portage/pkgs, build_image will fail if we
 # simply forget about the old location and start writing to the new.
@@ -278,6 +271,9 @@ EOF
 
 sudo cp /etc/portage/repos.conf/* "${BOARD_ETC}"/portage/repos.conf/
 
+# required when using --regen_configs_only
+sudo mkdir -p --mode=01777 "${BOARD_ROOT}"{/tmp,/var/tmp}
+
 # make it easy to find debug symbols
 sudo mkdir -p /usr/lib/debug/build
 sudo ln -sfT ${BOARD_ROOT}/usr/lib/debug /usr/lib/debug/${BOARD_ROOT}
@@ -322,6 +318,9 @@ if [[ ${FLAGS_regen_configs} -eq ${FLAGS_FALSE} ]]; then
       ${EMERGE_TOOLCHAIN_FLAGS} "${TOOLCHAIN_PKGS[@]}"
 fi
 
+# Setup BOARD_ROOT for QEMU user emulation.
+setup_qemu_static "${BOARD_ROOT}"
+
 if [ $FLAGS_default -eq $FLAGS_TRUE ] ; then
   echo $BOARD_VARIANT > "$GCLIENT_ROOT/src/scripts/.default_board"
 fi
@@ -338,9 +337,3 @@ if [ -n "${WORKING_ON}" ]; then
   info "Currently working on the following ebuilds for this board:"
   info "${WORKING_ON}"
 fi
-
-# Setup BOARD_ROOT for QEMU user emulation.
-setup_qemu_static "${BOARD_ROOT}"
-
-# Build grub platform modules if needed.
-sudo -E emerge --changed-use sys-boot/grub

update_chroot

@@ -222,6 +222,17 @@ if [[ "${FLAGS_skip_toolchain_update}" -eq "${FLAGS_FALSE}" && \
   for cross_chost in "${CROSS_CHOSTS[@]}"; do
     info "Updating cross ${cross_chost} toolchain"
     install_cross_toolchain "${cross_chost}" --quiet ${EMERGE_FLAGS}
+
+    # Set grub for arm64, not enabled by default since it requires the above
+    # cross toolchain which isn't in the SDK by default.
+    if [[ "${cross_chost}" == aarch64-* ]]; then
+      if [[ ! -d /etc/portage/package.use ]]; then
+        sudo mkdir /etc/portage/package.use
+      fi
+      sudo_clobber "/etc/portage/package.use/grub" \
+          <<<"sys-boot/grub grub_platforms_arm64"
+      sudo -E emerge --changed-use sys-boot/grub
+    fi
   done
 fi