Merge pull request #302 from marineam/dev

Developing in dev images

build_image

@@ -62,8 +62,6 @@ show_help_if_requested "$@"
 # not needed for the typical developer workflow.
 DEFINE_integer build_attempt 1 \
   "The build attempt for this image build."
-DEFINE_boolean fast ${FLAGS_TRUE} \
-  "Use the parallel_emerge wrapper script."
 DEFINE_integer jobs "${NUM_JOBS}" \
   "How many packages to build in parallel at maximum."
 DEFINE_boolean replace ${FLAGS_FALSE} \

build_library/build_image_util.sh

@@ -98,27 +98,26 @@ generate_update() {
 # to the basic emerge command.
 emerge_to_image() {
   local root_fs_dir="$1"; shift
-  local mask="${INSTALL_MASK:-$(portageq-$BOARD envvar PROD_INSTALL_MASK)}"
-  test -n "$mask" || die "PROD_INSTALL_MASK not defined"
 
-  local emerge_cmd
-  if [[ "${FLAGS_fast}" -eq "${FLAGS_TRUE}" ]]; then
-    emerge_cmd="$GCLIENT_ROOT/chromite/bin/parallel_emerge --board=$BOARD"
-  else
-    emerge_cmd="emerge-$BOARD"
-  fi
-  emerge_cmd+=" --root-deps=rdeps --usepkgonly -v"
-
-  if [[ $FLAGS_jobs -ne -1 ]]; then
-    emerge_cmd+=" --jobs=$FLAGS_jobs"
-  fi
-
-  sudo -E INSTALL_MASK="$mask" ${emerge_cmd} --root="${root_fs_dir}" "$@"
+  sudo -E ROOT="${root_fs_dir}" \
+      PORTAGE_CONFIGROOT="${BUILD_DIR}"/configroot \
+      emerge --root-deps=rdeps --usepkgonly --jobs=$FLAGS_jobs -v "$@"
 
   # Make sure profile.env and ld.so.cache has been generated
   sudo -E ROOT="${root_fs_dir}" env-update
 }
 
+# Switch to the dev or prod sub-profile
+set_image_profile() {
+  local suffix="$1"
+  local profile="${BUILD_DIR}/configroot/etc/portage/make.profile"
+  if [[ ! -d "${profile}/${suffix}" ]]; then
+      die "Not a valid profile: ${profile}/${suffix}"
+  fi
+  local realpath=$(readlink -f "${profile}/${suffix}")
+  ln -snf "${realpath}" "${profile}"
+}
+
 # Usage: systemd_enable /root default.target something.service
 # Or: systemd_enable /root default.target some@.service some@thing.service
 systemd_enable() {
@@ -160,6 +159,10 @@ start_image() {
 
   local disk_img="${BUILD_DIR}/${image_name}"
 
+  mkdir -p "${BUILD_DIR}"/configroot/etc/portage
+  ln -s "${BOARD_ROOT}"/etc/portage/* \
+      "${BUILD_DIR}"/configroot/etc/portage/
+
   info "Using image type ${disk_layout}"
   "${BUILD_LIBRARY_DIR}/disk_util" --disk_layout="${disk_layout}" \
       format "${disk_img}"
@@ -169,8 +172,7 @@ start_image() {
   trap "cleanup_mounts '${root_fs_dir}' && delete_prompt" EXIT
 
   # First thing first, install baselayout to create a working filesystem.
-  emerge-${BOARD} --root="${root_fs_dir}" \
-      --usepkgonly --oneshot --quiet --nodeps sys-apps/baselayout
+  emerge_to_image "${root_fs_dir}" --nodeps --oneshot sys-apps/baselayout
 
   # FIXME(marineam): Work around glibc setting EROOT=$ROOT
   # https://bugs.gentoo.org/show_bug.cgi?id=473728#c12
@@ -226,6 +228,7 @@ finish_image() {
     sudo fstrim "${root_fs_dir}/usr" || true
   fi
 
+  rm -rf "${BUILD_DIR}"/configroot
   cleanup_mounts "${root_fs_dir}"
   trap - EXIT
 }

build_library/dev_image_util.sh

@@ -2,10 +2,6 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-# Shell function library for functions specific to creating dev
-# images from base images.  The main function for export in this
-# library is 'install_dev_packages'.
-
 configure_dev_portage() {
     # Need profiles at the bare minimum
     local repo
@@ -17,7 +13,7 @@ configure_dev_portage() {
             "$1/var/lib/portage/${repo}"
     done
 
-    sudo mkdir -p "$1/etc/portage"
+    sudo mkdir -p "$1/etc/portage/repos.conf"
     sudo_clobber "$1/etc/portage/make.conf" <<EOF
 # make.conf for CoreOS dev images
 ARCH=$(get_board_arch $BOARD)
@@ -26,16 +22,36 @@ BOARD_USE="$BOARD"
 
 # Use /var/lib/portage instead of /usr/portage
 DISTDIR="/var/lib/portage/distfiles"
-PKGDIR="/var/lib/portage/packages"
+PKGDIR="/var/lib/portage/pkgs"
+PORT_LOGDIR="/var/log/portage"
 PORTDIR="/var/lib/portage/portage-stable"
 PORTDIR_OVERLAY="/var/lib/portage/coreos-overlay"
+PORTAGE_BINHOST="$(get_board_binhost $BOARD $COREOS_VERSION_ID)"
+EOF
+
+sudo_clobber "$1/etc/portage/repos.conf/coreos.conf" <<EOF
+[DEFAULT]
+main-repo = portage-stable
+
+[gentoo]
+disabled = true
+
+[coreos]
+location = /var/lib/portage/coreos-overlay
+sync-type = git
+sync-uri = https://github.com/coreos/coreos-overlay.git
+
+[portage-stable]
+location = /var/lib/portage/portage-stable
+sync-type = git
+sync-uri = https://github.com/coreos/portage-stable.git
 EOF
 
     # Now set the correct profile
     sudo PORTAGE_CONFIGROOT="$1" ROOT="$1" \
         PORTDIR="$1/var/lib/portage/portage-stable" \
         PORTDIR_OVERLAY="$1/var/lib/portage/coreos-overlay" \
-        eselect profile set --force $(get_board_profile $BOARD)
+        eselect profile set --force $(get_board_profile $BOARD)/dev
 }
 
 detect_dev_url() {
@@ -71,7 +87,8 @@ create_dev_image() {
 
   start_image "${image_name}" "${disk_layout}" "${root_fs_dir}" "${update_group}"
 
-  emerge_to_image "${root_fs_dir}" coreos-base/coreos-dev
+  set_image_profile dev
+  emerge_to_image "${root_fs_dir}" @system coreos-base/coreos-dev
   write_packages "${root_fs_dir}" "${BUILD_DIR}/${image_packages}"
 
   # Setup portage for emerge and gmerge

build_library/prod_image_util.sh

@@ -7,8 +7,8 @@
 # In prod images we only need the shared libraries.
 emerge_prod_gcc() {
     local root_fs_dir="$1"; shift
-    local mask="${INSTALL_MASK:-$(portageq-$BOARD envvar PROD_INSTALL_MASK)}"
-    test -n "$mask" || die "PROD_INSTALL_MASK not defined"
+    local mask="${INSTALL_MASK:-$(portageq-$BOARD envvar INSTALL_MASK)}"
+    test -n "$mask" || die "INSTALL_MASK not defined"
 
     mask="${mask}
         /usr/bin
@@ -38,6 +38,7 @@ create_prod_image() {
   start_image "${image_name}" "${disk_layout}" "${root_fs_dir}" "${update_group}"
 
   # Install minimal GCC (libs only) and then everything else
+  set_image_profile prod
   emerge_prod_gcc "${root_fs_dir}"
   emerge_to_image "${root_fs_dir}" coreos-base/coreos
   write_packages "${root_fs_dir}" "${BUILD_DIR}/${image_packages}"

build_packages

@@ -20,10 +20,6 @@ DEFINE_boolean getbinpkg "${FLAGS_TRUE}" \
   "Download binary packages from remote repository."
 DEFINE_boolean noworkon "${FLAGS_FALSE}" \
   "Don't force-build workon packages."
-DEFINE_boolean showoutput "${FLAGS_FALSE}" \
-  "Show all output from parallel_emerge."
-DEFINE_boolean withautotest "${FLAGS_TRUE}" \
-  "Build autotest client code."
 DEFINE_boolean fetchonly "${FLAGS_FALSE}" \
   "Don't build anything, instead only fetch what is needed."
 
@@ -39,8 +35,6 @@ up by the build_image script to put together a bootable Chrome OS image.
 
 If [packages] are specified, only build those specific packages (and any
 dependencies they might need).
-
-For the fastest builds, use --nowithautotest --noworkon.
 "
 show_help_if_requested "$@"
 
@@ -53,8 +47,6 @@ DEFINE_boolean toolchainpkgonly $FLAGS_FALSE \
   "Use binary packages only for the board toolchain."
 DEFINE_string accept_licenses "" \
   "Licenses to append to the accept list."
-DEFINE_boolean fast "${DEFAULT_FAST}" \
-  "Use the parallel_emerge wrapper script."
 DEFINE_integer jobs "${NUM_JOBS}" \
   "How many packages to build in parallel at maximum."
 DEFINE_boolean norebuild "${FLAGS_FALSE}" \
@@ -93,11 +85,6 @@ UPDATE_ARGS=( --regen_configs )
 if [[ -n ${FLAGS_accept_licenses} ]]; then
   UPDATE_ARGS+=( --accept_licenses "${FLAGS_accept_licenses}" )
 fi
-if [ "${FLAGS_fast}" -eq "${FLAGS_TRUE}" ]; then
-  UPDATE_ARGS+=( --fast )
-else
-  UPDATE_ARGS+=( --nofast )
-fi
 if [ "${FLAGS_usepkg}" -eq "${FLAGS_TRUE}" ]; then
   UPDATE_ARGS+=( --usepkg )
   if [[ "${FLAGS_getbinpkg}" -eq "${FLAGS_TRUE}" ]]; then
@@ -138,15 +125,7 @@ fi
 
 # Setup all the emerge command/flags.
 EMERGE_FLAGS=( -uDNv --backtrack=30 --select )
-
-if [[ "${FLAGS_fast}" -eq "${FLAGS_TRUE}" ]]; then
-  EMERGE_CMD=(
-    "${CHROMITE_BIN}/parallel_emerge"
-    --board=${FLAGS_board}
-  )
-else
-  EMERGE_CMD=( "emerge-${FLAGS_board}" )
-fi
+EMERGE_CMD=( "emerge-${FLAGS_board}" )
 if [[ "${FLAGS_fetchonly}" -eq "${FLAGS_TRUE}" ]]; then
   EMERGE_CMD+=( --fetchonly )
 fi
@@ -174,11 +153,6 @@ fi
 if [[ "${FLAGS_norebuild}" -eq "${FLAGS_FALSE}" ]]; then
   EMERGE_FLAGS+=( --rebuild-if-unbuilt )
 fi
-if [[ "${FLAGS_showoutput}" -eq "${FLAGS_TRUE}" && \
-      "${FLAGS_fast}" -eq "${FLAGS_TRUE}" ]]; then
-  # Only parallel_emerge supports --show-output.
-  EMERGE_FLAGS+=( --show-output )
-fi
 
 # Build cros_workon packages when they are changed.
 CROS_WORKON_PKGS=()
@@ -238,7 +212,8 @@ break_dep_loop sys-apps/util-linux udev
 break_dep_loop sys-apps/systemd cryptsetup
 
 info "Merging board packages now"
-sudo -E "${EMERGE_CMD[@]}" "${EMERGE_FLAGS[@]}" coreos-devel/board-packages
+sudo -E "${EMERGE_CMD[@]}" "${EMERGE_FLAGS[@]}" \
+    @system coreos-devel/board-packages
 
 info "Checking build root"
 test_image_content "${BOARD_ROOT}"

common.sh

@@ -390,9 +390,6 @@ if [[ -f ${GCLIENT_ROOT}/src/scripts/.default_board ]]; then
   fi
 fi
 
-# Disable --fast in most commands
-DEFAULT_FAST=${FLAGS_FALSE}
-
 # Directory to store built images.  Should be set by sourcing script when used.
 BUILD_DIR=
 

sdk_lib/make_chroot.sh

@@ -40,8 +40,6 @@ DEFINE_boolean getbinpkg $FLAGS_TRUE \
   "Download binary packages from remote repository."
 DEFINE_boolean delete $FLAGS_FALSE "Delete an existing chroot."
 DEFINE_boolean replace $FLAGS_FALSE "Overwrite existing chroot, if any."
-DEFINE_boolean fast "${DEFAULT_FAST}" \
-  "Use the parallel_emerge wrapper script."
 DEFINE_integer jobs "${NUM_JOBS}" \
   "How many packages to build in parallel at maximum."
 DEFINE_string stage3_path "" \
@@ -358,11 +356,6 @@ if [[ ${FLAGS_usepkg} -eq ${FLAGS_TRUE} ]]; then
 else
   UPDATE_ARGS+=( --nousepkg )
 fi
-if [[ ${FLAGS_fast} -eq ${FLAGS_TRUE} ]]; then
-  UPDATE_ARGS+=( --fast )
-else
-  UPDATE_ARGS+=( --nofast )
-fi
 if [[ "${FLAGS_jobs}" -ne -1 ]]; then
   UPDATE_ARGS+=( --jobs=${FLAGS_jobs} )
 fi

setup_board

@@ -37,8 +37,6 @@ DEFINE_string getbinpkgver "" \
   "Use binary packages from a specific version."
 DEFINE_boolean toolchainpkgonly $FLAGS_FALSE \
   "Use binary packages only for the board toolchain."
-DEFINE_boolean fast "${DEFAULT_FAST}" \
-  "Use the parallel_emerge wrapper script."
 DEFINE_integer jobs "${NUM_JOBS}" \
   "How many packages to build in parallel at maximum."
 DEFINE_boolean latest_toolchain $FLAGS_FALSE \
@@ -140,11 +138,6 @@ get_board_and_variant $FLAGS_board $FLAGS_variant
 
 # Before we can run any tools, we need to update chroot
 UPDATE_ARGS="--toolchain_boards=${BOARD}"
-if [ "${FLAGS_fast}" -eq "${FLAGS_TRUE}" ]; then
-  UPDATE_ARGS+=" --fast"
-else
-  UPDATE_ARGS+=" --nofast"
-fi
 if [ "${FLAGS_usepkg}" -eq "${FLAGS_TRUE}" ]; then
   UPDATE_ARGS+=" --usepkg"
   if [[ "${FLAGS_getbinpkg}" -eq "${FLAGS_TRUE}" ]]; then
@@ -174,7 +167,6 @@ esac
 COREOS_CONFIG="/usr/local/portage/coreos/coreos/config"
 BOARD_ROOT="/build/${BOARD_VARIANT}"
 BOARD_ETC="${BOARD_ROOT}/etc"
-BOARD_PROFILE="${BOARD_ETC}/portage/profile"
 BOARD_ARCH=$(get_board_arch "$BOARD")
 BOARD_CHOST=$(get_board_chost ${BOARD})
 PORTAGE_PROFILE=$(get_board_profile "$BOARD")
@@ -218,7 +210,7 @@ if [[ ! -L "${BOARD_ROOT}/packages" ]]; then
 fi
 
 info "Configuring portage in ${BOARD_ROOT}"
-sudo mkdir -p "${BOARD_PROFILE}"
+sudo mkdir -p "${BOARD_ETC}/portage/"{profile,repos.conf}
 sudo ROOT="${BOARD_ROOT}" eselect profile set --force "${PORTAGE_PROFILE}"
 
 # Cleanup/migrate from older make.conf files
@@ -258,6 +250,8 @@ EMERGE_DEFAULT_OPTS="--oneshot"
 source "${BOARD_ETC}/portage/make.conf.user"
 EOF
 
+sudo cp /etc/portage/repos.conf/* "${BOARD_ETC}"/portage/repos.conf/
+
 generate_all_wrappers
 
 if [[ ${FLAGS_regen_configs} -eq ${FLAGS_FALSE} ]]; then

update_chroot

@@ -28,8 +28,6 @@ show_help_if_requested "$@"
 # The following options are advanced options, only available to those willing
 # to read the source code. They are not shown in help output, since they are
 # not needed for the typical developer workflow.
-DEFINE_boolean fast ${DEFAULT_FAST} \
-  "Use the parallel_emerge wrapper script."
 DEFINE_integer jobs "${NUM_JOBS}" \
   "How many packages to build in parallel at maximum."
 DEFINE_boolean skip_toolchain_update $FLAGS_FALSE \
@@ -148,10 +146,6 @@ fi
 
 # Perform an update of coreos-devel/sdk-depends and world in the chroot.
 EMERGE_CMD="emerge"
-CHROMITE_BIN="${GCLIENT_ROOT}/chromite/bin"
-if [ "${FLAGS_fast}" -eq "${FLAGS_TRUE}" ]; then
-  EMERGE_CMD="${CHROMITE_BIN}/parallel_emerge"
-fi
 
 # In first pass, update portage and toolchains. Lagged updates of both
 # can cause serious issues later.
@@ -175,6 +169,7 @@ if [[ "${FLAGS_skip_toolchain_update}" -eq "${FLAGS_FALSE}" && \
 fi
 
 # Build cros_workon packages when they are changed.
+CHROMITE_BIN="${GCLIENT_ROOT}/chromite/bin"
 for pkg in $("${CHROMITE_BIN}/cros_list_modified_packages" --host); do
   EMERGE_FLAGS+=" --reinstall-atoms=${pkg} --usepkg-exclude=${pkg}"
 done