flatcar-scripts/setup_board

#!/bin/bash

# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

. "$(dirname "$0")/common.sh" || exit 1
. "${SRC_ROOT}/platform/dev/toolchain_utils.sh"

# Script must run inside the chroot
restart_in_chroot_if_needed "$@"

assert_not_root_user

# Developer-visible flags.
DEFINE_string board "$DEFAULT_BOARD" \
  "The name of the board to set up."
DEFINE_boolean default $FLAGS_FALSE \
  "Set board to the default board in your chroot"
DEFINE_boolean force $FLAGS_FALSE \
  "Force re-creating board root."
DEFINE_boolean usepkg $FLAGS_TRUE \
  "Use binary packages to bootstrap."

FLAGS_HELP="usage: $(basename $0) [flags]

setup_board sets up the sysroot for a particular board. This script is called
automatically when you run build_packages, so there is typically no need to
call it directly, unless you want to blow away your board (using --force).
"
show_help_if_requested "$@"

# The following options are advanced options, only available to those willing
# to read the source code. They are not shown in help output, since they are
# not needed for the typical developer workflow.
DEFINE_string accept_licenses "" \
  "Licenses to append to the accept list."
DEFINE_string board_overlay "" \
  "Location of the board overlay."
DEFINE_boolean fast ${DEFAULT_FAST} "Call many emerges in parallel"
DEFINE_integer jobs -1 \
  "How many packages to build in parallel at maximum."
DEFINE_boolean latest_toolchain $FLAGS_FALSE \
  "Use the latest ebuild for all toolchain packages (gcc, binutils, libc, \
kernel). This overrides the other toolchain version options."
DEFINE_string libc_version "[stable]" \
  "Version of libc to use."
DEFINE_string profile "" \
  "The portage configuration profile to use. Profile must be located in overlay-board/profiles"
DEFINE_boolean quiet $FLAGS_FALSE \
  "Don't print warnings when board already exists."
DEFINE_boolean skip_toolchain_update $FLAGS_FALSE \
  "Don't update toolchain automatically."
DEFINE_boolean skip_chroot_upgrade $FLAGS_FALSE \
  "Don't run the chroot upgrade automatically; use with care."
DEFINE_string toolchain "" \
  "Toolchain.  For example: i686-pc-linux-gnu, armv7a-softfloat-linux-gnueabi"
DEFINE_string variant "" \
  "Board variant."
DEFINE_boolean regen_configs ${FLAGS_FALSE} \
  "Regenerate all config files (useful for modifying profiles w/out rebuild)."

# The --reuse_pkgs_from_local_boards flag tells Portage to share binary
# packages between boards that are built locally, so that the total time
# required to build several boards is reduced. This flag is only useful
# when you are not able to use remote binary packages, since remote binary
# packages are usually more up to date than anything you have locally.
DEFINE_boolean reuse_pkgs_from_local_boards $FLAGS_FALSE \
  "Bootstrap from local packages instead of remote packages."


# builds wrappers like equery-arm-generic.
# args:
#   $1:  command to wrap
#   rest:  extra arguments to pass to the command
_generate_wrapper() {
  local command="${1}"
  shift
  local extra_args="$@"


  local target="/usr/local/bin/${command}-${BOARD_VARIANT}"
  sudo_clobber "${target}" <<EOF
#!/bin/bash

export CHROMEOS_ROOT="$GCLIENT_ROOT"
export CHOST="$FLAGS_toolchain"
export PORTAGE_CONFIGROOT="$BOARD_ROOT"
export SYSROOT="$BOARD_ROOT"
if [ -z "\$PORTAGE_USERNAME" ]; then
  export PORTAGE_USERNAME=\$(basename \$HOME)
fi
export ROOT="$BOARD_ROOT"
exec sudo -E ${command} ${extra_args} "\$@"
EOF
  # Note: parent will process these.
  wrappers+=( "${target}" )
  eval ${command^^}_WRAPPER="${target}" # ${foo^^} returns toupper($foo)
}

generate_all_wrappers() {
  local cmds=() wrappers=()
  local wrapper

  info "Generating wrapper scripts"

  wrapper="/usr/local/bin/pkg-config-${BOARD_VARIANT}"
  sudo_clobber "${wrapper}" <<EOF
#!/bin/bash

PKG_CONFIG_LIBDIR=\$(printf '%s:' "${BOARD_ROOT}"/usr/*/pkgconfig)
export PKG_CONFIG_LIBDIR

export PKG_CONFIG_SYSROOT_DIR="${BOARD_ROOT}"

exec pkg-config "\$@"
EOF
  wrappers+=( "${wrapper}" )

  for wrapper in 'emerge --root-deps' ebuild eclean equery portageq \
                 qcheck qfile qlist; do
    _generate_wrapper ${wrapper}
  done

  if [ "${CHOST}" != "$FLAGS_toolchain" ] ; then
    # TODO(cmasone): Do this more cleanly, if we figure out what "cleanly"
    # means.  Set up wrapper for pkg-config.  Point a board-specific wrapper
    # at the generic wrapper script created by crossdev-wrapper.
    cmds+=(
      "ln -sf '/usr/bin/cross-pkg-config' \
        '/usr/bin/${FLAGS_toolchain}-pkg-config'"
    )
  fi

  wrapper="/usr/local/bin/cros_workon-${BOARD_VARIANT}"
  sudo_clobber "${wrapper}" <<EOF
#!/bin/bash
exec cros_workon --board ${BOARD_VARIANT} "\$@"
EOF
  wrappers+=( "${wrapper}" )

  cmds+=(
    "chmod a+rx ${wrappers[*]}"
    "chown root:root ${wrappers[*]}"
  )

  sudo_multi "${cmds[@]}"
}

install_toolchain() {
  "${GCLIENT_ROOT}/src/scripts/install_toolchain" --board="${BOARD_VARIANT}"
}

# Print contents of $BOARD_ETC/make.conf.board. This file sources make.conf
# from each overlay, and then calculates the prebuilt URLs based on the
# various binhost configuration files.
print_board_make_conf() {
  echo "# AUTO-GENERATED FILE. DO NOT EDIT."
  echo
  echo "# Source make.conf from each overlay."
  local boto_config=""
  for overlay in $BOARD_OVERLAY_LIST; do
    if [[ -f "$overlay/make.conf" ]]; then
      echo source "$overlay/make.conf"
    fi
    if [[ -f "$overlay/googlestorage_account.boto" ]]; then
      boto_config="$overlay/googlestorage_account.boto"
    fi
  done
  local chromeos_overlay="$SRC_ROOT/private-overlays/chromeos-overlay"
  if [[ -f "$chromeos_overlay/googlestorage_account.boto" ]]; then
    boto_config="$chromeos_overlay/googlestorage_account.boto"
  fi
  local gsutil_cmd='gsutil cp \"${URI}\" \"${DISTDIR}/${FILE}\"'
  echo
  echo FETCHCOMMAND_GS="\"bash -c 'BOTO_CONFIG=$boto_config $gsutil_cmd'\""
  echo RESUMECOMMAND_GS='"$FETCHCOMMAND_GS"'
  echo
  if [[ -n ${FLAGS_accept_licenses} ]]; then
    echo "ACCEPT_LICENSE='${FLAGS_accept_licenses}'"
    echo
  fi
}

print_binhost_config() {
  if [ "$1" == "--local" ]; then
    cros_generate_local_binhosts --board "${BOARD_VARIANT}"
    echo 'PORTAGE_BINHOST="$LOCAL_BINHOST"'
    return 0
  elif [ $# -ne 0 ]; then
    die "Invalid arguments for print_binhost_config: $@"
  fi

  local binhost_dir="$CHROMIUMOS_OVERLAY/chromeos/binhost/target"
  local preflight_binhost="$binhost_dir/$BOARD-PREFLIGHT_BINHOST.conf"
  local chrome_binhost_suffix="LATEST_RELEASE_CHROME_BINHOST.conf"
  local chrome_binhost="$binhost_dir/$BOARD-$chrome_binhost_suffix"
  if [[ $ARCH = x86 && ! -f "$preflight_binhost" ]]; then
    preflight_binhost="$binhost_dir/x86-generic-PREFLIGHT_BINHOST.conf"
  elif [[ $ARCH = amd64 && ! -f "$preflight_binhost" ]]; then
    preflight_binhost="$binhost_dir/amd64-generic-PREFLIGHT_BINHOST.conf"
  fi
  if [[ $ARCH = x86 && ! -f "$chrome_binhost" ]]; then
    chrome_binhost="$binhost_dir/x86-generic-$chrome_binhost_suffix"
  elif [[ $ARCH = amd64 && ! -f "$chrome_binhost" ]]; then
    chrome_binhost="$binhost_dir/amd64-generic-$chrome_binhost_suffix"
  fi

  local partner_overlay="$SRC_ROOT/private-overlays/chromeos-partner-overlay"
  local internal_binhost_dir="$partner_overlay/chromeos/binhost/target"
  local internal_binhost="$internal_binhost_dir/$BOARD-PREFLIGHT_BINHOST.conf"

  cat <<'EOF'
# FULL_BINHOST is populated by the full builders. It is listed first because it
# is the lowest priority binhost. It is better to download packages from the
# preflight binhost because they are fresher packages.
PORTAGE_BINHOST="$FULL_BINHOST"

EOF

  if [[ -f "$preflight_binhost" && -z "$IGNORE_PREFLIGHT_BINHOST" ]]; then
    cat <<'EOF'
# PREFLIGHT_BINHOST is populated by the preflight builders. If the same
# package is provided by both the preflight and full binhosts, the package is
# downloaded from the preflight binhost.
EOF
    echo source "$preflight_binhost"
    echo 'PORTAGE_BINHOST="$PORTAGE_BINHOST $PREFLIGHT_BINHOST"'
    echo
  fi

  if [[ -f "$internal_binhost"  && -z "$IGNORE_PREFLIGHT_BINHOST" ]]; then
    cat <<'EOF'
# The internal PREFLIGHT_BINHOST is populated by the internal preflight
# builders. It takes priority over the public preflight binhost.
EOF
    echo source "$internal_binhost"
    echo 'PORTAGE_BINHOST="$PORTAGE_BINHOST $PREFLIGHT_BINHOST"'
    echo
  fi

  if [[ -f "$chrome_binhost" ]]; then
    cat <<'EOF'
# LATEST_RELEASE_CHROME_BINHOST provides prebuilts for chromeos-chrome only.
EOF
    echo source "$chrome_binhost"
    echo 'PORTAGE_BINHOST="$PORTAGE_BINHOST $LATEST_RELEASE_CHROME_BINHOST"'
  fi
}

print_board_binhost_config() {
  if [ "${FLAGS_reuse_pkgs_from_local_boards}" -eq "${FLAGS_TRUE}" ]; then
    print_binhost_config --local
  else
    print_binhost_config
  fi
}

# Parse command line flags
FLAGS "$@" || exit 1
eval set -- "${FLAGS_ARGV}"

# Only now can we die on error.  shflags functions leak non-zero error codes,
# so will die prematurely if 'switch_to_strict_mode' is specified before now.
switch_to_strict_mode

if [ -z "$FLAGS_board" ] ; then
  error "--board required."
  exit 1
fi

get_board_and_variant $FLAGS_board $FLAGS_variant

# Before we can run any tools, we need to update chroot
UPDATE_ARGS="--toolchain_boards=${BOARD}"
if [ "${FLAGS_fast}" -eq "${FLAGS_TRUE}" ]; then
  UPDATE_ARGS+=" --fast"
else
  UPDATE_ARGS+=" --nofast"
fi
if [ "${FLAGS_usepkg}" -eq "${FLAGS_TRUE}" ]; then
  UPDATE_ARGS+=" --usepkg"
else
  UPDATE_ARGS+=" --nousepkg"
fi
if [[ "${FLAGS_jobs}" -ne -1 ]]; then
  UPDATE_ARGS+=" --jobs=${FLAGS_jobs}"
fi
if [ "${FLAGS_skip_toolchain_update}" -eq "${FLAGS_TRUE}" ]; then
  UPDATE_ARGS+=" --skip_toolchain_update"
fi
if [ "${FLAGS_skip_chroot_upgrade}" -eq "${FLAGS_FALSE}" ] ; then
  "${SRC_ROOT}/scripts"/update_chroot ${UPDATE_ARGS}
fi

#
# Fetch the toolchain from the board overlay.
#
all_toolchains=( $(get_all_board_toolchains "${BOARD}") )
: ${FLAGS_toolchain:=${all_toolchains[0]}}

if [ -z "${FLAGS_toolchain}" ]; then
  error "No toolchain specified in board overlay or on command line."
  exit 1
fi

ARCH=$(get_board_arch "${BOARD}") || exit 1

case "$BOARD" in
  *-host)
    if [[ $FLAGS_usepkg -eq $FLAGS_TRUE ]]; then
      die_notrace "host boards only support --nousepkg"
    fi
    HOST_BOARD=true
    ;;
  *)
    HOST_BOARD=false
esac
# Locations we will need
BOARD_ROOT="/build/${BOARD_VARIANT}"
CROSSDEV_OVERLAY="/usr/local/portage/crossdev"
CHROMIUMOS_OVERLAY="/usr/local/portage/chromiumos"
CHROMIUMOS_CONFIG="${CHROMIUMOS_OVERLAY}/chromeos/config"
CHROMIUMOS_PROFILES="${CHROMIUMOS_OVERLAY}/profiles"
BOARD_ETC="${BOARD_ROOT}/etc"
BOARD_SETUP="${BOARD_ETC}/make.conf.board_setup"
BOARD_PROFILE="${BOARD_ETC}/portage/profile"

#
# Construct board overlay list.
#
BOARD_OVERLAY_LIST=$(cros_list_overlays \
  --board "$BOARD_VARIANT" \
  --board_overlay "$FLAGS_board_overlay")

eval $(portageq envvar -v CHOST PKGDIR)

if [ -d "${BOARD_ROOT}" ]; then
  if [[ ${FLAGS_force} -eq ${FLAGS_TRUE} ]]; then
    echo "--force set.  Re-creating ${BOARD_ROOT}..."
    # Removal takes long. Make it asynchronous.
    TEMP_DIR=`mktemp -d`
    sudo mv "${BOARD_ROOT}" "${TEMP_DIR}"
    sudo rm -rf "${TEMP_DIR}" &
  elif [[ ${FLAGS_regen_configs} -eq ${FLAGS_FALSE} ]]; then
    if ! ${HOST_BOARD}; then
      { print_board_make_conf; print_board_binhost_config; } | \
        sudo_clobber $BOARD_ETC/make.conf.board
    fi
    if [[ ${FLAGS_quiet} -eq ${FLAGS_FALSE} ]]; then
      warn "Board output directory '$BOARD_ROOT' already exists."
      warn "Not setting up board root. "
      warn "Use --force to clobber the board root and start again."
    fi
    # Update the users libc in their board if needed.
    install_toolchain
    exit 0
  fi
else
  # Regenerating configs w/out a board root doesn't make sense.
  FLAGS_regen_configs=${FLAGS_FALSE}
fi

cmds=(
  "mkdir -p '${BOARD_ROOT}' '${BOARD_ETC}' '${BOARD_PROFILE}' /usr/local/bin"
)

# Setup the make.confs. We use the following:
#    make.conf             <- Overall target make.conf [arm, x86, etc. version]
#    make.conf.board_setup <- Declares CHOST, ROOT, etc.
#    make.conf.common      <- Common settings across all targets
#    make.conf.board       <- Optional board-supplied make.conf
if ${HOST_BOARD}; then
  cmds+=(
    "ln -sf '${CHROMIUMOS_CONFIG}/make.conf.${BOARD}' \
      '${BOARD_ETC}/make.conf'"
    "cp -f '/etc/make.conf.host_setup' '${BOARD_ETC}/'"

    # Setting up symlinks for bootstrapping multilib.
    # See http://crosbug.com/14498
    "mkdir -p '${BOARD_ROOT}'{/usr,}/lib64"
    "ln -sfT lib64 '${BOARD_ROOT}/lib'"
    "ln -sfT lib64 '${BOARD_ROOT}/usr/lib'"

    # Copying some files for bootstrapping empty chroot.
    # See http://crosbug.com/14499
    "mkdir -p '${BOARD_ETC}'/{init.d,xml}"
    "cp /etc/xml/catalog '${BOARD_ETC}'/xml/"
    "cp /etc/init.d/functions.sh '${BOARD_ETC}'/init.d/"
  )
fi
cmds+=(
  "ln -sf '${CHROMIUMOS_CONFIG}/make.conf.common-target' \
    '${BOARD_ETC}/make.conf.common'"
)
sudo_multi "${cmds[@]}"

SAVED_VERSION=
if [[ ${FLAGS_regen_configs} -eq ${FLAGS_TRUE} ]]; then
  SAVED_VERSION=$(grep -s ^LIBC_VERSION= ${BOARD_SETUP} || true)
fi

sudo_clobber "${BOARD_SETUP}" <<EOF
# Created by setup_board
CHOST="${FLAGS_toolchain}"
ROOT="${BOARD_ROOT}/"
BOARD_OVERLAY="${BOARD_OVERLAY_LIST}"
MAKEOPTS="-j${NUM_JOBS}"
PKG_CONFIG="pkg-config-${BOARD_VARIANT}"
BOARD_USE="${BOARD_VARIANT}"
${SAVED_VERSION}
EOF

if ! ${HOST_BOARD}; then
  sudo ln -sf "${CHROMIUMOS_CONFIG}/make.conf.${ARCH}-target" \
    "${BOARD_ETC}/make.conf"
  print_board_make_conf | sudo_clobber $BOARD_ETC/make.conf.board

  # We install the toolchain related bits after the BOARD_ROOT, BOARD_PROFILE
  # and BOARD_ETC directories have been created.
  if [[ ${FLAGS_regen_configs} -eq ${FLAGS_FALSE} ]]; then
    install_toolchain
  fi
fi
# Setup make.globals and the profile.
cmds=(
  "touch /etc/make.conf.user"
  "ln -sf /etc/make.globals '${BOARD_ROOT}/etc/make.globals'"
  "ln -sf /etc/make.conf.user '${BOARD_ROOT}/etc/make.conf.user'"
  "mkdir -p '${BOARD_ROOT}/etc/portage/hooks'"
)
for d in "${SCRIPTS_DIR}"/hooks/*; do
  cmds+=( "ln -sfT '${d}' '${BOARD_ROOT}/etc/portage/hooks/${d##*/}'" )
done
sudo_multi "${cmds[@]}"

# Select the profile to build based on the board and profile passed to
# setup_board.  The developer can later change profiles by running
# cros_choose_profile manually.
if ! cros_choose_profile \
       --board "${FLAGS_board}" \
       --board_overlay "${FLAGS_board_overlay}" \
       --variant "${FLAGS_variant}" \
       --profile "${FLAGS_profile}"; then
  error "Selecting profile failed, removing incomplete board directory!"
  sudo rm -rf "${BOARD_ROOT}"
  exit 1
fi

generate_all_wrappers

if ${HOST_BOARD}; then
  EMERGE_CMD="emerge"
  if [[ $FLAGS_fast -eq $FLAGS_TRUE ]]; then
    EMERGE_CMD="${GCLIENT_ROOT}/chromite/bin/parallel_emerge"
  fi
  PACKAGES="system hard-host-depends world"

  # First, rebuild all packages from scratch. This is needed to make sure
  # we rebuild all chroot packages.
  sudo -E $EMERGE_CMD --emptytree --with-bdeps=y $PACKAGES
  sudo eclean -d packages

  # Next, install our rebuilt packages into our separate root.
  HOST_FLAGS="--root=$BOARD_ROOT --update --verbose --deep --root-deps"
  HOST_FLAGS+=" --with-bdeps=y --newuse --jobs=$NUM_JOBS --usepkgonly"
  sudo -E $EMERGE_CMD $HOST_FLAGS --select=y $PACKAGES
  sudo cp -a "${PKGDIR}" $BOARD_ROOT/packages

  # Install cross-compilers.
  COMPILERS=$(equery l cross-*/* --format='=$cpv')
  sudo -E $EMERGE_CMD $HOST_FLAGS --select=n $COMPILERS

  # Setup needed symlinks for cross-compilers.
  sudo mkdir -p $BOARD_ROOT/usr/local/portage
  sudo cp -a ${CROSSDEV_OVERLAY} $BOARD_ROOT/usr/local/portage

  # Setup crossdev configuration for categories.
  sudo cp -a /etc/portage/* $BOARD_ROOT/etc/portage

  # The new chroot should have gcc for each target. Make sure that
  # the latest one is correctly selected. Ignore cat errors as not
  # all overlays will have a toolchain.conf.
  ALL_TARGETS=$(cros_setup_toolchains --show-board-cfg=sdk)
  for target in ${ALL_TARGETS//,/ }; do
    libc=$(eval $(crossdev --show-target-cfg "${target}"); echo ${libc_pn})
    if [[ ${libc} == "glibc" ]] ; then
      # Install needed glibc tarball.
      cross_target_path=/var/lib/portage/pkgs/cross-${target}
      if [[ -e "$cross_target_path" ]] ; then
          sudo mkdir -p ${BOARD_ROOT}${cross_target_path}
          sudo cp -a ${cross_target_path}/glibc-* \
            ${BOARD_ROOT}${cross_target_path}
      fi
    fi

    CURRENT_GCC="$(gcc-config -c ${target})"
    sudo ROOT=${BOARD_ROOT} gcc-config ${CURRENT_GCC}
    CURRENT_BINUTILS="$(binutils-config -c ${target})"
    sudo ROOT=${BOARD_ROOT} binutils-config ${CURRENT_BINUTILS}
  done

  # Now cleanup paths referencing the ROOT from the *.la files.
  sudo find $BOARD_ROOT -type f -name '*.la' | xargs sudo \
      sed -i -e "s|$BOARD_ROOT/|/|g"
else
  # Now that the portageq-* wrappers have been created, we can add the binhost
  # config to make.conf.board.
  print_board_binhost_config | sudo_append $BOARD_ETC/make.conf.board

  if [[ ${FLAGS_regen_configs} -eq ${FLAGS_FALSE} ]]; then
    # Emerge the kernel headers into the board build root.  Use rdeps to
    # avoid pulling any spurious DEPEND things in that we don't care about.
    KERNEL_EMERGE_FLAGS="--select --quiet --root-deps=rdeps"
    if [[ "${FLAGS_usepkg}" -eq "${FLAGS_TRUE}" ||
          "${FLAGS_reuse_pkgs_from_local_boards}" -eq "${FLAGS_TRUE}" ]]; then
      KERNEL_EMERGE_FLAGS+=" --getbinpkg --usepkg"
    fi

    sudo -E "${EMERGE_WRAPPER}" ${KERNEL_EMERGE_FLAGS} \
      sys-kernel/linux-headers

    unset KERNEL_EMERGE_FLAGS
  fi
fi

if [ $FLAGS_default -eq $FLAGS_TRUE ] ; then
  echo $BOARD_VARIANT > "$GCLIENT_ROOT/src/scripts/.default_board"
fi

command_completed
echo "Done!"
echo "The SYSROOT is: ${BOARD_ROOT}"

# NOTE: Printing the working-on ebuilds does not only serve the informative
# purpose. It also causes the ${BOARD_ROOT}/etc/portage/package.* files to be
# regenerated.
WORKING_ON=$(cros_workon --board=${FLAGS_board} list)
if [ -n "${WORKING_ON}" ]; then
  echo
  echo "Currently working on the following ebuilds for this board:"
  echo "${WORKING_ON}"
fi