From cb782248d4a37725c07ff61a3d3e2152de1c74c9 Mon Sep 17 00:00:00 2001 From: Brian Harring Date: Tue, 13 Dec 2011 19:48:44 -0800 Subject: [PATCH] transfer setup_board and build_parents into crosutils Due to historical reasons, these scripts have existed at chromiumos-overlay/chrome/scripts, but have belonged in src/scripts (previously accessed by symlinks here pointing to their locations). Transfer them in, rather than relying on symlinks; this gets us atomic commits w/in crosutils in the process. Scripts were imported as of 604e3722af59164bc97aa5dcd1407e36495c96d7 BUG=chromium-os:24111 TEST=cbuildbot x86-generic-full Change-Id: I689f7e05a25d427e24372f206bdb0779bf857820 Reviewed-on: https://gerrit.chromium.org/gerrit/12893 Reviewed-by: David James Commit-Ready: Brian Harring Tested-by: Brian Harring --- build_packages | 222 ++++++++++++++- setup_board | 719 ++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 939 insertions(+), 2 deletions(-) mode change 120000 => 100755 build_packages mode change 120000 => 100755 setup_board diff --git a/build_packages b/build_packages deleted file mode 120000 index 94778a2726..0000000000 --- a/build_packages +++ /dev/null @@ -1 +0,0 @@ -../third_party/chromiumos-overlay/chromeos/scripts/build_packages \ No newline at end of file diff --git a/build_packages b/build_packages new file mode 100755 index 0000000000..edde767010 --- /dev/null +++ b/build_packages @@ -0,0 +1,221 @@ +#!/bin/bash + +# Copyright (c) 2011 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# Script to build the set of binary packages needed by Chrome OS. It will +# cross compile all of the packages into the given targets root and build +# binary packages as a side-effect. The output packages will be picked up +# by the build_image script to put together a bootable Chrome OS image. + +# Load common CrOS utilities. Inside the chroot this file is installed in +# /usr/lib/crosutils. Outside the chroot we find it relative to the script's +# location. +find_common_sh() { + local common_paths=(/usr/lib/crosutils $(dirname "$0")) + local path + + SCRIPT_ROOT= + for path in "${common_paths[@]}"; do + local common="${path}/common.sh" + if ([ -r "${common}" ] && . "${common}" && [ -d "${SCRIPTS_DIR}" ]); then + SCRIPT_ROOT=${path} + break + fi + done +} + +find_common_sh +. "${SCRIPT_ROOT}/common.sh" || ! echo "Unable to load common.sh" || exit 1 + +# Script must run inside the chroot +restart_in_chroot_if_needed "$@" + +get_default_board + +# Flags +DEFINE_string board "${DEFAULT_BOARD}" \ + "The board to build packages for." +# Deprecate chrome* options below once we have cbuild not passing these options +DEFINE_boolean chromefromsource "${FLAGS_FALSE}" \ + "Deprecated" +DEFINE_string chromebuild "" \ + "Deprecated" +DEFINE_string chromebase "" \ + "Deprecated" +DEFINE_boolean usepkg "${FLAGS_TRUE}" \ + "Use binary packages to bootstrap when possible." +DEFINE_boolean withdev "${FLAGS_TRUE}" \ + "Build useful developer friendly utilities." +DEFINE_boolean withautotest "${FLAGS_TRUE}" \ + "Build autotest client code." +DEFINE_integer jobs -1 \ + "How many packages to build in parallel at maximum." +DEFINE_integer retries -1 \ + "On build failure, the number of times to retry." +DEFINE_boolean withtest "${FLAGS_TRUE}" \ + "Build packages required for testing." +DEFINE_boolean withfactory "${FLAGS_TRUE}" \ + "Build factory installer." +DEFINE_boolean fast "${DEFAULT_FAST}" \ + "Call many emerges in parallel." +DEFINE_boolean norebuild "${FLAGS_FALSE}" \ + "Don't automatically rebuild dependencies." +DEFINE_boolean showoutput "${FLAGS_FALSE}" \ + "Show all output from parallel_emerge." +DEFINE_boolean noworkon "${FLAGS_FALSE}" \ + "Don't force-build workon packages." +DEFINE_boolean withdebug "${FLAGS_TRUE}" \ + "Build debug versions of Chromium-OS-specific packages." +DEFINE_boolean oldchromebinary "${FLAGS_FALSE}" \ + "Use the last prebuilt binary for Chrome produced by the buildbot." +DEFINE_boolean skip_toolchain_update "${FLAGS_FALSE}" \ + "Don't update toolchain automatically." + + +# Parse command line +FLAGS_HELP="usage: $0 [flags]" +FLAGS "$@" || exit 1 +eval set -- "${FLAGS_ARGV}" +check_flags_only_and_allow_null_arg "$@" && set -- + +# Die on any errors. +set -e + +# Right now build_packages has to be run from scripts/ +. ${SRC_ROOT}/third_party/chromiumos-overlay/chromeos/config/chromeos_version.sh + +if [[ -z "${FLAGS_board}" ]]; then + echo "Error: --board is required." + exit 1 +fi + +EMERGE_FLAGS="--backtrack=30" + +EMERGE_CMD="emerge" +EMERGE_BOARD_CMD="emerge-${FLAGS_board}" +if [[ "${FLAGS_fast}" -eq "${FLAGS_TRUE}" ]]; then + EMERGE_CMD="${GCLIENT_ROOT}/chromite/bin/parallel_emerge" + EMERGE_BOARD_CMD="${EMERGE_CMD} --board=${FLAGS_board}" +fi +if [[ -n "${EXTRA_BOARD_FLAGS}" ]]; then + EMERGE_BOARD_CMD="${EMERGE_BOARD_CMD} ${EXTRA_BOARD_FLAGS}" +fi + +if [[ "${FLAGS_usepkg}" -eq "${FLAGS_TRUE}" ]]; then + # Use binary packages. Include all build-time dependencies, + # so as to avoid unnecessary differences between source + # and binary builds. + EMERGE_FLAGS="${EMERGE_FLAGS} --getbinpkg --usepkg --with-bdeps y" +fi + +if [[ "${FLAGS_jobs}" -ne -1 ]]; then + EMERGE_JOBS="--jobs=${FLAGS_jobs}" + if [[ "${FLAGS_retries}" -eq -1 ]]; then + # The jobs flag can be flaky. Retry once by default, + # without the jobs flag. + FLAGS_retries=1 + fi +fi + +if [[ "${FLAGS_withdebug}" -eq "${FLAGS_FALSE}" ]]; then + export USE="${USE} -cros-debug" +fi + +${EMERGE_CMD} --info + +# Before we can run any tools, we need to update chroot or setup_board. +UPDATE_ARGS="" +if [ "${FLAGS_fast}" -eq "${FLAGS_TRUE}" ]; then + UPDATE_ARGS+=" --fast" +else + UPDATE_ARGS+=" --nofast" +fi +if [ "${FLAGS_usepkg}" -eq "${FLAGS_TRUE}" ]; then + UPDATE_ARGS+=" --usepkg" +else + UPDATE_ARGS+=" --nousepkg" +fi +if [ "${FLAGS_skip_toolchain_update}" -eq "${FLAGS_TRUE}" ]; then + UPDATE_ARGS+=" --skip_toolchain_update" +fi +${SCRIPTS_DIR}/setup_board --quiet --board=${FLAGS_board} ${UPDATE_ARGS} + +if [ "${FLAGS_noworkon}" -eq "${FLAGS_FALSE}" ]; then + # Always build cros-workon packages + CROS_WORKON_PKGS=$(cros_workon --board="${FLAGS_board}" list) +fi + +# TODO(anush): Make chrome a fake cros-workon package. +if [[ -n "${CHROME_ORIGIN}" ]]; then + CROS_WORKON_PKGS="${CROS_WORKON_PKGS} chromeos-base/chromeos-chrome" +fi + +PACKAGES="chromeos-base/chromeos" +if [[ "${FLAGS_withdev}" -eq "${FLAGS_TRUE}" ]]; then + PACKAGES="${PACKAGES} chromeos-base/chromeos-dev" +fi +if [[ "${FLAGS_withfactory}" -eq "${FLAGS_TRUE}" ]]; then + PACKAGES="${PACKAGES} chromeos-base/chromeos-factoryinstall" + PACKAGES="${PACKAGES} chromeos-base/factorytest-init" + PACKAGES="${PACKAGES} chromeos-base/chromeos-hwid" +fi +if [[ "${FLAGS_withtest}" -eq "${FLAGS_TRUE}" ]]; then + PACKAGES="${PACKAGES} chromeos-base/chromeos-test" +fi +if [[ "${FLAGS_withautotest}" -eq "${FLAGS_TRUE}" ]]; then + PACKAGES="${PACKAGES} chromeos-base/autotest-all" +fi + +# Verify that all packages can be emerged from scratch, without any +# backtracking. Only print the output if this step fails. +if ! OUTPUT=$(emerge-${FLAGS_board} -pe --backtrack=0 ${PACKAGES} 2>&1); then + printf "%s\n" "${OUTPUT}" + die "emerge detected broken ebuilds. See error message above." +fi + +if [[ "${FLAGS_fast}" -eq "${FLAGS_TRUE}" ]]; then + # These flags are specific to parallel_emerge + for pkg in ${CROS_WORKON_PKGS}; do + EMERGE_FLAGS="${EMERGE_FLAGS} --workon=${pkg}" + done + if [[ "${FLAGS_norebuild}" -eq "${FLAGS_FALSE}" ]]; then + EMERGE_FLAGS="${EMERGE_FLAGS} --rebuild" + fi + if [[ "${FLAGS_showoutput}" -eq "${FLAGS_TRUE}" ]]; then + EMERGE_FLAGS="${EMERGE_FLAGS} --show-output" + fi + if [[ "${FLAGS_oldchromebinary}" -eq "${FLAGS_TRUE}" ]]; then + EMERGE_FLAGS="${EMERGE_FLAGS} --force-remote-binary=chromeos-chrome" + EMERGE_FLAGS="${EMERGE_FLAGS} --force-remote-binary=libcros" + fi + eretry sudo -E ${EMERGE_BOARD_CMD} -uDNv ${EMERGE_FLAGS} ${PACKAGES} +else + echo "Building ${PACKAGES} ${CROS_WORKON_PKGS}" + if [[ "${FLAGS_usepkg}" -eq "${FLAGS_TRUE}" ]]; then + # Merge all packages, including binary versions of packages. + eretry ${EMERGE_BOARD_CMD} -uDNv ${EMERGE_FLAGS} ${PACKAGES} + if [[ -n "${CROS_WORKON_PKGS}" ]]; then + # Re-merge cros-workon packages, this time from source. This is + # inefficient, but regular emerge doesn't support any way of specifying + # that you want specific packages to be merged from source only. + eretry ${EMERGE_BOARD_CMD} --oneshot ${CROS_WORKON_PKGS} + fi + else + # Re-merge cros_workon packages, updating dependencies if necessary. + # --selective=n: Always re-merge cros_workon packages. + # -1: Don't add cros_workon packages to world file. + if [[ -n "${CROS_WORKON_PKGS}" ]]; then + eretry ${EMERGE_BOARD_CMD} -uDNv1 --selective=n ${CROS_WORKON_PKGS} + fi + + # Update the remainder of the packages. + eretry ${EMERGE_BOARD_CMD} -uDNv ${EMERGE_FLAGS} ${PACKAGES} + fi +fi + + +echo "Builds complete" +print_time_elapsed +echo "Done" diff --git a/setup_board b/setup_board deleted file mode 120000 index 28c83a346d..0000000000 --- a/setup_board +++ /dev/null @@ -1 +0,0 @@ -../third_party/chromiumos-overlay/chromeos/scripts/setup_board \ No newline at end of file diff --git a/setup_board b/setup_board new file mode 100755 index 0000000000..0320f3d2fd --- /dev/null +++ b/setup_board @@ -0,0 +1,718 @@ +#!/bin/bash + +# Copyright (c) 2011 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# This script sets up a the sysroot for a particular target board. + +# Load common CrOS utilities. Inside the chroot this file is installed in +# /usr/lib/crosutils. Outside the chroot we find it relative to the script's +# location. +find_common_sh() { + local common_paths=(/usr/lib/crosutils $(dirname "$0")) + local path + + SCRIPT_ROOT= + for path in "${common_paths[@]}"; do + local common="${path}/common.sh" + if ([ -r "${common}" ] && . "${common}" && [ -d "${SCRIPTS_DIR}" ]); then + SCRIPT_ROOT=${path} + break + fi + done +} + +find_common_sh +. "${SCRIPT_ROOT}/common.sh" || ! echo "Unable to load common.sh" || exit 1 + +# Script must run inside the chroot +restart_in_chroot_if_needed "$@" + +get_default_board + +# Flags +DEFINE_string board "$DEFAULT_BOARD" \ + "The name of the board to set up." +DEFINE_string build_root "/build" \ + "The root location for board sysroots." +DEFINE_string board_overlay "" \ + "Location of the board overlay." +DEFINE_string variant "" \ + "Board variant." +DEFINE_string toolchain "" \ + "Toolchain. For example: i686-pc-linux-gnu, armv7a-softfloat-linux-gnueabi" +DEFINE_boolean usepkg $FLAGS_TRUE \ + "Use binary packages to bootstrap." +DEFINE_boolean force $FLAGS_FALSE \ + "Force re-creating board root." +DEFINE_string binutils_version "2.21-r3" \ + "Version of binutils to use." +DEFINE_string gcc_version "4.6.0-r10" \ + "Version of gcc to use." +DEFINE_string libc_version "2.11.1-r3" \ + "Version of libc to use." +DEFINE_string kernel_version "2.6.38" \ + "Version of kernel headers to use." +DEFINE_boolean latest_toolchain $FLAGS_FALSE \ + "Use the latest ebuild for all toolchain packages (gcc, binutils, libc, \ +kernel). This overrides the other toolchain version options." +DEFINE_boolean default $FLAGS_FALSE \ + "Set board to the default board in your chroot" +DEFINE_boolean fast ${DEFAULT_FAST} "Call many emerges in parallel" +DEFINE_string profile "" \ + "The portage configuration profile to use. Profile must be located in overlay-board/profiles" +DEFINE_boolean quiet $FLAGS_FALSE \ + "Don't print warnings when board already exists." +DEFINE_boolean skip_toolchain_update $FLAGS_FALSE \ + "Don't update toolchain automatically." + +# builds wrappers like equery-arm-generic. +# args: +# $1: command to wrap +# rest: extra arguments to pass to the command +generate_wrapper() { + local command="${1}" + shift + local extra_args="$@" + + + local target="/usr/local/bin/${command}-${BOARD_VARIANT}" + sudo_clobber "${target}" < /dev/null 2>&1 +# Created by setup_board +CHOST="${FLAGS_toolchain}" +ROOT="${BOARD_ROOT}/" +BOARD_OVERLAY="${BOARD_OVERLAY_LIST}" +MAKEOPTS="-j${NUM_JOBS}" +PKG_CONFIG="pkg-config-${BOARD_VARIANT}" +BOARD_USE="${BOARD_VARIANT}" +EOF +if ! ${HOST_BOARD}; then + sudo ln -sf "${CHROMIUMOS_CONFIG}/make.conf.${ARCH}-target" \ + "${BOARD_ETC}/make.conf" + print_board_make_conf | sudo_clobber $BOARD_ETC/make.conf.board + print_board_binhost_config | sudo_append $BOARD_ETC/make.conf.board + + # We install the toolchain related bits after the BOARD_ROOT, BOARD_PROFILE + # and BOARD_ETC directories have been created. + install_toolchain_in_board +fi +# Setup make.globals and the profile. +sudo touch /etc/make.conf.user +sudo ln -sf /etc/make.globals "${BOARD_ROOT}/etc/make.globals" +sudo ln -sf /etc/make.conf.user "${BOARD_ROOT}/etc/make.conf.user" + +# Select the profile to build based on the board and profile passed to +# setup_board. The developer can later change profiles by running +# cros_choose_profile manually. +if ! cros_choose_profile \ + --board "${FLAGS_board}" \ + --build_root "${FLAGS_build_root}" \ + --board_overlay "${FLAGS_board_overlay}" \ + --variant "${FLAGS_variant}" \ + --profile "${FLAGS_profile}"; then + error "Selecting profile failed, removing incomplete board directory!" + sudo rm -rf "${BOARD_ROOT}" + exit 1 +fi + + +for wrapper in 'emerge --root-deps' ebuild eclean equery portageq; do + generate_wrapper $wrapper +done + +generate_pkgconfig_wrapper + +CROS_WORKON_WRAPPER="/usr/local/bin/cros_workon-${BOARD_VARIANT}" +cat < /dev/null 2>&1 +#!/bin/bash +exec cros_workon --board ${BOARD_VARIANT} "\$@" +EOF +sudo chmod +x "$CROS_WORKON_WRAPPER" + +# +# Emerge the kernel headers into the board build root. +# +EMERGE_FLAGS="" +if [[ $FLAGS_usepkg -eq $FLAGS_TRUE ]]; then + EMERGE_FLAGS="${EMERGE_FLAGS} --getbinpkg --usepkg" +fi + +if ${HOST_BOARD}; then + EMERGE_CMD="emerge" + if [[ $FLAGS_fast -eq $FLAGS_TRUE ]]; then + EMERGE_CMD="${GCLIENT_ROOT}/chromite/bin/parallel_emerge" + fi + PACKAGES="system hard-host-depends world" + + # First, rebuild all packages from scratch. This is needed to make sure + # we rebuild all chroot packages. + sudo -E $EMERGE_CMD --emptytree --with-bdeps=y $PACKAGES + sudo eclean -d packages + + # Next, install our rebuilt packages into our separate root. + HOST_FLAGS="--root=$BOARD_ROOT --update --verbose --deep --root-deps" + HOST_FLAGS+=" --with-bdeps=y --newuse --jobs=$NUM_JOBS --usepkgonly" + sudo -E $EMERGE_CMD $HOST_FLAGS $PACKAGES + sudo cp -a "${PKGDIR}" $BOARD_ROOT/packages + + # Install cross-compilers. + COMPILERS=$(equery l cross-*/* --format='=$cpv') + sudo -E $EMERGE_CMD $HOST_FLAGS --oneshot $COMPILERS + + # Setup needed symlinks for cross-compilers. + sudo mkdir -p $BOARD_ROOT/usr/local/portage + sudo cp -a /usr/local/portage/crossdev $BOARD_ROOT/usr/local/portage + + # Setup crossdev configuration for categories. + sudo cp -a /etc/portage/* $BOARD_ROOT/etc/portage + + # The new chroot should have gcc for each target. Make sure that + # the latest one is correctly selected. Ignore cat errors as not + # all overlays will have a toolchain.conf. + ALL_OVERLAYS=$(cros_overlay_list --all_boards) + TARGETS=$(cat $(printf '%s/toolchain.conf ' ${ALL_OVERLAYS}) 2>/dev/null | \ + sort -u) + for target in ${TARGETS}; do + # Install needed glibc tarball. + cross_target_path=/var/lib/portage/pkgs/cross-$target + if [[ -e "$cross_target_path" ]] ; then + sudo mkdir -p ${BOARD_ROOT}$cross_target_path + sudo cp -a ${cross_target_path}/glibc-* ${BOARD_ROOT}$cross_target_path + fi + + CURRENT_GCC="$(gcc-config -c ${target})" + sudo ROOT=${BOARD_ROOT} gcc-config ${CURRENT_GCC} + CURRENT_BINUTILS="$(binutils-config -c ${target})" + sudo ROOT=${BOARD_ROOT} binutils-config ${CURRENT_BINUTILS} + done + + # Now cleanup paths referencing the ROOT from the *.la files. + sudo find $BOARD_ROOT -type f -name '*.la' | xargs sudo \ + sed -i -e "s|$BOARD_ROOT/|/|g" +else + sudo -E "${EMERGE_WRAPPER}" ${EMERGE_FLAGS} chromeos-base/kernel-headers +fi + +if [ $FLAGS_default -eq $FLAGS_TRUE ] ; then + echo $BOARD_VARIANT > "$GCLIENT_ROOT/src/scripts/.default_board" +fi + +echo "Done!" +echo "The SYSROOT is: ${BOARD_ROOT}" + +# NOTE: Printing the working-on ebuilds does not only serve the informative +# purpose. It also causes the ${BOARD_ROOT}/etc/portage/package.* files to be +# regenerated. +WORKING_ON=$(cros_workon --board=${FLAGS_board} list) +if [ -n "${WORKING_ON}" ]; then + echo + echo "Currently working on the following ebuilds for this board:" + echo "${WORKING_ON}" +fi