From c503b0248be832e522b37f271ad7ba3dd13b4c7d Mon Sep 17 00:00:00 2001
From: Michael Marineau <mike@marineau.org>
Date: Tue, 22 Oct 2013 22:19:56 -0700
Subject: [PATCH 1/5] add(build_toolchains): New command to build cross and
 native target toolchains.

This replaces the cross-toolchain compile step in bootstrap_sdk and adds the
ability to build native toolchains using the cross toolchain. This is just
the first step towards actually providing the native toolchain in a container.
---
 build_library/catalyst_toolchains.sh | 136 +++++++++++++++++++++++++++
 build_library/toolchain_util.sh      |  50 ++++++++++
 build_toolchains                     |  54 +++++++++++
 3 files changed, 240 insertions(+)
 create mode 100644 build_library/catalyst_toolchains.sh
 create mode 100644 build_library/toolchain_util.sh
 create mode 100755 build_toolchains

diff --git a/build_library/catalyst_toolchains.sh b/build_library/catalyst_toolchains.sh
new file mode 100644
index 0000000000..28f15e8079
--- /dev/null
+++ b/build_library/catalyst_toolchains.sh
@@ -0,0 +1,136 @@
+#!/bin/bash
+
+set -e
+source /tmp/chroot-functions.sh
+source /tmp/toolchain_util.sh
+
+# A note on packages:
+# The default PKGDIR is /usr/portage/packages
+# To make sure things are uploaded to the correct places we split things up:
+# crossdev build packages use ${PKGDIR}/crossdev (uploaded to SDK location)
+# build deps in crossdev's sysroot use ${PKGDIR}/cross/${CHOST} (no upload)
+# native toolchains use ${PKGDIR}/native/${BOARD} (uploaded to board location)
+
+get_dependency_list() {
+    local ROOT="$1"
+    local IFS=$'| \t\n'
+    shift
+
+    PORTAGE_CONFIGROOT="$ROOT" SYSROOT="$ROOT" \
+        ROOT="$ROOT" emerge ${clst_myemergeopts} \
+        --pretend --with-bdeps=y --onlydeps --quiet \
+        "$@" | sed -e 's/.*\] \([^ :]*\).*/=\1/' |
+        egrep -v "(=$(echo "$*")-[0-9])"
+}
+
+configure_portage() {
+    local pkg_path="$1"
+    mkdir -p "${ROOT}/etc/portage"
+    rm -f "${ROOT}/etc/make.profile" "${ROOT}/etc/portage/make.profile"
+    # eselect will report "!!! Warning: Strange path." but that's OK
+    eselect profile set --force coreos:coreos/amd64/generic 
+    cat >"${ROOT}/etc/portage/make.conf" <<EOF
+CHOST=${CHOST}
+CBUILD=$(portageq envvar CBUILD)
+HOSTCC=\${CBUILD}-gcc
+ROOT="${ROOT}"
+SYSROOT="${SYSROOT}"
+PKG_CONFIG_PATH="\${SYSROOT}/usr/lib/pkgconfig/"
+PORTDIR="$(portageq envvar PORTDIR)"
+DISTDIR="\${PORTDIR}/distfiles"
+PKGDIR="\${PORTDIR}/packages/${pkg_path}"
+PORTDIR_OVERLAY="$(portageq envvar PORTDIR_OVERLAY)"
+EOF
+}
+
+build_cross_toolchain() {
+    local cross_chost="$1"
+    local cross_pkgs=( "${TOOLCHAIN_PKGS[@]/*\//cross-${cross_chost}/}" )
+    local PORTDIR="$(portageq envvar PORTDIR)"
+    local PKGDIR="${PORTDIR}/packages/crossdev"
+
+    PKGDIR="${PKGDIR}" crossdev \
+        --ov-output "/tmp/crossdev" --stable \
+        --portage "${clst_myemergeopts}" \
+        --init-target --target "${cross_chost}"
+
+    # If PKGCACHE is enabled check to see if binary packages are available.
+    # If so then don't perform a full bootstrap and just call emerge instead.
+    if [[ -n "${clst_PKGCACHE}" ]] && \
+        PKGDIR="${PKGDIR}" emerge ${clst_myemergeopts} \
+            --usepkgonly --binpkg-respect-use=y \
+            --pretend "${cross_pkgs[@]}" &>/dev/null
+    then
+        PKGDIR="${PKGDIR}" run_merge -u "${cross_pkgs[@]}"
+    else
+        PKGDIR="${PKGDIR}" crossdev \
+            --ov-output "/tmp/crossdev" --stable \
+            --env PKGDIR="${PORTDIR}/packages/crossdev" \
+            --portage "${clst_myemergeopts}" \
+            --stage4 --target "${cross_chost}"
+    fi
+
+    # Setup ccache for our shiny new toolchain
+    ccache-config --install-links "${cross_chost}"
+}
+
+configure_cross_root() {
+    local cross_chost="$1"
+    local ROOT="/usr/${cross_chost}"
+
+    CHOST="${cross_chost}" ROOT="$ROOT" SYSROOT="$ROOT" \
+        configure_portage "cross/${cross_chost}"
+
+    # In order to get a dependency list we must calculate it before
+    # updating package.provided. Otherwise portage will no-op.
+    get_dependency_list "$ROOT" "${TOOLCHAIN_PKGS[@]}" \
+        > "$ROOT/etc/portage/cross-${cross_chost}-depends"
+
+    # Add toolchain to packages.provided since they are on the host system
+    mkdir -p "$ROOT/etc/portage/profile/package.provided"
+    local native_pkg cross_pkg cross_pkg_version
+    for native_pkg in "${TOOLCHAIN_PKGS[@]}"; do
+        cross_pkg="${native_pkg/*\//cross-${cross_chost}/}"
+        cross_pkg_version=$(portageq match / "${cross_pkg}")
+        echo "${native_pkg%/*}/${cross_pkg_version#*/}"
+    done > "$ROOT/etc/portage/profile/package.provided/cross-${cross_chost}"
+}
+
+build_cross_libs() {
+    local cross_chost="$1"
+    local ROOT="/usr/${cross_chost}"
+    local cross_deps=$(<"$ROOT/etc/portage/cross-${cross_chost}-depends")
+
+    # --root is required because run_merge overrides ROOT=
+    PORTAGE_CONFIGROOT="$ROOT" run_merge -u --root="$ROOT" $cross_deps
+}
+
+configure_target_root() {
+    local board="$1"
+    local cross_chost=$(get_board_chost "$1")
+
+    CHOST="${cross_chost}" ROOT="/build/${board}" \
+        SYSROOT="/usr/${cross_chost}" configure_portage "target/${board}"
+}
+
+build_target_toolchain() {
+    local board="$1"
+    local ROOT="/build/${board}"
+
+    # --root is required because run_merge overrides ROOT=
+    PORTAGE_CONFIGROOT="$ROOT" \
+        run_merge -u --root="$ROOT" "${TOOLCHAIN_PKGS[@]}"
+}
+
+for cross_chost in $(get_chost_list); do
+    echo "Building cross toolchain for ${cross_chost}"
+    build_cross_toolchain "${cross_chost}"
+    configure_cross_root "${cross_chost}"
+    build_cross_libs "${cross_chost}"
+done
+
+for board in $(get_board_list); do
+    echo "Building native toolchain for ${bard}"
+    configure_target_root "${board}"
+    build_target_toolchain "${board}"
+done
diff --git a/build_library/toolchain_util.sh b/build_library/toolchain_util.sh
new file mode 100644
index 0000000000..e38be28b25
--- /dev/null
+++ b/build_library/toolchain_util.sh
@@ -0,0 +1,50 @@
+#!/bin/bash
+
+# Toolchain packages are treated a bit specially, since they take a
+# while to build and are generally more complicated to build they are
+# only built via catalyst and everyone else installs them as binpkgs.
+TOOLCHAIN_PKGS=(
+    sys-devel/binutils
+    sys-devel/gcc
+    sys-kernel/linux-headers
+    sys-libs/glibc
+)
+
+# Map board names to CHOSTs and portage profiles. This is the
+# definitive list, there is assorted code new and old that either
+# guesses or hard-code these. All that should migrate to this list.
+declare -A BOARD_CHOST BOARD_PROFILE
+BOARD_CHOST["amd64-generic"]="x86_64-cros-linux-gnu"
+BOARD_PROFILE["amd64-generic"]="coreos:coreos/amd64/generic"
+BOARD_NAMES=( "${!BOARD_CHOST[@]}" )
+
+get_board_list() {
+    local IFS=$'\n\t '
+    sort <<<"${BOARD_NAMES[*]}"
+}
+
+get_chost_list() {
+    local IFS=$'\n\t '
+    sort -u <<<"${BOARD_CHOST[*]}"
+}
+
+get_profile_list() {
+    local IFS=$'\n\t '
+    sort -u <<<"${BOARD_PROFILE[*]}"
+}
+
+get_board_chost() {
+    if [[ ${#BOARD_CHOST["$1"]} -ne 0 ]]; then
+        echo "${BOARD_CHOST["$1"]}"
+    else
+        die "Unknown board '$1'"
+    fi
+}
+
+get_board_profile() {
+    if [[ ${#BOARD_PROFILE["$1"]} -ne 0 ]]; then
+        echo "${BOARD_PROFILE["$1"]}"
+    else
+        die "Unknown board '$1'"
+    fi
+}
diff --git a/build_toolchains b/build_toolchains
new file mode 100755
index 0000000000..c22ed0977f
--- /dev/null
+++ b/build_toolchains
@@ -0,0 +1,54 @@
+#!/bin/bash
+#
+# Copyright (c) 2013 The CoreOS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+SCRIPT_ROOT=$(dirname $(readlink -f "$0"))
+. "${SCRIPT_ROOT}/common.sh" || exit 1
+
+TYPE="coreos-toolchains"
+ARCH=$(portageq envvar ARCH)
+DEFAULT_SEED="builds/coreos-sdk/stage4-${ARCH}-latest.tar.bz2"
+DEFAULT_PROFILE="coreos:default/linux/${ARCH}/10.0"
+FORCE_STAGES="stage4"
+
+. "${BUILD_LIBRARY_DIR}/catalyst.sh" || exit 1
+. "${BUILD_LIBRARY_DIR}/toolchain_util.sh" || exit 1
+
+# include upload options
+. "${BUILD_LIBRARY_DIR}/release_util.sh" || exit 1
+
+ROOT_OVERLAY="${TEMPDIR}/stage4-${ARCH}-$FLAGS_version-overlay"
+
+## Define the stage4 config template
+catalyst_stage4() {
+cat <<EOF
+target: stage4
+pkgcache_path: $BINPKGS
+stage4/packages: @system
+stage4/fsscript: ${BUILD_LIBRARY_DIR}/catalyst_toolchains.sh
+stage4/root_overlay: ${ROOT_OVERLAY}
+EOF
+catalyst_stage_default
+}
+
+catalyst_init "$@"
+check_gsutil_opts
+
+# toolchain_util.sh is required by catalyst_toolchains.sh
+mkdir -p "${ROOT_OVERLAY}/tmp"
+cp "${BUILD_LIBRARY_DIR}/toolchain_util.sh" "${ROOT_OVERLAY}/tmp"
+
+catalyst_build
+
+def_upload_path="${UPLOAD_ROOT}/sdk/${ARCH}/${FLAGS_version}"
+upload_files "packages" "${def_upload_path}" "pkgs/" "${BINPKGS}/crossdev"/*
+
+for board in $(get_board_list); do
+    board_packages="${BINPKGS}/target/${board}"
+    def_upload_path="${UPLOAD_ROOT}/${board}/${FLAGS_version}"
+    upload_files packages "${def_upload_path}" "pkgs/" "${board_packages}"/*
+done
+
+command_completed

From 8ee5f0013756558beae7a9394d28279a4eb65788 Mon Sep 17 00:00:00 2001
From: Michael Marineau <mike@marineau.org>
Date: Tue, 22 Oct 2013 22:25:22 -0700
Subject: [PATCH 2/5] fix(common.sh): Stop collecting stats and uploading them
 to Google.

Just... no.. in so many ways. Kinda dumb it took me this long to delete
this little chunk of rather annoying code.
---
 build_packages |  1 -
 common.sh      | 40 ----------------------------------------
 2 files changed, 41 deletions(-)

diff --git a/build_packages b/build_packages
index c7d5ae2951..4084ee3a4c 100755
--- a/build_packages
+++ b/build_packages
@@ -230,6 +230,5 @@ rm "${tmpfile}"
 trap - EXIT
 
 echo "Builds complete"
-EXTRA_COMMAND_STATS[package_count]=${package_count}
 command_completed
 echo "Done"
diff --git a/common.sh b/common.sh
index 4777180828..da89510005 100644
--- a/common.sh
+++ b/common.sh
@@ -769,10 +769,6 @@ print_time_elapsed() {
   fi
 }
 
-# Associative array for filling in extra command-specific stats before
-# calling command_completed.
-declare -A EXTRA_COMMAND_STATS
-
 # Save original command line.
 command_line_arr=( "$0" "$@" )
 
@@ -781,42 +777,6 @@ command_completed() {
   local run_time=$(get_elapsed_seconds)
   local cmd_base=$(basename "${command_line_arr[0]}")
   print_time_elapsed ${run_time} ${cmd_base}
-
-  # Prepare command stats in an associative array.  Additonal command-specific
-  # stats can be added through EXTRA_COMMAND_STATS associative array.
-  declare -A stats
-  stats=(
-    [cmd_line]=${command_line_arr[*]}
-    [cmd_base]=${cmd_base}
-    [cmd_args]=${command_line_arr[*]:1}
-    [run_time]=${run_time}
-    [username]=$(get_git_id)
-    [board]=${FLAGS_board}
-    [host]=$(hostname -f)
-    [cpu_count]=$(grep -c processor /proc/cpuinfo)
-    [cpu_type]=$(uname -p)
-  )
-
-  local attr
-  for attr in "${!EXTRA_COMMAND_STATS[@]}"; do
-    stats[${attr}]=${EXTRA_COMMAND_STATS[${attr}]}
-  done
-
-  # Prepare temporary file for stats.
-  local tmpfile=$(mktemp -t tmp.stats.XXXXXX)
-  trap "rm -f '${tmpfile}'" EXIT
-
-  # Write stats out to temporary file.
-  echo "Chromium OS Build Command Stats - Version 1" > "${tmpfile}"
-  for attr in "${!stats[@]}"; do
-    echo "${attr} ${stats[${attr}]}"
-  done >> "${tmpfile}"
-
-  # Call upload_command_stats on the stats file.  If it fails do not stop.
-  "${GCLIENT_ROOT}"/chromite/bin/upload_command_stats "${tmpfile}" || true
-
-  rm "${tmpfile}"
-  trap - EXIT
 }
 
 # The board and variant command line options can be used in a number of ways

From 31d5ced3df8714e87691ccec457a13e4bc5a1079 Mon Sep 17 00:00:00 2001
From: Michael Marineau <mike@marineau.org>
Date: Wed, 23 Oct 2013 11:58:19 -0700
Subject: [PATCH 3/5] fix(build_toolchains): Remove hard-coded portage profile

---
 build_library/catalyst_toolchains.sh | 14 +++++++++-----
 build_library/toolchain_util.sh      |  8 ++++++++
 2 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/build_library/catalyst_toolchains.sh b/build_library/catalyst_toolchains.sh
index 28f15e8079..78c32b7f24 100644
--- a/build_library/catalyst_toolchains.sh
+++ b/build_library/catalyst_toolchains.sh
@@ -25,10 +25,12 @@ get_dependency_list() {
 
 configure_portage() {
     local pkg_path="$1"
+    local profile="$2"
+
     mkdir -p "${ROOT}/etc/portage"
-    rm -f "${ROOT}/etc/make.profile" "${ROOT}/etc/portage/make.profile"
-    # eselect will report "!!! Warning: Strange path." but that's OK
-    eselect profile set --force coreos:coreos/amd64/generic 
+    echo "eselect will report '!!! Warning: Strange path.' but that's OK"
+    eselect profile set --force "$profile"
+
     cat >"${ROOT}/etc/portage/make.conf" <<EOF
 CHOST=${CHOST}
 CBUILD=$(portageq envvar CBUILD)
@@ -79,7 +81,8 @@ configure_cross_root() {
     local ROOT="/usr/${cross_chost}"
 
     CHOST="${cross_chost}" ROOT="$ROOT" SYSROOT="$ROOT" \
-        configure_portage "cross/${cross_chost}"
+        configure_portage "cross/${cross_chost}" \
+        "${CROSS_PROFILE[${cross_chost}]}"
 
     # In order to get a dependency list we must calculate it before
     # updating package.provided. Otherwise portage will no-op.
@@ -110,7 +113,8 @@ configure_target_root() {
     local cross_chost=$(get_board_chost "$1")
 
     CHOST="${cross_chost}" ROOT="/build/${board}" \
-        SYSROOT="/usr/${cross_chost}" configure_portage "target/${board}"
+        SYSROOT="/usr/${cross_chost}" configure_portage \
+        "target/${board}" "$(get_board_profile "${board}")"
 }
 
 build_target_toolchain() {
diff --git a/build_library/toolchain_util.sh b/build_library/toolchain_util.sh
index e38be28b25..716207a6fa 100644
--- a/build_library/toolchain_util.sh
+++ b/build_library/toolchain_util.sh
@@ -10,6 +10,14 @@ TOOLCHAIN_PKGS=(
     sys-libs/glibc
 )
 
+# Portage arguments to enforce the toolchain to only use binpkgs.
+TOOLCHAIN_BINONLY=( "${TOOLCHAIN_PKGS[@]/#/--useoldpkg-atoms=}" )
+
+# Portage profile to use for building out the cross compiler's SYSROOT.
+# This is only used as an intermediate step to be able to use the cross
+# compiler to build a full native toolchain. Packages are not uploaded.
+CROSS_PROFILE["x86_64-cros-linux-gnu"]="coreos:coreos/amd64/generic"
+
 # Map board names to CHOSTs and portage profiles. This is the
 # definitive list, there is assorted code new and old that either
 # guesses or hard-code these. All that should migrate to this list.

From 169cc17394677ecf115ca7a807754002ded9eebf Mon Sep 17 00:00:00 2001
From: Michael Marineau <mike@marineau.org>
Date: Wed, 23 Oct 2013 12:02:25 -0700
Subject: [PATCH 4/5] fix(build_toolchains): Fix simple logging typo

---
 build_library/catalyst_toolchains.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build_library/catalyst_toolchains.sh b/build_library/catalyst_toolchains.sh
index 78c32b7f24..aadd4c3ad4 100644
--- a/build_library/catalyst_toolchains.sh
+++ b/build_library/catalyst_toolchains.sh
@@ -134,7 +134,7 @@ for cross_chost in $(get_chost_list); do
 done
 
 for board in $(get_board_list); do
-    echo "Building native toolchain for ${bard}"
+    echo "Building native toolchain for ${board}"
     configure_target_root "${board}"
     build_target_toolchain "${board}"
 done

From 9187adb7ca5531ce63e4b2ab2e9c31df0803e9e6 Mon Sep 17 00:00:00 2001
From: Michael Marineau <mike@marineau.org>
Date: Wed, 23 Oct 2013 12:07:39 -0700
Subject: [PATCH 5/5] fix(catalyst_toolchains.sh): Fix error in doc comment

---
 build_library/catalyst_toolchains.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build_library/catalyst_toolchains.sh b/build_library/catalyst_toolchains.sh
index aadd4c3ad4..066b599cd1 100644
--- a/build_library/catalyst_toolchains.sh
+++ b/build_library/catalyst_toolchains.sh
@@ -9,7 +9,7 @@ source /tmp/toolchain_util.sh
 # To make sure things are uploaded to the correct places we split things up:
 # crossdev build packages use ${PKGDIR}/crossdev (uploaded to SDK location)
 # build deps in crossdev's sysroot use ${PKGDIR}/cross/${CHOST} (no upload)
-# native toolchains use ${PKGDIR}/native/${BOARD} (uploaded to board location)
+# native toolchains use ${PKGDIR}/target/${BOARD} (uploaded to board location)
 
 get_dependency_list() {
     local ROOT="$1"