From 7bf839c6fb3184a75f51680151c678f67748ee93 Mon Sep 17 00:00:00 2001
From: Michael Marineau <michael.marineau@coreos.com>
Date: Fri, 21 Jun 2013 12:53:15 -0400
Subject: [PATCH 1/3] add(bootstrap_sdk): Add a script for bootstrapping the
 sdk tarball

This uses Gentoo's catalyst for very thoroughly building images from
scratch. Using images based on this will eliminate some of the hackery
in make_chroot.sh for building up the sdk from a stock stage3 tarball.

For reference the procedure it performs is this:

1. snapshot: Grab a snapshot of portage-stable. Note that overalys are
   not snapshotted.
2. stage1: Using a "seed" tarball as a build environment, build a
   minimal root file system into a clean directory using ROOT=...
   and USE=-* The restricted USE flags are key be small and avoid
   circular dependencies.
3. stage2: Run portage-stable/scripts/bootstrap.sh
   This rebuilds the toolchain. Probably not strictly necessary most of
   the time but does super-duper-promise that the toolchain isn't linked
   to or otherwise influenced by whatever was in the "seed" tarball.
4. stage3: Run emerge -e system to rebuild everything using the fresh
   toolchain using the normal USE flags provided by the profile. This
   will also pull in assorted base system packages that weren't included
   in the minimal environment stage1 created.
5. stage4: Install any extra packages or other desired tweaks. For the
   sdk we just install all the packages normally make_chroot.sh does.
---
 bootstrap_sdk          |  54 ++++++++
 lib/catalyst.sh        | 284 +++++++++++++++++++++++++++++++++++++++++
 lib/catalyst_stage4.sh |   6 +
 lib/catalystrc         |  21 +++
 4 files changed, 365 insertions(+)
 create mode 100755 bootstrap_sdk
 create mode 100644 lib/catalyst.sh
 create mode 100644 lib/catalyst_stage4.sh
 create mode 100644 lib/catalystrc

diff --git a/bootstrap_sdk b/bootstrap_sdk
new file mode 100755
index 0000000000..5f20dd929b
--- /dev/null
+++ b/bootstrap_sdk
@@ -0,0 +1,54 @@
+#!/bin/bash
+#
+# Copyright (c) 2013 The CoreOS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+#
+# This uses Gentoo's catalyst for very thoroughly building images from
+# scratch. Using images based on this will eliminate some of the hackery
+# in make_chroot.sh for building up the sdk from a stock stage3 tarball.
+# 
+# For reference the procedure it performs is this:
+# 
+# 1. snapshot: Grab a snapshot of portage-stable. Note that overalys are
+#    not snapshotted.
+# 2. stage1: Using a "seed" tarball as a build environment, build a
+#    minimal root file system into a clean directory using ROOT=...
+#    and USE=-* The restricted USE flags are key be small and avoid
+#    circular dependencies.
+# 3. stage2: Run portage-stable/scripts/bootstrap.sh
+#    This rebuilds the toolchain. Probably not strictly necessary most of
+#    the time but does super-duper-promise that the toolchain isn't linked
+#    to or otherwise influenced by whatever was in the "seed" tarball.
+# 4. stage3: Run emerge -e system to rebuild everything using the fresh
+#    toolchain using the normal USE flags provided by the profile. This
+#    will also pull in assorted base system packages that weren't included
+#    in the minimal environment stage1 created.
+# 5. stage4: Install any extra packages or other desired tweaks. For the
+#    sdk we just install all the packages normally make_chroot.sh does.
+#
+# Usage: bootstrap_sdk [stage1 stage2 etc]
+# By default all four stages will be built using the latest stage4 as a seed.
+
+SCRIPT_ROOT=$(dirname $(readlink -f "$0"))
+. "${SCRIPT_ROOT}/common.sh" || exit 1
+
+TYPE="coreos-sdk"
+ARCH=$(portageq envvar ARCH)
+DEFAULT_PROFILE="coreos:default/linux/${ARCH}/10.0"
+
+. "${SCRIPT_ROOT}/lib/catalyst.sh" || exit 1
+
+## Define the stage4 config template
+catalyst_stage4() {
+cat <<EOF
+target: stage4
+pkgcache_path: $BINPKGS
+stage4/packages: dev-python/setuptools dev-vcs/git app-arch/pbzip2 net-misc/curl app-admin/sudo app-shells/bash-completion sys-devel/crossdev coreos-base/hard-host-depends
+stage4/fsscript: ${SCRIPT_ROOT}/lib/catalyst_stage4.sh
+EOF
+catalyst_stage_default
+}
+
+catalyst_init "$@"
+catalyst_build
diff --git a/lib/catalyst.sh b/lib/catalyst.sh
new file mode 100644
index 0000000000..73c195e256
--- /dev/null
+++ b/lib/catalyst.sh
@@ -0,0 +1,284 @@
+#!/bin/bash
+# Copyright (c) 2013 The CoreOS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# common.sh should be sourced first
+[[ -n "${DEFAULT_BUILD_ROOT}" ]] || exit 1
+
+# Default option values, may be provided before including this file
+: ${TYPE:="coreos-sdk"}
+: ${ARCH:=$(portageq envvar ARCH)}
+: ${DEFAULT_CATALYST_ROOT:="${DEFAULT_BUILD_ROOT}/catalyst"}
+: ${DEFAULT_SEED:="${DEFAULT_CATALYST_ROOT}/builds/${TYPE}/stage4-${ARCH}-latest.tar.bz2"}
+: ${DEFAULT_PROFILE:="coreos:default/linux/${ARCH}/10.0"}
+
+# Values set in catalyst_init, don't use till after calling it
+CATALYST_ROOT=
+DEBUG=
+BUILDS=
+BINPKGS=
+DISTDIR=
+TEMPDIR=
+STAGES=
+
+DEFINE_string catalyst_root "${DEFAULT_CATALYST_ROOT}" \
+    "Path to directory for all catalyst images and other files."
+DEFINE_string portage_stable "${SRC_ROOT}/third_party/portage-stable" \
+    "Path to the portage-stable git checkout."
+DEFINE_string coreos_overlay "${SRC_ROOT}/third_party/coreos-overlay" \
+    "Path to the coreos-overlay git checkout."
+DEFINE_string seed_tarball "${DEFAULT_SEED}" \
+    "Path to an existing stage tarball to start from."
+DEFINE_string version $(date -u +%Y%m%d%H%M%S) \
+    "Version to use for portage snapshot and stage tarballs."
+DEFINE_string profile "${DEFAULT_PROFILE}" \
+    "Portage profile, may be prefixed with repo:"
+DEFINE_boolean rebuild ${FLAGS_FALSE} \
+    "Rebuild and overwrite stages that already exist."
+DEFINE_integer make_jobs ${NUM_JOBS} "Tune make's concurrency."
+DEFINE_integer emerge_jobs ${NUM_JOBS} "Tune emerge's concurrency."
+DEFINE_boolean debug ${FLAGS_FALSE} "Enable verbose output from catalyst."
+
+#####################
+# CONFIG DEFINITIONS
+#
+# These templates can be expanded by calling the wrapping function after
+# catalyst_init has been called. They are written as files in write_configs
+# rather than passed as command line args for easier inspection and debugging
+# by hand, catalyst can get tricky.
+######################
+
+# Values to write to catalyst.conf
+catalyst_conf() {
+cat <<EOF
+# catalyst.conf
+contents="auto"
+digests="md5 sha1 sha512 whirlpool"
+hash_function="crc32"
+options="autoresume ccache kerncache pkgcache seedcache snapcache"
+sharedir="/usr/lib/catalyst"
+storedir="$CATALYST_ROOT"
+distdir="$DISTDIR"
+envscript="$TEMPDIR/catalystrc"
+port_logdir="$CATALYST_ROOT/log"
+portdir="$FLAGS_portage_stable"
+snapshot_cache="$CATALYST_ROOT/tmp/snapshot_cache"
+EOF
+}
+
+catalystrc() {
+cat "${SCRIPTS_DIR}/lib/catalystrc"
+echo "export MAKEOPTS=-j$FLAGS_make_jobs"
+echo "export EMERGE_DEFAULT_OPTS=--jobs=$FLAGS_emerge_jobs"
+}
+
+# Common values for all stage spec files
+catalyst_stage_default() {
+cat <<EOF
+subarch: $ARCH
+rel_type: $TYPE
+portage_overlay: $FLAGS_coreos_overlay
+profile: $FLAGS_profile
+snapshot: $FLAGS_version
+version_stamp: $FLAGS_version
+cflags: -O2 -pipe
+cxxflags: -O2 -pipe
+ldflags: -Wl,-O2 -Wl,--as-needed
+EOF
+}
+
+# Config values for each stage
+catalyst_stage1() {
+cat <<EOF
+target: stage1
+# stage1 packages aren't published, save in tmp
+pkgcache_path: ${TEMPDIR}/stage1-${ARCH}-packages
+EOF
+catalyst_stage_default
+}
+
+catalyst_stage2() {
+cat <<EOF
+target: stage2
+# stage2 packages aren't published, save in tmp
+pkgcache_path: ${TEMPDIR}/stage2-${ARCH}-packages
+EOF
+catalyst_stage_default
+}
+
+catalyst_stage3() {
+cat <<EOF
+target: stage3
+pkgcache_path: $BINPKGS
+EOF
+catalyst_stage_default
+}
+
+catalyst_stage4() {
+die "The calling script should redefine this function!"
+}
+
+##########################
+# END CONFIG DEFINITIONS #
+##########################
+
+# catalyst_init
+# Parses command ling arguments, validates them, and sets up environment
+# for the rest of the functions here. Should be the first thing called
+# after any script specific shflags DEFINE_* statements.
+# Usage: catalyst_init "$@"
+catalyst_init() {
+    FLAGS "$@" || exit 1
+    switch_to_strict_mode
+    eval set -- "${FLAGS_ARGV}"
+
+    if [[ $# -eq 0 ]]; then
+        STAGES="stage1 stage2 stage3 stage4"
+    else
+        for stage in "$@"; do
+            if [[ ! "$stage" =~ ^stage[1234]$ ]]; then
+                die_notrace "Invalid target name $stage"
+            fi
+        done
+        STAGES="$*"
+    fi
+
+    if [[ $(id -u) != 0 ]]; then
+        die_notrace "This script must be run as root."
+    fi
+
+    if ! which catalyst &>/dev/null; then
+        die_notrace "catalyst not found, not installed or bad PATH?"
+    fi
+
+    DEBUG=
+    if [[ ${FLAGS_debug} -eq ${FLAGS_TRUE} ]]; then
+        DEBUG="--debug --verbose"
+    fi
+
+    # Create output dir, expand path for easy comparison later
+    mkdir -p "$FLAGS_catalyst_root"
+    CATALYST_ROOT=$(readlink -f "$FLAGS_catalyst_root")
+
+    BUILDS="$CATALYST_ROOT/builds/$TYPE"
+    BINPKGS="$CATALYST_ROOT/packages/$TYPE"
+    TEMPDIR="$CATALYST_ROOT/tmp/$TYPE"
+    DISTDIR="$CATALYST_ROOT/distfiles"
+
+    # check for recent seed
+    if [[ ! -f "$FLAGS_seed_tarball" ]]; then
+        die_notrace "Seed tarball not found: $FLAGS_seed_tarball"
+    fi
+
+    # so far so good, expand path to work with weird comparison code below
+    FLAGS_seed_tarball=$(readlink -f "$FLAGS_seed_tarball")
+
+    if [[ ! "$FLAGS_seed_tarball" =~ .*\.tar\.bz2 ]]; then
+        die_notrace "Seed tarball doesn't end in .tar.bz2 :-/"
+    fi
+
+    # catalyst is obnoxious and wants the $TYPE/stage3-$VERSION part of the
+    # path, not the real path to the seed tarball. (Because it could be a
+    # directory under $TEMPDIR instead, aka the SEEDCACHE feature.)
+    if [[ "$FLAGS_seed_tarball" =~ "$CATALYST_ROOT/builds/".* ]]; then
+        SEED="${FLAGS_seed_tarball#$CATALYST_ROOT/builds/}"
+        SEED="${SEED%.tar.bz2}"
+    else
+        mkdir -p "$CATALYST_ROOT/builds/seed"
+        cp -n "$FLAGS_seed_tarball" "$CATALYST_ROOT/builds/seed"
+        SEED="seed/${FLAGS_seed_tarball##*/}"
+        SEED="${SEED%.tar.bz2}"
+    fi
+}
+
+write_configs() {
+    # No catalyst config option, so defined via environment
+    export CCACHE_DIR="$TEMPDIR/ccache"
+
+    info "Creating output directories..."
+    mkdir -p "$TEMPDIR" "$DISTDIR" "$CCACHE_DIR"
+    info "Writing out catalyst configs..."
+    info "    $TEMPDIR/catalyst.conf"
+    catalyst_conf > "$TEMPDIR/catalyst.conf"
+    info "    $TEMPDIR/catalystrc"
+    catalystrc > "$TEMPDIR/catalystrc"
+    info "    $TEMPDIR/stage1.spec"
+    catalyst_stage1 > "$TEMPDIR/stage1.spec"
+    info "    $TEMPDIR/stage2.spec"
+    catalyst_stage2 > "$TEMPDIR/stage2.spec"
+    info "    $TEMPDIR/stage3.spec"
+    catalyst_stage3 > "$TEMPDIR/stage3.spec"
+    info "    $TEMPDIR/stage4.spec"
+    catalyst_stage4 > "$TEMPDIR/stage4.spec"
+}
+
+build_stage() {
+    stage="$1"
+    srcpath="$2"
+    target_tarball="${stage}-${ARCH}-${FLAGS_version}.tar.bz2"
+
+    if [[ -f "$BUILDS/${target_tarball}" && $FLAGS_rebuild == $FLAGS_FALSE ]]
+    then
+        info "Skipping $stage, $target_tarball already exists."
+        return
+    fi
+
+    info "Starting $stage"
+    catalyst $DEBUG \
+        -c "$TEMPDIR/catalyst.conf" \
+        -f "$TEMPDIR/${stage}.spec" \
+        -C "source_subpath=$srcpath"
+    ln -sf "$stage-${ARCH}-${FLAGS_version}.tar.bz2" \
+        "$BUILDS/$stage-${ARCH}-latest.tar.bz2"
+    info "Finished building $target_tarball"
+}
+
+build_snapshot() {
+    if [[ -f "$CATALYST_ROOT/snapshots/portage-${FLAGS_version}.tar.bz2" ]]
+    then
+        info "Skipping snapshot, portage-${FLAGS_version}.tar.bz2 exists"
+    else
+        info "Creating snapshot portage-${FLAGS_version}.tar.bz2"
+        catalyst $DEBUG -c "$TEMPDIR/catalyst.conf" -s "$FLAGS_version"
+    fi
+}
+
+catalyst_build() {
+    # assert catalyst_init has been called
+    [[ -n "$CATALYST_ROOT" ]]
+
+    info "Building stages: $STAGES"
+    write_configs
+    build_snapshot
+
+    used_seed=0
+    if [[ "$STAGES" =~ stage1 ]]; then
+        build_stage stage1 "$SEED"
+        used_seed=1
+    fi
+
+    if [[ "$STAGES" =~ stage2 ]]; then
+        if [[ $used_seed -eq 1 ]]; then
+            SEED="${TYPE}/stage1-${ARCH}-latest"
+        fi
+        build_stage stage2 "$SEED"
+        used_seed=1
+    fi
+
+    if [[ "$STAGES" =~ stage3 ]]; then
+        if [[ $used_seed -eq 1 ]]; then
+            SEED="${TYPE}/stage2-${ARCH}-latest"
+        fi
+        build_stage stage3 "$SEED"
+        used_seed=1
+    fi
+
+    if [[ "$STAGES" =~ stage4 ]]; then
+        if [[ $used_seed -eq 1 ]]; then
+            SEED="${TYPE}/stage3-${ARCH}-latest"
+        fi
+        build_stage stage4 "$SEED"
+        used_seed=1
+    fi
+}
diff --git a/lib/catalyst_stage4.sh b/lib/catalyst_stage4.sh
new file mode 100644
index 0000000000..991bb05dad
--- /dev/null
+++ b/lib/catalyst_stage4.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+source /tmp/chroot-functions.sh
+
+echo "Double checking everything is fresh and happy."
+run_merge -uDN world
diff --git a/lib/catalystrc b/lib/catalystrc
new file mode 100644
index 0000000000..06f12c11cd
--- /dev/null
+++ b/lib/catalystrc
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+# setup some configs catalyst doesn't handle
+# i.e: update locale.gen to speed up builds
+maybe_append() {
+    [[ -f "$1" && ! -w "$1" ]] && return
+    if ! grep -q "^$2" "$1"; then
+        echo "$2" >> "$1"
+    fi
+}
+
+fixup_rootfs() {
+    mkdir -p "$1/etc" 2>/dev/null || return
+    maybe_append "$1/etc/locale.gen" "en_US ISO-8859-1"
+    maybe_append "$1/etc/locale.gen" "en_US.UTF-8 UTF-8"
+}
+
+# Fix both / and $ROOT (if it exists)
+fixup_rootfs
+[[ "${ROOT:-/}" != / ]] && fixup_rootfs "$ROOT"
+unset fixup_rootfs maybe_append

From c77b595d66812801ad7179f7e0bafbc61fd9a4b2 Mon Sep 17 00:00:00 2001
From: Michael Marineau <michael.marineau@coreos.com>
Date: Fri, 21 Jun 2013 19:54:50 -0400
Subject: [PATCH 2/3] fix(catalyst): Add --with-bdeps=y to stage4 upgrade

By default emerge will not install build dependencies if it doesn't need
them (i.e. installing a binary package) but we want to make sure
everything gets included in stage4 no matter how it was installed.
---
 lib/catalyst_stage4.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/catalyst_stage4.sh b/lib/catalyst_stage4.sh
index 991bb05dad..e2b540a6d2 100644
--- a/lib/catalyst_stage4.sh
+++ b/lib/catalyst_stage4.sh
@@ -3,4 +3,4 @@
 source /tmp/chroot-functions.sh
 
 echo "Double checking everything is fresh and happy."
-run_merge -uDN world
+run_merge -uDN --with-bdeps=y world

From f5e61302f17cdc81bd7b38c9bbaa8cf4482c7e58 Mon Sep 17 00:00:00 2001
From: Michael Marineau <michael.marineau@coreos.com>
Date: Fri, 21 Jun 2013 20:00:40 -0400
Subject: [PATCH 3/3] fix(make_chroot): Remove unnecessary package
 rebuilds/removals

These are not required when starting from the new sdk stage4 tarballs
instead of a stock gentoo stage3 tarball.
---
 sdk_lib/make_chroot.sh | 45 ------------------------------------------
 1 file changed, 45 deletions(-)

diff --git a/sdk_lib/make_chroot.sh b/sdk_lib/make_chroot.sh
index d5720fb301..7ed2a10654 100755
--- a/sdk_lib/make_chroot.sh
+++ b/sdk_lib/make_chroot.sh
@@ -427,36 +427,7 @@ echo STAGE3=$STAGE3 > $CHROOT_STATE
 info "Updating portage"
 early_enter_chroot emerge -uNv --quiet portage
 
-
-info "Unmerge openssh temporarily"
-early_enter_chroot emerge --unmerge net-misc/openssh
-
-
-info "Install python-2.6"
-early_enter_chroot emerge -uNv --quiet python:2.6 dev-python/setuptools
-early_enter_chroot eselect python set python2.6
-
-
-# Packages that inherit cros-workon commonly get a circular dependency
-# curl->openssl->git->curl that is broken by emerging an early version of git
-# without curl (and webdav that depends on it).
-# We also need to do this before the toolchain as those will sometimes also
-# fetch via remote git trees (for some bot configs).
-if [[ ! -e "${FLAGS_chroot}/usr/bin/git" ]]; then
-  info "Installing early git"
-  early_enter_chroot $EMERGE_CMD -uNv $USEPKG dev-vcs/git
-
-  early_enter_chroot $EMERGE_CMD -uNv $USEPKG --select $EMERGE_JOBS \
-      dev-libs/openssl net-misc/curl
-
-  # (Re-)emerge the full version of git.
-  info "Installing openssh"
-  early_enter_chroot $EMERGE_CMD -uNv $USEPKG net-misc/openssh --select $EMERGE_JOBS
-fi
-
-
 # Enable git terminal prompt
-early_enter_chroot $EMERGE_CMD -uNv $USEPKG app-shells/bash-completion
 early_enter_chroot eselect bashcomp enable --global git-prompt
 
 info "Updating host toolchain"
@@ -469,22 +440,6 @@ fi
 early_enter_chroot "${CHROOT_TRUNK_DIR}/chromite/bin/cros_setup_toolchains" \
     --hostonly "${TOOLCHAIN_ARGS[@]}"
 
-# dhcpcd is included in 'world' by the stage3 that we pull in for some reason.
-# We have no need to install it in our host environment, so pull it out here.
-info "Deselecting dhcpcd"
-early_enter_chroot $EMERGE_CMD --deselect dhcpcd
-
-# openrc is included in stage3. We don't need it.
-info "Unmerge openrc"
-early_enter_chroot $EMERGE_CMD --unmerge sys-apps/openrc sys-apps/sysvinit sys-fs/udev-init-scripts \
-	|| echo "openrc not installed, ignoring"
-
-early_enter_chroot INSTALL_MASK="" $EMERGE_CMD coreos-base/efunctions
-
-info "Running emerge curl sudo ..."
-early_enter_chroot $EMERGE_CMD -uNv $USEPKG --select $EMERGE_JOBS \
-  pbzip2 dev-libs/openssl net-misc/curl sudo
-
 if [ -n "${INITIALIZE_CHROOT}" ]; then
   # If we're creating a new chroot, we also want to set it to the latest
   # version.