From 7bf839c6fb3184a75f51680151c678f67748ee93 Mon Sep 17 00:00:00 2001
From: Michael Marineau <michael.marineau@coreos.com>
Date: Fri, 21 Jun 2013 12:53:15 -0400
Subject: [PATCH] add(bootstrap_sdk): Add a script for bootstrapping the sdk
 tarball

This uses Gentoo's catalyst for very thoroughly building images from
scratch. Using images based on this will eliminate some of the hackery
in make_chroot.sh for building up the sdk from a stock stage3 tarball.

For reference the procedure it performs is this:

1. snapshot: Grab a snapshot of portage-stable. Note that overalys are
   not snapshotted.
2. stage1: Using a "seed" tarball as a build environment, build a
   minimal root file system into a clean directory using ROOT=...
   and USE=-* The restricted USE flags are key be small and avoid
   circular dependencies.
3. stage2: Run portage-stable/scripts/bootstrap.sh
   This rebuilds the toolchain. Probably not strictly necessary most of
   the time but does super-duper-promise that the toolchain isn't linked
   to or otherwise influenced by whatever was in the "seed" tarball.
4. stage3: Run emerge -e system to rebuild everything using the fresh
   toolchain using the normal USE flags provided by the profile. This
   will also pull in assorted base system packages that weren't included
   in the minimal environment stage1 created.
5. stage4: Install any extra packages or other desired tweaks. For the
   sdk we just install all the packages normally make_chroot.sh does.
---
 bootstrap_sdk          |  54 ++++++++
 lib/catalyst.sh        | 284 +++++++++++++++++++++++++++++++++++++++++
 lib/catalyst_stage4.sh |   6 +
 lib/catalystrc         |  21 +++
 4 files changed, 365 insertions(+)
 create mode 100755 bootstrap_sdk
 create mode 100644 lib/catalyst.sh
 create mode 100644 lib/catalyst_stage4.sh
 create mode 100644 lib/catalystrc

diff --git a/bootstrap_sdk b/bootstrap_sdk
new file mode 100755
index 0000000000..5f20dd929b
--- /dev/null
+++ b/bootstrap_sdk
@@ -0,0 +1,54 @@
+#!/bin/bash
+#
+# Copyright (c) 2013 The CoreOS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+#
+# This uses Gentoo's catalyst for very thoroughly building images from
+# scratch. Using images based on this will eliminate some of the hackery
+# in make_chroot.sh for building up the sdk from a stock stage3 tarball.
+# 
+# For reference the procedure it performs is this:
+# 
+# 1. snapshot: Grab a snapshot of portage-stable. Note that overalys are
+#    not snapshotted.
+# 2. stage1: Using a "seed" tarball as a build environment, build a
+#    minimal root file system into a clean directory using ROOT=...
+#    and USE=-* The restricted USE flags are key be small and avoid
+#    circular dependencies.
+# 3. stage2: Run portage-stable/scripts/bootstrap.sh
+#    This rebuilds the toolchain. Probably not strictly necessary most of
+#    the time but does super-duper-promise that the toolchain isn't linked
+#    to or otherwise influenced by whatever was in the "seed" tarball.
+# 4. stage3: Run emerge -e system to rebuild everything using the fresh
+#    toolchain using the normal USE flags provided by the profile. This
+#    will also pull in assorted base system packages that weren't included
+#    in the minimal environment stage1 created.
+# 5. stage4: Install any extra packages or other desired tweaks. For the
+#    sdk we just install all the packages normally make_chroot.sh does.
+#
+# Usage: bootstrap_sdk [stage1 stage2 etc]
+# By default all four stages will be built using the latest stage4 as a seed.
+
+SCRIPT_ROOT=$(dirname $(readlink -f "$0"))
+. "${SCRIPT_ROOT}/common.sh" || exit 1
+
+TYPE="coreos-sdk"
+ARCH=$(portageq envvar ARCH)
+DEFAULT_PROFILE="coreos:default/linux/${ARCH}/10.0"
+
+. "${SCRIPT_ROOT}/lib/catalyst.sh" || exit 1
+
+## Define the stage4 config template
+catalyst_stage4() {
+cat <<EOF
+target: stage4
+pkgcache_path: $BINPKGS
+stage4/packages: dev-python/setuptools dev-vcs/git app-arch/pbzip2 net-misc/curl app-admin/sudo app-shells/bash-completion sys-devel/crossdev coreos-base/hard-host-depends
+stage4/fsscript: ${SCRIPT_ROOT}/lib/catalyst_stage4.sh
+EOF
+catalyst_stage_default
+}
+
+catalyst_init "$@"
+catalyst_build
diff --git a/lib/catalyst.sh b/lib/catalyst.sh
new file mode 100644
index 0000000000..73c195e256
--- /dev/null
+++ b/lib/catalyst.sh
@@ -0,0 +1,284 @@
+#!/bin/bash
+# Copyright (c) 2013 The CoreOS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# common.sh should be sourced first
+[[ -n "${DEFAULT_BUILD_ROOT}" ]] || exit 1
+
+# Default option values, may be provided before including this file
+: ${TYPE:="coreos-sdk"}
+: ${ARCH:=$(portageq envvar ARCH)}
+: ${DEFAULT_CATALYST_ROOT:="${DEFAULT_BUILD_ROOT}/catalyst"}
+: ${DEFAULT_SEED:="${DEFAULT_CATALYST_ROOT}/builds/${TYPE}/stage4-${ARCH}-latest.tar.bz2"}
+: ${DEFAULT_PROFILE:="coreos:default/linux/${ARCH}/10.0"}
+
+# Values set in catalyst_init, don't use till after calling it
+CATALYST_ROOT=
+DEBUG=
+BUILDS=
+BINPKGS=
+DISTDIR=
+TEMPDIR=
+STAGES=
+
+DEFINE_string catalyst_root "${DEFAULT_CATALYST_ROOT}" \
+    "Path to directory for all catalyst images and other files."
+DEFINE_string portage_stable "${SRC_ROOT}/third_party/portage-stable" \
+    "Path to the portage-stable git checkout."
+DEFINE_string coreos_overlay "${SRC_ROOT}/third_party/coreos-overlay" \
+    "Path to the coreos-overlay git checkout."
+DEFINE_string seed_tarball "${DEFAULT_SEED}" \
+    "Path to an existing stage tarball to start from."
+DEFINE_string version $(date -u +%Y%m%d%H%M%S) \
+    "Version to use for portage snapshot and stage tarballs."
+DEFINE_string profile "${DEFAULT_PROFILE}" \
+    "Portage profile, may be prefixed with repo:"
+DEFINE_boolean rebuild ${FLAGS_FALSE} \
+    "Rebuild and overwrite stages that already exist."
+DEFINE_integer make_jobs ${NUM_JOBS} "Tune make's concurrency."
+DEFINE_integer emerge_jobs ${NUM_JOBS} "Tune emerge's concurrency."
+DEFINE_boolean debug ${FLAGS_FALSE} "Enable verbose output from catalyst."
+
+#####################
+# CONFIG DEFINITIONS
+#
+# These templates can be expanded by calling the wrapping function after
+# catalyst_init has been called. They are written as files in write_configs
+# rather than passed as command line args for easier inspection and debugging
+# by hand, catalyst can get tricky.
+######################
+
+# Values to write to catalyst.conf
+catalyst_conf() {
+cat <<EOF
+# catalyst.conf
+contents="auto"
+digests="md5 sha1 sha512 whirlpool"
+hash_function="crc32"
+options="autoresume ccache kerncache pkgcache seedcache snapcache"
+sharedir="/usr/lib/catalyst"
+storedir="$CATALYST_ROOT"
+distdir="$DISTDIR"
+envscript="$TEMPDIR/catalystrc"
+port_logdir="$CATALYST_ROOT/log"
+portdir="$FLAGS_portage_stable"
+snapshot_cache="$CATALYST_ROOT/tmp/snapshot_cache"
+EOF
+}
+
+catalystrc() {
+cat "${SCRIPTS_DIR}/lib/catalystrc"
+echo "export MAKEOPTS=-j$FLAGS_make_jobs"
+echo "export EMERGE_DEFAULT_OPTS=--jobs=$FLAGS_emerge_jobs"
+}
+
+# Common values for all stage spec files
+catalyst_stage_default() {
+cat <<EOF
+subarch: $ARCH
+rel_type: $TYPE
+portage_overlay: $FLAGS_coreos_overlay
+profile: $FLAGS_profile
+snapshot: $FLAGS_version
+version_stamp: $FLAGS_version
+cflags: -O2 -pipe
+cxxflags: -O2 -pipe
+ldflags: -Wl,-O2 -Wl,--as-needed
+EOF
+}
+
+# Config values for each stage
+catalyst_stage1() {
+cat <<EOF
+target: stage1
+# stage1 packages aren't published, save in tmp
+pkgcache_path: ${TEMPDIR}/stage1-${ARCH}-packages
+EOF
+catalyst_stage_default
+}
+
+catalyst_stage2() {
+cat <<EOF
+target: stage2
+# stage2 packages aren't published, save in tmp
+pkgcache_path: ${TEMPDIR}/stage2-${ARCH}-packages
+EOF
+catalyst_stage_default
+}
+
+catalyst_stage3() {
+cat <<EOF
+target: stage3
+pkgcache_path: $BINPKGS
+EOF
+catalyst_stage_default
+}
+
+catalyst_stage4() {
+die "The calling script should redefine this function!"
+}
+
+##########################
+# END CONFIG DEFINITIONS #
+##########################
+
+# catalyst_init
+# Parses command ling arguments, validates them, and sets up environment
+# for the rest of the functions here. Should be the first thing called
+# after any script specific shflags DEFINE_* statements.
+# Usage: catalyst_init "$@"
+catalyst_init() {
+    FLAGS "$@" || exit 1
+    switch_to_strict_mode
+    eval set -- "${FLAGS_ARGV}"
+
+    if [[ $# -eq 0 ]]; then
+        STAGES="stage1 stage2 stage3 stage4"
+    else
+        for stage in "$@"; do
+            if [[ ! "$stage" =~ ^stage[1234]$ ]]; then
+                die_notrace "Invalid target name $stage"
+            fi
+        done
+        STAGES="$*"
+    fi
+
+    if [[ $(id -u) != 0 ]]; then
+        die_notrace "This script must be run as root."
+    fi
+
+    if ! which catalyst &>/dev/null; then
+        die_notrace "catalyst not found, not installed or bad PATH?"
+    fi
+
+    DEBUG=
+    if [[ ${FLAGS_debug} -eq ${FLAGS_TRUE} ]]; then
+        DEBUG="--debug --verbose"
+    fi
+
+    # Create output dir, expand path for easy comparison later
+    mkdir -p "$FLAGS_catalyst_root"
+    CATALYST_ROOT=$(readlink -f "$FLAGS_catalyst_root")
+
+    BUILDS="$CATALYST_ROOT/builds/$TYPE"
+    BINPKGS="$CATALYST_ROOT/packages/$TYPE"
+    TEMPDIR="$CATALYST_ROOT/tmp/$TYPE"
+    DISTDIR="$CATALYST_ROOT/distfiles"
+
+    # check for recent seed
+    if [[ ! -f "$FLAGS_seed_tarball" ]]; then
+        die_notrace "Seed tarball not found: $FLAGS_seed_tarball"
+    fi
+
+    # so far so good, expand path to work with weird comparison code below
+    FLAGS_seed_tarball=$(readlink -f "$FLAGS_seed_tarball")
+
+    if [[ ! "$FLAGS_seed_tarball" =~ .*\.tar\.bz2 ]]; then
+        die_notrace "Seed tarball doesn't end in .tar.bz2 :-/"
+    fi
+
+    # catalyst is obnoxious and wants the $TYPE/stage3-$VERSION part of the
+    # path, not the real path to the seed tarball. (Because it could be a
+    # directory under $TEMPDIR instead, aka the SEEDCACHE feature.)
+    if [[ "$FLAGS_seed_tarball" =~ "$CATALYST_ROOT/builds/".* ]]; then
+        SEED="${FLAGS_seed_tarball#$CATALYST_ROOT/builds/}"
+        SEED="${SEED%.tar.bz2}"
+    else
+        mkdir -p "$CATALYST_ROOT/builds/seed"
+        cp -n "$FLAGS_seed_tarball" "$CATALYST_ROOT/builds/seed"
+        SEED="seed/${FLAGS_seed_tarball##*/}"
+        SEED="${SEED%.tar.bz2}"
+    fi
+}
+
+write_configs() {
+    # No catalyst config option, so defined via environment
+    export CCACHE_DIR="$TEMPDIR/ccache"
+
+    info "Creating output directories..."
+    mkdir -p "$TEMPDIR" "$DISTDIR" "$CCACHE_DIR"
+    info "Writing out catalyst configs..."
+    info "    $TEMPDIR/catalyst.conf"
+    catalyst_conf > "$TEMPDIR/catalyst.conf"
+    info "    $TEMPDIR/catalystrc"
+    catalystrc > "$TEMPDIR/catalystrc"
+    info "    $TEMPDIR/stage1.spec"
+    catalyst_stage1 > "$TEMPDIR/stage1.spec"
+    info "    $TEMPDIR/stage2.spec"
+    catalyst_stage2 > "$TEMPDIR/stage2.spec"
+    info "    $TEMPDIR/stage3.spec"
+    catalyst_stage3 > "$TEMPDIR/stage3.spec"
+    info "    $TEMPDIR/stage4.spec"
+    catalyst_stage4 > "$TEMPDIR/stage4.spec"
+}
+
+build_stage() {
+    stage="$1"
+    srcpath="$2"
+    target_tarball="${stage}-${ARCH}-${FLAGS_version}.tar.bz2"
+
+    if [[ -f "$BUILDS/${target_tarball}" && $FLAGS_rebuild == $FLAGS_FALSE ]]
+    then
+        info "Skipping $stage, $target_tarball already exists."
+        return
+    fi
+
+    info "Starting $stage"
+    catalyst $DEBUG \
+        -c "$TEMPDIR/catalyst.conf" \
+        -f "$TEMPDIR/${stage}.spec" \
+        -C "source_subpath=$srcpath"
+    ln -sf "$stage-${ARCH}-${FLAGS_version}.tar.bz2" \
+        "$BUILDS/$stage-${ARCH}-latest.tar.bz2"
+    info "Finished building $target_tarball"
+}
+
+build_snapshot() {
+    if [[ -f "$CATALYST_ROOT/snapshots/portage-${FLAGS_version}.tar.bz2" ]]
+    then
+        info "Skipping snapshot, portage-${FLAGS_version}.tar.bz2 exists"
+    else
+        info "Creating snapshot portage-${FLAGS_version}.tar.bz2"
+        catalyst $DEBUG -c "$TEMPDIR/catalyst.conf" -s "$FLAGS_version"
+    fi
+}
+
+catalyst_build() {
+    # assert catalyst_init has been called
+    [[ -n "$CATALYST_ROOT" ]]
+
+    info "Building stages: $STAGES"
+    write_configs
+    build_snapshot
+
+    used_seed=0
+    if [[ "$STAGES" =~ stage1 ]]; then
+        build_stage stage1 "$SEED"
+        used_seed=1
+    fi
+
+    if [[ "$STAGES" =~ stage2 ]]; then
+        if [[ $used_seed -eq 1 ]]; then
+            SEED="${TYPE}/stage1-${ARCH}-latest"
+        fi
+        build_stage stage2 "$SEED"
+        used_seed=1
+    fi
+
+    if [[ "$STAGES" =~ stage3 ]]; then
+        if [[ $used_seed -eq 1 ]]; then
+            SEED="${TYPE}/stage2-${ARCH}-latest"
+        fi
+        build_stage stage3 "$SEED"
+        used_seed=1
+    fi
+
+    if [[ "$STAGES" =~ stage4 ]]; then
+        if [[ $used_seed -eq 1 ]]; then
+            SEED="${TYPE}/stage3-${ARCH}-latest"
+        fi
+        build_stage stage4 "$SEED"
+        used_seed=1
+    fi
+}
diff --git a/lib/catalyst_stage4.sh b/lib/catalyst_stage4.sh
new file mode 100644
index 0000000000..991bb05dad
--- /dev/null
+++ b/lib/catalyst_stage4.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+source /tmp/chroot-functions.sh
+
+echo "Double checking everything is fresh and happy."
+run_merge -uDN world
diff --git a/lib/catalystrc b/lib/catalystrc
new file mode 100644
index 0000000000..06f12c11cd
--- /dev/null
+++ b/lib/catalystrc
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+# setup some configs catalyst doesn't handle
+# i.e: update locale.gen to speed up builds
+maybe_append() {
+    [[ -f "$1" && ! -w "$1" ]] && return
+    if ! grep -q "^$2" "$1"; then
+        echo "$2" >> "$1"
+    fi
+}
+
+fixup_rootfs() {
+    mkdir -p "$1/etc" 2>/dev/null || return
+    maybe_append "$1/etc/locale.gen" "en_US ISO-8859-1"
+    maybe_append "$1/etc/locale.gen" "en_US.UTF-8 UTF-8"
+}
+
+# Fix both / and $ROOT (if it exists)
+fixup_rootfs
+[[ "${ROOT:-/}" != / ]] && fixup_rootfs "$ROOT"
+unset fixup_rootfs maybe_append