flatcar-scripts/bootstrap_sdk

#!/bin/bash
#
# Copyright (c) 2013 The CoreOS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
#
# This uses Gentoo's catalyst for very thoroughly building images from
# scratch. Using images based on this will eliminate some of the hackery
# in make_chroot.sh for building up the sdk from a stock stage3 tarball.
# 
# For reference the procedure it performs is this:
# 
# 1. snapshot: Grab a snapshot of portage-stable. Note that overalys are
#    not snapshotted.
# 2. stage1: Using a "seed" tarball as a build environment, build a
#    minimal root file system into a clean directory using ROOT=...
#    and USE=-* The restricted USE flags are key be small and avoid
#    circular dependencies.
# 3. stage2: Run portage-stable/scripts/bootstrap.sh
#    This rebuilds the toolchain. Probably not strictly necessary most of
#    the time but does super-duper-promise that the toolchain isn't linked
#    to or otherwise influenced by whatever was in the "seed" tarball.
# 4. stage3: Run emerge -e system to rebuild everything using the fresh
#    toolchain using the normal USE flags provided by the profile. This
#    will also pull in assorted base system packages that weren't included
#    in the minimal environment stage1 created.
# 5. stage4: Install any extra packages or other desired tweaks. For the
#    sdk we just install all the packages normally make_chroot.sh does.
#
# Usage: bootstrap_sdk [stage1 stage2 etc]
# By default all four stages will be built using the latest stage4 as a seed.

SCRIPT_ROOT=$(dirname $(readlink -f "$0"))
. "${SCRIPT_ROOT}/common.sh" || exit 1

TYPE="flatcar-sdk"

. "${BUILD_LIBRARY_DIR}/catalyst.sh" || exit 1

# include upload options
. "${BUILD_LIBRARY_DIR}/release_util.sh" || exit 1

## Define the stage4 config template
catalyst_stage4() {
cat <<EOF
target: stage4
pkgcache_path: $BINPKGS
stage4/packages: coreos-devel/sdk-depends
stage4/fsscript: ${BUILD_LIBRARY_DIR}/catalyst_sdk.sh
stage4/root_overlay: ${ROOT_OVERLAY}
stage4/empty: /etc/portage/repos.conf /root /usr/portage /var/cache/edb
stage4/rm: /etc/machine-id /etc/resolv.conf
EOF
catalyst_stage_default
}

# Switch to HTTP because early boostrap stages do not have SSL support.
GENTOO_MIRRORS=$(portageq envvar GENTOO_MIRRORS)
GENTOO_MIRRORS="${GENTOO_MIRRORS//https:\/\//http://}"
export GENTOO_MIRRORS

catalyst_init "$@"
check_gsutil_opts
ROOT_OVERLAY=${TEMPDIR}/stage4_overlay

if [[ "$STAGES" =~ stage4 ]]; then
    info "Setting release to ${FLATCAR_VERSION}"
    rm -rf "${ROOT_OVERLAY}"
    # need to setup the lib->lib64 symlink correctly
    libdir=$(get_sdk_libdir)
    mkdir -p "${ROOT_OVERLAY}/usr/${libdir}"
    if [[ "${libdir}" != lib ]]; then
        ln -s "${libdir}" "${ROOT_OVERLAY}/usr/lib"
    fi
    "${BUILD_LIBRARY_DIR}/set_lsb_release" \
        --root "${ROOT_OVERLAY}"
fi

# toolchain_util.sh is required by catalyst_sdk.sh
# To copy it, we need to create /tmp with the right permissions as it will be
# used in the exported chroot.
mkdir -p "${ROOT_OVERLAY}/tmp"
chmod 1777 "${ROOT_OVERLAY}/tmp"
cp "${BUILD_LIBRARY_DIR}/toolchain_util.sh" "${ROOT_OVERLAY}/tmp"


# Stage 1 uses a different ebuild repo snapshot as well as a different portage_overlay 
# the ones from the SDK where ./bootstrap_sdk is being executed -
# to buils a known-good stage 1 (see stages description at top
# of file). This prevents package upgrades from updated ebuild files in the portage/coreos
# to apply in stage 1.
# Stage 1 lacks proper isolation and will link all packages built against its own
# seed libraries instead of against libraries installed to /tmp/stage1root.
build_stage1() {
    local stage1_repos="$TEMPDIR/stage1-ebuild-repos"

    write_configs

    # use known-good gentoo base repo from SDK, coreos-overlay from manifest
    info "Creating stage 1 ebuild repos and stage 1 snapshot in '$stage1_repos'"
    rm -rf "$stage1_repos"
    mkdir "$stage1_repos"
    cp -R /var/gentoo/repos/gentoo "$stage1_repos"

    local overlay_revision=$(
        grep 'name="kinvolk/coreos-overlay"' /mnt/host/source/.repo/manifests/default.xml \
        | sed 's/.*revision="refs\/\(heads\/\)\{0,1\}\([^"]\+\)".*/\2/' )

    info "Using coreos-overlay revision '$overlay_revision'"

    mkdir "$stage1_repos/coreos-overlay"
    ( cd "$stage1_repos/coreos-overlay" \
         && git clone https://github.com/kinvolk/coreos-overlay.git . \
         && git checkout "$overlay_revision" )

    catalyst_conf > "$TEMPDIR/catalyst-stage1.conf"
    sed -i "s:^portdir.*:portdir=\"$stage1_repos/gentoo\":" \
         "$TEMPDIR/catalyst-stage1.conf"

    catalyst $DEBUG -c "$TEMPDIR/catalyst-stage1.conf" -s "$FLAGS_version-stage1"

    sed -i -e "s/^snapshot:.*/snapshot: $FLAGS_version-stage1/" \
           -e "s,^portage_overlay:.*,portage_overlay: $stage1_repos/coreos-overlay," \
        "$TEMPDIR/stage1.spec"

    build_stage stage1 "$SEED" "$TEMPDIR/catalyst-stage1.conf"
}

if [[ "$STAGES" =~ stage1 ]]; then
    build_stage1
    STAGES="${STAGES/stage1/}"
fi

catalyst_build

if [[ "$STAGES" =~ stage4 ]]; then
    info "Build complete! Changing output name to something more sensible."
    build_name="stage4-${ARCH}-${FLAGS_version}.tar.bz2"
    release_name="${TYPE}-${ARCH}-${FLAGS_version}.tar.bz2"
    build_image="${BUILDS}/${build_name}"
    release_image="${BUILDS}/${release_name}"
    build_contents="${build_image}.CONTENTS.gz"
    release_contents="${release_image}.CONTENTS.gz"
    build_digests="${build_image}.DIGESTS"
    release_digests="${release_image}.DIGESTS"
    ln -f "${build_image}" "${release_image}"
    ln -f "${build_contents}" "${release_contents}"
    sed -e "s/${build_name}/${release_name}/" \
        "${build_digests}" > "${release_digests}"

    # Validate we didn't break the DIGESTS with sed
    verify_digests "${release_image}" "${release_contents}"

    info "SDK ready: ${release_image}"

    def_upload_path="${UPLOAD_ROOT}/sdk/${ARCH}/${FLAGS_version}"
    sign_and_upload_files "tarball" "${def_upload_path}" "" \
        "${release_image}" "${release_contents}" "${release_digests}"
    sign_and_upload_files "packages" "${def_upload_path}" "pkgs/" \
        "${BINPKGS}"/*

    # Upload the SDK toolchain packages
    sign_and_upload_files "cross toolchain packages" "${def_upload_path}" \
        "toolchain/" "${BINPKGS}/crossdev"/*
fi

command_completed