mirror of
https://github.com/flatcar/scripts.git
synced 2026-05-04 19:56:32 +02:00
Brandon Philips
commit
27ea8c25a8
54
bootstrap_sdk
Executable file
54
bootstrap_sdk
Executable file
@ -0,0 +1,54 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Copyright (c) 2013 The CoreOS Authors. All rights reserved.
|
||||
# Use of this source code is governed by a BSD-style license that can be
|
||||
# found in the LICENSE file.
|
||||
#
|
||||
# This uses Gentoo's catalyst for very thoroughly building images from
|
||||
# scratch. Using images based on this will eliminate some of the hackery
|
||||
# in make_chroot.sh for building up the sdk from a stock stage3 tarball.
|
||||
#
|
||||
# For reference the procedure it performs is this:
|
||||
#
|
||||
# 1. snapshot: Grab a snapshot of portage-stable. Note that overalys are
|
||||
# not snapshotted.
|
||||
# 2. stage1: Using a "seed" tarball as a build environment, build a
|
||||
# minimal root file system into a clean directory using ROOT=...
|
||||
# and USE=-* The restricted USE flags are key be small and avoid
|
||||
# circular dependencies.
|
||||
# 3. stage2: Run portage-stable/scripts/bootstrap.sh
|
||||
# This rebuilds the toolchain. Probably not strictly necessary most of
|
||||
# the time but does super-duper-promise that the toolchain isn't linked
|
||||
# to or otherwise influenced by whatever was in the "seed" tarball.
|
||||
# 4. stage3: Run emerge -e system to rebuild everything using the fresh
|
||||
# toolchain using the normal USE flags provided by the profile. This
|
||||
# will also pull in assorted base system packages that weren't included
|
||||
# in the minimal environment stage1 created.
|
||||
# 5. stage4: Install any extra packages or other desired tweaks. For the
|
||||
# sdk we just install all the packages normally make_chroot.sh does.
|
||||
#
|
||||
# Usage: bootstrap_sdk [stage1 stage2 etc]
|
||||
# By default all four stages will be built using the latest stage4 as a seed.
|
||||
|
||||
SCRIPT_ROOT=$(dirname $(readlink -f "$0"))
|
||||
. "${SCRIPT_ROOT}/common.sh" || exit 1
|
||||
|
||||
TYPE="coreos-sdk"
|
||||
ARCH=$(portageq envvar ARCH)
|
||||
DEFAULT_PROFILE="coreos:default/linux/${ARCH}/10.0"
|
||||
|
||||
. "${SCRIPT_ROOT}/lib/catalyst.sh" || exit 1
|
||||
|
||||
## Define the stage4 config template
|
||||
catalyst_stage4() {
|
||||
cat <<EOF
|
||||
target: stage4
|
||||
pkgcache_path: $BINPKGS
|
||||
stage4/packages: dev-python/setuptools dev-vcs/git app-arch/pbzip2 net-misc/curl app-admin/sudo app-shells/bash-completion sys-devel/crossdev coreos-base/hard-host-depends
|
||||
stage4/fsscript: ${SCRIPT_ROOT}/lib/catalyst_stage4.sh
|
||||
EOF
|
||||
catalyst_stage_default
|
||||
}
|
||||
|
||||
catalyst_init "$@"
|
||||
catalyst_build
|
||||
284
lib/catalyst.sh
Normal file
284
lib/catalyst.sh
Normal file
@ -0,0 +1,284 @@
|
||||
#!/bin/bash
|
||||
# Copyright (c) 2013 The CoreOS Authors. All rights reserved.
|
||||
# Use of this source code is governed by a BSD-style license that can be
|
||||
# found in the LICENSE file.
|
||||
|
||||
# common.sh should be sourced first
|
||||
[[ -n "${DEFAULT_BUILD_ROOT}" ]] || exit 1
|
||||
|
||||
# Default option values, may be provided before including this file
|
||||
: ${TYPE:="coreos-sdk"}
|
||||
: ${ARCH:=$(portageq envvar ARCH)}
|
||||
: ${DEFAULT_CATALYST_ROOT:="${DEFAULT_BUILD_ROOT}/catalyst"}
|
||||
: ${DEFAULT_SEED:="${DEFAULT_CATALYST_ROOT}/builds/${TYPE}/stage4-${ARCH}-latest.tar.bz2"}
|
||||
: ${DEFAULT_PROFILE:="coreos:default/linux/${ARCH}/10.0"}
|
||||
|
||||
# Values set in catalyst_init, don't use till after calling it
|
||||
CATALYST_ROOT=
|
||||
DEBUG=
|
||||
BUILDS=
|
||||
BINPKGS=
|
||||
DISTDIR=
|
||||
TEMPDIR=
|
||||
STAGES=
|
||||
|
||||
DEFINE_string catalyst_root "${DEFAULT_CATALYST_ROOT}" \
|
||||
"Path to directory for all catalyst images and other files."
|
||||
DEFINE_string portage_stable "${SRC_ROOT}/third_party/portage-stable" \
|
||||
"Path to the portage-stable git checkout."
|
||||
DEFINE_string coreos_overlay "${SRC_ROOT}/third_party/coreos-overlay" \
|
||||
"Path to the coreos-overlay git checkout."
|
||||
DEFINE_string seed_tarball "${DEFAULT_SEED}" \
|
||||
"Path to an existing stage tarball to start from."
|
||||
DEFINE_string version $(date -u +%Y%m%d%H%M%S) \
|
||||
"Version to use for portage snapshot and stage tarballs."
|
||||
DEFINE_string profile "${DEFAULT_PROFILE}" \
|
||||
"Portage profile, may be prefixed with repo:"
|
||||
DEFINE_boolean rebuild ${FLAGS_FALSE} \
|
||||
"Rebuild and overwrite stages that already exist."
|
||||
DEFINE_integer make_jobs ${NUM_JOBS} "Tune make's concurrency."
|
||||
DEFINE_integer emerge_jobs ${NUM_JOBS} "Tune emerge's concurrency."
|
||||
DEFINE_boolean debug ${FLAGS_FALSE} "Enable verbose output from catalyst."
|
||||
|
||||
#####################
|
||||
# CONFIG DEFINITIONS
|
||||
#
|
||||
# These templates can be expanded by calling the wrapping function after
|
||||
# catalyst_init has been called. They are written as files in write_configs
|
||||
# rather than passed as command line args for easier inspection and debugging
|
||||
# by hand, catalyst can get tricky.
|
||||
######################
|
||||
|
||||
# Values to write to catalyst.conf
|
||||
catalyst_conf() {
|
||||
cat <<EOF
|
||||
# catalyst.conf
|
||||
contents="auto"
|
||||
digests="md5 sha1 sha512 whirlpool"
|
||||
hash_function="crc32"
|
||||
options="autoresume ccache kerncache pkgcache seedcache snapcache"
|
||||
sharedir="/usr/lib/catalyst"
|
||||
storedir="$CATALYST_ROOT"
|
||||
distdir="$DISTDIR"
|
||||
envscript="$TEMPDIR/catalystrc"
|
||||
port_logdir="$CATALYST_ROOT/log"
|
||||
portdir="$FLAGS_portage_stable"
|
||||
snapshot_cache="$CATALYST_ROOT/tmp/snapshot_cache"
|
||||
EOF
|
||||
}
|
||||
|
||||
catalystrc() {
|
||||
cat "${SCRIPTS_DIR}/lib/catalystrc"
|
||||
echo "export MAKEOPTS=-j$FLAGS_make_jobs"
|
||||
echo "export EMERGE_DEFAULT_OPTS=--jobs=$FLAGS_emerge_jobs"
|
||||
}
|
||||
|
||||
# Common values for all stage spec files
|
||||
catalyst_stage_default() {
|
||||
cat <<EOF
|
||||
subarch: $ARCH
|
||||
rel_type: $TYPE
|
||||
portage_overlay: $FLAGS_coreos_overlay
|
||||
profile: $FLAGS_profile
|
||||
snapshot: $FLAGS_version
|
||||
version_stamp: $FLAGS_version
|
||||
cflags: -O2 -pipe
|
||||
cxxflags: -O2 -pipe
|
||||
ldflags: -Wl,-O2 -Wl,--as-needed
|
||||
EOF
|
||||
}
|
||||
|
||||
# Config values for each stage
|
||||
catalyst_stage1() {
|
||||
cat <<EOF
|
||||
target: stage1
|
||||
# stage1 packages aren't published, save in tmp
|
||||
pkgcache_path: ${TEMPDIR}/stage1-${ARCH}-packages
|
||||
EOF
|
||||
catalyst_stage_default
|
||||
}
|
||||
|
||||
catalyst_stage2() {
|
||||
cat <<EOF
|
||||
target: stage2
|
||||
# stage2 packages aren't published, save in tmp
|
||||
pkgcache_path: ${TEMPDIR}/stage2-${ARCH}-packages
|
||||
EOF
|
||||
catalyst_stage_default
|
||||
}
|
||||
|
||||
catalyst_stage3() {
|
||||
cat <<EOF
|
||||
target: stage3
|
||||
pkgcache_path: $BINPKGS
|
||||
EOF
|
||||
catalyst_stage_default
|
||||
}
|
||||
|
||||
catalyst_stage4() {
|
||||
die "The calling script should redefine this function!"
|
||||
}
|
||||
|
||||
##########################
|
||||
# END CONFIG DEFINITIONS #
|
||||
##########################
|
||||
|
||||
# catalyst_init
|
||||
# Parses command ling arguments, validates them, and sets up environment
|
||||
# for the rest of the functions here. Should be the first thing called
|
||||
# after any script specific shflags DEFINE_* statements.
|
||||
# Usage: catalyst_init "$@"
|
||||
catalyst_init() {
|
||||
FLAGS "$@" || exit 1
|
||||
switch_to_strict_mode
|
||||
eval set -- "${FLAGS_ARGV}"
|
||||
|
||||
if [[ $# -eq 0 ]]; then
|
||||
STAGES="stage1 stage2 stage3 stage4"
|
||||
else
|
||||
for stage in "$@"; do
|
||||
if [[ ! "$stage" =~ ^stage[1234]$ ]]; then
|
||||
die_notrace "Invalid target name $stage"
|
||||
fi
|
||||
done
|
||||
STAGES="$*"
|
||||
fi
|
||||
|
||||
if [[ $(id -u) != 0 ]]; then
|
||||
die_notrace "This script must be run as root."
|
||||
fi
|
||||
|
||||
if ! which catalyst &>/dev/null; then
|
||||
die_notrace "catalyst not found, not installed or bad PATH?"
|
||||
fi
|
||||
|
||||
DEBUG=
|
||||
if [[ ${FLAGS_debug} -eq ${FLAGS_TRUE} ]]; then
|
||||
DEBUG="--debug --verbose"
|
||||
fi
|
||||
|
||||
# Create output dir, expand path for easy comparison later
|
||||
mkdir -p "$FLAGS_catalyst_root"
|
||||
CATALYST_ROOT=$(readlink -f "$FLAGS_catalyst_root")
|
||||
|
||||
BUILDS="$CATALYST_ROOT/builds/$TYPE"
|
||||
BINPKGS="$CATALYST_ROOT/packages/$TYPE"
|
||||
TEMPDIR="$CATALYST_ROOT/tmp/$TYPE"
|
||||
DISTDIR="$CATALYST_ROOT/distfiles"
|
||||
|
||||
# check for recent seed
|
||||
if [[ ! -f "$FLAGS_seed_tarball" ]]; then
|
||||
die_notrace "Seed tarball not found: $FLAGS_seed_tarball"
|
||||
fi
|
||||
|
||||
# so far so good, expand path to work with weird comparison code below
|
||||
FLAGS_seed_tarball=$(readlink -f "$FLAGS_seed_tarball")
|
||||
|
||||
if [[ ! "$FLAGS_seed_tarball" =~ .*\.tar\.bz2 ]]; then
|
||||
die_notrace "Seed tarball doesn't end in .tar.bz2 :-/"
|
||||
fi
|
||||
|
||||
# catalyst is obnoxious and wants the $TYPE/stage3-$VERSION part of the
|
||||
# path, not the real path to the seed tarball. (Because it could be a
|
||||
# directory under $TEMPDIR instead, aka the SEEDCACHE feature.)
|
||||
if [[ "$FLAGS_seed_tarball" =~ "$CATALYST_ROOT/builds/".* ]]; then
|
||||
SEED="${FLAGS_seed_tarball#$CATALYST_ROOT/builds/}"
|
||||
SEED="${SEED%.tar.bz2}"
|
||||
else
|
||||
mkdir -p "$CATALYST_ROOT/builds/seed"
|
||||
cp -n "$FLAGS_seed_tarball" "$CATALYST_ROOT/builds/seed"
|
||||
SEED="seed/${FLAGS_seed_tarball##*/}"
|
||||
SEED="${SEED%.tar.bz2}"
|
||||
fi
|
||||
}
|
||||
|
||||
write_configs() {
|
||||
# No catalyst config option, so defined via environment
|
||||
export CCACHE_DIR="$TEMPDIR/ccache"
|
||||
|
||||
info "Creating output directories..."
|
||||
mkdir -p "$TEMPDIR" "$DISTDIR" "$CCACHE_DIR"
|
||||
info "Writing out catalyst configs..."
|
||||
info " $TEMPDIR/catalyst.conf"
|
||||
catalyst_conf > "$TEMPDIR/catalyst.conf"
|
||||
info " $TEMPDIR/catalystrc"
|
||||
catalystrc > "$TEMPDIR/catalystrc"
|
||||
info " $TEMPDIR/stage1.spec"
|
||||
catalyst_stage1 > "$TEMPDIR/stage1.spec"
|
||||
info " $TEMPDIR/stage2.spec"
|
||||
catalyst_stage2 > "$TEMPDIR/stage2.spec"
|
||||
info " $TEMPDIR/stage3.spec"
|
||||
catalyst_stage3 > "$TEMPDIR/stage3.spec"
|
||||
info " $TEMPDIR/stage4.spec"
|
||||
catalyst_stage4 > "$TEMPDIR/stage4.spec"
|
||||
}
|
||||
|
||||
build_stage() {
|
||||
stage="$1"
|
||||
srcpath="$2"
|
||||
target_tarball="${stage}-${ARCH}-${FLAGS_version}.tar.bz2"
|
||||
|
||||
if [[ -f "$BUILDS/${target_tarball}" && $FLAGS_rebuild == $FLAGS_FALSE ]]
|
||||
then
|
||||
info "Skipping $stage, $target_tarball already exists."
|
||||
return
|
||||
fi
|
||||
|
||||
info "Starting $stage"
|
||||
catalyst $DEBUG \
|
||||
-c "$TEMPDIR/catalyst.conf" \
|
||||
-f "$TEMPDIR/${stage}.spec" \
|
||||
-C "source_subpath=$srcpath"
|
||||
ln -sf "$stage-${ARCH}-${FLAGS_version}.tar.bz2" \
|
||||
"$BUILDS/$stage-${ARCH}-latest.tar.bz2"
|
||||
info "Finished building $target_tarball"
|
||||
}
|
||||
|
||||
build_snapshot() {
|
||||
if [[ -f "$CATALYST_ROOT/snapshots/portage-${FLAGS_version}.tar.bz2" ]]
|
||||
then
|
||||
info "Skipping snapshot, portage-${FLAGS_version}.tar.bz2 exists"
|
||||
else
|
||||
info "Creating snapshot portage-${FLAGS_version}.tar.bz2"
|
||||
catalyst $DEBUG -c "$TEMPDIR/catalyst.conf" -s "$FLAGS_version"
|
||||
fi
|
||||
}
|
||||
|
||||
catalyst_build() {
|
||||
# assert catalyst_init has been called
|
||||
[[ -n "$CATALYST_ROOT" ]]
|
||||
|
||||
info "Building stages: $STAGES"
|
||||
write_configs
|
||||
build_snapshot
|
||||
|
||||
used_seed=0
|
||||
if [[ "$STAGES" =~ stage1 ]]; then
|
||||
build_stage stage1 "$SEED"
|
||||
used_seed=1
|
||||
fi
|
||||
|
||||
if [[ "$STAGES" =~ stage2 ]]; then
|
||||
if [[ $used_seed -eq 1 ]]; then
|
||||
SEED="${TYPE}/stage1-${ARCH}-latest"
|
||||
fi
|
||||
build_stage stage2 "$SEED"
|
||||
used_seed=1
|
||||
fi
|
||||
|
||||
if [[ "$STAGES" =~ stage3 ]]; then
|
||||
if [[ $used_seed -eq 1 ]]; then
|
||||
SEED="${TYPE}/stage2-${ARCH}-latest"
|
||||
fi
|
||||
build_stage stage3 "$SEED"
|
||||
used_seed=1
|
||||
fi
|
||||
|
||||
if [[ "$STAGES" =~ stage4 ]]; then
|
||||
if [[ $used_seed -eq 1 ]]; then
|
||||
SEED="${TYPE}/stage3-${ARCH}-latest"
|
||||
fi
|
||||
build_stage stage4 "$SEED"
|
||||
used_seed=1
|
||||
fi
|
||||
}
|
||||
6
lib/catalyst_stage4.sh
Normal file
6
lib/catalyst_stage4.sh
Normal file
@ -0,0 +1,6 @@
|
||||
#!/bin/bash
|
||||
|
||||
source /tmp/chroot-functions.sh
|
||||
|
||||
echo "Double checking everything is fresh and happy."
|
||||
run_merge -uDN --with-bdeps=y world
|
||||
21
lib/catalystrc
Normal file
21
lib/catalystrc
Normal file
@ -0,0 +1,21 @@
|
||||
#!/bin/bash
|
||||
|
||||
# setup some configs catalyst doesn't handle
|
||||
# i.e: update locale.gen to speed up builds
|
||||
maybe_append() {
|
||||
[[ -f "$1" && ! -w "$1" ]] && return
|
||||
if ! grep -q "^$2" "$1"; then
|
||||
echo "$2" >> "$1"
|
||||
fi
|
||||
}
|
||||
|
||||
fixup_rootfs() {
|
||||
mkdir -p "$1/etc" 2>/dev/null || return
|
||||
maybe_append "$1/etc/locale.gen" "en_US ISO-8859-1"
|
||||
maybe_append "$1/etc/locale.gen" "en_US.UTF-8 UTF-8"
|
||||
}
|
||||
|
||||
# Fix both / and $ROOT (if it exists)
|
||||
fixup_rootfs
|
||||
[[ "${ROOT:-/}" != / ]] && fixup_rootfs "$ROOT"
|
||||
unset fixup_rootfs maybe_append
|
||||
@ -427,36 +427,7 @@ echo STAGE3=$STAGE3 > $CHROOT_STATE
|
||||
info "Updating portage"
|
||||
early_enter_chroot emerge -uNv --quiet portage
|
||||
|
||||
|
||||
info "Unmerge openssh temporarily"
|
||||
early_enter_chroot emerge --unmerge net-misc/openssh
|
||||
|
||||
|
||||
info "Install python-2.6"
|
||||
early_enter_chroot emerge -uNv --quiet python:2.6 dev-python/setuptools
|
||||
early_enter_chroot eselect python set python2.6
|
||||
|
||||
|
||||
# Packages that inherit cros-workon commonly get a circular dependency
|
||||
# curl->openssl->git->curl that is broken by emerging an early version of git
|
||||
# without curl (and webdav that depends on it).
|
||||
# We also need to do this before the toolchain as those will sometimes also
|
||||
# fetch via remote git trees (for some bot configs).
|
||||
if [[ ! -e "${FLAGS_chroot}/usr/bin/git" ]]; then
|
||||
info "Installing early git"
|
||||
early_enter_chroot $EMERGE_CMD -uNv $USEPKG dev-vcs/git
|
||||
|
||||
early_enter_chroot $EMERGE_CMD -uNv $USEPKG --select $EMERGE_JOBS \
|
||||
dev-libs/openssl net-misc/curl
|
||||
|
||||
# (Re-)emerge the full version of git.
|
||||
info "Installing openssh"
|
||||
early_enter_chroot $EMERGE_CMD -uNv $USEPKG net-misc/openssh --select $EMERGE_JOBS
|
||||
fi
|
||||
|
||||
|
||||
# Enable git terminal prompt
|
||||
early_enter_chroot $EMERGE_CMD -uNv $USEPKG app-shells/bash-completion
|
||||
early_enter_chroot eselect bashcomp enable --global git-prompt
|
||||
|
||||
info "Updating host toolchain"
|
||||
@ -469,22 +440,6 @@ fi
|
||||
early_enter_chroot "${CHROOT_TRUNK_DIR}/chromite/bin/cros_setup_toolchains" \
|
||||
--hostonly "${TOOLCHAIN_ARGS[@]}"
|
||||
|
||||
# dhcpcd is included in 'world' by the stage3 that we pull in for some reason.
|
||||
# We have no need to install it in our host environment, so pull it out here.
|
||||
info "Deselecting dhcpcd"
|
||||
early_enter_chroot $EMERGE_CMD --deselect dhcpcd
|
||||
|
||||
# openrc is included in stage3. We don't need it.
|
||||
info "Unmerge openrc"
|
||||
early_enter_chroot $EMERGE_CMD --unmerge sys-apps/openrc sys-apps/sysvinit sys-fs/udev-init-scripts \
|
||||
|| echo "openrc not installed, ignoring"
|
||||
|
||||
early_enter_chroot INSTALL_MASK="" $EMERGE_CMD coreos-base/efunctions
|
||||
|
||||
info "Running emerge curl sudo ..."
|
||||
early_enter_chroot $EMERGE_CMD -uNv $USEPKG --select $EMERGE_JOBS \
|
||||
pbzip2 dev-libs/openssl net-misc/curl sudo
|
||||
|
||||
if [ -n "${INITIALIZE_CHROOT}" ]; then
|
||||
# If we're creating a new chroot, we also want to set it to the latest
|
||||
# version.
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user