mirror of
https://github.com/flatcar/scripts.git
synced 2025-08-07 21:16:57 +02:00
74a5e5fcc5
Sometimes the modification to stage1 repos is necessary. For example we will need to update baselayout package in stage1, so we get the cleaned up handling of PATH and ROOTPATH, which will allow a newer portage to succeed in calculating dependencies in stage2.
265 lines
11 KiB
Bash
Executable File
265 lines
11 KiB
Bash
Executable File
#!/bin/bash
|
|
#
|
|
# Copyright (c) 2013 The CoreOS Authors. All rights reserved.
|
|
# Use of this source code is governed by a BSD-style license that can be
|
|
# found in the LICENSE file.
|
|
#
|
|
# This uses Gentoo's catalyst for very thoroughly building images from
|
|
# scratch. Using images based on this will eliminate some of the hackery
|
|
# in make_chroot.sh for building up the sdk from a stock stage3 tarball.
|
|
#
|
|
# For reference the procedure it performs is this:
|
|
#
|
|
# 1. snapshot: Grab a snapshot of the portage-stable repo from
|
|
# the current SDK's /var/lib/gentoo/repos/gentoo.
|
|
# Alternatively, check out a git ref specified via --portage-ref.
|
|
#
|
|
# 2. stage1: Using a "seed" tarball as a build environment, build a
|
|
# minimal root file system into a clean directory using ROOT=...
|
|
# and USE=-* The restricted USE flags are key be small and avoid
|
|
# circular dependencies.
|
|
# NOTE that stage1 LACKS PROPER STAGE ISOLATION. Binaries produced in stage1
|
|
# will be linked against the SEED SDK libraries, NOT against libraries
|
|
# built in stage 1. See "stage_repo()" documentation further below for more.
|
|
# This stage uses:
|
|
# - portage-stable from the SDK's /var/lib/gentoo/repos/gentoo
|
|
# or a git ref via --stage1_portage_ref command line option
|
|
# - coreos-overlay from the SDK's /var/lib/gentoo/repos/coreos-overlay
|
|
# or a git ref via --stage1_overlay_ref command line option
|
|
# Command line option refs need caution though, since
|
|
# stage1 must not contain updated ebuilds (see build_stage1 below).
|
|
#
|
|
# 3. stage2: Run portage-stable/scripts/bootstrap.sh
|
|
# This rebuilds the toolchain using Gentoo bootstrapping, ensuring it's not linked
|
|
# to or otherwise influenced by whatever was in the "seed" tarball.
|
|
# The toolchain rebuild may contain updated package ebuilds from
|
|
# third_party/(portage-stable|coreos-overlay).
|
|
# This and all following stages use portage-stable and coreos-overlay
|
|
# from third_party/... (see 1.)
|
|
#
|
|
# 4. stage3: Run emerge -e system to rebuild everything using the fresh updated
|
|
# toolchain from 3., using the normal USE flags provided by the profile. This
|
|
# will also pull in assorted base system packages that weren't included
|
|
# in the minimal environment stage1 created.
|
|
#
|
|
# 5. stage4: Install any extra packages or other desired tweaks. For the
|
|
# sdk we just install all the packages normally make_chroot.sh does.
|
|
#
|
|
# Usage: bootstrap_sdk [stage1 stage2 etc]
|
|
# By default all four stages will be built using the latest stage4 as a seed.
|
|
|
|
SCRIPT_ROOT=$(dirname $(readlink -f "$0"))
|
|
. "${SCRIPT_ROOT}/common.sh" || exit 1
|
|
|
|
TYPE="flatcar-sdk"
|
|
|
|
. "${BUILD_LIBRARY_DIR}/catalyst.sh" || exit 1
|
|
|
|
# include upload options
|
|
. "${BUILD_LIBRARY_DIR}/release_util.sh" || exit 1
|
|
|
|
|
|
DEFINE_string stage1_portage_ref "" \
|
|
"Custom portage repo git ref to use in stage 1 (USE WITH CAUTION)"
|
|
DEFINE_string stage1_overlay_ref "" \
|
|
"Custom overlay repo git ref to use in stage 1 (USE WITH CAUTION)"
|
|
|
|
|
|
## Define the stage4 config template
|
|
catalyst_stage4() {
|
|
cat <<EOF
|
|
target: stage4
|
|
pkgcache_path: $BINPKGS
|
|
stage4/packages: coreos-devel/sdk-depends
|
|
stage4/fsscript: ${BUILD_LIBRARY_DIR}/catalyst_sdk.sh
|
|
stage4/root_overlay: ${ROOT_OVERLAY}
|
|
stage4/empty: /etc/portage/repos.conf /root /usr/portage /var/cache/edb
|
|
stage4/rm: /etc/machine-id /etc/resolv.conf
|
|
EOF
|
|
catalyst_stage_default
|
|
}
|
|
|
|
# Switch to HTTP because early boostrap stages do not have SSL support.
|
|
GENTOO_MIRRORS=$(portageq envvar GENTOO_MIRRORS)
|
|
GENTOO_MIRRORS="${GENTOO_MIRRORS//https:\/\//http://}"
|
|
export GENTOO_MIRRORS
|
|
|
|
catalyst_init "$@"
|
|
check_gsutil_opts
|
|
ROOT_OVERLAY=${TEMPDIR}/stage4_overlay
|
|
|
|
if [[ "$STAGES" =~ stage4 ]]; then
|
|
info "Setting release to ${FLATCAR_VERSION}"
|
|
rm -rf "${ROOT_OVERLAY}"
|
|
# need to setup the lib->lib64 symlink correctly
|
|
libdir=$(get_sdk_libdir)
|
|
mkdir -p "${ROOT_OVERLAY}/usr/${libdir}"
|
|
if [[ "${libdir}" != lib ]]; then
|
|
if [[ "$(get_sdk_symlink_lib)" == "yes" ]]; then
|
|
ln -s "${libdir}" "${ROOT_OVERLAY}/usr/lib"
|
|
else
|
|
mkdir -p "${ROOT_OVERLAY}/usr/lib"
|
|
fi
|
|
fi
|
|
"${BUILD_LIBRARY_DIR}/set_lsb_release" \
|
|
--root "${ROOT_OVERLAY}"
|
|
fi
|
|
|
|
# toolchain_util.sh is required by catalyst_sdk.sh
|
|
# To copy it, we need to create /tmp with the right permissions as it will be
|
|
# used in the exported chroot.
|
|
mkdir -p "${ROOT_OVERLAY}/tmp"
|
|
chmod 1777 "${ROOT_OVERLAY}/tmp"
|
|
cp "${BUILD_LIBRARY_DIR}/toolchain_util.sh" "${ROOT_OVERLAY}/tmp"
|
|
|
|
|
|
# Stage 1 uses "known-good" ebuild repos (both coreos-overlay and portage-stable)
|
|
# to build a minimal toolchain (USE="-*") for stage 2.
|
|
#
|
|
# No package updates must happen in stage 1, so we use the portage-stable and
|
|
# coreos-overlay repos included with the current SDK (from the SDK chroot's
|
|
# /var/lib/gentoo/repos/). "Current SDK" refers to the SDK we entered with
|
|
# 'cork enter', i.e. the SDK we run ./bootstrap_sdk in.
|
|
#
|
|
# Using ebuilds from the above mentioned sources will ensure that stage 1 builds
|
|
# a minimal stage 2 from known-good ebuild versions - the same ebuild versions
|
|
# that were used to build the very SDK we run ./bootstrap_sdk in.
|
|
#
|
|
# This is needed because stage 1 lacks proper isolation and will link all packages built for
|
|
# stage 2 against its own seed libraries ("/" in the catalyst chroot) instead of against libraries
|
|
# installed into the FS root of the stage 2 seed ("/tmp/stage1root" in the catalyst chroot).
|
|
# This is why we must prevent any updated package ebuilds to "leak" into stage 1, hence we use
|
|
# "known good" ebuild repo versions outlined above.
|
|
#
|
|
# In special circumstances it may be required to circumvent this and use git
|
|
# refs of either (or both) portage and overlay. The command line options
|
|
# --stage1-portage-ref and --stage1-overlay-ref may be used to specify
|
|
# a git ref known to work for stage1. In that case the stage1 seed (i.e. the seed SDK)
|
|
# will be updated prior to starting to build stage 2.
|
|
|
|
stage_repo() {
|
|
local repo="$1"
|
|
local gitref="$2"
|
|
local dest="$3"
|
|
local gitname="$repo"
|
|
|
|
if [ "$gitname" = "gentoo" ] ; then
|
|
gitname="portage-stable"
|
|
fi
|
|
|
|
if [ -z "$gitref" ]; then
|
|
cp -R "/var/gentoo/repos/${repo}" "$dest"
|
|
info "Using local SDK's ebuild repo '$repo' ('$gitname') in stage 1."
|
|
else
|
|
info "Using git ref '$gitref' for ebuild repo '$repo' ('$gitname') in stage 1."
|
|
mkdir "$dest/$repo"
|
|
( cd "$dest/$repo" \
|
|
&& git clone "https://github.com/flatcar/$gitname.git" . \
|
|
&& git fetch --all \
|
|
&& git checkout "$gitref" )
|
|
fi
|
|
(
|
|
set -euo pipefail
|
|
local repo_var hook name
|
|
|
|
# FLAGS_coreos_overlay for gitname coreos-overlay
|
|
repo_var="FLAGS_${gitname//-/_}"
|
|
shopt -s nullglob
|
|
for hook in "${FLAGS_coreos_overlay}/coreos/stage1_hooks/"*"-${gitname}.sh"; do
|
|
name=${hook##*/}
|
|
name=${name%"-${gitname}.sh"}
|
|
info "Invoking stage1 ${gitname} hook ${name} on ${dest}/${repo}"
|
|
"${hook}" "${dest}/${repo}" "${!repo_var}"
|
|
done
|
|
)
|
|
}
|
|
|
|
build_stage1() {
|
|
# First, write out the default 4-stage catalyst configuration files
|
|
write_configs
|
|
|
|
# Prepare local copies of both the "known-good" portage-stable and the
|
|
# "known-good" coreos-overlay ebuild repos
|
|
local stage1_repos="$TEMPDIR/stage1-ebuild-repos"
|
|
info "Creating stage 1 ebuild repos and stage 1 snapshot in '$stage1_repos'"
|
|
rm -rf "$stage1_repos"
|
|
mkdir "$stage1_repos"
|
|
|
|
# prepare ebuild repos for stage 1, either from the local SDK (default)
|
|
# or from git refs specified via command line flags
|
|
stage_repo "gentoo" "${FLAGS_stage1_portage_ref}" "$stage1_repos"
|
|
stage_repo "coreos-overlay" "${FLAGS_stage1_overlay_ref}" "$stage1_repos"
|
|
|
|
# Create a snapshot of "known-good" portage-stable repo copy for use in stage 1
|
|
# This requires us to create a custom catalyst config to point it to the
|
|
# repo copy we just created, for snapshotting.
|
|
catalyst_conf > "$TEMPDIR/catalyst-stage1.conf"
|
|
sed -i "s:^portdir.*:portdir=\"$stage1_repos/gentoo\":" \
|
|
"$TEMPDIR/catalyst-stage1.conf"
|
|
# take the "portage directory" (portage-stable copy) snapshot
|
|
catalyst \
|
|
"${DEBUG[@]}" \
|
|
--verbose \
|
|
--config "$TEMPDIR/catalyst-stage1.conf" \
|
|
--snapshot "$FLAGS_version-stage1"
|
|
|
|
# Update the stage 1 spec to use the "known-good" portage-stable snapshot
|
|
# and coreos-overlay copy repository versions from above.
|
|
sed -i -e "s/^snapshot:.*/snapshot: $FLAGS_version-stage1/" \
|
|
-e "s,^portage_overlay:.*,portage_overlay: $stage1_repos/coreos-overlay," \
|
|
"$TEMPDIR/stage1.spec"
|
|
|
|
# If we are to use a git ref for either ebuild repo we want to update the stage1 seed SDK
|
|
if [ -n "${FLAGS_stage1_portage_ref}" -o -n "${FLAGS_stage1_overlay_ref}" ] ; then
|
|
sed -i 's/^update_seed: no/update_seed: yes/' "$TEMPDIR/stage1.spec"
|
|
echo "update_seed_command: --update --deep --newuse --complete-graph --rebuild-if-new-ver --rebuild-exclude cross-*-cros-linux-gnu/* sys-devel/gcc " \
|
|
>>"$TEMPDIR/stage1.spec"
|
|
fi
|
|
|
|
# Finally, build stage 1
|
|
build_stage stage1 "$SEED" "$TEMPDIR/catalyst-stage1.conf"
|
|
}
|
|
|
|
if [[ "$STAGES" =~ stage1 ]]; then
|
|
build_stage1
|
|
STAGES="${STAGES/stage1/}"
|
|
SEED="${TYPE}/stage1-${ARCH}-latest"
|
|
fi
|
|
|
|
catalyst_build
|
|
|
|
if [[ "$STAGES" =~ stage4 ]]; then
|
|
info "Build complete! Changing output name to something more sensible."
|
|
build_name="stage4-${ARCH}-${FLAGS_version}.tar.bz2"
|
|
release_name="${TYPE}-${ARCH}-${FLAGS_version}.tar.bz2"
|
|
build_image="${BUILDS}/${build_name}"
|
|
release_image="${BUILDS}/${release_name}"
|
|
build_contents="${build_image}.CONTENTS.gz"
|
|
release_contents="${release_image}.CONTENTS.gz"
|
|
build_digests="${build_image}.DIGESTS"
|
|
release_digests="${release_image}.DIGESTS"
|
|
ln -f "${build_image}" "${release_image}"
|
|
ln -f "${build_contents}" "${release_contents}"
|
|
sed -e "s/${build_name}/${release_name}/" \
|
|
"${build_digests}" > "${release_digests}"
|
|
|
|
# Validate we didn't break the DIGESTS with sed
|
|
verify_digests "${release_image}" "${release_contents}"
|
|
|
|
info "SDK ready: ${release_image}"
|
|
|
|
def_upload_path="${UPLOAD_ROOT}/sdk/${ARCH}/${FLAGS_version}"
|
|
sign_and_upload_files "tarball" "${def_upload_path}" "" \
|
|
"${release_image}" "${release_contents}" "${release_digests}"
|
|
sign_and_upload_files "packages" "${def_upload_path}" "pkgs/" \
|
|
"${BINPKGS}"/*
|
|
|
|
if [ -d "${BINPKGS}/crossdev" ]; then
|
|
# Upload the SDK toolchain packages
|
|
sign_and_upload_files "cross toolchain packages" "${def_upload_path}" \
|
|
"toolchain/" "${BINPKGS}/crossdev"/*
|
|
fi
|
|
fi
|
|
|
|
command_completed
|