mirror of
https://github.com/flatcar/scripts.git
synced 2026-05-05 20:26:44 +02:00
Don't use repo snapshots for stage1 by updating seed the new way
This is what upstream Gentoo does. They would previously update the entire seed, but this took a long time. Our seeds are much bigger, so we kept repo snapshots to build stage1 against these instead. The new method of only rebuilding packages with changed sub-slots is a good compromise and removes the need to write stage1 hooks that selectively catch the repository up. This also avoids some conflicts by adding the `--ignore-world` option. Gentoo seeds have nothing in @world. We have much more, but none of that is needed for stage1. This continues to exclude cross-*-cros-linux-gnu/* as that is not needed for stage1. It now also excludes dev-lang/rust, because it is never a DEPEND, so it would not break other packages in this way. It may fail to run due to a sub-slot change in one of its own dependencies, but it is also unlikely to be needed in stage1 and it is not configured to use the system LLVM. If needs be, we could improve the behaviour of Portage's @changed-subslot to respect `--with-bdeps`. In my testing, it was unable to handle an SDK from 17 months ago, but one from 7 months ago did work. In practise, we will always use a much more recent one, which is far more likely to work. Signed-off-by: James Le Cuirot <jlecuirot@microsoft.com>
This commit is contained in:
James Le Cuirot
parent
9539132d3e
commit
a3015b199c
@ -59,12 +59,6 @@ TYPE="flatcar-sdk"
|
||||
. "${BUILD_LIBRARY_DIR}/release_util.sh" || exit 1
|
||||
|
||||
|
||||
DEFINE_string stage1_portage_path "" \
|
||||
"Path to custom portage ebuilds tree to use in stage 1 (DANGEROUS; USE WITH CAUTION)"
|
||||
DEFINE_string stage1_overlay_path "" \
|
||||
"Path to custom overlay ebuilds tree to use in stage 1 (DANGEROUS; USE WITH CAUTION)"
|
||||
|
||||
|
||||
## Define the stage4 config template
|
||||
catalyst_stage4() {
|
||||
cat <<EOF
|
||||
@ -142,85 +136,6 @@ cp "${BUILD_LIBRARY_DIR}/toolchain_util.sh" "${ROOT_OVERLAY}/tmp"
|
||||
# by stage one. Therefore, these binaries will cease to work in stage 2 when linked against
|
||||
# outdated "seed tarball" libraries which have been updated to newer versions in stage 1.
|
||||
|
||||
stage_repo() {
|
||||
local repo=${1}
|
||||
local path=${2}
|
||||
local dest=${3}
|
||||
local update_seed_file=${4}
|
||||
|
||||
if [ -z "$path" ]; then
|
||||
cp -R --reflink=auto "/var/gentoo/repos/${repo}" "$dest"
|
||||
info "Using local SDK's ebuild repo '$repo' in stage 1."
|
||||
else
|
||||
mkdir "$dest/$repo"
|
||||
cp -R --reflink=auto "${path}/"* "$dest/${repo}/"
|
||||
info "Using custom path '$path' for ebuild repo '$repo' in stage 1."
|
||||
info "This may break stage 2. YOU HAVE BEEN WARNED. You break it, you keep it."
|
||||
fi
|
||||
(
|
||||
set -euo pipefail
|
||||
local repo_var hook name
|
||||
|
||||
# FLAGS_flatcar_overlay for flatcar-overlay
|
||||
repo_var="FLAGS_${repo//-/_}"
|
||||
shopt -s nullglob
|
||||
for hook in "${FLAGS_flatcar_overlay}/coreos/stage1_hooks/"*"-${repo}.sh"; do
|
||||
name=${hook##*/}
|
||||
name=${name%"-${repo}.sh"}
|
||||
info "Invoking stage1 ${repo} hook ${name} on ${dest}/${repo}"
|
||||
"${hook}" "${dest}/${repo}" "${!repo_var}" "${update_seed_file}"
|
||||
done
|
||||
)
|
||||
}
|
||||
|
||||
build_stage1() {
|
||||
# First, write out the default catalyst configuration files
|
||||
write_configs
|
||||
|
||||
# Prepare local copies of both the "known-good" gentoo-subset and the
|
||||
# "known-good" flatcar-overlay ebuild repos
|
||||
local stage1_repos="$TEMPDIR/stage1-ebuild-repos"
|
||||
info "Creating stage 1 ebuild repos and stage 1 snapshot in '$stage1_repos'"
|
||||
rm -rf "$stage1_repos"
|
||||
mkdir "$stage1_repos"
|
||||
|
||||
# If the file exists and is not empty, seed will be updated.
|
||||
# Stage1 hooks may decide that the seed SDK needs updating.
|
||||
local update_seed_file="${TEMPDIR}/update_seed"
|
||||
|
||||
# prepare ebuild repos for stage 1, either from the local SDK (default)
|
||||
# or from custom paths specified via command line flags
|
||||
stage_repo "gentoo-subset" "${FLAGS_stage1_portage_path}" "$stage1_repos" "${update_seed_file}"
|
||||
stage_repo "flatcar-overlay" "${FLAGS_stage1_overlay_path}" "$stage1_repos" "${update_seed_file}"
|
||||
|
||||
# take the "portage directory" (gentoo-subset copy) snapshot
|
||||
build_snapshot "$stage1_repos/gentoo-subset" "${FLAGS_version}-stage1"
|
||||
|
||||
# Update the stage 1 spec to use the "known-good" gentoo-subset snapshot
|
||||
# and flatcar-overlay copy repository versions from above.
|
||||
sed -i -e "s/^snapshot_treeish:.*/snapshot_treeish: $FLAGS_version-stage1/" \
|
||||
-e "s,^repos:.*,repos: $stage1_repos/flatcar-overlay," \
|
||||
"$TEMPDIR/stage1.spec"
|
||||
|
||||
# If we are to use a custom path for either ebuild repo we want to update the stage1 seed SDK
|
||||
if [[ -n ${FLAGS_stage1_portage_path} ]] || [[ -n ${FLAGS_stage1_overlay_path} ]] || [[ -s ${update_seed_file} ]]; then
|
||||
sed -i 's/^update_seed: no/update_seed: yes/' "$TEMPDIR/stage1.spec"
|
||||
echo "update_seed_command: --update --deep --newuse --complete-graph --rebuild-if-new-ver --rebuild-exclude cross-*-cros-linux-gnu/* sys-devel/gcc " \
|
||||
>>"$TEMPDIR/stage1.spec"
|
||||
fi
|
||||
|
||||
rm -f "${update_seed_file}"
|
||||
|
||||
# Finally, build stage 1
|
||||
build_stage stage1
|
||||
}
|
||||
|
||||
if [[ "$STAGES" =~ stage1 ]]; then
|
||||
build_stage1
|
||||
STAGES="${STAGES/stage1/}"
|
||||
SEED="${TYPE}/stage1-${ARCH}-latest"
|
||||
fi
|
||||
|
||||
catalyst_build
|
||||
|
||||
if [[ "$STAGES" =~ stage4 ]]; then
|
||||
|
||||
@ -346,7 +346,7 @@ get_metadata() {
|
||||
local mirror="$(echo "${v}" | grep mirror:// | cut -d '/' -f 3)"
|
||||
if [ -n "${mirror}" ]; then
|
||||
# Take only first mirror, those not working should be removed
|
||||
local location="$(grep "^${mirror}"$'\t' /var/gentoo/repos/gentoo-subset/profiles/thirdpartymirrors | cut -d $'\t' -f 2- | cut -d ' ' -f 1 | tr -d $'\t')"
|
||||
local location="$(grep "^${mirror}"$'\t' /mnt/host/source/src/scripts/repos/gentoo-subset/profiles/thirdpartymirrors | cut -d $'\t' -f 2- | cut -d ' ' -f 1 | tr -d $'\t')"
|
||||
v="$(echo "${v}" | sed "s#mirror://${mirror}/#${location}#g")"
|
||||
fi
|
||||
new_val+="${v} "
|
||||
@ -472,7 +472,6 @@ EOF
|
||||
local license_dirs=(
|
||||
"/mnt/host/source/src/scripts/repos/flatcar-overlay/licenses/"
|
||||
"/mnt/host/source/src/scripts/repos/gentoo-subset/licenses/"
|
||||
"/var/gentoo/repos/gentoo/licenses/"
|
||||
"none"
|
||||
)
|
||||
for license_file in ${license_list}; do
|
||||
|
||||
@ -106,7 +106,8 @@ catalyst_stage1() {
|
||||
cat <<EOF
|
||||
# stage1 packages aren't published, save in tmp
|
||||
pkgcache_path: ${TEMPDIR}/stage1-${ARCH}-packages
|
||||
update_seed: no
|
||||
update_seed: yes
|
||||
update_seed_command: --exclude cross-*-cros-linux-gnu/* --exclude dev-lang/rust --ignore-world y --ignore-built-slot-operator-deps y @changed-subslot
|
||||
EOF
|
||||
catalyst_stage_default 1
|
||||
}
|
||||
@ -237,7 +238,7 @@ write_configs() {
|
||||
info "Configuring Portage..."
|
||||
cp -r "${BUILD_LIBRARY_DIR}"/portage/ "${TEMPDIR}/"
|
||||
|
||||
ln -sfT '/var/gentoo/repos/flatcar-overlay/coreos/user-patches' \
|
||||
ln -sfT '/mnt/host/source/src/scripts/repos/flatcar-overlay/coreos/user-patches' \
|
||||
"${TEMPDIR}"/portage/patches
|
||||
|
||||
[[ -n ${QEMU} ]] ||
|
||||
|
||||
@ -20,11 +20,3 @@ for cross_chost in $(get_chost_list); do
|
||||
PKGDIR="$(portageq envvar PKGDIR)/crossdev" \
|
||||
install_cross_rust "${cross_chost}" ${clst_myemergeopts}
|
||||
done
|
||||
|
||||
echo "Saving snapshot of repos for future SDK bootstraps"
|
||||
# Copy gentoo-subset and flatcar-overlay, which are under
|
||||
# /mnt/host/source/src/scripts/repos, into a local directory because they are
|
||||
# removed before archiving and we want to keep snapshots. These snapshots are
|
||||
# used by stage 1 in future bootstraps.
|
||||
mkdir -p /var/gentoo/repos
|
||||
cp -R /mnt/host/source/src/scripts/repos/{gentoo-subset,flatcar-overlay}/ /var/gentoo/repos/
|
||||
|
||||
@ -1,30 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
stage1_repo="${1}"
|
||||
new_repo="${2}"
|
||||
|
||||
good_version="3.6.8-r10"
|
||||
stage1_version=''
|
||||
|
||||
for f in "${stage1_repo}/sys-apps/baselayout/baselayout-"*'.ebuild'; do
|
||||
f="${f##*/}"
|
||||
if [[ "${f}" = *9999* ]]; then continue; fi
|
||||
f="${f%.ebuild}"
|
||||
f="${f#baselayout-}"
|
||||
stage1_version="${f}"
|
||||
done
|
||||
|
||||
if [[ -z "${stage1_version}" ]]; then exit 1; fi
|
||||
|
||||
older_version=$(printf '%s\n' "${stage1_version}" "${good_version}" | sort -V | head -n 1)
|
||||
|
||||
if [[ "${older_version}" = "${good_version}" ]]; then
|
||||
# Stage1 version is equal or newer than the good version, nothing
|
||||
# to do.
|
||||
exit 0
|
||||
fi
|
||||
|
||||
rm -rf "${stage1_repo}/sys-apps/baselayout"
|
||||
cp -a "${new_repo}/sys-apps/baselayout" "${stage1_repo}/sys-apps/baselayout"
|
||||
@ -1,19 +0,0 @@
|
||||
#!/bin/bash
|
||||
set -x
|
||||
set -euo pipefail
|
||||
|
||||
stage1_repo=${1}
|
||||
new_repo=${2}
|
||||
update_seed_file=${3}
|
||||
|
||||
cat=sys-libs
|
||||
pkg=libxcrypt
|
||||
|
||||
if [[ -d "${stage1_repo}/${cat}/${pkg}" ]]; then
|
||||
# libxcrypt is already a part of gentoo-subset, nothing to do
|
||||
exit 0
|
||||
fi
|
||||
|
||||
mkdir -p "${stage1_repo}/${cat}"
|
||||
cp -a "${new_repo}/${cat}/${pkg}" "${stage1_repo}/${cat}/${pkg}"
|
||||
echo x >"${update_seed_file}"
|
||||
@ -1,25 +0,0 @@
|
||||
#!/bin/bash
|
||||
set -x
|
||||
set -euo pipefail
|
||||
|
||||
stage1_repo="${1}"
|
||||
new_repo="${2}"
|
||||
parent_file='profiles/coreos/amd64/parent'
|
||||
old_parent_line='portage-stable:default/linux/amd64/17.0/no-multilib/hardened'
|
||||
stage1_parent="${stage1_repo}/${parent_file}"
|
||||
new_parent="${new_repo}/${parent_file}"
|
||||
|
||||
if [[ ! -e "${new_parent}" ]]; then
|
||||
echo "no file '${parent_file}' in new repo, nothing to do"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ ! -e "${stage1_parent}" ]]; then
|
||||
echo "no file '${parent_file}' in stage1 repo, nothing to do"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if grep --quiet --fixed-strings --line-regexp --regexp="${old_parent_line}" -- "${stage1_parent}"; then
|
||||
rm -f "${stage1_parent}"
|
||||
cp -a "${new_parent}" "${stage1_parent}"
|
||||
fi
|
||||
@ -1,53 +0,0 @@
|
||||
#!/bin/bash
|
||||
set -x
|
||||
set -euo pipefail
|
||||
|
||||
stage1_repo=${1}
|
||||
new_repo=${2}
|
||||
update_seed_file=${3}
|
||||
|
||||
base_profile_dir='profiles/coreos/base'
|
||||
|
||||
declare -A fixups_old=(
|
||||
['package.mask']='>=virtual/libcrypt-2'
|
||||
['package.unmask']='=virtual/libcrypt-1-r1'
|
||||
['package.use.force']='sys-libs/glibc crypt'
|
||||
['package.use.mask']='sys-libs/glibc -crypt'
|
||||
)
|
||||
|
||||
declare -A fixups_new=(
|
||||
['package.mask']='>=virtual/libcrypt-2'
|
||||
['package.unmask']='<virtual/libcrypt-2'
|
||||
['package.use.force']='sys-libs/glibc crypt'
|
||||
['package.use.mask']='sys-libs/glibc -crypt'
|
||||
)
|
||||
|
||||
for var_name in fixups_old fixups_new; do
|
||||
declare -n fixups="${var_name}"
|
||||
|
||||
skip=''
|
||||
for f in "${!fixups[@]}"; do
|
||||
l=${fixups["${f}"]}
|
||||
ff="${stage1_repo}/${base_profile_dir}/${f}"
|
||||
if ! grep --quiet --fixed-strings --line-regexp --regexp="${l}" -- "${ff}"; then
|
||||
# fixup not applicable, try next one
|
||||
skip=x
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ -n ${skip} ]]; then
|
||||
unset -n fixups
|
||||
continue
|
||||
fi
|
||||
|
||||
for f in "${!fixups[@]}"; do
|
||||
l=${fixups["${f}"]}
|
||||
ff="${stage1_repo}/${base_profile_dir}/${f}"
|
||||
ffb="${ff}.bak"
|
||||
mv "${ff}" "${ffb}"
|
||||
grep --invert-match --fixed-strings --line-regexp --regexp="${l}" -- "${ffb}" >"${ff}"
|
||||
done
|
||||
echo x >"${update_seed_file}"
|
||||
exit 0
|
||||
done
|
||||
@ -1,18 +0,0 @@
|
||||
The scripts in this directory are called by the SDK bootstrapping
|
||||
script when setting up the gentoo-subset and flatcar-overlay repos for
|
||||
the stage1 build. The scripts are invoked with two arguments - a path
|
||||
to the stage1 repository, and a path to the current repository. The
|
||||
difference between the two is that the stage1 repository is a copy of
|
||||
a repository saved in the seed SDK (thus it's going to be an older
|
||||
version of the repository), whereas the current repository is a
|
||||
repository that will be a base of the new SDK. The idea here is that
|
||||
something in the stage1 repository may be too old, thus it should be
|
||||
replaced with its equivalent from the current repository.
|
||||
|
||||
For more information about the bootstrap process, please see the
|
||||
`bootstrap_sdk` script in [the scripts
|
||||
repository](https://github.com/flatcar/scripts).
|
||||
|
||||
The script for gentoo-subset should end with `-gentoo-subset.sh`,
|
||||
and the script for flatcar-overlay with '-flatcar-overlay.sh`. For
|
||||
example: `0000-replace-ROOTPATH-flatcar-overlay.sh`.
|
||||
Loading…
x
Reference in New Issue
Block a user