mirror of
https://github.com/flatcar/scripts.git
synced 2025-08-07 21:16:57 +02:00
b2d6f7fc6e
Torcx manifest may contain paths and URLs as locations of packages. There are two kinds of packages - vendored and extra. Vendored packages normally have two locations - path to the directory inside the image where the package is (which is why it's called vendored), and a URL to the package on some remote server. Extra packages only have a URL. But the URLs are added only when we tell the build_torcx_store script to upload the packages at the same time, which is what the old build pipeline was doing. With the new pipeline, the upload happens as a separate step, thus the upload is disabled when invoking build_torcx_store, and so the packages are not getting URLs set. This change went unnoticed, because a kola test checking the generated torcx manifest was only checking if there is at least one location, either path or URL, and all the new releases have no extra packages, only vendored ones. When backporting the new pipeline to old LTS, the kola tests started to fail, because old LTS had one extra package, and this is how I noticed the problem.
288 lines
11 KiB
Bash
Executable File
288 lines
11 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
# Copyright (c) 2017 The CoreOS Authors. All rights reserved.
|
|
# Use of this source code is governed by a BSD-style license that can be
|
|
# found in the LICENSE file.
|
|
|
|
. "$(dirname "$0")/common.sh" || exit 1
|
|
|
|
# Script must run inside the chroot
|
|
assert_inside_chroot
|
|
|
|
assert_not_root_user
|
|
|
|
# Developer-visible flags.
|
|
DEFINE_string board "${DEFAULT_BOARD}" \
|
|
"The board to build packages for."
|
|
DEFINE_string output_root "${DEFAULT_BUILD_ROOT}/torcx" \
|
|
"Directory in which to place torcx stores and manifests (named by board/version)"
|
|
DEFINE_string extra_pkg_url "" \
|
|
"URL to directory where the torcx packages will be available for downloading"
|
|
|
|
# include upload options
|
|
. "${BUILD_LIBRARY_DIR}/release_util.sh" || exit 1
|
|
|
|
FLAGS_HELP="usage: $(basename $0) [flags] [images]
|
|
|
|
This script builds a collection of torcx images to be installed into a torcx
|
|
store. By default, all supported images are built, but a list of images can be
|
|
given as command arguments. Note that their order matters, since the version
|
|
specified last will get the default reference symlink.
|
|
"
|
|
show_help_if_requested "$@"
|
|
|
|
# The following options are advanced options, only available to those willing
|
|
# to read the source code. They are not shown in help output, since they are
|
|
# not needed for the typical developer workflow.
|
|
DEFINE_integer build_attempt 1 \
|
|
"The build attempt for this image build."
|
|
DEFINE_string group developer \
|
|
"The update group."
|
|
DEFINE_string version '' \
|
|
"Overrides version number in name to this version."
|
|
|
|
# Parse command line
|
|
FLAGS "$@" || exit 1
|
|
eval set -- "${FLAGS_ARGV}"
|
|
|
|
# Only now can we die on error. shflags functions leak non-zero error codes,
|
|
# so will die prematurely if 'switch_to_strict_mode' is specified before now.
|
|
switch_to_strict_mode
|
|
|
|
# Initialize upload options
|
|
check_gsutil_opts
|
|
|
|
# Define BUILD_DIR and set_build_symlinks.
|
|
. "${BUILD_LIBRARY_DIR}/toolchain_util.sh" || exit 1
|
|
. "${BUILD_LIBRARY_DIR}/board_options.sh" || exit 1
|
|
. "${BUILD_LIBRARY_DIR}/build_image_util.sh" || exit 1
|
|
. "${BUILD_LIBRARY_DIR}/torcx_manifest.sh" || exit 1
|
|
|
|
TORCX_CAS_ROOT="${FLAGS_output_root}/pkgs/${BOARD}"
|
|
|
|
# Build and install a package configured as part of a torcx image.
|
|
function torcx_build() (
|
|
tmproot=${1:?}
|
|
shift
|
|
pkgs=( "${@}" )
|
|
|
|
export LDFLAGS=-Wl,-rpath,/ORIGIN/../lib
|
|
export PKGDIR="${tmproot}/var/lib/portage/pkgs"
|
|
|
|
# Allow the meta-package to install bashrc to customize the builds.
|
|
[ -s "${tmproot}/etc/portage/bashrc" ] &&
|
|
. "${tmproot}/etc/portage/bashrc"
|
|
|
|
# Build binary packages using dev files in the board root.
|
|
emerge-${BOARD} \
|
|
--jobs="${NUM_JOBS}" \
|
|
--buildpkg \
|
|
--buildpkgonly \
|
|
--nodeps \
|
|
--oneshot \
|
|
--quiet \
|
|
--root-deps=rdeps \
|
|
"${pkgs[@]}"
|
|
|
|
# Install the binary packages in the temporary torcx image root.
|
|
emerge-${BOARD} \
|
|
--jobs="${NUM_JOBS}" \
|
|
--nodeps \
|
|
--oneshot \
|
|
--quiet \
|
|
--root="${tmproot}" \
|
|
--root-deps=rdeps \
|
|
--sysroot="${tmproot}" \
|
|
--usepkgonly \
|
|
"${pkgs[@]}"
|
|
)
|
|
|
|
# Create a torcx image from the given meta-package.
|
|
function torcx_package() {
|
|
local pkg="app-torcx/${1##*/}"
|
|
local name=${pkg%-[0-9]*}
|
|
local version=${pkg:${#name}+1}
|
|
local manifest_path="${2}"
|
|
local type="${3}"
|
|
local extra_pkg_url="${4}"
|
|
local deppkg digest file rpath sha512sum source_pkg rdepends tmproot tmppkgroot update_default tmpfile
|
|
local pkg_cas_file pkg_cas_root
|
|
local pkg_locations=()
|
|
local name=${name##*/}
|
|
local version=${version%%-r*}
|
|
|
|
# Run in a subshell to clean tmproot and tmppkgroot up without
|
|
# clobbering this shell's EXIT trap.
|
|
(
|
|
# Set up the base package layout to dump everything into /bin and /lib.
|
|
# tmproot is what the packages are installed into.
|
|
# A subset of the files from tmproot are then moved into tmppkgroot,
|
|
# which is then archived and uploaded.
|
|
tmproot=$(sudo mktemp --tmpdir="${BUILD_DIR}" -d)
|
|
tmppkgroot=$(sudo mktemp --tmpdir="${BUILD_DIR}" -d)
|
|
trap "sudo rm -rf '${tmproot}' '${tmppkgroot}'" EXIT
|
|
sudo chmod 0755 "${tmproot}" "${tmppkgroot}"
|
|
sudo mkdir -p "${tmproot}"/{.torcx,bin,lib,usr}
|
|
sudo ln -fns ../bin "${tmproot}/usr/bin"
|
|
sudo ln -fns ../lib "${tmproot}/usr/lib"
|
|
sudo ln -fns lib "${tmproot}/usr/lib64"
|
|
sudo ln -fns bin "${tmproot}/usr/sbin"
|
|
sudo ln -fns lib "${tmproot}/lib64"
|
|
sudo ln -fns bin "${tmproot}/sbin"
|
|
|
|
# Install the meta-package and its direct dependencies.
|
|
torcx_build "${tmproot}" "=${pkg}" $(torcx_dependencies "${pkg}")
|
|
|
|
# by convention, the first dependency in a torcx package is the primary
|
|
# source package
|
|
rdepends=($(torcx_dependencies "${pkg}"))
|
|
source_pkg="${rdepends[0]#=}"
|
|
|
|
# Pluck out shared libraries and SONAME links.
|
|
sudo mv "${tmproot}"/{lib,tmplib}
|
|
sudo rm -fr "${tmproot}/tmplib/debug"
|
|
sudo find "${tmproot}/tmplib" -name 'lib*.so' -type l -delete
|
|
sudo mkdir -p "${tmproot}/lib"
|
|
sudo find "${tmproot}/tmplib" -name 'lib*.so*' \
|
|
-exec mv -t "${tmproot}/lib/" {} +
|
|
|
|
# Rewrite any units for transparent activation from the torcx root.
|
|
if [ -e "${tmproot}/tmplib/systemd/system" ]
|
|
then
|
|
sudo mkdir -p "${tmproot}/lib/systemd"
|
|
sudo mv "${tmproot}/tmplib/systemd/system" \
|
|
"${tmproot}/lib/systemd/"
|
|
sudo find "${tmproot}/lib/systemd/system" -type f -exec sed -i \
|
|
-e '/^\[Unit]/aRequires=torcx.target\nAfter=torcx.target' \
|
|
-e '/^\[Service]/aEnvironmentFile=/run/metadata/torcx' \
|
|
-e "/^\[Service]/aEnvironment=TORCX_IMAGEDIR=/${name}" \
|
|
-e 's,/usr/s\?bin/,${TORCX_BINDIR}/,g' \
|
|
-e 's,^\([^ ]*=\)\(.{TORCX_BINDIR}\)/,\1/usr/bin/env PATH=\2:${PATH} \2/,' {} +
|
|
fi
|
|
|
|
# Network configuration can be installed unmodified.
|
|
if [ -e "${tmproot}/tmplib/systemd/network" ]
|
|
then
|
|
sudo mkdir -p "${tmproot}/lib/systemd"
|
|
sudo mv "${tmproot}/tmplib/systemd/network" \
|
|
"${tmproot}/lib/systemd/"
|
|
fi
|
|
|
|
# Rewrite RPATHs to use the real $ORIGIN value.
|
|
find -H "${tmproot}"/{bin,lib} -type f |
|
|
while read file
|
|
do
|
|
(
|
|
rpath=$(sudo patchelf --print-rpath "${file}" 2>/dev/null) &&
|
|
test "${rpath#/ORIGIN/}" != "${rpath}" &&
|
|
sudo patchelf --set-rpath "${rpath/#?/\$}" "${file}"
|
|
) || : # Set $? to 0 or the pipeline fails and -e quits.
|
|
done
|
|
|
|
# Move anything we plan to package to its root.
|
|
sudo mv "${tmproot}"/{.torcx,bin,lib} "${tmppkgroot}"
|
|
if [ -e "${tmproot}/usr/share" ]
|
|
then
|
|
sudo mkdir "${tmppkgroot}/usr"
|
|
sudo mv "${tmproot}/usr/share" "${tmppkgroot}/usr/"
|
|
fi
|
|
|
|
tmpfile="${BUILD_DIR}/${name}:${version}.torcx.tgz"
|
|
tar --force-local --selinux --xattrs -C "${tmppkgroot}" -czf "${tmpfile}" .
|
|
sha512sum=$(sha512sum "${tmpfile}" | awk '{print $1}')
|
|
|
|
# TODO(euank): this opaque digest, if it were reproducible, could save
|
|
# users from having to download things that haven't changed.
|
|
# For now, use the sha512sum of the final image.
|
|
# Ideally we should move to something more like a casync digest or tarsum.
|
|
# The reason this is currently not being done is because to do that we
|
|
# *MUST* ensure that a given pair of (digest, sha512sum) referenced in
|
|
# a previous torcx package remains correct.
|
|
# Because this code, as written, clobbers existing things with the same
|
|
# digest (but the sha512sum of the .torcx.tgz can differ, e.g. due to ctime)
|
|
# that property doesn't hold.
|
|
# To switch this back to a reprodicble digest, we *must* never clobber
|
|
# existing objects (and thus re-use their sha512sum here).
|
|
digest="${sha512sum}"
|
|
|
|
pkg_cas_root="${TORCX_CAS_ROOT}/${name}/${digest}"
|
|
pkg_cas_file="${pkg_cas_root}/${name}:${version}.torcx.tgz"
|
|
mkdir -p "${pkg_cas_root}"
|
|
mv "${tmpfile}" "${pkg_cas_file}"
|
|
|
|
update_default=false
|
|
if [[ "${type}" == "default" ]]; then
|
|
update_default=true
|
|
pkg_locations+=("/usr/share/torcx/store/${name}:${version}.torcx.tgz")
|
|
fi
|
|
if [[ "${FLAGS_upload}" -eq ${FLAGS_TRUE} ]]; then
|
|
pkg_locations+=("$(download_tectonic_torcx_url "pkgs/${BOARD}/${name}/${digest}/${name}:${version}.torcx.tgz")")
|
|
fi
|
|
if [[ -n "${extra_pkg_url}" ]]; then
|
|
pkg_locations+=("${extra_pkg_url}/${name}:${version}.torcx.tgz")
|
|
fi
|
|
torcx_manifest::add_pkg "${manifest_path}" \
|
|
"${name}" \
|
|
"${version}" \
|
|
"sha512-${sha512sum}" \
|
|
"${digest}" \
|
|
"${source_pkg}" \
|
|
"${pkg}" \
|
|
"${update_default}" \
|
|
"${pkg_locations[@]}"
|
|
)
|
|
}
|
|
|
|
# This list defines every torcx image that goes into the vendor store for the
|
|
# current branch's release version. Note that the default reference symlink
|
|
# for each package will point at the last version specified. This can handle
|
|
# swapping default package versions for different OS releases by reordering.
|
|
DEFAULT_IMAGES=(
|
|
=app-torcx/docker-20.10
|
|
)
|
|
|
|
# This list contains extra images which will be uploaded and included in the
|
|
# generated manifest, but won't be included in the vendor store.
|
|
EXTRA_IMAGES=(
|
|
)
|
|
|
|
mkdir -p "${BUILD_DIR}"
|
|
manifest_path="${BUILD_DIR}/torcx_manifest.json"
|
|
torcx_manifest::create_empty "${manifest_path}"
|
|
for pkg in "${@:-${DEFAULT_IMAGES[@]}}"; do
|
|
torcx_package "${pkg#=}" "${manifest_path}" "default" "${FLAGS_extra_pkg_url}"
|
|
done
|
|
for pkg in "${EXTRA_IMAGES[@]}"; do
|
|
torcx_package "${pkg#=}" "${manifest_path}" "extra" "${FLAGS_extra_pkg_url}"
|
|
done
|
|
|
|
set_build_symlinks latest "${FLAGS_group}-latest"
|
|
|
|
# Upload the pkgs referenced by this manifest
|
|
for pkg in $(torcx_manifest::get_pkg_names "${manifest_path}"); do
|
|
for digest in $(torcx_manifest::get_digests "${manifest_path}" "${pkg}"); do
|
|
# no need to sign; the manifest includes their shasum and is signed.
|
|
upload_files \
|
|
'torcx pkg' \
|
|
"${TORCX_UPLOAD_ROOT}/pkgs/${BOARD}/${pkg}/${digest}" \
|
|
"" \
|
|
"${TORCX_CAS_ROOT}/${pkg}/${digest}"/*.torcx.tgz
|
|
done
|
|
done
|
|
|
|
# Upload the manifest
|
|
# Note: the manifest is uploaded to 'UPLOAD_ROOT' rather than
|
|
# 'TORCX_UPLOAD_ROOT'.
|
|
# For non-release builds, those two locations will be the same, so it usually
|
|
# won't matter.
|
|
# However, for release builds, torcx packages may be uploaded directly to their
|
|
# final location, while the manifest still has to go through build bucket in
|
|
# order to get signed.
|
|
sign_and_upload_files \
|
|
'torcx manifest' \
|
|
"${UPLOAD_ROOT}/torcx/manifests/${BOARD}/${FLATCAR_VERSION}" \
|
|
"" \
|
|
"${manifest_path}"
|
|
|
|
# vim: tabstop=8 softtabstop=4 shiftwidth=8 expandtab
|