#!/bin/bash # Copyright (c) 2017 The CoreOS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. . "$(dirname "$0")/common.sh" || exit 1 # Script must run inside the chroot assert_inside_chroot assert_not_root_user # Developer-visible flags. DEFINE_string board "${DEFAULT_BOARD}" \ "The board to build packages for." DEFINE_string output_root "${DEFAULT_BUILD_ROOT}/torcx" \ "Directory in which to place torcx stores and manifests (named by board/version)" # include upload options . "${BUILD_LIBRARY_DIR}/release_util.sh" || exit 1 FLAGS_HELP="usage: $(basename $0) [flags] [images] This script builds a collection of torcx images to be installed into a torcx store. By default, all supported images are built, but a list of images can be given as command arguments. Note that their order matters, since the version specified last will get the default reference symlink. " show_help_if_requested "$@" # The following options are advanced options, only available to those willing # to read the source code. They are not shown in help output, since they are # not needed for the typical developer workflow. DEFINE_integer build_attempt 1 \ "The build attempt for this image build." DEFINE_string group developer \ "The update group." DEFINE_string version '' \ "Overrides version number in name to this version." # Parse command line FLAGS "$@" || exit 1 eval set -- "${FLAGS_ARGV}" # Only now can we die on error. shflags functions leak non-zero error codes, # so will die prematurely if 'switch_to_strict_mode' is specified before now. switch_to_strict_mode # Initialize upload options check_gsutil_opts # Define BUILD_DIR and set_build_symlinks. . "${BUILD_LIBRARY_DIR}/toolchain_util.sh" || exit 1 . "${BUILD_LIBRARY_DIR}/board_options.sh" || exit 1 . "${BUILD_LIBRARY_DIR}/build_image_util.sh" || exit 1 . "${BUILD_LIBRARY_DIR}/torcx_manifest.sh" || exit 1 TORCX_CAS_ROOT="${FLAGS_output_root}/pkgs/${BOARD}" # Print the first level of runtime dependencies for a torcx meta-package. function torcx_dependencies() ( pkg=${1:?} ebuild=$(equery-${BOARD} w "${pkg}") function inherit() { : ; } . "${ebuild}" echo ${RDEPEND} ) # Build and install a package configured as part of a torcx image. function torcx_build() ( pkg=${2:?} tmproot=${1:?} export LDFLAGS=-Wl,-rpath,/ORIGIN/../lib export PKGDIR="${tmproot}/var/lib/portage/pkgs" # Allow the meta-package to install bashrc to customize the builds. [ -s "${tmproot}/etc/portage/bashrc" ] && . "${tmproot}/etc/portage/bashrc" # Build binary packages using dev files in the board root. emerge-${BOARD} \ --buildpkg \ --buildpkgonly \ --nodeps \ --oneshot \ --quiet \ --root-deps=rdeps \ "${pkg}" # Install the binary packages in the temporary torcx image root. emerge-${BOARD} \ --nodeps \ --oneshot \ --quiet \ --root="${tmproot}" \ --root-deps=rdeps \ --sysroot="${tmproot}" \ --usepkgonly \ "${pkg}" ) # Create a torcx image from the given meta-package. function torcx_package() { local pkg="app-torcx/${1##*/}" local name=${pkg%-[0-9]*} local version=${pkg:${#name}+1} local manifest_path="${2}" local type="${3}" local deppkg digest file rpath sha512sum source_pkg rdepends tmproot tmppkgroot update_default tmpfile local pkg_cas_file pkg_cas_root local pkg_locations=() local name=${name##*/} local version=${version%%-r*} # Run in a subshell to clean tmproot and tmppkgroot up without # clobbering this shell's EXIT trap. ( # Set up the base package layout to dump everything into /bin and /lib. # tmproot is what the packages are installed into. # A subset of the files from tmproot are then moved into tmppkgroot, # which is then archived and uploaded. tmproot=$(sudo mktemp --tmpdir="${BUILD_DIR}" -d) tmppkgroot=$(sudo mktemp --tmpdir="${BUILD_DIR}" -d) trap "sudo rm -rf '${tmproot}' '${tmppkgroot}'" EXIT sudo chmod 0755 "${tmproot}" "${tmppkgroot}" sudo mkdir -p "${tmproot}"/{.torcx,bin,lib,usr} sudo ln -fns ../bin "${tmproot}/usr/bin" sudo ln -fns ../lib "${tmproot}/usr/lib" sudo ln -fns lib "${tmproot}/usr/lib64" sudo ln -fns bin "${tmproot}/usr/sbin" sudo ln -fns lib "${tmproot}/lib64" sudo ln -fns bin "${tmproot}/sbin" # Install the meta-package and its direct dependencies. for deppkg in "=${pkg}" $(torcx_dependencies "${pkg}") do torcx_build "${tmproot}" "${deppkg}" done # by convention, the first dependency in a torcx package is the primary # source package rdepends=($(torcx_dependencies "${pkg}")) source_pkg="${rdepends[0]#=}" # Pluck out shared libraries and SONAME links. sudo mv "${tmproot}"/{lib,tmplib} sudo rm -fr "${tmproot}/tmplib/debug" sudo find "${tmproot}/tmplib" -name 'lib*.so' -type l -delete sudo mkdir -p "${tmproot}/lib" sudo find "${tmproot}/tmplib" -name 'lib*.so*' \ -exec mv -t "${tmproot}/lib/" {} + # Rewrite any units for transparent activation from the torcx root. if [ -e "${tmproot}/tmplib/systemd/system" ] then sudo mkdir -p "${tmproot}/lib/systemd" sudo mv "${tmproot}/tmplib/systemd/system" \ "${tmproot}/lib/systemd/" sudo find "${tmproot}/lib/systemd/system" -type f -exec sed -i \ -e '/^\[Unit]/aRequires=torcx.target\nAfter=torcx.target' \ -e '/^\[Service]/aEnvironmentFile=/run/metadata/torcx' \ -e "/^\[Service]/aEnvironment=TORCX_IMAGEDIR=/${name}" \ -e 's,/usr/s\?bin/,${TORCX_BINDIR}/,g' \ -e 's,^\([^ ]*=\)\(.{TORCX_BINDIR}\)/,\1/usr/bin/env PATH=\2:${PATH} \2/,' {} + fi # Network configuration can be installed unmodified. if [ -e "${tmproot}/tmplib/systemd/network" ] then sudo mkdir -p "${tmproot}/lib/systemd" sudo mv "${tmproot}/tmplib/systemd/network" \ "${tmproot}/lib/systemd/" fi # Rewrite RPATHs to use the real $ORIGIN value. find -H "${tmproot}"/{bin,lib} -type f | while read file do ( rpath=$(sudo patchelf --print-rpath "${file}" 2>/dev/null) && test "${rpath#/ORIGIN/}" != "${rpath}" && sudo patchelf --set-rpath "${rpath/#?/\$}" "${file}" ) || : # Set $? to 0 or the pipeline fails and -e quits. done # Move anything we plan to package to its root. sudo mv "${tmproot}"/{.torcx,bin,lib} "${tmppkgroot}" if [ -e "${tmproot}/usr/share" ] then sudo mkdir "${tmppkgroot}/usr" sudo mv "${tmproot}/usr/share" "${tmppkgroot}/usr/" fi tmpfile="${BUILD_DIR}/${name}:${version}.torcx.tgz" tar --force-local --selinux --xattrs -C "${tmppkgroot}" -czf "${tmpfile}" . sha512sum=$(sha512sum "${tmpfile}" | awk '{print $1}') # TODO(euank): this opaque digest, if it were reproducible, could save # users from having to download things that haven't changed. # For now, use the sha512sum of the final image. # Ideally we should move to something more like a casync digest or tarsum. # The reason this is currently not being done is because to do that we # *MUST* ensure that a given pair of (digest, sha512sum) referenced in # a previous torcx package remains correct. # Because this code, as written, clobbers existing things with the same # digest (but the sha512sum of the .torcx.tgz can differ, e.g. due to ctime) # that property doesn't hold. # To switch this back to a reprodicble digest, we *must* never clobber # existing objects (and thus re-use their sha512sum here). digest="${sha512sum}" pkg_cas_root="${TORCX_CAS_ROOT}/${name}/${digest}" pkg_cas_file="${pkg_cas_root}/${name}:${version}.torcx.tgz" mkdir -p "${pkg_cas_root}" mv "${tmpfile}" "${pkg_cas_file}" update_default=false if [[ "${type}" == "default" ]]; then update_default=true pkg_locations+=("/usr/share/torcx/store/${name}:${version}.torcx.tgz") fi if [[ "${FLAGS_upload}" -eq ${FLAGS_TRUE} ]]; then pkg_locations+=("$(download_tectonic_torcx_url "pkgs/${BOARD}/${name}/${digest}/${name}:${version}.torcx.tgz")") fi torcx_manifest::add_pkg "${manifest_path}" \ "${name}" \ "${version}" \ "sha512-${sha512sum}" \ "${digest}" \ "${source_pkg}" \ "${update_default}" \ "${pkg_locations[@]}" ) } # This list defines every torcx image that goes into the vendor store for the # current branch's release version. Note that the default reference symlink # for each package will point at the last version specified. This can handle # swapping default package versions for different OS releases by reordering. DEFAULT_IMAGES=( =app-torcx/docker-20.10 ) # This list contains extra images which will be uploaded and included in the # generated manifest, but won't be included in the vendor store. EXTRA_IMAGES=( ) mkdir -p "${BUILD_DIR}" manifest_path="${BUILD_DIR}/torcx_manifest.json" torcx_manifest::create_empty "${manifest_path}" for pkg in "${@:-${DEFAULT_IMAGES[@]}}" ; do torcx_package "${pkg#=}" "${manifest_path}" "default" ; done for pkg in "${EXTRA_IMAGES[@]}" ; do torcx_package "${pkg#=}" "${manifest_path}" "extra" ; done set_build_symlinks latest "${FLAGS_group}-latest" # Upload the pkgs referenced by this manifest for pkg in $(torcx_manifest::get_pkg_names "${manifest_path}"); do for digest in $(torcx_manifest::get_digests "${manifest_path}" "${pkg}"); do # no need to sign; the manifest includes their shasum and is signed. upload_files \ 'torcx pkg' \ "${TORCX_UPLOAD_ROOT}/pkgs/${BOARD}/${pkg}/${digest}" \ "" \ "${TORCX_CAS_ROOT}/${pkg}/${digest}"/*.torcx.tgz done done # Upload the manifest # Note: the manifest is uploaded to 'UPLOAD_ROOT' rather than # 'TORCX_UPLOAD_ROOT'. # For non-release builds, those two locations will be the same, so it usually # won't matter. # However, for release builds, torcx packages may be uploaded directly to their # final location, while the manifest still has to go through build bucket in # order to get signed. sign_and_upload_files \ 'torcx manifest' \ "${UPLOAD_ROOT}/torcx/manifests/${BOARD}/${FLATCAR_VERSION}" \ "" \ "${manifest_path}" # vim: tabstop=8 softtabstop=4 shiftwidth=8 expandtab