#!/bin/bash # Copyright (c) 2017 The CoreOS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. . "$(dirname "$0")/common.sh" || exit 1 # Script must run inside the chroot restart_in_chroot_if_needed "$@" assert_not_root_user # Developer-visible flags. DEFINE_string board "${DEFAULT_BOARD}" \ "The board to build packages for." DEFINE_string output_root "${DEFAULT_BUILD_ROOT}/torcx" \ "Directory in which to place torcx stores and manifests (named by board/version)" # include upload options . "${BUILD_LIBRARY_DIR}/release_util.sh" || exit 1 FLAGS_HELP="usage: $(basename $0) [flags] [images] This script builds a collection of torcx images to be installed into a torcx store. By default, all supported images are built, but a list of images can be given as command arguments. Note that their order matters, since the version specified last will get the default reference symlink. " show_help_if_requested "$@" # The following options are advanced options, only available to those willing # to read the source code. They are not shown in help output, since they are # not needed for the typical developer workflow. DEFINE_integer build_attempt 1 \ "The build attempt for this image build." DEFINE_string group developer \ "The update group." DEFINE_string version '' \ "Overrides version number in name to this version." # Parse command line FLAGS "$@" || exit 1 eval set -- "${FLAGS_ARGV}" # Only now can we die on error. shflags functions leak non-zero error codes, # so will die prematurely if 'switch_to_strict_mode' is specified before now. switch_to_strict_mode # Initialize upload options check_gsutil_opts # Define BUILD_DIR and set_build_symlinks. . "${BUILD_LIBRARY_DIR}/toolchain_util.sh" || exit 1 . "${BUILD_LIBRARY_DIR}/board_options.sh" || exit 1 . "${BUILD_LIBRARY_DIR}/build_image_util.sh" || exit 1 . "${BUILD_LIBRARY_DIR}/torcx_manifest.sh" || exit 1 TORCX_CAS_ROOT="${FLAGS_output_root}/pkgs/${BOARD}" # Print the first level of runtime dependencies for a torcx meta-package. function torcx_dependencies() ( pkg=${1:?} ebuild=$(equery-${BOARD} w "${pkg}") function inherit() { : ; } . "${ebuild}" echo ${RDEPEND} ) # Build and install a package configured as part of a torcx image. function torcx_build() ( pkg=${2:?} tmproot=${1:?} export LDFLAGS=-Wl,-rpath,/ORIGIN/../lib export PKGDIR="${tmproot}/var/lib/portage/pkgs" # Allow the meta-package to install bashrc to customize the builds. [ -s "${tmproot}/etc/portage/bashrc" ] && . "${tmproot}/etc/portage/bashrc" emerge-${BOARD} \ --nodeps \ --oneshot \ --root="${tmproot}" \ --root-deps=rdeps \ --quiet \ "${pkg}" ) # Create a torcx image from the given meta-package. function torcx_package() { local pkg="app-torcx/${1##*/}" local name=${pkg%-[0-9]*} local version=${pkg:${#name}+1} local manifest_path="${2}" local type="${3}" local deppkg digest file rpath sha512sum source_pkg rdepends tmproot tmppkgroot update_default local pkg_cas_file pkg_cas_root local pkg_locations=() local name=${name##*/} local version=${version%%-r*} # Set up the base package layout to dump everything into /bin and /lib. # tmproot is what the packages are installed into. # A subset of the files from tmproot are then moved into tmppkgroot, # which is then archived and uploaded. tmproot=$(sudo mktemp --tmpdir="${BUILD_DIR}" -d) tmppkgroot=$(sudo mktemp --tmpdir="${BUILD_DIR}" -d) trap "sudo rm -rf '${tmproot}' '${tmppkgroot}'" EXIT RETURN sudo chmod 0755 "${tmproot}" "${tmppkgroot}" sudo mkdir -p "${tmproot}"/{.torcx,bin,lib,usr} sudo ln -fns ../bin "${tmproot}/usr/bin" sudo ln -fns ../lib "${tmproot}/usr/lib" sudo ln -fns lib "${tmproot}/usr/lib64" sudo ln -fns bin "${tmproot}/usr/sbin" sudo ln -fns lib "${tmproot}/lib64" sudo ln -fns bin "${tmproot}/sbin" # Install the meta-package and its direct dependencies. for deppkg in "=${pkg}" $(torcx_dependencies "${pkg}") do torcx_build "${tmproot}" "${deppkg}" done # by convention, the first dependency in a torcx package is the primary # source package rdepends=($(torcx_dependencies "${pkg}")) source_pkg="${rdepends[0]#=}" # Pluck out shared libraries and SONAME links. sudo mv "${tmproot}"/{lib,tmplib} sudo rm -fr "${tmproot}/tmplib/debug" sudo find "${tmproot}/tmplib" -name 'lib*.so' -type l -delete sudo mkdir -p "${tmproot}/lib" sudo find "${tmproot}/tmplib" -name 'lib*.so*' \ -exec mv -t "${tmproot}/lib/" {} + # Rewrite any units for transparent activation from the torcx root. if [ -e "${tmproot}/tmplib/systemd/system" ] then sudo mkdir -p "${tmproot}/lib/systemd" sudo mv "${tmproot}/tmplib/systemd/system" \ "${tmproot}/lib/systemd/" sudo find "${tmproot}/lib/systemd/system" -type f -exec sed -i \ -e '/^\[Unit]/aRequires=torcx.target\nAfter=torcx.target' \ -e '/^\[Service]/aEnvironmentFile=/run/metadata/torcx' \ -e 's,/usr/s\?bin/,${TORCX_BINDIR}/,g' \ -e 's,^\([^ ]*=\)\(.{TORCX_BINDIR}\)/,\1/usr/bin/env PATH=\2:${PATH} \2/,' {} + fi # Network configuration can be installed unmodified. if [ -e "${tmproot}/tmplib/systemd/network" ] then sudo mkdir -p "${tmproot}/lib/systemd" sudo mv "${tmproot}/tmplib/systemd/network" \ "${tmproot}/lib/systemd/" fi # Rewrite RPATHs to use the real $ORIGIN value. find -H "${tmproot}"/{bin,lib} -type f | while read file do rpath=$(sudo patchelf --print-rpath "${file}" 2>/dev/null) && test "${rpath#/ORIGIN/}" != "${rpath}" && sudo patchelf --set-rpath "${rpath/#?/\$}" "${file}" : # Set $? to 0 or the pipeline fails and -e quits. done # Move anything we plan to package to its root sudo mv "${tmproot}"/{.torcx,bin,lib} "${tmppkgroot}" # Create a reproducible digest by which this package will be uploaded # and referred digest=$(casync \ --digest=sha512-256 \ --without=selinux \ --with=flag-noatime \ --what=directory \ digest "${tmppkgroot}") pkg_cas_root="${TORCX_CAS_ROOT}/${name}/${digest}" pkg_cas_file="${pkg_cas_root}/${name}:${version}.torcx.tgz" # Create the cas store if it doesn't exist if [[ ! -d "${pkg_cas_root}" ]]; then mkdir -p "${TORCX_CAS_ROOT}/${name}" # Package the installed files. tmpcas="${BUILD_DIR}/${digest}" mkdir -p "${tmpcas}" tmpfile="${tmpcas}/${name}:${version}.torcx.tgz" tar --force-local -C "${tmppkgroot}" -czf "${tmpfile}" . # atomically move the whole directory in place from a tmp locatoin mv "${tmpcas}" "${pkg_cas_root}" fi [[ -f "${pkg_cas_file}" ]] || die "${pkg_cas_file} should exist but didn't" sha512sum=$(sha512sum "${pkg_cas_file}" | awk '{print $1}') update_default=false if [[ "${type}" == "default" ]]; then update_default=true pkg_locations+=("/usr/share/torcx/store/${name}:${version}.torcx.tgz") fi if [[ "${FLAGS_upload}" -eq ${FLAGS_TRUE} ]]; then pkg_locations+=("$(download_tectonic_torcx_url "pkgs/${BOARD}/${name}/${digest}/${name}:${version}.torcx.tgz")") fi torcx_manifest::add_pkg "${manifest_path}" \ "${name}" \ "${version}" \ "sha512-${sha512sum}" \ "${digest}" \ "${source_pkg}" \ "${update_default}" \ "${pkg_locations[@]}" trap - EXIT } # This list defines every torcx image that goes into the vendor store for the # current branch's release version. Note that the default reference symlink # for each package will point at the last version specified. This can handle # swapping default package versions for different OS releases by reordering. DEFAULT_IMAGES=( =app-torcx/docker-17.06 ) # This list contains extra images which will be uploaded and included in the # generated manifest, but won't be included in the vendor store. EXTRA_IMAGES=( =app-torcx/docker-1.12 ) mkdir -p "${BUILD_DIR}" manifest_path="${BUILD_DIR}/torcx_manifest.json" torcx_manifest::create_empty "${manifest_path}" for pkg in "${@:-${DEFAULT_IMAGES[@]}}" ; do torcx_package "${pkg#=}" "${manifest_path}" "default" ; done for pkg in "${EXTRA_IMAGES[@]}" ; do torcx_package "${pkg#=}" "${manifest_path}" "extra" ; done set_build_symlinks latest "${FLAGS_group}-latest" # Upload the pkgs referenced by this manifest for pkg in $(torcx_manifest::get_pkg_names "${manifest_path}"); do for digest in $(torcx_manifest::get_digests "${manifest_path}" "${pkg}"); do # no need to sign; the manifest includes their shasum and is signed. upload_files \ 'torcx pkg' \ "${TORCX_UPLOAD_ROOT}/pkgs/${BOARD}/${pkg}/${digest}" \ "" \ "${TORCX_CAS_ROOT}/${pkg}/${digest}"/*.torcx.tgz done done # Upload the manifest # Note: the manifest is uploaded to 'UPLOAD_ROOT' rather than # 'TORCX_UPLOAD_ROOT'. # For non-release builds, those two locations will be the same, so it usually # won't matter. # However, for release builds, torcx packages may be uploaded directly to their # final location, while the manifest still has to go through build bucket in # order to get signed. sign_and_upload_files \ 'torcx manifest' \ "${UPLOAD_ROOT}/torcx/manifests/${BOARD}/${COREOS_VERSION}" \ "" \ "${manifest_path}" # vim: tabstop=8 softtabstop=4 shiftwidth=8 expandtab