From 8e214f577919bddb96810a62efe987b4ef3ae630 Mon Sep 17 00:00:00 2001 From: David Michael Date: Thu, 18 May 2017 17:55:26 -0700 Subject: [PATCH 1/3] build_torcx_store: add a command for building torcx images This creates new torcx stores under src/build/torcx, similar to the layout of src/build/images. The script contains a list of packages to build so there is a deterministic set that gets branched with this repository. For development, a different list can also be given on the command line. Images are created by building only the direct dependencies of meta-packages under the app-torcx category. They use the board root as the sysroot for cross-compiling. The base layout of the installation root is structured so everything is under either /bin or /lib for simplicity. Any systemd units are rewritten so they depend on a successful torcx apply, and they inject the appropriate torcx runtime directory into the PATH. When ELF binaries are compiled, they are given an RPATH value of /ORIGIN/../lib which gets rewritten to $ORIGIN/../lib. The final value enables automatic dynamic linking with packaged libraries. The intermediate value is to avoid having to escape the $ through the various build system layers (which are different for each package) and it pretends to be an absolute path to silence security warnings. --- build_torcx_store | 174 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 174 insertions(+) create mode 100755 build_torcx_store diff --git a/build_torcx_store b/build_torcx_store new file mode 100755 index 0000000000..43f7fe1fc7 --- /dev/null +++ b/build_torcx_store @@ -0,0 +1,174 @@ +#!/bin/bash + +# Copyright (c) 2017 The CoreOS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +. "$(dirname "$0")/common.sh" || exit 1 + +# Script must run inside the chroot +restart_in_chroot_if_needed "$@" + +assert_not_root_user + +# Developer-visible flags. +DEFINE_string board "${DEFAULT_BOARD}" \ + "The board to build packages for." +DEFINE_string output_root "${DEFAULT_BUILD_ROOT}/torcx" \ + "Directory in which to place torcx stores (named by board/version)" + +# include upload options +. "${BUILD_LIBRARY_DIR}/release_util.sh" || exit 1 + +FLAGS_HELP="usage: $(basename $0) [flags] [images] + +This script builds a collection of torcx images to be installed into a torcx +store. By default, all supported images are built, but a list of images can be +given as command arguments. Note that their order matters, since the version +specified last will get the default reference symlink. +" +show_help_if_requested "$@" + +# The following options are advanced options, only available to those willing +# to read the source code. They are not shown in help output, since they are +# not needed for the typical developer workflow. +DEFINE_integer build_attempt 1 \ + "The build attempt for this image build." +DEFINE_string group developer \ + "The update group." +DEFINE_string version '' \ + "Overrides version number in name to this version." + +# Parse command line +FLAGS "$@" || exit 1 +eval set -- "${FLAGS_ARGV}" + +# Only now can we die on error. shflags functions leak non-zero error codes, +# so will die prematurely if 'switch_to_strict_mode' is specified before now. +switch_to_strict_mode + +# Define BUILD_DIR and set_build_symlinks. +. "${BUILD_LIBRARY_DIR}/toolchain_util.sh" || exit 1 +. "${BUILD_LIBRARY_DIR}/board_options.sh" || exit 1 +. "${BUILD_LIBRARY_DIR}/build_image_util.sh" || exit 1 + +# Print the first level of runtime dependencies for a torcx meta-package. +function torcx_dependencies() ( + pkg=${1:?} + ebuild=$(equery-${BOARD} w "${pkg}") + function inherit() { : ; } + . "${ebuild}" + echo ${RDEPEND} +) + +# Build and install a package configured as part of a torcx image. +function torcx_build() ( + pkg=${2:?} + tmproot=${1:?} + + export LDFLAGS=-Wl,-rpath,/ORIGIN/../lib + export PKGDIR="${tmproot}/var/lib/portage/pkgs" + + # Allow the meta-package to install bashrc to customize the builds. + [ -s "${tmproot}/etc/portage/bashrc" ] && + . "${tmproot}/etc/portage/bashrc" + + emerge-${BOARD} \ + --nodeps \ + --oneshot \ + --root="${tmproot}" \ + --root-deps=rdeps \ + "${pkg}" +) + +# Create a torcx image from the given meta-package. +function torcx_package() { + local pkg="app-torcx/${1##*/}" + local name=${pkg%-[0-9]*} + local version=${pkg:${#name}+1} + local deppkg file rpath tmproot + name=${name##*/} + version=${version%%-r*} + + # Set up the base package layout to dump everything into /bin and /lib. + tmproot=$(sudo mktemp --tmpdir="${BUILD_DIR}" -d) + trap "sudo rm -rf '${tmproot}'" EXIT RETURN + sudo chmod 0755 "${tmproot}" + sudo mkdir -p "${tmproot}"/{.torcx,bin,lib,usr} + sudo ln -fns ../bin "${tmproot}/usr/bin" + sudo ln -fns ../lib "${tmproot}/usr/lib" + sudo ln -fns lib "${tmproot}/usr/lib64" + sudo ln -fns bin "${tmproot}/usr/sbin" + sudo ln -fns lib "${tmproot}/lib64" + sudo ln -fns bin "${tmproot}/sbin" + + # Install the meta-package and its direct dependencies. + for deppkg in "=${pkg}" $(torcx_dependencies "${pkg}") + do + torcx_build "${tmproot}" "${deppkg}" + done + + # Pluck out shared libraries and SONAME links. + sudo mv "${tmproot}"/{lib,tmplib} + sudo rm -fr "${tmproot}/tmplib/debug" + sudo find "${tmproot}/tmplib" -name 'lib*.so' -type l -delete + sudo mkdir -p "${tmproot}/lib" + sudo find "${tmproot}/tmplib" -name 'lib*.so*' \ + -exec mv -t "${tmproot}/lib/" {} + + + # Rewrite any units for transparent activation from the torcx root. + if [ -e "${tmproot}/tmplib/systemd/system" ] + then + sudo mkdir -p "${tmproot}/lib/systemd" + sudo mv "${tmproot}/tmplib/systemd/system" \ + "${tmproot}/lib/systemd/" + sudo find "${tmproot}/lib/systemd/system" -type f -exec sed -i \ + -e '/^\[Unit]/aRequires=torcx.target\nAfter=torcx.target' \ + -e '/^\[Service]/aEnvironmentFile=/run/metadata/torcx' \ + -e 's,/usr/s\?bin/,${TORCX_BINDIR}/,g' \ + -e 's,^\([^ ]*=\)\(.{TORCX_BINDIR}\)/,\1/usr/bin/env PATH=\2:${PATH} \2/,' {} + + fi + + # Network configuration can be installed unmodified. + if [ -e "${tmproot}/tmplib/systemd/network" ] + then + sudo mkdir -p "${tmproot}/lib/systemd" + sudo mv "${tmproot}/tmplib/systemd/network" \ + "${tmproot}/lib/systemd/" + fi + + # Rewrite RPATHs to use the real $ORIGIN value. + find -H "${tmproot}"/{bin,lib} -type f | + while read file + do + rpath=$(sudo patchelf --print-rpath "${file}" 2>/dev/null) && + test "${rpath#/ORIGIN/}" != "${rpath}" && + sudo patchelf --set-rpath "${rpath/#?/\$}" "${file}" + : # Set $? to 0 or the pipeline fails and -e quits. + done + + # Package the installed files. + file="${BUILD_DIR}/${name}:${version}.torcx.tgz" + tar --force-local -C "${tmproot}" -czf "${file}" .torcx bin lib + ln -fns "${file##*/}" "${BUILD_DIR}/${name}:com.coreos.cl.torcx.tgz" + trap - EXIT +} + +# This list defines every torcx image that goes into the vendor store for the +# current branch's release version. Note that the default reference symlink +# for each package will point at the last version specified. This can handle +# swapping default package versions for different OS releases by reordering. +DEFAULT_IMAGES=( + =app-torcx/docker-17.05 +) + +mkdir -p "${BUILD_DIR}" +for pkg in "${@:-${DEFAULT_IMAGES[@]}}" ; do torcx_package "${pkg#=}" ; done + +set_build_symlinks latest "${FLAGS_group}-latest" + +sign_and_upload_files \ + 'torcx images' \ + "${UPLOAD_ROOT}/boards/${BOARD}/${COREOS_VERSION}" \ + torcx/ \ + "${BUILD_DIR}"/*.torcx.tgz From f91198ca0e7be7a3414e4367cda500683a73701c Mon Sep 17 00:00:00 2001 From: David Michael Date: Thu, 18 May 2017 18:11:11 -0700 Subject: [PATCH 2/3] build_packages: also build all torcx images by default The idea is that once the installed board packages have changed, the sysroot that was used to build any existing torcx packages may no longer be compatible. It will therefore run build_torcx_store, passing relevant options to it, to build a new torcx store of images with the current sysroot. This adds --skip_torcx_store to use the previous behavior. --- build_packages | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/build_packages b/build_packages index 9f2bfb3f4c..f869bd45ad 100755 --- a/build_packages +++ b/build_packages @@ -36,6 +36,8 @@ DEFINE_boolean skip_toolchain_update "${FLAGS_FALSE}" \ "Don't update toolchain automatically." DEFINE_boolean skip_chroot_upgrade "${FLAGS_FALSE}" \ "Don't run the chroot upgrade automatically; use with care." +DEFINE_boolean skip_torcx_store "${FLAGS_FALSE}" \ + "Don't build a new torcx store from the updated sysroot." # include upload options . "${BUILD_LIBRARY_DIR}/release_util.sh" || exit 1 @@ -251,5 +253,10 @@ test_image_content "${BOARD_ROOT}" # upload packages if enabled upload_packages +# Build a new torcx store with the updated packages, passing flags through. +if [ "${FLAGS_skip_torcx_store}" -eq "${FLAGS_FALSE}" ]; then + "${SCRIPTS_DIR}"/build_torcx_store --board="${BOARD}" +fi + info "Builds complete" command_completed From a0bebcc38b882fefb1264f9c0094eec466cf605a Mon Sep 17 00:00:00 2001 From: David Michael Date: Thu, 18 May 2017 18:17:59 -0700 Subject: [PATCH 3/3] build_image: copy torcx images into the vendor store This adds the option --torcx_store to specify the path to a directory containing torcx images to be baked into the OS image. A blank string can be given instead of a path to restore the previous behavior and leave an empty vendor store. The default value is the default path created by build_torcx_store, which is used when build_packages updates torcx images. This means that the current pattern "./build_packages && ./build_image prod" should result in a fully updated OS image with all torcx images available in the vendor store. --- build_image | 7 +++++++ build_library/build_image_util.sh | 6 ++++++ 2 files changed, 13 insertions(+) diff --git a/build_image b/build_image index d7f09fef37..c9d80c9e6c 100755 --- a/build_image +++ b/build_image @@ -32,6 +32,8 @@ DEFINE_string base_pkg "coreos-base/coreos" \ "The base portage package to base the build off of (only applies to prod images)" DEFINE_string base_dev_pkg "coreos-base/coreos-dev" \ "The base portage package to base the build off of (only applies to dev images)" +DEFINE_string torcx_store "${DEFAULT_BUILD_ROOT}/torcx/${DEFAULT_BOARD}/latest" \ + "Directory of torcx images to copy into the vendor store (or blank for none)" DEFINE_string output_root "${DEFAULT_BUILD_ROOT}/images" \ "Directory in which to place image result directories (named by version)" DEFINE_string disk_layout "" \ @@ -86,6 +88,11 @@ switch_to_strict_mode check_gsutil_opts +# Patch around default values not being able to depend on other flags. +if [ "x${FLAGS_torcx_store}" = "x${DEFAULT_BUILD_ROOT}/torcx/${DEFAULT_BOARD}/latest" ]; then + FLAGS_torcx_store="${DEFAULT_BUILD_ROOT}/torcx/${FLAGS_board}/latest" +fi + # If downloading packages is enabled ensure the board is configured properly. if [[ ${FLAGS_getbinpkg} -eq ${FLAGS_TRUE} ]]; then "${SRC_ROOT}/scripts/setup_board" --board="${FLAGS_board}" \ diff --git a/build_library/build_image_util.sh b/build_library/build_image_util.sh index 1e19fcb792..def5ab03c8 100755 --- a/build_library/build_image_util.sh +++ b/build_library/build_image_util.sh @@ -424,6 +424,12 @@ finish_image() { local install_grub=0 local disk_img="${BUILD_DIR}/${image_name}" + # Copy in a vendor torcx store if requested. + if [ -n "${FLAGS_torcx_store}" ]; then + sudo cp -dt "${root_fs_dir}"/usr/share/torcx/store \ + "${FLAGS_torcx_store}"/*.torcx.tgz + fi + # Only enable rootfs verification on prod builds. local disable_read_write="${FLAGS_FALSE}" if [[ "${IMAGE_BUILD_TYPE}" == "prod" ]]; then