From 0604b5c40a0e8085bfd06c806f746ae606dadfb5 Mon Sep 17 00:00:00 2001 From: Euan Kemp Date: Tue, 5 Sep 2017 17:39:29 -0700 Subject: [PATCH 01/11] build_library: add torcx_manifest helper This implements some basic functionality around working with torcx manifests. --- build_library/torcx_manifest.sh | 97 +++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 build_library/torcx_manifest.sh diff --git a/build_library/torcx_manifest.sh b/build_library/torcx_manifest.sh new file mode 100644 index 0000000000..fbc9cca0d0 --- /dev/null +++ b/build_library/torcx_manifest.sh @@ -0,0 +1,97 @@ +# Copyright (c) 2017 The Container Linux by CoreOS Authors. All rights +# reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# torcx_manifest.sh contains helper functions for creating, editing, and +# reading torcx manifest files. + +# create_empty creates an empty torcx manfiest at the given path. +function torcx_manifest::create_empty() { + local path="${1}" + jq '.' > "${path}" < "${path}" +} + +# get_pkg_names returns the list of packages in a given manifest. Each package +# may have one or more versions associated with it. +# +# Example: +# pkg_name_arr=($(torcx_manifest::get_pkg_names "torcx_manifest.json")) +function torcx_manifest::get_pkg_names() { + local file="${1}" + jq -r '.value.packages[].name' < "${file}" +} + +# get_digests returns the list of digests for a given package. +function torcx_manifest::get_digests() { + local file="${1}" + local name="${2}" + jq -r ".value.packages[] | select(.name == \"${name}\").versions[].casDigest" < "${file}" +} From fcef524c3a42e150a879e1f1aac51bc3605201a2 Mon Sep 17 00:00:00 2001 From: Euan Kemp Date: Mon, 4 Sep 2017 14:21:55 -0700 Subject: [PATCH 02/11] build_torcx_store: generate a torcx_manifest This modifies the `build_torcx_store` script to produce a manifest and cas-like structure of packages referenced by that manifest. It also removes the symlink creation logic (which will be re-added in build_image in coming commits). The concept of "extra packages", which are referenced in the manifest, but aren't installed in the rootfs, is also introduced. Since the logic of what to include in the rootfs is also extracted into build_image, supporting these "extra packages" isn't very complicated for this file. --- build_torcx_store | 109 +++++++++++++++++++++++++++++++++++++++------- 1 file changed, 94 insertions(+), 15 deletions(-) diff --git a/build_torcx_store b/build_torcx_store index 5cb68f9f34..f23cc79290 100755 --- a/build_torcx_store +++ b/build_torcx_store @@ -15,7 +15,7 @@ assert_not_root_user DEFINE_string board "${DEFAULT_BOARD}" \ "The board to build packages for." DEFINE_string output_root "${DEFAULT_BUILD_ROOT}/torcx" \ - "Directory in which to place torcx stores (named by board/version)" + "Directory in which to place torcx stores and manifests (named by board/version)" # include upload options . "${BUILD_LIBRARY_DIR}/release_util.sh" || exit 1 @@ -54,6 +54,9 @@ check_gsutil_opts . "${BUILD_LIBRARY_DIR}/toolchain_util.sh" || exit 1 . "${BUILD_LIBRARY_DIR}/board_options.sh" || exit 1 . "${BUILD_LIBRARY_DIR}/build_image_util.sh" || exit 1 +. "${BUILD_LIBRARY_DIR}/torcx_manifest.sh" || exit 1 + +TORCX_CAS_ROOT="${FLAGS_output_root}/pkgs/${BOARD}" # Print the first level of runtime dependencies for a torcx meta-package. function torcx_dependencies() ( @@ -90,14 +93,22 @@ function torcx_package() { local pkg="app-torcx/${1##*/}" local name=${pkg%-[0-9]*} local version=${pkg:${#name}+1} - local deppkg file rpath tmproot - name=${name##*/} - version=${version%%-r*} + local manifest_path="${2}" + local type="${3}" + local deppkg digest file rpath sha512sum source_pkg tmproot tmppkgroot update_default + local pkg_cas_file pkg_cas_root + local pkg_locations=() + local name=${name##*/} + local version=${version%%-r*} # Set up the base package layout to dump everything into /bin and /lib. + # tmproot is what the packages are installed into. + # A subset of the files from tmproot are then moved into tmppkgroot, + # which is then archived and uploaded. tmproot=$(sudo mktemp --tmpdir="${BUILD_DIR}" -d) - trap "sudo rm -rf '${tmproot}'" EXIT RETURN - sudo chmod 0755 "${tmproot}" + tmppkgroot=$(sudo mktemp --tmpdir="${BUILD_DIR}" -d) + trap "sudo rm -rf '${tmproot}' '${tmppkgroot}'" EXIT RETURN + sudo chmod 0755 "${tmproot}" "${tmppkgroot}" sudo mkdir -p "${tmproot}"/{.torcx,bin,lib,usr} sudo ln -fns ../bin "${tmproot}/usr/bin" sudo ln -fns ../lib "${tmproot}/usr/lib" @@ -109,6 +120,9 @@ function torcx_package() { # Install the meta-package and its direct dependencies. for deppkg in "=${pkg}" $(torcx_dependencies "${pkg}") do + # The first file in an app-torcx package is, by convention, + # considered the source package. + [ -z "${source_pkg}" ] && source_pkg="${deppkg#=}" torcx_build "${tmproot}" "${deppkg}" done @@ -151,10 +165,50 @@ function torcx_package() { : # Set $? to 0 or the pipeline fails and -e quits. done - # Package the installed files. - file="${BUILD_DIR}/${name}:${version}.torcx.tgz" - tar --force-local -C "${tmproot}" -czf "${file}" .torcx bin lib - ln -fns "${file##*/}" "${BUILD_DIR}/${name}:com.coreos.cl.torcx.tgz" + # Move anything we plan to package to its root + sudo mv "${tmproot}"/{.torcx,bin,lib} "${tmppkgroot}" + + # Create a reproducible digest by which this package will be uploaded + # and referred + digest=$(casync \ + --digest=sha512-256 \ + --without=selinux \ + --with=flag-noatime \ + --what=directory \ + digest "${tmppkgroot}") + + pkg_cas_root="${TORCX_CAS_ROOT}/${name}/${digest}" + pkg_cas_file="${pkg_cas_root}/${name}:${version}.torcx.tgz" + + # Create the cas store if it doesn't exist + if [[ ! -d "${pkg_cas_root}" ]]; then + mkdir -p "${TORCX_CAS_ROOT}/${name}" + # Package the installed files. + tmpcas="${BUILD_DIR}/${digest}" + mkdir -p "${tmpcas}" + tmpfile="${tmpcas}/${name}:${version}.torcx.tgz" + tar --force-local -C "${tmppkgroot}" -czf "${tmpfile}" . + # atomically move the whole directory in place from a tmp locatoin + mv "${tmpcas}" "${pkg_cas_root}" + fi + [[ -f "${pkg_cas_file}" ]] || die "${pkg_cas_file} should exist but didn't" + sha512sum=$(sha512sum "${pkg_cas_file}" | awk '{print $1}') + + update_default=false + if [[ "${type}" == "default" ]]; then + update_default=true + pkg_locations+=("/usr/share/torcx/store/${name}:${version}.torcx.tgz") + fi + # TODO: add upload bucket path to pkg_locations + torcx_manifest::add_pkg "${manifest_path}" \ + "${name}" \ + "${version}" \ + "sha512-${sha512sum}" \ + "${digest}" \ + "${source_pkg}" \ + "${update_default}" \ + "${pkg_locations[@]}" + trap - EXIT } @@ -166,13 +220,38 @@ DEFAULT_IMAGES=( =app-torcx/docker-17.06 ) +# This list contains extra images which will be uploaded and included in the +# generated manifest, but won't be included in the vendor store. +EXTRA_IMAGES=( + =app-torcx/docker-1.12 +) + mkdir -p "${BUILD_DIR}" -for pkg in "${@:-${DEFAULT_IMAGES[@]}}" ; do torcx_package "${pkg#=}" ; done +manifest_path="${BUILD_DIR}/torcx_manifest.json" +torcx_manifest::create_empty "${manifest_path}" +for pkg in "${@:-${DEFAULT_IMAGES[@]}}" ; do torcx_package "${pkg#=}" "${manifest_path}" "default" ; done +for pkg in "${EXTRA_IMAGES[@]}" ; do torcx_package "${pkg#=}" "${manifest_path}" "extra" ; done set_build_symlinks latest "${FLAGS_group}-latest" +# Upload the pkgs referenced by this manifest +for pkg in $(torcx_manifest::get_pkg_names "${manifest_path}"); do + pkg_name="${pkgs[0]}" + for digest in $(torcx_manifest::get_digests "${manifest_path}" "${pkg}"); do + # no need to sign; the manifest includes their shasum and is signed. + upload_files \ + 'torcx pkg' \ + "${UPLOAD_ROOT}/torcx/pkg/${BOARD}/${pkg_name}/${digest}" \ + "" \ + "${TORCX_CAS_ROOT}/${pkg_name}/${digest}"/*.torcx.tgz + done +done + +# Upload the manifest sign_and_upload_files \ - 'torcx images' \ - "${UPLOAD_ROOT}/boards/${BOARD}/${COREOS_VERSION}" \ - torcx/ \ - "${BUILD_DIR}"/*.torcx.tgz + 'torcx manifest' \ + "${UPLOAD_ROOT}/torcx/manifests/${BOARD}/${COREOS_VERSION}" \ + "" \ + "${manifest_path}" + +# vim: tabstop=8 softtabstop=4 shiftwidth=8 expandtab From 8f712b2af14d3e14d1d11daabf2129f24bf5f77d Mon Sep 17 00:00:00 2001 From: Euan Kemp Date: Wed, 6 Sep 2017 13:42:08 -0700 Subject: [PATCH 03/11] build_library: add torcx upload/download roots Torcx is special in that it wishes to be uploaded under a prefixed directory (torcx), typically wishes to be downloaded from there, but ultimately wants to be downloaded from a location without that prefix. In fact, I expect during a normal release process, it will be uploaded with that prefix to the build bucket, copied without that prefix to the final bucket (during pre-release), and then finally downloaded without the prefix. I think this set of variables ends up being the cleanest way to represent this complexity. --- build_library/release_util.sh | 42 +++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/build_library/release_util.sh b/build_library/release_util.sh index b2dde823c5..c48576f286 100644 --- a/build_library/release_util.sh +++ b/build_library/release_util.sh @@ -5,12 +5,14 @@ GSUTIL_OPTS= UPLOAD_ROOT= UPLOAD_PATH= +TORCX_UPLOAD_ROOT= UPLOAD_DEFAULT=${FLAGS_FALSE} # Default upload root can be overridden from the environment. _user="${USER}" [[ ${USER} == "root" ]] && _user="${SUDO_USER}" : ${COREOS_UPLOAD_ROOT:=gs://users.developer.core-os.net/${_user}} +: ${COREOS_TORCX_UPLOAD_ROOT:=${COREOS_UPLOAD_ROOT}/torcx} unset _user IMAGE_ZIPPER="lbzip2 --compress --keep" @@ -28,6 +30,12 @@ DEFINE_string download_root "" \ "HTTP download prefix, board/version/etc will be appended." DEFINE_string download_path "" \ "HTTP download path, overrides --download_root." +DEFINE_string torcx_upload_root "${COREOS_TORCX_UPLOAD_ROOT}" \ + "Tectonic torcx package and manifest Upload prefix. Must be a gs:// URL." +DEFINE_string tectonic_torcx_download_root "" \ + "HTTP download prefix for tectonic torcx packages and manifests." +DEFINE_string tectonic_torcx_download_path "" \ + "HTTP download path, overrides --tectonic_torcx_download_root." DEFINE_string sign "" \ "Sign all files to be uploaded with the given GPG key." DEFINE_string sign_digests "" \ @@ -48,6 +56,14 @@ check_gsutil_opts() { UPLOAD_ROOT="${FLAGS_upload_root%%/}" fi + if [[ -n "${FLAGS_torcx_upload_root}" ]]; then + if [[ "${FLAGS_torcx_upload_root}" != gs://* ]]; then + die_notrace "--torcx_upload_root must be a gs:// URL" + fi + # Make sure the path doesn't end with a slash + TORCX_UPLOAD_ROOT="${FLAGS_torcx_upload_root%%/}" + fi + if [[ -n "${FLAGS_upload_path}" ]]; then if [[ "${FLAGS_upload_path}" != gs://* ]]; then die_notrace "--upload_path must be a gs:// URL" @@ -229,3 +245,29 @@ download_image_url() { echo "${download_path}/$1" } + +# Translate the configured torcx upload URL to a download url +# This is similar to the download_image_url, other than assuming the release +# bucket is the tectonic_torcx one. +download_tectonic_torcx_url() { + if [[ ${FLAGS_upload} -ne ${FLAGS_TRUE} ]]; then + echo "$1" + return 0 + fi + + local download_root="${FLAGS_tectonic_torcx_download_root:-${TORCX_UPLOAD_ROOT}}" + + local download_path + if [[ -n "${FLAGS_tectonic_torcx_download_path}" ]]; then + download_path="${FLAGS_tectonic_torcx_download_path%%/}" + else + download_path="${download_root%%/}" + fi + + # Just in case download_root was set from UPLOAD_ROOT + if [[ "${download_path}" == gs://* ]]; then + download_path="http://${download_path#gs://}" + fi + + echo "${download_path}/$1" +} From c196be8798323bae0d74c7e2c44908da36845a87 Mon Sep 17 00:00:00 2001 From: Euan Kemp Date: Wed, 6 Sep 2017 13:46:13 -0700 Subject: [PATCH 04/11] build_torcx_store: add package 'url' location This also moves the 'torcx' prefixing logic over to the torcx upload root introduced in the release util library. It also corrects a bug in how the source package was being determined. --- build_torcx_store | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/build_torcx_store b/build_torcx_store index f23cc79290..27cf9f6eaa 100755 --- a/build_torcx_store +++ b/build_torcx_store @@ -95,7 +95,7 @@ function torcx_package() { local version=${pkg:${#name}+1} local manifest_path="${2}" local type="${3}" - local deppkg digest file rpath sha512sum source_pkg tmproot tmppkgroot update_default + local deppkg digest file rpath sha512sum source_pkg rdepends tmproot tmppkgroot update_default local pkg_cas_file pkg_cas_root local pkg_locations=() local name=${name##*/} @@ -120,12 +120,14 @@ function torcx_package() { # Install the meta-package and its direct dependencies. for deppkg in "=${pkg}" $(torcx_dependencies "${pkg}") do - # The first file in an app-torcx package is, by convention, - # considered the source package. - [ -z "${source_pkg}" ] && source_pkg="${deppkg#=}" torcx_build "${tmproot}" "${deppkg}" done + # by convention, the first dependency in a torcx package is the primary + # source package + rdepends=($(torcx_dependencies "${pkg}")) + source_pkg="${rdepends[0]#=}" + # Pluck out shared libraries and SONAME links. sudo mv "${tmproot}"/{lib,tmplib} sudo rm -fr "${tmproot}/tmplib/debug" @@ -199,7 +201,9 @@ function torcx_package() { update_default=true pkg_locations+=("/usr/share/torcx/store/${name}:${version}.torcx.tgz") fi - # TODO: add upload bucket path to pkg_locations + if [[ "${FLAGS_upload}" -eq ${FLAGS_TRUE} ]]; then + pkg_locations+=("$(download_tectonic_torcx_url "pkg/${BOARD}/${name}/${digest}/${name}:${version}.torcx.tgz")") + fi torcx_manifest::add_pkg "${manifest_path}" \ "${name}" \ "${version}" \ @@ -236,21 +240,20 @@ set_build_symlinks latest "${FLAGS_group}-latest" # Upload the pkgs referenced by this manifest for pkg in $(torcx_manifest::get_pkg_names "${manifest_path}"); do - pkg_name="${pkgs[0]}" for digest in $(torcx_manifest::get_digests "${manifest_path}" "${pkg}"); do # no need to sign; the manifest includes their shasum and is signed. upload_files \ 'torcx pkg' \ - "${UPLOAD_ROOT}/torcx/pkg/${BOARD}/${pkg_name}/${digest}" \ + "${TORCX_UPLOAD_ROOT}/pkg/${BOARD}/${pkg}/${digest}" \ "" \ - "${TORCX_CAS_ROOT}/${pkg_name}/${digest}"/*.torcx.tgz + "${TORCX_CAS_ROOT}/${pkg}/${digest}"/*.torcx.tgz done done # Upload the manifest sign_and_upload_files \ 'torcx manifest' \ - "${UPLOAD_ROOT}/torcx/manifests/${BOARD}/${COREOS_VERSION}" \ + "${TORCX_UPLOAD_ROOT}/manifests/${BOARD}/${COREOS_VERSION}" \ "" \ "${manifest_path}" From 31d7b40af3960b699330a76c1f1e8e503c05e07a Mon Sep 17 00:00:00 2001 From: Euan Kemp Date: Wed, 6 Sep 2017 15:21:38 -0700 Subject: [PATCH 05/11] build_library: add more torcx_manifest helpers These are useful for the build_image_util code soon to come. --- build_library/torcx_manifest.sh | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/build_library/torcx_manifest.sh b/build_library/torcx_manifest.sh index fbc9cca0d0..3d7a7e8771 100644 --- a/build_library/torcx_manifest.sh +++ b/build_library/torcx_manifest.sh @@ -89,9 +89,41 @@ function torcx_manifest::get_pkg_names() { jq -r '.value.packages[].name' < "${file}" } +# local_store_path returns the in-container-linux store path a given package + +# version combination should exist at. It returns the empty string if the +# package shouldn't exist on disk. +function torcx_manifest::local_store_path() { + local file="${1}" + local name="${2}" + local version="${3}" + jq -r ".value.packages[] | select(.name == \"${name}\") | .versions[] | select(.version == \"${version}\") | .locations[].path" < "${file}" +} + +# get_digest returns the cas digest for a given package version +function torcx_manifest::get_digest() { + local file="${1}" + local name="${2}" + local version="${3}" + jq -r ".value.packages[] | select(.name == \"${name}\") | .versions[] | select(.version == \"${version}\") | .casDigest" < "${file}" +} + # get_digests returns the list of digests for a given package. function torcx_manifest::get_digests() { local file="${1}" local name="${2}" jq -r ".value.packages[] | select(.name == \"${name}\").versions[].casDigest" < "${file}" } + +# get_versions returns the list of versions for a given package. +function torcx_manifest::get_versions() { + local file="${1}" + local name="${2}" + jq -r ".value.packages[] | select(.name == \"${name}\").versions[].version" < "${file}" +} + +# default_version returns the default version for a given package, or an empty string if there isn't one. +function torcx_manifest::default_version() { + local file="${1}" + local name="${2}" + jq -r ".value.packages[] | select(.name == \"${name}\").defaultVersion" < "${file}" +} From d3867403ed523f7b44011b9157a81dcf6608d5d4 Mon Sep 17 00:00:00 2001 From: Euan Kemp Date: Tue, 5 Sep 2017 18:06:30 -0700 Subject: [PATCH 06/11] build_image: populate torcx store from manifest This moves the default symlinking logic into build image as well. This assumes that a torcx store is available locally with all images referenced in the torcx manifest. This is accomplished with a highly-indented double-for-loop, but I think it's still decently readable. --- build_image | 11 +++++++---- build_library/build_image_util.sh | 23 +++++++++++++++++++---- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/build_image b/build_image index c9d80c9e6c..b7e63861b4 100755 --- a/build_image +++ b/build_image @@ -32,8 +32,10 @@ DEFINE_string base_pkg "coreos-base/coreos" \ "The base portage package to base the build off of (only applies to prod images)" DEFINE_string base_dev_pkg "coreos-base/coreos-dev" \ "The base portage package to base the build off of (only applies to dev images)" -DEFINE_string torcx_store "${DEFAULT_BUILD_ROOT}/torcx/${DEFAULT_BOARD}/latest" \ - "Directory of torcx images to copy into the vendor store (or blank for none)" +DEFINE_string torcx_manifest "${DEFAULT_BUILD_ROOT}/torcx/${DEFAULT_BOARD}/latest/torcx_manifest.json" \ + "The torcx manifest describing torcx packages for this image (or blank for none)" +DEFINE_string torcx_root "${DEFAULT_BUILD_ROOT}/torcx" \ + "Directory in which torcx packages can be found" DEFINE_string output_root "${DEFAULT_BUILD_ROOT}/images" \ "Directory in which to place image result directories (named by version)" DEFINE_string disk_layout "" \ @@ -89,8 +91,8 @@ switch_to_strict_mode check_gsutil_opts # Patch around default values not being able to depend on other flags. -if [ "x${FLAGS_torcx_store}" = "x${DEFAULT_BUILD_ROOT}/torcx/${DEFAULT_BOARD}/latest" ]; then - FLAGS_torcx_store="${DEFAULT_BUILD_ROOT}/torcx/${FLAGS_board}/latest" +if [ "x${FLAGS_torcx_manifest}" = "x${DEFAULT_BUILD_ROOT}/torcx/${DEFAULT_BOARD}/latest/torcx_manifest.json" ]; then + FLAGS_torcx_manifest="${DEFAULT_BUILD_ROOT}/torcx/${FLAGS_board}/latest/torcx_manifest.json" fi # If downloading packages is enabled ensure the board is configured properly. @@ -107,6 +109,7 @@ fi . "${BUILD_LIBRARY_DIR}/prod_image_util.sh" || exit 1 . "${BUILD_LIBRARY_DIR}/dev_image_util.sh" || exit 1 . "${BUILD_LIBRARY_DIR}/test_image_content.sh" || exit 1 +. "${BUILD_LIBRARY_DIR}/torcx_manifest.sh" || exit 1 . "${BUILD_LIBRARY_DIR}/vm_image_util.sh" || exit 1 PROD_IMAGE=0 diff --git a/build_library/build_image_util.sh b/build_library/build_image_util.sh index def5ab03c8..c70e2b0fe1 100755 --- a/build_library/build_image_util.sh +++ b/build_library/build_image_util.sh @@ -424,10 +424,25 @@ finish_image() { local install_grub=0 local disk_img="${BUILD_DIR}/${image_name}" - # Copy in a vendor torcx store if requested. - if [ -n "${FLAGS_torcx_store}" ]; then - sudo cp -dt "${root_fs_dir}"/usr/share/torcx/store \ - "${FLAGS_torcx_store}"/*.torcx.tgz + # Copy in packages from the torcx store that are marked as being on disk + if [ -n "${FLAGS_torcx_manifest}" ]; then + for pkg in $(torcx_manifest::get_pkg_names "${FLAGS_torcx_manifest}"); do + local default_version="$(torcx_manifest::default_version "${FLAGS_torcx_manifest}" "${pkg}")" + for version in $(torcx_manifest::get_versions "${FLAGS_torcx_manifest}" "${pkg}"); do + local on_disk_path="$(torcx_manifest::local_store_path "${FLAGS_torcx_manifest}" "${pkg}" "${version}")" + if [[ -n "${on_disk_path}" ]]; then + local casDigest="$(torcx_manifest::get_digest "${FLAGS_torcx_manifest}" "${pkg}" "${version}")" + sudo cp "${FLAGS_torcx_root}/pkgs/${BOARD}/${pkg}/${casDigest}/${pkg}:${version}.torcx.tgz" \ + "${root_fs_dir}${on_disk_path}" + + if [[ "${version}" == "${default_version}" ]]; then + # Create the default symlink for this package + sudo ln -fns "${on_disk_path##*/}" \ + "${root_fs_dir}/${on_disk_path%/*}/${pkg}:com.coreos.cl.torcx.tgz" + fi + fi + done + done fi # Only enable rootfs verification on prod builds. From 1989ff96d8369f77e0e14756d96b653b5bee9846 Mon Sep 17 00:00:00 2001 From: Euan Kemp Date: Wed, 6 Sep 2017 16:22:39 -0700 Subject: [PATCH 07/11] build_torcx_store: upload to 'pkgs' This is consistent with the local directory hierarchy --- build_torcx_store | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build_torcx_store b/build_torcx_store index 27cf9f6eaa..7a2de8451f 100755 --- a/build_torcx_store +++ b/build_torcx_store @@ -202,7 +202,7 @@ function torcx_package() { pkg_locations+=("/usr/share/torcx/store/${name}:${version}.torcx.tgz") fi if [[ "${FLAGS_upload}" -eq ${FLAGS_TRUE} ]]; then - pkg_locations+=("$(download_tectonic_torcx_url "pkg/${BOARD}/${name}/${digest}/${name}:${version}.torcx.tgz")") + pkg_locations+=("$(download_tectonic_torcx_url "pkgs/${BOARD}/${name}/${digest}/${name}:${version}.torcx.tgz")") fi torcx_manifest::add_pkg "${manifest_path}" \ "${name}" \ @@ -244,7 +244,7 @@ for pkg in $(torcx_manifest::get_pkg_names "${manifest_path}"); do # no need to sign; the manifest includes their shasum and is signed. upload_files \ 'torcx pkg' \ - "${TORCX_UPLOAD_ROOT}/pkg/${BOARD}/${pkg}/${digest}" \ + "${TORCX_UPLOAD_ROOT}/pkgs/${BOARD}/${pkg}/${digest}" \ "" \ "${TORCX_CAS_ROOT}/${pkg}/${digest}"/*.torcx.tgz done From cc23e8e49eb9a08390295e736f67db40972ee29d Mon Sep 17 00:00:00 2001 From: Euan Kemp Date: Wed, 6 Sep 2017 16:29:40 -0700 Subject: [PATCH 08/11] jenkins/packages: configure download root Pending jenkins also including this variable. --- jenkins/packages.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/jenkins/packages.sh b/jenkins/packages.sh index 6c57a8435c..70cf3fcd2b 100644 --- a/jenkins/packages.sh +++ b/jenkins/packages.sh @@ -56,6 +56,8 @@ script build_torcx_store \ --sign="${SIGNING_USER}" \ --sign_digests="${SIGNING_USER}" \ --upload_root="${UPLOAD_ROOT}" \ + --torcx_upload_root="${TORCX_PKG_DOWNLOAD_ROOT}" \ + --tectonic_torcx_download_root="${TECTONIC_TORCX_DOWNLOAD_ROOT}" \ --upload enter ccache --show-stats From f496072a10373a2d1e94c003edc3475f42dec67c Mon Sep 17 00:00:00 2001 From: Euan Kemp Date: Wed, 6 Sep 2017 16:30:41 -0700 Subject: [PATCH 09/11] jenkins/images: use torcx packages from manifest This accompanies the other changes around moving to a torcx manifest. --- jenkins/images.sh | 39 +++++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/jenkins/images.sh b/jenkins/images.sh index dea4784f66..3a48624364 100644 --- a/jenkins/images.sh +++ b/jenkins/images.sh @@ -40,26 +40,28 @@ else script set_official --board="${BOARD}" --noofficial fi -# Retrieve this version's torcx vendor store. +# Retrieve this version's torcx manifest +mkdir -p torcx/pkgs enter gsutil cp -r \ - "${DOWNLOAD_ROOT}/boards/${BOARD}/${COREOS_VERSION}/torcx" \ - /mnt/host/source/ -for image in torcx/*.torcx.tgz -do - gpg --verify "${image}.sig" -done + "${DOWNLOAD_ROOT}/torcx/manifests/${BOARD}/${COREOS_VERSION}/torcx_manifest.json"{,.sig} \ + /mnt/host/source/torcx/ +gpg --verify torcx/torcx_manifest.json.sig -# Work around the lack of symlink support in GCS. -shopt -s nullglob -for default in torcx/*:com.coreos.cl.torcx.tgz +# Download all cas references from the manifest and verify their checksums +# TODO: technically we can skip ones that don't have a 'path' since they're not +# included in the image. +while read name digest hash do - for image in torcx/*.torcx.tgz - do - [ "x${default}" != "x${image}" ] && - cmp --silent -- "${default}" "${image}" && - ln -fns "${image##*/}" "${default}" - done -done + mkdir -p "torcx/pkgs/${BOARD}/${name}/${digest}" + enter gsutil cp -r "${TORCX_PKG_DOWNLOAD_ROOT}/pkgs/${BOARD}/${name}/${digest}" \ + "/mnt/host/source/torcx/pkgs/${BOARD}/${name}/" + downloaded_hash=$(sha512sum "torcx/pkgs/${BOARD}/${name}/${digest}/"*.torcx.tgz | awk '{print $1}') + if [[ "sha512-${downloaded_hash}" != "${hash}" ]] + then + echo "Torcx package had wrong hash: ${downloaded_hash} instead of ${hash}" + exit 1 + fi +done < <(jq -r '.value.packages[] | . as $p | .name as $n | $p.versions[] | [.casDigest, .hash] | join(" ") | [$n, .] | join(" ")' "torcx/torcx_manifest.json") script build_image \ --board="${BOARD}" \ @@ -68,6 +70,7 @@ script build_image \ --getbinpkgver="${COREOS_VERSION}" \ --sign="${SIGNING_USER}" \ --sign_digests="${SIGNING_USER}" \ - --torcx_store=/mnt/host/source/torcx \ + --torcx_manifest=/mnt/host/source/torcx/torcx_manifest.json \ + --torcx_root=/mnt/host/source/torcx/ \ --upload_root="${UPLOAD_ROOT}" \ --upload prod container From ee541815032ecc3af2bc79cb1e435b4d1fab749f Mon Sep 17 00:00:00 2001 From: Euan Kemp Date: Wed, 6 Sep 2017 18:00:47 -0700 Subject: [PATCH 10/11] build_torcx_store: upload manifests to upload_root Rather than to a location special to them. The comment in the code explains the reasoning for this. --- build_torcx_store | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/build_torcx_store b/build_torcx_store index 7a2de8451f..8099f45d97 100755 --- a/build_torcx_store +++ b/build_torcx_store @@ -251,9 +251,16 @@ for pkg in $(torcx_manifest::get_pkg_names "${manifest_path}"); do done # Upload the manifest +# Note: the manifest is uploaded to 'UPLOAD_ROOT' rather than +# 'TORCX_UPLOAD_ROOT'. +# For non-release builds, those two locations will be the same, so it usually +# won't matter. +# However, for release builds, torcx packages may be uploaded directly to their +# final location, while the manifest still has to go through build bucket in +# order to get signed. sign_and_upload_files \ 'torcx manifest' \ - "${TORCX_UPLOAD_ROOT}/manifests/${BOARD}/${COREOS_VERSION}" \ + "${UPLOAD_ROOT}/torcx/manifests/${BOARD}/${COREOS_VERSION}" \ "" \ "${manifest_path}" From 677b5ae116dc07216041eb396641695ddea7f4b1 Mon Sep 17 00:00:00 2001 From: Euan Kemp Date: Fri, 8 Sep 2017 11:59:58 -0700 Subject: [PATCH 11/11] build_torcx_store: back-out reproducible digests Per the comment there, they were implemented in a broken fashion. This leaves the door open for using them in the future, but in the meanwhile simply uses the sha512sum as the digest (which solves the immediate issue). --- build_torcx_store | 41 +++++++++++++++++++---------------------- 1 file changed, 19 insertions(+), 22 deletions(-) diff --git a/build_torcx_store b/build_torcx_store index 8099f45d97..3cf2651429 100755 --- a/build_torcx_store +++ b/build_torcx_store @@ -170,31 +170,28 @@ function torcx_package() { # Move anything we plan to package to its root sudo mv "${tmproot}"/{.torcx,bin,lib} "${tmppkgroot}" - # Create a reproducible digest by which this package will be uploaded - # and referred - digest=$(casync \ - --digest=sha512-256 \ - --without=selinux \ - --with=flag-noatime \ - --what=directory \ - digest "${tmppkgroot}") + tmpfile="${BUILD_DIR}/${name}:${version}.torcx.tgz" + tar --force-local -C "${tmppkgroot}" -czf "${tmpfile}" . + sha512sum=$(sha512sum "${tmpfile}" | awk '{print $1}') + + # TODO(euank): this opaque digest, if it were reproducible, could save + # users from having to download things that haven't changed. + # For now, use the sha512sum of the final image. + # Ideally we should move to something more like a casync digest or tarsum. + # The reason this is currently not being done is because to do that we + # *MUST* ensure that a given pair of (digest, sha512sum) referenced in + # a previous torcx package remains correct. + # Because this code, as written, clobbers existing things with the same + # digest (but the sha512sum of the .torcx.tgz can differ, e.g. due to ctime) + # that property doesn't hold. + # To switch this back to a reprodicble digest, we *must* never clobber + # existing objects (and thus re-use their sha512sum here). + digest="${sha512sum}" pkg_cas_root="${TORCX_CAS_ROOT}/${name}/${digest}" pkg_cas_file="${pkg_cas_root}/${name}:${version}.torcx.tgz" - - # Create the cas store if it doesn't exist - if [[ ! -d "${pkg_cas_root}" ]]; then - mkdir -p "${TORCX_CAS_ROOT}/${name}" - # Package the installed files. - tmpcas="${BUILD_DIR}/${digest}" - mkdir -p "${tmpcas}" - tmpfile="${tmpcas}/${name}:${version}.torcx.tgz" - tar --force-local -C "${tmppkgroot}" -czf "${tmpfile}" . - # atomically move the whole directory in place from a tmp locatoin - mv "${tmpcas}" "${pkg_cas_root}" - fi - [[ -f "${pkg_cas_file}" ]] || die "${pkg_cas_file} should exist but didn't" - sha512sum=$(sha512sum "${pkg_cas_file}" | awk '{print $1}') + mkdir -p "${pkg_cas_root}" + mv "${tmpfile}" "${pkg_cas_file}" update_default=false if [[ "${type}" == "default" ]]; then