Merge pull request #737 from euank/torcx-cas

rework torcx uploading to include a manifest

build_image

@@ -32,8 +32,10 @@ DEFINE_string base_pkg "coreos-base/coreos" \
   "The base portage package to base the build off of (only applies to prod images)"
 DEFINE_string base_dev_pkg "coreos-base/coreos-dev" \
   "The base portage package to base the build off of (only applies to dev images)"
-DEFINE_string torcx_store "${DEFAULT_BUILD_ROOT}/torcx/${DEFAULT_BOARD}/latest" \
+DEFINE_string torcx_manifest "${DEFAULT_BUILD_ROOT}/torcx/${DEFAULT_BOARD}/latest/torcx_manifest.json" \
-  "Directory of torcx images to copy into the vendor store (or blank for none)"
+  "The torcx manifest describing torcx packages for this image (or blank for none)"
+DEFINE_string torcx_root "${DEFAULT_BUILD_ROOT}/torcx" \
+  "Directory in which torcx packages can be found"
 DEFINE_string output_root "${DEFAULT_BUILD_ROOT}/images" \
   "Directory in which to place image result directories (named by version)"
 DEFINE_string disk_layout "" \
@@ -89,8 +91,8 @@ switch_to_strict_mode
 check_gsutil_opts
 
 # Patch around default values not being able to depend on other flags.
-if [ "x${FLAGS_torcx_store}" = "x${DEFAULT_BUILD_ROOT}/torcx/${DEFAULT_BOARD}/latest" ]; then
+if [ "x${FLAGS_torcx_manifest}" = "x${DEFAULT_BUILD_ROOT}/torcx/${DEFAULT_BOARD}/latest/torcx_manifest.json" ]; then
-  FLAGS_torcx_store="${DEFAULT_BUILD_ROOT}/torcx/${FLAGS_board}/latest"
+  FLAGS_torcx_manifest="${DEFAULT_BUILD_ROOT}/torcx/${FLAGS_board}/latest/torcx_manifest.json"
 fi
 
 # If downloading packages is enabled ensure the board is configured properly.
@@ -107,6 +109,7 @@ fi
 . "${BUILD_LIBRARY_DIR}/prod_image_util.sh" || exit 1
 . "${BUILD_LIBRARY_DIR}/dev_image_util.sh" || exit 1
 . "${BUILD_LIBRARY_DIR}/test_image_content.sh" || exit 1
+. "${BUILD_LIBRARY_DIR}/torcx_manifest.sh" || exit 1
 . "${BUILD_LIBRARY_DIR}/vm_image_util.sh" || exit 1
 
 PROD_IMAGE=0

build_library/build_image_util.sh

@@ -424,10 +424,25 @@ finish_image() {
   local install_grub=0
   local disk_img="${BUILD_DIR}/${image_name}"
 
-  # Copy in a vendor torcx store if requested.
+  # Copy in packages from the torcx store that are marked as being on disk
-  if [ -n "${FLAGS_torcx_store}" ]; then
+  if [ -n "${FLAGS_torcx_manifest}" ]; then
-    sudo cp -dt "${root_fs_dir}"/usr/share/torcx/store \
+    for pkg in $(torcx_manifest::get_pkg_names "${FLAGS_torcx_manifest}"); do
-        "${FLAGS_torcx_store}"/*.torcx.tgz
+      local default_version="$(torcx_manifest::default_version "${FLAGS_torcx_manifest}" "${pkg}")"
+      for version in $(torcx_manifest::get_versions "${FLAGS_torcx_manifest}" "${pkg}"); do
+        local on_disk_path="$(torcx_manifest::local_store_path "${FLAGS_torcx_manifest}" "${pkg}" "${version}")"
+        if [[ -n "${on_disk_path}" ]]; then
+          local casDigest="$(torcx_manifest::get_digest "${FLAGS_torcx_manifest}" "${pkg}" "${version}")"
+          sudo cp "${FLAGS_torcx_root}/pkgs/${BOARD}/${pkg}/${casDigest}/${pkg}:${version}.torcx.tgz" \
+            "${root_fs_dir}${on_disk_path}"
+
+          if [[ "${version}" == "${default_version}" ]]; then
+            # Create the default symlink for this package
+            sudo ln -fns "${on_disk_path##*/}" \
+              "${root_fs_dir}/${on_disk_path%/*}/${pkg}:com.coreos.cl.torcx.tgz"
+          fi
+        fi
+      done
+    done
   fi
 
   # Only enable rootfs verification on prod builds.

build_library/release_util.sh

@@ -5,12 +5,14 @@
 GSUTIL_OPTS=
 UPLOAD_ROOT=
 UPLOAD_PATH=
+TORCX_UPLOAD_ROOT=
 UPLOAD_DEFAULT=${FLAGS_FALSE}
 
 # Default upload root can be overridden from the environment.
 _user="${USER}"
 [[ ${USER} == "root" ]] && _user="${SUDO_USER}"
 : ${COREOS_UPLOAD_ROOT:=gs://users.developer.core-os.net/${_user}}
+: ${COREOS_TORCX_UPLOAD_ROOT:=${COREOS_UPLOAD_ROOT}/torcx}
 unset _user
 
 IMAGE_ZIPPER="lbzip2 --compress --keep"
@@ -28,6 +30,12 @@ DEFINE_string download_root "" \
   "HTTP download prefix, board/version/etc will be appended."
 DEFINE_string download_path "" \
   "HTTP download path, overrides --download_root."
+DEFINE_string torcx_upload_root "${COREOS_TORCX_UPLOAD_ROOT}" \
+  "Tectonic torcx package and manifest Upload prefix. Must be a gs:// URL."
+DEFINE_string tectonic_torcx_download_root "" \
+  "HTTP download prefix for tectonic torcx packages and manifests."
+DEFINE_string tectonic_torcx_download_path "" \
+  "HTTP download path, overrides --tectonic_torcx_download_root."
 DEFINE_string sign "" \
   "Sign all files to be uploaded with the given GPG key."
 DEFINE_string sign_digests "" \
@@ -48,6 +56,14 @@ check_gsutil_opts() {
         UPLOAD_ROOT="${FLAGS_upload_root%%/}"
     fi
 
+    if [[ -n "${FLAGS_torcx_upload_root}" ]]; then
+        if [[ "${FLAGS_torcx_upload_root}" != gs://* ]]; then
+            die_notrace "--torcx_upload_root must be a gs:// URL"
+        fi
+        # Make sure the path doesn't end with a slash
+        TORCX_UPLOAD_ROOT="${FLAGS_torcx_upload_root%%/}"
+    fi
+
     if [[ -n "${FLAGS_upload_path}" ]]; then
         if [[ "${FLAGS_upload_path}" != gs://* ]]; then
             die_notrace "--upload_path must be a gs:// URL"
@@ -229,3 +245,29 @@ download_image_url() {
 
     echo "${download_path}/$1"
 }
+
+# Translate the configured torcx upload URL to a download url
+# This is similar to the download_image_url, other than assuming the release
+# bucket is the tectonic_torcx one.
+download_tectonic_torcx_url() {
+    if [[ ${FLAGS_upload} -ne ${FLAGS_TRUE} ]]; then
+        echo "$1"
+        return 0
+    fi
+
+    local download_root="${FLAGS_tectonic_torcx_download_root:-${TORCX_UPLOAD_ROOT}}"
+
+    local download_path
+    if [[ -n "${FLAGS_tectonic_torcx_download_path}" ]]; then
+        download_path="${FLAGS_tectonic_torcx_download_path%%/}"
+    else
+        download_path="${download_root%%/}"
+    fi
+
+    # Just in case download_root was set from UPLOAD_ROOT
+    if [[ "${download_path}" == gs://* ]]; then
+        download_path="http://${download_path#gs://}"
+    fi
+
+    echo "${download_path}/$1"
+}

build_library/torcx_manifest.sh

@@ -0,0 +1,129 @@
+# Copyright (c) 2017 The Container Linux by CoreOS Authors. All rights
+# reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# torcx_manifest.sh contains helper functions for creating, editing, and
+# reading torcx manifest files.
+
+# create_empty creates an empty torcx manfiest at the given path.
+function torcx_manifest::create_empty() {
+  local path="${1}"
+  jq '.' > "${path}" <<EOF
+{
+  "kind": "torcx-package-list-v0",
+  "value": {
+    "packages": []
+  }
+}
+EOF
+}
+
+# add_pkg adds a new version of a package to the torcx manifest specified by
+# path.
+# That manifest will be edited to include this version, with the associated
+# package of the given name being created as well if necessary.
+function torcx_manifest::add_pkg() {
+  path="${1}"; shift
+  name="${1}"; shift
+  version="${1}"; shift
+  pkg_hash="${1}"; shift
+  cas_digest="${1}"; shift
+  source_package="${1}"; shift
+  update_default="${1}"; shift
+
+  local manifest=$(cat "${path}")
+  local pkg_version_obj=$(jq '.' <<EOF
+{
+  "version": "${version}",
+  "hash": "${pkg_hash}",
+  "casDigest": "${cas_digest}",
+  "sourcePackage": "${source_package}",
+  "locations": []
+}
+EOF
+)
+
+  for location in "${@}"; do
+    if [[ "${location}" == /* ]]; then
+      # filepath
+      pkg_version_obj=$(jq ".locations |= . + [{\"path\": \"${location}\"}]" <(echo "${pkg_version_obj}"))
+    else
+      # url
+      pkg_version_obj=$(jq ".locations |= . + [{\"url\": \"${location}\"}]" <(echo "${pkg_version_obj}"))
+    fi
+  done
+
+
+  local existing_pkg="$(echo "${manifest}" | jq ".value.packages[] | select(.name == \"${name}\")")"
+
+  # If there isn't yet a package in the manifest for $name, initialize it to an empty one.
+  if [[ "${existing_pkg}" == "" ]]; then
+    pkg_json=$(cat <<EOF
+    {
+      "name": "${name}",
+      "versions": []
+    }
+EOF
+)
+    manifest="$(echo "${manifest}" | jq ".value.packages |= . + [${pkg_json}]")"
+  fi
+
+  if [[ "${update_default}" == "true" ]]; then
+    manifest="$(echo "${manifest}" | jq "(.value.packages[] | select(.name = \"${name}\") | .defaultVersion) |= \"${version}\"")"
+  fi
+
+  # append this specific package version to the manifest
+  manifest="$(echo "${manifest}" | jq "(.value.packages[] | select(.name = \"${name}\") | .versions) |= . + [${pkg_version_obj}]")"
+
+  echo "${manifest}" | jq '.' > "${path}"
+}
+
+# get_pkg_names returns the list of packages in a given manifest. Each package
+# may have one or more versions associated with it.
+#
+# Example:
+#     pkg_name_arr=($(torcx_manifest::get_pkg_names "torcx_manifest.json"))
+function torcx_manifest::get_pkg_names() {
+  local file="${1}"
+  jq -r '.value.packages[].name' < "${file}"
+}
+
+# local_store_path returns the in-container-linux store path a given package +
+# version combination should exist at. It returns the empty string if the
+# package shouldn't exist on disk.
+function torcx_manifest::local_store_path() {
+  local file="${1}"
+  local name="${2}"
+  local version="${3}"
+  jq -r ".value.packages[] | select(.name == \"${name}\") | .versions[] | select(.version == \"${version}\") | .locations[].path" < "${file}"
+}
+
+# get_digest returns the cas digest for a given package version
+function torcx_manifest::get_digest() {
+  local file="${1}"
+  local name="${2}"
+  local version="${3}"
+  jq -r ".value.packages[] | select(.name == \"${name}\") | .versions[] | select(.version == \"${version}\") | .casDigest" < "${file}"
+}
+
+# get_digests returns the list of digests for a given package. 
+function torcx_manifest::get_digests() {
+  local file="${1}"
+  local name="${2}"
+  jq -r ".value.packages[] | select(.name == \"${name}\").versions[].casDigest" < "${file}"
+}
+
+# get_versions returns the list of versions for a given package. 
+function torcx_manifest::get_versions() {
+  local file="${1}"
+  local name="${2}"
+  jq -r ".value.packages[] | select(.name == \"${name}\").versions[].version" < "${file}"
+}
+
+# default_version returns the default version for a given package, or an empty string if there isn't one.
+function torcx_manifest::default_version() {
+  local file="${1}"
+  local name="${2}"
+  jq -r ".value.packages[] | select(.name == \"${name}\").defaultVersion" < "${file}"
+}

build_torcx_store

@@ -15,7 +15,7 @@ assert_not_root_user
 DEFINE_string board "${DEFAULT_BOARD}" \
   "The board to build packages for."
 DEFINE_string output_root "${DEFAULT_BUILD_ROOT}/torcx" \
-  "Directory in which to place torcx stores (named by board/version)"
+  "Directory in which to place torcx stores and manifests (named by board/version)"
 
 # include upload options
 . "${BUILD_LIBRARY_DIR}/release_util.sh" || exit 1
@@ -54,6 +54,9 @@ check_gsutil_opts
 . "${BUILD_LIBRARY_DIR}/toolchain_util.sh" || exit 1
 . "${BUILD_LIBRARY_DIR}/board_options.sh" || exit 1
 . "${BUILD_LIBRARY_DIR}/build_image_util.sh" || exit 1
+. "${BUILD_LIBRARY_DIR}/torcx_manifest.sh" || exit 1
+
+TORCX_CAS_ROOT="${FLAGS_output_root}/pkgs/${BOARD}"
 
 # Print the first level of runtime dependencies for a torcx meta-package.
 function torcx_dependencies() (
@@ -90,14 +93,22 @@ function torcx_package() {
         local pkg="app-torcx/${1##*/}"
         local name=${pkg%-[0-9]*}
         local version=${pkg:${#name}+1}
-        local deppkg file rpath tmproot
+        local manifest_path="${2}"
-        name=${name##*/}
+        local type="${3}"
-        version=${version%%-r*}
+        local deppkg digest file rpath sha512sum source_pkg rdepends tmproot tmppkgroot update_default
+        local pkg_cas_file pkg_cas_root
+        local pkg_locations=()
+        local name=${name##*/}
+        local version=${version%%-r*}
 
         # Set up the base package layout to dump everything into /bin and /lib.
+        # tmproot is what the packages are installed into.
+        # A subset of the files from tmproot are then moved into tmppkgroot,
+        # which is then archived and uploaded.
         tmproot=$(sudo mktemp --tmpdir="${BUILD_DIR}" -d)
-        trap "sudo rm -rf '${tmproot}'" EXIT RETURN
+        tmppkgroot=$(sudo mktemp --tmpdir="${BUILD_DIR}" -d)
-        sudo chmod 0755 "${tmproot}"
+        trap "sudo rm -rf '${tmproot}' '${tmppkgroot}'" EXIT RETURN
+        sudo chmod 0755 "${tmproot}" "${tmppkgroot}"
         sudo mkdir -p "${tmproot}"/{.torcx,bin,lib,usr}
         sudo ln -fns ../bin "${tmproot}/usr/bin"
         sudo ln -fns ../lib "${tmproot}/usr/lib"
@@ -112,6 +123,11 @@ function torcx_package() {
                 torcx_build "${tmproot}" "${deppkg}"
         done
 
+        # by convention, the first dependency in a torcx package is the primary
+        # source package
+        rdepends=($(torcx_dependencies "${pkg}"))
+        source_pkg="${rdepends[0]#=}"
+
         # Pluck out shared libraries and SONAME links.
         sudo mv "${tmproot}"/{lib,tmplib}
         sudo rm -fr "${tmproot}/tmplib/debug"
@@ -151,10 +167,49 @@ function torcx_package() {
                 :  # Set $? to 0 or the pipeline fails and -e quits.
         done
 
-        # Package the installed files.
+        # Move anything we plan to package to its root
-        file="${BUILD_DIR}/${name}:${version}.torcx.tgz"
+        sudo mv "${tmproot}"/{.torcx,bin,lib} "${tmppkgroot}"
-        tar --force-local -C "${tmproot}" -czf "${file}" .torcx bin lib
+
-        ln -fns "${file##*/}" "${BUILD_DIR}/${name}:com.coreos.cl.torcx.tgz"
+        tmpfile="${BUILD_DIR}/${name}:${version}.torcx.tgz"
+        tar --force-local -C "${tmppkgroot}" -czf "${tmpfile}" .
+        sha512sum=$(sha512sum "${tmpfile}" | awk '{print $1}')
+
+        # TODO(euank): this opaque digest, if it were reproducible, could save
+        # users from having to download things that haven't changed.
+        # For now, use the sha512sum of the final image.
+        # Ideally we should move to something more like a casync digest or tarsum.
+        # The reason this is currently not being done is because to do that we
+        # *MUST* ensure that a given pair of (digest, sha512sum) referenced in
+        # a previous torcx package remains correct.
+        # Because this code, as written, clobbers existing things with the same
+        # digest (but the sha512sum of the .torcx.tgz can differ, e.g. due to ctime)
+        # that property doesn't hold.
+        # To switch this back to a reprodicble digest, we *must* never clobber
+        # existing objects (and thus re-use their sha512sum here).
+        digest="${sha512sum}"
+
+        pkg_cas_root="${TORCX_CAS_ROOT}/${name}/${digest}"
+        pkg_cas_file="${pkg_cas_root}/${name}:${version}.torcx.tgz"
+        mkdir -p "${pkg_cas_root}"
+        mv "${tmpfile}" "${pkg_cas_file}"
+
+        update_default=false
+        if [[ "${type}" == "default" ]]; then
+                update_default=true
+                pkg_locations+=("/usr/share/torcx/store/${name}:${version}.torcx.tgz")
+        fi
+        if [[ "${FLAGS_upload}" -eq ${FLAGS_TRUE} ]]; then
+                pkg_locations+=("$(download_tectonic_torcx_url "pkgs/${BOARD}/${name}/${digest}/${name}:${version}.torcx.tgz")")
+        fi
+        torcx_manifest::add_pkg "${manifest_path}" \
+            "${name}" \
+            "${version}" \
+            "sha512-${sha512sum}" \
+            "${digest}" \
+            "${source_pkg}" \
+            "${update_default}" \
+            "${pkg_locations[@]}"
+
         trap - EXIT
 }
 
@@ -166,13 +221,44 @@ DEFAULT_IMAGES=(
         =app-torcx/docker-17.06
 )
 
+# This list contains extra images which will be uploaded and included in the
+# generated manifest, but won't be included in the vendor store.
+EXTRA_IMAGES=(
+        =app-torcx/docker-1.12
+)
+
 mkdir -p "${BUILD_DIR}"
-for pkg in "${@:-${DEFAULT_IMAGES[@]}}" ; do torcx_package "${pkg#=}" ; done
+manifest_path="${BUILD_DIR}/torcx_manifest.json"
+torcx_manifest::create_empty "${manifest_path}"
+for pkg in "${@:-${DEFAULT_IMAGES[@]}}" ; do torcx_package "${pkg#=}" "${manifest_path}" "default" ; done
+for pkg in "${EXTRA_IMAGES[@]}" ; do torcx_package "${pkg#=}" "${manifest_path}" "extra" ; done
 
 set_build_symlinks latest "${FLAGS_group}-latest"
 
+# Upload the pkgs referenced by this manifest
+for pkg in $(torcx_manifest::get_pkg_names "${manifest_path}"); do
+        for digest in $(torcx_manifest::get_digests "${manifest_path}" "${pkg}"); do
+            # no need to sign; the manifest includes their shasum and is signed.
+            upload_files \
+                'torcx pkg' \
+                "${TORCX_UPLOAD_ROOT}/pkgs/${BOARD}/${pkg}/${digest}" \
+                "" \
+                "${TORCX_CAS_ROOT}/${pkg}/${digest}"/*.torcx.tgz
+      done
+done
+
+# Upload the manifest
+# Note: the manifest is uploaded to 'UPLOAD_ROOT' rather than
+# 'TORCX_UPLOAD_ROOT'.
+# For non-release builds, those two locations will be the same, so it usually
+# won't matter.
+# However, for release builds, torcx packages may be uploaded directly to their
+# final location, while the manifest still has to go through build bucket in
+# order to get signed.
 sign_and_upload_files \
-    'torcx images' \
+    'torcx manifest' \
-    "${UPLOAD_ROOT}/boards/${BOARD}/${COREOS_VERSION}" \
+    "${UPLOAD_ROOT}/torcx/manifests/${BOARD}/${COREOS_VERSION}" \
-    torcx/ \
+    "" \
-    "${BUILD_DIR}"/*.torcx.tgz
+    "${manifest_path}"
+
+# vim: tabstop=8 softtabstop=4 shiftwidth=8 expandtab

jenkins/images.sh

@@ -40,26 +40,28 @@ else
         script set_official --board="${BOARD}" --noofficial
 fi
 
-# Retrieve this version's torcx vendor store.
+# Retrieve this version's torcx manifest
+mkdir -p torcx/pkgs
 enter gsutil cp -r \
-    "${DOWNLOAD_ROOT}/boards/${BOARD}/${COREOS_VERSION}/torcx" \
+    "${DOWNLOAD_ROOT}/torcx/manifests/${BOARD}/${COREOS_VERSION}/torcx_manifest.json"{,.sig} \
-    /mnt/host/source/
+    /mnt/host/source/torcx/
-for image in torcx/*.torcx.tgz
+gpg --verify torcx/torcx_manifest.json.sig
-do
-        gpg --verify "${image}.sig"
-done
 
-# Work around the lack of symlink support in GCS.
+# Download all cas references from the manifest and verify their checksums
-shopt -s nullglob
+# TODO: technically we can skip ones that don't have a 'path' since they're not
-for default in torcx/*:com.coreos.cl.torcx.tgz
+# included in the image.
+while read name digest hash
 do
-        for image in torcx/*.torcx.tgz
+        mkdir -p "torcx/pkgs/${BOARD}/${name}/${digest}"
-        do
+        enter gsutil cp -r "${TORCX_PKG_DOWNLOAD_ROOT}/pkgs/${BOARD}/${name}/${digest}" \
-                [ "x${default}" != "x${image}" ] &&
+            "/mnt/host/source/torcx/pkgs/${BOARD}/${name}/"
-                cmp --silent -- "${default}" "${image}" &&
+        downloaded_hash=$(sha512sum "torcx/pkgs/${BOARD}/${name}/${digest}/"*.torcx.tgz | awk '{print $1}')
-                ln -fns "${image##*/}" "${default}"
+        if [[ "sha512-${downloaded_hash}" != "${hash}" ]]
-        done
+        then
-done
+                echo "Torcx package had wrong hash: ${downloaded_hash} instead of ${hash}"
+                exit 1
+        fi
+done < <(jq -r '.value.packages[] | . as $p | .name as $n | $p.versions[] | [.casDigest, .hash] | join(" ") | [$n, .] | join(" ")' "torcx/torcx_manifest.json")
 
 script build_image \
     --board="${BOARD}" \
@@ -68,6 +70,7 @@ script build_image \
     --getbinpkgver="${COREOS_VERSION}" \
     --sign="${SIGNING_USER}" \
     --sign_digests="${SIGNING_USER}" \
-    --torcx_store=/mnt/host/source/torcx \
+    --torcx_manifest=/mnt/host/source/torcx/torcx_manifest.json \
+    --torcx_root=/mnt/host/source/torcx/ \
     --upload_root="${UPLOAD_ROOT}" \
     --upload prod container

jenkins/packages.sh

@@ -56,6 +56,8 @@ script build_torcx_store \
     --sign="${SIGNING_USER}" \
     --sign_digests="${SIGNING_USER}" \
     --upload_root="${UPLOAD_ROOT}" \
+    --torcx_upload_root="${TORCX_PKG_DOWNLOAD_ROOT}" \
+    --tectonic_torcx_download_root="${TECTONIC_TORCX_DOWNLOAD_ROOT}" \
     --upload
 
 enter ccache --show-stats