diff --git a/build_image b/build_image index c9d80c9e6c..b7e63861b4 100755 --- a/build_image +++ b/build_image @@ -32,8 +32,10 @@ DEFINE_string base_pkg "coreos-base/coreos" \ "The base portage package to base the build off of (only applies to prod images)" DEFINE_string base_dev_pkg "coreos-base/coreos-dev" \ "The base portage package to base the build off of (only applies to dev images)" -DEFINE_string torcx_store "${DEFAULT_BUILD_ROOT}/torcx/${DEFAULT_BOARD}/latest" \ - "Directory of torcx images to copy into the vendor store (or blank for none)" +DEFINE_string torcx_manifest "${DEFAULT_BUILD_ROOT}/torcx/${DEFAULT_BOARD}/latest/torcx_manifest.json" \ + "The torcx manifest describing torcx packages for this image (or blank for none)" +DEFINE_string torcx_root "${DEFAULT_BUILD_ROOT}/torcx" \ + "Directory in which torcx packages can be found" DEFINE_string output_root "${DEFAULT_BUILD_ROOT}/images" \ "Directory in which to place image result directories (named by version)" DEFINE_string disk_layout "" \ @@ -89,8 +91,8 @@ switch_to_strict_mode check_gsutil_opts # Patch around default values not being able to depend on other flags. -if [ "x${FLAGS_torcx_store}" = "x${DEFAULT_BUILD_ROOT}/torcx/${DEFAULT_BOARD}/latest" ]; then - FLAGS_torcx_store="${DEFAULT_BUILD_ROOT}/torcx/${FLAGS_board}/latest" +if [ "x${FLAGS_torcx_manifest}" = "x${DEFAULT_BUILD_ROOT}/torcx/${DEFAULT_BOARD}/latest/torcx_manifest.json" ]; then + FLAGS_torcx_manifest="${DEFAULT_BUILD_ROOT}/torcx/${FLAGS_board}/latest/torcx_manifest.json" fi # If downloading packages is enabled ensure the board is configured properly. @@ -107,6 +109,7 @@ fi . "${BUILD_LIBRARY_DIR}/prod_image_util.sh" || exit 1 . "${BUILD_LIBRARY_DIR}/dev_image_util.sh" || exit 1 . "${BUILD_LIBRARY_DIR}/test_image_content.sh" || exit 1 +. "${BUILD_LIBRARY_DIR}/torcx_manifest.sh" || exit 1 . "${BUILD_LIBRARY_DIR}/vm_image_util.sh" || exit 1 PROD_IMAGE=0 diff --git a/build_library/build_image_util.sh b/build_library/build_image_util.sh index def5ab03c8..c70e2b0fe1 100755 --- a/build_library/build_image_util.sh +++ b/build_library/build_image_util.sh @@ -424,10 +424,25 @@ finish_image() { local install_grub=0 local disk_img="${BUILD_DIR}/${image_name}" - # Copy in a vendor torcx store if requested. - if [ -n "${FLAGS_torcx_store}" ]; then - sudo cp -dt "${root_fs_dir}"/usr/share/torcx/store \ - "${FLAGS_torcx_store}"/*.torcx.tgz + # Copy in packages from the torcx store that are marked as being on disk + if [ -n "${FLAGS_torcx_manifest}" ]; then + for pkg in $(torcx_manifest::get_pkg_names "${FLAGS_torcx_manifest}"); do + local default_version="$(torcx_manifest::default_version "${FLAGS_torcx_manifest}" "${pkg}")" + for version in $(torcx_manifest::get_versions "${FLAGS_torcx_manifest}" "${pkg}"); do + local on_disk_path="$(torcx_manifest::local_store_path "${FLAGS_torcx_manifest}" "${pkg}" "${version}")" + if [[ -n "${on_disk_path}" ]]; then + local casDigest="$(torcx_manifest::get_digest "${FLAGS_torcx_manifest}" "${pkg}" "${version}")" + sudo cp "${FLAGS_torcx_root}/pkgs/${BOARD}/${pkg}/${casDigest}/${pkg}:${version}.torcx.tgz" \ + "${root_fs_dir}${on_disk_path}" + + if [[ "${version}" == "${default_version}" ]]; then + # Create the default symlink for this package + sudo ln -fns "${on_disk_path##*/}" \ + "${root_fs_dir}/${on_disk_path%/*}/${pkg}:com.coreos.cl.torcx.tgz" + fi + fi + done + done fi # Only enable rootfs verification on prod builds. diff --git a/build_library/release_util.sh b/build_library/release_util.sh index b2dde823c5..c48576f286 100644 --- a/build_library/release_util.sh +++ b/build_library/release_util.sh @@ -5,12 +5,14 @@ GSUTIL_OPTS= UPLOAD_ROOT= UPLOAD_PATH= +TORCX_UPLOAD_ROOT= UPLOAD_DEFAULT=${FLAGS_FALSE} # Default upload root can be overridden from the environment. _user="${USER}" [[ ${USER} == "root" ]] && _user="${SUDO_USER}" : ${COREOS_UPLOAD_ROOT:=gs://users.developer.core-os.net/${_user}} +: ${COREOS_TORCX_UPLOAD_ROOT:=${COREOS_UPLOAD_ROOT}/torcx} unset _user IMAGE_ZIPPER="lbzip2 --compress --keep" @@ -28,6 +30,12 @@ DEFINE_string download_root "" \ "HTTP download prefix, board/version/etc will be appended." DEFINE_string download_path "" \ "HTTP download path, overrides --download_root." +DEFINE_string torcx_upload_root "${COREOS_TORCX_UPLOAD_ROOT}" \ + "Tectonic torcx package and manifest Upload prefix. Must be a gs:// URL." +DEFINE_string tectonic_torcx_download_root "" \ + "HTTP download prefix for tectonic torcx packages and manifests." +DEFINE_string tectonic_torcx_download_path "" \ + "HTTP download path, overrides --tectonic_torcx_download_root." DEFINE_string sign "" \ "Sign all files to be uploaded with the given GPG key." DEFINE_string sign_digests "" \ @@ -48,6 +56,14 @@ check_gsutil_opts() { UPLOAD_ROOT="${FLAGS_upload_root%%/}" fi + if [[ -n "${FLAGS_torcx_upload_root}" ]]; then + if [[ "${FLAGS_torcx_upload_root}" != gs://* ]]; then + die_notrace "--torcx_upload_root must be a gs:// URL" + fi + # Make sure the path doesn't end with a slash + TORCX_UPLOAD_ROOT="${FLAGS_torcx_upload_root%%/}" + fi + if [[ -n "${FLAGS_upload_path}" ]]; then if [[ "${FLAGS_upload_path}" != gs://* ]]; then die_notrace "--upload_path must be a gs:// URL" @@ -229,3 +245,29 @@ download_image_url() { echo "${download_path}/$1" } + +# Translate the configured torcx upload URL to a download url +# This is similar to the download_image_url, other than assuming the release +# bucket is the tectonic_torcx one. +download_tectonic_torcx_url() { + if [[ ${FLAGS_upload} -ne ${FLAGS_TRUE} ]]; then + echo "$1" + return 0 + fi + + local download_root="${FLAGS_tectonic_torcx_download_root:-${TORCX_UPLOAD_ROOT}}" + + local download_path + if [[ -n "${FLAGS_tectonic_torcx_download_path}" ]]; then + download_path="${FLAGS_tectonic_torcx_download_path%%/}" + else + download_path="${download_root%%/}" + fi + + # Just in case download_root was set from UPLOAD_ROOT + if [[ "${download_path}" == gs://* ]]; then + download_path="http://${download_path#gs://}" + fi + + echo "${download_path}/$1" +} diff --git a/build_library/torcx_manifest.sh b/build_library/torcx_manifest.sh new file mode 100644 index 0000000000..3d7a7e8771 --- /dev/null +++ b/build_library/torcx_manifest.sh @@ -0,0 +1,129 @@ +# Copyright (c) 2017 The Container Linux by CoreOS Authors. All rights +# reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# torcx_manifest.sh contains helper functions for creating, editing, and +# reading torcx manifest files. + +# create_empty creates an empty torcx manfiest at the given path. +function torcx_manifest::create_empty() { + local path="${1}" + jq '.' > "${path}" < "${path}" +} + +# get_pkg_names returns the list of packages in a given manifest. Each package +# may have one or more versions associated with it. +# +# Example: +# pkg_name_arr=($(torcx_manifest::get_pkg_names "torcx_manifest.json")) +function torcx_manifest::get_pkg_names() { + local file="${1}" + jq -r '.value.packages[].name' < "${file}" +} + +# local_store_path returns the in-container-linux store path a given package + +# version combination should exist at. It returns the empty string if the +# package shouldn't exist on disk. +function torcx_manifest::local_store_path() { + local file="${1}" + local name="${2}" + local version="${3}" + jq -r ".value.packages[] | select(.name == \"${name}\") | .versions[] | select(.version == \"${version}\") | .locations[].path" < "${file}" +} + +# get_digest returns the cas digest for a given package version +function torcx_manifest::get_digest() { + local file="${1}" + local name="${2}" + local version="${3}" + jq -r ".value.packages[] | select(.name == \"${name}\") | .versions[] | select(.version == \"${version}\") | .casDigest" < "${file}" +} + +# get_digests returns the list of digests for a given package. +function torcx_manifest::get_digests() { + local file="${1}" + local name="${2}" + jq -r ".value.packages[] | select(.name == \"${name}\").versions[].casDigest" < "${file}" +} + +# get_versions returns the list of versions for a given package. +function torcx_manifest::get_versions() { + local file="${1}" + local name="${2}" + jq -r ".value.packages[] | select(.name == \"${name}\").versions[].version" < "${file}" +} + +# default_version returns the default version for a given package, or an empty string if there isn't one. +function torcx_manifest::default_version() { + local file="${1}" + local name="${2}" + jq -r ".value.packages[] | select(.name == \"${name}\").defaultVersion" < "${file}" +} diff --git a/build_torcx_store b/build_torcx_store index 5cb68f9f34..3cf2651429 100755 --- a/build_torcx_store +++ b/build_torcx_store @@ -15,7 +15,7 @@ assert_not_root_user DEFINE_string board "${DEFAULT_BOARD}" \ "The board to build packages for." DEFINE_string output_root "${DEFAULT_BUILD_ROOT}/torcx" \ - "Directory in which to place torcx stores (named by board/version)" + "Directory in which to place torcx stores and manifests (named by board/version)" # include upload options . "${BUILD_LIBRARY_DIR}/release_util.sh" || exit 1 @@ -54,6 +54,9 @@ check_gsutil_opts . "${BUILD_LIBRARY_DIR}/toolchain_util.sh" || exit 1 . "${BUILD_LIBRARY_DIR}/board_options.sh" || exit 1 . "${BUILD_LIBRARY_DIR}/build_image_util.sh" || exit 1 +. "${BUILD_LIBRARY_DIR}/torcx_manifest.sh" || exit 1 + +TORCX_CAS_ROOT="${FLAGS_output_root}/pkgs/${BOARD}" # Print the first level of runtime dependencies for a torcx meta-package. function torcx_dependencies() ( @@ -90,14 +93,22 @@ function torcx_package() { local pkg="app-torcx/${1##*/}" local name=${pkg%-[0-9]*} local version=${pkg:${#name}+1} - local deppkg file rpath tmproot - name=${name##*/} - version=${version%%-r*} + local manifest_path="${2}" + local type="${3}" + local deppkg digest file rpath sha512sum source_pkg rdepends tmproot tmppkgroot update_default + local pkg_cas_file pkg_cas_root + local pkg_locations=() + local name=${name##*/} + local version=${version%%-r*} # Set up the base package layout to dump everything into /bin and /lib. + # tmproot is what the packages are installed into. + # A subset of the files from tmproot are then moved into tmppkgroot, + # which is then archived and uploaded. tmproot=$(sudo mktemp --tmpdir="${BUILD_DIR}" -d) - trap "sudo rm -rf '${tmproot}'" EXIT RETURN - sudo chmod 0755 "${tmproot}" + tmppkgroot=$(sudo mktemp --tmpdir="${BUILD_DIR}" -d) + trap "sudo rm -rf '${tmproot}' '${tmppkgroot}'" EXIT RETURN + sudo chmod 0755 "${tmproot}" "${tmppkgroot}" sudo mkdir -p "${tmproot}"/{.torcx,bin,lib,usr} sudo ln -fns ../bin "${tmproot}/usr/bin" sudo ln -fns ../lib "${tmproot}/usr/lib" @@ -112,6 +123,11 @@ function torcx_package() { torcx_build "${tmproot}" "${deppkg}" done + # by convention, the first dependency in a torcx package is the primary + # source package + rdepends=($(torcx_dependencies "${pkg}")) + source_pkg="${rdepends[0]#=}" + # Pluck out shared libraries and SONAME links. sudo mv "${tmproot}"/{lib,tmplib} sudo rm -fr "${tmproot}/tmplib/debug" @@ -151,10 +167,49 @@ function torcx_package() { : # Set $? to 0 or the pipeline fails and -e quits. done - # Package the installed files. - file="${BUILD_DIR}/${name}:${version}.torcx.tgz" - tar --force-local -C "${tmproot}" -czf "${file}" .torcx bin lib - ln -fns "${file##*/}" "${BUILD_DIR}/${name}:com.coreos.cl.torcx.tgz" + # Move anything we plan to package to its root + sudo mv "${tmproot}"/{.torcx,bin,lib} "${tmppkgroot}" + + tmpfile="${BUILD_DIR}/${name}:${version}.torcx.tgz" + tar --force-local -C "${tmppkgroot}" -czf "${tmpfile}" . + sha512sum=$(sha512sum "${tmpfile}" | awk '{print $1}') + + # TODO(euank): this opaque digest, if it were reproducible, could save + # users from having to download things that haven't changed. + # For now, use the sha512sum of the final image. + # Ideally we should move to something more like a casync digest or tarsum. + # The reason this is currently not being done is because to do that we + # *MUST* ensure that a given pair of (digest, sha512sum) referenced in + # a previous torcx package remains correct. + # Because this code, as written, clobbers existing things with the same + # digest (but the sha512sum of the .torcx.tgz can differ, e.g. due to ctime) + # that property doesn't hold. + # To switch this back to a reprodicble digest, we *must* never clobber + # existing objects (and thus re-use their sha512sum here). + digest="${sha512sum}" + + pkg_cas_root="${TORCX_CAS_ROOT}/${name}/${digest}" + pkg_cas_file="${pkg_cas_root}/${name}:${version}.torcx.tgz" + mkdir -p "${pkg_cas_root}" + mv "${tmpfile}" "${pkg_cas_file}" + + update_default=false + if [[ "${type}" == "default" ]]; then + update_default=true + pkg_locations+=("/usr/share/torcx/store/${name}:${version}.torcx.tgz") + fi + if [[ "${FLAGS_upload}" -eq ${FLAGS_TRUE} ]]; then + pkg_locations+=("$(download_tectonic_torcx_url "pkgs/${BOARD}/${name}/${digest}/${name}:${version}.torcx.tgz")") + fi + torcx_manifest::add_pkg "${manifest_path}" \ + "${name}" \ + "${version}" \ + "sha512-${sha512sum}" \ + "${digest}" \ + "${source_pkg}" \ + "${update_default}" \ + "${pkg_locations[@]}" + trap - EXIT } @@ -166,13 +221,44 @@ DEFAULT_IMAGES=( =app-torcx/docker-17.06 ) +# This list contains extra images which will be uploaded and included in the +# generated manifest, but won't be included in the vendor store. +EXTRA_IMAGES=( + =app-torcx/docker-1.12 +) + mkdir -p "${BUILD_DIR}" -for pkg in "${@:-${DEFAULT_IMAGES[@]}}" ; do torcx_package "${pkg#=}" ; done +manifest_path="${BUILD_DIR}/torcx_manifest.json" +torcx_manifest::create_empty "${manifest_path}" +for pkg in "${@:-${DEFAULT_IMAGES[@]}}" ; do torcx_package "${pkg#=}" "${manifest_path}" "default" ; done +for pkg in "${EXTRA_IMAGES[@]}" ; do torcx_package "${pkg#=}" "${manifest_path}" "extra" ; done set_build_symlinks latest "${FLAGS_group}-latest" +# Upload the pkgs referenced by this manifest +for pkg in $(torcx_manifest::get_pkg_names "${manifest_path}"); do + for digest in $(torcx_manifest::get_digests "${manifest_path}" "${pkg}"); do + # no need to sign; the manifest includes their shasum and is signed. + upload_files \ + 'torcx pkg' \ + "${TORCX_UPLOAD_ROOT}/pkgs/${BOARD}/${pkg}/${digest}" \ + "" \ + "${TORCX_CAS_ROOT}/${pkg}/${digest}"/*.torcx.tgz + done +done + +# Upload the manifest +# Note: the manifest is uploaded to 'UPLOAD_ROOT' rather than +# 'TORCX_UPLOAD_ROOT'. +# For non-release builds, those two locations will be the same, so it usually +# won't matter. +# However, for release builds, torcx packages may be uploaded directly to their +# final location, while the manifest still has to go through build bucket in +# order to get signed. sign_and_upload_files \ - 'torcx images' \ - "${UPLOAD_ROOT}/boards/${BOARD}/${COREOS_VERSION}" \ - torcx/ \ - "${BUILD_DIR}"/*.torcx.tgz + 'torcx manifest' \ + "${UPLOAD_ROOT}/torcx/manifests/${BOARD}/${COREOS_VERSION}" \ + "" \ + "${manifest_path}" + +# vim: tabstop=8 softtabstop=4 shiftwidth=8 expandtab diff --git a/jenkins/images.sh b/jenkins/images.sh index dea4784f66..3a48624364 100644 --- a/jenkins/images.sh +++ b/jenkins/images.sh @@ -40,26 +40,28 @@ else script set_official --board="${BOARD}" --noofficial fi -# Retrieve this version's torcx vendor store. +# Retrieve this version's torcx manifest +mkdir -p torcx/pkgs enter gsutil cp -r \ - "${DOWNLOAD_ROOT}/boards/${BOARD}/${COREOS_VERSION}/torcx" \ - /mnt/host/source/ -for image in torcx/*.torcx.tgz -do - gpg --verify "${image}.sig" -done + "${DOWNLOAD_ROOT}/torcx/manifests/${BOARD}/${COREOS_VERSION}/torcx_manifest.json"{,.sig} \ + /mnt/host/source/torcx/ +gpg --verify torcx/torcx_manifest.json.sig -# Work around the lack of symlink support in GCS. -shopt -s nullglob -for default in torcx/*:com.coreos.cl.torcx.tgz +# Download all cas references from the manifest and verify their checksums +# TODO: technically we can skip ones that don't have a 'path' since they're not +# included in the image. +while read name digest hash do - for image in torcx/*.torcx.tgz - do - [ "x${default}" != "x${image}" ] && - cmp --silent -- "${default}" "${image}" && - ln -fns "${image##*/}" "${default}" - done -done + mkdir -p "torcx/pkgs/${BOARD}/${name}/${digest}" + enter gsutil cp -r "${TORCX_PKG_DOWNLOAD_ROOT}/pkgs/${BOARD}/${name}/${digest}" \ + "/mnt/host/source/torcx/pkgs/${BOARD}/${name}/" + downloaded_hash=$(sha512sum "torcx/pkgs/${BOARD}/${name}/${digest}/"*.torcx.tgz | awk '{print $1}') + if [[ "sha512-${downloaded_hash}" != "${hash}" ]] + then + echo "Torcx package had wrong hash: ${downloaded_hash} instead of ${hash}" + exit 1 + fi +done < <(jq -r '.value.packages[] | . as $p | .name as $n | $p.versions[] | [.casDigest, .hash] | join(" ") | [$n, .] | join(" ")' "torcx/torcx_manifest.json") script build_image \ --board="${BOARD}" \ @@ -68,6 +70,7 @@ script build_image \ --getbinpkgver="${COREOS_VERSION}" \ --sign="${SIGNING_USER}" \ --sign_digests="${SIGNING_USER}" \ - --torcx_store=/mnt/host/source/torcx \ + --torcx_manifest=/mnt/host/source/torcx/torcx_manifest.json \ + --torcx_root=/mnt/host/source/torcx/ \ --upload_root="${UPLOAD_ROOT}" \ --upload prod container diff --git a/jenkins/packages.sh b/jenkins/packages.sh index 6c57a8435c..70cf3fcd2b 100644 --- a/jenkins/packages.sh +++ b/jenkins/packages.sh @@ -56,6 +56,8 @@ script build_torcx_store \ --sign="${SIGNING_USER}" \ --sign_digests="${SIGNING_USER}" \ --upload_root="${UPLOAD_ROOT}" \ + --torcx_upload_root="${TORCX_PKG_DOWNLOAD_ROOT}" \ + --tectonic_torcx_download_root="${TECTONIC_TORCX_DOWNLOAD_ROOT}" \ --upload enter ccache --show-stats