diff --git a/build_torcx_store b/build_torcx_store index 5cb68f9f34..f23cc79290 100755 --- a/build_torcx_store +++ b/build_torcx_store @@ -15,7 +15,7 @@ assert_not_root_user DEFINE_string board "${DEFAULT_BOARD}" \ "The board to build packages for." DEFINE_string output_root "${DEFAULT_BUILD_ROOT}/torcx" \ - "Directory in which to place torcx stores (named by board/version)" + "Directory in which to place torcx stores and manifests (named by board/version)" # include upload options . "${BUILD_LIBRARY_DIR}/release_util.sh" || exit 1 @@ -54,6 +54,9 @@ check_gsutil_opts . "${BUILD_LIBRARY_DIR}/toolchain_util.sh" || exit 1 . "${BUILD_LIBRARY_DIR}/board_options.sh" || exit 1 . "${BUILD_LIBRARY_DIR}/build_image_util.sh" || exit 1 +. "${BUILD_LIBRARY_DIR}/torcx_manifest.sh" || exit 1 + +TORCX_CAS_ROOT="${FLAGS_output_root}/pkgs/${BOARD}" # Print the first level of runtime dependencies for a torcx meta-package. function torcx_dependencies() ( @@ -90,14 +93,22 @@ function torcx_package() { local pkg="app-torcx/${1##*/}" local name=${pkg%-[0-9]*} local version=${pkg:${#name}+1} - local deppkg file rpath tmproot - name=${name##*/} - version=${version%%-r*} + local manifest_path="${2}" + local type="${3}" + local deppkg digest file rpath sha512sum source_pkg tmproot tmppkgroot update_default + local pkg_cas_file pkg_cas_root + local pkg_locations=() + local name=${name##*/} + local version=${version%%-r*} # Set up the base package layout to dump everything into /bin and /lib. + # tmproot is what the packages are installed into. + # A subset of the files from tmproot are then moved into tmppkgroot, + # which is then archived and uploaded. tmproot=$(sudo mktemp --tmpdir="${BUILD_DIR}" -d) - trap "sudo rm -rf '${tmproot}'" EXIT RETURN - sudo chmod 0755 "${tmproot}" + tmppkgroot=$(sudo mktemp --tmpdir="${BUILD_DIR}" -d) + trap "sudo rm -rf '${tmproot}' '${tmppkgroot}'" EXIT RETURN + sudo chmod 0755 "${tmproot}" "${tmppkgroot}" sudo mkdir -p "${tmproot}"/{.torcx,bin,lib,usr} sudo ln -fns ../bin "${tmproot}/usr/bin" sudo ln -fns ../lib "${tmproot}/usr/lib" @@ -109,6 +120,9 @@ function torcx_package() { # Install the meta-package and its direct dependencies. for deppkg in "=${pkg}" $(torcx_dependencies "${pkg}") do + # The first file in an app-torcx package is, by convention, + # considered the source package. + [ -z "${source_pkg}" ] && source_pkg="${deppkg#=}" torcx_build "${tmproot}" "${deppkg}" done @@ -151,10 +165,50 @@ function torcx_package() { : # Set $? to 0 or the pipeline fails and -e quits. done - # Package the installed files. - file="${BUILD_DIR}/${name}:${version}.torcx.tgz" - tar --force-local -C "${tmproot}" -czf "${file}" .torcx bin lib - ln -fns "${file##*/}" "${BUILD_DIR}/${name}:com.coreos.cl.torcx.tgz" + # Move anything we plan to package to its root + sudo mv "${tmproot}"/{.torcx,bin,lib} "${tmppkgroot}" + + # Create a reproducible digest by which this package will be uploaded + # and referred + digest=$(casync \ + --digest=sha512-256 \ + --without=selinux \ + --with=flag-noatime \ + --what=directory \ + digest "${tmppkgroot}") + + pkg_cas_root="${TORCX_CAS_ROOT}/${name}/${digest}" + pkg_cas_file="${pkg_cas_root}/${name}:${version}.torcx.tgz" + + # Create the cas store if it doesn't exist + if [[ ! -d "${pkg_cas_root}" ]]; then + mkdir -p "${TORCX_CAS_ROOT}/${name}" + # Package the installed files. + tmpcas="${BUILD_DIR}/${digest}" + mkdir -p "${tmpcas}" + tmpfile="${tmpcas}/${name}:${version}.torcx.tgz" + tar --force-local -C "${tmppkgroot}" -czf "${tmpfile}" . + # atomically move the whole directory in place from a tmp locatoin + mv "${tmpcas}" "${pkg_cas_root}" + fi + [[ -f "${pkg_cas_file}" ]] || die "${pkg_cas_file} should exist but didn't" + sha512sum=$(sha512sum "${pkg_cas_file}" | awk '{print $1}') + + update_default=false + if [[ "${type}" == "default" ]]; then + update_default=true + pkg_locations+=("/usr/share/torcx/store/${name}:${version}.torcx.tgz") + fi + # TODO: add upload bucket path to pkg_locations + torcx_manifest::add_pkg "${manifest_path}" \ + "${name}" \ + "${version}" \ + "sha512-${sha512sum}" \ + "${digest}" \ + "${source_pkg}" \ + "${update_default}" \ + "${pkg_locations[@]}" + trap - EXIT } @@ -166,13 +220,38 @@ DEFAULT_IMAGES=( =app-torcx/docker-17.06 ) +# This list contains extra images which will be uploaded and included in the +# generated manifest, but won't be included in the vendor store. +EXTRA_IMAGES=( + =app-torcx/docker-1.12 +) + mkdir -p "${BUILD_DIR}" -for pkg in "${@:-${DEFAULT_IMAGES[@]}}" ; do torcx_package "${pkg#=}" ; done +manifest_path="${BUILD_DIR}/torcx_manifest.json" +torcx_manifest::create_empty "${manifest_path}" +for pkg in "${@:-${DEFAULT_IMAGES[@]}}" ; do torcx_package "${pkg#=}" "${manifest_path}" "default" ; done +for pkg in "${EXTRA_IMAGES[@]}" ; do torcx_package "${pkg#=}" "${manifest_path}" "extra" ; done set_build_symlinks latest "${FLAGS_group}-latest" +# Upload the pkgs referenced by this manifest +for pkg in $(torcx_manifest::get_pkg_names "${manifest_path}"); do + pkg_name="${pkgs[0]}" + for digest in $(torcx_manifest::get_digests "${manifest_path}" "${pkg}"); do + # no need to sign; the manifest includes their shasum and is signed. + upload_files \ + 'torcx pkg' \ + "${UPLOAD_ROOT}/torcx/pkg/${BOARD}/${pkg_name}/${digest}" \ + "" \ + "${TORCX_CAS_ROOT}/${pkg_name}/${digest}"/*.torcx.tgz + done +done + +# Upload the manifest sign_and_upload_files \ - 'torcx images' \ - "${UPLOAD_ROOT}/boards/${BOARD}/${COREOS_VERSION}" \ - torcx/ \ - "${BUILD_DIR}"/*.torcx.tgz + 'torcx manifest' \ + "${UPLOAD_ROOT}/torcx/manifests/${BOARD}/${COREOS_VERSION}" \ + "" \ + "${manifest_path}" + +# vim: tabstop=8 softtabstop=4 shiftwidth=8 expandtab