From b2d6f7fc6e28b044da25027bf3a0e7b4d9ae4f75 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Thu, 21 Jul 2022 17:00:04 +0200 Subject: [PATCH] *: Allow specifying extra URLs for torcx packages Torcx manifest may contain paths and URLs as locations of packages. There are two kinds of packages - vendored and extra. Vendored packages normally have two locations - path to the directory inside the image where the package is (which is why it's called vendored), and a URL to the package on some remote server. Extra packages only have a URL. But the URLs are added only when we tell the build_torcx_store script to upload the packages at the same time, which is what the old build pipeline was doing. With the new pipeline, the upload happens as a separate step, thus the upload is disabled when invoking build_torcx_store, and so the packages are not getting URLs set. This change went unnoticed, because a kola test checking the generated torcx manifest was only checking if there is at least one location, either path or URL, and all the new releases have no extra packages, only vendored ones. When backporting the new pipeline to old LTS, the kola tests started to fail, because old LTS had one extra package, and this is how I noticed the problem. --- build_packages | 7 ++++++- build_torcx_store | 14 ++++++++++++-- ci-automation/packages.sh | 15 ++++++++++++++- 3 files changed, 32 insertions(+), 4 deletions(-) diff --git a/build_packages b/build_packages index c77f0c8dbd..1ad8800f48 100755 --- a/build_packages +++ b/build_packages @@ -40,6 +40,8 @@ DEFINE_string torcx_output_root "${DEFAULT_BUILD_ROOT}/torcx" \ "Directory in which to place torcx stores and manifests (named by board/version)" DEFINE_boolean skip_torcx_store "${FLAGS_FALSE}" \ "Don't build a new torcx store from the updated sysroot." +DEFINE_string torcx_extra_pkg_url "" \ + "URL to directory where the torcx packages will be available for downloading" # include upload options . "${BUILD_LIBRARY_DIR}/release_util.sh" || exit 1 @@ -303,7 +305,10 @@ upload_packages # Build a new torcx store with the updated packages, passing flags through. if [ "${FLAGS_skip_torcx_store}" -eq "${FLAGS_FALSE}" ]; then - "${SCRIPTS_DIR}"/build_torcx_store --board="${BOARD}" --output_root="${FLAGS_torcx_output_root}" + "${SCRIPTS_DIR}"/build_torcx_store \ + --board="${BOARD}" \ + --output_root="${FLAGS_torcx_output_root}" \ + --extra_pkg_url="${FLAGS_torcx_extra_pkg_url}" fi info "Builds complete" diff --git a/build_torcx_store b/build_torcx_store index c4b0f99ff9..6bc95b1ba2 100755 --- a/build_torcx_store +++ b/build_torcx_store @@ -16,6 +16,8 @@ DEFINE_string board "${DEFAULT_BOARD}" \ "The board to build packages for." DEFINE_string output_root "${DEFAULT_BUILD_ROOT}/torcx" \ "Directory in which to place torcx stores and manifests (named by board/version)" +DEFINE_string extra_pkg_url "" \ + "URL to directory where the torcx packages will be available for downloading" # include upload options . "${BUILD_LIBRARY_DIR}/release_util.sh" || exit 1 @@ -102,6 +104,7 @@ function torcx_package() { local version=${pkg:${#name}+1} local manifest_path="${2}" local type="${3}" + local extra_pkg_url="${4}" local deppkg digest file rpath sha512sum source_pkg rdepends tmproot tmppkgroot update_default tmpfile local pkg_cas_file pkg_cas_root local pkg_locations=() @@ -215,6 +218,9 @@ function torcx_package() { if [[ "${FLAGS_upload}" -eq ${FLAGS_TRUE} ]]; then pkg_locations+=("$(download_tectonic_torcx_url "pkgs/${BOARD}/${name}/${digest}/${name}:${version}.torcx.tgz")") fi + if [[ -n "${extra_pkg_url}" ]]; then + pkg_locations+=("${extra_pkg_url}/${name}:${version}.torcx.tgz") + fi torcx_manifest::add_pkg "${manifest_path}" \ "${name}" \ "${version}" \ @@ -243,8 +249,12 @@ EXTRA_IMAGES=( mkdir -p "${BUILD_DIR}" manifest_path="${BUILD_DIR}/torcx_manifest.json" torcx_manifest::create_empty "${manifest_path}" -for pkg in "${@:-${DEFAULT_IMAGES[@]}}" ; do torcx_package "${pkg#=}" "${manifest_path}" "default" ; done -for pkg in "${EXTRA_IMAGES[@]}" ; do torcx_package "${pkg#=}" "${manifest_path}" "extra" ; done +for pkg in "${@:-${DEFAULT_IMAGES[@]}}"; do + torcx_package "${pkg#=}" "${manifest_path}" "default" "${FLAGS_extra_pkg_url}" +done +for pkg in "${EXTRA_IMAGES[@]}"; do + torcx_package "${pkg#=}" "${manifest_path}" "extra" "${FLAGS_extra_pkg_url}" +done set_build_symlinks latest "${FLAGS_group}-latest" diff --git a/ci-automation/packages.sh b/ci-automation/packages.sh index 029b1f4c56..dcddb0dcbf 100644 --- a/ci-automation/packages.sh +++ b/ci-automation/packages.sh @@ -80,6 +80,18 @@ function _packages_build_impl() { local vernum="${FLATCAR_VERSION}" local docker_vernum="$(vernum_to_docker_image_version "${vernum}")" local packages_container="flatcar-packages-${arch}-${docker_vernum}" + local torcx_pkg_url="https://${BUILDCACHE_SERVER}/images/${arch}/${vernum}/torcx" + + source sdk_lib/sdk_container_common.sh + + if is_official "${vernum}"; then + # A channel returned by get_git_channel should not ever be + # "developer" here, because it's an official build done from + # one of the maintenance branches. So if the channel happens + # to be "developer", then you are doing it wrong (releasing + # from the main branch?). + torcx_pkg_url="https://$(get_git_channel).release.flatcar-linux.net/${arch}-usr/${vernum}/torcx" + fi # Build packages; store packages and torcx output in container ./run_sdk_container -x ./ci-cleanup.sh -n "${packages_container}" -v "${vernum}" \ @@ -88,7 +100,8 @@ function _packages_build_impl() { ./run_sdk_container -n "${packages_container}" -v "${vernum}" \ -C "${sdk_image}" \ ./build_packages --board="${arch}-usr" \ - --torcx_output_root="${CONTAINER_TORCX_ROOT}" + --torcx_output_root="${CONTAINER_TORCX_ROOT}" \ + --torcx_extra_pkg_url="${torcx_pkg_url}" # copy torcx manifest and docker tarball for publishing local torcx_tmp="__build__/torcx_tmp"