*: Allow specifying extra URLs for torcx packages

Torcx manifest may contain paths and URLs as locations of
packages. There are two kinds of packages - vendored and
extra. Vendored packages normally have two locations - path to the
directory inside the image where the package is (which is why it's
called vendored), and a URL to the package on some remote
server. Extra packages only have a URL. But the URLs are added only
when we tell the build_torcx_store script to upload the packages at
the same time, which is what the old build pipeline was doing. With
the new pipeline, the upload happens as a separate step, thus the
upload is disabled when invoking build_torcx_store, and so the
packages are not getting URLs set. This change went unnoticed, because
a kola test checking the generated torcx manifest was only checking if
there is at least one location, either path or URL, and all the new
releases have no extra packages, only vendored ones.

When backporting the new pipeline to old LTS, the kola tests started
to fail, because old LTS had one extra package, and this is how I
noticed the problem.

build_packages

@@ -40,6 +40,8 @@ DEFINE_string torcx_output_root "${DEFAULT_BUILD_ROOT}/torcx" \
   "Directory in which to place torcx stores and manifests (named by board/version)"
 DEFINE_boolean skip_torcx_store "${FLAGS_FALSE}" \
   "Don't build a new torcx store from the updated sysroot."
+DEFINE_string torcx_extra_pkg_url "" \
+  "URL to directory where the torcx packages will be available for downloading"
 
 # include upload options
 . "${BUILD_LIBRARY_DIR}/release_util.sh" || exit 1
@@ -303,7 +305,10 @@ upload_packages
 
 # Build a new torcx store with the updated packages, passing flags through.
 if [ "${FLAGS_skip_torcx_store}" -eq "${FLAGS_FALSE}" ]; then
-  "${SCRIPTS_DIR}"/build_torcx_store --board="${BOARD}" --output_root="${FLAGS_torcx_output_root}"
+  "${SCRIPTS_DIR}"/build_torcx_store \
+                  --board="${BOARD}" \
+                  --output_root="${FLAGS_torcx_output_root}" \
+                  --extra_pkg_url="${FLAGS_torcx_extra_pkg_url}"
 fi
 
 info "Builds complete"

build_torcx_store

@@ -16,6 +16,8 @@ DEFINE_string board "${DEFAULT_BOARD}" \
   "The board to build packages for."
 DEFINE_string output_root "${DEFAULT_BUILD_ROOT}/torcx" \
   "Directory in which to place torcx stores and manifests (named by board/version)"
+DEFINE_string extra_pkg_url "" \
+  "URL to directory where the torcx packages will be available for downloading"
 
 # include upload options
 . "${BUILD_LIBRARY_DIR}/release_util.sh" || exit 1
@@ -102,6 +104,7 @@ function torcx_package() {
         local version=${pkg:${#name}+1}
         local manifest_path="${2}"
         local type="${3}"
+        local extra_pkg_url="${4}"
         local deppkg digest file rpath sha512sum source_pkg rdepends tmproot tmppkgroot update_default tmpfile
         local pkg_cas_file pkg_cas_root
         local pkg_locations=()
@@ -215,6 +218,9 @@ function torcx_package() {
         if [[ "${FLAGS_upload}" -eq ${FLAGS_TRUE} ]]; then
                 pkg_locations+=("$(download_tectonic_torcx_url "pkgs/${BOARD}/${name}/${digest}/${name}:${version}.torcx.tgz")")
         fi
+        if [[ -n "${extra_pkg_url}" ]]; then
+            pkg_locations+=("${extra_pkg_url}/${name}:${version}.torcx.tgz")
+        fi
         torcx_manifest::add_pkg "${manifest_path}" \
             "${name}" \
             "${version}" \
@@ -243,8 +249,12 @@ EXTRA_IMAGES=(
 mkdir -p "${BUILD_DIR}"
 manifest_path="${BUILD_DIR}/torcx_manifest.json"
 torcx_manifest::create_empty "${manifest_path}"
-for pkg in "${@:-${DEFAULT_IMAGES[@]}}" ; do torcx_package "${pkg#=}" "${manifest_path}" "default" ; done
+for pkg in "${@:-${DEFAULT_IMAGES[@]}}"; do
-for pkg in "${EXTRA_IMAGES[@]}" ; do torcx_package "${pkg#=}" "${manifest_path}" "extra" ; done
+        torcx_package "${pkg#=}" "${manifest_path}" "default" "${FLAGS_extra_pkg_url}"
+done
+for pkg in "${EXTRA_IMAGES[@]}"; do
+        torcx_package "${pkg#=}" "${manifest_path}" "extra" "${FLAGS_extra_pkg_url}"
+done
 
 set_build_symlinks latest "${FLAGS_group}-latest"
 

ci-automation/packages.sh

@@ -80,6 +80,18 @@ function _packages_build_impl() {
     local vernum="${FLATCAR_VERSION}"
     local docker_vernum="$(vernum_to_docker_image_version "${vernum}")"
     local packages_container="flatcar-packages-${arch}-${docker_vernum}"
+    local torcx_pkg_url="https://${BUILDCACHE_SERVER}/images/${arch}/${vernum}/torcx"
+
+    source sdk_lib/sdk_container_common.sh
+
+    if is_official "${vernum}"; then
+        # A channel returned by get_git_channel should not ever be
+        # "developer" here, because it's an official build done from
+        # one of the maintenance branches. So if the channel happens
+        # to be "developer", then you are doing it wrong (releasing
+        # from the main branch?).
+        torcx_pkg_url="https://$(get_git_channel).release.flatcar-linux.net/${arch}-usr/${vernum}/torcx"
+    fi
 
     # Build packages; store packages and torcx output in container
     ./run_sdk_container -x ./ci-cleanup.sh -n "${packages_container}" -v "${vernum}" \
@@ -88,7 +100,8 @@ function _packages_build_impl() {
     ./run_sdk_container -n "${packages_container}" -v "${vernum}" \
         -C "${sdk_image}" \
         ./build_packages --board="${arch}-usr" \
-            --torcx_output_root="${CONTAINER_TORCX_ROOT}"
+            --torcx_output_root="${CONTAINER_TORCX_ROOT}" \
+            --torcx_extra_pkg_url="${torcx_pkg_url}"
 
     # copy torcx manifest and docker tarball for publishing
     local torcx_tmp="__build__/torcx_tmp"