Merge pull request #452 from flatcar-linux/krnowak/torcx-extra-urls

*: Allow specifying extra URLs for torcx packages

build_packages

@@ -40,6 +40,8 @@ DEFINE_string torcx_output_root "${DEFAULT_BUILD_ROOT}/torcx" \
   "Directory in which to place torcx stores and manifests (named by board/version)"
 DEFINE_boolean skip_torcx_store "${FLAGS_FALSE}" \
   "Don't build a new torcx store from the updated sysroot."
+DEFINE_string torcx_extra_pkg_url "" \
+  "URL to directory where the torcx packages will be available for downloading"
 
 # include upload options
 . "${BUILD_LIBRARY_DIR}/release_util.sh" || exit 1
@@ -303,7 +305,10 @@ upload_packages
 
 # Build a new torcx store with the updated packages, passing flags through.
 if [ "${FLAGS_skip_torcx_store}" -eq "${FLAGS_FALSE}" ]; then
-  "${SCRIPTS_DIR}"/build_torcx_store --board="${BOARD}" --output_root="${FLAGS_torcx_output_root}"
+  "${SCRIPTS_DIR}"/build_torcx_store \
+                  --board="${BOARD}" \
+                  --output_root="${FLAGS_torcx_output_root}" \
+                  --extra_pkg_url="${FLAGS_torcx_extra_pkg_url}"
 fi
 
 info "Builds complete"

build_torcx_store

@@ -16,6 +16,8 @@ DEFINE_string board "${DEFAULT_BOARD}" \
   "The board to build packages for."
 DEFINE_string output_root "${DEFAULT_BUILD_ROOT}/torcx" \
   "Directory in which to place torcx stores and manifests (named by board/version)"
+DEFINE_string extra_pkg_url "" \
+  "URL to directory where the torcx packages will be available for downloading"
 
 # include upload options
 . "${BUILD_LIBRARY_DIR}/release_util.sh" || exit 1
@@ -102,6 +104,7 @@ function torcx_package() {
         local version=${pkg:${#name}+1}
         local manifest_path="${2}"
         local type="${3}"
+        local extra_pkg_url="${4}"
         local deppkg digest file rpath sha512sum source_pkg rdepends tmproot tmppkgroot update_default tmpfile
         local pkg_cas_file pkg_cas_root
         local pkg_locations=()
@@ -215,6 +218,9 @@ function torcx_package() {
         if [[ "${FLAGS_upload}" -eq ${FLAGS_TRUE} ]]; then
                 pkg_locations+=("$(download_tectonic_torcx_url "pkgs/${BOARD}/${name}/${digest}/${name}:${version}.torcx.tgz")")
         fi
+        if [[ -n "${extra_pkg_url}" ]]; then
+            pkg_locations+=("${extra_pkg_url}/${name}:${version}.torcx.tgz")
+        fi
         torcx_manifest::add_pkg "${manifest_path}" \
             "${name}" \
             "${version}" \
@@ -243,8 +249,12 @@ EXTRA_IMAGES=(
 mkdir -p "${BUILD_DIR}"
 manifest_path="${BUILD_DIR}/torcx_manifest.json"
 torcx_manifest::create_empty "${manifest_path}"
-for pkg in "${@:-${DEFAULT_IMAGES[@]}}" ; do torcx_package "${pkg#=}" "${manifest_path}" "default" ; done
+for pkg in "${@:-${DEFAULT_IMAGES[@]}}"; do
-for pkg in "${EXTRA_IMAGES[@]}" ; do torcx_package "${pkg#=}" "${manifest_path}" "extra" ; done
+        torcx_package "${pkg#=}" "${manifest_path}" "default" "${FLAGS_extra_pkg_url}"
+done
+for pkg in "${EXTRA_IMAGES[@]}"; do
+        torcx_package "${pkg#=}" "${manifest_path}" "extra" "${FLAGS_extra_pkg_url}"
+done
 
 set_build_symlinks latest "${FLAGS_group}-latest"
 

ci-automation/packages.sh

@@ -80,6 +80,18 @@ function _packages_build_impl() {
     local vernum="${FLATCAR_VERSION}"
     local docker_vernum="$(vernum_to_docker_image_version "${vernum}")"
     local packages_container="flatcar-packages-${arch}-${docker_vernum}"
+    local torcx_pkg_url="https://${BUILDCACHE_SERVER}/images/${arch}/${vernum}/torcx"
+
+    source sdk_lib/sdk_container_common.sh
+
+    if is_official "${vernum}"; then
+        # A channel returned by get_git_channel should not ever be
+        # "developer" here, because it's an official build done from
+        # one of the maintenance branches. So if the channel happens
+        # to be "developer", then you are doing it wrong (releasing
+        # from the main branch?).
+        torcx_pkg_url="https://$(get_git_channel).release.flatcar-linux.net/${arch}-usr/${vernum}/torcx"
+    fi
 
     # Build packages; store packages and torcx output in container
     ./run_sdk_container -x ./ci-cleanup.sh -n "${packages_container}" -v "${vernum}" \
@@ -88,7 +100,8 @@ function _packages_build_impl() {
     ./run_sdk_container -n "${packages_container}" -v "${vernum}" \
         -C "${sdk_image}" \
         ./build_packages --board="${arch}-usr" \
-            --torcx_output_root="${CONTAINER_TORCX_ROOT}"
+            --torcx_output_root="${CONTAINER_TORCX_ROOT}" \
+            --torcx_extra_pkg_url="${torcx_pkg_url}"
 
     # copy torcx manifest and docker tarball for publishing
     local torcx_tmp="__build__/torcx_tmp"

ci-automation/test.sh

@@ -74,8 +74,11 @@
 # script would need to make anyway. For more information, please refer
 # to the vendor_test.sh file.
 
-# Download torcx package and manifest, add build cache URL to manifest
+# Download torcx manifest and modify URLs pointing to the origin
-#  so the docker.torcx-manifest-pkgs test can use it.
+# server to point to the build cache. This is because the tests for
+# releases are run before artifacts are uploaded to the origin
+# server. This would make kola's docker.torcx-manifest-pkgs test to
+# fail.
 function __prepare_torcx() {
     local arch="$1"
     local vernum="$2"
@@ -83,16 +86,28 @@ function __prepare_torcx() {
 
     copy_from_buildcache "images/${arch}/${vernum}/torcx/torcx_manifest.json" "${workdir}"
 
-    local docker_pkg
+    # Change URLs from:
-    docker_pkg="$(basename \
+    #
-                        "$(jq -r ".value.packages[0].versions[0].locations[0].path" \
+    # https://${channel}.release.flatcar-linux.net/${arch}-usr/${vernum}/torcx/…
-                        ${workdir}/torcx_manifest.json)")"
+    #
-
+    # to:
-    # Add docker package URL on build cache to manifest
+    #
-    local docker_url="http://${BUILDCACHE_SERVER}/images/${arch}/${vernum}/torcx/${docker_pkg}"
+    # https://bincache.flatcar-linux.net/images/${arch}/${vernum}/torcx/…
-    jq ".value.packages[0].versions[0].locations += [{\"url\" : \"${docker_url}\"}]" \
+    #
+    # This is done in two parts - replacing host part and arch part.
+    #
+    # Replace 'https://${channel}.release.flatcar-linux.net/' with
+    # 'https://bincache.flatcar-linux.net/' matching the initial "url"
+    # JSON key too.
+    local host_match='\("url":\s*"https://\)[a-z]\+\.release\([^/]\+/\)'
+    local host_replace='\1bincache\2'
+    # Replace '${arch}-usr/` part with 'images/${arch}/'.
+    local arch_match='\([a-z0-9]\+\)-usr/'
+    local arch_replace='images/\3/'
+    sed \
+        -e "s#${host_match}${arch_match}#${host_replace}${arch_replace}#g" \
         "${workdir}/torcx_manifest.json" \
-        > "${workdir}/torcx_manifest_new.json"
+        >"${workdir}/torcx_manifest_new.json"
 
     mv "${workdir}/torcx_manifest.json" "${workdir}/torcx_manifest.json.original"
     mv "${workdir}/torcx_manifest_new.json" "${workdir}/torcx_manifest.json"