diff --git a/build_packages b/build_packages index c77f0c8dbd..1ad8800f48 100755 --- a/build_packages +++ b/build_packages @@ -40,6 +40,8 @@ DEFINE_string torcx_output_root "${DEFAULT_BUILD_ROOT}/torcx" \ "Directory in which to place torcx stores and manifests (named by board/version)" DEFINE_boolean skip_torcx_store "${FLAGS_FALSE}" \ "Don't build a new torcx store from the updated sysroot." +DEFINE_string torcx_extra_pkg_url "" \ + "URL to directory where the torcx packages will be available for downloading" # include upload options . "${BUILD_LIBRARY_DIR}/release_util.sh" || exit 1 @@ -303,7 +305,10 @@ upload_packages # Build a new torcx store with the updated packages, passing flags through. if [ "${FLAGS_skip_torcx_store}" -eq "${FLAGS_FALSE}" ]; then - "${SCRIPTS_DIR}"/build_torcx_store --board="${BOARD}" --output_root="${FLAGS_torcx_output_root}" + "${SCRIPTS_DIR}"/build_torcx_store \ + --board="${BOARD}" \ + --output_root="${FLAGS_torcx_output_root}" \ + --extra_pkg_url="${FLAGS_torcx_extra_pkg_url}" fi info "Builds complete" diff --git a/build_torcx_store b/build_torcx_store index c4b0f99ff9..6bc95b1ba2 100755 --- a/build_torcx_store +++ b/build_torcx_store @@ -16,6 +16,8 @@ DEFINE_string board "${DEFAULT_BOARD}" \ "The board to build packages for." DEFINE_string output_root "${DEFAULT_BUILD_ROOT}/torcx" \ "Directory in which to place torcx stores and manifests (named by board/version)" +DEFINE_string extra_pkg_url "" \ + "URL to directory where the torcx packages will be available for downloading" # include upload options . "${BUILD_LIBRARY_DIR}/release_util.sh" || exit 1 @@ -102,6 +104,7 @@ function torcx_package() { local version=${pkg:${#name}+1} local manifest_path="${2}" local type="${3}" + local extra_pkg_url="${4}" local deppkg digest file rpath sha512sum source_pkg rdepends tmproot tmppkgroot update_default tmpfile local pkg_cas_file pkg_cas_root local pkg_locations=() @@ -215,6 +218,9 @@ function torcx_package() { if [[ "${FLAGS_upload}" -eq ${FLAGS_TRUE} ]]; then pkg_locations+=("$(download_tectonic_torcx_url "pkgs/${BOARD}/${name}/${digest}/${name}:${version}.torcx.tgz")") fi + if [[ -n "${extra_pkg_url}" ]]; then + pkg_locations+=("${extra_pkg_url}/${name}:${version}.torcx.tgz") + fi torcx_manifest::add_pkg "${manifest_path}" \ "${name}" \ "${version}" \ @@ -243,8 +249,12 @@ EXTRA_IMAGES=( mkdir -p "${BUILD_DIR}" manifest_path="${BUILD_DIR}/torcx_manifest.json" torcx_manifest::create_empty "${manifest_path}" -for pkg in "${@:-${DEFAULT_IMAGES[@]}}" ; do torcx_package "${pkg#=}" "${manifest_path}" "default" ; done -for pkg in "${EXTRA_IMAGES[@]}" ; do torcx_package "${pkg#=}" "${manifest_path}" "extra" ; done +for pkg in "${@:-${DEFAULT_IMAGES[@]}}"; do + torcx_package "${pkg#=}" "${manifest_path}" "default" "${FLAGS_extra_pkg_url}" +done +for pkg in "${EXTRA_IMAGES[@]}"; do + torcx_package "${pkg#=}" "${manifest_path}" "extra" "${FLAGS_extra_pkg_url}" +done set_build_symlinks latest "${FLAGS_group}-latest" diff --git a/ci-automation/packages.sh b/ci-automation/packages.sh index 029b1f4c56..dcddb0dcbf 100644 --- a/ci-automation/packages.sh +++ b/ci-automation/packages.sh @@ -80,6 +80,18 @@ function _packages_build_impl() { local vernum="${FLATCAR_VERSION}" local docker_vernum="$(vernum_to_docker_image_version "${vernum}")" local packages_container="flatcar-packages-${arch}-${docker_vernum}" + local torcx_pkg_url="https://${BUILDCACHE_SERVER}/images/${arch}/${vernum}/torcx" + + source sdk_lib/sdk_container_common.sh + + if is_official "${vernum}"; then + # A channel returned by get_git_channel should not ever be + # "developer" here, because it's an official build done from + # one of the maintenance branches. So if the channel happens + # to be "developer", then you are doing it wrong (releasing + # from the main branch?). + torcx_pkg_url="https://$(get_git_channel).release.flatcar-linux.net/${arch}-usr/${vernum}/torcx" + fi # Build packages; store packages and torcx output in container ./run_sdk_container -x ./ci-cleanup.sh -n "${packages_container}" -v "${vernum}" \ @@ -88,7 +100,8 @@ function _packages_build_impl() { ./run_sdk_container -n "${packages_container}" -v "${vernum}" \ -C "${sdk_image}" \ ./build_packages --board="${arch}-usr" \ - --torcx_output_root="${CONTAINER_TORCX_ROOT}" + --torcx_output_root="${CONTAINER_TORCX_ROOT}" \ + --torcx_extra_pkg_url="${torcx_pkg_url}" # copy torcx manifest and docker tarball for publishing local torcx_tmp="__build__/torcx_tmp" diff --git a/ci-automation/test.sh b/ci-automation/test.sh index f5d14d05bb..6be101ee16 100644 --- a/ci-automation/test.sh +++ b/ci-automation/test.sh @@ -74,8 +74,11 @@ # script would need to make anyway. For more information, please refer # to the vendor_test.sh file. -# Download torcx package and manifest, add build cache URL to manifest -# so the docker.torcx-manifest-pkgs test can use it. +# Download torcx manifest and modify URLs pointing to the origin +# server to point to the build cache. This is because the tests for +# releases are run before artifacts are uploaded to the origin +# server. This would make kola's docker.torcx-manifest-pkgs test to +# fail. function __prepare_torcx() { local arch="$1" local vernum="$2" @@ -83,16 +86,28 @@ function __prepare_torcx() { copy_from_buildcache "images/${arch}/${vernum}/torcx/torcx_manifest.json" "${workdir}" - local docker_pkg - docker_pkg="$(basename \ - "$(jq -r ".value.packages[0].versions[0].locations[0].path" \ - ${workdir}/torcx_manifest.json)")" - - # Add docker package URL on build cache to manifest - local docker_url="http://${BUILDCACHE_SERVER}/images/${arch}/${vernum}/torcx/${docker_pkg}" - jq ".value.packages[0].versions[0].locations += [{\"url\" : \"${docker_url}\"}]" \ + # Change URLs from: + # + # https://${channel}.release.flatcar-linux.net/${arch}-usr/${vernum}/torcx/… + # + # to: + # + # https://bincache.flatcar-linux.net/images/${arch}/${vernum}/torcx/… + # + # This is done in two parts - replacing host part and arch part. + # + # Replace 'https://${channel}.release.flatcar-linux.net/' with + # 'https://bincache.flatcar-linux.net/' matching the initial "url" + # JSON key too. + local host_match='\("url":\s*"https://\)[a-z]\+\.release\([^/]\+/\)' + local host_replace='\1bincache\2' + # Replace '${arch}-usr/` part with 'images/${arch}/'. + local arch_match='\([a-z0-9]\+\)-usr/' + local arch_replace='images/\3/' + sed \ + -e "s#${host_match}${arch_match}#${host_replace}${arch_replace}#g" \ "${workdir}/torcx_manifest.json" \ - > "${workdir}/torcx_manifest_new.json" + >"${workdir}/torcx_manifest_new.json" mv "${workdir}/torcx_manifest.json" "${workdir}/torcx_manifest.json.original" mv "${workdir}/torcx_manifest_new.json" "${workdir}/torcx_manifest.json"