From b2d6f7fc6e28b044da25027bf3a0e7b4d9ae4f75 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Thu, 21 Jul 2022 17:00:04 +0200 Subject: [PATCH 1/2] *: Allow specifying extra URLs for torcx packages Torcx manifest may contain paths and URLs as locations of packages. There are two kinds of packages - vendored and extra. Vendored packages normally have two locations - path to the directory inside the image where the package is (which is why it's called vendored), and a URL to the package on some remote server. Extra packages only have a URL. But the URLs are added only when we tell the build_torcx_store script to upload the packages at the same time, which is what the old build pipeline was doing. With the new pipeline, the upload happens as a separate step, thus the upload is disabled when invoking build_torcx_store, and so the packages are not getting URLs set. This change went unnoticed, because a kola test checking the generated torcx manifest was only checking if there is at least one location, either path or URL, and all the new releases have no extra packages, only vendored ones. When backporting the new pipeline to old LTS, the kola tests started to fail, because old LTS had one extra package, and this is how I noticed the problem. --- build_packages | 7 ++++++- build_torcx_store | 14 ++++++++++++-- ci-automation/packages.sh | 15 ++++++++++++++- 3 files changed, 32 insertions(+), 4 deletions(-) diff --git a/build_packages b/build_packages index c77f0c8dbd..1ad8800f48 100755 --- a/build_packages +++ b/build_packages @@ -40,6 +40,8 @@ DEFINE_string torcx_output_root "${DEFAULT_BUILD_ROOT}/torcx" \ "Directory in which to place torcx stores and manifests (named by board/version)" DEFINE_boolean skip_torcx_store "${FLAGS_FALSE}" \ "Don't build a new torcx store from the updated sysroot." +DEFINE_string torcx_extra_pkg_url "" \ + "URL to directory where the torcx packages will be available for downloading" # include upload options . "${BUILD_LIBRARY_DIR}/release_util.sh" || exit 1 @@ -303,7 +305,10 @@ upload_packages # Build a new torcx store with the updated packages, passing flags through. if [ "${FLAGS_skip_torcx_store}" -eq "${FLAGS_FALSE}" ]; then - "${SCRIPTS_DIR}"/build_torcx_store --board="${BOARD}" --output_root="${FLAGS_torcx_output_root}" + "${SCRIPTS_DIR}"/build_torcx_store \ + --board="${BOARD}" \ + --output_root="${FLAGS_torcx_output_root}" \ + --extra_pkg_url="${FLAGS_torcx_extra_pkg_url}" fi info "Builds complete" diff --git a/build_torcx_store b/build_torcx_store index c4b0f99ff9..6bc95b1ba2 100755 --- a/build_torcx_store +++ b/build_torcx_store @@ -16,6 +16,8 @@ DEFINE_string board "${DEFAULT_BOARD}" \ "The board to build packages for." DEFINE_string output_root "${DEFAULT_BUILD_ROOT}/torcx" \ "Directory in which to place torcx stores and manifests (named by board/version)" +DEFINE_string extra_pkg_url "" \ + "URL to directory where the torcx packages will be available for downloading" # include upload options . "${BUILD_LIBRARY_DIR}/release_util.sh" || exit 1 @@ -102,6 +104,7 @@ function torcx_package() { local version=${pkg:${#name}+1} local manifest_path="${2}" local type="${3}" + local extra_pkg_url="${4}" local deppkg digest file rpath sha512sum source_pkg rdepends tmproot tmppkgroot update_default tmpfile local pkg_cas_file pkg_cas_root local pkg_locations=() @@ -215,6 +218,9 @@ function torcx_package() { if [[ "${FLAGS_upload}" -eq ${FLAGS_TRUE} ]]; then pkg_locations+=("$(download_tectonic_torcx_url "pkgs/${BOARD}/${name}/${digest}/${name}:${version}.torcx.tgz")") fi + if [[ -n "${extra_pkg_url}" ]]; then + pkg_locations+=("${extra_pkg_url}/${name}:${version}.torcx.tgz") + fi torcx_manifest::add_pkg "${manifest_path}" \ "${name}" \ "${version}" \ @@ -243,8 +249,12 @@ EXTRA_IMAGES=( mkdir -p "${BUILD_DIR}" manifest_path="${BUILD_DIR}/torcx_manifest.json" torcx_manifest::create_empty "${manifest_path}" -for pkg in "${@:-${DEFAULT_IMAGES[@]}}" ; do torcx_package "${pkg#=}" "${manifest_path}" "default" ; done -for pkg in "${EXTRA_IMAGES[@]}" ; do torcx_package "${pkg#=}" "${manifest_path}" "extra" ; done +for pkg in "${@:-${DEFAULT_IMAGES[@]}}"; do + torcx_package "${pkg#=}" "${manifest_path}" "default" "${FLAGS_extra_pkg_url}" +done +for pkg in "${EXTRA_IMAGES[@]}"; do + torcx_package "${pkg#=}" "${manifest_path}" "extra" "${FLAGS_extra_pkg_url}" +done set_build_symlinks latest "${FLAGS_group}-latest" diff --git a/ci-automation/packages.sh b/ci-automation/packages.sh index 029b1f4c56..dcddb0dcbf 100644 --- a/ci-automation/packages.sh +++ b/ci-automation/packages.sh @@ -80,6 +80,18 @@ function _packages_build_impl() { local vernum="${FLATCAR_VERSION}" local docker_vernum="$(vernum_to_docker_image_version "${vernum}")" local packages_container="flatcar-packages-${arch}-${docker_vernum}" + local torcx_pkg_url="https://${BUILDCACHE_SERVER}/images/${arch}/${vernum}/torcx" + + source sdk_lib/sdk_container_common.sh + + if is_official "${vernum}"; then + # A channel returned by get_git_channel should not ever be + # "developer" here, because it's an official build done from + # one of the maintenance branches. So if the channel happens + # to be "developer", then you are doing it wrong (releasing + # from the main branch?). + torcx_pkg_url="https://$(get_git_channel).release.flatcar-linux.net/${arch}-usr/${vernum}/torcx" + fi # Build packages; store packages and torcx output in container ./run_sdk_container -x ./ci-cleanup.sh -n "${packages_container}" -v "${vernum}" \ @@ -88,7 +100,8 @@ function _packages_build_impl() { ./run_sdk_container -n "${packages_container}" -v "${vernum}" \ -C "${sdk_image}" \ ./build_packages --board="${arch}-usr" \ - --torcx_output_root="${CONTAINER_TORCX_ROOT}" + --torcx_output_root="${CONTAINER_TORCX_ROOT}" \ + --torcx_extra_pkg_url="${torcx_pkg_url}" # copy torcx manifest and docker tarball for publishing local torcx_tmp="__build__/torcx_tmp" From 1ecea3544f91e61322f0596c552bcd83704ea44c Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Tue, 6 Sep 2022 13:23:05 +0200 Subject: [PATCH 2/2] ci-automation: Change the way we prepare torcx manifest for testing Now URLs for torcx packages are always present in the torcx manifest, but for releases they may be pointing to the origin server where the packages will be eventually uploaded. At the time of running the tests, those packages are still only in the build cache, so change the URLs to point to the build cache, so the test can pass. --- ci-automation/test.sh | 37 ++++++++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/ci-automation/test.sh b/ci-automation/test.sh index f5d14d05bb..6be101ee16 100644 --- a/ci-automation/test.sh +++ b/ci-automation/test.sh @@ -74,8 +74,11 @@ # script would need to make anyway. For more information, please refer # to the vendor_test.sh file. -# Download torcx package and manifest, add build cache URL to manifest -# so the docker.torcx-manifest-pkgs test can use it. +# Download torcx manifest and modify URLs pointing to the origin +# server to point to the build cache. This is because the tests for +# releases are run before artifacts are uploaded to the origin +# server. This would make kola's docker.torcx-manifest-pkgs test to +# fail. function __prepare_torcx() { local arch="$1" local vernum="$2" @@ -83,16 +86,28 @@ function __prepare_torcx() { copy_from_buildcache "images/${arch}/${vernum}/torcx/torcx_manifest.json" "${workdir}" - local docker_pkg - docker_pkg="$(basename \ - "$(jq -r ".value.packages[0].versions[0].locations[0].path" \ - ${workdir}/torcx_manifest.json)")" - - # Add docker package URL on build cache to manifest - local docker_url="http://${BUILDCACHE_SERVER}/images/${arch}/${vernum}/torcx/${docker_pkg}" - jq ".value.packages[0].versions[0].locations += [{\"url\" : \"${docker_url}\"}]" \ + # Change URLs from: + # + # https://${channel}.release.flatcar-linux.net/${arch}-usr/${vernum}/torcx/… + # + # to: + # + # https://bincache.flatcar-linux.net/images/${arch}/${vernum}/torcx/… + # + # This is done in two parts - replacing host part and arch part. + # + # Replace 'https://${channel}.release.flatcar-linux.net/' with + # 'https://bincache.flatcar-linux.net/' matching the initial "url" + # JSON key too. + local host_match='\("url":\s*"https://\)[a-z]\+\.release\([^/]\+/\)' + local host_replace='\1bincache\2' + # Replace '${arch}-usr/` part with 'images/${arch}/'. + local arch_match='\([a-z0-9]\+\)-usr/' + local arch_replace='images/\3/' + sed \ + -e "s#${host_match}${arch_match}#${host_replace}${arch_replace}#g" \ "${workdir}/torcx_manifest.json" \ - > "${workdir}/torcx_manifest_new.json" + >"${workdir}/torcx_manifest_new.json" mv "${workdir}/torcx_manifest.json" "${workdir}/torcx_manifest.json.original" mv "${workdir}/torcx_manifest_new.json" "${workdir}/torcx_manifest.json"