From f373f06fb72197faa2927732d2ab31bba914ee36 Mon Sep 17 00:00:00 2001
From: Krzesimir Nowak <knowak@microsoft.com>
Date: Thu, 11 Sep 2025 15:29:09 +0200
Subject: [PATCH] *: Scavenge for configure logs and upload them to bincache

This searches for portage logs and configure logs after a build, and
uploads them to bincache. This is currently only done for SDK builds,
SDK container builds and package builds. This probably could be
extended to catch logs for sysext builds, but this was annoying to
implement, IIRC.

Signed-off-by: Krzesimir Nowak <knowak@microsoft.com>
---
 bootstrap_sdk_container               |  8 ++++-
 build_sdk_container_image             | 26 +++++++++++++-
 ci-automation/ci_automation_common.sh | 22 ++++++++++++
 ci-automation/garbage_collect.sh      |  2 ++
 ci-automation/packages.sh             | 37 +++++++++++++++-----
 ci-automation/sdk_bootstrap.sh        | 36 +++++++++++++++++---
 ci-automation/sdk_container.sh        | 34 +++++++++++++++----
 sdk_lib/Dockerfile.sdk-build          |  8 +----
 sdk_lib/sdk_container_common.sh       | 37 ++++++++++++++++++++
 sdk_lib/setup_boards.sh               | 49 +++++++++++++++++++++++++++
 10 files changed, 231 insertions(+), 28 deletions(-)
 create mode 100755 sdk_lib/setup_boards.sh

diff --git a/bootstrap_sdk_container b/bootstrap_sdk_container
index 6384e69e49..c197787179 100755
--- a/bootstrap_sdk_container
+++ b/bootstrap_sdk_container
@@ -11,6 +11,7 @@ source sdk_lib/sdk_container_common.sh
 
 seed_version=""
 target_version=""
+logdir=''
 
 declare -a cleanup
 
@@ -30,6 +31,7 @@ usage() {
     echo "      -x <cleanup-script> - For each resource generated during build (container etc.)"
     echo "                             add a cleanup line to <script> which, when run, will free"
     echo "                             the resource. Useful for CI."
+    echo "      -l <directory>      - Gather build logs here."
     echo "      -h                  - Print this help."
     echo
 }
@@ -38,6 +40,7 @@ usage() {
 while [ 0 -lt $# ] ; do
     case "$1" in
     -h) usage; exit 0;;
+    -l) logdir=${2}; shift 2;;
     -x) cleanup=("-x" "$2"); shift; shift;;
     *)  if [ -z "$seed_version" ] ; then
             seed_version="$1"
@@ -72,8 +75,11 @@ if $official; then
 fi
 
 # bootstrap_sdk needs FLATCAR_SDK_VERSION set to the seed version
+failed=''
 ./run_sdk_container "${cleanup[@]}" -V "$seed_version" -v "$target_version" \
-          sudo -E ./bootstrap_sdk
+          sudo -E ./bootstrap_sdk || failed=x
 
 # Update versionfile to the actual SDK version
 create_versionfile "${target_version}"
+
+if [[ -n ${failed} ]]; then exit 1; fi
diff --git a/build_sdk_container_image b/build_sdk_container_image
index 62146ab9b7..7321f09cd3 100755
--- a/build_sdk_container_image
+++ b/build_sdk_container_image
@@ -34,6 +34,7 @@ arch="amd64"
 official="0"
 tarball=""
 os_version=""
+logdir=""
 
 keep="false"
 cleanup=""
@@ -57,6 +58,7 @@ usage() {
     echo "                     add a cleanup line to <script> which, when run, will free"
     echo "                     the resource. Useful for CI."
     echo "      -h           - Print this help."
+    echo "      -l <dir>     - Put logs in this directory"
     echo
 }
 
@@ -66,6 +68,7 @@ while [ 0 -lt $# ] ; do
     case "$1" in
     -h) usage; exit 0;;
     -k) keep="true";      shift;;
+    -l) logdir="${2}";    shift; shift;;
     -v) os_version="$2";  shift; shift;;
     -x) cleanup="$2";     shift; shift;;
     *)  if [ -z "$tarball" ] ;  then
@@ -102,6 +105,10 @@ else
     official="0"
 fi
 
+if [[ -n ${logdir} ]]; then
+    mkdir -p "${logdir}"
+fi
+
 # --
 
 # import tarball
@@ -170,8 +177,19 @@ else
     if [ -n "$cleanup" ] ; then
         echo "$docker container rm -f '${toolchains_container}'" >> "$cleanup"
     fi
+    failed=''
     ./run_sdk_container -C "${import_image}" -n "${toolchains_container}" \
-            sudo ./build_toolchains --seed_tarball="./${tarball}"
+            sudo ./build_toolchains --seed_tarball="./${tarball}" || failed=x
+
+    if [[ -n ${logdir} ]]; then
+        if sudo test -d __build__/images/catalyst/log/coreos-toolchains; then
+            sudo cp -a __build__/images/catalyst/log/coreos-toolchains "${logdir}/coreos-toolchains"
+        fi
+        if sudo test -d __build__/images/catalyst/tmp/coreos-toolchains; then
+            scavenge_for_configure_logs --use-sudo __build__/images/catalyst/tmp/coreos-toolchains "${logdir}"
+        fi
+    fi
+    if [[ -n ${failed} ]]; then exit 1; fi
 
     # remove sdk tarball from scripts root so it's not part of the SDK container build context
     if [ -f "${tarball_copied}" ] ; then
@@ -216,6 +234,12 @@ else
                  .
 
     $docker stop "${binhost_container}"
+
+    if [[ -n ${logdir} ]]; then
+        $docker run --rm -v "${logdir}:/logdir" "${sdk_build_image}" ./sdk_lib/setup_boards.sh finish /logdir
+    else
+        $docker run --rm "${sdk_build_image}" ./sdk_lib/setup_boards.sh finish
+    fi
 fi
 
 # --
diff --git a/ci-automation/ci_automation_common.sh b/ci-automation/ci_automation_common.sh
index fb38d87e5a..1ded3d1e36 100644
--- a/ci-automation/ci_automation_common.sh
+++ b/ci-automation/ci_automation_common.sh
@@ -111,7 +111,29 @@ function copy_dir_from_buildcache() {
     rsync --partial -a -e "${sshcmd}" "${BUILDCACHE_USER}@${BUILDCACHE_SERVER}:${remote_path}" \
         "${local_path}"
 }
+# --
 
+# Returns true if the passed directory contains files that match any
+# of the passed glob strings. Remember to pass the glob as strings
+# (means: quote them, so globs are not resolved at the function
+# callsite).
+function dir_contains_globs() (
+    shopt -s nullglob
+
+    local dir=${1}; shift
+    # rest are globs
+
+    # this is in a subshell, so cd is safe - won't mess with working
+    # directory of the caller
+    cd "${dir}" 2>/dev/null || return 1
+    # for globs to be evaluated, we must not use quotes here
+    local -a files=( ${@} )
+
+    if [[ ${#files[@]} -gt 0 ]]; then
+        return 0
+    fi
+    return 1
+)
 # --
 
 function copy_to_buildcache() {
diff --git a/ci-automation/garbage_collect.sh b/ci-automation/garbage_collect.sh
index 437d8ce6bd..39398b333e 100644
--- a/ci-automation/garbage_collect.sh
+++ b/ci-automation/garbage_collect.sh
@@ -152,6 +152,7 @@ function _garbage_collect_impl() {
         rmpat="${rmpat} ${BUILDCACHE_PATH_PREFIX}/containers/${os_docker_vernum}/flatcar-images-*"
         rmpat="${rmpat} ${BUILDCACHE_PATH_PREFIX}/images/*/${os_vernum}/"
         rmpat="${rmpat} ${BUILDCACHE_PATH_PREFIX}/testing/${os_vernum}/"
+        rmpat="${rmpat} ${BUILDCACHE_PATH_PREFIX}/build-logs/${os_vernum}/"
 
         echo "## The following files will be removed ##"
         $sshcmd "${BUILDCACHE_USER}@${BUILDCACHE_SERVER}" \
@@ -211,6 +212,7 @@ function _garbage_collect_impl() {
                 "images/amd64" \
                 "images/arm64" \
                 "testing" \
+                "build-logs" \
                 ; do
         local fullpath="${BUILDCACHE_PATH_PREFIX}/${dir}"
         echo
diff --git a/ci-automation/packages.sh b/ci-automation/packages.sh
index e6e984f7f3..137ec48857 100644
--- a/ci-automation/packages.sh
+++ b/ci-automation/packages.sh
@@ -80,22 +80,43 @@ function _packages_build_impl() {
     local vernum="${FLATCAR_VERSION}"
     local docker_vernum="$(vernum_to_docker_image_version "${vernum}")"
     local packages_container="flatcar-packages-${arch}-${docker_vernum}"
+    local logs_tarball="packages-${arch}-logs-$(date --utc '+%F-%H%M-%S').tar.xz"
 
     source sdk_lib/sdk_container_common.sh
 
+    logdir=__build__/packages-logs-to-upload
+    mkdir -p "${logdir}"
+
     apply_local_patches
     # Build packages; store packages in container
+    failed=''
     ./run_sdk_container -x ./ci-cleanup.sh  -n "${packages_container}" -v "${vernum}" \
         -C "${sdk_image}" \
-        ./build_packages --board="${arch}-usr"
+        ./build_packages --board="${arch}-usr" || failed=x
 
-    # run_sdk_container updates the version file, use that version from here on
-    source sdk_container/.repo/manifests/version.txt
-    local vernum="${FLATCAR_VERSION}"
-    local docker_vernum="$(vernum_to_docker_image_version "${vernum}")"
-    local packages_image="flatcar-packages-${arch}"
+    # Copy the build logs to a directory accessible for us.
+    ./run_sdk_container -n "${packages_container}" -v "${vernum}" -C "${sdk_image}" \
+        bash -c "source sdk_lib/sdk_container_common.sh; cp -a /build/${arch@Q}-usr/var/log/portage ${logdir@Q}; scavenge_for_configure_logs /build/${arch@Q}-usr/var/tmp/portage ${logdir@Q}" || :
 
-    # generate image + push to build cache
-    docker_commit_to_buildcache "${packages_container}" "${packages_image}" "${docker_vernum}"
+    if [[ -z ${failed} ]]; then
+        # run_sdk_container updates the version file, use that version from here on
+        source sdk_container/.repo/manifests/version.txt
+        local vernum="${FLATCAR_VERSION}"
+        local docker_vernum="$(vernum_to_docker_image_version "${vernum}")"
+        local packages_image="flatcar-packages-${arch}"
+
+        # generate image + push to build cache
+        docker_commit_to_buildcache "${packages_container}" "${packages_image}" "${docker_vernum}"
+    fi
+    if dir_contains_globs "${logdir}" '*'; then
+        (
+            cd "${logdir}"
+            tar -cJf "${logs_tarball}" *
+            create_digests "${SIGNER}" "${logs_tarball}"
+            sign_artifacts "${SIGNER}" "${logs_tarball}"*
+            copy_to_buildcache "build-logs/${FLATCAR_SDK_VERSION}" "${logs_tarball}"*
+        )
+    fi
+    if [[ -n ${failed} ]]; then exit 1; fi
 }
 # --
diff --git a/ci-automation/sdk_bootstrap.sh b/ci-automation/sdk_bootstrap.sh
index 89d5b4bdc6..0837e16a61 100644
--- a/ci-automation/sdk_bootstrap.sh
+++ b/ci-automation/sdk_bootstrap.sh
@@ -176,13 +176,17 @@ function _sdk_bootstrap_impl() {
     fi
     apply_local_patches
 
-    ./bootstrap_sdk_container -x ./ci-cleanup.sh "${seed_version}" "${vernum}"
+    local failed=''
+    local logdir='__build__/sdk-bootstrap-logs-to-upload/'
+    mkdir -p "${logdir}"
+    ./bootstrap_sdk_container -l "${logdir}" -x ./ci-cleanup.sh "${seed_version}" "${vernum}" || failed=x
 
     # push SDK tarball to buildcache
     # Get Flatcar version number format (separator is '+' instead of '-',
     # equal to $(strip_version_prefix "$version")
     source sdk_container/.repo/manifests/version.txt
     local dest_tarball="flatcar-sdk-${ARCH}-${FLATCAR_SDK_VERSION}.tar.bz2"
+    local logs_tarball="sdk-bootstrap-logs-${ARCH}-$(date --utc '+%F-%H%M-%S').tar.xz"
 
     # change the owner of the files and directories in __build__ back
     # to ourselves, otherwise we could fail to sign the artifacts as
@@ -193,11 +197,33 @@ function _sdk_bootstrap_impl() {
     uid=$(id --user)
     gid=$(id --group)
     sudo chown --recursive "${uid}:${gid}" __build__
+    if [[ -z ${failed} ]]; then
+        (
+            cd "__build__/images/catalyst/builds/flatcar-sdk"
+            create_digests "${SIGNER}" "${dest_tarball}"
+            sign_artifacts "${SIGNER}" "${dest_tarball}"*
+            copy_to_buildcache "sdk/${ARCH}/${FLATCAR_SDK_VERSION}" "${dest_tarball}"*
+        )
+    fi
+
+    # collect logs
+    local catalyst_log='__build__/images/catalyst/log/flatcar-sdk'
+    if dir_contains_globs "${catalyst_log}" 'stage*'; then
+        cp -a "${catalyst_log}/stage"* "${logdir}"
+    fi
     (
-        cd "__build__/images/catalyst/builds/flatcar-sdk"
-        create_digests "${SIGNER}" "${dest_tarball}"
-        sign_artifacts "${SIGNER}" "${dest_tarball}"*
-        copy_to_buildcache "sdk/${ARCH}/${FLATCAR_SDK_VERSION}" "${dest_tarball}"*
+        source sdk_lib/sdk_container_common.sh
+        scavenge_for_configure_logs __build__/images/catalyst/tmp/flatcar-sdk "${logdir}"
     )
+    if dir_contains_globs "${logdir}" '*'; then
+        (
+            cd "${logdir}"
+            tar -cJf "${logs_tarball}" *
+            create_digests "${SIGNER}" "${logs_tarball}"
+            sign_artifacts "${SIGNER}" "${logs_tarball}"*
+            copy_to_buildcache "build-logs/${FLATCAR_SDK_VERSION}" "${logs_tarball}"*
+        )
+    fi
+    if [[ -n ${failed} ]]; then exit 1; fi
 }
 # --
diff --git a/ci-automation/sdk_container.sh b/ci-automation/sdk_container.sh
index e387558191..a0eb76189f 100644
--- a/ci-automation/sdk_container.sh
+++ b/ci-automation/sdk_container.sh
@@ -65,13 +65,35 @@ function _sdk_container_build_impl() {
     copy_from_buildcache "sdk/${ARCH}/${vernum}/${sdk_tarball}" "./__build__"
 
 
+    logdir=__build__/sdk-container-logs-to-upload
+    mkdir -p "${logdir}"
+    local failed=''
+    local logs_tarball="sdk-container-logs-${ARCH}-$(date --utc '+%F-%H%M-%S').tar.xz"
     # This will update the SDK_VERSION in versionfile
-    ./build_sdk_container_image -x ./ci-cleanup.sh ./__build__/"${sdk_tarball}"
+    ./build_sdk_container_image -l "${PWD}/${logdir}" -x ./ci-cleanup.sh ./__build__/"${sdk_tarball}" || failed=x
 
-    # push artifacts to build cache
-    local docker_vernum="$(vernum_to_docker_image_version "${vernum}")"
-    docker_image_to_buildcache "${CONTAINER_REGISTRY}/flatcar-sdk-all" "${docker_vernum}"
-    docker_image_to_buildcache "${CONTAINER_REGISTRY}/flatcar-sdk-amd64" "${docker_vernum}"
-    docker_image_to_buildcache "${CONTAINER_REGISTRY}/flatcar-sdk-arm64" "${docker_vernum}"
+    if [[ -z ${failed} ]]; then
+        # push artifacts to build cache
+        local docker_vernum="$(vernum_to_docker_image_version "${vernum}")"
+        docker_image_to_buildcache "${CONTAINER_REGISTRY}/flatcar-sdk-all" "${docker_vernum}"
+        docker_image_to_buildcache "${CONTAINER_REGISTRY}/flatcar-sdk-amd64" "${docker_vernum}"
+        docker_image_to_buildcache "${CONTAINER_REGISTRY}/flatcar-sdk-arm64" "${docker_vernum}"
+    fi
+    local uid
+    local gid
+    uid=$(id --user)
+    gid=$(id --group)
+    sudo chown --recursive "${uid}:${gid}" "${logdir}"
+    chmod --recursive a+rX,u+w "${logdir}"
+    if dir_contains_globs "${logdir}" '*'; then
+        (
+            cd "${logdir}"
+            tar -cJf "${logs_tarball}" *
+            create_digests "${SIGNER}" "${logs_tarball}"
+            sign_artifacts "${SIGNER}" "${logs_tarball}"*
+            copy_to_buildcache "build-logs/${FLATCAR_SDK_VERSION}" "${logs_tarball}"*
+        )
+    fi
+    if [[ -n ${failed} ]]; then exit 1; fi
 }
 # --
diff --git a/sdk_lib/Dockerfile.sdk-build b/sdk_lib/Dockerfile.sdk-build
index b61af9cd91..d532f70065 100644
--- a/sdk_lib/Dockerfile.sdk-build
+++ b/sdk_lib/Dockerfile.sdk-build
@@ -7,13 +7,7 @@ ARG OFFICIAL=0
 # mark build as official where appropriate
 RUN echo "export COREOS_OFFICIAL=$OFFICIAL" > /mnt/host/source/.env
 
-RUN /home/sdk/sdk_entry.sh ./setup_board --board="arm64-usr" --binhost="${BINHOST}/arm64-usr"
-RUN /home/sdk/sdk_entry.sh ./setup_board --board="arm64-usr" --regen_configs
-RUN /home/sdk/sdk_entry.sh ./build_packages --board="arm64-usr" --only_resolve_circular_deps
-
-RUN /home/sdk/sdk_entry.sh ./setup_board --board="amd64-usr" --binhost="${BINHOST}/amd64-usr"
-RUN /home/sdk/sdk_entry.sh ./setup_board --board="amd64-usr" --regen_configs
-RUN /home/sdk/sdk_entry.sh ./build_packages --board="amd64-usr" --only_resolve_circular_deps
+RUN /home/sdk/sdk_entry.sh ./sdk_lib/setup_boards.sh start "${BINHOST}"
 
 RUN rm /mnt/host/source/.env
 RUN rm -rf /home/sdk/toolchain-pkgs
diff --git a/sdk_lib/sdk_container_common.sh b/sdk_lib/sdk_container_common.sh
index 79d0abfe38..d1514a5f72 100644
--- a/sdk_lib/sdk_container_common.sh
+++ b/sdk_lib/sdk_container_common.sh
@@ -314,3 +314,40 @@ function gnupg_ssh_gcloud_mount_opts() {
         fi
     fi
 }
+
+function scavenge_for_configure_logs() {
+    local -a sudo_cmd=()
+    while [[ $# -gt 0 ]]; do
+        case ${1} in
+            --use-sudo) sudo_cmd=( sudo ); shift;;
+            --) shift; break;;
+            --*) echo "unknown flag for $0: $1" >&2; exit 1;;
+            *) break;;
+        esac
+    done
+    local dir=${1}; shift
+    local logdir=${1}; shift
+
+    # TODO: Add more interesting files
+    local -a interesting_files=( config.log CMakeConfigureLog.yaml meson-log.txt ) find_flags=()
+    for f in "${interesting_files[@]}"; do
+        if [[ ${#find_flags[@]} -ne 0 ]]; then
+            find_flags+=( '-o' )
+        fi
+        find_flags+=( '-name' "${f}" )
+    done
+    local -a logs
+    local l d
+    mapfile -t logs < <("${sudo_cmd[@]}" find "${dir}" "${find_flags[@]}")
+    for l in "${logs[@]}"; do
+        d=${l#"${dir}"}
+        d=${d#/}
+        if [[ ${d} = */* ]]; then
+            d=${d%/*}
+        else
+            d='.'
+        fi
+        "${sudo_cmd[@]}" mkdir -p "${logdir}/config-logs/${d}"
+        "${sudo_cmd[@]}" cp -a "${l}" "${logdir}/config-logs/${d}"
+    done
+}
diff --git a/sdk_lib/setup_boards.sh b/sdk_lib/setup_boards.sh
new file mode 100755
index 0000000000..f1ccc82dcf
--- /dev/null
+++ b/sdk_lib/setup_boards.sh
@@ -0,0 +1,49 @@
+#!/bin/bash
+
+# Script used in Dockerfile.sdk-build. The failure is indicated by the
+# BUILD_PACKAGES_FAILED file in the scripts directory in the built
+# docker image. We do it to be able to recover the build logs from
+# failed builds too.
+
+set -euo pipefail
+
+phase=${1}; shift
+case ${phase} in
+
+    start)
+        binhost=${1}; shift
+        if ! ( ./setup_board --board="arm64-usr" --binhost="${binhost}/arm64-usr" && \
+                   ./setup_board --board="arm64-usr" --regen_configs && \
+                   ./build_packages --board="arm64-usr" --only_resolve_circular_deps && \
+                   \
+                   ./setup_board --board="amd64-usr" --binhost="${binhost}/amd64-usr" && \
+                   ./setup_board --board="amd64-usr" --regen_configs && \
+                   ./build_packages --board="amd64-usr" --only_resolve_circular_deps ); then
+            touch BUILD_PACKAGES_FAILED
+        fi
+        exit 0
+        ;;
+
+    finish)
+        logdir=''
+        if [[ ${#} -gt 0 ]]; then
+            logdir=${1}; shift
+        fi
+        if [[ -n ${logdir} ]]; then
+            for arch in amd64 arm64; do
+                cp -a "/build/${arch}-usr/var/log/portage" "${logdir}/${arch}-package-logs"
+            done
+        fi
+        if [[ -e BUILD_PACKAGES_FAILED ]]; then
+            rm -f BUILD_PACKAGES_FAILED
+            exit 1
+        fi
+        exit 0
+        ;;
+
+    *)
+        echo "wrong phase ${phase@Q}" >&2
+        exit 1
+        ;;
+
+esac