*: Scavenge for configure logs and upload them to bincache

This searches for portage logs and configure logs after a build, and
uploads them to bincache. This is currently only done for SDK builds,
SDK container builds and package builds. This probably could be
extended to catch logs for sysext builds, but this was annoying to
implement, IIRC.

Signed-off-by: Krzesimir Nowak <knowak@microsoft.com>

bootstrap_sdk_container

@@ -11,6 +11,7 @@ source sdk_lib/sdk_container_common.sh
 
 seed_version=""
 target_version=""
+logdir=''
 
 declare -a cleanup
 
@@ -30,6 +31,7 @@ usage() {
     echo "      -x <cleanup-script> - For each resource generated during build (container etc.)"
     echo "                             add a cleanup line to <script> which, when run, will free"
     echo "                             the resource. Useful for CI."
+    echo "      -l <directory>      - Gather build logs here."
     echo "      -h                  - Print this help."
     echo
 }
@@ -38,6 +40,7 @@ usage() {
 while [ 0 -lt $# ] ; do
     case "$1" in
     -h) usage; exit 0;;
+    -l) logdir=${2}; shift 2;;
     -x) cleanup=("-x" "$2"); shift; shift;;
     *)  if [ -z "$seed_version" ] ; then
             seed_version="$1"
@@ -72,8 +75,11 @@ if $official; then
 fi
 
 # bootstrap_sdk needs FLATCAR_SDK_VERSION set to the seed version
+failed=''
 ./run_sdk_container "${cleanup[@]}" -V "$seed_version" -v "$target_version" \
-          sudo -E ./bootstrap_sdk
+          sudo -E ./bootstrap_sdk || failed=x
 
 # Update versionfile to the actual SDK version
 create_versionfile "${target_version}"
+
+if [[ -n ${failed} ]]; then exit 1; fi

build_sdk_container_image

@@ -34,6 +34,7 @@ arch="amd64"
 official="0"
 tarball=""
 os_version=""
+logdir=""
 
 keep="false"
 cleanup=""
@@ -57,6 +58,7 @@ usage() {
     echo "                     add a cleanup line to <script> which, when run, will free"
     echo "                     the resource. Useful for CI."
     echo "      -h           - Print this help."
+    echo "      -l <dir>     - Put logs in this directory"
     echo
 }
 
@@ -66,6 +68,7 @@ while [ 0 -lt $# ] ; do
     case "$1" in
     -h) usage; exit 0;;
     -k) keep="true";      shift;;
+    -l) logdir="${2}";    shift; shift;;
     -v) os_version="$2";  shift; shift;;
     -x) cleanup="$2";     shift; shift;;
     *)  if [ -z "$tarball" ] ;  then
@@ -102,6 +105,10 @@ else
     official="0"
 fi
 
+if [[ -n ${logdir} ]]; then
+    mkdir -p "${logdir}"
+fi
+
 # --
 
 # import tarball
@@ -170,8 +177,19 @@ else
     if [ -n "$cleanup" ] ; then
         echo "$docker container rm -f '${toolchains_container}'" >> "$cleanup"
     fi
+    failed=''
     ./run_sdk_container -C "${import_image}" -n "${toolchains_container}" \
-            sudo ./build_toolchains --seed_tarball="./${tarball}"
+            sudo ./build_toolchains --seed_tarball="./${tarball}" || failed=x
+
+    if [[ -n ${logdir} ]]; then
+        if sudo test -d __build__/images/catalyst/log/coreos-toolchains; then
+            sudo cp -a __build__/images/catalyst/log/coreos-toolchains "${logdir}/coreos-toolchains"
+        fi
+        if sudo test -d __build__/images/catalyst/tmp/coreos-toolchains; then
+            scavenge_for_configure_logs --use-sudo __build__/images/catalyst/tmp/coreos-toolchains "${logdir}"
+        fi
+    fi
+    if [[ -n ${failed} ]]; then exit 1; fi
 
     # remove sdk tarball from scripts root so it's not part of the SDK container build context
     if [ -f "${tarball_copied}" ] ; then
@@ -216,6 +234,12 @@ else
                  .
 
     $docker stop "${binhost_container}"
+
+    if [[ -n ${logdir} ]]; then
+        $docker run --rm -v "${logdir}:/logdir" "${sdk_build_image}" ./sdk_lib/setup_boards.sh finish /logdir
+    else
+        $docker run --rm "${sdk_build_image}" ./sdk_lib/setup_boards.sh finish
+    fi
 fi
 
 # --

ci-automation/ci_automation_common.sh

@@ -111,7 +111,29 @@ function copy_dir_from_buildcache() {
     rsync --partial -a -e "${sshcmd}" "${BUILDCACHE_USER}@${BUILDCACHE_SERVER}:${remote_path}" \
         "${local_path}"
 }
+# --
 
+# Returns true if the passed directory contains files that match any
+# of the passed glob strings. Remember to pass the glob as strings
+# (means: quote them, so globs are not resolved at the function
+# callsite).
+function dir_contains_globs() (
+    shopt -s nullglob
+
+    local dir=${1}; shift
+    # rest are globs
+
+    # this is in a subshell, so cd is safe - won't mess with working
+    # directory of the caller
+    cd "${dir}" 2>/dev/null || return 1
+    # for globs to be evaluated, we must not use quotes here
+    local -a files=( ${@} )
+
+    if [[ ${#files[@]} -gt 0 ]]; then
+        return 0
+    fi
+    return 1
+)
 # --
 
 function copy_to_buildcache() {

ci-automation/garbage_collect.sh

@@ -152,6 +152,7 @@ function _garbage_collect_impl() {
         rmpat="${rmpat} ${BUILDCACHE_PATH_PREFIX}/containers/${os_docker_vernum}/flatcar-images-*"
         rmpat="${rmpat} ${BUILDCACHE_PATH_PREFIX}/images/*/${os_vernum}/"
         rmpat="${rmpat} ${BUILDCACHE_PATH_PREFIX}/testing/${os_vernum}/"
+        rmpat="${rmpat} ${BUILDCACHE_PATH_PREFIX}/build-logs/${os_vernum}/"
 
         echo "## The following files will be removed ##"
         $sshcmd "${BUILDCACHE_USER}@${BUILDCACHE_SERVER}" \
@@ -211,6 +212,7 @@ function _garbage_collect_impl() {
                 "images/amd64" \
                 "images/arm64" \
                 "testing" \
+                "build-logs" \
                 ; do
         local fullpath="${BUILDCACHE_PATH_PREFIX}/${dir}"
         echo

ci-automation/packages.sh

@@ -80,22 +80,43 @@ function _packages_build_impl() {
     local vernum="${FLATCAR_VERSION}"
     local docker_vernum="$(vernum_to_docker_image_version "${vernum}")"
     local packages_container="flatcar-packages-${arch}-${docker_vernum}"
+    local logs_tarball="packages-${arch}-logs-$(date --utc '+%F-%H%M-%S').tar.xz"
 
     source sdk_lib/sdk_container_common.sh
 
+    logdir=__build__/packages-logs-to-upload
+    mkdir -p "${logdir}"
+
     apply_local_patches
     # Build packages; store packages in container
+    failed=''
     ./run_sdk_container -x ./ci-cleanup.sh  -n "${packages_container}" -v "${vernum}" \
         -C "${sdk_image}" \
-        ./build_packages --board="${arch}-usr"
+        ./build_packages --board="${arch}-usr" || failed=x
 
-    # run_sdk_container updates the version file, use that version from here on
-    source sdk_container/.repo/manifests/version.txt
-    local vernum="${FLATCAR_VERSION}"
-    local docker_vernum="$(vernum_to_docker_image_version "${vernum}")"
-    local packages_image="flatcar-packages-${arch}"
+    # Copy the build logs to a directory accessible for us.
+    ./run_sdk_container -n "${packages_container}" -v "${vernum}" -C "${sdk_image}" \
+        bash -c "source sdk_lib/sdk_container_common.sh; cp -a /build/${arch@Q}-usr/var/log/portage ${logdir@Q}; scavenge_for_configure_logs /build/${arch@Q}-usr/var/tmp/portage ${logdir@Q}" || :
 
-    # generate image + push to build cache
-    docker_commit_to_buildcache "${packages_container}" "${packages_image}" "${docker_vernum}"
+    if [[ -z ${failed} ]]; then
+        # run_sdk_container updates the version file, use that version from here on
+        source sdk_container/.repo/manifests/version.txt
+        local vernum="${FLATCAR_VERSION}"
+        local docker_vernum="$(vernum_to_docker_image_version "${vernum}")"
+        local packages_image="flatcar-packages-${arch}"
+
+        # generate image + push to build cache
+        docker_commit_to_buildcache "${packages_container}" "${packages_image}" "${docker_vernum}"
+    fi
+    if dir_contains_globs "${logdir}" '*'; then
+        (
+            cd "${logdir}"
+            tar -cJf "${logs_tarball}" *
+            create_digests "${SIGNER}" "${logs_tarball}"
+            sign_artifacts "${SIGNER}" "${logs_tarball}"*
+            copy_to_buildcache "build-logs/${FLATCAR_SDK_VERSION}" "${logs_tarball}"*
+        )
+    fi
+    if [[ -n ${failed} ]]; then exit 1; fi
 }
 # --

ci-automation/sdk_bootstrap.sh

@@ -176,13 +176,17 @@ function _sdk_bootstrap_impl() {
     fi
     apply_local_patches
 
-    ./bootstrap_sdk_container -x ./ci-cleanup.sh "${seed_version}" "${vernum}"
+    local failed=''
+    local logdir='__build__/sdk-bootstrap-logs-to-upload/'
+    mkdir -p "${logdir}"
+    ./bootstrap_sdk_container -l "${logdir}" -x ./ci-cleanup.sh "${seed_version}" "${vernum}" || failed=x
 
     # push SDK tarball to buildcache
     # Get Flatcar version number format (separator is '+' instead of '-',
     # equal to $(strip_version_prefix "$version")
     source sdk_container/.repo/manifests/version.txt
     local dest_tarball="flatcar-sdk-${ARCH}-${FLATCAR_SDK_VERSION}.tar.bz2"
+    local logs_tarball="sdk-bootstrap-logs-${ARCH}-$(date --utc '+%F-%H%M-%S').tar.xz"
 
     # change the owner of the files and directories in __build__ back
     # to ourselves, otherwise we could fail to sign the artifacts as
@@ -193,11 +197,33 @@ function _sdk_bootstrap_impl() {
     uid=$(id --user)
     gid=$(id --group)
     sudo chown --recursive "${uid}:${gid}" __build__
+    if [[ -z ${failed} ]]; then
+        (
+            cd "__build__/images/catalyst/builds/flatcar-sdk"
+            create_digests "${SIGNER}" "${dest_tarball}"
+            sign_artifacts "${SIGNER}" "${dest_tarball}"*
+            copy_to_buildcache "sdk/${ARCH}/${FLATCAR_SDK_VERSION}" "${dest_tarball}"*
+        )
+    fi
+
+    # collect logs
+    local catalyst_log='__build__/images/catalyst/log/flatcar-sdk'
+    if dir_contains_globs "${catalyst_log}" 'stage*'; then
+        cp -a "${catalyst_log}/stage"* "${logdir}"
+    fi
     (
-        cd "__build__/images/catalyst/builds/flatcar-sdk"
-        create_digests "${SIGNER}" "${dest_tarball}"
-        sign_artifacts "${SIGNER}" "${dest_tarball}"*
-        copy_to_buildcache "sdk/${ARCH}/${FLATCAR_SDK_VERSION}" "${dest_tarball}"*
+        source sdk_lib/sdk_container_common.sh
+        scavenge_for_configure_logs __build__/images/catalyst/tmp/flatcar-sdk "${logdir}"
     )
+    if dir_contains_globs "${logdir}" '*'; then
+        (
+            cd "${logdir}"
+            tar -cJf "${logs_tarball}" *
+            create_digests "${SIGNER}" "${logs_tarball}"
+            sign_artifacts "${SIGNER}" "${logs_tarball}"*
+            copy_to_buildcache "build-logs/${FLATCAR_SDK_VERSION}" "${logs_tarball}"*
+        )
+    fi
+    if [[ -n ${failed} ]]; then exit 1; fi
 }
 # --

ci-automation/sdk_container.sh

@@ -65,13 +65,35 @@ function _sdk_container_build_impl() {
     copy_from_buildcache "sdk/${ARCH}/${vernum}/${sdk_tarball}" "./__build__"
 
 
+    logdir=__build__/sdk-container-logs-to-upload
+    mkdir -p "${logdir}"
+    local failed=''
+    local logs_tarball="sdk-container-logs-${ARCH}-$(date --utc '+%F-%H%M-%S').tar.xz"
     # This will update the SDK_VERSION in versionfile
-    ./build_sdk_container_image -x ./ci-cleanup.sh ./__build__/"${sdk_tarball}"
+    ./build_sdk_container_image -l "${PWD}/${logdir}" -x ./ci-cleanup.sh ./__build__/"${sdk_tarball}" || failed=x
 
-    # push artifacts to build cache
-    local docker_vernum="$(vernum_to_docker_image_version "${vernum}")"
-    docker_image_to_buildcache "${CONTAINER_REGISTRY}/flatcar-sdk-all" "${docker_vernum}"
-    docker_image_to_buildcache "${CONTAINER_REGISTRY}/flatcar-sdk-amd64" "${docker_vernum}"
-    docker_image_to_buildcache "${CONTAINER_REGISTRY}/flatcar-sdk-arm64" "${docker_vernum}"
+    if [[ -z ${failed} ]]; then
+        # push artifacts to build cache
+        local docker_vernum="$(vernum_to_docker_image_version "${vernum}")"
+        docker_image_to_buildcache "${CONTAINER_REGISTRY}/flatcar-sdk-all" "${docker_vernum}"
+        docker_image_to_buildcache "${CONTAINER_REGISTRY}/flatcar-sdk-amd64" "${docker_vernum}"
+        docker_image_to_buildcache "${CONTAINER_REGISTRY}/flatcar-sdk-arm64" "${docker_vernum}"
+    fi
+    local uid
+    local gid
+    uid=$(id --user)
+    gid=$(id --group)
+    sudo chown --recursive "${uid}:${gid}" "${logdir}"
+    chmod --recursive a+rX,u+w "${logdir}"
+    if dir_contains_globs "${logdir}" '*'; then
+        (
+            cd "${logdir}"
+            tar -cJf "${logs_tarball}" *
+            create_digests "${SIGNER}" "${logs_tarball}"
+            sign_artifacts "${SIGNER}" "${logs_tarball}"*
+            copy_to_buildcache "build-logs/${FLATCAR_SDK_VERSION}" "${logs_tarball}"*
+        )
+    fi
+    if [[ -n ${failed} ]]; then exit 1; fi
 }
 # --

sdk_lib/Dockerfile.sdk-build

@@ -7,13 +7,7 @@ ARG OFFICIAL=0
 # mark build as official where appropriate
 RUN echo "export COREOS_OFFICIAL=$OFFICIAL" > /mnt/host/source/.env
 
-RUN /home/sdk/sdk_entry.sh ./setup_board --board="arm64-usr" --binhost="${BINHOST}/arm64-usr"
-RUN /home/sdk/sdk_entry.sh ./setup_board --board="arm64-usr" --regen_configs
-RUN /home/sdk/sdk_entry.sh ./build_packages --board="arm64-usr" --only_resolve_circular_deps
-
-RUN /home/sdk/sdk_entry.sh ./setup_board --board="amd64-usr" --binhost="${BINHOST}/amd64-usr"
-RUN /home/sdk/sdk_entry.sh ./setup_board --board="amd64-usr" --regen_configs
-RUN /home/sdk/sdk_entry.sh ./build_packages --board="amd64-usr" --only_resolve_circular_deps
+RUN /home/sdk/sdk_entry.sh ./sdk_lib/setup_boards.sh start "${BINHOST}"
 
 RUN rm /mnt/host/source/.env
 RUN rm -rf /home/sdk/toolchain-pkgs

sdk_lib/sdk_container_common.sh

@@ -314,3 +314,40 @@ function gnupg_ssh_gcloud_mount_opts() {
         fi
     fi
 }
+
+function scavenge_for_configure_logs() {
+    local -a sudo_cmd=()
+    while [[ $# -gt 0 ]]; do
+        case ${1} in
+            --use-sudo) sudo_cmd=( sudo ); shift;;
+            --) shift; break;;
+            --*) echo "unknown flag for $0: $1" >&2; exit 1;;
+            *) break;;
+        esac
+    done
+    local dir=${1}; shift
+    local logdir=${1}; shift
+
+    # TODO: Add more interesting files
+    local -a interesting_files=( config.log CMakeConfigureLog.yaml meson-log.txt ) find_flags=()
+    for f in "${interesting_files[@]}"; do
+        if [[ ${#find_flags[@]} -ne 0 ]]; then
+            find_flags+=( '-o' )
+        fi
+        find_flags+=( '-name' "${f}" )
+    done
+    local -a logs
+    local l d
+    mapfile -t logs < <("${sudo_cmd[@]}" find "${dir}" "${find_flags[@]}")
+    for l in "${logs[@]}"; do
+        d=${l#"${dir}"}
+        d=${d#/}
+        if [[ ${d} = */* ]]; then
+            d=${d%/*}
+        else
+            d='.'
+        fi
+        "${sudo_cmd[@]}" mkdir -p "${logdir}/config-logs/${d}"
+        "${sudo_cmd[@]}" cp -a "${l}" "${logdir}/config-logs/${d}"
+    done
+}

sdk_lib/setup_boards.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+# Script used in Dockerfile.sdk-build. The failure is indicated by the
+# BUILD_PACKAGES_FAILED file in the scripts directory in the built
+# docker image. We do it to be able to recover the build logs from
+# failed builds too.
+
+set -euo pipefail
+
+phase=${1}; shift
+case ${phase} in
+
+    start)
+        binhost=${1}; shift
+        if ! ( ./setup_board --board="arm64-usr" --binhost="${binhost}/arm64-usr" && \
+                   ./setup_board --board="arm64-usr" --regen_configs && \
+                   ./build_packages --board="arm64-usr" --only_resolve_circular_deps && \
+                   \
+                   ./setup_board --board="amd64-usr" --binhost="${binhost}/amd64-usr" && \
+                   ./setup_board --board="amd64-usr" --regen_configs && \
+                   ./build_packages --board="amd64-usr" --only_resolve_circular_deps ); then
+            touch BUILD_PACKAGES_FAILED
+        fi
+        exit 0
+        ;;
+
+    finish)
+        logdir=''
+        if [[ ${#} -gt 0 ]]; then
+            logdir=${1}; shift
+        fi
+        if [[ -n ${logdir} ]]; then
+            for arch in amd64 arm64; do
+                cp -a "/build/${arch}-usr/var/log/portage" "${logdir}/${arch}-package-logs"
+            done
+        fi
+        if [[ -e BUILD_PACKAGES_FAILED ]]; then
+            rm -f BUILD_PACKAGES_FAILED
+            exit 1
+        fi
+        exit 0
+        ;;
+
+    *)
+        echo "wrong phase ${phase@Q}" >&2
+        exit 1
+        ;;
+
+esac