Merge pull request #3984 from flatcar/krnowak/scavenge-logs

Drop dmesg, df and mount output, add build log scavenging and uploading

bootstrap_sdk_container

@@ -11,6 +11,7 @@ source sdk_lib/sdk_container_common.sh
 
 seed_version=""
 target_version=""
+logdir=''
 
 declare -a cleanup
 
@@ -30,6 +31,7 @@ usage() {
     echo "      -x <cleanup-script> - For each resource generated during build (container etc.)"
     echo "                             add a cleanup line to <script> which, when run, will free"
     echo "                             the resource. Useful for CI."
+    echo "      -l <directory>      - Gather build logs here."
     echo "      -h                  - Print this help."
     echo
 }
@@ -38,6 +40,7 @@ usage() {
 while [ 0 -lt $# ] ; do
     case "$1" in
     -h) usage; exit 0;;
+    -l) logdir=${2}; shift 2;;
     -x) cleanup=("-x" "$2"); shift; shift;;
     *)  if [ -z "$seed_version" ] ; then
             seed_version="$1"
@@ -72,8 +75,11 @@ if $official; then
 fi
 
 # bootstrap_sdk needs FLATCAR_SDK_VERSION set to the seed version
+failed=''
 ./run_sdk_container "${cleanup[@]}" -V "$seed_version" -v "$target_version" \
-          sudo -E ./bootstrap_sdk
+          sudo -E ./bootstrap_sdk || failed=x
 
 # Update versionfile to the actual SDK version
 create_versionfile "${target_version}"
+
+if [[ -n ${failed} ]]; then exit 1; fi

build_library/catalyst.sh

@@ -242,13 +242,13 @@ build_stage() {
     fi
 
     info "Starting $stage"
+    # Clean up possible leftovers from possible previous runs
+    rm -rf "$TEMPDIR/$stage-${ARCH}-${FLAGS_version}"
     catalyst \
         "${DEBUG[@]}" \
         --verbose \
         --config "$TEMPDIR/catalyst.conf" \
         --file "$TEMPDIR/${stage}.spec"
-    # Catalyst does not clean up after itself...
-    rm -rf "$TEMPDIR/$stage-${ARCH}-${FLAGS_version}"
     ln -sf "$stage-${ARCH}-${FLAGS_version}.tar.bz2" \
         "$BUILDS/$stage-${ARCH}-latest.tar.bz2"
     info "Finished building $target_tarball"

build_sdk_container_image

@@ -34,6 +34,7 @@ arch="amd64"
 official="0"
 tarball=""
 os_version=""
+logdir=""
 
 keep="false"
 cleanup=""
@@ -57,6 +58,7 @@ usage() {
     echo "                     add a cleanup line to <script> which, when run, will free"
     echo "                     the resource. Useful for CI."
     echo "      -h           - Print this help."
+    echo "      -l <dir>     - Put logs in this directory"
     echo
 }
 
@@ -66,6 +68,7 @@ while [ 0 -lt $# ] ; do
     case "$1" in
     -h) usage; exit 0;;
     -k) keep="true";      shift;;
+    -l) logdir="${2}";    shift; shift;;
     -v) os_version="$2";  shift; shift;;
     -x) cleanup="$2";     shift; shift;;
     *)  if [ -z "$tarball" ] ;  then
@@ -102,6 +105,10 @@ else
     official="0"
 fi
 
+if [[ -n ${logdir} ]]; then
+    mkdir -p "${logdir}"
+fi
+
 # --
 
 # import tarball
@@ -170,8 +177,19 @@ else
     if [ -n "$cleanup" ] ; then
         echo "$docker container rm -f '${toolchains_container}'" >> "$cleanup"
     fi
+    failed=''
     ./run_sdk_container -C "${import_image}" -n "${toolchains_container}" \
-            sudo ./build_toolchains --seed_tarball="./${tarball}"
+            sudo ./build_toolchains --seed_tarball="./${tarball}" || failed=x
+
+    if [[ -n ${logdir} ]]; then
+        if sudo test -d __build__/images/catalyst/log/coreos-toolchains; then
+            sudo cp -a __build__/images/catalyst/log/coreos-toolchains "${logdir}/coreos-toolchains"
+        fi
+        if sudo test -d __build__/images/catalyst/tmp/coreos-toolchains; then
+            scavenge_for_configure_logs --use-sudo __build__/images/catalyst/tmp/coreos-toolchains "${logdir}"
+        fi
+    fi
+    if [[ -n ${failed} ]]; then exit 1; fi
 
     # remove sdk tarball from scripts root so it's not part of the SDK container build context
     if [ -f "${tarball_copied}" ] ; then
@@ -216,6 +234,12 @@ else
                  .
 
     $docker stop "${binhost_container}"
+
+    if [[ -n ${logdir} ]]; then
+        $docker run --rm -v "${logdir}:/logdir" "${sdk_build_image}" ./sdk_lib/setup_boards.sh finish /logdir
+    else
+        $docker run --rm "${sdk_build_image}" ./sdk_lib/setup_boards.sh finish
+    fi
 fi
 
 # --

ci-automation/ci_automation_common.sh

@@ -111,7 +111,29 @@ function copy_dir_from_buildcache() {
     rsync --partial -a -e "${sshcmd}" "${BUILDCACHE_USER}@${BUILDCACHE_SERVER}:${remote_path}" \
         "${local_path}"
 }
+# --
 
+# Returns true if the passed directory contains files that match any
+# of the passed glob strings. Remember to pass the glob as strings
+# (means: quote them, so globs are not resolved at the function
+# callsite).
+function dir_contains_globs() (
+    shopt -s nullglob
+
+    local dir=${1}; shift
+    # rest are globs
+
+    # this is in a subshell, so cd is safe - won't mess with working
+    # directory of the caller
+    cd "${dir}" 2>/dev/null || return 1
+    # for globs to be evaluated, we must not use quotes here
+    local -a files=( ${@} )
+
+    if [[ ${#files[@]} -gt 0 ]]; then
+        return 0
+    fi
+    return 1
+)
 # --
 
 function copy_to_buildcache() {

ci-automation/garbage_collect.sh

@@ -152,6 +152,7 @@ function _garbage_collect_impl() {
         rmpat="${rmpat} ${BUILDCACHE_PATH_PREFIX}/containers/${os_docker_vernum}/flatcar-images-*"
         rmpat="${rmpat} ${BUILDCACHE_PATH_PREFIX}/images/*/${os_vernum}/"
         rmpat="${rmpat} ${BUILDCACHE_PATH_PREFIX}/testing/${os_vernum}/"
+        rmpat="${rmpat} ${BUILDCACHE_PATH_PREFIX}/build-logs/${os_vernum}/"
 
         echo "## The following files will be removed ##"
         $sshcmd "${BUILDCACHE_USER}@${BUILDCACHE_SERVER}" \
@@ -211,6 +212,7 @@ function _garbage_collect_impl() {
                 "images/amd64" \
                 "images/arm64" \
                 "testing" \
+                "build-logs" \
                 ; do
         local fullpath="${BUILDCACHE_PATH_PREFIX}/${dir}"
         echo

ci-automation/packages.sh

@@ -80,22 +80,43 @@ function _packages_build_impl() {
     local vernum="${FLATCAR_VERSION}"
     local docker_vernum="$(vernum_to_docker_image_version "${vernum}")"
     local packages_container="flatcar-packages-${arch}-${docker_vernum}"
+    local logs_tarball="packages-${arch}-logs-$(date --utc '+%F-%H%M-%S').tar.xz"
 
     source sdk_lib/sdk_container_common.sh
 
+    logdir=__build__/packages-logs-to-upload
+    mkdir -p "${logdir}"
+
     apply_local_patches
     # Build packages; store packages in container
+    failed=''
     ./run_sdk_container -x ./ci-cleanup.sh  -n "${packages_container}" -v "${vernum}" \
         -C "${sdk_image}" \
-        ./build_packages --board="${arch}-usr"
+        ./build_packages --board="${arch}-usr" || failed=x
 
-    # run_sdk_container updates the version file, use that version from here on
-    source sdk_container/.repo/manifests/version.txt
-    local vernum="${FLATCAR_VERSION}"
-    local docker_vernum="$(vernum_to_docker_image_version "${vernum}")"
-    local packages_image="flatcar-packages-${arch}"
+    # Copy the build logs to a directory accessible for us.
+    ./run_sdk_container -n "${packages_container}" -v "${vernum}" -C "${sdk_image}" \
+        bash -c "source sdk_lib/sdk_container_common.sh; cp -a /build/${arch@Q}-usr/var/log/portage ${logdir@Q}; scavenge_for_configure_logs /build/${arch@Q}-usr/var/tmp/portage ${logdir@Q}" || :
 
-    # generate image + push to build cache
-    docker_commit_to_buildcache "${packages_container}" "${packages_image}" "${docker_vernum}"
+    if [[ -z ${failed} ]]; then
+        # run_sdk_container updates the version file, use that version from here on
+        source sdk_container/.repo/manifests/version.txt
+        local vernum="${FLATCAR_VERSION}"
+        local docker_vernum="$(vernum_to_docker_image_version "${vernum}")"
+        local packages_image="flatcar-packages-${arch}"
+
+        # generate image + push to build cache
+        docker_commit_to_buildcache "${packages_container}" "${packages_image}" "${docker_vernum}"
+    fi
+    if dir_contains_globs "${logdir}" '*'; then
+        (
+            cd "${logdir}"
+            tar -cJf "${logs_tarball}" *
+            create_digests "${SIGNER}" "${logs_tarball}"
+            sign_artifacts "${SIGNER}" "${logs_tarball}"*
+            copy_to_buildcache "build-logs/${FLATCAR_SDK_VERSION}" "${logs_tarball}"*
+        )
+    fi
+    if [[ -n ${failed} ]]; then exit 1; fi
 }
 # --

ci-automation/sdk_bootstrap.sh

@@ -176,13 +176,17 @@ function _sdk_bootstrap_impl() {
     fi
     apply_local_patches
 
-    ./bootstrap_sdk_container -x ./ci-cleanup.sh "${seed_version}" "${vernum}"
+    local failed=''
+    local logdir='__build__/sdk-bootstrap-logs-to-upload/'
+    mkdir -p "${logdir}"
+    ./bootstrap_sdk_container -l "${logdir}" -x ./ci-cleanup.sh "${seed_version}" "${vernum}" || failed=x
 
     # push SDK tarball to buildcache
     # Get Flatcar version number format (separator is '+' instead of '-',
     # equal to $(strip_version_prefix "$version")
     source sdk_container/.repo/manifests/version.txt
     local dest_tarball="flatcar-sdk-${ARCH}-${FLATCAR_SDK_VERSION}.tar.bz2"
+    local logs_tarball="sdk-bootstrap-logs-${ARCH}-$(date --utc '+%F-%H%M-%S').tar.xz"
 
     # change the owner of the files and directories in __build__ back
     # to ourselves, otherwise we could fail to sign the artifacts as
@@ -193,11 +197,33 @@ function _sdk_bootstrap_impl() {
     uid=$(id --user)
     gid=$(id --group)
     sudo chown --recursive "${uid}:${gid}" __build__
+    if [[ -z ${failed} ]]; then
+        (
+            cd "__build__/images/catalyst/builds/flatcar-sdk"
+            create_digests "${SIGNER}" "${dest_tarball}"
+            sign_artifacts "${SIGNER}" "${dest_tarball}"*
+            copy_to_buildcache "sdk/${ARCH}/${FLATCAR_SDK_VERSION}" "${dest_tarball}"*
+        )
+    fi
+
+    # collect logs
+    local catalyst_log='__build__/images/catalyst/log/flatcar-sdk'
+    if dir_contains_globs "${catalyst_log}" 'stage*'; then
+        cp -a "${catalyst_log}/stage"* "${logdir}"
+    fi
     (
-        cd "__build__/images/catalyst/builds/flatcar-sdk"
-        create_digests "${SIGNER}" "${dest_tarball}"
-        sign_artifacts "${SIGNER}" "${dest_tarball}"*
-        copy_to_buildcache "sdk/${ARCH}/${FLATCAR_SDK_VERSION}" "${dest_tarball}"*
+        source sdk_lib/sdk_container_common.sh
+        scavenge_for_configure_logs __build__/images/catalyst/tmp/flatcar-sdk "${logdir}"
     )
+    if dir_contains_globs "${logdir}" '*'; then
+        (
+            cd "${logdir}"
+            tar -cJf "${logs_tarball}" *
+            create_digests "${SIGNER}" "${logs_tarball}"
+            sign_artifacts "${SIGNER}" "${logs_tarball}"*
+            copy_to_buildcache "build-logs/${FLATCAR_SDK_VERSION}" "${logs_tarball}"*
+        )
+    fi
+    if [[ -n ${failed} ]]; then exit 1; fi
 }
 # --

ci-automation/sdk_container.sh

@@ -65,13 +65,35 @@ function _sdk_container_build_impl() {
     copy_from_buildcache "sdk/${ARCH}/${vernum}/${sdk_tarball}" "./__build__"
 
 
+    logdir=__build__/sdk-container-logs-to-upload
+    mkdir -p "${logdir}"
+    local failed=''
+    local logs_tarball="sdk-container-logs-${ARCH}-$(date --utc '+%F-%H%M-%S').tar.xz"
     # This will update the SDK_VERSION in versionfile
-    ./build_sdk_container_image -x ./ci-cleanup.sh ./__build__/"${sdk_tarball}"
+    ./build_sdk_container_image -l "${PWD}/${logdir}" -x ./ci-cleanup.sh ./__build__/"${sdk_tarball}" || failed=x
 
-    # push artifacts to build cache
-    local docker_vernum="$(vernum_to_docker_image_version "${vernum}")"
-    docker_image_to_buildcache "${CONTAINER_REGISTRY}/flatcar-sdk-all" "${docker_vernum}"
-    docker_image_to_buildcache "${CONTAINER_REGISTRY}/flatcar-sdk-amd64" "${docker_vernum}"
-    docker_image_to_buildcache "${CONTAINER_REGISTRY}/flatcar-sdk-arm64" "${docker_vernum}"
+    if [[ -z ${failed} ]]; then
+        # push artifacts to build cache
+        local docker_vernum="$(vernum_to_docker_image_version "${vernum}")"
+        docker_image_to_buildcache "${CONTAINER_REGISTRY}/flatcar-sdk-all" "${docker_vernum}"
+        docker_image_to_buildcache "${CONTAINER_REGISTRY}/flatcar-sdk-amd64" "${docker_vernum}"
+        docker_image_to_buildcache "${CONTAINER_REGISTRY}/flatcar-sdk-arm64" "${docker_vernum}"
+    fi
+    local uid
+    local gid
+    uid=$(id --user)
+    gid=$(id --group)
+    sudo chown --recursive "${uid}:${gid}" "${logdir}"
+    chmod --recursive a+rX,u+w "${logdir}"
+    if dir_contains_globs "${logdir}" '*'; then
+        (
+            cd "${logdir}"
+            tar -cJf "${logs_tarball}" *
+            create_digests "${SIGNER}" "${logs_tarball}"
+            sign_artifacts "${SIGNER}" "${logs_tarball}"*
+            copy_to_buildcache "build-logs/${FLATCAR_SDK_VERSION}" "${logs_tarball}"*
+        )
+    fi
+    if [[ -n ${failed} ]]; then exit 1; fi
 }
 # --

common.sh

@@ -164,40 +164,9 @@ die_notrace() {
   for line in "$@"; do
     error "${DIE_PREFIX}${line}"
   done
-  if [[ ! -e "${SCRIPTS_DIR}/NO_DEBUG_OUTPUT_DELETE_ME" ]]; then
-      error "${DIE_PREFIX}!!!!!!!!!!!!!!!!!!!!!!!!!"
-      error "${DIE_PREFIX}!! BEGIN DEBUG OUTPUT: !!"
-      error "${DIE_PREFIX}!!!!!!!!!!!!!!!!!!!!!!!!!"
-      error
-      error "${DIE_PREFIX}== MOUNT =="
-      error "${DIE_PREFIX}==========="
-      error_command_output "${DIE_PREFIX}" mount
-      error
-      error "${DIE_PREFIX}== DF =="
-      error "${DIE_PREFIX}========"
-      error_command_output "${DIE_PREFIX}" df -h
-      error
-      error "${DIE_PREFIX}== DMESG =="
-      error "${DIE_PREFIX}==========="
-      error_command_output "${DIE_PREFIX}" sudo dmesg
-      error
-      error "${DIE_PREFIX}!!!!!!!!!!!!!!!!!!!!!!!"
-      error "${DIE_PREFIX}!! END DEBUG OUTPUT: !!"
-      error "${DIE_PREFIX}!!!!!!!!!!!!!!!!!!!!!!!"
-      touch "${SCRIPTS_DIR}/NO_DEBUG_OUTPUT_DELETE_ME"
-  fi
   exit 1
 }
 
-error_command_output() {
-    local prefix=${1}; shift
-    # rest are a command to execute
-    local REPLY
-    while read -r; do
-        error "${prefix}${REPLY}"
-    done < <("${@}" 2>&1)
-}
-
 # Simple version comparison routine
 # Note: not a true semver comparison and build revisions are ignored
 cmp_ver() {
@@ -325,8 +294,6 @@ BUILD_LIBRARY_DIR="${SCRIPTS_DIR}/build_library"
 REPO_CACHE_DIR="${REPO_ROOT}/.cache"
 REPO_MANIFESTS_DIR="${REPO_ROOT}/.repo/manifests"
 
-rm -f "${SCRIPTS_DIR}/NO_DEBUG_OUTPUT_DELETE_ME" || :
-
 # Source FLATCAR_VERSION_ID from manifest.
 if [[ -f "${REPO_MANIFESTS_DIR}/version.txt" ]]; then
   # The build id may be provided externally by the build system.

sdk_lib/Dockerfile.sdk-build

@@ -7,13 +7,7 @@ ARG OFFICIAL=0
 # mark build as official where appropriate
 RUN echo "export COREOS_OFFICIAL=$OFFICIAL" > /mnt/host/source/.env
 
-RUN /home/sdk/sdk_entry.sh ./setup_board --board="arm64-usr" --binhost="${BINHOST}/arm64-usr"
-RUN /home/sdk/sdk_entry.sh ./setup_board --board="arm64-usr" --regen_configs
-RUN /home/sdk/sdk_entry.sh ./build_packages --board="arm64-usr" --only_resolve_circular_deps
-
-RUN /home/sdk/sdk_entry.sh ./setup_board --board="amd64-usr" --binhost="${BINHOST}/amd64-usr"
-RUN /home/sdk/sdk_entry.sh ./setup_board --board="amd64-usr" --regen_configs
-RUN /home/sdk/sdk_entry.sh ./build_packages --board="amd64-usr" --only_resolve_circular_deps
+RUN /home/sdk/sdk_entry.sh ./sdk_lib/setup_boards.sh start "${BINHOST}"
 
 RUN rm /mnt/host/source/.env
 RUN rm -rf /home/sdk/toolchain-pkgs

sdk_lib/sdk_container_common.sh

@@ -314,3 +314,40 @@ function gnupg_ssh_gcloud_mount_opts() {
         fi
     fi
 }
+
+function scavenge_for_configure_logs() {
+    local -a sudo_cmd=()
+    while [[ $# -gt 0 ]]; do
+        case ${1} in
+            --use-sudo) sudo_cmd=( sudo ); shift;;
+            --) shift; break;;
+            --*) echo "unknown flag for $0: $1" >&2; exit 1;;
+            *) break;;
+        esac
+    done
+    local dir=${1}; shift
+    local logdir=${1}; shift
+
+    # TODO: Add more interesting files
+    local -a interesting_files=( config.log CMakeConfigureLog.yaml meson-log.txt ) find_flags=()
+    for f in "${interesting_files[@]}"; do
+        if [[ ${#find_flags[@]} -ne 0 ]]; then
+            find_flags+=( '-o' )
+        fi
+        find_flags+=( '-name' "${f}" )
+    done
+    local -a logs
+    local l d
+    mapfile -t logs < <("${sudo_cmd[@]}" find "${dir}" "${find_flags[@]}")
+    for l in "${logs[@]}"; do
+        d=${l#"${dir}"}
+        d=${d#/}
+        if [[ ${d} = */* ]]; then
+            d=${d%/*}
+        else
+            d='.'
+        fi
+        "${sudo_cmd[@]}" mkdir -p "${logdir}/config-logs/${d}"
+        "${sudo_cmd[@]}" cp -a "${l}" "${logdir}/config-logs/${d}"
+    done
+}

sdk_lib/setup_boards.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+# Script used in Dockerfile.sdk-build. The failure is indicated by the
+# BUILD_PACKAGES_FAILED file in the scripts directory in the built
+# docker image. We do it to be able to recover the build logs from
+# failed builds too.
+
+set -euo pipefail
+
+phase=${1}; shift
+case ${phase} in
+
+    start)
+        binhost=${1}; shift
+        if ! ( ./setup_board --board="arm64-usr" --binhost="${binhost}/arm64-usr" && \
+                   ./setup_board --board="arm64-usr" --regen_configs && \
+                   ./build_packages --board="arm64-usr" --only_resolve_circular_deps && \
+                   \
+                   ./setup_board --board="amd64-usr" --binhost="${binhost}/amd64-usr" && \
+                   ./setup_board --board="amd64-usr" --regen_configs && \
+                   ./build_packages --board="amd64-usr" --only_resolve_circular_deps ); then
+            touch BUILD_PACKAGES_FAILED
+        fi
+        exit 0
+        ;;
+
+    finish)
+        logdir=''
+        if [[ ${#} -gt 0 ]]; then
+            logdir=${1}; shift
+        fi
+        if [[ -n ${logdir} ]]; then
+            for arch in amd64 arm64; do
+                cp -a "/build/${arch}-usr/var/log/portage" "${logdir}/${arch}-package-logs"
+            done
+        fi
+        if [[ -e BUILD_PACKAGES_FAILED ]]; then
+            rm -f BUILD_PACKAGES_FAILED
+            exit 1
+        fi
+        exit 0
+        ;;
+
+    *)
+        echo "wrong phase ${phase@Q}" >&2
+        exit 1
+        ;;
+
+esac