From 28b26c87c19854021426927a882bb72e695874d1 Mon Sep 17 00:00:00 2001 From: Thilo Fromm Date: Wed, 15 Mar 2023 10:43:09 +0100 Subject: [PATCH 01/10] ci.yaml: re-use build container, finer grained artifact upload This change removes "docker commit" at the end of each step and instead makes build steps re-use the build container, saving some build time. It also makes artifact upload more granular, so build logs, images, and dev container can be downloaded individually. Lastly, it exports torcx tarball and binary packages as a separate artifact each, for successive re-use in the kola tests. --- .github/workflows/ci.yaml | 202 +++++++++++++++++++++++--------------- 1 file changed, 121 insertions(+), 81 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e008c7cbb7..58d01dd4db 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -3,11 +3,6 @@ on: pull_request: workflow_dispatch: inputs: - bincache_server: - description: | - Bincache server. - default: "bincache.flatcar-linux.net" - required: true image_formats: description: | Space-separated vendor formats to build. @@ -79,7 +74,6 @@ jobs: - name: Set environment shell: bash run: | - BUILDCACHE_SERVER="bincache.flatcar-linux.net" arch="${{ matrix.arch }}" COREOS_REMOTE="" COREOS_REF="" @@ -87,14 +81,12 @@ jobs: PORTAGE_REF="" IMAGE_FORMATS="qemu_uefi" - [ -z "${{ github.event.inputs.bincache_server }}" ] || BUILDCACHE_SERVER="${{ github.event.inputs.bincache_server }}" [ -z "${{ github.event.inputs.coreos_remote }}" ] || COREOS_REMOTE="${{ github.event.inputs.coreos_remote }}" [ -z "${{ github.event.inputs.coreos_ref }}" ] || COREOS_REF="${{ github.event.inputs.coreos_ref }}" [ -z "${{ github.event.inputs.portage_remote }}" ] || PORTAGE_REMOTE="${{ github.event.inputs.portage_remote }}" [ -z "${{ github.event.inputs.portage_ref }}" ] || PORTAGE_REF="${{ github.event.inputs.portage_ref }}" [ -z "${{ github.event.inputs.image_formats }}" ] || IMAGE_FORMATS="${{ github.event.inputs.image_formats }}" - echo "BUILDCACHE_SERVER=${BUILDCACHE_SERVER}" >> $GITHUB_ENV echo "arch=${arch}" >> $GITHUB_ENV echo "COREOS_REMOTE=${COREOS_REMOTE}" >> $GITHUB_ENV echo "COREOS_REF=${COREOS_REF}" >> $GITHUB_ENV @@ -102,6 +94,15 @@ jobs: echo "PORTAGE_REF=${PORTAGE_REF}" >> $GITHUB_ENV echo "IMAGE_FORMATS=${IMAGE_FORMATS}" >> $GITHUB_ENV + # Artifact root for images and torcx tarball as seen from within the container + echo "CI_CONTAINER_ARTIFACT_ROOT=/home/sdk/trunk/src/scripts/artifacts" >> $GITHUB_ENV + echo "CI_CONTAINER_TORCX_ROOT=/home/sdk/trunk/src/scripts/artifacts/torcx" >> $GITHUB_ENV + mkdir -p artifacts/torcx + + # Placeholder URL for run-kola-tests.yaml, "Extract artifacts" step which will replace + # this with its IP address. + echo "TORCX_TESTS_PACKAGE_URL=http://localhost:12345" >> $GITHUB_ENV + - name: Checkout submodules shell: bash run: | @@ -129,7 +130,7 @@ jobs: shell: bash run: | exec 2>&1 - set +x + set -x set -euo pipefail source ci-automation/ci_automation_common.sh @@ -144,92 +145,70 @@ jobs: docker_image_from_registry_or_buildcache "${sdk_name}" "${docker_sdk_vernum}" sdk_image="$(docker_image_fullname "${sdk_name}" "${docker_sdk_vernum}")" - vernum="${version#*-}" # remove main-,alpha-,beta-,stable-,lts- version tag - docker_vernum="$(vernum_to_docker_image_version "${vernum}")" - packages_container="flatcar-packages-${arch}-${docker_vernum}" + container_name="flatcar-ci-build" # Create version file ( source sdk_lib/sdk_container_common.sh create_versionfile "$sdk_version" "$version" ) - ./run_sdk_container -n "${packages_container}" -v "${version}" \ + + # Run the packages build. This will create the ci build container + # which will be re-used by subsequent build steps. + ./run_sdk_container -n "${container_name}" -v "${version}" \ -C "${sdk_image}" \ ./build_packages --board="${arch}-usr" \ - --torcx_output_root="${CONTAINER_TORCX_ROOT}" + --torcx_output_root="${CI_CONTAINER_TORCX_ROOT}" \ + --torcx_extra_pkg_url="${TORCX_TESTS_PACKAGE_URL}" - # copy torcx manifest and docker tarball for publishing - torcx_tmp="__build__/torcx_tmp" - rm -rf "${torcx_tmp}" - mkdir "${torcx_tmp}" - ./run_sdk_container -n "${packages_container}" -v "${version}" \ - -C "${sdk_image}" \ - cp -r "${CONTAINER_TORCX_ROOT}/" \ - "${torcx_tmp}" + # Copy logs + ./run_sdk_container -n "${container_name}" \ + tar -cJf ebuild_logs.tar.xz /build/${arch}-usr/var/log/portage \ + /build/${arch}-usr/var/tmp/portage - source sdk_container/.repo/manifests/version.txt - vernum="${FLATCAR_VERSION}" - docker_vernum="$(vernum_to_docker_image_version "${vernum}")" - packages_image="flatcar-packages-${arch}" + # Create binpkgs tarball for archiving as artifact later + ./run_sdk_container -n "${container_name}" \ + tar -C "/build/${arch}-usr/var/lib/portage/pkgs/" \ + -cvf binpkgs.tar . - echo "vernum=${vernum}" >> $GITHUB_ENV - echo "docker_vernum=${docker_vernum}" >> $GITHUB_ENV - echo "packages_image=${packages_image}" >> $GITHUB_ENV - echo "arch=${arch}" >> $GITHUB_ENV - echo "sdk_image=${sdk_image}" >> $GITHUB_ENV - echo "packages_container=${packages_container}" >> $GITHUB_ENV - docker commit "${packages_container}" "${packages_image}:${docker_vernum}" - docker rm -f "${packages_container}" + echo "container_name=${container_name}" >> "$GITHUB_ENV" + + - name: Upload build logs + uses: actions/upload-artifact@v3 + with: + retention-days: 7 + name: ${{ matrix.arch }}-build-logs + path: | + scripts/ebuild_logs.tar.xz - name: Build image shell: bash run: | set -euo pipefail - set +x + set -x echo 'channel="developer"' >> $GITHUB_ENV channel="developer" source ci-automation/ci_automation_common.sh - packages="flatcar-packages-${arch}" - packages_image="${packages}:${docker_vernum}" - image="flatcar-images-${arch}" - image_container="${image}-${docker_vernum}" official_arg="--noofficial" - echo "image=flatcar-images-${arch}" >> $GITHUB_ENV - echo "image_image=${image}:${docker_vernum}" >> $GITHUB_ENV - - ./run_sdk_container -x ./ci-cleanup.sh -n "${image_container}" -C "${packages_image}" \ - -v "${vernum}" \ - mkdir -p "${CONTAINER_IMAGE_ROOT}" - ./run_sdk_container -n "${image_container}" -C "${packages_image}" \ - -v "${vernum}" \ + ./run_sdk_container -n "${container_name}" \ ./set_official --board="${arch}-usr" "${official_arg}" - ./run_sdk_container -n "${image_container}" -C "${packages_image}" \ - -v "${vernum}" \ + ./run_sdk_container -n "${container_name}" \ ./build_image --board="${arch}-usr" --group="${channel}" \ - --output_root="${CONTAINER_IMAGE_ROOT}" \ - --torcx_root="${CONTAINER_TORCX_ROOT}" prodtar container + --output_root="${CI_CONTAINER_ARTIFACT_ROOT}" \ + --torcx_root="${CI_CONTAINER_TORCX_ROOT}" prodtar container - # Copy logs - ./run_sdk_container -n "${image_container}" -C "${packages_image}" -v "${vernum}" \ - tar -cJf ebuild_logs.tar.xz /build/${arch}-usr/var/log/portage \ - /build/${arch}-usr/var/tmp/portage - - docker commit "${image_container}" "${image}:${docker_vernum}" - docker rm -f "${image_container}" - - - name: Build VM image + - name: Build VM image(s) shell: bash run: | set -euo pipefail - set +x + set -x source ci-automation/ci_automation_common.sh - vms_container="flatcar-vms-${docker_vernum}" images_out="images" has_packet=0 @@ -254,35 +233,96 @@ jobs: for format in ${formats}; do echo " ################### VENDOR '${format}' ################### " - ./run_sdk_container -n "${vms_container}" -C "${image_image}" \ - -v "${vernum}" \ + ./run_sdk_container -n "${container_name}" \ ./image_to_vm.sh --format "${format}" --board="${arch}-usr" \ - --from "${CONTAINER_IMAGE_ROOT}/${arch}-usr/latest" \ + --from "${CI_CONTAINER_ARTIFACT_ROOT}/${arch}-usr/latest" \ --image_compression_formats=bz2 done - # copy resulting images - ./run_sdk_container -n "${vms_container}" \ - -v "${vernum}" \ - mv "${CONTAINER_IMAGE_ROOT}/${arch}-usr" "./${images_out}" + # upload-artifacts cannot handle artifact uploads from sym-linked directories (no, really) + # so we move things around. + mkdir -p artifacts/images + ( + cd artifacts/${arch}-usr/latest/ + mv * ../../images/ + ) - # remove symlinks before upload - find "./${images_out}" -type l -delete + # create a tarball for torcx package + JSON file because upload-artifacts cannot handle filenames containing colons + # (such as "docker:20.10.torcx.tgz") + mv artifacts/torcx/${arch}-usr/latest/torcx_manifest.json artifacts/torcx/pkgs/ + tar -C artifacts/torcx/pkgs/ -cvf torcx.tar . - docker rm -f "${vms_container}" - - name: Upload artifacts + - name: Upload binpkgs uses: actions/upload-artifact@v3 with: - name: images-${{ matrix.arch }} + retention-days: 7 + name: ${{ matrix.arch }}-binpkgs path: | - scripts/images/**/*.img.bz2 - scripts/images/**/*.bin.bz2 - scripts/images/**/flatcar_production_*_efi_*.fd - scripts/images/**/*.txt - scripts/images/**/flatcar_production_*.sh - scripts/images/**/flatcar_test_update.gz - scripts/ebuild_logs.tar.xz + scripts/binpkgs.tar + + - name: Upload update image (used with kola tests later) + uses: actions/upload-artifact@v3 + with: + retention-days: 7 + name: ${{ matrix.arch }}-test-update + path: | + scripts/artifacts/images/flatcar_test_update.gz + + - name: Upload generic image + uses: actions/upload-artifact@v3 + with: + retention-days: 7 + name: ${{ matrix.arch }}-generic-image + path: | + scripts/artifacts/images/flatcar_production_image.bin.bz2 + scripts/artifacts/images/flatcar_production_image.grub + scripts/artifacts/images/flatcar_production_image.shim + scripts/artifacts/images/flatcar_production_image.vmlinuz + scripts/artifacts/images/flatcar_production_image*.txt + scripts/artifacts/images/flatcar_production_image*.json + scripts/artifacts/images/flatcar_production_image_pcr_policy.zip + scripts/artifacts/images/flatcar_production_*_efi_*.fd + + - name: Upload developer container + uses: actions/upload-artifact@v3 + with: + retention-days: 7 + name: ${{ matrix.arch }}-devcontainer + path: | + scripts/artifacts/images/flatcar_developer_container* + + - name: Upload torcx tarball + uses: actions/upload-artifact@v3 + with: + retention-days: 7 + name: ${{ matrix.arch }}-torcx + path: | + scripts/torcx.tar + + # Clean up what we uploaded already so the "vendor images" wildcard + # works when uploading artifacts in the next step. + - name: Remove update, generic and devcontainer images + shell: bash + run: | + set -euo pipefail + set -x + rm -f artifacts/images/flatcar_test_update.gz \ + artifacts/images/flatcar_production_image* \ + artifacts/images/flatcar_developer_container* \ + artifacts/images/flatcar_production_update* + + - name: Upload vendor images + uses: actions/upload-artifact@v3 + with: + retention-days: 7 + name: ${{ matrix.arch }}-vm-images + path: | + scripts/artifacts/images/*.img.bz2 + scripts/artifacts/images/*.bin.bz2 + scripts/artifacts/images/flatcar_production_*_efi_*.fd + scripts/artifacts/images/*.txt + scripts/artifacts/images/flatcar_production_*.sh test: needs: packages From 27d540692fd320745952eac4a5aa0d25d1c5241f Mon Sep 17 00:00:00 2001 From: Thilo Fromm Date: Thu, 23 Mar 2023 17:39:52 +0100 Subject: [PATCH 02/10] run-kola-tests.yaml: use new artifacts, local web server This change updates the github actions kola test runner workflow to use the new, separated artifacts produced by ci.yaml. Further, it adds a fix for the devcontainer tests. Devcontainer and bin packages used in the devcontainer tests are now served from a local temporary web server. The change also adds the qemu_update test and provides the respective update payload. Lastly, the tests now use a local torcx_manifest.json produced by ci.yaml, which points to a torcx tarball also served by the local temporary web server. --- .github/workflows/run-kola-tests.yaml | 152 ++++++++++++++++++-- ci-automation/ci-config.env | 11 ++ ci-automation/vendor-testing/qemu.sh | 11 ++ ci-automation/vendor-testing/qemu_update.sh | 8 +- 4 files changed, 164 insertions(+), 18 deletions(-) diff --git a/.github/workflows/run-kola-tests.yaml b/.github/workflows/run-kola-tests.yaml index 9eb84816da..6c0ef80168 100644 --- a/.github/workflows/run-kola-tests.yaml +++ b/.github/workflows/run-kola-tests.yaml @@ -34,7 +34,7 @@ jobs: run: | sudo rm /bin/sh sudo ln -s /bin/bash /bin/sh - sudo apt-get install -y ca-certificates curl gnupg lsb-release qemu-system git bzip2 jq dnsmasq + sudo apt-get install -y ca-certificates curl gnupg lsb-release qemu-system git bzip2 jq dnsmasq python3 sudo systemctl stop dnsmasq sudo systemctl mask dnsmasq @@ -60,45 +60,156 @@ jobs: fetch-depth: 0 submodules: true - - name: Download artifact + - name: Download binpkgs if: ${{ !inputs.workflow_run_id }} uses: actions/download-artifact@v3 with: - name: images-${{ matrix.arch }} + name: ${{ matrix.arch }}-binpkgs - - name: Download artifacts from other workflow + - name: Download test update image + if: ${{ !inputs.workflow_run_id }} + uses: actions/download-artifact@v3 + with: + name: ${{ matrix.arch }}-test-update + + - name: Download generic image + if: ${{ !inputs.workflow_run_id }} + uses: actions/download-artifact@v3 + with: + name: ${{ matrix.arch }}-generic-image + + - name: Download developer container + if: ${{ !inputs.workflow_run_id }} + uses: actions/download-artifact@v3 + with: + name: ${{ matrix.arch }}-devcontainer + + - name: Download torcx tarball + if: ${{ !inputs.workflow_run_id }} + uses: actions/download-artifact@v3 + with: + name: ${{ matrix.arch }}-torcx + + - name: Download binpkgs from other workflow uses: gabriel-samfira/action-download-artifact@v5 if: ${{ inputs.workflow_run_id }} with: workflow: ${{ inputs.workflow_name_or_id }} workflow_conclusion: success run_id: ${{ inputs.workflow_run_id }} - name: images-${{ matrix.arch }} + name: ${{ matrix.arch }}-binpkgs + + - name: Download test update image from other workflow + uses: gabriel-samfira/action-download-artifact@v5 + if: ${{ inputs.workflow_run_id }} + with: + workflow: ${{ inputs.workflow_name_or_id }} + workflow_conclusion: success + run_id: ${{ inputs.workflow_run_id }} + name: ${{ matrix.arch }}-test-update + + - name: Download generic image from other workflow + uses: gabriel-samfira/action-download-artifact@v5 + if: ${{ inputs.workflow_run_id }} + with: + workflow: ${{ inputs.workflow_name_or_id }} + workflow_conclusion: success + run_id: ${{ inputs.workflow_run_id }} + name: ${{ matrix.arch }}-generic-image + + - name: Download developer container from other workflow + uses: gabriel-samfira/action-download-artifact@v5 + if: ${{ inputs.workflow_run_id }} + with: + workflow: ${{ inputs.workflow_name_or_id }} + workflow_conclusion: success + run_id: ${{ inputs.workflow_run_id }} + name: ${{ matrix.arch }}-devcontainer + + - name: Download torcx tarball from other workflow + uses: gabriel-samfira/action-download-artifact@v5 + if: ${{ inputs.workflow_run_id }} + with: + workflow: ${{ inputs.workflow_name_or_id }} + workflow_conclusion: success + run_id: ${{ inputs.workflow_run_id }} + name: ${{ matrix.arch }}-torcx + + - name: Extract artifacts + shell: bash + run: | + exec 2>&1 + set -x + set -euo pipefail + + # Set up a webserver for devcontainer and torcx tests. + # The respective tests will download devcontainer and torcx tarball via http. + # The devcontainer test will then run a build + # which will download and install binpkgs into the dev container. + # For the sake of that test we will serve both via a temporary local web server. + TESTS_WEBSERVER_WEBROOT="scripts/devcontainer-webroot" + default_rout_device="$(sudo ip -j route sh default |jq -r .[0].dev)" + TESTS_WEBSERVER_IP="$(sudo ip -j address show dev "${default_rout_device}" | jq -r .[0].addr_info[0].local)" + TESTS_WEBSERVER_PORT=12345 + echo "TESTS_WEBSERVER_WEBROOT=${TESTS_WEBSERVER_WEBROOT}" >> "$GITHUB_ENV" + echo "TESTS_WEBSERVER_IP=${TESTS_WEBSERVER_IP}" >> "$GITHUB_ENV" + echo "TESTS_WEBSERVER_PORT=${TESTS_WEBSERVER_PORT}" >> "$GITHUB_ENV" + + mkdir ${TESTS_WEBSERVER_WEBROOT} + mv flatcar_developer_container* ${TESTS_WEBSERVER_WEBROOT} + tar -C ${TESTS_WEBSERVER_WEBROOT} -xvf binpkgs.tar + + tar -C ${TESTS_WEBSERVER_WEBROOT} -xvf torcx.tar + + # Move torcx package into plain webroot + # (path consists of ///:.torcx.tar.gz) + mv "${TESTS_WEBSERVER_WEBROOT}/${{ matrix.arch }}-usr"/*/*/*.torcx.tgz \ + "${TESTS_WEBSERVER_WEBROOT}" + + # Update torcx.json's http URL to point to the webserver IP. + # ci.yaml defines the "localhost" placeholder in its "Set Environment" step. + sed -i "s,http://localhost:12345,http://${TESTS_WEBSERVER_IP}:${TESTS_WEBSERVER_PORT}," \ + "${TESTS_WEBSERVER_WEBROOT}/torcx_manifest.json" + cat "${TESTS_WEBSERVER_WEBROOT}/torcx_manifest.json" + + # Extract the generic image we'll use for qemu tests. + # Note that the qemu[_uefi] tests use the generic image instead of the + # qemu vendor VM image ("Astronaut: [...] Always have been."). + bzip2 --decompress flatcar_production_image.bin.bz2 + mv flatcar_production_image.bin flatcar_production_qemu_uefi_efi_code.fd scripts/ + + mv flatcar_test_update.gz scripts/ + - name: Run tests shell: bash run: | exec 2>&1 - set +x + set -x set -euo pipefail - # extract the image. - IMG_ARCHIVE=$(readlink -f images/**/flatcar_production_image.bin.bz2) - QEMU_UEFI_BIOS_FILE=$(readlink -f images/**/flatcar_production_qemu_uefi_efi_code.fd) - bzip2 --decompress ${IMG_ARCHIVE} - - cp ${IMG_ARCHIVE%%.bz2} ./scripts/ - cp ${QEMU_UEFI_BIOS_FILE} ./scripts/ + python3 -m http.server -d "${TESTS_WEBSERVER_WEBROOT}" -b "${TESTS_WEBSERVER_IP}" "${TESTS_WEBSERVER_PORT}" & pushd scripts source ci-automation/test.sh + # Provide our own torcx prepare function so we use our local manifest json. + # This is called by test_run below. + function __prepare_torcx() { + shift; shift # no need for arch or vernum + local destdir="$1" + cp "../${TESTS_WEBSERVER_WEBROOT}/torcx_manifest.json" "${destdir}" + } + PARALLEL_ARCH=10 cat > sdk_container/.env < Date: Thu, 23 Mar 2023 20:19:18 +0100 Subject: [PATCH 03/10] ci.yaml: run only when ready and / or review requested --- .github/workflows/ci.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 58d01dd4db..06f17bb054 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,6 +1,9 @@ name: "Run build" on: pull_request: + # Run when the PR is ready and each time a review is re-requested + # (i.e. after feedback has been addressed). + types: [review_requested, ready_for_review] workflow_dispatch: inputs: image_formats: @@ -35,6 +38,8 @@ permissions: {} jobs: packages: + # Do not run when still in draft mode but a review was requested anyway + if: github.event.pull_request.draft == false name: "Build Flatcar packages" runs-on: - self-hosted From 6f97e85590a87569d77770361395d30397f2f51a Mon Sep 17 00:00:00 2001 From: Thilo Fromm Date: Fri, 24 Mar 2023 07:35:20 +0100 Subject: [PATCH 04/10] dispatch-kola-tests.yaml: Elaborated comments on inputs --- .github/workflows/dispatch-kola-tests.yaml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dispatch-kola-tests.yaml b/.github/workflows/dispatch-kola-tests.yaml index a104c7c7ab..c970711d30 100644 --- a/.github/workflows/dispatch-kola-tests.yaml +++ b/.github/workflows/dispatch-kola-tests.yaml @@ -8,12 +8,15 @@ on: required: true default: ci.yaml description: | - The workflow ID from where we'll download the artifacts to be tested. + The workflow name or ID from where we'll download the artifacts to be tested. + E.g. the name of the YAML file (w/o path) of the respective workflow. workflow_run_id: type: string required: true description: | - The run ID of the workflow specified in workflow_name_or_id + The run ID of the workflow specified in workflow_name_or_id. + You can e.g. get this from a run's URL - + https://github.com/flatcar/scripts/actions/runs/ . permissions: {} From 43bbabec4bcdd07f42220c685886a669593cff65 Mon Sep 17 00:00:00 2001 From: Thilo Fromm Date: Mon, 27 Mar 2023 09:35:05 +0200 Subject: [PATCH 05/10] ci.yaml: extract + upload build logs also on failure Signed-off-by: Thilo Fromm --- .github/workflows/ci.yaml | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 06f17bb054..0a27cf1a6d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -151,6 +151,7 @@ jobs: sdk_image="$(docker_image_fullname "${sdk_name}" "${docker_sdk_vernum}")" container_name="flatcar-ci-build" + echo "container_name=${container_name}" >> "$GITHUB_ENV" # Create version file ( @@ -166,19 +167,24 @@ jobs: --torcx_output_root="${CI_CONTAINER_TORCX_ROOT}" \ --torcx_extra_pkg_url="${TORCX_TESTS_PACKAGE_URL}" - # Copy logs - ./run_sdk_container -n "${container_name}" \ - tar -cJf ebuild_logs.tar.xz /build/${arch}-usr/var/log/portage \ - /build/${arch}-usr/var/tmp/portage - # Create binpkgs tarball for archiving as artifact later ./run_sdk_container -n "${container_name}" \ tar -C "/build/${arch}-usr/var/lib/portage/pkgs/" \ -cvf binpkgs.tar . - echo "container_name=${container_name}" >> "$GITHUB_ENV" + - name: Extract build logs + if: always() + shell: bash + run: | + set -euo pipefail + set -x + # Copy logs + ./run_sdk_container -n "${container_name}" \ + tar -cJf ebuild_logs.tar.xz /build/${arch}-usr/var/log/portage \ + /build/${arch}-usr/var/tmp/portage - name: Upload build logs + if: always() uses: actions/upload-artifact@v3 with: retention-days: 7 From 3653c9a8e54a06f6eb373e070ec8eb91a497598b Mon Sep 17 00:00:00 2001 From: Thilo Fromm Date: Mon, 27 Mar 2023 18:14:33 +0200 Subject: [PATCH 06/10] [ci,run-kola-tests].yaml: use HEAD commit ref for PR builds Signed-off-by: Thilo Fromm --- .github/workflows/ci.yaml | 15 +++++++++++++++ .github/workflows/run-kola-tests.yaml | 16 ++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 0a27cf1a6d..a36e9df6de 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -76,6 +76,21 @@ jobs: fetch-depth: 0 submodules: true + # Hack alert: actions/checkout will check out the (disjunct) merge commit of a PR + # instead of its head commit. That commit is not connected to any branch. + # This causes breakage downstream e.g. when the devcontainer test wants to check out + # the ref in the scripts repo that corresponds to this build. + - name: If this is a PR build, use head commit instead of the merge commit + if: ${{ github.event.pull_request.head.sha }} + shell: bash + run: | + exec 2>&1 + set -x + set -euo pipefail + + git checkout ${{ github.event.pull_request.head.sha }} + git submodule update + - name: Set environment shell: bash run: | diff --git a/.github/workflows/run-kola-tests.yaml b/.github/workflows/run-kola-tests.yaml index 6c0ef80168..9e7104464f 100644 --- a/.github/workflows/run-kola-tests.yaml +++ b/.github/workflows/run-kola-tests.yaml @@ -60,6 +60,22 @@ jobs: fetch-depth: 0 submodules: true + # Hack alert: actions/checkout will check out the (disjunct) merge commit of a PR + # instead of its head commit. That commit is not connected to any branch. + # This is not technically necessary for the tests run but it is done to remain aligned + # with the ref. + - name: If this is a PR build, use head commit instead of the merge commit + if: ${{ github.event.pull_request.head.sha }} + shell: bash + run: | + exec 2>&1 + set -x + set -euo pipefail + + cd scripts + git checkout ${{ github.event.pull_request.head.sha }} + git submodule update + - name: Download binpkgs if: ${{ !inputs.workflow_run_id }} uses: actions/download-artifact@v3 From ae760903a3ac98d9d0a907ec02ed7e93cc804e34 Mon Sep 17 00:00:00 2001 From: Thilo Fromm Date: Tue, 28 Mar 2023 10:50:25 +0200 Subject: [PATCH 07/10] run-kola-tests: improve test results archive globs Signed-off-by: Thilo Fromm --- .github/workflows/run-kola-tests.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/run-kola-tests.yaml b/.github/workflows/run-kola-tests.yaml index 9e7104464f..e734af184a 100644 --- a/.github/workflows/run-kola-tests.yaml +++ b/.github/workflows/run-kola-tests.yaml @@ -251,11 +251,11 @@ jobs: with: name: ${{ matrix.arch }}-test-results path: | - scripts/__TESTS__ - scripts/results-.*.tap + scripts/__TESTS__/*/_kola_temp/ + scripts/results-*.tap - name: Create Test Summary if: always() uses: test-summary/action@v2 with: - paths: "scripts/results-.*.tap" + paths: "scripts/results-*.tap" From ab2000e3e46e47a3b12a520e495cbeb43ea1b9fd Mon Sep 17 00:00:00 2001 From: Thilo Fromm Date: Tue, 28 Mar 2023 21:09:11 +0200 Subject: [PATCH 08/10] run-kola-tests.yaml: fix test-summary TAP formatting Signed-off-by: Thilo Fromm --- .github/workflows/run-kola-tests.yaml | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/.github/workflows/run-kola-tests.yaml b/.github/workflows/run-kola-tests.yaml index e734af184a..82fe53dc34 100644 --- a/.github/workflows/run-kola-tests.yaml +++ b/.github/workflows/run-kola-tests.yaml @@ -254,8 +254,26 @@ jobs: scripts/__TESTS__/*/_kola_temp/ scripts/results-*.tap + - name: Patch TAP reports so test-summary can parse + if: always() + shell: bash + run: | + exec 2>&1 + set -x + set -euo pipefail + + cd scripts + for tap in results-*.tap; do + sumtap="test-summary-${tap}" + # If this is missing then test-summary assumes the TAP report + # is, in fact, XML. + # See https://github.com/flatcar/scripts/pull/696#discussion_r1151027499 + echo "TAP version 13" > "${sumtap}" + cat "${tap}" >> "${sumtap}" + done + - name: Create Test Summary if: always() uses: test-summary/action@v2 with: - paths: "scripts/results-*.tap" + paths: "scripts/test-summary-results-*.tap" From f07cb5f7815ed899d1b9675db9f306fef1354138 Mon Sep 17 00:00:00 2001 From: Thilo Fromm Date: Wed, 29 Mar 2023 12:59:34 +0200 Subject: [PATCH 09/10] tapfile_helper ff.: support TAP and Markdown output This change adds markdown output support to tapfile helper. tap_generate_report() has been refactored to use low-level output functions to write tests; TAP and markdown output is supported and both are generated by default. Also, it should be straightforward to add other output formats by implementing the respective low level print functions. The markdown output is now used by run-kola-tests.yaml to generate step output and, if run from a PR, add a comment with test results to the PR. Signed-off-by: Thilo Fromm --- .github/workflows/ci.yaml | 3 +- .github/workflows/dispatch-kola-tests.yaml | 3 +- .github/workflows/run-kola-tests.yaml | 54 +++---- ci-automation/ci-config.env | 5 + ci-automation/tapfile_helper_lib.sh | 176 ++++++++++++++++++--- ci-automation/test.sh | 8 +- ci-automation/test_update_reruns.sh | 8 +- 7 files changed, 204 insertions(+), 53 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a36e9df6de..2892c66970 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -34,7 +34,8 @@ concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }} cancel-in-progress: true -permissions: {} +permissions: + pull-requests: write jobs: packages: diff --git a/.github/workflows/dispatch-kola-tests.yaml b/.github/workflows/dispatch-kola-tests.yaml index c970711d30..edc9ca96b3 100644 --- a/.github/workflows/dispatch-kola-tests.yaml +++ b/.github/workflows/dispatch-kola-tests.yaml @@ -18,7 +18,8 @@ on: You can e.g. get this from a run's URL - https://github.com/flatcar/scripts/actions/runs/ . -permissions: {} +permissions: + pull-requests: write jobs: test: diff --git a/.github/workflows/run-kola-tests.yaml b/.github/workflows/run-kola-tests.yaml index 82fe53dc34..99db29bab4 100644 --- a/.github/workflows/run-kola-tests.yaml +++ b/.github/workflows/run-kola-tests.yaml @@ -26,6 +26,8 @@ jobs: fail-fast: false matrix: arch: ["amd64", "arm64"] + permissions: + pull-requests: write steps: - name: Prepare machine @@ -196,7 +198,6 @@ jobs: mv flatcar_test_update.gz scripts/ - - name: Run tests shell: bash run: | @@ -235,9 +236,8 @@ jobs: export MAX_RETRIES=5 export SKIP_COPY_TO_BINCACHE=1 - # run the test. + # run the tests. test_run ${{ matrix.arch }} qemu_uefi - test_run ${{ matrix.arch }} qemu_update # Stop the background webserver @@ -245,6 +245,26 @@ jobs: kill %1 set -e + - name: Create Test Summary + if: always() + shell: bash + run: | + exec 2>&1 + set -x + set -euo pipefail + + # qemu_update report includes all reqults of qemu_uefi as test results are + # stored in a temporary sqlite DB in scripts/ which is not deleted between test runs. + cp scripts/results-qemu_update.md ./test-results.md + cat test-results.md >> "$GITHUB_STEP_SUMMARY" + + - name: If started from a PR, post test summary to PR + if: ${{ github.event_name == 'pull_request' }} + uses: mshick/add-pr-comment@v2 + with: + if: always() + message-path: "test-results.md" + - name: Upload artifacts if: always() uses: actions/upload-artifact@v3 @@ -252,28 +272,8 @@ jobs: name: ${{ matrix.arch }}-test-results path: | scripts/__TESTS__/*/_kola_temp/ + scripts/__TESTS__/*/*.tap + scripts/__TESTS__/*/*.txt scripts/results-*.tap - - - name: Patch TAP reports so test-summary can parse - if: always() - shell: bash - run: | - exec 2>&1 - set -x - set -euo pipefail - - cd scripts - for tap in results-*.tap; do - sumtap="test-summary-${tap}" - # If this is missing then test-summary assumes the TAP report - # is, in fact, XML. - # See https://github.com/flatcar/scripts/pull/696#discussion_r1151027499 - echo "TAP version 13" > "${sumtap}" - cat "${tap}" >> "${sumtap}" - done - - - name: Create Test Summary - if: always() - uses: test-summary/action@v2 - with: - paths: "scripts/test-summary-results-*.tap" + scripts/results-*.md + test-results.md diff --git a/ci-automation/ci-config.env b/ci-automation/ci-config.env index fb7b476a57..944625fe45 100644 --- a/ci-automation/ci-config.env +++ b/ci-automation/ci-config.env @@ -47,6 +47,11 @@ CONTAINER_IMAGE_ROOT="/home/sdk/build/images" # echo "export PARALLEL_TESTS=\"5\"" > sdk_container/.env # to override the number of test cases to be run in parallel. +# -- General -- + +# "tap" for TAP reports, "md" for markdown are currently supported +TEST_REPORT_FORMATS=("tap" "md") + # -- QEMU -- QEMU_IMAGE_NAME=${QEMU_IMAGE_NAME:-flatcar_production_image.bin} diff --git a/ci-automation/tapfile_helper_lib.sh b/ci-automation/tapfile_helper_lib.sh index bbb7c6c4c5..8c837d8315 100644 --- a/ci-automation/tapfile_helper_lib.sh +++ b/ci-automation/tapfile_helper_lib.sh @@ -197,27 +197,167 @@ function tap_failed_tests_for_vendor() { } # -- +# TAP output format primitives for tap_generate_report() + +__tap_print_header() { + local arch="$1" + local version="$2" + local vendors="$3" + local count="$4" + + # We use count + 1 here because the very first "test result" will just print + # the list of platforms tested, not an actual test's result. + echo "1..$((count+1))" + echo "ok - Version: ${version}, Architecture: ${arch}" + echo " ---" + echo " Platforms tested: ${vendors}" + echo " ..." +} +# -- + +__tap_print_test_verdict() { + local verdict="$1" + local name="$2" + local succeded_vendors="$3" + local failed_vendors="$4" + + echo "${verdict} - ${test_name}" + echo " ---" + + if [ -n "${succeded_vendors}" ] ; then + echo " Succeeded: ${succeded_vendors}" + fi + if [ -n "${failed_vendors}" ] ; then + echo " Failed: ${failed_vendors}" + fi +} +# -- + +__tap_print_test_run_diag_output() { + local vendor="$1" + local run="$2" + echo " Error messages for ${vendor}, run ${run}:" + cat - +} +# -- + +__tap_finish_test_verdict() { + local verdict="$1" + local name="$2" + local succeded_vendors="$3" + local failed_vendors="$4" + echo " ..." +} +# -- + +__tap_finish_test_report() { + true +} +# -- + +# markdown output format primitives for tap_generate_report() + +__md_print_header() { + local arch="$1" + local version="$2" + local vendors="$3" + local count="$4" + + echo "### Test report for ${version} / ${arch}" + echo + echo "**Platforms tested** : ${vendors}" +} +# -- + +__md_print_test_verdict() { + local verdict="$1" + local name="$2" + local succeded_vendors="$3" + local failed_vendors="$4" + + v="![${verdict}](https://via.placeholder.com/50x20/00ff00/000000?text=PASS)" + if [ "${verdict}" = "not ok" ] ; then + v="![${verdict}](https://via.placeholder.com/50x20/ff0000/ffffff?text=FAIL)" + fi + + echo + echo -n "${v} **${name}**" + if [ -n "${succeded_vendors}" ] ; then + echo -n " 🟢 Succeeded: ${succeded_vendors}" + fi + if [ -n "${failed_vendors}" ] ; then + echo -n " ❌ Failed: ${failed_vendors}" + fi + echo + if [ "${verdict}" = "not ok" ] ; then + echo + echo "
" + echo + fi +} +# -- + +__md_print_test_run_diag_output() { + local vendor="$1" + local run="$2" + + echo "* Diagnostic output for ${vendor}, run ${run}" + echo + echo " \`\`\`" + cat - + echo " \`\`\`" + echo + +} +# -- +# +__md_finish_test_verdict() { + local verdict="$1" + local name="$2" + local succeded_vendors="$3" + local failed_vendors="$4" + if [ "${verdict}" = "not ok" ] ; then + echo + echo "
" + echo + fi +} +# -- + +__md_finish_test_report() { + true +} +# -- + + # Print the tap file from contents of the database. # INPUT: # 1: - Architecture to be included in the first line of the report # 2: - OS version tested, to be included in the first line of the report -# 3: - If set to "true" then debug output of transient test failures +# 3: - Output format of the report. "tap" and "markdown" are supported. +# 4: - If set to "true" then debug output of transient test failures # is included in the result report. function tap_generate_report() { local arch="$1" local version="$2" - local full_error_report="${3:-false}" + local format="$3" + local full_error_report="${4:-false}" + + case "${format}" in + tap) ;; + md) ;; + *) echo "ERROR: tap_generate_report() unknown format '${format}'" >&2 + return 1 + ;; + esac + local count count="$(__sqlite3_wrapper 'SELECT count(name) FROM test_case;')" local vendors vendors="$(__sqlite3_wrapper 'SELECT name FROM vendor;' | tr '\n' ' ')" - echo "1..$((count+1))" - echo "ok - Version: ${version}, Architecture: ${arch}" - echo " ---" - echo " Platforms tested: ${vendors}" - echo " ..." + __"${format}"_print_header "${arch}" "${version}" "${vendors}" "${count}" # Print result line for every test, including platforms it succeeded on # and transient failed runs. @@ -265,21 +405,17 @@ function tap_generate_report() { r=r ", " $2 else r="(" $2 ; } - END { if (t) print t r ")"; }' + END { if (t) print t " " r ")"; }' } - local succeded - succeded="$(list_runs 1)" + local succeeded + succeeded="$(list_runs 1)" local failed failed="$(list_runs 0)" - echo "${verdict} - ${test_name}" - echo " ---" - if [ -n "${succeded}" ] ; then - echo " Succeeded: ${succeded}" - fi + __"${format}"_print_test_verdict "${verdict}" "${test_name}" \ + "${succeeded}" "${failed}" if [ -n "${failed}" ] ; then - echo " Failed: ${failed}" if [ "${verdict}" = "not ok" -o "${full_error_report}" = "true" ] ; then # generate diagnostic output, per failed run. __sqlite3_wrapper -csv " @@ -291,7 +427,7 @@ function tap_generate_report() { ORDER BY t.run DESC;" | \ sed 's/,/ /' | \ while read -r vendor run; do - echo " Error messages for ${vendor}, run ${run}:" + { __sqlite3_wrapper -csv " SELECT t.output FROM test_run AS t, test_case AS c WHERE t.case_id=c.id @@ -299,10 +435,14 @@ function tap_generate_report() { AND t.run='${run}';" | \ sed 's/"/ /g' | \ awk '{print " L" NR ": \"" $0 "\""}' + } | __"${format}"_print_test_run_diag_output "${vendor}" "${run}" done fi fi - echo " ..." + __"${format}"_finish_test_verdict "${verdict}" "${test_name}" \ + "${succeeded}" "${failed}" done + + __"${format}"_finish_test_report } # -- diff --git a/ci-automation/test.sh b/ci-automation/test.sh index c9ec153a3d..2082aa83dd 100644 --- a/ci-automation/test.sh +++ b/ci-automation/test.sh @@ -165,8 +165,8 @@ function _test_run_impl() { # Make the torcx artifacts available to test implementation __prepare_torcx "${arch}" "${vernum}" "${work_dir}" - local tap_merged_summary="results-${image}.tap" - local tap_merged_detailed="results-${image}-detailed.tap" + local tap_merged_summary="results-${image}" + local tap_merged_detailed="results-${image}-detailed" local retry="" local success=false local print_give_up=true @@ -242,9 +242,9 @@ function _test_run_impl() { copy_to_buildcache "testing/${vernum}/${arch}/${image}" \ "${tests_dir}/"*.tap copy_to_buildcache "testing/${vernum}/${arch}/${image}" \ - "${tap_merged_summary}" + "${tap_merged_summary}"* copy_to_buildcache "testing/${vernum}/${arch}/${image}" \ - "${tap_merged_detailed}" + "${tap_merged_detailed}"* fi if ! $success; then return 1 diff --git a/ci-automation/test_update_reruns.sh b/ci-automation/test_update_reruns.sh index 2fcd79e543..0cd91f9ad4 100755 --- a/ci-automation/test_update_reruns.sh +++ b/ci-automation/test_update_reruns.sh @@ -19,8 +19,12 @@ failfile="$6" merged_summary="$7" merged_detailed="$8" +source ci-automation/ci-config.env source ci-automation/tapfile_helper_lib.sh tap_ingest_tapfile "${tapfile}" "${image}" "${retry}" tap_failed_tests_for_vendor "${image}" > "${failfile}" -tap_generate_report "${arch}" "${vernum}" > "${merged_summary}" -tap_generate_report "${arch}" "${vernum}" "true" > "${merged_detailed}" + +for format in "${TEST_REPORT_FORMATS[@]}"; do + tap_generate_report "${arch}" "${vernum}" "${format}" > "${merged_summary}.${format}" + tap_generate_report "${arch}" "${vernum}" "${format}" "true" > "${merged_detailed}.${format}" +done From f8bf7fa4c705a35f03af2dca2a1c1d32fe16a8fe Mon Sep 17 00:00:00 2001 From: Thilo Fromm Date: Thu, 30 Mar 2023 09:46:08 +0200 Subject: [PATCH 10/10] run-kola-tests.yaml: test report merge job Signed-off-by: Thilo Fromm --- .github/workflows/run-kola-tests.yaml | 130 +++++++++++++++++++++----- 1 file changed, 105 insertions(+), 25 deletions(-) diff --git a/.github/workflows/run-kola-tests.yaml b/.github/workflows/run-kola-tests.yaml index 99db29bab4..15a161c467 100644 --- a/.github/workflows/run-kola-tests.yaml +++ b/.github/workflows/run-kola-tests.yaml @@ -26,8 +26,6 @@ jobs: fail-fast: false matrix: arch: ["amd64", "arm64"] - permissions: - pull-requests: write steps: - name: Prepare machine @@ -245,35 +243,117 @@ jobs: kill %1 set -e - - name: Create Test Summary - if: always() - shell: bash - run: | - exec 2>&1 - set -x - set -euo pipefail - - # qemu_update report includes all reqults of qemu_uefi as test results are - # stored in a temporary sqlite DB in scripts/ which is not deleted between test runs. - cp scripts/results-qemu_update.md ./test-results.md - cat test-results.md >> "$GITHUB_STEP_SUMMARY" - - - name: If started from a PR, post test summary to PR - if: ${{ github.event_name == 'pull_request' }} - uses: mshick/add-pr-comment@v2 - with: - if: always() - message-path: "test-results.md" - - - name: Upload artifacts + - name: Upload detailed test logs if: always() uses: actions/upload-artifact@v3 with: - name: ${{ matrix.arch }}-test-results + name: ${{ matrix.arch }}-test-logs-and-results path: | scripts/__TESTS__/*/_kola_temp/ scripts/__TESTS__/*/*.tap scripts/__TESTS__/*/*.txt scripts/results-*.tap scripts/results-*.md - test-results.md + + - name: Upload raw TAP files of all runs for later merging + if: always() + uses: actions/upload-artifact@v3 + with: + name: ${{ matrix.arch }}-raw-tapfiles + path: | + scripts/__TESTS__/*/*.tap + + + merge_and_publish_results: + name: "Merge TAP reports and post results" + needs: tests + if: always() + runs-on: + - self-hosted + - debian + - kola + permissions: + pull-requests: write + + steps: + - name: Prepare machine + shell: bash + working-directory: ${{ github.workspace }} + run: | + sudo rm /bin/sh + sudo ln -s /bin/bash /bin/sh + sudo apt-get install -y ca-certificates curl gnupg lsb-release git bzip2 jq sqlite3 + + - uses: actions/checkout@v3 + with: + path: scripts + fetch-depth: 0 + submodules: true + + # Hack alert: actions/checkout will check out the (disjunct) merge commit of a PR + # instead of its head commit. That commit is not connected to any branch. + # This is not technically necessary for the tests run but it is done to remain aligned + # with the ref. + - name: If this is a PR build, use head commit instead of the merge commit + if: ${{ github.event.pull_request.head.sha }} + shell: bash + run: | + exec 2>&1 + set -x + set -euo pipefail + + cd scripts + git checkout ${{ github.event.pull_request.head.sha }} + git submodule update + + # This is clunky. Haven't figured out how to re-use matrix.arch here for downloads, + # so we download each arch individually. + - name: Download amd64 tapfiles + uses: actions/download-artifact@v3 + with: + name: amd64-raw-tapfiles + path: scripts/__TAP__/amd64 + + - name: Download arm64 tapfiles + uses: actions/download-artifact@v3 + with: + name: arm64-raw-tapfiles + path: scripts/__TAP__/arm64 + + - name: Create Test Summary + shell: bash + run: | + exec 2>&1 + set -x + set -euo pipefail + + cd scripts + + ls -laR __TAP__ + + source ci-automation/tapfile_helper_lib.sh + + all_archs="" + for arch in __TAP__/*; do + arch_name="$(basename "${arch}")" + all_archs="${all_archs} ${arch_name}" + for vendor in "${arch}"/*; do + vendor_name="$(basename "${vendor}")" + run=1 + for tap in "${vendor}"/*.tap; do + tap_ingest_tapfile "${tap}" "${vendor_name}-${arch_name}" "${run}" + ((run++)) + done + done + done + + source sdk_container/.repo/manifests/version.txt + tap_generate_report "${all_archs}" "${FLATCAR_VERSION}" "md" "true" > test-results.md + + cat test-results.md >> "$GITHUB_STEP_SUMMARY" + + - name: If started from a PR, post test summary to PR + if: ${{ github.event_name == 'pull_request' }} + uses: mshick/add-pr-comment@v2 + with: + message-path: "test-results.md"