From 69353bfa6df67d4537fc0b9b78e8edab63c629c8 Mon Sep 17 00:00:00 2001 From: Thilo Fromm Date: Wed, 15 Mar 2023 10:43:09 +0100 Subject: [PATCH 01/10] ci.yaml: re-use build container, finer grained artifact upload This change removes "docker commit" at the end of each step and instead makes build steps re-use the build container, saving some build time. It also makes artifact upload more granular, so build logs, images, and dev container can be downloaded individually. Lastly, it exports torcx tarball and binary packages as a separate artifact each, for successive re-use in the kola tests. --- .github/workflows/ci.yaml | 202 +++++++++++++++++++++++--------------- 1 file changed, 121 insertions(+), 81 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e008c7cbb7..58d01dd4db 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -3,11 +3,6 @@ on: pull_request: workflow_dispatch: inputs: - bincache_server: - description: | - Bincache server. - default: "bincache.flatcar-linux.net" - required: true image_formats: description: | Space-separated vendor formats to build. @@ -79,7 +74,6 @@ jobs: - name: Set environment shell: bash run: | - BUILDCACHE_SERVER="bincache.flatcar-linux.net" arch="${{ matrix.arch }}" COREOS_REMOTE="" COREOS_REF="" @@ -87,14 +81,12 @@ jobs: PORTAGE_REF="" IMAGE_FORMATS="qemu_uefi" - [ -z "${{ github.event.inputs.bincache_server }}" ] || BUILDCACHE_SERVER="${{ github.event.inputs.bincache_server }}" [ -z "${{ github.event.inputs.coreos_remote }}" ] || COREOS_REMOTE="${{ github.event.inputs.coreos_remote }}" [ -z "${{ github.event.inputs.coreos_ref }}" ] || COREOS_REF="${{ github.event.inputs.coreos_ref }}" [ -z "${{ github.event.inputs.portage_remote }}" ] || PORTAGE_REMOTE="${{ github.event.inputs.portage_remote }}" [ -z "${{ github.event.inputs.portage_ref }}" ] || PORTAGE_REF="${{ github.event.inputs.portage_ref }}" [ -z "${{ github.event.inputs.image_formats }}" ] || IMAGE_FORMATS="${{ github.event.inputs.image_formats }}" - echo "BUILDCACHE_SERVER=${BUILDCACHE_SERVER}" >> $GITHUB_ENV echo "arch=${arch}" >> $GITHUB_ENV echo "COREOS_REMOTE=${COREOS_REMOTE}" >> $GITHUB_ENV echo "COREOS_REF=${COREOS_REF}" >> $GITHUB_ENV @@ -102,6 +94,15 @@ jobs: echo "PORTAGE_REF=${PORTAGE_REF}" >> $GITHUB_ENV echo "IMAGE_FORMATS=${IMAGE_FORMATS}" >> $GITHUB_ENV + # Artifact root for images and torcx tarball as seen from within the container + echo "CI_CONTAINER_ARTIFACT_ROOT=/home/sdk/trunk/src/scripts/artifacts" >> $GITHUB_ENV + echo "CI_CONTAINER_TORCX_ROOT=/home/sdk/trunk/src/scripts/artifacts/torcx" >> $GITHUB_ENV + mkdir -p artifacts/torcx + + # Placeholder URL for run-kola-tests.yaml, "Extract artifacts" step which will replace + # this with its IP address. + echo "TORCX_TESTS_PACKAGE_URL=http://localhost:12345" >> $GITHUB_ENV + - name: Checkout submodules shell: bash run: | @@ -129,7 +130,7 @@ jobs: shell: bash run: | exec 2>&1 - set +x + set -x set -euo pipefail source ci-automation/ci_automation_common.sh @@ -144,92 +145,70 @@ jobs: docker_image_from_registry_or_buildcache "${sdk_name}" "${docker_sdk_vernum}" sdk_image="$(docker_image_fullname "${sdk_name}" "${docker_sdk_vernum}")" - vernum="${version#*-}" # remove main-,alpha-,beta-,stable-,lts- version tag - docker_vernum="$(vernum_to_docker_image_version "${vernum}")" - packages_container="flatcar-packages-${arch}-${docker_vernum}" + container_name="flatcar-ci-build" # Create version file ( source sdk_lib/sdk_container_common.sh create_versionfile "$sdk_version" "$version" ) - ./run_sdk_container -n "${packages_container}" -v "${version}" \ + + # Run the packages build. This will create the ci build container + # which will be re-used by subsequent build steps. + ./run_sdk_container -n "${container_name}" -v "${version}" \ -C "${sdk_image}" \ ./build_packages --board="${arch}-usr" \ - --torcx_output_root="${CONTAINER_TORCX_ROOT}" + --torcx_output_root="${CI_CONTAINER_TORCX_ROOT}" \ + --torcx_extra_pkg_url="${TORCX_TESTS_PACKAGE_URL}" - # copy torcx manifest and docker tarball for publishing - torcx_tmp="__build__/torcx_tmp" - rm -rf "${torcx_tmp}" - mkdir "${torcx_tmp}" - ./run_sdk_container -n "${packages_container}" -v "${version}" \ - -C "${sdk_image}" \ - cp -r "${CONTAINER_TORCX_ROOT}/" \ - "${torcx_tmp}" + # Copy logs + ./run_sdk_container -n "${container_name}" \ + tar -cJf ebuild_logs.tar.xz /build/${arch}-usr/var/log/portage \ + /build/${arch}-usr/var/tmp/portage - source sdk_container/.repo/manifests/version.txt - vernum="${FLATCAR_VERSION}" - docker_vernum="$(vernum_to_docker_image_version "${vernum}")" - packages_image="flatcar-packages-${arch}" + # Create binpkgs tarball for archiving as artifact later + ./run_sdk_container -n "${container_name}" \ + tar -C "/build/${arch}-usr/var/lib/portage/pkgs/" \ + -cvf binpkgs.tar . - echo "vernum=${vernum}" >> $GITHUB_ENV - echo "docker_vernum=${docker_vernum}" >> $GITHUB_ENV - echo "packages_image=${packages_image}" >> $GITHUB_ENV - echo "arch=${arch}" >> $GITHUB_ENV - echo "sdk_image=${sdk_image}" >> $GITHUB_ENV - echo "packages_container=${packages_container}" >> $GITHUB_ENV - docker commit "${packages_container}" "${packages_image}:${docker_vernum}" - docker rm -f "${packages_container}" + echo "container_name=${container_name}" >> "$GITHUB_ENV" + + - name: Upload build logs + uses: actions/upload-artifact@v3 + with: + retention-days: 7 + name: ${{ matrix.arch }}-build-logs + path: | + scripts/ebuild_logs.tar.xz - name: Build image shell: bash run: | set -euo pipefail - set +x + set -x echo 'channel="developer"' >> $GITHUB_ENV channel="developer" source ci-automation/ci_automation_common.sh - packages="flatcar-packages-${arch}" - packages_image="${packages}:${docker_vernum}" - image="flatcar-images-${arch}" - image_container="${image}-${docker_vernum}" official_arg="--noofficial" - echo "image=flatcar-images-${arch}" >> $GITHUB_ENV - echo "image_image=${image}:${docker_vernum}" >> $GITHUB_ENV - - ./run_sdk_container -x ./ci-cleanup.sh -n "${image_container}" -C "${packages_image}" \ - -v "${vernum}" \ - mkdir -p "${CONTAINER_IMAGE_ROOT}" - ./run_sdk_container -n "${image_container}" -C "${packages_image}" \ - -v "${vernum}" \ + ./run_sdk_container -n "${container_name}" \ ./set_official --board="${arch}-usr" "${official_arg}" - ./run_sdk_container -n "${image_container}" -C "${packages_image}" \ - -v "${vernum}" \ + ./run_sdk_container -n "${container_name}" \ ./build_image --board="${arch}-usr" --group="${channel}" \ - --output_root="${CONTAINER_IMAGE_ROOT}" \ - --torcx_root="${CONTAINER_TORCX_ROOT}" prodtar container + --output_root="${CI_CONTAINER_ARTIFACT_ROOT}" \ + --torcx_root="${CI_CONTAINER_TORCX_ROOT}" prodtar container - # Copy logs - ./run_sdk_container -n "${image_container}" -C "${packages_image}" -v "${vernum}" \ - tar -cJf ebuild_logs.tar.xz /build/${arch}-usr/var/log/portage \ - /build/${arch}-usr/var/tmp/portage - - docker commit "${image_container}" "${image}:${docker_vernum}" - docker rm -f "${image_container}" - - - name: Build VM image + - name: Build VM image(s) shell: bash run: | set -euo pipefail - set +x + set -x source ci-automation/ci_automation_common.sh - vms_container="flatcar-vms-${docker_vernum}" images_out="images" has_packet=0 @@ -254,35 +233,96 @@ jobs: for format in ${formats}; do echo " ################### VENDOR '${format}' ################### " - ./run_sdk_container -n "${vms_container}" -C "${image_image}" \ - -v "${vernum}" \ + ./run_sdk_container -n "${container_name}" \ ./image_to_vm.sh --format "${format}" --board="${arch}-usr" \ - --from "${CONTAINER_IMAGE_ROOT}/${arch}-usr/latest" \ + --from "${CI_CONTAINER_ARTIFACT_ROOT}/${arch}-usr/latest" \ --image_compression_formats=bz2 done - # copy resulting images - ./run_sdk_container -n "${vms_container}" \ - -v "${vernum}" \ - mv "${CONTAINER_IMAGE_ROOT}/${arch}-usr" "./${images_out}" + # upload-artifacts cannot handle artifact uploads from sym-linked directories (no, really) + # so we move things around. + mkdir -p artifacts/images + ( + cd artifacts/${arch}-usr/latest/ + mv * ../../images/ + ) - # remove symlinks before upload - find "./${images_out}" -type l -delete + # create a tarball for torcx package + JSON file because upload-artifacts cannot handle filenames containing colons + # (such as "docker:20.10.torcx.tgz") + mv artifacts/torcx/${arch}-usr/latest/torcx_manifest.json artifacts/torcx/pkgs/ + tar -C artifacts/torcx/pkgs/ -cvf torcx.tar . - docker rm -f "${vms_container}" - - name: Upload artifacts + - name: Upload binpkgs uses: actions/upload-artifact@v3 with: - name: images-${{ matrix.arch }} + retention-days: 7 + name: ${{ matrix.arch }}-binpkgs path: | - scripts/images/**/*.img.bz2 - scripts/images/**/*.bin.bz2 - scripts/images/**/flatcar_production_*_efi_*.fd - scripts/images/**/*.txt - scripts/images/**/flatcar_production_*.sh - scripts/images/**/flatcar_test_update.gz - scripts/ebuild_logs.tar.xz + scripts/binpkgs.tar + + - name: Upload update image (used with kola tests later) + uses: actions/upload-artifact@v3 + with: + retention-days: 7 + name: ${{ matrix.arch }}-test-update + path: | + scripts/artifacts/images/flatcar_test_update.gz + + - name: Upload generic image + uses: actions/upload-artifact@v3 + with: + retention-days: 7 + name: ${{ matrix.arch }}-generic-image + path: | + scripts/artifacts/images/flatcar_production_image.bin.bz2 + scripts/artifacts/images/flatcar_production_image.grub + scripts/artifacts/images/flatcar_production_image.shim + scripts/artifacts/images/flatcar_production_image.vmlinuz + scripts/artifacts/images/flatcar_production_image*.txt + scripts/artifacts/images/flatcar_production_image*.json + scripts/artifacts/images/flatcar_production_image_pcr_policy.zip + scripts/artifacts/images/flatcar_production_*_efi_*.fd + + - name: Upload developer container + uses: actions/upload-artifact@v3 + with: + retention-days: 7 + name: ${{ matrix.arch }}-devcontainer + path: | + scripts/artifacts/images/flatcar_developer_container* + + - name: Upload torcx tarball + uses: actions/upload-artifact@v3 + with: + retention-days: 7 + name: ${{ matrix.arch }}-torcx + path: | + scripts/torcx.tar + + # Clean up what we uploaded already so the "vendor images" wildcard + # works when uploading artifacts in the next step. + - name: Remove update, generic and devcontainer images + shell: bash + run: | + set -euo pipefail + set -x + rm -f artifacts/images/flatcar_test_update.gz \ + artifacts/images/flatcar_production_image* \ + artifacts/images/flatcar_developer_container* \ + artifacts/images/flatcar_production_update* + + - name: Upload vendor images + uses: actions/upload-artifact@v3 + with: + retention-days: 7 + name: ${{ matrix.arch }}-vm-images + path: | + scripts/artifacts/images/*.img.bz2 + scripts/artifacts/images/*.bin.bz2 + scripts/artifacts/images/flatcar_production_*_efi_*.fd + scripts/artifacts/images/*.txt + scripts/artifacts/images/flatcar_production_*.sh test: needs: packages From c6a15a41a94ea002cf1a311069b80a1d678b6204 Mon Sep 17 00:00:00 2001 From: Thilo Fromm Date: Thu, 23 Mar 2023 17:39:52 +0100 Subject: [PATCH 02/10] run-kola-tests.yaml: use new artifacts, local web server This change updates the github actions kola test runner workflow to use the new, separated artifacts produced by ci.yaml. Further, it adds a fix for the devcontainer tests. Devcontainer and bin packages used in the devcontainer tests are now served from a local temporary web server. The change also adds the qemu_update test and provides the respective update payload. Lastly, the tests now use a local torcx_manifest.json produced by ci.yaml, which points to a torcx tarball also served by the local temporary web server. --- .github/workflows/run-kola-tests.yaml | 152 ++++++++++++++++++-- ci-automation/ci-config.env | 11 ++ ci-automation/vendor-testing/qemu.sh | 13 +- ci-automation/vendor-testing/qemu_update.sh | 10 +- 4 files changed, 166 insertions(+), 20 deletions(-) diff --git a/.github/workflows/run-kola-tests.yaml b/.github/workflows/run-kola-tests.yaml index 1e636dc77c..9f57223c13 100644 --- a/.github/workflows/run-kola-tests.yaml +++ b/.github/workflows/run-kola-tests.yaml @@ -34,7 +34,7 @@ jobs: run: | sudo rm /bin/sh sudo ln -s /bin/bash /bin/sh - sudo apt-get install -y ca-certificates curl gnupg lsb-release qemu-system git bzip2 jq dnsmasq + sudo apt-get install -y ca-certificates curl gnupg lsb-release qemu-system git bzip2 jq dnsmasq python3 sudo systemctl stop dnsmasq sudo systemctl mask dnsmasq @@ -60,45 +60,156 @@ jobs: fetch-depth: 0 submodules: true - - name: Download artifact + - name: Download binpkgs if: ${{ !inputs.workflow_run_id }} uses: actions/download-artifact@v3 with: - name: images-${{ matrix.arch }} + name: ${{ matrix.arch }}-binpkgs - - name: Download artifacts from other workflow + - name: Download test update image + if: ${{ !inputs.workflow_run_id }} + uses: actions/download-artifact@v3 + with: + name: ${{ matrix.arch }}-test-update + + - name: Download generic image + if: ${{ !inputs.workflow_run_id }} + uses: actions/download-artifact@v3 + with: + name: ${{ matrix.arch }}-generic-image + + - name: Download developer container + if: ${{ !inputs.workflow_run_id }} + uses: actions/download-artifact@v3 + with: + name: ${{ matrix.arch }}-devcontainer + + - name: Download torcx tarball + if: ${{ !inputs.workflow_run_id }} + uses: actions/download-artifact@v3 + with: + name: ${{ matrix.arch }}-torcx + + - name: Download binpkgs from other workflow uses: gabriel-samfira/action-download-artifact@v5 if: ${{ inputs.workflow_run_id }} with: workflow: ${{ inputs.workflow_name_or_id }} workflow_conclusion: success run_id: ${{ inputs.workflow_run_id }} - name: images-${{ matrix.arch }} + name: ${{ matrix.arch }}-binpkgs + + - name: Download test update image from other workflow + uses: gabriel-samfira/action-download-artifact@v5 + if: ${{ inputs.workflow_run_id }} + with: + workflow: ${{ inputs.workflow_name_or_id }} + workflow_conclusion: success + run_id: ${{ inputs.workflow_run_id }} + name: ${{ matrix.arch }}-test-update + + - name: Download generic image from other workflow + uses: gabriel-samfira/action-download-artifact@v5 + if: ${{ inputs.workflow_run_id }} + with: + workflow: ${{ inputs.workflow_name_or_id }} + workflow_conclusion: success + run_id: ${{ inputs.workflow_run_id }} + name: ${{ matrix.arch }}-generic-image + + - name: Download developer container from other workflow + uses: gabriel-samfira/action-download-artifact@v5 + if: ${{ inputs.workflow_run_id }} + with: + workflow: ${{ inputs.workflow_name_or_id }} + workflow_conclusion: success + run_id: ${{ inputs.workflow_run_id }} + name: ${{ matrix.arch }}-devcontainer + + - name: Download torcx tarball from other workflow + uses: gabriel-samfira/action-download-artifact@v5 + if: ${{ inputs.workflow_run_id }} + with: + workflow: ${{ inputs.workflow_name_or_id }} + workflow_conclusion: success + run_id: ${{ inputs.workflow_run_id }} + name: ${{ matrix.arch }}-torcx + + - name: Extract artifacts + shell: bash + run: | + exec 2>&1 + set -x + set -euo pipefail + + # Set up a webserver for devcontainer and torcx tests. + # The respective tests will download devcontainer and torcx tarball via http. + # The devcontainer test will then run a build + # which will download and install binpkgs into the dev container. + # For the sake of that test we will serve both via a temporary local web server. + TESTS_WEBSERVER_WEBROOT="scripts/devcontainer-webroot" + default_rout_device="$(sudo ip -j route sh default |jq -r .[0].dev)" + TESTS_WEBSERVER_IP="$(sudo ip -j address show dev "${default_rout_device}" | jq -r .[0].addr_info[0].local)" + TESTS_WEBSERVER_PORT=12345 + echo "TESTS_WEBSERVER_WEBROOT=${TESTS_WEBSERVER_WEBROOT}" >> "$GITHUB_ENV" + echo "TESTS_WEBSERVER_IP=${TESTS_WEBSERVER_IP}" >> "$GITHUB_ENV" + echo "TESTS_WEBSERVER_PORT=${TESTS_WEBSERVER_PORT}" >> "$GITHUB_ENV" + + mkdir ${TESTS_WEBSERVER_WEBROOT} + mv flatcar_developer_container* ${TESTS_WEBSERVER_WEBROOT} + tar -C ${TESTS_WEBSERVER_WEBROOT} -xvf binpkgs.tar + + tar -C ${TESTS_WEBSERVER_WEBROOT} -xvf torcx.tar + + # Move torcx package into plain webroot + # (path consists of ///:.torcx.tar.gz) + mv "${TESTS_WEBSERVER_WEBROOT}/${{ matrix.arch }}-usr"/*/*/*.torcx.tgz \ + "${TESTS_WEBSERVER_WEBROOT}" + + # Update torcx.json's http URL to point to the webserver IP. + # ci.yaml defines the "localhost" placeholder in its "Set Environment" step. + sed -i "s,http://localhost:12345,http://${TESTS_WEBSERVER_IP}:${TESTS_WEBSERVER_PORT}," \ + "${TESTS_WEBSERVER_WEBROOT}/torcx_manifest.json" + cat "${TESTS_WEBSERVER_WEBROOT}/torcx_manifest.json" + + # Extract the generic image we'll use for qemu tests. + # Note that the qemu[_uefi] tests use the generic image instead of the + # qemu vendor VM image ("Astronaut: [...] Always have been."). + bzip2 --decompress flatcar_production_image.bin.bz2 + mv flatcar_production_image.bin flatcar_production_qemu_uefi_efi_code.fd scripts/ + + mv flatcar_test_update.gz scripts/ + - name: Run tests shell: bash run: | exec 2>&1 - set +x + set -x set -euo pipefail - # extract the image. - IMG_ARCHIVE=$(readlink -f images/**/flatcar_production_image.bin.bz2) - QEMU_UEFI_BIOS_FILE=$(readlink -f images/**/flatcar_production_qemu_uefi_efi_code.fd) - bzip2 --decompress ${IMG_ARCHIVE} - - cp ${IMG_ARCHIVE%%.bz2} ./scripts/ - cp ${QEMU_UEFI_BIOS_FILE} ./scripts/ + python3 -m http.server -d "${TESTS_WEBSERVER_WEBROOT}" -b "${TESTS_WEBSERVER_IP}" "${TESTS_WEBSERVER_PORT}" & pushd scripts source ci-automation/test.sh + # Provide our own torcx prepare function so we use our local manifest json. + # This is called by test_run below. + function __prepare_torcx() { + shift; shift # no need for arch or vernum + local destdir="$1" + cp "../${TESTS_WEBSERVER_WEBROOT}/torcx_manifest.json" "${destdir}" + } + PARALLEL_ARCH=10 cat > sdk_container/.env < Date: Thu, 23 Mar 2023 20:19:18 +0100 Subject: [PATCH 03/10] ci.yaml: run only when ready and / or review requested --- .github/workflows/ci.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 58d01dd4db..06f17bb054 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,6 +1,9 @@ name: "Run build" on: pull_request: + # Run when the PR is ready and each time a review is re-requested + # (i.e. after feedback has been addressed). + types: [review_requested, ready_for_review] workflow_dispatch: inputs: image_formats: @@ -35,6 +38,8 @@ permissions: {} jobs: packages: + # Do not run when still in draft mode but a review was requested anyway + if: github.event.pull_request.draft == false name: "Build Flatcar packages" runs-on: - self-hosted From b8a7333f448146d2e1bd6868d5bbd27c99c372a4 Mon Sep 17 00:00:00 2001 From: Thilo Fromm Date: Fri, 24 Mar 2023 07:35:20 +0100 Subject: [PATCH 04/10] dispatch-kola-tests.yaml: Elaborated comments on inputs --- .github/workflows/dispatch-kola-tests.yaml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dispatch-kola-tests.yaml b/.github/workflows/dispatch-kola-tests.yaml index a104c7c7ab..c970711d30 100644 --- a/.github/workflows/dispatch-kola-tests.yaml +++ b/.github/workflows/dispatch-kola-tests.yaml @@ -8,12 +8,15 @@ on: required: true default: ci.yaml description: | - The workflow ID from where we'll download the artifacts to be tested. + The workflow name or ID from where we'll download the artifacts to be tested. + E.g. the name of the YAML file (w/o path) of the respective workflow. workflow_run_id: type: string required: true description: | - The run ID of the workflow specified in workflow_name_or_id + The run ID of the workflow specified in workflow_name_or_id. + You can e.g. get this from a run's URL - + https://github.com/flatcar/scripts/actions/runs/ . permissions: {} From f1e3b305371dab1db08cc1c34353d6823eb078fb Mon Sep 17 00:00:00 2001 From: Thilo Fromm Date: Mon, 27 Mar 2023 09:35:05 +0200 Subject: [PATCH 05/10] ci.yaml: extract + upload build logs also on failure Signed-off-by: Thilo Fromm --- .github/workflows/ci.yaml | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 06f17bb054..0a27cf1a6d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -151,6 +151,7 @@ jobs: sdk_image="$(docker_image_fullname "${sdk_name}" "${docker_sdk_vernum}")" container_name="flatcar-ci-build" + echo "container_name=${container_name}" >> "$GITHUB_ENV" # Create version file ( @@ -166,19 +167,24 @@ jobs: --torcx_output_root="${CI_CONTAINER_TORCX_ROOT}" \ --torcx_extra_pkg_url="${TORCX_TESTS_PACKAGE_URL}" - # Copy logs - ./run_sdk_container -n "${container_name}" \ - tar -cJf ebuild_logs.tar.xz /build/${arch}-usr/var/log/portage \ - /build/${arch}-usr/var/tmp/portage - # Create binpkgs tarball for archiving as artifact later ./run_sdk_container -n "${container_name}" \ tar -C "/build/${arch}-usr/var/lib/portage/pkgs/" \ -cvf binpkgs.tar . - echo "container_name=${container_name}" >> "$GITHUB_ENV" + - name: Extract build logs + if: always() + shell: bash + run: | + set -euo pipefail + set -x + # Copy logs + ./run_sdk_container -n "${container_name}" \ + tar -cJf ebuild_logs.tar.xz /build/${arch}-usr/var/log/portage \ + /build/${arch}-usr/var/tmp/portage - name: Upload build logs + if: always() uses: actions/upload-artifact@v3 with: retention-days: 7 From bed3bf87d4e9a3df7665ee2bd975add9b46db0e6 Mon Sep 17 00:00:00 2001 From: Thilo Fromm Date: Mon, 27 Mar 2023 18:14:33 +0200 Subject: [PATCH 06/10] .yaml: use HEAD commit ref for PR builds Signed-off-by: Thilo Fromm --- .github/workflows/ci.yaml | 15 +++++++++++++++ .github/workflows/run-kola-tests.yaml | 16 ++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 0a27cf1a6d..a36e9df6de 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -76,6 +76,21 @@ jobs: fetch-depth: 0 submodules: true + # Hack alert: actions/checkout will check out the (disjunct) merge commit of a PR + # instead of its head commit. That commit is not connected to any branch. + # This causes breakage downstream e.g. when the devcontainer test wants to check out + # the ref in the scripts repo that corresponds to this build. + - name: If this is a PR build, use head commit instead of the merge commit + if: ${{ github.event.pull_request.head.sha }} + shell: bash + run: | + exec 2>&1 + set -x + set -euo pipefail + + git checkout ${{ github.event.pull_request.head.sha }} + git submodule update + - name: Set environment shell: bash run: | diff --git a/.github/workflows/run-kola-tests.yaml b/.github/workflows/run-kola-tests.yaml index 9f57223c13..482afda07b 100644 --- a/.github/workflows/run-kola-tests.yaml +++ b/.github/workflows/run-kola-tests.yaml @@ -60,6 +60,22 @@ jobs: fetch-depth: 0 submodules: true + # Hack alert: actions/checkout will check out the (disjunct) merge commit of a PR + # instead of its head commit. That commit is not connected to any branch. + # This is not technically necessary for the tests run but it is done to remain aligned + # with the ref. + - name: If this is a PR build, use head commit instead of the merge commit + if: ${{ github.event.pull_request.head.sha }} + shell: bash + run: | + exec 2>&1 + set -x + set -euo pipefail + + cd scripts + git checkout ${{ github.event.pull_request.head.sha }} + git submodule update + - name: Download binpkgs if: ${{ !inputs.workflow_run_id }} uses: actions/download-artifact@v3 From c4034d9a65079dcf96457d7689185ee6df051c9b Mon Sep 17 00:00:00 2001 From: Thilo Fromm Date: Tue, 28 Mar 2023 10:50:25 +0200 Subject: [PATCH 07/10] run-kola-tests: improve test results archive globs Signed-off-by: Thilo Fromm --- .github/workflows/run-kola-tests.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/run-kola-tests.yaml b/.github/workflows/run-kola-tests.yaml index 482afda07b..55b4097e23 100644 --- a/.github/workflows/run-kola-tests.yaml +++ b/.github/workflows/run-kola-tests.yaml @@ -248,11 +248,11 @@ jobs: with: name: ${{ matrix.arch }}-test-results path: | - scripts/__TESTS__ - scripts/results-.*.tap + scripts/__TESTS__/*/_kola_temp/ + scripts/results-*.tap - name: Create Test Summary if: always() uses: test-summary/action@v2 with: - paths: "scripts/results-.*.tap" + paths: "scripts/results-*.tap" From 138772c4e10ff126bd1975a7c76ae3f778712b76 Mon Sep 17 00:00:00 2001 From: Thilo Fromm Date: Tue, 28 Mar 2023 21:09:11 +0200 Subject: [PATCH 08/10] run-kola-tests.yaml: fix test-summary TAP formatting Signed-off-by: Thilo Fromm --- .github/workflows/run-kola-tests.yaml | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/.github/workflows/run-kola-tests.yaml b/.github/workflows/run-kola-tests.yaml index 55b4097e23..16de959a01 100644 --- a/.github/workflows/run-kola-tests.yaml +++ b/.github/workflows/run-kola-tests.yaml @@ -251,8 +251,26 @@ jobs: scripts/__TESTS__/*/_kola_temp/ scripts/results-*.tap + - name: Patch TAP reports so test-summary can parse + if: always() + shell: bash + run: | + exec 2>&1 + set -x + set -euo pipefail + + cd scripts + for tap in results-*.tap; do + sumtap="test-summary-${tap}" + # If this is missing then test-summary assumes the TAP report + # is, in fact, XML. + # See https://github.com/flatcar/scripts/pull/696#discussion_r1151027499 + echo "TAP version 13" > "${sumtap}" + cat "${tap}" >> "${sumtap}" + done + - name: Create Test Summary if: always() uses: test-summary/action@v2 with: - paths: "scripts/results-*.tap" + paths: "scripts/test-summary-results-*.tap" From e160917ec1e3a0e768da00759d4016fca9af360a Mon Sep 17 00:00:00 2001 From: Thilo Fromm Date: Wed, 29 Mar 2023 12:59:34 +0200 Subject: [PATCH 09/10] tapfile_helper ff.: support TAP and Markdown output This change adds markdown output support to tapfile helper. tap_generate_report() has been refactored to use low-level output functions to write tests; TAP and markdown output is supported and both are generated by default. Also, it should be straightforward to add other output formats by implementing the respective low level print functions. The markdown output is now used by run-kola-tests.yaml to generate step output and, if run from a PR, add a comment with test results to the PR. Signed-off-by: Thilo Fromm --- .github/workflows/ci.yaml | 3 +- .github/workflows/dispatch-kola-tests.yaml | 3 +- .github/workflows/run-kola-tests.yaml | 54 +++---- ci-automation/ci-config.env | 5 + ci-automation/tapfile_helper_lib.sh | 176 ++++++++++++++++++--- ci-automation/test.sh | 8 +- ci-automation/test_update_reruns.sh | 8 +- 7 files changed, 204 insertions(+), 53 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a36e9df6de..2892c66970 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -34,7 +34,8 @@ concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }} cancel-in-progress: true -permissions: {} +permissions: + pull-requests: write jobs: packages: diff --git a/.github/workflows/dispatch-kola-tests.yaml b/.github/workflows/dispatch-kola-tests.yaml index c970711d30..edc9ca96b3 100644 --- a/.github/workflows/dispatch-kola-tests.yaml +++ b/.github/workflows/dispatch-kola-tests.yaml @@ -18,7 +18,8 @@ on: You can e.g. get this from a run's URL - https://github.com/flatcar/scripts/actions/runs/ . -permissions: {} +permissions: + pull-requests: write jobs: test: diff --git a/.github/workflows/run-kola-tests.yaml b/.github/workflows/run-kola-tests.yaml index 16de959a01..caa2510bc5 100644 --- a/.github/workflows/run-kola-tests.yaml +++ b/.github/workflows/run-kola-tests.yaml @@ -26,6 +26,8 @@ jobs: fail-fast: false matrix: arch: ["amd64", "arm64"] + permissions: + pull-requests: write steps: - name: Prepare machine @@ -196,7 +198,6 @@ jobs: mv flatcar_test_update.gz scripts/ - - name: Run tests shell: bash run: | @@ -232,9 +233,8 @@ jobs: export MAX_RETRIES=5 export SKIP_COPY_TO_BINCACHE=1 - # run the test. + # run the tests. test_run ${{ matrix.arch }} qemu_uefi - test_run ${{ matrix.arch }} qemu_update # Stop the background webserver @@ -242,6 +242,26 @@ jobs: kill %1 set -e + - name: Create Test Summary + if: always() + shell: bash + run: | + exec 2>&1 + set -x + set -euo pipefail + + # qemu_update report includes all reqults of qemu_uefi as test results are + # stored in a temporary sqlite DB in scripts/ which is not deleted between test runs. + cp scripts/results-qemu_update.md ./test-results.md + cat test-results.md >> "$GITHUB_STEP_SUMMARY" + + - name: If started from a PR, post test summary to PR + if: ${{ github.event_name == 'pull_request' }} + uses: mshick/add-pr-comment@v2 + with: + if: always() + message-path: "test-results.md" + - name: Upload artifacts if: always() uses: actions/upload-artifact@v3 @@ -249,28 +269,8 @@ jobs: name: ${{ matrix.arch }}-test-results path: | scripts/__TESTS__/*/_kola_temp/ + scripts/__TESTS__/*/*.tap + scripts/__TESTS__/*/*.txt scripts/results-*.tap - - - name: Patch TAP reports so test-summary can parse - if: always() - shell: bash - run: | - exec 2>&1 - set -x - set -euo pipefail - - cd scripts - for tap in results-*.tap; do - sumtap="test-summary-${tap}" - # If this is missing then test-summary assumes the TAP report - # is, in fact, XML. - # See https://github.com/flatcar/scripts/pull/696#discussion_r1151027499 - echo "TAP version 13" > "${sumtap}" - cat "${tap}" >> "${sumtap}" - done - - - name: Create Test Summary - if: always() - uses: test-summary/action@v2 - with: - paths: "scripts/test-summary-results-*.tap" + scripts/results-*.md + test-results.md diff --git a/ci-automation/ci-config.env b/ci-automation/ci-config.env index bd15bff828..9f0306838a 100644 --- a/ci-automation/ci-config.env +++ b/ci-automation/ci-config.env @@ -47,6 +47,11 @@ CONTAINER_IMAGE_ROOT="/home/sdk/build/images" # echo "export PARALLEL_TESTS=\"5\"" > sdk_container/.env # to override the number of test cases to be run in parallel. +# -- General -- + +# "tap" for TAP reports, "md" for markdown are currently supported +TEST_REPORT_FORMATS=("tap" "md") + # -- QEMU -- QEMU_IMAGE_NAME=${QEMU_IMAGE_NAME:-flatcar_production_image.bin} diff --git a/ci-automation/tapfile_helper_lib.sh b/ci-automation/tapfile_helper_lib.sh index bbb7c6c4c5..8c837d8315 100644 --- a/ci-automation/tapfile_helper_lib.sh +++ b/ci-automation/tapfile_helper_lib.sh @@ -197,27 +197,167 @@ function tap_failed_tests_for_vendor() { } # -- +# TAP output format primitives for tap_generate_report() + +__tap_print_header() { + local arch="$1" + local version="$2" + local vendors="$3" + local count="$4" + + # We use count + 1 here because the very first "test result" will just print + # the list of platforms tested, not an actual test's result. + echo "1..$((count+1))" + echo "ok - Version: ${version}, Architecture: ${arch}" + echo " ---" + echo " Platforms tested: ${vendors}" + echo " ..." +} +# -- + +__tap_print_test_verdict() { + local verdict="$1" + local name="$2" + local succeded_vendors="$3" + local failed_vendors="$4" + + echo "${verdict} - ${test_name}" + echo " ---" + + if [ -n "${succeded_vendors}" ] ; then + echo " Succeeded: ${succeded_vendors}" + fi + if [ -n "${failed_vendors}" ] ; then + echo " Failed: ${failed_vendors}" + fi +} +# -- + +__tap_print_test_run_diag_output() { + local vendor="$1" + local run="$2" + echo " Error messages for ${vendor}, run ${run}:" + cat - +} +# -- + +__tap_finish_test_verdict() { + local verdict="$1" + local name="$2" + local succeded_vendors="$3" + local failed_vendors="$4" + echo " ..." +} +# -- + +__tap_finish_test_report() { + true +} +# -- + +# markdown output format primitives for tap_generate_report() + +__md_print_header() { + local arch="$1" + local version="$2" + local vendors="$3" + local count="$4" + + echo "### Test report for ${version} / ${arch}" + echo + echo "**Platforms tested** : ${vendors}" +} +# -- + +__md_print_test_verdict() { + local verdict="$1" + local name="$2" + local succeded_vendors="$3" + local failed_vendors="$4" + + v="![${verdict}](https://via.placeholder.com/50x20/00ff00/000000?text=PASS)" + if [ "${verdict}" = "not ok" ] ; then + v="![${verdict}](https://via.placeholder.com/50x20/ff0000/ffffff?text=FAIL)" + fi + + echo + echo -n "${v} **${name}**" + if [ -n "${succeded_vendors}" ] ; then + echo -n " 🟢 Succeeded: ${succeded_vendors}" + fi + if [ -n "${failed_vendors}" ] ; then + echo -n " ❌ Failed: ${failed_vendors}" + fi + echo + if [ "${verdict}" = "not ok" ] ; then + echo + echo "
" + echo + fi +} +# -- + +__md_print_test_run_diag_output() { + local vendor="$1" + local run="$2" + + echo "* Diagnostic output for ${vendor}, run ${run}" + echo + echo " \`\`\`" + cat - + echo " \`\`\`" + echo + +} +# -- +# +__md_finish_test_verdict() { + local verdict="$1" + local name="$2" + local succeded_vendors="$3" + local failed_vendors="$4" + if [ "${verdict}" = "not ok" ] ; then + echo + echo "
" + echo + fi +} +# -- + +__md_finish_test_report() { + true +} +# -- + + # Print the tap file from contents of the database. # INPUT: # 1: - Architecture to be included in the first line of the report # 2: - OS version tested, to be included in the first line of the report -# 3: - If set to "true" then debug output of transient test failures +# 3: - Output format of the report. "tap" and "markdown" are supported. +# 4: - If set to "true" then debug output of transient test failures # is included in the result report. function tap_generate_report() { local arch="$1" local version="$2" - local full_error_report="${3:-false}" + local format="$3" + local full_error_report="${4:-false}" + + case "${format}" in + tap) ;; + md) ;; + *) echo "ERROR: tap_generate_report() unknown format '${format}'" >&2 + return 1 + ;; + esac + local count count="$(__sqlite3_wrapper 'SELECT count(name) FROM test_case;')" local vendors vendors="$(__sqlite3_wrapper 'SELECT name FROM vendor;' | tr '\n' ' ')" - echo "1..$((count+1))" - echo "ok - Version: ${version}, Architecture: ${arch}" - echo " ---" - echo " Platforms tested: ${vendors}" - echo " ..." + __"${format}"_print_header "${arch}" "${version}" "${vendors}" "${count}" # Print result line for every test, including platforms it succeeded on # and transient failed runs. @@ -265,21 +405,17 @@ function tap_generate_report() { r=r ", " $2 else r="(" $2 ; } - END { if (t) print t r ")"; }' + END { if (t) print t " " r ")"; }' } - local succeded - succeded="$(list_runs 1)" + local succeeded + succeeded="$(list_runs 1)" local failed failed="$(list_runs 0)" - echo "${verdict} - ${test_name}" - echo " ---" - if [ -n "${succeded}" ] ; then - echo " Succeeded: ${succeded}" - fi + __"${format}"_print_test_verdict "${verdict}" "${test_name}" \ + "${succeeded}" "${failed}" if [ -n "${failed}" ] ; then - echo " Failed: ${failed}" if [ "${verdict}" = "not ok" -o "${full_error_report}" = "true" ] ; then # generate diagnostic output, per failed run. __sqlite3_wrapper -csv " @@ -291,7 +427,7 @@ function tap_generate_report() { ORDER BY t.run DESC;" | \ sed 's/,/ /' | \ while read -r vendor run; do - echo " Error messages for ${vendor}, run ${run}:" + { __sqlite3_wrapper -csv " SELECT t.output FROM test_run AS t, test_case AS c WHERE t.case_id=c.id @@ -299,10 +435,14 @@ function tap_generate_report() { AND t.run='${run}';" | \ sed 's/"/ /g' | \ awk '{print " L" NR ": \"" $0 "\""}' + } | __"${format}"_print_test_run_diag_output "${vendor}" "${run}" done fi fi - echo " ..." + __"${format}"_finish_test_verdict "${verdict}" "${test_name}" \ + "${succeeded}" "${failed}" done + + __"${format}"_finish_test_report } # -- diff --git a/ci-automation/test.sh b/ci-automation/test.sh index c9ec153a3d..2082aa83dd 100644 --- a/ci-automation/test.sh +++ b/ci-automation/test.sh @@ -165,8 +165,8 @@ function _test_run_impl() { # Make the torcx artifacts available to test implementation __prepare_torcx "${arch}" "${vernum}" "${work_dir}" - local tap_merged_summary="results-${image}.tap" - local tap_merged_detailed="results-${image}-detailed.tap" + local tap_merged_summary="results-${image}" + local tap_merged_detailed="results-${image}-detailed" local retry="" local success=false local print_give_up=true @@ -242,9 +242,9 @@ function _test_run_impl() { copy_to_buildcache "testing/${vernum}/${arch}/${image}" \ "${tests_dir}/"*.tap copy_to_buildcache "testing/${vernum}/${arch}/${image}" \ - "${tap_merged_summary}" + "${tap_merged_summary}"* copy_to_buildcache "testing/${vernum}/${arch}/${image}" \ - "${tap_merged_detailed}" + "${tap_merged_detailed}"* fi if ! $success; then return 1 diff --git a/ci-automation/test_update_reruns.sh b/ci-automation/test_update_reruns.sh index 2fcd79e543..0cd91f9ad4 100755 --- a/ci-automation/test_update_reruns.sh +++ b/ci-automation/test_update_reruns.sh @@ -19,8 +19,12 @@ failfile="$6" merged_summary="$7" merged_detailed="$8" +source ci-automation/ci-config.env source ci-automation/tapfile_helper_lib.sh tap_ingest_tapfile "${tapfile}" "${image}" "${retry}" tap_failed_tests_for_vendor "${image}" > "${failfile}" -tap_generate_report "${arch}" "${vernum}" > "${merged_summary}" -tap_generate_report "${arch}" "${vernum}" "true" > "${merged_detailed}" + +for format in "${TEST_REPORT_FORMATS[@]}"; do + tap_generate_report "${arch}" "${vernum}" "${format}" > "${merged_summary}.${format}" + tap_generate_report "${arch}" "${vernum}" "${format}" "true" > "${merged_detailed}.${format}" +done From 7cab408b5df4055da42ad7009dfa5f8a12c85cc6 Mon Sep 17 00:00:00 2001 From: Thilo Fromm Date: Thu, 30 Mar 2023 09:46:08 +0200 Subject: [PATCH 10/10] run-kola-tests.yaml: test report merge job Signed-off-by: Thilo Fromm --- .github/workflows/run-kola-tests.yaml | 130 +++++++++++++++++++++----- 1 file changed, 105 insertions(+), 25 deletions(-) diff --git a/.github/workflows/run-kola-tests.yaml b/.github/workflows/run-kola-tests.yaml index caa2510bc5..c62994c61b 100644 --- a/.github/workflows/run-kola-tests.yaml +++ b/.github/workflows/run-kola-tests.yaml @@ -26,8 +26,6 @@ jobs: fail-fast: false matrix: arch: ["amd64", "arm64"] - permissions: - pull-requests: write steps: - name: Prepare machine @@ -242,35 +240,117 @@ jobs: kill %1 set -e - - name: Create Test Summary - if: always() - shell: bash - run: | - exec 2>&1 - set -x - set -euo pipefail - - # qemu_update report includes all reqults of qemu_uefi as test results are - # stored in a temporary sqlite DB in scripts/ which is not deleted between test runs. - cp scripts/results-qemu_update.md ./test-results.md - cat test-results.md >> "$GITHUB_STEP_SUMMARY" - - - name: If started from a PR, post test summary to PR - if: ${{ github.event_name == 'pull_request' }} - uses: mshick/add-pr-comment@v2 - with: - if: always() - message-path: "test-results.md" - - - name: Upload artifacts + - name: Upload detailed test logs if: always() uses: actions/upload-artifact@v3 with: - name: ${{ matrix.arch }}-test-results + name: ${{ matrix.arch }}-test-logs-and-results path: | scripts/__TESTS__/*/_kola_temp/ scripts/__TESTS__/*/*.tap scripts/__TESTS__/*/*.txt scripts/results-*.tap scripts/results-*.md - test-results.md + + - name: Upload raw TAP files of all runs for later merging + if: always() + uses: actions/upload-artifact@v3 + with: + name: ${{ matrix.arch }}-raw-tapfiles + path: | + scripts/__TESTS__/*/*.tap + + + merge_and_publish_results: + name: "Merge TAP reports and post results" + needs: tests + if: always() + runs-on: + - self-hosted + - debian + - kola + permissions: + pull-requests: write + + steps: + - name: Prepare machine + shell: bash + working-directory: ${{ github.workspace }} + run: | + sudo rm /bin/sh + sudo ln -s /bin/bash /bin/sh + sudo apt-get install -y ca-certificates curl gnupg lsb-release git bzip2 jq sqlite3 + + - uses: actions/checkout@v3 + with: + path: scripts + fetch-depth: 0 + submodules: true + + # Hack alert: actions/checkout will check out the (disjunct) merge commit of a PR + # instead of its head commit. That commit is not connected to any branch. + # This is not technically necessary for the tests run but it is done to remain aligned + # with the ref. + - name: If this is a PR build, use head commit instead of the merge commit + if: ${{ github.event.pull_request.head.sha }} + shell: bash + run: | + exec 2>&1 + set -x + set -euo pipefail + + cd scripts + git checkout ${{ github.event.pull_request.head.sha }} + git submodule update + + # This is clunky. Haven't figured out how to re-use matrix.arch here for downloads, + # so we download each arch individually. + - name: Download amd64 tapfiles + uses: actions/download-artifact@v3 + with: + name: amd64-raw-tapfiles + path: scripts/__TAP__/amd64 + + - name: Download arm64 tapfiles + uses: actions/download-artifact@v3 + with: + name: arm64-raw-tapfiles + path: scripts/__TAP__/arm64 + + - name: Create Test Summary + shell: bash + run: | + exec 2>&1 + set -x + set -euo pipefail + + cd scripts + + ls -laR __TAP__ + + source ci-automation/tapfile_helper_lib.sh + + all_archs="" + for arch in __TAP__/*; do + arch_name="$(basename "${arch}")" + all_archs="${all_archs} ${arch_name}" + for vendor in "${arch}"/*; do + vendor_name="$(basename "${vendor}")" + run=1 + for tap in "${vendor}"/*.tap; do + tap_ingest_tapfile "${tap}" "${vendor_name}-${arch_name}" "${run}" + ((run++)) + done + done + done + + source sdk_container/.repo/manifests/version.txt + tap_generate_report "${all_archs}" "${FLATCAR_VERSION}" "md" "true" > test-results.md + + cat test-results.md >> "$GITHUB_STEP_SUMMARY" + + - name: If started from a PR, post test summary to PR + if: ${{ github.event_name == 'pull_request' }} + uses: mshick/add-pr-comment@v2 + with: + message-path: "test-results.md"