run-kola-tests.yaml: use new artifacts, local web server

This change updates the github actions kola test runner workflow to use
the new, separated artifacts produced by ci.yaml.

Further, it adds a fix for the devcontainer tests. Devcontainer and bin
packages used in the devcontainer tests are now served from a local
temporary web server.

The change also adds the qemu_update test and provides the respective
update payload.

Lastly, the tests now use a local torcx_manifest.json produced by
ci.yaml, which points to a torcx tarball also served by the local
temporary web server.

.github/workflows/run-kola-tests.yaml

@@ -34,7 +34,7 @@ jobs:
         run: |
           sudo rm /bin/sh
           sudo ln -s /bin/bash /bin/sh
-          sudo apt-get install -y ca-certificates curl gnupg lsb-release qemu-system git bzip2 jq dnsmasq
+          sudo apt-get install -y ca-certificates curl gnupg lsb-release qemu-system git bzip2 jq dnsmasq python3
           sudo systemctl stop dnsmasq
           sudo systemctl mask dnsmasq
 
@@ -60,45 +60,156 @@ jobs:
           fetch-depth: 0
           submodules: true
 
-      - name: Download artifact
+      - name: Download binpkgs
         if: ${{ !inputs.workflow_run_id }}
         uses: actions/download-artifact@v3
         with:
-          name: images-${{ matrix.arch }}
+          name: ${{ matrix.arch }}-binpkgs
 
-      - name: Download artifacts from other workflow
+      - name: Download test update image
+        if: ${{ !inputs.workflow_run_id }}
+        uses: actions/download-artifact@v3
+        with:
+          name: ${{ matrix.arch }}-test-update
+
+      - name: Download generic image
+        if: ${{ !inputs.workflow_run_id }}
+        uses: actions/download-artifact@v3
+        with:
+          name: ${{ matrix.arch }}-generic-image
+
+      - name: Download developer container
+        if: ${{ !inputs.workflow_run_id }}
+        uses: actions/download-artifact@v3
+        with:
+          name: ${{ matrix.arch }}-devcontainer
+
+      - name: Download torcx tarball
+        if: ${{ !inputs.workflow_run_id }}
+        uses: actions/download-artifact@v3
+        with:
+          name: ${{ matrix.arch }}-torcx
+
+      - name: Download binpkgs from other workflow
         uses: gabriel-samfira/action-download-artifact@v5
         if: ${{ inputs.workflow_run_id }}
         with:
           workflow: ${{ inputs.workflow_name_or_id }}
           workflow_conclusion: success
           run_id: ${{ inputs.workflow_run_id }}
-          name: images-${{ matrix.arch }}
+          name: ${{ matrix.arch }}-binpkgs
+
+      - name: Download test update image from other workflow
+        uses: gabriel-samfira/action-download-artifact@v5
+        if: ${{ inputs.workflow_run_id }}
+        with:
+          workflow: ${{ inputs.workflow_name_or_id }}
+          workflow_conclusion: success
+          run_id: ${{ inputs.workflow_run_id }}
+          name: ${{ matrix.arch }}-test-update
+
+      - name: Download generic image from other workflow
+        uses: gabriel-samfira/action-download-artifact@v5
+        if: ${{ inputs.workflow_run_id }}
+        with:
+          workflow: ${{ inputs.workflow_name_or_id }}
+          workflow_conclusion: success
+          run_id: ${{ inputs.workflow_run_id }}
+          name: ${{ matrix.arch }}-generic-image
+
+      - name: Download developer container from other workflow
+        uses: gabriel-samfira/action-download-artifact@v5
+        if: ${{ inputs.workflow_run_id }}
+        with:
+          workflow: ${{ inputs.workflow_name_or_id }}
+          workflow_conclusion: success
+          run_id: ${{ inputs.workflow_run_id }}
+          name: ${{ matrix.arch }}-devcontainer
+
+      - name: Download torcx tarball from other workflow
+        uses: gabriel-samfira/action-download-artifact@v5
+        if: ${{ inputs.workflow_run_id }}
+        with:
+          workflow: ${{ inputs.workflow_name_or_id }}
+          workflow_conclusion: success
+          run_id: ${{ inputs.workflow_run_id }}
+          name: ${{ matrix.arch }}-torcx
+
+      - name: Extract artifacts
+        shell: bash
+        run: |
+          exec 2>&1
+          set -x
+          set -euo pipefail
+
+          # Set up a webserver for devcontainer and torcx tests.
+          # The respective tests will download devcontainer and torcx tarball via http.
+          # The devcontainer test will then run a build
+          #   which will download and install binpkgs into the dev container.
+          # For the sake of that test we will serve both via a temporary local web server.
+          TESTS_WEBSERVER_WEBROOT="scripts/devcontainer-webroot"
+          default_rout_device="$(sudo ip -j route sh default |jq -r .[0].dev)"
+          TESTS_WEBSERVER_IP="$(sudo ip -j address show dev "${default_rout_device}" | jq -r .[0].addr_info[0].local)"
+          TESTS_WEBSERVER_PORT=12345
+          echo "TESTS_WEBSERVER_WEBROOT=${TESTS_WEBSERVER_WEBROOT}" >> "$GITHUB_ENV"
+          echo "TESTS_WEBSERVER_IP=${TESTS_WEBSERVER_IP}" >> "$GITHUB_ENV"
+          echo "TESTS_WEBSERVER_PORT=${TESTS_WEBSERVER_PORT}" >> "$GITHUB_ENV"
+
+          mkdir ${TESTS_WEBSERVER_WEBROOT}
+          mv flatcar_developer_container* ${TESTS_WEBSERVER_WEBROOT}
+          tar -C ${TESTS_WEBSERVER_WEBROOT} -xvf binpkgs.tar
+
+          tar -C ${TESTS_WEBSERVER_WEBROOT} -xvf torcx.tar
+
+          # Move torcx package into plain webroot
+          #  (path consists of <arch>/<packagename>/<checksum>/<packagename>:<version>.torcx.tar.gz)
+          mv "${TESTS_WEBSERVER_WEBROOT}/${{ matrix.arch }}-usr"/*/*/*.torcx.tgz \
+             "${TESTS_WEBSERVER_WEBROOT}"
+
+          # Update torcx.json's http URL to point to the webserver IP.
+          # ci.yaml defines the "localhost" placeholder in its "Set Environment" step.
+          sed -i "s,http://localhost:12345,http://${TESTS_WEBSERVER_IP}:${TESTS_WEBSERVER_PORT}," \
+                 "${TESTS_WEBSERVER_WEBROOT}/torcx_manifest.json"
+          cat "${TESTS_WEBSERVER_WEBROOT}/torcx_manifest.json"
+
+          # Extract the generic image we'll use for qemu tests.
+          # Note that the qemu[_uefi] tests use the generic image instead of the
+          #  qemu vendor VM image ("Astronaut: [...] Always have been.").
+          bzip2 --decompress flatcar_production_image.bin.bz2
+          mv flatcar_production_image.bin flatcar_production_qemu_uefi_efi_code.fd scripts/
+
+          mv flatcar_test_update.gz scripts/
+
 
       - name: Run tests
         shell: bash
         run: |
           exec 2>&1
-          set +x
+          set -x
           set -euo pipefail
 
-          # extract the image.
+          python3 -m http.server -d "${TESTS_WEBSERVER_WEBROOT}" -b "${TESTS_WEBSERVER_IP}" "${TESTS_WEBSERVER_PORT}" &
-          IMG_ARCHIVE=$(readlink -f images/**/flatcar_production_image.bin.bz2)
-          QEMU_UEFI_BIOS_FILE=$(readlink -f images/**/flatcar_production_qemu_uefi_efi_code.fd)
-          bzip2 --decompress ${IMG_ARCHIVE}
-
-          cp ${IMG_ARCHIVE%%.bz2} ./scripts/
-          cp ${QEMU_UEFI_BIOS_FILE} ./scripts/
 
           pushd scripts
           source ci-automation/test.sh
 
+          # Provide our own torcx prepare function so we use our local manifest json.
+          # This is called by test_run below.
+          function __prepare_torcx() {
+            shift; shift # no need for arch or vernum
+            local destdir="$1"
+            cp "../${TESTS_WEBSERVER_WEBROOT}/torcx_manifest.json" "${destdir}"
+          }
+
           PARALLEL_ARCH=10
 
           cat > sdk_container/.env <<EOF
           # export the QEMU_IMAGE_NAME to avoid to download it.
           export QEMU_IMAGE_NAME="/work/flatcar_production_image.bin"
           export QEMU_UEFI_BIOS="/work/flatcar_production_qemu_uefi_efi_code.fd"
+          export QEMU_UPDATE_PAYLOAD="/work/flatcar_test_update.gz"
+          export QEMU_DEVCONTAINER_URL="http://${TESTS_WEBSERVER_IP}:${TESTS_WEBSERVER_PORT}"
+          export QEMU_DEVCONTAINER_BINHOST_URL="http://${TESTS_WEBSERVER_IP}:${TESTS_WEBSERVER_PORT}"
           export PARALLEL_TESTS=${PARALLEL_ARCH}
           # The runner uses lxc containers for kola, and can't use loopback devices to
           # prepare the serial console setting - this means that kola may miss some errors
@@ -111,11 +222,24 @@ jobs:
           # run the test.
           test_run ${{ matrix.arch }} qemu_uefi
 
+          test_run ${{ matrix.arch }} qemu_update
+
+          # Stop the background webserver
+          set +e
+          kill %1
+          set -e
+
       - name: Upload artifacts
         if: always()
         uses: actions/upload-artifact@v3
         with:
-          name: test-results-${{ matrix.arch }}
+          name: ${{ matrix.arch }}-test-results
           path: |
             scripts/__TESTS__
             scripts/results-.*.tap
+
+      - name: Create Test Summary
+        if: always()
+        uses: test-summary/action@v2
+        with:
+          paths: "scripts/results-.*.tap"

ci-automation/ci-config.env

@@ -62,6 +62,17 @@ QEMU_BIOS="/usr/share/qemu/bios-256k.bin"
 # Published by vms.sh as part of the qemu vendor build.
 QEMU_UEFI_BIOS="${QEMU_UEFI_BIOS:-flatcar_production_qemu_uefi_efi_code.fd}"
 
+# Update payload for the qemu_update.sh test.
+# The default path set below is relative to TEST_WORK_DIR
+QEMU_UPDATE_PAYLOAD="tmp/flatcar_test_update.gz"
+
+# Devcontainer settings for isolated / local testing w/o a remote
+#  devcontainer server and/or binhost.
+# Can optionally be set to http / https URLs to pull the dev container
+#  and the binpackages from.
+QEMU_DEVCONTAINER_URL="${QEMU_DEVCONTAINER_URL:-}"
+QEMU_DEVCONTAINER_BINHOST_URL="${QEMU_DEVCONTAINER_BINHOST_URL:-}"
+
 
 # -- Equinix Metal --
 EQUINIXMETAL_PARALLEL="${PARALLEL_TESTS:-4}"

ci-automation/vendor-testing/qemu.sh

@@ -42,6 +42,16 @@ if [ "${CIA_TESTSCRIPT}" = "qemu_uefi.sh" ] ; then
     fi
 fi
 
+declare -a devcontainer_opts
+if [ -n "${QEMU_DEVCONTAINER_URL}" ] ; then
+    echo "++++ Using custom devcontainer URL '${QEMU_DEVCONTAINER_URL}'"
+    devcontainer_opts+=( "--devcontainer-url" "${QEMU_DEVCONTAINER_URL}" )
+fi
+if [ -n "${QEMU_DEVCONTAINER_BINHOST_URL}" ] ; then
+    echo "++++ Using custom devcontainer binhost '${QEMU_DEVCONTAINER_BINHOST_URL}'"
+    devcontainer_opts+=( "--devcontainer-binhost-url" "${QEMU_DEVCONTAINER_BINHOST_URL}" )
+fi
+
 set -x
 
 kola run \
@@ -53,6 +63,7 @@ kola run \
     --tapfile="${CIA_TAPFILE}" \
     --torcx-manifest="${CIA_TORCX_MANIFEST}" \
     ${QEMU_KOLA_SKIP_MANGLE:+--qemu-skip-mangle} \
+    "${devcontainer_opts[@]}" \
     "${@}"
 
 set +x

ci-automation/vendor-testing/qemu_update.sh

@@ -23,9 +23,9 @@ if [ "$*" != "" ] && [ "$*" != "*" ] && [[ "$*" != *"cl.update.payload" ]]; then
     exit 1
 fi
 
-mkdir -p tmp/
+mkdir -p "$(dirname ${QEMU_UPDATE_PAYLOAD})"
-if [ -f tmp/flatcar_test_update.gz ] ; then
+if [ -f "${QEMU_UPDATE_PAYLOAD}" ] ; then
-    echo "++++ ${CIA_TESTSCRIPT}: Using existing ./tmp/flatcar_test_update.gz for testing ${CIA_VERNUM} (${CIA_ARCH}) ++++"
+    echo "++++ ${CIA_TESTSCRIPT}: Using existing ${QEMU_UPDATE_PAYLOAD} for testing ${CIA_VERNUM} (${CIA_ARCH}) ++++"
 else
     echo "++++ ${CIA_TESTSCRIPT}: downloading flatcar_test_update.gz for ${CIA_VERNUM} (${CIA_ARCH}) ++++"
     copy_from_buildcache "images/${CIA_ARCH}/${CIA_VERNUM}/flatcar_test_update.gz" tmp/
@@ -105,7 +105,7 @@ run_kola_tests() {
       --qemu-image="${image}" \
       --tapfile="${instance_tapfile}" \
       --torcx-manifest="${CIA_TORCX_MANIFEST}" \
-      --update-payload=tmp/flatcar_test_update.gz \
+      --update-payload="${QEMU_UPDATE_PAYLOAD}" \
       ${QEMU_KOLA_SKIP_MANGLE:+--qemu-skip-mangle} \
       cl.update.payload
 }