From f9b13baeb5ef05fabf5230e1ce120449796c8c34 Mon Sep 17 00:00:00 2001 From: Mathieu Tortuyaux Date: Wed, 6 Jul 2022 16:46:19 +0200 Subject: [PATCH 1/7] .github/ci: add kola qemu test this initial attempt runs right after the "packages" jobs and downloads the resulting artifacts. The QEMU image is unzipped then the kola test is running using the ci-automation scripts. Signed-off-by: Mathieu Tortuyaux --- .github/workflows/ci.yaml | 50 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index ec456ec6f3..9adecf321c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -287,3 +287,53 @@ jobs: scripts/images/**/flatcar_production_*.sh scripts/images/**/flatcar_test_update.gz scripts/ebuild_logs.tar.xz + test: + name: "Run kola tests" + needs: packages + runs-on: + - debian + - kola + - self-hosted + - ${{ matrix.arch }} + strategy: + fail-fast: false + matrix: + arch: ["amd64", "arm64"] + defaults: + run: + working-directory: scripts + + steps: + - uses: actions/checkout@v3 + with: + path: scripts + fetch-depth: 0 + submodules: true + + # it downloads from the previous linked job. + - name: Download artifacts + uses: actions/download-artifact@v3 + with: + name: images-${{ matrix.arch }} + + - name: Run tests + shell: bash + run: | + exec 2>&1 + set +x + set -euo pipefail + + source ci-automation/test.sh + + # TODO: REMOVE ME (for debugging purposes) + ls -ali + find . -type f -name "*.bz2" + + # extract the image. + bzip2 --decompress scripts/images/**/flatcar_production_image.bin.bz2 + + # export the QEMU_IMAGE_NAME to avoid to download it. + export QEMU_IMAGE_NAME=$(readlink -f scripts/images/**/flatcar_production_image.bin) + + # run the test. + test_run ${{ matrix.arch }} qemu From dc8cf9c18d74833537958b3019681fb137644748 Mon Sep 17 00:00:00 2001 From: Gabriel Adrian Samfira Date: Mon, 11 Jul 2022 13:32:25 +0300 Subject: [PATCH 2/7] Add configuration options to test functions * Add SKIP_COPY_TO_BINCACHE environment variable that will skip uploading test results to bincache. This is useful if we want to upload test results as artifacts on github. * make QEMU_IMAGE_NAME configurable Signed-off-by: Gabriel Adrian Samfira --- ci-automation/ci-config.env | 2 +- ci-automation/test.sh | 26 ++++++++++++++++---------- ci-automation/vendor-testing/qemu.sh | 2 +- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/ci-automation/ci-config.env b/ci-automation/ci-config.env index d061a9bf8f..2b3289283b 100644 --- a/ci-automation/ci-config.env +++ b/ci-automation/ci-config.env @@ -45,7 +45,7 @@ CONTAINER_IMAGE_ROOT="/home/sdk/build/images" # -- QEMU -- -QEMU_IMAGE_NAME="flatcar_production_image.bin" +QEMU_IMAGE_NAME=${QEMU_IMAGE_NAME:-flatcar_production_image.bin} QEMU_PARALLEL="${PARALLEL_TESTS:-20}" # BIOS path within the SDK diff --git a/ci-automation/test.sh b/ci-automation/test.sh index 18b2ffdacd..9162aecfaf 100644 --- a/ci-automation/test.sh +++ b/ci-automation/test.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash -x # # Copyright (c) 2021 The Flatcar Maintainers. # Use of this source code is governed by a BSD-style license that can be @@ -120,6 +120,7 @@ function _test_run_impl() { fi local retries="${MAX_RETRIES:-20}" + local skip_copy_to_bincache=${SKIP_COPY_TO_BINCACHE:-0} source ci-automation/tapfile_helper_lib.sh source ci-automation/ci_automation_common.sh @@ -216,14 +217,19 @@ function _test_run_impl() { echo "########### All re-runs exhausted ($retries). Giving up. ###########" fi - # publish kola output, TAP files to build cache - copy_to_buildcache "testing/${vernum}/${arch}/${image}" \ - "${tests_dir}/_kola_temp" - copy_to_buildcache "testing/${vernum}/${arch}/${image}" \ - "${tests_dir}/"*.tap - copy_to_buildcache "testing/${vernum}/${arch}/${image}" \ - "${tap_merged_summary}" - copy_to_buildcache "testing/${vernum}/${arch}/${image}" \ - "${tap_merged_detailed}" + if [ ${skip_copy_to_bincache} -eq 0 ] + # publish kola output, TAP files to build cache + copy_to_buildcache "testing/${vernum}/${arch}/${image}" \ + "${tests_dir}/_kola_temp" + copy_to_buildcache "testing/${vernum}/${arch}/${image}" \ + "${tests_dir}/"*.tap + copy_to_buildcache "testing/${vernum}/${arch}/${image}" \ + "${tap_merged_summary}" + copy_to_buildcache "testing/${vernum}/${arch}/${image}" \ + "${tap_merged_detailed}" + fi + if ! $success; then + return 1 + fi } # -- diff --git a/ci-automation/vendor-testing/qemu.sh b/ci-automation/vendor-testing/qemu.sh index 4143609239..9b2e8dd750 100755 --- a/ci-automation/vendor-testing/qemu.sh +++ b/ci-automation/vendor-testing/qemu.sh @@ -22,7 +22,7 @@ fi # Fetch image and BIOS if not present if [ -f "${QEMU_IMAGE_NAME}" ] ; then - echo "++++ ${CIA_TESTSCRIPT}: Using existing ./${QEMU_IMAGE_NAME} for testing ${CIA_VERNUM} (${CIA_ARCH}) ++++" + echo "++++ ${CIA_TESTSCRIPT}: Using existing ${QEMU_IMAGE_NAME} for testing ${CIA_VERNUM} (${CIA_ARCH}) ++++" else echo "++++ ${CIA_TESTSCRIPT}: downloading ${QEMU_IMAGE_NAME} for ${CIA_VERNUM} (${CIA_ARCH}) ++++" copy_from_buildcache "images/${CIA_ARCH}/${CIA_VERNUM}/${QEMU_IMAGE_NAME}" . From 879e5a2b3053c0c15b6fc074c275ae21ff9cc223 Mon Sep 17 00:00:00 2001 From: Gabriel Adrian Samfira Date: Mon, 11 Jul 2022 13:34:24 +0300 Subject: [PATCH 3/7] Make the kola test workflow reusable * Adds a reusable workflow that can run tests * Adapts the ci.yaml to use reusable workflow * Creates a new workflow that helps trigger tests using an arbitrary workflow run. Signed-off-by: Gabriel Adrian Samfira --- .github/workflows/ci.yaml | 64 ++-------- .github/workflows/dispatch-kola-tests.yaml | 42 +++++++ .github/workflows/run-kola-tests.yaml | 137 +++++++++++++++++++++ 3 files changed, 189 insertions(+), 54 deletions(-) create mode 100644 .github/workflows/dispatch-kola-tests.yaml create mode 100644 .github/workflows/run-kola-tests.yaml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 9adecf321c..71b0069ba5 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -42,9 +42,9 @@ jobs: packages: name: "Build Flatcar packages" runs-on: + - self-hosted - debian - build - - self-hosted - x64 strategy: fail-fast: false @@ -61,18 +61,14 @@ jobs: run: | sudo rm /bin/sh sudo ln -s /bin/bash /bin/sh - sudo apt-get install \ - ca-certificates \ - curl \ - gnupg \ - lsb-release + sudo apt-get install -y ca-certificates curl gnupg lsb-release qemu-user-static git sudo mkdir -p /etc/apt/keyrings curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg echo \ "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \ $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null sudo apt-get update - sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin qemu-user-static git + sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin - uses: actions/checkout@v3 with: @@ -287,53 +283,13 @@ jobs: scripts/images/**/flatcar_production_*.sh scripts/images/**/flatcar_test_update.gz scripts/ebuild_logs.tar.xz + test: name: "Run kola tests" needs: packages - runs-on: - - debian - - kola - - self-hosted - - ${{ matrix.arch }} - strategy: - fail-fast: false - matrix: - arch: ["amd64", "arm64"] - defaults: - run: - working-directory: scripts - - steps: - - uses: actions/checkout@v3 - with: - path: scripts - fetch-depth: 0 - submodules: true - - # it downloads from the previous linked job. - - name: Download artifacts - uses: actions/download-artifact@v3 - with: - name: images-${{ matrix.arch }} - - - name: Run tests - shell: bash - run: | - exec 2>&1 - set +x - set -euo pipefail - - source ci-automation/test.sh - - # TODO: REMOVE ME (for debugging purposes) - ls -ali - find . -type f -name "*.bz2" - - # extract the image. - bzip2 --decompress scripts/images/**/flatcar_production_image.bin.bz2 - - # export the QEMU_IMAGE_NAME to avoid to download it. - export QEMU_IMAGE_NAME=$(readlink -f scripts/images/**/flatcar_production_image.bin) - - # run the test. - test_run ${{ matrix.arch }} qemu + uses: ./.github/workflows/run-kola-tests.yaml + with: + arch: '["amd64", "arm64"]' + workflow_name_or_id: ${{ github.event.workflow_run.workflow_id }} + workflow_run_id: ${{ github.event.workflow_run.id }} + parallel_tests: "2" diff --git a/.github/workflows/dispatch-kola-tests.yaml b/.github/workflows/dispatch-kola-tests.yaml new file mode 100644 index 0000000000..94c59c6884 --- /dev/null +++ b/.github/workflows/dispatch-kola-tests.yaml @@ -0,0 +1,42 @@ +name: Dispatch kola tests + +on: + workflow_dispatch: + inputs: + arch: + type: string + default: '["amd64", "arm64"]' + description: | + The architectures to run tests for. This is an array of architectures + that can be deserialized in JSON and results in an array of strings. + required: true + workflow_name_or_id: + type: string + required: true + default: ci.yaml + description: | + The workflow ID from where we'll download the artifacts to be tested. + workflow_run_id: + type: string + required: true + description: | + The run ID of the workflow specified in workflow_name_or_id + parallel_tests: + type: string + required: true + default: "2" + description: | + The number of parallel tests to run. + +permissions: {} + +jobs: + test: + name: "Trigger kola test" + uses: ./.github/workflows/run-kola-tests.yaml + with: + arch: ${{ github.event.inputs.arch }} + workflow_name_or_id: ${{ github.event.inputs.workflow_name_or_id }} + workflow_run_id: ${{ github.event.inputs.workflow_run_id }} + parallel_tests: ${{ github.event.inputs.parallel_tests }} + diff --git a/.github/workflows/run-kola-tests.yaml b/.github/workflows/run-kola-tests.yaml new file mode 100644 index 0000000000..1b5417d4bb --- /dev/null +++ b/.github/workflows/run-kola-tests.yaml @@ -0,0 +1,137 @@ +name: "Run kola tests" +on: + workflow_call: + inputs: + arch: + type: string + default: '["amd64", "arm64"]' + description: | + The architectures to run tests for. This is an array of architectures + that can be deserialized in JSON and results in an array of strings. + required: true + workflow_name_or_id: + type: string + required: true + default: ci.yaml + description: | + The workflow ID from where we'll download the artifacts to be tested. + workflow_run_id: + type: string + required: true + description: | + The run ID of the workflow specified in workflow_name_or_id + parallel_tests: + type: string + required: true + default: "2" + description: | + The number of parallel tests to run. + +jobs: + tests: + name: "Build Flatcar packages" + runs-on: + - self-hosted + - debian + - kola + - ${{ matrix.arch }} + strategy: + fail-fast: false + matrix: + arch: ${{fromJSON(github.event.inputs.arch)}} + + steps: + - name: Prepare machine + shell: bash + working-directory: ${{ github.workspace }} + run: | + sudo rm /bin/sh + sudo ln -s /bin/bash /bin/sh + sudo apt-get install -y ca-certificates curl gnupg lsb-release qemu-system qemu-user-static git bzip2 jq dnsmasq + sudo systemctl stop dnsmasq + sudo systemctl mask dnsmasq + + # Install Docker-CE + sudo mkdir -p /etc/apt/keyrings + curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg + echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \ + $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + sudo apt-get update + sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin + + # Install latest Go + VERSION=$(curl -s -L https://golang.org/dl/?mode=json | jq -r .[0].version) + curl -s -L https://dl.google.com/go/$VERSION.linux-amd64.tar.gz | sudo tar -xz -C /usr/local + sudo ln -s /usr/local/go/bin/go /usr/local/bin/ + sudo ln -s /usr/local/go/bin/gofmt /usr/local/bin/ + + # Set up MASQUERADE. Don't care much to secure it. + # This is needed for the VMs kola spins up to have internet access. + DEFAULT_ROUTE_DEVICE=$(ip -j route sh default |jq -r .[0].dev) + sudo iptables -t nat -I POSTROUTING -o $DEFAULT_ROUTE_DEVICE -j MASQUERADE + sudo iptables -I FORWARD -o $DEFAULT_ROUTE_DEVICE -j ACCEPT + sudo iptables -I FORWARD -i $DEFAULT_ROUTE_DEVICE -j ACCEPT + + # Enable IP forward + echo 'net.ipv4.ip_forward=1' | sudo tee /etc/sysctl.d/ip_forward.conf + # Enable unprivileged bpf + echo 'kernel.unprivileged_bpf_disabled=0' | sudo tee /etc/sysctl.d/enable_unprivileged_bpf.conf + sudo sysctl --system + sysctl -p + + # Enable extra SWAP + sudo fallocate /swap.img -l 8GiB + sudo chmod 600 /swap.img + sudo mkswap /swap.img + sudo swapon /swap.img + + + - uses: actions/checkout@v3 + with: + path: scripts + fetch-depth: 0 + submodules: true + + - name: Download artifacts + uses: gabriel-samfira/action-download-artifact@v5 + with: + workflow: ${{ github.event.inputs.workflow_name_or_id }} + workflow_conclusion: success + run_id: ${{ github.event.inputs.workflow_run_id }} + name: images-${{ matrix.arch }} + + - name: Run tests + shell: bash + run: | + exec 2>&1 + set +x + set -euo pipefail + + # extract the image. + IMG_ARCHIVE=$(readlink -f images/**/flatcar_production_image.bin.bz2) + bzip2 --decompress ${IMG_ARCHIVE} + + # export the QEMU_IMAGE_NAME to avoid to download it. + cp ${IMG_ARCHIVE%%.bz2} ./scripts/ + pushd scripts + pwd + source ci-automation/test.sh + cat > sdk_container/.env < Date: Mon, 11 Jul 2022 19:35:17 +0300 Subject: [PATCH 4/7] Remove customizations for now Remove arch and parallel options. Hardcode them for now. --- .github/workflows/ci.yaml | 2 -- .github/workflows/dispatch-kola-tests.yaml | 18 +----------------- .github/workflows/run-kola-tests.yaml | 17 ++--------------- 3 files changed, 3 insertions(+), 34 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 71b0069ba5..1ffa99e74a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -289,7 +289,5 @@ jobs: needs: packages uses: ./.github/workflows/run-kola-tests.yaml with: - arch: '["amd64", "arm64"]' workflow_name_or_id: ${{ github.event.workflow_run.workflow_id }} workflow_run_id: ${{ github.event.workflow_run.id }} - parallel_tests: "2" diff --git a/.github/workflows/dispatch-kola-tests.yaml b/.github/workflows/dispatch-kola-tests.yaml index 94c59c6884..a104c7c7ab 100644 --- a/.github/workflows/dispatch-kola-tests.yaml +++ b/.github/workflows/dispatch-kola-tests.yaml @@ -3,13 +3,6 @@ name: Dispatch kola tests on: workflow_dispatch: inputs: - arch: - type: string - default: '["amd64", "arm64"]' - description: | - The architectures to run tests for. This is an array of architectures - that can be deserialized in JSON and results in an array of strings. - required: true workflow_name_or_id: type: string required: true @@ -21,22 +14,13 @@ on: required: true description: | The run ID of the workflow specified in workflow_name_or_id - parallel_tests: - type: string - required: true - default: "2" - description: | - The number of parallel tests to run. permissions: {} jobs: test: - name: "Trigger kola test" + name: "Trigger kola test - test" uses: ./.github/workflows/run-kola-tests.yaml with: - arch: ${{ github.event.inputs.arch }} workflow_name_or_id: ${{ github.event.inputs.workflow_name_or_id }} workflow_run_id: ${{ github.event.inputs.workflow_run_id }} - parallel_tests: ${{ github.event.inputs.parallel_tests }} - diff --git a/.github/workflows/run-kola-tests.yaml b/.github/workflows/run-kola-tests.yaml index 1b5417d4bb..ed39632248 100644 --- a/.github/workflows/run-kola-tests.yaml +++ b/.github/workflows/run-kola-tests.yaml @@ -2,13 +2,6 @@ name: "Run kola tests" on: workflow_call: inputs: - arch: - type: string - default: '["amd64", "arm64"]' - description: | - The architectures to run tests for. This is an array of architectures - that can be deserialized in JSON and results in an array of strings. - required: true workflow_name_or_id: type: string required: true @@ -20,12 +13,6 @@ on: required: true description: | The run ID of the workflow specified in workflow_name_or_id - parallel_tests: - type: string - required: true - default: "2" - description: | - The number of parallel tests to run. jobs: tests: @@ -38,7 +25,7 @@ jobs: strategy: fail-fast: false matrix: - arch: ${{fromJSON(github.event.inputs.arch)}} + arch: ["amd64", "arm64"] steps: - name: Prepare machine @@ -119,7 +106,7 @@ jobs: source ci-automation/test.sh cat > sdk_container/.env < Date: Tue, 12 Jul 2022 19:46:27 +0300 Subject: [PATCH 5/7] Make QEMU_UEFI_BIOS configurable Signed-off-by: Gabriel Adrian Samfira --- ci-automation/ci-config.env | 2 +- ci-automation/test.sh | 2 +- ci-automation/vendor-testing/qemu.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ci-automation/ci-config.env b/ci-automation/ci-config.env index 2b3289283b..e418c06d2e 100644 --- a/ci-automation/ci-config.env +++ b/ci-automation/ci-config.env @@ -53,7 +53,7 @@ QEMU_BIOS="/usr/share/qemu/bios-256k.bin" # UEFI bios filename on build cache. # Published by vms.sh as part of the qemu vendor build. -QEMU_UEFI_BIOS="flatcar_production_qemu_uefi_efi_code.fd" +QEMU_UEFI_BIOS="${QEMU_UEFI_BIOS:-flatcar_production_qemu_uefi_efi_code.fd}" # -- Equinix Metal -- diff --git a/ci-automation/test.sh b/ci-automation/test.sh index 9162aecfaf..947f3dee8a 100644 --- a/ci-automation/test.sh +++ b/ci-automation/test.sh @@ -217,7 +217,7 @@ function _test_run_impl() { echo "########### All re-runs exhausted ($retries). Giving up. ###########" fi - if [ ${skip_copy_to_bincache} -eq 0 ] + if [ ${skip_copy_to_bincache} -eq 0 ];then # publish kola output, TAP files to build cache copy_to_buildcache "testing/${vernum}/${arch}/${image}" \ "${tests_dir}/_kola_temp" diff --git a/ci-automation/vendor-testing/qemu.sh b/ci-automation/vendor-testing/qemu.sh index 9b2e8dd750..a13e6d0495 100755 --- a/ci-automation/vendor-testing/qemu.sh +++ b/ci-automation/vendor-testing/qemu.sh @@ -32,7 +32,7 @@ bios="${QEMU_BIOS}" if [ "${CIA_TESTSCRIPT}" = "qemu_uefi.sh" ] ; then bios="${QEMU_UEFI_BIOS}" if [ -f "${bios}" ] ; then - echo "++++ ${CIA_TESTSCRIPT}: Using existing ./${bios} ++++" + echo "++++ ${CIA_TESTSCRIPT}: Using existing ${bios} ++++" else echo "++++ ${CIA_TESTSCRIPT}: downloading ${bios} for ${CIA_VERNUM} (${CIA_ARCH}) ++++" copy_from_buildcache "images/${CIA_ARCH}/${CIA_VERNUM}/${bios}" . From b518c3cdb8b78f8501760f94a56dcd2d0ecdd576 Mon Sep 17 00:00:00 2001 From: Gabriel Adrian Samfira Date: Wed, 13 Jul 2022 14:19:50 +0300 Subject: [PATCH 6/7] Disable image mangle in qemu tests Signed-off-by: Gabriel Adrian Samfira --- ci-automation/test.sh | 2 +- ci-automation/vendor-testing/qemu.sh | 1 + ci-automation/vendor-testing/qemu_update.sh | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/ci-automation/test.sh b/ci-automation/test.sh index 947f3dee8a..0dd0fce80c 100644 --- a/ci-automation/test.sh +++ b/ci-automation/test.sh @@ -1,4 +1,4 @@ -#!/bin/bash -x +#!/bin/bash # # Copyright (c) 2021 The Flatcar Maintainers. # Use of this source code is governed by a BSD-style license that can be diff --git a/ci-automation/vendor-testing/qemu.sh b/ci-automation/vendor-testing/qemu.sh index a13e6d0495..bf66dcb9f0 100755 --- a/ci-automation/vendor-testing/qemu.sh +++ b/ci-automation/vendor-testing/qemu.sh @@ -49,6 +49,7 @@ kola run \ --qemu-image="${QEMU_IMAGE_NAME}" \ --tapfile="${CIA_TAPFILE}" \ --torcx-manifest="${CIA_TORCX_MANIFEST}" \ + --qemu-skip-mangle \ "${@}" set +x diff --git a/ci-automation/vendor-testing/qemu_update.sh b/ci-automation/vendor-testing/qemu_update.sh index a21df0c822..2d8dea3a39 100755 --- a/ci-automation/vendor-testing/qemu_update.sh +++ b/ci-automation/vendor-testing/qemu_update.sh @@ -55,4 +55,5 @@ kola run \ --tapfile="${CIA_TAPFILE}" \ --torcx-manifest="${CIA_TORCX_MANIFEST}" \ --update-payload=tmp/flatcar_test_update.gz \ + --qemu-skip-mangle \ cl.update.payload From 340fd39a9f2e7287f5209c021dcbd74e6f96d6b4 Mon Sep 17 00:00:00 2001 From: Gabriel Adrian Samfira Date: Wed, 13 Jul 2022 14:20:12 +0300 Subject: [PATCH 7/7] Update kola test workflow * arm64 tests now run in LXD containers instead of VMs * parallelism increased for arm64 * some setup steps are skipped on arm64 Signed-off-by: Gabriel Adrian Samfira --- .github/workflows/ci.yaml | 5 +- .github/workflows/run-kola-tests.yaml | 81 ++++++++++++++++----------- 2 files changed, 50 insertions(+), 36 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 1ffa99e74a..43666ccd98 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -285,9 +285,6 @@ jobs: scripts/ebuild_logs.tar.xz test: - name: "Run kola tests" needs: packages + name: "Run kola tests" uses: ./.github/workflows/run-kola-tests.yaml - with: - workflow_name_or_id: ${{ github.event.workflow_run.workflow_id }} - workflow_run_id: ${{ github.event.workflow_run.id }} diff --git a/.github/workflows/run-kola-tests.yaml b/.github/workflows/run-kola-tests.yaml index ed39632248..56efbfe3ff 100644 --- a/.github/workflows/run-kola-tests.yaml +++ b/.github/workflows/run-kola-tests.yaml @@ -4,19 +4,19 @@ on: inputs: workflow_name_or_id: type: string - required: true + required: false default: ci.yaml description: | The workflow ID from where we'll download the artifacts to be tested. workflow_run_id: type: string - required: true + required: false description: | The run ID of the workflow specified in workflow_name_or_id jobs: tests: - name: "Build Flatcar packages" + name: "Run Kola tests" runs-on: - self-hosted - debian @@ -34,7 +34,7 @@ jobs: run: | sudo rm /bin/sh sudo ln -s /bin/bash /bin/sh - sudo apt-get install -y ca-certificates curl gnupg lsb-release qemu-system qemu-user-static git bzip2 jq dnsmasq + sudo apt-get install -y ca-certificates curl gnupg lsb-release qemu-system git bzip2 jq dnsmasq sudo systemctl stop dnsmasq sudo systemctl mask dnsmasq @@ -45,13 +45,7 @@ jobs: "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \ $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null sudo apt-get update - sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin - - # Install latest Go - VERSION=$(curl -s -L https://golang.org/dl/?mode=json | jq -r .[0].version) - curl -s -L https://dl.google.com/go/$VERSION.linux-amd64.tar.gz | sudo tar -xz -C /usr/local - sudo ln -s /usr/local/go/bin/go /usr/local/bin/ - sudo ln -s /usr/local/go/bin/gofmt /usr/local/bin/ + sudo apt-get install -y docker-ce docker-ce-cli containerd.io # Set up MASQUERADE. Don't care much to secure it. # This is needed for the VMs kola spins up to have internet access. @@ -60,19 +54,24 @@ jobs: sudo iptables -I FORWARD -o $DEFAULT_ROUTE_DEVICE -j ACCEPT sudo iptables -I FORWARD -i $DEFAULT_ROUTE_DEVICE -j ACCEPT - # Enable IP forward - echo 'net.ipv4.ip_forward=1' | sudo tee /etc/sysctl.d/ip_forward.conf - # Enable unprivileged bpf - echo 'kernel.unprivileged_bpf_disabled=0' | sudo tee /etc/sysctl.d/enable_unprivileged_bpf.conf - sudo sysctl --system - sysctl -p + # ARM64 tests run inside an LXD container instead of an LXD Virtual machine + # There are some limitations in terms of what we can customize. + if [ "${{ matrix.arch }}" == "amd64" ];then + # Enable IP forward + echo 'net.ipv4.ip_forward=1' | sudo tee /etc/sysctl.d/ip_forward.conf - # Enable extra SWAP - sudo fallocate /swap.img -l 8GiB - sudo chmod 600 /swap.img - sudo mkswap /swap.img - sudo swapon /swap.img + # Enable unprivileged bpf + echo 'kernel.unprivileged_bpf_disabled=0' | sudo tee /etc/sysctl.d/enable_unprivileged_bpf.conf + sudo sysctl --system + sysctl -p + + # Enable extra SWAP + sudo fallocate /swap.img -l 8GiB + sudo chmod 600 /swap.img + sudo mkswap /swap.img + sudo swapon /swap.img + fi - uses: actions/checkout@v3 with: @@ -80,12 +79,19 @@ jobs: fetch-depth: 0 submodules: true - - name: Download artifacts - uses: gabriel-samfira/action-download-artifact@v5 + - name: Download artifact + if: ${{ !inputs.workflow_run_id }} + uses: actions/download-artifact@v3 with: - workflow: ${{ github.event.inputs.workflow_name_or_id }} + name: images-${{ matrix.arch }} + + - name: Download artifacts from other workflow + uses: gabriel-samfira/action-download-artifact@v5 + if: ${{ inputs.workflow_run_id }} + with: + workflow: ${{ inputs.workflow_name_or_id }} workflow_conclusion: success - run_id: ${{ github.event.inputs.workflow_run_id }} + run_id: ${{ inputs.workflow_run_id }} name: images-${{ matrix.arch }} - name: Run tests @@ -97,21 +103,33 @@ jobs: # extract the image. IMG_ARCHIVE=$(readlink -f images/**/flatcar_production_image.bin.bz2) + QEMU_UEFI_BIOS_FILE=$(readlink -f images/**/flatcar_production_qemu_uefi_efi_code.fd) bzip2 --decompress ${IMG_ARCHIVE} - # export the QEMU_IMAGE_NAME to avoid to download it. cp ${IMG_ARCHIVE%%.bz2} ./scripts/ + cp ${QEMU_UEFI_BIOS_FILE} ./scripts/ + pushd scripts - pwd source ci-automation/test.sh + + PARALLEL_ARCH=2 + if [ "${{ matrix.arch }}" == "arm64" ];then + # ARM64 servers have more memory and CPUs + PARALLEL_ARCH=10 + fi + cat > sdk_container/.env <