mirrors/flatcar-scripts
1
0
Fork 0
mirror of https://github.com/flatcar/scripts.git synced 2025-09-23 14:41:31 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #696 from flatcar/t-lo/speed-up-ci-build

Browse Source 
Github Actions CI: streamline build, export all artifacts required by testing, improve testing
This commit is contained in:
Thilo Fromm 2023-03-30 13:36:45 +02:00 committed by GitHub
commit a8101a6d6c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 611 additions and 131 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

231
.github/workflows/ci.yaml vendored
View File

@ -1,13 +1,11 @@
name: "Run build"
on:
pull_request:
# Run when the PR is ready and each time a review is re-requested
# (i.e. after feedback has been addressed).
types: [review_requested, ready_for_review]
workflow_dispatch:
inputs:
bincache_server:
description: |
Bincache server.
default: "bincache.flatcar-linux.net"
required: true
image_formats:
description: |
Space-separated vendor formats to build.
@ -36,10 +34,13 @@ concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }}
cancel-in-progress: true
permissions: {}
permissions:
pull-requests: write
jobs:
packages:
# Do not run when still in draft mode but a review was requested anyway
if: github.event.pull_request.draft == false
name: "Build Flatcar packages"
runs-on:
- self-hosted
@ -76,10 +77,24 @@ jobs:
fetch-depth: 0
submodules: true
# Hack alert: actions/checkout will check out the (disjunct) merge commit of a PR
# instead of its head commit. That commit is not connected to any branch.
# This causes breakage downstream e.g. when the devcontainer test wants to check out
# the ref in the scripts repo that corresponds to this build.
- name: If this is a PR build, use head commit instead of the merge commit
if: ${{ github.event.pull_request.head.sha }}
shell: bash
run: |
exec 2>&1
set -x
set -euo pipefail
git checkout ${{ github.event.pull_request.head.sha }}
git submodule update
- name: Set environment
shell: bash
run: |
BUILDCACHE_SERVER="bincache.flatcar-linux.net"
arch="${{ matrix.arch }}"
COREOS_REMOTE=""
COREOS_REF=""
@ -87,14 +102,12 @@ jobs:
PORTAGE_REF=""
IMAGE_FORMATS="qemu_uefi"
[ -z "${{ github.event.inputs.bincache_server }}" ] || BUILDCACHE_SERVER="${{ github.event.inputs.bincache_server }}"
[ -z "${{ github.event.inputs.coreos_remote }}" ] || COREOS_REMOTE="${{ github.event.inputs.coreos_remote }}"
[ -z "${{ github.event.inputs.coreos_ref }}" ] || COREOS_REF="${{ github.event.inputs.coreos_ref }}"
[ -z "${{ github.event.inputs.portage_remote }}" ] || PORTAGE_REMOTE="${{ github.event.inputs.portage_remote }}"
[ -z "${{ github.event.inputs.portage_ref }}" ] || PORTAGE_REF="${{ github.event.inputs.portage_ref }}"
[ -z "${{ github.event.inputs.image_formats }}" ] || IMAGE_FORMATS="${{ github.event.inputs.image_formats }}"
echo "BUILDCACHE_SERVER=${BUILDCACHE_SERVER}" >> $GITHUB_ENV
echo "arch=${arch}" >> $GITHUB_ENV
echo "COREOS_REMOTE=${COREOS_REMOTE}" >> $GITHUB_ENV
echo "COREOS_REF=${COREOS_REF}" >> $GITHUB_ENV
@ -102,6 +115,15 @@ jobs:
echo "PORTAGE_REF=${PORTAGE_REF}" >> $GITHUB_ENV
echo "IMAGE_FORMATS=${IMAGE_FORMATS}" >> $GITHUB_ENV
# Artifact root for images and torcx tarball as seen from within the container
echo "CI_CONTAINER_ARTIFACT_ROOT=/home/sdk/trunk/src/scripts/artifacts" >> $GITHUB_ENV
echo "CI_CONTAINER_TORCX_ROOT=/home/sdk/trunk/src/scripts/artifacts/torcx" >> $GITHUB_ENV
mkdir -p artifacts/torcx
# Placeholder URL for run-kola-tests.yaml, "Extract artifacts" step which will replace
# this with its IP address.
echo "TORCX_TESTS_PACKAGE_URL=http://localhost:12345" >> $GITHUB_ENV
- name: Checkout submodules
shell: bash
run: |
@ -129,7 +151,7 @@ jobs:
shell: bash
run: |
exec 2>&1
set +x
set -x
set -euo pipefail
source ci-automation/ci_automation_common.sh
@ -144,92 +166,76 @@ jobs:
docker_image_from_registry_or_buildcache "${sdk_name}" "${docker_sdk_vernum}"
sdk_image="$(docker_image_fullname "${sdk_name}" "${docker_sdk_vernum}")"
vernum="${version#*-}" # remove main-,alpha-,beta-,stable-,lts- version tag
docker_vernum="$(vernum_to_docker_image_version "${vernum}")"
packages_container="flatcar-packages-${arch}-${docker_vernum}"
container_name="flatcar-ci-build"
echo "container_name=${container_name}" >> "$GITHUB_ENV"
# Create version file
(
source sdk_lib/sdk_container_common.sh
create_versionfile "$sdk_version" "$version"
)
./run_sdk_container -n "${packages_container}" -v "${version}" \
# Run the packages build. This will create the ci build container
# which will be re-used by subsequent build steps.
./run_sdk_container -n "${container_name}" -v "${version}" \
-C "${sdk_image}" \
./build_packages --board="${arch}-usr" \
--torcx_output_root="${CONTAINER_TORCX_ROOT}"
--torcx_output_root="${CI_CONTAINER_TORCX_ROOT}" \
--torcx_extra_pkg_url="${TORCX_TESTS_PACKAGE_URL}"
# copy torcx manifest and docker tarball for publishing
torcx_tmp="__build__/torcx_tmp"
rm -rf "${torcx_tmp}"
mkdir "${torcx_tmp}"
./run_sdk_container -n "${packages_container}" -v "${version}" \
-C "${sdk_image}" \
cp -r "${CONTAINER_TORCX_ROOT}/" \
"${torcx_tmp}"
# Create binpkgs tarball for archiving as artifact later
./run_sdk_container -n "${container_name}" \
tar -C "/build/${arch}-usr/var/lib/portage/pkgs/" \
-cvf binpkgs.tar .
source sdk_container/.repo/manifests/version.txt
vernum="${FLATCAR_VERSION}"
docker_vernum="$(vernum_to_docker_image_version "${vernum}")"
packages_image="flatcar-packages-${arch}"
- name: Extract build logs
if: always()
shell: bash
run: |
set -euo pipefail
set -x
# Copy logs
./run_sdk_container -n "${container_name}" \
tar -cJf ebuild_logs.tar.xz /build/${arch}-usr/var/log/portage \
/build/${arch}-usr/var/tmp/portage
echo "vernum=${vernum}" >> $GITHUB_ENV
echo "docker_vernum=${docker_vernum}" >> $GITHUB_ENV
echo "packages_image=${packages_image}" >> $GITHUB_ENV
echo "arch=${arch}" >> $GITHUB_ENV
echo "sdk_image=${sdk_image}" >> $GITHUB_ENV
echo "packages_container=${packages_container}" >> $GITHUB_ENV
docker commit "${packages_container}" "${packages_image}:${docker_vernum}"
docker rm -f "${packages_container}"
- name: Upload build logs
if: always()
uses: actions/upload-artifact@v3
with:
retention-days: 7
name: ${{ matrix.arch }}-build-logs
path: |
scripts/ebuild_logs.tar.xz
- name: Build image
shell: bash
run: |
set -euo pipefail
set +x
set -x
echo 'channel="developer"' >> $GITHUB_ENV
channel="developer"
source ci-automation/ci_automation_common.sh
packages="flatcar-packages-${arch}"
packages_image="${packages}:${docker_vernum}"
image="flatcar-images-${arch}"
image_container="${image}-${docker_vernum}"
official_arg="--noofficial"
echo "image=flatcar-images-${arch}" >> $GITHUB_ENV
echo "image_image=${image}:${docker_vernum}" >> $GITHUB_ENV
./run_sdk_container -x ./ci-cleanup.sh -n "${image_container}" -C "${packages_image}" \
-v "${vernum}" \
mkdir -p "${CONTAINER_IMAGE_ROOT}"
./run_sdk_container -n "${image_container}" -C "${packages_image}" \
-v "${vernum}" \
./run_sdk_container -n "${container_name}" \
./set_official --board="${arch}-usr" "${official_arg}"
./run_sdk_container -n "${image_container}" -C "${packages_image}" \
-v "${vernum}" \
./run_sdk_container -n "${container_name}" \
./build_image --board="${arch}-usr" --group="${channel}" \
--output_root="${CONTAINER_IMAGE_ROOT}" \
--torcx_root="${CONTAINER_TORCX_ROOT}" prodtar container
--output_root="${CI_CONTAINER_ARTIFACT_ROOT}" \
--torcx_root="${CI_CONTAINER_TORCX_ROOT}" prodtar container
# Copy logs
./run_sdk_container -n "${image_container}" -C "${packages_image}" -v "${vernum}" \
tar -cJf ebuild_logs.tar.xz /build/${arch}-usr/var/log/portage \
/build/${arch}-usr/var/tmp/portage
docker commit "${image_container}" "${image}:${docker_vernum}"
docker rm -f "${image_container}"
- name: Build VM image
- name: Build VM image(s)
shell: bash
run: |
set -euo pipefail
set +x
set -x
source ci-automation/ci_automation_common.sh
vms_container="flatcar-vms-${docker_vernum}"
images_out="images"
has_packet=0
@ -254,35 +260,96 @@ jobs:
for format in ${formats}; do
echo " ################### VENDOR '${format}' ################### "
./run_sdk_container -n "${vms_container}" -C "${image_image}" \
-v "${vernum}" \
./run_sdk_container -n "${container_name}" \
./image_to_vm.sh --format "${format}" --board="${arch}-usr" \
--from "${CONTAINER_IMAGE_ROOT}/${arch}-usr/latest" \
--from "${CI_CONTAINER_ARTIFACT_ROOT}/${arch}-usr/latest" \
--image_compression_formats=bz2
done
# copy resulting images
./run_sdk_container -n "${vms_container}" \
-v "${vernum}" \
mv "${CONTAINER_IMAGE_ROOT}/${arch}-usr" "./${images_out}"
# upload-artifacts cannot handle artifact uploads from sym-linked directories (no, really)
# so we move things around.
mkdir -p artifacts/images
(
cd artifacts/${arch}-usr/latest/
mv * ../../images/
)
# remove symlinks before upload
find "./${images_out}" -type l -delete
# create a tarball for torcx package + JSON file because upload-artifacts cannot handle filenames containing colons
# (such as "docker:20.10.torcx.tgz")
mv artifacts/torcx/${arch}-usr/latest/torcx_manifest.json artifacts/torcx/pkgs/
tar -C artifacts/torcx/pkgs/ -cvf torcx.tar .
docker rm -f "${vms_container}"
- name: Upload artifacts
- name: Upload binpkgs
uses: actions/upload-artifact@v3
with:
name: images-${{ matrix.arch }}
retention-days: 7
name: ${{ matrix.arch }}-binpkgs
path: |
scripts/images/**/*.img.bz2
scripts/images/**/*.bin.bz2
scripts/images/**/flatcar_production_*_efi_*.fd
scripts/images/**/*.txt
scripts/images/**/flatcar_production_*.sh
scripts/images/**/flatcar_test_update.gz
scripts/ebuild_logs.tar.xz
scripts/binpkgs.tar
- name: Upload update image (used with kola tests later)
uses: actions/upload-artifact@v3
with:
retention-days: 7
name: ${{ matrix.arch }}-test-update
path: |
scripts/artifacts/images/flatcar_test_update.gz
- name: Upload generic image
uses: actions/upload-artifact@v3
with:
retention-days: 7
name: ${{ matrix.arch }}-generic-image
path: |
scripts/artifacts/images/flatcar_production_image.bin.bz2
scripts/artifacts/images/flatcar_production_image.grub
scripts/artifacts/images/flatcar_production_image.shim
scripts/artifacts/images/flatcar_production_image.vmlinuz
scripts/artifacts/images/flatcar_production_image*.txt
scripts/artifacts/images/flatcar_production_image*.json
scripts/artifacts/images/flatcar_production_image_pcr_policy.zip
scripts/artifacts/images/flatcar_production_*_efi_*.fd
- name: Upload developer container
uses: actions/upload-artifact@v3
with:
retention-days: 7
name: ${{ matrix.arch }}-devcontainer
path: |
scripts/artifacts/images/flatcar_developer_container*
- name: Upload torcx tarball
uses: actions/upload-artifact@v3
with:
retention-days: 7
name: ${{ matrix.arch }}-torcx
path: |
scripts/torcx.tar
# Clean up what we uploaded already so the "vendor images" wildcard
# works when uploading artifacts in the next step.
- name: Remove update, generic and devcontainer images
shell: bash
run: |
set -euo pipefail
set -x
rm -f artifacts/images/flatcar_test_update.gz \
artifacts/images/flatcar_production_image* \
artifacts/images/flatcar_developer_container* \
artifacts/images/flatcar_production_update*
- name: Upload vendor images
uses: actions/upload-artifact@v3
with:
retention-days: 7
name: ${{ matrix.arch }}-vm-images
path: |
scripts/artifacts/images/*.img.bz2
scripts/artifacts/images/*.bin.bz2
scripts/artifacts/images/flatcar_production_*_efi_*.fd
scripts/artifacts/images/*.txt
scripts/artifacts/images/flatcar_production_*.sh
test:
needs: packages

10
.github/workflows/dispatch-kola-tests.yaml vendored
View File

@ -8,14 +8,18 @@ on:
required: true
default: ci.yaml
description: |
The workflow ID from where we'll download the artifacts to be tested.
The workflow name or ID from where we'll download the artifacts to be tested.
E.g. the name of the YAML file (w/o path) of the respective workflow.
workflow_run_id:
type: string
required: true
description: |
The run ID of the workflow specified in workflow_name_or_id
The run ID of the workflow specified in workflow_name_or_id.
You can e.g. get this from a run's URL -
https://github.com/flatcar/scripts/actions/runs/<ID> .
permissions: {}
permissions:
pull-requests: write
jobs:
test:

274
.github/workflows/run-kola-tests.yaml vendored
View File

@ -34,7 +34,7 @@ jobs:
run: |
sudo rm /bin/sh
sudo ln -s /bin/bash /bin/sh
sudo apt-get install -y ca-certificates curl gnupg lsb-release qemu-system git bzip2 jq dnsmasq
sudo apt-get install -y ca-certificates curl gnupg lsb-release qemu-system git bzip2 jq dnsmasq python3
sudo systemctl stop dnsmasq
sudo systemctl mask dnsmasq
@ -60,45 +60,171 @@ jobs:
fetch-depth: 0
submodules: true
- name: Download artifact
# Hack alert: actions/checkout will check out the (disjunct) merge commit of a PR
# instead of its head commit. That commit is not connected to any branch.
# This is not technically necessary for the tests run but it is done to remain aligned
# with the ref.
- name: If this is a PR build, use head commit instead of the merge commit
if: ${{ github.event.pull_request.head.sha }}
shell: bash
run: |
exec 2>&1
set -x
set -euo pipefail
cd scripts
git checkout ${{ github.event.pull_request.head.sha }}
git submodule update
- name: Download binpkgs
if: ${{ !inputs.workflow_run_id }}
uses: actions/download-artifact@v3
with:
name: images-${{ matrix.arch }}
name: ${{ matrix.arch }}-binpkgs
- name: Download artifacts from other workflow
- name: Download test update image
if: ${{ !inputs.workflow_run_id }}
uses: actions/download-artifact@v3
with:
name: ${{ matrix.arch }}-test-update
- name: Download generic image
if: ${{ !inputs.workflow_run_id }}
uses: actions/download-artifact@v3
with:
name: ${{ matrix.arch }}-generic-image
- name: Download developer container
if: ${{ !inputs.workflow_run_id }}
uses: actions/download-artifact@v3
with:
name: ${{ matrix.arch }}-devcontainer
- name: Download torcx tarball
if: ${{ !inputs.workflow_run_id }}
uses: actions/download-artifact@v3
with:
name: ${{ matrix.arch }}-torcx
- name: Download binpkgs from other workflow
uses: gabriel-samfira/action-download-artifact@v5
if: ${{ inputs.workflow_run_id }}
with:
workflow: ${{ inputs.workflow_name_or_id }}
workflow_conclusion: success
run_id: ${{ inputs.workflow_run_id }}
name: images-${{ matrix.arch }}
name: ${{ matrix.arch }}-binpkgs
- name: Download test update image from other workflow
uses: gabriel-samfira/action-download-artifact@v5
if: ${{ inputs.workflow_run_id }}
with:
workflow: ${{ inputs.workflow_name_or_id }}
workflow_conclusion: success
run_id: ${{ inputs.workflow_run_id }}
name: ${{ matrix.arch }}-test-update
- name: Download generic image from other workflow
uses: gabriel-samfira/action-download-artifact@v5
if: ${{ inputs.workflow_run_id }}
with:
workflow: ${{ inputs.workflow_name_or_id }}
workflow_conclusion: success
run_id: ${{ inputs.workflow_run_id }}
name: ${{ matrix.arch }}-generic-image
- name: Download developer container from other workflow
uses: gabriel-samfira/action-download-artifact@v5
if: ${{ inputs.workflow_run_id }}
with:
workflow: ${{ inputs.workflow_name_or_id }}
workflow_conclusion: success
run_id: ${{ inputs.workflow_run_id }}
name: ${{ matrix.arch }}-devcontainer
- name: Download torcx tarball from other workflow
uses: gabriel-samfira/action-download-artifact@v5
if: ${{ inputs.workflow_run_id }}
with:
workflow: ${{ inputs.workflow_name_or_id }}
workflow_conclusion: success
run_id: ${{ inputs.workflow_run_id }}
name: ${{ matrix.arch }}-torcx
- name: Extract artifacts
shell: bash
run: |
exec 2>&1
set -x
set -euo pipefail
# Set up a webserver for devcontainer and torcx tests.
# The respective tests will download devcontainer and torcx tarball via http.
# The devcontainer test will then run a build
# which will download and install binpkgs into the dev container.
# For the sake of that test we will serve both via a temporary local web server.
TESTS_WEBSERVER_WEBROOT="scripts/devcontainer-webroot"
default_rout_device="$(sudo ip -j route sh default |jq -r .[0].dev)"
TESTS_WEBSERVER_IP="$(sudo ip -j address show dev "${default_rout_device}" | jq -r .[0].addr_info[0].local)"
TESTS_WEBSERVER_PORT=12345
echo "TESTS_WEBSERVER_WEBROOT=${TESTS_WEBSERVER_WEBROOT}" >> "$GITHUB_ENV"
echo "TESTS_WEBSERVER_IP=${TESTS_WEBSERVER_IP}" >> "$GITHUB_ENV"
echo "TESTS_WEBSERVER_PORT=${TESTS_WEBSERVER_PORT}" >> "$GITHUB_ENV"
mkdir ${TESTS_WEBSERVER_WEBROOT}
mv flatcar_developer_container* ${TESTS_WEBSERVER_WEBROOT}
tar -C ${TESTS_WEBSERVER_WEBROOT} -xvf binpkgs.tar
tar -C ${TESTS_WEBSERVER_WEBROOT} -xvf torcx.tar
# Move torcx package into plain webroot
# (path consists of <arch>/<packagename>/<checksum>/<packagename>:<version>.torcx.tar.gz)
mv "${TESTS_WEBSERVER_WEBROOT}/${{ matrix.arch }}-usr"/*/*/*.torcx.tgz \
"${TESTS_WEBSERVER_WEBROOT}"
# Update torcx.json's http URL to point to the webserver IP.
# ci.yaml defines the "localhost" placeholder in its "Set Environment" step.
sed -i "s,http://localhost:12345,http://${TESTS_WEBSERVER_IP}:${TESTS_WEBSERVER_PORT}," \
"${TESTS_WEBSERVER_WEBROOT}/torcx_manifest.json"
cat "${TESTS_WEBSERVER_WEBROOT}/torcx_manifest.json"
# Extract the generic image we'll use for qemu tests.
# Note that the qemu[_uefi] tests use the generic image instead of the
# qemu vendor VM image ("Astronaut: [...] Always have been.").
bzip2 --decompress flatcar_production_image.bin.bz2
mv flatcar_production_image.bin flatcar_production_qemu_uefi_efi_code.fd scripts/
mv flatcar_test_update.gz scripts/
- name: Run tests
shell: bash
run: |
exec 2>&1
set +x
set -x
set -euo pipefail
# extract the image.
IMG_ARCHIVE=$(readlink -f images/**/flatcar_production_image.bin.bz2)
QEMU_UEFI_BIOS_FILE=$(readlink -f images/**/flatcar_production_qemu_uefi_efi_code.fd)
bzip2 --decompress ${IMG_ARCHIVE}
cp ${IMG_ARCHIVE%%.bz2} ./scripts/
cp ${QEMU_UEFI_BIOS_FILE} ./scripts/
python3 -m http.server -d "${TESTS_WEBSERVER_WEBROOT}" -b "${TESTS_WEBSERVER_IP}" "${TESTS_WEBSERVER_PORT}" &
pushd scripts
source ci-automation/test.sh
# Provide our own torcx prepare function so we use our local manifest json.
# This is called by test_run below.
function __prepare_torcx() {
shift; shift # no need for arch or vernum
local destdir="$1"
cp "../${TESTS_WEBSERVER_WEBROOT}/torcx_manifest.json" "${destdir}"
}
PARALLEL_ARCH=10
cat > sdk_container/.env <<EOF
# export the QEMU_IMAGE_NAME to avoid to download it.
export QEMU_IMAGE_NAME="/work/flatcar_production_image.bin"
export QEMU_UEFI_BIOS="/work/flatcar_production_qemu_uefi_efi_code.fd"
export QEMU_UPDATE_PAYLOAD="/work/flatcar_test_update.gz"
export QEMU_DEVCONTAINER_URL="http://${TESTS_WEBSERVER_IP}:${TESTS_WEBSERVER_PORT}"
export QEMU_DEVCONTAINER_BINHOST_URL="http://${TESTS_WEBSERVER_IP}:${TESTS_WEBSERVER_PORT}"
export PARALLEL_TESTS=${PARALLEL_ARCH}
# The runner uses lxc containers for kola, and can't use loopback devices to
# prepare the serial console setting - this means that kola may miss some errors
@ -108,14 +234,126 @@ jobs:
export MAX_RETRIES=5
export SKIP_COPY_TO_BINCACHE=1
# run the test.
# run the tests.
test_run ${{ matrix.arch }} qemu_uefi
test_run ${{ matrix.arch }} qemu_update
- name: Upload artifacts
# Stop the background webserver
set +e
kill %1
set -e
- name: Upload detailed test logs
if: always()
uses: actions/upload-artifact@v3
with:
name: test-results-${{ matrix.arch }}
name: ${{ matrix.arch }}-test-logs-and-results
path: |
scripts/__TESTS__
scripts/results-.*.tap
scripts/__TESTS__/*/_kola_temp/
scripts/__TESTS__/*/*.tap
scripts/__TESTS__/*/*.txt
scripts/results-*.tap
scripts/results-*.md
- name: Upload raw TAP files of all runs for later merging
if: always()
uses: actions/upload-artifact@v3
with:
name: ${{ matrix.arch }}-raw-tapfiles
path: |
scripts/__TESTS__/*/*.tap
merge_and_publish_results:
name: "Merge TAP reports and post results"
needs: tests
if: always()
runs-on:
- self-hosted
- debian
- kola
permissions:
pull-requests: write
steps:
- name: Prepare machine
shell: bash
working-directory: ${{ github.workspace }}
run: |
sudo rm /bin/sh
sudo ln -s /bin/bash /bin/sh
sudo apt-get install -y ca-certificates curl gnupg lsb-release git bzip2 jq sqlite3
- uses: actions/checkout@v3
with:
path: scripts
fetch-depth: 0
submodules: true
# Hack alert: actions/checkout will check out the (disjunct) merge commit of a PR
# instead of its head commit. That commit is not connected to any branch.
# This is not technically necessary for the tests run but it is done to remain aligned
# with the ref.
- name: If this is a PR build, use head commit instead of the merge commit
if: ${{ github.event.pull_request.head.sha }}
shell: bash
run: |
exec 2>&1
set -x
set -euo pipefail
cd scripts
git checkout ${{ github.event.pull_request.head.sha }}
git submodule update
# This is clunky. Haven't figured out how to re-use matrix.arch here for downloads,
# so we download each arch individually.
- name: Download amd64 tapfiles
uses: actions/download-artifact@v3
with:
name: amd64-raw-tapfiles
path: scripts/__TAP__/amd64
- name: Download arm64 tapfiles
uses: actions/download-artifact@v3
with:
name: arm64-raw-tapfiles
path: scripts/__TAP__/arm64
- name: Create Test Summary
shell: bash
run: |
exec 2>&1
set -x
set -euo pipefail
cd scripts
ls -laR __TAP__
source ci-automation/tapfile_helper_lib.sh
all_archs=""
for arch in __TAP__/*; do
arch_name="$(basename "${arch}")"
all_archs="${all_archs} ${arch_name}"
for vendor in "${arch}"/*; do
vendor_name="$(basename "${vendor}")"
run=1
for tap in "${vendor}"/*.tap; do
tap_ingest_tapfile "${tap}" "${vendor_name}-${arch_name}" "${run}"
((run++))
done
done
done
source sdk_container/.repo/manifests/version.txt
tap_generate_report "${all_archs}" "${FLATCAR_VERSION}" "md" "true" > test-results.md
cat test-results.md >> "$GITHUB_STEP_SUMMARY"
- name: If started from a PR, post test summary to PR
if: ${{ github.event_name == 'pull_request' }}
uses: mshick/add-pr-comment@v2
with:
message-path: "test-results.md"

16
ci-automation/ci-config.env
View File

@ -47,6 +47,11 @@ CONTAINER_IMAGE_ROOT="/home/sdk/build/images"
# echo "export PARALLEL_TESTS=\"5\"" > sdk_container/.env
# to override the number of test cases to be run in parallel.
# -- General --
# "tap" for TAP reports, "md" for markdown are currently supported
TEST_REPORT_FORMATS=("tap" "md")
# -- QEMU --
QEMU_IMAGE_NAME=${QEMU_IMAGE_NAME:-flatcar_production_image.bin}
@ -62,6 +67,17 @@ QEMU_BIOS="/usr/share/qemu/bios-256k.bin"
# Published by vms.sh as part of the qemu vendor build.
QEMU_UEFI_BIOS="${QEMU_UEFI_BIOS:-flatcar_production_qemu_uefi_efi_code.fd}"
# Update payload for the qemu_update.sh test.
# The default path set below is relative to TEST_WORK_DIR
QEMU_UPDATE_PAYLOAD="tmp/flatcar_test_update.gz"
# Devcontainer settings for isolated / local testing w/o a remote
# devcontainer server and/or binhost.
# Can optionally be set to http / https URLs to pull the dev container
# and the binpackages from.
QEMU_DEVCONTAINER_URL="${QEMU_DEVCONTAINER_URL:-}"
QEMU_DEVCONTAINER_BINHOST_URL="${QEMU_DEVCONTAINER_BINHOST_URL:-}"
# -- Equinix Metal --
EQUINIXMETAL_PARALLEL="${PARALLEL_TESTS:-4}"

176
ci-automation/tapfile_helper_lib.sh
View File

@ -197,27 +197,167 @@ function tap_failed_tests_for_vendor() {
}
# --
# TAP output format primitives for tap_generate_report()
__tap_print_header() {
local arch="$1"
local version="$2"
local vendors="$3"
local count="$4"
# We use count + 1 here because the very first "test result" will just print
# the list of platforms tested, not an actual test's result.
echo "1..$((count+1))"
echo "ok - Version: ${version}, Architecture: ${arch}"
echo " ---"
echo " Platforms tested: ${vendors}"
echo " ..."
}
# --
__tap_print_test_verdict() {
local verdict="$1"
local name="$2"
local succeded_vendors="$3"
local failed_vendors="$4"
echo "${verdict} - ${test_name}"
echo " ---"
if [ -n "${succeded_vendors}" ] ; then
echo " Succeeded: ${succeded_vendors}"
fi
if [ -n "${failed_vendors}" ] ; then
echo " Failed: ${failed_vendors}"
fi
}
# --
__tap_print_test_run_diag_output() {
local vendor="$1"
local run="$2"
echo " Error messages for ${vendor}, run ${run}:"
cat -
}
# --
__tap_finish_test_verdict() {
local verdict="$1"
local name="$2"
local succeded_vendors="$3"
local failed_vendors="$4"
echo " ..."
}
# --
__tap_finish_test_report() {
true
}
# --
# markdown output format primitives for tap_generate_report()
__md_print_header() {
local arch="$1"
local version="$2"
local vendors="$3"
local count="$4"
echo "### Test report for ${version} / ${arch}"
echo
echo "**Platforms tested** : ${vendors}"
}
# --
__md_print_test_verdict() {
local verdict="$1"
local name="$2"
local succeded_vendors="$3"
local failed_vendors="$4"
v="![${verdict}](https://via.placeholder.com/50x20/00ff00/000000?text=PASS)"
if [ "${verdict}" = "not ok" ] ; then
v="![${verdict}](https://via.placeholder.com/50x20/ff0000/ffffff?text=FAIL)"
fi
echo
echo -n "${v} **${name}**"
if [ -n "${succeded_vendors}" ] ; then
echo -n " 🟢 Succeeded: ${succeded_vendors}"
fi
if [ -n "${failed_vendors}" ] ; then
echo -n " ❌ Failed: ${failed_vendors}"
fi
echo
if [ "${verdict}" = "not ok" ] ; then
echo
echo "<details>"
echo
fi
}
# --
__md_print_test_run_diag_output() {
local vendor="$1"
local run="$2"
echo "* Diagnostic output for ${vendor}, run ${run}"
echo
echo " \`\`\`"
cat -
echo " \`\`\`"
echo
}
# --
#
__md_finish_test_verdict() {
local verdict="$1"
local name="$2"
local succeded_vendors="$3"
local failed_vendors="$4"
if [ "${verdict}" = "not ok" ] ; then
echo
echo "</details>"
echo
fi
}
# --
__md_finish_test_report() {
true
}
# --
# Print the tap file from contents of the database.
# INPUT:
# 1: <arch> - Architecture to be included in the first line of the report
# 2: <version> - OS version tested, to be included in the first line of the report
# 3: <include_transient_errors> - If set to "true" then debug output of transient test failures
# 3: <format> - Output format of the report. "tap" and "markdown" are supported.
# 4: <include_transient_errors> - If set to "true" then debug output of transient test failures
# is included in the result report.
function tap_generate_report() {
local arch="$1"
local version="$2"
local full_error_report="${3:-false}"
local format="$3"
local full_error_report="${4:-false}"
case "${format}" in
tap) ;;
md) ;;
*) echo "ERROR: tap_generate_report() unknown format '${format}'" >&2
return 1
;;
esac
local count
count="$(__sqlite3_wrapper 'SELECT count(name) FROM test_case;')"
local vendors
vendors="$(__sqlite3_wrapper 'SELECT name FROM vendor;' | tr '\n' ' ')"
echo "1..$((count+1))"
echo "ok - Version: ${version}, Architecture: ${arch}"
echo " ---"
echo " Platforms tested: ${vendors}"
echo " ..."
__"${format}"_print_header "${arch}" "${version}" "${vendors}" "${count}"
# Print result line for every test, including platforms it succeeded on
# and transient failed runs.
@ -265,21 +405,17 @@ function tap_generate_report() {
r=r ", " $2
else
r="(" $2 ; }
END { if (t) print t r ")"; }'
END { if (t) print t " " r ")"; }'
}
local succeded
succeded="$(list_runs 1)"
local succeeded
succeeded="$(list_runs 1)"
local failed
failed="$(list_runs 0)"
echo "${verdict} - ${test_name}"
echo " ---"
if [ -n "${succeded}" ] ; then
echo " Succeeded: ${succeded}"
fi
__"${format}"_print_test_verdict "${verdict}" "${test_name}" \
"${succeeded}" "${failed}"
if [ -n "${failed}" ] ; then
echo " Failed: ${failed}"
if [ "${verdict}" = "not ok" -o "${full_error_report}" = "true" ] ; then
# generate diagnostic output, per failed run.
__sqlite3_wrapper -csv "
@ -291,7 +427,7 @@ function tap_generate_report() {
ORDER BY t.run DESC;" | \
sed 's/,/ /' | \
while read -r vendor run; do
echo " Error messages for ${vendor}, run ${run}:"
{
__sqlite3_wrapper -csv "
SELECT t.output FROM test_run AS t, test_case AS c
WHERE t.case_id=c.id
@ -299,10 +435,14 @@ function tap_generate_report() {
AND t.run='${run}';" | \
sed 's/"/ /g' | \
awk '{print " L" NR ": \"" $0 "\""}'
} | __"${format}"_print_test_run_diag_output "${vendor}" "${run}"
done
fi
fi
echo " ..."
__"${format}"_finish_test_verdict "${verdict}" "${test_name}" \
"${succeeded}" "${failed}"
done
__"${format}"_finish_test_report
}
# --

8
ci-automation/test.sh
View File

@ -165,8 +165,8 @@ function _test_run_impl() {
# Make the torcx artifacts available to test implementation
__prepare_torcx "${arch}" "${vernum}" "${work_dir}"
local tap_merged_summary="results-${image}.tap"
local tap_merged_detailed="results-${image}-detailed.tap"
local tap_merged_summary="results-${image}"
local tap_merged_detailed="results-${image}-detailed"
local retry=""
local success=false
local print_give_up=true
@ -242,9 +242,9 @@ function _test_run_impl() {
copy_to_buildcache "testing/${vernum}/${arch}/${image}" \
"${tests_dir}/"*.tap
copy_to_buildcache "testing/${vernum}/${arch}/${image}" \
"${tap_merged_summary}"
"${tap_merged_summary}"*
copy_to_buildcache "testing/${vernum}/${arch}/${image}" \
"${tap_merged_detailed}"
"${tap_merged_detailed}"*
fi
if ! $success; then
return 1

8
ci-automation/test_update_reruns.sh
View File

@ -19,8 +19,12 @@ failfile="$6"
merged_summary="$7"
merged_detailed="$8"
source ci-automation/ci-config.env
source ci-automation/tapfile_helper_lib.sh
tap_ingest_tapfile "${tapfile}" "${image}" "${retry}"
tap_failed_tests_for_vendor "${image}" > "${failfile}"
tap_generate_report "${arch}" "${vernum}" > "${merged_summary}"
tap_generate_report "${arch}" "${vernum}" "true" > "${merged_detailed}"
for format in "${TEST_REPORT_FORMATS[@]}"; do
tap_generate_report "${arch}" "${vernum}" "${format}" > "${merged_summary}.${format}"
tap_generate_report "${arch}" "${vernum}" "${format}" "true" > "${merged_detailed}.${format}"
done

11
ci-automation/vendor-testing/qemu.sh
View File

@ -42,6 +42,16 @@ if [ "${CIA_TESTSCRIPT}" = "qemu_uefi.sh" ] ; then
fi
fi
declare -a devcontainer_opts
if [ -n "${QEMU_DEVCONTAINER_URL}" ] ; then
echo "++++ Using custom devcontainer URL '${QEMU_DEVCONTAINER_URL}'"
devcontainer_opts+=( "--devcontainer-url" "${QEMU_DEVCONTAINER_URL}" )
fi
if [ -n "${QEMU_DEVCONTAINER_BINHOST_URL}" ] ; then
echo "++++ Using custom devcontainer binhost '${QEMU_DEVCONTAINER_BINHOST_URL}'"
devcontainer_opts+=( "--devcontainer-binhost-url" "${QEMU_DEVCONTAINER_BINHOST_URL}" )
fi
set -x
kola run \
@ -53,6 +63,7 @@ kola run \
--tapfile="${CIA_TAPFILE}" \
--torcx-manifest="${CIA_TORCX_MANIFEST}" \
${QEMU_KOLA_SKIP_MANGLE:+--qemu-skip-mangle} \
"${devcontainer_opts[@]}" \
"${@}"
set +x

8
ci-automation/vendor-testing/qemu_update.sh
View File

@ -23,9 +23,9 @@ if [ "$*" != "" ] && [ "$*" != "*" ] && [[ "$*" != *"cl.update.payload" ]]; then
exit 1
fi
mkdir -p tmp/
if [ -f tmp/flatcar_test_update.gz ] ; then
echo "++++ ${CIA_TESTSCRIPT}: Using existing ./tmp/flatcar_test_update.gz for testing ${CIA_VERNUM} (${CIA_ARCH}) ++++"
mkdir -p "$(dirname ${QEMU_UPDATE_PAYLOAD})"
if [ -f "${QEMU_UPDATE_PAYLOAD}" ] ; then
echo "++++ ${CIA_TESTSCRIPT}: Using existing ${QEMU_UPDATE_PAYLOAD} for testing ${CIA_VERNUM} (${CIA_ARCH}) ++++"
else
echo "++++ ${CIA_TESTSCRIPT}: downloading flatcar_test_update.gz for ${CIA_VERNUM} (${CIA_ARCH}) ++++"
copy_from_buildcache "images/${CIA_ARCH}/${CIA_VERNUM}/flatcar_test_update.gz" tmp/
@ -105,7 +105,7 @@ run_kola_tests() {
--qemu-image="${image}" \
--tapfile="${instance_tapfile}" \
--torcx-manifest="${CIA_TORCX_MANIFEST}" \
--update-payload=tmp/flatcar_test_update.gz \
--update-payload="${QEMU_UPDATE_PAYLOAD}" \
${QEMU_KOLA_SKIP_MANGLE:+--qemu-skip-mangle} \
cl.update.payload
}