mirrors/flatcar-scripts
1
0
Fork 0
mirror of https://github.com/flatcar/scripts.git synced 2025-12-24 10:41:51 +01:00
Code Issues Packages Projects Releases Wiki Activity
flatcar-scripts/.github/workflows/ci.yaml
Thilo Fromm f07cb5f781 tapfile_helper ff.: support TAP and Markdown output 
This change adds markdown output support to tapfile helper.
tap_generate_report() has been refactored to use low-level output
functions to write tests; TAP and markdown output is supported and both
are generated by default. Also, it should be straightforward to add
other output formats by implementing the respective low level print
functions.

The markdown output is now used by run-kola-tests.yaml to generate step
output and, if run from a PR, add a comment with test results to the PR.

Signed-off-by: Thilo Fromm <thilofromm@microsoft.com>
2023-03-29 21:05:47 +02:00

358 lines
14 KiB
YAML
Raw Blame History

name: "Run build"
on:
pull_request:
# Run when the PR is ready and each time a review is re-requested
# (i.e. after feedback has been addressed).
types: [review_requested, ready_for_review]
workflow_dispatch:
inputs:
image_formats:
description: |
Space-separated vendor formats to build.
required: true
default: qemu_uefi
portage_remote:
description: |
The remote we should pull portage-stable from. This defaults to whatever the submodule is set to in this repo.
If triggered by a change in the portage repo, please set this to the remote which is proposing a change.
required: false
portage_ref:
description: |
This is the ref we will use to pull the changes from the portage_remote.
required: false
coreos_remote:
description: |
The remote we should pull coreos-overlay from. This defaults to whatever the submodule is set to in this repo.
If triggered by a change in the portage repo, please set this to the remote which is proposing a change.
required: false
coreos_ref:
description: |
This is the ref we will use to pull the changes from the coreos_remote.
required: false
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }}
cancel-in-progress: true
permissions:
pull-requests: write
jobs:
packages:
# Do not run when still in draft mode but a review was requested anyway
if: github.event.pull_request.draft == false
name: "Build Flatcar packages"
runs-on:
- self-hosted
- debian
- build
- x64
strategy:
fail-fast: false
matrix:
arch: ["amd64", "arm64"]
defaults:
run:
working-directory: scripts
steps:
- name: Prepare machine
shell: bash
working-directory: ${{ github.workspace }}
run: |
sudo rm /bin/sh
sudo ln -s /bin/bash /bin/sh
sudo apt-get install -y ca-certificates curl gnupg lsb-release qemu-user-static git
sudo mkdir -p /etc/apt/keyrings
curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
echo \
"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \
$(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
sudo apt-get update
sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin
- uses: actions/checkout@v3
with:
path: scripts
fetch-depth: 0
submodules: true
# Hack alert: actions/checkout will check out the (disjunct) merge commit of a PR
# instead of its head commit. That commit is not connected to any branch.
# This causes breakage downstream e.g. when the devcontainer test wants to check out
# the ref in the scripts repo that corresponds to this build.
- name: If this is a PR build, use head commit instead of the merge commit
if: ${{ github.event.pull_request.head.sha }}
shell: bash
run: |
exec 2>&1
set -x
set -euo pipefail
git checkout ${{ github.event.pull_request.head.sha }}
git submodule update
- name: Set environment
shell: bash
run: |
arch="${{ matrix.arch }}"
COREOS_REMOTE=""
COREOS_REF=""
PORTAGE_REMOTE=""
PORTAGE_REF=""
IMAGE_FORMATS="qemu_uefi"
[ -z "${{ github.event.inputs.coreos_remote }}" ] || COREOS_REMOTE="${{ github.event.inputs.coreos_remote }}"
[ -z "${{ github.event.inputs.coreos_ref }}" ] || COREOS_REF="${{ github.event.inputs.coreos_ref }}"
[ -z "${{ github.event.inputs.portage_remote }}" ] || PORTAGE_REMOTE="${{ github.event.inputs.portage_remote }}"
[ -z "${{ github.event.inputs.portage_ref }}" ] || PORTAGE_REF="${{ github.event.inputs.portage_ref }}"
[ -z "${{ github.event.inputs.image_formats }}" ] || IMAGE_FORMATS="${{ github.event.inputs.image_formats }}"
echo "arch=${arch}" >> $GITHUB_ENV
echo "COREOS_REMOTE=${COREOS_REMOTE}" >> $GITHUB_ENV
echo "COREOS_REF=${COREOS_REF}" >> $GITHUB_ENV
echo "PORTAGE_REMOTE=${PORTAGE_REMOTE}" >> $GITHUB_ENV
echo "PORTAGE_REF=${PORTAGE_REF}" >> $GITHUB_ENV
echo "IMAGE_FORMATS=${IMAGE_FORMATS}" >> $GITHUB_ENV
# Artifact root for images and torcx tarball as seen from within the container
echo "CI_CONTAINER_ARTIFACT_ROOT=/home/sdk/trunk/src/scripts/artifacts" >> $GITHUB_ENV
echo "CI_CONTAINER_TORCX_ROOT=/home/sdk/trunk/src/scripts/artifacts/torcx" >> $GITHUB_ENV
mkdir -p artifacts/torcx
# Placeholder URL for run-kola-tests.yaml, "Extract artifacts" step which will replace
# this with its IP address.
echo "TORCX_TESTS_PACKAGE_URL=http://localhost:12345" >> $GITHUB_ENV
- name: Checkout submodules
shell: bash
run: |
if [ "${COREOS_REMOTE}" != "" -a "${COREOS_REF}" != "" ]
then
REMOTE="${COREOS_REMOTE}"
REPO_PATH="sdk_container/src/third_party/coreos-overlay"
[[ "$REMOTE" == "https:*" ]] || REMOTE="https://github.com/${COREOS_REMOTE}"
git -C "$REPO_PATH" remote add test "$REMOTE"
git -C "$REPO_PATH" fetch test
git -C "$REPO_PATH" checkout "test/${COREOS_REF}"
fi
if [ "${PORTAGE_REMOTE}" != "" -a "${PORTAGE_REF}" != "" ]
then
REMOTE="${PORTAGE_REMOTE}"
REPO_PATH="sdk_container/src/third_party/portage-stable"
[[ "$REMOTE" == "https:*" ]] || REMOTE="https://github.com/${PORTAGE_REMOTE}"
git -C "$REPO_PATH" remote add test "$REMOTE"
git -C "$REPO_PATH" fetch test
git -C "$REPO_PATH" checkout "test/${PORTAGE_REF}"
fi
- name: Build packages
shell: bash
run: |
exec 2>&1
set -x
set -euo pipefail
source ci-automation/ci_automation_common.sh
source sdk_container/.repo/manifests/version.txt
version="alpha-$FLATCAR_VERSION_ID"
check_version_string "$version"
sdk_version="${FLATCAR_SDK_VERSION}"
sdk_name="flatcar-sdk-${arch}"
docker_sdk_vernum="$(vernum_to_docker_image_version "${sdk_version}")"
docker_image_from_registry_or_buildcache "${sdk_name}" "${docker_sdk_vernum}"
sdk_image="$(docker_image_fullname "${sdk_name}" "${docker_sdk_vernum}")"
container_name="flatcar-ci-build"
echo "container_name=${container_name}" >> "$GITHUB_ENV"
# Create version file
(
source sdk_lib/sdk_container_common.sh
create_versionfile "$sdk_version" "$version"
)
# Run the packages build. This will create the ci build container
# which will be re-used by subsequent build steps.
./run_sdk_container -n "${container_name}" -v "${version}" \
-C "${sdk_image}" \
./build_packages --board="${arch}-usr" \
--torcx_output_root="${CI_CONTAINER_TORCX_ROOT}" \
--torcx_extra_pkg_url="${TORCX_TESTS_PACKAGE_URL}"
# Create binpkgs tarball for archiving as artifact later
./run_sdk_container -n "${container_name}" \
tar -C "/build/${arch}-usr/var/lib/portage/pkgs/" \
-cvf binpkgs.tar .
- name: Extract build logs
if: always()
shell: bash
run: |
set -euo pipefail
set -x
# Copy logs
./run_sdk_container -n "${container_name}" \
tar -cJf ebuild_logs.tar.xz /build/${arch}-usr/var/log/portage \
/build/${arch}-usr/var/tmp/portage
- name: Upload build logs
if: always()
uses: actions/upload-artifact@v3
with:
retention-days: 7
name: ${{ matrix.arch }}-build-logs
path: |
scripts/ebuild_logs.tar.xz
- name: Build image
shell: bash
run: |
set -euo pipefail
set -x
echo 'channel="developer"' >> $GITHUB_ENV
channel="developer"
source ci-automation/ci_automation_common.sh
official_arg="--noofficial"
./run_sdk_container -n "${container_name}" \
./set_official --board="${arch}-usr" "${official_arg}"
./run_sdk_container -n "${container_name}" \
./build_image --board="${arch}-usr" --group="${channel}" \
--output_root="${CI_CONTAINER_ARTIFACT_ROOT}" \
--torcx_root="${CI_CONTAINER_TORCX_ROOT}" prodtar container
- name: Build VM image(s)
shell: bash
run: |
set -euo pipefail
set -x
source ci-automation/ci_automation_common.sh
images_out="images"
has_packet=0
has_pxe=0
formats="${IMAGE_FORMATS}"
for format in "${formats}";do
[[ "${format}" = 'packet' ]] || [[ "${format}" = 'equinix_metal' ]] && has_packet=1
[[ "${format}" = 'pxe' ]] && has_pxe=1
done
[[ ${has_packet} -eq 1 ]] && [[ ${has_pxe} -eq 0 ]] && set -- 'pxe' "${@}"
if echo "$formats" | tr ' ' '\n' | grep -q '^vmware'; then
formats=$(echo "$formats" | tr ' ' '\n' | sed '/vmware.*/d')
formats+=" vmware vmware_insecure vmware_ova vmware_raw"
fi
if echo "$formats" | tr ' ' '\n' | grep -q -P '^(ami|aws)'; then
formats=$(echo "$formats" | tr ' ' '\n' | sed '/ami.*/d' | sed '/aws/d')
formats+=" ami ami_vmdk"
fi
# Keep compatibility with SDK scripts where "equinix_metal" remains unknown.
formats=$(echo "$formats" | tr ' ' '\n' | sed 's/equinix_metal/packet/g')
for format in ${formats}; do
echo " ################### VENDOR '${format}' ################### "
./run_sdk_container -n "${container_name}" \
./image_to_vm.sh --format "${format}" --board="${arch}-usr" \
--from "${CI_CONTAINER_ARTIFACT_ROOT}/${arch}-usr/latest" \
--image_compression_formats=bz2
done
# upload-artifacts cannot handle artifact uploads from sym-linked directories (no, really)
# so we move things around.
mkdir -p artifacts/images
(
cd artifacts/${arch}-usr/latest/
mv * ../../images/
)
# create a tarball for torcx package + JSON file because upload-artifacts cannot handle filenames containing colons
# (such as "docker:20.10.torcx.tgz")
mv artifacts/torcx/${arch}-usr/latest/torcx_manifest.json artifacts/torcx/pkgs/
tar -C artifacts/torcx/pkgs/ -cvf torcx.tar .
- name: Upload binpkgs
uses: actions/upload-artifact@v3
with:
retention-days: 7
name: ${{ matrix.arch }}-binpkgs
path: |
scripts/binpkgs.tar
- name: Upload update image (used with kola tests later)
uses: actions/upload-artifact@v3
with:
retention-days: 7
name: ${{ matrix.arch }}-test-update
path: |
scripts/artifacts/images/flatcar_test_update.gz
- name: Upload generic image
uses: actions/upload-artifact@v3
with:
retention-days: 7
name: ${{ matrix.arch }}-generic-image
path: |
scripts/artifacts/images/flatcar_production_image.bin.bz2
scripts/artifacts/images/flatcar_production_image.grub
scripts/artifacts/images/flatcar_production_image.shim
scripts/artifacts/images/flatcar_production_image.vmlinuz
scripts/artifacts/images/flatcar_production_image*.txt
scripts/artifacts/images/flatcar_production_image*.json
scripts/artifacts/images/flatcar_production_image_pcr_policy.zip
scripts/artifacts/images/flatcar_production_*_efi_*.fd
- name: Upload developer container
uses: actions/upload-artifact@v3
with:
retention-days: 7
name: ${{ matrix.arch }}-devcontainer
path: |
scripts/artifacts/images/flatcar_developer_container*
- name: Upload torcx tarball
uses: actions/upload-artifact@v3
with:
retention-days: 7
name: ${{ matrix.arch }}-torcx
path: |
scripts/torcx.tar
# Clean up what we uploaded already so the "vendor images" wildcard
# works when uploading artifacts in the next step.
- name: Remove update, generic and devcontainer images
shell: bash
run: |
set -euo pipefail
set -x
rm -f artifacts/images/flatcar_test_update.gz \
artifacts/images/flatcar_production_image* \
artifacts/images/flatcar_developer_container* \
artifacts/images/flatcar_production_update*
- name: Upload vendor images
uses: actions/upload-artifact@v3
with:
retention-days: 7
name: ${{ matrix.arch }}-vm-images
path: |
scripts/artifacts/images/*.img.bz2
scripts/artifacts/images/*.bin.bz2
scripts/artifacts/images/flatcar_production_*_efi_*.fd
scripts/artifacts/images/*.txt
scripts/artifacts/images/flatcar_production_*.sh
test:
needs: packages
name: "Run kola tests"
uses: ./.github/workflows/run-kola-tests.yaml
Reference in New Issue View Git Blame Copy Permalink