From 28b26c87c19854021426927a882bb72e695874d1 Mon Sep 17 00:00:00 2001 From: Thilo Fromm Date: Wed, 15 Mar 2023 10:43:09 +0100 Subject: [PATCH] ci.yaml: re-use build container, finer grained artifact upload This change removes "docker commit" at the end of each step and instead makes build steps re-use the build container, saving some build time. It also makes artifact upload more granular, so build logs, images, and dev container can be downloaded individually. Lastly, it exports torcx tarball and binary packages as a separate artifact each, for successive re-use in the kola tests. --- .github/workflows/ci.yaml | 202 +++++++++++++++++++++++--------------- 1 file changed, 121 insertions(+), 81 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e008c7cbb7..58d01dd4db 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -3,11 +3,6 @@ on: pull_request: workflow_dispatch: inputs: - bincache_server: - description: | - Bincache server. - default: "bincache.flatcar-linux.net" - required: true image_formats: description: | Space-separated vendor formats to build. @@ -79,7 +74,6 @@ jobs: - name: Set environment shell: bash run: | - BUILDCACHE_SERVER="bincache.flatcar-linux.net" arch="${{ matrix.arch }}" COREOS_REMOTE="" COREOS_REF="" @@ -87,14 +81,12 @@ jobs: PORTAGE_REF="" IMAGE_FORMATS="qemu_uefi" - [ -z "${{ github.event.inputs.bincache_server }}" ] || BUILDCACHE_SERVER="${{ github.event.inputs.bincache_server }}" [ -z "${{ github.event.inputs.coreos_remote }}" ] || COREOS_REMOTE="${{ github.event.inputs.coreos_remote }}" [ -z "${{ github.event.inputs.coreos_ref }}" ] || COREOS_REF="${{ github.event.inputs.coreos_ref }}" [ -z "${{ github.event.inputs.portage_remote }}" ] || PORTAGE_REMOTE="${{ github.event.inputs.portage_remote }}" [ -z "${{ github.event.inputs.portage_ref }}" ] || PORTAGE_REF="${{ github.event.inputs.portage_ref }}" [ -z "${{ github.event.inputs.image_formats }}" ] || IMAGE_FORMATS="${{ github.event.inputs.image_formats }}" - echo "BUILDCACHE_SERVER=${BUILDCACHE_SERVER}" >> $GITHUB_ENV echo "arch=${arch}" >> $GITHUB_ENV echo "COREOS_REMOTE=${COREOS_REMOTE}" >> $GITHUB_ENV echo "COREOS_REF=${COREOS_REF}" >> $GITHUB_ENV @@ -102,6 +94,15 @@ jobs: echo "PORTAGE_REF=${PORTAGE_REF}" >> $GITHUB_ENV echo "IMAGE_FORMATS=${IMAGE_FORMATS}" >> $GITHUB_ENV + # Artifact root for images and torcx tarball as seen from within the container + echo "CI_CONTAINER_ARTIFACT_ROOT=/home/sdk/trunk/src/scripts/artifacts" >> $GITHUB_ENV + echo "CI_CONTAINER_TORCX_ROOT=/home/sdk/trunk/src/scripts/artifacts/torcx" >> $GITHUB_ENV + mkdir -p artifacts/torcx + + # Placeholder URL for run-kola-tests.yaml, "Extract artifacts" step which will replace + # this with its IP address. + echo "TORCX_TESTS_PACKAGE_URL=http://localhost:12345" >> $GITHUB_ENV + - name: Checkout submodules shell: bash run: | @@ -129,7 +130,7 @@ jobs: shell: bash run: | exec 2>&1 - set +x + set -x set -euo pipefail source ci-automation/ci_automation_common.sh @@ -144,92 +145,70 @@ jobs: docker_image_from_registry_or_buildcache "${sdk_name}" "${docker_sdk_vernum}" sdk_image="$(docker_image_fullname "${sdk_name}" "${docker_sdk_vernum}")" - vernum="${version#*-}" # remove main-,alpha-,beta-,stable-,lts- version tag - docker_vernum="$(vernum_to_docker_image_version "${vernum}")" - packages_container="flatcar-packages-${arch}-${docker_vernum}" + container_name="flatcar-ci-build" # Create version file ( source sdk_lib/sdk_container_common.sh create_versionfile "$sdk_version" "$version" ) - ./run_sdk_container -n "${packages_container}" -v "${version}" \ + + # Run the packages build. This will create the ci build container + # which will be re-used by subsequent build steps. + ./run_sdk_container -n "${container_name}" -v "${version}" \ -C "${sdk_image}" \ ./build_packages --board="${arch}-usr" \ - --torcx_output_root="${CONTAINER_TORCX_ROOT}" + --torcx_output_root="${CI_CONTAINER_TORCX_ROOT}" \ + --torcx_extra_pkg_url="${TORCX_TESTS_PACKAGE_URL}" - # copy torcx manifest and docker tarball for publishing - torcx_tmp="__build__/torcx_tmp" - rm -rf "${torcx_tmp}" - mkdir "${torcx_tmp}" - ./run_sdk_container -n "${packages_container}" -v "${version}" \ - -C "${sdk_image}" \ - cp -r "${CONTAINER_TORCX_ROOT}/" \ - "${torcx_tmp}" + # Copy logs + ./run_sdk_container -n "${container_name}" \ + tar -cJf ebuild_logs.tar.xz /build/${arch}-usr/var/log/portage \ + /build/${arch}-usr/var/tmp/portage - source sdk_container/.repo/manifests/version.txt - vernum="${FLATCAR_VERSION}" - docker_vernum="$(vernum_to_docker_image_version "${vernum}")" - packages_image="flatcar-packages-${arch}" + # Create binpkgs tarball for archiving as artifact later + ./run_sdk_container -n "${container_name}" \ + tar -C "/build/${arch}-usr/var/lib/portage/pkgs/" \ + -cvf binpkgs.tar . - echo "vernum=${vernum}" >> $GITHUB_ENV - echo "docker_vernum=${docker_vernum}" >> $GITHUB_ENV - echo "packages_image=${packages_image}" >> $GITHUB_ENV - echo "arch=${arch}" >> $GITHUB_ENV - echo "sdk_image=${sdk_image}" >> $GITHUB_ENV - echo "packages_container=${packages_container}" >> $GITHUB_ENV - docker commit "${packages_container}" "${packages_image}:${docker_vernum}" - docker rm -f "${packages_container}" + echo "container_name=${container_name}" >> "$GITHUB_ENV" + + - name: Upload build logs + uses: actions/upload-artifact@v3 + with: + retention-days: 7 + name: ${{ matrix.arch }}-build-logs + path: | + scripts/ebuild_logs.tar.xz - name: Build image shell: bash run: | set -euo pipefail - set +x + set -x echo 'channel="developer"' >> $GITHUB_ENV channel="developer" source ci-automation/ci_automation_common.sh - packages="flatcar-packages-${arch}" - packages_image="${packages}:${docker_vernum}" - image="flatcar-images-${arch}" - image_container="${image}-${docker_vernum}" official_arg="--noofficial" - echo "image=flatcar-images-${arch}" >> $GITHUB_ENV - echo "image_image=${image}:${docker_vernum}" >> $GITHUB_ENV - - ./run_sdk_container -x ./ci-cleanup.sh -n "${image_container}" -C "${packages_image}" \ - -v "${vernum}" \ - mkdir -p "${CONTAINER_IMAGE_ROOT}" - ./run_sdk_container -n "${image_container}" -C "${packages_image}" \ - -v "${vernum}" \ + ./run_sdk_container -n "${container_name}" \ ./set_official --board="${arch}-usr" "${official_arg}" - ./run_sdk_container -n "${image_container}" -C "${packages_image}" \ - -v "${vernum}" \ + ./run_sdk_container -n "${container_name}" \ ./build_image --board="${arch}-usr" --group="${channel}" \ - --output_root="${CONTAINER_IMAGE_ROOT}" \ - --torcx_root="${CONTAINER_TORCX_ROOT}" prodtar container + --output_root="${CI_CONTAINER_ARTIFACT_ROOT}" \ + --torcx_root="${CI_CONTAINER_TORCX_ROOT}" prodtar container - # Copy logs - ./run_sdk_container -n "${image_container}" -C "${packages_image}" -v "${vernum}" \ - tar -cJf ebuild_logs.tar.xz /build/${arch}-usr/var/log/portage \ - /build/${arch}-usr/var/tmp/portage - - docker commit "${image_container}" "${image}:${docker_vernum}" - docker rm -f "${image_container}" - - - name: Build VM image + - name: Build VM image(s) shell: bash run: | set -euo pipefail - set +x + set -x source ci-automation/ci_automation_common.sh - vms_container="flatcar-vms-${docker_vernum}" images_out="images" has_packet=0 @@ -254,35 +233,96 @@ jobs: for format in ${formats}; do echo " ################### VENDOR '${format}' ################### " - ./run_sdk_container -n "${vms_container}" -C "${image_image}" \ - -v "${vernum}" \ + ./run_sdk_container -n "${container_name}" \ ./image_to_vm.sh --format "${format}" --board="${arch}-usr" \ - --from "${CONTAINER_IMAGE_ROOT}/${arch}-usr/latest" \ + --from "${CI_CONTAINER_ARTIFACT_ROOT}/${arch}-usr/latest" \ --image_compression_formats=bz2 done - # copy resulting images - ./run_sdk_container -n "${vms_container}" \ - -v "${vernum}" \ - mv "${CONTAINER_IMAGE_ROOT}/${arch}-usr" "./${images_out}" + # upload-artifacts cannot handle artifact uploads from sym-linked directories (no, really) + # so we move things around. + mkdir -p artifacts/images + ( + cd artifacts/${arch}-usr/latest/ + mv * ../../images/ + ) - # remove symlinks before upload - find "./${images_out}" -type l -delete + # create a tarball for torcx package + JSON file because upload-artifacts cannot handle filenames containing colons + # (such as "docker:20.10.torcx.tgz") + mv artifacts/torcx/${arch}-usr/latest/torcx_manifest.json artifacts/torcx/pkgs/ + tar -C artifacts/torcx/pkgs/ -cvf torcx.tar . - docker rm -f "${vms_container}" - - name: Upload artifacts + - name: Upload binpkgs uses: actions/upload-artifact@v3 with: - name: images-${{ matrix.arch }} + retention-days: 7 + name: ${{ matrix.arch }}-binpkgs path: | - scripts/images/**/*.img.bz2 - scripts/images/**/*.bin.bz2 - scripts/images/**/flatcar_production_*_efi_*.fd - scripts/images/**/*.txt - scripts/images/**/flatcar_production_*.sh - scripts/images/**/flatcar_test_update.gz - scripts/ebuild_logs.tar.xz + scripts/binpkgs.tar + + - name: Upload update image (used with kola tests later) + uses: actions/upload-artifact@v3 + with: + retention-days: 7 + name: ${{ matrix.arch }}-test-update + path: | + scripts/artifacts/images/flatcar_test_update.gz + + - name: Upload generic image + uses: actions/upload-artifact@v3 + with: + retention-days: 7 + name: ${{ matrix.arch }}-generic-image + path: | + scripts/artifacts/images/flatcar_production_image.bin.bz2 + scripts/artifacts/images/flatcar_production_image.grub + scripts/artifacts/images/flatcar_production_image.shim + scripts/artifacts/images/flatcar_production_image.vmlinuz + scripts/artifacts/images/flatcar_production_image*.txt + scripts/artifacts/images/flatcar_production_image*.json + scripts/artifacts/images/flatcar_production_image_pcr_policy.zip + scripts/artifacts/images/flatcar_production_*_efi_*.fd + + - name: Upload developer container + uses: actions/upload-artifact@v3 + with: + retention-days: 7 + name: ${{ matrix.arch }}-devcontainer + path: | + scripts/artifacts/images/flatcar_developer_container* + + - name: Upload torcx tarball + uses: actions/upload-artifact@v3 + with: + retention-days: 7 + name: ${{ matrix.arch }}-torcx + path: | + scripts/torcx.tar + + # Clean up what we uploaded already so the "vendor images" wildcard + # works when uploading artifacts in the next step. + - name: Remove update, generic and devcontainer images + shell: bash + run: | + set -euo pipefail + set -x + rm -f artifacts/images/flatcar_test_update.gz \ + artifacts/images/flatcar_production_image* \ + artifacts/images/flatcar_developer_container* \ + artifacts/images/flatcar_production_update* + + - name: Upload vendor images + uses: actions/upload-artifact@v3 + with: + retention-days: 7 + name: ${{ matrix.arch }}-vm-images + path: | + scripts/artifacts/images/*.img.bz2 + scripts/artifacts/images/*.bin.bz2 + scripts/artifacts/images/flatcar_production_*_efi_*.fd + scripts/artifacts/images/*.txt + scripts/artifacts/images/flatcar_production_*.sh test: needs: packages