flatcar-scripts/.github/workflows/ci.yaml

name: "Run build"
on:
  workflow_dispatch:
    inputs:
      image_formats:
        type: string
        description: |
          Space-separated vendor formats to build.
        required: true
        default: qemu_uefi
      custom_sdk_version:
        type: string
        required: false
        description: |
          Custom SDK container version to use for this build.

  workflow_call:
    inputs:
      image_formats:
        type: string
        description: |
          Space-separated vendor formats to build.
        required: true
        default: qemu_uefi
      custom_sdk_version:
        type: string
        required: false
        description: |
          Custom SDK container version to use for this build.

permissions:
  pull-requests: write

jobs:
  packages:
    name: "Build Flatcar packages"
    runs-on:
      - self-hosted
      - debian
      - build
      - x64
    strategy:
      fail-fast: false
      matrix:
        arch: ["amd64", "arm64"]
    defaults:
      run:
        working-directory: scripts

    steps:
      - name: Prepare machine
        shell: bash
        working-directory: ${{ github.workspace }}
        run: |
          sudo rm /bin/sh
          sudo ln -s /bin/bash /bin/sh
          sudo apt-get update
          sudo apt-get install -y ca-certificates curl git gnupg lsb-release python3 qemu-user-static zstd
          sudo mkdir -p /etc/apt/keyrings
          curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
          echo \
            "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \
            $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
          sudo apt-get update
          sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin

      - name: Checkout scripts
        uses: actions/checkout@v3
        with:
          path: scripts
          fetch-depth: 0

      - name: Checkout build scripts
        uses: actions/checkout@v3
        with:
          repository: flatcar/flatcar-build-scripts
          path: flatcar-build-scripts

      # Hack alert: actions/checkout will check out the (disjunct) merge commit of a PR
      #  instead of its head commit. That commit is not connected to any branch.
      # This causes breakage downstream e.g. when the devcontainer test wants to check out
      #  the ref in the scripts repo that corresponds to this build.
      - name: If this is a PR build, use head commit instead of the merge commit
        if: ${{ github.event.pull_request.head.sha }}
        shell: bash
        run: |
          exec 2>&1
          set -x
          set -euo pipefail

          git checkout ${{ github.event.pull_request.head.sha }}

      - name: Set environment
        shell: bash
        run: |
          arch="${{ matrix.arch }}"
          echo "arch=${arch}" >> $GITHUB_ENV

          IMAGE_FORMATS="qemu_uefi"
          [ -z "${{ inputs.image_formats }}" ] || IMAGE_FORMATS="${{ inputs.image_formats }}"
          echo "IMAGE_FORMATS=${IMAGE_FORMATS}" >> $GITHUB_ENV

          # Artifact root for images and torcx tarball as seen from within the container
          echo "CI_CONTAINER_ARTIFACT_ROOT=/home/sdk/trunk/src/scripts/artifacts" >> $GITHUB_ENV
          echo "CI_CONTAINER_TORCX_ROOT=/home/sdk/trunk/src/scripts/artifacts/torcx" >> $GITHUB_ENV
          mkdir -p artifacts/torcx

          # Placeholder URL for run-kola-tests.yaml, "Extract artifacts" step which will replace
          #  this with its IP address.
          echo "TORCX_TESTS_PACKAGE_URL=http://localhost:12345" >> $GITHUB_ENV

          if [ -n "${{ inputs.custom_sdk_version }}" ] ; then
              echo "CUSTOM_SDK_VERSION=${{ inputs.custom_sdk_version }}" >> $GITHUB_ENV
          fi

      - name: Build packages
        shell: bash
        run: |
          exec 2>&1
          set -x
          set -euo pipefail

          # This is also done again in run-kola-tests.yaml because these changes here disappear
          source ci-automation/ci_automation_common.sh
          source sdk_container/.repo/manifests/version.txt

          version="alpha-$FLATCAR_VERSION_ID"
          check_version_string "$version"
          sdk_version="${CUSTOM_SDK_VERSION:-$FLATCAR_SDK_VERSION}"

          sdk_name="flatcar-sdk-${arch}"
          docker_sdk_vernum="$(vernum_to_docker_image_version "${sdk_version}")"
          docker_image_from_registry_or_buildcache "${sdk_name}" "${docker_sdk_vernum}"
          sdk_image="$(docker_image_fullname "${sdk_name}" "${docker_sdk_vernum}")"

          container_name="flatcar-ci-build"
          echo "container_name=${container_name}" >> "$GITHUB_ENV"

          # Create version file
          (
            source sdk_lib/sdk_container_common.sh
            create_versionfile "$sdk_version" "$version"
          )

          # Run the packages build. This will create the ci build container
          #  which will be re-used by subsequent build steps.
          ./run_sdk_container -n "${container_name}" -v "${version}" \
            -C "${sdk_image}" \
            ./build_packages --board="${arch}-usr" \
                --torcx_output_root="${CI_CONTAINER_TORCX_ROOT}" \
                --torcx_extra_pkg_url="${TORCX_TESTS_PACKAGE_URL}"

          # Create binpkgs tarball for archiving as artifact later
          ./run_sdk_container -n "${container_name}" \
              tar -C "/build/${arch}-usr/var/lib/portage/pkgs/" \
                -cvf binpkgs.tar .

      - name: Extract build logs
        if: always() && !cancelled()
        shell: bash
        run: |
          set -euo pipefail
          set -x
          # Copy logs
          ./run_sdk_container -n "${container_name}" \
              tar -cJf ebuild_logs.tar.xz /build/${arch}-usr/var/log/portage \
              /build/${arch}-usr/var/tmp/portage

      - name: Upload build logs
        if: always() && !cancelled()
        uses: actions/upload-artifact@v3
        with:
          retention-days: 7
          name: ${{ matrix.arch }}-build-logs
          path: |
            scripts/ebuild_logs.tar.xz

      - name: Build image
        shell: bash
        run: |
          set -euo pipefail
          set -x

          echo 'channel="developer"' >> $GITHUB_ENV
          channel="developer"

          source ci-automation/ci_automation_common.sh

          official_arg="--noofficial"

          ./run_sdk_container -n  "${container_name}" \
                  ./set_official --board="${arch}-usr" "${official_arg}"
          ./run_sdk_container -n "${container_name}" \
                  ./build_image --board="${arch}-usr" --group="${channel}" \
                                --output_root="${CI_CONTAINER_ARTIFACT_ROOT}" \
                                --torcx_root="${CI_CONTAINER_TORCX_ROOT}" prodtar container

      - name: Generate reports
        shell: bash
        run: |
          set -euo pipefail
          set -x

          source ci-automation/image_changes.sh

          channel=alpha
          vernum=$(source sdk_container/.repo/manifests/version.txt; echo "${FLATCAR_VERSION}")
          board="${arch}-usr"

          package_diff_env=(
              "FROM_B=file://${PWD}/artifacts/${arch}-usr/latest"
              # BOARD_B and CHANNEL_B are unused.
          )
          package_diff_params_b=(
              # The package-diff script appends version to the file
              # URL, but the directory with the image has no version
              # component at its end, so we use . as a version.
              '.'
          )
          size_changes_env=(
              # Nothing to add.
          )
          size_changes_params_b=(
              "local:${PWD}/artifacts/${arch}-usr/latest"
          )
          show_changes_env=(
              # Nothing to add.
              "SCRIPTS_REPO=scripts"
              "COREOS_OVERLAY_REPO=coreos-overlay"
              "PORTAGE_STABLE_REPO=portage-stable"
          )
          show_changes_params_overrides=(
              # We may not have a tag handy, so we tell show-changes
              # to use git HEAD as a reference to new changelog
              # entries.
              'NEW_VERSION=HEAD'
          )

          # Parent directory of the scripts repo, required by some other
          # script.
          work_directory='..'
          generate_image_changes_report \
              "${arch}" "${channel}" "${vernum}" 'image-changes-reports.txt' "../flatcar-build-scripts" "${work_directory}" \
              "${package_diff_env[@]}" --- "${package_diff_params_b[@]}" -- \
              "${size_changes_env[@]}" --- "${size_changes_params_b[@]}" -- \
              "${show_changes_env[@]}" --- "${show_changes_params_overrides[@]}"

      - name: Build VM image(s)
        shell: bash
        run: |
          set -euo pipefail
          set -x

          source ci-automation/ci_automation_common.sh

          images_out="images"

          has_packet=0
          has_pxe=0
          formats="${IMAGE_FORMATS}"
          for format in "${formats}";do
            [[ "${format}" = 'packet' ]] || [[ "${format}" = 'equinix_metal' ]] && has_packet=1
            [[ "${format}" = 'pxe' ]] && has_pxe=1
          done

          [[ ${has_packet} -eq 1 ]] && [[ ${has_pxe} -eq 0 ]] && set -- 'pxe' "${@}"
          if echo "$formats" | tr ' ' '\n' | grep -q '^vmware'; then
            formats=$(echo "$formats" | tr ' ' '\n' | sed '/vmware.*/d')
            formats+=" vmware vmware_insecure vmware_ova vmware_raw"
          fi
          if echo "$formats" | tr ' ' '\n' | grep -q -P '^(ami|aws)'; then
            formats=$(echo "$formats" | tr ' ' '\n' | sed '/ami.*/d' | sed '/aws/d')
            formats+=" ami ami_vmdk"
          fi
          # Keep compatibility with SDK scripts where "equinix_metal" remains unknown.
          formats=$(echo "$formats" | tr ' ' '\n' | sed 's/equinix_metal/packet/g')

          for format in ${formats}; do
              echo " ###################  VENDOR '${format}' ################### "
              ./run_sdk_container -n "${container_name}" \
                  ./image_to_vm.sh --format "${format}" --board="${arch}-usr" \
                      --from "${CI_CONTAINER_ARTIFACT_ROOT}/${arch}-usr/latest" \
                      --image_compression_formats=bz2
          done

          # upload-artifacts cannot handle artifact uploads from sym-linked directories (no, really)
          #  so we move things around.
          mkdir -p artifacts/images
          (
            cd artifacts/${arch}-usr/latest/
            mv * ../../images/
          )

          # create a tarball for torcx package + JSON file because upload-artifacts cannot handle filenames containing colons
          #  (such as "docker:20.10.torcx.tgz")
          mv artifacts/torcx/${arch}-usr/latest/torcx_manifest.json artifacts/torcx/pkgs/
          tar -C artifacts/torcx/pkgs/ -cvf torcx.tar .


      - name: Upload binpkgs
        uses: actions/upload-artifact@v3
        with:
          retention-days: 7
          name: ${{ matrix.arch }}-binpkgs
          path: |
            scripts/binpkgs.tar

      - name: Upload update image (used with kola tests later)
        uses: actions/upload-artifact@v3
        with:
          retention-days: 7
          name: ${{ matrix.arch }}-test-update
          path: |
            scripts/artifacts/images/flatcar_test_update.gz

      - name: Upload generic image
        uses: actions/upload-artifact@v3
        with:
          retention-days: 7
          name: ${{ matrix.arch }}-generic-image
          path: |
            scripts/artifacts/images/flatcar_production_image.bin.bz2
            scripts/artifacts/images/flatcar_production_image.grub
            scripts/artifacts/images/flatcar_production_image.shim
            scripts/artifacts/images/flatcar_production_image.vmlinuz
            scripts/artifacts/images/flatcar_production_image*.txt
            scripts/artifacts/images/flatcar_production_image*.json
            scripts/artifacts/images/flatcar_production_image_pcr_policy.zip
            scripts/artifacts/images/flatcar_production_*_efi_*.fd

      - name: Upload developer container
        uses: actions/upload-artifact@v3
        with:
          retention-days: 7
          name: ${{ matrix.arch }}-devcontainer
          path: |
            scripts/artifacts/images/flatcar_developer_container*

      - name: Upload torcx tarball
        uses: actions/upload-artifact@v3
        with:
          retention-days: 7
          name: ${{ matrix.arch }}-torcx
          path: |
            scripts/torcx.tar

      - name: Upload reports
        uses: actions/upload-artifact@v3
        with:
          retention-days: 7
          name: ${{ matrix.arch }}-image-changes-reports
          path: |
            scripts/image-changes-reports.txt

      # Clean up what we uploaded already so the "vendor images" wildcard
      #  works when uploading artifacts in the next step.
      - name: Remove update, generic and devcontainer images
        shell: bash
        run: |
          set -euo pipefail
          set -x
          rm -f artifacts/images/flatcar_test_update.gz \
                artifacts/images/flatcar_production_image* \
                artifacts/images/flatcar_developer_container* \
                artifacts/images/flatcar_production_update*

      - name: Upload vendor images
        uses: actions/upload-artifact@v3
        with:
          retention-days: 7
          name: ${{ matrix.arch }}-vm-images
          path: |
            scripts/artifacts/images/*.img.bz2
            scripts/artifacts/images/*.bin.bz2
            scripts/artifacts/images/flatcar_production_*_efi_*.fd
            scripts/artifacts/images/*.txt
            scripts/artifacts/images/flatcar_production_*.sh

  test:
    needs: packages
    name: "Run kola tests"
    uses: ./.github/workflows/run-kola-tests.yaml