New version: beta-3760.1.1

add zstd support to squashfs (release 3602)

.github/workflows/cacerts-release.yaml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out main scripts branch for GitHub workflow scripts only
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
         with:
           token: ${{ secrets.BOT_PR_TOKEN }}
           path: gha
@@ -23,7 +23,7 @@ jobs:
         run: gha/.github/workflows/figure-out-branch.sh '${{ matrix.channel }}'
       - name: Check out work scripts branch for updating
         if: steps.figure-out-branch.outputs.SKIP == 0
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
         with:
           token: ${{ secrets.BOT_PR_TOKEN }}
           path: work
@@ -57,7 +57,7 @@ jobs:
         run: gha/.github/workflows/cacerts-apply-patch.sh
       - name: Create pull request
         if: (steps.figure-out-branch.outputs.SKIP == 0) && (steps.apply-patch.outputs.UPDATE_NEEDED == 1)
-        uses: peter-evans/create-pull-request@v6
+        uses: peter-evans/create-pull-request@v5
         with:
           token: ${{ secrets.BOT_PR_TOKEN }}
           path: work
@@ -66,4 +66,3 @@ jobs:
           title: Update ca-certificates in ${{ steps.figure-out-branch.outputs.BRANCH }} from ${{ steps.apply-patch.outputs.VERSION_OLD }} to ${{ steps.nss-latest-release.outputs.NSS_VERSION }}
           body: Subject says it all.
           labels: ${{ steps.figure-out-branch.outputs.LABEL }}
-          signoff: true

.github/workflows/ci.yaml

@@ -7,7 +7,7 @@ on:
         description: |
           Space-separated vendor formats to build.
         required: true
-        default: qemu_uefi pxe
+        default: qemu_uefi
       custom_sdk_version:
         type: string
         required: false
@@ -21,7 +21,7 @@ on:
         description: |
           Space-separated vendor formats to build.
         required: true
-        default: qemu_uefi pxe
+        default: qemu_uefi
       custom_sdk_version:
         type: string
         required: false
@@ -34,7 +34,11 @@ permissions:
 jobs:
   packages:
     name: "Build Flatcar packages"
-    runs-on: oracle-vm-32cpu-128gb-x86-64
+    runs-on:
+      - self-hosted
+      - debian
+      - build
+      - x64
     strategy:
       fail-fast: false
       matrix:
@@ -51,19 +55,23 @@ jobs:
           sudo rm /bin/sh
           sudo ln -s /bin/bash /bin/sh
           sudo apt-get update
-          sudo apt-get install -y ca-certificates curl git gnupg lsb-release python3 python3-packaging qemu-user-static zstd
+          sudo apt-get install -y ca-certificates curl git gnupg lsb-release python3 qemu-user-static zstd
-
+          sudo mkdir -p /etc/apt/keyrings
-      - name: Set up Docker
+          curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
-        uses: docker/setup-docker-action@v4
+          echo \
+            "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \
+            $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+          sudo apt-get update
+          sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin
 
       - name: Checkout scripts
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
         with:
           path: scripts
           fetch-depth: 0
 
       - name: Checkout build scripts
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
         with:
           repository: flatcar/flatcar-build-scripts
           path: flatcar-build-scripts
@@ -88,12 +96,18 @@ jobs:
           arch="${{ matrix.arch }}"
           echo "arch=${arch}" >> $GITHUB_ENV
 
-          IMAGE_FORMATS="qemu_uefi pxe"
+          IMAGE_FORMATS="qemu_uefi"
           [ -z "${{ inputs.image_formats }}" ] || IMAGE_FORMATS="${{ inputs.image_formats }}"
           echo "IMAGE_FORMATS=${IMAGE_FORMATS}" >> $GITHUB_ENV
 
-          # Artifact root for images as seen from within the container
+          # Artifact root for images and torcx tarball as seen from within the container
           echo "CI_CONTAINER_ARTIFACT_ROOT=/home/sdk/trunk/src/scripts/artifacts" >> $GITHUB_ENV
+          echo "CI_CONTAINER_TORCX_ROOT=/home/sdk/trunk/src/scripts/artifacts/torcx" >> $GITHUB_ENV
+          mkdir -p artifacts/torcx
+
+          # Placeholder URL for run-kola-tests.yaml, "Extract artifacts" step which will replace
+          #  this with its IP address.
+          echo "TORCX_TESTS_PACKAGE_URL=http://localhost:12345" >> $GITHUB_ENV
 
           if [ -n "${{ inputs.custom_sdk_version }}" ] ; then
               echo "CUSTOM_SDK_VERSION=${{ inputs.custom_sdk_version }}" >> $GITHUB_ENV
@@ -132,7 +146,9 @@ jobs:
           #  which will be re-used by subsequent build steps.
           ./run_sdk_container -n "${container_name}" -v "${version}" \
             -C "${sdk_image}" \
-            ./build_packages --board="${arch}-usr"
+            ./build_packages --board="${arch}-usr" \
+                --torcx_output_root="${CI_CONTAINER_TORCX_ROOT}" \
+                --torcx_extra_pkg_url="${TORCX_TESTS_PACKAGE_URL}"
 
           # Create binpkgs tarball for archiving as artifact later
           ./run_sdk_container -n "${container_name}" \
@@ -152,7 +168,7 @@ jobs:
 
       - name: Upload build logs
         if: always() && !cancelled()
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v3
         with:
           retention-days: 7
           name: ${{ matrix.arch }}-build-logs
@@ -177,7 +193,57 @@ jobs:
           ./run_sdk_container -n "${container_name}" \
                   ./build_image --board="${arch}-usr" --group="${channel}" \
                                 --output_root="${CI_CONTAINER_ARTIFACT_ROOT}" \
-                                prodtar container sysext oem_sysext
+                                --torcx_root="${CI_CONTAINER_TORCX_ROOT}" prodtar container
+
+      - name: Generate reports
+        shell: bash
+        run: |
+          set -euo pipefail
+          set -x
+
+          source ci-automation/image_changes.sh
+
+          channel=alpha
+          vernum=$(source sdk_container/.repo/manifests/version.txt; echo "${FLATCAR_VERSION}")
+          board="${arch}-usr"
+
+          package_diff_env=(
+              "FROM_B=file://${PWD}/artifacts/${arch}-usr/latest"
+              # BOARD_B and CHANNEL_B are unused.
+          )
+          package_diff_params_b=(
+              # The package-diff script appends version to the file
+              # URL, but the directory with the image has no version
+              # component at its end, so we use . as a version.
+              '.'
+          )
+          size_changes_env=(
+              # Nothing to add.
+          )
+          size_changes_params_b=(
+              "local:${PWD}/artifacts/${arch}-usr/latest"
+          )
+          show_changes_env=(
+              # Nothing to add.
+              "SCRIPTS_REPO=scripts"
+              "COREOS_OVERLAY_REPO=coreos-overlay"
+              "PORTAGE_STABLE_REPO=portage-stable"
+          )
+          show_changes_params_overrides=(
+              # We may not have a tag handy, so we tell show-changes
+              # to use git HEAD as a reference to new changelog
+              # entries.
+              'NEW_VERSION=HEAD'
+          )
+
+          # Parent directory of the scripts repo, required by some other
+          # script.
+          work_directory='..'
+          generate_image_changes_report \
+              "${arch}" "${channel}" "${vernum}" 'image-changes-reports.txt' "../flatcar-build-scripts" "${work_directory}" \
+              "${package_diff_env[@]}" --- "${package_diff_params_b[@]}" -- \
+              "${size_changes_env[@]}" --- "${size_changes_params_b[@]}" -- \
+              "${show_changes_env[@]}" --- "${show_changes_params_overrides[@]}"
 
       - name: Build VM image(s)
         shell: bash
@@ -210,23 +276,13 @@ jobs:
           formats=$(echo "$formats" | tr ' ' '\n' | sed 's/equinix_metal/packet/g')
 
           for format in ${formats}; do
-              if [ "${format}" = qemu ] || [ "${format}" = qemu_uefi_secure ]; then
-                  continue
-              fi
               echo " ###################  VENDOR '${format}' ################### "
               ./run_sdk_container -n "${container_name}" \
                   ./image_to_vm.sh --format "${format}" --board="${arch}-usr" \
                       --from "${CI_CONTAINER_ARTIFACT_ROOT}/${arch}-usr/latest" \
-                      --image_compression_formats=none
+                      --image_compression_formats=bz2
           done
 
-          # Zip doesn't handle symlinks well, remove them
-          rm -f artifacts/${arch}-usr/latest/flatcar_production_{qemu,qemu_uefi_secure}_image.img*
-          # or create an explicit copy:
-          if [ -e artifacts/${arch}-usr/latest/flatcar_production_pxe.vmlinuz ]; then
-            rm -f artifacts/${arch}-usr/latest/flatcar_production_pxe.vmlinuz
-            cp artifacts/${arch}-usr/latest/flatcar_production_{image,pxe}.vmlinuz
-          fi
           # upload-artifacts cannot handle artifact uploads from sym-linked directories (no, really)
           #  so we move things around.
           mkdir -p artifacts/images
@@ -235,14 +291,14 @@ jobs:
             mv * ../../images/
           )
 
-      - name: Generate reports against last release
+          # create a tarball for torcx package + JSON file because upload-artifacts cannot handle filenames containing colons
-        run: .github/workflows/image_changes.sh ${{ matrix.arch }} release
+          #  (such as "docker:20.10.torcx.tgz")
+          mv artifacts/torcx/${arch}-usr/latest/torcx_manifest.json artifacts/torcx/pkgs/
+          tar -C artifacts/torcx/pkgs/ -cvf torcx.tar .
 
-      - name: Generate reports against last nightly
-        run: .github/workflows/image_changes.sh ${{ matrix.arch }} nightly
 
       - name: Upload binpkgs
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v3
         with:
           retention-days: 7
           name: ${{ matrix.arch }}-binpkgs
@@ -250,7 +306,7 @@ jobs:
             scripts/binpkgs.tar
 
       - name: Upload update image (used with kola tests later)
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v3
         with:
           retention-days: 7
           name: ${{ matrix.arch }}-test-update
@@ -258,36 +314,43 @@ jobs:
             scripts/artifacts/images/flatcar_test_update.gz
 
       - name: Upload generic image
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v3
         with:
           retention-days: 7
           name: ${{ matrix.arch }}-generic-image
           path: |
-            scripts/artifacts/images/flatcar_production_image.bin
+            scripts/artifacts/images/flatcar_production_image.bin.bz2
             scripts/artifacts/images/flatcar_production_image.grub
             scripts/artifacts/images/flatcar_production_image.shim
             scripts/artifacts/images/flatcar_production_image.vmlinuz
             scripts/artifacts/images/flatcar_production_image*.txt
             scripts/artifacts/images/flatcar_production_image*.json
             scripts/artifacts/images/flatcar_production_image_pcr_policy.zip
-            scripts/artifacts/images/flatcar_production_*_efi_*.qcow2
+            scripts/artifacts/images/flatcar_production_*_efi_*.fd
-            scripts/artifacts/images/flatcar_production_qemu.sh
 
       - name: Upload developer container
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v3
         with:
           retention-days: 7
           name: ${{ matrix.arch }}-devcontainer
           path: |
             scripts/artifacts/images/flatcar_developer_container*
 
+      - name: Upload torcx tarball
+        uses: actions/upload-artifact@v3
+        with:
+          retention-days: 7
+          name: ${{ matrix.arch }}-torcx
+          path: |
+            scripts/torcx.tar
+
       - name: Upload reports
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v3
         with:
           retention-days: 7
           name: ${{ matrix.arch }}-image-changes-reports
           path: |
-            scripts/image-changes-reports*.txt
+            scripts/image-changes-reports.txt
 
       # Clean up what we uploaded already so the "vendor images" wildcard
       #  works when uploading artifacts in the next step.
@@ -302,19 +365,16 @@ jobs:
                 artifacts/images/flatcar_production_update*
 
       - name: Upload vendor images
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v3
         with:
           retention-days: 7
           name: ${{ matrix.arch }}-vm-images
           path: |
-            scripts/artifacts/images/*.img
+            scripts/artifacts/images/*.img.bz2
-            scripts/artifacts/images/*.bin
+            scripts/artifacts/images/*.bin.bz2
-            scripts/artifacts/images/flatcar_production_*_efi_*.qcow2
+            scripts/artifacts/images/flatcar_production_*_efi_*.fd
             scripts/artifacts/images/*.txt
-            scripts/artifacts/images/flatcar-*.raw
             scripts/artifacts/images/flatcar_production_*.sh
-            scripts/artifacts/images/flatcar_production_pxe_image.cpio.gz
-            scripts/artifacts/images/flatcar_production_pxe.vmlinuz
 
   test:
     needs: packages

.github/workflows/common.sh

@@ -186,7 +186,7 @@ function commit_changes() {
   for dir; do
     git add "${dir}"
   done
-  git commit --signoff -m "${pkg}: Update from ${old_version} to ${new_version}"
+  git commit -m "${pkg}: Update from ${old_version} to ${new_version}"
 
   popd
 }

.github/workflows/containerd-apply-patch.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source "${GHA_SCRIPTS_DIR}/.github/workflows/common.sh"
+
+prepare_git_repo
+
+if ! check_remote_branch "containerd-${VERSION_NEW}-${TARGET_BRANCH}"; then
+    echo "remote branch already exists, nothing to do"
+    exit 0
+fi
+
+pushd "${SDK_OUTER_OVERLAY}"
+
+VERSION_OLD=$(sed -n "s/^DIST containerd-\([0-9]*\.[0-9]*\.[0-9]*\).*/\1/p" app-containers/containerd/Manifest | sort -ruV | head -n1)
+if [[ "${VERSION_NEW}" = "${VERSION_OLD}" ]]; then
+    echo "already the latest Containerd, nothing to do"
+    exit 0
+fi
+
+# we need to update not only the main ebuild file, but also its CONTAINERD_COMMIT,
+# which needs to point to COMMIT_HASH that matches with $VERSION_NEW from upstream containerd.
+containerdEbuildOldSymlink=$(get_ebuild_filename app-containers/containerd "${VERSION_OLD}")
+containerdEbuildNewSymlink="app-containers/containerd/containerd-${VERSION_NEW}.ebuild"
+containerdEbuildMain="app-containers/containerd/containerd-9999.ebuild"
+git mv "${containerdEbuildOldSymlink}" "${containerdEbuildNewSymlink}"
+sed -i "s/CONTAINERD_COMMIT=\"\(.*\)\"/CONTAINERD_COMMIT=\"${COMMIT_HASH}\"/g" "${containerdEbuildMain}"
+sed -i "s/v${VERSION_OLD}/v${VERSION_NEW}/g" "${containerdEbuildMain}"
+
+
+DOCKER_VERSION=$(sed -n "s/^DIST docker-\([0-9]*\.[0-9]*\.[0-9]*\).*/\1/p" app-containers/docker/Manifest | sort -ruV | head -n1)
+# torcx ebuild file has a docker version with only major and minor versions, like 19.03.
+versionTorcx=${DOCKER_VERSION%.*}
+torcxEbuildFile=$(get_ebuild_filename app-torcx/docker "${versionTorcx}")
+sed -i "s/containerd-${VERSION_OLD}/containerd-${VERSION_NEW}/g" "${torcxEbuildFile}"
+
+popd
+
+URL="https://github.com/containerd/containerd/releases/tag/v${VERSION_NEW}"
+
+generate_update_changelog 'containerd' "${VERSION_NEW}" "${URL}" 'containerd'
+
+commit_changes app-containers/containerd "${VERSION_OLD}" "${VERSION_NEW}" \
+               app-torcx/docker
+
+cleanup_repo
+
+echo "VERSION_OLD=${VERSION_OLD}" >>"${GITHUB_OUTPUT}"
+echo 'UPDATE_NEEDED=1' >>"${GITHUB_OUTPUT}"

.github/workflows/containerd-release-main.yaml

@@ -0,0 +1,50 @@
+name: Get the latest Containerd release for main
+on:
+  schedule:
+    - cron:  '00 8 * * 5'
+  workflow_dispatch:
+
+jobs:
+  get-containerd-release:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out scripts
+        uses: actions/checkout@v3
+        with:
+          token: ${{ secrets.BOT_PR_TOKEN }}
+          path: scripts
+      - name: Figure out latest Containerd release version
+        id: containerd-latest-release
+        run: |
+          versionCommitPair=( $(git ls-remote --tags https://github.com/containerd/containerd | grep 'refs/tags/v[0-9]*\.[0-9]*\.[0-9]*$' | sed -e 's#^\([0-9a-fA-F]*\)[[:space:]]*refs/tags/v\(.*\)$#\2 \1#g' | sort --reverse --unique --version-sort | head --lines 1) )
+
+          echo "VERSION_NEW=${versionCommitPair[0]}" >>"${GITHUB_OUTPUT}"
+          echo "COMMIT_HASH=${versionCommitPair[1]}" >>"${GITHUB_OUTPUT}"
+      - name: Set up Flatcar SDK
+        id: setup-flatcar-sdk
+        env:
+          WORK_SCRIPTS_DIR: "${{ github.workspace }}/scripts"
+          CHANNEL: main
+        run: scripts/.github/workflows/setup-flatcar-sdk.sh
+      - name: Apply patch for main
+        id: apply-patch-main
+        env:
+          GHA_SCRIPTS_DIR: "${{ github.workspace }}/scripts"
+          WORK_SCRIPTS_DIR: "${{ github.workspace }}/scripts"
+          VERSION_NEW: ${{ steps.containerd-latest-release.outputs.VERSION_NEW }}
+          COMMIT_HASH: ${{ steps.containerd-latest-release.outputs.COMMIT_HASH }}
+          PACKAGES_CONTAINER: ${{ steps.setup-flatcar-sdk.outputs.PACKAGES_CONTAINER }}
+          SDK_NAME: ${{ steps.setup-flatcar-sdk.outputs.SDK_NAME }}
+          TARGET_BRANCH: main
+        run: scripts/.github/workflows/containerd-apply-patch.sh
+      - name: Create pull request for main
+        uses: peter-evans/create-pull-request@v5
+        if: steps.apply-patch-main.outputs.UPDATE_NEEDED == 1
+        with:
+          token: ${{ secrets.BOT_PR_TOKEN }}
+          path: scripts
+          branch: "containerd-${{ steps.containerd-latest-release.outputs.VERSION_NEW }}-main"
+          base: main
+          title: Upgrade Containerd in main from ${{ steps.apply-patch-main.outputs.VERSION_OLD }} to ${{ steps.containerd-latest-release.outputs.VERSION_NEW }}
+          body: Subject says it all.
+          labels: main

.github/workflows/docker-apply-patch.sh

@@ -0,0 +1,72 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source "${GHA_SCRIPTS_DIR}/.github/workflows/common.sh"
+
+prepare_git_repo
+
+if ! check_remote_branch "docker-${VERSION_NEW}-${TARGET_BRANCH}"; then
+    echo "remote branch already exists, nothing to do"
+    exit 0
+fi
+
+pushd "${SDK_OUTER_OVERLAY}"
+
+VERSION_OLD=$(sed -n "s/^DIST docker-\([0-9]*.[0-9]*.[0-9]*\).*/\1/p" app-containers/docker/Manifest | sort -ruV | head -n1)
+if [[ "${VERSION_NEW}" = "${VERSION_OLD}" ]]; then
+    echo "already the latest Docker, nothing to do"
+    exit 0
+fi
+
+# we need to update not only the main ebuild file, but also its DOCKER_GITCOMMIT,
+# which needs to point to COMMIT_HASH that matches with $VERSION_NEW from upstream docker-ce.
+dockerEbuildOld=$(get_ebuild_filename app-containers/docker "${VERSION_OLD}")
+dockerEbuildNew="app-containers/docker/docker-${VERSION_NEW}.ebuild"
+git mv "${dockerEbuildOld}" "${dockerEbuildNew}"
+sed -i "s/GIT_COMMIT=\(.*\)/GIT_COMMIT=${COMMIT_HASH_MOBY}/g" "${dockerEbuildNew}"
+sed -i "s/v${VERSION_OLD}/v${VERSION_NEW}/g" "${dockerEbuildNew}"
+
+cliEbuildOld=$(get_ebuild_filename app-containers/docker-cli "${VERSION_OLD}")
+cliEbuildNew="app-containers/docker-cli/docker-cli-${VERSION_NEW}.ebuild"
+git mv "${cliEbuildOld}" "${cliEbuildNew}"
+sed -i "s/GIT_COMMIT=\(.*\)/GIT_COMMIT=${COMMIT_HASH_CLI}/g" "${cliEbuildNew}"
+sed -i "s/v${VERSION_OLD}/v${VERSION_NEW}/g" "${cliEbuildNew}"
+
+# torcx ebuild file has a docker version with only major and minor versions, like 19.03.
+versionTorcx=${VERSION_OLD%.*}
+torcxEbuildFile=$(get_ebuild_filename app-torcx/docker "${versionTorcx}")
+sed -i "s/docker-${VERSION_OLD}/docker-${VERSION_NEW}/g" "${torcxEbuildFile}"
+sed -i "s/docker-cli-${VERSION_OLD}/docker-cli-${VERSION_NEW}/g" "${torcxEbuildFile}"
+
+# update also docker versions used by the current runc ebuild file.
+versionRunc=$(sed -n "s/^DIST runc-\([0-9]*.[0-9]*.*\)\.tar.*/\1/p" app-containers/runc/Manifest | sort -ruV | head -n1)
+runcEbuildFile=$(get_ebuild_filename app-containers/runc "${versionRunc}")
+sed -i "s/github.com\/docker\/docker-ce\/blob\/v${VERSION_OLD}/github.com\/docker\/docker-ce\/blob\/v${VERSION_NEW}/g" ${runcEbuildFile}
+
+popd
+
+# URL for Docker release notes has a specific format of
+#   https://docs.docker.com/engine/release-notes/MAJOR.MINOR/#COMBINEDFULLVERSION
+# To get the subfolder part MAJOR.MINOR, drop the patchlevel of the semver.
+#   e.g. 20.10.23 -> 20.10
+# To get the combined full version, drop all dots from the full version.
+#   e.g. 20.10.23 -> 201023
+# So the result becomes like:
+#   https://docs.docker.com/engine/release-notes/20.10/#201023
+URLSUBFOLDER=${VERSION_NEW%.*}
+URLVERSION="${VERSION_NEW//./}"
+URL="https://docs.docker.com/engine/release-notes/${URLSUBFOLDER}/#${URLVERSION}"
+
+generate_update_changelog 'Docker' "${VERSION_NEW}" "${URL}" 'docker'
+
+regenerate_manifest app-containers/docker-cli "${VERSION_NEW}"
+commit_changes app-containers/docker "${VERSION_OLD}" "${VERSION_NEW}" \
+               app-containers/docker-cli \
+               app-torcx/docker \
+               app-containers/runc
+
+cleanup_repo
+
+echo "VERSION_OLD=${VERSION_OLD}" >>"${GITHUB_OUTPUT}"
+echo 'UPDATE_NEEDED=1' >>"${GITHUB_OUTPUT}"

.github/workflows/docker-release-main.yaml

@@ -0,0 +1,53 @@
+name: Get the latest Docker release for main
+on:
+  schedule:
+    - cron:  '35 7 * * 3'
+  workflow_dispatch:
+
+jobs:
+  get-docker-release:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out scripts
+        uses: actions/checkout@v3
+        with:
+          token: ${{ secrets.BOT_PR_TOKEN }}
+          path: scripts
+      - name: Figure out latest Docker release version
+        id: docker-latest-release
+        run: |
+          versionCommitPairMoby=( $(git ls-remote --tags https://github.com/moby/moby | grep 'refs/tags/v[0-9]*\.[0-9]*\.[0-9]*$' | sed -e 's#^\([0-9a-fA-F]*\)[[:space:]]*refs/tags/v\(.*\)$#\2 \1#g' | sort --reverse --unique --version-sort | head --lines 1) )
+          commitHashCLI=$(git ls-remote --tags https://github.com/docker/cli | grep 'refs/tags/v'"${versionCommitPairMoby[0]}"'$' | cut -f1)
+
+          echo "VERSION_NEW=${versionCommitPairMoby[0]}" >>"${GITHUB_OUTPUT}"
+          echo "COMMIT_HASH_MOBY=${versionCommitPairMoby[1]}" >>"${GITHUB_OUTPUT}"
+          echo "COMMIT_HASH_CLI=${commitHashCLI}" >>"${GITHUB_OUTPUT}"
+      - name: Set up Flatcar SDK
+        id: setup-flatcar-sdk
+        env:
+          WORK_SCRIPTS_DIR: "${{ github.workspace }}/scripts"
+          CHANNEL: main
+        run: scripts/.github/workflows/setup-flatcar-sdk.sh
+      - name: Apply patch for main
+        id: apply-patch-main
+        env:
+          GHA_SCRIPTS_DIR: "${{ github.workspace }}/scripts"
+          WORK_SCRIPTS_DIR: "${{ github.workspace }}/scripts"
+          VERSION_NEW: ${{ steps.docker-latest-release.outputs.VERSION_NEW }}
+          COMMIT_HASH_MOBY: ${{ steps.docker-latest-release.outputs.COMMIT_HASH_MOBY }}
+          COMMIT_HASH_CLI: ${{ steps.docker-latest-release.outputs.COMMIT_HASH_CLI }}
+          PACKAGES_CONTAINER: ${{ steps.setup-flatcar-sdk.outputs.PACKAGES_CONTAINER }}
+          SDK_NAME: ${{ steps.setup-flatcar-sdk.outputs.SDK_NAME }}
+          TARGET_BRANCH: main
+        run: scripts/.github/workflows/docker-apply-patch.sh
+      - name: Create pull request for main
+        uses: peter-evans/create-pull-request@v5
+        if: steps.apply-patch-main.outputs.UPDATE_NEEDED == 1
+        with:
+          token: ${{ secrets.BOT_PR_TOKEN }}
+          path: scripts
+          branch: docker-${{ steps.docker-latest-release.outputs.VERSION_NEW }}-main
+          base: main
+          title: Upgrade Docker in main from ${{ steps.apply-patch-main.outputs.VERSION_OLD }} to ${{ steps.docker-latest-release.outputs.VERSION_NEW }}
+          body: Subject says it all.
+          labels: main

.github/workflows/firmware-release-main.yaml

@@ -9,7 +9,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out scripts
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
         with:
           token: ${{ secrets.BOT_PR_TOKEN }}
           path: scripts
@@ -35,7 +35,7 @@ jobs:
           TARGET_BRANCH: main
         run: scripts/.github/workflows/firmware-apply-patch.sh
       - name: Create pull request for main
-        uses: peter-evans/create-pull-request@v6
+        uses: peter-evans/create-pull-request@v5
         if: steps.apply-patch-main.outputs.UPDATE_NEEDED == 1
         with:
           token: ${{ secrets.BOT_PR_TOKEN }}
@@ -45,4 +45,3 @@ jobs:
           title: Upgrade Linux Firmware in main from ${{ steps.apply-patch-main.outputs.VERSION_OLD }} to ${{ steps.firmware-latest-release.outputs.VERSION_NEW }}
           body: Subject says it all.
           labels: main
-          signoff: true

.github/workflows/go-apply-patch.sh

@@ -0,0 +1,74 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source "${GHA_SCRIPTS_DIR}/.github/workflows/common.sh"
+
+prepare_git_repo
+
+# create a mapping between short version and new version, e.g. 1.16 -> 1.16.3
+declare -A VERSIONS
+for version_new in ${VERSIONS_NEW}; do
+  version_new_trimmed="${version_new%.*}"
+  if [[ "${version_new_trimmed%.*}" = "${version_new_trimmed}" ]]; then
+    version_new_trimmed="${version_new}"
+  fi
+  VERSIONS["${version_new_trimmed}"]="${version_new}"
+done
+
+branch_name="go-$(join_by '-and-' ${VERSIONS_NEW})-main"
+
+if ! check_remote_branch "${branch_name}"; then
+  echo "remote branch already exists, nothing to do"
+  exit 0
+fi
+
+# Parse the Manifest file for already present source files and keep the latest version in the current series
+# DIST go1.17.src.tar.gz ... => 1.17
+# DIST go1.17.1.src.tar.gz ... => 1.17.1
+declare -a UPDATED_VERSIONS_OLD UPDATED_VERSIONS_NEW
+any_different=0
+for version_short in "${!VERSIONS[@]}"; do
+  pushd "${SDK_OUTER_OVERLAY}"
+  VERSION_NEW="${VERSIONS["${version_short}"]}"
+  VERSION_OLD=$(sed -n "s/^DIST go\(${version_short}\(\.*[0-9]*\)\?\)\.src.*/\1/p" dev-lang/go/Manifest | sort -ruV | head -n1)
+  if [[ -z "${VERSION_OLD}" ]]; then
+    echo "${version_short} is not packaged, skipping"
+    popd
+    continue
+  fi
+  if [[ "${VERSION_NEW}" = "${VERSION_OLD}" ]]; then
+    echo "${version_short} is already at the latest (${VERSION_NEW}), skipping"
+    popd
+    continue
+  fi
+  UPDATED_VERSIONS_OLD+=("${VERSION_OLD}")
+  UPDATED_VERSIONS_NEW+=("${VERSION_NEW}")
+
+  any_different=1
+  EBUILD_FILENAME=$(get_ebuild_filename dev-lang/go "${VERSION_OLD}")
+  git mv "${EBUILD_FILENAME}" "dev-lang/go/go-${VERSION_NEW}.ebuild"
+
+  popd
+
+  URL="https://go.dev/doc/devel/release#go${VERSION_NEW}"
+
+  generate_update_changelog 'Go' "${VERSION_NEW}" "${URL}" 'go'
+
+  commit_changes dev-lang/go "${VERSION_OLD}" "${VERSION_NEW}"
+done
+
+cleanup_repo
+
+if [[ $any_different -eq 0 ]]; then
+    echo "go packages were already at the latest versions, nothing to do"
+    exit 0
+fi
+
+vo_gh="$(join_by ' and ' "${UPDATED_VERSIONS_OLD[@]}")"
+vn_gh="$(join_by ' and ' "${UPDATED_VERSIONS_NEW[@]}")"
+
+echo "VERSIONS_OLD=${vo_gh}" >>"${GITHUB_OUTPUT}"
+echo "VERSIONS_NEW=${vn_gh}" >>"${GITHUB_OUTPUT}"
+echo "BRANCH_NAME=${branch_name}" >>"${GITHUB_OUTPUT}"
+echo 'UPDATE_NEEDED=1' >>"${GITHUB_OUTPUT}"

.github/workflows/go-current-major-versions.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source "${GHA_SCRIPTS_DIR}/.github/workflows/common.sh"
+
+pushd "${SDK_OUTER_OVERLAY}"
+
+versions=()
+for ebuild in dev-lang/go/go-*.ebuild; do
+    version="${ebuild##*/go-}" # 1.20.1-r1.ebuild or 1.19.ebuild
+    version="${version%.ebuild}" # 1.20.1-r1 or 1.19
+    version="${version%%-*}" # 1.20.1 or 1.19
+    short_version="${version%.*}" # 1.20 or 1
+    if [[ "${short_version%.*}" = "${short_version}" ]]; then
+        # fix short version
+        short_version="${version}"
+    fi
+
+    versions+=($(git ls-remote --tags https://github.com/golang/go | \
+                     cut -f2 | \
+                     sed --quiet "/refs\/tags\/go${short_version}\(\.[0-9]*\)\?$/s/^refs\/tags\/go//p" | \
+                     grep --extended-regexp --invert-match --regexp='(beta|rc)' | \
+                     sort --reverse --unique --version-sort | \
+                     head --lines=1))
+done
+
+popd
+
+echo "VERSIONS_NEW=${versions[*]}" >>"${GITHUB_OUTPUT}"

.github/workflows/go-release-main.yaml

@@ -0,0 +1,48 @@
+name: Get the latest Go release for main
+on:
+  schedule:
+    - cron:  '15 7 * * 1'
+  workflow_dispatch:
+
+jobs:
+  get-go-releases:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out scripts
+        uses: actions/checkout@v3
+        with:
+          token: ${{ secrets.BOT_PR_TOKEN }}
+          path: scripts
+      - name: Figure out latest Go release versions
+        id: go-latest-release
+        env:
+          GHA_SCRIPTS_DIR: "${{ github.workspace }}/scripts"
+          WORK_SCRIPTS_DIR: "${{ github.workspace }}/scripts"
+        run: scripts/.github/workflows/go-current-major-versions.sh
+      - name: Set up Flatcar SDK
+        id: setup-flatcar-sdk
+        env:
+          WORK_SCRIPTS_DIR: "${{ github.workspace }}/scripts"
+          CHANNEL: main
+        run: scripts/.github/workflows/setup-flatcar-sdk.sh
+      - name: Apply patch for main
+        id: apply-patch-main
+        env:
+          GHA_SCRIPTS_DIR: "${{ github.workspace }}/scripts"
+          WORK_SCRIPTS_DIR: "${{ github.workspace }}/scripts"
+          VERSIONS_NEW: ${{ steps.go-latest-release.outputs.VERSIONS_NEW }}
+          PACKAGES_CONTAINER: ${{ steps.setup-flatcar-sdk.outputs.PACKAGES_CONTAINER }}
+          SDK_NAME: ${{ steps.setup-flatcar-sdk.outputs.SDK_NAME }}
+          TARGET_BRANCH: main
+        run: scripts/.github/workflows/go-apply-patch.sh
+      - name: Create pull request for main
+        uses: peter-evans/create-pull-request@v5
+        if: steps.apply-patch-main.outputs.UPDATE_NEEDED == 1
+        with:
+          token: ${{ secrets.BOT_PR_TOKEN }}
+          path: scripts
+          branch: ${{ steps.apply-patch-main.outputs.BRANCH_NAME }}
+          base: main
+          title: Upgrade Go from ${{ steps.apply-patch-main.outputs.VERSIONS_OLD }} to ${{ steps.apply-patch-main.outputs.VERSIONS_NEW }}
+          body: Subject says it all.
+          labels: main

.github/workflows/image_changes.sh

@@ -1,43 +0,0 @@
-#!/bin/bash
-
-#set -x
-set -euo pipefail
-
-source ci-automation/image_changes.sh
-
-# Callback invoked by run_image_changes_job, read its docs to learn
-# about the details about the callback.
-function github_ricj_callback() {
-    package_diff_env+=(
-        "FROM_B=file://${PWD}/artifacts/images"
-        # BOARD_B and CHANNEL_B are unused.
-    )
-    package_diff_params+=(
-        # The package-diff script appends version to the file
-        # URL, but the directory with the image has no version
-        # component at its end, so we use . as a version.
-        '.'
-    )
-    # Nothing to add to size changes env.
-    size_changes_params+=(
-        "local:${PWD}/artifacts/images"
-    )
-    show_changes_env+=(
-        # Override the default locations of repositories.
-        "SCRIPTS_REPO=."
-        "COREOS_OVERLAY_REPO=../coreos-overlay"
-        "PORTAGE_STABLE_REPO=../portage-stable"
-    )
-    show_changes_params+=(
-        # We may not have a tag handy, so we tell show-changes
-        # to use git HEAD as a reference to new changelog
-        # entries.
-        'NEW_VERSION=HEAD'
-    )
-}
-
-arch=${1}; shift
-mode=${1}; shift
-report_file_name="image-changes-reports-${mode}.txt"
-
-run_image_changes_job "${arch}" "${mode}" "${report_file_name}" '../flatcar-build-scripts' github_ricj_callback

.github/workflows/kernel-apply-patch.sh

@@ -11,7 +11,6 @@ if ! check_remote_branch "linux-${VERSION_NEW}-${TARGET_BRANCH}"; then
     exit 0
 fi
 
-# Dive into ebuild repo section of SDK
 pushd "${SDK_OUTER_OVERLAY}"
 
 # trim the 3rd part in the input semver, e.g. from 5.4.1 to 5.4
@@ -25,19 +24,13 @@ if [[ "${VERSION_NEW}" = "${VERSION_OLD}" ]]; then
     exit 0
 fi
 
-extra_pkgs=(
+for pkg in sources modules kernel; do
-    sys-kernel/coreos-modules
+    pushd "sys-kernel/coreos-${pkg}"
-    sys-kernel/coreos-kernel
+    git mv "coreos-${pkg}"-*.ebuild "coreos-${pkg}-${VERSION_NEW}.ebuild"
-    app-emulation/hv-daemons
+    sed -i -e '/^COREOS_SOURCE_REVISION=/s/=.*/=""/' "coreos-${pkg}-${VERSION_NEW}.ebuild"
-)
+    popd
-
-for pkg in sys-kernel/coreos-{sources,modules,kernel} app-emulation/hv-daemons; do
-    pkg+=/${pkg##*/}
-    git mv "${pkg}"-*.ebuild "${pkg}-${VERSION_NEW}.ebuild"
-    sed -i -e '/^COREOS_SOURCE_REVISION=/s/=.*/=""/' "${pkg}-${VERSION_NEW}.ebuild"
 done
 
-# Leave ebuild repo section of SDK
 popd
 
 function get_lwn_link() {
@@ -77,7 +70,9 @@ URL=$(get_lwn_link "${VERSION_NEW}")
 
 generate_update_changelog 'Linux' "${VERSION_NEW}" "${URL}" 'linux' "${OLD_VERSIONS_AND_URLS[@]}"
 
-commit_changes sys-kernel/coreos-sources "${VERSION_OLD}" "${VERSION_NEW}" "${extra_pkgs[@]}"
+commit_changes sys-kernel/coreos-sources "${VERSION_OLD}" "${VERSION_NEW}" \
+               sys-kernel/coreos-modules \
+               sys-kernel/coreos-kernel
 
 cleanup_repo
 

.github/workflows/kernel-release.yaml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out main scripts branch for GitHub workflow scripts only
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
         with:
           token: ${{ secrets.BOT_PR_TOKEN }}
           path: gha
@@ -23,7 +23,7 @@ jobs:
         run: gha/.github/workflows/figure-out-branch.sh '${{ matrix.channel }}'
       - name: Check out work scripts branch for updating
         if: steps.figure-out-branch.outputs.SKIP == 0
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
         with:
           token: ${{ secrets.BOT_PR_TOKEN }}
           path: work
@@ -58,7 +58,7 @@ jobs:
         run: gha/.github/workflows/kernel-apply-patch.sh
       - name: Create pull request
         if: (steps.figure-out-branch.outputs.SKIP == 0) && (steps.apply-patch.outputs.UPDATE_NEEDED == 1)
-        uses: peter-evans/create-pull-request@v6
+        uses: peter-evans/create-pull-request@v5
         with:
           token: ${{ secrets.BOT_PR_TOKEN }}
           path: work
@@ -67,4 +67,3 @@ jobs:
           title: Upgrade Linux Kernel for ${{ steps.figure-out-branch.outputs.BRANCH }} from ${{ steps.apply-patch.outputs.VERSION_OLD }} to ${{ steps.kernel-latest-release.outputs.KERNEL_VERSION }}
           body: Subject says it all.
           labels: ${{ steps.figure-out-branch.outputs.LABEL }}
-          signoff: true

.github/workflows/mantle-releases-main.yml

@@ -45,7 +45,7 @@ jobs:
           fi
           echo "BRANCH=${branch}" >>"${GITHUB_OUTPUT}"
           echo "SKIP=${skip}" >>"${GITHUB_OUTPUT}"
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
         if: ${{ steps.figure-out-branch.outputs.SKIP == 0 }}
         with:
           token: ${{ secrets.BOT_PR_TOKEN }}
@@ -55,7 +55,7 @@ jobs:
         id: fetch-latest-mantle
         run: |
           set -euo pipefail
-          commit=$(git ls-remote  https://github.com/flatcar/mantle refs/heads/main | cut -f1)
+          commit=$(git ls-remote  https://github.com/flatcar/mantle refs/heads/flatcar-master | cut -f1)
           echo "COMMIT=${commit}" >>"${GITHUB_OUTPUT}"
       - name: Try to apply patch
         if: ${{ steps.figure-out-branch.outputs.SKIP == 0 }}
@@ -69,7 +69,7 @@ jobs:
           fi
       - name: Create pull request for branch
         if: ${{ steps.figure-out-branch.outputs.SKIP == 0 }}
-        uses: peter-evans/create-pull-request@v6
+        uses: peter-evans/create-pull-request@v4
         with:
           token: ${{ secrets.BOT_PR_TOKEN }}
           base: ${{ steps.figure-out-branch.outputs.BRANCH }}
@@ -79,4 +79,3 @@ jobs:
           title: Upgrade mantle container image to latest HEAD in ${{ steps.figure-out-branch.outputs.BRANCH }}
           commit-message: Update mantle container image to latest HEAD
           delete-branch: true
-          signoff: true

.github/workflows/portage-stable-packages-list

@@ -3,36 +3,26 @@
 acct-group/adm
 acct-group/audio
 acct-group/cdrom
-acct-group/clock
-acct-group/cuse
 acct-group/dialout
 acct-group/disk
 acct-group/dnsmasq
 acct-group/docker
 acct-group/floppy
-acct-group/incus
-acct-group/incus-admin
 acct-group/input
-acct-group/jobserver
 acct-group/kmem
 acct-group/kvm
 acct-group/lp
-acct-group/lxc
 acct-group/man
 acct-group/messagebus
-acct-group/named
 acct-group/netperf
 acct-group/nobody
 acct-group/ntp
-acct-group/openct
 acct-group/pcap
-acct-group/pcscd
 acct-group/polkitd
 acct-group/portage
 acct-group/render
 acct-group/root
 acct-group/sgx
-acct-group/shadow
 acct-group/sshd
 acct-group/systemd-coredump
 acct-group/systemd-journal
@@ -44,7 +34,6 @@ acct-group/systemd-timesync
 acct-group/tape
 acct-group/tss
 acct-group/tty
-acct-group/usb
 acct-group/users
 acct-group/utmp
 acct-group/uucp
@@ -52,16 +41,12 @@ acct-group/video
 acct-group/wheel
 
 acct-user/dnsmasq
-acct-user/lxc
 acct-user/man
 acct-user/messagebus
-acct-user/named
 acct-user/netperf
 acct-user/nobody
 acct-user/ntp
-acct-user/nvpd
 acct-user/pcap
-acct-user/pcscd
 acct-user/polkitd
 acct-user/portage
 acct-user/root
@@ -74,19 +59,12 @@ acct-user/systemd-resolve
 acct-user/systemd-timesync
 acct-user/tss
 
-app-admin/eselect
-app-admin/logrotate
-app-admin/perl-cleaner
-app-admin/sudo
-
 app-alternatives/awk
 app-alternatives/bc
 app-alternatives/bzip2
 app-alternatives/cpio
-app-alternatives/gpg
 app-alternatives/gzip
 app-alternatives/lex
-app-alternatives/ninja
 app-alternatives/sh
 app-alternatives/tar
 app-alternatives/yacc
@@ -96,12 +74,9 @@ app-arch/cpio
 app-arch/gzip
 app-arch/lbzip2
 app-arch/libarchive
-app-arch/lz4
-app-arch/lzop
 app-arch/ncompress
 app-arch/pbzip2
 app-arch/pigz
-app-arch/pixz
 app-arch/rpm2targz
 app-arch/sharutils
 app-arch/tar
@@ -112,345 +87,194 @@ app-arch/zstd
 
 app-cdr/cdrtools
 
-app-containers/aardvark-dns
-app-containers/catatonit
-app-containers/conmon
-app-containers/containerd
-app-containers/containers-common
-app-containers/containers-image
-app-containers/containers-shortnames
-app-containers/containers-storage
-app-containers/cri-tools
-app-containers/crun
-app-containers/docker
-app-containers/docker-buildx
-app-containers/docker-cli
-app-containers/incus
-app-containers/lxc
-app-containers/netavark
-app-containers/podman
-app-containers/runc
-app-containers/syft
-
 app-crypt/adcli
-app-crypt/argon2
-app-crypt/ccid
-app-crypt/gnupg
-app-crypt/gpgme
 app-crypt/libb2
 app-crypt/libmd
 app-crypt/mit-krb5
-app-crypt/p11-kit
 app-crypt/pinentry
 app-crypt/rhash
-app-crypt/sbsigntools
-app-crypt/tpm2-tools
-app-crypt/tpm2-tss
-app-crypt/trousers
-
-app-doc/eclass-manpages
 
 app-editors/nano
 app-editors/vim
 app-editors/vim-core
 
-app-emulation/open-vmdk
 app-emulation/qemu
 app-emulation/qemu-guest-agent
-app-emulation/virt-firmware
 
 app-eselect/eselect-iptables
-app-eselect/eselect-lib-bin-symlink
-app-eselect/eselect-pinentry
-app-eselect/eselect-python
-app-eselect/eselect-rust
-app-eselect/eselect-vi
 
 app-misc/c_rehash
 app-misc/editor-wrapper
-app-misc/jq
 app-misc/mime-types
 app-misc/pax-utils
 
 app-portage/elt-patches
-app-portage/gentoolkit
-app-portage/getuto
 app-portage/portage-utils
+app-portage/gentoolkit
 
 app-shells/bash
 app-shells/bash-completion
-app-shells/gentoo-bashcomp
 
 app-text/asciidoc
 app-text/build-docbook-catalog
 app-text/docbook-xml-dtd
 app-text/docbook-xsl-ns-stylesheets
 app-text/docbook-xsl-stylesheets
-app-text/mandoc
 app-text/manpager
-app-text/scdoc
 app-text/sgml-common
-app-text/xmlto
 
-app-vim/gentoo-syntax
+sec-keys/openpgp-keys-gentoo-release
 
-dev-build/autoconf
-dev-build/autoconf-archive
-dev-build/autoconf-wrapper
-dev-build/automake
-dev-build/automake-wrapper
-dev-build/cmake
-dev-build/gtk-doc-am
-dev-build/libtool
-dev-build/make
-dev-build/meson
-dev-build/meson-format-array
-dev-build/ninja
-
-dev-cpp/azure-core
-dev-cpp/azure-identity
-dev-cpp/azure-security-keyvault-certificates
-dev-cpp/azure-security-keyvault-keys
-dev-cpp/gflags
-dev-cpp/glog
 dev-cpp/gtest
 
-dev-db/etcd
 dev-db/sqlite
 
-dev-debug/gdb
-dev-debug/strace
-
-dev-embedded/u-boot-tools
-
-dev-go/go-md2man
-
 dev-lang/duktape
-dev-lang/go
 dev-lang/go-bootstrap
-dev-lang/nasm
+dev-lang/lua
 dev-lang/perl
 dev-lang/python
 dev-lang/python-exec
 dev-lang/python-exec-conf
-dev-lang/rust
-dev-lang/rust-bin
-dev-lang/rust-common
-dev-lang/swig
-dev-lang/tcl
 dev-lang/yasm
 
+dev-libs/boost
 dev-libs/cJSON
-dev-libs/cowsql
 dev-libs/cyrus-sasl
-dev-libs/dbus-glib
-dev-libs/ding-libs
 dev-libs/elfutils
 dev-libs/expat
 dev-libs/glib
 dev-libs/gmp
+dev-libs/gobject-introspection
 dev-libs/gobject-introspection-common
 dev-libs/inih
-dev-libs/jansson
-dev-libs/jose
-dev-libs/json-c
 dev-libs/jsoncpp
 dev-libs/libaio
 dev-libs/libassuan
 dev-libs/libbsd
 dev-libs/libdnet
-dev-libs/libev
-dev-libs/libevent
-dev-libs/libffi
 dev-libs/libgcrypt
 dev-libs/libgpg-error
 dev-libs/libksba
 dev-libs/libltdl
 dev-libs/libmspack
 dev-libs/libnl
-dev-libs/libp11
+dev-libs/libpcre
 dev-libs/libpcre2
 dev-libs/libpipeline
-dev-libs/libpwquality
-dev-libs/libsodium
 dev-libs/libtasn1
-dev-libs/libtraceevent
-dev-libs/libtracefs
-dev-libs/libunistring
 dev-libs/libusb
 dev-libs/libuv
-dev-libs/libverto
 dev-libs/libxml2
 dev-libs/libxslt
-dev-libs/libyaml
-dev-libs/lzo
-dev-libs/mpc
-dev-libs/mpdecimal
-dev-libs/mpfr
 dev-libs/nettle
-dev-libs/npth
-dev-libs/nspr
 dev-libs/oniguruma
-dev-libs/opensc
-dev-libs/openssl
 dev-libs/popt
 dev-libs/protobuf
-dev-libs/raft
-dev-libs/rapidjson
-dev-libs/tree-sitter
-dev-libs/tree-sitter-bash
 dev-libs/userspace-rcu
 dev-libs/xmlsec
-dev-libs/xxhash
-dev-libs/yajl
 
-dev-perl/File-Slurper
+dev-perl/File-Slurp
+dev-perl/Locale-gettext
 dev-perl/Parse-Yapp
 
-dev-python/backports-tarfile
+dev-python/autocommand
-dev-python/cachecontrol
+dev-python/boto
 dev-python/certifi
-dev-python/cffi
-dev-python/chardet
-dev-python/charset-normalizer
-dev-python/colorama
 dev-python/crcmod
-dev-python/cryptography
 dev-python/cython
-dev-python/dependency-groups
-dev-python/distlib
 dev-python/distro
 dev-python/docutils
-dev-python/editables
-dev-python/ensurepip-pip
-dev-python/ensurepip-setuptools
 dev-python/fasteners
-dev-python/fastjsonschema
 dev-python/flit-core
 dev-python/gentoo-common
 dev-python/gpep517
-dev-python/hatch-vcs
+dev-python/inflect
-dev-python/hatchling
-dev-python/idna
 dev-python/installer
-dev-python/jaraco-collections
 dev-python/jaraco-context
 dev-python/jaraco-functools
 dev-python/jaraco-text
-dev-python/jinja2
+dev-python/jinja
-dev-python/lark
 dev-python/lazy-object-proxy
-dev-python/linkify-it-py
 dev-python/lxml
-dev-python/markdown-it-py
 dev-python/markupsafe
-dev-python/mdurl
 dev-python/more-itertools
-dev-python/msgpack
+dev-python/nspektr
+dev-python/ordered-set
 dev-python/packaging
-dev-python/pathspec
-dev-python/pefile
-dev-python/pip
 dev-python/platformdirs
-dev-python/pluggy
-dev-python/ply
-dev-python/poetry-core
-dev-python/pycparser
 dev-python/pydecomp
 dev-python/pygments
-dev-python/pyproject-hooks
+dev-python/pyparsing
-dev-python/pysocks
-dev-python/requests
-dev-python/resolvelib
-dev-python/rich
 dev-python/setuptools
 dev-python/setuptools-scm
 dev-python/six
 dev-python/snakeoil
 dev-python/tomli
-dev-python/tomli-w
-dev-python/tree-sitter
-dev-python/trove-classifiers
-dev-python/truststore
 dev-python/typing-extensions
-dev-python/uc-micro-py
-dev-python/urllib3
 dev-python/wheel
 
+dev-util/b2
 dev-util/bpftool
-dev-util/bsdiff
 dev-util/catalyst
-dev-util/debugedit
+dev-util/checkbashisms
+dev-util/cmake
+dev-util/cmocka
+dev-util/desktop-file-utils
 dev-util/gdbus-codegen
 dev-util/glib-utils
 dev-util/gperf
-dev-util/maturin
+dev-util/gtk-doc-am
+dev-util/meson
+dev-util/meson-format-array
+dev-util/ninja
 dev-util/pahole
 dev-util/patchelf
 dev-util/patchutils
 dev-util/perf
-dev-util/pkgcheck
 dev-util/pkgconf
 dev-util/re2c
-dev-util/xdelta
+dev-util/strace
-dev-util/xxd
 
 dev-vcs/git
+dev-vcs/repo
 
 eclass/acct-group.eclass
 eclass/acct-user.eclass
 eclass/alternatives.eclass
 eclass/app-alternatives.eclass
 eclass/autotools.eclass
-eclass/bash-completion-r1.eclass
+# Still has some Flatcar modifications, will need to upstream it first.
-eclass/branding.eclass
+#
-eclass/cargo.eclass
+# eclass/bash-completion-r1.eclass
-eclass/check-reqs.eclass
 eclass/cmake-multilib.eclass
 eclass/cmake.eclass
-eclass/crossdev.eclass
-eclass/db-use.eclass
 eclass/desktop.eclass
-eclass/dist-kernel-utils.eclass
 eclass/distutils-r1.eclass
-eclass/dot-a.eclass
+eclass/eapi7-ver.eclass
 eclass/eapi8-dosym.eclass
-eclass/eapi9-pipestatus.eclass
-eclass/eapi9-ver.eclass
 eclass/edo.eclass
 eclass/edos2unix.eclass
 eclass/elisp-common.eclass
+eclass/epatch.eclass
+eclass/eqawarn.eclass
 eclass/estack.eclass
+eclass/eutils.eclass
 eclass/fcaps.eclass
 eclass/flag-o-matic.eclass
 eclass/git-r3.eclass
 eclass/gnome.org.eclass
-eclass/gnome2-utils.eclass
 eclass/gnuconfig.eclass
-eclass/go-env.eclass
-eclass/go-module.eclass
-eclass/golang-base.eclass
-eclass/golang-vcs-snapshot.eclass
-eclass/golang-vcs.eclass
-eclass/guile-single.eclass
-eclass/guile-utils.eclass
 eclass/java-pkg-opt-2.eclass
 eclass/java-utils-2.eclass
 eclass/kernel-2.eclass
 eclass/libtool.eclass
 eclass/linux-info.eclass
-eclass/linux-mod-r1.eclass
 eclass/linux-mod.eclass
-eclass/llvm-r1.eclass
-eclass/llvm-utils.eclass
 eclass/llvm.eclass
-eclass/lua-single.eclass
+eclass/ltprune.eclass
-eclass/lua-utils.eclass
-eclass/mercurial.eclass
 eclass/meson-multilib.eclass
 eclass/meson.eclass
-eclass/mono-env.eclass
-eclass/mount-boot-utils.eclass
 eclass/mount-boot.eclass
 eclass/multibuild.eclass
 eclass/multilib-build.eclass
@@ -460,11 +284,9 @@ eclass/multiprocessing.eclass
 eclass/ninja-utils.eclass
 eclass/optfeature.eclass
 eclass/out-of-source-utils.eclass
-eclass/out-of-source.eclass
 eclass/pam.eclass
 eclass/pax-utils.eclass
 eclass/perl-functions.eclass
-eclass/perl-module.eclass
 eclass/plocale.eclass
 eclass/portability.eclass
 eclass/prefix.eclass
@@ -474,96 +296,60 @@ eclass/python-any-r1.eclass
 eclass/python-r1.eclass
 eclass/python-single-r1.eclass
 eclass/python-utils-r1.eclass
-eclass/qmake-utils.eclass
 eclass/readme.gentoo-r1.eclass
-eclass/rpm.eclass
-eclass/ruby-single.eclass
-eclass/ruby-utils.eclass
-eclass/rust-toolchain.eclass
-eclass/rust.eclass
 eclass/savedconfig.eclass
-eclass/secureboot.eclass
 eclass/selinux-policy-2.eclass
-eclass/sgml-catalog-r1.eclass
-eclass/shell-completion.eclass
-eclass/ssl-cert.eclass
 eclass/strip-linguas.eclass
-eclass/subversion.eclass
-eclass/sysroot.eclass
 eclass/systemd.eclass
 eclass/tmpfiles.eclass
 eclass/toolchain-autoconf.eclass
 eclass/toolchain-funcs.eclass
 eclass/toolchain.eclass
-eclass/tree-sitter-grammar.eclass
 eclass/udev.eclass
-eclass/unpacker.eclass
 eclass/user-info.eclass
-eclass/usr-ldscript.eclass
+# This file is modified by us to be an empty file, so can't be synced for now.
+#
+# eclass/usr-ldscript.eclass
 eclass/vcs-clean.eclass
-eclass/vcs-snapshot.eclass
 eclass/verify-sig.eclass
+eclass/versionator.eclass
 eclass/vim-doc.eclass
 eclass/vim-plugin.eclass
 eclass/virtualx.eclass
 eclass/waf-utils.eclass
 eclass/wrapper.eclass
 eclass/xdg-utils.eclass
-eclass/xdg.eclass
 eclass/xorg-3.eclass
 
 licenses
 
 media-libs/libpng
 
-net-analyzer/netperf
+net-analyzer/nmap
-net-analyzer/openbsd-netcat
-net-analyzer/tcpdump
 net-analyzer/traceroute
 
-net-dialup/lrzsz
+net-dns/bind-tools
-net-dialup/minicom
-
-net-dns/bind
 net-dns/c-ares
 net-dns/dnsmasq
-net-dns/libidn2
 
-net-firewall/conntrack-tools
-net-firewall/ebtables
 net-firewall/ipset
-net-firewall/iptables
-net-firewall/nftables
 
 net-fs/cifs-utils
-net-fs/nfs-utils
-net-fs/samba
 
 net-libs/gnutls
 net-libs/libmicrohttpd
-net-libs/libmnl
-net-libs/libnetfilter_conntrack
-net-libs/libnetfilter_cthelper
-net-libs/libnetfilter_cttimeout
-net-libs/libnetfilter_queue
-net-libs/libnfnetlink
 net-libs/libnftnl
+net-libs/libnsl
 net-libs/libpcap
-net-libs/libpsl
 net-libs/libslirp
-net-libs/libtirpc
 net-libs/nghttp2
 net-libs/rpcsvc-proto
 
 net-misc/bridge-utils
-net-misc/chrony
 net-misc/curl
 net-misc/ethertypes
 net-misc/iperf
 net-misc/iputils
-net-misc/ntp
-net-misc/openssh
-net-misc/passt
 net-misc/rsync
 net-misc/socat
 net-misc/wget
@@ -574,29 +360,28 @@ net-nds/rpcbind
 
 net-vpn/wireguard-tools
 
+perl-core/File-Temp
+
 profiles
 
-scripts
+# The bootstrap script has some modifications, so we can't sync scripts directory yet.
-
+#
-sec-keys/openpgp-keys-gentoo-release
+# scripts
 
 sec-policy/selinux-base
 sec-policy/selinux-base-policy
 sec-policy/selinux-container
 sec-policy/selinux-dbus
-sec-policy/selinux-policykit
 sec-policy/selinux-sssd
 sec-policy/selinux-unconfined
 
+
 sys-apps/acl
 sys-apps/attr
-sys-apps/azure-vm-utils
 sys-apps/bubblewrap
-sys-apps/busybox
 sys-apps/checkpolicy
 sys-apps/config-site
 sys-apps/coreutils
-sys-apps/dbus
 sys-apps/debianutils
 sys-apps/diffutils
 sys-apps/dtc
@@ -615,119 +400,87 @@ sys-apps/iproute2
 sys-apps/iucode_tool
 sys-apps/kbd
 sys-apps/kexec-tools
-sys-apps/keyutils
 sys-apps/kmod
 sys-apps/less
 sys-apps/locale-gen
-sys-apps/lsb-release
 sys-apps/lshw
+sys-apps/makedev
 sys-apps/man-db
 sys-apps/man-pages
 sys-apps/miscfiles
 sys-apps/net-tools
 sys-apps/nvme-cli
 sys-apps/pciutils
-sys-apps/pcsc-lite
-sys-apps/pkgcore
 sys-apps/portage
 sys-apps/pv
 sys-apps/sandbox
 sys-apps/sed
 sys-apps/semodule-utils
-sys-apps/shadow
 sys-apps/smartmontools
-sys-apps/systemd
 sys-apps/texinfo
 sys-apps/usbutils
 sys-apps/util-linux
 sys-apps/which
-sys-apps/zram-generator
 
-sys-auth/pambase
-sys-auth/polkit
-sys-auth/sssd
-
-sys-block/open-iscsi
 sys-block/open-isns
 sys-block/parted
 sys-block/thin-provisioning-tools
 
-sys-boot/efibootmgr
+sys-devel/autoconf
-sys-boot/gnu-efi
+sys-devel/autoconf-archive
-sys-boot/grub
+sys-devel/autoconf-wrapper
-sys-boot/mokutil
+sys-devel/automake
-
+sys-devel/automake-wrapper
 sys-devel/bc
 sys-devel/binutils
 sys-devel/binutils-config
 sys-devel/bison
 sys-devel/crossdev
-sys-devel/dwz
 sys-devel/flex
 sys-devel/gcc
 sys-devel/gcc-config
+sys-devel/gdb
 sys-devel/gettext
 sys-devel/gnuconfig
+sys-devel/libtool
 sys-devel/m4
 sys-devel/patch
 
-sys-firmware/edk2-bin
+sys-firmware/edk2-ovmf-bin
 sys-firmware/intel-microcode
 sys-firmware/ipxe
 sys-firmware/seabios-bin
 sys-firmware/sgabios
 
-sys-fs/btrfs-progs
 sys-fs/cryptsetup
 sys-fs/dosfstools
 sys-fs/e2fsprogs
-sys-fs/erofs-utils
 sys-fs/fuse
 sys-fs/fuse-common
-sys-fs/fuse-overlayfs
-sys-fs/inotify-tools
 sys-fs/lsscsi
-sys-fs/lvm2
-sys-fs/lxcfs
-sys-fs/mdadm
 sys-fs/mtools
 sys-fs/multipath-tools
 sys-fs/quota
-sys-fs/squashfs-tools
-sys-fs/squashfs-tools-ng
 sys-fs/xfsprogs
-sys-fs/zfs
-sys-fs/zfs-kmod
 
-sys-kernel/dracut
 sys-kernel/linux-headers
 
 sys-libs/binutils-libs
-sys-libs/cracklib
-sys-libs/efivar
 sys-libs/gdbm
-sys-libs/glibc
+sys-libs/ldb
 sys-libs/libcap
 sys-libs/libcap-ng
 sys-libs/libnvme
 sys-libs/libseccomp
 sys-libs/libselinux
 sys-libs/libsepol
-sys-libs/libunwind
-sys-libs/liburing
-sys-libs/libxcrypt
 sys-libs/ncurses
-sys-libs/pam
 sys-libs/readline
 sys-libs/talloc
 sys-libs/tdb
 sys-libs/tevent
-sys-libs/timezone-data
 sys-libs/zlib
 
-sys-power/acpid
-
-sys-process/audit
 sys-process/lsof
 sys-process/procps
 sys-process/psmisc
@@ -736,34 +489,28 @@ sys-process/tini
 virtual/acl
 virtual/dev-manager
 virtual/editor
-virtual/krb5
-virtual/ldb
 virtual/libc
 virtual/libcrypt
 virtual/libelf
-virtual/libiconv
-virtual/libintl
-virtual/libudev
 virtual/libusb
 virtual/man
 virtual/openssh
 virtual/os-headers
 virtual/package-manager
-virtual/pager
 virtual/perl-Carp
+virtual/perl-Data-Dumper
 virtual/perl-Encode
 virtual/perl-Exporter
 virtual/perl-ExtUtils-MakeMaker
+virtual/perl-File-Spec
+virtual/perl-File-Temp
+virtual/perl-Getopt-Long
+virtual/perl-IO
 virtual/pkgconfig
-virtual/resolvconf
 virtual/service-manager
 virtual/ssh
 virtual/tmpfiles
-virtual/udev
-virtual/zlib
 
-x11-drivers/nvidia-drivers
+x11-base/xorg-proto
 
 x11-libs/pixman
-
-x11-misc/makedepend

.github/workflows/pr-comment-build-dispatcher.yaml

@@ -13,7 +13,7 @@ concurrency:
 jobs:
   run_pre_checks:
     # Only run if this is a PR comment that contains a valid command
-    if: ${{ github.event.issue.pull_request && (contains(github.event.comment.body, '/build-image') || contains(github.event.comment.body, '/update-sdk')) }}
+    if: ${{ github.event.issue.pull_request }} && ( contains(github.event.comment.body, '/build-image') || contains(github.event.comment.body, '/update-sdk'))
     name: Check if commenter is in the Flatcar maintainers team
     outputs:
       maintainers: steps.step1.output.maintainers
@@ -77,4 +77,4 @@ jobs:
     uses: ./.github/workflows/ci.yaml
     with:
       custom_sdk_version: ${{ needs.update_sdk.outputs.sdk_version }}
-      image_formats: qemu_uefi pxe
+      image_formats: qemu_uefi

.github/workflows/pr-workflows.yaml

@@ -46,4 +46,4 @@ jobs:
     uses: ./.github/workflows/ci.yaml
     with:
       custom_sdk_version: ${{ needs.update_sdk.outputs.sdk_version }}
-      image_formats: qemu_uefi pxe
+      image_formats: qemu_uefi

.github/workflows/run-kola-tests.yaml

@@ -17,11 +17,15 @@ on:
 jobs:
   tests:
     name: "Run Kola tests"
-    runs-on: oracle-vm-32cpu-128gb-x86-64
+    runs-on:
+      - self-hosted
+      - debian
+      - kola
+      - ${{ matrix.arch }}
     strategy:
       fail-fast: false
       matrix:
-        arch: ["amd64"]
+        arch: ["amd64", "arm64"]
 
     steps:
       - name: Prepare machine
@@ -30,7 +34,18 @@ jobs:
         run: |
           sudo rm /bin/sh
           sudo ln -s /bin/bash /bin/sh
-          sudo apt update && sudo apt install -y ca-certificates curl gnupg lsb-release qemu-system git bzip2 jq dnsmasq python3 zstd iproute2 iptables
+          sudo apt-get install -y ca-certificates curl gnupg lsb-release qemu-system git bzip2 jq dnsmasq python3 zstd
+          sudo systemctl stop dnsmasq
+          sudo systemctl mask dnsmasq
+
+          # Install Docker-CE
+          sudo mkdir -p /etc/apt/keyrings
+          curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
+          echo \
+            "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \
+            $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+          sudo apt-get update
+          sudo apt-get install -y docker-ce docker-ce-cli containerd.io
 
           # Set up MASQUERADE. Don't care much to secure it.
           # This is needed for the VMs kola spins up to have internet access.
@@ -39,10 +54,7 @@ jobs:
           sudo iptables -I FORWARD -o $DEFAULT_ROUTE_DEVICE -j ACCEPT
           sudo iptables -I FORWARD -i $DEFAULT_ROUTE_DEVICE -j ACCEPT
 
-      - name: Set up Docker
+      - uses: actions/checkout@v3
-        uses: docker/setup-docker-action@v4
-
-      - uses: actions/checkout@v4
         with:
           path: scripts
           fetch-depth: 0
@@ -65,28 +77,34 @@ jobs:
 
       - name: Download binpkgs
         if: ${{ !inputs.workflow_run_id }}
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v3
         with:
           name: ${{ matrix.arch }}-binpkgs
 
       - name: Download test update image
         if: ${{ !inputs.workflow_run_id }}
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v3
         with:
           name: ${{ matrix.arch }}-test-update
 
       - name: Download generic image
         if: ${{ !inputs.workflow_run_id }}
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v3
         with:
           name: ${{ matrix.arch }}-generic-image
 
       - name: Download developer container
         if: ${{ !inputs.workflow_run_id }}
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v3
         with:
           name: ${{ matrix.arch }}-devcontainer
 
+      - name: Download torcx tarball
+        if: ${{ !inputs.workflow_run_id }}
+        uses: actions/download-artifact@v3
+        with:
+          name: ${{ matrix.arch }}-torcx
+
       - name: Download binpkgs from other workflow
         uses: gabriel-samfira/action-download-artifact@v5
         if: ${{ inputs.workflow_run_id }}
@@ -123,6 +141,15 @@ jobs:
           run_id: ${{ inputs.workflow_run_id }}
           name: ${{ matrix.arch }}-devcontainer
 
+      - name: Download torcx tarball from other workflow
+        uses: gabriel-samfira/action-download-artifact@v5
+        if: ${{ inputs.workflow_run_id }}
+        with:
+          workflow: ${{ inputs.workflow_name_or_id }}
+          workflow_conclusion: success
+          run_id: ${{ inputs.workflow_run_id }}
+          name: ${{ matrix.arch }}-torcx
+
       - name: Extract artifacts
         shell: bash
         run: |
@@ -130,8 +157,8 @@ jobs:
           set -x
           set -euo pipefail
 
-          # Set up a webserver for devcontainer tests.
+          # Set up a webserver for devcontainer and torcx tests.
-          # The respective tests will download devcontainer via http.
+          # The respective tests will download devcontainer and torcx tarball via http.
           # The devcontainer test will then run a build
           #   which will download and install binpkgs into the dev container.
           # For the sake of that test we will serve both via a temporary local web server.
@@ -147,10 +174,24 @@ jobs:
           mv flatcar_developer_container* ${TESTS_WEBSERVER_WEBROOT}
           tar -C ${TESTS_WEBSERVER_WEBROOT} -xvf binpkgs.tar
 
+          tar -C ${TESTS_WEBSERVER_WEBROOT} -xvf torcx.tar
+
+          # Move torcx package into plain webroot
+          #  (path consists of <arch>/<packagename>/<checksum>/<packagename>:<version>.torcx.tar.gz)
+          mv "${TESTS_WEBSERVER_WEBROOT}/${{ matrix.arch }}-usr"/*/*/*.torcx.tgz \
+             "${TESTS_WEBSERVER_WEBROOT}"
+
+          # Update torcx.json's http URL to point to the webserver IP.
+          # ci.yaml defines the "localhost" placeholder in its "Set Environment" step.
+          sed -i "s,http://localhost:12345,http://${TESTS_WEBSERVER_IP}:${TESTS_WEBSERVER_PORT}," \
+                 "${TESTS_WEBSERVER_WEBROOT}/torcx_manifest.json"
+          cat "${TESTS_WEBSERVER_WEBROOT}/torcx_manifest.json"
+
           # Extract the generic image we'll use for qemu tests.
           # Note that the qemu[_uefi] tests use the generic image instead of the
           #  qemu vendor VM image ("Astronaut: [...] Always have been.").
-          mv flatcar_production_image.bin flatcar_production_qemu_uefi_efi_code.qcow2 flatcar_production_qemu_uefi_efi_vars.qcow2 scripts/
+          bzip2 --decompress flatcar_production_image.bin.bz2
+          mv flatcar_production_image.bin flatcar_production_qemu_uefi_efi_code.fd scripts/
 
           mv flatcar_test_update.gz scripts/
 
@@ -180,13 +221,20 @@ jobs:
 
           source ci-automation/test.sh
 
-          PARALLEL_ARCH=5
+          # Provide our own torcx prepare function so we use our local manifest json.
+          # This is called by test_run below.
+          function __prepare_torcx() {
+            shift; shift # no need for arch or vernum
+            local destdir="$1"
+            cp "../${TESTS_WEBSERVER_WEBROOT}/torcx_manifest.json" "${destdir}"
+          }
+
+          PARALLEL_ARCH=10
 
           cat > sdk_container/.env <<EOF
           # export the QEMU_IMAGE_NAME to avoid to download it.
           export QEMU_IMAGE_NAME="/work/flatcar_production_image.bin"
-          export QEMU_UEFI_FIRMWARE="/work/flatcar_production_qemu_uefi_efi_code.qcow2"
+          export QEMU_UEFI_BIOS="/work/flatcar_production_qemu_uefi_efi_code.fd"
-          export QEMU_UEFI_OVMF_VARS="/work/flatcar_production_qemu_uefi_efi_vars.qcow2"
           export QEMU_UPDATE_PAYLOAD="/work/flatcar_test_update.gz"
           export QEMU_DEVCONTAINER_URL="http://${TESTS_WEBSERVER_IP}:${TESTS_WEBSERVER_PORT}"
           export QEMU_DEVCONTAINER_BINHOST_URL="http://${TESTS_WEBSERVER_IP}:${TESTS_WEBSERVER_PORT}"
@@ -210,7 +258,7 @@ jobs:
 
       - name: Upload detailed test logs
         if: always() && !cancelled()
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v3
         with:
           name: ${{ matrix.arch }}-test-logs-and-results
           path: |
@@ -222,7 +270,7 @@ jobs:
 
       - name: Upload raw TAP files of all runs for later merging
         if: always() && !cancelled()
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v3
         with:
           name: ${{ matrix.arch }}-raw-tapfiles
           path: |
@@ -233,7 +281,10 @@ jobs:
     name: "Merge TAP reports and post results"
     needs: tests
     if: always() && !cancelled()
-    runs-on: oracle-vm-32cpu-128gb-x86-64
+    runs-on:
+      - self-hosted
+      - debian
+      - kola
     permissions:
       pull-requests: write
 
@@ -244,9 +295,9 @@ jobs:
         run: |
           sudo rm /bin/sh
           sudo ln -s /bin/bash /bin/sh
-          sudo apt update && sudo apt install -y ca-certificates curl gnupg lsb-release git bzip2 jq sqlite3
+          sudo apt-get install -y ca-certificates curl gnupg lsb-release git bzip2 jq sqlite3
 
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
         with:
           path: scripts
           fetch-depth: 0
@@ -271,11 +322,17 @@ jobs:
       # This is clunky. Haven't figured out how to re-use matrix.arch here for downloads,
       #  so we download each arch individually.
       - name: Download amd64 tapfiles
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v3
         with:
           name: amd64-raw-tapfiles
           path: scripts/__TAP__/amd64
 
+      - name: Download arm64 tapfiles
+        uses: actions/download-artifact@v3
+        with:
+          name: arm64-raw-tapfiles
+          path: scripts/__TAP__/arm64
+
       - name: Create Test Summary
         shell: bash
         run: |

.github/workflows/runc-apply-patch.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source "${GHA_SCRIPTS_DIR}/.github/workflows/common.sh"
+
+prepare_git_repo
+
+if ! check_remote_branch "runc-${VERSION_NEW}-${TARGET_BRANCH}"; then
+    echo "remote branch already exists, nothing to do"
+    exit 0
+fi
+
+pushd "${SDK_OUTER_OVERLAY}"
+
+# Get the newest runc version, including official releases and rc
+# versions. We need some sed tweaks like replacing dots with
+# underscores, adding trailing underscore, sort, and trim the trailing
+# underscore and replace other underscores with dots again, so that
+# sort -V can properly sort "1.0.0" as newer than "1.0.0-rc95" and
+# "0.0.2.1" as newer than "0.0.2".
+VERSION_OLD=$(sed -n "s/^DIST runc-\([0-9]*\.[0-9]*.*\)\.tar.*/\1_/p" app-containers/runc/Manifest | tr '.' '_' | sort -ruV | sed -e 's/_$//' | tr '_' '.' | head -n1)
+if [[ "${VERSION_NEW}" = "${VERSION_OLD}" ]]; then
+    echo "already the latest Runc, nothing to do"
+    exit 0
+fi
+
+runcEbuildOld=$(get_ebuild_filename app-containers/runc "${VERSION_OLD}")
+runcEbuildNew="app-containers/runc/runc-${VERSION_NEW}.ebuild"
+git mv "${runcEbuildOld}" "${runcEbuildNew}"
+sed -i "s/${VERSION_OLD}/${VERSION_NEW}/g" "${runcEbuildNew}"
+sed -i "s/COMMIT_ID=\"\(.*\)\"/COMMIT_ID=\"${COMMIT_HASH}\"/g" "${runcEbuildNew}"
+
+# update also runc versions used by docker and containerd
+sed -i "s/runc-${VERSION_OLD}/runc-${VERSION_NEW}/g" app-containers/containerd/containerd-9999.ebuild
+
+dockerVersion=$(sed -n "s/^DIST docker-\([0-9]*.[0-9]*.[0-9]*\).*/\1/p" app-containers/docker/Manifest | sort -ruV | head -n1)
+
+# torcx ebuild file has a docker version with only major and minor versions, like 19.03.
+versionTorcx=${dockerVersion%.*}
+torcxEbuildFile=$(get_ebuild_filename app-torcx/docker "${versionTorcx}")
+sed -i "s/runc-${VERSION_OLD}/runc-${VERSION_NEW}/g" "${torcxEbuildFile}"
+
+popd
+
+URL="https://github.com/opencontainers/runc/releases/tag/v${VERSION_NEW}"
+
+generate_update_changelog 'runc' "${VERSION_NEW}" "${URL}" 'runc'
+
+commit_changes app-containers/runc "${VERSION_OLD}" "${VERSION_NEW}" \
+               app-containers/containerd \
+               app-torcx/docker
+
+cleanup_repo
+
+echo "VERSION_OLD=${VERSION_OLD}" >>"${GITHUB_OUTPUT}"
+echo 'UPDATE_NEEDED=1' >>"${GITHUB_OUTPUT}"

.github/workflows/runc-release-main.yaml

@@ -0,0 +1,65 @@
+name: Get the latest Runc release for main
+on:
+  schedule:
+    - cron:  '50 7 * * 4'
+  workflow_dispatch:
+
+jobs:
+  get-runc-release:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out scripts
+        uses: actions/checkout@v3
+        with:
+          token: ${{ secrets.BOT_PR_TOKEN }}
+          path: scripts
+      - name: Figure out latest Runc release version
+        id: runc-latest-release
+        run: |
+          REMOTE='https://github.com/opencontainers/runc'
+          # Get the newest runc version, including official releases
+          # and rc versions. We need some sed tweaks like replacing
+          # dots with underscores, adding trailing underscore, sort,
+          # and trim the trailing underscore and replace other
+          # underscores with dots again, so that sort -V can properly
+          # sort "1.0.0" as newer than "1.0.0-rc95" and "0.0.2.1" as
+          # newer than "0.0.2".
+          versionCommitPair=( $(git ls-remote --tags "${REMOTE}" | grep 'refs/tags/v[a-z0-9._-]*$' | sed -e 's#^\([0-9a-fA-F]*\)[[:space:]]*refs/tags/v\(.*\)$#\2_ \1#g' -e 's/\./_/g' | sort --reverse --unique --version-sort --key=1,1 | sed -e 's/_ / /' -e 's/_/./g' | head --lines=1) )
+          versionNew="${versionCommitPair[0]}"
+          # Gentoo expects an underline between version and rc, so
+          # "1.1.0-rc.1" becomes "1.1.0_rc.1".
+          versionNew="${versionNew//-/_}"
+          # Gentoo expects no separators between rc and the number, so
+          # "1.1.0_rc.1" becomes "1.1.0_rc1"
+          versionNew="${versionNew//rc./rc}"
+          commitHash="${versionCommitPair[1]}"
+          echo "VERSION_NEW=${versionNew}" >>"${GITHUB_OUTPUT}"
+          echo "COMMIT_HASH=${commitHash}" >>"${GITHUB_OUTPUT}"
+      - name: Set up Flatcar SDK
+        id: setup-flatcar-sdk
+        env:
+          WORK_SCRIPTS_DIR: "${{ github.workspace }}/scripts"
+          CHANNEL: main
+        run: scripts/.github/workflows/setup-flatcar-sdk.sh
+      - name: Apply patch for main
+        id: apply-patch-main
+        env:
+          GHA_SCRIPTS_DIR: "${{ github.workspace }}/scripts"
+          WORK_SCRIPTS_DIR: "${{ github.workspace }}/scripts"
+          VERSION_NEW: ${{ steps.runc-latest-release.outputs.VERSION_NEW }}
+          COMMIT_HASH: ${{ steps.runc-latest-release.outputs.COMMIT_HASH }}
+          PACKAGES_CONTAINER: ${{ steps.setup-flatcar-sdk.outputs.PACKAGES_CONTAINER }}
+          SDK_NAME: ${{ steps.setup-flatcar-sdk.outputs.SDK_NAME }}
+          TARGET_BRANCH: main
+        run: scripts/.github/workflows/runc-apply-patch.sh
+      - name: Create pull request for main
+        uses: peter-evans/create-pull-request@v5
+        if: steps.apply-patch-main.outputs.UPDATE_NEEDED == 1
+        with:
+          token: ${{ secrets.BOT_PR_TOKEN }}
+          path: scripts
+          branch: runc-${{ steps.runc-latest-release.outputs.VERSION_NEW }}-main
+          base: main
+          title: Upgrade Runc in main from ${{ steps.apply-patch-main.outputs.VERSION_OLD }} to ${{ steps.runc-latest-release.outputs.VERSION_NEW }}
+          body: Subject says it all.
+          labels: main

.github/workflows/rust-apply-patch.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source "${GHA_SCRIPTS_DIR}/.github/workflows/common.sh"
+
+prepare_git_repo
+
+if ! check_remote_branch "rust-${VERSION_NEW}-${TARGET_BRANCH}"; then
+    echo "remote branch already exists, nothing to do"
+    exit 0
+fi
+
+pushd "${SDK_OUTER_OVERLAY}"
+
+VERSION_OLD=$(sed -n "s/^DIST rustc-\(1\.[0-9]*\.[0-9]*\).*/\1/p" dev-lang/rust/Manifest | sort -ruV | head -n1)
+if [[ "${VERSION_NEW}" = "${VERSION_OLD}" ]]; then
+    echo "already the latest Rust, nothing to do"
+    exit 0
+fi
+
+# Replace (dev-lang/virtual)/rust versions in profiles/, e.g. package.accept_keywords.
+# Try to match all kinds of version specifiers, e.g. >=, <=, =, ~.
+find profiles -name 'package.*' | xargs sed -i "s/\([><]*=\|~\)*dev-lang\/rust-\S\+/\1dev-lang\/rust-${VERSION_NEW}/"
+find profiles -name 'package.*' | xargs sed -i "s/\([><]*=\|~\)*virtual\/rust-\S\+/\1virtual\/rust-${VERSION_NEW}/"
+
+EBUILD_FILENAME=$(get_ebuild_filename dev-lang/rust "${VERSION_OLD}")
+git mv "${EBUILD_FILENAME}" "dev-lang/rust/rust-${VERSION_NEW}.ebuild"
+EBUILD_FILENAME=$(get_ebuild_filename virtual/rust "${VERSION_OLD}")
+git mv "${EBUILD_FILENAME}" "virtual/rust/rust-${VERSION_NEW}.ebuild"
+
+popd
+
+URL="https://github.com/rust-lang/rust/releases/tag/${VERSION_NEW}"
+
+generate_update_changelog 'Rust' "${VERSION_NEW}" "${URL}" 'rust'
+
+commit_changes dev-lang/rust "${VERSION_OLD}" "${VERSION_NEW}" \
+               profiles \
+               virtual/rust
+
+cleanup_repo
+
+echo "VERSION_OLD=${VERSION_OLD}" >>"${GITHUB_OUTPUT}"
+echo 'UPDATE_NEEDED=1' >>"${GITHUB_OUTPUT}"

.github/workflows/rust-release-main.yaml

@@ -0,0 +1,48 @@
+name: Get the latest Rust release for main
+on:
+  schedule:
+    - cron:  '20 7 * * 2'
+  workflow_dispatch:
+
+jobs:
+  get-rust-release:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out scripts
+        uses: actions/checkout@v3
+        with:
+          token: ${{ secrets.BOT_PR_TOKEN }}
+          path: scripts
+      - name: Figure out latest Rust release version
+        id: rust-latest-release
+        run: |
+          version=$(git ls-remote --tags 'https://github.com/rust-lang/rust' | cut -f2 | sed -n "/refs\/tags\/1\.[0-9]*\.[0-9]*$/s/^refs\/tags\///p" | sort -ruV | head -n1)
+          echo "VERSION_NEW=${version}" >>"${GITHUB_OUTPUT}"
+      - name: Set up Flatcar SDK
+        id: setup-flatcar-sdk
+        env:
+          WORK_SCRIPTS_DIR: "${{ github.workspace }}/scripts"
+          CHANNEL: main
+        run: scripts/.github/workflows/setup-flatcar-sdk.sh
+      - name: Apply patch for main
+        id: apply-patch-main
+        env:
+          GHA_SCRIPTS_DIR: "${{ github.workspace }}/scripts"
+          WORK_SCRIPTS_DIR: "${{ github.workspace }}/scripts"
+          VERSION_NEW: ${{ steps.rust-latest-release.outputs.VERSION_NEW }}
+          PACKAGES_CONTAINER: ${{ steps.setup-flatcar-sdk.outputs.PACKAGES_CONTAINER }}
+          SDK_NAME: ${{ steps.setup-flatcar-sdk.outputs.SDK_NAME }}
+          TARGET_BRANCH: main
+        run: scripts/.github/workflows/rust-apply-patch.sh
+      - name: Create pull request for main
+        id: create-pull-request
+        uses: peter-evans/create-pull-request@v5
+        if: steps.apply-patch-main.outputs.UPDATE_NEEDED == 1
+        with:
+          token: ${{ secrets.BOT_PR_TOKEN }}
+          path: scripts
+          branch: rust-${{ steps.rust-latest-release.outputs.VERSION_NEW }}-main
+          base: main
+          title: Upgrade dev-lang/rust and virtual/rust in main from ${{ steps.apply-patch-main.outputs.VERSION_OLD }} to ${{ steps.rust-latest-release.outputs.VERSION_NEW }}
+          body: Subject says it all.
+          labels: main

.github/workflows/update-metadata-glsa.yaml

@@ -9,7 +9,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out scripts
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
         with:
           token: ${{ secrets.BOT_PR_TOKEN }}
       - name: Update GLSA metadata
@@ -22,7 +22,7 @@ jobs:
           todaydate=$(date +%Y-%m-%d)
           echo "TODAYDATE=${todaydate}" >>"${GITHUB_OUTPUT}"
       - name: Create pull request for main branch
-        uses: peter-evans/create-pull-request@v6
+        uses: peter-evans/create-pull-request@v5
         with:
           token: ${{ secrets.BOT_PR_TOKEN }}
           branch: buildbot/monthly-glsa-metadata-updates-${{steps.update-glsa-metadata.outputs.TODAYDATE }}
@@ -33,4 +33,3 @@ jobs:
           commit-message: "portage-stable/metadata: Monthly GLSA metadata updates"
           author: Flatcar Buildbot <buildbot@flatcar-linux.org>
           labels: main
-          signoff: true

.github/workflows/update-portage-stable-packages-from-list.yaml

@@ -9,12 +9,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out scripts
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           path: ./scripts
       - name: Check out Gentoo
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
         with:
           repository: gentoo/gentoo
           path: gentoo
@@ -25,7 +25,7 @@ jobs:
           fetch-depth: 250000
           ref: master
       - name: Check out build scripts
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
         with:
           repository: flatcar/flatcar-build-scripts
           path: flatcar-build-scripts
@@ -68,7 +68,7 @@ jobs:
           echo "UPDATED=${updated}" >>"${GITHUB_OUTPUT}"
           echo "TODAYDATE=${todaydate}" >>"${GITHUB_OUTPUT}"
       - name: Create pull request for main branch
-        uses: peter-evans/create-pull-request@v6
+        uses: peter-evans/create-pull-request@v5
         if: steps.update-listed-packages.outputs.UPDATED == 1
         with:
           token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/update-sdk.yaml

@@ -39,7 +39,7 @@ jobs:
     name: "Build an updated SDK container image"
     runs-on:
       - self-hosted
-      - ubuntu
+      - debian
       - build
       - x64
     strategy:
@@ -59,11 +59,15 @@ jobs:
           sudo rm /bin/sh
           sudo ln -s /bin/bash /bin/sh
           sudo apt-get install -y ca-certificates curl gnupg lsb-release qemu-user-static git jq openssh-client rsync zstd
+          sudo mkdir -p /etc/apt/keyrings
+          curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
+          echo \
+            "deb [signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \
+            $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+          sudo apt-get update
+          sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin
 
-      - name: Set up Docker
+      - uses: actions/checkout@v3
-        uses: docker/setup-docker-action@v4
-
-      - uses: actions/checkout@v4
         id: step2
         with:
           path: scripts

.github/workflows/vmware-release-main.yaml

@@ -9,7 +9,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out scripts
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
         with:
           token: ${{ secrets.BOT_PR_TOKEN }}
           path: scripts
@@ -38,7 +38,7 @@ jobs:
           TARGET_BRANCH: main
         run: scripts/.github/workflows/vmware-apply-patch.sh
       - name: Create pull request for main
-        uses: peter-evans/create-pull-request@v6
+        uses: peter-evans/create-pull-request@v5
         if: steps.apply-patch-main.outputs.UPDATE_NEEDED == 1
         with:
           token: ${{ secrets.BOT_PR_TOKEN }}
@@ -48,4 +48,3 @@ jobs:
           title: Upgrade open-vm-tools in main from ${{ steps.apply-patch-main.outputs.VERSION_OLD }} to ${{ steps.openvmtools-latest-release.outputs.VERSION_NEW }}
           body: Subject says it all.
           labels: main
-          signoff: true

CODEOWNERS

@@ -1,5 +0,0 @@
-# CODEOWNERS file for scripts
-# This file defines who is responsible for code review
-# See: https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners
-
-* @flatcar/flatcar-maintainers

CODE_OF_CONDUCT.md

@@ -1,9 +0,0 @@
-# Code of Conduct
-
-The Flatcar project follows the [CNCF Code of Conduct](https://github.com/cncf/foundation/blob/main/code-of-conduct.md).
-
-For details on how we uphold community standards across all Flatcar repositories, please see the [main Flatcar Code of Conduct](https://github.com/flatcar/Flatcar/blob/main/CODE_OF_CONDUCT.md).
-
-## Reporting
-
-If you experience or witness unacceptable behavior, please report it following the process outlined in the [CNCF Code of Conduct](https://github.com/cncf/foundation/blob/main/code-of-conduct.md).

CONTRIBUTING.md

@@ -1,15 +1,71 @@
-Welcome! We're so glad you're here and interested in contributing to Flatcar! 💖
+# How to Contribute
 
-Whether you're fixing a bug, adding a feature, or improving docs — we appreciate you!
+CoreOS projects are [Apache 2.0 licensed](LICENSE) and accept contributions via
+GitHub pull requests.  This document outlines some of the conventions on
+development workflow, commit message formatting, contact points and other
+resources to make it easier to get your contribution accepted.
 
-For more detailed guidelines (finding issues, community meetings, PR lifecycle, commit message format, and more), check out the [main Flatcar CONTRIBUTING guide](https://github.com/flatcar/Flatcar/blob/main/CONTRIBUTING.md).
+# Certificate of Origin
 
-If you want to file an issue for any Flatcar repository, please use the [central Flatcar issue tracker](https://github.com/flatcar/Flatcar/issues).
+By contributing to this project you agree to the Developer Certificate of
+Origin (DCO). This document was created by the Linux Kernel community and is a
+simple statement that you, as a contributor, have the legal right to make the
+contribution. See the [DCO](DCO) file for details.
 
----
+# Email and Chat
 
-## Repository Specific Guidelines
+The project currently uses the general CoreOS email list and IRC channel:
+- Email: [coreos-dev](https://groups.google.com/forum/#!forum/coreos-dev)
+- IRC: #[coreos](irc://irc.freenode.org:6667/#coreos) IRC channel on freenode.org
 
-Any guidelines specific to this repository that are not covered in the main contribution guide will be listed here.
+Please avoid emailing maintainers found in the MAINTAINERS file directly. They
+are very busy and read the mailing lists.
 
-<!-- Add repo-specific guidelines below this line -->
+## Getting Started
+
+- Fork the repository on GitHub
+- Read the [README](README.md) for build and test instructions
+- Play with the project, submit bugs, submit patches!
+
+## Contribution Flow
+
+This is a rough outline of what a contributor's workflow looks like:
+
+- Create a topic branch from where you want to base your work (usually master).
+- Make commits of logical units.
+- Make sure your commit messages are in the proper format (see below).
+- Push your changes to a topic branch in your fork of the repository.
+- Make sure the tests pass, and add any new tests as appropriate.
+- Submit a pull request to the original repository.
+
+Thanks for your contributions!
+
+### Format of the Commit Message
+
+We follow a rough convention for commit messages that is designed to answer two
+questions: what changed and why. The subject line should feature the what and
+the body of the commit should describe the why.
+
+```
+scripts: add the test-cluster command
+
+this uses tmux to setup a test cluster that you can easily kill and
+start for debugging.
+
+Fixes #38
+```
+
+The format can be described more formally as follows:
+
+```
+<subsystem>: <what changed>
+<BLANK LINE>
+<why this change was made>
+<BLANK LINE>
+<footer>
+```
+
+The first line is the subject and should be no longer than 70 characters, the
+second line is always blank, and other lines should be wrapped at 80 characters.
+This allows the message to be easier to read on GitHub as well as in various
+git tools.

GOVERNANCE.md

@@ -1,11 +0,0 @@
-# Governance
-
-For details on the Flatcar project governance model, decision-making process, and roles, please see the [main Flatcar Governance document](https://github.com/flatcar/Flatcar/blob/main/governance.md).
-
----
-
-## Repository-Specific Governance
-
-Any governance details specific to this repository will be listed here.
-
-<!-- Add repo-specific governance notes below this line -->

MAINTAINERS.md

@@ -1,11 +1,9 @@
 # Maintainers
 
-For the current list of maintainers and their responsibilities, please see the [main Flatcar MAINTAINERS file](https://github.com/flatcar/Flatcar/blob/main/MAINTAINERS.md).
+* Kai Lüke @pothos
+* Gabriel Samfira @gabriel-samfira
+* Thilo Fromm @t-lo
 
----
+See [Governance](https://github.com/flatcar/Flatcar/blob/main/governance.md) for governance, commit, and vote guidelines as well as maintainer responsibilities. Everybody listed in this file is a committer as per governance definition.
 
-## Repository-Specific Maintainers
+The contents of this file are synchronized from [Flatcar/MAINTAINERS.md](https://github.com/flatcar/Flatcar/blob/main/MAINTAINERS.md).
-
-Any maintainers specific to this repository will be listed here.
-
-<!-- Add repo-specific maintainers below this line -->

PREFIX.md

@@ -12,7 +12,7 @@ Before prefix build support are considered stable, the below must be implemented
    Prefix builds currently use the SDK cross toolchains (`/usr/<arch>-gnu/`) instead of board toolchains in `/build/<board>`.
    Prefix builds must be integrated with the board toolchains and stop using `cb-emerge` before considered stable.
 3. Add prefix wrappers for all portage tools (similar to board wrappers), not just `emerge`.
-4. Add test cases for prefix builds to [mantle/kola](https://github.com/flatcar/mantle/tree/main/kola).
+4. Add test cases for prefix builds to [mantle/kola](https://github.com/flatcar/mantle/tree/flatcar-master/kola).
 
 ## About
 

README.md

@@ -1,18 +1,3 @@
-<div style="text-align: center">
-
-[![Flatcar OS](https://img.shields.io/badge/Flatcar-Website-blue?logo=data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0idXRmLTgiPz4NCjwhLS0gR2VuZXJhdG9yOiBBZG9iZSBJbGx1c3RyYXRvciAyNi4wLjMsIFNWRyBFeHBvcnQgUGx1Zy1JbiAuIFNWRyBWZXJzaW9uOiA2LjAwIEJ1aWxkIDApICAtLT4NCjxzdmcgdmVyc2lvbj0iMS4wIiBpZD0ia2F0bWFuXzEiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyIgeG1sbnM6eGxpbms9Imh0dHA6Ly93d3cudzMub3JnLzE5OTkveGxpbmsiIHg9IjBweCIgeT0iMHB4Ig0KCSB2aWV3Qm94PSIwIDAgODAwIDYwMCIgc3R5bGU9ImVuYWJsZS1iYWNrZ3JvdW5kOm5ldyAwIDAgODAwIDYwMDsiIHhtbDpzcGFjZT0icHJlc2VydmUiPg0KPHN0eWxlIHR5cGU9InRleHQvY3NzIj4NCgkuc3Qwe2ZpbGw6IzA5QkFDODt9DQo8L3N0eWxlPg0KPHBhdGggY2xhc3M9InN0MCIgZD0iTTQ0MCwxODIuOGgtMTUuOXYxNS45SDQ0MFYxODIuOHoiLz4NCjxwYXRoIGNsYXNzPSJzdDAiIGQ9Ik00MDAuNSwzMTcuOWgtMzEuOXYxNS45aDMxLjlWMzE3Ljl6Ii8+DQo8cGF0aCBjbGFzcz0ic3QwIiBkPSJNNTQzLjgsMzE3LjlINTEydjE1LjloMzEuOVYzMTcuOXoiLz4NCjxwYXRoIGNsYXNzPSJzdDAiIGQ9Ik02NTUuMiw0MjAuOXYtOTUuNGgtMTUuOXY5NS40aC0xNS45VjI2MmgtMzEuOVYxMzQuOEgyMDkuNFYyNjJoLTMxLjl2MTU5aC0xNS45di05NS40aC0xNnY5NS40aC0xNS45djMxLjINCgloMzEuOXYxNS44aDQ3Ljh2LTE1LjhoMTUuOXYxNS44SDI3M3YtMTUuOGgyNTQuOHYxNS44aDQ3Ljh2LTE1LjhoMTUuOXYxNS44aDQ3Ljh2LTE1LjhoMzEuOXYtMzEuMkg2NTUuMnogTTQ4Ny44LDE1MWg3OS42djMxLjgNCgloLTIzLjZ2NjMuNkg1MTJ2LTYzLjZoLTI0LjJMNDg3LjgsMTUxTDQ4Ny44LDE1MXogTTIzMywyMTQuNlYxNTFoNjMuN3YyMy41aC0zMS45djE1LjhoMzEuOXYyNC4yaC0zMS45djMxLjhIMjMzVjIxNC42eiBNMzA1LDMxNy45DQoJdjE1LjhoLTQ3Ljh2MzEuOEgzMDV2NDcuN2gtOTUuNVYyODYuMUgzMDVMMzA1LDMxNy45eiBNMzEyLjYsMjQ2LjRWMTUxaDMxLjl2NjMuNmgzMS45djMxLjhMMzEyLjYsMjQ2LjRMMzEyLjYsMjQ2LjRMMzEyLjYsMjQ2LjR6DQoJIE00NDguMywzMTcuOXY5NS40aC00Ny44di00Ny43aC0zMS45djQ3LjdoLTQ3LjhWMzAyaDE1Ljl2LTE1LjhoOTUuNVYzMDJoMTUuOUw0NDguMywzMTcuOXogTTQ0MCwyNDYuNHYtMzEuOGgtMTUuOXYzMS44aC0zMS45DQoJdi03OS41aDE1Ljl2LTE1LjhoNDcuOHYxNS44aDE1Ljl2NzkuNUg0NDB6IE01OTEuNiwzMTcuOXY0Ny43aC0xNS45djE1LjhoMTUuOXYzMS44aC00Ny44di0zMS43SDUyOHYtMTUuOGgtMTUuOXY0Ny43aC00Ny44VjI4Ni4xDQoJaDEyNy4zVjMxNy45eiIvPg0KPC9zdmc+DQo=)](https://www.flatcar.org/)
-[![Discord](https://img.shields.io/badge/Discord-Chat%20with%20us!-5865F2?logo=discord)](https://discord.gg/PMYjFUsJyq)
-[![Matrix](https://img.shields.io/badge/Matrix-Chat%20with%20us!-green?logo=matrix)](https://app.element.io/#/room/#flatcar:matrix.org)
-[![Slack](https://img.shields.io/badge/Slack-Chat%20with%20us!-4A154B?logo=slack)](https://kubernetes.slack.com/archives/C03GQ8B5XNJ)
-[![Twitter Follow](https://img.shields.io/twitter/follow/flatcar?style=social)](https://x.com/flatcar)
-[![Mastodon Follow](https://img.shields.io/badge/Mastodon-Follow-6364FF?logo=mastodon)](https://hachyderm.io/@flatcar)
-[![Bluesky](https://img.shields.io/badge/Bluesky-Follow-0285FF?logo=bluesky)](https://bsky.app/profile/flatcar.org)
-[![OpenSSF Best Practices](https://www.bestpractices.dev/projects/10926/badge)](https://www.bestpractices.dev/projects/10926)
-
-
-> **Note:** To file an issue for any Flatcar repository, please use the [central Flatcar issue tracker](https://github.com/flatcar/Flatcar/issues).
-</div>
-
 # Flatcar Container Linux SDK scripts
 
 Welcome to the scripts repo, your starting place for most things here in the Flatcar Container Linux SDK. To get started you can find our documentation on [the Flatcar docs website][flatcar-docs].
@@ -106,20 +91,6 @@ To clone the scripts repo and pick a version:
   * list releases (e.g. all Alpha releases): `git tag -l alpha-*`
   * check out the release version, e.g. `3033.0.0`: `git checkout 3033.0.0`
 
-### Working with forks
-
-When using GitHub's "fork" feature, please **make sure to fork all branches**, not just `main`. Forking only `main` is the default on GitHub.
-
-The SDK container wrapper script `run_sdk_container` requires release tags in our release branches and fails to start if no release branch is present (see e.g. https://github.com/flatcar/Flatcar/issues/1705).
-If you have forked manually, please make sure to include all tags. You can retrofit upstream tags to a fork by using e.g.:
-
-```bash
-git remote add upstream https://github.com/flatcar/scripts.git
-git fetch --tags upstream
-```
-
-This is necessary because the SDK uses `git describe --tags` to determine the current version, and forks don't include the original repository's tags by default.
-
 To use the SDK container:
 * Fetch image and start the SDK container: `./run_sdk_container -t`
   This will fetch the container image of the "scripts" repo's release version you checked out.
@@ -155,13 +126,3 @@ The script `./bootstrap_sdk_container` bootstraps a new SDK tarball using an exi
 # Automation stubs for continuous integration
 
 Script stubs for various build stages can be found in the [ci-automation](ci-automation) folder. These are helpful for gluing Flatcar Container Linux builds to a continuous integration system.
-
----
-
-## Community & Project Documentation
-
-- [Contributing Guidelines](CONTRIBUTING.md) — How to contribute, find issues, and submit pull requests
-- [Code of Conduct](CODE_OF_CONDUCT.md) — Standards for respectful and inclusive community participation
-- [Security Policy](SECURITY.md) — How to report vulnerabilities and security-related information
-- [Maintainers](MAINTAINERS.md) — Current project maintainers and their responsibilities
-- [Governance](GOVERNANCE.md) — Project governance model, decision-making process, and roles

SECURITY.md

@@ -1,15 +0,0 @@
-# Security Policy
-
-The Flatcar project takes security seriously. We appreciate your efforts to responsibly disclose your findings.
-
-For our full security policy, supported versions, and how to report a vulnerability, please see the [main Flatcar Security Policy](https://github.com/flatcar/Flatcar/blob/main/SECURITY.md).
-
-**Please do not open public issues for security vulnerabilities.**
-
----
-
-## Repository-Specific Security Notes
-
-Any security considerations specific to this repository will be listed here.
-
-<!-- Add repo-specific security notes below this line -->

bash_completion

@@ -106,28 +106,124 @@ _autotest_complete() {
   _complete_board_sysroot_flag && return 0
 }
 
-# Complete flatcar_workon's <command> argument.
+# Complete cros_workon's <command> argument.
 #
 # TODO(petkov): We should probably extract the list of commands from
-# flatcar_workon --help, just like we do for flags (see _flag_complete).
+# cros_workon --help, just like we do for flags (see _flag_complete).
 #
 # TODO(petkov): Currently, this assumes that the command is the first
 # argument. In practice, the command is the first non-flag
 # argument. I.e., this should be fixed to support something like
-# "flatcar_workon --all list".
+# "cros_workon --all list".
-_complete_flatcar_workon_command() {
+_complete_cros_workon_command() {
   [ ${COMP_CWORD} -eq 1 ] || return 1
   local command="${COMP_WORDS[1]}"
-  COMPREPLY=($(compgen -W "start stop list" -- "$command"))
+  COMPREPLY=($(compgen -W "start stop list iterate" -- "$command"))
   return 0
 }
 
-# Complete flatcar_workon arguments.
+# Prints the full path to the cros_workon executable, handling tilde
-_flatcar_workon() {
+# expansion for the current user.
+_cros_workon_executable() {
+  local cros_workon="${COMP_WORDS[0]}"
+  if [[ "$cros_workon" == '~/'* ]]; then
+    cros_workon="$HOME/${cros_workon#'~/'}"
+  fi
+  echo "$cros_workon"
+}
+
+# Lists the workon (or live, if --all is passed in) ebuilds. Lists
+# both the full names (e.g., chromeos-base/metrics) as well as just
+# the ebuild names (e.g., metrics).
+_cros_workon_list() {
+  local cros_workon=$(_cros_workon_executable)
+  ${cros_workon} list $1 | sed 's,\(.\+\)/\(.\+\),\1/\2 \2,'
+}
+
+# Completes the current cros_workon argument assuming it's a
+# package/ebuild name.
+_complete_cros_workon_package() {
+  [ ${COMP_CWORD} -gt 1 ] || return 1
+  local package="${COMP_WORDS[COMP_CWORD]}"
+  local command="${COMP_WORDS[1]}"
+  # If "start", complete based on all workon packages.
+  if [[ ${command} == "start" ]]; then
+    COMPREPLY=($(compgen -W "$(_cros_workon_list --all)" -- "$package"))
+    return 0
+  fi
+  # If "stop" or "iterate", complete based on all live packages.
+  if [[ ${command} == "stop" ]] || [[ ${command} == "iterate" ]]; then
+    COMPREPLY=($(compgen -W "$(_cros_workon_list)" -- "$package"))
+    return 0
+  fi
+  return 1
+}
+
+# Complete cros_workon arguments.
+_cros_workon() {
   COMPREPLY=()
   _flag_complete && return 0
   _complete_board_sysroot_flag && return 0
-  _complete_flatcar_workon_command && return 0
+  _complete_cros_workon_command && return 0
+  _complete_cros_workon_package && return 0
+  return 0
+}
+
+_list_repo_commands() {
+  local repo=${COMP_WORDS[0]}
+  "$repo" help --all | grep -E '^  ' | sed 's/  \([^ ]\+\) .\+/\1/'
+}
+
+_list_repo_branches() {
+  local repo=${COMP_WORDS[0]}
+  "$repo" branches 2>&1 | grep \| | sed 's/[ *][Pp ] *\([^ ]\+\) .*/\1/'
+}
+
+_list_repo_projects() {
+  local repo=${COMP_WORDS[0]}
+  "$repo" manifest -o /dev/stdout 2> /dev/null \
+    | grep 'project name=' \
+    | sed 's/.\+name="\([^"]\+\)".\+/\1/'
+}
+
+# Complete repo's <command> argument.
+_complete_repo_command() {
+  [ ${COMP_CWORD} -eq 1 ] || return 1
+  local command=${COMP_WORDS[1]}
+  COMPREPLY=($(compgen -W "$(_list_repo_commands)" -- "$command"))
+  return 0
+}
+
+_complete_repo_arg() {
+  [ ${COMP_CWORD} -gt 1 ] || return 1
+  local command=${COMP_WORDS[1]}
+  local current=${COMP_WORDS[COMP_CWORD]}
+  if [[ ${command} == "abandon" ]]; then
+    if [[ ${COMP_CWORD} -eq 2 ]]; then
+      COMPREPLY=($(compgen -W "$(_list_repo_branches)" -- "$current"))
+    else
+      COMPREPLY=($(compgen -W "$(_list_repo_projects)" -- "$current"))
+    fi
+    return 0
+  fi
+  if [[ ${command} == "help" ]]; then
+    [ ${COMP_CWORD} -eq 2 ] && \
+      COMPREPLY=($(compgen -W "$(_list_repo_commands)" -- "$current"))
+    return 0
+  fi
+  if [[ ${command} == "start" ]]; then
+    [ ${COMP_CWORD} -gt 2 ] && \
+      COMPREPLY=($(compgen -W "$(_list_repo_projects)" -- "$current"))
+    return 0
+  fi
+  return 1
+}
+
+# Complete repo arguments.
+_complete_repo() {
+  COMPREPLY=()
+  _complete_repo_command && return 0
+  _complete_repo_arg && return 0
   return 0
 }
 
@@ -138,7 +234,8 @@ complete -o bashdefault -o default -F _board_sysroot \
   image_to_usb.sh \
   mod_image_for_test.sh
 complete -o bashdefault -o default -o nospace -F _autotest_complete autotest
-complete -F _flatcar_workon flatcar_workon
+complete -F _cros_workon cros_workon
+complete -F _complete_repo repo
 
 ###  Local Variables:
 ###  mode: shell-script

bootstrap_sdk

@@ -4,30 +4,48 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 #
-# This uses Gentoo's catalyst for very thoroughly building images from scratch.
+# This uses Gentoo's catalyst for very thoroughly building images from
+# scratch. Using images based on this will eliminate some of the hackery
+# in make_chroot.sh for building up the sdk from a stock stage3 tarball.
+# 
 # For reference the procedure it performs is this:
+# 
+# 1. snapshot: Grab a snapshot of the portage-stable repo from
+#     the current SDK's /var/lib/gentoo/repos/gentoo.
+#     Alternatively, check out a git ref specified via --portage-ref.
 #
-# 1. seed: Take a recent SDK, dev container, or custom tarball as a seed to
+# 2. stage1: Using a "seed" tarball as a build environment, build a
-#     build stage 1 with. Before proceeding, update relevant packages that have
+#     minimal root file system into a clean directory using ROOT=...
-#     changed sub-slot to avoid missing library issues later in the build.
+#     and USE=-* The restricted USE flags are key be small and avoid
-#
+#     circular dependencies.
-# 2. stage1: Using the above seed tarball as a build environment, build a
-#     minimal root file system into a clean directory using ROOT=... and USE=-*
-#     The restricted USE flags are key be small and avoid circular dependencies.
 #     NOTE that stage1 LACKS PROPER STAGE ISOLATION. Binaries produced in stage1
-#     will be linked against the SEED SDK libraries, NOT against libraries built
+#      will be linked against the SEED SDK libraries, NOT against libraries
-#     in stage 1.
+#      built in stage 1. See "stage_repo()" documentation further below for more.
+#     This stage uses:
+#     - portage-stable from the SDK's /var/lib/gentoo/repos/gentoo
+#        or a custom path via --stage1_portage_path command line option
+#     - coreos-overlay  from the SDK's /var/lib/gentoo/repos/coreos-overlay
+#        or a custom path via --stage1_overlay_path command line option
+#     Command line option refs need caution though, since
+#     stage1 must not contain updated ebuilds (see build_stage1 below).
 #
-# 3. stage2: This is skipped as recommended by upstream Gentoo.
+# 3. stage2: Run portage-stable/scripts/bootstrap.sh
+#     This rebuilds the toolchain using Gentoo bootstrapping, ensuring it's not linked
+#     to or otherwise influenced by whatever was in the "seed" tarball.
+#     The toolchain rebuild may contain updated package ebuilds from
+#     third_party/(portage-stable|coreos-overlay).
+#     This and all following stages use portage-stable and coreos-overlay
+#     from third_party/... (see 1.)
 #
-# 4. stage3: Run emerge -e system to rebuild everything using the normal USE
+# 4. stage3: Run emerge -e system to rebuild everything using the fresh updated
-#     flags provided by the profile. This will also pull in assorted base system
+#     toolchain from 3., using the normal USE flags provided by the profile. This
-#     packages that weren't included in the minimal environment stage1 created.
+#     will also pull in assorted base system packages that weren't included
+#     in the minimal environment stage1 created.
 #
 # 5. stage4: Install any extra packages or other desired tweaks. For the
 #     sdk we just install all the packages normally make_chroot.sh does.
 #
-# Usage: bootstrap_sdk [stage1 stage3 etc]
+# Usage: bootstrap_sdk [stage1 stage2 etc]
 # By default all four stages will be built using the latest stage4 as a seed.
 
 SCRIPT_ROOT=$(dirname $(readlink -f "$0"))
@@ -41,17 +59,24 @@ TYPE="flatcar-sdk"
 . "${BUILD_LIBRARY_DIR}/release_util.sh" || exit 1
 
 
+DEFINE_string stage1_portage_path "" \
+  "Path to custom portage ebuilds tree to use in stage 1 (DANGEROUS; USE WITH CAUTION)"
+DEFINE_string stage1_overlay_path "" \
+  "Path to custom overlay ebuilds tree to use in stage 1 (DANGEROUS; USE WITH CAUTION)"
+
+
 ## Define the stage4 config template
 catalyst_stage4() {
 cat <<EOF
+target: stage4
 pkgcache_path: $BINPKGS
 stage4/packages: coreos-devel/sdk-depends
 stage4/fsscript: ${BUILD_LIBRARY_DIR}/catalyst_sdk.sh
 stage4/root_overlay: ${ROOT_OVERLAY}
-stage4/empty: /root /var/cache/edb
+stage4/empty: /etc/portage/repos.conf /root /usr/portage /var/cache/edb
 stage4/rm: /etc/machine-id /etc/resolv.conf
 EOF
-catalyst_stage_default 4
+catalyst_stage_default
 }
 
 # Switch to HTTP because early boostrap stages do not have SSL support.
@@ -60,6 +85,7 @@ GENTOO_MIRRORS="${GENTOO_MIRRORS//https:\/\//http://}"
 export GENTOO_MIRRORS
 
 catalyst_init "$@"
+check_gsutil_opts
 ROOT_OVERLAY=${TEMPDIR}/stage4_overlay
 
 if [[ "$STAGES" =~ stage4 ]]; then
@@ -86,6 +112,120 @@ mkdir -p "${ROOT_OVERLAY}/tmp"
 chmod 1777 "${ROOT_OVERLAY}/tmp"
 cp "${BUILD_LIBRARY_DIR}/toolchain_util.sh" "${ROOT_OVERLAY}/tmp"
 
+
+# Stage 1 uses "known-good" ebuilds (from both coreos-overlay and portage-stable)
+#  to build a minimal toolchain (USE="-*") for stage 2.
+#
+# No package updates must happen in stage 1, so we use the portage-stable and
+#  coreos-overlay paths included with the current SDK (from the SDK chroot's
+#  /var/lib/gentoo/repos/). "Current SDK" refers to the SDK we entered with
+#  'cork enter', i.e. the SDK we run ./bootstrap_sdk in.
+#
+# Using ebuilds from the above mentioned sources will ensure that stage 1 builds
+#  a minimal stage 2 from known-good ebuild versions - the same ebuild versions
+#  that were used to build the very SDK we run ./bootstrap_sdk in.
+#
+# DANGER ZONE
+#
+# Stage 1 lacks proper isolation and will link all packages built for
+#  stage 2 against its own seed libraries ("/" in the catalyst chroot) instead of against libraries
+#  installed into the FS root of the stage 2 seed ("/tmp/stage1root" in the catalyst chroot).
+#  This is why we must prevent any updated package ebuilds to "leak" into stage 1, hence we use
+#  "known good" ebuild repo versions outlined above.
+#
+# In special circumstances it may be required to circumvent this and use custom paths
+#  for either (or both) portage and overlay. The command line options
+#  --stage1-portage-path and --stage1-overlay-path may be used to specify
+#  a repo path known to work for stage1. In that case the stage1 seed (i.e. the seed SDK)
+#  will be updated prior to starting to build stage 2.
+#  NOTE that this should never be used to introduce library updates in stage 1. All binaries
+#  produced in stage 1 are linked against libraries in the seed tarball, NOT libraries produced
+#  by stage one. Therefore, these binaries will cease to work in stage 2 when linked against
+#  outdated "seed tarball" libraries which have been updated to newer versions in stage 1.
+
+stage_repo() {
+    local repo="$1"
+    local path="$2"
+    local dest="$3"
+    local gitname="$repo"
+
+    if [ "$gitname" = "gentoo" ] ; then
+        gitname="portage-stable"
+    fi
+
+    if [ -z "$path" ]; then
+        cp -R "/var/gentoo/repos/${repo}" "$dest"
+        info "Using local SDK's ebuild repo '$repo' ('$gitname') in stage 1."
+    else
+        mkdir "$dest/$repo"
+        cp -R "${path}/"* "$dest/${repo}/"
+        info "Using custom path '$path' for ebuild repo '$repo' ('$gitname') in stage 1."
+        info "This may break stage 2. YOU HAVE BEEN WARNED. You break it, you keep it."
+    fi
+    (
+        set -euo pipefail
+        local repo_var hook name
+
+        # FLAGS_coreos_overlay for gitname coreos-overlay
+        repo_var="FLAGS_${gitname//-/_}"
+        shopt -s nullglob
+        for hook in "${FLAGS_coreos_overlay}/coreos/stage1_hooks/"*"-${gitname}.sh"; do
+            name=${hook##*/}
+            name=${name%"-${gitname}.sh"}
+            info "Invoking stage1 ${gitname} hook ${name} on ${dest}/${repo}"
+            "${hook}" "${dest}/${repo}" "${!repo_var}"
+        done
+    )
+}
+
+build_stage1() {
+    # First, write out the default 4-stage catalyst configuration files
+    write_configs
+
+    # Prepare local copies of both the "known-good" portage-stable and the
+    #  "known-good" coreos-overlay ebuild repos
+    local stage1_repos="$TEMPDIR/stage1-ebuild-repos"
+    info "Creating stage 1 ebuild repos and stage 1 snapshot in '$stage1_repos'"
+    rm -rf "$stage1_repos"
+    mkdir "$stage1_repos"
+
+    # prepare ebuild repos for stage 1, either from the local SDK (default)
+    #  or from custom paths specified via command line flags
+    stage_repo "gentoo" "${FLAGS_stage1_portage_path}" "$stage1_repos"
+    stage_repo "coreos-overlay" "${FLAGS_stage1_overlay_path}" "$stage1_repos"
+
+    # Create a snapshot of "known-good" portage-stable repo copy for use in stage 1
+    #  This requires us to create a custom catalyst config to point it to the
+    #  repo copy we just created, for snapshotting.
+    catalyst_conf > "$TEMPDIR/catalyst-stage1.conf"
+    sed -i "s:^portdir.*:portdir=\"$stage1_repos/gentoo\":" \
+         "$TEMPDIR/catalyst-stage1.conf"
+    # take the "portage directory" (portage-stable copy) snapshot
+    build_snapshot "${TEMPDIR}/catalyst-stage1.conf" "${FLAGS_version}-stage1"
+
+    # Update the stage 1 spec to use the "known-good" portage-stable snapshot
+    #  and coreos-overlay copy repository versions from above.
+    sed -i -e "s/^snapshot:.*/snapshot: $FLAGS_version-stage1/" \
+           -e "s,^portage_overlay:.*,portage_overlay: $stage1_repos/coreos-overlay," \
+        "$TEMPDIR/stage1.spec"
+
+    # If we are to use a custom path for either ebuild repo we want to update the stage1 seed SDK
+    if [ -n "${FLAGS_stage1_portage_path}" -o -n "${FLAGS_stage1_overlay_path}" ] ; then
+        sed -i 's/^update_seed: no/update_seed: yes/' "$TEMPDIR/stage1.spec"
+        echo "update_seed_command: --update --deep --newuse --complete-graph --rebuild-if-new-ver --rebuild-exclude cross-*-cros-linux-gnu/* sys-devel/gcc " \
+            >>"$TEMPDIR/stage1.spec"
+    fi
+
+    # Finally, build stage 1
+    build_stage stage1 "$SEED" "$TEMPDIR/catalyst-stage1.conf"
+}
+
+if [[ "$STAGES" =~ stage1 ]]; then
+    build_stage1
+    STAGES="${STAGES/stage1/}"
+    SEED="${TYPE}/stage1-${ARCH}-latest"
+fi
+
 catalyst_build
 
 if [[ "$STAGES" =~ stage4 ]]; then
@@ -107,6 +247,18 @@ if [[ "$STAGES" =~ stage4 ]]; then
     verify_digests "${release_image}" "${release_contents}"
 
     info "SDK ready: ${release_image}"
+
+    def_upload_path="${UPLOAD_ROOT}/sdk/${ARCH}/${FLAGS_version}"
+    sign_and_upload_files "tarball" "${def_upload_path}" "" \
+        "${release_image}" "${release_contents}" "${release_digests}"
+    sign_and_upload_files "packages" "${def_upload_path}" "pkgs/" \
+        "${BINPKGS}"/*
+
+    if [ -d "${BINPKGS}/crossdev" ]; then
+        # Upload the SDK toolchain packages
+        sign_and_upload_files "cross toolchain packages" "${def_upload_path}" \
+            "toolchain/" "${BINPKGS}/crossdev"/*
+    fi
 fi
 
 command_completed

build_dev_binpkgs

@@ -1,87 +0,0 @@
-#!/bin/bash
-# Copyright (c) 2023 by the Flatcar Maintainers.
-# Use of this source code is governed by the Apache 2.0 license.
-
-. "$(dirname "$0")/common.sh" || exit 1
-
-# Script must run inside the chroot
-assert_inside_chroot
-assert_not_root_user
-
-# Dependencies and packages to include by default.
-packages_default=( "coreos-devel/board-packages" )
-
-# Packages that are rdeps of the above but should not be included.
-# (mostly large packages, e.g. programming languages etc.)
-skip_packages_default="dev-lang/rust,dev-lang/rust-bin,dev-lang/go,dev-lang/go-bootstrap,dev-go/go-md2man"
-
-
-# Developer-visible flags.
-DEFINE_string board "${DEFAULT_BOARD}" \
-  "The board to build packages for."
-DEFINE_string skip_packages "${skip_packages_default}" \
-  "Comma-separated list of packages in the dependency tree to skip."
-DEFINE_boolean pretend "${FLAGS_FALSE}" \
-  "List packages that would be built but do not actually build."
-
-FLAGS_HELP="usage: $(basename "$0") [flags] [packages]
-
-build_dev_binpkgs builds binary packages for all dependencies of [packages]
-that are not present in '/build/<board>/var/lib/portage/pkgs/'.
-Useful for publishing a complete set of packages to a binhost.
-
-[packages] defaults to '${packages_default[*]}' if not specified.
-"
-
-# Parse command line
-FLAGS "$@" || exit 1
-eval set -- "${FLAGS_ARGV}"
-
-# Die on any errors.
-switch_to_strict_mode
-
-if [[ $# -eq 0 ]]; then
-  set -- "${packages_default[@]}"
-fi
-# --
-
-function my_board_emerge() {
-  PORTAGE_CONFIGROOT="/build/${FLAGS_board}" SYSROOT="${SYSROOT:-/build/${FLAGS_board}}" ROOT="/build/${FLAGS_board}" sudo -E emerge "${@}"
-}
-# --
-
-pkg_build_list=()
-pkg_skipped_list=()
-
-info "Collecting list of binpkgs to build"
-
-# Normally, BDEPENDs are only installed to the SDK, but the point of this script
-# is to install them to the board root because the dev container uses a board
-# profile. This is easily achieved using --root-deps. Since it is still the SDK
-# doing the building, which might have different package versions available to
-# the board profile, we have to be careful not to include SDK BDEPENDs in the
-# list of binary packages to publish, hence the sed call.
-while read -r pkg; do
-  [[ -f /build/${FLAGS_board}/var/lib/portage/pkgs/${pkg}.tbz2 ]] && continue
-  IFS=,
-  for s in ${FLAGS_skip_packages}; do
-    if [[ ${pkg} == ${s}-* ]] ; then
-      pkg_skipped_list+=("${pkg}")
-      continue 2
-    fi
-  done
-  unset IFS
-  pkg_build_list+=("=${pkg}")
-  echo "      =${pkg}"
-done < <(my_board_emerge --pretend --emptytree --root-deps "${@}" |
-         sed -n "/\[ebuild .* to \/build\/${FLAGS_board}\/ /s/^\[[^]]\+\] \([^ :]\+\)*:.*/\1/p")
-# --
-
-if [[ ${#pkg_skipped_list[@]} -gt 0 ]]; then
-  info "Skipping binpkgs '${pkg_skipped_list[*]}' because these are in the skip list."
-fi
-
-pretend=""
-[[ ${FLAGS_pretend} -eq ${FLAGS_TRUE} ]] && pretend="--pretend"
-
-my_board_emerge --buildpkg ${pretend} "${pkg_build_list[@]}"

build_docker_aci

@@ -0,0 +1,110 @@
+#!/bin/bash
+
+# Copyright (c) 2016 The CoreOS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# This is a wrapper around the ebuild_aci_util.sh functions to set up the
+# necessary environment, similar to the build_image script.
+
+SCRIPT_ROOT=$(dirname $(readlink -f "$0"))
+. "${SCRIPT_ROOT}/common.sh" || exit 1
+
+# Script must run inside the chroot
+assert_inside_chroot
+
+assert_not_root_user
+
+# Developer-visible flags.
+DEFINE_string board "${DEFAULT_BOARD}" \
+  "The board to build an image for."
+DEFINE_string build_dir "" \
+  "Directory in which to place image result directories (named by version)"
+DEFINE_boolean getbinpkg "${FLAGS_FALSE}" \
+  "Download binary packages from remote repository."
+DEFINE_string getbinpkgver "" \
+  "Use binary packages from a specific version."
+
+FLAGS_HELP="USAGE: build_docker_aci [flags] [docker version] [aci version number].
+This script is used to build a CoreOS docker-skim ACI.
+
+The docker version should identify an existent ebuild (i.e.
+app-containers/docker-\$version).
+
+The aci version number is an atomically incrementing number that will be
+appended to the aci version (to create e.g. :v1.12.6_coreos.0).
+
+Examples:
+
+build_docker_aci --board=amd64-usr --build_dir=<build_dir> 1.12.6 0
+...
+"
+show_help_if_requested "$@"
+
+# The following options are advanced options, only available to those willing
+# to read the source code. They are not shown in help output, since they are
+# not needed for the typical developer workflow.
+DEFINE_integer build_attempt 1 \
+  "The build attempt for this image build."
+DEFINE_string group "docker-aci" \
+  "The update group (not used for actual updates here)"
+DEFINE_string output_root "${DEFAULT_BUILD_ROOT}/images" \
+  "Directory in which to place image result directories (named by version)"
+DEFINE_string version "" \
+  "Sets the docker version to build."
+DEFINE_integer aci_version "" \
+  "Sets the aci version tag identifier."
+
+# Parse command line.
+FLAGS "$@" || exit 1
+[ -z "${FLAGS_ARGV}" ] && echo 'No version given' && exit 0
+eval set -- "${FLAGS_ARGV}"
+
+version="${1:?Docker version}"
+aci_version="${2:?Docker version}"
+
+
+# Only now can we die on error.  shflags functions leak non-zero error codes,
+# so will die prematurely if 'switch_to_strict_mode' is specified before now.
+switch_to_strict_mode
+
+# If downloading packages is enabled ensure the board is configured properly.
+if [[ ${FLAGS_getbinpkg} -eq ${FLAGS_TRUE} ]]; then
+    "${SRC_ROOT}/scripts/setup_board" --board="${FLAGS_board}" \
+      --getbinpkgver="${FLAGS_getbinpkgver}" --regen_configs_only
+fi
+
+# N.B.  Ordering matters for some of the libraries below, because
+# some of the files contain initialization used by later files.
+. "${BUILD_LIBRARY_DIR}/toolchain_util.sh" || exit 1
+. "${BUILD_LIBRARY_DIR}/board_options.sh" || exit 1
+. "${BUILD_LIBRARY_DIR}/build_image_util.sh" || exit 1
+. "${BUILD_LIBRARY_DIR}/prod_image_util.sh" || exit 1
+. "${BUILD_LIBRARY_DIR}/test_image_content.sh" || exit 1
+. "${BUILD_LIBRARY_DIR}/ebuild_aci_util.sh" || exit 1
+
+BUILD_DIR=${FLAGS_build_dir:-$BUILD_DIR}
+
+case "${version}" in
+    1.12.[0-9]*)
+        packaged_files=( 
+            "/usr/bin/docker"
+            "/usr/bin/dockerd"
+            "/usr/bin/docker-containerd"
+            "/usr/bin/docker-containerd-shim"
+            "/usr/bin/docker-proxy"
+            "/usr/bin/docker-runc"
+            "/usr/lib/flatcar/dockerd"
+        )
+        ebuild_aci_create "users.developer.core-os.net/skim/docker" \
+            "coreos_docker-${BOARD}-${version}_coreos.${aci_version}" \
+            "app-containers/docker" \
+            "${version}" \
+            "${aci_version}" \
+            "${packaged_files[@]}"
+        ;;
+    *)
+        1>&2 echo "Unrecognized version; please enter a supported version"
+        exit 1
+        ;;
+esac

build_image

@@ -33,24 +33,24 @@ DEFINE_string base_pkg "coreos-base/coreos" \
   "The base portage package to base the build off of (only applies to prod images)"
 DEFINE_string base_dev_pkg "coreos-base/coreos-dev" \
   "The base portage package to base the build off of (only applies to dev containers)"
-DEFINE_string base_sysexts "containerd-flatcar|app-containers/containerd,docker-flatcar|app-containers/docker&app-containers/docker-cli&app-containers/docker-buildx" \
+DEFINE_string torcx_manifest "${DEFAULT_BUILD_ROOT}/torcx/${DEFAULT_BOARD}/latest/torcx_manifest.json" \
-  "Comma-separated list of name:package[&package[&package]] - build 'package' (a single package or a list of packages separated by '&') into sysext 'name', and include with OS image and update payload. Must be in order of dependencies, base sysexts come first."
+  "The torcx manifest describing torcx packages for this image (or blank for none)"
+DEFINE_string torcx_root "${DEFAULT_BUILD_ROOT}/torcx" \
+  "Directory in which torcx packages can be found. Will update the default --torcx_manifest if set."
 DEFINE_string output_root "${DEFAULT_BUILD_ROOT}/images" \
   "Directory in which to place image result directories (named by version)"
 DEFINE_string disk_layout "" \
   "The disk layout type to use for this image."
 DEFINE_string group "${DEFAULT_GROUP}" \
   "The update group."
+DEFINE_boolean generate_update "${FLAGS_FALSE}" \
+  "Generate update payload. (prod only)"
 DEFINE_boolean extract_update "${FLAGS_TRUE}" \
-  "Extract the /usr partition for generating updates. Only valid for the prod image."
+  "Extract the /usr partition for generating updates."
-DEFINE_boolean generate_update "${FLAGS_TRUE}" \
-  "Generate update payload for testing. The update is signed with a dev key. The kernel is signed with a dev key (unofficial builds) or not at all (official builds). Only valid for the prod image. Implies --extract_update."
 DEFINE_string developer_data "" \
   "Insert a custom cloudinit file into the image."
 DEFINE_string devcontainer_binhost "${DEFAULT_DEVCONTAINER_BINHOST}" \
   "Override portage binhost configuration used in development container."
-DEFINE_string oem_sysexts "everything!" \
-  "A comma-separated list of OEMs to build, by default build all the OEM sysexts. Used only if building OEM sysexts"
 
 # include upload options
 . "${BUILD_LIBRARY_DIR}/release_util.sh" || exit 1
@@ -62,12 +62,10 @@ different forms.  This scripts can be used to build the following:
 prod - Production image for CoreOS. This image is for booting (default if no argument is given).
 prodtar - Production container tar ball (implies prod). This can e.g. be used to run the Flatcar production image as a container (run machinectl import-tar or docker import).
 container - Developer image with single filesystem, bootable by nspawn.
-sysext - Build extra sysexts (podman, python, zfs, etc.).
-oem_sysext - Build OEM sysexts for all supported platforms.
 
 Examples:
 
-build_image --board=<board> [prod] [prodtar] [container] [sysext] [oem_sysext] - builds developer and production images/tars.
+build_image --board=<board> [prod] [prodtar] [container] - builds developer and production images/tars.
 ...
 "
 show_help_if_requested "$@"
@@ -85,12 +83,19 @@ DEFINE_string version "" \
 # Parse command line.
 FLAGS "$@" || exit 1
 
-eval set -- "${FLAGS_ARGV:-prod oem_sysext}"
+eval set -- "${FLAGS_ARGV:-prod}"
 
 # Only now can we die on error.  shflags functions leak non-zero error codes,
 # so will die prematurely if 'switch_to_strict_mode' is specified before now.
 switch_to_strict_mode
 
+check_gsutil_opts
+
+# Patch around default values not being able to depend on other flags.
+if [ "x${FLAGS_torcx_manifest}" = "x${DEFAULT_BUILD_ROOT}/torcx/${DEFAULT_BOARD}/latest/torcx_manifest.json" ]; then
+  FLAGS_torcx_manifest="${FLAGS_torcx_root}/${FLAGS_board}/latest/torcx_manifest.json"
+fi
+
 # If downloading packages is enabled ensure the board is configured properly.
 if [[ ${FLAGS_getbinpkg} -eq ${FLAGS_TRUE} ]]; then
   "${SRC_ROOT}/scripts/setup_board" --board="${FLAGS_board}" \
@@ -105,22 +110,17 @@ fi
 . "${BUILD_LIBRARY_DIR}/prod_image_util.sh" || exit 1
 . "${BUILD_LIBRARY_DIR}/dev_container_util.sh" || exit 1
 . "${BUILD_LIBRARY_DIR}/test_image_content.sh" || exit 1
+. "${BUILD_LIBRARY_DIR}/torcx_manifest.sh" || exit 1
 . "${BUILD_LIBRARY_DIR}/vm_image_util.sh" || exit 1
-. "${BUILD_LIBRARY_DIR}/extra_sysexts.sh" || exit 1
-. "${BUILD_LIBRARY_DIR}/oem_sysexts.sh" || exit 1
 
 PROD_IMAGE=0
 PROD_TAR=0
 CONTAINER=0
-SYSEXT=0
-OEM_SYSEXT=0
 for arg in "$@"; do
   case "${arg}" in
     prod) PROD_IMAGE=1 ;;
     prodtar) PROD_IMAGE=1 PROD_TAR=1 ;;
     container) CONTAINER=1 ;;
-    sysext) SYSEXT=1 ;;
-    oem_sysext) OEM_SYSEXT=1 ;;
     *)    die_notrace "Unknown image type ${arg}" ;;
   esac
 done
@@ -132,7 +132,7 @@ if [[ ${skip_test_build_root} -ne 1 ]]; then
 fi
 
 # Handle existing directory.
-if [[ -e "${BUILD_DIR}" ]] && [[ "${PROD_IMAGE}" = 1 ]]; then
+if [[ -e "${BUILD_DIR}" ]]; then
   if [[ ${FLAGS_replace} -eq ${FLAGS_TRUE} ]]; then
     sudo rm -rf "${BUILD_DIR}"
   else
@@ -146,11 +146,6 @@ fi
 # Create the output directory and temporary mount points.
 mkdir -p "${BUILD_DIR}"
 
-# --generate_update implies --extract_update.
-if [[ ${FLAGS_generate_update} -eq ${FLAGS_TRUE} ]]; then
-  FLAGS_extract_update=${FLAGS_TRUE}
-fi
-
 DISK_LAYOUT="${FLAGS_disk_layout:-base}"
 CONTAINER_LAYOUT="${FLAGS_disk_layout:-container}"
 
@@ -180,25 +175,20 @@ fi
 
 if [[ "${PROD_IMAGE}" -eq 1 ]]; then
   IMAGE_BUILD_TYPE="prod"
-  create_prod_image ${FLATCAR_PRODUCTION_IMAGE_NAME} ${DISK_LAYOUT} ${FLAGS_group} ${FLAGS_base_pkg} ${FLAGS_base_sysexts}
+  create_prod_image ${FLATCAR_PRODUCTION_IMAGE_NAME} ${DISK_LAYOUT} ${FLAGS_group} ${FLAGS_base_pkg}
-  if [[ ${FLAGS_extract_update} -eq ${FLAGS_TRUE} ]]; then
+  if [[ ${FLAGS_generate_update} -eq ${FLAGS_TRUE} ]]; then
+    generate_update "${FLATCAR_PRODUCTION_IMAGE_NAME}" ${DISK_LAYOUT}
+  elif [[ ${FLAGS_extract_update} -eq ${FLAGS_TRUE} ]]; then
     extract_update "${FLATCAR_PRODUCTION_IMAGE_NAME}" "${DISK_LAYOUT}"
   fi
-  if [[ ${FLAGS_generate_update} -eq ${FLAGS_TRUE} && ${COREOS_OFFICIAL:-0} -ne 1 ]]; then
-    generate_update "${FLATCAR_PRODUCTION_IMAGE_NAME}" "${DISK_LAYOUT}"
-  fi
   if [[ "${PROD_TAR}" -eq 1 ]]; then
     create_prod_tar ${FLATCAR_PRODUCTION_IMAGE_NAME}
   fi
 fi
-if [[ "${SYSEXT}" -eq 1 ]]; then
-  create_prod_sysexts "${FLATCAR_PRODUCTION_IMAGE_NAME}"
-fi
-if [[ "${OEM_SYSEXT}" -eq 1 ]]; then
-  create_oem_sysexts "${FLATCAR_PRODUCTION_IMAGE_NAME}" "${FLAGS_oem_sysexts}"
-fi
 
-if [[ ${FLAGS_extract_update} -eq ${FLAGS_TRUE} ]]; then
+if [[ ${FLAGS_generate_update} -eq ${FLAGS_TRUE} ]] || \
+   [[ ${FLAGS_extract_update} -eq ${FLAGS_TRUE} ]]
+then
   zip_update_tools
 fi
 
@@ -214,6 +204,8 @@ FLATCAR_BUILD_ID="${FLATCAR_BUILD_ID}"
 FLATCAR_SDK_VERSION=${FLATCAR_SDK_VERSION}
 EOF
 
+upload_image "${BUILD_DIR}/version.txt"
+
 # Create a named symlink.
 set_build_symlinks latest "${FLAGS_group}-latest"
 
@@ -240,3 +232,5 @@ if [[ "${PROD_IMAGE}" -eq 1 ]]; then
 fi
 
 command_completed
+
+

build_library/build_image_util.sh

@@ -20,7 +20,6 @@ BUILD_DIR="${FLAGS_output_root}/${BOARD}/${IMAGE_SUBDIR}"
 OUTSIDE_OUTPUT_DIR="../build/images/${BOARD}/${IMAGE_SUBDIR}"
 
 source "${BUILD_LIBRARY_DIR}/reports_util.sh" || exit 1
-source "${BUILD_LIBRARY_DIR}/sbsign_util.sh" || exit 1
 
 set_build_symlinks() {
     local build=$(basename ${BUILD_DIR})
@@ -61,34 +60,34 @@ delete_prompt() {
 extract_update() {
   local image_name="$1"
   local disk_layout="$2"
-  local update="${BUILD_DIR}/${image_name%_image.bin}_update.bin"
+  local update_path="${BUILD_DIR}/${image_name%_image.bin}_update.bin"
+  local digest_path="${update_path}.DIGESTS"
 
   "${BUILD_LIBRARY_DIR}/disk_util" --disk_layout="${disk_layout}" \
-    extract "${BUILD_DIR}/${image_name}" "USR-A" "${update}"
+    extract "${BUILD_DIR}/${image_name}" "USR-A" "${update_path}"
 
   # Compress image
-  files_to_evaluate+=( "${update}" )
+  files_to_evaluate+=( "${update_path}" )
-  compress_disk_images files_to_evaluate
+  declare -a compressed_images
-}
+  declare -a extra_files
+  compress_disk_images files_to_evaluate compressed_images extra_files
 
-generate_update() {
+  # Upload compressed image
-  local image_name="$1"
+  upload_image -d "${digest_path}" "${compressed_images[@]}" "${extra_files[@]}"
-  local disk_layout="$2"
-  local image_kernel="${BUILD_DIR}/${image_name%.bin}.vmlinuz"
-  local update="${BUILD_DIR}/${image_name%_image.bin}_update.bin"
-  local devkey="/usr/share/update_engine/update-payload-key.key.pem"
 
-  # Extract the partition if it isn't extracted already.
+  # Upload legacy digests
-  [[ -s ${update} ]] ||
+  upload_legacy_digests "${digest_path}" compressed_images
-    "${BUILD_LIBRARY_DIR}/disk_util" --disk_layout="${disk_layout}" \
-      extract "${BUILD_DIR}/${image_name}" "USR-A" "${update}"
 
-  echo "Generating update payload, signed with a dev key"
+  # For production as well as dev builds we generate a dev-key-signed update
+  # payload for running tests (the signature won't be accepted by production systems).
+  local update_test="${BUILD_DIR}/flatcar_test_update.gz"
   delta_generator \
-      -private_key "${devkey}" \
+      -private_key "/usr/share/update_engine/update-payload-key.key.pem" \
-      -new_image "${update}" \
+      -new_image "${update_path}" \
-      -new_kernel "${image_kernel}" \
+      -new_kernel "${BUILD_DIR}/${image_name%.bin}.vmlinuz" \
-      -out_file "${BUILD_DIR}/flatcar_test_update.gz"
+      -out_file "${update_test}"
+
+  upload_image "${update_test}"
 }
 
 zip_update_tools() {
@@ -97,9 +96,42 @@ zip_update_tools() {
 
   info "Generating update tools zip"
   # Make sure some vars this script needs are exported
-  local -x REPO_MANIFESTS_DIR=${REPO_MANIFESTS_DIR} SCRIPTS_DIR=${SCRIPTS_DIR}
+  export REPO_MANIFESTS_DIR SCRIPTS_DIR
   "${BUILD_LIBRARY_DIR}/generate_au_zip.py" \
     --arch "$(get_sdk_arch)" --output-dir "${BUILD_DIR}" --zip-name "${update_zip}"
+
+  upload_image "${BUILD_DIR}/${update_zip}"
+}
+
+generate_update() {
+  local image_name="$1"
+  local disk_layout="$2"
+  local image_kernel="${BUILD_DIR}/${image_name%.bin}.vmlinuz"
+  local update_prefix="${image_name%_image.bin}_update"
+  local update="${BUILD_DIR}/${update_prefix}"
+  local devkey="/usr/share/update_engine/update-payload-key.key.pem"
+
+  echo "Generating update payload, signed with a dev key"
+  "${BUILD_LIBRARY_DIR}/disk_util" --disk_layout="${disk_layout}" \
+    extract "${BUILD_DIR}/${image_name}" "USR-A" "${update}.bin"
+  delta_generator \
+      -private_key "${devkey}" \
+      -new_image "${update}.bin" \
+      -new_kernel "${image_kernel}" \
+      -out_file "${update}.gz"
+
+  # Compress image
+  declare -a files_to_evaluate
+  declare -a compressed_images
+  declare -a extra_files
+  files_to_evaluate+=( "${update}.bin" )
+  compress_disk_images files_to_evaluate compressed_images extra_files
+
+  # Upload images
+  upload_image -d "${update}.DIGESTS" "${update}".{gz,zip} "${compressed_images[@]}" "${extra_files[@]}"
+
+  # Upload legacy digests
+  upload_legacy_digests "${update}.DIGESTS" compressed_images
 }
 
 # ldconfig cannot generate caches for non-native arches.
@@ -150,14 +182,9 @@ emerge_to_image() {
   fi
 
   sudo -E ROOT="${root_fs_dir}" \
-      FEATURES="-ebuild-locks -merge-wait" \
+      FEATURES="-ebuild-locks" \
       PORTAGE_CONFIGROOT="${BUILD_DIR}"/configroot \
-      emerge \
+      emerge --root-deps=rdeps --usepkgonly --jobs="${NUM_JOBS}" --verbose "$@"
-      --usepkgonly \
-      --binpkg-respect-use=y \
-      --jobs="${NUM_JOBS}" \
-      --verbose \
-      "$@"
 
   # Shortcut if this was just baselayout
   [[ "$*" == *sys-apps/baselayout ]] && return
@@ -171,6 +198,26 @@ emerge_to_image() {
       test_image_content "${root_fs_dir}"
 }
 
+# emerge_to_image without a rootfs check; you should use emerge_to_image unless
+# here's a good reason not to.
+emerge_to_image_unchecked() {
+  local root_fs_dir="$1"; shift
+
+  if [[ ${FLAGS_getbinpkg} -eq ${FLAGS_TRUE} ]]; then
+    set -- --getbinpkg "$@"
+  fi
+
+  sudo -E ROOT="${root_fs_dir}" \
+      PORTAGE_CONFIGROOT="${BUILD_DIR}"/configroot \
+      emerge --root-deps=rdeps --usepkgonly --jobs="${NUM_JOBS}" --verbose "$@"
+
+  # Shortcut if this was just baselayout
+  [[ "$*" == *sys-apps/baselayout ]] && return
+
+  # Make sure profile.env has been generated
+  sudo -E ROOT="${root_fs_dir}" env-update --no-ldconfig
+}
+
 # Switch to the dev or prod sub-profile
 set_image_profile() {
   local suffix="$1"
@@ -213,8 +260,8 @@ image_packages_portage() {
     ROOT="$1" PORTAGE_CONFIGROOT="${BUILD_DIR}"/configroot \
         equery --no-color list --format '$cpv::$repo' '*'
 }
-
+# List packages implicitly contained in rootfs, such as in torcx packages or
-# List packages implicitly contained in rootfs, such as in initramfs.
+# initramfs.
 image_packages_implicit() {
     local profile="${BUILD_DIR}/configroot/etc/portage/profile"
 
@@ -243,6 +290,11 @@ image_packages_implicit() {
             query_available_package "${pkg}"
         done < "${profile}/package.provided"
     fi
+
+    # Include source packages of all torcx images installed on disk.
+    [ -z "${FLAGS_torcx_manifest}" ] ||
+    torcx_manifest::sources_on_disk "${FLAGS_torcx_manifest}" |
+    while read pkg ; do query_available_package "${pkg}" ; done
 }
 
 # Generate a list of packages installed in an image.
@@ -262,7 +314,7 @@ write_packages() {
 # Generate an SPDX SBOM using syft
 write_sbom() {
     info "Writing ${2##*/}"
-    sudo syft scan "${1}" -o spdx-json="$2"
+    sudo syft packages "${1}" -o spdx-json="$2"
 }
 
 # Get metadata $key for package $pkg installed under $prefix
@@ -289,16 +341,18 @@ get_metadata() {
     if [ "${key}" = "SRC_URI" ]; then
         local package_name="$(echo "${pkg%%:*}" | cut -d / -f 2)"
         local ebuild_path="${prefix}/var/db/pkg/${pkg%%:*}/${package_name}.ebuild"
+        # SRC_URI is empty for the special github.com/flatcar projects
         if [ -z "${val}" ]; then
             # The grep invocation gives errors when the ebuild file is not present.
             # This can happen when the binary packages from ./build_packages are outdated.
-            val="$(grep "EGIT_REPO_URI=" "${ebuild_path}" | cut -d '"' -f 2)"
+            val="$(grep "CROS_WORKON_PROJECT=" "${ebuild_path}" | cut -d '"' -f 2)"
             if [ -n "${val}" ]; then
-                # If using git, then the package was probably pinned to a commit.
+                val="https://github.com/${val}"
+                # All github.com/flatcar projects specify their commit
                 local commit=""
-                commit="$(grep "EGIT_COMMIT=" "${ebuild_path}" | cut -d '"' -f 2)"
+                commit="$(grep "CROS_WORKON_COMMIT=" "${ebuild_path}" | cut -d '"' -f 2)"
                 if [ -n "${commit}" ]; then
-                    val="${val%.git}/commit/${commit}"
+                    val="${val}/commit/${commit}"
                 fi
             fi
         fi
@@ -307,13 +361,17 @@ get_metadata() {
             # Do not attempt to postprocess by resolving ${P} and friends because it does not affect production images
             val="$(cat "${ebuild_path}" | tr '\n' ' ' | grep -P -o 'SRC_URI=".*?"' | cut -d '"' -f 2)"
         fi
+        # Some packages use nothing from the above but EGIT_REPO_URI (currently only app-crypt/go-tspi)
+        if [ -z "${val}" ]; then
+            val="$(grep "EGIT_REPO_URI=" "${ebuild_path}" | cut -d '"' -f 2)"
+        fi
         # Replace all mirror://MIRRORNAME/ parts with the actual URL prefix of the mirror
         new_val=""
         for v in ${val}; do
             local mirror="$(echo "${v}" | grep mirror:// | cut -d '/' -f 3)"
             if [ -n "${mirror}" ]; then
                 # Take only first mirror, those not working should be removed
-                local location="$(grep "^${mirror}"$'\t' /mnt/host/source/src/third_party/portage-stable/profiles/thirdpartymirrors | cut -d $'\t' -f 2- | cut -d ' ' -f 1 | tr -d $'\t')"
+                local location="$(grep "^${mirror}"$'\t' /var/gentoo/repos/gentoo/profiles/thirdpartymirrors | cut -d $'\t' -f 2- | cut -d ' ' -f 1 | tr -d $'\t')"
                 v="$(echo "${v}" | sed "s#mirror://${mirror}/#${location}#g")"
             fi
             new_val+="${v} "
@@ -438,7 +496,8 @@ EOF
     license_list="$(jq -r '.[] | "\(.licenses | .[])"' "${json_input}" | sort | uniq)"
     local license_dirs=(
         "/mnt/host/source/src/third_party/coreos-overlay/licenses/"
-        "/mnt/host/source/src/third_party/portage-stable/licenses/"
+        "/mnt/host/source/src/third_party/portage-stable/"
+        "/var/gentoo/repos/gentoo/licenses/"
         "none"
     )
     for license_file in ${license_list}; do
@@ -458,6 +517,8 @@ EOF
 # Add /usr/share/SLSA reports for packages indirectly contained within the rootfs
 # If the package is available in BOARD_ROOT accesses it from there, otherwise
 # needs to download binpkg.
+# Reports for torcx packages are also included when adding the torcx package to
+# rootfs.
 insert_extra_slsa() {
   info "Inserting additional SLSA file"
   local rootfs="$1"
@@ -475,8 +536,7 @@ insert_extra_slsa() {
     if [ -f "${binpkg}" ]; then
       info "Found ${atom} at ${binpkg}"
       qtbz2 -O -t "${binpkg}" | \
-        lbzcat -d -c - | \
+        sudo tar -C "${rootfs}" -xj --wildcards './usr/share/SLSA'
-        sudo tar -C "${rootfs}" -x --wildcards './usr/share/SLSA'
       continue
     fi
     warn "Missing SLSA information for ${atom}"
@@ -485,7 +545,7 @@ insert_extra_slsa() {
 
 # Add an entry to the image's package.provided
 package_provided() {
-    local p profile="${BUILD_DIR}/configroot/etc/portage/profile"
+    local p profile="${BUILD_DIR}/configroot/etc/portage/profile"    
     for p in "$@"; do
         info "Writing $p to package.provided and soname.provided"
         echo "$p" >> "${profile}/package.provided"
@@ -562,12 +622,31 @@ finish_image() {
   local image_initrd_contents="${11}"
   local image_initrd_contents_wtd="${12}"
   local image_disk_space_usage="${13}"
-  local image_realinitrd_contents="${14}"
-  local image_realinitrd_contents_wtd="${15}"
 
   local install_grub=0
   local disk_img="${BUILD_DIR}/${image_name}"
 
+  # Copy in packages from the torcx store that are marked as being on disk
+  if [ -n "${FLAGS_torcx_manifest}" ]; then
+    for pkg in $(torcx_manifest::get_pkg_names "${FLAGS_torcx_manifest}"); do
+      local default_version="$(torcx_manifest::default_version "${FLAGS_torcx_manifest}" "${pkg}")"
+      for version in $(torcx_manifest::get_versions "${FLAGS_torcx_manifest}" "${pkg}"); do
+        local on_disk_path="$(torcx_manifest::local_store_path "${FLAGS_torcx_manifest}" "${pkg}" "${version}")"
+        if [[ -n "${on_disk_path}" ]]; then
+          local casDigest="$(torcx_manifest::get_digest "${FLAGS_torcx_manifest}" "${pkg}" "${version}")"
+          sudo cp "${FLAGS_torcx_root}/pkgs/${BOARD}/${pkg}/${casDigest}/${pkg}:${version}.torcx.tgz" \
+            "${root_fs_dir}${on_disk_path}"
+          sudo tar xf "${root_fs_dir}${on_disk_path}" -C "${root_fs_dir}" --wildcards "./usr/share/SLSA"
+          if [[ "${version}" == "${default_version}" ]]; then
+            # Create the default symlink for this package
+            sudo ln -fns "${on_disk_path##*/}" \
+              "${root_fs_dir}/${on_disk_path%/*}/${pkg}:com.coreos.cl.torcx.tgz"
+          fi
+        fi
+      done
+    done
+  fi
+
   # Only enable rootfs verification on prod builds.
   local disable_read_write="${FLAGS_FALSE}"
   if [[ "${IMAGE_BUILD_TYPE}" == "prod" ]]; then
@@ -624,7 +703,7 @@ finish_image() {
   # --allow-user=root
   # --allow-user=core
   mapfile -t allowed_users < <(grep '^COPY_USERS=' "${root_fs_dir}/sbin/flatcar-tmpfiles" | sed -e 's/.*="\([^"]*\)"/\1/' | tr '|' '\n' | sed -e 's/^/--allow-user=/')
-  mapfile -t allowed_groups < <(grep '^COPY_GROUPS=' "${root_fs_dir}/sbin/flatcar-tmpfiles" | sed -e 's/.*="\([^"]*\)"/\1/' | tr '|' '\n' | sed -e 's/^/--allow-group=/')
+  mapfile -t allowed_users < <(grep '^COPY_GROUPS=' "${root_fs_dir}/sbin/flatcar-tmpfiles" | sed -e 's/.*="\([^"]*\)"/\1/' | tr '|' '\n' | sed -e 's/^/--allow-group=/')
   sudo "${BUILD_LIBRARY_DIR}/gen_tmpfiles.py" --root="${root_fs_dir}" \
       --output="${root_fs_dir}/usr/lib/tmpfiles.d/base_image_var.conf" \
       "${ignores[@]}" "${allowed_users[@]}" "${allowed_groups[@]}" "${root_fs_dir}/var"
@@ -708,17 +787,6 @@ EOF
     sudo setfiles -Dv -r "${root_fs_dir}" "${root_fs_dir}"/etc/selinux/mcs/contexts/files/file_contexts "${root_fs_dir}"/etc
   fi
 
-  # Temporary hack: set group ownership of /etc/{g,}shadow to the
-  # shadow group, that way unix_chkpwd, chage and expiry can act on
-  # those files.
-  #
-  # This permissions setting should likely be done in some ebuild, but
-  # currently files in /usr/share/baselayout are installed by the
-  # baselayout package, we don't want to add more deps to it.
-  sudo chgrp \
-      --reference="${root_fs_dir}/usr/bin/chage" \
-      "${root_fs_dir}"/{etc,usr/share/baselayout}/{g,}shadow
-
   # Backup the /etc contents to /usr/share/flatcar/etc to serve as
   # source for creating missing files. Make sure that the preexisting
   # /usr/share/flatcar/etc does not have any meaningful (non-empty)
@@ -728,35 +796,12 @@ EOF
   if [[ $(sudo find "${root_fs_dir}/usr/share/flatcar/etc" -size +0 ! -type d 2>/dev/null | wc -l) -gt 0 ]]; then
     die "Unexpected non-empty files in ${root_fs_dir}/usr/share/flatcar/etc"
   fi
-  # Some backwards-compat symlinks still use this folder as target,
-  # we can't remove it yet
   sudo rm -rf "${root_fs_dir}/usr/share/flatcar/etc"
   sudo cp -a "${root_fs_dir}/etc" "${root_fs_dir}/usr/share/flatcar/etc"
-  # Now set up a default confext and enable it.
-  # It's important to use dm-verity not only for stricter image policies
-  # but also because it allows us the refresh to identify this image and
-  # skip setting it up again in the final boot, which not only saves us
-  # a daemon-reload during boot but also from /etc contents shortly
-  # disappearing until systemd-sysext uses mount beneath for an atomic
-  # remount. Instead of a temporary directory we first prepare it as
-  # folder and then convert it to a DDI and remove the folder.
-  sudo rm -rf "${root_fs_dir}/usr/lib/confexts/00-flatcar-default"
-  sudo mkdir -p "${root_fs_dir}/usr/lib/confexts/00-flatcar-default"
-  # Do a copy because we keep /etc for the flatcar (.tar) container and the developer container
-  sudo cp -a "${root_fs_dir}/etc" "${root_fs_dir}/usr/lib/confexts/00-flatcar-default/etc"
-  sudo mkdir -p "${root_fs_dir}/usr/lib/confexts/00-flatcar-default/etc/extension-release.d/"
-  echo ID=_any | sudo tee "${root_fs_dir}/usr/lib/confexts/00-flatcar-default/etc/extension-release.d/extension-release.00-flatcar-default" > /dev/null
-  sudo systemd-repart \
-    --private-key="${SYSEXT_SIGNING_KEY_DIR}/sysexts.key" \
-    --certificate="${SYSEXT_SIGNING_KEY_DIR}/sysexts.crt" \
-    --make-ddi=confext \
-    --copy-source="${root_fs_dir}/usr/lib/confexts/00-flatcar-default" \
-    "${root_fs_dir}/usr/lib/confexts/00-flatcar-default.raw"
-  sudo rm -rf "${root_fs_dir}/usr/lib/confexts/00-flatcar-default"
 
-  # Remove the rootfs state as it should be recreated through tmpfiles
+  # Remove the rootfs state as it should be recreated through the
-  # (and for /etc we use a confext) and may not be present on updating machines.
+  # tmpfiles and may not be present on updating machines. This
-  # This makes sure our tests cover the case of missing files in the
+  # makes sure our tests cover the case of missing files in the
   # rootfs and don't rely on the new image. Not done for the developer
   # container.
   if [[ -n "${image_kernel}" ]]; then
@@ -809,11 +854,13 @@ EOF
         seek=${verity_offset} count=64 bs=1 status=none
   fi
 
-  # Sign the kernel after /usr is in a consistent state and verity is
+  # Sign the kernel after /usr is in a consistent state and verity is calculated
-  # calculated. Only for unofficial builds as official builds get signed later.
   if [[ ${COREOS_OFFICIAL:-0} -ne 1 ]]; then
-    do_sbsign --output "${root_fs_dir}/boot/flatcar/vmlinuz-a"{,}
+      sudo sbsign --key /usr/share/sb_keys/DB.key \
-    cleanup_sbsign_certs
+	   --cert /usr/share/sb_keys/DB.crt \
+	   "${root_fs_dir}/boot/flatcar/vmlinuz-a"
+      sudo mv "${root_fs_dir}/boot/flatcar/vmlinuz-a.signed" \
+	   "${root_fs_dir}/boot/flatcar/vmlinuz-a"
   fi
 
   if [[ -n "${image_kernel}" ]]; then
@@ -868,7 +915,7 @@ EOF
 
     info "Generating $pcr_policy"
     pushd "${BUILD_DIR}" >/dev/null
-    zip --quiet -r -9 "${pcr_policy}" pcrs
+    zip --quiet -r -9 "${BUILD_DIR}/${pcr_policy}" pcrs
     popd >/dev/null
     rm -rf "${BUILD_DIR}/pcrs"
   fi
@@ -893,20 +940,6 @@ EOF
       rm -rf "${BUILD_DIR}/tmp_initrd_contents"
   fi
 
-  if [[ -n ${image_realinitrd_contents} || -n ${image_realinitrd_contents_wtd} ]]; then
-        mkdir -p "${BUILD_DIR}/tmp_initrd_contents"
-        sudo mount "${root_fs_dir}/usr/lib/flatcar/bootengine.img" "${BUILD_DIR}/tmp_initrd_contents"
-        if [[ -n ${image_realinitrd_contents} ]]; then
-            write_contents "${BUILD_DIR}/tmp_initrd_contents" "${BUILD_DIR}/${image_realinitrd_contents}"
-        fi
-
-        if [[ -n ${image_realinitrd_contents_wtd} ]]; then
-            write_contents_with_technical_details "${BUILD_DIR}/tmp_initrd_contents" "${BUILD_DIR}/${image_realinitrd_contents_wtd}"
-        fi
-        sudo umount "${BUILD_DIR}/tmp_initrd_contents"
-        rm -rf "${BUILD_DIR}/tmp_initrd_contents"
-    fi
-
   if [[ -n "${image_disk_space_usage}" ]]; then
       write_disk_space_usage "${root_fs_dir}" "${BUILD_DIR}/${image_disk_space_usage}"
   fi
@@ -914,67 +947,3 @@ EOF
   cleanup_mounts "${root_fs_dir}"
   trap - EXIT
 }
-
-sbsign_image() {
-  local image_name="$1"
-  local disk_layout="$2"
-  local root_fs_dir="$3"
-  local image_kernel="$4"
-  local pcr_policy="$5"
-  local image_grub="$6"
-
-  local disk_img="${BUILD_DIR}/${image_name}"
-  local EFI_ARCH
-
-  case "${BOARD}" in
-    amd64-usr) EFI_ARCH="x64" ;;
-    arm64-usr) EFI_ARCH="aa64" ;;
-    *) die "Unknown board ${BOARD@Q}" ;;
-  esac
-
-  "${BUILD_LIBRARY_DIR}/disk_util" --disk_layout="${disk_layout}" \
-      mount "${disk_img}" "${root_fs_dir}"
-  trap "cleanup_mounts '${root_fs_dir}'; cleanup_sbsign_certs" EXIT
-
-  # Sign the kernel with the shim-embedded key.
-  do_sbsign --output "${root_fs_dir}/boot/flatcar/vmlinuz-a"{,}
-
-  if [[ -n "${image_kernel}" ]]; then
-    # copying kernel from vfat so ignore the permissions
-    cp --no-preserve=mode \
-        "${root_fs_dir}/boot/flatcar/vmlinuz-a" \
-        "${BUILD_DIR}/${image_kernel}"
-  fi
-
-  # Sign GRUB and mokmanager(mm) with the shim-embedded key.
-  do_sbsign --output "${root_fs_dir}/boot/EFI/boot/grub${EFI_ARCH}.efi"{,}
-  do_sbsign --output "${root_fs_dir}/boot/EFI/boot/mm${EFI_ARCH}.efi"{,}
-
-  # copying from vfat so ignore permissions
-  if [[ -n "${image_grub}" ]]; then
-    cp --no-preserve=mode "${root_fs_dir}/boot/EFI/boot/grub${EFI_ARCH}.efi" \
-        "${BUILD_DIR}/${image_grub}"
-  fi
-
-  if [[ -n "${pcr_policy}" ]]; then
-    mkdir -p "${BUILD_DIR}/pcrs"
-    "${BUILD_LIBRARY_DIR}"/generate_kernel_hash.py \
-        "${root_fs_dir}/boot/flatcar/vmlinuz-a" "${FLATCAR_VERSION}" \
-        >"${BUILD_DIR}/pcrs/kernel.config"
-  fi
-
-  cleanup_mounts "${root_fs_dir}"
-  cleanup_sbsign_certs
-  trap - EXIT
-
-  if [[ -n "${pcr_policy}" ]]; then
-    "${BUILD_LIBRARY_DIR}"/generate_grub_hashes.py \
-        "${disk_img}" /usr/lib/grub/ "${BUILD_DIR}/pcrs" "${FLATCAR_VERSION}"
-
-    info "Generating $pcr_policy"
-    pushd "${BUILD_DIR}" >/dev/null
-    zip --quiet -r -9 "${BUILD_DIR}/${pcr_policy}" pcrs
-    popd >/dev/null
-    rm -rf "${BUILD_DIR}/pcrs"
-  fi
-}

build_library/catalyst.sh

@@ -55,15 +55,17 @@ DEFINE_boolean debug ${FLAGS_FALSE} "Enable verbose output from catalyst."
 catalyst_conf() {
 cat <<EOF
 # catalyst.conf
-digests=["md5", "sha1", "sha512", "blake2b"]
+contents="auto"
-options=["pkgcache"]
+digests="md5 sha1 sha512 whirlpool"
+hash_function="crc32"
+options="pkgcache"
 sharedir="/usr/share/catalyst"
 storedir="$CATALYST_ROOT"
 distdir="$DISTDIR"
 envscript="$TEMPDIR/catalystrc"
 port_logdir="$CATALYST_ROOT/log"
-repo_basedir="/mnt/host/source/src/third_party"
+portdir="$FLAGS_portage_stable"
-repo_name="portage-stable"
+snapshot_cache="$CATALYST_ROOT/tmp/snapshot_cache"
 EOF
 }
 
@@ -80,42 +82,61 @@ export ac_cv_posix_semaphores_enabled=yes
 EOF
 }
 
-# Common values for all stage spec files. Takes a stage number and,
+repos_conf() {
-# optionally, a profile name as parameters.
+cat <<EOF
+[DEFAULT]
+main-repo = portage-stable
+
+[coreos]
+location = /var/gentoo/repos/local
+
+[portage-stable]
+location = /var/gentoo/repos/gentoo
+EOF
+}
+
+# Common values for all stage spec files
 catalyst_stage_default() {
 cat <<EOF
-target: stage$1
 subarch: $ARCH
 rel_type: $TYPE
 portage_confdir: $TEMPDIR/portage
-repos: $FLAGS_coreos_overlay
+portage_overlay: $FLAGS_coreos_overlay
-keep_repos: portage-stable coreos-overlay
+profile: $FLAGS_profile
-profile: ${2:-$FLAGS_profile}
+snapshot: $FLAGS_version
-snapshot_treeish: $FLAGS_version
 version_stamp: $FLAGS_version
 cflags: -O2 -pipe
 cxxflags: -O2 -pipe
 ldflags: -Wl,-O2 -Wl,--as-needed
-source_subpath: ${SEED}
 EOF
 }
 
 # Config values for each stage
 catalyst_stage1() {
 cat <<EOF
+target: stage1
 # stage1 packages aren't published, save in tmp
 pkgcache_path: ${TEMPDIR}/stage1-${ARCH}-packages
-update_seed: yes
+update_seed: no
-update_seed_command: --exclude cross-*-cros-linux-gnu/* --exclude dev-lang/rust --exclude dev-lang/rust-bin --ignore-world y --ignore-built-slot-operator-deps y @changed-subslot
 EOF
-catalyst_stage_default 1 "${FLAGS_profile}/transition"
+catalyst_stage_default
+}
+
+catalyst_stage2() {
+cat <<EOF
+target: stage2
+# stage2 packages aren't published, save in tmp
+pkgcache_path: ${TEMPDIR}/stage2-${ARCH}-packages
+EOF
+catalyst_stage_default
 }
 
 catalyst_stage3() {
 cat <<EOF
+target: stage3
 pkgcache_path: $BINPKGS
 EOF
-catalyst_stage_default 3
+catalyst_stage_default
 }
 
 catalyst_stage4() {
@@ -141,10 +162,10 @@ catalyst_init() {
     if [[ -n "${FORCE_STAGES}" ]]; then
         STAGES="${FORCE_STAGES}"
     elif [[ $# -eq 0 ]]; then
-        STAGES="stage1 stage3 stage4"
+        STAGES="stage1 stage2 stage3 stage4"
     else
         for stage in "$@"; do
-            if [[ ! "$stage" =~ ^stage[134]$ ]]; then
+            if [[ ! "$stage" =~ ^stage[1234]$ ]]; then
                 die_notrace "Invalid target name $stage"
             fi
         done
@@ -159,11 +180,6 @@ catalyst_init() {
         die_notrace "catalyst not found, not installed or bad PATH?"
     fi
 
-    # Before doing anything else, ensure we have at least Catalyst 4.
-    if catalyst --version | grep -q "Catalyst [0-3]\."; then
-        emerge --verbose "--jobs=${NUM_JOBS}" --oneshot ">=dev-util/catalyst-4" || exit 1
-    fi
-
     DEBUG=()
     if [[ ${FLAGS_debug} -eq ${FLAGS_TRUE} ]]; then
         DEBUG=("--debug")
@@ -191,8 +207,8 @@ catalyst_init() {
     # so far so good, expand path to work with weird comparison code below
     FLAGS_seed_tarball=$(readlink -f "$FLAGS_seed_tarball")
 
-    if [[ ! "$FLAGS_seed_tarball" =~ .\.tar\.(bz2|xz) ]]; then
+    if [[ ! "$FLAGS_seed_tarball" =~ .*\.tar\.bz2 ]]; then
-        die_notrace "Seed tarball doesn't end in .tar.bz2 or .tar.xz :-/"
+        die_notrace "Seed tarball doesn't end in .tar.bz2 :-/"
     fi
 
     # catalyst is obnoxious and wants the $TYPE/stage3-$VERSION part of the
@@ -200,41 +216,49 @@ catalyst_init() {
     # directory under $TEMPDIR instead, aka the SEEDCACHE feature.)
     if [[ "$FLAGS_seed_tarball" =~ "$CATALYST_ROOT/builds/".* ]]; then
         SEED="${FLAGS_seed_tarball#$CATALYST_ROOT/builds/}"
-        SEED="${SEED%.tar.*}"
+        SEED="${SEED%.tar.bz2}"
     else
         mkdir -p "$CATALYST_ROOT/builds/seed"
         cp -n "$FLAGS_seed_tarball" "$CATALYST_ROOT/builds/seed"
         SEED="seed/${FLAGS_seed_tarball##*/}"
-        SEED="${SEED%.tar.*}"
+        SEED="${SEED%.tar.bz2}"
     fi
 }
 
 write_configs() {
     info "Creating output directories..."
-    mkdir -m 775 -p "$DISTDIR"
+    mkdir -m 775 -p "$TEMPDIR/portage/repos.conf" "$DISTDIR"
     chown portage:portage "$DISTDIR"
     info "Writing out catalyst configs..."
     info "    catalyst.conf"
     catalyst_conf > "$TEMPDIR/catalyst.conf"
     info "    catalystrc"
     catalystrc > "$TEMPDIR/catalystrc"
+    info "    portage/repos.conf/coreos.conf"
+    repos_conf > "$TEMPDIR/portage/repos.conf/coreos.conf"
     info "    stage1.spec"
     catalyst_stage1 > "$TEMPDIR/stage1.spec"
-
+    info "    stage2.spec"
-    info "Configuring Portage..."
+    catalyst_stage2 > "$TEMPDIR/stage2.spec"
-    cp -r "${BUILD_LIBRARY_DIR}"/portage/ "${TEMPDIR}/"
+    info "    stage3.spec"
-
+    catalyst_stage3 > "$TEMPDIR/stage3.spec"
-    ln -sfT '/mnt/host/source/src/third_party/coreos-overlay/coreos/user-patches' \
+    info "    stage4.spec"
-        "${TEMPDIR}"/portage/patches
+    catalyst_stage4 > "$TEMPDIR/stage4.spec"
+    info "Putting a symlink to user patches..."
+    ln -sfT '/var/gentoo/repos/local/coreos/user-patches' \
+        "$TEMPDIR/portage/patches"
 }
 
 build_stage() {
-    local stage catalyst_conf target_tarball
+    local stage srcpath catalyst_conf target_tarball
 
     stage="$1"
-    catalyst_conf="$TEMPDIR/catalyst.conf"
+    srcpath="$2"
+    catalyst_conf="$3"
     target_tarball="${stage}-${ARCH}-${FLAGS_version}.tar.bz2"
 
+    [ -z "$catalyst_conf" ] && catalyst_conf="$TEMPDIR/catalyst.conf"
+
     if [[ -f "$BUILDS/${target_tarball}" && $FLAGS_rebuild == $FLAGS_FALSE ]]
     then
         info "Skipping $stage, $target_tarball already exists."
@@ -246,7 +270,8 @@ build_stage() {
         "${DEBUG[@]}" \
         --verbose \
         --config "$TEMPDIR/catalyst.conf" \
-        --file "$TEMPDIR/${stage}.spec"
+        --file "$TEMPDIR/${stage}.spec" \
+        --cli "source_subpath=$srcpath"
     # Catalyst does not clean up after itself...
     rm -rf "$TEMPDIR/$stage-${ARCH}-${FLAGS_version}"
     ln -sf "$stage-${ARCH}-${FLAGS_version}.tar.bz2" \
@@ -255,19 +280,46 @@ build_stage() {
 }
 
 build_snapshot() {
-    local repo_dir snapshot snapshots_dir snapshot_path
+    local catalyst_conf snapshot snapshots_dir snapshot_base snapshot_path
 
-    repo_dir=${1:-"${FLAGS_portage_stable}"}
+    catalyst_conf=${1:-"${TEMPDIR}/catalyst.conf"}
     snapshot=${2:-"${FLAGS_version}"}
     snapshots_dir="${CATALYST_ROOT}/snapshots"
-    snapshot_path="${snapshots_dir}/portage-stable-${snapshot}.sqfs"
+    snapshot_base="${snapshots_dir}/gentoo-${snapshot}"
-    if [[ -f ${snapshot_path} && $FLAGS_rebuild == $FLAGS_FALSE ]]
+    snapshot_path="${snapshot_base}.tar.bz2"
+    if [[ -f "${snapshot_path}" && $FLAGS_rebuild == $FLAGS_FALSE ]]
     then
         info "Skipping snapshot, ${snapshot_path} exists"
     else
         info "Creating snapshot ${snapshot_path}"
-        mkdir -p "${snapshot_path%/*}"
+        catalyst \
-        tar -c -C "${repo_dir}" . | tar2sqfs "${snapshot_path}" -q -f -j1 -c gzip
+            "${DEBUG[@]}" \
+            --verbose \
+            --config "${catalyst_conf}" \
+            --snapshot "${snapshot}"
+    fi
+    local f
+    local to_remove=()
+    # This will expand to at least our just built snapshot tarball, so
+    # no nullglob is needed here.
+    for f in "${snapshot_base}".*; do
+        case "${f}" in
+            "${snapshot_path}")
+                # Our snapshot, keep it as is.
+                :
+                ;;
+            *.CONTENTS|*.CONTENTS.gz|*.DIGESTS)
+                # These can stay, catalyst is not bothered by those.
+                :
+                ;;
+            *)
+                to_remove+=("${f}")
+                ;;
+        esac
+    done
+    if [[ ${#to_remove[@]} -gt 0 ]]; then
+        info "$(printf '%s\n' 'Found spurious files in snapshots directory that may confuse Catalyst, removing them:' "${to_remove[@]}")"
+        rm -rf "${to_remove[@]}"
     fi
 }
 
@@ -283,17 +335,23 @@ catalyst_build() {
 
     used_seed=0
     if [[ "$STAGES" =~ stage1 ]]; then
-        build_stage stage1
+        build_stage stage1 "$SEED"
+        used_seed=1
+    fi
+
+    if [[ "$STAGES" =~ stage2 ]]; then
+        if [[ $used_seed -eq 1 ]]; then
+            SEED="${TYPE}/stage1-${ARCH}-latest"
+        fi
+        build_stage stage2 "$SEED"
         used_seed=1
     fi
 
     if [[ "$STAGES" =~ stage3 ]]; then
         if [[ $used_seed -eq 1 ]]; then
-            SEED="${TYPE}/stage1-${ARCH}-latest"
+            SEED="${TYPE}/stage2-${ARCH}-latest"
         fi
-        info "    stage3.spec"
+        build_stage stage3 "$SEED"
-        catalyst_stage3 > "$TEMPDIR/stage3.spec"
-        build_stage stage3
         used_seed=1
     fi
 
@@ -301,12 +359,10 @@ catalyst_build() {
         if [[ $used_seed -eq 1 ]]; then
             SEED="${TYPE}/stage3-${ARCH}-latest"
         fi
-        info "    stage4.spec"
+        build_stage stage4 "$SEED"
-        catalyst_stage4 > "$TEMPDIR/stage4.spec"
-        build_stage stage4
         used_seed=1
     fi
 
     # Cleanup snapshots, we don't use them
-    rm -rf "$CATALYST_ROOT/snapshots/${FLAGS_portage_stable##*/}-${FLAGS_version}.sqfs"*
+    rm -rf "$CATALYST_ROOT/snapshots/gentoo-${FLAGS_version}.tar.bz2"*
 }

build_library/catalyst_sdk.sh

@@ -4,9 +4,6 @@ set -e
 source /tmp/chroot-functions.sh
 source /tmp/toolchain_util.sh
 
-ln -vsfT "$(portageq get_repo_path / coreos-overlay)/coreos/user-patches" \
-    /etc/portage/patches
-
 echo "Double checking everything is fresh and happy."
 run_merge -uDN --with-bdeps=y world
 
@@ -14,12 +11,20 @@ echo "Setting the default Python interpreter"
 eselect python update
 
 echo "Building cross toolchain for the SDK."
-configure_crossdev_overlay / /usr/local/portage/crossdev
+configure_crossdev_overlay / /tmp/crossdev
 
 for cross_chost in $(get_chost_list); do
     echo "Building cross toolchain for ${cross_chost}"
     PKGDIR="$(portageq envvar PKGDIR)/crossdev" \
         install_cross_toolchain "${cross_chost}" ${clst_myemergeopts}
+    PKGDIR="$(portageq envvar PKGDIR)/crossdev" \
+        install_cross_rust "${cross_chost}" ${clst_myemergeopts}
 done
 
-PKGDIR="$(portageq envvar PKGDIR)/crossdev" install_cross_rust ${clst_myemergeopts}
+echo "Saving snapshot of coreos-overlay repo for future SDK bootstraps"
+# Copy coreos-overlay, which is in /var/gentoo/repos/local/, into a
+# local directory.  /var/gentoo/repos/local/ is removed before archiving
+# and we want to keep a snapshot. This snapshot is used - alongside
+# /var/gentoo/repos/gentoo - by stage 1 of future bootstraps.
+mkdir -p /var/gentoo/repos/coreos-overlay
+cp -R /var/gentoo/repos/local/* /var/gentoo/repos/coreos-overlay

build_library/catalyst_toolchains.sh

@@ -28,40 +28,16 @@ build_target_toolchain() {
     local ROOT="/build/${board}"
     local SYSROOT="/usr/$(get_board_chost "${board}")"
 
-    function btt_emerge() {
+    mkdir -p "${ROOT}/usr"
-        # --root is required because run_merge overrides ROOT=
+    cp -at "${ROOT}" "${SYSROOT}"/lib*
-        PORTAGE_CONFIGROOT="$ROOT" run_merge --root="$ROOT" --sysroot="$ROOT" "${@}"
+    cp -at "${ROOT}"/usr "${SYSROOT}"/usr/include "${SYSROOT}"/usr/lib*
-    }
 
-    # install baselayout first so we have the basic directory
+    # --root is required because run_merge overrides ROOT=
-    # structure for libraries and binaries copied from sysroot
+    PORTAGE_CONFIGROOT="$ROOT" \
-    btt_emerge --oneshot --nodeps sys-apps/baselayout
+        run_merge -u --root="$ROOT" --sysroot="$ROOT" "${TOOLCHAIN_PKGS[@]}"
-
-    # copy libraries, binaries and header files from sysroot to root -
-    # sysroot may be using split-usr, whereas root does not, so take
-    # this into account
-    (
-        shopt -s nullglob
-        local d f
-        local -a files
-        for d in "${SYSROOT}"/{,usr/}{bin,sbin,lib*}; do
-            if [[ ! -d ${d} ]]; then
-                continue
-            fi
-            files=( "${d}"/* )
-            if [[ ${#files[@]} -gt 0 ]]; then
-                f=${d##*/}
-                cp -at "${ROOT}/usr/${f}" "${files[@]}"
-            fi
-        done
-        cp -at "${ROOT}"/usr "${SYSROOT}"/usr/include
-    )
-
-    btt_emerge --update "${TOOLCHAIN_PKGS[@]}"
-    unset -f btt_emerge
 }
 
-configure_crossdev_overlay / /usr/local/portage/crossdev
+configure_crossdev_overlay / /tmp/crossdev
 
 for board in $(get_board_list); do
     echo "Building native toolchain for ${board}"

build_library/dev_container_util.sh

@@ -38,27 +38,26 @@ CHOST=$(get_board_chost $BOARD)
 DISTDIR="/var/lib/portage/distfiles"
 PKGDIR="/var/lib/portage/pkgs"
 PORT_LOGDIR="/var/log/portage"
-PORTAGE_BINHOST="$(get_binhost_url "${binhost}" "${update_group}" 'pkgs')"
+PORTAGE_BINHOST="$(get_binhost_url "${binhost}" "${update_group}" 'pkgs')
+$(get_binhost_url "${binhost}" "${update_group}" 'toolchain')"
 EOF
 
-    sudo_clobber "${root_fs_dir}/etc/portage/repos.conf/portage-stable.conf" <<EOF
+    sudo_clobber "${root_fs_dir}/etc/portage/repos.conf/coreos.conf" <<EOF
 [DEFAULT]
 main-repo = portage-stable
 
+[coreos]
+location = /var/lib/portage/coreos-overlay
+
 [portage-stable]
 location = /var/lib/portage/portage-stable
-EOF
-
-    sudo_clobber "${root_fs_dir}/etc/portage/repos.conf/coreos-overlay.conf" <<EOF
-[coreos-overlay]
-location = /var/lib/portage/coreos-overlay
 EOF
 
     # Now set the correct profile, we do not use the eselect tool - it
     # does not seem to be usable outside of the chroot without using
     # deprecated PORTDIR and PORTDIR_OVERLAY environment variables.
     local profile_name=$(get_board_profile "${BOARD}")
-    # Turn coreos-overlay:coreos/amd64/generic into coreos/amd64/generic/dev
+    # Turn coreos:coreos/amd64/generic into coreos/amd64/generic/dev
     profile_name="${profile_name#*:}/dev"
     local profile_directory="${root_fs_dir}/var/lib/portage/coreos-overlay/profiles/${profile_name}"
     if [[ ! -d "${profile_directory}" ]]; then
@@ -81,9 +80,7 @@ create_dev_container() {
   fi
 
   info "Building developer image ${image_name}"
-  # The "dev-image-rootfs" directory name is important - it is used to
+  local root_fs_dir="${BUILD_DIR}/rootfs"
-  # determine the package target in coreos/base/profile.bashrc
-  local root_fs_dir="${BUILD_DIR}/dev-image-rootfs"
   local image_contents="${image_name%.bin}_contents.txt"
   local image_contents_wtd="${image_name%.bin}_contents_wtd.txt"
   local image_packages="${image_name%.bin}_packages.txt"
@@ -116,6 +113,20 @@ create_dev_container() {
   finish_image "${image_name}" "${disk_layout}" "${root_fs_dir}" "${image_contents}" "${image_contents_wtd}"
 
   declare -a files_to_evaluate
+  declare -a compressed_images
+  declare -a extra_files
+
   files_to_evaluate+=( "${BUILD_DIR}/${image_name}" )
-  compress_disk_images files_to_evaluate
+  compress_disk_images files_to_evaluate compressed_images extra_files
+
+  upload_image -d "${BUILD_DIR}/${image_name}.DIGESTS" \
+      "${BUILD_DIR}/${image_contents}" \
+      "${BUILD_DIR}/${image_contents_wtd}" \
+      "${BUILD_DIR}/${image_packages}" \
+      "${BUILD_DIR}/${image_licenses}" \
+      "${compressed_images[@]}" \
+      "${extra_files[@]}"
+
+  # Upload legacy digests
+  upload_legacy_digests "${BUILD_DIR}/${image_name}.DIGESTS" compressed_images
 }

build_library/disk_layout.json

@@ -13,10 +13,10 @@
         "label":"EFI-SYSTEM",
         "fs_label":"EFI-SYSTEM",
         "type":"efi",
-        "blocks":"2097152",
+        "blocks":"262144",
         "fs_type":"vfat",
         "mount":"/boot",
-        "features": []
+        "features": ["hybrid"]
       },
       "2":{
         "label":"BIOS-BOOT",
@@ -27,11 +27,9 @@
         "label":"USR-A",
         "uuid":"7130c94a-213a-4e5a-8e26-6cce9662f132",
         "type":"flatcar-rootfs",
-        "blocks":"4194304",
+        "blocks":"2097152",
-        "extract_blocks":"2097152",
         "fs_blocks":"260094",
-        "fs_type":"btrfs",
+        "fs_type":"ext2",
-        "fs_compression":"zstd",
         "mount":"/usr",
         "features": ["prioritize", "verity"]
       },
@@ -39,8 +37,7 @@
         "label":"USR-B",
         "uuid":"e03dd35c-7c2d-4a47-b3fe-27f15780a57c",
         "type":"flatcar-rootfs",
-        "blocks":"4194304",
+        "blocks":"2097152",
-        "extract_blocks":"2097152",
         "fs_blocks":"262144"
       },
       "5":{
@@ -53,7 +50,7 @@
         "label":"OEM",
         "fs_label":"OEM",
         "type":"data",
-        "blocks":"2097152",
+        "blocks":"262144",
         "fs_type":"btrfs",
         "fs_compression":"zlib",
         "mount":"/oem"
@@ -72,7 +69,7 @@
         "label":"ROOT",
         "fs_label":"ROOT",
         "type":"flatcar-resize",
-        "blocks":"3653632",
+        "blocks":"4427776",
         "fs_type":"ext4",
         "mount":"/"
       }
@@ -88,7 +85,7 @@
       "9":{
         "label":"ROOT",
         "fs_label":"ROOT",
-        "blocks":"50876416"
+        "blocks":"58875904"
       }
     },
     "vagrant":{

build_library/disk_util

@@ -40,10 +40,10 @@ def LoadPartitionConfig(options):
       '_comment', 'type', 'num', 'label', 'blocks', 'block_size', 'fs_blocks',
       'fs_block_size', 'fs_type', 'features', 'uuid', 'part_alignment', 'mount',
       'binds', 'fs_subvolume', 'fs_bytes_per_inode', 'fs_inode_size', 'fs_label',
-      'fs_compression', 'extract_blocks'))
+      'fs_compression'))
   integer_layout_keys = set((
       'blocks', 'block_size', 'fs_blocks', 'fs_block_size', 'part_alignment',
-      'fs_bytes_per_inode', 'fs_inode_size', 'extract_blocks'))
+      'fs_bytes_per_inode', 'fs_inode_size'))
   required_layout_keys = set(('type', 'num', 'label', 'blocks'))
 
   filename = options.disk_layout_file
@@ -136,13 +136,6 @@ def LoadPartitionConfig(options):
       part.setdefault('fs_block_size', metadata['fs_block_size'])
       part.setdefault('fs_blocks', part['bytes'] // part['fs_block_size'])
       part['fs_bytes'] = part['fs_blocks'] * part['fs_block_size']
-      # The partition may specify extract_blocks to limit what content gets
-      # extracted. The use case is the /usr partition where we can grow the
-      # partition but can't directly grow the filesystem and the update
-      # payload until all (or most) nodes are running the partition layout
-      # with the grown /usr partition (which can take a few years).
-      if part.get('extract_blocks', None):
-        part['extract_bytes'] = part['extract_blocks'] * metadata['block_size']
 
       if part['fs_bytes'] > part['bytes']:
         raise InvalidLayout(
@@ -610,7 +603,7 @@ def Mount(options):
     if options.read_only or ('verity' in mount.get('features', []) and not options.writable_verity):
       mount_opts.append('ro')
       if mount.get('fs_type', None) == 'btrfs':
-        mount_opts.append('rescue=nologreplay')
+        mount_opts.append('norecovery')
 
     if mount.get('fs_subvolume', None):
       mount_opts.append('subvol=%s' % mount['fs_subvolume'])
@@ -806,7 +799,7 @@ def Verity(options):
                   '--hash-offset', part['fs_bytes'],
                   loop_dev, loop_dev]).decode('utf8')
       print(verityout.strip())
-      m = re.search(r'Root hash:\s+([a-f0-9]{64})$', verityout, re.IGNORECASE|re.MULTILINE)
+      m = re.search("Root hash:\s+([a-f0-9]{64})$", verityout, re.IGNORECASE|re.MULTILINE)
       if not m:
           raise Exception("Failed to parse verity output!")
 
@@ -830,7 +823,6 @@ def Extract(options):
   if not part['image_compat']:
     raise InvalidLayout("Disk layout is incompatible with existing image")
 
-  extract_size = part.get('extract_bytes', part['image_bytes'])
   subprocess.check_call(['dd',
                          'bs=10MB',
                          'iflag=count_bytes,skip_bytes',
@@ -839,7 +831,7 @@ def Extract(options):
                          'if=%s' % options.disk_image,
                          'of=%s' % options.output,
                          'skip=%s' % part['image_first_byte'],
-                         'count=%s' % extract_size])
+                         'count=%s' % part['image_bytes']])
 
 
 def GetPartitionByNumber(partitions, num):

build_library/ebuild_aci_manifest.in

@@ -0,0 +1,14 @@
+{
+  "acKind": "ImageManifest",
+  "acVersion": "0.8.6",
+  "name": "@ACI_NAME@",
+  "labels": [
+    {"name": "arch", "value": "@ACI_ARCH@"},
+    {"name": "os", "value": "linux"},
+    {"name": "version", "value": "@ACI_VERSION@"}
+  ],
+  "app": {
+    "user": "0",
+    "group": "0"
+  }
+}

build_library/ebuild_aci_util.sh

@@ -0,0 +1,97 @@
+# Copyright (c) 2016 The CoreOS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# Expects BOARD, BUILD_DIR, BUILD_LIBRARY_DIR, and FLATCAR_VERSION in env.
+
+# Copied from create_prod_image()
+create_ebuild_aci_image() {
+    local image_name="$1"
+    local disk_layout="$2"
+    local update_group="$3"
+    local pkg="$4"
+
+    info "Building ACI staging image ${image_name}"
+    local root_fs_dir="${BUILD_DIR}/rootfs"
+    local image_contents="${image_name%.bin}_contents.txt"
+    local image_packages="${image_name%.bin}_packages.txt"
+    local image_licenses="${image_name%.bin}_licenses.json"
+
+    start_image \
+        "${image_name}" "${disk_layout}" "${root_fs_dir}" "${update_group}"
+
+    # Install minimal GCC (libs only) and then everything else
+    extract_prod_gcc "${root_fs_dir}"
+
+    emerge_to_image_unchecked "${root_fs_dir}" "${pkg}"
+    run_ldconfig "${root_fs_dir}"
+    write_packages "${root_fs_dir}" "${BUILD_DIR}/${image_packages}"
+    write_licenses "${root_fs_dir}" "${BUILD_DIR}/${image_licenses}"
+    insert_licenses "${BUILD_DIR}/${image_licenses}" "${root_fs_dir}"
+
+    cleanup_mounts "${root_fs_dir}"
+    trap - EXIT
+}
+
+ebuild_aci_write_manifest() {
+    local manifest="${1?No output path was specified}"
+    local name="${2?No ACI name was specified}"
+    local version="${3?No ACI version was specified}"
+    local appc_arch=
+
+    case "${BOARD}" in
+        amd64-usr) appc_arch=amd64 ;;
+        arm64-usr) appc_arch=aarch64 ;;
+        *) die_notrace "Cannot map \"${BOARD}\" to an appc arch" ;;
+    esac
+
+    sudo cp "${BUILD_LIBRARY_DIR}/ebuild_aci_manifest.in" "${manifest}"
+    sudo sed "${manifest}" -i \
+        -e "s,@ACI_NAME@,${name}," \
+        -e "s,@ACI_VERSION@,${version}," \
+        -e "s,@ACI_ARCH@,${appc_arch},"
+}
+
+ebuild_aci_create() {
+    local aciroot="${BUILD_DIR}"
+    local aci_name="${1?No aci name was specified}"; shift
+    local output_image="${1?No output file specified}"; shift
+    local pkg="${1?No package given}"; shift
+    local version="${1?No package version given}"; shift
+    local extra_version="${1?No extra version number given}"; shift
+    local pkg_files=( "${@}" )
+
+    local staging_image="flatcar_pkg_staging_aci_stage.bin"
+
+    local ebuild_atom="=${pkg}-${version}"
+
+    local ebuild=$(equery-"${BOARD}" w "${ebuild_atom}" 2>/dev/null)
+    [ -n "${ebuild}" ] || die_notrace "No ebuild exists for ebuild \"${pkg}\""
+
+    # Build a staging image for this ebuild.
+    create_ebuild_aci_image "${staging_image}" container stable "${ebuild_atom}"
+
+    # Remount the staging image to brutalize the rootfs for broken services.
+    "${BUILD_LIBRARY_DIR}/disk_util" --disk_layout=container \
+        mount "${BUILD_DIR}/${staging_image}" "${aciroot}/rootfs"
+    trap "cleanup_mounts '${aciroot}/rootfs' && delete_prompt" EXIT
+
+    # Substitute variables into the manifest to produce the final version.
+    ebuild_aci_write_manifest \
+        "${aciroot}/manifest" \
+        "${aci_name}" \
+        "${version}_flatcar.${extra_version}"
+
+    local pkg_files_in_rootfs=( "${pkg_files[@]/#/rootfs}" )
+
+    # Write a tar ACI file containing the manifest and desired parts of the mounted rootfs
+    sudo tar -C "${aciroot}" -hczf "${BUILD_DIR}/${output_image}.aci" \
+        manifest ${pkg_files_in_rootfs[@]}
+
+    # Unmount the staging image, and delete it to save space.
+    cleanup_mounts "${aciroot}/rootfs"
+    trap - EXIT
+    rm -f "${BUILD_DIR}/${staging_image}"
+
+    echo "Created aci for ${pkg}-${version}: ${BUILD_DIR}/${output_image}.aci"
+}

build_library/extra_sysexts.sh

@@ -1,29 +0,0 @@
-EXTRA_SYSEXTS=(
-  "overlaybd|sys-fs/overlaybd,app-containers/accelerated-container-image"
-  "incus|app-containers/incus"
-  "nvidia-drivers-535|x11-drivers/nvidia-drivers:0/535|-kernel-open persistenced|amd64"
-  "nvidia-drivers-535-open|x11-drivers/nvidia-drivers:0/535|kernel-open persistenced|amd64"
-  "nvidia-drivers-550|x11-drivers/old-nvidia-drivers:0/550|-kernel-open persistenced|amd64"
-  "nvidia-drivers-550-open|x11-drivers/old-nvidia-drivers:0/550|kernel-open persistenced|amd64"
-  "nvidia-drivers-570|x11-drivers/nvidia-drivers:0/570|-kernel-open persistenced|amd64"
-  "nvidia-drivers-570-open|x11-drivers/nvidia-drivers:0/570|kernel-open persistenced|amd64"
-  "podman|app-containers/podman,net-misc/passt"
-  "python|dev-lang/python,dev-python/pip"
-  "zfs|sys-fs/zfs"
-)
-
-_get_unversioned_sysext_packages_unsorted() {
-  for sysext in "${EXTRA_SYSEXTS[@]}"; do
-    IFS="|" read -r _ PACKAGE_ATOMS _ <<< "$sysext"
-
-    IFS=,
-    for atom in $PACKAGE_ATOMS; do
-       qatom "$atom" -F "%{CATEGORY}/%{PN}"
-    done
-    unset IFS
-  done
-}
-
-get_unversioned_sysext_packages() {
-  _get_unversioned_sysext_packages_unsorted | sort | uniq
-}

build_library/extract-initramfs-from-vmlinuz.sh

@@ -7,39 +7,51 @@
 # This will create one or more out-dir/rootfs-N directories that contain the contents of the initramfs.
 
 set -euo pipefail
-
+# check for unzstd. Will abort the script with an error message if the tool is not present.
-# check for xzcat. Will abort the script with an error message if the tool is not present.
+unzstd -V >/dev/null
-xzcat -V >/dev/null
-
 fail() {
     echo "${*}" >&2
     exit 1
 }
 
-find_xz_headers() {
+# Stolen from extract-vmlinux and modified.
-    grep --fixed-strings --text --byte-offset --only-matching $'\xFD\x37\x7A\x58\x5A\x00' "$1" | cut -d: -f1
+try_decompress() {
+    local header="${1}"
+    local no_idea="${2}"
+    local tool="${3}"
+    local image="${4}"
+    local tmp="${5}"
+    local output_basename="${6}"
+
+    local pos
+    local tool_filename=$(echo "${tool}" | cut -f1 -d' ')
+    # The obscure use of the "tr" filter is to work around older versions of
+    # "grep" that report the byte offset of the line instead of the pattern.
+
+    # Try to find the header and decompress from here.
+    for pos in $(tr "${header}\n${no_idea}" "\n${no_idea}=" < "${image}" |
+                     grep --text --byte-offset --only-matching "^${no_idea}")
+    do
+        pos=${pos%%:*}
+        # Disable error handling, because we will be potentially
+        # giving the tool garbage or a valid archive with some garbage
+        # appended to it. So let the tool extract the valid archive
+        # and then complain about the garbage at the end, but don't
+        # fail the script because of it.
+        set +e; tail "-c+${pos}" "${image}" | "${tool}" >"${tmp}/out" 2>/dev/null; set -e;
+        if [ -s "${tmp}/out" ]; then
+            mv "${tmp}/out" "${output_basename}-${tool_filename}-at-${pos}"
+        else
+            rm -f "${tmp}/out"
+        fi
+    done
 }
 
-decompress_at() {
+try_unzstd_decompress() {
-    # Data may not really be a valid xz, so allow for errors.
+    local image="${1}"
-    tail "-c+$((${2%:*} + 1))" "$1" | xzcat 2>/dev/null || true
+    local tmp="${2}"
-}
+    local output_basename="${3}"
-
+    try_decompress '(\265/\375' xxx unzstd "${image}" "${tmp}" "${output_basename}"
-try_extract() {
-    # cpio can do strange things when given garbage, so do a basic check.
-    [[ $(head -c6 "$1") == 070701 ]] || return 0
-
-    while {
-        # cpio needs the directory to exist first. Fail if it's already there.
-        { mkdir "${out}/rootfs-${ROOTFS_IDX}" || return $?; } &&
-        # There may be multiple concatenated archives so try cpio till it fails.
-        cpio --quiet --extract --make-directories --directory="${out}/rootfs-${ROOTFS_IDX}" --nonmatching 'dev/*' 2>/dev/null
-    }; do
-        ROOTFS_IDX=$(( ROOTFS_IDX + 1 ))
-    done < "$1"
-
-    # Last cpio attempt may or may not leave an empty directory.
-    rmdir "${out}/rootfs-${ROOTFS_IDX}" 2>/dev/null || ROOTFS_IDX=$(( ROOTFS_IDX + 1 ))
 }
 
 me="${0##*/}"
@@ -53,22 +65,39 @@ if [[ ! -s "${image}" ]]; then
 fi
 mkdir -p "${out}"
 
-tmp=$(mktemp --directory -t eifv-XXXXXX)
+tmp=$(mktemp --directory /tmp/eifv-XXXXXX)
-trap 'rm -rf -- "${tmp}"' EXIT
+trap "rm -rf ${tmp}" EXIT
+
+tmp_dec="${tmp}/decompress"
+mkdir "${tmp_dec}"
+fr_prefix="${tmp}/first-round"
+
 ROOTFS_IDX=0
-
+perform_round() {
-# arm64 kernels are not compressed, so try decompressing once.
+    local image="${1}"
-# Other kernels are compressed, so also try decompressing twice.
+    local tmp_dec="${2}"
-for OFF1 in $(find_xz_headers "${image}")
+    local round_prefix="${3}"
-do
+    try_unzstd_decompress "${image}" "${tmp_dec}" "${round_prefix}"
-    decompress_at "${image}" "${OFF1}" > "${tmp}/initrd.maybe_cpio_or_elf"
+    for rnd in "${round_prefix}"*; do
-    try_extract "${tmp}/initrd.maybe_cpio_or_elf"
+        if [[ $(file --brief "${rnd}") =~ 'cpio archive' ]]; then
-
+            mkdir -p "${out}/rootfs-${ROOTFS_IDX}"
-    for OFF2 in $(find_xz_headers "${tmp}/initrd.maybe_cpio_or_elf")
+            while cpio --quiet --extract --make-directories --directory="${out}/rootfs-${ROOTFS_IDX}" --nonmatching 'dev/*'; do
-    do
+                ROOTFS_IDX=$(( ROOTFS_IDX + 1 ))
-        decompress_at "${tmp}/initrd.maybe_cpio_or_elf" "${OFF2}" > "${tmp}/initrd.maybe_cpio"
+                mkdir -p "${out}/rootfs-${ROOTFS_IDX}"
-        try_extract "${tmp}/initrd.maybe_cpio"
+            done <${rnd}
+            rmdir "${out}/rootfs-${ROOTFS_IDX}"
+        fi
     done
+}
+
+shopt -s nullglob
+perform_round "${image}" "${tmp_dec}" "${fr_prefix}"
+for fr in "${fr_prefix}"*; do
+    fr_files="${fr}-files"
+    fr_dec="${fr_files}/decompress"
+    mkdir -p "${fr_dec}"
+    sr_prefix="${fr_files}/second-round"
+    perform_round "${fr}" "${fr_dec}" "${sr_prefix}"
 done
 
 if [[ ${ROOTFS_IDX} -eq 0 ]]; then

build_library/flatcar-sb-dev-shim-2025.cert

@@ -1,22 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIEPDCCAySgAwIBAgICCSkwDQYJKoZIhvcNAQELBQAwPTE7MDkGA1UEAxMyRmxhdGNhciBDb250
-YWluZXIgTGludXggU2VjdXJlIEJvb3QgRGV2ZWxvcG1lbnQgQ0EwHhcNMjUwMzIwMTE1NzI5WhcN
-MjgwMzIwMTE1NzI5WjBRMSAwHgYDVQQKExdGbGF0Y2FyIENvbnRhaW5lciBMaW51eDEtMCsGA1UE
-AxMkRmxhdGNhciBDb250YWluZXIgTGludXggU2hpbSBTaWduaW5nMIICIjANBgkqhkiG9w0BAQEF
-AAOCAg8AMIICCgKCAgEA1/GCCSfkqRgSgSqphcfkBgRVxhdhYwlTm4DMeIet/15kPEQ8h8zGm5Js
-DhYYBKJfeGCM36/pBFT61KcpOTcxuEg2VKm2zOLsGfxymZjWln1Y3nUPiWx6AY/CRM6g2vYgXYIj
-x40aJN73usdRmdk6mVssKMMokkYFuH7eOxgWCkGtBbu/UZ/MU0VfdAc12EIuk/K4LMjSFpOitH2x
-mAvFobB8YAYzwhVybNl8etXUS+I3HjCUAwl0ly/fv4Pjb8LODI22jkPV/2X1OxG59wHOxsiNSBvd
-8szcYAH49iHg2bMVljsjtnEA7b51r4I6HJWlvTOc9Z3+jVz9mPXVlh6GEOzSVMBV7KsxkWeQdoUf
-8cQm+tqdfG2xVJUAWCil7xZAk1/l5C2fWgkRHX7fmF71ZDWW240iJvKRuA1/MlU5HlZfQk0EjgYv
-VZpwklpygn5bHbzquFlqwDhmtypULfTZ/NHnf1ygRuzwi7n/RTlZMziveNIj/yJBXoXdHlta8yDo
-VfV8G/m19z+YPW3gET2H1UwU656axcw7wUspndmuZySqqHl0yTDi/B1s8lT8+VxK4dol+GVIvys3
-zD6/K5J11YbsGydogBWSjir60ObWzloPLd8cQ0OXwHddZy5fFrfHgoTfrCacAOvcYynmwoHLHwwQ
-RVtC/X7MH4R2fIcvtAUCAwEAAaMyMDAwCQYDVR0TBAIwADATBgNVHSUEDDAKBggrBgEFBQcDAzAO
-BgNVHQ8BAf8EBAMCAb4wDQYJKoZIhvcNAQELBQADggEBAGdP0xWGtfrCwPTL/m/2dJDx0VWnMf7C
-sAHNmlTji7d7bO7tI7h5RVj664z2GUgjpYlnCMAiDqutG3Uksrxq59lXaV2q4em4clZtnIWPwJ5V
-UcySW5VePkTekJHzS27KjNG/l6audfutM6GkKIMjMxJE1M/a5v+FsHF9taFEJrjJDPRD7gi/c75H
-sqW8C0hwcm/6/+yaoQte6ufTZu1TFacbXPEp0cZ4JHjxILYxXNIn6x2PUFMFo1XLhjOAIC67AaUk
-/qNhqmhxD3yYhagamvPKN9mV0qlqv1tw61XYvJwL5eDfSgtQXCiZlXjQWu+lysF3p2pH7lyGdzGr
-19/6sbQ=
------END CERTIFICATE-----

build_library/generate_au_zip.py

@@ -22,6 +22,8 @@ SCRIPTS_DIR = os.environ['SCRIPTS_DIR']
 # GLOBALS
 STATIC_FILES = ['%s/version.txt' % REPO_MANIFESTS_DIR,
                 '%s/common.sh' % SCRIPTS_DIR,
+                '%s/core_pre_alpha' % SCRIPTS_DIR,
+                '%s/core_roller_upload' % SCRIPTS_DIR,
                 '%s/core_sign_update' % SCRIPTS_DIR,
                 ]
 
@@ -88,8 +90,8 @@ def _SplitAndStrip(data):
     if 'not found' in line:
       raise _LibNotFound(line)
     line = re.sub('.*not a dynamic executable.*', '', line)
-    line = re.sub(r'.* =>\s+', '', line)
+    line = re.sub('.* =>\s+', '', line)
-    line = re.sub(r'\(0x.*\)\s?', '', line)
+    line = re.sub('\(0x.*\)\s?', '', line)
     line = line.strip()
     if not len(line):
       continue

build_library/generate_grub_hashes.py

@@ -40,13 +40,13 @@ with open(os.path.join(outputdir, "grub_modules.config"), "w") as f:
     f.write(json.dumps({"9": {"binaryvalues": [{"prefix": "grub_module", "values": hashvalues}]}}))
 
 with open(os.path.join(outputdir, "kernel_cmdline.config"), "w") as f:
-    f.write(json.dumps({"8": {"asciivalues": [{"prefix": "grub_kernel_cmdline", "values": [{"value": r"rootflags=rw mount.usrflags=ro BOOT_IMAGE=/flatcar/vmlinuz-[ab] mount.usr=PARTUUID=\S{36} rootflags=rw mount.usrflags=ro consoleblank=0 root=LABEL=ROOT (console=\S+)? (flatcar.autologin=\S+)? verity.usrhash=\\S{64}", "description": "Flatcar kernel command line %s" % version}]}]}}))
+    f.write(json.dumps({"8": {"asciivalues": [{"prefix": "grub_kernel_cmdline", "values": [{"value": "rootflags=rw mount.usrflags=ro BOOT_IMAGE=/flatcar/vmlinuz-[ab] mount.usr=PARTUUID=\S{36} rootflags=rw mount.usrflags=ro consoleblank=0 root=LABEL=ROOT (console=\S+)? (flatcar.autologin=\S+)? verity.usrhash=\\S{64}", "description": "Flatcar kernel command line %s" % version}]}]}}))
 
-commands = [{"value": r'\[.*\]', "description": "Flatcar Grub configuration %s" % version},
+commands = [{"value": '\[.*\]', "description": "Flatcar Grub configuration %s" % version},
             {"value": 'gptprio.next -d usr -u usr_uuid', "description": "Flatcar Grub configuration %s" % version},
             {"value": 'insmod all_video', "description": "Flatcar Grub configuration %s" % version},
-            {"value": r'linux /flatcar/vmlinuz-[ab] rootflags=rw mount.usrflags=ro consoleblank=0 root=LABEL=ROOT (console=\S+)? (flatcar.autologin=\S+)?', "description": "Flatcar Grub configuration %s" % version},
+            {"value": 'linux /flatcar/vmlinuz-[ab] rootflags=rw mount.usrflags=ro consoleblank=0 root=LABEL=ROOT (console=\S+)? (flatcar.autologin=\S+)?', "description": "Flatcar Grub configuration %s" % version},
-            {"value": r'menuentry Flatcar \S+ --id=flatcar\S* {', "description": "Flatcar Grub configuration %s" % version},
+            {"value": 'menuentry Flatcar \S+ --id=flatcar\S* {', "description": "Flatcar Grub configuration %s" % version},
             {"value": 'search --no-floppy --set randomize_disk_guid --disk-uuid 00000000-0000-0000-0000-000000000001', "description": "Flatcar Grub configuration %s" % version},
             {"value": 'search --no-floppy --set oem --part-label OEM --hint hd0,gpt1', "description": "Flatcar Grub configuration %s" % version},
             {"value": 'set .+', "description": "Flatcar Grub configuration %s" % version},

build_library/grub.cfg

@@ -9,9 +9,6 @@ insmod all_video
 
 # Default menuentry id and boot timeout
 set default="flatcar"
-# Retry default boot entry - this will decrement the gpt tries counter and
-# switch to previous entry when all attempts are exhausted.
-set fallback="0 0 0"
 set timeout=1
 
 # Default kernel args for root filesystem, console, and Flatcar.
@@ -26,6 +23,18 @@ set linux_append=""
 
 set secure_boot="0"
 
+if [ "$grub_platform" = "efi" ]; then
+   getenv -e SecureBoot -g 8be4df61-93ca-11d2-aa0d-00e098032b8c -b sb
+   getenv -e SetupMode -g 8be4df61-93ca-11d2-aa0d-00e098032b8c -b setupmode
+   if [ "$sb" = "01" -a "$setupmode" = "00" ]; then
+      set secure_boot="1"
+      getenv -e NetBootVerificationKey -g b8ade7d5-d400-4213-8d15-d47be0a621bf -b gpgpubkey
+      if [ "$gpgpubkey" != "" ]; then
+         trust_var gpgpubkey
+      fi
+   fi
+fi
+
 if [ "$net_default_server" != "" ]; then
    smbios --type 1 --get-uuid 8 --set uuid
    smbios --type 1 --get-string 7 --set serial
@@ -79,7 +88,7 @@ if [ -z "$linux_console" ]; then
         terminal_output console serial_com0
     elif [ "$grub_platform" = efi ]; then
         if [ "$grub_cpu" = arm64 ]; then
-            set linux_console="console=ttyAMA0,115200n8 console=tty0"
+            set linux_console="console=ttyAMA0,115200n8"
         else
             set linux_console="console=ttyS0,115200n8 console=tty0"
        fi

build_library/grub_install.sh

@@ -35,54 +35,52 @@ switch_to_strict_mode
 # must be sourced after flags are parsed.
 . "${BUILD_LIBRARY_DIR}/toolchain_util.sh" || exit 1
 . "${BUILD_LIBRARY_DIR}/board_options.sh" || exit 1
-. "${BUILD_LIBRARY_DIR}/sbsign_util.sh" || exit 1
-
-SBSIGN_DB_KEY="${SBSIGN_DB_KEY:-/usr/share/sb_keys/DB.key}"
-SBSIGN_DB_CERT="${SBSIGN_DB_CERT:-/usr/share/sb_keys/DB.crt}"
 
 # Our GRUB lives under flatcar/grub so new pygrub versions cannot find grub.cfg
 GRUB_DIR="flatcar/grub/${FLAGS_target}"
 
+# GRUB install location inside the SDK
+GRUB_SRC="/usr/lib/grub/${FLAGS_target}"
+
 # Modules required to boot a standard CoreOS configuration
-CORE_MODULES=( normal search test fat part_gpt search_fs_uuid xzio search_part_label terminal gptprio configfile memdisk tar echo read btrfs )
+CORE_MODULES=( normal search test fat part_gpt search_fs_uuid gzio search_part_label terminal gptprio configfile memdisk tar echo read )
 
-SBAT_ARG=()
+# Name of the core image, depends on target
+CORE_NAME=
+
+# Whether the SDK's grub or the board root's grub is used. Once amd64 is
+# fixed up the board root's grub will always be used.
+BOARD_GRUB=0
 
 case "${FLAGS_target}" in
-    x86_64-efi)
-        EFI_ARCH="x64"
-        ;;
-    arm64-efi)
-        EFI_ARCH="aa64"
-        ;;
-esac
-
-case "${FLAGS_target}" in
-    x86_64-efi|arm64-efi)
-        GRUB_IMAGE="EFI/boot/grub${EFI_ARCH}.efi"
-        CORE_MODULES+=( serial linux efi_gop efinet pgp http tftp tpm )
-        SBAT_ARG=( --sbat "${BOARD_ROOT}/usr/share/grub/sbat.csv" )
-        ;;
     i386-pc)
-        GRUB_IMAGE="${GRUB_DIR}/core.img"
         CORE_MODULES+=( biosdisk serial )
+        CORE_NAME="core.img"
+        ;;
+    x86_64-efi)
+	CORE_MODULES+=( serial efi_gop efinet pgp http tftp )
+        CORE_NAME="core.efi"
         ;;
     x86_64-xen)
-        GRUB_IMAGE="xen/pvboot-x86_64.elf"
+        CORE_NAME="core.elf"
+        ;;
+    arm64-efi)
+        CORE_MODULES+=( serial linux efi_gop efinet pgp http tftp )
+        CORE_NAME="core.efi"
+        BOARD_GRUB=1
         ;;
     *)
         die_notrace "Unknown GRUB target ${FLAGS_target}"
         ;;
 esac
 
-info "Updating GRUB in ${BOARD_ROOT}"
+if [[ $BOARD_GRUB -eq 1 ]]; then
-emerge-${BOARD} \
+    info "Updating GRUB in ${BOARD_ROOT}"
-        --nodeps --select --verbose --update --getbinpkg --usepkgonly --newuse \
+    emerge-${BOARD} \
-        sys-boot/grub \
+           --nodeps --select --verbose --update --getbinpkg --usepkgonly --newuse \
-        sys-boot/shim \
+           sys-boot/grub
-        sys-boot/shim-signed
+    GRUB_SRC="${BOARD_ROOT}/usr/lib/grub/${FLAGS_target}"
-
+fi
-GRUB_SRC="${BOARD_ROOT}/usr/lib/grub/${FLAGS_target}"
 [[ -d "${GRUB_SRC}" ]] || die "GRUB not installed at ${GRUB_SRC}"
 
 # In order for grub-setup-bios to properly detect the layout of the disk
@@ -95,7 +93,6 @@ ESP_DIR=
 LOOP_DEV=
 
 cleanup() {
-    cleanup_sbsign_certs
     if [[ -d "${ESP_DIR}" ]]; then
         if mountpoint -q "${ESP_DIR}"; then
             sudo umount "${ESP_DIR}"
@@ -129,32 +126,21 @@ done
 if [[ -z ${MOUNTED} ]]; then
     failboat "${LOOP_DEV}p1 where art thou? udev has forsaken us!"
 fi
-sudo mkdir -p "${ESP_DIR}/${GRUB_DIR}" "${ESP_DIR}/${GRUB_IMAGE%/*}"
+sudo mkdir -p "${ESP_DIR}/${GRUB_DIR}"
 
-# Additional GRUB modules cannot be loaded with Secure Boot enabled, so only
+info "Compressing modules in ${GRUB_DIR}"
-# copy and compress these for target that don't support it.
+for file in "${GRUB_SRC}"/*{.lst,.mod}; do
-case "${FLAGS_target}" in
+    out="${ESP_DIR}/${GRUB_DIR}/${file##*/}"
-    x86_64-efi|arm64-efi) : ;;
+    gzip --best --stdout "${file}" | sudo_clobber "${out}"
-    *)
+done
-        info "Compressing modules in ${GRUB_DIR}"
-        for file in "${GRUB_SRC}"/*{.lst,.mod}; do
-            for core_mod in "${CORE_MODULES[@]}"; do
-                [[ ${file} == ${GRUB_SRC}/${core_mod}.mod ]] && continue 2
-            done
-            out="${ESP_DIR}/${GRUB_DIR}/${file##*/}"
-            xz --stdout "${file}" | sudo_clobber "${out}"
-        done
-        ;;
-esac
 
 info "Generating ${GRUB_DIR}/load.cfg"
 # Include a small initial config in the core image to search for the ESP
 # by filesystem ID in case the platform doesn't provide the boot disk.
-# $root points to memdisk here so instead use hd0,gpt1 as a hint so it is
+# The existing $root value is given as a hint so it is searched first.
-# searched first.
 ESP_FSID=$(sudo grub-probe -t fs_uuid -d "${LOOP_DEV}p1")
 sudo_clobber "${ESP_DIR}/${GRUB_DIR}/load.cfg" <<EOF
-search.fs_uuid ${ESP_FSID} root hd0,gpt1
+search.fs_uuid ${ESP_FSID} root \$root
 set prefix=(memdisk)
 set
 EOF
@@ -178,55 +164,21 @@ if [[ ! -f "${ESP_DIR}/flatcar/grub/grub.cfg.tar" ]]; then
     fi
 
     sudo tar cf "${ESP_DIR}/flatcar/grub/grub.cfg.tar" \
-      -C "${GRUB_TEMP_DIR}" "grub.cfg"
+	 -C "${GRUB_TEMP_DIR}" "grub.cfg"
 fi
 
-info "Generating ${GRUB_IMAGE}"
+info "Generating ${GRUB_DIR}/${CORE_NAME}"
 sudo grub-mkimage \
-    --compression=xz \
+    --compression=auto \
     --format "${FLAGS_target}" \
     --directory "${GRUB_SRC}" \
     --config "${ESP_DIR}/${GRUB_DIR}/load.cfg" \
     --memdisk "${ESP_DIR}/flatcar/grub/grub.cfg.tar" \
-    "${SBAT_ARG[@]}" \
+    --output "${ESP_DIR}/${GRUB_DIR}/${CORE_NAME}" \
-    --output "${ESP_DIR}/${GRUB_IMAGE}" \
     "${CORE_MODULES[@]}"
 
 # Now target specific steps to make the system bootable
 case "${FLAGS_target}" in
-    x86_64-efi|arm64-efi)
-        info "Installing default ${FLAGS_target} UEFI bootloader."
-
-        if [[ ${COREOS_OFFICIAL:-0} -ne 1 ]]; then
-            # Sign GRUB and mokmanager(mm) with the shim-embedded key.
-            do_sbsign --output "${ESP_DIR}/${GRUB_IMAGE}"{,}
-            do_sbsign --output "${ESP_DIR}/EFI/boot/mm${EFI_ARCH}.efi" \
-                "${BOARD_ROOT}/usr/lib/shim/mm${EFI_ARCH}.efi"
-
-            # Unofficial build: Sign shim with our development key.
-            sudo sbsign \
-                --key "${SBSIGN_DB_KEY}" \
-                --cert "${SBSIGN_DB_CERT}" \
-                --output "${ESP_DIR}/EFI/boot/boot${EFI_ARCH}.efi" \
-                "${BOARD_ROOT}/usr/lib/shim/shim${EFI_ARCH}.efi"
-        else
-            # Official build: Copy signed shim and mm for signing later.
-            sudo cp "${BOARD_ROOT}/usr/lib/shim/mm${EFI_ARCH}.efi" \
-                "${ESP_DIR}/EFI/boot/mm${EFI_ARCH}.efi"
-            sudo cp "${BOARD_ROOT}/usr/lib/shim/shim${EFI_ARCH}.efi.signed" \
-                "${ESP_DIR}/EFI/boot/boot${EFI_ARCH}.efi"
-        fi
-
-        # copying from vfat so ignore permissions
-        if [[ -n ${FLAGS_copy_efi_grub} ]]; then
-            cp --no-preserve=mode "${ESP_DIR}/${GRUB_IMAGE}" \
-                "${FLAGS_copy_efi_grub}"
-        fi
-        if [[ -n ${FLAGS_copy_shim} ]]; then
-            cp --no-preserve=mode "${ESP_DIR}/EFI/boot/boot${EFI_ARCH}.efi" \
-                "${FLAGS_copy_shim}"
-        fi
-        ;;
     i386-pc)
         info "Installing MBR and the BIOS Boot partition."
         sudo cp "${GRUB_SRC}/boot.img" "${ESP_DIR}/${GRUB_DIR}"
@@ -237,12 +189,56 @@ case "${FLAGS_target}" in
         sudo dd bs=448 count=1 status=none if="${LOOP_DEV}" \
             of="${ESP_DIR}/${GRUB_DIR}/mbr.bin"
         ;;
+    x86_64-efi)
+        info "Installing default x86_64 UEFI bootloader."
+        sudo mkdir -p "${ESP_DIR}/EFI/boot"
+	# Use the test keys for signing unofficial builds
+	if [[ ${COREOS_OFFICIAL:-0} -ne 1 ]]; then
+            sudo sbsign --key /usr/share/sb_keys/DB.key \
+		--cert /usr/share/sb_keys/DB.crt \
+                    "${ESP_DIR}/${GRUB_DIR}/${CORE_NAME}"
+            sudo cp "${ESP_DIR}/${GRUB_DIR}/${CORE_NAME}.signed" \
+                "${ESP_DIR}/EFI/boot/grub.efi"
+            sudo sbsign --key /usr/share/sb_keys/DB.key \
+                 --cert /usr/share/sb_keys/DB.crt \
+                 --output "${ESP_DIR}/EFI/boot/bootx64.efi" \
+                 "/usr/lib/shim/shim.efi"
+        else
+            sudo cp "${ESP_DIR}/${GRUB_DIR}/${CORE_NAME}" \
+                "${ESP_DIR}/EFI/boot/grub.efi"
+            sudo cp "/usr/lib/shim/shim.efi" \
+                "${ESP_DIR}/EFI/boot/bootx64.efi"
+	fi
+        # copying from vfat so ignore permissions
+        if [[ -n "${FLAGS_copy_efi_grub}" ]]; then
+            cp --no-preserve=mode "${ESP_DIR}/EFI/boot/grub.efi" \
+                "${FLAGS_copy_efi_grub}"
+        fi
+        if [[ -n "${FLAGS_copy_shim}" ]]; then
+            cp --no-preserve=mode "${ESP_DIR}/EFI/boot/bootx64.efi" \
+                "${FLAGS_copy_shim}"
+        fi
+        ;;
     x86_64-xen)
         info "Installing default x86_64 Xen bootloader."
-        sudo mkdir -p "${ESP_DIR}/boot/grub"
+        sudo mkdir -p "${ESP_DIR}/xen" "${ESP_DIR}/boot/grub"
+        sudo cp "${ESP_DIR}/${GRUB_DIR}/${CORE_NAME}" \
+            "${ESP_DIR}/xen/pvboot-x86_64.elf"
         sudo cp "${BUILD_LIBRARY_DIR}/menu.lst" \
             "${ESP_DIR}/boot/grub/menu.lst"
         ;;
+    arm64-efi)
+        info "Installing default arm64 UEFI bootloader."
+        sudo mkdir -p "${ESP_DIR}/EFI/boot"
+        #FIXME(andrejro): shim not ported to aarch64
+        sudo cp "${ESP_DIR}/${GRUB_DIR}/${CORE_NAME}" \
+            "${ESP_DIR}/EFI/boot/bootaa64.efi"
+        if [[ -n "${FLAGS_copy_efi_grub}" ]]; then
+            # copying from vfat so ignore permissions
+            cp --no-preserve=mode "${ESP_DIR}/EFI/boot/bootaa64.efi" \
+                "${FLAGS_copy_efi_grub}"
+        fi
+        ;;
 esac
 
 cleanup

build_library/modify_image_util.sh

@@ -0,0 +1,116 @@
+#!/bin/bash
+
+# Copyright (c) 2014 The CoreOS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# Shell library for modifying an image built with build_image.
+
+start_modify_image() {
+    # Default to the most recent image
+    if [[ -z "${FLAGS_from}" ]] ; then
+        FLAGS_from="$(${SCRIPT_ROOT}/get_latest_image.sh --board=${FLAGS_board})"
+    else
+        FLAGS_from="$(readlink -f "${FLAGS_from}")"
+    fi
+
+    local src_image="${FLAGS_from}/${FLATCAR_PRODUCTION_IMAGE_NAME}"
+    if [[ ! -f "${src_image}" ]]; then
+        die_notrace "Source image does not exist: ${src_image}"
+    fi
+
+    # Source should include version.txt, switch to its version information
+    if [[ ! -f "${FLAGS_from}/version.txt" ]]; then
+        die_notrace "Source version info does not exist: ${FLAGS_from}/version.txt"
+    fi
+    source "${FLAGS_from}/version.txt"
+    FLATCAR_VERSION_STRING="${FLATCAR_VERSION}"
+
+    # Load after version.txt to set the correct output paths
+    . "${BUILD_LIBRARY_DIR}/toolchain_util.sh"
+    . "${BUILD_LIBRARY_DIR}/board_options.sh"
+    . "${BUILD_LIBRARY_DIR}/build_image_util.sh"
+
+    # Handle existing directory.
+    if [[ -e "${BUILD_DIR}" ]]; then
+        if [[ ${FLAGS_replace} -eq ${FLAGS_TRUE} ]]; then
+            sudo rm -rf "${BUILD_DIR}"
+        else
+            error "Directory ${BUILD_DIR} already exists."
+            error "Use --build_attempt option to specify an unused attempt."
+            error "Or use --replace if you want to overwrite this directory."
+            die "Unwilling to overwrite ${BUILD_DIR}."
+        fi
+    fi
+
+    # Create the output directory and temporary mount points.
+    DST_IMAGE="${BUILD_DIR}/${FLATCAR_PRODUCTION_IMAGE_NAME}"
+    ROOT_FS_DIR="${BUILD_DIR}/rootfs"
+    mkdir -p "${ROOT_FS_DIR}"
+
+    info "Copying from ${FLAGS_from}"
+    cp "${src_image}" "${DST_IMAGE}"
+
+    # Copy all extra useful things, these do not need to be modified.
+    local update_prefix="${FLATCAR_PRODUCTION_IMAGE_NAME%_image.bin}_update"
+    local production_prefix="${FLATCAR_PRODUCTION_IMAGE_NAME%.bin}"
+    local container_prefix="${FLATCAR_DEVELOPER_CONTAINER_NAME%.bin}"
+    local pcr_data="${FLATCAR_PRODUCTION_IMAGE_NAME%.bin}_pcr_policy.zip"
+    EXTRA_FILES=(
+        "version.txt"
+        "${update_prefix}.bin"
+        "${update_prefix}.zip"
+        "${pcr_data}"
+        "${production_prefix}_contents.txt"
+        "${production_prefix}_packages.txt"
+        "${production_prefix}_kernel_config.txt"
+        "${FLATCAR_DEVELOPER_CONTAINER_NAME}"
+        "${container_prefix}_contents.txt"
+        "${container_prefix}_packages.txt"
+        )
+    for filename in "${EXTRA_FILES[@]}"; do
+        if [[ -e "${FLAGS_from}/${filename}" ]]; then
+            cp "${FLAGS_from}/${filename}" "${BUILD_DIR}/${filename}"
+        fi
+    done
+
+    "${BUILD_LIBRARY_DIR}/disk_util" --disk_layout="${FLAGS_disk_layout}" \
+            mount "${DST_IMAGE}" "${ROOT_FS_DIR}"
+    trap "cleanup_mounts '${ROOT_FS_DIR}'" EXIT
+}
+
+finish_modify_image() {
+    cleanup_mounts "${ROOT_FS_DIR}"
+    trap - EXIT
+
+
+    declare -a files_to_evaluate
+    declare -a compressed_images
+    declare -a extra_files
+
+    files_to_evaluate+=( "${DST_IMAGE}" )
+    compress_disk_images files_to_evaluate compressed_images extra_files
+
+    upload_image -d "${DST_IMAGE}.DIGESTS" \
+        "${compressed_images[@]}" \
+        "${extra_files[@]}"
+
+    # Upload legacy digests
+    upload_legacy_digests "${DST_IMAGE}.DIGESTS" compressed_images
+
+    for filename in "${EXTRA_FILES[@]}"; do
+        if [[ -e "${BUILD_DIR}/${filename}" ]]; then
+            upload_image "${BUILD_DIR}/${filename}"
+        fi
+    done
+
+    set_build_symlinks "${FLAGS_group}-latest"
+
+    info "Done. Updated image is in ${BUILD_DIR}"
+    cat << EOF
+To convert it to a virtual machine image, use:
+  ./image_to_vm.sh --from=${OUTSIDE_OUTPUT_DIR} --board=${BOARD}
+
+The default type is qemu, see ./image_to_vm.sh --help for other options.
+EOF
+}

build_library/oem_aci_util.sh

@@ -0,0 +1,124 @@
+# Copyright (c) 2016 The CoreOS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# Expects BOARD, BUILD_DIR, BUILD_LIBRARY_DIR, and FLATCAR_VERSION in env.
+
+# There must be a manifest template included with the ebuild at
+# files/manifest.in, which will have some variable values substituted before
+# being written into place for the ACI.  Optionally, a shell script can also be
+# included at files/manglefs.sh to be run after all packages are installed.  It
+# is intended to be used to make modifications to the file system layout and
+# program paths that some included agent software might expect.
+
+# Copied from create_prod_image()
+create_oem_aci_image() {
+    local image_name="$1"
+    local disk_layout="$2"
+    local update_group="$3"
+    local base_pkg="${4?No base package was specified}"
+
+    info "Building OEM ACI staging image ${image_name}"
+    local root_fs_dir="${BUILD_DIR}/rootfs"
+    local image_contents="${image_name%.bin}_contents.txt"
+    local image_packages="${image_name%.bin}_packages.txt"
+    local image_licenses="${image_name%.bin}_licenses.json"
+
+    start_image \
+        "${image_name}" "${disk_layout}" "${root_fs_dir}" "${update_group}"
+
+    # Install minimal GCC (libs only) and then everything else
+    set_image_profile oem-aci
+    extract_prod_gcc "${root_fs_dir}"
+    emerge_to_image "${root_fs_dir}" "${base_pkg}"
+    run_ldconfig "${root_fs_dir}"
+    write_packages "${root_fs_dir}" "${BUILD_DIR}/${image_packages}"
+    write_licenses "${root_fs_dir}" "${BUILD_DIR}/${image_licenses}"
+    insert_licenses "${BUILD_DIR}/${image_licenses}" "${root_fs_dir}"
+
+    # clean-ups of things we do not need
+    sudo rm ${root_fs_dir}/etc/csh.env
+    sudo rm -rf ${root_fs_dir}/etc/env.d
+    sudo rm -rf ${root_fs_dir}/var/db/pkg
+
+    sudo mv ${root_fs_dir}/etc/profile.env \
+        ${root_fs_dir}/usr/share/baselayout/profile.env
+
+    # Move the ld.so configs into /usr so they can be symlinked from /
+    sudo mv ${root_fs_dir}/etc/ld.so.conf ${root_fs_dir}/usr/lib
+    sudo mv ${root_fs_dir}/etc/ld.so.conf.d ${root_fs_dir}/usr/lib
+
+    sudo ln --symbolic ../usr/lib/ld.so.conf ${root_fs_dir}/etc/ld.so.conf
+
+    # Add a tmpfiles rule that symlink ld.so.conf from /usr into /
+    sudo tee "${root_fs_dir}/usr/lib/tmpfiles.d/baselayout-ldso.conf" \
+        > /dev/null <<EOF
+L+  /etc/ld.so.conf     -   -   -   -   ../usr/lib/ld.so.conf
+EOF
+
+    # Move the PAM configuration into /usr
+    sudo mkdir -p ${root_fs_dir}/usr/lib/pam.d
+    sudo mv -n ${root_fs_dir}/etc/pam.d/* ${root_fs_dir}/usr/lib/pam.d/
+    sudo rmdir ${root_fs_dir}/etc/pam.d
+
+    # Take the non-kernel-related bits from finish_image().
+    rm -rf "${BUILD_DIR}"/configroot
+    cleanup_mounts "${root_fs_dir}"
+    trap - EXIT
+}
+
+oem_aci_write_manifest() {
+    local manifest_template="${1?No input path was specified}"
+    local manifest="${2?No output path was specified}"
+    local name="${3?No ACI name was specified}"
+    local appc_arch=
+
+    case "${BOARD}" in
+        amd64-usr) appc_arch=amd64 ;;
+        arm64-usr) appc_arch=aarch64 ;;
+        *) die_notrace "Cannot map \"${BOARD}\" to an appc arch" ;;
+    esac
+
+    sudo cp "${manifest_template}" "${manifest}"
+    sudo sed "${manifest}" -i \
+        -e "s,@ACI_NAME@,${name}," \
+        -e "s,@ACI_VERSION@,${FLATCAR_VERSION}," \
+        -e "s,@ACI_ARCH@,${appc_arch},"
+}
+
+oem_aci_create() {
+    local aciroot="${BUILD_DIR}"
+    local oem="${1?No OEM was specified}"
+    local base_pkg="coreos-base/coreos-oem-${oem}"
+    local ebuild=$(equery-"${BOARD}" w "${base_pkg}" 2>/dev/null)
+    local staging_image="coreos_oem_${oem}_aci_stage.bin"
+
+    [ -n "${ebuild}" ] || die_notrace "No ebuild exists for OEM \"${oem}\""
+    grep -Fqs '(meta package)' "${ebuild}" ||
+        die_notrace "The \"${base_pkg}\" ebuild is not a meta package"
+
+    # Build a staging image for this OEM.
+    create_oem_aci_image "${staging_image}" container stable "${base_pkg}"
+
+    # Remount the staging image to brutalize the rootfs for broken services.
+    "${BUILD_LIBRARY_DIR}/disk_util" --disk_layout=container \
+        mount "${BUILD_DIR}/${staging_image}" "${aciroot}/rootfs"
+    trap "cleanup_mounts '${aciroot}/rootfs' && delete_prompt" EXIT
+    [ -r "${ebuild%/*}/files/manglefs.sh" ] &&
+        sudo sh -c "cd '${aciroot}/rootfs' && . '${ebuild%/*}/files/manglefs.sh'"
+
+    # Substitute variables into the OEM manifest to produce the final version.
+    oem_aci_write_manifest \
+        "${ebuild%/*}/files/manifest.in" \
+        "${aciroot}/manifest" \
+        "coreos.com/oem-${oem}"
+
+    # Write a tar ACI file containing the manifest and mounted rootfs contents.
+    sudo tar -C "${aciroot}" -czf "${BUILD_DIR}/flatcar-oem-${oem}.aci" \
+        manifest rootfs
+
+    # Unmount the staging image, and delete it to save space.
+    cleanup_mounts "${aciroot}/rootfs"
+    trap - EXIT
+    rm -f "${BUILD_DIR}/${staging_image}"
+}

build_library/oem_sysexts.sh

@@ -1,83 +0,0 @@
-#!/bin/bash
-# OEM sysext helpers.
-
-# Auto-detect scripts repo root from this file's location.
-# oem_sysexts.sh is at: <scripts_repo>/build_library/oem_sysexts.sh
-_OEM_SYSEXTS_SCRIPTS_ROOT="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")/..")"
-
-get_oem_overlay_root() {
-  local overlay_root="/mnt/host/source/src/third_party/coreos-overlay"
-
-  if [[ ! -d "${overlay_root}" ]]; then
-    overlay_root="${_OEM_SYSEXTS_SCRIPTS_ROOT}/sdk_container/src/third_party/coreos-overlay"
-  fi
-
-  if [[ ! -d "${overlay_root}" ]]; then
-    echo "No coreos-overlay repo found (tried SDK and ${_OEM_SYSEXTS_SCRIPTS_ROOT})" >&2
-    exit 1
-  fi
-
-  printf '%s' "${overlay_root}"
-}
-
-_get_oem_ids() {
-  local arch list_var_name
-  arch=${1}; shift
-  list_var_name=${1}; shift
-
-  local overlay_root
-  overlay_root=$(get_oem_overlay_root)
-
-  local -a ebuilds=("${overlay_root}/coreos-base/common-oem-files/common-oem-files-"*'.ebuild')
-  if [[ ${#ebuilds[@]} -eq 0 ]] || [[ ! -e ${ebuilds[0]} ]]; then
-    echo "No coreos-base/common-oem-files ebuilds?!" >&2
-    exit 1
-  fi
-
-  # This defines local COMMON_OEMIDS, AMD64_ONLY_OEMIDS,
-  # ARM64_ONLY_OEMIDS and OEMIDS variable. We don't use the last
-  # one. Also defines global-by-default EAPI, which we make local
-  # here to avoid making it global.
-  local EAPI
-  source "${ebuilds[0]}" flatcar-local-variables
-
-  local -n arch_oemids_ref="${arch^^}_ONLY_OEMIDS"
-  local all_oemids=(
-    "${COMMON_OEMIDS[@]}"
-    "${arch_oemids_ref[@]}"
-  )
-
-  mapfile -t "${list_var_name}" < <(printf '%s\n' "${all_oemids[@]}" | sort)
-}
-
-# Gets a list of OEMs that are using sysexts.
-#
-# 1 - arch
-# 2 - name of an array variable to store the result in
-get_oem_id_list() {
-  _get_oem_ids "$@"
-}
-
-# Gets a list of OEM sysext descriptors.
-#
-# 1 - arch
-# 2 - name of an array variable to store the result in
-#
-# Format: "name|metapackage|useflags"
-get_oem_sysext_matrix() {
-  local arch list_var_name
-  arch=${1}; shift
-  list_var_name=${1}; shift
-
-  local -a oem_ids
-  _get_oem_ids "${arch}" oem_ids
-
-  local -a matrix=()
-  local oem_id
-  for oem_id in "${oem_ids[@]}"; do
-    matrix+=("oem-${oem_id}|coreos-base/oem-${oem_id}|${oem_id}")
-  done
-
-  local -n matrix_ref="${list_var_name}"
-  matrix_ref=("${matrix[@]}")
-}

build_library/prefix_util.sh

@@ -108,7 +108,7 @@ function create_make_conf() {
     final)
       filepath="${FINALROOT}${EPREFIX}/etc/portage/make.conf"
       dir="${FINALDIR}"
-      emerge_opts="--usepkgonly"
+      emerge_opts="--root-deps=rdeps --usepkgonly"
       ;;
   esac
 
@@ -128,6 +128,7 @@ EMERGE_DEFAULT_OPTS=${emerge_opts@Q}
 
 USE="
 -desktop
+-ensurepip
 -installkernel
 -llvm
 -nls

build_library/prod_image_util.sh

@@ -3,8 +3,6 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-source "${BUILD_LIBRARY_DIR}/oem_sysexts.sh" || exit 1
-
 # Lookup the current version of a binary package, downloading it if needed.
 # Usage: get_binary_pkg some-pkg/name
 # Prints: some-pkg/name-1.2.3
@@ -46,8 +44,7 @@ extract_prod_gcc() {
     #  /usr/lib/gcc/x86_64-cros-linux-gnu/$version/*
     # Instead we extract them to plain old /usr/lib
     qtbz2 -O -t "${pkg}" | \
-        lbzcat -d -c - | \
+        sudo tar -C "${root_fs_dir}" -xj \
-        sudo tar -C "${root_fs_dir}" -x \
         --transform 's#/usr/lib/.*/#/usr/lib64/#' \
         --wildcards './usr/lib/gcc/*.so*' \
         --wildcards './usr/share/SLSA'
@@ -65,13 +62,8 @@ create_prod_image() {
     exit 1
   fi
 
-  local base_sysexts="$5"
-
   info "Building production image ${image_name}"
-  # The "prod-image-rootfs" directory name is important - it is used
+  local root_fs_dir="${BUILD_DIR}/rootfs"
-  # to determine the package target in coreos/base/profile.bashrc
-  local root_fs_dir="${BUILD_DIR}/prod-image-rootfs"
-  local root_fs_sysexts_output_dir="${BUILD_DIR}/rootfs-included-sysexts"
   local image_contents="${image_name%.bin}_contents.txt"
   local image_contents_wtd="${image_name%.bin}_contents_wtd.txt"
   local image_packages="${image_name%.bin}_packages.txt"
@@ -85,8 +77,7 @@ create_prod_image() {
   local image_initrd_contents="${image_name%.bin}_initrd_contents.txt"
   local image_initrd_contents_wtd="${image_name%.bin}_initrd_contents_wtd.txt"
   local image_disk_usage="${image_name%.bin}_disk_usage.txt"
-  local image_realinitrd_contents="${image_name%.bin}_realinitrd_contents.txt"
+  local image_pkgdb="${image_name%.bin}_pkgdb.tar.xz"
-  local image_realinitrd_contents_wtd="${image_name%.bin}_realinitrd_contents_wtd.txt"
   local image_sysext_base="${image_name%.bin}_sysext.squashfs"
 
   start_image "${image_name}" "${disk_layout}" "${root_fs_dir}" "${update_group}"
@@ -97,31 +88,9 @@ create_prod_image() {
   emerge_to_image "${root_fs_dir}" "${base_pkg}"
   run_ldconfig "${root_fs_dir}"
   run_localedef "${root_fs_dir}"
-
-  local root_with_everything="${root_fs_dir}"
-
-  # Call helper script for adding sysexts to the base OS.
-  # Helper will generate a rootfs dir with all packages (base OS and sysexts) included.
-  local root_sysext_mergedir="${BUILD_DIR}/rootfs-with-sysext-pkgs"
-  if [[ -n "${base_sysexts}" ]] ; then
-    "${BUILD_LIBRARY_DIR}/sysext_prod_builder" \
-        "${BOARD}" "${BUILD_DIR}" "${root_fs_dir}" \
-        "${root_sysext_mergedir}" \
-        "${root_fs_sysexts_output_dir}" \
-        "${base_sysexts}"
-    root_with_everything="${root_sysext_mergedir}"
-  fi
-
-
-  write_sbom "${root_with_everything}" "${BUILD_DIR}/${image_sbom}"
-  write_licenses "${root_with_everything}" "${BUILD_DIR}/${image_licenses}"
-
-  if [[ -n "${base_sysexts}" ]] ; then
-    sudo rm -rf "${root_sysext_mergedir}"
-  fi
-
   write_packages "${root_fs_dir}" "${BUILD_DIR}/${image_packages}"
-
+  write_sbom "${root_fs_dir}" "${BUILD_DIR}/${image_sbom}"
+  write_licenses "${root_fs_dir}" "${BUILD_DIR}/${image_licenses}"
   insert_licenses "${BUILD_DIR}/${image_licenses}" "${root_fs_dir}"
   insert_extra_slsa "${root_fs_dir}"
 
@@ -133,11 +102,12 @@ create_prod_image() {
           || die_notrace "coreos-au-key is missing the 'official' use flag"
   fi
 
+  tar -cf "${BUILD_DIR}/${image_pkgdb}" -C "${root_fs_dir}" var/cache/edb var/db/pkg
   sudo cp -a "${root_fs_dir}" "${BUILD_DIR}/root_fs_dir2"
   sudo rsync -a --delete  "${BUILD_DIR}/configroot/etc/portage" "${BUILD_DIR}/root_fs_dir2/etc"
-  sudo mksquashfs "${BUILD_DIR}/root_fs_dir2"  "${BUILD_DIR}/${image_sysext_base}" -noappend -xattrs-exclude '^btrfs.'
+  sudo mksquashfs "${BUILD_DIR}/root_fs_dir2"  "${BUILD_DIR}/${image_sysext_base}" -noappend
   sudo rm -rf "${BUILD_DIR}/root_fs_dir2"
-
+  
   # clean-ups of things we do not need
   sudo rm ${root_fs_dir}/etc/csh.env
   sudo rm -rf ${root_fs_dir}/etc/env.d
@@ -160,25 +130,14 @@ create_prod_image() {
 L+  /etc/ld.so.conf     -   -   -   -   ../usr/lib/ld.so.conf
 EOF
 
-  local -a bad_pam_files
+  # Move the PAM configuration into /usr
-  mapfile -t -d '' bad_pam_files < <(find "${root_fs_dir}"/etc/security "${root_fs_dir}"/etc/pam.d ! -type d ! -name '.keep*' -print0)
+  sudo mkdir -p ${root_fs_dir}/usr/lib/pam.d
-  if [[ ${#bad_pam_files[@]} -gt 0 ]]; then
+  sudo mv -n ${root_fs_dir}/etc/pam.d/* ${root_fs_dir}/usr/lib/pam.d/
-    error "Found following PAM config files: ${bad_pam_files[@]#"${root_fs_dir}"}"
+  sudo rmdir ${root_fs_dir}/etc/pam.d
-    error "Expected them to be either removed or, better, vendored (/etc/pam.d files should be in /usr/lib/pam, /etc/security files should be in /usr/lib/pam/security)."
-    error "Vendoring can be done with vendorize_pam_files inside a post_src_install hook for the package that installed the config file."
-    die "PAM config errors spotted"
-  fi
 
   # Remove source locale data, only need to ship the compiled archive.
   sudo rm -rf ${root_fs_dir}/usr/share/i18n/
 
-  # Inject ephemeral sysext signing certificate
-  sudo mkdir -p "${root_fs_dir}/usr/lib/verity.d"
-  sudo cp "${SYSEXT_SIGNING_KEY_DIR}/sysexts.crt" "${root_fs_dir}/usr/lib/verity.d"
-
-  # Finish image will move files from /etc to /usr/share/flatcar/etc.
-  # Note that image filesystem contents generated by finish_image will not
-  # include sysext contents (only the sysext squashfs files themselves).
   finish_image \
       "${image_name}" \
       "${disk_layout}" \
@@ -192,21 +151,40 @@ EOF
       "${image_kconfig}" \
       "${image_initrd_contents}" \
       "${image_initrd_contents_wtd}" \
-      "${image_disk_usage}" \
+      "${image_disk_usage}"
-      "${image_realinitrd_contents}" \
-      "${image_realinitrd_contents_wtd}"
 
-  # Official builds will sign and upload these files later, so remove them to
+  # Upload
-  # prevent them from being uploaded now.
+  local to_upload=(
-  if [[ ${COREOS_OFFICIAL:-0} -eq 1 ]]; then
+    "${BUILD_DIR}/${image_contents}"
-    rm -v \
+    "${BUILD_DIR}/${image_contents_wtd}"
-        "${BUILD_DIR}/${image_kernel}" \
+    "${BUILD_DIR}/${image_packages}"
-        "${BUILD_DIR}/${image_pcr_policy}" \
+    "${BUILD_DIR}/${image_sbom}"
-        "${BUILD_DIR}/${image_grub}"
+    "${BUILD_DIR}/${image_licenses}"
-  fi
+    "${BUILD_DIR}/${image_kernel}"
+    "${BUILD_DIR}/${image_pcr_policy}"
+    "${BUILD_DIR}/${image_grub}"
+    "${BUILD_DIR}/${image_kconfig}"
+    "${BUILD_DIR}/${image_initrd_contents}"
+    "${BUILD_DIR}/${image_initrd_contents_wtd}"
+    "${BUILD_DIR}/${image_disk_usage}"
+    "${BUILD_DIR}/${image_sysext_base}"
+  )
 
   local files_to_evaluate=( "${BUILD_DIR}/${image_name}" )
-  compress_disk_images files_to_evaluate
+  declare -a compressed_images
+  declare -a extra_files
+  compress_disk_images files_to_evaluate compressed_images extra_files
+  to_upload+=( "${compressed_images[@]}" )
+  to_upload+=( "${extra_files[@]}" )
+
+  # FIXME(bgilbert): no shim on arm64
+  if [[ -f "${BUILD_DIR}/${image_shim}" ]]; then
+    to_upload+=("${BUILD_DIR}/${image_shim}")
+  fi
+  upload_image -d "${BUILD_DIR}/${image_name}.DIGESTS" "${to_upload[@]}"
+
+  # Upload legacy digests
+  upload_legacy_digests "${BUILD_DIR}/${image_name}.DIGESTS" compressed_images
 }
 
 create_prod_tar() {
@@ -223,136 +201,5 @@ create_prod_tar() {
   sudo umount "/mnt/${lodevbase}p9"
   sudo rmdir "/mnt/${lodevbase}p9"
   sudo losetup --detach "${lodev}"
-}
+  upload_image "${container}"
-
-create_prod_sysexts() {
-  local image_name="$1"
-  local image_sysext_base="${image_name%.bin}_sysext.squashfs"
-  for sysext in "${EXTRA_SYSEXTS[@]}"; do
-    local name pkgs useflags arches
-    IFS="|" read -r name pkgs useflags arches <<< "$sysext"
-    name="flatcar-$name"
-    local pkg_array=(${pkgs//,/ })
-    local arch_array=(${arches//,/ })
-    local useflags_array=(${useflags//,/ })
-
-    local mangle_script="${BUILD_LIBRARY_DIR}/sysext_mangle_${name}"
-    if [[ ! -x "${mangle_script}" ]]; then
-      mangle_script=
-    fi
-
-    if [[ -n "$arches" ]]; then
-      should_skip=1
-      for arch in "${arch_array[@]}"; do
-        if [[ $arch == "$ARCH" ]]; then
-          should_skip=0
-        fi
-      done
-      if [[ $should_skip -eq 1 ]]; then
-        continue
-      fi
-    fi
-
-    sudo rm -f "${BUILD_DIR}/${name}.raw" \
-	"${BUILD_DIR}/flatcar-test-update-${name}.gz" \
-	"${BUILD_DIR}/${name}_*"
-    # we use -E to pass the USE flags, but also MODULES_SIGN variables
-    #
-    # The --install_root_basename="${name}-extra-sysext-rootfs" flag
-    # is important - it sets the name of a rootfs directory, which is
-    # used to determine the package target in
-    # coreos/base/profile.bashrc
-    USE="${useflags_array[*]}" sudo -E "${SCRIPT_ROOT}/build_sysext" --board="${BOARD}" \
-        --squashfs_base="${BUILD_DIR}/${image_sysext_base}" \
-        --image_builddir="${BUILD_DIR}" \
-        --install_root_basename="${name}-extra-sysext-rootfs" \
-        ${mangle_script:+--manglefs_script=${mangle_script}} \
-        "${name}" "${pkg_array[@]}"
-    delta_generator \
-      -private_key "/usr/share/update_engine/update-payload-key.key.pem" \
-      -new_image "${BUILD_DIR}/${name}.raw" \
-      -out_file "${BUILD_DIR}/flatcar_test_update-${name}.gz"
-  done
-}
-
-create_oem_sysexts() {
-  local image_name=${1}; shift
-  local requested_oem_sysexts_csv=${1}; shift
-  local image_sysext_base="${image_name%.bin}_sysext.squashfs"
-  local overlay_path
-  overlay_path=$(portageq get_repo_path / coreos-overlay)
-
-  local -a oem_sysexts
-  get_oem_sysext_matrix "${ARCH}" oem_sysexts
-  if [[ ${requested_oem_sysexts_csv} != 'everything!' ]]; then
-      local -a all_oems requested_oems invalid_oems
-      all_oems=( "${oem_sysexts[@]}" )
-      all_oems=( "${all_oems[@]%%|*}" )
-      all_oems=( "${all_oems[@]#oem-}" )
-      mapfile -t requested_oems <<<"${requested_oem_sysexts_csv//,/$'\n'}"
-      mapfile -t invalid_oems < <(comm -23 <(printf '%s\n' "${requested_oems[@]}" | sort -u) <(printf '%s\n' "${all_oems[@]}" | sort -u))
-      if [[ ${#invalid_oems[@]} -gt 0 ]]; then
-          die "Requested OEMs to build sysexts for are invalid: ${invalid_oems[*]}, valid OEMs are ${all_oems[*]}"
-      fi
-      mapfile -t oem_sysexts < <(printf '%s\n' "${oem_sysexts[@]}" | grep '^oem-\('"${requested_oem_sysexts_csv//,/'\|'}"'\)|')
-  fi
-
-  local sysext name metapkg useflags
-  for sysext in "${oem_sysexts[@]}"; do
-    IFS="|" read -r name metapkg useflags <<< "${sysext}"
-
-    # Check for manglefs script in the package's files directory
-    local mangle_script="${overlay_path}/${metapkg}/files/manglefs.sh"
-    if [[ ! -x "${mangle_script}" ]]; then
-      mangle_script=
-    fi
-
-    sudo rm -f "${BUILD_DIR}/${name}.raw" \
-        "${BUILD_DIR}/flatcar_test_update-${name}.gz" \
-        "${BUILD_DIR}/${name}_"*
-
-    info "Building OEM sysext ${name} with USE=${useflags}"
-    # The --install_root_basename="${name}-oem-sysext-rootfs" flag is
-    # important - it sets the name of a rootfs directory, which is
-    # used to determine the package target in
-    # coreos/base/profile.bashrc
-    #
-    # OEM sysexts use no compression here since they will be stored
-    # in a compressed OEM partition.
-    USE="${useflags}" sudo -E "${SCRIPT_ROOT}/build_sysext" --board="${BOARD}" \
-        --squashfs_base="${BUILD_DIR}/${image_sysext_base}" \
-        --image_builddir="${BUILD_DIR}" \
-        --metapkgs="${metapkg}" \
-        --install_root_basename="${name}-oem-sysext-rootfs" \
-        --compression=none \
-        ${mangle_script:+--manglefs_script="${mangle_script}"} \
-        "${name}"
-    delta_generator \
-      -private_key "/usr/share/update_engine/update-payload-key.key.pem" \
-      -new_image "${BUILD_DIR}/${name}.raw" \
-      -out_file "${BUILD_DIR}/flatcar_test_update-${name}.gz"
-  done
-}
-
-sbsign_prod_image() {
-  local image_name="$1"
-  local disk_layout="$2"
-
-  info "Signing production image ${image_name} for Secure Boot"
-  local root_fs_dir="${BUILD_DIR}/rootfs"
-  local image_prefix="${image_name%.bin}"
-  local image_kernel="${image_prefix}.vmlinuz"
-  local image_pcr_policy="${image_prefix}_pcr_policy.zip"
-  local image_grub="${image_prefix}.grub"
-
-  sbsign_image \
-      "${image_name}" \
-      "${disk_layout}" \
-      "${root_fs_dir}" \
-      "${image_kernel}" \
-      "${image_pcr_policy}" \
-      "${image_grub}"
-
-  local files_to_evaluate=( "${BUILD_DIR}/${image_name}" )
-  compress_disk_images files_to_evaluate
 }

build_library/qemu_template.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/sh
 
 SCRIPT_DIR="$(dirname "$0")"
 VM_BOARD=
@@ -17,11 +17,7 @@ SSH_KEYS=""
 CLOUD_CONFIG_FILE=""
 IGNITION_CONFIG_FILE=""
 CONFIG_IMAGE=""
-SWTPM_DIR=
 SAFE_ARGS=0
-FORWARDED_PORTS=""
-PRIMARY_DISK_OPTS=""
-DISKS=()
 USAGE="Usage: $0 [-a authorized_keys] [--] [qemu options...]
 Options:
     -i FILE     File containing an Ignition config
@@ -29,25 +25,7 @@ Options:
     -u FILE     Cloudinit user-data as either a cloud config or script.
     -c FILE     Config drive as an iso or fat filesystem image.
     -a FILE     SSH public keys for login access. [~/.ssh/id_{dsa,rsa}.pub]
-    -d DISK     Setup additional disk. Can be used multiple times to
-                setup multiple disks. The value is a path to an image
-                file, optionally followed by a comma and options to
-                pass to virtio-blk-pci device. For example -d
-                /tmp/qcow2-disk,serial=secondary.
-    -D OPTS     Additional virtio-blk-pci options for primary
-                disk. For example serial=primary-disk.
     -p PORT     The port on localhost to map to the VM's sshd. [2222]
-    -I FILE     Set a custom image file.
-    -f PORT     Forward host_port:guest_port.
-    -M MB       Set VM memory in MBs.
-    -T DIR      Add a software TPM2 device through swtpm which stores secrets
-                and the control socket to the given directory. This may need
-                some configuration first with 'swtpm_setup --tpmstate DIR ...'
-                (see https://github.com/stefanberger/swtpm/wiki/Certificates-created-by-swtpm_setup).
-    -R FILE     Set up pflash ro content, e.g., for UEFI (with -W).
-    -W FILE     Set up pflash rw content, e.g., for UEFI (with -R).
-    -K FILE     Set kernel for direct boot used to simulate a PXE boot (with -r).
-    -r FILE     Set initrd for direct boot used to simulate a PXE boot (with -K).
     -s          Safe settings: single simple cpu and no KVM.
     -h          this ;-)
 
@@ -64,8 +42,8 @@ used as an explicit separator. See the qemu(1) man page for more details.
 "
 
 die(){
-    echo "${1}"
+	echo "${1}"
-    exit 1
+	exit 1
 }
 
 check_conflict() {
@@ -92,42 +70,12 @@ while [ $# -ge 1 ]; do
             check_conflict
             SSH_KEYS="$2"
             shift 2 ;;
-        -d|-disk)
-            DISKS+=( "$2" )
-            shift 2 ;;
-        -D|-image-disk-opts)
-            PRIMARY_DISK_OPTS="$2"
-            shift 2 ;;
         -p|-ssh-port)
             SSH_PORT="$2"
             shift 2 ;;
-        -f|-forward-port)
-            FORWARDED_PORTS="${FORWARDED_PORTS} $2"
-            shift 2 ;;
         -s|-safe)
             SAFE_ARGS=1
             shift ;;
-        -I|-image-file)
-            VM_IMAGE="$2"
-            shift 2 ;;
-        -M|-memory)
-            VM_MEMORY="$2"
-            shift 2 ;;
-        -T|-tpm)
-            SWTPM_DIR="$2"
-            shift 2 ;;
-        -R|-pflash-ro)
-            VM_PFLASH_RO="$2"
-            shift 2 ;;
-        -W|-pflash-rw)
-            VM_PFLASH_RW="$2"
-            shift 2 ;;
-        -K|-kernel-file)
-            VM_KERNEL="$2"
-            shift 2 ;;
-        -r|-initrd-file)
-            VM_INITRD="$2"
-            shift 2 ;;
         -v|-verbose)
             set -x
             shift ;;
@@ -161,29 +109,6 @@ write_ssh_keys() {
     sed -e 's/^/ - /'
 }
 
-if [ -n "${SWTPM_DIR}" ]; then
-    mkdir -p "${SWTPM_DIR}"
-    if ! command -v swtpm >/dev/null; then
-        echo "$0: swtpm command not found!" >&2
-        exit 1
-    fi
-    case "${VM_BOARD}" in
-        amd64-usr)
-            TPM_DEV=tpm-tis ;;
-        arm64-usr)
-            TPM_DEV=tpm-tis-device ;;
-        *) die "Unsupported arch" ;;
-    esac
-    SWTPM_SOCK="${SWTPM_DIR}/socket"
-    swtpm socket --tpmstate "dir=${SWTPM_DIR}" --ctrl "type=unixio,path=${SWTPM_SOCK},terminate" --tpm2 &
-    SWTPM_PROC=$!
-    PARENT=$$
-    # The swtpm process exits if qemu disconnects but if we never started qemu because
-    # this script fails or qemu failed to start, we need to kill the process.
-    # The EXIT trap is already in use by the config drive cleanup and anyway doesn't work with kill -9.
-    (while [ -e "/proc/${PARENT}" ]; do sleep 1; done; kill "${SWTPM_PROC}" 2>/dev/null; exit 0) &
-    set -- -chardev "socket,id=chrtpm,path=${SWTPM_SOCK}" -tpmdev emulator,id=tpm0,chardev=chrtpm -device "${TPM_DEV}",tpmdev=tpm0 "$@"
-fi
 
 if [ -z "${CONFIG_IMAGE}" ]; then
     CONFIG_DRIVE=$(mktemp -d)
@@ -223,15 +148,6 @@ if [ -z "${CONFIG_IMAGE}" ]; then
     fi
 fi
 
-# Process port forwards
-QEMU_FORWARDED_PORTS=""
-for port in ${FORWARDED_PORTS}; do
-    host_port=${port%:*}
-    guest_port=${port#*:}
-    QEMU_FORWARDED_PORTS="${QEMU_FORWARDED_PORTS},hostfwd=tcp::${host_port}-:${guest_port}"
-done
-QEMU_FORWARDED_PORTS="${QEMU_FORWARDED_PORTS#,}"
-
 # Start assembling our default command line arguments
 if [ "${SAFE_ARGS}" -eq 1 ]; then
     # Disable KVM, for testing things like UEFI which don't like it
@@ -239,16 +155,12 @@ if [ "${SAFE_ARGS}" -eq 1 ]; then
 else
     case "${VM_BOARD}+$(uname -m)" in
         amd64-usr+x86_64)
-            set -- -global ICH9-LPC.disable_s3=1 \
-                   -global driver=cfi.pflash01,property=secure,value=on \
-                   "$@"
             # Emulate the host CPU closely in both features and cores.
-            set -- -machine q35,accel=kvm:hvf:tcg,smm=on -cpu host -smp "${VM_NCPUS}" "$@"
+            set -- -machine accel=kvm:hvf:tcg -cpu host -smp "${VM_NCPUS}" "$@" ;;
-            ;;
         amd64-usr+*)
-            set -- -machine q35 -cpu kvm64 -smp 1 -nographic "$@" ;;
+            set -- -machine pc-q35-2.8 -cpu kvm64 -smp 1 -nographic "$@" ;;
-        arm64-usr+aarch64|arm64-usr+arm64)
+        arm64-usr+aarch64)
-            set -- -machine virt,accel=kvm:hvf:tcg,gic-version=3 -cpu host -smp "${VM_NCPUS}" -nographic "$@" ;;
+            set -- -machine virt,accel=kvm,gic-version=3 -cpu host -smp "${VM_NCPUS}" -nographic "$@" ;;
         arm64-usr+*)
             if test "${VM_NCPUS}" -gt 4 ; then
                 VM_NCPUS=4
@@ -273,36 +185,23 @@ if [ -n "${CONFIG_IMAGE}" ]; then
 fi
 
 if [ -n "${VM_IMAGE}" ]; then
-    if [[ ,${PRIMARY_DISK_OPTS}, = *,drive=* || ,${PRIMARY_DISK_OPTS}, = *,bootindex=* ]]; then
+    case "${VM_BOARD}" in
-        die "Can't override drive or bootindex options for primary disk"
+        amd64-usr)
-    fi
+            set -- -drive if=virtio,file="${SCRIPT_DIR}/${VM_IMAGE}" "$@" ;;
-    set -- -drive if=none,id=blk,file="${VM_IMAGE}" \
+        arm64-usr)
-        -device virtio-blk-pci,drive=blk,bootindex=1${PRIMARY_DISK_OPTS:+,}${PRIMARY_DISK_OPTS:-} "$@"
+            set -- -drive if=none,id=blk,file="${SCRIPT_DIR}/${VM_IMAGE}" \
+            -device virtio-blk-device,drive=blk "$@"
+            ;;
+        *) die "Unsupported arch" ;;
+    esac
 fi
 
-declare -i id_counter=1
-
-for disk in "${DISKS[@]}"; do
-    disk_id="flatcar-extra-disk-$((id_counter++))"
-    if [[ ${disk} = *,* ]]; then
-        disk_path=${disk%%,*}
-        disk_opts=${disk#*,}
-    else
-        disk_path=${disk}
-        disk_opts=
-    fi
-    set -- \
-        -drive "if=none,id=${disk_id},file=${disk_path}" \
-        -device "virtio-blk-pci,drive=${disk_id}${disk_opts:+,}${disk_opts:-}" \
-        "${@}"
-done
-
 if [ -n "${VM_KERNEL}" ]; then
-    set -- -kernel "${VM_KERNEL}" "$@"
+    set -- -kernel "${SCRIPT_DIR}/${VM_KERNEL}" "$@"
 fi
 
 if [ -n "${VM_INITRD}" ]; then
-    set -- -initrd "${VM_INITRD}" "$@"
+    set -- -initrd "${SCRIPT_DIR}/${VM_INITRD}" "$@"
 fi
 
 if [ -n "${VM_UUID}" ]; then
@@ -311,13 +210,13 @@ fi
 
 if [ -n "${VM_CDROM}" ]; then
     set -- -boot order=d \
-        -drive file="${VM_CDROM}",media=cdrom,format=raw "$@"
+	-drive file="${SCRIPT_DIR}/${VM_CDROM}",media=cdrom,format=raw "$@"
 fi
 
 if [ -n "${VM_PFLASH_RO}" ] && [ -n "${VM_PFLASH_RW}" ]; then
     set -- \
-        -drive if=pflash,unit=0,file="${VM_PFLASH_RO}",format=qcow2,readonly=on \
+        -drive if=pflash,file="${SCRIPT_DIR}/${VM_PFLASH_RO}",format=raw,readonly=on \
-        -drive if=pflash,unit=1,file="${VM_PFLASH_RW}",format=qcow2 "$@"
+        -drive if=pflash,file="${SCRIPT_DIR}/${VM_PFLASH_RW}",format=raw "$@"
 fi
 
 if [ -n "${IGNITION_CONFIG_FILE}" ]; then
@@ -326,18 +225,25 @@ fi
 
 case "${VM_BOARD}" in
     amd64-usr)
-        QEMU_BIN=qemu-system-x86_64 ;;
+        # Default to KVM, fall back on full emulation
+        qemu-system-x86_64 \
+            -name "$VM_NAME" \
+            -m ${VM_MEMORY} \
+            -netdev user,id=eth0,hostfwd=tcp::"${SSH_PORT}"-:22,hostname="${VM_NAME}" \
+            -device virtio-net-pci,netdev=eth0 \
+            -object rng-random,filename=/dev/urandom,id=rng0 -device virtio-rng-pci,rng=rng0 \
+            "$@"
+        ;;
     arm64-usr)
-        QEMU_BIN=qemu-system-aarch64 ;;
+        qemu-system-aarch64 \
+            -name "$VM_NAME" \
+            -m ${VM_MEMORY} \
+            -netdev user,id=eth0,hostfwd=tcp::"${SSH_PORT}"-:22,hostname="${VM_NAME}" \
+            -device virtio-net-device,netdev=eth0 \
+            -object rng-random,filename=/dev/urandom,id=rng0 -device virtio-rng-pci,rng=rng0 \
+            "$@"
+        ;;
     *) die "Unsupported arch" ;;
 esac
 
-"$QEMU_BIN" \
-    -name "$VM_NAME" \
-    -m ${VM_MEMORY} \
-    -netdev user,id=eth0${QEMU_FORWARDED_PORTS:+,}${QEMU_FORWARDED_PORTS},hostfwd=tcp::"${SSH_PORT}"-:22,hostname="${VM_NAME}" \
-    -device virtio-net-pci,netdev=eth0 \
-    -object rng-random,filename=/dev/urandom,id=rng0 -device virtio-rng-pci,rng=rng0 \
-    "$@"
-
 exit $?

build_library/release_util.sh

@@ -2,8 +2,44 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+GSUTIL_OPTS=
+UPLOAD_ROOT=
+UPLOAD_PATH=
+TORCX_UPLOAD_ROOT=
+UPLOAD_DEFAULT=${FLAGS_FALSE}
 DEFAULT_IMAGE_COMPRESSION_FORMAT="bz2"
 
+# Default upload root can be overridden from the environment.
+_user="${USER}"
+[[ ${USER} == "root" ]] && _user="${SUDO_USER}"
+: ${FLATCAR_UPLOAD_ROOT:=gs://users.developer.core-os.net/${_user}}
+: ${FLATCAR_TORCX_UPLOAD_ROOT:=${FLATCAR_UPLOAD_ROOT}/torcx}
+unset _user
+
+DEFINE_boolean parallel ${FLAGS_TRUE} \
+  "Enable parallelism in gsutil."
+DEFINE_boolean upload ${UPLOAD_DEFAULT} \
+  "Upload all packages/images via gsutil."
+DEFINE_boolean private ${FLAGS_TRUE} \
+  "Upload the image as a private object."
+DEFINE_string upload_root "${FLATCAR_UPLOAD_ROOT}" \
+  "Upload prefix, board/version/etc will be appended. Must be a gs:// URL."
+DEFINE_string upload_path "" \
+  "Full upload path, overrides --upload_root. Must be a full gs:// URL."
+DEFINE_string download_root "" \
+  "HTTP download prefix, board/version/etc will be appended."
+DEFINE_string download_path "" \
+  "HTTP download path, overrides --download_root."
+DEFINE_string torcx_upload_root "${FLATCAR_TORCX_UPLOAD_ROOT}" \
+  "Tectonic torcx package and manifest Upload prefix. Must be a gs:// URL."
+DEFINE_string tectonic_torcx_download_root "" \
+  "HTTP download prefix for tectonic torcx packages and manifests."
+DEFINE_string tectonic_torcx_download_path "" \
+  "HTTP download path, overrides --tectonic_torcx_download_root."
+DEFINE_string sign "" \
+  "Sign all files to be uploaded with the given GPG key."
+DEFINE_string sign_digests "" \
+  "Sign image DIGESTS files with the given GPG key."
 DEFINE_string image_compression_formats "${DEFAULT_IMAGE_COMPRESSION_FORMAT}" \
   "Compress the resulting images using thise formats. This option acceps a list of comma separated values. Options are: none, bz2, gz, zip, zst"
 DEFINE_boolean only_store_compressed ${FLAGS_TRUE} \
@@ -39,21 +75,9 @@ compress_file() {
         ;;
     esac
 
-    # Check if symlink in which case we set up a "compressed" symlink
+    ${IMAGE_ZIPPER} -f "${filepath}" 2>&1 >/dev/null || die "failed to compress ${filepath}"
-    local compressed_name="${filepath}.${compression_format}"
-    if [ -L "${filepath}" ]; then
-        # We could also test if the target exists and otherwise do the compression
-        # but we might then end up with two different compressed artifacts
-        local link_target
-        link_target=$(readlink -f "${filepath}")
-        local target_basename
-        target_basename=$(basename "${link_target}")
-        ln -fs "${target_basename}.${compression_format}" "${compressed_name}"
-    else
-        ${IMAGE_ZIPPER} -f "${filepath}" 2>&1 >/dev/null || die "failed to compress ${filepath}"
-    fi
 
-    echo -n "${compressed_name}"
+    echo -n "${filepath}.${compression_format}"
 }
 
 compress_disk_images() {
@@ -61,11 +85,19 @@ compress_disk_images() {
     # among them.
     local -n local_files_to_evaluate="$1"
 
-    info "Compressing ${#local_files_to_evaluate[@]} images"
+    # An array that will hold the path on disk to the resulting disk image archives.
+    # Multiple compression formats may be requested, so this array may hold
+    # multiple archives for the same image.
+    local -n local_resulting_archives="$2"
+
+    # Files that did not match the filter for disk images.
+    local -n local_extra_files="$3"
+
+    info "Compressing images"
     # We want to compress images, but we also want to remove the uncompressed files
     # from the list of uploadable files.
     for filename in "${local_files_to_evaluate[@]}"; do
-        if [[ "${filename}" =~ \.(img|bin|vdi|vhd|vhdx|vmdk|qcow[2]?)$ ]]; then
+        if [[ "${filename}" =~ \.(img|bin|vdi|vhd|vmdk)$ ]]; then
             # Parse the formats as an array. This will yield an extra empty
             # array element at the end.
             readarray -td, FORMATS<<<"${FLAGS_image_compression_formats},"
@@ -74,14 +106,12 @@ compress_disk_images() {
 
             # An associative array we set an element on whenever we process a format.
             # This way we don't process the same format twice. A unique for array elements.
-            # (But first we need to unset the previous loop or we can only compress a single
-            # file per list of files).
-            unset processed_format
             declare -A processed_format
             for format in "${FORMATS[@]}";do
                 if [ -z "${processed_format[${format}]}" ]; then
                     info "Compressing ${filename##*/} to ${format}"
                     COMPRESSED_FILENAME=$(compress_file "${filename}" "${format}")
+                    local_resulting_archives+=( "$COMPRESSED_FILENAME" )
                     processed_format["${format}"]=1
                 fi
             done
@@ -91,11 +121,281 @@ compress_disk_images() {
                [ "${filename##*/}" != "flatcar_production_image.bin" ] &&
                [ "${filename##*/}" != "flatcar_production_update.bin" ] &&
                ! echo "${FORMATS[@]}" | grep -q "none"; then
-                info "Removing ${filename}"
                 rm "${filename}"
-            else
-                info "Keeping ${filename}"
             fi
+        else
+            local_extra_files+=( "${filename}" )            
         fi
     done
 }
+
+upload_legacy_digests() {
+    [[ ${FLAGS_upload} -eq ${FLAGS_TRUE} ]] || return 0
+
+    local local_digest_file="$1"
+    local -n local_compressed_files="$2"
+
+    [[ "${#local_compressed_files[@]}" -gt 0 ]] || return 0
+
+    # Upload legacy digests
+    declare -a digests_to_upload
+    for file in "${local_compressed_files[@]}";do
+        legacy_digest_file="${file}.DIGESTS"
+        cp "${local_digest_file}" "${legacy_digest_file}"
+        digests_to_upload+=( "${legacy_digest_file}" )
+    done
+    local def_upload_path="${UPLOAD_ROOT}/boards/${BOARD}/${FLATCAR_VERSION}"
+    upload_files "digests" "${def_upload_path}" "" "${digests_to_upload[@]}"
+}
+
+check_gsutil_opts() {
+    [[ ${FLAGS_upload} -eq ${FLAGS_TRUE} ]] || return 0
+
+    if [[ ${FLAGS_parallel} -eq ${FLAGS_TRUE} ]]; then
+        GSUTIL_OPTS="-m"
+    fi
+
+    if [[ -n "${FLAGS_upload_root}" ]]; then
+        if [[ "${FLAGS_upload_root}" != gs://* ]] \
+           && [[ "${FLAGS_upload_root}" != rsync://* ]] ; then
+            die_notrace "--upload_root must be a gs:// or rsync:// URL"
+        fi
+        # Make sure the path doesn't end with a slash
+        UPLOAD_ROOT="${FLAGS_upload_root%%/}"
+    fi
+
+    if [[ -n "${FLAGS_torcx_upload_root}" ]]; then
+        if [[ "${FLAGS_torcx_upload_root}" != gs://* ]] \
+           && [[ "${FLAGS_torcx_upload_root}" != rsync://* ]] ; then
+            die_notrace "--torcx_upload_root must be a gs:// or rsync:// URL"
+        fi
+        # Make sure the path doesn't end with a slash
+        TORCX_UPLOAD_ROOT="${FLAGS_torcx_upload_root%%/}"
+    fi
+
+    if [[ -n "${FLAGS_upload_path}" ]]; then
+        if [[ "${FLAGS_upload_path}" != gs://* ]] \
+           && [[ "${FLAGS_upload_path}" != rsync://* ]] ; then
+            die_notrace "--upload_path must be a gs:// or rsync:// URL"
+        fi
+        # Make sure the path doesn't end with a slash
+        UPLOAD_PATH="${FLAGS_upload_path%%/}"
+    fi
+
+    # Ensure scripts run via sudo can use the user's gsutil/boto configuration.
+    if [[ -n "${SUDO_USER}" ]]; then
+        : ${BOTO_PATH:="$HOME/.boto:/home/$SUDO_USER/.boto"}
+        export BOTO_PATH
+    fi
+}
+
+# Generic upload function
+# Usage: upload_files "file type" "${UPLOAD_ROOT}/default/path" "" files...
+#  arg1: file type reported via log
+#  arg2: default upload path, overridden by --upload_path
+#  arg3: upload path suffix that can't be overridden, must end in /
+#  argv: remaining args are files or directories to upload
+upload_files() {
+    [[ ${FLAGS_upload} -eq ${FLAGS_TRUE} ]] || return 0
+
+    local msg="$1"
+    local local_upload_path="$2"
+    local extra_upload_suffix="$3"
+    shift 3
+
+    if [[ -n "${UPLOAD_PATH}" ]]; then
+        local_upload_path="${UPLOAD_PATH}"
+    fi
+
+    if [[ -n "${extra_upload_suffix}" && "${extra_upload_suffix}" != */ ]]
+    then
+        die "upload suffix '${extra_upload_suffix}' doesn't end in /"
+    fi
+
+    info "Uploading ${msg} to ${local_upload_path}"
+
+    if [[ "${local_upload_path}" = 'rsync://'* ]]; then
+        local rsync_upload_path="${local_upload_path#rsync://}"
+        local sshcmd="ssh -o BatchMode=yes "
+              sshcmd="$sshcmd -o StrictHostKeyChecking=no"
+              sshcmd="$sshcmd -o UserKnownHostsFile=/dev/null"
+              sshcmd="$sshcmd -o NumberOfPasswordPrompts=0"
+
+        # ensure the target path exists
+        local sshuserhost="${rsync_upload_path%:*}"
+        local destpath="${rsync_upload_path#*:}"
+        ${sshcmd} "${sshuserhost}" \
+            "mkdir -p ${destpath}/${extra_upload_suffix}"
+
+        # now sync
+        rsync -Pav -e "${sshcmd}" "$@" \
+            "${rsync_upload_path}/${extra_upload_suffix}"
+    else
+        gsutil ${GSUTIL_OPTS} cp -R "$@" \
+            "${local_upload_path}/${extra_upload_suffix}"
+    fi
+}
+
+
+# Identical to upload_files but GPG signs every file if enabled.
+# Usage: sign_and_upload_files "file type" "${UPLOAD_ROOT}/default/path" "" files...
+#  arg1: file type reported via log
+#  arg2: default upload path, overridden by --upload_path
+#  arg3: upload path suffix that can't be overridden, must end in /
+#  argv: remaining args are files or directories to upload
+sign_and_upload_files() {
+    [[ ${FLAGS_upload} -eq ${FLAGS_TRUE} ]] || return 0
+
+    local msg="$1"
+    local path="$2"
+    local suffix="$3"
+    shift 3
+
+    # run a subshell to possibly clean the temporary directory with
+    # signatures without clobbering the global EXIT trap
+    (
+    # Create simple GPG detached signature for all uploads.
+    local sigs=()
+    if [[ -n "${FLAGS_sign}" ]]; then
+        local file
+        local sigfile
+        local sigdir=$(mktemp --directory)
+        trap "rm -rf ${sigdir}" EXIT
+        for file in "$@"; do
+            if [[ "${file}" =~ \.(asc|gpg|sig)$ ]]; then
+                continue
+            fi
+
+            for sigfile in $(find "${file}" ! -type d); do
+                mkdir -p "${sigdir}/${sigfile%/*}"
+                gpg --batch --local-user "${FLAGS_sign}" \
+                    --output "${sigdir}/${sigfile}.sig" \
+                    --detach-sign "${sigfile}" || die "gpg failed"
+            done
+
+            [ -d "${file}" ] &&
+            sigs+=( "${sigdir}/${file}" ) ||
+            sigs+=( "${sigdir}/${file}.sig" )
+        done
+    fi
+
+    upload_files "${msg}" "${path}" "${suffix}" "$@" "${sigs[@]}"
+    )
+}
+
+upload_packages() {
+    [[ ${FLAGS_upload} -eq ${FLAGS_TRUE} ]] || return 0
+    [[ -n "${BOARD}" ]] || die "board_options.sh must be sourced first"
+
+    local board_packages="${1:-"${BOARD_ROOT}/packages"}"
+    local def_upload_path="${UPLOAD_ROOT}/boards/${BOARD}/${FLATCAR_VERSION}"
+    sign_and_upload_files packages ${def_upload_path} "pkgs/" \
+        "${board_packages}"/*
+}
+
+# Upload a set of files (usually images) and digest, optionally w/ gpg sig
+# If more than one file is specified -d must be the first argument
+# Usage: upload_image [-d file.DIGESTS] file1 [file2...]
+upload_image() {
+    [[ ${FLAGS_upload} -eq ${FLAGS_TRUE} ]] || return 0
+    [[ -n "${BOARD}" ]] || die "board_options.sh must be sourced first"
+
+    # The name to use for .DIGESTS and .DIGESTS.asc must be explicit if
+    # there is more than one file to upload to avoid potential confusion.
+    local digests
+    if [[ "$1" == "-d" ]]; then
+        [[ -n "$2" ]] || die "-d requires an argument"
+        digests="$2"
+        shift 2
+    else
+        [[ $# -eq 1 ]] || die "-d is required for multi-file uploads"
+        # digests is assigned after image is possibly compressed/renamed
+    fi
+
+    local uploads=()
+    local filename
+    for filename in "$@"; do
+        if [[ ! -f "${filename}" ]]; then
+            die "File '${filename}' does not exist!"
+        fi
+        uploads+=( "${filename}" )
+    done
+
+    if [[ -z "${digests}" ]]; then
+        digests="${uploads[0]}.DIGESTS"
+    fi
+
+    # For consistency generate a .DIGESTS file similar to the one catalyst
+    # produces for the SDK tarballs and up upload it too.
+    make_digests -d "${digests}" "${uploads[@]}"
+    uploads+=( "${digests}" )
+
+    # Create signature as ...DIGESTS.asc as Gentoo does.
+    if [[ -n "${FLAGS_sign_digests}" ]]; then
+      rm -f "${digests}.asc"
+      gpg --batch --local-user "${FLAGS_sign_digests}" \
+          --clearsign "${digests}" || die "gpg failed"
+      uploads+=( "${digests}.asc" )
+    fi
+
+    local log_msg=$(basename "$digests" .DIGESTS)
+    local def_upload_path="${UPLOAD_ROOT}/boards/${BOARD}/${FLATCAR_VERSION}"
+    sign_and_upload_files "${log_msg}" "${def_upload_path}" "" "${uploads[@]}"
+}
+
+# Translate the configured upload URL to a download URL
+# Usage: download_image_url "path/suffix"
+download_image_url() {
+    if [[ ${FLAGS_upload} -ne ${FLAGS_TRUE} ]]; then
+        echo "$1"
+        return 0
+    fi
+
+    local download_root="${FLAGS_download_root:-${UPLOAD_ROOT}}"
+
+    local download_path
+    local download_channel
+    if [[ -n "${FLAGS_download_path}" ]]; then
+        download_path="${FLAGS_download_path%%/}"
+    elif [[ "${download_root}" == *flatcar-jenkins* ]]; then
+        download_channel="${download_root##*/}"
+        download_root="gs://${download_channel}.release.flatcar-linux.net"
+        # Official release download paths don't include the boards directory
+        download_path="${download_root%%/}/${BOARD}/${FLATCAR_VERSION}"
+    else
+        download_path="${download_root%%/}/boards/${BOARD}/${FLATCAR_VERSION}"
+    fi
+
+    # Just in case download_root was set from UPLOAD_ROOT
+    if [[ "${download_path}" == gs://* ]]; then
+        download_path="https://${download_path#gs://}"
+    fi
+
+    echo "${download_path}/$1"
+}
+
+# Translate the configured torcx upload URL to a download url
+# This is similar to the download_image_url, other than assuming the release
+# bucket is the tectonic_torcx one.
+download_tectonic_torcx_url() {
+    if [[ ${FLAGS_upload} -ne ${FLAGS_TRUE} ]]; then
+        echo "$1"
+        return 0
+    fi
+
+    local download_root="${FLAGS_tectonic_torcx_download_root:-${TORCX_UPLOAD_ROOT}}"
+
+    local download_path
+    if [[ -n "${FLAGS_tectonic_torcx_download_path}" ]]; then
+        download_path="${FLAGS_tectonic_torcx_download_path%%/}"
+    else
+        download_path="${download_root%%/}"
+    fi
+
+    # Just in case download_root was set from UPLOAD_ROOT
+    if [[ "${download_path}" == gs://* ]]; then
+        download_path="http://${download_path#gs://}"
+    fi
+
+    echo "${download_path}/$1"
+}

build_library/reports_util.sh

@@ -33,7 +33,6 @@ write_contents() {
     # %l - symlink target (empty if not a symlink)
     sudo TZ=UTC find -printf \
         '%M %2n %-7u %-7g %7s %TY-%Tm-%Td %TH:%TM ./%P -> %l\n' \
-        | sort --key=8 \
         | sed -e 's/ -> $//' >"${output}"
     popd >/dev/null
 }
@@ -58,8 +57,7 @@ write_contents_with_technical_details() {
     # %s - size in bytes
     # %P - file's path
     sudo find -printf \
-        '%M %D %i %n %s ./%P\n' \
+        '%M %D %i %n %s ./%P\n' >"${output}"
-        | sort --key=6 >"${output}"
     popd >/dev/null
 }
 

build_library/sbsign_util.sh

@@ -1,55 +0,0 @@
-# Copyright (c) 2024 The Flatcar Maintainers.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-if [[ ${COREOS_OFFICIAL:-0} -ne 1 ]]; then
-    SBSIGN_KEY="${SBSIGN_KEY:-/usr/share/sb_keys/shim.key}"
-    SBSIGN_CERT="${SBSIGN_CERT:-/usr/share/sb_keys/shim.pem}"
-else
-    SBSIGN_KEY="pkcs11:token=flatcar-secure-boot-prod-2026-04"
-    unset SBSIGN_CERT
-fi
-
-PKCS11_MODULE_PATH="/usr/$(get_sdk_libdir)/pkcs11/azure-keyvault-pkcs11.so"
-
-PKCS11_ENV=(
-    AZURE_KEYVAULT_URL="https://flatcar-hsm0001.vault.azure.net/"
-    PKCS11_MODULE_PATH="${PKCS11_MODULE_PATH}"
-    AZURE_KEYVAULT_PKCS11_DEBUG=1
-)
-
-get_sbsign_cert() {
-    if [[ ${SBSIGN_KEY} != pkcs11:* || -s ${SBSIGN_CERT-} ]]; then
-        return
-    fi
-
-    SBSIGN_CERT=$(mktemp -t signing-cert.XXXXXXXXXX.pem)
-    info "Fetching ${SBSIGN_KEY} from Azure"
-
-    # Needs Key Vault Reader role.
-    env "${PKCS11_ENV[@]}" p11-kit export-object \
-        --provider "${PKCS11_MODULE_PATH}" \
-        "${SBSIGN_KEY};type=cert" \
-        | tee "${SBSIGN_CERT}"
-}
-
-cleanup_sbsign_certs() {
-    if [[ ${SBSIGN_CERT-} == "${TMPDIR-/tmp}"/* ]]; then
-        rm -f -- "${SBSIGN_CERT}"
-    fi
-}
-
-do_sbsign() {
-    get_sbsign_cert
-    info "Signing ${@:$#} with ${SBSIGN_KEY}"
-
-    if [[ ${SBSIGN_KEY} == pkcs11:* ]]; then
-        set -- --engine pkcs11 "${@}"
-    fi
-
-    # Needs Key Vault Crypto User role.
-    sudo env "${PKCS11_ENV[@]}" sbsign \
-        --key "${SBSIGN_KEY}" \
-        --cert "${SBSIGN_CERT}" \
-        "${@}"
-}

build_library/set_lsb_release

@@ -25,38 +25,40 @@ ROOT_FS_DIR="$FLAGS_root"
 [ -n "$ROOT_FS_DIR" ] || die "--root is required."
 [ -d "$ROOT_FS_DIR" ] || die "Root FS does not exist? ($ROOT_FS_DIR)"
 
-# These variables are set in the base profile.
+OS_NAME="Flatcar Container Linux by Kinvolk"
-eval $("portageq${FLAGS_board:+-}${FLAGS_board}" envvar -v BRANDING_OS_\*)
+OS_CODENAME="Oklo"
-BRANDING_OS_PRETTY_NAME="${BRANDING_OS_NAME} ${FLATCAR_VERSION}"
+OS_ID="flatcar"
+OS_ID_LIKE="coreos"
+OS_PRETTY_NAME="$OS_NAME $FLATCAR_VERSION (${OS_CODENAME})"
 
 FLATCAR_APPID="{e96281a6-d1af-4bde-9a0a-97b76e56dc57}"
 
 # DISTRIB_* are the standard lsb-release names
 sudo mkdir -p "${ROOT_FS_DIR}/usr/share/flatcar" "${ROOT_FS_DIR}/etc/flatcar"
 sudo_clobber "${ROOT_FS_DIR}/usr/share/flatcar/lsb-release" <<EOF
-DISTRIB_ID="$BRANDING_OS_NAME"
+DISTRIB_ID="$OS_NAME"
 DISTRIB_RELEASE=$FLATCAR_VERSION
-DISTRIB_DESCRIPTION="$BRANDING_OS_PRETTY_NAME"
+DISTRIB_CODENAME="$OS_CODENAME"
+DISTRIB_DESCRIPTION="$OS_PRETTY_NAME"
 EOF
 sudo ln -sf "../usr/share/flatcar/lsb-release" "${ROOT_FS_DIR}/etc/lsb-release"
 
 # And the new standard, os-release
 # https://www.freedesktop.org/software/systemd/man/os-release.html
 sudo_clobber "${ROOT_FS_DIR}/usr/lib/os-release" <<EOF
-NAME="$BRANDING_OS_NAME"
+NAME="$OS_NAME"
-ID="$BRANDING_OS_ID"
+ID=$OS_ID
-ID_LIKE="$BRANDING_OS_ID_LIKE"
+ID_LIKE=$OS_ID_LIKE
-VERSION="$FLATCAR_VERSION"
+VERSION=$FLATCAR_VERSION
-VERSION_ID="$FLATCAR_VERSION_ID"
+VERSION_ID=$FLATCAR_VERSION_ID
-BUILD_ID="$FLATCAR_BUILD_ID"
+BUILD_ID=$FLATCAR_BUILD_ID
-SYSEXT_LEVEL="1.0"
+SYSEXT_LEVEL=1.0
-PRETTY_NAME="$BRANDING_OS_PRETTY_NAME"
+PRETTY_NAME="$OS_PRETTY_NAME"
 ANSI_COLOR="38;5;75"
-HOME_URL="$BRANDING_OS_HOME_URL"
+HOME_URL="https://flatcar.org/"
-BUG_REPORT_URL="$BRANDING_OS_BUG_REPORT_URL"
+BUG_REPORT_URL="https://issues.flatcar.org"
-SUPPORT_URL="$BRANDING_OS_SUPPORT_URL"
 FLATCAR_BOARD="$FLAGS_board"
-CPE_NAME="cpe:2.3:o:${BRANDING_OS_ID}-linux:${BRANDING_OS_ID}_linux:${FLATCAR_VERSION}:*:*:*:*:*:*:*"
+CPE_NAME="cpe:2.3:o:${OS_ID}-linux:${OS_ID}_linux:${FLATCAR_VERSION}:*:*:*:*:*:*:*"
 EOF
 sudo ln -sf "../usr/lib/os-release" "${ROOT_FS_DIR}/etc/os-release"
 sudo ln -sf "../../lib/os-release" "${ROOT_FS_DIR}/usr/share/flatcar/os-release"

build_library/sysext_mangle_containerd-flatcar

@@ -1,23 +0,0 @@
-#!/bin/bash
-
-set -euo pipefail
-rootfs="${1}"
-
-pushd "${rootfs}"
-
-# No manpages on Flatcar, no need to ship "stress" tool
-rm -rf ./usr/{bin/{containerd-stress,gen-manpages},lib/debug/}
-
-dir=$(dirname "${BASH_SOURCE[0]}")
-files_dir="${dir}/../sdk_container/src/third_party/coreos-overlay/coreos/sysext/containerd"
-
-echo ">>> NOTICE $0: installing extra files from '${files_dir}'"
-# ATTENTION: don't preserve ownership as repo is owned by sdk user
-cp -vdR --preserve=mode,timestamps "${files_dir}/"* ./
-
-install -D -m0644 /dev/stdin ./usr/lib/systemd/system/multi-user.target.d/10-containerd-service.conf <<EOF
-[Unit]
-Upholds=containerd.service
-EOF
-
-popd

build_library/sysext_mangle_docker-flatcar

@@ -1,21 +0,0 @@
-#!/bin/bash
-
-set -euo pipefail
-rootfs="${1}"
-
-# Remove debug and contrib
-echo ">>> NOTICE: $0: removing '/usr/lib/debug/', '/usr/share/docker/contrib' from sysext"
-rm -rf "${rootfs}/usr/lib/debug/" "${rootfs}/usr/share/docker/contrib/"
-# For Docker 27.2.1, two files are symlinked to /usr/share/docker/contrib
-# There were previously shipped directly in /usr/share/docker/contrib folder
-rm -f "${rootfs}/usr/bin/dockerd-rootless-setuptool.sh" "${rootfs}/usr/bin/dockerd-rootless.sh"
-
-script_root="$(cd "$(dirname "$0")/../"; pwd)"
-files_dir="${script_root}/sdk_container/src/third_party/coreos-overlay/coreos/sysext/docker"
-
-echo ">>> NOTICE $0: installing extra files from '${files_dir}'"
-# ATTENTION: don't preserve ownership as repo is owned by sdk user
-cp -vdR --preserve=mode,timestamps "${files_dir}/"* "${rootfs}"
-
-mkdir -p "${rootfs}/usr/lib/systemd/system/sockets.target.d"
-{ echo "[Unit]"; echo "Upholds=docker.socket"; } > "${rootfs}/usr/lib/systemd/system/sockets.target.d/10-docker-socket.conf"

build_library/sysext_mangle_flatcar-incus

@@ -1,27 +0,0 @@
-#!/bin/bash
-
-set -euo pipefail
-rootfs="${1}"
-
-pushd "${rootfs}"
-
-rm -rf ./usr/{lib/debug,lib64/pkgconfig,include}/
-
-pushd ./usr/lib/systemd/system
-mkdir -p "multi-user.target.d"
-{ echo "[Unit]"; echo "Upholds=incus.service"; } > "multi-user.target.d/10-incus.conf"
-popd
-
-mkdir -p ./usr/lib/tmpfiles.d
-pushd ./usr/lib/tmpfiles.d
-cat <<EOF >./10-incus.conf
-d  /var/lib/lxc/rootfs	0755  root  root  -  -
-EOF
-popd
-
-# Add 'core' user to 'incus-admin' group to avoid prefixing
-# all commands with sudo.
-mkdir -p ./usr/lib/userdb/
-echo " " > ./usr/lib/userdb/core:incus-admin.membership
-
-popd

build_library/sysext_mangle_flatcar-nvidia-drivers-535

@@ -1,14 +0,0 @@
-#!/bin/bash
-
-set -euo pipefail
-
-SCRIPT_NAME=$(basename "$(realpath "${BASH_SOURCE[0]}")")
-SYSEXT_NAME=${SCRIPT_NAME#sysext_mangle_}
-SYSEXT_NAME=${SYSEXT_NAME%.sh}
-DIR=$(dirname "$(realpath "${BASH_SOURCE[0]}")")
-. "$DIR/sysext_mangle_kmod"
-
-rootfs="${1}"
-
-cd "${rootfs}"
-configure_modprobe "$SYSEXT_NAME"

build_library/sysext_mangle_flatcar-nvidia-drivers-535-open

@@ -1 +0,0 @@
-sysext_mangle_flatcar-nvidia-drivers-535

build_library/sysext_mangle_flatcar-nvidia-drivers-550

@@ -1 +0,0 @@
-sysext_mangle_flatcar-nvidia-drivers-535

build_library/sysext_mangle_flatcar-nvidia-drivers-550-open

@@ -1 +0,0 @@
-sysext_mangle_flatcar-nvidia-drivers-535

build_library/sysext_mangle_flatcar-nvidia-drivers-570

@@ -1 +0,0 @@
-sysext_mangle_flatcar-nvidia-drivers-535

build_library/sysext_mangle_flatcar-nvidia-drivers-570-open

@@ -1 +0,0 @@
-sysext_mangle_flatcar-nvidia-drivers-535

build_library/sysext_mangle_flatcar-overlaybd

@@ -1,15 +0,0 @@
-#!/bin/bash
-
-set -euo pipefail
-rootfs="${1}"
-
-pushd "${rootfs}"
-
-rm -rf ./usr/lib/debug/
-
-pushd ./usr/lib/systemd/system
-mkdir -p "multi-user.target.d"
-{ echo "[Unit]"; echo "Upholds=overlaybd-tcmu.service overlaybd-snapshotter.service"; } > "multi-user.target.d/10-overlaybd.conf"
-popd
-
-popd

build_library/sysext_mangle_flatcar-podman

@@ -1,18 +0,0 @@
-#!/bin/bash
-
-set -euo pipefail
-rootfs="${1}"
-
-pushd "${rootfs}"
-
-rm -rf ./usr/{lib/debug,lib64/cmake,lib64/pkgconfig,include,share/aclocal,share/fish}/
-
-mkdir -p ./usr/share/podman/etc
-cp -a ./etc/{fuse.conf,containers} ./usr/share/podman/etc/
-
-cat <<EOF >>./usr/lib/tmpfiles.d/podman.conf
-C /etc/containers - - - - /usr/share/podman/etc/containers
-C /etc/fuse.conf - - - - /usr/share/podman/etc/fuse.conf
-EOF
-
-popd

build_library/sysext_mangle_flatcar-python

@@ -1,20 +0,0 @@
-#!/bin/bash
-
-set -euo pipefail
-rootfs="${1}"
-
-pushd "${rootfs}"
-
-rm -rf ./usr/{lib/debug,share,include,lib64/pkgconfig}
-
-# Remove test stuff from python - it's quite large.
-for p in ./usr/lib/python*; do
-    if [[ ! -d ${p} ]]; then
-        continue
-    fi
-    # find directories named tests or test and remove them (-prune
-    # avoids searching below those directories)
-    find "${p}" \( -name tests -o -name test \) -type d -prune -exec rm -rf '{}' '+'
-done
-
-popd

build_library/sysext_mangle_flatcar-zfs

@@ -1,47 +0,0 @@
-#!/bin/bash
-
-set -euo pipefail
-rootfs="${1}"
-
-DIR="$(dirname "$(realpath "${BASH_SOURCE[0]}")")"
-. "$DIR/sysext_mangle_kmod"
-
-pushd "${rootfs}"
-
-rm -rf ./usr/{lib/debug/,lib64/cmake/,include/}
-rm -rf ./usr/lib/dracut/
-rm -rf ./usr/share/initramfs-tools
-rm -rf ./usr/src
-
-mkdir -p ./usr/share/zfs/etc
-rm -rf ./etc/{csh.env,environment.d/,profile.env}
-cp -a ./etc/. ./usr/share/zfs/etc/
-
-pushd ./usr/lib/systemd/system
-while read cmd unit; do
-  if [ "$cmd" = enable ]; then
-    target=$(awk -F= '/WantedBy/ { print $2 }' $unit)
-    mkdir -p "${target}.wants"
-    ln -svr "${unit}" "${target}".wants/
-  fi
-done < <(grep -v '^#' "${rootfs}"/usr/lib/systemd/system-preset/50-zfs.preset)
-mkdir -p "multi-user.target.d"
-{ echo "[Unit]"; echo "Upholds=zfs.target"; } > "multi-user.target.d/10-zfs.conf"
-popd
-
-mkdir -p ./usr/lib/tmpfiles.d
-cat <<EOF >./usr/lib/tmpfiles.d/10-zfs.conf
-d  /etc/zfs                                                 0755  root  root  -  -
-L  /etc/zfs/zed.d                                           -     -     -     -  /usr/share/zfs/etc/zfs/zed.d
-L  /etc/zfs/zfs-functions                                   -     -     -     -  /usr/share/zfs/etc/zfs/zfs-functions
-L  /etc/zfs/zpool.d                                         -     -     -     -  /usr/share/zfs/etc/zfs/zpool.d
-C  /etc/systemd/system/systemd-udevd.service.d/10-zfs.conf  -     -     -     -  /usr/lib/systemd/system/systemd-udevd.service.d/10-zfs.conf
-EOF
-
-mkdir -p ./usr/lib/systemd/system/systemd-udevd.service.d
-cat <<EOF >./usr/lib/systemd/system/systemd-udevd.service.d/10-zfs.conf
-[Unit]
-After=systemd-sysext.service
-EOF
-configure_modprobe flatcar-zfs
-popd

build_library/sysext_mangle_kmod

@@ -1,48 +0,0 @@
-#!/bin/bash
-
-configure_modprobe() {
-  local sysext_name="${1}"
-  shift
-
-  local module_directories=(./usr/lib/modules/*-flatcar/)
-
-  mkdir -p ./usr/lib/modprobe.d/
-  for module_name in $(find "${module_directories[@]}" -type f \( -name "*.ko" -o -name "*.ko.*" \) -printf "%f\n" | sed -E 's/\.ko(\.\w+)?$//'); do
-    cat <<EOF >> "./usr/lib/modprobe.d/10-${sysext_name}-kmod-sysext.conf"
-install $module_name /usr/libexec/_${sysext_name}_modprobe_helper $module_name
-remove $module_name /usr/libexec/_${sysext_name}_modprobe_helper -r $module_name
-EOF
-  done
-
-  mkdir -p ./usr/libexec/
-  install -m0755 -D /dev/stdin "./usr/libexec/_${sysext_name}_modprobe_helper" <<'EOF'
-#!/bin/bash
-
-set -euo pipefail
-
-action="Loading"
-for arg in "$@"; do
-    if [[ $arg == "-r" ]]; then
-        action="Unloading"
-    fi
-done
-echo "$action kernel module from a sysext..."
-
-KMOD_PATH=/usr/lib/modules/$(uname -r)
-TMP_DIR=$(mktemp -d)
-trap "rm -rf -- '${TMP_DIR}'" EXIT
-mkdir "${TMP_DIR}"/{upper,work}
-
-unshare -m bash -s -- "${@}" <<FOE
-set -euo pipefail
-if ! mountpoint -q "${KMOD_PATH}"; then
-    mount -t overlay overlay -o lowerdir="${KMOD_PATH}",upperdir="${TMP_DIR}"/upper,workdir="${TMP_DIR}"/work "${KMOD_PATH}"
-    depmod
-fi
-modprobe --ignore-install "\${@}"
-FOE
-EOF
-
-  # prevent the sysext from masking /usr/lib/modules/*-flatcar/modules.XXX
-  find "${module_directories[@]}" -maxdepth 1 -mindepth 1 -type f -delete
-}

build_library/sysext_prod_builder

@@ -1,185 +0,0 @@
-#!/bin/bash
-# Copyright (c) 2023 by the Flatcar Maintainers.
-# Use of this source code is governed by the Apache 2.0 license.
-
-# Helper script for building OS images w/ sysexts included.
-# Called by build_image -> prod_image_util.sh.
-# This is a separate script mainly so we can trap EXIT and clean up our mounts
-#  without interfering with traps set by build_image.
-
-# We're in build_library/, script root is one up
-SCRIPT_ROOT="$(cd "$(dirname "$(readlink -f "$0")")/../"; pwd)"
-. "${SCRIPT_ROOT}/common.sh" || exit 1
-
-# Script must run inside the chroot
-assert_inside_chroot
-switch_to_strict_mode
-
-. "${BUILD_LIBRARY_DIR}/build_image_util.sh" || exit 1
-
-# Create a sysext from a package and install it to the OS image.
-# Conventions:
-# - For each <group>/<package>, <group>_<package>_pkginfo will be built. Can be used in subsequent calls
-#   to build dependent sysexts.
-# - If ${BUILD_LIBRARY_DIR}/sysext_mangle_<group>_<package> exists it will be used as FS mangle script
-#   when building the sysext.
-create_prod_sysext() {
-  local BOARD="$1"
-  local output_dir="$2"
-  local workdir="$3"
-  local base_sysext="$4"
-  local install_root="$5"
-  local name="$6"
-  local grp_pkgs="$7"
-  local pkginfo="${8:-}"
-
-  local -a build_sysext_opts=()
-
-  local -a grp_pkg
-  mapfile -t grp_pkg <<<"${grp_pkgs//&/$'\n'}"
-  local msg="Installing ${grp_pkg[*]} in sysext ${name}.raw"
-
-  # Include previous sysexts' pkginfo if supplied
-  if [[ -n "${pkginfo}" ]] ; then
-    if [[ ! -f "${output_dir}/${pkginfo}" ]] ; then
-      die "Sysext build '${name}': unable to find package info at '${output_dir}/${pkginfo}'."
-    fi
-    msg="${msg} w/ package info '${pkginfo}'"
-    build_sysext_opts+=( "--base_pkginfo=${output_dir}/${pkginfo}" )
-  fi
-
-  # Include FS mangle script if present
-  if [[ -x "${BUILD_LIBRARY_DIR}/sysext_mangle_${name}" ]] ; then
-    build_sysext_opts+=( "--manglefs_script=${BUILD_LIBRARY_DIR}/sysext_mangle_${name}" )
-    msg="${msg}, FS mangle script 'sysext_mangle_${name}'"
-  fi
-
-  info "${msg}."
-
-  # Pass the build ID extracted from root FS to build_sysext. This prevents common.sh
-  #   in build_sysext to generate a (timestamp based) build ID during a DEV build of a
-  #   release tag (which breaks its version check).
-  #
-  # The --install_root_basename="${name}-base-sysext-rootfs" flag is
-  # important - it sets the name of a rootfs directory, which is used
-  # to determine the package target in coreos/base/profile.bashrc
-  #
-  # Built-in sysexts are stored in the compressed /usr partition, so we
-  # disable compression to avoid double-compression.
-  sudo -E "FLATCAR_BUILD_ID=$FLATCAR_BUILD_ID" "${SCRIPTS_DIR}/build_sysext" \
-            --board="${BOARD}" \
-            --image_builddir="${workdir}/sysext-build" \
-            --squashfs_base="${base_sysext}" \
-            --generate_pkginfo \
-            --compression=none \
-            --install_root_basename="${name}-base-sysext-rootfs" \
-            "${build_sysext_opts[@]}" \
-            "${name}" "${grp_pkg[@]}"
-
-  sudo mv "${workdir}/sysext-build/${name}.raw" "${workdir}/sysext-build/${name}_pkginfo.raw" \
-                              "${workdir}/sysext-build/${name}"_*.txt "${output_dir}"
-
-  sudo mkdir -p "${install_root}"/usr/share/flatcar/sysext
-  sudo install -m 0644 -D "${output_dir}/${name}.raw" "${install_root}"/usr/share/flatcar/sysext/
-
-  sudo mkdir -p "${install_root}"/etc/extensions/
-  sudo ln -sf "/usr/share/flatcar/sysext/${name}.raw" "${install_root}/etc/extensions/${name}.raw"
-}
-# --
-
-BOARD="$1"
-BUILD_DIR="$2"
-root_fs_dir="$3"
-
-merged_rootfs_dir="$4"
-sysext_output_dir="$5"
-
-sysexts_list="$6"
-
-grp_pkg=""
-prev_pkginfo=""
-sysext_workdir="${BUILD_DIR}/prod-sysext-work"
-sysext_mountdir="${BUILD_DIR}/prod-sysext-work/mounts"
-sysext_base="${sysext_workdir}/base-os.squashfs"
-
-function cleanup() {
-  IFS=':' read -r -a mounted_sysexts <<< "$sysext_lowerdirs"
-  # skip the rootfs
-  mounted_sysexts=("${mounted_sysexts[@]:1}")
-
-  for sysext in "${mounted_sysexts[@]}"; do
-    sudo systemd-dissect --umount --rmdir "$sysext"
-  done
-
-  sudo umount "${sysext_mountdir}"/* || true
-  rm -rf "${sysext_workdir}" || true
-}
-# --
-
-trap cleanup EXIT
-
-rm -rf "${sysext_workdir}" "${sysext_output_dir}"
-mkdir "${sysext_workdir}" "${sysext_output_dir}"
-
-info "creating temporary base OS squashfs"
-sudo mksquashfs "${root_fs_dir}" "${sysext_base}" -noappend -xattrs-exclude '^btrfs.'
-
-# Build sysexts on top of root fs and mount sysexts' squashfs + pkginfo squashfs
-# for combined overlay later.
-prev_pkginfo=""
-sysext_lowerdirs="${sysext_mountdir}/rootfs-lower"
-mkdir -p "${sysext_mountdir}"
-for sysext in ${sysexts_list//,/ }; do
-  # format is "<name>:<group>/<package>"
-  name="${sysext%|*}"
-  grp_pkg="${sysext#*|}"
-  create_prod_sysext "${BOARD}" \
-                     "${sysext_output_dir}" \
-                     "${sysext_workdir}" \
-                     "${sysext_base}" \
-                     "${root_fs_dir}"\
-                     "${name}" \
-                     "${grp_pkg}" \
-                     "${prev_pkginfo}"
-
-  sudo systemd-dissect \
-    --read-only \
-    --mount \
-    --mkdir \
-    --image-policy='root=encrypted+unprotected+absent:usr=encrypted+unprotected+absent' \
-    "${sysext_output_dir}/${name}.raw" \
-    "${sysext_mountdir}/${name}"
-
-  sudo systemd-dissect \
-    --read-only \
-    --mount \
-    --mkdir \
-    --image-policy='root=encrypted+unprotected+absent:usr=encrypted+unprotected+absent' \
-    "${sysext_output_dir}/${name}_pkginfo.raw" \
-    "${sysext_mountdir}/${name}_pkginfo"
-
-  sysext_lowerdirs="${sysext_lowerdirs}:${sysext_mountdir}/${name}"
-  sysext_lowerdirs="${sysext_lowerdirs}:${sysext_mountdir}/${name}_pkginfo"
-
-  prev_pkginfo="${name}_pkginfo.raw"
-done
-
-# Mount the combined overlay (base OS, sysexts, and syset pkginfos) and copy a snapshot
-# into the designated output dir for upper layers to process.
-mkdir -p "${sysext_mountdir}/rootfs-lower" 
-sudo mount -rt squashfs -o loop,nodev "${sysext_base}" "${sysext_mountdir}/rootfs-lower"
-
-# Mount overlay for report generation
-mkdir -p "${sysext_workdir}/.work"
-mkdir -p "${sysext_mountdir}/rootfs-upper" 
-sudo mount -t overlay overlay \
-         -o lowerdir="${sysext_lowerdirs}",upperdir="${sysext_mountdir}/rootfs-upper",workdir="${sysext_workdir}/.work" \
-         "${sysext_mountdir}/rootfs-upper"
-
-
-sudo rm -rf "${merged_rootfs_dir}"
-sudo cp -a "${sysext_mountdir}/rootfs-upper" "${merged_rootfs_dir}"
-
-
-cleanup
-trap -- EXIT

build_library/test_image_content.sh

@@ -4,7 +4,7 @@
 
 GLSA_ALLOWLIST=(
 	201412-09 # incompatible CA certificate version numbers
-	202407-05 # ebuild of sys-auth/sssd already has a custom patch to fix CVE-2021-3621
+	202209-12 # grub 2.06 is still in progress
 )
 
 glsa_image() {

build_library/toolchain_util.sh

@@ -14,18 +14,18 @@ TOOLCHAIN_PKGS=(
 # This is only used as an intermediate step to be able to use the cross
 # compiler to build a full native toolchain. Packages are not uploaded.
 declare -A CROSS_PROFILES
-CROSS_PROFILES["x86_64-cros-linux-gnu"]="coreos-overlay:coreos/amd64/generic"
+CROSS_PROFILES["x86_64-cros-linux-gnu"]="coreos:coreos/amd64/generic"
-CROSS_PROFILES["aarch64-cros-linux-gnu"]="coreos-overlay:coreos/arm64/generic"
+CROSS_PROFILES["aarch64-cros-linux-gnu"]="coreos:coreos/arm64/generic"
 
 # Map board names to CHOSTs and portage profiles. This is the
 # definitive list, there is assorted code new and old that either
 # guesses or hard-code these. All that should migrate to this list.
 declare -A BOARD_CHOSTS BOARD_PROFILES
 BOARD_CHOSTS["amd64-usr"]="x86_64-cros-linux-gnu"
-BOARD_PROFILES["amd64-usr"]="coreos-overlay:coreos/amd64/generic"
+BOARD_PROFILES["amd64-usr"]="coreos:coreos/amd64/generic"
 
 BOARD_CHOSTS["arm64-usr"]="aarch64-cros-linux-gnu"
-BOARD_PROFILES["arm64-usr"]="coreos-overlay:coreos/arm64/generic"
+BOARD_PROFILES["arm64-usr"]="coreos:coreos/arm64/generic"
 
 BOARD_NAMES=( "${!BOARD_CHOSTS[@]}" )
 
@@ -130,24 +130,37 @@ get_board_profile() {
     done
 }
 
-# Usage: get_board_binhost board [version...]
+# Usage: get_board_binhost [-t] board [version...]
+# -t: toolchain only, full rebuilds re-using toolchain pkgs
 # If no versions are specified the current and SDK versions are used.
 get_board_binhost() {
-    local board ver
+    local toolchain_only=0 board ver
+    if [[ "$1" == "-t" ]]; then
+        toolchain_only=1
+        shift
+    fi
     board="$1"
     shift
 
+    local pkgs_include_toolchain=0
     if [[ $# -eq 0 ]]; then
         if [[ "${FLATCAR_BUILD_ID}" =~ ^nightly-.*$ ]] ; then
             # containerised nightly build; this uses [VERSION]-[BUILD_ID] for binpkg url
+            #  and toolchain packages are at the same location as OS image ones
             set -- "${FLATCAR_VERSION_ID}+${FLATCAR_BUILD_ID}"
+            pkgs_include_toolchain=1
         else
             set -- "${FLATCAR_VERSION_ID}"
         fi
     fi
 
     for ver in "$@"; do
-        echo "${FLATCAR_DEV_BUILDS}/boards/${board}/${ver}/pkgs/"
+        if [[ $toolchain_only -eq 0 ]]; then
+            echo "${FLATCAR_DEV_BUILDS}/boards/${board}/${ver}/pkgs/"
+        fi
+        if [[ $pkgs_include_toolchain -eq 0 ]]; then
+            echo "${FLATCAR_DEV_BUILDS}/boards/${board}/${ver}/toolchain/"
+        fi
     done
 }
 
@@ -156,7 +169,7 @@ get_sdk_arch() {
 }
 
 get_sdk_profile() {
-    echo "coreos-overlay:coreos/$(get_sdk_arch)/sdk"
+    echo "coreos:coreos/$(get_sdk_arch)/sdk"
 }
 
 get_sdk_libdir() {
@@ -183,9 +196,13 @@ get_sdk_binhost() {
         FLATCAR_DEV_BUILDS_SDK="${FLATCAR_DEV_BUILDS_SDK-${SETTING_BINPKG_SERVER_PROD}/sdk}"
     fi
     for ver in "$@"; do
+        # Usually only crossdev needs to be fetched from /toolchain/ in the setup_board step.
         # The entry for /pkgs/ is there if something needs to be reinstalled in the SDK
         # but normally it is not needed because everything is already part of the tarball.
+        # To install the crossdev Rust package, /toolchain-arm64/ is derived from /toolchain/
+        # when necessary in install_cross_toolchain().
         if curl -Ifs -o /dev/null "${FLATCAR_DEV_BUILDS_SDK}/${arch}/${ver}/pkgs/"; then
+            echo "${FLATCAR_DEV_BUILDS_SDK}/${arch}/${ver}/toolchain/"
             echo "${FLATCAR_DEV_BUILDS_SDK}/${arch}/${ver}/pkgs/"
         fi
     done
@@ -227,7 +244,7 @@ configure_crossdev_overlay() {
     echo "x-crossdev" | \
         "${sudo[@]}" tee "${root}${location}/profiles/repo_name" > /dev/null
     "${sudo[@]}" tee "${root}${location}/metadata/layout.conf" > /dev/null <<EOF
-masters = portage-stable coreos-overlay
+masters = portage-stable coreos
 use-manifests = true
 thin-manifests = true
 EOF
@@ -246,7 +263,7 @@ _get_dependency_list() {
     local IFS=$'| \t\n'
 
     PORTAGE_CONFIGROOT="$ROOT" emerge "$@" --pretend \
-        --emptytree --onlydeps --quiet | \
+        --emptytree --root-deps=rdeps --onlydeps --quiet | \
         egrep "$ROOT" |
         sed -e 's/[^]]*\] \([^ :]*\).*/=\1/' |
         egrep -v "=($(echo "${pkgs[*]}"))-[0-9]"
@@ -254,7 +271,7 @@ _get_dependency_list() {
 
 # Configure a new ROOT
 # Values are copied from the environment or the current host configuration.
-# Usage: CBUILD=foo-bar-linux-gnu ROOT=/foo/bar SYSROOT=/foo/bar configure_portage coreos-overlay:some/profile
+# Usage: CBUILD=foo-bar-linux-gnu ROOT=/foo/bar SYSROOT=/foo/bar configure_portage coreos:some/profile
 # Note: if using portageq to get CBUILD it must be called before CHOST is set.
 _configure_sysroot() {
     local profile="$1"
@@ -267,14 +284,10 @@ _configure_sysroot() {
 
     "${sudo[@]}" mkdir -p "${ROOT}/etc/portage/"{profile,repos.conf}
     "${sudo[@]}" cp /etc/portage/repos.conf/* "${ROOT}/etc/portage/repos.conf/"
-    # set PORTAGE_CONFIGROOT to tell eselect to modify the profile
+    "${sudo[@]}" eselect profile set --force "$profile"
-    # inside /build/<arch>-usr, but set ROOT to /, so eselect will
-    # actually find the profile which is outside /build/<arch>-usr,
-    # set SYSROOT to / as well, because it must match ROOT
-    "${sudo[@]}" PORTAGE_CONFIGROOT=${ROOT} SYSROOT=/ ROOT=/ eselect profile set --force "$profile"
 
     local coreos_path
-    coreos_path=$(portageq get_repo_path "${ROOT}" coreos-overlay)
+    coreos_path=$(portageq get_repo_path "${ROOT}" coreos)
     "${sudo[@]}" ln -sfT "${coreos_path}/coreos/user-patches" "${ROOT}/etc/portage/patches"
 
     echo "Writing make.conf for the sysroot ${SYSROOT}, root ${ROOT}"
@@ -317,7 +330,7 @@ _get_cross_pkgs_for_emerge_and_crossdev() {
     local -n gcpfeac_emerge_atoms_var_ref="${gcpfeac_emerge_atoms_var_name}"
     local -n gcpfeac_crossdev_pkg_flags_var_ref="${gcpfeac_crossdev_pkg_flags_var_name}"
 
-    local -a all_pkgs=( "${TOOLCHAIN_PKGS[@]}" dev-debug/gdb )
+    local -a all_pkgs=( "${TOOLCHAIN_PKGS[@]}" sys-devel/gdb )
     local -A crossdev_flags_map=(
         [binutils]=--binutils
         [gdb]=--gdb
@@ -397,11 +410,20 @@ install_cross_toolchain() {
     else
         echo "Installing existing binaries"
         "${sudo[@]}" emerge "${emerge_flags[@]}" "${emerge_atoms[@]}"
+        if [ "${cbuild}" = "x86_64-pc-linux-gnu" ] && [ "${cross_chost}" = aarch64-cros-linux-gnu ] && \
+           [ ! -d /usr/lib/rust-*/rustlib/aarch64-unknown-linux-gnu ] && [ ! -d /usr/lib/rustlib/aarch64-unknown-linux-gnu ]; then
+          # If no aarch64 folder exists, warn about the situation but don't compile Rust here or download it as binary package
+          echo "WARNING: No aarch64 cross-compilation Rust libraries found!"
+          echo "In case building fails, make sure the old Rust version is deleted with: sudo emerge --unmerge virtual/rust dev-lang/rust"
+          echo "Then install it again with: sudo emerge ${emerge_flags[@]} virtual/rust"
+          echo "This will download the binary package or build from source."
+        fi
     fi
 
     # Setup environment and wrappers for our shiny new toolchain
     binutils_set_latest_profile "${cross_chost}"
     gcc_set_latest_profile "${cross_chost}"
+    "${sudo[@]}" CC_QUIET=1 sysroot-config --install-links "${cross_chost}"
 }
 
 # Build/install toolchain dependencies into the cross sysroot for a
@@ -450,22 +472,22 @@ install_cross_libs() {
 }
 
 install_cross_rust() {
+    local cross_chost="$1"; shift
+    local emerge_flags=( "$@" --binpkg-respect-use=y --update )
+    local cbuild="$(portageq envvar CBUILD)"
+
     # may be called from either catalyst (root) or upgrade_chroot (user)
     local sudo=("env")
     if [[ $(id -u) -ne 0 ]]; then
         sudo=("sudo" "-E")
     fi
 
-    echo "Installing dev-lang/rust with (potentially outdated) cross targets."
+    if [ "${cbuild}" = "x86_64-pc-linux-gnu" ] && [ "${cross_chost}" = "aarch64-cros-linux-gnu" ]; then
-    "${sudo[@]}" emerge "${emerge_flags[@]}" --binpkg-respect-use=y --update dev-lang/rust
+        echo "Building Rust for arm64"
-
+        # If no aarch64 folder exists, try to remove any existing Rust packages.
-    [[
+        [ ! -d /usr/lib/rustlib/aarch64-unknown-linux-gnu ] && ("${sudo[@]}" emerge --unmerge dev-lang/rust || true)
-       -d /usr/lib/rustlib/x86_64-unknown-linux-gnu &&
+        "${sudo[@]}" emerge "${emerge_flags[@]}" dev-lang/rust
-       -d /usr/lib/rustlib/aarch64-unknown-linux-gnu
+    fi
-    ]] && return
-
-    echo "Rebuilding dev-lang/rust with updated cross targets."
-    "${sudo[@]}" emerge "${emerge_flags[@]}" --usepkg=n dev-lang/rust
 }
 
 # Update to the latest binutils profile for a given CHOST if required
@@ -490,14 +512,10 @@ binutils_set_latest_profile() {
 # The extra flag can be blank, hardenednopie, and so on. See gcc-config -l
 # Usage: gcc_get_latest_profile chost [extra]
 gcc_get_latest_profile() {
-    local prefix=${1}
+    local prefix="${1}-"
-    local suffix=${2+-${2}}
+    local suffix="${2+-$2}"
     local status
-    NO_COLOR=1 gcc-config --list-profiles | \
+    gcc-config -l | cut -d' ' -f3 | grep "^${prefix}[0-9\\.]*${suffix}$" | tail -n1
-        sed -e 's/^\s*//' | \
-        cut -d' ' -f2 | \
-        grep "^${prefix}-[0-9\\.]*${suffix}$" | \
-        tail -n1
 
     # return 1 if anything in the above pipe failed
     for status in ${PIPESTATUS[@]}; do

build_library/torcx_manifest.sh

@@ -0,0 +1,150 @@
+# Copyright (c) 2017 The Container Linux by CoreOS Authors. All rights
+# reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# torcx_manifest.sh contains helper functions for creating, editing, and
+# reading torcx manifest files.
+
+# create_empty creates an empty torcx manfiest at the given path.
+function torcx_manifest::create_empty() {
+  local path="${1}"
+  jq '.' > "${path}" <<EOF
+{
+  "kind": "torcx-package-list-v0",
+  "value": {
+    "packages": []
+  }
+}
+EOF
+}
+
+# add_pkg adds a new version of a package to the torcx manifest specified by
+# path.
+# That manifest will be edited to include this version, with the associated
+# package of the given name being created as well if necessary.
+function torcx_manifest::add_pkg() {
+  path="${1}"; shift
+  name="${1}"; shift
+  version="${1}"; shift
+  pkg_hash="${1}"; shift
+  cas_digest="${1}"; shift
+  source_package="${1}"; shift
+  meta_package="${1}"; shift
+  update_default="${1}"; shift
+
+  local manifest=$(cat "${path}")
+  local pkg_version_obj=$(jq '.' <<EOF
+{
+  "version": "${version}",
+  "hash": "${pkg_hash}",
+  "casDigest": "${cas_digest}",
+  "sourcePackage": "${source_package}",
+  "metaPackage": "${meta_package}",
+  "locations": []
+}
+EOF
+)
+
+  for location in "${@}"; do
+    if [[ "${location}" == /* ]]; then
+      # filepath
+      pkg_version_obj=$(jq ".locations |= . + [{\"path\": \"${location}\"}]" <(echo "${pkg_version_obj}"))
+    else
+      # url
+      pkg_version_obj=$(jq ".locations |= . + [{\"url\": \"${location}\"}]" <(echo "${pkg_version_obj}"))
+    fi
+  done
+
+
+  local existing_pkg="$(echo "${manifest}" | jq ".value.packages[] | select(.name == \"${name}\")")"
+
+  # If there isn't yet a package in the manifest for $name, initialize it to an empty one.
+  if [[ "${existing_pkg}" == "" ]]; then
+    pkg_json=$(cat <<EOF
+    {
+      "name": "${name}",
+      "versions": []
+    }
+EOF
+)
+    manifest="$(echo "${manifest}" | jq ".value.packages |= . + [${pkg_json}]")"
+  fi
+
+  if [[ "${update_default}" == "true" ]]; then
+    manifest="$(echo "${manifest}" | jq "(.value.packages[] | select(.name = \"${name}\") | .defaultVersion) |= \"${version}\"")"
+  fi
+
+  # append this specific package version to the manifest
+  manifest="$(echo "${manifest}" | jq "(.value.packages[] | select(.name = \"${name}\") | .versions) |= . + [${pkg_version_obj}]")"
+
+  echo "${manifest}" | jq '.' > "${path}"
+}
+
+# get_pkg_names returns the list of packages in a given manifest. Each package
+# may have one or more versions associated with it.
+#
+# Example:
+#     pkg_name_arr=($(torcx_manifest::get_pkg_names "torcx_manifest.json"))
+function torcx_manifest::get_pkg_names() {
+  local file="${1}"
+  jq -r '.value.packages[].name' < "${file}"
+}
+
+# local_store_path returns the in-container-linux store path a given package +
+# version combination should exist at. It returns the empty string if the
+# package shouldn't exist on disk.
+function torcx_manifest::local_store_path() {
+  local file="${1}"
+  local name="${2}"
+  local version="${3}"
+  jq -r ".value.packages[] | select(.name == \"${name}\") | .versions[] | select(.version == \"${version}\") | .locations[] | select(.path).path" < "${file}"
+}
+
+# get_digest returns the cas digest for a given package version
+function torcx_manifest::get_digest() {
+  local file="${1}"
+  local name="${2}"
+  local version="${3}"
+  jq -r ".value.packages[] | select(.name == \"${name}\") | .versions[] | select(.version == \"${version}\") | .casDigest" < "${file}"
+}
+
+# get_digests returns the list of digests for a given package. 
+function torcx_manifest::get_digests() {
+  local file="${1}"
+  local name="${2}"
+  jq -r ".value.packages[] | select(.name == \"${name}\").versions[].casDigest" < "${file}"
+}
+
+# get_versions returns the list of versions for a given package. 
+function torcx_manifest::get_versions() {
+  local file="${1}"
+  local name="${2}"
+  jq -r ".value.packages[] | select(.name == \"${name}\").versions[].version" < "${file}"
+}
+
+# default_version returns the default version for a given package, or an empty string if there isn't one.
+function torcx_manifest::default_version() {
+  local file="${1}"
+  local name="${2}"
+  jq -r ".value.packages[] | select(.name == \"${name}\").defaultVersion" < "${file}"
+}
+
+# sources_on_disk returns the list of source packages of all torcx images installed on disk
+function torcx_manifest::sources_on_disk() {
+  local file="${1}"
+  local torcx_pkg=""
+  jq -r ".value.packages[].versions[] | select(.locations[].path).metaPackage" < "${file}" |
+  while read torcx_pkg; do
+    torcx_dependencies "${torcx_pkg}" | tr ' ' '\n'
+  done
+}
+
+# Print the first level of runtime dependencies for a torcx meta-package.
+function torcx_dependencies() (
+  pkg=${1:?}
+  ebuild=$(equery-${BOARD} w "${pkg}")
+  function inherit() { : ; }
+  . "${ebuild}"
+  echo ${RDEPEND}
+)

build_library/vm_image_util.sh

@@ -6,7 +6,6 @@
 # Default values use the format IMG_DEFAULT_<opt>.
 
 VALID_IMG_TYPES=(
-    akamai
     ami
     ami_vmdk
     azure
@@ -16,23 +15,19 @@ VALID_IMG_TYPES=(
     digitalocean
     exoscale
     gce
-    hetzner
     hyperv
-    hyperv_vhdx
     iso
-    kubevirt
     openstack
     openstack_mini
     packet
     parallels
-    proxmoxve
     pxe
+    qemu
     qemu_uefi
+    qemu_uefi_secure
     rackspace
     rackspace_onmetal
     rackspace_vhd
-    scaleway
-    stackit
     vagrant
     vagrant_parallels
     vagrant_virtualbox
@@ -43,29 +38,22 @@ VALID_IMG_TYPES=(
     vmware_ova
     vmware_raw
     xen
-    nutanix
 )
 
 #list of oem package names, minus the oem- prefix
 VALID_OEM_PACKAGES=(
-    akamai
     azure
     cloudsigma
     cloudstack
     digitalocean
     exoscale
     gce
-    hetzner
     hyperv
-    kubevirt
     openstack
     packet
-    proxmoxve
     qemu
     rackspace
     rackspace-onmetal
-    scaleway
-    stackit
     vagrant
     vagrant-key
     vagrant-virtualbox
@@ -119,9 +107,6 @@ IMG_DEFAULT_FS_HOOK=
 # May be raw, qcow2 (qemu), or vmdk (vmware, virtualbox)
 IMG_DEFAULT_DISK_FORMAT=raw
 
-# Extension to set before the compression extension.
-IMG_DEFAULT_DISK_EXTENSION=
-
 # Name of the partition layout from disk_layout.json
 IMG_DEFAULT_DISK_LAYOUT=base
 
@@ -132,12 +117,19 @@ IMG_DEFAULT_CONF_FORMAT=
 IMG_DEFAULT_BUNDLE_FORMAT=
 
 # Memory size to use in any config files
-IMG_DEFAULT_MEM=2048
+IMG_DEFAULT_MEM=1024
 
 # Number of CPUs to use in any config files
 IMG_DEFAULT_CPUS=2
 
 ## qemu
+IMG_qemu_DISK_FORMAT=qcow2
+IMG_qemu_DISK_LAYOUT=vm
+IMG_qemu_CONF_FORMAT=qemu
+IMG_qemu_OEM_USE=qemu
+IMG_qemu_OEM_PACKAGE=common-oem-files
+IMG_qemu_OEM_SYSEXT=oem-qemu
+
 IMG_qemu_uefi_DISK_FORMAT=qcow2
 IMG_qemu_uefi_DISK_LAYOUT=vm
 IMG_qemu_uefi_CONF_FORMAT=qemu_uefi
@@ -145,6 +137,13 @@ IMG_qemu_uefi_OEM_USE=qemu
 IMG_qemu_uefi_OEM_PACKAGE=common-oem-files
 IMG_qemu_uefi_OEM_SYSEXT=oem-qemu
 
+IMG_qemu_uefi_secure_DISK_FORMAT=qcow2
+IMG_qemu_uefi_secure_DISK_LAYOUT=vm
+IMG_qemu_uefi_secure_CONF_FORMAT=qemu_uefi_secure
+IMG_qemu_uefi_secure_OEM_USE=qemu
+IMG_qemu_uefi_secure_OEM_PACKAGE=common-oem-files
+IMG_qemu_uefi_secure_OEM_SYSEXT=oem-qemu
+
 ## xen
 IMG_xen_CONF_FORMAT=xl
 
@@ -225,11 +224,9 @@ IMG_ami_vmdk_DISK_FORMAT=vmdk_stream
 IMG_ami_vmdk_OEM_USE=ami
 IMG_ami_vmdk_OEM_PACKAGE=common-oem-files
 IMG_ami_vmdk_SYSEXT=oem-ami
-IMG_ami_vmdk_DISK_LAYOUT=vm
 IMG_ami_OEM_USE=ami
 IMG_ami_OEM_PACKAGE=common-oem-files
 IMG_ami_OEM_SYSEXT=oem-ami
-IMG_ami_DISK_LAYOUT=vm
 
 ## openstack
 IMG_openstack_DISK_FORMAT=qcow2
@@ -259,9 +256,8 @@ IMG_iso_MEM=2048
 ## gce, image tarball
 IMG_gce_DISK_LAYOUT=vm
 IMG_gce_CONF_FORMAT=gce
-IMG_gce_OEM_PACKAGE=common-oem-files
+IMG_gce_OEM_PACKAGE=oem-gce
-IMG_gce_OEM_USE=gce
+IMG_gce_OEM_ACI=gce
-IMG_gce_OEM_SYSEXT=oem-gce
 
 ## rackspace
 IMG_rackspace_OEM_PACKAGE=oem-rackspace
@@ -294,23 +290,9 @@ IMG_azure_OEM_USE=azure
 IMG_azure_OEM_PACKAGE=common-oem-files
 IMG_azure_OEM_SYSEXT=oem-azure
 
-## hetzner
-IMG_hetzner_DISK_LAYOUT=vm
-IMG_hetzner_OEM_USE=hetzner
-IMG_hetzner_OEM_PACKAGE=common-oem-files
-IMG_hetzner_OEM_SYSEXT=oem-hetzner
-
 ## hyper-v
 IMG_hyperv_DISK_FORMAT=vhd
-IMG_hyperv_OEM_USE=hyperv
+IMG_hyperv_OEM_PACKAGE=oem-hyperv
-IMG_hyperv_OEM_PACKAGE=common-oem-files
-IMG_hyperv_OEM_SYSEXT=oem-hyperv
-
-## hyper-v vhdx
-IMG_hyperv_vhdx_DISK_FORMAT=vhdx
-IMG_hyperv_vhdx_OEM_USE=hyperv
-IMG_hyperv_vhdx_OEM_PACKAGE=common-oem-files
-IMG_hyperv_vhdx_OEM_SYSEXT=oem-hyperv
 
 ## cloudsigma
 IMG_cloudsigma_DISK_FORMAT=qcow2
@@ -321,49 +303,6 @@ IMG_packet_OEM_PACKAGE=common-oem-files
 IMG_packet_OEM_SYSEXT=oem-packet
 IMG_packet_OEM_USE=packet
 
-## scaleway
-IMG_scaleway_DISK_FORMAT=qcow2
-IMG_scaleway_DISK_LAYOUT=vm
-IMG_scaleway_OEM_PACKAGE=common-oem-files
-IMG_scaleway_OEM_USE=scaleway
-IMG_scaleway_OEM_SYSEXT=oem-scaleway
-IMG_scaleway_DISK_EXTENSION=qcow2
-
-## stackit
-IMG_stackit_DISK_FORMAT=qcow2
-IMG_stackit_DISK_LAYOUT=vm
-IMG_stackit_OEM_PACKAGE=common-oem-files
-IMG_stackit_OEM_USE=stackit
-IMG_stackit_OEM_SYSEXT=oem-stackit
-
-## kubevirt
-IMG_kubevirt_DISK_FORMAT=qcow2
-IMG_kubevirt_DISK_LAYOUT=vm
-IMG_kubevirt_OEM_PACKAGE=common-oem-files
-IMG_kubevirt_OEM_USE=kubevirt
-IMG_kubevirt_OEM_SYSEXT=oem-kubevirt
-IMG_kubevirt_DISK_EXTENSION=qcow2
-
-## akamai (Linode)
-IMG_akamai_DISK_LAYOUT=vm
-IMG_akamai_OEM_PACKAGE=common-oem-files
-IMG_akamai_OEM_USE=akamai
-IMG_akamai_OEM_SYSEXT=oem-akamai
-
-# proxmoxve
-IMG_proxmoxve_DISK_FORMAT=qcow2
-IMG_proxmoxve_DISK_LAYOUT=vm
-IMG_proxmoxve_OEM_PACKAGE=common-oem-files
-IMG_proxmoxve_OEM_USE=proxmoxve
-IMG_proxmoxve_OEM_SYSEXT=oem-proxmoxve
-
-## nutanix
-IMG_nutanix_DISK_FORMAT=qcow2
-IMG_nutanix_DISK_LAYOUT=vm
-IMG_nutanix_OEM_USE=nutanix
-IMG_nutanix_OEM_PACKAGE=common-oem-files
-IMG_nutanix_OEM_SYSEXT=oem-nutanix
-
 ###########################################################
 
 # Print the default vm type for the specified board
@@ -371,7 +310,7 @@ get_default_vm_type() {
     local board="$1"
     case "$board" in
     amd64-usr)
-        echo "qemu_uefi"
+        echo "qemu"
         ;;
     arm64-usr)
         echo "qemu_uefi"
@@ -474,11 +413,6 @@ _dst_path() {
 # Get the proper disk format extension.
 _disk_ext() {
     local disk_format=$(_get_vm_opt DISK_FORMAT)
-    local disk_extension=$(_get_vm_opt DISK_EXTENSION)
-    if [[ -n ${disk_extension} ]]; then
-	echo "${disk_extension}"
-	return 0
-    fi
     case ${disk_format} in
         raw) echo bin;;
         qcow2) echo img;;
@@ -487,9 +421,7 @@ _disk_ext() {
         vmdk_scsi) echo vmdk;;
         vmdk_stream) echo vmdk;;
         hdd) echo hdd;;
-        vhd) echo vhd;;
+        vhd*) echo vhd;;
-        vhd_fixed) echo vhd;;
-        vhdx) echo vhdx;;
         *) echo "${disk_format}";;
     esac
 }
@@ -536,10 +468,7 @@ setup_disk_image() {
 install_oem_package() {
     local oem_pkg=$(_get_vm_opt OEM_PACKAGE)
     local oem_use=$(_get_vm_opt OEM_USE)
-    # The "${VM_IMG_TYPE}-oem-image-rootfs" directory name is
+    local oem_tmp="${VM_TMP_DIR}/oem"
-    # important - it is used to determine the package target in
-    # coreos/base/profile.bashrc
-    local oem_tmp="${VM_TMP_DIR}/${VM_IMG_TYPE}-oem-image-rootfs"
 
     if [[ -z "${oem_pkg}" ]]; then
         return 0
@@ -562,14 +491,43 @@ install_oem_package() {
     info "Installing ${oem_pkg} to OEM partition"
     USE="${oem_use}" emerge-${BOARD} \
         --root="${oem_tmp}" --sysroot="${oem_tmp}" \
-        --usepkgonly ${getbinpkg} \
+        --root-deps=rdeps --usepkgonly ${getbinpkg} \
         --verbose --jobs=2 "${oem_pkg}"
     sudo rsync -a "${oem_tmp}/oem/" "${VM_TMP_ROOT}/oem/"
     sudo rm -rf "${oem_tmp}"
 }
 
-# Install the prebuilt OEM sysext file into the OEM partition.
+# Write the OEM ACI file into the OEM partition.
-# The sysext should have been built by 'build_image oem_sysext'.
+install_oem_aci() {
+    local oem_aci=$(_get_vm_opt OEM_ACI)
+    local aci_dir="${FLAGS_to}/oem-${oem_aci}-aci"
+    local aci_path="${aci_dir}/flatcar-oem-${oem_aci}.aci"
+    local binpkgflags=(--nogetbinpkg)
+
+    [ -n "${oem_aci}" ] || return 0
+
+    [ "${FLAGS_getbinpkg}" = "${FLAGS_TRUE}" ] &&
+    binpkgflags=(--getbinpkg --getbinpkgver="${FLAGS_getbinpkgver}")
+
+    # Build an OEM ACI if necessary, supplying build environment flags.
+    [ -e "${aci_path}" ] &&
+    info "ACI ${aci_path} exists; reusing it" ||
+    "${SCRIPT_ROOT}/build_oem_aci" \
+        --board="${BOARD}" \
+        --build_dir="${aci_dir}" \
+        "${binpkgflags[@]}" \
+        "${oem_aci}"
+
+    info "Installing ${oem_aci} OEM ACI"
+    sudo install -Dpm 0644 \
+        "${aci_path}" \
+        "${VM_TMP_ROOT}/oem/flatcar-oem-${oem_aci}.aci" ||
+    die "Could not install ${oem_aci} OEM ACI"
+    # Remove aci_dir if building ACI and installing it succeeded
+    rm -rf "${aci_dir}"
+}
+
+# Write the OEM sysext file into the OEM partition.
 install_oem_sysext() {
     local oem_sysext=$(_get_vm_opt OEM_SYSEXT)
 
@@ -577,24 +535,54 @@ install_oem_sysext() {
         return 0
     fi
 
-    local prebuilt_sysext_filename="${oem_sysext}.raw"
+    local built_sysext_dir="${FLAGS_to}/${oem_sysext}-sysext"
-    local prebuilt_sysext_path="${FLAGS_from}/${prebuilt_sysext_filename}"
+    local built_sysext_filename="${oem_sysext}.raw"
+    local built_sysext_path="${built_sysext_dir}/${built_sysext_filename}"
     local version="${FLATCAR_VERSION}"
-
+    local metapkg="coreos-base/${oem_sysext}"
-    if [[ ! -f "${prebuilt_sysext_path}" ]]; then
+    local build_sysext_flags=(
-        die "Prebuilt OEM sysext not found at ${prebuilt_sysext_path}. Run 'build_image oem_sysext' first."
+        --board="${BOARD}"
+        --squashfs_base="${VM_SRC_SYSEXT_IMG}"
+        --image_builddir="${built_sysext_dir}"
+        --metapkgs="${metapkg}"
+    )
+    local overlay_path mangle_fs
+    overlay_path=$(portageq get_repo_path / coreos)
+    mangle_fs="${overlay_path}/${metapkg}/files/manglefs.sh"
+    if [[ -x "${mangle_fs}" ]]; then
+        build_sysext_flags+=(
+            --manglefs_script="${mangle_fs}"
+        )
     fi
 
+    mkdir -p "${built_sysext_dir}"
+    sudo "${build_sysext_env[@]}" "${SCRIPT_ROOT}/build_sysext" "${build_sysext_flags[@]}" "${oem_sysext}"
+
     local installed_sysext_oem_dir='/oem/sysext'
     local installed_sysext_file_prefix="${oem_sysext}-${version}"
     local installed_sysext_filename="${installed_sysext_file_prefix}.raw"
     local installed_sysext_abspath="${installed_sysext_oem_dir}/${installed_sysext_filename}"
-
+    info "Installing ${oem_sysext} sysext"
-    info "Installing ${oem_sysext} sysext from prebuilt image"
     sudo install -Dpm 0644 \
-         "${prebuilt_sysext_path}" \
+         "${built_sysext_path}" \
          "${VM_TMP_ROOT}${installed_sysext_abspath}" ||
         die "Could not install ${oem_sysext} sysext"
+    # Move sysext image and reports to a destination directory to
+    # upload them, thus making them available as separate artifacts to
+    # download.
+    local upload_dir to_move
+    upload_dir="$(_dst_dir)"
+    for to_move in "${built_sysext_dir}/${oem_sysext}"*; do
+        mv "${to_move}" "${upload_dir}/${to_move##*/}"
+    done
+    # Generate dev-key-signed update payload for testing
+    delta_generator \
+      -private_key "/usr/share/update_engine/update-payload-key.key.pem" \
+      -new_image "${upload_dir}/${built_sysext_filename}" \
+      -out_file "${upload_dir}/flatcar_test_update-${oem_sysext}.gz"
+    # Remove sysext_dir if building sysext and installing it
+    # succeeded.
+    rm -rf "${built_sysext_dir}"
 
     # Mark the installed sysext as active.
     sudo touch "${VM_TMP_ROOT}${installed_sysext_oem_dir}/active-${oem_sysext}"
@@ -627,21 +615,6 @@ write_vm_disk() {
     info "Writing $disk_format image $(basename "${VM_DST_IMG}")"
     _write_${disk_format}_disk "${VM_TMP_IMG}" "${VM_DST_IMG}"
 
-    # We now only support building qemu_uefi and set up symlinks
-    # for the qemu and qemu_uefi_secure images
-    if [ "${VM_IMG_TYPE}" = qemu_uefi ]; then
-        local qemu="${VM_DST_IMG/qemu_uefi/qemu}"
-        local qemu_uefi_secure="${VM_DST_IMG/qemu_uefi/qemu_uefi_secure}"
-        local target_basename
-        target_basename=$(basename "${VM_DST_IMG}")
-        if [ "${BOARD}" = amd64-usr ]; then
-          ln -fs "${target_basename}" "${qemu}"
-          VM_GENERATED_FILES+=( "${qemu}" )
-        fi
-        ln -fs "${target_basename}" "${qemu_uefi_secure}"
-        VM_GENERATED_FILES+=( "${qemu_uefi_secure}" )
-    fi
-
     # Add disk image to final file list if it isn't going to be bundled
     if [[ -z "$(_get_vm_opt BUNDLE_FORMAT)" ]]; then
         VM_GENERATED_FILES+=( "${VM_DST_IMG}" )
@@ -667,11 +640,6 @@ _write_vhd_fixed_disk() {
     assert_image_size "$2" vpc
 }
 
-_write_vhdx_disk() {
-    qemu-img convert -f raw "$1" -O vhdx -o subformat=dynamic "$2"
-    assert_image_size "$2" vhdx
-}
-
 _write_vmdk_ide_disk() {
     qemu-img convert -f raw "$1" -O vmdk -o adapter_type=ide "$2"
     assert_image_size "$2" vmdk
@@ -711,23 +679,13 @@ _write_cpio_common() {
     echo "/.noupdate f 444 root root echo -n" >"${VM_TMP_DIR}/extra"
 
     # Set correct group for PXE/ISO, which has no writeable /etc
-    echo /share/flatcar/update.conf f 644 root root \
+    echo /usr/share/flatcar/update.conf f 644 root root \
         "sed -e 's/GROUP=.*$/GROUP=${VM_GROUP}/' ${base_dir}/share/flatcar/update.conf" \
         >> "${VM_TMP_DIR}/extra"
 
-    local -a mksquashfs_opts=(
-        -pf "${VM_TMP_DIR}/extra"
-        -xattrs-exclude '^btrfs.'
-        # mksquashfs doesn't like overwriting existing files with
-        # pseudo-files, so tell it to ignore the existing file instead
-        #
-        # also, this must be the last option
-        -e share/flatcar/update.conf
-    )
-
     # Build the squashfs, embed squashfs into a gzipped cpio
     pushd "${cpio_target}" >/dev/null
-    sudo mksquashfs "${base_dir}" "./usr.squashfs" "${mksquashfs_opts[@]}"
+    sudo mksquashfs "${base_dir}" "./usr.squashfs" -pf "${VM_TMP_DIR}/extra"
     find . | cpio -o -H newc | gzip > "$2"
     popd >/dev/null
 
@@ -742,15 +700,15 @@ _write_cpio_disk() {
     local grub_name="$(_dst_name "_grub.efi")"
     _write_cpio_common $@
     # Pull the kernel and loader out of the filesystem
-    ln -fs flatcar_production_image.vmlinuz "${dst_dir}/${vmlinuz_name}"
+    cp "${base_dir}"/boot/flatcar/vmlinuz-a "${dst_dir}/${vmlinuz_name}"
 
-    local efi_file
+    local grub_arch
     case $BOARD in
-        amd64-usr) efi_file="grubx64.efi" ;;
+        amd64-usr) grub_arch="x86_64-efi" ;;
-        arm64-usr) efi_file="bootaa64.efi" ;;
+        arm64-usr) grub_arch="arm64-efi" ;;
     esac
 
-    cp "${base_dir}/boot/EFI/boot/${efi_file}" "${dst_dir}/${grub_name}"
+    cp "${base_dir}/boot/flatcar/grub/${grub_arch}/core.efi" "${dst_dir}/${grub_name}"
     VM_GENERATED_FILES+=( "${dst_dir}/${vmlinuz_name}" "${dst_dir}/${grub_name}" )
 }
 
@@ -801,17 +759,18 @@ _write_qemu_common() {
         -e "s%^VM_MEMORY=.*%VM_MEMORY='${vm_mem}'%" \
         -e "s%^VM_BOARD=.*%VM_BOARD='${BOARD}'%" \
         "${BUILD_LIBRARY_DIR}/qemu_template.sh" > "${script}"
+    checkbashisms --posix "${script}" || die
     chmod +x "${script}"
 
     cat >"${VM_README}" <<EOF
 If you have qemu installed (or in the SDK), you can start the image with:
   cd path/to/image
-  ./$(basename "${script}") -display curses
+  ./$(basename "${script}") -curses
 
 If you need to use a different ssh key or different ssh port:
-  ./$(basename "${script}") -a ~/.ssh/authorized_keys -p 2223 -- -display curses
+  ./$(basename "${script}") -a ~/.ssh/authorized_keys -p 2223 -- -curses
 
-If you rather you can use the -nographic option instad of '-display curses'. In this
+If you rather you can use the -nographic option instad of -curses. In this
 mode you can switch from the vm to the qemu monitor console with: Ctrl-a c
 See the qemu man page for more details on the monitor console.
 
@@ -827,82 +786,52 @@ _write_qemu_conf() {
     local dst_name=$(basename "$VM_DST_IMG")
 
     _write_qemu_common "${script}"
-    sed -e "s%^VM_IMAGE=.*%VM_IMAGE=\"\${SCRIPT_DIR}/${dst_name}\"%" -i "${script}"
+    sed -e "s%^VM_IMAGE=.*%VM_IMAGE='${dst_name}'%" -i "${script}"
 }
 
 _write_qemu_uefi_conf() {
-    local flash_ro="$(_dst_name "_efi_code.qcow2")"
-    local flash_rw="$(_dst_name "_efi_vars.qcow2")"
     local script="$(_dst_dir)/$(_dst_name ".sh")"
 
     _write_qemu_conf
 
+    local flash_ro="$(_dst_name "_efi_code.fd")"
+    local flash_rw="$(_dst_name "_efi_vars.fd")"
+
     case $BOARD in
         amd64-usr)
-            cp "/usr/share/edk2/OvmfX64/OVMF_CODE_4M.qcow2" "$(_dst_dir)/${flash_ro}"
+            cp "/usr/share/edk2-ovmf/OVMF_CODE.fd" "$(_dst_dir)/${flash_ro}"
-            cp "/usr/share/edk2/OvmfX64/OVMF_VARS_4M.qcow2" "$(_dst_dir)/${flash_rw}"
+            cp "/usr/share/edk2-ovmf/OVMF_VARS.fd" "$(_dst_dir)/${flash_rw}"
             ;;
         arm64-usr)
-            cp "/usr/share/edk2/ArmVirtQemu-AARCH64/QEMU_EFI.qcow2" "$(_dst_dir)/${flash_ro}"
+            # Get edk2 files into local build workspace.
-            cp "/usr/share/edk2/ArmVirtQemu-AARCH64/QEMU_VARS.qcow2" "$(_dst_dir)/${flash_rw}"
+            info "Updating edk2 in /build/${BOARD}"
+            emerge-${BOARD} --nodeps --select --verbose --update --getbinpkg --newuse sys-firmware/edk2-aarch64
+            # Create 64MiB flash device image files.
+            dd if=/dev/zero bs=1M count=64 of="$(_dst_dir)/${flash_rw}" \
+                status=none
+            cp "/build/${BOARD}/usr/share/edk2-aarch64/QEMU_EFI.fd" \
+                "$(_dst_dir)/${flash_ro}.work"
+            truncate --reference="$(_dst_dir)/${flash_rw}" \
+                "$(_dst_dir)/${flash_ro}.work"
+            mv "$(_dst_dir)/${flash_ro}.work" "$(_dst_dir)/${flash_ro}"
             ;;
     esac
 
-    sed -e "s%^VM_PFLASH_RO=.*%VM_PFLASH_RO=\"\${SCRIPT_DIR}/${flash_ro}\"%" \
+    sed -e "s%^VM_PFLASH_RO=.*%VM_PFLASH_RO='${flash_ro}'%" \
-        -e "s%^VM_PFLASH_RW=.*%VM_PFLASH_RW=\"\${SCRIPT_DIR}/${flash_rw}\"%" -i "${script}"
+        -e "s%^VM_PFLASH_RW=.*%VM_PFLASH_RW='${flash_rw}'%" -i "${script}"
     VM_GENERATED_FILES+=( "$(_dst_dir)/${flash_ro}" "$(_dst_dir)/${flash_rw}" )
-
-    # We now only support building qemu_uefi and generate the
-    # other artifacts from here
-    if [ "${VM_IMG_TYPE}" = qemu_uefi ]; then
-      local qemu="${VM_DST_IMG/qemu_uefi/qemu}"
-      local qemu_uefi_secure="${VM_DST_IMG/qemu_uefi/qemu_uefi_secure}"
-      local qemu_name="${VM_NAME/qemu_uefi/qemu}"
-      local qemu_uefi_secure_name="${VM_NAME/qemu_uefi/qemu_uefi_secure}"
-      if [ "${BOARD}" = amd64-usr ]; then
-        VM_IMG_TYPE=qemu VM_DST_IMG="${qemu}" VM_NAME="${qemu_name}" _write_qemu_conf
-      fi
-      VM_IMG_TYPE=qemu_uefi_secure VM_DST_IMG="${qemu_uefi_secure}" VM_NAME="${qemu_uefi_secure_name}" _write_qemu_uefi_secure_conf
-    fi
 }
 
 _write_qemu_uefi_secure_conf() {
-    local flash_rw="$(_dst_name "_efi_vars.qcow2")"
+    local flash_rw="$(_dst_name "_efi_vars.fd")"
-    local flash_ro="$(_dst_name "_efi_code.qcow2")"
-    local script="$(_dst_dir)/$(_dst_name ".sh")"
-    local owner="00000000-0000-0000-0000-000000000000"
-    local flash_in
 
     _write_qemu_uefi_conf
-
+    cert-to-efi-sig-list "/usr/share/sb_keys/PK.crt" "${VM_TMP_DIR}/PK.esl"
-    case $BOARD in
+    cert-to-efi-sig-list "/usr/share/sb_keys/KEK.crt" "${VM_TMP_DIR}/KEK.esl"
-        amd64-usr)
+    cert-to-efi-sig-list "/usr/share/sb_keys/DB.crt" "${VM_TMP_DIR}/DB.esl"
-            cp "/usr/share/edk2/OvmfX64/OVMF_CODE_4M.secboot.qcow2" "$(_dst_dir)/${flash_ro}"
+    flash-var "$(_dst_dir)/${flash_rw}" "PK" "${VM_TMP_DIR}/PK.esl"
-            flash_in="/usr/share/edk2/OvmfX64/OVMF_VARS_4M.secboot.qcow2"
+    flash-var "$(_dst_dir)/${flash_rw}" "KEK" "${VM_TMP_DIR}/KEK.esl"
-            ;;
+    flash-var "$(_dst_dir)/${flash_rw}" "db" "${VM_TMP_DIR}/DB.esl"
-        arm64-usr)
-            # This firmware is not considered secure due to the lack of an SMM
-            # implementation, which is needed to protect the variable store, but
-            # it's only supposed to be used for testing anyway.
-            cp "/usr/share/edk2/ArmVirtQemu-AARCH64/QEMU_EFI.secboot_INSECURE.qcow2" "$(_dst_dir)/${flash_ro}"
-            flash_in="/usr/share/edk2/ArmVirtQemu-AARCH64/QEMU_VARS.secboot_INSECURE.qcow2"
-            ;;
-    esac
-
-    # TODO: Remove the temporary flatcar shim signing cert
-    local _sb_db_cert="${SBSIGN_DB_CERT:-/usr/share/sb_keys/DB.crt}"
-    local _sb_extra_db_certs=()
-    if [[ -z ${SBSIGN_DB_CERT:-} ]]; then
-        # Default behavior: include the temporary dev shim cert alongside DB.crt
-        _sb_extra_db_certs=( --add-db "${owner}" "${BUILD_LIBRARY_DIR}/flatcar-sb-dev-shim-2025.cert" )
-    fi
-    virt-fw-vars \
-        --input "${flash_in}" \
-        --output "$(_dst_dir)/${flash_rw}" \
-        --add-db "${owner}" "${_sb_db_cert}" \
-        "${_sb_extra_db_certs[@]}"
-
-    sed -e "s%^SECURE_BOOT=.*%SECURE_BOOT=1%" -i "${script}"
 }
 
 _write_pxe_conf() {
@@ -911,13 +840,13 @@ _write_pxe_conf() {
     local dst_name=$(basename "$VM_DST_IMG")
 
     _write_qemu_common "${script}"
-    sed -e "s%^VM_KERNEL=.*%VM_KERNEL=\"\${SCRIPT_DIR}/${vmlinuz_name}\"%" \
+    sed -e "s%^VM_KERNEL=.*%VM_KERNEL='${vmlinuz_name}'%" \
-        -e "s%^VM_INITRD=.*%VM_INITRD=\"\${SCRIPT_DIR}/${dst_name}\"%" -i "${script}"
+        -e "s%^VM_INITRD=.*%VM_INITRD='${dst_name}'%" -i "${script}"
 
     cat >>"${VM_README}" <<EOF
 
 You can pass extra kernel parameters with -append, for example:
-  ./$(basename "${script}") -display curses -append 'sshkey="PUT AN SSH KEY HERE"'
+  ./$(basename "${script}") -curses -append 'sshkey="PUT AN SSH KEY HERE"'
 
 When using -nographic or -serial you must also enable the serial console:
   ./$(basename "${script}") -nographic -append 'console=ttyS0,115200n8'
@@ -937,7 +866,7 @@ _write_iso_conf() {
     local script="$(_dst_dir)/$(_dst_name ".sh")"
     local dst_name=$(basename "$VM_DST_IMG")
     _write_qemu_common "${script}"
-    sed -e "s%^VM_CDROM=.*%VM_CDROM=\"\${SCRIPT_DIR}/${dst_name}\"%" -i "${script}"
+    sed -e "s%^VM_CDROM=.*%VM_CDROM='${dst_name}'%" -i "${script}"
 }
 
 # Generate the vmware config file
@@ -1238,7 +1167,7 @@ EOF
     "version": "${FLATCAR_VERSION_ID}",
     "providers": [{
       "name": "${provider}",
-      "url": "https://${BUILDCACHE_SERVER:-bincache.flatcar-linux.net}/images/${BOARD%-usr}/${FLATCAR_VERSION}/$(_dst_name ".box")",
+      "url": "$(download_image_url "$(_dst_name ".box")")",
       "checksum_type": "sha256",
       "checksum": "$(sha256sum "${box}" | awk '{print $1}')"
     }]
@@ -1285,6 +1214,53 @@ vm_cleanup() {
     sudo rm -rf "${VM_TMP_DIR}"
 }
 
+vm_upload() {
+
+    declare -a legacy_uploads
+    declare -a uploadable_files
+    declare -a compressed_images
+    declare -a image_files
+    declare -a digest_uploads
+
+    compress_disk_images VM_GENERATED_FILES compressed_images uploadable_files
+
+    if [ "${#compressed_images[@]}" -gt 0 ]; then
+        uploadable_files+=( "${compressed_images[@]}" )
+        legacy_uploads+=( "${compressed_images[@]}" )
+    fi
+
+    local digests="$(_dst_dir)/$(_dst_name .DIGESTS)"
+    upload_image -d "${digests}" "${uploadable_files[@]}"
+
+    [[ -e "${digests}" ]] || return 0
+
+    # Since depending on the ordering of $VM_GENERATED_FILES is brittle only
+    # use it if $VM_DST_IMG isn't included in the uploaded files.
+    if [ "${#legacy_uploads[@]}" -eq 0 ];then
+        legacy_uploads+=( "${VM_GENERATED_FILES[0]}" )
+    fi
+
+    for legacy_upload in "${legacy_uploads[@]}";do
+        local legacy_digest_file="${legacy_upload}.DIGESTS"
+        [[ "${legacy_digest_file}" == "${digests}" ]] && continue
+
+        cp "${digests}" "${legacy_digest_file}"
+        digest_uploads+=( "${legacy_digest_file}" )
+
+        if [[ -e "${digests}.asc" ]]; then
+            digest_uploads+=( "${legacy_digest_file}.asc" )
+            cp "${digests}.asc" "${legacy_digest_file}.asc"
+        fi
+    done
+
+    if [ "${#digest_uploads[@]}" -gt 0 ];then
+        legacy_uploads+=( "${digest_uploads[@]}" )
+    fi
+
+    local def_upload_path="${UPLOAD_ROOT}/boards/${BOARD}/${FLATCAR_VERSION}"
+    upload_files "$(_dst_name)" "${def_upload_path}" "" "${legacy_uploads[@]}"
+}
+
 print_readme() {
     local filename
     info "Files written to $(relpath "$(dirname "${VM_DST_IMG}")")"

build_oem_aci

@@ -0,0 +1,78 @@
+#!/bin/bash
+
+# Copyright (c) 2016 The CoreOS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# This is a wrapper around the oem_aci_util.sh functions to set up the
+# necessary environment, similar to the build_image script.
+
+SCRIPT_ROOT=$(dirname $(readlink -f "$0"))
+. "${SCRIPT_ROOT}/common.sh" || exit 1
+
+# Script must run inside the chroot
+assert_inside_chroot
+
+assert_not_root_user
+
+# Developer-visible flags.
+DEFINE_string board "${DEFAULT_BOARD}" \
+  "The board to build an image for."
+DEFINE_string build_dir "" \
+  "Directory in which to place image result directories (named by version)"
+DEFINE_boolean getbinpkg "${FLAGS_FALSE}" \
+  "Download binary packages from remote repository."
+DEFINE_string getbinpkgver "" \
+  "Use binary packages from a specific version."
+
+FLAGS_HELP="USAGE: build_oem_aci [flags] [oem name].
+This script is used to build a CoreOS OEM ACI.
+
+Examples:
+
+build_oem_aci --board=amd64-usr --build_dir=<build_dir> gce
+...
+"
+show_help_if_requested "$@"
+
+# The following options are advanced options, only available to those willing
+# to read the source code. They are not shown in help output, since they are
+# not needed for the typical developer workflow.
+DEFINE_integer build_attempt 1 \
+  "The build attempt for this image build."
+DEFINE_string group "oem-aci" \
+  "The update group (not used for actual updates here)"
+DEFINE_string output_root "${DEFAULT_BUILD_ROOT}/images" \
+  "Directory in which to place image result directories (named by version)"
+DEFINE_string version "" \
+  "Overrides version number in name to this version."
+
+# Parse command line.
+FLAGS "$@" || exit 1
+[ -z "${FLAGS_ARGV}" ] && echo 'No OEM given' && exit 0
+eval set -- "${FLAGS_ARGV}"
+
+# Only now can we die on error.  shflags functions leak non-zero error codes,
+# so will die prematurely if 'switch_to_strict_mode' is specified before now.
+switch_to_strict_mode
+
+# If downloading packages is enabled ensure the board is configured properly.
+if [[ ${FLAGS_getbinpkg} -eq ${FLAGS_TRUE} ]]; then
+    "${SRC_ROOT}/scripts/setup_board" --board="${FLAGS_board}" \
+      --getbinpkgver="${FLAGS_getbinpkgver}" --regen_configs_only
+fi
+
+# N.B.  Ordering matters for some of the libraries below, because
+# some of the files contain initialization used by later files.
+. "${BUILD_LIBRARY_DIR}/toolchain_util.sh" || exit 1
+. "${BUILD_LIBRARY_DIR}/board_options.sh" || exit 1
+. "${BUILD_LIBRARY_DIR}/build_image_util.sh" || exit 1
+. "${BUILD_LIBRARY_DIR}/prod_image_util.sh" || exit 1
+. "${BUILD_LIBRARY_DIR}/test_image_content.sh" || exit 1
+. "${BUILD_LIBRARY_DIR}/oem_aci_util.sh" || exit 1
+
+BUILD_DIR=${FLAGS_build_dir:-$BUILD_DIR}
+
+for oem
+do oem_aci_create "${oem}"
+done

build_packages

@@ -24,8 +24,10 @@ DEFINE_boolean getbinpkg "${FLAGS_TRUE}" \
   "Download binary packages from remote repository."
 DEFINE_string getbinpkgver "" \
   "Use binary packages from a specific version."
+DEFINE_boolean toolchainpkgonly "${FLAGS_FALSE}" \
+  "Use binary packages only for the board toolchain."
 DEFINE_boolean workon "${FLAGS_TRUE}" \
-  "Automatically rebuild updated flatcar-workon packages."
+  "Automatically rebuild updated cros-workon packages."
 DEFINE_boolean fetchonly "${FLAGS_FALSE}" \
   "Don't build anything, instead only fetch what is needed."
 DEFINE_boolean rebuild "${FLAGS_FALSE}" \
@@ -34,10 +36,12 @@ DEFINE_boolean skip_toolchain_update "${FLAGS_FALSE}" \
   "Don't update toolchain automatically."
 DEFINE_boolean skip_chroot_upgrade "${FLAGS_FALSE}" \
   "Don't run the chroot upgrade automatically; use with care."
-DEFINE_boolean only_resolve_circular_deps "${FLAGS_FALSE}" \
+DEFINE_string torcx_output_root "${DEFAULT_BUILD_ROOT}/torcx" \
-  "Don't build all packages; only resolve circular dependencies, then stop."
+  "Directory in which to place torcx stores and manifests (named by board/version)"
-DEFINE_boolean debug_emerge "${FLAGS_FALSE}" \
+DEFINE_boolean skip_torcx_store "${FLAGS_FALSE}" \
-  "Enable debug output for emerge."
+  "Don't build a new torcx store from the updated sysroot."
+DEFINE_string torcx_extra_pkg_url "" \
+  "URL to directory where the torcx packages will be available for downloading"
 
 # include upload options
 . "${BUILD_LIBRARY_DIR}/release_util.sh" || exit 1
@@ -84,6 +88,8 @@ if [[ "${FLAGS_usepkgonly}" -eq "${FLAGS_TRUE}" ]]; then
   FLAGS_workon="${FLAGS_FALSE}"
 fi
 
+check_gsutil_opts
+
 # Before we can run any tools, we need to update chroot or setup_board.
 UPDATE_ARGS=( --regen_configs )
 if [ "${FLAGS_usepkg}" -eq "${FLAGS_TRUE}" ]; then
@@ -98,6 +104,11 @@ if [ "${FLAGS_usepkg}" -eq "${FLAGS_TRUE}" ]; then
   else
     UPDATE_ARGS+=( --nogetbinpkg )
   fi
+  if [[ "${FLAGS_toolchainpkgonly}" -eq "${FLAGS_TRUE}" ]]; then
+    UPDATE_ARGS+=( --toolchainpkgonly )
+  else
+    UPDATE_ARGS+=( --notoolchainpkgonly )
+  fi
   if [[ -n "${FLAGS_getbinpkgver}" ]]; then
     UPDATE_ARGS+=( --getbinpkgver="${FLAGS_getbinpkgver}" )
   fi
@@ -117,8 +128,6 @@ fi
 . "${BUILD_LIBRARY_DIR}/toolchain_util.sh" || exit 1
 . "${BUILD_LIBRARY_DIR}/board_options.sh" || exit 1
 . "${BUILD_LIBRARY_DIR}/test_image_content.sh" || exit 1
-. "${BUILD_LIBRARY_DIR}/extra_sysexts.sh" || exit 1
-. "${BUILD_LIBRARY_DIR}/oem_sysexts.sh" || exit 1
 
 # Setup all the emerge command/flags.
 EMERGE_FLAGS=( --update --deep --newuse --verbose --backtrack=30 --select )
@@ -158,20 +167,16 @@ if [[ "${FLAGS_rebuild}" -eq "${FLAGS_TRUE}" ]]; then
   EMERGE_FLAGS+=( --rebuild-if-unbuilt )
 fi
 
-if [[ "${FLAGS_debug_emerge}" -eq "${FLAGS_TRUE}" ]]; then
+# Build cros_workon packages when they are changed.
-  EMERGE_FLAGS+=( --debug )
+CROS_WORKON_PKGS=()
+if [ "${FLAGS_workon}" -eq "${FLAGS_TRUE}" ]; then
+  CROS_WORKON_PKGS+=( $("${SRC_ROOT}/scripts/cros_workon" list --board=${FLAGS_board}) )
 fi
 
-# Build flatcar_workon packages when they are changed.
+if [[ ${#CROS_WORKON_PKGS[@]} -gt 0 ]]; then
-WORKON_PKGS=()
-if [[ ${FLAGS_workon} -eq "${FLAGS_TRUE}" ]]; then
-  mapfile -t WORKON_PKGS < <("${SRC_ROOT}"/scripts/flatcar_workon list --board="${FLAGS_board}")
-fi
-
-if [[ ${#WORKON_PKGS[@]} -gt 0 ]]; then
   EMERGE_FLAGS+=(
-    --reinstall-atoms="${WORKON_PKGS[*]}"
+    --reinstall-atoms="${CROS_WORKON_PKGS[*]}"
-    --usepkg-exclude="${WORKON_PKGS[*]}"
+    --usepkg-exclude="${CROS_WORKON_PKGS[*]}"
   )
 fi
 
@@ -267,80 +272,18 @@ if [[ "${FLAGS_usepkgonly}" -eq "${FLAGS_FALSE}" ]]; then
   # lvm2[udev] -> virtual/udev -> systemd[cryptsetup] -> cryptsetup -> lvm2
   # lvm2[systemd] -> systemd[cryptsetup] -> cryptsetup -> lvm2
   # systemd[cryptsetup] -> cryptsetup[udev] -> virtual/udev -> systemd
-  # systemd[tpm] -> tpm2-tss -> util-linux[udev] -> virtual/udev -> systemd
+  break_dep_loop sys-apps/util-linux udev,systemd,cryptsetup \
-  # curl[http2] -> nghttp2[systemd] -> systemd[curl] -> curl
-  # sys-libs/pam[systemd] -> sys-apps/systemd[pam] -> sys-libs/pam
-  #   dropping USE=pam from sys-apps/systemd requires dropping
-  #   USE=systemd from sys-auth/pambase
-  # sys-auth/pambase[sssd] -> sys-auth/sssd -> sys-apps/shadow[pam] -> sys-auth/pambase
-  break_dep_loop sys-apps/util-linux cryptsetup,systemd,udev \
                  sys-fs/cryptsetup udev \
-                 sys-fs/lvm2 systemd,udev \
+                 sys-fs/lvm2 udev,systemd \
-                 sys-apps/systemd cryptsetup,pam,tpm \
+                 sys-apps/systemd cryptsetup
-                 net-misc/curl http2 \
-                 net-libs/nghttp2 systemd \
-                 sys-libs/pam systemd \
-                 sys-auth/pambase sssd,systemd
-fi
-
-if [[ "${FLAGS_only_resolve_circular_deps}" -eq "${FLAGS_TRUE}" ]]; then
-  info "Circular dependencies resolved. Stopping as requested."
-  exit
 fi
 
 export KBUILD_BUILD_USER="${BUILD_USER:-build}"
 export KBUILD_BUILD_HOST="${BUILD_HOST:-pony-truck.infra.kinvolk.io}"
 
-# Build sysext packages from an array of sysext definitions.
-# Usage: build_sysext_packages "description" "${SYSEXT_ARRAY[@]}"
-# Array format: "name|packages|useflags|arches"
-build_sysext_packages() {
-  local description="$1"
-  shift
-  local sysexts=("$@")
-
-  info "Merging ${description} packages now"
-  for sysext in "${sysexts[@]}"; do
-    local sysext_name package_atoms useflags arches
-    IFS="|" read -r sysext_name package_atoms useflags arches <<< "$sysext"
-    [[ -z ${arches} || ,${arches}, == *,"${ARCH}",* ]] || continue
-
-    info "Building packages for $sysext_name sysext with USE=$useflags"
-    IFS=,
-    for package in $package_atoms; do
-       # --buildpkgonly does not install dependencies, so we install them
-       # separately before building the binary package
-       sudo --preserve-env=MODULES_SIGN_KEY,MODULES_SIGN_CERT \
-         env USE="$useflags" FEATURES="-ebuild-locks binpkg-multi-instance" "${EMERGE_CMD[@]}" \
-         "${EMERGE_FLAGS[@]}" \
-         --quiet \
-         --onlydeps \
-         --binpkg-respect-use=y \
-         "${package}"
-
-       sudo --preserve-env=MODULES_SIGN_KEY,MODULES_SIGN_CERT \
-         env USE="$useflags" FEATURES="-ebuild-locks binpkg-multi-instance" "${EMERGE_CMD[@]}" \
-         "${EMERGE_FLAGS[@]}" \
-         --quiet \
-         --buildpkgonly \
-         --binpkg-respect-use=y \
-         "${package}"
-    done
-    unset IFS
-  done
-}
-
 info "Merging board packages now"
 sudo -E "${EMERGE_CMD[@]}" "${EMERGE_FLAGS[@]}" "$@"
 
-build_sysext_packages "extra sysexts" "${EXTRA_SYSEXTS[@]}"
-
-declare -a oem_sysexts
-get_oem_sysext_matrix "${ARCH}" oem_sysexts
-if [[ ${#oem_sysexts[@]} -gt 0 ]]; then
-  build_sysext_packages "OEM sysexts" "${oem_sysexts[@]}"
-fi
-
 info "Removing obsolete packages"
 # The return value of emerge is not clearly reliable. It may fail with
 # an output like following:
@@ -375,19 +318,21 @@ if [[ "${FLAGS_usepkgonly}" -eq "${FLAGS_FALSE}" ]]; then
   fi
 fi
 
-exclusions_file=$(mktemp)
+eclean-$BOARD -d packages
-if [ ! -f "$exclusions_file" ]; then
-    die_notrace "Couldn't create temporary exclusions file $exclusions_file for eclean"
-fi
-get_unversioned_sysext_packages > "$exclusions_file"
-eclean-"$BOARD" -d --exclude-file="$exclusions_file" packages
-rm -f "$exclusions_file"
-# run eclean again, this time without the --deep option, to clean old versions
-# of sysext packages (those, for which .ebuild file no  longer exists)
-eclean-"$BOARD" packages
 
 info "Checking build root"
 test_image_content "${BOARD_ROOT}"
 
+# upload packages if enabled
+upload_packages
+
+# Build a new torcx store with the updated packages, passing flags through.
+if [ "${FLAGS_skip_torcx_store}" -eq "${FLAGS_FALSE}" ]; then
+  "${SCRIPTS_DIR}"/build_torcx_store \
+                  --board="${BOARD}" \
+                  --output_root="${FLAGS_torcx_output_root}" \
+                  --extra_pkg_url="${FLAGS_torcx_extra_pkg_url}"
+fi
+
 info "Builds complete"
 command_completed

build_sdk_container_image

@@ -137,7 +137,7 @@ else
     if [ -n "$cleanup" ] ; then
         echo "$docker image rm -f '${import_image}'" >> "$cleanup"
     fi
-    docker_build -t "$import_image" \
+    $docker build -t "$import_image" \
                  --build-arg VERSION="${docker_vernum}" \
                  -f sdk_lib/Dockerfile.sdk-import \
                  .
@@ -208,7 +208,7 @@ else
     if [ -n "$cleanup" ] ; then
         echo "$docker image rm -f '${sdk_build_image}'" >> "$cleanup"
     fi
-    docker_build -t "${sdk_build_image}" \
+    $docker build -t "${sdk_build_image}" \
                  --build-arg VERSION="${docker_vernum}" \
                  --build-arg BINHOST="http://${binhost}" \
                  --build-arg OFFICIAL="${official}" \
@@ -231,7 +231,7 @@ for a in all arm64 amd64; do
         arm64) rmarch="amd64-usr"; rmcross="x86_64-cros-linux-gnu";;
         amd64) rmarch="arm64-usr"; rmcross="aarch64-cros-linux-gnu";;
     esac
-    docker_build -t "$sdk_container_common_registry/flatcar-sdk-${a}:${docker_vernum}" \
+    $docker build -t "$sdk_container_common_registry/flatcar-sdk-${a}:${docker_vernum}" \
                  --build-arg VERSION="${docker_vernum}" \
                  --build-arg RMARCH="${rmarch}" \
                  --build-arg RMCROSS="${rmcross}" \

build_sysext

@@ -7,7 +7,6 @@
 # Script to generate sysext. See systemd-sysext(8). Prerequisite is
 # that you've run build_packages and build_image.
 
-
 SCRIPT_ROOT=$(dirname "$(readlink -f "$0")")
 . "${SCRIPT_ROOT}/common.sh" || exit 1
 
@@ -16,7 +15,6 @@ assert_inside_chroot
 assert_root_user
 
 default_imagedir="$(readlink -f "${SCRIPT_ROOT}/../build/images")/<BOARD>/latest/"
-default_install_root_basename='install-root'
 
 # All these are used to set up the 'BUILD_DIR' variable
 DEFINE_string board "${DEFAULT_BOARD}" \
@@ -27,22 +25,10 @@ DEFINE_string squashfs_base '' \
   "The path to the squashfs base image. Defaults to the most current image built in '${default_imagedir}/${FLATCAR_PRODUCTION_IMAGE_SYSEXT_BASE}'."
 DEFINE_string image_builddir '' \
   "Custom directory to build the sysext in. Defaults to a 'sysext' sub-directory of the directory the squashfs base image resides in; '${default_imagedir}/sysext' by default."
-DEFINE_boolean strip_binaries "${FLAGS_FALSE}" \
-  "After installation, scan sysext root for unstripped binaries and strip these. WARNING - this can subtly break some packages, e.g. Docker (see https://github.com/moby/moby/blob/master/project/PACKAGERS.md#stripping-binaries)."
 DEFINE_string manglefs_script '' \
   "A path to executable that will customize the rootfs of the sysext image."
-DEFINE_boolean generate_pkginfo "${FLAGS_FALSE}" \
-  "Generate an additional squashfs '<sysext_name>_pkginfo.raw' with portage package meta-information (/var/db ...). Useful for creating sysext dependencies; see 'base_pkginfo' below."
-DEFINE_string base_pkginfo "" \
-  "Colon-separated list of pkginfo squashfs paths / files generated via 'generate_pkginfo' to base this sysext on. The corresponding base sysexts are expected to be merged with the sysext generated."
-DEFINE_string compression "lz4hc" \
-  "Compression to use for sysext EROFS image. Options: 'lz4', 'lz4hc', 'zstd', or 'none'. Default is 'lz4hc'."
-DEFINE_string mkerofs_opts "" \
-  "Additional mkfs.erofs options to pass via SYSTEMD_REPART_MKFS_OPTIONS_EROFS. If not specified, defaults are used based on compression type."
 DEFINE_boolean ignore_version_mismatch "${FLAGS_FALSE}" \
   "Ignore version mismatch between SDK board packages and base squashfs. DANGEROUS."
-DEFINE_string install_root_basename "${default_install_root_basename}" \
-  "Name of a root directory where packages will be installed. ${default_install_root_basename@Q} by default."
 
 FLAGS_HELP="USAGE: build_sysext [flags] <sysext_name> <binary_package> [<binary_package> ...]
 
@@ -87,10 +73,6 @@ FLAGS "$@" || exit 1
 
 eval set -- "${FLAGS_ARGV}"
 
-# Only now can we die on error.  shflags functions leak non-zero error codes,
-# so will die prematurely if 'switch_to_strict_mode' is specified before now.
-switch_to_strict_mode -uo pipefail
-
 # Validate command line parameters
 
 SYSEXTNAME="${1:-}"
@@ -133,20 +115,17 @@ _get_sysext_arch() {
     fi
 }
 
+set -euo pipefail
+
 cleanup() {
   local dirs=(
     "${BUILD_DIR}/fs-root"
-    "${BUILD_DIR}/${FLAGS_install_root_basename}"
+    "${BUILD_DIR}/install-root"
     "${BUILD_DIR}/workdir"
     "${BUILD_DIR}/img-rootfs"
   )
   umount "${dirs[@]}" 2>/dev/null || true
   rm -rf "${dirs[@]}" || true
-  if [[ -d "${BUILD_DIR}/base-pkginfo" ]] ; then
-    umount "${BUILD_DIR}/base-pkginfo"/* 2>/dev/null || true
-    rm -rf "${BUILD_DIR}/base-pkginfo" || true
-  fi
-  rm -rf "${BUILD_DIR}/img-pkginfo"
 }
 
 # Set up trap to execute cleanup() on script exit
@@ -155,46 +134,12 @@ trap cleanup EXIT
 ARCH=$(_get_sysext_arch "${FLAGS_board}")
 cleanup
 
-# If we need to handle pkginfo squashfs files, create mount points under
-# ${BUILD_DIR}/base-pkginfo, mount the squashfs images, and add the mount paths to
-# the list of lowerdirs.
-pkginfo_lowerdirs=""
-if [[ -n "${FLAGS_base_pkginfo}" ]] ; then
-  for entry in $(echo ${FLAGS_base_pkginfo} | sed 's/:/ /g'); do
-    ppath="$(readlink -f "${entry}")"
-    if [[ ! -f "${ppath}" ]] ; then
-      error "--base_pkginfo contains invalid entries."
-      error "Pkginfo file '${ppath}' does not exist."
-      die "Full --base_pkginfo: '${FLAGS_base_pkginfo}'"
-    fi
-
-    pfile="$(basename "${ppath}")"
-    pmdir="${BUILD_DIR}/base-pkginfo/${pfile}"
-    mkdir -p "${pmdir}"
-    mount -rt squashfs -o loop,nodev "${ppath}" "${pmdir}"
-    pkginfo_lowerdirs="${pkginfo_lowerdirs}:${pmdir}"
-    info "Added packageinfo from '${ppath}' to base layers."
-  done
-fi
-
 mkdir "${BUILD_DIR}/fs-root"
 mount -rt squashfs -o loop,nodev "${FLAGS_squashfs_base}" "${BUILD_DIR}/fs-root"
-mkdir "${BUILD_DIR}/${FLAGS_install_root_basename}"
+mkdir "${BUILD_DIR}/install-root"
 mkdir "${BUILD_DIR}/workdir"
-mount -t overlay overlay -o lowerdir="${BUILD_DIR}/fs-root${pkginfo_lowerdirs}",upperdir="${BUILD_DIR}/${FLAGS_install_root_basename}",workdir="${BUILD_DIR}/workdir" "${BUILD_DIR}/${FLAGS_install_root_basename}"
+mount -t overlay overlay -o lowerdir="${BUILD_DIR}/fs-root",upperdir="${BUILD_DIR}/install-root",workdir="${BUILD_DIR}/workdir" "${BUILD_DIR}/install-root"
-
+VERSION_BOARD=$(grep "^VERSION=" ${BUILD_DIR}/fs-root/usr/lib/os-release | cut -d = -f 2-)
-REPO_BUILD_ID=$(source "${REPO_MANIFESTS_DIR}/version.txt"; echo "$FLATCAR_BUILD_ID")
-REPO_FLATCAR_VERSION=$(source "${REPO_MANIFESTS_DIR}/version.txt"; echo "$FLATCAR_VERSION")
-VERSION_BOARD=$(source "${BUILD_DIR}/fs-root/usr/lib/os-release" && echo "$VERSION")
-
-if [[ -z $REPO_BUILD_ID ]] && [[ ${COREOS_OFFICIAL:-0} -ne 1 ]]; then
-    BASE_SQUASHFS_BUILD_ID=$(source "${BUILD_DIR}/fs-root/usr/lib/os-release" && echo -n "$BUILD_ID")
-    info "This is a dev rebuild of an official release tag: No BUILD ID set in '${REPO_MANIFESTS_DIR}/version.txt'.  Will use base squashfs BUILD ID for version check."
-    info "Repo root FLATCAR_VERSION is '$REPO_FLATCAR_VERSION', squashfs build ID is '$BASE_SQUASHFS_BUILD_ID'"
-    FLATCAR_VERSION="${REPO_FLATCAR_VERSION}${BASE_SQUASHFS_BUILD_ID:++}${BASE_SQUASHFS_BUILD_ID}"
-    info "Setting FLATCAR_VERSION to '$FLATCAR_VERSION'"
-fi
-
 if [ "$VERSION_BOARD" != "$FLATCAR_VERSION" ]; then
   warn "Base squashfs version: $VERSION_BOARD"
   warn "SDK board packages version: $FLATCAR_VERSION"
@@ -216,129 +161,54 @@ if [[ ${#} -lt 1 ]]; then
    show_help_if_requested -h
 fi
 
-info "Building '${SYSEXTNAME}' sysext with (meta-)packages '${@}' in '${BUILD_DIR}' using '${FLAGS_compression}' compression".
+info "Building '${SYSEXTNAME}' with (meta-)packages '${@}' in '${BUILD_DIR}'".
 
 for package; do
   echo "Installing package into sysext image: $package"
-  FEATURES="-ebuild-locks binpkg-multi-instance" emerge \
+  FEATURES="-ebuild-locks" emerge \
-    --root="${BUILD_DIR}/${FLAGS_install_root_basename}" \
+    --root="${BUILD_DIR}/install-root" \
     --config-root="/build/${FLAGS_board}"  \
     --sysroot="/build/${FLAGS_board}"  \
+    --root-deps=rdeps \
     --usepkgonly \
-    --binpkg-respect-use=y \
-    --getbinpkg \
     --verbose \
-    --jobs=${NUM_JOBS} \
     "${package}"
 done
 
-# Make squashfs generation more reproducible.
-export SOURCE_DATE_EPOCH=$(stat -c '%Y' "${BUILD_DIR}/fs-root/usr/lib/os-release")
-
 # Unmount in order to get rid of the overlay
-umount "${BUILD_DIR}/${FLAGS_install_root_basename}"
+umount "${BUILD_DIR}/install-root"
 umount "${BUILD_DIR}/fs-root"
 
-if [[ "$FLAGS_generate_pkginfo" = "${FLAGS_TRUE}" ]] ; then
-  info "  Creating pkginfo squashfs '${BUILD_DIR}/${SYSEXTNAME}_pkginfo.raw'"
-  mkdir -p "${BUILD_DIR}/img-pkginfo/var/db"
-  cp -R "${BUILD_DIR}/${FLAGS_install_root_basename}/var/db/pkg" "${BUILD_DIR}/img-pkginfo/var/db/"
-  mksquashfs "${BUILD_DIR}/img-pkginfo" "${BUILD_DIR}/${SYSEXTNAME}_pkginfo.raw" \
-              -noappend -xattrs-exclude '^btrfs.' -comp zstd -Xcompression-level 22 -b 512k
-fi
-
-info "Writing ${SYSEXTNAME}_packages.txt"
-ROOT="${BUILD_DIR}/${FLAGS_install_root_basename}" PORTAGE_CONFIGROOT="/build/${FLAGS_board}" \
-      equery --no-color list --format '$cpv::$repo' '*' > "${BUILD_DIR}/${SYSEXTNAME}_packages.txt"
-
-
-if [[ "${FLAGS_strip_binaries}" = "${FLAGS_TRUE}" ]]; then
-    chost="$("portageq-${BOARD}" envvar CHOST)"
-    strip="${chost}-strip"
-
-    info "Stripping all non-stripped binaries in sysext using '${strip}'"
-
-    # Find all non-stripped binaries, remove ':' from filepath, and strip 'em
-    find "${BUILD_DIR}/${FLAGS_install_root_basename}" -exec file \{\} \; \
-         | awk '/not stripped/ {print substr($1, 1, length($1)-1)}' \
-         | while read bin; do
-                info "     ${strip} ${bin}"
-                "${strip}" "${bin}"
-           done
-fi
-
 if [[ -n "${FLAGS_manglefs_script}" ]]; then
   if [[ ! -x "${FLAGS_manglefs_script}" ]]; then
     die "${FLAGS_manglefs_script} is not executable"
   fi
-  "${FLAGS_manglefs_script}" "${BUILD_DIR}/${FLAGS_install_root_basename}"
+  "${FLAGS_manglefs_script}" "${BUILD_DIR}/install-root"
 fi
 
 info "Removing non-/usr directories from sysext image"
-for entry in "${BUILD_DIR}/${FLAGS_install_root_basename}"/*; do
+for entry in "${BUILD_DIR}/install-root"/*; do
   if [[ "${entry}" = */usr ]]; then
     continue
   fi
   info "  Removing ${entry##*/}"
   rm -rf "${entry}"
 done
-mkdir -p "${BUILD_DIR}/${FLAGS_install_root_basename}/usr/lib/extension-release.d"
+mkdir -p "${BUILD_DIR}/install-root/usr/lib/extension-release.d"
 version_field="${VERSION_FIELD_OVERRIDE:-VERSION_ID=${FLATCAR_VERSION_ID}}"
 all_fields=(
   'ID=flatcar'
   "${version_field}"
   "ARCHITECTURE=${ARCH}"
-  "EXTENSION_RELOAD_MANAGER=1"
 )
-printf '%s\n' "${all_fields[@]}" >"${BUILD_DIR}/${FLAGS_install_root_basename}/usr/lib/extension-release.d/extension-release.${SYSEXTNAME}"
+printf '%s\n' "${all_fields[@]}" >"${BUILD_DIR}/install-root/usr/lib/extension-release.d/extension-release.${SYSEXTNAME}"
-
+mksquashfs "${BUILD_DIR}/install-root" "${BUILD_DIR}/${SYSEXTNAME}.raw" -noappend
-info "Removing opaque directory markers to always merge all contents"
+rm -rf "${BUILD_DIR}"/{fs-root,install-root,workdir}
-find "${BUILD_DIR}/${FLAGS_install_root_basename}" -xdev -type d -exec sh -c 'if [ "$(attr -R -q -g overlay.opaque {} 2>/dev/null)" = y ]; then attr -R -r overlay.opaque {}; fi' \;
-
-info "Checking for invalid file ownership"
-invalid_files=$(find "${BUILD_DIR}/${FLAGS_install_root_basename}" -user sdk -or -group sdk)
-if [[ -n "${invalid_files}" ]]; then
-  die "Invalid file ownership: ${invalid_files}"
-fi
-
-# Set up EROFS compression options based on compression type
-if [[ "${FLAGS_compression}" != "none" ]]; then
-  export SYSTEMD_REPART_MKFS_OPTIONS_EROFS="-z${FLAGS_compression}"
-
-  if [[ -n "${FLAGS_mkerofs_opts}" ]]; then
-    # User provided custom options
-    export SYSTEMD_REPART_MKFS_OPTIONS_EROFS="${SYSTEMD_REPART_MKFS_OPTIONS_EROFS} ${FLAGS_mkerofs_opts}"
-  elif [[ "${FLAGS_compression}" = "lz4hc" ]]; then
-    # Default options for lz4hc
-    export SYSTEMD_REPART_MKFS_OPTIONS_EROFS="${SYSTEMD_REPART_MKFS_OPTIONS_EROFS},12 -C65536 -Efragments,ztailpacking"
-  elif [[ "${FLAGS_compression}" = "zstd" ]]; then
-    # Default options for zstd
-    export SYSTEMD_REPART_MKFS_OPTIONS_EROFS="${SYSTEMD_REPART_MKFS_OPTIONS_EROFS},level=22 -C524288 -Efragments,ztailpacking"
-  fi
-  info "Building sysext with ${FLAGS_compression} compression"
-else
-  info "Building sysext without compression (built-in sysexts)"
-fi
-
-systemd-repart \
-  --private-key="${SYSEXT_SIGNING_KEY_DIR}/sysexts.key" \
-  --certificate="${SYSEXT_SIGNING_KEY_DIR}/sysexts.crt" \
-  --make-ddi=sysext \
-  --copy-source="${BUILD_DIR}/${FLAGS_install_root_basename}" \
-  "${BUILD_DIR}/${SYSEXTNAME}.raw"
-
-rm -rf "${BUILD_DIR}"/{fs-root,"${FLAGS_install_root_basename}",workdir}
 
 # Generate reports
 mkdir "${BUILD_DIR}/img-rootfs"
-systemd-dissect --read-only \
+mount -rt squashfs -o loop,nodev "${BUILD_DIR}/${SYSEXTNAME}.raw" "${BUILD_DIR}/img-rootfs"
-  --mount \
-  --mkdir \
-  --image-policy='root=encrypted+unprotected+absent:usr=encrypted+unprotected+absent' \
-  "${BUILD_DIR}/${SYSEXTNAME}.raw" \
-  "${BUILD_DIR}/img-rootfs"
-
 write_contents "${BUILD_DIR}/img-rootfs" "${BUILD_DIR}/${SYSEXTNAME}_contents.txt"
 write_contents_with_technical_details "${BUILD_DIR}/img-rootfs" "${BUILD_DIR}/${SYSEXTNAME}_contents_wtd.txt"
 write_disk_space_usage_in_paths "${BUILD_DIR}/img-rootfs" "${BUILD_DIR}/${SYSEXTNAME}_disk_usage.txt"
-systemd-dissect --umount --rmdir "${BUILD_DIR}/img-rootfs"
+umount "${BUILD_DIR}/img-rootfs"

build_toolchains

@@ -18,12 +18,13 @@ FORCE_STAGES="stage4"
 ## Define the stage4 config template
 catalyst_stage4() {
 cat <<EOF
+target: stage4
 pkgcache_path: $BINPKGS
 stage4/packages: @system
 stage4/fsscript: ${BUILD_LIBRARY_DIR}/catalyst_toolchains.sh
 stage4/root_overlay: ${ROOT_OVERLAY}
 EOF
-catalyst_stage_default 4
+catalyst_stage_default
 }
 create_provenance_overlay() {
   local root_overlay="$1"
@@ -45,6 +46,7 @@ create_provenance_overlay() {
 }
 
 catalyst_init "$@"
+check_gsutil_opts
 
 ROOT_OVERLAY="${TEMPDIR}/stage4-${ARCH}-$FLAGS_version-overlay"
 
@@ -54,4 +56,13 @@ cp "${BUILD_LIBRARY_DIR}/toolchain_util.sh" "${ROOT_OVERLAY}/tmp"
 create_provenance_overlay "${ROOT_OVERLAY}"
 
 catalyst_build
+
+# TODO: Actually just TOOLCHAIN_PKGS and the exact dependencies should be uploaded
+for board in $(get_board_list); do
+    board_packages="${BINPKGS}/target/${board}"
+    def_upload_path="${UPLOAD_ROOT}/boards/${board}/${FLAGS_version}"
+    sign_and_upload_files "board toolchain packages" "${def_upload_path}" \
+        "toolchain/" "${board_packages}"/*
+done
+
 command_completed

build_torcx_store

@@ -0,0 +1,287 @@
+#!/bin/bash
+
+# Copyright (c) 2017 The CoreOS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+. "$(dirname "$0")/common.sh" || exit 1
+
+# Script must run inside the chroot
+assert_inside_chroot
+
+assert_not_root_user
+
+# Developer-visible flags.
+DEFINE_string board "${DEFAULT_BOARD}" \
+  "The board to build packages for."
+DEFINE_string output_root "${DEFAULT_BUILD_ROOT}/torcx" \
+  "Directory in which to place torcx stores and manifests (named by board/version)"
+DEFINE_string extra_pkg_url "" \
+  "URL to directory where the torcx packages will be available for downloading"
+
+# include upload options
+. "${BUILD_LIBRARY_DIR}/release_util.sh" || exit 1
+
+FLAGS_HELP="usage: $(basename $0) [flags] [images]
+
+This script builds a collection of torcx images to be installed into a torcx
+store.  By default, all supported images are built, but a list of images can be
+given as command arguments.  Note that their order matters, since the version
+specified last will get the default reference symlink.
+"
+show_help_if_requested "$@"
+
+# The following options are advanced options, only available to those willing
+# to read the source code. They are not shown in help output, since they are
+# not needed for the typical developer workflow.
+DEFINE_integer build_attempt 1 \
+  "The build attempt for this image build."
+DEFINE_string group developer \
+  "The update group."
+DEFINE_string version '' \
+  "Overrides version number in name to this version."
+
+# Parse command line
+FLAGS "$@" || exit 1
+eval set -- "${FLAGS_ARGV}"
+
+# Only now can we die on error.  shflags functions leak non-zero error codes,
+# so will die prematurely if 'switch_to_strict_mode' is specified before now.
+switch_to_strict_mode
+
+# Initialize upload options
+check_gsutil_opts
+
+# Define BUILD_DIR and set_build_symlinks.
+. "${BUILD_LIBRARY_DIR}/toolchain_util.sh" || exit 1
+. "${BUILD_LIBRARY_DIR}/board_options.sh" || exit 1
+. "${BUILD_LIBRARY_DIR}/build_image_util.sh" || exit 1
+. "${BUILD_LIBRARY_DIR}/torcx_manifest.sh" || exit 1
+
+TORCX_CAS_ROOT="${FLAGS_output_root}/pkgs/${BOARD}"
+
+# Build and install a package configured as part of a torcx image.
+function torcx_build() (
+        tmproot=${1:?}
+        shift
+        pkgs=( "${@}" )
+
+        export LDFLAGS=-Wl,-rpath,/ORIGIN/../lib
+        export PKGDIR="${tmproot}/var/lib/portage/pkgs"
+
+        # Allow the meta-package to install bashrc to customize the builds.
+        [ -s "${tmproot}/etc/portage/bashrc" ] &&
+        . "${tmproot}/etc/portage/bashrc"
+
+        # Build binary packages using dev files in the board root.
+        emerge-${BOARD} \
+            --jobs="${NUM_JOBS}" \
+            --buildpkg \
+            --buildpkgonly \
+            --nodeps \
+            --oneshot \
+            --verbose \
+            --root-deps=rdeps \
+            "${pkgs[@]}"
+
+        # Install the binary packages in the temporary torcx image root.
+        emerge-${BOARD} \
+            --jobs="${NUM_JOBS}" \
+            --nodeps \
+            --oneshot \
+            --verbose \
+            --root="${tmproot}" \
+            --root-deps=rdeps \
+            --sysroot="${tmproot}" \
+            --usepkgonly \
+            "${pkgs[@]}"
+)
+
+# Create a torcx image from the given meta-package.
+function torcx_package() {
+        local pkg="app-torcx/${1##*/}"
+        local name=${pkg%-[0-9]*}
+        local version=${pkg:${#name}+1}
+        local manifest_path="${2}"
+        local type="${3}"
+        local extra_pkg_url="${4}"
+        local deppkg digest file rpath sha512sum source_pkg rdepends tmproot tmppkgroot update_default tmpfile
+        local pkg_cas_file pkg_cas_root
+        local pkg_locations=()
+        local name=${name##*/}
+        local version=${version%%-r*}
+
+        # Run in a subshell to clean tmproot and tmppkgroot up without
+        # clobbering this shell's EXIT trap.
+        (
+        # Set up the base package layout to dump everything into /bin and /lib.
+        # tmproot is what the packages are installed into.
+        # A subset of the files from tmproot are then moved into tmppkgroot,
+        # which is then archived and uploaded.
+        tmproot=$(sudo mktemp --tmpdir="${BUILD_DIR}" -d)
+        tmppkgroot=$(sudo mktemp --tmpdir="${BUILD_DIR}" -d)
+        trap "sudo rm -rf '${tmproot}' '${tmppkgroot}'" EXIT
+        sudo chmod 0755 "${tmproot}" "${tmppkgroot}"
+        sudo mkdir -p "${tmproot}"/{.torcx,bin,lib,usr}
+        sudo ln -fns ../bin "${tmproot}/usr/bin"
+        sudo ln -fns ../lib "${tmproot}/usr/lib"
+        sudo ln -fns lib "${tmproot}/usr/lib64"
+        sudo ln -fns bin "${tmproot}/usr/sbin"
+        sudo ln -fns lib "${tmproot}/lib64"
+        sudo ln -fns bin "${tmproot}/sbin"
+
+        # Install the meta-package and its direct dependencies.
+        torcx_build "${tmproot}" "=${pkg}" $(torcx_dependencies "${pkg}")
+
+        # by convention, the first dependency in a torcx package is the primary
+        # source package
+        rdepends=($(torcx_dependencies "${pkg}"))
+        source_pkg="${rdepends[0]#=}"
+
+        # Pluck out shared libraries and SONAME links.
+        sudo mv "${tmproot}"/{lib,tmplib}
+        sudo rm -fr "${tmproot}/tmplib/debug"
+        sudo find "${tmproot}/tmplib" -name 'lib*.so' -type l -delete
+        sudo mkdir -p "${tmproot}/lib"
+        sudo find "${tmproot}/tmplib" -name 'lib*.so*' \
+            -exec mv -t "${tmproot}/lib/" {} +
+
+        # Rewrite any units for transparent activation from the torcx root.
+        if [ -e "${tmproot}/tmplib/systemd/system" ]
+        then
+                sudo mkdir -p "${tmproot}/lib/systemd"
+                sudo mv "${tmproot}/tmplib/systemd/system" \
+                    "${tmproot}/lib/systemd/"
+                sudo find "${tmproot}/lib/systemd/system" -type f -exec sed -i \
+                    -e '/^\[Unit]/aRequires=torcx.target\nAfter=torcx.target' \
+                    -e '/^\[Service]/aEnvironmentFile=/run/metadata/torcx' \
+                    -e "/^\[Service]/aEnvironment=TORCX_IMAGEDIR=/${name}" \
+                    -e 's,/usr/s\?bin/,${TORCX_BINDIR}/,g' \
+                    -e 's,^\([^ ]*=\)\(.{TORCX_BINDIR}\)/,\1/usr/bin/env PATH=\2:${PATH} \2/,' {} +
+        fi
+
+        # Network configuration can be installed unmodified.
+        if [ -e "${tmproot}/tmplib/systemd/network" ]
+        then
+                sudo mkdir -p "${tmproot}/lib/systemd"
+                sudo mv "${tmproot}/tmplib/systemd/network" \
+                    "${tmproot}/lib/systemd/"
+        fi
+
+        # Rewrite RPATHs to use the real $ORIGIN value.
+        find -H "${tmproot}"/{bin,lib} -type f |
+        while read file
+        do
+                (
+                rpath=$(sudo patchelf --print-rpath "${file}" 2>/dev/null) &&
+                test "${rpath#/ORIGIN/}" != "${rpath}" &&
+                sudo patchelf --set-rpath "${rpath/#?/\$}" "${file}"
+                ) || :  # Set $? to 0 or the pipeline fails and -e quits.
+        done
+
+        # Move anything we plan to package to its root.
+        sudo mv "${tmproot}"/{.torcx,bin,lib} "${tmppkgroot}"
+        if [ -e "${tmproot}/usr/share" ]
+        then
+                sudo mkdir "${tmppkgroot}/usr"
+                sudo mv "${tmproot}/usr/share" "${tmppkgroot}/usr/"
+        fi
+
+        tmpfile="${BUILD_DIR}/${name}:${version}.torcx.tgz"
+        tar --force-local --selinux --xattrs -C "${tmppkgroot}" -czf "${tmpfile}" .
+        sha512sum=$(sha512sum "${tmpfile}" | awk '{print $1}')
+
+        # TODO(euank): this opaque digest, if it were reproducible, could save
+        # users from having to download things that haven't changed.
+        # For now, use the sha512sum of the final image.
+        # Ideally we should move to something more like a casync digest or tarsum.
+        # The reason this is currently not being done is because to do that we
+        # *MUST* ensure that a given pair of (digest, sha512sum) referenced in
+        # a previous torcx package remains correct.
+        # Because this code, as written, clobbers existing things with the same
+        # digest (but the sha512sum of the .torcx.tgz can differ, e.g. due to ctime)
+        # that property doesn't hold.
+        # To switch this back to a reprodicble digest, we *must* never clobber
+        # existing objects (and thus re-use their sha512sum here).
+        digest="${sha512sum}"
+
+        pkg_cas_root="${TORCX_CAS_ROOT}/${name}/${digest}"
+        pkg_cas_file="${pkg_cas_root}/${name}:${version}.torcx.tgz"
+        mkdir -p "${pkg_cas_root}"
+        mv "${tmpfile}" "${pkg_cas_file}"
+
+        update_default=false
+        if [[ "${type}" == "default" ]]; then
+                update_default=true
+                pkg_locations+=("/usr/share/torcx/store/${name}:${version}.torcx.tgz")
+        fi
+        if [[ "${FLAGS_upload}" -eq ${FLAGS_TRUE} ]]; then
+                pkg_locations+=("$(download_tectonic_torcx_url "pkgs/${BOARD}/${name}/${digest}/${name}:${version}.torcx.tgz")")
+        fi
+        if [[ -n "${extra_pkg_url}" ]]; then
+            pkg_locations+=("${extra_pkg_url}/${name}:${version}.torcx.tgz")
+        fi
+        torcx_manifest::add_pkg "${manifest_path}" \
+            "${name}" \
+            "${version}" \
+            "sha512-${sha512sum}" \
+            "${digest}" \
+            "${source_pkg}" \
+            "${pkg}" \
+            "${update_default}" \
+            "${pkg_locations[@]}"
+        )
+}
+
+# This list defines every torcx image that goes into the vendor store for the
+# current branch's release version.  Note that the default reference symlink
+# for each package will point at the last version specified.  This can handle
+# swapping default package versions for different OS releases by reordering.
+DEFAULT_IMAGES=(
+        =app-torcx/docker-20.10
+)
+
+# This list contains extra images which will be uploaded and included in the
+# generated manifest, but won't be included in the vendor store.
+EXTRA_IMAGES=(
+)
+
+mkdir -p "${BUILD_DIR}"
+manifest_path="${BUILD_DIR}/torcx_manifest.json"
+torcx_manifest::create_empty "${manifest_path}"
+for pkg in "${@:-${DEFAULT_IMAGES[@]}}"; do
+        torcx_package "${pkg#=}" "${manifest_path}" "default" "${FLAGS_extra_pkg_url}"
+done
+for pkg in "${EXTRA_IMAGES[@]}"; do
+        torcx_package "${pkg#=}" "${manifest_path}" "extra" "${FLAGS_extra_pkg_url}"
+done
+
+set_build_symlinks latest "${FLAGS_group}-latest"
+
+# Upload the pkgs referenced by this manifest
+for pkg in $(torcx_manifest::get_pkg_names "${manifest_path}"); do
+        for digest in $(torcx_manifest::get_digests "${manifest_path}" "${pkg}"); do
+            # no need to sign; the manifest includes their shasum and is signed.
+            upload_files \
+                'torcx pkg' \
+                "${TORCX_UPLOAD_ROOT}/pkgs/${BOARD}/${pkg}/${digest}" \
+                "" \
+                "${TORCX_CAS_ROOT}/${pkg}/${digest}"/*.torcx.tgz
+      done
+done
+
+# Upload the manifest
+# Note: the manifest is uploaded to 'UPLOAD_ROOT' rather than
+# 'TORCX_UPLOAD_ROOT'.
+# For non-release builds, those two locations will be the same, so it usually
+# won't matter.
+# However, for release builds, torcx packages may be uploaded directly to their
+# final location, while the manifest still has to go through build bucket in
+# order to get signed.
+sign_and_upload_files \
+    'torcx manifest' \
+    "${UPLOAD_ROOT}/torcx/manifests/${BOARD}/${FLATCAR_VERSION}" \
+    "" \
+    "${manifest_path}"
+
+# vim: tabstop=8 softtabstop=4 shiftwidth=8 expandtab

changelog/bugfixes/2023-10-17-cloudsigma-tty-raw.md

@@ -1 +0,0 @@
-- Set TTY used for fetching server_context to RAW mode before running cloudinit on cloudsigma ([scripts#1280](https://github.com/flatcar/scripts/pull/1280))

changelog/bugfixes/2023-12-11-update-extension-custom-url.md

@@ -1 +0,0 @@
-- Fixed supplying extension update payloads with a custom base URL in Nebraska ([Flatcar#1281](https://github.com/flatcar/Flatcar/issues/1281))

changelog/bugfixes/2024-01-15-aws-ssm-agent.md

@@ -1 +0,0 @@
-- AWS: Fixed the Amazon SSM agent that was crashing. ([Flatcar#1307](https://github.com/flatcar/Flatcar/issues/1307))

changelog/bugfixes/2024-01-15-coreos-cloudinit.md

@@ -1 +0,0 @@
-- Fixed a bug resulting in coreos-cloudinit resetting the instance hostname to 'localhost' if no metadata could be found ([coreos-cloudinit#25](https://github.com/flatcar/coreos-cloudinit/pull/25))

changelog/bugfixes/2024-01-16-self-hosted-oem-update-payloads.md

@@ -1 +0,0 @@
-- Fixed the handling of OEM update payloads in a Nebraska response with self-hosted packages ([ue-rs#49](https://github.com/flatcar/ue-rs/pull/49))

changelog/bugfixes/2024-01-24-update-proxy-setup.md

@@ -1 +0,0 @@
-- Forwarded the proxy environment variables of `update-engine.service` to the postinstall script to support fetching OEM systemd-sysext payloads through a proxy ([Flatcar#1326](https://github.com/flatcar/Flatcar/issues/1326))

changelog/bugfixes/2024-02-06-cloudsigma-remove-custom-cloudinit-service-config.md

@@ -1 +0,0 @@
-- Removed custom CloudSigma coreos-cloudinit service configuration since it will be called with the cloudsigma oem anyway. The restart of the service can also cause the serial port to be stuck in an nondeterministic state which breaks future runs.

changelog/bugfixes/2024-02-08-airgapped-updates.md

@@ -1 +0,0 @@
-- Added a workaround for old airgapped/proxied update-engine clients to be able to update to this release ([Flatcar#1332](https://github.com/flatcar/Flatcar/issues/1332), [update_engine#38](https://github.com/flatcar/update_engine/pull/38))