feat: add fallback if S3 is missbehaving

Add fallback to direct asset download in case of S3 issues.

Signed-off-by: Mateusz Urbanek <mateusz.urbanek@siderolabs.com>

.dockerignore

@@ -1,18 +1,18 @@
 # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
 #
-# Generated on 2024-03-15T10:45:09Z by kres latest.
+# Generated on 2025-08-14T09:17:28Z by kres 9f63e23-dirty.
 
 *
 !cmd
 !internal
 !pkg
-!hack
 !go.mod
 !go.sum
 !.golangci.yml
 !CHANGELOG.md
 !README.md
 !.markdownlint.json
+!hack/govulncheck.sh
 !tailwind.config.js
 !package.json
 !package-lock.json

.github/workflows/ci.yaml

@@ -1,6 +1,6 @@
 # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
 #
-# Generated on 2025-07-22T11:53:08Z by kres b869533.
+# Generated on 2025-08-13T16:38:32Z by kres 9f63e23.
 
 concurrency:
   group: ${{ github.head_ref || github.run_id }}
@@ -78,17 +78,34 @@ jobs:
       - name: unit-tests-race
         run: |
           make unit-tests-race
-      - name: integration
+      - name: integration-direct
         if: github.event_name == 'pull_request'
         env:
           REGISTRY: registry.dev.siderolabs.io
-          TEST_FLAGS: -test.schematic-service-repository=127.0.0.1:5100/image-factory/schematic -test.installer-external-repository=127.0.0.1:5100/siderolabs -test.installer-internal-repository=127.0.0.1:5100/siderolabs -test.cache-repository=127.0.0.1:5100/image-factory/cache
+          RUN_TESTS: TestIntegrationDirect
+          TEST_FLAGS: -test.schematic-service-repository=127.0.0.1:5100/image-factory/schematic -test.installer-external-repository=127.0.0.1:5100/siderolabs -test.installer-internal-repository=127.0.0.1:5100/siderolabs -test.cache-repository=127.0.0.1:5100/image-factory/cache -test.signing-cache-repository=127.0.0.1:5100/image-factory/signing-cache
         run: |
-          make integration
+          make integration-direct
+      - name: integration-s3
+        if: github.event_name == 'pull_request'
+        env:
+          REGISTRY: registry.dev.siderolabs.io
+          RUN_TESTS: TestIntegrationS3
+          TEST_FLAGS: -test.schematic-service-repository=127.0.0.1:5100/image-factory/schematic -test.installer-external-repository=127.0.0.1:5100/siderolabs -test.installer-internal-repository=127.0.0.1:5100/siderolabs -test.cache-repository=127.0.0.1:5100/image-factory/cache -test.signing-cache-repository=127.0.0.1:5100/image-factory/signing-cache
+        run: |
+          make integration-s3
+      - name: integration-cdn
+        if: github.event_name == 'pull_request'
+        env:
+          REGISTRY: registry.dev.siderolabs.io
+          RUN_TESTS: TestIntegrationCDN
+          TEST_FLAGS: -test.schematic-service-repository=127.0.0.1:5100/image-factory/schematic -test.installer-external-repository=127.0.0.1:5100/siderolabs -test.installer-internal-repository=127.0.0.1:5100/siderolabs -test.cache-repository=127.0.0.1:5100/image-factory/cache -test.signing-cache-repository=127.0.0.1:5100/image-factory/signing-cache
+        run: |
+          make integration-cdn
       - name: coverage
         uses: codecov/codecov-action@v5
         with:
-          files: _out/coverage-unit-tests.txt,_out/coverage-integration.txt
+          files: _out/coverage-unit-tests.txt,_out/coverage-integration-direct.txt,_out/coverage-integration-s3.txt,_out/coverage-integration-cdn.txt
           token: ${{ secrets.CODECOV_TOKEN }}
         timeout-minutes: 3
       - name: image-factory
@@ -155,7 +172,7 @@ jobs:
           files: |-
             _out/image-factory-*
             _out/sha*.txt
-  integration-talos-main:
+  integration-cdn-talos-main:
     runs-on:
       - self-hosted
       - generic
@@ -200,9 +217,114 @@ jobs:
           driver: remote
           endpoint: tcp://buildkit-amd64.ci.svc.cluster.local:1234
         timeout-minutes: 10
-      - name: integration-talos-main
+      - name: integration-cdn-talos-main
         env:
           REGISTRY: registry.dev.siderolabs.io
-          TEST_FLAGS: -test.schematic-service-repository=registry.dev.siderolabs.io/image-factory/schematic -test.installer-external-repository=registry.dev.siderolabs.io/siderolabs -test.installer-internal-repository=registry.dev.siderolabs.io/siderolabs -test.cache-repository=registry.dev.siderolabs.io/image-factory/cache
+          RUN_TESTS: TestIntegrationCDN
+          TEST_FLAGS: -test.schematic-service-repository=registry.dev.siderolabs.io/image-factory/schematic -test.installer-external-repository=registry.dev.siderolabs.io/siderolabs -test.installer-internal-repository=registry.dev.siderolabs.io/siderolabs -test.cache-repository=registry.dev.siderolabs.io/image-factory/cache -test.signing-cache-repository=127.0.0.1:5100/image-factory/signing-cache
         run: |
-          make integration-talos-main
+          make integration-cdn-talos-main
+  integration-direct-talos-main:
+    runs-on:
+      - self-hosted
+      - generic
+    if: contains(fromJSON(needs.default.outputs.labels), 'integration/talos-main')
+    needs:
+      - default
+    steps:
+      - name: gather-system-info
+        id: system-info
+        uses: kenchan0130/actions-system-info@v1.3.1
+        continue-on-error: true
+      - name: print-system-info
+        run: |
+          MEMORY_GB=$((${{ steps.system-info.outputs.totalmem }}/1024/1024/1024))
+
+          OUTPUTS=(
+            "CPU Core: ${{ steps.system-info.outputs.cpu-core }}"
+            "CPU Model: ${{ steps.system-info.outputs.cpu-model }}"
+            "Hostname: ${{ steps.system-info.outputs.hostname }}"
+            "NodeName: ${NODE_NAME}"
+            "Kernel release: ${{ steps.system-info.outputs.kernel-release }}"
+            "Kernel version: ${{ steps.system-info.outputs.kernel-version }}"
+            "Name: ${{ steps.system-info.outputs.name }}"
+            "Platform: ${{ steps.system-info.outputs.platform }}"
+            "Release: ${{ steps.system-info.outputs.release }}"
+            "Total memory: ${MEMORY_GB} GB"
+          )
+
+          for OUTPUT in "${OUTPUTS[@]}";do
+            echo "${OUTPUT}"
+          done
+        continue-on-error: true
+      - name: checkout
+        uses: actions/checkout@v4
+      - name: Unshallow
+        run: |
+          git fetch --prune --unshallow
+      - name: Set up Docker Buildx
+        id: setup-buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          driver: remote
+          endpoint: tcp://buildkit-amd64.ci.svc.cluster.local:1234
+        timeout-minutes: 10
+      - name: integration-direct-talos-main
+        env:
+          REGISTRY: registry.dev.siderolabs.io
+          RUN_TESTS: TestIntegrationDirect
+          TEST_FLAGS: -test.schematic-service-repository=registry.dev.siderolabs.io/image-factory/schematic -test.installer-external-repository=registry.dev.siderolabs.io/siderolabs -test.installer-internal-repository=registry.dev.siderolabs.io/siderolabs -test.cache-repository=registry.dev.siderolabs.io/image-factory/cache -test.signing-cache-repository=127.0.0.1:5100/image-factory/signing-cache
+        run: |
+          make integration-direct-talos-main
+  integration-s3-talos-main:
+    runs-on:
+      - self-hosted
+      - generic
+    if: contains(fromJSON(needs.default.outputs.labels), 'integration/talos-main')
+    needs:
+      - default
+    steps:
+      - name: gather-system-info
+        id: system-info
+        uses: kenchan0130/actions-system-info@v1.3.1
+        continue-on-error: true
+      - name: print-system-info
+        run: |
+          MEMORY_GB=$((${{ steps.system-info.outputs.totalmem }}/1024/1024/1024))
+
+          OUTPUTS=(
+            "CPU Core: ${{ steps.system-info.outputs.cpu-core }}"
+            "CPU Model: ${{ steps.system-info.outputs.cpu-model }}"
+            "Hostname: ${{ steps.system-info.outputs.hostname }}"
+            "NodeName: ${NODE_NAME}"
+            "Kernel release: ${{ steps.system-info.outputs.kernel-release }}"
+            "Kernel version: ${{ steps.system-info.outputs.kernel-version }}"
+            "Name: ${{ steps.system-info.outputs.name }}"
+            "Platform: ${{ steps.system-info.outputs.platform }}"
+            "Release: ${{ steps.system-info.outputs.release }}"
+            "Total memory: ${MEMORY_GB} GB"
+          )
+
+          for OUTPUT in "${OUTPUTS[@]}";do
+            echo "${OUTPUT}"
+          done
+        continue-on-error: true
+      - name: checkout
+        uses: actions/checkout@v4
+      - name: Unshallow
+        run: |
+          git fetch --prune --unshallow
+      - name: Set up Docker Buildx
+        id: setup-buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          driver: remote
+          endpoint: tcp://buildkit-amd64.ci.svc.cluster.local:1234
+        timeout-minutes: 10
+      - name: integration-s3-talos-main
+        env:
+          REGISTRY: registry.dev.siderolabs.io
+          RUN_TESTS: TestIntegrationS3
+          TEST_FLAGS: -test.schematic-service-repository=registry.dev.siderolabs.io/image-factory/schematic -test.installer-external-repository=registry.dev.siderolabs.io/siderolabs -test.installer-internal-repository=registry.dev.siderolabs.io/siderolabs -test.cache-repository=registry.dev.siderolabs.io/image-factory/cache -test.signing-cache-repository=127.0.0.1:5100/image-factory/signing-cache
+        run: |
+          make integration-s3-talos-main

.github/workflows/integration-cdn-talos-main-cron.yaml

@@ -0,0 +1,61 @@
+# THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
+#
+# Generated on 2025-08-13T15:31:47Z by kres 9f63e23.
+
+concurrency:
+  group: ${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+"on":
+  schedule:
+    - cron: 30 7 * * *
+name: integration-cdn-talos-main-cron
+jobs:
+  default:
+    runs-on:
+      - self-hosted
+      - generic
+    steps:
+      - name: gather-system-info
+        id: system-info
+        uses: kenchan0130/actions-system-info@v1.3.1
+        continue-on-error: true
+      - name: print-system-info
+        run: |
+          MEMORY_GB=$((${{ steps.system-info.outputs.totalmem }}/1024/1024/1024))
+
+          OUTPUTS=(
+            "CPU Core: ${{ steps.system-info.outputs.cpu-core }}"
+            "CPU Model: ${{ steps.system-info.outputs.cpu-model }}"
+            "Hostname: ${{ steps.system-info.outputs.hostname }}"
+            "NodeName: ${NODE_NAME}"
+            "Kernel release: ${{ steps.system-info.outputs.kernel-release }}"
+            "Kernel version: ${{ steps.system-info.outputs.kernel-version }}"
+            "Name: ${{ steps.system-info.outputs.name }}"
+            "Platform: ${{ steps.system-info.outputs.platform }}"
+            "Release: ${{ steps.system-info.outputs.release }}"
+            "Total memory: ${MEMORY_GB} GB"
+          )
+
+          for OUTPUT in "${OUTPUTS[@]}";do
+            echo "${OUTPUT}"
+          done
+        continue-on-error: true
+      - name: checkout
+        uses: actions/checkout@v4
+      - name: Unshallow
+        run: |
+          git fetch --prune --unshallow
+      - name: Set up Docker Buildx
+        id: setup-buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          driver: remote
+          endpoint: tcp://buildkit-amd64.ci.svc.cluster.local:1234
+        timeout-minutes: 10
+      - name: integration-cdn-talos-main
+        env:
+          REGISTRY: registry.dev.siderolabs.io
+          RUN_TESTS: TestIntegrationCDN
+          TEST_FLAGS: -test.schematic-service-repository=registry.dev.siderolabs.io/image-factory/schematic -test.installer-external-repository=registry.dev.siderolabs.io/siderolabs -test.installer-internal-repository=registry.dev.siderolabs.io/siderolabs -test.cache-repository=registry.dev.siderolabs.io/image-factory/cache -test.signing-cache-repository=127.0.0.1:5100/image-factory/signing-cache
+        run: |
+          make integration-cdn-talos-main

.github/workflows/integration-direct-talos-main-cron.yaml

@@ -0,0 +1,61 @@
+# THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
+#
+# Generated on 2025-08-13T15:31:47Z by kres 9f63e23.
+
+concurrency:
+  group: ${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+"on":
+  schedule:
+    - cron: 30 7 * * *
+name: integration-direct-talos-main-cron
+jobs:
+  default:
+    runs-on:
+      - self-hosted
+      - generic
+    steps:
+      - name: gather-system-info
+        id: system-info
+        uses: kenchan0130/actions-system-info@v1.3.1
+        continue-on-error: true
+      - name: print-system-info
+        run: |
+          MEMORY_GB=$((${{ steps.system-info.outputs.totalmem }}/1024/1024/1024))
+
+          OUTPUTS=(
+            "CPU Core: ${{ steps.system-info.outputs.cpu-core }}"
+            "CPU Model: ${{ steps.system-info.outputs.cpu-model }}"
+            "Hostname: ${{ steps.system-info.outputs.hostname }}"
+            "NodeName: ${NODE_NAME}"
+            "Kernel release: ${{ steps.system-info.outputs.kernel-release }}"
+            "Kernel version: ${{ steps.system-info.outputs.kernel-version }}"
+            "Name: ${{ steps.system-info.outputs.name }}"
+            "Platform: ${{ steps.system-info.outputs.platform }}"
+            "Release: ${{ steps.system-info.outputs.release }}"
+            "Total memory: ${MEMORY_GB} GB"
+          )
+
+          for OUTPUT in "${OUTPUTS[@]}";do
+            echo "${OUTPUT}"
+          done
+        continue-on-error: true
+      - name: checkout
+        uses: actions/checkout@v4
+      - name: Unshallow
+        run: |
+          git fetch --prune --unshallow
+      - name: Set up Docker Buildx
+        id: setup-buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          driver: remote
+          endpoint: tcp://buildkit-amd64.ci.svc.cluster.local:1234
+        timeout-minutes: 10
+      - name: integration-direct-talos-main
+        env:
+          REGISTRY: registry.dev.siderolabs.io
+          RUN_TESTS: TestIntegrationDirect
+          TEST_FLAGS: -test.schematic-service-repository=registry.dev.siderolabs.io/image-factory/schematic -test.installer-external-repository=registry.dev.siderolabs.io/siderolabs -test.installer-internal-repository=registry.dev.siderolabs.io/siderolabs -test.cache-repository=registry.dev.siderolabs.io/image-factory/cache -test.signing-cache-repository=127.0.0.1:5100/image-factory/signing-cache
+        run: |
+          make integration-direct-talos-main

.github/workflows/integration-s3-talos-main-cron.yaml

@@ -1,6 +1,6 @@
 # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
 #
-# Generated on 2025-07-22T11:53:08Z by kres b869533.
+# Generated on 2025-08-13T15:31:47Z by kres 9f63e23.
 
 concurrency:
   group: ${{ github.head_ref || github.run_id }}
@@ -8,7 +8,7 @@ concurrency:
 "on":
   schedule:
     - cron: 30 7 * * *
-name: integration-talos-main-cron
+name: integration-s3-talos-main-cron
 jobs:
   default:
     runs-on:
@@ -52,9 +52,10 @@ jobs:
           driver: remote
           endpoint: tcp://buildkit-amd64.ci.svc.cluster.local:1234
         timeout-minutes: 10
-      - name: integration-talos-main
+      - name: integration-s3-talos-main
         env:
           REGISTRY: registry.dev.siderolabs.io
-          TEST_FLAGS: -test.schematic-service-repository=registry.dev.siderolabs.io/image-factory/schematic -test.installer-external-repository=registry.dev.siderolabs.io/siderolabs -test.installer-internal-repository=registry.dev.siderolabs.io/siderolabs -test.cache-repository=registry.dev.siderolabs.io/image-factory/cache
+          RUN_TESTS: TestIntegrationS3
+          TEST_FLAGS: -test.schematic-service-repository=registry.dev.siderolabs.io/image-factory/schematic -test.installer-external-repository=registry.dev.siderolabs.io/siderolabs -test.installer-internal-repository=registry.dev.siderolabs.io/siderolabs -test.cache-repository=registry.dev.siderolabs.io/image-factory/cache -test.signing-cache-repository=127.0.0.1:5100/image-factory/signing-cache
         run: |
-          make integration-talos-main
+          make integration-s3-talos-main

.github/workflows/lock.yml

@@ -1,6 +1,6 @@
 # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
 #
-# Generated on 2025-07-22T11:53:08Z by kres b869533.
+# Generated on 2025-08-13T14:42:25Z by kres 9f63e23.
 
 "on":
   schedule:

.github/workflows/slack-notify-ci-failure.yaml

@@ -0,0 +1,92 @@
+# THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
+#
+# Generated on 2025-08-13T14:42:25Z by kres 9f63e23.
+
+"on":
+  workflow_run:
+    workflows:
+      - default
+      - integration-cdn-talos-main-cron
+      - integration-direct-talos-main-cron
+      - integration-s3-talos-main-cron
+    types:
+      - completed
+    branches:
+      - main
+name: slack-notify-failure
+jobs:
+  slack-notify:
+    runs-on:
+      - self-hosted
+      - generic
+    if: github.event.workflow_run.conclusion == 'failure' && github.event.workflow_run.event != 'pull_request'
+    steps:
+      - name: Slack Notify
+        uses: slackapi/slack-github-action@v2
+        with:
+          method: chat.postMessage
+          payload: |
+            {
+                "channel": "ci-failure",
+                "text": "${{ github.event.workflow_run.conclusion }} - ${{ github.repository }}",
+                "icon_emoji": "${{ github.event.workflow_run.conclusion == 'success' && ':white_check_mark:' || github.event.workflow_run.conclusion == 'failure' && ':x:' || ':warning:' }}",
+                "username": "GitHub Actions",
+                "attachments": [
+                    {
+                        "blocks": [
+                            {
+                                "fields": [
+                                    {
+                                        "text": "${{ github.event.workflow_run.event == 'pull_request' && format('*Pull Request:* {0} (`{1}`)\n<{2}/pull/{3}|{4}>', github.repository, github.ref_name, github.event.repository.html_url, steps.get-pr-number.outputs.pull_request_number, github.event.workflow_run.display_title) || format('*Build:* {0} (`{1}`)\n<{2}/commit/{3}|{4}>', github.repository, github.ref_name, github.event.repository.html_url, github.sha, github.event.workflow_run.display_title) }}",
+                                        "type": "mrkdwn"
+                                    },
+                                    {
+                                        "text": "*Status:*\n`${{ github.event.workflow_run.conclusion }}`",
+                                        "type": "mrkdwn"
+                                    }
+                                ],
+                                "type": "section"
+                            },
+                            {
+                                "fields": [
+                                    {
+                                        "text": "*Author:*\n`${{ github.actor }}`",
+                                        "type": "mrkdwn"
+                                    },
+                                    {
+                                        "text": "*Event:*\n`${{ github.event.workflow_run.event }}`",
+                                        "type": "mrkdwn"
+                                    }
+                                ],
+                                "type": "section"
+                            },
+                            {
+                                "type": "divider"
+                            },
+                            {
+                                "elements": [
+                                    {
+                                        "text": {
+                                            "text": "Logs",
+                                            "type": "plain_text"
+                                        },
+                                        "type": "button",
+                                        "url": "${{ github.event.workflow_run.html_url }}"
+                                    },
+                                    {
+                                        "text": {
+                                            "text": "Commit",
+                                            "type": "plain_text"
+                                        },
+                                        "type": "button",
+                                        "url": "${{ github.event.repository.html_url }}/commit/${{ github.sha }}"
+                                    }
+                                ],
+                                "type": "actions"
+                            }
+                        ],
+                        "color": "${{ github.event.workflow_run.conclusion == 'success' && '#2EB886' || github.event.workflow_run.conclusion == 'failure' && '#A30002' || '#FFCC00' }}"
+                    }
+                ]
+            }
+          token: ${{ secrets.SLACK_BOT_TOKEN_V2 }}

.github/workflows/slack-notify.yaml

@@ -1,12 +1,14 @@
 # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
 #
-# Generated on 2025-07-22T11:53:08Z by kres b869533.
+# Generated on 2025-08-13T14:42:25Z by kres 9f63e23.
 
 "on":
   workflow_run:
     workflows:
       - default
-      - integration-talos-main-cron
+      - integration-cdn-talos-main-cron
+      - integration-direct-talos-main-cron
+      - integration-s3-talos-main-cron
     types:
       - completed
 name: slack-notify
@@ -30,64 +32,66 @@ jobs:
           method: chat.postMessage
           payload: |
             {
-                "channel": "proj-talos-maintainers",
+                "channel": "ci-all",
+                "text": "${{ github.event.workflow_run.conclusion }} - ${{ github.repository }}",
+                "icon_emoji": "${{ github.event.workflow_run.conclusion == 'success' && ':white_check_mark:' || github.event.workflow_run.conclusion == 'failure' && ':x:' || ':warning:' }}",
+                "username": "GitHub Actions",
                 "attachments": [
                     {
-                        "color": "${{ github.event.workflow_run.conclusion == 'success' && '#2EB886' || github.event.workflow_run.conclusion == 'failure' && '#A30002' || '#FFCC00' }}",
-                        "fallback": "test",
                         "blocks": [
                             {
-                                "type": "section",
                                 "fields": [
                                     {
-                                        "type": "mrkdwn",
+                                        "text": "${{ github.event.workflow_run.event == 'pull_request' && format('*Pull Request:* {0} (`{1}`)\n<{2}/pull/{3}|{4}>', github.repository, github.ref_name, github.event.repository.html_url, steps.get-pr-number.outputs.pull_request_number, github.event.workflow_run.display_title) || format('*Build:* {0} (`{1}`)\n<{2}/commit/{3}|{4}>', github.repository, github.ref_name, github.event.repository.html_url, github.sha, github.event.workflow_run.display_title) }}",
-                                        "text": "${{ github.event.workflow_run.event == 'pull_request' && format('*Pull Request:* {0} (`{1}`)\n<{2}/pull/{3}|{4}>', github.repository, github.ref_name, github.event.repository.html_url, steps.get-pr-number.outputs.pull_request_number, github.event.workflow_run.display_title) || format('*Build:* {0} (`{1}`)\n<{2}/commit/{3}|{4}>', github.repository, github.ref_name, github.event.repository.html_url, github.sha, github.event.workflow_run.display_title) }}"
+                                        "type": "mrkdwn"
                                     },
                                     {
-                                        "type": "mrkdwn",
+                                        "text": "*Status:*\n`${{ github.event.workflow_run.conclusion }}`",
-                                        "text": "*Status:*\n`${{ github.event.workflow_run.conclusion }}`"
+                                        "type": "mrkdwn"
                                     }
-                                ]
+                                ],
+                                "type": "section"
                             },
                             {
-                                "type": "section",
                                 "fields": [
                                     {
-                                        "type": "mrkdwn",
+                                        "text": "*Author:*\n`${{ github.actor }}`",
-                                        "text": "*Author:*\n`${{ github.actor }}`"
+                                        "type": "mrkdwn"
                                     },
                                     {
-                                        "type": "mrkdwn",
+                                        "text": "*Event:*\n`${{ github.event.workflow_run.event }}`",
-                                        "text": "*Event:*\n`${{ github.event.workflow_run.event }}`"
+                                        "type": "mrkdwn"
                                     }
-                                ]
+                                ],
+                                "type": "section"
                             },
                             {
                                 "type": "divider"
                             },
                             {
-                                "type": "actions",
                                 "elements": [
                                     {
-                                        "type": "button",
                                         "text": {
-                                            "type": "plain_text",
+                                            "text": "Logs",
-                                            "text": "Logs"
+                                            "type": "plain_text"
                                         },
+                                        "type": "button",
                                         "url": "${{ github.event.workflow_run.html_url }}"
                                     },
                                     {
-                                        "type": "button",
                                         "text": {
-                                            "type": "plain_text",
+                                            "text": "Commit",
-                                            "text": "Commit"
+                                            "type": "plain_text"
                                         },
+                                        "type": "button",
                                         "url": "${{ github.event.repository.html_url }}/commit/${{ github.sha }}"
                                     }
-                                ]
+                                ],
+                                "type": "actions"
+                            }
+                        ],
+                        "color": "${{ github.event.workflow_run.conclusion == 'success' && '#2EB886' || github.event.workflow_run.conclusion == 'failure' && '#A30002' || '#FFCC00' }}"
                     }
                 ]
             }
-                ]
+          token: ${{ secrets.SLACK_BOT_TOKEN_V2 }}
-            }
-          token: ${{ secrets.SLACK_BOT_TOKEN }}

.github/workflows/stale.yml

@@ -1,6 +1,6 @@
 # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
 #
-# Generated on 2025-07-22T11:53:08Z by kres b869533.
+# Generated on 2025-08-13T14:42:25Z by kres 9f63e23.
 
 "on":
   schedule:

.kres.yaml

@@ -2,6 +2,11 @@ kind: golang.Generate
 spec:
   versionPackagePath: internal/version
 ---
+kind: golang.GoVulnCheck
+spec:
+  ignore:
+    - GO-2025-3770
+---
 kind: golang.Build
 spec:
   outputs:
@@ -55,15 +60,31 @@ spec:
       toplevel: true
     - name: imager-tools
       toplevel: true
-    - name: integration.test
+    - name: integration-cdn.test
       toplevel: true
-    - name: integration
+    - name: integration-direct.test
+      toplevel: true
+    - name: integration-s3.test
+      toplevel: true
+    - name: integration-direct
+      toplevel: true
+      dependants:
+        - coverage
+    - name: integration-s3
+      toplevel: true
+      dependants:
+        - coverage
+    - name: integration-cdn
       toplevel: true
       dependants:
         - coverage
     - name: update-to-talos-main
       toplevel: true
-    - name: integration-talos-main
+    - name: integration-cdn-talos-main
+      toplevel: true
+    - name: integration-direct-talos-main
+      toplevel: true
+    - name: integration-s3-talos-main
       toplevel: true
     - name: tailwind
       toplevel: true
@@ -280,27 +301,83 @@ spec:
               dst: /
 ---
 kind: custom.Step
-name: integration.test
+name: integration-direct.test
 spec:
   docker:
     enabled: true
     stages:
-      - name: integration-build
+      - name: integration-direct-build
         description: builds the integration test binary
         from: base
         steps:
           - script:
-              command: go test -c -covermode=atomic -coverpkg=./... -tags integration ./internal/integration
+              command: go test -c -covermode=atomic -coverpkg=./... -tags integration,integration_direct ./internal/integration
               cache:
                 - /root/.cache/go-build
                 - /go/pkg
-      - name: integration.test
+      - name: integration-direct.test
         description: copies out the integration test binary
         steps:
           - copy:
-              from: integration-build
+              from: integration-direct-build
               src: /src/integration.test
-              dst: /integration.test
+              dst: /integration-direct.test
+  makefile:
+    enabled: true
+    phony: true
+    script:
+      - "@$(MAKE) local-$@ DEST=$(ARTIFACTS)"
+---
+kind: custom.Step
+name: integration-s3.test
+spec:
+  docker:
+    enabled: true
+    stages:
+      - name: integration-s3-build
+        description: builds the integration test binary
+        from: base
+        steps:
+          - script:
+              command: go test -c -covermode=atomic -coverpkg=./... -tags integration,integration_s3 ./internal/integration
+              cache:
+                - /root/.cache/go-build
+                - /go/pkg
+      - name: integration-s3.test
+        description: copies out the integration test binary
+        steps:
+          - copy:
+              from: integration-s3-build
+              src: /src/integration.test
+              dst: /integration-s3.test
+  makefile:
+    enabled: true
+    phony: true
+    script:
+      - "@$(MAKE) local-$@ DEST=$(ARTIFACTS)"
+---
+kind: custom.Step
+name: integration-cdn.test
+spec:
+  docker:
+    enabled: true
+    stages:
+      - name: integration-cdn-build
+        description: builds the integration test binary
+        from: base
+        steps:
+          - script:
+              command: go test -c -covermode=atomic -coverpkg=./... -tags integration,integration_cdn ./internal/integration
+              cache:
+                - /root/.cache/go-build
+                - /go/pkg
+      - name: integration-cdn.test
+        description: copies out the integration test binary
+        steps:
+          - copy:
+              from: integration-cdn-build
+              src: /src/integration.test
+              dst: /integration-cdn.test
   makefile:
     enabled: true
     phony: true
@@ -345,16 +422,62 @@ spec:
       - '@$(MAKE) local-copy-out-go-mod DEST=. TARGET_ARGS="--no-cache-filter=update-to-talos-main"'
 ---
 kind: custom.Step
-name: integration
+name: integration-direct
 spec:
   makefile:
     enabled: true
     phony: true
     depends:
-      - integration.test
+      - integration-direct.test
+    script:
+      - "@$(MAKE) image-image-factory PUSH=true"
+      - docker pull $(REGISTRY)/$(USERNAME)/image-factory:$(TAG)
+      - docker rm -f local-if || true
+      - docker run -d -p 5100:5000 --name=local-if registry:3
+      - docker run --rm --net=host --privileged -v /dev:/dev -v /var/run:/var/run -v $(PWD)/$(ARTIFACTS)/:/out/ -v $(PWD)/$(ARTIFACTS)/integration-direct.test:/bin/integration-direct.test:ro --entrypoint /bin/integration-direct.test $(REGISTRY)/$(USERNAME)/image-factory:$(TAG) -test.v $(TEST_FLAGS) -test.coverprofile=/out/coverage-integration-direct.txt -test.run $(RUN_TESTS)
+      - docker rm -f local-if
+  ghaction:
+    enabled: true
+    condition: on-pull-request
+    environment:
+      REGISTRY: registry.dev.siderolabs.io
+      RUN_TESTS: TestIntegrationDirect
+      TEST_FLAGS: "-test.schematic-service-repository=127.0.0.1:5100/image-factory/schematic -test.installer-external-repository=127.0.0.1:5100/siderolabs -test.installer-internal-repository=127.0.0.1:5100/siderolabs -test.cache-repository=127.0.0.1:5100/image-factory/cache -test.signing-cache-repository=127.0.0.1:5100/image-factory/signing-cache"
+---
+kind: custom.Step
+name: integration-s3
+spec:
+  makefile:
+    enabled: true
+    phony: true
+    depends:
+      - integration-s3.test
+    script:
+      - "@$(MAKE) image-image-factory PUSH=true"
+      - docker pull $(REGISTRY)/$(USERNAME)/image-factory:$(TAG)
+      - docker rm -f local-if || true
+      - docker run -d -p 5100:5000 --name=local-if registry:3
+      - docker run --rm --net=host --privileged -v /dev:/dev -v /var/run:/var/run -v $(PWD)/$(ARTIFACTS)/:/out/ -v $(PWD)/$(ARTIFACTS)/integration-s3.test:/bin/integration-s3.test:ro --entrypoint /bin/integration-s3.test $(REGISTRY)/$(USERNAME)/image-factory:$(TAG) -test.v $(TEST_FLAGS) -test.coverprofile=/out/coverage-integration-s3.txt -test.run $(RUN_TESTS)
+      - docker rm -f local-if
+  ghaction:
+    enabled: true
+    condition: on-pull-request
+    environment:
+      REGISTRY: registry.dev.siderolabs.io
+      RUN_TESTS: TestIntegrationS3
+      TEST_FLAGS: "-test.schematic-service-repository=127.0.0.1:5100/image-factory/schematic -test.installer-external-repository=127.0.0.1:5100/siderolabs -test.installer-internal-repository=127.0.0.1:5100/siderolabs -test.cache-repository=127.0.0.1:5100/image-factory/cache -test.signing-cache-repository=127.0.0.1:5100/image-factory/signing-cache"
+---
+kind: custom.Step
+name: integration-cdn
+spec:
+  makefile:
+    enabled: true
+    phony: true
+    depends:
+      - integration-cdn.test
     variables:
       - name: RUN_TESTS
-        defaultValue: TestIntegration
+        defaultValue: TestIntegrationCDN
       - name: TEST_FLAGS
         defaultValue: ""
     script:
@@ -362,38 +485,87 @@ spec:
       - docker pull $(REGISTRY)/$(USERNAME)/image-factory:$(TAG)
       - docker rm -f local-if || true
       - docker run -d -p 5100:5000 --name=local-if registry:3
-      - docker run --rm --net=host --privileged -v /dev:/dev -v /var/run:/var/run -v $(PWD)/$(ARTIFACTS)/:/out/ -v $(PWD)/$(ARTIFACTS)/integration.test:/bin/integration.test:ro --entrypoint /bin/integration.test $(REGISTRY)/$(USERNAME)/image-factory:$(TAG) -test.v $(TEST_FLAGS) -test.coverprofile=/out/coverage-integration.txt -test.run $(RUN_TESTS)
+      - docker run --rm --net=host --privileged -v /dev:/dev -v /var/run:/var/run -v $(PWD)/$(ARTIFACTS)/:/out/ -v $(PWD)/$(ARTIFACTS)/integration-cdn.test:/bin/integration-cdn.test:ro --entrypoint /bin/integration-cdn.test $(REGISTRY)/$(USERNAME)/image-factory:$(TAG) -test.v $(TEST_FLAGS) -test.coverprofile=/out/coverage-integration-cdn.txt -test.run $(RUN_TESTS)
       - docker rm -f local-if
   ghaction:
     enabled: true
     condition: on-pull-request
     environment:
       REGISTRY: registry.dev.siderolabs.io
-      TEST_FLAGS: "-test.schematic-service-repository=127.0.0.1:5100/image-factory/schematic -test.installer-external-repository=127.0.0.1:5100/siderolabs -test.installer-internal-repository=127.0.0.1:5100/siderolabs -test.cache-repository=127.0.0.1:5100/image-factory/cache"
+      RUN_TESTS: TestIntegrationCDN
+      TEST_FLAGS: "-test.schematic-service-repository=127.0.0.1:5100/image-factory/schematic -test.installer-external-repository=127.0.0.1:5100/siderolabs -test.installer-internal-repository=127.0.0.1:5100/siderolabs -test.cache-repository=127.0.0.1:5100/image-factory/cache -test.signing-cache-repository=127.0.0.1:5100/image-factory/signing-cache"
 ---
 kind: custom.Step
-name: integration-talos-main
+name: integration-direct-talos-main
 spec:
   makefile:
     enabled: true
     phony: true
     depends:
       - update-to-talos-main
-    variables:
-      - name: RUN_TESTS
-        defaultValue: TestIntegration
-      - name: TEST_FLAGS
-        defaultValue: ""
     script:
-      - "@$(MAKE) integration"
+      - "@$(MAKE) integration-direct"
   ghaction:
     enabled: true
     cronOnly: true
     environment:
       REGISTRY: registry.dev.siderolabs.io
-      TEST_FLAGS: "-test.schematic-service-repository=registry.dev.siderolabs.io/image-factory/schematic -test.installer-external-repository=registry.dev.siderolabs.io/siderolabs -test.installer-internal-repository=registry.dev.siderolabs.io/siderolabs -test.cache-repository=registry.dev.siderolabs.io/image-factory/cache"
+      RUN_TESTS: TestIntegrationDirect
+      TEST_FLAGS: "-test.schematic-service-repository=registry.dev.siderolabs.io/image-factory/schematic -test.installer-external-repository=registry.dev.siderolabs.io/siderolabs -test.installer-internal-repository=registry.dev.siderolabs.io/siderolabs -test.cache-repository=registry.dev.siderolabs.io/image-factory/cache -test.signing-cache-repository=127.0.0.1:5100/image-factory/signing-cache"
     jobs:
-      - name: integration-talos-main
+      - name: integration-direct-talos-main
+        runnerLabels:
+          - generic
+        triggerLabels:
+          - integration/talos-main
+        crons:
+          - "30 7 * * *"
+---
+kind: custom.Step
+name: integration-s3-talos-main
+spec:
+  makefile:
+    enabled: true
+    phony: true
+    depends:
+      - update-to-talos-main
+    script:
+      - "@$(MAKE) integration-s3"
+  ghaction:
+    enabled: true
+    cronOnly: true
+    environment:
+      REGISTRY: registry.dev.siderolabs.io
+      RUN_TESTS: TestIntegrationS3
+      TEST_FLAGS: "-test.schematic-service-repository=registry.dev.siderolabs.io/image-factory/schematic -test.installer-external-repository=registry.dev.siderolabs.io/siderolabs -test.installer-internal-repository=registry.dev.siderolabs.io/siderolabs -test.cache-repository=registry.dev.siderolabs.io/image-factory/cache -test.signing-cache-repository=127.0.0.1:5100/image-factory/signing-cache"
+    jobs:
+      - name: integration-s3-talos-main
+        runnerLabels:
+          - generic
+        triggerLabels:
+          - integration/talos-main
+        crons:
+          - "30 7 * * *"
+---
+kind: custom.Step
+name: integration-cdn-talos-main
+spec:
+  makefile:
+    enabled: true
+    phony: true
+    depends:
+      - update-to-talos-main
+    script:
+      - "@$(MAKE) integration-cdn"
+  ghaction:
+    enabled: true
+    cronOnly: true
+    environment:
+      REGISTRY: registry.dev.siderolabs.io
+      RUN_TESTS: TestIntegrationCDN
+      TEST_FLAGS: "-test.schematic-service-repository=registry.dev.siderolabs.io/image-factory/schematic -test.installer-external-repository=registry.dev.siderolabs.io/siderolabs -test.installer-internal-repository=registry.dev.siderolabs.io/siderolabs -test.cache-repository=registry.dev.siderolabs.io/image-factory/cache -test.signing-cache-repository=127.0.0.1:5100/image-factory/signing-cache"
+    jobs:
+      - name: integration-cdn-talos-main
         runnerLabels:
           - generic
         triggerLabels:
@@ -410,7 +582,9 @@ kind: service.CodeCov
 spec:
   targetThreshold: 50
   inputPaths:
-    - coverage-integration.txt
+    - coverage-integration-direct.txt
+    - coverage-integration-s3.txt
+    - coverage-integration-cdn.txt
 ---
 kind: custom.Step
 name: tailwind
@@ -434,8 +608,6 @@ spec:
               src: package.json package-lock.json
               dst: .
           - script:
-              cache:
-                - /src/node_modules
               command: bun install
       - name: tailwind-update
         description: "tailwind update"
@@ -448,8 +620,6 @@ spec:
               src: internal/frontend/http
               dst: internal/frontend/http
           - script:
-              cache:
-                - /src/node_modules
               command: node_modules/.bin/tailwindcss -i internal/frontend/http/css/input.css -o internal/frontend/http/css/output.css --minify
       - name: tailwind-copy
         description: "Copies assets"

Dockerfile

@@ -2,7 +2,7 @@
 
 # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
 #
-# Generated on 2025-07-29T12:16:54Z by kres dd1ed6f.
+# Generated on 2025-08-14T09:35:32Z by kres df7e867-dirty.
 
 ARG TOOLCHAIN
 ARG PKGS_PREFIX
@@ -83,11 +83,11 @@ FROM ${PKGS_PREFIX}/zstd:${PKGS} AS pkg-zstd
 FROM --platform=${BUILDPLATFORM} docker.io/oven/bun:1.2.4-alpine AS tailwind-base
 WORKDIR /src
 COPY package.json package-lock.json .
-RUN --mount=type=cache,target=/src/node_modules,id=image-factory/src/node_modules bun install
+RUN bun install
 
 # base toolchain image
 FROM --platform=${BUILDPLATFORM} ${TOOLCHAIN} AS toolchain
-RUN apk --update --no-cache add bash curl build-base jq protoc protobuf-dev
+RUN apk --update --no-cache add bash build-base curl jq protoc protobuf-dev
 
 # copies the imager tools
 FROM scratch AS imager-tools
@@ -127,7 +127,7 @@ COPY --from=pkg-zstd / /
 FROM tailwind-base AS tailwind-update
 COPY tailwind.config.js .
 COPY internal/frontend/http internal/frontend/http
-RUN --mount=type=cache,target=/src/node_modules,id=image-factory/src/node_modules node_modules/.bin/tailwindcss -i internal/frontend/http/css/input.css -o internal/frontend/http/css/output.css --minify
+RUN node_modules/.bin/tailwindcss -i internal/frontend/http/css/input.css -o internal/frontend/http/css/output.css --minify
 
 # build tools
 FROM --platform=${BUILDPLATFORM} toolchain AS tools
@@ -177,8 +177,16 @@ RUN mkdir -p internal/version/data && \
     echo -n ${TAG} > internal/version/data/tag
 
 # builds the integration test binary
-FROM base AS integration-build
+FROM base AS integration-cdn-build
-RUN --mount=type=cache,target=/root/.cache/go-build,id=image-factory/root/.cache/go-build --mount=type=cache,target=/go/pkg,id=image-factory/go/pkg go test -c -covermode=atomic -coverpkg=./... -tags integration ./internal/integration
+RUN --mount=type=cache,target=/root/.cache/go-build,id=image-factory/root/.cache/go-build --mount=type=cache,target=/go/pkg,id=image-factory/go/pkg go test -c -covermode=atomic -coverpkg=./... -tags integration,integration_cdn ./internal/integration
+
+# builds the integration test binary
+FROM base AS integration-direct-build
+RUN --mount=type=cache,target=/root/.cache/go-build,id=image-factory/root/.cache/go-build --mount=type=cache,target=/go/pkg,id=image-factory/go/pkg go test -c -covermode=atomic -coverpkg=./... -tags integration,integration_direct ./internal/integration
+
+# builds the integration test binary
+FROM base AS integration-s3-build
+RUN --mount=type=cache,target=/root/.cache/go-build,id=image-factory/root/.cache/go-build --mount=type=cache,target=/go/pkg,id=image-factory/go/pkg go test -c -covermode=atomic -coverpkg=./... -tags integration,integration_s3 ./internal/integration
 
 # runs gofumpt
 FROM base AS lint-gofumpt
@@ -191,11 +199,19 @@ COPY .golangci.yml .
 ENV GOGC=50
 RUN --mount=type=cache,target=/root/.cache/go-build,id=image-factory/root/.cache/go-build --mount=type=cache,target=/root/.cache/golangci-lint,id=image-factory/root/.cache/golangci-lint,sharing=locked --mount=type=cache,target=/go/pkg,id=image-factory/go/pkg golangci-lint run --config .golangci.yml
 
+# runs golangci-lint fmt
+FROM base AS lint-golangci-lint-fmt-run
+WORKDIR /src
+COPY .golangci.yml .
+ENV GOGC=50
+RUN --mount=type=cache,target=/root/.cache/go-build,id=image-factory/root/.cache/go-build --mount=type=cache,target=/root/.cache/golangci-lint,id=image-factory/root/.cache/golangci-lint,sharing=locked --mount=type=cache,target=/go/pkg,id=image-factory/go/pkg golangci-lint fmt --config .golangci.yml
+RUN --mount=type=cache,target=/root/.cache/go-build,id=image-factory/root/.cache/go-build --mount=type=cache,target=/root/.cache/golangci-lint,id=image-factory/root/.cache/golangci-lint,sharing=locked --mount=type=cache,target=/go/pkg,id=image-factory/go/pkg golangci-lint run --fix --issues-exit-code 0 --config .golangci.yml
+
 # runs govulncheck
 FROM base AS lint-govulncheck
+COPY --chmod=0755 hack/govulncheck.sh ./hack/govulncheck.sh
 WORKDIR /src
-COPY ./hack/govulncheck.sh ./hack/govulncheck.sh
+RUN --mount=type=cache,target=/root/.cache/go-build,id=image-factory/root/.cache/go-build --mount=type=cache,target=/go/pkg,id=image-factory/go/pkg ./hack/govulncheck.sh -exclude 'GO-2025-3770' ./...
-RUN --mount=type=cache,target=/root/.cache/go-build,id=image-factory/root/.cache/go-build --mount=type=cache,target=/go/pkg,id=image-factory/go/pkg ./hack/govulncheck.sh ./...
 
 # runs unit-tests with race detector
 FROM base AS unit-tests-race
@@ -221,8 +237,20 @@ RUN echo -n 'undefined' > internal/version/data/sha && \
     echo -n ${ABBREV_TAG} > internal/version/data/tag
 
 # copies out the integration test binary
-FROM scratch AS integration.test
+FROM scratch AS integration-cdn.test
-COPY --from=integration-build /src/integration.test /integration.test
+COPY --from=integration-cdn-build /src/integration.test /integration-cdn.test
+
+# copies out the integration test binary
+FROM scratch AS integration-direct.test
+COPY --from=integration-direct-build /src/integration.test /integration-direct.test
+
+# copies out the integration test binary
+FROM scratch AS integration-s3.test
+COPY --from=integration-s3-build /src/integration.test /integration-s3.test
+
+# clean golangci-lint fmt output
+FROM scratch AS lint-golangci-lint-fmt
+COPY --from=lint-golangci-lint-fmt-run /src .
 
 FROM scratch AS unit-tests
 COPY --from=unit-tests-run /src/coverage.txt /coverage-unit-tests.txt
@@ -278,3 +306,4 @@ COPY --from=image-factory image-factory-linux-${TARGETARCH} /usr/bin/image-facto
 COPY --from=imager-tools / /
 LABEL org.opencontainers.image.source=https://github.com/siderolabs/image-factory
 ENTRYPOINT ["/usr/bin/image-factory"]
+

Makefile

@@ -1,6 +1,6 @@
 # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
 #
-# Generated on 2025-07-29T12:16:54Z by kres dd1ed6f.
+# Generated on 2025-08-14T09:04:59Z by kres 9f63e23-dirty.
 
 # common variables
 
@@ -80,7 +80,7 @@ TOOLCHAIN ?= docker.io/golang:1.24-alpine
 
 PKGS_PREFIX ?= ghcr.io/siderolabs
 PKGS ?= v1.11.0
-RUN_TESTS ?= TestIntegration
+RUN_TESTS ?= TestIntegrationCDN
 TEST_FLAGS ?=
 
 # help menu
@@ -178,6 +178,9 @@ generate:  ## Generate .proto definitions.
 lint-golangci-lint:  ## Runs golangci-lint linter.
 	@$(MAKE) target-$@
 
+lint-golangci-lint-fmt:  ## Runs golangci-lint formatter and tries to fix issues automatically.
+	@$(MAKE) local-$@ DEST=.
+
 lint-gofumpt:  ## Runs gofumpt linter.
 	@$(MAKE) target-$@
 
@@ -204,13 +207,31 @@ unit-tests:  ## Performs unit tests
 unit-tests-race:  ## Performs unit tests with race detection enabled.
 	@$(MAKE) target-$@
 
-.PHONY: integration
+.PHONY: integration-direct
-integration: integration.test
+integration-direct: integration-direct.test
 	@$(MAKE) image-image-factory PUSH=true
 	docker pull $(REGISTRY)/$(USERNAME)/image-factory:$(TAG)
 	docker rm -f local-if || true
 	docker run -d -p 5100:5000 --name=local-if registry:3
-	docker run --rm --net=host --privileged -v /dev:/dev -v /var/run:/var/run -v $(PWD)/$(ARTIFACTS)/:/out/ -v $(PWD)/$(ARTIFACTS)/integration.test:/bin/integration.test:ro --entrypoint /bin/integration.test $(REGISTRY)/$(USERNAME)/image-factory:$(TAG) -test.v $(TEST_FLAGS) -test.coverprofile=/out/coverage-integration.txt -test.run $(RUN_TESTS)
+	docker run --rm --net=host --privileged -v /dev:/dev -v /var/run:/var/run -v $(PWD)/$(ARTIFACTS)/:/out/ -v $(PWD)/$(ARTIFACTS)/integration-direct.test:/bin/integration-direct.test:ro --entrypoint /bin/integration-direct.test $(REGISTRY)/$(USERNAME)/image-factory:$(TAG) -test.v $(TEST_FLAGS) -test.coverprofile=/out/coverage-integration-direct.txt -test.run $(RUN_TESTS)
+	docker rm -f local-if
+
+.PHONY: integration-s3
+integration-s3: integration-s3.test
+	@$(MAKE) image-image-factory PUSH=true
+	docker pull $(REGISTRY)/$(USERNAME)/image-factory:$(TAG)
+	docker rm -f local-if || true
+	docker run -d -p 5100:5000 --name=local-if registry:3
+	docker run --rm --net=host --privileged -v /dev:/dev -v /var/run:/var/run -v $(PWD)/$(ARTIFACTS)/:/out/ -v $(PWD)/$(ARTIFACTS)/integration-s3.test:/bin/integration-s3.test:ro --entrypoint /bin/integration-s3.test $(REGISTRY)/$(USERNAME)/image-factory:$(TAG) -test.v $(TEST_FLAGS) -test.coverprofile=/out/coverage-integration-s3.txt -test.run $(RUN_TESTS)
+	docker rm -f local-if
+
+.PHONY: integration-cdn
+integration-cdn: integration-cdn.test
+	@$(MAKE) image-image-factory PUSH=true
+	docker pull $(REGISTRY)/$(USERNAME)/image-factory:$(TAG)
+	docker rm -f local-if || true
+	docker run -d -p 5100:5000 --name=local-if registry:3
+	docker run --rm --net=host --privileged -v /dev:/dev -v /var/run:/var/run -v $(PWD)/$(ARTIFACTS)/:/out/ -v $(PWD)/$(ARTIFACTS)/integration-cdn.test:/bin/integration-cdn.test:ro --entrypoint /bin/integration-cdn.test $(REGISTRY)/$(USERNAME)/image-factory:$(TAG) -test.v $(TEST_FLAGS) -test.coverprofile=/out/coverage-integration-cdn.txt -test.run $(RUN_TESTS)
 	docker rm -f local-if
 
 .PHONY: $(ARTIFACTS)/image-factory-linux-amd64
@@ -247,17 +268,33 @@ imager-base:
 .PHONY: imager-tools
 imager-tools:
 
-.PHONY: integration.test
+.PHONY: integration-cdn.test
-integration.test:
+integration-cdn.test:
+	@$(MAKE) local-$@ DEST=$(ARTIFACTS)
+
+.PHONY: integration-direct.test
+integration-direct.test:
+	@$(MAKE) local-$@ DEST=$(ARTIFACTS)
+
+.PHONY: integration-s3.test
+integration-s3.test:
 	@$(MAKE) local-$@ DEST=$(ARTIFACTS)
 
 .PHONY: update-to-talos-main
 update-to-talos-main:
 	@$(MAKE) local-copy-out-go-mod DEST=. TARGET_ARGS="--no-cache-filter=update-to-talos-main"
 
-.PHONY: integration-talos-main
+.PHONY: integration-cdn-talos-main
-integration-talos-main: update-to-talos-main
+integration-cdn-talos-main: update-to-talos-main
-	@$(MAKE) integration
+	@$(MAKE) integration-cdn
+
+.PHONY: integration-direct-talos-main
+integration-direct-talos-main: update-to-talos-main
+	@$(MAKE) integration-direct
+
+.PHONY: integration-s3-talos-main
+integration-s3-talos-main: update-to-talos-main
+	@$(MAKE) integration-s3
 
 .PHONY: tailwind
 tailwind:

cmd/image-factory/cmd/options.go

@@ -99,6 +99,10 @@ type Options struct { //nolint:govet
 	// Leave empty to disable.
 	MetricsListenAddr string
 
+	// MetricsNamespace is the namespace for Prometheus metrics.
+	// It's not user-configurable, but set by the image factory tests.
+	MetricsNamespace string
+
 	// SecureBoot settings.
 	SecureBoot SecureBootOptions
 }

cmd/image-factory/cmd/service.go

@@ -123,6 +123,7 @@ func RunFactory(ctx context.Context, logger *zap.Logger, opts Options) error {
 
 	frontendOptions.RemoteOptions = append(frontendOptions.RemoteOptions, remoteOptions()...)
 	frontendOptions.RegistryRefreshInterval = opts.RegistryRefreshInterval
+	frontendOptions.MetricsNamespace = opts.MetricsNamespace
 
 	frontendHTTP, err := frontendhttp.NewFrontend(logger, configFactory, assetBuilder, artifactsManager, secureBootService, frontendOptions)
 	if err != nil {
@@ -329,7 +330,9 @@ func buildAssetBuilder(logger *zap.Logger, artifactsManager *artifacts.Manager,
 	}
 
 	builderOptions := asset.Options{
+		MetricsNamespace:   opts.MetricsNamespace,
 		AllowedConcurrency: opts.AssetBuildMaxConcurrency,
+		GetAfterPut:        opts.CacheS3Enabled,
 	}
 
 	builder, err := asset.NewBuilder(logger, artifactsManager, cache, builderOptions)
@@ -359,7 +362,9 @@ func buildSchematicFactory(logger *zap.Logger, opts Options) (*schematic.Factory
 		return nil, fmt.Errorf("failed to initialize registry storage: %w", err)
 	}
 
-	factory := schematic.NewFactory(logger, schematiccache.NewCache(storage), schematic.Options{})
+	c := schematiccache.NewCache(storage, schematiccache.Options{MetricsNamespace: opts.MetricsNamespace})
+
+	factory := schematic.NewFactory(logger, c, schematic.Options{MetricsNamespace: opts.MetricsNamespace})
 
 	prometheus.MustRegister(factory)
 

hack/govulncheck.sh

@@ -1,28 +1,48 @@
-#!/usr/bin/env bash
+#!/bin/bash
 # Source: https://github.com/tianon/gosu/blob/e157efb/govulncheck-with-excludes.sh
 # Licensed under the Apache License, Version 2.0
 # Copyright Tianon Gravi
 set -Eeuo pipefail
 
-# a wrapper / replacement for "govulncheck" which allows for excluding vulnerabilities
+exclude_arg=""
+pass_args=()
+
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+        -exclude)
+            exclude_arg="$2"
+            shift 2
+            ;;
+        *)
+            pass_args+=("$1")
+            shift
+            ;;
+    esac
+done
+
+if [[ -n "$exclude_arg" ]]; then
+    excludeVulns="$(jq -nc --arg list "$exclude_arg" '$list | split(",")')"
+else
+    excludeVulns="[]"
+fi
 
-excludeVulns="$(jq -nc '[
-	"GO-2025-3770",
-	empty # trailing comma hack (makes diffs smaller)
-]')"
 export excludeVulns
 
+# Debug print
+echo "excludeVulns = $excludeVulns"
+echo "Passing args: ${pass_args[*]}"
+
 if ! command -v govulncheck > /dev/null; then
     printf "govulncheck not installed"
     exit 1
 fi
 
-if out="$(govulncheck "$@")"; then
+if out="$(govulncheck "${pass_args[@]}")"; then
 	printf '%s\n' "$out"
 	exit 0
 fi
 
-json="$(govulncheck -json "$@")"
+json="$(govulncheck -json "${pass_args[@]}")"
 
 vulns="$(jq <<<"$json" -cs '
 	(

internal/asset/asset.go

@@ -37,20 +37,26 @@ type BootAsset interface {
 
 // Builder is the asset builder.
 type Builder struct {
-	logger           *zap.Logger
+	metricConcurrencyLatency prometheus.Histogram
 	cache                    cache.Cache
-	artifactsManager *artifacts.Manager
+	metricBuildLatency       prometheus.Histogram
 	sf                       singleflight.Group
+	metricAssetsCached       *prometheus.CounterVec
+	logger                   *zap.Logger
+	metricAssetsBuilt        *prometheus.CounterVec
+	metricAssetBytesCached   *prometheus.CounterVec
+	metricAssetBytesBuilt    *prometheus.CounterVec
+	metricAssetCachedErrors  *prometheus.CounterVec
 	semaphore                chan struct{}
-
+	artifactsManager         *artifacts.Manager
-	metricAssetsCached, metricAssetsBuilt         *prometheus.CounterVec
+	getAfterPut              bool
-	metricAssetBytesCached, metricAssetBytesBuilt *prometheus.CounterVec
-	metricConcurrencyLatency, metricBuildLatency  prometheus.Histogram
 }
 
 // Options configures the asset builder.
 type Options struct {
+	MetricsNamespace   string
 	AllowedConcurrency int
+	GetAfterPut        bool
 }
 
 // NewBuilder creates a new asset builder.
@@ -60,11 +66,13 @@ func NewBuilder(logger *zap.Logger, artifactsManager *artifacts.Manager, cache c
 		cache:            cache,
 		artifactsManager: artifactsManager,
 		semaphore:        make(chan struct{}, options.AllowedConcurrency),
+		getAfterPut:      options.GetAfterPut,
 
 		metricAssetsCached: prometheus.NewCounterVec(
 			prometheus.CounterOpts{
 				Name:      "image_factory_assets_delivered_cached_total",
 				Help:      "Number of assets retrieved from cache.",
+				Namespace: options.MetricsNamespace,
 			},
 			[]string{"talos_version", "output_kind", "arch"},
 		),
@@ -72,6 +80,7 @@ func NewBuilder(logger *zap.Logger, artifactsManager *artifacts.Manager, cache c
 			prometheus.CounterOpts{
 				Name:      "image_factory_assets_built_total",
 				Help:      "Number of assets built (missing in the cache).",
+				Namespace: options.MetricsNamespace,
 			},
 			[]string{"talos_version", "output_kind", "arch"},
 		),
@@ -79,6 +88,7 @@ func NewBuilder(logger *zap.Logger, artifactsManager *artifacts.Manager, cache c
 			prometheus.CounterOpts{
 				Name:      "image_factory_assets_cached_bytes_total",
 				Help:      "Number of bytes delivered with cached assets.",
+				Namespace: options.MetricsNamespace,
 			},
 			[]string{"talos_version", "output_kind", "arch"},
 		),
@@ -86,6 +96,15 @@ func NewBuilder(logger *zap.Logger, artifactsManager *artifacts.Manager, cache c
 			prometheus.CounterOpts{
 				Name:      "image_factory_assets_built_bytes_total",
 				Help:      "Number of bytes delivered with assets built (missing in the cache).",
+				Namespace: options.MetricsNamespace,
+			},
+			[]string{"talos_version", "output_kind", "arch"},
+		),
+		metricAssetCachedErrors: prometheus.NewCounterVec(
+			prometheus.CounterOpts{
+				Name:      "image_factory_assets_cached_errors_total",
+				Help:      "Number of errors retrieving assets from cache.",
+				Namespace: options.MetricsNamespace,
 			},
 			[]string{"talos_version", "output_kind", "arch"},
 		),
@@ -94,6 +113,7 @@ func NewBuilder(logger *zap.Logger, artifactsManager *artifacts.Manager, cache c
 				Name:      "image_factory_assets_concurrency_latency_seconds",
 				Help:      "Latency of asset build related to the concurrency limit (waiting for available workers).",
 				Buckets:   []float64{1, 10, 60, 180, 600},
+				Namespace: options.MetricsNamespace,
 			},
 		),
 		metricBuildLatency: prometheus.NewHistogram(
@@ -101,6 +121,7 @@ func NewBuilder(logger *zap.Logger, artifactsManager *artifacts.Manager, cache c
 				Name:      "image_factory_assets_build_latency_seconds",
 				Help:      "Latency of asset build for the build itself (excluding waiting for available workers).",
 				Buckets:   []float64{1, 10, 60, 180, 600},
+				Namespace: options.MetricsNamespace,
 			},
 		),
 	}, nil
@@ -179,6 +200,9 @@ func (b *Builder) buildAndCache(profileHash string, prof profile.Profile, versio
 		b.logger.Error("error putting asset to cache", zap.Error(err), zap.String("profile_hash", profileHash))
 	}
 
+	// if the asset delivery requires a redirect to the cache, we need to return the cached asset
+	// so that the client can download it directly from the cache.
+	if b.getAfterPut {
 		cachedAsset, err := b.cache.Get(ctx, profileHash)
 		if err == nil {
 			b.metricAssetsCached.WithLabelValues(versionString, prof.Output.Kind.String(), prof.Arch).Inc()
@@ -187,7 +211,13 @@ func (b *Builder) buildAndCache(profileHash string, prof profile.Profile, versio
 			return cachedAsset, nil
 		}
 
-	return nil, fmt.Errorf("error getting asset from cache: %w", err)
+		// if we failed to get the asset from cache, return the built asset anyway
+		b.logger.Warn("failed to get object from cache, falling back to direct delivery", zap.Error(err))
+
+		b.metricAssetCachedErrors.WithLabelValues(versionString, prof.Output.Kind.String(), prof.Arch).Inc()
+	}
+
+	return asset, nil
 }
 
 // build the asset using Talos imager.
@@ -285,6 +315,8 @@ func (b *Builder) Collect(ch chan<- prometheus.Metric) {
 	b.metricAssetBytesBuilt.Collect(ch)
 	b.metricAssetBytesCached.Collect(ch)
 
+	b.metricAssetCachedErrors.Collect(ch)
+
 	b.metricBuildLatency.Collect(ch)
 	b.metricConcurrencyLatency.Collect(ch)
 }

internal/frontend/http/http.go

@@ -55,14 +55,12 @@ type Frontend struct {
 
 // Options configures the HTTP frontend.
 type Options struct {
+	CacheSigningKey             crypto.PrivateKey
 	ExternalURL                 *url.URL
 	ExternalPXEURL              *url.URL
-
 	InstallerInternalRepository name.Repository
 	InstallerExternalRepository name.Repository
-
+	MetricsNamespace            string
-	CacheSigningKey crypto.PrivateKey
-
 	RemoteOptions               []remote.Option
 	RegistryRefreshInterval     time.Duration
 }
@@ -105,7 +103,9 @@ func NewFrontend(
 
 	// monitoring middleware
 	mdlw := middleware.New(middleware.Config{
-		Recorder: metrics.NewRecorder(metrics.Config{}),
+		Recorder: metrics.NewRecorder(metrics.Config{
+			Prefix: opts.MetricsNamespace,
+		}),
 	})
 
 	registerRoute := func(registrator func(string, httprouter.Handle), path string, handler func(ctx context.Context, w http.ResponseWriter, r *http.Request, p httprouter.Params) error) {

internal/integration/integration_test.go

@@ -46,7 +46,6 @@ func setupFactory(t *testing.T, options cmd.Options) (context.Context, string) {
 	options.SchematicServiceRepository = schematicFactoryRepositoryFlag
 	options.InstallerExternalRepository = installerExternalRepository
 	options.InstallerInternalRepository = installerInternalRepository
-	options.CacheRepository = cacheRepository
 	options.RegistryRefreshInterval = time.Minute // use a short interval for the tests
 
 	setupSecureBoot(t, &options)
@@ -121,14 +120,11 @@ func healthcheck(url string) func() error {
 }
 
 const (
-	bucket       = "image-factory"
-	bucketPrefix = "/" + bucket
-
 	s3Access = "AKIA6Z4C7N3S2JD3JH9A"
 	s3Secret = "y1rE4xZnqO6xvM7L0jFD3EXAMPLEnG4K2vOfLp8Iv9"
 )
 
-func setupS3(t *testing.T, pool *dockertest.Pool) string {
+func setupS3(t *testing.T, pool *dockertest.Pool, bucket string) string {
 	t.Helper()
 
 	_, port, err := net.SplitHostPort(findListenAddr(t))
@@ -167,16 +163,13 @@ func setupS3(t *testing.T, pool *dockertest.Pool) string {
 	})
 	require.NoError(t, err)
 
-	t.Setenv("AWS_ACCESS_KEY_ID", s3Access)
-	t.Setenv("AWS_SECRET_ACCESS_KEY", s3Secret)
-
 	return endpoint
 }
 
 //go:embed testdata/templates/nginx.sh
 var nginxConfigTemplate string
 
-func setupMockCDN(t *testing.T, pool *dockertest.Pool, s3 string) string {
+func setupMockCDN(t *testing.T, pool *dockertest.Pool, s3, bucket string) string {
 	t.Helper()
 
 	_, port, err := net.SplitHostPort(findListenAddr(t))
@@ -249,19 +242,7 @@ func findListenAddr(t *testing.T) string {
 	return addr
 }
 
-func TestIntegration(t *testing.T) {
+func commonTest(t *testing.T, options cmd.Options) {
-	pool := docker(t)
-	options := cmd.DefaultOptions
-
-	options.CacheS3Enabled = true
-	options.CacheS3Bucket = bucket
-	options.InsecureCacheS3 = true
-	options.CacheS3Endpoint = setupS3(t, pool)
-
-	options.CacheCDNEnabled = true
-	options.CacheCDNTrimPrefix = bucketPrefix
-	options.CacheCDNHost = setupMockCDN(t, pool, options.CacheS3Endpoint)
-
 	ctx, listenAddr := setupFactory(t, options)
 	baseURL := "http://" + listenAddr
 
@@ -313,6 +294,7 @@ var (
 	installerExternalRepository    string
 	installerInternalRepository    string
 	cacheRepository                string
+	signingCacheRepository         string
 )
 
 func init() {
@@ -321,4 +303,5 @@ func init() {
 	flag.StringVar(&installerExternalRepository, "test.installer-external-repository", cmd.DefaultOptions.InstallerExternalRepository, "image repository for the installer (external)")
 	flag.StringVar(&installerInternalRepository, "test.installer-internal-repository", cmd.DefaultOptions.InstallerInternalRepository, "image repository for the installer (internal)")
 	flag.StringVar(&cacheRepository, "test.cache-repository", cmd.DefaultOptions.CacheRepository, "image repository for cached boot assets")
+	flag.StringVar(&signingCacheRepository, "test.signing-cache-repository", cmd.DefaultOptions.CacheRepository+"sign", "image repository for signatures of cached boot assets (used for S3+CDN tests)")
 }

internal/integration/main_cdn_test.go

@@ -0,0 +1,40 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+//go:build integration_cdn
+
+package integration_test
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/siderolabs/image-factory/cmd/image-factory/cmd"
+)
+
+func TestIntegrationCDN(t *testing.T) {
+	pool := docker(t)
+
+	// set up S3 access credentials for the tests, those are shared across all tests
+	t.Setenv("AWS_ACCESS_KEY_ID", s3Access)
+	t.Setenv("AWS_SECRET_ACCESS_KEY", s3Secret)
+
+	t.Run("S3+CDN", func(t *testing.T) {
+		options := cmd.DefaultOptions
+
+		options.CacheRepository = signingCacheRepository
+		options.MetricsNamespace = "test_s3_cdn"
+
+		options.CacheS3Enabled = true
+		options.CacheS3Bucket = "test-s3-cdn"
+		options.InsecureCacheS3 = true
+		options.CacheS3Endpoint = setupS3(t, pool, options.CacheS3Bucket)
+
+		options.CacheCDNEnabled = true
+		options.CacheCDNTrimPrefix = fmt.Sprintf("/%s", options.CacheS3Bucket)
+		options.CacheCDNHost = setupMockCDN(t, pool, options.CacheS3Endpoint, options.CacheS3Bucket)
+
+		commonTest(t, options)
+	})
+}

internal/integration/main_direct_test.go

@@ -0,0 +1,22 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+//go:build integration_direct
+
+package integration_test
+
+import (
+	"testing"
+
+	"github.com/siderolabs/image-factory/cmd/image-factory/cmd"
+)
+
+func TestIntegrationDirect(t *testing.T) {
+	options := cmd.DefaultOptions
+
+	options.CacheRepository = cacheRepository
+	options.MetricsNamespace = ""
+
+	commonTest(t, options)
+}

internal/integration/main_s3_test.go

@@ -0,0 +1,35 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+//go:build integration_s3
+
+package integration_test
+
+import (
+	"testing"
+
+	"github.com/siderolabs/image-factory/cmd/image-factory/cmd"
+)
+
+func TestIntegrationS3(t *testing.T) {
+	pool := docker(t)
+
+	// set up S3 access credentials for the tests, those are shared across all tests
+	t.Setenv("AWS_ACCESS_KEY_ID", s3Access)
+	t.Setenv("AWS_SECRET_ACCESS_KEY", s3Secret)
+
+	t.Run("S3", func(t *testing.T) {
+		options := cmd.DefaultOptions
+
+		options.CacheRepository = signingCacheRepository
+		options.MetricsNamespace = "test_s3"
+
+		options.CacheS3Enabled = true
+		options.CacheS3Bucket = "test-s3"
+		options.InsecureCacheS3 = true
+		options.CacheS3Endpoint = setupS3(t, pool, options.CacheS3Bucket)
+
+		commonTest(t, options)
+	})
+}

internal/schematic/schematic.go

@@ -17,15 +17,18 @@ import (
 
 // Factory is the schematic factory.
 type Factory struct {
-	options Options
-	logger  *zap.Logger
 	storage         storage.Storage
-
+	metricGet       prometheus.Counter
-	metricGet, metricCreate, metricDuplicate prometheus.Counter
+	metricCreate    prometheus.Counter
+	metricDuplicate prometheus.Counter
+	logger          *zap.Logger
+	options         Options
 }
 
 // Options for the schematic factory.
-type Options struct{}
+type Options struct {
+	MetricsNamespace string
+}
 
 // NewFactory creates a new schematic factory.
 func NewFactory(logger *zap.Logger, storage storage.Storage, options Options) *Factory {
@@ -37,14 +40,17 @@ func NewFactory(logger *zap.Logger, storage storage.Storage, options Options) *F
 		metricGet: prometheus.NewCounter(prometheus.CounterOpts{
 			Name:      "image_factory_schematic_get_total",
 			Help:      "Number of times schematics were retrieved.",
+			Namespace: options.MetricsNamespace,
 		}),
 		metricCreate: prometheus.NewCounter(prometheus.CounterOpts{
 			Name:      "image_factory_schematic_create_total",
 			Help:      "Number of new schematics created.",
+			Namespace: options.MetricsNamespace,
 		}),
 		metricDuplicate: prometheus.NewCounter(prometheus.CounterOpts{
 			Name:      "image_factory_schematic_duplicate_create_total",
 			Help:      "Number of new schematics which were created as duplicate.",
+			Namespace: options.MetricsNamespace,
 		}),
 	}
 }

internal/schematic/storage/cache/cache.go

@@ -17,6 +17,11 @@ import (
 	"github.com/siderolabs/image-factory/internal/schematic/storage"
 )
 
+// Options configures the storage.
+type Options struct {
+	MetricsNamespace string
+}
+
 // Storage is a schematic storage in-memory cache.
 type Storage struct {
 	underlying storage.Storage
@@ -29,13 +34,14 @@ type Storage struct {
 }
 
 // NewCache returns a new cache storage.
-func NewCache(underlying storage.Storage) *Storage {
+func NewCache(underlying storage.Storage, options Options) *Storage {
 	return &Storage{
 		underlying: underlying,
 		m:          map[string]optional.Optional[[]byte]{},
 		metricCacheSize: prometheus.NewGauge(prometheus.GaugeOpts{
 			Name:      "image_factory_schematic_cache_size",
 			Help:      "Number of schematics in in-memory cache.",
+			Namespace: options.MetricsNamespace,
 		}),
 	}
 }

internal/schematic/storage/cache/cache_test.go

@@ -55,7 +55,7 @@ func TestStorage(t *testing.T) {
 	defer cancel()
 
 	underlying := &mockStorage{}
-	strg := cache.NewCache(underlying)
+	strg := cache.NewCache(underlying, cache.Options{})
 
 	v, err := strg.Get(ctx, "foo")
 	require.NoError(t, err)