feat: add fallback if S3 is missbehaving

Add fallback to direct asset download in case of S3 issues.

Signed-off-by: Mateusz Urbanek <mateusz.urbanek@siderolabs.com>
This commit is contained in:
Mateusz Urbanek 2025-08-13 13:28:21 +02:00
parent 9760ab0fee
commit a1e37078e1
No known key found for this signature in database
GPG Key ID: F16F84591E26D77F
24 changed files with 912 additions and 182 deletions

View File

@ -1,18 +1,18 @@
# THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
# #
# Generated on 2024-03-15T10:45:09Z by kres latest. # Generated on 2025-08-14T09:17:28Z by kres 9f63e23-dirty.
* *
!cmd !cmd
!internal !internal
!pkg !pkg
!hack
!go.mod !go.mod
!go.sum !go.sum
!.golangci.yml !.golangci.yml
!CHANGELOG.md !CHANGELOG.md
!README.md !README.md
!.markdownlint.json !.markdownlint.json
!hack/govulncheck.sh
!tailwind.config.js !tailwind.config.js
!package.json !package.json
!package-lock.json !package-lock.json

View File

@ -1,6 +1,6 @@
# THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
# #
# Generated on 2025-07-22T11:53:08Z by kres b869533. # Generated on 2025-08-13T16:38:32Z by kres 9f63e23.
concurrency: concurrency:
group: ${{ github.head_ref || github.run_id }} group: ${{ github.head_ref || github.run_id }}
@ -78,17 +78,34 @@ jobs:
- name: unit-tests-race - name: unit-tests-race
run: | run: |
make unit-tests-race make unit-tests-race
- name: integration - name: integration-direct
if: github.event_name == 'pull_request' if: github.event_name == 'pull_request'
env: env:
REGISTRY: registry.dev.siderolabs.io REGISTRY: registry.dev.siderolabs.io
TEST_FLAGS: -test.schematic-service-repository=127.0.0.1:5100/image-factory/schematic -test.installer-external-repository=127.0.0.1:5100/siderolabs -test.installer-internal-repository=127.0.0.1:5100/siderolabs -test.cache-repository=127.0.0.1:5100/image-factory/cache RUN_TESTS: TestIntegrationDirect
TEST_FLAGS: -test.schematic-service-repository=127.0.0.1:5100/image-factory/schematic -test.installer-external-repository=127.0.0.1:5100/siderolabs -test.installer-internal-repository=127.0.0.1:5100/siderolabs -test.cache-repository=127.0.0.1:5100/image-factory/cache -test.signing-cache-repository=127.0.0.1:5100/image-factory/signing-cache
run: | run: |
make integration make integration-direct
- name: integration-s3
if: github.event_name == 'pull_request'
env:
REGISTRY: registry.dev.siderolabs.io
RUN_TESTS: TestIntegrationS3
TEST_FLAGS: -test.schematic-service-repository=127.0.0.1:5100/image-factory/schematic -test.installer-external-repository=127.0.0.1:5100/siderolabs -test.installer-internal-repository=127.0.0.1:5100/siderolabs -test.cache-repository=127.0.0.1:5100/image-factory/cache -test.signing-cache-repository=127.0.0.1:5100/image-factory/signing-cache
run: |
make integration-s3
- name: integration-cdn
if: github.event_name == 'pull_request'
env:
REGISTRY: registry.dev.siderolabs.io
RUN_TESTS: TestIntegrationCDN
TEST_FLAGS: -test.schematic-service-repository=127.0.0.1:5100/image-factory/schematic -test.installer-external-repository=127.0.0.1:5100/siderolabs -test.installer-internal-repository=127.0.0.1:5100/siderolabs -test.cache-repository=127.0.0.1:5100/image-factory/cache -test.signing-cache-repository=127.0.0.1:5100/image-factory/signing-cache
run: |
make integration-cdn
- name: coverage - name: coverage
uses: codecov/codecov-action@v5 uses: codecov/codecov-action@v5
with: with:
files: _out/coverage-unit-tests.txt,_out/coverage-integration.txt files: _out/coverage-unit-tests.txt,_out/coverage-integration-direct.txt,_out/coverage-integration-s3.txt,_out/coverage-integration-cdn.txt
token: ${{ secrets.CODECOV_TOKEN }} token: ${{ secrets.CODECOV_TOKEN }}
timeout-minutes: 3 timeout-minutes: 3
- name: image-factory - name: image-factory
@ -155,7 +172,7 @@ jobs:
files: |- files: |-
_out/image-factory-* _out/image-factory-*
_out/sha*.txt _out/sha*.txt
integration-talos-main: integration-cdn-talos-main:
runs-on: runs-on:
- self-hosted - self-hosted
- generic - generic
@ -200,9 +217,114 @@ jobs:
driver: remote driver: remote
endpoint: tcp://buildkit-amd64.ci.svc.cluster.local:1234 endpoint: tcp://buildkit-amd64.ci.svc.cluster.local:1234
timeout-minutes: 10 timeout-minutes: 10
- name: integration-talos-main - name: integration-cdn-talos-main
env: env:
REGISTRY: registry.dev.siderolabs.io REGISTRY: registry.dev.siderolabs.io
TEST_FLAGS: -test.schematic-service-repository=registry.dev.siderolabs.io/image-factory/schematic -test.installer-external-repository=registry.dev.siderolabs.io/siderolabs -test.installer-internal-repository=registry.dev.siderolabs.io/siderolabs -test.cache-repository=registry.dev.siderolabs.io/image-factory/cache RUN_TESTS: TestIntegrationCDN
TEST_FLAGS: -test.schematic-service-repository=registry.dev.siderolabs.io/image-factory/schematic -test.installer-external-repository=registry.dev.siderolabs.io/siderolabs -test.installer-internal-repository=registry.dev.siderolabs.io/siderolabs -test.cache-repository=registry.dev.siderolabs.io/image-factory/cache -test.signing-cache-repository=127.0.0.1:5100/image-factory/signing-cache
run: | run: |
make integration-talos-main make integration-cdn-talos-main
integration-direct-talos-main:
runs-on:
- self-hosted
- generic
if: contains(fromJSON(needs.default.outputs.labels), 'integration/talos-main')
needs:
- default
steps:
- name: gather-system-info
id: system-info
uses: kenchan0130/actions-system-info@v1.3.1
continue-on-error: true
- name: print-system-info
run: |
MEMORY_GB=$((${{ steps.system-info.outputs.totalmem }}/1024/1024/1024))
OUTPUTS=(
"CPU Core: ${{ steps.system-info.outputs.cpu-core }}"
"CPU Model: ${{ steps.system-info.outputs.cpu-model }}"
"Hostname: ${{ steps.system-info.outputs.hostname }}"
"NodeName: ${NODE_NAME}"
"Kernel release: ${{ steps.system-info.outputs.kernel-release }}"
"Kernel version: ${{ steps.system-info.outputs.kernel-version }}"
"Name: ${{ steps.system-info.outputs.name }}"
"Platform: ${{ steps.system-info.outputs.platform }}"
"Release: ${{ steps.system-info.outputs.release }}"
"Total memory: ${MEMORY_GB} GB"
)
for OUTPUT in "${OUTPUTS[@]}";do
echo "${OUTPUT}"
done
continue-on-error: true
- name: checkout
uses: actions/checkout@v4
- name: Unshallow
run: |
git fetch --prune --unshallow
- name: Set up Docker Buildx
id: setup-buildx
uses: docker/setup-buildx-action@v3
with:
driver: remote
endpoint: tcp://buildkit-amd64.ci.svc.cluster.local:1234
timeout-minutes: 10
- name: integration-direct-talos-main
env:
REGISTRY: registry.dev.siderolabs.io
RUN_TESTS: TestIntegrationDirect
TEST_FLAGS: -test.schematic-service-repository=registry.dev.siderolabs.io/image-factory/schematic -test.installer-external-repository=registry.dev.siderolabs.io/siderolabs -test.installer-internal-repository=registry.dev.siderolabs.io/siderolabs -test.cache-repository=registry.dev.siderolabs.io/image-factory/cache -test.signing-cache-repository=127.0.0.1:5100/image-factory/signing-cache
run: |
make integration-direct-talos-main
integration-s3-talos-main:
runs-on:
- self-hosted
- generic
if: contains(fromJSON(needs.default.outputs.labels), 'integration/talos-main')
needs:
- default
steps:
- name: gather-system-info
id: system-info
uses: kenchan0130/actions-system-info@v1.3.1
continue-on-error: true
- name: print-system-info
run: |
MEMORY_GB=$((${{ steps.system-info.outputs.totalmem }}/1024/1024/1024))
OUTPUTS=(
"CPU Core: ${{ steps.system-info.outputs.cpu-core }}"
"CPU Model: ${{ steps.system-info.outputs.cpu-model }}"
"Hostname: ${{ steps.system-info.outputs.hostname }}"
"NodeName: ${NODE_NAME}"
"Kernel release: ${{ steps.system-info.outputs.kernel-release }}"
"Kernel version: ${{ steps.system-info.outputs.kernel-version }}"
"Name: ${{ steps.system-info.outputs.name }}"
"Platform: ${{ steps.system-info.outputs.platform }}"
"Release: ${{ steps.system-info.outputs.release }}"
"Total memory: ${MEMORY_GB} GB"
)
for OUTPUT in "${OUTPUTS[@]}";do
echo "${OUTPUT}"
done
continue-on-error: true
- name: checkout
uses: actions/checkout@v4
- name: Unshallow
run: |
git fetch --prune --unshallow
- name: Set up Docker Buildx
id: setup-buildx
uses: docker/setup-buildx-action@v3
with:
driver: remote
endpoint: tcp://buildkit-amd64.ci.svc.cluster.local:1234
timeout-minutes: 10
- name: integration-s3-talos-main
env:
REGISTRY: registry.dev.siderolabs.io
RUN_TESTS: TestIntegrationS3
TEST_FLAGS: -test.schematic-service-repository=registry.dev.siderolabs.io/image-factory/schematic -test.installer-external-repository=registry.dev.siderolabs.io/siderolabs -test.installer-internal-repository=registry.dev.siderolabs.io/siderolabs -test.cache-repository=registry.dev.siderolabs.io/image-factory/cache -test.signing-cache-repository=127.0.0.1:5100/image-factory/signing-cache
run: |
make integration-s3-talos-main

View File

@ -0,0 +1,61 @@
# THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
#
# Generated on 2025-08-13T15:31:47Z by kres 9f63e23.
concurrency:
group: ${{ github.head_ref || github.run_id }}
cancel-in-progress: true
"on":
schedule:
- cron: 30 7 * * *
name: integration-cdn-talos-main-cron
jobs:
default:
runs-on:
- self-hosted
- generic
steps:
- name: gather-system-info
id: system-info
uses: kenchan0130/actions-system-info@v1.3.1
continue-on-error: true
- name: print-system-info
run: |
MEMORY_GB=$((${{ steps.system-info.outputs.totalmem }}/1024/1024/1024))
OUTPUTS=(
"CPU Core: ${{ steps.system-info.outputs.cpu-core }}"
"CPU Model: ${{ steps.system-info.outputs.cpu-model }}"
"Hostname: ${{ steps.system-info.outputs.hostname }}"
"NodeName: ${NODE_NAME}"
"Kernel release: ${{ steps.system-info.outputs.kernel-release }}"
"Kernel version: ${{ steps.system-info.outputs.kernel-version }}"
"Name: ${{ steps.system-info.outputs.name }}"
"Platform: ${{ steps.system-info.outputs.platform }}"
"Release: ${{ steps.system-info.outputs.release }}"
"Total memory: ${MEMORY_GB} GB"
)
for OUTPUT in "${OUTPUTS[@]}";do
echo "${OUTPUT}"
done
continue-on-error: true
- name: checkout
uses: actions/checkout@v4
- name: Unshallow
run: |
git fetch --prune --unshallow
- name: Set up Docker Buildx
id: setup-buildx
uses: docker/setup-buildx-action@v3
with:
driver: remote
endpoint: tcp://buildkit-amd64.ci.svc.cluster.local:1234
timeout-minutes: 10
- name: integration-cdn-talos-main
env:
REGISTRY: registry.dev.siderolabs.io
RUN_TESTS: TestIntegrationCDN
TEST_FLAGS: -test.schematic-service-repository=registry.dev.siderolabs.io/image-factory/schematic -test.installer-external-repository=registry.dev.siderolabs.io/siderolabs -test.installer-internal-repository=registry.dev.siderolabs.io/siderolabs -test.cache-repository=registry.dev.siderolabs.io/image-factory/cache -test.signing-cache-repository=127.0.0.1:5100/image-factory/signing-cache
run: |
make integration-cdn-talos-main

View File

@ -0,0 +1,61 @@
# THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
#
# Generated on 2025-08-13T15:31:47Z by kres 9f63e23.
concurrency:
group: ${{ github.head_ref || github.run_id }}
cancel-in-progress: true
"on":
schedule:
- cron: 30 7 * * *
name: integration-direct-talos-main-cron
jobs:
default:
runs-on:
- self-hosted
- generic
steps:
- name: gather-system-info
id: system-info
uses: kenchan0130/actions-system-info@v1.3.1
continue-on-error: true
- name: print-system-info
run: |
MEMORY_GB=$((${{ steps.system-info.outputs.totalmem }}/1024/1024/1024))
OUTPUTS=(
"CPU Core: ${{ steps.system-info.outputs.cpu-core }}"
"CPU Model: ${{ steps.system-info.outputs.cpu-model }}"
"Hostname: ${{ steps.system-info.outputs.hostname }}"
"NodeName: ${NODE_NAME}"
"Kernel release: ${{ steps.system-info.outputs.kernel-release }}"
"Kernel version: ${{ steps.system-info.outputs.kernel-version }}"
"Name: ${{ steps.system-info.outputs.name }}"
"Platform: ${{ steps.system-info.outputs.platform }}"
"Release: ${{ steps.system-info.outputs.release }}"
"Total memory: ${MEMORY_GB} GB"
)
for OUTPUT in "${OUTPUTS[@]}";do
echo "${OUTPUT}"
done
continue-on-error: true
- name: checkout
uses: actions/checkout@v4
- name: Unshallow
run: |
git fetch --prune --unshallow
- name: Set up Docker Buildx
id: setup-buildx
uses: docker/setup-buildx-action@v3
with:
driver: remote
endpoint: tcp://buildkit-amd64.ci.svc.cluster.local:1234
timeout-minutes: 10
- name: integration-direct-talos-main
env:
REGISTRY: registry.dev.siderolabs.io
RUN_TESTS: TestIntegrationDirect
TEST_FLAGS: -test.schematic-service-repository=registry.dev.siderolabs.io/image-factory/schematic -test.installer-external-repository=registry.dev.siderolabs.io/siderolabs -test.installer-internal-repository=registry.dev.siderolabs.io/siderolabs -test.cache-repository=registry.dev.siderolabs.io/image-factory/cache -test.signing-cache-repository=127.0.0.1:5100/image-factory/signing-cache
run: |
make integration-direct-talos-main

View File

@ -1,6 +1,6 @@
# THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
# #
# Generated on 2025-07-22T11:53:08Z by kres b869533. # Generated on 2025-08-13T15:31:47Z by kres 9f63e23.
concurrency: concurrency:
group: ${{ github.head_ref || github.run_id }} group: ${{ github.head_ref || github.run_id }}
@ -8,7 +8,7 @@ concurrency:
"on": "on":
schedule: schedule:
- cron: 30 7 * * * - cron: 30 7 * * *
name: integration-talos-main-cron name: integration-s3-talos-main-cron
jobs: jobs:
default: default:
runs-on: runs-on:
@ -52,9 +52,10 @@ jobs:
driver: remote driver: remote
endpoint: tcp://buildkit-amd64.ci.svc.cluster.local:1234 endpoint: tcp://buildkit-amd64.ci.svc.cluster.local:1234
timeout-minutes: 10 timeout-minutes: 10
- name: integration-talos-main - name: integration-s3-talos-main
env: env:
REGISTRY: registry.dev.siderolabs.io REGISTRY: registry.dev.siderolabs.io
TEST_FLAGS: -test.schematic-service-repository=registry.dev.siderolabs.io/image-factory/schematic -test.installer-external-repository=registry.dev.siderolabs.io/siderolabs -test.installer-internal-repository=registry.dev.siderolabs.io/siderolabs -test.cache-repository=registry.dev.siderolabs.io/image-factory/cache RUN_TESTS: TestIntegrationS3
TEST_FLAGS: -test.schematic-service-repository=registry.dev.siderolabs.io/image-factory/schematic -test.installer-external-repository=registry.dev.siderolabs.io/siderolabs -test.installer-internal-repository=registry.dev.siderolabs.io/siderolabs -test.cache-repository=registry.dev.siderolabs.io/image-factory/cache -test.signing-cache-repository=127.0.0.1:5100/image-factory/signing-cache
run: | run: |
make integration-talos-main make integration-s3-talos-main

View File

@ -1,6 +1,6 @@
# THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
# #
# Generated on 2025-07-22T11:53:08Z by kres b869533. # Generated on 2025-08-13T14:42:25Z by kres 9f63e23.
"on": "on":
schedule: schedule:

View File

@ -0,0 +1,92 @@
# THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
#
# Generated on 2025-08-13T14:42:25Z by kres 9f63e23.
"on":
workflow_run:
workflows:
- default
- integration-cdn-talos-main-cron
- integration-direct-talos-main-cron
- integration-s3-talos-main-cron
types:
- completed
branches:
- main
name: slack-notify-failure
jobs:
slack-notify:
runs-on:
- self-hosted
- generic
if: github.event.workflow_run.conclusion == 'failure' && github.event.workflow_run.event != 'pull_request'
steps:
- name: Slack Notify
uses: slackapi/slack-github-action@v2
with:
method: chat.postMessage
payload: |
{
"channel": "ci-failure",
"text": "${{ github.event.workflow_run.conclusion }} - ${{ github.repository }}",
"icon_emoji": "${{ github.event.workflow_run.conclusion == 'success' && ':white_check_mark:' || github.event.workflow_run.conclusion == 'failure' && ':x:' || ':warning:' }}",
"username": "GitHub Actions",
"attachments": [
{
"blocks": [
{
"fields": [
{
"text": "${{ github.event.workflow_run.event == 'pull_request' && format('*Pull Request:* {0} (`{1}`)\n<{2}/pull/{3}|{4}>', github.repository, github.ref_name, github.event.repository.html_url, steps.get-pr-number.outputs.pull_request_number, github.event.workflow_run.display_title) || format('*Build:* {0} (`{1}`)\n<{2}/commit/{3}|{4}>', github.repository, github.ref_name, github.event.repository.html_url, github.sha, github.event.workflow_run.display_title) }}",
"type": "mrkdwn"
},
{
"text": "*Status:*\n`${{ github.event.workflow_run.conclusion }}`",
"type": "mrkdwn"
}
],
"type": "section"
},
{
"fields": [
{
"text": "*Author:*\n`${{ github.actor }}`",
"type": "mrkdwn"
},
{
"text": "*Event:*\n`${{ github.event.workflow_run.event }}`",
"type": "mrkdwn"
}
],
"type": "section"
},
{
"type": "divider"
},
{
"elements": [
{
"text": {
"text": "Logs",
"type": "plain_text"
},
"type": "button",
"url": "${{ github.event.workflow_run.html_url }}"
},
{
"text": {
"text": "Commit",
"type": "plain_text"
},
"type": "button",
"url": "${{ github.event.repository.html_url }}/commit/${{ github.sha }}"
}
],
"type": "actions"
}
],
"color": "${{ github.event.workflow_run.conclusion == 'success' && '#2EB886' || github.event.workflow_run.conclusion == 'failure' && '#A30002' || '#FFCC00' }}"
}
]
}
token: ${{ secrets.SLACK_BOT_TOKEN_V2 }}

View File

@ -1,12 +1,14 @@
# THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
# #
# Generated on 2025-07-22T11:53:08Z by kres b869533. # Generated on 2025-08-13T14:42:25Z by kres 9f63e23.
"on": "on":
workflow_run: workflow_run:
workflows: workflows:
- default - default
- integration-talos-main-cron - integration-cdn-talos-main-cron
- integration-direct-talos-main-cron
- integration-s3-talos-main-cron
types: types:
- completed - completed
name: slack-notify name: slack-notify
@ -30,64 +32,66 @@ jobs:
method: chat.postMessage method: chat.postMessage
payload: | payload: |
{ {
"channel": "proj-talos-maintainers", "channel": "ci-all",
"text": "${{ github.event.workflow_run.conclusion }} - ${{ github.repository }}",
"icon_emoji": "${{ github.event.workflow_run.conclusion == 'success' && ':white_check_mark:' || github.event.workflow_run.conclusion == 'failure' && ':x:' || ':warning:' }}",
"username": "GitHub Actions",
"attachments": [ "attachments": [
{ {
"color": "${{ github.event.workflow_run.conclusion == 'success' && '#2EB886' || github.event.workflow_run.conclusion == 'failure' && '#A30002' || '#FFCC00' }}",
"fallback": "test",
"blocks": [ "blocks": [
{ {
"type": "section",
"fields": [ "fields": [
{ {
"type": "mrkdwn", "text": "${{ github.event.workflow_run.event == 'pull_request' && format('*Pull Request:* {0} (`{1}`)\n<{2}/pull/{3}|{4}>', github.repository, github.ref_name, github.event.repository.html_url, steps.get-pr-number.outputs.pull_request_number, github.event.workflow_run.display_title) || format('*Build:* {0} (`{1}`)\n<{2}/commit/{3}|{4}>', github.repository, github.ref_name, github.event.repository.html_url, github.sha, github.event.workflow_run.display_title) }}",
"text": "${{ github.event.workflow_run.event == 'pull_request' && format('*Pull Request:* {0} (`{1}`)\n<{2}/pull/{3}|{4}>', github.repository, github.ref_name, github.event.repository.html_url, steps.get-pr-number.outputs.pull_request_number, github.event.workflow_run.display_title) || format('*Build:* {0} (`{1}`)\n<{2}/commit/{3}|{4}>', github.repository, github.ref_name, github.event.repository.html_url, github.sha, github.event.workflow_run.display_title) }}" "type": "mrkdwn"
}, },
{ {
"type": "mrkdwn", "text": "*Status:*\n`${{ github.event.workflow_run.conclusion }}`",
"text": "*Status:*\n`${{ github.event.workflow_run.conclusion }}`" "type": "mrkdwn"
} }
] ],
"type": "section"
}, },
{ {
"type": "section",
"fields": [ "fields": [
{ {
"type": "mrkdwn", "text": "*Author:*\n`${{ github.actor }}`",
"text": "*Author:*\n`${{ github.actor }}`" "type": "mrkdwn"
}, },
{ {
"type": "mrkdwn", "text": "*Event:*\n`${{ github.event.workflow_run.event }}`",
"text": "*Event:*\n`${{ github.event.workflow_run.event }}`" "type": "mrkdwn"
} }
] ],
"type": "section"
}, },
{ {
"type": "divider" "type": "divider"
}, },
{ {
"type": "actions",
"elements": [ "elements": [
{ {
"type": "button",
"text": { "text": {
"type": "plain_text", "text": "Logs",
"text": "Logs" "type": "plain_text"
}, },
"type": "button",
"url": "${{ github.event.workflow_run.html_url }}" "url": "${{ github.event.workflow_run.html_url }}"
}, },
{ {
"type": "button",
"text": { "text": {
"type": "plain_text", "text": "Commit",
"text": "Commit" "type": "plain_text"
}, },
"type": "button",
"url": "${{ github.event.repository.html_url }}/commit/${{ github.sha }}" "url": "${{ github.event.repository.html_url }}/commit/${{ github.sha }}"
} }
] ],
"type": "actions"
} }
] ],
"color": "${{ github.event.workflow_run.conclusion == 'success' && '#2EB886' || github.event.workflow_run.conclusion == 'failure' && '#A30002' || '#FFCC00' }}"
} }
] ]
} }
token: ${{ secrets.SLACK_BOT_TOKEN }} token: ${{ secrets.SLACK_BOT_TOKEN_V2 }}

View File

@ -1,6 +1,6 @@
# THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
# #
# Generated on 2025-07-22T11:53:08Z by kres b869533. # Generated on 2025-08-13T14:42:25Z by kres 9f63e23.
"on": "on":
schedule: schedule:

View File

@ -2,6 +2,11 @@ kind: golang.Generate
spec: spec:
versionPackagePath: internal/version versionPackagePath: internal/version
--- ---
kind: golang.GoVulnCheck
spec:
ignore:
- GO-2025-3770
---
kind: golang.Build kind: golang.Build
spec: spec:
outputs: outputs:
@ -55,15 +60,31 @@ spec:
toplevel: true toplevel: true
- name: imager-tools - name: imager-tools
toplevel: true toplevel: true
- name: integration.test - name: integration-cdn.test
toplevel: true toplevel: true
- name: integration - name: integration-direct.test
toplevel: true
- name: integration-s3.test
toplevel: true
- name: integration-direct
toplevel: true
dependants:
- coverage
- name: integration-s3
toplevel: true
dependants:
- coverage
- name: integration-cdn
toplevel: true toplevel: true
dependants: dependants:
- coverage - coverage
- name: update-to-talos-main - name: update-to-talos-main
toplevel: true toplevel: true
- name: integration-talos-main - name: integration-cdn-talos-main
toplevel: true
- name: integration-direct-talos-main
toplevel: true
- name: integration-s3-talos-main
toplevel: true toplevel: true
- name: tailwind - name: tailwind
toplevel: true toplevel: true
@ -280,27 +301,83 @@ spec:
dst: / dst: /
--- ---
kind: custom.Step kind: custom.Step
name: integration.test name: integration-direct.test
spec: spec:
docker: docker:
enabled: true enabled: true
stages: stages:
- name: integration-build - name: integration-direct-build
description: builds the integration test binary description: builds the integration test binary
from: base from: base
steps: steps:
- script: - script:
command: go test -c -covermode=atomic -coverpkg=./... -tags integration ./internal/integration command: go test -c -covermode=atomic -coverpkg=./... -tags integration,integration_direct ./internal/integration
cache: cache:
- /root/.cache/go-build - /root/.cache/go-build
- /go/pkg - /go/pkg
- name: integration.test - name: integration-direct.test
description: copies out the integration test binary description: copies out the integration test binary
steps: steps:
- copy: - copy:
from: integration-build from: integration-direct-build
src: /src/integration.test src: /src/integration.test
dst: /integration.test dst: /integration-direct.test
makefile:
enabled: true
phony: true
script:
- "@$(MAKE) local-$@ DEST=$(ARTIFACTS)"
---
kind: custom.Step
name: integration-s3.test
spec:
docker:
enabled: true
stages:
- name: integration-s3-build
description: builds the integration test binary
from: base
steps:
- script:
command: go test -c -covermode=atomic -coverpkg=./... -tags integration,integration_s3 ./internal/integration
cache:
- /root/.cache/go-build
- /go/pkg
- name: integration-s3.test
description: copies out the integration test binary
steps:
- copy:
from: integration-s3-build
src: /src/integration.test
dst: /integration-s3.test
makefile:
enabled: true
phony: true
script:
- "@$(MAKE) local-$@ DEST=$(ARTIFACTS)"
---
kind: custom.Step
name: integration-cdn.test
spec:
docker:
enabled: true
stages:
- name: integration-cdn-build
description: builds the integration test binary
from: base
steps:
- script:
command: go test -c -covermode=atomic -coverpkg=./... -tags integration,integration_cdn ./internal/integration
cache:
- /root/.cache/go-build
- /go/pkg
- name: integration-cdn.test
description: copies out the integration test binary
steps:
- copy:
from: integration-cdn-build
src: /src/integration.test
dst: /integration-cdn.test
makefile: makefile:
enabled: true enabled: true
phony: true phony: true
@ -345,16 +422,62 @@ spec:
- '@$(MAKE) local-copy-out-go-mod DEST=. TARGET_ARGS="--no-cache-filter=update-to-talos-main"' - '@$(MAKE) local-copy-out-go-mod DEST=. TARGET_ARGS="--no-cache-filter=update-to-talos-main"'
--- ---
kind: custom.Step kind: custom.Step
name: integration name: integration-direct
spec: spec:
makefile: makefile:
enabled: true enabled: true
phony: true phony: true
depends: depends:
- integration.test - integration-direct.test
script:
- "@$(MAKE) image-image-factory PUSH=true"
- docker pull $(REGISTRY)/$(USERNAME)/image-factory:$(TAG)
- docker rm -f local-if || true
- docker run -d -p 5100:5000 --name=local-if registry:3
- docker run --rm --net=host --privileged -v /dev:/dev -v /var/run:/var/run -v $(PWD)/$(ARTIFACTS)/:/out/ -v $(PWD)/$(ARTIFACTS)/integration-direct.test:/bin/integration-direct.test:ro --entrypoint /bin/integration-direct.test $(REGISTRY)/$(USERNAME)/image-factory:$(TAG) -test.v $(TEST_FLAGS) -test.coverprofile=/out/coverage-integration-direct.txt -test.run $(RUN_TESTS)
- docker rm -f local-if
ghaction:
enabled: true
condition: on-pull-request
environment:
REGISTRY: registry.dev.siderolabs.io
RUN_TESTS: TestIntegrationDirect
TEST_FLAGS: "-test.schematic-service-repository=127.0.0.1:5100/image-factory/schematic -test.installer-external-repository=127.0.0.1:5100/siderolabs -test.installer-internal-repository=127.0.0.1:5100/siderolabs -test.cache-repository=127.0.0.1:5100/image-factory/cache -test.signing-cache-repository=127.0.0.1:5100/image-factory/signing-cache"
---
kind: custom.Step
name: integration-s3
spec:
makefile:
enabled: true
phony: true
depends:
- integration-s3.test
script:
- "@$(MAKE) image-image-factory PUSH=true"
- docker pull $(REGISTRY)/$(USERNAME)/image-factory:$(TAG)
- docker rm -f local-if || true
- docker run -d -p 5100:5000 --name=local-if registry:3
- docker run --rm --net=host --privileged -v /dev:/dev -v /var/run:/var/run -v $(PWD)/$(ARTIFACTS)/:/out/ -v $(PWD)/$(ARTIFACTS)/integration-s3.test:/bin/integration-s3.test:ro --entrypoint /bin/integration-s3.test $(REGISTRY)/$(USERNAME)/image-factory:$(TAG) -test.v $(TEST_FLAGS) -test.coverprofile=/out/coverage-integration-s3.txt -test.run $(RUN_TESTS)
- docker rm -f local-if
ghaction:
enabled: true
condition: on-pull-request
environment:
REGISTRY: registry.dev.siderolabs.io
RUN_TESTS: TestIntegrationS3
TEST_FLAGS: "-test.schematic-service-repository=127.0.0.1:5100/image-factory/schematic -test.installer-external-repository=127.0.0.1:5100/siderolabs -test.installer-internal-repository=127.0.0.1:5100/siderolabs -test.cache-repository=127.0.0.1:5100/image-factory/cache -test.signing-cache-repository=127.0.0.1:5100/image-factory/signing-cache"
---
kind: custom.Step
name: integration-cdn
spec:
makefile:
enabled: true
phony: true
depends:
- integration-cdn.test
variables: variables:
- name: RUN_TESTS - name: RUN_TESTS
defaultValue: TestIntegration defaultValue: TestIntegrationCDN
- name: TEST_FLAGS - name: TEST_FLAGS
defaultValue: "" defaultValue: ""
script: script:
@ -362,38 +485,87 @@ spec:
- docker pull $(REGISTRY)/$(USERNAME)/image-factory:$(TAG) - docker pull $(REGISTRY)/$(USERNAME)/image-factory:$(TAG)
- docker rm -f local-if || true - docker rm -f local-if || true
- docker run -d -p 5100:5000 --name=local-if registry:3 - docker run -d -p 5100:5000 --name=local-if registry:3
- docker run --rm --net=host --privileged -v /dev:/dev -v /var/run:/var/run -v $(PWD)/$(ARTIFACTS)/:/out/ -v $(PWD)/$(ARTIFACTS)/integration.test:/bin/integration.test:ro --entrypoint /bin/integration.test $(REGISTRY)/$(USERNAME)/image-factory:$(TAG) -test.v $(TEST_FLAGS) -test.coverprofile=/out/coverage-integration.txt -test.run $(RUN_TESTS) - docker run --rm --net=host --privileged -v /dev:/dev -v /var/run:/var/run -v $(PWD)/$(ARTIFACTS)/:/out/ -v $(PWD)/$(ARTIFACTS)/integration-cdn.test:/bin/integration-cdn.test:ro --entrypoint /bin/integration-cdn.test $(REGISTRY)/$(USERNAME)/image-factory:$(TAG) -test.v $(TEST_FLAGS) -test.coverprofile=/out/coverage-integration-cdn.txt -test.run $(RUN_TESTS)
- docker rm -f local-if - docker rm -f local-if
ghaction: ghaction:
enabled: true enabled: true
condition: on-pull-request condition: on-pull-request
environment: environment:
REGISTRY: registry.dev.siderolabs.io REGISTRY: registry.dev.siderolabs.io
TEST_FLAGS: "-test.schematic-service-repository=127.0.0.1:5100/image-factory/schematic -test.installer-external-repository=127.0.0.1:5100/siderolabs -test.installer-internal-repository=127.0.0.1:5100/siderolabs -test.cache-repository=127.0.0.1:5100/image-factory/cache" RUN_TESTS: TestIntegrationCDN
TEST_FLAGS: "-test.schematic-service-repository=127.0.0.1:5100/image-factory/schematic -test.installer-external-repository=127.0.0.1:5100/siderolabs -test.installer-internal-repository=127.0.0.1:5100/siderolabs -test.cache-repository=127.0.0.1:5100/image-factory/cache -test.signing-cache-repository=127.0.0.1:5100/image-factory/signing-cache"
--- ---
kind: custom.Step kind: custom.Step
name: integration-talos-main name: integration-direct-talos-main
spec: spec:
makefile: makefile:
enabled: true enabled: true
phony: true phony: true
depends: depends:
- update-to-talos-main - update-to-talos-main
variables:
- name: RUN_TESTS
defaultValue: TestIntegration
- name: TEST_FLAGS
defaultValue: ""
script: script:
- "@$(MAKE) integration" - "@$(MAKE) integration-direct"
ghaction: ghaction:
enabled: true enabled: true
cronOnly: true cronOnly: true
environment: environment:
REGISTRY: registry.dev.siderolabs.io REGISTRY: registry.dev.siderolabs.io
TEST_FLAGS: "-test.schematic-service-repository=registry.dev.siderolabs.io/image-factory/schematic -test.installer-external-repository=registry.dev.siderolabs.io/siderolabs -test.installer-internal-repository=registry.dev.siderolabs.io/siderolabs -test.cache-repository=registry.dev.siderolabs.io/image-factory/cache" RUN_TESTS: TestIntegrationDirect
TEST_FLAGS: "-test.schematic-service-repository=registry.dev.siderolabs.io/image-factory/schematic -test.installer-external-repository=registry.dev.siderolabs.io/siderolabs -test.installer-internal-repository=registry.dev.siderolabs.io/siderolabs -test.cache-repository=registry.dev.siderolabs.io/image-factory/cache -test.signing-cache-repository=127.0.0.1:5100/image-factory/signing-cache"
jobs: jobs:
- name: integration-talos-main - name: integration-direct-talos-main
runnerLabels:
- generic
triggerLabels:
- integration/talos-main
crons:
- "30 7 * * *"
---
kind: custom.Step
name: integration-s3-talos-main
spec:
makefile:
enabled: true
phony: true
depends:
- update-to-talos-main
script:
- "@$(MAKE) integration-s3"
ghaction:
enabled: true
cronOnly: true
environment:
REGISTRY: registry.dev.siderolabs.io
RUN_TESTS: TestIntegrationS3
TEST_FLAGS: "-test.schematic-service-repository=registry.dev.siderolabs.io/image-factory/schematic -test.installer-external-repository=registry.dev.siderolabs.io/siderolabs -test.installer-internal-repository=registry.dev.siderolabs.io/siderolabs -test.cache-repository=registry.dev.siderolabs.io/image-factory/cache -test.signing-cache-repository=127.0.0.1:5100/image-factory/signing-cache"
jobs:
- name: integration-s3-talos-main
runnerLabels:
- generic
triggerLabels:
- integration/talos-main
crons:
- "30 7 * * *"
---
kind: custom.Step
name: integration-cdn-talos-main
spec:
makefile:
enabled: true
phony: true
depends:
- update-to-talos-main
script:
- "@$(MAKE) integration-cdn"
ghaction:
enabled: true
cronOnly: true
environment:
REGISTRY: registry.dev.siderolabs.io
RUN_TESTS: TestIntegrationCDN
TEST_FLAGS: "-test.schematic-service-repository=registry.dev.siderolabs.io/image-factory/schematic -test.installer-external-repository=registry.dev.siderolabs.io/siderolabs -test.installer-internal-repository=registry.dev.siderolabs.io/siderolabs -test.cache-repository=registry.dev.siderolabs.io/image-factory/cache -test.signing-cache-repository=127.0.0.1:5100/image-factory/signing-cache"
jobs:
- name: integration-cdn-talos-main
runnerLabels: runnerLabels:
- generic - generic
triggerLabels: triggerLabels:
@ -410,7 +582,9 @@ kind: service.CodeCov
spec: spec:
targetThreshold: 50 targetThreshold: 50
inputPaths: inputPaths:
- coverage-integration.txt - coverage-integration-direct.txt
- coverage-integration-s3.txt
- coverage-integration-cdn.txt
--- ---
kind: custom.Step kind: custom.Step
name: tailwind name: tailwind
@ -434,8 +608,6 @@ spec:
src: package.json package-lock.json src: package.json package-lock.json
dst: . dst: .
- script: - script:
cache:
- /src/node_modules
command: bun install command: bun install
- name: tailwind-update - name: tailwind-update
description: "tailwind update" description: "tailwind update"
@ -448,8 +620,6 @@ spec:
src: internal/frontend/http src: internal/frontend/http
dst: internal/frontend/http dst: internal/frontend/http
- script: - script:
cache:
- /src/node_modules
command: node_modules/.bin/tailwindcss -i internal/frontend/http/css/input.css -o internal/frontend/http/css/output.css --minify command: node_modules/.bin/tailwindcss -i internal/frontend/http/css/input.css -o internal/frontend/http/css/output.css --minify
- name: tailwind-copy - name: tailwind-copy
description: "Copies assets" description: "Copies assets"

View File

@ -2,7 +2,7 @@
# THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
# #
# Generated on 2025-07-29T12:16:54Z by kres dd1ed6f. # Generated on 2025-08-14T09:35:32Z by kres df7e867-dirty.
ARG TOOLCHAIN ARG TOOLCHAIN
ARG PKGS_PREFIX ARG PKGS_PREFIX
@ -83,11 +83,11 @@ FROM ${PKGS_PREFIX}/zstd:${PKGS} AS pkg-zstd
FROM --platform=${BUILDPLATFORM} docker.io/oven/bun:1.2.4-alpine AS tailwind-base FROM --platform=${BUILDPLATFORM} docker.io/oven/bun:1.2.4-alpine AS tailwind-base
WORKDIR /src WORKDIR /src
COPY package.json package-lock.json . COPY package.json package-lock.json .
RUN --mount=type=cache,target=/src/node_modules,id=image-factory/src/node_modules bun install RUN bun install
# base toolchain image # base toolchain image
FROM --platform=${BUILDPLATFORM} ${TOOLCHAIN} AS toolchain FROM --platform=${BUILDPLATFORM} ${TOOLCHAIN} AS toolchain
RUN apk --update --no-cache add bash curl build-base jq protoc protobuf-dev RUN apk --update --no-cache add bash build-base curl jq protoc protobuf-dev
# copies the imager tools # copies the imager tools
FROM scratch AS imager-tools FROM scratch AS imager-tools
@ -127,7 +127,7 @@ COPY --from=pkg-zstd / /
FROM tailwind-base AS tailwind-update FROM tailwind-base AS tailwind-update
COPY tailwind.config.js . COPY tailwind.config.js .
COPY internal/frontend/http internal/frontend/http COPY internal/frontend/http internal/frontend/http
RUN --mount=type=cache,target=/src/node_modules,id=image-factory/src/node_modules node_modules/.bin/tailwindcss -i internal/frontend/http/css/input.css -o internal/frontend/http/css/output.css --minify RUN node_modules/.bin/tailwindcss -i internal/frontend/http/css/input.css -o internal/frontend/http/css/output.css --minify
# build tools # build tools
FROM --platform=${BUILDPLATFORM} toolchain AS tools FROM --platform=${BUILDPLATFORM} toolchain AS tools
@ -141,15 +141,15 @@ ENV GOEXPERIMENT=${GOEXPERIMENT}
ENV GOPATH=/go ENV GOPATH=/go
ARG DEEPCOPY_VERSION ARG DEEPCOPY_VERSION
RUN --mount=type=cache,target=/root/.cache/go-build,id=image-factory/root/.cache/go-build --mount=type=cache,target=/go/pkg,id=image-factory/go/pkg go install github.com/siderolabs/deep-copy@${DEEPCOPY_VERSION} \ RUN --mount=type=cache,target=/root/.cache/go-build,id=image-factory/root/.cache/go-build --mount=type=cache,target=/go/pkg,id=image-factory/go/pkg go install github.com/siderolabs/deep-copy@${DEEPCOPY_VERSION} \
&& mv /go/bin/deep-copy /bin/deep-copy && mv /go/bin/deep-copy /bin/deep-copy
ARG GOLANGCILINT_VERSION ARG GOLANGCILINT_VERSION
RUN --mount=type=cache,target=/root/.cache/go-build,id=image-factory/root/.cache/go-build --mount=type=cache,target=/go/pkg,id=image-factory/go/pkg go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@${GOLANGCILINT_VERSION} \ RUN --mount=type=cache,target=/root/.cache/go-build,id=image-factory/root/.cache/go-build --mount=type=cache,target=/go/pkg,id=image-factory/go/pkg go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@${GOLANGCILINT_VERSION} \
&& mv /go/bin/golangci-lint /bin/golangci-lint && mv /go/bin/golangci-lint /bin/golangci-lint
RUN --mount=type=cache,target=/root/.cache/go-build,id=image-factory/root/.cache/go-build --mount=type=cache,target=/go/pkg,id=image-factory/go/pkg go install golang.org/x/vuln/cmd/govulncheck@latest \ RUN --mount=type=cache,target=/root/.cache/go-build,id=image-factory/root/.cache/go-build --mount=type=cache,target=/go/pkg,id=image-factory/go/pkg go install golang.org/x/vuln/cmd/govulncheck@latest \
&& mv /go/bin/govulncheck /bin/govulncheck && mv /go/bin/govulncheck /bin/govulncheck
ARG GOFUMPT_VERSION ARG GOFUMPT_VERSION
RUN go install mvdan.cc/gofumpt@${GOFUMPT_VERSION} \ RUN go install mvdan.cc/gofumpt@${GOFUMPT_VERSION} \
&& mv /go/bin/gofumpt /bin/gofumpt && mv /go/bin/gofumpt /bin/gofumpt
# Copies assets # Copies assets
FROM scratch AS tailwind-copy FROM scratch AS tailwind-copy
@ -177,8 +177,16 @@ RUN mkdir -p internal/version/data && \
echo -n ${TAG} > internal/version/data/tag echo -n ${TAG} > internal/version/data/tag
# builds the integration test binary # builds the integration test binary
FROM base AS integration-build FROM base AS integration-cdn-build
RUN --mount=type=cache,target=/root/.cache/go-build,id=image-factory/root/.cache/go-build --mount=type=cache,target=/go/pkg,id=image-factory/go/pkg go test -c -covermode=atomic -coverpkg=./... -tags integration ./internal/integration RUN --mount=type=cache,target=/root/.cache/go-build,id=image-factory/root/.cache/go-build --mount=type=cache,target=/go/pkg,id=image-factory/go/pkg go test -c -covermode=atomic -coverpkg=./... -tags integration,integration_cdn ./internal/integration
# builds the integration test binary
FROM base AS integration-direct-build
RUN --mount=type=cache,target=/root/.cache/go-build,id=image-factory/root/.cache/go-build --mount=type=cache,target=/go/pkg,id=image-factory/go/pkg go test -c -covermode=atomic -coverpkg=./... -tags integration,integration_direct ./internal/integration
# builds the integration test binary
FROM base AS integration-s3-build
RUN --mount=type=cache,target=/root/.cache/go-build,id=image-factory/root/.cache/go-build --mount=type=cache,target=/go/pkg,id=image-factory/go/pkg go test -c -covermode=atomic -coverpkg=./... -tags integration,integration_s3 ./internal/integration
# runs gofumpt # runs gofumpt
FROM base AS lint-gofumpt FROM base AS lint-gofumpt
@ -191,11 +199,19 @@ COPY .golangci.yml .
ENV GOGC=50 ENV GOGC=50
RUN --mount=type=cache,target=/root/.cache/go-build,id=image-factory/root/.cache/go-build --mount=type=cache,target=/root/.cache/golangci-lint,id=image-factory/root/.cache/golangci-lint,sharing=locked --mount=type=cache,target=/go/pkg,id=image-factory/go/pkg golangci-lint run --config .golangci.yml RUN --mount=type=cache,target=/root/.cache/go-build,id=image-factory/root/.cache/go-build --mount=type=cache,target=/root/.cache/golangci-lint,id=image-factory/root/.cache/golangci-lint,sharing=locked --mount=type=cache,target=/go/pkg,id=image-factory/go/pkg golangci-lint run --config .golangci.yml
# runs golangci-lint fmt
FROM base AS lint-golangci-lint-fmt-run
WORKDIR /src
COPY .golangci.yml .
ENV GOGC=50
RUN --mount=type=cache,target=/root/.cache/go-build,id=image-factory/root/.cache/go-build --mount=type=cache,target=/root/.cache/golangci-lint,id=image-factory/root/.cache/golangci-lint,sharing=locked --mount=type=cache,target=/go/pkg,id=image-factory/go/pkg golangci-lint fmt --config .golangci.yml
RUN --mount=type=cache,target=/root/.cache/go-build,id=image-factory/root/.cache/go-build --mount=type=cache,target=/root/.cache/golangci-lint,id=image-factory/root/.cache/golangci-lint,sharing=locked --mount=type=cache,target=/go/pkg,id=image-factory/go/pkg golangci-lint run --fix --issues-exit-code 0 --config .golangci.yml
# runs govulncheck # runs govulncheck
FROM base AS lint-govulncheck FROM base AS lint-govulncheck
COPY --chmod=0755 hack/govulncheck.sh ./hack/govulncheck.sh
WORKDIR /src WORKDIR /src
COPY ./hack/govulncheck.sh ./hack/govulncheck.sh RUN --mount=type=cache,target=/root/.cache/go-build,id=image-factory/root/.cache/go-build --mount=type=cache,target=/go/pkg,id=image-factory/go/pkg ./hack/govulncheck.sh -exclude 'GO-2025-3770' ./...
RUN --mount=type=cache,target=/root/.cache/go-build,id=image-factory/root/.cache/go-build --mount=type=cache,target=/go/pkg,id=image-factory/go/pkg ./hack/govulncheck.sh ./...
# runs unit-tests with race detector # runs unit-tests with race detector
FROM base AS unit-tests-race FROM base AS unit-tests-race
@ -221,8 +237,20 @@ RUN echo -n 'undefined' > internal/version/data/sha && \
echo -n ${ABBREV_TAG} > internal/version/data/tag echo -n ${ABBREV_TAG} > internal/version/data/tag
# copies out the integration test binary # copies out the integration test binary
FROM scratch AS integration.test FROM scratch AS integration-cdn.test
COPY --from=integration-build /src/integration.test /integration.test COPY --from=integration-cdn-build /src/integration.test /integration-cdn.test
# copies out the integration test binary
FROM scratch AS integration-direct.test
COPY --from=integration-direct-build /src/integration.test /integration-direct.test
# copies out the integration test binary
FROM scratch AS integration-s3.test
COPY --from=integration-s3-build /src/integration.test /integration-s3.test
# clean golangci-lint fmt output
FROM scratch AS lint-golangci-lint-fmt
COPY --from=lint-golangci-lint-fmt-run /src .
FROM scratch AS unit-tests FROM scratch AS unit-tests
COPY --from=unit-tests-run /src/coverage.txt /coverage-unit-tests.txt COPY --from=unit-tests-run /src/coverage.txt /coverage-unit-tests.txt
@ -278,3 +306,4 @@ COPY --from=image-factory image-factory-linux-${TARGETARCH} /usr/bin/image-facto
COPY --from=imager-tools / / COPY --from=imager-tools / /
LABEL org.opencontainers.image.source=https://github.com/siderolabs/image-factory LABEL org.opencontainers.image.source=https://github.com/siderolabs/image-factory
ENTRYPOINT ["/usr/bin/image-factory"] ENTRYPOINT ["/usr/bin/image-factory"]

View File

@ -1,6 +1,6 @@
# THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
# #
# Generated on 2025-07-29T12:16:54Z by kres dd1ed6f. # Generated on 2025-08-14T09:04:59Z by kres 9f63e23-dirty.
# common variables # common variables
@ -80,7 +80,7 @@ TOOLCHAIN ?= docker.io/golang:1.24-alpine
PKGS_PREFIX ?= ghcr.io/siderolabs PKGS_PREFIX ?= ghcr.io/siderolabs
PKGS ?= v1.11.0 PKGS ?= v1.11.0
RUN_TESTS ?= TestIntegration RUN_TESTS ?= TestIntegrationCDN
TEST_FLAGS ?= TEST_FLAGS ?=
# help menu # help menu
@ -178,6 +178,9 @@ generate: ## Generate .proto definitions.
lint-golangci-lint: ## Runs golangci-lint linter. lint-golangci-lint: ## Runs golangci-lint linter.
@$(MAKE) target-$@ @$(MAKE) target-$@
lint-golangci-lint-fmt: ## Runs golangci-lint formatter and tries to fix issues automatically.
@$(MAKE) local-$@ DEST=.
lint-gofumpt: ## Runs gofumpt linter. lint-gofumpt: ## Runs gofumpt linter.
@$(MAKE) target-$@ @$(MAKE) target-$@
@ -204,13 +207,31 @@ unit-tests: ## Performs unit tests
unit-tests-race: ## Performs unit tests with race detection enabled. unit-tests-race: ## Performs unit tests with race detection enabled.
@$(MAKE) target-$@ @$(MAKE) target-$@
.PHONY: integration .PHONY: integration-direct
integration: integration.test integration-direct: integration-direct.test
@$(MAKE) image-image-factory PUSH=true @$(MAKE) image-image-factory PUSH=true
docker pull $(REGISTRY)/$(USERNAME)/image-factory:$(TAG) docker pull $(REGISTRY)/$(USERNAME)/image-factory:$(TAG)
docker rm -f local-if || true docker rm -f local-if || true
docker run -d -p 5100:5000 --name=local-if registry:3 docker run -d -p 5100:5000 --name=local-if registry:3
docker run --rm --net=host --privileged -v /dev:/dev -v /var/run:/var/run -v $(PWD)/$(ARTIFACTS)/:/out/ -v $(PWD)/$(ARTIFACTS)/integration.test:/bin/integration.test:ro --entrypoint /bin/integration.test $(REGISTRY)/$(USERNAME)/image-factory:$(TAG) -test.v $(TEST_FLAGS) -test.coverprofile=/out/coverage-integration.txt -test.run $(RUN_TESTS) docker run --rm --net=host --privileged -v /dev:/dev -v /var/run:/var/run -v $(PWD)/$(ARTIFACTS)/:/out/ -v $(PWD)/$(ARTIFACTS)/integration-direct.test:/bin/integration-direct.test:ro --entrypoint /bin/integration-direct.test $(REGISTRY)/$(USERNAME)/image-factory:$(TAG) -test.v $(TEST_FLAGS) -test.coverprofile=/out/coverage-integration-direct.txt -test.run $(RUN_TESTS)
docker rm -f local-if
.PHONY: integration-s3
integration-s3: integration-s3.test
@$(MAKE) image-image-factory PUSH=true
docker pull $(REGISTRY)/$(USERNAME)/image-factory:$(TAG)
docker rm -f local-if || true
docker run -d -p 5100:5000 --name=local-if registry:3
docker run --rm --net=host --privileged -v /dev:/dev -v /var/run:/var/run -v $(PWD)/$(ARTIFACTS)/:/out/ -v $(PWD)/$(ARTIFACTS)/integration-s3.test:/bin/integration-s3.test:ro --entrypoint /bin/integration-s3.test $(REGISTRY)/$(USERNAME)/image-factory:$(TAG) -test.v $(TEST_FLAGS) -test.coverprofile=/out/coverage-integration-s3.txt -test.run $(RUN_TESTS)
docker rm -f local-if
.PHONY: integration-cdn
integration-cdn: integration-cdn.test
@$(MAKE) image-image-factory PUSH=true
docker pull $(REGISTRY)/$(USERNAME)/image-factory:$(TAG)
docker rm -f local-if || true
docker run -d -p 5100:5000 --name=local-if registry:3
docker run --rm --net=host --privileged -v /dev:/dev -v /var/run:/var/run -v $(PWD)/$(ARTIFACTS)/:/out/ -v $(PWD)/$(ARTIFACTS)/integration-cdn.test:/bin/integration-cdn.test:ro --entrypoint /bin/integration-cdn.test $(REGISTRY)/$(USERNAME)/image-factory:$(TAG) -test.v $(TEST_FLAGS) -test.coverprofile=/out/coverage-integration-cdn.txt -test.run $(RUN_TESTS)
docker rm -f local-if docker rm -f local-if
.PHONY: $(ARTIFACTS)/image-factory-linux-amd64 .PHONY: $(ARTIFACTS)/image-factory-linux-amd64
@ -247,17 +268,33 @@ imager-base:
.PHONY: imager-tools .PHONY: imager-tools
imager-tools: imager-tools:
.PHONY: integration.test .PHONY: integration-cdn.test
integration.test: integration-cdn.test:
@$(MAKE) local-$@ DEST=$(ARTIFACTS)
.PHONY: integration-direct.test
integration-direct.test:
@$(MAKE) local-$@ DEST=$(ARTIFACTS)
.PHONY: integration-s3.test
integration-s3.test:
@$(MAKE) local-$@ DEST=$(ARTIFACTS) @$(MAKE) local-$@ DEST=$(ARTIFACTS)
.PHONY: update-to-talos-main .PHONY: update-to-talos-main
update-to-talos-main: update-to-talos-main:
@$(MAKE) local-copy-out-go-mod DEST=. TARGET_ARGS="--no-cache-filter=update-to-talos-main" @$(MAKE) local-copy-out-go-mod DEST=. TARGET_ARGS="--no-cache-filter=update-to-talos-main"
.PHONY: integration-talos-main .PHONY: integration-cdn-talos-main
integration-talos-main: update-to-talos-main integration-cdn-talos-main: update-to-talos-main
@$(MAKE) integration @$(MAKE) integration-cdn
.PHONY: integration-direct-talos-main
integration-direct-talos-main: update-to-talos-main
@$(MAKE) integration-direct
.PHONY: integration-s3-talos-main
integration-s3-talos-main: update-to-talos-main
@$(MAKE) integration-s3
.PHONY: tailwind .PHONY: tailwind
tailwind: tailwind:

View File

@ -99,6 +99,10 @@ type Options struct { //nolint:govet
// Leave empty to disable. // Leave empty to disable.
MetricsListenAddr string MetricsListenAddr string
// MetricsNamespace is the namespace for Prometheus metrics.
// It's not user-configurable, but set by the image factory tests.
MetricsNamespace string
// SecureBoot settings. // SecureBoot settings.
SecureBoot SecureBootOptions SecureBoot SecureBootOptions
} }

View File

@ -123,6 +123,7 @@ func RunFactory(ctx context.Context, logger *zap.Logger, opts Options) error {
frontendOptions.RemoteOptions = append(frontendOptions.RemoteOptions, remoteOptions()...) frontendOptions.RemoteOptions = append(frontendOptions.RemoteOptions, remoteOptions()...)
frontendOptions.RegistryRefreshInterval = opts.RegistryRefreshInterval frontendOptions.RegistryRefreshInterval = opts.RegistryRefreshInterval
frontendOptions.MetricsNamespace = opts.MetricsNamespace
frontendHTTP, err := frontendhttp.NewFrontend(logger, configFactory, assetBuilder, artifactsManager, secureBootService, frontendOptions) frontendHTTP, err := frontendhttp.NewFrontend(logger, configFactory, assetBuilder, artifactsManager, secureBootService, frontendOptions)
if err != nil { if err != nil {
@ -329,7 +330,9 @@ func buildAssetBuilder(logger *zap.Logger, artifactsManager *artifacts.Manager,
} }
builderOptions := asset.Options{ builderOptions := asset.Options{
MetricsNamespace: opts.MetricsNamespace,
AllowedConcurrency: opts.AssetBuildMaxConcurrency, AllowedConcurrency: opts.AssetBuildMaxConcurrency,
GetAfterPut: opts.CacheS3Enabled,
} }
builder, err := asset.NewBuilder(logger, artifactsManager, cache, builderOptions) builder, err := asset.NewBuilder(logger, artifactsManager, cache, builderOptions)
@ -359,7 +362,9 @@ func buildSchematicFactory(logger *zap.Logger, opts Options) (*schematic.Factory
return nil, fmt.Errorf("failed to initialize registry storage: %w", err) return nil, fmt.Errorf("failed to initialize registry storage: %w", err)
} }
factory := schematic.NewFactory(logger, schematiccache.NewCache(storage), schematic.Options{}) c := schematiccache.NewCache(storage, schematiccache.Options{MetricsNamespace: opts.MetricsNamespace})
factory := schematic.NewFactory(logger, c, schematic.Options{MetricsNamespace: opts.MetricsNamespace})
prometheus.MustRegister(factory) prometheus.MustRegister(factory)

View File

@ -1,28 +1,48 @@
#!/usr/bin/env bash #!/bin/bash
# Source: https://github.com/tianon/gosu/blob/e157efb/govulncheck-with-excludes.sh # Source: https://github.com/tianon/gosu/blob/e157efb/govulncheck-with-excludes.sh
# Licensed under the Apache License, Version 2.0 # Licensed under the Apache License, Version 2.0
# Copyright Tianon Gravi # Copyright Tianon Gravi
set -Eeuo pipefail set -Eeuo pipefail
# a wrapper / replacement for "govulncheck" which allows for excluding vulnerabilities exclude_arg=""
pass_args=()
while [[ $# -gt 0 ]]; do
case "$1" in
-exclude)
exclude_arg="$2"
shift 2
;;
*)
pass_args+=("$1")
shift
;;
esac
done
if [[ -n "$exclude_arg" ]]; then
excludeVulns="$(jq -nc --arg list "$exclude_arg" '$list | split(",")')"
else
excludeVulns="[]"
fi
excludeVulns="$(jq -nc '[
"GO-2025-3770",
empty # trailing comma hack (makes diffs smaller)
]')"
export excludeVulns export excludeVulns
# Debug print
echo "excludeVulns = $excludeVulns"
echo "Passing args: ${pass_args[*]}"
if ! command -v govulncheck > /dev/null; then if ! command -v govulncheck > /dev/null; then
printf "govulncheck not installed" printf "govulncheck not installed"
exit 1 exit 1
fi fi
if out="$(govulncheck "$@")"; then if out="$(govulncheck "${pass_args[@]}")"; then
printf '%s\n' "$out" printf '%s\n' "$out"
exit 0 exit 0
fi fi
json="$(govulncheck -json "$@")" json="$(govulncheck -json "${pass_args[@]}")"
vulns="$(jq <<<"$json" -cs ' vulns="$(jq <<<"$json" -cs '
( (

View File

@ -37,20 +37,26 @@ type BootAsset interface {
// Builder is the asset builder. // Builder is the asset builder.
type Builder struct { type Builder struct {
logger *zap.Logger metricConcurrencyLatency prometheus.Histogram
cache cache.Cache cache cache.Cache
artifactsManager *artifacts.Manager metricBuildLatency prometheus.Histogram
sf singleflight.Group sf singleflight.Group
semaphore chan struct{} metricAssetsCached *prometheus.CounterVec
logger *zap.Logger
metricAssetsCached, metricAssetsBuilt *prometheus.CounterVec metricAssetsBuilt *prometheus.CounterVec
metricAssetBytesCached, metricAssetBytesBuilt *prometheus.CounterVec metricAssetBytesCached *prometheus.CounterVec
metricConcurrencyLatency, metricBuildLatency prometheus.Histogram metricAssetBytesBuilt *prometheus.CounterVec
metricAssetCachedErrors *prometheus.CounterVec
semaphore chan struct{}
artifactsManager *artifacts.Manager
getAfterPut bool
} }
// Options configures the asset builder. // Options configures the asset builder.
type Options struct { type Options struct {
MetricsNamespace string
AllowedConcurrency int AllowedConcurrency int
GetAfterPut bool
} }
// NewBuilder creates a new asset builder. // NewBuilder creates a new asset builder.
@ -60,47 +66,62 @@ func NewBuilder(logger *zap.Logger, artifactsManager *artifacts.Manager, cache c
cache: cache, cache: cache,
artifactsManager: artifactsManager, artifactsManager: artifactsManager,
semaphore: make(chan struct{}, options.AllowedConcurrency), semaphore: make(chan struct{}, options.AllowedConcurrency),
getAfterPut: options.GetAfterPut,
metricAssetsCached: prometheus.NewCounterVec( metricAssetsCached: prometheus.NewCounterVec(
prometheus.CounterOpts{ prometheus.CounterOpts{
Name: "image_factory_assets_delivered_cached_total", Name: "image_factory_assets_delivered_cached_total",
Help: "Number of assets retrieved from cache.", Help: "Number of assets retrieved from cache.",
Namespace: options.MetricsNamespace,
}, },
[]string{"talos_version", "output_kind", "arch"}, []string{"talos_version", "output_kind", "arch"},
), ),
metricAssetsBuilt: prometheus.NewCounterVec( metricAssetsBuilt: prometheus.NewCounterVec(
prometheus.CounterOpts{ prometheus.CounterOpts{
Name: "image_factory_assets_built_total", Name: "image_factory_assets_built_total",
Help: "Number of assets built (missing in the cache).", Help: "Number of assets built (missing in the cache).",
Namespace: options.MetricsNamespace,
}, },
[]string{"talos_version", "output_kind", "arch"}, []string{"talos_version", "output_kind", "arch"},
), ),
metricAssetBytesCached: prometheus.NewCounterVec( metricAssetBytesCached: prometheus.NewCounterVec(
prometheus.CounterOpts{ prometheus.CounterOpts{
Name: "image_factory_assets_cached_bytes_total", Name: "image_factory_assets_cached_bytes_total",
Help: "Number of bytes delivered with cached assets.", Help: "Number of bytes delivered with cached assets.",
Namespace: options.MetricsNamespace,
}, },
[]string{"talos_version", "output_kind", "arch"}, []string{"talos_version", "output_kind", "arch"},
), ),
metricAssetBytesBuilt: prometheus.NewCounterVec( metricAssetBytesBuilt: prometheus.NewCounterVec(
prometheus.CounterOpts{ prometheus.CounterOpts{
Name: "image_factory_assets_built_bytes_total", Name: "image_factory_assets_built_bytes_total",
Help: "Number of bytes delivered with assets built (missing in the cache).", Help: "Number of bytes delivered with assets built (missing in the cache).",
Namespace: options.MetricsNamespace,
},
[]string{"talos_version", "output_kind", "arch"},
),
metricAssetCachedErrors: prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "image_factory_assets_cached_errors_total",
Help: "Number of errors retrieving assets from cache.",
Namespace: options.MetricsNamespace,
}, },
[]string{"talos_version", "output_kind", "arch"}, []string{"talos_version", "output_kind", "arch"},
), ),
metricConcurrencyLatency: prometheus.NewHistogram( metricConcurrencyLatency: prometheus.NewHistogram(
prometheus.HistogramOpts{ prometheus.HistogramOpts{
Name: "image_factory_assets_concurrency_latency_seconds", Name: "image_factory_assets_concurrency_latency_seconds",
Help: "Latency of asset build related to the concurrency limit (waiting for available workers).", Help: "Latency of asset build related to the concurrency limit (waiting for available workers).",
Buckets: []float64{1, 10, 60, 180, 600}, Buckets: []float64{1, 10, 60, 180, 600},
Namespace: options.MetricsNamespace,
}, },
), ),
metricBuildLatency: prometheus.NewHistogram( metricBuildLatency: prometheus.NewHistogram(
prometheus.HistogramOpts{ prometheus.HistogramOpts{
Name: "image_factory_assets_build_latency_seconds", Name: "image_factory_assets_build_latency_seconds",
Help: "Latency of asset build for the build itself (excluding waiting for available workers).", Help: "Latency of asset build for the build itself (excluding waiting for available workers).",
Buckets: []float64{1, 10, 60, 180, 600}, Buckets: []float64{1, 10, 60, 180, 600},
Namespace: options.MetricsNamespace,
}, },
), ),
}, nil }, nil
@ -179,15 +200,24 @@ func (b *Builder) buildAndCache(profileHash string, prof profile.Profile, versio
b.logger.Error("error putting asset to cache", zap.Error(err), zap.String("profile_hash", profileHash)) b.logger.Error("error putting asset to cache", zap.Error(err), zap.String("profile_hash", profileHash))
} }
cachedAsset, err := b.cache.Get(ctx, profileHash) // if the asset delivery requires a redirect to the cache, we need to return the cached asset
if err == nil { // so that the client can download it directly from the cache.
b.metricAssetsCached.WithLabelValues(versionString, prof.Output.Kind.String(), prof.Arch).Inc() if b.getAfterPut {
b.metricAssetBytesCached.WithLabelValues(versionString, prof.Output.Kind.String(), prof.Arch).Add(float64(asset.Size())) cachedAsset, err := b.cache.Get(ctx, profileHash)
if err == nil {
b.metricAssetsCached.WithLabelValues(versionString, prof.Output.Kind.String(), prof.Arch).Inc()
b.metricAssetBytesCached.WithLabelValues(versionString, prof.Output.Kind.String(), prof.Arch).Add(float64(asset.Size()))
return cachedAsset, nil return cachedAsset, nil
}
// if we failed to get the asset from cache, return the built asset anyway
b.logger.Warn("failed to get object from cache, falling back to direct delivery", zap.Error(err))
b.metricAssetCachedErrors.WithLabelValues(versionString, prof.Output.Kind.String(), prof.Arch).Inc()
} }
return nil, fmt.Errorf("error getting asset from cache: %w", err) return asset, nil
} }
// build the asset using Talos imager. // build the asset using Talos imager.
@ -285,6 +315,8 @@ func (b *Builder) Collect(ch chan<- prometheus.Metric) {
b.metricAssetBytesBuilt.Collect(ch) b.metricAssetBytesBuilt.Collect(ch)
b.metricAssetBytesCached.Collect(ch) b.metricAssetBytesCached.Collect(ch)
b.metricAssetCachedErrors.Collect(ch)
b.metricBuildLatency.Collect(ch) b.metricBuildLatency.Collect(ch)
b.metricConcurrencyLatency.Collect(ch) b.metricConcurrencyLatency.Collect(ch)
} }

View File

@ -55,16 +55,14 @@ type Frontend struct {
// Options configures the HTTP frontend. // Options configures the HTTP frontend.
type Options struct { type Options struct {
ExternalURL *url.URL CacheSigningKey crypto.PrivateKey
ExternalPXEURL *url.URL ExternalURL *url.URL
ExternalPXEURL *url.URL
InstallerInternalRepository name.Repository InstallerInternalRepository name.Repository
InstallerExternalRepository name.Repository InstallerExternalRepository name.Repository
MetricsNamespace string
CacheSigningKey crypto.PrivateKey RemoteOptions []remote.Option
RegistryRefreshInterval time.Duration
RemoteOptions []remote.Option
RegistryRefreshInterval time.Duration
} }
// NewFrontend creates a new HTTP frontend. // NewFrontend creates a new HTTP frontend.
@ -105,7 +103,9 @@ func NewFrontend(
// monitoring middleware // monitoring middleware
mdlw := middleware.New(middleware.Config{ mdlw := middleware.New(middleware.Config{
Recorder: metrics.NewRecorder(metrics.Config{}), Recorder: metrics.NewRecorder(metrics.Config{
Prefix: opts.MetricsNamespace,
}),
}) })
registerRoute := func(registrator func(string, httprouter.Handle), path string, handler func(ctx context.Context, w http.ResponseWriter, r *http.Request, p httprouter.Params) error) { registerRoute := func(registrator func(string, httprouter.Handle), path string, handler func(ctx context.Context, w http.ResponseWriter, r *http.Request, p httprouter.Params) error) {

View File

@ -46,7 +46,6 @@ func setupFactory(t *testing.T, options cmd.Options) (context.Context, string) {
options.SchematicServiceRepository = schematicFactoryRepositoryFlag options.SchematicServiceRepository = schematicFactoryRepositoryFlag
options.InstallerExternalRepository = installerExternalRepository options.InstallerExternalRepository = installerExternalRepository
options.InstallerInternalRepository = installerInternalRepository options.InstallerInternalRepository = installerInternalRepository
options.CacheRepository = cacheRepository
options.RegistryRefreshInterval = time.Minute // use a short interval for the tests options.RegistryRefreshInterval = time.Minute // use a short interval for the tests
setupSecureBoot(t, &options) setupSecureBoot(t, &options)
@ -121,14 +120,11 @@ func healthcheck(url string) func() error {
} }
const ( const (
bucket = "image-factory"
bucketPrefix = "/" + bucket
s3Access = "AKIA6Z4C7N3S2JD3JH9A" s3Access = "AKIA6Z4C7N3S2JD3JH9A"
s3Secret = "y1rE4xZnqO6xvM7L0jFD3EXAMPLEnG4K2vOfLp8Iv9" s3Secret = "y1rE4xZnqO6xvM7L0jFD3EXAMPLEnG4K2vOfLp8Iv9"
) )
func setupS3(t *testing.T, pool *dockertest.Pool) string { func setupS3(t *testing.T, pool *dockertest.Pool, bucket string) string {
t.Helper() t.Helper()
_, port, err := net.SplitHostPort(findListenAddr(t)) _, port, err := net.SplitHostPort(findListenAddr(t))
@ -167,16 +163,13 @@ func setupS3(t *testing.T, pool *dockertest.Pool) string {
}) })
require.NoError(t, err) require.NoError(t, err)
t.Setenv("AWS_ACCESS_KEY_ID", s3Access)
t.Setenv("AWS_SECRET_ACCESS_KEY", s3Secret)
return endpoint return endpoint
} }
//go:embed testdata/templates/nginx.sh //go:embed testdata/templates/nginx.sh
var nginxConfigTemplate string var nginxConfigTemplate string
func setupMockCDN(t *testing.T, pool *dockertest.Pool, s3 string) string { func setupMockCDN(t *testing.T, pool *dockertest.Pool, s3, bucket string) string {
t.Helper() t.Helper()
_, port, err := net.SplitHostPort(findListenAddr(t)) _, port, err := net.SplitHostPort(findListenAddr(t))
@ -249,19 +242,7 @@ func findListenAddr(t *testing.T) string {
return addr return addr
} }
func TestIntegration(t *testing.T) { func commonTest(t *testing.T, options cmd.Options) {
pool := docker(t)
options := cmd.DefaultOptions
options.CacheS3Enabled = true
options.CacheS3Bucket = bucket
options.InsecureCacheS3 = true
options.CacheS3Endpoint = setupS3(t, pool)
options.CacheCDNEnabled = true
options.CacheCDNTrimPrefix = bucketPrefix
options.CacheCDNHost = setupMockCDN(t, pool, options.CacheS3Endpoint)
ctx, listenAddr := setupFactory(t, options) ctx, listenAddr := setupFactory(t, options)
baseURL := "http://" + listenAddr baseURL := "http://" + listenAddr
@ -313,6 +294,7 @@ var (
installerExternalRepository string installerExternalRepository string
installerInternalRepository string installerInternalRepository string
cacheRepository string cacheRepository string
signingCacheRepository string
) )
func init() { func init() {
@ -321,4 +303,5 @@ func init() {
flag.StringVar(&installerExternalRepository, "test.installer-external-repository", cmd.DefaultOptions.InstallerExternalRepository, "image repository for the installer (external)") flag.StringVar(&installerExternalRepository, "test.installer-external-repository", cmd.DefaultOptions.InstallerExternalRepository, "image repository for the installer (external)")
flag.StringVar(&installerInternalRepository, "test.installer-internal-repository", cmd.DefaultOptions.InstallerInternalRepository, "image repository for the installer (internal)") flag.StringVar(&installerInternalRepository, "test.installer-internal-repository", cmd.DefaultOptions.InstallerInternalRepository, "image repository for the installer (internal)")
flag.StringVar(&cacheRepository, "test.cache-repository", cmd.DefaultOptions.CacheRepository, "image repository for cached boot assets") flag.StringVar(&cacheRepository, "test.cache-repository", cmd.DefaultOptions.CacheRepository, "image repository for cached boot assets")
flag.StringVar(&signingCacheRepository, "test.signing-cache-repository", cmd.DefaultOptions.CacheRepository+"sign", "image repository for signatures of cached boot assets (used for S3+CDN tests)")
} }

View File

@ -0,0 +1,40 @@
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
//go:build integration_cdn
package integration_test
import (
"fmt"
"testing"
"github.com/siderolabs/image-factory/cmd/image-factory/cmd"
)
func TestIntegrationCDN(t *testing.T) {
pool := docker(t)
// set up S3 access credentials for the tests, those are shared across all tests
t.Setenv("AWS_ACCESS_KEY_ID", s3Access)
t.Setenv("AWS_SECRET_ACCESS_KEY", s3Secret)
t.Run("S3+CDN", func(t *testing.T) {
options := cmd.DefaultOptions
options.CacheRepository = signingCacheRepository
options.MetricsNamespace = "test_s3_cdn"
options.CacheS3Enabled = true
options.CacheS3Bucket = "test-s3-cdn"
options.InsecureCacheS3 = true
options.CacheS3Endpoint = setupS3(t, pool, options.CacheS3Bucket)
options.CacheCDNEnabled = true
options.CacheCDNTrimPrefix = fmt.Sprintf("/%s", options.CacheS3Bucket)
options.CacheCDNHost = setupMockCDN(t, pool, options.CacheS3Endpoint, options.CacheS3Bucket)
commonTest(t, options)
})
}

View File

@ -0,0 +1,22 @@
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
//go:build integration_direct
package integration_test
import (
"testing"
"github.com/siderolabs/image-factory/cmd/image-factory/cmd"
)
func TestIntegrationDirect(t *testing.T) {
options := cmd.DefaultOptions
options.CacheRepository = cacheRepository
options.MetricsNamespace = ""
commonTest(t, options)
}

View File

@ -0,0 +1,35 @@
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
//go:build integration_s3
package integration_test
import (
"testing"
"github.com/siderolabs/image-factory/cmd/image-factory/cmd"
)
func TestIntegrationS3(t *testing.T) {
pool := docker(t)
// set up S3 access credentials for the tests, those are shared across all tests
t.Setenv("AWS_ACCESS_KEY_ID", s3Access)
t.Setenv("AWS_SECRET_ACCESS_KEY", s3Secret)
t.Run("S3", func(t *testing.T) {
options := cmd.DefaultOptions
options.CacheRepository = signingCacheRepository
options.MetricsNamespace = "test_s3"
options.CacheS3Enabled = true
options.CacheS3Bucket = "test-s3"
options.InsecureCacheS3 = true
options.CacheS3Endpoint = setupS3(t, pool, options.CacheS3Bucket)
commonTest(t, options)
})
}

View File

@ -17,15 +17,18 @@ import (
// Factory is the schematic factory. // Factory is the schematic factory.
type Factory struct { type Factory struct {
options Options storage storage.Storage
logger *zap.Logger metricGet prometheus.Counter
storage storage.Storage metricCreate prometheus.Counter
metricDuplicate prometheus.Counter
metricGet, metricCreate, metricDuplicate prometheus.Counter logger *zap.Logger
options Options
} }
// Options for the schematic factory. // Options for the schematic factory.
type Options struct{} type Options struct {
MetricsNamespace string
}
// NewFactory creates a new schematic factory. // NewFactory creates a new schematic factory.
func NewFactory(logger *zap.Logger, storage storage.Storage, options Options) *Factory { func NewFactory(logger *zap.Logger, storage storage.Storage, options Options) *Factory {
@ -35,16 +38,19 @@ func NewFactory(logger *zap.Logger, storage storage.Storage, options Options) *F
logger: logger.With(zap.String("factory", "schematic")), logger: logger.With(zap.String("factory", "schematic")),
metricGet: prometheus.NewCounter(prometheus.CounterOpts{ metricGet: prometheus.NewCounter(prometheus.CounterOpts{
Name: "image_factory_schematic_get_total", Name: "image_factory_schematic_get_total",
Help: "Number of times schematics were retrieved.", Help: "Number of times schematics were retrieved.",
Namespace: options.MetricsNamespace,
}), }),
metricCreate: prometheus.NewCounter(prometheus.CounterOpts{ metricCreate: prometheus.NewCounter(prometheus.CounterOpts{
Name: "image_factory_schematic_create_total", Name: "image_factory_schematic_create_total",
Help: "Number of new schematics created.", Help: "Number of new schematics created.",
Namespace: options.MetricsNamespace,
}), }),
metricDuplicate: prometheus.NewCounter(prometheus.CounterOpts{ metricDuplicate: prometheus.NewCounter(prometheus.CounterOpts{
Name: "image_factory_schematic_duplicate_create_total", Name: "image_factory_schematic_duplicate_create_total",
Help: "Number of new schematics which were created as duplicate.", Help: "Number of new schematics which were created as duplicate.",
Namespace: options.MetricsNamespace,
}), }),
} }
} }

View File

@ -17,6 +17,11 @@ import (
"github.com/siderolabs/image-factory/internal/schematic/storage" "github.com/siderolabs/image-factory/internal/schematic/storage"
) )
// Options configures the storage.
type Options struct {
MetricsNamespace string
}
// Storage is a schematic storage in-memory cache. // Storage is a schematic storage in-memory cache.
type Storage struct { type Storage struct {
underlying storage.Storage underlying storage.Storage
@ -29,13 +34,14 @@ type Storage struct {
} }
// NewCache returns a new cache storage. // NewCache returns a new cache storage.
func NewCache(underlying storage.Storage) *Storage { func NewCache(underlying storage.Storage, options Options) *Storage {
return &Storage{ return &Storage{
underlying: underlying, underlying: underlying,
m: map[string]optional.Optional[[]byte]{}, m: map[string]optional.Optional[[]byte]{},
metricCacheSize: prometheus.NewGauge(prometheus.GaugeOpts{ metricCacheSize: prometheus.NewGauge(prometheus.GaugeOpts{
Name: "image_factory_schematic_cache_size", Name: "image_factory_schematic_cache_size",
Help: "Number of schematics in in-memory cache.", Help: "Number of schematics in in-memory cache.",
Namespace: options.MetricsNamespace,
}), }),
} }
} }

View File

@ -55,7 +55,7 @@ func TestStorage(t *testing.T) {
defer cancel() defer cancel()
underlying := &mockStorage{} underlying := &mockStorage{}
strg := cache.NewCache(underlying) strg := cache.NewCache(underlying, cache.Options{})
v, err := strg.Get(ctx, "foo") v, err := strg.Get(ctx, "foo")
require.NoError(t, err) require.NoError(t, err)