From a1e37078e10bae58d8ee3f117cdbc405de35e65c Mon Sep 17 00:00:00 2001 From: Mateusz Urbanek Date: Wed, 13 Aug 2025 13:28:21 +0200 Subject: [PATCH] feat: add fallback if S3 is missbehaving Add fallback to direct asset download in case of S3 issues. Signed-off-by: Mateusz Urbanek --- .dockerignore | 4 +- .github/workflows/ci.yaml | 140 ++++++++++- .../integration-cdn-talos-main-cron.yaml | 61 +++++ .../integration-direct-talos-main-cron.yaml | 61 +++++ ...ml => integration-s3-talos-main-cron.yaml} | 11 +- .github/workflows/lock.yml | 2 +- .../workflows/slack-notify-ci-failure.yaml | 92 +++++++ .github/workflows/slack-notify.yaml | 58 ++--- .github/workflows/stale.yml | 2 +- .kres.yaml | 226 +++++++++++++++--- Dockerfile | 57 +++-- Makefile | 57 ++++- cmd/image-factory/cmd/options.go | 4 + cmd/image-factory/cmd/service.go | 7 +- hack/govulncheck.sh | 36 ++- internal/asset/asset.go | 90 ++++--- internal/frontend/http/http.go | 18 +- internal/integration/integration_test.go | 27 +-- internal/integration/main_cdn_test.go | 40 ++++ internal/integration/main_direct_test.go | 22 ++ internal/integration/main_s3_test.go | 35 +++ internal/schematic/schematic.go | 30 ++- internal/schematic/storage/cache/cache.go | 12 +- .../schematic/storage/cache/cache_test.go | 2 +- 24 files changed, 912 insertions(+), 182 deletions(-) create mode 100644 .github/workflows/integration-cdn-talos-main-cron.yaml create mode 100644 .github/workflows/integration-direct-talos-main-cron.yaml rename .github/workflows/{integration-talos-main-cron.yaml => integration-s3-talos-main-cron.yaml} (87%) create mode 100644 .github/workflows/slack-notify-ci-failure.yaml create mode 100644 internal/integration/main_cdn_test.go create mode 100644 internal/integration/main_direct_test.go create mode 100644 internal/integration/main_s3_test.go diff --git a/.dockerignore b/.dockerignore index 6898018..a6ffcdd 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,18 +1,18 @@ # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. # -# Generated on 2024-03-15T10:45:09Z by kres latest. +# Generated on 2025-08-14T09:17:28Z by kres 9f63e23-dirty. * !cmd !internal !pkg -!hack !go.mod !go.sum !.golangci.yml !CHANGELOG.md !README.md !.markdownlint.json +!hack/govulncheck.sh !tailwind.config.js !package.json !package-lock.json diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index fbe17af..309dee7 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,6 +1,6 @@ # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. # -# Generated on 2025-07-22T11:53:08Z by kres b869533. +# Generated on 2025-08-13T16:38:32Z by kres 9f63e23. concurrency: group: ${{ github.head_ref || github.run_id }} @@ -78,17 +78,34 @@ jobs: - name: unit-tests-race run: | make unit-tests-race - - name: integration + - name: integration-direct if: github.event_name == 'pull_request' env: REGISTRY: registry.dev.siderolabs.io - TEST_FLAGS: -test.schematic-service-repository=127.0.0.1:5100/image-factory/schematic -test.installer-external-repository=127.0.0.1:5100/siderolabs -test.installer-internal-repository=127.0.0.1:5100/siderolabs -test.cache-repository=127.0.0.1:5100/image-factory/cache + RUN_TESTS: TestIntegrationDirect + TEST_FLAGS: -test.schematic-service-repository=127.0.0.1:5100/image-factory/schematic -test.installer-external-repository=127.0.0.1:5100/siderolabs -test.installer-internal-repository=127.0.0.1:5100/siderolabs -test.cache-repository=127.0.0.1:5100/image-factory/cache -test.signing-cache-repository=127.0.0.1:5100/image-factory/signing-cache run: | - make integration + make integration-direct + - name: integration-s3 + if: github.event_name == 'pull_request' + env: + REGISTRY: registry.dev.siderolabs.io + RUN_TESTS: TestIntegrationS3 + TEST_FLAGS: -test.schematic-service-repository=127.0.0.1:5100/image-factory/schematic -test.installer-external-repository=127.0.0.1:5100/siderolabs -test.installer-internal-repository=127.0.0.1:5100/siderolabs -test.cache-repository=127.0.0.1:5100/image-factory/cache -test.signing-cache-repository=127.0.0.1:5100/image-factory/signing-cache + run: | + make integration-s3 + - name: integration-cdn + if: github.event_name == 'pull_request' + env: + REGISTRY: registry.dev.siderolabs.io + RUN_TESTS: TestIntegrationCDN + TEST_FLAGS: -test.schematic-service-repository=127.0.0.1:5100/image-factory/schematic -test.installer-external-repository=127.0.0.1:5100/siderolabs -test.installer-internal-repository=127.0.0.1:5100/siderolabs -test.cache-repository=127.0.0.1:5100/image-factory/cache -test.signing-cache-repository=127.0.0.1:5100/image-factory/signing-cache + run: | + make integration-cdn - name: coverage uses: codecov/codecov-action@v5 with: - files: _out/coverage-unit-tests.txt,_out/coverage-integration.txt + files: _out/coverage-unit-tests.txt,_out/coverage-integration-direct.txt,_out/coverage-integration-s3.txt,_out/coverage-integration-cdn.txt token: ${{ secrets.CODECOV_TOKEN }} timeout-minutes: 3 - name: image-factory @@ -155,7 +172,7 @@ jobs: files: |- _out/image-factory-* _out/sha*.txt - integration-talos-main: + integration-cdn-talos-main: runs-on: - self-hosted - generic @@ -200,9 +217,114 @@ jobs: driver: remote endpoint: tcp://buildkit-amd64.ci.svc.cluster.local:1234 timeout-minutes: 10 - - name: integration-talos-main + - name: integration-cdn-talos-main env: REGISTRY: registry.dev.siderolabs.io - TEST_FLAGS: -test.schematic-service-repository=registry.dev.siderolabs.io/image-factory/schematic -test.installer-external-repository=registry.dev.siderolabs.io/siderolabs -test.installer-internal-repository=registry.dev.siderolabs.io/siderolabs -test.cache-repository=registry.dev.siderolabs.io/image-factory/cache + RUN_TESTS: TestIntegrationCDN + TEST_FLAGS: -test.schematic-service-repository=registry.dev.siderolabs.io/image-factory/schematic -test.installer-external-repository=registry.dev.siderolabs.io/siderolabs -test.installer-internal-repository=registry.dev.siderolabs.io/siderolabs -test.cache-repository=registry.dev.siderolabs.io/image-factory/cache -test.signing-cache-repository=127.0.0.1:5100/image-factory/signing-cache run: | - make integration-talos-main + make integration-cdn-talos-main + integration-direct-talos-main: + runs-on: + - self-hosted + - generic + if: contains(fromJSON(needs.default.outputs.labels), 'integration/talos-main') + needs: + - default + steps: + - name: gather-system-info + id: system-info + uses: kenchan0130/actions-system-info@v1.3.1 + continue-on-error: true + - name: print-system-info + run: | + MEMORY_GB=$((${{ steps.system-info.outputs.totalmem }}/1024/1024/1024)) + + OUTPUTS=( + "CPU Core: ${{ steps.system-info.outputs.cpu-core }}" + "CPU Model: ${{ steps.system-info.outputs.cpu-model }}" + "Hostname: ${{ steps.system-info.outputs.hostname }}" + "NodeName: ${NODE_NAME}" + "Kernel release: ${{ steps.system-info.outputs.kernel-release }}" + "Kernel version: ${{ steps.system-info.outputs.kernel-version }}" + "Name: ${{ steps.system-info.outputs.name }}" + "Platform: ${{ steps.system-info.outputs.platform }}" + "Release: ${{ steps.system-info.outputs.release }}" + "Total memory: ${MEMORY_GB} GB" + ) + + for OUTPUT in "${OUTPUTS[@]}";do + echo "${OUTPUT}" + done + continue-on-error: true + - name: checkout + uses: actions/checkout@v4 + - name: Unshallow + run: | + git fetch --prune --unshallow + - name: Set up Docker Buildx + id: setup-buildx + uses: docker/setup-buildx-action@v3 + with: + driver: remote + endpoint: tcp://buildkit-amd64.ci.svc.cluster.local:1234 + timeout-minutes: 10 + - name: integration-direct-talos-main + env: + REGISTRY: registry.dev.siderolabs.io + RUN_TESTS: TestIntegrationDirect + TEST_FLAGS: -test.schematic-service-repository=registry.dev.siderolabs.io/image-factory/schematic -test.installer-external-repository=registry.dev.siderolabs.io/siderolabs -test.installer-internal-repository=registry.dev.siderolabs.io/siderolabs -test.cache-repository=registry.dev.siderolabs.io/image-factory/cache -test.signing-cache-repository=127.0.0.1:5100/image-factory/signing-cache + run: | + make integration-direct-talos-main + integration-s3-talos-main: + runs-on: + - self-hosted + - generic + if: contains(fromJSON(needs.default.outputs.labels), 'integration/talos-main') + needs: + - default + steps: + - name: gather-system-info + id: system-info + uses: kenchan0130/actions-system-info@v1.3.1 + continue-on-error: true + - name: print-system-info + run: | + MEMORY_GB=$((${{ steps.system-info.outputs.totalmem }}/1024/1024/1024)) + + OUTPUTS=( + "CPU Core: ${{ steps.system-info.outputs.cpu-core }}" + "CPU Model: ${{ steps.system-info.outputs.cpu-model }}" + "Hostname: ${{ steps.system-info.outputs.hostname }}" + "NodeName: ${NODE_NAME}" + "Kernel release: ${{ steps.system-info.outputs.kernel-release }}" + "Kernel version: ${{ steps.system-info.outputs.kernel-version }}" + "Name: ${{ steps.system-info.outputs.name }}" + "Platform: ${{ steps.system-info.outputs.platform }}" + "Release: ${{ steps.system-info.outputs.release }}" + "Total memory: ${MEMORY_GB} GB" + ) + + for OUTPUT in "${OUTPUTS[@]}";do + echo "${OUTPUT}" + done + continue-on-error: true + - name: checkout + uses: actions/checkout@v4 + - name: Unshallow + run: | + git fetch --prune --unshallow + - name: Set up Docker Buildx + id: setup-buildx + uses: docker/setup-buildx-action@v3 + with: + driver: remote + endpoint: tcp://buildkit-amd64.ci.svc.cluster.local:1234 + timeout-minutes: 10 + - name: integration-s3-talos-main + env: + REGISTRY: registry.dev.siderolabs.io + RUN_TESTS: TestIntegrationS3 + TEST_FLAGS: -test.schematic-service-repository=registry.dev.siderolabs.io/image-factory/schematic -test.installer-external-repository=registry.dev.siderolabs.io/siderolabs -test.installer-internal-repository=registry.dev.siderolabs.io/siderolabs -test.cache-repository=registry.dev.siderolabs.io/image-factory/cache -test.signing-cache-repository=127.0.0.1:5100/image-factory/signing-cache + run: | + make integration-s3-talos-main diff --git a/.github/workflows/integration-cdn-talos-main-cron.yaml b/.github/workflows/integration-cdn-talos-main-cron.yaml new file mode 100644 index 0000000..c58dc2c --- /dev/null +++ b/.github/workflows/integration-cdn-talos-main-cron.yaml @@ -0,0 +1,61 @@ +# THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. +# +# Generated on 2025-08-13T15:31:47Z by kres 9f63e23. + +concurrency: + group: ${{ github.head_ref || github.run_id }} + cancel-in-progress: true +"on": + schedule: + - cron: 30 7 * * * +name: integration-cdn-talos-main-cron +jobs: + default: + runs-on: + - self-hosted + - generic + steps: + - name: gather-system-info + id: system-info + uses: kenchan0130/actions-system-info@v1.3.1 + continue-on-error: true + - name: print-system-info + run: | + MEMORY_GB=$((${{ steps.system-info.outputs.totalmem }}/1024/1024/1024)) + + OUTPUTS=( + "CPU Core: ${{ steps.system-info.outputs.cpu-core }}" + "CPU Model: ${{ steps.system-info.outputs.cpu-model }}" + "Hostname: ${{ steps.system-info.outputs.hostname }}" + "NodeName: ${NODE_NAME}" + "Kernel release: ${{ steps.system-info.outputs.kernel-release }}" + "Kernel version: ${{ steps.system-info.outputs.kernel-version }}" + "Name: ${{ steps.system-info.outputs.name }}" + "Platform: ${{ steps.system-info.outputs.platform }}" + "Release: ${{ steps.system-info.outputs.release }}" + "Total memory: ${MEMORY_GB} GB" + ) + + for OUTPUT in "${OUTPUTS[@]}";do + echo "${OUTPUT}" + done + continue-on-error: true + - name: checkout + uses: actions/checkout@v4 + - name: Unshallow + run: | + git fetch --prune --unshallow + - name: Set up Docker Buildx + id: setup-buildx + uses: docker/setup-buildx-action@v3 + with: + driver: remote + endpoint: tcp://buildkit-amd64.ci.svc.cluster.local:1234 + timeout-minutes: 10 + - name: integration-cdn-talos-main + env: + REGISTRY: registry.dev.siderolabs.io + RUN_TESTS: TestIntegrationCDN + TEST_FLAGS: -test.schematic-service-repository=registry.dev.siderolabs.io/image-factory/schematic -test.installer-external-repository=registry.dev.siderolabs.io/siderolabs -test.installer-internal-repository=registry.dev.siderolabs.io/siderolabs -test.cache-repository=registry.dev.siderolabs.io/image-factory/cache -test.signing-cache-repository=127.0.0.1:5100/image-factory/signing-cache + run: | + make integration-cdn-talos-main diff --git a/.github/workflows/integration-direct-talos-main-cron.yaml b/.github/workflows/integration-direct-talos-main-cron.yaml new file mode 100644 index 0000000..f59a4db --- /dev/null +++ b/.github/workflows/integration-direct-talos-main-cron.yaml @@ -0,0 +1,61 @@ +# THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. +# +# Generated on 2025-08-13T15:31:47Z by kres 9f63e23. + +concurrency: + group: ${{ github.head_ref || github.run_id }} + cancel-in-progress: true +"on": + schedule: + - cron: 30 7 * * * +name: integration-direct-talos-main-cron +jobs: + default: + runs-on: + - self-hosted + - generic + steps: + - name: gather-system-info + id: system-info + uses: kenchan0130/actions-system-info@v1.3.1 + continue-on-error: true + - name: print-system-info + run: | + MEMORY_GB=$((${{ steps.system-info.outputs.totalmem }}/1024/1024/1024)) + + OUTPUTS=( + "CPU Core: ${{ steps.system-info.outputs.cpu-core }}" + "CPU Model: ${{ steps.system-info.outputs.cpu-model }}" + "Hostname: ${{ steps.system-info.outputs.hostname }}" + "NodeName: ${NODE_NAME}" + "Kernel release: ${{ steps.system-info.outputs.kernel-release }}" + "Kernel version: ${{ steps.system-info.outputs.kernel-version }}" + "Name: ${{ steps.system-info.outputs.name }}" + "Platform: ${{ steps.system-info.outputs.platform }}" + "Release: ${{ steps.system-info.outputs.release }}" + "Total memory: ${MEMORY_GB} GB" + ) + + for OUTPUT in "${OUTPUTS[@]}";do + echo "${OUTPUT}" + done + continue-on-error: true + - name: checkout + uses: actions/checkout@v4 + - name: Unshallow + run: | + git fetch --prune --unshallow + - name: Set up Docker Buildx + id: setup-buildx + uses: docker/setup-buildx-action@v3 + with: + driver: remote + endpoint: tcp://buildkit-amd64.ci.svc.cluster.local:1234 + timeout-minutes: 10 + - name: integration-direct-talos-main + env: + REGISTRY: registry.dev.siderolabs.io + RUN_TESTS: TestIntegrationDirect + TEST_FLAGS: -test.schematic-service-repository=registry.dev.siderolabs.io/image-factory/schematic -test.installer-external-repository=registry.dev.siderolabs.io/siderolabs -test.installer-internal-repository=registry.dev.siderolabs.io/siderolabs -test.cache-repository=registry.dev.siderolabs.io/image-factory/cache -test.signing-cache-repository=127.0.0.1:5100/image-factory/signing-cache + run: | + make integration-direct-talos-main diff --git a/.github/workflows/integration-talos-main-cron.yaml b/.github/workflows/integration-s3-talos-main-cron.yaml similarity index 87% rename from .github/workflows/integration-talos-main-cron.yaml rename to .github/workflows/integration-s3-talos-main-cron.yaml index 79bd6cc..ce04dbc 100644 --- a/.github/workflows/integration-talos-main-cron.yaml +++ b/.github/workflows/integration-s3-talos-main-cron.yaml @@ -1,6 +1,6 @@ # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. # -# Generated on 2025-07-22T11:53:08Z by kres b869533. +# Generated on 2025-08-13T15:31:47Z by kres 9f63e23. concurrency: group: ${{ github.head_ref || github.run_id }} @@ -8,7 +8,7 @@ concurrency: "on": schedule: - cron: 30 7 * * * -name: integration-talos-main-cron +name: integration-s3-talos-main-cron jobs: default: runs-on: @@ -52,9 +52,10 @@ jobs: driver: remote endpoint: tcp://buildkit-amd64.ci.svc.cluster.local:1234 timeout-minutes: 10 - - name: integration-talos-main + - name: integration-s3-talos-main env: REGISTRY: registry.dev.siderolabs.io - TEST_FLAGS: -test.schematic-service-repository=registry.dev.siderolabs.io/image-factory/schematic -test.installer-external-repository=registry.dev.siderolabs.io/siderolabs -test.installer-internal-repository=registry.dev.siderolabs.io/siderolabs -test.cache-repository=registry.dev.siderolabs.io/image-factory/cache + RUN_TESTS: TestIntegrationS3 + TEST_FLAGS: -test.schematic-service-repository=registry.dev.siderolabs.io/image-factory/schematic -test.installer-external-repository=registry.dev.siderolabs.io/siderolabs -test.installer-internal-repository=registry.dev.siderolabs.io/siderolabs -test.cache-repository=registry.dev.siderolabs.io/image-factory/cache -test.signing-cache-repository=127.0.0.1:5100/image-factory/signing-cache run: | - make integration-talos-main + make integration-s3-talos-main diff --git a/.github/workflows/lock.yml b/.github/workflows/lock.yml index c4d522a..d3f8ffc 100644 --- a/.github/workflows/lock.yml +++ b/.github/workflows/lock.yml @@ -1,6 +1,6 @@ # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. # -# Generated on 2025-07-22T11:53:08Z by kres b869533. +# Generated on 2025-08-13T14:42:25Z by kres 9f63e23. "on": schedule: diff --git a/.github/workflows/slack-notify-ci-failure.yaml b/.github/workflows/slack-notify-ci-failure.yaml new file mode 100644 index 0000000..e851cae --- /dev/null +++ b/.github/workflows/slack-notify-ci-failure.yaml @@ -0,0 +1,92 @@ +# THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. +# +# Generated on 2025-08-13T14:42:25Z by kres 9f63e23. + +"on": + workflow_run: + workflows: + - default + - integration-cdn-talos-main-cron + - integration-direct-talos-main-cron + - integration-s3-talos-main-cron + types: + - completed + branches: + - main +name: slack-notify-failure +jobs: + slack-notify: + runs-on: + - self-hosted + - generic + if: github.event.workflow_run.conclusion == 'failure' && github.event.workflow_run.event != 'pull_request' + steps: + - name: Slack Notify + uses: slackapi/slack-github-action@v2 + with: + method: chat.postMessage + payload: | + { + "channel": "ci-failure", + "text": "${{ github.event.workflow_run.conclusion }} - ${{ github.repository }}", + "icon_emoji": "${{ github.event.workflow_run.conclusion == 'success' && ':white_check_mark:' || github.event.workflow_run.conclusion == 'failure' && ':x:' || ':warning:' }}", + "username": "GitHub Actions", + "attachments": [ + { + "blocks": [ + { + "fields": [ + { + "text": "${{ github.event.workflow_run.event == 'pull_request' && format('*Pull Request:* {0} (`{1}`)\n<{2}/pull/{3}|{4}>', github.repository, github.ref_name, github.event.repository.html_url, steps.get-pr-number.outputs.pull_request_number, github.event.workflow_run.display_title) || format('*Build:* {0} (`{1}`)\n<{2}/commit/{3}|{4}>', github.repository, github.ref_name, github.event.repository.html_url, github.sha, github.event.workflow_run.display_title) }}", + "type": "mrkdwn" + }, + { + "text": "*Status:*\n`${{ github.event.workflow_run.conclusion }}`", + "type": "mrkdwn" + } + ], + "type": "section" + }, + { + "fields": [ + { + "text": "*Author:*\n`${{ github.actor }}`", + "type": "mrkdwn" + }, + { + "text": "*Event:*\n`${{ github.event.workflow_run.event }}`", + "type": "mrkdwn" + } + ], + "type": "section" + }, + { + "type": "divider" + }, + { + "elements": [ + { + "text": { + "text": "Logs", + "type": "plain_text" + }, + "type": "button", + "url": "${{ github.event.workflow_run.html_url }}" + }, + { + "text": { + "text": "Commit", + "type": "plain_text" + }, + "type": "button", + "url": "${{ github.event.repository.html_url }}/commit/${{ github.sha }}" + } + ], + "type": "actions" + } + ], + "color": "${{ github.event.workflow_run.conclusion == 'success' && '#2EB886' || github.event.workflow_run.conclusion == 'failure' && '#A30002' || '#FFCC00' }}" + } + ] + } + token: ${{ secrets.SLACK_BOT_TOKEN_V2 }} diff --git a/.github/workflows/slack-notify.yaml b/.github/workflows/slack-notify.yaml index 93afc83..75857d7 100644 --- a/.github/workflows/slack-notify.yaml +++ b/.github/workflows/slack-notify.yaml @@ -1,12 +1,14 @@ # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. # -# Generated on 2025-07-22T11:53:08Z by kres b869533. +# Generated on 2025-08-13T14:42:25Z by kres 9f63e23. "on": workflow_run: workflows: - default - - integration-talos-main-cron + - integration-cdn-talos-main-cron + - integration-direct-talos-main-cron + - integration-s3-talos-main-cron types: - completed name: slack-notify @@ -30,64 +32,66 @@ jobs: method: chat.postMessage payload: | { - "channel": "proj-talos-maintainers", + "channel": "ci-all", + "text": "${{ github.event.workflow_run.conclusion }} - ${{ github.repository }}", + "icon_emoji": "${{ github.event.workflow_run.conclusion == 'success' && ':white_check_mark:' || github.event.workflow_run.conclusion == 'failure' && ':x:' || ':warning:' }}", + "username": "GitHub Actions", "attachments": [ { - "color": "${{ github.event.workflow_run.conclusion == 'success' && '#2EB886' || github.event.workflow_run.conclusion == 'failure' && '#A30002' || '#FFCC00' }}", - "fallback": "test", "blocks": [ { - "type": "section", "fields": [ { - "type": "mrkdwn", - "text": "${{ github.event.workflow_run.event == 'pull_request' && format('*Pull Request:* {0} (`{1}`)\n<{2}/pull/{3}|{4}>', github.repository, github.ref_name, github.event.repository.html_url, steps.get-pr-number.outputs.pull_request_number, github.event.workflow_run.display_title) || format('*Build:* {0} (`{1}`)\n<{2}/commit/{3}|{4}>', github.repository, github.ref_name, github.event.repository.html_url, github.sha, github.event.workflow_run.display_title) }}" + "text": "${{ github.event.workflow_run.event == 'pull_request' && format('*Pull Request:* {0} (`{1}`)\n<{2}/pull/{3}|{4}>', github.repository, github.ref_name, github.event.repository.html_url, steps.get-pr-number.outputs.pull_request_number, github.event.workflow_run.display_title) || format('*Build:* {0} (`{1}`)\n<{2}/commit/{3}|{4}>', github.repository, github.ref_name, github.event.repository.html_url, github.sha, github.event.workflow_run.display_title) }}", + "type": "mrkdwn" }, { - "type": "mrkdwn", - "text": "*Status:*\n`${{ github.event.workflow_run.conclusion }}`" + "text": "*Status:*\n`${{ github.event.workflow_run.conclusion }}`", + "type": "mrkdwn" } - ] + ], + "type": "section" }, { - "type": "section", "fields": [ { - "type": "mrkdwn", - "text": "*Author:*\n`${{ github.actor }}`" + "text": "*Author:*\n`${{ github.actor }}`", + "type": "mrkdwn" }, { - "type": "mrkdwn", - "text": "*Event:*\n`${{ github.event.workflow_run.event }}`" + "text": "*Event:*\n`${{ github.event.workflow_run.event }}`", + "type": "mrkdwn" } - ] + ], + "type": "section" }, { "type": "divider" }, { - "type": "actions", "elements": [ { - "type": "button", "text": { - "type": "plain_text", - "text": "Logs" + "text": "Logs", + "type": "plain_text" }, + "type": "button", "url": "${{ github.event.workflow_run.html_url }}" }, { - "type": "button", "text": { - "type": "plain_text", - "text": "Commit" + "text": "Commit", + "type": "plain_text" }, + "type": "button", "url": "${{ github.event.repository.html_url }}/commit/${{ github.sha }}" } - ] + ], + "type": "actions" } - ] + ], + "color": "${{ github.event.workflow_run.conclusion == 'success' && '#2EB886' || github.event.workflow_run.conclusion == 'failure' && '#A30002' || '#FFCC00' }}" } ] } - token: ${{ secrets.SLACK_BOT_TOKEN }} + token: ${{ secrets.SLACK_BOT_TOKEN_V2 }} diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 13e7467..76f41bd 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -1,6 +1,6 @@ # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. # -# Generated on 2025-07-22T11:53:08Z by kres b869533. +# Generated on 2025-08-13T14:42:25Z by kres 9f63e23. "on": schedule: diff --git a/.kres.yaml b/.kres.yaml index da1f4ff..a118aee 100644 --- a/.kres.yaml +++ b/.kres.yaml @@ -2,6 +2,11 @@ kind: golang.Generate spec: versionPackagePath: internal/version --- +kind: golang.GoVulnCheck +spec: + ignore: + - GO-2025-3770 +--- kind: golang.Build spec: outputs: @@ -55,15 +60,31 @@ spec: toplevel: true - name: imager-tools toplevel: true - - name: integration.test + - name: integration-cdn.test toplevel: true - - name: integration + - name: integration-direct.test + toplevel: true + - name: integration-s3.test + toplevel: true + - name: integration-direct + toplevel: true + dependants: + - coverage + - name: integration-s3 + toplevel: true + dependants: + - coverage + - name: integration-cdn toplevel: true dependants: - coverage - name: update-to-talos-main toplevel: true - - name: integration-talos-main + - name: integration-cdn-talos-main + toplevel: true + - name: integration-direct-talos-main + toplevel: true + - name: integration-s3-talos-main toplevel: true - name: tailwind toplevel: true @@ -280,27 +301,83 @@ spec: dst: / --- kind: custom.Step -name: integration.test +name: integration-direct.test spec: docker: enabled: true stages: - - name: integration-build + - name: integration-direct-build description: builds the integration test binary from: base steps: - script: - command: go test -c -covermode=atomic -coverpkg=./... -tags integration ./internal/integration + command: go test -c -covermode=atomic -coverpkg=./... -tags integration,integration_direct ./internal/integration cache: - /root/.cache/go-build - /go/pkg - - name: integration.test + - name: integration-direct.test description: copies out the integration test binary steps: - copy: - from: integration-build + from: integration-direct-build src: /src/integration.test - dst: /integration.test + dst: /integration-direct.test + makefile: + enabled: true + phony: true + script: + - "@$(MAKE) local-$@ DEST=$(ARTIFACTS)" +--- +kind: custom.Step +name: integration-s3.test +spec: + docker: + enabled: true + stages: + - name: integration-s3-build + description: builds the integration test binary + from: base + steps: + - script: + command: go test -c -covermode=atomic -coverpkg=./... -tags integration,integration_s3 ./internal/integration + cache: + - /root/.cache/go-build + - /go/pkg + - name: integration-s3.test + description: copies out the integration test binary + steps: + - copy: + from: integration-s3-build + src: /src/integration.test + dst: /integration-s3.test + makefile: + enabled: true + phony: true + script: + - "@$(MAKE) local-$@ DEST=$(ARTIFACTS)" +--- +kind: custom.Step +name: integration-cdn.test +spec: + docker: + enabled: true + stages: + - name: integration-cdn-build + description: builds the integration test binary + from: base + steps: + - script: + command: go test -c -covermode=atomic -coverpkg=./... -tags integration,integration_cdn ./internal/integration + cache: + - /root/.cache/go-build + - /go/pkg + - name: integration-cdn.test + description: copies out the integration test binary + steps: + - copy: + from: integration-cdn-build + src: /src/integration.test + dst: /integration-cdn.test makefile: enabled: true phony: true @@ -345,16 +422,62 @@ spec: - '@$(MAKE) local-copy-out-go-mod DEST=. TARGET_ARGS="--no-cache-filter=update-to-talos-main"' --- kind: custom.Step -name: integration +name: integration-direct spec: makefile: enabled: true phony: true depends: - - integration.test + - integration-direct.test + script: + - "@$(MAKE) image-image-factory PUSH=true" + - docker pull $(REGISTRY)/$(USERNAME)/image-factory:$(TAG) + - docker rm -f local-if || true + - docker run -d -p 5100:5000 --name=local-if registry:3 + - docker run --rm --net=host --privileged -v /dev:/dev -v /var/run:/var/run -v $(PWD)/$(ARTIFACTS)/:/out/ -v $(PWD)/$(ARTIFACTS)/integration-direct.test:/bin/integration-direct.test:ro --entrypoint /bin/integration-direct.test $(REGISTRY)/$(USERNAME)/image-factory:$(TAG) -test.v $(TEST_FLAGS) -test.coverprofile=/out/coverage-integration-direct.txt -test.run $(RUN_TESTS) + - docker rm -f local-if + ghaction: + enabled: true + condition: on-pull-request + environment: + REGISTRY: registry.dev.siderolabs.io + RUN_TESTS: TestIntegrationDirect + TEST_FLAGS: "-test.schematic-service-repository=127.0.0.1:5100/image-factory/schematic -test.installer-external-repository=127.0.0.1:5100/siderolabs -test.installer-internal-repository=127.0.0.1:5100/siderolabs -test.cache-repository=127.0.0.1:5100/image-factory/cache -test.signing-cache-repository=127.0.0.1:5100/image-factory/signing-cache" +--- +kind: custom.Step +name: integration-s3 +spec: + makefile: + enabled: true + phony: true + depends: + - integration-s3.test + script: + - "@$(MAKE) image-image-factory PUSH=true" + - docker pull $(REGISTRY)/$(USERNAME)/image-factory:$(TAG) + - docker rm -f local-if || true + - docker run -d -p 5100:5000 --name=local-if registry:3 + - docker run --rm --net=host --privileged -v /dev:/dev -v /var/run:/var/run -v $(PWD)/$(ARTIFACTS)/:/out/ -v $(PWD)/$(ARTIFACTS)/integration-s3.test:/bin/integration-s3.test:ro --entrypoint /bin/integration-s3.test $(REGISTRY)/$(USERNAME)/image-factory:$(TAG) -test.v $(TEST_FLAGS) -test.coverprofile=/out/coverage-integration-s3.txt -test.run $(RUN_TESTS) + - docker rm -f local-if + ghaction: + enabled: true + condition: on-pull-request + environment: + REGISTRY: registry.dev.siderolabs.io + RUN_TESTS: TestIntegrationS3 + TEST_FLAGS: "-test.schematic-service-repository=127.0.0.1:5100/image-factory/schematic -test.installer-external-repository=127.0.0.1:5100/siderolabs -test.installer-internal-repository=127.0.0.1:5100/siderolabs -test.cache-repository=127.0.0.1:5100/image-factory/cache -test.signing-cache-repository=127.0.0.1:5100/image-factory/signing-cache" +--- +kind: custom.Step +name: integration-cdn +spec: + makefile: + enabled: true + phony: true + depends: + - integration-cdn.test variables: - name: RUN_TESTS - defaultValue: TestIntegration + defaultValue: TestIntegrationCDN - name: TEST_FLAGS defaultValue: "" script: @@ -362,38 +485,87 @@ spec: - docker pull $(REGISTRY)/$(USERNAME)/image-factory:$(TAG) - docker rm -f local-if || true - docker run -d -p 5100:5000 --name=local-if registry:3 - - docker run --rm --net=host --privileged -v /dev:/dev -v /var/run:/var/run -v $(PWD)/$(ARTIFACTS)/:/out/ -v $(PWD)/$(ARTIFACTS)/integration.test:/bin/integration.test:ro --entrypoint /bin/integration.test $(REGISTRY)/$(USERNAME)/image-factory:$(TAG) -test.v $(TEST_FLAGS) -test.coverprofile=/out/coverage-integration.txt -test.run $(RUN_TESTS) + - docker run --rm --net=host --privileged -v /dev:/dev -v /var/run:/var/run -v $(PWD)/$(ARTIFACTS)/:/out/ -v $(PWD)/$(ARTIFACTS)/integration-cdn.test:/bin/integration-cdn.test:ro --entrypoint /bin/integration-cdn.test $(REGISTRY)/$(USERNAME)/image-factory:$(TAG) -test.v $(TEST_FLAGS) -test.coverprofile=/out/coverage-integration-cdn.txt -test.run $(RUN_TESTS) - docker rm -f local-if ghaction: enabled: true condition: on-pull-request environment: REGISTRY: registry.dev.siderolabs.io - TEST_FLAGS: "-test.schematic-service-repository=127.0.0.1:5100/image-factory/schematic -test.installer-external-repository=127.0.0.1:5100/siderolabs -test.installer-internal-repository=127.0.0.1:5100/siderolabs -test.cache-repository=127.0.0.1:5100/image-factory/cache" + RUN_TESTS: TestIntegrationCDN + TEST_FLAGS: "-test.schematic-service-repository=127.0.0.1:5100/image-factory/schematic -test.installer-external-repository=127.0.0.1:5100/siderolabs -test.installer-internal-repository=127.0.0.1:5100/siderolabs -test.cache-repository=127.0.0.1:5100/image-factory/cache -test.signing-cache-repository=127.0.0.1:5100/image-factory/signing-cache" --- kind: custom.Step -name: integration-talos-main +name: integration-direct-talos-main spec: makefile: enabled: true phony: true depends: - update-to-talos-main - variables: - - name: RUN_TESTS - defaultValue: TestIntegration - - name: TEST_FLAGS - defaultValue: "" script: - - "@$(MAKE) integration" + - "@$(MAKE) integration-direct" ghaction: enabled: true cronOnly: true environment: REGISTRY: registry.dev.siderolabs.io - TEST_FLAGS: "-test.schematic-service-repository=registry.dev.siderolabs.io/image-factory/schematic -test.installer-external-repository=registry.dev.siderolabs.io/siderolabs -test.installer-internal-repository=registry.dev.siderolabs.io/siderolabs -test.cache-repository=registry.dev.siderolabs.io/image-factory/cache" + RUN_TESTS: TestIntegrationDirect + TEST_FLAGS: "-test.schematic-service-repository=registry.dev.siderolabs.io/image-factory/schematic -test.installer-external-repository=registry.dev.siderolabs.io/siderolabs -test.installer-internal-repository=registry.dev.siderolabs.io/siderolabs -test.cache-repository=registry.dev.siderolabs.io/image-factory/cache -test.signing-cache-repository=127.0.0.1:5100/image-factory/signing-cache" jobs: - - name: integration-talos-main + - name: integration-direct-talos-main + runnerLabels: + - generic + triggerLabels: + - integration/talos-main + crons: + - "30 7 * * *" +--- +kind: custom.Step +name: integration-s3-talos-main +spec: + makefile: + enabled: true + phony: true + depends: + - update-to-talos-main + script: + - "@$(MAKE) integration-s3" + ghaction: + enabled: true + cronOnly: true + environment: + REGISTRY: registry.dev.siderolabs.io + RUN_TESTS: TestIntegrationS3 + TEST_FLAGS: "-test.schematic-service-repository=registry.dev.siderolabs.io/image-factory/schematic -test.installer-external-repository=registry.dev.siderolabs.io/siderolabs -test.installer-internal-repository=registry.dev.siderolabs.io/siderolabs -test.cache-repository=registry.dev.siderolabs.io/image-factory/cache -test.signing-cache-repository=127.0.0.1:5100/image-factory/signing-cache" + jobs: + - name: integration-s3-talos-main + runnerLabels: + - generic + triggerLabels: + - integration/talos-main + crons: + - "30 7 * * *" +--- +kind: custom.Step +name: integration-cdn-talos-main +spec: + makefile: + enabled: true + phony: true + depends: + - update-to-talos-main + script: + - "@$(MAKE) integration-cdn" + ghaction: + enabled: true + cronOnly: true + environment: + REGISTRY: registry.dev.siderolabs.io + RUN_TESTS: TestIntegrationCDN + TEST_FLAGS: "-test.schematic-service-repository=registry.dev.siderolabs.io/image-factory/schematic -test.installer-external-repository=registry.dev.siderolabs.io/siderolabs -test.installer-internal-repository=registry.dev.siderolabs.io/siderolabs -test.cache-repository=registry.dev.siderolabs.io/image-factory/cache -test.signing-cache-repository=127.0.0.1:5100/image-factory/signing-cache" + jobs: + - name: integration-cdn-talos-main runnerLabels: - generic triggerLabels: @@ -410,7 +582,9 @@ kind: service.CodeCov spec: targetThreshold: 50 inputPaths: - - coverage-integration.txt + - coverage-integration-direct.txt + - coverage-integration-s3.txt + - coverage-integration-cdn.txt --- kind: custom.Step name: tailwind @@ -434,8 +608,6 @@ spec: src: package.json package-lock.json dst: . - script: - cache: - - /src/node_modules command: bun install - name: tailwind-update description: "tailwind update" @@ -448,8 +620,6 @@ spec: src: internal/frontend/http dst: internal/frontend/http - script: - cache: - - /src/node_modules command: node_modules/.bin/tailwindcss -i internal/frontend/http/css/input.css -o internal/frontend/http/css/output.css --minify - name: tailwind-copy description: "Copies assets" diff --git a/Dockerfile b/Dockerfile index 6fee844..b892cda 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. # -# Generated on 2025-07-29T12:16:54Z by kres dd1ed6f. +# Generated on 2025-08-14T09:35:32Z by kres df7e867-dirty. ARG TOOLCHAIN ARG PKGS_PREFIX @@ -83,11 +83,11 @@ FROM ${PKGS_PREFIX}/zstd:${PKGS} AS pkg-zstd FROM --platform=${BUILDPLATFORM} docker.io/oven/bun:1.2.4-alpine AS tailwind-base WORKDIR /src COPY package.json package-lock.json . -RUN --mount=type=cache,target=/src/node_modules,id=image-factory/src/node_modules bun install +RUN bun install # base toolchain image FROM --platform=${BUILDPLATFORM} ${TOOLCHAIN} AS toolchain -RUN apk --update --no-cache add bash curl build-base jq protoc protobuf-dev +RUN apk --update --no-cache add bash build-base curl jq protoc protobuf-dev # copies the imager tools FROM scratch AS imager-tools @@ -127,7 +127,7 @@ COPY --from=pkg-zstd / / FROM tailwind-base AS tailwind-update COPY tailwind.config.js . COPY internal/frontend/http internal/frontend/http -RUN --mount=type=cache,target=/src/node_modules,id=image-factory/src/node_modules node_modules/.bin/tailwindcss -i internal/frontend/http/css/input.css -o internal/frontend/http/css/output.css --minify +RUN node_modules/.bin/tailwindcss -i internal/frontend/http/css/input.css -o internal/frontend/http/css/output.css --minify # build tools FROM --platform=${BUILDPLATFORM} toolchain AS tools @@ -141,15 +141,15 @@ ENV GOEXPERIMENT=${GOEXPERIMENT} ENV GOPATH=/go ARG DEEPCOPY_VERSION RUN --mount=type=cache,target=/root/.cache/go-build,id=image-factory/root/.cache/go-build --mount=type=cache,target=/go/pkg,id=image-factory/go/pkg go install github.com/siderolabs/deep-copy@${DEEPCOPY_VERSION} \ - && mv /go/bin/deep-copy /bin/deep-copy + && mv /go/bin/deep-copy /bin/deep-copy ARG GOLANGCILINT_VERSION RUN --mount=type=cache,target=/root/.cache/go-build,id=image-factory/root/.cache/go-build --mount=type=cache,target=/go/pkg,id=image-factory/go/pkg go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@${GOLANGCILINT_VERSION} \ - && mv /go/bin/golangci-lint /bin/golangci-lint + && mv /go/bin/golangci-lint /bin/golangci-lint RUN --mount=type=cache,target=/root/.cache/go-build,id=image-factory/root/.cache/go-build --mount=type=cache,target=/go/pkg,id=image-factory/go/pkg go install golang.org/x/vuln/cmd/govulncheck@latest \ - && mv /go/bin/govulncheck /bin/govulncheck + && mv /go/bin/govulncheck /bin/govulncheck ARG GOFUMPT_VERSION RUN go install mvdan.cc/gofumpt@${GOFUMPT_VERSION} \ - && mv /go/bin/gofumpt /bin/gofumpt + && mv /go/bin/gofumpt /bin/gofumpt # Copies assets FROM scratch AS tailwind-copy @@ -177,8 +177,16 @@ RUN mkdir -p internal/version/data && \ echo -n ${TAG} > internal/version/data/tag # builds the integration test binary -FROM base AS integration-build -RUN --mount=type=cache,target=/root/.cache/go-build,id=image-factory/root/.cache/go-build --mount=type=cache,target=/go/pkg,id=image-factory/go/pkg go test -c -covermode=atomic -coverpkg=./... -tags integration ./internal/integration +FROM base AS integration-cdn-build +RUN --mount=type=cache,target=/root/.cache/go-build,id=image-factory/root/.cache/go-build --mount=type=cache,target=/go/pkg,id=image-factory/go/pkg go test -c -covermode=atomic -coverpkg=./... -tags integration,integration_cdn ./internal/integration + +# builds the integration test binary +FROM base AS integration-direct-build +RUN --mount=type=cache,target=/root/.cache/go-build,id=image-factory/root/.cache/go-build --mount=type=cache,target=/go/pkg,id=image-factory/go/pkg go test -c -covermode=atomic -coverpkg=./... -tags integration,integration_direct ./internal/integration + +# builds the integration test binary +FROM base AS integration-s3-build +RUN --mount=type=cache,target=/root/.cache/go-build,id=image-factory/root/.cache/go-build --mount=type=cache,target=/go/pkg,id=image-factory/go/pkg go test -c -covermode=atomic -coverpkg=./... -tags integration,integration_s3 ./internal/integration # runs gofumpt FROM base AS lint-gofumpt @@ -191,11 +199,19 @@ COPY .golangci.yml . ENV GOGC=50 RUN --mount=type=cache,target=/root/.cache/go-build,id=image-factory/root/.cache/go-build --mount=type=cache,target=/root/.cache/golangci-lint,id=image-factory/root/.cache/golangci-lint,sharing=locked --mount=type=cache,target=/go/pkg,id=image-factory/go/pkg golangci-lint run --config .golangci.yml +# runs golangci-lint fmt +FROM base AS lint-golangci-lint-fmt-run +WORKDIR /src +COPY .golangci.yml . +ENV GOGC=50 +RUN --mount=type=cache,target=/root/.cache/go-build,id=image-factory/root/.cache/go-build --mount=type=cache,target=/root/.cache/golangci-lint,id=image-factory/root/.cache/golangci-lint,sharing=locked --mount=type=cache,target=/go/pkg,id=image-factory/go/pkg golangci-lint fmt --config .golangci.yml +RUN --mount=type=cache,target=/root/.cache/go-build,id=image-factory/root/.cache/go-build --mount=type=cache,target=/root/.cache/golangci-lint,id=image-factory/root/.cache/golangci-lint,sharing=locked --mount=type=cache,target=/go/pkg,id=image-factory/go/pkg golangci-lint run --fix --issues-exit-code 0 --config .golangci.yml + # runs govulncheck FROM base AS lint-govulncheck +COPY --chmod=0755 hack/govulncheck.sh ./hack/govulncheck.sh WORKDIR /src -COPY ./hack/govulncheck.sh ./hack/govulncheck.sh -RUN --mount=type=cache,target=/root/.cache/go-build,id=image-factory/root/.cache/go-build --mount=type=cache,target=/go/pkg,id=image-factory/go/pkg ./hack/govulncheck.sh ./... +RUN --mount=type=cache,target=/root/.cache/go-build,id=image-factory/root/.cache/go-build --mount=type=cache,target=/go/pkg,id=image-factory/go/pkg ./hack/govulncheck.sh -exclude 'GO-2025-3770' ./... # runs unit-tests with race detector FROM base AS unit-tests-race @@ -221,8 +237,20 @@ RUN echo -n 'undefined' > internal/version/data/sha && \ echo -n ${ABBREV_TAG} > internal/version/data/tag # copies out the integration test binary -FROM scratch AS integration.test -COPY --from=integration-build /src/integration.test /integration.test +FROM scratch AS integration-cdn.test +COPY --from=integration-cdn-build /src/integration.test /integration-cdn.test + +# copies out the integration test binary +FROM scratch AS integration-direct.test +COPY --from=integration-direct-build /src/integration.test /integration-direct.test + +# copies out the integration test binary +FROM scratch AS integration-s3.test +COPY --from=integration-s3-build /src/integration.test /integration-s3.test + +# clean golangci-lint fmt output +FROM scratch AS lint-golangci-lint-fmt +COPY --from=lint-golangci-lint-fmt-run /src . FROM scratch AS unit-tests COPY --from=unit-tests-run /src/coverage.txt /coverage-unit-tests.txt @@ -278,3 +306,4 @@ COPY --from=image-factory image-factory-linux-${TARGETARCH} /usr/bin/image-facto COPY --from=imager-tools / / LABEL org.opencontainers.image.source=https://github.com/siderolabs/image-factory ENTRYPOINT ["/usr/bin/image-factory"] + diff --git a/Makefile b/Makefile index 3db55f5..44594c3 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. # -# Generated on 2025-07-29T12:16:54Z by kres dd1ed6f. +# Generated on 2025-08-14T09:04:59Z by kres 9f63e23-dirty. # common variables @@ -80,7 +80,7 @@ TOOLCHAIN ?= docker.io/golang:1.24-alpine PKGS_PREFIX ?= ghcr.io/siderolabs PKGS ?= v1.11.0 -RUN_TESTS ?= TestIntegration +RUN_TESTS ?= TestIntegrationCDN TEST_FLAGS ?= # help menu @@ -178,6 +178,9 @@ generate: ## Generate .proto definitions. lint-golangci-lint: ## Runs golangci-lint linter. @$(MAKE) target-$@ +lint-golangci-lint-fmt: ## Runs golangci-lint formatter and tries to fix issues automatically. + @$(MAKE) local-$@ DEST=. + lint-gofumpt: ## Runs gofumpt linter. @$(MAKE) target-$@ @@ -204,13 +207,31 @@ unit-tests: ## Performs unit tests unit-tests-race: ## Performs unit tests with race detection enabled. @$(MAKE) target-$@ -.PHONY: integration -integration: integration.test +.PHONY: integration-direct +integration-direct: integration-direct.test @$(MAKE) image-image-factory PUSH=true docker pull $(REGISTRY)/$(USERNAME)/image-factory:$(TAG) docker rm -f local-if || true docker run -d -p 5100:5000 --name=local-if registry:3 - docker run --rm --net=host --privileged -v /dev:/dev -v /var/run:/var/run -v $(PWD)/$(ARTIFACTS)/:/out/ -v $(PWD)/$(ARTIFACTS)/integration.test:/bin/integration.test:ro --entrypoint /bin/integration.test $(REGISTRY)/$(USERNAME)/image-factory:$(TAG) -test.v $(TEST_FLAGS) -test.coverprofile=/out/coverage-integration.txt -test.run $(RUN_TESTS) + docker run --rm --net=host --privileged -v /dev:/dev -v /var/run:/var/run -v $(PWD)/$(ARTIFACTS)/:/out/ -v $(PWD)/$(ARTIFACTS)/integration-direct.test:/bin/integration-direct.test:ro --entrypoint /bin/integration-direct.test $(REGISTRY)/$(USERNAME)/image-factory:$(TAG) -test.v $(TEST_FLAGS) -test.coverprofile=/out/coverage-integration-direct.txt -test.run $(RUN_TESTS) + docker rm -f local-if + +.PHONY: integration-s3 +integration-s3: integration-s3.test + @$(MAKE) image-image-factory PUSH=true + docker pull $(REGISTRY)/$(USERNAME)/image-factory:$(TAG) + docker rm -f local-if || true + docker run -d -p 5100:5000 --name=local-if registry:3 + docker run --rm --net=host --privileged -v /dev:/dev -v /var/run:/var/run -v $(PWD)/$(ARTIFACTS)/:/out/ -v $(PWD)/$(ARTIFACTS)/integration-s3.test:/bin/integration-s3.test:ro --entrypoint /bin/integration-s3.test $(REGISTRY)/$(USERNAME)/image-factory:$(TAG) -test.v $(TEST_FLAGS) -test.coverprofile=/out/coverage-integration-s3.txt -test.run $(RUN_TESTS) + docker rm -f local-if + +.PHONY: integration-cdn +integration-cdn: integration-cdn.test + @$(MAKE) image-image-factory PUSH=true + docker pull $(REGISTRY)/$(USERNAME)/image-factory:$(TAG) + docker rm -f local-if || true + docker run -d -p 5100:5000 --name=local-if registry:3 + docker run --rm --net=host --privileged -v /dev:/dev -v /var/run:/var/run -v $(PWD)/$(ARTIFACTS)/:/out/ -v $(PWD)/$(ARTIFACTS)/integration-cdn.test:/bin/integration-cdn.test:ro --entrypoint /bin/integration-cdn.test $(REGISTRY)/$(USERNAME)/image-factory:$(TAG) -test.v $(TEST_FLAGS) -test.coverprofile=/out/coverage-integration-cdn.txt -test.run $(RUN_TESTS) docker rm -f local-if .PHONY: $(ARTIFACTS)/image-factory-linux-amd64 @@ -247,17 +268,33 @@ imager-base: .PHONY: imager-tools imager-tools: -.PHONY: integration.test -integration.test: +.PHONY: integration-cdn.test +integration-cdn.test: + @$(MAKE) local-$@ DEST=$(ARTIFACTS) + +.PHONY: integration-direct.test +integration-direct.test: + @$(MAKE) local-$@ DEST=$(ARTIFACTS) + +.PHONY: integration-s3.test +integration-s3.test: @$(MAKE) local-$@ DEST=$(ARTIFACTS) .PHONY: update-to-talos-main update-to-talos-main: @$(MAKE) local-copy-out-go-mod DEST=. TARGET_ARGS="--no-cache-filter=update-to-talos-main" -.PHONY: integration-talos-main -integration-talos-main: update-to-talos-main - @$(MAKE) integration +.PHONY: integration-cdn-talos-main +integration-cdn-talos-main: update-to-talos-main + @$(MAKE) integration-cdn + +.PHONY: integration-direct-talos-main +integration-direct-talos-main: update-to-talos-main + @$(MAKE) integration-direct + +.PHONY: integration-s3-talos-main +integration-s3-talos-main: update-to-talos-main + @$(MAKE) integration-s3 .PHONY: tailwind tailwind: diff --git a/cmd/image-factory/cmd/options.go b/cmd/image-factory/cmd/options.go index baf2412..d7253c2 100644 --- a/cmd/image-factory/cmd/options.go +++ b/cmd/image-factory/cmd/options.go @@ -99,6 +99,10 @@ type Options struct { //nolint:govet // Leave empty to disable. MetricsListenAddr string + // MetricsNamespace is the namespace for Prometheus metrics. + // It's not user-configurable, but set by the image factory tests. + MetricsNamespace string + // SecureBoot settings. SecureBoot SecureBootOptions } diff --git a/cmd/image-factory/cmd/service.go b/cmd/image-factory/cmd/service.go index 0ffe404..3cf2248 100644 --- a/cmd/image-factory/cmd/service.go +++ b/cmd/image-factory/cmd/service.go @@ -123,6 +123,7 @@ func RunFactory(ctx context.Context, logger *zap.Logger, opts Options) error { frontendOptions.RemoteOptions = append(frontendOptions.RemoteOptions, remoteOptions()...) frontendOptions.RegistryRefreshInterval = opts.RegistryRefreshInterval + frontendOptions.MetricsNamespace = opts.MetricsNamespace frontendHTTP, err := frontendhttp.NewFrontend(logger, configFactory, assetBuilder, artifactsManager, secureBootService, frontendOptions) if err != nil { @@ -329,7 +330,9 @@ func buildAssetBuilder(logger *zap.Logger, artifactsManager *artifacts.Manager, } builderOptions := asset.Options{ + MetricsNamespace: opts.MetricsNamespace, AllowedConcurrency: opts.AssetBuildMaxConcurrency, + GetAfterPut: opts.CacheS3Enabled, } builder, err := asset.NewBuilder(logger, artifactsManager, cache, builderOptions) @@ -359,7 +362,9 @@ func buildSchematicFactory(logger *zap.Logger, opts Options) (*schematic.Factory return nil, fmt.Errorf("failed to initialize registry storage: %w", err) } - factory := schematic.NewFactory(logger, schematiccache.NewCache(storage), schematic.Options{}) + c := schematiccache.NewCache(storage, schematiccache.Options{MetricsNamespace: opts.MetricsNamespace}) + + factory := schematic.NewFactory(logger, c, schematic.Options{MetricsNamespace: opts.MetricsNamespace}) prometheus.MustRegister(factory) diff --git a/hack/govulncheck.sh b/hack/govulncheck.sh index b1fa1d7..121c50b 100755 --- a/hack/govulncheck.sh +++ b/hack/govulncheck.sh @@ -1,28 +1,48 @@ -#!/usr/bin/env bash +#!/bin/bash # Source: https://github.com/tianon/gosu/blob/e157efb/govulncheck-with-excludes.sh # Licensed under the Apache License, Version 2.0 # Copyright Tianon Gravi set -Eeuo pipefail -# a wrapper / replacement for "govulncheck" which allows for excluding vulnerabilities +exclude_arg="" +pass_args=() + +while [[ $# -gt 0 ]]; do + case "$1" in + -exclude) + exclude_arg="$2" + shift 2 + ;; + *) + pass_args+=("$1") + shift + ;; + esac +done + +if [[ -n "$exclude_arg" ]]; then + excludeVulns="$(jq -nc --arg list "$exclude_arg" '$list | split(",")')" +else + excludeVulns="[]" +fi -excludeVulns="$(jq -nc '[ - "GO-2025-3770", - empty # trailing comma hack (makes diffs smaller) -]')" export excludeVulns +# Debug print +echo "excludeVulns = $excludeVulns" +echo "Passing args: ${pass_args[*]}" + if ! command -v govulncheck > /dev/null; then printf "govulncheck not installed" exit 1 fi -if out="$(govulncheck "$@")"; then +if out="$(govulncheck "${pass_args[@]}")"; then printf '%s\n' "$out" exit 0 fi -json="$(govulncheck -json "$@")" +json="$(govulncheck -json "${pass_args[@]}")" vulns="$(jq <<<"$json" -cs ' ( diff --git a/internal/asset/asset.go b/internal/asset/asset.go index 425ef60..9a278cf 100644 --- a/internal/asset/asset.go +++ b/internal/asset/asset.go @@ -37,20 +37,26 @@ type BootAsset interface { // Builder is the asset builder. type Builder struct { - logger *zap.Logger - cache cache.Cache - artifactsManager *artifacts.Manager - sf singleflight.Group - semaphore chan struct{} - - metricAssetsCached, metricAssetsBuilt *prometheus.CounterVec - metricAssetBytesCached, metricAssetBytesBuilt *prometheus.CounterVec - metricConcurrencyLatency, metricBuildLatency prometheus.Histogram + metricConcurrencyLatency prometheus.Histogram + cache cache.Cache + metricBuildLatency prometheus.Histogram + sf singleflight.Group + metricAssetsCached *prometheus.CounterVec + logger *zap.Logger + metricAssetsBuilt *prometheus.CounterVec + metricAssetBytesCached *prometheus.CounterVec + metricAssetBytesBuilt *prometheus.CounterVec + metricAssetCachedErrors *prometheus.CounterVec + semaphore chan struct{} + artifactsManager *artifacts.Manager + getAfterPut bool } // Options configures the asset builder. type Options struct { + MetricsNamespace string AllowedConcurrency int + GetAfterPut bool } // NewBuilder creates a new asset builder. @@ -60,47 +66,62 @@ func NewBuilder(logger *zap.Logger, artifactsManager *artifacts.Manager, cache c cache: cache, artifactsManager: artifactsManager, semaphore: make(chan struct{}, options.AllowedConcurrency), + getAfterPut: options.GetAfterPut, metricAssetsCached: prometheus.NewCounterVec( prometheus.CounterOpts{ - Name: "image_factory_assets_delivered_cached_total", - Help: "Number of assets retrieved from cache.", + Name: "image_factory_assets_delivered_cached_total", + Help: "Number of assets retrieved from cache.", + Namespace: options.MetricsNamespace, }, []string{"talos_version", "output_kind", "arch"}, ), metricAssetsBuilt: prometheus.NewCounterVec( prometheus.CounterOpts{ - Name: "image_factory_assets_built_total", - Help: "Number of assets built (missing in the cache).", + Name: "image_factory_assets_built_total", + Help: "Number of assets built (missing in the cache).", + Namespace: options.MetricsNamespace, }, []string{"talos_version", "output_kind", "arch"}, ), metricAssetBytesCached: prometheus.NewCounterVec( prometheus.CounterOpts{ - Name: "image_factory_assets_cached_bytes_total", - Help: "Number of bytes delivered with cached assets.", + Name: "image_factory_assets_cached_bytes_total", + Help: "Number of bytes delivered with cached assets.", + Namespace: options.MetricsNamespace, }, []string{"talos_version", "output_kind", "arch"}, ), metricAssetBytesBuilt: prometheus.NewCounterVec( prometheus.CounterOpts{ - Name: "image_factory_assets_built_bytes_total", - Help: "Number of bytes delivered with assets built (missing in the cache).", + Name: "image_factory_assets_built_bytes_total", + Help: "Number of bytes delivered with assets built (missing in the cache).", + Namespace: options.MetricsNamespace, + }, + []string{"talos_version", "output_kind", "arch"}, + ), + metricAssetCachedErrors: prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "image_factory_assets_cached_errors_total", + Help: "Number of errors retrieving assets from cache.", + Namespace: options.MetricsNamespace, }, []string{"talos_version", "output_kind", "arch"}, ), metricConcurrencyLatency: prometheus.NewHistogram( prometheus.HistogramOpts{ - Name: "image_factory_assets_concurrency_latency_seconds", - Help: "Latency of asset build related to the concurrency limit (waiting for available workers).", - Buckets: []float64{1, 10, 60, 180, 600}, + Name: "image_factory_assets_concurrency_latency_seconds", + Help: "Latency of asset build related to the concurrency limit (waiting for available workers).", + Buckets: []float64{1, 10, 60, 180, 600}, + Namespace: options.MetricsNamespace, }, ), metricBuildLatency: prometheus.NewHistogram( prometheus.HistogramOpts{ - Name: "image_factory_assets_build_latency_seconds", - Help: "Latency of asset build for the build itself (excluding waiting for available workers).", - Buckets: []float64{1, 10, 60, 180, 600}, + Name: "image_factory_assets_build_latency_seconds", + Help: "Latency of asset build for the build itself (excluding waiting for available workers).", + Buckets: []float64{1, 10, 60, 180, 600}, + Namespace: options.MetricsNamespace, }, ), }, nil @@ -179,15 +200,24 @@ func (b *Builder) buildAndCache(profileHash string, prof profile.Profile, versio b.logger.Error("error putting asset to cache", zap.Error(err), zap.String("profile_hash", profileHash)) } - cachedAsset, err := b.cache.Get(ctx, profileHash) - if err == nil { - b.metricAssetsCached.WithLabelValues(versionString, prof.Output.Kind.String(), prof.Arch).Inc() - b.metricAssetBytesCached.WithLabelValues(versionString, prof.Output.Kind.String(), prof.Arch).Add(float64(asset.Size())) + // if the asset delivery requires a redirect to the cache, we need to return the cached asset + // so that the client can download it directly from the cache. + if b.getAfterPut { + cachedAsset, err := b.cache.Get(ctx, profileHash) + if err == nil { + b.metricAssetsCached.WithLabelValues(versionString, prof.Output.Kind.String(), prof.Arch).Inc() + b.metricAssetBytesCached.WithLabelValues(versionString, prof.Output.Kind.String(), prof.Arch).Add(float64(asset.Size())) - return cachedAsset, nil + return cachedAsset, nil + } + + // if we failed to get the asset from cache, return the built asset anyway + b.logger.Warn("failed to get object from cache, falling back to direct delivery", zap.Error(err)) + + b.metricAssetCachedErrors.WithLabelValues(versionString, prof.Output.Kind.String(), prof.Arch).Inc() } - return nil, fmt.Errorf("error getting asset from cache: %w", err) + return asset, nil } // build the asset using Talos imager. @@ -285,6 +315,8 @@ func (b *Builder) Collect(ch chan<- prometheus.Metric) { b.metricAssetBytesBuilt.Collect(ch) b.metricAssetBytesCached.Collect(ch) + b.metricAssetCachedErrors.Collect(ch) + b.metricBuildLatency.Collect(ch) b.metricConcurrencyLatency.Collect(ch) } diff --git a/internal/frontend/http/http.go b/internal/frontend/http/http.go index a836345..6c82101 100644 --- a/internal/frontend/http/http.go +++ b/internal/frontend/http/http.go @@ -55,16 +55,14 @@ type Frontend struct { // Options configures the HTTP frontend. type Options struct { - ExternalURL *url.URL - ExternalPXEURL *url.URL - + CacheSigningKey crypto.PrivateKey + ExternalURL *url.URL + ExternalPXEURL *url.URL InstallerInternalRepository name.Repository InstallerExternalRepository name.Repository - - CacheSigningKey crypto.PrivateKey - - RemoteOptions []remote.Option - RegistryRefreshInterval time.Duration + MetricsNamespace string + RemoteOptions []remote.Option + RegistryRefreshInterval time.Duration } // NewFrontend creates a new HTTP frontend. @@ -105,7 +103,9 @@ func NewFrontend( // monitoring middleware mdlw := middleware.New(middleware.Config{ - Recorder: metrics.NewRecorder(metrics.Config{}), + Recorder: metrics.NewRecorder(metrics.Config{ + Prefix: opts.MetricsNamespace, + }), }) registerRoute := func(registrator func(string, httprouter.Handle), path string, handler func(ctx context.Context, w http.ResponseWriter, r *http.Request, p httprouter.Params) error) { diff --git a/internal/integration/integration_test.go b/internal/integration/integration_test.go index 3ecf6ea..1aa23e6 100644 --- a/internal/integration/integration_test.go +++ b/internal/integration/integration_test.go @@ -46,7 +46,6 @@ func setupFactory(t *testing.T, options cmd.Options) (context.Context, string) { options.SchematicServiceRepository = schematicFactoryRepositoryFlag options.InstallerExternalRepository = installerExternalRepository options.InstallerInternalRepository = installerInternalRepository - options.CacheRepository = cacheRepository options.RegistryRefreshInterval = time.Minute // use a short interval for the tests setupSecureBoot(t, &options) @@ -121,14 +120,11 @@ func healthcheck(url string) func() error { } const ( - bucket = "image-factory" - bucketPrefix = "/" + bucket - s3Access = "AKIA6Z4C7N3S2JD3JH9A" s3Secret = "y1rE4xZnqO6xvM7L0jFD3EXAMPLEnG4K2vOfLp8Iv9" ) -func setupS3(t *testing.T, pool *dockertest.Pool) string { +func setupS3(t *testing.T, pool *dockertest.Pool, bucket string) string { t.Helper() _, port, err := net.SplitHostPort(findListenAddr(t)) @@ -167,16 +163,13 @@ func setupS3(t *testing.T, pool *dockertest.Pool) string { }) require.NoError(t, err) - t.Setenv("AWS_ACCESS_KEY_ID", s3Access) - t.Setenv("AWS_SECRET_ACCESS_KEY", s3Secret) - return endpoint } //go:embed testdata/templates/nginx.sh var nginxConfigTemplate string -func setupMockCDN(t *testing.T, pool *dockertest.Pool, s3 string) string { +func setupMockCDN(t *testing.T, pool *dockertest.Pool, s3, bucket string) string { t.Helper() _, port, err := net.SplitHostPort(findListenAddr(t)) @@ -249,19 +242,7 @@ func findListenAddr(t *testing.T) string { return addr } -func TestIntegration(t *testing.T) { - pool := docker(t) - options := cmd.DefaultOptions - - options.CacheS3Enabled = true - options.CacheS3Bucket = bucket - options.InsecureCacheS3 = true - options.CacheS3Endpoint = setupS3(t, pool) - - options.CacheCDNEnabled = true - options.CacheCDNTrimPrefix = bucketPrefix - options.CacheCDNHost = setupMockCDN(t, pool, options.CacheS3Endpoint) - +func commonTest(t *testing.T, options cmd.Options) { ctx, listenAddr := setupFactory(t, options) baseURL := "http://" + listenAddr @@ -313,6 +294,7 @@ var ( installerExternalRepository string installerInternalRepository string cacheRepository string + signingCacheRepository string ) func init() { @@ -321,4 +303,5 @@ func init() { flag.StringVar(&installerExternalRepository, "test.installer-external-repository", cmd.DefaultOptions.InstallerExternalRepository, "image repository for the installer (external)") flag.StringVar(&installerInternalRepository, "test.installer-internal-repository", cmd.DefaultOptions.InstallerInternalRepository, "image repository for the installer (internal)") flag.StringVar(&cacheRepository, "test.cache-repository", cmd.DefaultOptions.CacheRepository, "image repository for cached boot assets") + flag.StringVar(&signingCacheRepository, "test.signing-cache-repository", cmd.DefaultOptions.CacheRepository+"sign", "image repository for signatures of cached boot assets (used for S3+CDN tests)") } diff --git a/internal/integration/main_cdn_test.go b/internal/integration/main_cdn_test.go new file mode 100644 index 0000000..18d58e6 --- /dev/null +++ b/internal/integration/main_cdn_test.go @@ -0,0 +1,40 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//go:build integration_cdn + +package integration_test + +import ( + "fmt" + "testing" + + "github.com/siderolabs/image-factory/cmd/image-factory/cmd" +) + +func TestIntegrationCDN(t *testing.T) { + pool := docker(t) + + // set up S3 access credentials for the tests, those are shared across all tests + t.Setenv("AWS_ACCESS_KEY_ID", s3Access) + t.Setenv("AWS_SECRET_ACCESS_KEY", s3Secret) + + t.Run("S3+CDN", func(t *testing.T) { + options := cmd.DefaultOptions + + options.CacheRepository = signingCacheRepository + options.MetricsNamespace = "test_s3_cdn" + + options.CacheS3Enabled = true + options.CacheS3Bucket = "test-s3-cdn" + options.InsecureCacheS3 = true + options.CacheS3Endpoint = setupS3(t, pool, options.CacheS3Bucket) + + options.CacheCDNEnabled = true + options.CacheCDNTrimPrefix = fmt.Sprintf("/%s", options.CacheS3Bucket) + options.CacheCDNHost = setupMockCDN(t, pool, options.CacheS3Endpoint, options.CacheS3Bucket) + + commonTest(t, options) + }) +} diff --git a/internal/integration/main_direct_test.go b/internal/integration/main_direct_test.go new file mode 100644 index 0000000..14be4f4 --- /dev/null +++ b/internal/integration/main_direct_test.go @@ -0,0 +1,22 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//go:build integration_direct + +package integration_test + +import ( + "testing" + + "github.com/siderolabs/image-factory/cmd/image-factory/cmd" +) + +func TestIntegrationDirect(t *testing.T) { + options := cmd.DefaultOptions + + options.CacheRepository = cacheRepository + options.MetricsNamespace = "" + + commonTest(t, options) +} diff --git a/internal/integration/main_s3_test.go b/internal/integration/main_s3_test.go new file mode 100644 index 0000000..e91224c --- /dev/null +++ b/internal/integration/main_s3_test.go @@ -0,0 +1,35 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//go:build integration_s3 + +package integration_test + +import ( + "testing" + + "github.com/siderolabs/image-factory/cmd/image-factory/cmd" +) + +func TestIntegrationS3(t *testing.T) { + pool := docker(t) + + // set up S3 access credentials for the tests, those are shared across all tests + t.Setenv("AWS_ACCESS_KEY_ID", s3Access) + t.Setenv("AWS_SECRET_ACCESS_KEY", s3Secret) + + t.Run("S3", func(t *testing.T) { + options := cmd.DefaultOptions + + options.CacheRepository = signingCacheRepository + options.MetricsNamespace = "test_s3" + + options.CacheS3Enabled = true + options.CacheS3Bucket = "test-s3" + options.InsecureCacheS3 = true + options.CacheS3Endpoint = setupS3(t, pool, options.CacheS3Bucket) + + commonTest(t, options) + }) +} diff --git a/internal/schematic/schematic.go b/internal/schematic/schematic.go index 4b13c55..a94dfe8 100644 --- a/internal/schematic/schematic.go +++ b/internal/schematic/schematic.go @@ -17,15 +17,18 @@ import ( // Factory is the schematic factory. type Factory struct { - options Options - logger *zap.Logger - storage storage.Storage - - metricGet, metricCreate, metricDuplicate prometheus.Counter + storage storage.Storage + metricGet prometheus.Counter + metricCreate prometheus.Counter + metricDuplicate prometheus.Counter + logger *zap.Logger + options Options } // Options for the schematic factory. -type Options struct{} +type Options struct { + MetricsNamespace string +} // NewFactory creates a new schematic factory. func NewFactory(logger *zap.Logger, storage storage.Storage, options Options) *Factory { @@ -35,16 +38,19 @@ func NewFactory(logger *zap.Logger, storage storage.Storage, options Options) *F logger: logger.With(zap.String("factory", "schematic")), metricGet: prometheus.NewCounter(prometheus.CounterOpts{ - Name: "image_factory_schematic_get_total", - Help: "Number of times schematics were retrieved.", + Name: "image_factory_schematic_get_total", + Help: "Number of times schematics were retrieved.", + Namespace: options.MetricsNamespace, }), metricCreate: prometheus.NewCounter(prometheus.CounterOpts{ - Name: "image_factory_schematic_create_total", - Help: "Number of new schematics created.", + Name: "image_factory_schematic_create_total", + Help: "Number of new schematics created.", + Namespace: options.MetricsNamespace, }), metricDuplicate: prometheus.NewCounter(prometheus.CounterOpts{ - Name: "image_factory_schematic_duplicate_create_total", - Help: "Number of new schematics which were created as duplicate.", + Name: "image_factory_schematic_duplicate_create_total", + Help: "Number of new schematics which were created as duplicate.", + Namespace: options.MetricsNamespace, }), } } diff --git a/internal/schematic/storage/cache/cache.go b/internal/schematic/storage/cache/cache.go index 7ef9f85..5c0b3e4 100644 --- a/internal/schematic/storage/cache/cache.go +++ b/internal/schematic/storage/cache/cache.go @@ -17,6 +17,11 @@ import ( "github.com/siderolabs/image-factory/internal/schematic/storage" ) +// Options configures the storage. +type Options struct { + MetricsNamespace string +} + // Storage is a schematic storage in-memory cache. type Storage struct { underlying storage.Storage @@ -29,13 +34,14 @@ type Storage struct { } // NewCache returns a new cache storage. -func NewCache(underlying storage.Storage) *Storage { +func NewCache(underlying storage.Storage, options Options) *Storage { return &Storage{ underlying: underlying, m: map[string]optional.Optional[[]byte]{}, metricCacheSize: prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "image_factory_schematic_cache_size", - Help: "Number of schematics in in-memory cache.", + Name: "image_factory_schematic_cache_size", + Help: "Number of schematics in in-memory cache.", + Namespace: options.MetricsNamespace, }), } } diff --git a/internal/schematic/storage/cache/cache_test.go b/internal/schematic/storage/cache/cache_test.go index c358aa5..fcc6e34 100644 --- a/internal/schematic/storage/cache/cache_test.go +++ b/internal/schematic/storage/cache/cache_test.go @@ -55,7 +55,7 @@ func TestStorage(t *testing.T) { defer cancel() underlying := &mockStorage{} - strg := cache.NewCache(underlying) + strg := cache.NewCache(underlying, cache.Options{}) v, err := strg.Get(ctx, "foo") require.NoError(t, err)