From ccd55cc8fb5fddaab91ffc817649ca05fa82702b Mon Sep 17 00:00:00 2001 From: Artem Chernyshev Date: Wed, 4 Jun 2025 21:16:10 +0300 Subject: [PATCH] feat: rewrite Omni config management Omni can now be configured via a config file instead of the command line flags. The flags `--config-path` will now read the config provided in the YAML format. The config structure was completely changed. It was not public before, so it's fine to ignore backward compatibility. The command line flags were not changed. Signed-off-by: Artem Chernyshev --- .github/workflows/ci.yaml | 32 +- .github/workflows/e2e-backups-cron.yaml | 4 +- .../workflows/e2e-forced-removal-cron.yaml | 4 +- .github/workflows/e2e-scaling-cron.yaml | 4 +- .github/workflows/e2e-short-cron.yaml | 4 +- .../workflows/e2e-short-secureboot-cron.yaml | 4 +- .github/workflows/e2e-templates-cron.yaml | 4 +- .github/workflows/e2e-upgrades-cron.yaml | 4 +- .../workflows/e2e-workload-proxy-cron.yaml | 4 +- .github/workflows/helm.yaml | 4 +- .kres.yaml | 1 + Dockerfile | 12 +- Makefile | 12 +- client/go.mod | 2 +- client/go.sum | 4 +- cmd/omni/cmd/cmd.go | 956 ++++++++++-------- cmd/omni/cmd/run.go | 157 ++- go.mod | 9 +- go.sum | 16 +- hack/test/integration.sh | 1 + internal/backend/factory/factory.go | 24 +- internal/backend/factory/factory_test.go | 5 +- internal/backend/grpc/auth.go | 2 +- internal/backend/grpc/configs_test.go | 4 +- internal/backend/grpc/grpc.go | 2 +- internal/backend/grpc/management.go | 4 +- internal/backend/grpc/serviceaccount.go | 6 +- internal/backend/installimage/installimage.go | 2 +- .../backend/runtime/kubernetes/kubernetes.go | 4 +- .../omni/cluster_machine_config.go | 20 +- .../omni/cluster_machine_config_test.go | 2 +- .../controllers/omni/installation_media.go | 2 +- .../internal/loadbalancer/loadbalancer.go | 10 +- .../controllers/omni/kubernetes_status.go | 2 +- .../runtime/omni/controllers/omni/versions.go | 6 +- internal/backend/runtime/omni/export_test.go | 4 +- internal/backend/runtime/omni/loader.go | 12 +- .../runtime/omni/migration/migration_test.go | 2 +- internal/backend/runtime/omni/omni.go | 26 +- internal/backend/runtime/omni/state.go | 18 +- internal/backend/runtime/omni/state_etcd.go | 20 +- .../backend/runtime/omni/state_etcd_test.go | 40 +- .../backend/runtime/omni/state_validation.go | 4 +- .../runtime/omni/state_validation_test.go | 18 +- .../backend/runtime/omni/virtual/state.go | 2 +- internal/backend/runtime/talos/talos.go | 4 +- internal/backend/saml/saml.go | 2 +- internal/backend/server.go | 383 ++----- internal/backend/services/http_server.go | 233 +++++ .../{proxy_server.go => services/proxy.go} | 52 +- internal/backend/workloadproxy/handler.go | 2 +- internal/pkg/auth/config.go | 8 +- internal/pkg/auth/config_test.go | 56 +- internal/pkg/config/auth.go | 75 +- internal/pkg/config/backup.go | 41 + internal/pkg/config/config.go | 537 ++++------ internal/pkg/config/config_test.go | 90 ++ internal/pkg/config/debug.go | 22 + internal/pkg/config/features.go | 15 + internal/pkg/config/logs.go | 67 ++ internal/pkg/config/services.go | 273 +++++ internal/pkg/config/storage.go | 57 ++ internal/pkg/config/testdata/backups.yaml | 69 ++ internal/pkg/config/testdata/config-full.yaml | 120 +++ .../pkg/config/testdata/conflicting-auth.yaml | 72 ++ .../testdata/invalid-join-token-mode.yaml | 71 ++ .../pkg/config/testdata/unknown-keys.yaml | 124 +++ internal/pkg/features/features.go | 12 +- internal/pkg/siderolink/loghandler.go | 2 +- internal/pkg/siderolink/loghandler_test.go | 44 +- internal/pkg/siderolink/machines.go | 26 +- internal/pkg/siderolink/manager.go | 36 +- internal/pkg/siderolink/siderolink_test.go | 8 +- 73 files changed, 2628 insertions(+), 1352 deletions(-) create mode 100644 internal/backend/services/http_server.go rename internal/backend/{proxy_server.go => services/proxy.go} (70%) create mode 100644 internal/pkg/config/backup.go create mode 100644 internal/pkg/config/config_test.go create mode 100644 internal/pkg/config/debug.go create mode 100644 internal/pkg/config/features.go create mode 100644 internal/pkg/config/logs.go create mode 100644 internal/pkg/config/services.go create mode 100644 internal/pkg/config/storage.go create mode 100644 internal/pkg/config/testdata/backups.yaml create mode 100644 internal/pkg/config/testdata/config-full.yaml create mode 100644 internal/pkg/config/testdata/conflicting-auth.yaml create mode 100644 internal/pkg/config/testdata/invalid-join-token-mode.yaml create mode 100644 internal/pkg/config/testdata/unknown-keys.yaml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 9c49f19f..3358df75 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,6 +1,6 @@ # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. # -# Generated on 2025-06-02T21:18:31Z by kres 99b55ad-dirty. +# Generated on 2025-06-06T17:20:38Z by kres fc6afbe-dirty. name: default concurrency: @@ -34,7 +34,7 @@ jobs: steps: - name: gather-system-info id: system-info - uses: kenchan0130/actions-system-info@v1.3.0 + uses: kenchan0130/actions-system-info@v1.3.1 continue-on-error: true - name: print-system-info run: | @@ -154,21 +154,21 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} registry: ghcr.io username: ${{ github.repository_owner }} - - name: image-integration-test + - name: image-omni-integration-test run: | - make image-integration-test - - name: push-integration-test + make image-omni-integration-test + - name: push-omni-integration-test if: github.event_name != 'pull_request' env: PUSH: "true" run: | - make image-integration-test - - name: push-integration-test-latest + make image-omni-integration-test + - name: push-omni-integration-test-latest if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main' env: PUSH: "true" run: | - make image-integration-test IMAGE_TAG=latest + make image-omni-integration-test IMAGE_TAG=latest - name: run-integration-test if: github.event_name == 'pull_request' env: @@ -246,7 +246,7 @@ jobs: steps: - name: gather-system-info id: system-info - uses: kenchan0130/actions-system-info@v1.3.0 + uses: kenchan0130/actions-system-info@v1.3.1 continue-on-error: true - name: print-system-info run: | @@ -325,7 +325,7 @@ jobs: steps: - name: gather-system-info id: system-info - uses: kenchan0130/actions-system-info@v1.3.0 + uses: kenchan0130/actions-system-info@v1.3.1 continue-on-error: true - name: print-system-info run: | @@ -404,7 +404,7 @@ jobs: steps: - name: gather-system-info id: system-info - uses: kenchan0130/actions-system-info@v1.3.0 + uses: kenchan0130/actions-system-info@v1.3.1 continue-on-error: true - name: print-system-info run: | @@ -483,7 +483,7 @@ jobs: steps: - name: gather-system-info id: system-info - uses: kenchan0130/actions-system-info@v1.3.0 + uses: kenchan0130/actions-system-info@v1.3.1 continue-on-error: true - name: print-system-info run: | @@ -562,7 +562,7 @@ jobs: steps: - name: gather-system-info id: system-info - uses: kenchan0130/actions-system-info@v1.3.0 + uses: kenchan0130/actions-system-info@v1.3.1 continue-on-error: true - name: print-system-info run: | @@ -642,7 +642,7 @@ jobs: steps: - name: gather-system-info id: system-info - uses: kenchan0130/actions-system-info@v1.3.0 + uses: kenchan0130/actions-system-info@v1.3.1 continue-on-error: true - name: print-system-info run: | @@ -721,7 +721,7 @@ jobs: steps: - name: gather-system-info id: system-info - uses: kenchan0130/actions-system-info@v1.3.0 + uses: kenchan0130/actions-system-info@v1.3.1 continue-on-error: true - name: print-system-info run: | @@ -800,7 +800,7 @@ jobs: steps: - name: gather-system-info id: system-info - uses: kenchan0130/actions-system-info@v1.3.0 + uses: kenchan0130/actions-system-info@v1.3.1 continue-on-error: true - name: print-system-info run: | diff --git a/.github/workflows/e2e-backups-cron.yaml b/.github/workflows/e2e-backups-cron.yaml index 404081c9..edd30919 100644 --- a/.github/workflows/e2e-backups-cron.yaml +++ b/.github/workflows/e2e-backups-cron.yaml @@ -1,6 +1,6 @@ # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. # -# Generated on 2025-06-02T21:18:31Z by kres 99b55ad-dirty. +# Generated on 2025-06-06T17:20:38Z by kres fc6afbe-dirty. name: e2e-backups-cron concurrency: @@ -17,7 +17,7 @@ jobs: steps: - name: gather-system-info id: system-info - uses: kenchan0130/actions-system-info@v1.3.0 + uses: kenchan0130/actions-system-info@v1.3.1 continue-on-error: true - name: print-system-info run: | diff --git a/.github/workflows/e2e-forced-removal-cron.yaml b/.github/workflows/e2e-forced-removal-cron.yaml index 8aa72ca7..6423e9ca 100644 --- a/.github/workflows/e2e-forced-removal-cron.yaml +++ b/.github/workflows/e2e-forced-removal-cron.yaml @@ -1,6 +1,6 @@ # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. # -# Generated on 2025-06-02T21:18:31Z by kres 99b55ad-dirty. +# Generated on 2025-06-06T17:20:38Z by kres fc6afbe-dirty. name: e2e-forced-removal-cron concurrency: @@ -17,7 +17,7 @@ jobs: steps: - name: gather-system-info id: system-info - uses: kenchan0130/actions-system-info@v1.3.0 + uses: kenchan0130/actions-system-info@v1.3.1 continue-on-error: true - name: print-system-info run: | diff --git a/.github/workflows/e2e-scaling-cron.yaml b/.github/workflows/e2e-scaling-cron.yaml index 3d1ff156..5ea2f82d 100644 --- a/.github/workflows/e2e-scaling-cron.yaml +++ b/.github/workflows/e2e-scaling-cron.yaml @@ -1,6 +1,6 @@ # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. # -# Generated on 2025-06-02T21:18:31Z by kres 99b55ad-dirty. +# Generated on 2025-06-06T17:20:38Z by kres fc6afbe-dirty. name: e2e-scaling-cron concurrency: @@ -17,7 +17,7 @@ jobs: steps: - name: gather-system-info id: system-info - uses: kenchan0130/actions-system-info@v1.3.0 + uses: kenchan0130/actions-system-info@v1.3.1 continue-on-error: true - name: print-system-info run: | diff --git a/.github/workflows/e2e-short-cron.yaml b/.github/workflows/e2e-short-cron.yaml index dd548d15..5fb14daf 100644 --- a/.github/workflows/e2e-short-cron.yaml +++ b/.github/workflows/e2e-short-cron.yaml @@ -1,6 +1,6 @@ # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. # -# Generated on 2025-06-02T21:18:31Z by kres 99b55ad-dirty. +# Generated on 2025-06-06T17:20:38Z by kres fc6afbe-dirty. name: e2e-short-cron concurrency: @@ -17,7 +17,7 @@ jobs: steps: - name: gather-system-info id: system-info - uses: kenchan0130/actions-system-info@v1.3.0 + uses: kenchan0130/actions-system-info@v1.3.1 continue-on-error: true - name: print-system-info run: | diff --git a/.github/workflows/e2e-short-secureboot-cron.yaml b/.github/workflows/e2e-short-secureboot-cron.yaml index 19ab09b2..a844816e 100644 --- a/.github/workflows/e2e-short-secureboot-cron.yaml +++ b/.github/workflows/e2e-short-secureboot-cron.yaml @@ -1,6 +1,6 @@ # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. # -# Generated on 2025-06-02T21:18:31Z by kres 99b55ad-dirty. +# Generated on 2025-06-06T17:20:38Z by kres fc6afbe-dirty. name: e2e-short-secureboot-cron concurrency: @@ -17,7 +17,7 @@ jobs: steps: - name: gather-system-info id: system-info - uses: kenchan0130/actions-system-info@v1.3.0 + uses: kenchan0130/actions-system-info@v1.3.1 continue-on-error: true - name: print-system-info run: | diff --git a/.github/workflows/e2e-templates-cron.yaml b/.github/workflows/e2e-templates-cron.yaml index e4d960a4..bc233fd8 100644 --- a/.github/workflows/e2e-templates-cron.yaml +++ b/.github/workflows/e2e-templates-cron.yaml @@ -1,6 +1,6 @@ # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. # -# Generated on 2025-06-02T21:18:31Z by kres 99b55ad-dirty. +# Generated on 2025-06-06T17:20:38Z by kres fc6afbe-dirty. name: e2e-templates-cron concurrency: @@ -17,7 +17,7 @@ jobs: steps: - name: gather-system-info id: system-info - uses: kenchan0130/actions-system-info@v1.3.0 + uses: kenchan0130/actions-system-info@v1.3.1 continue-on-error: true - name: print-system-info run: | diff --git a/.github/workflows/e2e-upgrades-cron.yaml b/.github/workflows/e2e-upgrades-cron.yaml index 97484433..20daebf4 100644 --- a/.github/workflows/e2e-upgrades-cron.yaml +++ b/.github/workflows/e2e-upgrades-cron.yaml @@ -1,6 +1,6 @@ # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. # -# Generated on 2025-06-02T21:18:31Z by kres 99b55ad-dirty. +# Generated on 2025-06-06T17:20:38Z by kres fc6afbe-dirty. name: e2e-upgrades-cron concurrency: @@ -17,7 +17,7 @@ jobs: steps: - name: gather-system-info id: system-info - uses: kenchan0130/actions-system-info@v1.3.0 + uses: kenchan0130/actions-system-info@v1.3.1 continue-on-error: true - name: print-system-info run: | diff --git a/.github/workflows/e2e-workload-proxy-cron.yaml b/.github/workflows/e2e-workload-proxy-cron.yaml index 78c3dca9..06d51fde 100644 --- a/.github/workflows/e2e-workload-proxy-cron.yaml +++ b/.github/workflows/e2e-workload-proxy-cron.yaml @@ -1,6 +1,6 @@ # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. # -# Generated on 2025-06-02T21:18:31Z by kres 99b55ad-dirty. +# Generated on 2025-06-06T17:20:38Z by kres fc6afbe-dirty. name: e2e-workload-proxy-cron concurrency: @@ -17,7 +17,7 @@ jobs: steps: - name: gather-system-info id: system-info - uses: kenchan0130/actions-system-info@v1.3.0 + uses: kenchan0130/actions-system-info@v1.3.1 continue-on-error: true - name: print-system-info run: | diff --git a/.github/workflows/helm.yaml b/.github/workflows/helm.yaml index f9042c89..9bfbed96 100644 --- a/.github/workflows/helm.yaml +++ b/.github/workflows/helm.yaml @@ -1,6 +1,6 @@ # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. # -# Generated on 2025-06-02T21:18:31Z by kres 99b55ad-dirty. +# Generated on 2025-06-06T17:20:38Z by kres fc6afbe-dirty. name: helm concurrency: @@ -31,7 +31,7 @@ jobs: steps: - name: gather-system-info id: system-info - uses: kenchan0130/actions-system-info@v1.3.0 + uses: kenchan0130/actions-system-info@v1.3.1 continue-on-error: true - name: print-system-info run: | diff --git a/.kres.yaml b/.kres.yaml index ec7d1391..0a13df44 100644 --- a/.kres.yaml +++ b/.kres.yaml @@ -25,6 +25,7 @@ spec: - path: internal/integration name: integration-test enableDockerImage: true + imageName: omni-integration-test outputs: linux-amd64: GOOS: linux diff --git a/Dockerfile b/Dockerfile index 53bf28f1..5a01b746 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,8 @@ -# syntax = docker/dockerfile-upstream:1.15.1-labs +# syntax = docker/dockerfile-upstream:1.16.0-labs # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. # -# Generated on 2025-06-02T21:18:31Z by kres 99b55ad-dirty. +# Generated on 2025-06-06T17:20:38Z by kres fc6afbe-dirty. ARG JS_TOOLCHAIN ARG TOOLCHAIN @@ -20,7 +20,7 @@ ENV GOPATH=/go ENV PATH=${PATH}:/usr/local/go/bin # runs markdownlint -FROM docker.io/oven/bun:1.2.13-alpine AS lint-markdown +FROM docker.io/oven/bun:1.2.15-alpine AS lint-markdown WORKDIR /src RUN bun i markdownlint-cli@0.45.0 sentences-per-line@0.3.0 COPY .markdownlint.json . @@ -586,13 +586,13 @@ COPY --from=omnictl-linux-amd64 / / COPY --from=omnictl-linux-arm64 / / COPY --from=omnictl-windows-amd64.exe / / -FROM scratch AS image-integration-test +FROM scratch AS image-omni-integration-test ARG TARGETARCH -COPY --from=integration-test integration-test-linux-${TARGETARCH} /integration-test +COPY --from=integration-test integration-test-linux-${TARGETARCH} /omni-integration-test COPY --from=image-fhs / / COPY --from=image-ca-certificates / / LABEL org.opencontainers.image.source=https://github.com/siderolabs/omni -ENTRYPOINT ["/integration-test"] +ENTRYPOINT ["/omni-integration-test"] FROM scratch AS image-omni ARG TARGETARCH diff --git a/Makefile b/Makefile index 86b34bd8..d50e546a 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. # -# Generated on 2025-06-02T21:18:31Z by kres 99b55ad-dirty. +# Generated on 2025-06-06T17:20:38Z by kres fc6afbe-dirty. # common variables @@ -79,7 +79,7 @@ COMMON_ARGS += --build-arg=DEEPCOPY_VERSION="$(DEEPCOPY_VERSION)" COMMON_ARGS += --build-arg=GOLANGCILINT_VERSION="$(GOLANGCILINT_VERSION)" COMMON_ARGS += --build-arg=GOFUMPT_VERSION="$(GOFUMPT_VERSION)" COMMON_ARGS += --build-arg=TESTPKGS="$(TESTPKGS)" -JS_TOOLCHAIN ?= docker.io/oven/bun:1.2.13-alpine +JS_TOOLCHAIN ?= docker.io/oven/bun:1.2.15-alpine TOOLCHAIN ?= docker.io/golang:1.24-alpine # extra variables @@ -148,7 +148,7 @@ else GO_LDFLAGS += -s endif -all: unit-tests-frontend lint-eslint frontend unit-tests-client unit-tests acompat make-cookies omni image-omni omnictl helm integration-test image-integration-test lint +all: unit-tests-frontend lint-eslint frontend unit-tests-client unit-tests acompat make-cookies omni image-omni omnictl helm integration-test image-omni-integration-test lint $(ARTIFACTS): ## Creates artifacts directory. @mkdir -p $(ARTIFACTS) @@ -386,9 +386,9 @@ integration-test-linux-arm64: $(ARTIFACTS)/integration-test-linux-arm64 ## Buil .PHONY: integration-test integration-test: integration-test-darwin-amd64 integration-test-darwin-arm64 integration-test-linux-amd64 integration-test-linux-arm64 ## Builds executables for integration-test. -.PHONY: image-integration-test -image-integration-test: ## Builds image for integration-test. - @$(MAKE) registry-$@ IMAGE_NAME="integration-test" +.PHONY: image-omni-integration-test +image-omni-integration-test: ## Builds image for omni-integration-test. + @$(MAKE) registry-$@ IMAGE_NAME="omni-integration-test" .PHONY: dev-server dev-server: diff --git a/client/go.mod b/client/go.mod index 2dda505e..b3fbe441 100644 --- a/client/go.mod +++ b/client/go.mod @@ -28,7 +28,7 @@ require ( github.com/mattn/go-isatty v0.0.20 github.com/planetscale/vtprotobuf v0.6.1-0.20241121165744-79df5c4772f2 github.com/sergi/go-diff v1.3.1 - github.com/siderolabs/gen v0.8.1 + github.com/siderolabs/gen v0.8.2 github.com/siderolabs/go-api-signature v0.3.6 github.com/siderolabs/go-kubeconfig v0.1.1 github.com/siderolabs/go-pointer v1.0.1 diff --git a/client/go.sum b/client/go.sum index 969e73bb..bd3635af 100644 --- a/client/go.sum +++ b/client/go.sum @@ -164,8 +164,8 @@ github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= github.com/siderolabs/crypto v0.5.1 h1:aZEUTZBoP8rH+0TqQAlUgazriPh89MrXf4R+th+m6ps= github.com/siderolabs/crypto v0.5.1/go.mod h1:7RHC7eUKBx6RLS2lDaNXrQ83zY9iPH/aQSTxk1I4/j4= -github.com/siderolabs/gen v0.8.1 h1:i26KvarXfkXYqsmIicjGr2DN68uuuKDse8J4Z2J0O/Y= -github.com/siderolabs/gen v0.8.1/go.mod h1:CIhFWgYkOKtxKD8Zct5NcJMgRzefnF2XIqeGOA+um0U= +github.com/siderolabs/gen v0.8.2 h1:ZUSyehTbqm6h6K7bMyGN2X99Z7q8hg9KU0XLqRaebp0= +github.com/siderolabs/gen v0.8.2/go.mod h1:CRrktDXQf3yDJI7xKv+cDYhBbKdfd/YE16OpgcHoT9E= github.com/siderolabs/go-api-signature v0.3.6 h1:wDIsXbpl7Oa/FXvxB6uz4VL9INA9fmr3EbmjEZYFJrU= github.com/siderolabs/go-api-signature v0.3.6/go.mod h1:hoH13AfunHflxbXfh+NoploqV13ZTDfQ1mQJWNVSW9U= github.com/siderolabs/go-kubeconfig v0.1.1 h1:tZlgpelj/OqrcHVUwISPN0NRgObcflpH9WtE41mtQZ0= diff --git a/cmd/omni/cmd/cmd.go b/cmd/omni/cmd/cmd.go index c6f2464b..bb4af2ca 100644 --- a/cmd/omni/cmd/cmd.go +++ b/cmd/omni/cmd/cmd.go @@ -14,9 +14,6 @@ import ( "sync" "syscall" - "github.com/cosi-project/runtime/pkg/resource" - "github.com/cosi-project/runtime/pkg/state" - "github.com/prometheus/client_golang/prometheus" "github.com/siderolabs/gen/ensure" "github.com/siderolabs/go-debug" "github.com/spf13/cobra" @@ -24,36 +21,17 @@ import ( "go.uber.org/zap/zapcore" "github.com/siderolabs/omni/client/pkg/constants" - authres "github.com/siderolabs/omni/client/pkg/omni/resources/auth" - omnires "github.com/siderolabs/omni/client/pkg/omni/resources/omni" "github.com/siderolabs/omni/client/pkg/panichandler" - "github.com/siderolabs/omni/internal/backend" - "github.com/siderolabs/omni/internal/backend/discovery" - "github.com/siderolabs/omni/internal/backend/dns" - "github.com/siderolabs/omni/internal/backend/imagefactory" - "github.com/siderolabs/omni/internal/backend/logging" - "github.com/siderolabs/omni/internal/backend/resourcelogger" - "github.com/siderolabs/omni/internal/backend/runtime/omni" - "github.com/siderolabs/omni/internal/backend/runtime/omni/virtual" - "github.com/siderolabs/omni/internal/backend/runtime/talos" - "github.com/siderolabs/omni/internal/backend/workloadproxy" - "github.com/siderolabs/omni/internal/pkg/auth" - "github.com/siderolabs/omni/internal/pkg/auth/user" "github.com/siderolabs/omni/internal/pkg/config" - "github.com/siderolabs/omni/internal/pkg/ctxstore" - "github.com/siderolabs/omni/internal/pkg/features" - "github.com/siderolabs/omni/internal/pkg/siderolink" "github.com/siderolabs/omni/internal/version" ) -func runDebugServer(ctx context.Context, logger *zap.Logger) { - const debugAddr = ":9980" - +func runDebugServer(ctx context.Context, logger *zap.Logger, bindEndpoint string) { debugLogFunc := func(msg string) { logger.Info(msg) } - if err := debug.ListenAndServe(ctx, debugAddr, debugLogFunc); err != nil { + if err := debug.ListenAndServe(ctx, bindEndpoint, debugLogFunc); err != nil { logger.Panic("failed to start debug server", zap.Error(err)) } } @@ -104,138 +82,26 @@ var rootCmd = &cobra.Command{ cancel() }, logger) - return RunService(ctx, logger, config.Config) + var configs []*config.Params + + if rootCmdArgs.configPath != "" { + var cfg *config.Params + + cfg, err = config.LoadFromFile(rootCmdArgs.configPath) + if err != nil { + return err + } + + configs = append(configs, cfg) + } + + return RunService(ctx, logger, append(configs, cmdConfig)...) }, } -func runWithState(logger *zap.Logger) func(context.Context, state.State, *virtual.State) error { - return func(ctx context.Context, resourceState state.State, virtualState *virtual.State) error { - auditWrap, auditErr := omni.NewAuditWrap(resourceState, config.Config, logger) - if auditErr != nil { - return auditErr - } - - resourceState = auditWrap.WrapState(resourceState) - - talosClientFactory := talos.NewClientFactory(resourceState, logger) - prometheus.MustRegister(talosClientFactory) - - dnsService := dns.NewService(resourceState, logger) - workloadProxyReconciler := workloadproxy.NewReconciler(logger.With(logging.Component("workload_proxy_reconciler")), zapcore.DebugLevel) - - var resourceLogger *resourcelogger.Logger - - if len(config.Config.LogResourceUpdatesTypes) > 0 { - var err error - - resourceLogger, err = resourcelogger.New(ctx, resourceState, logger.With(logging.Component("resourcelogger")), - config.Config.LogResourceUpdatesLogLevel, config.Config.LogResourceUpdatesTypes...) - if err != nil { - return fmt.Errorf("failed to set up resource logger: %w", err) - } - } - - imageFactoryClient, err := imagefactory.NewClient(resourceState, config.Config.ImageFactoryBaseURL) - if err != nil { - return fmt.Errorf("failed to set up image factory client: %w", err) - } - - linkCounterDeltaCh := make(chan siderolink.LinkCounterDeltas) - siderolinkEventsCh := make(chan *omnires.MachineStatusSnapshot) - installEventCh := make(chan resource.ID) - - discoveryClientCache := discovery.NewClientCache(logger.With(logging.Component("discovery_client_factory"))) - defer discoveryClientCache.Close() - - prometheus.MustRegister(discoveryClientCache) - - omniRuntime, err := omni.New(talosClientFactory, dnsService, workloadProxyReconciler, resourceLogger, - imageFactoryClient, linkCounterDeltaCh, siderolinkEventsCh, installEventCh, resourceState, virtualState, - prometheus.DefaultRegisterer, discoveryClientCache, logger.With(logging.Component("omni_runtime"))) - if err != nil { - return fmt.Errorf("failed to set up the controller runtime: %w", err) - } - - machineMap := siderolink.NewMachineMap(siderolink.NewStateStorage(omniRuntime.State())) - - logHandler, err := siderolink.NewLogHandler( - machineMap, - resourceState, - &config.Config.MachineLogConfig, - logger.With(logging.Component("siderolink_log_handler")), - ) - if err != nil { - return fmt.Errorf("failed to set up log handler: %w", err) - } - - talosRuntime := talos.New(talosClientFactory, logger) - - err = user.EnsureInitialResources(ctx, omniRuntime.State(), logger, config.Config.InitialUsers) - if err != nil { - return fmt.Errorf("failed to write initial user resources to state: %w", err) - } - - authConfig, err := auth.EnsureAuthConfigResource(ctx, omniRuntime.State(), logger, config.Config.Auth) - if err != nil { - return fmt.Errorf("failed to write Auth0 parameters to state: %w", err) - } - - if err = features.UpdateResources(ctx, omniRuntime.State(), logger); err != nil { - return fmt.Errorf("failed to update features config resources: %w", err) - } - - ctx = ctxstore.WithValue(ctx, auth.EnabledAuthContextKey{Enabled: authres.Enabled(authConfig)}) - - handler, err := backend.NewFrontendHandler(rootCmdArgs.frontendDst, logger) - if err != nil { - return fmt.Errorf("failed to set up frontend handler: %w", err) - } - - server, err := backend.NewServer( - rootCmdArgs.bindAddress, - rootCmdArgs.metricsBindAddress, - rootCmdArgs.k8sProxyBindAddress, - rootCmdArgs.pprofBindAddress, - dnsService, - workloadProxyReconciler, - imageFactoryClient, - linkCounterDeltaCh, - siderolinkEventsCh, - installEventCh, - omniRuntime, - talosRuntime, - logHandler, - authConfig, - rootCmdArgs.keyFile, - rootCmdArgs.certFile, - backend.NewProxyServer(rootCmdArgs.frontendBind, handler, rootCmdArgs.keyFile, rootCmdArgs.certFile), - auditWrap, - logger, - ) - if err != nil { - return fmt.Errorf("failed to create server: %w", err) - } - - if err := server.Run(ctx); err != nil { - return fmt.Errorf("failed to run server: %w", err) - } - - return nil - } -} - var rootCmdArgs struct { - bindAddress string - frontendBind string - frontendDst string - k8sProxyBindAddress string - metricsBindAddress string - pprofBindAddress string - keyFile string - certFile string - registryMirrors []string - - debug bool + configPath string + debug bool } // RootCmd returns the root command. @@ -243,325 +109,603 @@ func RootCmd() *cobra.Command { return initOnce() } var initOnce = sync.OnceValue(func() *cobra.Command { rootCmd.Flags().BoolVar(&rootCmdArgs.debug, "debug", false, "enable debug logs.") - rootCmd.Flags().StringVar(&rootCmdArgs.bindAddress, "bind-addr", "0.0.0.0:8080", "start HTTP server on the defined address.") - rootCmd.Flags().StringVar(&rootCmdArgs.frontendDst, "frontend-dst", "", "destination address non API requests from proxy server.") - rootCmd.Flags().StringVar(&rootCmdArgs.frontendBind, "frontend-bind", "", "proxy server which will redirect all non API requests to the definied frontend server.") - rootCmd.Flags().StringVar(&rootCmdArgs.metricsBindAddress, "metrics-bind-addr", "0.0.0.0:2122", "start Prometheus HTTP server on the defined address.") - rootCmd.Flags().StringVar(&rootCmdArgs.pprofBindAddress, "pprof-bind-addr", "", "start pprof HTTP server on the defined address (\"\" if disabled).") - rootCmd.Flags().StringVar(&rootCmdArgs.k8sProxyBindAddress, "k8s-proxy-bind-addr", "0.0.0.0:8095", "start Kubernetes workload proxy on the defined address.") - rootCmd.Flags().StringSliceVar(&rootCmdArgs.registryMirrors, "registry-mirror", []string{}, "list of registry mirrors to use in format: =") - rootCmd.Flags().StringVar(&config.Config.AccountID, "account-id", config.Config.AccountID, "instance account ID, should never be changed.") - rootCmd.Flags().StringVar(&config.Config.Name, "name", config.Config.Name, "instance user-facing name.") - rootCmd.Flags().StringVar(&config.Config.APIURL, "advertised-api-url", config.Config.APIURL, "advertised API frontend URL.") - rootCmd.Flags().StringVar(&config.Config.KubernetesProxyURL, "advertised-kubernetes-proxy-url", config.Config.KubernetesProxyURL, "advertised Kubernetes proxy URL.") - rootCmd.Flags().BoolVar(&config.Config.SiderolinkDisableLastEndpoint, "siderolink-disable-last-endpoint", false, "do not populate last known peer endpoint for the WireGuard peers") + rootCmd.Flags().StringVar(&config.Config.Account.ID, "account-id", config.Config.Account.ID, "instance account ID, should never be changed.") + rootCmd.Flags().StringVar(&config.Config.Account.Name, "name", config.Config.Account.Name, "instance user-facing name.") + + defineServiceFlags() + defineAuthFlags() + defineLogsFlags() + defineStorageFlags() + defineRegistriesFlags() + defineFeatureFlags() + defineDebugFlags() + defineEtcdBackupsFlags() + + rootCmd.Flags().StringVar(&rootCmdArgs.configPath, "config-path", "", "load the config from the file, flags have bigger priority") + + return rootCmd +}) + +var cmdConfig = config.InitDefault() + +func defineServiceFlags() { + // API rootCmd.Flags().StringVar( - &config.Config.SiderolinkWireguardAdvertisedAddress, - "siderolink-wireguard-advertised-addr", - config.Config.SiderolinkWireguardAdvertisedAddress, - "advertised wireguard address which is passed down to the nodes.") - rootCmd.Flags().StringVar(&config.Config.SiderolinkWireguardBindAddress, "siderolink-wireguard-bind-addr", config.Config.SiderolinkWireguardBindAddress, "SideroLink WireGuard bind address.") - rootCmd.Flags().BoolVar(&config.Config.SiderolinkUseGRPCTunnel, "siderolink-use-grpc-tunnel", false, "use gRPC tunnel to wrap WireGuard traffic instead of UDP. When enabled, "+ - "the SideroLink connections from Talos machines will be configured to use the tunnel mode, regardless of their individual configuration. ") + &cmdConfig.Services.API.BindEndpoint, + "bind-addr", + cmdConfig.Services.API.BindEndpoint, + "start HTTP + API server on the defined address.", + ) + rootCmd.Flags().StringVar( + &cmdConfig.Services.API.AdvertisedURL, + "advertised-api-url", + cmdConfig.Services.API.AdvertisedURL, + "advertised API frontend URL.", + ) + rootCmd.Flags().StringVar( + &cmdConfig.Services.API.KeyFile, + "key", + "", + "TLS key file", + ) + rootCmd.Flags().StringVar( + &cmdConfig.Services.API.CertFile, + "cert", + "", + "TLS cert file", + ) - rootCmd.Flags().StringVar(&config.Config.MachineAPIBindAddress, "siderolink-api-bind-addr", config.Config.MachineAPIBindAddress, "SideroLink provision bind address.") - rootCmd.Flags().StringVar(&config.Config.MachineAPICertFile, "siderolink-api-cert", config.Config.MachineAPICertFile, "SideroLink TLS cert file path.") - rootCmd.Flags().StringVar(&config.Config.MachineAPIKeyFile, "siderolink-api-key", config.Config.MachineAPIKeyFile, "SideroLink TLS key file path.") + // Metrics + rootCmd.Flags().StringVar( + &cmdConfig.Services.Metrics.BindEndpoint, + "metrics-bind-addr", + cmdConfig.Services.Metrics.BindEndpoint, + "start Prometheus HTTP server on the defined address.", + ) + + // KubernetesProxy + rootCmd.Flags().StringVar( + &cmdConfig.Services.KubernetesProxy.BindEndpoint, + "k8s-proxy-bind-addr", cmdConfig.Services.KubernetesProxy.BindEndpoint, + "start Kubernetes proxy on the defined address.", + ) + + rootCmd.Flags().StringVar( + &cmdConfig.Services.KubernetesProxy.AdvertisedURL, + "advertised-kubernetes-proxy-url", + cmdConfig.Services.KubernetesProxy.AdvertisedURL, + "advertised Kubernetes proxy URL.", + ) + + // Siderolink + rootCmd.Flags().StringVar( + &cmdConfig.Services.Siderolink.WireGuard.BindEndpoint, + "siderolink-wireguard-bind-addr", + cmdConfig.Services.Siderolink.WireGuard.BindEndpoint, + "SideroLink WireGuard bind address.", + ) + rootCmd.Flags().StringVar( + &cmdConfig.Services.Siderolink.WireGuard.AdvertisedEndpoint, + "siderolink-wireguard-advertised-addr", + cmdConfig.Services.Siderolink.WireGuard.AdvertisedEndpoint, + "advertised wireguard address which is passed down to the nodes.", + ) + rootCmd.Flags().BoolVar( + &cmdConfig.Services.Siderolink.DisableLastEndpoint, + "siderolink-disable-last-endpoint", + false, + "do not populate last known peer endpoint for the WireGuard peers", + ) + rootCmd.Flags().BoolVar( + &cmdConfig.Services.Siderolink.UseGRPCTunnel, + "siderolink-use-grpc-tunnel", + false, + "use gRPC tunnel to wrap WireGuard traffic instead of UDP. When enabled, "+ + "the SideroLink connections from Talos machines will be configured to use the tunnel mode, regardless of their individual configuration. ", + ) + rootCmd.Flags().IntVar( + &cmdConfig.Services.Siderolink.EventSinkPort, + "event-sink-port", + cmdConfig.Services.Siderolink.EventSinkPort, + "event sink bind port.", + ) + rootCmd.Flags().IntVar( + &cmdConfig.Services.Siderolink.LogServerPort, + "log-server-port", + cmdConfig.Services.Siderolink.LogServerPort, + "port for TCP log server", + ) + rootCmd.Flags().Var( + &cmdConfig.Services.Siderolink.JoinTokensMode, + "join-tokens-mode", + "configures Talos machine join flow to use secure node tokens", + ) + + // MachineAPI + for _, prefix := range []string{"siderolink", "machine"} { + rootCmd.Flags().StringVar( + &cmdConfig.Services.MachineAPI.BindEndpoint, + fmt.Sprintf("%s-api-bind-addr", prefix), + cmdConfig.Services.MachineAPI.BindEndpoint, + "machine API provision bind address.", + ) + rootCmd.Flags().StringVar( + &cmdConfig.Services.MachineAPI.CertFile, + fmt.Sprintf("%s-api-cert", prefix), + cmdConfig.Services.MachineAPI.CertFile, + "machine API TLS cert file path.", + ) + rootCmd.Flags().StringVar( + &cmdConfig.Services.MachineAPI.KeyFile, + fmt.Sprintf("%s-api-key", prefix), + cmdConfig.Services.MachineAPI.KeyFile, + "machine API TLS cert file path.", + ) + rootCmd.Flags().StringVar( + &cmdConfig.Services.MachineAPI.AdvertisedURL, + fmt.Sprintf("%s-api-advertised-url", prefix), + cmdConfig.Services.MachineAPI.AdvertisedURL, + "machine API advertised API URL.", + ) + } rootCmd.Flags().MarkDeprecated("siderolink-api-bind-addr", "--deprecated, use --machine-api-bind-addr") //nolint:errcheck rootCmd.Flags().MarkDeprecated("siderolink-api-cert", "deprecated, use --machine-api-cert") //nolint:errcheck rootCmd.Flags().MarkDeprecated("siderolink-api-key", "deprecated, use --machine-api-key") //nolint:errcheck - rootCmd.Flags().StringVar(&config.Config.MachineAPIBindAddress, "machine-api-bind-addr", config.Config.MachineAPIBindAddress, "machine API bind address.") - rootCmd.Flags().StringVar(&config.Config.MachineAPICertFile, "machine-api-cert", config.Config.MachineAPICertFile, "machine API TLS cert file path.") - rootCmd.Flags().StringVar(&config.Config.MachineAPIKeyFile, "machine-api-key", config.Config.MachineAPIKeyFile, "machine API TLS key file path.") + // LoadBalancer + rootCmd.Flags().IntVar( + &cmdConfig.Services.LoadBalancer.MinPort, + "lb-min-port", + cmdConfig.Services.LoadBalancer.MinPort, + "cluster load balancer port range min value.", + ) + rootCmd.Flags().IntVar( + &cmdConfig.Services.LoadBalancer.MaxPort, + "lb-max-port", + cmdConfig.Services.LoadBalancer.MaxPort, + "cluster load balancer port range max value.", + ) - rootCmd.Flags().IntVar(&config.Config.EventSinkPort, "event-sink-port", config.Config.EventSinkPort, "event sink bind port.") - rootCmd.Flags().StringVar(&config.Config.SideroLinkAPIURL, "siderolink-api-advertised-url", config.Config.SideroLinkAPIURL, "SideroLink advertised API URL.") - rootCmd.Flags().IntVar(&config.Config.LoadBalancer.MinPort, "lb-min-port", config.Config.LoadBalancer.MinPort, "cluster load balancer port range min value.") - rootCmd.Flags().IntVar(&config.Config.LoadBalancer.MaxPort, "lb-max-port", config.Config.LoadBalancer.MaxPort, "cluster load balancer port range max value.") - rootCmd.Flags().IntVar(&config.Config.LogServerPort, "log-server-port", config.Config.LogServerPort, "port for TCP log server") + rootCmd.Flags().IntVar( + &cmdConfig.Services.LocalResourceService.Port, + "local-resource-server-port", + cmdConfig.Services.LocalResourceService.Port, + "port for local read-only public resource server.", + ) - rootCmd.Flags().IntVar(&config.Config.MachineLogConfig.BufferInitialCapacity, "machine-log-buffer-capacity", - config.Config.MachineLogConfig.BufferInitialCapacity, "initial buffer capacity for machine logs in bytes") - rootCmd.Flags().IntVar(&config.Config.MachineLogConfig.BufferMaxCapacity, "machine-log-buffer-max-capacity", - config.Config.MachineLogConfig.BufferMaxCapacity, "max buffer capacity for machine logs in bytes") - rootCmd.Flags().IntVar(&config.Config.MachineLogConfig.BufferSafetyGap, "machine-log-buffer-safe-gap", - config.Config.MachineLogConfig.BufferSafetyGap, "safety gap for machine log buffer in bytes") - rootCmd.Flags().IntVar(&config.Config.MachineLogConfig.NumCompressedChunks, "machine-log-num-compressed-chunks", - config.Config.MachineLogConfig.NumCompressedChunks, "number of compressed log chunks to keep") - rootCmd.Flags().BoolVar(&config.Config.MachineLogConfig.StorageEnabled, "machine-log-storage-enabled", - config.Config.MachineLogConfig.StorageEnabled, "enable machine log storage") - rootCmd.Flags().StringVar(&config.Config.MachineLogConfig.StoragePath, "machine-log-storage-path", - config.Config.MachineLogConfig.StoragePath, "path of the directory for storing machine logs") - rootCmd.Flags().DurationVar(&config.Config.MachineLogConfig.StorageFlushPeriod, "machine-log-storage-flush-period", - config.Config.MachineLogConfig.StorageFlushPeriod, "period for flushing machine logs to disk") - rootCmd.Flags().Float64Var(&config.Config.MachineLogConfig.StorageFlushJitter, "machine-log-storage-flush-jitter", - config.Config.MachineLogConfig.StorageFlushJitter, "jitter for the machine log storage flush period") + rootCmd.Flags().BoolVar( + &cmdConfig.Services.LocalResourceService.Enabled, + "local-resource-server-enabled", + cmdConfig.Services.LocalResourceService.Enabled, + "enable local read-only public resource server.", + ) + + // Embedded discovery service + rootCmd.Flags().BoolVar( + &cmdConfig.Services.EmbeddedDiscoveryService.Enabled, + "embedded-discovery-service-enabled", + cmdConfig.Services.EmbeddedDiscoveryService.Enabled, + "enable embedded discovery service, binds only to the SideroLink WireGuard address", + ) + + rootCmd.Flags().IntVar( + &cmdConfig.Services.EmbeddedDiscoveryService.Port, + "embedded-discovery-service-endpoint", + cmdConfig.Services.EmbeddedDiscoveryService.Port, + "embedded discovery service port to listen on", + ) + rootCmd.Flags().MarkDeprecated("embedded-discovery-service-endpoint", "use --embedded-discovery-service-port") //nolint:errcheck + + rootCmd.Flags().IntVar( + &cmdConfig.Services.EmbeddedDiscoveryService.Port, + "embedded-discovery-service-port", + cmdConfig.Services.EmbeddedDiscoveryService.Port, + "embedded discovery service port to listen on", + ) + rootCmd.Flags().BoolVar( + &cmdConfig.Services.EmbeddedDiscoveryService.SnapshotsEnabled, + "embedded-discovery-service-snapshots-enabled", + cmdConfig.Services.EmbeddedDiscoveryService.SnapshotsEnabled, + "enable snapshots for the embedded discovery service", + ) + rootCmd.Flags().StringVar( + &cmdConfig.Services.EmbeddedDiscoveryService.SnapshotsPath, + "embedded-discovery-service-snapshot-path", + cmdConfig.Services.EmbeddedDiscoveryService.SnapshotsPath, + "path to the file for storing the embedded discovery service state", + ) + rootCmd.Flags().DurationVar( + &cmdConfig.Services.EmbeddedDiscoveryService.SnapshotsInterval, + "embedded-discovery-service-snapshot-interval", + cmdConfig.Services.EmbeddedDiscoveryService.SnapshotsInterval, + "interval for saving the embedded discovery service state", + ) + rootCmd.Flags().StringVar( + &cmdConfig.Services.EmbeddedDiscoveryService.LogLevel, + "embedded-discovery-service-log-level", + cmdConfig.Services.EmbeddedDiscoveryService.LogLevel, + "log level for the embedded discovery service - it has no effect if it is lower (more verbose) than the main log level", + ) + + // DevServerProxy + rootCmd.Flags().StringVar( + &cmdConfig.Services.DevServerProxy.ProxyTo, "frontend-dst", "", + "destination address non API requests from proxy server.") + rootCmd.Flags().StringVar( + &cmdConfig.Services.DevServerProxy.BindEndpoint, "frontend-bind", "", + "proxy server which will redirect all non API requests to the definied frontend server.") +} + +func defineAuthFlags() { + // Auth0 + rootCmd.Flags().BoolVar(&cmdConfig.Auth.Auth0.Enabled, "auth-auth0-enabled", cmdConfig.Auth.Auth0.Enabled, + "enable Auth0 authentication. Once set to true, it cannot be set back to false.") + rootCmd.Flags().StringVar(&cmdConfig.Auth.Auth0.ClientID, "auth-auth0-client-id", cmdConfig.Auth.Auth0.ClientID, "Auth0 application client ID.") + rootCmd.Flags().StringVar(&cmdConfig.Auth.Auth0.Domain, "auth-auth0-domain", cmdConfig.Auth.Auth0.Domain, "Auth0 application domain.") + rootCmd.Flags().BoolVar(&cmdConfig.Auth.Auth0.UseFormData, "auth-auth0-use-form-data", cmdConfig.Auth.Auth0.UseFormData, + "When true, data to the token endpoint is transmitted as x-www-form-urlencoded data instead of JSON. The default is false") + + // Webauthn + rootCmd.Flags().BoolVar(&cmdConfig.Auth.WebAuthn.Enabled, "auth-webauthn-enabled", cmdConfig.Auth.WebAuthn.Enabled, + "enable WebAuthn authentication. Once set to true, it cannot be set back to false.") + rootCmd.Flags().BoolVar(&cmdConfig.Auth.WebAuthn.Required, "auth-webauthn-required", cmdConfig.Auth.WebAuthn.Required, + "require WebAuthn authentication. Once set to true, it cannot be set back to false.") + + rootCmd.Flags().BoolVar(&cmdConfig.Auth.SAML.Enabled, "auth-saml-enabled", cmdConfig.Auth.SAML.Enabled, + "enabled SAML authentication.", + ) + rootCmd.Flags().StringVar(&cmdConfig.Auth.SAML.MetadataURL, "auth-saml-url", cmdConfig.Auth.SAML.MetadataURL, "SAML identity provider metadata URL (mutually exclusive with --auth-saml-metadata") + rootCmd.Flags().StringVar(&cmdConfig.Auth.SAML.Metadata, "auth-saml-metadata", cmdConfig.Auth.SAML.Metadata, + "SAML identity provider metadata file path (mutually exclusive with --auth-saml-url).", + ) + rootCmd.Flags().Var( + &cmdConfig.Auth.SAML.LabelRules, + "auth-saml-label-rules", + "defines mapping of SAML assertion attributes into Omni identity labels", + ) + rootCmd.Flags().StringSliceVar( + &cmdConfig.Auth.Auth0.InitialUsers, + "initial-users", + cmdConfig.Auth.Auth0.InitialUsers, + "initial set of user emails. these users will be created on startup.", + ) + rootCmd.Flags().DurationVar( + &cmdConfig.Auth.KeyPruner.Interval, + "public-key-pruning-interval", + cmdConfig.Auth.KeyPruner.Interval, + "interval between public key pruning runs.", + ) + rootCmd.Flags().BoolVar( + &cmdConfig.Auth.Suspended, + "suspended", + cmdConfig.Auth.Suspended, + "start omni in suspended (read-only) mode.", + ) + + rootCmd.Flags().BoolVar( + &cmdConfig.Auth.InitialServiceAccount.Enabled, + "create-initial-service-account", + cmdConfig.Auth.InitialServiceAccount.Enabled, + "create and dump a service account credentials on the first start of Omni", + ) + + rootCmd.Flags().StringVar( + &cmdConfig.Auth.InitialServiceAccount.KeyPath, + "initial-service-account-key-path", + cmdConfig.Auth.InitialServiceAccount.KeyPath, + "dump the initial service account key into the path", + ) + + rootCmd.Flags().StringVar( + &cmdConfig.Auth.InitialServiceAccount.Role, + "initial-service-account-role", + cmdConfig.Auth.InitialServiceAccount.Role, + "the initial service account access role", + ) + + rootCmd.Flags().DurationVar( + &cmdConfig.Auth.InitialServiceAccount.Lifetime, + "initial-service-account-lifetime", + cmdConfig.Auth.InitialServiceAccount.Lifetime, + "the lifetime duration of the initial service account key", + ) + + rootCmd.MarkFlagsMutuallyExclusive("auth-saml-url", "auth-saml-metadata") +} + +func defineLogsFlags() { + rootCmd.Flags().IntVar( + &cmdConfig.Logs.Machine.BufferInitialCapacity, "machine-log-buffer-capacity", + cmdConfig.Logs.Machine.BufferInitialCapacity, "initial buffer capacity for machine logs in bytes") + rootCmd.Flags().IntVar(&cmdConfig.Logs.Machine.BufferMaxCapacity, "machine-log-buffer-max-capacity", + cmdConfig.Logs.Machine.BufferMaxCapacity, "max buffer capacity for machine logs in bytes") + rootCmd.Flags().IntVar(&cmdConfig.Logs.Machine.BufferSafetyGap, "machine-log-buffer-safe-gap", + cmdConfig.Logs.Machine.BufferSafetyGap, "safety gap for machine log buffer in bytes") + rootCmd.Flags().IntVar(&cmdConfig.Logs.Machine.Storage.NumCompressedChunks, "machine-log-num-compressed-chunks", + cmdConfig.Logs.Machine.Storage.NumCompressedChunks, "number of compressed log chunks to keep") + rootCmd.Flags().BoolVar(&cmdConfig.Logs.Machine.Storage.Enabled, "machine-log-storage-enabled", + cmdConfig.Logs.Machine.Storage.Enabled, "enable machine log storage") + rootCmd.Flags().StringVar(&cmdConfig.Logs.Machine.Storage.Path, "machine-log-storage-path", + cmdConfig.Logs.Machine.Storage.Path, "path of the directory for storing machine logs") + rootCmd.Flags().DurationVar(&cmdConfig.Logs.Machine.Storage.FlushPeriod, "machine-log-storage-flush-period", + cmdConfig.Logs.Machine.Storage.FlushPeriod, "period for flushing machine logs to disk") + rootCmd.Flags().Float64Var(&cmdConfig.Logs.Machine.Storage.FlushJitter, "machine-log-storage-flush-jitter", + cmdConfig.Logs.Machine.Storage.FlushJitter, "jitter for the machine log storage flush period") + + rootCmd.Flags().StringSliceVar( + &cmdConfig.Logs.ResourceLogger.Types, + "log-resource-updates-types", + cmdConfig.Logs.ResourceLogger.Types, + "list of resource types whose updates should be logged", + ) + rootCmd.Flags().StringVar( + &cmdConfig.Logs.ResourceLogger.LogLevel, + "log-resource-updates-log-level", + cmdConfig.Logs.ResourceLogger.LogLevel, + "log level for resource updates", + ) + + rootCmd.Flags().StringVar( + &cmdConfig.Logs.Audit.Path, + "audit-log-dir", + cmdConfig.Logs.Audit.Path, + "Directory for audit log storage", + ) + + rootCmd.Flags().BoolVar( + &cmdConfig.Logs.Stripe.Enabled, + "enable-stripe-reporting", + cmdConfig.Logs.Stripe.Enabled, + "enable Stripe machine usage reporting", + ) // keep the old flags for backwards-compatibility { - rootCmd.Flags().BoolVar(&config.Config.MachineLogConfig.StorageEnabled, "log-storage-enabled", config.Config.MachineLogConfig.StorageEnabled, "enable machine log storage") - rootCmd.Flags().StringVar(&config.Config.MachineLogConfig.StoragePath, "log-storage-path", config.Config.MachineLogConfig.StoragePath, + rootCmd.Flags().BoolVar(&cmdConfig.Logs.Machine.Storage.Enabled, "log-storage-enabled", cmdConfig.Logs.Machine.Storage.Enabled, "enable machine log storage") + rootCmd.Flags().StringVar(&cmdConfig.Logs.Machine.Storage.Path, "log-storage-path", cmdConfig.Logs.Machine.Storage.Path, "path of the directory for storing machine logs") - rootCmd.Flags().DurationVar(&config.Config.MachineLogConfig.StorageFlushPeriod, "log-storage-flush-period", config.Config.MachineLogConfig.StorageFlushPeriod, + rootCmd.Flags().DurationVar(&cmdConfig.Logs.Machine.Storage.FlushPeriod, "log-storage-flush-period", cmdConfig.Logs.Machine.Storage.FlushPeriod, "period for flushing machine logs to disk") rootCmd.Flags().MarkDeprecated("log-storage-enabled", "use --machine-log-storage-enabled") //nolint:errcheck rootCmd.Flags().MarkDeprecated("log-storage-path", "use --machine-log-storage-path") //nolint:errcheck rootCmd.Flags().MarkDeprecated("log-storage-flush-period", "use --machine-log-storage-flush-period") //nolint:errcheck } +} - rootCmd.Flags().BoolVar(&config.Config.Auth.Auth0.Enabled, "auth-auth0-enabled", config.Config.Auth.Auth0.Enabled, - "enable Auth0 authentication. Once set to true, it cannot be set back to false.") - rootCmd.Flags().StringVar(&config.Config.Auth.Auth0.ClientID, "auth-auth0-client-id", config.Config.Auth.Auth0.ClientID, "Auth0 application client ID.") - rootCmd.Flags().StringVar(&config.Config.Auth.Auth0.Domain, "auth-auth0-domain", config.Config.Auth.Auth0.Domain, "Auth0 application domain.") - rootCmd.Flags().BoolVar(&config.Config.Auth.Auth0.UseFormData, "auth-auth0-use-form-data", config.Config.Auth.Auth0.UseFormData, - "When true, data to the token endpoint is transmitted as x-www-form-urlencoded data instead of JSON. The default is false") - - rootCmd.Flags().BoolVar(&config.Config.Auth.WebAuthn.Enabled, "auth-webauthn-enabled", config.Config.Auth.WebAuthn.Enabled, - "enable WebAuthn authentication. Once set to true, it cannot be set back to false.") - rootCmd.Flags().BoolVar(&config.Config.Auth.WebAuthn.Required, "auth-webauthn-required", config.Config.Auth.WebAuthn.Required, - "require WebAuthn authentication. Once set to true, it cannot be set back to false.") - - rootCmd.Flags().BoolVar(&config.Config.Auth.SAML.Enabled, "auth-saml-enabled", config.Config.Auth.SAML.Enabled, - "enabled SAML authentication.", - ) - rootCmd.Flags().StringVar(&config.Config.Auth.SAML.URL, "auth-saml-url", config.Config.Auth.SAML.URL, "SAML identity provider metadata URL (mutually exclusive with --auth-saml-metadata") - rootCmd.Flags().StringVar(&config.Config.Auth.SAML.Metadata, "auth-saml-metadata", config.Config.Auth.SAML.Metadata, - "SAML identity provider metadata file path (mutually exclusive with --auth-saml-url).", - ) - rootCmd.Flags().Var(&config.Config.Auth.SAML.LabelRules, "auth-saml-label-rules", "defines mapping of SAML assertion attributes into Omni identity labels") - - rootCmd.Flags().StringSliceVar(&config.Config.InitialUsers, "initial-users", config.Config.InitialUsers, "initial set of user emails. these users will be created on startup.") - - rootCmd.Flags().StringVar(&config.Config.Storage.Kind, "storage-kind", config.Config.Storage.Kind, "storage type: etcd|boltdb.") - rootCmd.Flags().BoolVar(&config.Config.Storage.Etcd.Embedded, "etcd-embedded", config.Config.Storage.Etcd.Embedded, "use embedded etcd server.") - rootCmd.Flags().BoolVar(&config.Config.Storage.Etcd.EmbeddedUnsafeFsync, "etcd-embedded-unsafe-fsync", config.Config.Storage.Etcd.EmbeddedUnsafeFsync, - "disable fsync in the embedded etcd server (dangerous).") - rootCmd.Flags().StringSliceVar(&config.Config.Storage.Etcd.Endpoints, "etcd-endpoints", config.Config.Storage.Etcd.Endpoints, "external etcd endpoints.") - rootCmd.Flags().DurationVar(&config.Config.Storage.Etcd.DialKeepAliveTime, - "etcd-dial-keepalive-time", config.Config.Storage.Etcd.DialKeepAliveTime, "external etcd client keep-alive time (interval).") - rootCmd.Flags().DurationVar(&config.Config.Storage.Etcd.DialKeepAliveTimeout, - "etcd-dial-keepalive-timeout", config.Config.Storage.Etcd.DialKeepAliveTimeout, "external etcd client keep-alive timeout.") - rootCmd.Flags().StringVar(&config.Config.Storage.Etcd.CAPath, "etcd-ca-path", config.Config.Storage.Etcd.CAPath, "external etcd CA path.") - rootCmd.Flags().StringVar(&config.Config.Storage.Etcd.CertPath, "etcd-client-cert-path", config.Config.Storage.Etcd.CertPath, "external etcd client cert path.") - rootCmd.Flags().StringVar(&config.Config.Storage.Etcd.KeyPath, "etcd-client-key-path", config.Config.Storage.Etcd.KeyPath, "external etcd client key path.") - - rootCmd.Flags().StringVar(&config.Config.SecondaryStorage.Path, "secondary-storage-path", config.Config.SecondaryStorage.Path, - "path of the file for boltdb-backed secondary storage for frequently updated data.") - - rootCmd.Flags().StringVar(&config.Config.TalosRegistry, "talos-installer-registry", config.Config.TalosRegistry, "Talos installer image registry.") - rootCmd.Flags().StringVar(&config.Config.KubernetesRegistry, "kubernetes-registry", config.Config.KubernetesRegistry, "Kubernetes container registry.") - rootCmd.Flags().StringVar(&config.Config.ImageFactoryBaseURL, "image-factory-address", config.Config.ImageFactoryBaseURL, "Image factory base URL to use.") - rootCmd.Flags().StringVar(&config.Config.ImageFactoryPXEBaseURL, "image-factory-pxe-address", config.Config.ImageFactoryPXEBaseURL, "Image factory pxe base URL to use.") - +func defineStorageFlags() { rootCmd.Flags().StringVar( - &config.Config.Storage.Etcd.PrivateKeySource, - "private-key-source", - config.Config.Storage.Etcd.PrivateKeySource, - "file containing private key to use for decrypting master key slot.", + &cmdConfig.Storage.Default.Kind, + "storage-kind", + cmdConfig.Storage.Default.Kind, + "storage type: etcd|boltdb.", ) - rootCmd.Flags().StringSliceVar( - &config.Config.Storage.Etcd.PublicKeyFiles, - "public-key-files", - config.Config.Storage.Etcd.PublicKeyFiles, - "list of paths to files containing public keys to use for encrypting keys slots.", + rootCmd.Flags().BoolVar( + &cmdConfig.Storage.Default.Etcd.Embedded, + "etcd-embedded", + cmdConfig.Storage.Default.Etcd.Embedded, + "use embedded etcd server.", + ) + rootCmd.Flags().BoolVar( + &cmdConfig.Storage.Default.Etcd.EmbeddedUnsafeFsync, + "etcd-embedded-unsafe-fsync", + cmdConfig.Storage.Default.Etcd.EmbeddedUnsafeFsync, + "disable fsync in the embedded etcd server (dangerous).", ) - rootCmd.Flags().DurationVar( - &config.Config.KeyPruner.Interval, - "public-key-pruning-interval", - config.Config.KeyPruner.Interval, - "interval between public key pruning runs.", - ) - - rootCmd.Flags().BoolVar(&config.Config.Auth.Suspended, "suspended", config.Config.Auth.Suspended, "start omni in suspended (read-only) mode.") - - rootCmd.Flags().BoolVar(&config.Config.EnableTalosPreReleaseVersions, "enable-talos-pre-release-versions", config.Config.EnableTalosPreReleaseVersions, - "make Omni version discovery controler include Talos pre-release versions.") - - rootCmd.Flags().BoolVar(&config.Config.WorkloadProxying.Enabled, "workload-proxying-enabled", config.Config.WorkloadProxying.Enabled, "enable workload proxying feature.") - rootCmd.Flags().StringVar(&config.Config.WorkloadProxying.Subdomain, "workload-proxying-subdomain", config.Config.WorkloadProxying.Subdomain, "workload proxying subdomain.") - - rootCmd.Flags().BoolVar(&config.Config.ConfigDataCompression.Enabled, "config-data-compression-enabled", config.Config.ConfigDataCompression.Enabled, "enable config data compression.") - - rootCmd.Flags().IntVar(&config.Config.LocalResourceServerPort, "local-resource-server-port", config.Config.LocalResourceServerPort, "port for local read-only public resource server.") - - ensure.NoError(rootCmd.MarkFlagRequired("private-key-source")) ensure.NoError(rootCmd.Flags().MarkHidden("etcd-embedded-unsafe-fsync")) - rootCmd.Flags().StringVar(&rootCmdArgs.keyFile, "key", "", "TLS key file") - rootCmd.Flags().StringVar(&rootCmdArgs.certFile, "cert", "", "TLS cert file") + rootCmd.Flags().StringSliceVar( + &cmdConfig.Storage.Default.Etcd.Endpoints, + "etcd-endpoints", + cmdConfig.Storage.Default.Etcd.Endpoints, + "external etcd endpoints.", + ) + rootCmd.Flags().DurationVar( + &cmdConfig.Storage.Default.Etcd.DialKeepAliveTime, + "etcd-dial-keepalive-time", + cmdConfig.Storage.Default.Etcd.DialKeepAliveTime, + "external etcd client keep-alive time (interval).", + ) + rootCmd.Flags().DurationVar( + &cmdConfig.Storage.Default.Etcd.DialKeepAliveTimeout, + "etcd-dial-keepalive-timeout", + cmdConfig.Storage.Default.Etcd.DialKeepAliveTimeout, + "external etcd client keep-alive timeout.", + ) + rootCmd.Flags().StringVar( + &cmdConfig.Storage.Default.Etcd.CAFile, + "etcd-ca-path", + cmdConfig.Storage.Default.Etcd.CAFile, + "external etcd CA path.", + ) + rootCmd.Flags().StringVar( + &cmdConfig.Storage.Default.Etcd.CertFile, + "etcd-client-cert-path", + cmdConfig.Storage.Default.Etcd.CertFile, + "external etcd client cert path.", + ) + rootCmd.Flags().StringVar( + &cmdConfig.Storage.Default.Etcd.KeyFile, + "etcd-client-key-path", + cmdConfig.Storage.Default.Etcd.KeyFile, + "external etcd client key path.", + ) + rootCmd.Flags().StringVar( + &cmdConfig.Storage.Default.Etcd.PrivateKeySource, + "private-key-source", + cmdConfig.Storage.Default.Etcd.PrivateKeySource, + "file containing private key to use for decrypting master key slot.", + ) + + ensure.NoError(rootCmd.MarkFlagRequired("private-key-source")) + + rootCmd.Flags().StringSliceVar( + &cmdConfig.Storage.Default.Etcd.PublicKeyFiles, + "public-key-files", + cmdConfig.Storage.Default.Etcd.PublicKeyFiles, + "list of paths to files containing public keys to use for encrypting keys slots.", + ) + rootCmd.Flags().StringVar( + &cmdConfig.Storage.Secondary.Path, + "secondary-storage-path", + cmdConfig.Storage.Secondary.Path, + "path of the file for boltdb-backed secondary storage for frequently updated data.", + ) +} + +func defineRegistriesFlags() { + rootCmd.Flags().StringVar( + &cmdConfig.Registries.Talos, + "talos-installer-registry", + cmdConfig.Registries.Talos, + "Talos installer image registry.", + ) + rootCmd.Flags().StringVar( + &cmdConfig.Registries.Kubernetes, + "kubernetes-registry", + cmdConfig.Registries.Kubernetes, + "Kubernetes container registry.", + ) + rootCmd.Flags().StringVar( + &cmdConfig.Registries.ImageFactoryBaseURL, + "image-factory-address", + cmdConfig.Registries.ImageFactoryBaseURL, + "Image factory base URL to use.", + ) + rootCmd.Flags().StringVar( + &cmdConfig.Registries.ImageFactoryPXEBaseURL, + "image-factory-pxe-address", + cmdConfig.Registries.ImageFactoryPXEBaseURL, + "Image factory pxe base URL to use.", + ) + + rootCmd.Flags().StringSliceVar( + &cmdConfig.Registries.Mirrors, + "registry-mirror", + []string{}, + "list of registry mirrors to use in format: =", + ) +} + +func defineFeatureFlags() { + rootCmd.Flags().BoolVar( + &cmdConfig.Features.EnableTalosPreReleaseVersions, + "enable-talos-pre-release-versions", + cmdConfig.Features.EnableTalosPreReleaseVersions, + "make Omni version discovery controler include Talos pre-release versions.", + ) rootCmd.Flags().BoolVar( - &config.Config.EtcdBackup.S3Enabled, + &cmdConfig.Features.EnableConfigDataCompression, + "config-data-compression-enabled", + cmdConfig.Features.EnableConfigDataCompression, + "enable config data compression.", + ) + + rootCmd.Flags().BoolVar( + &cmdConfig.Services.WorkloadProxy.Enabled, + "workload-proxying-enabled", + cmdConfig.Services.WorkloadProxy.Enabled, + "enable workload proxying feature.", + ) + + rootCmd.Flags().StringVar( + &cmdConfig.Services.WorkloadProxy.Subdomain, + "workload-proxying-subdomain", + cmdConfig.Services.WorkloadProxy.Subdomain, + "workload proxying subdomain.", + ) + + rootCmd.Flags().BoolVar( + &cmdConfig.Features.EnableBreakGlassConfigs, + "enable-break-glass-configs", + cmdConfig.Features.EnableBreakGlassConfigs, + "Allows downloading admin Talos and Kubernetes configs.", + ) + + rootCmd.Flags().BoolVar( + &cmdConfig.Features.DisableControllerRuntimeCache, + "disable-controller-runtime-cache", + cmdConfig.Features.DisableControllerRuntimeCache, + "disable watch-based cache for controller-runtime (affects performance)", + ) +} + +func defineDebugFlags() { + rootCmd.Flags().StringVar( + &cmdConfig.Debug.Pprof.Endpoint, + "pprof-bind-addr", + cmdConfig.Debug.Pprof.Endpoint, + "start pprof HTTP server on the defined address (\"\" if disabled).", + ) + + rootCmd.Flags().StringVar( + &cmdConfig.Debug.Server.Endpoint, + "debug-server-endpoint", + cmdConfig.Debug.Server.Endpoint, + "start debug HTTP server on the defined address (\"\" if disabled).", + ) +} + +func defineEtcdBackupsFlags() { + rootCmd.Flags().BoolVar( + &cmdConfig.EtcdBackup.S3Enabled, "etcd-backup-s3", - config.Config.EtcdBackup.S3Enabled, + cmdConfig.EtcdBackup.S3Enabled, "S3 will be used for cluster etcd backups", ) rootCmd.Flags().StringVar( - &config.Config.EtcdBackup.LocalPath, + &cmdConfig.EtcdBackup.LocalPath, "etcd-backup-local-path", - config.Config.EtcdBackup.LocalPath, + cmdConfig.EtcdBackup.LocalPath, "path to local directory for cluster etcd backups", ) - rootCmd.MarkFlagsMutuallyExclusive("etcd-backup-s3", "etcd-backup-local-path") - rootCmd.MarkFlagsMutuallyExclusive("auth-saml-url", "auth-saml-metadata") - rootCmd.Flags().DurationVar( - &config.Config.EtcdBackup.TickInterval, + &cmdConfig.EtcdBackup.TickInterval, "etcd-backup-tick-interval", - config.Config.EtcdBackup.TickInterval, + cmdConfig.EtcdBackup.TickInterval, "interval between etcd backups ticks (controller events to check if any cluster needs to be backed up)", ) rootCmd.Flags().DurationVar( - &config.Config.EtcdBackup.Jitter, + &cmdConfig.EtcdBackup.Jitter, "etcd-backup-jitter", - config.Config.EtcdBackup.Jitter, + cmdConfig.EtcdBackup.Jitter, "jitter for etcd backups, randomly added/subtracted from the interval between automatic etcd backups", ) rootCmd.Flags().DurationVar( - &config.Config.EtcdBackup.MinInterval, + &cmdConfig.EtcdBackup.MinInterval, "etcd-backup-min-interval", - config.Config.EtcdBackup.MinInterval, + cmdConfig.EtcdBackup.MinInterval, "minimal interval between etcd backups", ) rootCmd.Flags().DurationVar( - &config.Config.EtcdBackup.MaxInterval, + &cmdConfig.EtcdBackup.MaxInterval, "etcd-backup-max-interval", - config.Config.EtcdBackup.MaxInterval, + cmdConfig.EtcdBackup.MaxInterval, "maximal interval between etcd backups", ) rootCmd.Flags().Uint64Var( - &config.Config.EtcdBackup.UploadLimitMbps, + &cmdConfig.EtcdBackup.UploadLimitMbps, "etcd-backup-upload-limit-mbps", - config.Config.EtcdBackup.UploadLimitMbps, + cmdConfig.EtcdBackup.UploadLimitMbps, "throughput limit in Mbps for etcd backup uploads, zero means unlimited", ) rootCmd.Flags().Uint64Var( - &config.Config.EtcdBackup.DownloadLimitMbps, + &cmdConfig.EtcdBackup.DownloadLimitMbps, "etcd-backup-download-limit-mbps", - config.Config.EtcdBackup.DownloadLimitMbps, + cmdConfig.EtcdBackup.DownloadLimitMbps, "throughput limit in Mbps for etcd backup downloads, zero means unlimited", ) - rootCmd.Flags().StringSliceVar(&config.Config.LogResourceUpdatesTypes, - "log-resource-updates-types", - config.Config.LogResourceUpdatesTypes, - "list of resource types whose updates should be logged", - ) - rootCmd.Flags().StringVar(&config.Config.LogResourceUpdatesLogLevel, - "log-resource-updates-log-level", - config.Config.LogResourceUpdatesLogLevel, - "log level for resource updates", - ) - - rootCmd.Flags().BoolVar(&config.Config.DisableControllerRuntimeCache, - "disable-controller-runtime-cache", - config.Config.DisableControllerRuntimeCache, - "disable watch-based cache for controller-runtime (affects performance)", - ) - - rootCmd.Flags().BoolVar( - &config.Config.EmbeddedDiscoveryService.Enabled, - "embedded-discovery-service-enabled", - config.Config.EmbeddedDiscoveryService.Enabled, - "enable embedded discovery service, binds only to the SideroLink WireGuard address", - ) - rootCmd.Flags().IntVar( - &config.Config.EmbeddedDiscoveryService.Port, - "embedded-discovery-service-endpoint", - config.Config.EmbeddedDiscoveryService.Port, - "embedded discovery service port to listen on", - ) - rootCmd.Flags().BoolVar( - &config.Config.EmbeddedDiscoveryService.SnapshotsEnabled, - "embedded-discovery-service-snapshots-enabled", - config.Config.EmbeddedDiscoveryService.SnapshotsEnabled, - "enable snapshots for the embedded discovery service", - ) - rootCmd.Flags().StringVar( - &config.Config.EmbeddedDiscoveryService.SnapshotPath, - "embedded-discovery-service-snapshot-path", - config.Config.EmbeddedDiscoveryService.SnapshotPath, - "path to the file for storing the embedded discovery service state", - ) - rootCmd.Flags().DurationVar( - &config.Config.EmbeddedDiscoveryService.SnapshotInterval, - "embedded-discovery-service-snapshot-interval", - config.Config.EmbeddedDiscoveryService.SnapshotInterval, - "interval for saving the embedded discovery service state", - ) - rootCmd.Flags().StringVar( - &config.Config.EmbeddedDiscoveryService.LogLevel, - "embedded-discovery-service-log-level", - config.Config.EmbeddedDiscoveryService.LogLevel, - "log level for the embedded discovery service - it has no effect if it is lower (more verbose) than the main log level", - ) - - rootCmd.Flags().BoolVar( - &config.Config.EnableBreakGlassConfigs, - "enable-break-glass-configs", - config.Config.EnableBreakGlassConfigs, - "Allows downloading admin Talos and Kubernetes configs.", - ) - - rootCmd.Flags().StringVar( - &config.Config.AuditLogDir, - "audit-log-dir", - config.Config.AuditLogDir, - "Directory for audit log storage", - ) - - rootCmd.Flags().BoolVar( - &config.Config.InitialServiceAccount.Enabled, - "create-initial-service-account", - config.Config.InitialServiceAccount.Enabled, - "create and dump a service account credentials on the first start of Omni", - ) - - rootCmd.Flags().StringVar( - &config.Config.InitialServiceAccount.KeyPath, - "initial-service-account-key-path", - config.Config.InitialServiceAccount.KeyPath, - "dump the initial service account key into the path", - ) - - rootCmd.Flags().StringVar( - &config.Config.InitialServiceAccount.Role, - "initial-service-account-role", - config.Config.InitialServiceAccount.Role, - "the initial service account access role", - ) - - rootCmd.Flags().DurationVar( - &config.Config.InitialServiceAccount.Lifetime, - "initial-service-account-lifetime", - config.Config.InitialServiceAccount.Lifetime, - "the lifetime duration of the initial service account key", - ) - - rootCmd.Flags().BoolVar( - &config.Config.EnableStripeReporting, - "enable-stripe-reporting", - config.Config.EnableStripeReporting, - "enable Stripe machine usage reporting", - ) - - rootCmd.Flags().Var( - &config.Config.JoinTokensMode, - "join-tokens-mode", - "configures Talos machine join flow to use secure node tokens", - ) - - return rootCmd -}) + rootCmd.MarkFlagsMutuallyExclusive("etcd-backup-s3", "etcd-backup-local-path") +} diff --git a/cmd/omni/cmd/run.go b/cmd/omni/cmd/run.go index fb319cc1..e6c236c4 100644 --- a/cmd/omni/cmd/run.go +++ b/cmd/omni/cmd/run.go @@ -8,46 +8,64 @@ package cmd import ( "context" "fmt" - "strings" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" "github.com/go-logr/zapr" "github.com/prometheus/client_golang/prometheus" - "github.com/siderolabs/talos/pkg/machinery/config/generate" + "github.com/siderolabs/talos/pkg/machinery/config/merge" "go.uber.org/zap" "go.uber.org/zap/zapcore" - "gopkg.in/yaml.v3" "k8s.io/klog/v2" "github.com/siderolabs/omni/client/pkg/compression" "github.com/siderolabs/omni/client/pkg/constants" + authres "github.com/siderolabs/omni/client/pkg/omni/resources/auth" + omnires "github.com/siderolabs/omni/client/pkg/omni/resources/omni" "github.com/siderolabs/omni/client/pkg/panichandler" + "github.com/siderolabs/omni/internal/backend" + "github.com/siderolabs/omni/internal/backend/discovery" + "github.com/siderolabs/omni/internal/backend/dns" + "github.com/siderolabs/omni/internal/backend/imagefactory" "github.com/siderolabs/omni/internal/backend/logging" + "github.com/siderolabs/omni/internal/backend/resourcelogger" "github.com/siderolabs/omni/internal/backend/runtime/omni" + "github.com/siderolabs/omni/internal/backend/runtime/omni/virtual" + "github.com/siderolabs/omni/internal/backend/runtime/talos" + "github.com/siderolabs/omni/internal/backend/workloadproxy" + "github.com/siderolabs/omni/internal/pkg/auth" "github.com/siderolabs/omni/internal/pkg/auth/actor" + "github.com/siderolabs/omni/internal/pkg/auth/user" "github.com/siderolabs/omni/internal/pkg/config" + "github.com/siderolabs/omni/internal/pkg/ctxstore" + "github.com/siderolabs/omni/internal/pkg/features" + "github.com/siderolabs/omni/internal/pkg/siderolink" "github.com/siderolabs/omni/internal/version" ) // RunService starts the main Omni server. -func RunService(ctx context.Context, logger *zap.Logger, params *config.Params) error { +func RunService(ctx context.Context, logger *zap.Logger, paramsList ...*config.Params) error { cfg := config.InitDefault() - raw, err := yaml.Marshal(params) - if err != nil { - return err + for _, params := range paramsList { + if err := merge.Merge(cfg, params); err != nil { + return err + } } - if err = yaml.Unmarshal(raw, &cfg); err != nil { + cfg.PopulateFallbacks() + + if err := cfg.Validate(); err != nil { return err } config.Config = cfg - if err = compression.InitConfig(config.Config.ConfigDataCompression.Enabled); err != nil { + if err := compression.InitConfig(config.Config.Features.EnableConfigDataCompression); err != nil { return err } - logger.Info("initialized resource compression config", zap.Bool("enabled", config.Config.ConfigDataCompression.Enabled)) + logger.Info("initialized resource compression config", zap.Bool("enabled", config.Config.Features.EnableConfigDataCompression)) // set kubernetes logger to use warn log level and use zap klog.SetLogger(zapr.NewLogger(logger.WithOptions(zap.IncreaseLevel(zapcore.WarnLevel)).With(logging.Component("kubernetes")))) @@ -56,32 +74,127 @@ func RunService(ctx context.Context, logger *zap.Logger, params *config.Params) logger.Warn("running debug build") } - for _, registryMirror := range rootCmdArgs.registryMirrors { - hostname, endpoint, ok := strings.Cut(registryMirror, "=") - if !ok { - return fmt.Errorf("invalid registry mirror spec: %q", registryMirror) - } - - config.Config.DefaultConfigGenOptions = append(config.Config.DefaultConfigGenOptions, generate.WithRegistryMirror(hostname, endpoint)) - } - logger.Info("starting Omni", zap.String("version", version.Tag)) logger.Debug("using config", zap.Any("config", config.Config)) - if cfg.RunDebugServer { + if cfg.Debug.Server.Endpoint != "" { panichandler.Go(func() { - runDebugServer(ctx, logger) + runDebugServer(ctx, logger, cfg.Debug.Server.Endpoint) }, logger) } // this global context propagates into all controllers and any other background activities ctx = actor.MarkContextAsInternalActor(ctx) - err = omni.NewState(ctx, config.Config, logger, prometheus.DefaultRegisterer, runWithState(logger)) + err := omni.NewState(ctx, config.Config, logger, prometheus.DefaultRegisterer, runWithState(logger)) if err != nil { return fmt.Errorf("failed to run Omni: %w", err) } return nil } + +func runWithState(logger *zap.Logger) func(context.Context, state.State, *virtual.State) error { + return func(ctx context.Context, resourceState state.State, virtualState *virtual.State) error { + auditWrap, auditErr := omni.NewAuditWrap(resourceState, config.Config, logger) + if auditErr != nil { + return auditErr + } + + resourceState = auditWrap.WrapState(resourceState) + + talosClientFactory := talos.NewClientFactory(resourceState, logger) + prometheus.MustRegister(talosClientFactory) + + dnsService := dns.NewService(resourceState, logger) + workloadProxyReconciler := workloadproxy.NewReconciler(logger.With(logging.Component("workload_proxy_reconciler")), zapcore.DebugLevel) + + var resourceLogger *resourcelogger.Logger + + if len(config.Config.Logs.ResourceLogger.Types) > 0 { + var err error + + resourceLogger, err = resourcelogger.New(ctx, resourceState, logger.With(logging.Component("resourcelogger")), + config.Config.Logs.ResourceLogger.LogLevel, config.Config.Logs.ResourceLogger.Types...) + if err != nil { + return fmt.Errorf("failed to set up resource logger: %w", err) + } + } + + imageFactoryClient, err := imagefactory.NewClient(resourceState, config.Config.Registries.ImageFactoryBaseURL) + if err != nil { + return fmt.Errorf("failed to set up image factory client: %w", err) + } + + linkCounterDeltaCh := make(chan siderolink.LinkCounterDeltas) + siderolinkEventsCh := make(chan *omnires.MachineStatusSnapshot) + installEventCh := make(chan resource.ID) + + discoveryClientCache := discovery.NewClientCache(logger.With(logging.Component("discovery_client_factory"))) + defer discoveryClientCache.Close() + + prometheus.MustRegister(discoveryClientCache) + + omniRuntime, err := omni.New(talosClientFactory, dnsService, workloadProxyReconciler, resourceLogger, + imageFactoryClient, linkCounterDeltaCh, siderolinkEventsCh, installEventCh, resourceState, virtualState, + prometheus.DefaultRegisterer, discoveryClientCache, logger.With(logging.Component("omni_runtime"))) + if err != nil { + return fmt.Errorf("failed to set up the controller runtime: %w", err) + } + + machineMap := siderolink.NewMachineMap(siderolink.NewStateStorage(omniRuntime.State())) + + logHandler, err := siderolink.NewLogHandler( + machineMap, + resourceState, + &config.Config.Logs.Machine, + logger.With(logging.Component("siderolink_log_handler")), + ) + if err != nil { + return fmt.Errorf("failed to set up log handler: %w", err) + } + + talosRuntime := talos.New(talosClientFactory, logger) + + err = user.EnsureInitialResources(ctx, omniRuntime.State(), logger, config.Config.Auth.Auth0.InitialUsers) + if err != nil { + return fmt.Errorf("failed to write initial user resources to state: %w", err) + } + + authConfig, err := auth.EnsureAuthConfigResource(ctx, omniRuntime.State(), logger, config.Config.Auth) + if err != nil { + return fmt.Errorf("failed to write Auth0 parameters to state: %w", err) + } + + if err = features.UpdateResources(ctx, omniRuntime.State(), logger); err != nil { + return fmt.Errorf("failed to update features config resources: %w", err) + } + + ctx = ctxstore.WithValue(ctx, auth.EnabledAuthContextKey{Enabled: authres.Enabled(authConfig)}) + + server, err := backend.NewServer( + dnsService, + workloadProxyReconciler, + imageFactoryClient, + linkCounterDeltaCh, + siderolinkEventsCh, + installEventCh, + omniRuntime, + talosRuntime, + logHandler, + authConfig, + auditWrap, + logger, + ) + if err != nil { + return fmt.Errorf("failed to create server: %w", err) + } + + if err := server.Run(ctx); err != nil { + return fmt.Errorf("failed to run server: %w", err) + } + + return nil + } +} diff --git a/go.mod b/go.mod index c948388e..caabd014 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/siderolabs/omni go 1.24.2 replace ( + github.com/siderolabs/gen => github.com/unix4ever/gen v0.0.0-20250606184729-3e319e7e52c5 // use nested module github.com/siderolabs/omni/client => ./client // forked go-yaml that introduces RawYAML interface, which can be used to populate YAML fields using bytes @@ -37,6 +38,7 @@ require ( github.com/gertd/go-pluralize v0.2.1 github.com/go-jose/go-jose/v4 v4.1.0 github.com/go-logr/zapr v1.3.0 + github.com/go-playground/validator/v10 v10.26.0 github.com/golang-jwt/jwt/v4 v4.5.2 github.com/google/go-cmp v0.7.0 github.com/google/go-containerregistry v0.20.3 @@ -113,6 +115,8 @@ require ( sigs.k8s.io/controller-runtime v0.20.4 ) +require github.com/go-logr/logr v1.4.2 + require ( cel.dev/expr v0.23.1 // indirect github.com/ProtonMail/go-crypto v1.2.0 // indirect @@ -153,13 +157,15 @@ require ( github.com/evanphx/json-patch v5.9.11+incompatible // indirect github.com/fatih/color v1.18.0 // indirect github.com/fxamacker/cbor/v2 v2.8.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.8 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/go-chi/chi/v5 v5.2.1 // indirect - github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/jsonpointer v0.21.1 // indirect github.com/go-openapi/jsonreference v0.21.0 // indirect github.com/go-openapi/swag v0.23.1 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/btree v1.1.3 // indirect @@ -184,6 +190,7 @@ require ( github.com/josharian/native v1.1.0 // indirect github.com/jsimonetti/rtnetlink/v2 v2.0.3 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/leodido/go-urn v1.4.0 // indirect github.com/mailru/easyjson v0.9.0 // indirect github.com/mattermost/xml-roundtrip-validator v0.1.0 // indirect github.com/mattn/go-colorable v0.1.14 // indirect diff --git a/go.sum b/go.sum index ae20ef96..cb3a2a50 100644 --- a/go.sum +++ b/go.sum @@ -152,6 +152,8 @@ github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/fxamacker/cbor/v2 v2.8.0 h1:fFtUGXUzXPHTIUdne5+zzMPTfffl3RD5qYnkY40vtxU= github.com/fxamacker/cbor/v2 v2.8.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/gabriel-vasile/mimetype v1.4.8 h1:FfZ3gj38NjllZIeJAmMhr+qKL8Wu+nOoI3GqacKw1NM= +github.com/gabriel-vasile/mimetype v1.4.8/go.mod h1:ByKUIKGjh1ODkGM1asKUbQZOLGrPjydw3hYPU2YU9t8= github.com/gertd/go-pluralize v0.2.1 h1:M3uASbVjMnTsPb0PNqg+E/24Vwigyo/tvyMTtAlLgiA= github.com/gertd/go-pluralize v0.2.1/go.mod h1:rbYaKDbsXxmRfr8uygAEKhOWsjyrrqrkHVpZvoOp8zk= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= @@ -175,6 +177,14 @@ github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4= github.com/go-openapi/swag v0.23.1 h1:lpsStH0n2ittzTnbaSloVZLuB5+fvSY/+hnagBjSNZU= github.com/go-openapi/swag v0.23.1/go.mod h1:STZs8TbRvEQQKUA+JZNAm3EWlgaOBGpyFDqQnDHMef0= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.26.0 h1:SP05Nqhjcvz81uJaRfEV0YBSSSGMc/iMaVtFbr3Sw2k= +github.com/go-playground/validator/v10 v10.26.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= @@ -302,6 +312,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4= github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= github.com/mattermost/xml-roundtrip-validator v0.1.0 h1:RXbVD2UAl7A7nOTR4u7E3ILa4IbtvKBHw64LDsmu9hU= @@ -401,8 +413,6 @@ github.com/siderolabs/discovery-client v0.1.11 h1:Au+7QZ+CIB6g4C7ZCC4m5Ai5Uso1g/ github.com/siderolabs/discovery-client v0.1.11/go.mod h1:Iw5XUphGNNV0m2czHjbj9aLhQvfM8hYEfWCc6fdQ4ko= github.com/siderolabs/discovery-service v1.0.10 h1:GSd5p+bC+PJjIpCqiDtVFKKU18LpsS6jmv+3OF55+Bw= github.com/siderolabs/discovery-service v1.0.10/go.mod h1:tzeHcfftQQHZSShuSTcrgIN3BY6fmhlum/Z9yOJ61lk= -github.com/siderolabs/gen v0.8.1 h1:i26KvarXfkXYqsmIicjGr2DN68uuuKDse8J4Z2J0O/Y= -github.com/siderolabs/gen v0.8.1/go.mod h1:CIhFWgYkOKtxKD8Zct5NcJMgRzefnF2XIqeGOA+um0U= github.com/siderolabs/go-api-signature v0.3.6 h1:wDIsXbpl7Oa/FXvxB6uz4VL9INA9fmr3EbmjEZYFJrU= github.com/siderolabs/go-api-signature v0.3.6/go.mod h1:hoH13AfunHflxbXfh+NoploqV13ZTDfQ1mQJWNVSW9U= github.com/siderolabs/go-circular v0.2.3 h1:GKkA1Tw79kEFGtWdl7WTxEUTbwtklITeiRT0V1McHrA= @@ -477,6 +487,8 @@ github.com/stripe/stripe-go/v76 v76.25.0 h1:kmDoOTvdQSTQssQzWZQQkgbAR2Q8eXdMWbN/ github.com/stripe/stripe-go/v76 v76.25.0/go.mod h1:rw1MxjlAKKcZ+3FOXgTHgwiOa2ya6CPq6ykpJ0Q6Po4= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk= +github.com/unix4ever/gen v0.0.0-20250606184729-3e319e7e52c5 h1:+dggEUpyHfLUWap8CITVdbPdZrv2tH7m2uqjOoQqin4= +github.com/unix4ever/gen v0.0.0-20250606184729-3e319e7e52c5/go.mod h1:CRrktDXQf3yDJI7xKv+cDYhBbKdfd/YE16OpgcHoT9E= github.com/unix4ever/yaml v0.0.0-20220527175918-f17b0f05cf2c h1:Vn6nVVu9MdOYvXPkJP83iX5jVIfvxFC9v9xIKb+DlaQ= github.com/unix4ever/yaml v0.0.0-20220527175918-f17b0f05cf2c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= github.com/vbatts/tar-split v0.12.1 h1:CqKoORW7BUWBe7UL/iqTVvkTBOF8UvOMKOIZykxnnbo= diff --git a/hack/test/integration.sh b/hack/test/integration.sh index dc78a7a6..a0d60cc3 100755 --- a/hack/test/integration.sh +++ b/hack/test/integration.sh @@ -146,6 +146,7 @@ SIDEROLINK_DEV_JOIN_TOKEN="${JOIN_TOKEN}" \ --cert hack/certs/localhost.pem \ --etcd-embedded-unsafe-fsync=true \ --etcd-backup-s3 \ + --join-tokens-mode strict \ --audit-log-dir /tmp/omni-data/audit-log \ --enable-talos-pre-release-versions="${ENABLE_TALOS_PRERELEASE_VERSIONS}" \ "${REGISTRY_MIRROR_FLAGS[@]}" \ diff --git a/internal/backend/factory/factory.go b/internal/backend/factory/factory.go index aede0a50..bdf57c78 100644 --- a/internal/backend/factory/factory.go +++ b/internal/backend/factory/factory.go @@ -30,8 +30,18 @@ import ( // Handler of image requests. type Handler struct { - State state.State - Logger *zap.Logger + state state.State + logger *zap.Logger + config *config.Registries +} + +// NewHandler creates a new factory proxy handler. +func NewHandler(state state.State, logger *zap.Logger, config *config.Registries) *Handler { + return &Handler{ + state: state, + logger: logger, + config: config, + } } func setContentHeaders(w http.ResponseWriter, contentType, filename string) { @@ -45,7 +55,7 @@ func httpNotFound(w http.ResponseWriter) { } func (handler *Handler) handleError(msg string, w http.ResponseWriter, err error) { - handler.Logger.Error(msg, zap.Error(err)) + handler.logger.Error(msg, zap.Error(err)) switch status.Code(err) { //nolint:exhaustive case codes.Unauthenticated: @@ -71,7 +81,7 @@ func (handler *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } - params, err := parseRequest(r, handler.State) + params, err := parseRequest(r, handler.state, handler.config) if err != nil { if errors.Is(err, errNotFound) { httpNotFound(w) @@ -84,7 +94,7 @@ func (handler *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } - handler.Logger.Info("proxy request", zap.String("url", params.ProxyURL)) + handler.logger.Info("proxy request", zap.String("url", params.ProxyURL)) proxyReq, err := http.NewRequestWithContext(r.Context(), r.Method, params.ProxyURL, nil) if err != nil { @@ -138,7 +148,7 @@ type ProxyParams struct { DestinationFilename string } -func parseRequest(r *http.Request, st state.State) (*ProxyParams, error) { +func parseRequest(r *http.Request, st state.State, config *config.Registries) (*ProxyParams, error) { segments := strings.Split(strings.Trim(r.URL.Path, "/"), "/") if len(segments) < 4 { @@ -177,7 +187,7 @@ func parseRequest(r *http.Request, st state.State) (*ProxyParams, error) { DestinationFilename: fmt.Sprintf("%s-%s.%s", media.TypedSpec().Value.DestFilePrefix, srcFilename, media.TypedSpec().Value.Extension), } - proxyURL, err := url.Parse(config.Config.ImageFactoryBaseURL) + proxyURL, err := url.Parse(config.ImageFactoryBaseURL) if err != nil { return nil, err } diff --git a/internal/backend/factory/factory_test.go b/internal/backend/factory/factory_test.go index e0443fb2..a0d5fbd9 100644 --- a/internal/backend/factory/factory_test.go +++ b/internal/backend/factory/factory_test.go @@ -20,6 +20,7 @@ import ( "github.com/siderolabs/omni/client/pkg/omni/resources" "github.com/siderolabs/omni/client/pkg/omni/resources/omni" "github.com/siderolabs/omni/internal/backend/factory" + "github.com/siderolabs/omni/internal/pkg/config" ) func TestParseRequest(t *testing.T) { @@ -92,7 +93,9 @@ func TestParseRequest(t *testing.T) { request, err := http.NewRequestWithContext(ctx, http.MethodHead, "https://localhost"+tt.incomingURL, nil) require.NoError(err) - params, err := factory.ParseRequest(request, state) + params, err := factory.ParseRequest(request, state, &config.Registries{ + ImageFactoryBaseURL: "https://factory.talos.dev", + }) if tt.shouldFail { require.Error(err) diff --git a/internal/backend/grpc/auth.go b/internal/backend/grpc/auth.go index 8fed5597..a04eea64 100644 --- a/internal/backend/grpc/auth.go +++ b/internal/backend/grpc/auth.go @@ -269,7 +269,7 @@ func verifiedEmail(ctx context.Context) (string, error) { } func (s *authServer) buildLoginURL(pgpKeyID string) (string, error) { - loginURL, err := url.Parse(config.Config.APIURL) + loginURL, err := url.Parse(config.Config.Services.API.URL()) if err != nil { return "", err } diff --git a/internal/backend/grpc/configs_test.go b/internal/backend/grpc/configs_test.go index 0c774c34..b351a206 100644 --- a/internal/backend/grpc/configs_test.go +++ b/internal/backend/grpc/configs_test.go @@ -92,10 +92,10 @@ func TestGenerateConfigs(t *testing.T) { require.Equal(t, codes.PermissionDenied, status.Code(err), err) }) - config.Config.EnableBreakGlassConfigs = true + config.Config.Features.EnableBreakGlassConfigs = true defer func() { - config.Config.EnableBreakGlassConfigs = false + config.Config.Features.EnableBreakGlassConfigs = false }() t.Run("kubeconfig enabled no cluster", func(t *testing.T) { diff --git a/internal/backend/grpc/grpc.go b/internal/backend/grpc/grpc.go index f0e32ac0..146df67d 100644 --- a/internal/backend/grpc/grpc.go +++ b/internal/backend/grpc/grpc.go @@ -52,7 +52,7 @@ func MakeServiceServers( logger *zap.Logger, auditor AuditLogger, ) iter.Seq2[ServiceServer, error] { - dest, err := generateDest(config.Config.APIURL) + dest, err := generateDest(config.Config.Services.API.URL()) if err != nil { return func(yield func(ServiceServer, error) bool) { yield(nil, fmt.Errorf("error generating destination: %w", err)) diff --git a/internal/backend/grpc/management.go b/internal/backend/grpc/management.go index 8542569d..689ba7d7 100644 --- a/internal/backend/grpc/management.go +++ b/internal/backend/grpc/management.go @@ -330,7 +330,7 @@ func getBreakGlass(ctx context.Context, clusterName string) ([]byte, error) { } func (s *managementServer) breakGlassTalosconfig(ctx context.Context, raw bool) (*management.TalosconfigResponse, error) { - if !constants.IsDebugBuild && !config.Config.EnableBreakGlassConfigs { + if !constants.IsDebugBuild && !config.Config.Features.EnableBreakGlassConfigs { return nil, status.Error(codes.PermissionDenied, "not allowed") } @@ -372,7 +372,7 @@ func (s *managementServer) breakGlassKubeconfig(ctx context.Context) (*managemen return nil, err } - if !constants.IsDebugBuild && !config.Config.EnableBreakGlassConfigs { + if !constants.IsDebugBuild && !config.Config.Features.EnableBreakGlassConfigs { return nil, status.Error(codes.PermissionDenied, "not allowed") } diff --git a/internal/backend/grpc/serviceaccount.go b/internal/backend/grpc/serviceaccount.go index 28ccfaf0..9f9da823 100644 --- a/internal/backend/grpc/serviceaccount.go +++ b/internal/backend/grpc/serviceaccount.go @@ -278,7 +278,7 @@ func (s *managementServer) generateServiceAccountJWT(ctx context.Context, req *m now := time.Now() token := jwt.NewWithClaims(signingMethod, jwt.MapClaims{ "iat": now.Unix(), - "iss": fmt.Sprintf("omni-%s-service-account-issuer", config.Config.Name), + "iss": fmt.Sprintf("omni-%s-service-account-issuer", config.Config.Account.Name), "exp": now.Add(req.GetServiceAccountTtl().AsDuration()).Unix(), "sub": req.GetServiceAccountUser(), "groups": req.GetServiceAccountGroups(), @@ -292,7 +292,7 @@ func (s *managementServer) generateServiceAccountJWT(ctx context.Context, req *m } func (s *managementServer) buildServiceAccountKubeconfig(cluster, user, token string) ([]byte, error) { - clusterName := config.Config.Name + "-" + cluster + "-" + user + clusterName := config.Config.Account.Name + "-" + cluster + "-" + user contextName := clusterName conf := clientcmdapi.Config{ @@ -301,7 +301,7 @@ func (s *managementServer) buildServiceAccountKubeconfig(cluster, user, token st CurrentContext: contextName, Clusters: map[string]*clientcmdapi.Cluster{ clusterName: { - Server: config.Config.KubernetesProxyURL, + Server: config.Config.Services.KubernetesProxy.URL(), }, }, Contexts: map[string]*clientcmdapi.Context{ diff --git a/internal/backend/installimage/installimage.go b/internal/backend/installimage/installimage.go index e364a8c5..6837b62a 100644 --- a/internal/backend/installimage/installimage.go +++ b/internal/backend/installimage/installimage.go @@ -74,5 +74,5 @@ func Build(imageFactoryHost string, resID resource.ID, installImage *specs.Machi return imageFactoryHost + "/" + installerName + "/" + schematicID + ":" + desiredTalosVersion, nil } - return appconfig.Config.TalosRegistry + ":" + desiredTalosVersion, nil + return appconfig.Config.Registries.Talos + ":" + desiredTalosVersion, nil } diff --git a/internal/backend/runtime/kubernetes/kubernetes.go b/internal/backend/runtime/kubernetes/kubernetes.go index 77b0d3b7..b4b6f646 100644 --- a/internal/backend/runtime/kubernetes/kubernetes.go +++ b/internal/backend/runtime/kubernetes/kubernetes.go @@ -303,11 +303,11 @@ func (r *Runtime) GetOIDCKubeconfig(context *common.Context, identity string, ex Identity string ExtraOptions []string }{ - InstanceName: config.Config.Name, + InstanceName: config.Config.Account.Name, ClusterName: context.Name, EndpointOIDC: issuerEndpoint, - KubernetesProxyEndpoint: config.Config.KubernetesProxyURL, + KubernetesProxyEndpoint: config.Config.Services.KubernetesProxy.URL(), ClientID: external.DefaultClientID, Identity: identity, diff --git a/internal/backend/runtime/omni/controllers/omni/cluster_machine_config.go b/internal/backend/runtime/omni/controllers/omni/cluster_machine_config.go index 2ce73277..e9efe7cf 100644 --- a/internal/backend/runtime/omni/controllers/omni/cluster_machine_config.go +++ b/internal/backend/runtime/omni/controllers/omni/cluster_machine_config.go @@ -12,6 +12,7 @@ import ( "errors" "fmt" "slices" + "strings" "text/template" "github.com/cosi-project/runtime/pkg/controller" @@ -52,7 +53,7 @@ const ClusterMachineConfigControllerName = "ClusterMachineConfigController" type ClusterMachineConfigController = qtransform.QController[*omni.ClusterMachine, *omni.ClusterMachineConfig] // NewClusterMachineConfigController initializes ClusterMachineConfigController. -func NewClusterMachineConfigController(imageFactoryHost string, defaultGenOptions []generate.Option, eventSinkPort int) *ClusterMachineConfigController { +func NewClusterMachineConfigController(imageFactoryHost string, registryMirrors []string, eventSinkPort int) *ClusterMachineConfigController { return qtransform.NewQController( qtransform.Settings[*omni.ClusterMachine, *omni.ClusterMachineConfig]{ Name: ClusterMachineConfigControllerName, @@ -63,7 +64,7 @@ func NewClusterMachineConfigController(imageFactoryHost string, defaultGenOption return omni.NewClusterMachine(resources.DefaultNamespace, machineConfig.Metadata().ID()) }, TransformFunc: func(ctx context.Context, r controller.Reader, logger *zap.Logger, clusterMachine *omni.ClusterMachine, machineConfig *omni.ClusterMachineConfig) error { - return reconcileClusterMachineConfig(ctx, r, logger, clusterMachine, machineConfig, defaultGenOptions, eventSinkPort, imageFactoryHost) + return reconcileClusterMachineConfig(ctx, r, logger, clusterMachine, machineConfig, registryMirrors, eventSinkPort, imageFactoryHost) }, }, qtransform.WithExtraMappedInput( @@ -104,7 +105,7 @@ func reconcileClusterMachineConfig( logger *zap.Logger, clusterMachine *omni.ClusterMachine, machineConfig *omni.ClusterMachineConfig, - defaultGenOptions []generate.Option, + registryMirrors []string, eventSinkPort int, imageFactoryHost string, ) error { @@ -247,8 +248,19 @@ func reconcileClusterMachineConfig( imageFactoryHost: imageFactoryHost, } + configGenOptions := make([]generate.Option, 0, len(registryMirrors)) + + for _, registryMirror := range registryMirrors { + hostname, endpoint, ok := strings.Cut(registryMirror, "=") + if !ok { + return fmt.Errorf("invalid registry mirror spec: %q", registryMirror) + } + + configGenOptions = append(configGenOptions, generate.WithRegistryMirror(hostname, endpoint)) + } + data, err := helper.generateConfig(clusterMachine, clusterMachineConfigPatches, secrets, loadBalancerConfig, - cluster, clusterConfigVersion, machineConfigGenOptions, defaultGenOptions, connectionParams, link, eventSinkPort) + cluster, clusterConfigVersion, machineConfigGenOptions, configGenOptions, connectionParams, link, eventSinkPort) if err != nil { machineConfig.TypedSpec().Value.GenerationError = err.Error() diff --git a/internal/backend/runtime/omni/controllers/omni/cluster_machine_config_test.go b/internal/backend/runtime/omni/controllers/omni/cluster_machine_config_test.go index 7d48cb05..62c609c8 100644 --- a/internal/backend/runtime/omni/controllers/omni/cluster_machine_config_test.go +++ b/internal/backend/runtime/omni/controllers/omni/cluster_machine_config_test.go @@ -103,7 +103,7 @@ func (suite *ClusterMachineConfigSuite) TestReconcile() { ) } - newImage := fmt.Sprintf("%s:v1.0.2", conf.Config.TalosRegistry) + newImage := fmt.Sprintf("%s:v1.0.2", conf.Config.Registries.Talos) _, err = safe.StateUpdateWithConflicts(suite.ctx, suite.state, omni.NewClusterMachineConfigPatches(resources.DefaultNamespace, machines[0].Metadata().ID()).Metadata(), func(config *omni.ClusterMachineConfigPatches) error { diff --git a/internal/backend/runtime/omni/controllers/omni/installation_media.go b/internal/backend/runtime/omni/controllers/omni/installation_media.go index af7b8dfa..2a86a6d0 100644 --- a/internal/backend/runtime/omni/controllers/omni/installation_media.go +++ b/internal/backend/runtime/omni/controllers/omni/installation_media.go @@ -396,7 +396,7 @@ func (ctrl *InstallationMediaController) Run(ctx context.Context, r controller.R newMedia.TypedSpec().Value.Name = m.Name newMedia.TypedSpec().Value.Profile = m.Profile newMedia.TypedSpec().Value.ContentType = m.ContentType - newMedia.TypedSpec().Value.DestFilePrefix = fmt.Sprintf("%s-omni-%s", fname.srcPrefix, config.Config.Name) + newMedia.TypedSpec().Value.DestFilePrefix = fmt.Sprintf("%s-omni-%s", fname.srcPrefix, config.Config.Account.Name) newMedia.TypedSpec().Value.Extension = fname.extension newMedia.TypedSpec().Value.NoSecureBoot = m.SBC newMedia.TypedSpec().Value.MinTalosVersion = m.MinTalosVersion diff --git a/internal/backend/runtime/omni/controllers/omni/internal/loadbalancer/loadbalancer.go b/internal/backend/runtime/omni/controllers/omni/internal/loadbalancer/loadbalancer.go index 2ab9fd98..20eb50f7 100644 --- a/internal/backend/runtime/omni/controllers/omni/internal/loadbalancer/loadbalancer.go +++ b/internal/backend/runtime/omni/controllers/omni/internal/loadbalancer/loadbalancer.go @@ -30,12 +30,12 @@ func DefaultNew(bindAddress string, bindPort int, logger *zap.Logger) (LoadBalan bindAddress, bindPort, logger.WithOptions(zap.IncreaseLevel(zap.ErrorLevel)), // silence the load balancer logs - controlplane.WithDialTimeout(config.Config.LoadBalancer.DialTimeout), - controlplane.WithKeepAlivePeriod(config.Config.LoadBalancer.KeepAlivePeriod), - controlplane.WithTCPUserTimeout(config.Config.LoadBalancer.TCPUserTimeout), + controlplane.WithDialTimeout(config.Config.Services.LoadBalancer.DialTimeout), + controlplane.WithKeepAlivePeriod(config.Config.Services.LoadBalancer.KeepAlivePeriod), + controlplane.WithTCPUserTimeout(config.Config.Services.LoadBalancer.TCPUserTimeout), controlplane.WithHealthCheckOptions( - upstream.WithHealthcheckInterval(config.Config.LoadBalancer.HealthCheckInterval), - upstream.WithHealthcheckTimeout(config.Config.LoadBalancer.HealthCheckTimeout), + upstream.WithHealthcheckInterval(config.Config.Services.LoadBalancer.HealthCheckInterval), + upstream.WithHealthcheckTimeout(config.Config.Services.LoadBalancer.HealthCheckTimeout), ), ) } diff --git a/internal/backend/runtime/omni/controllers/omni/kubernetes_status.go b/internal/backend/runtime/omni/controllers/omni/kubernetes_status.go index a480b5a4..1f4d49d1 100644 --- a/internal/backend/runtime/omni/controllers/omni/kubernetes_status.go +++ b/internal/backend/runtime/omni/controllers/omni/kubernetes_status.go @@ -402,7 +402,7 @@ func (ctrl *KubernetesStatusController) startWatcher(ctx context.Context, logger w.podsSync(ctx, notifyCh) }, logger) - if config.Config.WorkloadProxying.Enabled { + if config.Config.Services.WorkloadProxy.Enabled { w.serviceFactory = informers.NewSharedInformerFactory(w.client.Clientset(), 0) if _, err = w.serviceFactory.Core().V1().Services().Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ diff --git a/internal/backend/runtime/omni/controllers/omni/versions.go b/internal/backend/runtime/omni/controllers/omni/versions.go index 00726500..d4ca7f5d 100644 --- a/internal/backend/runtime/omni/controllers/omni/versions.go +++ b/internal/backend/runtime/omni/controllers/omni/versions.go @@ -239,12 +239,12 @@ func forAllCompatibleVersions( func (ctrl *VersionsController) reconcileTalosVersions(ctx context.Context, r controller.Runtime, k8sVersions []*compatibility.KubernetesVersion, logger *zap.Logger) error { tracker := trackResource(r, resources.DefaultNamespace, omni.TalosVersionType) - allVersions, err := ctrl.fetchTalosVersions(ctx, config.Config.ImageFactoryBaseURL) + allVersions, err := ctrl.fetchTalosVersions(ctx, config.Config.Registries.ImageFactoryBaseURL) if err != nil { return err } - talosVersions := ctrl.getVersionsAfter(allVersions, minDiscoveredTalosVersion, config.Config.EnableTalosPreReleaseVersions) + talosVersions := ctrl.getVersionsAfter(allVersions, minDiscoveredTalosVersion, config.Config.Features.EnableTalosPreReleaseVersions) talosVersions = xslices.FilterInPlace(talosVersions, func(v string) bool { return consts.DenylistedTalosVersions.IsAllowed(v) @@ -286,7 +286,7 @@ func (ctrl *VersionsController) reconcileTalosVersions(ctx context.Context, r co func (ctrl *VersionsController) reconcileKubernetesVersions(ctx context.Context, r controller.Runtime, logger *zap.Logger) ([]string, error) { tracker := trackResource(r, resources.DefaultNamespace, omni.KubernetesVersionType) - allVersions, err := ctrl.fetchVersionsFromRegistry(ctx, config.Config.KubernetesRegistry) + allVersions, err := ctrl.fetchVersionsFromRegistry(ctx, config.Config.Registries.Kubernetes) if err != nil { return nil, err } diff --git a/internal/backend/runtime/omni/export_test.go b/internal/backend/runtime/omni/export_test.go index 3baddd57..24f99c06 100644 --- a/internal/backend/runtime/omni/export_test.go +++ b/internal/backend/runtime/omni/export_test.go @@ -30,7 +30,7 @@ func EtcdElections(ctx context.Context, client *clientv3.Client, electionKey str return etcdElections(ctx, client, electionKey, logger, f) } -func ClusterValidationOptions(st state.State, etcdBackupConfig config.EtcdBackupParams, embeddedDiscoveryServiceConfig config.EmbeddedDiscoveryServiceParams) []validated.StateOption { +func ClusterValidationOptions(st state.State, etcdBackupConfig config.EtcdBackup, embeddedDiscoveryServiceConfig config.EmbeddedDiscoveryService) []validated.StateOption { return clusterValidationOptions(st, etcdBackupConfig, embeddedDiscoveryServiceConfig) } @@ -50,7 +50,7 @@ func MachineClassValidationOptions(st state.State) []validated.StateOption { return machineClassValidationOptions(st) } -func IdentityValidationOptions(samlConfig config.SAMLParams) []validated.StateOption { +func IdentityValidationOptions(samlConfig config.SAML) []validated.StateOption { return identityValidationOptions(samlConfig) } diff --git a/internal/backend/runtime/omni/loader.go b/internal/backend/runtime/omni/loader.go index adc478fc..b7856e77 100644 --- a/internal/backend/runtime/omni/loader.go +++ b/internal/backend/runtime/omni/loader.go @@ -18,6 +18,7 @@ import ( "go.uber.org/zap" "github.com/siderolabs/omni/internal/backend/runtime/keyprovider" + "github.com/siderolabs/omni/internal/pkg/config" ) // Loader is an interface that returns a private key. @@ -102,12 +103,19 @@ func makeVaultHTTPLoader(source string, logger *zap.Logger) (Loader, error) { token, ok := os.LookupEnv("VAULT_TOKEN") if !ok { - return nil, errors.New("VAULT_TOKEN is not set") + token = config.Config.Storage.Vault.Token + + if token == "" { + return nil, errors.New("VAULT_TOKEN is not set") + } } addr, ok := os.LookupEnv("VAULT_ADDR") if !ok { - return nil, errors.New("VAULT_ADDR is not set") + addr = config.Config.Storage.Vault.URL + if addr == "" { + return nil, errors.New("VAULT_ADDR is not set") + } } return &VaultHTTPLoader{ diff --git a/internal/backend/runtime/omni/migration/migration_test.go b/internal/backend/runtime/omni/migration/migration_test.go index e136685f..22718e91 100644 --- a/internal/backend/runtime/omni/migration/migration_test.go +++ b/internal/backend/runtime/omni/migration/migration_test.go @@ -481,7 +481,7 @@ func (suite *MigrationSuite) TestUpdateConfigPatchLabels() { ctx := suite.T().Context() cluster := omni.NewCluster(resources.DefaultNamespace, "cluster") - cluster.TypedSpec().Value.InstallImage = fmt.Sprintf("%s:v%s", config.Config.TalosRegistry, constants.DefaultTalosVersion) //nolint:staticcheck + cluster.TypedSpec().Value.InstallImage = fmt.Sprintf("%s:v%s", config.Config.Registries.Talos, constants.DefaultTalosVersion) //nolint:staticcheck machineSet := omni.NewMachineSet(resources.DefaultNamespace, "machine-set") machineSet.Metadata().Labels().Set("cluster", cluster.Metadata().ID()) diff --git a/internal/backend/runtime/omni/omni.go b/internal/backend/runtime/omni/omni.go index 9b88a786..6518aea3 100644 --- a/internal/backend/runtime/omni/omni.go +++ b/internal/backend/runtime/omni/omni.go @@ -93,7 +93,7 @@ func New(talosClientFactory *talos.ClientFactory, dnsService *dns.Service, workl ) (*Runtime, error) { var opts []options.Option - if !config.Config.DisableControllerRuntimeCache { + if !config.Config.Features.DisableControllerRuntimeCache { opts = append(opts, safe.WithResourceCache[*omni.BackupData](), safe.WithResourceCache[*omni.ConfigPatch](), @@ -215,7 +215,7 @@ func New(talosClientFactory *talos.ClientFactory, dnsService *dns.Service, workl TalosClientFactory: talosClientFactory, NodeResolver: dnsService, }), - omnictrl.NewKubernetesStatusController(config.Config.APIURL, config.Config.WorkloadProxying.Subdomain), + omnictrl.NewKubernetesStatusController(config.Config.Services.API.URL(), config.Config.Services.WorkloadProxy.Subdomain), &omnictrl.LoadBalancerController{}, &omnictrl.MachineSetNodeController{}, &omnictrl.MachineSetDestroyStatusController{}, @@ -224,12 +224,12 @@ func New(talosClientFactory *talos.ClientFactory, dnsService *dns.Service, workl &omnictrl.MachineStatusMetricsController{}, &omnictrl.VersionsController{}, omnictrl.NewClusterLoadBalancerController( - config.Config.LoadBalancer.MinPort, - config.Config.LoadBalancer.MaxPort, + config.Config.Services.LoadBalancer.MinPort, + config.Config.Services.LoadBalancer.MaxPort, ), &omnictrl.InstallationMediaController{}, omnictrl.NewKeyPrunerController( - config.Config.KeyPruner.Interval, + config.Config.Auth.KeyPruner.Interval, ), &omnictrl.OngoingTaskController{}, omnictrl.NewMachineRequestStatusCleanupController(), @@ -247,18 +247,22 @@ func New(talosClientFactory *talos.ClientFactory, dnsService *dns.Service, workl omnictrl.NewClusterConfigVersionController(), omnictrl.NewClusterEndpointController(), omnictrl.NewClusterKubernetesNodesController(), - omnictrl.NewClusterMachineConfigController(imageFactoryHost, config.Config.DefaultConfigGenOptions, config.Config.EventSinkPort), + omnictrl.NewClusterMachineConfigController( + imageFactoryHost, + config.Config.Registries.Mirrors, + config.Config.Services.Siderolink.EventSinkPort, + ), omnictrl.NewClusterMachineTeardownController(), omnictrl.NewMachineConfigGenOptionsController(), omnictrl.NewMachineStatusController(imageFactoryClient), omnictrl.NewClusterMachineConfigStatusController(imageFactoryHost), omnictrl.NewClusterMachineEncryptionKeyController(), omnictrl.NewClusterMachineStatusController(), - omnictrl.NewClusterStatusController(config.Config.EmbeddedDiscoveryService.Enabled), + omnictrl.NewClusterStatusController(config.Config.Services.EmbeddedDiscoveryService.Enabled), omnictrl.NewClusterDiagnosticsController(), omnictrl.NewClusterUUIDController(), omnictrl.NewControlPlaneStatusController(), - omnictrl.NewDiscoveryServiceConfigPatchController(config.Config.EmbeddedDiscoveryService.Port), + omnictrl.NewDiscoveryServiceConfigPatchController(config.Config.Services.EmbeddedDiscoveryService.Port), omnictrl.NewKubernetesNodeAuditController(nil, time.Minute), omnictrl.NewEtcdBackupEncryptionController(), omnictrl.NewClusterWorkloadProxyStatusController(workloadProxyReconciler), @@ -288,7 +292,7 @@ func New(talosClientFactory *talos.ClientFactory, dnsService *dns.Service, workl omnictrl.NewLinkStatusController[*siderolinkresources.Link](peers), omnictrl.NewLinkStatusController[*siderolinkresources.PendingMachine](peers), omnictrl.NewPendingMachineStatusController(), - omnictrl.NewMaintenanceConfigStatusController(nil, siderolink.ListenHost, config.Config.EventSinkPort, config.Config.LogServerPort), + omnictrl.NewMaintenanceConfigStatusController(nil, siderolink.ListenHost, config.Config.Services.Siderolink.EventSinkPort, config.Config.Services.Siderolink.LogServerPort), omnictrl.NewDiscoveryAffiliateDeleteTaskController(clockwork.NewRealClock(), discoveryClientCache), omnictrl.NewInfraProviderCombinedStatusController(), omnictrl.NewServiceAccountStatusController(), @@ -300,7 +304,7 @@ func New(talosClientFactory *talos.ClientFactory, dnsService *dns.Service, workl ) } - if config.Config.EnableStripeReporting { + if config.Config.Logs.Stripe.Enabled { stripeAPIKey, ok := os.LookupEnv("STRIPE_API_KEY") if !ok { return nil, fmt.Errorf("environment variable STRIPE_API_KEY is not set") @@ -359,7 +363,7 @@ func New(talosClientFactory *talos.ClientFactory, dnsService *dns.Service, workl metricsRegistry.MustRegister(expvarCollector) validationOptions := slices.Concat( - clusterValidationOptions(resourceState, config.Config.EtcdBackup, config.Config.EmbeddedDiscoveryService), + clusterValidationOptions(resourceState, config.Config.EtcdBackup, config.Config.Services.EmbeddedDiscoveryService), relationLabelsValidationOptions(), accessPolicyValidationOptions(), authorizationValidationOptions(resourceState), diff --git a/internal/backend/runtime/omni/state.go b/internal/backend/runtime/omni/state.go index 8b79670a..940f6c3f 100644 --- a/internal/backend/runtime/omni/state.go +++ b/internal/backend/runtime/omni/state.go @@ -66,19 +66,19 @@ func NewState(ctx context.Context, params *config.Params, logger *zap.Logger, me ) } - switch params.Storage.Kind { + switch params.Storage.Default.Kind { case "boltdb": - return buildBoltPersistentState(ctx, params.Storage.Boltdb.Path, logger, stateFunc) + return buildBoltPersistentState(ctx, params.Storage.Default.Boltdb.Path, logger, stateFunc) case "etcd": return buildEtcdPersistentState(ctx, params, logger, stateFunc) default: - return fmt.Errorf("unknown storage kind %q", params.Storage.Kind) + return fmt.Errorf("unknown storage kind %q", params.Storage.Default.Kind) } } func newNamespacedState(params *config.Params, primaryStorageCoreState state.CoreState, virtualState *virtual.State, logger *zap.Logger) (*namespaced.State, func(), error) { secondaryStorageCoreState, secondaryStorageBackingStore, err := newBoltPersistentState( - params.SecondaryStorage.Path, &bbolt.Options{ + params.Storage.Secondary.Path, &bbolt.Options{ NoSync: true, // we do not need fsync for the secondary storage }, true, logger) if err != nil { @@ -166,7 +166,7 @@ func initResources(ctx context.Context, resourceState state.State, logger *zap.L sysVersion := system.NewSysVersion(resources.EphemeralNamespace, system.SysVersionID) sysVersion.TypedSpec().Value.BackendVersion = version.Tag - sysVersion.TypedSpec().Value.InstanceName = config.Config.Name + sysVersion.TypedSpec().Value.InstanceName = config.Config.Account.Name sysVersion.TypedSpec().Value.BackendApiVersion = version.API if err := resourceState.Create(ctx, sysVersion); err != nil { @@ -190,22 +190,22 @@ func stateWithMetrics(namespacedState *namespaced.State, metricsRegistry prometh // NewAuditWrap creates a new audit wrap. func NewAuditWrap(resState state.State, params *config.Params, logger *zap.Logger) (*AuditWrap, error) { - if params.AuditLogDir == "" { + if params.Logs.Audit.Path == "" { logger.Info("audit log disabled") return &AuditWrap{state: resState}, nil } - logger.Info("audit log enabled", zap.String("dir", params.AuditLogDir)) + logger.Info("audit log enabled", zap.String("dir", params.Logs.Audit.Path)) - a, err := audit.NewLog(params.AuditLogDir, logger) + a, err := audit.NewLog(params.Logs.Audit.Path, logger) if err != nil { return nil, err } hooks.Init(a) - return &AuditWrap{state: resState, log: a, dir: params.AuditLogDir}, nil + return &AuditWrap{state: resState, log: a, dir: params.Logs.Audit.Path}, nil } // AuditWrap is builder/wrapper for creating logged access to Omni and Talos nodes. diff --git a/internal/backend/runtime/omni/state_etcd.go b/internal/backend/runtime/omni/state_etcd.go index c6338373..d25820c9 100644 --- a/internal/backend/runtime/omni/state_etcd.go +++ b/internal/backend/runtime/omni/state_etcd.go @@ -42,16 +42,16 @@ import ( const compressionThresholdBytes = 2048 func buildEtcdPersistentState(ctx context.Context, params *config.Params, logger *zap.Logger, f func(context.Context, namespaced.StateBuilder) error) error { - prefix := fmt.Sprintf("/omni/%s", url.PathEscape(params.AccountID)) + prefix := fmt.Sprintf("/omni/%s", url.PathEscape(params.Account.ID)) - return getEtcdClient(ctx, ¶ms.Storage.Etcd, logger, func(ctx context.Context, etcdClient *clientv3.Client) error { + return getEtcdClient(ctx, ¶ms.Storage.Default.Etcd, logger, func(ctx context.Context, etcdClient *clientv3.Client) error { return etcdElections(ctx, etcdClient, prefix, logger, func(ctx context.Context, _ *clientv3.Client) error { - cipher, err := makeCipher(params.AccountID, params.Storage.Etcd, etcdClient, logger) //nolint:contextcheck + cipher, err := makeCipher(params.Account.ID, params.Storage.Default.Etcd, etcdClient, logger) //nolint:contextcheck if err != nil { return err } - salt := sha256.Sum256([]byte(params.AccountID)) + salt := sha256.Sum256([]byte(params.Account.ID)) etcdState := etcd.NewState( etcdClient, @@ -261,15 +261,15 @@ func getExternalEtcdClient(ctx context.Context, params *config.EtcdParams, logge logger.Info("starting etcd client", zap.Strings("endpoints", params.Endpoints), - zap.String("cert_path", params.CertPath), - zap.String("key_path", params.KeyPath), - zap.String("ca_path", params.CAPath), + zap.String("cert_path", params.CertFile), + zap.String("key_path", params.KeyFile), + zap.String("ca_path", params.CAFile), ) tlsInfo := transport.TLSInfo{ - CertFile: params.CertPath, - KeyFile: params.KeyPath, - TrustedCAFile: params.CAPath, + CertFile: params.CertFile, + KeyFile: params.KeyFile, + TrustedCAFile: params.CAFile, } tlsConfig, err := tlsInfo.ClientConfig() diff --git a/internal/backend/runtime/omni/state_etcd_test.go b/internal/backend/runtime/omni/state_etcd_test.go index 5bb37a29..1458a635 100644 --- a/internal/backend/runtime/omni/state_etcd_test.go +++ b/internal/backend/runtime/omni/state_etcd_test.go @@ -123,14 +123,18 @@ func TestEtcdInitialization(t *testing.T) { for _, step := range steps { res := t.Run(step.name, func(t *testing.T) { err := omniruntime.BuildEtcdPersistentState(t.Context(), &config.Params{ - Name: "instance-name", - Storage: config.StorageParams{ - Etcd: config.EtcdParams{ - Embedded: true, - EmbeddedDBPath: etcdDir, - PrivateKeySource: step.args.privateKeySource, - PublicKeyFiles: step.args.publicKeyFiles, - Endpoints: []string{"http://localhost:0"}, + Account: config.Account{ + Name: "instance-name", + }, + Storage: config.Storage{ + Default: config.StorageDefault{ + Etcd: config.EtcdParams{ + Embedded: true, + EmbeddedDBPath: etcdDir, + PrivateKeySource: step.args.privateKeySource, + PublicKeyFiles: step.args.publicKeyFiles, + Endpoints: []string{"http://localhost:0"}, + }, }, }, }, zaptest.NewLogger(t), func(context.Context, namespaced.StateBuilder) error { @@ -190,14 +194,18 @@ func TestEncryptDecrypt(t *testing.T) { for _, step := range steps { res := t.Run(step.name, func(t *testing.T) { err := omniruntime.BuildEtcdPersistentState(t.Context(), &config.Params{ - Name: "instance-name", - Storage: config.StorageParams{ - Etcd: config.EtcdParams{ - Embedded: true, - EmbeddedDBPath: etcdDir, - PrivateKeySource: step.args.privateKeySource, - PublicKeyFiles: step.args.publicKeyFiles, - Endpoints: []string{"http://localhost:0"}, + Account: config.Account{ + Name: "instance-name", + }, + Storage: config.Storage{ + Default: config.StorageDefault{ + Etcd: config.EtcdParams{ + Embedded: true, + EmbeddedDBPath: etcdDir, + PrivateKeySource: step.args.privateKeySource, + PublicKeyFiles: step.args.publicKeyFiles, + Endpoints: []string{"http://localhost:0"}, + }, }, }, }, zaptest.NewLogger(t), diff --git a/internal/backend/runtime/omni/state_validation.go b/internal/backend/runtime/omni/state_validation.go index 6f123739..a2c2ad0c 100644 --- a/internal/backend/runtime/omni/state_validation.go +++ b/internal/backend/runtime/omni/state_validation.go @@ -42,7 +42,7 @@ import ( // Validation is only syntactic - they are checked whether they are valid semver strings. // //nolint:gocognit,gocyclo,cyclop -func clusterValidationOptions(st state.State, etcdBackupConfig config.EtcdBackupParams, embeddedDiscoveryServiceConfig config.EmbeddedDiscoveryServiceParams) []validated.StateOption { +func clusterValidationOptions(st state.State, etcdBackupConfig config.EtcdBackup, embeddedDiscoveryServiceConfig config.EmbeddedDiscoveryService) []validated.StateOption { validateVersions := func(ctx context.Context, existingRes *omni.Cluster, res *omni.Cluster, skipTalosVersion, skipKubernetesVersion bool) error { if skipTalosVersion && skipKubernetesVersion { return nil @@ -710,7 +710,7 @@ func hasUppercaseLetters(s string) bool { return false } -func identityValidationOptions(samlConfig config.SAMLParams) []validated.StateOption { +func identityValidationOptions(samlConfig config.SAML) []validated.StateOption { return []validated.StateOption{ validated.WithCreateValidations(validated.NewCreateValidationForType(func(ctx context.Context, res *authres.Identity, _ ...state.CreateOption) error { var errs error diff --git a/internal/backend/runtime/omni/state_validation_test.go b/internal/backend/runtime/omni/state_validation_test.go index 65825df4..c5cc77ca 100644 --- a/internal/backend/runtime/omni/state_validation_test.go +++ b/internal/backend/runtime/omni/state_validation_test.go @@ -51,14 +51,14 @@ func TestClusterValidation(t *testing.T) { talos15 := "1.5.0" - etcdBackupConfig := config.EtcdBackupParams{ + etcdBackupConfig := config.EtcdBackup{ TickInterval: time.Minute, MinInterval: time.Hour, MaxInterval: 24 * time.Hour, } innerSt := state.WrapCore(namespaced.NewState(inmem.Build)) - st := validated.NewState(innerSt, omni.ClusterValidationOptions(state.WrapCore(innerSt), etcdBackupConfig, config.EmbeddedDiscoveryServiceParams{})...) + st := validated.NewState(innerSt, omni.ClusterValidationOptions(state.WrapCore(innerSt), etcdBackupConfig, config.EmbeddedDiscoveryService{})...) talosVersion1 := omnires.NewTalosVersion(resources.DefaultNamespace, "1.4.0") talosVersion1.TypedSpec().Value.CompatibleKubernetesVersions = []string{"1.27.0", "1.27.1"} @@ -172,9 +172,9 @@ func TestClusterUseEmbeddedDiscoveryServiceValidation(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second) t.Cleanup(cancel) - buildState := func(conf config.EmbeddedDiscoveryServiceParams) (inner, outer state.State) { + buildState := func(conf config.EmbeddedDiscoveryService) (inner, outer state.State) { innerSt := state.WrapCore(namespaced.NewState(inmem.Build)) - st := validated.NewState(innerSt, omni.ClusterValidationOptions(state.WrapCore(innerSt), config.EtcdBackupParams{}, conf)...) + st := validated.NewState(innerSt, omni.ClusterValidationOptions(state.WrapCore(innerSt), config.EtcdBackup{}, conf)...) return innerSt, state.WrapCore(st) } @@ -182,7 +182,7 @@ func TestClusterUseEmbeddedDiscoveryServiceValidation(t *testing.T) { t.Run("disabled instance-wide - create", func(t *testing.T) { t.Parallel() - _, st := buildState(config.EmbeddedDiscoveryServiceParams{ + _, st := buildState(config.EmbeddedDiscoveryService{ Enabled: false, }) @@ -202,7 +202,7 @@ func TestClusterUseEmbeddedDiscoveryServiceValidation(t *testing.T) { t.Parallel() // prepare a cluster which has the feature enabled, while it is disabled instance-wide - innerSt, st := buildState(config.EmbeddedDiscoveryServiceParams{ + innerSt, st := buildState(config.EmbeddedDiscoveryService{ Enabled: false, }) @@ -231,7 +231,7 @@ func TestClusterUseEmbeddedDiscoveryServiceValidation(t *testing.T) { t.Run("enabled instance-wide", func(t *testing.T) { t.Parallel() - _, st := buildState(config.EmbeddedDiscoveryServiceParams{ + _, st := buildState(config.EmbeddedDiscoveryService{ Enabled: true, }) @@ -580,7 +580,7 @@ func TestIdentitySAMLValidation(t *testing.T) { t.Cleanup(cancel) innerSt := state.WrapCore(namespaced.NewState(inmem.Build)) - st := validated.NewState(innerSt, omni.IdentityValidationOptions(config.SAMLParams{ + st := validated.NewState(innerSt, omni.IdentityValidationOptions(config.SAML{ Enabled: true, })...) @@ -633,7 +633,7 @@ func TestCreateIdentityValidation(t *testing.T) { t.Cleanup(cancel) innerSt := state.WrapCore(namespaced.NewState(inmem.Build)) - st := validated.NewState(innerSt, omni.IdentityValidationOptions(config.SAMLParams{})...) + st := validated.NewState(innerSt, omni.IdentityValidationOptions(config.SAML{})...) assert := assert.New(t) diff --git a/internal/backend/runtime/omni/virtual/state.go b/internal/backend/runtime/omni/virtual/state.go index de6f70b1..2dfbfdbd 100644 --- a/internal/backend/runtime/omni/virtual/state.go +++ b/internal/backend/runtime/omni/virtual/state.go @@ -305,7 +305,7 @@ func (v *State) advertisedEndpoints(_ context.Context, ptr resource.Pointer) (*v res := virtual.NewAdvertisedEndpoints() - res.TypedSpec().Value.GrpcApiUrl = config.Config.APIURL + res.TypedSpec().Value.GrpcApiUrl = config.Config.Services.API.URL() return res, nil } diff --git a/internal/backend/runtime/talos/talos.go b/internal/backend/runtime/talos/talos.go index 7e0e1127..6de0f3f9 100644 --- a/internal/backend/runtime/talos/talos.go +++ b/internal/backend/runtime/talos/talos.go @@ -199,8 +199,8 @@ func (r *Runtime) GetTalosconfigRaw(context *common.Context, identity string) ([ Identity: identity, } - contextName := config.Config.Name - apiURL := config.Config.APIURL + contextName := config.Config.Account.Name + apiURL := config.Config.Services.API.URL() cluster := "" diff --git a/internal/backend/saml/saml.go b/internal/backend/saml/saml.go index 945fdf56..bd2af73e 100644 --- a/internal/backend/saml/saml.go +++ b/internal/backend/saml/saml.go @@ -32,7 +32,7 @@ func NewHandler(state state.State, cfg *specs.AuthConfigSpec_SAML, logger *zap.L return nil, err } - rootURL, err := url.Parse(config.Config.APIURL) + rootURL, err := url.Parse(config.Config.Services.API.URL()) if err != nil { return nil, err } diff --git a/internal/backend/server.go b/internal/backend/server.go index e79b1671..417fad72 100644 --- a/internal/backend/server.go +++ b/internal/backend/server.go @@ -9,7 +9,6 @@ package backend import ( "compress/gzip" "context" - "crypto/tls" "encoding/json" "errors" "fmt" @@ -22,7 +21,6 @@ import ( "os" "strconv" "strings" - "sync" "syscall" "time" @@ -35,7 +33,6 @@ import ( "github.com/cosi-project/runtime/pkg/state" protobufserver "github.com/cosi-project/runtime/pkg/state/protobuf/server" "github.com/crewjam/saml/samlsp" - "github.com/fsnotify/fsnotify" grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" grpc_zap "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap" grpc_recovery "github.com/grpc-ecosystem/go-grpc-middleware/recovery" @@ -63,7 +60,6 @@ import ( "github.com/siderolabs/omni/client/pkg/omni/resources" authres "github.com/siderolabs/omni/client/pkg/omni/resources/auth" omnires "github.com/siderolabs/omni/client/pkg/omni/resources/omni" - "github.com/siderolabs/omni/client/pkg/panichandler" "github.com/siderolabs/omni/internal/backend/debug" "github.com/siderolabs/omni/internal/backend/dns" "github.com/siderolabs/omni/internal/backend/factory" @@ -80,6 +76,7 @@ import ( "github.com/siderolabs/omni/internal/backend/runtime/omni" "github.com/siderolabs/omni/internal/backend/runtime/talos" "github.com/siderolabs/omni/internal/backend/saml" + "github.com/siderolabs/omni/internal/backend/services" "github.com/siderolabs/omni/internal/backend/workloadproxy" "github.com/siderolabs/omni/internal/frontend" "github.com/siderolabs/omni/internal/memconn" @@ -116,19 +113,16 @@ type Server struct { siderolinkEventsCh chan<- *omnires.MachineStatusSnapshot installEventCh chan<- resource.ID - proxyServer Proxy - bindAddress string - metricsBindAddress string - pprofBindAddress string - k8sProxyBindAddress string - keyFile string - certFile string - workloadProxyKey []byte + pprofBindAddress string + apiService config.Service + metricsService config.Service + devServerProxy config.DevServerProxyService + k8sProxyService config.KubernetesProxyService + workloadProxyKey []byte } // NewServer creates new HTTP server. func NewServer( - bindAddress, metricsBindAddress, k8sProxyBindAddress, pprofBindAddress string, dnsService *dns.Service, workloadProxyReconciler *workloadproxy.Reconciler, imageFactoryClient *imagefactory.Client, @@ -139,8 +133,6 @@ func NewServer( talosRuntime *talos.Runtime, logHandler *siderolink.LogHandler, authConfig *authres.Config, - keyFile, certFile string, - proxyServer Proxy, auditor Auditor, logger *zap.Logger, ) (*Server, error) { @@ -156,13 +148,11 @@ func NewServer( linkCounterDeltaCh: linkCounterDeltaCh, siderolinkEventsCh: siderolinkEventsCh, installEventCh: installEventCh, - proxyServer: proxyServer, - bindAddress: bindAddress, - metricsBindAddress: metricsBindAddress, - pprofBindAddress: pprofBindAddress, - k8sProxyBindAddress: k8sProxyBindAddress, - keyFile: keyFile, - certFile: certFile, + devServerProxy: config.Config.Services.DevServerProxy, + apiService: config.Config.Services.API, + metricsService: config.Config.Services.Metrics, + pprofBindAddress: config.Config.Debug.Pprof.Endpoint, + k8sProxyService: config.Config.Services.KubernetesProxy, } k8sruntime, err := kubernetes.New(omniRuntime.State()) @@ -237,28 +227,22 @@ func (s *Server) Run(ctx context.Context) error { return err } - var crtData *certData - - if s.certFile != "" && s.keyFile != "" { - crtData = &certData{certFile: s.certFile, keyFile: s.keyFile} - } - workloadProxyHandler, err := s.workloadProxyHandler(mux) if err != nil { return fmt.Errorf("failed to create workload proxy handler: %w", err) } - apiSrv := s.makeAPIServer(workloadProxyHandler, proxyServer, crtData) + apiSrv := s.makeAPIServer(workloadProxyHandler, proxyServer) fns := []func() error{ func() error { return proxyServer.Serve(ctx, gtwyDialsTo) }, func() error { return actualSrv.Serve(ctx, prxDialsTo) }, func() error { return apiSrv.Run(ctx) }, - func() error { return runMetricsServer(ctx, s.metricsBindAddress, s.logger) }, + func() error { return s.runMetricsServer(ctx) }, func() error { - return runK8sProxyServer(ctx, s.k8sProxyBindAddress, oidcStorage, crtData, s.omniRuntime.State(), s.auditor, s.logger) + return s.runK8sProxyServer(ctx, oidcStorage) }, - func() error { return s.proxyServer.Run(ctx, apiSrv.Handler(), s.logger) }, + func() error { return s.runDevProxyServer(ctx, apiSrv.Handler()) }, func() error { return s.logHandler.Start(ctx) }, func() error { return s.runMachineAPI(ctx) }, func() error { return s.auditor.RunCleanup(ctx) }, @@ -272,11 +256,13 @@ func (s *Server) Run(ctx context.Context) error { eg.Go(fn) } - if err = runLocalResourceServer(ctx, runtimeState, serverOptions, eg, s.logger); err != nil { - return fmt.Errorf("failed to run local resource server: %w", err) + if config.Config.Services.LocalResourceService.Enabled { + if err = runLocalResourceServer(ctx, runtimeState, serverOptions, eg, s.logger); err != nil { + return fmt.Errorf("failed to run local resource server: %w", err) + } } - if config.Config.EmbeddedDiscoveryService.Enabled { + if config.Config.Services.EmbeddedDiscoveryService.Enabled { eg.Go(func() error { if err = runEmbeddedDiscoveryService(ctx, s.logger); err != nil { return fmt.Errorf("failed to run discovery server over Siderolink: %w", err) @@ -286,7 +272,7 @@ func (s *Server) Run(ctx context.Context) error { }) } - if config.Config.InitialServiceAccount.Enabled { + if config.Config.Auth.InitialServiceAccount.Enabled { if err = s.createInitialServiceAccount(ctx); err != nil { return err } @@ -298,10 +284,11 @@ func (s *Server) Run(ctx context.Context) error { func (s *Server) makeMux(oidcProvider *oidc.Provider) (*http.ServeMux, error) { imageFactoryHandler := handler.NewAuthConfig( handler.NewSignature( - &factory.Handler{ - State: s.omniRuntime.State(), - Logger: s.logger.With(logging.Component("factory_proxy")), - }, + factory.NewHandler( + s.omniRuntime.State(), + s.logger.With(logging.Component("factory_proxy")), + &config.Config.Registries, + ), s.authenticatorFunc(), s.logger, ), @@ -557,15 +544,15 @@ func (s *Server) authenticatorFunc() auth.AuthenticatorFunc { } func (s *Server) runMachineAPI(ctx context.Context) error { - wgAddress := config.Config.SiderolinkWireguardBindAddress + wgAddress := config.Config.Services.Siderolink.WireGuard.BindEndpoint params := siderolink.Params{ WireguardEndpoint: wgAddress, - AdvertisedEndpoint: config.Config.SiderolinkWireguardAdvertisedAddress, - APIEndpoint: config.Config.MachineAPIBindAddress, - Cert: config.Config.MachineAPICertFile, - Key: config.Config.MachineAPIKeyFile, - EventSinkPort: strconv.Itoa(config.Config.EventSinkPort), + AdvertisedEndpoint: config.Config.Services.Siderolink.WireGuard.AdvertisedEndpoint, + MachineAPIEndpoint: config.Config.Services.MachineAPI.BindEndpoint, + MachineAPITLSCert: config.Config.Services.MachineAPI.CertFile, + MachineAPITLSKey: config.Config.Services.MachineAPI.KeyFile, + EventSinkPort: strconv.Itoa(config.Config.Services.Siderolink.EventSinkPort), } omniState := s.omniRuntime.State() @@ -614,9 +601,9 @@ func (s *Server) runMachineAPI(ctx context.Context) error { eg.Go(func() error { return slink.Run(groupCtx, siderolink.ListenHost, - strconv.Itoa(config.Config.EventSinkPort), + strconv.Itoa(config.Config.Services.Siderolink.EventSinkPort), strconv.Itoa(talosconstants.TrustdPort), - strconv.Itoa(config.Config.LogServerPort), + strconv.Itoa(config.Config.Services.Siderolink.LogServerPort), ) }) @@ -637,7 +624,7 @@ func (s *Server) workloadProxyHandler(next http.Handler) (http.Handler, error) { return nil, fmt.Errorf("failed to create pgp signature validator: %w", err) } - mainURL, err := url.Parse(config.Config.APIURL) + mainURL, err := url.Parse(config.Config.Services.API.URL()) if err != nil { return nil, fmt.Errorf("failed to parse API URL: %w", err) } @@ -647,15 +634,15 @@ func (s *Server) workloadProxyHandler(next http.Handler) (http.Handler, error) { s.workloadProxyReconciler, pgpSignatureValidator, mainURL, - config.Config.WorkloadProxying.Subdomain, + config.Config.Services.WorkloadProxy.Subdomain, s.logger.With(logging.Component("workload_proxy_handler")), s.workloadProxyKey, ) } -func (s *Server) makeAPIServer(regular http.Handler, grpcServer *grpcServer, data *certData) *apiServer { +func (s *Server) makeAPIServer(regular http.Handler, grpcServer *grpcServer) *apiServer { wrap := func(fn func(w http.ResponseWriter, req *http.Request)) http.Handler { - if data != nil { + if s.apiService.IsSecure() { return http.HandlerFunc(fn) } @@ -663,40 +650,40 @@ func (s *Server) makeAPIServer(regular http.Handler, grpcServer *grpcServer, dat return h2c.NewHandler(http.HandlerFunc(fn), &http2.Server{}) } - srv := &http.Server{ - Addr: s.bindAddress, - Handler: wrap(func(w http.ResponseWriter, req *http.Request) { - if req.ProtoMajor == 2 && strings.HasPrefix( - req.Header.Get("Content-Type"), "application/grpc") { - // grpcServer provides top-level gRPC proxy handler. - grpcServer.ServeHTTP(w, setRealIPRequest(req)) + handler := wrap(func(w http.ResponseWriter, req *http.Request) { + if req.ProtoMajor == 2 && strings.HasPrefix( + req.Header.Get("Content-Type"), "application/grpc") { + // grpcServer provides top-level gRPC proxy handler. + grpcServer.ServeHTTP(w, setRealIPRequest(req)) - return - } + return + } - // handler contains "regular" HTTP handlers - regular.ServeHTTP(w, req) - }), - } + // handler contains "regular" HTTP handlers + regular.ServeHTTP(w, req) + }) return &apiServer{ - srv: srv, - cert: data, - logger: s.logger.With(zap.String("server", s.bindAddress), zap.String("server_type", "api")), + srv: services.NewFromConfig( + &s.apiService, + handler, + ), + handler: handler, + logger: s.logger.With(zap.String("server", s.apiService.BindEndpoint), zap.String("server_type", "api")), } } type apiServer struct { - srv *http.Server - cert *certData - logger *zap.Logger + srv *services.Server + handler http.Handler + logger *zap.Logger } func (s *apiServer) Run(ctx context.Context) error { - return (&server{server: s.srv, certData: s.cert}).Run(ctx, s.logger) + return s.srv.Run(ctx, s.logger) } -func (s *apiServer) Handler() http.Handler { return s.srv.Handler } +func (s *apiServer) Handler() http.Handler { return s.handler } func recoveryHandler(logger *zap.Logger) grpc_recovery.RecoveryHandlerFunc { return func(p any) error { @@ -865,33 +852,32 @@ func getOmnictlDownloads(dir string) (http.Handler, error) { return http.FileServer(http.Dir(dir)), nil } -func runMetricsServer(ctx context.Context, bindAddress string, logger *zap.Logger) error { +func (s *Server) runDevProxyServer(ctx context.Context, next http.Handler) error { + handler, err := services.NewFrontendHandler(s.devServerProxy.ProxyTo, s.logger) + if err != nil { + return fmt.Errorf("failed to set up frontend handler: %w", err) + } + + return services.NewProxy(s.devServerProxy, handler).Run(ctx, next, s.logger) +} + +func (s *Server) runMetricsServer(ctx context.Context) error { var metricsMux http.ServeMux metricsMux.Handle("/metrics", promhttp.Handler()) - metricsServer := &http.Server{ - Addr: bindAddress, - Handler: &metricsMux, - } + logger := s.logger.With(zap.String("server", s.metricsService.BindEndpoint), zap.String("server_type", "metrics")) - logger = logger.With(zap.String("server", bindAddress), zap.String("server_type", "metrics")) - - return (&server{server: metricsServer}).Run(ctx, logger) + return services.NewFromConfig(&s.metricsService, &metricsMux).Run(ctx, logger) } type oidcStore interface { GetPublicKeyByID(keyID string) (any, error) } -func runK8sProxyServer( +func (s *Server) runK8sProxyServer( ctx context.Context, - bindAddress string, oidcStorage oidcStore, - data *certData, - runtimeState state.State, - wrapper k8sproxy.MiddlewareWrapper, - logger *zap.Logger, ) error { keyFunc := func(_ context.Context, keyID string) (any, error) { return oidcStorage.GetPublicKeyByID(keyID) @@ -900,7 +886,7 @@ func runK8sProxyServer( clusterUUIDResolver := func(ctx context.Context, clusterID string) (resource.ID, error) { ctx = actor.MarkContextAsInternalActor(ctx) - uuid, resolveErr := safe.StateGetByID[*omnires.ClusterUUID](ctx, runtimeState, clusterID) + uuid, resolveErr := safe.StateGetByID[*omnires.ClusterUUID](ctx, s.omniRuntime.State(), clusterID) if resolveErr != nil { return "", fmt.Errorf("failed to resolve cluster ID to UUID: %w", resolveErr) } @@ -908,7 +894,7 @@ func runK8sProxyServer( return uuid.TypedSpec().Value.Uuid, nil } - k8sProxyHandler, err := k8sproxy.NewHandler(keyFunc, clusterUUIDResolver, wrapper, logger) + k8sProxyHandler, err := k8sproxy.NewHandler(keyFunc, clusterUUIDResolver, s.auditor, s.logger) if err != nil { return err } @@ -918,19 +904,14 @@ func runK8sProxyServer( k8sProxy := monitoring.NewHandler( logging.NewHandler( k8sProxyHandler, - logger.With(zap.String("handler", "k8s_proxy")), + s.logger.With(zap.String("handler", "k8s_proxy")), ), prometheus.Labels{"handler": "k8s-proxy"}, ) - k8sProxyServer := &http.Server{ - Addr: bindAddress, - Handler: k8sProxy, - } + logger := s.logger.With(zap.String("server", s.k8sProxyService.BindEndpoint), zap.String("server_type", "k8s_proxy")) - logger = logger.With(zap.String("server", bindAddress), zap.String("server_type", "k8s_proxy")) - - return (&server{server: k8sProxyServer, certData: data}).Run(ctx, logger) + return services.NewFromConfig(&s.k8sProxyService, k8sProxy).Run(ctx, logger) } // setRealIPRequest extracts ip from the request and sets it to the X-Forwarded-For header if there is no @@ -952,183 +933,8 @@ func setRealIPRequest(req *http.Request) *http.Request { return newReq } -type server struct { - server *http.Server - certData *certData -} - -type certData struct { - cert tls.Certificate - certFile string - keyFile string - mu sync.Mutex - loaded bool -} - -func (c *certData) load() error { - cert, err := tls.LoadX509KeyPair(c.certFile, c.keyFile) - if err != nil { - return err - } - - c.mu.Lock() - defer c.mu.Unlock() - - c.loaded = true - c.cert = cert - - return nil -} - -func (c *certData) getCert() (*tls.Certificate, error) { - c.mu.Lock() - defer c.mu.Unlock() - - if !c.loaded { - return nil, fmt.Errorf("the cert wasn't loaded yet") - } - - return &c.cert, nil -} - -func (c *certData) runWatcher(ctx context.Context, logger *zap.Logger) error { - w, err := fsnotify.NewWatcher() - if err != nil { - return fmt.Errorf("error creating fsnotify watcher: %w", err) - } - defer w.Close() //nolint:errcheck - - if err = w.Add(c.certFile); err != nil { - return fmt.Errorf("error adding watch for file %s: %w", c.certFile, err) - } - - if err = w.Add(c.keyFile); err != nil { - return fmt.Errorf("error adding watch for file %s: %w", c.keyFile, err) - } - - handleEvent := func(e fsnotify.Event) error { - defer func() { - if err = c.load(); err != nil { - logger.Error("failed to load certs", zap.Error(err)) - - return - } - - logger.Info("reloaded certs") - }() - - if !e.Has(fsnotify.Remove) && !e.Has(fsnotify.Rename) { - return nil - } - - if err = w.Remove(e.Name); err != nil { - logger.Error("failed to remove file watch, it may have been deleted", zap.String("file", e.Name), zap.Error(err)) - } - - if err = w.Add(e.Name); err != nil { - return fmt.Errorf("error adding watch for file %s: %w", e.Name, err) - } - - return nil - } - - for { - select { - case e := <-w.Events: - if err = handleEvent(e); err != nil { - return err - } - case err = <-w.Errors: - return fmt.Errorf("received fsnotify error: %w", err) - case <-ctx.Done(): - return nil - } - } -} - -func (s *server) Run(ctx context.Context, logger *zap.Logger) error { - logger.Info("server starting") - defer logger.Info("server stopped") - - stop := xcontext.AfterFuncSync(ctx, func() { //nolint:contextcheck - logger.Info("server stopping") - - shutdownCtx, shutdownCtxCancel := context.WithTimeout(context.Background(), 5*time.Second) - defer shutdownCtxCancel() - - err := s.shutdown(shutdownCtx) - if err != nil { - logger.Error("failed to gracefully stop server", zap.Error(err)) - } - }) - - defer stop() - - if err := s.listenAndServe(ctx, logger); err != nil && !errors.Is(err, http.ErrServerClosed) { - return fmt.Errorf("failed to serve: %w", err) - } - - return nil -} - -func (s *server) listenAndServe(ctx context.Context, logger *zap.Logger) error { - if s.certData == nil { - return s.server.ListenAndServe() - } - - if err := s.certData.load(); err != nil { - return err - } - - s.server.TLSConfig = &tls.Config{ - GetCertificate: func(*tls.ClientHelloInfo) (*tls.Certificate, error) { - return s.certData.getCert() - }, - } - - ctx, cancel := context.WithCancel(ctx) - defer cancel() - - eg := panichandler.NewErrGroup() - - eg.Go(func() error { - for { - err := s.certData.runWatcher(ctx, logger) - - if err == nil { - return nil - } - - logger.Error("cert watcher crashed, restarting in 5 seconds", zap.Error(err)) - - time.Sleep(time.Second * 5) - } - }) - - eg.Go(func() error { - defer cancel() - - return s.server.ListenAndServeTLS("", "") - }) - - return eg.Wait() -} - -func (s *server) shutdown(ctx context.Context) error { - err := s.server.Shutdown(ctx) - if !errors.Is(ctx.Err(), err) { - return err - } - - if closeErr := s.server.Close(); closeErr != nil { - return fmt.Errorf("failed to close server: %w", closeErr) - } - - return err -} - func runLocalResourceServer(ctx context.Context, st state.CoreState, serverOptions []grpc.ServerOption, eg *errgroup.Group, logger *zap.Logger) error { - listener, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", config.Config.LocalResourceServerPort)) + listener, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", config.Config.Services.LocalResourceService.Port)) if err != nil { return fmt.Errorf("failed to listen: %w", err) } @@ -1191,7 +997,7 @@ func runLocalResourceServer(ctx context.Context, st state.CoreState, serverOptio } func (s *Server) createInitialServiceAccount(ctx context.Context) error { - serviceAccountEmail := config.Config.InitialServiceAccount.Name + access.ServiceAccountNameSuffix + serviceAccountEmail := config.Config.Auth.InitialServiceAccount.Name + access.ServiceAccountNameSuffix identity, err := safe.ReaderGetByID[*authres.Identity](ctx, s.omniRuntime.State(), serviceAccountEmail) if err != nil && !state.IsNotFoundError(err) { @@ -1203,7 +1009,7 @@ func (s *Server) createInitialServiceAccount(ctx context.Context) error { return nil } - key, err := pgp.GenerateKey(config.Config.InitialServiceAccount.Name, "automation initial", serviceAccountEmail, config.Config.InitialServiceAccount.Lifetime) + key, err := pgp.GenerateKey(config.Config.Auth.InitialServiceAccount.Name, "automation initial", serviceAccountEmail, config.Config.Auth.InitialServiceAccount.Lifetime) if err != nil { return fmt.Errorf("failed to create initial service account key, generate failed: %w", err) } @@ -1216,8 +1022,8 @@ func (s *Server) createInitialServiceAccount(ctx context.Context) error { _, err = serviceaccountmgmt.Create( ctx, s.omniRuntime.State(), - config.Config.InitialServiceAccount.Name, - config.Config.InitialServiceAccount.Role, + config.Config.Auth.InitialServiceAccount.Name, + config.Config.Auth.InitialServiceAccount.Role, false, []byte(k), ) @@ -1225,15 +1031,15 @@ func (s *Server) createInitialServiceAccount(ctx context.Context) error { return fmt.Errorf("failed to create initial service account key: %w", err) } - data, err := serviceaccount.Encode(config.Config.InitialServiceAccount.Name, key) + data, err := serviceaccount.Encode(config.Config.Auth.InitialServiceAccount.Name, key) if err != nil { return fmt.Errorf("failed to create initial service account key, failed to encode: %w", err) } - if err = os.WriteFile(config.Config.InitialServiceAccount.KeyPath, []byte(data), 0o640); err != nil { + if err = os.WriteFile(config.Config.Auth.InitialServiceAccount.KeyPath, []byte(data), 0o640); err != nil { return fmt.Errorf( "failed to create initial service account key, failed to write key to path %q: %w", - config.Config.InitialServiceAccount.KeyPath, + config.Config.Auth.InitialServiceAccount.KeyPath, err, ) } @@ -1243,7 +1049,7 @@ func (s *Server) createInitialServiceAccount(ctx context.Context) error { // runEmbeddedDiscoveryService runs an embedded discovery service over Siderolink. func runEmbeddedDiscoveryService(ctx context.Context, logger *zap.Logger) error { - logLevel, err := zapcore.ParseLevel(config.Config.EmbeddedDiscoveryService.LogLevel) + logLevel, err := zapcore.ParseLevel(config.Config.Services.EmbeddedDiscoveryService.LogLevel) if err != nil { logLevel = zapcore.WarnLevel @@ -1254,13 +1060,13 @@ func runEmbeddedDiscoveryService(ctx context.Context, logger *zap.Logger) error if err = retry.Constant(30*time.Second, retry.WithUnits(time.Second)).RetryWithContext(ctx, func(context.Context) error { err = service.Run(ctx, service.Options{ - ListenAddr: net.JoinHostPort(siderolink.ListenHost, strconv.Itoa(config.Config.EmbeddedDiscoveryService.Port)), + ListenAddr: net.JoinHostPort(siderolink.ListenHost, strconv.Itoa(config.Config.Services.EmbeddedDiscoveryService.Port)), GCInterval: time.Minute, MetricsRegisterer: registerer, - SnapshotsEnabled: config.Config.EmbeddedDiscoveryService.SnapshotsEnabled, - SnapshotInterval: config.Config.EmbeddedDiscoveryService.SnapshotInterval, - SnapshotPath: config.Config.EmbeddedDiscoveryService.SnapshotPath, + SnapshotsEnabled: config.Config.Services.EmbeddedDiscoveryService.SnapshotsEnabled, + SnapshotInterval: config.Config.Services.EmbeddedDiscoveryService.SnapshotsInterval, + SnapshotPath: config.Config.Services.EmbeddedDiscoveryService.SnapshotsPath, }, logger.WithOptions(zap.IncreaseLevel(logLevel)).With(logging.Component("discovery_service"))) if errors.Is(err, syscall.EADDRNOTAVAIL) { @@ -1284,14 +1090,9 @@ func runPprofServer(ctx context.Context, bindAddress string, l *zap.Logger) erro mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) mux.HandleFunc("/debug/pprof/trace", pprof.Trace) - srv := &http.Server{ - Addr: bindAddress, - Handler: mux, - } - l = l.With(zap.String("server", bindAddress), zap.String("server_type", "pprof")) - return (&server{server: srv}).Run(ctx, l) + return services.NewInsecure(bindAddress, mux).Run(ctx, l) } //nolint:unparam diff --git a/internal/backend/services/http_server.go b/internal/backend/services/http_server.go new file mode 100644 index 00000000..7c30fecd --- /dev/null +++ b/internal/backend/services/http_server.go @@ -0,0 +1,233 @@ +// Copyright (c) 2025 Sidero Labs, Inc. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. + +// Package services contains HTTP servers. +package services + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "net/http" + "sync" + "time" + + "github.com/fsnotify/fsnotify" + "go.uber.org/zap" + + "github.com/siderolabs/omni/client/pkg/panichandler" + "github.com/siderolabs/omni/internal/pkg/config" + "github.com/siderolabs/omni/internal/pkg/xcontext" +) + +// NewFromConfig creates a new Server from the config.Service. +func NewFromConfig(service config.HTTPService, handler http.Handler) *Server { + var cert *certData + + if service.IsSecure() { + cert = &certData{ + certFile: service.GetCertFile(), + keyFile: service.GetKeyFile(), + } + } + + return &Server{ + server: &http.Server{ + Addr: service.GetBindEndpoint(), + Handler: handler, + }, + certData: cert, + } +} + +// NewInsecure creates a new Server. +func NewInsecure(endpoint string, handler http.Handler) *Server { + return &Server{ + server: &http.Server{ + Addr: endpoint, + Handler: handler, + }, + } +} + +// Server is the HTTP server. +type Server struct { + server *http.Server + certData *certData +} + +type certData struct { + cert tls.Certificate + certFile string + keyFile string + mu sync.Mutex + loaded bool +} + +func (c *certData) load() error { + cert, err := tls.LoadX509KeyPair(c.certFile, c.keyFile) + if err != nil { + return err + } + + c.mu.Lock() + defer c.mu.Unlock() + + c.loaded = true + c.cert = cert + + return nil +} + +func (c *certData) getCert() (*tls.Certificate, error) { + c.mu.Lock() + defer c.mu.Unlock() + + if !c.loaded { + return nil, fmt.Errorf("the cert wasn't loaded yet") + } + + return &c.cert, nil +} + +func (c *certData) runWatcher(ctx context.Context, logger *zap.Logger) error { + w, err := fsnotify.NewWatcher() + if err != nil { + return fmt.Errorf("error creating fsnotify watcher: %w", err) + } + defer w.Close() //nolint:errcheck + + if err = w.Add(c.certFile); err != nil { + return fmt.Errorf("error adding watch for file %s: %w", c.certFile, err) + } + + if err = w.Add(c.keyFile); err != nil { + return fmt.Errorf("error adding watch for file %s: %w", c.keyFile, err) + } + + handleEvent := func(e fsnotify.Event) error { + defer func() { + if err = c.load(); err != nil { + logger.Error("failed to load certs", zap.Error(err)) + + return + } + + logger.Info("reloaded certs") + }() + + if !e.Has(fsnotify.Remove) && !e.Has(fsnotify.Rename) { + return nil + } + + if err = w.Remove(e.Name); err != nil { + logger.Error("failed to remove file watch, it may have been deleted", zap.String("file", e.Name), zap.Error(err)) + } + + if err = w.Add(e.Name); err != nil { + return fmt.Errorf("error adding watch for file %s: %w", e.Name, err) + } + + return nil + } + + for { + select { + case e := <-w.Events: + if err = handleEvent(e); err != nil { + return err + } + case err = <-w.Errors: + return fmt.Errorf("received fsnotify error: %w", err) + case <-ctx.Done(): + return nil + } + } +} + +// Run the server. +func (s *Server) Run(ctx context.Context, logger *zap.Logger) error { + logger.Info("server starting", zap.Bool("secure", s.certData != nil)) + defer logger.Info("server stopped") + + stop := xcontext.AfterFuncSync(ctx, func() { //nolint:contextcheck + logger.Info("server stopping") + + shutdownCtx, shutdownCtxCancel := context.WithTimeout(context.Background(), 5*time.Second) + defer shutdownCtxCancel() + + err := s.shutdown(shutdownCtx) + if err != nil { + logger.Error("failed to gracefully stop server", zap.Error(err)) + } + }) + + defer stop() + + if err := s.listenAndServe(ctx, logger); err != nil && !errors.Is(err, http.ErrServerClosed) { + logger.Error("failed to serve", zap.Error(err)) + + return fmt.Errorf("failed to serve: %w", err) + } + + return nil +} + +func (s *Server) listenAndServe(ctx context.Context, logger *zap.Logger) error { + if s.certData == nil { + return s.server.ListenAndServe() + } + + if err := s.certData.load(); err != nil { + return fmt.Errorf("failed to load certs: %w", err) + } + + s.server.TLSConfig = &tls.Config{ + GetCertificate: func(*tls.ClientHelloInfo) (*tls.Certificate, error) { + return s.certData.getCert() + }, + } + + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + eg := panichandler.NewErrGroup() + + eg.Go(func() error { + for { + err := s.certData.runWatcher(ctx, logger) + + if err == nil { + return nil + } + + logger.Error("cert watcher crashed, restarting in 5 seconds", zap.Error(err)) + + time.Sleep(time.Second * 5) + } + }) + + eg.Go(func() error { + defer cancel() + + return s.server.ListenAndServeTLS("", "") + }) + + return eg.Wait() +} + +func (s *Server) shutdown(ctx context.Context) error { + err := s.server.Shutdown(ctx) + if !errors.Is(ctx.Err(), err) { + return err + } + + if closeErr := s.server.Close(); closeErr != nil { + return fmt.Errorf("failed to close server: %w", closeErr) + } + + return err +} diff --git a/internal/backend/proxy_server.go b/internal/backend/services/proxy.go similarity index 70% rename from internal/backend/proxy_server.go rename to internal/backend/services/proxy.go index 0d252b49..9b87daaa 100644 --- a/internal/backend/proxy_server.go +++ b/internal/backend/services/proxy.go @@ -3,7 +3,7 @@ // Use of this software is governed by the Business Source License // included in the LICENSE file. -package backend +package services import ( "context" @@ -16,6 +16,8 @@ import ( "time" "go.uber.org/zap" + + "github.com/siderolabs/omni/internal/pkg/config" ) // Proxy is a proxy server. @@ -23,28 +25,24 @@ type Proxy interface { Run(ctx context.Context, next http.Handler, logger *zap.Logger) error } -// NewProxyServer creates a new proxy server. If the destination is empty, the proxy server will be a no-op. -func NewProxyServer(bindAddr string, proxyTo http.Handler, keyFile, certFile string) Proxy { +// NewProxy creates a new proxy server. If the destination is empty, the proxy server will be a no-op. +func NewProxy(config config.DevServerProxyService, handler http.Handler) Proxy { switch { - case bindAddr == "": + case config.BindEndpoint == "": return &nopProxy{reason: "bind address is empty"} - case proxyTo == nopHandler: + case config.ProxyTo == "": return &nopProxy{reason: "proxy destination is empty"} default: return &httpProxy{ - bindAddr: bindAddr, - proxyTo: proxyTo, - keyFile: keyFile, - certFile: certFile, + config: config, + proxyTo: handler, } } } type httpProxy struct { - proxyTo http.Handler - bindAddr string - keyFile string - certFile string + proxyTo http.Handler + config config.DevServerProxyService } func hasPrefix(s string, prefixes ...string) bool { @@ -58,26 +56,20 @@ func hasPrefix(s string, prefixes ...string) bool { } func (prx *httpProxy) Run(ctx context.Context, next http.Handler, logger *zap.Logger) error { - srv := &server{ - server: &http.Server{ - Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if hasPrefix(r.URL.Path, "/api/", "/omnictl/", "/talosctl/", "/image/") { - next.ServeHTTP(w, r) + srv := NewFromConfig( + &prx.config, + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if hasPrefix(r.URL.Path, "/api/", "/omnictl/", "/talosctl/", "/image/") { + next.ServeHTTP(w, r) - return - } + return + } - prx.proxyTo.ServeHTTP(w, r) - }), - Addr: prx.bindAddr, - }, - certData: &certData{ - certFile: prx.certFile, - keyFile: prx.keyFile, - }, - } + prx.proxyTo.ServeHTTP(w, r) + }), + ) - logger = logger.With(zap.String("server", prx.bindAddr), zap.String("server_type", "proxy_server")) + logger = logger.With(zap.String("server", prx.config.BindEndpoint), zap.String("server_type", "proxy_server")) return srv.Run(ctx, logger) } diff --git a/internal/backend/workloadproxy/handler.go b/internal/backend/workloadproxy/handler.go index 265646fb..105e12ec 100644 --- a/internal/backend/workloadproxy/handler.go +++ b/internal/backend/workloadproxy/handler.go @@ -220,7 +220,7 @@ func (h *HTTPHandler) getSignatureCookies(request *http.Request) (publicKeyID st } func (h *HTTPHandler) redirectToLogin(writer http.ResponseWriter, request *http.Request) { - loginURL, err := url.Parse(config.Config.APIURL) + loginURL, err := url.Parse(config.Config.Services.API.URL()) if err != nil { h.logger.Warn("failed to redirect to login", zap.Error(err)) diff --git a/internal/pkg/auth/config.go b/internal/pkg/auth/config.go index aae74b82..edaf3470 100644 --- a/internal/pkg/auth/config.go +++ b/internal/pkg/auth/config.go @@ -20,7 +20,7 @@ import ( ) // EnsureAuthConfigResource creates/configures the auth config resource. -func EnsureAuthConfigResource(ctx context.Context, st state.State, logger *zap.Logger, authParams config.AuthParams) (*auth.Config, error) { +func EnsureAuthConfigResource(ctx context.Context, st state.State, logger *zap.Logger, authParams config.Auth) (*auth.Config, error) { err := validateParams(authParams) if err != nil { return nil, err @@ -46,7 +46,7 @@ func EnsureAuthConfigResource(ctx context.Context, st state.State, logger *zap.L res.TypedSpec().Value.Auth0.ClientId = authParams.Auth0.ClientID res.TypedSpec().Value.Auth0.UseFormData = authParams.Auth0.UseFormData res.TypedSpec().Value.Saml.Enabled = authParams.SAML.Enabled - res.TypedSpec().Value.Saml.Url = authParams.SAML.URL + res.TypedSpec().Value.Saml.Url = authParams.SAML.MetadataURL res.TypedSpec().Value.Saml.Metadata = authParams.SAML.Metadata res.TypedSpec().Value.Saml.LabelRules = authParams.SAML.LabelRules @@ -105,7 +105,7 @@ func EnsureAuthConfigResource(ctx context.Context, st state.State, logger *zap.L return authConfig, nil } -func validateParams(authParams config.AuthParams) error { +func validateParams(authParams config.Auth) error { if !authParams.SAML.Enabled && !authParams.Auth0.Enabled && !authParams.WebAuthn.Enabled { return errors.New("no authentication is enabled") } @@ -114,7 +114,7 @@ func validateParams(authParams config.AuthParams) error { return errors.New("both auth0 and SAML auth are enabled, only one can be enabled at the same time") } - if authParams.SAML.Enabled && authParams.SAML.URL == "" && authParams.SAML.Metadata == "" { + if authParams.SAML.Enabled && authParams.SAML.MetadataURL == "" && authParams.SAML.Metadata == "" { return errors.New("SAML is enabled but neither URL nor metadata is set") } diff --git a/internal/pkg/auth/config_test.go b/internal/pkg/auth/config_test.go index 0770e179..d37b8e8f 100644 --- a/internal/pkg/auth/config_test.go +++ b/internal/pkg/auth/config_test.go @@ -31,8 +31,8 @@ func TestEnsureAuthConfigResource(t *testing.T) { for _, tt := range []struct { //nolint:govet name string - initialConfig config.AuthParams - updatedConfig *config.AuthParams + initialConfig config.Auth + updatedConfig *config.Auth expected *specs.AuthConfigSpec expectInitError bool expectUpdateError bool @@ -43,8 +43,8 @@ func TestEnsureAuthConfigResource(t *testing.T) { }, { name: "enable auth0", - initialConfig: config.AuthParams{ - Auth0: config.Auth0Params{ + initialConfig: config.Auth{ + Auth0: config.Auth0{ Enabled: true, ClientID: "client-id", Domain: "domain", @@ -62,8 +62,8 @@ func TestEnsureAuthConfigResource(t *testing.T) { }, { name: "enable webauthn", - initialConfig: config.AuthParams{ - WebAuthn: config.WebAuthnParams{ + initialConfig: config.Auth{ + WebAuthn: config.WebAuthn{ Enabled: true, }, }, @@ -77,14 +77,14 @@ func TestEnsureAuthConfigResource(t *testing.T) { }, { name: "make webauthn not required", - initialConfig: config.AuthParams{ - WebAuthn: config.WebAuthnParams{ + initialConfig: config.Auth{ + WebAuthn: config.WebAuthn{ Enabled: true, Required: true, }, }, - updatedConfig: &config.AuthParams{ - WebAuthn: config.WebAuthnParams{ + updatedConfig: &config.Auth{ + WebAuthn: config.WebAuthn{ Enabled: true, Required: false, }, @@ -99,14 +99,14 @@ func TestEnsureAuthConfigResource(t *testing.T) { }, { name: "fail to disable webauthn", - initialConfig: config.AuthParams{ - WebAuthn: config.WebAuthnParams{ + initialConfig: config.Auth{ + WebAuthn: config.WebAuthn{ Enabled: true, Required: true, }, }, - updatedConfig: &config.AuthParams{ - WebAuthn: config.WebAuthnParams{ + updatedConfig: &config.Auth{ + WebAuthn: config.WebAuthn{ Enabled: false, }, }, @@ -114,29 +114,29 @@ func TestEnsureAuthConfigResource(t *testing.T) { }, { name: "fail to disable auth0", - initialConfig: config.AuthParams{ - Auth0: config.Auth0Params{ + initialConfig: config.Auth{ + Auth0: config.Auth0{ Enabled: true, ClientID: "client-id", Domain: "domain", }, }, - updatedConfig: &config.AuthParams{}, + updatedConfig: &config.Auth{}, expectUpdateError: true, }, { name: "switch from auth0 to SAML", - initialConfig: config.AuthParams{ - Auth0: config.Auth0Params{ + initialConfig: config.Auth{ + Auth0: config.Auth0{ Enabled: true, ClientID: "client-id", Domain: "domain", }, }, - updatedConfig: &config.AuthParams{ - SAML: config.SAMLParams{ - Enabled: true, - URL: "http://samltest.sp/idp", + updatedConfig: &config.Auth{ + SAML: config.SAML{ + Enabled: true, + MetadataURL: "http://samltest.sp/idp", }, }, expected: &specs.AuthConfigSpec{ @@ -150,13 +150,13 @@ func TestEnsureAuthConfigResource(t *testing.T) { }, { name: "fail to disable SAML", - initialConfig: config.AuthParams{ - SAML: config.SAMLParams{ - Enabled: true, - URL: "http://samltest.sp/idp", + initialConfig: config.Auth{ + SAML: config.SAML{ + Enabled: true, + MetadataURL: "http://samltest.sp/idp", }, }, - updatedConfig: &config.AuthParams{}, + updatedConfig: &config.Auth{}, expectUpdateError: true, }, } { diff --git a/internal/pkg/config/auth.go b/internal/pkg/config/auth.go index 80b523fc..e4c64556 100644 --- a/internal/pkg/config/auth.go +++ b/internal/pkg/config/auth.go @@ -5,39 +5,74 @@ package config -import "encoding/json" +import ( + "encoding/json" + "time" +) -// AuthParams configures authentication. +// Auth configures authentication. // //nolint:govet -type AuthParams struct { - Auth0 Auth0Params `yaml:"auth0"` - WebAuthn WebAuthnParams `yaml:"webauthn"` - SAML SAMLParams `yaml:"saml"` +type Auth struct { + // Auth0 auth type configuration. + Auth0 Auth0 `yaml:"auth0" validate:"excluded_with=SAML"` + // WebAuthn auth type configuration. + WebAuthn WebAuthn `yaml:"webauthn"` + // SAML auth type configuration. + SAML SAML `yaml:"saml" validate:"excluded_with=Auth0"` + // KeyPruner automatically removes the unused public keys registered in Omni. + KeyPruner KeyPrunerConfig `yaml:"keyPruner"` + + // Suspended makes the account readonly. Suspended bool `yaml:"suspended"` + + // InitialServiceAccount creates a service account on the first Omni start up. + // Writes the service account key to the file defined in the keyPath param. + // + // This service account can be used if it is required to run omnictl or a provider using + // some automation scripts. + InitialServiceAccount InitialServiceAccount `yaml:"initialServiceAccount"` } -// Auth0Params holds configuration parameters for Auth0. -type Auth0Params struct { - Domain string `yaml:"domain"` - ClientID string `yaml:"clientID"` - UseFormData bool `yaml:"useFormData"` - Enabled bool `yaml:"enabled"` +// Auth0 holds configuration parameters for Auth0. +// +//nolint:govet +type Auth0 struct { + // InitialUsers adds the user to the account on the first Omni start up. + InitialUsers []string `yaml:"initialUsers"` + Domain string `yaml:"domain"` + ClientID string `yaml:"clientID"` + UseFormData bool `yaml:"useFormData"` + Enabled bool `yaml:"enabled"` } -// WebAuthnParams holds configuration parameters for WebAuthn. -type WebAuthnParams struct { +// WebAuthn holds configuration parameters for WebAuthn. +type WebAuthn struct { Enabled bool `yaml:"enabled"` Required bool `yaml:"required"` } -// SAMLParams holds configuration parameters for SAML auth. -type SAMLParams struct { - LabelRules SAMLLabelRules `yaml:"labelRules"` - URL string `yaml:"url"` - Metadata string `yaml:"metadata"` - Enabled bool `yaml:"enabled"` +// SAML holds configuration parameters for SAML auth. +type SAML struct { + LabelRules SAMLLabelRules `yaml:"labelRules"` + MetadataURL string `yaml:"url" validate:"excluded_with=Metadata"` + Metadata string `yaml:"metadata" validate:"excluded_with=MetadataURL"` + Enabled bool `yaml:"enabled"` +} + +// KeyPrunerConfig defines key pruner configs. +type KeyPrunerConfig struct { + Interval time.Duration `yaml:"interval"` +} + +// InitialServiceAccount allows creating a service account for automated omnictl runs on the Omni service deployment. +type InitialServiceAccount struct { + Role string `yaml:"role"` + KeyPath string `yaml:"keyPath"` + Name string `yaml:"name"` + Lifetime time.Duration `yaml:"lifetime"` + Enabled bool `yaml:"enabled"` } // SAMLLabelRules defines mapping of SAML assertion attributes to Omni identity labels. diff --git a/internal/pkg/config/backup.go b/internal/pkg/config/backup.go new file mode 100644 index 00000000..3894b60c --- /dev/null +++ b/internal/pkg/config/backup.go @@ -0,0 +1,41 @@ +// Copyright (c) 2025 Sidero Labs, Inc. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. + +package config + +import ( + "errors" + "time" +) + +// EtcdBackup defines etcd backup configs. +type EtcdBackup struct { + LocalPath string `yaml:"localPath" validate:"excluded_with=S3Enabled"` + S3Enabled bool `yaml:"s3Enabled" validate:"excluded_with=LocalPath"` + TickInterval time.Duration `yaml:"tickInterval"` + MinInterval time.Duration `yaml:"minInterval"` + MaxInterval time.Duration `yaml:"maxInterval"` + UploadLimitMbps uint64 `yaml:"uploadLimitMbps"` + DownloadLimitMbps uint64 `yaml:"downloadLimitMbps"` + Jitter time.Duration `yaml:"jitter"` +} + +// GetStorageType returns the storage type. +func (ebp EtcdBackup) GetStorageType() (EtcdBackupStorage, error) { + if ebp.LocalPath != "" && ebp.S3Enabled { + return "", errors.New("both localPath and s3 are set") + } + + switch { + case ebp.LocalPath == "" && !ebp.S3Enabled: + return EtcdBackupTypeS3, nil + case ebp.LocalPath != "": + return EtcdBackupTypeFS, nil + case ebp.S3Enabled: + return EtcdBackupTypeS3, nil + default: + return "", errors.New("unknown backup storage type") + } +} diff --git a/internal/pkg/config/config.go b/internal/pkg/config/config.go index a256db45..6fd77da5 100644 --- a/internal/pkg/config/config.go +++ b/internal/pkg/config/config.go @@ -7,15 +7,17 @@ package config import ( + "bytes" "errors" "fmt" + "io" "net" "net/url" - "slices" - "strings" + "os" "time" - "github.com/siderolabs/talos/pkg/machinery/config/generate" + "github.com/go-playground/validator/v10" + "github.com/siderolabs/gen/xyaml" "go.uber.org/zap/zapcore" consts "github.com/siderolabs/omni/client/pkg/constants" @@ -27,251 +29,84 @@ const ( wireguardDefaultPort = "50180" ) +// FromBytes loads the config from bytes. +func FromBytes(data []byte) (*Params, error) { + return parseConfig(bytes.NewBuffer(data)) +} + +// LoadFromFile loads the config from the file. +func LoadFromFile(path string) (*Params, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + + defer f.Close() //nolint:errcheck + + return parseConfig(f) +} + +func parseConfig(r io.Reader) (*Params, error) { + data, err := io.ReadAll(r) + if err != nil { + return nil, err + } + + var config Params + + if err := xyaml.UnmarshalStrict(data, &config); err != nil { + return nil, err + } + + return &config, nil +} + // Params defines application configs. // //nolint:govet type Params struct { - // AccountID is the stable identifier of the instance. + Account Account `yaml:"account" validate:"required"` + Services Services `yaml:"services" validate:"required"` + Auth Auth `yaml:"auth" validate:"required"` + Logs Logs `yaml:"logs" validate:"required"` + Storage Storage `yaml:"storage" validate:"required"` + EtcdBackup EtcdBackup `yaml:"etcdBackup"` + Registries Registries `yaml:"registries" validate:"required"` + Debug Debug `yaml:"debug"` + Features Features `yaml:"features"` +} + +// Validate Omni params. +func (p *Params) Validate() error { + validate := validator.New(validator.WithRequiredStructEnabled()) + + return validate.Struct(p) +} + +// Account defines Omni account settings. +type Account struct { + // ID is the stable identifier of the instance. // // Omni will use that to build paths to etcd storage, etc. - AccountID string `yaml:"accountID"` + ID string `yaml:"id" validate:"required"` // Name is the user-facing name of the instance. // // Omni will use to present some information to the user. // Name can be changed at any time. - Name string `yaml:"name"` - - APIURL string `yaml:"apiURL"` - MachineAPIBindAddress string `yaml:"apiBindAddress"` - MachineAPICertFile string `yaml:"apiCertFile"` - MachineAPIKeyFile string `yaml:"apiKeyFile"` - - KubernetesProxyURL string `yaml:"kubernetesProxyURL"` - SiderolinkEnabled bool `yaml:"siderolinkEnabled"` - SiderolinkWireguardBindAddress string `yaml:"siderolinkWireguardBindAddress"` - SiderolinkWireguardAdvertisedAddress string `yaml:"siderolinkWireguardAdvertisedAddress"` - SiderolinkDisableLastEndpoint bool `yaml:"siderolinkDisableLastEndpoint"` - SiderolinkUseGRPCTunnel bool `yaml:"siderolinkUseGRPCTunnel"` - RunDebugServer bool `yaml:"runDebugServer"` - - EventSinkPort int `yaml:"eventSinkPort"` - SideroLinkAPIURL string `yaml:"siderolinkAPIURL"` - LoadBalancer LoadBalancerParams `yaml:"loadbalancer"` - LogServerPort int `yaml:"logServerPort"` - - MachineLogConfig MachineLogConfigParams `yaml:"machineLogConfig"` - - Auth AuthParams `yaml:"auth"` - - InitialUsers []string `yaml:"initialUsers"` - - TalosRegistry string `yaml:"talosRegistry"` - - KubernetesRegistry string `yaml:"kubernetesRegistry"` - - ImageFactoryBaseURL string `yaml:"imageFactoryAddress"` - ImageFactoryPXEBaseURL string `yaml:"imageFactoryProxyAddress"` - - Storage StorageParams `yaml:"storage"` - - SecondaryStorage BoltDBParams `yaml:"secondaryStorage"` - - DefaultConfigGenOptions []generate.Option `yaml:"-" json:"-"` - - KeyPruner KeyPrunerParams `yaml:"keyPruner"` - - EnableTalosPreReleaseVersions bool `yaml:"enableTalosPreReleaseVersions"` - - WorkloadProxying WorkloadProxyingParams `yaml:"workloadProxying"` - - ConfigDataCompression ConfigDataCompressionParams `yaml:"configDataCompression"` - - LocalResourceServerPort int `yaml:"localResourceServerPort"` - - EtcdBackup EtcdBackupParams `yaml:"etcdBackup"` - - DisableControllerRuntimeCache bool `yaml:"disableControllerRuntimeCache"` - - LogResourceUpdatesTypes []string `yaml:"logResourceUpdatesTypes"` - LogResourceUpdatesLogLevel string `yaml:"logResourceUpdatesLogLevel"` - - EmbeddedDiscoveryService EmbeddedDiscoveryServiceParams `yaml:"embeddedDiscoveryService"` - - EnableBreakGlassConfigs bool `yaml:"enableBreakGlassConfigs"` - - AuditLogDir string `yaml:"auditLogDir"` - - InitialServiceAccount InitialServiceAccount `yaml:"initialServiceAccount"` - - EnableStripeReporting bool `yaml:"enableStripeReporting"` - - JoinTokensMode JoinTokensMode `yaml:"joinTokensMode"` + Name string `yaml:"name" validate:"required"` } -// JoinTokensMode is the join token operation mode config. -// -//nolint:recvcheck -type JoinTokensMode string +// Registries configures docker registries to be used for the Talos and Kubernetes images. +// Also it has URLs for the image factory. +type Registries struct { + Talos string `yaml:"talos" validate:"required"` + Kubernetes string `yaml:"kubernetes" validate:"required"` -// String implements pflag.Value. -func (s JoinTokensMode) String() string { - return string(s) -} + ImageFactoryBaseURL string `yaml:"imageFactoryBaseURL" validate:"required"` + ImageFactoryPXEBaseURL string `yaml:"imageFactoryPXEBaseURL"` -// Set implements pflag.Value. -func (s *JoinTokensMode) Set(value string) error { - if !slices.Contains(s.values(), value) { - return fmt.Errorf("should be one of %s", strings.Join(s.values(), ", ")) - } - - *s = JoinTokensMode(value) - - return nil -} - -// Type implements pflag.Value. -func (s JoinTokensMode) Type() string { - return fmt.Sprintf("[%s]", strings.Join(s.values(), ",")) -} - -func (JoinTokensMode) values() []string { - return []string{JoinTokensModeLegacyOnly, JoinTokensModeLegacyAllowed, JoinTokensModeStrict} -} - -const ( - // JoinTokensModeLegacyOnly disables node unique token flow, uses only join token when letting the machine into the system. - JoinTokensModeLegacyOnly = "legacy" - // JoinTokensModeLegacyAllowed allows joining Talos nodes which do not support node unique token flow - // uses unique token flow only for the machines which support it. - JoinTokensModeLegacyAllowed = "legacyAllowed" - // JoinTokensModeStrict rejects the machines that do not support node unique tokens flow. - JoinTokensModeStrict = "strict" -) - -// InitialServiceAccount allows creating a service account for automated omnictl runs on the Omni service deployment. -type InitialServiceAccount struct { - Role string - KeyPath string - Name string - Lifetime time.Duration - Enabled bool -} - -// EmbeddedDiscoveryServiceParams defines embedded discovery service configs. -type EmbeddedDiscoveryServiceParams struct { - SnapshotPath string `yaml:"snapshotPath"` - LogLevel string `yaml:"logLevel"` - Enabled bool `yaml:"enabled"` - SnapshotsEnabled bool `yaml:"snapshotsEnabled"` - Port int `yaml:"port"` - SnapshotInterval time.Duration `yaml:"snapshotInterval"` -} - -// EtcdBackupParams defines etcd backup configs. -type EtcdBackupParams struct { - LocalPath string `yaml:"localPath"` - S3Enabled bool `yaml:"s3Enabled"` - TickInterval time.Duration `yaml:"tickInterval"` - MinInterval time.Duration `yaml:"minInterval"` - MaxInterval time.Duration `yaml:"maxInterval"` - UploadLimitMbps uint64 `yaml:"uploadLimitMbps"` - DownloadLimitMbps uint64 `yaml:"downloadLimitMbps"` - Jitter time.Duration `yaml:"jitter"` -} - -// GetStorageType returns the storage type. -func (ebp EtcdBackupParams) GetStorageType() (EtcdBackupStorage, error) { - if ebp.LocalPath != "" && ebp.S3Enabled { - return "", errors.New("both localPath and s3 are set") - } - - switch { - case ebp.LocalPath == "" && !ebp.S3Enabled: - return EtcdBackupTypeS3, nil - case ebp.LocalPath != "": - return EtcdBackupTypeFS, nil - case ebp.S3Enabled: - return EtcdBackupTypeS3, nil - default: - return "", errors.New("unknown backup storage type") - } -} - -// WorkloadProxyingParams defines workload proxying configs. -type WorkloadProxyingParams struct { - Subdomain string `yaml:"subdomain"` - Enabled bool `yaml:"enabled"` -} - -// ConfigDataCompressionParams defines config data compression configs. -// -//nolint:revive -type ConfigDataCompressionParams struct { - Enabled bool `yaml:"enabled"` -} - -// LoadBalancerParams defines load balancer configs. -type LoadBalancerParams struct { - MinPort int `yaml:"minPort"` - MaxPort int `yaml:"maxPort"` - - DialTimeout time.Duration `yaml:"dialTimeout"` - KeepAlivePeriod time.Duration `yaml:"keepAlivePeriod"` - TCPUserTimeout time.Duration `yaml:"tcpUserTimeout"` - - HealthCheckInterval time.Duration `yaml:"healthCheckInterval"` - HealthCheckTimeout time.Duration `yaml:"healthCheckTimeout"` -} - -// StorageParams defines storage configs. -type StorageParams struct { - // Kind can be either 'boltdb' or 'etcd'. - Kind string `yaml:"kind"` - Boltdb BoltDBParams `yaml:"boltdb"` - Etcd EtcdParams `yaml:"etcd"` -} - -// BoltDBParams defines boltdb storage configs. -type BoltDBParams struct { - Path string `yaml:"path"` -} - -// EtcdParams defines etcd storage configs. -type EtcdParams struct { ///nolint:govet - // External etcd: list of endpoints, as host:port pairs. - Endpoints []string `yaml:"endpoints"` - DialKeepAliveTime time.Duration `yaml:"dialKeepAliveTime"` - DialKeepAliveTimeout time.Duration `yaml:"dialKeepAliveTimeout"` - CAPath string `yaml:"caPath"` - CertPath string `yaml:"certPath"` - KeyPath string `yaml:"keyPath"` - - // Use embedded etcd server (no clustering). - Embedded bool `yaml:"embedded"` - EmbeddedDBPath string `yaml:"embeddedDBPath"` - EmbeddedUnsafeFsync bool `yaml:"embeddedUnsafeFsync"` - - PrivateKeySource string `yaml:"privateKeySource"` - PublicKeyFiles []string `yaml:"publicKeysFiles"` -} - -// KeyPrunerParams defines key pruner configs. -type KeyPrunerParams struct { - Interval time.Duration `yaml:"interval"` -} - -// MachineLogConfigParams defines log storage configuration. -type MachineLogConfigParams struct { - StoragePath string `yaml:"directory"` - - BufferInitialCapacity int `yaml:"bufferInitialCapacity"` - BufferMaxCapacity int `yaml:"bufferMaxCapacity"` - BufferSafetyGap int `yaml:"bufferSafetyGap"` - NumCompressedChunks int `yaml:"numCompressedChunks"` - - StorageFlushPeriod time.Duration `yaml:"flushPeriod"` - StorageFlushJitter float64 `yaml:"flushJitter"` - StorageEnabled bool `yaml:"enabled"` + // Mirrors enables registry mirrors for all Talos machines connected to Omni. + Mirrors []string `yaml:"mirrors"` } var ( @@ -283,11 +118,11 @@ var ( // GetImageFactoryPXEBaseURL reads image factory PXE address from the args. func (p *Params) GetImageFactoryPXEBaseURL() (*url.URL, error) { - if p.ImageFactoryPXEBaseURL != "" { - return url.Parse(p.ImageFactoryPXEBaseURL) + if p.Registries.ImageFactoryPXEBaseURL != "" { + return url.Parse(p.Registries.ImageFactoryPXEBaseURL) } - url, err := url.Parse(p.ImageFactoryBaseURL) + url, err := url.Parse(p.Registries.ImageFactoryBaseURL) if err != nil { return nil, fmt.Errorf("invalid URL specified for the image factory: %w", err) } @@ -297,24 +132,9 @@ func (p *Params) GetImageFactoryPXEBaseURL() (*url.URL, error) { return url, nil } -// GetAdvertisedAPIHost returns the advertised host (IP or domain) of the API without the port. -func (p *Params) GetAdvertisedAPIHost() (string, error) { - apiURL, err := url.Parse(p.SideroLinkAPIURL) - if err != nil { - return "", err - } - - apiHost, _, err := net.SplitHostPort(apiURL.Host) - if err != nil { - apiHost = apiURL.Host - } - - return apiHost, nil -} - // GetOIDCIssuerEndpoint returns the OIDC issuer endpoint. func (p *Params) GetOIDCIssuerEndpoint() (string, error) { - u, err := url.Parse(p.APIURL) + u, err := url.Parse(p.Services.API.URL()) if err != nil { return "", err } @@ -327,109 +147,152 @@ func (p *Params) GetOIDCIssuerEndpoint() (string, error) { return u.String(), nil } +// PopulateFallbacks in the config file. +func (p *Params) PopulateFallbacks() { + // copy the keys from the main API server if kubernetes proxy doesn't have certs defined explicitly. + if !p.Services.KubernetesProxy.IsSecure() { + p.Services.KubernetesProxy.CertFile = p.Services.API.CertFile + p.Services.KubernetesProxy.KeyFile = p.Services.API.KeyFile + } + + // copy the keys from the main API server if dev server proxy doesn't have certs defined explicitly. + if !p.Services.DevServerProxy.IsSecure() { + p.Services.DevServerProxy.CertFile = p.Services.API.CertFile + p.Services.DevServerProxy.KeyFile = p.Services.API.KeyFile + } +} + // InitDefault creates the default config. func InitDefault() *Params { return &Params{ - AccountID: "edd2822a-7834-4fe0-8172-cc5581f13a8d", - Name: "default", - APIURL: fmt.Sprintf("http://%s", net.JoinHostPort("localhost", "8080")), - KubernetesProxyURL: fmt.Sprintf("https://%s", net.JoinHostPort("localhost", "8095")), - SiderolinkEnabled: true, - SiderolinkWireguardBindAddress: net.JoinHostPort("0.0.0.0", wireguardDefaultPort), - SiderolinkWireguardAdvertisedAddress: net.JoinHostPort(localIP, wireguardDefaultPort), - MachineAPIBindAddress: net.JoinHostPort(localIP, "8090"), - EventSinkPort: 8090, - SideroLinkAPIURL: fmt.Sprintf("grpc://%s", net.JoinHostPort(localIP, "8090")), - LoadBalancer: LoadBalancerParams{ - MinPort: 10000, - MaxPort: 35000, - - DialTimeout: 15 * time.Second, - KeepAlivePeriod: 30 * time.Second, - TCPUserTimeout: 30 * time.Second, - - HealthCheckInterval: 20 * time.Second, - HealthCheckTimeout: 15 * time.Second, + Account: Account{ + ID: "edd2822a-7834-4fe0-8172-cc5581f13a8d", + Name: "default", }, - KeyPruner: KeyPrunerParams{ - Interval: 10 * time.Minute, - }, - LogServerPort: 8092, - MachineLogConfig: MachineLogConfigParams{ - BufferInitialCapacity: 16384, - BufferMaxCapacity: 131072, - BufferSafetyGap: 256, - NumCompressedChunks: 5, - StorageEnabled: true, - StoragePath: "_out/logs", - StorageFlushPeriod: 10 * time.Minute, - StorageFlushJitter: 0.1, - }, - TalosRegistry: consts.TalosRegistry, - KubernetesRegistry: consts.KubernetesRegistry, - ImageFactoryBaseURL: consts.ImageFactoryBaseURL, - Storage: StorageParams{ - Kind: "etcd", - Boltdb: BoltDBParams{ - Path: "_out/omni.db", + Services: Services{ + API: Service{ + BindEndpoint: net.JoinHostPort("localhost", "8080"), }, - Etcd: EtcdParams{ - Endpoints: []string{"http://localhost:2379"}, - DialKeepAliveTime: 30 * time.Second, - DialKeepAliveTimeout: 5 * time.Second, - CAPath: "etcd/ca.crt", - CertPath: "etcd/client.crt", - KeyPath: "etcd/client.key", + KubernetesProxy: KubernetesProxyService{ + BindEndpoint: net.JoinHostPort("localhost", "8095"), + }, + Metrics: Service{ + BindEndpoint: net.JoinHostPort("0.0.0.0", "2122"), + }, + Siderolink: SiderolinkService{ + WireGuard: SiderolinkWireGuard{ + BindEndpoint: net.JoinHostPort("0.0.0.0", wireguardDefaultPort), + AdvertisedEndpoint: net.JoinHostPort(localIP, wireguardDefaultPort), + }, + EventSinkPort: 8090, + LogServerPort: 8092, + JoinTokensMode: JoinTokensModeLegacyOnly, + }, + MachineAPI: MachineAPI{ + BindEndpoint: net.JoinHostPort(localIP, "8090"), + }, + LoadBalancer: LoadBalancerService{ + MinPort: 10000, + MaxPort: 35000, - Embedded: true, - EmbeddedDBPath: "_out/etcd/", + DialTimeout: 15 * time.Second, + KeepAlivePeriod: 30 * time.Second, + TCPUserTimeout: 30 * time.Second, + + HealthCheckInterval: 20 * time.Second, + HealthCheckTimeout: 15 * time.Second, + }, + LocalResourceService: LocalResourceService{ + Enabled: true, + Port: 8081, + }, + EmbeddedDiscoveryService: EmbeddedDiscoveryService{ + Enabled: true, + Port: 8093, + SnapshotsEnabled: true, + SnapshotsPath: "_out/secondary-storage/discovery-service-state.binpb", + SnapshotsInterval: 10 * time.Minute, + LogLevel: zapcore.WarnLevel.String(), + }, + WorkloadProxy: WorkloadProxy{ + Subdomain: "proxy-us", + Enabled: true, }, }, - - SecondaryStorage: BoltDBParams{ - Path: "_out/secondary-storage/bolt.db", + Auth: Auth{ + KeyPruner: KeyPrunerConfig{ + Interval: 10 * time.Minute, + }, + InitialServiceAccount: InitialServiceAccount{ + Enabled: false, + Role: string(role.Admin), + KeyPath: "_out/initial-service-account-key", + Name: "automation", + Lifetime: time.Hour, + }, }, - - WorkloadProxying: WorkloadProxyingParams{ - Enabled: true, - Subdomain: "proxy-us", + Registries: Registries{ + Talos: consts.TalosRegistry, + Kubernetes: consts.KubernetesRegistry, + ImageFactoryBaseURL: consts.ImageFactoryBaseURL, }, - - ConfigDataCompression: ConfigDataCompressionParams{ - Enabled: true, + Logs: Logs{ + Audit: LogsAudit{ + Path: "_out/audit", + }, + ResourceLogger: ResourceLoggerConfig{ + LogLevel: zapcore.InfoLevel.String(), + Types: common.UserManagedResourceTypes, + }, + Machine: LogsMachine{ + BufferInitialCapacity: 16384, + BufferMaxCapacity: 131072, + BufferSafetyGap: 256, + Storage: LogsMachineStorage{ + Enabled: true, + Path: "_out/logs", + FlushPeriod: 10 * time.Minute, + FlushJitter: 0.1, + NumCompressedChunks: 5, + }, + }, }, + Storage: Storage{ + Secondary: BoltDB{ + Path: "_out/secondary-storage/bolt.db", + }, + Default: StorageDefault{ + Kind: "etcd", + Boltdb: BoltDB{ + Path: "_out/omni.db", + }, + Etcd: EtcdParams{ + Endpoints: []string{"http://localhost:2379"}, + DialKeepAliveTime: 30 * time.Second, + DialKeepAliveTimeout: 5 * time.Second, + CAFile: "etcd/ca.crt", + CertFile: "etcd/client.crt", + KeyFile: "etcd/client.key", - InitialServiceAccount: InitialServiceAccount{ - Enabled: false, - Role: string(role.Admin), - KeyPath: "_out/initial-service-account-key", - Name: "automation", - Lifetime: time.Hour, + Embedded: true, + EmbeddedDBPath: "_out/etcd/", + }, + }, }, - - LocalResourceServerPort: 8081, - - EtcdBackup: EtcdBackupParams{ + Features: Features{ + EnableConfigDataCompression: true, + }, + EtcdBackup: EtcdBackup{ TickInterval: time.Minute, MinInterval: time.Hour, MaxInterval: 24 * time.Hour, Jitter: 10 * time.Minute, }, - - LogResourceUpdatesLogLevel: zapcore.InfoLevel.String(), - LogResourceUpdatesTypes: common.UserManagedResourceTypes, - - EmbeddedDiscoveryService: EmbeddedDiscoveryServiceParams{ - Enabled: true, - Port: 8093, - SnapshotsEnabled: true, - SnapshotPath: "_out/secondary-storage/discovery-service-state.binpb", - SnapshotInterval: 10 * time.Minute, - LogLevel: zapcore.WarnLevel.String(), + Debug: Debug{ + Server: DebugServer{ + Endpoint: ":9988", + }, }, - - JoinTokensMode: JoinTokensModeLegacyOnly, - RunDebugServer: true, } } diff --git a/internal/pkg/config/config_test.go b/internal/pkg/config/config_test.go new file mode 100644 index 00000000..a9d8b163 --- /dev/null +++ b/internal/pkg/config/config_test.go @@ -0,0 +1,90 @@ +// Copyright (c) 2025 Sidero Labs, Inc. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. + +// Package config contains the application config loading functions. +package config_test + +import ( + _ "embed" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/siderolabs/omni/internal/pkg/config" +) + +//go:embed testdata/config-full.yaml +var configFull []byte + +//go:embed testdata/invalid-join-token-mode.yaml +var configInvalidJoinTokenMode []byte + +//go:embed testdata/conflicting-auth.yaml +var conflictingAuth []byte + +//go:embed testdata/backups.yaml +var backups []byte + +//go:embed testdata/unknown-keys.yaml +var unknownKeys []byte + +func TestValidateConfig(t *testing.T) { + for _, tt := range []struct { + name string + validateErr string + loadErr string + config []byte + }{ + { + name: "empty", + config: []byte("{}"), + validateErr: "required", + }, + { + name: "full", + config: configFull, + }, + { + name: "invalid join tokens mode", + config: configInvalidJoinTokenMode, + validateErr: "JoinTokensMode", + }, + { + name: "conflicting auth", + config: conflictingAuth, + validateErr: "Field validation for 'Auth0' failed", + }, + { + name: "conflicting backups", + config: backups, + validateErr: "Field validation for 'LocalPath' failed", + }, + { + name: "unknown keys", + config: unknownKeys, + loadErr: "unknown keys found", + }, + } { + t.Run(tt.name, func(t *testing.T) { + cfg, err := config.FromBytes(tt.config) + if tt.loadErr != "" { + require.ErrorContains(t, err, tt.loadErr) + + return + } + + require.NoError(t, err) + + err = cfg.Validate() + if tt.validateErr != "" { + require.ErrorContains(t, err, tt.validateErr) + + return + } + + require.NoError(t, err) + }) + } +} diff --git a/internal/pkg/config/debug.go b/internal/pkg/config/debug.go new file mode 100644 index 00000000..64014f4f --- /dev/null +++ b/internal/pkg/config/debug.go @@ -0,0 +1,22 @@ +// Copyright (c) 2025 Sidero Labs, Inc. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. + +package config + +// Debug configures debugging tools of the Omni instance. +type Debug struct { + Server DebugServer `yaml:"server"` + Pprof DebugPprof `yaml:"pprof"` +} + +// DebugServer enables the debug server. +type DebugServer struct { + Endpoint string `yaml:"endpoint"` +} + +// DebugPprof enables pprof server. +type DebugPprof struct { + Endpoint string `yaml:"endpoint"` +} diff --git a/internal/pkg/config/features.go b/internal/pkg/config/features.go new file mode 100644 index 00000000..37031ea0 --- /dev/null +++ b/internal/pkg/config/features.go @@ -0,0 +1,15 @@ +// Copyright (c) 2025 Sidero Labs, Inc. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. + +package config + +// Features contains all Omni feature flags. +type Features struct { + EnableTalosPreReleaseVersions bool `yaml:"enableTalosPreReleaseVersions"` + EnableBreakGlassConfigs bool `yaml:"enableBreakGlassConfigs"` + EnableConfigDataCompression bool `yaml:"enableConfigDataCompression"` + + DisableControllerRuntimeCache bool `yaml:"disableControllerRuntimeCache"` +} diff --git a/internal/pkg/config/logs.go b/internal/pkg/config/logs.go new file mode 100644 index 00000000..0c13f0e5 --- /dev/null +++ b/internal/pkg/config/logs.go @@ -0,0 +1,67 @@ +// Copyright (c) 2025 Sidero Labs, Inc. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. + +package config + +import "time" + +// Logs configures logging of the Omni instance. +// +//nolint:govet +type Logs struct { + // Machine configures Talos machine logs handler. + Machine LogsMachine `yaml:"machine"` + // Audit configures audit logs handler. + Audit LogsAudit `yaml:"audit"` + // ResourceLogger configures resource logger. + ResourceLogger ResourceLoggerConfig `yaml:"resourceLogger"` + // Stripe enables reporting to stripe. + Stripe LogsStripe `yaml:"stripe"` +} + +// LogsMachine configures Talos machine logs handler. +type LogsMachine struct { + // Storage configures persistent machine log storage of the Omni instance. + Storage LogsMachineStorage `yaml:"storage"` + + BufferInitialCapacity int `yaml:"bufferInitialCapacity"` + BufferMaxCapacity int `yaml:"bufferMaxCapacity"` + BufferSafetyGap int `yaml:"bufferSafetyGap"` +} + +// LogsMachineStorage configures the machine logs storage. +// +//nolint:govet +type LogsMachineStorage struct { + Enabled bool `yaml:"enabled"` + // Path to store the logs in. + Path string `yaml:"path"` + // FlushPeriod is the period to use to flush the logs to disk. + FlushPeriod time.Duration `yaml:"flushPeriod"` + // FlushJitter flush period jitter. + FlushJitter float64 `yaml:"flushJitter"` + // NumCompressedChunks is the count of log chunks to keep in the logs history. + NumCompressedChunks int `yaml:"numCompressedChunks"` +} + +// LogsAudit configures audit logs peristence. +type LogsAudit struct { + // Path to store the audit logs in. + Path string `yaml:"path"` +} + +// ResourceLoggerConfig is the config for the Omni resource logger. +// This is the debug tool, that allows logging all resource changes to the stdout. +type ResourceLoggerConfig struct { + // LogLevel is the level of the logs to use when writing the data. + LogLevel string `yaml:"logLevel"` + // Types is the list of the resource types to log to stdout. + Types []string `yaml:"types"` +} + +// LogsStripe report usage metrics to stripe. +type LogsStripe struct { + Enabled bool `yaml:"enabled"` +} diff --git a/internal/pkg/config/services.go b/internal/pkg/config/services.go new file mode 100644 index 00000000..c92821b9 --- /dev/null +++ b/internal/pkg/config/services.go @@ -0,0 +1,273 @@ +// Copyright (c) 2025 Sidero Labs, Inc. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. + +package config + +import ( + "fmt" + "slices" + "strings" + "time" +) + +// HTTPService defines the interface for HTTP like services. +type HTTPService interface { + URL() string + GetCertFile() string + GetKeyFile() string + GetBindEndpoint() string + IsSecure() bool +} + +// Services configs. +// +//nolint:govet +type Services struct { + // API is the Omni gRPC API service, gateway and the frontend. + API Service `yaml:"api"` + // DevServerProxy is used in Omni development and allows proxying through Omni to the node JS dev server. + DevServerProxy DevServerProxyService `yaml:"devServerProxy"` + // Metrics exposes prometheus metrics. + Metrics Service `yaml:"metrics"` + // KubernetesProxy proxies the Kubernetes API to the clusters managed by Omni. + KubernetesProxy KubernetesProxyService `yaml:"kubernetesProxy"` + // Siderolink manages WireGuard connections to the Talos machines connected to Omni. + Siderolink SiderolinkService `yaml:"siderolink"` + // MachineAPI is the public API of Omni that helps to establish WireGuard connections. + MachineAPI MachineAPI `yaml:"machineAPI"` + // LocalResourceService runs COSI API service that gives readonly access to all resources. + LocalResourceService LocalResourceService `yaml:"localResourceService"` + // EmbeddedDiscoveryService runs https://discovery.talos.dev/ inside Omni. + EmbeddedDiscoveryService EmbeddedDiscoveryService `yaml:"embeddedDiscoveryService"` + // LoadBalancer configures Omni Kubernetes loadbalancer runner. + LoadBalancer LoadBalancerService `yaml:"loadBalancer"` + // WorkloadProxy runs the workload proxy service in Omni. + WorkloadProxy WorkloadProxy `yaml:"workloadProxy"` +} + +// Service is the base service config. +type Service struct { + BindEndpoint string `yaml:"endpoint"` + // AdvertisedURL should be used when Omni runs behind an ingress. + // This value is used in the machine join config, kernel params and schematics generation. + AdvertisedURL string `yaml:"advertisedURL"` + // CertFile is the TLS cert. + CertFile string `yaml:"certFile"` + // KeyFile is the TLS key. + KeyFile string `yaml:"keyFile"` +} + +// GetBindEndpoint implements HTTPService. +func (s *Service) GetBindEndpoint() string { + return s.BindEndpoint +} + +// GetCertFile implements HTTPService. +func (s *Service) GetCertFile() string { + return s.CertFile +} + +// GetKeyFile implements HTTPService. +func (s *Service) GetKeyFile() string { + return s.KeyFile +} + +// IsSecure returns true if both cert file and key file are present. +func (s *Service) IsSecure() bool { + return s.CertFile != "" && s.KeyFile != "" +} + +// URL gets the URL from the endpoint. +func (s *Service) URL() string { + if s.AdvertisedURL != "" { + return s.AdvertisedURL + } + + schema := "http" + if s.IsSecure() { + schema = "https" + } + + return fmt.Sprintf("%s://%s", schema, s.BindEndpoint) +} + +// DevServerProxyService is used in Omni development and allows proxying through Omni to the node JS dev server. +type DevServerProxyService struct { + Service `yaml:",inline"` + ProxyTo string `yaml:"proxyTo"` +} + +// KubernetesProxyService is the base service config. +type KubernetesProxyService struct { + BindEndpoint string `yaml:"endpoint"` + // AdvertisedURL should be used when Omni runs behind an ingress. + // This value is used in the machine join config, kernel params and schematics generation. + AdvertisedURL string `yaml:"advertisedURL"` + // CertFile is the TLS cert. + CertFile string `yaml:"certFile" validate:"required"` + // KeyFile is the TLS key. + KeyFile string `yaml:"keyFile" validate:"required"` +} + +// GetBindEndpoint implements HTTPService. +func (ks *KubernetesProxyService) GetBindEndpoint() string { + return ks.BindEndpoint +} + +// GetCertFile implements HTTPKubernetesProxyService. +func (ks *KubernetesProxyService) GetCertFile() string { + return ks.CertFile +} + +// GetKeyFile implements HTTPKubernetesProxyService. +func (ks *KubernetesProxyService) GetKeyFile() string { + return ks.KeyFile +} + +// IsSecure returns true if both cert file and key file are present. +func (ks *KubernetesProxyService) IsSecure() bool { + return ks.CertFile != "" && ks.KeyFile != "" +} + +// URL returns kubernetes services URL. +// It is always HTTPS. +func (ks *KubernetesProxyService) URL() string { + if ks.AdvertisedURL != "" { + return ks.AdvertisedURL + } + + return fmt.Sprintf("https://%s", ks.BindEndpoint) +} + +// SiderolinkService manages WireGuard connections to the Talos machines connected to Omni. +type SiderolinkService struct { + WireGuard SiderolinkWireGuard `yaml:"wireGuard"` + // JoinTokensMode controls Talos machine join tokens operation mode. + // - strict - only for Talos >= 1.6.x + // - legacyAllowed - relies on the legacy join tokens mode for Talos < 1.6.x (less secure, use only if Talos upgrade is not an option) + // - legacy - does not use node unique join tokens mode + JoinTokensMode JoinTokensMode `yaml:"joinTokensMode" validate:"oneof=strict legacyAllowed legacy"` + // DisableLastEndpoint disables populating last known peer endpoint for the WireGuard peers. + // Using last known peer endpoints helps Omni quicker re-establish WireGuard connection to the nodes + // after it is restarted. + // Enable this flag if Omni runs behind the ingress and doesn't see the real node IPs. + DisableLastEndpoint bool `yaml:"disableLastEndpoint"` + // UsegRPCTunnel forces using WireGuard over gRPC for all machines on the account. + UseGRPCTunnel bool `yaml:"useGRPCTunnel"` + // EventSinkPort is the port where Talos nodes send Talos events. + // This port is only open on the WireGuard tunnel Omni endpoint. + EventSinkPort int `yaml:"eventSinkPort"` + // LogServerPort is the port where Talos nodes send console logs. + // This port is only open on the WireGuard tunnel Omni endpoint. + LogServerPort int `yaml:"logServerPort"` +} + +// SiderolinkWireGuard defines siderolink wireguard endpoint config. +type SiderolinkWireGuard struct { + BindEndpoint string `yaml:"endpoint"` + AdvertisedEndpoint string `yaml:"advertisedEndpoint"` +} + +// MachineAPI is the public API of Omni that helps to establish WireGuard connections. +// This API used to exchange WireGuard keys, assign IP addresses. +// If gRPC tunnel mode is used, WireGuard traffic goes over this endpoint too. +type MachineAPI Service + +// URL composes URL for Talos to connect. +func (m MachineAPI) URL() string { + if m.AdvertisedURL != "" { + return m.AdvertisedURL + } + + schema := "grpc" + if m.CertFile != "" && m.KeyFile != "" { + schema = "https" + } + + return fmt.Sprintf("%s://%s", schema, m.BindEndpoint) +} + +// LocalResourceService runs COSI API service that gives readonly access to all resources. +type LocalResourceService struct { + Enabled bool `yaml:"enabled"` + Port int `yaml:"port"` +} + +// EmbeddedDiscoveryService runs https://discovery.talos.dev/ inside Omni. +// Discovery service is only available inside the WireGuard tunnel +// +//nolint:govet +type EmbeddedDiscoveryService struct { + Enabled bool `yaml:"enabled"` + Port int `yaml:"port"` + + // SnapshotsEnabled turns on the discovery service persistence. + SnapshotsEnabled bool `yaml:"snapshotsEnabled"` + // SnapshotsPath is the path on disk where to store the discovery service state. + SnapshotsPath string `yaml:"snapshotsPath"` + SnapshotsInterval time.Duration `yaml:"snapshotsInterval"` + LogLevel string `yaml:"logLevel"` +} + +// LoadBalancerService configures Omni Kubernetes loadbalancer. +type LoadBalancerService struct { + // MinPort is the minimum port number used for load balancer endpoints. + MinPort int `yaml:"minPort"` + // MaxPort is the maximum port number used for load balancer endpoints. + MaxPort int `yaml:"maxPort"` + + DialTimeout time.Duration `yaml:"dialTimeout"` + KeepAlivePeriod time.Duration `yaml:"keepAlivePeriod"` + TCPUserTimeout time.Duration `yaml:"tcpUserTimeout"` + + HealthCheckInterval time.Duration `yaml:"healthCheckInterval"` + HealthCheckTimeout time.Duration `yaml:"healthCheckTimeout"` +} + +// WorkloadProxy configures workload proxy. +type WorkloadProxy struct { + Subdomain string `yaml:"subdomain"` + Enabled bool `yaml:"enabled"` +} + +// JoinTokensMode is the join token operation mode config. +// +//nolint:recvcheck +type JoinTokensMode string + +// String implements pflag.Value. +func (s JoinTokensMode) String() string { + return string(s) +} + +// Set implements pflag.Value. +func (s *JoinTokensMode) Set(value string) error { + if !slices.Contains(s.values(), value) { + return fmt.Errorf("should be one of %s", strings.Join(s.values(), ", ")) + } + + *s = JoinTokensMode(value) + + return nil +} + +// Type implements pflag.Value. +func (s JoinTokensMode) Type() string { + return fmt.Sprintf("[%s]", strings.Join(s.values(), ",")) +} + +func (JoinTokensMode) values() []string { + return []string{JoinTokensModeLegacyOnly, JoinTokensModeLegacyAllowed, JoinTokensModeStrict} +} + +const ( + // JoinTokensModeLegacyOnly disables node unique token flow, uses only join token when letting the machine into the system. + JoinTokensModeLegacyOnly = "legacy" + // JoinTokensModeLegacyAllowed allows joining Talos nodes which do not support node unique token flow + // uses unique token flow only for the machines which support it. + JoinTokensModeLegacyAllowed = "legacyAllowed" + // JoinTokensModeStrict rejects the machines that do not support node unique tokens flow. + JoinTokensModeStrict = "strict" +) diff --git a/internal/pkg/config/storage.go b/internal/pkg/config/storage.go new file mode 100644 index 00000000..269785d7 --- /dev/null +++ b/internal/pkg/config/storage.go @@ -0,0 +1,57 @@ +// Copyright (c) 2025 Sidero Labs, Inc. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. + +package config + +import "time" + +// Storage defines Omni COSI state storage configuration. +type Storage struct { + // Vault configuration where the state encryption keys are present. + Vault Vault `yaml:"vault"` + // Secondary storage is used to store the metrics and any frequently changed resources + // which might overflow etcd resource history. + Secondary BoltDB `yaml:"secondary" validate:"required"` + // Default is the storage used for the default resource namespace in Omni. + Default StorageDefault `yaml:"default" validate:"required"` +} + +// Vault allows setting vault configuration through the config file. +type Vault struct { + URL string `yaml:"url"` + Token string `yaml:"token"` +} + +// StorageDefault defines storage configs. +type StorageDefault struct { + // Kind can be either 'boltdb' or 'etcd'. + Kind string `yaml:"kind" validate:"oneof=etcd boltdb"` + Boltdb BoltDB `yaml:"boltdb"` + Etcd EtcdParams `yaml:"etcd"` +} + +// BoltDB defines boltdb storage configs. +type BoltDB struct { + Path string `yaml:"path"` +} + +// EtcdParams defines etcd storage configs. +type EtcdParams struct { ///nolint:govet + // External etcd: list of endpoints, as host:port pairs. + Endpoints []string `yaml:"endpoints"` + DialKeepAliveTime time.Duration `yaml:"dialKeepAliveTime"` + DialKeepAliveTimeout time.Duration `yaml:"dialKeepAliveTimeout"` + CAFile string `yaml:"caFile"` + CertFile string `yaml:"certFile"` + KeyFile string `yaml:"keyFile"` + + // Use embedded etcd server (no clustering). + Embedded bool `yaml:"embedded"` + EmbeddedDBPath string `yaml:"embeddedDBPath"` + EmbeddedUnsafeFsync bool `yaml:"embeddedUnsafeFsync"` + + PrivateKeySource string `yaml:"privateKeySource" validate:"required"` + PublicKeyFiles []string `yaml:"publicKeyFiles"` +} diff --git a/internal/pkg/config/testdata/backups.yaml b/internal/pkg/config/testdata/backups.yaml new file mode 100644 index 00000000..7da50f3b --- /dev/null +++ b/internal/pkg/config/testdata/backups.yaml @@ -0,0 +1,69 @@ +--- +account: + id: "uuid" + name: "artem" + +services: + api: + endpoint: 0.0.0.0:8099 + kubernetesProxy: + endpoint: 0.0.0.0:8095 + certFile: certFile + keyFile: keyFile + siderolink: + joinTokensMode: strict + +auth: + keyPruner: + interval: 1m + auth0: + enabled: true + +logs: + machine: + storage: + enabled: true + path: "_out/logs" + flushPeriod: 10m + flushJitter: 0.1 + audit: + path: _out/audit + resourceLogger: + types: + - Links.omni.siderolabs.dev + logLevel: Info + stripe: + enabled: true + +registries: + talos: ghcr.io/siderolabs/installer + kubernetes: ghcr.io/siderolabs/kubelet + imageFactoryBaseURL: https://factory.talos.dev + +storage: + vault: + url: http://127.0.0.1:8200 + token: dev-o-token + secondary: + path: "_out/secondary-storage/bolt.db" + default: + kind: etcd + boltdb: + path: "_out/omni.db" + etcd: + endpoints: + - http://localhost:2379 + dialKeepAliveTime: 30s + dialKeepAliveTimeout: 5s + caFile: etcd/ca.crt + certFile: etcd/client.crt + keyFile: etcd/client.key + embedded: true + privateKeySource: "vault://secret/omni-private-key" + publicKeyFiles: + - "internal/backend/runtime/omni/testdata/pgp/new_key.public" + embeddedUnsafeFsync: true + embeddedDBPath: _out/etcd/ +etcdBackup: + s3Enabled: true + localPath: "/hi" diff --git a/internal/pkg/config/testdata/config-full.yaml b/internal/pkg/config/testdata/config-full.yaml new file mode 100644 index 00000000..dc33e9e3 --- /dev/null +++ b/internal/pkg/config/testdata/config-full.yaml @@ -0,0 +1,120 @@ +--- +account: + id: "uuid" + name: "artem" + +services: + api: + endpoint: 0.0.0.0:8099 + metrics: + endpoint: 0.0.0.0:2122 + kubernetesProxy: + endpoint: 0.0.0.0:8095 + certFile: certFile + keyFile: keyFile + siderolink: + wireGuard: + endpoint: localhost:50180 + advertisedEndpoint: 192.168.88.219:50180 + disableLastEndpoint: true + useGRPCTunnel: true + eventSinkPort: 8091 + logServerPort: 8092 + joinTokensMode: strict + machineAPI: + endpoint: 0.0.0.0:8090 + advertisedURL: "grpc://192.168.88.219:8090" + certFile: hack/certs/api.cert + keyFile: hack/certs/api.key + localResourceService: + enabled: false + port: 8081 + embeddedDiscoveryService: + enabled: true + port: 8093 + snapshotsEnabled: true + snapshotsPath: "_out/secondary-storage/discovery-service-state.binpb" + logLevel: Warn + loadBalancer: + minPort: 10000 + maxPort: 20000 + dialTimeout: 30s + devServerProxy: + endpoint: 0.0.0.0:8120 + workloadProxy: + enabled: true + subdomain: "proxy-us" + +debug: + server: + endpoint: 0.0.0.0:9988 + pprof: + endpoint: 0.0.0.0:8124 + +auth: + keyPruner: + interval: 1m + auth0: + enabled: true + clientID: TODO + domain: TODO + initialUsers: + - test-user@siderolabs.com + initialServiceAccount: + enabled: true + role: Admin + keyPath: _out/test-sa + name: tests + lifetime: 1m + +registries: + talos: ghcr.io/siderolabs/installer + kubernetes: ghcr.io/siderolabs/kubelet + imageFactoryBaseURL: https://factory.talos.dev + +storage: + vault: + url: http://127.0.0.1:8200 + token: dev-o-token + secondary: + path: "_out/secondary-storage/bolt.db" + default: + kind: etcd + boltdb: + path: "_out/omni.db" + etcd: + endpoints: + - http://localhost:2379 + dialKeepAliveTime: 30s + dialKeepAliveTimeout: 5s + caFile: etcd/ca.crt + certFile: etcd/client.crt + keyFile: etcd/client.key + embedded: true + privateKeySource: "vault://secret/omni-private-key" + publicKeyFiles: + - "internal/backend/runtime/omni/testdata/pgp/new_key.public" + embeddedUnsafeFsync: true + embeddedDBPath: _out/etcd/ + +logs: + machine: + storage: + enabled: true + path: "_out/logs" + flushPeriod: 10m + flushJitter: 0.1 + audit: + path: _out/audit + resourceLogger: + types: + - Links.omni.siderolabs.dev + logLevel: Info + stripe: + enabled: true + +features: + enableTalosPreReleaseVersions: true + enableConfigDataCompression: true + enableBreakGlassConfigs: true + disableControllerRuntimeCache: false diff --git a/internal/pkg/config/testdata/conflicting-auth.yaml b/internal/pkg/config/testdata/conflicting-auth.yaml new file mode 100644 index 00000000..b049430c --- /dev/null +++ b/internal/pkg/config/testdata/conflicting-auth.yaml @@ -0,0 +1,72 @@ +--- +account: + id: "uuid" + name: "artem" + +services: + api: + endpoint: 0.0.0.0:8099 + kubernetesProxy: + endpoint: 0.0.0.0:8095 + certFile: certFile + keyFile: keyFile + +auth: + keyPruner: + interval: 1m + auth0: + enabled: true + saml: + enabled: true + +registries: + talos: ghcr.io/siderolabs/installer + kubernetes: ghcr.io/siderolabs/kubelet + imageFactoryBaseURL: https://factory.talos.dev + +storage: + vault: + url: http://127.0.0.1:8200 + token: dev-o-token + secondary: + path: "_out/secondary-storage/bolt.db" + default: + kind: etcd + boltdb: + path: "_out/omni.db" + etcd: + endpoints: + - http://localhost:2379 + dialKeepAliveTime: 30s + dialKeepAliveTimeout: 5s + caFile: etcd/ca.crt + certFile: etcd/client.crt + keyFile: etcd/client.key + embedded: true + privateKeySource: "vault://secret/omni-private-key" + publicKeyFiles: + - "internal/backend/runtime/omni/testdata/pgp/new_key.public" + embeddedUnsafeFsync: true + embeddedDBPath: _out/etcd/ + +logs: + machine: + storage: + enabled: true + path: "_out/logs" + flushPeriod: 10m + flushJitter: 0.1 + audit: + path: _out/audit + resourceLogger: + types: + - Links.omni.siderolabs.dev + logLevel: Info + stripe: + enabled: true + +features: + enableTalosPreReleaseVersions: true + enableConfigDataCompression: true + enableBreakGlassConfigs: true + disableControllerRuntimeCache: false diff --git a/internal/pkg/config/testdata/invalid-join-token-mode.yaml b/internal/pkg/config/testdata/invalid-join-token-mode.yaml new file mode 100644 index 00000000..19d9c69c --- /dev/null +++ b/internal/pkg/config/testdata/invalid-join-token-mode.yaml @@ -0,0 +1,71 @@ + +--- +account: + id: "uuid" + name: "artem" + +services: + api: + endpoint: 0.0.0.0:8099 + kubernetesProxy: + endpoint: 0.0.0.0:8095 + certFile: certFile + keyFile: keyFile + siderolink: + joinTokensMode: nope + +auth: + keyPruner: + interval: 1m + +registries: + talos: ghcr.io/siderolabs/installer + kubernetes: ghcr.io/siderolabs/kubelet + imageFactoryBaseURL: https://factory.talos.dev + +storage: + vault: + url: http://127.0.0.1:8200 + token: dev-o-token + secondary: + path: "_out/secondary-storage/bolt.db" + default: + kind: etcd + boltdb: + path: "_out/omni.db" + etcd: + endpoints: + - http://localhost:2379 + dialKeepAliveTime: 30s + dialKeepAliveTimeout: 5s + caFile: etcd/ca.crt + certFile: etcd/client.crt + keyFile: etcd/client.key + embedded: true + privateKeySource: "vault://secret/omni-private-key" + publicKeyFiles: + - "internal/backend/runtime/omni/testdata/pgp/new_key.public" + embeddedUnsafeFsync: true + embeddedDBPath: _out/etcd/ + +logs: + machine: + storage: + enabled: true + path: "_out/logs" + flushPeriod: 10m + flushJitter: 0.1 + audit: + path: _out/audit + resourceLogger: + types: + - Links.omni.siderolabs.dev + logLevel: Info + stripe: + enabled: true + +features: + enableTalosPreReleaseVersions: true + enableConfigDataCompression: true + enableBreakGlassConfigs: true + disableControllerRuntimeCache: false diff --git a/internal/pkg/config/testdata/unknown-keys.yaml b/internal/pkg/config/testdata/unknown-keys.yaml new file mode 100644 index 00000000..8d33590f --- /dev/null +++ b/internal/pkg/config/testdata/unknown-keys.yaml @@ -0,0 +1,124 @@ +--- +account: + id: "uuid" + name: "artem" + +services: + api: + endpoint: 0.0.0.0:8099 + metrics: + endpoint: 0.0.0.0:2122 + kubernetesProxy: + endpoint: 0.0.0.0:8095 + certFile: certFile + keyFile: keyFile + siderolink: + wireGuard: + endpoint: localhost:50180 + advertisedEndpoint: 192.168.88.219:50180 + disableLastEndpoint: true + useGRPCTunnel: true + eventSinkPort: 8091 + logServerPort: 8092 + joinTokensMode: strict + machineAPI: + endpoint: 0.0.0.0:8090 + advertisedURL: "grpc://192.168.88.219:8090" + certFile: hack/certs/api.cert + keyFile: hack/certs/api.key + localResourceService: + enabled: false + port: 8081 + embeddedDiscoveryService: + enabled: true + port: 8093 + snapshotsEnabled: true + snapshotsPath: "_out/secondary-storage/discovery-service-state.binpb" + logLevel: Warn + loadBalancer: + minPort: 10000 + maxPort: 20000 + dialTimeout: 30s + devServerProxy: + endpoint: 0.0.0.0:8120 + workloadProxy: + enabled: true + subdomain: "proxy-us" + +debug: + server: + enabled: true + endpoint: 0.0.0.0:9988 + pprof: + enabled: true + endpoint: 0.0.0.0:8124 + +auth: + keyPruner: + interval: 1m + auth0: + enabled: true + clientID: TODO + domain: TODO + initialUsers: + - test-user@siderolabs.com + initialServiceAccount: + enabled: true + role: Admin + keyPath: _out/test-sa + name: tests + lifetime: 1m + +registries: + talos: ghcr.io/siderolabs/installer + kubernetes: ghcr.io/siderolabs/kubelet + imageFactoryBaseURL: https://factory.talos.dev + +storage: + vault: + url: http://127.0.0.1:8200 + token: dev-o-token + secondary: + path: "_out/secondary-storage/bolt.db" + default: + kind: etcd + boltdb: + path: "_out/omni.db" + etcd: + endpoints: + - http://localhost:2379 + dialKeepAliveTime: 30s + dialKeepAliveTimeout: 5s + caFile: etcd/ca.crt + certFile: etcd/client.crt + keyFile: etcd/client.key + embedded: true + privateKeySource: "vault://secret/omni-private-key" + publicKeyFiles: + - "internal/backend/runtime/omni/testdata/pgp/new_key.public" + embeddedUnsafeFsync: true + embeddedDBPath: _out/etcd/ + +logs: + machine: + storage: + enabled: true + path: "_out/logs" + flushPeriod: 10m + flushJitter: 0.1 + audit: + path: _out/audit + resourceLogger: + types: + - Links.omni.siderolabs.dev + logLevel: Info + stripe: + enabled: true + +features: + enableTalosPreReleaseVersions: true + enableConfigDataCompression: true + enableBreakGlassConfigs: true + disableControllerRuntimeCache: false + +johndoe: unknown diff --git a/internal/pkg/features/features.go b/internal/pkg/features/features.go index 5264f4f2..fc6cbf5d 100644 --- a/internal/pkg/features/features.go +++ b/internal/pkg/features/features.go @@ -24,16 +24,16 @@ import ( // UpdateResources creates or updates the features omni.FeaturesConfig resource with the current feature flags. func UpdateResources(ctx context.Context, st state.State, logger *zap.Logger) error { updateFeaturesConfig := func(res *omni.FeaturesConfig) error { - res.TypedSpec().Value.EnableWorkloadProxying = config.Config.WorkloadProxying.Enabled - res.TypedSpec().Value.EmbeddedDiscoveryService = config.Config.EmbeddedDiscoveryService.Enabled + res.TypedSpec().Value.EnableWorkloadProxying = config.Config.Services.WorkloadProxy.Enabled + res.TypedSpec().Value.EmbeddedDiscoveryService = config.Config.Services.EmbeddedDiscoveryService.Enabled res.TypedSpec().Value.EtcdBackupSettings = &specs.EtcdBackupSettings{ TickInterval: durationpb.New(config.Config.EtcdBackup.TickInterval), MinInterval: durationpb.New(config.Config.EtcdBackup.MinInterval), MaxInterval: durationpb.New(config.Config.EtcdBackup.MaxInterval), } - res.TypedSpec().Value.AuditLogEnabled = config.Config.AuditLogDir != "" - res.TypedSpec().Value.ImageFactoryBaseUrl = config.Config.ImageFactoryBaseURL + res.TypedSpec().Value.AuditLogEnabled = config.Config.Logs.Audit.Path != "" + res.TypedSpec().Value.ImageFactoryBaseUrl = config.Config.Registries.ImageFactoryBaseURL return nil } @@ -55,7 +55,7 @@ func UpdateResources(ctx context.Context, st state.State, logger *zap.Logger) er return fmt.Errorf("failed to create features config: %w", err) } - logger.Info("created features config resource", zap.Bool("enable_workload_proxying", config.Config.WorkloadProxying.Enabled)) + logger.Info("created features config resource", zap.Bool("enable_workload_proxying", config.Config.Services.WorkloadProxy.Enabled)) return nil } @@ -64,7 +64,7 @@ func UpdateResources(ctx context.Context, st state.State, logger *zap.Logger) er return fmt.Errorf("failed to update features config: %w", err) } - logger.Info("updated features config resource", zap.Bool("enable_workload_proxying", config.Config.WorkloadProxying.Enabled)) + logger.Info("updated features config resource", zap.Bool("enable_workload_proxying", config.Config.Services.WorkloadProxy.Enabled)) return nil } diff --git a/internal/pkg/siderolink/loghandler.go b/internal/pkg/siderolink/loghandler.go index c85ac161..ecd9db6f 100644 --- a/internal/pkg/siderolink/loghandler.go +++ b/internal/pkg/siderolink/loghandler.go @@ -23,7 +23,7 @@ import ( ) // NewLogHandler returns a new LogHandler. -func NewLogHandler(machineMap *MachineMap, omniState state.State, storageConfig *config.MachineLogConfigParams, logger *zap.Logger) (*LogHandler, error) { +func NewLogHandler(machineMap *MachineMap, omniState state.State, storageConfig *config.LogsMachine, logger *zap.Logger) (*LogHandler, error) { cache, err := NewMachineCache(storageConfig, logger) if err != nil { return nil, fmt.Errorf("failed to create machine cache: %w", err) diff --git a/internal/pkg/siderolink/loghandler_test.go b/internal/pkg/siderolink/loghandler_test.go index da15e626..b9a83e90 100644 --- a/internal/pkg/siderolink/loghandler_test.go +++ b/internal/pkg/siderolink/loghandler_test.go @@ -35,13 +35,15 @@ import ( ) func TestLogHandler_HandleMessage(t *testing.T) { - storageConfig := config.MachineLogConfigParams{ + storageConfig := config.LogsMachine{ BufferInitialCapacity: 16, BufferMaxCapacity: 128, BufferSafetyGap: 16, - NumCompressedChunks: 5, - StorageFlushPeriod: time.Second, - StorageEnabled: false, + Storage: config.LogsMachineStorage{ + NumCompressedChunks: 5, + FlushPeriod: time.Second, + Enabled: false, + }, } t.Run("empty log message", func(t *testing.T) { @@ -189,11 +191,13 @@ func TestLogHandlerStorage(t *testing.T) { tempDir := t.TempDir() logger := zaptest.NewLogger(t) - logHandler, err := siderolink.NewLogHandler(machineMap, st, &config.MachineLogConfigParams{ - StorageEnabled: true, - StoragePath: tempDir, - StorageFlushPeriod: 2 * time.Second, - NumCompressedChunks: 5, + logHandler, err := siderolink.NewLogHandler(machineMap, st, &config.LogsMachine{ + Storage: config.LogsMachineStorage{ + Enabled: true, + Path: tempDir, + FlushPeriod: 2 * time.Second, + NumCompressedChunks: 5, + }, BufferInitialCapacity: 16, BufferMaxCapacity: 128, BufferSafetyGap: 16, @@ -274,11 +278,13 @@ func TestLogHandlerStorageLegacyMigration(t *testing.T) { require.NoError(t, os.WriteFile(legacyLogPath, legacyLog, 0o644)) require.NoError(t, os.WriteFile(legacyLogHashPath, []byte(hex.EncodeToString(legacyHash[:])), 0o644)) - logHandler, err := siderolink.NewLogHandler(machineMap, st, &config.MachineLogConfigParams{ - StorageEnabled: true, - StoragePath: tempDir, - StorageFlushPeriod: 1 * time.Second, - NumCompressedChunks: 5, + logHandler, err := siderolink.NewLogHandler(machineMap, st, &config.LogsMachine{ + Storage: config.LogsMachineStorage{ + Enabled: true, + Path: tempDir, + FlushPeriod: 1 * time.Second, + NumCompressedChunks: 5, + }, BufferInitialCapacity: 16, BufferMaxCapacity: 128, BufferSafetyGap: 16, @@ -327,10 +333,12 @@ func TestLogHandlerStorageDisabled(t *testing.T) { require.NoError(t, st.Create(ctx, omni.NewMachine(resources.DefaultNamespace, "machine-2"))) tempDir := t.TempDir() - storageConfig := config.MachineLogConfigParams{ - StorageEnabled: false, - StoragePath: tempDir, - StorageFlushPeriod: 100 * time.Millisecond, + storageConfig := config.LogsMachine{ + Storage: config.LogsMachineStorage{ + Enabled: false, + Path: tempDir, + FlushPeriod: 100 * time.Millisecond, + }, } handler, err := siderolink.NewLogHandler(machineMap, st, &storageConfig, zaptest.NewLogger(t)) diff --git a/internal/pkg/siderolink/machines.go b/internal/pkg/siderolink/machines.go index e2b0b81c..6e21fc2f 100644 --- a/internal/pkg/siderolink/machines.go +++ b/internal/pkg/siderolink/machines.go @@ -36,14 +36,14 @@ import ( type MachineCache struct { machineBuffers containers.LazyMap[MachineID, *circular.Buffer] logger *zap.Logger - logStorageConfig *config.MachineLogConfigParams + logStorageConfig *config.LogsMachine compressor *zstd.Compressor mx sync.Mutex inited bool } // NewMachineCache returns a new MachineCache. -func NewMachineCache(logStorageConfig *config.MachineLogConfigParams, logger *zap.Logger) (*MachineCache, error) { +func NewMachineCache(logStorageConfig *config.LogsMachine, logger *zap.Logger) (*MachineCache, error) { compressor, err := zstd.NewCompressor() if err != nil { return nil, fmt.Errorf("failed to create log compressor: %w", err) @@ -92,13 +92,13 @@ func (m *MachineCache) GetBuffer(id MachineID) (*circular.Buffer, error) { return val, nil } - if !m.logStorageConfig.StorageEnabled { + if !m.logStorageConfig.Storage.Enabled { return nil, &BufferNotFoundError{id: id} } // Probe the file system to check if a log file exists for this machine. // Check both for the old (/path/machine-id.log) and the new (/path/machine-id.log.NUM) format. - glob := filepath.Join(m.logStorageConfig.StoragePath, string(id)+".log*") + glob := filepath.Join(m.logStorageConfig.Storage.Path, string(id)+".log*") matches, err := filepath.Glob(glob) if err != nil { @@ -149,7 +149,7 @@ func (m *MachineCache) Remove(id MachineID) error { m.machineBuffers.Remove(id) - matches, err := filepath.Glob(filepath.Join(m.logStorageConfig.StoragePath, string(id)+".log*")) + matches, err := filepath.Glob(filepath.Join(m.logStorageConfig.Storage.Path, string(id)+".log*")) if err != nil { return fmt.Errorf("failed to list log files for machine %q: %w", id, err) } @@ -194,15 +194,15 @@ func (m *MachineCache) init() { circular.WithInitialCapacity(m.logStorageConfig.BufferInitialCapacity), circular.WithMaxCapacity(m.logStorageConfig.BufferMaxCapacity), circular.WithSafetyGap(m.logStorageConfig.BufferSafetyGap), - circular.WithNumCompressedChunks(m.logStorageConfig.NumCompressedChunks, m.compressor), + circular.WithNumCompressedChunks(m.logStorageConfig.Storage.NumCompressedChunks, m.compressor), circular.WithLogger(m.logger), } - if m.logStorageConfig.StorageEnabled { + if m.logStorageConfig.Storage.Enabled { bufferOpts = append(bufferOpts, circular.WithPersistence(circular.PersistenceOptions{ - ChunkPath: filepath.Join(m.logStorageConfig.StoragePath, string(id)+".log"), - FlushInterval: m.logStorageConfig.StorageFlushPeriod, - FlushJitter: m.logStorageConfig.StorageFlushJitter, + ChunkPath: filepath.Join(m.logStorageConfig.Storage.Path, string(id)+".log"), + FlushInterval: m.logStorageConfig.Storage.FlushPeriod, + FlushJitter: m.logStorageConfig.Storage.FlushJitter, })) } @@ -211,7 +211,7 @@ func (m *MachineCache) init() { return nil, fmt.Errorf("failed to create circular buffer for machine '%s': %w", id, err) } - if m.logStorageConfig.StorageEnabled { + if m.logStorageConfig.Storage.Enabled { loadLegacyLogs(m.logStorageConfig, id, buffer, m.logger) } @@ -327,8 +327,8 @@ func IsBufferNotFoundError(err error) bool { // It removes the old log file and its hash file regardless of the result. // // It is a best-effort function and does not return any error. -func loadLegacyLogs(config *config.MachineLogConfigParams, id MachineID, writer io.Writer, logger *zap.Logger) { - filePath := filepath.Join(config.StoragePath, fmt.Sprintf("%s.log", id)) +func loadLegacyLogs(config *config.LogsMachine, id MachineID, writer io.Writer, logger *zap.Logger) { + filePath := filepath.Join(config.Storage.Path, fmt.Sprintf("%s.log", id)) shaSumPath := filePath + ".sha256sum" defer func() { diff --git a/internal/pkg/siderolink/manager.go b/internal/pkg/siderolink/manager.go index 59632249..09bb0e90 100644 --- a/internal/pkg/siderolink/manager.go +++ b/internal/pkg/siderolink/manager.go @@ -109,8 +109,8 @@ func NewManager( provisionServer: NewProvisionHandler( logger, state, - config.Config.JoinTokensMode, - config.Config.SiderolinkUseGRPCTunnel, + config.Config.Services.Siderolink.JoinTokensMode, + config.Config.Services.Siderolink.UseGRPCTunnel, ), } @@ -192,34 +192,34 @@ func generateJoinToken() (string, error) { type Params struct { WireguardEndpoint string AdvertisedEndpoint string - APIEndpoint string - Cert string - Key string + MachineAPIEndpoint string + MachineAPITLSCert string + MachineAPITLSKey string EventSinkPort string } // NewListener creates a new listener. func (p *Params) NewListener() (net.Listener, error) { - if p.APIEndpoint == "" { + if p.MachineAPIEndpoint == "" { return nil, errors.New("no siderolink API endpoint specified") } switch { - case p.Cert == "" && p.Key == "": + case p.MachineAPITLSCert == "" && p.MachineAPITLSKey == "": // no key, no cert - use plain TCP - return net.Listen("tcp", p.APIEndpoint) - case p.Cert == "": + return net.Listen("tcp", p.MachineAPIEndpoint) + case p.MachineAPITLSCert == "": return nil, errors.New("siderolink cert is required") - case p.Key == "": + case p.MachineAPITLSKey == "": return nil, errors.New("siderolink key is required") } - cert, err := tls.LoadX509KeyPair(p.Cert, p.Key) + cert, err := tls.LoadX509KeyPair(p.MachineAPITLSCert, p.MachineAPITLSKey) if err != nil { return nil, fmt.Errorf("failed to load siderolink cert/key: %w", err) } - return tls.Listen("tcp", p.APIEndpoint, &tls.Config{ + return tls.Listen("tcp", p.MachineAPIEndpoint, &tls.Config{ Certificates: []tls.Certificate{cert}, NextProtos: []string{"h2"}, }) @@ -541,7 +541,7 @@ func (manager *Manager) pollWireguardPeers(ctx context.Context) error { spec.Connected = sinceLastHandshake < wireguard.PeerDownInterval } - if config.Config.SiderolinkDisableLastEndpoint { + if config.Config.Services.Siderolink.DisableLastEndpoint { spec.LastEndpoint = "" return nil @@ -579,12 +579,12 @@ func (manager *Manager) updateConnectionParams(ctx context.Context, siderolinkCo if _, err = safe.StateUpdateWithConflicts(ctx, manager.state, connectionParams.Metadata(), func(res *siderolink.ConnectionParams) error { spec := res.TypedSpec().Value - spec.ApiEndpoint = config.Config.SideroLinkAPIURL + spec.ApiEndpoint = config.Config.Services.MachineAPI.URL() spec.JoinToken = siderolinkConfig.TypedSpec().Value.JoinToken spec.WireguardEndpoint = siderolinkConfig.TypedSpec().Value.AdvertisedEndpoint - spec.UseGrpcTunnel = config.Config.SiderolinkUseGRPCTunnel - spec.LogsPort = int32(config.Config.LogServerPort) - spec.EventsPort = int32(config.Config.EventSinkPort) + spec.UseGrpcTunnel = config.Config.Services.Siderolink.UseGRPCTunnel + spec.LogsPort = int32(config.Config.Services.Siderolink.LogServerPort) + spec.EventsPort = int32(config.Config.Services.Siderolink.EventSinkPort) var url string @@ -604,7 +604,7 @@ func (manager *Manager) updateConnectionParams(ctx context.Context, siderolinkCo talosconstants.KernelParamLoggingKernel, net.JoinHostPort( siderolinkConfig.TypedSpec().Value.ServerAddress, - strconv.Itoa(config.Config.LogServerPort), + strconv.Itoa(config.Config.Services.Siderolink.LogServerPort), ), ) diff --git a/internal/pkg/siderolink/siderolink_test.go b/internal/pkg/siderolink/siderolink_test.go index b6620aee..63fa0c23 100644 --- a/internal/pkg/siderolink/siderolink_test.go +++ b/internal/pkg/siderolink/siderolink_test.go @@ -121,8 +121,8 @@ func (suite *SiderolinkSuite) SetupTest() { params := sideromanager.Params{ WireguardEndpoint: "127.0.0.1:0", - AdvertisedEndpoint: config.Config.SiderolinkWireguardAdvertisedAddress + "," + TestIP, - APIEndpoint: "127.0.0.1:0", + AdvertisedEndpoint: config.Config.Services.Siderolink.WireGuard.AdvertisedEndpoint + "," + TestIP, + MachineAPIEndpoint: "127.0.0.1:0", } nodeUniqueToken, err := jointoken.NewNodeUniqueToken("fingerprint", "test-token").Encode() @@ -321,7 +321,7 @@ func (suite *SiderolinkSuite) TestNodeWithSeveralAdvertisedIPs() { }, ))(suite.T()) - require.Equal(suite.T(), []string{config.Config.SiderolinkWireguardAdvertisedAddress, TestIP}, resp.GetEndpoints()) + require.Equal(suite.T(), []string{config.Config.Services.Siderolink.WireGuard.AdvertisedEndpoint, TestIP}, resp.GetEndpoints()) } func (suite *SiderolinkSuite) TestVirtualNodes() { @@ -403,7 +403,7 @@ func (suite *SiderolinkSuite) TestVirtualNodes() { expectedResp := resp.CloneVT() expectedResp.GrpcPeerAddrPort = "" - expectedResp.ServerEndpoint = pb.MakeEndpoints(config.Config.SiderolinkWireguardAdvertisedAddress, TestIP) + expectedResp.ServerEndpoint = pb.MakeEndpoints(config.Config.Services.Siderolink.WireGuard.AdvertisedEndpoint, TestIP) suite.Assert().NoError(err)