diff --git a/Dockerfile b/Dockerfile index fccd1a405..9ddf76d99 100644 --- a/Dockerfile +++ b/Dockerfile @@ -479,7 +479,7 @@ RUN mkdir -pv /rootfs/opt/{containerd/bin,containerd/lib} COPY --chmod=0644 hack/containerd.toml /rootfs/etc/containerd/config.toml COPY --chmod=0644 hack/cri-containerd.toml /rootfs/etc/cri/containerd.toml COPY --chmod=0644 hack/cri-plugin.part /rootfs/etc/cri/conf.d/00-base.part -RUN touch /rootfs/etc/{extensions.yaml,resolv.conf,hosts,os-release,machine-id,cri/conf.d/cri.toml,cri/conf.d/01-registries.part} +RUN touch /rootfs/etc/{extensions.yaml,resolv.conf,hosts,os-release,machine-id,cri/conf.d/cri.toml,cri/conf.d/01-registries.part,cri/conf.d/20-customization.part} RUN ln -s ca-certificates /rootfs/etc/ssl/certs/ca-certificates.crt RUN ln -s /etc/ssl /rootfs/etc/pki RUN ln -s /etc/ssl /rootfs/usr/share/ca-certificates @@ -526,7 +526,7 @@ RUN mkdir -pv /rootfs/opt/{containerd/bin,containerd/lib} COPY --chmod=0644 hack/containerd.toml /rootfs/etc/containerd/config.toml COPY --chmod=0644 hack/cri-containerd.toml /rootfs/etc/cri/containerd.toml COPY --chmod=0644 hack/cri-plugin.part /rootfs/etc/cri/conf.d/00-base.part -RUN touch /rootfs/etc/{extensions.yaml,resolv.conf,hosts,os-release,machine-id,cri/conf.d/cri.toml,cri/conf.d/01-registries.part} +RUN touch /rootfs/etc/{extensions.yaml,resolv.conf,hosts,os-release,machine-id,cri/conf.d/cri.toml,cri/conf.d/01-registries.part,cri/conf.d/20-customization.part} RUN ln -s /etc/ssl /rootfs/etc/pki RUN ln -s ca-certificates /rootfs/etc/ssl/certs/ca-certificates.crt RUN ln -s /etc/ssl /rootfs/usr/share/ca-certificates diff --git a/hack/cri-containerd.toml b/hack/cri-containerd.toml index 86676f61f..501ba3975 100644 --- a/hack/cri-containerd.toml +++ b/hack/cri-containerd.toml @@ -7,7 +7,6 @@ disabled_plugins = [ imports = [ "/etc/cri/conf.d/cri.toml", - "/var/cri/conf.d/*.toml", # deprecated ] [debug] diff --git a/hack/release.toml b/hack/release.toml index a646b22fc..eb5a9baa4 100644 --- a/hack/release.toml +++ b/hack/release.toml @@ -225,6 +225,14 @@ machine: Changes to the node labels will be applied immediately without `kubelet` restart. Talos keeps track of the owned node labels in the `talos.dev/owned-labels` annotation. +""" + + [notes.criconfig] + title = "CRI Configuration Overrides" + description = """\ +Talos no longer supports CRI config overrides placed in `/var/cri/conf.d` directory. + +[New way](https://www.talos.dev/v1.3/talos-guides/configuration/containerd/) correctly handles merging of containerd/CRI plugin configuration. """ [make_deps] diff --git a/internal/app/machined/pkg/controllers/files/cri_config_parts.go b/internal/app/machined/pkg/controllers/files/cri_config_parts.go index 6ab5b099a..14d25c2ea 100644 --- a/internal/app/machined/pkg/controllers/files/cri_config_parts.go +++ b/internal/app/machined/pkg/controllers/files/cri_config_parts.go @@ -7,15 +7,14 @@ package files import ( "context" "fmt" - "os" "path/filepath" "sort" "github.com/cosi-project/runtime/pkg/controller" "github.com/cosi-project/runtime/pkg/resource" - "github.com/siderolabs/go-pointer" "go.uber.org/zap" + "github.com/siderolabs/talos/internal/pkg/toml" "github.com/siderolabs/talos/pkg/machinery/constants" "github.com/siderolabs/talos/pkg/machinery/resources/files" ) @@ -37,7 +36,6 @@ func (ctrl *CRIConfigPartsController) Inputs() []controller.Input { { Namespace: files.NamespaceName, Type: files.EtcFileStatusType, - ID: pointer.To(constants.CRIRegistryConfigPart), // watch only registry configuration which might be updated Kind: controller.InputWeak, }, } @@ -74,24 +72,16 @@ func (ctrl *CRIConfigPartsController) Run(ctx context.Context, r controller.Runt sort.Strings(parts) - var contents []byte - - for _, part := range parts { - var partContents []byte - - partContents, err = os.ReadFile(part) - if err != nil { - return err - } - - contents = append(contents, append([]byte("\n## "+part+"\n\n"), partContents...)...) + out, err := toml.Merge(parts) + if err != nil { + return err } if err := r.Modify(ctx, files.NewEtcFileSpec(files.NamespaceName, constants.CRIConfig), func(r resource.Resource) error { spec := r.(*files.EtcFileSpec).TypedSpec() - spec.Contents = contents + spec.Contents = out spec.Mode = 0o600 return nil diff --git a/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer_tasks.go b/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer_tasks.go index 98ee16aa1..43e77eb51 100644 --- a/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer_tasks.go +++ b/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer_tasks.go @@ -70,6 +70,7 @@ import ( "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1/machine" "github.com/siderolabs/talos/pkg/machinery/constants" "github.com/siderolabs/talos/pkg/machinery/kernel" + resourcefiles "github.com/siderolabs/talos/pkg/machinery/resources/files" "github.com/siderolabs/talos/pkg/machinery/resources/k8s" resourceruntime "github.com/siderolabs/talos/pkg/machinery/resources/runtime" "github.com/siderolabs/talos/pkg/version" @@ -1066,6 +1067,15 @@ func WriteUserFiles(seq runtime.Sequence, data interface{}) (runtime.TaskExecuti continue } + // CRI configuration customization + if f.Path() == filepath.Join("/etc", constants.CRICustomizationConfigPart) { + if err = injectCRIConfigPatch(ctx, r.State().V1Alpha2().Resources(), []byte(f.Content())); err != nil { + result = multierror.Append(result, err) + } + + continue + } + // Determine if supplied path is in /var or not. // If not, we'll write it to /var anyways and bind mount below p := f.Path() @@ -1115,6 +1125,56 @@ func WriteUserFiles(seq runtime.Sequence, data interface{}) (runtime.TaskExecuti }, "writeUserFiles" } +func injectCRIConfigPatch(ctx context.Context, st state.State, content []byte) error { + // limit overall waiting time + ctx, cancel := context.WithTimeout(ctx, time.Minute) + defer cancel() + + ch := make(chan state.Event) + + // wait for the CRI config to be created + if err := st.Watch(ctx, resourcefiles.NewEtcFileSpec(resourcefiles.NamespaceName, constants.CRIConfig).Metadata(), ch); err != nil { + return err + } + + // first update should be received about the existing resource + select { + case <-ch: + case <-ctx.Done(): + return ctx.Err() + } + + etcFileSpec := resourcefiles.NewEtcFileSpec(resourcefiles.NamespaceName, constants.CRICustomizationConfigPart) + etcFileSpec.TypedSpec().Mode = 0o600 + etcFileSpec.TypedSpec().Contents = content + + if err := st.Create(ctx, etcFileSpec); err != nil { + return err + } + + // wait for the CRI config parts controller to generate the merged file + var version resource.Version + + select { + case ev := <-ch: + version = ev.Resource.Metadata().Version() + case <-ctx.Done(): + return ctx.Err() + } + + // wait for the file to be rendered + _, err := st.WatchFor(ctx, resourcefiles.NewEtcFileStatus(resourcefiles.NamespaceName, constants.CRIConfig).Metadata(), state.WithCondition(func(r resource.Resource) (bool, error) { + fileStatus, ok := r.(*resourcefiles.EtcFileStatus) + if !ok { + return false, nil + } + + return fileStatus.TypedSpec().SpecVersion == version.String(), nil + })) + + return err +} + //nolint:deadcode,unused func doesNotExists(p string) (err error) { _, err = os.Stat(p) diff --git a/internal/pkg/toml/merge.go b/internal/pkg/toml/merge.go new file mode 100644 index 000000000..b44da6332 --- /dev/null +++ b/internal/pkg/toml/merge.go @@ -0,0 +1,48 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package toml + +import ( + "bytes" + "fmt" + + "github.com/BurntSushi/toml" + + "github.com/siderolabs/talos/pkg/machinery/config/merge" +) + +// Merge several TOML documents in files into one. +// +// Merge process relies on generic map[string]interface{} merge which might not always be correct. +func Merge(parts []string) ([]byte, error) { + merged := map[string]interface{}{} + + var header []byte + + for _, part := range parts { + partial := map[string]interface{}{} + + if _, err := toml.DecodeFile(part, &partial); err != nil { + return nil, fmt.Errorf("error decoding %q: %w", part, err) + } + + if err := merge.Merge(merged, partial); err != nil { + return nil, fmt.Errorf("error merging %q: %w", part, err) + } + + header = append(header, []byte(fmt.Sprintf("## %s\n", part))...) + } + + var out bytes.Buffer + + _, _ = out.Write(header) //nolint:errcheck + _ = out.WriteByte('\n') //nolint:errcheck + + if err := toml.NewEncoder(&out).Encode(merged); err != nil { + return nil, fmt.Errorf("error encoding merged config: %w", err) + } + + return out.Bytes(), nil +} diff --git a/internal/pkg/toml/merge_test.go b/internal/pkg/toml/merge_test.go new file mode 100644 index 000000000..7c5cf791b --- /dev/null +++ b/internal/pkg/toml/merge_test.go @@ -0,0 +1,29 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package toml_test + +import ( + _ "embed" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/siderolabs/talos/internal/pkg/toml" +) + +//go:embed testdata/expected.toml +var expected []byte + +func TestMerge(t *testing.T) { + out, err := toml.Merge([]string{ + "testdata/1.toml", + "testdata/2.toml", + "testdata/3.toml", + }) + require.NoError(t, err) + + assert.Equal(t, expected, out) +} diff --git a/internal/pkg/toml/testdata/1.toml b/internal/pkg/toml/testdata/1.toml new file mode 100644 index 000000000..fbfe25411 --- /dev/null +++ b/internal/pkg/toml/testdata/1.toml @@ -0,0 +1,5 @@ +version = 2 + +[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + discard_unpacked_layers = true diff --git a/internal/pkg/toml/testdata/2.toml b/internal/pkg/toml/testdata/2.toml new file mode 100644 index 000000000..abdc95154 --- /dev/null +++ b/internal/pkg/toml/testdata/2.toml @@ -0,0 +1,5 @@ +[plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".registry] + config_path = "/etc/cri/conf.d/hosts" + [plugins."io.containerd.grpc.v1.cri".registry.configs] diff --git a/internal/pkg/toml/testdata/3.toml b/internal/pkg/toml/testdata/3.toml new file mode 100644 index 000000000..3e6569645 --- /dev/null +++ b/internal/pkg/toml/testdata/3.toml @@ -0,0 +1,6 @@ +[metrics] + address = "0.0.0.0:11234" + +[plugins] + [plugins."io.containerd.grpc.v1.cri"] + sandbox_image = "registry.k8s.io/pause:3.8" diff --git a/internal/pkg/toml/testdata/expected.toml b/internal/pkg/toml/testdata/expected.toml new file mode 100644 index 000000000..7748750c4 --- /dev/null +++ b/internal/pkg/toml/testdata/expected.toml @@ -0,0 +1,20 @@ +## testdata/1.toml +## testdata/2.toml +## testdata/3.toml + +version = 2 + +[metrics] + address = "0.0.0.0:11234" + +[plugins] + [plugins."io.containerd.grpc.v1.cri"] + sandbox_image = "registry.k8s.io/pause:3.8" + [plugins."io.containerd.grpc.v1.cri".containerd] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + discard_unpacked_layers = true + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".registry] + config_path = "/etc/cri/conf.d/hosts" + [plugins."io.containerd.grpc.v1.cri".registry.configs] diff --git a/internal/pkg/toml/toml.go b/internal/pkg/toml/toml.go new file mode 100644 index 000000000..2bc51cb7a --- /dev/null +++ b/internal/pkg/toml/toml.go @@ -0,0 +1,6 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package toml provides utility functions for TOML handling. +package toml diff --git a/pkg/machinery/constants/constants.go b/pkg/machinery/constants/constants.go index fcf513d23..a05853b74 100644 --- a/pkg/machinery/constants/constants.go +++ b/pkg/machinery/constants/constants.go @@ -446,6 +446,9 @@ const ( // CRIRegistryConfigPart is the path to the CRI generated registry configuration relative to /etc. CRIRegistryConfigPart = "cri/conf.d/01-registries.part" + // CRICustomizationConfigPart is the path to the CRI generated registry configuration relative to /etc. + CRICustomizationConfigPart = "cri/conf.d/20-customization.part" + // TalosConfigEnvVar is the environment variable for setting the Talos configuration file path. TalosConfigEnvVar = "TALOSCONFIG" diff --git a/website/content/v1.3/talos-guides/configuration/containerd.md b/website/content/v1.3/talos-guides/configuration/containerd.md index bf396a041..f3e6229b9 100644 --- a/website/content/v1.3/talos-guides/configuration/containerd.md +++ b/website/content/v1.3/talos-guides/configuration/containerd.md @@ -5,27 +5,28 @@ aliases: - ../../guides/configuring-containerd --- -The base containerd configuration expects to merge in any additional configs present in `/var/cri/conf.d/*.toml`. +The base containerd configuration expects to merge in any additional configs present in `/etc/cri/conf.d/20-customization.part`. -## An example of exposing metrics +## Examples -Into each machine config, add the following: +### Exposing Metrics + +Patch the machine config by adding the following: ```yaml machine: - ... files: - content: | [metrics] address = "0.0.0.0:11234" - path: /var/cri/conf.d/metrics.toml + path: /etc/cri/conf.d/20-customization.part op: create ``` -Create cluster like normal and see that metrics are now present on this port: +Once the server reboots, metrics are now available: ```bash -$ curl 127.0.0.1:11234/v1/metrics +$ curl ${IP}:11234/v1/metrics # HELP container_blkio_io_service_bytes_recursive_bytes The blkio io service bytes recursive # TYPE container_blkio_io_service_bytes_recursive_bytes gauge container_blkio_io_service_bytes_recursive_bytes{container_id="0677d73196f5f4be1d408aab1c4125cf9e6c458a4bea39e590ac779709ffbe14",device="/dev/dm-0",major="253",minor="0",namespace="k8s.io",op="Async"} 0 @@ -33,3 +34,32 @@ container_blkio_io_service_bytes_recursive_bytes{container_id="0677d73196f5f4be1 ... ... ``` + +### Pause Image + +This change is often required for air-gapped environments, as `containerd` CRI plugin has a reference to the `pause` image which is used +to create pods, and it can't be controlled with Kubernetes pod definitions. + +```yaml +machine: + files: + - content: | + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + sandbox_image = "registry.k8s.io/pause:3.8" + path: /etc/cri/conf.d/20-customization.part + op: create +``` + +Now the `pause` image is set to `registry.k8s.io/pause:3.8`: + +```bash +$ talosctl containers --kubernetes +NODE NAMESPACE ID IMAGE PID STATUS +172.20.0.5 k8s.io kube-system/kube-flannel-6hfck registry.k8s.io/pause:3.8 1773 SANDBOX_READY +172.20.0.5 k8s.io └─ kube-system/kube-flannel-6hfck:install-cni ghcr.io/siderolabs/install-cni:v1.3.0-alpha.0-2-gb155fa0 0 CONTAINER_EXITED +172.20.0.5 k8s.io └─ kube-system/kube-flannel-6hfck:install-config ghcr.io/siderolabs/flannel:v0.20.1 0 CONTAINER_EXITED +172.20.0.5 k8s.io └─ kube-system/kube-flannel-6hfck:kube-flannel ghcr.io/siderolabs/flannel:v0.20.1 2092 CONTAINER_RUNNING +172.20.0.5 k8s.io kube-system/kube-proxy-xp7jq registry.k8s.io/pause:3.8 1780 SANDBOX_READY +172.20.0.5 k8s.io └─ kube-system/kube-proxy-xp7jq:kube-proxy k8s.gcr.io/kube-proxy:v1.26.0-alpha.3 1843 CONTAINER_RUNNING +```