feat: implement CRI configuration customization

This is tricky, as containerd doesn't merge itself plugin configuration
across multiple files. TOML can't load configuration correctly from
concatenated files.

Fixes #6390

Signed-off-by: Andrey Smirnov <andrey.smirnov@talos-systems.com>

Dockerfile

@@ -479,7 +479,7 @@ RUN mkdir -pv /rootfs/opt/{containerd/bin,containerd/lib}
 COPY --chmod=0644 hack/containerd.toml /rootfs/etc/containerd/config.toml
 COPY --chmod=0644 hack/cri-containerd.toml /rootfs/etc/cri/containerd.toml
 COPY --chmod=0644 hack/cri-plugin.part /rootfs/etc/cri/conf.d/00-base.part
-RUN touch /rootfs/etc/{extensions.yaml,resolv.conf,hosts,os-release,machine-id,cri/conf.d/cri.toml,cri/conf.d/01-registries.part}
+RUN touch /rootfs/etc/{extensions.yaml,resolv.conf,hosts,os-release,machine-id,cri/conf.d/cri.toml,cri/conf.d/01-registries.part,cri/conf.d/20-customization.part}
 RUN ln -s ca-certificates /rootfs/etc/ssl/certs/ca-certificates.crt
 RUN ln -s /etc/ssl /rootfs/etc/pki
 RUN ln -s /etc/ssl /rootfs/usr/share/ca-certificates
@@ -526,7 +526,7 @@ RUN mkdir -pv /rootfs/opt/{containerd/bin,containerd/lib}
 COPY --chmod=0644 hack/containerd.toml /rootfs/etc/containerd/config.toml
 COPY --chmod=0644 hack/cri-containerd.toml /rootfs/etc/cri/containerd.toml
 COPY --chmod=0644 hack/cri-plugin.part /rootfs/etc/cri/conf.d/00-base.part
-RUN touch /rootfs/etc/{extensions.yaml,resolv.conf,hosts,os-release,machine-id,cri/conf.d/cri.toml,cri/conf.d/01-registries.part}
+RUN touch /rootfs/etc/{extensions.yaml,resolv.conf,hosts,os-release,machine-id,cri/conf.d/cri.toml,cri/conf.d/01-registries.part,cri/conf.d/20-customization.part}
 RUN ln -s /etc/ssl /rootfs/etc/pki
 RUN ln -s ca-certificates /rootfs/etc/ssl/certs/ca-certificates.crt
 RUN ln -s /etc/ssl /rootfs/usr/share/ca-certificates

hack/cri-containerd.toml

@@ -7,7 +7,6 @@ disabled_plugins = [
 
 imports = [
     "/etc/cri/conf.d/cri.toml",
-    "/var/cri/conf.d/*.toml", # deprecated
 ]
 
 [debug]

hack/release.toml

@@ -225,6 +225,14 @@ machine:
 Changes to the node labels will be applied immediately without `kubelet` restart.
 
 Talos keeps track of the owned node labels in the `talos.dev/owned-labels` annotation.
+"""
+
+    [notes.criconfig]
+        title = "CRI Configuration Overrides"
+        description = """\
+Talos no longer supports CRI config overrides placed in `/var/cri/conf.d` directory.
+
+[New way](https://www.talos.dev/v1.3/talos-guides/configuration/containerd/) correctly handles merging of containerd/CRI plugin configuration.
 """
 
 [make_deps]

internal/app/machined/pkg/controllers/files/cri_config_parts.go

@@ -7,15 +7,14 @@ package files
 import (
 	"context"
 	"fmt"
-	"os"
 	"path/filepath"
 	"sort"
 
 	"github.com/cosi-project/runtime/pkg/controller"
 	"github.com/cosi-project/runtime/pkg/resource"
-	"github.com/siderolabs/go-pointer"
 	"go.uber.org/zap"
 
+	"github.com/siderolabs/talos/internal/pkg/toml"
 	"github.com/siderolabs/talos/pkg/machinery/constants"
 	"github.com/siderolabs/talos/pkg/machinery/resources/files"
 )
@@ -37,7 +36,6 @@ func (ctrl *CRIConfigPartsController) Inputs() []controller.Input {
 		{
 			Namespace: files.NamespaceName,
 			Type:      files.EtcFileStatusType,
-			ID:        pointer.To(constants.CRIRegistryConfigPart), // watch only registry configuration which might be updated
 			Kind:      controller.InputWeak,
 		},
 	}
@@ -74,24 +72,16 @@ func (ctrl *CRIConfigPartsController) Run(ctx context.Context, r controller.Runt
 
 		sort.Strings(parts)
 
-		var contents []byte
+		out, err := toml.Merge(parts)
-
+		if err != nil {
-		for _, part := range parts {
+			return err
-			var partContents []byte
-
-			partContents, err = os.ReadFile(part)
-			if err != nil {
-				return err
-			}
-
-			contents = append(contents, append([]byte("\n## "+part+"\n\n"), partContents...)...)
 		}
 
 		if err := r.Modify(ctx, files.NewEtcFileSpec(files.NamespaceName, constants.CRIConfig),
 			func(r resource.Resource) error {
 				spec := r.(*files.EtcFileSpec).TypedSpec()
 
-				spec.Contents = contents
+				spec.Contents = out
 				spec.Mode = 0o600
 
 				return nil

internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer_tasks.go

@@ -70,6 +70,7 @@ import (
 	"github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1/machine"
 	"github.com/siderolabs/talos/pkg/machinery/constants"
 	"github.com/siderolabs/talos/pkg/machinery/kernel"
+	resourcefiles "github.com/siderolabs/talos/pkg/machinery/resources/files"
 	"github.com/siderolabs/talos/pkg/machinery/resources/k8s"
 	resourceruntime "github.com/siderolabs/talos/pkg/machinery/resources/runtime"
 	"github.com/siderolabs/talos/pkg/version"
@@ -1066,6 +1067,15 @@ func WriteUserFiles(seq runtime.Sequence, data interface{}) (runtime.TaskExecuti
 				continue
 			}
 
+			// CRI configuration customization
+			if f.Path() == filepath.Join("/etc", constants.CRICustomizationConfigPart) {
+				if err = injectCRIConfigPatch(ctx, r.State().V1Alpha2().Resources(), []byte(f.Content())); err != nil {
+					result = multierror.Append(result, err)
+				}
+
+				continue
+			}
+
 			// Determine if supplied path is in /var or not.
 			// If not, we'll write it to /var anyways and bind mount below
 			p := f.Path()
@@ -1115,6 +1125,56 @@ func WriteUserFiles(seq runtime.Sequence, data interface{}) (runtime.TaskExecuti
 	}, "writeUserFiles"
 }
 
+func injectCRIConfigPatch(ctx context.Context, st state.State, content []byte) error {
+	// limit overall waiting time
+	ctx, cancel := context.WithTimeout(ctx, time.Minute)
+	defer cancel()
+
+	ch := make(chan state.Event)
+
+	// wait for the CRI config to be created
+	if err := st.Watch(ctx, resourcefiles.NewEtcFileSpec(resourcefiles.NamespaceName, constants.CRIConfig).Metadata(), ch); err != nil {
+		return err
+	}
+
+	// first update should be received about the existing resource
+	select {
+	case <-ch:
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+
+	etcFileSpec := resourcefiles.NewEtcFileSpec(resourcefiles.NamespaceName, constants.CRICustomizationConfigPart)
+	etcFileSpec.TypedSpec().Mode = 0o600
+	etcFileSpec.TypedSpec().Contents = content
+
+	if err := st.Create(ctx, etcFileSpec); err != nil {
+		return err
+	}
+
+	// wait for the CRI config parts controller to generate the merged file
+	var version resource.Version
+
+	select {
+	case ev := <-ch:
+		version = ev.Resource.Metadata().Version()
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+
+	// wait for the file to be rendered
+	_, err := st.WatchFor(ctx, resourcefiles.NewEtcFileStatus(resourcefiles.NamespaceName, constants.CRIConfig).Metadata(), state.WithCondition(func(r resource.Resource) (bool, error) {
+		fileStatus, ok := r.(*resourcefiles.EtcFileStatus)
+		if !ok {
+			return false, nil
+		}
+
+		return fileStatus.TypedSpec().SpecVersion == version.String(), nil
+	}))
+
+	return err
+}
+
 //nolint:deadcode,unused
 func doesNotExists(p string) (err error) {
 	_, err = os.Stat(p)

internal/pkg/toml/merge.go

@@ -0,0 +1,48 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package toml
+
+import (
+	"bytes"
+	"fmt"
+
+	"github.com/BurntSushi/toml"
+
+	"github.com/siderolabs/talos/pkg/machinery/config/merge"
+)
+
+// Merge several TOML documents in files into one.
+//
+// Merge process relies on generic map[string]interface{} merge which might not always be correct.
+func Merge(parts []string) ([]byte, error) {
+	merged := map[string]interface{}{}
+
+	var header []byte
+
+	for _, part := range parts {
+		partial := map[string]interface{}{}
+
+		if _, err := toml.DecodeFile(part, &partial); err != nil {
+			return nil, fmt.Errorf("error decoding %q: %w", part, err)
+		}
+
+		if err := merge.Merge(merged, partial); err != nil {
+			return nil, fmt.Errorf("error merging %q: %w", part, err)
+		}
+
+		header = append(header, []byte(fmt.Sprintf("## %s\n", part))...)
+	}
+
+	var out bytes.Buffer
+
+	_, _ = out.Write(header) //nolint:errcheck
+	_ = out.WriteByte('\n')  //nolint:errcheck
+
+	if err := toml.NewEncoder(&out).Encode(merged); err != nil {
+		return nil, fmt.Errorf("error encoding merged config: %w", err)
+	}
+
+	return out.Bytes(), nil
+}

internal/pkg/toml/merge_test.go

@@ -0,0 +1,29 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package toml_test
+
+import (
+	_ "embed"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/siderolabs/talos/internal/pkg/toml"
+)
+
+//go:embed testdata/expected.toml
+var expected []byte
+
+func TestMerge(t *testing.T) {
+	out, err := toml.Merge([]string{
+		"testdata/1.toml",
+		"testdata/2.toml",
+		"testdata/3.toml",
+	})
+	require.NoError(t, err)
+
+	assert.Equal(t, expected, out)
+}

internal/pkg/toml/testdata/1.toml

@@ -0,0 +1,5 @@
+version = 2
+
+[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+    runtime_type = "io.containerd.runc.v2"
+    discard_unpacked_layers = true

internal/pkg/toml/testdata/2.toml

@@ -0,0 +1,5 @@
+[plugins]
+  [plugins."io.containerd.grpc.v1.cri"]
+    [plugins."io.containerd.grpc.v1.cri".registry]
+      config_path = "/etc/cri/conf.d/hosts"
+      [plugins."io.containerd.grpc.v1.cri".registry.configs]

internal/pkg/toml/testdata/3.toml

@@ -0,0 +1,6 @@
+[metrics]
+  address = "0.0.0.0:11234"
+
+[plugins]
+   [plugins."io.containerd.grpc.v1.cri"]
+    sandbox_image = "registry.k8s.io/pause:3.8"

internal/pkg/toml/testdata/expected.toml

@@ -0,0 +1,20 @@
+## testdata/1.toml
+## testdata/2.toml
+## testdata/3.toml
+
+version = 2
+
+[metrics]
+  address = "0.0.0.0:11234"
+
+[plugins]
+  [plugins."io.containerd.grpc.v1.cri"]
+    sandbox_image = "registry.k8s.io/pause:3.8"
+    [plugins."io.containerd.grpc.v1.cri".containerd]
+      [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+          discard_unpacked_layers = true
+          runtime_type = "io.containerd.runc.v2"
+    [plugins."io.containerd.grpc.v1.cri".registry]
+      config_path = "/etc/cri/conf.d/hosts"
+      [plugins."io.containerd.grpc.v1.cri".registry.configs]

internal/pkg/toml/toml.go

@@ -0,0 +1,6 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+// Package toml provides utility functions for TOML handling.
+package toml

pkg/machinery/constants/constants.go

@@ -446,6 +446,9 @@ const (
 	// CRIRegistryConfigPart is the path to the CRI generated registry configuration relative to /etc.
 	CRIRegistryConfigPart = "cri/conf.d/01-registries.part"
 
+	// CRICustomizationConfigPart is the path to the CRI generated registry configuration relative to /etc.
+	CRICustomizationConfigPart = "cri/conf.d/20-customization.part"
+
 	// TalosConfigEnvVar is the environment variable for setting the Talos configuration file path.
 	TalosConfigEnvVar = "TALOSCONFIG"
 

website/content/v1.3/talos-guides/configuration/containerd.md

@@ -5,27 +5,28 @@ aliases:
   - ../../guides/configuring-containerd
 ---
 
-The base containerd configuration expects to merge in any additional configs present in `/var/cri/conf.d/*.toml`.
+The base containerd configuration expects to merge in any additional configs present in `/etc/cri/conf.d/20-customization.part`.
 
-## An example of exposing metrics
+## Examples
 
-Into each machine config, add the following:
+### Exposing Metrics
+
+Patch the machine config by adding the following:
 
 ```yaml
 machine:
-  ...
   files:
     - content: |
         [metrics]
           address = "0.0.0.0:11234"
-      path: /var/cri/conf.d/metrics.toml
+      path: /etc/cri/conf.d/20-customization.part
       op: create
 ```
 
-Create cluster like normal and see that metrics are now present on this port:
+Once the server reboots, metrics are now available:
 
 ```bash
-$ curl 127.0.0.1:11234/v1/metrics
+$ curl ${IP}:11234/v1/metrics
 # HELP container_blkio_io_service_bytes_recursive_bytes The blkio io service bytes recursive
 # TYPE container_blkio_io_service_bytes_recursive_bytes gauge
 container_blkio_io_service_bytes_recursive_bytes{container_id="0677d73196f5f4be1d408aab1c4125cf9e6c458a4bea39e590ac779709ffbe14",device="/dev/dm-0",major="253",minor="0",namespace="k8s.io",op="Async"} 0
@@ -33,3 +34,32 @@ container_blkio_io_service_bytes_recursive_bytes{container_id="0677d73196f5f4be1
 ...
 ...
 ```
+
+### Pause Image
+
+This change is often required for air-gapped environments, as `containerd` CRI plugin has a reference to the `pause` image which is used
+to create pods, and it can't be controlled with Kubernetes pod definitions.
+
+```yaml
+machine:
+  files:
+    - content: |
+        [plugins]
+          [plugins."io.containerd.grpc.v1.cri"]
+            sandbox_image = "registry.k8s.io/pause:3.8"
+      path: /etc/cri/conf.d/20-customization.part
+      op: create
+```
+
+Now the `pause` image is set to `registry.k8s.io/pause:3.8`:
+
+```bash
+$ talosctl containers --kubernetes
+NODE         NAMESPACE   ID                                                 IMAGE                                                      PID    STATUS
+172.20.0.5   k8s.io      kube-system/kube-flannel-6hfck                     registry.k8s.io/pause:3.8                                  1773   SANDBOX_READY
+172.20.0.5   k8s.io      └─ kube-system/kube-flannel-6hfck:install-cni      ghcr.io/siderolabs/install-cni:v1.3.0-alpha.0-2-gb155fa0   0      CONTAINER_EXITED
+172.20.0.5   k8s.io      └─ kube-system/kube-flannel-6hfck:install-config   ghcr.io/siderolabs/flannel:v0.20.1                         0      CONTAINER_EXITED
+172.20.0.5   k8s.io      └─ kube-system/kube-flannel-6hfck:kube-flannel     ghcr.io/siderolabs/flannel:v0.20.1                         2092   CONTAINER_RUNNING
+172.20.0.5   k8s.io      kube-system/kube-proxy-xp7jq                       registry.k8s.io/pause:3.8                                  1780   SANDBOX_READY
+172.20.0.5   k8s.io      └─ kube-system/kube-proxy-xp7jq:kube-proxy         k8s.gcr.io/kube-proxy:v1.26.0-alpha.3                      1843   CONTAINER_RUNNING
+```