mirrors/talos
1
0
Fork 0
mirror of https://github.com/siderolabs/talos.git synced 2025-10-23 21:41:11 +02:00
Code Issues Packages Projects Releases Wiki Activity

feat: implement CRI configuration customization

Browse Source 
This is tricky, as containerd doesn't merge itself plugin configuration
across multiple files. TOML can't load configuration correctly from
concatenated files.

Fixes #6390

Signed-off-by: Andrey Smirnov <andrey.smirnov@talos-systems.com>
This commit is contained in:
Andrey Smirnov 2022-11-15 20:21:59 +04:00
parent e1e340bdd9
commit 6ffc381c59
No known key found for this signature in database
GPG Key ID: 7B26396447AB6DFD
14 changed files with 234 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Dockerfile
View File

@ -479,7 +479,7 @@ RUN mkdir -pv /rootfs/opt/{containerd/bin,containerd/lib}
COPY --chmod=0644 hack/containerd.toml /rootfs/etc/containerd/config.toml
COPY --chmod=0644 hack/cri-containerd.toml /rootfs/etc/cri/containerd.toml
COPY --chmod=0644 hack/cri-plugin.part /rootfs/etc/cri/conf.d/00-base.part
RUN touch /rootfs/etc/{extensions.yaml,resolv.conf,hosts,os-release,machine-id,cri/conf.d/cri.toml,cri/conf.d/01-registries.part}
RUN touch /rootfs/etc/{extensions.yaml,resolv.conf,hosts,os-release,machine-id,cri/conf.d/cri.toml,cri/conf.d/01-registries.part,cri/conf.d/20-customization.part}
RUN ln -s ca-certificates /rootfs/etc/ssl/certs/ca-certificates.crt
RUN ln -s /etc/ssl /rootfs/etc/pki
RUN ln -s /etc/ssl /rootfs/usr/share/ca-certificates
@ -526,7 +526,7 @@ RUN mkdir -pv /rootfs/opt/{containerd/bin,containerd/lib}
COPY --chmod=0644 hack/containerd.toml /rootfs/etc/containerd/config.toml
COPY --chmod=0644 hack/cri-containerd.toml /rootfs/etc/cri/containerd.toml
COPY --chmod=0644 hack/cri-plugin.part /rootfs/etc/cri/conf.d/00-base.part
RUN touch /rootfs/etc/{extensions.yaml,resolv.conf,hosts,os-release,machine-id,cri/conf.d/cri.toml,cri/conf.d/01-registries.part}
RUN touch /rootfs/etc/{extensions.yaml,resolv.conf,hosts,os-release,machine-id,cri/conf.d/cri.toml,cri/conf.d/01-registries.part,cri/conf.d/20-customization.part}
RUN ln -s /etc/ssl /rootfs/etc/pki
RUN ln -s ca-certificates /rootfs/etc/ssl/certs/ca-certificates.crt
RUN ln -s /etc/ssl /rootfs/usr/share/ca-certificates

1
hack/cri-containerd.toml
View File

@ -7,7 +7,6 @@ disabled_plugins = [
imports = [
"/etc/cri/conf.d/cri.toml",
"/var/cri/conf.d/*.toml", # deprecated
]
[debug]

8
hack/release.toml
View File

@ -225,6 +225,14 @@ machine:
Changes to the node labels will be applied immediately without `kubelet` restart.
Talos keeps track of the owned node labels in the `talos.dev/owned-labels` annotation.
"""
[notes.criconfig]
title = "CRI Configuration Overrides"
description = """\
Talos no longer supports CRI config overrides placed in `/var/cri/conf.d` directory.
[New way](https://www.talos.dev/v1.3/talos-guides/configuration/containerd/) correctly handles merging of containerd/CRI plugin configuration.
"""
[make_deps]

20
internal/app/machined/pkg/controllers/files/cri_config_parts.go
View File

@ -7,15 +7,14 @@ package files
import (
"context"
"fmt"
"os"
"path/filepath"
"sort"
"github.com/cosi-project/runtime/pkg/controller"
"github.com/cosi-project/runtime/pkg/resource"
"github.com/siderolabs/go-pointer"
"go.uber.org/zap"
"github.com/siderolabs/talos/internal/pkg/toml"
"github.com/siderolabs/talos/pkg/machinery/constants"
"github.com/siderolabs/talos/pkg/machinery/resources/files"
)
@ -37,7 +36,6 @@ func (ctrl *CRIConfigPartsController) Inputs() []controller.Input {
{
Namespace: files.NamespaceName,
Type: files.EtcFileStatusType,
ID: pointer.To(constants.CRIRegistryConfigPart), // watch only registry configuration which might be updated
Kind: controller.InputWeak,
},
}
@ -74,24 +72,16 @@ func (ctrl *CRIConfigPartsController) Run(ctx context.Context, r controller.Runt
sort.Strings(parts)
var contents []byte
for _, part := range parts {
var partContents []byte
partContents, err = os.ReadFile(part)
if err != nil {
return err
}
contents = append(contents, append([]byte("\n## "+part+"\n\n"), partContents...)...)
out, err := toml.Merge(parts)
if err != nil {
return err
}
if err := r.Modify(ctx, files.NewEtcFileSpec(files.NamespaceName, constants.CRIConfig),
func(r resource.Resource) error {
spec := r.(*files.EtcFileSpec).TypedSpec()
spec.Contents = contents
spec.Contents = out
spec.Mode = 0o600
return nil

60
internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer_tasks.go
View File

@ -70,6 +70,7 @@ import (
"github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1/machine"
"github.com/siderolabs/talos/pkg/machinery/constants"
"github.com/siderolabs/talos/pkg/machinery/kernel"
resourcefiles "github.com/siderolabs/talos/pkg/machinery/resources/files"
"github.com/siderolabs/talos/pkg/machinery/resources/k8s"
resourceruntime "github.com/siderolabs/talos/pkg/machinery/resources/runtime"
"github.com/siderolabs/talos/pkg/version"
@ -1066,6 +1067,15 @@ func WriteUserFiles(seq runtime.Sequence, data interface{}) (runtime.TaskExecuti
continue
}
// CRI configuration customization
if f.Path() == filepath.Join("/etc", constants.CRICustomizationConfigPart) {
if err = injectCRIConfigPatch(ctx, r.State().V1Alpha2().Resources(), []byte(f.Content())); err != nil {
result = multierror.Append(result, err)
}
continue
}
// Determine if supplied path is in /var or not.
// If not, we'll write it to /var anyways and bind mount below
p := f.Path()
@ -1115,6 +1125,56 @@ func WriteUserFiles(seq runtime.Sequence, data interface{}) (runtime.TaskExecuti
}, "writeUserFiles"
}
func injectCRIConfigPatch(ctx context.Context, st state.State, content []byte) error {
// limit overall waiting time
ctx, cancel := context.WithTimeout(ctx, time.Minute)
defer cancel()
ch := make(chan state.Event)
// wait for the CRI config to be created
if err := st.Watch(ctx, resourcefiles.NewEtcFileSpec(resourcefiles.NamespaceName, constants.CRIConfig).Metadata(), ch); err != nil {
return err
}
// first update should be received about the existing resource
select {
case <-ch:
case <-ctx.Done():
return ctx.Err()
}
etcFileSpec := resourcefiles.NewEtcFileSpec(resourcefiles.NamespaceName, constants.CRICustomizationConfigPart)
etcFileSpec.TypedSpec().Mode = 0o600
etcFileSpec.TypedSpec().Contents = content
if err := st.Create(ctx, etcFileSpec); err != nil {
return err
}
// wait for the CRI config parts controller to generate the merged file
var version resource.Version
select {
case ev := <-ch:
version = ev.Resource.Metadata().Version()
case <-ctx.Done():
return ctx.Err()
}
// wait for the file to be rendered
_, err := st.WatchFor(ctx, resourcefiles.NewEtcFileStatus(resourcefiles.NamespaceName, constants.CRIConfig).Metadata(), state.WithCondition(func(r resource.Resource) (bool, error) {
fileStatus, ok := r.(*resourcefiles.EtcFileStatus)
if !ok {
return false, nil
}
return fileStatus.TypedSpec().SpecVersion == version.String(), nil
}))
return err
}
//nolint:deadcode,unused
func doesNotExists(p string) (err error) {
_, err = os.Stat(p)

48
internal/pkg/toml/merge.go Normal file
View File

@ -0,0 +1,48 @@
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
package toml
import (
"bytes"
"fmt"
"github.com/BurntSushi/toml"
"github.com/siderolabs/talos/pkg/machinery/config/merge"
)
// Merge several TOML documents in files into one.
//
// Merge process relies on generic map[string]interface{} merge which might not always be correct.
func Merge(parts []string) ([]byte, error) {
merged := map[string]interface{}{}
var header []byte
for _, part := range parts {
partial := map[string]interface{}{}
if _, err := toml.DecodeFile(part, &partial); err != nil {
return nil, fmt.Errorf("error decoding %q: %w", part, err)
}
if err := merge.Merge(merged, partial); err != nil {
return nil, fmt.Errorf("error merging %q: %w", part, err)
}
header = append(header, []byte(fmt.Sprintf("## %s\n", part))...)
}
var out bytes.Buffer
_, _ = out.Write(header) //nolint:errcheck
_ = out.WriteByte('\n') //nolint:errcheck
if err := toml.NewEncoder(&out).Encode(merged); err != nil {
return nil, fmt.Errorf("error encoding merged config: %w", err)
}
return out.Bytes(), nil
}

29
internal/pkg/toml/merge_test.go Normal file
View File

@ -0,0 +1,29 @@
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
package toml_test
import (
_ "embed"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/siderolabs/talos/internal/pkg/toml"
)
//go:embed testdata/expected.toml
var expected []byte
func TestMerge(t *testing.T) {
out, err := toml.Merge([]string{
"testdata/1.toml",
"testdata/2.toml",
"testdata/3.toml",
})
require.NoError(t, err)
assert.Equal(t, expected, out)
}

5
internal/pkg/toml/testdata/1.toml vendored Normal file
View File

@ -0,0 +1,5 @@
version = 2
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
runtime_type = "io.containerd.runc.v2"
discard_unpacked_layers = true

5
internal/pkg/toml/testdata/2.toml vendored Normal file
View File

@ -0,0 +1,5 @@
[plugins]
[plugins."io.containerd.grpc.v1.cri"]
[plugins."io.containerd.grpc.v1.cri".registry]
config_path = "/etc/cri/conf.d/hosts"
[plugins."io.containerd.grpc.v1.cri".registry.configs]

6
internal/pkg/toml/testdata/3.toml vendored Normal file
View File

@ -0,0 +1,6 @@
[metrics]
address = "0.0.0.0:11234"
[plugins]
[plugins."io.containerd.grpc.v1.cri"]
sandbox_image = "registry.k8s.io/pause:3.8"

20
internal/pkg/toml/testdata/expected.toml vendored Normal file
View File

@ -0,0 +1,20 @@
## testdata/1.toml
## testdata/2.toml
## testdata/3.toml
version = 2
[metrics]
address = "0.0.0.0:11234"
[plugins]
[plugins."io.containerd.grpc.v1.cri"]
sandbox_image = "registry.k8s.io/pause:3.8"
[plugins."io.containerd.grpc.v1.cri".containerd]
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
discard_unpacked_layers = true
runtime_type = "io.containerd.runc.v2"
[plugins."io.containerd.grpc.v1.cri".registry]
config_path = "/etc/cri/conf.d/hosts"
[plugins."io.containerd.grpc.v1.cri".registry.configs]

6
internal/pkg/toml/toml.go Normal file
View File

@ -0,0 +1,6 @@
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
// Package toml provides utility functions for TOML handling.
package toml

3
pkg/machinery/constants/constants.go
View File

@ -446,6 +446,9 @@ const (
// CRIRegistryConfigPart is the path to the CRI generated registry configuration relative to /etc.
CRIRegistryConfigPart = "cri/conf.d/01-registries.part"
// CRICustomizationConfigPart is the path to the CRI generated registry configuration relative to /etc.
CRICustomizationConfigPart = "cri/conf.d/20-customization.part"
// TalosConfigEnvVar is the environment variable for setting the Talos configuration file path.
TalosConfigEnvVar = "TALOSCONFIG"

44
website/content/v1.3/talos-guides/configuration/containerd.md
View File

@ -5,27 +5,28 @@ aliases:
- ../../guides/configuring-containerd
---
The base containerd configuration expects to merge in any additional configs present in `/var/cri/conf.d/*.toml`.
The base containerd configuration expects to merge in any additional configs present in `/etc/cri/conf.d/20-customization.part`.
## An example of exposing metrics
## Examples
Into each machine config, add the following:
### Exposing Metrics
Patch the machine config by adding the following:
```yaml
machine:
...
files:
- content: |
[metrics]
address = "0.0.0.0:11234"
path: /var/cri/conf.d/metrics.toml
path: /etc/cri/conf.d/20-customization.part
op: create
```
Create cluster like normal and see that metrics are now present on this port:
Once the server reboots, metrics are now available:
```bash
$ curl 127.0.0.1:11234/v1/metrics
$ curl ${IP}:11234/v1/metrics
# HELP container_blkio_io_service_bytes_recursive_bytes The blkio io service bytes recursive
# TYPE container_blkio_io_service_bytes_recursive_bytes gauge
container_blkio_io_service_bytes_recursive_bytes{container_id="0677d73196f5f4be1d408aab1c4125cf9e6c458a4bea39e590ac779709ffbe14",device="/dev/dm-0",major="253",minor="0",namespace="k8s.io",op="Async"} 0
@ -33,3 +34,32 @@ container_blkio_io_service_bytes_recursive_bytes{container_id="0677d73196f5f4be1
...
...
```
### Pause Image
This change is often required for air-gapped environments, as `containerd` CRI plugin has a reference to the `pause` image which is used
to create pods, and it can't be controlled with Kubernetes pod definitions.
```yaml
machine:
files:
- content: |
[plugins]
[plugins."io.containerd.grpc.v1.cri"]
sandbox_image = "registry.k8s.io/pause:3.8"
path: /etc/cri/conf.d/20-customization.part
op: create
```
Now the `pause` image is set to `registry.k8s.io/pause:3.8`:
```bash
$ talosctl containers --kubernetes
NODE NAMESPACE ID IMAGE PID STATUS
172.20.0.5 k8s.io kube-system/kube-flannel-6hfck registry.k8s.io/pause:3.8 1773 SANDBOX_READY
172.20.0.5 k8s.io └─ kube-system/kube-flannel-6hfck:install-cni ghcr.io/siderolabs/install-cni:v1.3.0-alpha.0-2-gb155fa0 0 CONTAINER_EXITED
172.20.0.5 k8s.io └─ kube-system/kube-flannel-6hfck:install-config ghcr.io/siderolabs/flannel:v0.20.1 0 CONTAINER_EXITED
172.20.0.5 k8s.io └─ kube-system/kube-flannel-6hfck:kube-flannel ghcr.io/siderolabs/flannel:v0.20.1 2092 CONTAINER_RUNNING
172.20.0.5 k8s.io kube-system/kube-proxy-xp7jq registry.k8s.io/pause:3.8 1780 SANDBOX_READY
172.20.0.5 k8s.io └─ kube-system/kube-proxy-xp7jq:kube-proxy k8s.gcr.io/kube-proxy:v1.26.0-alpha.3 1843 CONTAINER_RUNNING
```