mirror of
https://github.com/siderolabs/talos.git
synced 2025-10-28 15:01:13 +01:00
chore: add custom node taints
This PR adds support for custom node taints. Refer to `nodeTaints` in the `configuration` for more information. Closes #7581 Signed-off-by: Dmitriy Matrenichev <dmitry.matrenichev@siderolabs.com>
This commit is contained in:
parent
8e23074665
commit
dd45dd06cf
@ -7,6 +7,7 @@ package k8s
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
|
||||||
"github.com/cosi-project/runtime/pkg/controller"
|
"github.com/cosi-project/runtime/pkg/controller"
|
||||||
"github.com/cosi-project/runtime/pkg/safe"
|
"github.com/cosi-project/runtime/pkg/safe"
|
||||||
@ -53,7 +54,7 @@ func (ctrl *NodeTaintSpecController) Outputs() []controller.Output {
|
|||||||
// Run implements controller.Controller interface.
|
// Run implements controller.Controller interface.
|
||||||
//
|
//
|
||||||
//nolint:gocyclo
|
//nolint:gocyclo
|
||||||
func (ctrl *NodeTaintSpecController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error {
|
func (ctrl *NodeTaintSpecController) Run(ctx context.Context, r controller.Runtime, _ *zap.Logger) error {
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
@ -68,16 +69,24 @@ func (ctrl *NodeTaintSpecController) Run(ctx context.Context, r controller.Runti
|
|||||||
|
|
||||||
r.StartTrackingOutputs()
|
r.StartTrackingOutputs()
|
||||||
|
|
||||||
if cfg != nil && cfg.Config().Machine() != nil && cfg.Config().Cluster() != nil {
|
if cfg != nil && cfg.Config().Machine() != nil {
|
||||||
if cfg.Config().Machine().Type().IsControlPlane() && !cfg.Config().Cluster().ScheduleOnControlPlanes() {
|
if cfg.Config().Cluster() != nil {
|
||||||
if err = safe.WriterModify(ctx, r, k8s.NewNodeTaintSpec(constants.LabelNodeRoleControlPlane), func(k *k8s.NodeTaintSpec) error {
|
if cfg.Config().Machine().Type().IsControlPlane() && !cfg.Config().Cluster().ScheduleOnControlPlanes() {
|
||||||
k.TypedSpec().Key = constants.LabelNodeRoleControlPlane
|
if err = createTaint(ctx, r, constants.LabelNodeRoleControlPlane, "", string(v1.TaintEffectNoSchedule)); err != nil {
|
||||||
k.TypedSpec().Value = ""
|
return err
|
||||||
k.TypedSpec().Effect = string(v1.TaintEffectNoSchedule)
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return nil
|
for key, val := range cfg.Config().Machine().NodeTaints() {
|
||||||
}); err != nil {
|
value, effect, found := strings.Cut(val, ":")
|
||||||
return fmt.Errorf("error updating node taint spec: %w", err)
|
if !found {
|
||||||
|
effect = value
|
||||||
|
value = ""
|
||||||
|
}
|
||||||
|
|
||||||
|
if err = createTaint(ctx, r, key, value, effect); err != nil {
|
||||||
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -87,3 +96,17 @@ func (ctrl *NodeTaintSpecController) Run(ctx context.Context, r controller.Runti
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func createTaint(ctx context.Context, r controller.Runtime, key string, val string, effect string) error {
|
||||||
|
if err := safe.WriterModify(ctx, r, k8s.NewNodeTaintSpec(key), func(k *k8s.NodeTaintSpec) error {
|
||||||
|
k.TypedSpec().Key = key
|
||||||
|
k.TypedSpec().Value = val
|
||||||
|
k.TypedSpec().Effect = effect
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}); err != nil {
|
||||||
|
return fmt.Errorf("error updating node taint spec: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|||||||
@ -11,6 +11,7 @@ import (
|
|||||||
"github.com/cosi-project/runtime/pkg/resource/rtestutils"
|
"github.com/cosi-project/runtime/pkg/resource/rtestutils"
|
||||||
"github.com/cosi-project/runtime/pkg/safe"
|
"github.com/cosi-project/runtime/pkg/safe"
|
||||||
"github.com/cosi-project/runtime/pkg/state"
|
"github.com/cosi-project/runtime/pkg/state"
|
||||||
|
"github.com/siderolabs/gen/xslices"
|
||||||
"github.com/siderolabs/go-pointer"
|
"github.com/siderolabs/go-pointer"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/suite"
|
"github.com/stretchr/testify/suite"
|
||||||
@ -43,16 +44,19 @@ func TestNodeTaintsSuite(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func (suite *NodeTaintsSuite) updateMachineConfig(machineType machine.Type, allowScheduling bool) {
|
func (suite *NodeTaintsSuite) updateMachineConfig(machineType machine.Type, allowScheduling bool, taints ...customTaint) {
|
||||||
cfg, err := safe.StateGetByID[*config.MachineConfig](suite.Ctx(), suite.State(), config.V1Alpha1ID)
|
cfg, err := safe.StateGetByID[*config.MachineConfig](suite.Ctx(), suite.State(), config.V1Alpha1ID)
|
||||||
if err != nil && !state.IsNotFoundError(err) {
|
if err != nil && !state.IsNotFoundError(err) {
|
||||||
suite.Require().NoError(err)
|
suite.Require().NoError(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
nodeTaints := xslices.ToMap(taints, func(t customTaint) (string, string) { return t.key, t.value })
|
||||||
|
|
||||||
if cfg == nil {
|
if cfg == nil {
|
||||||
cfg = config.NewMachineConfig(container.NewV1Alpha1(&v1alpha1.Config{
|
cfg = config.NewMachineConfig(container.NewV1Alpha1(&v1alpha1.Config{
|
||||||
MachineConfig: &v1alpha1.MachineConfig{
|
MachineConfig: &v1alpha1.MachineConfig{
|
||||||
MachineType: machineType.String(),
|
MachineType: machineType.String(),
|
||||||
|
MachineNodeTaints: nodeTaints,
|
||||||
},
|
},
|
||||||
ClusterConfig: &v1alpha1.ClusterConfig{
|
ClusterConfig: &v1alpha1.ClusterConfig{
|
||||||
AllowSchedulingOnControlPlanes: pointer.To(allowScheduling),
|
AllowSchedulingOnControlPlanes: pointer.To(allowScheduling),
|
||||||
@ -63,6 +67,7 @@ func (suite *NodeTaintsSuite) updateMachineConfig(machineType machine.Type, allo
|
|||||||
} else {
|
} else {
|
||||||
cfg.Container().RawV1Alpha1().ClusterConfig.AllowSchedulingOnControlPlanes = pointer.To(allowScheduling)
|
cfg.Container().RawV1Alpha1().ClusterConfig.AllowSchedulingOnControlPlanes = pointer.To(allowScheduling)
|
||||||
cfg.Container().RawV1Alpha1().MachineConfig.MachineType = machineType.String()
|
cfg.Container().RawV1Alpha1().MachineConfig.MachineType = machineType.String()
|
||||||
|
cfg.Container().RawV1Alpha1().MachineConfig.MachineNodeTaints = nodeTaints
|
||||||
suite.Require().NoError(suite.State().Update(suite.Ctx(), cfg))
|
suite.Require().NoError(suite.State().Update(suite.Ctx(), cfg))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -86,3 +91,28 @@ func (suite *NodeTaintsSuite) TestControlplane() {
|
|||||||
|
|
||||||
rtestutils.AssertNoResource[*k8s.NodeTaintSpec](suite.Ctx(), suite.T(), suite.State(), constants.LabelNodeRoleControlPlane)
|
rtestutils.AssertNoResource[*k8s.NodeTaintSpec](suite.Ctx(), suite.T(), suite.State(), constants.LabelNodeRoleControlPlane)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (suite *NodeTaintsSuite) TestCustomTaints() {
|
||||||
|
const customTaintKey = "key1"
|
||||||
|
|
||||||
|
suite.updateMachineConfig(machine.TypeControlPlane, false, customTaint{
|
||||||
|
key: customTaintKey,
|
||||||
|
value: "value1:NoSchedule",
|
||||||
|
})
|
||||||
|
|
||||||
|
rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []string{customTaintKey},
|
||||||
|
func(labelSpec *k8s.NodeTaintSpec, asrt *assert.Assertions) {
|
||||||
|
asrt.Equal(customTaintKey, labelSpec.TypedSpec().Key)
|
||||||
|
asrt.Equal("value1", labelSpec.TypedSpec().Value)
|
||||||
|
asrt.Equal(string(v1.TaintEffectNoSchedule), labelSpec.TypedSpec().Effect)
|
||||||
|
})
|
||||||
|
|
||||||
|
suite.updateMachineConfig(machine.TypeControlPlane, false)
|
||||||
|
|
||||||
|
rtestutils.AssertNoResource[*k8s.NodeTaintSpec](suite.Ctx(), suite.T(), suite.State(), customTaintKey)
|
||||||
|
}
|
||||||
|
|
||||||
|
type customTaint struct {
|
||||||
|
key string
|
||||||
|
value string
|
||||||
|
}
|
||||||
|
|||||||
@ -147,6 +147,7 @@ func (r *Runtime) CanApplyImmediate(cfg config.Provider) error {
|
|||||||
newConfig.MachineConfig.MachinePods = currentConfig.MachineConfig.MachinePods
|
newConfig.MachineConfig.MachinePods = currentConfig.MachineConfig.MachinePods
|
||||||
newConfig.MachineConfig.MachineSeccompProfiles = currentConfig.MachineConfig.MachineSeccompProfiles
|
newConfig.MachineConfig.MachineSeccompProfiles = currentConfig.MachineConfig.MachineSeccompProfiles
|
||||||
newConfig.MachineConfig.MachineNodeLabels = currentConfig.MachineConfig.MachineNodeLabels
|
newConfig.MachineConfig.MachineNodeLabels = currentConfig.MachineConfig.MachineNodeLabels
|
||||||
|
newConfig.MachineConfig.MachineNodeTaints = currentConfig.MachineConfig.MachineNodeTaints
|
||||||
|
|
||||||
if newConfig.MachineConfig.MachineFeatures != nil {
|
if newConfig.MachineConfig.MachineFeatures != nil {
|
||||||
if currentConfig.MachineConfig.MachineFeatures != nil {
|
if currentConfig.MachineConfig.MachineFeatures != nil {
|
||||||
|
|||||||
@ -63,6 +63,8 @@ func (suite *NodeLabelsSuite) TestUpdateWorker() {
|
|||||||
suite.testUpdate(node, false)
|
suite.testUpdate(node, false)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const metadataKeyName = "metadata.name="
|
||||||
|
|
||||||
// testUpdate cycles through a set of node label updates reverting the change in the end.
|
// testUpdate cycles through a set of node label updates reverting the change in the end.
|
||||||
func (suite *NodeLabelsSuite) testUpdate(node string, isControlplane bool) {
|
func (suite *NodeLabelsSuite) testUpdate(node string, isControlplane bool) {
|
||||||
k8sNode, err := suite.GetK8sNodeByInternalIP(suite.ctx, node)
|
k8sNode, err := suite.GetK8sNodeByInternalIP(suite.ctx, node)
|
||||||
@ -71,7 +73,7 @@ func (suite *NodeLabelsSuite) testUpdate(node string, isControlplane bool) {
|
|||||||
suite.T().Logf("updating labels on node %q (%q)", node, k8sNode.Name)
|
suite.T().Logf("updating labels on node %q (%q)", node, k8sNode.Name)
|
||||||
|
|
||||||
watcher, err := suite.Clientset.CoreV1().Nodes().Watch(suite.ctx, metav1.ListOptions{
|
watcher, err := suite.Clientset.CoreV1().Nodes().Watch(suite.ctx, metav1.ListOptions{
|
||||||
FieldSelector: "metadata.name=" + k8sNode.Name,
|
FieldSelector: metadataKeyName + k8sNode.Name,
|
||||||
Watch: true,
|
Watch: true,
|
||||||
})
|
})
|
||||||
suite.Require().NoError(err)
|
suite.Require().NoError(err)
|
||||||
@ -135,7 +137,7 @@ func (suite *NodeLabelsSuite) TestAllowScheduling() {
|
|||||||
suite.T().Logf("updating taints on node %q (%q)", node, k8sNode.Name)
|
suite.T().Logf("updating taints on node %q (%q)", node, k8sNode.Name)
|
||||||
|
|
||||||
watcher, err := suite.Clientset.CoreV1().Nodes().Watch(suite.ctx, metav1.ListOptions{
|
watcher, err := suite.Clientset.CoreV1().Nodes().Watch(suite.ctx, metav1.ListOptions{
|
||||||
FieldSelector: "metadata.name=" + k8sNode.Name,
|
FieldSelector: metadataKeyName + k8sNode.Name,
|
||||||
Watch: true,
|
Watch: true,
|
||||||
})
|
})
|
||||||
suite.Require().NoError(err)
|
suite.Require().NoError(err)
|
||||||
@ -160,9 +162,9 @@ outer:
|
|||||||
select {
|
select {
|
||||||
case ev := <-watcher.ResultChan():
|
case ev := <-watcher.ResultChan():
|
||||||
k8sNode, ok := ev.Object.(*v1.Node)
|
k8sNode, ok := ev.Object.(*v1.Node)
|
||||||
suite.Require().True(ok, "watch event is not of type v1.Node")
|
suite.Require().Truef(ok, "watch event is not of type v1.Node but was %T", ev.Object)
|
||||||
|
|
||||||
suite.T().Logf("labels %v, taints %v", k8sNode.Labels, k8sNode.Spec.Taints)
|
suite.T().Logf("labels %#v, taints %#v", k8sNode.Labels, k8sNode.Spec.Taints)
|
||||||
|
|
||||||
for k, v := range expectedLabels {
|
for k, v := range expectedLabels {
|
||||||
if v == "" {
|
if v == "" {
|
||||||
@ -175,7 +177,7 @@ outer:
|
|||||||
}
|
}
|
||||||
|
|
||||||
if k8sNode.Labels[k] != v {
|
if k8sNode.Labels[k] != v {
|
||||||
suite.T().Logf("label %q is not %q", k, v)
|
suite.T().Logf("label %q is %q but expected %q", k, k8sNode.Labels[k], v)
|
||||||
|
|
||||||
continue outer
|
continue outer
|
||||||
}
|
}
|
||||||
|
|||||||
189
internal/integration/api/node-taints.go
Normal file
189
internal/integration/api/node-taints.go
Normal file
@ -0,0 +1,189 @@
|
|||||||
|
// This Source Code Form is subject to the terms of the Mozilla Public
|
||||||
|
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||||
|
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||||
|
|
||||||
|
//go:build integration_api
|
||||||
|
|
||||||
|
package api
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/siderolabs/gen/maps"
|
||||||
|
"github.com/siderolabs/gen/xslices"
|
||||||
|
"github.com/siderolabs/gen/xtesting/must"
|
||||||
|
v1 "k8s.io/api/core/v1"
|
||||||
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||||
|
"k8s.io/apimachinery/pkg/watch"
|
||||||
|
|
||||||
|
"github.com/siderolabs/talos/internal/integration/base"
|
||||||
|
machineapi "github.com/siderolabs/talos/pkg/machinery/api/machine"
|
||||||
|
"github.com/siderolabs/talos/pkg/machinery/client"
|
||||||
|
"github.com/siderolabs/talos/pkg/machinery/config/container"
|
||||||
|
"github.com/siderolabs/talos/pkg/machinery/config/machine"
|
||||||
|
"github.com/siderolabs/talos/pkg/machinery/constants"
|
||||||
|
)
|
||||||
|
|
||||||
|
// NodeTaintsSuite verifies updating node taints via machine config.
|
||||||
|
type NodeTaintsSuite struct {
|
||||||
|
base.K8sSuite
|
||||||
|
|
||||||
|
ctx context.Context //nolint:containedctx
|
||||||
|
ctxCancel context.CancelFunc
|
||||||
|
}
|
||||||
|
|
||||||
|
// SuiteName ...
|
||||||
|
func (suite *NodeTaintsSuite) SuiteName() string {
|
||||||
|
return "api.NodeTaintsSuite"
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetupTest ...
|
||||||
|
func (suite *NodeTaintsSuite) SetupTest() {
|
||||||
|
// make sure API calls have timeout
|
||||||
|
suite.ctx, suite.ctxCancel = context.WithTimeout(context.Background(), 5*time.Minute)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TearDownTest ...
|
||||||
|
func (suite *NodeTaintsSuite) TearDownTest() {
|
||||||
|
if suite.ctxCancel != nil {
|
||||||
|
suite.ctxCancel()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestUpdateControlPlane verifies node taints updates on control plane nodes.
|
||||||
|
func (suite *NodeTaintsSuite) TestUpdateControlPlane() {
|
||||||
|
node := suite.RandomDiscoveredNodeInternalIP(machine.TypeControlPlane)
|
||||||
|
|
||||||
|
suite.testUpdate(node)
|
||||||
|
}
|
||||||
|
|
||||||
|
// testUpdate cycles through a set of node taints updates reverting the change in the end.
|
||||||
|
func (suite *NodeTaintsSuite) testUpdate(node string) {
|
||||||
|
k8sNode, err := suite.GetK8sNodeByInternalIP(suite.ctx, node)
|
||||||
|
suite.Require().NoError(err)
|
||||||
|
|
||||||
|
suite.T().Logf("updating taints on node %q (%q)", node, k8sNode.Name)
|
||||||
|
|
||||||
|
watcher, err := suite.Clientset.CoreV1().Nodes().Watch(suite.ctx, metav1.ListOptions{
|
||||||
|
FieldSelector: metadataKeyName + k8sNode.Name,
|
||||||
|
Watch: true,
|
||||||
|
})
|
||||||
|
suite.Require().NoError(err)
|
||||||
|
|
||||||
|
defer watcher.Stop()
|
||||||
|
|
||||||
|
// set two new taints
|
||||||
|
suite.setNodeTaints(node, map[string]string{
|
||||||
|
"talos.dev/test1": "value1:NoSchedule",
|
||||||
|
"talos.dev/test2": "NoSchedule",
|
||||||
|
})
|
||||||
|
|
||||||
|
suite.waitUntil(watcher, map[string]string{
|
||||||
|
constants.LabelNodeRoleControlPlane: "NoSchedule",
|
||||||
|
"talos.dev/test1": "value1:NoSchedule",
|
||||||
|
"talos.dev/test2": "NoSchedule",
|
||||||
|
})
|
||||||
|
|
||||||
|
// remove one taint
|
||||||
|
suite.setNodeTaints(node, map[string]string{
|
||||||
|
"talos.dev/test1": "value1:NoSchedule",
|
||||||
|
})
|
||||||
|
|
||||||
|
suite.waitUntil(watcher, map[string]string{
|
||||||
|
constants.LabelNodeRoleControlPlane: "NoSchedule",
|
||||||
|
"talos.dev/test1": "value1:NoSchedule",
|
||||||
|
})
|
||||||
|
|
||||||
|
// remove all taints
|
||||||
|
suite.setNodeTaints(node, nil)
|
||||||
|
|
||||||
|
suite.waitUntil(watcher, map[string]string{
|
||||||
|
constants.LabelNodeRoleControlPlane: "NoSchedule",
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (suite *NodeTaintsSuite) waitUntil(watcher watch.Interface, expectedTaints map[string]string) {
|
||||||
|
outer:
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case ev := <-watcher.ResultChan():
|
||||||
|
k8sNode, ok := ev.Object.(*v1.Node)
|
||||||
|
suite.Require().Truef(ok, "watch event is not of type v1.Node but was %T", ev.Object)
|
||||||
|
|
||||||
|
suite.T().Logf("labels %#v, taints %#v", k8sNode.Labels, k8sNode.Spec.Taints)
|
||||||
|
|
||||||
|
taints := xslices.ToMap(k8sNode.Spec.Taints, func(t v1.Taint) (string, string) {
|
||||||
|
switch {
|
||||||
|
case t.Value == "":
|
||||||
|
return t.Key, string(t.Effect)
|
||||||
|
case t.Effect == "":
|
||||||
|
return t.Key, t.Value
|
||||||
|
default:
|
||||||
|
return t.Key, strings.Join([]string{t.Value, string(t.Effect)}, ":")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
expectedTaintsKeys := maps.Keys(expectedTaints)
|
||||||
|
|
||||||
|
slices.Sort(expectedTaintsKeys)
|
||||||
|
|
||||||
|
for _, key := range expectedTaintsKeys {
|
||||||
|
actualValue, ok := taints[key]
|
||||||
|
if !ok {
|
||||||
|
suite.T().Logf("taint %q is not present", key)
|
||||||
|
|
||||||
|
continue outer
|
||||||
|
}
|
||||||
|
|
||||||
|
expectedValue := expectedTaints[key]
|
||||||
|
|
||||||
|
if expectedValue != actualValue {
|
||||||
|
suite.T().Logf("expected taint %q to be %q but was %q", key, expectedValue, actualValue)
|
||||||
|
|
||||||
|
continue outer
|
||||||
|
}
|
||||||
|
|
||||||
|
delete(taints, key)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(taints) > 0 {
|
||||||
|
keys := maps.Keys(taints)
|
||||||
|
|
||||||
|
slices.Sort(keys)
|
||||||
|
|
||||||
|
suite.T().Logf("taints %v are still present", keys)
|
||||||
|
|
||||||
|
continue outer
|
||||||
|
}
|
||||||
|
|
||||||
|
return
|
||||||
|
case <-suite.ctx.Done():
|
||||||
|
suite.T().Fatal("timeout")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (suite *NodeTaintsSuite) setNodeTaints(nodeIP string, nodeTaints map[string]string) {
|
||||||
|
nodeCtx := client.WithNode(suite.ctx, nodeIP)
|
||||||
|
|
||||||
|
nodeConfig := must.Value(suite.ReadConfigFromNode(nodeCtx))(suite.T())
|
||||||
|
|
||||||
|
nodeConfigRaw := nodeConfig.RawV1Alpha1()
|
||||||
|
suite.Require().NotNil(nodeConfigRaw, "node config is not of type v1alpha1.Config")
|
||||||
|
|
||||||
|
nodeConfigRaw.MachineConfig.MachineNodeTaints = nodeTaints
|
||||||
|
|
||||||
|
bytes := must.Value(container.NewV1Alpha1(nodeConfigRaw).Bytes())(suite.T())
|
||||||
|
|
||||||
|
must.Value(suite.Client.ApplyConfiguration(nodeCtx, &machineapi.ApplyConfigurationRequest{
|
||||||
|
Data: bytes,
|
||||||
|
Mode: machineapi.ApplyConfigurationRequest_NO_REBOOT,
|
||||||
|
}))(suite.T())
|
||||||
|
}
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
allSuites = append(allSuites, new(NodeTaintsSuite))
|
||||||
|
}
|
||||||
@ -42,6 +42,7 @@ type MachineConfig interface {
|
|||||||
Kernel() Kernel
|
Kernel() Kernel
|
||||||
SeccompProfiles() []SeccompProfile
|
SeccompProfiles() []SeccompProfile
|
||||||
NodeLabels() NodeLabels
|
NodeLabels() NodeLabels
|
||||||
|
NodeTaints() NodeTaints
|
||||||
}
|
}
|
||||||
|
|
||||||
// SeccompProfile defines the requirements for a config that pertains to seccomp
|
// SeccompProfile defines the requirements for a config that pertains to seccomp
|
||||||
@ -54,6 +55,9 @@ type SeccompProfile interface {
|
|||||||
// NodeLabels defines the labels that should be set on a node.
|
// NodeLabels defines the labels that should be set on a node.
|
||||||
type NodeLabels map[string]string
|
type NodeLabels map[string]string
|
||||||
|
|
||||||
|
// NodeTaints defines the taints that should be set on a node.
|
||||||
|
type NodeTaints map[string]string
|
||||||
|
|
||||||
// Disk represents the options available for partitioning, formatting, and
|
// Disk represents the options available for partitioning, formatting, and
|
||||||
// mounting extra disks.
|
// mounting extra disks.
|
||||||
type Disk interface {
|
type Disk interface {
|
||||||
|
|||||||
@ -8,6 +8,7 @@ import (
|
|||||||
"net/url"
|
"net/url"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"github.com/siderolabs/gen/xtesting/must"
|
||||||
"github.com/siderolabs/go-pointer"
|
"github.com/siderolabs/go-pointer"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
@ -34,7 +35,7 @@ func TestNew(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
sideroLinkCfg := siderolink.NewConfigV1Alpha1()
|
sideroLinkCfg := siderolink.NewConfigV1Alpha1()
|
||||||
sideroLinkCfg.APIUrlConfig.URL = must(url.Parse("https://siderolink.api/join?jointoken=secret&user=alice"))
|
sideroLinkCfg.APIUrlConfig.URL = must.Value(url.Parse("https://siderolink.api/join?jointoken=secret&user=alice"))(t)
|
||||||
|
|
||||||
cfg, err := container.New(v1alpha1Cfg, sideroLinkCfg)
|
cfg, err := container.New(v1alpha1Cfg, sideroLinkCfg)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@ -81,7 +82,7 @@ func TestValidate(t *testing.T) {
|
|||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
sideroLinkCfg := siderolink.NewConfigV1Alpha1()
|
sideroLinkCfg := siderolink.NewConfigV1Alpha1()
|
||||||
sideroLinkCfg.APIUrlConfig.URL = must(url.Parse("https://siderolink.api/?jointoken=secret&user=alice"))
|
sideroLinkCfg.APIUrlConfig.URL = must.Value(url.Parse("https://siderolink.api/?jointoken=secret&user=alice"))(t)
|
||||||
|
|
||||||
invalidSideroLinkCfg := siderolink.NewConfigV1Alpha1()
|
invalidSideroLinkCfg := siderolink.NewConfigV1Alpha1()
|
||||||
|
|
||||||
@ -89,7 +90,7 @@ func TestValidate(t *testing.T) {
|
|||||||
ClusterConfig: &v1alpha1.ClusterConfig{
|
ClusterConfig: &v1alpha1.ClusterConfig{
|
||||||
ControlPlane: &v1alpha1.ControlPlaneConfig{
|
ControlPlane: &v1alpha1.ControlPlaneConfig{
|
||||||
Endpoint: &v1alpha1.Endpoint{
|
Endpoint: &v1alpha1.Endpoint{
|
||||||
URL: must(url.Parse("https://localhost:6443")),
|
URL: must.Value(url.Parse("https://localhost:6443"))(t),
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
@ -159,14 +160,6 @@ func TestValidate(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func must[T any](t T, err error) T {
|
|
||||||
if err != nil {
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return t
|
|
||||||
}
|
|
||||||
|
|
||||||
type validationMode struct{}
|
type validationMode struct{}
|
||||||
|
|
||||||
func (validationMode) String() string {
|
func (validationMode) String() string {
|
||||||
|
|||||||
@ -2096,6 +2096,18 @@
|
|||||||
"description": "Configures the node labels for the machine.\n",
|
"description": "Configures the node labels for the machine.\n",
|
||||||
"markdownDescription": "Configures the node labels for the machine.",
|
"markdownDescription": "Configures the node labels for the machine.",
|
||||||
"x-intellij-html-description": "\u003cp\u003eConfigures the node labels for the machine.\u003c/p\u003e\n"
|
"x-intellij-html-description": "\u003cp\u003eConfigures the node labels for the machine.\u003c/p\u003e\n"
|
||||||
|
},
|
||||||
|
"nodeTaints": {
|
||||||
|
"patternProperties": {
|
||||||
|
".*": {
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"type": "object",
|
||||||
|
"title": "nodeTaints",
|
||||||
|
"description": "Configures the node taints for the machine. Effect is optional.\n",
|
||||||
|
"markdownDescription": "Configures the node taints for the machine. Effect is optional.",
|
||||||
|
"x-intellij-html-description": "\u003cp\u003eConfigures the node taints for the machine. Effect is optional.\u003c/p\u003e\n"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"additionalProperties": false,
|
"additionalProperties": false,
|
||||||
|
|||||||
@ -90,6 +90,11 @@ func (m *MachineConfig) NodeLabels() config.NodeLabels {
|
|||||||
return m.MachineNodeLabels
|
return m.MachineNodeLabels
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// NodeTaints implements the config.Provider interface.
|
||||||
|
func (m *MachineConfig) NodeTaints() config.NodeTaints {
|
||||||
|
return m.MachineNodeTaints
|
||||||
|
}
|
||||||
|
|
||||||
// Cluster implements the config.Provider interface.
|
// Cluster implements the config.Provider interface.
|
||||||
func (c *Config) Cluster() config.ClusterConfig {
|
func (c *Config) Cluster() config.ClusterConfig {
|
||||||
if c == nil || c.ClusterConfig == nil {
|
if c == nil || c.ClusterConfig == nil {
|
||||||
|
|||||||
@ -281,6 +281,12 @@ type MachineConfig struct {
|
|||||||
// - name: node labels example.
|
// - name: node labels example.
|
||||||
// value: 'map[string]string{"exampleLabel": "exampleLabelValue"}'
|
// value: 'map[string]string{"exampleLabel": "exampleLabelValue"}'
|
||||||
MachineNodeLabels map[string]string `yaml:"nodeLabels,omitempty"`
|
MachineNodeLabels map[string]string `yaml:"nodeLabels,omitempty"`
|
||||||
|
// description: |
|
||||||
|
// Configures the node taints for the machine. Effect is optional.
|
||||||
|
// examples:
|
||||||
|
// - name: node taints example.
|
||||||
|
// value: 'map[string]string{"exampleTaint": "exampleTaintValue:NoSchedule"}'
|
||||||
|
MachineNodeTaints map[string]string `yaml:"nodeTaints,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// MachineSeccompProfile defines seccomp profiles for the machine.
|
// MachineSeccompProfile defines seccomp profiles for the machine.
|
||||||
|
|||||||
@ -255,6 +255,13 @@ func (MachineConfig) Doc() *encoder.Doc {
|
|||||||
Description: "Configures the node labels for the machine.",
|
Description: "Configures the node labels for the machine.",
|
||||||
Comments: [3]string{"" /* encoder.HeadComment */, "Configures the node labels for the machine." /* encoder.LineComment */, "" /* encoder.FootComment */},
|
Comments: [3]string{"" /* encoder.HeadComment */, "Configures the node labels for the machine." /* encoder.LineComment */, "" /* encoder.FootComment */},
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
Name: "nodeTaints",
|
||||||
|
Type: "map[string]string",
|
||||||
|
Note: "",
|
||||||
|
Description: "Configures the node taints for the machine. Effect is optional.",
|
||||||
|
Comments: [3]string{"" /* encoder.HeadComment */, "Configures the node taints for the machine. Effect is optional." /* encoder.LineComment */, "" /* encoder.FootComment */},
|
||||||
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -284,6 +291,7 @@ func (MachineConfig) Doc() *encoder.Doc {
|
|||||||
doc.Fields[20].AddExample("", machineKernelExample())
|
doc.Fields[20].AddExample("", machineKernelExample())
|
||||||
doc.Fields[21].AddExample("", machineSeccompExample())
|
doc.Fields[21].AddExample("", machineSeccompExample())
|
||||||
doc.Fields[22].AddExample("node labels example.", map[string]string{"exampleLabel": "exampleLabelValue"})
|
doc.Fields[22].AddExample("node labels example.", map[string]string{"exampleLabel": "exampleLabelValue"})
|
||||||
|
doc.Fields[23].AddExample("node taints example.", map[string]string{"exampleTaint": "exampleTaintValue:NoSchedule"})
|
||||||
|
|
||||||
return doc
|
return doc
|
||||||
}
|
}
|
||||||
|
|||||||
@ -286,6 +286,10 @@ func (c *Config) Validate(mode validation.RuntimeMode, options ...validation.Opt
|
|||||||
result = multierror.Append(result, fmt.Errorf("invalid machine node labels: %w", err))
|
result = multierror.Append(result, fmt.Errorf("invalid machine node labels: %w", err))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if err := labels.ValidateTaints(c.MachineConfig.MachineNodeTaints); err != nil {
|
||||||
|
result = multierror.Append(result, fmt.Errorf("invalid machine node taints: %w", err))
|
||||||
|
}
|
||||||
|
|
||||||
if c.Machine().Features().KubernetesTalosAPIAccess().Enabled() {
|
if c.Machine().Features().KubernetesTalosAPIAccess().Enabled() {
|
||||||
if !c.Machine().Features().RBACEnabled() {
|
if !c.Machine().Features().RBACEnabled() {
|
||||||
result = multierror.Append(result, fmt.Errorf("feature API RBAC should be enabled when Kubernetes Talos API Access feature is enabled"))
|
result = multierror.Append(result, fmt.Errorf("feature API RBAC should be enabled when Kubernetes Talos API Access feature is enabled"))
|
||||||
|
|||||||
@ -1600,6 +1600,13 @@ func (in *MachineConfig) DeepCopyInto(out *MachineConfig) {
|
|||||||
(*out)[key] = val
|
(*out)[key] = val
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if in.MachineNodeTaints != nil {
|
||||||
|
in, out := &in.MachineNodeTaints, &out.MachineNodeTaints
|
||||||
|
*out = make(map[string]string, len(*in))
|
||||||
|
for key, val := range *in {
|
||||||
|
(*out)[key] = val
|
||||||
|
}
|
||||||
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -1005,6 +1005,13 @@ var UdevdDroppedCapabilities = map[string]struct{}{
|
|||||||
"cap_sys_boot": {},
|
"cap_sys_boot": {},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ValidEffects is the set of valid taint effects.
|
||||||
|
var ValidEffects = []string{
|
||||||
|
"NoSchedule",
|
||||||
|
"PreferNoSchedule",
|
||||||
|
"NoExecute",
|
||||||
|
}
|
||||||
|
|
||||||
// OSReleaseTemplate is the template for /etc/os-release.
|
// OSReleaseTemplate is the template for /etc/os-release.
|
||||||
const OSReleaseTemplate = `NAME="{{ .Name }}"
|
const OSReleaseTemplate = `NAME="{{ .Name }}"
|
||||||
ID={{ .ID }}
|
ID={{ .ID }}
|
||||||
|
|||||||
@ -10,11 +10,13 @@ package labels
|
|||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"regexp"
|
"regexp"
|
||||||
"sort"
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/hashicorp/go-multierror"
|
"github.com/hashicorp/go-multierror"
|
||||||
"github.com/siderolabs/gen/maps"
|
"github.com/siderolabs/gen/maps"
|
||||||
|
|
||||||
|
"github.com/siderolabs/talos/pkg/machinery/constants"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Validate validates that a set of labels are correctly defined.
|
// Validate validates that a set of labels are correctly defined.
|
||||||
@ -22,7 +24,7 @@ func Validate(labels map[string]string) error {
|
|||||||
var multiErr *multierror.Error
|
var multiErr *multierror.Error
|
||||||
|
|
||||||
keys := maps.Keys(labels)
|
keys := maps.Keys(labels)
|
||||||
sort.Strings(keys)
|
slices.Sort(keys)
|
||||||
|
|
||||||
for _, k := range keys {
|
for _, k := range keys {
|
||||||
if err := ValidateQualifiedName(k); err != nil {
|
if err := ValidateQualifiedName(k); err != nil {
|
||||||
@ -130,3 +132,41 @@ func ValidateLabelValue(value string) error {
|
|||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ValidateTaints validates that a set of taints is correctly defined.
|
||||||
|
func ValidateTaints(taints map[string]string) error {
|
||||||
|
var multiErr *multierror.Error
|
||||||
|
|
||||||
|
keys := maps.Keys(taints)
|
||||||
|
slices.Sort(keys)
|
||||||
|
|
||||||
|
for _, k := range keys {
|
||||||
|
if err := ValidateQualifiedName(k); err != nil {
|
||||||
|
multiErr = multierror.Append(multiErr, err)
|
||||||
|
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
val, effect, found := strings.Cut(taints[k], ":")
|
||||||
|
if !found {
|
||||||
|
effect = val
|
||||||
|
}
|
||||||
|
|
||||||
|
// validate that the taint has a valid effect, which is required to add the taint
|
||||||
|
if !slices.Contains(constants.ValidEffects, effect) {
|
||||||
|
multiErr = multierror.Append(multiErr, fmt.Errorf("invalid taint effect: %q", effect))
|
||||||
|
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if found {
|
||||||
|
if err := ValidateLabelValue(val); err != nil {
|
||||||
|
multiErr = multierror.Append(multiErr, err)
|
||||||
|
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return multiErr.ErrorOrNil()
|
||||||
|
}
|
||||||
|
|||||||
@ -55,3 +55,44 @@ func TestValidate(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestValidateTaints(t *testing.T) {
|
||||||
|
for _, tt := range []struct {
|
||||||
|
name string
|
||||||
|
taints map[string]string
|
||||||
|
|
||||||
|
expectedError string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "empty",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "valid",
|
||||||
|
taints: map[string]string{
|
||||||
|
"foor": "bar:NoExecute",
|
||||||
|
"doo": "NoExecute",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid",
|
||||||
|
taints: map[string]string{
|
||||||
|
strings.Repeat("a", 64): "bar",
|
||||||
|
"bar": strings.Repeat("a", 64),
|
||||||
|
"foo": "bar:NoExecute:NoSchedule",
|
||||||
|
"loo": "bar:",
|
||||||
|
"zoo": "bar:NoExocute",
|
||||||
|
"koo": "key",
|
||||||
|
},
|
||||||
|
expectedError: "6 errors occurred:\n\t* name is too long: \"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\" (limit is 63)\n\t* invalid taint effect: \"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\"\n\t* invalid taint effect: \"NoExecute:NoSchedule\"\n\t* invalid taint effect: \"key\"\n\t* invalid taint effect: \"\"\n\t* invalid taint effect: \"NoExocute\"\n\n", //nolint:lll
|
||||||
|
},
|
||||||
|
} {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
err := labels.ValidateTaints(tt.taints)
|
||||||
|
if tt.expectedError != "" {
|
||||||
|
assert.EqualError(t, err, tt.expectedError)
|
||||||
|
} else {
|
||||||
|
assert.NoError(t, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@ -433,6 +433,10 @@ seccompProfiles:
|
|||||||
nodeLabels:
|
nodeLabels:
|
||||||
exampleLabel: exampleLabelValue
|
exampleLabel: exampleLabelValue
|
||||||
{{< /highlight >}}</details> | |
|
{{< /highlight >}}</details> | |
|
||||||
|
|`nodeTaints` |map[string]string |Configures the node taints for the machine. Effect is optional. <details><summary>Show example(s)</summary>{{< highlight yaml >}}
|
||||||
|
nodeTaints:
|
||||||
|
exampleTaint: exampleTaintValue:NoSchedule
|
||||||
|
{{< /highlight >}}</details> | |
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@ -2096,6 +2096,18 @@
|
|||||||
"description": "Configures the node labels for the machine.\n",
|
"description": "Configures the node labels for the machine.\n",
|
||||||
"markdownDescription": "Configures the node labels for the machine.",
|
"markdownDescription": "Configures the node labels for the machine.",
|
||||||
"x-intellij-html-description": "\u003cp\u003eConfigures the node labels for the machine.\u003c/p\u003e\n"
|
"x-intellij-html-description": "\u003cp\u003eConfigures the node labels for the machine.\u003c/p\u003e\n"
|
||||||
|
},
|
||||||
|
"nodeTaints": {
|
||||||
|
"patternProperties": {
|
||||||
|
".*": {
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"type": "object",
|
||||||
|
"title": "nodeTaints",
|
||||||
|
"description": "Configures the node taints for the machine. Effect is optional.\n",
|
||||||
|
"markdownDescription": "Configures the node taints for the machine. Effect is optional.",
|
||||||
|
"x-intellij-html-description": "\u003cp\u003eConfigures the node taints for the machine. Effect is optional.\u003c/p\u003e\n"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"additionalProperties": false,
|
"additionalProperties": false,
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user