talos/internal/app/machined/pkg/controllers/k8s/manifest_test.go

// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

//nolint:dupl
package k8s_test

import (
	"context"
	"fmt"
	"log"
	"reflect"
	"sync"
	"testing"
	"time"

	"github.com/cosi-project/runtime/pkg/controller/runtime"
	"github.com/cosi-project/runtime/pkg/resource"
	"github.com/cosi-project/runtime/pkg/state"
	"github.com/cosi-project/runtime/pkg/state/impl/inmem"
	"github.com/cosi-project/runtime/pkg/state/impl/namespaced"
	"github.com/stretchr/testify/suite"
	"github.com/talos-systems/go-retry/retry"

	k8sadapter "github.com/talos-systems/talos/internal/app/machined/pkg/adapters/k8s"
	k8sctrl "github.com/talos-systems/talos/internal/app/machined/pkg/controllers/k8s"
	"github.com/talos-systems/talos/pkg/logging"
	"github.com/talos-systems/talos/pkg/machinery/constants"
	"github.com/talos-systems/talos/pkg/machinery/resources/config"
	"github.com/talos-systems/talos/pkg/machinery/resources/k8s"
	"github.com/talos-systems/talos/pkg/machinery/resources/secrets"
)

type ManifestSuite struct {
	suite.Suite

	state state.State

	runtime *runtime.Runtime
	wg      sync.WaitGroup

	ctx       context.Context
	ctxCancel context.CancelFunc
}

func (suite *ManifestSuite) SetupTest() {
	suite.ctx, suite.ctxCancel = context.WithTimeout(context.Background(), 3*time.Minute)

	suite.state = state.WrapCore(namespaced.NewState(inmem.Build))

	var err error

	suite.runtime, err = runtime.NewRuntime(suite.state, logging.Wrap(log.Writer()))
	suite.Require().NoError(err)

	suite.Require().NoError(suite.runtime.RegisterController(&k8sctrl.ManifestController{}))

	suite.startRuntime()
}

func (suite *ManifestSuite) startRuntime() {
	suite.wg.Add(1)

	go func() {
		defer suite.wg.Done()

		suite.Assert().NoError(suite.runtime.Run(suite.ctx))
	}()
}

//nolint:dupl
func (suite *ManifestSuite) assertManifests(manifests []string) error {
	resources, err := suite.state.List(suite.ctx, resource.NewMetadata(k8s.ControlPlaneNamespaceName, k8s.ManifestType, "", resource.VersionUndefined))
	if err != nil {
		return err
	}

	ids := make([]string, 0, len(resources.Items))

	for _, res := range resources.Items {
		ids = append(ids, res.Metadata().ID())
	}

	if !reflect.DeepEqual(manifests, ids) {
		return retry.ExpectedError(fmt.Errorf("expected %q, got %q", manifests, ids))
	}

	return nil
}

var defaultManifestSpec = config.K8sManifestsSpec{
	Server:        "127.0.0.1",
	ClusterDomain: "cluster.",

	PodCIDRs: []string{constants.DefaultIPv4PodNet},

	ProxyEnabled: true,
	ProxyImage:   "foo/bar",
	ProxyArgs: []string{
		fmt.Sprintf("--cluster-cidr=%s", constants.DefaultIPv4PodNet),
		"--hostname-override=$(NODE_NAME)",
		"--kubeconfig=/etc/kubernetes/kubeconfig",
		"--proxy-mode=iptables",
		"--conntrack-max-per-core=0",
	},

	CoreDNSEnabled: true,
	CoreDNSImage:   "foo/bar",

	DNSServiceIP: "192.168.0.1",

	FlannelEnabled:  true,
	FlannelImage:    "foo/bar",
	FlannelCNIImage: "foo/bar",

	PodSecurityPolicyEnabled: true,
}

func (suite *ManifestSuite) TestReconcileDefaults() {
	rootSecrets := secrets.NewKubernetesRoot(secrets.KubernetesRootID)
	manifestConfig := config.NewK8sManifests()
	manifestConfig.SetManifests(defaultManifestSpec)

	suite.Require().NoError(suite.state.Create(suite.ctx, rootSecrets))
	suite.Require().NoError(suite.state.Create(suite.ctx, manifestConfig))

	suite.Assert().NoError(retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry(
		func() error {
			return suite.assertManifests(
				[]string{
					"00-kubelet-bootstrapping-token",
					"01-csr-approver-role-binding",
					"01-csr-node-bootstrap",
					"01-csr-renewal-role-binding",
					"02-kube-system-sa-role-binding",
					"03-default-pod-security-policy",
					"05-flannel",
					"10-kube-proxy",
					"11-core-dns",
					"11-core-dns-svc",
					"11-kube-config-in-cluster",
				},
			)
		},
	))
}

func (suite *ManifestSuite) TestReconcileDisableKubeProxy() {
	rootSecrets := secrets.NewKubernetesRoot(secrets.KubernetesRootID)
	manifestConfig := config.NewK8sManifests()
	spec := defaultManifestSpec
	spec.ProxyEnabled = false
	manifestConfig.SetManifests(spec)

	suite.Require().NoError(suite.state.Create(suite.ctx, rootSecrets))
	suite.Require().NoError(suite.state.Create(suite.ctx, manifestConfig))

	suite.Assert().NoError(retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry(
		func() error {
			return suite.assertManifests(
				[]string{
					"00-kubelet-bootstrapping-token",
					"01-csr-approver-role-binding",
					"01-csr-node-bootstrap",
					"01-csr-renewal-role-binding",
					"02-kube-system-sa-role-binding",
					"03-default-pod-security-policy",
					"05-flannel",
					"11-core-dns",
					"11-core-dns-svc",
					"11-kube-config-in-cluster",
				},
			)
		},
	))
}

func (suite *ManifestSuite) TestReconcileKubeProxyExtraArgs() {
	rootSecrets := secrets.NewKubernetesRoot(secrets.KubernetesRootID)
	manifestConfig := config.NewK8sManifests()
	spec := defaultManifestSpec
	spec.ProxyArgs = append(spec.ProxyArgs, "--bind-address=\"::\"")
	manifestConfig.SetManifests(spec)

	suite.Require().NoError(suite.state.Create(suite.ctx, rootSecrets))
	suite.Require().NoError(suite.state.Create(suite.ctx, manifestConfig))

	suite.Assert().NoError(retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry(
		func() error {
			return suite.assertManifests(
				[]string{
					"00-kubelet-bootstrapping-token",
					"01-csr-approver-role-binding",
					"01-csr-node-bootstrap",
					"01-csr-renewal-role-binding",
					"02-kube-system-sa-role-binding",
					"03-default-pod-security-policy",
					"05-flannel",
					"10-kube-proxy",
					"11-core-dns",
					"11-core-dns-svc",
					"11-kube-config-in-cluster",
				},
			)
		},
	))

	r, err := suite.state.Get(suite.ctx, resource.NewMetadata(k8s.ControlPlaneNamespaceName, k8s.ManifestType, "10-kube-proxy", resource.VersionUndefined))
	suite.Require().NoError(err)

	manifest := r.(*k8s.Manifest) //nolint:errcheck,forcetypeassert
	suite.Assert().Len(k8sadapter.Manifest(manifest).Objects(), 3)

	suite.Assert().Equal("DaemonSet", k8sadapter.Manifest(manifest).Objects()[0].GetKind())

	ds := k8sadapter.Manifest(manifest).Objects()[0].Object
	containerSpec := ds["spec"].(map[string]interface{})["template"].(map[string]interface{})["spec"].(map[string]interface{})["containers"].([]interface{})[0]
	args := containerSpec.(map[string]interface{})["command"].([]interface{}) //nolint:errcheck,forcetypeassert

	suite.Assert().Equal("--bind-address=\"::\"", args[len(args)-1])
}

func (suite *ManifestSuite) TestReconcileDisablePSP() {
	rootSecrets := secrets.NewKubernetesRoot(secrets.KubernetesRootID)
	manifestConfig := config.NewK8sManifests()
	spec := defaultManifestSpec
	spec.PodSecurityPolicyEnabled = false
	manifestConfig.SetManifests(spec)

	suite.Require().NoError(suite.state.Create(suite.ctx, rootSecrets))
	suite.Require().NoError(suite.state.Create(suite.ctx, manifestConfig))

	suite.Assert().NoError(retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry(
		func() error {
			return suite.assertManifests(
				[]string{
					"00-kubelet-bootstrapping-token",
					"01-csr-approver-role-binding",
					"01-csr-node-bootstrap",
					"01-csr-renewal-role-binding",
					"02-kube-system-sa-role-binding",
					"05-flannel",
					"10-kube-proxy",
					"11-core-dns",
					"11-core-dns-svc",
					"11-kube-config-in-cluster",
				},
			)
		},
	))
}

func (suite *ManifestSuite) TearDownTest() {
	suite.T().Log("tear down")

	suite.ctxCancel()

	suite.wg.Wait()

	// trigger updates in resources to stop watch loops
	suite.Assert().NoError(suite.state.Create(context.Background(), secrets.NewKubernetesRoot("-")))
	suite.Assert().NoError(suite.state.Create(context.Background(), config.NewK8sControlPlaneAPIServer()))
}

func TestManifestSuite(t *testing.T) {
	suite.Run(t, new(ManifestSuite))
}