talos/internal/app/machined/pkg/controllers/secrets/kubernetes_test.go

// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

//nolint:dupl
package secrets_test

import (
	"context"
	stdlibx509 "crypto/x509"
	"fmt"
	"log"
	"net"
	"net/url"
	"sync"
	"testing"
	"time"

	"github.com/cosi-project/runtime/pkg/controller/runtime"
	"github.com/cosi-project/runtime/pkg/resource"
	"github.com/cosi-project/runtime/pkg/state"
	"github.com/cosi-project/runtime/pkg/state/impl/inmem"
	"github.com/cosi-project/runtime/pkg/state/impl/namespaced"
	"github.com/stretchr/testify/suite"
	"github.com/talos-systems/crypto/x509"
	"github.com/talos-systems/go-retry/retry"
	"k8s.io/client-go/tools/clientcmd"

	secretsctrl "github.com/talos-systems/talos/internal/app/machined/pkg/controllers/secrets"
	"github.com/talos-systems/talos/pkg/logging"
	"github.com/talos-systems/talos/pkg/machinery/config/types/v1alpha1/machine"
	"github.com/talos-systems/talos/pkg/machinery/constants"
	"github.com/talos-systems/talos/pkg/machinery/resources/config"
	"github.com/talos-systems/talos/pkg/machinery/resources/network"
	"github.com/talos-systems/talos/pkg/machinery/resources/secrets"
	timeresource "github.com/talos-systems/talos/pkg/machinery/resources/time"
)

type KubernetesSuite struct {
	suite.Suite

	state state.State

	runtime *runtime.Runtime
	wg      sync.WaitGroup

	ctx       context.Context
	ctxCancel context.CancelFunc
}

func (suite *KubernetesSuite) SetupTest() {
	suite.ctx, suite.ctxCancel = context.WithTimeout(context.Background(), 3*time.Minute)

	suite.state = state.WrapCore(namespaced.NewState(inmem.Build))

	var err error

	suite.runtime, err = runtime.NewRuntime(suite.state, logging.Wrap(log.Writer()))
	suite.Require().NoError(err)

	suite.Require().NoError(suite.runtime.RegisterController(&secretsctrl.KubernetesController{}))

	suite.startRuntime()
}

func (suite *KubernetesSuite) startRuntime() {
	suite.wg.Add(1)

	go func() {
		defer suite.wg.Done()

		suite.Assert().NoError(suite.runtime.Run(suite.ctx))
	}()
}

func (suite *KubernetesSuite) TestReconcile() {
	rootSecrets := secrets.NewKubernetesRoot(secrets.KubernetesRootID)

	k8sCA, err := x509.NewSelfSignedCertificateAuthority(
		x509.Organization("kubernetes"),
		x509.ECDSA(true),
	)
	suite.Require().NoError(err)

	aggregatorCA, err := x509.NewSelfSignedCertificateAuthority(
		x509.Organization("kubernetes"),
		x509.ECDSA(true),
	)
	suite.Require().NoError(err)

	serviceAccount, err := x509.NewECDSAKey()
	suite.Require().NoError(err)

	rootSecrets.TypedSpec().Name = "cluster1"
	rootSecrets.TypedSpec().Endpoint, err = url.Parse("https://some.url:6443/")
	suite.Require().NoError(err)

	rootSecrets.TypedSpec().CA = &x509.PEMEncodedCertificateAndKey{
		Crt: k8sCA.CrtPEM,
		Key: k8sCA.KeyPEM,
	}
	rootSecrets.TypedSpec().AggregatorCA = &x509.PEMEncodedCertificateAndKey{
		Crt: aggregatorCA.CrtPEM,
		Key: aggregatorCA.KeyPEM,
	}
	rootSecrets.TypedSpec().ServiceAccount = &x509.PEMEncodedKey{
		Key: serviceAccount.KeyPEM,
	}
	rootSecrets.TypedSpec().CertSANs = []string{"example.com"}
	rootSecrets.TypedSpec().APIServerIPs = []net.IP{net.ParseIP("10.4.3.2"), net.ParseIP("10.2.1.3")}
	rootSecrets.TypedSpec().DNSDomain = "cluster.remote"
	suite.Require().NoError(suite.state.Create(suite.ctx, rootSecrets))

	machineType := config.NewMachineType()
	machineType.SetMachineType(machine.TypeControlPlane)
	suite.Require().NoError(suite.state.Create(suite.ctx, machineType))

	networkStatus := network.NewStatus(network.NamespaceName, network.StatusID)
	networkStatus.TypedSpec().AddressReady = true
	networkStatus.TypedSpec().HostnameReady = true
	suite.Require().NoError(suite.state.Create(suite.ctx, networkStatus))

	certSANs := secrets.NewCertSAN(secrets.NamespaceName, secrets.CertSANKubernetesID)
	certSANs.TypedSpec().Append(
		"example.com",
		"foo",
		"foo.example.com",
		"kubernetes",
		"kubernetes.default",
		"kubernetes.default.svc",
		"kubernetes.default.svc.cluster.remote",
		"localhost",
		"some.url",
		"10.2.1.3",
		"10.4.3.2",
		"172.16.0.1",
	)
	suite.Require().NoError(suite.state.Create(suite.ctx, certSANs))

	timeSync := timeresource.NewStatus()
	timeSync.SetStatus(timeresource.StatusSpec{
		Synced: true,
	})
	suite.Require().NoError(suite.state.Create(suite.ctx, timeSync))

	suite.Assert().NoError(retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry(
		func() error {
			certs, err := suite.state.Get(suite.ctx, resource.NewMetadata(secrets.NamespaceName, secrets.KubernetesType, secrets.KubernetesID, resource.VersionUndefined))
			if err != nil {
				if state.IsNotFoundError(err) {
					return retry.ExpectedError(err)
				}

				return err
			}

			kubernetesCerts := certs.(*secrets.Kubernetes).Certs()

			apiCert, err := kubernetesCerts.APIServer.GetCert()
			suite.Require().NoError(err)

			suite.Assert().Equal(
				[]string{
					"example.com",
					"foo",
					"foo.example.com",
					"kubernetes",
					"kubernetes.default",
					"kubernetes.default.svc",
					"kubernetes.default.svc.cluster.remote",
					"localhost",
					"some.url",
				}, apiCert.DNSNames)
			suite.Assert().Equal("[10.2.1.3 10.4.3.2 172.16.0.1]", fmt.Sprintf("%v", apiCert.IPAddresses))

			suite.Assert().Equal("kube-apiserver", apiCert.Subject.CommonName)
			suite.Assert().Equal([]string{"kube-master"}, apiCert.Subject.Organization)

			suite.Assert().Equal(stdlibx509.KeyUsageDigitalSignature|stdlibx509.KeyUsageKeyEncipherment, apiCert.KeyUsage)
			suite.Assert().Equal([]stdlibx509.ExtKeyUsage{stdlibx509.ExtKeyUsageServerAuth}, apiCert.ExtKeyUsage)

			clientCert, err := kubernetesCerts.APIServerKubeletClient.GetCert()
			suite.Require().NoError(err)

			suite.Assert().Empty(clientCert.DNSNames)
			suite.Assert().Empty(clientCert.IPAddresses)

			suite.Assert().Equal(constants.KubernetesAPIServerKubeletClientCommonName, clientCert.Subject.CommonName)
			suite.Assert().Equal([]string{constants.KubernetesAdminCertOrganization}, clientCert.Subject.Organization)

			suite.Assert().Equal(stdlibx509.KeyUsageDigitalSignature|stdlibx509.KeyUsageKeyEncipherment, clientCert.KeyUsage)
			suite.Assert().Equal([]stdlibx509.ExtKeyUsage{stdlibx509.ExtKeyUsageClientAuth}, clientCert.ExtKeyUsage)

			frontProxyCert, err := kubernetesCerts.FrontProxy.GetCert()
			suite.Require().NoError(err)

			suite.Assert().Empty(frontProxyCert.DNSNames)
			suite.Assert().Empty(frontProxyCert.IPAddresses)

			suite.Assert().Equal("front-proxy-client", frontProxyCert.Subject.CommonName)
			suite.Assert().Empty(frontProxyCert.Subject.Organization)

			suite.Assert().Equal(stdlibx509.KeyUsageDigitalSignature|stdlibx509.KeyUsageKeyEncipherment, frontProxyCert.KeyUsage)
			suite.Assert().Equal([]stdlibx509.ExtKeyUsage{stdlibx509.ExtKeyUsageClientAuth}, frontProxyCert.ExtKeyUsage)

			for _, kubeconfig := range []string{kubernetesCerts.ControllerManagerKubeconfig, kubernetesCerts.SchedulerKubeconfig, kubernetesCerts.AdminKubeconfig} {
				config, err := clientcmd.Load([]byte(kubeconfig))
				suite.Require().NoError(err)

				suite.Assert().NoError(clientcmd.ConfirmUsable(*config, config.CurrentContext))
			}

			return nil
		},
	))
}

func (suite *KubernetesSuite) TearDownTest() {
	suite.T().Log("tear down")

	suite.ctxCancel()

	suite.wg.Wait()
}

func TestKubernetesSuite(t *testing.T) {
	suite.Run(t, new(KubernetesSuite))
}