sidero/sfyra/pkg/tests/server_class.go

// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

package tests

import (
	"context"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"sort"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"github.com/talos-systems/go-retry/retry"
	"github.com/talos-systems/talos/pkg/machinery/config/configloader"
	talosconfig "github.com/talos-systems/talos/pkg/machinery/config/types/v1alpha1"
	apiextensions "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/api/meta"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/client-go/discovery"
	"k8s.io/client-go/discovery/cached/memory"
	"k8s.io/client-go/dynamic"
	"k8s.io/client-go/restmapper"
	capiclient "sigs.k8s.io/cluster-api/cmd/clusterctl/client"
	"sigs.k8s.io/controller-runtime/pkg/client"

	"github.com/talos-systems/sidero/app/metal-controller-manager/api/v1alpha1"
	"github.com/talos-systems/sidero/sfyra/pkg/capi"
	"github.com/talos-systems/sidero/sfyra/pkg/constants"
	"github.com/talos-systems/sidero/sfyra/pkg/talos"
	"github.com/talos-systems/sidero/sfyra/pkg/vm"
)

const (
	defaultServerClassName  = "default"
	workloadServerClassName = "workload"
)

// TestServerClassDefault verifies server class creation.
func TestServerClassDefault(ctx context.Context, metalClient client.Client, vmSet *vm.Set) TestFunc {
	return func(t *testing.T) {
		classSpec := v1alpha1.ServerClassSpec{
			Qualifiers: v1alpha1.Qualifiers{
				CPU: []v1alpha1.CPUInformation{
					{
						Manufacturer: "QEMU",
					},
				},
			},
		}

		serverClass, err := createServerClass(ctx, metalClient, defaultServerClassName, classSpec)
		require.NoError(t, err)

		numNodes := len(vmSet.Nodes())

		// wait for the server class to gather all nodes (all nodes should match)
		require.NoError(t, retry.Constant(2*time.Minute, retry.WithUnits(10*time.Second)).Retry(func() error {
			if err := metalClient.Get(ctx, types.NamespacedName{Name: defaultServerClassName}, &serverClass); err != nil {
				return retry.UnexpectedError(err)
			}

			if len(serverClass.Status.ServersAvailable)+len(serverClass.Status.ServersInUse) != numNodes {
				return retry.ExpectedError(fmt.Errorf("%d + %d != %d", len(serverClass.Status.ServersAvailable), len(serverClass.Status.ServersInUse), numNodes))
			}

			return nil
		}))

		assert.Len(t, append(serverClass.Status.ServersAvailable, serverClass.Status.ServersInUse...), numNodes)

		nodes := vmSet.Nodes()
		expectedUUIDs := make([]string, len(nodes))

		for i := range nodes {
			expectedUUIDs[i] = nodes[i].UUID.String()
		}

		actualUUIDs := append(serverClass.Status.ServersAvailable, serverClass.Status.ServersInUse...)

		sort.Strings(expectedUUIDs)
		sort.Strings(actualUUIDs)

		assert.Equal(t, expectedUUIDs, actualUUIDs)

		_, err = createServerClass(ctx, metalClient, workloadServerClassName, classSpec)
		require.NoError(t, err)
	}
}

// TestServerClassPatch verifies config patches work at the server level.
func TestServerClassPatch(ctx context.Context, metalClient client.Client, cluster talos.Cluster, capiManager *capi.Manager) TestFunc {
	return func(t *testing.T) {
		// Create dummy serverclass + a server
		dummySpec := v1alpha1.ServerSpec{
			CPU: &v1alpha1.CPUInformation{
				Manufacturer: "DummyCPU",
			},
			Accepted: true,
		}

		dummyServer, err := createDummyServer(ctx, metalClient, "dummyserver-0", dummySpec)
		require.NoError(t, err)

		//nolint: errcheck
		defer metalClient.Delete(ctx, &dummyServer)

		installConfig := talosconfig.InstallConfig{
			InstallExtraKernelArgs: []string{
				"woot",
			},
		}

		installPatch := configPatchToJSON(t, &installConfig)

		classSpec := v1alpha1.ServerClassSpec{
			ConfigPatches: []v1alpha1.ConfigPatches{
				{
					Op:    "replace",
					Path:  "/machine/install",
					Value: apiextensions.JSON{Raw: installPatch},
				},
			},
			Qualifiers: v1alpha1.Qualifiers{
				CPU: []v1alpha1.CPUInformation{
					{
						Manufacturer: "DummyCPU",
					},
				},
			},
		}

		dummyClass, err := createServerClass(ctx, metalClient, "dummyservers", classSpec)
		require.NoError(t, err)

		//nolint: errcheck
		defer metalClient.Delete(ctx, &dummyClass)

		// Create "cluster" using serverclass above
		kubeconfig, err := capiManager.GetKubeconfig(ctx)
		require.NoError(t, err)

		config, err := cluster.KubernetesClient().K8sRestConfig(ctx)
		require.NoError(t, err)

		capiClient := capiManager.GetManagerClient()

		nodeCountCP := int64(1)
		nodeCountWorker := int64(0)

		os.Setenv("CONTROL_PLANE_ENDPOINT", "localhost")
		os.Setenv("CONTROL_PLANE_PORT", "11111")
		os.Setenv("CONTROL_PLANE_SERVERCLASS", "dummyservers")
		os.Setenv("WORKER_SERVERCLASS", "dummyservers")
		// TODO: make it configurable
		os.Setenv("KUBERNETES_VERSION", "v1.19.0")

		templateOptions := capiclient.GetClusterTemplateOptions{
			Kubeconfig:               kubeconfig,
			ClusterName:              "serverclass-config-patch-test",
			ControlPlaneMachineCount: &nodeCountCP,
			WorkerMachineCount:       &nodeCountWorker,
		}

		template, err := capiClient.GetClusterTemplate(templateOptions)
		require.NoError(t, err)

		dc, err := discovery.NewDiscoveryClientForConfig(config)
		require.NoError(t, err)

		mapper := restmapper.NewDeferredDiscoveryRESTMapper(memory.NewMemCacheClient(dc))

		dyn, err := dynamic.NewForConfig(config)
		require.NoError(t, err)

		for _, obj := range template.Objs() {
			var mapping *meta.RESTMapping

			mapping, err = mapper.RESTMapping(obj.GroupVersionKind().GroupKind(), obj.GroupVersionKind().Version)
			require.NoError(t, err)

			var dr dynamic.ResourceInterface
			if mapping.Scope.Name() == meta.RESTScopeNameNamespace {
				// namespaced resources should specify the namespace
				dr = dyn.Resource(mapping.Resource).Namespace(obj.GetNamespace())
			} else {
				// for cluster-wide resources
				dr = dyn.Resource(mapping.Resource)
			}

			obj := obj

			_, err = dr.Create(ctx, &obj, metav1.CreateOptions{})
			require.NoError(t, err)

			if obj.GroupVersionKind().Kind == "Cluster" {
				//nolint: errcheck
				defer dr.Delete(ctx, obj.GetName(), metav1.DeleteOptions{})
			}
		}

		// Wait for metadata server to return a 200 for that UUID, then verify it has patch.
		metadataEndpoint := fmt.Sprintf("http://%s:9091/configdata?uuid=dummyserver-0", cluster.SideroComponentsIP())

		var metadataBytes []byte

		require.NoError(t, retry.Constant(5*time.Minute, retry.WithUnits(10*time.Second)).Retry(func() error {
			req, _ := http.NewRequestWithContext(ctx, "GET", metadataEndpoint, nil)
			client := &http.Client{}
			response, err := client.Do(req)
			if err != nil {
				return retry.UnexpectedError(err)
			}

			if response.StatusCode != http.StatusOK {
				return retry.ExpectedError(fmt.Errorf("metadata not yet present: %d", response.StatusCode))
			}

			defer response.Body.Close()
			metadataBytes, err = ioutil.ReadAll(response.Body)
			if err != nil {
				return retry.UnexpectedError(err)
			}
			return nil
		}))

		configProvider, err := configloader.NewFromBytes(metadataBytes)
		require.NoError(t, err)

		switch configProvider.Version() {
		case "v1alpha1":
			config, ok := configProvider.(*talosconfig.Config)
			if !ok {
				t.Error("unable to case config")
			}

			require.Len(t, config.MachineConfig.MachineInstall.InstallExtraKernelArgs, 1)

			if config.MachineConfig.MachineInstall.InstallExtraKernelArgs[0] != "woot" {
				t.Error("unable to validate server class patch was applied to kernel args")
			}

		default:
			t.Error("unknown config type")
		}
	}
}

func createServerClass(ctx context.Context, metalClient client.Client, name string, spec v1alpha1.ServerClassSpec) (v1alpha1.ServerClass, error) {
	var retClass v1alpha1.ServerClass

	if err := metalClient.Get(ctx, types.NamespacedName{Name: name}, &retClass); err != nil {
		if !apierrors.IsNotFound(err) {
			return retClass, nil
		}

		retClass.APIVersion = constants.SideroAPIVersion
		retClass.Name = name
		retClass.Spec = spec

		err = metalClient.Create(ctx, &retClass)
		if err != nil {
			return retClass, err
		}
	}

	return retClass, nil
}