From ad5f1ed9a941384576cace3a807aad1a9bd6fb0b Mon Sep 17 00:00:00 2001 From: Andrey Smirnov Date: Thu, 26 Nov 2020 18:18:26 +0300 Subject: [PATCH] test: add a test to deploy and destroy workload cluster This cluster has a single controlplane node just for the sake of testing with another cluster in place. Signed-off-by: Andrey Smirnov --- ...cture.cluster.x-k8s.io_serverbindings.yaml | 20 +-- sfyra/pkg/capi/cluster.go | 12 ++ sfyra/pkg/tests/cluster_utils.go | 157 ++++++++++++++++++ sfyra/pkg/tests/management_cluster.go | 106 +----------- sfyra/pkg/tests/server_class.go | 12 +- sfyra/pkg/tests/tests.go | 4 + sfyra/pkg/tests/workload_cluster.go | 31 ++++ 7 files changed, 224 insertions(+), 118 deletions(-) create mode 100644 sfyra/pkg/tests/cluster_utils.go create mode 100644 sfyra/pkg/tests/workload_cluster.go diff --git a/app/cluster-api-provider-sidero/config/crd/bases/infrastructure.cluster.x-k8s.io_serverbindings.yaml b/app/cluster-api-provider-sidero/config/crd/bases/infrastructure.cluster.x-k8s.io_serverbindings.yaml index f4209b3..eb9fd10 100644 --- a/app/cluster-api-provider-sidero/config/crd/bases/infrastructure.cluster.x-k8s.io_serverbindings.yaml +++ b/app/cluster-api-provider-sidero/config/crd/bases/infrastructure.cluster.x-k8s.io_serverbindings.yaml @@ -21,26 +21,26 @@ spec: jsonPath: .status.ready name: Ready type: string - - description: Metal Machine - jsonPath: .spec.metalMachineRef.name - name: MetalMachine - priority: 1 - type: string - description: Server ID jsonPath: .metadata.name name: Server priority: 1 type: string - - description: Cluster to which this ServerBinding belongs - jsonPath: .metadata.labels.cluster\.x-k8s\.io/cluster-name - name: Cluster - priority: 1 - type: string - description: Server Class jsonPath: .spec.serverClassRef.name name: ServerClass priority: 1 type: string + - description: Metal Machine + jsonPath: .spec.metalMachineRef.name + name: MetalMachine + priority: 1 + type: string + - description: Cluster to which this ServerBinding belongs + jsonPath: .metadata.labels.cluster\.x-k8s\.io/cluster-name + name: Cluster + priority: 1 + type: string name: v1alpha3 schema: openAPIV3Schema: diff --git a/sfyra/pkg/capi/cluster.go b/sfyra/pkg/capi/cluster.go index 1f68054..4184292 100644 --- a/sfyra/pkg/capi/cluster.go +++ b/sfyra/pkg/capi/cluster.go @@ -14,6 +14,7 @@ import ( cabpt "github.com/talos-systems/cluster-api-bootstrap-provider-talos/api/v1alpha3" cacpt "github.com/talos-systems/cluster-api-control-plane-provider-talos/api/v1alpha3" + "github.com/talos-systems/go-retry/retry" taloscluster "github.com/talos-systems/talos/pkg/cluster" talosclusterapi "github.com/talos-systems/talos/pkg/machinery/api/cluster" talosclient "github.com/talos-systems/talos/pkg/machinery/client" @@ -104,6 +105,10 @@ func NewCluster(ctx context.Context, metalClient runtimeclient.Reader, clusterNa continue } + if !metalMachine.DeletionTimestamp.IsZero() { + continue + } + var server metal.Server if err := metalClient.Get(ctx, types.NamespacedName{Namespace: metalMachine.Spec.ServerRef.Namespace, Name: metalMachine.Spec.ServerRef.Name}, &server); err != nil { @@ -177,6 +182,13 @@ func NewCluster(ctx context.Context, metalClient runtimeclient.Reader, clusterNa // Health runs the healthcheck for the cluster. func (cluster *Cluster) Health(ctx context.Context) error { + return retry.Constant(5*time.Minute, retry.WithUnits(10*time.Second)).Retry(func() error { + // retry health checks as sometimes bootstrap bootkube issues break the check + return retry.ExpectedError(cluster.health(ctx)) + }) +} + +func (cluster *Cluster) health(ctx context.Context) error { resp, err := cluster.client.ClusterHealthCheck(talosclient.WithNodes(ctx, cluster.controlPlaneNodes[0]), 3*time.Minute, &talosclusterapi.ClusterInfo{ ControlPlaneNodes: cluster.controlPlaneNodes, WorkerNodes: cluster.workerNodes, diff --git a/sfyra/pkg/tests/cluster_utils.go b/sfyra/pkg/tests/cluster_utils.go new file mode 100644 index 0000000..44561df --- /dev/null +++ b/sfyra/pkg/tests/cluster_utils.go @@ -0,0 +1,157 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package tests + +import ( + "context" + "fmt" + "os" + "strconv" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/talos-systems/go-retry/retry" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/discovery" + "k8s.io/client-go/discovery/cached/memory" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/restmapper" + "sigs.k8s.io/cluster-api/api/v1alpha3" + capiclient "sigs.k8s.io/cluster-api/cmd/clusterctl/client" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/talos-systems/sidero/sfyra/pkg/capi" + "github.com/talos-systems/sidero/sfyra/pkg/loadbalancer" + "github.com/talos-systems/sidero/sfyra/pkg/talos" + "github.com/talos-systems/sidero/sfyra/pkg/vm" +) + +func deployCluster(ctx context.Context, t *testing.T, metalClient client.Client, capiCluster talos.Cluster, vmSet *vm.Set, + capiManager *capi.Manager, clusterName, serverClassName string, loadbalancerPort int, controlPlaneNodes, workerNodes int64) (*loadbalancer.ControlPlane, *capi.Cluster) { + t.Logf("deploying cluster %q from server class %q with loadbalancer port %d", clusterName, serverClassName, loadbalancerPort) + + kubeconfig, err := capiManager.GetKubeconfig(ctx) + require.NoError(t, err) + + config, err := capiCluster.KubernetesClient().K8sRestConfig(ctx) + require.NoError(t, err) + + capiClient := capiManager.GetManagerClient() + + loadbalancer, err := loadbalancer.NewControlPlane(metalClient, vmSet.BridgeIP(), loadbalancerPort, "default", clusterName, false) + require.NoError(t, err) + + os.Setenv("CONTROL_PLANE_ENDPOINT", vmSet.BridgeIP().String()) + os.Setenv("CONTROL_PLANE_PORT", strconv.Itoa(loadbalancerPort)) + os.Setenv("CONTROL_PLANE_SERVERCLASS", serverClassName) + os.Setenv("WORKER_SERVERCLASS", serverClassName) + // TODO: make it configurable + os.Setenv("KUBERNETES_VERSION", "v1.19.4") + + templateOptions := capiclient.GetClusterTemplateOptions{ + Kubeconfig: kubeconfig, + ClusterName: clusterName, + ControlPlaneMachineCount: &controlPlaneNodes, + WorkerMachineCount: &workerNodes, + } + + template, err := capiClient.GetClusterTemplate(templateOptions) + require.NoError(t, err) + + dc, err := discovery.NewDiscoveryClientForConfig(config) + require.NoError(t, err) + + mapper := restmapper.NewDeferredDiscoveryRESTMapper(memory.NewMemCacheClient(dc)) + + dyn, err := dynamic.NewForConfig(config) + require.NoError(t, err) + + for _, obj := range template.Objs() { + var mapping *meta.RESTMapping + + mapping, err = mapper.RESTMapping(obj.GroupVersionKind().GroupKind(), obj.GroupVersionKind().Version) + require.NoError(t, err) + + var dr dynamic.ResourceInterface + if mapping.Scope.Name() == meta.RESTScopeNameNamespace { + // namespaced resources should specify the namespace + dr = dyn.Resource(mapping.Resource).Namespace(obj.GetNamespace()) + } else { + // for cluster-wide resources + dr = dyn.Resource(mapping.Resource) + } + + var data []byte + + data, err = obj.MarshalJSON() + require.NoError(t, err) + + t.Logf("applying %s", string(data)) + + obj := obj + + _, err = dr.Create(ctx, &obj, metav1.CreateOptions{ + FieldManager: "sfyra", + }) + if err != nil { + if apierrors.IsAlreadyExists(err) { + _, err = dr.Patch(ctx, obj.GetName(), types.ApplyPatchType, data, metav1.PatchOptions{ + FieldManager: "sfyra", + }) + } + } + + require.NoError(t, err) + } + + t.Log("waiting for the cluster to be provisioned") + + require.NoError(t, retry.Constant(10*time.Minute, retry.WithUnits(10*time.Second), retry.WithErrorLogging(true)).Retry(func() error { + return capi.CheckClusterReady(ctx, metalClient, clusterName) + })) + + t.Log("verifying cluster health") + + deployedCluster, err := capi.NewCluster(ctx, metalClient, clusterName, vmSet.BridgeIP()) + require.NoError(t, err) + + require.NoError(t, deployedCluster.Health(ctx)) + + return loadbalancer, deployedCluster +} + +func deleteCluster(ctx context.Context, t *testing.T, metalClient client.Client, clusterName string) { + var cluster v1alpha3.Cluster + + err := metalClient.Get(ctx, types.NamespacedName{Namespace: "default", Name: clusterName}, &cluster) + require.NoError(t, err) + + t.Logf("deleting cluster %q", clusterName) + + err = metalClient.Delete(ctx, &cluster) + require.NoError(t, err) + + require.NoError(t, retry.Constant(3*time.Minute, retry.WithUnits(10*time.Second)).Retry(func() error { + err = metalClient.Get(ctx, types.NamespacedName{Namespace: "default", Name: clusterName}, &cluster) + if err == nil { + err = metalClient.Delete(ctx, &cluster) + if err != nil { + return retry.UnexpectedError(err) + } + + return retry.ExpectedError(fmt.Errorf("cluster is not deleted yet")) + } + + if apierrors.IsNotFound(err) { + return nil + } + + return retry.UnexpectedError(err) + })) +} diff --git a/sfyra/pkg/tests/management_cluster.go b/sfyra/pkg/tests/management_cluster.go index 7006e48..fd2b464 100644 --- a/sfyra/pkg/tests/management_cluster.go +++ b/sfyra/pkg/tests/management_cluster.go @@ -6,26 +6,11 @@ package tests import ( "context" - "os" - "strconv" "testing" - "time" - "github.com/stretchr/testify/require" - "github.com/talos-systems/go-retry/retry" - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/meta" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/discovery" - "k8s.io/client-go/discovery/cached/memory" - "k8s.io/client-go/dynamic" - "k8s.io/client-go/restmapper" - capiclient "sigs.k8s.io/cluster-api/cmd/clusterctl/client" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/talos-systems/sidero/sfyra/pkg/capi" - "github.com/talos-systems/sidero/sfyra/pkg/loadbalancer" "github.com/talos-systems/sidero/sfyra/pkg/talos" "github.com/talos-systems/sidero/sfyra/pkg/vm" ) @@ -36,97 +21,8 @@ const ( ) // TestManagementCluster deploys the management cluster via CAPI. -// -//nolint: gocognit func TestManagementCluster(ctx context.Context, metalClient client.Client, cluster talos.Cluster, vmSet *vm.Set, capiManager *capi.Manager) TestFunc { return func(t *testing.T) { - kubeconfig, err := capiManager.GetKubeconfig(ctx) - require.NoError(t, err) - - config, err := cluster.KubernetesClient().K8sRestConfig(ctx) - require.NoError(t, err) - - capiClient := capiManager.GetManagerClient() - - nodeCount := int64(1) - - _, err = loadbalancer.NewControlPlane(metalClient, vmSet.BridgeIP(), managementClusterLBPort, "default", managementClusterName, false) - require.NoError(t, err) - - os.Setenv("CONTROL_PLANE_ENDPOINT", vmSet.BridgeIP().String()) - os.Setenv("CONTROL_PLANE_PORT", strconv.Itoa(managementClusterLBPort)) - os.Setenv("CONTROL_PLANE_SERVERCLASS", serverClassName) - os.Setenv("WORKER_SERVERCLASS", serverClassName) - // TODO: make it configurable - os.Setenv("KUBERNETES_VERSION", "v1.19.0") - - templateOptions := capiclient.GetClusterTemplateOptions{ - Kubeconfig: kubeconfig, - ClusterName: managementClusterName, - ControlPlaneMachineCount: &nodeCount, - WorkerMachineCount: &nodeCount, - } - - template, err := capiClient.GetClusterTemplate(templateOptions) - require.NoError(t, err) - - dc, err := discovery.NewDiscoveryClientForConfig(config) - require.NoError(t, err) - - mapper := restmapper.NewDeferredDiscoveryRESTMapper(memory.NewMemCacheClient(dc)) - - dyn, err := dynamic.NewForConfig(config) - require.NoError(t, err) - - for _, obj := range template.Objs() { - var mapping *meta.RESTMapping - - mapping, err = mapper.RESTMapping(obj.GroupVersionKind().GroupKind(), obj.GroupVersionKind().Version) - require.NoError(t, err) - - var dr dynamic.ResourceInterface - if mapping.Scope.Name() == meta.RESTScopeNameNamespace { - // namespaced resources should specify the namespace - dr = dyn.Resource(mapping.Resource).Namespace(obj.GetNamespace()) - } else { - // for cluster-wide resources - dr = dyn.Resource(mapping.Resource) - } - - var data []byte - - data, err = obj.MarshalJSON() - require.NoError(t, err) - - t.Logf("applying %s", string(data)) - - obj := obj - - _, err = dr.Create(ctx, &obj, metav1.CreateOptions{ - FieldManager: "sfyra", - }) - if err != nil { - if apierrors.IsAlreadyExists(err) { - _, err = dr.Patch(ctx, obj.GetName(), types.ApplyPatchType, data, metav1.PatchOptions{ - FieldManager: "sfyra", - }) - } - } - - require.NoError(t, err) - } - - t.Log("waiting for the cluster to be provisioned") - - require.NoError(t, retry.Constant(10*time.Minute, retry.WithUnits(10*time.Second)).Retry(func() error { - return capi.CheckClusterReady(ctx, metalClient, managementClusterName) - })) - - t.Log("verifying cluster health") - - cluster, err := capi.NewCluster(ctx, metalClient, managementClusterName, vmSet.BridgeIP()) - require.NoError(t, err) - - require.NoError(t, cluster.Health(ctx)) + deployCluster(ctx, t, metalClient, cluster, vmSet, capiManager, managementClusterName, defaultServerClassName, managementClusterLBPort, 1, 1) } } diff --git a/sfyra/pkg/tests/server_class.go b/sfyra/pkg/tests/server_class.go index c1cbb0c..3714689 100644 --- a/sfyra/pkg/tests/server_class.go +++ b/sfyra/pkg/tests/server_class.go @@ -38,7 +38,10 @@ import ( "github.com/talos-systems/sidero/sfyra/pkg/vm" ) -const serverClassName = "default" +const ( + defaultServerClassName = "default" + workloadServerClassName = "workload" +) // TestServerClassDefault verifies server class creation. func TestServerClassDefault(ctx context.Context, metalClient client.Client, vmSet *vm.Set) TestFunc { @@ -53,14 +56,14 @@ func TestServerClassDefault(ctx context.Context, metalClient client.Client, vmSe }, } - serverClass, err := createServerClass(ctx, metalClient, serverClassName, classSpec) + serverClass, err := createServerClass(ctx, metalClient, defaultServerClassName, classSpec) require.NoError(t, err) numNodes := len(vmSet.Nodes()) // wait for the server class to gather all nodes (all nodes should match) require.NoError(t, retry.Constant(2*time.Minute, retry.WithUnits(10*time.Second)).Retry(func() error { - if err := metalClient.Get(ctx, types.NamespacedName{Name: serverClassName}, &serverClass); err != nil { + if err := metalClient.Get(ctx, types.NamespacedName{Name: defaultServerClassName}, &serverClass); err != nil { return retry.UnexpectedError(err) } @@ -86,6 +89,9 @@ func TestServerClassDefault(ctx context.Context, metalClient client.Client, vmSe sort.Strings(actualUUIDs) assert.Equal(t, expectedUUIDs, actualUUIDs) + + _, err = createServerClass(ctx, metalClient, workloadServerClassName, classSpec) + require.NoError(t, err) } } diff --git a/sfyra/pkg/tests/tests.go b/sfyra/pkg/tests/tests.go index 7ac5358..0456d66 100644 --- a/sfyra/pkg/tests/tests.go +++ b/sfyra/pkg/tests/tests.go @@ -119,6 +119,10 @@ func Run(ctx context.Context, cluster talos.Cluster, vmSet *vm.Set, capiManager "TestServerReset", TestServerReset(ctx, metalClient), }, + { + "TestWorkloadCluster", + TestWorkloadCluster(ctx, metalClient, cluster, vmSet, capiManager), + }, } testsToRun := []testing.InternalTest{} diff --git a/sfyra/pkg/tests/workload_cluster.go b/sfyra/pkg/tests/workload_cluster.go new file mode 100644 index 0000000..e75c292 --- /dev/null +++ b/sfyra/pkg/tests/workload_cluster.go @@ -0,0 +1,31 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package tests + +import ( + "context" + "testing" + + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/talos-systems/sidero/sfyra/pkg/capi" + "github.com/talos-systems/sidero/sfyra/pkg/talos" + "github.com/talos-systems/sidero/sfyra/pkg/vm" +) + +const ( + workloadClusterName = "workload-cluster" + workloadClusterLBPort = 20000 +) + +// TestWorkloadCluster deploys and destroys the workload cluster via CAPI. +func TestWorkloadCluster(ctx context.Context, metalClient client.Client, cluster talos.Cluster, vmSet *vm.Set, capiManager *capi.Manager) TestFunc { + return func(t *testing.T) { + loadbalancer, _ := deployCluster(ctx, t, metalClient, cluster, vmSet, capiManager, workloadClusterName, workloadServerClassName, workloadClusterLBPort, 1, 0) + defer loadbalancer.Close() + + deleteCluster(ctx, t, metalClient, workloadClusterName) + } +}