test: implement node discovery for integration tests

This adds support for node discovery for API-based tests, but discovery
is based on k8s state. Discovery can be overridden if we provide a list
of node IPs as a flag.

Also adds a test for K8s API server version.

Signed-off-by: Andrey Smirnov <smirnov.andrey@gmail.com>

cmd/osctl/cmd/root.go

@@ -17,7 +17,6 @@ import (
 	"syscall"
 
 	"github.com/spf13/cobra"
-	"google.golang.org/grpc/metadata"
 
 	"github.com/talos-systems/talos/cmd/osctl/pkg/client"
 	"github.com/talos-systems/talos/cmd/osctl/pkg/helpers"
@@ -113,9 +112,7 @@ func Execute() {
 // setupClient wraps common code to initialize osd client
 func setupClient(action func(*client.Client)) {
 	// Update context with grpc metadata for proxy/relay requests
-	md := metadata.New(make(map[string]string))
+	globalCtx = client.WithTargets(globalCtx, target...)
-	md.Set("targets", target...)
-	globalCtx = metadata.NewOutgoingContext(globalCtx, md)
 
 	t, creds, err := client.NewClientTargetAndCredentialsFromConfig(talosconfig, cmdcontext)
 	if err != nil {

cmd/osctl/pkg/client/context.go

@@ -0,0 +1,19 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package client
+
+import (
+	"context"
+
+	"google.golang.org/grpc/metadata"
+)
+
+// WithTargets wraps the context with metadata to send request to set of nodes.
+func WithTargets(ctx context.Context, target ...string) context.Context {
+	md := metadata.New(nil)
+	md.Set("targets", target...)
+
+	return metadata.NewOutgoingContext(ctx, md)
+}

internal/integration/api/version.go

@@ -9,6 +9,7 @@ package api
 import (
 	"context"
 
+	"github.com/talos-systems/talos/cmd/osctl/pkg/client"
 	"github.com/talos-systems/talos/internal/integration/base"
 )
 
@@ -30,6 +31,24 @@ func (suite *VersionSuite) TestExpectedVersionMaster() {
 	suite.Assert().Equal(suite.Version, v.Response[0].Version.Tag)
 }
 
+// TestSameVersionCluster verifies that all the nodes are on the same version
+func (suite *VersionSuite) TestSameVersionCluster() {
+	nodes := suite.DiscoverNodes()
+	suite.Require().NotEmpty(nodes)
+
+	ctx := client.WithTargets(context.Background(), nodes...)
+
+	v, err := suite.Client.Version(ctx)
+	suite.Require().NoError(err)
+
+	suite.Require().Len(v.Response, len(nodes))
+
+	expectedVersion := v.Response[0].Version.Tag
+	for _, version := range v.Response {
+		suite.Assert().Equal(expectedVersion, version.Version.Tag)
+	}
+}
+
 func init() {
 	allSuites = append(allSuites, new(VersionSuite))
 }

internal/integration/base/api.go

@@ -34,6 +34,32 @@ func (apiSuite *APISuite) SetupSuite() {
 	apiSuite.Require().NoError(err)
 }
 
+// DiscoverNodes provides list of Talos nodes in the cluster.
+//
+// As there's no way to provide this functionality via Talos API, it works the following way:
+// 1. If there's a provided list of nodes, it's used.
+// 2. If integration test was compiled with k8s support, k8s is used.
+func (apiSuite *APISuite) DiscoverNodes() []string {
+	discoveredNodes := apiSuite.TalosSuite.DiscoverNodes()
+	if discoveredNodes != nil {
+		return discoveredNodes
+	}
+
+	var err error
+
+	apiSuite.discoveredNodes, err = discoverNodesK8s(apiSuite.Client)
+	if err != nil {
+		apiSuite.Require().Error(err)
+	}
+
+	if apiSuite.discoveredNodes == nil {
+		// still no nodes, skip the test
+		apiSuite.T().Skip("no nodes were discovered")
+	}
+
+	return apiSuite.discoveredNodes
+}
+
 // TearDownSuite closes Talos API client
 func (apiSuite *APISuite) TearDownSuite() {
 	if apiSuite.Client != nil {

internal/integration/base/base.go

@@ -11,12 +11,29 @@ package base
 type TalosSuite struct {
 	// Target is address of master node, if not set config is used
 	Target string
+	// Nodes is a list of Talos cluster addresses (overrides discovery if set)
+	Nodes []string
 	// TalosConfig is a path to talosconfig
 	TalosConfig string
 	// Version is the (expected) version of Talos tests are running against
 	Version string
 	// OsctlPath is path to osctl binary
 	OsctlPath string
+
+	discoveredNodes []string
+}
+
+// DiscoverNodes provides basic functionality to discover cluster nodes via test settings.
+//
+// This method is overridden in specific suites to allow for specific discovery.
+func (talosSuite *TalosSuite) DiscoverNodes() []string {
+	if talosSuite.discoveredNodes == nil {
+		if talosSuite.Nodes != nil {
+			talosSuite.discoveredNodes = talosSuite.Nodes
+		}
+	}
+
+	return talosSuite.discoveredNodes
 }
 
 // ConfiguredSuite expects config to be set before running

internal/integration/base/discovery_k8s.go

@@ -0,0 +1,55 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+// +build integration_k8s
+
+package base
+
+import (
+	"context"
+
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
+	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
+
+	"github.com/talos-systems/talos/cmd/osctl/pkg/client"
+)
+
+func discoverNodesK8s(client *client.Client) ([]string, error) {
+	kubeconfig, err := client.Kubeconfig(context.Background())
+	if err != nil {
+		return nil, err
+	}
+
+	config, err := clientcmd.BuildConfigFromKubeconfigGetter("", func() (*clientcmdapi.Config, error) {
+		return clientcmd.Load(kubeconfig)
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	clientset, err := kubernetes.NewForConfig(config)
+	if err != nil {
+		return nil, err
+	}
+
+	nodes, err := clientset.CoreV1().Nodes().List(metav1.ListOptions{})
+	if err != nil {
+		return nil, err
+	}
+
+	var result []string
+
+	for _, node := range nodes.Items {
+		for _, nodeAddress := range node.Status.Addresses {
+			if nodeAddress.Type == v1.NodeInternalIP {
+				result = append(result, nodeAddress.Address)
+			}
+		}
+	}
+
+	return result, nil
+}

internal/integration/base/discovery_nok8s.go

@@ -0,0 +1,13 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+// +build integration,!integration_k8s
+
+package base
+
+import "github.com/talos-systems/talos/cmd/osctl/pkg/client"
+
+func discoverNodesK8s(client *client.Client) ([]string, error) {
+	return nil, nil
+}

internal/integration/base/k8s.go

@@ -9,6 +9,7 @@ package base
 import (
 	"context"
 
+	"k8s.io/client-go/discovery"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/clientcmd"
 	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
@@ -18,7 +19,8 @@ import (
 type K8sSuite struct {
 	APISuite
 
-	Clientset *kubernetes.Clientset
+	Clientset       *kubernetes.Clientset
+	DiscoveryClient *discovery.DiscoveryClient
 }
 
 // SetupSuite initializes Kubernetes client
@@ -35,4 +37,7 @@ func (k8sSuite *K8sSuite) SetupSuite() {
 
 	k8sSuite.Clientset, err = kubernetes.NewForConfig(config)
 	k8sSuite.Require().NoError(err)
+
+	k8sSuite.DiscoveryClient, err = discovery.NewDiscoveryClientForConfig(config)
+	k8sSuite.Require().NoError(err)
 }

internal/integration/flags_test.go

@@ -0,0 +1,21 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+// +build integration
+
+package integration_test
+
+import "strings"
+
+type stringList []string
+
+func (l *stringList) String() string {
+	return strings.Join(*l, ",")
+}
+
+func (l *stringList) Set(value string) error {
+	*l = append(*l, strings.Split(value, ",")...)
+
+	return nil
+}

internal/integration/integration_test.go

@@ -30,6 +30,7 @@ var allSuites []suite.TestingSuite
 var (
 	talosConfig     string
 	target          string
+	nodes           stringList
 	expectedVersion string
 	osctlPath       string
 )
@@ -43,6 +44,7 @@ func TestIntegration(t *testing.T) {
 		if configuredSuite, ok := s.(base.ConfiguredSuite); ok {
 			configuredSuite.SetConfig(base.TalosSuite{
 				Target:      target,
+				Nodes:       []string(nodes),
 				TalosConfig: talosConfig,
 				Version:     expectedVersion,
 				OsctlPath:   osctlPath,
@@ -75,6 +77,7 @@ func init() {
 
 	flag.StringVar(&talosConfig, "talos.config", defaultTalosConfig, "The path to the Talos configuration file")
 	flag.StringVar(&target, "talos.target", "", "target the specificed node")
+	flag.Var(&nodes, "talos.nodes", "list of node addresses (overrides discovery)")
 	flag.StringVar(&expectedVersion, "talos.version", version.Tag, "expected Talos version")
 	flag.StringVar(&osctlPath, "talos.osctlpath", "osctl", "The path to 'osctl' binary")
 

internal/integration/k8s/version.go

@@ -29,11 +29,19 @@ func (suite *VersionSuite) SuiteName() string {
 
 // TestExpectedVersion verifies that node versions matches expected
 func (suite *VersionSuite) TestExpectedVersion() {
+	// verify k8s version (api server)
+	apiServerVersion, err := suite.DiscoveryClient.ServerVersion()
+	suite.Require().NoError(err)
+
+	expectedApiServerVersion := fmt.Sprintf("v%s", constants.DefaultKubernetesVersion)
+	suite.Assert().Equal(expectedApiServerVersion, apiServerVersion.GitVersion)
+
 	v, err := suite.Client.Version(context.Background())
 	suite.Require().NoError(err)
 
 	checkKernelVersion := v.Response[0].Platform != nil && v.Response[0].Platform.Mode != runtime.Container.String()
 
+	// verify each node (kubelet version, Talos version, etc.)
 	nodes, err := suite.Clientset.CoreV1().Nodes().List(metav1.ListOptions{})
 	suite.Require().NoError(err)