diff --git a/cmd/osctl/cmd/root.go b/cmd/osctl/cmd/root.go index 3c52e1f9e..b075a96af 100644 --- a/cmd/osctl/cmd/root.go +++ b/cmd/osctl/cmd/root.go @@ -17,7 +17,6 @@ import ( "syscall" "github.com/spf13/cobra" - "google.golang.org/grpc/metadata" "github.com/talos-systems/talos/cmd/osctl/pkg/client" "github.com/talos-systems/talos/cmd/osctl/pkg/helpers" @@ -113,9 +112,7 @@ func Execute() { // setupClient wraps common code to initialize osd client func setupClient(action func(*client.Client)) { // Update context with grpc metadata for proxy/relay requests - md := metadata.New(make(map[string]string)) - md.Set("targets", target...) - globalCtx = metadata.NewOutgoingContext(globalCtx, md) + globalCtx = client.WithTargets(globalCtx, target...) t, creds, err := client.NewClientTargetAndCredentialsFromConfig(talosconfig, cmdcontext) if err != nil { diff --git a/cmd/osctl/pkg/client/context.go b/cmd/osctl/pkg/client/context.go new file mode 100644 index 000000000..cccf46723 --- /dev/null +++ b/cmd/osctl/pkg/client/context.go @@ -0,0 +1,19 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package client + +import ( + "context" + + "google.golang.org/grpc/metadata" +) + +// WithTargets wraps the context with metadata to send request to set of nodes. +func WithTargets(ctx context.Context, target ...string) context.Context { + md := metadata.New(nil) + md.Set("targets", target...) + + return metadata.NewOutgoingContext(ctx, md) +} diff --git a/internal/integration/api/version.go b/internal/integration/api/version.go index 6cdc702c1..c07682f40 100644 --- a/internal/integration/api/version.go +++ b/internal/integration/api/version.go @@ -9,6 +9,7 @@ package api import ( "context" + "github.com/talos-systems/talos/cmd/osctl/pkg/client" "github.com/talos-systems/talos/internal/integration/base" ) @@ -30,6 +31,24 @@ func (suite *VersionSuite) TestExpectedVersionMaster() { suite.Assert().Equal(suite.Version, v.Response[0].Version.Tag) } +// TestSameVersionCluster verifies that all the nodes are on the same version +func (suite *VersionSuite) TestSameVersionCluster() { + nodes := suite.DiscoverNodes() + suite.Require().NotEmpty(nodes) + + ctx := client.WithTargets(context.Background(), nodes...) + + v, err := suite.Client.Version(ctx) + suite.Require().NoError(err) + + suite.Require().Len(v.Response, len(nodes)) + + expectedVersion := v.Response[0].Version.Tag + for _, version := range v.Response { + suite.Assert().Equal(expectedVersion, version.Version.Tag) + } +} + func init() { allSuites = append(allSuites, new(VersionSuite)) } diff --git a/internal/integration/base/api.go b/internal/integration/base/api.go index b6bcedad1..6a52d2861 100644 --- a/internal/integration/base/api.go +++ b/internal/integration/base/api.go @@ -34,6 +34,32 @@ func (apiSuite *APISuite) SetupSuite() { apiSuite.Require().NoError(err) } +// DiscoverNodes provides list of Talos nodes in the cluster. +// +// As there's no way to provide this functionality via Talos API, it works the following way: +// 1. If there's a provided list of nodes, it's used. +// 2. If integration test was compiled with k8s support, k8s is used. +func (apiSuite *APISuite) DiscoverNodes() []string { + discoveredNodes := apiSuite.TalosSuite.DiscoverNodes() + if discoveredNodes != nil { + return discoveredNodes + } + + var err error + + apiSuite.discoveredNodes, err = discoverNodesK8s(apiSuite.Client) + if err != nil { + apiSuite.Require().Error(err) + } + + if apiSuite.discoveredNodes == nil { + // still no nodes, skip the test + apiSuite.T().Skip("no nodes were discovered") + } + + return apiSuite.discoveredNodes +} + // TearDownSuite closes Talos API client func (apiSuite *APISuite) TearDownSuite() { if apiSuite.Client != nil { diff --git a/internal/integration/base/base.go b/internal/integration/base/base.go index 666d739cb..038e626cd 100644 --- a/internal/integration/base/base.go +++ b/internal/integration/base/base.go @@ -11,12 +11,29 @@ package base type TalosSuite struct { // Target is address of master node, if not set config is used Target string + // Nodes is a list of Talos cluster addresses (overrides discovery if set) + Nodes []string // TalosConfig is a path to talosconfig TalosConfig string // Version is the (expected) version of Talos tests are running against Version string // OsctlPath is path to osctl binary OsctlPath string + + discoveredNodes []string +} + +// DiscoverNodes provides basic functionality to discover cluster nodes via test settings. +// +// This method is overridden in specific suites to allow for specific discovery. +func (talosSuite *TalosSuite) DiscoverNodes() []string { + if talosSuite.discoveredNodes == nil { + if talosSuite.Nodes != nil { + talosSuite.discoveredNodes = talosSuite.Nodes + } + } + + return talosSuite.discoveredNodes } // ConfiguredSuite expects config to be set before running diff --git a/internal/integration/base/discovery_k8s.go b/internal/integration/base/discovery_k8s.go new file mode 100644 index 000000000..b0c4863a3 --- /dev/null +++ b/internal/integration/base/discovery_k8s.go @@ -0,0 +1,55 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// +build integration_k8s + +package base + +import ( + "context" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" + + "github.com/talos-systems/talos/cmd/osctl/pkg/client" +) + +func discoverNodesK8s(client *client.Client) ([]string, error) { + kubeconfig, err := client.Kubeconfig(context.Background()) + if err != nil { + return nil, err + } + + config, err := clientcmd.BuildConfigFromKubeconfigGetter("", func() (*clientcmdapi.Config, error) { + return clientcmd.Load(kubeconfig) + }) + if err != nil { + return nil, err + } + + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + return nil, err + } + + nodes, err := clientset.CoreV1().Nodes().List(metav1.ListOptions{}) + if err != nil { + return nil, err + } + + var result []string + + for _, node := range nodes.Items { + for _, nodeAddress := range node.Status.Addresses { + if nodeAddress.Type == v1.NodeInternalIP { + result = append(result, nodeAddress.Address) + } + } + } + + return result, nil +} diff --git a/internal/integration/base/discovery_nok8s.go b/internal/integration/base/discovery_nok8s.go new file mode 100644 index 000000000..7b1e0a002 --- /dev/null +++ b/internal/integration/base/discovery_nok8s.go @@ -0,0 +1,13 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// +build integration,!integration_k8s + +package base + +import "github.com/talos-systems/talos/cmd/osctl/pkg/client" + +func discoverNodesK8s(client *client.Client) ([]string, error) { + return nil, nil +} diff --git a/internal/integration/base/k8s.go b/internal/integration/base/k8s.go index 8d8bf1072..fec8be4f8 100644 --- a/internal/integration/base/k8s.go +++ b/internal/integration/base/k8s.go @@ -9,6 +9,7 @@ package base import ( "context" + "k8s.io/client-go/discovery" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/clientcmd" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" @@ -18,7 +19,8 @@ import ( type K8sSuite struct { APISuite - Clientset *kubernetes.Clientset + Clientset *kubernetes.Clientset + DiscoveryClient *discovery.DiscoveryClient } // SetupSuite initializes Kubernetes client @@ -35,4 +37,7 @@ func (k8sSuite *K8sSuite) SetupSuite() { k8sSuite.Clientset, err = kubernetes.NewForConfig(config) k8sSuite.Require().NoError(err) + + k8sSuite.DiscoveryClient, err = discovery.NewDiscoveryClientForConfig(config) + k8sSuite.Require().NoError(err) } diff --git a/internal/integration/flags_test.go b/internal/integration/flags_test.go new file mode 100644 index 000000000..ac5491a77 --- /dev/null +++ b/internal/integration/flags_test.go @@ -0,0 +1,21 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// +build integration + +package integration_test + +import "strings" + +type stringList []string + +func (l *stringList) String() string { + return strings.Join(*l, ",") +} + +func (l *stringList) Set(value string) error { + *l = append(*l, strings.Split(value, ",")...) + + return nil +} diff --git a/internal/integration/integration_test.go b/internal/integration/integration_test.go index af83258c5..049c1b223 100644 --- a/internal/integration/integration_test.go +++ b/internal/integration/integration_test.go @@ -30,6 +30,7 @@ var allSuites []suite.TestingSuite var ( talosConfig string target string + nodes stringList expectedVersion string osctlPath string ) @@ -43,6 +44,7 @@ func TestIntegration(t *testing.T) { if configuredSuite, ok := s.(base.ConfiguredSuite); ok { configuredSuite.SetConfig(base.TalosSuite{ Target: target, + Nodes: []string(nodes), TalosConfig: talosConfig, Version: expectedVersion, OsctlPath: osctlPath, @@ -75,6 +77,7 @@ func init() { flag.StringVar(&talosConfig, "talos.config", defaultTalosConfig, "The path to the Talos configuration file") flag.StringVar(&target, "talos.target", "", "target the specificed node") + flag.Var(&nodes, "talos.nodes", "list of node addresses (overrides discovery)") flag.StringVar(&expectedVersion, "talos.version", version.Tag, "expected Talos version") flag.StringVar(&osctlPath, "talos.osctlpath", "osctl", "The path to 'osctl' binary") diff --git a/internal/integration/k8s/version.go b/internal/integration/k8s/version.go index 6e383fd31..f32aaba06 100644 --- a/internal/integration/k8s/version.go +++ b/internal/integration/k8s/version.go @@ -29,11 +29,19 @@ func (suite *VersionSuite) SuiteName() string { // TestExpectedVersion verifies that node versions matches expected func (suite *VersionSuite) TestExpectedVersion() { + // verify k8s version (api server) + apiServerVersion, err := suite.DiscoveryClient.ServerVersion() + suite.Require().NoError(err) + + expectedApiServerVersion := fmt.Sprintf("v%s", constants.DefaultKubernetesVersion) + suite.Assert().Equal(expectedApiServerVersion, apiServerVersion.GitVersion) + v, err := suite.Client.Version(context.Background()) suite.Require().NoError(err) checkKernelVersion := v.Response[0].Platform != nil && v.Response[0].Platform.Mode != runtime.Container.String() + // verify each node (kubelet version, Talos version, etc.) nodes, err := suite.Clientset.CoreV1().Nodes().List(metav1.ListOptions{}) suite.Require().NoError(err)