diff --git a/cmd/node/nodeCreate.go b/cmd/node/nodeCreate.go index 736c5a56..31cbec44 100644 --- a/cmd/node/nodeCreate.go +++ b/cmd/node/nodeCreate.go @@ -50,10 +50,17 @@ func NewCmdNodeCreate() *cobra.Command { Long: `Create a new containerized k3s node (k3s in docker).`, Args: cobra.ExactArgs(1), // exactly one name accepted // TODO: if not specified, inherit from cluster that the node shall belong to, if that is specified Run: func(cmd *cobra.Command, args []string) { - nodes, cluster := parseCreateNodeCmd(cmd, args) - if err := k3dc.NodeAddToClusterMulti(cmd.Context(), runtimes.SelectedRuntime, nodes, cluster, createNodeOpts); err != nil { - l.Log().Errorf("Failed to add nodes to cluster '%s'", cluster.Name) - l.Log().Fatalln(err) + nodes, clusterName := parseCreateNodeCmd(cmd, args) + if strings.HasPrefix(clusterName, "https://") { + l.Log().Infof("Adding %d node(s) to the remote cluster '%s'...", len(nodes), clusterName) + if err := k3dc.NodeAddToClusterMultiRemote(cmd.Context(), runtimes.SelectedRuntime, nodes, clusterName, createNodeOpts); err != nil { + l.Log().Fatalf("failed to add %d node(s) to the remote cluster '%s': %v", len(nodes), clusterName, err) + } + } else { + l.Log().Infof("Adding %d node(s) to the runtime local cluster '%s'...", len(nodes), clusterName) + if err := k3dc.NodeAddToClusterMulti(cmd.Context(), runtimes.SelectedRuntime, nodes, &k3d.Cluster{Name: clusterName}, createNodeOpts); err != nil { + l.Log().Fatalf("failed to add %d node(s) to the runtime local cluster '%s': %v", len(nodes), clusterName, err) + } } l.Log().Infof("Successfully created %d node(s)!", len(nodes)) }, @@ -79,12 +86,16 @@ func NewCmdNodeCreate() *cobra.Command { cmd.Flags().StringSliceP("runtime-label", "", []string{}, "Specify container runtime labels in format \"foo=bar\"") cmd.Flags().StringSliceP("k3s-node-label", "", []string{}, "Specify k3s node labels in format \"foo=bar\"") + cmd.Flags().StringSliceP("network", "n", []string{}, "Add node to (another) runtime network") + + cmd.Flags().StringVarP(&createNodeOpts.ClusterToken, "token", "t", "", "Override cluster token (required when connecting to an external cluster)") + // done return cmd } -// parseCreateNodeCmd parses the command input into variables required to create a cluster -func parseCreateNodeCmd(cmd *cobra.Command, args []string) ([]*k3d.Node, *k3d.Cluster) { +// parseCreateNodeCmd parses the command input into variables required to create a node +func parseCreateNodeCmd(cmd *cobra.Command, args []string) ([]*k3d.Node, string) { // --replicas replicas, err := cmd.Flags().GetInt("replicas") @@ -116,9 +127,6 @@ func parseCreateNodeCmd(cmd *cobra.Command, args []string) ([]*k3d.Node, *k3d.Cl if err != nil { l.Log().Fatalln(err) } - cluster := &k3d.Cluster{ - Name: clusterName, - } // --memory memory, err := cmd.Flags().GetString("memory") @@ -166,6 +174,12 @@ func parseCreateNodeCmd(cmd *cobra.Command, args []string) ([]*k3d.Node, *k3d.Cl k3sNodeLabels[labelSplitted[0]] = labelSplitted[1] } + // --network + networks, err := cmd.Flags().GetStringSlice("network") + if err != nil { + l.Log().Fatalf("failed to get --network string slice flag: %v", err) + } + // generate list of nodes nodes := []*k3d.Node{} for i := 0; i < replicas; i++ { @@ -177,9 +191,10 @@ func parseCreateNodeCmd(cmd *cobra.Command, args []string) ([]*k3d.Node, *k3d.Cl RuntimeLabels: runtimeLabels, Restart: true, Memory: memory, + Networks: networks, } nodes = append(nodes, node) } - return nodes, cluster + return nodes, clusterName } diff --git a/pkg/client/cluster.go b/pkg/client/cluster.go index 93d4d076..6b18c9cb 100644 --- a/pkg/client/cluster.go +++ b/pkg/client/cluster.go @@ -372,7 +372,7 @@ ClusterCreatOpts: // connection url is always the name of the first server node (index 0) // TODO: change this to the server loadbalancer connectionURL := fmt.Sprintf("https://%s:%s", GenerateNodeName(cluster.Name, k3d.ServerRole, 0), k3d.DefaultAPIPort) clusterCreateOpts.GlobalLabels[k3d.LabelClusterURL] = connectionURL - clusterCreateOpts.GlobalEnv = append(clusterCreateOpts.GlobalEnv, fmt.Sprintf("K3S_TOKEN=%s", cluster.Token)) + clusterCreateOpts.GlobalEnv = append(clusterCreateOpts.GlobalEnv, fmt.Sprintf("%s=%s", k3d.K3sEnvClusterToken, cluster.Token)) nodeSetup := func(node *k3d.Node) error { // cluster specific settings @@ -406,12 +406,12 @@ ClusterCreatOpts: // the cluster has an init server node, but its not this one, so connect it to the init node if cluster.InitNode != nil && !node.ServerOpts.IsInit { - node.Env = append(node.Env, fmt.Sprintf("K3S_URL=%s", connectionURL)) + node.Env = append(node.Env, fmt.Sprintf("%s=%s", k3d.K3sEnvClusterConnectURL, connectionURL)) node.RuntimeLabels[k3d.LabelServerIsInit] = "false" // set label, that this server node is not the init server } } else if node.Role == k3d.AgentRole { - node.Env = append(node.Env, fmt.Sprintf("K3S_URL=%s", connectionURL)) + node.Env = append(node.Env, fmt.Sprintf("%s=%s", k3d.K3sEnvClusterConnectURL, connectionURL)) } node.Networks = []string{cluster.Network.Name} diff --git a/pkg/client/node.go b/pkg/client/node.go index b6910ee5..2bc9d6cb 100644 --- a/pkg/client/node.go +++ b/pkg/client/node.go @@ -59,8 +59,12 @@ func NodeAddToCluster(ctx context.Context, runtime runtimes.Runtime, node *k3d.N return fmt.Errorf("Failed to find specified cluster '%s': %w", targetClusterName, err) } - // network - node.Networks = []string{cluster.Network.Name} + // networks: ensure that cluster network is on index 0 + networks := []string{cluster.Network.Name} + if node.Networks != nil { + networks = append(networks, node.Networks...) + } + node.Networks = networks // skeleton if node.RuntimeLabels == nil { @@ -163,22 +167,30 @@ func NodeAddToCluster(ctx context.Context, runtime runtimes.Runtime, node *k3d.N node = srcNode - l.Log().Debugf("Resulting node %+v", node) + l.Log().Tracef("Resulting node %+v", node) - k3sURLFound := false - for _, envVar := range node.Env { - if strings.HasPrefix(envVar, "K3S_URL") { - k3sURLFound = true - break + k3sURLEnvFound := false + k3sTokenEnvFoundIndex := -1 + for index, envVar := range node.Env { + if strings.HasPrefix(envVar, k3d.K3sEnvClusterConnectURL) { + k3sURLEnvFound = true + } + if strings.HasPrefix(envVar, k3d.K3sEnvClusterToken) { + k3sTokenEnvFoundIndex = index } } - if !k3sURLFound { + if !k3sURLEnvFound { if url, ok := node.RuntimeLabels[k3d.LabelClusterURL]; ok { - node.Env = append(node.Env, fmt.Sprintf("K3S_URL=%s", url)) + node.Env = append(node.Env, fmt.Sprintf("%s=%s", k3d.K3sEnvClusterConnectURL, url)) } else { l.Log().Warnln("Failed to find K3S_URL value!") } } + if k3sTokenEnvFoundIndex != -1 && createNodeOpts.ClusterToken != "" { + l.Log().Debugln("Overriding copied cluster token with value from nodeCreateOpts...") + node.Env[k3sTokenEnvFoundIndex] = fmt.Sprintf("%s=%s", k3d.K3sEnvClusterToken, createNodeOpts.ClusterToken) + node.RuntimeLabels[k3d.LabelClusterToken] = createNodeOpts.ClusterToken + } // add node actions if len(registryConfigBytes) != 0 { @@ -217,6 +229,33 @@ func NodeAddToCluster(ctx context.Context, runtime runtimes.Runtime, node *k3d.N return nil } +func NodeAddToClusterRemote(ctx context.Context, runtime runtimes.Runtime, node *k3d.Node, clusterRef string, createNodeOpts k3d.NodeCreateOpts) error { + // runtime labels + if node.RuntimeLabels == nil { + node.RuntimeLabels = map[string]string{} + } + + node.FillRuntimeLabels() + + node.RuntimeLabels[k3d.LabelClusterName] = clusterRef + node.RuntimeLabels[k3d.LabelClusterURL] = clusterRef + node.RuntimeLabels[k3d.LabelClusterExternal] = "true" + node.RuntimeLabels[k3d.LabelClusterToken] = createNodeOpts.ClusterToken + + if node.Env == nil { + node.Env = []string{} + } + + node.Env = append(node.Env, fmt.Sprintf("%s=%s", k3d.K3sEnvClusterConnectURL, clusterRef)) + node.Env = append(node.Env, fmt.Sprintf("%s=%s", k3d.K3sEnvClusterToken, createNodeOpts.ClusterToken)) + + if err := NodeRun(ctx, runtime, node, createNodeOpts); err != nil { + return fmt.Errorf("failed to run node '%s': %w", node.Name, err) + } + + return nil +} + // NodeAddToClusterMulti adds multiple nodes to a chosen cluster func NodeAddToClusterMulti(ctx context.Context, runtime runtimes.Runtime, nodes []*k3d.Node, cluster *k3d.Cluster, createNodeOpts k3d.NodeCreateOpts) error { if createNodeOpts.Timeout > 0*time.Second { @@ -233,7 +272,28 @@ func NodeAddToClusterMulti(ctx context.Context, runtime runtimes.Runtime, nodes }) } if err := nodeWaitGroup.Wait(); err != nil { - return fmt.Errorf("Failed to add one or more nodes: %w", err) + return fmt.Errorf("failed to add one or more nodes: %w", err) + } + + return nil +} + +func NodeAddToClusterMultiRemote(ctx context.Context, runtime runtimes.Runtime, nodes []*k3d.Node, clusterRef string, createNodeOpts k3d.NodeCreateOpts) error { + if createNodeOpts.Timeout > 0*time.Second { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, createNodeOpts.Timeout) + defer cancel() + } + + nodeWaitGroup, ctx := errgroup.WithContext(ctx) + for _, node := range nodes { + currentNode := node + nodeWaitGroup.Go(func() error { + return NodeAddToClusterRemote(ctx, runtime, currentNode, clusterRef, createNodeOpts) + }) + } + if err := nodeWaitGroup.Wait(); err != nil { + return fmt.Errorf("failed to add one or more nodes: %w", err) } return nil diff --git a/pkg/client/tools.go b/pkg/client/tools.go index e07b2faf..91f5bf3c 100644 --- a/pkg/client/tools.go +++ b/pkg/client/tools.go @@ -245,7 +245,7 @@ func runToolsNode(ctx context.Context, runtime runtimes.Runtime, cluster *k3d.Cl Networks: []string{network}, Cmd: []string{}, Args: []string{"noop"}, - RuntimeLabels: k3d.DefaultRuntimeLabels, + RuntimeLabels: labels, } node.RuntimeLabels[k3d.LabelClusterName] = cluster.Name if err := NodeRun(ctx, runtime, node, k3d.NodeCreateOpts{}); err != nil { diff --git a/pkg/runtimes/docker/network.go b/pkg/runtimes/docker/network.go index 0a912295..3149095e 100644 --- a/pkg/runtimes/docker/network.go +++ b/pkg/runtimes/docker/network.go @@ -157,6 +157,11 @@ func (d Docker) CreateNetworkIfNotPresent(ctx context.Context, inNet *k3d.Cluste return existingNet, true, nil } + labels := make(map[string]string, 0) + for k, v := range k3d.DefaultRuntimeLabels { + labels[k] = v + } + // (3) Create a new network netCreateOpts := types.NetworkCreate{ Driver: "bridge", @@ -164,7 +169,7 @@ func (d Docker) CreateNetworkIfNotPresent(ctx context.Context, inNet *k3d.Cluste "com.docker.network.bridge.enable_ip_masquerade": "true", }, CheckDuplicate: true, - Labels: k3d.DefaultRuntimeLabels, + Labels: labels, } // we want a managed (user-defined) network, but user didn't specify a subnet, so we try to auto-generate one diff --git a/pkg/runtimes/docker/node.go b/pkg/runtimes/docker/node.go index e839b24b..b2b8f7e4 100644 --- a/pkg/runtimes/docker/node.go +++ b/pkg/runtimes/docker/node.go @@ -186,7 +186,7 @@ func getContainersByLabel(ctx context.Context, labels map[string]string) ([]type All: true, }) if err != nil { - return nil, fmt.Errorf("Failed to list containers: %+v", err) + return nil, fmt.Errorf("failed to list containers: %w", err) } return containers, nil diff --git a/pkg/types/node.go b/pkg/types/node.go index d1a6c55b..a5fc2001 100644 --- a/pkg/types/node.go +++ b/pkg/types/node.go @@ -22,17 +22,18 @@ THE SOFTWARE. package types func (node *Node) FillRuntimeLabels() { - labels := make(map[string]string) + if node.RuntimeLabels == nil { + node.RuntimeLabels = make(map[string]string) + } for k, v := range DefaultRuntimeLabels { - labels[k] = v + node.RuntimeLabels[k] = v } for k, v := range DefaultRuntimeLabelsVar { - labels[k] = v + node.RuntimeLabels[k] = v } for k, v := range node.RuntimeLabels { - labels[k] = v + node.RuntimeLabels[k] = v } - node.RuntimeLabels = labels // second most important: the node role label node.RuntimeLabels[LabelRole] = string(node.Role) diff --git a/pkg/types/types.go b/pkg/types/types.go index 74784bf2..d98c9112 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -106,6 +106,7 @@ const ( LabelClusterName string = "k3d.cluster" LabelClusterURL string = "k3d.cluster.url" LabelClusterToken string = "k3d.cluster.token" + LabelClusterExternal string = "k3d.cluster.external" LabelImageVolume string = "k3d.cluster.imageVolume" LabelNetworkExternal string = "k3d.cluster.network.external" LabelNetwork string = "k3d.cluster.network" @@ -137,9 +138,16 @@ var DefaultTmpfsMounts = []string{ // DefaultNodeEnv defines some default environment variables that should be set on every node var DefaultNodeEnv = []string{ - "K3S_KUBECONFIG_OUTPUT=/output/kubeconfig.yaml", + fmt.Sprintf("%s=/output/kubeconfig.yaml", K3sEnvKubeconfigOutput), } +// k3s environment variables +const ( + K3sEnvClusterToken string = "K3S_TOKEN" + K3sEnvClusterConnectURL string = "K3S_URL" + K3sEnvKubeconfigOutput string = "K3S_KUBECONFIG_OUTPUT" +) + // DefaultK3dInternalHostRecord defines the default /etc/hosts entry for the k3d host const DefaultK3dInternalHostRecord = "host.k3d.internal" @@ -216,6 +224,7 @@ type NodeCreateOpts struct { Timeout time.Duration NodeHooks []NodeHook `yaml:"nodeHooks,omitempty" json:"nodeHooks,omitempty"` EnvironmentInfo *EnvironmentInfo + ClusterToken string } // NodeStartOpts describes a set of options one can set when (re-)starting a node