[Feature] CreateNode: add token and network flags and allow remote cluster (#734)

- `--cluster` flag parsed for `https://` prefix and node creation treated differently accordingly
- new `--network` string array flag to add the node to multiple networks (primary network when adding to a remote cluster)
- new `--token` flag to provide the cluster token

cmd/node/nodeCreate.go

@@ -50,10 +50,17 @@ func NewCmdNodeCreate() *cobra.Command {
 		Long:  `Create a new containerized k3s node (k3s in docker).`,
 		Args:  cobra.ExactArgs(1), // exactly one name accepted // TODO: if not specified, inherit from cluster that the node shall belong to, if that is specified
 		Run: func(cmd *cobra.Command, args []string) {
-			nodes, cluster := parseCreateNodeCmd(cmd, args)
+			nodes, clusterName := parseCreateNodeCmd(cmd, args)
-			if err := k3dc.NodeAddToClusterMulti(cmd.Context(), runtimes.SelectedRuntime, nodes, cluster, createNodeOpts); err != nil {
+			if strings.HasPrefix(clusterName, "https://") {
-				l.Log().Errorf("Failed to add nodes to cluster '%s'", cluster.Name)
+				l.Log().Infof("Adding %d node(s) to the remote cluster '%s'...", len(nodes), clusterName)
-				l.Log().Fatalln(err)
+				if err := k3dc.NodeAddToClusterMultiRemote(cmd.Context(), runtimes.SelectedRuntime, nodes, clusterName, createNodeOpts); err != nil {
+					l.Log().Fatalf("failed to add %d node(s) to the remote cluster '%s': %v", len(nodes), clusterName, err)
+				}
+			} else {
+				l.Log().Infof("Adding %d node(s) to the runtime local cluster '%s'...", len(nodes), clusterName)
+				if err := k3dc.NodeAddToClusterMulti(cmd.Context(), runtimes.SelectedRuntime, nodes, &k3d.Cluster{Name: clusterName}, createNodeOpts); err != nil {
+					l.Log().Fatalf("failed to add %d node(s) to the runtime local cluster '%s': %v", len(nodes), clusterName, err)
+				}
 			}
 			l.Log().Infof("Successfully created %d node(s)!", len(nodes))
 		},
@@ -79,12 +86,16 @@ func NewCmdNodeCreate() *cobra.Command {
 	cmd.Flags().StringSliceP("runtime-label", "", []string{}, "Specify container runtime labels in format \"foo=bar\"")
 	cmd.Flags().StringSliceP("k3s-node-label", "", []string{}, "Specify k3s node labels in format \"foo=bar\"")
 
+	cmd.Flags().StringSliceP("network", "n", []string{}, "Add node to (another) runtime network")
+
+	cmd.Flags().StringVarP(&createNodeOpts.ClusterToken, "token", "t", "", "Override cluster token (required when connecting to an external cluster)")
+
 	// done
 	return cmd
 }
 
-// parseCreateNodeCmd parses the command input into variables required to create a cluster
+// parseCreateNodeCmd parses the command input into variables required to create a node
-func parseCreateNodeCmd(cmd *cobra.Command, args []string) ([]*k3d.Node, *k3d.Cluster) {
+func parseCreateNodeCmd(cmd *cobra.Command, args []string) ([]*k3d.Node, string) {
 
 	// --replicas
 	replicas, err := cmd.Flags().GetInt("replicas")
@@ -116,9 +127,6 @@ func parseCreateNodeCmd(cmd *cobra.Command, args []string) ([]*k3d.Node, *k3d.Cl
 	if err != nil {
 		l.Log().Fatalln(err)
 	}
-	cluster := &k3d.Cluster{
-		Name: clusterName,
-	}
 
 	// --memory
 	memory, err := cmd.Flags().GetString("memory")
@@ -166,6 +174,12 @@ func parseCreateNodeCmd(cmd *cobra.Command, args []string) ([]*k3d.Node, *k3d.Cl
 		k3sNodeLabels[labelSplitted[0]] = labelSplitted[1]
 	}
 
+	// --network
+	networks, err := cmd.Flags().GetStringSlice("network")
+	if err != nil {
+		l.Log().Fatalf("failed to get --network string slice flag: %v", err)
+	}
+
 	// generate list of nodes
 	nodes := []*k3d.Node{}
 	for i := 0; i < replicas; i++ {
@@ -177,9 +191,10 @@ func parseCreateNodeCmd(cmd *cobra.Command, args []string) ([]*k3d.Node, *k3d.Cl
 			RuntimeLabels: runtimeLabels,
 			Restart:       true,
 			Memory:        memory,
+			Networks:      networks,
 		}
 		nodes = append(nodes, node)
 	}
 
-	return nodes, cluster
+	return nodes, clusterName
 }

pkg/client/cluster.go

@@ -372,7 +372,7 @@ ClusterCreatOpts:
 	// connection url is always the name of the first server node (index 0) // TODO: change this to the server loadbalancer
 	connectionURL := fmt.Sprintf("https://%s:%s", GenerateNodeName(cluster.Name, k3d.ServerRole, 0), k3d.DefaultAPIPort)
 	clusterCreateOpts.GlobalLabels[k3d.LabelClusterURL] = connectionURL
-	clusterCreateOpts.GlobalEnv = append(clusterCreateOpts.GlobalEnv, fmt.Sprintf("K3S_TOKEN=%s", cluster.Token))
+	clusterCreateOpts.GlobalEnv = append(clusterCreateOpts.GlobalEnv, fmt.Sprintf("%s=%s", k3d.K3sEnvClusterToken, cluster.Token))
 
 	nodeSetup := func(node *k3d.Node) error {
 		// cluster specific settings
@@ -406,12 +406,12 @@ ClusterCreatOpts:
 
 			// the cluster has an init server node, but its not this one, so connect it to the init node
 			if cluster.InitNode != nil && !node.ServerOpts.IsInit {
-				node.Env = append(node.Env, fmt.Sprintf("K3S_URL=%s", connectionURL))
+				node.Env = append(node.Env, fmt.Sprintf("%s=%s", k3d.K3sEnvClusterConnectURL, connectionURL))
 				node.RuntimeLabels[k3d.LabelServerIsInit] = "false" // set label, that this server node is not the init server
 			}
 
 		} else if node.Role == k3d.AgentRole {
-			node.Env = append(node.Env, fmt.Sprintf("K3S_URL=%s", connectionURL))
+			node.Env = append(node.Env, fmt.Sprintf("%s=%s", k3d.K3sEnvClusterConnectURL, connectionURL))
 		}
 
 		node.Networks = []string{cluster.Network.Name}

pkg/client/node.go

@@ -59,8 +59,12 @@ func NodeAddToCluster(ctx context.Context, runtime runtimes.Runtime, node *k3d.N
 		return fmt.Errorf("Failed to find specified cluster '%s': %w", targetClusterName, err)
 	}
 
-	// network
+	// networks: ensure that cluster network is on index 0
-	node.Networks = []string{cluster.Network.Name}
+	networks := []string{cluster.Network.Name}
+	if node.Networks != nil {
+		networks = append(networks, node.Networks...)
+	}
+	node.Networks = networks
 
 	// skeleton
 	if node.RuntimeLabels == nil {
@@ -163,22 +167,30 @@ func NodeAddToCluster(ctx context.Context, runtime runtimes.Runtime, node *k3d.N
 
 	node = srcNode
 
-	l.Log().Debugf("Resulting node %+v", node)
+	l.Log().Tracef("Resulting node %+v", node)
 
-	k3sURLFound := false
+	k3sURLEnvFound := false
-	for _, envVar := range node.Env {
+	k3sTokenEnvFoundIndex := -1
-		if strings.HasPrefix(envVar, "K3S_URL") {
+	for index, envVar := range node.Env {
-			k3sURLFound = true
+		if strings.HasPrefix(envVar, k3d.K3sEnvClusterConnectURL) {
-			break
+			k3sURLEnvFound = true
+		}
+		if strings.HasPrefix(envVar, k3d.K3sEnvClusterToken) {
+			k3sTokenEnvFoundIndex = index
 		}
 	}
-	if !k3sURLFound {
+	if !k3sURLEnvFound {
 		if url, ok := node.RuntimeLabels[k3d.LabelClusterURL]; ok {
-			node.Env = append(node.Env, fmt.Sprintf("K3S_URL=%s", url))
+			node.Env = append(node.Env, fmt.Sprintf("%s=%s", k3d.K3sEnvClusterConnectURL, url))
 		} else {
 			l.Log().Warnln("Failed to find K3S_URL value!")
 		}
 	}
+	if k3sTokenEnvFoundIndex != -1 && createNodeOpts.ClusterToken != "" {
+		l.Log().Debugln("Overriding copied cluster token with value from nodeCreateOpts...")
+		node.Env[k3sTokenEnvFoundIndex] = fmt.Sprintf("%s=%s", k3d.K3sEnvClusterToken, createNodeOpts.ClusterToken)
+		node.RuntimeLabels[k3d.LabelClusterToken] = createNodeOpts.ClusterToken
+	}
 
 	// add node actions
 	if len(registryConfigBytes) != 0 {
@@ -217,6 +229,33 @@ func NodeAddToCluster(ctx context.Context, runtime runtimes.Runtime, node *k3d.N
 	return nil
 }
 
+func NodeAddToClusterRemote(ctx context.Context, runtime runtimes.Runtime, node *k3d.Node, clusterRef string, createNodeOpts k3d.NodeCreateOpts) error {
+	// runtime labels
+	if node.RuntimeLabels == nil {
+		node.RuntimeLabels = map[string]string{}
+	}
+
+	node.FillRuntimeLabels()
+
+	node.RuntimeLabels[k3d.LabelClusterName] = clusterRef
+	node.RuntimeLabels[k3d.LabelClusterURL] = clusterRef
+	node.RuntimeLabels[k3d.LabelClusterExternal] = "true"
+	node.RuntimeLabels[k3d.LabelClusterToken] = createNodeOpts.ClusterToken
+
+	if node.Env == nil {
+		node.Env = []string{}
+	}
+
+	node.Env = append(node.Env, fmt.Sprintf("%s=%s", k3d.K3sEnvClusterConnectURL, clusterRef))
+	node.Env = append(node.Env, fmt.Sprintf("%s=%s", k3d.K3sEnvClusterToken, createNodeOpts.ClusterToken))
+
+	if err := NodeRun(ctx, runtime, node, createNodeOpts); err != nil {
+		return fmt.Errorf("failed to run node '%s': %w", node.Name, err)
+	}
+
+	return nil
+}
+
 // NodeAddToClusterMulti adds multiple nodes to a chosen cluster
 func NodeAddToClusterMulti(ctx context.Context, runtime runtimes.Runtime, nodes []*k3d.Node, cluster *k3d.Cluster, createNodeOpts k3d.NodeCreateOpts) error {
 	if createNodeOpts.Timeout > 0*time.Second {
@@ -233,7 +272,28 @@ func NodeAddToClusterMulti(ctx context.Context, runtime runtimes.Runtime, nodes
 		})
 	}
 	if err := nodeWaitGroup.Wait(); err != nil {
-		return fmt.Errorf("Failed to add one or more nodes: %w", err)
+		return fmt.Errorf("failed to add one or more nodes: %w", err)
+	}
+
+	return nil
+}
+
+func NodeAddToClusterMultiRemote(ctx context.Context, runtime runtimes.Runtime, nodes []*k3d.Node, clusterRef string, createNodeOpts k3d.NodeCreateOpts) error {
+	if createNodeOpts.Timeout > 0*time.Second {
+		var cancel context.CancelFunc
+		ctx, cancel = context.WithTimeout(ctx, createNodeOpts.Timeout)
+		defer cancel()
+	}
+
+	nodeWaitGroup, ctx := errgroup.WithContext(ctx)
+	for _, node := range nodes {
+		currentNode := node
+		nodeWaitGroup.Go(func() error {
+			return NodeAddToClusterRemote(ctx, runtime, currentNode, clusterRef, createNodeOpts)
+		})
+	}
+	if err := nodeWaitGroup.Wait(); err != nil {
+		return fmt.Errorf("failed to add one or more nodes: %w", err)
 	}
 
 	return nil

pkg/client/tools.go

@@ -245,7 +245,7 @@ func runToolsNode(ctx context.Context, runtime runtimes.Runtime, cluster *k3d.Cl
 		Networks:      []string{network},
 		Cmd:           []string{},
 		Args:          []string{"noop"},
-		RuntimeLabels: k3d.DefaultRuntimeLabels,
+		RuntimeLabels: labels,
 	}
 	node.RuntimeLabels[k3d.LabelClusterName] = cluster.Name
 	if err := NodeRun(ctx, runtime, node, k3d.NodeCreateOpts{}); err != nil {

pkg/runtimes/docker/network.go

@@ -157,6 +157,11 @@ func (d Docker) CreateNetworkIfNotPresent(ctx context.Context, inNet *k3d.Cluste
 		return existingNet, true, nil
 	}
 
+	labels := make(map[string]string, 0)
+	for k, v := range k3d.DefaultRuntimeLabels {
+		labels[k] = v
+	}
+
 	// (3) Create a new network
 	netCreateOpts := types.NetworkCreate{
 		Driver: "bridge",
@@ -164,7 +169,7 @@ func (d Docker) CreateNetworkIfNotPresent(ctx context.Context, inNet *k3d.Cluste
 			"com.docker.network.bridge.enable_ip_masquerade": "true",
 		},
 		CheckDuplicate: true,
-		Labels:         k3d.DefaultRuntimeLabels,
+		Labels:         labels,
 	}
 
 	// we want a managed (user-defined) network, but user didn't specify a subnet, so we try to auto-generate one

pkg/runtimes/docker/node.go

@@ -186,7 +186,7 @@ func getContainersByLabel(ctx context.Context, labels map[string]string) ([]type
 		All:     true,
 	})
 	if err != nil {
-		return nil, fmt.Errorf("Failed to list containers: %+v", err)
+		return nil, fmt.Errorf("failed to list containers: %w", err)
 	}
 
 	return containers, nil

pkg/types/node.go

@@ -22,17 +22,18 @@ THE SOFTWARE.
 package types
 
 func (node *Node) FillRuntimeLabels() {
-	labels := make(map[string]string)
+	if node.RuntimeLabels == nil {
+		node.RuntimeLabels = make(map[string]string)
+	}
 	for k, v := range DefaultRuntimeLabels {
-		labels[k] = v
+		node.RuntimeLabels[k] = v
 	}
 	for k, v := range DefaultRuntimeLabelsVar {
-		labels[k] = v
+		node.RuntimeLabels[k] = v
 	}
 	for k, v := range node.RuntimeLabels {
-		labels[k] = v
+		node.RuntimeLabels[k] = v
 	}
-	node.RuntimeLabels = labels
 	// second most important: the node role label
 	node.RuntimeLabels[LabelRole] = string(node.Role)
 

pkg/types/types.go

@@ -106,6 +106,7 @@ const (
 	LabelClusterName          string = "k3d.cluster"
 	LabelClusterURL           string = "k3d.cluster.url"
 	LabelClusterToken         string = "k3d.cluster.token"
+	LabelClusterExternal      string = "k3d.cluster.external"
 	LabelImageVolume          string = "k3d.cluster.imageVolume"
 	LabelNetworkExternal      string = "k3d.cluster.network.external"
 	LabelNetwork              string = "k3d.cluster.network"
@@ -137,9 +138,16 @@ var DefaultTmpfsMounts = []string{
 
 // DefaultNodeEnv defines some default environment variables that should be set on every node
 var DefaultNodeEnv = []string{
-	"K3S_KUBECONFIG_OUTPUT=/output/kubeconfig.yaml",
+	fmt.Sprintf("%s=/output/kubeconfig.yaml", K3sEnvKubeconfigOutput),
 }
 
+// k3s environment variables
+const (
+	K3sEnvClusterToken      string = "K3S_TOKEN"
+	K3sEnvClusterConnectURL string = "K3S_URL"
+	K3sEnvKubeconfigOutput  string = "K3S_KUBECONFIG_OUTPUT"
+)
+
 // DefaultK3dInternalHostRecord defines the default /etc/hosts entry for the k3d host
 const DefaultK3dInternalHostRecord = "host.k3d.internal"
 
@@ -216,6 +224,7 @@ type NodeCreateOpts struct {
 	Timeout         time.Duration
 	NodeHooks       []NodeHook `yaml:"nodeHooks,omitempty" json:"nodeHooks,omitempty"`
 	EnvironmentInfo *EnvironmentInfo
+	ClusterToken    string
 }
 
 // NodeStartOpts describes a set of options one can set when (re-)starting a node