mirrors/k3d
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

[Enhancement] Refactoring: normalize label flags (k3s node & runtime) (#598, @ejose19)

Browse Source
This commit is contained in:
ejose19 2021-05-19 09:20:47 -03:00 committed by iwilltry42
parent 261ac0faf4
commit bfead49c46
No known key found for this signature in database
GPG Key ID: 7BA57AD1CFF16110
30 changed files with 361 additions and 185 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
cmd/cluster/clusterCreate.go
View File

@ -283,8 +283,11 @@ func NewCmdClusterCreate() *cobra.Command {
cmd.Flags().StringArrayP("port", "p", nil, "Map ports from the node containers to the host (Format: `[HOST:][HOSTPORT:]CONTAINERPORT[/PROTOCOL][@NODEFILTER]`)\n - Example: `k3d cluster create --agents 2 -p 8080:80@agent[0] -p 8081@agent[1]`")
_ = ppViper.BindPFlag("cli.ports", cmd.Flags().Lookup("port"))
cmd.Flags().StringArrayP("label", "l", nil, "Add label to node container (Format: `KEY[=VALUE][@NODEFILTER[;NODEFILTER...]]`\n - Example: `k3d cluster create --agents 2 -l \"my.label@agent[0,1]\" -l \"other.label=somevalue@server[0]\"`")
_ = ppViper.BindPFlag("cli.labels", cmd.Flags().Lookup("label"))
cmd.Flags().StringArrayP("k3s-node-label", "", nil, "Add label to k3s node (Format: `KEY[=VALUE][@NODEFILTER[;NODEFILTER...]]`\n - Example: `k3d cluster create --agents 2 --k3s-node-label \"my.label@agent[0,1]\" --k3s-node-label \"other.label=somevalue@server[0]\"`")
_ = ppViper.BindPFlag("cli.k3s-node-labels", cmd.Flags().Lookup("k3s-node-label"))
cmd.Flags().StringArrayP("runtime-label", "", nil, "Add label to container runtime (Format: `KEY[=VALUE][@NODEFILTER[;NODEFILTER...]]`\n - Example: `k3d cluster create --agents 2 --runtime-label \"my.label@agent[0,1]\" --runtime-label \"other.label=somevalue@server[0]\"`")
_ = ppViper.BindPFlag("cli.runtime-labels", cmd.Flags().Lookup("runtime-label"))
/* k3s */
cmd.Flags().StringArray("k3s-arg", nil, "Additional args passed to k3s command (Format: `ARG@NODEFILTER[;@NODEFILTER]`)\n - Example: `k3d cluster create --k3s-arg \"--disable=traefik@server[0]\"")
@ -481,10 +484,10 @@ func applyCLIOverrides(cfg conf.SimpleConfig) (conf.SimpleConfig, error) {
log.Tracef("PortFilterMap: %+v", portFilterMap)
// --label
// labelFilterMap will add container label to applied node filters
labelFilterMap := make(map[string][]string, 1)
for _, labelFlag := range ppViper.GetStringSlice("cli.labels") {
// --k3s-node-label
// k3sNodeLabelFilterMap will add k3s node label to applied node filters
k3sNodeLabelFilterMap := make(map[string][]string, 1)
for _, labelFlag := range ppViper.GetStringSlice("cli.k3s-node-labels") {
// split node filter from the specified label
label, nodeFilters, err := cliutil.SplitFiltersFromFlag(labelFlag)
@ -493,21 +496,51 @@ func applyCLIOverrides(cfg conf.SimpleConfig) (conf.SimpleConfig, error) {
}
// create new entry or append filter to existing entry
if _, exists := labelFilterMap[label]; exists {
labelFilterMap[label] = append(labelFilterMap[label], nodeFilters...)
if _, exists := k3sNodeLabelFilterMap[label]; exists {
k3sNodeLabelFilterMap[label] = append(k3sNodeLabelFilterMap[label], nodeFilters...)
} else {
labelFilterMap[label] = nodeFilters
k3sNodeLabelFilterMap[label] = nodeFilters
}
}
for label, nodeFilters := range labelFilterMap {
cfg.Labels = append(cfg.Labels, conf.LabelWithNodeFilters{
for label, nodeFilters := range k3sNodeLabelFilterMap {
cfg.Options.K3sOptions.NodeLabels = append(cfg.Options.K3sOptions.NodeLabels, conf.LabelWithNodeFilters{
Label: label,
NodeFilters: nodeFilters,
})
}
log.Tracef("LabelFilterMap: %+v", labelFilterMap)
log.Tracef("K3sNodeLabelFilterMap: %+v", k3sNodeLabelFilterMap)
// --runtime-label
// runtimeLabelFilterMap will add container runtime label to applied node filters
runtimeLabelFilterMap := make(map[string][]string, 1)
for _, labelFlag := range ppViper.GetStringSlice("cli.runtime-labels") {
// split node filter from the specified label
label, nodeFilters, err := cliutil.SplitFiltersFromFlag(labelFlag)
if err != nil {
log.Fatalln(err)
}
cliutil.ValidateRuntimeLabelKey(strings.Split(label, "=")[0])
// create new entry or append filter to existing entry
if _, exists := runtimeLabelFilterMap[label]; exists {
runtimeLabelFilterMap[label] = append(runtimeLabelFilterMap[label], nodeFilters...)
} else {
runtimeLabelFilterMap[label] = nodeFilters
}
}
for label, nodeFilters := range runtimeLabelFilterMap {
cfg.Options.Runtime.Labels = append(cfg.Options.Runtime.Labels, conf.LabelWithNodeFilters{
Label: label,
NodeFilters: nodeFilters,
})
}
log.Tracef("RuntimeLabelFilterMap: %+v", runtimeLabelFilterMap)
// --env
// envFilterMap will add container env vars to applied node filters

37
cmd/node/nodeCreate.go
View File

@ -30,6 +30,7 @@ import (
dockerunits "github.com/docker/go-units"
"github.com/rancher/k3d/v4/cmd/util"
cliutil "github.com/rancher/k3d/v4/cmd/util"
k3dc "github.com/rancher/k3d/v4/pkg/client"
"github.com/rancher/k3d/v4/pkg/runtimes"
k3d "github.com/rancher/k3d/v4/pkg/types"
@ -74,6 +75,7 @@ func NewCmdNodeCreate() *cobra.Command {
cmd.Flags().BoolVar(&createNodeOpts.Wait, "wait", false, "Wait for the node(s) to be ready before returning.")
cmd.Flags().DurationVar(&createNodeOpts.Timeout, "timeout", 0*time.Second, "Maximum waiting time for '--wait' before canceling/returning.")
cmd.Flags().StringSliceP("runtime-label", "", []string{}, "Specify container runtime labels in format \"foo=bar\"")
cmd.Flags().StringSliceP("k3s-node-label", "", []string{}, "Specify k3s node labels in format \"foo=bar\"")
// done
@ -127,9 +129,30 @@ func parseCreateNodeCmd(cmd *cobra.Command, args []string) ([]*k3d.Node, *k3d.Cl
log.Errorf("Provided memory limit value is invalid")
}
// --runtime-label
runtimeLabelsFlag, err := cmd.Flags().GetStringSlice("runtime-label")
if err != nil {
log.Errorln("No runtime-label specified")
log.Fatalln(err)
}
runtimeLabels := make(map[string]string, len(runtimeLabelsFlag)+1)
for _, label := range runtimeLabelsFlag {
labelSplitted := strings.Split(label, "=")
if len(labelSplitted) != 2 {
log.Fatalf("unknown runtime-label format format: %s, use format \"foo=bar\"", label)
}
cliutil.ValidateRuntimeLabelKey(labelSplitted[0])
runtimeLabels[labelSplitted[0]] = labelSplitted[1]
}
// Internal k3d runtime labels take precedence over user-defined labels
runtimeLabels[k3d.LabelRole] = roleStr
// --k3s-node-label
k3sNodeLabelsFlag, err := cmd.Flags().GetStringSlice("k3s-node-label")
if err != nil {
log.Errorln("No node-label specified")
log.Errorln("No k3s-node-label specified")
log.Fatalln(err)
}
@ -137,7 +160,7 @@ func parseCreateNodeCmd(cmd *cobra.Command, args []string) ([]*k3d.Node, *k3d.Cl
for _, label := range k3sNodeLabelsFlag {
labelSplitted := strings.Split(label, "=")
if len(labelSplitted) != 2 {
log.Fatalf("unknown label format format: %s, use format \"foo=bar\"", label)
log.Fatalf("unknown k3s-node-label format format: %s, use format \"foo=bar\"", label)
}
k3sNodeLabels[labelSplitted[0]] = labelSplitted[1]
}
@ -146,13 +169,11 @@ func parseCreateNodeCmd(cmd *cobra.Command, args []string) ([]*k3d.Node, *k3d.Cl
nodes := []*k3d.Node{}
for i := 0; i < replicas; i++ {
node := &k3d.Node{
Name: fmt.Sprintf("%s-%s-%d", k3d.DefaultObjectNamePrefix, args[0], i),
Role: role,
Image: image,
Labels: map[string]string{
k3d.LabelRole: roleStr,
},
Name: fmt.Sprintf("%s-%s-%d", k3d.DefaultObjectNamePrefix, args[0], i),
Role: role,
Image: image,
K3sNodeLabels: k3sNodeLabels,
RuntimeLabels: runtimeLabels,
Restart: true,
Memory: memory,
}

2
cmd/node/nodeList.go
View File

@ -88,7 +88,7 @@ func NewCmdNodeList() *cobra.Command {
fmt.Fprintf(tabwriter, "%s\t%s\t%s\t%s\n",
strings.TrimPrefix(node.Name, "/"),
string(node.Role),
node.Labels[k3d.LabelClusterName],
node.RuntimeLabels[k3d.LabelClusterName],
node.State.Status)
}))
},

4
cmd/registry/registryList.go
View File

@ -88,8 +88,8 @@ func NewCmdRegistryList() *cobra.Command {
util.PrintNodes(existingNodes, registryListFlags.output,
headers, util.NodePrinterFunc(func(tabwriter *tabwriter.Writer, node *k3d.Node) {
cluster := "*"
if _, ok := node.Labels[k3d.LabelClusterName]; ok {
cluster = node.Labels[k3d.LabelClusterName]
if _, ok := node.RuntimeLabels[k3d.LabelClusterName]; ok {
cluster = node.RuntimeLabels[k3d.LabelClusterName]
}
fmt.Fprintf(tabwriter, "%s\t%s\t%s\t%s\n",
strings.TrimPrefix(node.Name, "/"),

35
cmd/util/runtimeLabels.go Normal file
View File

@ -0,0 +1,35 @@
/*
Copyright © 2020 The k3d Author(s)
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
package util
import (
"strings"
log "github.com/sirupsen/logrus"
)
// validateRuntimeLabelKey validates a given label key is not reserved for internal k3d usage
func ValidateRuntimeLabelKey(labelKey string) {
if strings.HasPrefix(labelKey, "k3s.") || strings.HasPrefix(labelKey, "k3d.") || labelKey == "app" {
log.Fatalf("runtime label \"%s\" is reserved for internal usage", labelKey)
}
}

40
pkg/client/cluster.go
View File

@ -381,13 +381,13 @@ ClusterCreatOpts:
nodeSetup := func(node *k3d.Node, suffix int) error {
// cluster specific settings
if node.Labels == nil {
node.Labels = make(map[string]string) // TODO: maybe create an init function?
if node.RuntimeLabels == nil {
node.RuntimeLabels = make(map[string]string) // TODO: maybe create an init function?
}
// ensure global labels
for k, v := range clusterCreateOpts.GlobalLabels {
node.Labels[k] = v
node.RuntimeLabels[k] = v
}
// ensure global env
@ -404,7 +404,7 @@ ClusterCreatOpts:
cluster.Network.IPAM.IPsUsed = append(cluster.Network.IPAM.IPsUsed, ip) // make sure that we're not reusing the same IP next time
node.IP.Static = true
node.IP.IP = ip
node.Labels[k3d.LabelNodeStaticIP] = ip.String()
node.RuntimeLabels[k3d.LabelNodeStaticIP] = ip.String()
}
node.ServerOpts.KubeAPI = cluster.KubeAPI
@ -412,7 +412,7 @@ ClusterCreatOpts:
// the cluster has an init server node, but its not this one, so connect it to the init node
if cluster.InitNode != nil && !node.ServerOpts.IsInit {
node.Env = append(node.Env, fmt.Sprintf("K3S_URL=%s", connectionURL))
node.Labels[k3d.LabelServerIsInit] = "false" // set label, that this server node is not the init server
node.RuntimeLabels[k3d.LabelServerIsInit] = "false" // set label, that this server node is not the init server
}
} else if node.Role == k3d.AgentRole {
@ -446,10 +446,10 @@ ClusterCreatOpts:
if cluster.InitNode != nil {
log.Infoln("Creating initializing server node")
cluster.InitNode.Args = append(cluster.InitNode.Args, "--cluster-init")
if cluster.InitNode.Labels == nil {
cluster.InitNode.Labels = map[string]string{}
if cluster.InitNode.RuntimeLabels == nil {
cluster.InitNode.RuntimeLabels = map[string]string{}
}
cluster.InitNode.Labels[k3d.LabelServerIsInit] = "true" // set label, that this server node is the init server
cluster.InitNode.RuntimeLabels[k3d.LabelServerIsInit] = "true" // set label, that this server node is the init server
// in case the LoadBalancer was disabled, expose the API Port on the initializing server node
if clusterCreateOpts.DisableLoadBalancer {
@ -547,10 +547,10 @@ ClusterCreatOpts:
fmt.Sprintf("PORTS=%s", strings.Join(ports, ",")),
fmt.Sprintf("WORKER_PROCESSES=%d", len(ports)),
},
Role: k3d.LoadBalancerRole,
Labels: clusterCreateOpts.GlobalLabels, // TODO: createLoadBalancer: add more expressive labels
Networks: []string{cluster.Network.Name},
Restart: true,
Role: k3d.LoadBalancerRole,
RuntimeLabels: clusterCreateOpts.GlobalLabels, // TODO: createLoadBalancer: add more expressive labels
Networks: []string{cluster.Network.Name},
Restart: true,
}
if len(udp_ports) > 0 {
lbNode.Env = append(lbNode.Env, fmt.Sprintf("UDP_PORTS=%s", strings.Join(udp_ports, ",")))
@ -673,7 +673,7 @@ func ClusterDelete(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clus
// ClusterList returns a list of all existing clusters
func ClusterList(ctx context.Context, runtime k3drt.Runtime) ([]*k3d.Cluster, error) {
log.Traceln("Listing Clusters...")
nodes, err := runtime.GetNodesByLabel(ctx, k3d.DefaultObjectLabels)
nodes, err := runtime.GetNodesByLabel(ctx, k3d.DefaultRuntimeLabels)
if err != nil {
log.Errorln("Failed to get clusters")
return nil, err
@ -691,7 +691,7 @@ func ClusterList(ctx context.Context, runtime k3drt.Runtime) ([]*k3d.Cluster, er
log.Tracef("Found %d cluster-internal nodes", len(nodes))
if log.GetLevel() == log.TraceLevel {
for _, node := range nodes {
log.Tracef("Found cluster-internal node %s of role %s belonging to cluster %s", node.Name, node.Role, node.Labels[k3d.LabelClusterName])
log.Tracef("Found cluster-internal node %s of role %s belonging to cluster %s", node.Name, node.Role, node.RuntimeLabels[k3d.LabelClusterName])
}
}
@ -700,7 +700,7 @@ func ClusterList(ctx context.Context, runtime k3drt.Runtime) ([]*k3d.Cluster, er
for _, node := range nodes {
clusterExists := false
for _, cluster := range clusters {
if node.Labels[k3d.LabelClusterName] == cluster.Name { // TODO: handle case, where this label doesn't exist
if node.RuntimeLabels[k3d.LabelClusterName] == cluster.Name { // TODO: handle case, where this label doesn't exist
cluster.Nodes = append(cluster.Nodes, node)
clusterExists = true
break
@ -709,7 +709,7 @@ func ClusterList(ctx context.Context, runtime k3drt.Runtime) ([]*k3d.Cluster, er
// cluster is not in the list yet, so we add it with the current node as its first member
if !clusterExists {
clusters = append(clusters, &k3d.Cluster{
Name: node.Labels[k3d.LabelClusterName],
Name: node.RuntimeLabels[k3d.LabelClusterName],
Nodes: []*k3d.Node{node},
})
}
@ -734,7 +734,7 @@ func populateClusterFieldsFromLabels(cluster *k3d.Cluster) error {
// get the name of the cluster network
if cluster.Network.Name == "" {
if networkName, ok := node.Labels[k3d.LabelNetwork]; ok {
if networkName, ok := node.RuntimeLabels[k3d.LabelNetwork]; ok {
cluster.Network.Name = networkName
}
}
@ -742,7 +742,7 @@ func populateClusterFieldsFromLabels(cluster *k3d.Cluster) error {
// check if the network is external
// since the struct value is a bool, initialized as false, we cannot check if it's unset
if !cluster.Network.External && !networkExternalSet {
if networkExternalString, ok := node.Labels[k3d.LabelNetworkExternal]; ok {
if networkExternalString, ok := node.RuntimeLabels[k3d.LabelNetworkExternal]; ok {
if networkExternal, err := strconv.ParseBool(networkExternalString); err == nil {
cluster.Network.External = networkExternal
networkExternalSet = true
@ -752,14 +752,14 @@ func populateClusterFieldsFromLabels(cluster *k3d.Cluster) error {
// get image volume // TODO: enable external image volumes the same way we do it with networks
if cluster.ImageVolume == "" {
if imageVolumeName, ok := node.Labels[k3d.LabelImageVolume]; ok {
if imageVolumeName, ok := node.RuntimeLabels[k3d.LabelImageVolume]; ok {
cluster.ImageVolume = imageVolumeName
}
}
// get k3s cluster's token
if cluster.Token == "" {
if token, ok := node.Labels[k3d.LabelClusterToken]; ok {
if token, ok := node.RuntimeLabels[k3d.LabelClusterToken]; ok {
cluster.Token = token
}
}

8
pkg/client/kubeconfig.go
View File

@ -131,11 +131,11 @@ func KubeconfigGet(ctx context.Context, runtime runtimes.Runtime, cluster *k3d.C
APIHost := k3d.DefaultAPIHost
for _, server := range serverNodes {
if _, ok := server.Labels[k3d.LabelServerAPIPort]; ok {
if _, ok := server.RuntimeLabels[k3d.LabelServerAPIPort]; ok {
chosenServer = server
APIPort = server.Labels[k3d.LabelServerAPIPort]
if _, ok := server.Labels[k3d.LabelServerAPIHost]; ok {
APIHost = server.Labels[k3d.LabelServerAPIHost]
APIPort = server.RuntimeLabels[k3d.LabelServerAPIPort]
if _, ok := server.RuntimeLabels[k3d.LabelServerAPIHost]; ok {
APIHost = server.RuntimeLabels[k3d.LabelServerAPIHost]
}
break
}

40
pkg/client/node.go
View File

@ -59,8 +59,8 @@ func NodeAddToCluster(ctx context.Context, runtime runtimes.Runtime, node *k3d.N
node.Networks = []string{cluster.Network.Name}
// skeleton
if node.Labels == nil {
node.Labels = map[string]string{
if node.RuntimeLabels == nil {
node.RuntimeLabels = map[string]string{
k3d.LabelRole: string(node.Role),
}
}
@ -141,7 +141,7 @@ func NodeAddToCluster(ctx context.Context, runtime runtimes.Runtime, node *k3d.N
}
}
if !k3sURLFound {
if url, ok := node.Labels[k3d.LabelClusterURL]; ok {
if url, ok := node.RuntimeLabels[k3d.LabelClusterURL]; ok {
node.Env = append(node.Env, fmt.Sprintf("K3S_URL=%s", url))
} else {
log.Warnln("Failed to find K3S_URL value!")
@ -381,18 +381,22 @@ func NodeCreate(ctx context.Context, runtime runtimes.Runtime, node *k3d.Node, c
// ### Labels ###
labels := make(map[string]string)
for k, v := range k3d.DefaultObjectLabels {
for k, v := range k3d.DefaultRuntimeLabels {
labels[k] = v
}
for k, v := range k3d.DefaultObjectLabelsVar {
for k, v := range k3d.DefaultRuntimeLabelsVar {
labels[k] = v
}
for k, v := range node.Labels {
for k, v := range node.RuntimeLabels {
labels[k] = v
}
node.Labels = labels
node.RuntimeLabels = labels
// second most important: the node role label
node.Labels[k3d.LabelRole] = string(node.Role)
node.RuntimeLabels[k3d.LabelRole] = string(node.Role)
for k, v := range node.K3sNodeLabels {
node.Args = append(node.Args, "--node-label", fmt.Sprintf("%s=%s", k, v))
}
// ### Environment ###
node.Env = append(node.Env, k3d.DefaultNodeEnv...) // append default node env vars
@ -469,7 +473,7 @@ func NodeDelete(ctx context.Context, runtime runtimes.Runtime, node *k3d.Node, o
// update the server loadbalancer
if !opts.SkipLBUpdate && (node.Role == k3d.ServerRole || node.Role == k3d.AgentRole) {
cluster, err := ClusterGet(ctx, runtime, &k3d.Cluster{Name: node.Labels[k3d.LabelClusterName]})
cluster, err := ClusterGet(ctx, runtime, &k3d.Cluster{Name: node.RuntimeLabels[k3d.LabelClusterName]})
if err != nil {
log.Errorf("Failed to find cluster for node '%s'", node.Name)
return err
@ -493,10 +497,6 @@ func patchAgentSpec(node *k3d.Node) error {
node.Cmd = []string{"agent"}
}
for k, v := range node.K3sNodeLabels {
node.Args = append(node.Args, "--node-label", fmt.Sprintf("%s=%s", k, v))
}
return nil
}
@ -509,9 +509,9 @@ func patchServerSpec(node *k3d.Node, runtime runtimes.Runtime) error {
// Add labels and TLS SAN for the exposed API
// FIXME: For now, the labels concerning the API on the server nodes are only being used for configuring the kubeconfig
node.Labels[k3d.LabelServerAPIHostIP] = node.ServerOpts.KubeAPI.Binding.HostIP // TODO: maybe get docker machine IP here
node.Labels[k3d.LabelServerAPIHost] = node.ServerOpts.KubeAPI.Host
node.Labels[k3d.LabelServerAPIPort] = node.ServerOpts.KubeAPI.Binding.HostPort
node.RuntimeLabels[k3d.LabelServerAPIHostIP] = node.ServerOpts.KubeAPI.Binding.HostIP // TODO: maybe get docker machine IP here
node.RuntimeLabels[k3d.LabelServerAPIHost] = node.ServerOpts.KubeAPI.Host
node.RuntimeLabels[k3d.LabelServerAPIPort] = node.ServerOpts.KubeAPI.Binding.HostPort
// If the runtime is docker, attempt to use the docker host
if runtime == runtimes.Docker {
@ -519,19 +519,19 @@ func patchServerSpec(node *k3d.Node, runtime runtimes.Runtime) error {
if dockerHost != "" {
dockerHost = strings.Split(dockerHost, ":")[0] // remove the port
log.Tracef("Using docker host %s", dockerHost)
node.Labels[k3d.LabelServerAPIHostIP] = dockerHost
node.Labels[k3d.LabelServerAPIHost] = dockerHost
node.RuntimeLabels[k3d.LabelServerAPIHostIP] = dockerHost
node.RuntimeLabels[k3d.LabelServerAPIHost] = dockerHost
}
}
node.Args = append(node.Args, "--tls-san", node.Labels[k3d.LabelServerAPIHost]) // add TLS SAN for non default host name
node.Args = append(node.Args, "--tls-san", node.RuntimeLabels[k3d.LabelServerAPIHost]) // add TLS SAN for non default host name
return nil
}
// NodeList returns a list of all existing clusters
func NodeList(ctx context.Context, runtime runtimes.Runtime) ([]*k3d.Node, error) {
nodes, err := runtime.GetNodesByLabel(ctx, k3d.DefaultObjectLabels)
nodes, err := runtime.GetNodesByLabel(ctx, k3d.DefaultRuntimeLabels)
if err != nil {
log.Errorln("Failed to get nodes")
return nil, err

10
pkg/client/registry.go
View File

@ -77,7 +77,7 @@ func RegistryCreate(ctx context.Context, runtime runtimes.Runtime, reg *k3d.Regi
}
// setup the node labels
registryNode.Labels = map[string]string{
registryNode.RuntimeLabels = map[string]string{
k3d.LabelClusterName: reg.ClusterRef,
k3d.LabelRole: string(k3d.RegistryRole),
k3d.LabelRegistryHost: reg.ExposureOpts.Host, // TODO: docker machine host?
@ -85,11 +85,11 @@ func RegistryCreate(ctx context.Context, runtime runtimes.Runtime, reg *k3d.Regi
k3d.LabelRegistryPortExternal: reg.ExposureOpts.Binding.HostPort,
k3d.LabelRegistryPortInternal: reg.ExposureOpts.Port.Port(),
}
for k, v := range k3d.DefaultObjectLabels {
registryNode.Labels[k] = v
for k, v := range k3d.DefaultRuntimeLabels {
registryNode.RuntimeLabels[k] = v
}
for k, v := range k3d.DefaultObjectLabelsVar {
registryNode.Labels[k] = v
for k, v := range k3d.DefaultRuntimeLabelsVar {
registryNode.RuntimeLabels[k] = v
}
// port

20
pkg/config/config_test.go
View File

@ -64,12 +64,6 @@ func TestReadSimpleConfig(t *testing.T) {
NodeFilters: []string{"loadbalancer"},
},
},
Labels: []conf.LabelWithNodeFilters{
{
Label: "foo=bar",
NodeFilters: []string{"server[0]", "loadbalancer"},
},
},
Env: []conf.EnvVarWithNodeFilters{
{
EnvVar: "bar=baz",
@ -90,11 +84,25 @@ func TestReadSimpleConfig(t *testing.T) {
NodeFilters: []string{"server[*]"},
},
},
NodeLabels: []conf.LabelWithNodeFilters{
{
Label: "foo=bar",
NodeFilters: []string{"server[0]", "loadbalancer"},
},
},
},
KubeconfigOptions: conf.SimpleConfigOptionsKubeconfig{
UpdateDefaultKubeconfig: true,
SwitchCurrentContext: true,
},
Runtime: conf.SimpleConfigOptionsRuntime{
Labels: []conf.LabelWithNodeFilters{
{
Label: "foo=bar",
NodeFilters: []string{"server[0]", "loadbalancer"},
},
},
},
},
}

16
pkg/config/test_assets/config_test_simple.yaml
View File

@ -22,11 +22,6 @@ env:
- envVar: bar=baz
nodeFilters:
- all
labels:
- label: foo=bar
nodeFilters:
- "server[0]"
- loadbalancer
options:
k3d:
@ -39,6 +34,17 @@ options:
- arg: --tls-san=127.0.0.1
nodeFilters:
- "server[*]"
nodeLabels:
- label: foo=bar
nodeFilters:
- server[0]
- loadbalancer
kubeconfig:
updateDefaultKubeconfig: true
switchCurrentContext: true
runtime:
labels:
- label: foo=bar
nodeFilters:
- server[0]
- loadbalancer

18
pkg/config/test_assets/config_test_simple_invalid_servers.yaml
View File

@ -22,11 +22,6 @@ env:
- envVar: bar=baz
nodeFilters:
- all
labels:
- label: foo=bar
nodeFilters:
- server[0]
- loadbalancer
options:
k3d:
@ -39,6 +34,17 @@ options:
- arg: --tls-san=127.0.0.1
nodeFilters:
- "server[*]"
nodeLabels:
- label: foo=bar
nodeFilters:
- server[0]
- loadbalancer
kubeconfig:
updateDefaultKubeconfig: true
switchCurrentContext: true
switchCurrentContext: true
runtime:
labels:
- label: foo=bar
nodeFilters:
- server[0]
- loadbalancer

44
pkg/config/transform.go
View File

@ -190,23 +190,47 @@ func TransformSimpleToClusterConfig(ctx context.Context, runtime runtimes.Runtim
}
}
// -> LABELS
for _, labelWithNodeFilters := range simpleConfig.Labels {
if len(labelWithNodeFilters.NodeFilters) == 0 && nodeCount > 1 {
return nil, fmt.Errorf("Labelmapping '%s' lacks a node filter, but there's more than one node", labelWithNodeFilters.Label)
// -> K3S NODE LABELS
for _, k3sNodeLabelWithNodeFilters := range simpleConfig.Options.K3sOptions.NodeLabels {
if len(k3sNodeLabelWithNodeFilters.NodeFilters) == 0 && nodeCount > 1 {
return nil, fmt.Errorf("K3sNodeLabelmapping '%s' lacks a node filter, but there's more than one node", k3sNodeLabelWithNodeFilters.Label)
}
nodes, err := util.FilterNodes(nodeList, labelWithNodeFilters.NodeFilters)
nodes, err := util.FilterNodes(nodeList, k3sNodeLabelWithNodeFilters.NodeFilters)
if err != nil {
return nil, err
}
for _, node := range nodes {
if node.Labels == nil {
node.Labels = make(map[string]string) // ensure that the map is initialized
if node.K3sNodeLabels == nil {
node.K3sNodeLabels = make(map[string]string) // ensure that the map is initialized
}
k, v := util.SplitLabelKeyValue(labelWithNodeFilters.Label)
node.Labels[k] = v
k, v := util.SplitLabelKeyValue(k3sNodeLabelWithNodeFilters.Label)
node.K3sNodeLabels[k] = v
}
}
// -> RUNTIME LABELS
for _, runtimeLabelWithNodeFilters := range simpleConfig.Options.Runtime.Labels {
if len(runtimeLabelWithNodeFilters.NodeFilters) == 0 && nodeCount > 1 {
return nil, fmt.Errorf("RuntimeLabelmapping '%s' lacks a node filter, but there's more than one node", runtimeLabelWithNodeFilters.Label)
}
nodes, err := util.FilterNodes(nodeList, runtimeLabelWithNodeFilters.NodeFilters)
if err != nil {
return nil, err
}
for _, node := range nodes {
if node.RuntimeLabels == nil {
node.RuntimeLabels = make(map[string]string) // ensure that the map is initialized
}
k, v := util.SplitLabelKeyValue(runtimeLabelWithNodeFilters.Label)
cliutil.ValidateRuntimeLabelKey(k)
node.RuntimeLabels[k] = v
}
}
@ -260,7 +284,7 @@ func TransformSimpleToClusterConfig(ctx context.Context, runtime runtimes.Runtim
}
// ensure, that we have the default object labels
for k, v := range k3d.DefaultObjectLabels {
for k, v := range k3d.DefaultRuntimeLabels {
clusterCreateOpts.GlobalLabels[k] = v
}

9
pkg/config/v1alpha3/migrations.go
View File

@ -49,6 +49,15 @@ func MigrateV1Alpha2(input configtypes.Config) (configtypes.Config, error) {
return nil, err
}
cfg.Options.Runtime.Labels = []LabelWithNodeFilters{}
for _, label := range input.(v1alpha2.SimpleConfig).Labels {
cfg.Options.Runtime.Labels = append(cfg.Options.Runtime.Labels, LabelWithNodeFilters{
Label: label.Label,
NodeFilters: label.NodeFilters,
})
}
cfg.Options.K3sOptions.ExtraArgs = []K3sArgWithNodeFilters{}
for _, arg := range input.(v1alpha2.SimpleConfig).Options.K3sOptions.ExtraServerArgs {

52
pkg/config/v1alpha3/schema.json
View File

@ -108,21 +108,6 @@
"additionalProperties": false
}
},
"labels": {
"type": "array",
"items": {
"type": "object",
"properties": {
"label": {
"type": "string"
},
"nodeFilters": {
"$ref": "#/definitions/nodeFilters"
}
},
"additionalProperties": false
}
},
"options": {
"type": "object",
"properties": {
@ -170,12 +155,24 @@
"properties": {
"arg": {
"type": "string",
"examples": [
"--tls-san=127.0.0.1",
"--disable=traefik"
]
"examples": ["--tls-san=127.0.0.1", "--disable=traefik"]
},
"nodeFilters": {
"nodeFilters": {
"$ref": "#/definitions/nodeFilters"
}
},
"additionalProperties": false
}
},
"nodeLabels": {
"type": "array",
"items": {
"type": "object",
"properties": {
"label": {
"type": "string"
},
"nodeFilters": {
"$ref": "#/definitions/nodeFilters"
}
},
@ -210,6 +207,21 @@
},
"agentsMemory": {
"type": "string"
},
"labels": {
"type": "array",
"items": {
"type": "object",
"properties": {
"label": {
"type": "string"
},
"nodeFilters": {
"$ref": "#/definitions/nodeFilters"
}
},
"additionalProperties": false
}
}
}
}

11
pkg/config/v1alpha3/types.go
View File

@ -95,9 +95,10 @@ type SimpleConfigOptions struct {
}
type SimpleConfigOptionsRuntime struct {
GPURequest string `mapstructure:"gpuRequest" yaml:"gpuRequest"`
ServersMemory string `mapstructure:"serversMemory" yaml:"serversMemory"`
AgentsMemory string `mapstructure:"agentsMemory" yaml:"agentsMemory"`
GPURequest string `mapstructure:"gpuRequest" yaml:"gpuRequest"`
ServersMemory string `mapstructure:"serversMemory" yaml:"serversMemory"`
AgentsMemory string `mapstructure:"agentsMemory" yaml:"agentsMemory"`
Labels []LabelWithNodeFilters `mapstructure:"labels" yaml:"labels"`
}
type SimpleConfigOptionsK3d struct {
@ -111,7 +112,8 @@ type SimpleConfigOptionsK3d struct {
}
type SimpleConfigOptionsK3s struct {
ExtraArgs []K3sArgWithNodeFilters `mapstructure:"extraArgs" yaml:"extraArgs"`
ExtraArgs []K3sArgWithNodeFilters `mapstructure:"extraArgs" yaml:"extraArgs"`
NodeLabels []LabelWithNodeFilters `mapstructure:"nodeLabels" yaml:"nodeLabels"`
}
// SimpleConfig describes the toplevel k3d configuration file.
@ -127,7 +129,6 @@ type SimpleConfig struct {
ClusterToken string `mapstructure:"token" yaml:"clusterToken" json:"clusterToken,omitempty"` // default: auto-generated
Volumes []VolumeWithNodeFilters `mapstructure:"volumes" yaml:"volumes" json:"volumes,omitempty"`
Ports []PortWithNodeFilters `mapstructure:"ports" yaml:"ports" json:"ports,omitempty"`
Labels []LabelWithNodeFilters `mapstructure:"labels" yaml:"labels" json:"labels,omitempty"`
Options SimpleConfigOptions `mapstructure:"options" yaml:"options" json:"options,omitempty"`
Env []EnvVarWithNodeFilters `mapstructure:"env" yaml:"env" json:"env,omitempty"`
Registries struct {

2
pkg/runtimes/docker/container.go
View File

@ -151,7 +151,7 @@ func getNodeContainer(ctx context.Context, node *k3d.Node) (*types.Container, er
// (1) list containers which have the default k3d labels attached
filters := filters.NewArgs()
for k, v := range node.Labels {
for k, v := range node.RuntimeLabels {
filters.Add("label", fmt.Sprintf("%s=%s", k, v))
}

2
pkg/runtimes/docker/network.go
View File

@ -147,7 +147,7 @@ func (d Docker) CreateNetworkIfNotPresent(ctx context.Context, inNet *k3d.Cluste
// (3) Create a new network
netCreateOpts := types.NetworkCreate{
CheckDuplicate: true,
Labels: k3d.DefaultObjectLabels,
Labels: k3d.DefaultRuntimeLabels,
}
// we want a managed (user-defined) network, but user didn't specify a subnet, so we try to auto-generate one

2
pkg/runtimes/docker/node.go
View File

@ -178,7 +178,7 @@ func getContainersByLabel(ctx context.Context, labels map[string]string) ([]type
// (1) list containers which have the default k3d labels attached
filters := filters.NewArgs()
for k, v := range k3d.DefaultObjectLabels {
for k, v := range k3d.DefaultRuntimeLabels {
filters.Add("label", fmt.Sprintf("%s=%s", k, v))
}
for k, v := range labels {

44
pkg/runtimes/docker/translate.go
View File

@ -73,7 +73,7 @@ func TranslateNodeToContainer(node *k3d.Node) (*NodeInDocker, error) {
containerConfig.Env = node.Env
/* Labels */
containerConfig.Labels = node.Labels // has to include the role
containerConfig.Labels = node.RuntimeLabels // has to include the role
/* Auto-Restart */
if node.Restart {
@ -162,10 +162,10 @@ func TranslateNodeToContainer(node *k3d.Node) (*NodeInDocker, error) {
// TranslateContainerToNode translates a docker container object into a k3d node representation
func TranslateContainerToNode(cont *types.Container) (*k3d.Node, error) {
node := &k3d.Node{
Name: strings.TrimPrefix(cont.Names[0], "/"), // container name with leading '/' cut off
Image: cont.Image,
Labels: cont.Labels,
Role: k3d.NodeRoles[cont.Labels[k3d.LabelRole]],
Name: strings.TrimPrefix(cont.Names[0], "/"), // container name with leading '/' cut off
Image: cont.Image,
RuntimeLabels: cont.Labels,
Role: k3d.NodeRoles[cont.Labels[k3d.LabelRole]],
// TODO: all the rest
}
return node, nil
@ -175,7 +175,7 @@ func TranslateContainerToNode(cont *types.Container) (*k3d.Node, error) {
func TranslateContainerDetailsToNode(containerDetails types.ContainerJSON) (*k3d.Node, error) {
// first, make sure, that it's actually a k3d managed container by checking if it has all the default labels
for k, v := range k3d.DefaultObjectLabels {
for k, v := range k3d.DefaultRuntimeLabels {
log.Tracef("TranslateContainerDetailsToNode: Checking for default object label %s=%s on container %s", k, v, containerDetails.Name)
found := false
for lk, lv := range containerDetails.Config.Labels {
@ -273,22 +273,22 @@ func TranslateContainerDetailsToNode(containerDetails types.ContainerJSON) (*k3d
}
node := &k3d.Node{
Name: strings.TrimPrefix(containerDetails.Name, "/"), // container name with leading '/' cut off
Role: k3d.NodeRoles[containerDetails.Config.Labels[k3d.LabelRole]],
Image: containerDetails.Image,
Volumes: containerDetails.HostConfig.Binds,
Env: env,
Cmd: containerDetails.Config.Cmd,
Args: []string{}, // empty, since Cmd already contains flags
Ports: containerDetails.HostConfig.PortBindings,
Restart: restart,
Created: containerDetails.Created,
Labels: labels,
Networks: orderedNetworks,
ServerOpts: serverOpts,
AgentOpts: k3d.AgentOpts{},
State: nodeState,
Memory: memoryStr,
Name: strings.TrimPrefix(containerDetails.Name, "/"), // container name with leading '/' cut off
Role: k3d.NodeRoles[containerDetails.Config.Labels[k3d.LabelRole]],
Image: containerDetails.Image,
Volumes: containerDetails.HostConfig.Binds,
Env: env,
Cmd: containerDetails.Config.Cmd,
Args: []string{}, // empty, since Cmd already contains flags
Ports: containerDetails.HostConfig.PortBindings,
Restart: restart,
Created: containerDetails.Created,
RuntimeLabels: labels,
Networks: orderedNetworks,
ServerOpts: serverOpts,
AgentOpts: k3d.AgentOpts{},
State: nodeState,
Memory: memoryStr,
}
return node, nil
}

6
pkg/runtimes/docker/translate_test.go
View File

@ -52,9 +52,9 @@ func TestTranslateNodeToContainer(t *testing.T) {
},
},
},
Restart: true,
Labels: map[string]string{k3d.LabelRole: string(k3d.ServerRole), "test_key_1": "test_val_1"},
Networks: []string{"mynet"},
Restart: true,
RuntimeLabels: map[string]string{k3d.LabelRole: string(k3d.ServerRole), "test_key_1": "test_val_1"},
Networks: []string{"mynet"},
}
init := true

2
pkg/runtimes/docker/util.go
View File

@ -44,7 +44,7 @@ import (
// GetDefaultObjectLabelsFilter returns docker type filters created from k3d labels
func GetDefaultObjectLabelsFilter(clusterName string) filters.Args {
filters := filters.NewArgs()
for key, value := range k3d.DefaultObjectLabels {
for key, value := range k3d.DefaultRuntimeLabels {
filters.Add("label", fmt.Sprintf("%s=%s", key, value))
}
filters.Add("label", fmt.Sprintf("%s=%s", k3d.LabelClusterName, clusterName))

4
pkg/runtimes/docker/volume.go
View File

@ -49,10 +49,10 @@ func (d Docker) CreateVolume(ctx context.Context, name string, labels map[string
DriverOpts: map[string]string{},
}
for k, v := range k3d.DefaultObjectLabels {
for k, v := range k3d.DefaultRuntimeLabels {
volumeCreateOptions.Labels[k] = v
}
for k, v := range k3d.DefaultObjectLabelsVar {
for k, v := range k3d.DefaultRuntimeLabelsVar {
volumeCreateOptions.Labels[k] = v
}

24
pkg/tools/tools.go
View File

@ -65,7 +65,7 @@ func ImageImportIntoClusterMulti(ctx context.Context, runtime runtimes.Runtime,
var ok bool
for _, node := range cluster.Nodes {
if node.Role == k3d.ServerRole || node.Role == k3d.AgentRole {
if imageVolume, ok = node.Labels[k3d.LabelImageVolume]; ok {
if imageVolume, ok = node.RuntimeLabels[k3d.LabelImageVolume]; ok {
break
}
}
@ -276,23 +276,23 @@ func containsVersionPart(imageTag string) bool {
// runToolsNode will start a new k3d tools container and connect it to the network of the chosen cluster
func runToolsNode(ctx context.Context, runtime runtimes.Runtime, cluster *k3d.Cluster, network string, volumes []string) (*k3d.Node, error) {
labels := map[string]string{}
for k, v := range k3d.DefaultObjectLabels {
for k, v := range k3d.DefaultRuntimeLabels {
labels[k] = v
}
for k, v := range k3d.DefaultObjectLabelsVar {
for k, v := range k3d.DefaultRuntimeLabelsVar {
labels[k] = v
}
node := &k3d.Node{
Name: fmt.Sprintf("%s-%s-tools", k3d.DefaultObjectNamePrefix, cluster.Name),
Image: fmt.Sprintf("%s:%s", k3d.DefaultToolsImageRepo, version.GetHelperImageVersion()),
Role: k3d.NoRole,
Volumes: volumes,
Networks: []string{network},
Cmd: []string{},
Args: []string{"noop"},
Labels: k3d.DefaultObjectLabels,
Name: fmt.Sprintf("%s-%s-tools", k3d.DefaultObjectNamePrefix, cluster.Name),
Image: fmt.Sprintf("%s:%s", k3d.DefaultToolsImageRepo, version.GetHelperImageVersion()),
Role: k3d.NoRole,
Volumes: volumes,
Networks: []string{network},
Cmd: []string{},
Args: []string{"noop"},
RuntimeLabels: k3d.DefaultRuntimeLabels,
}
node.Labels[k3d.LabelClusterName] = cluster.Name
node.RuntimeLabels[k3d.LabelClusterName] = cluster.Name
if err := k3dc.NodeRun(ctx, runtime, node, k3d.NodeCreateOpts{}); err != nil {
log.Errorf("Failed to create tools container for cluster '%s'", cluster.Name)
return node, err

10
pkg/types/types.go
View File

@ -105,13 +105,13 @@ var ClusterExternalNodeRoles = []Role{
RegistryRole,
}
// DefaultObjectLabels specifies a set of labels that will be attached to k3d objects by default
var DefaultObjectLabels = map[string]string{
// DefaultRuntimeLabels specifies a set of labels that will be attached to k3d runtime objects by default
var DefaultRuntimeLabels = map[string]string{
"app": "k3d",
}
// DefaultObjectLabelsVar specifies a set of labels that will be attached to k3d objects by default but are not static (e.g. across k3d versions)
var DefaultObjectLabelsVar = map[string]string{
// DefaultRuntimeLabelsVar specifies a set of labels that will be attached to k3d runtime objects by default but are not static (e.g. across k3d versions)
var DefaultRuntimeLabelsVar = map[string]string{
"k3d.version": version.GetVersion(),
}
@ -339,7 +339,7 @@ type Node struct {
Ports nat.PortMap `yaml:"portMappings" json:"portMappings,omitempty"`
Restart bool `yaml:"restart" json:"restart,omitempty"`
Created string `yaml:"created" json:"created,omitempty"`
Labels map[string]string // filled automatically
RuntimeLabels map[string]string `yaml:"runtimeLabels" json:"runtimeLabels,omitempty"`
K3sNodeLabels map[string]string `yaml:"k3sNodeLabels" json:"k3sNodeLabels,omitempty"`
Networks []string // filled automatically
ExtraHosts []string // filled automatically

18
tests/assets/config_test_simple.yaml
View File

@ -22,11 +22,6 @@ env:
- envVar: bar=baz,bob
nodeFilters:
- all
labels:
- label: foo=bar
nodeFilters:
- server[0]
- loadbalancer
registries:
create: true
use: []
@ -47,6 +42,17 @@ options:
- arg: --tls-san=127.0.0.1
nodeFilters:
- server[*]
nodeLabels:
- label: foo=bar
nodeFilters:
- server[0]
- loadbalancer
kubeconfig:
updateDefaultKubeconfig: true
switchCurrentContext: true
switchCurrentContext: true
runtime:
labels:
- label: foo=bar
nodeFilters:
- server[0]
- loadbalancer

13
tests/assets/config_test_simple_migration_v1alpha3.yaml
View File

@ -22,11 +22,6 @@ env:
- envVar: bar=baz,bob
nodeFilters:
- all
labels:
- label: foo=bar
nodeFilters:
- server[0]
- loadbalancer
registries:
create: true
use: []
@ -49,4 +44,10 @@ options:
- server[*]
kubeconfig:
updateDefaultKubeconfig: true
switchCurrentContext: true
switchCurrentContext: true
runtime:
labels:
- label: foo=bar
nodeFilters:
- server[0]
- loadbalancer

6
tests/common.sh
View File

@ -175,4 +175,10 @@ docker_assert_container_label() {
# $1 = container/node name
# $2 = label to assert
docker inspect --format '{{ range $k, $v := .Config.Labels }}{{ printf "%s=%s\n" $k $v }}{{ end }}' "$1" | grep -E "^$2$"
}
k3s_assert_node_label() {
# $1 = node name
# $2 = label to assert
kubectl get node "$1" --output go-template='{{ range $k, $v := .metadata.labels }}{{ printf "%s=%s\n" $k $v }}{{ end }}' | grep -E "^$2$"
}

6
tests/test_config_file.sh
View File

@ -18,7 +18,7 @@ fi
export CURRENT_STAGE="Test | config-file | $K3S_IMAGE_TAG"
clustername="ConfigTest"
clustername="configtest"
highlight "[START] ConfigTest $EXTRA_TITLE"
@ -45,6 +45,10 @@ exec_in_node "k3d-$clustername-server-0" "env" | grep "bar=baz,bob" || failed "E
info "Ensuring that container labels have been set as stated in the config"
docker_assert_container_label "k3d-$clustername-server-0" "foo=bar" || failed "Expected label 'foo=bar' not present on container/node k3d-$clustername-server-0"
## K3s Node Labels
info "Ensuring that k3s node labels have been set as stated in the config"
k3s_assert_node_label "k3d-$clustername-server-0" "foo=bar" || failed "Expected label 'foo=bar' not present on node k3d-$clustername-server-0"
## Registry Node
info "Ensuring, that we have a registry node present"
$EXE node list "k3d-$clustername-registry" || failed "Expected k3d-$clustername-registry to be present"

4
tests/test_config_with_overrides.sh
View File

@ -44,6 +44,10 @@ exec_in_node "k3d-$clustername-agent-1" "env" | grep "x=y" || failed "Expected e
info "Ensuring that container labels have been set as stated in the config"
docker_assert_container_label "k3d-$clustername-server-0" "foo=bar" || failed "Expected label 'foo=bar' not present on container/node k3d-$clustername-server-0"
## K3s Node Labels
info "Ensuring that k3s node labels have been set as stated in the config"
k3s_assert_node_label "k3d-$clustername-server-0" "foo=bar" || failed "Expected label 'foo=bar' not present on node k3d-$clustername-server-0"
## Registry Node
info "Ensuring, that we DO NOT have a registry node present"
$EXE node list "k3d-$clustername-registry" && failed "Expected k3d-$clustername-registry to NOT be present"