fix: proper handling of registries and networks when deleting clusters/nodes

CHANGELOG.md

@@ -14,7 +14,7 @@
 - `pkg/cluster` is now `pkg/client`
 - `ClusterCreate` and `NodeCreate` don't start the entities (containers) anymore
   - `ClusterRun` and `NodeRun` orchestrate the new Create and Start functionality
-- `NodeDelete` now takes an additional `NodeDeleteOpts` struct to toggle specific steps
+- `NodeDelete`/`ClusterDelete` now take an additional `NodeDeleteOpts`/`ClusterDeleteOpts` struct to toggle specific steps
 - NodeSpec now features a list of networks (required for registries)
 - New config flow: CLIConfig (SimpleConfig) -> ClusterConfig -> Cluster + Opts
 

cmd/cluster/clusterCreate.go

@@ -133,7 +133,7 @@ func NewCmdClusterCreate() *cobra.Command {
 				}
 				// rollback if creation failed
 				log.Errorln("Failed to create cluster >>> Rolling Back")
-				if err := k3dCluster.ClusterDelete(cmd.Context(), runtimes.SelectedRuntime, &clusterConfig.Cluster); err != nil {
+				if err := k3dCluster.ClusterDelete(cmd.Context(), runtimes.SelectedRuntime, &clusterConfig.Cluster, k3d.ClusterDeleteOpts{SkipRegistryCheck: true}); err != nil {
 					log.Errorln(err)
 					log.Fatalln("Cluster creation FAILED, also FAILED to rollback changes!")
 				}

cmd/cluster/clusterDelete.go

@@ -54,7 +54,7 @@ func NewCmdClusterDelete() *cobra.Command {
 				log.Infoln("No clusters found")
 			} else {
 				for _, c := range clusters {
-					if err := client.ClusterDelete(cmd.Context(), runtimes.SelectedRuntime, c); err != nil {
+					if err := client.ClusterDelete(cmd.Context(), runtimes.SelectedRuntime, c, k3d.ClusterDeleteOpts{SkipRegistryCheck: false}); err != nil {
 						log.Fatalln(err)
 					}
 					log.Infoln("Removing cluster details from default kubeconfig...")

cmd/node/nodeDelete.go

@@ -49,12 +49,7 @@ func NewCmdNodeDelete() *cobra.Command {
 		Run: func(cmd *cobra.Command, args []string) {
 
 			nodes := parseDeleteNodeCmd(cmd, args, &flags)
-			nodeDeleteOpts := k3d.NodeDeleteOpts{
+			nodeDeleteOpts := k3d.NodeDeleteOpts{SkipLBUpdate: flags.All} // do not update LB, if we're deleting all nodes anyway
-				SkipRegistryCheck: true,
-			}
-			if flags.All {
-				nodeDeleteOpts.SkipLBUpdate = true
-			}
 
 			if len(nodes) == 0 {
 				log.Infoln("No nodes found")

cmd/registry/registryDelete.go

@@ -53,7 +53,7 @@ func NewCmdRegistryDelete() *cobra.Command {
 				log.Infoln("No registries found")
 			} else {
 				for _, node := range nodes {
-					if err := client.NodeDelete(cmd.Context(), runtimes.SelectedRuntime, node, k3d.NodeDeleteOpts{SkipLBUpdate: true, SkipRegistryCheck: true}); err != nil {
+					if err := client.NodeDelete(cmd.Context(), runtimes.SelectedRuntime, node, k3d.NodeDeleteOpts{SkipLBUpdate: true}); err != nil {
 						log.Fatalln(err)
 					}
 				}

pkg/client/cluster.go

@@ -269,8 +269,8 @@ func ClusterPrepNetwork(ctx context.Context, runtime k3drt.Runtime, cluster *k3d
 		log.Errorln("Failed to create cluster network")
 		return err
 	}
-	cluster.Network.Name = networkID
+	clusterCreateOpts.GlobalLabels[k3d.LabelNetworkID] = networkID
-	clusterCreateOpts.GlobalLabels[k3d.LabelNetwork] = networkID
+	clusterCreateOpts.GlobalLabels[k3d.LabelNetwork] = cluster.Network.Name
 	clusterCreateOpts.GlobalLabels[k3d.LabelNetworkExternal] = strconv.FormatBool(cluster.Network.External)
 	if networkExists {
 		clusterCreateOpts.GlobalLabels[k3d.LabelNetworkExternal] = "true" // if the network wasn't created, we say that it's managed externally (important for cluster deletion)
@@ -528,7 +528,7 @@ ClusterCreatOpts:
 }
 
 // ClusterDelete deletes an existing cluster
-func ClusterDelete(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluster) error {
+func ClusterDelete(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluster, opts k3d.ClusterDeleteOpts) error {
 
 	log.Infof("Deleting cluster '%s'", cluster.Name)
 	cluster, err := ClusterGet(ctx, runtime, cluster)
@@ -539,6 +539,35 @@ func ClusterDelete(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clus
 
 	failed := 0
 	for _, node := range cluster.Nodes {
+		// registry: only delete, if not connected to other networks
+		if node.Role == k3d.RegistryRole && !opts.SkipRegistryCheck {
+			log.Tracef("Registry Node has %d networks: %+v", len(node.Networks), node)
+
+			// check if node is connected to other networks, that are not
+			// - the cluster network
+			// - default docker networks
+			// -> if so, disconnect it from the cluster network and continue
+			connectedToOtherNet := false
+			for _, net := range node.Networks {
+				if net == cluster.Network.Name {
+					continue
+				}
+				if net == "bridge" || net == "host" {
+					continue
+				}
+				log.Tracef("net: %s", net)
+				connectedToOtherNet = true
+				break
+			}
+			if connectedToOtherNet {
+				log.Infof("Registry %s is also connected to other (non-default) networks (%+v), not deleting it...", node.Name, node.Networks)
+				if err := runtime.DisconnectNodeFromNetwork(ctx, node, cluster.Network.Name); err != nil {
+					log.Warnf("Failed to disconnect registry %s from cluster network %s", node.Name, cluster.Network.Name)
+				}
+				continue
+			}
+		}
+
 		if err := NodeDelete(ctx, runtime, node, k3d.NodeDeleteOpts{SkipLBUpdate: true}); err != nil {
 			log.Warningf("Failed to delete node '%s': Try to delete it manually", node.Name)
 			failed++

pkg/client/node.go

@@ -289,11 +289,6 @@ func NodeCreate(ctx context.Context, runtime runtimes.Runtime, node *k3d.Node, c
 // NodeDelete deletes an existing node
 func NodeDelete(ctx context.Context, runtime runtimes.Runtime, node *k3d.Node, opts k3d.NodeDeleteOpts) error {
 
-	// registry: only delete, if not connected to other networks
-	if !opts.SkipRegistryCheck && node.Role == k3d.RegistryRole {
-		log.Debugln("NodeDelete special case: registry -> not yet implemented")
-	}
-
 	// delete node
 	if err := runtime.DeleteNode(ctx, node); err != nil {
 		log.Error(err)

pkg/runtimes/docker/network.go

@@ -158,6 +158,7 @@ func (d Docker) ConnectNodeToNetwork(ctx context.Context, node *k3d.Node, networ
 
 // DisconnectNodeFromNetwork disconnects a node from a network (u don't say :O)
 func (d Docker) DisconnectNodeFromNetwork(ctx context.Context, node *k3d.Node, networkName string) error {
+	log.Debugf("Disconnecting node %s from network %s...", node.Name, networkName)
 	// get container
 	container, err := getNodeContainer(ctx, node)
 	if err != nil {

pkg/types/types.go

@@ -116,6 +116,7 @@ const (
 	LabelImageVolume          string = "k3d.cluster.imageVolume"
 	LabelNetworkExternal      string = "k3d.cluster.network.external"
 	LabelNetwork              string = "k3d.cluster.network"
+	LabelNetworkID            string = "k3d.cluster.network.id"
 	LabelRole                 string = "k3d.role"
 	LabelServerAPIPort        string = "k3d.server.api.port"
 	LabelServerAPIHost        string = "k3d.server.api.host"
@@ -208,6 +209,11 @@ type ClusterStartOpts struct {
 	NodeHooks     []NodeHook `yaml:"nodeHooks,omitempty" json:"nodeHooks,omitempty"`
 }
 
+// ClusterDeleteOpts describe a set of options one can set when deleting a cluster
+type ClusterDeleteOpts struct {
+	SkipRegistryCheck bool // skip checking if this is a registry (and act accordingly)
+}
+
 // NodeCreateOpts describes a set of options one can set when creating a new node
 type NodeCreateOpts struct {
 	Wait      bool
@@ -225,7 +231,6 @@ type NodeStartOpts struct {
 // NodeDeleteOpts describes a set of options one can set when deleting a node
 type NodeDeleteOpts struct {
 	SkipLBUpdate bool // skip updating the loadbalancer
-	SkipRegistryCheck bool // skip checking if this is a registry (and act accordingly)
 }
 
 // NodeHookAction is an interface to implement actions that should trigger at specific points of the node lifecycle