[Enhancement] clusterDelete: proper node and network handling (#437)

This comes with several fixes/improvements

- only consider containers that have the default object label (app=k3d)
- handle network deletion
  - check if there are other k3d containers connected
  - if there are only registries, disconnect them
  - if there are non-registry nodes, leave everything as it is
  - if there are any containers connected, that are not automatically
  disconnected, log a warning and continue

cmd/cluster/clusterDelete.go

@@ -100,6 +100,7 @@ func parseDeleteClusterCmd(cmd *cobra.Command, args []string) []*k3d.Cluster {
 	if all, err := cmd.Flags().GetBool("all"); err != nil {
 		log.Fatalln(err)
 	} else if all {
+		log.Infoln("Deleting all clusters...")
 		clusters, err = client.ClusterList(cmd.Context(), runtimes.SelectedRuntime)
 		if err != nil {
 			log.Fatalln(err)

docs/usage/guides/registries.md

@@ -67,7 +67,7 @@ Finally, we can create the cluster, mounting the CA file in the path we specifie
 
 ### Using k3d-managed registries
 
-!!! info "Not ported yet"
+!!! info "Just ported!"
       The k3d-managed registry is available again as of k3d v4.0.0 (January 2021)
 
 #### Create a dedicated registry together with your cluster

pkg/client/cluster.go

@@ -39,6 +39,7 @@ import (
 	config "github.com/rancher/k3d/v4/pkg/config/v1alpha1"
 	k3drt "github.com/rancher/k3d/v4/pkg/runtimes"
 	"github.com/rancher/k3d/v4/pkg/runtimes/docker"
+	runtimeErr "github.com/rancher/k3d/v4/pkg/runtimes/errors"
 	"github.com/rancher/k3d/v4/pkg/types"
 	k3d "github.com/rancher/k3d/v4/pkg/types"
 	"github.com/rancher/k3d/v4/pkg/util"
@@ -100,7 +101,6 @@ func ClusterRun(ctx context.Context, runtime k3drt.Runtime, clusterConfig *confi
 
 // ClusterPrep takes care of the steps required before creating/starting the cluster containers
 func ClusterPrep(ctx context.Context, runtime k3drt.Runtime, clusterConfig *config.ClusterConfig) error {
-
 	/*
 	 * Set up contexts
 	 * Used for (early) termination (across API boundaries)
@@ -139,6 +139,7 @@ func ClusterPrep(ctx context.Context, runtime k3drt.Runtime, clusterConfig *conf
 
 	// Ensure referenced registries
 	for _, reg := range clusterConfig.ClusterCreateOpts.Registries.Use {
+		log.Debugf("Trying to find registry %s", reg.Host)
 		regNode, err := runtime.GetNode(ctx, &k3d.Node{Name: reg.Host})
 		if err != nil {
 			return fmt.Errorf("Failed to find registry node '%s': %+v", reg.Host, err)
@@ -243,7 +244,6 @@ func ClusterPrepImageVolume(ctx context.Context, runtime k3drt.Runtime, cluster
 	 * Cluster-Wide volumes
 	 * - image volume (for importing images)
 	 */
-
 	imageVolumeName := fmt.Sprintf("%s-%s-images", k3d.DefaultObjectNamePrefix, cluster.Name)
 	if err := runtime.CreateVolume(ctx, imageVolumeName, map[string]string{k3d.LabelClusterName: cluster.Name}); err != nil {
 		log.Errorf("Failed to create image volume '%s' for cluster '%s'", imageVolumeName, cluster.Name)
@@ -468,7 +468,7 @@ ClusterCreatOpts:
 					fmt.Sprintf("WORKER_PROCESSES=%d", len(strings.Split(ports, ","))),
 				},
 				Role:    k3d.LoadBalancerRole,
-				Labels:  k3d.DefaultObjectLabels, // TODO: createLoadBalancer: add more expressive labels
+				Labels:  clusterCreateOpts.GlobalLabels, // TODO: createLoadBalancer: add more expressive labels
 				Network: cluster.Network.Name,
 				Restart: true,
 			}
@@ -491,6 +491,10 @@ ClusterCreatOpts:
 func ClusterDelete(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluster) error {
 
 	log.Infof("Deleting cluster '%s'", cluster.Name)
+	cluster, err := ClusterGet(ctx, runtime, cluster)
+	if err != nil {
+		return err
+	}
 	log.Debugf("Cluster Details: %+v", cluster)
 
 	failed := 0
@@ -507,8 +511,32 @@ func ClusterDelete(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clus
 		if !cluster.Network.External {
 			log.Infof("Deleting cluster network '%s'", cluster.Network.Name)
 			if err := runtime.DeleteNetwork(ctx, cluster.Network.Name); err != nil {
-				if strings.HasSuffix(err.Error(), "active endpoints") {
-					log.Warningf("Failed to delete cluster network '%s' because it's still in use: is there another cluster using it?", cluster.Network.Name)
+				if errors.Is(err, runtimeErr.ErrRuntimeNetworkNotEmpty) { // there are still containers connected to that network
+
+					connectedNodes, err := runtime.GetNodesInNetwork(ctx, cluster.Network.Name) // check, if there are any k3d nodes connected to the cluster
+					if err != nil {
+						log.Warningf("Failed to check cluster network for connected nodes: %+v", err)
+					}
+
+					if len(connectedNodes) > 0 { // there are still k3d-managed containers (aka nodes) connected to the network
+						connectedRegistryNodes := util.FilterNodesByRole(connectedNodes, k3d.RegistryRole)
+						if len(connectedRegistryNodes) == len(connectedNodes) { // only registry node(s) left in the network
+							for _, node := range connectedRegistryNodes {
+								log.Debugf("Disconnecting registry node %s from the network...", node.Name)
+								if err := runtime.DisconnectNodeFromNetwork(ctx, node, cluster.Network.Name); err != nil {
+									log.Warnf("Failed to disconnect registry %s from network %s", node.Name, cluster.Network.Name)
+								} else {
+									if err := runtime.DeleteNetwork(ctx, cluster.Network.Name); err != nil {
+										log.Warningf("Failed to delete cluster network, even after disconnecting registry node(s): %+v", err)
+									}
+								}
+							}
+						} else { // besides the registry node(s), there are still other nodes... maybe they still need a registry
+							log.Debugf("There are some non-registry nodes left in the network")
+						}
+					} else {
+						log.Warningf("Failed to delete cluster network '%s' because it's still in use: is there another cluster using it?", cluster.Network.Name)
+					}
 				} else {
 					log.Warningf("Failed to delete cluster network '%s': '%+v'", cluster.Network.Name, err)
 				}
@@ -535,14 +563,29 @@ func ClusterDelete(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clus
 
 // ClusterList returns a list of all existing clusters
 func ClusterList(ctx context.Context, runtime k3drt.Runtime) ([]*k3d.Cluster, error) {
+	log.Traceln("Listing Clusters...")
 	nodes, err := runtime.GetNodesByLabel(ctx, k3d.DefaultObjectLabels)
 	if err != nil {
 		log.Errorln("Failed to get clusters")
 		return nil, err
 	}
 
+	log.Debugf("Found %d nodes", len(nodes))
+	if log.GetLevel() == log.TraceLevel {
+		for _, node := range nodes {
+			log.Tracef("Found node %s of role %s", node.Name, node.Role)
+		}
+	}
+
 	nodes = NodeFilterByRoles(nodes, k3d.ClusterInternalNodeRoles, k3d.ClusterExternalNodeRoles)
 
+	log.Tracef("Found %d cluster-internal nodes", len(nodes))
+	if log.GetLevel() == log.TraceLevel {
+		for _, node := range nodes {
+			log.Tracef("Found cluster-internal node %s of role %s belonging to cluster %s", node.Name, node.Role, node.Labels[k3d.LabelClusterName])
+		}
+	}
+
 	clusters := []*k3d.Cluster{}
 	// for each node, check, if we can add it to a cluster or add the cluster if it doesn't exist yet
 	for _, node := range nodes {
@@ -570,6 +613,7 @@ func ClusterList(ctx context.Context, runtime k3drt.Runtime) ([]*k3d.Cluster, er
 			log.Warnln(err)
 		}
 	}
+	log.Debugf("Found %d clusters", len(clusters))
 	return clusters, nil
 }
 

pkg/config/transform.go

@@ -222,6 +222,11 @@ func TransformSimpleToClusterConfig(ctx context.Context, runtime runtimes.Runtim
 		GlobalEnv:           []string{},          // empty init
 	}
 
+	// ensure, that we have the default object labels
+	for k, v := range k3d.DefaultObjectLabels {
+		clusterCreateOpts.GlobalLabels[k] = v
+	}
+
 	/*
 	 * Registries
 	 */

pkg/runtimes/containerd/network.go

@@ -41,3 +41,8 @@ func (d Containerd) DeleteNetwork(ctx context.Context, ID string) error {
 func (d Containerd) ConnectNodeToNetwork(ctx context.Context, node *k3d.Node, network string) error {
 	return nil
 }
+
+// DisconnectNodeFromNetwork disconnects a node from a network (u don't say :O)
+func (d Containerd) DisconnectNodeFromNetwork(ctx context.Context, node *k3d.Node, network string) error {
+	return nil
+}

pkg/runtimes/containerd/node.go

@@ -137,3 +137,8 @@ func (d Containerd) ExecInNode(ctx context.Context, node *k3d.Node, cmd []string
 func (d Containerd) ExecInNodeGetLogs(ctx context.Context, node *k3d.Node, cmd []string) (*bufio.Reader, error) {
 	return nil, nil
 }
+
+// GetNodesInNetwork returns all the nodes connected to a given network
+func (d Containerd) GetNodesInNetwork(ctx context.Context, network string) ([]*k3d.Node, error) {
+	return nil, nil
+}

pkg/runtimes/docker/network.go

@@ -25,12 +25,14 @@ import (
 	"context"
 	"fmt"
 	"net"
+	"strings"
 
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/api/types/network"
 	"github.com/docker/docker/client"
 
+	runtimeErr "github.com/rancher/k3d/v4/pkg/runtimes/errors"
 	k3d "github.com/rancher/k3d/v4/pkg/types"
 	log "github.com/sirupsen/logrus"
 )
@@ -94,7 +96,13 @@ func (d Docker) DeleteNetwork(ctx context.Context, ID string) error {
 	defer docker.Close()
 
 	// (3) delete network
-	return docker.NetworkRemove(ctx, ID)
+	if err := docker.NetworkRemove(ctx, ID); err != nil {
+		if strings.HasSuffix(err.Error(), "active endpoints") {
+			return runtimeErr.ErrRuntimeNetworkNotEmpty
+		}
+		return err
+	}
+	return nil
 }
 
 // GetNetwork gets information about a network by its ID
@@ -147,3 +155,29 @@ func (d Docker) ConnectNodeToNetwork(ctx context.Context, node *k3d.Node, networ
 	// connect container to network
 	return docker.NetworkConnect(ctx, networkResource.ID, container.ID, &network.EndpointSettings{})
 }
+
+// DisconnectNodeFromNetwork disconnects a node from a network (u don't say :O)
+func (d Docker) DisconnectNodeFromNetwork(ctx context.Context, node *k3d.Node, networkName string) error {
+	// get container
+	container, err := getNodeContainer(ctx, node)
+	if err != nil {
+		return err
+	}
+
+	// get docker client
+	docker, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
+	if err != nil {
+		log.Errorln("Failed to create docker client")
+		return err
+	}
+	defer docker.Close()
+
+	// get network
+	networkResource, err := GetNetwork(ctx, networkName)
+	if err != nil {
+		log.Errorf("Failed to get network '%s'", networkName)
+		return err
+	}
+
+	return docker.NetworkDisconnect(ctx, networkResource.ID, container.ID, true)
+}

pkg/runtimes/docker/node.go

@@ -25,6 +25,7 @@ package docker
 import (
 	"bufio"
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -33,6 +34,7 @@ import (
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/client"
+	runtimeErr "github.com/rancher/k3d/v4/pkg/runtimes/errors"
 	k3d "github.com/rancher/k3d/v4/pkg/types"
 	log "github.com/sirupsen/logrus"
 )
@@ -59,6 +61,7 @@ func (d Docker) CreateNode(ctx context.Context, node *k3d.Node) error {
 
 // DeleteNode deletes a node
 func (d Docker) DeleteNode(ctx context.Context, nodeSpec *k3d.Node) error {
+	log.Debugf("Deleting node %s ...", nodeSpec.Name)
 	return removeContainer(ctx, nodeSpec.Name)
 }
 
@@ -218,7 +221,7 @@ func (d Docker) GetNode(ctx context.Context, node *k3d.Node) (*k3d.Node, error)
 
 	node, err = TranslateContainerDetailsToNode(containerDetails)
 	if err != nil {
-		log.Errorf("Failed to translate container details for node '%s' to node object", node.Name)
+		log.Errorf("Failed to translate container '%s' to node object", containerDetails.Name)
 		return node, err
 	}
 
@@ -396,10 +399,44 @@ func executeInNode(ctx context.Context, node *k3d.Node, cmd []string) (*types.Hi
 		if execInfo.ExitCode == 0 { // success
 			log.Debugf("Exec process in node '%s' exited with '0'", node.Name)
 			return &execConnection, nil
-		} else { // failed
-			return &execConnection, fmt.Errorf("Exec process in node '%s' failed with exit code '%d'", node.Name, execInfo.ExitCode)
 		}
+		return &execConnection, fmt.Errorf("Exec process in node '%s' failed with exit code '%d'", node.Name, execInfo.ExitCode)
+	}
+}
 
+// GetNodesInNetwork returns all the nodes connected to a given network
+func (d Docker) GetNodesInNetwork(ctx context.Context, network string) ([]*k3d.Node, error) {
+	// create docker client
+	docker, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
+	if err != nil {
+		log.Errorln("Failed to create docker client")
+		return nil, err
+	}
+	defer docker.Close()
+
+	net, err := GetNetwork(ctx, network)
+	if err != nil {
+		return nil, err
 	}
 
+	connectedNodes := []*k3d.Node{}
+
+	// loop over list of containers connected to this cluster and transform them into nodes internally
+	for cID := range net.Containers {
+		containerDetails, err := getContainerDetails(ctx, cID)
+		if err != nil {
+			return nil, err
+		}
+		node, err := TranslateContainerDetailsToNode(containerDetails)
+		if err != nil {
+			if errors.Is(err, runtimeErr.ErrRuntimeContainerUnknown) {
+				log.Tracef("GetNodesInNetwork: inspected non-k3d-managed container %s", containerDetails.Name)
+				continue
+			}
+			return nil, err
+		}
+		connectedNodes = append(connectedNodes, node)
+	}
+
+	return connectedNodes, nil
 }

pkg/runtimes/docker/translate.go

@@ -31,6 +31,7 @@ import (
 	docker "github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/network"
 	"github.com/docker/go-connections/nat"
+	runtimeErr "github.com/rancher/k3d/v4/pkg/runtimes/errors"
 	k3d "github.com/rancher/k3d/v4/pkg/types"
 	log "github.com/sirupsen/logrus"
 
@@ -135,6 +136,22 @@ func TranslateContainerToNode(cont *types.Container) (*k3d.Node, error) {
 // TranslateContainerDetailsToNode translates a docker containerJSON object into a k3d node representation
 func TranslateContainerDetailsToNode(containerDetails types.ContainerJSON) (*k3d.Node, error) {
 
+	// first, make sure, that it's actually a k3d managed container by checking if it has all the default labels
+	for k, v := range k3d.DefaultObjectLabels {
+		log.Tracef("TranslateContainerDetailsToNode: Checking for default object label %s=%s", k, v)
+		found := false
+		for lk, lv := range containerDetails.Config.Labels {
+			if lk == k && lv == v {
+				found = true
+				break
+			}
+		}
+		if !found {
+			log.Debugf("Container %s is missing default label %s=%s in label set %+v", containerDetails.Name, k, v, containerDetails.Config.Labels)
+			return nil, runtimeErr.ErrRuntimeContainerUnknown
+		}
+	}
+
 	// restart -> we only set 'unless-stopped' upon cluster creation
 	restart := false
 	if containerDetails.HostConfig.RestartPolicy.IsAlways() || containerDetails.HostConfig.RestartPolicy.IsUnlessStopped() {

pkg/runtimes/docker/volume.go

@@ -45,11 +45,12 @@ func (d Docker) CreateVolume(ctx context.Context, name string, labels map[string
 	// (1) create volume
 	volumeCreateOptions := volume.VolumeCreateBody{
 		Name:       name,
-		Labels:     k3d.DefaultObjectLabels,
+		Labels:     labels,
 		Driver:     "local", // TODO: allow setting driver + opts
 		DriverOpts: map[string]string{},
 	}
-	for k, v := range labels {
+
+	for k, v := range k3d.DefaultObjectLabels {
 		volumeCreateOptions.Labels[k] = v
 	}
 

pkg/runtimes/errors/errors.go

@@ -0,0 +1,30 @@
+/*
+Copyright © 2020 The k3d Author(s)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+package runtimes
+
+import "errors"
+
+// ErrRuntimeNetworkNotEmpty describes an error that occurs because a network still has containers connected to it (e.g. cannot be deleted)
+var ErrRuntimeNetworkNotEmpty = errors.New("network not empty")
+
+// ErrRuntimeContainerUnknown describes the situation, where we're inspecting a container that's not obviously managed by k3d
+var ErrRuntimeContainerUnknown = errors.New("container not managed by k3d: missing default label(s)")

pkg/runtimes/runtime.go

@@ -56,6 +56,7 @@ type Runtime interface {
 	GetNodesByLabel(context.Context, map[string]string) ([]*k3d.Node, error)
 	GetNode(context.Context, *k3d.Node) (*k3d.Node, error)
 	GetNodeStatus(context.Context, *k3d.Node) (bool, string, error)
+	GetNodesInNetwork(context.Context, string) ([]*k3d.Node, error)
 	CreateNetworkIfNotPresent(context.Context, string) (string, bool, error) // @return NETWORK_NAME, EXISTS, ERROR
 	GetKubeconfig(context.Context, *k3d.Node) (io.ReadCloser, error)
 	DeleteNetwork(context.Context, string) error
@@ -72,7 +73,8 @@ type Runtime interface {
 	CopyToNode(context.Context, string, string, *k3d.Node) error  // @param context, source, destination, node
 	WriteToNode(context.Context, []byte, string, *k3d.Node) error // @param context, content, destination, node
 	GetHostIP(context.Context, string) (net.IP, error)
-	ConnectNodeToNetwork(context.Context, *k3d.Node, string) error // @param context, node, network name
+	ConnectNodeToNetwork(context.Context, *k3d.Node, string) error      // @param context, node, network name
+	DisconnectNodeFromNetwork(context.Context, *k3d.Node, string) error // @param context, node, network name
 }
 
 // GetRuntime checks, if a given name is represented by an implemented k3d runtime and returns it

pkg/util/filter.go

@@ -174,3 +174,14 @@ func FilterNodes(nodes []*k3d.Node, filters []string) ([]*k3d.Node, error) {
 
 	return filteredNodes, nil
 }
+
+// FilterNodesByRole returns a stripped list of nodes which do match the given role
+func FilterNodesByRole(nodes []*k3d.Node, role k3d.Role) []*k3d.Node {
+	filteredNodes := []*k3d.Node{}
+	for _, node := range nodes {
+		if node.Role == role {
+			filteredNodes = append(filteredNodes, node)
+		}
+	}
+	return filteredNodes
+}

version/version.go

@@ -35,7 +35,7 @@ var Version string
 var HelperVersionOverride string
 
 // K3sVersion should contain the latest version tag of k3s (hardcoded at build time)
-var K3sVersion = "rancher/k3s:v1.20.0-k3s2"
+var K3sVersion = "v1.20.0-k3s2"
 
 // GetVersion returns the version for cli, it gets it from "git describe --tags" or returns "dev" when doing simple go build
 func GetVersion() string {