Options for using a specific volume in the registry

New --registry-volume option for using an specific volume in
the registry.

Signed-off-by: Alvaro Saurin <alvaro.saurin@gmail.com>

cli/commands.go

@@ -249,6 +249,7 @@ func CreateCluster(c *cli.Context) error {
 		RegistryEnabled:    c.Bool("enable-registry"),
 		RegistryName:       c.String("registry-name"),
 		RegistryPort:       c.Int("registry-port"),
+		RegistryVolume:     c.String("registry-volume"),
 		ServerArgs:         k3sServerArgs,
 		Volumes:            volumesSpec,
 	}
@@ -379,7 +380,7 @@ func DeleteCluster(c *cli.Context) error {
 			return fmt.Errorf(" Couldn't remove server for cluster %s\n%+v", cluster.name, err)
 		}
 
-		if err := disconnectRegistryFromNetwork(cluster.name); err != nil {
+		if err := disconnectRegistryFromNetwork(cluster.name, c.IsSet("keep-registry-volume")); err != nil {
 			log.Warningf("Couldn't disconnect Registry from network %s\n%+v", cluster.name, err)
 		}
 

cli/registry.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"path"
+	"strconv"
 	"time"
 
 	"github.com/docker/docker/api/types"
@@ -27,11 +28,21 @@ const defaultRegistryPort = 5000
 
 const defaultFullRegistriesPath = "/etc/rancher/k3s/registries.yaml"
 
-var defaultRegistryLabels = map[string]string{
+const defaultRegistryMountPath = "/var/lib/registry"
+
+// default labels assigned to the registry container
+var defaultRegistryContainerLabels = map[string]string{
 	"app":       "k3d",
 	"component": "registry",
 }
 
+// default labels assigned to the registry volume
+var defaultRegistryVolumeLabels = map[string]string{
+	"app":       "k3d",
+	"component": "registry",
+	"managed":   "true",
+}
+
 // NOTE: structs copied from https://github.com/rancher/k3s/blob/master/pkg/agent/templates/registry.go
 //       for avoiding a dependencies nightmare...
 
@@ -137,7 +148,7 @@ func createRegistry(spec ClusterSpec) (string, error) {
 	containerLabels := make(map[string]string)
 
 	// add a standard list of labels to our registry
-	for k, v := range defaultRegistryLabels {
+	for k, v := range defaultRegistryContainerLabels {
 		containerLabels[k] = v
 	}
 	containerLabels["created"] = time.Now().Format("2006-01-02 15:04:05")
@@ -160,6 +171,32 @@ func createRegistry(spec ClusterSpec) (string, error) {
 	}
 
 	spec.Volumes = &Volumes{} // we do not need in the registry any of the volumes used by the other containers
+	if spec.RegistryVolume != "" {
+		vol, err := getVolume(spec.RegistryVolume, map[string]string{})
+		if err != nil {
+			return "", fmt.Errorf(" Couldn't check if volume %s exists: %w", spec.RegistryVolume, err)
+		}
+		if vol != nil {
+			log.Printf("Using existing volume %s for the Registry\n", spec.RegistryVolume)
+		} else {
+			log.Printf("Creating Registry volume %s...\n", spec.RegistryVolume)
+
+			// assign some labels (so we can recognize the volume later on)
+			volLabels := map[string]string{
+				"registry-name": spec.RegistryName,
+				"registry-port": strconv.Itoa(spec.RegistryPort),
+			}
+			for k, v := range defaultRegistryVolumeLabels {
+				volLabels[k] = v
+			}
+			_, err := createVolume(spec.RegistryVolume, volLabels)
+			if err != nil {
+				return "", fmt.Errorf(" Couldn't create volume %s for registry: %w", spec.RegistryVolume, err)
+			}
+		}
+		mount := fmt.Sprintf("%s:%s", spec.RegistryVolume, defaultRegistryMountPath)
+		hostConfig.Binds = []string{mount}
+	}
 
 	// connect the registry to this k3d network
 	networkingConfig := &network.NetworkingConfig{
@@ -200,7 +237,7 @@ func getRegistryContainer() (string, error) {
 	cFilter := filters.NewArgs()
 	cFilter.Add("name", defaultRegistryContainerName)
 	// filter with the standard list of labels of our registry
-	for k, v := range defaultRegistryLabels {
+	for k, v := range defaultRegistryContainerLabels {
 		cFilter.Add("label", fmt.Sprintf("%s=%s", k, v))
 	}
 
@@ -224,7 +261,7 @@ func connectRegistryToNetwork(ID string, networkID string, aliases []string) err
 
 // disconnectRegistryFromNetwork disconnects the Registry from a Network
 // if the Registry container is not connected to any more networks, it is stopped
-func disconnectRegistryFromNetwork(name string) error {
+func disconnectRegistryFromNetwork(name string, keepRegistryVolume bool) error {
 	// disconnect the registry from this cluster's network
 	netName := k3dNetworkName(name)
 	cid, err := getRegistryContainer()
@@ -248,9 +285,33 @@ func disconnectRegistryFromNetwork(name string) error {
 	}
 	if len(networks) == 0 {
 		log.Printf("...Removing the Registry\n")
+		volName, err := getVolumeMountedIn(cid, defaultRegistryMountPath)
+		if err != nil {
+			log.Printf("...warning: could not detect registry volume\n")
+		}
+
 		if err := removeContainer(cid); err != nil {
 			log.Println(err)
 		}
+
+		// check if the volume mounted in /var/lib/registry was managed by us. In that case (and only if
+		// the user does not want to keep the volume alive), delete the registry volume
+		if volName != "" {
+			vol, err := getVolume(volName, defaultRegistryVolumeLabels)
+			if err != nil {
+				return fmt.Errorf(" Couldn't remove volume for registry %s\n%w", defaultRegistryContainerName, err)
+			}
+			if vol != nil {
+				if keepRegistryVolume {
+					log.Printf("...(keeping the Registry volume %s)\n", volName)
+				} else {
+					log.Printf("...Removing the Registry volume %s\n", volName)
+					if err := deleteVolume(volName); err != nil {
+						return fmt.Errorf(" Couldn't remove volume for registry %s\n%w", defaultRegistryContainerName, err)
+					}
+				}
+			}
+		}
 	}
 
 	return nil

cli/types.go

@@ -48,6 +48,7 @@ type ClusterSpec struct {
 	RegistryEnabled    bool
 	RegistryName       string
 	RegistryPort       int
+	RegistryVolume     string
 	ServerArgs         []string
 	Volumes            *Volumes
 }

cli/volume.go

@@ -18,9 +18,8 @@ type Volumes struct {
 	GroupSpecificVolumes map[string][]string
 }
 
-// createImageVolume will create a new docker volume used for storing image tarballs that can be loaded into the clusters
+// createVolume will create a new docker volume
-func createImageVolume(clusterName string) (types.Volume, error) {
+func createVolume(volName string, volLabels map[string]string) (types.Volume, error) {
-
 	var vol types.Volume
 
 	ctx := context.Background()
@@ -29,43 +28,99 @@ func createImageVolume(clusterName string) (types.Volume, error) {
 		return vol, fmt.Errorf(" Couldn't create docker client\n%+v", err)
 	}
 
-	volName := fmt.Sprintf("k3d-%s-images", clusterName)
-
 	volumeCreateOptions := volume.VolumeCreateBody{
-		Name: volName,
+		Name:       volName,
-		Labels: map[string]string{
+		Labels:     volLabels,
-			"app":     "k3d",
-			"cluster": clusterName,
-		},
 		Driver:     "local", //TODO: allow setting driver + opts
 		DriverOpts: map[string]string{},
 	}
 	vol, err = docker.VolumeCreate(ctx, volumeCreateOptions)
 	if err != nil {
-		return vol, fmt.Errorf("failed to create image volume [%s] for cluster [%s]\n%+v", volName, clusterName, err)
+		return vol, fmt.Errorf("failed to create image volume [%s]\n%+v", volName, err)
 	}
 
 	return vol, nil
 }
 
-// deleteImageVolume will delete the volume we created for sharing images with this cluster
+// deleteVolume will delete a volume
-func deleteImageVolume(clusterName string) error {
+func deleteVolume(volName string) error {
-
 	ctx := context.Background()
 	docker, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
 	if err != nil {
 		return fmt.Errorf(" Couldn't create docker client\n%+v", err)
 	}
 
-	volName := fmt.Sprintf("k3d-%s-images", clusterName)
-
 	if err = docker.VolumeRemove(ctx, volName, true); err != nil {
-		return fmt.Errorf(" Couldn't remove volume [%s] for cluster [%s]\n%+v", volName, clusterName, err)
+		return fmt.Errorf(" Couldn't remove volume [%s]\n%+v", volName, err)
 	}
 
 	return nil
 }
 
+// getVolume checks if a docker volume exists. The volume can be specified with a name and/or some labels.
+func getVolume(volName string, volLabels map[string]string) (*types.Volume, error) {
+	ctx := context.Background()
+
+	docker, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
+	if err != nil {
+		return nil, fmt.Errorf(" Couldn't create docker client: %w", err)
+	}
+
+	vFilter := filters.NewArgs()
+	if volName != "" {
+		vFilter.Add("name", volName)
+	}
+	for k, v := range volLabels {
+		vFilter.Add("label", fmt.Sprintf("%s=%s", k, v))
+	}
+
+	volumes, err := docker.VolumeList(ctx, vFilter)
+	if err != nil {
+		return nil, fmt.Errorf(" Couldn't list volumes: %w", err)
+	}
+	if len(volumes.Volumes) == 0 {
+		return nil, nil
+	}
+	return volumes.Volumes[0], nil
+}
+
+// getVolumeMountedIn gets the volume that is mounted in some container in some path
+func getVolumeMountedIn(ID string, path string) (string, error) {
+	ctx := context.Background()
+
+	docker, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
+	if err != nil {
+		return "", fmt.Errorf(" Couldn't create docker client: %w", err)
+	}
+
+	c, err := docker.ContainerInspect(ctx, ID)
+	if err != nil {
+		return "", fmt.Errorf(" Couldn't inspect container %s: %w", ID, err)
+	}
+	for _, mount := range c.Mounts {
+		if mount.Destination == path {
+			return mount.Name, nil
+		}
+	}
+	return "", nil
+}
+
+// createImageVolume will create a new docker volume used for storing image tarballs that can be loaded into the clusters
+func createImageVolume(clusterName string) (types.Volume, error) {
+	volName := fmt.Sprintf("k3d-%s-images", clusterName)
+	volLabels := map[string]string{
+		"app":     "k3d",
+		"cluster": clusterName,
+	}
+	return createVolume(volName, volLabels)
+}
+
+// deleteImageVolume will delete the volume we created for sharing images with this cluster
+func deleteImageVolume(clusterName string) error {
+	volName := fmt.Sprintf("k3d-%s-images", clusterName)
+	return deleteVolume(volName)
+}
+
 // getImageVolume returns the docker volume object representing the imagevolume for the cluster
 func getImageVolume(clusterName string) (types.Volume, error) {
 	var vol types.Volume

main.go

@@ -136,6 +136,10 @@ func main() {
 					Value: defaultRegistryPort,
 					Usage: "Port of the local registry container",
 				},
+				cli.StringFlag{
+					Name:  "registry-volume",
+					Usage: "Use a specific volume for the registry storage (will be created if not existing)",
+				},
 				cli.StringFlag{
 					Name:  "registries-file",
 					Usage: "registries.yaml config file",
@@ -219,6 +223,10 @@ func main() {
 					Name:  "prune",
 					Usage: "Disconnect any other non-k3d containers in the network before deleting the cluster",
 				},
+				cli.BoolFlag{
+					Name:  "keep-registry-volume",
+					Usage: "Do not delete the registry volume",
+				},
 			},
 			Action: run.DeleteCluster,
 		},

tests/02-registry.sh

@@ -33,7 +33,7 @@ else
 fi
 
 info "Creating two clusters (with a registry)..."
-$EXE create --wait 60 --name "c1" --api-port 6443 --enable-registry || failed "could not create cluster c1"
+$EXE create --wait 60 --name "c1" --api-port 6443 --enable-registry --registry-volume "reg-vol" || failed "could not create cluster c1"
 $EXE create --wait 60 --name "c2" --api-port 6444 --enable-registry --registries-file "$REGISTRIES_YAML" || failed "could not create cluster c2"
 
 check_k3d_clusters "c1" "c2" || failed "error checking cluster"
@@ -43,10 +43,14 @@ check_registry || abort "local registry not available at $REGISTRY"
 passed "Local registry running at $REGISTRY"
 
 info "Deleting c1 cluster: the registry should remain..."
-$EXE delete --name "c1" || failed "could not delete the cluster c1"
+$EXE delete --name "c1" --keep-registry-volume || failed "could not delete the cluster c1"
 check_registry || abort "local registry not available at $REGISTRY after removing c1"
 passed "The local registry is still running"
 
+info "Checking that the reg-vol still exists after removing c1"
+check_volume_exists "reg-vol" || abort "the registry volume 'reg-vol' does not seem to exist"
+passed "reg-vol still exists"
+
 info "Pulling a test image..."
 docker pull $TEST_IMAGE
 docker tag $TEST_IMAGE $REGISTRY/$TEST_IMAGE
@@ -84,8 +88,18 @@ kubectl --kubeconfig=$($EXE get-kubeconfig --name "c2") wait --for=condition=ava
 passed "Local registry seems to be usable"
 
 info "Deleting c2 cluster: the registry should be removed now..."
-$EXE delete --name "c2" || failed "could not delete the cluster c2"
+$EXE delete --name "c2" --keep-registry-volume || failed "could not delete the cluster c2"
 check_registry && abort "local registry still running at $REGISTRY"
 passed "The local registry has been removed"
 
+info "Creating a new clusters that uses a registry with an existsing 'reg-vol' volume..."
+check_volume_exists "reg-vol" || abort "the registry volume 'reg-vol' does not exist"
+$EXE create --wait 60 --name "c3" --api-port 6445 --enable-registry --registry-volume "reg-vol" || failed "could not create cluster c3"
+
+info "Deleting c3 cluster: the registry should be removed and, this time, the volume too..."
+$EXE delete --name "c3" || failed "could not delete the cluster c3"
+check_volume_exists "reg-vol" && abort "the registry volume 'reg-vol' still exists"
+passed "'reg-vol' has been removed"
+
 exit 0
+

tests/common.sh

@@ -83,3 +83,6 @@ check_registry() {
   check_url $REGISTRY/v2/_catalog
 }
 
+check_volume_exists() {
+  docker volume inspect "$1" >/dev/null 2>&1
+}