From 5b7ec7e29d02baf3467596752925aa5570e74d0c Mon Sep 17 00:00:00 2001 From: Alvaro Saurin Date: Mon, 2 Mar 2020 13:04:48 +0100 Subject: [PATCH] Options for using a specific volume in the registry New --registry-volume option for using an specific volume in the registry. Signed-off-by: Alvaro Saurin --- cli/commands.go | 3 +- cli/registry.go | 69 ++++++++++++++++++++++++++++++++-- cli/types.go | 1 + cli/volume.go | 89 +++++++++++++++++++++++++++++++++++--------- main.go | 8 ++++ tests/02-registry.sh | 20 ++++++++-- tests/common.sh | 3 ++ 7 files changed, 168 insertions(+), 25 deletions(-) diff --git a/cli/commands.go b/cli/commands.go index f3298549..b8e7344c 100644 --- a/cli/commands.go +++ b/cli/commands.go @@ -249,6 +249,7 @@ func CreateCluster(c *cli.Context) error { RegistryEnabled: c.Bool("enable-registry"), RegistryName: c.String("registry-name"), RegistryPort: c.Int("registry-port"), + RegistryVolume: c.String("registry-volume"), ServerArgs: k3sServerArgs, Volumes: volumesSpec, } @@ -379,7 +380,7 @@ func DeleteCluster(c *cli.Context) error { return fmt.Errorf(" Couldn't remove server for cluster %s\n%+v", cluster.name, err) } - if err := disconnectRegistryFromNetwork(cluster.name); err != nil { + if err := disconnectRegistryFromNetwork(cluster.name, c.IsSet("keep-registry-volume")); err != nil { log.Warningf("Couldn't disconnect Registry from network %s\n%+v", cluster.name, err) } diff --git a/cli/registry.go b/cli/registry.go index 7431d1ba..c6ac56db 100644 --- a/cli/registry.go +++ b/cli/registry.go @@ -5,6 +5,7 @@ import ( "fmt" "io/ioutil" "path" + "strconv" "time" "github.com/docker/docker/api/types" @@ -27,11 +28,21 @@ const defaultRegistryPort = 5000 const defaultFullRegistriesPath = "/etc/rancher/k3s/registries.yaml" -var defaultRegistryLabels = map[string]string{ +const defaultRegistryMountPath = "/var/lib/registry" + +// default labels assigned to the registry container +var defaultRegistryContainerLabels = map[string]string{ "app": "k3d", "component": "registry", } +// default labels assigned to the registry volume +var defaultRegistryVolumeLabels = map[string]string{ + "app": "k3d", + "component": "registry", + "managed": "true", +} + // NOTE: structs copied from https://github.com/rancher/k3s/blob/master/pkg/agent/templates/registry.go // for avoiding a dependencies nightmare... @@ -137,7 +148,7 @@ func createRegistry(spec ClusterSpec) (string, error) { containerLabels := make(map[string]string) // add a standard list of labels to our registry - for k, v := range defaultRegistryLabels { + for k, v := range defaultRegistryContainerLabels { containerLabels[k] = v } containerLabels["created"] = time.Now().Format("2006-01-02 15:04:05") @@ -160,6 +171,32 @@ func createRegistry(spec ClusterSpec) (string, error) { } spec.Volumes = &Volumes{} // we do not need in the registry any of the volumes used by the other containers + if spec.RegistryVolume != "" { + vol, err := getVolume(spec.RegistryVolume, map[string]string{}) + if err != nil { + return "", fmt.Errorf(" Couldn't check if volume %s exists: %w", spec.RegistryVolume, err) + } + if vol != nil { + log.Printf("Using existing volume %s for the Registry\n", spec.RegistryVolume) + } else { + log.Printf("Creating Registry volume %s...\n", spec.RegistryVolume) + + // assign some labels (so we can recognize the volume later on) + volLabels := map[string]string{ + "registry-name": spec.RegistryName, + "registry-port": strconv.Itoa(spec.RegistryPort), + } + for k, v := range defaultRegistryVolumeLabels { + volLabels[k] = v + } + _, err := createVolume(spec.RegistryVolume, volLabels) + if err != nil { + return "", fmt.Errorf(" Couldn't create volume %s for registry: %w", spec.RegistryVolume, err) + } + } + mount := fmt.Sprintf("%s:%s", spec.RegistryVolume, defaultRegistryMountPath) + hostConfig.Binds = []string{mount} + } // connect the registry to this k3d network networkingConfig := &network.NetworkingConfig{ @@ -200,7 +237,7 @@ func getRegistryContainer() (string, error) { cFilter := filters.NewArgs() cFilter.Add("name", defaultRegistryContainerName) // filter with the standard list of labels of our registry - for k, v := range defaultRegistryLabels { + for k, v := range defaultRegistryContainerLabels { cFilter.Add("label", fmt.Sprintf("%s=%s", k, v)) } @@ -224,7 +261,7 @@ func connectRegistryToNetwork(ID string, networkID string, aliases []string) err // disconnectRegistryFromNetwork disconnects the Registry from a Network // if the Registry container is not connected to any more networks, it is stopped -func disconnectRegistryFromNetwork(name string) error { +func disconnectRegistryFromNetwork(name string, keepRegistryVolume bool) error { // disconnect the registry from this cluster's network netName := k3dNetworkName(name) cid, err := getRegistryContainer() @@ -248,9 +285,33 @@ func disconnectRegistryFromNetwork(name string) error { } if len(networks) == 0 { log.Printf("...Removing the Registry\n") + volName, err := getVolumeMountedIn(cid, defaultRegistryMountPath) + if err != nil { + log.Printf("...warning: could not detect registry volume\n") + } + if err := removeContainer(cid); err != nil { log.Println(err) } + + // check if the volume mounted in /var/lib/registry was managed by us. In that case (and only if + // the user does not want to keep the volume alive), delete the registry volume + if volName != "" { + vol, err := getVolume(volName, defaultRegistryVolumeLabels) + if err != nil { + return fmt.Errorf(" Couldn't remove volume for registry %s\n%w", defaultRegistryContainerName, err) + } + if vol != nil { + if keepRegistryVolume { + log.Printf("...(keeping the Registry volume %s)\n", volName) + } else { + log.Printf("...Removing the Registry volume %s\n", volName) + if err := deleteVolume(volName); err != nil { + return fmt.Errorf(" Couldn't remove volume for registry %s\n%w", defaultRegistryContainerName, err) + } + } + } + } } return nil diff --git a/cli/types.go b/cli/types.go index a8020835..dac7e1c9 100644 --- a/cli/types.go +++ b/cli/types.go @@ -48,6 +48,7 @@ type ClusterSpec struct { RegistryEnabled bool RegistryName string RegistryPort int + RegistryVolume string ServerArgs []string Volumes *Volumes } diff --git a/cli/volume.go b/cli/volume.go index a172ab20..eb45ff5d 100644 --- a/cli/volume.go +++ b/cli/volume.go @@ -18,9 +18,8 @@ type Volumes struct { GroupSpecificVolumes map[string][]string } -// createImageVolume will create a new docker volume used for storing image tarballs that can be loaded into the clusters -func createImageVolume(clusterName string) (types.Volume, error) { - +// createVolume will create a new docker volume +func createVolume(volName string, volLabels map[string]string) (types.Volume, error) { var vol types.Volume ctx := context.Background() @@ -29,43 +28,99 @@ func createImageVolume(clusterName string) (types.Volume, error) { return vol, fmt.Errorf(" Couldn't create docker client\n%+v", err) } - volName := fmt.Sprintf("k3d-%s-images", clusterName) - volumeCreateOptions := volume.VolumeCreateBody{ - Name: volName, - Labels: map[string]string{ - "app": "k3d", - "cluster": clusterName, - }, + Name: volName, + Labels: volLabels, Driver: "local", //TODO: allow setting driver + opts DriverOpts: map[string]string{}, } vol, err = docker.VolumeCreate(ctx, volumeCreateOptions) if err != nil { - return vol, fmt.Errorf("failed to create image volume [%s] for cluster [%s]\n%+v", volName, clusterName, err) + return vol, fmt.Errorf("failed to create image volume [%s]\n%+v", volName, err) } return vol, nil } -// deleteImageVolume will delete the volume we created for sharing images with this cluster -func deleteImageVolume(clusterName string) error { - +// deleteVolume will delete a volume +func deleteVolume(volName string) error { ctx := context.Background() docker, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) if err != nil { return fmt.Errorf(" Couldn't create docker client\n%+v", err) } - volName := fmt.Sprintf("k3d-%s-images", clusterName) - if err = docker.VolumeRemove(ctx, volName, true); err != nil { - return fmt.Errorf(" Couldn't remove volume [%s] for cluster [%s]\n%+v", volName, clusterName, err) + return fmt.Errorf(" Couldn't remove volume [%s]\n%+v", volName, err) } return nil } +// getVolume checks if a docker volume exists. The volume can be specified with a name and/or some labels. +func getVolume(volName string, volLabels map[string]string) (*types.Volume, error) { + ctx := context.Background() + + docker, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) + if err != nil { + return nil, fmt.Errorf(" Couldn't create docker client: %w", err) + } + + vFilter := filters.NewArgs() + if volName != "" { + vFilter.Add("name", volName) + } + for k, v := range volLabels { + vFilter.Add("label", fmt.Sprintf("%s=%s", k, v)) + } + + volumes, err := docker.VolumeList(ctx, vFilter) + if err != nil { + return nil, fmt.Errorf(" Couldn't list volumes: %w", err) + } + if len(volumes.Volumes) == 0 { + return nil, nil + } + return volumes.Volumes[0], nil +} + +// getVolumeMountedIn gets the volume that is mounted in some container in some path +func getVolumeMountedIn(ID string, path string) (string, error) { + ctx := context.Background() + + docker, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) + if err != nil { + return "", fmt.Errorf(" Couldn't create docker client: %w", err) + } + + c, err := docker.ContainerInspect(ctx, ID) + if err != nil { + return "", fmt.Errorf(" Couldn't inspect container %s: %w", ID, err) + } + for _, mount := range c.Mounts { + if mount.Destination == path { + return mount.Name, nil + } + } + return "", nil +} + +// createImageVolume will create a new docker volume used for storing image tarballs that can be loaded into the clusters +func createImageVolume(clusterName string) (types.Volume, error) { + volName := fmt.Sprintf("k3d-%s-images", clusterName) + volLabels := map[string]string{ + "app": "k3d", + "cluster": clusterName, + } + return createVolume(volName, volLabels) +} + +// deleteImageVolume will delete the volume we created for sharing images with this cluster +func deleteImageVolume(clusterName string) error { + volName := fmt.Sprintf("k3d-%s-images", clusterName) + return deleteVolume(volName) +} + // getImageVolume returns the docker volume object representing the imagevolume for the cluster func getImageVolume(clusterName string) (types.Volume, error) { var vol types.Volume diff --git a/main.go b/main.go index e1f46742..a8c59f97 100644 --- a/main.go +++ b/main.go @@ -136,6 +136,10 @@ func main() { Value: defaultRegistryPort, Usage: "Port of the local registry container", }, + cli.StringFlag{ + Name: "registry-volume", + Usage: "Use a specific volume for the registry storage (will be created if not existing)", + }, cli.StringFlag{ Name: "registries-file", Usage: "registries.yaml config file", @@ -219,6 +223,10 @@ func main() { Name: "prune", Usage: "Disconnect any other non-k3d containers in the network before deleting the cluster", }, + cli.BoolFlag{ + Name: "keep-registry-volume", + Usage: "Do not delete the registry volume", + }, }, Action: run.DeleteCluster, }, diff --git a/tests/02-registry.sh b/tests/02-registry.sh index 78ae1711..30a4afc6 100755 --- a/tests/02-registry.sh +++ b/tests/02-registry.sh @@ -33,7 +33,7 @@ else fi info "Creating two clusters (with a registry)..." -$EXE create --wait 60 --name "c1" --api-port 6443 --enable-registry || failed "could not create cluster c1" +$EXE create --wait 60 --name "c1" --api-port 6443 --enable-registry --registry-volume "reg-vol" || failed "could not create cluster c1" $EXE create --wait 60 --name "c2" --api-port 6444 --enable-registry --registries-file "$REGISTRIES_YAML" || failed "could not create cluster c2" check_k3d_clusters "c1" "c2" || failed "error checking cluster" @@ -43,10 +43,14 @@ check_registry || abort "local registry not available at $REGISTRY" passed "Local registry running at $REGISTRY" info "Deleting c1 cluster: the registry should remain..." -$EXE delete --name "c1" || failed "could not delete the cluster c1" +$EXE delete --name "c1" --keep-registry-volume || failed "could not delete the cluster c1" check_registry || abort "local registry not available at $REGISTRY after removing c1" passed "The local registry is still running" +info "Checking that the reg-vol still exists after removing c1" +check_volume_exists "reg-vol" || abort "the registry volume 'reg-vol' does not seem to exist" +passed "reg-vol still exists" + info "Pulling a test image..." docker pull $TEST_IMAGE docker tag $TEST_IMAGE $REGISTRY/$TEST_IMAGE @@ -84,8 +88,18 @@ kubectl --kubeconfig=$($EXE get-kubeconfig --name "c2") wait --for=condition=ava passed "Local registry seems to be usable" info "Deleting c2 cluster: the registry should be removed now..." -$EXE delete --name "c2" || failed "could not delete the cluster c2" +$EXE delete --name "c2" --keep-registry-volume || failed "could not delete the cluster c2" check_registry && abort "local registry still running at $REGISTRY" passed "The local registry has been removed" +info "Creating a new clusters that uses a registry with an existsing 'reg-vol' volume..." +check_volume_exists "reg-vol" || abort "the registry volume 'reg-vol' does not exist" +$EXE create --wait 60 --name "c3" --api-port 6445 --enable-registry --registry-volume "reg-vol" || failed "could not create cluster c3" + +info "Deleting c3 cluster: the registry should be removed and, this time, the volume too..." +$EXE delete --name "c3" || failed "could not delete the cluster c3" +check_volume_exists "reg-vol" && abort "the registry volume 'reg-vol' still exists" +passed "'reg-vol' has been removed" + exit 0 + diff --git a/tests/common.sh b/tests/common.sh index 94009aa9..aec70553 100755 --- a/tests/common.sh +++ b/tests/common.sh @@ -83,3 +83,6 @@ check_registry() { check_url $REGISTRY/v2/_catalog } +check_volume_exists() { + docker volume inspect "$1" >/dev/null 2>&1 +}