mirrors/k3d
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Options for using a specific volume in the registry

Browse Source 
New --registry-volume option for using an specific volume in
the registry.

Signed-off-by: Alvaro Saurin <alvaro.saurin@gmail.com>
This commit is contained in:
Alvaro Saurin 2020-03-02 13:04:48 +01:00
parent 0ec51e1318
commit 5b7ec7e29d
No known key found for this signature in database
GPG Key ID: 7B38F4A5C322A955
7 changed files with 168 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
cli/commands.go
View File

@ -249,6 +249,7 @@ func CreateCluster(c *cli.Context) error {
RegistryEnabled: c.Bool("enable-registry"),
RegistryName: c.String("registry-name"),
RegistryPort: c.Int("registry-port"),
RegistryVolume: c.String("registry-volume"),
ServerArgs: k3sServerArgs,
Volumes: volumesSpec,
}
@ -379,7 +380,7 @@ func DeleteCluster(c *cli.Context) error {
return fmt.Errorf(" Couldn't remove server for cluster %s\n%+v", cluster.name, err)
}
if err := disconnectRegistryFromNetwork(cluster.name); err != nil {
if err := disconnectRegistryFromNetwork(cluster.name, c.IsSet("keep-registry-volume")); err != nil {
log.Warningf("Couldn't disconnect Registry from network %s\n%+v", cluster.name, err)
}

69
cli/registry.go
View File

@ -5,6 +5,7 @@ import (
"fmt"
"io/ioutil"
"path"
"strconv"
"time"
"github.com/docker/docker/api/types"
@ -27,11 +28,21 @@ const defaultRegistryPort = 5000
const defaultFullRegistriesPath = "/etc/rancher/k3s/registries.yaml"
var defaultRegistryLabels = map[string]string{
const defaultRegistryMountPath = "/var/lib/registry"
// default labels assigned to the registry container
var defaultRegistryContainerLabels = map[string]string{
"app": "k3d",
"component": "registry",
}
// default labels assigned to the registry volume
var defaultRegistryVolumeLabels = map[string]string{
"app": "k3d",
"component": "registry",
"managed": "true",
}
// NOTE: structs copied from https://github.com/rancher/k3s/blob/master/pkg/agent/templates/registry.go
// for avoiding a dependencies nightmare...
@ -137,7 +148,7 @@ func createRegistry(spec ClusterSpec) (string, error) {
containerLabels := make(map[string]string)
// add a standard list of labels to our registry
for k, v := range defaultRegistryLabels {
for k, v := range defaultRegistryContainerLabels {
containerLabels[k] = v
}
containerLabels["created"] = time.Now().Format("2006-01-02 15:04:05")
@ -160,6 +171,32 @@ func createRegistry(spec ClusterSpec) (string, error) {
}
spec.Volumes = &Volumes{} // we do not need in the registry any of the volumes used by the other containers
if spec.RegistryVolume != "" {
vol, err := getVolume(spec.RegistryVolume, map[string]string{})
if err != nil {
return "", fmt.Errorf(" Couldn't check if volume %s exists: %w", spec.RegistryVolume, err)
}
if vol != nil {
log.Printf("Using existing volume %s for the Registry\n", spec.RegistryVolume)
} else {
log.Printf("Creating Registry volume %s...\n", spec.RegistryVolume)
// assign some labels (so we can recognize the volume later on)
volLabels := map[string]string{
"registry-name": spec.RegistryName,
"registry-port": strconv.Itoa(spec.RegistryPort),
}
for k, v := range defaultRegistryVolumeLabels {
volLabels[k] = v
}
_, err := createVolume(spec.RegistryVolume, volLabels)
if err != nil {
return "", fmt.Errorf(" Couldn't create volume %s for registry: %w", spec.RegistryVolume, err)
}
}
mount := fmt.Sprintf("%s:%s", spec.RegistryVolume, defaultRegistryMountPath)
hostConfig.Binds = []string{mount}
}
// connect the registry to this k3d network
networkingConfig := &network.NetworkingConfig{
@ -200,7 +237,7 @@ func getRegistryContainer() (string, error) {
cFilter := filters.NewArgs()
cFilter.Add("name", defaultRegistryContainerName)
// filter with the standard list of labels of our registry
for k, v := range defaultRegistryLabels {
for k, v := range defaultRegistryContainerLabels {
cFilter.Add("label", fmt.Sprintf("%s=%s", k, v))
}
@ -224,7 +261,7 @@ func connectRegistryToNetwork(ID string, networkID string, aliases []string) err
// disconnectRegistryFromNetwork disconnects the Registry from a Network
// if the Registry container is not connected to any more networks, it is stopped
func disconnectRegistryFromNetwork(name string) error {
func disconnectRegistryFromNetwork(name string, keepRegistryVolume bool) error {
// disconnect the registry from this cluster's network
netName := k3dNetworkName(name)
cid, err := getRegistryContainer()
@ -248,9 +285,33 @@ func disconnectRegistryFromNetwork(name string) error {
}
if len(networks) == 0 {
log.Printf("...Removing the Registry\n")
volName, err := getVolumeMountedIn(cid, defaultRegistryMountPath)
if err != nil {
log.Printf("...warning: could not detect registry volume\n")
}
if err := removeContainer(cid); err != nil {
log.Println(err)
}
// check if the volume mounted in /var/lib/registry was managed by us. In that case (and only if
// the user does not want to keep the volume alive), delete the registry volume
if volName != "" {
vol, err := getVolume(volName, defaultRegistryVolumeLabels)
if err != nil {
return fmt.Errorf(" Couldn't remove volume for registry %s\n%w", defaultRegistryContainerName, err)
}
if vol != nil {
if keepRegistryVolume {
log.Printf("...(keeping the Registry volume %s)\n", volName)
} else {
log.Printf("...Removing the Registry volume %s\n", volName)
if err := deleteVolume(volName); err != nil {
return fmt.Errorf(" Couldn't remove volume for registry %s\n%w", defaultRegistryContainerName, err)
}
}
}
}
}
return nil

1
cli/types.go
View File

@ -48,6 +48,7 @@ type ClusterSpec struct {
RegistryEnabled bool
RegistryName string
RegistryPort int
RegistryVolume string
ServerArgs []string
Volumes *Volumes
}

89
cli/volume.go
View File

@ -18,9 +18,8 @@ type Volumes struct {
GroupSpecificVolumes map[string][]string
}
// createImageVolume will create a new docker volume used for storing image tarballs that can be loaded into the clusters
func createImageVolume(clusterName string) (types.Volume, error) {
// createVolume will create a new docker volume
func createVolume(volName string, volLabels map[string]string) (types.Volume, error) {
var vol types.Volume
ctx := context.Background()
@ -29,43 +28,99 @@ func createImageVolume(clusterName string) (types.Volume, error) {
return vol, fmt.Errorf(" Couldn't create docker client\n%+v", err)
}
volName := fmt.Sprintf("k3d-%s-images", clusterName)
volumeCreateOptions := volume.VolumeCreateBody{
Name: volName,
Labels: map[string]string{
"app": "k3d",
"cluster": clusterName,
},
Name: volName,
Labels: volLabels,
Driver: "local", //TODO: allow setting driver + opts
DriverOpts: map[string]string{},
}
vol, err = docker.VolumeCreate(ctx, volumeCreateOptions)
if err != nil {
return vol, fmt.Errorf("failed to create image volume [%s] for cluster [%s]\n%+v", volName, clusterName, err)
return vol, fmt.Errorf("failed to create image volume [%s]\n%+v", volName, err)
}
return vol, nil
}
// deleteImageVolume will delete the volume we created for sharing images with this cluster
func deleteImageVolume(clusterName string) error {
// deleteVolume will delete a volume
func deleteVolume(volName string) error {
ctx := context.Background()
docker, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
if err != nil {
return fmt.Errorf(" Couldn't create docker client\n%+v", err)
}
volName := fmt.Sprintf("k3d-%s-images", clusterName)
if err = docker.VolumeRemove(ctx, volName, true); err != nil {
return fmt.Errorf(" Couldn't remove volume [%s] for cluster [%s]\n%+v", volName, clusterName, err)
return fmt.Errorf(" Couldn't remove volume [%s]\n%+v", volName, err)
}
return nil
}
// getVolume checks if a docker volume exists. The volume can be specified with a name and/or some labels.
func getVolume(volName string, volLabels map[string]string) (*types.Volume, error) {
ctx := context.Background()
docker, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
if err != nil {
return nil, fmt.Errorf(" Couldn't create docker client: %w", err)
}
vFilter := filters.NewArgs()
if volName != "" {
vFilter.Add("name", volName)
}
for k, v := range volLabels {
vFilter.Add("label", fmt.Sprintf("%s=%s", k, v))
}
volumes, err := docker.VolumeList(ctx, vFilter)
if err != nil {
return nil, fmt.Errorf(" Couldn't list volumes: %w", err)
}
if len(volumes.Volumes) == 0 {
return nil, nil
}
return volumes.Volumes[0], nil
}
// getVolumeMountedIn gets the volume that is mounted in some container in some path
func getVolumeMountedIn(ID string, path string) (string, error) {
ctx := context.Background()
docker, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
if err != nil {
return "", fmt.Errorf(" Couldn't create docker client: %w", err)
}
c, err := docker.ContainerInspect(ctx, ID)
if err != nil {
return "", fmt.Errorf(" Couldn't inspect container %s: %w", ID, err)
}
for _, mount := range c.Mounts {
if mount.Destination == path {
return mount.Name, nil
}
}
return "", nil
}
// createImageVolume will create a new docker volume used for storing image tarballs that can be loaded into the clusters
func createImageVolume(clusterName string) (types.Volume, error) {
volName := fmt.Sprintf("k3d-%s-images", clusterName)
volLabels := map[string]string{
"app": "k3d",
"cluster": clusterName,
}
return createVolume(volName, volLabels)
}
// deleteImageVolume will delete the volume we created for sharing images with this cluster
func deleteImageVolume(clusterName string) error {
volName := fmt.Sprintf("k3d-%s-images", clusterName)
return deleteVolume(volName)
}
// getImageVolume returns the docker volume object representing the imagevolume for the cluster
func getImageVolume(clusterName string) (types.Volume, error) {
var vol types.Volume

8
main.go
View File

@ -136,6 +136,10 @@ func main() {
Value: defaultRegistryPort,
Usage: "Port of the local registry container",
},
cli.StringFlag{
Name: "registry-volume",
Usage: "Use a specific volume for the registry storage (will be created if not existing)",
},
cli.StringFlag{
Name: "registries-file",
Usage: "registries.yaml config file",
@ -219,6 +223,10 @@ func main() {
Name: "prune",
Usage: "Disconnect any other non-k3d containers in the network before deleting the cluster",
},
cli.BoolFlag{
Name: "keep-registry-volume",
Usage: "Do not delete the registry volume",
},
},
Action: run.DeleteCluster,
},

20
tests/02-registry.sh
View File

@ -33,7 +33,7 @@ else
fi
info "Creating two clusters (with a registry)..."
$EXE create --wait 60 --name "c1" --api-port 6443 --enable-registry || failed "could not create cluster c1"
$EXE create --wait 60 --name "c1" --api-port 6443 --enable-registry --registry-volume "reg-vol" || failed "could not create cluster c1"
$EXE create --wait 60 --name "c2" --api-port 6444 --enable-registry --registries-file "$REGISTRIES_YAML" || failed "could not create cluster c2"
check_k3d_clusters "c1" "c2" || failed "error checking cluster"
@ -43,10 +43,14 @@ check_registry || abort "local registry not available at $REGISTRY"
passed "Local registry running at $REGISTRY"
info "Deleting c1 cluster: the registry should remain..."
$EXE delete --name "c1" || failed "could not delete the cluster c1"
$EXE delete --name "c1" --keep-registry-volume || failed "could not delete the cluster c1"
check_registry || abort "local registry not available at $REGISTRY after removing c1"
passed "The local registry is still running"
info "Checking that the reg-vol still exists after removing c1"
check_volume_exists "reg-vol" || abort "the registry volume 'reg-vol' does not seem to exist"
passed "reg-vol still exists"
info "Pulling a test image..."
docker pull $TEST_IMAGE
docker tag $TEST_IMAGE $REGISTRY/$TEST_IMAGE
@ -84,8 +88,18 @@ kubectl --kubeconfig=$($EXE get-kubeconfig --name "c2") wait --for=condition=ava
passed "Local registry seems to be usable"
info "Deleting c2 cluster: the registry should be removed now..."
$EXE delete --name "c2" || failed "could not delete the cluster c2"
$EXE delete --name "c2" --keep-registry-volume || failed "could not delete the cluster c2"
check_registry && abort "local registry still running at $REGISTRY"
passed "The local registry has been removed"
info "Creating a new clusters that uses a registry with an existsing 'reg-vol' volume..."
check_volume_exists "reg-vol" || abort "the registry volume 'reg-vol' does not exist"
$EXE create --wait 60 --name "c3" --api-port 6445 --enable-registry --registry-volume "reg-vol" || failed "could not create cluster c3"
info "Deleting c3 cluster: the registry should be removed and, this time, the volume too..."
$EXE delete --name "c3" || failed "could not delete the cluster c3"
check_volume_exists "reg-vol" && abort "the registry volume 'reg-vol' still exists"
passed "'reg-vol' has been removed"
exit 0

3
tests/common.sh
View File

@ -83,3 +83,6 @@ check_registry() {
check_url $REGISTRY/v2/_catalog
}
check_volume_exists() {
docker volume inspect "$1" >/dev/null 2>&1
}