Merge pull request #91 from rancher/feature/import-images-helper

[Feature] import images from docker daemon into k3d using a helper container

cli/cluster.go

@@ -14,8 +14,7 @@ import (
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/client"
-
+	homedir "github.com/mitchellh/go-homedir"
-	"github.com/mitchellh/go-homedir"
 	"github.com/olekukonko/tablewriter"
 )
 
@@ -68,6 +67,10 @@ func createClusterDir(name string) {
 	if err := createDirIfNotExists(clusterPath); err != nil {
 		log.Fatalf("ERROR: couldn't create cluster directory [%s] -> %+v", clusterPath, err)
 	}
+	// create subdir for sharing container images
+	if err := createDirIfNotExists(clusterPath + "/images"); err != nil {
+		log.Fatalf("ERROR: couldn't create cluster sub-directory [%s] -> %+v", clusterPath+"/images", err)
+	}
 }
 
 // deleteClusterDir contrary to createClusterDir, this deletes the cluster directory under $HOME/.config/k3d/<cluster_name>

cli/commands.go

@@ -100,7 +100,7 @@ func CreateCluster(c *cli.Context) error {
 	if c.IsSet("port") {
 		log.Println("INFO: As of v2.0.0 --port will be used for arbitrary port mapping. Please use --api-port/-a instead for configuring the Api Port")
 	}
-	apiPort, err := parseApiPort(c.String("api-port"))
+	apiPort, err := parseAPIPort(c.String("api-port"))
 	if err != nil {
 		return err
 	}
@@ -111,7 +111,7 @@ func CreateCluster(c *cli.Context) error {
 	if apiPort.Host == "" {
 		apiPort.Host, err = getDockerMachineIp()
 		// IP address is the same as the host
-		apiPort.HostIp = apiPort.Host
+		apiPort.HostIP = apiPort.Host
 		// In case of error, Log a warning message, and continue on. Since it more likely caused by a miss configured
 		// DOCKER_MACHINE_NAME environment variable.
 		if err != nil {
@@ -135,9 +135,18 @@ func CreateCluster(c *cli.Context) error {
 		log.Fatal(err)
 	}
 
+	// create a docker volume for sharing image tarballs with the cluster
+	imageVolume, err := createImageVolume(c.String("name"))
+	log.Println("Created docker volume ", imageVolume.Name)
+	if err != nil {
+		return err
+	}
+	volumes := c.StringSlice("volume")
+	volumes = append(volumes, fmt.Sprintf("%s:/images", imageVolume.Name))
+
 	clusterSpec := &ClusterSpec{
 		AgentArgs:         []string{},
-		ApiPort:           *apiPort,
+		APIPort:           *apiPort,
 		AutoRestart:       c.Bool("auto-restart"),
 		ClusterName:       c.String("name"),
 		Env:               env,
@@ -146,11 +155,15 @@ func CreateCluster(c *cli.Context) error {
 		PortAutoOffset:    c.Int("port-auto-offset"),
 		ServerArgs:        k3sServerArgs,
 		Verbose:           c.GlobalBool("verbose"),
-		Volumes:           c.StringSlice("volume"),
+		Volumes:           volumes,
 	}
 
 	// create the server
 	log.Printf("Creating cluster [%s]", c.String("name"))
+
+	// create the directory where we will put the kubeconfig file by default (when running `k3d get-config`)
+	createClusterDir(c.String("name"))
+
 	dockerID, err := createServer(clusterSpec)
 	if err != nil {
 		deleteCluster()
@@ -192,10 +205,6 @@ func CreateCluster(c *cli.Context) error {
 		time.Sleep(1 * time.Second)
 	}
 
-	// create the directory where we will put the kubeconfig file by default (when running `k3d get-config`)
-	// TODO: this can probably be moved to `k3d get-config` or be removed in a different approach
-	createClusterDir(c.String("name"))
-
 	// spin up the worker nodes
 	// TODO: do this concurrently in different goroutines
 	if c.Int("workers") > 0 {
@@ -241,8 +250,8 @@ func DeleteCluster(c *cli.Context) error {
 				}
 			}
 		}
-		log.Println("...Removing server")
 		deleteClusterDir(cluster.name)
+		log.Println("...Removing server")
 		if err := removeContainer(cluster.server.ID); err != nil {
 			return fmt.Errorf("ERROR: Couldn't remove server for cluster %s\n%+v", cluster.name, err)
 		}
@@ -251,6 +260,11 @@ func DeleteCluster(c *cli.Context) error {
 			log.Printf("WARNING: couldn't delete cluster network for cluster %s\n%+v", cluster.name, err)
 		}
 
+		log.Println("...Removing docker image volume")
+		if err := deleteImageVolume(cluster.name); err != nil {
+			log.Printf("WARNING: couldn't delete image docker volume for cluster %s\n%+v", cluster.name, err)
+		}
+
 		log.Printf("SUCCESS: removed cluster [%s]", cluster.name)
 	}
 
@@ -362,3 +376,14 @@ func GetKubeConfig(c *cli.Context) error {
 func Shell(c *cli.Context) error {
 	return subShell(c.String("name"), c.String("shell"), c.String("command"))
 }
+
+// ImportImage saves an image locally and imports it into the k3d containers
+func ImportImage(c *cli.Context) error {
+	images := make([]string, 0)
+	if strings.Contains(c.Args().First(), ",") {
+		images = append(images, strings.Split(c.Args().First(), ",")...)
+	} else {
+		images = append(images, c.Args()...)
+	}
+	return importImage(c.String("name"), images, c.Bool("no-remove"))
+}

cli/container.go

@@ -22,7 +22,7 @@ import (
 
 type ClusterSpec struct {
 	AgentArgs         []string
-	ApiPort           apiPort
+	APIPort           apiPort
 	AutoRestart       bool
 	ClusterName       string
 	Env               []string
@@ -94,14 +94,14 @@ func createServer(spec *ClusterSpec) (string, error) {
 		return "", err
 	}
 
-	hostIp := "0.0.0.0"
+	hostIP := "0.0.0.0"
 	containerLabels["apihost"] = "localhost"
-	if spec.ApiPort.Host != "" {
+	if spec.APIPort.Host != "" {
-		hostIp = spec.ApiPort.HostIp
+		hostIP = spec.APIPort.HostIP
-		containerLabels["apihost"] = spec.ApiPort.Host
+		containerLabels["apihost"] = spec.APIPort.Host
 	}
 
-	apiPortSpec := fmt.Sprintf("%s:%s:%s/tcp", hostIp, spec.ApiPort.Port, spec.ApiPort.Port)
+	apiPortSpec := fmt.Sprintf("%s:%s:%s/tcp", hostIP, spec.APIPort.Port, spec.APIPort.Port)
 
 	serverPorts = append(serverPorts, apiPortSpec)
 
@@ -157,7 +157,7 @@ func createWorker(spec *ClusterSpec, postfix int) (string, error) {
 
 	containerName := GetContainerName("worker", spec.ClusterName, postfix)
 
-	env := append(spec.Env, fmt.Sprintf("K3S_URL=https://k3d-%s-server:%s", spec.ClusterName, spec.ApiPort.Port))
+	env := append(spec.Env, fmt.Sprintf("K3S_URL=https://k3d-%s-server:%s", spec.ClusterName, spec.APIPort.Port))
 
 	// ports to be assigned to the server belong to roles
 	// all, server or <server-container-name>
@@ -230,7 +230,7 @@ func removeContainer(ID string) error {
 	}
 
 	if err := docker.ContainerRemove(ctx, ID, options); err != nil {
-		return fmt.Errorf("FAILURE: couldn't delete container [%s] -> %+v", ID, err)
+		return fmt.Errorf("ERROR: couldn't delete container [%s] -> %+v", ID, err)
 	}
 	return nil
 }

cli/image.go

@@ -0,0 +1,209 @@
+package run
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"strings"
+	"time"
+
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/network"
+	"github.com/docker/docker/client"
+)
+
+const (
+	imageBasePathRemote = "/images"
+	k3dToolsImage       = "docker.io/iwilltry42/k3d-tools:v0.0.1"
+)
+
+func importImage(clusterName string, images []string, noRemove bool) error {
+	// get a docker client
+	ctx := context.Background()
+	docker, err := client.NewEnvClient()
+	if err != nil {
+		return fmt.Errorf("ERROR: couldn't create docker client\n%+v", err)
+	}
+
+	// get cluster directory to temporarily save the image tarball there
+	imageVolume, err := getImageVolume(clusterName)
+	if err != nil {
+		return fmt.Errorf("ERROR: couldn't get image volume for cluster [%s]\n%+v", clusterName, err)
+	}
+
+	//*** first, save the images using the local docker daemon
+	log.Printf("INFO: Saving images %s from local docker daemon...", images)
+	toolsContainerName := fmt.Sprintf("k3d-%s-tools", clusterName)
+	tarFileName := fmt.Sprintf("%s/k3d-%s-images-%s.tar", imageBasePathRemote, clusterName, time.Now().Format("20060102150405"))
+
+	// create a tools container to get the tarball into the named volume
+	containerConfig := container.Config{
+		Hostname: toolsContainerName,
+		Image:    k3dToolsImage,
+		Labels: map[string]string{
+			"app":       "k3d",
+			"cluster":   clusterName,
+			"component": "tools",
+		},
+		Cmd:          append([]string{"save-image", "-d", tarFileName}, images...),
+		AttachStdout: true,
+		AttachStderr: true,
+	}
+	hostConfig := container.HostConfig{
+		Binds: []string{
+			"/var/run/docker.sock:/var/run/docker.sock",
+			fmt.Sprintf("%s:%s:rw", imageVolume.Name, imageBasePathRemote),
+		},
+	}
+
+	toolsContainerID, err := startContainer(false, &containerConfig, &hostConfig, &network.NetworkingConfig{}, toolsContainerName)
+	if err != nil {
+		return err
+	}
+
+	defer func() {
+		if err = docker.ContainerRemove(ctx, toolsContainerID, types.ContainerRemoveOptions{
+			Force: true,
+		}); err != nil {
+			log.Println(fmt.Errorf("WARN: couldn't remove tools container\n%+v", err))
+		}
+	}()
+
+	// loop to wait for tools container to exit (failed or successfully saved images)
+	for {
+		cont, err := docker.ContainerInspect(ctx, toolsContainerID)
+		if err != nil {
+			return fmt.Errorf("ERROR: couldn't get helper container's exit code\n%+v", err)
+		}
+		if !cont.State.Running { // container finished...
+			if cont.State.ExitCode == 0 { // ...successfully
+				log.Println("INFO: saved images to shared docker volume")
+				break
+			} else if cont.State.ExitCode != 0 { // ...failed
+				errTxt := "ERROR: helper container failed to save images"
+				logReader, err := docker.ContainerLogs(ctx, toolsContainerID, types.ContainerLogsOptions{
+					ShowStdout: true,
+					ShowStderr: true,
+				})
+				if err != nil {
+					return fmt.Errorf("%s\n> couldn't get logs from helper container\n%+v", errTxt, err)
+				}
+				logs, err := ioutil.ReadAll(logReader) // let's show somw logs indicating what happened
+				if err != nil {
+					return fmt.Errorf("%s\n> couldn't get logs from helper container\n%+v", errTxt, err)
+				}
+				return fmt.Errorf("%s -> Logs from [%s]:\n>>>>>>\n%s\n<<<<<<", errTxt, toolsContainerName, string(logs))
+			}
+		}
+		time.Sleep(time.Second / 2) // wait for half a second so we don't spam the docker API too much
+	}
+
+	// Get the container IDs for all containers in the cluster
+	clusters, err := getClusters(false, clusterName)
+	if err != nil {
+		return fmt.Errorf("ERROR: couldn't get cluster by name [%s]\n%+v", clusterName, err)
+	}
+	containerList := []types.Container{clusters[clusterName].server}
+	containerList = append(containerList, clusters[clusterName].workers...)
+
+	// *** second, import the images using ctr in the k3d nodes
+
+	// create exec configuration
+	cmd := []string{"ctr", "image", "import", tarFileName}
+	execConfig := types.ExecConfig{
+		AttachStderr: true,
+		AttachStdout: true,
+		Cmd:          cmd,
+		Tty:          true,
+		Detach:       true,
+	}
+
+	execAttachConfig := types.ExecConfig{
+		Tty: true,
+	}
+
+	execStartConfig := types.ExecStartCheck{
+		Tty: true,
+	}
+
+	// import in each node separately
+	// TODO: import concurrently using goroutines or find a way to share the image cache
+	for _, container := range containerList {
+
+		containerName := container.Names[0][1:] // trimming the leading "/" from name
+		log.Printf("INFO: Importing images %s in container [%s]", images, containerName)
+
+		// create exec configuration
+		execResponse, err := docker.ContainerExecCreate(ctx, container.ID, execConfig)
+		if err != nil {
+			return fmt.Errorf("ERROR: Failed to create exec command for container [%s]\n%+v", containerName, err)
+		}
+
+		// attach to exec process in container
+		containerConnection, err := docker.ContainerExecAttach(ctx, execResponse.ID, execAttachConfig)
+		if err != nil {
+			return fmt.Errorf("ERROR: couldn't attach to container [%s]\n%+v", containerName, err)
+		}
+		defer containerConnection.Close()
+
+		// start exec
+		err = docker.ContainerExecStart(ctx, execResponse.ID, execStartConfig)
+		if err != nil {
+			return fmt.Errorf("ERROR: couldn't execute command in container [%s]\n%+v", containerName, err)
+		}
+
+		// get output from container
+		content, err := ioutil.ReadAll(containerConnection.Reader)
+		if err != nil {
+			return fmt.Errorf("ERROR: couldn't read output from container [%s]\n%+v", containerName, err)
+		}
+
+		// example output "unpacking image........ ...done"
+		if !strings.Contains(string(content), "done") {
+			return fmt.Errorf("ERROR: seems like something went wrong using `ctr image import` in container [%s]. Full output below:\n%s", containerName, string(content))
+		}
+	}
+
+	log.Printf("INFO: Successfully imported images %s in all nodes of cluster [%s]", images, clusterName)
+
+	// remove tarball from inside the server container
+	if !noRemove {
+		log.Println("INFO: Cleaning up tarball")
+
+		execID, err := docker.ContainerExecCreate(ctx, clusters[clusterName].server.ID, types.ExecConfig{
+			Cmd: []string{"rm", "-f", tarFileName},
+		})
+		if err != nil {
+			log.Printf("WARN: failed to delete tarball: couldn't create remove in container [%s]\n%+v", clusters[clusterName].server.ID, err)
+		}
+		err = docker.ContainerExecStart(ctx, execID.ID, types.ExecStartCheck{
+			Detach: true,
+		})
+		if err != nil {
+			log.Printf("WARN: couldn't start tarball deletion action\n%+v", err)
+		}
+
+		for {
+			execInspect, err := docker.ContainerExecInspect(ctx, execID.ID)
+			if err != nil {
+				log.Printf("WARN: couldn't verify deletion of tarball\n%+v", err)
+			}
+
+			if !execInspect.Running {
+				if execInspect.ExitCode == 0 {
+					log.Println("INFO: deleted tarball")
+					break
+				} else {
+					log.Println("WARN: failed to delete tarball")
+					break
+				}
+			}
+		}
+	}
+
+	log.Println("INFO: ...Done")
+
+	return nil
+}

cli/util.go

@@ -11,7 +11,7 @@ import (
 
 type apiPort struct {
 	Host   string
-	HostIp string
+	HostIP string
 	Port   string
 }
 
@@ -90,7 +90,7 @@ func ValidateHostname(name string) error {
 	return nil
 }
 
-func parseApiPort(portSpec string) (*apiPort, error) {
+func parseAPIPort(portSpec string) (*apiPort, error) {
 	var port *apiPort
 	split := strings.Split(portSpec, ":")
 	if len(split) > 2 {
@@ -105,7 +105,7 @@ func parseApiPort(portSpec string) (*apiPort, error) {
 		if err != nil {
 			return nil, err
 		}
-		port = &apiPort{Host: split[0], HostIp: addrs[0], Port: split[1]}
+		port = &apiPort{Host: split[0], HostIP: addrs[0], Port: split[1]}
 	}
 
 	// Verify 'port' is an integer and within port ranges

cli/volume.go

@@ -0,0 +1,92 @@
+package run
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/docker/docker/api/types/volume"
+	"github.com/docker/docker/client"
+)
+
+// createImageVolume will create a new docker volume used for storing image tarballs that can be loaded into the clusters
+func createImageVolume(clusterName string) (types.Volume, error) {
+
+	var vol types.Volume
+
+	ctx := context.Background()
+	docker, err := client.NewEnvClient()
+	if err != nil {
+		return vol, fmt.Errorf("ERROR: couldn't create docker client\n%+v", err)
+	}
+
+	volName := fmt.Sprintf("k3d-%s-images", clusterName)
+
+	volumeCreateOptions := volume.VolumesCreateBody{
+		Name: volName,
+		Labels: map[string]string{
+			"app":     "k3d",
+			"cluster": clusterName,
+		},
+		Driver:     "local", //TODO: allow setting driver + opts
+		DriverOpts: map[string]string{},
+	}
+	vol, err = docker.VolumeCreate(ctx, volumeCreateOptions)
+	if err != nil {
+		return vol, fmt.Errorf("ERROR: failed to create image volume [%s] for cluster [%s]\n%+v", volName, clusterName, err)
+	}
+
+	return vol, nil
+}
+
+// deleteImageVolume will delete the volume we created for sharing images with this cluster
+func deleteImageVolume(clusterName string) error {
+
+	ctx := context.Background()
+	docker, err := client.NewEnvClient()
+	if err != nil {
+		return fmt.Errorf("ERROR: couldn't create docker client\n%+v", err)
+	}
+
+	volName := fmt.Sprintf("k3d-%s-images", clusterName)
+
+	if err = docker.VolumeRemove(ctx, volName, true); err != nil {
+		return fmt.Errorf("ERROR: couldn't remove volume [%s] for cluster [%s]\n%+v", volName, clusterName, err)
+	}
+
+	return nil
+}
+
+// getImageVolume returns the docker volume object representing the imagevolume for the cluster
+func getImageVolume(clusterName string) (types.Volume, error) {
+	var vol types.Volume
+	volName := fmt.Sprintf("k3d-%s-images", clusterName)
+
+	ctx := context.Background()
+	docker, err := client.NewEnvClient()
+	if err != nil {
+		return vol, fmt.Errorf("ERROR: couldn't create docker client\n%+v", err)
+	}
+
+	filters := filters.NewArgs()
+	filters.Add("label", "app=k3d")
+	filters.Add("label", fmt.Sprintf("cluster=%s", clusterName))
+	volumeList, err := docker.VolumeList(ctx, filters)
+	if err != nil {
+		return vol, fmt.Errorf("ERROR: couldn't get volumes for cluster [%s]\n%+v ", clusterName, err)
+	}
+	volFound := false
+	for _, volume := range volumeList.Volumes {
+		if volume.Name == volName {
+			vol = *volume
+			volFound = true
+			break
+		}
+	}
+	if !volFound {
+		return vol, fmt.Errorf("ERROR: didn't find volume [%s] in list of volumes returned for cluster [%s]", volName, clusterName)
+	}
+
+	return vol, nil
+}

main.go

@@ -214,6 +214,24 @@ func main() {
 			},
 			Action: run.GetKubeConfig,
 		},
+		{
+			// get-kubeconfig grabs the kubeconfig from the cluster and prints the path to it
+			Name:    "import-images",
+			Aliases: []string{"i"},
+			Usage:   "Import a comma- or space-separated list of container images from your local docker daemon into the cluster",
+			Flags: []cli.Flag{
+				cli.StringFlag{
+					Name:  "name, n, cluster, c",
+					Value: defaultK3sClusterName,
+					Usage: "Name of the cluster",
+				},
+				cli.BoolFlag{
+					Name:  "no-remove, no-rm, keep, k",
+					Usage: "Disable automatic removal of the tarball",
+				},
+			},
+			Action: run.ImportImage,
+		},
 	}
 
 	// Global flags

vendor/modules.txt

@@ -8,6 +8,7 @@ github.com/docker/docker/api/types
 github.com/docker/docker/api/types/container
 github.com/docker/docker/api/types/filters
 github.com/docker/docker/api/types/network
+github.com/docker/docker/api/types/volume
 github.com/docker/docker/client
 github.com/docker/docker/api/types/mount
 github.com/docker/docker/api/types/registry
@@ -18,7 +19,6 @@ github.com/docker/docker/api/types/versions
 github.com/docker/docker/api/types/events
 github.com/docker/docker/api/types/reference
 github.com/docker/docker/api/types/time
-github.com/docker/docker/api/types/volume
 github.com/docker/docker/pkg/tlsconfig
 # github.com/docker/go-connections v0.4.0
 github.com/docker/go-connections/nat