[Enhancement/Fix] Properly use env/runtime info and inject dns accordingly (#758)

- make use of environment and runtime info
- DfD: use host.docker.internal
- All other cases: use Docker network Gateway
- k3d-tools: based on alpine to have `getent` present

pkg/client/cluster.go

@@ -60,6 +60,9 @@ func ClusterRun(ctx context.Context, runtime k3drt.Runtime, clusterConfig *confi
 		return fmt.Errorf("Failed Cluster Preparation: %+v", err)
 	}
 
+	// Create tools-node for later steps
+	go EnsureToolsNode(ctx, runtime, &clusterConfig.Cluster)
+
 	/*
 	 * Step 1: Create Containers
 	 */
@@ -295,6 +298,7 @@ func ClusterPrepImageVolume(ctx context.Context, runtime k3drt.Runtime, cluster
 	}
 
 	clusterCreateOpts.GlobalLabels[k3d.LabelImageVolume] = imageVolumeName
+	cluster.ImageVolume = imageVolumeName
 
 	// attach volume to nodes
 	for _, node := range cluster.Nodes {
@@ -812,12 +816,12 @@ func GenerateNodeName(cluster string, role k3d.Role, suffix int) string {
 }
 
 // ClusterStart starts a whole cluster (i.e. all nodes of the cluster)
-func ClusterStart(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluster, startClusterOpts types.ClusterStartOpts) error {
+func ClusterStart(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluster, clusterStartOpts types.ClusterStartOpts) error {
 	l.Log().Infof("Starting cluster '%s'", cluster.Name)
 
-	if startClusterOpts.Timeout > 0*time.Second {
+	if clusterStartOpts.Timeout > 0*time.Second {
 		var cancel context.CancelFunc
-		ctx, cancel = context.WithTimeout(ctx, startClusterOpts.Timeout)
+		ctx, cancel = context.WithTimeout(ctx, clusterStartOpts.Timeout)
 		defer cancel()
 	}
 
@@ -860,9 +864,9 @@ func ClusterStart(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clust
 		l.Log().Infoln("Starting the initializing server...")
 		if err := NodeStart(ctx, runtime, initNode, &k3d.NodeStartOpts{
 			Wait:            true, // always wait for the init node
-			NodeHooks:       startClusterOpts.NodeHooks,
+			NodeHooks:       clusterStartOpts.NodeHooks,
 			ReadyLogMessage: "Running kube-apiserver", // initNode means, that we're using etcd -> this will need quorum, so "k3s is up and running" won't happen right now
-			EnvironmentInfo: startClusterOpts.EnvironmentInfo,
+			EnvironmentInfo: clusterStartOpts.EnvironmentInfo,
 		}); err != nil {
 			return fmt.Errorf("Failed to start initializing server node: %+v", err)
 		}
@@ -875,8 +879,8 @@ func ClusterStart(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clust
 	for _, serverNode := range servers {
 		if err := NodeStart(ctx, runtime, serverNode, &k3d.NodeStartOpts{
 			Wait:            true,
-			NodeHooks:       startClusterOpts.NodeHooks,
+			NodeHooks:       clusterStartOpts.NodeHooks,
-			EnvironmentInfo: startClusterOpts.EnvironmentInfo,
+			EnvironmentInfo: clusterStartOpts.EnvironmentInfo,
 		}); err != nil {
 			return fmt.Errorf("Failed to start server %s: %+v", serverNode.Name, err)
 		}
@@ -894,8 +898,8 @@ func ClusterStart(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clust
 		agentWG.Go(func() error {
 			return NodeStart(aCtx, runtime, currentAgentNode, &k3d.NodeStartOpts{
 				Wait:            true,
-				NodeHooks:       startClusterOpts.NodeHooks,
+				NodeHooks:       clusterStartOpts.NodeHooks,
-				EnvironmentInfo: startClusterOpts.EnvironmentInfo,
+				EnvironmentInfo: clusterStartOpts.EnvironmentInfo,
 			})
 		})
 	}
@@ -915,7 +919,7 @@ func ClusterStart(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clust
 		helperWG.Go(func() error {
 			nodeStartOpts := &k3d.NodeStartOpts{
 				NodeHooks:       currentHelperNode.HookActions,
-				EnvironmentInfo: startClusterOpts.EnvironmentInfo,
+				EnvironmentInfo: clusterStartOpts.EnvironmentInfo,
 			}
 			if currentHelperNode.Role == k3d.LoadBalancerRole {
 				nodeStartOpts.Wait = true
@@ -936,7 +940,7 @@ func ClusterStart(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clust
 	/*** DNS ***/
 
 	// add /etc/hosts and CoreDNS entry for host.k3d.internal, referring to the host system
-	if err := prepInjectHostIP(ctx, runtime, cluster); err != nil {
+	if err := prepInjectHostIP(ctx, runtime, cluster, &clusterStartOpts); err != nil {
 		return fmt.Errorf("failed to inject host IP: %w", err)
 	}
 
@@ -1007,33 +1011,27 @@ func corednsAddHost(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clu
 }
 
 // prepInjectHostIP adds /etc/hosts and CoreDNS entry for host.k3d.internal, referring to the host system
-func prepInjectHostIP(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluster) error {
+func prepInjectHostIP(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluster, clusterStartOpts *k3d.ClusterStartOpts) error {
 	if cluster.Network.Name == "host" {
 		l.Log().Tracef("Not injecting hostIP as clusternetwork is 'host'")
 		return nil
 	}
 	l.Log().Infoln("Trying to get IP of the docker host and inject it into the cluster as 'host.k3d.internal' for easy access")
-	hostIP, err := GetHostIP(ctx, runtime, cluster)
+	hostIP := clusterStartOpts.EnvironmentInfo.HostGateway
-	if err != nil {
+	hostsEntry := fmt.Sprintf("%s %s", hostIP.String(), k3d.DefaultK3dInternalHostRecord)
-		l.Log().Warnf("Failed to get HostIP to inject as host.k3d.internal: %+v", err)
+	l.Log().Debugf("Adding extra host entry '%s'...", hostsEntry)
-		return nil
+	for _, node := range cluster.Nodes {
-	}
+		if err := runtime.ExecInNode(ctx, node, []string{"sh", "-c", fmt.Sprintf("echo '%s' >> /etc/hosts", hostsEntry)}); err != nil {
-	if hostIP != nil {
+			return fmt.Errorf("failed to add extra entry '%s' to /etc/hosts in node '%s': %w", hostsEntry, node.Name, err)
-		hostsEntry := fmt.Sprintf("%s %s", hostIP.String(), k3d.DefaultK3dInternalHostRecord)
-		l.Log().Debugf("Adding extra host entry '%s'...", hostsEntry)
-		for _, node := range cluster.Nodes {
-			if err := runtime.ExecInNode(ctx, node, []string{"sh", "-c", fmt.Sprintf("echo '%s' >> /etc/hosts", hostsEntry)}); err != nil {
-				return fmt.Errorf("failed to add extra entry '%s' to /etc/hosts in node '%s': %w", hostsEntry, node.Name, err)
-			}
 		}
-		l.Log().Debugf("Successfully added host record \"%s\" to /etc/hosts in all nodes", hostsEntry)
+	}
+	l.Log().Debugf("Successfully added host record \"%s\" to /etc/hosts in all nodes", hostsEntry)
 
-		err := corednsAddHost(ctx, runtime, cluster, hostIP.String(), k3d.DefaultK3dInternalHostRecord)
+	err := corednsAddHost(ctx, runtime, cluster, hostIP.String(), k3d.DefaultK3dInternalHostRecord)
-		if err != nil {
+	if err != nil {
-			return fmt.Errorf("failed to inject host record \"%s\" into CoreDNS ConfigMap: %w", hostsEntry, err)
+		return fmt.Errorf("failed to inject host record \"%s\" into CoreDNS ConfigMap: %w", hostsEntry, err)
-		}
-		l.Log().Debugf("Successfully added host record \"%s\" to the CoreDNS ConfigMap ", hostsEntry)
 	}
+	l.Log().Debugf("Successfully added host record \"%s\" to the CoreDNS ConfigMap ", hostsEntry)
 	return nil
 }
 

pkg/client/environment.go

@@ -32,14 +32,23 @@ import (
 )
 
 func GatherEnvironmentInfo(ctx context.Context, runtime runtimes.Runtime, cluster *k3d.Cluster) (*k3d.EnvironmentInfo, error) {
+
+	envInfo := &k3d.EnvironmentInfo{}
+
+	rtimeInfo, err := runtime.Info()
+	if err != nil {
+		return nil, err
+	}
+	envInfo.RuntimeInfo = *rtimeInfo
+
 	l.Log().Infof("Using the k3d-tools node to gather environment information")
 	toolsNode, err := EnsureToolsNode(ctx, runtime, cluster)
 	if err != nil {
 		return nil, err
 	}
-	defer NodeDelete(ctx, runtime, toolsNode, k3d.NodeDeleteOpts{SkipLBUpdate: true})
+	defer func() {
-
+		go NodeDelete(ctx, runtime, toolsNode, k3d.NodeDeleteOpts{SkipLBUpdate: true})
-	envInfo := &k3d.EnvironmentInfo{}
+	}()
 
 	hostIP, err := GetHostIP(ctx, runtime, cluster)
 	if err != nil {

pkg/client/host.go

@@ -27,51 +27,74 @@ import (
 	"fmt"
 	"net"
 	"regexp"
-	"runtime"
+	goruntime "runtime"
+	"strings"
 
 	l "github.com/rancher/k3d/v5/pkg/logger"
-	rt "github.com/rancher/k3d/v5/pkg/runtimes"
+	"github.com/rancher/k3d/v5/pkg/runtimes"
 	k3d "github.com/rancher/k3d/v5/pkg/types"
 	"github.com/rancher/k3d/v5/pkg/util"
 )
 
-var nsLookupAddressRegexp = regexp.MustCompile(`^Address:\s+(?P<ip>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})$`)
+type ResolveHostCmd struct {
+	Cmd        string
+	LogMatcher *regexp.Regexp
+}
+
+var (
+	ResolveHostCmdNSLookup = ResolveHostCmd{
+		Cmd:        "nslookup %s",
+		LogMatcher: regexp.MustCompile(`^Address:\s+(?P<ip>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})$`),
+	}
+
+	ResolveHostCmdGetEnt = ResolveHostCmd{
+		Cmd:        "getent ahostsv4 '%s'",
+		LogMatcher: regexp.MustCompile(`(?P<ip>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})\s+STREAM.+`), // e.g. `192.168.47.4   STREAM host.docker.internal`,
+	}
+)
 
 // GetHostIP returns the routable IP address to be able to access services running on the host system from inside the cluster.
 // This depends on the Operating System and the chosen Runtime.
-func GetHostIP(ctx context.Context, rtime rt.Runtime, cluster *k3d.Cluster) (net.IP, error) {
+func GetHostIP(ctx context.Context, runtime runtimes.Runtime, cluster *k3d.Cluster) (net.IP, error) {
+
+	rtimeInfo, err := runtime.Info()
+	if err != nil {
+		return nil, err
+	}
+
+	l.Log().Tracef("GOOS: %s / Runtime OS: %s (%s)", goruntime.GOOS, rtimeInfo.OSType, rtimeInfo.OS)
+
+	isDockerDesktop := func(os string) bool {
+		return strings.ToLower(os) == "docker desktop"
+	}
 
 	// Docker Runtime
-	if rtime == rt.Docker {
+	if runtime == runtimes.Docker {
-
-		l.Log().Tracef("Runtime GOOS: %s", runtime.GOOS)
-
-		// "native" Docker on Linux
-		if runtime.GOOS == "linux" {
-			ip, err := rtime.GetHostIP(ctx, cluster.Network.Name)
-			if err != nil {
-				return nil, fmt.Errorf("runtime failed to get host IP: %w", err)
-			}
-			return ip, nil
-		}
 
 		// Docker (for Desktop) on MacOS or Windows
-		if runtime.GOOS == "windows" || runtime.GOOS == "darwin" {
+		if isDockerDesktop(rtimeInfo.OS) {
 
-			toolsNode, err := EnsureToolsNode(ctx, rtime, cluster)
+			toolsNode, err := EnsureToolsNode(ctx, runtime, cluster)
 			if err != nil {
 				return nil, fmt.Errorf("failed to ensure that k3d-tools node is running to get host IP :%w", err)
 			}
 
-			ip, err := resolveHostnameFromInside(ctx, rtime, toolsNode, "host.docker.internal")
+			ip, err := resolveHostnameFromInside(ctx, runtime, toolsNode, "host.docker.internal", ResolveHostCmdGetEnt)
-			if err != nil {
+			if err == nil {
-				return nil, fmt.Errorf("failed to resolve 'host.docker.internal' from inside the k3d-tools node: %w", err)
+				return ip, nil
 			}
-			return ip, nil
+
+			l.Log().Warnf("failed to resolve 'host.docker.internal' from inside the k3d-tools node: %v", err)
+
 		}
 
-		// Catch all other GOOS cases
+		l.Log().Infof("HostIP: using network gateway...")
-		return nil, fmt.Errorf("GetHostIP only implemented for Linux, MacOS (Darwin) and Windows")
+		ip, err := runtime.GetHostIP(ctx, cluster.Network.Name)
+		if err != nil {
+			return nil, fmt.Errorf("runtime failed to get host IP: %w", err)
+		}
+
+		return ip, nil
 
 	}
 
@@ -80,9 +103,9 @@ func GetHostIP(ctx context.Context, rtime rt.Runtime, cluster *k3d.Cluster) (net
 
 }
 
-func resolveHostnameFromInside(ctx context.Context, rtime rt.Runtime, node *k3d.Node, hostname string) (net.IP, error) {
+func resolveHostnameFromInside(ctx context.Context, rtime runtimes.Runtime, node *k3d.Node, hostname string, cmd ResolveHostCmd) (net.IP, error) {
 
-	logreader, execErr := rtime.ExecInNodeGetLogs(ctx, node, []string{"sh", "-c", fmt.Sprintf("nslookup %s", hostname)})
+	logreader, execErr := rtime.ExecInNodeGetLogs(ctx, node, []string{"sh", "-c", fmt.Sprintf(cmd.Cmd, hostname)})
 
 	if logreader == nil {
 		if execErr != nil {
@@ -105,12 +128,12 @@ func resolveHostnameFromInside(ctx context.Context, rtime rt.Runtime, node *k3d.
 	}
 	for scanner.Scan() {
 		l.Log().Tracef("Scanning Log Line '%s'", scanner.Text())
-		match := nsLookupAddressRegexp.FindStringSubmatch(scanner.Text())
+		match := cmd.LogMatcher.FindStringSubmatch(scanner.Text())
 		if len(match) == 0 {
 			continue
 		}
 		l.Log().Tracef("-> Match(es): '%+v'", match)
-		submatches = util.MapSubexpNames(nsLookupAddressRegexp.SubexpNames(), match)
+		submatches = util.MapSubexpNames(cmd.LogMatcher.SubexpNames(), match)
 		l.Log().Tracef(" -> Submatch(es): %+v", submatches)
 		break
 	}
@@ -118,7 +141,7 @@ func resolveHostnameFromInside(ctx context.Context, rtime rt.Runtime, node *k3d.
 		if execErr != nil {
 			l.Log().Errorln(execErr)
 		}
-		return nil, fmt.Errorf("Failed to read address for '%s' from nslookup response", hostname)
+		return nil, fmt.Errorf("Failed to read address for '%s' from command output", hostname)
 	}
 
 	l.Log().Debugf("Hostname '%s' -> Address '%s'", hostname, submatches["ip"])

pkg/client/tools.go

@@ -256,35 +256,41 @@ func runToolsNode(ctx context.Context, runtime runtimes.Runtime, cluster *k3d.Cl
 }
 
 func EnsureToolsNode(ctx context.Context, runtime runtimes.Runtime, cluster *k3d.Cluster) (*k3d.Node, error) {
-	var err error
-
-	cluster, err = ClusterGet(ctx, runtime, cluster)
-	if err != nil {
-		return nil, fmt.Errorf("failed to retrieve cluster '%s': %w", cluster.Name, err)
-	}
-
-	if cluster.Network.Name == "" {
-		return nil, fmt.Errorf("failed to get network for cluster '%s'", cluster.Name)
-	}
-
-	var imageVolume string
-	var ok bool
-	for _, node := range cluster.Nodes {
-		if node.Role == k3d.ServerRole || node.Role == k3d.AgentRole {
-			if imageVolume, ok = node.RuntimeLabels[k3d.LabelImageVolume]; ok {
-				break
-			}
-		}
-	}
-	if imageVolume == "" {
-		return nil, fmt.Errorf("Failed to find image volume for cluster '%s'", cluster.Name)
-	}
-
-	l.Log().Debugf("Attaching to cluster's image volume '%s'", imageVolume)
 
 	var toolsNode *k3d.Node
-	toolsNode, err = runtime.GetNode(ctx, &k3d.Node{Name: fmt.Sprintf("%s-%s-tools", k3d.DefaultObjectNamePrefix, cluster.Name)})
+	toolsNode, err := runtime.GetNode(ctx, &k3d.Node{Name: fmt.Sprintf("%s-%s-tools", k3d.DefaultObjectNamePrefix, cluster.Name)})
 	if err != nil || toolsNode == nil {
+
+		// Get more info on the cluster, if required
+		var imageVolume string
+		if cluster.Network.Name == "" || cluster.ImageVolume == "" {
+			l.Log().Debugf("Gathering some more info about the cluster before creating the tools node...")
+			var err error
+			cluster, err = ClusterGet(ctx, runtime, cluster)
+			if err != nil {
+				return nil, fmt.Errorf("failed to retrieve cluster: %w", err)
+			}
+
+			if cluster.Network.Name == "" {
+				return nil, fmt.Errorf("failed to get network for cluster '%s'", cluster.Name)
+			}
+
+			var ok bool
+			for _, node := range cluster.Nodes {
+				if node.Role == k3d.ServerRole || node.Role == k3d.AgentRole {
+					if imageVolume, ok = node.RuntimeLabels[k3d.LabelImageVolume]; ok {
+						break
+					}
+				}
+			}
+			if imageVolume == "" {
+				return nil, fmt.Errorf("Failed to find image volume for cluster '%s'", cluster.Name)
+			}
+			l.Log().Debugf("Attaching to cluster's image volume '%s'", imageVolume)
+			cluster.ImageVolume = imageVolume
+		}
+
+		// start tools node
 		l.Log().Infoln("Starting new tools node...")
 		toolsNode, err = runToolsNode(
 			ctx,
@@ -292,7 +298,7 @@ func EnsureToolsNode(ctx context.Context, runtime runtimes.Runtime, cluster *k3d
 			cluster,
 			cluster.Network.Name,
 			[]string{
-				fmt.Sprintf("%s:%s", imageVolume, k3d.DefaultImageVolumeMountPath),
+				fmt.Sprintf("%s:%s", cluster.ImageVolume, k3d.DefaultImageVolumeMountPath),
 				fmt.Sprintf("%s:%s", runtime.GetRuntimePath(), runtime.GetRuntimePath()),
 			})
 		if err != nil {

pkg/runtimes/docker/host.go

@@ -25,19 +25,14 @@ import (
 	"context"
 	"fmt"
 	"net"
-	"runtime"
 )
 
 // GetHostIP returns the IP of the docker host (routable from inside the containers)
 func (d Docker) GetHostIP(ctx context.Context, network string) (net.IP, error) {
-	if runtime.GOOS == "linux" {
+	ip, err := GetGatewayIP(ctx, network)
-		ip, err := GetGatewayIP(ctx, network)
+	if err != nil {
-		if err != nil {
+		return nil, fmt.Errorf("failed to get gateway IP of docker network '%s': %w", network, err)
-			return nil, fmt.Errorf("failed to get gateway IP of docker network '%s': %w", network, err)
-		}
-		return ip, nil
 	}
-
+	return ip, nil
-	return nil, fmt.Errorf("Docker Runtime: GetHostIP only implemented for Linux")
 
 }

pkg/types/types.go

@@ -28,6 +28,7 @@ import (
 	"time"
 
 	"github.com/docker/go-connections/nat"
+	runtimeTypes "github.com/rancher/k3d/v5/pkg/runtimes/types"
 	"github.com/rancher/k3d/v5/pkg/types/k3s"
 	"github.com/rancher/k3d/v5/version"
 	"inet.af/netaddr"
@@ -424,4 +425,5 @@ type RegistryExternal struct {
 
 type EnvironmentInfo struct {
 	HostGateway net.IP
+	RuntimeInfo runtimeTypes.RuntimeInfo
 }

tools/Dockerfile

@@ -1,13 +1,16 @@
-FROM golang:1.17 as builder
+FROM golang:1.17-alpine as builder
 ARG GIT_TAG
 WORKDIR /app
 COPY . .
+RUN apk update && apk add make bash git
 ENV GIT_TAG=${GIT_TAG}
 ENV GO111MODULE=on
 ENV CGO_ENABLED=0
 RUN make build
 
-FROM busybox:1.31
+FROM alpine:3.14
+RUN apk update && apk add bash
 WORKDIR /app
 COPY --from=builder /app/bin/k3d-tools .
 ENTRYPOINT [ "/app/k3d-tools"]
+