diff --git a/pkg/client/cluster.go b/pkg/client/cluster.go index e52aebcf..275ceac6 100644 --- a/pkg/client/cluster.go +++ b/pkg/client/cluster.go @@ -60,6 +60,9 @@ func ClusterRun(ctx context.Context, runtime k3drt.Runtime, clusterConfig *confi return fmt.Errorf("Failed Cluster Preparation: %+v", err) } + // Create tools-node for later steps + go EnsureToolsNode(ctx, runtime, &clusterConfig.Cluster) + /* * Step 1: Create Containers */ @@ -295,6 +298,7 @@ func ClusterPrepImageVolume(ctx context.Context, runtime k3drt.Runtime, cluster } clusterCreateOpts.GlobalLabels[k3d.LabelImageVolume] = imageVolumeName + cluster.ImageVolume = imageVolumeName // attach volume to nodes for _, node := range cluster.Nodes { @@ -812,12 +816,12 @@ func GenerateNodeName(cluster string, role k3d.Role, suffix int) string { } // ClusterStart starts a whole cluster (i.e. all nodes of the cluster) -func ClusterStart(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluster, startClusterOpts types.ClusterStartOpts) error { +func ClusterStart(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluster, clusterStartOpts types.ClusterStartOpts) error { l.Log().Infof("Starting cluster '%s'", cluster.Name) - if startClusterOpts.Timeout > 0*time.Second { + if clusterStartOpts.Timeout > 0*time.Second { var cancel context.CancelFunc - ctx, cancel = context.WithTimeout(ctx, startClusterOpts.Timeout) + ctx, cancel = context.WithTimeout(ctx, clusterStartOpts.Timeout) defer cancel() } @@ -860,9 +864,9 @@ func ClusterStart(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clust l.Log().Infoln("Starting the initializing server...") if err := NodeStart(ctx, runtime, initNode, &k3d.NodeStartOpts{ Wait: true, // always wait for the init node - NodeHooks: startClusterOpts.NodeHooks, + NodeHooks: clusterStartOpts.NodeHooks, ReadyLogMessage: "Running kube-apiserver", // initNode means, that we're using etcd -> this will need quorum, so "k3s is up and running" won't happen right now - EnvironmentInfo: startClusterOpts.EnvironmentInfo, + EnvironmentInfo: clusterStartOpts.EnvironmentInfo, }); err != nil { return fmt.Errorf("Failed to start initializing server node: %+v", err) } @@ -875,8 +879,8 @@ func ClusterStart(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clust for _, serverNode := range servers { if err := NodeStart(ctx, runtime, serverNode, &k3d.NodeStartOpts{ Wait: true, - NodeHooks: startClusterOpts.NodeHooks, - EnvironmentInfo: startClusterOpts.EnvironmentInfo, + NodeHooks: clusterStartOpts.NodeHooks, + EnvironmentInfo: clusterStartOpts.EnvironmentInfo, }); err != nil { return fmt.Errorf("Failed to start server %s: %+v", serverNode.Name, err) } @@ -894,8 +898,8 @@ func ClusterStart(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clust agentWG.Go(func() error { return NodeStart(aCtx, runtime, currentAgentNode, &k3d.NodeStartOpts{ Wait: true, - NodeHooks: startClusterOpts.NodeHooks, - EnvironmentInfo: startClusterOpts.EnvironmentInfo, + NodeHooks: clusterStartOpts.NodeHooks, + EnvironmentInfo: clusterStartOpts.EnvironmentInfo, }) }) } @@ -915,7 +919,7 @@ func ClusterStart(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clust helperWG.Go(func() error { nodeStartOpts := &k3d.NodeStartOpts{ NodeHooks: currentHelperNode.HookActions, - EnvironmentInfo: startClusterOpts.EnvironmentInfo, + EnvironmentInfo: clusterStartOpts.EnvironmentInfo, } if currentHelperNode.Role == k3d.LoadBalancerRole { nodeStartOpts.Wait = true @@ -936,7 +940,7 @@ func ClusterStart(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clust /*** DNS ***/ // add /etc/hosts and CoreDNS entry for host.k3d.internal, referring to the host system - if err := prepInjectHostIP(ctx, runtime, cluster); err != nil { + if err := prepInjectHostIP(ctx, runtime, cluster, &clusterStartOpts); err != nil { return fmt.Errorf("failed to inject host IP: %w", err) } @@ -1007,33 +1011,27 @@ func corednsAddHost(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clu } // prepInjectHostIP adds /etc/hosts and CoreDNS entry for host.k3d.internal, referring to the host system -func prepInjectHostIP(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluster) error { +func prepInjectHostIP(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluster, clusterStartOpts *k3d.ClusterStartOpts) error { if cluster.Network.Name == "host" { l.Log().Tracef("Not injecting hostIP as clusternetwork is 'host'") return nil } l.Log().Infoln("Trying to get IP of the docker host and inject it into the cluster as 'host.k3d.internal' for easy access") - hostIP, err := GetHostIP(ctx, runtime, cluster) - if err != nil { - l.Log().Warnf("Failed to get HostIP to inject as host.k3d.internal: %+v", err) - return nil - } - if hostIP != nil { - hostsEntry := fmt.Sprintf("%s %s", hostIP.String(), k3d.DefaultK3dInternalHostRecord) - l.Log().Debugf("Adding extra host entry '%s'...", hostsEntry) - for _, node := range cluster.Nodes { - if err := runtime.ExecInNode(ctx, node, []string{"sh", "-c", fmt.Sprintf("echo '%s' >> /etc/hosts", hostsEntry)}); err != nil { - return fmt.Errorf("failed to add extra entry '%s' to /etc/hosts in node '%s': %w", hostsEntry, node.Name, err) - } + hostIP := clusterStartOpts.EnvironmentInfo.HostGateway + hostsEntry := fmt.Sprintf("%s %s", hostIP.String(), k3d.DefaultK3dInternalHostRecord) + l.Log().Debugf("Adding extra host entry '%s'...", hostsEntry) + for _, node := range cluster.Nodes { + if err := runtime.ExecInNode(ctx, node, []string{"sh", "-c", fmt.Sprintf("echo '%s' >> /etc/hosts", hostsEntry)}); err != nil { + return fmt.Errorf("failed to add extra entry '%s' to /etc/hosts in node '%s': %w", hostsEntry, node.Name, err) } - l.Log().Debugf("Successfully added host record \"%s\" to /etc/hosts in all nodes", hostsEntry) + } + l.Log().Debugf("Successfully added host record \"%s\" to /etc/hosts in all nodes", hostsEntry) - err := corednsAddHost(ctx, runtime, cluster, hostIP.String(), k3d.DefaultK3dInternalHostRecord) - if err != nil { - return fmt.Errorf("failed to inject host record \"%s\" into CoreDNS ConfigMap: %w", hostsEntry, err) - } - l.Log().Debugf("Successfully added host record \"%s\" to the CoreDNS ConfigMap ", hostsEntry) + err := corednsAddHost(ctx, runtime, cluster, hostIP.String(), k3d.DefaultK3dInternalHostRecord) + if err != nil { + return fmt.Errorf("failed to inject host record \"%s\" into CoreDNS ConfigMap: %w", hostsEntry, err) } + l.Log().Debugf("Successfully added host record \"%s\" to the CoreDNS ConfigMap ", hostsEntry) return nil } diff --git a/pkg/client/environment.go b/pkg/client/environment.go index 2294e8d5..b16f9f0f 100644 --- a/pkg/client/environment.go +++ b/pkg/client/environment.go @@ -32,14 +32,23 @@ import ( ) func GatherEnvironmentInfo(ctx context.Context, runtime runtimes.Runtime, cluster *k3d.Cluster) (*k3d.EnvironmentInfo, error) { + + envInfo := &k3d.EnvironmentInfo{} + + rtimeInfo, err := runtime.Info() + if err != nil { + return nil, err + } + envInfo.RuntimeInfo = *rtimeInfo + l.Log().Infof("Using the k3d-tools node to gather environment information") toolsNode, err := EnsureToolsNode(ctx, runtime, cluster) if err != nil { return nil, err } - defer NodeDelete(ctx, runtime, toolsNode, k3d.NodeDeleteOpts{SkipLBUpdate: true}) - - envInfo := &k3d.EnvironmentInfo{} + defer func() { + go NodeDelete(ctx, runtime, toolsNode, k3d.NodeDeleteOpts{SkipLBUpdate: true}) + }() hostIP, err := GetHostIP(ctx, runtime, cluster) if err != nil { diff --git a/pkg/client/host.go b/pkg/client/host.go index 2a14ab52..37e6bb4f 100644 --- a/pkg/client/host.go +++ b/pkg/client/host.go @@ -27,51 +27,74 @@ import ( "fmt" "net" "regexp" - "runtime" + goruntime "runtime" + "strings" l "github.com/rancher/k3d/v5/pkg/logger" - rt "github.com/rancher/k3d/v5/pkg/runtimes" + "github.com/rancher/k3d/v5/pkg/runtimes" k3d "github.com/rancher/k3d/v5/pkg/types" "github.com/rancher/k3d/v5/pkg/util" ) -var nsLookupAddressRegexp = regexp.MustCompile(`^Address:\s+(?P\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})$`) +type ResolveHostCmd struct { + Cmd string + LogMatcher *regexp.Regexp +} + +var ( + ResolveHostCmdNSLookup = ResolveHostCmd{ + Cmd: "nslookup %s", + LogMatcher: regexp.MustCompile(`^Address:\s+(?P\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})$`), + } + + ResolveHostCmdGetEnt = ResolveHostCmd{ + Cmd: "getent ahostsv4 '%s'", + LogMatcher: regexp.MustCompile(`(?P\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})\s+STREAM.+`), // e.g. `192.168.47.4 STREAM host.docker.internal`, + } +) // GetHostIP returns the routable IP address to be able to access services running on the host system from inside the cluster. // This depends on the Operating System and the chosen Runtime. -func GetHostIP(ctx context.Context, rtime rt.Runtime, cluster *k3d.Cluster) (net.IP, error) { +func GetHostIP(ctx context.Context, runtime runtimes.Runtime, cluster *k3d.Cluster) (net.IP, error) { + + rtimeInfo, err := runtime.Info() + if err != nil { + return nil, err + } + + l.Log().Tracef("GOOS: %s / Runtime OS: %s (%s)", goruntime.GOOS, rtimeInfo.OSType, rtimeInfo.OS) + + isDockerDesktop := func(os string) bool { + return strings.ToLower(os) == "docker desktop" + } // Docker Runtime - if rtime == rt.Docker { - - l.Log().Tracef("Runtime GOOS: %s", runtime.GOOS) - - // "native" Docker on Linux - if runtime.GOOS == "linux" { - ip, err := rtime.GetHostIP(ctx, cluster.Network.Name) - if err != nil { - return nil, fmt.Errorf("runtime failed to get host IP: %w", err) - } - return ip, nil - } + if runtime == runtimes.Docker { // Docker (for Desktop) on MacOS or Windows - if runtime.GOOS == "windows" || runtime.GOOS == "darwin" { + if isDockerDesktop(rtimeInfo.OS) { - toolsNode, err := EnsureToolsNode(ctx, rtime, cluster) + toolsNode, err := EnsureToolsNode(ctx, runtime, cluster) if err != nil { return nil, fmt.Errorf("failed to ensure that k3d-tools node is running to get host IP :%w", err) } - ip, err := resolveHostnameFromInside(ctx, rtime, toolsNode, "host.docker.internal") - if err != nil { - return nil, fmt.Errorf("failed to resolve 'host.docker.internal' from inside the k3d-tools node: %w", err) + ip, err := resolveHostnameFromInside(ctx, runtime, toolsNode, "host.docker.internal", ResolveHostCmdGetEnt) + if err == nil { + return ip, nil } - return ip, nil + + l.Log().Warnf("failed to resolve 'host.docker.internal' from inside the k3d-tools node: %v", err) + } - // Catch all other GOOS cases - return nil, fmt.Errorf("GetHostIP only implemented for Linux, MacOS (Darwin) and Windows") + l.Log().Infof("HostIP: using network gateway...") + ip, err := runtime.GetHostIP(ctx, cluster.Network.Name) + if err != nil { + return nil, fmt.Errorf("runtime failed to get host IP: %w", err) + } + + return ip, nil } @@ -80,9 +103,9 @@ func GetHostIP(ctx context.Context, rtime rt.Runtime, cluster *k3d.Cluster) (net } -func resolveHostnameFromInside(ctx context.Context, rtime rt.Runtime, node *k3d.Node, hostname string) (net.IP, error) { +func resolveHostnameFromInside(ctx context.Context, rtime runtimes.Runtime, node *k3d.Node, hostname string, cmd ResolveHostCmd) (net.IP, error) { - logreader, execErr := rtime.ExecInNodeGetLogs(ctx, node, []string{"sh", "-c", fmt.Sprintf("nslookup %s", hostname)}) + logreader, execErr := rtime.ExecInNodeGetLogs(ctx, node, []string{"sh", "-c", fmt.Sprintf(cmd.Cmd, hostname)}) if logreader == nil { if execErr != nil { @@ -105,12 +128,12 @@ func resolveHostnameFromInside(ctx context.Context, rtime rt.Runtime, node *k3d. } for scanner.Scan() { l.Log().Tracef("Scanning Log Line '%s'", scanner.Text()) - match := nsLookupAddressRegexp.FindStringSubmatch(scanner.Text()) + match := cmd.LogMatcher.FindStringSubmatch(scanner.Text()) if len(match) == 0 { continue } l.Log().Tracef("-> Match(es): '%+v'", match) - submatches = util.MapSubexpNames(nsLookupAddressRegexp.SubexpNames(), match) + submatches = util.MapSubexpNames(cmd.LogMatcher.SubexpNames(), match) l.Log().Tracef(" -> Submatch(es): %+v", submatches) break } @@ -118,7 +141,7 @@ func resolveHostnameFromInside(ctx context.Context, rtime rt.Runtime, node *k3d. if execErr != nil { l.Log().Errorln(execErr) } - return nil, fmt.Errorf("Failed to read address for '%s' from nslookup response", hostname) + return nil, fmt.Errorf("Failed to read address for '%s' from command output", hostname) } l.Log().Debugf("Hostname '%s' -> Address '%s'", hostname, submatches["ip"]) diff --git a/pkg/client/tools.go b/pkg/client/tools.go index 69efa69f..8e1dd503 100644 --- a/pkg/client/tools.go +++ b/pkg/client/tools.go @@ -256,35 +256,41 @@ func runToolsNode(ctx context.Context, runtime runtimes.Runtime, cluster *k3d.Cl } func EnsureToolsNode(ctx context.Context, runtime runtimes.Runtime, cluster *k3d.Cluster) (*k3d.Node, error) { - var err error - - cluster, err = ClusterGet(ctx, runtime, cluster) - if err != nil { - return nil, fmt.Errorf("failed to retrieve cluster '%s': %w", cluster.Name, err) - } - - if cluster.Network.Name == "" { - return nil, fmt.Errorf("failed to get network for cluster '%s'", cluster.Name) - } - - var imageVolume string - var ok bool - for _, node := range cluster.Nodes { - if node.Role == k3d.ServerRole || node.Role == k3d.AgentRole { - if imageVolume, ok = node.RuntimeLabels[k3d.LabelImageVolume]; ok { - break - } - } - } - if imageVolume == "" { - return nil, fmt.Errorf("Failed to find image volume for cluster '%s'", cluster.Name) - } - - l.Log().Debugf("Attaching to cluster's image volume '%s'", imageVolume) var toolsNode *k3d.Node - toolsNode, err = runtime.GetNode(ctx, &k3d.Node{Name: fmt.Sprintf("%s-%s-tools", k3d.DefaultObjectNamePrefix, cluster.Name)}) + toolsNode, err := runtime.GetNode(ctx, &k3d.Node{Name: fmt.Sprintf("%s-%s-tools", k3d.DefaultObjectNamePrefix, cluster.Name)}) if err != nil || toolsNode == nil { + + // Get more info on the cluster, if required + var imageVolume string + if cluster.Network.Name == "" || cluster.ImageVolume == "" { + l.Log().Debugf("Gathering some more info about the cluster before creating the tools node...") + var err error + cluster, err = ClusterGet(ctx, runtime, cluster) + if err != nil { + return nil, fmt.Errorf("failed to retrieve cluster: %w", err) + } + + if cluster.Network.Name == "" { + return nil, fmt.Errorf("failed to get network for cluster '%s'", cluster.Name) + } + + var ok bool + for _, node := range cluster.Nodes { + if node.Role == k3d.ServerRole || node.Role == k3d.AgentRole { + if imageVolume, ok = node.RuntimeLabels[k3d.LabelImageVolume]; ok { + break + } + } + } + if imageVolume == "" { + return nil, fmt.Errorf("Failed to find image volume for cluster '%s'", cluster.Name) + } + l.Log().Debugf("Attaching to cluster's image volume '%s'", imageVolume) + cluster.ImageVolume = imageVolume + } + + // start tools node l.Log().Infoln("Starting new tools node...") toolsNode, err = runToolsNode( ctx, @@ -292,7 +298,7 @@ func EnsureToolsNode(ctx context.Context, runtime runtimes.Runtime, cluster *k3d cluster, cluster.Network.Name, []string{ - fmt.Sprintf("%s:%s", imageVolume, k3d.DefaultImageVolumeMountPath), + fmt.Sprintf("%s:%s", cluster.ImageVolume, k3d.DefaultImageVolumeMountPath), fmt.Sprintf("%s:%s", runtime.GetRuntimePath(), runtime.GetRuntimePath()), }) if err != nil { diff --git a/pkg/runtimes/docker/host.go b/pkg/runtimes/docker/host.go index dd67f77c..5b5a6bed 100644 --- a/pkg/runtimes/docker/host.go +++ b/pkg/runtimes/docker/host.go @@ -25,19 +25,14 @@ import ( "context" "fmt" "net" - "runtime" ) // GetHostIP returns the IP of the docker host (routable from inside the containers) func (d Docker) GetHostIP(ctx context.Context, network string) (net.IP, error) { - if runtime.GOOS == "linux" { - ip, err := GetGatewayIP(ctx, network) - if err != nil { - return nil, fmt.Errorf("failed to get gateway IP of docker network '%s': %w", network, err) - } - return ip, nil + ip, err := GetGatewayIP(ctx, network) + if err != nil { + return nil, fmt.Errorf("failed to get gateway IP of docker network '%s': %w", network, err) } - - return nil, fmt.Errorf("Docker Runtime: GetHostIP only implemented for Linux") + return ip, nil } diff --git a/pkg/types/types.go b/pkg/types/types.go index 1035ad3f..9a6374c8 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -28,6 +28,7 @@ import ( "time" "github.com/docker/go-connections/nat" + runtimeTypes "github.com/rancher/k3d/v5/pkg/runtimes/types" "github.com/rancher/k3d/v5/pkg/types/k3s" "github.com/rancher/k3d/v5/version" "inet.af/netaddr" @@ -424,4 +425,5 @@ type RegistryExternal struct { type EnvironmentInfo struct { HostGateway net.IP + RuntimeInfo runtimeTypes.RuntimeInfo } diff --git a/tools/Dockerfile b/tools/Dockerfile index 674b2f57..761622d3 100644 --- a/tools/Dockerfile +++ b/tools/Dockerfile @@ -1,13 +1,16 @@ -FROM golang:1.17 as builder +FROM golang:1.17-alpine as builder ARG GIT_TAG WORKDIR /app COPY . . +RUN apk update && apk add make bash git ENV GIT_TAG=${GIT_TAG} ENV GO111MODULE=on ENV CGO_ENABLED=0 RUN make build -FROM busybox:1.31 +FROM alpine:3.14 +RUN apk update && apk add bash WORKDIR /app COPY --from=builder /app/bin/k3d-tools . ENTRYPOINT [ "/app/k3d-tools"] +