mirrors/k3d
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

[Enhancement/Fix] Properly use env/runtime info and inject dns accordingly (#758)

Browse Source 
- make use of environment and runtime info
- DfD: use host.docker.internal
- All other cases: use Docker network Gateway
- k3d-tools: based on alpine to have `getent` present
This commit is contained in:
Thorsten Klein 2021-09-23 12:31:26 +02:00 committed by iwilltry42
parent f801e46e9e
commit 67d8c8c84f
No known key found for this signature in database
GPG Key ID: 7BA57AD1CFF16110
7 changed files with 136 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
pkg/client/cluster.go
View File

@ -60,6 +60,9 @@ func ClusterRun(ctx context.Context, runtime k3drt.Runtime, clusterConfig *confi
return fmt.Errorf("Failed Cluster Preparation: %+v", err)
}
// Create tools-node for later steps
go EnsureToolsNode(ctx, runtime, &clusterConfig.Cluster)
/*
* Step 1: Create Containers
*/
@ -295,6 +298,7 @@ func ClusterPrepImageVolume(ctx context.Context, runtime k3drt.Runtime, cluster
}
clusterCreateOpts.GlobalLabels[k3d.LabelImageVolume] = imageVolumeName
cluster.ImageVolume = imageVolumeName
// attach volume to nodes
for _, node := range cluster.Nodes {
@ -812,12 +816,12 @@ func GenerateNodeName(cluster string, role k3d.Role, suffix int) string {
}
// ClusterStart starts a whole cluster (i.e. all nodes of the cluster)
func ClusterStart(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluster, startClusterOpts types.ClusterStartOpts) error {
func ClusterStart(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluster, clusterStartOpts types.ClusterStartOpts) error {
l.Log().Infof("Starting cluster '%s'", cluster.Name)
if startClusterOpts.Timeout > 0*time.Second {
if clusterStartOpts.Timeout > 0*time.Second {
var cancel context.CancelFunc
ctx, cancel = context.WithTimeout(ctx, startClusterOpts.Timeout)
ctx, cancel = context.WithTimeout(ctx, clusterStartOpts.Timeout)
defer cancel()
}
@ -860,9 +864,9 @@ func ClusterStart(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clust
l.Log().Infoln("Starting the initializing server...")
if err := NodeStart(ctx, runtime, initNode, &k3d.NodeStartOpts{
Wait: true, // always wait for the init node
NodeHooks: startClusterOpts.NodeHooks,
NodeHooks: clusterStartOpts.NodeHooks,
ReadyLogMessage: "Running kube-apiserver", // initNode means, that we're using etcd -> this will need quorum, so "k3s is up and running" won't happen right now
EnvironmentInfo: startClusterOpts.EnvironmentInfo,
EnvironmentInfo: clusterStartOpts.EnvironmentInfo,
}); err != nil {
return fmt.Errorf("Failed to start initializing server node: %+v", err)
}
@ -875,8 +879,8 @@ func ClusterStart(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clust
for _, serverNode := range servers {
if err := NodeStart(ctx, runtime, serverNode, &k3d.NodeStartOpts{
Wait: true,
NodeHooks: startClusterOpts.NodeHooks,
EnvironmentInfo: startClusterOpts.EnvironmentInfo,
NodeHooks: clusterStartOpts.NodeHooks,
EnvironmentInfo: clusterStartOpts.EnvironmentInfo,
}); err != nil {
return fmt.Errorf("Failed to start server %s: %+v", serverNode.Name, err)
}
@ -894,8 +898,8 @@ func ClusterStart(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clust
agentWG.Go(func() error {
return NodeStart(aCtx, runtime, currentAgentNode, &k3d.NodeStartOpts{
Wait: true,
NodeHooks: startClusterOpts.NodeHooks,
EnvironmentInfo: startClusterOpts.EnvironmentInfo,
NodeHooks: clusterStartOpts.NodeHooks,
EnvironmentInfo: clusterStartOpts.EnvironmentInfo,
})
})
}
@ -915,7 +919,7 @@ func ClusterStart(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clust
helperWG.Go(func() error {
nodeStartOpts := &k3d.NodeStartOpts{
NodeHooks: currentHelperNode.HookActions,
EnvironmentInfo: startClusterOpts.EnvironmentInfo,
EnvironmentInfo: clusterStartOpts.EnvironmentInfo,
}
if currentHelperNode.Role == k3d.LoadBalancerRole {
nodeStartOpts.Wait = true
@ -936,7 +940,7 @@ func ClusterStart(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clust
/*** DNS ***/
// add /etc/hosts and CoreDNS entry for host.k3d.internal, referring to the host system
if err := prepInjectHostIP(ctx, runtime, cluster); err != nil {
if err := prepInjectHostIP(ctx, runtime, cluster, &clusterStartOpts); err != nil {
return fmt.Errorf("failed to inject host IP: %w", err)
}
@ -1007,33 +1011,27 @@ func corednsAddHost(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clu
}
// prepInjectHostIP adds /etc/hosts and CoreDNS entry for host.k3d.internal, referring to the host system
func prepInjectHostIP(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluster) error {
func prepInjectHostIP(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluster, clusterStartOpts *k3d.ClusterStartOpts) error {
if cluster.Network.Name == "host" {
l.Log().Tracef("Not injecting hostIP as clusternetwork is 'host'")
return nil
}
l.Log().Infoln("Trying to get IP of the docker host and inject it into the cluster as 'host.k3d.internal' for easy access")
hostIP, err := GetHostIP(ctx, runtime, cluster)
if err != nil {
l.Log().Warnf("Failed to get HostIP to inject as host.k3d.internal: %+v", err)
return nil
}
if hostIP != nil {
hostsEntry := fmt.Sprintf("%s %s", hostIP.String(), k3d.DefaultK3dInternalHostRecord)
l.Log().Debugf("Adding extra host entry '%s'...", hostsEntry)
for _, node := range cluster.Nodes {
if err := runtime.ExecInNode(ctx, node, []string{"sh", "-c", fmt.Sprintf("echo '%s' >> /etc/hosts", hostsEntry)}); err != nil {
return fmt.Errorf("failed to add extra entry '%s' to /etc/hosts in node '%s': %w", hostsEntry, node.Name, err)
}
hostIP := clusterStartOpts.EnvironmentInfo.HostGateway
hostsEntry := fmt.Sprintf("%s %s", hostIP.String(), k3d.DefaultK3dInternalHostRecord)
l.Log().Debugf("Adding extra host entry '%s'...", hostsEntry)
for _, node := range cluster.Nodes {
if err := runtime.ExecInNode(ctx, node, []string{"sh", "-c", fmt.Sprintf("echo '%s' >> /etc/hosts", hostsEntry)}); err != nil {
return fmt.Errorf("failed to add extra entry '%s' to /etc/hosts in node '%s': %w", hostsEntry, node.Name, err)
}
l.Log().Debugf("Successfully added host record \"%s\" to /etc/hosts in all nodes", hostsEntry)
}
l.Log().Debugf("Successfully added host record \"%s\" to /etc/hosts in all nodes", hostsEntry)
err := corednsAddHost(ctx, runtime, cluster, hostIP.String(), k3d.DefaultK3dInternalHostRecord)
if err != nil {
return fmt.Errorf("failed to inject host record \"%s\" into CoreDNS ConfigMap: %w", hostsEntry, err)
}
l.Log().Debugf("Successfully added host record \"%s\" to the CoreDNS ConfigMap ", hostsEntry)
err := corednsAddHost(ctx, runtime, cluster, hostIP.String(), k3d.DefaultK3dInternalHostRecord)
if err != nil {
return fmt.Errorf("failed to inject host record \"%s\" into CoreDNS ConfigMap: %w", hostsEntry, err)
}
l.Log().Debugf("Successfully added host record \"%s\" to the CoreDNS ConfigMap ", hostsEntry)
return nil
}

15
pkg/client/environment.go
View File

@ -32,14 +32,23 @@ import (
)
func GatherEnvironmentInfo(ctx context.Context, runtime runtimes.Runtime, cluster *k3d.Cluster) (*k3d.EnvironmentInfo, error) {
envInfo := &k3d.EnvironmentInfo{}
rtimeInfo, err := runtime.Info()
if err != nil {
return nil, err
}
envInfo.RuntimeInfo = *rtimeInfo
l.Log().Infof("Using the k3d-tools node to gather environment information")
toolsNode, err := EnsureToolsNode(ctx, runtime, cluster)
if err != nil {
return nil, err
}
defer NodeDelete(ctx, runtime, toolsNode, k3d.NodeDeleteOpts{SkipLBUpdate: true})
envInfo := &k3d.EnvironmentInfo{}
defer func() {
go NodeDelete(ctx, runtime, toolsNode, k3d.NodeDeleteOpts{SkipLBUpdate: true})
}()
hostIP, err := GetHostIP(ctx, runtime, cluster)
if err != nil {

81
pkg/client/host.go
View File

@ -27,51 +27,74 @@ import (
"fmt"
"net"
"regexp"
"runtime"
goruntime "runtime"
"strings"
l "github.com/rancher/k3d/v5/pkg/logger"
rt "github.com/rancher/k3d/v5/pkg/runtimes"
"github.com/rancher/k3d/v5/pkg/runtimes"
k3d "github.com/rancher/k3d/v5/pkg/types"
"github.com/rancher/k3d/v5/pkg/util"
)
var nsLookupAddressRegexp = regexp.MustCompile(`^Address:\s+(?P<ip>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})$`)
type ResolveHostCmd struct {
Cmd string
LogMatcher *regexp.Regexp
}
var (
ResolveHostCmdNSLookup = ResolveHostCmd{
Cmd: "nslookup %s",
LogMatcher: regexp.MustCompile(`^Address:\s+(?P<ip>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})$`),
}
ResolveHostCmdGetEnt = ResolveHostCmd{
Cmd: "getent ahostsv4 '%s'",
LogMatcher: regexp.MustCompile(`(?P<ip>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})\s+STREAM.+`), // e.g. `192.168.47.4 STREAM host.docker.internal`,
}
)
// GetHostIP returns the routable IP address to be able to access services running on the host system from inside the cluster.
// This depends on the Operating System and the chosen Runtime.
func GetHostIP(ctx context.Context, rtime rt.Runtime, cluster *k3d.Cluster) (net.IP, error) {
func GetHostIP(ctx context.Context, runtime runtimes.Runtime, cluster *k3d.Cluster) (net.IP, error) {
rtimeInfo, err := runtime.Info()
if err != nil {
return nil, err
}
l.Log().Tracef("GOOS: %s / Runtime OS: %s (%s)", goruntime.GOOS, rtimeInfo.OSType, rtimeInfo.OS)
isDockerDesktop := func(os string) bool {
return strings.ToLower(os) == "docker desktop"
}
// Docker Runtime
if rtime == rt.Docker {
l.Log().Tracef("Runtime GOOS: %s", runtime.GOOS)
// "native" Docker on Linux
if runtime.GOOS == "linux" {
ip, err := rtime.GetHostIP(ctx, cluster.Network.Name)
if err != nil {
return nil, fmt.Errorf("runtime failed to get host IP: %w", err)
}
return ip, nil
}
if runtime == runtimes.Docker {
// Docker (for Desktop) on MacOS or Windows
if runtime.GOOS == "windows" || runtime.GOOS == "darwin" {
if isDockerDesktop(rtimeInfo.OS) {
toolsNode, err := EnsureToolsNode(ctx, rtime, cluster)
toolsNode, err := EnsureToolsNode(ctx, runtime, cluster)
if err != nil {
return nil, fmt.Errorf("failed to ensure that k3d-tools node is running to get host IP :%w", err)
}
ip, err := resolveHostnameFromInside(ctx, rtime, toolsNode, "host.docker.internal")
if err != nil {
return nil, fmt.Errorf("failed to resolve 'host.docker.internal' from inside the k3d-tools node: %w", err)
ip, err := resolveHostnameFromInside(ctx, runtime, toolsNode, "host.docker.internal", ResolveHostCmdGetEnt)
if err == nil {
return ip, nil
}
return ip, nil
l.Log().Warnf("failed to resolve 'host.docker.internal' from inside the k3d-tools node: %v", err)
}
// Catch all other GOOS cases
return nil, fmt.Errorf("GetHostIP only implemented for Linux, MacOS (Darwin) and Windows")
l.Log().Infof("HostIP: using network gateway...")
ip, err := runtime.GetHostIP(ctx, cluster.Network.Name)
if err != nil {
return nil, fmt.Errorf("runtime failed to get host IP: %w", err)
}
return ip, nil
}
@ -80,9 +103,9 @@ func GetHostIP(ctx context.Context, rtime rt.Runtime, cluster *k3d.Cluster) (net
}
func resolveHostnameFromInside(ctx context.Context, rtime rt.Runtime, node *k3d.Node, hostname string) (net.IP, error) {
func resolveHostnameFromInside(ctx context.Context, rtime runtimes.Runtime, node *k3d.Node, hostname string, cmd ResolveHostCmd) (net.IP, error) {
logreader, execErr := rtime.ExecInNodeGetLogs(ctx, node, []string{"sh", "-c", fmt.Sprintf("nslookup %s", hostname)})
logreader, execErr := rtime.ExecInNodeGetLogs(ctx, node, []string{"sh", "-c", fmt.Sprintf(cmd.Cmd, hostname)})
if logreader == nil {
if execErr != nil {
@ -105,12 +128,12 @@ func resolveHostnameFromInside(ctx context.Context, rtime rt.Runtime, node *k3d.
}
for scanner.Scan() {
l.Log().Tracef("Scanning Log Line '%s'", scanner.Text())
match := nsLookupAddressRegexp.FindStringSubmatch(scanner.Text())
match := cmd.LogMatcher.FindStringSubmatch(scanner.Text())
if len(match) == 0 {
continue
}
l.Log().Tracef("-> Match(es): '%+v'", match)
submatches = util.MapSubexpNames(nsLookupAddressRegexp.SubexpNames(), match)
submatches = util.MapSubexpNames(cmd.LogMatcher.SubexpNames(), match)
l.Log().Tracef(" -> Submatch(es): %+v", submatches)
break
}
@ -118,7 +141,7 @@ func resolveHostnameFromInside(ctx context.Context, rtime rt.Runtime, node *k3d.
if execErr != nil {
l.Log().Errorln(execErr)
}
return nil, fmt.Errorf("Failed to read address for '%s' from nslookup response", hostname)
return nil, fmt.Errorf("Failed to read address for '%s' from command output", hostname)
}
l.Log().Debugf("Hostname '%s' -> Address '%s'", hostname, submatches["ip"])

60
pkg/client/tools.go
View File

@ -256,35 +256,41 @@ func runToolsNode(ctx context.Context, runtime runtimes.Runtime, cluster *k3d.Cl
}
func EnsureToolsNode(ctx context.Context, runtime runtimes.Runtime, cluster *k3d.Cluster) (*k3d.Node, error) {
var err error
cluster, err = ClusterGet(ctx, runtime, cluster)
if err != nil {
return nil, fmt.Errorf("failed to retrieve cluster '%s': %w", cluster.Name, err)
}
if cluster.Network.Name == "" {
return nil, fmt.Errorf("failed to get network for cluster '%s'", cluster.Name)
}
var imageVolume string
var ok bool
for _, node := range cluster.Nodes {
if node.Role == k3d.ServerRole || node.Role == k3d.AgentRole {
if imageVolume, ok = node.RuntimeLabels[k3d.LabelImageVolume]; ok {
break
}
}
}
if imageVolume == "" {
return nil, fmt.Errorf("Failed to find image volume for cluster '%s'", cluster.Name)
}
l.Log().Debugf("Attaching to cluster's image volume '%s'", imageVolume)
var toolsNode *k3d.Node
toolsNode, err = runtime.GetNode(ctx, &k3d.Node{Name: fmt.Sprintf("%s-%s-tools", k3d.DefaultObjectNamePrefix, cluster.Name)})
toolsNode, err := runtime.GetNode(ctx, &k3d.Node{Name: fmt.Sprintf("%s-%s-tools", k3d.DefaultObjectNamePrefix, cluster.Name)})
if err != nil || toolsNode == nil {
// Get more info on the cluster, if required
var imageVolume string
if cluster.Network.Name == "" || cluster.ImageVolume == "" {
l.Log().Debugf("Gathering some more info about the cluster before creating the tools node...")
var err error
cluster, err = ClusterGet(ctx, runtime, cluster)
if err != nil {
return nil, fmt.Errorf("failed to retrieve cluster: %w", err)
}
if cluster.Network.Name == "" {
return nil, fmt.Errorf("failed to get network for cluster '%s'", cluster.Name)
}
var ok bool
for _, node := range cluster.Nodes {
if node.Role == k3d.ServerRole || node.Role == k3d.AgentRole {
if imageVolume, ok = node.RuntimeLabels[k3d.LabelImageVolume]; ok {
break
}
}
}
if imageVolume == "" {
return nil, fmt.Errorf("Failed to find image volume for cluster '%s'", cluster.Name)
}
l.Log().Debugf("Attaching to cluster's image volume '%s'", imageVolume)
cluster.ImageVolume = imageVolume
}
// start tools node
l.Log().Infoln("Starting new tools node...")
toolsNode, err = runToolsNode(
ctx,
@ -292,7 +298,7 @@ func EnsureToolsNode(ctx context.Context, runtime runtimes.Runtime, cluster *k3d
cluster,
cluster.Network.Name,
[]string{
fmt.Sprintf("%s:%s", imageVolume, k3d.DefaultImageVolumeMountPath),
fmt.Sprintf("%s:%s", cluster.ImageVolume, k3d.DefaultImageVolumeMountPath),
fmt.Sprintf("%s:%s", runtime.GetRuntimePath(), runtime.GetRuntimePath()),
})
if err != nil {

13
pkg/runtimes/docker/host.go
View File

@ -25,19 +25,14 @@ import (
"context"
"fmt"
"net"
"runtime"
)
// GetHostIP returns the IP of the docker host (routable from inside the containers)
func (d Docker) GetHostIP(ctx context.Context, network string) (net.IP, error) {
if runtime.GOOS == "linux" {
ip, err := GetGatewayIP(ctx, network)
if err != nil {
return nil, fmt.Errorf("failed to get gateway IP of docker network '%s': %w", network, err)
}
return ip, nil
ip, err := GetGatewayIP(ctx, network)
if err != nil {
return nil, fmt.Errorf("failed to get gateway IP of docker network '%s': %w", network, err)
}
return nil, fmt.Errorf("Docker Runtime: GetHostIP only implemented for Linux")
return ip, nil
}

2
pkg/types/types.go
View File

@ -28,6 +28,7 @@ import (
"time"
"github.com/docker/go-connections/nat"
runtimeTypes "github.com/rancher/k3d/v5/pkg/runtimes/types"
"github.com/rancher/k3d/v5/pkg/types/k3s"
"github.com/rancher/k3d/v5/version"
"inet.af/netaddr"
@ -424,4 +425,5 @@ type RegistryExternal struct {
type EnvironmentInfo struct {
HostGateway net.IP
RuntimeInfo runtimeTypes.RuntimeInfo
}

7
tools/Dockerfile
View File

@ -1,13 +1,16 @@
FROM golang:1.17 as builder
FROM golang:1.17-alpine as builder
ARG GIT_TAG
WORKDIR /app
COPY . .
RUN apk update && apk add make bash git
ENV GIT_TAG=${GIT_TAG}
ENV GO111MODULE=on
ENV CGO_ENABLED=0
RUN make build
FROM busybox:1.31
FROM alpine:3.14
RUN apk update && apk add bash
WORKDIR /app
COPY --from=builder /app/bin/k3d-tools .
ENTRYPOINT [ "/app/k3d-tools"]