diff --git a/cmd/util/filter.go b/cmd/util/filter.go index 50965303..0e5858a8 100644 --- a/cmd/util/filter.go +++ b/cmd/util/filter.go @@ -30,6 +30,8 @@ import ( k3d "github.com/rancher/k3d/v3/pkg/types" + "github.com/rancher/k3d/v3/pkg/util" + "regexp" ) @@ -99,7 +101,7 @@ func FilterNodes(nodes []*k3d.Node, filters []string) ([]*k3d.Node, error) { } // map capturing group names to submatches - submatches := mapSubexpNames(filterRegexp.SubexpNames(), match) + submatches := util.MapSubexpNames(filterRegexp.SubexpNames(), match) // if one of the filters is 'all', we only return this and drop all others if submatches["group"] == "all" { diff --git a/cmd/util/util.go b/cmd/util/util.go index be1ec9c9..cdf4ba59 100644 --- a/cmd/util/util.go +++ b/cmd/util/util.go @@ -20,13 +20,3 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ package util - -// mapSubexpNames maps regex capturing group names to corresponding matches -func mapSubexpNames(names, matches []string) map[string]string { - //names, matches = names[1:], matches[1:] - nameMatchMap := make(map[string]string, len(matches)) - for index := range names { - nameMatchMap[names[index]] = matches[index] - } - return nameMatchMap -} diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index c3b1d76c..508bf515 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -262,6 +262,23 @@ func ClusterCreate(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clus } } + /* + * Networking Magic + */ + + // add extra host + hostIP, err := GetHostIP(ctx, runtime, cluster) + if err != nil { + return err + } + hostsEntry := fmt.Sprintf("%s %s", hostIP, k3d.DefaultK3dInternalHostRecord) + log.Debugf("Adding extra host entry '%s'", hostsEntry) + for _, node := range cluster.Nodes { + if err := runtime.ExecInNode(ctx, node, []string{"sh", "-c", fmt.Sprintf("echo '%s' >> /etc/hosts", hostsEntry)}); err != nil { + log.Warnf("Failed to add extra entry '%s' to /etc/hosts in node '%s'", hostsEntry, node.Name) + } + } + /* * Auxiliary Containers */ diff --git a/pkg/cluster/host.go b/pkg/cluster/host.go new file mode 100644 index 00000000..a2ab6ef7 --- /dev/null +++ b/pkg/cluster/host.go @@ -0,0 +1,102 @@ +/* +Copyright © 2020 The k3d Author(s) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ +package cluster + +import ( + "bufio" + "context" + "fmt" + "net" + "regexp" + "runtime" + + rt "github.com/rancher/k3d/v3/pkg/runtimes" + k3d "github.com/rancher/k3d/v3/pkg/types" + "github.com/rancher/k3d/v3/pkg/util" + log "github.com/sirupsen/logrus" +) + +var nsLookupAddressRegexp = regexp.MustCompile(`^Address:\s+(?P\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})$`) + +// GetHostIP returns the routable IP address to be able to access services running on the host system from inside the cluster. +// This depends on the Operating System and the chosen Runtime. +func GetHostIP(ctx context.Context, rtime rt.Runtime, cluster *k3d.Cluster) (net.IP, error) { + + // Docker Runtime + if rtime == rt.Docker { + + log.Debugf("Runtime GOOS: %s", runtime.GOOS) + + // "native" Docker on Linux + if runtime.GOOS == "linux" { + ip, err := rtime.GetHostIP(ctx, cluster.Network.Name) + if err != nil { + return nil, err + } + return ip, nil + } + + // Docker (for Desktop) on MacOS or Windows + if runtime.GOOS == "windows" || runtime.GOOS == "darwin" { + ip, err := resolveHostnameFromInside(ctx, rtime, cluster.Nodes[0], "host.docker.internal") + if err != nil { + return nil, err + } + return ip, nil + } + + // Catch all other GOOS cases + return nil, fmt.Errorf("GetHostIP only implemented for Linux, MacOS (Darwin) and Windows") + + } + + // Catch all other runtime selections + return nil, fmt.Errorf("GetHostIP only implemented for the docker runtime") + +} + +func resolveHostnameFromInside(ctx context.Context, rtime rt.Runtime, node *k3d.Node, hostname string) (net.IP, error) { + + logreader, err := rtime.ExecInNodeGetLogs(ctx, node, []string{"sh", "-c", fmt.Sprintf("nslookup %s", hostname)}) + if err != nil { + return nil, err + } + + submatches := map[string]string{} + scanner := bufio.NewScanner(logreader) + for scanner.Scan() { + match := nsLookupAddressRegexp.FindStringSubmatch(scanner.Text()) + if len(match) == 0 { + continue + } + submatches = util.MapSubexpNames(nsLookupAddressRegexp.SubexpNames(), match) + break + } + if _, ok := submatches["ip"]; !ok { + return nil, fmt.Errorf("Failed to read address for '%s' from nslookup response", hostname) + } + + log.Debugf("Hostname '%s' -> Address '%s'", hostname, submatches["ip"]) + + return net.ParseIP(submatches["ip"]), nil + +} diff --git a/pkg/runtimes/containerd/host.go b/pkg/runtimes/containerd/host.go new file mode 100644 index 00000000..091b7172 --- /dev/null +++ b/pkg/runtimes/containerd/host.go @@ -0,0 +1,32 @@ +/* +Copyright © 2020 The k3d Author(s) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ +package containerd + +import ( + "context" + "net" +) + +// GetHostIP returns the IP of the containerd host +func (d Containerd) GetHostIP(ctx context.Context, network string) (net.IP, error) { + return nil, nil +} diff --git a/pkg/runtimes/containerd/node.go b/pkg/runtimes/containerd/node.go index 6589d539..a6fd19a7 100644 --- a/pkg/runtimes/containerd/node.go +++ b/pkg/runtimes/containerd/node.go @@ -23,6 +23,7 @@ THE SOFTWARE. package containerd import ( + "bufio" "context" "io" "time" @@ -126,3 +127,8 @@ func (d Containerd) GetNodeLogs(ctx context.Context, node *k3d.Node, since time. func (d Containerd) ExecInNode(ctx context.Context, node *k3d.Node, cmd []string) error { return nil } + +// ExecInNodeGetLogs execs a command inside a node and returns its logreader +func (d Containerd) ExecInNodeGetLogs(ctx context.Context, node *k3d.Node, cmd []string) (*bufio.Reader, error) { + return nil, nil +} diff --git a/pkg/runtimes/docker/container.go b/pkg/runtimes/docker/container.go index 61b06d07..54038858 100644 --- a/pkg/runtimes/docker/container.go +++ b/pkg/runtimes/docker/container.go @@ -147,7 +147,7 @@ func getNodeContainer(ctx context.Context, node *k3d.Node) (*types.Container, er for k, v := range node.Labels { filters.Add("label", fmt.Sprintf("%s=%s", k, v)) } - // See https://github.com/moby/moby/issues/29997 for explanation around initial / + // See https://github.com/moby/moby/issues/29997 for explanation around initial / filters.Add("name", fmt.Sprintf("^/?%s$", node.Name)) // regex filtering for exact name match containers, err := docker.ContainerList(ctx, types.ContainerListOptions{ diff --git a/pkg/runtimes/docker/host.go b/pkg/runtimes/docker/host.go new file mode 100644 index 00000000..f28cf8c0 --- /dev/null +++ b/pkg/runtimes/docker/host.go @@ -0,0 +1,43 @@ +/* +Copyright © 2020 The k3d Author(s) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ +package docker + +import ( + "context" + "fmt" + "net" + "runtime" +) + +// GetHostIP returns the IP of the docker host (routable from inside the containers) +func (d Docker) GetHostIP(ctx context.Context, network string) (net.IP, error) { + if runtime.GOOS == "linux" { + ip, err := GetGatewayIP(ctx, network) + if err != nil { + return nil, err + } + return ip, nil + } + + return nil, fmt.Errorf("Docker Runtime: GetHostIP only implemented for Linux") + +} diff --git a/pkg/runtimes/docker/network.go b/pkg/runtimes/docker/network.go index 7815aa93..5343d4d9 100644 --- a/pkg/runtimes/docker/network.go +++ b/pkg/runtimes/docker/network.go @@ -23,6 +23,7 @@ package docker import ( "context" + "net" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/filters" @@ -104,3 +105,15 @@ func GetNetwork(ctx context.Context, ID string) (types.NetworkResource, error) { defer docker.Close() return docker.NetworkInspect(ctx, ID, types.NetworkInspectOptions{}) } + +// GetGatewayIP returns the IP of the network gateway +func GetGatewayIP(ctx context.Context, network string) (net.IP, error) { + bridgeNetwork, err := GetNetwork(ctx, network) + if err != nil { + return nil, err + } + + gatewayIP := net.ParseIP(bridgeNetwork.IPAM.Config[0].Gateway) + + return gatewayIP, nil +} diff --git a/pkg/runtimes/docker/node.go b/pkg/runtimes/docker/node.go index 157f08ac..7ec2106e 100644 --- a/pkg/runtimes/docker/node.go +++ b/pkg/runtimes/docker/node.go @@ -23,6 +23,7 @@ THE SOFTWARE. package docker import ( + "bufio" "context" "fmt" "io" @@ -303,22 +304,36 @@ func (d Docker) GetNodeLogs(ctx context.Context, node *k3d.Node, since time.Time return logreader, nil } +// ExecInNodeGetLogs executes a command inside a node and returns the logs to the caller, e.g. to parse them +func (d Docker) ExecInNodeGetLogs(ctx context.Context, node *k3d.Node, cmd []string) (*bufio.Reader, error) { + resp, err := executeInNode(ctx, node, cmd) + if err != nil { + return nil, err + } + return resp.Reader, nil +} + // ExecInNode execs a command inside a node func (d Docker) ExecInNode(ctx context.Context, node *k3d.Node, cmd []string) error { + _, err := executeInNode(ctx, node, cmd) + return err +} + +func executeInNode(ctx context.Context, node *k3d.Node, cmd []string) (*types.HijackedResponse, error) { log.Debugf("Executing command '%+v' in node '%s'", cmd, node.Name) // get the container for the given node container, err := getNodeContainer(ctx, node) if err != nil { - return err + return nil, err } // create docker client docker, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) if err != nil { log.Errorln("Failed to create docker client") - return err + return nil, err } defer docker.Close() @@ -332,7 +347,7 @@ func (d Docker) ExecInNode(ctx context.Context, node *k3d.Node, cmd []string) er }) if err != nil { log.Errorf("Failed to create exec config for node '%s'", node.Name) - return err + return nil, err } execConnection, err := docker.ContainerExecAttach(ctx, exec.ID, types.ExecStartCheck{ @@ -340,13 +355,12 @@ func (d Docker) ExecInNode(ctx context.Context, node *k3d.Node, cmd []string) er }) if err != nil { log.Errorf("Failed to connect to exec process in node '%s'", node.Name) - return err + return nil, err } - defer execConnection.Close() if err := docker.ContainerExecStart(ctx, exec.ID, types.ExecStartCheck{Tty: true}); err != nil { log.Errorf("Failed to start exec process in node '%s'", node.Name) - return err + return nil, err } for { @@ -354,7 +368,7 @@ func (d Docker) ExecInNode(ctx context.Context, node *k3d.Node, cmd []string) er execInfo, err := docker.ContainerExecInspect(ctx, exec.ID) if err != nil { log.Errorf("Failed to inspect exec process in node '%s'", node.Name) - return err + return &execConnection, err } // if still running, continue loop @@ -375,13 +389,14 @@ func (d Docker) ExecInNode(ctx context.Context, node *k3d.Node, cmd []string) er logs, err := ioutil.ReadAll(execConnection.Reader) if err != nil { log.Errorf("Failed to get logs from node '%s'", node.Name) - return err + return &execConnection, err } - return fmt.Errorf("Logs from failed access process:\n%s", string(logs)) + return &execConnection, fmt.Errorf("Logs from failed access process:\n%s", string(logs)) } } - return nil + return &execConnection, nil + } diff --git a/pkg/runtimes/docker/translate.go b/pkg/runtimes/docker/translate.go index 8889bc52..d5e483f4 100644 --- a/pkg/runtimes/docker/translate.go +++ b/pkg/runtimes/docker/translate.go @@ -41,7 +41,8 @@ func TranslateNodeToContainer(node *k3d.Node) (*NodeInDocker, error) { /* initialize everything that we need */ containerConfig := docker.Config{} hostConfig := docker.HostConfig{ - Init: &[]bool{true}[0], + Init: &[]bool{true}[0], + ExtraHosts: node.ExtraHosts, } networkingConfig := network.NetworkingConfig{} diff --git a/pkg/runtimes/runtime.go b/pkg/runtimes/runtime.go index e7456dc8..ebc14674 100644 --- a/pkg/runtimes/runtime.go +++ b/pkg/runtimes/runtime.go @@ -22,9 +22,11 @@ THE SOFTWARE. package runtimes import ( + "bufio" "context" "fmt" "io" + "net" "time" "github.com/rancher/k3d/v3/pkg/runtimes/containerd" @@ -63,9 +65,11 @@ type Runtime interface { GetVolume(string) (string, error) GetRuntimePath() string // returns e.g. '/var/run/docker.sock' for a default docker setup ExecInNode(context.Context, *k3d.Node, []string) error + ExecInNodeGetLogs(context.Context, *k3d.Node, []string) (*bufio.Reader, error) GetNodeLogs(context.Context, *k3d.Node, time.Time) (io.ReadCloser, error) GetImages(context.Context) ([]string, error) CopyToNode(context.Context, string, string, *k3d.Node) error + GetHostIP(context.Context, string) (net.IP, error) } // GetRuntime checks, if a given name is represented by an implemented k3d runtime and returns it diff --git a/pkg/types/types.go b/pkg/types/types.go index d1fa269a..f8441a76 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -108,6 +108,9 @@ var DefaultNodeEnv = []string{ "K3S_KUBECONFIG_OUTPUT=/output/kubeconfig.yaml", } +// DefaultK3dInternalHostRecord defines the default /etc/hosts entry for the k3d host +const DefaultK3dInternalHostRecord = "host.k3d.internal" + // DefaultImageVolumeMountPath defines the mount path inside k3d nodes where we will mount the shared image volume by default const DefaultImageVolumeMountPath = "/k3d/images" @@ -234,6 +237,7 @@ type Node struct { Restart bool `yaml:"restart" json:"restart,omitempty"` Labels map[string]string // filled automatically Network string // filled automatically + ExtraHosts []string // filled automatically ServerOpts ServerOpts `yaml:"server_opts" json:"serverOpts,omitempty"` AgentOpts AgentOpts `yaml:"agent_opts" json:"agentOpts,omitempty"` State NodeState // filled automatically diff --git a/pkg/util/regexp.go b/pkg/util/regexp.go new file mode 100644 index 00000000..a7530a94 --- /dev/null +++ b/pkg/util/regexp.go @@ -0,0 +1,33 @@ +/* +Copyright © 2020 The k3d Author(s) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +package util + +// MapSubexpNames maps regex capturing group names to corresponding matches +func MapSubexpNames(names, matches []string) map[string]string { + //names, matches = names[1:], matches[1:] + nameMatchMap := make(map[string]string, len(matches)) + for index := range names { + nameMatchMap[names[index]] = matches[index] + } + return nameMatchMap +}