mirrors/k3d
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

[Refactoring/Preparation] use real port structs as prep for registries (#427)

Browse Source
This commit is contained in:
Thorsten Klein 2020-12-11 16:36:10 +01:00 committed by GitHub
parent d042c79df2
commit c44c576d69
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 236 additions and 192 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Makefile
View File

@ -49,6 +49,7 @@ E2E_LOG_LEVEL ?= WARN
E2E_SKIP ?= E2E_SKIP ?=
E2E_EXTRA ?= E2E_EXTRA ?=
E2E_RUNNER_START_TIMEOUT ?= 10 E2E_RUNNER_START_TIMEOUT ?= 10
E2E_HELPER_IMAGE_TAG ?=
########## Go Build Options ########## ########## Go Build Options ##########
# Build targets # Build targets
@ -168,7 +169,7 @@ test:
e2e: build-docker-dind e2e: build-docker-dind
@echo "Running e2e tests in k3d:$(K3D_IMAGE_TAG)" @echo "Running e2e tests in k3d:$(K3D_IMAGE_TAG)"
LOG_LEVEL="$(E2E_LOG_LEVEL)" E2E_SKIP="$(E2E_SKIP)" E2E_EXTRA="$(E2E_EXTRA)" E2E_RUNNER_START_TIMEOUT=$(E2E_RUNNER_START_TIMEOUT) tests/dind.sh "${K3D_IMAGE_TAG}-dind" LOG_LEVEL="$(E2E_LOG_LEVEL)" E2E_SKIP="$(E2E_SKIP)" E2E_EXTRA="$(E2E_EXTRA)" E2E_RUNNER_START_TIMEOUT=$(E2E_RUNNER_START_TIMEOUT) E2E_HELPER_IMAGE_TAG="$(E2E_HELPER_IMAGE_TAG)" tests/dind.sh "${K3D_IMAGE_TAG}-dind"
ci-tests: fmt check e2e ci-tests: fmt check e2e

34
cmd/cluster/clusterCreate.go
View File

@ -51,17 +51,12 @@ Every cluster will consist of one or more containers:
// flags that go through some pre-processing before transforming them to config // flags that go through some pre-processing before transforming them to config
type preProcessedFlags struct { type preProcessedFlags struct {
APIPort string APIPort string
Volumes []string Volumes []string
Ports []string Ports []string
Labels []string Labels []string
Env []string Env []string
} RegistryUse []string
// registry
type registryFlags struct {
Use []string
Create bool
} }
// NewCmdClusterCreate returns a new cobra command // NewCmdClusterCreate returns a new cobra command
@ -70,7 +65,6 @@ func NewCmdClusterCreate() *cobra.Command {
cliConfig := &conf.SimpleConfig{} cliConfig := &conf.SimpleConfig{}
var configFile string var configFile string
ppFlags := &preProcessedFlags{} ppFlags := &preProcessedFlags{}
regFlags := &registryFlags{}
// create new command // create new command
cmd := &cobra.Command{ cmd := &cobra.Command{
@ -213,11 +207,8 @@ func NewCmdClusterCreate() *cobra.Command {
} }
/* Registry */ /* Registry */
cmd.Flags().StringArrayVar(&regFlags.Use, "registry-use", nil, "Connect to one or more registries running locally") cmd.Flags().StringArrayVar(&cliConfig.Registries.Use, "registry-use", nil, "Connect to one or more k3d-managed registries running locally")
if err := cmd.Flags().MarkHidden("registry-use"); err != nil { cmd.Flags().BoolVar(&cliConfig.Registries.Create, "registry-create", false, "Create a k3d-managed registry and connect it to the cluster")
log.Fatalln("Failed to mark flag `cluster create --registry-use` as hidden")
}
cmd.Flags().BoolVar(&cliConfig.Registries.Create, "registry-create", false, "Create a registry and connect it to the cluster")
/* Multi Server Configuration */ /* Multi Server Configuration */
@ -265,11 +256,15 @@ func parseCreateClusterCmd(cmd *cobra.Command, args []string, cliConfig *conf.Si
// -> API-PORT // -> API-PORT
// parse the port mapping // parse the port mapping
exposeAPI, err := cliutil.ParseExposePort(ppFlags.APIPort) exposeAPI, err := cliutil.ParsePortExposureSpec(ppFlags.APIPort, k3d.DefaultAPIPort)
if err != nil { if err != nil {
log.Fatalln(err) log.Fatalln(err)
} }
cliConfig.ExposeAPI = exposeAPI cliConfig.ExposeAPI = conf.SimpleExposureOpts{
Host: exposeAPI.Host,
HostIP: exposeAPI.Binding.HostIP,
HostPort: exposeAPI.Binding.HostPort,
}
// -> VOLUMES // -> VOLUMES
// volumeFilterMap will map volume mounts to applied node filters // volumeFilterMap will map volume mounts to applied node filters
@ -384,5 +379,4 @@ func parseCreateClusterCmd(cmd *cobra.Command, args []string, cliConfig *conf.Si
} }
log.Tracef("EnvFilterMap: %+v", envFilterMap) log.Tracef("EnvFilterMap: %+v", envFilterMap)
} }

4
cmd/registry/registryCreate.go
View File

@ -101,7 +101,7 @@ func parseCreateRegistryCmd(cmd *cobra.Command, args []string, flags *regCreateF
} }
// --port // --port
exposePort, err := cliutil.ParseExposePort(ppFlags.Port) exposePort, err := cliutil.ParsePortExposureSpec(ppFlags.Port, k3d.DefaultRegistryPort)
if err != nil { if err != nil {
log.Errorln("Failed to parse registry port") log.Errorln("Failed to parse registry port")
log.Fatalln(err) log.Fatalln(err)
@ -113,5 +113,5 @@ func parseCreateRegistryCmd(cmd *cobra.Command, args []string, flags *regCreateF
registryName = fmt.Sprintf("%s-%s", k3d.DefaultObjectNamePrefix, args[0]) registryName = fmt.Sprintf("%s-%s", k3d.DefaultObjectNamePrefix, args[0])
} }
return &k3d.Registry{Host: registryName, Image: flags.Image, Port: k3d.MappedPort{InternalPort: k3d.DefaultRegistryPort, ExternalPort: exposePort}}, clusters return &k3d.Registry{Host: registryName, Image: flags.Image, ExposureOpts: *exposePort}, clusters
} }

85
cmd/util/ports.go
View File

@ -24,60 +24,83 @@ package util
import ( import (
"fmt" "fmt"
"net" "net"
"regexp"
"strconv" "strconv"
"strings"
"github.com/docker/go-connections/nat"
k3d "github.com/rancher/k3d/v4/pkg/types" k3d "github.com/rancher/k3d/v4/pkg/types"
"github.com/rancher/k3d/v4/pkg/util"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
) )
// ParseExposePort parses/validates a string to create an exposePort struct from it var apiPortRegexp = regexp.MustCompile(`^(?P<hostref>(?P<hostip>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})|(?P<hostname>\S+):)?(?P<port>(\d{1,5}|random))$`)
func ParseExposePort(portString string) (k3d.ExposedPort, error) {
var exposePort k3d.ExposedPort // ParsePortExposureSpec parses/validates a string to create an exposePort struct from it
func ParsePortExposureSpec(exposedPortSpec, internalPort string) (*k3d.ExposureOpts, error) {
split := strings.Split(portString, ":") match := apiPortRegexp.FindStringSubmatch(exposedPortSpec)
if len(split) > 2 {
log.Errorln("Failed to parse API Port specification") if len(match) == 0 {
return exposePort, fmt.Errorf("api-port format error") log.Errorln("Failed to parse Port Exposure specification")
return nil, fmt.Errorf("Port Exposure Spec format error: Must be [(HostIP|HostName):]HostPort")
} }
if len(split) == 1 { submatches := util.MapSubexpNames(apiPortRegexp.SubexpNames(), match)
exposePort = k3d.ExposedPort{Port: split[0]}
} else { // no port specified (or not matched via regex)
// Make sure 'host' can be resolved to an IP address if submatches["port"] == "" {
addrs, err := net.LookupHost(split[0]) return nil, fmt.Errorf("Failed to find port in Port Exposure spec '%s'", exposedPortSpec)
}
api := &k3d.ExposureOpts{}
// check if there's a host reference
if submatches["hostname"] != "" {
log.Tracef("Port Exposure: found hostname: %s", submatches["hostname"])
addrs, err := net.LookupHost(submatches["hostname"])
if err != nil { if err != nil {
return exposePort, err return nil, fmt.Errorf("Failed to lookup host '%s' specified for Port Exposure: %+v", submatches["hostname"], err)
} }
exposePort = k3d.ExposedPort{Host: split[0], HostIP: addrs[0], Port: split[1]} api.Host = submatches["hostname"]
submatches["hostip"] = addrs[0] // set hostip to the resolved address
} }
// Verify 'port' is an integer and within port ranges realPortString := ""
if exposePort.Port == "" || exposePort.Port == "random" {
log.Debugf("API-Port Mapping didn't specify hostPort, choosing one randomly...") if submatches["hostip"] == "" {
submatches["hostip"] = k3d.DefaultAPIHost
}
// start with the IP, if there is any
if submatches["hostip"] != "" {
realPortString += submatches["hostip"] + ":"
}
// port: get a free one if there's none defined or set to random
if submatches["port"] == "" || submatches["port"] == "random" {
log.Debugf("Port Exposure Mapping didn't specify hostPort, choosing one randomly...")
freePort, err := GetFreePort() freePort, err := GetFreePort()
if err != nil || freePort == 0 { if err != nil || freePort == 0 {
log.Warnf("Failed to get random free port:\n%+v", err) log.Warnf("Failed to get random free port: %+v", err)
log.Warnf("Falling back to default port %s (may be blocked though)...", k3d.DefaultAPIPort) log.Warnf("Falling back to internal port %s (may be blocked though)...", internalPort)
exposePort.Port = k3d.DefaultAPIPort submatches["port"] = internalPort
} else { } else {
exposePort.Port = strconv.Itoa(freePort) submatches["port"] = strconv.Itoa(freePort)
log.Debugf("Got free port for API: '%d'", freePort) log.Debugf("Got free port for Port Exposure: '%d'", freePort)
} }
} }
p, err := strconv.Atoi(exposePort.Port)
realPortString += fmt.Sprintf("%s:%s/tcp", submatches["port"], internalPort)
portMapping, err := nat.ParsePortSpec(realPortString)
if err != nil { if err != nil {
log.Errorln("Failed to parse port mapping") return nil, fmt.Errorf("Failed to parse port spec for Port Exposure '%s': %+v", realPortString, err)
return exposePort, err
} }
if p < 0 || p > 65535 { api.Port = portMapping[0].Port // there can be only one due to our regexp
log.Errorln("Failed to parse API Port specification") api.Binding = portMapping[0].Binding
return exposePort, fmt.Errorf("Port value '%d' out of range", p)
}
return exposePort, nil return api, nil
} }

43
pkg/client/cluster.go
View File

@ -33,6 +33,7 @@ import (
gort "runtime" gort "runtime"
"github.com/docker/go-connections/nat"
"github.com/imdario/mergo" "github.com/imdario/mergo"
"github.com/rancher/k3d/v4/pkg/actions" "github.com/rancher/k3d/v4/pkg/actions"
config "github.com/rancher/k3d/v4/pkg/config/v1alpha1" config "github.com/rancher/k3d/v4/pkg/config/v1alpha1"
@ -276,7 +277,7 @@ ClusterCreatOpts:
/* /*
* Docker Machine Special Configuration * Docker Machine Special Configuration
*/ */
if cluster.ExposeAPI.Host == k3d.DefaultAPIHost && runtime == k3drt.Docker { if cluster.KubeAPI.Host == k3d.DefaultAPIHost && runtime == k3drt.Docker {
if gort.GOOS == "windows" || gort.GOOS == "darwin" { if gort.GOOS == "windows" || gort.GOOS == "darwin" {
log.Tracef("Running on %s: checking if it's using docker-machine", gort.GOOS) log.Tracef("Running on %s: checking if it's using docker-machine", gort.GOOS)
machineIP, err := runtime.(docker.Docker).GetDockerMachineIP() machineIP, err := runtime.(docker.Docker).GetDockerMachineIP()
@ -284,8 +285,8 @@ ClusterCreatOpts:
log.Warnf("Using docker-machine, but failed to get it's IP: %+v", err) log.Warnf("Using docker-machine, but failed to get it's IP: %+v", err)
} else if machineIP != "" { } else if machineIP != "" {
log.Infof("Using the docker-machine IP %s to connect to the Kubernetes API", machineIP) log.Infof("Using the docker-machine IP %s to connect to the Kubernetes API", machineIP)
cluster.ExposeAPI.Host = machineIP cluster.KubeAPI.Host = machineIP
cluster.ExposeAPI.HostIP = machineIP cluster.KubeAPI.Binding.HostIP = machineIP
} else { } else {
log.Traceln("Not using docker-machine") log.Traceln("Not using docker-machine")
} }
@ -330,7 +331,7 @@ ClusterCreatOpts:
// node role specific settings // node role specific settings
if node.Role == k3d.ServerRole { if node.Role == k3d.ServerRole {
node.ServerOpts.ExposeAPI = cluster.ExposeAPI node.ServerOpts.KubeAPI = cluster.KubeAPI
// the cluster has an init server node, but its not this one, so connect it to the init node // the cluster has an init server node, but its not this one, so connect it to the init node
if cluster.InitNode != nil && !node.ServerOpts.IsInit { if cluster.InitNode != nil && !node.ServerOpts.IsInit {
@ -371,7 +372,7 @@ ClusterCreatOpts:
// in case the LoadBalancer was disabled, expose the API Port on the initializing server node // in case the LoadBalancer was disabled, expose the API Port on the initializing server node
if clusterCreateOpts.DisableLoadBalancer { if clusterCreateOpts.DisableLoadBalancer {
cluster.InitNode.Ports = append(cluster.InitNode.Ports, fmt.Sprintf("%s:%s:%s/tcp", cluster.ExposeAPI.Host, cluster.ExposeAPI.Port, k3d.DefaultAPIPort)) cluster.InitNode.Ports[k3d.DefaultAPIPort] = []nat.PortBinding{cluster.KubeAPI.Binding}
} }
if err := nodeSetup(cluster.InitNode, serverCount); err != nil { if err := nodeSetup(cluster.InitNode, serverCount); err != nil {
@ -390,7 +391,7 @@ ClusterCreatOpts:
continue continue
} else if serverCount == 0 && clusterCreateOpts.DisableLoadBalancer { } else if serverCount == 0 && clusterCreateOpts.DisableLoadBalancer {
// if this is the first server node and the server loadbalancer is disabled, expose the API Port on this server node // if this is the first server node and the server loadbalancer is disabled, expose the API Port on this server node
node.Ports = append(node.Ports, fmt.Sprintf("%s:%s:%s/tcp", cluster.ExposeAPI.Host, cluster.ExposeAPI.Port, k3d.DefaultAPIPort)) node.Ports[k3d.DefaultAPIPort] = []nat.PortBinding{cluster.KubeAPI.Binding}
} }
time.Sleep(1 * time.Second) // FIXME: arbitrary wait for one second to avoid race conditions of servers registering time.Sleep(1 * time.Second) // FIXME: arbitrary wait for one second to avoid race conditions of servers registering
@ -431,34 +432,20 @@ ClusterCreatOpts:
// generate comma-separated list of extra ports to forward // generate comma-separated list of extra ports to forward
ports := k3d.DefaultAPIPort ports := k3d.DefaultAPIPort
for _, portString := range cluster.ServerLoadBalancer.Ports { for exposedPort := range cluster.ServerLoadBalancer.Ports {
split := strings.Split(portString, ":") ports += "," + exposedPort.Port()
port := split[len(split)-1]
if strings.Contains(port, "-") {
split := strings.Split(port, "-")
start, err := strconv.Atoi(split[0])
if err != nil {
log.Errorf("Failed to parse port mapping for loadbalancer '%s'", port)
return err
}
end, err := strconv.Atoi(split[1])
if err != nil {
log.Errorf("Failed to parse port mapping for loadbalancer '%s'", port)
return err
}
for i := start; i <= end; i++ {
ports += "," + strconv.Itoa(i)
}
} else {
ports += "," + port
}
} }
if cluster.ServerLoadBalancer.Ports == nil {
cluster.ServerLoadBalancer.Ports = nat.PortMap{}
}
cluster.ServerLoadBalancer.Ports[k3d.DefaultAPIPort] = []nat.PortBinding{cluster.KubeAPI.Binding}
// Create LB as a modified node with loadbalancerRole // Create LB as a modified node with loadbalancerRole
lbNode := &k3d.Node{ lbNode := &k3d.Node{
Name: fmt.Sprintf("%s-%s-serverlb", k3d.DefaultObjectNamePrefix, cluster.Name), Name: fmt.Sprintf("%s-%s-serverlb", k3d.DefaultObjectNamePrefix, cluster.Name),
Image: fmt.Sprintf("%s:%s", k3d.DefaultLBImageRepo, version.GetHelperImageVersion()), Image: fmt.Sprintf("%s:%s", k3d.DefaultLBImageRepo, version.GetHelperImageVersion()),
Ports: append(cluster.ServerLoadBalancer.Ports, fmt.Sprintf("%s:%s:%s/tcp", cluster.ExposeAPI.Host, cluster.ExposeAPI.Port, k3d.DefaultAPIPort)), Ports: cluster.ServerLoadBalancer.Ports,
Env: []string{ Env: []string{
fmt.Sprintf("SERVERS=%s", servers), fmt.Sprintf("SERVERS=%s", servers),
fmt.Sprintf("PORTS=%s", ports), fmt.Sprintf("PORTS=%s", ports),

8
pkg/client/node.go
View File

@ -330,11 +330,11 @@ func patchServerSpec(node *k3d.Node) error {
// Add labels and TLS SAN for the exposed API // Add labels and TLS SAN for the exposed API
// FIXME: For now, the labels concerning the API on the server nodes are only being used for configuring the kubeconfig // FIXME: For now, the labels concerning the API on the server nodes are only being used for configuring the kubeconfig
node.Labels[k3d.LabelServerAPIHostIP] = node.ServerOpts.ExposeAPI.HostIP // TODO: maybe get docker machine IP here node.Labels[k3d.LabelServerAPIHostIP] = node.ServerOpts.KubeAPI.Binding.HostIP // TODO: maybe get docker machine IP here
node.Labels[k3d.LabelServerAPIHost] = node.ServerOpts.ExposeAPI.Host node.Labels[k3d.LabelServerAPIHost] = node.ServerOpts.KubeAPI.Host
node.Labels[k3d.LabelServerAPIPort] = node.ServerOpts.ExposeAPI.Port node.Labels[k3d.LabelServerAPIPort] = node.ServerOpts.KubeAPI.Binding.HostPort
node.Args = append(node.Args, "--tls-san", node.ServerOpts.ExposeAPI.Host) // add TLS SAN for non default host name node.Args = append(node.Args, "--tls-san", node.ServerOpts.KubeAPI.Host) // add TLS SAN for non default host name
return nil return nil
} }

37
pkg/client/registry.go
View File

@ -25,6 +25,7 @@ import (
"context" "context"
"fmt" "fmt"
"github.com/docker/go-connections/nat"
"github.com/rancher/k3d/v4/pkg/runtimes" "github.com/rancher/k3d/v4/pkg/runtimes"
k3d "github.com/rancher/k3d/v4/pkg/types" k3d "github.com/rancher/k3d/v4/pkg/types"
"github.com/rancher/k3d/v4/pkg/types/k3s" "github.com/rancher/k3d/v4/pkg/types/k3s"
@ -71,19 +72,19 @@ func RegistryCreate(ctx context.Context, runtime runtimes.Runtime, reg *k3d.Regi
// setup the node labels // setup the node labels
registryNode.Labels = map[string]string{ registryNode.Labels = map[string]string{
k3d.LabelRole: string(k3d.RegistryRole), k3d.LabelRole: string(k3d.RegistryRole),
k3d.LabelRegistryHost: reg.Port.ExternalPort.Host, // TODO: docker machine host? k3d.LabelRegistryHost: reg.ExposureOpts.Host, // TODO: docker machine host?
k3d.LabelRegistryHostIP: reg.Port.ExternalPort.HostIP, k3d.LabelRegistryHostIP: reg.ExposureOpts.Binding.HostIP,
k3d.LabelRegistryPort: reg.Port.ExternalPort.Port, k3d.LabelRegistryPortExternal: reg.ExposureOpts.Binding.HostPort,
k3d.LabelRegistryPortInternal: reg.ExposureOpts.Port.Port(),
} }
for k, v := range k3d.DefaultObjectLabels { for k, v := range k3d.DefaultObjectLabels {
registryNode.Labels[k] = v registryNode.Labels[k] = v
} }
// port // port
registryNode.Ports = []string{ registryNode.Ports = nat.PortMap{}
fmt.Sprintf("%s:%s:%s/tcp", reg.Port.ExternalPort.HostIP, reg.Port.ExternalPort.Port, k3d.DefaultRegistryPort), registryNode.Ports[reg.ExposureOpts.Port] = []nat.PortBinding{reg.ExposureOpts.Binding}
}
// create the registry node // create the registry node
log.Infof("Creating node '%s'", registryNode.Name) log.Infof("Creating node '%s'", registryNode.Name)
@ -136,8 +137,8 @@ func RegistryGenerateK3sConfig(ctx context.Context, registries []*k3d.Registry)
regConf := &k3s.Registry{} regConf := &k3s.Registry{}
for _, reg := range registries { for _, reg := range registries {
internalAddress := fmt.Sprintf("%s:%s", reg.Host, reg.Port.InternalPort) internalAddress := fmt.Sprintf("%s:%s", reg.Host, reg.ExposureOpts.Port.Port())
externalAddress := fmt.Sprintf("%s:%s", reg.Host, reg.Port.ExternalPort.Port) externalAddress := fmt.Sprintf("%s:%s", reg.Host, reg.ExposureOpts.Binding.HostPort)
// init mirrors if nil // init mirrors if nil
if regConf.Mirrors == nil { if regConf.Mirrors == nil {
@ -159,3 +160,21 @@ func RegistryGenerateK3sConfig(ctx context.Context, registries []*k3d.Registry)
return regConf, nil return regConf, nil
} }
// RegistryGet gets a registry node by name and returns it as a registry object
func RegistryGet(ctx context.Context, runtime runtimes.Runtime, name string) (*k3d.Registry, error) {
regNode, err := runtime.GetNode(ctx, &k3d.Node{
Name: name,
Role: k3d.RegistryRole,
})
if err != nil {
return nil, fmt.Errorf("Failed to find registry '%s': %+v", name, err)
}
registry := &k3d.Registry{
Host: regNode.Name,
}
// TODO: finish RegistryGet
return registry, nil
}

17
pkg/config/config_test.go
View File

@ -33,19 +33,20 @@ import (
func TestReadSimpleConfig(t *testing.T) { func TestReadSimpleConfig(t *testing.T) {
exposedAPI := conf.SimpleExposureOpts{}
exposedAPI.HostIP = "0.0.0.0"
exposedAPI.HostPort = "6443"
expectedConfig := conf.SimpleConfig{ expectedConfig := conf.SimpleConfig{
TypeMeta: conf.TypeMeta{ TypeMeta: conf.TypeMeta{
APIVersion: "k3d.io/v1alpha1", APIVersion: "k3d.io/v1alpha1",
Kind: "Simple", Kind: "Simple",
}, },
Name: "test", Name: "test",
Servers: 1, Servers: 1,
Agents: 2, Agents: 2,
ExposeAPI: k3d.ExposedPort{ ExposeAPI: exposedAPI,
HostIP: "0.0.0.0", Image: "rancher/k3s:latest",
Port: "6443",
},
Image: "rancher/k3s:latest",
Volumes: []conf.VolumeWithNodeFilters{ Volumes: []conf.VolumeWithNodeFilters{
{ {
Volume: "/my/path:/some/path", Volume: "/my/path:/some/path",

4
pkg/config/test_assets/config_test_simple.yaml
View File

@ -3,9 +3,9 @@ kind: Simple
name: test name: test
servers: 1 servers: 1
agents: 2 agents: 2
exposeAPI: kubeAPI:
hostIP: "0.0.0.0" hostIP: "0.0.0.0"
port: "6443" hostPort: "6443"
image: rancher/k3s:latest image: rancher/k3s:latest
volumes: volumes:
- volume: /my/path:/some/path - volume: /my/path:/some/path

50
pkg/config/transform.go
View File

@ -26,6 +26,7 @@ import (
"context" "context"
"fmt" "fmt"
"github.com/docker/go-connections/nat"
cliutil "github.com/rancher/k3d/v4/cmd/util" // TODO: move parseapiport to pkg cliutil "github.com/rancher/k3d/v4/cmd/util" // TODO: move parseapiport to pkg
conf "github.com/rancher/k3d/v4/pkg/config/v1alpha1" conf "github.com/rancher/k3d/v4/pkg/config/v1alpha1"
"github.com/rancher/k3d/v4/pkg/runtimes" "github.com/rancher/k3d/v4/pkg/runtimes"
@ -61,12 +62,21 @@ func TransformSimpleToClusterConfig(ctx context.Context, runtime runtimes.Runtim
simpleConfig.ExposeAPI.HostIP = k3d.DefaultAPIHost simpleConfig.ExposeAPI.HostIP = k3d.DefaultAPIHost
} }
kubeAPIExposureOpts := &k3d.ExposureOpts{
Host: simpleConfig.ExposeAPI.Host,
}
kubeAPIExposureOpts.Port = k3d.DefaultAPIPort
kubeAPIExposureOpts.Binding = nat.PortBinding{
HostIP: simpleConfig.ExposeAPI.HostIP,
HostPort: simpleConfig.ExposeAPI.HostPort,
}
// FILL CLUSTER CONFIG // FILL CLUSTER CONFIG
newCluster := k3d.Cluster{ newCluster := k3d.Cluster{
Name: simpleConfig.Name, Name: simpleConfig.Name,
Network: clusterNetwork, Network: clusterNetwork,
Token: simpleConfig.ClusterToken, Token: simpleConfig.ClusterToken,
ExposeAPI: simpleConfig.ExposeAPI, KubeAPI: kubeAPIExposureOpts,
} }
// -> NODES // -> NODES
@ -142,7 +152,20 @@ func TransformSimpleToClusterConfig(ctx context.Context, runtime runtimes.Runtim
} }
for _, node := range nodes { for _, node := range nodes {
node.Ports = append(node.Ports, portWithNodeFilters.Port) portmappings, err := nat.ParsePortSpec(portWithNodeFilters.Port)
if err != nil {
return nil, fmt.Errorf("Failed to parse port spec '%s': %+v", portWithNodeFilters.Port, err)
}
if node.Ports == nil {
node.Ports = nat.PortMap{}
}
for _, pm := range portmappings {
if _, exists := node.Ports[pm.Port]; exists {
node.Ports[pm.Port] = append(node.Ports[pm.Port], pm.Binding)
} else {
node.Ports[pm.Port] = []nat.PortBinding{pm.Binding}
}
}
} }
} }
@ -200,18 +223,15 @@ func TransformSimpleToClusterConfig(ctx context.Context, runtime runtimes.Runtim
/* /*
* Registries * Registries
*/ */
regPort, err := cliutil.ParseExposePort("random")
if err != nil {
return nil, fmt.Errorf("Failed to get port for registry: %+v", err)
}
if simpleConfig.Registries.Create { if simpleConfig.Registries.Create {
regPort, err := cliutil.ParsePortExposureSpec("random", k3d.DefaultRegistryPort)
if err != nil {
return nil, fmt.Errorf("Failed to get port for registry: %+v", err)
}
clusterCreateOpts.Registries.Create = &k3d.Registry{ clusterCreateOpts.Registries.Create = &k3d.Registry{
Host: fmt.Sprintf("%s-%s-registry", k3d.DefaultObjectNamePrefix, newCluster.Name), Host: fmt.Sprintf("%s-%s-registry", k3d.DefaultObjectNamePrefix, newCluster.Name),
Image: fmt.Sprintf("%s:%s", k3d.DefaultRegistryImageRepo, k3d.DefaultRegistryImageTag), Image: fmt.Sprintf("%s:%s", k3d.DefaultRegistryImageRepo, k3d.DefaultRegistryImageTag),
Port: k3d.MappedPort{ ExposureOpts: *regPort,
InternalPort: k3d.DefaultRegistryPort,
ExternalPort: regPort,
},
} }
} }

13
pkg/config/v1alpha1/types.go
View File

@ -116,7 +116,7 @@ type SimpleConfig struct {
Name string `mapstructure:"name" yaml:"name" json:"name,omitempty"` Name string `mapstructure:"name" yaml:"name" json:"name,omitempty"`
Servers int `mapstructure:"servers" yaml:"servers" json:"servers,omitempty"` //nolint:lll // default 1 Servers int `mapstructure:"servers" yaml:"servers" json:"servers,omitempty"` //nolint:lll // default 1
Agents int `mapstructure:"agents" yaml:"agents" json:"agents,omitempty"` //nolint:lll // default 0 Agents int `mapstructure:"agents" yaml:"agents" json:"agents,omitempty"` //nolint:lll // default 0
ExposeAPI k3d.ExposedPort `mapstructure:"exposeAPI" yaml:"exposeAPI" json:"exposeAPI,omitempty"` ExposeAPI SimpleExposureOpts `mapstructure:"kubeAPI" yaml:"kubeAPI" json:"kubeAPI,omitempty"`
Image string `mapstructure:"image" yaml:"image" json:"image,omitempty"` Image string `mapstructure:"image" yaml:"image" json:"image,omitempty"`
Network string `mapstructure:"network" yaml:"network" json:"network,omitempty"` Network string `mapstructure:"network" yaml:"network" json:"network,omitempty"`
ClusterToken string `mapstructure:"clusterToken" yaml:"clusterToken" json:"clusterToken,omitempty"` // default: auto-generated ClusterToken string `mapstructure:"clusterToken" yaml:"clusterToken" json:"clusterToken,omitempty"` // default: auto-generated
@ -126,11 +126,18 @@ type SimpleConfig struct {
Options SimpleConfigOptions `mapstructure:"options" yaml:"options" json:"options,omitempty"` Options SimpleConfigOptions `mapstructure:"options" yaml:"options" json:"options,omitempty"`
Env []EnvVarWithNodeFilters `mapstructure:"env" yaml:"env" json:"env,omitempty"` Env []EnvVarWithNodeFilters `mapstructure:"env" yaml:"env" json:"env,omitempty"`
Registries struct { Registries struct {
Use []*k3d.Registry `mapstructure:"use" yaml:"use,omitempty" json:"use,omitempty"` Use []string `mapstructure:"use" yaml:"use,omitempty" json:"use,omitempty"`
Create bool `mapstructure:"create" yaml:"create,omitempty" json:"create,omitempty"` Create bool `mapstructure:"create" yaml:"create,omitempty" json:"create,omitempty"`
} `mapstructure:"registries" yaml:"registries,omitempty" json:"registries,omitempty"` } `mapstructure:"registries" yaml:"registries,omitempty" json:"registries,omitempty"`
} }
// SimpleExposureOpts provides a simplified syntax compared to the original k3d.ExposureOpts
type SimpleExposureOpts struct {
Host string `mapstructure:"host" yaml:"host,omitempty" json:"host,omitempty"`
HostIP string `mapstructure:"hostIP" yaml:"hostIP,omitempty" json:"hostIP,omitempty"`
HostPort string `mapstructure:"hostPort" yaml:"hostPort,omitempty" json:"hostPort,omitempty"`
}
// GetKind implements Config.GetKind // GetKind implements Config.GetKind
func (c SimpleConfig) GetKind() string { func (c SimpleConfig) GetKind() string {
return "Cluster" return "Cluster"

9
pkg/config/validate.go
View File

@ -57,7 +57,7 @@ func ValidateClusterConfig(ctx context.Context, runtime runtimes.Runtime, config
} }
// API-Port cannot be changed when using network=host // API-Port cannot be changed when using network=host
if config.Cluster.Network.Name == "host" && config.Cluster.ExposeAPI.Port != k3d.DefaultAPIPort { if config.Cluster.Network.Name == "host" && config.Cluster.KubeAPI.Port.Port() != k3d.DefaultAPIPort {
// in hostNetwork mode, we're not going to map a hostport. Here it should always use 6443. // in hostNetwork mode, we're not going to map a hostport. Here it should always use 6443.
// Note that hostNetwork mode is super inflexible and since we don't change the backend port (on the container), it will only be one hostmode cluster allowed. // Note that hostNetwork mode is super inflexible and since we don't change the backend port (on the container), it will only be one hostmode cluster allowed.
return fmt.Errorf("The API Port can not be changed when using 'host' network") return fmt.Errorf("The API Port can not be changed when using 'host' network")
@ -78,13 +78,6 @@ func ValidateClusterConfig(ctx context.Context, runtime runtimes.Runtime, config
return err return err
} }
} }
// ports
for _, port := range node.Ports {
if err := util.ValidatePortMap(port); err != nil {
return err
}
}
} }
return nil return nil

28
pkg/runtimes/docker/translate.go
View File

@ -94,13 +94,14 @@ func TranslateNodeToContainer(node *k3d.Node) (*NodeInDocker, error) {
// containerConfig.Volumes = map[string]struct{}{} // TODO: do we need this? We only used binds before // containerConfig.Volumes = map[string]struct{}{} // TODO: do we need this? We only used binds before
/* Ports */ /* Ports */
exposedPorts, portBindings, err := nat.ParsePortSpecs(node.Ports) exposedPorts := nat.PortSet{}
if err != nil { for ep := range node.Ports {
log.Errorf("Failed to parse port specs '%v'", node.Ports) if _, exists := exposedPorts[ep]; !exists {
return nil, err exposedPorts[ep] = struct{}{}
}
} }
containerConfig.ExposedPorts = exposedPorts containerConfig.ExposedPorts = exposedPorts
hostConfig.PortBindings = portBindings hostConfig.PortBindings = node.Ports
/* Network */ /* Network */
networkingConfig.EndpointsConfig = map[string]*network.EndpointSettings{ networkingConfig.EndpointsConfig = map[string]*network.EndpointSettings{
node.Network: {}, node.Network: {},
@ -135,14 +136,6 @@ func TranslateContainerToNode(cont *types.Container) (*k3d.Node, error) {
// TranslateContainerDetailsToNode translates a docker containerJSON object into a k3d node representation // TranslateContainerDetailsToNode translates a docker containerJSON object into a k3d node representation
func TranslateContainerDetailsToNode(containerDetails types.ContainerJSON) (*k3d.Node, error) { func TranslateContainerDetailsToNode(containerDetails types.ContainerJSON) (*k3d.Node, error) {
// translate portMap to string representation
ports := []string{}
for containerPort, portBindingList := range containerDetails.HostConfig.PortBindings {
for _, hostInfo := range portBindingList {
ports = append(ports, fmt.Sprintf("%s:%s:%s", hostInfo.HostIP, hostInfo.HostPort, containerPort))
}
}
// restart -> we only set 'unless-stopped' upon cluster creation // restart -> we only set 'unless-stopped' upon cluster creation
restart := false restart := false
if containerDetails.HostConfig.RestartPolicy.IsAlways() || containerDetails.HostConfig.RestartPolicy.IsUnlessStopped() { if containerDetails.HostConfig.RestartPolicy.IsAlways() || containerDetails.HostConfig.RestartPolicy.IsUnlessStopped() {
@ -159,13 +152,14 @@ func TranslateContainerDetailsToNode(containerDetails types.ContainerJSON) (*k3d
// serverOpts // serverOpts
serverOpts := k3d.ServerOpts{IsInit: false} serverOpts := k3d.ServerOpts{IsInit: false}
serverOpts.KubeAPI = &k3d.ExposureOpts{}
for k, v := range containerDetails.Config.Labels { for k, v := range containerDetails.Config.Labels {
if k == k3d.LabelServerAPIHostIP { if k == k3d.LabelServerAPIHostIP {
serverOpts.ExposeAPI.HostIP = v serverOpts.KubeAPI.Binding.HostIP = v
} else if k == k3d.LabelServerAPIHost { } else if k == k3d.LabelServerAPIHost {
serverOpts.ExposeAPI.Host = v serverOpts.KubeAPI.Host = v
} else if k == k3d.LabelServerAPIPort { } else if k == k3d.LabelServerAPIPort {
serverOpts.ExposeAPI.Port = v serverOpts.KubeAPI.Binding.HostPort = v
} }
} }
@ -199,7 +193,7 @@ func TranslateContainerDetailsToNode(containerDetails types.ContainerJSON) (*k3d
Env: env, Env: env,
Cmd: containerDetails.Config.Cmd, Cmd: containerDetails.Config.Cmd,
Args: []string{}, // empty, since Cmd already contains flags Args: []string{}, // empty, since Cmd already contains flags
Ports: ports, Ports: containerDetails.HostConfig.PortBindings,
Restart: restart, Restart: restart,
Labels: labels, Labels: labels,
Network: clusterNetwork, Network: clusterNetwork,

9
pkg/runtimes/docker/translate_test.go
View File

@ -43,7 +43,14 @@ func TestTranslateNodeToContainer(t *testing.T) {
Env: []string{"TEST_KEY_1=TEST_VAL_1"}, Env: []string{"TEST_KEY_1=TEST_VAL_1"},
Cmd: []string{"server", "--https-listen-port=6443"}, Cmd: []string{"server", "--https-listen-port=6443"},
Args: []string{"--some-boolflag"}, Args: []string{"--some-boolflag"},
Ports: []string{"0.0.0.0:6443:6443/tcp"}, Ports: nat.PortMap{
"6443/tcp": []nat.PortBinding{
{
HostIP: "0.0.0.0",
HostPort: "6443",
},
},
},
Restart: true, Restart: true,
Labels: map[string]string{k3d.LabelRole: string(k3d.ServerRole), "test_key_1": "test_val_1"}, Labels: map[string]string{k3d.LabelRole: string(k3d.ServerRole), "test_key_1": "test_val_1"},
} }

67
pkg/types/types.go
View File

@ -25,6 +25,8 @@ import (
"context" "context"
"fmt" "fmt"
"time" "time"
"github.com/docker/go-connections/nat"
) )
// DefaultClusterName specifies the default name used for newly created clusters // DefaultClusterName specifies the default name used for newly created clusters
@ -106,19 +108,20 @@ var DefaultObjectLabels = map[string]string{
// List of k3d technical label name // List of k3d technical label name
const ( const (
LabelClusterName string = "k3d.cluster" LabelClusterName string = "k3d.cluster"
LabelClusterURL string = "k3d.cluster.url" LabelClusterURL string = "k3d.cluster.url"
LabelClusterToken string = "k3d.cluster.token" LabelClusterToken string = "k3d.cluster.token"
LabelImageVolume string = "k3d.cluster.imageVolume" LabelImageVolume string = "k3d.cluster.imageVolume"
LabelNetworkExternal string = "k3d.cluster.network.external" LabelNetworkExternal string = "k3d.cluster.network.external"
LabelNetwork string = "k3d.cluster.network" LabelNetwork string = "k3d.cluster.network"
LabelRole string = "k3d.role" LabelRole string = "k3d.role"
LabelServerAPIPort string = "k3d.server.api.port" LabelServerAPIPort string = "k3d.server.api.port"
LabelServerAPIHost string = "k3d.server.api.host" LabelServerAPIHost string = "k3d.server.api.host"
LabelServerAPIHostIP string = "k3d.server.api.hostIP" LabelServerAPIHostIP string = "k3d.server.api.hostIP"
LabelRegistryHost string = "k3d.registry.host" LabelRegistryHost string = "k3d.registry.host"
LabelRegistryHostIP string = "k3d.registry.hostIP" LabelRegistryHostIP string = "k3d.registry.hostIP"
LabelRegistryPort string = "k3s.registry.port" LabelRegistryPortExternal string = "k3s.registry.port.external"
LabelRegistryPortInternal string = "k3s.registry.port.internal"
) )
// DefaultRoleCmds maps the node roles to their respective default commands // DefaultRoleCmds maps the node roles to their respective default commands
@ -240,7 +243,7 @@ type Cluster struct {
Nodes []*Node `yaml:"nodes" json:"nodes,omitempty"` Nodes []*Node `yaml:"nodes" json:"nodes,omitempty"`
InitNode *Node // init server node InitNode *Node // init server node
ExternalDatastore *ExternalDatastore `yaml:"externalDatastore,omitempty" json:"externalDatastore,omitempty"` ExternalDatastore *ExternalDatastore `yaml:"externalDatastore,omitempty" json:"externalDatastore,omitempty"`
ExposeAPI ExposedPort `yaml:"exposeAPI" json:"exposeAPI,omitempty"` KubeAPI *ExposureOpts `yaml:"kubeAPI" json:"kubeAPI,omitempty"`
ServerLoadBalancer *Node `yaml:"serverLoadbalancer,omitempty" json:"serverLoadBalancer,omitempty"` ServerLoadBalancer *Node `yaml:"serverLoadbalancer,omitempty" json:"serverLoadBalancer,omitempty"`
ImageVolume string `yaml:"imageVolume" json:"imageVolume,omitempty"` ImageVolume string `yaml:"imageVolume" json:"imageVolume,omitempty"`
} }
@ -294,7 +297,7 @@ type Node struct {
Env []string `yaml:"env" json:"env,omitempty"` Env []string `yaml:"env" json:"env,omitempty"`
Cmd []string // filled automatically based on role Cmd []string // filled automatically based on role
Args []string `yaml:"extraArgs" json:"extraArgs,omitempty"` Args []string `yaml:"extraArgs" json:"extraArgs,omitempty"`
Ports []string `yaml:"portMappings" json:"portMappings,omitempty"` Ports nat.PortMap `yaml:"portMappings" json:"portMappings,omitempty"`
Restart bool `yaml:"restart" json:"restart,omitempty"` Restart bool `yaml:"restart" json:"restart,omitempty"`
Labels map[string]string // filled automatically Labels map[string]string // filled automatically
Network string // filled automatically Network string // filled automatically
@ -307,8 +310,14 @@ type Node struct {
// ServerOpts describes some additional server role specific opts // ServerOpts describes some additional server role specific opts
type ServerOpts struct { type ServerOpts struct {
IsInit bool `yaml:"isInitializingServer" json:"isInitializingServer,omitempty"` IsInit bool `yaml:"isInitializingServer" json:"isInitializingServer,omitempty"`
ExposeAPI ExposedPort // filled automatically KubeAPI *ExposureOpts `yaml:"kubeAPI" json:"kubeAPI"`
}
// ExposureOpts describes settings that the user can set for accessing the Kubernetes API
type ExposureOpts struct {
nat.PortMapping // filled automatically (reference to normal portmapping)
Host string `yaml:"host,omitempty" json:"host,omitempty"`
} }
// ExternalDatastore describes an external datastore used for HA/multi-server clusters // ExternalDatastore describes an external datastore used for HA/multi-server clusters
@ -320,19 +329,6 @@ type ExternalDatastore struct {
Network string `yaml:"network" json:"network,omitempty"` Network string `yaml:"network" json:"network,omitempty"`
} }
// MappedPort combines an internal port mapped to an exposed port
type MappedPort struct {
InternalPort string `yaml:"internal,omitempty" json:"internal,omitempty"`
ExternalPort ExposedPort `yaml:"expose,omitempty" json:"expose,omitempty"`
}
// ExposedPort describes a port exposed on the host system
type ExposedPort struct {
Host string `yaml:"host" json:"host,omitempty"`
HostIP string `yaml:"hostIP" json:"hostIP,omitempty"`
Port string `yaml:"port" json:"port"`
}
// AgentOpts describes some additional agent role specific opts // AgentOpts describes some additional agent role specific opts
type AgentOpts struct{} type AgentOpts struct{}
@ -362,11 +358,12 @@ const (
// Registry describes a k3d-managed registry // Registry describes a k3d-managed registry
type Registry struct { type Registry struct {
ClusterRef string // filled automatically -> if created with a cluster ClusterRef string // filled automatically -> if created with a cluster
Host string `yaml:"host" json:"host"` Protocol string `yaml:"protocol,omitempty" json:"protocol,omitempty"` // default: http
Image string `yaml:"image,omitempty" json:"image,omitempty"` Host string `yaml:"host" json:"host"`
Port MappedPort `yaml:"port" json:"port"` Image string `yaml:"image,omitempty" json:"image,omitempty"`
Options struct { ExposureOpts ExposureOpts `yaml:"expose" json:"expose"`
Options struct {
ConfigFile string `yaml:"configFile,omitempty" json:"configFile,omitempty"` ConfigFile string `yaml:"configFile,omitempty" json:"configFile,omitempty"`
Proxy struct { Proxy struct {
RemoteURL string `yaml:"remoteURL" json:"remoteURL"` RemoteURL string `yaml:"remoteURL" json:"remoteURL"`

5
pkg/util/ports.go
View File

@ -28,11 +28,6 @@ import (
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
) )
// ValidatePortMap validates a port mapping
func ValidatePortMap(portmap string) error {
return nil // TODO: ValidatePortMap: add validation of port mapping
}
// GetFreePort tries to fetch an open port from the OS-Kernel // GetFreePort tries to fetch an open port from the OS-Kernel
func GetFreePort() (int, error) { func GetFreePort() (int, error) {
tcpAddress, err := net.ResolveTCPAddr("tcp", "localhost:0") tcpAddress, err := net.ResolveTCPAddr("tcp", "localhost:0")

12
tests/dind.sh
View File

@ -46,7 +46,13 @@ until docker inspect "$k3de2e" | jq ".[0].State.Running" && docker logs "$k3de2e
done done
# build helper container images # build helper container images
docker exec --workdir /src "$k3de2e" make build-helper-images if [ -z "$E2E_HELPER_IMAGE_TAG" ]; then
docker exec --workdir /src "$k3de2e" make build-helper-images
# execute tests
docker exec "$k3de2e" /src/tests/runner.sh
else
# execute tests
docker exec -e "K3D_HELPER_IMAGE_TAG=$E2E_HELPER_IMAGE_TAG" "$k3de2e" /src/tests/runner.sh
fi
# execute tests
docker exec "$k3de2e" /src/tests/runner.sh