mirrors/k3d
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

[Refactoring/Preparation] use real port structs as prep for registries (#427)

Browse Source
This commit is contained in:
Thorsten Klein 2020-12-11 16:36:10 +01:00 committed by GitHub
parent d042c79df2
commit c44c576d69
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 236 additions and 192 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Makefile
View File

@ -49,6 +49,7 @@ E2E_LOG_LEVEL ?= WARN
E2E_SKIP ?=
E2E_EXTRA ?=
E2E_RUNNER_START_TIMEOUT ?= 10
E2E_HELPER_IMAGE_TAG ?=
########## Go Build Options ##########
# Build targets
@ -168,7 +169,7 @@ test:
e2e: build-docker-dind
@echo "Running e2e tests in k3d:$(K3D_IMAGE_TAG)"
LOG_LEVEL="$(E2E_LOG_LEVEL)" E2E_SKIP="$(E2E_SKIP)" E2E_EXTRA="$(E2E_EXTRA)" E2E_RUNNER_START_TIMEOUT=$(E2E_RUNNER_START_TIMEOUT) tests/dind.sh "${K3D_IMAGE_TAG}-dind"
LOG_LEVEL="$(E2E_LOG_LEVEL)" E2E_SKIP="$(E2E_SKIP)" E2E_EXTRA="$(E2E_EXTRA)" E2E_RUNNER_START_TIMEOUT=$(E2E_RUNNER_START_TIMEOUT) E2E_HELPER_IMAGE_TAG="$(E2E_HELPER_IMAGE_TAG)" tests/dind.sh "${K3D_IMAGE_TAG}-dind"
ci-tests: fmt check e2e

34
cmd/cluster/clusterCreate.go
View File

@ -51,17 +51,12 @@ Every cluster will consist of one or more containers:
// flags that go through some pre-processing before transforming them to config
type preProcessedFlags struct {
APIPort string
Volumes []string
Ports []string
Labels []string
Env []string
}
// registry
type registryFlags struct {
Use []string
Create bool
APIPort string
Volumes []string
Ports []string
Labels []string
Env []string
RegistryUse []string
}
// NewCmdClusterCreate returns a new cobra command
@ -70,7 +65,6 @@ func NewCmdClusterCreate() *cobra.Command {
cliConfig := &conf.SimpleConfig{}
var configFile string
ppFlags := &preProcessedFlags{}
regFlags := &registryFlags{}
// create new command
cmd := &cobra.Command{
@ -213,11 +207,8 @@ func NewCmdClusterCreate() *cobra.Command {
}
/* Registry */
cmd.Flags().StringArrayVar(&regFlags.Use, "registry-use", nil, "Connect to one or more registries running locally")
if err := cmd.Flags().MarkHidden("registry-use"); err != nil {
log.Fatalln("Failed to mark flag `cluster create --registry-use` as hidden")
}
cmd.Flags().BoolVar(&cliConfig.Registries.Create, "registry-create", false, "Create a registry and connect it to the cluster")
cmd.Flags().StringArrayVar(&cliConfig.Registries.Use, "registry-use", nil, "Connect to one or more k3d-managed registries running locally")
cmd.Flags().BoolVar(&cliConfig.Registries.Create, "registry-create", false, "Create a k3d-managed registry and connect it to the cluster")
/* Multi Server Configuration */
@ -265,11 +256,15 @@ func parseCreateClusterCmd(cmd *cobra.Command, args []string, cliConfig *conf.Si
// -> API-PORT
// parse the port mapping
exposeAPI, err := cliutil.ParseExposePort(ppFlags.APIPort)
exposeAPI, err := cliutil.ParsePortExposureSpec(ppFlags.APIPort, k3d.DefaultAPIPort)
if err != nil {
log.Fatalln(err)
}
cliConfig.ExposeAPI = exposeAPI
cliConfig.ExposeAPI = conf.SimpleExposureOpts{
Host: exposeAPI.Host,
HostIP: exposeAPI.Binding.HostIP,
HostPort: exposeAPI.Binding.HostPort,
}
// -> VOLUMES
// volumeFilterMap will map volume mounts to applied node filters
@ -384,5 +379,4 @@ func parseCreateClusterCmd(cmd *cobra.Command, args []string, cliConfig *conf.Si
}
log.Tracef("EnvFilterMap: %+v", envFilterMap)
}

4
cmd/registry/registryCreate.go
View File

@ -101,7 +101,7 @@ func parseCreateRegistryCmd(cmd *cobra.Command, args []string, flags *regCreateF
}
// --port
exposePort, err := cliutil.ParseExposePort(ppFlags.Port)
exposePort, err := cliutil.ParsePortExposureSpec(ppFlags.Port, k3d.DefaultRegistryPort)
if err != nil {
log.Errorln("Failed to parse registry port")
log.Fatalln(err)
@ -113,5 +113,5 @@ func parseCreateRegistryCmd(cmd *cobra.Command, args []string, flags *regCreateF
registryName = fmt.Sprintf("%s-%s", k3d.DefaultObjectNamePrefix, args[0])
}
return &k3d.Registry{Host: registryName, Image: flags.Image, Port: k3d.MappedPort{InternalPort: k3d.DefaultRegistryPort, ExternalPort: exposePort}}, clusters
return &k3d.Registry{Host: registryName, Image: flags.Image, ExposureOpts: *exposePort}, clusters
}

85
cmd/util/ports.go
View File

@ -24,60 +24,83 @@ package util
import (
"fmt"
"net"
"regexp"
"strconv"
"strings"
"github.com/docker/go-connections/nat"
k3d "github.com/rancher/k3d/v4/pkg/types"
"github.com/rancher/k3d/v4/pkg/util"
log "github.com/sirupsen/logrus"
)
// ParseExposePort parses/validates a string to create an exposePort struct from it
func ParseExposePort(portString string) (k3d.ExposedPort, error) {
var apiPortRegexp = regexp.MustCompile(`^(?P<hostref>(?P<hostip>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})|(?P<hostname>\S+):)?(?P<port>(\d{1,5}|random))$`)
var exposePort k3d.ExposedPort
// ParsePortExposureSpec parses/validates a string to create an exposePort struct from it
func ParsePortExposureSpec(exposedPortSpec, internalPort string) (*k3d.ExposureOpts, error) {
split := strings.Split(portString, ":")
if len(split) > 2 {
log.Errorln("Failed to parse API Port specification")
return exposePort, fmt.Errorf("api-port format error")
match := apiPortRegexp.FindStringSubmatch(exposedPortSpec)
if len(match) == 0 {
log.Errorln("Failed to parse Port Exposure specification")
return nil, fmt.Errorf("Port Exposure Spec format error: Must be [(HostIP|HostName):]HostPort")
}
if len(split) == 1 {
exposePort = k3d.ExposedPort{Port: split[0]}
} else {
// Make sure 'host' can be resolved to an IP address
addrs, err := net.LookupHost(split[0])
submatches := util.MapSubexpNames(apiPortRegexp.SubexpNames(), match)
// no port specified (or not matched via regex)
if submatches["port"] == "" {
return nil, fmt.Errorf("Failed to find port in Port Exposure spec '%s'", exposedPortSpec)
}
api := &k3d.ExposureOpts{}
// check if there's a host reference
if submatches["hostname"] != "" {
log.Tracef("Port Exposure: found hostname: %s", submatches["hostname"])
addrs, err := net.LookupHost(submatches["hostname"])
if err != nil {
return exposePort, err
return nil, fmt.Errorf("Failed to lookup host '%s' specified for Port Exposure: %+v", submatches["hostname"], err)
}
exposePort = k3d.ExposedPort{Host: split[0], HostIP: addrs[0], Port: split[1]}
api.Host = submatches["hostname"]
submatches["hostip"] = addrs[0] // set hostip to the resolved address
}
// Verify 'port' is an integer and within port ranges
if exposePort.Port == "" || exposePort.Port == "random" {
log.Debugf("API-Port Mapping didn't specify hostPort, choosing one randomly...")
realPortString := ""
if submatches["hostip"] == "" {
submatches["hostip"] = k3d.DefaultAPIHost
}
// start with the IP, if there is any
if submatches["hostip"] != "" {
realPortString += submatches["hostip"] + ":"
}
// port: get a free one if there's none defined or set to random
if submatches["port"] == "" || submatches["port"] == "random" {
log.Debugf("Port Exposure Mapping didn't specify hostPort, choosing one randomly...")
freePort, err := GetFreePort()
if err != nil || freePort == 0 {
log.Warnf("Failed to get random free port:\n%+v", err)
log.Warnf("Falling back to default port %s (may be blocked though)...", k3d.DefaultAPIPort)
exposePort.Port = k3d.DefaultAPIPort
log.Warnf("Failed to get random free port: %+v", err)
log.Warnf("Falling back to internal port %s (may be blocked though)...", internalPort)
submatches["port"] = internalPort
} else {
exposePort.Port = strconv.Itoa(freePort)
log.Debugf("Got free port for API: '%d'", freePort)
submatches["port"] = strconv.Itoa(freePort)
log.Debugf("Got free port for Port Exposure: '%d'", freePort)
}
}
p, err := strconv.Atoi(exposePort.Port)
realPortString += fmt.Sprintf("%s:%s/tcp", submatches["port"], internalPort)
portMapping, err := nat.ParsePortSpec(realPortString)
if err != nil {
log.Errorln("Failed to parse port mapping")
return exposePort, err
return nil, fmt.Errorf("Failed to parse port spec for Port Exposure '%s': %+v", realPortString, err)
}
if p < 0 || p > 65535 {
log.Errorln("Failed to parse API Port specification")
return exposePort, fmt.Errorf("Port value '%d' out of range", p)
}
api.Port = portMapping[0].Port // there can be only one due to our regexp
api.Binding = portMapping[0].Binding
return exposePort, nil
return api, nil
}

43
pkg/client/cluster.go
View File

@ -33,6 +33,7 @@ import (
gort "runtime"
"github.com/docker/go-connections/nat"
"github.com/imdario/mergo"
"github.com/rancher/k3d/v4/pkg/actions"
config "github.com/rancher/k3d/v4/pkg/config/v1alpha1"
@ -276,7 +277,7 @@ ClusterCreatOpts:
/*
* Docker Machine Special Configuration
*/
if cluster.ExposeAPI.Host == k3d.DefaultAPIHost && runtime == k3drt.Docker {
if cluster.KubeAPI.Host == k3d.DefaultAPIHost && runtime == k3drt.Docker {
if gort.GOOS == "windows" || gort.GOOS == "darwin" {
log.Tracef("Running on %s: checking if it's using docker-machine", gort.GOOS)
machineIP, err := runtime.(docker.Docker).GetDockerMachineIP()
@ -284,8 +285,8 @@ ClusterCreatOpts:
log.Warnf("Using docker-machine, but failed to get it's IP: %+v", err)
} else if machineIP != "" {
log.Infof("Using the docker-machine IP %s to connect to the Kubernetes API", machineIP)
cluster.ExposeAPI.Host = machineIP
cluster.ExposeAPI.HostIP = machineIP
cluster.KubeAPI.Host = machineIP
cluster.KubeAPI.Binding.HostIP = machineIP
} else {
log.Traceln("Not using docker-machine")
}
@ -330,7 +331,7 @@ ClusterCreatOpts:
// node role specific settings
if node.Role == k3d.ServerRole {
node.ServerOpts.ExposeAPI = cluster.ExposeAPI
node.ServerOpts.KubeAPI = cluster.KubeAPI
// the cluster has an init server node, but its not this one, so connect it to the init node
if cluster.InitNode != nil && !node.ServerOpts.IsInit {
@ -371,7 +372,7 @@ ClusterCreatOpts:
// in case the LoadBalancer was disabled, expose the API Port on the initializing server node
if clusterCreateOpts.DisableLoadBalancer {
cluster.InitNode.Ports = append(cluster.InitNode.Ports, fmt.Sprintf("%s:%s:%s/tcp", cluster.ExposeAPI.Host, cluster.ExposeAPI.Port, k3d.DefaultAPIPort))
cluster.InitNode.Ports[k3d.DefaultAPIPort] = []nat.PortBinding{cluster.KubeAPI.Binding}
}
if err := nodeSetup(cluster.InitNode, serverCount); err != nil {
@ -390,7 +391,7 @@ ClusterCreatOpts:
continue
} else if serverCount == 0 && clusterCreateOpts.DisableLoadBalancer {
// if this is the first server node and the server loadbalancer is disabled, expose the API Port on this server node
node.Ports = append(node.Ports, fmt.Sprintf("%s:%s:%s/tcp", cluster.ExposeAPI.Host, cluster.ExposeAPI.Port, k3d.DefaultAPIPort))
node.Ports[k3d.DefaultAPIPort] = []nat.PortBinding{cluster.KubeAPI.Binding}
}
time.Sleep(1 * time.Second) // FIXME: arbitrary wait for one second to avoid race conditions of servers registering
@ -431,34 +432,20 @@ ClusterCreatOpts:
// generate comma-separated list of extra ports to forward
ports := k3d.DefaultAPIPort
for _, portString := range cluster.ServerLoadBalancer.Ports {
split := strings.Split(portString, ":")
port := split[len(split)-1]
if strings.Contains(port, "-") {
split := strings.Split(port, "-")
start, err := strconv.Atoi(split[0])
if err != nil {
log.Errorf("Failed to parse port mapping for loadbalancer '%s'", port)
return err
}
end, err := strconv.Atoi(split[1])
if err != nil {
log.Errorf("Failed to parse port mapping for loadbalancer '%s'", port)
return err
}
for i := start; i <= end; i++ {
ports += "," + strconv.Itoa(i)
}
} else {
ports += "," + port
}
for exposedPort := range cluster.ServerLoadBalancer.Ports {
ports += "," + exposedPort.Port()
}
if cluster.ServerLoadBalancer.Ports == nil {
cluster.ServerLoadBalancer.Ports = nat.PortMap{}
}
cluster.ServerLoadBalancer.Ports[k3d.DefaultAPIPort] = []nat.PortBinding{cluster.KubeAPI.Binding}
// Create LB as a modified node with loadbalancerRole
lbNode := &k3d.Node{
Name: fmt.Sprintf("%s-%s-serverlb", k3d.DefaultObjectNamePrefix, cluster.Name),
Image: fmt.Sprintf("%s:%s", k3d.DefaultLBImageRepo, version.GetHelperImageVersion()),
Ports: append(cluster.ServerLoadBalancer.Ports, fmt.Sprintf("%s:%s:%s/tcp", cluster.ExposeAPI.Host, cluster.ExposeAPI.Port, k3d.DefaultAPIPort)),
Ports: cluster.ServerLoadBalancer.Ports,
Env: []string{
fmt.Sprintf("SERVERS=%s", servers),
fmt.Sprintf("PORTS=%s", ports),

8
pkg/client/node.go
View File

@ -330,11 +330,11 @@ func patchServerSpec(node *k3d.Node) error {
// Add labels and TLS SAN for the exposed API
// FIXME: For now, the labels concerning the API on the server nodes are only being used for configuring the kubeconfig
node.Labels[k3d.LabelServerAPIHostIP] = node.ServerOpts.ExposeAPI.HostIP // TODO: maybe get docker machine IP here
node.Labels[k3d.LabelServerAPIHost] = node.ServerOpts.ExposeAPI.Host
node.Labels[k3d.LabelServerAPIPort] = node.ServerOpts.ExposeAPI.Port
node.Labels[k3d.LabelServerAPIHostIP] = node.ServerOpts.KubeAPI.Binding.HostIP // TODO: maybe get docker machine IP here
node.Labels[k3d.LabelServerAPIHost] = node.ServerOpts.KubeAPI.Host
node.Labels[k3d.LabelServerAPIPort] = node.ServerOpts.KubeAPI.Binding.HostPort
node.Args = append(node.Args, "--tls-san", node.ServerOpts.ExposeAPI.Host) // add TLS SAN for non default host name
node.Args = append(node.Args, "--tls-san", node.ServerOpts.KubeAPI.Host) // add TLS SAN for non default host name
return nil
}

37
pkg/client/registry.go
View File

@ -25,6 +25,7 @@ import (
"context"
"fmt"
"github.com/docker/go-connections/nat"
"github.com/rancher/k3d/v4/pkg/runtimes"
k3d "github.com/rancher/k3d/v4/pkg/types"
"github.com/rancher/k3d/v4/pkg/types/k3s"
@ -71,19 +72,19 @@ func RegistryCreate(ctx context.Context, runtime runtimes.Runtime, reg *k3d.Regi
// setup the node labels
registryNode.Labels = map[string]string{
k3d.LabelRole: string(k3d.RegistryRole),
k3d.LabelRegistryHost: reg.Port.ExternalPort.Host, // TODO: docker machine host?
k3d.LabelRegistryHostIP: reg.Port.ExternalPort.HostIP,
k3d.LabelRegistryPort: reg.Port.ExternalPort.Port,
k3d.LabelRole: string(k3d.RegistryRole),
k3d.LabelRegistryHost: reg.ExposureOpts.Host, // TODO: docker machine host?
k3d.LabelRegistryHostIP: reg.ExposureOpts.Binding.HostIP,
k3d.LabelRegistryPortExternal: reg.ExposureOpts.Binding.HostPort,
k3d.LabelRegistryPortInternal: reg.ExposureOpts.Port.Port(),
}
for k, v := range k3d.DefaultObjectLabels {
registryNode.Labels[k] = v
}
// port
registryNode.Ports = []string{
fmt.Sprintf("%s:%s:%s/tcp", reg.Port.ExternalPort.HostIP, reg.Port.ExternalPort.Port, k3d.DefaultRegistryPort),
}
registryNode.Ports = nat.PortMap{}
registryNode.Ports[reg.ExposureOpts.Port] = []nat.PortBinding{reg.ExposureOpts.Binding}
// create the registry node
log.Infof("Creating node '%s'", registryNode.Name)
@ -136,8 +137,8 @@ func RegistryGenerateK3sConfig(ctx context.Context, registries []*k3d.Registry)
regConf := &k3s.Registry{}
for _, reg := range registries {
internalAddress := fmt.Sprintf("%s:%s", reg.Host, reg.Port.InternalPort)
externalAddress := fmt.Sprintf("%s:%s", reg.Host, reg.Port.ExternalPort.Port)
internalAddress := fmt.Sprintf("%s:%s", reg.Host, reg.ExposureOpts.Port.Port())
externalAddress := fmt.Sprintf("%s:%s", reg.Host, reg.ExposureOpts.Binding.HostPort)
// init mirrors if nil
if regConf.Mirrors == nil {
@ -159,3 +160,21 @@ func RegistryGenerateK3sConfig(ctx context.Context, registries []*k3d.Registry)
return regConf, nil
}
// RegistryGet gets a registry node by name and returns it as a registry object
func RegistryGet(ctx context.Context, runtime runtimes.Runtime, name string) (*k3d.Registry, error) {
regNode, err := runtime.GetNode(ctx, &k3d.Node{
Name: name,
Role: k3d.RegistryRole,
})
if err != nil {
return nil, fmt.Errorf("Failed to find registry '%s': %+v", name, err)
}
registry := &k3d.Registry{
Host: regNode.Name,
}
// TODO: finish RegistryGet
return registry, nil
}

17
pkg/config/config_test.go
View File

@ -33,19 +33,20 @@ import (
func TestReadSimpleConfig(t *testing.T) {
exposedAPI := conf.SimpleExposureOpts{}
exposedAPI.HostIP = "0.0.0.0"
exposedAPI.HostPort = "6443"
expectedConfig := conf.SimpleConfig{
TypeMeta: conf.TypeMeta{
APIVersion: "k3d.io/v1alpha1",
Kind: "Simple",
},
Name: "test",
Servers: 1,
Agents: 2,
ExposeAPI: k3d.ExposedPort{
HostIP: "0.0.0.0",
Port: "6443",
},
Image: "rancher/k3s:latest",
Name: "test",
Servers: 1,
Agents: 2,
ExposeAPI: exposedAPI,
Image: "rancher/k3s:latest",
Volumes: []conf.VolumeWithNodeFilters{
{
Volume: "/my/path:/some/path",

4
pkg/config/test_assets/config_test_simple.yaml
View File

@ -3,9 +3,9 @@ kind: Simple
name: test
servers: 1
agents: 2
exposeAPI:
kubeAPI:
hostIP: "0.0.0.0"
port: "6443"
hostPort: "6443"
image: rancher/k3s:latest
volumes:
- volume: /my/path:/some/path

50
pkg/config/transform.go
View File

@ -26,6 +26,7 @@ import (
"context"
"fmt"
"github.com/docker/go-connections/nat"
cliutil "github.com/rancher/k3d/v4/cmd/util" // TODO: move parseapiport to pkg
conf "github.com/rancher/k3d/v4/pkg/config/v1alpha1"
"github.com/rancher/k3d/v4/pkg/runtimes"
@ -61,12 +62,21 @@ func TransformSimpleToClusterConfig(ctx context.Context, runtime runtimes.Runtim
simpleConfig.ExposeAPI.HostIP = k3d.DefaultAPIHost
}
kubeAPIExposureOpts := &k3d.ExposureOpts{
Host: simpleConfig.ExposeAPI.Host,
}
kubeAPIExposureOpts.Port = k3d.DefaultAPIPort
kubeAPIExposureOpts.Binding = nat.PortBinding{
HostIP: simpleConfig.ExposeAPI.HostIP,
HostPort: simpleConfig.ExposeAPI.HostPort,
}
// FILL CLUSTER CONFIG
newCluster := k3d.Cluster{
Name: simpleConfig.Name,
Network: clusterNetwork,
Token: simpleConfig.ClusterToken,
ExposeAPI: simpleConfig.ExposeAPI,
Name: simpleConfig.Name,
Network: clusterNetwork,
Token: simpleConfig.ClusterToken,
KubeAPI: kubeAPIExposureOpts,
}
// -> NODES
@ -142,7 +152,20 @@ func TransformSimpleToClusterConfig(ctx context.Context, runtime runtimes.Runtim
}
for _, node := range nodes {
node.Ports = append(node.Ports, portWithNodeFilters.Port)
portmappings, err := nat.ParsePortSpec(portWithNodeFilters.Port)
if err != nil {
return nil, fmt.Errorf("Failed to parse port spec '%s': %+v", portWithNodeFilters.Port, err)
}
if node.Ports == nil {
node.Ports = nat.PortMap{}
}
for _, pm := range portmappings {
if _, exists := node.Ports[pm.Port]; exists {
node.Ports[pm.Port] = append(node.Ports[pm.Port], pm.Binding)
} else {
node.Ports[pm.Port] = []nat.PortBinding{pm.Binding}
}
}
}
}
@ -200,18 +223,15 @@ func TransformSimpleToClusterConfig(ctx context.Context, runtime runtimes.Runtim
/*
* Registries
*/
regPort, err := cliutil.ParseExposePort("random")
if err != nil {
return nil, fmt.Errorf("Failed to get port for registry: %+v", err)
}
if simpleConfig.Registries.Create {
regPort, err := cliutil.ParsePortExposureSpec("random", k3d.DefaultRegistryPort)
if err != nil {
return nil, fmt.Errorf("Failed to get port for registry: %+v", err)
}
clusterCreateOpts.Registries.Create = &k3d.Registry{
Host: fmt.Sprintf("%s-%s-registry", k3d.DefaultObjectNamePrefix, newCluster.Name),
Image: fmt.Sprintf("%s:%s", k3d.DefaultRegistryImageRepo, k3d.DefaultRegistryImageTag),
Port: k3d.MappedPort{
InternalPort: k3d.DefaultRegistryPort,
ExternalPort: regPort,
},
Host: fmt.Sprintf("%s-%s-registry", k3d.DefaultObjectNamePrefix, newCluster.Name),
Image: fmt.Sprintf("%s:%s", k3d.DefaultRegistryImageRepo, k3d.DefaultRegistryImageTag),
ExposureOpts: *regPort,
}
}

13
pkg/config/v1alpha1/types.go
View File

@ -116,7 +116,7 @@ type SimpleConfig struct {
Name string `mapstructure:"name" yaml:"name" json:"name,omitempty"`
Servers int `mapstructure:"servers" yaml:"servers" json:"servers,omitempty"` //nolint:lll // default 1
Agents int `mapstructure:"agents" yaml:"agents" json:"agents,omitempty"` //nolint:lll // default 0
ExposeAPI k3d.ExposedPort `mapstructure:"exposeAPI" yaml:"exposeAPI" json:"exposeAPI,omitempty"`
ExposeAPI SimpleExposureOpts `mapstructure:"kubeAPI" yaml:"kubeAPI" json:"kubeAPI,omitempty"`
Image string `mapstructure:"image" yaml:"image" json:"image,omitempty"`
Network string `mapstructure:"network" yaml:"network" json:"network,omitempty"`
ClusterToken string `mapstructure:"clusterToken" yaml:"clusterToken" json:"clusterToken,omitempty"` // default: auto-generated
@ -126,11 +126,18 @@ type SimpleConfig struct {
Options SimpleConfigOptions `mapstructure:"options" yaml:"options" json:"options,omitempty"`
Env []EnvVarWithNodeFilters `mapstructure:"env" yaml:"env" json:"env,omitempty"`
Registries struct {
Use []*k3d.Registry `mapstructure:"use" yaml:"use,omitempty" json:"use,omitempty"`
Create bool `mapstructure:"create" yaml:"create,omitempty" json:"create,omitempty"`
Use []string `mapstructure:"use" yaml:"use,omitempty" json:"use,omitempty"`
Create bool `mapstructure:"create" yaml:"create,omitempty" json:"create,omitempty"`
} `mapstructure:"registries" yaml:"registries,omitempty" json:"registries,omitempty"`
}
// SimpleExposureOpts provides a simplified syntax compared to the original k3d.ExposureOpts
type SimpleExposureOpts struct {
Host string `mapstructure:"host" yaml:"host,omitempty" json:"host,omitempty"`
HostIP string `mapstructure:"hostIP" yaml:"hostIP,omitempty" json:"hostIP,omitempty"`
HostPort string `mapstructure:"hostPort" yaml:"hostPort,omitempty" json:"hostPort,omitempty"`
}
// GetKind implements Config.GetKind
func (c SimpleConfig) GetKind() string {
return "Cluster"

9
pkg/config/validate.go
View File

@ -57,7 +57,7 @@ func ValidateClusterConfig(ctx context.Context, runtime runtimes.Runtime, config
}
// API-Port cannot be changed when using network=host
if config.Cluster.Network.Name == "host" && config.Cluster.ExposeAPI.Port != k3d.DefaultAPIPort {
if config.Cluster.Network.Name == "host" && config.Cluster.KubeAPI.Port.Port() != k3d.DefaultAPIPort {
// in hostNetwork mode, we're not going to map a hostport. Here it should always use 6443.
// Note that hostNetwork mode is super inflexible and since we don't change the backend port (on the container), it will only be one hostmode cluster allowed.
return fmt.Errorf("The API Port can not be changed when using 'host' network")
@ -78,13 +78,6 @@ func ValidateClusterConfig(ctx context.Context, runtime runtimes.Runtime, config
return err
}
}
// ports
for _, port := range node.Ports {
if err := util.ValidatePortMap(port); err != nil {
return err
}
}
}
return nil

28
pkg/runtimes/docker/translate.go
View File

@ -94,13 +94,14 @@ func TranslateNodeToContainer(node *k3d.Node) (*NodeInDocker, error) {
// containerConfig.Volumes = map[string]struct{}{} // TODO: do we need this? We only used binds before
/* Ports */
exposedPorts, portBindings, err := nat.ParsePortSpecs(node.Ports)
if err != nil {
log.Errorf("Failed to parse port specs '%v'", node.Ports)
return nil, err
exposedPorts := nat.PortSet{}
for ep := range node.Ports {
if _, exists := exposedPorts[ep]; !exists {
exposedPorts[ep] = struct{}{}
}
}
containerConfig.ExposedPorts = exposedPorts
hostConfig.PortBindings = portBindings
hostConfig.PortBindings = node.Ports
/* Network */
networkingConfig.EndpointsConfig = map[string]*network.EndpointSettings{
node.Network: {},
@ -135,14 +136,6 @@ func TranslateContainerToNode(cont *types.Container) (*k3d.Node, error) {
// TranslateContainerDetailsToNode translates a docker containerJSON object into a k3d node representation
func TranslateContainerDetailsToNode(containerDetails types.ContainerJSON) (*k3d.Node, error) {
// translate portMap to string representation
ports := []string{}
for containerPort, portBindingList := range containerDetails.HostConfig.PortBindings {
for _, hostInfo := range portBindingList {
ports = append(ports, fmt.Sprintf("%s:%s:%s", hostInfo.HostIP, hostInfo.HostPort, containerPort))
}
}
// restart -> we only set 'unless-stopped' upon cluster creation
restart := false
if containerDetails.HostConfig.RestartPolicy.IsAlways() || containerDetails.HostConfig.RestartPolicy.IsUnlessStopped() {
@ -159,13 +152,14 @@ func TranslateContainerDetailsToNode(containerDetails types.ContainerJSON) (*k3d
// serverOpts
serverOpts := k3d.ServerOpts{IsInit: false}
serverOpts.KubeAPI = &k3d.ExposureOpts{}
for k, v := range containerDetails.Config.Labels {
if k == k3d.LabelServerAPIHostIP {
serverOpts.ExposeAPI.HostIP = v
serverOpts.KubeAPI.Binding.HostIP = v
} else if k == k3d.LabelServerAPIHost {
serverOpts.ExposeAPI.Host = v
serverOpts.KubeAPI.Host = v
} else if k == k3d.LabelServerAPIPort {
serverOpts.ExposeAPI.Port = v
serverOpts.KubeAPI.Binding.HostPort = v
}
}
@ -199,7 +193,7 @@ func TranslateContainerDetailsToNode(containerDetails types.ContainerJSON) (*k3d
Env: env,
Cmd: containerDetails.Config.Cmd,
Args: []string{}, // empty, since Cmd already contains flags
Ports: ports,
Ports: containerDetails.HostConfig.PortBindings,
Restart: restart,
Labels: labels,
Network: clusterNetwork,

9
pkg/runtimes/docker/translate_test.go
View File

@ -43,7 +43,14 @@ func TestTranslateNodeToContainer(t *testing.T) {
Env: []string{"TEST_KEY_1=TEST_VAL_1"},
Cmd: []string{"server", "--https-listen-port=6443"},
Args: []string{"--some-boolflag"},
Ports: []string{"0.0.0.0:6443:6443/tcp"},
Ports: nat.PortMap{
"6443/tcp": []nat.PortBinding{
{
HostIP: "0.0.0.0",
HostPort: "6443",
},
},
},
Restart: true,
Labels: map[string]string{k3d.LabelRole: string(k3d.ServerRole), "test_key_1": "test_val_1"},
}

67
pkg/types/types.go
View File

@ -25,6 +25,8 @@ import (
"context"
"fmt"
"time"
"github.com/docker/go-connections/nat"
)
// DefaultClusterName specifies the default name used for newly created clusters
@ -106,19 +108,20 @@ var DefaultObjectLabels = map[string]string{
// List of k3d technical label name
const (
LabelClusterName string = "k3d.cluster"
LabelClusterURL string = "k3d.cluster.url"
LabelClusterToken string = "k3d.cluster.token"
LabelImageVolume string = "k3d.cluster.imageVolume"
LabelNetworkExternal string = "k3d.cluster.network.external"
LabelNetwork string = "k3d.cluster.network"
LabelRole string = "k3d.role"
LabelServerAPIPort string = "k3d.server.api.port"
LabelServerAPIHost string = "k3d.server.api.host"
LabelServerAPIHostIP string = "k3d.server.api.hostIP"
LabelRegistryHost string = "k3d.registry.host"
LabelRegistryHostIP string = "k3d.registry.hostIP"
LabelRegistryPort string = "k3s.registry.port"
LabelClusterName string = "k3d.cluster"
LabelClusterURL string = "k3d.cluster.url"
LabelClusterToken string = "k3d.cluster.token"
LabelImageVolume string = "k3d.cluster.imageVolume"
LabelNetworkExternal string = "k3d.cluster.network.external"
LabelNetwork string = "k3d.cluster.network"
LabelRole string = "k3d.role"
LabelServerAPIPort string = "k3d.server.api.port"
LabelServerAPIHost string = "k3d.server.api.host"
LabelServerAPIHostIP string = "k3d.server.api.hostIP"
LabelRegistryHost string = "k3d.registry.host"
LabelRegistryHostIP string = "k3d.registry.hostIP"
LabelRegistryPortExternal string = "k3s.registry.port.external"
LabelRegistryPortInternal string = "k3s.registry.port.internal"
)
// DefaultRoleCmds maps the node roles to their respective default commands
@ -240,7 +243,7 @@ type Cluster struct {
Nodes []*Node `yaml:"nodes" json:"nodes,omitempty"`
InitNode *Node // init server node
ExternalDatastore *ExternalDatastore `yaml:"externalDatastore,omitempty" json:"externalDatastore,omitempty"`
ExposeAPI ExposedPort `yaml:"exposeAPI" json:"exposeAPI,omitempty"`
KubeAPI *ExposureOpts `yaml:"kubeAPI" json:"kubeAPI,omitempty"`
ServerLoadBalancer *Node `yaml:"serverLoadbalancer,omitempty" json:"serverLoadBalancer,omitempty"`
ImageVolume string `yaml:"imageVolume" json:"imageVolume,omitempty"`
}
@ -294,7 +297,7 @@ type Node struct {
Env []string `yaml:"env" json:"env,omitempty"`
Cmd []string // filled automatically based on role
Args []string `yaml:"extraArgs" json:"extraArgs,omitempty"`
Ports []string `yaml:"portMappings" json:"portMappings,omitempty"`
Ports nat.PortMap `yaml:"portMappings" json:"portMappings,omitempty"`
Restart bool `yaml:"restart" json:"restart,omitempty"`
Labels map[string]string // filled automatically
Network string // filled automatically
@ -307,8 +310,14 @@ type Node struct {
// ServerOpts describes some additional server role specific opts
type ServerOpts struct {
IsInit bool `yaml:"isInitializingServer" json:"isInitializingServer,omitempty"`
ExposeAPI ExposedPort // filled automatically
IsInit bool `yaml:"isInitializingServer" json:"isInitializingServer,omitempty"`
KubeAPI *ExposureOpts `yaml:"kubeAPI" json:"kubeAPI"`
}
// ExposureOpts describes settings that the user can set for accessing the Kubernetes API
type ExposureOpts struct {
nat.PortMapping // filled automatically (reference to normal portmapping)
Host string `yaml:"host,omitempty" json:"host,omitempty"`
}
// ExternalDatastore describes an external datastore used for HA/multi-server clusters
@ -320,19 +329,6 @@ type ExternalDatastore struct {
Network string `yaml:"network" json:"network,omitempty"`
}
// MappedPort combines an internal port mapped to an exposed port
type MappedPort struct {
InternalPort string `yaml:"internal,omitempty" json:"internal,omitempty"`
ExternalPort ExposedPort `yaml:"expose,omitempty" json:"expose,omitempty"`
}
// ExposedPort describes a port exposed on the host system
type ExposedPort struct {
Host string `yaml:"host" json:"host,omitempty"`
HostIP string `yaml:"hostIP" json:"hostIP,omitempty"`
Port string `yaml:"port" json:"port"`
}
// AgentOpts describes some additional agent role specific opts
type AgentOpts struct{}
@ -362,11 +358,12 @@ const (
// Registry describes a k3d-managed registry
type Registry struct {
ClusterRef string // filled automatically -> if created with a cluster
Host string `yaml:"host" json:"host"`
Image string `yaml:"image,omitempty" json:"image,omitempty"`
Port MappedPort `yaml:"port" json:"port"`
Options struct {
ClusterRef string // filled automatically -> if created with a cluster
Protocol string `yaml:"protocol,omitempty" json:"protocol,omitempty"` // default: http
Host string `yaml:"host" json:"host"`
Image string `yaml:"image,omitempty" json:"image,omitempty"`
ExposureOpts ExposureOpts `yaml:"expose" json:"expose"`
Options struct {
ConfigFile string `yaml:"configFile,omitempty" json:"configFile,omitempty"`
Proxy struct {
RemoteURL string `yaml:"remoteURL" json:"remoteURL"`

5
pkg/util/ports.go
View File

@ -28,11 +28,6 @@ import (
log "github.com/sirupsen/logrus"
)
// ValidatePortMap validates a port mapping
func ValidatePortMap(portmap string) error {
return nil // TODO: ValidatePortMap: add validation of port mapping
}
// GetFreePort tries to fetch an open port from the OS-Kernel
func GetFreePort() (int, error) {
tcpAddress, err := net.ResolveTCPAddr("tcp", "localhost:0")

12
tests/dind.sh
View File

@ -46,7 +46,13 @@ until docker inspect "$k3de2e" | jq ".[0].State.Running" && docker logs "$k3de2e
done
# build helper container images
docker exec --workdir /src "$k3de2e" make build-helper-images
if [ -z "$E2E_HELPER_IMAGE_TAG" ]; then
docker exec --workdir /src "$k3de2e" make build-helper-images
# execute tests
docker exec "$k3de2e" /src/tests/runner.sh
else
# execute tests
docker exec -e "K3D_HELPER_IMAGE_TAG=$E2E_HELPER_IMAGE_TAG" "$k3de2e" /src/tests/runner.sh
fi
# execute tests
docker exec "$k3de2e" /src/tests/runner.sh