diff --git a/cmd/create/createCluster.go b/cmd/create/createCluster.go index 1bfc9aa1..1f670914 100644 --- a/cmd/create/createCluster.go +++ b/cmd/create/createCluster.go @@ -22,8 +22,6 @@ THE SOFTWARE. package create import ( - "fmt" - "github.com/spf13/cobra" "github.com/rancher/k3d/pkg/cluster" @@ -120,6 +118,10 @@ func parseCreateClusterCmd(cmd *cobra.Command, args []string) (runtimes.Runtime, if err != nil { log.Fatalln(err) } + exposeAPI, err := cluster.ParseAPIPort(apiPort) + if err != nil { + log.Fatalln(err) + } // --volume volumes, err := cmd.Flags().GetStringSlice("volume") @@ -144,14 +146,13 @@ func parseCreateClusterCmd(cmd *cobra.Command, args []string) (runtimes.Runtime, // -> master nodes for i := 0; i < masterCount; i++ { node := k3d.Node{ - Role: k3d.MasterRole, - Image: image, - Volumes: volumes, // add only volumes for `all`, `masters`, `master[i]` and `k3d--master-` - Labels: make(map[string]string, 1), + Role: k3d.MasterRole, + Image: image, + Volumes: volumes, // add only volumes for `all`, `masters`, `master[i]` and `k3d--master-` + MasterOpts: k3d.MasterOpts{}, } if i == 0 { - node.Ports = append(node.Ports, fmt.Sprintf("0.0.0.0:%s:6443/tcp", apiPort)) // TODO: update (choose interface, enable more than one master) and get '6443' from defaultport variable - node.Labels["k3d.master.apiPort"] = apiPort + node.MasterOpts.ExposeAPI = exposeAPI } cluster.Nodes = append(cluster.Nodes, node) } @@ -164,6 +165,7 @@ func parseCreateClusterCmd(cmd *cobra.Command, args []string) (runtimes.Runtime, Volumes: volumes, // add only volumes for `all`, `workers`, `worker[i]` and `k3d--worker-` } cluster.Nodes = append(cluster.Nodes, node) + } return runtime, cluster diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index a1b49604..31a52d04 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -71,7 +71,9 @@ func CreateCluster(cluster *k3d.Cluster, runtime k3drt.Runtime) error { for _, node := range cluster.Nodes { // cluster specific settings - node.Labels = make(map[string]string) + if node.Labels == nil { + node.Labels = make(map[string]string) // TODO: maybe create an init function? + } node.Labels["k3d.cluster"] = cluster.Name node.Env = append(node.Env, fmt.Sprintf("K3S_CLUSTER_SECRET=%s", cluster.Secret)) diff --git a/pkg/cluster/kubeconfig.go b/pkg/cluster/kubeconfig.go index 3bc4d142..e123b76b 100644 --- a/pkg/cluster/kubeconfig.go +++ b/pkg/cluster/kubeconfig.go @@ -23,8 +23,10 @@ package cluster import ( "bytes" + "fmt" "io/ioutil" "os" + "strings" "github.com/rancher/k3d/pkg/runtimes" k3d "github.com/rancher/k3d/pkg/types" @@ -33,14 +35,41 @@ import ( // GetKubeconfig grabs the kubeconfig file from /output from a master node container and puts it into a local directory func GetKubeconfig(runtime runtimes.Runtime, cluster *k3d.Cluster) ([]byte, error) { + // get all master nodes for the selected cluster masterNodes, err := runtime.GetNodesByLabel(map[string]string{"k3d.cluster": cluster.Name, "k3d.role": string(k3d.MasterRole)}) if err != nil { - log.Errorln("Failed to get masternodes") + log.Errorln("Failed to get master nodes") return nil, err } - reader, err := runtime.GetKubeconfig(masterNodes[0]) + if len(masterNodes) == 0 { + return nil, fmt.Errorf("Didn't find any master node") + } + + // prefer a master node, which actually has the port exposed + var chosenMaster *k3d.Node + chosenMaster = nil + APIPort := "6443" // TODO: use default from types + APIHost := "localhost" // TODO: use default from types + + for _, master := range masterNodes { + if _, ok := master.Labels["k3d.master.api.port"]; ok { + chosenMaster = master + APIPort = master.Labels["k3d.master.api.port"] + if _, ok := master.Labels["k3d.master.api.host"]; ok { + APIHost = master.Labels["k3d.master.api.host"] + } + break + } + } + + if chosenMaster == nil { + chosenMaster = masterNodes[0] + } + + // get the kubeconfig from the first master node + reader, err := runtime.GetKubeconfig(chosenMaster) if err != nil { - log.Errorf("Failed to get kubeconfig from node '%s'", masterNodes[0].Name) + log.Errorf("Failed to get kubeconfig from node '%s'", chosenMaster) return nil, err } defer reader.Close() @@ -51,10 +80,13 @@ func GetKubeconfig(runtime runtimes.Runtime, cluster *k3d.Cluster) ([]byte, erro return nil, err } - // write to file, skipping the first 512 bytes which contain file metadata - // and trimming any NULL characters + // drop the first 512 bytes which contain file metadata + // and trim any NULL characters trimBytes := bytes.Trim(readBytes[512:], "\x00") + // replace host and port where the API is exposed with what we've found in the master node labels (or use the default) + trimBytes = []byte(strings.Replace(string(trimBytes), "localhost:6443", fmt.Sprintf("%s:%s", APIHost, APIPort), 1)) // replace localhost:6443 with localhost: in kubeconfig + return trimBytes, nil } diff --git a/pkg/cluster/node.go b/pkg/cluster/node.go index b46158f6..a5ace7d9 100644 --- a/pkg/cluster/node.go +++ b/pkg/cluster/node.go @@ -114,5 +114,26 @@ func patchWorkerSpec(node *k3d.Node) error { func patchMasterSpec(node *k3d.Node) error { node.Args = append([]string{"server"}, node.Args...) node.Labels["k3d.role"] = string(k3d.MasterRole) // TODO: maybe put those in a global var DefaultMasterNodeSpec? + + hostIP := "0.0.0.0" // TODO: from defaults + apiPort := "6443" // TODO: from defaults + + if node.MasterOpts.ExposeAPI.Port != "" { + + apiPort = node.MasterOpts.ExposeAPI.Port + node.Labels["k3d.master.api.port"] = node.MasterOpts.ExposeAPI.Port + + if node.MasterOpts.ExposeAPI.Host != "" { + + hostIP = node.MasterOpts.ExposeAPI.HostIP + node.Labels["k3d.master.api.hostIP"] = node.MasterOpts.ExposeAPI.HostIP // TODO: maybe get docker machine IP here + + node.Labels["k3d.master.api.host"] = node.MasterOpts.ExposeAPI.Host + + } + + } + + node.Ports = append(node.Ports, fmt.Sprintf("%s:%s:6443/tcp", hostIP, apiPort)) // TODO: get '6443' from defaultport variable return nil } diff --git a/pkg/cluster/ports.go b/pkg/cluster/ports.go new file mode 100644 index 00000000..5744d97f --- /dev/null +++ b/pkg/cluster/ports.go @@ -0,0 +1,69 @@ +/* +Copyright © 2019 Thorsten Klein + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ +package cluster + +import ( + "fmt" + "net" + "strconv" + "strings" + + k3d "github.com/rancher/k3d/pkg/types" + log "github.com/sirupsen/logrus" +) + +// ParseAPIPort parses/validates a string to create an exposeAPI struct from it +func ParseAPIPort(portString string) (k3d.ExposeAPI, error) { + + var exposeAPI k3d.ExposeAPI + + split := strings.Split(portString, ":") + if len(split) > 2 { + log.Errorln("Failed to parse API Port specification") + return exposeAPI, fmt.Errorf("api-port format error") + } + + if len(split) == 1 { + exposeAPI = k3d.ExposeAPI{Port: split[0]} + } else { + // Make sure 'host' can be resolved to an IP address + addrs, err := net.LookupHost(split[0]) + if err != nil { + return exposeAPI, err + } + exposeAPI = k3d.ExposeAPI{Host: split[0], HostIP: addrs[0], Port: split[1]} + } + + // Verify 'port' is an integer and within port ranges + p, err := strconv.Atoi(exposeAPI.Port) + if err != nil { + return exposeAPI, err + } + + if p < 0 || p > 65535 { + log.Errorln("Failed to parse API Port specification") + return exposeAPI, fmt.Errorf("port value '%d' out of range", p) + } + + return exposeAPI, nil + +} diff --git a/pkg/types/types.go b/pkg/types/types.go index eb9a24cb..31547834 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -79,17 +79,35 @@ type Cluster struct { // Node describes a k3d node type Node struct { - Name string `yaml:"name" json:"name,omitempty"` - Role Role `yaml:"role" json:"role,omitempty"` - Image string `yaml:"image" json:"image,omitempty"` - Volumes []string `yaml:"volumes" json:"volumes,omitempty"` - Env []string `yaml:"env" json:"env,omitempty"` - Cmd []string // filled automatically based on role - Args []string `yaml:"extra_args" json:"extraArgs,omitempty"` - Ports []string `yaml:"port_mappings" json:"portMappings,omitempty"` // TODO: make a struct out of this? - Restart bool `yaml:"restart" json:"restart,omitempty"` - Labels map[string]string // filled automatically - Network string // filled automatically + Name string `yaml:"name" json:"name,omitempty"` + Role Role `yaml:"role" json:"role,omitempty"` + Image string `yaml:"image" json:"image,omitempty"` + Volumes []string `yaml:"volumes" json:"volumes,omitempty"` + Env []string `yaml:"env" json:"env,omitempty"` + Cmd []string // filled automatically based on role + Args []string `yaml:"extra_args" json:"extraArgs,omitempty"` + Ports []string `yaml:"port_mappings" json:"portMappings,omitempty"` // TODO: make a struct out of this? + Restart bool `yaml:"restart" json:"restart,omitempty"` + Labels map[string]string // filled automatically + Network string // filled automatically + MasterOpts MasterOpts `yaml:"master_opts" json:"masterOpts,omitempty"` + WorkerOpts WorkerOpts `yaml:"worker_opts" json:"workerOpts,omitempty"` +} + +// MasterOpts describes some additional master role specific opts +type MasterOpts struct { + ExposeAPI ExposeAPI `yaml:"expose_api" json:"exposeAPI,omitempty"` +} + +// ExposeAPI describes specs needed to expose the API-Server +type ExposeAPI struct { + Host string `yaml:"host" json:"host,omitempty"` + HostIP string `yaml:"host_ip" json:"hostIP,omitempty"` + Port string `yaml:"port" json:"port"` +} + +// WorkerOpts describes some additional worker role specific opts +type WorkerOpts struct { } // GetDefaultObjectName prefixes the passed name with the default prefix