diff --git a/cmd/create/createCluster.go b/cmd/create/createCluster.go index d9deb005..ede38a60 100644 --- a/cmd/create/createCluster.go +++ b/cmd/create/createCluster.go @@ -69,7 +69,7 @@ func NewCmdCreateCluster() *cobra.Command { /********* * Flags * *********/ - cmd.Flags().StringArrayP("api-port", "a", []string{"6443"}, "Specify the Kubernetes API server port (Format: `--api-port [HOST:]HOSTPORT[@NODEFILTER]`\n - Example: `k3d create -m 3 -a 0.0.0.0:6550@master[0] -a 0.0.0.0:6551@master[1]` ") + cmd.Flags().StringP("api-port", "a", k3d.DefaultAPIPort, "Specify the Kubernetes API server port (Format: `--api-port [HOST:]HOSTPORT`\n - Example: `k3d create -m 3 -a 0.0.0.0:6550` ") cmd.Flags().IntP("masters", "m", 1, "Specify how many masters you want to create") cmd.Flags().IntP("workers", "w", 0, "Specify how many workers you want to create") cmd.Flags().String("image", fmt.Sprintf("%s:%s", k3d.DefaultK3sImageRepo, version.GetK3sVersion(false)), "Specify k3s image that you want to use for the nodes") @@ -83,10 +83,6 @@ func NewCmdCreateCluster() *cobra.Command { cmd.Flags().BoolVar(&createClusterOpts.DisableImageVolume, "no-image-volume", false, "Disable the creation of a volume for importing images") /* Multi Master Configuration */ - // multi-master - general - // TODO: implement load-balancer/proxy for multi-master setups - cmd.Flags().BoolVar(&createClusterOpts.DisableLoadbalancer, "no-lb", false, "[WIP] Disable automatic deployment of a load balancer in Multi-Master setups") - cmd.Flags().String("lb-port", "0.0.0.0:6443", "[WIP] Specify port to be exposed by the master load balancer (Format: `[HOST:]HOSTPORT)") // multi-master - datastore // TODO: implement multi-master setups with external data store @@ -174,65 +170,22 @@ func parseCreateClusterCmd(cmd *cobra.Command, args []string, createClusterOpts } // --api-port - apiPortFlags, err := cmd.Flags().GetStringArray("api-port") + apiPort, err := cmd.Flags().GetString("api-port") if err != nil { log.Fatalln(err) } - // error out if we have more api-ports than masters specified - if len(apiPortFlags) > masterCount { - log.Fatalf("Cannot expose more api-ports than master nodes exist (%d > %d)", len(apiPortFlags), masterCount) - } - - ipPortCombinations := map[string]struct{}{} // only for finding duplicates - apiPortFilters := map[string]struct{}{} // only for deduplication - exposeAPIToFiltersMap := map[k3d.ExposeAPI][]string{} - for _, apiPortFlag := range apiPortFlags { - - // split the flag value from the node filter - apiPortString, filters, err := cliutil.SplitFiltersFromFlag(apiPortFlag) - if err != nil { - log.Fatalln(err) - } - - // if there's only one master node, we don't need a node filter, but if there's more than one, we need exactly one node filter per api-port flag - if len(filters) > 1 || (len(filters) == 0 && masterCount > 1) { - log.Fatalf("Exactly one node filter required per '--api-port' flag, but got %d on flag %s", len(filters), apiPortFlag) - } - - // add default, if no filter was set and we only have a single master node - if len(filters) == 0 && masterCount == 1 { - filters = []string{"master[0]"} - } - - // only one api-port mapping allowed per master node - if _, exists := apiPortFilters[filters[0]]; exists { - log.Fatalf("Cannot assign multiple api-port mappings to the same node: duplicate '%s'", filters[0]) - } - apiPortFilters[filters[0]] = struct{}{} - - // parse the port mapping - exposeAPI, err := cliutil.ParseAPIPort(apiPortString) - if err != nil { - log.Fatalln(err) - } - - // error out on duplicates - ipPort := fmt.Sprintf("%s:%s", exposeAPI.HostIP, exposeAPI.Port) - if _, exists := ipPortCombinations[ipPort]; exists { - log.Fatalf("Duplicate IP:PORT combination '%s' for the Api Port is not allowed", ipPort) - } - ipPortCombinations[ipPort] = struct{}{} - - // add to map - exposeAPIToFiltersMap[exposeAPI] = filters - } - - // --lb-port - lbPort, err := cmd.Flags().GetString("lb-port") + // parse the port mapping + exposeAPI, err := cliutil.ParseAPIPort(apiPort) if err != nil { log.Fatalln(err) } + if exposeAPI.Host == "" { + exposeAPI.Host = k3d.DefaultAPIHost + } + if exposeAPI.HostIP == "" { + exposeAPI.HostIP = k3d.DefaultAPIHost + } // --datastore-endpoint datastoreEndpoint, err := cmd.Flags().GetString("datastore-endpoint") @@ -319,6 +272,7 @@ func parseCreateClusterCmd(cmd *cobra.Command, args []string, createClusterOpts Network: network, Secret: secret, CreateClusterOpts: createClusterOpts, + ExposeAPI: exposeAPI, } // generate list of nodes @@ -336,7 +290,7 @@ func parseCreateClusterCmd(cmd *cobra.Command, args []string, createClusterOpts MasterOpts: k3d.MasterOpts{}, } - // TODO: by default, we don't expose an API port, even if we only have a single master: should we change that? + // TODO: by default, we don't expose an API port: should we change that? // -> if we want to change that, simply add the exposeAPI struct here // first master node will be init node if we have more than one master specified but no external datastore @@ -363,20 +317,6 @@ func parseCreateClusterCmd(cmd *cobra.Command, args []string, createClusterOpts cluster.Nodes = append(cluster.Nodes, &node) } - // add masterOpts - for exposeAPI, filters := range exposeAPIToFiltersMap { - nodes, err := cliutil.FilterNodes(cluster.Nodes, filters) - if err != nil { - log.Fatalln(err) - } - for _, node := range nodes { - if node.Role != k3d.MasterRole { - log.Fatalf("Node returned by filters '%+v' for exposing the API is not a master node", filters) - } - node.MasterOpts.ExposeAPI = exposeAPI - } - } - // append volumes for volume, filters := range volumeFilterMap { nodes, err := cliutil.FilterNodes(cluster.Nodes, filters) @@ -405,15 +345,7 @@ func parseCreateClusterCmd(cmd *cobra.Command, args []string, createClusterOpts /********************** * Utility Containers * **********************/ - - // TODO: create load balancer and other util containers // TODO: for now, this will only work with the docker provider (?) -> can replace dynamic docker lookup with static traefik config (?) - if masterCount > 1 && !createClusterOpts.DisableLoadbalancer { // TODO: add traefik to the same network and add traefik labels to the master node containers - log.Debugln("Creating LB in front of master nodes") - cluster.MasterLoadBalancer = &k3d.ClusterLoadbalancer{ - Image: k3d.DefaultLBImage, - ExposedPort: lbPort, - } - } + // ... return cluster } diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 4e1db6df..eb4701e5 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -117,6 +117,8 @@ func CreateCluster(cluster *k3d.Cluster, runtime k3drt.Runtime) error { // node role specific settings if node.Role == k3d.MasterRole { + node.MasterOpts.ExposeAPI = cluster.ExposeAPI + // the cluster has an init master node, but its not this one, so connect it to the init node if cluster.InitNode != nil && !node.MasterOpts.IsInit { node.Args = append(node.Args, "--server", fmt.Sprintf("https://%s:%d", cluster.InitNode.Name, 6443)) @@ -245,6 +247,37 @@ initNodeFinished: } } + /* + * Auxiliary Containers + */ + // MasterLoadBalancer + servers := "" + for _, node := range cluster.Nodes { + if node.Role == k3d.MasterRole { + log.Debugf("Node NAME: %s", node.Name) + if servers == "" { + servers = node.Name + } else { + servers = fmt.Sprintf("%s,%s", servers, node.Name) + } + } + } + lbNode := &k3d.Node{ + Name: fmt.Sprintf("%s-%s-masterlb", k3d.DefaultObjectNamePrefix, cluster.Name), + Image: k3d.DefaultLBImage, + Ports: []string{fmt.Sprintf("%s:%s:%s/tcp", cluster.ExposeAPI.Host, cluster.ExposeAPI.Port, k3d.DefaultAPIPort)}, + Env: []string{ + fmt.Sprintf("SERVERS=%s", servers), + fmt.Sprintf("PORT=%s", k3d.DefaultAPIPort), + }, + Role: k3d.NoRole, + Labels: k3d.DefaultObjectLabels, // TODO: createLoadBalancer: add more expressive labels + Network: cluster.Network.Name, + } + if err := CreateNode(lbNode, runtime); err != nil { + log.Errorln("Failed to create loadbalancer") + return err + } return nil } diff --git a/pkg/cluster/kubeconfig.go b/pkg/cluster/kubeconfig.go index c2f06712..caf06c3a 100644 --- a/pkg/cluster/kubeconfig.go +++ b/pkg/cluster/kubeconfig.go @@ -37,6 +37,7 @@ import ( // GetKubeconfig grabs the kubeconfig file from /output from a master node container and puts it into a local directory func GetKubeconfig(runtime runtimes.Runtime, cluster *k3d.Cluster) ([]byte, error) { // get all master nodes for the selected cluster + // TODO: getKubeconfig: we should make sure, that the master node we're trying to getch is actually running masterNodes, err := runtime.GetNodesByLabel(map[string]string{"k3d.cluster": cluster.Name, "k3d.role": string(k3d.MasterRole)}) if err != nil { log.Errorln("Failed to get master nodes") @@ -49,8 +50,8 @@ func GetKubeconfig(runtime runtimes.Runtime, cluster *k3d.Cluster) ([]byte, erro // prefer a master node, which actually has the port exposed var chosenMaster *k3d.Node chosenMaster = nil - APIPort := "6443" // TODO: use default from types - APIHost := "localhost" // TODO: use default from types + APIPort := k3d.DefaultAPIPort + APIHost := k3d.DefaultAPIHost for _, master := range masterNodes { if _, ok := master.Labels["k3d.master.api.port"]; ok { diff --git a/pkg/cluster/node.go b/pkg/cluster/node.go index cb139cea..b7fdc511 100644 --- a/pkg/cluster/node.go +++ b/pkg/cluster/node.go @@ -125,8 +125,6 @@ func CreateNode(node *k3d.Node, runtime runtimes.Runtime) error { return err } log.Debugf("spec = %+v\n", node) - } else { - return fmt.Errorf("Unknown node role '%s'", node.Role) } /* @@ -164,19 +162,14 @@ func patchMasterSpec(node *k3d.Node) error { // role label node.Labels["k3d.role"] = string(k3d.MasterRole) // TODO: maybe put those in a global var DefaultMasterNodeSpec? - // extra settings to expose the API port (if wanted) - if node.MasterOpts.ExposeAPI.Port != "" { - if node.MasterOpts.ExposeAPI.Host == "" { - node.MasterOpts.ExposeAPI.Host = "0.0.0.0" - } - node.Labels["k3d.master.api.hostIP"] = node.MasterOpts.ExposeAPI.HostIP // TODO: maybe get docker machine IP here + // Add labels and TLS SAN for the exposed API + // FIXME: For now, the labels concerning the API on the master nodes are only being used for configuring the kubeconfig + node.Labels["k3d.master.api.hostIP"] = node.MasterOpts.ExposeAPI.HostIP // TODO: maybe get docker machine IP here + node.Labels["k3d.master.api.host"] = node.MasterOpts.ExposeAPI.Host + node.Labels["k3d.master.api.port"] = node.MasterOpts.ExposeAPI.Port - node.Labels["k3d.master.api.host"] = node.MasterOpts.ExposeAPI.Host + node.Args = append(node.Args, "--tls-san", node.MasterOpts.ExposeAPI.Host) // add TLS SAN for non default host name - node.Args = append(node.Args, "--tls-san", node.MasterOpts.ExposeAPI.Host) // add TLS SAN for non default host name - node.Labels["k3d.master.api.port"] = node.MasterOpts.ExposeAPI.Port - node.Ports = append(node.Ports, fmt.Sprintf("%s:%s:6443/tcp", node.MasterOpts.ExposeAPI.Host, node.MasterOpts.ExposeAPI.Port)) // TODO: get '6443' from defaultport variable - } return nil } diff --git a/pkg/types/types.go b/pkg/types/types.go index 7296de3f..466e0d88 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -38,7 +38,7 @@ const DefaultClusterNameMaxLength = 32 const DefaultK3sImageRepo = "docker.io/rancher/k3s" // DefaultLBImage defines the default cluster load balancer image -const DefaultLBImage = "docker.io/library/traefik:v2.0" +const DefaultLBImage = "docker.io/iwilltry42/k3d-proxy:v0.0.1" // DefaultObjectNamePrefix defines the name prefix for every object created by k3d const DefaultObjectNamePrefix = "k3d" @@ -50,7 +50,8 @@ type Role string const ( MasterRole Role = "master" WorkerRole Role = "worker" - NoRole Role = "nope" + NoRole Role = "noRole" + ProxyRole Role = "proxy" ) // DefaultK3dRoles defines the roles available for nodes @@ -87,13 +88,18 @@ const DefaultConfigDirName = ".k3d" // should end up in $HOME/ // DefaultKubeconfigPrefix defines the default prefix for kubeconfig files const DefaultKubeconfigPrefix = DefaultObjectNamePrefix + "-kubeconfig" +// DefaultAPIPort defines the default Kubernetes API Port +const DefaultAPIPort = "6443" + +// DefaultAPIHost defines the default host (IP) for the Kubernetes API +const DefaultAPIHost = "0.0.0.0" + // CreateClusterOpts describe a set of options one can set when creating a cluster type CreateClusterOpts struct { - DisableImageVolume bool - DisableLoadbalancer bool - WaitForMaster int - K3sServerArgs []string - K3sAgentArgs []string + DisableImageVolume bool + WaitForMaster int + K3sServerArgs []string + K3sAgentArgs []string } // ClusterNetwork describes a network which a cluster is running in @@ -104,14 +110,14 @@ type ClusterNetwork struct { // Cluster describes a k3d cluster type Cluster struct { - Name string `yaml:"name" json:"name,omitempty"` - Network ClusterNetwork `yaml:"network" json:"network,omitempty"` - Secret string `yaml:"cluster_secret" json:"clusterSecret,omitempty"` - Nodes []*Node `yaml:"nodes" json:"nodes,omitempty"` - InitNode *Node // init master node - MasterLoadBalancer *ClusterLoadbalancer `yaml:"master_loadbalancer" json:"masterLoadBalancer,omitempty"` - ExternalDatastore ExternalDatastore `yaml:"external_datastore" json:"externalDatastore,omitempty"` - CreateClusterOpts *CreateClusterOpts `yaml:"options" json:"options,omitempty"` + Name string `yaml:"name" json:"name,omitempty"` + Network ClusterNetwork `yaml:"network" json:"network,omitempty"` + Secret string `yaml:"cluster_secret" json:"clusterSecret,omitempty"` + Nodes []*Node `yaml:"nodes" json:"nodes,omitempty"` + InitNode *Node // init master node + ExternalDatastore ExternalDatastore `yaml:"external_datastore" json:"externalDatastore,omitempty"` + CreateClusterOpts *CreateClusterOpts `yaml:"options" json:"options,omitempty"` + ExposeAPI ExposeAPI `yaml:"expose_api" json:"exposeAPI,omitempty"` } // Node describes a k3d node @@ -133,8 +139,8 @@ type Node struct { // MasterOpts describes some additional master role specific opts type MasterOpts struct { - ExposeAPI ExposeAPI `yaml:"expose_api" json:"exposeAPI,omitempty"` IsInit bool `yaml:"is_initializing_master" json:"isInitializingMaster,omitempty"` + ExposeAPI ExposeAPI // filled automatically } // ExternalDatastore describes an external datastore used for HA/multi-master clusters @@ -160,9 +166,3 @@ type WorkerOpts struct{} func GetDefaultObjectName(name string) string { return fmt.Sprintf("%s-%s", DefaultObjectNamePrefix, name) } - -// ClusterLoadbalancer describes a loadbalancer deployed in front of a multi-master cluster -type ClusterLoadbalancer struct { - Image string - ExposedPort string `yaml:"exposed_port" json:"exposedPort,omitempty"` -} diff --git a/proxy/Dockerfile b/proxy/Dockerfile new file mode 100644 index 00000000..b76fb638 --- /dev/null +++ b/proxy/Dockerfile @@ -0,0 +1,13 @@ +FROM nginx:1.16.0-alpine + +RUN apk -U --no-cache add curl ca-certificates\ + && mkdir -p /etc/confd \ + && curl -sLf https://github.com/kelseyhightower/confd/releases/download/v0.16.0/confd-0.16.0-linux-amd64 > /usr/bin/confd \ + && chmod +x /usr/bin/confd \ + && apk del curl + +COPY templates /etc/confd/templates/ +COPY conf.d /etc/confd/conf.d/ +COPY nginx-proxy /usr/bin/ + +ENTRYPOINT nginx-proxy \ No newline at end of file diff --git a/proxy/conf.d/nginx.toml b/proxy/conf.d/nginx.toml new file mode 100644 index 00000000..344166c6 --- /dev/null +++ b/proxy/conf.d/nginx.toml @@ -0,0 +1,7 @@ +[template] +src = "nginx.tmpl" +dest = "/etc/nginx/nginx.conf" +keys = [ + "SERVERS", + "PORT", +] \ No newline at end of file diff --git a/proxy/nginx-proxy b/proxy/nginx-proxy new file mode 100755 index 00000000..8afd20eb --- /dev/null +++ b/proxy/nginx-proxy @@ -0,0 +1,7 @@ +#!/bin/sh + +# Run confd +confd -onetime -backend env + +# Start nginx +nginx -g 'daemon off;' \ No newline at end of file diff --git a/proxy/templates/nginx.tmpl b/proxy/templates/nginx.tmpl new file mode 100644 index 00000000..c156a9f6 --- /dev/null +++ b/proxy/templates/nginx.tmpl @@ -0,0 +1,23 @@ +error_log stderr notice; + +worker_processes auto; +events { + multi_accept on; + use epoll; + worker_connections 1024; +} + +stream { + upstream kube_apiserver { + {{ $servers := split (getenv "SERVERS") "," }}{{range $servers}} + server {{.}}:{{getenv "PORT"}}; + {{end}} + } + + server { + listen {{getenv "PORT"}}; + proxy_pass kube_apiserver; + proxy_timeout 30; + proxy_connect_timeout 2s; + } +} \ No newline at end of file