generate new config file for lb and add helper functions to get images from env if needed

pkg/client/cluster.go

@@ -507,9 +507,16 @@ ClusterCreatOpts:
 	 */
 	// *** ServerLoadBalancer ***
 	if !clusterCreateOpts.DisableLoadBalancer {
-		if err := LoadbalancerCreate(ctx, runtime, cluster, &k3d.LoadbalancerCreateOpts{Labels: clusterCreateOpts.GlobalLabels}); err != nil {
+		node, nodeCreateOpts, err := LoadbalancerPrepare(ctx, runtime, cluster, &k3d.LoadbalancerCreateOpts{Labels: clusterCreateOpts.GlobalLabels})
+		if err != nil {
 			return err
 		}
+		if err := NodeCreate(ctx, runtime, node, *nodeCreateOpts); err != nil {
+			log.Errorln("Failed to create loadbalancer")
+			return err
+		}
+		log.Debugf("Created loadbalancer '%s'", node.Name)
+		return err
 	}
 
 	return nil

pkg/client/loadbalancer.go

@@ -29,10 +29,10 @@ import (
 	"strings"
 
 	"github.com/docker/go-connections/nat"
+	"github.com/rancher/k3d/v4/pkg/actions"
 	"github.com/rancher/k3d/v4/pkg/runtimes"
 	"github.com/rancher/k3d/v4/pkg/types"
 	k3d "github.com/rancher/k3d/v4/pkg/types"
-	"github.com/rancher/k3d/v4/version"
 	log "github.com/sirupsen/logrus"
 	"sigs.k8s.io/yaml"
 )
@@ -112,28 +112,28 @@ func GetLoadbalancerConfig(ctx context.Context, runtime runtimes.Runtime, cluste
 	return currentConfig, nil
 }
 
-func LoadbalancerCreate(ctx context.Context, runtime runtimes.Runtime, cluster *types.Cluster, opts *k3d.LoadbalancerCreateOpts) error {
-	// Generate a comma-separated list of server/server names to pass to the LB container
-	servers := ""
+func LoadbalancerPrepare(ctx context.Context, runtime runtimes.Runtime, cluster *types.Cluster, opts *k3d.LoadbalancerCreateOpts) (*k3d.Node, *k3d.NodeCreateOpts, error) {
+
+	lbConfig := k3d.LoadbalancerConfig{
+		Ports:    map[string][]string{},
+		Settings: k3d.LoadBalancerSettings{},
+	}
+
+	// get list of server nodes
+	servers := []string{}
 	for _, node := range cluster.Nodes {
 		if node.Role == k3d.ServerRole {
-			if servers == "" {
-				servers = node.Name
-			} else {
-				servers = fmt.Sprintf("%s,%s", servers, node.Name)
-			}
+			servers = append(servers, node.Name)
 		}
 	}
 
-	// generate comma-separated list of extra ports to forward
-	ports := []string{k3d.DefaultAPIPort}
-	var udp_ports []string
+	// Default API Port proxied to the server nodes
+	lbConfig.Ports[fmt.Sprintf("%s.tcp", k3d.DefaultAPIPort)] = servers
+
+	// generate comma-separated list of extra ports to forward // TODO: no default targets?
 	for exposedPort := range cluster.ServerLoadBalancer.Ports {
-		if exposedPort.Proto() == "udp" {
-			udp_ports = append(udp_ports, exposedPort.Port())
-			continue
-		}
-		ports = append(ports, exposedPort.Port())
+		// TODO: catch duplicates here?
+		lbConfig.Ports[fmt.Sprintf("%s.%s", exposedPort.Port(), exposedPort.Proto())] = servers
 	}
 
 	if cluster.ServerLoadBalancer.Ports == nil {
@@ -143,28 +143,36 @@ func LoadbalancerCreate(ctx context.Context, runtime runtimes.Runtime, cluster *
 
 	// Create LB as a modified node with loadbalancerRole
 	lbNode := &k3d.Node{
-		Name:  fmt.Sprintf("%s-%s-serverlb", k3d.DefaultObjectNamePrefix, cluster.Name),
-		Image: fmt.Sprintf("%s:%s", k3d.DefaultLBImageRepo, version.GetHelperImageVersion()),
-		Ports: cluster.ServerLoadBalancer.Ports,
-		Env: []string{
-			fmt.Sprintf("SERVERS=%s", servers),
-			fmt.Sprintf("PORTS=%s", strings.Join(ports, ",")),
-			fmt.Sprintf("WORKER_PROCESSES=%d", len(ports)),
-		},
+		Name:          fmt.Sprintf("%s-%s-serverlb", k3d.DefaultObjectNamePrefix, cluster.Name),
+		Image:         k3d.GetLoadbalancerImage(),
+		Ports:         cluster.ServerLoadBalancer.Ports,
 		Role:          k3d.LoadBalancerRole,
 		RuntimeLabels: opts.Labels, // TODO: createLoadBalancer: add more expressive labels
 		Networks:      []string{cluster.Network.Name},
 		Restart:       true,
 	}
-	if len(udp_ports) > 0 {
-		lbNode.Env = append(lbNode.Env, fmt.Sprintf("UDP_PORTS=%s", strings.Join(udp_ports, ",")))
-	}
 	cluster.Nodes = append(cluster.Nodes, lbNode) // append lbNode to list of cluster nodes, so it will be considered during rollback
 	log.Infof("Creating LoadBalancer '%s'", lbNode.Name)
-	if err := NodeCreate(ctx, runtime, lbNode, k3d.NodeCreateOpts{}); err != nil {
-		log.Errorln("Failed to create loadbalancer")
-		return err
+
+	// some additional nginx settings
+	lbConfig.Settings.WorkerProcesses = k3d.DefaultLoadbalancerWorkerProcesses + len(cluster.ServerLoadBalancer.Ports)*len(servers)
+
+	// prepare to write config to lb container
+	configyaml, err := yaml.Marshal(lbConfig)
+	if err != nil {
+		return nil, nil, err
 	}
-	log.Debugf("Created loadbalancer '%s'", lbNode.Name)
-	return nil
+
+	writeLbConfigAction := k3d.NodeHook{
+		Stage: k3d.LifecycleStagePreStart,
+		Action: actions.WriteFileAction{
+			Runtime: runtime,
+			Dest:    k3d.DefaultLoadbalancerConfigPath,
+			Mode:    0744,
+			Content: configyaml,
+		},
+	}
+
+	return lbNode, &k3d.NodeCreateOpts{NodeHooks: []k3d.NodeHook{writeLbConfigAction}}, nil
+
 }

pkg/types/images.go

@@ -0,0 +1,48 @@
+/*
+Copyright © 2020-2021 The k3d Author(s)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+package types
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/rancher/k3d/v4/version"
+	log "github.com/sirupsen/logrus"
+)
+
+func GetLoadbalancerImage() string {
+	if img := os.Getenv("K3D_IMAGE_LOADBALANCER"); img != "" {
+		log.Infof("Loadbalancer image set from env var $K3D_IMAGE_LOADBALANCER: %s", img)
+		return img
+	}
+
+	return fmt.Sprintf("%s:%s", DefaultLBImageRepo, version.GetHelperImageVersion())
+}
+
+func GetToolsImage() string {
+	if img := os.Getenv("K3D_IMAGE_TOOLS"); img != "" {
+		log.Infof("Tools image set from env var $K3D_IMAGE_TOOLS: %s", img)
+		return img
+	}
+
+	return fmt.Sprintf("%s:%s", DefaultToolsImageRepo, version.GetHelperImageVersion())
+}

pkg/types/types.go

@@ -442,10 +442,18 @@ type RegistryExternal struct {
  * 		- k3d-k3s-default-agent-1
  */
 type LoadbalancerConfig struct {
-	Ports map[string][]string `yaml:"ports"`
+	Ports    map[string][]string  `yaml:"ports"`
+	Settings LoadBalancerSettings `yaml:"settings"`
 }
 
-const DefaultLoadbalancerConfigPath = "/etc/confd/portmap.yaml"
+type LoadBalancerSettings struct {
+	WorkerProcesses int `yaml:"workerProcesses"`
+}
+
+const (
+	DefaultLoadbalancerConfigPath      = "/etc/confd/portmap.yaml"
+	DefaultLoadbalancerWorkerProcesses = 1024
+)
 
 type LoadbalancerCreateOpts struct {
 	Labels map[string]string

proxy/conf.d/nginx.toml

@@ -2,5 +2,6 @@
 src = "nginx.tmpl"
 dest = "/etc/nginx/nginx.conf"
 keys = [
-    "ports"
+    "ports",
+    "settings"
 ]

proxy/templates/nginx.tmpl

@@ -5,16 +5,13 @@
 #             #######             #
 ###################################
 
-{{- $servers := split (getenv "SERVERS") "," -}}
-{{- $ports := split (getenv "PORTS") "," -}}
-{{- $udp_ports := split (getenv "UDP_PORTS") "," -}}
 error_log stderr notice;
 
 worker_processes auto;
 events {
   multi_accept on;
   use epoll;
-  worker_connections {{ add 1024 (len $ports) }};
+  worker_connections {{ getv "/settings/workerProcesses" }};
 }
 
 stream {

proxy/test/portmap.yaml

@@ -4,4 +4,7 @@ ports:
     - server-1
   4321.udp:
     - agent-0
-    - agent-1
+    - agent-1
+
+settings:
+  workerProcesses: 1030