cmd/k8s-operator: allow specifying replicas for connectors

This commit adds a `replicas` field to the `Connector` custom resource that
allows users to specify the number of desired replicas deployed for their
connectors.

This allows users to deploy exit nodes, subnet routers and app connectors
in a highly available fashion.

Signed-off-by: David Bond <davidsbond93@gmail.com>

cmd/k8s-operator/connector.go

@@ -25,7 +25,6 @@
 	"k8s.io/client-go/tools/record"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
-
 	tsoperator "tailscale.com/k8s-operator"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
 	"tailscale.com/kube/kubetypes"
@@ -188,7 +187,13 @@ func (a *ConnectorReconciler) maybeProvisionConnector(ctx context.Context, logge
 		}
 	}
 
+	var replicas int32 = 1
+	if cn.Spec.Replicas != nil {
+		replicas = *cn.Spec.Replicas
+	}
+
 	sts := &tailscaleSTSConfig{
+		Replicas:            replicas,
 		ParentResourceName:  cn.Name,
 		ParentResourceUID:   string(cn.UID),
 		Hostname:            hostname,
@@ -219,16 +224,19 @@ func (a *ConnectorReconciler) maybeProvisionConnector(ctx context.Context, logge
 	} else {
 		a.exitNodes.Remove(cn.UID)
 	}
+
 	if cn.Spec.SubnetRouter != nil {
 		a.subnetRouters.Add(cn.GetUID())
 	} else {
 		a.subnetRouters.Remove(cn.GetUID())
 	}
+
 	if cn.Spec.AppConnector != nil {
 		a.appConnectors.Add(cn.GetUID())
 	} else {
 		a.appConnectors.Remove(cn.GetUID())
 	}
+
 	a.mu.Unlock()
 	gaugeConnectorSubnetRouterResources.Set(int64(a.subnetRouters.Len()))
 	gaugeConnectorExitNodeResources.Set(int64(a.exitNodes.Len()))
@@ -244,6 +252,7 @@ func (a *ConnectorReconciler) maybeProvisionConnector(ctx context.Context, logge
 		return err
 	}
 
+	// TODO(davidsbond): device info needs to handle multiple replicas
 	dev, err := a.ssr.DeviceInfo(ctx, crl, logger)
 	if err != nil {
 		return err

cmd/k8s-operator/connector_test.go

@@ -20,6 +20,7 @@
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
 	"tailscale.com/kube/kubetypes"
 	"tailscale.com/tstest"
+	"tailscale.com/types/ptr"
 	"tailscale.com/util/mak"
 )
 
@@ -36,6 +37,7 @@ func TestConnector(t *testing.T) {
 			APIVersion: "tailscale.com/v1alpha1",
 		},
 		Spec: tsapi.ConnectorSpec{
+			Replicas: ptr.To[int32](3),
 			SubnetRouter: &tsapi.SubnetRouter{
 				AdvertiseRoutes: []tsapi.Route{"10.40.0.0/14"},
 			},
@@ -56,6 +58,7 @@ func TestConnector(t *testing.T) {
 	cl := tstest.NewClock(tstest.ClockOpts{})
 	cr := &ConnectorReconciler{
 		Client:   fc,
+		recorder: record.NewFakeRecorder(10),
 		ssr: &tailscaleSTSReconciler{
 			Client:            fc,
 			tsClient:          ft,
@@ -78,6 +81,7 @@ func TestConnector(t *testing.T) {
 		isExitNode:   true,
 		subnetRoutes: "10.40.0.0/14",
 		app:          kubetypes.AppConnector,
+		replicas:     cn.Spec.Replicas,
 	}
 	expectEqual(t, fc, expectedSecret(t, fc, opts))
 	expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs)
@@ -156,6 +160,7 @@ func TestConnector(t *testing.T) {
 			APIVersion: "tailscale.io/v1alpha1",
 		},
 		Spec: tsapi.ConnectorSpec{
+			Replicas: ptr.To[int32](3),
 			SubnetRouter: &tsapi.SubnetRouter{
 				AdvertiseRoutes: []tsapi.Route{"10.40.0.0/14"},
 			},
@@ -174,6 +179,7 @@ func TestConnector(t *testing.T) {
 		subnetRoutes: "10.40.0.0/14",
 		hostname:     "test-connector",
 		app:          kubetypes.AppConnector,
+		replicas:     cn.Spec.Replicas,
 	}
 	expectEqual(t, fc, expectedSecret(t, fc, opts))
 	expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs)
@@ -217,6 +223,7 @@ func TestConnectorWithProxyClass(t *testing.T) {
 			APIVersion: "tailscale.io/v1alpha1",
 		},
 		Spec: tsapi.ConnectorSpec{
+			Replicas: ptr.To[int32](3),
 			SubnetRouter: &tsapi.SubnetRouter{
 				AdvertiseRoutes: []tsapi.Route{"10.40.0.0/14"},
 			},
@@ -260,6 +267,7 @@ func TestConnectorWithProxyClass(t *testing.T) {
 		isExitNode:   true,
 		subnetRoutes: "10.40.0.0/14",
 		app:          kubetypes.AppConnector,
+		replicas:     cn.Spec.Replicas,
 	}
 	expectEqual(t, fc, expectedSecret(t, fc, opts))
 	expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs)
@@ -311,6 +319,7 @@ func TestConnectorWithAppConnector(t *testing.T) {
 			APIVersion: "tailscale.io/v1alpha1",
 		},
 		Spec: tsapi.ConnectorSpec{
+			Replicas:     ptr.To[int32](3),
 			AppConnector: &tsapi.AppConnector{},
 		},
 	}
@@ -350,6 +359,7 @@ func TestConnectorWithAppConnector(t *testing.T) {
 		hostname:       "test-connector",
 		app:            kubetypes.AppConnector,
 		isAppConnector: true,
+		replicas:       cn.Spec.Replicas,
 	}
 	expectEqual(t, fc, expectedSecret(t, fc, opts))
 	expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs)

cmd/k8s-operator/deploy/crds/tailscale.com_connectors.yaml

@@ -125,6 +125,13 @@ spec:
                     resources created for this Connector. If unset, the operator will
                     create resources with the default configuration.
                   type: string
+                replicas:
+                  description: |-
+                    Replicas specifies how many replicas to create the StatefulSet with.
+                    Defaults to 1.
+                  type: integer
+                  format: int32
+                  minimum: 0
                 subnetRouter:
                   description: |-
                     SubnetRouter defines subnet routes that the Connector device should

cmd/k8s-operator/deploy/manifests/operator.yaml

@@ -150,6 +150,13 @@ spec:
                                     resources created for this Connector. If unset, the operator will
                                     create resources with the default configuration.
                                 type: string
+                            replicas:
+                                description: |-
+                                    Replicas specifies how many replicas to create the StatefulSet with.
+                                    Defaults to 1.
+                                format: int32
+                                minimum: 0
+                                type: integer
                             subnetRouter:
                                 description: |-
                                     SubnetRouter defines subnet routes that the Connector device should

cmd/k8s-operator/ingress.go

@@ -212,6 +212,7 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 	hostname := hostnameForIngress(ing)
 
 	sts := &tailscaleSTSConfig{
+		Replicas:            1,
 		Hostname:            hostname,
 		ParentResourceName:  ing.Name,
 		ParentResourceUID:   string(ing.UID),

cmd/k8s-operator/sts.go

@@ -13,6 +13,7 @@
 	"fmt"
 	"net/http"
 	"os"
+	"path"
 	"slices"
 	"strconv"
 	"strings"
@@ -114,6 +115,7 @@
 )
 
 type tailscaleSTSConfig struct {
+	Replicas            int32
 	ParentResourceName  string
 	ParentResourceUID   string
 	ChildResourceLabels map[string]string
@@ -205,11 +207,12 @@ func (a *tailscaleSTSReconciler) Provision(ctx context.Context, logger *zap.Suga
 	}
 	sts.ProxyClass = proxyClass
 
-	secretName, _, err := a.createOrGetSecret(ctx, logger, sts, hsvc)
+	secretNames, err := a.createOrGetSecrets(ctx, logger, sts, hsvc)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create or get API key secret: %w", err)
 	}
-	_, err = a.reconcileSTS(ctx, logger, sts, hsvc, secretName)
+
+	_, err = a.reconcileSTS(ctx, logger, sts, hsvc, secretNames)
 	if err != nil {
 		return nil, fmt.Errorf("failed to reconcile statefulset: %w", err)
 	}
@@ -339,23 +342,26 @@ func (a *tailscaleSTSReconciler) reconcileHeadlessService(ctx context.Context, l
 	return createOrUpdate(ctx, a.Client, a.operatorNamespace, hsvc, func(svc *corev1.Service) { svc.Spec = hsvc.Spec })
 }
 
-func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger *zap.SugaredLogger, stsC *tailscaleSTSConfig, hsvc *corev1.Service) (secretName string, configs tailscaledConfigs, _ error) {
+func (a *tailscaleSTSReconciler) createOrGetSecrets(ctx context.Context, logger *zap.SugaredLogger, stsC *tailscaleSTSConfig, hsvc *corev1.Service) ([]string, error) {
+	secretNames := make([]string, stsC.Replicas)
+
+	for i := range stsC.Replicas {
 		secret := &corev1.Secret{
 			ObjectMeta: metav1.ObjectMeta{
-			// Hardcode a -0 suffix so that in future, if we support
-			// multiple StatefulSet replicas, we can provision -N for
-			// those.
-			Name:      hsvc.Name + "-0",
+				Name:      hsvc.Name + "-" + strconv.FormatInt(int64(i), 10),
 				Namespace: a.operatorNamespace,
 				Labels:    stsC.ChildResourceLabels,
 			},
 		}
+
+		secretNames[i] = secret.Name
+
 		var orig *corev1.Secret // unmodified copy of secret
 		if err := a.Get(ctx, client.ObjectKeyFromObject(secret), secret); err == nil {
 			logger.Debugf("secret %s/%s already exists", secret.GetNamespace(), secret.GetName())
 			orig = secret.DeepCopy()
 		} else if !apierrors.IsNotFound(err) {
-		return "", nil, err
+			return nil, err
 		}
 
 		var authKey string
@@ -365,14 +371,16 @@ func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger *
 			// ACME account key.
 			sts, err := getSingleObject[appsv1.StatefulSet](ctx, a.Client, a.operatorNamespace, stsC.ChildResourceLabels)
 			if err != nil {
-			return "", nil, err
+				return nil, err
 			}
+
 			if sts != nil {
 				// StatefulSet exists, so we have already created the secret.
 				// If the secret is missing, they should delete the StatefulSet.
 				logger.Errorf("Tailscale proxy secret doesn't exist, but the corresponding StatefulSet %s/%s already does. Something is wrong, please delete the StatefulSet.", sts.GetNamespace(), sts.GetName())
-			return "", nil, nil
+				return nil, nil
 			}
+
 			// Create API Key secret which is going to be used by the statefulset
 			// to authenticate with Tailscale.
 			logger.Debugf("creating authkey for new tailscale proxy")
@@ -382,21 +390,24 @@ func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger *
 			}
 			authKey, err = newAuthKey(ctx, a.tsClient, tags)
 			if err != nil {
-			return "", nil, err
+				return nil, err
 			}
 		}
+
 		configs, err := tailscaledConfig(stsC, authKey, orig)
 		if err != nil {
-		return "", nil, fmt.Errorf("error creating tailscaled config: %w", err)
+			return nil, fmt.Errorf("error creating tailscaled config: %w", err)
 		}
+
 		latest := tailcfg.CapabilityVersion(-1)
 		var latestConfig ipn.ConfigVAlpha
 		for key, val := range configs {
 			fn := tsoperator.TailscaledConfigFileName(key)
 			b, err := json.Marshal(val)
 			if err != nil {
-			return "", nil, fmt.Errorf("error marshalling tailscaled config: %w", err)
+				return nil, fmt.Errorf("error marshalling tailscaled config: %w", err)
 			}
+
 			mak.Set(&secret.StringData, fn, string(b))
 			if key > latest {
 				latest = key
@@ -407,23 +418,26 @@ func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger *
 		if stsC.ServeConfig != nil {
 			j, err := json.Marshal(stsC.ServeConfig)
 			if err != nil {
-			return "", nil, err
+				return nil, err
 			}
+
 			mak.Set(&secret.StringData, "serve-config", string(j))
 		}
 
 		if orig != nil {
 			logger.Debugf("patching the existing proxy Secret with tailscaled config %s", sanitizeConfigBytes(latestConfig))
 			if err := a.Patch(ctx, secret, client.MergeFrom(orig)); err != nil {
-			return "", nil, err
+				return nil, err
 			}
 		} else {
 			logger.Debugf("creating a new Secret for the proxy with tailscaled config %s", sanitizeConfigBytes(latestConfig))
 			if err := a.Create(ctx, secret); err != nil {
-			return "", nil, err
+				return nil, err
 			}
 		}
-	return secret.Name, configs, nil
+	}
+
+	return secretNames, nil
 }
 
 // sanitizeConfigBytes returns ipn.ConfigVAlpha in string form with redacted
@@ -534,7 +548,7 @@ func newAuthKey(ctx context.Context, tsClient tsClient, tags []string) (string,
 //go:embed deploy/manifests/userspace-proxy.yaml
 var userspaceProxyYaml []byte
 
-func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.SugaredLogger, sts *tailscaleSTSConfig, headlessSvc *corev1.Service, proxySecret string) (*appsv1.StatefulSet, error) {
+func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.SugaredLogger, sts *tailscaleSTSConfig, headlessSvc *corev1.Service, proxySecrets []string) (*appsv1.StatefulSet, error) {
 	ss := new(appsv1.StatefulSet)
 	if sts.ServeConfig != nil && sts.ForwardClusterTrafficViaL7IngressProxy != true { // If forwarding cluster traffic via is required we need non-userspace + NET_ADMIN + forwarding
 		if err := yaml.Unmarshal(userspaceProxyYaml, &ss); err != nil {
@@ -573,18 +587,22 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
 		pod.Labels[key] = val // sync StatefulSet labels to Pod to make it easier for users to select the Pod
 	}
 
+	if sts.Replicas > 0 {
+		ss.Spec.Replicas = ptr.To(sts.Replicas)
+	}
+
 	// Generic containerboot configuration options.
 	container.Env = append(container.Env,
 		corev1.EnvVar{
 			Name:  "TS_KUBE_SECRET",
-			Value: proxySecret,
+			Value: "$(POD_NAME)",
 		},
 		corev1.EnvVar{
-			// New style is in the form of cap-<capability-version>.hujson.
 			Name:  "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR",
-			Value: "/etc/tsconfig",
+			Value: "/etc/tsconfig/$(POD_NAME)",
 		},
 	)
+
 	if sts.ForwardClusterTrafficViaL7IngressProxy {
 		container.Env = append(container.Env, corev1.EnvVar{
 			Name:  "EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS",
@@ -592,20 +610,23 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
 		})
 	}
 
+	for i, secret := range proxySecrets {
 		configVolume := corev1.Volume{
-		Name: "tailscaledconfig",
+			Name: "tailscaledconfig-" + strconv.Itoa(i),
 			VolumeSource: corev1.VolumeSource{
 				Secret: &corev1.SecretVolumeSource{
-				SecretName: proxySecret,
+					SecretName: secret,
 				},
 			},
 		}
+
 		pod.Spec.Volumes = append(ss.Spec.Template.Spec.Volumes, configVolume)
 		container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{
-		Name:      "tailscaledconfig",
+			Name:      fmt.Sprintf("tailscaledconfig-%d", i),
 			ReadOnly:  true,
-		MountPath: "/etc/tsconfig",
+			MountPath: path.Join("/etc/tsconfig/", secret),
 		})
+	}
 
 	if a.tsFirewallMode != "" {
 		container.Env = append(container.Env, corev1.EnvVar{
@@ -643,24 +664,29 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
 	} else if sts.ServeConfig != nil {
 		container.Env = append(container.Env, corev1.EnvVar{
 			Name:  "TS_SERVE_CONFIG",
-			Value: "/etc/tailscaled/serve-config",
+			Value: "/etc/tailscaled/$(POD_NAME)-serve-config",
 		})
+
+		for _, secret := range proxySecrets {
 			container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{
-			Name:      "serve-config",
+				Name:      "serve-config-" + secret,
 				ReadOnly:  true,
 				MountPath: "/etc/tailscaled",
 			})
+
 			pod.Spec.Volumes = append(ss.Spec.Template.Spec.Volumes, corev1.Volume{
-			Name: "serve-config",
+				Name: "serve-config-" + secret,
 				VolumeSource: corev1.VolumeSource{
 					Secret: &corev1.SecretVolumeSource{
-					SecretName: proxySecret,
+						SecretName: "secret",
 						Items:      []corev1.KeyToPath{{Key: "serve-config", Path: "serve-config"}},
 					},
 				},
 			})
 		}
 
+	}
+
 	app, err := appInfoForProxy(sts)
 	if err != nil {
 		// No need to error out if now or in future we end up in a

cmd/k8s-operator/testutils_test.go

@@ -69,7 +69,7 @@ type configOpts struct {
 	shouldRemoveAuthKey                            bool
 	secretExtraData                                map[string][]byte
 	resourceVersion                                string
-
+	replicas                                       *int32
 	enableMetrics                                  bool
 	serviceMonitorLabels                           tsapi.Labels
 }
@@ -202,7 +202,7 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef
 			},
 		},
 		Spec: appsv1.StatefulSetSpec{
-			Replicas: ptr.To[int32](1),
+			Replicas: opts.replicas,
 			Selector: &metav1.LabelSelector{
 				MatchLabels: map[string]string{"app": "1234-UID"},
 			},

k8s-operator/api.md

@@ -120,6 +120,7 @@ _Appears in:_
 | `subnetRouter` _[SubnetRouter](#subnetrouter)_ | SubnetRouter defines subnet routes that the Connector device should<br />expose to tailnet as a Tailscale subnet router.<br />https://tailscale.com/kb/1019/subnets/<br />If this field is unset, the device does not get configured as a Tailscale subnet router.<br />This field is mutually exclusive with the appConnector field. |  |  |
 | `appConnector` _[AppConnector](#appconnector)_ | AppConnector defines whether the Connector device should act as a Tailscale app connector. A Connector that is<br />configured as an app connector cannot be a subnet router or an exit node. If this field is unset, the<br />Connector does not act as an app connector.<br />Note that you will need to manually configure the permissions and the domains for the app connector via the<br />Admin panel.<br />Note also that the main tested and supported use case of this config option is to deploy an app connector on<br />Kubernetes to access SaaS applications available on the public internet. Using the app connector to expose<br />cluster workloads or other internal workloads to tailnet might work, but this is not a use case that we have<br />tested or optimised for.<br />If you are using the app connector to access SaaS applications because you need a predictable egress IP that<br />can be whitelisted, it is also your responsibility to ensure that cluster traffic from the connector flows<br />via that predictable IP, for example by enforcing that cluster egress traffic is routed via an egress NAT<br />device with a static IP address.<br />https://tailscale.com/kb/1281/app-connectors |  |  |
 | `exitNode` _boolean_ | ExitNode defines whether the Connector device should act as a Tailscale exit node. Defaults to false.<br />This field is mutually exclusive with the appConnector field.<br />https://tailscale.com/kb/1103/exit-nodes |  |  |
+| `replicas` _integer_ | Replicas specifies how many replicas to create the StatefulSet with.<br />Defaults to 1. |  | Minimum: 0 <br /> |
 
 
 #### ConnectorStatus

k8s-operator/apis/v1alpha1/types_connector.go

@@ -113,6 +113,12 @@ type ConnectorSpec struct {
 	// https://tailscale.com/kb/1103/exit-nodes
 	// +optional
 	ExitNode bool `json:"exitNode"`
+
+	// Replicas specifies how many replicas to create the StatefulSet with.
+	// Defaults to 1.
+	// +optional
+	// +kubebuilder:validation:Minimum=0
+	Replicas *int32 `json:"replicas,omitempty"`
 }
 
 // SubnetRouter defines subnet routes that should be exposed to tailnet via a

k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go

@@ -110,6 +110,11 @@ func (in *ConnectorSpec) DeepCopyInto(out *ConnectorSpec) {
 		*out = new(AppConnector)
 		(*in).DeepCopyInto(*out)
 	}
+	if in.Replicas != nil {
+		in, out := &in.Replicas, &out.Replicas
+		*out = new(int32)
+		**out = **in
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConnectorSpec.