cmd/k8s-operator: ensures proxy can run on GKE Autopilot

legacy iptables need NET_RAW cap that is dropped on Autopilot pods by default + privileged pods are not allowed on Autopilot

Updates #8184

Signed-off-by: Irbe Krumina <irbe@tailscale.com>

cmd/k8s-operator/manifests/proxy.yaml

@@ -35,3 +35,4 @@ spec:
             capabilities:
               add:
                 - NET_ADMIN
+                - NET_RAW

cmd/k8s-operator/operator.go

@@ -53,6 +53,7 @@ func main() {
 		priorityClassName  = defaultEnv("PROXY_PRIORITY_CLASS_NAME", "")
 		tags               = defaultEnv("PROXY_TAGS", "tag:k8s")
 		shouldRunAuthProxy = defaultBool("AUTH_PROXY", false)
+		runInRestrictedEnv = defaultBool("RUN_IN_RESTRICTED_ENV", false)
 	)
 
 	var opts []kzap.Opts
@@ -73,7 +74,7 @@ func main() {
 	if shouldRunAuthProxy {
 		launchAuthProxy(zlog, restConfig, s)
 	}
-	startReconcilers(zlog, s, tsNamespace, restConfig, tsClient, image, priorityClassName, tags)
+	startReconcilers(zlog, s, tsNamespace, restConfig, tsClient, image, priorityClassName, tags, runInRestrictedEnv)
 }
 
 // initTSNet initializes the tsnet.Server and logs in to Tailscale. It uses the
@@ -182,7 +183,7 @@ waitOnline:
 
 // startReconcilers starts the controller-runtime manager and registers the
 // ServiceReconciler.
-func startReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string, restConfig *rest.Config, tsClient *tailscale.Client, image, priorityClassName, tags string) {
+func startReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string, restConfig *rest.Config, tsClient *tailscale.Client, image, priorityClassName, tags string, runInRestrictedEnv bool) {
 	var (
 		isDefaultLoadBalancer = defaultBool("OPERATOR_DEFAULT_LOAD_BALANCER", false)
 	)
@@ -231,6 +232,7 @@ func startReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace stri
 		operatorNamespace:      tsNamespace,
 		proxyImage:             image,
 		proxyPriorityClassName: priorityClassName,
+		runInRestrictedEnv:     runInRestrictedEnv,
 	}
 	err = builder.
 		ControllerManagedBy(mgr).

cmd/k8s-operator/operator_test.go

@@ -767,6 +767,54 @@ func TestCustomPriorityClassName(t *testing.T) {
 
 	expectEqual(t, fc, expectedSTS(shortName, fullName, "custom-priority-class-name", "tailscale-critical"))
 }
+func TestRunInRestrictedEnv(t *testing.T) {
+	fc := fake.NewFakeClient()
+	ft := &fakeTSClient{}
+	zl, err := zap.NewDevelopment()
+	if err != nil {
+		t.Fatal(err)
+	}
+	sr := &ServiceReconciler{
+		Client: fc,
+		ssr: &tailscaleSTSReconciler{
+			Client:             fc,
+			tsClient:           ft,
+			defaultTags:        []string{"tag:k8s"},
+			operatorNamespace:  "operator-ns",
+			proxyImage:         "tailscale/tailscale",
+			runInRestrictedEnv: true,
+		},
+		logger: zl.Sugar(),
+	}
+
+	// Create a service that we should manage, and check that the initial round
+	// of objects looks right.
+	mustCreate(t, fc, &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test",
+			Namespace: "default",
+			// The apiserver is supposed to set the UID, but the fake client
+			// doesn't. So, set it explicitly because other code later depends
+			// on it being set.
+			UID: types.UID("1234-UID"),
+			Annotations: map[string]string{
+				"tailscale.com/expose": "true",
+			},
+		},
+		Spec: corev1.ServiceSpec{
+			ClusterIP: "10.20.30.40",
+			Type:      corev1.ServiceTypeClusterIP,
+		},
+	})
+
+	expectReconciled(t, sr, "default", "test")
+
+	fullName, shortName := findGenName(t, fc, "default", "test")
+	sts := expectedSTS(shortName, fullName, "default-test", "")
+	sts.Spec.Template.Spec.InitContainers = nil
+
+	expectEqual(t, fc, sts)
+}
 
 func TestDefaultLoadBalancer(t *testing.T) {
 	fc := fake.NewFakeClient()
@@ -920,7 +968,7 @@ func expectedSTS(stsName, secretName, hostname, priorityClassName string) *appsv
 							},
 							SecurityContext: &corev1.SecurityContext{
 								Capabilities: &corev1.Capabilities{
-									Add: []corev1.Capability{"NET_ADMIN"},
+									Add: []corev1.Capability{"NET_ADMIN", "NET_RAW"},
 								},
 							},
 							ImagePullPolicy: "Always",
@@ -989,7 +1037,7 @@ func expectedEgressSTS(stsName, secretName, tailnetTargetIP, hostname, priorityC
 							},
 							SecurityContext: &corev1.SecurityContext{
 								Capabilities: &corev1.Capabilities{
-									Add: []corev1.Capability{"NET_ADMIN"},
+									Add: []corev1.Capability{"NET_ADMIN", "NET_RAW"},
 								},
 							},
 							ImagePullPolicy: "Always",

cmd/k8s-operator/sts.go

@@ -78,6 +78,7 @@ type tailscaleSTSReconciler struct {
 	operatorNamespace      string
 	proxyImage             string
 	proxyPriorityClassName string
+	runInRestrictedEnv     bool
 }
 
 // IsHTTPSEnabledOnTailnet reports whether HTTPS is enabled on the tailnet.
@@ -381,6 +382,15 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
 		"app": sts.ParentResourceUID,
 	}
 	ss.Spec.Template.Spec.PriorityClassName = a.proxyPriorityClassName
+
+	// If we run in a restricted env, do not run the privileged syctler init
+	// container that ensures IP forwarding. In most cases this will work
+	// anyway without extra action from users as the forwarding is usually
+	// enabled in kube containers. If it was not, there is an extra check in
+	// containerboot that will error out.
+	if a.runInRestrictedEnv {
+		ss.Spec.Template.Spec.InitContainers = nil
+	}
 	logger.Debugf("reconciling statefulset %s/%s", ss.GetNamespace(), ss.GetName())
 	return createOrUpdate(ctx, a.Client, a.operatorNamespace, &ss, func(s *appsv1.StatefulSet) { s.Spec = ss.Spec })
 }