mirrors/tailscale
1
0
Fork 0
mirror of https://github.com/tailscale/tailscale.git synced 2025-09-20 21:21:23 +02:00
Code Issues Packages Projects Releases Wiki Activity

cmd/{k8s-operator,k8s-proxy}: add kube-apiserver ProxyGroup type (#16266)

Browse Source 
Adds a new k8s-proxy command to convert operator's in-process proxy to
a separately deployable type of ProxyGroup: kube-apiserver. k8s-proxy
reads in a new config file written by the operator, modelled on tailscaled's
conffile but with some modifications to ensure multiple versions of the
config can co-exist within a file. This should make it much easier to
support reading that config file from a Kube Secret with a stable file name.

To avoid needing to give the operator ClusterRole{,Binding} permissions,
the helm chart now optionally deploys a new static ServiceAccount for
the API Server proxy to use if in auth mode.

Proxies deployed by kube-apiserver ProxyGroups currently work the same as
the operator's in-process proxy. They do not yet leverage Tailscale Services
for presenting a single HA DNS name.

Updates #13358

Change-Id: Ib6ead69b2173c5e1929f3c13fb48a9a5362195d8
Signed-off-by: Tom Proctor <tomhjp@users.noreply.github.com>
This commit is contained in:
Tom Proctor 2025-07-09 09:21:56 +01:00 committed by GitHub
parent 90bf0a97b3
commit 4dfed6b146
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
31 changed files with 1788 additions and 351 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
Makefile
View File

@ -92,38 +92,38 @@ pushspk: spk ## Push and install synology package on ${SYNO_HOST} host
scp tailscale.spk root@${SYNO_HOST}:
ssh root@${SYNO_HOST} /usr/syno/bin/synopkg install tailscale.spk
publishdevimage: ## Build and publish tailscale image to location specified by ${REPO}
@test -n "${REPO}" || (echo "REPO=... required; e.g. REPO=ghcr.io/${USER}/tailscale" && exit 1)
@test "${REPO}" != "tailscale/tailscale" || (echo "REPO=... must not be tailscale/tailscale" && exit 1)
@test "${REPO}" != "ghcr.io/tailscale/tailscale" || (echo "REPO=... must not be ghcr.io/tailscale/tailscale" && exit 1)
@test "${REPO}" != "tailscale/k8s-operator" || (echo "REPO=... must not be tailscale/k8s-operator" && exit 1)
@test "${REPO}" != "ghcr.io/tailscale/k8s-operator" || (echo "REPO=... must not be ghcr.io/tailscale/k8s-operator" && exit 1)
.PHONY: check-image-repo
check-image-repo:
@if [ -z "$(REPO)" ]; then \
echo "REPO=... required; e.g. REPO=ghcr.io/$$USER/tailscale" >&2; \
exit 1; \
fi
@for repo in tailscale/tailscale ghcr.io/tailscale/tailscale \
tailscale/k8s-operator ghcr.io/tailscale/k8s-operator \
tailscale/k8s-nameserver ghcr.io/tailscale/k8s-nameserver \
tailscale/tsidp ghcr.io/tailscale/tsidp \
tailscale/k8s-proxy ghcr.io/tailscale/k8s-proxy; do \
if [ "$(REPO)" = "$$repo" ]; then \
echo "REPO=... must not be $$repo" >&2; \
exit 1; \
fi; \
done
publishdevimage: check-image-repo ## Build and publish tailscale image to location specified by ${REPO}
TAGS="${TAGS}" REPOS=${REPO} PLATFORM=${PLATFORM} PUSH=true TARGET=client ./build_docker.sh
publishdevoperator: ## Build and publish k8s-operator image to location specified by ${REPO}
@test -n "${REPO}" || (echo "REPO=... required; e.g. REPO=ghcr.io/${USER}/tailscale" && exit 1)
@test "${REPO}" != "tailscale/tailscale" || (echo "REPO=... must not be tailscale/tailscale" && exit 1)
@test "${REPO}" != "ghcr.io/tailscale/tailscale" || (echo "REPO=... must not be ghcr.io/tailscale/tailscale" && exit 1)
@test "${REPO}" != "tailscale/k8s-operator" || (echo "REPO=... must not be tailscale/k8s-operator" && exit 1)
@test "${REPO}" != "ghcr.io/tailscale/k8s-operator" || (echo "REPO=... must not be ghcr.io/tailscale/k8s-operator" && exit 1)
publishdevoperator: check-image-repo ## Build and publish k8s-operator image to location specified by ${REPO}
TAGS="${TAGS}" REPOS=${REPO} PLATFORM=${PLATFORM} PUSH=true TARGET=k8s-operator ./build_docker.sh
publishdevnameserver: ## Build and publish k8s-nameserver image to location specified by ${REPO}
@test -n "${REPO}" || (echo "REPO=... required; e.g. REPO=ghcr.io/${USER}/tailscale" && exit 1)
@test "${REPO}" != "tailscale/tailscale" || (echo "REPO=... must not be tailscale/tailscale" && exit 1)
@test "${REPO}" != "ghcr.io/tailscale/tailscale" || (echo "REPO=... must not be ghcr.io/tailscale/tailscale" && exit 1)
@test "${REPO}" != "tailscale/k8s-nameserver" || (echo "REPO=... must not be tailscale/k8s-nameserver" && exit 1)
@test "${REPO}" != "ghcr.io/tailscale/k8s-nameserver" || (echo "REPO=... must not be ghcr.io/tailscale/k8s-nameserver" && exit 1)
publishdevnameserver: check-image-repo ## Build and publish k8s-nameserver image to location specified by ${REPO}
TAGS="${TAGS}" REPOS=${REPO} PLATFORM=${PLATFORM} PUSH=true TARGET=k8s-nameserver ./build_docker.sh
publishdevtsidp: ## Build and publish tsidp image to location specified by ${REPO}
@test -n "${REPO}" || (echo "REPO=... required; e.g. REPO=ghcr.io/${USER}/tailscale" && exit 1)
@test "${REPO}" != "tailscale/tailscale" || (echo "REPO=... must not be tailscale/tailscale" && exit 1)
@test "${REPO}" != "ghcr.io/tailscale/tailscale" || (echo "REPO=... must not be ghcr.io/tailscale/tailscale" && exit 1)
@test "${REPO}" != "tailscale/tsidp" || (echo "REPO=... must not be tailscale/tsidp" && exit 1)
@test "${REPO}" != "ghcr.io/tailscale/tsidp" || (echo "REPO=... must not be ghcr.io/tailscale/tsidp" && exit 1)
publishdevtsidp: check-image-repo ## Build and publish tsidp image to location specified by ${REPO}
TAGS="${TAGS}" REPOS=${REPO} PLATFORM=${PLATFORM} PUSH=true TARGET=tsidp ./build_docker.sh
publishdevproxy: check-image-repo ## Build and publish k8s-proxy image to location specified by ${REPO}
TAGS="${TAGS}" REPOS=${REPO} PLATFORM=${PLATFORM} PUSH=true TARGET=k8s-proxy ./build_docker.sh
.PHONY: sshintegrationtest
sshintegrationtest: ## Run the SSH integration tests in various Docker containers
@GOOS=linux GOARCH=amd64 ./tool/go test -tags integrationtest -c ./ssh/tailssh -o ssh/tailssh/testcontainers/tailssh.test && \

18
build_docker.sh
View File

@ -118,6 +118,24 @@ case "$TARGET" in
--annotations="${ANNOTATIONS}" \
/usr/local/bin/tsidp
;;
k8s-proxy)
DEFAULT_REPOS="tailscale/k8s-proxy"
REPOS="${REPOS:-${DEFAULT_REPOS}}"
go run github.com/tailscale/mkctr \
--gopaths="tailscale.com/cmd/k8s-proxy:/usr/local/bin/k8s-proxy" \
--ldflags=" \
-X tailscale.com/version.longStamp=${VERSION_LONG} \
-X tailscale.com/version.shortStamp=${VERSION_SHORT} \
-X tailscale.com/version.gitCommitStamp=${VERSION_GIT_HASH}" \
--base="${BASE}" \
--tags="${TAGS}" \
--gotags="ts_kube,ts_package_container" \
--repos="${REPOS}" \
--push="${PUSH}" \
--target="${PLATFORM}" \
--annotations="${ANNOTATIONS}" \
/usr/local/bin/k8s-proxy
;;
*)
echo "unknown target: $TARGET"
exit 1

3
cmd/k8s-operator/depaware.txt
View File

@ -200,7 +200,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
github.com/tailscale/goupnp/scpd from github.com/tailscale/goupnp
github.com/tailscale/goupnp/soap from github.com/tailscale/goupnp+
github.com/tailscale/goupnp/ssdp from github.com/tailscale/goupnp
github.com/tailscale/hujson from tailscale.com/ipn/conffile
github.com/tailscale/hujson from tailscale.com/ipn/conffile+
L 💣 github.com/tailscale/netlink from tailscale.com/net/routetable+
L 💣 github.com/tailscale/netlink/nl from github.com/tailscale/netlink
github.com/tailscale/peercred from tailscale.com/ipn/ipnauth
@ -822,6 +822,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
tailscale.com/k8s-operator/sessionrecording/ws from tailscale.com/k8s-operator/sessionrecording
tailscale.com/kube/egressservices from tailscale.com/cmd/k8s-operator
tailscale.com/kube/ingressservices from tailscale.com/cmd/k8s-operator
tailscale.com/kube/k8s-proxy/conf from tailscale.com/cmd/k8s-operator
tailscale.com/kube/kubeapi from tailscale.com/ipn/store/kubestore+
tailscale.com/kube/kubeclient from tailscale.com/ipn/store/kubestore
tailscale.com/kube/kubetypes from tailscale.com/cmd/k8s-operator+

16
cmd/k8s-operator/deploy/chart/templates/apiserverproxy-rbac.yaml
View File

@ -1,7 +1,16 @@
# Copyright (c) Tailscale Inc & AUTHORS
# SPDX-License-Identifier: BSD-3-Clause
{{ if eq .Values.apiServerProxyConfig.mode "true" }}
# If old setting used, enable both old (operator) and new (ProxyGroup) workflows.
# If new setting used, enable only new workflow.
{{ if or (eq .Values.apiServerProxyConfig.mode "true")
(eq .Values.apiServerProxyConfig.allowImpersonation "true") }}
apiVersion: v1
kind: ServiceAccount
metadata:
name: kube-apiserver-auth-proxy
namespace: {{ .Release.Namespace }}
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
@ -16,9 +25,14 @@ kind: ClusterRoleBinding
metadata:
name: tailscale-auth-proxy
subjects:
{{- if eq .Values.apiServerProxyConfig.mode "true" }}
- kind: ServiceAccount
name: operator
namespace: {{ .Release.Namespace }}
{{- end }}
- kind: ServiceAccount
name: kube-apiserver-auth-proxy
namespace: {{ .Release.Namespace }}
roleRef:
kind: ClusterRole
name: tailscale-auth-proxy

16
cmd/k8s-operator/deploy/chart/values.yaml
View File

@ -92,6 +92,13 @@ ingressClass:
# If you need more configuration options, take a look at ProxyClass:
# https://tailscale.com/kb/1445/kubernetes-operator-customization#cluster-resource-customization-using-proxyclass-custom-resource
proxyConfig:
# Configure the proxy image to use instead of the default tailscale/tailscale:latest.
# Applying a ProxyClass with `spec.statefulSet.pod.tailscaleContainer.image`
# set will override any defaults here.
#
# Note that ProxyGroups of type "kube-apiserver" use a different default image,
# tailscale/k8s-proxy:latest, and it is currently only possible to override
# that image via the same ProxyClass field.
image:
# Repository defaults to DockerHub, but images are also synced to ghcr.io/tailscale/tailscale.
repository: tailscale/tailscale
@ -115,6 +122,15 @@ proxyConfig:
# Kubernetes API server.
# https://tailscale.com/kb/1437/kubernetes-operator-api-server-proxy
apiServerProxyConfig:
# Set to "true" to create the ClusterRole permissions required for the API
# server proxy's auth mode. In auth mode, the API server proxy impersonates
# groups and users based on tailnet ACL grants. Required for ProxyGroups of
# type "kube-apiserver" running in auth mode.
allowImpersonation: "false" # "true", "false"
# If true or noauth, the operator will run an in-process API server proxy.
# You can deploy a ProxyGroup of type "kube-apiserver" to run a high
# availability set of API server proxies instead.
mode: "false" # "true", "false", "noauth"
imagePullSecrets: []

46
cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml
View File

@ -1379,12 +1379,21 @@ spec:
type: string
image:
description: |-
Container image name. By default images are pulled from
docker.io/tailscale/tailscale, but the official images are also
available at ghcr.io/tailscale/tailscale. Specifying image name here
will override any proxy image values specified via the Kubernetes
operator's Helm chart values or PROXY_IMAGE env var in the operator
Deployment.
Container image name. By default images are pulled from docker.io/tailscale,
but the official images are also available at ghcr.io/tailscale.
For all uses except on ProxyGroups of type "kube-apiserver", this image must
be either tailscale/tailscale, or an equivalent mirror of that image.
To apply to ProxyGroups of type "kube-apiserver", this image must be
tailscale/k8s-proxy or a mirror of that image.
For "tailscale/tailscale"-based proxies, specifying image name here will
override any proxy image values specified via the Kubernetes operator's
Helm chart values or PROXY_IMAGE env var in the operator Deployment.
For "tailscale/k8s-proxy"-based proxies, there is currently no way to
configure your own default, and this field is the only way to use a
custom image.
https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#image
type: string
imagePullPolicy:
@ -1655,7 +1664,9 @@ spec:
PodSecurityContext, the value specified in SecurityContext takes precedence.
type: string
tailscaleInitContainer:
description: Configuration for the proxy init container that enables forwarding.
description: |-
Configuration for the proxy init container that enables forwarding.
Not valid to apply to ProxyGroups of type "kube-apiserver".
type: object
properties:
debug:
@ -1709,12 +1720,21 @@ spec:
type: string
image:
description: |-
Container image name. By default images are pulled from
docker.io/tailscale/tailscale, but the official images are also
available at ghcr.io/tailscale/tailscale. Specifying image name here
will override any proxy image values specified via the Kubernetes
operator's Helm chart values or PROXY_IMAGE env var in the operator
Deployment.
Container image name. By default images are pulled from docker.io/tailscale,
but the official images are also available at ghcr.io/tailscale.
For all uses except on ProxyGroups of type "kube-apiserver", this image must
be either tailscale/tailscale, or an equivalent mirror of that image.
To apply to ProxyGroups of type "kube-apiserver", this image must be
tailscale/k8s-proxy or a mirror of that image.
For "tailscale/tailscale"-based proxies, specifying image name here will
override any proxy image values specified via the Kubernetes operator's
Helm chart values or PROXY_IMAGE env var in the operator Deployment.
For "tailscale/k8s-proxy"-based proxies, there is currently no way to
configure your own default, and this field is the only way to use a
custom image.
https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#image
type: string
imagePullPolicy:

19
cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml
View File

@ -77,6 +77,22 @@ spec:
must not start with a dash and must be between 1 and 62 characters long.
type: string
pattern: ^[a-z0-9][a-z0-9-]{0,61}$
kubeAPIServer:
description: |-
KubeAPIServer contains configuration specific to the kube-apiserver
ProxyGroup type. This field is only used when Type is set to "kube-apiserver".
type: object
properties:
mode:
description: |-
Mode to run the API server proxy in. Supported modes are auth and noauth.
In auth mode, requests from the tailnet proxied over to the Kubernetes
API server are additionally impersonated using the sender's tailnet identity.
If not specified, defaults to auth mode.
type: string
enum:
- auth
- noauth
proxyClass:
description: |-
ProxyClass is the name of the ProxyClass custom resource that contains
@ -106,12 +122,13 @@ spec:
pattern: ^tag:[a-zA-Z][a-zA-Z0-9-]*$
type:
description: |-
Type of the ProxyGroup proxies. Supported types are egress and ingress.
Type of the ProxyGroup proxies. Supported types are egress, ingress, and kube-apiserver.
Type is immutable once a ProxyGroup is created.
type: string
enum:
- egress
- ingress
- kube-apiserver
x-kubernetes-validations:
- rule: self == oldSelf
message: ProxyGroup type is immutable

9
cmd/k8s-operator/deploy/manifests/authproxy-rbac.yaml
View File

@ -1,6 +1,12 @@
# Copyright (c) Tailscale Inc & AUTHORS
# SPDX-License-Identifier: BSD-3-Clause
apiVersion: v1
kind: ServiceAccount
metadata:
name: kube-apiserver-auth-proxy
namespace: tailscale
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
@ -18,6 +24,9 @@ subjects:
- kind: ServiceAccount
name: operator
namespace: tailscale
- kind: ServiceAccount
name: kube-apiserver-auth-proxy
namespace: tailscale
roleRef:
kind: ClusterRole
name: tailscale-auth-proxy

65
cmd/k8s-operator/deploy/manifests/operator.yaml
View File

@ -1852,12 +1852,21 @@ spec:
type: array
image:
description: |-
Container image name. By default images are pulled from
docker.io/tailscale/tailscale, but the official images are also
available at ghcr.io/tailscale/tailscale. Specifying image name here
will override any proxy image values specified via the Kubernetes
operator's Helm chart values or PROXY_IMAGE env var in the operator
Deployment.
Container image name. By default images are pulled from docker.io/tailscale,
but the official images are also available at ghcr.io/tailscale.
For all uses except on ProxyGroups of type "kube-apiserver", this image must
be either tailscale/tailscale, or an equivalent mirror of that image.
To apply to ProxyGroups of type "kube-apiserver", this image must be
tailscale/k8s-proxy or a mirror of that image.
For "tailscale/tailscale"-based proxies, specifying image name here will
override any proxy image values specified via the Kubernetes operator's
Helm chart values or PROXY_IMAGE env var in the operator Deployment.
For "tailscale/k8s-proxy"-based proxies, there is currently no way to
configure your own default, and this field is the only way to use a
custom image.
https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#image
type: string
imagePullPolicy:
@ -2129,7 +2138,9 @@ spec:
type: object
type: object
tailscaleInitContainer:
description: Configuration for the proxy init container that enables forwarding.
description: |-
Configuration for the proxy init container that enables forwarding.
Not valid to apply to ProxyGroups of type "kube-apiserver".
properties:
debug:
description: |-
@ -2182,12 +2193,21 @@ spec:
type: array
image:
description: |-
Container image name. By default images are pulled from
docker.io/tailscale/tailscale, but the official images are also
available at ghcr.io/tailscale/tailscale. Specifying image name here
will override any proxy image values specified via the Kubernetes
operator's Helm chart values or PROXY_IMAGE env var in the operator
Deployment.
Container image name. By default images are pulled from docker.io/tailscale,
but the official images are also available at ghcr.io/tailscale.
For all uses except on ProxyGroups of type "kube-apiserver", this image must
be either tailscale/tailscale, or an equivalent mirror of that image.
To apply to ProxyGroups of type "kube-apiserver", this image must be
tailscale/k8s-proxy or a mirror of that image.
For "tailscale/tailscale"-based proxies, specifying image name here will
override any proxy image values specified via the Kubernetes operator's
Helm chart values or PROXY_IMAGE env var in the operator Deployment.
For "tailscale/k8s-proxy"-based proxies, there is currently no way to
configure your own default, and this field is the only way to use a
custom image.
https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#image
type: string
imagePullPolicy:
@ -2904,6 +2924,22 @@ spec:
must not start with a dash and must be between 1 and 62 characters long.
pattern: ^[a-z0-9][a-z0-9-]{0,61}$
type: string
kubeAPIServer:
description: |-
KubeAPIServer contains configuration specific to the kube-apiserver
ProxyGroup type. This field is only used when Type is set to "kube-apiserver".
properties:
mode:
description: |-
Mode to run the API server proxy in. Supported modes are auth and noauth.
In auth mode, requests from the tailnet proxied over to the Kubernetes
API server are additionally impersonated using the sender's tailnet identity.
If not specified, defaults to auth mode.
enum:
- auth
- noauth
type: string
type: object
proxyClass:
description: |-
ProxyClass is the name of the ProxyClass custom resource that contains
@ -2933,11 +2969,12 @@ spec:
type: array
type:
description: |-
Type of the ProxyGroup proxies. Supported types are egress and ingress.
Type of the ProxyGroup proxies. Supported types are egress, ingress, and kube-apiserver.
Type is immutable once a ProxyGroup is created.
enum:
- egress
- ingress
- kube-apiserver
type: string
x-kubernetes-validations:
- message: ProxyGroup type is immutable

6
cmd/k8s-operator/ingress-for-pg.go
View File

@ -239,7 +239,7 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin
// This checks and ensures that Tailscale Service's owner references are updated
// for this Ingress and errors if that is not possible (i.e. because it
// appears that the Tailscale Service has been created by a non-operator actor).
updatedAnnotations, err := r.ownerAnnotations(existingTSSvc)
updatedAnnotations, err := ownerAnnotations(r.operatorID, existingTSSvc)
if err != nil {
const instr = "To proceed, you can either manually delete the existing Tailscale Service or choose a different MagicDNS name at `.spec.tls.hosts[0] in the Ingress definition"
msg := fmt.Sprintf("error ensuring ownership of Tailscale Service %s: %v. %s", hostname, err, instr)
@ -867,9 +867,9 @@ type OwnerRef struct {
// nil, but does not contain an owner reference we return an error as this likely means
// that the Service was created by somthing other than a Tailscale
// Kubernetes operator.
func (r *HAIngressReconciler) ownerAnnotations(svc *tailscale.VIPService) (map[string]string, error) {
func ownerAnnotations(operatorID string, svc *tailscale.VIPService) (map[string]string, error) {
ref := OwnerRef{
OperatorID: r.operatorID,
OperatorID: operatorID,
}
if svc == nil {
c := ownerAnnotationValue{OwnerRefs: []OwnerRef{ref}}

49
cmd/k8s-operator/ingress-for-pg_test.go
View File

@ -12,8 +12,10 @@ import (
"maps"
"reflect"
"slices"
"strings"
"testing"
"github.com/google/go-cmp/cmp"
"go.uber.org/zap"
corev1 "k8s.io/api/core/v1"
networkingv1 "k8s.io/api/networking/v1"
@ -650,6 +652,53 @@ func TestIngressPGReconciler_MultiCluster(t *testing.T) {
}
}
func TestOwnerAnnotations(t *testing.T) {
singleSelfOwner := map[string]string{
ownerAnnotation: `{"ownerRefs":[{"operatorID":"self-id"}]}`,
}
for name, tc := range map[string]struct {
svc *tailscale.VIPService
wantAnnotations map[string]string
wantErr string
}{
"no_svc": {
svc: nil,
wantAnnotations: singleSelfOwner,
},
"empty_svc": {
svc: &tailscale.VIPService{},
wantErr: "likely a resource created by something other than the Tailscale Kubernetes operator",
},
"already_owner": {
svc: &tailscale.VIPService{
Annotations: singleSelfOwner,
},
wantAnnotations: singleSelfOwner,
},
"add_owner": {
svc: &tailscale.VIPService{
Annotations: map[string]string{
ownerAnnotation: `{"ownerRefs":[{"operatorID":"operator-2"}]}`,
},
},
wantAnnotations: map[string]string{
ownerAnnotation: `{"ownerRefs":[{"operatorID":"operator-2"},{"operatorID":"self-id"}]}`,
},
},
} {
t.Run(name, func(t *testing.T) {
got, err := ownerAnnotations("self-id", tc.svc)
if tc.wantErr != "" && !strings.Contains(err.Error(), tc.wantErr) {
t.Errorf("ownerAnnotations() error = %v, wantErr %v", err, tc.wantErr)
}
if diff := cmp.Diff(tc.wantAnnotations, got); diff != "" {
t.Errorf("ownerAnnotations() mismatch (-want +got):\n%s", diff)
}
})
}
}
func populateTLSSecret(ctx context.Context, c client.Client, pgName, domain string) error {
secret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{

64
cmd/k8s-operator/operator.go
View File

@ -26,6 +26,7 @@ import (
networkingv1 "k8s.io/api/networking/v1"
rbacv1 "k8s.io/api/rbac/v1"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
apiequality "k8s.io/apimachinery/pkg/api/equality"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
klabels "k8s.io/apimachinery/pkg/labels"
@ -77,6 +78,7 @@ func main() {
tsNamespace = defaultEnv("OPERATOR_NAMESPACE", "")
tslogging = defaultEnv("OPERATOR_LOGGING", "info")
image = defaultEnv("PROXY_IMAGE", "tailscale/tailscale:latest")
k8sProxyImage = defaultEnv("K8S_PROXY_IMAGE", "tailscale/k8s-proxy:latest")
priorityClassName = defaultEnv("PROXY_PRIORITY_CLASS_NAME", "")
tags = defaultEnv("PROXY_TAGS", "tag:k8s")
tsFirewallMode = defaultEnv("PROXY_FIREWALL_MODE", "")
@ -110,17 +112,27 @@ func main() {
// The operator can run either as a plain operator or it can
// additionally act as api-server proxy
// https://tailscale.com/kb/1236/kubernetes-operator/?q=kubernetes#accessing-the-kubernetes-control-plane-using-an-api-server-proxy.
mode := apiproxy.ParseAPIProxyMode()
if mode == apiproxy.APIServerProxyModeDisabled {
mode := parseAPIProxyMode()
if mode == apiServerProxyModeDisabled {
hostinfo.SetApp(kubetypes.AppOperator)
} else {
hostinfo.SetApp(kubetypes.AppAPIServerProxy)
hostinfo.SetApp(kubetypes.AppInProcessAPIServerProxy)
}
s, tsc := initTSNet(zlog, loginServer)
defer s.Close()
restConfig := config.GetConfigOrDie()
apiproxy.MaybeLaunchAPIServerProxy(zlog, restConfig, s, mode)
if mode != apiServerProxyModeDisabled {
ap, err := apiproxy.NewAPIServerProxy(zlog, restConfig, s, mode == apiServerProxyModeEnabled)
if err != nil {
zlog.Fatalf("error creating API server proxy: %v", err)
}
go func() {
if err := ap.Run(context.Background()); err != nil {
zlog.Fatalf("error running API server proxy: %v", err)
}
}()
}
rOpts := reconcilerOpts{
log: zlog,
tsServer: s,
@ -128,6 +140,7 @@ func main() {
tailscaleNamespace: tsNamespace,
restConfig: restConfig,
proxyImage: image,
k8sProxyImage: k8sProxyImage,
proxyPriorityClassName: priorityClassName,
proxyActAsDefaultLoadBalancer: isDefaultLoadBalancer,
proxyTags: tags,
@ -415,7 +428,6 @@ func runReconcilers(opts reconcilerOpts) {
Complete(&HAServiceReconciler{
recorder: eventRecorder,
tsClient: opts.tsClient,
tsnetServer: opts.tsServer,
defaultTags: strings.Split(opts.proxyTags, ","),
Client: mgr.GetClient(),
logger: opts.log.Named("service-pg-reconciler"),
@ -625,13 +637,14 @@ func runReconcilers(opts reconcilerOpts) {
ownedByProxyGroupFilter := handler.EnqueueRequestForOwner(mgr.GetScheme(), mgr.GetRESTMapper(), &tsapi.ProxyGroup{})
proxyClassFilterForProxyGroup := handler.EnqueueRequestsFromMapFunc(proxyClassHandlerForProxyGroup(mgr.GetClient(), startlog))
nodeFilterForProxyGroup := handler.EnqueueRequestsFromMapFunc(nodeHandlerForProxyGroup(mgr.GetClient(), opts.defaultProxyClass, startlog))
saFilterForProxyGroup := handler.EnqueueRequestsFromMapFunc(serviceAccountHandlerForProxyGroup(mgr.GetClient(), startlog))
err = builder.ControllerManagedBy(mgr).
For(&tsapi.ProxyGroup{}).
Named("proxygroup-reconciler").
Watches(&corev1.Service{}, ownedByProxyGroupFilter).
Watches(&appsv1.StatefulSet{}, ownedByProxyGroupFilter).
Watches(&corev1.ConfigMap{}, ownedByProxyGroupFilter).
Watches(&corev1.ServiceAccount{}, ownedByProxyGroupFilter).
Watches(&corev1.ServiceAccount{}, saFilterForProxyGroup).
Watches(&corev1.Secret{}, ownedByProxyGroupFilter).
Watches(&rbacv1.Role{}, ownedByProxyGroupFilter).
Watches(&rbacv1.RoleBinding{}, ownedByProxyGroupFilter).
@ -645,7 +658,8 @@ func runReconcilers(opts reconcilerOpts) {
tsClient: opts.tsClient,
tsNamespace: opts.tailscaleNamespace,
proxyImage: opts.proxyImage,
tsProxyImage: opts.proxyImage,
k8sProxyImage: opts.k8sProxyImage,
defaultTags: strings.Split(opts.proxyTags, ","),
tsFirewallMode: opts.proxyFirewallMode,
defaultProxyClass: opts.defaultProxyClass,
@ -668,6 +682,7 @@ type reconcilerOpts struct {
tailscaleNamespace string // namespace in which operator resources will be deployed
restConfig *rest.Config // config for connecting to the kube API server
proxyImage string // <proxy-image-repo>:<proxy-image-tag>
k8sProxyImage string // <k8s-proxy-image-repo>:<k8s-proxy-image-tag>
// proxyPriorityClassName isPriorityClass to be set for proxy Pods. This
// is a legacy mechanism for cluster resource configuration options -
// going forward use ProxyClass.
@ -996,8 +1011,8 @@ func nodeHandlerForProxyGroup(cl client.Client, defaultProxyClass string, logger
}
// proxyClassHandlerForProxyGroup returns a handler that, for a given ProxyClass,
// returns a list of reconcile requests for all Connectors that have
// .spec.proxyClass set.
// returns a list of reconcile requests for all ProxyGroups that have
// .spec.proxyClass set to that ProxyClass.
func proxyClassHandlerForProxyGroup(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc {
return func(ctx context.Context, o client.Object) []reconcile.Request {
pgList := new(tsapi.ProxyGroupList)
@ -1016,6 +1031,37 @@ func proxyClassHandlerForProxyGroup(cl client.Client, logger *zap.SugaredLogger)
}
}
// serviceAccountHandlerForProxyGroup returns a handler that, for a given ServiceAccount,
// returns a list of reconcile requests for all ProxyGroups that use that ServiceAccount.
// For most ProxyGroups, this will be a dedicated ServiceAccount owned by a specific
// ProxyGroup. But for kube-apiserver ProxyGroups running in auth mode, they use a shared
// static ServiceAccount named "kube-apiserver-auth-proxy".
func serviceAccountHandlerForProxyGroup(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc {
return func(ctx context.Context, o client.Object) []reconcile.Request {
pgList := new(tsapi.ProxyGroupList)
if err := cl.List(ctx, pgList); err != nil {
logger.Debugf("error listing ProxyGroups for ServiceAccount: %v", err)
return nil
}
reqs := make([]reconcile.Request, 0)
saName := o.GetName()
for _, pg := range pgList.Items {
if saName == authAPIServerProxySAName && isAuthAPIServerProxy(&pg) {
reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&pg)})
}
expectedOwner := pgOwnerReference(&pg)[0]
saOwnerRefs := o.GetOwnerReferences()
for _, ref := range saOwnerRefs {
if apiequality.Semantic.DeepEqual(ref, expectedOwner) {
reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&pg)})
break
}
}
}
return reqs
}
}
// serviceHandlerForIngress returns a handler for Service events for ingress
// reconciler that ensures that if the Service associated with an event is of
// interest to the reconciler, the associated Ingress(es) gets be reconciled.

61
cmd/k8s-operator/proxy.go Normal file
View File

@ -0,0 +1,61 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
//go:build !plan9
package main
import (
"fmt"
"log"
"os"
)
type apiServerProxyMode int
func (a apiServerProxyMode) String() string {
switch a {
case apiServerProxyModeDisabled:
return "disabled"
case apiServerProxyModeEnabled:
return "auth"
case apiServerProxyModeNoAuth:
return "noauth"
default:
return "unknown"
}
}
const (
apiServerProxyModeDisabled apiServerProxyMode = iota
apiServerProxyModeEnabled
apiServerProxyModeNoAuth
)
func parseAPIProxyMode() apiServerProxyMode {
haveAuthProxyEnv := os.Getenv("AUTH_PROXY") != ""
haveAPIProxyEnv := os.Getenv("APISERVER_PROXY") != ""
switch {
case haveAPIProxyEnv && haveAuthProxyEnv:
log.Fatal("AUTH_PROXY (deprecated) and APISERVER_PROXY are mutually exclusive, please unset AUTH_PROXY")
case haveAuthProxyEnv:
var authProxyEnv = defaultBool("AUTH_PROXY", false) // deprecated
if authProxyEnv {
return apiServerProxyModeEnabled
}
return apiServerProxyModeDisabled
case haveAPIProxyEnv:
var apiProxyEnv = defaultEnv("APISERVER_PROXY", "") // true, false or "noauth"
switch apiProxyEnv {
case "true":
return apiServerProxyModeEnabled
case "false", "":
return apiServerProxyModeDisabled
case "noauth":
return apiServerProxyModeNoAuth
default:
panic(fmt.Sprintf("unknown APISERVER_PROXY value %q", apiProxyEnv))
}
}
return apiServerProxyModeDisabled
}

196
cmd/k8s-operator/proxygroup.go
View File

@ -17,6 +17,7 @@ import (
"strings"
"sync"
dockerref "github.com/distribution/reference"
"go.uber.org/zap"
xslices "golang.org/x/exp/slices"
appsv1 "k8s.io/api/apps/v1"
@ -36,9 +37,11 @@ import (
tsoperator "tailscale.com/k8s-operator"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/kube/egressservices"
"tailscale.com/kube/k8s-proxy/conf"
"tailscale.com/kube/kubetypes"
"tailscale.com/tailcfg"
"tailscale.com/tstime"
"tailscale.com/types/opt"
"tailscale.com/types/ptr"
"tailscale.com/util/clientmetric"
"tailscale.com/util/mak"
@ -48,7 +51,9 @@ import (
const (
reasonProxyGroupCreationFailed = "ProxyGroupCreationFailed"
reasonProxyGroupReady = "ProxyGroupReady"
reasonProxyGroupAvailable = "ProxyGroupAvailable"
reasonProxyGroupCreating = "ProxyGroupCreating"
reasonProxyGroupInvalid = "ProxyGroupInvalid"
// Copied from k8s.io/apiserver/pkg/registry/generic/registry/store.go@cccad306d649184bf2a0e319ba830c53f65c445c
optimisticLockErrorMsg = "the object has been modified; please apply your changes to the latest version and try again"
@ -63,12 +68,14 @@ const (
//
// tailcfg.CurrentCapabilityVersion was 106 when the ProxyGroup controller was
// first introduced.
pgMinCapabilityVersion = 106
pgMinCapabilityVersion = 106
kubeAPIServerConfigFile = "config.hujson"
)
var (
gaugeEgressProxyGroupResources = clientmetric.NewGauge(kubetypes.MetricProxyGroupEgressCount)
gaugeIngressProxyGroupResources = clientmetric.NewGauge(kubetypes.MetricProxyGroupIngressCount)
gaugeEgressProxyGroupResources = clientmetric.NewGauge(kubetypes.MetricProxyGroupEgressCount)
gaugeIngressProxyGroupResources = clientmetric.NewGauge(kubetypes.MetricProxyGroupIngressCount)
gaugeAPIServerProxyGroupResources = clientmetric.NewGauge(kubetypes.MetricProxyGroupAPIServerCount)
)
// ProxyGroupReconciler ensures cluster resources for a ProxyGroup definition.
@ -81,15 +88,17 @@ type ProxyGroupReconciler struct {
// User-specified defaults from the helm installation.
tsNamespace string
proxyImage string
tsProxyImage string
k8sProxyImage string
defaultTags []string
tsFirewallMode string
defaultProxyClass string
loginServer string
mu sync.Mutex // protects following
egressProxyGroups set.Slice[types.UID] // for egress proxygroups gauge
ingressProxyGroups set.Slice[types.UID] // for ingress proxygroups gauge
mu sync.Mutex // protects following
egressProxyGroups set.Slice[types.UID] // for egress proxygroups gauge
ingressProxyGroups set.Slice[types.UID] // for ingress proxygroups gauge
apiServerProxyGroups set.Slice[types.UID] // for kube-apiserver proxygroups gauge
}
func (r *ProxyGroupReconciler) logger(name string) *zap.SugaredLogger {
@ -170,7 +179,6 @@ func (r *ProxyGroupReconciler) reconcilePG(ctx context.Context, pg *tsapi.ProxyG
if err != nil {
return r.notReadyErrf(pg, "error getting ProxyGroup's ProxyClass %q: %w", proxyClassName, err)
}
validateProxyClassForPG(logger, pg, proxyClass)
if !tsoperator.ProxyClassIsReady(proxyClass) {
msg := fmt.Sprintf("the ProxyGroup's ProxyClass %q is not yet in a ready state, waiting...", proxyClassName)
logger.Info(msg)
@ -178,6 +186,10 @@ func (r *ProxyGroupReconciler) reconcilePG(ctx context.Context, pg *tsapi.ProxyG
}
}
if err := r.validate(ctx, pg, proxyClass, logger); err != nil {
return r.notReady(reasonProxyGroupInvalid, fmt.Sprintf("invalid ProxyGroup spec: %v", err))
}
staticEndpoints, nrr, err := r.maybeProvision(ctx, pg, proxyClass)
if err != nil {
if strings.Contains(err.Error(), optimisticLockErrorMsg) {
@ -192,11 +204,7 @@ func (r *ProxyGroupReconciler) reconcilePG(ctx context.Context, pg *tsapi.ProxyG
return staticEndpoints, nrr, nil
}
// validateProxyClassForPG applies custom validation logic for ProxyClass applied to ProxyGroup.
func validateProxyClassForPG(logger *zap.SugaredLogger, pg *tsapi.ProxyGroup, pc *tsapi.ProxyClass) {
if pg.Spec.Type == tsapi.ProxyGroupTypeIngress {
return
}
func (r *ProxyGroupReconciler) validate(ctx context.Context, pg *tsapi.ProxyGroup, pc *tsapi.ProxyClass, logger *zap.SugaredLogger) error {
// Our custom logic for ensuring minimum downtime ProxyGroup update rollouts relies on the local health check
// beig accessible on the replica Pod IP:9002. This address can also be modified by users, via
// TS_LOCAL_ADDR_PORT env var.
@ -208,13 +216,70 @@ func validateProxyClassForPG(logger *zap.SugaredLogger, pg *tsapi.ProxyGroup, pc
// shouldn't need to set their own).
//
// TODO(irbekrm): maybe disallow configuring this env var in future (in Tailscale 1.84 or later).
if hasLocalAddrPortSet(pc) {
if pg.Spec.Type == tsapi.ProxyGroupTypeEgress && hasLocalAddrPortSet(pc) {
msg := fmt.Sprintf("ProxyClass %s applied to an egress ProxyGroup has TS_LOCAL_ADDR_PORT env var set to a custom value."+
"This will disable the ProxyGroup graceful failover mechanism, so you might experience downtime when ProxyGroup pods are restarted."+
"In future we will remove the ability to set custom TS_LOCAL_ADDR_PORT for egress ProxyGroups."+
"Please raise an issue if you expect that this will cause issues for your workflow.", pc.Name)
logger.Warn(msg)
}
// image is the value of pc.Spec.StatefulSet.Pod.TailscaleContainer.Image or ""
// imagePath is a slash-delimited path ending with the image name, e.g.
// "tailscale/tailscale" or maybe "k8s-proxy" if hosted at example.com/k8s-proxy.
var image, imagePath string
if pc != nil &&
pc.Spec.StatefulSet != nil &&
pc.Spec.StatefulSet.Pod != nil &&
pc.Spec.StatefulSet.Pod.TailscaleContainer != nil &&
pc.Spec.StatefulSet.Pod.TailscaleContainer.Image != "" {
image, err := dockerref.ParseNormalizedNamed(pc.Spec.StatefulSet.Pod.TailscaleContainer.Image)
if err != nil {
// Shouldn't be possible as the ProxyClass won't be marked ready
// without successfully parsing the image.
return fmt.Errorf("error parsing %q as a container image reference: %w", pc.Spec.StatefulSet.Pod.TailscaleContainer.Image, err)
}
imagePath = dockerref.Path(image)
}
var errs []error
if isAuthAPIServerProxy(pg) {
// Validate that the static ServiceAccount already exists.
sa := &corev1.ServiceAccount{}
if err := r.Get(ctx, types.NamespacedName{Namespace: r.tsNamespace, Name: authAPIServerProxySAName}, sa); err != nil {
if !apierrors.IsNotFound(err) {
return fmt.Errorf("error validating that ServiceAccount %q exists: %w", authAPIServerProxySAName, err)
}
errs = append(errs, fmt.Errorf("the ServiceAccount %q used for the API server proxy in auth mode does not exist but "+
"should have been created during operator installation; use apiServerProxyConfig.allowImpersonation=true "+
"in the helm chart, or authproxy-rbac.yaml from the static manifests", authAPIServerProxySAName))
}
} else {
// Validate that the ServiceAccount we create won't overwrite the static one.
// TODO(tomhjp): This doesn't cover other controllers that could create a
// ServiceAccount. Perhaps should have some guards to ensure that an update
// would never change the ownership of a resource we expect to already be owned.
if pgServiceAccountName(pg) == authAPIServerProxySAName {
errs = append(errs, fmt.Errorf("the name of the ProxyGroup %q conflicts with the static ServiceAccount used for the API server proxy in auth mode", pg.Name))
}
}
if pg.Spec.Type == tsapi.ProxyGroupTypeKubernetesAPIServer {
if strings.HasSuffix(imagePath, "tailscale") {
errs = append(errs, fmt.Errorf("the configured ProxyClass %q specifies to use image %q but expected a %q image for ProxyGroup of type %q", pc.Name, image, "k8s-proxy", pg.Spec.Type))
}
if pc != nil && pc.Spec.StatefulSet != nil && pc.Spec.StatefulSet.Pod != nil && pc.Spec.StatefulSet.Pod.TailscaleInitContainer != nil {
errs = append(errs, fmt.Errorf("the configured ProxyClass %q specifies Tailscale init container config, but ProxyGroups of type %q do not use init containers", pc.Name, pg.Spec.Type))
}
} else {
if strings.HasSuffix(imagePath, "k8s-proxy") {
errs = append(errs, fmt.Errorf("the configured ProxyClass %q specifies to use image %q but expected a %q image for ProxyGroup of type %q", pc.Name, image, "tailscale", pg.Spec.Type))
}
}
return errors.Join(errs...)
}
func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.ProxyGroup, proxyClass *tsapi.ProxyClass) (map[string][]netip.AddrPort, *notReadyReason, error) {
@ -263,14 +328,21 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro
return r.notReadyErrf(pg, "error provisioning state Secrets: %w", err)
}
}
sa := pgServiceAccount(pg, r.tsNamespace)
if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, sa, func(s *corev1.ServiceAccount) {
s.ObjectMeta.Labels = sa.ObjectMeta.Labels
s.ObjectMeta.Annotations = sa.ObjectMeta.Annotations
s.ObjectMeta.OwnerReferences = sa.ObjectMeta.OwnerReferences
}); err != nil {
return r.notReadyErrf(pg, "error provisioning ServiceAccount: %w", err)
// auth mode kube-apiserver ProxyGroups use a statically created
// ServiceAccount to keep ClusterRole creation permissions limited to the
// helm chart installer.
if !isAuthAPIServerProxy(pg) {
sa := pgServiceAccount(pg, r.tsNamespace)
if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, sa, func(s *corev1.ServiceAccount) {
s.ObjectMeta.Labels = sa.ObjectMeta.Labels
s.ObjectMeta.Annotations = sa.ObjectMeta.Annotations
s.ObjectMeta.OwnerReferences = sa.ObjectMeta.OwnerReferences
}); err != nil {
return r.notReadyErrf(pg, "error provisioning ServiceAccount: %w", err)
}
}
role := pgRole(pg, r.tsNamespace)
if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, role, func(r *rbacv1.Role) {
r.ObjectMeta.Labels = role.ObjectMeta.Labels
@ -280,6 +352,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro
}); err != nil {
return r.notReadyErrf(pg, "error provisioning Role: %w", err)
}
roleBinding := pgRoleBinding(pg, r.tsNamespace)
if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, roleBinding, func(r *rbacv1.RoleBinding) {
r.ObjectMeta.Labels = roleBinding.ObjectMeta.Labels
@ -290,6 +363,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro
}); err != nil {
return r.notReadyErrf(pg, "error provisioning RoleBinding: %w", err)
}
if pg.Spec.Type == tsapi.ProxyGroupTypeEgress {
cm, hp := pgEgressCM(pg, r.tsNamespace)
if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, cm, func(existing *corev1.ConfigMap) {
@ -300,6 +374,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro
return r.notReadyErrf(pg, "error provisioning egress ConfigMap %q: %w", cm.Name, err)
}
}
if pg.Spec.Type == tsapi.ProxyGroupTypeIngress {
cm := pgIngressCM(pg, r.tsNamespace)
if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, cm, func(existing *corev1.ConfigMap) {
@ -309,7 +384,12 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro
return r.notReadyErrf(pg, "error provisioning ingress ConfigMap %q: %w", cm.Name, err)
}
}
ss, err := pgStatefulSet(pg, r.tsNamespace, r.proxyImage, r.tsFirewallMode, tailscaledPort, proxyClass)
defaultImage := r.tsProxyImage
if pg.Spec.Type == tsapi.ProxyGroupTypeKubernetesAPIServer {
defaultImage = r.k8sProxyImage
}
ss, err := pgStatefulSet(pg, r.tsNamespace, defaultImage, r.tsFirewallMode, tailscaledPort, proxyClass)
if err != nil {
return r.notReadyErrf(pg, "error generating StatefulSet spec: %w", err)
}
@ -371,7 +451,7 @@ func (r *ProxyGroupReconciler) maybeUpdateStatus(ctx context.Context, logger *za
if len(devices) > 0 {
status = metav1.ConditionTrue
if len(devices) == desiredReplicas {
reason = reasonProxyGroupReady
reason = reasonProxyGroupAvailable
}
}
tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, status, reason, message, 0, r.clock, logger)
@ -702,17 +782,57 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p
return nil, err
}
configs, err := pgTailscaledConfig(pg, proxyClass, i, authKey, endpoints[nodePortSvcName], existingAdvertiseServices, r.loginServer)
if err != nil {
return nil, fmt.Errorf("error creating tailscaled config: %w", err)
}
if pg.Spec.Type == tsapi.ProxyGroupTypeKubernetesAPIServer {
hostname := pgHostname(pg, i)
for cap, cfg := range configs {
cfgJSON, err := json.Marshal(cfg)
if err != nil {
return nil, fmt.Errorf("error marshalling tailscaled config: %w", err)
if authKey == nil && existingCfgSecret != nil {
deviceAuthed := false
for _, d := range pg.Status.Devices {
if d.Hostname == hostname {
deviceAuthed = true
break
}
}
if !deviceAuthed {
existingCfg := conf.ConfigV1Alpha1{}
if err := json.Unmarshal(existingCfgSecret.Data[kubeAPIServerConfigFile], &existingCfg); err != nil {
return nil, fmt.Errorf("error unmarshalling existing config: %w", err)
}
if existingCfg.AuthKey != nil {
authKey = existingCfg.AuthKey
}
}
}
cfg := conf.VersionedConfig{
Version: "v1alpha1",
ConfigV1Alpha1: &conf.ConfigV1Alpha1{
Hostname: &hostname,
State: ptr.To(fmt.Sprintf("kube:%s", pgPodName(pg.Name, i))),
App: ptr.To(kubetypes.AppProxyGroupKubeAPIServer),
AuthKey: authKey,
KubeAPIServer: &conf.KubeAPIServer{
AuthMode: opt.NewBool(isAuthAPIServerProxy(pg)),
},
},
}
cfgB, err := json.Marshal(cfg)
if err != nil {
return nil, fmt.Errorf("error marshalling k8s-proxy config: %w", err)
}
mak.Set(&cfgSecret.Data, kubeAPIServerConfigFile, cfgB)
} else {
configs, err := pgTailscaledConfig(pg, proxyClass, i, authKey, endpoints[nodePortSvcName], existingAdvertiseServices, r.loginServer)
if err != nil {
return nil, fmt.Errorf("error creating tailscaled config: %w", err)
}
for cap, cfg := range configs {
cfgJSON, err := json.Marshal(cfg)
if err != nil {
return nil, fmt.Errorf("error marshalling tailscaled config: %w", err)
}
mak.Set(&cfgSecret.Data, tsoperator.TailscaledConfigFileName(cap), cfgJSON)
}
mak.Set(&cfgSecret.Data, tsoperator.TailscaledConfigFileName(cap), cfgJSON)
}
if existingCfgSecret != nil {
@ -834,9 +954,12 @@ func (r *ProxyGroupReconciler) ensureAddedToGaugeForProxyGroup(pg *tsapi.ProxyGr
r.egressProxyGroups.Add(pg.UID)
case tsapi.ProxyGroupTypeIngress:
r.ingressProxyGroups.Add(pg.UID)
case tsapi.ProxyGroupTypeKubernetesAPIServer:
r.apiServerProxyGroups.Add(pg.UID)
}
gaugeEgressProxyGroupResources.Set(int64(r.egressProxyGroups.Len()))
gaugeIngressProxyGroupResources.Set(int64(r.ingressProxyGroups.Len()))
gaugeAPIServerProxyGroupResources.Set(int64(r.apiServerProxyGroups.Len()))
}
// ensureRemovedFromGaugeForProxyGroup ensures the gauge metric for the ProxyGroup resource type is updated when the
@ -847,9 +970,12 @@ func (r *ProxyGroupReconciler) ensureRemovedFromGaugeForProxyGroup(pg *tsapi.Pro
r.egressProxyGroups.Remove(pg.UID)
case tsapi.ProxyGroupTypeIngress:
r.ingressProxyGroups.Remove(pg.UID)
case tsapi.ProxyGroupTypeKubernetesAPIServer:
r.apiServerProxyGroups.Remove(pg.UID)
}
gaugeEgressProxyGroupResources.Set(int64(r.egressProxyGroups.Len()))
gaugeIngressProxyGroupResources.Set(int64(r.ingressProxyGroups.Len()))
gaugeAPIServerProxyGroupResources.Set(int64(r.apiServerProxyGroups.Len()))
}
func pgTailscaledConfig(pg *tsapi.ProxyGroup, pc *tsapi.ProxyClass, idx int32, authKey *string, staticEndpoints []netip.AddrPort, oldAdvertiseServices []string, loginServer string) (tailscaledConfigs, error) {
@ -858,7 +984,7 @@ func pgTailscaledConfig(pg *tsapi.ProxyGroup, pc *tsapi.ProxyClass, idx int32, a
AcceptDNS: "false",
AcceptRoutes: "false", // AcceptRoutes defaults to true
Locked: "false",
Hostname: ptr.To(fmt.Sprintf("%s-%d", pg.Name, idx)),
Hostname: ptr.To(pgHostname(pg, idx)),
AdvertiseServices: oldAdvertiseServices,
AuthKey: authKey,
}
@ -867,10 +993,6 @@ func pgTailscaledConfig(pg *tsapi.ProxyGroup, pc *tsapi.ProxyClass, idx int32, a
conf.ServerURL = &loginServer
}
if pg.Spec.HostnamePrefix != "" {
conf.Hostname = ptr.To(fmt.Sprintf("%s-%d", pg.Spec.HostnamePrefix, idx))
}
if shouldAcceptRoutes(pc) {
conf.AcceptRoutes = "true"
}

162
cmd/k8s-operator/proxygroup_specs.go
View File

@ -7,6 +7,7 @@ package main
import (
"fmt"
"path/filepath"
"slices"
"strconv"
"strings"
@ -28,6 +29,9 @@ const (
// deletionGracePeriodSeconds is set to 6 minutes to ensure that the pre-stop hook of these proxies have enough chance to terminate gracefully.
deletionGracePeriodSeconds int64 = 360
staticEndpointPortName = "static-endpoint-port"
// authAPIServerProxySAName is the ServiceAccount deployed by the helm chart
// if apiServerProxy.authEnabled is true.
authAPIServerProxySAName = "kube-apiserver-auth-proxy"
)
func pgNodePortServiceName(proxyGroupName string, replica int32) string {
@ -61,6 +65,9 @@ func pgNodePortService(pg *tsapi.ProxyGroup, name string, namespace string) *cor
// Returns the base StatefulSet definition for a ProxyGroup. A ProxyClass may be
// applied over the top after.
func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string, port *uint16, proxyClass *tsapi.ProxyClass) (*appsv1.StatefulSet, error) {
if pg.Spec.Type == tsapi.ProxyGroupTypeKubernetesAPIServer {
return kubeAPIServerStatefulSet(pg, namespace, image)
}
ss := new(appsv1.StatefulSet)
if err := yaml.Unmarshal(proxyYaml, &ss); err != nil {
return nil, fmt.Errorf("failed to unmarshal proxy spec: %w", err)
@ -167,6 +174,7 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string
Value: "$(POD_NAME)",
},
{
// TODO(tomhjp): This is tsrecorder-specific and does nothing. Delete.
Name: "TS_STATE",
Value: "kube:$(POD_NAME)",
},
@ -264,9 +272,124 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string
// gracefully.
ss.Spec.Template.DeletionGracePeriodSeconds = ptr.To(deletionGracePeriodSeconds)
}
return ss, nil
}
func kubeAPIServerStatefulSet(pg *tsapi.ProxyGroup, namespace, image string) (*appsv1.StatefulSet, error) {
sts := &appsv1.StatefulSet{
ObjectMeta: metav1.ObjectMeta{
Name: pg.Name,
Namespace: namespace,
Labels: pgLabels(pg.Name, nil),
OwnerReferences: pgOwnerReference(pg),
},
Spec: appsv1.StatefulSetSpec{
Replicas: ptr.To(pgReplicas(pg)),
Selector: &metav1.LabelSelector{
MatchLabels: pgLabels(pg.Name, nil),
},
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Name: pg.Name,
Namespace: namespace,
Labels: pgLabels(pg.Name, nil),
DeletionGracePeriodSeconds: ptr.To[int64](10),
},
Spec: corev1.PodSpec{
ServiceAccountName: pgServiceAccountName(pg),
Containers: []corev1.Container{
{
Name: mainContainerName,
Image: image,
Env: []corev1.EnvVar{
{
// Used as default hostname and in Secret names.
Name: "POD_NAME",
ValueFrom: &corev1.EnvVarSource{
FieldRef: &corev1.ObjectFieldSelector{
FieldPath: "metadata.name",
},
},
},
{
// Used by kubeclient to post Events about the Pod's lifecycle.
Name: "POD_UID",
ValueFrom: &corev1.EnvVarSource{
FieldRef: &corev1.ObjectFieldSelector{
FieldPath: "metadata.uid",
},
},
},
{
// Used in an interpolated env var if metrics enabled.
Name: "POD_IP",
ValueFrom: &corev1.EnvVarSource{
FieldRef: &corev1.ObjectFieldSelector{
FieldPath: "status.podIP",
},
},
},
{
// Included for completeness with POD_IP and easier backwards compatibility in future.
Name: "POD_IPS",
ValueFrom: &corev1.EnvVarSource{
FieldRef: &corev1.ObjectFieldSelector{
FieldPath: "status.podIPs",
},
},
},
{
Name: "TS_K8S_PROXY_CONFIG",
Value: filepath.Join("/etc/tsconfig/$(POD_NAME)/", kubeAPIServerConfigFile),
},
},
VolumeMounts: func() []corev1.VolumeMount {
var mounts []corev1.VolumeMount
// TODO(tomhjp): Read config directly from the Secret instead.
for i := range pgReplicas(pg) {
mounts = append(mounts, corev1.VolumeMount{
Name: fmt.Sprintf("k8s-proxy-config-%d", i),
ReadOnly: true,
MountPath: fmt.Sprintf("/etc/tsconfig/%s-%d", pg.Name, i),
})
}
return mounts
}(),
Ports: []corev1.ContainerPort{
{
Name: "k8s-proxy",
ContainerPort: 443,
Protocol: corev1.ProtocolTCP,
},
},
},
},
Volumes: func() []corev1.Volume {
var volumes []corev1.Volume
for i := range pgReplicas(pg) {
volumes = append(volumes, corev1.Volume{
Name: fmt.Sprintf("k8s-proxy-config-%d", i),
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: pgConfigSecretName(pg.Name, i),
},
},
})
}
return volumes
}(),
},
},
},
}
return sts, nil
}
func pgServiceAccount(pg *tsapi.ProxyGroup, namespace string) *corev1.ServiceAccount {
return &corev1.ServiceAccount{
ObjectMeta: metav1.ObjectMeta{
@ -305,8 +428,8 @@ func pgRole(pg *tsapi.ProxyGroup, namespace string) *rbacv1.Role {
ResourceNames: func() (secrets []string) {
for i := range pgReplicas(pg) {
secrets = append(secrets,
pgConfigSecretName(pg.Name, i), // Config with auth key.
fmt.Sprintf("%s-%d", pg.Name, i), // State.
pgConfigSecretName(pg.Name, i), // Config with auth key.
pgPodName(pg.Name, i), // State.
)
}
return secrets
@ -336,7 +459,7 @@ func pgRoleBinding(pg *tsapi.ProxyGroup, namespace string) *rbacv1.RoleBinding {
Subjects: []rbacv1.Subject{
{
Kind: "ServiceAccount",
Name: pg.Name,
Name: pgServiceAccountName(pg),
Namespace: namespace,
},
},
@ -347,6 +470,27 @@ func pgRoleBinding(pg *tsapi.ProxyGroup, namespace string) *rbacv1.RoleBinding {
}
}
// kube-apiserver proxies in auth mode use a static ServiceAccount. Everything
// else uses a per-ProxyGroup ServiceAccount.
func pgServiceAccountName(pg *tsapi.ProxyGroup) string {
if isAuthAPIServerProxy(pg) {
return authAPIServerProxySAName
}
return pg.Name
}
func isAuthAPIServerProxy(pg *tsapi.ProxyGroup) bool {
if pg.Spec.Type != tsapi.ProxyGroupTypeKubernetesAPIServer {
return false
}
// The default is auth mode.
return pg.Spec.KubeAPIServer == nil ||
pg.Spec.KubeAPIServer.Mode == nil ||
*pg.Spec.KubeAPIServer.Mode == tsapi.APIServerProxyModeAuth
}
func pgStateSecrets(pg *tsapi.ProxyGroup, namespace string) (secrets []*corev1.Secret) {
for i := range pgReplicas(pg) {
secrets = append(secrets, &corev1.Secret{
@ -418,6 +562,18 @@ func pgReplicas(pg *tsapi.ProxyGroup) int32 {
return 2
}
func pgPodName(pgName string, i int32) string {
return fmt.Sprintf("%s-%d", pgName, i)
}
func pgHostname(pg *tsapi.ProxyGroup, i int32) string {
if pg.Spec.HostnamePrefix != "" {
return fmt.Sprintf("%s-%d", pg.Spec.HostnamePrefix, i)
}
return fmt.Sprintf("%s-%d", pg.Name, i)
}
func pgConfigSecretName(pgName string, i int32) string {
return fmt.Sprintf("%s-%d-config", pgName, i)
}

249
cmd/k8s-operator/proxygroup_test.go
View File

@ -629,7 +629,7 @@ func TestProxyGroupWithStaticEndpoints(t *testing.T) {
reconciler := &ProxyGroupReconciler{
tsNamespace: tsNamespace,
proxyImage: testProxyImage,
tsProxyImage: testProxyImage,
defaultTags: []string{"tag:test-tag"},
tsFirewallMode: "auto",
defaultProxyClass: "default-pc",
@ -772,7 +772,7 @@ func TestProxyGroupWithStaticEndpoints(t *testing.T) {
t.Run("delete_and_cleanup", func(t *testing.T) {
reconciler := &ProxyGroupReconciler{
tsNamespace: tsNamespace,
proxyImage: testProxyImage,
tsProxyImage: testProxyImage,
defaultTags: []string{"tag:test-tag"},
tsFirewallMode: "auto",
defaultProxyClass: "default-pc",
@ -832,7 +832,7 @@ func TestProxyGroup(t *testing.T) {
cl := tstest.NewClock(tstest.ClockOpts{})
reconciler := &ProxyGroupReconciler{
tsNamespace: tsNamespace,
proxyImage: testProxyImage,
tsProxyImage: testProxyImage,
defaultTags: []string{"tag:test-tag"},
tsFirewallMode: "auto",
defaultProxyClass: "default-pc",
@ -915,7 +915,7 @@ func TestProxyGroup(t *testing.T) {
},
}
tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, reasonProxyGroupReady, reasonProxyGroupReady, 0, cl, zl.Sugar())
tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, reasonProxyGroupReady, "2/2 ProxyGroup pods running", 0, cl, zl.Sugar())
tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, reasonProxyGroupAvailable, "2/2 ProxyGroup pods running", 0, cl, zl.Sugar())
expectEqual(t, fc, pg)
expectProxyGroupResources(t, fc, pg, true, pc)
})
@ -934,7 +934,7 @@ func TestProxyGroup(t *testing.T) {
addNodeIDToStateSecrets(t, fc, pg)
expectReconciled(t, reconciler, "", pg.Name)
tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, reasonProxyGroupReady, reasonProxyGroupReady, 0, cl, zl.Sugar())
tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, reasonProxyGroupReady, "3/3 ProxyGroup pods running", 0, cl, zl.Sugar())
tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, reasonProxyGroupAvailable, "3/3 ProxyGroup pods running", 0, cl, zl.Sugar())
pg.Status.Devices = append(pg.Status.Devices, tsapi.TailnetDevice{
Hostname: "hostname-nodeid-2",
TailnetIPs: []string{"1.2.3.4", "::1"},
@ -952,7 +952,7 @@ func TestProxyGroup(t *testing.T) {
expectReconciled(t, reconciler, "", pg.Name)
pg.Status.Devices = pg.Status.Devices[:1] // truncate to only the first device.
tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, reasonProxyGroupReady, "1/1 ProxyGroup pods running", 0, cl, zl.Sugar())
tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, reasonProxyGroupAvailable, "1/1 ProxyGroup pods running", 0, cl, zl.Sugar())
expectEqual(t, fc, pg)
expectProxyGroupResources(t, fc, pg, true, pc)
})
@ -1025,12 +1025,12 @@ func TestProxyGroupTypes(t *testing.T) {
zl, _ := zap.NewDevelopment()
reconciler := &ProxyGroupReconciler{
tsNamespace: tsNamespace,
proxyImage: testProxyImage,
Client: fc,
l: zl.Sugar(),
tsClient: &fakeTSClient{},
clock: tstest.NewClock(tstest.ClockOpts{}),
tsNamespace: tsNamespace,
tsProxyImage: testProxyImage,
Client: fc,
l: zl.Sugar(),
tsClient: &fakeTSClient{},
clock: tstest.NewClock(tstest.ClockOpts{}),
}
t.Run("egress_type", func(t *testing.T) {
@ -1047,7 +1047,7 @@ func TestProxyGroupTypes(t *testing.T) {
mustCreate(t, fc, pg)
expectReconciled(t, reconciler, "", pg.Name)
verifyProxyGroupCounts(t, reconciler, 0, 1)
verifyProxyGroupCounts(t, reconciler, 0, 1, 0)
sts := &appsv1.StatefulSet{}
if err := fc.Get(t.Context(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil {
@ -1161,7 +1161,7 @@ func TestProxyGroupTypes(t *testing.T) {
}
expectReconciled(t, reconciler, "", pg.Name)
verifyProxyGroupCounts(t, reconciler, 1, 2)
verifyProxyGroupCounts(t, reconciler, 1, 2, 0)
sts := &appsv1.StatefulSet{}
if err := fc.Get(t.Context(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil {
@ -1198,6 +1198,44 @@ func TestProxyGroupTypes(t *testing.T) {
t.Errorf("unexpected volume mounts (-want +got):\n%s", diff)
}
})
t.Run("kubernetes_api_server_type", func(t *testing.T) {
pg := &tsapi.ProxyGroup{
ObjectMeta: metav1.ObjectMeta{
Name: "test-k8s-apiserver",
UID: "test-k8s-apiserver-uid",
},
Spec: tsapi.ProxyGroupSpec{
Type: tsapi.ProxyGroupTypeKubernetesAPIServer,
Replicas: ptr.To[int32](2),
KubeAPIServer: &tsapi.KubeAPIServerConfig{
Mode: ptr.To(tsapi.APIServerProxyModeNoAuth),
},
},
}
if err := fc.Create(t.Context(), pg); err != nil {
t.Fatal(err)
}
expectReconciled(t, reconciler, "", pg.Name)
verifyProxyGroupCounts(t, reconciler, 1, 2, 1)
sts := &appsv1.StatefulSet{}
if err := fc.Get(t.Context(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil {
t.Fatalf("failed to get StatefulSet: %v", err)
}
// Verify the StatefulSet configuration for KubernetesAPIServer type.
if sts.Spec.Template.Spec.Containers[0].Name != mainContainerName {
t.Errorf("unexpected container name %s, want %s", sts.Spec.Template.Spec.Containers[0].Name, mainContainerName)
}
if sts.Spec.Template.Spec.Containers[0].Ports[0].ContainerPort != 443 {
t.Errorf("unexpected container port %d, want 443", sts.Spec.Template.Spec.Containers[0].Ports[0].ContainerPort)
}
if sts.Spec.Template.Spec.Containers[0].Ports[0].Name != "k8s-proxy" {
t.Errorf("unexpected port name %s, want k8s-proxy", sts.Spec.Template.Spec.Containers[0].Ports[0].Name)
}
})
}
func TestIngressAdvertiseServicesConfigPreserved(t *testing.T) {
@ -1206,12 +1244,12 @@ func TestIngressAdvertiseServicesConfigPreserved(t *testing.T) {
WithStatusSubresource(&tsapi.ProxyGroup{}).
Build()
reconciler := &ProxyGroupReconciler{
tsNamespace: tsNamespace,
proxyImage: testProxyImage,
Client: fc,
l: zap.Must(zap.NewDevelopment()).Sugar(),
tsClient: &fakeTSClient{},
clock: tstest.NewClock(tstest.ClockOpts{}),
tsNamespace: tsNamespace,
tsProxyImage: testProxyImage,
Client: fc,
l: zap.Must(zap.NewDevelopment()).Sugar(),
tsClient: &fakeTSClient{},
clock: tstest.NewClock(tstest.ClockOpts{}),
}
existingServices := []string{"svc1", "svc2"}
@ -1272,6 +1310,170 @@ func TestIngressAdvertiseServicesConfigPreserved(t *testing.T) {
})
}
func TestValidateProxyGroup(t *testing.T) {
type testCase struct {
typ tsapi.ProxyGroupType
pgName string
image string
noauth bool
initContainer bool
staticSAExists bool
expectedErrs int
}
for name, tc := range map[string]testCase{
"default_ingress": {
typ: tsapi.ProxyGroupTypeIngress,
},
"default_kube": {
typ: tsapi.ProxyGroupTypeKubernetesAPIServer,
staticSAExists: true,
},
"default_kube_noauth": {
typ: tsapi.ProxyGroupTypeKubernetesAPIServer,
noauth: true,
// Does not require the static ServiceAccount to exist.
},
"kube_static_sa_missing": {
typ: tsapi.ProxyGroupTypeKubernetesAPIServer,
staticSAExists: false,
expectedErrs: 1,
},
"kube_noauth_would_overwrite_static_sa": {
typ: tsapi.ProxyGroupTypeKubernetesAPIServer,
staticSAExists: true,
noauth: true,
pgName: authAPIServerProxySAName,
expectedErrs: 1,
},
"ingress_would_overwrite_static_sa": {
typ: tsapi.ProxyGroupTypeIngress,
staticSAExists: true,
pgName: authAPIServerProxySAName,
expectedErrs: 1,
},
"tailscale_image_for_kube_pg_1": {
typ: tsapi.ProxyGroupTypeKubernetesAPIServer,
staticSAExists: true,
image: "example.com/tailscale/tailscale",
expectedErrs: 1,
},
"tailscale_image_for_kube_pg_2": {
typ: tsapi.ProxyGroupTypeKubernetesAPIServer,
staticSAExists: true,
image: "example.com/tailscale",
expectedErrs: 1,
},
"tailscale_image_for_kube_pg_3": {
typ: tsapi.ProxyGroupTypeKubernetesAPIServer,
staticSAExists: true,
image: "example.com/tailscale/tailscale:latest",
expectedErrs: 1,
},
"tailscale_image_for_kube_pg_4": {
typ: tsapi.ProxyGroupTypeKubernetesAPIServer,
staticSAExists: true,
image: "tailscale/tailscale",
expectedErrs: 1,
},
"k8s_proxy_image_for_ingress_pg": {
typ: tsapi.ProxyGroupTypeIngress,
image: "example.com/k8s-proxy",
expectedErrs: 1,
},
"init_container_for_kube_pg": {
typ: tsapi.ProxyGroupTypeKubernetesAPIServer,
staticSAExists: true,
initContainer: true,
expectedErrs: 1,
},
"init_container_for_ingress_pg": {
typ: tsapi.ProxyGroupTypeIngress,
initContainer: true,
},
"init_container_for_egress_pg": {
typ: tsapi.ProxyGroupTypeEgress,
initContainer: true,
},
} {
t.Run(name, func(t *testing.T) {
pc := &tsapi.ProxyClass{
ObjectMeta: metav1.ObjectMeta{
Name: "some-pc",
},
Spec: tsapi.ProxyClassSpec{
StatefulSet: &tsapi.StatefulSet{
Pod: &tsapi.Pod{},
},
},
}
if tc.image != "" {
pc.Spec.StatefulSet.Pod.TailscaleContainer = &tsapi.Container{
Image: tc.image,
}
}
if tc.initContainer {
pc.Spec.StatefulSet.Pod.TailscaleInitContainer = &tsapi.Container{}
}
pgName := "some-pg"
if tc.pgName != "" {
pgName = tc.pgName
}
pg := &tsapi.ProxyGroup{
ObjectMeta: metav1.ObjectMeta{
Name: pgName,
},
Spec: tsapi.ProxyGroupSpec{
Type: tc.typ,
},
}
if tc.noauth {
pg.Spec.KubeAPIServer = &tsapi.KubeAPIServerConfig{
Mode: ptr.To(tsapi.APIServerProxyModeNoAuth),
}
}
var objs []client.Object
if tc.staticSAExists {
objs = append(objs, &corev1.ServiceAccount{
ObjectMeta: metav1.ObjectMeta{
Name: authAPIServerProxySAName,
Namespace: tsNamespace,
},
})
}
r := ProxyGroupReconciler{
tsNamespace: tsNamespace,
Client: fake.NewClientBuilder().
WithObjects(objs...).
Build(),
}
logger, _ := zap.NewDevelopment()
err := r.validate(t.Context(), pg, pc, logger.Sugar())
if tc.expectedErrs == 0 {
if err != nil {
t.Fatalf("expected no errors, got: %v", err)
}
// Test finished.
return
}
if err == nil {
t.Fatalf("expected %d errors, got none", tc.expectedErrs)
}
type unwrapper interface {
Unwrap() []error
}
errs := err.(unwrapper)
if len(errs.Unwrap()) != tc.expectedErrs {
t.Fatalf("expected %d errors, got %d: %v", tc.expectedErrs, len(errs.Unwrap()), err)
}
})
}
}
func proxyClassesForLEStagingTest() (*tsapi.ProxyClass, *tsapi.ProxyClass, *tsapi.ProxyClass) {
pcLEStaging := &tsapi.ProxyClass{
ObjectMeta: metav1.ObjectMeta{
@ -1326,7 +1528,7 @@ func setProxyClassReady(t *testing.T, fc client.Client, cl *tstest.Clock, name s
return pc
}
func verifyProxyGroupCounts(t *testing.T, r *ProxyGroupReconciler, wantIngress, wantEgress int) {
func verifyProxyGroupCounts(t *testing.T, r *ProxyGroupReconciler, wantIngress, wantEgress, wantAPIServer int) {
t.Helper()
if r.ingressProxyGroups.Len() != wantIngress {
t.Errorf("expected %d ingress proxy groups, got %d", wantIngress, r.ingressProxyGroups.Len())
@ -1334,6 +1536,9 @@ func verifyProxyGroupCounts(t *testing.T, r *ProxyGroupReconciler, wantIngress,
if r.egressProxyGroups.Len() != wantEgress {
t.Errorf("expected %d egress proxy groups, got %d", wantEgress, r.egressProxyGroups.Len())
}
if r.apiServerProxyGroups.Len() != wantAPIServer {
t.Errorf("expected %d kube-apiserver proxy groups, got %d", wantAPIServer, r.apiServerProxyGroups.Len())
}
}
func verifyEnvVar(t *testing.T, sts *appsv1.StatefulSet, name, expectedValue string) {
@ -1512,7 +1717,7 @@ func TestProxyGroupLetsEncryptStaging(t *testing.T) {
reconciler := &ProxyGroupReconciler{
tsNamespace: tsNamespace,
proxyImage: testProxyImage,
tsProxyImage: testProxyImage,
defaultTags: []string{"tag:test"},
defaultProxyClass: tt.defaultProxyClass,
Client: fc,

12
cmd/k8s-operator/sts.go
View File

@ -102,6 +102,8 @@ const (
defaultLocalAddrPort = 9002 // metrics and health check port
letsEncryptStagingEndpoint = "https://acme-staging-v02.api.letsencrypt.org/directory"
mainContainerName = "tailscale"
)
var (
@ -761,7 +763,7 @@ func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet,
}
if pc.Spec.UseLetsEncryptStagingEnvironment && (stsCfg.proxyType == proxyTypeIngressResource || stsCfg.proxyType == string(tsapi.ProxyGroupTypeIngress)) {
for i, c := range ss.Spec.Template.Spec.Containers {
if c.Name == "tailscale" {
if isMainContainer(&c) {
ss.Spec.Template.Spec.Containers[i].Env = append(ss.Spec.Template.Spec.Containers[i].Env, corev1.EnvVar{
Name: "TS_DEBUG_ACME_DIRECTORY_URL",
Value: letsEncryptStagingEndpoint,
@ -829,7 +831,7 @@ func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet,
return base
}
for i, c := range ss.Spec.Template.Spec.Containers {
if c.Name == "tailscale" {
if isMainContainer(&c) {
ss.Spec.Template.Spec.Containers[i] = updateContainer(wantsPod.TailscaleContainer, ss.Spec.Template.Spec.Containers[i])
break
}
@ -847,7 +849,7 @@ func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet,
func enableEndpoints(ss *appsv1.StatefulSet, metrics, debug bool) {
for i, c := range ss.Spec.Template.Spec.Containers {
if c.Name == "tailscale" {
if isMainContainer(&c) {
if debug {
ss.Spec.Template.Spec.Containers[i].Env = append(ss.Spec.Template.Spec.Containers[i].Env,
// Serve tailscaled's debug metrics on on
@ -902,6 +904,10 @@ func enableEndpoints(ss *appsv1.StatefulSet, metrics, debug bool) {
}
}
func isMainContainer(c *corev1.Container) bool {
return c.Name == mainContainerName
}
// tailscaledConfig takes a proxy config, a newly generated auth key if generated and a Secret with the previous proxy
// state and auth key and returns tailscaled config files for currently supported proxy versions.
func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *corev1.Secret) (tailscaledConfigs, error) {

60
cmd/k8s-operator/svc-for-pg.go
View File

@ -60,7 +60,6 @@ type HAServiceReconciler struct {
recorder record.EventRecorder
logger *zap.SugaredLogger
tsClient tsClient
tsnetServer tsnetServer
tsNamespace string
lc localClient
defaultTags []string
@ -221,7 +220,7 @@ func (r *HAServiceReconciler) maybeProvision(ctx context.Context, hostname strin
// This checks and ensures that Tailscale Service's owner references are updated
// for this Service and errors if that is not possible (i.e. because it
// appears that the Tailscale Service has been created by a non-operator actor).
updatedAnnotations, err := r.ownerAnnotations(existingTSSvc)
updatedAnnotations, err := ownerAnnotations(r.operatorID, existingTSSvc)
if err != nil {
instr := fmt.Sprintf("To proceed, you can either manually delete the existing Tailscale Service or choose a different hostname with the '%s' annotaion", AnnotationHostname)
msg := fmt.Sprintf("error ensuring ownership of Tailscale Service %s: %v. %s", hostname, err, instr)
@ -395,7 +394,7 @@ func (r *HAServiceReconciler) maybeCleanup(ctx context.Context, hostname string,
serviceName := tailcfg.ServiceName("svc:" + hostname)
// 1. Clean up the Tailscale Service.
svcChanged, err = r.cleanupTailscaleService(ctx, serviceName, logger)
svcChanged, err = cleanupTailscaleService(ctx, r.tsClient, serviceName, r.operatorID, logger)
if err != nil {
return false, fmt.Errorf("error deleting Tailscale Service: %w", err)
}
@ -456,7 +455,7 @@ func (r *HAServiceReconciler) maybeCleanupProxyGroup(ctx context.Context, proxyG
return false, fmt.Errorf("failed to update tailscaled config services: %w", err)
}
svcsChanged, err = r.cleanupTailscaleService(ctx, tailcfg.ServiceName(tsSvcName), logger)
svcsChanged, err = cleanupTailscaleService(ctx, r.tsClient, tailcfg.ServiceName(tsSvcName), r.operatorID, logger)
if err != nil {
return false, fmt.Errorf("deleting Tailscale Service %q: %w", tsSvcName, err)
}
@ -529,8 +528,8 @@ func (r *HAServiceReconciler) tailnetCertDomain(ctx context.Context) (string, er
// If a Tailscale Service is found, but contains other owner references, only removes this operator's owner reference.
// If a Tailscale Service by the given name is not found or does not contain this operator's owner reference, do nothing.
// It returns true if an existing Tailscale Service was updated to remove owner reference, as well as any error that occurred.
func (r *HAServiceReconciler) cleanupTailscaleService(ctx context.Context, name tailcfg.ServiceName, logger *zap.SugaredLogger) (updated bool, err error) {
svc, err := r.tsClient.GetVIPService(ctx, name)
func cleanupTailscaleService(ctx context.Context, tsClient tsClient, name tailcfg.ServiceName, operatorID string, logger *zap.SugaredLogger) (updated bool, err error) {
svc, err := tsClient.GetVIPService(ctx, name)
if isErrorFeatureFlagNotEnabled(err) {
msg := fmt.Sprintf("Unable to proceed with cleanup: %s.", msgFeatureFlagNotEnabled)
logger.Warn(msg)
@ -563,14 +562,14 @@ func (r *HAServiceReconciler) cleanupTailscaleService(ctx context.Context, name
// cluster before deleting the Ingress. Perhaps the comparison could be
// 'if or.OperatorID == r.operatorID || or.ingressUID == r.ingressUID'.
ix := slices.IndexFunc(o.OwnerRefs, func(or OwnerRef) bool {
return or.OperatorID == r.operatorID
return or.OperatorID == operatorID
})
if ix == -1 {
return false, nil
}
if len(o.OwnerRefs) == 1 {
logger.Infof("Deleting Tailscale Service %q", name)
return false, r.tsClient.DeleteVIPService(ctx, name)
return false, tsClient.DeleteVIPService(ctx, name)
}
o.OwnerRefs = slices.Delete(o.OwnerRefs, ix, ix+1)
logger.Infof("Updating Tailscale Service %q", name)
@ -579,7 +578,7 @@ func (r *HAServiceReconciler) cleanupTailscaleService(ctx context.Context, name
return false, fmt.Errorf("error marshalling updated Tailscale Service owner reference: %w", err)
}
svc.Annotations[ownerAnnotation] = string(json)
return true, r.tsClient.CreateOrUpdateVIPService(ctx, svc)
return true, tsClient.CreateOrUpdateVIPService(ctx, svc)
}
func (a *HAServiceReconciler) backendRoutesSetup(ctx context.Context, serviceName, replicaName, pgName string, wantsCfg *ingressservices.Config, logger *zap.SugaredLogger) (bool, error) {
@ -742,49 +741,6 @@ func (a *HAServiceReconciler) numberPodsAdvertising(ctx context.Context, pgName
return count, nil
}
// ownerAnnotations returns the updated annotations required to ensure this
// instance of the operator is included as an owner. If the Tailscale Service is not
// nil, but does not contain an owner we return an error as this likely means
// that the Tailscale Service was created by something other than a Tailscale
// Kubernetes operator.
func (r *HAServiceReconciler) ownerAnnotations(svc *tailscale.VIPService) (map[string]string, error) {
ref := OwnerRef{
OperatorID: r.operatorID,
}
if svc == nil {
c := ownerAnnotationValue{OwnerRefs: []OwnerRef{ref}}
json, err := json.Marshal(c)
if err != nil {
return nil, fmt.Errorf("[unexpected] unable to marshal Tailscale Service owner annotation contents: %w, please report this", err)
}
return map[string]string{
ownerAnnotation: string(json),
}, nil
}
o, err := parseOwnerAnnotation(svc)
if err != nil {
return nil, err
}
if o == nil || len(o.OwnerRefs) == 0 {
return nil, fmt.Errorf("Tailscale Service %s exists, but does not contain owner annotation with owner references; not proceeding as this is likely a resource created by something other than the Tailscale Kubernetes operator", svc.Name)
}
if slices.Contains(o.OwnerRefs, ref) { // up to date
return svc.Annotations, nil
}
o.OwnerRefs = append(o.OwnerRefs, ref)
json, err := json.Marshal(o)
if err != nil {
return nil, fmt.Errorf("error marshalling updated owner references: %w", err)
}
newAnnots := make(map[string]string, len(svc.Annotations)+1)
for k, v := range svc.Annotations {
newAnnots[k] = v
}
newAnnots[ownerAnnotation] = string(json)
return newAnnots, nil
}
// dnsNameForService returns the DNS name for the given Tailscale Service name.
func (r *HAServiceReconciler) dnsNameForService(ctx context.Context, svc tailcfg.ServiceName) (string, error) {
s := svc.WithoutPrefix()

2
cmd/k8s-operator/svc-for-pg_test.go
View File

@ -187,7 +187,6 @@ func setupServiceTest(t *testing.T) (*HAServiceReconciler, *corev1.Secret, clien
if err := fc.Status().Update(context.Background(), pg); err != nil {
t.Fatal(err)
}
fakeTsnetServer := &fakeTSNetServer{certDomains: []string{"foo.com"}}
ft := &fakeTSClient{}
zl, err := zap.NewDevelopment()
@ -210,7 +209,6 @@ func setupServiceTest(t *testing.T) (*HAServiceReconciler, *corev1.Secret, clien
clock: cl,
defaultTags: []string{"tag:k8s"},
tsNamespace: "operator-ns",
tsnetServer: fakeTsnetServer,
logger: zl.Sugar(),
recorder: record.NewFakeRecorder(10),
lc: lc,

197
cmd/k8s-proxy/k8s-proxy.go Normal file
View File

@ -0,0 +1,197 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
//go:build !plan9
// k8s-proxy proxies between tailnet and Kubernetes cluster traffic.
// Currently, it only supports proxying tailnet clients to the Kubernetes API
// server.
package main
import (
"context"
"errors"
"fmt"
"os"
"os/signal"
"syscall"
"time"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
"golang.org/x/sync/errgroup"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
"tailscale.com/hostinfo"
"tailscale.com/ipn"
"tailscale.com/ipn/store"
apiproxy "tailscale.com/k8s-operator/api-proxy"
"tailscale.com/kube/k8s-proxy/conf"
"tailscale.com/kube/state"
"tailscale.com/tsnet"
)
func main() {
logger := zap.Must(zap.NewProduction()).Sugar()
defer logger.Sync()
if err := run(logger); err != nil {
logger.Fatal(err.Error())
}
}
func run(logger *zap.SugaredLogger) error {
var (
configFile = os.Getenv("TS_K8S_PROXY_CONFIG")
podUID = os.Getenv("POD_UID")
)
if configFile == "" {
return errors.New("TS_K8S_PROXY_CONFIG unset")
}
// TODO(tomhjp): Support reloading config.
// TODO(tomhjp): Support reading config from a Secret.
cfg, err := conf.Load(configFile)
if err != nil {
return fmt.Errorf("error loading config file %q: %w", configFile, err)
}
if cfg.Parsed.LogLevel != nil {
level, err := zapcore.ParseLevel(*cfg.Parsed.LogLevel)
if err != nil {
return fmt.Errorf("error parsing log level %q: %w", *cfg.Parsed.LogLevel, err)
}
logger = logger.WithOptions(zap.IncreaseLevel(level))
}
if cfg.Parsed.App != nil {
hostinfo.SetApp(*cfg.Parsed.App)
}
st, err := getStateStore(cfg.Parsed.State, logger)
if err != nil {
return err
}
// If Pod UID unset, assume we're running outside of a cluster/not managed
// by the operator, so no need to set additional state keys.
if podUID != "" {
if err := state.SetInitialKeys(st, podUID); err != nil {
return fmt.Errorf("error setting initial state: %w", err)
}
}
var authKey string
if cfg.Parsed.AuthKey != nil {
authKey = *cfg.Parsed.AuthKey
}
ts := &tsnet.Server{
Logf: logger.Named("tsnet").Debugf,
UserLogf: logger.Named("tsnet").Infof,
Store: st,
AuthKey: authKey,
}
if cfg.Parsed.Hostname != nil {
ts.Hostname = *cfg.Parsed.Hostname
}
// ctx to live for the lifetime of the process.
ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer cancel()
// Make sure we crash loop if Up doesn't complete in reasonable time.
upCtx, upCancel := context.WithTimeout(ctx, time.Minute)
defer upCancel()
if _, err := ts.Up(upCtx); err != nil {
return fmt.Errorf("error starting tailscale server: %w", err)
}
defer ts.Close()
group, groupCtx := errgroup.WithContext(ctx)
// Setup for updating state keys.
if podUID != "" {
lc, err := ts.LocalClient()
if err != nil {
return fmt.Errorf("error getting local client: %w", err)
}
w, err := lc.WatchIPNBus(groupCtx, ipn.NotifyInitialNetMap)
if err != nil {
return fmt.Errorf("error watching IPN bus: %w", err)
}
defer w.Close()
group.Go(func() error {
if err := state.KeepKeysUpdated(st, w.Next); err != nil && err != groupCtx.Err() {
return fmt.Errorf("error keeping state keys updated: %w", err)
}
return nil
})
}
// Setup for the API server proxy.
restConfig, err := getRestConfig(logger)
if err != nil {
return fmt.Errorf("error getting rest config: %w", err)
}
authMode := true
if cfg.Parsed.KubeAPIServer != nil {
v, ok := cfg.Parsed.KubeAPIServer.AuthMode.Get()
if ok {
authMode = v
}
}
ap, err := apiproxy.NewAPIServerProxy(logger.Named("apiserver-proxy"), restConfig, ts, authMode)
if err != nil {
return fmt.Errorf("error creating api server proxy: %w", err)
}
// TODO(tomhjp): Work out whether we should use TS_CERT_SHARE_MODE or not,
// and possibly issue certs upfront here before serving.
group.Go(func() error {
if err := ap.Run(groupCtx); err != nil {
return fmt.Errorf("error running API server proxy: %w", err)
}
return nil
})
return group.Wait()
}
func getStateStore(path *string, logger *zap.SugaredLogger) (ipn.StateStore, error) {
p := "mem:"
if path != nil {
p = *path
} else {
logger.Warn("No state Secret provided; using in-memory store, which will lose state on restart")
}
st, err := store.New(logger.Errorf, p)
if err != nil {
return nil, fmt.Errorf("error creating state store: %w", err)
}
return st, nil
}
func getRestConfig(logger *zap.SugaredLogger) (*rest.Config, error) {
restConfig, err := rest.InClusterConfig()
switch err {
case nil:
return restConfig, nil
case rest.ErrNotInCluster:
logger.Info("Not running in-cluster, falling back to kubeconfig")
default:
return nil, fmt.Errorf("error getting in-cluster config: %w", err)
}
loadingRules := clientcmd.NewDefaultClientConfigLoadingRules()
clientConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, nil)
restConfig, err = clientConfig.ClientConfig()
if err != nil {
return nil, fmt.Errorf("error loading kubeconfig: %w", err)
}
return restConfig, nil
}

29
k8s-operator/api-proxy/env.go
View File

@ -1,29 +0,0 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
//go:build !plan9
package apiproxy
import (
"os"
"tailscale.com/types/opt"
)
func defaultBool(envName string, defVal bool) bool {
vs := os.Getenv(envName)
if vs == "" {
return defVal
}
v, _ := opt.Bool(vs).Get()
return v
}
func defaultEnv(envName, defVal string) string {
v := os.Getenv(envName)
if v == "" {
return defVal
}
return v
}

187
k8s-operator/api-proxy/proxy.go
View File

@ -6,17 +6,17 @@
package apiproxy
import (
"context"
"crypto/tls"
"errors"
"fmt"
"log"
"net/http"
"net/http/httputil"
"net/netip"
"net/url"
"os"
"strings"
"time"
"github.com/pkg/errors"
"go.uber.org/zap"
"k8s.io/client-go/rest"
"k8s.io/client-go/transport"
@ -37,123 +37,52 @@ var (
whoIsKey = ctxkey.New("", (*apitype.WhoIsResponse)(nil))
)
type APIServerProxyMode int
func (a APIServerProxyMode) String() string {
switch a {
case APIServerProxyModeDisabled:
return "disabled"
case APIServerProxyModeEnabled:
return "auth"
case APIServerProxyModeNoAuth:
return "noauth"
default:
return "unknown"
}
}
const (
APIServerProxyModeDisabled APIServerProxyMode = iota
APIServerProxyModeEnabled
APIServerProxyModeNoAuth
)
func ParseAPIProxyMode() APIServerProxyMode {
haveAuthProxyEnv := os.Getenv("AUTH_PROXY") != ""
haveAPIProxyEnv := os.Getenv("APISERVER_PROXY") != ""
switch {
case haveAPIProxyEnv && haveAuthProxyEnv:
log.Fatal("AUTH_PROXY and APISERVER_PROXY are mutually exclusive")
case haveAuthProxyEnv:
var authProxyEnv = defaultBool("AUTH_PROXY", false) // deprecated
if authProxyEnv {
return APIServerProxyModeEnabled
}
return APIServerProxyModeDisabled
case haveAPIProxyEnv:
var apiProxyEnv = defaultEnv("APISERVER_PROXY", "") // true, false or "noauth"
switch apiProxyEnv {
case "true":
return APIServerProxyModeEnabled
case "false", "":
return APIServerProxyModeDisabled
case "noauth":
return APIServerProxyModeNoAuth
default:
panic(fmt.Sprintf("unknown APISERVER_PROXY value %q", apiProxyEnv))
}
}
return APIServerProxyModeDisabled
}
// maybeLaunchAPIServerProxy launches the auth proxy, which is a small HTTP server
// that authenticates requests using the Tailscale LocalAPI and then proxies
// them to the kube-apiserver.
func MaybeLaunchAPIServerProxy(zlog *zap.SugaredLogger, restConfig *rest.Config, s *tsnet.Server, mode APIServerProxyMode) {
if mode == APIServerProxyModeDisabled {
return
}
startlog := zlog.Named("launchAPIProxy")
if mode == APIServerProxyModeNoAuth {
// NewAPIServerProxy creates a new APIServerProxy that's ready to start once Run
// is called. No network traffic will flow until Run is called.
//
// authMode controls how the proxy behaves:
// - true: the proxy is started and requests are impersonated using the
// caller's Tailscale identity and the rules defined in the tailnet ACLs.
// - false: the proxy is started and requests are passed through to the
// Kubernetes API without any auth modifications.
func NewAPIServerProxy(zlog *zap.SugaredLogger, restConfig *rest.Config, ts *tsnet.Server, authMode bool) (*APIServerProxy, error) {
if !authMode {
restConfig = rest.AnonymousClientConfig(restConfig)
}
cfg, err := restConfig.TransportConfig()
if err != nil {
startlog.Fatalf("could not get rest.TransportConfig(): %v", err)
return nil, fmt.Errorf("could not get rest.TransportConfig(): %w", err)
}
// Kubernetes uses SPDY for exec and port-forward, however SPDY is
// incompatible with HTTP/2; so disable HTTP/2 in the proxy.
tr := http.DefaultTransport.(*http.Transport).Clone()
tr.TLSClientConfig, err = transport.TLSConfigFor(cfg)
if err != nil {
startlog.Fatalf("could not get transport.TLSConfigFor(): %v", err)
return nil, fmt.Errorf("could not get transport.TLSConfigFor(): %w", err)
}
tr.TLSNextProto = make(map[string]func(authority string, c *tls.Conn) http.RoundTripper)
rt, err := transport.HTTPWrappersForConfig(cfg, tr)
if err != nil {
startlog.Fatalf("could not get rest.TransportConfig(): %v", err)
return nil, fmt.Errorf("could not get rest.TransportConfig(): %w", err)
}
go runAPIServerProxy(s, rt, zlog.Named("apiserver-proxy"), mode, restConfig.Host)
}
// runAPIServerProxy runs an HTTP server that authenticates requests using the
// Tailscale LocalAPI and then proxies them to the Kubernetes API.
// It listens on :443 and uses the Tailscale HTTPS certificate.
// s will be started if it is not already running.
// rt is used to proxy requests to the Kubernetes API.
//
// mode controls how the proxy behaves:
// - apiserverProxyModeDisabled: the proxy is not started.
// - apiserverProxyModeEnabled: the proxy is started and requests are impersonated using the
// caller's identity from the Tailscale LocalAPI.
// - apiserverProxyModeNoAuth: the proxy is started and requests are not impersonated and
// are passed through to the Kubernetes API.
//
// It never returns.
func runAPIServerProxy(ts *tsnet.Server, rt http.RoundTripper, log *zap.SugaredLogger, mode APIServerProxyMode, host string) {
if mode == APIServerProxyModeDisabled {
return
}
ln, err := ts.Listen("tcp", ":443")
u, err := url.Parse(restConfig.Host)
if err != nil {
log.Fatalf("could not listen on :443: %v", err)
return nil, fmt.Errorf("failed to parse URL %w", err)
}
u, err := url.Parse(host)
if err != nil {
log.Fatalf("runAPIServerProxy: failed to parse URL %v", err)
if u.Scheme == "" || u.Host == "" {
return nil, fmt.Errorf("the API server proxy requires host and scheme but got: %q", restConfig.Host)
}
lc, err := ts.LocalClient()
if err != nil {
log.Fatalf("could not get local client: %v", err)
return nil, fmt.Errorf("could not get local client: %w", err)
}
ap := &apiserverProxy{
log: log,
ap := &APIServerProxy{
log: zlog,
lc: lc,
mode: mode,
authMode: authMode,
upstreamURL: u,
ts: ts,
}
@ -164,41 +93,69 @@ func runAPIServerProxy(ts *tsnet.Server, rt http.RoundTripper, log *zap.SugaredL
Transport: rt,
}
return ap, nil
}
// Run starts the HTTP server that authenticates requests using the
// Tailscale LocalAPI and then proxies them to the Kubernetes API.
// It listens on :443 and uses the Tailscale HTTPS certificate.
//
// It return when ctx is cancelled or ServeTLS fails.
func (ap *APIServerProxy) Run(ctx context.Context) error {
ln, err := ap.ts.Listen("tcp", ":443")
if err != nil {
return fmt.Errorf("could not listen on :443: %v", err)
}
mux := http.NewServeMux()
mux.HandleFunc("/", ap.serveDefault)
mux.HandleFunc("POST /api/v1/namespaces/{namespace}/pods/{pod}/exec", ap.serveExecSPDY)
mux.HandleFunc("GET /api/v1/namespaces/{namespace}/pods/{pod}/exec", ap.serveExecWS)
hs := &http.Server{
ap.hs = &http.Server{
// Kubernetes uses SPDY for exec and port-forward, however SPDY is
// incompatible with HTTP/2; so disable HTTP/2 in the proxy.
TLSConfig: &tls.Config{
GetCertificate: lc.GetCertificate,
GetCertificate: ap.lc.GetCertificate,
NextProtos: []string{"http/1.1"},
},
TLSNextProto: make(map[string]func(*http.Server, *tls.Conn, http.Handler)),
Handler: mux,
}
log.Infof("API server proxy in %q mode is listening on %s", mode, ln.Addr())
if err := hs.ServeTLS(ln, "", ""); err != nil {
log.Fatalf("runAPIServerProxy: failed to serve %v", err)
errs := make(chan error)
go func() {
ap.log.Infof("API server proxy is listening on %s with auth mode: %v", ln.Addr(), ap.authMode)
if err := ap.hs.ServeTLS(ln, "", ""); err != nil && err != http.ErrServerClosed {
errs <- fmt.Errorf("failed to serve: %w", err)
}
}()
select {
case <-ctx.Done():
shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
return ap.hs.Shutdown(shutdownCtx)
case err := <-errs:
return err
}
}
// apiserverProxy is an [net/http.Handler] that authenticates requests using the Tailscale
// APIServerProxy is an [net/http.Handler] that authenticates requests using the Tailscale
// LocalAPI and then proxies them to the Kubernetes API.
type apiserverProxy struct {
type APIServerProxy struct {
log *zap.SugaredLogger
lc *local.Client
rp *httputil.ReverseProxy
mode APIServerProxyMode
authMode bool
ts *tsnet.Server
hs *http.Server
upstreamURL *url.URL
}
// serveDefault is the default handler for Kubernetes API server requests.
func (ap *apiserverProxy) serveDefault(w http.ResponseWriter, r *http.Request) {
func (ap *APIServerProxy) serveDefault(w http.ResponseWriter, r *http.Request) {
who, err := ap.whoIs(r)
if err != nil {
ap.authError(w, err)
@ -210,17 +167,17 @@ func (ap *apiserverProxy) serveDefault(w http.ResponseWriter, r *http.Request) {
// serveExecSPDY serves 'kubectl exec' requests for sessions streamed over SPDY,
// optionally configuring the kubectl exec sessions to be recorded.
func (ap *apiserverProxy) serveExecSPDY(w http.ResponseWriter, r *http.Request) {
func (ap *APIServerProxy) serveExecSPDY(w http.ResponseWriter, r *http.Request) {
ap.execForProto(w, r, ksr.SPDYProtocol)
}
// serveExecWS serves 'kubectl exec' requests for sessions streamed over WebSocket,
// optionally configuring the kubectl exec sessions to be recorded.
func (ap *apiserverProxy) serveExecWS(w http.ResponseWriter, r *http.Request) {
func (ap *APIServerProxy) serveExecWS(w http.ResponseWriter, r *http.Request) {
ap.execForProto(w, r, ksr.WSProtocol)
}
func (ap *apiserverProxy) execForProto(w http.ResponseWriter, r *http.Request, proto ksr.Protocol) {
func (ap *APIServerProxy) execForProto(w http.ResponseWriter, r *http.Request, proto ksr.Protocol) {
const (
podNameKey = "pod"
namespaceNameKey = "namespace"
@ -282,10 +239,10 @@ func (ap *apiserverProxy) execForProto(w http.ResponseWriter, r *http.Request, p
ap.rp.ServeHTTP(h, r.WithContext(whoIsKey.WithValue(r.Context(), who)))
}
func (h *apiserverProxy) addImpersonationHeadersAsRequired(r *http.Request) {
r.URL.Scheme = h.upstreamURL.Scheme
r.URL.Host = h.upstreamURL.Host
if h.mode == APIServerProxyModeNoAuth {
func (ap *APIServerProxy) addImpersonationHeadersAsRequired(r *http.Request) {
r.URL.Scheme = ap.upstreamURL.Scheme
r.URL.Host = ap.upstreamURL.Host
if !ap.authMode {
// If we are not providing authentication, then we are just
// proxying to the Kubernetes API, so we don't need to do
// anything else.
@ -310,16 +267,16 @@ func (h *apiserverProxy) addImpersonationHeadersAsRequired(r *http.Request) {
}
// Now add the impersonation headers that we want.
if err := addImpersonationHeaders(r, h.log); err != nil {
log.Print("failed to add impersonation headers: ", err.Error())
if err := addImpersonationHeaders(r, ap.log); err != nil {
ap.log.Errorf("failed to add impersonation headers: %v", err)
}
}
func (ap *apiserverProxy) whoIs(r *http.Request) (*apitype.WhoIsResponse, error) {
func (ap *APIServerProxy) whoIs(r *http.Request) (*apitype.WhoIsResponse, error) {
return ap.lc.WhoIs(r.Context(), r.RemoteAddr)
}
func (ap *apiserverProxy) authError(w http.ResponseWriter, err error) {
func (ap *APIServerProxy) authError(w http.ResponseWriter, err error) {
ap.log.Errorf("failed to authenticate caller: %v", err)
http.Error(w, "failed to authenticate caller", http.StatusInternalServerError)
}

40
k8s-operator/api.md
View File

@ -21,6 +21,21 @@
#### APIServerProxyMode
_Underlying type:_ _string_
_Validation:_
- Enum: [auth noauth]
- Type: string
_Appears in:_
- [KubeAPIServerConfig](#kubeapiserverconfig)
#### AppConnector
@ -142,7 +157,7 @@ _Appears in:_
| Field | Description | Default | Validation |
| --- | --- | --- | --- |
| `env` _[Env](#env) array_ | List of environment variables to set in the container.<br />https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#environment-variables<br />Note that environment variables provided here will take precedence<br />over Tailscale-specific environment variables set by the operator,<br />however running proxies with custom values for Tailscale environment<br />variables (i.e TS_USERSPACE) is not recommended and might break in<br />the future. | | |
| `image` _string_ | Container image name. By default images are pulled from<br />docker.io/tailscale/tailscale, but the official images are also<br />available at ghcr.io/tailscale/tailscale. Specifying image name here<br />will override any proxy image values specified via the Kubernetes<br />operator's Helm chart values or PROXY_IMAGE env var in the operator<br />Deployment.<br />https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#image | | |
| `image` _string_ | Container image name. By default images are pulled from docker.io/tailscale,<br />but the official images are also available at ghcr.io/tailscale.<br />For all uses except on ProxyGroups of type "kube-apiserver", this image must<br />be either tailscale/tailscale, or an equivalent mirror of that image.<br />To apply to ProxyGroups of type "kube-apiserver", this image must be<br />tailscale/k8s-proxy or a mirror of that image.<br />For "tailscale/tailscale"-based proxies, specifying image name here will<br />override any proxy image values specified via the Kubernetes operator's<br />Helm chart values or PROXY_IMAGE env var in the operator Deployment.<br />For "tailscale/k8s-proxy"-based proxies, there is currently no way to<br />configure your own default, and this field is the only way to use a<br />custom image.<br />https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#image | | |
| `imagePullPolicy` _[PullPolicy](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.3/#pullpolicy-v1-core)_ | Image pull policy. One of Always, Never, IfNotPresent. Defaults to Always.<br />https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#image | | Enum: [Always Never IfNotPresent] <br /> |
| `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.3/#resourcerequirements-v1-core)_ | Container resource requirements.<br />By default Tailscale Kubernetes operator does not apply any resource<br />requirements. The amount of resources required wil depend on the<br />amount of resources the operator needs to parse, usage patterns and<br />cluster size.<br />https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#resources | | |
| `securityContext` _[SecurityContext](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.3/#securitycontext-v1-core)_ | Container security context.<br />Security context specified here will override the security context set by the operator.<br />By default the operator sets the Tailscale container and the Tailscale init container to privileged<br />for proxies created for Tailscale ingress and egress Service, Connector and ProxyGroup.<br />You can reduce the permissions of the Tailscale container to cap NET_ADMIN by<br />installing device plugin in your cluster and configuring the proxies tun device to be created<br />by the device plugin, see https://github.com/tailscale/tailscale/issues/10814#issuecomment-2479977752<br />https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#security-context | | |
@ -313,6 +328,22 @@ _Appears in:_
#### KubeAPIServerConfig
KubeAPIServerConfig contains configuration specific to the kube-apiserver ProxyGroup type.
_Appears in:_
- [ProxyGroupSpec](#proxygroupspec)
| Field | Description | Default | Validation |
| --- | --- | --- | --- |
| `mode` _[APIServerProxyMode](#apiserverproxymode)_ | Mode to run the API server proxy in. Supported modes are auth and noauth.<br />In auth mode, requests from the tailnet proxied over to the Kubernetes<br />API server are additionally impersonated using the sender's tailnet identity.<br />If not specified, defaults to auth mode. | | Enum: [auth noauth] <br />Type: string <br /> |
#### LabelValue
_Underlying type:_ _string_
@ -459,7 +490,7 @@ _Appears in:_
| `annotations` _object (keys:string, values:string)_ | Annotations that will be added to the proxy Pod.<br />Any annotations specified here will be merged with the default<br />annotations applied to the Pod by the Tailscale Kubernetes operator.<br />Annotations must be valid Kubernetes annotations.<br />https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#syntax-and-character-set | | |
| `affinity` _[Affinity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.3/#affinity-v1-core)_ | Proxy Pod's affinity rules.<br />By default, the Tailscale Kubernetes operator does not apply any affinity rules.<br />https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#affinity | | |
| `tailscaleContainer` _[Container](#container)_ | Configuration for the proxy container running tailscale. | | |
| `tailscaleInitContainer` _[Container](#container)_ | Configuration for the proxy init container that enables forwarding. | | |
| `tailscaleInitContainer` _[Container](#container)_ | Configuration for the proxy init container that enables forwarding.<br />Not valid to apply to ProxyGroups of type "kube-apiserver". | | |
| `securityContext` _[PodSecurityContext](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.3/#podsecuritycontext-v1-core)_ | Proxy Pod's security context.<br />By default Tailscale Kubernetes operator does not apply any Pod<br />security context.<br />https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#security-context-2 | | |
| `imagePullSecrets` _[LocalObjectReference](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.3/#localobjectreference-v1-core) array_ | Proxy Pod's image pull Secrets.<br />https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#PodSpec | | |
| `nodeName` _string_ | Proxy Pod's node name.<br />https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#scheduling | | |
@ -638,11 +669,12 @@ _Appears in:_
| Field | Description | Default | Validation |
| --- | --- | --- | --- |
| `type` _[ProxyGroupType](#proxygrouptype)_ | Type of the ProxyGroup proxies. Supported types are egress and ingress.<br />Type is immutable once a ProxyGroup is created. | | Enum: [egress ingress] <br />Type: string <br /> |
| `type` _[ProxyGroupType](#proxygrouptype)_ | Type of the ProxyGroup proxies. Supported types are egress, ingress, and kube-apiserver.<br />Type is immutable once a ProxyGroup is created. | | Enum: [egress ingress kube-apiserver] <br />Type: string <br /> |
| `tags` _[Tags](#tags)_ | Tags that the Tailscale devices will be tagged with. Defaults to [tag:k8s].<br />If you specify custom tags here, make sure you also make the operator<br />an owner of these tags.<br />See https://tailscale.com/kb/1236/kubernetes-operator/#setting-up-the-kubernetes-operator.<br />Tags cannot be changed once a ProxyGroup device has been created.<br />Tag values must be in form ^tag:[a-zA-Z][a-zA-Z0-9-]*$. | | Pattern: `^tag:[a-zA-Z][a-zA-Z0-9-]*$` <br />Type: string <br /> |
| `replicas` _integer_ | Replicas specifies how many replicas to create the StatefulSet with.<br />Defaults to 2. | | Minimum: 0 <br /> |
| `hostnamePrefix` _[HostnamePrefix](#hostnameprefix)_ | HostnamePrefix is the hostname prefix to use for tailnet devices created<br />by the ProxyGroup. Each device will have the integer number from its<br />StatefulSet pod appended to this prefix to form the full hostname.<br />HostnamePrefix can contain lower case letters, numbers and dashes, it<br />must not start with a dash and must be between 1 and 62 characters long. | | Pattern: `^[a-z0-9][a-z0-9-]{0,61}$` <br />Type: string <br /> |
| `proxyClass` _string_ | ProxyClass is the name of the ProxyClass custom resource that contains<br />configuration options that should be applied to the resources created<br />for this ProxyGroup. If unset, and there is no default ProxyClass<br />configured, the operator will create resources with the default<br />configuration. | | |
| `kubeAPIServer` _[KubeAPIServerConfig](#kubeapiserverconfig)_ | KubeAPIServer contains configuration specific to the kube-apiserver<br />ProxyGroup type. This field is only used when Type is set to "kube-apiserver". | | |
#### ProxyGroupStatus
@ -669,7 +701,7 @@ _Underlying type:_ _string_
_Validation:_
- Enum: [egress ingress]
- Enum: [egress ingress kube-apiserver]
- Type: string
_Appears in:_

22
k8s-operator/apis/v1alpha1/types_proxyclass.go
View File

@ -264,6 +264,7 @@ type Pod struct {
// +optional
TailscaleContainer *Container `json:"tailscaleContainer,omitempty"`
// Configuration for the proxy init container that enables forwarding.
// Not valid to apply to ProxyGroups of type "kube-apiserver".
// +optional
TailscaleInitContainer *Container `json:"tailscaleInitContainer,omitempty"`
// Proxy Pod's security context.
@ -364,12 +365,21 @@ type Container struct {
// the future.
// +optional
Env []Env `json:"env,omitempty"`
// Container image name. By default images are pulled from
// docker.io/tailscale/tailscale, but the official images are also
// available at ghcr.io/tailscale/tailscale. Specifying image name here
// will override any proxy image values specified via the Kubernetes
// operator's Helm chart values or PROXY_IMAGE env var in the operator
// Deployment.
// Container image name. By default images are pulled from docker.io/tailscale,
// but the official images are also available at ghcr.io/tailscale.
//
// For all uses except on ProxyGroups of type "kube-apiserver", this image must
// be either tailscale/tailscale, or an equivalent mirror of that image.
// To apply to ProxyGroups of type "kube-apiserver", this image must be
// tailscale/k8s-proxy or a mirror of that image.
//
// For "tailscale/tailscale"-based proxies, specifying image name here will
// override any proxy image values specified via the Kubernetes operator's
// Helm chart values or PROXY_IMAGE env var in the operator Deployment.
// For "tailscale/k8s-proxy"-based proxies, there is currently no way to
// configure your own default, and this field is the only way to use a
// custom image.
//
// https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#image
// +optional
Image string `json:"image,omitempty"`

33
k8s-operator/apis/v1alpha1/types_proxygroup.go
View File

@ -49,7 +49,7 @@ type ProxyGroupList struct {
}
type ProxyGroupSpec struct {
// Type of the ProxyGroup proxies. Supported types are egress and ingress.
// Type of the ProxyGroup proxies. Supported types are egress, ingress, and kube-apiserver.
// Type is immutable once a ProxyGroup is created.
// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="ProxyGroup type is immutable"
Type ProxyGroupType `json:"type"`
@ -84,6 +84,11 @@ type ProxyGroupSpec struct {
// configuration.
// +optional
ProxyClass string `json:"proxyClass,omitempty"`
// KubeAPIServer contains configuration specific to the kube-apiserver
// ProxyGroup type. This field is only used when Type is set to "kube-apiserver".
// +optional
KubeAPIServer *KubeAPIServerConfig `json:"kubeAPIServer,omitempty"`
}
type ProxyGroupStatus struct {
@ -122,14 +127,34 @@ type TailnetDevice struct {
}
// +kubebuilder:validation:Type=string
// +kubebuilder:validation:Enum=egress;ingress
// +kubebuilder:validation:Enum=egress;ingress;kube-apiserver
type ProxyGroupType string
const (
ProxyGroupTypeEgress ProxyGroupType = "egress"
ProxyGroupTypeIngress ProxyGroupType = "ingress"
ProxyGroupTypeEgress ProxyGroupType = "egress"
ProxyGroupTypeIngress ProxyGroupType = "ingress"
ProxyGroupTypeKubernetesAPIServer ProxyGroupType = "kube-apiserver"
)
// +kubebuilder:validation:Type=string
// +kubebuilder:validation:Enum=auth;noauth
type APIServerProxyMode string
const (
APIServerProxyModeAuth APIServerProxyMode = "auth"
APIServerProxyModeNoAuth APIServerProxyMode = "noauth"
)
// +kubebuilder:validation:Type=string
// +kubebuilder:validation:Pattern=`^[a-z0-9][a-z0-9-]{0,61}$`
type HostnamePrefix string
// KubeAPIServerConfig contains configuration specific to the kube-apiserver ProxyGroup type.
type KubeAPIServerConfig struct {
// Mode to run the API server proxy in. Supported modes are auth and noauth.
// In auth mode, requests from the tailnet proxied over to the Kubernetes
// API server are additionally impersonated using the sender's tailnet identity.
// If not specified, defaults to auth mode.
// +optional
Mode *APIServerProxyMode `json:"mode,omitempty"`
}

25
k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go
View File

@ -316,6 +316,26 @@ func (in *Env) DeepCopy() *Env {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *KubeAPIServerConfig) DeepCopyInto(out *KubeAPIServerConfig) {
*out = *in
if in.Mode != nil {
in, out := &in.Mode, &out.Mode
*out = new(APIServerProxyMode)
**out = **in
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeAPIServerConfig.
func (in *KubeAPIServerConfig) DeepCopy() *KubeAPIServerConfig {
if in == nil {
return nil
}
out := new(KubeAPIServerConfig)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in Labels) DeepCopyInto(out *Labels) {
{
@ -731,6 +751,11 @@ func (in *ProxyGroupSpec) DeepCopyInto(out *ProxyGroupSpec) {
*out = new(int32)
**out = **in
}
if in.KubeAPIServer != nil {
in, out := &in.KubeAPIServer, &out.KubeAPIServer
*out = new(KubeAPIServerConfig)
(*in).DeepCopyInto(*out)
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxyGroupSpec.

101
kube/k8s-proxy/conf/conf.go Normal file
View File

@ -0,0 +1,101 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
//go:build !plan9
// Package conf contains code to load, manipulate, and access config file
// settings for k8s-proxy.
package conf
import (
"encoding/json"
"fmt"
"os"
"github.com/tailscale/hujson"
"tailscale.com/types/opt"
)
const v1Alpha1 = "v1alpha1"
// Config describes a config file.
type Config struct {
Path string // disk path of HuJSON
Raw []byte // raw bytes from disk, in HuJSON form
Std []byte // standardized JSON form
Version string // "v1alpha1"
// Parsed is the parsed config, converted from its on-disk version to the
// latest known format.
Parsed ConfigV1Alpha1
}
// VersionedConfig allows specifying config at the root of the object, or in
// a versioned sub-object.
// e.g. {"version": "v1alpha1", "authKey": "abc123"}
// or {"version": "v1beta1", "a-beta-config": "a-beta-value", "v1alpha1": {"authKey": "abc123"}}
type VersionedConfig struct {
Version string `json:",omitempty"` // "v1alpha1"
// Latest version of the config.
*ConfigV1Alpha1
// Backwards compatibility version(s) of the config. Fields and sub-fields
// from here should only be added to, never changed in place.
V1Alpha1 *ConfigV1Alpha1 `json:",omitempty"`
// V1Beta1 *ConfigV1Beta1 `json:",omitempty"` // Not yet used.
}
type ConfigV1Alpha1 struct {
AuthKey *string `json:",omitempty"` // Tailscale auth key to use.
Hostname *string `json:",omitempty"` // Tailscale device hostname.
State *string `json:",omitempty"` // Path to the Tailscale state.
LogLevel *string `json:",omitempty"` // "debug", "info". Defaults to "info".
App *string `json:",omitempty"` // e.g. kubetypes.AppProxyGroupKubeAPIServer
KubeAPIServer *KubeAPIServer `json:",omitempty"` // Config specific to the API Server proxy.
}
type KubeAPIServer struct {
AuthMode opt.Bool `json:",omitempty"`
}
// Load reads and parses the config file at the provided path on disk.
func Load(path string) (c Config, err error) {
c.Path = path
c.Raw, err = os.ReadFile(path)
if err != nil {
return c, fmt.Errorf("error reading config file %q: %w", path, err)
}
c.Std, err = hujson.Standardize(c.Raw)
if err != nil {
return c, fmt.Errorf("error parsing config file %q HuJSON/JSON: %w", path, err)
}
var ver VersionedConfig
if err := json.Unmarshal(c.Std, &ver); err != nil {
return c, fmt.Errorf("error parsing config file %q: %w", path, err)
}
rootV1Alpha1 := (ver.Version == v1Alpha1)
backCompatV1Alpha1 := (ver.V1Alpha1 != nil)
switch {
case ver.Version == "":
return c, fmt.Errorf("error parsing config file %q: no \"version\" field provided", path)
case rootV1Alpha1 && backCompatV1Alpha1:
// Exactly one of these should be set.
return c, fmt.Errorf("error parsing config file %q: both root and v1alpha1 config provided", path)
case rootV1Alpha1 != backCompatV1Alpha1:
c.Version = v1Alpha1
switch {
case rootV1Alpha1 && ver.ConfigV1Alpha1 != nil:
c.Parsed = *ver.ConfigV1Alpha1
case backCompatV1Alpha1:
c.Parsed = *ver.V1Alpha1
default:
c.Parsed = ConfigV1Alpha1{}
}
default:
return c, fmt.Errorf("error parsing config file %q: unsupported \"version\" value %q; want \"%s\"", path, ver.Version, v1Alpha1)
}
return c, nil
}

86
kube/k8s-proxy/conf/conf_test.go Normal file
View File

@ -0,0 +1,86 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
//go:build !plan9
package conf
import (
"os"
"path/filepath"
"strings"
"testing"
"github.com/google/go-cmp/cmp"
"tailscale.com/types/ptr"
)
// Test that the config file can be at the root of the object, or in a versioned sub-object.
// or {"version": "v1beta1", "a-beta-config": "a-beta-value", "v1alpha1": {"authKey": "abc123"}}
func TestVersionedConfig(t *testing.T) {
testCases := map[string]struct {
inputConfig string
expectedConfig ConfigV1Alpha1
expectedError string
}{
"root_config_v1alpha1": {
inputConfig: `{"version": "v1alpha1", "authKey": "abc123"}`,
expectedConfig: ConfigV1Alpha1{AuthKey: ptr.To("abc123")},
},
"backwards_compat_v1alpha1_config": {
// Client doesn't know about v1beta1, so it should read in v1alpha1.
inputConfig: `{"version": "v1beta1", "beta-key": "beta-value", "authKey": "def456", "v1alpha1": {"authKey": "abc123"}}`,
expectedConfig: ConfigV1Alpha1{AuthKey: ptr.To("abc123")},
},
"unknown_key_allowed": {
// Adding new keys to the config doesn't require a version bump.
inputConfig: `{"version": "v1alpha1", "unknown-key": "unknown-value", "authKey": "abc123"}`,
expectedConfig: ConfigV1Alpha1{AuthKey: ptr.To("abc123")},
},
"version_only_no_authkey": {
inputConfig: `{"version": "v1alpha1"}`,
expectedConfig: ConfigV1Alpha1{},
},
"both_config_v1alpha1": {
inputConfig: `{"version": "v1alpha1", "authKey": "abc123", "v1alpha1": {"authKey": "def456"}}`,
expectedError: "both root and v1alpha1 config provided",
},
"empty_config": {
inputConfig: `{}`,
expectedError: `no "version" field provided`,
},
"v1beta1_without_backwards_compat": {
inputConfig: `{"version": "v1beta1", "beta-key": "beta-value", "authKey": "def456"}`,
expectedError: `unsupported "version" value "v1beta1"; want "v1alpha1"`,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "config.json")
if err := os.WriteFile(path, []byte(tc.inputConfig), 0644); err != nil {
t.Fatalf("failed to write config file: %v", err)
}
cfg, err := Load(path)
switch {
case tc.expectedError == "" && err != nil:
t.Fatalf("unexpected error: %v", err)
case tc.expectedError != "":
if err == nil {
t.Fatalf("expected error %q, got nil", tc.expectedError)
} else if !strings.Contains(err.Error(), tc.expectedError) {
t.Fatalf("expected error %q, got %q", tc.expectedError, err.Error())
}
return
}
if cfg.Version != "v1alpha1" {
t.Fatalf("expected version %q, got %q", "v1alpha1", cfg.Version)
}
// Diff actual vs expected config.
if diff := cmp.Diff(cfg.Parsed, tc.expectedConfig); diff != "" {
t.Fatalf("Unexpected parsed config (-got +want):\n%s", diff)
}
})
}
}

18
kube/kubetypes/types.go
View File

@ -5,14 +5,15 @@ package kubetypes
const (
// Hostinfo App values for the Tailscale Kubernetes Operator components.
AppOperator = "k8s-operator"
AppAPIServerProxy = "k8s-operator-proxy"
AppIngressProxy = "k8s-operator-ingress-proxy"
AppIngressResource = "k8s-operator-ingress-resource"
AppEgressProxy = "k8s-operator-egress-proxy"
AppConnector = "k8s-operator-connector-resource"
AppProxyGroupEgress = "k8s-operator-proxygroup-egress"
AppProxyGroupIngress = "k8s-operator-proxygroup-ingress"
AppOperator = "k8s-operator"
AppInProcessAPIServerProxy = "k8s-operator-proxy"
AppIngressProxy = "k8s-operator-ingress-proxy"
AppIngressResource = "k8s-operator-ingress-resource"
AppEgressProxy = "k8s-operator-egress-proxy"
AppConnector = "k8s-operator-connector-resource"
AppProxyGroupEgress = "k8s-operator-proxygroup-egress"
AppProxyGroupIngress = "k8s-operator-proxygroup-ingress"
AppProxyGroupKubeAPIServer = "k8s-operator-proxygroup-kube-apiserver"
// Clientmetrics for Tailscale Kubernetes Operator components
MetricIngressProxyCount = "k8s_ingress_proxies" // L3
@ -29,6 +30,7 @@ const (
MetricEgressServiceCount = "k8s_egress_service_resources"
MetricProxyGroupEgressCount = "k8s_proxygroup_egress_resources"
MetricProxyGroupIngressCount = "k8s_proxygroup_ingress_resources"
MetricProxyGroupAPIServerCount = "k8s_proxygroup_kube_apiserver_resources"
// Keys that containerboot writes to state file that can be used to determine its state.
// fields set in Tailscale state Secret. These are mostly used by the Tailscale Kubernetes operator to determine

97
kube/state/state.go Normal file
View File

@ -0,0 +1,97 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
//go:build !plan9
// Package state updates state keys for tailnet client devices managed by the
// operator. These keys are used to signal readiness, metadata, and current
// configuration state to the operator. Client packages deployed by the operator
// include containerboot, tsrecorder, and k8s-proxy, but currently containerboot
// has its own implementation to manage the same keys.
package state
import (
"encoding/json"
"fmt"
"tailscale.com/ipn"
"tailscale.com/kube/kubetypes"
"tailscale.com/tailcfg"
"tailscale.com/util/deephash"
)
const (
keyPodUID = ipn.StateKey(kubetypes.KeyPodUID)
keyCapVer = ipn.StateKey(kubetypes.KeyCapVer)
keyDeviceID = ipn.StateKey(kubetypes.KeyDeviceID)
keyDeviceIPs = ipn.StateKey(kubetypes.KeyDeviceIPs)
keyDeviceFQDN = ipn.StateKey(kubetypes.KeyDeviceFQDN)
)
// SetInitialKeys sets Pod UID and cap ver and clears tailnet device state
// keys to help stop the operator using stale tailnet device state.
func SetInitialKeys(store ipn.StateStore, podUID string) error {
// Clear device state keys first so the operator knows if the pod UID
// matches, the other values are definitely not stale.
for _, key := range []ipn.StateKey{keyDeviceID, keyDeviceFQDN, keyDeviceIPs} {
if _, err := store.ReadState(key); err == nil {
if err := store.WriteState(key, nil); err != nil {
return fmt.Errorf("error writing %q to state store: %w", key, err)
}
}
}
if err := store.WriteState(keyPodUID, []byte(podUID)); err != nil {
return fmt.Errorf("error writing pod UID to state store: %w", err)
}
if err := store.WriteState(keyCapVer, fmt.Appendf(nil, "%d", tailcfg.CurrentCapabilityVersion)); err != nil {
return fmt.Errorf("error writing capability version to state store: %w", err)
}
return nil
}
// KeepKeysUpdated sets state store keys consistent with containerboot to
// signal proxy readiness to the operator. It runs until its context is
// cancelled or it hits an error. The passed in next function is expected to be
// from a local.IPNBusWatcher that is at least subscribed to
// ipn.NotifyInitialNetMap.
func KeepKeysUpdated(store ipn.StateStore, next func() (ipn.Notify, error)) error {
var currentDeviceID, currentDeviceIPs, currentDeviceFQDN deephash.Sum
for {
n, err := next() // Blocks on a streaming LocalAPI HTTP call.
if err != nil {
return err
}
if n.NetMap == nil {
continue
}
if deviceID := n.NetMap.SelfNode.StableID(); deephash.Update(&currentDeviceID, &deviceID) {
if err := store.WriteState(keyDeviceID, []byte(deviceID)); err != nil {
return fmt.Errorf("failed to store device ID in state: %w", err)
}
}
if fqdn := n.NetMap.SelfNode.Name(); deephash.Update(&currentDeviceFQDN, &fqdn) {
if err := store.WriteState(keyDeviceFQDN, []byte(fqdn)); err != nil {
return fmt.Errorf("failed to store device FQDN in state: %w", err)
}
}
if addrs := n.NetMap.SelfNode.Addresses(); deephash.Update(&currentDeviceIPs, &addrs) {
var deviceIPs []string
for _, addr := range addrs.AsSlice() {
deviceIPs = append(deviceIPs, addr.Addr().String())
}
deviceIPsValue, err := json.Marshal(deviceIPs)
if err != nil {
return err
}
if err := store.WriteState(keyDeviceIPs, deviceIPsValue); err != nil {
return fmt.Errorf("failed to store device IPs in state: %w", err)
}
}
}
}

203
kube/state/state_test.go Normal file
View File

@ -0,0 +1,203 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
//go:build !plan9
package state
import (
"bytes"
"fmt"
"net/netip"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"tailscale.com/ipn"
"tailscale.com/ipn/store"
"tailscale.com/tailcfg"
"tailscale.com/types/logger"
"tailscale.com/types/netmap"
)
func TestSetInitialStateKeys(t *testing.T) {
var (
podUID = []byte("test-pod-uid")
expectedCapVer = fmt.Appendf(nil, "%d", tailcfg.CurrentCapabilityVersion)
)
for name, tc := range map[string]struct {
initial map[ipn.StateKey][]byte
expected map[ipn.StateKey][]byte
}{
"empty_initial": {
initial: map[ipn.StateKey][]byte{},
expected: map[ipn.StateKey][]byte{
keyPodUID: podUID,
keyCapVer: expectedCapVer,
},
},
"existing_pod_uid_and_capver": {
initial: map[ipn.StateKey][]byte{
keyPodUID: podUID,
keyCapVer: expectedCapVer,
},
expected: map[ipn.StateKey][]byte{
keyPodUID: podUID,
keyCapVer: expectedCapVer,
},
},
"all_keys_preexisting": {
initial: map[ipn.StateKey][]byte{
keyPodUID: podUID,
keyCapVer: expectedCapVer,
keyDeviceID: []byte("existing-device-id"),
keyDeviceFQDN: []byte("existing-device-fqdn"),
keyDeviceIPs: []byte(`["1.2.3.4"]`),
},
expected: map[ipn.StateKey][]byte{
keyPodUID: podUID,
keyCapVer: expectedCapVer,
keyDeviceID: nil,
keyDeviceFQDN: nil,
keyDeviceIPs: nil,
},
},
} {
t.Run(name, func(t *testing.T) {
store, err := store.New(logger.Discard, "mem:")
if err != nil {
t.Fatalf("error creating in-memory store: %v", err)
}
for key, value := range tc.initial {
if err := store.WriteState(key, value); err != nil {
t.Fatalf("error writing initial state key %q: %v", key, err)
}
}
if err := SetInitialKeys(store, string(podUID)); err != nil {
t.Fatalf("setInitialStateKeys failed: %v", err)
}
actual := make(map[ipn.StateKey][]byte)
for expectedKey, expectedValue := range tc.expected {
actualValue, err := store.ReadState(expectedKey)
if err != nil {
t.Errorf("error reading state key %q: %v", expectedKey, err)
continue
}
actual[expectedKey] = actualValue
if !bytes.Equal(actualValue, expectedValue) {
t.Errorf("state key %q mismatch: expected %q, got %q", expectedKey, expectedValue, actualValue)
}
}
if diff := cmp.Diff(actual, tc.expected); diff != "" {
t.Errorf("state keys mismatch (-got +want):\n%s", diff)
}
})
}
}
func TestKeepStateKeysUpdated(t *testing.T) {
store, err := store.New(logger.Discard, "mem:")
if err != nil {
t.Fatalf("error creating in-memory store: %v", err)
}
nextWaiting := make(chan struct{})
go func() {
<-nextWaiting // Acknowledge the initial signal.
}()
notifyCh := make(chan ipn.Notify)
next := func() (ipn.Notify, error) {
nextWaiting <- struct{}{} // Send signal to test that state is consistent.
return <-notifyCh, nil // Wait for test input.
}
errs := make(chan error, 1)
go func() {
err := KeepKeysUpdated(store, next)
if err != nil {
errs <- fmt.Errorf("keepStateKeysUpdated returned with error: %w", err)
}
}()
for _, tc := range []struct {
name string
notify ipn.Notify
expected map[ipn.StateKey][]byte
}{
{
name: "initial_not_authed",
notify: ipn.Notify{},
expected: map[ipn.StateKey][]byte{
keyDeviceID: nil,
keyDeviceFQDN: nil,
keyDeviceIPs: nil,
},
},
{
name: "authed",
notify: ipn.Notify{
NetMap: &netmap.NetworkMap{
SelfNode: (&tailcfg.Node{
StableID: "TESTCTRL00000001",
Name: "test-node.test.ts.net",
Addresses: []netip.Prefix{netip.MustParsePrefix("100.64.0.1/32"), netip.MustParsePrefix("fd7a:115c:a1e0:ab12:4843:cd96:0:1/128")},
}).View(),
},
},
expected: map[ipn.StateKey][]byte{
keyDeviceID: []byte("TESTCTRL00000001"),
keyDeviceFQDN: []byte("test-node.test.ts.net"),
keyDeviceIPs: []byte(`["100.64.0.1","fd7a:115c:a1e0:ab12:4843:cd96:0:1"]`),
},
},
{
name: "updated_fields",
notify: ipn.Notify{
NetMap: &netmap.NetworkMap{
SelfNode: (&tailcfg.Node{
StableID: "TESTCTRL00000001",
Name: "updated.test.ts.net",
Addresses: []netip.Prefix{netip.MustParsePrefix("100.64.0.250/32")},
}).View(),
},
},
expected: map[ipn.StateKey][]byte{
keyDeviceID: []byte("TESTCTRL00000001"),
keyDeviceFQDN: []byte("updated.test.ts.net"),
keyDeviceIPs: []byte(`["100.64.0.250"]`),
},
},
} {
t.Run(tc.name, func(t *testing.T) {
// Send test input.
select {
case notifyCh <- tc.notify:
case <-errs:
t.Fatal("keepStateKeysUpdated returned before test input")
case <-time.After(5 * time.Second):
t.Fatal("timed out waiting for next() to be called again")
}
// Wait for next() to be called again so we know the goroutine has
// processed the event.
select {
case <-nextWaiting:
case <-errs:
t.Fatal("keepStateKeysUpdated returned before test input")
case <-time.After(5 * time.Second):
t.Fatal("timed out waiting for next() to be called again")
}
for key, value := range tc.expected {
got, _ := store.ReadState(key)
if !bytes.Equal(got, value) {
t.Errorf("state key %q mismatch: expected %q, got %q", key, value, got)
}
}
})
}
}