mirror of
https://github.com/tailscale/tailscale.git
synced 2026-04-30 18:02:37 +02:00
85d6ba9473
This commit modifies the kubernetes operator to use the `tailscale-client-go-v2` package instead of the internal tailscale client it was previously using. This now gives us the ability to expand out custom resources and features as they become available via the API module. The tailnet reconciler has also been modified to manage clients as tailnets are created and removed, providing each subsequent reconciler with a single `ClientProvider` that obtains a tailscale client for the respective tailnet by name, or the operator's default when presented with a blank string. Fixes: https://github.com/tailscale/corp/issues/38418 Signed-off-by: David Bond <davidsbond93@gmail.com>
1016 lines
31 KiB
Go
1016 lines
31 KiB
Go
// Copyright (c) Tailscale Inc & contributors
|
|
// SPDX-License-Identifier: BSD-3-Clause
|
|
|
|
//go:build !plan9
|
|
|
|
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"maps"
|
|
"net/http"
|
|
"net/netip"
|
|
"path"
|
|
"reflect"
|
|
"slices"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/google/go-cmp/cmp"
|
|
"go.uber.org/zap"
|
|
appsv1 "k8s.io/api/apps/v1"
|
|
corev1 "k8s.io/api/core/v1"
|
|
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
|
"k8s.io/apimachinery/pkg/api/resource"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
|
"k8s.io/apimachinery/pkg/types"
|
|
"k8s.io/client-go/tools/record"
|
|
"sigs.k8s.io/controller-runtime/pkg/client"
|
|
"sigs.k8s.io/controller-runtime/pkg/reconcile"
|
|
"tailscale.com/client/tailscale/v2"
|
|
|
|
"tailscale.com/ipn"
|
|
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
|
|
"tailscale.com/k8s-operator/tsclient"
|
|
"tailscale.com/kube/kubetypes"
|
|
"tailscale.com/util/mak"
|
|
)
|
|
|
|
const (
|
|
vipTestIP = "5.6.7.8"
|
|
)
|
|
|
|
// confgOpts contains configuration options for creating cluster resources for
|
|
// Tailscale proxies.
|
|
type configOpts struct {
|
|
stsName string
|
|
secretName string
|
|
hostname string
|
|
namespace string
|
|
tailscaleNamespace string
|
|
namespaced bool
|
|
parentType string
|
|
proxyType string
|
|
priorityClassName string
|
|
firewallMode string
|
|
tailnetTargetIP string
|
|
tailnetTargetFQDN string
|
|
clusterTargetIP string
|
|
clusterTargetDNS string
|
|
subnetRoutes string
|
|
isExitNode bool
|
|
isAppConnector bool
|
|
serveConfig *ipn.ServeConfig
|
|
shouldEnableForwardingClusterTrafficViaIngress bool
|
|
proxyClass string // configuration from the named ProxyClass should be applied to proxy resources
|
|
app string
|
|
shouldRemoveAuthKey bool
|
|
secretExtraData map[string][]byte
|
|
resourceVersion string
|
|
replicas *int32
|
|
enableMetrics bool
|
|
serviceMonitorLabels tsapi.Labels
|
|
}
|
|
|
|
func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.StatefulSet {
|
|
t.Helper()
|
|
zl, err := zap.NewDevelopment()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
tsContainer := corev1.Container{
|
|
Name: "tailscale",
|
|
Image: "tailscale/tailscale",
|
|
Env: []corev1.EnvVar{
|
|
{Name: "TS_USERSPACE", Value: "false"},
|
|
{Name: "POD_IP", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "status.podIP"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}},
|
|
{Name: "POD_NAME", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "metadata.name"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}},
|
|
{Name: "POD_UID", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "metadata.uid"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}},
|
|
{Name: "TS_KUBE_SECRET", Value: "$(POD_NAME)"},
|
|
{Name: "TS_EXPERIMENTAL_SERVICE_AUTO_ADVERTISEMENT", Value: "false"},
|
|
{Name: "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR", Value: "/etc/tsconfig/$(POD_NAME)"},
|
|
{Name: "TS_DEBUG_ACME_FORCE_RENEWAL", Value: "true"},
|
|
},
|
|
SecurityContext: &corev1.SecurityContext{
|
|
Privileged: new(true),
|
|
},
|
|
Resources: corev1.ResourceRequirements{
|
|
Requests: corev1.ResourceList{
|
|
corev1.ResourceCPU: resource.MustParse("1m"),
|
|
corev1.ResourceMemory: resource.MustParse("1Mi"),
|
|
},
|
|
},
|
|
ImagePullPolicy: "Always",
|
|
}
|
|
if opts.shouldEnableForwardingClusterTrafficViaIngress {
|
|
tsContainer.Env = append(tsContainer.Env, corev1.EnvVar{
|
|
Name: "EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS",
|
|
Value: "true",
|
|
})
|
|
}
|
|
var annots map[string]string
|
|
var volumes []corev1.Volume
|
|
volumes = []corev1.Volume{
|
|
{
|
|
Name: "tailscaledconfig-0",
|
|
VolumeSource: corev1.VolumeSource{
|
|
Secret: &corev1.SecretVolumeSource{
|
|
SecretName: opts.secretName,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
tsContainer.VolumeMounts = []corev1.VolumeMount{{
|
|
Name: "tailscaledconfig-0",
|
|
ReadOnly: true,
|
|
MountPath: "/etc/tsconfig/" + opts.secretName,
|
|
}}
|
|
if opts.firewallMode != "" {
|
|
tsContainer.Env = append(tsContainer.Env, corev1.EnvVar{
|
|
Name: "TS_DEBUG_FIREWALL_MODE",
|
|
Value: opts.firewallMode,
|
|
})
|
|
}
|
|
if opts.tailnetTargetIP != "" {
|
|
mak.Set(&annots, "tailscale.com/operator-last-set-ts-tailnet-target-ip", opts.tailnetTargetIP)
|
|
tsContainer.Env = append(tsContainer.Env, corev1.EnvVar{
|
|
Name: "TS_TAILNET_TARGET_IP",
|
|
Value: opts.tailnetTargetIP,
|
|
})
|
|
} else if opts.tailnetTargetFQDN != "" {
|
|
mak.Set(&annots, "tailscale.com/operator-last-set-ts-tailnet-target-fqdn", opts.tailnetTargetFQDN)
|
|
tsContainer.Env = append(tsContainer.Env, corev1.EnvVar{
|
|
Name: "TS_TAILNET_TARGET_FQDN",
|
|
Value: opts.tailnetTargetFQDN,
|
|
})
|
|
|
|
} else if opts.clusterTargetIP != "" {
|
|
tsContainer.Env = append(tsContainer.Env, corev1.EnvVar{
|
|
Name: "TS_DEST_IP",
|
|
Value: opts.clusterTargetIP,
|
|
})
|
|
mak.Set(&annots, "tailscale.com/operator-last-set-cluster-ip", opts.clusterTargetIP)
|
|
} else if opts.clusterTargetDNS != "" {
|
|
tsContainer.Env = append(tsContainer.Env, corev1.EnvVar{
|
|
Name: "TS_EXPERIMENTAL_DEST_DNS_NAME",
|
|
Value: opts.clusterTargetDNS,
|
|
})
|
|
mak.Set(&annots, "tailscale.com/operator-last-set-cluster-dns-name", opts.clusterTargetDNS)
|
|
}
|
|
if opts.serveConfig != nil {
|
|
tsContainer.Env = append(tsContainer.Env, corev1.EnvVar{
|
|
Name: "TS_SERVE_CONFIG",
|
|
Value: "/etc/tailscaled/$(POD_NAME)/serve-config",
|
|
})
|
|
volumes = append(volumes, corev1.Volume{
|
|
Name: "serve-config-0",
|
|
VolumeSource: corev1.VolumeSource{
|
|
Secret: &corev1.SecretVolumeSource{
|
|
SecretName: opts.secretName,
|
|
Items: []corev1.KeyToPath{{
|
|
Key: "serve-config",
|
|
Path: "serve-config",
|
|
}},
|
|
},
|
|
},
|
|
})
|
|
tsContainer.VolumeMounts = append(tsContainer.VolumeMounts, corev1.VolumeMount{Name: "serve-config-0", ReadOnly: true, MountPath: path.Join("/etc/tailscaled", opts.secretName)})
|
|
}
|
|
tsContainer.Env = append(tsContainer.Env, corev1.EnvVar{
|
|
Name: "TS_INTERNAL_APP",
|
|
Value: opts.app,
|
|
})
|
|
if opts.enableMetrics {
|
|
tsContainer.Env = append(tsContainer.Env,
|
|
corev1.EnvVar{
|
|
Name: "TS_DEBUG_ADDR_PORT",
|
|
Value: "$(POD_IP):9001"},
|
|
corev1.EnvVar{
|
|
Name: "TS_TAILSCALED_EXTRA_ARGS",
|
|
Value: "--debug=$(TS_DEBUG_ADDR_PORT)",
|
|
},
|
|
corev1.EnvVar{
|
|
Name: "TS_LOCAL_ADDR_PORT",
|
|
Value: "$(POD_IP):9002",
|
|
},
|
|
corev1.EnvVar{
|
|
Name: "TS_ENABLE_METRICS",
|
|
Value: "true",
|
|
},
|
|
)
|
|
tsContainer.Ports = append(tsContainer.Ports,
|
|
corev1.ContainerPort{Name: "debug", ContainerPort: 9001, Protocol: "TCP"},
|
|
corev1.ContainerPort{Name: "metrics", ContainerPort: 9002, Protocol: "TCP"},
|
|
)
|
|
}
|
|
ss := &appsv1.StatefulSet{
|
|
TypeMeta: metav1.TypeMeta{
|
|
Kind: "StatefulSet",
|
|
APIVersion: "apps/v1",
|
|
},
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: opts.stsName,
|
|
Namespace: "operator-ns",
|
|
Labels: map[string]string{
|
|
"tailscale.com/managed": "true",
|
|
"tailscale.com/parent-resource": "test",
|
|
"tailscale.com/parent-resource-ns": opts.namespace,
|
|
"tailscale.com/parent-resource-type": opts.parentType,
|
|
},
|
|
},
|
|
Spec: appsv1.StatefulSetSpec{
|
|
Replicas: opts.replicas,
|
|
Selector: &metav1.LabelSelector{
|
|
MatchLabels: map[string]string{"app": "1234-UID"},
|
|
},
|
|
ServiceName: opts.stsName,
|
|
Template: corev1.PodTemplateSpec{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Annotations: annots,
|
|
DeletionGracePeriodSeconds: new(int64(10)),
|
|
Labels: map[string]string{
|
|
"tailscale.com/managed": "true",
|
|
"tailscale.com/parent-resource": "test",
|
|
"tailscale.com/parent-resource-ns": opts.namespace,
|
|
"tailscale.com/parent-resource-type": opts.parentType,
|
|
"app": "1234-UID",
|
|
},
|
|
},
|
|
Spec: corev1.PodSpec{
|
|
ServiceAccountName: "proxies",
|
|
PriorityClassName: opts.priorityClassName,
|
|
InitContainers: []corev1.Container{
|
|
{
|
|
Name: "sysctler",
|
|
Image: "tailscale/tailscale",
|
|
Command: []string{"/bin/sh", "-c"},
|
|
Args: []string{"sysctl -w net.ipv4.ip_forward=1 && if sysctl net.ipv6.conf.all.forwarding; then sysctl -w net.ipv6.conf.all.forwarding=1; fi"},
|
|
SecurityContext: &corev1.SecurityContext{
|
|
Privileged: new(true),
|
|
},
|
|
},
|
|
},
|
|
Containers: []corev1.Container{tsContainer},
|
|
Volumes: volumes,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
// If opts.proxyClass is set, retrieve the ProxyClass and apply
|
|
// configuration from that to the StatefulSet.
|
|
if opts.proxyClass != "" {
|
|
t.Logf("applying configuration from ProxyClass %s", opts.proxyClass)
|
|
proxyClass := new(tsapi.ProxyClass)
|
|
if err := cl.Get(context.Background(), types.NamespacedName{Name: opts.proxyClass}, proxyClass); err != nil {
|
|
t.Fatalf("error getting ProxyClass: %v", err)
|
|
}
|
|
return applyProxyClassToStatefulSet(proxyClass, ss, new(tailscaleSTSConfig), zl.Sugar())
|
|
}
|
|
return ss
|
|
}
|
|
|
|
func expectedSTSUserspace(t *testing.T, cl client.Client, opts configOpts) *appsv1.StatefulSet {
|
|
t.Helper()
|
|
zl, err := zap.NewDevelopment()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
tsContainer := corev1.Container{
|
|
Name: "tailscale",
|
|
Image: "tailscale/tailscale",
|
|
Env: []corev1.EnvVar{
|
|
{Name: "TS_USERSPACE", Value: "true"},
|
|
{Name: "POD_IP", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "status.podIP"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}},
|
|
{Name: "POD_NAME", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "metadata.name"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}},
|
|
{Name: "POD_UID", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "metadata.uid"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}},
|
|
{Name: "TS_KUBE_SECRET", Value: "$(POD_NAME)"},
|
|
{Name: "TS_EXPERIMENTAL_SERVICE_AUTO_ADVERTISEMENT", Value: "false"},
|
|
{Name: "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR", Value: "/etc/tsconfig/$(POD_NAME)"},
|
|
{Name: "TS_DEBUG_ACME_FORCE_RENEWAL", Value: "true"},
|
|
{Name: "TS_SERVE_CONFIG", Value: "/etc/tailscaled/$(POD_NAME)/serve-config"},
|
|
{Name: "TS_INTERNAL_APP", Value: opts.app},
|
|
},
|
|
ImagePullPolicy: "Always",
|
|
VolumeMounts: []corev1.VolumeMount{
|
|
{Name: "tailscaledconfig-0", ReadOnly: true, MountPath: path.Join("/etc/tsconfig", opts.secretName)},
|
|
{Name: "serve-config-0", ReadOnly: true, MountPath: path.Join("/etc/tailscaled", opts.secretName)},
|
|
},
|
|
Resources: corev1.ResourceRequirements{
|
|
Requests: corev1.ResourceList{
|
|
corev1.ResourceCPU: resource.MustParse("1m"),
|
|
corev1.ResourceMemory: resource.MustParse("1Mi"),
|
|
},
|
|
},
|
|
}
|
|
if opts.enableMetrics {
|
|
tsContainer.Env = append(tsContainer.Env,
|
|
corev1.EnvVar{
|
|
Name: "TS_DEBUG_ADDR_PORT",
|
|
Value: "$(POD_IP):9001"},
|
|
corev1.EnvVar{
|
|
Name: "TS_TAILSCALED_EXTRA_ARGS",
|
|
Value: "--debug=$(TS_DEBUG_ADDR_PORT)",
|
|
},
|
|
corev1.EnvVar{
|
|
Name: "TS_LOCAL_ADDR_PORT",
|
|
Value: "$(POD_IP):9002",
|
|
},
|
|
corev1.EnvVar{
|
|
Name: "TS_ENABLE_METRICS",
|
|
Value: "true",
|
|
},
|
|
)
|
|
tsContainer.Ports = append(tsContainer.Ports, corev1.ContainerPort{
|
|
Name: "debug", ContainerPort: 9001, Protocol: "TCP"},
|
|
corev1.ContainerPort{Name: "metrics", ContainerPort: 9002, Protocol: "TCP"},
|
|
)
|
|
}
|
|
volumes := []corev1.Volume{
|
|
{
|
|
Name: "tailscaledconfig-0",
|
|
VolumeSource: corev1.VolumeSource{
|
|
Secret: &corev1.SecretVolumeSource{
|
|
SecretName: opts.secretName,
|
|
},
|
|
},
|
|
},
|
|
{
|
|
Name: "serve-config-0",
|
|
VolumeSource: corev1.VolumeSource{
|
|
Secret: &corev1.SecretVolumeSource{
|
|
SecretName: opts.secretName,
|
|
Items: []corev1.KeyToPath{{Key: "serve-config", Path: "serve-config"}},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
ss := &appsv1.StatefulSet{
|
|
TypeMeta: metav1.TypeMeta{
|
|
Kind: "StatefulSet",
|
|
APIVersion: "apps/v1",
|
|
},
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: opts.stsName,
|
|
Namespace: "operator-ns",
|
|
Labels: map[string]string{
|
|
"tailscale.com/managed": "true",
|
|
"tailscale.com/parent-resource": "test",
|
|
"tailscale.com/parent-resource-ns": opts.namespace,
|
|
"tailscale.com/parent-resource-type": opts.parentType,
|
|
},
|
|
},
|
|
Spec: appsv1.StatefulSetSpec{
|
|
Replicas: new(int32(1)),
|
|
Selector: &metav1.LabelSelector{
|
|
MatchLabels: map[string]string{"app": "1234-UID"},
|
|
},
|
|
ServiceName: opts.stsName,
|
|
Template: corev1.PodTemplateSpec{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
DeletionGracePeriodSeconds: new(int64(10)),
|
|
Labels: map[string]string{
|
|
"tailscale.com/managed": "true",
|
|
"tailscale.com/parent-resource": "test",
|
|
"tailscale.com/parent-resource-ns": opts.namespace,
|
|
"tailscale.com/parent-resource-type": opts.parentType,
|
|
"app": "1234-UID",
|
|
},
|
|
},
|
|
Spec: corev1.PodSpec{
|
|
ServiceAccountName: "proxies",
|
|
PriorityClassName: opts.priorityClassName,
|
|
Containers: []corev1.Container{tsContainer},
|
|
Volumes: volumes,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
// If opts.proxyClass is set, retrieve the ProxyClass and apply
|
|
// configuration from that to the StatefulSet.
|
|
if opts.proxyClass != "" {
|
|
t.Logf("applying configuration from ProxyClass %s", opts.proxyClass)
|
|
proxyClass := new(tsapi.ProxyClass)
|
|
if err := cl.Get(context.Background(), types.NamespacedName{Name: opts.proxyClass}, proxyClass); err != nil {
|
|
t.Fatalf("error getting ProxyClass: %v", err)
|
|
}
|
|
return applyProxyClassToStatefulSet(proxyClass, ss, new(tailscaleSTSConfig), zl.Sugar())
|
|
}
|
|
return ss
|
|
}
|
|
|
|
func expectedHeadlessService(name string, parentType string) *corev1.Service {
|
|
return &corev1.Service{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: name,
|
|
GenerateName: "ts-test-",
|
|
Namespace: "operator-ns",
|
|
Labels: map[string]string{
|
|
"tailscale.com/managed": "true",
|
|
"tailscale.com/parent-resource": "test",
|
|
"tailscale.com/parent-resource-ns": "default",
|
|
"tailscale.com/parent-resource-type": parentType,
|
|
},
|
|
},
|
|
Spec: corev1.ServiceSpec{
|
|
Selector: map[string]string{
|
|
"app": "1234-UID",
|
|
},
|
|
ClusterIP: "None",
|
|
IPFamilyPolicy: new(corev1.IPFamilyPolicyPreferDualStack),
|
|
},
|
|
}
|
|
}
|
|
|
|
func expectedMetricsService(opts configOpts) *corev1.Service {
|
|
labels := metricsLabels(opts)
|
|
selector := map[string]string{
|
|
"tailscale.com/managed": "true",
|
|
"tailscale.com/parent-resource": "test",
|
|
"tailscale.com/parent-resource-type": opts.parentType,
|
|
}
|
|
if opts.namespaced {
|
|
selector["tailscale.com/parent-resource-ns"] = opts.namespace
|
|
}
|
|
return &corev1.Service{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: metricsResourceName(opts.stsName),
|
|
Namespace: opts.tailscaleNamespace,
|
|
Labels: labels,
|
|
},
|
|
Spec: corev1.ServiceSpec{
|
|
Selector: selector,
|
|
Type: corev1.ServiceTypeClusterIP,
|
|
Ports: []corev1.ServicePort{{Protocol: "TCP", Port: 9002, Name: "metrics"}},
|
|
},
|
|
}
|
|
}
|
|
|
|
func metricsLabels(opts configOpts) map[string]string {
|
|
promJob := fmt.Sprintf("ts_%s_default_test", opts.proxyType)
|
|
if !opts.namespaced {
|
|
promJob = fmt.Sprintf("ts_%s_test", opts.proxyType)
|
|
}
|
|
labels := map[string]string{
|
|
"tailscale.com/managed": "true",
|
|
"tailscale.com/metrics-target": opts.stsName,
|
|
"ts_prom_job": promJob,
|
|
"ts_proxy_type": opts.proxyType,
|
|
"ts_proxy_parent_name": "test",
|
|
}
|
|
if opts.namespaced {
|
|
labels["ts_proxy_parent_namespace"] = "default"
|
|
}
|
|
return labels
|
|
}
|
|
|
|
func expectedServiceMonitor(t *testing.T, opts configOpts) *unstructured.Unstructured {
|
|
t.Helper()
|
|
smLabels := metricsLabels(opts)
|
|
if len(opts.serviceMonitorLabels) != 0 {
|
|
smLabels = mergeMapKeys(smLabels, opts.serviceMonitorLabels.Parse())
|
|
}
|
|
name := metricsResourceName(opts.stsName)
|
|
sm := &ServiceMonitor{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: name,
|
|
Namespace: opts.tailscaleNamespace,
|
|
Labels: smLabels,
|
|
ResourceVersion: opts.resourceVersion,
|
|
OwnerReferences: []metav1.OwnerReference{{APIVersion: "v1", Kind: "Service", Name: name, BlockOwnerDeletion: new(true), Controller: new(true)}},
|
|
},
|
|
TypeMeta: metav1.TypeMeta{
|
|
Kind: "ServiceMonitor",
|
|
APIVersion: "monitoring.coreos.com/v1",
|
|
},
|
|
Spec: ServiceMonitorSpec{
|
|
Selector: metav1.LabelSelector{MatchLabels: metricsLabels(opts)},
|
|
Endpoints: []ServiceMonitorEndpoint{{
|
|
Port: "metrics",
|
|
}},
|
|
NamespaceSelector: ServiceMonitorNamespaceSelector{
|
|
MatchNames: []string{opts.tailscaleNamespace},
|
|
},
|
|
JobLabel: "ts_prom_job",
|
|
TargetLabels: []string{
|
|
"ts_proxy_parent_name",
|
|
"ts_proxy_parent_namespace",
|
|
"ts_proxy_type",
|
|
},
|
|
},
|
|
}
|
|
u, err := serviceMonitorToUnstructured(sm)
|
|
if err != nil {
|
|
t.Fatalf("error converting ServiceMonitor to unstructured: %v", err)
|
|
}
|
|
return u
|
|
}
|
|
|
|
func expectedSecret(t *testing.T, cl client.Client, opts configOpts) *corev1.Secret {
|
|
t.Helper()
|
|
s := &corev1.Secret{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: opts.secretName,
|
|
Namespace: "operator-ns",
|
|
},
|
|
}
|
|
if opts.serveConfig != nil {
|
|
serveConfigBs, err := json.Marshal(opts.serveConfig)
|
|
if err != nil {
|
|
t.Fatalf("error marshalling serve config: %v", err)
|
|
}
|
|
mak.Set(&s.StringData, "serve-config", string(serveConfigBs))
|
|
}
|
|
conf := &ipn.ConfigVAlpha{
|
|
Version: "alpha0",
|
|
AcceptDNS: "false",
|
|
Hostname: &opts.hostname,
|
|
Locked: "false",
|
|
AuthKey: new("new-authkey"),
|
|
AcceptRoutes: "false",
|
|
AppConnector: &ipn.AppConnectorPrefs{Advertise: false},
|
|
NoStatefulFiltering: "true",
|
|
}
|
|
if opts.proxyClass != "" {
|
|
t.Logf("applying configuration from ProxyClass %s", opts.proxyClass)
|
|
proxyClass := new(tsapi.ProxyClass)
|
|
if err := cl.Get(context.Background(), types.NamespacedName{Name: opts.proxyClass}, proxyClass); err != nil {
|
|
t.Fatalf("error getting ProxyClass: %v", err)
|
|
}
|
|
if proxyClass.Spec.TailscaleConfig != nil && proxyClass.Spec.TailscaleConfig.AcceptRoutes {
|
|
conf.AcceptRoutes = "true"
|
|
}
|
|
}
|
|
if opts.shouldRemoveAuthKey {
|
|
conf.AuthKey = nil
|
|
}
|
|
if opts.isAppConnector {
|
|
conf.AppConnector = &ipn.AppConnectorPrefs{Advertise: true}
|
|
}
|
|
var routes []netip.Prefix
|
|
if opts.subnetRoutes != "" || opts.isExitNode {
|
|
r := opts.subnetRoutes
|
|
if opts.isExitNode {
|
|
r = "0.0.0.0/0,::/0," + r
|
|
}
|
|
for rr := range strings.SplitSeq(r, ",") {
|
|
prefix, err := netip.ParsePrefix(rr)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
routes = append(routes, prefix)
|
|
}
|
|
}
|
|
conf.AdvertiseRoutes = routes
|
|
bnn, err := json.Marshal(conf)
|
|
if err != nil {
|
|
t.Fatalf("error marshalling tailscaled config")
|
|
}
|
|
conf.AppConnector = nil
|
|
bn, err := json.Marshal(conf)
|
|
if err != nil {
|
|
t.Fatalf("error marshalling tailscaled config")
|
|
}
|
|
mak.Set(&s.StringData, "cap-95.hujson", string(bn))
|
|
mak.Set(&s.StringData, "cap-107.hujson", string(bnn))
|
|
labels := map[string]string{
|
|
"tailscale.com/managed": "true",
|
|
"tailscale.com/parent-resource": "test",
|
|
"tailscale.com/parent-resource-ns": "default",
|
|
"tailscale.com/parent-resource-type": opts.parentType,
|
|
}
|
|
if opts.parentType == "connector" {
|
|
labels["tailscale.com/parent-resource-ns"] = "" // Connector is cluster scoped
|
|
}
|
|
s.Labels = labels
|
|
for key, val := range opts.secretExtraData {
|
|
mak.Set(&s.Data, key, val)
|
|
}
|
|
return s
|
|
}
|
|
|
|
func findNoGenName(t *testing.T, client client.Client, ns, name, typ string) {
|
|
t.Helper()
|
|
labels := map[string]string{
|
|
kubetypes.LabelManaged: "true",
|
|
LabelParentName: name,
|
|
LabelParentNamespace: ns,
|
|
LabelParentType: typ,
|
|
}
|
|
s, err := getSingleObject[corev1.Secret](context.Background(), client, "operator-ns", labels)
|
|
if err != nil {
|
|
t.Fatalf("finding secrets for %q: %v", name, err)
|
|
}
|
|
if s != nil {
|
|
t.Fatalf("found unexpected secret with name %q", s.GetName())
|
|
}
|
|
}
|
|
|
|
func findGenName(t *testing.T, client client.Client, ns, name, typ string) (full, noSuffix string) {
|
|
t.Helper()
|
|
labels := map[string]string{
|
|
kubetypes.LabelManaged: "true",
|
|
LabelParentName: name,
|
|
LabelParentNamespace: ns,
|
|
LabelParentType: typ,
|
|
}
|
|
s, err := getSingleObject[corev1.Secret](context.Background(), client, "operator-ns", labels)
|
|
if err != nil {
|
|
t.Fatalf("finding secret for %q: %v", name, err)
|
|
}
|
|
if s == nil {
|
|
t.Fatalf("no secret found for %q %s %+#v", name, ns, labels)
|
|
}
|
|
return s.GetName(), strings.TrimSuffix(s.GetName(), "-0")
|
|
}
|
|
|
|
func findGenNames(t *testing.T, cl client.Client, ns, name, typ string) []string {
|
|
t.Helper()
|
|
labels := map[string]string{
|
|
kubetypes.LabelManaged: "true",
|
|
LabelParentName: name,
|
|
LabelParentNamespace: ns,
|
|
LabelParentType: typ,
|
|
}
|
|
|
|
var list corev1.SecretList
|
|
if err := cl.List(t.Context(), &list, client.InNamespace(ns), client.MatchingLabels(labels)); err != nil {
|
|
t.Fatalf("finding secrets for %q: %v", name, err)
|
|
}
|
|
|
|
if len(list.Items) == 0 {
|
|
t.Fatalf("no secrets found for %q %s %+#v", name, ns, labels)
|
|
}
|
|
|
|
names := make([]string, len(list.Items))
|
|
for i, secret := range list.Items {
|
|
names[i] = secret.GetName()
|
|
}
|
|
|
|
return names
|
|
}
|
|
|
|
func mustCreate(t *testing.T, client client.Client, obj client.Object) {
|
|
t.Helper()
|
|
if err := client.Create(context.Background(), obj); err != nil {
|
|
t.Fatalf("creating %q: %v", obj.GetName(), err)
|
|
}
|
|
}
|
|
func mustCreateAll(t *testing.T, client client.Client, objs ...client.Object) {
|
|
t.Helper()
|
|
for _, obj := range objs {
|
|
mustCreate(t, client, obj)
|
|
}
|
|
}
|
|
|
|
func mustDeleteAll(t *testing.T, client client.Client, objs ...client.Object) {
|
|
t.Helper()
|
|
for _, obj := range objs {
|
|
if err := client.Delete(context.Background(), obj); err != nil {
|
|
t.Fatalf("deleting %q: %v", obj.GetName(), err)
|
|
}
|
|
}
|
|
}
|
|
|
|
func mustUpdate[T any, O ptrObject[T]](t *testing.T, client client.Client, ns, name string, update func(O)) {
|
|
t.Helper()
|
|
obj := O(new(T))
|
|
if err := client.Get(context.Background(), types.NamespacedName{
|
|
Name: name,
|
|
Namespace: ns,
|
|
}, obj); err != nil {
|
|
t.Fatalf("getting %q: %v", name, err)
|
|
}
|
|
update(obj)
|
|
if err := client.Update(context.Background(), obj); err != nil {
|
|
t.Fatalf("updating %q: %v", name, err)
|
|
}
|
|
}
|
|
|
|
func mustUpdateStatus[T any, O ptrObject[T]](t *testing.T, client client.Client, ns, name string, update func(O)) {
|
|
t.Helper()
|
|
obj := O(new(T))
|
|
if err := client.Get(context.Background(), types.NamespacedName{
|
|
Name: name,
|
|
Namespace: ns,
|
|
}, obj); err != nil {
|
|
t.Fatalf("getting %q: %v", name, err)
|
|
}
|
|
update(obj)
|
|
if err := client.Status().Update(context.Background(), obj); err != nil {
|
|
t.Fatalf("updating %q: %v", name, err)
|
|
}
|
|
}
|
|
|
|
// expectEqual accepts a Kubernetes object and a Kubernetes client. It tests
|
|
// whether an object with equivalent contents can be retrieved by the passed
|
|
// client. If you want to NOT test some object fields for equality, use the
|
|
// modify func to ensure that they are removed from the cluster object and the
|
|
// object passed as 'want'. If no such modifications are needed, you can pass
|
|
// nil in place of the modify function.
|
|
func expectEqual[T any, O ptrObject[T]](t *testing.T, client client.Client, want O, modifiers ...func(O)) {
|
|
t.Helper()
|
|
got := O(new(T))
|
|
if err := client.Get(context.Background(), types.NamespacedName{
|
|
Name: want.GetName(),
|
|
Namespace: want.GetNamespace(),
|
|
}, got); err != nil {
|
|
t.Fatalf("getting %q: %v", want.GetName(), err)
|
|
}
|
|
// The resource version changes eagerly whenever the operator does even a
|
|
// no-op update. Asserting a specific value leads to overly brittle tests,
|
|
// so just remove it from both got and want.
|
|
got.SetResourceVersion("")
|
|
want.SetResourceVersion("")
|
|
for _, modifier := range modifiers {
|
|
modifier(want)
|
|
modifier(got)
|
|
}
|
|
if diff := cmp.Diff(got, want); diff != "" {
|
|
t.Fatalf("unexpected %s (-got +want):\n%s", reflect.TypeOf(want).Elem().Name(), diff)
|
|
}
|
|
}
|
|
|
|
func expectEqualUnstructured(t *testing.T, client client.Client, want *unstructured.Unstructured) {
|
|
t.Helper()
|
|
got := &unstructured.Unstructured{}
|
|
got.SetGroupVersionKind(want.GroupVersionKind())
|
|
if err := client.Get(context.Background(), types.NamespacedName{
|
|
Name: want.GetName(),
|
|
Namespace: want.GetNamespace(),
|
|
}, got); err != nil {
|
|
t.Fatalf("getting %q: %v", want.GetName(), err)
|
|
}
|
|
if diff := cmp.Diff(got, want); diff != "" {
|
|
t.Fatalf("unexpected contents of Unstructured (-got +want):\n%s", diff)
|
|
}
|
|
}
|
|
|
|
func expectMissing[T any, O ptrObject[T]](t *testing.T, client client.Client, ns, name string) {
|
|
t.Helper()
|
|
obj := O(new(T))
|
|
err := client.Get(context.Background(), types.NamespacedName{
|
|
Name: name,
|
|
Namespace: ns,
|
|
}, obj)
|
|
if !apierrors.IsNotFound(err) {
|
|
t.Fatalf("%s %s/%s unexpectedly present, wanted missing", reflect.TypeOf(obj).Elem().Name(), ns, name)
|
|
}
|
|
}
|
|
|
|
func expectReconciled(t *testing.T, sr reconcile.Reconciler, ns, name string) {
|
|
t.Helper()
|
|
req := reconcile.Request{
|
|
NamespacedName: types.NamespacedName{
|
|
Namespace: ns,
|
|
Name: name,
|
|
},
|
|
}
|
|
res, err := sr.Reconcile(context.Background(), req)
|
|
if err != nil {
|
|
t.Fatalf("Reconcile: unexpected error: %v", err)
|
|
}
|
|
if res.Requeue {
|
|
t.Fatalf("unexpected immediate requeue")
|
|
}
|
|
if res.RequeueAfter != 0 {
|
|
t.Fatalf("unexpected timed requeue (%v)", res.RequeueAfter)
|
|
}
|
|
}
|
|
|
|
func expectRequeue(t *testing.T, sr reconcile.Reconciler, ns, name string) {
|
|
t.Helper()
|
|
req := reconcile.Request{
|
|
NamespacedName: types.NamespacedName{
|
|
Name: name,
|
|
Namespace: ns,
|
|
},
|
|
}
|
|
res, err := sr.Reconcile(context.Background(), req)
|
|
if err != nil {
|
|
t.Fatalf("Reconcile: unexpected error: %v", err)
|
|
}
|
|
if res.RequeueAfter == 0 {
|
|
t.Fatalf("expected timed requeue, got success")
|
|
}
|
|
}
|
|
func expectError(t *testing.T, sr reconcile.Reconciler, ns, name string) {
|
|
t.Helper()
|
|
req := reconcile.Request{
|
|
NamespacedName: types.NamespacedName{
|
|
Name: name,
|
|
Namespace: ns,
|
|
},
|
|
}
|
|
_, err := sr.Reconcile(context.Background(), req)
|
|
if err == nil {
|
|
t.Error("Reconcile: expected error but did not get one")
|
|
}
|
|
}
|
|
|
|
// expectEvents accepts a test recorder and a list of events, tests that expected
|
|
// events are sent down the recorder's channel. Waits for 5s for each event.
|
|
func expectEvents(t *testing.T, rec *record.FakeRecorder, wantsEvents []string) {
|
|
t.Helper()
|
|
// Events are not expected to arrive in order.
|
|
seenEvents := make([]string, 0)
|
|
for range len(wantsEvents) {
|
|
timer := time.NewTimer(time.Second * 5)
|
|
defer timer.Stop()
|
|
select {
|
|
case gotEvent := <-rec.Events:
|
|
found := false
|
|
if slices.Contains(wantsEvents, gotEvent) {
|
|
found = true
|
|
seenEvents = append(seenEvents, gotEvent)
|
|
}
|
|
if !found {
|
|
t.Errorf("got unexpected event %q, expected events: %+#v", gotEvent, wantsEvents)
|
|
}
|
|
case <-timer.C:
|
|
t.Errorf("timeout waiting for an event, wants events %#+v, got events %+#v", wantsEvents, seenEvents)
|
|
}
|
|
}
|
|
}
|
|
|
|
type (
|
|
fakeTSClient struct {
|
|
sync.Mutex
|
|
loginURL string
|
|
keyRequests []tailscale.KeyCapabilities
|
|
deleted []string
|
|
devices []tailscale.Device
|
|
vipServices map[string]tailscale.VIPService
|
|
}
|
|
|
|
fakeVIPServices struct {
|
|
mu sync.RWMutex
|
|
vipServices map[string]tailscale.VIPService
|
|
}
|
|
|
|
fakeKeys struct {
|
|
keyRequests *[]tailscale.KeyCapabilities
|
|
}
|
|
|
|
fakeDevices struct {
|
|
deleted *[]string
|
|
devices *[]tailscale.Device
|
|
}
|
|
)
|
|
|
|
func (c *fakeTSClient) VIPServices() tsclient.VIPServiceResource {
|
|
return &fakeVIPServices{
|
|
vipServices: c.vipServices,
|
|
}
|
|
}
|
|
|
|
func (m *fakeVIPServices) List(_ context.Context) ([]tailscale.VIPService, error) {
|
|
m.mu.RLock()
|
|
defer m.mu.RUnlock()
|
|
|
|
if len(m.vipServices) == 0 {
|
|
return nil, tailscale.APIError{Status: http.StatusNotFound}
|
|
}
|
|
|
|
return slices.Collect(maps.Values(m.vipServices)), nil
|
|
}
|
|
|
|
func (m *fakeVIPServices) Delete(_ context.Context, name string) error {
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
|
|
if _, ok := m.vipServices[name]; !ok {
|
|
return tailscale.APIError{Status: http.StatusNotFound}
|
|
}
|
|
|
|
delete(m.vipServices, name)
|
|
return nil
|
|
}
|
|
|
|
func (m *fakeVIPServices) Get(_ context.Context, name string) (*tailscale.VIPService, error) {
|
|
if svc, ok := m.vipServices[name]; ok {
|
|
return &svc, nil
|
|
}
|
|
|
|
return nil, tailscale.APIError{Status: http.StatusNotFound}
|
|
}
|
|
|
|
func (m *fakeVIPServices) CreateOrUpdate(_ context.Context, svc tailscale.VIPService) error {
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
|
|
if svc.Addrs == nil {
|
|
svc.Addrs = []string{vipTestIP}
|
|
}
|
|
|
|
m.vipServices[svc.Name] = svc
|
|
return nil
|
|
}
|
|
|
|
func (c *fakeTSClient) Devices() tsclient.DeviceResource {
|
|
return &fakeDevices{
|
|
deleted: &c.deleted,
|
|
devices: &c.devices,
|
|
}
|
|
}
|
|
|
|
func (m *fakeDevices) Delete(_ context.Context, id string) error {
|
|
*m.deleted = append(*m.deleted, id)
|
|
|
|
return tailscale.APIError{Status: http.StatusNotFound}
|
|
}
|
|
|
|
func (m *fakeDevices) List(_ context.Context, _ ...tailscale.ListDevicesOptions) ([]tailscale.Device, error) {
|
|
return *m.devices, nil
|
|
}
|
|
|
|
func (m *fakeDevices) Get(_ context.Context, id string) (*tailscale.Device, error) {
|
|
if m.devices == nil {
|
|
return nil, tailscale.APIError{Status: http.StatusNotFound}
|
|
}
|
|
|
|
for _, dev := range *m.devices {
|
|
if dev.ID == id {
|
|
return &dev, nil
|
|
}
|
|
}
|
|
|
|
return nil, tailscale.APIError{Status: http.StatusNotFound}
|
|
}
|
|
|
|
func (c *fakeTSClient) Keys() tsclient.KeyResource {
|
|
return &fakeKeys{
|
|
keyRequests: &c.keyRequests,
|
|
}
|
|
}
|
|
|
|
func (m *fakeKeys) CreateAuthKey(_ context.Context, ckr tailscale.CreateKeyRequest) (*tailscale.Key, error) {
|
|
*m.keyRequests = append(*m.keyRequests, ckr.Capabilities)
|
|
|
|
return &tailscale.Key{Key: "new-authkey"}, nil
|
|
}
|
|
|
|
func (m *fakeKeys) List(_ context.Context, _ bool) ([]tailscale.Key, error) {
|
|
return nil, nil
|
|
}
|
|
|
|
func (c *fakeTSClient) LoginURL() string {
|
|
return c.loginURL
|
|
}
|
|
|
|
type fakeTSNetServer struct {
|
|
certDomains []string
|
|
}
|
|
|
|
func (f *fakeTSNetServer) CertDomains() []string {
|
|
return f.certDomains
|
|
}
|
|
|
|
func removeResourceReqs(sts *appsv1.StatefulSet) {
|
|
if sts != nil {
|
|
sts.Spec.Template.Spec.Resources = nil
|
|
}
|
|
}
|
|
|
|
func removeTargetPortsFromSvc(svc *corev1.Service) {
|
|
newPorts := make([]corev1.ServicePort, 0)
|
|
for _, p := range svc.Spec.Ports {
|
|
newPorts = append(newPorts, corev1.ServicePort{Protocol: p.Protocol, Port: p.Port, Name: p.Name})
|
|
}
|
|
svc.Spec.Ports = newPorts
|
|
}
|
|
|
|
func removeAuthKeyIfExistsModifier(t *testing.T) func(s *corev1.Secret) {
|
|
return func(secret *corev1.Secret) {
|
|
t.Helper()
|
|
if len(secret.StringData["cap-95.hujson"]) != 0 {
|
|
conf := &ipn.ConfigVAlpha{}
|
|
if err := json.Unmarshal([]byte(secret.StringData["cap-95.hujson"]), conf); err != nil {
|
|
t.Fatalf("error umarshalling 'cap-95.hujson' contents: %v", err)
|
|
}
|
|
conf.AuthKey = nil
|
|
b, err := json.Marshal(conf)
|
|
if err != nil {
|
|
t.Fatalf("error marshalling 'cap-95.huson' contents: %v", err)
|
|
}
|
|
mak.Set(&secret.StringData, "cap-95.hujson", string(b))
|
|
}
|
|
if len(secret.StringData["cap-107.hujson"]) != 0 {
|
|
conf := &ipn.ConfigVAlpha{}
|
|
if err := json.Unmarshal([]byte(secret.StringData["cap-107.hujson"]), conf); err != nil {
|
|
t.Fatalf("error umarshalling 'cap-107.hujson' contents: %v", err)
|
|
}
|
|
conf.AuthKey = nil
|
|
b, err := json.Marshal(conf)
|
|
if err != nil {
|
|
t.Fatalf("error marshalling 'cap-107.huson' contents: %v", err)
|
|
}
|
|
mak.Set(&secret.StringData, "cap-107.hujson", string(b))
|
|
}
|
|
}
|
|
}
|