// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package e2e import ( "context" "fmt" "net/http" "strings" "testing" "time" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" "tailscale.com/client/tailscale/v2" kube "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/kube/kubetypes" "tailscale.com/tsnet" "tailscale.com/tstest" "tailscale.com/util/httpm" ) // See [TestMain] for test requirements. func TestL3Ingress(t *testing.T) { if tnClient == nil { t.Skip("TestL3Ingress requires a working tailnet client") } // Apply nginx createAndCleanup(t, kubeClient, nginxDeployment(ns, "nginx")) // Apply service to expose it as ingress name := generateName("test-ingress") svc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: ns, Annotations: map[string]string{ "tailscale.com/expose": "true", }, }, Spec: corev1.ServiceSpec{ Selector: map[string]string{ "app.kubernetes.io/name": "nginx", }, Ports: []corev1.ServicePort{ { Name: "http", Protocol: "TCP", Port: 80, }, }, }, } createAndCleanup(t, kubeClient, svc) if err := tstest.WaitFor(time.Minute, func() error { maybeReadySvc := &corev1.Service{ObjectMeta: objectMeta(ns, name)} if err := get(t.Context(), kubeClient, maybeReadySvc); err != nil { return err } isReady := kube.SvcIsReady(maybeReadySvc) if isReady { t.Log("Service is ready") return nil } return fmt.Errorf("Service is not ready yet") }); err != nil { t.Fatalf("error waiting for the Service to become Ready: %v", err) } // Get the DNS name for the Service from the associated Secret. var fqdn string if err := tstest.WaitFor(time.Minute, func() error { var secrets corev1.SecretList if err := kubeClient.List(t.Context(), &secrets, client.InNamespace("tailscale"), client.MatchingLabels{ "tailscale.com/parent-resource": name, "tailscale.com/parent-resource-ns": ns, }, ); err != nil { return err } if len(secrets.Items) == 0 { return fmt.Errorf("Service not ready yet") } fqdn = strings.TrimSuffix(string(secrets.Items[0].Data[kubetypes.KeyDeviceFQDN]), ".") if fqdn != "" { t.Log("Got DNS name for Service") return nil } return fmt.Errorf("device FQDN not set yet") }); err != nil { t.Fatalf("error waiting for DNS Name for Service: %v", err) } if err := testIngressIsReachable(t, newHTTPClient(tnClient), fmt.Sprintf("http://%s:80", fqdn)); err != nil { t.Fatal(err) } } func TestL3HAIngress(t *testing.T) { if tnClient == nil { t.Skip("TestL3HAIngress requires a working tailnet client") } // Apply nginx. createAndCleanup(t, kubeClient, nginxDeployment(ns, "nginx")) // Create an ingress ProxyGroup. createAndCleanup(t, kubeClient, &tsapi.ProxyGroup{ ObjectMeta: metav1.ObjectMeta{ Name: "ingress", }, Spec: tsapi.ProxyGroupSpec{ Type: tsapi.ProxyGroupTypeIngress, }, }) // Apply a Service to expose nginx via the ProxyGroup. name := generateName("test-ingress") svc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: ns, Annotations: map[string]string{ "tailscale.com/proxy-group": "ingress", }, }, Spec: corev1.ServiceSpec{ Type: corev1.ServiceTypeLoadBalancer, LoadBalancerClass: new("tailscale"), Selector: map[string]string{ "app.kubernetes.io/name": "nginx", }, Ports: []corev1.ServicePort{ { Name: "http", Protocol: "TCP", Port: 80, }, }, }, } createAndCleanup(t, kubeClient, svc) var svcIPv4 string forceReconcile := triggerReconcile(t, client.ObjectKey{Namespace: ns, Name: name}, &corev1.Service{}, 30*time.Second) // Wait for Service to be ready if err := tstest.WaitFor(5*time.Minute, func() error { maybeReadySvc := &corev1.Service{ObjectMeta: objectMeta(ns, name)} forceReconcile() if err := get(t.Context(), kubeClient, maybeReadySvc); err != nil { return err } for _, cond := range maybeReadySvc.Status.Conditions { if cond.Type == string(tsapi.IngressSvcConfigured) && cond.Status == metav1.ConditionTrue { if len(maybeReadySvc.Status.LoadBalancer.Ingress) == 0 { return fmt.Errorf("Service does not have an IP assigned yet") } svcIPv4 = maybeReadySvc.Status.LoadBalancer.Ingress[0].IP t.Log("Service is ready") return nil } } return fmt.Errorf("Service is not ready yet") }); err != nil { t.Fatalf("error waiting for the Service to become ready: %v", err) } if err := testIngressIsReachable(t, newHTTPClient(tnClient), fmt.Sprintf("http://%s:80", svcIPv4)); err != nil { t.Fatal(err) } } func TestL7Ingress(t *testing.T) { if tnClient == nil { t.Skip("TestL7Ingress requires a working tailnet client") } // Apply nginx Deployment and Service. createAndCleanup(t, kubeClient, nginxDeployment(ns, "nginx")) createAndCleanup(t, kubeClient, &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "nginx", Namespace: ns, }, Spec: corev1.ServiceSpec{ Selector: map[string]string{ "app.kubernetes.io/name": "nginx", }, Ports: []corev1.ServicePort{ { Name: "http", Port: 80, }, }, }, }) // Apply Ingress to expose nginx. name := generateName("test-ingress") ingress := l7Ingress(ns, name, map[string]string{}) createAndCleanup(t, kubeClient, ingress) t.Log("Waiting for the Ingress to be ready...") hostname, err := waitForIngressHostname(t, ns, name) if err != nil { t.Fatalf("error waiting for Ingress hostname: %v", err) } if err := testIngressIsReachable(t, newHTTPClient(tnClient), fmt.Sprintf("https://%s:443", hostname)); err != nil { t.Fatal(err) } } func TestL7HAIngress(t *testing.T) { if tnClient == nil { t.Skip("TestL7HAIngress requires a working tailnet client") } // Apply nginx Deployment and Service. createAndCleanup(t, kubeClient, nginxDeployment(ns, "nginx")) createAndCleanup(t, kubeClient, &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "nginx", Namespace: ns, }, Spec: corev1.ServiceSpec{ Selector: map[string]string{ "app.kubernetes.io/name": "nginx", }, Ports: []corev1.ServicePort{ { Name: "http", Port: 80, }, }, }, }) // Create ProxyGroup that the Ingress will reference. createAndCleanup(t, kubeClient, &tsapi.ProxyGroup{ ObjectMeta: metav1.ObjectMeta{ Name: "ingress", }, Spec: tsapi.ProxyGroupSpec{ Type: tsapi.ProxyGroupTypeIngress, }, }) // Apply Ingress to expose nginx. name := generateName("test-ingress") ingress := l7Ingress(ns, name, map[string]string{"tailscale.com/proxy-group": "ingress"}) createAndCleanup(t, kubeClient, ingress) t.Log("Waiting for the Ingress to be ready...") hostname, err := waitForIngressHostname(t, ns, name) if err != nil { t.Fatalf("error waiting for Ingress hostname: %v", err) } if err := testIngressIsReachable(t, newHTTPClient(tnClient), fmt.Sprintf("https://%s:443", hostname)); err != nil { t.Fatal(err) } } func TestL7HAIngressMultiTailnet(t *testing.T) { if tnClient == nil || secondTNClient == nil { t.Skip("TestL7HAMultiTailnet requires a working tailnet client for a first and second tailnet") } // Apply nginx Deployment and Service. createAndCleanup(t, kubeClient, nginxDeployment(ns, "nginx")) createAndCleanup(t, kubeClient, &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "nginx", Namespace: ns, }, Spec: corev1.ServiceSpec{ Selector: map[string]string{ "app.kubernetes.io/name": "nginx", }, Ports: []corev1.ServicePort{ { Name: "http", Port: 80, }, }, }, }) // Create Ingress ProxyGroup for each Tailnet. firstTailnetPG := &tsapi.ProxyGroup{ ObjectMeta: metav1.ObjectMeta{ Name: "first-tailnet", }, Spec: tsapi.ProxyGroupSpec{ Type: tsapi.ProxyGroupTypeIngress, }, } createAndCleanup(t, kubeClient, firstTailnetPG) secondTailnetPG := &tsapi.ProxyGroup{ ObjectMeta: metav1.ObjectMeta{ Name: "second-tailnet", }, Spec: tsapi.ProxyGroupSpec{ Type: tsapi.ProxyGroupTypeIngress, Tailnet: "second-tailnet", }, } createAndCleanup(t, kubeClient, secondTailnetPG) if err := verifyProxyGroupTailnet(t, firstTailnetPG, tnClient); err != nil { t.Fatalf("verifying ProxyGroup %s is registered to the correct tailnet: %v", firstTailnetPG.Name, err) } if err := verifyProxyGroupTailnet(t, secondTailnetPG, secondTNClient); err != nil { t.Fatalf("verifying ProxyGroup %s is registered to the correct tailnet: %v", secondTailnetPG.Name, err) } // Apply Ingress to expose nginx. name := generateName("test-ingress") ingress := l7Ingress(ns, name, map[string]string{ "tailscale.com/proxy-group": "second-tailnet", }) createAndCleanup(t, kubeClient, ingress) // Check that the tailscale (VIP) Service has been created in the expected Tailnet. svcName := "svc:" + name if err := tstest.WaitFor(3*time.Minute, func() error { _, err := secondTSClient.VIPServices().Get(t.Context(), svcName) if tailscale.IsNotFound(err) { return fmt.Errorf("Tailscale service %q not yet in expected tailnet", svcName) } return err }); err != nil { t.Fatalf("Tailscale service %q never appeared in expected tailnet: %v", svcName, err) } hostname, err := waitForIngressHostname(t, ns, name) if err != nil { t.Fatalf("error waiting for Ingress hostname: %v", err) } if err := testIngressIsReachable(t, newHTTPClient(secondTNClient), fmt.Sprintf("https://%s:443", hostname)); err != nil { t.Fatal(err) } } func l7Ingress(namespace, name string, annotations map[string]string) *networkingv1.Ingress { ingress := &networkingv1.Ingress{ ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: namespace, Annotations: annotations, }, Spec: networkingv1.IngressSpec{ IngressClassName: new("tailscale"), TLS: []networkingv1.IngressTLS{ {Hosts: []string{name}}, }, Rules: []networkingv1.IngressRule{ { IngressRuleValue: networkingv1.IngressRuleValue{ HTTP: &networkingv1.HTTPIngressRuleValue{ Paths: []networkingv1.HTTPIngressPath{ { Path: "/", PathType: new(networkingv1.PathTypePrefix), Backend: networkingv1.IngressBackend{ Service: &networkingv1.IngressServiceBackend{ Name: "nginx", Port: networkingv1.ServiceBackendPort{ Number: 80, }, }, }, }, }, }, }, }, }, }, } return ingress } func nginxDeployment(namespace, name string) *appsv1.Deployment { return &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: namespace, Labels: map[string]string{ "app.kubernetes.io/name": "nginx", }, }, Spec: appsv1.DeploymentSpec{ Replicas: new(int32(1)), Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{ "app.kubernetes.io/name": "nginx", }, }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ "app.kubernetes.io/name": "nginx", }, }, Spec: corev1.PodSpec{ Containers: []corev1.Container{ { Name: "nginx", Image: "nginx", }, }, }, }, }, } } // triggerReconcile triggers an expected reconcile for the given object if // none occurs. This is needed when running some tests against devcontrol, // where the final change that should trigger a reconcile does not always do so. // This has not been reproducible in a real tailnet environment, so a // workaround that runs only when using devcontrol is acceptable. func triggerReconcile(t testing.TB, key client.ObjectKey, obj client.Object, after time.Duration) func() { if !*fDevcontrol { return func() {} } triggerAt := time.Now().Add(after) var triggered bool return func() { if triggered || !time.Now().After(triggerAt) { return } if err := kubeClient.Get(t.Context(), key, obj); err != nil { t.Logf("failed to get %s: %v", key, err) return } ann := obj.GetAnnotations() if ann == nil { ann = map[string]string{} } ann["tailscale.com/trigger-reconcile"] = "true" obj.SetAnnotations(ann) if err := kubeClient.Update(t.Context(), obj); err != nil { t.Logf("failed to update %s: %v", key, err) return } triggered = true } } func testIngressIsReachable(t *testing.T, httpClient *http.Client, url string) error { t.Helper() var resp *http.Response if err := tstest.WaitFor(time.Minute, func() error { req, err := http.NewRequest(httpm.GET, url, nil) if err != nil { return err } ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) defer cancel() resp, err = httpClient.Do(req.WithContext(ctx)) if err != nil { return err } resp.Body.Close() return nil }); err != nil { return fmt.Errorf("error trying to reach %s: %w", url, err) } if resp.StatusCode != http.StatusOK { return fmt.Errorf("unexpected status from %s: %d", url, resp.StatusCode) } return nil } // verifyProxyGroupTailnet verifies that a ProxyGroup is registered to the correct tailnet. // This is done by getting the expected tailnet domain for the tailnet client, // and comparing this with the actual device fqdn in the ProxyGroup state secret. func verifyProxyGroupTailnet(t *testing.T, pg *tsapi.ProxyGroup, cl *tsnet.Server) error { t.Helper() // Determine the expected tailnet Magic DNS Name. lc, err := cl.LocalClient() if err != nil { return err } status, err := lc.Status(t.Context()) if err != nil { return err } _, expectedTailnet, ok := strings.Cut(strings.TrimSuffix(status.Self.DNSName, "."), ".") if !ok { return fmt.Errorf("unexpected DNSName format %q", status.Self.DNSName) } // Read the device FQDN from the first state secret for the ProxyGroup, // and verify that this matches the expected tailnet. if err := tstest.WaitFor(3*time.Minute, func() error { var secrets corev1.SecretList if err := kubeClient.List(t.Context(), &secrets, client.InNamespace("tailscale"), client.MatchingLabels{ kubetypes.LabelSecretType: kubetypes.LabelSecretTypeState, "tailscale.com/parent-resource-type": "proxygroup", "tailscale.com/parent-resource": pg.Name, }, ); err != nil { return err } if len(secrets.Items) == 0 { return fmt.Errorf("no state secrets found for ProxyGroup %q yet", pg.Name) } fqdn := strings.TrimSuffix(string(secrets.Items[0].Data[kubetypes.KeyDeviceFQDN]), ".") _, tailnet, ok := strings.Cut(fqdn, ".") if !ok { return fmt.Errorf("ProxyGroup %q: device FQDN %q has no domain yet", pg.Name, fqdn) } if tailnet != expectedTailnet { return fmt.Errorf("ProxyGroup %q on wrong tailnet: got domain %q, want %q", pg.Name, tailnet, expectedTailnet) } return nil }); err != nil { return fmt.Errorf("ProxyGroup %q not on expected tailnet: %v", pg.Name, err) } return nil } func waitForIngressHostname(t *testing.T, namespace, name string) (string, error) { t.Helper() var hostname string forceReconcile := triggerReconcile(t, client.ObjectKey{Namespace: namespace, Name: name}, &networkingv1.Ingress{}, 30*time.Second) if err := tstest.WaitFor(5*time.Minute, func() error { forceReconcile() ing := &networkingv1.Ingress{} if err := kubeClient.Get(t.Context(), client.ObjectKey{ Namespace: namespace, Name: name, }, ing); err != nil { return err } if len(ing.Status.LoadBalancer.Ingress) == 0 || ing.Status.LoadBalancer.Ingress[0].Hostname == "" { return fmt.Errorf("Ingress not ready yet") } hostname = ing.Status.LoadBalancer.Ingress[0].Hostname t.Log("Ingress is ready") return nil }); err != nil { return "", fmt.Errorf("Ingress %s/%s never got a hostname: %w", namespace, name, err) } return hostname, nil }