diff --git a/.golangci.yml b/.golangci.yml index 29a2f9a1c..4413041e1 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -245,6 +245,10 @@ linters: text: Function 'buildConstructor' has too many statements linters: - funlen + - path: pkg/provider/kubernetes/ingress-nginx/kubernetes.go + text: Function 'loadConfiguration' has too many statements + linters: + - funlen - path: pkg/tracing/haystack/logger.go linters: - goprintffuncname diff --git a/docs/content/reference/install-configuration/providers/kubernetes/kubernetes-ingress-nginx.md b/docs/content/reference/install-configuration/providers/kubernetes/kubernetes-ingress-nginx.md new file mode 100644 index 000000000..6170be7ff --- /dev/null +++ b/docs/content/reference/install-configuration/providers/kubernetes/kubernetes-ingress-nginx.md @@ -0,0 +1,112 @@ +--- +title: "Traefik Kubernetes Ingress NGINX Documentation" +description: "Understand the requirements, routing configuration, and how to set up the Kubernetes Ingress NGINX provider. Read the technical documentation." +--- + +# Traefik & Ingresses with NGINX Annotations + +The experimental Traefik Kubernetes Ingress NGINX provider is a Kubernetes Ingress controller; i.e, +it manages access to cluster services by supporting the [Ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/) specification. +It also supports some of the [ingress-nginx](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/annotations/) annotations on ingresses to customize their behavior. + +!!! warning "Ingress Discovery" + + The Kubernetes Ingress NGINX provider is discovering by default all Ingresses in the cluster, + which may lead to duplicated routers if you are also using the Kubernetes Ingress provider. + We recommend to use IngressClass for the Ingresses you want to be handled by this provider, + or to use the `watchNamespace` or `watchNamespaceSelector` options to limit the discovery of Ingresses to a specific namespace or set of namespaces. + +## Configuration Example + +As this provider is an experimental feature, it needs to be enabled in the experimental and in the provider sections of the configuration. +You can enable the Kubernetes Ingress NGINX provider as detailed below: + +```yaml tab="File (YAML)" +experimental: + kubernetesIngressNGINX: true + +providers: + kubernetesIngressNGINX: {} +``` + +```toml tab="File (TOML)" +[experimental.kubernetesIngressNGINX] + +[providers.kubernetesIngressNGINX] +``` + +```bash tab="CLI" +--experimental.kubernetesingressnginx=true +--providers.kubernetesingressnginx=true +``` + +The provider then watches for incoming ingresses events, such as the example below, +and derives the corresponding dynamic configuration from it, +which in turn creates the resulting routers, services, handlers, etc. + +## Configuration Options + + +| Field | Description | Default | Required | +|:------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:--------|:---------| +| `providers.providersThrottleDuration` | Minimum amount of time to wait for, after a configuration reload, before taking into account any new configuration refresh event.
If multiple events occur within this time, only the most recent one is taken into account, and all others are discarded.
**This option cannot be set per provider, but the throttling algorithm applies to each of them independently.** | 2s | No | +| `providers.kubernetesIngressNGINX.endpoint` | Server endpoint URL.
More information [here](#endpoint). | "" | No | +| `providers.kubernetesIngressNGINX.token` | Bearer token used for the Kubernetes client configuration. | "" | No | +| `providers.kubernetesIngressNGINX.certAuthFilePath` | Path to the certificate authority file.
Used for the Kubernetes client configuration. | "" | No | +| `providers.kubernetesIngressNGINX.throttleDuration` | Minimum amount of time to wait between two Kubernetes events before producing a new configuration.
This prevents a Kubernetes cluster that updates many times per second from continuously changing your Traefik configuration.
If empty, every event is caught. | 0s | No | +| `providers.kubernetesIngressNGINX.watchNamespace` | Namespace the controller watches for updates to Kubernetes objects. All namespaces are watched if this parameter is left empty. | "" | No | +| `providers.kubernetesIngressNGINX.watchNamespaceSelector` | Selector selects namespaces the controller watches for updates to Kubernetes objects. | "" | No | +| `providers.kubernetesIngressNGINX.ingressClass` | Name of the ingress class this controller satisfies. | "" | No | +| `providers.kubernetesIngressNGINX.controllerClass` | Ingress Class Controller value this controller satisfies. | "" | No | +| `providers.kubernetesIngressNGINX.watchIngressWithoutClass` | Define if Ingress Controller should also watch for Ingresses without an IngressClass or the annotation specified. | false | No | +| `providers.kubernetesIngressNGINX.ingressClassByName` | Define if Ingress Controller should watch for Ingress Class by Name together with Controller Class. | false | No | +| `providers.kubernetesIngressNGINX.publishService` | Service fronting the Ingress controller. Takes the form namespace/name. | "" | No | +| `providers.kubernetesIngressNGINX.publishStatusAddress` | Customized address (or addresses, separated by comma) to set as the load-balancer status of Ingress objects this controller satisfies. | "" | No | +| `providers.kubernetesIngressNGINX.defaultBackendService` | Service used to serve HTTP requests not matching any known server name (catch-all). Takes the form 'namespace/name'. | "" | No | +| `providers.kubernetesIngressNGINX.disableSvcExternalName` | Disable support for Services of type ExternalName. | false | No | + + + +### `endpoint` + +The Kubernetes server endpoint URL. + +When deployed into Kubernetes, Traefik reads the environment variables `KUBERNETES_SERVICE_HOST` +and `KUBERNETES_SERVICE_PORT` or `KUBECONFIG` to construct the endpoint. + +The access token is looked up in `/var/run/secrets/kubernetes.io/serviceaccount/token` +and the SSL CA certificate in `/var/run/secrets/kubernetes.io/serviceaccount/ca.crt`. +Both are mounted automatically when deployed inside Kubernetes. + +The endpoint may be specified to override the environment variable values inside +a cluster. + +When the environment variables are not found, Traefik tries to connect to the +Kubernetes API server with an external-cluster client. + +In this case, the endpoint is required. +Specifically, it may be set to the URL used by `kubectl proxy` to connect to a Kubernetes +cluster using the granted authentication and authorization of the associated kubeconfig. + +```yaml tab="File (YAML)" +providers: + kubernetesIngressNGINX: + endpoint: "http://localhost:8080" + # ... +``` + +```toml tab="File (TOML)" +[providers.kubernetesIngressNGINX] + endpoint = "http://localhost:8080" + # ... +``` + +```bash tab="CLI" +--providers.kubernetesingressnginx.endpoint=http://localhost:8080 +``` + +## Routing Configuration + +See the dedicated section in [routing](../../../routing-configuration/kubernetes/ingress-nginx.md). + +{!traefik-for-business-applications.md!} diff --git a/docs/content/reference/routing-configuration/kubernetes/ingress-nginx.md b/docs/content/reference/routing-configuration/kubernetes/ingress-nginx.md new file mode 100644 index 000000000..619ddaa54 --- /dev/null +++ b/docs/content/reference/routing-configuration/kubernetes/ingress-nginx.md @@ -0,0 +1,394 @@ +--- +title: "Traefik Kubernetes Ingress NGINX Routing Configuration" +description: "Understand the routing configuration for the Kubernetes Ingress NGINX Controller and Traefik Proxy. Read the technical documentation." +--- + +# Traefik & Ingresses with NGINX Annotations + +The experimental Kubernetes Controller for Ingresses with NGINX annotations. +{: .subtitle } + +!!! warning "Ingress Discovery" + + The Kubernetes Ingress NGINX provider is discovering by default all Ingresses in the cluster, + which may lead to duplicated routers if you are also using the Kubernetes Ingress provider. + We recommend to use IngressClass for the Ingresses you want to be handled by this provider, + or to use the `watchNamespace` or `watchNamespaceSelector` options to limit the discovery of Ingresses to a specific namespace or set of namespaces. + +## Routing Configuration + +The Kubernetes Ingress NGINX provider watches for incoming ingresses events, such as the example below, +and derives the corresponding dynamic configuration from it, +which in turn will create the resulting routers, services, handlers, etc. + +## Configuration Example + +??? example "Configuring Kubernetes Ingress NGINX Controller" + + ```yaml tab="RBAC" + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: traefik-ingress-controller + rules: + - apiGroups: + - "" + resources: + - namespaces + verbs: + - get + - apiGroups: + - "" + resources: + - configmaps + - pods + - secrets + - endpoints + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch + - apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - networking.k8s.io + resources: + - ingresses/status + verbs: + - update + - apiGroups: + - networking.k8s.io + resources: + - ingressclasses + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch + - get + + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: traefik-ingress-controller + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: traefik-ingress-controller + subjects: + - kind: ServiceAccount + name: traefik-ingress-controller + namespace: default + ``` + + ```yaml tab="Traefik" + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: traefik-ingress-controller + + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: traefik + labels: + app: traefik + + spec: + replicas: 1 + selector: + matchLabels: + app: traefik + template: + metadata: + labels: + app: traefik + spec: + serviceAccountName: traefik-ingress-controller + containers: + - name: traefik + image: traefik:v3.4 + args: + - --entryPoints.web.address=:80 + - --providers.kubernetesingressnginx + ports: + - name: web + containerPort: 80 + + --- + apiVersion: v1 + kind: Service + metadata: + name: traefik + spec: + type: LoadBalancer + selector: + app: traefik + ports: + - name: web + port: 80 + targetPort: 80 + ``` + + ```yaml tab="Whoami" + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: whoami + labels: + app: whoami + + spec: + replicas: 2 + selector: + matchLabels: + app: whoami + template: + metadata: + labels: + app: whoami + spec: + containers: + - name: whoami + image: traefik/whoami + ports: + - containerPort: 80 + + --- + apiVersion: v1 + kind: Service + metadata: + name: whoami + + spec: + selector: + app: whoami + ports: + - name: http + port: 80 + ``` + + ```yaml tab="Ingress" + --- + apiVersion: networking.k8s.io/v1 + kind: IngressClass + metadata: + name: nginx + spec: + controller: k8s.io/ingress-nginx + + --- + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: myingress + + spec: + ingressClassName: nginx + rules: + - host: whoami.localhost + http: + paths: + - path: /bar + pathType: Exact + backend: + service: + name: whoami + port: + number: 80 + - path: /foo + pathType: Exact + backend: + service: + name: whoami + port: + number: 80 + ``` + +## Annotations Support + +This section lists all known NGINX Ingress annotations, split between those currently implemented (with limitations if any) and those not implemented. +Limitations or behavioral differences are indicated where relevant. + +!!! warning "Global configuration" + + Traefik does not expose all global configuration options to control default behaviors for ingresses. + + Some behaviors that are globally configurable in NGINX (such as default SSL redirect, rate limiting, or affinity) are currently not supported and cannot be overridden per-ingress as in NGINX. + +### Caveats and Key Behavioral Differences + +- **Authentication**: Forward auth behaves differently and session caching is not supported. NGINX supports sub-request based auth, while Traefik forwards the original request. +- **Session Affinity**: Only persistent mode is supported. +- **Leader Election**: Not supported; no cluster mode with leader election. +- **Default Backend**: Only `defaultBackend` in Ingress spec is supported; the annotation is ignored. +- **Load Balancing**: Only round_robin is supported; EWMA and IP hash are not supported. +- **CORS**: NGINX responds with all configured headers unconditionally; Traefik handles headers differently between pre-flight and regular requests. +- **TLS/Backend Protocols**: AUTO_HTTP, FCGI and some TLS options are not supported in Traefik. +- **Path Handling**: Traefik preserves trailing slashes by default; NGINX removes them unless configured otherwise. + +### Supported NGINX Annotations + +| Annotation | Limitations / Notes | +|-------------------------------------------------------|--------------------------------------------------------------------------------------------| +| `nginx.ingress.kubernetes.io/affinity` | | +| `nginx.ingress.kubernetes.io/affinity-mode` | Only persistent mode supported; balanced/canary not supported. | +| `nginx.ingress.kubernetes.io/auth-type` | | +| `nginx.ingress.kubernetes.io/auth-secret` | | +| `nginx.ingress.kubernetes.io/auth-secret-type` | | +| `nginx.ingress.kubernetes.io/auth-realm` | | +| `nginx.ingress.kubernetes.io/auth-url` | Only URL and response headers copy supported. Forward auth behaves differently than NGINX. | +| `nginx.ingress.kubernetes.io/auth-method` | | +| `nginx.ingress.kubernetes.io/auth-response-headers` | | +| `nginx.ingress.kubernetes.io/ssl-redirect` | Cannot opt-out per route if enabled globally. | +| `nginx.ingress.kubernetes.io/force-ssl-redirect` | Cannot opt-out per route if enabled globally. | +| `nginx.ingress.kubernetes.io/ssl-passthrough` | Some differences in SNI/default backend handling. | +| `nginx.ingress.kubernetes.io/use-regex` | | +| `nginx.ingress.kubernetes.io/session-cookie-name` | | +| `nginx.ingress.kubernetes.io/session-cookie-path` | | +| `nginx.ingress.kubernetes.io/session-cookie-domain` | | +| `nginx.ingress.kubernetes.io/session-cookie-samesite` | | +| `nginx.ingress.kubernetes.io/load-balance` | Only round_robin supported; ewma and IP hash not supported. | +| `nginx.ingress.kubernetes.io/backend-protocol` | FCGI and AUTO_HTTP not supported. | +| `nginx.ingress.kubernetes.io/enable-cors` | Partial support. | +| `nginx.ingress.kubernetes.io/cors-allow-credentials` | | +| `nginx.ingress.kubernetes.io/cors-allow-headers` | | +| `nginx.ingress.kubernetes.io/cors-allow-methods` | | +| `nginx.ingress.kubernetes.io/cors-allow-origin` | | +| `nginx.ingress.kubernetes.io/cors-max-age` | | +| `nginx.ingress.kubernetes.io/proxy-ssl-server-name` | | +| `nginx.ingress.kubernetes.io/proxy-ssl-name` | | +| `nginx.ingress.kubernetes.io/proxy-ssl-verify` | | +| `nginx.ingress.kubernetes.io/proxy-ssl-secret` | | +| `nginx.ingress.kubernetes.io/service-upstream` | | + +### Unsupported NGINX Annotations + +All other NGINX annotations not listed above, including but not limited to: + +| Annotation | Notes | +|-----------------------------------------------------------------------------|------------------------------------------------------| +| `nginx.ingress.kubernetes.io/app-root` | Not supported. | +| `nginx.ingress.kubernetes.io/affinity-canary-behavior` | Not supported. | +| `nginx.ingress.kubernetes.io/auth-tls-secret` | Not supported. | +| `nginx.ingress.kubernetes.io/auth-tls-verify-depth` | Not supported. | +| `nginx.ingress.kubernetes.io/auth-tls-verify-client` | Not supported. | +| `nginx.ingress.kubernetes.io/auth-tls-error-page` | Not supported. | +| `nginx.ingress.kubernetes.io/auth-tls-pass-certificate-to-upstream` | Not supported. | +| `nginx.ingress.kubernetes.io/auth-tls-match-cn` | Not supported. | +| `nginx.ingress.kubernetes.io/auth-cache-key` | Not supported. | +| `nginx.ingress.kubernetes.io/auth-cache-duration` | Not supported. | +| `nginx.ingress.kubernetes.io/auth-keepalive` | Not supported. | +| `nginx.ingress.kubernetes.io/auth-keepalive-share-vars` | Not supported. | +| `nginx.ingress.kubernetes.io/auth-keepalive-requests` | Not supported. | +| `nginx.ingress.kubernetes.io/auth-keepalive-timeout` | Not supported. | +| `nginx.ingress.kubernetes.io/auth-proxy-set-headers` | Not supported. | +| `nginx.ingress.kubernetes.io/auth-snippet` | Not supported. | +| `nginx.ingress.kubernetes.io/enable-global-auth` | Not supported. | +| `nginx.ingress.kubernetes.io/canary` | Not supported. | +| `nginx.ingress.kubernetes.io/canary-by-header` | Not supported. | +| `nginx.ingress.kubernetes.io/canary-by-header-value` | Not supported. | +| `nginx.ingress.kubernetes.io/canary-by-header-pattern` | Not supported. | +| `nginx.ingress.kubernetes.io/canary-by-cookie` | Not supported. | +| `nginx.ingress.kubernetes.io/canary-weight` | Not supported. | +| `nginx.ingress.kubernetes.io/canary-weight-total` | Not supported. | +| `nginx.ingress.kubernetes.io/client-body-buffer-size` | Not supported. | +| `nginx.ingress.kubernetes.io/configuration-snippet` | Not supported. | +| `nginx.ingress.kubernetes.io/custom-http-errors` | Not supported. | +| `nginx.ingress.kubernetes.io/disable-proxy-intercept-errors` | Not supported. | +| `nginx.ingress.kubernetes.io/default-backend` | Not supported; use `defaultBackend` in Ingress spec. | +| `nginx.ingress.kubernetes.io/limit-rate-after` | Not supported. | +| `nginx.ingress.kubernetes.io/limit-rate` | Not supported. | +| `nginx.ingress.kubernetes.io/limit-whitelist` | Not supported. | +| `nginx.ingress.kubernetes.io/limit-rps` | Not supported. | +| `nginx.ingress.kubernetes.io/limit-rpm` | Not supported. | +| `nginx.ingress.kubernetes.io/limit-burst-multiplier` | Not supported. | +| `nginx.ingress.kubernetes.io/limit-connections` | Not supported. | +| `nginx.ingress.kubernetes.io/global-rate-limit` | Not supported. | +| `nginx.ingress.kubernetes.io/global-rate-limit-window` | Not supported. | +| `nginx.ingress.kubernetes.io/global-rate-limit-key` | Not supported. | +| `nginx.ingress.kubernetes.io/global-rate-limit-ignored-cidrs` | Not supported. | +| `nginx.ingress.kubernetes.io/permanent-redirect` | Not supported. | +| `nginx.ingress.kubernetes.io/permanent-redirect-code` | Not supported. | +| `nginx.ingress.kubernetes.io/temporal-redirect` | Not supported. | +| `nginx.ingress.kubernetes.io/preserve-trailing-slash` | Not supported; Traefik preserves by default. | +| `nginx.ingress.kubernetes.io/proxy-cookie-domain` | Not supported. | +| `nginx.ingress.kubernetes.io/proxy-cookie-path` | Not supported. | +| `nginx.ingress.kubernetes.io/proxy-connect-timeout` | Not supported. | +| `nginx.ingress.kubernetes.io/proxy-send-timeout` | Not supported. | +| `nginx.ingress.kubernetes.io/proxy-read-timeout` | Not supported. | +| `nginx.ingress.kubernetes.io/proxy-next-upstream` | Not supported. | +| `nginx.ingress.kubernetes.io/proxy-next-upstream-timeout` | Not supported. | +| `nginx.ingress.kubernetes.io/proxy-next-upstream-tries` | Not supported. | +| `nginx.ingress.kubernetes.io/proxy-request-buffering` | Not supported. | +| `nginx.ingress.kubernetes.io/proxy-redirect-from` | Not supported. | +| `nginx.ingress.kubernetes.io/proxy-redirect-to` | Not supported. | +| `nginx.ingress.kubernetes.io/proxy-http-version` | Not supported. | +| `nginx.ingress.kubernetes.io/proxy-ssl-ciphers` | Not supported. | +| `nginx.ingress.kubernetes.io/proxy-ssl-verify-depth` | Not supported. | +| `nginx.ingress.kubernetes.io/proxy-ssl-protocols` | Not supported. | +| `nginx.ingress.kubernetes.io/enable-rewrite-log` | Not supported. | +| `nginx.ingress.kubernetes.io/rewrite-target` | Not supported. | +| `nginx.ingress.kubernetes.io/satisfy` | Not supported. | +| `nginx.ingress.kubernetes.io/server-alias` | Not supported. | +| `nginx.ingress.kubernetes.io/server-snippet` | Not supported. | +| `nginx.ingress.kubernetes.io/session-cookie-conditional-samesite-none` | Not supported. | +| `nginx.ingress.kubernetes.io/session-cookie-expires` | Not supported. | +| `nginx.ingress.kubernetes.io/session-cookie-change-on-failure` | Not supported. | +| `nginx.ingress.kubernetes.io/ssl-ciphers` | Not supported. | +| `nginx.ingress.kubernetes.io/ssl-prefer-server-ciphers` | Not supported. | +| `nginx.ingress.kubernetes.io/connection-proxy-header` | Not supported. | +| `nginx.ingress.kubernetes.io/enable-access-log` | Not supported. | +| `nginx.ingress.kubernetes.io/enable-opentracing` | Not supported. | +| `nginx.ingress.kubernetes.io/opentracing-trust-incoming-span` | Not supported. | +| `nginx.ingress.kubernetes.io/enable-opentelemetry` | Not supported. | +| `nginx.ingress.kubernetes.io/opentelemetry-trust-incoming-span` | Not supported. | +| `nginx.ingress.kubernetes.io/enable-modsecurity` | Not supported. | +| `nginx.ingress.kubernetes.io/enable-owasp-core-rules` | Not supported. | +| `nginx.ingress.kubernetes.io/modsecurity-transaction-id` | Not supported. | +| `nginx.ingress.kubernetes.io/modsecurity-snippet` | Not supported. | +| `nginx.ingress.kubernetes.io/mirror-request-body` | Not supported. | +| `nginx.ingress.kubernetes.io/mirror-target` | Not supported. | +| `nginx.ingress.kubernetes.io/mirror-host` | Not supported. | +| `nginx.ingress.kubernetes.io/x-forwarded-prefix` | Not supported. | +| `nginx.ingress.kubernetes.io/upstream-hash-by` | Not supported. | +| `nginx.ingress.kubernetes.io/upstream-vhost` | Not supported. | +| `nginx.ingress.kubernetes.io/denylist-source-range` | Not supported. | +| `nginx.ingress.kubernetes.io/whitelist-source-range` | Not supported. | +| `nginx.ingress.kubernetes.io/proxy-buffering` | Not supported. | +| `nginx.ingress.kubernetes.io/proxy-buffers-number` | Not supported. | +| `nginx.ingress.kubernetes.io/proxy-buffer-size` | Not supported. | +| `nginx.ingress.kubernetes.io/proxy-max-temp-file-size` | Not supported. | +| `nginx.ingress.kubernetes.io/stream-snippet` | Not supported. | diff --git a/docs/content/reference/static-configuration/cli-ref.md b/docs/content/reference/static-configuration/cli-ref.md index e0b1b0c73..cfc377935 100644 --- a/docs/content/reference/static-configuration/cli-ref.md +++ b/docs/content/reference/static-configuration/cli-ref.md @@ -339,6 +339,9 @@ Enable debug mode for the FastProxy implementation. (Default: ```false```) `--experimental.kubernetesgateway`: (Deprecated) Allow the Kubernetes gateway api provider usage. (Default: ```false```) +`--experimental.kubernetesingressnginx`: +Allow the Kubernetes Ingress NGINX provider usage. (Default: ```false```) + `--experimental.localplugins.`: Local plugins configuration. (Default: ```false```) @@ -1047,6 +1050,51 @@ Ingress refresh throttle duration (Default: ```0```) `--providers.kubernetesingress.token`: Kubernetes bearer token (not needed for in-cluster client). It accepts either a token value or a file path to the token. +`--providers.kubernetesingressnginx`: +Enable Kubernetes Ingress NGINX provider. (Default: ```false```) + +`--providers.kubernetesingressnginx.certauthfilepath`: +Kubernetes certificate authority file path (not needed for in-cluster client). + +`--providers.kubernetesingressnginx.controllerclass`: +Ingress Class Controller value this controller satisfies. (Default: ```k8s.io/ingress-nginx```) + +`--providers.kubernetesingressnginx.defaultbackendservice`: +Service used to serve HTTP requests not matching any known server name (catch-all). Takes the form 'namespace/name'. + +`--providers.kubernetesingressnginx.disablesvcexternalname`: +Disable support for Services of type ExternalName. (Default: ```false```) + +`--providers.kubernetesingressnginx.endpoint`: +Kubernetes server endpoint (required for external cluster client). + +`--providers.kubernetesingressnginx.ingressclass`: +Name of the ingress class this controller satisfies. (Default: ```nginx```) + +`--providers.kubernetesingressnginx.ingressclassbyname`: +Define if Ingress Controller should watch for Ingress Class by Name together with Controller Class. (Default: ```false```) + +`--providers.kubernetesingressnginx.publishservice`: +Service fronting the Ingress controller. Takes the form 'namespace/name'. + +`--providers.kubernetesingressnginx.publishstatusaddress`: +Customized address (or addresses, separated by comma) to set as the load-balancer status of Ingress objects this controller satisfies. + +`--providers.kubernetesingressnginx.throttleduration`: +Ingress refresh throttle duration. (Default: ```0```) + +`--providers.kubernetesingressnginx.token`: +Kubernetes bearer token (not needed for in-cluster client). It accepts either a token value or a file path to the token. + +`--providers.kubernetesingressnginx.watchingresswithoutclass`: +Define if Ingress Controller should also watch for Ingresses without an IngressClass or the annotation specified. (Default: ```false```) + +`--providers.kubernetesingressnginx.watchnamespace`: +Namespace the controller watches for updates to Kubernetes objects. All namespaces are watched if this parameter is left empty. + +`--providers.kubernetesingressnginx.watchnamespaceselector`: +Selector selects namespaces the controller watches for updates to Kubernetes objects. + `--providers.nomad`: Enable Nomad backend with default settings. (Default: ```false```) diff --git a/docs/content/reference/static-configuration/env-ref.md b/docs/content/reference/static-configuration/env-ref.md index ca423ab23..7a3f59fcb 100644 --- a/docs/content/reference/static-configuration/env-ref.md +++ b/docs/content/reference/static-configuration/env-ref.md @@ -339,6 +339,9 @@ Enable debug mode for the FastProxy implementation. (Default: ```false```) `TRAEFIK_EXPERIMENTAL_KUBERNETESGATEWAY`: (Deprecated) Allow the Kubernetes gateway api provider usage. (Default: ```false```) +`TRAEFIK_EXPERIMENTAL_KUBERNETESINGRESSNGINX`: +Allow the Kubernetes Ingress NGINX provider usage. (Default: ```false```) + `TRAEFIK_EXPERIMENTAL_LOCALPLUGINS_`: Local plugins configuration. (Default: ```false```) @@ -999,6 +1002,51 @@ Kubernetes bearer token (not needed for in-cluster client). It accepts either a `TRAEFIK_PROVIDERS_KUBERNETESINGRESS`: Enable Kubernetes backend with default settings. (Default: ```false```) +`TRAEFIK_PROVIDERS_KUBERNETESINGRESSNGINX`: +Enable Kubernetes Ingress NGINX provider. (Default: ```false```) + +`TRAEFIK_PROVIDERS_KUBERNETESINGRESSNGINX_CERTAUTHFILEPATH`: +Kubernetes certificate authority file path (not needed for in-cluster client). + +`TRAEFIK_PROVIDERS_KUBERNETESINGRESSNGINX_CONTROLLERCLASS`: +Ingress Class Controller value this controller satisfies. (Default: ```k8s.io/ingress-nginx```) + +`TRAEFIK_PROVIDERS_KUBERNETESINGRESSNGINX_DEFAULTBACKENDSERVICE`: +Service used to serve HTTP requests not matching any known server name (catch-all). Takes the form 'namespace/name'. + +`TRAEFIK_PROVIDERS_KUBERNETESINGRESSNGINX_DISABLESVCEXTERNALNAME`: +Disable support for Services of type ExternalName. (Default: ```false```) + +`TRAEFIK_PROVIDERS_KUBERNETESINGRESSNGINX_ENDPOINT`: +Kubernetes server endpoint (required for external cluster client). + +`TRAEFIK_PROVIDERS_KUBERNETESINGRESSNGINX_INGRESSCLASS`: +Name of the ingress class this controller satisfies. (Default: ```nginx```) + +`TRAEFIK_PROVIDERS_KUBERNETESINGRESSNGINX_INGRESSCLASSBYNAME`: +Define if Ingress Controller should watch for Ingress Class by Name together with Controller Class. (Default: ```false```) + +`TRAEFIK_PROVIDERS_KUBERNETESINGRESSNGINX_PUBLISHSERVICE`: +Service fronting the Ingress controller. Takes the form 'namespace/name'. + +`TRAEFIK_PROVIDERS_KUBERNETESINGRESSNGINX_PUBLISHSTATUSADDRESS`: +Customized address (or addresses, separated by comma) to set as the load-balancer status of Ingress objects this controller satisfies. + +`TRAEFIK_PROVIDERS_KUBERNETESINGRESSNGINX_THROTTLEDURATION`: +Ingress refresh throttle duration. (Default: ```0```) + +`TRAEFIK_PROVIDERS_KUBERNETESINGRESSNGINX_TOKEN`: +Kubernetes bearer token (not needed for in-cluster client). It accepts either a token value or a file path to the token. + +`TRAEFIK_PROVIDERS_KUBERNETESINGRESSNGINX_WATCHINGRESSWITHOUTCLASS`: +Define if Ingress Controller should also watch for Ingresses without an IngressClass or the annotation specified. (Default: ```false```) + +`TRAEFIK_PROVIDERS_KUBERNETESINGRESSNGINX_WATCHNAMESPACE`: +Namespace the controller watches for updates to Kubernetes objects. All namespaces are watched if this parameter is left empty. + +`TRAEFIK_PROVIDERS_KUBERNETESINGRESSNGINX_WATCHNAMESPACESELECTOR`: +Selector selects namespaces the controller watches for updates to Kubernetes objects. + `TRAEFIK_PROVIDERS_KUBERNETESINGRESS_ALLOWEMPTYSERVICES`: Allow creation of services without endpoints. (Default: ```false```) diff --git a/docs/content/reference/static-configuration/file.toml b/docs/content/reference/static-configuration/file.toml index 4839306e8..515a55687 100644 --- a/docs/content/reference/static-configuration/file.toml +++ b/docs/content/reference/static-configuration/file.toml @@ -143,6 +143,21 @@ ip = "foobar" hostname = "foobar" publishedService = "foobar" + [providers.kubernetesIngressNGINX] + endpoint = "foobar" + token = "foobar" + certAuthFilePath = "foobar" + throttleDuration = "42s" + watchNamespace = "foobar" + watchNamespaceSelector = "foobar" + ingressClass = "foobar" + controllerClass = "foobar" + watchIngressWithoutClass = true + ingressClassByName = true + publishService = "foobar" + publishStatusAddress = ["foobar", "foobar"] + defaultBackendService = "foobar" + disableSvcExternalName = true [providers.kubernetesCRD] endpoint = "foobar" token = "foobar" @@ -572,6 +587,7 @@ [experimental] abortOnPluginFailure = true otlplogs = true + kubernetesIngressNGINX = true kubernetesGateway = true [experimental.plugins] [experimental.plugins.Descriptor0] diff --git a/docs/content/reference/static-configuration/file.yaml b/docs/content/reference/static-configuration/file.yaml index 4f302ead3..f36478611 100644 --- a/docs/content/reference/static-configuration/file.yaml +++ b/docs/content/reference/static-configuration/file.yaml @@ -158,6 +158,23 @@ providers: disableClusterScopeResources: true nativeLBByDefault: true strictPrefixMatching: true + kubernetesIngressNGINX: + endpoint: foobar + token: foobar + certAuthFilePath: foobar + throttleDuration: 42s + watchNamespace: foobar + watchNamespaceSelector: foobar + ingressClass: foobar + controllerClass: foobar + watchIngressWithoutClass: true + ingressClassByName: true + publishService: foobar + publishStatusAddress: + - foobar + - foobar + defaultBackendService: foobar + disableSvcExternalName: true kubernetesCRD: endpoint: foobar token: foobar @@ -670,6 +687,7 @@ experimental: fastProxy: debug: true otlplogs: true + kubernetesIngressNGINX: true kubernetesGateway: true core: defaultRuleSyntax: foobar diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 587b0a3c0..7a53171a6 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -79,6 +79,7 @@ nav: - 'Swarm': 'providers/swarm.md' - 'Kubernetes IngressRoute': 'providers/kubernetes-crd.md' - 'Kubernetes Ingress': 'providers/kubernetes-ingress.md' + - 'Kubernetes Ingress NGINX': 'reference/install-configuration/providers/kubernetes/kubernetes-ingress-nginx.md' - 'Kubernetes Gateway API': 'providers/kubernetes-gateway.md' - 'Consul Catalog': 'providers/consul-catalog.md' - 'Nomad': 'providers/nomad.md' @@ -99,6 +100,7 @@ nav: - 'Swarm': 'routing/providers/swarm.md' - 'Kubernetes IngressRoute': 'routing/providers/kubernetes-crd.md' - 'Kubernetes Ingress': 'routing/providers/kubernetes-ingress.md' + - 'Kubernetes Ingress NGINX': 'reference/routing-configuration/kubernetes/ingress-nginx.md' - 'Kubernetes Gateway API': 'routing/providers/kubernetes-gateway.md' - 'Consul Catalog': 'routing/providers/consul-catalog.md' - 'Nomad': 'routing/providers/nomad.md' @@ -205,6 +207,7 @@ nav: - 'Kubernetes Gateway API' : 'reference/install-configuration/providers/kubernetes/kubernetes-gateway.md' - 'Kubernetes CRD' : 'reference/install-configuration/providers/kubernetes/kubernetes-crd.md' - 'Kubernetes Ingress' : 'reference/install-configuration/providers/kubernetes/kubernetes-ingress.md' + - 'Kubernetes Ingress NGINX' : 'reference/install-configuration/providers/kubernetes/kubernetes-ingress-nginx.md' - 'Docker': 'reference/install-configuration/providers/docker.md' - 'Swarm': 'reference/install-configuration/providers/swarm.md' - 'Hashicorp': @@ -307,6 +310,7 @@ nav: - 'UDP' : - 'IngressRouteUDP' : 'reference/routing-configuration/kubernetes/crd/udp/ingressrouteudp.md' - 'Ingress' : 'reference/routing-configuration/kubernetes/ingress.md' + - 'Ingress NGINX' : 'reference/routing-configuration/kubernetes/ingress-nginx.md' - 'Label & Tag Providers' : - 'Docker' : 'reference/routing-configuration/other-providers/docker.md' - 'Swarm' : 'reference/routing-configuration/other-providers/swarm.md' diff --git a/pkg/config/static/experimental.go b/pkg/config/static/experimental.go index f88564637..dba89fec8 100644 --- a/pkg/config/static/experimental.go +++ b/pkg/config/static/experimental.go @@ -4,11 +4,12 @@ import "github.com/traefik/traefik/v3/pkg/plugins" // Experimental experimental Traefik features. type Experimental struct { - Plugins map[string]plugins.Descriptor `description:"Plugins configuration." json:"plugins,omitempty" toml:"plugins,omitempty" yaml:"plugins,omitempty" export:"true"` - LocalPlugins map[string]plugins.LocalDescriptor `description:"Local plugins configuration." json:"localPlugins,omitempty" toml:"localPlugins,omitempty" yaml:"localPlugins,omitempty" export:"true"` - AbortOnPluginFailure bool `description:"Defines whether all plugins must be loaded successfully for Traefik to start." json:"abortOnPluginFailure,omitempty" toml:"abortOnPluginFailure,omitempty" yaml:"abortOnPluginFailure,omitempty" export:"true"` - FastProxy *FastProxyConfig `description:"Enables the FastProxy implementation." json:"fastProxy,omitempty" toml:"fastProxy,omitempty" yaml:"fastProxy,omitempty" label:"allowEmpty" file:"allowEmpty" export:"true"` - OTLPLogs bool `description:"Enables the OpenTelemetry logs integration." json:"otlplogs,omitempty" toml:"otlplogs,omitempty" yaml:"otlplogs,omitempty" export:"true"` + Plugins map[string]plugins.Descriptor `description:"Plugins configuration." json:"plugins,omitempty" toml:"plugins,omitempty" yaml:"plugins,omitempty" export:"true"` + LocalPlugins map[string]plugins.LocalDescriptor `description:"Local plugins configuration." json:"localPlugins,omitempty" toml:"localPlugins,omitempty" yaml:"localPlugins,omitempty" export:"true"` + AbortOnPluginFailure bool `description:"Defines whether all plugins must be loaded successfully for Traefik to start." json:"abortOnPluginFailure,omitempty" toml:"abortOnPluginFailure,omitempty" yaml:"abortOnPluginFailure,omitempty" export:"true"` + FastProxy *FastProxyConfig `description:"Enables the FastProxy implementation." json:"fastProxy,omitempty" toml:"fastProxy,omitempty" yaml:"fastProxy,omitempty" label:"allowEmpty" file:"allowEmpty" export:"true"` + OTLPLogs bool `description:"Enables the OpenTelemetry logs integration." json:"otlplogs,omitempty" toml:"otlplogs,omitempty" yaml:"otlplogs,omitempty" export:"true"` + KubernetesIngressNGINX bool `description:"Allow the Kubernetes Ingress NGINX provider usage." json:"kubernetesIngressNGINX,omitempty" toml:"kubernetesIngressNGINX,omitempty" yaml:"kubernetesIngressNGINX,omitempty" export:"true"` // Deprecated: KubernetesGateway provider is not an experimental feature starting with v3.1. Please remove its usage from the static configuration. KubernetesGateway bool `description:"(Deprecated) Allow the Kubernetes gateway api provider usage." json:"kubernetesGateway,omitempty" toml:"kubernetesGateway,omitempty" yaml:"kubernetesGateway,omitempty" export:"true"` diff --git a/pkg/config/static/static_config.go b/pkg/config/static/static_config.go index 907d56ba9..e448ea305 100644 --- a/pkg/config/static/static_config.go +++ b/pkg/config/static/static_config.go @@ -22,6 +22,7 @@ import ( "github.com/traefik/traefik/v3/pkg/provider/kubernetes/crd" "github.com/traefik/traefik/v3/pkg/provider/kubernetes/gateway" "github.com/traefik/traefik/v3/pkg/provider/kubernetes/ingress" + ingressnginx "github.com/traefik/traefik/v3/pkg/provider/kubernetes/ingress-nginx" "github.com/traefik/traefik/v3/pkg/provider/kv/consul" "github.com/traefik/traefik/v3/pkg/provider/kv/etcd" "github.com/traefik/traefik/v3/pkg/provider/kv/redis" @@ -233,19 +234,20 @@ type Providers struct { Docker *docker.Provider `description:"Enable Docker backend with default settings." json:"docker,omitempty" toml:"docker,omitempty" yaml:"docker,omitempty" label:"allowEmpty" file:"allowEmpty" export:"true"` Swarm *docker.SwarmProvider `description:"Enable Docker Swarm backend with default settings." json:"swarm,omitempty" toml:"swarm,omitempty" yaml:"swarm,omitempty" label:"allowEmpty" file:"allowEmpty" export:"true"` - File *file.Provider `description:"Enable File backend with default settings." json:"file,omitempty" toml:"file,omitempty" yaml:"file,omitempty" export:"true"` - KubernetesIngress *ingress.Provider `description:"Enable Kubernetes backend with default settings." json:"kubernetesIngress,omitempty" toml:"kubernetesIngress,omitempty" yaml:"kubernetesIngress,omitempty" label:"allowEmpty" file:"allowEmpty" export:"true"` - KubernetesCRD *crd.Provider `description:"Enable Kubernetes backend with default settings." json:"kubernetesCRD,omitempty" toml:"kubernetesCRD,omitempty" yaml:"kubernetesCRD,omitempty" label:"allowEmpty" file:"allowEmpty" export:"true"` - KubernetesGateway *gateway.Provider `description:"Enable Kubernetes gateway api provider with default settings." json:"kubernetesGateway,omitempty" toml:"kubernetesGateway,omitempty" yaml:"kubernetesGateway,omitempty" label:"allowEmpty" file:"allowEmpty" export:"true"` - Rest *rest.Provider `description:"Enable Rest backend with default settings." json:"rest,omitempty" toml:"rest,omitempty" yaml:"rest,omitempty" label:"allowEmpty" file:"allowEmpty" export:"true"` - ConsulCatalog *consulcatalog.ProviderBuilder `description:"Enable ConsulCatalog backend with default settings." json:"consulCatalog,omitempty" toml:"consulCatalog,omitempty" yaml:"consulCatalog,omitempty" label:"allowEmpty" file:"allowEmpty" export:"true"` - Nomad *nomad.ProviderBuilder `description:"Enable Nomad backend with default settings." json:"nomad,omitempty" toml:"nomad,omitempty" yaml:"nomad,omitempty" label:"allowEmpty" file:"allowEmpty" export:"true"` - Ecs *ecs.Provider `description:"Enable AWS ECS backend with default settings." json:"ecs,omitempty" toml:"ecs,omitempty" yaml:"ecs,omitempty" label:"allowEmpty" file:"allowEmpty" export:"true"` - Consul *consul.ProviderBuilder `description:"Enable Consul backend with default settings." json:"consul,omitempty" toml:"consul,omitempty" yaml:"consul,omitempty" label:"allowEmpty" file:"allowEmpty" export:"true"` - Etcd *etcd.Provider `description:"Enable Etcd backend with default settings." json:"etcd,omitempty" toml:"etcd,omitempty" yaml:"etcd,omitempty" label:"allowEmpty" file:"allowEmpty" export:"true"` - ZooKeeper *zk.Provider `description:"Enable ZooKeeper backend with default settings." json:"zooKeeper,omitempty" toml:"zooKeeper,omitempty" yaml:"zooKeeper,omitempty" label:"allowEmpty" file:"allowEmpty" export:"true"` - Redis *redis.Provider `description:"Enable Redis backend with default settings." json:"redis,omitempty" toml:"redis,omitempty" yaml:"redis,omitempty" label:"allowEmpty" file:"allowEmpty" export:"true"` - HTTP *http.Provider `description:"Enable HTTP backend with default settings." json:"http,omitempty" toml:"http,omitempty" yaml:"http,omitempty" label:"allowEmpty" file:"allowEmpty" export:"true"` + File *file.Provider `description:"Enable File backend with default settings." json:"file,omitempty" toml:"file,omitempty" yaml:"file,omitempty" export:"true"` + KubernetesIngress *ingress.Provider `description:"Enable Kubernetes backend with default settings." json:"kubernetesIngress,omitempty" toml:"kubernetesIngress,omitempty" yaml:"kubernetesIngress,omitempty" label:"allowEmpty" file:"allowEmpty" export:"true"` + KubernetesIngressNGINX *ingressnginx.Provider `description:"Enable Kubernetes Ingress NGINX provider." json:"kubernetesIngressNGINX,omitempty" toml:"kubernetesIngressNGINX,omitempty" yaml:"kubernetesIngressNGINX,omitempty" label:"allowEmpty" file:"allowEmpty" export:"true"` + KubernetesCRD *crd.Provider `description:"Enable Kubernetes backend with default settings." json:"kubernetesCRD,omitempty" toml:"kubernetesCRD,omitempty" yaml:"kubernetesCRD,omitempty" label:"allowEmpty" file:"allowEmpty" export:"true"` + KubernetesGateway *gateway.Provider `description:"Enable Kubernetes gateway api provider with default settings." json:"kubernetesGateway,omitempty" toml:"kubernetesGateway,omitempty" yaml:"kubernetesGateway,omitempty" label:"allowEmpty" file:"allowEmpty" export:"true"` + Rest *rest.Provider `description:"Enable Rest backend with default settings." json:"rest,omitempty" toml:"rest,omitempty" yaml:"rest,omitempty" label:"allowEmpty" file:"allowEmpty" export:"true"` + ConsulCatalog *consulcatalog.ProviderBuilder `description:"Enable ConsulCatalog backend with default settings." json:"consulCatalog,omitempty" toml:"consulCatalog,omitempty" yaml:"consulCatalog,omitempty" label:"allowEmpty" file:"allowEmpty" export:"true"` + Nomad *nomad.ProviderBuilder `description:"Enable Nomad backend with default settings." json:"nomad,omitempty" toml:"nomad,omitempty" yaml:"nomad,omitempty" label:"allowEmpty" file:"allowEmpty" export:"true"` + Ecs *ecs.Provider `description:"Enable AWS ECS backend with default settings." json:"ecs,omitempty" toml:"ecs,omitempty" yaml:"ecs,omitempty" label:"allowEmpty" file:"allowEmpty" export:"true"` + Consul *consul.ProviderBuilder `description:"Enable Consul backend with default settings." json:"consul,omitempty" toml:"consul,omitempty" yaml:"consul,omitempty" label:"allowEmpty" file:"allowEmpty" export:"true"` + Etcd *etcd.Provider `description:"Enable Etcd backend with default settings." json:"etcd,omitempty" toml:"etcd,omitempty" yaml:"etcd,omitempty" label:"allowEmpty" file:"allowEmpty" export:"true"` + ZooKeeper *zk.Provider `description:"Enable ZooKeeper backend with default settings." json:"zooKeeper,omitempty" toml:"zooKeeper,omitempty" yaml:"zooKeeper,omitempty" label:"allowEmpty" file:"allowEmpty" export:"true"` + Redis *redis.Provider `description:"Enable Redis backend with default settings." json:"redis,omitempty" toml:"redis,omitempty" yaml:"redis,omitempty" label:"allowEmpty" file:"allowEmpty" export:"true"` + HTTP *http.Provider `description:"Enable HTTP backend with default settings." json:"http,omitempty" toml:"http,omitempty" yaml:"http,omitempty" label:"allowEmpty" file:"allowEmpty" export:"true"` Plugin map[string]PluginConf `description:"Plugins configuration." json:"plugin,omitempty" toml:"plugin,omitempty" yaml:"plugin,omitempty"` } @@ -391,6 +393,16 @@ func (c *Configuration) ValidateConfiguration() error { } } + if c.Providers != nil && c.Providers.KubernetesIngressNGINX != nil { + if c.Experimental == nil || !c.Experimental.KubernetesIngressNGINX { + return errors.New("the experimental KubernetesIngressNGINX feature must be enabled to use the KubernetesIngressNGINX provider") + } + + if c.Providers.KubernetesIngressNGINX.WatchNamespace != "" && c.Providers.KubernetesIngressNGINX.WatchNamespaceSelector != "" { + return errors.New("watchNamespace and watchNamespaceSelector options are mutually exclusive") + } + } + if c.AccessLog != nil && c.AccessLog.OTLP != nil { if c.Experimental == nil || !c.Experimental.OTLPLogs { return errors.New("the experimental OTLPLogs feature must be enabled to use OTLP access logging") diff --git a/pkg/provider/aggregator/aggregator.go b/pkg/provider/aggregator/aggregator.go index 9f8d21308..fb2b1a6dc 100644 --- a/pkg/provider/aggregator/aggregator.go +++ b/pkg/provider/aggregator/aggregator.go @@ -92,6 +92,10 @@ func NewProviderAggregator(conf static.Providers) *ProviderAggregator { p.quietAddProvider(conf.KubernetesIngress) } + if conf.KubernetesIngressNGINX != nil { + p.quietAddProvider(conf.KubernetesIngressNGINX) + } + if conf.KubernetesCRD != nil { p.quietAddProvider(conf.KubernetesCRD) } diff --git a/pkg/provider/kubernetes/ingress-nginx/annotations.go b/pkg/provider/kubernetes/ingress-nginx/annotations.go new file mode 100644 index 000000000..6536abed5 --- /dev/null +++ b/pkg/provider/kubernetes/ingress-nginx/annotations.go @@ -0,0 +1,115 @@ +package ingressnginx + +import ( + "errors" + "reflect" + "strconv" + "strings" + + netv1 "k8s.io/api/networking/v1" +) + +type ingressConfig struct { + AuthType *string `annotation:"nginx.ingress.kubernetes.io/auth-type"` + AuthSecret *string `annotation:"nginx.ingress.kubernetes.io/auth-secret"` + AuthRealm *string `annotation:"nginx.ingress.kubernetes.io/auth-realm"` + AuthSecretType *string `annotation:"nginx.ingress.kubernetes.io/auth-secret-type"` + + AuthURL *string `annotation:"nginx.ingress.kubernetes.io/auth-url"` + AuthResponseHeaders *string `annotation:"nginx.ingress.kubernetes.io/auth-response-headers"` + + ForceSSLRedirect *bool `annotation:"nginx.ingress.kubernetes.io/force-ssl-redirect"` + SSLRedirect *bool `annotation:"nginx.ingress.kubernetes.io/ssl-redirect"` + + SSLPassthrough *bool `annotation:"nginx.ingress.kubernetes.io/ssl-passthrough"` + + UseRegex *bool `annotation:"nginx.ingress.kubernetes.io/use-regex"` + + Affinity *string `annotation:"nginx.ingress.kubernetes.io/affinity"` + SessionCookieName *string `annotation:"nginx.ingress.kubernetes.io/session-cookie-name"` + SessionCookieSecure *bool `annotation:"nginx.ingress.kubernetes.io/session-cookie-secure"` + SessionCookiePath *string `annotation:"nginx.ingress.kubernetes.io/session-cookie-path"` + SessionCookieDomain *string `annotation:"nginx.ingress.kubernetes.io/session-cookie-domain"` + SessionCookieSameSite *string `annotation:"nginx.ingress.kubernetes.io/session-cookie-samesite"` + SessionCookieMaxAge *int `annotation:"nginx.ingress.kubernetes.io/session-cookie-max-age"` + + ServiceUpstream *bool `annotation:"nginx.ingress.kubernetes.io/service-upstream"` + + BackendProtocol *string `annotation:"nginx.ingress.kubernetes.io/backend-protocol"` + + ProxySSLSecret *string `annotation:"nginx.ingress.kubernetes.io/proxy-ssl-secret"` + ProxySSLVerify *string `annotation:"nginx.ingress.kubernetes.io/proxy-ssl-verify"` + ProxySSLName *string `annotation:"nginx.ingress.kubernetes.io/proxy-ssl-name"` + ProxySSLServerName *string `annotation:"nginx.ingress.kubernetes.io/proxy-ssl-server-name"` + + EnableCORS *bool `annotation:"nginx.ingress.kubernetes.io/enable-cors"` + EnableCORSAllowCredentials *bool `annotation:"nginx.ingress.kubernetes.io/cors-allow-credentials"` + CORSExposeHeaders *[]string `annotation:"nginx.ingress.kubernetes.io/cors-expose-headers"` + CORSAllowHeaders *[]string `annotation:"nginx.ingress.kubernetes.io/cors-allow-headers"` + CORSAllowMethods *[]string `annotation:"nginx.ingress.kubernetes.io/cors-allow-methods"` + CORSAllowOrigin *[]string `annotation:"nginx.ingress.kubernetes.io/cors-allow-origin"` + CORSMaxAge *int `annotation:"nginx.ingress.kubernetes.io/cors-max-age"` +} + +// parseIngressConfig parses the annotations from an Ingress object into an ingressConfig struct. +func parseIngressConfig(ing *netv1.Ingress) (ingressConfig, error) { + cfg := ingressConfig{} + cfgType := reflect.TypeOf(cfg) + cfgValue := reflect.ValueOf(&cfg).Elem() + + for i := range cfgType.NumField() { + field := cfgType.Field(i) + annotation := field.Tag.Get("annotation") + if annotation == "" { + continue + } + + val, ok := ing.GetAnnotations()[annotation] + if !ok { + continue + } + + switch field.Type.Elem().Kind() { + case reflect.String: + cfgValue.Field(i).Set(reflect.ValueOf(&val)) + case reflect.Bool: + parsed, err := strconv.ParseBool(val) + if err == nil { + cfgValue.Field(i).Set(reflect.ValueOf(&parsed)) + } + case reflect.Int: + parsed, err := strconv.Atoi(val) + if err == nil { + cfgValue.Field(i).Set(reflect.ValueOf(&parsed)) + } + case reflect.Slice: + if field.Type.Elem().Elem().Kind() == reflect.String { + // Handle slice of strings + var slice []string + elements := strings.Split(val, ",") + for _, elt := range elements { + slice = append(slice, strings.TrimSpace(elt)) + } + cfgValue.Field(i).Set(reflect.ValueOf(&slice)) + } else { + return cfg, errors.New("unsupported slice type in annotations") + } + default: + return cfg, errors.New("unsupported kind") + } + } + + return cfg, nil +} + +// parseBackendProtocol parses the backend protocol annotation and returns the corresponding protocol string. +func parseBackendProtocol(bp string) string { + switch strings.ToUpper(bp) { + case "HTTPS", "GRPCS": + return "https" + case "GRPC": + return "h2c" + default: + return "http" + } +} diff --git a/pkg/provider/kubernetes/ingress-nginx/annotations_test.go b/pkg/provider/kubernetes/ingress-nginx/annotations_test.go new file mode 100644 index 000000000..36ce89434 --- /dev/null +++ b/pkg/provider/kubernetes/ingress-nginx/annotations_test.go @@ -0,0 +1,76 @@ +package ingressnginx + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + netv1 "k8s.io/api/networking/v1" + "k8s.io/utils/ptr" +) + +func Test_parseIngressConfig(t *testing.T) { + tests := []struct { + desc string + annotations map[string]string + expected ingressConfig + }{ + { + desc: "all fields set", + annotations: map[string]string{ + "nginx.ingress.kubernetes.io/ssl-passthrough": "true", + "nginx.ingress.kubernetes.io/affinity": "cookie", + "nginx.ingress.kubernetes.io/session-cookie-name": "mycookie", + "nginx.ingress.kubernetes.io/session-cookie-secure": "true", + "nginx.ingress.kubernetes.io/session-cookie-path": "/foo", + "nginx.ingress.kubernetes.io/session-cookie-domain": "example.com", + "nginx.ingress.kubernetes.io/session-cookie-samesite": "Strict", + "nginx.ingress.kubernetes.io/session-cookie-max-age": "3600", + "nginx.ingress.kubernetes.io/backend-protocol": "HTTPS", + "nginx.ingress.kubernetes.io/cors-expose-headers": "foo, bar", + }, + expected: ingressConfig{ + SSLPassthrough: ptr.To(true), + Affinity: ptr.To("cookie"), + SessionCookieName: ptr.To("mycookie"), + SessionCookieSecure: ptr.To(true), + SessionCookiePath: ptr.To("/foo"), + SessionCookieDomain: ptr.To("example.com"), + SessionCookieSameSite: ptr.To("Strict"), + SessionCookieMaxAge: ptr.To(3600), + BackendProtocol: ptr.To("HTTPS"), + CORSExposeHeaders: ptr.To([]string{"foo", "bar"}), + }, + }, + { + desc: "missing fields", + annotations: map[string]string{ + "nginx.ingress.kubernetes.io/ssl-passthrough": "false", + }, + expected: ingressConfig{ + SSLPassthrough: ptr.To(false), + }, + }, + { + desc: "invalid bool and int", + annotations: map[string]string{ + "nginx.ingress.kubernetes.io/ssl-passthrough": "notabool", + "nginx.ingress.kubernetes.io/session-cookie-max-age (in seconds)": "notanint", + }, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + t.Parallel() + + var ing netv1.Ingress + ing.SetAnnotations(test.annotations) + + cfg, err := parseIngressConfig(&ing) + require.NoError(t, err) + + assert.Equal(t, test.expected, cfg) + }) + } +} diff --git a/pkg/provider/kubernetes/ingress-nginx/client.go b/pkg/provider/kubernetes/ingress-nginx/client.go new file mode 100644 index 000000000..93eae4a09 --- /dev/null +++ b/pkg/provider/kubernetes/ingress-nginx/client.go @@ -0,0 +1,384 @@ +package ingressnginx + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "runtime" + "slices" + "time" + + "github.com/rs/zerolog/log" + "github.com/traefik/traefik/v3/pkg/provider/kubernetes/k8s" + "github.com/traefik/traefik/v3/pkg/types" + traefikversion "github.com/traefik/traefik/v3/pkg/version" + corev1 "k8s.io/api/core/v1" + discoveryv1 "k8s.io/api/discovery/v1" + netv1 "k8s.io/api/networking/v1" + kerror "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" + kinformers "k8s.io/client-go/informers" + kclientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" +) + +const ( + resyncPeriod = 10 * time.Minute + defaultTimeout = 5 * time.Second +) + +type clientWrapper struct { + clientset kclientset.Interface + clusterScopeFactory kinformers.SharedInformerFactory + factoriesKube map[string]kinformers.SharedInformerFactory + factoriesSecret map[string]kinformers.SharedInformerFactory + factoriesIngress map[string]kinformers.SharedInformerFactory + isNamespaceAll bool + watchedNamespaces []string + + ignoreIngressClasses bool +} + +// newInClusterClient returns a new Provider client that is expected to run +// inside the cluster. +func newInClusterClient(endpoint string) (*clientWrapper, error) { + config, err := rest.InClusterConfig() + if err != nil { + return nil, fmt.Errorf("failed to create in-cluster configuration: %w", err) + } + + if endpoint != "" { + config.Host = endpoint + } + + return createClientFromConfig(config) +} + +func newExternalClusterClientFromFile(file string) (*clientWrapper, error) { + configFromFlags, err := clientcmd.BuildConfigFromFlags("", file) + if err != nil { + return nil, err + } + return createClientFromConfig(configFromFlags) +} + +// newExternalClusterClient returns a new Provider client that may run outside +// of the cluster. +// The endpoint parameter must not be empty. +func newExternalClusterClient(endpoint, caFilePath string, token types.FileOrContent) (*clientWrapper, error) { + if endpoint == "" { + return nil, errors.New("endpoint missing for external cluster client") + } + + tokenData, err := token.Read() + if err != nil { + return nil, fmt.Errorf("read token: %w", err) + } + + config := &rest.Config{ + Host: endpoint, + BearerToken: string(tokenData), + } + + if caFilePath != "" { + caData, err := os.ReadFile(caFilePath) + if err != nil { + return nil, fmt.Errorf("failed to read CA file %s: %w", caFilePath, err) + } + + config.TLSClientConfig = rest.TLSClientConfig{CAData: caData} + } + return createClientFromConfig(config) +} + +func createClientFromConfig(c *rest.Config) (*clientWrapper, error) { + c.UserAgent = fmt.Sprintf( + "%s/%s (%s/%s) kubernetes/ingress", + filepath.Base(os.Args[0]), + traefikversion.Version, + runtime.GOOS, + runtime.GOARCH, + ) + + clientset, err := kclientset.NewForConfig(c) + if err != nil { + return nil, err + } + + return newClient(clientset), nil +} + +func newClient(clientSet kclientset.Interface) *clientWrapper { + return &clientWrapper{ + clientset: clientSet, + factoriesSecret: make(map[string]kinformers.SharedInformerFactory), + factoriesIngress: make(map[string]kinformers.SharedInformerFactory), + factoriesKube: make(map[string]kinformers.SharedInformerFactory), + } +} + +// WatchAll starts namespace-specific controllers for all relevant kinds. +func (c *clientWrapper) WatchAll(ctx context.Context, namespace, namespaceSelector string) (<-chan interface{}, error) { + stopCh := ctx.Done() + eventCh := make(chan interface{}, 1) + eventHandler := &k8s.ResourceEventHandler{Ev: eventCh} + + c.ignoreIngressClasses = false + _, err := c.clientset.NetworkingV1().IngressClasses().List(ctx, metav1.ListOptions{Limit: 1}) + if err != nil { + if !kerror.IsNotFound(err) { + if kerror.IsForbidden(err) { + c.ignoreIngressClasses = true + } + } + } + + if namespaceSelector != "" { + ns, err := c.clientset.CoreV1().Namespaces().List(ctx, metav1.ListOptions{LabelSelector: namespaceSelector}) + if err != nil { + return nil, fmt.Errorf("listing namespaces: %w", err) + } + for _, item := range ns.Items { + c.watchedNamespaces = append(c.watchedNamespaces, item.Name) + } + } else { + c.isNamespaceAll = namespace == metav1.NamespaceAll + c.watchedNamespaces = []string{namespace} + } + + notOwnedByHelm := func(opts *metav1.ListOptions) { + opts.LabelSelector = "owner!=helm" + } + + for _, ns := range c.watchedNamespaces { + factoryIngress := kinformers.NewSharedInformerFactoryWithOptions(c.clientset, resyncPeriod, kinformers.WithNamespace(ns)) + + _, err := factoryIngress.Networking().V1().Ingresses().Informer().AddEventHandler(eventHandler) + if err != nil { + return nil, err + } + + c.factoriesIngress[ns] = factoryIngress + + factoryKube := kinformers.NewSharedInformerFactoryWithOptions(c.clientset, resyncPeriod, kinformers.WithNamespace(ns)) + _, err = factoryKube.Core().V1().Services().Informer().AddEventHandler(eventHandler) + if err != nil { + return nil, err + } + _, err = factoryKube.Discovery().V1().EndpointSlices().Informer().AddEventHandler(eventHandler) + if err != nil { + return nil, err + } + c.factoriesKube[ns] = factoryKube + + factorySecret := kinformers.NewSharedInformerFactoryWithOptions(c.clientset, resyncPeriod, kinformers.WithNamespace(ns), kinformers.WithTweakListOptions(notOwnedByHelm)) + _, err = factorySecret.Core().V1().Secrets().Informer().AddEventHandler(eventHandler) + if err != nil { + return nil, err + } + c.factoriesSecret[ns] = factorySecret + } + + for _, ns := range c.watchedNamespaces { + c.factoriesIngress[ns].Start(stopCh) + c.factoriesKube[ns].Start(stopCh) + c.factoriesSecret[ns].Start(stopCh) + } + + for _, ns := range c.watchedNamespaces { + for t, ok := range c.factoriesIngress[ns].WaitForCacheSync(stopCh) { + if !ok { + return nil, fmt.Errorf("timed out waiting for controller caches to sync %s in namespace %q", t.String(), ns) + } + } + + for t, ok := range c.factoriesKube[ns].WaitForCacheSync(stopCh) { + if !ok { + return nil, fmt.Errorf("timed out waiting for controller caches to sync %s in namespace %q", t.String(), ns) + } + } + + for t, ok := range c.factoriesSecret[ns].WaitForCacheSync(stopCh) { + if !ok { + return nil, fmt.Errorf("timed out waiting for controller caches to sync %s in namespace %q", t.String(), ns) + } + } + } + + c.clusterScopeFactory = kinformers.NewSharedInformerFactory(c.clientset, resyncPeriod) + + if !c.ignoreIngressClasses { + _, err = c.clusterScopeFactory.Networking().V1().IngressClasses().Informer().AddEventHandler(eventHandler) + if err != nil { + return nil, err + } + } + + c.clusterScopeFactory.Start(stopCh) + + for t, ok := range c.clusterScopeFactory.WaitForCacheSync(stopCh) { + if !ok { + return nil, fmt.Errorf("timed out waiting for controller caches to sync %s", t.String()) + } + } + + return eventCh, nil +} + +func (c *clientWrapper) ListIngressClasses() ([]*netv1.IngressClass, error) { + if c.ignoreIngressClasses { + return []*netv1.IngressClass{}, nil + } + + return c.clusterScopeFactory.Networking().V1().IngressClasses().Lister().List(labels.Everything()) +} + +// ListIngresses returns all Ingresses for observed namespaces in the cluster. +func (c *clientWrapper) ListIngresses() []*netv1.Ingress { + var results []*netv1.Ingress + + for ns, factory := range c.factoriesIngress { + // networking + listNew, err := factory.Networking().V1().Ingresses().Lister().List(labels.Everything()) + if err != nil { + log.Error().Err(err).Msgf("Failed to list ingresses in namespace %s", ns) + continue + } + + results = append(results, listNew...) + } + + return results +} + +// UpdateIngressStatus updates an Ingress with a provided status. +func (c *clientWrapper) UpdateIngressStatus(src *netv1.Ingress, ingStatus []netv1.IngressLoadBalancerIngress) error { + if !c.isWatchedNamespace(src.Namespace) { + return fmt.Errorf("failed to get ingress %s/%s: namespace is not within watched namespaces", src.Namespace, src.Name) + } + + ing, err := c.factoriesIngress[c.lookupNamespace(src.Namespace)].Networking().V1().Ingresses().Lister().Ingresses(src.Namespace).Get(src.Name) + if err != nil { + return fmt.Errorf("failed to get ingress %s/%s: %w", src.Namespace, src.Name, err) + } + + logger := log.With().Str("namespace", ing.Namespace).Str("ingress", ing.Name).Logger() + + if isLoadBalancerIngressEquals(ing.Status.LoadBalancer.Ingress, ingStatus) { + logger.Debug().Msg("Skipping ingress status update") + return nil + } + + ingCopy := ing.DeepCopy() + ingCopy.Status = netv1.IngressStatus{LoadBalancer: netv1.IngressLoadBalancerStatus{Ingress: ingStatus}} + + ctx, cancel := context.WithTimeout(context.Background(), defaultTimeout) + defer cancel() + + _, err = c.clientset.NetworkingV1().Ingresses(ingCopy.Namespace).UpdateStatus(ctx, ingCopy, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("failed to update ingress status %s/%s: %w", src.Namespace, src.Name, err) + } + + logger.Info().Msg("Updated ingress status") + return nil +} + +// GetService returns the named service from the given namespace. +func (c *clientWrapper) GetService(namespace, name string) (*corev1.Service, error) { + if !c.isWatchedNamespace(namespace) { + return nil, fmt.Errorf("failed to get service %s/%s: namespace is not within watched namespaces", namespace, name) + } + + return c.factoriesKube[c.lookupNamespace(namespace)].Core().V1().Services().Lister().Services(namespace).Get(name) +} + +// GetEndpointSlicesForService returns the EndpointSlices for the given service name in the given namespace. +func (c *clientWrapper) GetEndpointSlicesForService(namespace, serviceName string) ([]*discoveryv1.EndpointSlice, error) { + if !c.isWatchedNamespace(namespace) { + return nil, fmt.Errorf("failed to get endpointslices for service %s/%s: namespace is not within watched namespaces", namespace, serviceName) + } + + serviceLabelRequirement, err := labels.NewRequirement(discoveryv1.LabelServiceName, selection.Equals, []string{serviceName}) + if err != nil { + return nil, fmt.Errorf("failed to create service label selector requirement: %w", err) + } + serviceSelector := labels.NewSelector() + serviceSelector = serviceSelector.Add(*serviceLabelRequirement) + + return c.factoriesKube[c.lookupNamespace(namespace)].Discovery().V1().EndpointSlices().Lister().EndpointSlices(namespace).List(serviceSelector) +} + +// GetSecret returns the named secret from the given namespace. +func (c *clientWrapper) GetSecret(namespace, name string) (*corev1.Secret, error) { + if !c.isWatchedNamespace(namespace) { + return nil, fmt.Errorf("failed to get secret %s/%s: namespace is not within watched namespaces", namespace, name) + } + + return c.factoriesSecret[c.lookupNamespace(namespace)].Core().V1().Secrets().Lister().Secrets(namespace).Get(name) +} + +// lookupNamespace returns the lookup namespace key for the given namespace. +// When listening on all namespaces, it returns the client-go identifier ("") +// for all-namespaces. Otherwise, it returns the given namespace. +// The distinction is necessary because we index all informers on the special +// identifier iff all-namespaces are requested but receive specific namespace +// identifiers from the Kubernetes API, so we have to bridge this gap. +func (c *clientWrapper) lookupNamespace(ns string) string { + if c.isNamespaceAll { + return metav1.NamespaceAll + } + return ns +} + +// isWatchedNamespace checks to ensure that the namespace is being watched before we request +// it to ensure we don't panic by requesting an out-of-watch object. +func (c *clientWrapper) isWatchedNamespace(ns string) bool { + if c.isNamespaceAll { + return true + } + + return slices.Contains(c.watchedNamespaces, ns) +} + +// isLoadBalancerIngressEquals returns true if the given slices are equal, false otherwise. +func isLoadBalancerIngressEquals(aSlice, bSlice []netv1.IngressLoadBalancerIngress) bool { + if len(aSlice) != len(bSlice) { + return false + } + + aMap := make(map[string]struct{}) + for _, aIngress := range aSlice { + aMap[aIngress.Hostname+aIngress.IP] = struct{}{} + } + + for _, bIngress := range bSlice { + if _, exists := aMap[bIngress.Hostname+bIngress.IP]; !exists { + return false + } + } + + return true +} + +// filterIngressClass return a slice containing IngressClass matching either the annotation name or the controller. +func filterIngressClass(ingressClasses []*netv1.IngressClass, ingressClassByName bool, ingressClass, controllerClass string) []*netv1.IngressClass { + var filteredIngressClasses []*netv1.IngressClass + for _, ic := range ingressClasses { + if ingressClassByName && ic.Name == ingressClass { + return append(filteredIngressClasses, ic) + } + + if ic.Spec.Controller == controllerClass { + filteredIngressClasses = append(filteredIngressClasses, ic) + continue + } + } + + return filteredIngressClasses +} diff --git a/pkg/provider/kubernetes/ingress-nginx/client_test.go b/pkg/provider/kubernetes/ingress-nginx/client_test.go new file mode 100644 index 000000000..e6ed09596 --- /dev/null +++ b/pkg/provider/kubernetes/ingress-nginx/client_test.go @@ -0,0 +1,270 @@ +package ingressnginx + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + discoveryv1 "k8s.io/api/discovery/v1" + netv1 "k8s.io/api/networking/v1" + kerror "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kversion "k8s.io/apimachinery/pkg/version" + discoveryfake "k8s.io/client-go/discovery/fake" + kubefake "k8s.io/client-go/kubernetes/fake" +) + +func TestIsLoadBalancerIngressEquals(t *testing.T) { + testCases := []struct { + desc string + aSlice []netv1.IngressLoadBalancerIngress + bSlice []netv1.IngressLoadBalancerIngress + expectedEqual bool + }{ + { + desc: "both slices are empty", + expectedEqual: true, + }, + { + desc: "not the same length", + bSlice: []netv1.IngressLoadBalancerIngress{ + {IP: "192.168.1.1", Hostname: "traefik"}, + }, + expectedEqual: false, + }, + { + desc: "same ordered content", + aSlice: []netv1.IngressLoadBalancerIngress{ + {IP: "192.168.1.1", Hostname: "traefik"}, + }, + bSlice: []netv1.IngressLoadBalancerIngress{ + {IP: "192.168.1.1", Hostname: "traefik"}, + }, + expectedEqual: true, + }, + { + desc: "same unordered content", + aSlice: []netv1.IngressLoadBalancerIngress{ + {IP: "192.168.1.1", Hostname: "traefik"}, + {IP: "192.168.1.2", Hostname: "traefik2"}, + }, + bSlice: []netv1.IngressLoadBalancerIngress{ + {IP: "192.168.1.2", Hostname: "traefik2"}, + {IP: "192.168.1.1", Hostname: "traefik"}, + }, + expectedEqual: true, + }, + { + desc: "different ordered content", + aSlice: []netv1.IngressLoadBalancerIngress{ + {IP: "192.168.1.1", Hostname: "traefik"}, + {IP: "192.168.1.2", Hostname: "traefik2"}, + }, + bSlice: []netv1.IngressLoadBalancerIngress{ + {IP: "192.168.1.1", Hostname: "traefik"}, + {IP: "192.168.1.2", Hostname: "traefik"}, + }, + expectedEqual: false, + }, + { + desc: "different unordered content", + aSlice: []netv1.IngressLoadBalancerIngress{ + {IP: "192.168.1.1", Hostname: "traefik"}, + {IP: "192.168.1.2", Hostname: "traefik2"}, + }, + bSlice: []netv1.IngressLoadBalancerIngress{ + {IP: "192.168.1.2", Hostname: "traefik3"}, + {IP: "192.168.1.1", Hostname: "traefik"}, + }, + expectedEqual: false, + }, + } + + for _, test := range testCases { + t.Run(test.desc, func(t *testing.T) { + t.Parallel() + + gotEqual := isLoadBalancerIngressEquals(test.aSlice, test.bSlice) + assert.Equal(t, test.expectedEqual, gotEqual) + }) + } +} + +func TestClientIgnoresHelmOwnedSecrets(t *testing.T) { + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "secret", + }, + } + helmSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "helm-secret", + Labels: map[string]string{ + "owner": "helm", + }, + }, + } + + kubeClient := kubefake.NewClientset(helmSecret, secret) + + discovery, _ := kubeClient.Discovery().(*discoveryfake.FakeDiscovery) + discovery.FakedServerVersion = &kversion.Info{ + GitVersion: "v1.19", + } + + client := newClient(kubeClient) + + eventCh, err := client.WatchAll(t.Context(), "", "") + require.NoError(t, err) + + select { + case event := <-eventCh: + secret, ok := event.(*corev1.Secret) + require.True(t, ok) + + assert.NotEqual(t, "helm-secret", secret.Name) + case <-time.After(50 * time.Millisecond): + assert.Fail(t, "expected to receive event for secret") + } + + select { + case <-eventCh: + assert.Fail(t, "received more than one event") + case <-time.After(50 * time.Millisecond): + } + + _, err = client.GetSecret("default", "secret") + require.NoError(t, err) + + _, err = client.GetSecret("default", "helm-secret") + assert.True(t, kerror.IsNotFound(err)) +} + +func TestClientIgnoresEmptyEndpointSliceUpdates(t *testing.T) { + emptyEndpointSlice := &discoveryv1.EndpointSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: "empty-endpointslice", + Namespace: "test", + ResourceVersion: "1244", + Annotations: map[string]string{ + "test-annotation": "_", + }, + }, + } + + samplePortName := "testing" + samplePortNumber := int32(1337) + samplePortProtocol := corev1.ProtocolTCP + sampleAddressReady := true + filledEndpointSlice := &discoveryv1.EndpointSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: "filled-endpointslice", + Namespace: "test", + ResourceVersion: "1234", + }, + AddressType: discoveryv1.AddressTypeIPv4, + Endpoints: []discoveryv1.Endpoint{{ + Addresses: []string{"10.13.37.1"}, + Conditions: discoveryv1.EndpointConditions{ + Ready: &sampleAddressReady, + }, + }}, + Ports: []discoveryv1.EndpointPort{{ + Name: &samplePortName, + Port: &samplePortNumber, + Protocol: &samplePortProtocol, + }}, + } + + kubeClient := kubefake.NewClientset(emptyEndpointSlice, filledEndpointSlice) + + discovery, _ := kubeClient.Discovery().(*discoveryfake.FakeDiscovery) + discovery.FakedServerVersion = &kversion.Info{ + GitVersion: "v1.19", + } + + client := newClient(kubeClient) + + eventCh, err := client.WatchAll(t.Context(), "", "") + require.NoError(t, err) + + select { + case event := <-eventCh: + ep, ok := event.(*discoveryv1.EndpointSlice) + require.True(t, ok) + + assert.True(t, ep.Name == "empty-endpointslice" || ep.Name == "filled-endpointslice") + case <-time.After(50 * time.Millisecond): + assert.Fail(t, "expected to receive event for endpointslices") + } + + emptyEndpointSlice, err = kubeClient.DiscoveryV1().EndpointSlices("test").Get(t.Context(), "empty-endpointslice", metav1.GetOptions{}) + assert.NoError(t, err) + + // Update endpoint annotation and resource version (apparently not done by fake client itself) + // to show an update that should not trigger an update event on our eventCh. + // This reflects the behavior of kubernetes controllers which use endpoint annotations for leader election. + emptyEndpointSlice.Annotations["test-annotation"] = "___" + emptyEndpointSlice.ResourceVersion = "1245" + _, err = kubeClient.DiscoveryV1().EndpointSlices("test").Update(t.Context(), emptyEndpointSlice, metav1.UpdateOptions{}) + require.NoError(t, err) + + select { + case event := <-eventCh: + ep, ok := event.(*discoveryv1.EndpointSlice) + require.True(t, ok) + + assert.Fail(t, "didn't expect to receive event for empty endpointslice update", ep.Name) + case <-time.After(50 * time.Millisecond): + } + + filledEndpointSlice, err = kubeClient.DiscoveryV1().EndpointSlices("test").Get(t.Context(), "filled-endpointslice", metav1.GetOptions{}) + assert.NoError(t, err) + + filledEndpointSlice.Endpoints[0].Addresses[0] = "10.13.37.2" + filledEndpointSlice.ResourceVersion = "1235" + _, err = kubeClient.DiscoveryV1().EndpointSlices("test").Update(t.Context(), filledEndpointSlice, metav1.UpdateOptions{}) + require.NoError(t, err) + + select { + case event := <-eventCh: + ep, ok := event.(*discoveryv1.EndpointSlice) + require.True(t, ok) + + assert.Equal(t, "filled-endpointslice", ep.Name) + case <-time.After(50 * time.Millisecond): + assert.Fail(t, "expected to receive event for filled endpointslice") + } + + select { + case <-eventCh: + assert.Fail(t, "received more than one event") + case <-time.After(50 * time.Millisecond): + } + + newPortNumber := int32(42) + filledEndpointSlice.Ports[0].Port = &newPortNumber + filledEndpointSlice.ResourceVersion = "1236" + _, err = kubeClient.DiscoveryV1().EndpointSlices("test").Update(t.Context(), filledEndpointSlice, metav1.UpdateOptions{}) + require.NoError(t, err) + + select { + case event := <-eventCh: + ep, ok := event.(*discoveryv1.EndpointSlice) + require.True(t, ok) + + assert.Equal(t, "filled-endpointslice", ep.Name) + case <-time.After(50 * time.Millisecond): + assert.Fail(t, "expected to receive event for filled endpointslice") + } + + select { + case <-eventCh: + assert.Fail(t, "received more than one event") + case <-time.After(50 * time.Millisecond): + } +} diff --git a/pkg/provider/kubernetes/ingress-nginx/convert.go b/pkg/provider/kubernetes/ingress-nginx/convert.go new file mode 100644 index 000000000..f2a27e034 --- /dev/null +++ b/pkg/provider/kubernetes/ingress-nginx/convert.go @@ -0,0 +1,69 @@ +package ingressnginx + +import ( + "errors" + + corev1 "k8s.io/api/core/v1" + netv1 "k8s.io/api/networking/v1" +) + +type marshaler interface { + Marshal() ([]byte, error) +} + +type unmarshaler interface { + Unmarshal(data []byte) error +} + +type LoadBalancerIngress interface { + corev1.LoadBalancerIngress | netv1.IngressLoadBalancerIngress +} + +// convertSlice converts slice of LoadBalancerIngress to slice of LoadBalancerIngress. +// O (Bar), I (Foo) => []Bar. +func convertSlice[O LoadBalancerIngress, I LoadBalancerIngress](loadBalancerIngresses []I) ([]O, error) { + var results []O + + for _, loadBalancerIngress := range loadBalancerIngresses { + mar, ok := any(&loadBalancerIngress).(marshaler) + if !ok { + // All the pointer of types related to the interface LoadBalancerIngress are compatible with the interface marshaler. + continue + } + + um, err := convert[O](mar) + if err != nil { + return nil, err + } + + v, ok := any(*um).(O) + if !ok { + continue + } + + results = append(results, v) + } + + return results, nil +} + +// convert must only be used with unmarshaler and marshaler compatible types. +func convert[T any](input marshaler) (*T, error) { + data, err := input.Marshal() + if err != nil { + return nil, err + } + + var output T + um, ok := any(&output).(unmarshaler) + if !ok { + return nil, errors.New("the output type doesn't implement unmarshaler interface") + } + + err = um.Unmarshal(data) + if err != nil { + return nil, err + } + + return &output, nil +} diff --git a/pkg/provider/kubernetes/ingress-nginx/convert_test.go b/pkg/provider/kubernetes/ingress-nginx/convert_test.go new file mode 100644 index 000000000..a0bff8834 --- /dev/null +++ b/pkg/provider/kubernetes/ingress-nginx/convert_test.go @@ -0,0 +1,77 @@ +package ingressnginx + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + netv1 "k8s.io/api/networking/v1" + "k8s.io/utils/ptr" +) + +func Test_convertSlice_corev1_to_networkingv1(t *testing.T) { + g := []corev1.LoadBalancerIngress{ + { + IP: "132456", + Hostname: "foo", + Ports: []corev1.PortStatus{ + { + Port: 123, + Protocol: "https", + Error: ptr.To("test"), + }, + }, + }, + } + + actual, err := convertSlice[netv1.IngressLoadBalancerIngress](g) + require.NoError(t, err) + + expected := []netv1.IngressLoadBalancerIngress{ + { + IP: "132456", + Hostname: "foo", + Ports: []netv1.IngressPortStatus{ + { + Port: 123, + Protocol: "https", + Error: ptr.To("test"), + }, + }, + }, + } + + assert.Equal(t, expected, actual) +} + +func Test_convert(t *testing.T) { + g := &corev1.LoadBalancerIngress{ + IP: "132456", + Hostname: "foo", + Ports: []corev1.PortStatus{ + { + Port: 123, + Protocol: "https", + Error: ptr.To("test"), + }, + }, + } + + actual, err := convert[netv1.IngressLoadBalancerIngress](g) + require.NoError(t, err) + + expected := &netv1.IngressLoadBalancerIngress{ + IP: "132456", + Hostname: "foo", + Ports: []netv1.IngressPortStatus{ + { + Port: 123, + Protocol: "https", + Error: ptr.To("test"), + }, + }, + } + + assert.Equal(t, expected, actual) +} diff --git a/pkg/provider/kubernetes/ingress-nginx/fixtures/ingressclasses.yml b/pkg/provider/kubernetes/ingress-nginx/fixtures/ingressclasses.yml new file mode 100644 index 000000000..33e88a25d --- /dev/null +++ b/pkg/provider/kubernetes/ingress-nginx/fixtures/ingressclasses.yml @@ -0,0 +1,7 @@ +--- +apiVersion: networking.k8s.io/v1 +kind: IngressClass +metadata: + name: nginx +spec: + controller: k8s.io/ingress-nginx \ No newline at end of file diff --git a/pkg/provider/kubernetes/ingress-nginx/fixtures/ingresses/01-ingress-with-basicauth.yml b/pkg/provider/kubernetes/ingress-nginx/fixtures/ingresses/01-ingress-with-basicauth.yml new file mode 100644 index 000000000..cd750a9f1 --- /dev/null +++ b/pkg/provider/kubernetes/ingress-nginx/fixtures/ingresses/01-ingress-with-basicauth.yml @@ -0,0 +1,37 @@ +--- +kind: Ingress +apiVersion: networking.k8s.io/v1 +metadata: + name: ingress-with-basicauth + namespace: default + annotations: + # Configuration basic authentication for the Ingress + nginx.ingress.kubernetes.io/auth-type: "basic" + nginx.ingress.kubernetes.io/auth-secret-type: "auth-file" + nginx.ingress.kubernetes.io/auth-secret: "default/basic-auth" + nginx.ingress.kubernetes.io/auth-realm: "Authentication Required" + +spec: + ingressClassName: nginx + rules: + - host: whoami.localhost + http: + paths: + - path: /basicauth + pathType: Exact + backend: + service: + name: whoami + port: + number: 80 + +--- +kind: Secret +apiVersion: v1 +metadata: + name: basic-auth + namespace: default +type: Opaque +data: + # user:password + auth: dXNlcjp7U0hBfVc2cGg1TW01UHo4R2dpVUxiUGd6RzM3bWo5Zz0= \ No newline at end of file diff --git a/pkg/provider/kubernetes/ingress-nginx/fixtures/ingresses/02-ingress-with-forwardauth.yml b/pkg/provider/kubernetes/ingress-nginx/fixtures/ingresses/02-ingress-with-forwardauth.yml new file mode 100644 index 000000000..220499792 --- /dev/null +++ b/pkg/provider/kubernetes/ingress-nginx/fixtures/ingresses/02-ingress-with-forwardauth.yml @@ -0,0 +1,24 @@ +--- +kind: Ingress +apiVersion: networking.k8s.io/v1 +metadata: + name: ingress-with-forwardauth + namespace: default + annotations: + nginx.ingress.kubernetes.io/auth-url: "http://whoami.default.svc/" + nginx.ingress.kubernetes.io/auth-method: "GET" + nginx.ingress.kubernetes.io/auth-response-headers: "X-Foo" + +spec: + ingressClassName: nginx + rules: + - host: whoami.localhost + http: + paths: + - path: /forwardauth + pathType: Exact + backend: + service: + name: whoami + port: + number: 80 diff --git a/pkg/provider/kubernetes/ingress-nginx/fixtures/ingresses/03-ingress-with-ssl-redirect.yml b/pkg/provider/kubernetes/ingress-nginx/fixtures/ingresses/03-ingress-with-ssl-redirect.yml new file mode 100644 index 000000000..57f8dd420 --- /dev/null +++ b/pkg/provider/kubernetes/ingress-nginx/fixtures/ingresses/03-ingress-with-ssl-redirect.yml @@ -0,0 +1,75 @@ +--- +kind: Ingress +apiVersion: networking.k8s.io/v1 +metadata: + name: ingress-with-ssl-redirect + namespace: default + + +spec: + ingressClassName: nginx + rules: + - host: sslredirect.localhost + http: + paths: + - path: / + pathType: Exact + backend: + service: + name: whoami + port: + number: 80 + tls: + - hosts: + - sslredirect.localhost + secretName: whoami-tls + +--- +kind: Ingress +apiVersion: networking.k8s.io/v1 +metadata: + name: ingress-without-ssl-redirect + namespace: default + annotations: + nginx.ingress.kubernetes.io/ssl-redirect: "false" + +spec: + ingressClassName: nginx + rules: + - host: withoutsslredirect.localhost + http: + paths: + - path: / + pathType: Exact + backend: + service: + name: whoami + port: + number: 80 + tls: + - hosts: + - withoutsslredirect.localhost + secretName: whoami-tls + +--- +kind: Ingress +apiVersion: networking.k8s.io/v1 +metadata: + name: ingress-with-force-ssl-redirect + namespace: default + annotations: + nginx.ingress.kubernetes.io/force-ssl-redirect: "true" + +spec: + ingressClassName: nginx + rules: + - host: forcesslredirect.localhost + http: + paths: + - path: / + pathType: Exact + backend: + service: + name: whoami + port: + number: 80 diff --git a/pkg/provider/kubernetes/ingress-nginx/fixtures/ingresses/04-ingress-with-ssl-passthrough.yml b/pkg/provider/kubernetes/ingress-nginx/fixtures/ingresses/04-ingress-with-ssl-passthrough.yml new file mode 100644 index 000000000..70f531c7c --- /dev/null +++ b/pkg/provider/kubernetes/ingress-nginx/fixtures/ingresses/04-ingress-with-ssl-passthrough.yml @@ -0,0 +1,22 @@ +--- +kind: Ingress +apiVersion: networking.k8s.io/v1 +metadata: + name: ingress-with-ssl-passthrough + namespace: default + annotations: + nginx.ingress.kubernetes.io/ssl-passthrough: "true" + +spec: + ingressClassName: nginx + rules: + - host: passthrough.whoami.localhost + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: whoami-tls + port: + number: 443 \ No newline at end of file diff --git a/pkg/provider/kubernetes/ingress-nginx/fixtures/ingresses/05-ingress-with-default-backend.yml b/pkg/provider/kubernetes/ingress-nginx/fixtures/ingresses/05-ingress-with-default-backend.yml new file mode 100644 index 000000000..8809e7164 --- /dev/null +++ b/pkg/provider/kubernetes/ingress-nginx/fixtures/ingresses/05-ingress-with-default-backend.yml @@ -0,0 +1,24 @@ +--- +kind: Ingress +apiVersion: networking.k8s.io/v1 +metadata: + name: ingress-with-default-backend + namespace: default + +spec: + defaultBackend: + service: + name: whoami-default + port: + number: 80 + + rules: + - http: + paths: + - path: / + pathType: Exact + backend: + service: + name: whoami + port: + number: 80 diff --git a/pkg/provider/kubernetes/ingress-nginx/fixtures/ingresses/05-ingress-with-default-backend2.yml b/pkg/provider/kubernetes/ingress-nginx/fixtures/ingresses/05-ingress-with-default-backend2.yml new file mode 100644 index 000000000..a9c350168 --- /dev/null +++ b/pkg/provider/kubernetes/ingress-nginx/fixtures/ingresses/05-ingress-with-default-backend2.yml @@ -0,0 +1,108 @@ + +--- +kind: Ingress +apiVersion: networking.k8s.io/v1 +metadata: + name: ingress-with-default-backend2 + namespace: default +# annotations: +# nginx.ingress.kubernetes.io/force-ssl-redirect: "true" + +# annotations: +## Configuration basic authentication for the Ingress +# nginx.ingress.kubernetes.io/auth-type: "basic" +# nginx.ingress.kubernetes.io/auth-secret-type: "auth-file" +# nginx.ingress.kubernetes.io/auth-secret: "default/basic-auth" +# nginx.ingress.kubernetes.io/auth-realm: "Authentication Required" + +spec: + defaultBackend: + service: + name: whoami-default2 + port: + number: 80 + + rules: + - host: dd.localhost + http: + paths: + - path: / + pathType: Exact + backend: + service: + name: whoami + port: + number: 443 +# +# tls: +# - hosts: +# - dd.localhost +# secretName: whoami-tls +# +#--- +#kind: Secret +#apiVersion: v1 +#metadata: +# name: whoami-tls +# namespace: default +# +#type: opaque +#stringData: +# tls.crt: | +# -----BEGIN CERTIFICATE----- +# MIIEXjCCAsagAwIBAgIQAJmtU2qHBlD9D2HZFZLMeDANBgkqhkiG9w0BAQsFADCB +# jzEeMBwGA1UEChMVbWtjZXJ0IGRldmVsb3BtZW50IENBMTIwMAYDVQQLDClyb21h +# aW5AY29udGFpbm91cy5ob21lIChSb21haW4gVHJpYm90dMOpKTE5MDcGA1UEAwww +# bWtjZXJ0IHJvbWFpbkBjb250YWlub3VzLmhvbWUgKFJvbWFpbiBUcmlib3R0w6kp +# MB4XDTI1MDYxMDE1NDE0NFoXDTI3MDkxMDE1NDE0NFowXzEnMCUGA1UEChMebWtj +# ZXJ0IGRldmVsb3BtZW50IGNlcnRpZmljYXRlMTQwMgYDVQQLDCtyb21haW5ATWFj +# Qm9vay1Qcm8ubG9jYWwgKFJvbWFpbiBUcmlib3R0w6kpMIIBIjANBgkqhkiG9w0B +# AQEFAAOCAQ8AMIIBCgKCAQEAq3dajz+RgY+VUXvKKtHFFVd+0URcpDRgN+SJOxP/ +# 1uZG2U57DMvTiVy6zfpYo7QPzyEAUwbRTMMgxZV5oy1JPkGzV5kc08GUT3Lh1Azf +# LVPX/K1nA+k7p9+kuMsfkHVABMawRpnWo215T9pjGaTKERA2EaNvrSdq73k6raVn +# DnnmvUgWGPvxTetaLu0AVQscGyrTfQNMB8BwC+JEQJKocenJ0ve5l9/yv9543P2G +# 6UcOv71lDOBNPyltrc4sXfGC2vB1APbp80BVfkZDiF+8Gr8wGJrkd75Esp/xetFV +# yZ6NKO9ZsGZ2E14/qxfvASHGNFNQJafqhnuGbmky8AeaawIDAQABo2UwYzAOBgNV +# HQ8BAf8EBAMCBaAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwHwYDVR0jBBgwFoAUjIHl +# 1gcu+iVVHCicC14yHQiRojgwGwYDVR0RBBQwEoIQd2hvYW1pLmxvY2FsaG9zdDAN +# BgkqhkiG9w0BAQsFAAOCAYEAo/f0ADJwnkOakCHcYCNSqRY/VzRIQSQK3wfDq3bD +# 8EDxGrGPYHOIL+u/Up4RO2/9vLEnFpWb30A8z/qZTKKD+rMuU3qTcCJ2tsB3DAIV +# T+b2GmJYjURf1gqe/NNXnzZqgkoP+bHx6iNvDr1kmc3pZshayz+FxzNmjgpbKl2G +# SgfFLnJDm7hwTC9JFoPyzb586Q0OGQKCJpDMy6pi1MAQl2RWiKyrgo1mhYnSxQmI +# qbJbxYlegRRQQPD6YEJcL5lwILVW3TXcGrK+zuMD+xWznDTBg2BxbF2umG8jmXPH +# 04gRfjlMNLEYSrNEU8EOa/lXebcxnlz6meFOgfYKmSHxL+kwjTUuppDV/qP9U+VS +# /ozJ85VS8iEx1obqZGgqgwcBKMRYzuRnW1XEScGUOK9/cs9mGoXG9uafKb7ekFQc +# wU0j0FoUVzc50WWEjCGFU/dS/2HXUXU/Rcf+uULC10ORplwrB5XdXZYxowh7T//U +# yh86E4+M0LHyZH0vUwDoBk/1 +# -----END CERTIFICATE----- +# +# +# tls.key: | +# -----BEGIN PRIVATE KEY----- +# MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCrd1qPP5GBj5VR +# e8oq0cUVV37RRFykNGA35Ik7E//W5kbZTnsMy9OJXLrN+lijtA/PIQBTBtFMwyDF +# lXmjLUk+QbNXmRzTwZRPcuHUDN8tU9f8rWcD6Tun36S4yx+QdUAExrBGmdajbXlP +# 2mMZpMoREDYRo2+tJ2rveTqtpWcOeea9SBYY+/FN61ou7QBVCxwbKtN9A0wHwHAL +# 4kRAkqhx6cnS97mX3/K/3njc/YbpRw6/vWUM4E0/KW2tzixd8YLa8HUA9unzQFV+ +# RkOIX7wavzAYmuR3vkSyn/F60VXJno0o71mwZnYTXj+rF+8BIcY0U1Alp+qGe4Zu +# aTLwB5prAgMBAAECggEAVNpLxnf+2c7kZd6MvYPxtA4IhCcAcYI5228NOl87TG3I +# weFEo6B6no91IlmxY9HHwQjj0DKfgQ1POnguKcJPbK+2wLLUwTYa3vZLK1TzXMsR +# J8noINda3kiei5R5mlNryvFIaqfWwCl8zzeTsy0JkkgjebcXnOjU0o17rFMeHNsH +# A3iFWWnHtJkn2OaVtOOgsyjJ9oAnGX0AE4cVp7ZZTerpaYTXzkCphbwRi00IEbCk +# 1bn7gPcBQRoxs12GJUUuy/sopQRA51PE//CnV2pkGuDFWBhFBBKYdsHaTUwmTb5P +# l6S5CuCtw44NkTPetTe2sn9DpOIlR7PmojQndmKkgQKBgQDJ6/RDueJCBxSYS6bh +# 7dTPRphJvntoJHs9Q/NNjKdQhxv0vIIdtRk88Q2qhjjlCzHb1RtD5Jsl+D+TxOdG +# wR1/E8+hdbRKv+WACywa38aBPuZSEj89bnyPyQfzs5TtzD5JsdUHT4l5Eudth6Gv +# w14dFKria8WiEd7X2GodnlZX/wKBgQDZY1QBNjAHsi7QJJSvbPKwK8RygvdNJEem +# FYxhjtHzOfUttjyDXDSGheY3/VzKi2rGgVAHLi+qbvwkURn4qT3xtV5Lpi+BLWHP +# Gwepisd9P5TrN0DGQojWjzYatN9MYRzX0JynIB+alabN2bG7kfPPsHikAA7pRxLH +# 7EwMBDGdlQKBgBqd9uoCk9e+VTGqL0py7m2QUbzO1jepL3GpBmZ/lwKffMjrHH/M +# ApKs9+81mERhEGZ5FgoCFY2Qxti0yQPjqv64XtNaz7RWzWrujhbQzrr0zqmc7Cct +# 7E+L4Xd3gbdDCCbwwTMgge+q1UTz7xVbPIm60rfcGwY9MtHjHkHfQGSDAoGBAIA/ +# CAT6+dTgepuSqSDg7j+eYnOH7etVlutVVQ8M2bFbJNiF5Sc900L1ZX7seryHCUP4 +# b8T8q2Qpu5iVO/QlrASXkfyhGu9jXYt4D8omtE+gnfMyEoWkJOQncqzIvd9qf0CW +# soQqAFsLJG/WmPLmRObm3hUqb6GRq3PEZIzGQJsNAoGBAJEN0ZkrIkNK+Jjd1oNB +# AnwgLA0qyAHqJxPig45Nudhb6Jw4ub/hKG9bCrLpcBM57Lue535e2HtQ5Ed22Pim +# 0m7bQkvrIQYjflW99RsfkiH5qJsiTy9O92iKgGtJAJ80vTkIggAbsnzOHlZvR0Fr +# +GhYvMt0TxpugicUqguSSUZp +# -----END PRIVATE KEY----- diff --git a/pkg/provider/kubernetes/ingress-nginx/fixtures/ingresses/06-ingress-with-sticky.yml b/pkg/provider/kubernetes/ingress-nginx/fixtures/ingresses/06-ingress-with-sticky.yml new file mode 100644 index 000000000..ac56745cc --- /dev/null +++ b/pkg/provider/kubernetes/ingress-nginx/fixtures/ingresses/06-ingress-with-sticky.yml @@ -0,0 +1,28 @@ +--- +kind: Ingress +apiVersion: networking.k8s.io/v1 +metadata: + name: ingress-with-sticky + namespace: default + annotations: + nginx.ingress.kubernetes.io/affinity: cookie + nginx.ingress.kubernetes.io/session-cookie-name: foobar + nginx.ingress.kubernetes.io/session-cookie-secure: "true" + nginx.ingress.kubernetes.io/session-cookie-path: "/foobar" + nginx.ingress.kubernetes.io/session-cookie-domain: "foo.localhost" + nginx.ingress.kubernetes.io/session-cookie-samesite: "None" + nginx.ingress.kubernetes.io/session-cookie-max-age: "42" + +spec: + ingressClassName: nginx + rules: + - host: sticky.localhost + http: + paths: + - path: / + pathType: Exact + backend: + service: + name: whoami + port: + number: 80 diff --git a/pkg/provider/kubernetes/ingress-nginx/fixtures/ingresses/07-ingress-with-proxy-ssl.yml b/pkg/provider/kubernetes/ingress-nginx/fixtures/ingresses/07-ingress-with-proxy-ssl.yml new file mode 100644 index 000000000..4d3f4dc63 --- /dev/null +++ b/pkg/provider/kubernetes/ingress-nginx/fixtures/ingresses/07-ingress-with-proxy-ssl.yml @@ -0,0 +1,37 @@ +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: ingress-with-proxy-ssl + namespace: default + annotations: + nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" # HTTP, HTTPS, AUTO_HTTP, GRPC, GRPCS and FCGI + nginx.ingress.kubernetes.io/proxy-ssl-secret: "default/ingress-with-proxy-ssl" + nginx.ingress.kubernetes.io/proxy-ssl-verify: "on" + nginx.ingress.kubernetes.io/proxy-ssl-verify-depth: "1" + nginx.ingress.kubernetes.io/proxy-ssl-server-name: "whoami.localhost" + nginx.ingress.kubernetes.io/proxy-ssl-name: "whoami.localhost" + +spec: + ingressClassName: nginx + rules: + - host: proxy-ssl.localhost + http: + paths: + - path: / + pathType: Exact + backend: + service: + name: whoami-tls + port: + number: 443 + +--- +kind: Secret +apiVersion: v1 +metadata: + namespace: default + name: ingress-with-proxy-ssl + +data: + ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0t diff --git a/pkg/provider/kubernetes/ingress-nginx/fixtures/ingresses/08-ingress-with-cors.yml b/pkg/provider/kubernetes/ingress-nginx/fixtures/ingresses/08-ingress-with-cors.yml new file mode 100644 index 000000000..b862b9bf6 --- /dev/null +++ b/pkg/provider/kubernetes/ingress-nginx/fixtures/ingresses/08-ingress-with-cors.yml @@ -0,0 +1,28 @@ +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: ingress-with-cors + namespace: default + annotations: + nginx.ingress.kubernetes.io/enable-cors: "true" + nginx.ingress.kubernetes.io/cors-allow-credentials: "true" + nginx.ingress.kubernetes.io/cors-expose-headers: "X-Forwarded-For, X-Forwarded-Host" + nginx.ingress.kubernetes.io/cors-allow-headers: "X-Foo" + nginx.ingress.kubernetes.io/cors-allow-methods: "PUT, GET, POST, OPTIONS" + nginx.ingress.kubernetes.io/cors-allow-origin: "*" + nginx.ingress.kubernetes.io/cors-max-age: "42" + +spec: + ingressClassName: nginx + rules: + - host: cors.localhost + http: + paths: + - path: / + pathType: Exact + backend: + service: + name: whoami + port: + number: 80 diff --git a/pkg/provider/kubernetes/ingress-nginx/fixtures/ingresses/09-ingress-with-service-upstream.yml b/pkg/provider/kubernetes/ingress-nginx/fixtures/ingresses/09-ingress-with-service-upstream.yml new file mode 100644 index 000000000..2a2adcec2 --- /dev/null +++ b/pkg/provider/kubernetes/ingress-nginx/fixtures/ingresses/09-ingress-with-service-upstream.yml @@ -0,0 +1,22 @@ +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: ingress-with-service-upstream + namespace: default + annotations: + nginx.ingress.kubernetes.io/service-upstream: "true" + +spec: + ingressClassName: nginx + rules: + - host: service-upstream.localhost + http: + paths: + - path: / + pathType: Exact + backend: + service: + name: whoami + port: + number: 80 diff --git a/pkg/provider/kubernetes/ingress-nginx/fixtures/secrets.yml b/pkg/provider/kubernetes/ingress-nginx/fixtures/secrets.yml new file mode 100644 index 000000000..7c53f7fb0 --- /dev/null +++ b/pkg/provider/kubernetes/ingress-nginx/fixtures/secrets.yml @@ -0,0 +1,9 @@ +kind: Secret +apiVersion: v1 +metadata: + namespace: default + name: whoami-tls + +data: + tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0t + tls.key: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0t diff --git a/pkg/provider/kubernetes/ingress-nginx/fixtures/services.yml b/pkg/provider/kubernetes/ingress-nginx/fixtures/services.yml new file mode 100644 index 000000000..b658083aa --- /dev/null +++ b/pkg/provider/kubernetes/ingress-nginx/fixtures/services.yml @@ -0,0 +1,80 @@ +kind: Service +apiVersion: v1 +metadata: + name: whoami + namespace: default + +spec: + clusterIP: 10.10.10.1 + ports: + - name: web2 + protocol: TCP + port: 8000 + targetPort: web2 + - name: web + protocol: TCP + port: 80 + targetPort: web + selector: + app: whoami + task: whoami + +--- +kind: EndpointSlice +apiVersion: discovery.k8s.io/v1 +metadata: + name: whoami + namespace: default + labels: + kubernetes.io/service-name: whoami + +addressType: IPv4 +ports: + - name: web + port: 80 + - name: web2 + port: 8000 +endpoints: + - addresses: + - 10.10.0.1 + - 10.10.0.2 + conditions: + ready: true + +--- +apiVersion: v1 +kind: Service +metadata: + name: whoami-tls + namespace: default + +spec: + ports: + - name: websecure + protocol: TCP + appProtocol: https + port: 443 + targetPort: websecure + selector: + app: whoami-tls + task: whoami + +--- +kind: EndpointSlice +apiVersion: discovery.k8s.io/v1 +metadata: + name: whoami-tls + namespace: default + labels: + kubernetes.io/service-name: whoami-tls + +addressType: IPv4 +ports: + - name: websecure + port: 8443 +endpoints: + - addresses: + - 10.10.0.5 + - 10.10.0.6 + conditions: + ready: true diff --git a/pkg/provider/kubernetes/ingress-nginx/kubernetes.go b/pkg/provider/kubernetes/ingress-nginx/kubernetes.go new file mode 100644 index 000000000..d35837947 --- /dev/null +++ b/pkg/provider/kubernetes/ingress-nginx/kubernetes.go @@ -0,0 +1,1118 @@ +package ingressnginx + +import ( + "context" + "errors" + "fmt" + "maps" + "math" + "net" + "os" + "regexp" + "slices" + "strconv" + "strings" + "time" + + "github.com/cenkalti/backoff/v4" + "github.com/mitchellh/hashstructure" + "github.com/rs/zerolog/log" + ptypes "github.com/traefik/paerser/types" + "github.com/traefik/traefik/v3/pkg/config/dynamic" + "github.com/traefik/traefik/v3/pkg/job" + "github.com/traefik/traefik/v3/pkg/logs" + "github.com/traefik/traefik/v3/pkg/provider" + "github.com/traefik/traefik/v3/pkg/provider/kubernetes/k8s" + "github.com/traefik/traefik/v3/pkg/safe" + "github.com/traefik/traefik/v3/pkg/tls" + "github.com/traefik/traefik/v3/pkg/types" + corev1 "k8s.io/api/core/v1" + netv1 "k8s.io/api/networking/v1" + "k8s.io/utils/ptr" +) + +const ( + providerName = "kubernetesingressnginx" + + annotationIngressClass = "kubernetes.io/ingress.class" + + defaultControllerName = "k8s.io/ingress-nginx" + defaultAnnotationValue = "nginx" + + defaultBackendName = "default-backend" + defaultBackendTLSName = "default-backend-tls" +) + +type backendAddress struct { + Address string + Fenced bool +} + +type namedServersTransport struct { + Name string + ServersTransport *dynamic.ServersTransport +} + +type certBlocks struct { + CA *types.FileOrContent + Certificate *tls.Certificate +} + +// Provider holds configurations of the provider. +type Provider struct { + Endpoint string `description:"Kubernetes server endpoint (required for external cluster client)." json:"endpoint,omitempty" toml:"endpoint,omitempty" yaml:"endpoint,omitempty"` + Token types.FileOrContent `description:"Kubernetes bearer token (not needed for in-cluster client). It accepts either a token value or a file path to the token." json:"token,omitempty" toml:"token,omitempty" yaml:"token,omitempty" loggable:"false"` + CertAuthFilePath string `description:"Kubernetes certificate authority file path (not needed for in-cluster client)." json:"certAuthFilePath,omitempty" toml:"certAuthFilePath,omitempty" yaml:"certAuthFilePath,omitempty"` + ThrottleDuration ptypes.Duration `description:"Ingress refresh throttle duration." json:"throttleDuration,omitempty" toml:"throttleDuration,omitempty" yaml:"throttleDuration,omitempty" export:"true"` + + WatchNamespace string `description:"Namespace the controller watches for updates to Kubernetes objects. All namespaces are watched if this parameter is left empty." json:"watchNamespace,omitempty" toml:"watchNamespace,omitempty" yaml:"watchNamespace,omitempty" export:"true"` + WatchNamespaceSelector string `description:"Selector selects namespaces the controller watches for updates to Kubernetes objects." json:"watchNamespaceSelector,omitempty" toml:"watchNamespaceSelector,omitempty" yaml:"watchNamespaceSelector,omitempty" export:"true"` + + IngressClass string `description:"Name of the ingress class this controller satisfies." json:"ingressClass,omitempty" toml:"ingressClass,omitempty" yaml:"ingressClass,omitempty" export:"true"` + ControllerClass string `description:"Ingress Class Controller value this controller satisfies." json:"controllerClass,omitempty" toml:"controllerClass,omitempty" yaml:"controllerClass,omitempty" export:"true"` + WatchIngressWithoutClass bool `description:"Define if Ingress Controller should also watch for Ingresses without an IngressClass or the annotation specified." json:"watchIngressWithoutClass,omitempty" toml:"watchIngressWithoutClass,omitempty" yaml:"watchIngressWithoutClass,omitempty" export:"true"` + IngressClassByName bool `description:"Define if Ingress Controller should watch for Ingress Class by Name together with Controller Class." json:"ingressClassByName,omitempty" toml:"ingressClassByName,omitempty" yaml:"ingressClassByName,omitempty" export:"true"` + + // TODO: support report-node-internal-ip-address and update-status. + PublishService string `description:"Service fronting the Ingress controller. Takes the form 'namespace/name'." json:"publishService,omitempty" toml:"publishService,omitempty" yaml:"publishService,omitempty" export:"true"` + PublishStatusAddress []string `description:"Customized address (or addresses, separated by comma) to set as the load-balancer status of Ingress objects this controller satisfies." json:"publishStatusAddress,omitempty" toml:"publishStatusAddress,omitempty" yaml:"publishStatusAddress,omitempty"` + + DefaultBackendService string `description:"Service used to serve HTTP requests not matching any known server name (catch-all). Takes the form 'namespace/name'." json:"defaultBackendService,omitempty" toml:"defaultBackendService,omitempty" yaml:"defaultBackendService,omitempty" export:"true"` + DisableSvcExternalName bool `description:"Disable support for Services of type ExternalName." json:"disableSvcExternalName,omitempty" toml:"disableSvcExternalName,omitempty" yaml:"disableSvcExternalName,omitempty" export:"true"` + + defaultBackendServiceNamespace string + defaultBackendServiceName string + + k8sClient *clientWrapper + lastConfiguration safe.Safe +} + +func (p *Provider) SetDefaults() { + p.IngressClass = defaultAnnotationValue + p.ControllerClass = defaultControllerName +} + +// Init the provider. +func (p *Provider) Init() error { + // Validates and parses the default backend configuration. + if p.DefaultBackendService != "" { + parts := strings.Split(p.DefaultBackendService, "/") + if len(parts) != 2 { + return fmt.Errorf("invalid default backend service format: %s, expected 'namespace/name'", p.DefaultBackendService) + } + p.defaultBackendServiceNamespace = parts[0] + p.defaultBackendServiceName = parts[1] + } + + // Initializes Kubernetes client. + var err error + p.k8sClient, err = p.newK8sClient() + if err != nil { + return fmt.Errorf("creating kubernetes client: %w", err) + } + + return nil +} + +// Provide allows the k8s provider to provide configurations to traefik using the given configuration channel. +func (p *Provider) Provide(configurationChan chan<- dynamic.Message, pool *safe.Pool) error { + logger := log.With().Str(logs.ProviderName, providerName).Logger() + ctxLog := logger.WithContext(context.Background()) + + pool.GoCtx(func(ctxPool context.Context) { + operation := func() error { + eventsChan, err := p.k8sClient.WatchAll(ctxPool, p.WatchNamespace, p.WatchNamespaceSelector) + if err != nil { + logger.Error().Err(err).Msg("Error watching kubernetes events") + timer := time.NewTimer(1 * time.Second) + select { + case <-timer.C: + return err + case <-ctxPool.Done(): + return nil + } + } + + throttleDuration := time.Duration(p.ThrottleDuration) + throttledChan := throttleEvents(ctxLog, throttleDuration, pool, eventsChan) + if throttledChan != nil { + eventsChan = throttledChan + } + + for { + select { + case <-ctxPool.Done(): + return nil + case event := <-eventsChan: + // Note that event is the *first* event that came in during this + // throttling interval -- if we're hitting our throttle, we may have + // dropped events. This is fine, because we don't treat different + // event types differently. But if we do in the future, we'll need to + // track more information about the dropped events. + conf := p.loadConfiguration(ctxLog) + + confHash, err := hashstructure.Hash(conf, nil) + switch { + case err != nil: + logger.Error().Msg("Unable to hash the configuration") + case p.lastConfiguration.Get() == confHash: + logger.Debug().Msgf("Skipping Kubernetes event kind %T", event) + default: + p.lastConfiguration.Set(confHash) + configurationChan <- dynamic.Message{ + ProviderName: providerName, + Configuration: conf, + } + } + + // If we're throttling, we sleep here for the throttle duration to + // enforce that we don't refresh faster than our throttle. time.Sleep + // returns immediately if p.ThrottleDuration is 0 (no throttle). + time.Sleep(throttleDuration) + } + } + } + + notify := func(err error, time time.Duration) { + logger.Error().Err(err).Msgf("Provider error, retrying in %s", time) + } + + err := backoff.RetryNotify(safe.OperationWithRecover(operation), backoff.WithContext(job.NewBackOff(backoff.NewExponentialBackOff()), ctxPool), notify) + if err != nil { + logger.Error().Err(err).Msg("Cannot retrieve data") + } + }) + + return nil +} + +func (p *Provider) newK8sClient() (*clientWrapper, error) { + withEndpoint := "" + if p.Endpoint != "" { + withEndpoint = fmt.Sprintf(" with endpoint %v", p.Endpoint) + } + + switch { + case os.Getenv("KUBERNETES_SERVICE_HOST") != "" && os.Getenv("KUBERNETES_SERVICE_PORT") != "": + log.Info().Msgf("Creating in-cluster Provider client%s", withEndpoint) + return newInClusterClient(p.Endpoint) + case os.Getenv("KUBECONFIG") != "": + log.Info().Msgf("Creating cluster-external Provider client from KUBECONFIG %s", os.Getenv("KUBECONFIG")) + return newExternalClusterClientFromFile(os.Getenv("KUBECONFIG")) + default: + log.Info().Msgf("Creating cluster-external Provider client%s", withEndpoint) + return newExternalClusterClient(p.Endpoint, p.CertAuthFilePath, p.Token) + } +} + +func (p *Provider) loadConfiguration(ctx context.Context) *dynamic.Configuration { + conf := &dynamic.Configuration{ + HTTP: &dynamic.HTTPConfiguration{ + Routers: map[string]*dynamic.Router{}, + Middlewares: map[string]*dynamic.Middleware{}, + Services: map[string]*dynamic.Service{}, + ServersTransports: map[string]*dynamic.ServersTransport{}, + }, + TCP: &dynamic.TCPConfiguration{ + Routers: map[string]*dynamic.TCPRouter{}, + Services: map[string]*dynamic.TCPService{}, + }, + } + + // We configure the default backend when it is configured at the provider level. + if p.defaultBackendServiceNamespace != "" && p.defaultBackendServiceName != "" { + ib := netv1.IngressBackend{Service: &netv1.IngressServiceBackend{Name: p.defaultBackendServiceName}} + svc, err := p.buildService(p.defaultBackendServiceNamespace, ib, ingressConfig{}) + if err != nil { + log.Ctx(ctx).Error().Err(err).Msg("Cannot build default backend service") + return conf + } + + // Add the default backend service router to the configuration. + conf.HTTP.Routers[defaultBackendName] = &dynamic.Router{ + Rule: "PathPrefix(`/`)", + // "default" stands for the default rule syntax in Traefik v3, i.e. the v3 syntax. + RuleSyntax: "default", + Priority: math.MinInt32, + Service: defaultBackendName, + } + + conf.HTTP.Routers[defaultBackendTLSName] = &dynamic.Router{ + Rule: "PathPrefix(`/`)", + // "default" stands for the default rule syntax in Traefik v3, i.e. the v3 syntax. + RuleSyntax: "default", + Priority: math.MinInt32, + Service: defaultBackendName, + TLS: &dynamic.RouterTLSConfig{}, + } + + conf.HTTP.Services[defaultBackendName] = svc + } + + var ingressClasses []*netv1.IngressClass + ics, err := p.k8sClient.ListIngressClasses() + if err != nil { + log.Ctx(ctx).Warn().Err(err).Msg("Failed to list ingress classes") + } + ingressClasses = filterIngressClass(ics, p.IngressClassByName, p.IngressClass, p.ControllerClass) + + ingresses := p.k8sClient.ListIngresses() + + uniqCerts := make(map[string]*tls.CertAndStores) + for _, ingress := range ingresses { + logger := log.Ctx(ctx).With().Str("ingress", ingress.Name).Str("namespace", ingress.Namespace).Logger() + ctxIngress := logger.WithContext(ctx) + + if !p.shouldProcessIngress(ingress, ingressClasses) { + continue + } + + ingressConfig, err := parseIngressConfig(ingress) + if err != nil { + logger.Error().Err(err).Msg("Error parsing ingress configuration") + continue + } + + if err := p.updateIngressStatus(ingress); err != nil { + logger.Error().Err(err).Msg("Error while updating ingress status") + } + + var hasTLS bool + if len(ingress.Spec.TLS) > 0 { + hasTLS = true + if err := p.loadCertificates(ctxIngress, ingress, uniqCerts); err != nil { + logger.Error().Err(err).Msg("Error configuring TLS") + continue + } + } + + namedServersTransport, err := p.buildServersTransport(ingress.Namespace, ingress.Name, ingressConfig) + if err != nil { + logger.Error().Err(err).Msg("Ignoring Ingress cannot create proxy SSL configuration") + continue + } + + var defaultBackendService *dynamic.Service + if ingress.Spec.DefaultBackend != nil && ingress.Spec.DefaultBackend.Service != nil { + var err error + defaultBackendService, err = p.buildService(ingress.Namespace, *ingress.Spec.DefaultBackend, ingressConfig) + if err != nil { + logger.Error(). + Str("serviceName", ingress.Spec.DefaultBackend.Service.Name). + Str("servicePort", ingress.Spec.DefaultBackend.Service.Port.String()). + Err(err). + Msg("Cannot create default backend service") + } + } + + if defaultBackendService != nil && len(ingress.Spec.Rules) == 0 { + rt := &dynamic.Router{ + Rule: "PathPrefix(`/`)", + // "default" stands for the default rule syntax in Traefik v3, i.e. the v3 syntax. + RuleSyntax: "default", + Priority: math.MinInt32, + Service: defaultBackendName, + } + + if err := p.applyMiddlewares(ingress.Namespace, defaultBackendName, ingressConfig, hasTLS, rt, conf); err != nil { + logger.Error().Err(err).Msg("Error applying middlewares") + } + + conf.HTTP.Routers[defaultBackendName] = rt + + rtTLS := &dynamic.Router{ + Rule: "PathPrefix(`/`)", + // "default" stands for the default rule syntax in Traefik v3, i.e. the v3 syntax. + RuleSyntax: "default", + Priority: math.MinInt32, + Service: defaultBackendName, + TLS: &dynamic.RouterTLSConfig{}, + } + + if err := p.applyMiddlewares(ingress.Namespace, defaultBackendTLSName, ingressConfig, false, rtTLS, conf); err != nil { + logger.Error().Err(err).Msg("Error applying middlewares") + } + + conf.HTTP.Routers[defaultBackendTLSName] = rtTLS + + if namedServersTransport != nil && defaultBackendService.LoadBalancer != nil { + defaultBackendService.LoadBalancer.ServersTransport = namedServersTransport.Name + conf.HTTP.ServersTransports[namedServersTransport.Name] = namedServersTransport.ServersTransport + } + conf.HTTP.Services[defaultBackendName] = defaultBackendService + } + + for ri, rule := range ingress.Spec.Rules { + if ptr.Deref(ingressConfig.SSLPassthrough, false) { + if rule.Host == "" { + logger.Error().Err(err).Msg("Cannot process ssl-passthrough for rule without host") + continue + } + + var backend *netv1.IngressBackend + if rule.HTTP != nil { + for _, path := range rule.HTTP.Paths { + if path.Path == "/" { + backend = &path.Backend + break + } + } + } else if ingress.Spec.DefaultBackend != nil { + // Passthrough with the default backend if no HTTP section. + backend = ingress.Spec.DefaultBackend + } + + if backend == nil { + logger.Error().Msgf("No backend found for ssl-passthrough for rule with host %q", rule.Host) + continue + } + + service, err := p.buildPassthroughService(ingress.Namespace, *backend, ingressConfig) + if err != nil { + logger.Error().Err(err).Msgf("Cannot create passthrough service for %s", backend.Service.Name) + continue + } + + port := backend.Service.Port.Name + if len(backend.Service.Port.Name) == 0 { + port = strconv.Itoa(int(backend.Service.Port.Number)) + } + + serviceName := provider.Normalize(ingress.Namespace + "-" + backend.Service.Name + "-" + port) + conf.TCP.Services[serviceName] = service + + routerKey := strings.TrimPrefix(provider.Normalize(ingress.Namespace+"-"+ingress.Name+"-"+rule.Host), "-") + + conf.TCP.Routers[routerKey] = &dynamic.TCPRouter{ + Rule: fmt.Sprintf("HostSNI(`%s`)", rule.Host), + // "default" stands for the default rule syntax in Traefik v3, i.e. the v3 syntax. + RuleSyntax: "default", + Service: serviceName, + TLS: &dynamic.RouterTCPTLSConfig{ + Passthrough: true, + }, + } + + continue + } + + if defaultBackendService != nil && rule.Host != "" { + key := provider.Normalize(ingress.Namespace + "-" + ingress.Name + "-default-backend") + + rt := &dynamic.Router{ + Rule: buildHostRule(rule.Host), + // "default" stands for the default rule syntax in Traefik v3, i.e. the v3 syntax. + RuleSyntax: "default", + Service: key, + } + + if err := p.applyMiddlewares(ingress.Namespace, key, ingressConfig, hasTLS, rt, conf); err != nil { + logger.Error().Err(err).Msg("Error applying middlewares") + } + + conf.HTTP.Routers[key] = rt + + rtTLS := &dynamic.Router{ + Rule: buildHostRule(rule.Host), + // "default" stands for the default rule syntax in Traefik v3, i.e. the v3 syntax. + RuleSyntax: "default", + Service: key, + TLS: &dynamic.RouterTLSConfig{}, + } + + if err := p.applyMiddlewares(ingress.Namespace, key+"-tls", ingressConfig, false, rtTLS, conf); err != nil { + logger.Error().Err(err).Msg("Error applying middlewares") + } + + conf.HTTP.Routers[key+"-tls"] = rtTLS + + if namedServersTransport != nil && defaultBackendService.LoadBalancer != nil { + defaultBackendService.LoadBalancer.ServersTransport = namedServersTransport.Name + conf.HTTP.ServersTransports[namedServersTransport.Name] = namedServersTransport.ServersTransport + } + + conf.HTTP.Services[key] = defaultBackendService + } + + if rule.HTTP == nil { + continue + } + + for pi, pa := range rule.HTTP.Paths { + // As NGINX we are ignoring resource backend. + // An Ingress backend must have se service or a resource definition. + if pa.Backend.Service == nil { + logger.Error().Str("path", pa.Path). + Err(err).Msg("Ignoring path with no service backend") + continue + } + + portString := pa.Backend.Service.Port.Name + if len(pa.Backend.Service.Port.Name) == 0 { + portString = strconv.Itoa(int(pa.Backend.Service.Port.Number)) + } + + // TODO: if no service, do not add middlewares and 503. + serviceName := provider.Normalize(ingress.Namespace + "-" + pa.Backend.Service.Name + "-" + portString) + + service, err := p.buildService(ingress.Namespace, pa.Backend, ingressConfig) + if err != nil { + logger.Error(). + Str("serviceName", pa.Backend.Service.Name). + Str("servicePort", pa.Backend.Service.Port.String()). + Err(err). + Msg("Cannot create service") + continue + } + + rt := &dynamic.Router{ + Rule: buildRule(rule.Host, pa, ingressConfig), + // "default" stands for the default rule syntax in Traefik v3, i.e. the v3 syntax. + RuleSyntax: "default", + Service: serviceName, + } + if hasTLS { + rt.TLS = &dynamic.RouterTLSConfig{} + } + + routerKey := provider.Normalize(fmt.Sprintf("%s-%s-rule-%d-path-%d", ingress.Namespace, ingress.Name, ri, pi)) + + conf.HTTP.Routers[routerKey] = rt + conf.HTTP.Services[serviceName] = service + + if namedServersTransport != nil && service.LoadBalancer != nil { + service.LoadBalancer.ServersTransport = namedServersTransport.Name + conf.HTTP.ServersTransports[namedServersTransport.Name] = namedServersTransport.ServersTransport + } + + if err := p.applyMiddlewares(ingress.Namespace, routerKey, ingressConfig, hasTLS, rt, conf); err != nil { + logger.Error().Err(err).Msg("Error applying middlewares") + } + } + } + } + + conf.TLS = &dynamic.TLSConfiguration{ + Certificates: slices.Collect(maps.Values(uniqCerts)), + } + + return conf +} + +func (p *Provider) buildServersTransport(namespace, name string, cfg ingressConfig) (*namedServersTransport, error) { + scheme := parseBackendProtocol(ptr.Deref(cfg.BackendProtocol, "HTTP")) + if scheme != "https" { + return nil, nil + } + + nst := &namedServersTransport{ + Name: provider.Normalize(namespace + "-" + name), + ServersTransport: &dynamic.ServersTransport{ + ServerName: ptr.Deref(cfg.ProxySSLName, ptr.Deref(cfg.ProxySSLServerName, "")), + InsecureSkipVerify: strings.ToLower(ptr.Deref(cfg.ProxySSLVerify, "off")) == "on", + }, + } + + if sslSecret := ptr.Deref(cfg.ProxySSLSecret, ""); sslSecret != "" { + parts := strings.Split(sslSecret, "/") + if len(parts) != 2 { + return nil, fmt.Errorf("malformed proxy SSL secret: %s, expected namespace/name", sslSecret) + } + + blocks, err := p.certificateBlocks(parts[0], parts[1]) + if err != nil { + return nil, fmt.Errorf("getting certificate blocks: %w", err) + } + + if blocks.CA != nil { + nst.ServersTransport.RootCAs = []types.FileOrContent{*blocks.CA} + } + + if blocks.Certificate != nil { + nst.ServersTransport.Certificates = []tls.Certificate{*blocks.Certificate} + } + } + + return nst, nil +} + +func (p *Provider) buildService(namespace string, backend netv1.IngressBackend, cfg ingressConfig) (*dynamic.Service, error) { + backendAddresses, err := p.getBackendAddresses(namespace, backend, cfg) + if err != nil { + return nil, fmt.Errorf("getting backend addresses: %w", err) + } + + lb := &dynamic.ServersLoadBalancer{} + lb.SetDefaults() + + if ptr.Deref(cfg.Affinity, "") != "" { + lb.Sticky = &dynamic.Sticky{ + Cookie: &dynamic.Cookie{ + Name: ptr.Deref(cfg.SessionCookieName, "INGRESSCOOKIE"), + Secure: ptr.Deref(cfg.SessionCookieSecure, false), + HTTPOnly: true, // Default value in Nginx. + SameSite: strings.ToLower(ptr.Deref(cfg.SessionCookieSameSite, "")), + MaxAge: ptr.Deref(cfg.SessionCookieMaxAge, 0), + Path: ptr.To(ptr.Deref(cfg.SessionCookiePath, "/")), + Domain: ptr.Deref(cfg.SessionCookieDomain, ""), + }, + } + } + + scheme := parseBackendProtocol(ptr.Deref(cfg.BackendProtocol, "HTTP")) + + svc := &dynamic.Service{LoadBalancer: lb} + for _, addr := range backendAddresses { + svc.LoadBalancer.Servers = append(svc.LoadBalancer.Servers, dynamic.Server{ + URL: fmt.Sprintf("%s://%s", scheme, addr.Address), + }) + } + + return svc, nil +} + +func (p *Provider) buildPassthroughService(namespace string, backend netv1.IngressBackend, cfg ingressConfig) (*dynamic.TCPService, error) { + backendAddresses, err := p.getBackendAddresses(namespace, backend, cfg) + if err != nil { + return nil, fmt.Errorf("getting backend addresses: %w", err) + } + + lb := &dynamic.TCPServersLoadBalancer{} + for _, addr := range backendAddresses { + lb.Servers = append(lb.Servers, dynamic.TCPServer{ + Address: addr.Address, + }) + } + + return &dynamic.TCPService{LoadBalancer: lb}, nil +} + +func (p *Provider) getBackendAddresses(namespace string, backend netv1.IngressBackend, cfg ingressConfig) ([]backendAddress, error) { + service, err := p.k8sClient.GetService(namespace, backend.Service.Name) + if err != nil { + return nil, fmt.Errorf("getting service: %w", err) + } + + if p.DisableSvcExternalName && service.Spec.Type == corev1.ServiceTypeExternalName { + return nil, errors.New("externalName services not allowed") + } + + var portName string + var portSpec corev1.ServicePort + var match bool + for _, p := range service.Spec.Ports { + // A port with number 0 or an empty name is not allowed, this case is there for the default backend service. + if (backend.Service.Port.Number == 0 && backend.Service.Port.Name == "") || + (backend.Service.Port.Number == p.Port || (backend.Service.Port.Name == p.Name && len(p.Name) > 0)) { + portName = p.Name + portSpec = p + match = true + break + } + } + if !match { + return nil, errors.New("service port not found") + } + + if service.Spec.Type == corev1.ServiceTypeExternalName { + return []backendAddress{{Address: net.JoinHostPort(service.Spec.ExternalName, strconv.Itoa(int(portSpec.Port)))}}, nil + } + + // When service upstream is set to true we return the service ClusterIP as the backend address. + if ptr.Deref(cfg.ServiceUpstream, false) { + return []backendAddress{{Address: net.JoinHostPort(service.Spec.ClusterIP, strconv.Itoa(int(portSpec.Port)))}}, nil + } + + endpointSlices, err := p.k8sClient.GetEndpointSlicesForService(namespace, backend.Service.Name) + if err != nil { + return nil, fmt.Errorf("getting endpointslices: %w", err) + } + + var addresses []backendAddress + uniqAddresses := map[string]struct{}{} + for _, endpointSlice := range endpointSlices { + var port int32 + for _, p := range endpointSlice.Ports { + if portName == *p.Name { + port = *p.Port + break + } + } + if port == 0 { + continue + } + + for _, endpoint := range endpointSlice.Endpoints { + if !k8s.EndpointServing(endpoint) { + continue + } + + for _, address := range endpoint.Addresses { + if _, ok := uniqAddresses[address]; ok { + continue + } + + uniqAddresses[address] = struct{}{} + addresses = append(addresses, backendAddress{ + Address: net.JoinHostPort(address, strconv.Itoa(int(port))), + Fenced: ptr.Deref(endpoint.Conditions.Terminating, false) && ptr.Deref(endpoint.Conditions.Serving, false), + }) + } + } + } + + return addresses, nil +} + +func (p *Provider) updateIngressStatus(ing *netv1.Ingress) error { + if p.PublishService == "" && len(p.PublishStatusAddress) == 0 { + // Nothing to do, no PublishService or PublishStatusAddress defined. + return nil + } + + if len(p.PublishStatusAddress) > 0 { + ingStatus := make([]netv1.IngressLoadBalancerIngress, 0, len(p.PublishStatusAddress)) + for _, nameOrIP := range p.PublishStatusAddress { + if net.ParseIP(nameOrIP) != nil { + ingStatus = append(ingStatus, netv1.IngressLoadBalancerIngress{IP: nameOrIP}) + continue + } + + ingStatus = append(ingStatus, netv1.IngressLoadBalancerIngress{Hostname: nameOrIP}) + } + + return p.k8sClient.UpdateIngressStatus(ing, ingStatus) + } + + serviceInfo := strings.Split(p.PublishService, "/") + if len(serviceInfo) != 2 { + return fmt.Errorf("parsing publishService, 'namespace/service' format expected: %s", p.PublishService) + } + + serviceNamespace, serviceName := serviceInfo[0], serviceInfo[1] + + service, err := p.k8sClient.GetService(serviceNamespace, serviceName) + if err != nil { + return fmt.Errorf("getting service: %w", err) + } + + var ingressStatus []netv1.IngressLoadBalancerIngress + + switch service.Spec.Type { + case corev1.ServiceTypeExternalName: + ingressStatus = []netv1.IngressLoadBalancerIngress{{ + Hostname: service.Spec.ExternalName, + }} + + case corev1.ServiceTypeClusterIP: + ingressStatus = []netv1.IngressLoadBalancerIngress{{ + IP: service.Spec.ClusterIP, + }} + + case corev1.ServiceTypeNodePort: + if service.Spec.ExternalIPs == nil { + ingressStatus = []netv1.IngressLoadBalancerIngress{{ + IP: service.Spec.ClusterIP, + }} + } else { + ingressStatus = make([]netv1.IngressLoadBalancerIngress, 0, len(service.Spec.ExternalIPs)) + for _, ip := range service.Spec.ExternalIPs { + ingressStatus = append(ingressStatus, netv1.IngressLoadBalancerIngress{IP: ip}) + } + } + + case corev1.ServiceTypeLoadBalancer: + ingressStatus, err = convertSlice[netv1.IngressLoadBalancerIngress](service.Status.LoadBalancer.Ingress) + if err != nil { + return fmt.Errorf("converting ingress loadbalancer status: %w", err) + } + for _, ip := range service.Spec.ExternalIPs { + // Avoid duplicates in the ingress status. + var found bool + for _, status := range ingressStatus { + if status.IP == ip || status.Hostname == ip { + found = true + continue + } + } + if !found { + ingressStatus = append(ingressStatus, netv1.IngressLoadBalancerIngress{IP: ip}) + } + } + } + + return p.k8sClient.UpdateIngressStatus(ing, ingressStatus) +} + +func (p *Provider) shouldProcessIngress(ingress *netv1.Ingress, ingressClasses []*netv1.IngressClass) bool { + if len(ingressClasses) > 0 && ingress.Spec.IngressClassName != nil { + return slices.ContainsFunc(ingressClasses, func(ic *netv1.IngressClass) bool { + return *ingress.Spec.IngressClassName == ic.ObjectMeta.Name + }) + } + + if class, ok := ingress.Annotations[annotationIngressClass]; ok { + return class == p.IngressClass + } + + return p.WatchIngressWithoutClass +} + +func (p *Provider) loadCertificates(ctx context.Context, ingress *netv1.Ingress, uniqCerts map[string]*tls.CertAndStores) error { + for _, t := range ingress.Spec.TLS { + if t.SecretName == "" { + log.Ctx(ctx).Debug().Msg("Skipping TLS sub-section: No secret name provided") + continue + } + + certKey := ingress.Namespace + "-" + t.SecretName + if _, certExists := uniqCerts[certKey]; !certExists { + blocks, err := p.certificateBlocks(ingress.Namespace, t.SecretName) + if err != nil { + return fmt.Errorf("getting certificate blocks: %w", err) + } + + if blocks.Certificate == nil { + return fmt.Errorf("no keypair found in secret %s/%s", ingress.Namespace, t.SecretName) + } + + uniqCerts[certKey] = &tls.CertAndStores{ + Certificate: *blocks.Certificate, + } + } + } + + return nil +} + +func (p *Provider) applyMiddlewares(namespace, routerKey string, ingressConfig ingressConfig, hasTLS bool, rt *dynamic.Router, conf *dynamic.Configuration) error { + if err := p.applyBasicAuthConfiguration(namespace, routerKey, ingressConfig, rt, conf); err != nil { + return fmt.Errorf("applying basic auth configuration: %w", err) + } + + if err := applyForwardAuthConfiguration(routerKey, ingressConfig, rt, conf); err != nil { + return fmt.Errorf("applying forward auth configuration: %w", err) + } + + applyCORSConfiguration(routerKey, ingressConfig, rt, conf) + + // Apply SSL redirect is mandatory to be applied after all other middlewares. + // TODO: check how to remove this, and create the HTTP router elsewhere. + applySSLRedirectConfiguration(routerKey, ingressConfig, hasTLS, rt, conf) + + return nil +} + +func (p *Provider) applyBasicAuthConfiguration(namespace, routerName string, ingressConfig ingressConfig, rt *dynamic.Router, conf *dynamic.Configuration) error { + if ingressConfig.AuthType == nil { + return nil + } + + authType := ptr.Deref(ingressConfig.AuthType, "") + if authType != "basic" && authType != "digest" { + return fmt.Errorf("invalid auth-type %q, must be 'basic' or 'digest'", authType) + } + + authSecret := ptr.Deref(ingressConfig.AuthSecret, "") + if authSecret == "" { + return fmt.Errorf("invalid auth-secret %q, must not be empty", authSecret) + } + + authSecretParts := strings.Split(authSecret, "/") + if len(authSecretParts) > 2 { + return fmt.Errorf("invalid auth secret %q", authSecret) + } + + secretName := authSecretParts[0] + secretNamespace := namespace + if len(authSecretParts) == 2 { + secretNamespace = authSecretParts[0] + secretName = authSecretParts[1] + } + + secret, err := p.k8sClient.GetSecret(secretNamespace, secretName) + if err != nil { + return fmt.Errorf("getting secret %s: %w", authSecret, err) + } + + authSecretType := ptr.Deref(ingressConfig.AuthSecretType, "auth-file") + if authSecretType != "auth-file" && authSecretType != "auth-map" { + return fmt.Errorf("invalid auth-secret-type %q, must be 'auth-file' or 'auth-map'", authSecretType) + } + + users, err := basicAuthUsers(secret, authSecretType) + if err != nil { + return fmt.Errorf("getting users from secret %s: %w", authSecret, err) + } + + realm := ptr.Deref(ingressConfig.AuthRealm, "") + + switch authType { + case "basic": + basicMiddlewareName := routerName + "-basic-auth" + conf.HTTP.Middlewares[basicMiddlewareName] = &dynamic.Middleware{ + BasicAuth: &dynamic.BasicAuth{ + Users: users, + Realm: realm, + RemoveHeader: false, + }, + } + rt.Middlewares = append(rt.Middlewares, basicMiddlewareName) + + case "digest": + digestMiddlewareName := routerName + "-digest-auth" + conf.HTTP.Middlewares[digestMiddlewareName] = &dynamic.Middleware{ + DigestAuth: &dynamic.DigestAuth{ + Users: users, + Realm: realm, + RemoveHeader: false, + }, + } + rt.Middlewares = append(rt.Middlewares, digestMiddlewareName) + } + + return nil +} + +func (p *Provider) certificateBlocks(namespace, name string) (*certBlocks, error) { + secret, err := p.k8sClient.GetSecret(namespace, name) + if err != nil { + return nil, fmt.Errorf("fetching secret %s/%s: %w", namespace, name, err) + } + + certBytes, hasCert := secret.Data[corev1.TLSCertKey] + keyBytes, hasKey := secret.Data[corev1.TLSPrivateKeyKey] + caBytes, hasCA := secret.Data[corev1.ServiceAccountRootCAKey] + + if !hasCert && !hasKey && !hasCA { + return nil, errors.New("secret does not contain a keypair or CA certificate") + } + + var blocks certBlocks + if hasCA { + if len(caBytes) == 0 { + return nil, errors.New("secret contains an empty CA certificate") + } + + ca := types.FileOrContent(caBytes) + blocks.CA = &ca + } + + if hasKey && hasCert { + if len(certBytes) == 0 { + return nil, errors.New("secret contains an empty certificate") + } + if len(keyBytes) == 0 { + return nil, errors.New("secret contains an empty key") + } + blocks.Certificate = &tls.Certificate{ + CertFile: types.FileOrContent(certBytes), + KeyFile: types.FileOrContent(keyBytes), + } + } + + return &blocks, nil +} + +func applyCORSConfiguration(routerName string, ingressConfig ingressConfig, rt *dynamic.Router, conf *dynamic.Configuration) { + if !ptr.Deref(ingressConfig.EnableCORS, false) { + return + } + + corsMiddlewareName := routerName + "-cors" + conf.HTTP.Middlewares[corsMiddlewareName] = &dynamic.Middleware{ + Headers: &dynamic.Headers{ + AccessControlAllowCredentials: ptr.Deref(ingressConfig.EnableCORSAllowCredentials, true), + AccessControlExposeHeaders: ptr.Deref(ingressConfig.CORSExposeHeaders, []string{}), + AccessControlAllowHeaders: ptr.Deref(ingressConfig.CORSAllowHeaders, []string{"DNT", "Keep-Alive", "User-Agent", "X-Requested-With", "If-Modified-Since", "Cache-Control", "Content-Type", "Range,Authorization"}), + AccessControlAllowMethods: ptr.Deref(ingressConfig.CORSAllowMethods, []string{"GET", "PUT", "POST", "DELETE", "PATCH", "OPTIONS"}), + AccessControlAllowOriginList: ptr.Deref(ingressConfig.CORSAllowOrigin, []string{"*"}), + AccessControlMaxAge: int64(ptr.Deref(ingressConfig.CORSMaxAge, 1728000)), + }, + } + + rt.Middlewares = append(rt.Middlewares, corsMiddlewareName) +} + +func applySSLRedirectConfiguration(routerName string, ingressConfig ingressConfig, hasTLS bool, rt *dynamic.Router, conf *dynamic.Configuration) { + var forceSSLRedirect bool + if ingressConfig.ForceSSLRedirect != nil { + forceSSLRedirect = *ingressConfig.ForceSSLRedirect + } + + sslRedirect := ptr.Deref(ingressConfig.SSLRedirect, hasTLS) + + if !forceSSLRedirect && !sslRedirect { + if hasTLS { + httpRouter := &dynamic.Router{ + Rule: rt.Rule, + // "default" stands for the default rule syntax in Traefik v3, i.e. the v3 syntax. + RuleSyntax: "default", + Middlewares: rt.Middlewares, + Service: rt.Service, + } + + conf.HTTP.Routers[routerName+"-http"] = httpRouter + } + + return + } + + redirectRouter := &dynamic.Router{ + Rule: rt.Rule, + // "default" stands for the default rule syntax in Traefik v3, i.e. the v3 syntax. + RuleSyntax: "default", + Service: "noop@internal", + } + + redirectMiddlewareName := routerName + "-redirect-scheme" + conf.HTTP.Middlewares[redirectMiddlewareName] = &dynamic.Middleware{ + RedirectScheme: &dynamic.RedirectScheme{ + Scheme: "https", + Permanent: true, + }, + } + redirectRouter.Middlewares = append(redirectRouter.Middlewares, redirectMiddlewareName) + + conf.HTTP.Routers[routerName+"-redirect"] = redirectRouter +} + +func applyForwardAuthConfiguration(routerName string, ingressConfig ingressConfig, rt *dynamic.Router, conf *dynamic.Configuration) error { + if ingressConfig.AuthURL == nil { + return nil + } + + if *ingressConfig.AuthURL == "" { + return errors.New("empty auth-url found in ingress annotations") + } + + authResponseHeaders := strings.Split(ptr.Deref(ingressConfig.AuthResponseHeaders, ""), ",") + + forwardMiddlewareName := routerName + "-forward-auth" + conf.HTTP.Middlewares[forwardMiddlewareName] = &dynamic.Middleware{ + ForwardAuth: &dynamic.ForwardAuth{ + Address: *ingressConfig.AuthURL, + AuthResponseHeaders: authResponseHeaders, + }, + } + rt.Middlewares = append(rt.Middlewares, forwardMiddlewareName) + + return nil +} + +func basicAuthUsers(secret *corev1.Secret, authSecretType string) (dynamic.Users, error) { + var users dynamic.Users + if authSecretType == "auth-map" { + if len(secret.Data) == 0 { + return nil, fmt.Errorf("secret %s/%s does not contain any user credentials", secret.Namespace, secret.Name) + } + + for user, pass := range secret.Data { + users = append(users, user+":"+string(pass)) + } + + return users, nil + } + + // Default to auth-file type. + authFileContent, ok := secret.Data["auth"] + if !ok { + return nil, fmt.Errorf("secret %s/%s does not contain auth-file content key `auth`", secret.Namespace, secret.Name) + } + + // Trim lines and filter out blanks + rawLines := strings.Split(string(authFileContent), "\n") + for _, rawLine := range rawLines { + line := strings.TrimSpace(rawLine) + if line != "" && !strings.HasPrefix(line, "#") { + users = append(users, line) + } + } + + return users, nil +} + +func buildRule(host string, pa netv1.HTTPIngressPath, config ingressConfig) string { + var rules []string + if len(host) > 0 { + rules = append(rules, buildHostRule(host)) + } + + if len(pa.Path) > 0 { + pathType := ptr.Deref(pa.PathType, netv1.PathTypePrefix) + if pathType == netv1.PathTypeImplementationSpecific { + pathType = netv1.PathTypePrefix + } + + switch pathType { + case netv1.PathTypeExact: + rules = append(rules, fmt.Sprintf("Path(`%s`)", pa.Path)) + case netv1.PathTypePrefix: + if ptr.Deref(config.UseRegex, false) { + rules = append(rules, fmt.Sprintf("PathRegexp(`^%s`)", regexp.QuoteMeta(pa.Path))) + } else { + rules = append(rules, buildPrefixRule(pa.Path)) + } + } + } + + return strings.Join(rules, " && ") +} + +func buildHostRule(host string) string { + if strings.HasPrefix(host, "*.") { + host = strings.Replace(regexp.QuoteMeta(host), `\*\.`, `[a-zA-Z0-9-]+\.`, 1) + return fmt.Sprintf("HostRegexp(`^%s$`)", host) + } + + return fmt.Sprintf("Host(`%s`)", host) +} + +// buildPrefixRule is a helper function to build a path prefix rule that matches path prefix split by `/`. +// For example, the paths `/abc`, `/abc/`, and `/abc/def` would all match the prefix `/abc`, +// but the path `/abcd` would not. See TestStrictPrefixMatchingRule() for more examples. +// +// "PathPrefix" in Kubernetes Gateway API is semantically equivalent to the "Prefix" path type in the +// Kubernetes Ingress API. +func buildPrefixRule(path string) string { + if path == "/" { + return "PathPrefix(`/`)" + } + + path = strings.TrimSuffix(path, "/") + return fmt.Sprintf("(Path(`%[1]s`) || PathPrefix(`%[1]s/`))", path) +} + +func throttleEvents(ctx context.Context, throttleDuration time.Duration, pool *safe.Pool, eventsChan <-chan interface{}) chan interface{} { + if throttleDuration == 0 { + return nil + } + + // Create a buffered channel to hold the pending event (if we're delaying processing the event due to throttling). + eventsChanBuffered := make(chan interface{}, 1) + + // Run a goroutine that reads events from eventChan and does a + // non-blocking write to pendingEvent. This guarantees that writing to + // eventChan will never block, and that pendingEvent will have + // something in it if there's been an event since we read from that channel. + pool.GoCtx(func(ctxPool context.Context) { + for { + select { + case <-ctxPool.Done(): + return + case nextEvent := <-eventsChan: + select { + case eventsChanBuffered <- nextEvent: + default: + // We already have an event in eventsChanBuffered, so we'll + // do a refresh as soon as our throttle allows us to. It's fine + // to drop the event and keep whatever's in the buffer -- we + // don't do different things for different events. + log.Ctx(ctx).Debug().Msgf("Dropping event kind %T due to throttling", nextEvent) + } + } + } + }) + + return eventsChanBuffered +} diff --git a/pkg/provider/kubernetes/ingress-nginx/kubernetes_test.go b/pkg/provider/kubernetes/ingress-nginx/kubernetes_test.go new file mode 100644 index 000000000..37f819300 --- /dev/null +++ b/pkg/provider/kubernetes/ingress-nginx/kubernetes_test.go @@ -0,0 +1,605 @@ +package ingressnginx + +import ( + "math" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/traefik/traefik/v3/pkg/config/dynamic" + "github.com/traefik/traefik/v3/pkg/provider/kubernetes/k8s" + "github.com/traefik/traefik/v3/pkg/tls" + "github.com/traefik/traefik/v3/pkg/types" + "k8s.io/apimachinery/pkg/runtime" + kubefake "k8s.io/client-go/kubernetes/fake" + "k8s.io/utils/ptr" +) + +func TestLoadIngresses(t *testing.T) { + testCases := []struct { + desc string + ingressClass string + defaultBackendServiceName string + defaultBackendServiceNamespace string + paths []string + expected *dynamic.Configuration + }{ + { + desc: "Empty, no IngressClass", + paths: []string{ + "services.yml", + "ingresses/01-ingress-with-basicauth.yml", + }, + expected: &dynamic.Configuration{ + TCP: &dynamic.TCPConfiguration{ + Routers: map[string]*dynamic.TCPRouter{}, + Services: map[string]*dynamic.TCPService{}, + }, + HTTP: &dynamic.HTTPConfiguration{ + Routers: map[string]*dynamic.Router{}, + Middlewares: map[string]*dynamic.Middleware{}, + Services: map[string]*dynamic.Service{}, + ServersTransports: map[string]*dynamic.ServersTransport{}, + }, + TLS: &dynamic.TLSConfiguration{}, + }, + }, + { + desc: "Basic Auth", + paths: []string{ + "services.yml", + "ingressclasses.yml", + "ingresses/01-ingress-with-basicauth.yml", + }, + expected: &dynamic.Configuration{ + TCP: &dynamic.TCPConfiguration{ + Routers: map[string]*dynamic.TCPRouter{}, + Services: map[string]*dynamic.TCPService{}, + }, + HTTP: &dynamic.HTTPConfiguration{ + Routers: map[string]*dynamic.Router{ + "default-ingress-with-basicauth-rule-0-path-0": { + Rule: "Host(`whoami.localhost`) && Path(`/basicauth`)", + RuleSyntax: "default", + Middlewares: []string{"default-ingress-with-basicauth-rule-0-path-0-basic-auth"}, + Service: "default-whoami-80", + }, + }, + Middlewares: map[string]*dynamic.Middleware{ + "default-ingress-with-basicauth-rule-0-path-0-basic-auth": { + BasicAuth: &dynamic.BasicAuth{ + Users: dynamic.Users{ + "user:{SHA}W6ph5Mm5Pz8GgiULbPgzG37mj9g=", + }, + Realm: "Authentication Required", + }, + }, + }, + Services: map[string]*dynamic.Service{ + "default-whoami-80": { + LoadBalancer: &dynamic.ServersLoadBalancer{ + Servers: []dynamic.Server{ + { + URL: "http://10.10.0.1:80", + }, + { + URL: "http://10.10.0.2:80", + }, + }, + Strategy: "wrr", + PassHostHeader: ptr.To(true), + ResponseForwarding: &dynamic.ResponseForwarding{ + FlushInterval: dynamic.DefaultFlushInterval, + }, + }, + }, + }, + ServersTransports: map[string]*dynamic.ServersTransport{}, + }, + TLS: &dynamic.TLSConfiguration{}, + }, + }, + { + desc: "Forward Auth", + paths: []string{ + "services.yml", + "ingressclasses.yml", + "ingresses/02-ingress-with-forwardauth.yml", + }, + expected: &dynamic.Configuration{ + TCP: &dynamic.TCPConfiguration{ + Routers: map[string]*dynamic.TCPRouter{}, + Services: map[string]*dynamic.TCPService{}, + }, + HTTP: &dynamic.HTTPConfiguration{ + Routers: map[string]*dynamic.Router{ + "default-ingress-with-forwardauth-rule-0-path-0": { + Rule: "Host(`whoami.localhost`) && Path(`/forwardauth`)", + RuleSyntax: "default", + Middlewares: []string{"default-ingress-with-forwardauth-rule-0-path-0-forward-auth"}, + Service: "default-whoami-80", + }, + }, + Middlewares: map[string]*dynamic.Middleware{ + "default-ingress-with-forwardauth-rule-0-path-0-forward-auth": { + ForwardAuth: &dynamic.ForwardAuth{ + Address: "http://whoami.default.svc/", + AuthResponseHeaders: []string{"X-Foo"}, + }, + }, + }, + Services: map[string]*dynamic.Service{ + "default-whoami-80": { + LoadBalancer: &dynamic.ServersLoadBalancer{ + Servers: []dynamic.Server{ + { + URL: "http://10.10.0.1:80", + }, + { + URL: "http://10.10.0.2:80", + }, + }, + Strategy: "wrr", + PassHostHeader: ptr.To(true), + ResponseForwarding: &dynamic.ResponseForwarding{ + FlushInterval: dynamic.DefaultFlushInterval, + }, + }, + }, + }, + ServersTransports: map[string]*dynamic.ServersTransport{}, + }, + TLS: &dynamic.TLSConfiguration{}, + }, + }, + { + desc: "SSL Redirect", + paths: []string{ + "services.yml", + "secrets.yml", + "ingressclasses.yml", + "ingresses/03-ingress-with-ssl-redirect.yml", + }, + expected: &dynamic.Configuration{ + TCP: &dynamic.TCPConfiguration{ + Routers: map[string]*dynamic.TCPRouter{}, + Services: map[string]*dynamic.TCPService{}, + }, + HTTP: &dynamic.HTTPConfiguration{ + Routers: map[string]*dynamic.Router{ + "default-ingress-with-ssl-redirect-rule-0-path-0": { + Rule: "Host(`sslredirect.localhost`) && Path(`/`)", + RuleSyntax: "default", + TLS: &dynamic.RouterTLSConfig{}, + Service: "default-whoami-80", + }, + "default-ingress-with-ssl-redirect-rule-0-path-0-redirect": { + Rule: "Host(`sslredirect.localhost`) && Path(`/`)", + RuleSyntax: "default", + Middlewares: []string{"default-ingress-with-ssl-redirect-rule-0-path-0-redirect-scheme"}, + Service: "noop@internal", + }, + "default-ingress-without-ssl-redirect-rule-0-path-0-http": { + Rule: "Host(`withoutsslredirect.localhost`) && Path(`/`)", + RuleSyntax: "default", + Service: "default-whoami-80", + }, + "default-ingress-without-ssl-redirect-rule-0-path-0": { + Rule: "Host(`withoutsslredirect.localhost`) && Path(`/`)", + RuleSyntax: "default", + TLS: &dynamic.RouterTLSConfig{}, + Service: "default-whoami-80", + }, + "default-ingress-with-force-ssl-redirect-rule-0-path-0": { + Rule: "Host(`forcesslredirect.localhost`) && Path(`/`)", + RuleSyntax: "default", + Service: "default-whoami-80", + }, + "default-ingress-with-force-ssl-redirect-rule-0-path-0-redirect": { + Rule: "Host(`forcesslredirect.localhost`) && Path(`/`)", + RuleSyntax: "default", + Middlewares: []string{"default-ingress-with-force-ssl-redirect-rule-0-path-0-redirect-scheme"}, + Service: "noop@internal", + }, + }, + Middlewares: map[string]*dynamic.Middleware{ + "default-ingress-with-ssl-redirect-rule-0-path-0-redirect-scheme": { + RedirectScheme: &dynamic.RedirectScheme{ + Scheme: "https", + Permanent: true, + }, + }, + "default-ingress-with-force-ssl-redirect-rule-0-path-0-redirect-scheme": { + RedirectScheme: &dynamic.RedirectScheme{ + Scheme: "https", + Permanent: true, + }, + }, + }, + Services: map[string]*dynamic.Service{ + "default-whoami-80": { + LoadBalancer: &dynamic.ServersLoadBalancer{ + Servers: []dynamic.Server{ + { + URL: "http://10.10.0.1:80", + }, + { + URL: "http://10.10.0.2:80", + }, + }, + Strategy: "wrr", + PassHostHeader: ptr.To(true), + ResponseForwarding: &dynamic.ResponseForwarding{ + FlushInterval: dynamic.DefaultFlushInterval, + }, + }, + }, + }, + ServersTransports: map[string]*dynamic.ServersTransport{}, + }, + TLS: &dynamic.TLSConfiguration{ + Certificates: []*tls.CertAndStores{ + { + Certificate: tls.Certificate{ + CertFile: "-----BEGIN CERTIFICATE-----", + KeyFile: "-----BEGIN CERTIFICATE-----", + }, + }, + }, + }, + }, + }, + { + desc: "SSL Passthrough", + paths: []string{ + "services.yml", + "secrets.yml", + "ingressclasses.yml", + "ingresses/04-ingress-with-ssl-passthrough.yml", + }, + expected: &dynamic.Configuration{ + TCP: &dynamic.TCPConfiguration{ + Routers: map[string]*dynamic.TCPRouter{ + "default-ingress-with-ssl-passthrough-passthrough-whoami-localhost": { + Rule: "HostSNI(`passthrough.whoami.localhost`)", + RuleSyntax: "default", + TLS: &dynamic.RouterTCPTLSConfig{ + Passthrough: true, + }, + Service: "default-whoami-tls-443", + }, + }, + Services: map[string]*dynamic.TCPService{ + "default-whoami-tls-443": { + LoadBalancer: &dynamic.TCPServersLoadBalancer{ + Servers: []dynamic.TCPServer{ + { + Address: "10.10.0.5:8443", + }, + { + Address: "10.10.0.6:8443", + }, + }, + }, + }, + }, + }, + HTTP: &dynamic.HTTPConfiguration{ + Routers: map[string]*dynamic.Router{}, + Middlewares: map[string]*dynamic.Middleware{}, + Services: map[string]*dynamic.Service{}, + ServersTransports: map[string]*dynamic.ServersTransport{}, + }, + TLS: &dynamic.TLSConfiguration{}, + }, + }, + { + desc: "Sticky Sessions", + paths: []string{ + "services.yml", + "secrets.yml", + "ingressclasses.yml", + "ingresses/06-ingress-with-sticky.yml", + }, + expected: &dynamic.Configuration{ + TCP: &dynamic.TCPConfiguration{ + Routers: map[string]*dynamic.TCPRouter{}, + Services: map[string]*dynamic.TCPService{}, + }, + HTTP: &dynamic.HTTPConfiguration{ + Routers: map[string]*dynamic.Router{ + "default-ingress-with-sticky-rule-0-path-0": { + Rule: "Host(`sticky.localhost`) && Path(`/`)", + RuleSyntax: "default", + Service: "default-whoami-80", + }, + }, + Middlewares: map[string]*dynamic.Middleware{}, + Services: map[string]*dynamic.Service{ + "default-whoami-80": { + LoadBalancer: &dynamic.ServersLoadBalancer{ + Servers: []dynamic.Server{ + { + URL: "http://10.10.0.1:80", + }, + { + URL: "http://10.10.0.2:80", + }, + }, + Strategy: "wrr", + PassHostHeader: ptr.To(true), + ResponseForwarding: &dynamic.ResponseForwarding{ + FlushInterval: dynamic.DefaultFlushInterval, + }, + Sticky: &dynamic.Sticky{ + Cookie: &dynamic.Cookie{ + Name: "foobar", + Domain: "foo.localhost", + HTTPOnly: true, + MaxAge: 42, + Path: ptr.To("/foobar"), + SameSite: "none", + Secure: true, + }, + }, + }, + }, + }, + ServersTransports: map[string]*dynamic.ServersTransport{}, + }, + TLS: &dynamic.TLSConfiguration{}, + }, + }, + { + desc: "Proxy SSL", + paths: []string{ + "services.yml", + "secrets.yml", + "ingressclasses.yml", + "ingresses/07-ingress-with-proxy-ssl.yml", + }, + expected: &dynamic.Configuration{ + TCP: &dynamic.TCPConfiguration{ + Routers: map[string]*dynamic.TCPRouter{}, + Services: map[string]*dynamic.TCPService{}, + }, + HTTP: &dynamic.HTTPConfiguration{ + Routers: map[string]*dynamic.Router{ + "default-ingress-with-proxy-ssl-rule-0-path-0": { + Rule: "Host(`proxy-ssl.localhost`) && Path(`/`)", + RuleSyntax: "default", + Service: "default-whoami-tls-443", + }, + }, + Middlewares: map[string]*dynamic.Middleware{}, + Services: map[string]*dynamic.Service{ + "default-whoami-tls-443": { + LoadBalancer: &dynamic.ServersLoadBalancer{ + Servers: []dynamic.Server{ + { + URL: "https://10.10.0.5:8443", + }, + { + URL: "https://10.10.0.6:8443", + }, + }, + Strategy: "wrr", + PassHostHeader: ptr.To(true), + ResponseForwarding: &dynamic.ResponseForwarding{ + FlushInterval: dynamic.DefaultFlushInterval, + }, + ServersTransport: "default-ingress-with-proxy-ssl", + }, + }, + }, + ServersTransports: map[string]*dynamic.ServersTransport{ + "default-ingress-with-proxy-ssl": { + ServerName: "whoami.localhost", + InsecureSkipVerify: true, + RootCAs: []types.FileOrContent{"-----BEGIN CERTIFICATE-----"}, + }, + }, + }, + TLS: &dynamic.TLSConfiguration{}, + }, + }, + { + desc: "CORS", + paths: []string{ + "services.yml", + "ingressclasses.yml", + "ingresses/08-ingress-with-cors.yml", + }, + expected: &dynamic.Configuration{ + TCP: &dynamic.TCPConfiguration{ + Routers: map[string]*dynamic.TCPRouter{}, + Services: map[string]*dynamic.TCPService{}, + }, + HTTP: &dynamic.HTTPConfiguration{ + Routers: map[string]*dynamic.Router{ + "default-ingress-with-cors-rule-0-path-0": { + Rule: "Host(`cors.localhost`) && Path(`/`)", + RuleSyntax: "default", + Middlewares: []string{"default-ingress-with-cors-rule-0-path-0-cors"}, + Service: "default-whoami-80", + }, + }, + Middlewares: map[string]*dynamic.Middleware{ + "default-ingress-with-cors-rule-0-path-0-cors": { + Headers: &dynamic.Headers{ + AccessControlAllowCredentials: true, + AccessControlAllowHeaders: []string{"X-Foo"}, + AccessControlAllowMethods: []string{"PUT", "GET", "POST", "OPTIONS"}, + AccessControlAllowOriginList: []string{"*"}, + AccessControlExposeHeaders: []string{"X-Forwarded-For", "X-Forwarded-Host"}, + AccessControlMaxAge: 42, + }, + }, + }, + Services: map[string]*dynamic.Service{ + "default-whoami-80": { + LoadBalancer: &dynamic.ServersLoadBalancer{ + Servers: []dynamic.Server{ + { + URL: "http://10.10.0.1:80", + }, + { + URL: "http://10.10.0.2:80", + }, + }, + Strategy: "wrr", + PassHostHeader: ptr.To(true), + ResponseForwarding: &dynamic.ResponseForwarding{ + FlushInterval: dynamic.DefaultFlushInterval, + }, + }, + }, + }, + ServersTransports: map[string]*dynamic.ServersTransport{}, + }, + TLS: &dynamic.TLSConfiguration{}, + }, + }, + { + desc: "Service Upstream", + paths: []string{ + "services.yml", + "ingressclasses.yml", + "ingresses/09-ingress-with-service-upstream.yml", + }, + expected: &dynamic.Configuration{ + TCP: &dynamic.TCPConfiguration{ + Routers: map[string]*dynamic.TCPRouter{}, + Services: map[string]*dynamic.TCPService{}, + }, + HTTP: &dynamic.HTTPConfiguration{ + Routers: map[string]*dynamic.Router{ + "default-ingress-with-service-upstream-rule-0-path-0": { + Rule: "Host(`service-upstream.localhost`) && Path(`/`)", + RuleSyntax: "default", + Service: "default-whoami-80", + }, + }, + Middlewares: map[string]*dynamic.Middleware{}, + Services: map[string]*dynamic.Service{ + "default-whoami-80": { + LoadBalancer: &dynamic.ServersLoadBalancer{ + Servers: []dynamic.Server{ + { + URL: "http://10.10.10.1:80", + }, + }, + Strategy: "wrr", + PassHostHeader: ptr.To(true), + ResponseForwarding: &dynamic.ResponseForwarding{ + FlushInterval: dynamic.DefaultFlushInterval, + }, + }, + }, + }, + ServersTransports: map[string]*dynamic.ServersTransport{}, + }, + TLS: &dynamic.TLSConfiguration{}, + }, + }, + { + desc: "Default Backend", + defaultBackendServiceName: "whoami", + defaultBackendServiceNamespace: "default", + paths: []string{ + "services.yml", + }, + expected: &dynamic.Configuration{ + TCP: &dynamic.TCPConfiguration{ + Routers: map[string]*dynamic.TCPRouter{}, + Services: map[string]*dynamic.TCPService{}, + }, + HTTP: &dynamic.HTTPConfiguration{ + Routers: map[string]*dynamic.Router{ + "default-backend": { + Rule: "PathPrefix(`/`)", + RuleSyntax: "default", + Priority: math.MinInt32, + Service: "default-backend", + }, + "default-backend-tls": { + Rule: "PathPrefix(`/`)", + RuleSyntax: "default", + Priority: math.MinInt32, + TLS: &dynamic.RouterTLSConfig{}, + Service: "default-backend", + }, + }, + Middlewares: map[string]*dynamic.Middleware{}, + Services: map[string]*dynamic.Service{ + "default-backend": { + LoadBalancer: &dynamic.ServersLoadBalancer{ + Servers: []dynamic.Server{ + { + URL: "http://10.10.0.1:8000", + }, + { + URL: "http://10.10.0.2:8000", + }, + }, + Strategy: "wrr", + PassHostHeader: ptr.To(true), + ResponseForwarding: &dynamic.ResponseForwarding{ + FlushInterval: dynamic.DefaultFlushInterval, + }, + }, + }, + }, + ServersTransports: map[string]*dynamic.ServersTransport{}, + }, + TLS: &dynamic.TLSConfiguration{}, + }, + }, + } + + for _, test := range testCases { + t.Run(test.desc, func(t *testing.T) { + t.Parallel() + + k8sObjects := readResources(t, test.paths) + kubeClient := kubefake.NewClientset(k8sObjects...) + client := newClient(kubeClient) + + eventCh, err := client.WatchAll(t.Context(), "", "") + require.NoError(t, err) + + if len(k8sObjects) > 0 { + // just wait for the first event + <-eventCh + } + + p := Provider{ + k8sClient: client, + defaultBackendServiceName: test.defaultBackendServiceName, + defaultBackendServiceNamespace: test.defaultBackendServiceNamespace, + } + p.SetDefaults() + + conf := p.loadConfiguration(t.Context()) + assert.Equal(t, test.expected, conf) + }) + } +} + +func readResources(t *testing.T, paths []string) []runtime.Object { + t.Helper() + + var k8sObjects []runtime.Object + for _, path := range paths { + yamlContent, err := os.ReadFile(filepath.FromSlash("./fixtures/" + path)) + if err != nil { + panic(err) + } + + k8sObjects = append(k8sObjects, k8s.MustParseYaml(yamlContent)...) + } + + return k8sObjects +}