From c4898b7e980035c3978876f5d434ca61b58715de Mon Sep 17 00:00:00 2001 From: Dave Salisbury Date: Sun, 18 Apr 2021 14:13:53 +1000 Subject: [PATCH 01/32] Plumb in filtering on ingress class name --- main.go | 1 + pkg/apis/externaldns/types.go | 3 +++ source/ingress.go | 32 +++++++++++++++++++++++++++++++- source/store.go | 3 ++- 4 files changed, 37 insertions(+), 2 deletions(-) diff --git a/main.go b/main.go index 5843a583f..b10c3372a 100644 --- a/main.go +++ b/main.go @@ -104,6 +104,7 @@ func main() { Namespace: cfg.Namespace, AnnotationFilter: cfg.AnnotationFilter, LabelFilter: cfg.LabelFilter, + IngressClassNameFilter: cfg.IngressClassNameFilter, FQDNTemplate: cfg.FQDNTemplate, CombineFQDNAndAnnotation: cfg.CombineFQDNAndAnnotation, IgnoreHostnameAnnotation: cfg.IgnoreHostnameAnnotation, diff --git a/pkg/apis/externaldns/types.go b/pkg/apis/externaldns/types.go index 89e053b86..e9153ab7c 100644 --- a/pkg/apis/externaldns/types.go +++ b/pkg/apis/externaldns/types.go @@ -53,6 +53,7 @@ type Config struct { Namespace string AnnotationFilter string LabelFilter string + IngressClassNameFilter []string FQDNTemplate string CombineFQDNAndAnnotation bool IgnoreHostnameAnnotation bool @@ -186,6 +187,7 @@ var defaultConfig = &Config{ Namespace: "", AnnotationFilter: "", LabelFilter: "", + IngressClassNameFilter: nil, FQDNTemplate: "", CombineFQDNAndAnnotation: false, IgnoreHostnameAnnotation: false, @@ -362,6 +364,7 @@ func (cfg *Config) ParseFlags(args []string) error { app.Flag("namespace", "Limit sources of endpoints to a specific namespace (default: all namespaces)").Default(defaultConfig.Namespace).StringVar(&cfg.Namespace) app.Flag("annotation-filter", "Filter sources managed by external-dns via annotation using label selector semantics (default: all sources)").Default(defaultConfig.AnnotationFilter).StringVar(&cfg.AnnotationFilter) app.Flag("label-filter", "Filter sources managed by external-dns via label selector when listing all resources; currently only supported by source CRD").Default(defaultConfig.LabelFilter).StringVar(&cfg.LabelFilter) + app.Flag("ingress-class-filter", "Filter ingresses to just these ingress class(es)").StringsVar(&cfg.IngressClassNameFilter) app.Flag("fqdn-template", "A templated string that's used to generate DNS names from sources that don't define a hostname themselves, or to add a hostname suffix when paired with the fake source (optional). Accepts comma separated list for multiple global FQDN.").Default(defaultConfig.FQDNTemplate).StringVar(&cfg.FQDNTemplate) app.Flag("combine-fqdn-annotation", "Combine FQDN template and Annotations instead of overwriting").BoolVar(&cfg.CombineFQDNAndAnnotation) app.Flag("ignore-hostname-annotation", "Ignore hostname annotation when generating DNS names, valid only when using fqdn-template is set (optional, default: false)").BoolVar(&cfg.IgnoreHostnameAnnotation) diff --git a/source/ingress.go b/source/ingress.go index e81f524d1..01b987300 100644 --- a/source/ingress.go +++ b/source/ingress.go @@ -54,6 +54,7 @@ type ingressSource struct { client kubernetes.Interface namespace string annotationFilter string + ingressClassNameFilter []string fqdnTemplate *template.Template combineFQDNAnnotation bool ignoreHostnameAnnotation bool @@ -63,7 +64,7 @@ type ingressSource struct { } // NewIngressSource creates a new ingressSource with the given config. -func NewIngressSource(kubeClient kubernetes.Interface, namespace, annotationFilter string, fqdnTemplate string, combineFqdnAnnotation bool, ignoreHostnameAnnotation bool, ignoreIngressTLSSpec bool, ignoreIngressRulesSpec bool) (Source, error) { +func NewIngressSource(kubeClient kubernetes.Interface, namespace, annotationFilter string, fqdnTemplate string, combineFqdnAnnotation bool, ignoreHostnameAnnotation bool, ignoreIngressTLSSpec bool, ignoreIngressRulesSpec bool, ingressClassNameFilter []string) (Source, error) { tmpl, err := parseTemplate(fqdnTemplate) if err != nil { return nil, err @@ -94,6 +95,7 @@ func NewIngressSource(kubeClient kubernetes.Interface, namespace, annotationFilt client: kubeClient, namespace: namespace, annotationFilter: annotationFilter, + ingressClassNameFilter: ingressClassNameFilter, fqdnTemplate: tmpl, combineFQDNAnnotation: combineFqdnAnnotation, ignoreHostnameAnnotation: ignoreHostnameAnnotation, @@ -116,6 +118,11 @@ func (sc *ingressSource) Endpoints(ctx context.Context) ([]*endpoint.Endpoint, e return nil, err } + ingresses, err = sc.filterByIngressClass(ingresses) + if err != nil { + return nil, err + } + endpoints := []*endpoint.Endpoint{} for _, ing := range ingresses { @@ -210,6 +217,29 @@ func (sc *ingressSource) filterByAnnotations(ingresses []*networkv1.Ingress) ([] return filteredList, nil } +// filterByIngressClass filters a list of ingresses based on a required ingress +// class +func (sc *ingressSource) filterByIngressClass(ingresses []*v1beta1.Ingress) ([]*v1beta1.Ingress, error) { + // if no class is specified then there's nothing to do + if sc.ingressClassNameFilter == nil { + return ingresses, nil + } + + filteredList := []*v1beta1.Ingress{} + + for _, ingress := range ingresses { + for _, nameFilter := range sc.ingressClassNameFilter { + // include ingress if its annotations match the selector + if ingress.Spec.IngressClassName != nil && nameFilter == *ingress.Spec.IngressClassName { + filteredList = append(filteredList, ingress) + break + } + } + } + + return filteredList, nil +} + func (sc *ingressSource) setResourceLabel(ingress *networkv1.Ingress, endpoints []*endpoint.Endpoint) { for _, ep := range endpoints { ep.Labels[endpoint.ResourceLabelKey] = fmt.Sprintf("ingress/%s/%s", ingress.Namespace, ingress.Name) diff --git a/source/store.go b/source/store.go index 3b8372255..5cb2d47bf 100644 --- a/source/store.go +++ b/source/store.go @@ -43,6 +43,7 @@ type Config struct { Namespace string AnnotationFilter string LabelFilter string + IngressClassNameFilter []string FQDNTemplate string CombineFQDNAndAnnotation bool IgnoreHostnameAnnotation bool @@ -189,7 +190,7 @@ func BuildWithConfig(source string, p ClientGenerator, cfg *Config) (Source, err if err != nil { return nil, err } - return NewIngressSource(client, cfg.Namespace, cfg.AnnotationFilter, cfg.FQDNTemplate, cfg.CombineFQDNAndAnnotation, cfg.IgnoreHostnameAnnotation, cfg.IgnoreIngressTLSSpec, cfg.IgnoreIngressRulesSpec) + return NewIngressSource(client, cfg.Namespace, cfg.AnnotationFilter, cfg.FQDNTemplate, cfg.CombineFQDNAndAnnotation, cfg.IgnoreHostnameAnnotation, cfg.IgnoreIngressTLSSpec, cfg.IgnoreIngressRulesSpec, cfg.IngressClassNameFilter) case "pod": client, err := p.KubeClient() if err != nil { From 115e2501af1e9b4dc3a37bbc08a648e9031e24c5 Mon Sep 17 00:00:00 2001 From: Dave Salisbury Date: Sun, 18 Apr 2021 16:27:01 +1000 Subject: [PATCH 02/32] Add ingressClassNameFilter testing --- source/ingress_test.go | 47 +++++++++++++++++++++++++++++++++++------- 1 file changed, 40 insertions(+), 7 deletions(-) diff --git a/source/ingress_test.go b/source/ingress_test.go index c9d35494c..4b22be091 100644 --- a/source/ingress_test.go +++ b/source/ingress_test.go @@ -63,6 +63,7 @@ func (suite *IngressSuite) SetupTest() { false, false, false, + []string{}, ) suite.NoError(err, "should initialize ingress source") } @@ -144,6 +145,7 @@ func TestNewIngressSource(t *testing.T) { false, false, false, + []string{}, ) if ti.expectError { assert.Error(t, err) @@ -358,6 +360,7 @@ func testIngressEndpoints(t *testing.T) { ignoreHostnameAnnotation bool ignoreIngressTLSSpec bool ignoreIngressRulesSpec bool + ingressClassNameFilter []string }{ { title: "no ingress", @@ -1169,6 +1172,33 @@ func testIngressEndpoints(t *testing.T) { }, }, }, + { + title: "ingressClassName filtering", + targetNamespace: "", + ingressClassNameFilter: []string{"public"}, + ingressItems: []fakeIngress{ + { + name: "fake-public", + namespace: namespace, + tlsdnsnames: [][]string{{"example.org"}}, + ips: []string{"1.2.3.4"}, + ingressClassName: "public", + }, + { + name: "fake-internal", + namespace: namespace, + tlsdnsnames: [][]string{{"int.example.org"}}, + ips: []string{"2.3.4.5"}, + ingressClassName: "internal", + }, + }, + expected: []*endpoint.Endpoint{ + { + DNSName: "example.org", + Targets: endpoint.Targets{"1.2.3.4"}, + }, + }, + }, } { ti := ti t.Run(ti.title, func(t *testing.T) { @@ -1189,6 +1219,7 @@ func testIngressEndpoints(t *testing.T) { ti.ignoreHostnameAnnotation, ti.ignoreIngressTLSSpec, ti.ignoreIngressRulesSpec, + ti.ingressClassNameFilter, ) // Informer cache has all of the ingresses. Retrieve and validate their endpoints. res, err := source.Endpoints(context.Background()) @@ -1204,13 +1235,14 @@ func testIngressEndpoints(t *testing.T) { // ingress specific helper functions type fakeIngress struct { - dnsnames []string - tlsdnsnames [][]string - ips []string - hostnames []string - namespace string - name string - annotations map[string]string + dnsnames []string + tlsdnsnames [][]string + ips []string + hostnames []string + namespace string + name string + annotations map[string]string + ingressClassName string } func (ing fakeIngress) Ingress() *networkv1.Ingress { @@ -1222,6 +1254,7 @@ func (ing fakeIngress) Ingress() *networkv1.Ingress { }, Spec: networkv1.IngressSpec{ Rules: []networkv1.IngressRule{}, + IngressClassName: &ing.ingressClassName, }, Status: networkv1.IngressStatus{ LoadBalancer: v1.LoadBalancerStatus{ From c823eba1392ecd493c5e33f598799a41d657ddd7 Mon Sep 17 00:00:00 2001 From: Dave Salisbury Date: Sat, 2 Oct 2021 14:34:34 +1000 Subject: [PATCH 03/32] ingress source: update code for networkv1 --- source/ingress.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/ingress.go b/source/ingress.go index 01b987300..184ce299c 100644 --- a/source/ingress.go +++ b/source/ingress.go @@ -219,13 +219,13 @@ func (sc *ingressSource) filterByAnnotations(ingresses []*networkv1.Ingress) ([] // filterByIngressClass filters a list of ingresses based on a required ingress // class -func (sc *ingressSource) filterByIngressClass(ingresses []*v1beta1.Ingress) ([]*v1beta1.Ingress, error) { +func (sc *ingressSource) filterByIngressClass(ingresses []*networkv1.Ingress) ([]*networkv1.Ingress, error) { // if no class is specified then there's nothing to do if sc.ingressClassNameFilter == nil { return ingresses, nil } - filteredList := []*v1beta1.Ingress{} + filteredList := []*networkv1.Ingress{} for _, ingress := range ingresses { for _, nameFilter := range sc.ingressClassNameFilter { From 0b6c67fe56d5dee40cafa4d2922cb80716477a07 Mon Sep 17 00:00:00 2001 From: Dave Salisbury Date: Sat, 2 Oct 2021 14:34:49 +1000 Subject: [PATCH 04/32] cli args: rename ingress-class-filter to ingress-classes --- pkg/apis/externaldns/types.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/apis/externaldns/types.go b/pkg/apis/externaldns/types.go index e9153ab7c..4210f1df8 100644 --- a/pkg/apis/externaldns/types.go +++ b/pkg/apis/externaldns/types.go @@ -364,7 +364,7 @@ func (cfg *Config) ParseFlags(args []string) error { app.Flag("namespace", "Limit sources of endpoints to a specific namespace (default: all namespaces)").Default(defaultConfig.Namespace).StringVar(&cfg.Namespace) app.Flag("annotation-filter", "Filter sources managed by external-dns via annotation using label selector semantics (default: all sources)").Default(defaultConfig.AnnotationFilter).StringVar(&cfg.AnnotationFilter) app.Flag("label-filter", "Filter sources managed by external-dns via label selector when listing all resources; currently only supported by source CRD").Default(defaultConfig.LabelFilter).StringVar(&cfg.LabelFilter) - app.Flag("ingress-class-filter", "Filter ingresses to just these ingress class(es)").StringsVar(&cfg.IngressClassNameFilter) + app.Flag("ingress-classes", "Restrict ingress source to just these classes (defaults to any class)").StringsVar(&cfg.IngressClassNameFilter) app.Flag("fqdn-template", "A templated string that's used to generate DNS names from sources that don't define a hostname themselves, or to add a hostname suffix when paired with the fake source (optional). Accepts comma separated list for multiple global FQDN.").Default(defaultConfig.FQDNTemplate).StringVar(&cfg.FQDNTemplate) app.Flag("combine-fqdn-annotation", "Combine FQDN template and Annotations instead of overwriting").BoolVar(&cfg.CombineFQDNAndAnnotation) app.Flag("ignore-hostname-annotation", "Ignore hostname annotation when generating DNS names, valid only when using fqdn-template is set (optional, default: false)").BoolVar(&cfg.IgnoreHostnameAnnotation) From da71b3fff860ee65e71ca5d89196e821a775ca43 Mon Sep 17 00:00:00 2001 From: Dave Salisbury Date: Sat, 2 Oct 2021 15:13:45 +1000 Subject: [PATCH 05/32] ingress source: improve class name filtering and logging --- source/ingress.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/source/ingress.go b/source/ingress.go index 184ce299c..e3aa63ff2 100644 --- a/source/ingress.go +++ b/source/ingress.go @@ -220,7 +220,7 @@ func (sc *ingressSource) filterByAnnotations(ingresses []*networkv1.Ingress) ([] // filterByIngressClass filters a list of ingresses based on a required ingress // class func (sc *ingressSource) filterByIngressClass(ingresses []*networkv1.Ingress) ([]*networkv1.Ingress, error) { - // if no class is specified then there's nothing to do + // if no class filter is specified then there's nothing to do if sc.ingressClassNameFilter == nil { return ingresses, nil } @@ -228,13 +228,22 @@ func (sc *ingressSource) filterByIngressClass(ingresses []*networkv1.Ingress) ([ filteredList := []*networkv1.Ingress{} for _, ingress := range ingresses { + // we have a filter class but this ingress doesn't have its class set + if ingress.Spec.IngressClassName == nil { + log.Debugf("Ignoring ingress %s/%s because ingressClassName is not set", ingress.Namespace, ingress.Name) + } + + var matched = false; for _, nameFilter := range sc.ingressClassNameFilter { - // include ingress if its annotations match the selector - if ingress.Spec.IngressClassName != nil && nameFilter == *ingress.Spec.IngressClassName { + if nameFilter == *ingress.Spec.IngressClassName { filteredList = append(filteredList, ingress) + matched = true; break } } + if matched == false { + log.Debugf("Ignoring ingress %s/%s because ingressClassName '%s' is not in specified list", ingress.Namespace, ingress.Name, *ingress.Spec.IngressClassName) + } } return filteredList, nil From 8da6f99857dc405bc4a0c50a7c416a86e421ea36 Mon Sep 17 00:00:00 2001 From: Dave Salisbury Date: Sat, 2 Oct 2021 15:21:37 +1000 Subject: [PATCH 06/32] Rename ingressClassNameFilter to ingressClassNames ...and update the help text to specify use more clearly --- main.go | 2 +- pkg/apis/externaldns/types.go | 6 +++--- source/ingress.go | 10 +++++----- source/ingress_test.go | 2 +- source/store.go | 4 ++-- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/main.go b/main.go index b10c3372a..bc26e0f4d 100644 --- a/main.go +++ b/main.go @@ -104,7 +104,7 @@ func main() { Namespace: cfg.Namespace, AnnotationFilter: cfg.AnnotationFilter, LabelFilter: cfg.LabelFilter, - IngressClassNameFilter: cfg.IngressClassNameFilter, + IngressClassNames: cfg.IngressClassNames, FQDNTemplate: cfg.FQDNTemplate, CombineFQDNAndAnnotation: cfg.CombineFQDNAndAnnotation, IgnoreHostnameAnnotation: cfg.IgnoreHostnameAnnotation, diff --git a/pkg/apis/externaldns/types.go b/pkg/apis/externaldns/types.go index 4210f1df8..f9df5d933 100644 --- a/pkg/apis/externaldns/types.go +++ b/pkg/apis/externaldns/types.go @@ -53,7 +53,7 @@ type Config struct { Namespace string AnnotationFilter string LabelFilter string - IngressClassNameFilter []string + IngressClassNames []string FQDNTemplate string CombineFQDNAndAnnotation bool IgnoreHostnameAnnotation bool @@ -187,7 +187,7 @@ var defaultConfig = &Config{ Namespace: "", AnnotationFilter: "", LabelFilter: "", - IngressClassNameFilter: nil, + IngressClassNames: nil, FQDNTemplate: "", CombineFQDNAndAnnotation: false, IgnoreHostnameAnnotation: false, @@ -364,7 +364,7 @@ func (cfg *Config) ParseFlags(args []string) error { app.Flag("namespace", "Limit sources of endpoints to a specific namespace (default: all namespaces)").Default(defaultConfig.Namespace).StringVar(&cfg.Namespace) app.Flag("annotation-filter", "Filter sources managed by external-dns via annotation using label selector semantics (default: all sources)").Default(defaultConfig.AnnotationFilter).StringVar(&cfg.AnnotationFilter) app.Flag("label-filter", "Filter sources managed by external-dns via label selector when listing all resources; currently only supported by source CRD").Default(defaultConfig.LabelFilter).StringVar(&cfg.LabelFilter) - app.Flag("ingress-classes", "Restrict ingress source to just these classes (defaults to any class)").StringsVar(&cfg.IngressClassNameFilter) + app.Flag("ingress-class", "Require an ingress to have this class name (defaults to any class; specify multiple times to allow more than one class)").StringsVar(&cfg.IngressClassNames) app.Flag("fqdn-template", "A templated string that's used to generate DNS names from sources that don't define a hostname themselves, or to add a hostname suffix when paired with the fake source (optional). Accepts comma separated list for multiple global FQDN.").Default(defaultConfig.FQDNTemplate).StringVar(&cfg.FQDNTemplate) app.Flag("combine-fqdn-annotation", "Combine FQDN template and Annotations instead of overwriting").BoolVar(&cfg.CombineFQDNAndAnnotation) app.Flag("ignore-hostname-annotation", "Ignore hostname annotation when generating DNS names, valid only when using fqdn-template is set (optional, default: false)").BoolVar(&cfg.IgnoreHostnameAnnotation) diff --git a/source/ingress.go b/source/ingress.go index e3aa63ff2..0f27d6bb8 100644 --- a/source/ingress.go +++ b/source/ingress.go @@ -54,7 +54,7 @@ type ingressSource struct { client kubernetes.Interface namespace string annotationFilter string - ingressClassNameFilter []string + ingressClassNames []string fqdnTemplate *template.Template combineFQDNAnnotation bool ignoreHostnameAnnotation bool @@ -64,7 +64,7 @@ type ingressSource struct { } // NewIngressSource creates a new ingressSource with the given config. -func NewIngressSource(kubeClient kubernetes.Interface, namespace, annotationFilter string, fqdnTemplate string, combineFqdnAnnotation bool, ignoreHostnameAnnotation bool, ignoreIngressTLSSpec bool, ignoreIngressRulesSpec bool, ingressClassNameFilter []string) (Source, error) { +func NewIngressSource(kubeClient kubernetes.Interface, namespace, annotationFilter string, fqdnTemplate string, combineFqdnAnnotation bool, ignoreHostnameAnnotation bool, ignoreIngressTLSSpec bool, ignoreIngressRulesSpec bool, ingressClassNames []string) (Source, error) { tmpl, err := parseTemplate(fqdnTemplate) if err != nil { return nil, err @@ -95,7 +95,7 @@ func NewIngressSource(kubeClient kubernetes.Interface, namespace, annotationFilt client: kubeClient, namespace: namespace, annotationFilter: annotationFilter, - ingressClassNameFilter: ingressClassNameFilter, + ingressClassNames: ingressClassNames, fqdnTemplate: tmpl, combineFQDNAnnotation: combineFqdnAnnotation, ignoreHostnameAnnotation: ignoreHostnameAnnotation, @@ -221,7 +221,7 @@ func (sc *ingressSource) filterByAnnotations(ingresses []*networkv1.Ingress) ([] // class func (sc *ingressSource) filterByIngressClass(ingresses []*networkv1.Ingress) ([]*networkv1.Ingress, error) { // if no class filter is specified then there's nothing to do - if sc.ingressClassNameFilter == nil { + if sc.ingressClassNames == nil { return ingresses, nil } @@ -234,7 +234,7 @@ func (sc *ingressSource) filterByIngressClass(ingresses []*networkv1.Ingress) ([ } var matched = false; - for _, nameFilter := range sc.ingressClassNameFilter { + for _, nameFilter := range sc.ingressClassNames { if nameFilter == *ingress.Spec.IngressClassName { filteredList = append(filteredList, ingress) matched = true; diff --git a/source/ingress_test.go b/source/ingress_test.go index 4b22be091..804f35fa9 100644 --- a/source/ingress_test.go +++ b/source/ingress_test.go @@ -360,7 +360,7 @@ func testIngressEndpoints(t *testing.T) { ignoreHostnameAnnotation bool ignoreIngressTLSSpec bool ignoreIngressRulesSpec bool - ingressClassNameFilter []string + ingressClassNames []string }{ { title: "no ingress", diff --git a/source/store.go b/source/store.go index 5cb2d47bf..9e5743ffb 100644 --- a/source/store.go +++ b/source/store.go @@ -43,7 +43,7 @@ type Config struct { Namespace string AnnotationFilter string LabelFilter string - IngressClassNameFilter []string + IngressClassNames []string FQDNTemplate string CombineFQDNAndAnnotation bool IgnoreHostnameAnnotation bool @@ -190,7 +190,7 @@ func BuildWithConfig(source string, p ClientGenerator, cfg *Config) (Source, err if err != nil { return nil, err } - return NewIngressSource(client, cfg.Namespace, cfg.AnnotationFilter, cfg.FQDNTemplate, cfg.CombineFQDNAndAnnotation, cfg.IgnoreHostnameAnnotation, cfg.IgnoreIngressTLSSpec, cfg.IgnoreIngressRulesSpec, cfg.IngressClassNameFilter) + return NewIngressSource(client, cfg.Namespace, cfg.AnnotationFilter, cfg.FQDNTemplate, cfg.CombineFQDNAndAnnotation, cfg.IgnoreHostnameAnnotation, cfg.IgnoreIngressTLSSpec, cfg.IgnoreIngressRulesSpec, cfg.IngressClassNames) case "pod": client, err := p.KubeClient() if err != nil { From ac0c4be36ad8a308287e178f090c66cf15c64ecc Mon Sep 17 00:00:00 2001 From: Dave Salisbury Date: Sat, 2 Oct 2021 15:23:48 +1000 Subject: [PATCH 07/32] Update tests for ingress class filtering --- source/ingress_test.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/source/ingress_test.go b/source/ingress_test.go index 804f35fa9..9f1590a4c 100644 --- a/source/ingress_test.go +++ b/source/ingress_test.go @@ -1175,7 +1175,7 @@ func testIngressEndpoints(t *testing.T) { { title: "ingressClassName filtering", targetNamespace: "", - ingressClassNameFilter: []string{"public"}, + ingressClassNames: []string{"public", "dmz"}, ingressItems: []fakeIngress{ { name: "fake-public", @@ -1191,12 +1191,23 @@ func testIngressEndpoints(t *testing.T) { ips: []string{"2.3.4.5"}, ingressClassName: "internal", }, + { + name: "fake-dmz", + namespace: namespace, + tlsdnsnames: [][]string{{"dmz.example.org"}}, + ips: []string{"3.4.5.6"}, + ingressClassName: "dmz", + }, }, expected: []*endpoint.Endpoint{ { DNSName: "example.org", Targets: endpoint.Targets{"1.2.3.4"}, }, + { + DNSName: "dmz.example.org", + Targets: endpoint.Targets{"3.4.5.6"}, + }, }, }, } { @@ -1219,7 +1230,7 @@ func testIngressEndpoints(t *testing.T) { ti.ignoreHostnameAnnotation, ti.ignoreIngressTLSSpec, ti.ignoreIngressRulesSpec, - ti.ingressClassNameFilter, + ti.ingressClassNames, ) // Informer cache has all of the ingresses. Retrieve and validate their endpoints. res, err := source.Endpoints(context.Background()) From 71a672fe720c5ca99e7b019e086ceb563ec52560 Mon Sep 17 00:00:00 2001 From: Dave Salisbury Date: Sat, 2 Oct 2021 16:02:33 +1000 Subject: [PATCH 08/32] ingress source: check for duplicate classname configs we don't want to get incompatible restrictions by letting someone set "--ingress-class" and --annotation-filter="kubernetes.io/ingress.class" at the same time --- source/ingress.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/source/ingress.go b/source/ingress.go index 0f27d6bb8..bb0bf2961 100644 --- a/source/ingress.go +++ b/source/ingress.go @@ -18,6 +18,7 @@ package source import ( "context" + "errors" "fmt" "sort" "strings" @@ -70,6 +71,21 @@ func NewIngressSource(kubeClient kubernetes.Interface, namespace, annotationFilt return nil, err } + // ensure that ingress class is only set in either the ingressClassNames or + // annotationFilter but not both + if ingressClassNames != nil && annotationFilter != "" { + selector, err := getLabelSelector(annotationFilter) + if err != nil { + return nil, err + } + + requirements, _ := selector.Requirements() + for _, requirement := range requirements { + if requirement.Key() == "kubernetes.io/ingress.class" { + return nil, errors.New("--ingress-class is mutually exclusive with kubernetes.io/ingress.class annotation") + } + } + } // Use shared informer to listen for add/update/delete of ingresses in the specified namespace. // Set resync period to 0, to prevent processing when nothing has changed. informerFactory := kubeinformers.NewSharedInformerFactoryWithOptions(kubeClient, 0, kubeinformers.WithNamespace(namespace)) From 901effbca5207e2f8e49d6e2e066be9e8470b0be Mon Sep 17 00:00:00 2001 From: Dave Salisbury Date: Sat, 2 Oct 2021 16:32:40 +1000 Subject: [PATCH 09/32] ingress source: ingressClassNames now feed into annotation filter --- source/ingress.go | 27 +++++++++++++++++++-------- source/ingress_test.go | 13 +++++++++++++ 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/source/ingress.go b/source/ingress.go index bb0bf2961..58fb119be 100644 --- a/source/ingress.go +++ b/source/ingress.go @@ -27,6 +27,7 @@ import ( log "github.com/sirupsen/logrus" networkv1 "k8s.io/api/networking/v1" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" "k8s.io/apimachinery/pkg/util/wait" kubeinformers "k8s.io/client-go/informers" netinformers "k8s.io/client-go/informers/networking/v1" @@ -241,24 +242,34 @@ func (sc *ingressSource) filterByIngressClass(ingresses []*networkv1.Ingress) ([ return ingresses, nil } + classNameReq, err := labels.NewRequirement("kubernetes.io/ingress.class", selection.In, sc.ingressClassNames) + if err != nil { + return nil, errors.New("Failed to create selector requirement from ingress class names") + } + + selector := labels.NewSelector() + selector = selector.Add(*classNameReq) + filteredList := []*networkv1.Ingress{} for _, ingress := range ingresses { - // we have a filter class but this ingress doesn't have its class set - if ingress.Spec.IngressClassName == nil { - log.Debugf("Ignoring ingress %s/%s because ingressClassName is not set", ingress.Namespace, ingress.Name) - } - var matched = false; + for _, nameFilter := range sc.ingressClassNames { - if nameFilter == *ingress.Spec.IngressClassName { - filteredList = append(filteredList, ingress) + if ingress.Spec.IngressClassName != nil && nameFilter == *ingress.Spec.IngressClassName { matched = true; + } else if matchLabelSelector(selector, ingress.Annotations) { + matched = true; + } + + if matched == true { + filteredList = append(filteredList, ingress) break } } + if matched == false { - log.Debugf("Ignoring ingress %s/%s because ingressClassName '%s' is not in specified list", ingress.Namespace, ingress.Name, *ingress.Spec.IngressClassName) + log.Debugf("Discarding ingress %s/%s because it does not match required ingress classes %v", ingress.Namespace, ingress.Name, sc.ingressClassNames) } } diff --git a/source/ingress_test.go b/source/ingress_test.go index 9f1590a4c..9cf60d963 100644 --- a/source/ingress_test.go +++ b/source/ingress_test.go @@ -1198,6 +1198,15 @@ func testIngressEndpoints(t *testing.T) { ips: []string{"3.4.5.6"}, ingressClassName: "dmz", }, + { + name: "annotated-dmz", + namespace: namespace, + tlsdnsnames: [][]string{{"annodmz.example.org"}}, + ips: []string{"4.5.6.7"}, + annotations: map[string]string{ + "kubernetes.io/ingress.class": "dmz", + }, + }, }, expected: []*endpoint.Endpoint{ { @@ -1208,6 +1217,10 @@ func testIngressEndpoints(t *testing.T) { DNSName: "dmz.example.org", Targets: endpoint.Targets{"3.4.5.6"}, }, + { + DNSName: "annodmz.example.org", + Targets: endpoint.Targets{"4.5.6.7"}, + }, }, }, } { From 807e213590b01e01e52060f564e63fbb8ce7bba5 Mon Sep 17 00:00:00 2001 From: Dave Salisbury Date: Sat, 2 Oct 2021 16:37:02 +1000 Subject: [PATCH 10/32] docs: add info about new --ingress-class --- docs/faq.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/faq.md b/docs/faq.md index 67662a782..2ce8ab48a 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -254,13 +254,16 @@ Sometimes you need to run an internal and an external dns service. The internal one should provision hostnames used on the internal network (perhaps inside a VPC), and the external one to expose DNS to the internet. -To do this with ExternalDNS you can use the `--annotation-filter` to specifically tie an instance of ExternalDNS to +To do this with ExternalDNS you can use the `--ingress-class` to specifically tie an instance of ExternalDNS to an instance of a ingress controller. Let's assume you have two ingress controllers `nginx-internal` and `nginx-external` -then you can start two ExternalDNS providers one with `--annotation-filter=kubernetes.io/ingress.class in (nginx-internal)` -and one with `--annotation-filter=kubernetes.io/ingress.class in (nginx-external)`. +then you can start two ExternalDNS providers one with `--ingress-class=nginx-internal` and one with `--ingress-class=nginx-external`. -If you need to search for multiple values of said annotation, you can provide a comma separated list, like so: -`--annotation-filter=kubernetes.io/ingress.class in (nginx-internal, alb-ingress-internal)`. +If you need to search for multiple ingress classes, you can specify the argument multiple times, like so: +`--ingress-class=nginx-internal --ingress-class=alb-ingress-internal`. + +The `--ingress-class` argument will check both the ingressClassName field as well as the deprecated `kubernetes.io/ingress.class` annotation. + +Note: the `--ingress-class` argument cannot be used at the same time as a `kubernetes.io/ingress.class` annotation filter; if you do this an error will be raised. Beware when using multiple sources, e.g. `--source=service --source=ingress`, `--annotation-filter` will filter every given source objects. If you need to filter only one specific source you have to run a separated external dns service containing only the wanted `--source` and `--annotation-filter`. From 8ec342a1c6ac3c95540ee87c4f3c94afe9f15635 Mon Sep 17 00:00:00 2001 From: Dave Salisbury Date: Sat, 2 Oct 2021 16:41:39 +1000 Subject: [PATCH 11/32] docs: format ingressClassName as inline code --- docs/faq.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/faq.md b/docs/faq.md index 2ce8ab48a..2ce3ed481 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -261,7 +261,7 @@ then you can start two ExternalDNS providers one with `--ingress-class=nginx-int If you need to search for multiple ingress classes, you can specify the argument multiple times, like so: `--ingress-class=nginx-internal --ingress-class=alb-ingress-internal`. -The `--ingress-class` argument will check both the ingressClassName field as well as the deprecated `kubernetes.io/ingress.class` annotation. +The `--ingress-class` argument will check both the `ingressClassName` field as well as the deprecated `kubernetes.io/ingress.class` annotation. Note: the `--ingress-class` argument cannot be used at the same time as a `kubernetes.io/ingress.class` annotation filter; if you do this an error will be raised. From a24217a720278d84eee2d9881e952f87a3d307e1 Mon Sep 17 00:00:00 2001 From: Dave Salisbury Date: Sat, 2 Oct 2021 16:49:51 +1000 Subject: [PATCH 12/32] ingress source: use shorthand bool comparison --- source/ingress.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/ingress.go b/source/ingress.go index 58fb119be..b92e4475d 100644 --- a/source/ingress.go +++ b/source/ingress.go @@ -262,13 +262,13 @@ func (sc *ingressSource) filterByIngressClass(ingresses []*networkv1.Ingress) ([ matched = true; } - if matched == true { + if matched { filteredList = append(filteredList, ingress) break } } - if matched == false { + if !matched { log.Debugf("Discarding ingress %s/%s because it does not match required ingress classes %v", ingress.Namespace, ingress.Name, sc.ingressClassNames) } } From 0aa3d55450cf06161583156ff63b61c73473fff2 Mon Sep 17 00:00:00 2001 From: Dave Salisbury Date: Sat, 2 Oct 2021 16:50:35 +1000 Subject: [PATCH 13/32] ingress source: pass back err when requirement fails --- source/ingress.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/ingress.go b/source/ingress.go index b92e4475d..371ba9248 100644 --- a/source/ingress.go +++ b/source/ingress.go @@ -244,7 +244,7 @@ func (sc *ingressSource) filterByIngressClass(ingresses []*networkv1.Ingress) ([ classNameReq, err := labels.NewRequirement("kubernetes.io/ingress.class", selection.In, sc.ingressClassNames) if err != nil { - return nil, errors.New("Failed to create selector requirement from ingress class names") + return nil, err } selector := labels.NewSelector() From 6c4a450b254bdfc23f4243f6bef657e406ecb028 Mon Sep 17 00:00:00 2001 From: Dave Salisbury Date: Sat, 2 Oct 2021 16:54:42 +1000 Subject: [PATCH 14/32] gofmt simplify new ingress-class changes --- pkg/apis/externaldns/types.go | 2 +- source/ingress.go | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/apis/externaldns/types.go b/pkg/apis/externaldns/types.go index f9df5d933..a59e356be 100644 --- a/pkg/apis/externaldns/types.go +++ b/pkg/apis/externaldns/types.go @@ -364,7 +364,7 @@ func (cfg *Config) ParseFlags(args []string) error { app.Flag("namespace", "Limit sources of endpoints to a specific namespace (default: all namespaces)").Default(defaultConfig.Namespace).StringVar(&cfg.Namespace) app.Flag("annotation-filter", "Filter sources managed by external-dns via annotation using label selector semantics (default: all sources)").Default(defaultConfig.AnnotationFilter).StringVar(&cfg.AnnotationFilter) app.Flag("label-filter", "Filter sources managed by external-dns via label selector when listing all resources; currently only supported by source CRD").Default(defaultConfig.LabelFilter).StringVar(&cfg.LabelFilter) - app.Flag("ingress-class", "Require an ingress to have this class name (defaults to any class; specify multiple times to allow more than one class)").StringsVar(&cfg.IngressClassNames) + app.Flag("ingress-class", "Require an ingress to have this class name (defaults to any class; specify multiple times to allow more than one class)").StringsVar(&cfg.IngressClassNames) app.Flag("fqdn-template", "A templated string that's used to generate DNS names from sources that don't define a hostname themselves, or to add a hostname suffix when paired with the fake source (optional). Accepts comma separated list for multiple global FQDN.").Default(defaultConfig.FQDNTemplate).StringVar(&cfg.FQDNTemplate) app.Flag("combine-fqdn-annotation", "Combine FQDN template and Annotations instead of overwriting").BoolVar(&cfg.CombineFQDNAndAnnotation) app.Flag("ignore-hostname-annotation", "Ignore hostname annotation when generating DNS names, valid only when using fqdn-template is set (optional, default: false)").BoolVar(&cfg.IgnoreHostnameAnnotation) diff --git a/source/ingress.go b/source/ingress.go index 371ba9248..33a16c3b8 100644 --- a/source/ingress.go +++ b/source/ingress.go @@ -56,7 +56,7 @@ type ingressSource struct { client kubernetes.Interface namespace string annotationFilter string - ingressClassNames []string + ingressClassNames []string fqdnTemplate *template.Template combineFQDNAnnotation bool ignoreHostnameAnnotation bool @@ -135,7 +135,7 @@ func (sc *ingressSource) Endpoints(ctx context.Context) ([]*endpoint.Endpoint, e return nil, err } - ingresses, err = sc.filterByIngressClass(ingresses) + ingresses, err = sc.filterByIngressClass(ingresses) if err != nil { return nil, err } @@ -253,13 +253,13 @@ func (sc *ingressSource) filterByIngressClass(ingresses []*networkv1.Ingress) ([ filteredList := []*networkv1.Ingress{} for _, ingress := range ingresses { - var matched = false; + var matched = false for _, nameFilter := range sc.ingressClassNames { if ingress.Spec.IngressClassName != nil && nameFilter == *ingress.Spec.IngressClassName { - matched = true; + matched = true } else if matchLabelSelector(selector, ingress.Annotations) { - matched = true; + matched = true } if matched { From 31bc5bb0778d9dd06ab93b0c26b08fd50eb5ddba Mon Sep 17 00:00:00 2001 From: Dave Salisbury Date: Sat, 2 Oct 2021 17:08:54 +1000 Subject: [PATCH 15/32] ingress source: fix broken NewIngressSource test and add an extra one for the mutual exclusivity of ingressClassNames and ingress.class annotationFilters --- source/ingress_test.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/source/ingress_test.go b/source/ingress_test.go index 9cf60d963..f79a7b222 100644 --- a/source/ingress_test.go +++ b/source/ingress_test.go @@ -100,6 +100,7 @@ func TestNewIngressSource(t *testing.T) { fqdnTemplate string combineFQDNAndAnnotation bool expectError bool + ingressClassNames []string }{ { title: "invalid template", @@ -131,6 +132,17 @@ func TestNewIngressSource(t *testing.T) { expectError: false, annotationFilter: "kubernetes.io/ingress.class=nginx", }, + { + title: "non-empty ingress class name list", + expectError: false, + ingressClassNames: []string{"internal", "external"}, + }, + { + title: "ingress class name and annotation filter jointly specified", + expectError: true, + ingressClassNames: []string{"internal", "external"}, + annotationFilter: "kubernetes.io/ingress.class=nginx", + }, } { ti := ti t.Run(ti.title, func(t *testing.T) { @@ -145,7 +157,7 @@ func TestNewIngressSource(t *testing.T) { false, false, false, - []string{}, + ti.ingressClassNames, ) if ti.expectError { assert.Error(t, err) From 99c10f56c634ffcaaa6a4f775d156fbdd81222c9 Mon Sep 17 00:00:00 2001 From: Dave Salisbury Date: Mon, 14 Feb 2022 16:37:28 +1100 Subject: [PATCH 16/32] ingress source: improve mutual exclusive error message --- source/ingress.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/ingress.go b/source/ingress.go index 005739210..56833e15c 100644 --- a/source/ingress.go +++ b/source/ingress.go @@ -83,7 +83,7 @@ func NewIngressSource(ctx context.Context, kubeClient kubernetes.Interface, name requirements, _ := selector.Requirements() for _, requirement := range requirements { if requirement.Key() == "kubernetes.io/ingress.class" { - return nil, errors.New("--ingress-class is mutually exclusive with kubernetes.io/ingress.class annotation") + return nil, errors.New("--ingress-class is mutually exclusive with the kubernetes.io/ingress.class annotation filter") } } } From 15f27aafd3517531978685a3039a054df2c0b985 Mon Sep 17 00:00:00 2001 From: Dave Salisbury Date: Mon, 14 Feb 2022 16:48:56 +1100 Subject: [PATCH 17/32] faq: extend notes about annotation filter risks --- docs/faq.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/faq.md b/docs/faq.md index 9cda9c59f..10a36422f 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -269,10 +269,12 @@ The `--ingress-class` argument will check both the `ingressClassName` field as w Note: the `--ingress-class` argument cannot be used at the same time as a `kubernetes.io/ingress.class` annotation filter; if you do this an error will be raised. -Beware when using multiple sources, e.g. `--source=service --source=ingress`, `--annotation-filter` will filter every given source objects. -If you need to filter only one specific source you have to run a separated external dns service containing only the wanted `--source` and `--annotation-filter`. +If you use annotations to indicate different ingress classes in your cluster, you can instead use an `--annotation-filter` argument to restrict which objects ExternalDNS considers; for example, `--annotation-filter=kubernetes.io/ingress.class in (public,dmz)`. -**Note:** Filtering based on annotation means that the external-dns controller will receive all resources of that kind and then filter on the client-side. +However, beware when using annotation filters with multiple sources, e.g. `--source=service --source=ingress`, since `--annotation-filter` will filter every given source objects. +If you need to use annotation filters against a specific source you have to run a separated external dns service containing only the wanted `--source` and `--annotation-filter`. + +**Note:** Filtering based on annotation or ingress class name means that the external-dns controller will receive all resources of that kind and then filter on the client-side. In larger clusters with many resources which change frequently this can cause performance issues. If only some resources need to be managed by an instance of external-dns then label filtering can be used instead of annotation filtering. This means that only those resources which match the selector specified in `--label-filter` will be passed to the controller. From f76382a5adb1e92e821978b6b9ee298cb9514ac0 Mon Sep 17 00:00:00 2001 From: Dave Salisbury Date: Mon, 14 Feb 2022 16:57:01 +1100 Subject: [PATCH 18/32] docs: update public-private-route53 example for ingress-class filtering --- docs/tutorials/public-private-route53.md | 32 ++++++++++++++++-------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/docs/tutorials/public-private-route53.md b/docs/tutorials/public-private-route53.md index 764a7e747..b8110ecbd 100644 --- a/docs/tutorials/public-private-route53.md +++ b/docs/tutorials/public-private-route53.md @@ -213,7 +213,7 @@ spec: Consult [AWS ExternalDNS setup docs](aws.md) for installation guidelines. -In ExternalDNS containers args, make sure to specify `annotation-filter` and `aws-zone-type`: +In ExternalDNS containers args, make sure to specify `aws-zone-type` and either `ingress-class` or `annotation-filter` (depending on whether your cluster makes use of `ingressClassName`): ```yaml apiVersion: apps/v1beta2 @@ -241,7 +241,9 @@ spec: - --provider=aws - --registry=txt - --txt-owner-id=external-dns - - --annotation-filter=kubernetes.io/ingress.class in (external-ingress) + - --ingress-class=external-ingress + # ... or, if you use annotations for ingress classes + # - --annotation-filter=kubernetes.io/ingress.class in (external-ingress) - --aws-zone-type=public image: k8s.gcr.io/external-dns/external-dns:v0.7.6 name: external-dns-public @@ -251,7 +253,7 @@ spec: Consult [AWS ExternalDNS setup docs](aws.md) for installation guidelines. -In ExternalDNS containers args, make sure to specify `annotation-filter` and `aws-zone-type`: +In ExternalDNS containers args, make sure to specify `aws-zone-type` and either `ingress-class` or `annotation-filter` (depending on whether your cluster makes use of `ingressClassName`): ```yaml apiVersion: apps/v1beta2 @@ -279,7 +281,9 @@ spec: - --provider=aws - --registry=txt - --txt-owner-id=dev.k8s.nexus - - --annotation-filter=kubernetes.io/ingress.class in (internal-ingress) + - --ingress-class=internal-ingress + # ... or, if you use annotations for ingress classes + # - --annotation-filter=kubernetes.io/ingress.class in (internal-ingress) - --aws-zone-type=private image: k8s.gcr.io/external-dns/external-dns:v0.7.6 name: external-dns-private @@ -287,20 +291,23 @@ spec: ## Create application Service definitions -For this setup to work, you've to create two Service definitions for your application. +For this setup to work, you need to create two Ingress definitions for your application. -At first, create public Service definition: +At first, create public Ingress definition (make sure to un-comment either the `annotations` or `ingressClassName` lines): ```yaml apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - annotations: - kubernetes.io/ingress.class: "external-ingress" + # uncomment if you use annotations for ingress classes + # annotations: + # kubernetes.io/ingress.class: "external-ingress" labels: app: app name: app-public spec: + # uncomment if you use ingressClassName + # ingressClassName: external-ingress rules: - host: app.domain.com http: @@ -310,18 +317,21 @@ spec: servicePort: 80 ``` -Then create private Service definition: +Then create private Ingress definition (again, make sure to un-comment either the `annotations` or `ingressClassName` lines): ```yaml apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - annotations: - kubernetes.io/ingress.class: "internal-ingress" + # uncomment if you use annotations for ingress classes + # annotations: + # kubernetes.io/ingress.class: "internal-ingress" labels: app: app name: app-private spec: + # uncomment if you use ingressClassName + # ingressClassName: internal-ingress rules: - host: app.domain.com http: From 07dc39abc7a7ae4e7625e08921922f3d8fa05d9f Mon Sep 17 00:00:00 2001 From: Ole Markus With Date: Sun, 20 Nov 2022 15:40:00 +0100 Subject: [PATCH 19/32] If controller error propagates all the way out, bail execution --- controller/controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/controller/controller.go b/controller/controller.go index 528d870fa..742c6f2c7 100644 --- a/controller/controller.go +++ b/controller/controller.go @@ -292,7 +292,7 @@ func (c *Controller) Run(ctx context.Context) { for { if c.ShouldRunOnce(time.Now()) { if err := c.RunOnce(ctx); err != nil { - log.Error(err) + log.Fatal(err) } } select { From 4c546fb73a4cf526e3913b996091043054ccec48 Mon Sep 17 00:00:00 2001 From: Maurice Meyer Date: Fri, 31 Mar 2023 17:34:20 +0200 Subject: [PATCH 20/32] docs: add example for provider specific config --- docs/contributing/crd-source/dnsendpoint-example.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/contributing/crd-source/dnsendpoint-example.yaml b/docs/contributing/crd-source/dnsendpoint-example.yaml index 894301d8a..2ed7d7fa0 100644 --- a/docs/contributing/crd-source/dnsendpoint-example.yaml +++ b/docs/contributing/crd-source/dnsendpoint-example.yaml @@ -9,3 +9,7 @@ spec: recordType: A targets: - 192.168.99.216 + # Provider specific configurations are set like an annotation would on other sources + providerSpecific: + - name: external-dns.alpha.kubernetes.io/cloudflare-proxied + value: "true" From 2554f9f879d3364156f878f4db619224a69aa7a3 Mon Sep 17 00:00:00 2001 From: Viacheslav Sychov Date: Fri, 28 Apr 2023 20:29:54 +0200 Subject: [PATCH 21/32] #1828: Support encrypted DNS txt records Signed-off-by: Viacheslav Sychov --- docs/registry.md | 65 +++++++++++++++++ endpoint/crypto.go | 134 ++++++++++++++++++++++++++++++++++ endpoint/crypto_test.go | 58 +++++++++++++++ endpoint/labels.go | 56 +++++++++++++- endpoint/labels_test.go | 71 +++++++++++++----- main.go | 2 +- pkg/apis/externaldns/types.go | 6 ++ provider/awssd/aws_sd.go | 2 +- registry/aws_sd_registry.go | 4 +- registry/txt.go | 23 ++++-- registry/txt_test.go | 54 ++++++++------ 11 files changed, 423 insertions(+), 52 deletions(-) create mode 100644 endpoint/crypto.go create mode 100644 endpoint/crypto_test.go diff --git a/docs/registry.md b/docs/registry.md index 615ba305a..3ff20b93a 100644 --- a/docs/registry.md +++ b/docs/registry.md @@ -13,3 +13,68 @@ The controller will try to create the "new format" TXT records if they are not p Later on, the old format will be dropped and only the new format will be kept (-). Cleanup will be done by controller itself. + +### Encryption of TXT Records +TXT records may contain sensitive information, such as the internal ingress name or namespace, which attackers could exploit to gather information about your infrastructure. +By encrypting TXT records, you can protect this information from unauthorized access. It is strongly recommended to encrypt all TXT records to prevent potential security breaches. + +To enable encryption of TXT records, you can use the following parameters: +- `--txt-encrypt-enabled=true` +- `--txt-encrypt-aes-key=32bytesKey` (used for AES-256-GCM encryption and should be exactly 32 bytes) + +Note that the key used for encryption should be a secure key and properly managed to ensure the security of your TXT records. + +### Generating TXT encryption AES key +Python +```python +python -c 'import os,base64; print(base64.urlsafe_b64encode(os.urandom(32)).decode())' +``` + +Bash +```shell +dd if=/dev/urandom bs=32 count=1 2>/dev/null | base64 | tr -d -- '\n' | tr -- '+/' '-_'; echo +``` + +OpenSSL +```shell +openssl rand -base64 32 | tr -- '+/' '-_' +``` + +PowerShell +```powershell +# Add System.Web assembly to session, just in case +Add-Type -AssemblyName System.Web +[Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes([System.Web.Security.Membership]::GeneratePassword(32,4))).Replace("+","-").Replace("/","_") +``` + +Terraform +```hcl +resource "random_password" "txt_key" { + length = 32 + override_special = "-_" +} +``` + +### Manually Encrypt/Decrypt TXT Records + +In some cases, you may need to edit labels generated by External-DNS, and in such cases, you can use simple Golang code to do that. + +```go +package main + +import ( + "fmt" + "sigs.k8s.io/external-dns/endpoint" +) + +func main() { + key := []byte("testtesttesttesttesttesttesttest") + encrypted, _ := endpoint.EncryptText( + "heritage=external-dns,external-dns/owner=example,external-dns/resource=ingress/default/example", + key, + nil, + ) + decrypted, _, _ := endpoint.DecryptText(encrypted, key) + fmt.Println(decrypted) +} +``` diff --git a/endpoint/crypto.go b/endpoint/crypto.go new file mode 100644 index 000000000..1d6ebd1dd --- /dev/null +++ b/endpoint/crypto.go @@ -0,0 +1,134 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package endpoint + +import ( + "bytes" + "compress/gzip" + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "encoding/base64" + "fmt" + "io" + + log "github.com/sirupsen/logrus" +) + +// EncryptText gzip input data and encrypts it using the supplied AES key +func EncryptText(text string, aesKey []byte, nonceEncoded []byte) (string, error) { + block, err := aes.NewCipher(aesKey) + if err != nil { + return "", err + } + + gcm, err := cipher.NewGCM(block) + if err != nil { + return "", err + } + + nonce := make([]byte, gcm.NonceSize()) + if nonceEncoded == nil { + if _, err = io.ReadFull(rand.Reader, nonce); err != nil { + return "", err + } + } else { + if _, err = base64.StdEncoding.Decode(nonce, nonceEncoded); err != nil { + return "", err + } + } + + data, err := compressData([]byte(text)) + if err != nil { + return "", err + } + + cipherData := gcm.Seal(nonce, nonce, data, nil) + return base64.StdEncoding.EncodeToString(cipherData), nil +} + +// DecryptText decrypt gziped data using a supplied AES encryption key ang ungzip it +// in case of decryption failed, will return original input and decryption error +func DecryptText(text string, aesKey []byte) (decryptResult string, encryptNonce string, err error) { + block, err := aes.NewCipher(aesKey) + if err != nil { + return "", "", err + } + gcm, err := cipher.NewGCM(block) + if err != nil { + return "", "", err + } + nonceSize := gcm.NonceSize() + data, err := base64.StdEncoding.DecodeString(text) + if err != nil { + return "", "", err + } + if len(data) <= nonceSize { + return "", "", fmt.Errorf("the encoded data from text %#v is shorter than %#v bytes and can't be decoded", text, nonceSize) + } + nonce, ciphertext := data[:nonceSize], data[nonceSize:] + plaindata, err := gcm.Open(nil, nonce, ciphertext, nil) + if err != nil { + return "", "", err + } + plaindata, err = decompressData(plaindata) + if err != nil { + log.Debugf("Failed to decompress data based on the base64 encoded text %#v. Got error %#v.", text, err) + return "", "", err + } + + return string(plaindata), base64.StdEncoding.EncodeToString(nonce), nil +} + +// decompressData gzip compressed data +func decompressData(data []byte) (resData []byte, err error) { + gz, err := gzip.NewReader(bytes.NewBuffer(data)) + if err != nil { + return nil, err + } + defer gz.Close() + var b bytes.Buffer + if _, err = b.ReadFrom(gz); err != nil { + return nil, err + } + + return b.Bytes(), nil +} + +// compressData by gzip, for minify data stored in registry +func compressData(data []byte) (compressedData []byte, err error) { + var b bytes.Buffer + gz, err := gzip.NewWriterLevel(&b, gzip.BestCompression) + if err != nil { + return nil, err + } + + defer gz.Close() + if _, err = gz.Write(data); err != nil { + return nil, err + } + + if err = gz.Flush(); err != nil { + return nil, err + } + + if err = gz.Close(); err != nil { + return nil, err + } + + return b.Bytes(), nil +} diff --git a/endpoint/crypto_test.go b/endpoint/crypto_test.go new file mode 100644 index 000000000..880afcce3 --- /dev/null +++ b/endpoint/crypto_test.go @@ -0,0 +1,58 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package endpoint + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestEncrypt(t *testing.T) { + // Verify that text encryption and decryption works + aesKey := []byte("s%zF`.*'5`9.AhI2!B,.~hmbs^.*TL?;") + plaintext := "Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum." + encryptedtext, err := EncryptText(plaintext, aesKey, nil) + require.NoError(t, err) + decryptedtext, _, err := DecryptText(encryptedtext, aesKey) + require.NoError(t, err) + if plaintext != decryptedtext { + t.Errorf("Original plain text %#v differs from the resulting decrypted text %#v", plaintext, decryptedtext) + } + + // Verify that decrypt returns an error and empty data if wrong AES encryption key is used + decryptedtext, _, err = DecryptText(encryptedtext, []byte("s'J!jD`].LC?g&Oa11AgTub,j48ts/96")) + require.Error(t, err) + if decryptedtext != "" { + t.Error("Data decryption failed, empty string should be as result") + } + + // Verify that decrypt returns an error and empty data if unencrypted input is is supplied + decryptedtext, _, err = DecryptText(plaintext, aesKey) + require.Error(t, err) + if decryptedtext != "" { + t.Errorf("Data decryption failed, empty string should be as result") + } + + // Verify that a known encrypted text is decrypted to what is expected + encryptedtext = "0Mfzf6wsN8llrfX0ucDZ6nlc2+QiQfKKedjPPLu5atb2I35L9nUZeJcCnuLVW7CVW3K0h94vSuBLdXnMrj8Vcm0M09shxaoF48IcCpD03XtQbKXqk2hPbsW6+JybvplHIQGr16/PcjUSObGmR9yjf38+qEltApkKvrPjsyw43BX4eE10rL0Bln33UJD7/w+zazRDPFlAcbGtkt0ETKHnvyB3/aCddLipvrhjCXj2ZY/ktRF6h716kJRgXU10dCIQHFYU45MIdxI+k10HK3yZqhI2V0Gp2xjrFV/LRQ7/OS9SFee4asPWUYxbCEsnOzp8qc0dCPFSo1dtADzWnUZnsAcbnjtudT4milfLJc5CxDk1v3ykqQ/ajejwHjWQ7b8U6AsTErbezfdcqrb5IzkLgHb5TosnfrdDmNc9GcKfpsrCHbVY8KgNwMVdtwavLv7d9WM6sooUlZ3t0sABGkzagXQmPRvwLnkSOlie5XrnzWo8/8/4UByLga29CaXO" + decryptedtext, _, err = DecryptText(encryptedtext, aesKey) + require.NoError(t, err) + if decryptedtext != plaintext { + t.Error("Decryption of text didn't result in expected plaintext result.") + } +} diff --git a/endpoint/labels.go b/endpoint/labels.go index 797190db4..c65333dc6 100644 --- a/endpoint/labels.go +++ b/endpoint/labels.go @@ -17,6 +17,8 @@ limitations under the License. package endpoint import ( + log "github.com/sirupsen/logrus" + "errors" "fmt" "sort" @@ -41,6 +43,9 @@ const ( // DualstackLabelKey is the name of the label that identifies dualstack endpoints DualstackLabelKey = "dualstack" + + // txtEncryptionNonce label for keep same nonce for same txt records, for prevent different result of encryption for same txt record, it can cause issues for some providers + txtEncryptionNonce = "txt-encryption-nonce" ) // Labels store metadata related to the endpoint @@ -55,7 +60,7 @@ func NewLabels() Labels { // NewLabelsFromString constructs endpoints labels from a provided format string // if heritage set to another value is found then error is returned // no heritage automatically assumes is not owned by external-dns and returns invalidHeritage error -func NewLabelsFromString(labelText string) (Labels, error) { +func NewLabelsFromStringPlain(labelText string) (Labels, error) { endpointLabels := map[string]string{} labelText = strings.Trim(labelText, "\"") // drop quotes tokens := strings.Split(labelText, ",") @@ -85,9 +90,26 @@ func NewLabelsFromString(labelText string) (Labels, error) { return endpointLabels, nil } -// Serialize transforms endpoints labels into a external-dns recognizable format string +func NewLabelsFromString(labelText string, aesKey []byte) (Labels, error) { + if len(aesKey) != 0 { + decryptedText, encryptionNonce, err := DecryptText(strings.Trim(labelText, "\""), aesKey) + //in case if we have decryption error, just try process original text + //decryption errors should be ignored here, because we can already have plain-text labels in registry + if err == nil { + labels, err := NewLabelsFromStringPlain(decryptedText) + if err == nil { + labels[txtEncryptionNonce] = encryptionNonce + } + + return labels, err + } + } + return NewLabelsFromStringPlain(labelText) +} + +// SerializePlain transforms endpoints labels into a external-dns recognizable format string // withQuotes adds additional quotes -func (l Labels) Serialize(withQuotes bool) string { +func (l Labels) SerializePlain(withQuotes bool) string { var tokens []string tokens = append(tokens, fmt.Sprintf("heritage=%s", heritage)) var keys []string @@ -104,3 +126,31 @@ func (l Labels) Serialize(withQuotes bool) string { } return strings.Join(tokens, ",") } + +// Serialize same to SerializePlain, but encrypt data, if encryption enabled +func (l Labels) Serialize(withQuotes bool, txtEncryptEnabled bool, aesKey []byte) string { + if !txtEncryptEnabled { + return l.SerializePlain(withQuotes) + } + + var encryptionNonce []byte = nil + if extractedNonce, nonceExists := l[txtEncryptionNonce]; nonceExists { + encryptionNonce = []byte(extractedNonce) + delete(l, txtEncryptionNonce) + } + + text := l.SerializePlain(false) + log.Debugf("Encrypt the serialized text %#v before returning it.", text) + var err error + text, err = EncryptText(text, aesKey, encryptionNonce) + + if err != nil { + log.Fatalf("Failed to encrypt the text %#v using the encryption key %#v. Got error %#v.", text, aesKey, err) + } + + if withQuotes { + text = fmt.Sprintf("\"%s\"", text) + } + log.Debugf("Serialized text after encryption is %#v.", text) + return text +} diff --git a/endpoint/labels_test.go b/endpoint/labels_test.go index 9386a23e6..394635b71 100644 --- a/endpoint/labels_test.go +++ b/endpoint/labels_test.go @@ -25,14 +25,18 @@ import ( type LabelsSuite struct { suite.Suite - foo Labels - fooAsText string - fooAsTextWithQuotes string - barText string - barTextAsMap Labels - noHeritageText string - wrongHeritageText string - multipleHeritageText string // considered invalid + aesKey []byte + foo Labels + fooAsText string + fooAsTextWithQuotes string + fooAsTextEncrypted string + fooAsTextWithQuotesEncrypted string + barText string + barTextEncrypted string + barTextAsMap Labels + noHeritageText string + wrongHeritageText string + multipleHeritageText string // considered invalid } func (suite *LabelsSuite) SetupTest() { @@ -40,48 +44,79 @@ func (suite *LabelsSuite) SetupTest() { "owner": "foo-owner", "resource": "foo-resource", } + suite.aesKey = []byte(")K_Fy|?Z.64#UuHm`}[d!GC%WJM_fs{_") suite.fooAsText = "heritage=external-dns,external-dns/owner=foo-owner,external-dns/resource=foo-resource" suite.fooAsTextWithQuotes = fmt.Sprintf(`"%s"`, suite.fooAsText) - + suite.fooAsTextEncrypted = `+lvP8q9KHJ6BS6O81i2Q6DLNdf2JSKy8j/gbZKviTZlGYj7q+yDoYMgkQ1hPn6urtGllM5bfFMcaaHto52otQtiOYrX8990J3kQqg4s47m3bH3Ejl8RSxSSuWJM3HJtPghQzYg0/LSOsdQ0=` + suite.fooAsTextWithQuotesEncrypted = fmt.Sprintf(`"%s"`, suite.fooAsTextEncrypted) suite.barTextAsMap = map[string]string{ "owner": "bar-owner", "resource": "bar-resource", "new-key": "bar-new-key", } suite.barText = "heritage=external-dns,,external-dns/owner=bar-owner,external-dns/resource=bar-resource,external-dns/new-key=bar-new-key,random=stuff,no-equal-sign,," // also has some random gibberish - + suite.barTextEncrypted = "yi6vVATlgYN0enXBIupVK2atNUKtajofWMroWtvZjUanFZXlWvqjJPpjmMd91kv86bZj+syQEP0uR3TK6eFVV7oKFh/NxYyh238FjZ+25zlXW9TgbLoMalUNOkhKFdfXkLeeaqJjePB59t+kQBYX+ZEryK652asPs6M+xTIvtg07N7WWZ6SjJujm0RRISg==" suite.noHeritageText = "external-dns/owner=random-owner" suite.wrongHeritageText = "heritage=mate,external-dns/owner=random-owner" suite.multipleHeritageText = "heritage=mate,heritage=external-dns,external-dns/owner=random-owner" } func (suite *LabelsSuite) TestSerialize() { - suite.Equal(suite.fooAsText, suite.foo.Serialize(false), "should serializeLabel") - suite.Equal(suite.fooAsTextWithQuotes, suite.foo.Serialize(true), "should serializeLabel") + suite.Equal(suite.fooAsText, suite.foo.SerializePlain(false), "should serializeLabel") + suite.Equal(suite.fooAsTextWithQuotes, suite.foo.SerializePlain(true), "should serializeLabel") + suite.Equal(suite.fooAsText, suite.foo.Serialize(false, false, nil), "should serializeLabel") + suite.Equal(suite.fooAsTextWithQuotes, suite.foo.Serialize(true, false, nil), "should serializeLabel") + suite.Equal(suite.fooAsText, suite.foo.Serialize(false, false, suite.aesKey), "should serializeLabel") + suite.Equal(suite.fooAsTextWithQuotes, suite.foo.Serialize(true, false, suite.aesKey), "should serializeLabel") + suite.NotEqual(suite.fooAsText, suite.foo.Serialize(false, true, suite.aesKey), "should serializeLabel and encrypt") + suite.NotEqual(suite.fooAsTextWithQuotes, suite.foo.Serialize(true, true, suite.aesKey), "should serializeLabel and encrypt") +} + +func (suite *LabelsSuite) TestEncryptionNonceReUsage() { + foo, err := NewLabelsFromString(suite.fooAsTextEncrypted, suite.aesKey) + suite.NoError(err, "should succeed for valid label text") + serialized := foo.Serialize(false, true, suite.aesKey) + suite.Equal(serialized, suite.fooAsTextEncrypted, "serialized result should be equal") } func (suite *LabelsSuite) TestDeserialize() { - foo, err := NewLabelsFromString(suite.fooAsText) + foo, err := NewLabelsFromStringPlain(suite.fooAsText) suite.NoError(err, "should succeed for valid label text") suite.Equal(suite.foo, foo, "should reconstruct original label map") - foo, err = NewLabelsFromString(suite.fooAsTextWithQuotes) + foo, err = NewLabelsFromStringPlain(suite.fooAsTextWithQuotes) suite.NoError(err, "should succeed for valid label text") suite.Equal(suite.foo, foo, "should reconstruct original label map") - bar, err := NewLabelsFromString(suite.barText) + foo, err = NewLabelsFromString(suite.fooAsTextEncrypted, suite.aesKey) + suite.NoError(err, "should succeed for valid encrypted label text") + for key, val := range suite.foo { + suite.Equal(val, foo[key], "should contains all keys from original label map") + } + + foo, err = NewLabelsFromString(suite.fooAsTextWithQuotesEncrypted, suite.aesKey) + suite.NoError(err, "should succeed for valid encrypted label text") + for key, val := range suite.foo { + suite.Equal(val, foo[key], "should contains all keys from original label map") + } + + bar, err := NewLabelsFromStringPlain(suite.barText) suite.NoError(err, "should succeed for valid label text") suite.Equal(suite.barTextAsMap, bar, "should reconstruct original label map") - noHeritage, err := NewLabelsFromString(suite.noHeritageText) + bar, err = NewLabelsFromString(suite.barText, suite.aesKey) + suite.NoError(err, "should succeed for valid encrypted label text") + suite.Equal(suite.barTextAsMap, bar, "should reconstruct original label map") + + noHeritage, err := NewLabelsFromStringPlain(suite.noHeritageText) suite.Equal(ErrInvalidHeritage, err, "should fail if no heritage is found") suite.Nil(noHeritage, "should return nil") - wrongHeritage, err := NewLabelsFromString(suite.wrongHeritageText) + wrongHeritage, err := NewLabelsFromStringPlain(suite.wrongHeritageText) suite.Equal(ErrInvalidHeritage, err, "should fail if wrong heritage is found") suite.Nil(wrongHeritage, "if error should return nil") - multipleHeritage, err := NewLabelsFromString(suite.multipleHeritageText) + multipleHeritage, err := NewLabelsFromStringPlain(suite.multipleHeritageText) suite.Equal(ErrInvalidHeritage, err, "should fail if multiple heritage is found") suite.Nil(multipleHeritage, "if error should return nil") } diff --git a/main.go b/main.go index 960045ce9..0e6e37e28 100644 --- a/main.go +++ b/main.go @@ -382,7 +382,7 @@ func main() { case "noop": r, err = registry.NewNoopRegistry(p) case "txt": - r, err = registry.NewTXTRegistry(p, cfg.TXTPrefix, cfg.TXTSuffix, cfg.TXTOwnerID, cfg.TXTCacheInterval, cfg.TXTWildcardReplacement, cfg.ManagedDNSRecordTypes) + r, err = registry.NewTXTRegistry(p, cfg.TXTPrefix, cfg.TXTSuffix, cfg.TXTOwnerID, cfg.TXTCacheInterval, cfg.TXTWildcardReplacement, cfg.ManagedDNSRecordTypes, cfg.TXTEncryptEnabled, []byte(cfg.TXTEncryptAESKey)) case "aws-sd": r, err = registry.NewAWSSDRegistry(p.(*awssd.AWSSDProvider), cfg.TXTOwnerID) default: diff --git a/pkg/apis/externaldns/types.go b/pkg/apis/externaldns/types.go index eefd2b2bf..a4ca2ab66 100644 --- a/pkg/apis/externaldns/types.go +++ b/pkg/apis/externaldns/types.go @@ -147,6 +147,8 @@ type Config struct { TXTOwnerID string TXTPrefix string TXTSuffix string + TXTEncryptEnabled bool + TXTEncryptAESKey string Interval time.Duration MinEventSyncInterval time.Duration Once bool @@ -295,6 +297,8 @@ var defaultConfig = &Config{ TXTCacheInterval: 0, TXTWildcardReplacement: "", MinEventSyncInterval: 5 * time.Second, + TXTEncryptEnabled: false, + TXTEncryptAESKey: "", Interval: time.Minute, Once: false, DryRun: false, @@ -570,6 +574,8 @@ func (cfg *Config) ParseFlags(args []string) error { app.Flag("txt-prefix", "When using the TXT registry, a custom string that's prefixed to each ownership DNS record (optional). Could contain record type template like '%{record_type}-prefix-'. Mutual exclusive with txt-suffix!").Default(defaultConfig.TXTPrefix).StringVar(&cfg.TXTPrefix) app.Flag("txt-suffix", "When using the TXT registry, a custom string that's suffixed to the host portion of each ownership DNS record (optional). Could contain record type template like '-%{record_type}-suffix'. Mutual exclusive with txt-prefix!").Default(defaultConfig.TXTSuffix).StringVar(&cfg.TXTSuffix) app.Flag("txt-wildcard-replacement", "When using the TXT registry, a custom string that's used instead of an asterisk for TXT records corresponding to wildcard DNS records (optional)").Default(defaultConfig.TXTWildcardReplacement).StringVar(&cfg.TXTWildcardReplacement) + app.Flag("txt-encrypt-enabled", "When using the TXT registry, set if TXT records should be encrypted before stored (default: disabled)").BoolVar(&cfg.TXTEncryptEnabled) + app.Flag("txt-encrypt-aes-key", "When using the TXT registry, set TXT record decryption and encryption 32 byte aes key (required when --txt-encrypt=true)").Default(defaultConfig.TXTEncryptAESKey).StringVar(&cfg.TXTEncryptAESKey) // Flags related to the main control loop app.Flag("txt-cache-interval", "The interval between cache synchronizations in duration format (default: disabled)").Default(defaultConfig.TXTCacheInterval.String()).DurationVar(&cfg.TXTCacheInterval) diff --git a/provider/awssd/aws_sd.go b/provider/awssd/aws_sd.go index ecdb7efc5..97cca81cc 100644 --- a/provider/awssd/aws_sd.go +++ b/provider/awssd/aws_sd.go @@ -509,7 +509,7 @@ func (p *AWSSDProvider) DeleteService(service *sd.Service) error { // convert ownerID string to service description format label := endpoint.NewLabels() label[endpoint.OwnerLabelKey] = p.ownerID - label[endpoint.AWSSDDescriptionLabel] = label.Serialize(false) + label[endpoint.AWSSDDescriptionLabel] = label.SerializePlain(false) if strings.HasPrefix(aws.StringValue(service.Description), label[endpoint.AWSSDDescriptionLabel]) { log.Infof("Deleting service \"%s\"", *service.Name) diff --git a/registry/aws_sd_registry.go b/registry/aws_sd_registry.go index a2ff3350d..f0fef584d 100644 --- a/registry/aws_sd_registry.go +++ b/registry/aws_sd_registry.go @@ -55,7 +55,7 @@ func (sdr *AWSSDRegistry) Records(ctx context.Context) ([]*endpoint.Endpoint, er } for _, record := range records { - labels, err := endpoint.NewLabelsFromString(record.Labels[endpoint.AWSSDDescriptionLabel]) + labels, err := endpoint.NewLabelsFromStringPlain(record.Labels[endpoint.AWSSDDescriptionLabel]) if err != nil { // if we fail to parse the output then simply assume the endpoint is not managed by any instance of External DNS record.Labels = endpoint.NewLabels() @@ -96,7 +96,7 @@ func (sdr *AWSSDRegistry) updateLabels(endpoints []*endpoint.Endpoint) { ep.Labels = make(map[string]string) } ep.Labels[endpoint.OwnerLabelKey] = sdr.ownerID - ep.Labels[endpoint.AWSSDDescriptionLabel] = ep.Labels.Serialize(false) + ep.Labels[endpoint.AWSSDDescriptionLabel] = ep.Labels.SerializePlain(false) } } diff --git a/registry/txt.go b/registry/txt.go index 48dd028d5..2c7cccbdf 100644 --- a/registry/txt.go +++ b/registry/txt.go @@ -52,15 +52,27 @@ type TXTRegistry struct { // missingTXTRecords stores TXT records which are missing after the migration to the new format missingTXTRecords []*endpoint.Endpoint + + // encrypt text records + txtEncryptEnabled bool + txtEncryptAESKey []byte } const keySuffixAAAA = ":AAAA" // NewTXTRegistry returns new TXTRegistry object -func NewTXTRegistry(provider provider.Provider, txtPrefix, txtSuffix, ownerID string, cacheInterval time.Duration, txtWildcardReplacement string, managedRecordTypes []string) (*TXTRegistry, error) { +func NewTXTRegistry(provider provider.Provider, txtPrefix, txtSuffix, ownerID string, cacheInterval time.Duration, txtWildcardReplacement string, managedRecordTypes []string, txtEncryptEnabled bool, txtEncryptAESKey []byte) (*TXTRegistry, error) { if ownerID == "" { return nil, errors.New("owner id cannot be empty") } + if len(txtEncryptAESKey) == 0 { + txtEncryptAESKey = nil + } else if len(txtEncryptAESKey) != 32 { + return nil, errors.New("the AES Encryption key must have a length of 32 bytes") + } + if txtEncryptEnabled && txtEncryptAESKey == nil { + return nil, errors.New("the AES Encryption key must be set when TXT record encryption is enabled") + } if len(txtPrefix) > 0 && len(txtSuffix) > 0 { return nil, errors.New("txt-prefix and txt-suffix are mutual exclusive") @@ -75,6 +87,8 @@ func NewTXTRegistry(provider provider.Provider, txtPrefix, txtSuffix, ownerID st cacheInterval: cacheInterval, wildcardReplacement: txtWildcardReplacement, managedRecordTypes: managedRecordTypes, + txtEncryptEnabled: txtEncryptEnabled, + txtEncryptAESKey: txtEncryptAESKey, }, nil } @@ -114,7 +128,7 @@ func (im *TXTRegistry) Records(ctx context.Context) ([]*endpoint.Endpoint, error continue } // We simply assume that TXT records for the registry will always have only one target. - labels, err := endpoint.NewLabelsFromString(record.Targets[0]) + labels, err := endpoint.NewLabelsFromString(record.Targets[0], im.txtEncryptAESKey) if err == endpoint.ErrInvalidHeritage { // if no heritage is found or it is invalid // case when value of txt record cannot be identified @@ -205,7 +219,7 @@ func (im *TXTRegistry) generateTXTRecord(r *endpoint.Endpoint) []*endpoint.Endpo if r.RecordType != endpoint.RecordTypeAAAA { // old TXT record format - txt := endpoint.NewEndpoint(im.mapper.toTXTName(r.DNSName), endpoint.RecordTypeTXT, r.Labels.Serialize(true)) + txt := endpoint.NewEndpoint(im.mapper.toTXTName(r.DNSName), endpoint.RecordTypeTXT, r.Labels.Serialize(true, im.txtEncryptEnabled, im.txtEncryptAESKey)) if txt != nil { txt.WithSetIdentifier(r.SetIdentifier) txt.Labels[endpoint.OwnedRecordLabelKey] = r.DNSName @@ -213,9 +227,8 @@ func (im *TXTRegistry) generateTXTRecord(r *endpoint.Endpoint) []*endpoint.Endpo endpoints = append(endpoints, txt) } } - // new TXT record format (containing record type) - txtNew := endpoint.NewEndpoint(im.mapper.toNewTXTName(r.DNSName, r.RecordType), endpoint.RecordTypeTXT, r.Labels.Serialize(true)) + txtNew := endpoint.NewEndpoint(im.mapper.toNewTXTName(r.DNSName, r.RecordType), endpoint.RecordTypeTXT, r.Labels.Serialize(true, im.txtEncryptEnabled, im.txtEncryptAESKey)) if txtNew != nil { txtNew.WithSetIdentifier(r.SetIdentifier) txtNew.Labels[endpoint.OwnedRecordLabelKey] = r.DNSName diff --git a/registry/txt_test.go b/registry/txt_test.go index 2f7297fb1..80075c81d 100644 --- a/registry/txt_test.go +++ b/registry/txt_test.go @@ -46,20 +46,20 @@ func TestTXTRegistry(t *testing.T) { func testTXTRegistryNew(t *testing.T) { p := inmemory.NewInMemoryProvider() - _, err := NewTXTRegistry(p, "txt", "", "", time.Hour, "", []string{}) + _, err := NewTXTRegistry(p, "txt", "", "", time.Hour, "", []string{}, false, nil) require.Error(t, err) - _, err = NewTXTRegistry(p, "", "txt", "", time.Hour, "", []string{}) + _, err = NewTXTRegistry(p, "", "txt", "", time.Hour, "", []string{}, false, nil) require.Error(t, err) - r, err := NewTXTRegistry(p, "txt", "", "owner", time.Hour, "", []string{}) + r, err := NewTXTRegistry(p, "txt", "", "owner", time.Hour, "", []string{}, false, nil) require.NoError(t, err) assert.Equal(t, p, r.provider) - r, err = NewTXTRegistry(p, "", "txt", "owner", time.Hour, "", []string{}) + r, err = NewTXTRegistry(p, "", "txt", "owner", time.Hour, "", []string{}, false, nil) require.NoError(t, err) - _, err = NewTXTRegistry(p, "txt", "txt", "owner", time.Hour, "", []string{}) + _, err = NewTXTRegistry(p, "txt", "txt", "owner", time.Hour, "", []string{}, false, nil) require.Error(t, err) _, ok := r.mapper.(affixNameMapper) @@ -67,7 +67,17 @@ func testTXTRegistryNew(t *testing.T) { assert.Equal(t, "owner", r.ownerID) assert.Equal(t, p, r.provider) - r, err = NewTXTRegistry(p, "", "", "owner", time.Hour, "", []string{}) + aesKey := []byte(";k&l)nUC/33:{?d{3)54+,AD?]SX%yh^") + _, err = NewTXTRegistry(p, "", "", "owner", time.Hour, "", []string{}, false, nil) + require.NoError(t, err) + + _, err = NewTXTRegistry(p, "", "", "owner", time.Hour, "", []string{}, false, aesKey) + require.NoError(t, err) + + _, err = NewTXTRegistry(p, "", "", "owner", time.Hour, "", []string{}, true, nil) + require.Error(t, err) + + r, err = NewTXTRegistry(p, "", "", "owner", time.Hour, "", []string{}, true, aesKey) require.NoError(t, err) _, ok = r.mapper.(affixNameMapper) @@ -203,13 +213,13 @@ func testTXTRegistryRecordsPrefixed(t *testing.T) { }, } - r, _ := NewTXTRegistry(p, "txt.", "", "owner", time.Hour, "wc", []string{}) + r, _ := NewTXTRegistry(p, "txt.", "", "owner", time.Hour, "wc", []string{}, false, nil) records, _ := r.Records(ctx) assert.True(t, testutils.SameEndpoints(records, expectedRecords)) // Ensure prefix is case-insensitive - r, _ = NewTXTRegistry(p, "TxT.", "", "owner", time.Hour, "", []string{}) + r, _ = NewTXTRegistry(p, "TxT.", "", "owner", time.Hour, "", []string{}, false, nil) records, _ = r.Records(ctx) assert.True(t, testutils.SameEndpointLabels(records, expectedRecords)) @@ -328,13 +338,13 @@ func testTXTRegistryRecordsSuffixed(t *testing.T) { }, } - r, _ := NewTXTRegistry(p, "", "-txt", "owner", time.Hour, "", []string{}) + r, _ := NewTXTRegistry(p, "", "-txt", "owner", time.Hour, "", []string{}, false, nil) records, _ := r.Records(ctx) assert.True(t, testutils.SameEndpoints(records, expectedRecords)) // Ensure prefix is case-insensitive - r, _ = NewTXTRegistry(p, "", "-TxT", "owner", time.Hour, "", []string{}) + r, _ = NewTXTRegistry(p, "", "-TxT", "owner", time.Hour, "", []string{}, false, nil) records, _ = r.Records(ctx) assert.True(t, testutils.SameEndpointLabels(records, expectedRecords)) @@ -429,7 +439,7 @@ func testTXTRegistryRecordsNoPrefix(t *testing.T) { }, } - r, _ := NewTXTRegistry(p, "", "", "owner", time.Hour, "", []string{}) + r, _ := NewTXTRegistry(p, "", "", "owner", time.Hour, "", []string{}, false, nil) records, _ := r.Records(ctx) assert.True(t, testutils.SameEndpoints(records, expectedRecords)) @@ -472,7 +482,7 @@ func testTXTRegistryApplyChangesWithPrefix(t *testing.T) { newEndpointWithOwner("txt.cname-multiple.test-zone.example.org", "\"heritage=external-dns,external-dns/owner=owner\"", endpoint.RecordTypeTXT, "").WithSetIdentifier("test-set-2"), }, }) - r, _ := NewTXTRegistry(p, "txt.", "", "owner", time.Hour, "", []string{}) + r, _ := NewTXTRegistry(p, "txt.", "", "owner", time.Hour, "", []string{}, false, nil) changes := &plan.Changes{ Create: []*endpoint.Endpoint{ @@ -561,7 +571,7 @@ func testTXTRegistryApplyChangesWithTemplatedPrefix(t *testing.T) { p.ApplyChanges(ctx, &plan.Changes{ Create: []*endpoint.Endpoint{}, }) - r, _ := NewTXTRegistry(p, "prefix%{record_type}.", "", "owner", time.Hour, "", []string{}) + r, _ := NewTXTRegistry(p, "prefix%{record_type}.", "", "owner", time.Hour, "", []string{}, false, nil) changes := &plan.Changes{ Create: []*endpoint.Endpoint{ newEndpointWithOwnerResource("new-record-1.test-zone.example.org", "new-loadbalancer-1.lb.com", endpoint.RecordTypeCNAME, "", "ingress/default/my-ingress"), @@ -605,7 +615,7 @@ func testTXTRegistryApplyChangesWithTemplatedSuffix(t *testing.T) { p.OnApplyChanges = func(ctx context.Context, got *plan.Changes) { assert.Equal(t, ctxEndpoints, ctx.Value(provider.RecordsContextKey)) } - r, _ := NewTXTRegistry(p, "", "-%{record_type}suffix", "owner", time.Hour, "", []string{}) + r, _ := NewTXTRegistry(p, "", "-%{record_type}suffix", "owner", time.Hour, "", []string{}, false, nil) changes := &plan.Changes{ Create: []*endpoint.Endpoint{ newEndpointWithOwnerResource("new-record-1.test-zone.example.org", "new-loadbalancer-1.lb.com", endpoint.RecordTypeCNAME, "", "ingress/default/my-ingress"), @@ -671,7 +681,7 @@ func testTXTRegistryApplyChangesWithSuffix(t *testing.T) { newEndpointWithOwner("cname-multiple-txt.test-zone.example.org", "\"heritage=external-dns,external-dns/owner=owner\"", endpoint.RecordTypeTXT, "").WithSetIdentifier("test-set-2"), }, }) - r, _ := NewTXTRegistry(p, "", "-txt", "owner", time.Hour, "wildcard", []string{}) + r, _ := NewTXTRegistry(p, "", "-txt", "owner", time.Hour, "wildcard", []string{}, false, nil) changes := &plan.Changes{ Create: []*endpoint.Endpoint{ @@ -775,7 +785,7 @@ func testTXTRegistryApplyChangesNoPrefix(t *testing.T) { newEndpointWithOwner("cname-foobar.test-zone.example.org", "\"heritage=external-dns,external-dns/owner=owner\"", endpoint.RecordTypeTXT, ""), }, }) - r, _ := NewTXTRegistry(p, "", "", "owner", time.Hour, "", []string{}) + r, _ := NewTXTRegistry(p, "", "", "owner", time.Hour, "", []string{}, false, nil) changes := &plan.Changes{ Create: []*endpoint.Endpoint{ @@ -941,7 +951,7 @@ func testTXTRegistryMissingRecordsNoPrefix(t *testing.T) { }, } - r, _ := NewTXTRegistry(p, "", "", "owner", time.Hour, "wc", []string{endpoint.RecordTypeCNAME, endpoint.RecordTypeA, endpoint.RecordTypeNS}) + r, _ := NewTXTRegistry(p, "", "", "owner", time.Hour, "wc", []string{endpoint.RecordTypeCNAME, endpoint.RecordTypeA, endpoint.RecordTypeNS}, false, nil) records, _ := r.Records(ctx) missingRecords := r.MissingRecords() @@ -1045,7 +1055,7 @@ func testTXTRegistryMissingRecordsWithPrefix(t *testing.T) { }, } - r, _ := NewTXTRegistry(p, "txt.", "", "owner", time.Hour, "wc", []string{endpoint.RecordTypeCNAME, endpoint.RecordTypeA, endpoint.RecordTypeNS}) + r, _ := NewTXTRegistry(p, "txt.", "", "owner", time.Hour, "wc", []string{endpoint.RecordTypeCNAME, endpoint.RecordTypeA, endpoint.RecordTypeNS}, false, nil) records, _ := r.Records(ctx) missingRecords := r.MissingRecords() @@ -1199,7 +1209,7 @@ func TestNewTXTScheme(t *testing.T) { newEndpointWithOwner("cname-foobar.test-zone.example.org", "\"heritage=external-dns,external-dns/owner=owner\"", endpoint.RecordTypeTXT, ""), }, }) - r, _ := NewTXTRegistry(p, "", "", "owner", time.Hour, "", []string{}) + r, _ := NewTXTRegistry(p, "", "", "owner", time.Hour, "", []string{}, false, nil) changes := &plan.Changes{ Create: []*endpoint.Endpoint{ @@ -1275,7 +1285,7 @@ func TestGenerateTXT(t *testing.T) { } p := inmemory.NewInMemoryProvider() p.CreateZone(testZone) - r, _ := NewTXTRegistry(p, "", "", "owner", time.Hour, "", []string{}) + r, _ := NewTXTRegistry(p, "", "", "owner", time.Hour, "", []string{}, false, nil) gotTXT := r.generateTXTRecord(record) assert.Equal(t, expectedTXT, gotTXT) } @@ -1294,7 +1304,7 @@ func TestGenerateTXTForAAAA(t *testing.T) { } p := inmemory.NewInMemoryProvider() p.CreateZone(testZone) - r, _ := NewTXTRegistry(p, "", "", "owner", time.Hour, "", []string{}) + r, _ := NewTXTRegistry(p, "", "", "owner", time.Hour, "", []string{}, false, nil) gotTXT := r.generateTXTRecord(record) assert.Equal(t, expectedTXT, gotTXT) } @@ -1311,7 +1321,7 @@ func TestFailGenerateTXT(t *testing.T) { expectedTXT := []*endpoint.Endpoint{} p := inmemory.NewInMemoryProvider() p.CreateZone(testZone) - r, _ := NewTXTRegistry(p, "", "", "owner", time.Hour, "", []string{}) + r, _ := NewTXTRegistry(p, "", "", "owner", time.Hour, "", []string{}, false, nil) gotTXT := r.generateTXTRecord(cnameRecord) assert.Equal(t, expectedTXT, gotTXT) } From 4a2b9c60aba171f2c409150308ed134a80652473 Mon Sep 17 00:00:00 2001 From: Arnaud Lefray Date: Thu, 4 May 2023 15:10:27 +0200 Subject: [PATCH 22/32] ingress: improve ingress class name filter testing Signed-off-by: Arnaud Lefray --- source/ingress.go | 7 ++- source/ingress_test.go | 96 +++++++++++++++++++++++++++++++++--------- 2 files changed, 82 insertions(+), 21 deletions(-) diff --git a/source/ingress.go b/source/ingress.go index 56833e15c..bf204a266 100644 --- a/source/ingress.go +++ b/source/ingress.go @@ -45,6 +45,8 @@ const ( // Possible values for the ingress-hostname-source annotation IngressHostnameSourceAnnotationOnlyValue = "annotation-only" IngressHostnameSourceDefinedHostsOnlyValue = "defined-hosts-only" + + IngressClassAnnotationKey = "kubernetes.io/ingress.class" ) // ingressSource is an implementation of Source for Kubernetes ingress objects. @@ -238,11 +240,11 @@ func (sc *ingressSource) filterByAnnotations(ingresses []*networkv1.Ingress) ([] // class func (sc *ingressSource) filterByIngressClass(ingresses []*networkv1.Ingress) ([]*networkv1.Ingress, error) { // if no class filter is specified then there's nothing to do - if sc.ingressClassNames == nil { + if len(sc.ingressClassNames) == 0 { return ingresses, nil } - classNameReq, err := labels.NewRequirement("kubernetes.io/ingress.class", selection.In, sc.ingressClassNames) + classNameReq, err := labels.NewRequirement(IngressClassAnnotationKey, selection.In, sc.ingressClassNames) if err != nil { return nil, err } @@ -258,6 +260,7 @@ func (sc *ingressSource) filterByIngressClass(ingresses []*networkv1.Ingress) ([ for _, nameFilter := range sc.ingressClassNames { if ingress.Spec.IngressClassName != nil && nameFilter == *ingress.Spec.IngressClassName { matched = true + } else if matchLabelSelector(selector, ingress.Annotations) { matched = true } diff --git a/source/ingress_test.go b/source/ingress_test.go index a3bfc457a..8d3ae4175 100644 --- a/source/ingress_test.go +++ b/source/ingress_test.go @@ -136,15 +136,15 @@ func TestNewIngressSource(t *testing.T) { annotationFilter: "kubernetes.io/ingress.class=nginx", }, { - title: "non-empty ingress class name list", - expectError: false, + title: "non-empty ingress class name list", + expectError: false, ingressClassNames: []string{"internal", "external"}, }, { - title: "ingress class name and annotation filter jointly specified", - expectError: true, + title: "ingress class name and annotation filter jointly specified", + expectError: true, ingressClassNames: []string{"internal", "external"}, - annotationFilter: "kubernetes.io/ingress.class=nginx", + annotationFilter: "kubernetes.io/ingress.class=nginx", }, } { ti := ti @@ -378,7 +378,7 @@ func testIngressEndpoints(t *testing.T) { ignoreIngressTLSSpec bool ignoreIngressRulesSpec bool ingressLabelSelector labels.Selector - ingressClassNames []string + ingressClassNames []string }{ { title: "no ingress", @@ -1191,16 +1191,22 @@ func testIngressEndpoints(t *testing.T) { }, }, { - title: "ingressClassName filtering", - targetNamespace: "", - ingressClassNames: []string{"public", "dmz"}, + title: "ingressClassName filtering", + targetNamespace: "", + ingressClassNames: []string{"public", "dmz"}, ingressItems: []fakeIngress{ + { + name: "none", + namespace: namespace, + tlsdnsnames: [][]string{{"none.example.org"}}, + ips: []string{"1.0.0.0"}, + }, { name: "fake-public", namespace: namespace, tlsdnsnames: [][]string{{"example.org"}}, ips: []string{"1.2.3.4"}, - ingressClassName: "public", + ingressClassName: "public", // match }, { name: "fake-internal", @@ -1214,17 +1220,57 @@ func testIngressEndpoints(t *testing.T) { namespace: namespace, tlsdnsnames: [][]string{{"dmz.example.org"}}, ips: []string{"3.4.5.6"}, - ingressClassName: "dmz", + ingressClassName: "dmz", // match }, { - name: "annotated-dmz", - namespace: namespace, - tlsdnsnames: [][]string{{"annodmz.example.org"}}, - ips: []string{"4.5.6.7"}, + name: "annotated-dmz", + namespace: namespace, + tlsdnsnames: [][]string{{"annodmz.example.org"}}, + ips: []string{"4.5.6.7"}, annotations: map[string]string{ - "kubernetes.io/ingress.class": "dmz", + "kubernetes.io/ingress.class": "dmz", // match }, }, + { + name: "fake-internal-annotated-dmz", + namespace: namespace, + tlsdnsnames: [][]string{{"int-annodmz.example.org"}}, + ips: []string{"5.6.7.8"}, + annotations: map[string]string{ + "kubernetes.io/ingress.class": "dmz", // match + }, + ingressClassName: "internal", + }, + { + name: "fake-dmz-annotated-internal", + namespace: namespace, + tlsdnsnames: [][]string{{"dmz-annoint.example.org"}}, + ips: []string{"6.7.8.9"}, + annotations: map[string]string{ + "kubernetes.io/ingress.class": "internal", + }, + ingressClassName: "dmz", // match + }, + { + name: "empty-annotated-dmz", + namespace: namespace, + tlsdnsnames: [][]string{{"empty-annotdmz.example.org"}}, + ips: []string{"7.8.9.0"}, + annotations: map[string]string{ + "kubernetes.io/ingress.class": "dmz", // match + }, + ingressClassName: "", + }, + { + name: "empty-annotated-internal", + namespace: namespace, + tlsdnsnames: [][]string{{"empty-annotint.example.org"}}, + ips: []string{"8.9.0.1"}, + annotations: map[string]string{ + "kubernetes.io/ingress.class": "internal", + }, + ingressClassName: "", + }, }, expected: []*endpoint.Endpoint{ { @@ -1239,9 +1285,21 @@ func testIngressEndpoints(t *testing.T) { DNSName: "annodmz.example.org", Targets: endpoint.Targets{"4.5.6.7"}, }, + { + DNSName: "int-annodmz.example.org", + Targets: endpoint.Targets{"5.6.7.8"}, + }, + { + DNSName: "dmz-annoint.example.org", + Targets: endpoint.Targets{"6.7.8.9"}, + }, + { + DNSName: "empty-annotdmz.example.org", + Targets: endpoint.Targets{"7.8.9.0"}, + }, }, - }, - { + }, + { ingressLabelSelector: labels.SelectorFromSet(labels.Set{"app": "web-external"}), title: "ingress with matching labels", targetNamespace: "", @@ -1339,7 +1397,7 @@ func (ing fakeIngress) Ingress() *networkv1.Ingress { Labels: ing.labels, }, Spec: networkv1.IngressSpec{ - Rules: []networkv1.IngressRule{}, + Rules: []networkv1.IngressRule{}, IngressClassName: &ing.ingressClassName, }, Status: networkv1.IngressStatus{ From 04d7ed8ad71aa6ef4aab380836b593996e159e04 Mon Sep 17 00:00:00 2001 From: Arnaud Lefray Date: Thu, 4 May 2023 15:11:50 +0200 Subject: [PATCH 23/32] docs: update class name filtering documentation --- docs/faq.md | 20 ++++++++++---------- docs/tutorials/public-private-route53.md | 5 ++--- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/docs/faq.md b/docs/faq.md index 10a36422f..dda8ed9db 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -258,26 +258,26 @@ Sometimes you need to run an internal and an external dns service. The internal one should provision hostnames used on the internal network (perhaps inside a VPC), and the external one to expose DNS to the internet. -To do this with ExternalDNS you can use the `--ingress-class` to specifically tie an instance of ExternalDNS to -an instance of a ingress controller. Let's assume you have two ingress controllers `nginx-internal` and `nginx-external` -then you can start two ExternalDNS providers one with `--ingress-class=nginx-internal` and one with `--ingress-class=nginx-external`. +To do this with ExternalDNS you can use the `--ingress-class` flag to specifically tie an instance of ExternalDNS to an instance of a ingress controller. +Let's assume you have two ingress controllers, `nginx-internal` and `nginx-external`. +You can then start two ExternalDNS providers, one with `--ingress-class=nginx-internal` and one with `--ingress-class=nginx-external`. -If you need to search for multiple ingress classes, you can specify the argument multiple times, like so: +If you need to search for multiple ingress classes, you can specify the flag multiple times, like so: `--ingress-class=nginx-internal --ingress-class=alb-ingress-internal`. -The `--ingress-class` argument will check both the `ingressClassName` field as well as the deprecated `kubernetes.io/ingress.class` annotation. +The `--ingress-class` flag will check both the `ingressClassName` field and the deprecated `kubernetes.io/ingress.class` annotation. -Note: the `--ingress-class` argument cannot be used at the same time as a `kubernetes.io/ingress.class` annotation filter; if you do this an error will be raised. +Note: the `--ingress-class` flag cannot be used at the same time as a `kubernetes.io/ingress.class` annotation filter; if you do this an error will be raised. -If you use annotations to indicate different ingress classes in your cluster, you can instead use an `--annotation-filter` argument to restrict which objects ExternalDNS considers; for example, `--annotation-filter=kubernetes.io/ingress.class in (public,dmz)`. +If you use annotations to indicate different ingress classes in your cluster, you can instead use an `--annotation-filter` flag to restrict which objects ExternalDNS considers; for example, `--annotation-filter=kubernetes.io/ingress.class in (public,dmz)`. However, beware when using annotation filters with multiple sources, e.g. `--source=service --source=ingress`, since `--annotation-filter` will filter every given source objects. If you need to use annotation filters against a specific source you have to run a separated external dns service containing only the wanted `--source` and `--annotation-filter`. **Note:** Filtering based on annotation or ingress class name means that the external-dns controller will receive all resources of that kind and then filter on the client-side. -In larger clusters with many resources which change frequently this can cause performance issues. If only some resources need to be managed by an instance -of external-dns then label filtering can be used instead of annotation filtering. This means that only those resources which match the selector specified -in `--label-filter` will be passed to the controller. +In larger clusters with many resources which change frequently this can cause performance issues. +If only some resources need to be managed by an instance of external-dns then label filtering can be used instead of ingress class filtering (or legacy annotation filtering). +This means that only those resources which match the selector specified in `--label-filter` will be passed to the controller. ### How do I specify that I want the DNS record to point to either the Node's public or private IP when it has both? diff --git a/docs/tutorials/public-private-route53.md b/docs/tutorials/public-private-route53.md index b8110ecbd..02ec1a54d 100644 --- a/docs/tutorials/public-private-route53.md +++ b/docs/tutorials/public-private-route53.md @@ -351,12 +351,12 @@ metadata: certmanager.k8s.io/acme-challenge-type: "dns01" certmanager.k8s.io/acme-dns01-provider: "route53" certmanager.k8s.io/cluster-issuer: "letsencrypt-production" - kubernetes.io/ingress.class: "external-ingress" kubernetes.io/tls-acme: "true" labels: app: app name: app-public spec: + ingressClassName: "external-ingress" rules: - host: app.domain.com http: @@ -376,12 +376,11 @@ And reuse the requested certificate in private Service definition: apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - annotations: - kubernetes.io/ingress.class: "internal-ingress" labels: app: app name: app-private spec: + ingressClassName: "internal-ingress" rules: - host: app.domain.com http: From e9fd86035d1c17a8c8c843a712a7502c6d58a298 Mon Sep 17 00:00:00 2001 From: Arnaud Lefray Date: Thu, 4 May 2023 15:44:14 +0200 Subject: [PATCH 24/32] fix: add missing record check to ingress tests --- source/ingress_test.go | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/source/ingress_test.go b/source/ingress_test.go index dd3a6debc..b76dd439a 100644 --- a/source/ingress_test.go +++ b/source/ingress_test.go @@ -1319,28 +1319,34 @@ func testIngressEndpoints(t *testing.T) { }, expected: []*endpoint.Endpoint{ { - DNSName: "example.org", - Targets: endpoint.Targets{"1.2.3.4"}, + DNSName: "example.org", + RecordType: endpoint.RecordTypeA, + Targets: endpoint.Targets{"1.2.3.4"}, }, { - DNSName: "dmz.example.org", - Targets: endpoint.Targets{"3.4.5.6"}, + DNSName: "dmz.example.org", + RecordType: endpoint.RecordTypeA, + Targets: endpoint.Targets{"3.4.5.6"}, }, { - DNSName: "annodmz.example.org", - Targets: endpoint.Targets{"4.5.6.7"}, + DNSName: "annodmz.example.org", + RecordType: endpoint.RecordTypeA, + Targets: endpoint.Targets{"4.5.6.7"}, }, { - DNSName: "int-annodmz.example.org", - Targets: endpoint.Targets{"5.6.7.8"}, + DNSName: "int-annodmz.example.org", + RecordType: endpoint.RecordTypeA, + Targets: endpoint.Targets{"5.6.7.8"}, }, { - DNSName: "dmz-annoint.example.org", - Targets: endpoint.Targets{"6.7.8.9"}, + DNSName: "dmz-annoint.example.org", + RecordType: endpoint.RecordTypeA, + Targets: endpoint.Targets{"6.7.8.9"}, }, { - DNSName: "empty-annotdmz.example.org", - Targets: endpoint.Targets{"7.8.9.0"}, + DNSName: "empty-annotdmz.example.org", + RecordType: endpoint.RecordTypeA, + Targets: endpoint.Targets{"7.8.9.0"}, }, }, }, From e3842a4be60b713d9454ada503c2af45f77d599f Mon Sep 17 00:00:00 2001 From: Arnaud Lefray Date: Fri, 5 May 2023 08:37:12 +0200 Subject: [PATCH 25/32] Apply suggestions from code review Co-authored-by: John Gardiner Myers --- docs/faq.md | 2 +- docs/tutorials/public-private-route53.md | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/docs/faq.md b/docs/faq.md index cfcc13ba9..527db7ccd 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -273,7 +273,7 @@ Note: the `--ingress-class` flag cannot be used at the same time as a `kubernete If you use annotations to indicate different ingress classes in your cluster, you can instead use an `--annotation-filter` flag to restrict which objects ExternalDNS considers; for example, `--annotation-filter=kubernetes.io/ingress.class in (public,dmz)`. -However, beware when using annotation filters with multiple sources, e.g. `--source=service --source=ingress`, since `--annotation-filter` will filter every given source objects. +However, beware when using annotation filters with multiple sources, e.g. `--source=service --source=ingress`, since `--annotation-filter` will filter every given source object. If you need to use annotation filters against a specific source you have to run a separated external dns service containing only the wanted `--source` and `--annotation-filter`. **Note:** Filtering based on annotation or ingress class name means that the external-dns controller will receive all resources of that kind and then filter on the client-side. diff --git a/docs/tutorials/public-private-route53.md b/docs/tutorials/public-private-route53.md index 3b950b109..9ceb43195 100644 --- a/docs/tutorials/public-private-route53.md +++ b/docs/tutorials/public-private-route53.md @@ -242,8 +242,6 @@ spec: - --registry=txt - --txt-owner-id=external-dns - --ingress-class=external-ingress - # ... or, if you use annotations for ingress classes - # - --annotation-filter=kubernetes.io/ingress.class in (external-ingress) - --aws-zone-type=public image: registry.k8s.io/external-dns/external-dns:v0.13.4 name: external-dns-public @@ -293,7 +291,7 @@ spec: For this setup to work, you need to create two Ingress definitions for your application. -At first, create public Ingress definition (make sure to un-comment either the `annotations` or `ingressClassName` lines): +At first, create a public Ingress definition (make sure to un-comment either the `annotations` or `ingressClassName` lines): ```yaml apiVersion: networking.k8s.io/v1 @@ -320,7 +318,7 @@ spec: pathType: Prefix ``` -Then create private Ingress definition (again, make sure to un-comment either the `annotations` or `ingressClassName` lines): +Then create a private Ingress definition (again, make sure to un-comment either the `annotations` or `ingressClassName` lines): ```yaml apiVersion: networking.k8s.io/v1 From 42077cd35a2b6ebefaaee60003523291da3639de Mon Sep 17 00:00:00 2001 From: Arnaud Lefray Date: Fri, 5 May 2023 16:08:04 +0200 Subject: [PATCH 26/32] ingress: ignore annotation when ingressclassname is non empty --- source/ingress.go | 7 ++++--- source/ingress_test.go | 9 ++------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/source/ingress.go b/source/ingress.go index bf204a266..af7a9dc99 100644 --- a/source/ingress.go +++ b/source/ingress.go @@ -258,9 +258,10 @@ func (sc *ingressSource) filterByIngressClass(ingresses []*networkv1.Ingress) ([ var matched = false for _, nameFilter := range sc.ingressClassNames { - if ingress.Spec.IngressClassName != nil && nameFilter == *ingress.Spec.IngressClassName { - matched = true - + if ingress.Spec.IngressClassName != nil && len(*ingress.Spec.IngressClassName) > 0 { + if nameFilter == *ingress.Spec.IngressClassName { + matched = true + } } else if matchLabelSelector(selector, ingress.Annotations) { matched = true } diff --git a/source/ingress_test.go b/source/ingress_test.go index b76dd439a..2be3eddfd 100644 --- a/source/ingress_test.go +++ b/source/ingress_test.go @@ -1282,7 +1282,7 @@ func testIngressEndpoints(t *testing.T) { tlsdnsnames: [][]string{{"int-annodmz.example.org"}}, ips: []string{"5.6.7.8"}, annotations: map[string]string{ - "kubernetes.io/ingress.class": "dmz", // match + "kubernetes.io/ingress.class": "dmz", // match but ignored (non-empty ingressClassName) }, ingressClassName: "internal", }, @@ -1302,7 +1302,7 @@ func testIngressEndpoints(t *testing.T) { tlsdnsnames: [][]string{{"empty-annotdmz.example.org"}}, ips: []string{"7.8.9.0"}, annotations: map[string]string{ - "kubernetes.io/ingress.class": "dmz", // match + "kubernetes.io/ingress.class": "dmz", // match (empty ingressClassName) }, ingressClassName: "", }, @@ -1333,11 +1333,6 @@ func testIngressEndpoints(t *testing.T) { RecordType: endpoint.RecordTypeA, Targets: endpoint.Targets{"4.5.6.7"}, }, - { - DNSName: "int-annodmz.example.org", - RecordType: endpoint.RecordTypeA, - Targets: endpoint.Targets{"5.6.7.8"}, - }, { DNSName: "dmz-annoint.example.org", RecordType: endpoint.RecordTypeA, From f42f3705c5d213a2fed5fdef4abf6c3c736a740e Mon Sep 17 00:00:00 2001 From: Arnaud Lefray Date: Fri, 5 May 2023 16:11:26 +0200 Subject: [PATCH 27/32] docs: replace mentions of ingress.class annotations for the spec.ingressClassName field --- docs/faq.md | 16 +++++++++----- docs/tutorials/alibabacloud.md | 3 +-- .../tutorials/aws-load-balancer-controller.md | 5 +---- docs/tutorials/aws.md | 6 ++--- docs/tutorials/azure-private-dns.md | 3 +-- docs/tutorials/coredns.md | 3 +-- docs/tutorials/exoscale.md | 2 +- docs/tutorials/kube-ingress-aws.md | 5 ----- docs/tutorials/nginx-ingress.md | 4 ---- docs/tutorials/public-private-route53.md | 22 +++++-------------- docs/tutorials/rdns.md | 3 +-- 11 files changed, 25 insertions(+), 47 deletions(-) diff --git a/docs/faq.md b/docs/faq.md index 527db7ccd..adefacb81 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -257,8 +257,7 @@ spec: ### Running an internal and external dns service Sometimes you need to run an internal and an external dns service. -The internal one should provision hostnames used on the internal network (perhaps inside a VPC), and the external -one to expose DNS to the internet. +The internal one should provision hostnames used on the internal network (perhaps inside a VPC), and the external one to expose DNS to the internet. To do this with ExternalDNS you can use the `--ingress-class` flag to specifically tie an instance of ExternalDNS to an instance of a ingress controller. Let's assume you have two ingress controllers, `nginx-internal` and `nginx-external`. @@ -267,16 +266,21 @@ You can then start two ExternalDNS providers, one with `--ingress-class=nginx-in If you need to search for multiple ingress classes, you can specify the flag multiple times, like so: `--ingress-class=nginx-internal --ingress-class=alb-ingress-internal`. -The `--ingress-class` flag will check both the `ingressClassName` field and the deprecated `kubernetes.io/ingress.class` annotation. +The `--ingress-class` flag will check both the `spec.ingressClassName` field and the deprecated `kubernetes.io/ingress.class` annotation. +The `spec.ingressClassName` tasks precedence over the annotation if both are supplied. -Note: the `--ingress-class` flag cannot be used at the same time as a `kubernetes.io/ingress.class` annotation filter; if you do this an error will be raised. +**Backward compatibility** -If you use annotations to indicate different ingress classes in your cluster, you can instead use an `--annotation-filter` flag to restrict which objects ExternalDNS considers; for example, `--annotation-filter=kubernetes.io/ingress.class in (public,dmz)`. +The previous `--annotation-filter` flag can still be used to restrict which objects ExternalDNS considers; for example, `--annotation-filter=kubernetes.io/ingress.class in (public,dmz)`. However, beware when using annotation filters with multiple sources, e.g. `--source=service --source=ingress`, since `--annotation-filter` will filter every given source object. If you need to use annotation filters against a specific source you have to run a separated external dns service containing only the wanted `--source` and `--annotation-filter`. -**Note:** Filtering based on annotation or ingress class name means that the external-dns controller will receive all resources of that kind and then filter on the client-side. +Note: the `--ingress-class` flag cannot be used at the same time as the `--annotation-filter=kubernetes.io/ingress.class in (...)` flag; if you do this an error will be raised. + +**Performance considerations** + +Filtering based on ingress class name or annotations means that the external-dns controller will receive all resources of that kind and then filter on the client-side. In larger clusters with many resources which change frequently this can cause performance issues. If only some resources need to be managed by an instance of external-dns then label filtering can be used instead of ingress class filtering (or legacy annotation filtering). This means that only those resources which match the selector specified in `--label-filter` will be passed to the controller. diff --git a/docs/tutorials/alibabacloud.md b/docs/tutorials/alibabacloud.md index ee0477ede..f7653237e 100644 --- a/docs/tutorials/alibabacloud.md +++ b/docs/tutorials/alibabacloud.md @@ -233,9 +233,8 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: foo - annotations: - kubernetes.io/ingress.class: "nginx" # use the one that corresponds to your ingress controller. spec: + ingressClassName: nginx # use the one that corresponds to your ingress controller. rules: - host: foo.external-dns-test.com http: diff --git a/docs/tutorials/aws-load-balancer-controller.md b/docs/tutorials/aws-load-balancer-controller.md index 9e66b1e6d..98bc5da69 100644 --- a/docs/tutorials/aws-load-balancer-controller.md +++ b/docs/tutorials/aws-load-balancer-controller.md @@ -24,7 +24,7 @@ as Kubernetes does with the AWS cloud provider. In the examples that follow, it is assumed that you configured the ALB Ingress Controller with the `ingress-class=alb` argument (not to be confused with the same argument to ExternalDNS) so that the controller will only respect Ingress -objects with the `kubernetes.io/ingress.class` annotation set to "alb". +objects with the `ingressClassName` field set to "alb". ## Deploy an example application @@ -80,7 +80,6 @@ kind: Ingress metadata: annotations: alb.ingress.kubernetes.io/scheme: internet-facing - kubernetes.io/ingress.class: alb name: echoserver spec: ingressClassName: alb @@ -120,7 +119,6 @@ metadata: annotations: alb.ingress.kubernetes.io/scheme: internet-facing external-dns.alpha.kubernetes.io/hostname: echoserver.mycluster.example.org, echoserver.example.org - kubernetes.io/ingress.class: alb name: echoserver spec: ingressClassName: alb @@ -159,7 +157,6 @@ metadata: annotations: alb.ingress.kubernetes.io/scheme: internet-facing alb.ingress.kubernetes.io/ip-address-type: dualstack - kubernetes.io/ingress.class: alb name: echoserver spec: ingressClassName: alb diff --git a/docs/tutorials/aws.md b/docs/tutorials/aws.md index d5e7ad92b..e749c16d7 100644 --- a/docs/tutorials/aws.md +++ b/docs/tutorials/aws.md @@ -739,9 +739,8 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: nginx - annotations: - kubernetes.io/ingress.class: "nginx" # use the one that corresponds to your ingress controller. spec: + ingressClassName: nginx rules: - host: server.example.com http: @@ -936,7 +935,8 @@ Running several fast polling ExternalDNS instances in a given account can easily * `--source=ingress --source=service` - specify multiple times for multiple sources * `--namespace=my-app` * `--label-filter=app in (my-app)` - * `--annotation-filter=kubernetes.io/ingress.class in (nginx-external)` - note that this filter would apply to services too.. + * `--ingress-class=nginx-external` + * `--annotation-filter=kubernetes.io/ingress.class in (nginx-external)` - note that this filter would apply to services too.. (deprecated in favor of `--ingress-class`) * Limit services watched by type (not applicable to ingress or other types) * `--service-type-filter=LoadBalancer` default `all` * Limit the hosted zones considered diff --git a/docs/tutorials/azure-private-dns.md b/docs/tutorials/azure-private-dns.md index 640036e46..0e85d9026 100644 --- a/docs/tutorials/azure-private-dns.md +++ b/docs/tutorials/azure-private-dns.md @@ -416,9 +416,8 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: nginx - annotations: - kubernetes.io/ingress.class: nginx spec: + ingressClassName: nginx rules: - host: server.example.com http: diff --git a/docs/tutorials/coredns.md b/docs/tutorials/coredns.md index f2c11b8c2..5cd1223f2 100644 --- a/docs/tutorials/coredns.md +++ b/docs/tutorials/coredns.md @@ -198,9 +198,8 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: nginx - annotations: - kubernetes.io/ingress.class: "nginx" spec: + ingressClassName: nginx rules: - host: nginx.example.org http: diff --git a/docs/tutorials/exoscale.md b/docs/tutorials/exoscale.md index d1e93cb0f..047c1d6b8 100644 --- a/docs/tutorials/exoscale.md +++ b/docs/tutorials/exoscale.md @@ -109,9 +109,9 @@ kind: Ingress metadata: name: nginx annotations: - kubernetes.io/ingress.class: nginx external-dns.alpha.kubernetes.io/target: {{ Elastic-IP-address }} spec: + ingressClassName: nginx rules: - host: via-ingress.example.com http: diff --git a/docs/tutorials/kube-ingress-aws.md b/docs/tutorials/kube-ingress-aws.md index bea5e56ac..5cf37d4ec 100644 --- a/docs/tutorials/kube-ingress-aws.md +++ b/docs/tutorials/kube-ingress-aws.md @@ -141,8 +141,6 @@ Create the following Ingress to expose the echoserver application to the Interne apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - annotations: - kubernetes.io/ingress.class: skipper name: echoserver spec: ingressClassName: skipper @@ -181,7 +179,6 @@ kind: Ingress metadata: annotations: external-dns.alpha.kubernetes.io/hostname: echoserver.mycluster.example.org, echoserver.example.org - kubernetes.io/ingress.class: skipper name: echoserver spec: ingressClassName: skipper @@ -218,7 +215,6 @@ kind: Ingress metadata: annotations: alb.ingress.kubernetes.io/ip-address-type: dualstack - kubernetes.io/ingress.class: skipper name: echoserver spec: ingressClassName: skipper @@ -256,7 +252,6 @@ kind: Ingress metadata: annotations: zalando.org/aws-load-balancer-type: nlb - kubernetes.io/ingress.class: skipper name: echoserver spec: ingressClassName: skipper diff --git a/docs/tutorials/nginx-ingress.md b/docs/tutorials/nginx-ingress.md index f6c170b4c..ce79f24d9 100644 --- a/docs/tutorials/nginx-ingress.md +++ b/docs/tutorials/nginx-ingress.md @@ -294,8 +294,6 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: nginx - annotations: - kubernetes.io/ingress.class: nginx spec: ingressClassName: nginx rules: @@ -595,8 +593,6 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: nginx - annotations: - kubernetes.io/ingress.class: nginx spec: ingressClassName: nginx rules: diff --git a/docs/tutorials/public-private-route53.md b/docs/tutorials/public-private-route53.md index 9ceb43195..66071e13d 100644 --- a/docs/tutorials/public-private-route53.md +++ b/docs/tutorials/public-private-route53.md @@ -213,7 +213,7 @@ spec: Consult [AWS ExternalDNS setup docs](aws.md) for installation guidelines. -In ExternalDNS containers args, make sure to specify `aws-zone-type` and either `ingress-class` or `annotation-filter` (depending on whether your cluster makes use of `ingressClassName`): +In ExternalDNS containers args, make sure to specify `aws-zone-type` and `ingress-class`: ```yaml apiVersion: apps/v1beta2 @@ -251,7 +251,7 @@ spec: Consult [AWS ExternalDNS setup docs](aws.md) for installation guidelines. -In ExternalDNS containers args, make sure to specify `aws-zone-type` and either `ingress-class` or `annotation-filter` (depending on whether your cluster makes use of `ingressClassName`): +In ExternalDNS containers args, make sure to specify `aws-zone-type` and `ingress-class`: ```yaml apiVersion: apps/v1beta2 @@ -280,8 +280,6 @@ spec: - --registry=txt - --txt-owner-id=dev.k8s.nexus - --ingress-class=internal-ingress - # ... or, if you use annotations for ingress classes - # - --annotation-filter=kubernetes.io/ingress.class in (internal-ingress) - --aws-zone-type=private image: registry.k8s.io/external-dns/external-dns:v0.13.4 name: external-dns-private @@ -291,21 +289,17 @@ spec: For this setup to work, you need to create two Ingress definitions for your application. -At first, create a public Ingress definition (make sure to un-comment either the `annotations` or `ingressClassName` lines): +At first, create a public Ingress definition: ```yaml apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - # uncomment if you use annotations for ingress classes - # annotations: - # kubernetes.io/ingress.class: "external-ingress" labels: app: app name: app-public spec: - # uncomment if you use ingressClassName - # ingressClassName: external-ingress + ingressClassName: external-ingress rules: - host: app.domain.com http: @@ -318,21 +312,17 @@ spec: pathType: Prefix ``` -Then create a private Ingress definition (again, make sure to un-comment either the `annotations` or `ingressClassName` lines): +Then create a private Ingress definition: ```yaml apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - # uncomment if you use annotations for ingress classes - # annotations: - # kubernetes.io/ingress.class: "internal-ingress" labels: app: app name: app-private spec: - # uncomment if you use ingressClassName - # ingressClassName: internal-ingress + ingressClassName: internal-ingress rules: - host: app.domain.com http: diff --git a/docs/tutorials/rdns.md b/docs/tutorials/rdns.md index 684c7c64d..529b1765c 100644 --- a/docs/tutorials/rdns.md +++ b/docs/tutorials/rdns.md @@ -142,9 +142,8 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: nginx - annotations: - kubernetes.io/ingress.class: "nginx" spec: + ingressClassName: nginx rules: - host: nginx.lb.rancher.cloud http: From 683663e9c21f0ed9a2cff073f555baec6752dadc Mon Sep 17 00:00:00 2001 From: John Gardiner Myers Date: Tue, 2 May 2023 22:55:08 -0700 Subject: [PATCH 28/32] IPv6 internal node IPs are usable externally --- docs/tutorials/nodes.md | 5 +- source/compatibility.go | 10 +-- source/node.go | 36 ++++++---- source/node_test.go | 57 +++++++++++++++- source/pod.go | 52 ++++++++------- source/pod_test.go | 142 +++++++++++++++++++++++++++++++++++++++- source/service.go | 21 ++++-- source/service_test.go | 37 +++++++++++ source/shared_test.go | 5 +- 9 files changed, 314 insertions(+), 51 deletions(-) diff --git a/docs/tutorials/nodes.md b/docs/tutorials/nodes.md index 46f21da5d..b99a2f9ca 100644 --- a/docs/tutorials/nodes.md +++ b/docs/tutorials/nodes.md @@ -3,8 +3,9 @@ This tutorial describes how to configure ExternalDNS to use the cluster nodes as source. Using nodes (`--source=node`) as source is possible to synchronize a DNS zone with the nodes of a cluster. -The node source adds an `A` record per each node `externalIP` (if not found, node's `internalIP` is used). -The TTL record can be set with the `external-dns.alpha.kubernetes.io/ttl` node annotation. +The node source adds an `A` record per each node `externalIP` (if not found, any IPv4 `internalIP` is used instead). +It also adds an `AAAA` record per each node IPv6 `internalIP`. +The TTL of the records can be set with the `external-dns.alpha.kubernetes.io/ttl` node annotation. ## Manifest (for cluster without RBAC enabled) diff --git a/source/compatibility.go b/source/compatibility.go index bc6e19abf..1953b76ca 100644 --- a/source/compatibility.go +++ b/source/compatibility.go @@ -157,11 +157,13 @@ func legacyEndpointsFromDNSControllerNodePortService(svc *v1.Service, sc *servic continue } for _, address := range node.Status.Addresses { - if address.Type == v1.NodeExternalIP && isExternal { - endpoints = append(endpoints, endpoint.NewEndpoint(hostname, endpoint.RecordTypeA, address.Address)) + recordType := suitableType(address.Address) + // IPv6 addresses are labeled as NodeInternalIP despite being usable externally as well. + if isExternal && (address.Type == v1.NodeExternalIP || (address.Type == v1.NodeInternalIP && recordType == endpoint.RecordTypeAAAA)) { + endpoints = append(endpoints, endpoint.NewEndpoint(hostname, recordType, address.Address)) } - if address.Type == v1.NodeInternalIP && isInternal { - endpoints = append(endpoints, endpoint.NewEndpoint(hostname, endpoint.RecordTypeA, address.Address)) + if isInternal && address.Type == v1.NodeInternalIP { + endpoints = append(endpoints, endpoint.NewEndpoint(hostname, recordType, address.Address)) } } } diff --git a/source/node.go b/source/node.go index b0e672d73..39135ceaf 100644 --- a/source/node.go +++ b/source/node.go @@ -76,6 +76,11 @@ func NewNodeSource(ctx context.Context, kubeClient kubernetes.Interface, annotat }, nil } +type endpointsKey struct { + dnsName string + recordType string +} + // Endpoints returns endpoint objects for each service that should be processed. func (ns *nodeSource) Endpoints(ctx context.Context) ([]*endpoint.Endpoint, error) { nodes, err := ns.nodeInformer.Lister().List(labels.Everything()) @@ -88,7 +93,7 @@ func (ns *nodeSource) Endpoints(ctx context.Context) ([]*endpoint.Endpoint, erro return nil, err } - endpoints := map[string]*endpoint.Endpoint{} + endpoints := map[endpointsKey]*endpoint.Endpoint{} // create endpoints for all nodes for _, node := range nodes { @@ -109,8 +114,7 @@ func (ns *nodeSource) Endpoints(ctx context.Context) ([]*endpoint.Endpoint, erro // create new endpoint with the information we already have ep := &endpoint.Endpoint{ - RecordType: "A", // hardcoded DNS record type - RecordTTL: ttl, + RecordTTL: ttl, } if ns.fqdnTemplate != nil { @@ -134,14 +138,19 @@ func (ns *nodeSource) Endpoints(ctx context.Context) ([]*endpoint.Endpoint, erro return nil, fmt.Errorf("failed to get node address from %s: %s", node.Name, err.Error()) } - ep.Targets = endpoint.Targets(addrs) ep.Labels = endpoint.NewLabels() - - log.Debugf("adding endpoint %s", ep) - if _, ok := endpoints[ep.DNSName]; ok { - endpoints[ep.DNSName].Targets = append(endpoints[ep.DNSName].Targets, ep.Targets...) - } else { - endpoints[ep.DNSName] = ep + for _, addr := range addrs { + log.Debugf("adding endpoint %s target %s", ep, addr) + key := endpointsKey{ + dnsName: ep.DNSName, + recordType: suitableType(addr), + } + if _, ok := endpoints[key]; !ok { + epCopy := *ep + epCopy.RecordType = key.recordType + endpoints[key] = &epCopy + } + endpoints[key].Targets = append(endpoints[key].Targets, addr) } } @@ -163,13 +172,18 @@ func (ns *nodeSource) nodeAddresses(node *v1.Node) ([]string, error) { v1.NodeExternalIP: {}, v1.NodeInternalIP: {}, } + var ipv6Addresses []string for _, addr := range node.Status.Addresses { addresses[addr.Type] = append(addresses[addr.Type], addr.Address) + // IPv6 addresses are labeled as NodeInternalIP despite being usable externally as well. + if addr.Type == v1.NodeInternalIP && suitableType(addr.Address) == endpoint.RecordTypeAAAA { + ipv6Addresses = append(ipv6Addresses, addr.Address) + } } if len(addresses[v1.NodeExternalIP]) > 0 { - return addresses[v1.NodeExternalIP], nil + return append(addresses[v1.NodeExternalIP], ipv6Addresses...), nil } if len(addresses[v1.NodeInternalIP]) > 0 { diff --git a/source/node_test.go b/source/node_test.go index 901c1baa1..885d9f54e 100644 --- a/source/node_test.go +++ b/source/node_test.go @@ -127,6 +127,19 @@ func testNodeSourceEndpoints(t *testing.T) { }, false, }, + { + "ipv6 node with fqdn returns one endpoint", + "", + "", + "node1.example.org", + []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "2001:DB8::8"}}, + map[string]string{}, + map[string]string{}, + []*endpoint.Endpoint{ + {RecordType: "AAAA", DNSName: "node1.example.org", Targets: endpoint.Targets{"2001:DB8::8"}}, + }, + false, + }, { "node with fqdn template returns endpoint with expanded hostname", "", @@ -166,6 +179,20 @@ func testNodeSourceEndpoints(t *testing.T) { }, false, }, + { + "node with fqdn template returns two endpoints with dual-stack IP addresses and expanded hostname", + "", + "{{.Name}}.example.org", + "node1", + []v1.NodeAddress{{Type: v1.NodeExternalIP, Address: "1.2.3.4"}, {Type: v1.NodeInternalIP, Address: "2001:DB8::8"}}, + map[string]string{}, + map[string]string{}, + []*endpoint.Endpoint{ + {RecordType: "A", DNSName: "node1.example.org", Targets: endpoint.Targets{"1.2.3.4"}}, + {RecordType: "AAAA", DNSName: "node1.example.org", Targets: endpoint.Targets{"2001:DB8::8"}}, + }, + false, + }, { "node with both external and internal IP returns an endpoint with external IP", "", @@ -179,6 +206,20 @@ func testNodeSourceEndpoints(t *testing.T) { }, false, }, + { + "node with both external, internal, and IPv6 IP returns endpoints with external IPs", + "", + "", + "node1", + []v1.NodeAddress{{Type: v1.NodeExternalIP, Address: "1.2.3.4"}, {Type: v1.NodeInternalIP, Address: "2.3.4.5"}, {Type: v1.NodeInternalIP, Address: "2001:DB8::8"}}, + map[string]string{}, + map[string]string{}, + []*endpoint.Endpoint{ + {RecordType: "A", DNSName: "node1", Targets: endpoint.Targets{"1.2.3.4"}}, + {RecordType: "AAAA", DNSName: "node1", Targets: endpoint.Targets{"2001:DB8::8"}}, + }, + false, + }, { "node with only internal IP returns an endpoint with internal IP", "", @@ -192,6 +233,20 @@ func testNodeSourceEndpoints(t *testing.T) { }, false, }, + { + "node with only internal IPs returns endpoints with internal IPs", + "", + "", + "node1", + []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "2.3.4.5"}, {Type: v1.NodeInternalIP, Address: "2001:DB8::8"}}, + map[string]string{}, + map[string]string{}, + []*endpoint.Endpoint{ + {RecordType: "A", DNSName: "node1", Targets: endpoint.Targets{"2.3.4.5"}}, + {RecordType: "AAAA", DNSName: "node1", Targets: endpoint.Targets{"2001:DB8::8"}}, + }, + false, + }, { "node with neither external nor internal IP returns no endpoints", "", @@ -318,7 +373,7 @@ func testNodeSourceEndpoints(t *testing.T) { false, }, { - "node with nil Lables returns valid endpoint", + "node with nil Labels returns valid endpoint", "", "", "node1", diff --git a/source/pod.go b/source/pod.go index 36e6ffe50..87772d75b 100644 --- a/source/pod.go +++ b/source/pod.go @@ -76,13 +76,18 @@ func NewPodSource(ctx context.Context, kubeClient kubernetes.Interface, namespac func (*podSource) AddEventHandler(ctx context.Context, handler func()) { } +type endpointKey struct { + domain string + recordType string +} + func (ps *podSource) Endpoints(ctx context.Context) ([]*endpoint.Endpoint, error) { pods, err := ps.podInformer.Lister().Pods(ps.namespace).List(labels.Everything()) if err != nil { return nil, err } - domains := make(map[string][]string) + endpointMap := make(map[endpointKey][]string) for _, pod := range pods { if !pod.Spec.HostNetwork { log.Debugf("skipping pod %s. hostNetwork=false", pod.Name) @@ -90,50 +95,51 @@ func (ps *podSource) Endpoints(ctx context.Context) ([]*endpoint.Endpoint, error } if domain, ok := pod.Annotations[internalHostnameAnnotationKey]; ok { - if _, ok := domains[domain]; !ok { - domains[domain] = []string{} - } - domains[domain] = append(domains[domain], pod.Status.PodIP) + addToEndpointMap(endpointMap, domain, suitableType(pod.Status.PodIP), pod.Status.PodIP) } if domain, ok := pod.Annotations[hostnameAnnotationKey]; ok { - if _, ok := domains[domain]; !ok { - domains[domain] = []string{} - } - node, _ := ps.nodeInformer.Lister().Get(pod.Spec.NodeName) for _, address := range node.Status.Addresses { - if address.Type == corev1.NodeExternalIP { - domains[domain] = append(domains[domain], address.Address) + recordType := suitableType(address.Address) + // IPv6 addresses are labeled as NodeInternalIP despite being usable externally as well. + if address.Type == corev1.NodeExternalIP || (address.Type == corev1.NodeInternalIP && recordType == endpoint.RecordTypeAAAA) { + addToEndpointMap(endpointMap, domain, recordType, address.Address) } } } if ps.compatibility == "kops-dns-controller" { if domain, ok := pod.Annotations[kopsDNSControllerInternalHostnameAnnotationKey]; ok { - if _, ok := domains[domain]; !ok { - domains[domain] = []string{} - } - domains[domain] = append(domains[domain], pod.Status.PodIP) + addToEndpointMap(endpointMap, domain, suitableType(pod.Status.PodIP), pod.Status.PodIP) } if domain, ok := pod.Annotations[kopsDNSControllerHostnameAnnotationKey]; ok { - if _, ok := domains[domain]; !ok { - domains[domain] = []string{} - } - node, _ := ps.nodeInformer.Lister().Get(pod.Spec.NodeName) for _, address := range node.Status.Addresses { - if address.Type == corev1.NodeExternalIP { - domains[domain] = append(domains[domain], address.Address) + recordType := suitableType(address.Address) + // IPv6 addresses are labeled as NodeInternalIP despite being usable externally as well. + if address.Type == corev1.NodeExternalIP || (address.Type == corev1.NodeInternalIP && recordType == endpoint.RecordTypeAAAA) { + addToEndpointMap(endpointMap, domain, recordType, address.Address) } } } } } endpoints := []*endpoint.Endpoint{} - for domain, targets := range domains { - endpoints = append(endpoints, endpoint.NewEndpoint(domain, endpoint.RecordTypeA, targets...)) + for key, targets := range endpointMap { + endpoints = append(endpoints, endpoint.NewEndpoint(key.domain, key.recordType, targets...)) } return endpoints, nil } + +func addToEndpointMap(endpointMap map[endpointKey][]string, domain string, recordType string, address string) { + key := endpointKey{ + domain: domain, + recordType: recordType, + } + if _, ok := endpointMap[key]; !ok { + endpointMap[key] = []string{} + } + endpointMap[key] = append(endpointMap[key], address) +} diff --git a/source/pod_test.go b/source/pod_test.go index c138aaf5f..5a57aedc2 100644 --- a/source/pod_test.go +++ b/source/pod_test.go @@ -41,7 +41,7 @@ func TestPodSource(t *testing.T) { pods []*corev1.Pod }{ { - "create records based on pod's external and internal IPs", + "create IPv4 records based on pod's external and internal IPs", "", "", []*endpoint.Endpoint{ @@ -111,7 +111,7 @@ func TestPodSource(t *testing.T) { }, }, { - "create records based on pod's external and internal IPs using DNS Controller annotations", + "create IPv4 records based on pod's external and internal IPs using DNS Controller annotations", "", "kops-dns-controller", []*endpoint.Endpoint{ @@ -180,12 +180,149 @@ func TestPodSource(t *testing.T) { }, }, }, + { + "create IPv6 records based on pod's external and internal IPs", + "", + "", + []*endpoint.Endpoint{ + {DNSName: "a.foo.example.org", Targets: endpoint.Targets{"2001:DB8::1", "2001:DB8::2"}, RecordType: endpoint.RecordTypeAAAA}, + {DNSName: "internal.a.foo.example.org", Targets: endpoint.Targets{"2001:DB8::1", "2001:DB8::2"}, RecordType: endpoint.RecordTypeAAAA}, + }, + false, + []*corev1.Node{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "my-node1", + }, + Status: corev1.NodeStatus{ + Addresses: []corev1.NodeAddress{ + {Type: corev1.NodeInternalIP, Address: "2001:DB8::1"}, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "my-node2", + }, + Status: corev1.NodeStatus{ + Addresses: []corev1.NodeAddress{ + {Type: corev1.NodeInternalIP, Address: "2001:DB8::2"}, + }, + }, + }, + }, + []*corev1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "my-pod1", + Namespace: "kube-system", + Annotations: map[string]string{ + internalHostnameAnnotationKey: "internal.a.foo.example.org", + hostnameAnnotationKey: "a.foo.example.org", + }, + }, + Spec: corev1.PodSpec{ + HostNetwork: true, + NodeName: "my-node1", + }, + Status: corev1.PodStatus{ + PodIP: "2001:DB8::1", + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "my-pod2", + Namespace: "kube-system", + Annotations: map[string]string{ + internalHostnameAnnotationKey: "internal.a.foo.example.org", + hostnameAnnotationKey: "a.foo.example.org", + }, + }, + Spec: corev1.PodSpec{ + HostNetwork: true, + NodeName: "my-node2", + }, + Status: corev1.PodStatus{ + PodIP: "2001:DB8::2", + }, + }, + }, + }, + { + "create IPv6 records based on pod's external and internal IPs using DNS Controller annotations", + "", + "kops-dns-controller", + []*endpoint.Endpoint{ + {DNSName: "a.foo.example.org", Targets: endpoint.Targets{"2001:DB8::1", "2001:DB8::2"}, RecordType: endpoint.RecordTypeAAAA}, + {DNSName: "internal.a.foo.example.org", Targets: endpoint.Targets{"2001:DB8::1", "2001:DB8::2"}, RecordType: endpoint.RecordTypeAAAA}, + }, + false, + []*corev1.Node{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "my-node1", + }, + Status: corev1.NodeStatus{ + Addresses: []corev1.NodeAddress{ + {Type: corev1.NodeInternalIP, Address: "2001:DB8::1"}, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "my-node2", + }, + Status: corev1.NodeStatus{ + Addresses: []corev1.NodeAddress{ + {Type: corev1.NodeInternalIP, Address: "2001:DB8::2"}, + }, + }, + }, + }, + []*corev1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "my-pod1", + Namespace: "kube-system", + Annotations: map[string]string{ + kopsDNSControllerInternalHostnameAnnotationKey: "internal.a.foo.example.org", + kopsDNSControllerHostnameAnnotationKey: "a.foo.example.org", + }, + }, + Spec: corev1.PodSpec{ + HostNetwork: true, + NodeName: "my-node1", + }, + Status: corev1.PodStatus{ + PodIP: "2001:DB8::1", + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "my-pod2", + Namespace: "kube-system", + Annotations: map[string]string{ + kopsDNSControllerInternalHostnameAnnotationKey: "internal.a.foo.example.org", + kopsDNSControllerHostnameAnnotationKey: "a.foo.example.org", + }, + }, + Spec: corev1.PodSpec{ + HostNetwork: true, + NodeName: "my-node2", + }, + Status: corev1.PodStatus{ + PodIP: "2001:DB8::2", + }, + }, + }, + }, { "create multiple records", "", "", []*endpoint.Endpoint{ {DNSName: "a.foo.example.org", Targets: endpoint.Targets{"54.10.11.1"}, RecordType: endpoint.RecordTypeA}, + {DNSName: "a.foo.example.org", Targets: endpoint.Targets{"2001:DB8::1"}, RecordType: endpoint.RecordTypeAAAA}, {DNSName: "b.foo.example.org", Targets: endpoint.Targets{"54.10.11.2"}, RecordType: endpoint.RecordTypeA}, }, false, @@ -197,6 +334,7 @@ func TestPodSource(t *testing.T) { Status: corev1.NodeStatus{ Addresses: []corev1.NodeAddress{ {Type: corev1.NodeExternalIP, Address: "54.10.11.1"}, + {Type: corev1.NodeInternalIP, Address: "2001:DB8::1"}, {Type: corev1.NodeInternalIP, Address: "10.0.1.1"}, }, }, diff --git a/source/service.go b/source/service.go index 925c327f2..58270cdcc 100644 --- a/source/service.go +++ b/source/service.go @@ -216,7 +216,10 @@ func (sc *serviceSource) Endpoints(ctx context.Context) ([]*endpoint.Endpoint, e }) // Use stable sort to not disrupt the order of services sort.SliceStable(endpoints, func(i, j int) bool { - return endpoints[i].DNSName < endpoints[j].DNSName + if endpoints[i].DNSName != endpoints[j].DNSName { + return endpoints[i].DNSName < endpoints[j].DNSName + } + return endpoints[i].RecordType < endpoints[j].RecordType }) mergedEndpoints := []*endpoint.Endpoint{} mergedEndpoints = append(mergedEndpoints, endpoints[0]) @@ -308,8 +311,8 @@ func (sc *serviceSource) extractHeadlessEndpoints(svc *v1.Service, hostname stri return endpoints } for _, address := range node.Status.Addresses { - if address.Type == v1.NodeExternalIP { - targets = endpoint.Targets{address.Address} + if address.Type == v1.NodeExternalIP || (address.Type == v1.NodeInternalIP && suitableType(address.Address) == endpoint.RecordTypeAAAA) { + targets = append(targets, address.Address) log.Debugf("Generating matching endpoint %s with NodeExternalIP %s", headlessDomain, address.Address) } } @@ -499,7 +502,7 @@ func (sc *serviceSource) generateEndpoints(svc *v1.Service, hostname string, pro log.Errorf("Unable to extract targets from service %s/%s error: %v", svc.Namespace, svc.Name, err) return endpoints } - endpoints = append(endpoints, sc.extractNodePortEndpoints(svc, targets, hostname, ttl)...) + endpoints = append(endpoints, sc.extractNodePortEndpoints(svc, hostname, ttl)...) case v1.ServiceTypeExternalName: targets = append(targets, extractServiceExternalName(svc)...) } @@ -587,6 +590,7 @@ func (sc *serviceSource) extractNodePortTargets(svc *v1.Service) (endpoint.Targe var ( internalIPs endpoint.Targets externalIPs endpoint.Targets + ipv6IPs endpoint.Targets nodes []*v1.Node err error ) @@ -634,24 +638,27 @@ func (sc *serviceSource) extractNodePortTargets(svc *v1.Service) (endpoint.Targe externalIPs = append(externalIPs, address.Address) case v1.NodeInternalIP: internalIPs = append(internalIPs, address.Address) + if suitableType(address.Address) == endpoint.RecordTypeAAAA { + ipv6IPs = append(ipv6IPs, address.Address) + } } } } access := getAccessFromAnnotations(svc.Annotations) if access == "public" { - return externalIPs, nil + return append(externalIPs, ipv6IPs...), nil } if access == "private" { return internalIPs, nil } if len(externalIPs) > 0 { - return externalIPs, nil + return append(externalIPs, ipv6IPs...), nil } return internalIPs, nil } -func (sc *serviceSource) extractNodePortEndpoints(svc *v1.Service, nodeTargets endpoint.Targets, hostname string, ttl endpoint.TTL) []*endpoint.Endpoint { +func (sc *serviceSource) extractNodePortEndpoints(svc *v1.Service, hostname string, ttl endpoint.TTL) []*endpoint.Endpoint { var endpoints []*endpoint.Endpoint for _, port := range svc.Spec.Ports { diff --git a/source/service_test.go b/source/service_test.go index 98b400fca..45737adb7 100644 --- a/source/service_test.go +++ b/source/service_test.go @@ -1518,6 +1518,7 @@ func TestServiceSourceNodePortServices(t *testing.T) { expected: []*endpoint.Endpoint{ {DNSName: "_foo._tcp.foo.example.org", Targets: endpoint.Targets{"0 50 30192 foo.example.org"}, RecordType: endpoint.RecordTypeSRV}, {DNSName: "foo.example.org", Targets: endpoint.Targets{"54.10.11.1", "54.10.11.2"}, RecordType: endpoint.RecordTypeA}, + {DNSName: "foo.example.org", Targets: endpoint.Targets{"2001:DB8::1", "2001:DB8::2"}, RecordType: endpoint.RecordTypeAAAA}, }, nodes: []*v1.Node{{ ObjectMeta: metav1.ObjectMeta{ @@ -1527,6 +1528,7 @@ func TestServiceSourceNodePortServices(t *testing.T) { Addresses: []v1.NodeAddress{ {Type: v1.NodeExternalIP, Address: "54.10.11.1"}, {Type: v1.NodeInternalIP, Address: "10.0.1.1"}, + {Type: v1.NodeInternalIP, Address: "2001:DB8::1"}, }, }, }, { @@ -1537,6 +1539,7 @@ func TestServiceSourceNodePortServices(t *testing.T) { Addresses: []v1.NodeAddress{ {Type: v1.NodeExternalIP, Address: "54.10.11.2"}, {Type: v1.NodeInternalIP, Address: "10.0.1.2"}, + {Type: v1.NodeInternalIP, Address: "2001:DB8::2"}, }, }, }}, @@ -1559,6 +1562,7 @@ func TestServiceSourceNodePortServices(t *testing.T) { Addresses: []v1.NodeAddress{ {Type: v1.NodeExternalIP, Address: "54.10.11.1"}, {Type: v1.NodeInternalIP, Address: "10.0.1.1"}, + {Type: v1.NodeInternalIP, Address: "2001:DB8::1"}, }, }, }, { @@ -1569,6 +1573,7 @@ func TestServiceSourceNodePortServices(t *testing.T) { Addresses: []v1.NodeAddress{ {Type: v1.NodeExternalIP, Address: "54.10.11.2"}, {Type: v1.NodeInternalIP, Address: "10.0.1.2"}, + {Type: v1.NodeInternalIP, Address: "2001:DB8::2"}, }, }, }}, @@ -1584,6 +1589,7 @@ func TestServiceSourceNodePortServices(t *testing.T) { expected: []*endpoint.Endpoint{ {DNSName: "_foo._tcp.foo.bar.example.com", Targets: endpoint.Targets{"0 50 30192 foo.bar.example.com"}, RecordType: endpoint.RecordTypeSRV}, {DNSName: "foo.bar.example.com", Targets: endpoint.Targets{"54.10.11.1", "54.10.11.2"}, RecordType: endpoint.RecordTypeA}, + {DNSName: "foo.bar.example.com", Targets: endpoint.Targets{"2001:DB8::1", "2001:DB8::2"}, RecordType: endpoint.RecordTypeAAAA}, }, nodes: []*v1.Node{{ ObjectMeta: metav1.ObjectMeta{ @@ -1593,6 +1599,7 @@ func TestServiceSourceNodePortServices(t *testing.T) { Addresses: []v1.NodeAddress{ {Type: v1.NodeExternalIP, Address: "54.10.11.1"}, {Type: v1.NodeInternalIP, Address: "10.0.1.1"}, + {Type: v1.NodeInternalIP, Address: "2001:DB8::1"}, }, }, }, { @@ -1603,6 +1610,7 @@ func TestServiceSourceNodePortServices(t *testing.T) { Addresses: []v1.NodeAddress{ {Type: v1.NodeExternalIP, Address: "54.10.11.2"}, {Type: v1.NodeInternalIP, Address: "10.0.1.2"}, + {Type: v1.NodeInternalIP, Address: "2001:DB8::2"}, }, }, }}, @@ -1619,6 +1627,7 @@ func TestServiceSourceNodePortServices(t *testing.T) { expected: []*endpoint.Endpoint{ {DNSName: "_foo._tcp.foo.example.org", Targets: endpoint.Targets{"0 50 30192 foo.example.org"}, RecordType: endpoint.RecordTypeSRV}, {DNSName: "foo.example.org", Targets: endpoint.Targets{"10.0.1.1", "10.0.1.2"}, RecordType: endpoint.RecordTypeA}, + {DNSName: "foo.example.org", Targets: endpoint.Targets{"2001:DB8::1", "2001:DB8::2"}, RecordType: endpoint.RecordTypeAAAA}, }, nodes: []*v1.Node{{ ObjectMeta: metav1.ObjectMeta{ @@ -1627,6 +1636,7 @@ func TestServiceSourceNodePortServices(t *testing.T) { Status: v1.NodeStatus{ Addresses: []v1.NodeAddress{ {Type: v1.NodeInternalIP, Address: "10.0.1.1"}, + {Type: v1.NodeInternalIP, Address: "2001:DB8::1"}, }, }, }, { @@ -1636,6 +1646,7 @@ func TestServiceSourceNodePortServices(t *testing.T) { Status: v1.NodeStatus{ Addresses: []v1.NodeAddress{ {Type: v1.NodeInternalIP, Address: "10.0.1.2"}, + {Type: v1.NodeInternalIP, Address: "2001:DB8::2"}, }, }, }}, @@ -1652,6 +1663,7 @@ func TestServiceSourceNodePortServices(t *testing.T) { expected: []*endpoint.Endpoint{ {DNSName: "_foo._tcp.foo.example.org", Targets: endpoint.Targets{"0 50 30192 foo.example.org"}, RecordType: endpoint.RecordTypeSRV}, {DNSName: "foo.example.org", Targets: endpoint.Targets{"54.10.11.2"}, RecordType: endpoint.RecordTypeA}, + {DNSName: "foo.example.org", Targets: endpoint.Targets{"2001:DB8::2"}, RecordType: endpoint.RecordTypeAAAA}, }, nodes: []*v1.Node{{ ObjectMeta: metav1.ObjectMeta{ @@ -1661,6 +1673,7 @@ func TestServiceSourceNodePortServices(t *testing.T) { Addresses: []v1.NodeAddress{ {Type: v1.NodeExternalIP, Address: "54.10.11.1"}, {Type: v1.NodeInternalIP, Address: "10.0.1.1"}, + {Type: v1.NodeInternalIP, Address: "2001:DB8::1"}, }, }, }, { @@ -1671,6 +1684,7 @@ func TestServiceSourceNodePortServices(t *testing.T) { Addresses: []v1.NodeAddress{ {Type: v1.NodeExternalIP, Address: "54.10.11.2"}, {Type: v1.NodeInternalIP, Address: "10.0.1.2"}, + {Type: v1.NodeInternalIP, Address: "2001:DB8::2"}, }, }, }}, @@ -1691,6 +1705,7 @@ func TestServiceSourceNodePortServices(t *testing.T) { expected: []*endpoint.Endpoint{ {DNSName: "_foo._tcp.foo.example.org", Targets: endpoint.Targets{"0 50 30192 foo.example.org"}, RecordType: endpoint.RecordTypeSRV}, {DNSName: "foo.example.org", Targets: endpoint.Targets{"54.10.11.2"}, RecordType: endpoint.RecordTypeA}, + {DNSName: "foo.example.org", Targets: endpoint.Targets{"2001:DB8::2"}, RecordType: endpoint.RecordTypeAAAA}, }, nodes: []*v1.Node{{ ObjectMeta: metav1.ObjectMeta{ @@ -1700,6 +1715,7 @@ func TestServiceSourceNodePortServices(t *testing.T) { Addresses: []v1.NodeAddress{ {Type: v1.NodeExternalIP, Address: "54.10.11.1"}, {Type: v1.NodeInternalIP, Address: "10.0.1.1"}, + {Type: v1.NodeInternalIP, Address: "2001:DB8::1"}, }, }, }, { @@ -1710,6 +1726,7 @@ func TestServiceSourceNodePortServices(t *testing.T) { Addresses: []v1.NodeAddress{ {Type: v1.NodeExternalIP, Address: "54.10.11.2"}, {Type: v1.NodeInternalIP, Address: "10.0.1.2"}, + {Type: v1.NodeInternalIP, Address: "2001:DB8::2"}, }, }, }}, @@ -1731,6 +1748,7 @@ func TestServiceSourceNodePortServices(t *testing.T) { expected: []*endpoint.Endpoint{ {DNSName: "_foo._tcp.foo.example.org", Targets: endpoint.Targets{"0 50 30192 foo.example.org"}, RecordType: endpoint.RecordTypeSRV}, {DNSName: "foo.example.org", Targets: endpoint.Targets{"10.0.1.1", "10.0.1.2"}, RecordType: endpoint.RecordTypeA}, + {DNSName: "foo.example.org", Targets: endpoint.Targets{"2001:DB8::1", "2001:DB8::2"}, RecordType: endpoint.RecordTypeAAAA}, }, nodes: []*v1.Node{{ ObjectMeta: metav1.ObjectMeta{ @@ -1740,6 +1758,7 @@ func TestServiceSourceNodePortServices(t *testing.T) { Addresses: []v1.NodeAddress{ {Type: v1.NodeExternalIP, Address: "54.10.11.1"}, {Type: v1.NodeInternalIP, Address: "10.0.1.1"}, + {Type: v1.NodeInternalIP, Address: "2001:DB8::1"}, }, }, }, { @@ -1750,6 +1769,7 @@ func TestServiceSourceNodePortServices(t *testing.T) { Addresses: []v1.NodeAddress{ {Type: v1.NodeExternalIP, Address: "54.10.11.2"}, {Type: v1.NodeInternalIP, Address: "10.0.1.2"}, + {Type: v1.NodeInternalIP, Address: "2001:DB8::2"}, }, }, }}, @@ -1768,6 +1788,7 @@ func TestServiceSourceNodePortServices(t *testing.T) { expected: []*endpoint.Endpoint{ {DNSName: "_foo._tcp.foo.example.org", Targets: endpoint.Targets{"0 50 30192 foo.example.org"}, RecordType: endpoint.RecordTypeSRV}, {DNSName: "foo.example.org", Targets: endpoint.Targets{"54.10.11.1", "54.10.11.2"}, RecordType: endpoint.RecordTypeA}, + {DNSName: "foo.example.org", Targets: endpoint.Targets{"2001:DB8::1", "2001:DB8::2"}, RecordType: endpoint.RecordTypeAAAA}, }, nodes: []*v1.Node{{ ObjectMeta: metav1.ObjectMeta{ @@ -1777,6 +1798,7 @@ func TestServiceSourceNodePortServices(t *testing.T) { Addresses: []v1.NodeAddress{ {Type: v1.NodeExternalIP, Address: "54.10.11.1"}, {Type: v1.NodeInternalIP, Address: "10.0.1.1"}, + {Type: v1.NodeInternalIP, Address: "2001:DB8::1"}, }, }, }, { @@ -1787,6 +1809,7 @@ func TestServiceSourceNodePortServices(t *testing.T) { Addresses: []v1.NodeAddress{ {Type: v1.NodeExternalIP, Address: "54.10.11.2"}, {Type: v1.NodeInternalIP, Address: "10.0.1.2"}, + {Type: v1.NodeInternalIP, Address: "2001:DB8::2"}, }, }, }}, @@ -1804,7 +1827,9 @@ func TestServiceSourceNodePortServices(t *testing.T) { }, expected: []*endpoint.Endpoint{ {DNSName: "internal.foo.example.org", RecordType: endpoint.RecordTypeA, Targets: endpoint.Targets{"10.0.1.1"}}, + {DNSName: "internal.foo.example.org", RecordType: endpoint.RecordTypeAAAA, Targets: endpoint.Targets{"2001:DB8::1"}}, {DNSName: "internal.bar.example.org", RecordType: endpoint.RecordTypeA, Targets: endpoint.Targets{"10.0.1.1"}}, + {DNSName: "internal.bar.example.org", RecordType: endpoint.RecordTypeAAAA, Targets: endpoint.Targets{"2001:DB8::1"}}, }, nodes: []*v1.Node{{ ObjectMeta: metav1.ObjectMeta{ @@ -1817,6 +1842,7 @@ func TestServiceSourceNodePortServices(t *testing.T) { Addresses: []v1.NodeAddress{ {Type: v1.NodeExternalIP, Address: "54.10.11.1"}, {Type: v1.NodeInternalIP, Address: "10.0.1.1"}, + {Type: v1.NodeInternalIP, Address: "2001:DB8::1"}, }, }, }, { @@ -1830,6 +1856,7 @@ func TestServiceSourceNodePortServices(t *testing.T) { Addresses: []v1.NodeAddress{ {Type: v1.NodeExternalIP, Address: "54.10.11.2"}, {Type: v1.NodeInternalIP, Address: "10.0.1.2"}, + {Type: v1.NodeInternalIP, Address: "2001:DB8::2"}, }, }, }}, @@ -1846,7 +1873,9 @@ func TestServiceSourceNodePortServices(t *testing.T) { }, expected: []*endpoint.Endpoint{ {DNSName: "internal.foo.example.org", RecordType: endpoint.RecordTypeA, Targets: endpoint.Targets{"10.0.1.1", "10.0.1.2"}}, + {DNSName: "internal.foo.example.org", RecordType: endpoint.RecordTypeAAAA, Targets: endpoint.Targets{"2001:DB8::1", "2001:DB8::2"}}, {DNSName: "internal.bar.example.org", RecordType: endpoint.RecordTypeA, Targets: endpoint.Targets{"10.0.1.1", "10.0.1.2"}}, + {DNSName: "internal.bar.example.org", RecordType: endpoint.RecordTypeAAAA, Targets: endpoint.Targets{"2001:DB8::1", "2001:DB8::2"}}, }, nodes: []*v1.Node{{ ObjectMeta: metav1.ObjectMeta{ @@ -1859,6 +1888,7 @@ func TestServiceSourceNodePortServices(t *testing.T) { Addresses: []v1.NodeAddress{ {Type: v1.NodeExternalIP, Address: "54.10.11.1"}, {Type: v1.NodeInternalIP, Address: "10.0.1.1"}, + {Type: v1.NodeInternalIP, Address: "2001:DB8::1"}, }, }, }, { @@ -1872,6 +1902,7 @@ func TestServiceSourceNodePortServices(t *testing.T) { Addresses: []v1.NodeAddress{ {Type: v1.NodeExternalIP, Address: "54.10.11.2"}, {Type: v1.NodeInternalIP, Address: "10.0.1.2"}, + {Type: v1.NodeInternalIP, Address: "2001:DB8::2"}, }, }, }}, @@ -1888,7 +1919,9 @@ func TestServiceSourceNodePortServices(t *testing.T) { }, expected: []*endpoint.Endpoint{ {DNSName: "foo.example.org", RecordType: endpoint.RecordTypeA, Targets: endpoint.Targets{"54.10.11.1", "54.10.11.2"}}, + {DNSName: "foo.example.org", RecordType: endpoint.RecordTypeAAAA, Targets: endpoint.Targets{"2001:DB8::1", "2001:DB8::2"}}, {DNSName: "bar.example.org", RecordType: endpoint.RecordTypeA, Targets: endpoint.Targets{"54.10.11.1", "54.10.11.2"}}, + {DNSName: "bar.example.org", RecordType: endpoint.RecordTypeAAAA, Targets: endpoint.Targets{"2001:DB8::1", "2001:DB8::2"}}, }, nodes: []*v1.Node{{ ObjectMeta: metav1.ObjectMeta{ @@ -1901,6 +1934,7 @@ func TestServiceSourceNodePortServices(t *testing.T) { Addresses: []v1.NodeAddress{ {Type: v1.NodeExternalIP, Address: "54.10.11.1"}, {Type: v1.NodeInternalIP, Address: "10.0.1.1"}, + {Type: v1.NodeInternalIP, Address: "2001:DB8::1"}, }, }, }, { @@ -1914,6 +1948,7 @@ func TestServiceSourceNodePortServices(t *testing.T) { Addresses: []v1.NodeAddress{ {Type: v1.NodeExternalIP, Address: "54.10.11.2"}, {Type: v1.NodeInternalIP, Address: "10.0.1.2"}, + {Type: v1.NodeInternalIP, Address: "2001:DB8::2"}, }, }, }}, @@ -1942,6 +1977,7 @@ func TestServiceSourceNodePortServices(t *testing.T) { Addresses: []v1.NodeAddress{ {Type: v1.NodeExternalIP, Address: "54.10.11.1"}, {Type: v1.NodeInternalIP, Address: "10.0.1.1"}, + {Type: v1.NodeInternalIP, Address: "2001:DB8::1"}, }, }, }, { @@ -1955,6 +1991,7 @@ func TestServiceSourceNodePortServices(t *testing.T) { Addresses: []v1.NodeAddress{ {Type: v1.NodeExternalIP, Address: "54.10.11.2"}, {Type: v1.NodeInternalIP, Address: "10.0.1.2"}, + {Type: v1.NodeInternalIP, Address: "2001:DB8::2"}, }, }, }}, diff --git a/source/shared_test.go b/source/shared_test.go index 9cdc58d22..11828dbe2 100644 --- a/source/shared_test.go +++ b/source/shared_test.go @@ -29,11 +29,14 @@ func sortEndpoints(endpoints []*endpoint.Endpoint) { sort.Strings([]string(ep.Targets)) } sort.Slice(endpoints, func(i, k int) bool { - // Sort by DNSName and Targets + // Sort by DNSName, RecordType, and Targets ei, ek := endpoints[i], endpoints[k] if ei.DNSName != ek.DNSName { return ei.DNSName < ek.DNSName } + if ei.RecordType != ek.RecordType { + return ei.RecordType < ek.RecordType + } // Targets are sorted ahead of time. for j, ti := range ei.Targets { if j >= len(ek.Targets) { From 7feb8d67e9173eee9f5114de31007a42a85f1e04 Mon Sep 17 00:00:00 2001 From: Arnaud Lefray Date: Tue, 9 May 2023 08:45:14 +0200 Subject: [PATCH 29/32] docs: remove product mention. --- docs/faq.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/faq.md b/docs/faq.md index adefacb81..43731a371 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -260,11 +260,11 @@ Sometimes you need to run an internal and an external dns service. The internal one should provision hostnames used on the internal network (perhaps inside a VPC), and the external one to expose DNS to the internet. To do this with ExternalDNS you can use the `--ingress-class` flag to specifically tie an instance of ExternalDNS to an instance of a ingress controller. -Let's assume you have two ingress controllers, `nginx-internal` and `nginx-external`. -You can then start two ExternalDNS providers, one with `--ingress-class=nginx-internal` and one with `--ingress-class=nginx-external`. +Let's assume you have two ingress controllers, `internal` and `external`. +You can then start two ExternalDNS providers, one with `--ingress-class=internal` and one with `--ingress-class=external`. If you need to search for multiple ingress classes, you can specify the flag multiple times, like so: -`--ingress-class=nginx-internal --ingress-class=alb-ingress-internal`. +`--ingress-class=internal --ingress-class=external`. The `--ingress-class` flag will check both the `spec.ingressClassName` field and the deprecated `kubernetes.io/ingress.class` annotation. The `spec.ingressClassName` tasks precedence over the annotation if both are supplied. From 7b940026bef96a78d37a841671cf2b6f69550145 Mon Sep 17 00:00:00 2001 From: Arnaud Lefray Date: Tue, 9 May 2023 08:46:56 +0200 Subject: [PATCH 30/32] docs(aws): remove mention of deprecated annotation filter --- docs/tutorials/aws.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/tutorials/aws.md b/docs/tutorials/aws.md index e749c16d7..f77f8017f 100644 --- a/docs/tutorials/aws.md +++ b/docs/tutorials/aws.md @@ -936,7 +936,6 @@ Running several fast polling ExternalDNS instances in a given account can easily * `--namespace=my-app` * `--label-filter=app in (my-app)` * `--ingress-class=nginx-external` - * `--annotation-filter=kubernetes.io/ingress.class in (nginx-external)` - note that this filter would apply to services too.. (deprecated in favor of `--ingress-class`) * Limit services watched by type (not applicable to ingress or other types) * `--service-type-filter=LoadBalancer` default `all` * Limit the hosted zones considered From 4745ddbb0eca0fc9d1571bb77c7aec148d662c99 Mon Sep 17 00:00:00 2001 From: John Gardiner Myers Date: Tue, 9 May 2023 19:42:56 -0700 Subject: [PATCH 31/32] Address review comment --- source/node.go | 9 ++------- source/pod.go | 9 ++------- source/source.go | 6 ++++++ 3 files changed, 10 insertions(+), 14 deletions(-) diff --git a/source/node.go b/source/node.go index 39135ceaf..5e287e9a0 100644 --- a/source/node.go +++ b/source/node.go @@ -76,11 +76,6 @@ func NewNodeSource(ctx context.Context, kubeClient kubernetes.Interface, annotat }, nil } -type endpointsKey struct { - dnsName string - recordType string -} - // Endpoints returns endpoint objects for each service that should be processed. func (ns *nodeSource) Endpoints(ctx context.Context) ([]*endpoint.Endpoint, error) { nodes, err := ns.nodeInformer.Lister().List(labels.Everything()) @@ -93,7 +88,7 @@ func (ns *nodeSource) Endpoints(ctx context.Context) ([]*endpoint.Endpoint, erro return nil, err } - endpoints := map[endpointsKey]*endpoint.Endpoint{} + endpoints := map[endpointKey]*endpoint.Endpoint{} // create endpoints for all nodes for _, node := range nodes { @@ -141,7 +136,7 @@ func (ns *nodeSource) Endpoints(ctx context.Context) ([]*endpoint.Endpoint, erro ep.Labels = endpoint.NewLabels() for _, addr := range addrs { log.Debugf("adding endpoint %s target %s", ep, addr) - key := endpointsKey{ + key := endpointKey{ dnsName: ep.DNSName, recordType: suitableType(addr), } diff --git a/source/pod.go b/source/pod.go index 87772d75b..123468539 100644 --- a/source/pod.go +++ b/source/pod.go @@ -76,11 +76,6 @@ func NewPodSource(ctx context.Context, kubeClient kubernetes.Interface, namespac func (*podSource) AddEventHandler(ctx context.Context, handler func()) { } -type endpointKey struct { - domain string - recordType string -} - func (ps *podSource) Endpoints(ctx context.Context) ([]*endpoint.Endpoint, error) { pods, err := ps.podInformer.Lister().Pods(ps.namespace).List(labels.Everything()) if err != nil { @@ -128,14 +123,14 @@ func (ps *podSource) Endpoints(ctx context.Context) ([]*endpoint.Endpoint, error } endpoints := []*endpoint.Endpoint{} for key, targets := range endpointMap { - endpoints = append(endpoints, endpoint.NewEndpoint(key.domain, key.recordType, targets...)) + endpoints = append(endpoints, endpoint.NewEndpoint(key.dnsName, key.recordType, targets...)) } return endpoints, nil } func addToEndpointMap(endpointMap map[endpointKey][]string, domain string, recordType string, address string) { key := endpointKey{ - domain: domain, + dnsName: domain, recordType: recordType, } if _, ok := endpointMap[key]; !ok { diff --git a/source/source.go b/source/source.go index 8573772c3..91b83bb4f 100644 --- a/source/source.go +++ b/source/source.go @@ -86,6 +86,12 @@ type Source interface { AddEventHandler(context.Context, func()) } +// endpointKey is the type of a map key for separating endpoints or targets. +type endpointKey struct { + dnsName string + recordType string +} + func getTTLFromAnnotations(annotations map[string]string) (endpoint.TTL, error) { ttlNotConfigured := endpoint.TTL(0) ttlAnnotation, exists := annotations[ttlAnnotationKey] From e03df8b50497a6977dd360dd49878a5c981bb9bf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 10 May 2023 04:00:34 +0000 Subject: [PATCH 32/32] build(deps): bump alpine from 3.17 to 3.18 Bumps alpine from 3.17 to 3.18. --- updated-dependencies: - dependency-name: alpine dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 55b72749d..8590ad633 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,7 +25,7 @@ RUN go mod download COPY . . -FROM alpine:3.17 +FROM alpine:3.18 RUN apk update && apk add "libcrypto3>=3.0.8-r1" "libssl3>=3.0.8-r1" && rm -rf /var/cache/apt/*