Merge pull request #2153 from RentTheRunway/falconertc/clouddns_visibility_filter

CloudDNS: Allow filtering for private and public zones

README.md

@@ -139,7 +139,7 @@ The following tutorials are provided:
 * [Dyn](docs/tutorials/dyn.md)
 * [Exoscale](docs/tutorials/exoscale.md)
 * [ExternalName Services](docs/tutorials/externalname.md)
-* Google Container Engine
+* Google Kubernetes Engine
 	* [Using Google's Default Ingress Controller](docs/tutorials/gke.md)
 	* [Using the Nginx Ingress Controller](docs/tutorials/nginx-ingress.md)
 * [Headless Services](docs/tutorials/hostport.md)

docs/faq.md

@@ -2,7 +2,7 @@
 
 ### How is ExternalDNS useful to me?
 
-You've probably created many deployments. Typically, you expose your deployment to the Internet by creating a Service with `type=LoadBalancer`. Depending on your environment, this usually assigns a random publicly available endpoint to your service that you can access from anywhere in the world. On Google Container Engine, this is a public IP address:
+You've probably created many deployments. Typically, you expose your deployment to the Internet by creating a Service with `type=LoadBalancer`. Depending on your environment, this usually assigns a random publicly available endpoint to your service that you can access from anywhere in the world. On Google Kubernetes Engine, this is a public IP address:
 
 ```console
 $ kubectl get svc
@@ -54,7 +54,7 @@ Yes, you can. Pass in a comma separated list to `--fqdn-template`. Beaware this
 
 ### Which Service and Ingress controllers are supported?
 
-Regarding Services, we'll support the OSI Layer 4 load balancers that Kubernetes creates on AWS and Google Container Engine, and possibly other clusters running on Google Compute Engine.
+Regarding Services, we'll support the OSI Layer 4 load balancers that Kubernetes creates on AWS and Google Kubernetes Engine, and possibly other clusters running on Google Compute Engine.
 
 Regarding Ingress, we'll support:
 * Google's Ingress Controller on GKE that integrates with their Layer 7 load balancers (GLBC)

docs/tutorials/gke.md

@@ -1,4 +1,4 @@
-# Setting up ExternalDNS on Google Container Engine
+# Setting up ExternalDNS on Google Kubernetes Engine
 
 This tutorial describes how to setup ExternalDNS for usage within a GKE cluster. Make sure to use **>=0.4** version of ExternalDNS for this tutorial
 
@@ -123,6 +123,7 @@ spec:
         - --domain-filter=external-dns-test.gcp.zalan.do # will make ExternalDNS see only the hosted zones matching provided domain, omit to process all available hosted zones
         - --provider=google
 #        - --google-project=zalando-external-dns-test # Use this to specify a project different from the one external-dns is running inside
+        - --google-zone-visibility=private # Use this to filter to only zones with this visibility. Set to either 'public' or 'private'. Omitting will match public and private zones
         - --policy=upsert-only # would prevent ExternalDNS from deleting any records, omit to enable full synchronization
         - --registry=txt
         - --txt-owner-id=my-identifier

main.go

@@ -218,7 +218,7 @@ func main() {
 	case "rcodezero":
 		p, err = rcode0.NewRcodeZeroProvider(domainFilter, cfg.DryRun, cfg.RcodezeroTXTEncrypt)
 	case "google":
-		p, err = google.NewGoogleProvider(ctx, cfg.GoogleProject, domainFilter, zoneIDFilter, cfg.GoogleBatchChangeSize, cfg.GoogleBatchChangeInterval, cfg.DryRun)
+		p, err = google.NewGoogleProvider(ctx, cfg.GoogleProject, domainFilter, zoneIDFilter, cfg.GoogleBatchChangeSize, cfg.GoogleBatchChangeInterval, cfg.GoogleZoneVisibility, cfg.DryRun)
 	case "digitalocean":
 		p, err = digitalocean.NewDigitalOceanProvider(ctx, domainFilter, cfg.DryRun, cfg.DigitalOceanAPIPageSize)
 	case "hetzner":

pkg/apis/externaldns/types.go

@@ -67,6 +67,7 @@ type Config struct {
 	GoogleProject                     string
 	GoogleBatchChangeSize             int
 	GoogleBatchChangeInterval         time.Duration
+	GoogleZoneVisibility              string
 	DomainFilter                      []string
 	ExcludeDomains                    []string
 	RegexDomainFilter                 *regexp.Regexp
@@ -198,6 +199,7 @@ var defaultConfig = &Config{
 	GoogleProject:               "",
 	GoogleBatchChangeSize:       1000,
 	GoogleBatchChangeInterval:   time.Second,
+	GoogleZoneVisibility:        "",
 	DomainFilter:                []string{},
 	ExcludeDomains:              []string{},
 	RegexDomainFilter:           regexp.MustCompile(""),
@@ -387,6 +389,7 @@ func (cfg *Config) ParseFlags(args []string) error {
 	app.Flag("google-project", "When using the Google provider, current project is auto-detected, when running on GCP. Specify other project with this. Must be specified when running outside GCP.").Default(defaultConfig.GoogleProject).StringVar(&cfg.GoogleProject)
 	app.Flag("google-batch-change-size", "When using the Google provider, set the maximum number of changes that will be applied in each batch.").Default(strconv.Itoa(defaultConfig.GoogleBatchChangeSize)).IntVar(&cfg.GoogleBatchChangeSize)
 	app.Flag("google-batch-change-interval", "When using the Google provider, set the interval between batch changes.").Default(defaultConfig.GoogleBatchChangeInterval.String()).DurationVar(&cfg.GoogleBatchChangeInterval)
+	app.Flag("google-zone-visibility", "When using the Google provider, filter for zones with this visibility (optional, options: public, private)").Default(defaultConfig.GoogleZoneVisibility).EnumVar(&cfg.GoogleZoneVisibility, "", "public", "private")
 	app.Flag("alibaba-cloud-config-file", "When using the Alibaba Cloud provider, specify the Alibaba Cloud configuration file (required when --provider=alibabacloud").Default(defaultConfig.AlibabaCloudConfigFile).StringVar(&cfg.AlibabaCloudConfigFile)
 	app.Flag("alibaba-cloud-zone-type", "When using the Alibaba Cloud provider, filter for zones of this type (optional, options: public, private)").Default(defaultConfig.AlibabaCloudZoneType).EnumVar(&cfg.AlibabaCloudZoneType, "", "public", "private")
 	app.Flag("aws-zone-type", "When using the AWS provider, filter for zones of this type (optional, options: public, private)").Default(defaultConfig.AWSZoneType).EnumVar(&cfg.AWSZoneType, "", "public", "private")

pkg/apis/externaldns/types_test.go

@@ -46,6 +46,7 @@ var (
 		GoogleProject:               "",
 		GoogleBatchChangeSize:       1000,
 		GoogleBatchChangeInterval:   time.Second,
+		GoogleZoneVisibility:        "",
 		DomainFilter:                []string{""},
 		ExcludeDomains:              []string{""},
 		RegexDomainFilter:           regexp.MustCompile(""),
@@ -134,6 +135,7 @@ var (
 		GoogleProject:               "project",
 		GoogleBatchChangeSize:       100,
 		GoogleBatchChangeInterval:   time.Second * 2,
+		GoogleZoneVisibility:        "private",
 		DomainFilter:                []string{"example.org", "company.com"},
 		ExcludeDomains:              []string{"xapi.example.org", "xapi.company.com"},
 		RegexDomainFilter:           regexp.MustCompile("(example\\.org|company\\.com)$"),
@@ -249,6 +251,7 @@ func TestParseFlags(t *testing.T) {
 				"--google-project=project",
 				"--google-batch-change-size=100",
 				"--google-batch-change-interval=2s",
+				"--google-zone-visibility=private",
 				"--azure-config-file=azure.json",
 				"--azure-resource-group=arg",
 				"--azure-subscription-id=arg",
@@ -351,6 +354,7 @@ func TestParseFlags(t *testing.T) {
 				"EXTERNAL_DNS_GOOGLE_PROJECT":                  "project",
 				"EXTERNAL_DNS_GOOGLE_BATCH_CHANGE_SIZE":        "100",
 				"EXTERNAL_DNS_GOOGLE_BATCH_CHANGE_INTERVAL":    "2s",
+				"EXTERNAL_DNS_GOOGLE_ZONE_VISIBILITY":          "private",
 				"EXTERNAL_DNS_AZURE_CONFIG_FILE":               "azure.json",
 				"EXTERNAL_DNS_AZURE_RESOURCE_GROUP":            "arg",
 				"EXTERNAL_DNS_AZURE_SUBSCRIPTION_ID":           "arg",

provider/google/google.go

@@ -110,6 +110,8 @@ type GoogleProvider struct {
 	batchChangeInterval time.Duration
 	// only consider hosted zones managing domains ending in this suffix
 	domainFilter endpoint.DomainFilter
+	// filter for zones based on visibility
+	zoneTypeFilter provider.ZoneTypeFilter
 	// only consider hosted zones ending with this zone id
 	zoneIDFilter provider.ZoneIDFilter
 	// A client for managing resource record sets
@@ -123,7 +125,7 @@ type GoogleProvider struct {
 }
 
 // NewGoogleProvider initializes a new Google CloudDNS based Provider.
-func NewGoogleProvider(ctx context.Context, project string, domainFilter endpoint.DomainFilter, zoneIDFilter provider.ZoneIDFilter, batchChangeSize int, batchChangeInterval time.Duration, dryRun bool) (*GoogleProvider, error) {
+func NewGoogleProvider(ctx context.Context, project string, domainFilter endpoint.DomainFilter, zoneIDFilter provider.ZoneIDFilter, batchChangeSize int, batchChangeInterval time.Duration, zoneVisibility string, dryRun bool) (*GoogleProvider, error) {
 	gcloud, err := google.DefaultClient(ctx, dns.NdevClouddnsReadwriteScope)
 	if err != nil {
 		return nil, err
@@ -149,12 +151,15 @@ func NewGoogleProvider(ctx context.Context, project string, domainFilter endpoin
 		}
 	}
 
+	zoneTypeFilter := provider.NewZoneTypeFilter(zoneVisibility)
+
 	provider := &GoogleProvider{
 		project:                  project,
 		dryRun:                   dryRun,
 		batchChangeSize:          batchChangeSize,
 		batchChangeInterval:      batchChangeInterval,
 		domainFilter:             domainFilter,
+		zoneTypeFilter:           zoneTypeFilter,
 		zoneIDFilter:             zoneIDFilter,
 		resourceRecordSetsClient: resourceRecordSetsService{dnsClient.ResourceRecordSets},
 		managedZonesClient:       managedZonesService{dnsClient.ManagedZones},
@@ -171,11 +176,11 @@ func (p *GoogleProvider) Zones(ctx context.Context) (map[string]*dns.ManagedZone
 
 	f := func(resp *dns.ManagedZonesListResponse) error {
 		for _, zone := range resp.ManagedZones {
-			if p.domainFilter.Match(zone.DnsName) && (p.zoneIDFilter.Match(fmt.Sprintf("%v", zone.Id)) || p.zoneIDFilter.Match(fmt.Sprintf("%v", zone.Name))) {
+			if p.domainFilter.Match(zone.DnsName) && p.zoneTypeFilter.Match(zone.Visibility) && (p.zoneIDFilter.Match(fmt.Sprintf("%v", zone.Id)) || p.zoneIDFilter.Match(fmt.Sprintf("%v", zone.Name))) {
 				zones[zone.Name] = zone
-				log.Debugf("Matched %s (zone: %s)", zone.DnsName, zone.Name)
+				log.Debugf("Matched %s (zone: %s) (visibility: %s)", zone.DnsName, zone.Name, zone.Visibility)
 			} else {
-				log.Debugf("Filtered %s (zone: %s)", zone.DnsName, zone.Name)
+				log.Debugf("Filtered %s (zone: %s) (visibility: %s)", zone.DnsName, zone.Name, zone.Visibility)
 			}
 		}
 

provider/google/google_test.go

@@ -194,24 +194,46 @@ func hasTrailingDot(target string) bool {
 }
 
 func TestGoogleZonesIDFilter(t *testing.T) {
-	provider := newGoogleProviderZoneOverlap(t, endpoint.NewDomainFilter([]string{"cluster.local."}), provider.NewZoneIDFilter([]string{"10002"}), false, []*endpoint.Endpoint{})
+	provider := newGoogleProviderZoneOverlap(t, endpoint.NewDomainFilter([]string{"cluster.local."}), provider.NewZoneIDFilter([]string{"10002"}), provider.NewZoneTypeFilter(""), false, []*endpoint.Endpoint{})
 
 	zones, err := provider.Zones(context.Background())
 	require.NoError(t, err)
 
 	validateZones(t, zones, map[string]*dns.ManagedZone{
-		"internal-2": {Name: "internal-2", DnsName: "cluster.local.", Id: 10002},
+		"internal-2": {Name: "internal-2", DnsName: "cluster.local.", Id: 10002, Visibility: "private"},
 	})
 }
 
 func TestGoogleZonesNameFilter(t *testing.T) {
-	provider := newGoogleProviderZoneOverlap(t, endpoint.NewDomainFilter([]string{"cluster.local."}), provider.NewZoneIDFilter([]string{"internal-2"}), false, []*endpoint.Endpoint{})
+	provider := newGoogleProviderZoneOverlap(t, endpoint.NewDomainFilter([]string{"cluster.local."}), provider.NewZoneIDFilter([]string{"internal-2"}), provider.NewZoneTypeFilter(""), false, []*endpoint.Endpoint{})
 
 	zones, err := provider.Zones(context.Background())
 	require.NoError(t, err)
 
 	validateZones(t, zones, map[string]*dns.ManagedZone{
-		"internal-2": {Name: "internal-2", DnsName: "cluster.local.", Id: 10002},
+		"internal-2": {Name: "internal-2", DnsName: "cluster.local.", Id: 10002, Visibility: "private"},
+	})
+}
+
+func TestGoogleZonesVisibilityFilterPublic(t *testing.T) {
+	provider := newGoogleProviderZoneOverlap(t, endpoint.NewDomainFilter([]string{"cluster.local."}), provider.NewZoneIDFilter([]string{"split-horizon-1"}), provider.NewZoneTypeFilter("public"), false, []*endpoint.Endpoint{})
+
+	zones, err := provider.Zones(context.Background())
+	require.NoError(t, err)
+
+	validateZones(t, zones, map[string]*dns.ManagedZone{
+		"split-horizon-1": {Name: "split-horizon-1", DnsName: "cluster.local.", Id: 10001, Visibility: "public"},
+	})
+}
+
+func TestGoogleZonesVisibilityFilterPrivate(t *testing.T) {
+	provider := newGoogleProviderZoneOverlap(t, endpoint.NewDomainFilter([]string{"cluster.local."}), provider.NewZoneIDFilter([]string{"split-horizon-1"}), provider.NewZoneTypeFilter("public"), false, []*endpoint.Endpoint{})
+
+	zones, err := provider.Zones(context.Background())
+	require.NoError(t, err)
+
+	validateZones(t, zones, map[string]*dns.ManagedZone{
+		"split-horizon-1": {Name: "split-horizon-1", DnsName: "cluster.local.", Id: 10001, Visibility: "public"},
 	})
 }
 
@@ -650,6 +672,7 @@ func validateZones(t *testing.T, zones map[string]*dns.ManagedZone, expected map
 func validateZone(t *testing.T, zone *dns.ManagedZone, expected *dns.ManagedZone) {
 	assert.Equal(t, expected.Name, zone.Name)
 	assert.Equal(t, expected.DnsName, zone.DnsName)
+	assert.Equal(t, expected.Visibility, zone.Visibility)
 }
 
 func validateChange(t *testing.T, change *dns.Change, expected *dns.Change) {
@@ -672,33 +695,51 @@ func validateChangeRecord(t *testing.T, record *dns.ResourceRecordSet, expected
 	assert.Equal(t, expected.Type, record.Type)
 }
 
-func newGoogleProviderZoneOverlap(t *testing.T, domainFilter endpoint.DomainFilter, zoneIDFilter provider.ZoneIDFilter, dryRun bool, records []*endpoint.Endpoint) *GoogleProvider {
+func newGoogleProviderZoneOverlap(t *testing.T, domainFilter endpoint.DomainFilter, zoneIDFilter provider.ZoneIDFilter, zoneTypeFilter provider.ZoneTypeFilter, dryRun bool, records []*endpoint.Endpoint) *GoogleProvider {
 	provider := &GoogleProvider{
 		project:                  "zalando-external-dns-test",
 		dryRun:                   false,
 		domainFilter:             domainFilter,
 		zoneIDFilter:             zoneIDFilter,
+		zoneTypeFilter:           zoneTypeFilter,
 		resourceRecordSetsClient: &mockResourceRecordSetsClient{},
 		managedZonesClient:       &mockManagedZonesClient{},
 		changesClient:            &mockChangesClient{},
 	}
 
 	createZone(t, provider, &dns.ManagedZone{
-		Name:    "internal-1",
+		Name:       "internal-1",
-		DnsName: "cluster.local.",
+		DnsName:    "cluster.local.",
-		Id:      10001,
+		Id:         10001,
+		Visibility: "private",
 	})
 
 	createZone(t, provider, &dns.ManagedZone{
-		Name:    "internal-2",
+		Name:       "internal-2",
-		DnsName: "cluster.local.",
+		DnsName:    "cluster.local.",
-		Id:      10002,
+		Id:         10002,
+		Visibility: "private",
 	})
 
 	createZone(t, provider, &dns.ManagedZone{
-		Name:    "internal-3",
+		Name:       "internal-3",
-		DnsName: "cluster.local.",
+		DnsName:    "cluster.local.",
-		Id:      10003,
+		Id:         10003,
+		Visibility: "private",
+	})
+
+	createZone(t, provider, &dns.ManagedZone{
+		Name:       "split-horizon-1",
+		DnsName:    "cluster.local.",
+		Id:         10004,
+		Visibility: "public",
+	})
+
+	createZone(t, provider, &dns.ManagedZone{
+		Name:       "split-horizon-1",
+		DnsName:    "cluster.local.",
+		Id:         10004,
+		Visibility: "private",
 	})
 
 	provider.dryRun = dryRun

provider/zone_type_filter.go

@@ -37,24 +37,34 @@ func NewZoneTypeFilter(zoneType string) ZoneTypeFilter {
 }
 
 // Match checks whether a zone matches the zone type that's filtered for.
-func (f ZoneTypeFilter) Match(zone *route53.HostedZone) bool {
+func (f ZoneTypeFilter) Match(rawZoneType interface{}) bool {
 	// An empty zone filter includes all hosted zones.
 	if f.zoneType == "" {
 		return true
 	}
 
-	// If the zone has no config we assume it's a public zone since the config's field
+	switch zoneType := rawZoneType.(type) {
-	// `PrivateZone` is false by default in go.
-	if zone.Config == nil {
-		return f.zoneType == zoneTypePublic
-	}
-
 	// Given a zone type we return true if the given zone matches this type.
-	switch f.zoneType {
+	case string:
-	case zoneTypePublic:
+		switch f.zoneType {
-		return !aws.BoolValue(zone.Config.PrivateZone)
+		case zoneTypePublic:
-	case zoneTypePrivate:
+			return zoneType == zoneTypePublic
-		return aws.BoolValue(zone.Config.PrivateZone)
+		case zoneTypePrivate:
+			return zoneType == zoneTypePrivate
+		}
+	case *route53.HostedZone:
+		// If the zone has no config we assume it's a public zone since the config's field
+		// `PrivateZone` is false by default in go.
+		if zoneType.Config == nil {
+			return f.zoneType == zoneTypePublic
+		}
+
+		switch f.zoneType {
+		case zoneTypePublic:
+			return !aws.BoolValue(zoneType.Config.PrivateZone)
+		case zoneTypePrivate:
+			return aws.BoolValue(zoneType.Config.PrivateZone)
+		}
 	}
 
 	// We return false on any other path, e.g. unknown zone type filter value.

provider/zone_type_filter_test.go

@@ -26,46 +26,38 @@ import (
 )
 
 func TestZoneTypeFilterMatch(t *testing.T) {
-	publicZone := &route53.HostedZone{Config: &route53.HostedZoneConfig{PrivateZone: aws.Bool(false)}}
+	publicZoneStr := "public"
-	privateZone := &route53.HostedZone{Config: &route53.HostedZoneConfig{PrivateZone: aws.Bool(true)}}
+	privateZoneStr := "private"
+	publicZoneAWS := &route53.HostedZone{Config: &route53.HostedZoneConfig{PrivateZone: aws.Bool(false)}}
+	privateZoneAWS := &route53.HostedZone{Config: &route53.HostedZoneConfig{PrivateZone: aws.Bool(true)}}
 
 	for _, tc := range []struct {
 		zoneTypeFilter string
-		zone           *route53.HostedZone
 		matches        bool
+		zones          []interface{}
 	}{
 		{
-			"", publicZone, true,
+			"", true, []interface{}{ publicZoneStr, privateZoneStr, &route53.HostedZone{} },
 		},
 		{
-			"", privateZone, true,
+			"public", true, []interface{}{ publicZoneStr, publicZoneAWS, &route53.HostedZone{} },
 		},
 		{
-			"public", publicZone, true,
+			"public", false, []interface{}{ privateZoneStr, privateZoneAWS },
 		},
 		{
-			"public", privateZone, false,
+			"private", true, []interface{}{ privateZoneStr, privateZoneAWS },
 		},
 		{
-			"private", publicZone, false,
+			"private", false, []interface{}{ publicZoneStr, publicZoneAWS, &route53.HostedZone{} },
 		},
 		{
-			"private", privateZone, true,
+			"unknown", false, []interface{}{ publicZoneStr },
-		},
-		{
-			"unknown", publicZone, false,
-		},
-		{
-			"", &route53.HostedZone{}, true,
-		},
-		{
-			"public", &route53.HostedZone{}, true,
-		},
-		{
-			"private", &route53.HostedZone{}, false,
 		},
 	} {
 		zoneTypeFilter := NewZoneTypeFilter(tc.zoneTypeFilter)
-		assert.Equal(t, tc.matches, zoneTypeFilter.Match(tc.zone))
+		for _, zone := range tc.zones {
+			assert.Equal(t, tc.matches, zoneTypeFilter.Match(zone))
+		}
 	}
 }