aws: cache zones list

When it syncs AWS DNS with k8s cluster content (at `--interval`), external-dns submits two distinct Route53 API calls:
* to fetch available zones (eg. for tag based zones discovery, or when zones are created after exernal-dns started),
* to fetch relevant zones' resource records.

Each call taxes the Route53 APIs calls budget (5 API calls per second per AWS account/region hard limit), increasing the probability of being throttled.
Changing synchronization interval would mitigate those calls' impact, but at the cost of keeping stale records for a longer time.

For most practical uses cases, zones list aren't expected to change frequently.
Even less so when external-dns is provided an explicit, static zones set (`--zone-id-filter` rather than `--aws-zone-tags`).

Using a zones list cache halves the number of Route53 read API calls.

CHANGELOG.md

@@ -15,6 +15,7 @@
 - Add tutorial for GKE with workload identity (#1765) @ddgenome
 - Fix NodePort with externaltrafficpolicy targets duplication @codearky
 - Update contributing section in README (#1760) @seanmalloy
+- Option to cache AWS zones list @bpineau
 
 ## v0.7.3 - 2020-08-05
 

docs/tutorials/aws.md

@@ -420,3 +420,10 @@ Give ExternalDNS some time to clean up the DNS records for you. Then delete the
 ```console
 $ aws route53 delete-hosted-zone --id /hostedzone/ZEWFWZ4R16P7IB
 ```
+
+## Throttling
+
+Route53 has a [5 API requests per second per account hard quota](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/DNSLimitations.html#limits-api-requests-route-53).
+Running several fast polling ExternalDNS instances in a given account can easily hit that limit. Some ways to circumvent that issue includes:
+* Augment the synchronization interval (`--interval`), at the cost of slower changes propagation.
+* If the ExternalDNS managed zones list doesn't change frequently, set `--aws-zones-cache-duration` (zones list cache time-to-live) to a larger value. Note that zones list cache can be disabled with `--aws-zones-cache-duration=0s`.

main.go

@@ -175,6 +175,7 @@ func main() {
 				APIRetries:           cfg.AWSAPIRetries,
 				PreferCNAME:          cfg.AWSPreferCNAME,
 				DryRun:               cfg.DryRun,
+				ZoneCacheDuration:    cfg.AWSZoneCacheDuration,
 			},
 		)
 	case "aws-sd":

pkg/apis/externaldns/types.go

@@ -72,6 +72,7 @@ type Config struct {
 	AWSEvaluateTargetHealth           bool
 	AWSAPIRetries                     int
 	AWSPreferCNAME                    bool
+	AWSZoneCacheDuration              time.Duration
 	AzureConfigFile                   string
 	AzureResourceGroup                string
 	AzureSubscriptionID               string
@@ -176,6 +177,7 @@ var defaultConfig = &Config{
 	AWSEvaluateTargetHealth:     true,
 	AWSAPIRetries:               3,
 	AWSPreferCNAME:              false,
+	AWSZoneCacheDuration:        0 * time.Second,
 	AzureConfigFile:             "/etc/kubernetes/azure.json",
 	AzureResourceGroup:          "",
 	AzureSubscriptionID:         "",
@@ -335,6 +337,7 @@ func (cfg *Config) ParseFlags(args []string) error {
 	app.Flag("aws-evaluate-target-health", "When using the AWS provider, set whether to evaluate the health of a DNS target (default: enabled, disable with --no-aws-evaluate-target-health)").Default(strconv.FormatBool(defaultConfig.AWSEvaluateTargetHealth)).BoolVar(&cfg.AWSEvaluateTargetHealth)
 	app.Flag("aws-api-retries", "When using the AWS provider, set the maximum number of retries for API calls before giving up.").Default(strconv.Itoa(defaultConfig.AWSAPIRetries)).IntVar(&cfg.AWSAPIRetries)
 	app.Flag("aws-prefer-cname", "When using the AWS provider, prefer using CNAME instead of ALIAS (default: disabled)").BoolVar(&cfg.AWSPreferCNAME)
+	app.Flag("aws-zones-cache-duration", "When using the AWS provider, set the zones list cache TTL (0s to disable).").Default(defaultConfig.AWSZoneCacheDuration.String()).DurationVar(&cfg.AWSZoneCacheDuration)
 	app.Flag("azure-config-file", "When using the Azure provider, specify the Azure configuration file (required when --provider=azure").Default(defaultConfig.AzureConfigFile).StringVar(&cfg.AzureConfigFile)
 	app.Flag("azure-resource-group", "When using the Azure provider, override the Azure resource group to use (required when --provider=azure-private-dns)").Default(defaultConfig.AzureResourceGroup).StringVar(&cfg.AzureResourceGroup)
 	app.Flag("azure-subscription-id", "When using the Azure provider, specify the Azure configuration file (required when --provider=azure-private-dns)").Default(defaultConfig.AzureSubscriptionID).StringVar(&cfg.AzureSubscriptionID)

pkg/apis/externaldns/types_test.go

@@ -54,6 +54,7 @@ var (
 		AWSEvaluateTargetHealth:     true,
 		AWSAPIRetries:               3,
 		AWSPreferCNAME:              false,
+		AWSZoneCacheDuration:        0 * time.Second,
 		AzureConfigFile:             "/etc/kubernetes/azure.json",
 		AzureResourceGroup:          "",
 		AzureSubscriptionID:         "",
@@ -129,6 +130,7 @@ var (
 		AWSEvaluateTargetHealth:     false,
 		AWSAPIRetries:               13,
 		AWSPreferCNAME:              true,
+		AWSZoneCacheDuration:        10 * time.Second,
 		AzureConfigFile:             "azure.json",
 		AzureResourceGroup:          "arg",
 		AzureSubscriptionID:         "arg",
@@ -261,6 +263,7 @@ func TestParseFlags(t *testing.T) {
 				"--aws-batch-change-interval=2s",
 				"--aws-api-retries=13",
 				"--aws-prefer-cname",
+				"--aws-zones-cache-duration=10s",
 				"--no-aws-evaluate-target-health",
 				"--policy=upsert-only",
 				"--registry=noop",
@@ -348,6 +351,7 @@ func TestParseFlags(t *testing.T) {
 				"EXTERNAL_DNS_AWS_EVALUATE_TARGET_HEALTH":      "0",
 				"EXTERNAL_DNS_AWS_API_RETRIES":                 "13",
 				"EXTERNAL_DNS_AWS_PREFER_CNAME":                "true",
+				"EXTERNAL_DNS_AWS_ZONES_CACHE_DURATION":        "10s",
 				"EXTERNAL_DNS_POLICY":                          "upsert-only",
 				"EXTERNAL_DNS_REGISTRY":                        "noop",
 				"EXTERNAL_DNS_TXT_OWNER_ID":                    "owner-1",

provider/aws/aws.go

@@ -117,6 +117,12 @@ type Route53API interface {
 	ListTagsForResourceWithContext(ctx context.Context, input *route53.ListTagsForResourceInput, opts ...request.Option) (*route53.ListTagsForResourceOutput, error)
 }
 
+type zonesListCache struct {
+	age      time.Time
+	duration time.Duration
+	zones    map[string]*route53.HostedZone
+}
+
 // AWSProvider is an implementation of Provider for AWS Route53.
 type AWSProvider struct {
 	provider.BaseProvider
@@ -134,6 +140,7 @@ type AWSProvider struct {
 	// filter hosted zones by tags
 	zoneTagFilter provider.ZoneTagFilter
 	preferCNAME   bool
+	zonesCache    *zonesListCache
 }
 
 // AWSConfig contains configuration to create a new AWS provider.
@@ -149,6 +156,7 @@ type AWSConfig struct {
 	APIRetries           int
 	PreferCNAME          bool
 	DryRun               bool
+	ZoneCacheDuration    time.Duration
 }
 
 // NewAWSProvider initializes a new AWS Route53 based Provider.
@@ -188,6 +196,7 @@ func NewAWSProvider(awsConfig AWSConfig) (*AWSProvider, error) {
 		evaluateTargetHealth: awsConfig.EvaluateTargetHealth,
 		preferCNAME:          awsConfig.PreferCNAME,
 		dryRun:               awsConfig.DryRun,
+		zonesCache:           &zonesListCache{duration: awsConfig.ZoneCacheDuration},
 	}
 
 	return provider, nil
@@ -195,6 +204,12 @@ func NewAWSProvider(awsConfig AWSConfig) (*AWSProvider, error) {
 
 // Zones returns the list of hosted zones.
 func (p *AWSProvider) Zones(ctx context.Context) (map[string]*route53.HostedZone, error) {
+	if p.zonesCache.zones != nil && time.Since(p.zonesCache.age) < p.zonesCache.duration {
+		log.Debug("Using cached zones list")
+		return p.zonesCache.zones, nil
+	}
+	log.Debug("Refreshing zones list cache")
+
 	zones := make(map[string]*route53.HostedZone)
 
 	var tagErr error
@@ -242,6 +257,11 @@ func (p *AWSProvider) Zones(ctx context.Context) (map[string]*route53.HostedZone
 		log.Debugf("Considering zone: %s (domain: %s)", aws.StringValue(zone.Id), aws.StringValue(zone.Name))
 	}
 
+	if p.zonesCache.duration > time.Duration(0) {
+		p.zonesCache.zones = zones
+		p.zonesCache.age = time.Now()
+	}
+
 	return zones, nil
 }
 

provider/aws/aws_test.go

@@ -500,6 +500,7 @@ func TestAWSApplyChanges(t *testing.T) {
 
 		ctx := tt.setup(provider)
 
+		provider.zonesCache = &zonesListCache{duration: 0 * time.Minute}
 		counter := NewRoute53APICounter(provider.client)
 		provider.client = counter
 		require.NoError(t, provider.ApplyChanges(ctx, changes))
@@ -1200,6 +1201,7 @@ func newAWSProviderWithTagFilter(t *testing.T, domainFilter endpoint.DomainFilte
 		zoneTypeFilter:       zoneTypeFilter,
 		zoneTagFilter:        zoneTagFilter,
 		dryRun:               false,
+		zonesCache:           &zonesListCache{duration: 1 * time.Minute},
 	}
 
 	createAWSZone(t, provider, &route53.HostedZone{