diff --git a/main.go b/main.go index 1f08a82c1..3bbb9ad34 100644 --- a/main.go +++ b/main.go @@ -142,7 +142,7 @@ func main() { case "rcodezero": p, err = provider.NewRcodeZeroProvider(domainFilter, cfg.DryRun, cfg.RcodezeroTXTEncrypt) case "google": - p, err = provider.NewGoogleProvider(cfg.GoogleProject, domainFilter, zoneIDFilter, cfg.GoogleBatchChangeSize, cfg.DryRun) + p, err = provider.NewGoogleProvider(cfg.GoogleProject, domainFilter, zoneIDFilter, cfg.GoogleBatchChangeSize, cfg.GoogleBatchChangeInterval, cfg.DryRun) case "digitalocean": p, err = provider.NewDigitalOceanProvider(domainFilter, cfg.DryRun) case "linode": diff --git a/pkg/apis/externaldns/types.go b/pkg/apis/externaldns/types.go index 0f976aa9d..e89bb40b3 100644 --- a/pkg/apis/externaldns/types.go +++ b/pkg/apis/externaldns/types.go @@ -55,6 +55,7 @@ type Config struct { Provider string GoogleProject string GoogleBatchChangeSize int + GoogleBatchChangeInterval time.Duration DomainFilter []string ExcludeDomains []string ZoneIDFilter []string @@ -147,6 +148,7 @@ var defaultConfig = &Config{ Provider: "", GoogleProject: "", GoogleBatchChangeSize: 1000, + GoogleBatchChangeInterval: time.Second, DomainFilter: []string{}, ExcludeDomains: []string{}, AlibabaCloudConfigFile: "/etc/kubernetes/alibaba-cloud.json", @@ -293,6 +295,7 @@ func (cfg *Config) ParseFlags(args []string) error { app.Flag("zone-id-filter", "Filter target zones by hosted zone id; specify multiple times for multiple zones (optional)").Default("").StringsVar(&cfg.ZoneIDFilter) app.Flag("google-project", "When using the Google provider, current project is auto-detected, when running on GCP. Specify other project with this. Must be specified when running outside GCP.").Default(defaultConfig.GoogleProject).StringVar(&cfg.GoogleProject) app.Flag("google-batch-change-size", "When using the Google provider, set the maximum number of changes that will be applied in each batch.").Default(strconv.Itoa(defaultConfig.GoogleBatchChangeSize)).IntVar(&cfg.GoogleBatchChangeSize) + app.Flag("google-batch-change-interval", "When using the Google provider, set the interval between batch changes.").Default(defaultConfig.GoogleBatchChangeInterval.String()).DurationVar(&cfg.GoogleBatchChangeInterval) app.Flag("alibaba-cloud-config-file", "When using the Alibaba Cloud provider, specify the Alibaba Cloud configuration file (required when --provider=alibabacloud").Default(defaultConfig.AlibabaCloudConfigFile).StringVar(&cfg.AlibabaCloudConfigFile) app.Flag("alibaba-cloud-zone-type", "When using the Alibaba Cloud provider, filter for zones of this type (optional, options: public, private)").Default(defaultConfig.AlibabaCloudZoneType).EnumVar(&cfg.AlibabaCloudZoneType, "", "public", "private") app.Flag("aws-zone-type", "When using the AWS provider, filter for zones of this type (optional, options: public, private)").Default(defaultConfig.AWSZoneType).EnumVar(&cfg.AWSZoneType, "", "public", "private") diff --git a/pkg/apis/externaldns/types_test.go b/pkg/apis/externaldns/types_test.go index 19d9f4509..3134b5194 100644 --- a/pkg/apis/externaldns/types_test.go +++ b/pkg/apis/externaldns/types_test.go @@ -41,6 +41,7 @@ var ( Provider: "google", GoogleProject: "", GoogleBatchChangeSize: 1000, + GoogleBatchChangeInterval: time.Second, DomainFilter: []string{""}, ExcludeDomains: []string{""}, ZoneIDFilter: []string{""}, @@ -106,6 +107,7 @@ var ( Provider: "google", GoogleProject: "project", GoogleBatchChangeSize: 100, + GoogleBatchChangeInterval: time.Second * 2, DomainFilter: []string{"example.org", "company.com"}, ExcludeDomains: []string{"xapi.example.org", "xapi.company.com"}, ZoneIDFilter: []string{"/hostedzone/ZTST1", "/hostedzone/ZTST2"}, @@ -177,6 +179,7 @@ var ( Provider: "google", GoogleProject: "", GoogleBatchChangeSize: 1000, + GoogleBatchChangeInterval: time.Second, DomainFilter: []string{""}, ExcludeDomains: []string{""}, ZoneIDFilter: []string{""}, @@ -260,6 +263,7 @@ func TestParseFlags(t *testing.T) { "--provider=google", "--google-project=project", "--google-batch-change-size=100", + "--google-batch-change-interval=2s", "--azure-config-file=azure.json", "--azure-resource-group=arg", "--cloudflare-proxied", @@ -326,74 +330,75 @@ func TestParseFlags(t *testing.T) { title: "override everything via environment variables", args: []string{}, envVars: map[string]string{ - "EXTERNAL_DNS_MASTER": "http://127.0.0.1:8080", - "EXTERNAL_DNS_KUBECONFIG": "/some/path", - "EXTERNAL_DNS_REQUEST_TIMEOUT": "77s", - "EXTERNAL_DNS_ISTIO_INGRESS_GATEWAY": "istio-other/istio-otheringressgateway", - "EXTERNAL_DNS_CONTOUR_LOAD_BALANCER": "heptio-contour-other/contour-other", - "EXTERNAL_DNS_SOURCE": "service\ningress\nconnector", - "EXTERNAL_DNS_NAMESPACE": "namespace", - "EXTERNAL_DNS_FQDN_TEMPLATE": "{{.Name}}.service.example.com", - "EXTERNAL_DNS_IGNORE_HOSTNAME_ANNOTATION": "1", - "EXTERNAL_DNS_COMPATIBILITY": "mate", - "EXTERNAL_DNS_PROVIDER": "google", - "EXTERNAL_DNS_GOOGLE_PROJECT": "project", - "EXTERNAL_DNS_GOOGLE_BATCH_CHANGE_SIZE": "100", - "EXTERNAL_DNS_AZURE_CONFIG_FILE": "azure.json", - "EXTERNAL_DNS_AZURE_RESOURCE_GROUP": "arg", - "EXTERNAL_DNS_CLOUDFLARE_PROXIED": "1", - "EXTERNAL_DNS_CLOUDFLARE_ZONES_PER_PAGE": "20", - "EXTERNAL_DNS_COREDNS_PREFIX": "/coredns/", - "EXTERNAL_DNS_INFOBLOX_GRID_HOST": "127.0.0.1", - "EXTERNAL_DNS_INFOBLOX_WAPI_PORT": "8443", - "EXTERNAL_DNS_INFOBLOX_WAPI_USERNAME": "infoblox", - "EXTERNAL_DNS_INFOBLOX_WAPI_PASSWORD": "infoblox", - "EXTERNAL_DNS_INFOBLOX_WAPI_VERSION": "2.6.1", - "EXTERNAL_DNS_INFOBLOX_VIEW": "internal", - "EXTERNAL_DNS_INFOBLOX_SSL_VERIFY": "0", - "EXTERNAL_DNS_INFOBLOX_MAX_RESULTS": "2000", - "EXTERNAL_DNS_OCI_CONFIG_FILE": "oci.yaml", - "EXTERNAL_DNS_INMEMORY_ZONE": "example.org\ncompany.com", - "EXTERNAL_DNS_DOMAIN_FILTER": "example.org\ncompany.com", - "EXTERNAL_DNS_EXCLUDE_DOMAINS": "xapi.example.org\nxapi.company.com", - "EXTERNAL_DNS_PDNS_SERVER": "http://ns.example.com:8081", - "EXTERNAL_DNS_PDNS_API_KEY": "some-secret-key", - "EXTERNAL_DNS_PDNS_TLS_ENABLED": "1", - "EXTERNAL_DNS_RDNS_ROOT_DOMAIN": "lb.rancher.cloud", - "EXTERNAL_DNS_TLS_CA": "/path/to/ca.crt", - "EXTERNAL_DNS_TLS_CLIENT_CERT": "/path/to/cert.pem", - "EXTERNAL_DNS_TLS_CLIENT_CERT_KEY": "/path/to/key.pem", - "EXTERNAL_DNS_ZONE_ID_FILTER": "/hostedzone/ZTST1\n/hostedzone/ZTST2", - "EXTERNAL_DNS_AWS_ZONE_TYPE": "private", - "EXTERNAL_DNS_AWS_ZONE_TAGS": "tag=foo", - "EXTERNAL_DNS_AWS_ASSUME_ROLE": "some-other-role", - "EXTERNAL_DNS_AWS_BATCH_CHANGE_SIZE": "100", - "EXTERNAL_DNS_AWS_BATCH_CHANGE_INTERVAL": "2s", - "EXTERNAL_DNS_AWS_EVALUATE_TARGET_HEALTH": "0", - "EXTERNAL_DNS_AWS_API_RETRIES": "13", - "EXTERNAL_DNS_AWS_PREFER_CNAME": "true", - "EXTERNAL_DNS_POLICY": "upsert-only", - "EXTERNAL_DNS_REGISTRY": "noop", - "EXTERNAL_DNS_TXT_OWNER_ID": "owner-1", - "EXTERNAL_DNS_TXT_PREFIX": "associated-txt-record", - "EXTERNAL_DNS_TXT_CACHE_INTERVAL": "12h", - "EXTERNAL_DNS_INTERVAL": "10m", - "EXTERNAL_DNS_ONCE": "1", - "EXTERNAL_DNS_DRY_RUN": "1", - "EXTERNAL_DNS_LOG_FORMAT": "json", - "EXTERNAL_DNS_METRICS_ADDRESS": "127.0.0.1:9099", - "EXTERNAL_DNS_LOG_LEVEL": "debug", - "EXTERNAL_DNS_CONNECTOR_SOURCE_SERVER": "localhost:8081", - "EXTERNAL_DNS_EXOSCALE_ENDPOINT": "https://api.foo.ch/dns", - "EXTERNAL_DNS_EXOSCALE_APIKEY": "1", - "EXTERNAL_DNS_EXOSCALE_APISECRET": "2", - "EXTERNAL_DNS_CRD_SOURCE_APIVERSION": "test.k8s.io/v1alpha1", - "EXTERNAL_DNS_CRD_SOURCE_KIND": "Endpoint", - "EXTERNAL_DNS_RCODEZERO_TXT_ENCRYPT": "1", - "EXTERNAL_DNS_NS1_ENDPOINT": "https://api.example.com/v1", - "EXTERNAL_DNS_NS1_IGNORESSL": "1", - "EXTERNAL_DNS_TRANSIP_ACCOUNT": "transip", - "EXTERNAL_DNS_TRANSIP_KEYFILE": "/path/to/transip.key", + "EXTERNAL_DNS_MASTER": "http://127.0.0.1:8080", + "EXTERNAL_DNS_KUBECONFIG": "/some/path", + "EXTERNAL_DNS_REQUEST_TIMEOUT": "77s", + "EXTERNAL_DNS_ISTIO_INGRESS_GATEWAY": "istio-other/istio-otheringressgateway", + "EXTERNAL_DNS_CONTOUR_LOAD_BALANCER": "heptio-contour-other/contour-other", + "EXTERNAL_DNS_SOURCE": "service\ningress\nconnector", + "EXTERNAL_DNS_NAMESPACE": "namespace", + "EXTERNAL_DNS_FQDN_TEMPLATE": "{{.Name}}.service.example.com", + "EXTERNAL_DNS_IGNORE_HOSTNAME_ANNOTATION": "1", + "EXTERNAL_DNS_COMPATIBILITY": "mate", + "EXTERNAL_DNS_PROVIDER": "google", + "EXTERNAL_DNS_GOOGLE_PROJECT": "project", + "EXTERNAL_DNS_GOOGLE_BATCH_CHANGE_SIZE": "100", + "EXTERNAL_DNS_GOOGLE_BATCH_CHANGE_INTERVAL": "2s", + "EXTERNAL_DNS_AZURE_CONFIG_FILE": "azure.json", + "EXTERNAL_DNS_AZURE_RESOURCE_GROUP": "arg", + "EXTERNAL_DNS_CLOUDFLARE_PROXIED": "1", + "EXTERNAL_DNS_CLOUDFLARE_ZONES_PER_PAGE": "20", + "EXTERNAL_DNS_COREDNS_PREFIX": "/coredns/", + "EXTERNAL_DNS_INFOBLOX_GRID_HOST": "127.0.0.1", + "EXTERNAL_DNS_INFOBLOX_WAPI_PORT": "8443", + "EXTERNAL_DNS_INFOBLOX_WAPI_USERNAME": "infoblox", + "EXTERNAL_DNS_INFOBLOX_WAPI_PASSWORD": "infoblox", + "EXTERNAL_DNS_INFOBLOX_WAPI_VERSION": "2.6.1", + "EXTERNAL_DNS_INFOBLOX_VIEW": "internal", + "EXTERNAL_DNS_INFOBLOX_SSL_VERIFY": "0", + "EXTERNAL_DNS_INFOBLOX_MAX_RESULTS": "2000", + "EXTERNAL_DNS_OCI_CONFIG_FILE": "oci.yaml", + "EXTERNAL_DNS_INMEMORY_ZONE": "example.org\ncompany.com", + "EXTERNAL_DNS_DOMAIN_FILTER": "example.org\ncompany.com", + "EXTERNAL_DNS_EXCLUDE_DOMAINS": "xapi.example.org\nxapi.company.com", + "EXTERNAL_DNS_PDNS_SERVER": "http://ns.example.com:8081", + "EXTERNAL_DNS_PDNS_API_KEY": "some-secret-key", + "EXTERNAL_DNS_PDNS_TLS_ENABLED": "1", + "EXTERNAL_DNS_RDNS_ROOT_DOMAIN": "lb.rancher.cloud", + "EXTERNAL_DNS_TLS_CA": "/path/to/ca.crt", + "EXTERNAL_DNS_TLS_CLIENT_CERT": "/path/to/cert.pem", + "EXTERNAL_DNS_TLS_CLIENT_CERT_KEY": "/path/to/key.pem", + "EXTERNAL_DNS_ZONE_ID_FILTER": "/hostedzone/ZTST1\n/hostedzone/ZTST2", + "EXTERNAL_DNS_AWS_ZONE_TYPE": "private", + "EXTERNAL_DNS_AWS_ZONE_TAGS": "tag=foo", + "EXTERNAL_DNS_AWS_ASSUME_ROLE": "some-other-role", + "EXTERNAL_DNS_AWS_BATCH_CHANGE_SIZE": "100", + "EXTERNAL_DNS_AWS_BATCH_CHANGE_INTERVAL": "2s", + "EXTERNAL_DNS_AWS_EVALUATE_TARGET_HEALTH": "0", + "EXTERNAL_DNS_AWS_API_RETRIES": "13", + "EXTERNAL_DNS_AWS_PREFER_CNAME": "true", + "EXTERNAL_DNS_POLICY": "upsert-only", + "EXTERNAL_DNS_REGISTRY": "noop", + "EXTERNAL_DNS_TXT_OWNER_ID": "owner-1", + "EXTERNAL_DNS_TXT_PREFIX": "associated-txt-record", + "EXTERNAL_DNS_TXT_CACHE_INTERVAL": "12h", + "EXTERNAL_DNS_INTERVAL": "10m", + "EXTERNAL_DNS_ONCE": "1", + "EXTERNAL_DNS_DRY_RUN": "1", + "EXTERNAL_DNS_LOG_FORMAT": "json", + "EXTERNAL_DNS_METRICS_ADDRESS": "127.0.0.1:9099", + "EXTERNAL_DNS_LOG_LEVEL": "debug", + "EXTERNAL_DNS_CONNECTOR_SOURCE_SERVER": "localhost:8081", + "EXTERNAL_DNS_EXOSCALE_ENDPOINT": "https://api.foo.ch/dns", + "EXTERNAL_DNS_EXOSCALE_APIKEY": "1", + "EXTERNAL_DNS_EXOSCALE_APISECRET": "2", + "EXTERNAL_DNS_CRD_SOURCE_APIVERSION": "test.k8s.io/v1alpha1", + "EXTERNAL_DNS_CRD_SOURCE_KIND": "Endpoint", + "EXTERNAL_DNS_RCODEZERO_TXT_ENCRYPT": "1", + "EXTERNAL_DNS_NS1_ENDPOINT": "https://api.example.com/v1", + "EXTERNAL_DNS_NS1_IGNORESSL": "1", + "EXTERNAL_DNS_TRANSIP_ACCOUNT": "transip", + "EXTERNAL_DNS_TRANSIP_KEYFILE": "/path/to/transip.key", }, expected: overriddenConfig, }, diff --git a/provider/google.go b/provider/google.go index 85c3a9084..49ece4f0a 100644 --- a/provider/google.go +++ b/provider/google.go @@ -21,6 +21,7 @@ import ( "fmt" "sort" "strings" + "time" "cloud.google.com/go/compute/metadata" "github.com/linki/instrumented_http" @@ -106,6 +107,8 @@ type GoogleProvider struct { dryRun bool // Max batch size to submit to Google Cloud DNS per transaction. batchChangeSize int + // Interval between batch updates. + batchChangeInterval time.Duration // only consider hosted zones managing domains ending in this suffix domainFilter DomainFilter // only consider hosted zones ending with this zone id @@ -119,7 +122,7 @@ type GoogleProvider struct { } // NewGoogleProvider initializes a new Google CloudDNS based Provider. -func NewGoogleProvider(project string, domainFilter DomainFilter, zoneIDFilter ZoneIDFilter, batchChangeSize int, dryRun bool) (*GoogleProvider, error) { +func NewGoogleProvider(project string, domainFilter DomainFilter, zoneIDFilter ZoneIDFilter, batchChangeSize int, batchChangeInterval time.Duration, dryRun bool) (*GoogleProvider, error) { gcloud, err := google.DefaultClient(context.TODO(), dns.NdevClouddnsReadwriteScope) if err != nil { return nil, err @@ -149,6 +152,7 @@ func NewGoogleProvider(project string, domainFilter DomainFilter, zoneIDFilter Z project: project, dryRun: dryRun, batchChangeSize: batchChangeSize, + batchChangeInterval: batchChangeInterval, domainFilter: domainFilter, zoneIDFilter: zoneIDFilter, resourceRecordSetsClient: resourceRecordSetsService{dnsClient.ResourceRecordSets}, @@ -310,6 +314,8 @@ func (p *GoogleProvider) submitChange(change *dns.Change) error { if _, err := p.changesClient.Create(p.project, zone, c).Do(); err != nil { return err } + + time.Sleep(p.batchChangeInterval) } }