mirror of
https://github.com/kubernetes-sigs/external-dns.git
synced 2025-08-06 09:36:58 +02:00
Merge pull request #1248 from vdesjardins/clouddns-batching
Google Provider: add support for batching updates
This commit is contained in:
commit
e9c834ae9b
2
main.go
2
main.go
@ -142,7 +142,7 @@ func main() {
|
||||
case "rcodezero":
|
||||
p, err = provider.NewRcodeZeroProvider(domainFilter, cfg.DryRun, cfg.RcodezeroTXTEncrypt)
|
||||
case "google":
|
||||
p, err = provider.NewGoogleProvider(cfg.GoogleProject, domainFilter, zoneIDFilter, cfg.DryRun)
|
||||
p, err = provider.NewGoogleProvider(cfg.GoogleProject, domainFilter, zoneIDFilter, cfg.GoogleBatchChangeSize, cfg.GoogleBatchChangeInterval, cfg.DryRun)
|
||||
case "digitalocean":
|
||||
p, err = provider.NewDigitalOceanProvider(domainFilter, cfg.DryRun)
|
||||
case "linode":
|
||||
|
@ -54,6 +54,8 @@ type Config struct {
|
||||
ConnectorSourceServer string
|
||||
Provider string
|
||||
GoogleProject string
|
||||
GoogleBatchChangeSize int
|
||||
GoogleBatchChangeInterval time.Duration
|
||||
DomainFilter []string
|
||||
ExcludeDomains []string
|
||||
ZoneIDFilter []string
|
||||
@ -145,6 +147,8 @@ var defaultConfig = &Config{
|
||||
ConnectorSourceServer: "localhost:8080",
|
||||
Provider: "",
|
||||
GoogleProject: "",
|
||||
GoogleBatchChangeSize: 1000,
|
||||
GoogleBatchChangeInterval: time.Second,
|
||||
DomainFilter: []string{},
|
||||
ExcludeDomains: []string{},
|
||||
AlibabaCloudConfigFile: "/etc/kubernetes/alibaba-cloud.json",
|
||||
@ -290,6 +294,8 @@ func (cfg *Config) ParseFlags(args []string) error {
|
||||
app.Flag("exclude-domains", "Exclude subdomains (optional)").Default("").StringsVar(&cfg.ExcludeDomains)
|
||||
app.Flag("zone-id-filter", "Filter target zones by hosted zone id; specify multiple times for multiple zones (optional)").Default("").StringsVar(&cfg.ZoneIDFilter)
|
||||
app.Flag("google-project", "When using the Google provider, current project is auto-detected, when running on GCP. Specify other project with this. Must be specified when running outside GCP.").Default(defaultConfig.GoogleProject).StringVar(&cfg.GoogleProject)
|
||||
app.Flag("google-batch-change-size", "When using the Google provider, set the maximum number of changes that will be applied in each batch.").Default(strconv.Itoa(defaultConfig.GoogleBatchChangeSize)).IntVar(&cfg.GoogleBatchChangeSize)
|
||||
app.Flag("google-batch-change-interval", "When using the Google provider, set the interval between batch changes.").Default(defaultConfig.GoogleBatchChangeInterval.String()).DurationVar(&cfg.GoogleBatchChangeInterval)
|
||||
app.Flag("alibaba-cloud-config-file", "When using the Alibaba Cloud provider, specify the Alibaba Cloud configuration file (required when --provider=alibabacloud").Default(defaultConfig.AlibabaCloudConfigFile).StringVar(&cfg.AlibabaCloudConfigFile)
|
||||
app.Flag("alibaba-cloud-zone-type", "When using the Alibaba Cloud provider, filter for zones of this type (optional, options: public, private)").Default(defaultConfig.AlibabaCloudZoneType).EnumVar(&cfg.AlibabaCloudZoneType, "", "public", "private")
|
||||
app.Flag("aws-zone-type", "When using the AWS provider, filter for zones of this type (optional, options: public, private)").Default(defaultConfig.AWSZoneType).EnumVar(&cfg.AWSZoneType, "", "public", "private")
|
||||
|
@ -40,6 +40,8 @@ var (
|
||||
Compatibility: "",
|
||||
Provider: "google",
|
||||
GoogleProject: "",
|
||||
GoogleBatchChangeSize: 1000,
|
||||
GoogleBatchChangeInterval: time.Second,
|
||||
DomainFilter: []string{""},
|
||||
ExcludeDomains: []string{""},
|
||||
ZoneIDFilter: []string{""},
|
||||
@ -104,6 +106,8 @@ var (
|
||||
Compatibility: "mate",
|
||||
Provider: "google",
|
||||
GoogleProject: "project",
|
||||
GoogleBatchChangeSize: 100,
|
||||
GoogleBatchChangeInterval: time.Second * 2,
|
||||
DomainFilter: []string{"example.org", "company.com"},
|
||||
ExcludeDomains: []string{"xapi.example.org", "xapi.company.com"},
|
||||
ZoneIDFilter: []string{"/hostedzone/ZTST1", "/hostedzone/ZTST2"},
|
||||
@ -174,6 +178,8 @@ var (
|
||||
Compatibility: "",
|
||||
Provider: "google",
|
||||
GoogleProject: "",
|
||||
GoogleBatchChangeSize: 1000,
|
||||
GoogleBatchChangeInterval: time.Second,
|
||||
DomainFilter: []string{""},
|
||||
ExcludeDomains: []string{""},
|
||||
ZoneIDFilter: []string{""},
|
||||
@ -256,6 +262,8 @@ func TestParseFlags(t *testing.T) {
|
||||
"--compatibility=mate",
|
||||
"--provider=google",
|
||||
"--google-project=project",
|
||||
"--google-batch-change-size=100",
|
||||
"--google-batch-change-interval=2s",
|
||||
"--azure-config-file=azure.json",
|
||||
"--azure-resource-group=arg",
|
||||
"--cloudflare-proxied",
|
||||
@ -322,73 +330,75 @@ func TestParseFlags(t *testing.T) {
|
||||
title: "override everything via environment variables",
|
||||
args: []string{},
|
||||
envVars: map[string]string{
|
||||
"EXTERNAL_DNS_MASTER": "http://127.0.0.1:8080",
|
||||
"EXTERNAL_DNS_KUBECONFIG": "/some/path",
|
||||
"EXTERNAL_DNS_REQUEST_TIMEOUT": "77s",
|
||||
"EXTERNAL_DNS_ISTIO_INGRESS_GATEWAY": "istio-other/istio-otheringressgateway",
|
||||
"EXTERNAL_DNS_CONTOUR_LOAD_BALANCER": "heptio-contour-other/contour-other",
|
||||
"EXTERNAL_DNS_SOURCE": "service\ningress\nconnector",
|
||||
"EXTERNAL_DNS_NAMESPACE": "namespace",
|
||||
"EXTERNAL_DNS_FQDN_TEMPLATE": "{{.Name}}.service.example.com",
|
||||
"EXTERNAL_DNS_IGNORE_HOSTNAME_ANNOTATION": "1",
|
||||
"EXTERNAL_DNS_COMPATIBILITY": "mate",
|
||||
"EXTERNAL_DNS_PROVIDER": "google",
|
||||
"EXTERNAL_DNS_GOOGLE_PROJECT": "project",
|
||||
"EXTERNAL_DNS_AZURE_CONFIG_FILE": "azure.json",
|
||||
"EXTERNAL_DNS_AZURE_RESOURCE_GROUP": "arg",
|
||||
"EXTERNAL_DNS_CLOUDFLARE_PROXIED": "1",
|
||||
"EXTERNAL_DNS_CLOUDFLARE_ZONES_PER_PAGE": "20",
|
||||
"EXTERNAL_DNS_COREDNS_PREFIX": "/coredns/",
|
||||
"EXTERNAL_DNS_INFOBLOX_GRID_HOST": "127.0.0.1",
|
||||
"EXTERNAL_DNS_INFOBLOX_WAPI_PORT": "8443",
|
||||
"EXTERNAL_DNS_INFOBLOX_WAPI_USERNAME": "infoblox",
|
||||
"EXTERNAL_DNS_INFOBLOX_WAPI_PASSWORD": "infoblox",
|
||||
"EXTERNAL_DNS_INFOBLOX_WAPI_VERSION": "2.6.1",
|
||||
"EXTERNAL_DNS_INFOBLOX_VIEW": "internal",
|
||||
"EXTERNAL_DNS_INFOBLOX_SSL_VERIFY": "0",
|
||||
"EXTERNAL_DNS_INFOBLOX_MAX_RESULTS": "2000",
|
||||
"EXTERNAL_DNS_OCI_CONFIG_FILE": "oci.yaml",
|
||||
"EXTERNAL_DNS_INMEMORY_ZONE": "example.org\ncompany.com",
|
||||
"EXTERNAL_DNS_DOMAIN_FILTER": "example.org\ncompany.com",
|
||||
"EXTERNAL_DNS_EXCLUDE_DOMAINS": "xapi.example.org\nxapi.company.com",
|
||||
"EXTERNAL_DNS_PDNS_SERVER": "http://ns.example.com:8081",
|
||||
"EXTERNAL_DNS_PDNS_API_KEY": "some-secret-key",
|
||||
"EXTERNAL_DNS_PDNS_TLS_ENABLED": "1",
|
||||
"EXTERNAL_DNS_RDNS_ROOT_DOMAIN": "lb.rancher.cloud",
|
||||
"EXTERNAL_DNS_TLS_CA": "/path/to/ca.crt",
|
||||
"EXTERNAL_DNS_TLS_CLIENT_CERT": "/path/to/cert.pem",
|
||||
"EXTERNAL_DNS_TLS_CLIENT_CERT_KEY": "/path/to/key.pem",
|
||||
"EXTERNAL_DNS_ZONE_ID_FILTER": "/hostedzone/ZTST1\n/hostedzone/ZTST2",
|
||||
"EXTERNAL_DNS_AWS_ZONE_TYPE": "private",
|
||||
"EXTERNAL_DNS_AWS_ZONE_TAGS": "tag=foo",
|
||||
"EXTERNAL_DNS_AWS_ASSUME_ROLE": "some-other-role",
|
||||
"EXTERNAL_DNS_AWS_BATCH_CHANGE_SIZE": "100",
|
||||
"EXTERNAL_DNS_AWS_BATCH_CHANGE_INTERVAL": "2s",
|
||||
"EXTERNAL_DNS_AWS_EVALUATE_TARGET_HEALTH": "0",
|
||||
"EXTERNAL_DNS_AWS_API_RETRIES": "13",
|
||||
"EXTERNAL_DNS_AWS_PREFER_CNAME": "true",
|
||||
"EXTERNAL_DNS_POLICY": "upsert-only",
|
||||
"EXTERNAL_DNS_REGISTRY": "noop",
|
||||
"EXTERNAL_DNS_TXT_OWNER_ID": "owner-1",
|
||||
"EXTERNAL_DNS_TXT_PREFIX": "associated-txt-record",
|
||||
"EXTERNAL_DNS_TXT_CACHE_INTERVAL": "12h",
|
||||
"EXTERNAL_DNS_INTERVAL": "10m",
|
||||
"EXTERNAL_DNS_ONCE": "1",
|
||||
"EXTERNAL_DNS_DRY_RUN": "1",
|
||||
"EXTERNAL_DNS_LOG_FORMAT": "json",
|
||||
"EXTERNAL_DNS_METRICS_ADDRESS": "127.0.0.1:9099",
|
||||
"EXTERNAL_DNS_LOG_LEVEL": "debug",
|
||||
"EXTERNAL_DNS_CONNECTOR_SOURCE_SERVER": "localhost:8081",
|
||||
"EXTERNAL_DNS_EXOSCALE_ENDPOINT": "https://api.foo.ch/dns",
|
||||
"EXTERNAL_DNS_EXOSCALE_APIKEY": "1",
|
||||
"EXTERNAL_DNS_EXOSCALE_APISECRET": "2",
|
||||
"EXTERNAL_DNS_CRD_SOURCE_APIVERSION": "test.k8s.io/v1alpha1",
|
||||
"EXTERNAL_DNS_CRD_SOURCE_KIND": "Endpoint",
|
||||
"EXTERNAL_DNS_RCODEZERO_TXT_ENCRYPT": "1",
|
||||
"EXTERNAL_DNS_NS1_ENDPOINT": "https://api.example.com/v1",
|
||||
"EXTERNAL_DNS_NS1_IGNORESSL": "1",
|
||||
"EXTERNAL_DNS_TRANSIP_ACCOUNT": "transip",
|
||||
"EXTERNAL_DNS_TRANSIP_KEYFILE": "/path/to/transip.key",
|
||||
"EXTERNAL_DNS_MASTER": "http://127.0.0.1:8080",
|
||||
"EXTERNAL_DNS_KUBECONFIG": "/some/path",
|
||||
"EXTERNAL_DNS_REQUEST_TIMEOUT": "77s",
|
||||
"EXTERNAL_DNS_ISTIO_INGRESS_GATEWAY": "istio-other/istio-otheringressgateway",
|
||||
"EXTERNAL_DNS_CONTOUR_LOAD_BALANCER": "heptio-contour-other/contour-other",
|
||||
"EXTERNAL_DNS_SOURCE": "service\ningress\nconnector",
|
||||
"EXTERNAL_DNS_NAMESPACE": "namespace",
|
||||
"EXTERNAL_DNS_FQDN_TEMPLATE": "{{.Name}}.service.example.com",
|
||||
"EXTERNAL_DNS_IGNORE_HOSTNAME_ANNOTATION": "1",
|
||||
"EXTERNAL_DNS_COMPATIBILITY": "mate",
|
||||
"EXTERNAL_DNS_PROVIDER": "google",
|
||||
"EXTERNAL_DNS_GOOGLE_PROJECT": "project",
|
||||
"EXTERNAL_DNS_GOOGLE_BATCH_CHANGE_SIZE": "100",
|
||||
"EXTERNAL_DNS_GOOGLE_BATCH_CHANGE_INTERVAL": "2s",
|
||||
"EXTERNAL_DNS_AZURE_CONFIG_FILE": "azure.json",
|
||||
"EXTERNAL_DNS_AZURE_RESOURCE_GROUP": "arg",
|
||||
"EXTERNAL_DNS_CLOUDFLARE_PROXIED": "1",
|
||||
"EXTERNAL_DNS_CLOUDFLARE_ZONES_PER_PAGE": "20",
|
||||
"EXTERNAL_DNS_COREDNS_PREFIX": "/coredns/",
|
||||
"EXTERNAL_DNS_INFOBLOX_GRID_HOST": "127.0.0.1",
|
||||
"EXTERNAL_DNS_INFOBLOX_WAPI_PORT": "8443",
|
||||
"EXTERNAL_DNS_INFOBLOX_WAPI_USERNAME": "infoblox",
|
||||
"EXTERNAL_DNS_INFOBLOX_WAPI_PASSWORD": "infoblox",
|
||||
"EXTERNAL_DNS_INFOBLOX_WAPI_VERSION": "2.6.1",
|
||||
"EXTERNAL_DNS_INFOBLOX_VIEW": "internal",
|
||||
"EXTERNAL_DNS_INFOBLOX_SSL_VERIFY": "0",
|
||||
"EXTERNAL_DNS_INFOBLOX_MAX_RESULTS": "2000",
|
||||
"EXTERNAL_DNS_OCI_CONFIG_FILE": "oci.yaml",
|
||||
"EXTERNAL_DNS_INMEMORY_ZONE": "example.org\ncompany.com",
|
||||
"EXTERNAL_DNS_DOMAIN_FILTER": "example.org\ncompany.com",
|
||||
"EXTERNAL_DNS_EXCLUDE_DOMAINS": "xapi.example.org\nxapi.company.com",
|
||||
"EXTERNAL_DNS_PDNS_SERVER": "http://ns.example.com:8081",
|
||||
"EXTERNAL_DNS_PDNS_API_KEY": "some-secret-key",
|
||||
"EXTERNAL_DNS_PDNS_TLS_ENABLED": "1",
|
||||
"EXTERNAL_DNS_RDNS_ROOT_DOMAIN": "lb.rancher.cloud",
|
||||
"EXTERNAL_DNS_TLS_CA": "/path/to/ca.crt",
|
||||
"EXTERNAL_DNS_TLS_CLIENT_CERT": "/path/to/cert.pem",
|
||||
"EXTERNAL_DNS_TLS_CLIENT_CERT_KEY": "/path/to/key.pem",
|
||||
"EXTERNAL_DNS_ZONE_ID_FILTER": "/hostedzone/ZTST1\n/hostedzone/ZTST2",
|
||||
"EXTERNAL_DNS_AWS_ZONE_TYPE": "private",
|
||||
"EXTERNAL_DNS_AWS_ZONE_TAGS": "tag=foo",
|
||||
"EXTERNAL_DNS_AWS_ASSUME_ROLE": "some-other-role",
|
||||
"EXTERNAL_DNS_AWS_BATCH_CHANGE_SIZE": "100",
|
||||
"EXTERNAL_DNS_AWS_BATCH_CHANGE_INTERVAL": "2s",
|
||||
"EXTERNAL_DNS_AWS_EVALUATE_TARGET_HEALTH": "0",
|
||||
"EXTERNAL_DNS_AWS_API_RETRIES": "13",
|
||||
"EXTERNAL_DNS_AWS_PREFER_CNAME": "true",
|
||||
"EXTERNAL_DNS_POLICY": "upsert-only",
|
||||
"EXTERNAL_DNS_REGISTRY": "noop",
|
||||
"EXTERNAL_DNS_TXT_OWNER_ID": "owner-1",
|
||||
"EXTERNAL_DNS_TXT_PREFIX": "associated-txt-record",
|
||||
"EXTERNAL_DNS_TXT_CACHE_INTERVAL": "12h",
|
||||
"EXTERNAL_DNS_INTERVAL": "10m",
|
||||
"EXTERNAL_DNS_ONCE": "1",
|
||||
"EXTERNAL_DNS_DRY_RUN": "1",
|
||||
"EXTERNAL_DNS_LOG_FORMAT": "json",
|
||||
"EXTERNAL_DNS_METRICS_ADDRESS": "127.0.0.1:9099",
|
||||
"EXTERNAL_DNS_LOG_LEVEL": "debug",
|
||||
"EXTERNAL_DNS_CONNECTOR_SOURCE_SERVER": "localhost:8081",
|
||||
"EXTERNAL_DNS_EXOSCALE_ENDPOINT": "https://api.foo.ch/dns",
|
||||
"EXTERNAL_DNS_EXOSCALE_APIKEY": "1",
|
||||
"EXTERNAL_DNS_EXOSCALE_APISECRET": "2",
|
||||
"EXTERNAL_DNS_CRD_SOURCE_APIVERSION": "test.k8s.io/v1alpha1",
|
||||
"EXTERNAL_DNS_CRD_SOURCE_KIND": "Endpoint",
|
||||
"EXTERNAL_DNS_RCODEZERO_TXT_ENCRYPT": "1",
|
||||
"EXTERNAL_DNS_NS1_ENDPOINT": "https://api.example.com/v1",
|
||||
"EXTERNAL_DNS_NS1_IGNORESSL": "1",
|
||||
"EXTERNAL_DNS_TRANSIP_ACCOUNT": "transip",
|
||||
"EXTERNAL_DNS_TRANSIP_KEYFILE": "/path/to/transip.key",
|
||||
},
|
||||
expected: overriddenConfig,
|
||||
},
|
||||
|
@ -19,7 +19,9 @@ package provider
|
||||
import (
|
||||
goctx "context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"cloud.google.com/go/compute/metadata"
|
||||
"github.com/linki/instrumented_http"
|
||||
@ -104,6 +106,10 @@ type GoogleProvider struct {
|
||||
project string
|
||||
// Enabled dry-run will print any modifying actions rather than execute them.
|
||||
dryRun bool
|
||||
// Max batch size to submit to Google Cloud DNS per transaction.
|
||||
batchChangeSize int
|
||||
// Interval between batch updates.
|
||||
batchChangeInterval time.Duration
|
||||
// only consider hosted zones managing domains ending in this suffix
|
||||
domainFilter DomainFilter
|
||||
// only consider hosted zones ending with this zone id
|
||||
@ -117,7 +123,7 @@ type GoogleProvider struct {
|
||||
}
|
||||
|
||||
// NewGoogleProvider initializes a new Google CloudDNS based Provider.
|
||||
func NewGoogleProvider(project string, domainFilter DomainFilter, zoneIDFilter ZoneIDFilter, dryRun bool) (*GoogleProvider, error) {
|
||||
func NewGoogleProvider(project string, domainFilter DomainFilter, zoneIDFilter ZoneIDFilter, batchChangeSize int, batchChangeInterval time.Duration, dryRun bool) (*GoogleProvider, error) {
|
||||
gcloud, err := google.DefaultClient(context.TODO(), dns.NdevClouddnsReadwriteScope)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@ -146,6 +152,8 @@ func NewGoogleProvider(project string, domainFilter DomainFilter, zoneIDFilter Z
|
||||
provider := &GoogleProvider{
|
||||
project: project,
|
||||
dryRun: dryRun,
|
||||
batchChangeSize: batchChangeSize,
|
||||
batchChangeInterval: batchChangeInterval,
|
||||
domainFilter: domainFilter,
|
||||
zoneIDFilter: zoneIDFilter,
|
||||
resourceRecordSetsClient: resourceRecordSetsService{dnsClient.ResourceRecordSets},
|
||||
@ -290,29 +298,104 @@ func (p *GoogleProvider) submitChange(change *dns.Change) error {
|
||||
// separate into per-zone change sets to be passed to the API.
|
||||
changes := separateChange(zones, change)
|
||||
|
||||
for z, c := range changes {
|
||||
log.Infof("Change zone: %v", z)
|
||||
for _, del := range c.Deletions {
|
||||
log.Infof("Del records: %s %s %s %d", del.Name, del.Type, del.Rrdatas, del.Ttl)
|
||||
}
|
||||
for _, add := range c.Additions {
|
||||
log.Infof("Add records: %s %s %s %d", add.Name, add.Type, add.Rrdatas, add.Ttl)
|
||||
}
|
||||
}
|
||||
for zone, change := range changes {
|
||||
for batch, c := range batchChange(change, p.batchChangeSize) {
|
||||
log.Infof("Change zone: %v batch #%d", zone, batch)
|
||||
for _, del := range c.Deletions {
|
||||
log.Infof("Del records: %s %s %s %d", del.Name, del.Type, del.Rrdatas, del.Ttl)
|
||||
}
|
||||
for _, add := range c.Additions {
|
||||
log.Infof("Add records: %s %s %s %d", add.Name, add.Type, add.Rrdatas, add.Ttl)
|
||||
}
|
||||
|
||||
if p.dryRun {
|
||||
return nil
|
||||
}
|
||||
if p.dryRun {
|
||||
continue
|
||||
}
|
||||
|
||||
for z, c := range changes {
|
||||
if _, err := p.changesClient.Create(p.project, z, c).Do(); err != nil {
|
||||
return err
|
||||
if _, err := p.changesClient.Create(p.project, zone, c).Do(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
time.Sleep(p.batchChangeInterval)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// batchChange seperates a zone in multiple transaction.
|
||||
func batchChange(change *dns.Change, batchSize int) []*dns.Change {
|
||||
changes := []*dns.Change{}
|
||||
|
||||
if batchSize == 0 {
|
||||
return append(changes, change)
|
||||
}
|
||||
|
||||
type dnsChange struct {
|
||||
additions []*dns.ResourceRecordSet
|
||||
deletions []*dns.ResourceRecordSet
|
||||
}
|
||||
|
||||
changesByName := map[string]*dnsChange{}
|
||||
|
||||
for _, a := range change.Additions {
|
||||
change, ok := changesByName[a.Name]
|
||||
if !ok {
|
||||
change = &dnsChange{}
|
||||
changesByName[a.Name] = change
|
||||
}
|
||||
|
||||
change.additions = append(change.additions, a)
|
||||
}
|
||||
|
||||
for _, a := range change.Deletions {
|
||||
change, ok := changesByName[a.Name]
|
||||
if !ok {
|
||||
change = &dnsChange{}
|
||||
changesByName[a.Name] = change
|
||||
}
|
||||
|
||||
change.deletions = append(change.deletions, a)
|
||||
}
|
||||
|
||||
names := make([]string, 0)
|
||||
for v := range changesByName {
|
||||
names = append(names, v)
|
||||
}
|
||||
sort.Strings(names)
|
||||
|
||||
currentChange := &dns.Change{}
|
||||
var totalChanges int
|
||||
for _, name := range names {
|
||||
c := changesByName[name]
|
||||
|
||||
totalChangesByName := len(c.additions) + len(c.deletions)
|
||||
|
||||
if totalChangesByName > batchSize {
|
||||
log.Warnf("Total changes for %s exceeds max batch size of %d, total changes: %d", name,
|
||||
batchSize, totalChangesByName)
|
||||
continue
|
||||
}
|
||||
|
||||
if totalChanges+totalChangesByName > batchSize {
|
||||
totalChanges = 0
|
||||
changes = append(changes, currentChange)
|
||||
currentChange = &dns.Change{}
|
||||
}
|
||||
|
||||
currentChange.Additions = append(currentChange.Additions, c.additions...)
|
||||
currentChange.Deletions = append(currentChange.Deletions, c.deletions...)
|
||||
|
||||
totalChanges += totalChangesByName
|
||||
}
|
||||
|
||||
if totalChanges > 0 {
|
||||
changes = append(changes, currentChange)
|
||||
}
|
||||
|
||||
return changes
|
||||
}
|
||||
|
||||
// separateChange separates a multi-zone change into a single change per zone.
|
||||
func separateChange(zones map[string]*dns.ManagedZone, change *dns.Change) map[string]*dns.Change {
|
||||
changes := make(map[string]*dns.Change)
|
||||
|
@ -19,6 +19,7 @@ package provider
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@ -36,8 +37,9 @@ import (
|
||||
)
|
||||
|
||||
var (
|
||||
testZones = map[string]*dns.ManagedZone{}
|
||||
testRecords = map[string]map[string]*dns.ResourceRecordSet{}
|
||||
testZones = map[string]*dns.ManagedZone{}
|
||||
testRecords = map[string]map[string]*dns.ResourceRecordSet{}
|
||||
googleDefaultBatchChangeSize = 4000
|
||||
)
|
||||
|
||||
type mockManagedZonesCreateCall struct {
|
||||
@ -551,6 +553,94 @@ func TestSeparateChanges(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func TestGoogleBatchChangeSet(t *testing.T) {
|
||||
cs := &dns.Change{}
|
||||
|
||||
for i := 1; i <= googleDefaultBatchChangeSize; i += 2 {
|
||||
cs.Additions = append(cs.Additions, &dns.ResourceRecordSet{
|
||||
Name: fmt.Sprintf("host-%d.example.org.", i),
|
||||
Ttl: 2,
|
||||
})
|
||||
cs.Deletions = append(cs.Deletions, &dns.ResourceRecordSet{
|
||||
Name: fmt.Sprintf("host-%d.example.org.", i),
|
||||
Ttl: 20,
|
||||
})
|
||||
}
|
||||
|
||||
batchCs := batchChange(cs, googleDefaultBatchChangeSize)
|
||||
|
||||
require.Equal(t, 1, len(batchCs))
|
||||
|
||||
sortChangesByName(cs)
|
||||
validateChange(t, batchCs[0], cs)
|
||||
}
|
||||
|
||||
func TestGoogleBatchChangeSetExceeding(t *testing.T) {
|
||||
cs := &dns.Change{}
|
||||
const testCount = 50
|
||||
const testLimit = 11
|
||||
const expectedBatchCount = 5
|
||||
const expectedChangesCount = 10
|
||||
|
||||
for i := 1; i <= testCount; i += 2 {
|
||||
cs.Additions = append(cs.Additions, &dns.ResourceRecordSet{
|
||||
Name: fmt.Sprintf("host-%d.example.org.", i),
|
||||
Ttl: 2,
|
||||
})
|
||||
cs.Deletions = append(cs.Deletions, &dns.ResourceRecordSet{
|
||||
Name: fmt.Sprintf("host-%d.example.org.", i),
|
||||
Ttl: 20,
|
||||
})
|
||||
}
|
||||
|
||||
batchCs := batchChange(cs, testLimit)
|
||||
|
||||
require.Equal(t, expectedBatchCount, len(batchCs))
|
||||
|
||||
dnsChange := &dns.Change{}
|
||||
for _, c := range batchCs {
|
||||
dnsChange.Additions = append(dnsChange.Additions, c.Additions...)
|
||||
dnsChange.Deletions = append(dnsChange.Deletions, c.Deletions...)
|
||||
}
|
||||
|
||||
require.Equal(t, len(cs.Additions), len(dnsChange.Additions))
|
||||
require.Equal(t, len(cs.Deletions), len(dnsChange.Deletions))
|
||||
|
||||
sortChangesByName(cs)
|
||||
sortChangesByName(dnsChange)
|
||||
|
||||
validateChange(t, dnsChange, cs)
|
||||
}
|
||||
|
||||
func TestGoogleBatchChangeSetExceedingNameChange(t *testing.T) {
|
||||
cs := &dns.Change{}
|
||||
const testCount = 10
|
||||
const testLimit = 1
|
||||
|
||||
cs.Additions = append(cs.Additions, &dns.ResourceRecordSet{
|
||||
Name: "host-1.example.org.",
|
||||
Ttl: 2,
|
||||
})
|
||||
cs.Deletions = append(cs.Deletions, &dns.ResourceRecordSet{
|
||||
Name: "host-1.example.org.",
|
||||
Ttl: 20,
|
||||
})
|
||||
|
||||
batchCs := batchChange(cs, testLimit)
|
||||
|
||||
require.Equal(t, 0, len(batchCs))
|
||||
}
|
||||
|
||||
func sortChangesByName(cs *dns.Change) {
|
||||
sort.SliceStable(cs.Additions, func(i, j int) bool {
|
||||
return cs.Additions[i].Name < cs.Additions[j].Name
|
||||
})
|
||||
|
||||
sort.SliceStable(cs.Deletions, func(i, j int) bool {
|
||||
return cs.Deletions[i].Name < cs.Deletions[j].Name
|
||||
})
|
||||
}
|
||||
|
||||
func validateZones(t *testing.T, zones map[string]*dns.ManagedZone, expected map[string]*dns.ManagedZone) {
|
||||
require.Len(t, zones, len(expected))
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user