From 370bae6dd38d39f1473535d791b185842c96df6f Mon Sep 17 00:00:00 2001 From: njuettner Date: Wed, 16 Jan 2019 15:55:28 +0100 Subject: [PATCH] Cloudflare pagination for zones --- Gopkg.lock | 6 +-- Gopkg.toml | 2 +- main.go | 2 +- pkg/apis/externaldns/types.go | 3 ++ pkg/apis/externaldns/types_test.go | 4 ++ provider/cloudflare.go | 43 +++++++++++++-------- provider/cloudflare_test.go | 61 +++++++++++++++++++++++++++--- 7 files changed, 95 insertions(+), 26 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index d5b1ec160..470fba59c 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -128,12 +128,12 @@ revision = "4c0e84591b9aa9e6dcfdf3e020114cd81f89d5f9" [[projects]] - digest = "1:85fd00554a6ed5b33687684b76635d532c74141508b5bce2843d85e8a3c9dc91" + branch = "master" + digest = "1:3e90c0d9954bf53a2061c4a0f193e6569c9ab2118c8f3a250871498f6b4645e5" name = "github.com/cloudflare/cloudflare-go" packages = ["."] pruneopts = "" - revision = "4c6994ac3877fbb627766edadc67f4e816e8c890" - version = "v0.7.4" + revision = "0c85496d873009e53e64d391ade643e7ac02e0d4" [[projects]] digest = "1:eaeede87b418b97f9dee473f8940fd9b65ca5cdac0503350c7c8f8965ea3cf4d" diff --git a/Gopkg.toml b/Gopkg.toml index 17cc1e3cc..6590be2e0 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -21,8 +21,8 @@ ignored = ["github.com/kubernetes/repo-infra/kazel"] version = "~1.13.7" [[constraint]] + branch = "master" name = "github.com/cloudflare/cloudflare-go" - version = "0.7.3" [[constraint]] name = "github.com/digitalocean/godo" diff --git a/main.go b/main.go index 56d4e5777..7c95bf790 100644 --- a/main.go +++ b/main.go @@ -129,7 +129,7 @@ func main() { case "azure": p, err = provider.NewAzureProvider(cfg.AzureConfigFile, domainFilter, zoneIDFilter, cfg.AzureResourceGroup, cfg.DryRun) case "cloudflare": - p, err = provider.NewCloudFlareProvider(domainFilter, zoneIDFilter, cfg.CloudflareProxied, cfg.DryRun) + p, err = provider.NewCloudFlareProvider(domainFilter, zoneIDFilter, cfg.CloudflareZonesPerPage, cfg.CloudflareProxied, cfg.DryRun) case "google": p, err = provider.NewGoogleProvider(cfg.GoogleProject, domainFilter, zoneIDFilter, cfg.DryRun) case "digitalocean": diff --git a/pkg/apis/externaldns/types.go b/pkg/apis/externaldns/types.go index 5de4aca5c..6d079e054 100644 --- a/pkg/apis/externaldns/types.go +++ b/pkg/apis/externaldns/types.go @@ -64,6 +64,7 @@ type Config struct { AzureConfigFile string AzureResourceGroup string CloudflareProxied bool + CloudflareZonesPerPage int InfobloxGridHost string InfobloxWapiPort int InfobloxWapiUsername string @@ -136,6 +137,7 @@ var defaultConfig = &Config{ AzureConfigFile: "/etc/kubernetes/azure.json", AzureResourceGroup: "", CloudflareProxied: false, + CloudflareZonesPerPage: 50, InfobloxGridHost: "", InfobloxWapiPort: 443, InfobloxWapiUsername: "admin", @@ -251,6 +253,7 @@ func (cfg *Config) ParseFlags(args []string) error { app.Flag("azure-config-file", "When using the Azure provider, specify the Azure configuration file (required when --provider=azure").Default(defaultConfig.AzureConfigFile).StringVar(&cfg.AzureConfigFile) app.Flag("azure-resource-group", "When using the Azure provider, override the Azure resource group to use (optional)").Default(defaultConfig.AzureResourceGroup).StringVar(&cfg.AzureResourceGroup) app.Flag("cloudflare-proxied", "When using the Cloudflare provider, specify if the proxy mode must be enabled (default: disabled)").BoolVar(&cfg.CloudflareProxied) + app.Flag("cloudflare-zones-per-page", "When using the Cloudflare provider, specify how many zones per page listed, max. possible 50 (default: 50)").Default(strconv.Itoa(defaultConfig.CloudflareZonesPerPage)).IntVar(&cfg.CloudflareZonesPerPage) app.Flag("infoblox-grid-host", "When using the Infoblox provider, specify the Grid Manager host (required when --provider=infoblox)").Default(defaultConfig.InfobloxGridHost).StringVar(&cfg.InfobloxGridHost) app.Flag("infoblox-wapi-port", "When using the Infoblox provider, specify the WAPI port (default: 443)").Default(strconv.Itoa(defaultConfig.InfobloxWapiPort)).IntVar(&cfg.InfobloxWapiPort) app.Flag("infoblox-wapi-username", "When using the Infoblox provider, specify the WAPI username (default: admin)").Default(defaultConfig.InfobloxWapiUsername).StringVar(&cfg.InfobloxWapiUsername) diff --git a/pkg/apis/externaldns/types_test.go b/pkg/apis/externaldns/types_test.go index 8a1f1854f..0d5f6f30e 100644 --- a/pkg/apis/externaldns/types_test.go +++ b/pkg/apis/externaldns/types_test.go @@ -51,6 +51,7 @@ var ( AzureConfigFile: "/etc/kubernetes/azure.json", AzureResourceGroup: "", CloudflareProxied: false, + CloudflareZonesPerPage: 50, InfobloxGridHost: "", InfobloxWapiPort: 443, InfobloxWapiUsername: "admin", @@ -103,6 +104,7 @@ var ( AzureConfigFile: "azure.json", AzureResourceGroup: "arg", CloudflareProxied: true, + CloudflareZonesPerPage: 20, InfobloxGridHost: "127.0.0.1", InfobloxWapiPort: 8443, InfobloxWapiUsername: "infoblox", @@ -171,6 +173,7 @@ func TestParseFlags(t *testing.T) { "--azure-config-file=azure.json", "--azure-resource-group=arg", "--cloudflare-proxied", + "--cloudflare-zones-per-page=20", "--infoblox-grid-host=127.0.0.1", "--infoblox-wapi-port=8443", "--infoblox-wapi-username=infoblox", @@ -234,6 +237,7 @@ func TestParseFlags(t *testing.T) { "EXTERNAL_DNS_AZURE_CONFIG_FILE": "azure.json", "EXTERNAL_DNS_AZURE_RESOURCE_GROUP": "arg", "EXTERNAL_DNS_CLOUDFLARE_PROXIED": "1", + "EXTERNAL_DNS_CLOUDFLARE_ZONES_PER_PAGE": "20", "EXTERNAL_DNS_INFOBLOX_GRID_HOST": "127.0.0.1", "EXTERNAL_DNS_INFOBLOX_WAPI_PORT": "8443", "EXTERNAL_DNS_INFOBLOX_WAPI_USERNAME": "infoblox", diff --git a/provider/cloudflare.go b/provider/cloudflare.go index 75e24f414..73bc4b9ed 100644 --- a/provider/cloudflare.go +++ b/provider/cloudflare.go @@ -17,6 +17,7 @@ limitations under the License. package provider import ( + "context" "fmt" "os" "strings" @@ -53,6 +54,7 @@ type cloudFlareDNS interface { UserDetails() (cloudflare.User, error) ZoneIDByName(zoneName string) (string, error) ListZones(zoneID ...string) ([]cloudflare.Zone, error) + ListZonesContext(ctx context.Context, opts ...cloudflare.ReqOption) (cloudflare.ZonesResponse, error) DNSRecords(zoneID string, rr cloudflare.DNSRecord) ([]cloudflare.DNSRecord, error) CreateDNSRecord(zoneID string, rr cloudflare.DNSRecord) (*cloudflare.DNSRecordResponse, error) DeleteDNSRecord(zoneID, recordID string) error @@ -89,14 +91,19 @@ func (z zoneService) DeleteDNSRecord(zoneID, recordID string) error { return z.service.DeleteDNSRecord(zoneID, recordID) } +func (z zoneService) ListZonesContext(ctx context.Context, opts ...cloudflare.ReqOption) (cloudflare.ZonesResponse, error) { + return z.service.ListZonesContext(ctx, opts...) +} + // CloudFlareProvider is an implementation of Provider for CloudFlare DNS. type CloudFlareProvider struct { Client cloudFlareDNS // only consider hosted zones managing domains ending in this suffix - domainFilter DomainFilter - zoneIDFilter ZoneIDFilter - proxied bool - DryRun bool + domainFilter DomainFilter + zoneIDFilter ZoneIDFilter + proxied bool + DryRun bool + PaginationOptions cloudflare.PaginationOptions } // cloudFlareChange differentiates between ChangActions @@ -106,7 +113,7 @@ type cloudFlareChange struct { } // NewCloudFlareProvider initializes a new CloudFlare DNS based Provider. -func NewCloudFlareProvider(domainFilter DomainFilter, zoneIDFilter ZoneIDFilter, proxied bool, dryRun bool) (*CloudFlareProvider, error) { +func NewCloudFlareProvider(domainFilter DomainFilter, zoneIDFilter ZoneIDFilter, zonesPerPage int, proxied bool, dryRun bool) (*CloudFlareProvider, error) { // initialize via API email and API key and returns new API object config, err := cloudflare.New(os.Getenv("CF_API_KEY"), os.Getenv("CF_API_EMAIL")) if err != nil { @@ -119,6 +126,9 @@ func NewCloudFlareProvider(domainFilter DomainFilter, zoneIDFilter ZoneIDFilter, zoneIDFilter: zoneIDFilter, proxied: proxied, DryRun: dryRun, + PaginationOptions: cloudflare.PaginationOptions{ + PerPage: zonesPerPage, + }, } return provider, nil } @@ -126,22 +136,23 @@ func NewCloudFlareProvider(domainFilter DomainFilter, zoneIDFilter ZoneIDFilter, // Zones returns the list of hosted zones. func (p *CloudFlareProvider) Zones() ([]cloudflare.Zone, error) { result := []cloudflare.Zone{} - - zones, err := p.Client.ListZones() + ctx := context.TODO() + zonesResponse, err := p.Client.ListZonesContext(ctx, cloudflare.WithPagination(p.PaginationOptions)) if err != nil { return nil, err } + for pages := 0; pages < zonesResponse.ResultInfo.TotalPages; pages++ { + for _, zone := range zonesResponse.Result { + if !p.domainFilter.Match(zone.Name) { + continue + } - for _, zone := range zones { - if !p.domainFilter.Match(zone.Name) { - continue + if !p.zoneIDFilter.Match(zone.ID) { + continue + } + + result = append(result, zone) } - - if !p.zoneIDFilter.Match(zone.ID) { - continue - } - - result = append(result, zone) } return result, nil diff --git a/provider/cloudflare_test.go b/provider/cloudflare_test.go index cfa54ca21..6c1dfc34c 100644 --- a/provider/cloudflare_test.go +++ b/provider/cloudflare_test.go @@ -17,15 +17,14 @@ limitations under the License. package provider import ( + "context" "fmt" "os" "testing" + cloudflare "github.com/cloudflare/cloudflare-go" "github.com/kubernetes-incubator/external-dns/endpoint" "github.com/kubernetes-incubator/external-dns/plan" - - cloudflare "github.com/cloudflare/cloudflare-go" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -66,6 +65,17 @@ func (m *mockCloudFlareClient) ListZones(zoneID ...string) ([]cloudflare.Zone, e return []cloudflare.Zone{{ID: "1234567890", Name: "ext-dns-test.zalando.to."}, {ID: "1234567891", Name: "foo.com."}}, nil } +func (m *mockCloudFlareClient) ListZonesContext(ctx context.Context, opts ...cloudflare.ReqOption) (cloudflare.ZonesResponse, error) { + return cloudflare.ZonesResponse{ + Result: []cloudflare.Zone{ + {ID: "1234567890", Name: "ext-dns-test.zalando.to."}, + {ID: "1234567891", Name: "foo.com."}}, + ResultInfo: cloudflare.ResultInfo{ + TotalPages: 1, + }, + }, nil +} + type mockCloudFlareUserDetailsFail struct{} func (m *mockCloudFlareUserDetailsFail) CreateDNSRecord(zoneID string, rr cloudflare.DNSRecord) (*cloudflare.DNSRecordResponse, error) { @@ -96,6 +106,10 @@ func (m *mockCloudFlareUserDetailsFail) ListZones(zoneID ...string) ([]cloudflar return []cloudflare.Zone{{Name: "ext-dns-test.zalando.to."}}, nil } +func (m *mockCloudFlareUserDetailsFail) ListZonesContext(ctx context.Context, opts ...cloudflare.ReqOption) (cloudflare.ZonesResponse, error) { + return cloudflare.ZonesResponse{}, nil +} + type mockCloudFlareCreateZoneFail struct{} func (m *mockCloudFlareCreateZoneFail) CreateDNSRecord(zoneID string, rr cloudflare.DNSRecord) (*cloudflare.DNSRecordResponse, error) { @@ -155,6 +169,17 @@ func (m *mockCloudFlareDNSRecordsFail) ListZones(zoneID ...string) ([]cloudflare return []cloudflare.Zone{{Name: "ext-dns-test.zalando.to."}}, nil } +func (m *mockCloudFlareDNSRecordsFail) ListZonesContext(ctx context.Context, opts ...cloudflare.ReqOption) (cloudflare.ZonesResponse, error) { + return cloudflare.ZonesResponse{ + Result: []cloudflare.Zone{ + {ID: "1234567890", Name: "ext-dns-test.zalando.to."}, + {ID: "1234567891", Name: "foo.com."}}, + ResultInfo: cloudflare.ResultInfo{ + TotalPages: 1, + }, + }, nil +} + type mockCloudFlareZoneIDByNameFail struct{} func (m *mockCloudFlareZoneIDByNameFail) CreateDNSRecord(zoneID string, rr cloudflare.DNSRecord) (*cloudflare.DNSRecordResponse, error) { @@ -245,6 +270,10 @@ func (m *mockCloudFlareListZonesFail) ListZones(zoneID ...string) ([]cloudflare. return []cloudflare.Zone{{}}, fmt.Errorf("no zones available") } +func (m *mockCloudFlareListZonesFail) ListZonesContext(ctx context.Context, opts ...cloudflare.ReqOption) (cloudflare.ZonesResponse, error) { + return cloudflare.ZonesResponse{}, fmt.Errorf("no zones available") +} + type mockCloudFlareCreateRecordsFail struct{} func (m *mockCloudFlareCreateRecordsFail) CreateDNSRecord(zoneID string, rr cloudflare.DNSRecord) (*cloudflare.DNSRecordResponse, error) { @@ -275,6 +304,10 @@ func (m *mockCloudFlareCreateRecordsFail) ListZones(zoneID ...string) ([]cloudfl return []cloudflare.Zone{{}}, fmt.Errorf("no zones available") } +func (m *mockCloudFlareCreateRecordsFail) ListZonesContext(ctx context.Context, opts ...cloudflare.ReqOption) (cloudflare.ZonesResponse, error) { + return cloudflare.ZonesResponse{}, nil +} + type mockCloudFlareDeleteRecordsFail struct{} func (m *mockCloudFlareDeleteRecordsFail) CreateDNSRecord(zoneID string, rr cloudflare.DNSRecord) (*cloudflare.DNSRecordResponse, error) { @@ -305,6 +338,10 @@ func (m *mockCloudFlareDeleteRecordsFail) ListZones(zoneID ...string) ([]cloudfl return []cloudflare.Zone{{Name: "ext-dns-test.zalando.to."}}, nil } +func (m *mockCloudFlareDeleteRecordsFail) ListZonesContext(ctx context.Context, opts ...cloudflare.ReqOption) (cloudflare.ZonesResponse, error) { + return cloudflare.ZonesResponse{}, nil +} + type mockCloudFlareUpdateRecordsFail struct{} func (m *mockCloudFlareUpdateRecordsFail) CreateDNSRecord(zoneID string, rr cloudflare.DNSRecord) (*cloudflare.DNSRecordResponse, error) { @@ -335,6 +372,10 @@ func (m *mockCloudFlareUpdateRecordsFail) ListZones(zoneID ...string) ([]cloudfl return []cloudflare.Zone{{Name: "ext-dns-test.zalando.to."}}, nil } +func (m *mockCloudFlareUpdateRecordsFail) ListZonesContext(ctx context.Context, opts ...cloudflare.ReqOption) (cloudflare.ZonesResponse, error) { + return cloudflare.ZonesResponse{}, nil +} + func TestNewCloudFlareChanges(t *testing.T) { expect := []struct { Name string @@ -439,13 +480,23 @@ func TestRecords(t *testing.T) { func TestNewCloudFlareProvider(t *testing.T) { _ = os.Setenv("CF_API_KEY", "xxxxxxxxxxxxxxxxx") _ = os.Setenv("CF_API_EMAIL", "test@test.com") - _, err := NewCloudFlareProvider(NewDomainFilter([]string{"ext-dns-test.zalando.to."}), NewZoneIDFilter([]string{""}), false, true) + _, err := NewCloudFlareProvider( + NewDomainFilter([]string{"ext-dns-test.zalando.to."}), + NewZoneIDFilter([]string{""}), + 1, + false, + true) if err != nil { t.Errorf("should not fail, %s", err) } _ = os.Unsetenv("CF_API_KEY") _ = os.Unsetenv("CF_API_EMAIL") - _, err = NewCloudFlareProvider(NewDomainFilter([]string{"ext-dns-test.zalando.to."}), NewZoneIDFilter([]string{""}), false, true) + _, err = NewCloudFlareProvider( + NewDomainFilter([]string{"ext-dns-test.zalando.to."}), + NewZoneIDFilter([]string{""}), + 50, + false, + true) if err == nil { t.Errorf("expected to fail") }