Cloudflare pagination for zones

Gopkg.lock

@@ -128,12 +128,12 @@
   revision = "4c0e84591b9aa9e6dcfdf3e020114cd81f89d5f9"
 
 [[projects]]
-  digest = "1:85fd00554a6ed5b33687684b76635d532c74141508b5bce2843d85e8a3c9dc91"
+  branch = "master"
+  digest = "1:3e90c0d9954bf53a2061c4a0f193e6569c9ab2118c8f3a250871498f6b4645e5"
   name = "github.com/cloudflare/cloudflare-go"
   packages = ["."]
   pruneopts = ""
-  revision = "4c6994ac3877fbb627766edadc67f4e816e8c890"
+  revision = "0c85496d873009e53e64d391ade643e7ac02e0d4"
-  version = "v0.7.4"
 
 [[projects]]
   digest = "1:eaeede87b418b97f9dee473f8940fd9b65ca5cdac0503350c7c8f8965ea3cf4d"

Gopkg.toml

@@ -21,8 +21,8 @@ ignored = ["github.com/kubernetes/repo-infra/kazel"]
   version = "~1.13.7"
 
 [[constraint]]
+  branch = "master"
   name = "github.com/cloudflare/cloudflare-go"
-  version = "0.7.3"
 
 [[constraint]]
   name = "github.com/digitalocean/godo"

main.go

@@ -129,7 +129,7 @@ func main() {
 	case "azure":
 		p, err = provider.NewAzureProvider(cfg.AzureConfigFile, domainFilter, zoneIDFilter, cfg.AzureResourceGroup, cfg.DryRun)
 	case "cloudflare":
-		p, err = provider.NewCloudFlareProvider(domainFilter, zoneIDFilter, cfg.CloudflareProxied, cfg.DryRun)
+		p, err = provider.NewCloudFlareProvider(domainFilter, zoneIDFilter, cfg.CloudflareZonesPerPage, cfg.CloudflareProxied, cfg.DryRun)
 	case "google":
 		p, err = provider.NewGoogleProvider(cfg.GoogleProject, domainFilter, zoneIDFilter, cfg.DryRun)
 	case "digitalocean":

pkg/apis/externaldns/types.go

@@ -64,6 +64,7 @@ type Config struct {
 	AzureConfigFile          string
 	AzureResourceGroup       string
 	CloudflareProxied        bool
+	CloudflareZonesPerPage   int
 	InfobloxGridHost         string
 	InfobloxWapiPort         int
 	InfobloxWapiUsername     string
@@ -136,6 +137,7 @@ var defaultConfig = &Config{
 	AzureConfigFile:          "/etc/kubernetes/azure.json",
 	AzureResourceGroup:       "",
 	CloudflareProxied:        false,
+	CloudflareZonesPerPage:   50,
 	InfobloxGridHost:         "",
 	InfobloxWapiPort:         443,
 	InfobloxWapiUsername:     "admin",
@@ -251,6 +253,7 @@ func (cfg *Config) ParseFlags(args []string) error {
 	app.Flag("azure-config-file", "When using the Azure provider, specify the Azure configuration file (required when --provider=azure").Default(defaultConfig.AzureConfigFile).StringVar(&cfg.AzureConfigFile)
 	app.Flag("azure-resource-group", "When using the Azure provider, override the Azure resource group to use (optional)").Default(defaultConfig.AzureResourceGroup).StringVar(&cfg.AzureResourceGroup)
 	app.Flag("cloudflare-proxied", "When using the Cloudflare provider, specify if the proxy mode must be enabled (default: disabled)").BoolVar(&cfg.CloudflareProxied)
+	app.Flag("cloudflare-zones-per-page", "When using the Cloudflare provider, specify how many zones per page listed, max. possible 50 (default: 50)").Default(strconv.Itoa(defaultConfig.CloudflareZonesPerPage)).IntVar(&cfg.CloudflareZonesPerPage)
 	app.Flag("infoblox-grid-host", "When using the Infoblox provider, specify the Grid Manager host (required when --provider=infoblox)").Default(defaultConfig.InfobloxGridHost).StringVar(&cfg.InfobloxGridHost)
 	app.Flag("infoblox-wapi-port", "When using the Infoblox provider, specify the WAPI port (default: 443)").Default(strconv.Itoa(defaultConfig.InfobloxWapiPort)).IntVar(&cfg.InfobloxWapiPort)
 	app.Flag("infoblox-wapi-username", "When using the Infoblox provider, specify the WAPI username (default: admin)").Default(defaultConfig.InfobloxWapiUsername).StringVar(&cfg.InfobloxWapiUsername)

pkg/apis/externaldns/types_test.go

@@ -51,6 +51,7 @@ var (
 		AzureConfigFile:         "/etc/kubernetes/azure.json",
 		AzureResourceGroup:      "",
 		CloudflareProxied:       false,
+		CloudflareZonesPerPage:  50,
 		InfobloxGridHost:        "",
 		InfobloxWapiPort:        443,
 		InfobloxWapiUsername:    "admin",
@@ -103,6 +104,7 @@ var (
 		AzureConfigFile:         "azure.json",
 		AzureResourceGroup:      "arg",
 		CloudflareProxied:       true,
+		CloudflareZonesPerPage:  20,
 		InfobloxGridHost:        "127.0.0.1",
 		InfobloxWapiPort:        8443,
 		InfobloxWapiUsername:    "infoblox",
@@ -171,6 +173,7 @@ func TestParseFlags(t *testing.T) {
 				"--azure-config-file=azure.json",
 				"--azure-resource-group=arg",
 				"--cloudflare-proxied",
+				"--cloudflare-zones-per-page=20",
 				"--infoblox-grid-host=127.0.0.1",
 				"--infoblox-wapi-port=8443",
 				"--infoblox-wapi-username=infoblox",
@@ -234,6 +237,7 @@ func TestParseFlags(t *testing.T) {
 				"EXTERNAL_DNS_AZURE_CONFIG_FILE":          "azure.json",
 				"EXTERNAL_DNS_AZURE_RESOURCE_GROUP":       "arg",
 				"EXTERNAL_DNS_CLOUDFLARE_PROXIED":         "1",
+				"EXTERNAL_DNS_CLOUDFLARE_ZONES_PER_PAGE":  "20",
 				"EXTERNAL_DNS_INFOBLOX_GRID_HOST":         "127.0.0.1",
 				"EXTERNAL_DNS_INFOBLOX_WAPI_PORT":         "8443",
 				"EXTERNAL_DNS_INFOBLOX_WAPI_USERNAME":     "infoblox",

provider/cloudflare.go

@@ -17,6 +17,7 @@ limitations under the License.
 package provider
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"strings"
@@ -53,6 +54,7 @@ type cloudFlareDNS interface {
 	UserDetails() (cloudflare.User, error)
 	ZoneIDByName(zoneName string) (string, error)
 	ListZones(zoneID ...string) ([]cloudflare.Zone, error)
+	ListZonesContext(ctx context.Context, opts ...cloudflare.ReqOption) (cloudflare.ZonesResponse, error)
 	DNSRecords(zoneID string, rr cloudflare.DNSRecord) ([]cloudflare.DNSRecord, error)
 	CreateDNSRecord(zoneID string, rr cloudflare.DNSRecord) (*cloudflare.DNSRecordResponse, error)
 	DeleteDNSRecord(zoneID, recordID string) error
@@ -89,14 +91,19 @@ func (z zoneService) DeleteDNSRecord(zoneID, recordID string) error {
 	return z.service.DeleteDNSRecord(zoneID, recordID)
 }
 
+func (z zoneService) ListZonesContext(ctx context.Context, opts ...cloudflare.ReqOption) (cloudflare.ZonesResponse, error) {
+	return z.service.ListZonesContext(ctx, opts...)
+}
+
 // CloudFlareProvider is an implementation of Provider for CloudFlare DNS.
 type CloudFlareProvider struct {
 	Client cloudFlareDNS
 	// only consider hosted zones managing domains ending in this suffix
-	domainFilter DomainFilter
+	domainFilter      DomainFilter
-	zoneIDFilter ZoneIDFilter
+	zoneIDFilter      ZoneIDFilter
-	proxied      bool
+	proxied           bool
-	DryRun       bool
+	DryRun            bool
+	PaginationOptions cloudflare.PaginationOptions
 }
 
 // cloudFlareChange differentiates between ChangActions
@@ -106,7 +113,7 @@ type cloudFlareChange struct {
 }
 
 // NewCloudFlareProvider initializes a new CloudFlare DNS based Provider.
-func NewCloudFlareProvider(domainFilter DomainFilter, zoneIDFilter ZoneIDFilter, proxied bool, dryRun bool) (*CloudFlareProvider, error) {
+func NewCloudFlareProvider(domainFilter DomainFilter, zoneIDFilter ZoneIDFilter, zonesPerPage int, proxied bool, dryRun bool) (*CloudFlareProvider, error) {
 	// initialize via API email and API key and returns new API object
 	config, err := cloudflare.New(os.Getenv("CF_API_KEY"), os.Getenv("CF_API_EMAIL"))
 	if err != nil {
@@ -119,6 +126,9 @@ func NewCloudFlareProvider(domainFilter DomainFilter, zoneIDFilter ZoneIDFilter,
 		zoneIDFilter: zoneIDFilter,
 		proxied:      proxied,
 		DryRun:       dryRun,
+		PaginationOptions: cloudflare.PaginationOptions{
+			PerPage: zonesPerPage,
+		},
 	}
 	return provider, nil
 }
@@ -126,22 +136,23 @@ func NewCloudFlareProvider(domainFilter DomainFilter, zoneIDFilter ZoneIDFilter,
 // Zones returns the list of hosted zones.
 func (p *CloudFlareProvider) Zones() ([]cloudflare.Zone, error) {
 	result := []cloudflare.Zone{}
-
+	ctx := context.TODO()
-	zones, err := p.Client.ListZones()
+	zonesResponse, err := p.Client.ListZonesContext(ctx, cloudflare.WithPagination(p.PaginationOptions))
 	if err != nil {
 		return nil, err
 	}
+	for pages := 0; pages < zonesResponse.ResultInfo.TotalPages; pages++ {
+		for _, zone := range zonesResponse.Result {
+			if !p.domainFilter.Match(zone.Name) {
+				continue
+			}
 
-	for _, zone := range zones {
+			if !p.zoneIDFilter.Match(zone.ID) {
-		if !p.domainFilter.Match(zone.Name) {
+				continue
-			continue
+			}
+
+			result = append(result, zone)
 		}
-
-		if !p.zoneIDFilter.Match(zone.ID) {
-			continue
-		}
-
-		result = append(result, zone)
 	}
 
 	return result, nil

provider/cloudflare_test.go

@@ -17,15 +17,14 @@ limitations under the License.
 package provider
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"testing"
 
+	cloudflare "github.com/cloudflare/cloudflare-go"
 	"github.com/kubernetes-incubator/external-dns/endpoint"
 	"github.com/kubernetes-incubator/external-dns/plan"
-
-	cloudflare "github.com/cloudflare/cloudflare-go"
-
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -66,6 +65,17 @@ func (m *mockCloudFlareClient) ListZones(zoneID ...string) ([]cloudflare.Zone, e
 	return []cloudflare.Zone{{ID: "1234567890", Name: "ext-dns-test.zalando.to."}, {ID: "1234567891", Name: "foo.com."}}, nil
 }
 
+func (m *mockCloudFlareClient) ListZonesContext(ctx context.Context, opts ...cloudflare.ReqOption) (cloudflare.ZonesResponse, error) {
+	return cloudflare.ZonesResponse{
+		Result: []cloudflare.Zone{
+			{ID: "1234567890", Name: "ext-dns-test.zalando.to."},
+			{ID: "1234567891", Name: "foo.com."}},
+		ResultInfo: cloudflare.ResultInfo{
+			TotalPages: 1,
+		},
+	}, nil
+}
+
 type mockCloudFlareUserDetailsFail struct{}
 
 func (m *mockCloudFlareUserDetailsFail) CreateDNSRecord(zoneID string, rr cloudflare.DNSRecord) (*cloudflare.DNSRecordResponse, error) {
@@ -96,6 +106,10 @@ func (m *mockCloudFlareUserDetailsFail) ListZones(zoneID ...string) ([]cloudflar
 	return []cloudflare.Zone{{Name: "ext-dns-test.zalando.to."}}, nil
 }
 
+func (m *mockCloudFlareUserDetailsFail) ListZonesContext(ctx context.Context, opts ...cloudflare.ReqOption) (cloudflare.ZonesResponse, error) {
+	return cloudflare.ZonesResponse{}, nil
+}
+
 type mockCloudFlareCreateZoneFail struct{}
 
 func (m *mockCloudFlareCreateZoneFail) CreateDNSRecord(zoneID string, rr cloudflare.DNSRecord) (*cloudflare.DNSRecordResponse, error) {
@@ -155,6 +169,17 @@ func (m *mockCloudFlareDNSRecordsFail) ListZones(zoneID ...string) ([]cloudflare
 	return []cloudflare.Zone{{Name: "ext-dns-test.zalando.to."}}, nil
 }
 
+func (m *mockCloudFlareDNSRecordsFail) ListZonesContext(ctx context.Context, opts ...cloudflare.ReqOption) (cloudflare.ZonesResponse, error) {
+	return cloudflare.ZonesResponse{
+		Result: []cloudflare.Zone{
+			{ID: "1234567890", Name: "ext-dns-test.zalando.to."},
+			{ID: "1234567891", Name: "foo.com."}},
+		ResultInfo: cloudflare.ResultInfo{
+			TotalPages: 1,
+		},
+	}, nil
+}
+
 type mockCloudFlareZoneIDByNameFail struct{}
 
 func (m *mockCloudFlareZoneIDByNameFail) CreateDNSRecord(zoneID string, rr cloudflare.DNSRecord) (*cloudflare.DNSRecordResponse, error) {
@@ -245,6 +270,10 @@ func (m *mockCloudFlareListZonesFail) ListZones(zoneID ...string) ([]cloudflare.
 	return []cloudflare.Zone{{}}, fmt.Errorf("no zones available")
 }
 
+func (m *mockCloudFlareListZonesFail) ListZonesContext(ctx context.Context, opts ...cloudflare.ReqOption) (cloudflare.ZonesResponse, error) {
+	return cloudflare.ZonesResponse{}, fmt.Errorf("no zones available")
+}
+
 type mockCloudFlareCreateRecordsFail struct{}
 
 func (m *mockCloudFlareCreateRecordsFail) CreateDNSRecord(zoneID string, rr cloudflare.DNSRecord) (*cloudflare.DNSRecordResponse, error) {
@@ -275,6 +304,10 @@ func (m *mockCloudFlareCreateRecordsFail) ListZones(zoneID ...string) ([]cloudfl
 	return []cloudflare.Zone{{}}, fmt.Errorf("no zones available")
 }
 
+func (m *mockCloudFlareCreateRecordsFail) ListZonesContext(ctx context.Context, opts ...cloudflare.ReqOption) (cloudflare.ZonesResponse, error) {
+	return cloudflare.ZonesResponse{}, nil
+}
+
 type mockCloudFlareDeleteRecordsFail struct{}
 
 func (m *mockCloudFlareDeleteRecordsFail) CreateDNSRecord(zoneID string, rr cloudflare.DNSRecord) (*cloudflare.DNSRecordResponse, error) {
@@ -305,6 +338,10 @@ func (m *mockCloudFlareDeleteRecordsFail) ListZones(zoneID ...string) ([]cloudfl
 	return []cloudflare.Zone{{Name: "ext-dns-test.zalando.to."}}, nil
 }
 
+func (m *mockCloudFlareDeleteRecordsFail) ListZonesContext(ctx context.Context, opts ...cloudflare.ReqOption) (cloudflare.ZonesResponse, error) {
+	return cloudflare.ZonesResponse{}, nil
+}
+
 type mockCloudFlareUpdateRecordsFail struct{}
 
 func (m *mockCloudFlareUpdateRecordsFail) CreateDNSRecord(zoneID string, rr cloudflare.DNSRecord) (*cloudflare.DNSRecordResponse, error) {
@@ -335,6 +372,10 @@ func (m *mockCloudFlareUpdateRecordsFail) ListZones(zoneID ...string) ([]cloudfl
 	return []cloudflare.Zone{{Name: "ext-dns-test.zalando.to."}}, nil
 }
 
+func (m *mockCloudFlareUpdateRecordsFail) ListZonesContext(ctx context.Context, opts ...cloudflare.ReqOption) (cloudflare.ZonesResponse, error) {
+	return cloudflare.ZonesResponse{}, nil
+}
+
 func TestNewCloudFlareChanges(t *testing.T) {
 	expect := []struct {
 		Name string
@@ -439,13 +480,23 @@ func TestRecords(t *testing.T) {
 func TestNewCloudFlareProvider(t *testing.T) {
 	_ = os.Setenv("CF_API_KEY", "xxxxxxxxxxxxxxxxx")
 	_ = os.Setenv("CF_API_EMAIL", "test@test.com")
-	_, err := NewCloudFlareProvider(NewDomainFilter([]string{"ext-dns-test.zalando.to."}), NewZoneIDFilter([]string{""}), false, true)
+	_, err := NewCloudFlareProvider(
+		NewDomainFilter([]string{"ext-dns-test.zalando.to."}),
+		NewZoneIDFilter([]string{""}),
+		1,
+		false,
+		true)
 	if err != nil {
 		t.Errorf("should not fail, %s", err)
 	}
 	_ = os.Unsetenv("CF_API_KEY")
 	_ = os.Unsetenv("CF_API_EMAIL")
-	_, err = NewCloudFlareProvider(NewDomainFilter([]string{"ext-dns-test.zalando.to."}), NewZoneIDFilter([]string{""}), false, true)
+	_, err = NewCloudFlareProvider(
+		NewDomainFilter([]string{"ext-dns-test.zalando.to."}),
+		NewZoneIDFilter([]string{""}),
+		50,
+		false,
+		true)
 	if err == nil {
 		t.Errorf("expected to fail")
 	}