Add OVH API rate limiting option

go.mod

@@ -51,6 +51,7 @@ require (
 	github.com/vinyldns/go-vinyldns v0.0.0-20190611170422-7119fe55ed92
 	github.com/vultr/govultr v0.3.2
 	go.etcd.io/etcd v0.5.0-alpha.5.0.20200401174654-e694b7bb0875
+	go.uber.org/ratelimit v0.1.0
 	golang.org/x/net v0.0.0-20200202094626-16171245cfb2
 	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
 	google.golang.org/api v0.15.0

go.sum

@@ -550,6 +550,8 @@ go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/multierr v0.0.0-20180122172545-ddea229ff1df/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
 go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
+go.uber.org/ratelimit v0.1.0 h1:U2AruXqeTb4Eh9sYQSTrMhH8Cb7M0Ian2ibBOnBcnAw=
+go.uber.org/ratelimit v0.1.0/go.mod h1:2X8KaoNd1J0lZV+PxJk/5+DGbO/tpwLR1m++a7FnB/Y=
 go.uber.org/zap v0.0.0-20180814183419-67bc79d13d15/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM=

main.go

@@ -200,7 +200,7 @@ func main() {
 	case "digitalocean":
 		p, err = digitalocean.NewDigitalOceanProvider(ctx, domainFilter, cfg.DryRun, cfg.DigitalOceanAPIPageSize)
 	case "ovh":
-		p, err = ovh.NewOVHProvider(ctx, domainFilter, cfg.OVHEndpoint, cfg.DryRun)
+		p, err = ovh.NewOVHProvider(ctx, domainFilter, cfg.OVHEndpoint, cfg.OVHApiRateLimit, cfg.DryRun)
 	case "linode":
 		p, err = linode.NewLinodeProvider(domainFilter, cfg.DryRun, externaldns.Version)
 	case "dnsimple":

pkg/apis/externaldns/types.go

@@ -99,6 +99,7 @@ type Config struct {
 	OCIConfigFile                     string
 	InMemoryZones                     []string
 	OVHEndpoint                       string
+	OVHApiRateLimit                   int
 	PDNSServer                        string
 	PDNSAPIKey                        string `secure:"yes"`
 	PDNSTLSEnabled                    bool
@@ -197,6 +198,7 @@ var defaultConfig = &Config{
 	OCIConfigFile:               "/etc/kubernetes/oci.yaml",
 	InMemoryZones:               []string{},
 	OVHEndpoint:                 "ovh-eu",
+	OVHApiRateLimit:             20,
 	PDNSServer:                  "http://localhost:8081",
 	PDNSAPIKey:                  "",
 	PDNSTLSEnabled:              false,
@@ -360,6 +362,7 @@ func (cfg *Config) ParseFlags(args []string) error {
 	app.Flag("rcodezero-txt-encrypt", "When using the Rcodezero provider with txt registry option, set if TXT rrs are encrypted (default: false)").Default(strconv.FormatBool(defaultConfig.RcodezeroTXTEncrypt)).BoolVar(&cfg.RcodezeroTXTEncrypt)
 	app.Flag("inmemory-zone", "Provide a list of pre-configured zones for the inmemory provider; specify multiple times for multiple zones (optional)").Default("").StringsVar(&cfg.InMemoryZones)
 	app.Flag("ovh-endpoint", "When using the OVH provider, specify the endpoint (default: ovh-eu)").Default(defaultConfig.OVHEndpoint).StringVar(&cfg.OVHEndpoint)
+	app.Flag("ovh-api-rate-limit", "When using the OVH provider, specify the API request rate limit, X operations by seconds (default: 20)").Default(strconv.Itoa(defaultConfig.OVHApiRateLimit)).IntVar(&cfg.OVHApiRateLimit)
 	app.Flag("pdns-server", "When using the PowerDNS/PDNS provider, specify the URL to the pdns server (required when --provider=pdns)").Default(defaultConfig.PDNSServer).StringVar(&cfg.PDNSServer)
 	app.Flag("pdns-api-key", "When using the PowerDNS/PDNS provider, specify the API key to use to authorize requests (required when --provider=pdns)").Default(defaultConfig.PDNSAPIKey).StringVar(&cfg.PDNSAPIKey)
 	app.Flag("pdns-tls-enabled", "When using the PowerDNS/PDNS provider, specify whether to use TLS (default: false, requires --tls-ca, optionally specify --tls-client-cert and --tls-client-cert-key)").Default(strconv.FormatBool(defaultConfig.PDNSTLSEnabled)).BoolVar(&cfg.PDNSTLSEnabled)

pkg/apis/externaldns/types_test.go

@@ -75,6 +75,7 @@ var (
 		OCIConfigFile:               "/etc/kubernetes/oci.yaml",
 		InMemoryZones:               []string{""},
 		OVHEndpoint:                 "ovh-eu",
+		OVHApiRateLimit:             20,
 		PDNSServer:                  "http://localhost:8081",
 		PDNSAPIKey:                  "",
 		Policy:                      "sync",
@@ -149,6 +150,7 @@ var (
 		OCIConfigFile:               "oci.yaml",
 		InMemoryZones:               []string{"example.org", "company.com"},
 		OVHEndpoint:                 "ovh-ca",
+		OVHApiRateLimit:             42,
 		PDNSServer:                  "http://ns.example.com:8081",
 		PDNSAPIKey:                  "some-secret-key",
 		PDNSTLSEnabled:              true,
@@ -237,6 +239,7 @@ func TestParseFlags(t *testing.T) {
 				"--inmemory-zone=example.org",
 				"--inmemory-zone=company.com",
 				"--ovh-endpoint=ovh-ca",
+				"--ovh-api-rate-limit=42",
 				"--pdns-server=http://ns.example.com:8081",
 				"--pdns-api-key=some-secret-key",
 				"--pdns-tls-enabled",
@@ -326,6 +329,7 @@ func TestParseFlags(t *testing.T) {
 				"EXTERNAL_DNS_OCI_CONFIG_FILE":                 "oci.yaml",
 				"EXTERNAL_DNS_INMEMORY_ZONE":                   "example.org\ncompany.com",
 				"EXTERNAL_DNS_OVH_ENDPOINT":                    "ovh-ca",
+				"EXTERNAL_DNS_OVH_API_RATE_LIMIT":              "42",
 				"EXTERNAL_DNS_DOMAIN_FILTER":                   "example.org\ncompany.com",
 				"EXTERNAL_DNS_EXCLUDE_DOMAINS":                 "xapi.example.org\nxapi.company.com",
 				"EXTERNAL_DNS_PDNS_SERVER":                     "http://ns.example.com:8081",

provider/ovh/ovh.go

@@ -29,6 +29,8 @@ import (
 	"sigs.k8s.io/external-dns/endpoint"
 	"sigs.k8s.io/external-dns/plan"
 	"sigs.k8s.io/external-dns/provider"
+
+	"go.uber.org/ratelimit"
 )
 
 const (
@@ -50,6 +52,8 @@ type OVHProvider struct {
 
 	client ovhClient
 
+	apiRateLimiter ratelimit.Limiter
+
 	domainFilter endpoint.DomainFilter
 	DryRun       bool
 }
@@ -79,7 +83,7 @@ type ovhChange struct {
 }
 
 // NewOVHProvider initializes a new OVH DNS based Provider.
-func NewOVHProvider(ctx context.Context, domainFilter endpoint.DomainFilter, endpoint string, dryRun bool) (*OVHProvider, error) {
+func NewOVHProvider(ctx context.Context, domainFilter endpoint.DomainFilter, endpoint string, apiRateLimit int, dryRun bool) (*OVHProvider, error) {
 	client, err := ovh.NewEndpointClient(endpoint)
 	if err != nil {
 		return nil, err
@@ -91,6 +95,7 @@ func NewOVHProvider(ctx context.Context, domainFilter endpoint.DomainFilter, end
 	return &OVHProvider{
 		client:         client,
 		domainFilter:   domainFilter,
+		apiRateLimiter: ratelimit.New(apiRateLimit),
 		DryRun:         dryRun,
 	}, nil
 }
@@ -149,10 +154,14 @@ func (p *OVHProvider) ApplyChanges(ctx context.Context, changes *plan.Changes) e
 
 func (p *OVHProvider) refresh(zone string) error {
 	log.Debugf("OVH: Refresh %s zone", zone)
+
+	p.apiRateLimiter.Take()
 	return p.client.Post(fmt.Sprintf("/domain/zone/%s/refresh", zone), nil, nil)
 }
 
 func (p *OVHProvider) change(change ovhChange) error {
+	p.apiRateLimiter.Take()
+
 	switch change.Action {
 	case ovhCreate:
 		log.Debugf("OVH: Add an entry to %s", change.String())
@@ -194,6 +203,7 @@ func (p *OVHProvider) zones() ([]string, error) {
 	zones := []string{}
 	filteredZones := []string{}
 
+	p.apiRateLimiter.Take()
 	if err := p.client.Get("/domain/zone", &zones); err != nil {
 		return nil, err
 	}
@@ -213,6 +223,8 @@ func (p *OVHProvider) records(ctx *context.Context, zone *string, records chan<-
 	eg, _ := errgroup.WithContext(*ctx)
 
 	log.Debugf("OVH: Getting records for %s", *zone)
+
+	p.apiRateLimiter.Take()
 	if err := p.client.Get(fmt.Sprintf("/domain/zone/%s/record", *zone), &recordsIds); err != nil {
 		return err
 	}
@@ -236,6 +248,8 @@ func (p *OVHProvider) record(zone *string, id uint64, records chan<- ovhRecord)
 	record := ovhRecord{}
 
 	log.Debugf("OVH: Getting record %d for %s", id, *zone)
+
+	p.apiRateLimiter.Take()
 	if err := p.client.Get(fmt.Sprintf("/domain/zone/%s/record/%d", *zone, id), &record); err != nil {
 		return err
 	}

provider/ovh/ovh_test.go

@@ -25,6 +25,7 @@ import (
 	"github.com/ovh/go-ovh/ovh"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
+	"go.uber.org/ratelimit"
 	"sigs.k8s.io/external-dns/endpoint"
 	"sigs.k8s.io/external-dns/plan"
 )
@@ -59,6 +60,7 @@ func TestOvhZones(t *testing.T) {
 	client := new(mockOvhClient)
 	provider := &OVHProvider{
 		client:         client,
+		apiRateLimiter: ratelimit.New(10),
 		domainFilter:   endpoint.NewDomainFilter([]string{"com"}),
 	}
 
@@ -81,7 +83,7 @@ func TestOvhZones(t *testing.T) {
 func TestOvhZoneRecords(t *testing.T) {
 	assert := assert.New(t)
 	client := new(mockOvhClient)
-	provider := &OVHProvider{client: client}
+	provider := &OVHProvider{client: client, apiRateLimiter: ratelimit.New(10)}
 
 	// Basic zones records
 	client.On("Get", "/domain/zone").Return([]string{"example.org"}, nil).Once()
@@ -125,7 +127,7 @@ func TestOvhZoneRecords(t *testing.T) {
 func TestOvhRecords(t *testing.T) {
 	assert := assert.New(t)
 	client := new(mockOvhClient)
-	provider := &OVHProvider{client: client}
+	provider := &OVHProvider{client: client, apiRateLimiter: ratelimit.New(10)}
 
 	// Basic zones records
 	client.On("Get", "/domain/zone").Return([]string{"example.org", "example.net"}, nil).Once()
@@ -158,7 +160,7 @@ func TestOvhRecords(t *testing.T) {
 
 func TestOvhRefresh(t *testing.T) {
 	client := new(mockOvhClient)
-	provider := &OVHProvider{client: client}
+	provider := &OVHProvider{client: client, apiRateLimiter: ratelimit.New(10)}
 
 	// Basic zone refresh
 	client.On("Post", "/domain/zone/example.net/refresh", nil).Return(nil, nil).Once()
@@ -199,7 +201,7 @@ func TestOvhNewChange(t *testing.T) {
 func TestOvhApplyChanges(t *testing.T) {
 	assert := assert.New(t)
 	client := new(mockOvhClient)
-	provider := &OVHProvider{client: client}
+	provider := &OVHProvider{client: client, apiRateLimiter: ratelimit.New(10)}
 	changes := plan.Changes{
 		Create: []*endpoint.Endpoint{
 			{DNSName: ".example.net", RecordType: "A", RecordTTL: 10, Targets: []string{"203.0.113.42"}},
@@ -252,7 +254,7 @@ func TestOvhApplyChanges(t *testing.T) {
 func TestOvhChange(t *testing.T) {
 	assert := assert.New(t)
 	client := new(mockOvhClient)
-	provider := &OVHProvider{client: client}
+	provider := &OVHProvider{client: client, apiRateLimiter: ratelimit.New(10)}
 
 	// Record creation
 	client.On("Post", "/domain/zone/example.net/record", ovhRecordFields{SubDomain: "ovh"}).Return(nil, nil).Once()