From ba5afe9518dd6c9d653cc1a512ca41b82c8587f8 Mon Sep 17 00:00:00 2001 From: Hugome Date: Sun, 7 Jun 2020 05:00:36 +0200 Subject: [PATCH] Add OVH API rate limiting option --- go.mod | 1 + go.sum | 2 ++ main.go | 2 +- pkg/apis/externaldns/types.go | 3 +++ pkg/apis/externaldns/types_test.go | 4 ++++ provider/ovh/ovh.go | 22 ++++++++++++++++++---- provider/ovh/ovh_test.go | 16 +++++++++------- 7 files changed, 38 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 59025b32c..f867c11fe 100644 --- a/go.mod +++ b/go.mod @@ -51,6 +51,7 @@ require ( github.com/vinyldns/go-vinyldns v0.0.0-20190611170422-7119fe55ed92 github.com/vultr/govultr v0.3.2 go.etcd.io/etcd v0.5.0-alpha.5.0.20200401174654-e694b7bb0875 + go.uber.org/ratelimit v0.1.0 golang.org/x/net v0.0.0-20200202094626-16171245cfb2 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 google.golang.org/api v0.15.0 diff --git a/go.sum b/go.sum index a4af0ab59..4747dabaf 100644 --- a/go.sum +++ b/go.sum @@ -550,6 +550,8 @@ go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/multierr v0.0.0-20180122172545-ddea229ff1df/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/ratelimit v0.1.0 h1:U2AruXqeTb4Eh9sYQSTrMhH8Cb7M0Ian2ibBOnBcnAw= +go.uber.org/ratelimit v0.1.0/go.mod h1:2X8KaoNd1J0lZV+PxJk/5+DGbO/tpwLR1m++a7FnB/Y= go.uber.org/zap v0.0.0-20180814183419-67bc79d13d15/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= diff --git a/main.go b/main.go index 0c98bf053..80d3dcf23 100644 --- a/main.go +++ b/main.go @@ -200,7 +200,7 @@ func main() { case "digitalocean": p, err = digitalocean.NewDigitalOceanProvider(ctx, domainFilter, cfg.DryRun, cfg.DigitalOceanAPIPageSize) case "ovh": - p, err = ovh.NewOVHProvider(ctx, domainFilter, cfg.OVHEndpoint, cfg.DryRun) + p, err = ovh.NewOVHProvider(ctx, domainFilter, cfg.OVHEndpoint, cfg.OVHApiRateLimit, cfg.DryRun) case "linode": p, err = linode.NewLinodeProvider(domainFilter, cfg.DryRun, externaldns.Version) case "dnsimple": diff --git a/pkg/apis/externaldns/types.go b/pkg/apis/externaldns/types.go index c2c6c06c3..3017a35bc 100644 --- a/pkg/apis/externaldns/types.go +++ b/pkg/apis/externaldns/types.go @@ -99,6 +99,7 @@ type Config struct { OCIConfigFile string InMemoryZones []string OVHEndpoint string + OVHApiRateLimit int PDNSServer string PDNSAPIKey string `secure:"yes"` PDNSTLSEnabled bool @@ -197,6 +198,7 @@ var defaultConfig = &Config{ OCIConfigFile: "/etc/kubernetes/oci.yaml", InMemoryZones: []string{}, OVHEndpoint: "ovh-eu", + OVHApiRateLimit: 20, PDNSServer: "http://localhost:8081", PDNSAPIKey: "", PDNSTLSEnabled: false, @@ -360,6 +362,7 @@ func (cfg *Config) ParseFlags(args []string) error { app.Flag("rcodezero-txt-encrypt", "When using the Rcodezero provider with txt registry option, set if TXT rrs are encrypted (default: false)").Default(strconv.FormatBool(defaultConfig.RcodezeroTXTEncrypt)).BoolVar(&cfg.RcodezeroTXTEncrypt) app.Flag("inmemory-zone", "Provide a list of pre-configured zones for the inmemory provider; specify multiple times for multiple zones (optional)").Default("").StringsVar(&cfg.InMemoryZones) app.Flag("ovh-endpoint", "When using the OVH provider, specify the endpoint (default: ovh-eu)").Default(defaultConfig.OVHEndpoint).StringVar(&cfg.OVHEndpoint) + app.Flag("ovh-api-rate-limit", "When using the OVH provider, specify the API request rate limit, X operations by seconds (default: 20)").Default(strconv.Itoa(defaultConfig.OVHApiRateLimit)).IntVar(&cfg.OVHApiRateLimit) app.Flag("pdns-server", "When using the PowerDNS/PDNS provider, specify the URL to the pdns server (required when --provider=pdns)").Default(defaultConfig.PDNSServer).StringVar(&cfg.PDNSServer) app.Flag("pdns-api-key", "When using the PowerDNS/PDNS provider, specify the API key to use to authorize requests (required when --provider=pdns)").Default(defaultConfig.PDNSAPIKey).StringVar(&cfg.PDNSAPIKey) app.Flag("pdns-tls-enabled", "When using the PowerDNS/PDNS provider, specify whether to use TLS (default: false, requires --tls-ca, optionally specify --tls-client-cert and --tls-client-cert-key)").Default(strconv.FormatBool(defaultConfig.PDNSTLSEnabled)).BoolVar(&cfg.PDNSTLSEnabled) diff --git a/pkg/apis/externaldns/types_test.go b/pkg/apis/externaldns/types_test.go index 929f5f1cc..506ea2f43 100644 --- a/pkg/apis/externaldns/types_test.go +++ b/pkg/apis/externaldns/types_test.go @@ -75,6 +75,7 @@ var ( OCIConfigFile: "/etc/kubernetes/oci.yaml", InMemoryZones: []string{""}, OVHEndpoint: "ovh-eu", + OVHApiRateLimit: 20, PDNSServer: "http://localhost:8081", PDNSAPIKey: "", Policy: "sync", @@ -149,6 +150,7 @@ var ( OCIConfigFile: "oci.yaml", InMemoryZones: []string{"example.org", "company.com"}, OVHEndpoint: "ovh-ca", + OVHApiRateLimit: 42, PDNSServer: "http://ns.example.com:8081", PDNSAPIKey: "some-secret-key", PDNSTLSEnabled: true, @@ -237,6 +239,7 @@ func TestParseFlags(t *testing.T) { "--inmemory-zone=example.org", "--inmemory-zone=company.com", "--ovh-endpoint=ovh-ca", + "--ovh-api-rate-limit=42", "--pdns-server=http://ns.example.com:8081", "--pdns-api-key=some-secret-key", "--pdns-tls-enabled", @@ -326,6 +329,7 @@ func TestParseFlags(t *testing.T) { "EXTERNAL_DNS_OCI_CONFIG_FILE": "oci.yaml", "EXTERNAL_DNS_INMEMORY_ZONE": "example.org\ncompany.com", "EXTERNAL_DNS_OVH_ENDPOINT": "ovh-ca", + "EXTERNAL_DNS_OVH_API_RATE_LIMIT": "42", "EXTERNAL_DNS_DOMAIN_FILTER": "example.org\ncompany.com", "EXTERNAL_DNS_EXCLUDE_DOMAINS": "xapi.example.org\nxapi.company.com", "EXTERNAL_DNS_PDNS_SERVER": "http://ns.example.com:8081", diff --git a/provider/ovh/ovh.go b/provider/ovh/ovh.go index a32260e1b..fdb324bf7 100644 --- a/provider/ovh/ovh.go +++ b/provider/ovh/ovh.go @@ -29,6 +29,8 @@ import ( "sigs.k8s.io/external-dns/endpoint" "sigs.k8s.io/external-dns/plan" "sigs.k8s.io/external-dns/provider" + + "go.uber.org/ratelimit" ) const ( @@ -50,6 +52,8 @@ type OVHProvider struct { client ovhClient + apiRateLimiter ratelimit.Limiter + domainFilter endpoint.DomainFilter DryRun bool } @@ -79,7 +83,7 @@ type ovhChange struct { } // NewOVHProvider initializes a new OVH DNS based Provider. -func NewOVHProvider(ctx context.Context, domainFilter endpoint.DomainFilter, endpoint string, dryRun bool) (*OVHProvider, error) { +func NewOVHProvider(ctx context.Context, domainFilter endpoint.DomainFilter, endpoint string, apiRateLimit int, dryRun bool) (*OVHProvider, error) { client, err := ovh.NewEndpointClient(endpoint) if err != nil { return nil, err @@ -89,9 +93,10 @@ func NewOVHProvider(ctx context.Context, domainFilter endpoint.DomainFilter, end return nil, ErrNoDryRun } return &OVHProvider{ - client: client, - domainFilter: domainFilter, - DryRun: dryRun, + client: client, + domainFilter: domainFilter, + apiRateLimiter: ratelimit.New(apiRateLimit), + DryRun: dryRun, }, nil } @@ -149,10 +154,14 @@ func (p *OVHProvider) ApplyChanges(ctx context.Context, changes *plan.Changes) e func (p *OVHProvider) refresh(zone string) error { log.Debugf("OVH: Refresh %s zone", zone) + + p.apiRateLimiter.Take() return p.client.Post(fmt.Sprintf("/domain/zone/%s/refresh", zone), nil, nil) } func (p *OVHProvider) change(change ovhChange) error { + p.apiRateLimiter.Take() + switch change.Action { case ovhCreate: log.Debugf("OVH: Add an entry to %s", change.String()) @@ -194,6 +203,7 @@ func (p *OVHProvider) zones() ([]string, error) { zones := []string{} filteredZones := []string{} + p.apiRateLimiter.Take() if err := p.client.Get("/domain/zone", &zones); err != nil { return nil, err } @@ -213,6 +223,8 @@ func (p *OVHProvider) records(ctx *context.Context, zone *string, records chan<- eg, _ := errgroup.WithContext(*ctx) log.Debugf("OVH: Getting records for %s", *zone) + + p.apiRateLimiter.Take() if err := p.client.Get(fmt.Sprintf("/domain/zone/%s/record", *zone), &recordsIds); err != nil { return err } @@ -236,6 +248,8 @@ func (p *OVHProvider) record(zone *string, id uint64, records chan<- ovhRecord) record := ovhRecord{} log.Debugf("OVH: Getting record %d for %s", id, *zone) + + p.apiRateLimiter.Take() if err := p.client.Get(fmt.Sprintf("/domain/zone/%s/record/%d", *zone, id), &record); err != nil { return err } diff --git a/provider/ovh/ovh_test.go b/provider/ovh/ovh_test.go index 59b336941..b7d5dd78d 100644 --- a/provider/ovh/ovh_test.go +++ b/provider/ovh/ovh_test.go @@ -25,6 +25,7 @@ import ( "github.com/ovh/go-ovh/ovh" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" + "go.uber.org/ratelimit" "sigs.k8s.io/external-dns/endpoint" "sigs.k8s.io/external-dns/plan" ) @@ -58,8 +59,9 @@ func TestOvhZones(t *testing.T) { assert := assert.New(t) client := new(mockOvhClient) provider := &OVHProvider{ - client: client, - domainFilter: endpoint.NewDomainFilter([]string{"com"}), + client: client, + apiRateLimiter: ratelimit.New(10), + domainFilter: endpoint.NewDomainFilter([]string{"com"}), } // Basic zones @@ -81,7 +83,7 @@ func TestOvhZones(t *testing.T) { func TestOvhZoneRecords(t *testing.T) { assert := assert.New(t) client := new(mockOvhClient) - provider := &OVHProvider{client: client} + provider := &OVHProvider{client: client, apiRateLimiter: ratelimit.New(10)} // Basic zones records client.On("Get", "/domain/zone").Return([]string{"example.org"}, nil).Once() @@ -125,7 +127,7 @@ func TestOvhZoneRecords(t *testing.T) { func TestOvhRecords(t *testing.T) { assert := assert.New(t) client := new(mockOvhClient) - provider := &OVHProvider{client: client} + provider := &OVHProvider{client: client, apiRateLimiter: ratelimit.New(10)} // Basic zones records client.On("Get", "/domain/zone").Return([]string{"example.org", "example.net"}, nil).Once() @@ -158,7 +160,7 @@ func TestOvhRecords(t *testing.T) { func TestOvhRefresh(t *testing.T) { client := new(mockOvhClient) - provider := &OVHProvider{client: client} + provider := &OVHProvider{client: client, apiRateLimiter: ratelimit.New(10)} // Basic zone refresh client.On("Post", "/domain/zone/example.net/refresh", nil).Return(nil, nil).Once() @@ -199,7 +201,7 @@ func TestOvhNewChange(t *testing.T) { func TestOvhApplyChanges(t *testing.T) { assert := assert.New(t) client := new(mockOvhClient) - provider := &OVHProvider{client: client} + provider := &OVHProvider{client: client, apiRateLimiter: ratelimit.New(10)} changes := plan.Changes{ Create: []*endpoint.Endpoint{ {DNSName: ".example.net", RecordType: "A", RecordTTL: 10, Targets: []string{"203.0.113.42"}}, @@ -252,7 +254,7 @@ func TestOvhApplyChanges(t *testing.T) { func TestOvhChange(t *testing.T) { assert := assert.New(t) client := new(mockOvhClient) - provider := &OVHProvider{client: client} + provider := &OVHProvider{client: client, apiRateLimiter: ratelimit.New(10)} // Record creation client.On("Post", "/domain/zone/example.net/record", ovhRecordFields{SubDomain: "ovh"}).Return(nil, nil).Once()