mirrors/external-dns
1
0
Fork 0
mirror of https://github.com/kubernetes-sigs/external-dns.git synced 2025-08-06 09:36:58 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #4021 from danie1sullivan/fix-dynamodb-registry-loop

Browse Source 
fix: dynamodb registry when statements over 25
This commit is contained in:
Kubernetes Prow Robot 2023-11-10 10:02:39 +01:00 committed by GitHub
commit 4941d0cf3e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 126 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
registry/dynamodb.go
View File

@ -68,6 +68,9 @@ type DynamoDBRegistry struct {
const dynamodbAttributeMigrate = "dynamodb/needs-migration"
// DynamoDB allows a maximum batch size of 25 items.
var dynamodbMaxBatchSize uint8 = 25
// NewDynamoDBRegistry returns a new DynamoDBRegistry object.
func NewDynamoDBRegistry(provider provider.Provider, ownerID string, dynamodbAPI DynamoDBAPI, table string, txtPrefix, txtSuffix, txtWildcardReplacement string, managedRecordTypes, excludeRecordTypes []string, txtEncryptAESKey []byte, cacheInterval time.Duration) (*DynamoDBRegistry, error) {
if ownerID == "" {
@ -477,9 +480,9 @@ func (im *DynamoDBRegistry) appendDelete(statements []*dynamodb.BatchStatementRe
func (im *DynamoDBRegistry) executeStatements(ctx context.Context, statements []*dynamodb.BatchStatementRequest, handleErr func(request *dynamodb.BatchStatementRequest, response *dynamodb.BatchStatementResponse) error) error {
for len(statements) > 0 {
var chunk []*dynamodb.BatchStatementRequest
if len(statements) > 25 {
chunk = chunk[:25]
statements = statements[25:]
if len(statements) > int(dynamodbMaxBatchSize) {
chunk = statements[:dynamodbMaxBatchSize]
statements = statements[dynamodbMaxBatchSize:]
} else {
chunk = statements
statements = nil

120
registry/dynamodb_test.go
View File

@ -217,6 +217,7 @@ func TestDynamoDBRegistryRecords(t *testing.T) {
func TestDynamoDBRegistryApplyChanges(t *testing.T) {
for _, tc := range []struct {
name string
maxBatchSize uint8
stubConfig DynamoDBStubConfig
addRecords []*endpoint.Endpoint
changes plan.Changes
@ -294,6 +295,118 @@ func TestDynamoDBRegistryApplyChanges(t *testing.T) {
},
},
},
{
name: "create more entries than DynamoDB batch size limit",
maxBatchSize: 2,
changes: plan.Changes{
Create: []*endpoint.Endpoint{
{
DNSName: "new1.test-zone.example.org",
Targets: endpoint.Targets{"new1.loadbalancer.com"},
RecordType: endpoint.RecordTypeCNAME,
SetIdentifier: "set-new",
Labels: map[string]string{
endpoint.ResourceLabelKey: "ingress/default/new1-ingress",
},
},
{
DNSName: "new2.test-zone.example.org",
Targets: endpoint.Targets{"new2.loadbalancer.com"},
RecordType: endpoint.RecordTypeCNAME,
SetIdentifier: "set-new",
Labels: map[string]string{
endpoint.ResourceLabelKey: "ingress/default/new2-ingress",
},
},
{
DNSName: "new3.test-zone.example.org",
Targets: endpoint.Targets{"new3.loadbalancer.com"},
RecordType: endpoint.RecordTypeCNAME,
SetIdentifier: "set-new",
Labels: map[string]string{
endpoint.ResourceLabelKey: "ingress/default/new3-ingress",
},
},
},
},
stubConfig: DynamoDBStubConfig{
ExpectInsert: map[string]map[string]string{
"new1.test-zone.example.org#CNAME#set-new": {endpoint.ResourceLabelKey: "ingress/default/new1-ingress"},
"new2.test-zone.example.org#CNAME#set-new": {endpoint.ResourceLabelKey: "ingress/default/new2-ingress"},
"new3.test-zone.example.org#CNAME#set-new": {endpoint.ResourceLabelKey: "ingress/default/new3-ingress"},
},
ExpectDelete: sets.New("quux.test-zone.example.org#A#set-2"),
},
expectedRecords: []*endpoint.Endpoint{
{
DNSName: "foo.test-zone.example.org",
Targets: endpoint.Targets{"foo.loadbalancer.com"},
RecordType: endpoint.RecordTypeCNAME,
Labels: map[string]string{
endpoint.OwnerLabelKey: "",
},
},
{
DNSName: "bar.test-zone.example.org",
Targets: endpoint.Targets{"my-domain.com"},
RecordType: endpoint.RecordTypeCNAME,
Labels: map[string]string{
endpoint.OwnerLabelKey: "test-owner",
endpoint.ResourceLabelKey: "ingress/default/my-ingress",
},
},
{
DNSName: "baz.test-zone.example.org",
Targets: endpoint.Targets{"1.1.1.1"},
RecordType: endpoint.RecordTypeA,
SetIdentifier: "set-1",
Labels: map[string]string{
endpoint.OwnerLabelKey: "test-owner",
endpoint.ResourceLabelKey: "ingress/default/my-ingress",
},
},
{
DNSName: "baz.test-zone.example.org",
Targets: endpoint.Targets{"2.2.2.2"},
RecordType: endpoint.RecordTypeA,
SetIdentifier: "set-2",
Labels: map[string]string{
endpoint.OwnerLabelKey: "test-owner",
endpoint.ResourceLabelKey: "ingress/default/other-ingress",
},
},
{
DNSName: "new1.test-zone.example.org",
Targets: endpoint.Targets{"new1.loadbalancer.com"},
RecordType: endpoint.RecordTypeCNAME,
SetIdentifier: "set-new",
Labels: map[string]string{
endpoint.OwnerLabelKey: "test-owner",
endpoint.ResourceLabelKey: "ingress/default/new1-ingress",
},
},
{
DNSName: "new2.test-zone.example.org",
Targets: endpoint.Targets{"new2.loadbalancer.com"},
RecordType: endpoint.RecordTypeCNAME,
SetIdentifier: "set-new",
Labels: map[string]string{
endpoint.OwnerLabelKey: "test-owner",
endpoint.ResourceLabelKey: "ingress/default/new2-ingress",
},
},
{
DNSName: "new3.test-zone.example.org",
Targets: endpoint.Targets{"new3.loadbalancer.com"},
RecordType: endpoint.RecordTypeCNAME,
SetIdentifier: "set-new",
Labels: map[string]string{
endpoint.OwnerLabelKey: "test-owner",
endpoint.ResourceLabelKey: "ingress/default/new3-ingress",
},
},
},
},
{
name: "create orphaned",
changes: plan.Changes{
@ -911,6 +1024,11 @@ func TestDynamoDBRegistryApplyChanges(t *testing.T) {
},
} {
t.Run(tc.name, func(t *testing.T) {
originalMaxBatchSize := dynamodbMaxBatchSize
if tc.maxBatchSize > 0 {
dynamodbMaxBatchSize = tc.maxBatchSize
}
api, p := newDynamoDBAPIStub(t, &tc.stubConfig)
if len(tc.addRecords) > 0 {
_ = p.(*wrappedProvider).Provider.ApplyChanges(context.Background(), &plan.Changes{
@ -945,6 +1063,8 @@ func TestDynamoDBRegistryApplyChanges(t *testing.T) {
if tc.expectedError == "" {
assert.Empty(t, r.orphanedLabels)
}
dynamodbMaxBatchSize = originalMaxBatchSize
})
}
}