mirror of
https://github.com/kubernetes-sigs/external-dns.git
synced 2025-08-06 17:46:57 +02:00
feat(aws): fetch zones with tags batching (#5058)
* feat(aws-provider): aws tags batching Signed-off-by: ivan katliarchuk <ivan.katliarchuk@gmail.com> * feat(aws-provider): aws tags batching Signed-off-by: ivan katliarchuk <ivan.katliarchuk@gmail.com> * wip Signed-off-by: ivan katliarchuk <ivan.katliarchuk@gmail.com> * chore(aws-provider): aws tags batching functionality Signed-off-by: ivan katliarchuk <ivan.katliarchuk@gmail.com> * chore(aws-provider): aws tags batching functionality Signed-off-by: ivan katliarchuk <ivan.katliarchuk@gmail.com> * chore(aws-provider): aws tags batching functionality Signed-off-by: ivan katliarchuk <ivan.katliarchuk@gmail.com> * chore(aws-provider): aws tags batching functionality Signed-off-by: ivan katliarchuk <ivan.katliarchuk@gmail.com> * chore(aws-provider): aws tags batching functionality Signed-off-by: ivan katliarchuk <ivan.katliarchuk@gmail.com> * chore(aws-provider): aws tags batching functionality Signed-off-by: ivan katliarchuk <ivan.katliarchuk@gmail.com> * chore(aws-provider): aws tags batching functionality Signed-off-by: ivan katliarchuk <ivan.katliarchuk@gmail.com> * chore(aws-provider): aws tags batching functionality Signed-off-by: ivan katliarchuk <ivan.katliarchuk@gmail.com> * chore(aws-provider): aws tags batching functionality Signed-off-by: ivan katliarchuk <ivan.katliarchuk@gmail.com> * chore(aws-provider): aws tags batching functionality * chore(aws-provider): aws tags batching functionality Signed-off-by: ivan katliarchuk <ivan.katliarchuk@gmail.com> * feat(aws-provider): fetch zones with aws tags batching Signed-off-by: ivan katliarchuk <ivan.katliarchuk@gmail.com> * feat(aws-provider): fetch zones with aws tags batching Signed-off-by: ivan katliarchuk <ivan.katliarchuk@gmail.com> * chore(aws-provider): aws tags batching functionality Signed-off-by: ivan katliarchuk <ivan.katliarchuk@gmail.com> --------- Signed-off-by: ivan katliarchuk <ivan.katliarchuk@gmail.com>
This commit is contained in:
parent
36e5b84093
commit
d4a66bdb66
2
.github/workflows/lint-test-chart.yaml
vendored
2
.github/workflows/lint-test-chart.yaml
vendored
@ -28,7 +28,7 @@ jobs:
|
|||||||
working-directory: charts/external-dns
|
working-directory: charts/external-dns
|
||||||
run: |
|
run: |
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
helm plugin install https://github.com/losisin/helm-values-schema-json.git
|
helm plugin install https://github.com/losisin/helm-values-schema-json.git
|
||||||
helm schema
|
helm schema
|
||||||
if [[ -n "$(git status --porcelain --untracked-files=no)" ]]
|
if [[ -n "$(git status --porcelain --untracked-files=no)" ]]
|
||||||
|
177
docs/tutorials/aws-filters.md
Normal file
177
docs/tutorials/aws-filters.md
Normal file
@ -0,0 +1,177 @@
|
|||||||
|
# AWS Filters
|
||||||
|
|
||||||
|
This document provides guidance on filtering AWS zones using various strategies and flags.
|
||||||
|
|
||||||
|
## Strategies for Scoping Zones
|
||||||
|
|
||||||
|
> Without specifying these flags, management applies to all zones.
|
||||||
|
|
||||||
|
In order to manage specific zones, there is a possibility to combine multiple options
|
||||||
|
|
||||||
|
| Argument | Description | Flow Control |
|
||||||
|
|:---------------------------|:-----------------------------------------------------------|:------------:|
|
||||||
|
| `--zone-id-filter` | Specify multiple times if needed | OR |
|
||||||
|
| `--domain-filter` | By domain suffix - specify multiple times if needed | OR |
|
||||||
|
| `--regex-domain-filter` | By domain suffix but as a regex - overrides domain-filter | AND |
|
||||||
|
| `--exclude-domains` | To exclude a domain or subdomain | OR |
|
||||||
|
| `--regex-domain-exclusion` | Subtracts its matches from `regex-domain-filter`'s matches | AND |
|
||||||
|
| `--aws-zone-type` | Only sync zones of this type `[public\|private]` | OR |
|
||||||
|
| `--aws-zone-tags` | Only sync zones with this tag | AND |
|
||||||
|
|
||||||
|
Minimum required configuration
|
||||||
|
|
||||||
|
```sh
|
||||||
|
args:
|
||||||
|
--provider=aws
|
||||||
|
--registry=txt
|
||||||
|
--source=service
|
||||||
|
```
|
||||||
|
|
||||||
|
### Filter by Zone Type
|
||||||
|
|
||||||
|
> If this flag is not specified, management applies to both public and private zones.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
args:
|
||||||
|
--aws-zone-type=private|public # choose between public or private
|
||||||
|
...
|
||||||
|
```
|
||||||
|
|
||||||
|
### Filter by Domain
|
||||||
|
|
||||||
|
> Specify multiple times if needed.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
args:
|
||||||
|
--domain-filter=example.com
|
||||||
|
--domain-filter=.paradox.example.com
|
||||||
|
...
|
||||||
|
```
|
||||||
|
|
||||||
|
Example `--domain-filter=example.com` will allow for zone `example.com` and any zones that end in `.example.com`, including `an.example.com`, i.e., the subdomains of example.com.
|
||||||
|
|
||||||
|
When there are multiple domains, filter `--domain-filter=example.com` will match domains `example.com`, `ex.par.example.com`, `par.example.com`, `x.par.eu-west-1.example.com`.
|
||||||
|
|
||||||
|
And if the filter is prepended with `.` e.g., `--domain-filter=.example.com` it will allow *only* zones that end in `.example.com`, i.e., the subdomains of example.com but not the `example.com` zone itself. Example result: `ex.par.eu-west-1.example.com`, `ex.par.example.com`, `par.example.com`.
|
||||||
|
|
||||||
|
> Note: if you prepend the filter with ".", it will not attempt to match parent zones.
|
||||||
|
|
||||||
|
### Filter by Zone ID
|
||||||
|
|
||||||
|
> Specify multiple times if needed, the flow logic is OR
|
||||||
|
|
||||||
|
```sh
|
||||||
|
args:
|
||||||
|
--zone-id-filter=ABCDEF12345678
|
||||||
|
--zone-id-filter=XYZDEF12345888
|
||||||
|
...
|
||||||
|
```
|
||||||
|
|
||||||
|
### Filter by Tag
|
||||||
|
|
||||||
|
> Specify multiple times if needed, the flow logic is AND
|
||||||
|
|
||||||
|
Keys only
|
||||||
|
|
||||||
|
```sh
|
||||||
|
args:
|
||||||
|
--aws-zone-tags=owner
|
||||||
|
--aws-zone-tags=vertical
|
||||||
|
```
|
||||||
|
|
||||||
|
Or specify keys with values
|
||||||
|
|
||||||
|
```sh
|
||||||
|
args:
|
||||||
|
--aws-zone-tags=owner=k8s
|
||||||
|
--aws-zone-tags=vertical=k8s
|
||||||
|
```
|
||||||
|
|
||||||
|
Can't specify multiple or separate values with commas: `key1=val1,key2=val2` at the moment.
|
||||||
|
Filter only by value `--aws-zone-tags==tag-value` is not supported.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
args:
|
||||||
|
--aws-zone-tags=team=k8s,vertical=platform # this is not supported
|
||||||
|
--aws-zone-tags==tag-value # this is not supported
|
||||||
|
```
|
||||||
|
|
||||||
|
## Filtering Workflows
|
||||||
|
|
||||||
|
***Filtering Sequence***
|
||||||
|
|
||||||
|
The diagram describes the sequence for filtering AWS zones.
|
||||||
|
|
||||||
|
```mermaid
|
||||||
|
flowchart TD
|
||||||
|
A["zones"] --> B{"Is zonesCache valid?"}
|
||||||
|
B -- Yes --> C["Return cached zones"]
|
||||||
|
B -- No --> D["Initialize zones map"]
|
||||||
|
D --> E["For each profile and client"]
|
||||||
|
E --> F["Create paginator"]
|
||||||
|
F --> G{"Has more pages?"}
|
||||||
|
G -- Yes --> H["Get next page"]
|
||||||
|
H --> I["For each zone in page"]
|
||||||
|
I --> J{"Match zoneIDFilter?"}
|
||||||
|
J -- No --> G
|
||||||
|
J -- Yes --> K{"Match zoneTypeFilter?"}
|
||||||
|
K -- No --> G
|
||||||
|
K -- Yes --> L{"Match domainFilter?"}
|
||||||
|
L -- No --> M{"zoneMatchParent?"}
|
||||||
|
M -- No --> G
|
||||||
|
M -- Yes --> N{"Match domainFilter parent?"}
|
||||||
|
N -- No --> G
|
||||||
|
N -- Yes --> O{"zoneTagFilter specified?"}
|
||||||
|
O -- Yes --> P["Add zone to zonesToValidate"]
|
||||||
|
O -- No --> Q["Add zone to zones map"]
|
||||||
|
P --> Q
|
||||||
|
Q --> G
|
||||||
|
G -- No --> R{"zonesToValidate not empty?"}
|
||||||
|
R -- Yes --> S["Get tags for zones"]
|
||||||
|
S --> T["For each zone and tags"]
|
||||||
|
T --> U{"Match zoneTagFilter?"}
|
||||||
|
U -- No --> V["Delete zone from zones map"]
|
||||||
|
U -- Yes --> W["Keep zone in zones map"]
|
||||||
|
V --> W
|
||||||
|
W --> R
|
||||||
|
R -- No --> X["Update zonesCache"]
|
||||||
|
X --> Y["Return zones"]
|
||||||
|
```
|
||||||
|
|
||||||
|
***Filtering Flow***
|
||||||
|
|
||||||
|
The is a sequence diagram that describes the interaction between `external-dns`, `AWSProvider`, and `Route53Client`
|
||||||
|
during the filtering process. Here is a high-level description:
|
||||||
|
|
||||||
|
```mermaid
|
||||||
|
sequenceDiagram
|
||||||
|
participant external-dns
|
||||||
|
participant AWSProvider
|
||||||
|
participant Route53Client
|
||||||
|
|
||||||
|
external-dns->>AWSProvider: zones
|
||||||
|
alt Cache is valid
|
||||||
|
AWSProvider-->>external-dns: return cached zones
|
||||||
|
else
|
||||||
|
|
||||||
|
AWSProvider->>Route53Client: ListHostedZonesPaginator
|
||||||
|
loop While paginator.HasMorePages
|
||||||
|
Route53Client->>AWSProvider: paginator.NextPage
|
||||||
|
alt ThrottlingException
|
||||||
|
AWSProvider->>external-dns: error
|
||||||
|
else
|
||||||
|
AWSProvider-->>external-dns: return error
|
||||||
|
end
|
||||||
|
AWSProvider->>AWSProvider: Filter zones
|
||||||
|
alt Tags need validation
|
||||||
|
AWSProvider->>Route53Client: ListTagsForResources
|
||||||
|
Route53Client->>AWSProvider: return tags
|
||||||
|
AWSProvider->>AWSProvider: Validate tags
|
||||||
|
end
|
||||||
|
end
|
||||||
|
alt Cache duration > 0
|
||||||
|
AWSProvider->>AWSProvider: Update cache
|
||||||
|
end
|
||||||
|
AWSProvider-->>external-dns: return zones
|
||||||
|
end
|
||||||
|
```
|
@ -1014,97 +1014,3 @@ Because those limits are in place, `aws-batch-change-size` can be set to any val
|
|||||||
## Using CRD source to manage DNS records in AWS
|
## Using CRD source to manage DNS records in AWS
|
||||||
|
|
||||||
Please refer to the [CRD source documentation](../sources/crd.md#example) for more information.
|
Please refer to the [CRD source documentation](../sources/crd.md#example) for more information.
|
||||||
|
|
||||||
## Strategies for Scoping Zones
|
|
||||||
|
|
||||||
> Without specifying these flags, management applies to all zones.
|
|
||||||
|
|
||||||
In order to manage specific zones, you may need to combine multiple options
|
|
||||||
|
|
||||||
| Argument | Description | Flow Control |
|
|
||||||
|:----------------------------|:----------------------------------------------------------------------------|:------------:|
|
|
||||||
| `--zone-id-filter` | Specify multiple times if needed | OR |
|
|
||||||
| `--domain-filter` | By domain suffix - specify multiple times if needed | OR |
|
|
||||||
| `--regex-domain-filter` | By domain suffix but as a regex - overrides domain-filter | AND |
|
|
||||||
| `--exclude-domains` | To exclude a domain or subdomain | OR |
|
|
||||||
| `--regex-domain-exclusion` | Subtracts its matches from `regex-domain-filter`'s matches | AND |
|
|
||||||
| `--aws-zone-type` | Only sync zones of this type `[public\|private]` | OR |
|
|
||||||
| `--aws-zone-tags` | Only sync zones with this tag | AND |
|
|
||||||
|
|
||||||
Minimum required configuration
|
|
||||||
|
|
||||||
```sh
|
|
||||||
args:
|
|
||||||
--provider=aws
|
|
||||||
--registry=txt
|
|
||||||
--source=service
|
|
||||||
```
|
|
||||||
|
|
||||||
### Filter by Zone Type
|
|
||||||
|
|
||||||
> If this flag is not specified, management applies to both public and private zones.
|
|
||||||
|
|
||||||
```sh
|
|
||||||
args:
|
|
||||||
--aws-zone-type=private|public # choose between public or private
|
|
||||||
...
|
|
||||||
```
|
|
||||||
|
|
||||||
### Filter by Domain
|
|
||||||
|
|
||||||
> Specify multiple times if needed.
|
|
||||||
|
|
||||||
```sh
|
|
||||||
args:
|
|
||||||
--domain-filter=example.com
|
|
||||||
--domain-filter=.paradox.example.com
|
|
||||||
...
|
|
||||||
```
|
|
||||||
|
|
||||||
Example `--domain-filter=example.com` will allow for zone `example.com` and any zones that end in `.example.com`, including `an.example.com`, i.e., the subdomains of example.com.
|
|
||||||
|
|
||||||
When there are multiple domains, filter `--domain-filter=example.com` will match domains `example.com`, `ex.par.example.com`, `par.example.com`, `x.par.eu-west-1.example.com`.
|
|
||||||
|
|
||||||
And if the filter is prepended with `.` e.g., `--domain-filter=.example.com` it will allow *only* zones that end in `.example.com`, i.e., the subdomains of example.com but not the `example.com` zone itself. Example result: `ex.par.eu-west-1.example.com`, `ex.par.example.com`, `par.example.com`.
|
|
||||||
|
|
||||||
> Note: if you prepend the filter with ".", it will not attempt to match parent zones.
|
|
||||||
|
|
||||||
### Filter by Zone ID
|
|
||||||
|
|
||||||
> Specify multiple times if needed, the flow logic is OR
|
|
||||||
|
|
||||||
```sh
|
|
||||||
args:
|
|
||||||
--zone-id-filter=ABCDEF12345678
|
|
||||||
--zone-id-filter=XYZDEF12345888
|
|
||||||
...
|
|
||||||
```
|
|
||||||
|
|
||||||
### Filter by Tag
|
|
||||||
|
|
||||||
> Specify multiple times if needed, the flow logic is AND
|
|
||||||
|
|
||||||
Keys only
|
|
||||||
|
|
||||||
```sh
|
|
||||||
args:
|
|
||||||
--aws-zone-tags=owner
|
|
||||||
--aws-zone-tags=vertical
|
|
||||||
```
|
|
||||||
|
|
||||||
Or specify keys with values
|
|
||||||
|
|
||||||
```sh
|
|
||||||
args:
|
|
||||||
--aws-zone-tags=owner=k8s
|
|
||||||
--aws-zone-tags=vertical=k8s
|
|
||||||
```
|
|
||||||
|
|
||||||
Can't specify multiple or separate values with commas: `key1=val1,key2=val2` at the moment.
|
|
||||||
Filter only by value `--aws-zone-tags==tag-value` is not supported.
|
|
||||||
|
|
||||||
```sh
|
|
||||||
args:
|
|
||||||
--aws-zone-tags=team=k8s,vertical=platform # this is not supported
|
|
||||||
--aws-zone-tags==tag-value # this is not supported
|
|
||||||
```
|
|
||||||
|
@ -62,6 +62,9 @@ const (
|
|||||||
providerSpecificMultiValueAnswer = "aws/multi-value-answer"
|
providerSpecificMultiValueAnswer = "aws/multi-value-answer"
|
||||||
providerSpecificHealthCheckID = "aws/health-check-id"
|
providerSpecificHealthCheckID = "aws/health-check-id"
|
||||||
sameZoneAlias = "same-zone"
|
sameZoneAlias = "same-zone"
|
||||||
|
// Currently supported up to 10 health checks or hosted zones.
|
||||||
|
// https://docs.aws.amazon.com/Route53/latest/APIReference/API_ListTagsForResources.html#API_ListTagsForResources_RequestSyntax
|
||||||
|
batchSize = 10
|
||||||
)
|
)
|
||||||
|
|
||||||
// see elb: https://docs.aws.amazon.com/general/latest/gr/elb.html
|
// see elb: https://docs.aws.amazon.com/general/latest/gr/elb.html
|
||||||
@ -205,7 +208,7 @@ type Route53API interface {
|
|||||||
ChangeResourceRecordSets(ctx context.Context, input *route53.ChangeResourceRecordSetsInput, optFns ...func(options *route53.Options)) (*route53.ChangeResourceRecordSetsOutput, error)
|
ChangeResourceRecordSets(ctx context.Context, input *route53.ChangeResourceRecordSetsInput, optFns ...func(options *route53.Options)) (*route53.ChangeResourceRecordSetsOutput, error)
|
||||||
CreateHostedZone(ctx context.Context, input *route53.CreateHostedZoneInput, optFns ...func(*route53.Options)) (*route53.CreateHostedZoneOutput, error)
|
CreateHostedZone(ctx context.Context, input *route53.CreateHostedZoneInput, optFns ...func(*route53.Options)) (*route53.CreateHostedZoneOutput, error)
|
||||||
ListHostedZones(ctx context.Context, input *route53.ListHostedZonesInput, optFns ...func(options *route53.Options)) (*route53.ListHostedZonesOutput, error)
|
ListHostedZones(ctx context.Context, input *route53.ListHostedZonesInput, optFns ...func(options *route53.Options)) (*route53.ListHostedZonesOutput, error)
|
||||||
ListTagsForResource(ctx context.Context, input *route53.ListTagsForResourceInput, optFns ...func(options *route53.Options)) (*route53.ListTagsForResourceOutput, error)
|
ListTagsForResources(ctx context.Context, input *route53.ListTagsForResourcesInput, optFns ...func(options *route53.Options)) (*route53.ListTagsForResourcesOutput, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Route53Change wrapper to handle ownership relation throughout the provider implementation
|
// Route53Change wrapper to handle ownership relation throughout the provider implementation
|
||||||
@ -231,6 +234,29 @@ func (cs Route53Changes) Route53Changes() []route53types.Change {
|
|||||||
return ret
|
return ret
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type zoneTags map[string]map[string]string
|
||||||
|
|
||||||
|
// filterZonesByTags filters the provided zones map by matching the tags against the provider's zoneTagFilter.
|
||||||
|
// It removes any zones from the map that do not match the filter criteria.
|
||||||
|
func (z zoneTags) filterZonesByTags(p *AWSProvider, zones map[string]*profiledZone) {
|
||||||
|
for zone, tags := range z {
|
||||||
|
if !p.zoneTagFilter.Match(tags) {
|
||||||
|
delete(zones, zone)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// append adds tags to the ZoneTags for a given zoneID.
|
||||||
|
func (z zoneTags) append(id string, tags []route53types.Tag) {
|
||||||
|
zoneId := fmt.Sprintf("/hostedzone/%s", id)
|
||||||
|
if _, exists := z[zoneId]; !exists {
|
||||||
|
z[zoneId] = make(map[string]string)
|
||||||
|
}
|
||||||
|
for _, tag := range tags {
|
||||||
|
z[zoneId][*tag.Key] = *tag.Value
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
type zonesListCache struct {
|
type zonesListCache struct {
|
||||||
age time.Time
|
age time.Time
|
||||||
duration time.Duration
|
duration time.Duration
|
||||||
@ -328,7 +354,6 @@ func (p *AWSProvider) zones(ctx context.Context) (map[string]*profiledZone, erro
|
|||||||
zones := make(map[string]*profiledZone)
|
zones := make(map[string]*profiledZone)
|
||||||
|
|
||||||
for profile, client := range p.clients {
|
for profile, client := range p.clients {
|
||||||
var tagErr error
|
|
||||||
paginator := route53.NewListHostedZonesPaginator(client, &route53.ListHostedZonesInput{})
|
paginator := route53.NewListHostedZonesPaginator(client, &route53.ListHostedZonesInput{})
|
||||||
|
|
||||||
for paginator.HasMorePages() {
|
for paginator.HasMorePages() {
|
||||||
@ -342,6 +367,7 @@ func (p *AWSProvider) zones(ctx context.Context) (map[string]*profiledZone, erro
|
|||||||
// nothing to do here. Falling through to general error handling
|
// nothing to do here. Falling through to general error handling
|
||||||
return nil, provider.NewSoftError(fmt.Errorf("failed to list hosted zones: %w", err))
|
return nil, provider.NewSoftError(fmt.Errorf("failed to list hosted zones: %w", err))
|
||||||
}
|
}
|
||||||
|
var zonesToTagFilter []string
|
||||||
for _, zone := range resp.HostedZones {
|
for _, zone := range resp.HostedZones {
|
||||||
if !p.zoneIDFilter.Match(*zone.Id) {
|
if !p.zoneIDFilter.Match(*zone.Id) {
|
||||||
continue
|
continue
|
||||||
@ -360,16 +386,8 @@ func (p *AWSProvider) zones(ctx context.Context) (map[string]*profiledZone, erro
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Only fetch tags if a tag filter was specified
|
|
||||||
if !p.zoneTagFilter.IsEmpty() {
|
if !p.zoneTagFilter.IsEmpty() {
|
||||||
tags, err := p.tagsForZone(ctx, *zone.Id, profile)
|
zonesToTagFilter = append(zonesToTagFilter, cleanZoneID(*zone.Id))
|
||||||
if err != nil {
|
|
||||||
tagErr = err
|
|
||||||
break
|
|
||||||
}
|
|
||||||
if !p.zoneTagFilter.Match(tags) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
zones[*zone.Id] = &profiledZone{
|
zones[*zone.Id] = &profiledZone{
|
||||||
@ -377,14 +395,21 @@ func (p *AWSProvider) zones(ctx context.Context) (map[string]*profiledZone, erro
|
|||||||
zone: &zone,
|
zone: &zone,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
if tagErr != nil {
|
if len(zonesToTagFilter) > 0 {
|
||||||
return nil, provider.NewSoftErrorf("failed to list zones tags: %w", tagErr)
|
if zTags, err := p.tagsForZone(ctx, zonesToTagFilter, profile); err != nil {
|
||||||
|
return nil, provider.NewSoftErrorf("failed to list tags for zones %w", err)
|
||||||
|
} else {
|
||||||
|
zTags.filterZonesByTags(p, zones)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, zone := range zones {
|
if log.IsLevelEnabled(log.DebugLevel) {
|
||||||
log.Debugf("Considering zone: %s (domain: %s)", *zone.zone.Id, *zone.zone.Name)
|
for _, zone := range zones {
|
||||||
|
log.Debugf("Considering zone: %s (domain: %s)", *zone.zone.Id, *zone.zone.Name)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if p.zonesCache.duration > time.Duration(0) {
|
if p.zonesCache.duration > time.Duration(0) {
|
||||||
@ -636,6 +661,7 @@ func (p *AWSProvider) submitChanges(ctx context.Context, changes Route53Changes,
|
|||||||
}
|
}
|
||||||
|
|
||||||
var failedZones []string
|
var failedZones []string
|
||||||
|
debugLevel := log.DebugLevel
|
||||||
for z, cs := range changesByZone {
|
for z, cs := range changesByZone {
|
||||||
log := log.WithFields(log.Fields{
|
log := log.WithFields(log.Fields{
|
||||||
"zoneName": *zones[z].zone.Name,
|
"zoneName": *zones[z].zone.Name,
|
||||||
@ -678,10 +704,11 @@ func (p *AWSProvider) submitChanges(ctx context.Context, changes Route53Changes,
|
|||||||
|
|
||||||
if len(changesByOwnership) > 1 {
|
if len(changesByOwnership) > 1 {
|
||||||
log.Debug("Trying to submit change sets one-by-one instead")
|
log.Debug("Trying to submit change sets one-by-one instead")
|
||||||
|
|
||||||
for _, changes := range changesByOwnership {
|
for _, changes := range changesByOwnership {
|
||||||
for _, c := range changes {
|
if log.Logger.IsLevelEnabled(debugLevel) {
|
||||||
log.Debugf("Desired change: %s %s %s", c.Action, *c.ResourceRecordSet.Name, c.ResourceRecordSet.Type)
|
for _, c := range changes {
|
||||||
|
log.Debugf("Desired change: %s %s %s", c.Action, *c.ResourceRecordSet.Name, c.ResourceRecordSet.Type)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
params.ChangeBatch = &route53types.ChangeBatch{
|
params.ChangeBatch = &route53types.ChangeBatch{
|
||||||
Changes: changes.Route53Changes(),
|
Changes: changes.Route53Changes(),
|
||||||
@ -940,23 +967,29 @@ func groupChangesByNameAndOwnershipRelation(cs Route53Changes) map[string]Route5
|
|||||||
return changesByOwnership
|
return changesByOwnership
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *AWSProvider) tagsForZone(ctx context.Context, zoneID string, profile string) (map[string]string, error) {
|
func (p *AWSProvider) tagsForZone(ctx context.Context, zoneIDs []string, profile string) (zoneTags, error) {
|
||||||
client := p.clients[profile]
|
client := p.clients[profile]
|
||||||
|
|
||||||
// TODO: this will make single API request for each zone, which consumes API requests
|
result := zoneTags{}
|
||||||
// more effective way is to batch requests https://github.com/aws/aws-sdk-go-v2/blob/ed8a3caa0df9ce36a5b60aebeee201187098d205/service/route53/api_op_ListTagsForResources.go#L37
|
|
||||||
response, err := client.ListTagsForResource(ctx, &route53.ListTagsForResourceInput{
|
for i := 0; i < len(zoneIDs); i += batchSize {
|
||||||
ResourceType: route53types.TagResourceTypeHostedzone,
|
batch := zoneIDs[i:min(i+batchSize, len(zoneIDs))]
|
||||||
ResourceId: aws.String(cleanZoneID(zoneID)),
|
if len(batch) == 0 {
|
||||||
})
|
break
|
||||||
if err != nil {
|
}
|
||||||
return nil, provider.NewSoftErrorf("failed to list tags for zone %s: %w", zoneID, err)
|
response, err := client.ListTagsForResources(ctx, &route53.ListTagsForResourcesInput{
|
||||||
|
ResourceType: route53types.TagResourceTypeHostedzone,
|
||||||
|
ResourceIds: batch,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, provider.NewSoftErrorf("failed to list tags for zones. %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, res := range response.ResourceTagSets {
|
||||||
|
result.append(*res.ResourceId, res.Tags)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
tagMap := map[string]string{}
|
return result, nil
|
||||||
for _, tag := range response.ResourceTagSet.Tags {
|
|
||||||
tagMap[*tag.Key] = *tag.Value
|
|
||||||
}
|
|
||||||
return tagMap, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// count bytes for all changes values
|
// count bytes for all changes values
|
||||||
|
@ -18,6 +18,8 @@ package aws
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
log "github.com/sirupsen/logrus"
|
log "github.com/sirupsen/logrus"
|
||||||
@ -57,7 +59,36 @@ func TestAWSZonesFilterWithTags(t *testing.T) {
|
|||||||
z, err := provider.Zones(ctx)
|
z, err := provider.Zones(ctx)
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
assert.EqualValues(t, 24, len(z))
|
assert.EqualValues(t, 24, len(z))
|
||||||
assert.Equal(t, 169, stub.calls["listtagsforresource"])
|
assert.Equal(t, 17, stub.calls["listtagsforresource"])
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAWSZonesFiltersWithTags(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
filters []string
|
||||||
|
want, calls int
|
||||||
|
}{
|
||||||
|
{[]string{"owner=ext-dns"}, 169, 17},
|
||||||
|
{[]string{"domain=n3.n2.n1.ex.com"}, 1, 17},
|
||||||
|
{[]string{"parentdomain=n3.n2.n1.ex.com"}, 1, 17},
|
||||||
|
{[]string{"vpcid=vpc-not-exists"}, 0, 17},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
tName := fmt.Sprintf("filters=%s and zones=%d", strings.Join(tt.filters, ","), tt.want)
|
||||||
|
t.Run(tName, func(t *testing.T) {
|
||||||
|
var zones HostedZones
|
||||||
|
unmarshalTestHelper("/fixtures/160-plus-zones.yaml", &zones, t)
|
||||||
|
|
||||||
|
stub := NewRoute53APIFixtureStub(&zones)
|
||||||
|
provider := providerFilters(stub,
|
||||||
|
WithZoneTagFilters(tt.filters),
|
||||||
|
)
|
||||||
|
z, err := provider.Zones(context.Background())
|
||||||
|
assert.NoError(t, err)
|
||||||
|
assert.EqualValues(t, tt.want, len(z))
|
||||||
|
assert.EqualValues(t, tt.calls, stub.calls["listtagsforresource"])
|
||||||
|
})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAWSZonesSecondRequestHitsTheCache(t *testing.T) {
|
func TestAWSZonesSecondRequestHitsTheCache(t *testing.T) {
|
||||||
|
@ -134,9 +134,9 @@ func (c *Route53APICounter) ListHostedZones(ctx context.Context, input *route53.
|
|||||||
return c.wrapped.ListHostedZones(ctx, input, optFns...)
|
return c.wrapped.ListHostedZones(ctx, input, optFns...)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Route53APICounter) ListTagsForResource(ctx context.Context, input *route53.ListTagsForResourceInput, optFns ...func(options *route53.Options)) (*route53.ListTagsForResourceOutput, error) {
|
func (c *Route53APICounter) ListTagsForResources(ctx context.Context, input *route53.ListTagsForResourcesInput, optFns ...func(options *route53.Options)) (*route53.ListTagsForResourcesOutput, error) {
|
||||||
c.calls["ListTagsForResource"]++
|
c.calls["ListTagsForResource"]++
|
||||||
return c.wrapped.ListTagsForResource(ctx, input, optFns...)
|
return c.wrapped.ListTagsForResources(ctx, input, optFns...)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Route53 stores wildcards escaped: http://docs.aws.amazon.com/Route53/latest/DeveloperGuide/DomainNameFormat.html?shortFooter=true#domain-name-format-asterisk
|
// Route53 stores wildcards escaped: http://docs.aws.amazon.com/Route53/latest/DeveloperGuide/DomainNameFormat.html?shortFooter=true#domain-name-format-asterisk
|
||||||
@ -161,22 +161,26 @@ func specialCharactersEscape(s string) string {
|
|||||||
return result.String()
|
return result.String()
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *Route53APIStub) ListTagsForResource(ctx context.Context, input *route53.ListTagsForResourceInput, optFns ...func(options *route53.Options)) (*route53.ListTagsForResourceOutput, error) {
|
func (r *Route53APIStub) ListTagsForResources(ctx context.Context, input *route53.ListTagsForResourcesInput, optFns ...func(options *route53.Options)) (*route53.ListTagsForResourcesOutput, error) {
|
||||||
if input.ResourceType == route53types.TagResourceTypeHostedzone {
|
if input.ResourceType == route53types.TagResourceTypeHostedzone {
|
||||||
zoneId := fmt.Sprintf("/%s/%s", input.ResourceType, *input.ResourceId)
|
var sets []route53types.ResourceTagSet
|
||||||
if strings.Contains(zoneId, "ext-dns-test-error-on-list-tags") {
|
for _, el := range input.ResourceIds {
|
||||||
return nil, fmt.Errorf("operation error Route53APIStub: ListTagsForResource")
|
zoneId := fmt.Sprintf("/%s/%s", input.ResourceType, el)
|
||||||
|
if strings.Contains(zoneId, "ext-dns-test-error-on-list-tags") {
|
||||||
|
return nil, fmt.Errorf("operation error Route53APIStub: ListTagsForResource")
|
||||||
|
}
|
||||||
|
|
||||||
|
if r.zoneTags[zoneId] != nil {
|
||||||
|
sets = append(sets, route53types.ResourceTagSet{
|
||||||
|
ResourceId: &el,
|
||||||
|
ResourceType: route53types.TagResourceTypeHostedzone,
|
||||||
|
Tags: r.zoneTags[zoneId],
|
||||||
|
})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
tags := r.zoneTags[zoneId]
|
return &route53.ListTagsForResourcesOutput{ResourceTagSets: sets}, nil
|
||||||
return &route53.ListTagsForResourceOutput{
|
|
||||||
ResourceTagSet: &route53types.ResourceTagSet{
|
|
||||||
ResourceId: input.ResourceId,
|
|
||||||
ResourceType: input.ResourceType,
|
|
||||||
Tags: tags,
|
|
||||||
},
|
|
||||||
}, nil
|
|
||||||
}
|
}
|
||||||
return &route53.ListTagsForResourceOutput{}, nil
|
return &route53.ListTagsForResourcesOutput{}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *Route53APIStub) ChangeResourceRecordSets(ctx context.Context, input *route53.ChangeResourceRecordSetsInput, optFns ...func(options *route53.Options)) (*route53.ChangeResourceRecordSetsOutput, error) {
|
func (r *Route53APIStub) ChangeResourceRecordSets(ctx context.Context, input *route53.ChangeResourceRecordSetsInput, optFns ...func(options *route53.Options)) (*route53.ChangeResourceRecordSetsOutput, error) {
|
||||||
@ -333,7 +337,8 @@ func TestAWSZones(t *testing.T) {
|
|||||||
{"private filter", provider.NewZoneIDFilter([]string{}), provider.NewZoneTypeFilter("private"), provider.NewZoneTagFilter([]string{}), privateZones},
|
{"private filter", provider.NewZoneIDFilter([]string{}), provider.NewZoneTypeFilter("private"), provider.NewZoneTagFilter([]string{}), privateZones},
|
||||||
{"unknown filter", provider.NewZoneIDFilter([]string{}), provider.NewZoneTypeFilter("unknown"), provider.NewZoneTagFilter([]string{}), noZones},
|
{"unknown filter", provider.NewZoneIDFilter([]string{}), provider.NewZoneTypeFilter("unknown"), provider.NewZoneTagFilter([]string{}), noZones},
|
||||||
{"zone id filter", provider.NewZoneIDFilter([]string{"/hostedzone/zone-3.ext-dns-test-2.teapot.zalan.do."}), provider.NewZoneTypeFilter(""), provider.NewZoneTagFilter([]string{}), privateZones},
|
{"zone id filter", provider.NewZoneIDFilter([]string{"/hostedzone/zone-3.ext-dns-test-2.teapot.zalan.do."}), provider.NewZoneTypeFilter(""), provider.NewZoneTagFilter([]string{}), privateZones},
|
||||||
{"tag filter", provider.NewZoneIDFilter([]string{}), provider.NewZoneTypeFilter(""), provider.NewZoneTagFilter([]string{"zone=3"}), privateZones},
|
{"tag filter zero zone match", provider.NewZoneIDFilter([]string{}), provider.NewZoneTypeFilter(""), provider.NewZoneTagFilter([]string{"zone=not-exists"}), noZones},
|
||||||
|
{"tag filter single zone match", provider.NewZoneIDFilter([]string{}), provider.NewZoneTypeFilter(""), provider.NewZoneTagFilter([]string{"zone=3"}), privateZones},
|
||||||
} {
|
} {
|
||||||
t.Run(ti.msg, func(t *testing.T) {
|
t.Run(ti.msg, func(t *testing.T) {
|
||||||
provider, _ := newAWSProviderWithTagFilter(t, endpoint.NewDomainFilter([]string{"ext-dns-test-2.teapot.zalan.do."}), ti.zoneIDFilter, ti.zoneTypeFilter, ti.zoneTagFilter, defaultEvaluateTargetHealth, false, nil)
|
provider, _ := newAWSProviderWithTagFilter(t, endpoint.NewDomainFilter([]string{"ext-dns-test-2.teapot.zalan.do."}), ti.zoneIDFilter, ti.zoneTypeFilter, ti.zoneTagFilter, defaultEvaluateTargetHealth, false, nil)
|
||||||
@ -364,7 +369,7 @@ func TestAWSZonesWithTagFilterError(t *testing.T) {
|
|||||||
})
|
})
|
||||||
_, err := provider.Zones(context.Background())
|
_, err := provider.Zones(context.Background())
|
||||||
require.Error(t, err)
|
require.Error(t, err)
|
||||||
require.ErrorContains(t, err, "failed to list tags for zone /hostedzone/zone-2.ext-dns-test-error-on-list-tags")
|
require.ErrorContains(t, err, "failed to list tags for zones")
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAWSRecordsFilter(t *testing.T) {
|
func TestAWSRecordsFilter(t *testing.T) {
|
||||||
|
@ -124,16 +124,21 @@ func (r Route53APIFixtureStub) ListHostedZones(ctx context.Context, input *route
|
|||||||
return output, nil
|
return output, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r Route53APIFixtureStub) ListTagsForResource(ctx context.Context, input *route53.ListTagsForResourceInput, optFns ...func(options *route53.Options)) (*route53.ListTagsForResourceOutput, error) {
|
func (r Route53APIFixtureStub) ListTagsForResources(ctx context.Context, input *route53.ListTagsForResourcesInput, optFns ...func(options *route53.Options)) (*route53.ListTagsForResourcesOutput, error) {
|
||||||
r.calls["listtagsforresource"]++
|
r.calls["listtagsforresource"]++
|
||||||
tags := r.zoneTags[*input.ResourceId]
|
|
||||||
return &route53.ListTagsForResourceOutput{
|
var sets []route53types.ResourceTagSet
|
||||||
ResourceTagSet: &route53types.ResourceTagSet{
|
|
||||||
ResourceId: input.ResourceId,
|
for _, el := range input.ResourceIds {
|
||||||
ResourceType: input.ResourceType,
|
if r.zoneTags[el] != nil {
|
||||||
Tags: tags,
|
sets = append(sets, route53types.ResourceTagSet{
|
||||||
},
|
ResourceId: &el,
|
||||||
}, nil
|
ResourceType: route53types.TagResourceTypeHostedzone,
|
||||||
|
Tags: r.zoneTags[el],
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return &route53.ListTagsForResourcesOutput{ResourceTagSets: sets}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func unmarshalTestHelper(input string, obj any, t *testing.T) {
|
func unmarshalTestHelper(input string, obj any, t *testing.T) {
|
||||||
|
Loading…
Reference in New Issue
Block a user