cloudflare: bugfix - do not attempt to create unconfigured empty custom hostnames; improve tests; streamline logic (#5146)

improve test coverage

test the edge case when the custom hostname has changed during the record deletion

don't use custom hostnames if Cloudflare for SaaS fails to authenticate

Use new --cloudflare-custom-hostnames flag to enable cloudflare custom hostnames support

custom hostnames flags --cloudflare-custom-hostnames-min-tls-version and --cloudflare-custom-hostnames-certificate-authority support

markdown lint

Update cloudflare.md

docs/flags.md

@@ -87,6 +87,9 @@
 | `--tencent-cloud-config-file="/etc/kubernetes/tencent-cloud.json"` | When using the Tencent Cloud provider, specify the Tencent Cloud configuration file (required when --provider=tencentcloud) |
 | `--tencent-cloud-zone-type=` | When using the Tencent Cloud provider, filter for zones with visibility (optional, options: public, private) |
 | `--[no-]cloudflare-proxied` | When using the Cloudflare provider, specify if the proxy mode must be enabled (default: disabled) |
+| `--[no-]cloudflare-custom-hostnames` | When using the Cloudflare provider, specify if the Custom Hostnames feature will be used. Requires "Cloudflare for SaaS" enabled. (default: disabled) |
+| `--cloudflare-custom-hostnames-min-tls-version=1.0` | When using the Cloudflare provider with the Custom Hostnames, specify which Minimum TLS Version will be used by default. (default: 1.0, options: 1.0, 1.1, 1.2, 1.3) |
+| `--cloudflare-custom-hostnames-certificate-authority=google` | When using the Cloudflare provider with the Custom Hostnames, specify which Cerrtificate Authority will be used by default. (default: google, options: google, ssl_com, lets_encrypt) |
 | `--cloudflare-dns-records-per-page=100` | When using the Cloudflare provider, specify how many DNS records listed per page, max possible 5,000 (default: 100) |
 | `--cloudflare-region-key=CLOUDFLARE-REGION-KEY` | When using the Cloudflare provider, specify the region (default: earth) |
 | `--coredns-prefix="/skydns/"` | When using the CoreDNS provider, specify the prefix name |

docs/tutorials/cloudflare.md

@@ -310,7 +310,14 @@ If not set the value will default to `global`.
 
 ## Setting cloudflare-custom-hostname
 
-Using the `external-dns.alpha.kubernetes.io/cloudflare-custom-hostname: "<custom hostname>"` annotation, you can have [custom hostnames](https://developers.cloudflare.com/cloudflare-for-platforms/cloudflare-for-saas/domain-support/) automatically managed for A/CNAME record as a custom origin.
+You can automatically configure custom hostnames for A/CNAME DNS records (as custom origins) using the `--cloudflare-custom-hostnames` flag and the `external-dns.alpha.kubernetes.io/cloudflare-custom-hostname: "<custom hostname>"` annotation.
+
+See [Cloudflare for Platforms](https://developers.cloudflare.com/cloudflare-for-platforms/cloudflare-for-saas/domain-support/) for more information on custom hostnames.
+
+This feature is disabled by default and supports the `--cloudflare-custom-hostnames-min-tls-version` and `--cloudflare-custom-hostnames-certificate-authority` flags.
+
+The custom hostname DNS must resolve to the Cloudflare DNS record (`external-dns.alpha.kubernetes.io/hostname`) for automatic certificate validation via the HTTP method. It's important to note that the TXT method does not allow automatic validation and is not supported.
+
 Requires [Cloudflare for SaaS](https://developers.cloudflare.com/cloudflare-for-platforms/cloudflare-for-saas/) product and "SSL and Certificates" API permission.
 
 ## Using CRD source to manage DNS records in Cloudflare

main.go

@@ -250,7 +250,18 @@ func main() {
 	case "civo":
 		p, err = civo.NewCivoProvider(domainFilter, cfg.DryRun)
 	case "cloudflare":
-		p, err = cloudflare.NewCloudFlareProvider(domainFilter, zoneIDFilter, cfg.CloudflareProxied, cfg.DryRun, cfg.CloudflareDNSRecordsPerPage, cfg.CloudflareRegionKey)
+		p, err = cloudflare.NewCloudFlareProvider(
+			domainFilter,
+			zoneIDFilter,
+			cfg.CloudflareProxied,
+			cfg.DryRun,
+			cfg.CloudflareDNSRecordsPerPage,
+			cfg.CloudflareRegionKey,
+			cloudflare.CustomHostnamesConfig{
+				Enabled:              cfg.CloudflareCustomHostnames,
+				MinTLSVersion:        cfg.CloudflareCustomHostnamesMinTLSVersion,
+				CertificateAuthority: cfg.CloudflareCustomHostnamesCertificateAuthority,
+			})
 	case "google":
 		p, err = google.NewGoogleProvider(ctx, cfg.GoogleProject, domainFilter, zoneIDFilter, cfg.GoogleBatchChangeSize, cfg.GoogleBatchChangeInterval, cfg.GoogleZoneVisibility, cfg.DryRun)
 	case "digitalocean":

pkg/apis/externaldns/types.go

@@ -111,6 +111,9 @@ type Config struct {
 	AzureActiveDirectoryAuthorityHost             string
 	AzureZonesCacheDuration                       time.Duration
 	CloudflareProxied                             bool
+	CloudflareCustomHostnames                     bool
+	CloudflareCustomHostnamesMinTLSVersion        string
+	CloudflareCustomHostnamesCertificateAuthority string
 	CloudflareDNSRecordsPerPage                   int
 	CloudflareRegionKey                           string
 	CoreDNSPrefix                                 string
@@ -274,6 +277,9 @@ var defaultConfig = &Config{
 	AzureSubscriptionID:                    "",
 	AzureZonesCacheDuration:                0 * time.Second,
 	CloudflareProxied:                      false,
+	CloudflareCustomHostnames:              false,
+	CloudflareCustomHostnamesMinTLSVersion: "1.0",
+	CloudflareCustomHostnamesCertificateAuthority: "google",
 	CloudflareDNSRecordsPerPage:                   100,
 	CloudflareRegionKey:                           "earth",
 	CoreDNSPrefix:                                 "/skydns/",
@@ -518,6 +524,9 @@ func App(cfg *Config) *kingpin.Application {
 	app.Flag("tencent-cloud-zone-type", "When using the Tencent Cloud provider, filter for zones with visibility (optional, options: public, private)").Default(defaultConfig.TencentCloudZoneType).EnumVar(&cfg.TencentCloudZoneType, "", "public", "private")
 
 	app.Flag("cloudflare-proxied", "When using the Cloudflare provider, specify if the proxy mode must be enabled (default: disabled)").BoolVar(&cfg.CloudflareProxied)
+	app.Flag("cloudflare-custom-hostnames", "When using the Cloudflare provider, specify if the Custom Hostnames feature will be used. Requires \"Cloudflare for SaaS\" enabled. (default: disabled)").BoolVar(&cfg.CloudflareCustomHostnames)
+	app.Flag("cloudflare-custom-hostnames-min-tls-version", "When using the Cloudflare provider with the Custom Hostnames, specify which Minimum TLS Version will be used by default. (default: 1.0, options: 1.0, 1.1, 1.2, 1.3)").Default("1.0").EnumVar(&cfg.CloudflareCustomHostnamesMinTLSVersion, "1.0", "1.1", "1.2", "1.3")
+	app.Flag("cloudflare-custom-hostnames-certificate-authority", "When using the Cloudflare provider with the Custom Hostnames, specify which Cerrtificate Authority will be used by default. (default: google, options: google, ssl_com, lets_encrypt)").Default("google").EnumVar(&cfg.CloudflareCustomHostnamesCertificateAuthority, "google", "ssl_com", "lets_encrypt")
 	app.Flag("cloudflare-dns-records-per-page", "When using the Cloudflare provider, specify how many DNS records listed per page, max possible 5,000 (default: 100)").Default(strconv.Itoa(defaultConfig.CloudflareDNSRecordsPerPage)).IntVar(&cfg.CloudflareDNSRecordsPerPage)
 	app.Flag("cloudflare-region-key", "When using the Cloudflare provider, specify the region (default: earth)").StringVar(&cfg.CloudflareRegionKey)
 	app.Flag("coredns-prefix", "When using the CoreDNS provider, specify the prefix name").Default(defaultConfig.CoreDNSPrefix).StringVar(&cfg.CoreDNSPrefix)

pkg/apis/externaldns/types_test.go

@@ -74,6 +74,9 @@ var (
 		AzureResourceGroup:                     "",
 		AzureSubscriptionID:                    "",
 		CloudflareProxied:                      false,
+		CloudflareCustomHostnames:              false,
+		CloudflareCustomHostnamesMinTLSVersion: "1.0",
+		CloudflareCustomHostnamesCertificateAuthority: "google",
 		CloudflareDNSRecordsPerPage:                   100,
 		CloudflareRegionKey:                           "",
 		CoreDNSPrefix:                                 "/skydns/",
@@ -179,6 +182,9 @@ var (
 		AzureResourceGroup:                     "arg",
 		AzureSubscriptionID:                    "arg",
 		CloudflareProxied:                      true,
+		CloudflareCustomHostnames:              true,
+		CloudflareCustomHostnamesMinTLSVersion: "1.3",
+		CloudflareCustomHostnamesCertificateAuthority: "google",
 		CloudflareDNSRecordsPerPage:                   5000,
 		CloudflareRegionKey:                           "us",
 		CoreDNSPrefix:                                 "/coredns/",
@@ -287,6 +293,9 @@ func TestParseFlags(t *testing.T) {
 				"--azure-resource-group=arg",
 				"--azure-subscription-id=arg",
 				"--cloudflare-proxied",
+				"--cloudflare-custom-hostnames",
+				"--cloudflare-custom-hostnames-min-tls-version=1.3",
+				"--cloudflare-custom-hostnames-certificate-authority=google",
 				"--cloudflare-dns-records-per-page=5000",
 				"--cloudflare-region-key=us",
 				"--coredns-prefix=/coredns/",
@@ -413,6 +422,9 @@ func TestParseFlags(t *testing.T) {
 				"EXTERNAL_DNS_AZURE_RESOURCE_GROUP":                              "arg",
 				"EXTERNAL_DNS_AZURE_SUBSCRIPTION_ID":                             "arg",
 				"EXTERNAL_DNS_CLOUDFLARE_PROXIED":                                "1",
+				"EXTERNAL_DNS_CLOUDFLARE_CUSTOM_HOSTNAMES":                       "1",
+				"EXTERNAL_DNS_CLOUDFLARE_CUSTOM_HOSTNAMES_MIN_TLS_VERSION":       "1.3",
+				"EXTERNAL_DNS_CLOUDFLARE_CUSTOM_HOSTNAMES_CERTIFICATE_AUTHORITY": "google",
 				"EXTERNAL_DNS_CLOUDFLARE_DNS_RECORDS_PER_PAGE":                   "5000",
 				"EXTERNAL_DNS_CLOUDFLARE_REGION_KEY":                             "us",
 				"EXTERNAL_DNS_COREDNS_PREFIX":                                    "/coredns/",

provider/cloudflare/cloudflare.go

@@ -64,6 +64,12 @@ var recordTypeProxyNotSupported = map[string]bool{
 	"SRV": true,
 }
 
+type CustomHostnamesConfig struct {
+	Enabled              bool
+	MinTLSVersion        string
+	CertificateAuthority string
+}
+
 var recordTypeCustomHostnameSupported = map[string]bool{
 	"A":     true,
 	"CNAME": true,
@@ -152,6 +158,7 @@ type CloudFlareProvider struct {
 	domainFilter          endpoint.DomainFilter
 	zoneIDFilter          provider.ZoneIDFilter
 	proxiedByDefault      bool
+	CustomHostnamesConfig CustomHostnamesConfig
 	DryRun                bool
 	DNSRecordsPerPage     int
 	RegionKey             string
@@ -163,6 +170,7 @@ type cloudFlareChange struct {
 	ResourceRecord     cloudflare.DNSRecord
 	RegionalHostname   cloudflare.RegionalHostname
 	CustomHostname     cloudflare.CustomHostname
+	CustomHostnamePrev string
 }
 
 // RecordParamsTypes is a typeset of the possible Record Params that can be passed to cloudflare-go library
@@ -201,7 +209,7 @@ func getCreateDNSRecordParam(cfc cloudFlareChange) cloudflare.CreateDNSRecordPar
 }
 
 // NewCloudFlareProvider initializes a new CloudFlare DNS based Provider.
-func NewCloudFlareProvider(domainFilter endpoint.DomainFilter, zoneIDFilter provider.ZoneIDFilter, proxiedByDefault bool, dryRun bool, dnsRecordsPerPage int, regionKey string) (*CloudFlareProvider, error) {
+func NewCloudFlareProvider(domainFilter endpoint.DomainFilter, zoneIDFilter provider.ZoneIDFilter, proxiedByDefault bool, dryRun bool, dnsRecordsPerPage int, regionKey string, customHostnamesConfig CustomHostnamesConfig) (*CloudFlareProvider, error) {
 	// initialize via chosen auth method and returns new API object
 	var (
 		config *cloudflare.API
@@ -229,6 +237,7 @@ func NewCloudFlareProvider(domainFilter endpoint.DomainFilter, zoneIDFilter prov
 		domainFilter:          domainFilter,
 		zoneIDFilter:          zoneIDFilter,
 		proxiedByDefault:      proxiedByDefault,
+		CustomHostnamesConfig: customHostnamesConfig,
 		DryRun:                dryRun,
 		DNSRecordsPerPage:     dnsRecordsPerPage,
 		RegionKey:             regionKey,
@@ -319,7 +328,7 @@ func (p *CloudFlareProvider) ApplyChanges(ctx context.Context, changes *plan.Cha
 
 	for _, endpoint := range changes.Create {
 		for _, target := range endpoint.Targets {
-			cloudflareChanges = append(cloudflareChanges, p.newCloudFlareChange(cloudFlareCreate, endpoint, target))
+			cloudflareChanges = append(cloudflareChanges, p.newCloudFlareChange(cloudFlareCreate, endpoint, target, nil))
 		}
 	}
 
@@ -329,21 +338,21 @@ func (p *CloudFlareProvider) ApplyChanges(ctx context.Context, changes *plan.Cha
 		add, remove, leave := provider.Difference(current.Targets, desired.Targets)
 
 		for _, a := range remove {
-			cloudflareChanges = append(cloudflareChanges, p.newCloudFlareChange(cloudFlareDelete, current, a))
+			cloudflareChanges = append(cloudflareChanges, p.newCloudFlareChange(cloudFlareDelete, current, a, current))
 		}
 
 		for _, a := range add {
-			cloudflareChanges = append(cloudflareChanges, p.newCloudFlareChange(cloudFlareCreate, desired, a))
+			cloudflareChanges = append(cloudflareChanges, p.newCloudFlareChange(cloudFlareCreate, desired, a, current))
 		}
 
 		for _, a := range leave {
-			cloudflareChanges = append(cloudflareChanges, p.newCloudFlareChange(cloudFlareUpdate, desired, a))
+			cloudflareChanges = append(cloudflareChanges, p.newCloudFlareChange(cloudFlareUpdate, desired, a, current))
 		}
 	}
 
 	for _, endpoint := range changes.Delete {
 		for _, target := range endpoint.Targets {
-			cloudflareChanges = append(cloudflareChanges, p.newCloudFlareChange(cloudFlareDelete, endpoint, target))
+			cloudflareChanges = append(cloudflareChanges, p.newCloudFlareChange(cloudFlareDelete, endpoint, target, nil))
 		}
 	}
 
@@ -367,16 +376,6 @@ func (p *CloudFlareProvider) submitChanges(ctx context.Context, changes []*cloud
 
 	var failedZones []string
 	for zoneID, changes := range changesByZone {
-		records, err := p.listDNSRecordsWithAutoPagination(ctx, zoneID)
-		if err != nil {
-			return fmt.Errorf("could not fetch records from zone, %v", err)
-		}
-
-		chs, chErr := p.listCustomHostnamesWithPagination(ctx, zoneID)
-		if chErr != nil {
-			return fmt.Errorf("could not fetch custom hostnames from zone, %v", chErr)
-		}
-
 		var failedChange bool
 		for _, change := range changes {
 			logFields := log.Fields{
@@ -394,34 +393,37 @@ func (p *CloudFlareProvider) submitChanges(ctx context.Context, changes []*cloud
 			}
 
 			resourceContainer := cloudflare.ZoneIdentifier(zoneID)
+			records, err := p.listDNSRecordsWithAutoPagination(ctx, zoneID)
+			if err != nil {
+				return fmt.Errorf("could not fetch records from zone, %v", err)
+			}
+			chs, chErr := p.listCustomHostnamesWithPagination(ctx, zoneID)
+			if chErr != nil {
+				return fmt.Errorf("could not fetch custom hostnames from zone, %v", chErr)
+			}
 			if change.Action == cloudFlareUpdate {
 				if recordTypeCustomHostnameSupported[change.ResourceRecord.Type] {
-					chID, oldCh := p.getCustomHostnameIDbyOrigin(chs, change.ResourceRecord.Name)
+					prevCh := change.CustomHostnamePrev
-					if chID == "" && change.CustomHostname.Hostname != "" {
+					newCh := change.CustomHostname.Hostname
-						log.WithFields(logFields).Infof("Adding custom hostname %v", change.CustomHostname.Hostname)
+					if prevCh != "" {
+						prevChID, _ := p.getCustomHostnameOrigin(chs, prevCh)
+						if prevChID != "" && prevCh != newCh {
+							log.WithFields(logFields).Infof("Removing previous custom hostname %v/%v", prevChID, prevCh)
+							chErr := p.Client.DeleteCustomHostname(ctx, zoneID, prevChID)
+							if chErr != nil {
+								failedChange = true
+								log.WithFields(logFields).Errorf("failed to remove previous custom hostname %v/%v: %v", prevChID, prevCh, chErr)
+							}
+						}
+					}
+					if newCh != "" {
+						if prevCh != newCh {
+							log.WithFields(logFields).Infof("Adding custom hostname %v", newCh)
 							_, chErr := p.Client.CreateCustomHostname(ctx, zoneID, change.CustomHostname)
 							if chErr != nil {
 								failedChange = true
-							log.WithFields(logFields).Errorf("failed to add custom hostname %v: %v", change.CustomHostname.Hostname, chErr)
+								log.WithFields(logFields).Errorf("failed to add custom hostname %v: %v", newCh, chErr)
 							}
-					} else if chID != "" && oldCh != "" && change.CustomHostname.Hostname == "" {
-						log.WithFields(logFields).Infof("Removing custom hostname %v", change.CustomHostname.Hostname)
-						chErr := p.Client.DeleteCustomHostname(ctx, zoneID, chID)
-						if chErr != nil {
-							failedChange = true
-							log.WithFields(logFields).Errorf("failed to remove custom hostname %v: %v", change.CustomHostname.Hostname, chErr)
-						}
-					} else if chID != "" && change.CustomHostname.Hostname != "" && oldCh != change.CustomHostname.Hostname {
-						log.WithFields(logFields).Infof("Replacing custom hostname: %v/%v to %v", chID, oldCh, change.CustomHostname.Hostname)
-						chDelErr := p.Client.DeleteCustomHostname(ctx, zoneID, chID)
-						if chDelErr != nil {
-							failedChange = true
-							log.WithFields(logFields).Errorf("failed to remove replacing custom hostname %v/%v: %v", chID, oldCh, chDelErr)
-						}
-						_, chAddErr := p.Client.CreateCustomHostname(ctx, zoneID, change.CustomHostname)
-						if chAddErr != nil {
-							failedChange = true
-							log.WithFields(logFields).Errorf("failed to add replacing custom hostname %v: %v", change.CustomHostname.Hostname, chAddErr)
 						}
 					}
 				}
@@ -455,14 +457,19 @@ func (p *CloudFlareProvider) submitChanges(ctx context.Context, changes []*cloud
 					failedChange = true
 					log.WithFields(logFields).Errorf("failed to delete record: %v", err)
 				}
-				chID, oldCh := p.getCustomHostnameIDbyOrigin(chs, change.ResourceRecord.Name)
+				if change.CustomHostname.Hostname == "" {
+					continue
+				}
+				log.WithFields(logFields).Infof("Deleting custom hostname %v", change.CustomHostname.Hostname)
+				chID, _ := p.getCustomHostnameOrigin(chs, change.CustomHostname.Hostname)
 				if chID == "" {
+					log.WithFields(logFields).Infof("Custom hostname %v not found", change.CustomHostname.Hostname)
 					continue
 				}
 				chErr := p.Client.DeleteCustomHostname(ctx, zoneID, chID)
 				if chErr != nil {
 					failedChange = true
-					log.WithFields(logFields).Errorf("failed to delete custom hostname %v/%v: %v", chID, oldCh, chErr)
+					log.WithFields(logFields).Errorf("failed to delete custom hostname %v/%v: %v", chID, change.CustomHostname.Hostname, chErr)
 				}
 			} else if change.Action == cloudFlareCreate {
 				recordParam := getCreateDNSRecordParam(*change)
@@ -471,7 +478,16 @@ func (p *CloudFlareProvider) submitChanges(ctx context.Context, changes []*cloud
 					failedChange = true
 					log.WithFields(logFields).Errorf("failed to create record: %v", err)
 				}
+				if change.CustomHostname.Hostname == "" {
+					continue
+				}
 				log.WithFields(logFields).Infof("Creating custom hostname %v", change.CustomHostname.Hostname)
+				chID, chOrigin := p.getCustomHostnameOrigin(chs, change.CustomHostname.Hostname)
+				if chID != "" {
+					failedChange = true
+					log.WithFields(logFields).Errorf("failed to create custom hostname, %v already exists for origin %v", change.CustomHostname.Hostname, chOrigin)
+					continue
+				}
 				_, chErr := p.Client.CreateCustomHostname(ctx, zoneID, change.CustomHostname)
 				if chErr != nil {
 					failedChange = true
@@ -537,16 +553,16 @@ func (p *CloudFlareProvider) getRecordID(records []cloudflare.DNSRecord, record
 	return ""
 }
 
-func (p *CloudFlareProvider) getCustomHostnameIDbyOrigin(chs []cloudflare.CustomHostname, origin string) (string, string) {
+func (p *CloudFlareProvider) getCustomHostnameOrigin(chs []cloudflare.CustomHostname, hostname string) (string, string) {
 	for _, zoneCh := range chs {
-		if zoneCh.CustomOriginServer == origin {
+		if zoneCh.Hostname == hostname {
-			return zoneCh.ID, zoneCh.Hostname
+			return zoneCh.ID, zoneCh.CustomOriginServer
 		}
 	}
 	return "", ""
 }
 
-func (p *CloudFlareProvider) newCloudFlareChange(action string, endpoint *endpoint.Endpoint, target string) *cloudFlareChange {
+func (p *CloudFlareProvider) newCloudFlareChange(action string, endpoint *endpoint.Endpoint, target string, current *endpoint.Endpoint) *cloudFlareChange {
 	ttl := defaultCloudFlareRecordTTL
 	proxied := shouldBeProxied(endpoint, p.proxiedByDefault)
 
@@ -554,6 +570,19 @@ func (p *CloudFlareProvider) newCloudFlareChange(action string, endpoint *endpoi
 		ttl = int(endpoint.RecordTTL)
 	}
 	dt := time.Now()
+
+	customHostnamePrev := ""
+	newCustomHostname := cloudflare.CustomHostname{}
+	if p.CustomHostnamesConfig.Enabled {
+		if current != nil {
+			customHostnamePrev = getEndpointCustomHostname(current)
+		}
+		newCustomHostname = cloudflare.CustomHostname{
+			Hostname:           getEndpointCustomHostname(endpoint),
+			CustomOriginServer: endpoint.DNSName,
+			SSL:                getCustomHostnamesSSLOptions(endpoint, p.CustomHostnamesConfig),
+		}
+	}
 	return &cloudFlareChange{
 		Action: action,
 		ResourceRecord: cloudflare.DNSRecord{
@@ -573,19 +602,8 @@ func (p *CloudFlareProvider) newCloudFlareChange(action string, endpoint *endpoi
 			RegionKey: p.RegionKey,
 			CreatedOn: &dt,
 		},
-		CustomHostname: cloudflare.CustomHostname{
+		CustomHostnamePrev: customHostnamePrev,
-			Hostname:           getEndpointCustomHostname(endpoint),
+		CustomHostname:     newCustomHostname,
-			CustomOriginServer: endpoint.DNSName,
-			SSL: &cloudflare.CustomHostnameSSL{
-				Type:                 "dv",
-				Method:               "http",
-				CertificateAuthority: "google",
-				BundleMethod:         "ubiquitous",
-				Settings: cloudflare.CustomHostnameSSLSettings{
-					MinTLSVersion: "1.0",
-				},
-			},
-		},
 	}
 }
 
@@ -618,6 +636,9 @@ func (p *CloudFlareProvider) listDNSRecordsWithAutoPagination(ctx context.Contex
 
 // listCustomHostnamesWithPagination performs automatic pagination of results on requests to cloudflare.CustomHostnames
 func (p *CloudFlareProvider) listCustomHostnamesWithPagination(ctx context.Context, zoneID string) ([]cloudflare.CustomHostname, error) {
+	if !p.CustomHostnamesConfig.Enabled {
+		return nil, nil
+	}
 	var chs []cloudflare.CustomHostname
 	resultInfo := cloudflare.ResultInfo{Page: 1}
 	for {
@@ -643,6 +664,18 @@ func (p *CloudFlareProvider) listCustomHostnamesWithPagination(ctx context.Conte
 	return chs, nil
 }
 
+func getCustomHostnamesSSLOptions(endpoint *endpoint.Endpoint, customHostnamesConfig CustomHostnamesConfig) *cloudflare.CustomHostnameSSL {
+	return &cloudflare.CustomHostnameSSL{
+		Type:                 "dv",
+		Method:               "http",
+		CertificateAuthority: customHostnamesConfig.CertificateAuthority,
+		BundleMethod:         "ubiquitous",
+		Settings: cloudflare.CustomHostnameSSLSettings{
+			MinTLSVersion: customHostnamesConfig.MinTLSVersion,
+		},
+	}
+}
+
 func shouldBeProxied(endpoint *endpoint.Endpoint, proxiedByDefault bool) bool {
 	proxied := proxiedByDefault
 
@@ -695,7 +728,7 @@ func groupByNameAndTypeWithCustomHostnames(records []cloudflare.DNSRecord, chs [
 	// map custom origin to custom hostname, custom origin should match to a dns record
 	customOriginServers := map[string]string{}
 
-	// only one latest custom hostname for a dns record would work
+	// only one latest custom hostname for a dns record would work; noop (chs is empty) if custom hostnames feature is not in use
 	for _, c := range chs {
 		customOriginServers[c.CustomOriginServer] = c.Hostname
 	}
@@ -721,9 +754,10 @@ func groupByNameAndTypeWithCustomHostnames(records []cloudflare.DNSRecord, chs [
 		if ep == nil {
 			continue
 		}
-		ep.WithProviderSpecific(source.CloudflareProxiedKey, strconv.FormatBool(proxied))
+		ep = ep.WithProviderSpecific(source.CloudflareProxiedKey, strconv.FormatBool(proxied))
+		// noop (customOriginServers is empty) if custom hostnames feature is not in use
 		if customHostname, ok := customOriginServers[records[0].Name]; ok {
-			ep.WithProviderSpecific(source.CloudflareCustomHostnameKey, customHostname)
+			ep = ep.WithProviderSpecific(source.CloudflareCustomHostnameKey, customHostname)
 		}
 
 		endpoints = append(endpoints, ep)

provider/cloudflare/cloudflare_test.go

@@ -26,10 +26,12 @@ import (
 	"testing"
 
 	cloudflare "github.com/cloudflare/cloudflare-go"
+	log "github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
 
 	"github.com/maxatome/go-testdeep/td"
 	"sigs.k8s.io/external-dns/endpoint"
+	"sigs.k8s.io/external-dns/internal/testutils"
 	"sigs.k8s.io/external-dns/plan"
 	"sigs.k8s.io/external-dns/provider"
 )
@@ -112,6 +114,7 @@ func getDNSRecordFromRecordParams(rp any) cloudflare.DNSRecord {
 	switch params := rp.(type) {
 	case cloudflare.CreateDNSRecordParams:
 		return cloudflare.DNSRecord{
+			ID:      params.ID,
 			Name:    params.Name,
 			TTL:     params.TTL,
 			Proxied: params.Proxied,
@@ -120,6 +123,7 @@ func getDNSRecordFromRecordParams(rp any) cloudflare.DNSRecord {
 		}
 	case cloudflare.UpdateDNSRecordParams:
 		return cloudflare.DNSRecord{
+			ID:      params.ID,
 			Name:    params.Name,
 			TTL:     params.TTL,
 			Proxied: params.Proxied,
@@ -131,16 +135,23 @@ func getDNSRecordFromRecordParams(rp any) cloudflare.DNSRecord {
 	}
 }
 
+func generateDNSRecordID(rrtype string, name string, content string) string {
+	return fmt.Sprintf("%s-%s-%s", name, rrtype, content)
+}
+
 func (m *mockCloudFlareClient) CreateDNSRecord(ctx context.Context, rc *cloudflare.ResourceContainer, rp cloudflare.CreateDNSRecordParams) (cloudflare.DNSRecord, error) {
 	recordData := getDNSRecordFromRecordParams(rp)
+	if recordData.ID == "" {
+		recordData.ID = generateDNSRecordID(recordData.Type, recordData.Name, recordData.Content)
+	}
 	m.Actions = append(m.Actions, MockAction{
 		Name:       "Create",
 		ZoneId:     rc.Identifier,
-		RecordId:   rp.ID,
+		RecordId:   recordData.ID,
 		RecordData: recordData,
 	})
 	if zone, ok := m.Records[rc.Identifier]; ok {
-		zone[rp.ID] = recordData
+		zone[recordData.ID] = recordData
 	}
 
 	if recordData.Name == "newerror.bar.com" {
@@ -156,6 +167,10 @@ func (m *mockCloudFlareClient) ListDNSRecords(ctx context.Context, rc *cloudflar
 	result := []cloudflare.DNSRecord{}
 	if zone, ok := m.Records[rc.Identifier]; ok {
 		for _, record := range zone {
+			if strings.HasPrefix(record.Name, "newerror-list-") {
+				m.DeleteDNSRecord(ctx, rc, record.ID)
+				return nil, &cloudflare.ResultInfo{}, errors.New("failed to list erroring DNS record")
+			}
 			result = append(result, record)
 		}
 	}
@@ -200,6 +215,9 @@ func (m *mockCloudFlareClient) UpdateDNSRecord(ctx context.Context, rc *cloudfla
 	})
 	if zone, ok := m.Records[rc.Identifier]; ok {
 		if _, ok := zone[rp.ID]; ok {
+			if strings.HasPrefix(recordData.Name, "newerror-update-") {
+				return errors.New("failed to update erroring DNS record")
+			}
 			zone[rp.ID] = recordData
 		}
 	}
@@ -226,7 +244,11 @@ func (m *mockCloudFlareClient) DeleteDNSRecord(ctx context.Context, rc *cloudfla
 	})
 	if zone, ok := m.Records[rc.Identifier]; ok {
 		if _, ok := zone[recordID]; ok {
+			name := zone[recordID].Name
 			delete(zone, recordID)
+			if strings.HasPrefix(name, "newerror-delete-") {
+				return errors.New("failed to delete erroring DNS record")
+			}
 			return nil
 		}
 	}
@@ -240,6 +262,10 @@ func (m *mockCloudFlareClient) UserDetails(ctx context.Context) (cloudflare.User
 func (m *mockCloudFlareClient) CustomHostnames(ctx context.Context, zoneID string, page int, filter cloudflare.CustomHostname) ([]cloudflare.CustomHostname, cloudflare.ResultInfo, error) {
 	var err error = nil
 
+	if strings.HasPrefix(zoneID, "newerror-") {
+		return nil, cloudflare.ResultInfo{}, errors.New("failed to list custom hostnames")
+	}
+
 	if page != 1 || filter.Hostname != "" {
 		err = errors.New("pages and filters are not supported for custom hostnames mock test")
 	}
@@ -247,6 +273,10 @@ func (m *mockCloudFlareClient) CustomHostnames(ctx context.Context, zoneID strin
 	result := []cloudflare.CustomHostname{}
 	if zone, ok := m.customHostnames[zoneID]; ok {
 		for _, ch := range zone {
+			if strings.HasPrefix(ch.Hostname, "newerror-list-") {
+				m.DeleteCustomHostname(ctx, zoneID, ch.ID)
+				return nil, cloudflare.ResultInfo{}, errors.New("failed to list erroring custom hostname")
+			}
 			result = append(result, ch)
 		}
 	}
@@ -262,10 +292,15 @@ func (m *mockCloudFlareClient) CustomHostnames(ctx context.Context, zoneID strin
 }
 
 func (m *mockCloudFlareClient) CreateCustomHostname(ctx context.Context, zoneID string, ch cloudflare.CustomHostname) (*cloudflare.CustomHostnameResponse, error) {
+	if ch.Hostname == "" || ch.CustomOriginServer == "" || ch.Hostname == "newerror-create.foo.fancybar.com" {
+		return nil, fmt.Errorf("Invalid custom hostname or origin hostname")
+	}
 	if _, ok := m.customHostnames[zoneID]; !ok {
 		m.customHostnames[zoneID] = map[string]cloudflare.CustomHostname{}
 	}
-	m.customHostnames[zoneID][ch.ID] = ch
+	var newCustomHostname cloudflare.CustomHostname = ch
+	newCustomHostname.ID = fmt.Sprintf("ID-%s", ch.Hostname)
+	m.customHostnames[zoneID][newCustomHostname.ID] = newCustomHostname
 	return &cloudflare.CustomHostnameResponse{}, nil
 }
 
@@ -273,9 +308,11 @@ func (m *mockCloudFlareClient) DeleteCustomHostname(ctx context.Context, zoneID
 	if zone, ok := m.customHostnames[zoneID]; ok {
 		if _, ok := zone[customHostnameID]; ok {
 			delete(zone, customHostnameID)
-			return nil
 		}
 	}
+	if customHostnameID == "ID-newerror-delete.foo.fancybar.com" {
+		return fmt.Errorf("Invalid custom hostname to delete")
+	}
 	return nil
 }
 
@@ -342,6 +379,16 @@ func (m *mockCloudFlareClient) ZoneDetails(ctx context.Context, zoneID string) (
 	return cloudflare.Zone{}, errors.New("Unknown zoneID: " + zoneID)
 }
 
+func (p *CloudFlareProvider) getCustomHostnameIDbyCustomHostnameAndOrigin(chs []cloudflare.CustomHostname, customHostname string, origin string) (string, string) {
+	for _, zoneCh := range chs {
+		if zoneCh.Hostname == customHostname && zoneCh.CustomOriginServer == origin {
+			return zoneCh.ID, zoneCh.Hostname
+
+		}
+	}
+	return "", ""
+}
+
 func AssertActions(t *testing.T, provider *CloudFlareProvider, endpoints []*endpoint.Endpoint, actions []MockAction, managedRecords []string, args ...interface{}) {
 	t.Helper()
 
@@ -401,7 +448,9 @@ func TestCloudflareA(t *testing.T) {
 		{
 			Name:     "Create",
 			ZoneId:   "001",
+			RecordId: generateDNSRecordID("A", "bar.com", "127.0.0.1"),
 			RecordData: cloudflare.DNSRecord{
+				ID:      generateDNSRecordID("A", "bar.com", "127.0.0.1"),
 				Type:    "A",
 				Name:    "bar.com",
 				Content: "127.0.0.1",
@@ -412,7 +461,9 @@ func TestCloudflareA(t *testing.T) {
 		{
 			Name:     "Create",
 			ZoneId:   "001",
+			RecordId: generateDNSRecordID("A", "bar.com", "127.0.0.2"),
 			RecordData: cloudflare.DNSRecord{
+				ID:      generateDNSRecordID("A", "bar.com", "127.0.0.2"),
 				Type:    "A",
 				Name:    "bar.com",
 				Content: "127.0.0.2",
@@ -438,7 +489,9 @@ func TestCloudflareCname(t *testing.T) {
 		{
 			Name:     "Create",
 			ZoneId:   "001",
+			RecordId: generateDNSRecordID("CNAME", "cname.bar.com", "google.com"),
 			RecordData: cloudflare.DNSRecord{
+				ID:      generateDNSRecordID("CNAME", "cname.bar.com", "google.com"),
 				Type:    "CNAME",
 				Name:    "cname.bar.com",
 				Content: "google.com",
@@ -449,7 +502,9 @@ func TestCloudflareCname(t *testing.T) {
 		{
 			Name:     "Create",
 			ZoneId:   "001",
+			RecordId: generateDNSRecordID("CNAME", "cname.bar.com", "facebook.com"),
 			RecordData: cloudflare.DNSRecord{
+				ID:      generateDNSRecordID("CNAME", "cname.bar.com", "facebook.com"),
 				Type:    "CNAME",
 				Name:    "cname.bar.com",
 				Content: "facebook.com",
@@ -476,7 +531,9 @@ func TestCloudflareCustomTTL(t *testing.T) {
 		{
 			Name:     "Create",
 			ZoneId:   "001",
+			RecordId: generateDNSRecordID("A", "ttl.bar.com", "127.0.0.1"),
 			RecordData: cloudflare.DNSRecord{
+				ID:      generateDNSRecordID("A", "ttl.bar.com", "127.0.0.1"),
 				Type:    "A",
 				Name:    "ttl.bar.com",
 				Content: "127.0.0.1",
@@ -502,7 +559,9 @@ func TestCloudflareProxiedDefault(t *testing.T) {
 		{
 			Name:     "Create",
 			ZoneId:   "001",
+			RecordId: generateDNSRecordID("A", "bar.com", "127.0.0.1"),
 			RecordData: cloudflare.DNSRecord{
+				ID:      generateDNSRecordID("A", "bar.com", "127.0.0.1"),
 				Type:    "A",
 				Name:    "bar.com",
 				Content: "127.0.0.1",
@@ -534,7 +593,9 @@ func TestCloudflareProxiedOverrideTrue(t *testing.T) {
 		{
 			Name:     "Create",
 			ZoneId:   "001",
+			RecordId: generateDNSRecordID("A", "bar.com", "127.0.0.1"),
 			RecordData: cloudflare.DNSRecord{
+				ID:      generateDNSRecordID("A", "bar.com", "127.0.0.1"),
 				Type:    "A",
 				Name:    "bar.com",
 				Content: "127.0.0.1",
@@ -566,7 +627,9 @@ func TestCloudflareProxiedOverrideFalse(t *testing.T) {
 		{
 			Name:     "Create",
 			ZoneId:   "001",
+			RecordId: generateDNSRecordID("A", "bar.com", "127.0.0.1"),
 			RecordData: cloudflare.DNSRecord{
+				ID:      generateDNSRecordID("A", "bar.com", "127.0.0.1"),
 				Type:    "A",
 				Name:    "bar.com",
 				Content: "127.0.0.1",
@@ -598,7 +661,9 @@ func TestCloudflareProxiedOverrideIllegal(t *testing.T) {
 		{
 			Name:     "Create",
 			ZoneId:   "001",
+			RecordId: generateDNSRecordID("A", "bar.com", "127.0.0.1"),
 			RecordData: cloudflare.DNSRecord{
+				ID:      generateDNSRecordID("A", "bar.com", "127.0.0.1"),
 				Type:    "A",
 				Name:    "bar.com",
 				Content: "127.0.0.1",
@@ -631,11 +696,12 @@ func TestCloudflareSetProxied(t *testing.T) {
 	}
 
 	for _, testCase := range testCases {
+		target := "127.0.0.1"
 		endpoints := []*endpoint.Endpoint{
 			{
 				RecordType: testCase.recordType,
 				DNSName:    testCase.domain,
-				Targets:    endpoint.Targets{"127.0.0.1"},
+				Targets:    endpoint.Targets{target},
 				ProviderSpecific: endpoint.ProviderSpecific{
 					endpoint.ProviderSpecificProperty{
 						Name:  "external-dns.alpha.kubernetes.io/cloudflare-proxied",
@@ -644,12 +710,14 @@ func TestCloudflareSetProxied(t *testing.T) {
 				},
 			},
 		}
-
+		expectedID := fmt.Sprintf("%s-%s-%s", testCase.domain, testCase.recordType, target)
 		AssertActions(t, &CloudFlareProvider{}, endpoints, []MockAction{
 			{
 				Name:     "Create",
 				ZoneId:   "001",
+				RecordId: expectedID,
 				RecordData: cloudflare.DNSRecord{
+					ID:      expectedID,
 					Type:    testCase.recordType,
 					Name:    testCase.domain,
 					Content: "127.0.0.1",
@@ -795,7 +863,8 @@ func TestCloudflareProvider(t *testing.T) {
 		false,
 		true,
 		5000,
-		"")
+		"",
+		CustomHostnamesConfig{Enabled: false})
 	if err != nil {
 		t.Errorf("should not fail, %s", err)
 	}
@@ -812,7 +881,8 @@ func TestCloudflareProvider(t *testing.T) {
 		false,
 		true,
 		5000,
-		"")
+		"",
+		CustomHostnamesConfig{Enabled: false})
 	if err != nil {
 		t.Errorf("should not fail, %s", err)
 	}
@@ -826,7 +896,8 @@ func TestCloudflareProvider(t *testing.T) {
 		false,
 		true,
 		5000,
-		"")
+		"",
+		CustomHostnamesConfig{Enabled: false})
 	if err != nil {
 		t.Errorf("should not fail, %s", err)
 	}
@@ -839,7 +910,8 @@ func TestCloudflareProvider(t *testing.T) {
 		false,
 		true,
 		5000,
-		"")
+		"",
+		CustomHostnamesConfig{Enabled: false})
 	if err == nil {
 		t.Errorf("expected to fail")
 	}
@@ -879,7 +951,9 @@ func TestCloudflareApplyChanges(t *testing.T) {
 		{
 			Name:     "Create",
 			ZoneId:   "001",
+			RecordId: generateDNSRecordID("", "new.bar.com", "target"),
 			RecordData: cloudflare.DNSRecord{
+				ID:      generateDNSRecordID("", "new.bar.com", "target"),
 				Name:    "new.bar.com",
 				Content: "target",
 				TTL:     1,
@@ -889,7 +963,9 @@ func TestCloudflareApplyChanges(t *testing.T) {
 		{
 			Name:     "Create",
 			ZoneId:   "001",
+			RecordId: generateDNSRecordID("", "foobar.bar.com", "target-new"),
 			RecordData: cloudflare.DNSRecord{
+				ID:      generateDNSRecordID("", "foobar.bar.com", "target-new"),
 				Name:    "foobar.bar.com",
 				Content: "target-new",
 				TTL:     1,
@@ -1368,7 +1444,9 @@ func TestCloudflareComplexUpdate(t *testing.T) {
 		{
 			Name:     "Create",
 			ZoneId:   "001",
+			RecordId: generateDNSRecordID("A", "foobar.bar.com", "2.3.4.5"),
 			RecordData: cloudflare.DNSRecord{
+				ID:      generateDNSRecordID("A", "foobar.bar.com", "2.3.4.5"),
 				Name:    "foobar.bar.com",
 				Type:    "A",
 				Content: "2.3.4.5",
@@ -1381,6 +1459,7 @@ func TestCloudflareComplexUpdate(t *testing.T) {
 			ZoneId:   "001",
 			RecordId: "1234567890",
 			RecordData: cloudflare.DNSRecord{
+				ID:      "1234567890",
 				Name:    "foobar.bar.com",
 				Type:    "A",
 				Content: "1.2.3.4",
@@ -1451,7 +1530,14 @@ func TestCustomTTLWithEnabledProxyNotChanged(t *testing.T) {
 func TestCloudFlareProvider_Region(t *testing.T) {
 	_ = os.Setenv("CF_API_TOKEN", "abc123def")
 	_ = os.Setenv("CF_API_EMAIL", "test@test.com")
-	provider, err := NewCloudFlareProvider(endpoint.NewDomainFilter([]string{"example.com"}), provider.ZoneIDFilter{}, true, false, 50, "us")
+	provider, err := NewCloudFlareProvider(
+		endpoint.NewDomainFilter([]string{"example.com"}),
+		provider.ZoneIDFilter{},
+		true,
+		false,
+		50,
+		"us",
+		CustomHostnamesConfig{Enabled: false})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -1482,7 +1568,14 @@ func TestCloudFlareProvider_updateDataLocalizationRegionalHostnameParams(t *test
 func TestCloudFlareProvider_newCloudFlareChange(t *testing.T) {
 	_ = os.Setenv("CF_API_KEY", "xxxxxxxxxxxxxxxxx")
 	_ = os.Setenv("CF_API_EMAIL", "test@test.com")
-	provider, err := NewCloudFlareProvider(endpoint.NewDomainFilter([]string{"example.com"}), provider.ZoneIDFilter{}, true, false, 50, "us")
+	provider, err := NewCloudFlareProvider(
+		endpoint.NewDomainFilter([]string{"example.com"}),
+		provider.ZoneIDFilter{},
+		true,
+		false,
+		50,
+		"us",
+		CustomHostnamesConfig{Enabled: false})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -1492,7 +1585,7 @@ func TestCloudFlareProvider_newCloudFlareChange(t *testing.T) {
 		Targets: []string{"192.0.2.1"},
 	}
 
-	change := provider.newCloudFlareChange(cloudFlareCreate, endpoint, endpoint.Targets[0])
+	change := provider.newCloudFlareChange(cloudFlareCreate, endpoint, endpoint.Targets[0], nil)
 	if change.RegionalHostname.RegionKey != "us" {
 		t.Errorf("expected region key to be 'us', but got '%s'", change.RegionalHostname.RegionKey)
 	}
@@ -1621,33 +1714,353 @@ func TestCloudFlareProvider_submitChangesApex(t *testing.T) {
 	}
 }
 
-func TestCloudflareCustomHostnameOperations(t *testing.T) {
+func TestCloudflareZoneRecordsFail(t *testing.T) {
-	client := NewMockCloudFlareClientWithRecords(map[string][]cloudflare.DNSRecord{
+	client := &mockCloudFlareClient{
-		"001": ExampleDomain,
+		Zones: map[string]string{
-	})
+			"newerror-001": "bar.com",
-	provider := &CloudFlareProvider{
+		},
+		Records:         map[string]map[string]cloudflare.DNSRecord{},
+		customHostnames: map[string]map[string]cloudflare.CustomHostname{},
+	}
+	failingProvider := &CloudFlareProvider{
 		Client:                client,
+		CustomHostnamesConfig: CustomHostnamesConfig{Enabled: true},
 	}
 	ctx := context.Background()
 
+	_, err := failingProvider.Records(ctx)
+	if err == nil {
+		t.Errorf("should fail - invalid zone id, %s", err)
+	}
+}
+
+func TestCloudflareDNSRecordsOperationsFail(t *testing.T) {
+	client := NewMockCloudFlareClient()
+	provider := &CloudFlareProvider{
+		Client:                client,
+		CustomHostnamesConfig: CustomHostnamesConfig{Enabled: true},
+	}
+	ctx := context.Background()
+	domainFilter := endpoint.NewDomainFilter([]string{"bar.com"})
+
+	testFailCases := []struct {
+		Name                    string
+		Endpoints               []*endpoint.Endpoint
+		ExpectedCustomHostnames map[string]string
+		shouldFail              bool
+	}{
+		{
+			Name: "failing to create dns record",
+			Endpoints: []*endpoint.Endpoint{
+				{
+					DNSName:    "newerror.bar.com",
+					Targets:    endpoint.Targets{"1.2.3.4"},
+					RecordType: endpoint.RecordTypeA,
+					RecordTTL:  endpoint.TTL(defaultCloudFlareRecordTTL),
+					Labels:     endpoint.Labels{},
+				},
+			},
+			shouldFail: true,
+		},
+		{
+			Name: "failing to list DNS record",
+			Endpoints: []*endpoint.Endpoint{
+				{
+					DNSName:    "newerror-list-1.foo.bar.com",
+					Targets:    endpoint.Targets{"1.2.3.4"},
+					RecordType: endpoint.RecordTypeA,
+					RecordTTL:  endpoint.TTL(defaultCloudFlareRecordTTL),
+					Labels:     endpoint.Labels{},
+				},
+			},
+			shouldFail: true,
+		},
+		{
+			Name: "create failing to update DNS record",
+			Endpoints: []*endpoint.Endpoint{
+				{
+					DNSName:    "newerror-update-1.foo.bar.com",
+					Targets:    endpoint.Targets{"1.2.3.4"},
+					RecordType: endpoint.RecordTypeA,
+					RecordTTL:  endpoint.TTL(defaultCloudFlareRecordTTL),
+					Labels:     endpoint.Labels{},
+				},
+			},
+			shouldFail: false,
+		},
+		{
+			Name: "failing to update DNS record",
+			Endpoints: []*endpoint.Endpoint{
+				{
+					DNSName:    "newerror-update-1.foo.bar.com",
+					Targets:    endpoint.Targets{"1.2.3.4"},
+					RecordType: endpoint.RecordTypeA,
+					RecordTTL:  1234,
+					Labels:     endpoint.Labels{},
+				},
+			},
+			shouldFail: true,
+		},
+		{
+			Name: "create failing to delete DNS record",
+			Endpoints: []*endpoint.Endpoint{
+				{
+					DNSName:    "newerror-delete-1.foo.bar.com",
+					Targets:    endpoint.Targets{"1.2.3.4"},
+					RecordType: endpoint.RecordTypeA,
+					RecordTTL:  1234,
+					Labels:     endpoint.Labels{},
+				},
+			},
+			shouldFail: false,
+		},
+		{
+			Name:       "failing to delete erroring DNS record",
+			Endpoints:  []*endpoint.Endpoint{},
+			shouldFail: true,
+		},
+	}
+
+	for _, tc := range testFailCases {
 		records, err := provider.Records(ctx)
 		if err != nil {
 			t.Errorf("should not fail, %s", err)
 		}
 
+		endpoints, err := provider.AdjustEndpoints(tc.Endpoints)
+
+		assert.NoError(t, err)
+		plan := &plan.Plan{
+			Current:        records,
+			Desired:        endpoints,
+			DomainFilter:   endpoint.MatchAllDomainFilters{&domainFilter},
+			ManagedRecords: []string{endpoint.RecordTypeA, endpoint.RecordTypeCNAME},
+		}
+
+		planned := plan.Calculate()
+
+		err = provider.ApplyChanges(context.Background(), planned.Changes)
+		if err == nil && tc.shouldFail {
+			t.Errorf("should fail - %s, %s", tc.Name, err)
+		} else if err != nil && !tc.shouldFail {
+			t.Errorf("should not fail - %s, %s", tc.Name, err)
+		}
+	}
+}
+
+func TestCloudflareCustomHostnameOperations(t *testing.T) {
+	client := NewMockCloudFlareClient()
+	provider := &CloudFlareProvider{
+		Client:                client,
+		CustomHostnamesConfig: CustomHostnamesConfig{Enabled: true},
+	}
+	ctx := context.Background()
 	domainFilter := endpoint.NewDomainFilter([]string{"bar.com"})
 
+	testFailCases := []struct {
+		Name                    string
+		Endpoints               []*endpoint.Endpoint
+		ExpectedCustomHostnames map[string]string
+		shouldFail              bool
+	}{
+		{
+			Name: "failing to create custom hostname on record creation",
+			Endpoints: []*endpoint.Endpoint{
+				{
+					DNSName:    "create.foo.bar.com",
+					Targets:    endpoint.Targets{"1.2.3.4"},
+					RecordType: endpoint.RecordTypeA,
+					RecordTTL:  endpoint.TTL(defaultCloudFlareRecordTTL),
+					Labels:     endpoint.Labels{},
+					ProviderSpecific: endpoint.ProviderSpecific{
+						{
+							Name:  "external-dns.alpha.kubernetes.io/cloudflare-custom-hostname",
+							Value: "newerror-create.foo.fancybar.com",
+						},
+					},
+				},
+			},
+			shouldFail: true,
+		},
+		{
+			Name: "add custom hostname to more than one endpoint",
+			Endpoints: []*endpoint.Endpoint{
+				{
+					DNSName:    "fail.foo.bar.com",
+					Targets:    endpoint.Targets{"1.2.3.4", "2.3.4.5"},
+					RecordType: endpoint.RecordTypeA,
+					RecordTTL:  endpoint.TTL(defaultCloudFlareRecordTTL),
+					Labels:     endpoint.Labels{},
+					ProviderSpecific: endpoint.ProviderSpecific{
+						{
+							Name:  "external-dns.alpha.kubernetes.io/cloudflare-custom-hostname",
+							Value: "fail.foo.fancybar.com",
+						},
+					},
+				},
+			},
+			shouldFail: true,
+		},
+		{
+			Name: "failing to update custom hostname",
+			Endpoints: []*endpoint.Endpoint{
+				{
+					DNSName:    "fail.foo.bar.com",
+					Targets:    endpoint.Targets{"1.2.3.4"},
+					RecordType: endpoint.RecordTypeA,
+					RecordTTL:  endpoint.TTL(defaultCloudFlareRecordTTL),
+					Labels:     endpoint.Labels{},
+					ProviderSpecific: endpoint.ProviderSpecific{
+						{
+							Name:  "external-dns.alpha.kubernetes.io/cloudflare-custom-hostname",
+							Value: "newerror-create.foo.fancybar.com",
+						},
+					},
+				},
+			},
+			shouldFail: true,
+		},
+		{
+			Name: "adding failing to list custom hostname",
+			Endpoints: []*endpoint.Endpoint{
+				{
+					DNSName:    "fail.list.foo.bar.com",
+					Targets:    endpoint.Targets{"1.2.3.4"},
+					RecordType: endpoint.RecordTypeA,
+					RecordTTL:  endpoint.TTL(defaultCloudFlareRecordTTL),
+					Labels:     endpoint.Labels{},
+					ProviderSpecific: endpoint.ProviderSpecific{
+						{
+							Name:  "external-dns.alpha.kubernetes.io/cloudflare-custom-hostname",
+							Value: "newerror-list-1.foo.fancybar.com",
+						},
+					},
+				},
+			},
+			shouldFail: true,
+		},
+		{
+			Name: "adding normal custom hostname",
+			Endpoints: []*endpoint.Endpoint{
+				{
+					DNSName:    "b.foo.bar.com",
+					Targets:    endpoint.Targets{"1.2.3.4"},
+					RecordType: endpoint.RecordTypeA,
+					RecordTTL:  endpoint.TTL(defaultCloudFlareRecordTTL),
+					Labels:     endpoint.Labels{},
+					ProviderSpecific: endpoint.ProviderSpecific{
+						{
+							Name:  "external-dns.alpha.kubernetes.io/cloudflare-custom-hostname",
+							Value: "b.foo.fancybar.com",
+						},
+					},
+				},
+			},
+			shouldFail: false,
+		},
+		{
+			Name: "updating to erroring custom hostname",
+			Endpoints: []*endpoint.Endpoint{
+				{
+					DNSName:    "b.foo.bar.com",
+					Targets:    endpoint.Targets{"1.2.3.4"},
+					RecordType: endpoint.RecordTypeA,
+					RecordTTL:  endpoint.TTL(defaultCloudFlareRecordTTL),
+					Labels:     endpoint.Labels{},
+					ProviderSpecific: endpoint.ProviderSpecific{
+						{
+							Name:  "external-dns.alpha.kubernetes.io/cloudflare-custom-hostname",
+							Value: "newerror-create.foo.fancybar.com",
+						},
+					},
+				},
+			},
+			shouldFail: true,
+		},
+		{
+			Name: "set to custom hostname which would error on removing",
+			Endpoints: []*endpoint.Endpoint{
+				{
+					DNSName:    "b.foo.bar.com",
+					Targets:    endpoint.Targets{"1.2.3.4"},
+					RecordType: endpoint.RecordTypeA,
+					RecordTTL:  endpoint.TTL(defaultCloudFlareRecordTTL),
+					Labels:     endpoint.Labels{},
+					ProviderSpecific: endpoint.ProviderSpecific{
+						{
+							Name:  "external-dns.alpha.kubernetes.io/cloudflare-custom-hostname",
+							Value: "newerror-delete.foo.fancybar.com",
+						},
+					},
+				},
+			},
+			shouldFail: false,
+		},
+		{
+			Name: "delete erroring on remove custom hostname",
+			Endpoints: []*endpoint.Endpoint{
+				{
+					DNSName:    "b.foo.bar.com",
+					Targets:    endpoint.Targets{"1.2.3.4"},
+					RecordType: endpoint.RecordTypeA,
+					RecordTTL:  endpoint.TTL(defaultCloudFlareRecordTTL),
+					Labels:     endpoint.Labels{},
+				},
+			},
+			shouldFail: true,
+		},
+		{
+			Name: "create erroring to remove custom hostname on record deletion",
+			Endpoints: []*endpoint.Endpoint{
+				{
+					DNSName:    "b.foo.bar.com",
+					Targets:    endpoint.Targets{"1.2.3.4"},
+					RecordType: endpoint.RecordTypeA,
+					RecordTTL:  endpoint.TTL(defaultCloudFlareRecordTTL),
+					Labels:     endpoint.Labels{},
+					ProviderSpecific: endpoint.ProviderSpecific{
+						{
+							Name:  "external-dns.alpha.kubernetes.io/cloudflare-custom-hostname",
+							Value: "newerror-delete.foo.fancybar.com",
+						},
+					},
+				},
+			},
+			shouldFail: false,
+		},
+		{
+			Name:       "failing to remove custom hostname on record deletion",
+			Endpoints:  []*endpoint.Endpoint{},
+			shouldFail: true,
+		},
+	}
+
 	testCases := []struct {
 		Name                    string
 		Endpoints               []*endpoint.Endpoint
 		ExpectedCustomHostnames map[string]string
 	}{
+		{
+			Name: "add A record without custom hostname",
+			Endpoints: []*endpoint.Endpoint{
+				{
+					DNSName:    "nocustomhostname.foo.bar.com",
+					Targets:    endpoint.Targets{"1.2.3.4"},
+					RecordType: endpoint.RecordTypeA,
+					RecordTTL:  endpoint.TTL(defaultCloudFlareRecordTTL),
+					Labels:     endpoint.Labels{},
+				},
+			},
+			ExpectedCustomHostnames: map[string]string{
+				"nocustomhostname.foo.bar.com": "",
+			},
+		},
 		{
 			Name: "add custom hostname",
 			Endpoints: []*endpoint.Endpoint{
 				{
 					DNSName:    "a.foo.bar.com",
-					Targets:    endpoint.Targets{"1.2.3.4", "2.3.4.5"},
+					Targets:    endpoint.Targets{"1.2.3.4"},
 					RecordType: endpoint.RecordTypeA,
 					RecordTTL:  endpoint.TTL(defaultCloudFlareRecordTTL),
 					Labels:     endpoint.Labels{},
@@ -1682,7 +2095,7 @@ func TestCloudflareCustomHostnameOperations(t *testing.T) {
 			Endpoints: []*endpoint.Endpoint{
 				{
 					DNSName:    "a.foo.bar.com",
-					Targets:    endpoint.Targets{"1.2.3.4", "2.3.4.5"},
+					Targets:    endpoint.Targets{"1.2.3.4"},
 					RecordType: endpoint.RecordTypeA,
 					RecordTTL:  endpoint.TTL(defaultCloudFlareRecordTTL),
 					Labels:     endpoint.Labels{},
@@ -1694,24 +2107,60 @@ func TestCloudflareCustomHostnameOperations(t *testing.T) {
 					},
 				},
 			},
-			ExpectedCustomHostnames: map[string]string{"a.foo.bar.com": "a2.foo.fancybar.com"},
+			ExpectedCustomHostnames: map[string]string{
+				"a.foo.bar.com": "a2.foo.fancybar.com",
+			},
 		},
 		{
 			Name: "delete custom hostname",
 			Endpoints: []*endpoint.Endpoint{
+
 				{
 					DNSName:    "a.foo.bar.com",
-					Targets:    endpoint.Targets{"1.2.3.4", "2.3.4.5"},
+					Targets:    endpoint.Targets{"1.2.3.4"},
 					RecordType: endpoint.RecordTypeA,
 					RecordTTL:  endpoint.TTL(defaultCloudFlareRecordTTL),
 					Labels:     endpoint.Labels{},
 				},
 			},
-			ExpectedCustomHostnames: map[string]string{"a.foo.bar.com": ""},
+			ExpectedCustomHostnames: map[string]string{
+				"a.foo.bar.com": "",
+			},
 		},
 	}
 
+	for _, tc := range testFailCases {
+		records, err := provider.Records(ctx)
+		if err != nil {
+			t.Errorf("should not fail, %s", err)
+		}
+
+		endpoints, err := provider.AdjustEndpoints(tc.Endpoints)
+
+		assert.NoError(t, err)
+		plan := &plan.Plan{
+			Current:        records,
+			Desired:        endpoints,
+			DomainFilter:   endpoint.MatchAllDomainFilters{&domainFilter},
+			ManagedRecords: []string{endpoint.RecordTypeA, endpoint.RecordTypeCNAME},
+		}
+
+		planned := plan.Calculate()
+
+		err = provider.ApplyChanges(context.Background(), planned.Changes)
+		if err == nil && tc.shouldFail {
+			t.Errorf("should fail - %s, %s", tc.Name, err)
+		} else if err != nil && !tc.shouldFail {
+			t.Errorf("should not fail - %s, %s", tc.Name, err)
+		}
+	}
+
 	for _, tc := range testCases {
+		records, err := provider.Records(ctx)
+		if err != nil {
+			t.Errorf("should not fail, %s", err)
+		}
+
 		endpoints, err := provider.AdjustEndpoints(tc.Endpoints)
 
 		assert.NoError(t, err)
@@ -1726,17 +2175,106 @@ func TestCloudflareCustomHostnameOperations(t *testing.T) {
 
 		err = provider.ApplyChanges(context.Background(), planned.Changes)
 		if err != nil {
-			t.Errorf("should not fail, %s", err)
+			t.Errorf("should not fail - %s, %s", tc.Name, err)
 		}
 
 		chs, chErr := provider.listCustomHostnamesWithPagination(ctx, "001")
 		if chErr != nil {
-			t.Errorf("should not fail, %s", chErr)
+			t.Errorf("should not fail - %s, %s", tc.Name, chErr)
 		}
 
-		for k, v := range tc.ExpectedCustomHostnames {
+		for expectedOrigin, expectedCustomHostname := range tc.ExpectedCustomHostnames {
-			_, ch := provider.getCustomHostnameIDbyOrigin(chs, k)
+			_, ch := provider.getCustomHostnameIDbyCustomHostnameAndOrigin(chs, expectedCustomHostname, expectedOrigin)
-			assert.Equal(t, v, ch)
+			assert.Equal(t, expectedCustomHostname, ch)
 		}
 	}
 }
+
+func TestCloudflareCustomHostnameNotFoundOnRecordDeletion(t *testing.T) {
+	client := NewMockCloudFlareClient()
+	provider := &CloudFlareProvider{
+		Client:                client,
+		CustomHostnamesConfig: CustomHostnamesConfig{Enabled: true},
+	}
+	ctx := context.Background()
+	zoneID := "001"
+	domainFilter := endpoint.NewDomainFilter([]string{"bar.com"})
+
+	testCases := []struct {
+		Name                    string
+		Endpoints               []*endpoint.Endpoint
+		ExpectedCustomHostnames map[string]string
+		preApplyHook            bool
+	}{
+		{
+			Name: "create DNS record with custom hostname",
+			Endpoints: []*endpoint.Endpoint{
+				{
+					DNSName:    "create.foo.bar.com",
+					Targets:    endpoint.Targets{"1.2.3.4"},
+					RecordType: endpoint.RecordTypeA,
+					RecordTTL:  endpoint.TTL(defaultCloudFlareRecordTTL),
+					Labels:     endpoint.Labels{},
+					ProviderSpecific: endpoint.ProviderSpecific{
+						{
+							Name:  "external-dns.alpha.kubernetes.io/cloudflare-custom-hostname",
+							Value: "newerror-getCustomHostnameOrigin.foo.fancybar.com",
+						},
+					},
+				},
+			},
+			preApplyHook: false,
+		},
+		{
+			Name:         "remove DNS record with unexpectedly missing custom hostname",
+			Endpoints:    []*endpoint.Endpoint{},
+			preApplyHook: true,
+		},
+	}
+
+	b := testutils.LogsToBuffer(log.InfoLevel, t)
+	for _, tc := range testCases {
+		records, err := provider.Records(ctx)
+		if err != nil {
+			t.Errorf("should not fail, %s", err)
+		}
+
+		endpoints, err := provider.AdjustEndpoints(tc.Endpoints)
+
+		assert.NoError(t, err)
+		plan := &plan.Plan{
+			Current:        records,
+			Desired:        endpoints,
+			DomainFilter:   endpoint.MatchAllDomainFilters{&domainFilter},
+			ManagedRecords: []string{endpoint.RecordTypeA, endpoint.RecordTypeCNAME},
+		}
+
+		planned := plan.Calculate()
+
+		// manually corrupt custom hostname before the deletion step
+		// the purpose is to cause getCustomHostnameOrigin() to fail on change.Action == cloudFlareDelete
+		if tc.preApplyHook {
+			chs, chErr := provider.listCustomHostnamesWithPagination(ctx, zoneID)
+			if chErr != nil {
+				t.Errorf("should not fail - %s, %s", tc.Name, chErr)
+			}
+			chID, _ := provider.getCustomHostnameOrigin(chs, "newerror-getCustomHostnameOrigin.foo.fancybar.com")
+			if chID != "" {
+				t.Logf("corrupting custom hostname %v", chID)
+				oldCh := client.customHostnames[zoneID][chID]
+				ch := cloudflare.CustomHostname{
+					Hostname:           "corrupted-newerror-getCustomHostnameOrigin.foo.fancybar.com",
+					CustomOriginServer: oldCh.CustomOriginServer,
+					SSL:                oldCh.SSL,
+				}
+				client.customHostnames[zoneID][chID] = ch
+			}
+		}
+
+		err = provider.ApplyChanges(context.Background(), planned.Changes)
+		if err != nil {
+			t.Errorf("should not fail - %s, %s", tc.Name, err)
+		}
+	}
+	assert.Contains(t, b.String(), "level=info msg=\"Custom hostname newerror-getCustomHostnameOrigin.foo.fancybar.com not found\" action=DELETE record=create.foo.bar.com")
+}