feat(azure): add support for workload identity

Signed-off-by: Igor Beliakov <demtis.register@gmail.com>
This commit is contained in:
Igor Beliakov 2023-09-01 19:08:42 +02:00
parent b2de466aa5
commit a1c2775d45
2 changed files with 167 additions and 10 deletions

View File

@ -53,6 +53,7 @@ The following fields are used:
* `aadClientID` and `aadClientSecret` are associated with the Service Principal. This is only used with Service Principal method documented in the next section. * `aadClientID` and `aadClientSecret` are associated with the Service Principal. This is only used with Service Principal method documented in the next section.
* `useManagedIdentityExtension` - this is set to `true` if you use either AKS Kubelet Identity or AAD Pod Identities methods documented in the next section. * `useManagedIdentityExtension` - this is set to `true` if you use either AKS Kubelet Identity or AAD Pod Identities methods documented in the next section.
* `userAssignedIdentityID` - this contains the client id from the Managed identitty when using the AAD Pod Identities method documented in the next setion. * `userAssignedIdentityID` - this contains the client id from the Managed identitty when using the AAD Pod Identities method documented in the next setion.
* `useWorkloadIdentityExtension` - this is set to `true` if you use Workload Identity method documented in the next section.
The Azure DNS provider expects, by default, that the configuration file is at `/etc/kubernetes/azure.json`. This can be overridden with the `--azure-config-file` option when starting ExternalDNS. The Azure DNS provider expects, by default, that the configuration file is at `/etc/kubernetes/azure.json`. This can be overridden with the `--azure-config-file` option when starting ExternalDNS.
@ -63,6 +64,7 @@ ExternalDNS needs permissions to make changes to the Azure DNS zone. There are t
- [Service Principal](#service-principal) - [Service Principal](#service-principal)
- [Managed Identity Using AKS Kubelet Identity](#managed-identity-using-aks-kubelet-identity) - [Managed Identity Using AKS Kubelet Identity](#managed-identity-using-aks-kubelet-identity)
- [Managed Identity Using AAD Pod Identities](#managed-identity-using-aad-pod-identities) - [Managed Identity Using AAD Pod Identities](#managed-identity-using-aad-pod-identities)
- [Managed Identity Using Workload Identity](#managed-identity-using-workload-identity)
### Service Principal ### Service Principal
@ -319,6 +321,136 @@ kubectl patch deployment external-dns --namespace "default" --patch \
'{"spec": {"template": {"metadata": {"labels": {"aadpodidbinding": "external-dns"}}}}}' '{"spec": {"template": {"metadata": {"labels": {"aadpodidbinding": "external-dns"}}}}}'
``` ```
### Managed identity using Workload Identity
For this process, we will create a [managed identity](https://docs.microsoft.com//azure/active-directory/managed-identities-azure-resources/overview) that will be explicitly used by the ExternalDNS container. This process is somewhat similar to Pod Identity except that this managed identity is associated with a kubernetes service account.
#### Deploy OIDC issuer and Workload Identity services
Update your cluster to install [OIDC Issuer](https://learn.microsoft.com/en-us/azure/aks/use-oidc-issuer) and [Workload Identity](https://learn.microsoft.com/en-us/azure/aks/workload-identity-deploy-cluster):
```bash
$ AZURE_AKS_RESOURCE_GROUP="my-aks-cluster-group" # name of resource group where aks cluster was created
$ AZURE_AKS_CLUSTER_NAME="my-aks-cluster" # name of aks cluster previously created
$ az aks update --resource-group ${AZURE_AKS_RESOURCE_GROUP} --name ${AZURE_AKS_CLUSTER_NAME} --enable-oidc-issuer --enable-workload-identity
```
#### Create a managed identity
Create a managed identity:
```bash
$ IDENTITY_RESOURCE_GROUP=$AZURE_AKS_RESOURCE_GROUP # custom group or reuse AKS group
$ IDENTITY_NAME="example-com-identity"
# create a managed identity
$ az identity create --resource-group "${IDENTITY_RESOURCE_GROUP}" --name "${IDENTITY_NAME}"
```
#### Assign a role to the managed identity
Grant access to Azure DNS zone for the managed identity:
```bash
$ AZURE_DNS_ZONE_RESOURCE_GROUP="MyDnsResourceGroup" # name of resource group where dns zone is hosted
$ AZURE_DNS_ZONE="example.com" # DNS zone name like example.com or sub.example.com
# fetch identity client id from managed identity created earlier
$ IDENTITY_CLIENT_ID=$(az identity show --resource-group "${IDENTITY_RESOURCE_GROUP}" \
--name "${IDENTITY_NAME}" --query "clientId" --output tsv)
# fetch DNS id used to grant access to the managed identity
$ DNS_ID=$(az network dns zone show --name "${AZURE_DNS_ZONE}" \
--resource-group "${AZURE_DNS_ZONE_RESOURCE_GROUP}" --query "id" --output tsv)
$ RESOURCE_GROUP_ID=$(az group show --name "${AZURE_DNS_ZONE_RESOURCE_GROUP}" --query "id" --output tsv)
$ az role assignment create --role "DNS Zone Contributor" \
--assignee "${IDENTITY_CLIENT_ID}" --scope "${DNS_ID}"
$ az role assignment create --role "Reader" \
--assignee "${IDENTITY_CLIENT_ID}" --scope "${RESOURCE_GROUP_ID}"
```
#### Create a federated identity credential
A binding between the managed identity and the ExternalDNS service account needs to be setup by creating a federated identity resource:
```bash
$ OIDC_ISSUER_URL="$(az aks show -n myAKSCluster -g myResourceGroup --query "oidcIssuerProfile.issuerUrl" -otsv)"
$ az identity federated-credential create --name ${IDENTITY_NAME} --identity-name ${IDENTITY_NAME} --resource-group $AZURE_AKS_RESOURCE_GROUP} --issuer "$OIDC_ISSUER_URL" --subject "system:serviceaccount:default:external-dns"
```
NOTE: make sure federated credential refers to correct namespace and service account (`system:serviceaccount:<NAMESPACE>:<SERVICE_ACCOUNT>`)
#### helm
When deploying external-dns with helm, here are the parameters you need to pass:
```yaml
fullnameOverride: external-dns
serviceAccount:
annotations:
azure.workload.identity/client-id: <IDENTITY_CLIENT_ID>
podLabels:
azure.workload.identity/use: "true"
provider: azure
secretConfiguration:
enabled: true
mountPath: "/etc/kubernetes/"
data:
azure.json: |
{
"subscriptionId": "<SUBSCRIPTION_ID>",
"resourceGroup": "<AZURE_DNS_ZONE_RESOURCE_GROUP>",
"useWorkloadIdentityExtension": true
}
```
NOTE: make sure the pod is restarted whenever you make a configuration change.
#### kubectl (alternative)
##### Create a configuration file for the managed identity
Create the file `azure.json` with the values from previous steps:
```bash
cat <<-EOF > /local/path/to/azure.json
{
"subscriptionId": "$(az account show --query id -o tsv)",
"resourceGroup": "$AZURE_DNS_ZONE_RESOURCE_GROUP",
"useWorkloadIdentityExtension": true
}
EOF
```
Use the `azure.json` file to create a Kubernetes secret:
```bash
$ kubectl create secret generic azure-config-file --namespace "default" --from-file /local/path/to/azure.json
```
##### Update labels and annotations on ExternalDNS service account
To instruct Workload Identity webhook to inject a projected token into the ExternalDNS pod, the pod needs to have a label `azure.workload.identity/use: "true"` (before Workload Identity 1.0.0, this label was supposed to be set on the service account instead). Also, the service account needs to have an annotation `azure.workload.identity/client-id: <IDENTITY_CLIENT_ID>`:
To patch the existing serviceaccount and deployment, use the following command:
```bash
$ kubectl patch serviceaccount external-dns --namespace "default" --patch \
"{\"metadata\": {\"annotations\": {\"azure.workload.identity/client-id\": \"${IDENTITY_CLIENT_ID}\"}}}"
$ kubectl patch deployment external-dns --namespace "default" --patch \
'{"spec": {"template": {"metadata": {"labels": {\"azure.workload.identity/use\": \"true\"}}}}}'
```
NOTE: it's also possible to specify (or override) ClientID through `UserAssignedIdentityID` field in `azure.json`.
NOTE: make sure the pod is restarted whenever you make a configuration change.
## Ingress used with ExternalDNS ## Ingress used with ExternalDNS
This deployment assumes that you will be using nginx-ingress. When using nginx-ingress do not deploy it as a Daemon Set. This causes nginx-ingress to write the Cluster IP of the backend pods in the ingress status.loadbalancer.ip property which then has external-dns write the Cluster IP(s) in DNS vs. the nginx-ingress service external IP. This deployment assumes that you will be using nginx-ingress. When using nginx-ingress do not deploy it as a Daemon Set. This causes nginx-ingress to write the Cluster IP of the backend pods in the ingress status.loadbalancer.ip property which then has external-dns write the Cluster IP(s) in DNS vs. the nginx-ingress service external IP.
@ -651,6 +783,6 @@ $ az group delete --name "MyDnsResourceGroup"
## More tutorials ## More tutorials
A video explanantion is available here: https://www.youtube.com/watch?v=VSn6DPKIhM8&list=PLpbcUe4chE79sB7Jg7B4z3HytqUUEwcNE A video explanantion is available here: https://www.youtube.com/watch?v=VSn6DPKIhM8&list=PLpbcUe4chE79sB7Jg7B4z3HytqUUEwcNE
![image](https://user-images.githubusercontent.com/6548359/235437721-87611869-75f2-4f32-bb35-9da585e46299.png) ![image](https://user-images.githubusercontent.com/6548359/235437721-87611869-75f2-4f32-bb35-9da585e46299.png)

View File

@ -30,15 +30,16 @@ import (
// config represents common config items for Azure DNS and Azure Private DNS // config represents common config items for Azure DNS and Azure Private DNS
type config struct { type config struct {
Cloud string `json:"cloud" yaml:"cloud"` Cloud string `json:"cloud" yaml:"cloud"`
TenantID string `json:"tenantId" yaml:"tenantId"` TenantID string `json:"tenantId" yaml:"tenantId"`
SubscriptionID string `json:"subscriptionId" yaml:"subscriptionId"` SubscriptionID string `json:"subscriptionId" yaml:"subscriptionId"`
ResourceGroup string `json:"resourceGroup" yaml:"resourceGroup"` ResourceGroup string `json:"resourceGroup" yaml:"resourceGroup"`
Location string `json:"location" yaml:"location"` Location string `json:"location" yaml:"location"`
ClientID string `json:"aadClientId" yaml:"aadClientId"` ClientID string `json:"aadClientId" yaml:"aadClientId"`
ClientSecret string `json:"aadClientSecret" yaml:"aadClientSecret"` ClientSecret string `json:"aadClientSecret" yaml:"aadClientSecret"`
UseManagedIdentityExtension bool `json:"useManagedIdentityExtension" yaml:"useManagedIdentityExtension"` UseManagedIdentityExtension bool `json:"useManagedIdentityExtension" yaml:"useManagedIdentityExtension"`
UserAssignedIdentityID string `json:"userAssignedIdentityID" yaml:"userAssignedIdentityID"` UseWorkloadIdentityExtension bool `json:"useWorkloadIdentityExtension" yaml:"useWorkloadIdentityExtension"`
UserAssignedIdentityID string `json:"userAssignedIdentityID" yaml:"userAssignedIdentityID"`
} }
func getConfig(configFile, resourceGroup, userAssignedIdentityClientID string) (*config, error) { func getConfig(configFile, resourceGroup, userAssignedIdentityClientID string) (*config, error) {
@ -93,6 +94,30 @@ func getCredentials(cfg config) (azcore.TokenCredential, error) {
return cred, nil return cred, nil
} }
// Try to retrieve token with Workload Identity.
if cfg.UseWorkloadIdentityExtension {
log.Info("Using workload identity extension to retrieve access token for Azure API.")
wiOpt := azidentity.WorkloadIdentityCredentialOptions{
ClientOptions: azcore.ClientOptions{
Cloud: cloudCfg,
},
// In a standard scenario, Client ID and Tenant ID are expected to be read from environment variables.
// Though, in certain cases, it might be important to have an option to override those (e.g. when AZURE_TENANT_ID is not set
// through a webhook or azure.workload.identity/client-id service account annotation is absent). When any of those values are
// empty in our config, they will automatically be read from environment variables by azidentity
TenantID: cfg.TenantID,
ClientID: cfg.ClientID,
}
cred, err := azidentity.NewWorkloadIdentityCredential(&wiOpt)
if err != nil {
return nil, fmt.Errorf("failed to create a workload identity token: %w", err)
}
return cred, nil
}
// Try to retrieve token with MSI. // Try to retrieve token with MSI.
if cfg.UseManagedIdentityExtension { if cfg.UseManagedIdentityExtension {
log.Info("Using managed identity extension to retrieve access token for Azure API.") log.Info("Using managed identity extension to retrieve access token for Azure API.")