feat: add azkv.environmentType (#1469)

users of USGovCloud, ChinaCloud, GermanCloud need slightly different
configuration for AADEndpoint and keyvault resource.

This is based on CSI Secret Store Azure KV driver,

Signed-off-by: Moritz Johner <beller.moritz@googlemail.com>

Signed-off-by: Moritz Johner <beller.moritz@googlemail.com>

apis/externalsecrets/v1beta1/secretstore_azurekv_types.go

@@ -34,6 +34,20 @@ const (
 	AzureWorkloadIdentity AzureAuthType = "WorkloadIdentity"
 )
 
+// AzureEnvironmentType specifies the Azure cloud environment endpoints to use for
+// connecting and authenticating with Azure. By default it points to the public cloud AAD endpoint.
+// The following endpoints are available, also see here: https://github.com/Azure/go-autorest/blob/main/autorest/azure/environments.go#L152
+// PublicCloud, USGovernmentCloud, ChinaCloud, GermanCloud
+// +kubebuilder:validation:Enum=PublicCloud;USGovernmentCloud;ChinaCloud;GermanCloud
+type AzureEnvironmentType string
+
+const (
+	AzureEnvironmentPublicCloud       AzureEnvironmentType = "PublicCloud"
+	AzureEnvironmentUSGovernmentCloud AzureEnvironmentType = "USGovernmentCloud"
+	AzureEnvironmentChinaCloud        AzureEnvironmentType = "ChinaCloud"
+	AzureEnvironmentGermanCloud       AzureEnvironmentType = "GermanCloud"
+)
+
 // Configures an store to sync secrets using Azure KV.
 type AzureKVProvider struct {
 	// Auth type defines how to authenticate to the keyvault service.
@@ -51,6 +65,13 @@ type AzureKVProvider struct {
 	// +optional
 	TenantID *string `json:"tenantId,omitempty"`
 
+	// EnvironmentType specifies the Azure cloud environment endpoints to use for
+	// connecting and authenticating with Azure. By default it points to the public cloud AAD endpoint.
+	// The following endpoints are available, also see here: https://github.com/Azure/go-autorest/blob/main/autorest/azure/environments.go#L152
+	// PublicCloud, USGovernmentCloud, ChinaCloud, GermanCloud
+	// +kubebuilder:default=PublicCloud
+	EnvironmentType AzureEnvironmentType `json:"environmentType,omitempty"`
+
 	// Auth configures how the operator authenticates with Azure. Required for ServicePrincipal auth type.
 	// +optional
 	AuthSecretRef *AzureKVAuth `json:"authSecretRef,omitempty"`

config/crds/bases/external-secrets.io_clustersecretstores.yaml

@@ -1734,6 +1734,19 @@ spec:
                         - ManagedIdentity
                         - WorkloadIdentity
                         type: string
+                      environmentType:
+                        default: PublicCloud
+                        description: 'EnvironmentType specifies the Azure cloud environment
+                          endpoints to use for connecting and authenticating with
+                          Azure. By default it points to the public cloud AAD endpoint.
+                          The following endpoints are available, also see here: https://github.com/Azure/go-autorest/blob/main/autorest/azure/environments.go#L152
+                          PublicCloud, USGovernmentCloud, ChinaCloud, GermanCloud'
+                        enum:
+                        - PublicCloud
+                        - USGovernmentCloud
+                        - ChinaCloud
+                        - GermanCloud
+                        type: string
                       identityId:
                         description: If multiple Managed Identity is assigned to the
                           pod, you can select the one to be used

config/crds/bases/external-secrets.io_secretstores.yaml

@@ -1734,6 +1734,19 @@ spec:
                         - ManagedIdentity
                         - WorkloadIdentity
                         type: string
+                      environmentType:
+                        default: PublicCloud
+                        description: 'EnvironmentType specifies the Azure cloud environment
+                          endpoints to use for connecting and authenticating with
+                          Azure. By default it points to the public cloud AAD endpoint.
+                          The following endpoints are available, also see here: https://github.com/Azure/go-autorest/blob/main/autorest/azure/environments.go#L152
+                          PublicCloud, USGovernmentCloud, ChinaCloud, GermanCloud'
+                        enum:
+                        - PublicCloud
+                        - USGovernmentCloud
+                        - ChinaCloud
+                        - GermanCloud
+                        type: string
                       identityId:
                         description: If multiple Managed Identity is assigned to the
                           pod, you can select the one to be used

deploy/crds/bundle.yaml

@@ -1654,6 +1654,15 @@ spec:
                             - ManagedIdentity
                             - WorkloadIdentity
                           type: string
+                        environmentType:
+                          default: PublicCloud
+                          description: 'EnvironmentType specifies the Azure cloud environment endpoints to use for connecting and authenticating with Azure. By default it points to the public cloud AAD endpoint. The following endpoints are available, also see here: https://github.com/Azure/go-autorest/blob/main/autorest/azure/environments.go#L152 PublicCloud, USGovernmentCloud, ChinaCloud, GermanCloud'
+                          enum:
+                            - PublicCloud
+                            - USGovernmentCloud
+                            - ChinaCloud
+                            - GermanCloud
+                          type: string
                         identityId:
                           description: If multiple Managed Identity is assigned to the pod, you can select the one to be used
                           type: string
@@ -4374,6 +4383,15 @@ spec:
                             - ManagedIdentity
                             - WorkloadIdentity
                           type: string
+                        environmentType:
+                          default: PublicCloud
+                          description: 'EnvironmentType specifies the Azure cloud environment endpoints to use for connecting and authenticating with Azure. By default it points to the public cloud AAD endpoint. The following endpoints are available, also see here: https://github.com/Azure/go-autorest/blob/main/autorest/azure/environments.go#L152 PublicCloud, USGovernmentCloud, ChinaCloud, GermanCloud'
+                          enum:
+                            - PublicCloud
+                            - USGovernmentCloud
+                            - ChinaCloud
+                            - GermanCloud
+                          type: string
                         identityId:
                           description: If multiple Managed Identity is assigned to the pod, you can select the one to be used
                           type: string

docs/provider-azure-key-vault.md

@@ -11,6 +11,20 @@ We support Service Principals, Managed Identity and Workload Identity authentica
 
 To use Managed Identity authentication, you should use [aad-pod-identity](https://azure.github.io/aad-pod-identity/docs/) to assign the identity to external-secrets operator. To add the selector to external-secrets operator, use `podLabels` in your values.yaml in case of Helm installation of external-secrets.
 
+We support connecting to different cloud flavours azure supports: `PublicCloud`, `USGovernmentCloud`, `ChinaCloud` and `GermanCloud`. You have to specify the `environmentType` and point to the correct cloud flavour. This defaults to `PublicCloud`.
+
+```
+apiVersion: external-secrets.io/v1beta1
+kind: SecretStore
+metadata:
+  name: azure-backend
+spec:
+  provider:
+    azurekv:
+      # PublicCloud, USGovernmentCloud, ChinaCloud, GermanCloud
+      environmentType: PublicCloud # default
+```
+
 Minimum required permissions are `Get` over secret and certificate permissions. This can be done by adding a Key Vault access policy:
 
 ```sh

pkg/provider/azure/keyvault/keyvault.go

@@ -27,6 +27,7 @@ import (
 	"github.com/Azure/azure-sdk-for-go/profiles/latest/keyvault/keyvault"
 	"github.com/Azure/go-autorest/autorest"
 	"github.com/Azure/go-autorest/autorest/adal"
+	"github.com/Azure/go-autorest/autorest/azure"
 	kvauth "github.com/Azure/go-autorest/autorest/azure/auth"
 	"github.com/AzureAD/microsoft-authentication-library-for-go/apps/confidential"
 	"github.com/tidwall/gjson"
@@ -48,7 +49,6 @@ const (
 	defaultObjType       = "secret"
 	objectTypeCert       = "cert"
 	objectTypeKey        = "key"
-	vaultResource        = "https://vault.azure.net"
 	azureDefaultAudience = "api://AzureADTokenExchange"
 	annotationClientID   = "azure.workload.identity/client-id"
 	annotationTenantID   = "azure.workload.identity/tenant-id"
@@ -423,6 +423,8 @@ func getSecretMapProperties(tags map[string]*string, key, property string) map[s
 }
 
 func (a *Azure) authorizerForWorkloadIdentity(ctx context.Context, tokenProvider tokenProviderFunc) (autorest.Authorizer, error) {
+	aadEndpoint := aadEndpointForProviderConfig(a.provider)
+	kvResource := kvResourceForProviderConfig(a.provider)
 	// if no serviceAccountRef was provided
 	// we expect certain env vars to be present.
 	// They are set by the azure workload identity webhook.
@@ -437,7 +439,7 @@ func (a *Azure) authorizerForWorkloadIdentity(ctx context.Context, tokenProvider
 		if err != nil {
 			return nil, fmt.Errorf(errReadTokenFile, tokenFilePath, err)
 		}
-		tp, err := tokenProvider(ctx, string(token), clientID, tenantID)
+		tp, err := tokenProvider(ctx, string(token), clientID, tenantID, aadEndpoint, kvResource)
 		if err != nil {
 			return nil, err
 		}
@@ -467,7 +469,7 @@ func (a *Azure) authorizerForWorkloadIdentity(ctx context.Context, tokenProvider
 	if err != nil {
 		return nil, err
 	}
-	tp, err := tokenProvider(ctx, token, clientID, tenantID)
+	tp, err := tokenProvider(ctx, token, clientID, tenantID, aadEndpoint, kvResource)
 	if err != nil {
 		return nil, err
 	}
@@ -491,26 +493,27 @@ type tokenProvider struct {
 	accessToken string
 }
 
-type tokenProviderFunc func(ctx context.Context, token, clientID, tenantID string) (adal.OAuthTokenProvider, error)
+type tokenProviderFunc func(ctx context.Context, token, clientID, tenantID, aadEndpoint, kvResource string) (adal.OAuthTokenProvider, error)
 
-func newTokenProvider(ctx context.Context, token, clientID, tenantID string) (adal.OAuthTokenProvider, error) {
+func newTokenProvider(ctx context.Context, token, clientID, tenantID, aadEndpoint, kvResource string) (adal.OAuthTokenProvider, error) {
 	// exchange token with Azure AccessToken
 	cred, err := confidential.NewCredFromAssertion(token)
 	if err != nil {
 		return nil, err
 	}
-
-	// AZURE_AUTHORITY_HOST
-
 	cClient, err := confidential.New(clientID, cred, confidential.WithAuthority(
-		fmt.Sprintf("https://login.microsoftonline.com/%s/oauth2/token", tenantID),
+		fmt.Sprintf("%s%s/oauth2/token", aadEndpoint, tenantID),
 	))
 	if err != nil {
 		return nil, err
 	}
-
+	scope := kvResource
+	// .default needs to be added to the scope
+	if !strings.Contains(kvResource, ".default") {
+		scope = fmt.Sprintf("%s/.default", kvResource)
+	}
 	authRes, err := cClient.AcquireTokenByCredential(ctx, []string{
-		"https://vault.azure.net/.default",
+		scope,
 	})
 	if err != nil {
 		return nil, err
@@ -526,7 +529,7 @@ func (t *tokenProvider) OAuthToken() string {
 
 func (a *Azure) authorizerForManagedIdentity() (autorest.Authorizer, error) {
 	msiConfig := kvauth.NewMSIConfig()
-	msiConfig.Resource = vaultResource
+	msiConfig.Resource = kvResourceForProviderConfig(a.provider)
 	if a.provider.IdentityID != nil {
 		msiConfig.ClientID = *a.provider.IdentityID
 	}
@@ -555,9 +558,9 @@ func (a *Azure) authorizerForServicePrincipal(ctx context.Context) (autorest.Aut
 	if err != nil {
 		return nil, err
 	}
-
 	clientCredentialsConfig := kvauth.NewClientCredentialsConfig(cid, csec, *a.provider.TenantID)
-	clientCredentialsConfig.Resource = vaultResource
+	clientCredentialsConfig.Resource = kvResourceForProviderConfig(a.provider)
+	clientCredentialsConfig.AADEndpoint = aadEndpointForProviderConfig(a.provider)
 	return clientCredentialsConfig.Authorizer()
 }
 
@@ -591,6 +594,38 @@ func (a *Azure) Validate() (esv1beta1.ValidationResult, error) {
 	return esv1beta1.ValidationResultReady, nil
 }
 
+func aadEndpointForProviderConfig(prov *esv1beta1.AzureKVProvider) string {
+	switch prov.EnvironmentType {
+	case esv1beta1.AzureEnvironmentPublicCloud:
+		return azure.PublicCloud.ActiveDirectoryEndpoint
+	case esv1beta1.AzureEnvironmentChinaCloud:
+		return azure.ChinaCloud.ActiveDirectoryEndpoint
+	case esv1beta1.AzureEnvironmentUSGovernmentCloud:
+		return azure.USGovernmentCloud.ActiveDirectoryEndpoint
+	case esv1beta1.AzureEnvironmentGermanCloud:
+		return azure.GermanCloud.ActiveDirectoryEndpoint
+	default:
+		return azure.PublicCloud.ActiveDirectoryEndpoint
+	}
+}
+
+func kvResourceForProviderConfig(prov *esv1beta1.AzureKVProvider) string {
+	var res string
+	switch prov.EnvironmentType {
+	case esv1beta1.AzureEnvironmentPublicCloud:
+		res = azure.PublicCloud.KeyVaultEndpoint
+	case esv1beta1.AzureEnvironmentChinaCloud:
+		res = azure.ChinaCloud.KeyVaultEndpoint
+	case esv1beta1.AzureEnvironmentUSGovernmentCloud:
+		res = azure.USGovernmentCloud.KeyVaultEndpoint
+	case esv1beta1.AzureEnvironmentGermanCloud:
+		res = azure.GermanCloud.KeyVaultEndpoint
+	default:
+		res = azure.PublicCloud.KeyVaultEndpoint
+	}
+	return strings.TrimSuffix(res, "/")
+}
+
 func getObjType(ref esv1beta1.ExternalSecretDataRemoteRef) (string, string) {
 	objectType := defaultObjType
 

pkg/provider/azure/keyvault/keyvault_auth_test.go

@@ -193,7 +193,7 @@ func TestGetAuthorizorForWorkloadIdentity(t *testing.T) {
 				kubeClient: awsauthfake.NewCreateTokenMock(saToken),
 				provider:   store.Spec.Provider.AzureKV,
 			}
-			tokenProvider := func(ctx context.Context, token, clientID, tenantID string) (adal.OAuthTokenProvider, error) {
+			tokenProvider := func(ctx context.Context, token, clientID, tenantID, aadEndpoint, kvResource string) (adal.OAuthTokenProvider, error) {
 				tassert.Equal(t, token, saToken)
 				tassert.Equal(t, clientID, clientID)
 				tassert.Equal(t, tenantID, tenantID)