Merge pull request #805 from external-secrets/feature/azure-getallsecrets

Implement GetAllSecrets for Azure Key Vault

pkg/controllers/externalsecret/externalsecret_controller.go

@@ -47,7 +47,6 @@ const (
 
 	errGetES                 = "could not get ExternalSecret"
 	errConvert               = "could not apply conversion strategy to keys: %v"
-	errFindSecretKey         = "could not find secret %v: %v"
 	errUpdateSecret          = "could not update Secret"
 	errPatchStatus           = "unable to patch status"
 	errGetSecretStore        = "could not get SecretStore %q, %w"
@@ -66,7 +65,6 @@ const (
 	errPolicyMergeGetSecret  = "unable to get secret %s: %w"
 	errPolicyMergeMutate     = "unable to mutate secret %s: %w"
 	errPolicyMergePatch      = "unable to patch secret %s: %w"
-	errGetSecretKey          = "key %q from ExternalSecret %q: %w"
 	errTplCMMissingKey       = "error in configmap %s: missing key %s"
 	errTplSecMissingKey      = "error in secret %s: missing key %s"
 )
@@ -412,7 +410,7 @@ func (r *Reconciler) getProviderSecretData(ctx context.Context, providerClient e
 		if remoteRef.Find != nil {
 			secretMap, err = providerClient.GetAllSecrets(ctx, *remoteRef.Find)
 			if err != nil {
-				return nil, fmt.Errorf(errFindSecretKey, externalSecret.Name, err)
+				return nil, err
 			}
 			secretMap, err = utils.ConvertKeys(remoteRef.Find.ConversionStrategy, secretMap)
 			if err != nil {
@@ -421,7 +419,7 @@ func (r *Reconciler) getProviderSecretData(ctx context.Context, providerClient e
 		} else if remoteRef.Extract != nil {
 			secretMap, err = providerClient.GetSecretMap(ctx, *remoteRef.Extract)
 			if err != nil {
-				return nil, fmt.Errorf(errGetSecretKey, remoteRef.Extract.Key, externalSecret.Name, err)
+				return nil, err
 			}
 			secretMap, err = utils.ConvertKeys(remoteRef.Extract.ConversionStrategy, secretMap)
 			if err != nil {
@@ -435,7 +433,7 @@ func (r *Reconciler) getProviderSecretData(ctx context.Context, providerClient e
 	for _, secretRef := range externalSecret.Spec.Data {
 		secretData, err := providerClient.GetSecret(ctx, secretRef.RemoteRef)
 		if err != nil {
-			return nil, fmt.Errorf(errGetSecretKey, secretRef.RemoteRef.Key, externalSecret.Name, err)
+			return nil, err
 		}
 
 		providerData[secretRef.SecretKey] = secretData

pkg/provider/azure/keyvault/fake/fake.go

@@ -65,3 +65,11 @@ func (mc *AzureMockClient) WithCertificate(serviceURL, secretName, secretVersion
 		}
 	}
 }
+
+func (mc *AzureMockClient) WithList(serviceURL string, apiOutput keyvault.SecretListResultIterator, err error) {
+	if mc != nil {
+		mc.getSecretsComplete = func(ctx context.Context, vaultBaseURL string, maxresults *int32) (result keyvault.SecretListResultIterator, err error) {
+			return apiOutput, err
+		}
+	}
+}

pkg/provider/azure/keyvault/keyvault.go

@@ -20,6 +20,8 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"path"
+	"regexp"
 	"strings"
 
 	"github.com/Azure/azure-sdk-for-go/profiles/latest/keyvault/keyvault"
@@ -189,10 +191,42 @@ func (a *Azure) ValidateStore(store esv1beta1.GenericStore) error {
 	return nil
 }
 
-// Empty GetAllSecrets.
+// Implements store.Client.GetAllSecrets Interface.
+// Retrieves a map[string][]byte with the secret names as key and the secret itself as the calue.
 func (a *Azure) GetAllSecrets(ctx context.Context, ref esv1beta1.ExternalSecretFind) (map[string][]byte, error) {
-	// TO be implemented
-	return nil, fmt.Errorf("GetAllSecrets not implemented")
+	basicClient := a.baseClient
+	secretsMap := make(map[string][]byte)
+	checkTags := len(ref.Tags) > 0
+	checkName := ref.Name != nil && len(ref.Name.RegExp) > 0
+
+	secretListIter, err := basicClient.GetSecretsComplete(context.Background(), *a.provider.VaultURL, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	for secretListIter.NotDone() {
+		secretList := secretListIter.Response().Value
+		for _, secret := range *secretList {
+			ok, secretName := isValidSecret(checkTags, checkName, ref, secret)
+			if !ok {
+				continue
+			}
+
+			secretResp, err := basicClient.GetSecret(context.Background(), *a.provider.VaultURL, secretName, "")
+			if err != nil {
+				return nil, err
+			}
+
+			secretValue := *secretResp.Value
+			secretsMap[secretName] = []byte(secretValue)
+		}
+
+		err = secretListIter.Next()
+		if err != nil {
+			return nil, err
+		}
+	}
+	return secretsMap, nil
 }
 
 // Implements store.Client.GetSecret Interface.
@@ -459,3 +493,36 @@ func getObjType(ref esv1beta1.ExternalSecretDataRemoteRef) (string, string) {
 	}
 	return objectType, secretName
 }
+
+func isValidSecret(checkTags, checkName bool, ref esv1beta1.ExternalSecretFind, secret keyvault.SecretItem) (bool, string) {
+	if secret.ID == nil || !*secret.Attributes.Enabled {
+		return false, ""
+	}
+
+	if checkTags && !okByTags(ref, secret) {
+		return false, ""
+	}
+
+	secretName := path.Base(*secret.ID)
+	if checkName && !okByName(ref, secretName) {
+		return false, ""
+	}
+
+	return true, secretName
+}
+
+func okByName(ref esv1beta1.ExternalSecretFind, secretName string) bool {
+	matches, _ := regexp.MatchString(ref.Name.RegExp, secretName)
+	return matches
+}
+
+func okByTags(ref esv1beta1.ExternalSecretFind, secret keyvault.SecretItem) bool {
+	tagsFound := true
+	for k, v := range ref.Tags {
+		if val, ok := secret.Tags[k]; !ok || *val != v {
+			tagsFound = false
+			break
+		}
+	}
+	return tagsFound
+}

pkg/provider/azure/keyvault/keyvault_test.go

@@ -17,6 +17,7 @@ package keyvault
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"reflect"
 	"testing"
@@ -36,10 +37,12 @@ type secretManagerTestCase struct {
 	secretVersion  string
 	serviceURL     string
 	ref            *esv1beta1.ExternalSecretDataRemoteRef
+	refFind        *esv1beta1.ExternalSecretFind
 	apiErr         error
 	secretOutput   keyvault.SecretBundle
 	keyOutput      keyvault.KeyBundle
 	certOutput     keyvault.CertificateBundle
+	listOutput     keyvault.SecretListResultIterator
 	expectError    string
 	expectedSecret string
 	// for testing secretmap
@@ -53,6 +56,7 @@ func makeValidSecretManagerTestCase() *secretManagerTestCase {
 		secretName:     "MySecret",
 		secretVersion:  "",
 		ref:            makeValidRef(),
+		refFind:        makeValidFind(),
 		secretOutput:   keyvault.SecretBundle{Value: &secretString},
 		serviceURL:     "",
 		apiErr:         nil,
@@ -75,6 +79,7 @@ func makeValidSecretManagerTestCaseCustom(tweaks ...func(smtc *secretManagerTest
 	smtc.mockClient.WithValue(smtc.serviceURL, smtc.secretName, smtc.secretVersion, smtc.secretOutput, smtc.apiErr)
 	smtc.mockClient.WithKey(smtc.serviceURL, smtc.secretName, smtc.secretVersion, smtc.keyOutput, smtc.apiErr)
 	smtc.mockClient.WithCertificate(smtc.serviceURL, smtc.secretName, smtc.secretVersion, smtc.certOutput, smtc.apiErr)
+	smtc.mockClient.WithList(smtc.serviceURL, smtc.listOutput, smtc.apiErr)
 
 	return smtc
 }
@@ -86,6 +91,11 @@ const (
 	jsonSingleTestString = `{"Name": "External", "LastName": "Secret" }`
 	keyName              = "key/keyname"
 	certName             = "cert/certname"
+	secretString         = "changedvalue"
+	unexpectedError      = "[%d] unexpected error: %s, expected: '%s'"
+	unexpectedSecretData = "[%d] unexpected secret data: expected %#v, got %#v"
+	secretName           = "example-1"
+	fakeURL              = "noop"
 )
 
 func newKVJWK(b []byte) *keyvault.JSONWebKey {
@@ -137,7 +147,7 @@ func TestAzureKeyVaultSecretManagerGetSecret(t *testing.T) {
 		}
 		smtc.ref.Property = "Age"
 		smtc.expectError = fmt.Sprintf("property %s does not exist in key %s", smtc.ref.Property, smtc.ref.Key)
-		smtc.apiErr = fmt.Errorf(smtc.expectError)
+		smtc.apiErr = errors.New(smtc.expectError)
 	}
 
 	// // good case: key set
@@ -191,7 +201,7 @@ func TestAzureKeyVaultSecretManagerGetSecret(t *testing.T) {
 	}
 
 	sm := Azure{
-		provider: &esv1beta1.AzureKVProvider{VaultURL: pointer.StringPtr("noop")},
+		provider: &esv1beta1.AzureKVProvider{VaultURL: pointer.StringPtr(fakeURL)},
 	}
 	for k, v := range successCases {
 		sm.baseClient = v.mockClient
@@ -245,7 +255,7 @@ func TestAzureKeyVaultSecretManagerGetSecretMap(t *testing.T) {
 		}
 		smtc.ref.Property = "Age"
 		smtc.expectError = fmt.Sprintf("property %s does not exist in key %s", smtc.ref.Property, smtc.ref.Key)
-		smtc.apiErr = fmt.Errorf(smtc.expectError)
+		smtc.apiErr = errors.New(smtc.expectError)
 	}
 
 	badPubRSAKey := func(smtc *secretManagerTestCase) {
@@ -287,7 +297,7 @@ func TestAzureKeyVaultSecretManagerGetSecretMap(t *testing.T) {
 	}
 
 	sm := Azure{
-		provider: &esv1beta1.AzureKVProvider{VaultURL: pointer.StringPtr("noop")},
+		provider: &esv1beta1.AzureKVProvider{VaultURL: pointer.StringPtr(fakeURL)},
 	}
 	for k, v := range successCases {
 		sm.baseClient = v.mockClient
@@ -301,10 +311,175 @@ func TestAzureKeyVaultSecretManagerGetSecretMap(t *testing.T) {
 	}
 }
 
+func TestAzureKeyVaultSecretManagerGetAllSecrets(t *testing.T) {
+	secretString := secretString
+	secretName := secretName
+	wrongName := "not-valid"
+	environment := "dev"
+	author := "seb"
+	enabled := true
+
+	getNextPage := func(ctx context.Context, list keyvault.SecretListResult) (result keyvault.SecretListResult, err error) {
+		return keyvault.SecretListResult{
+			Value:    nil,
+			NextLink: nil,
+		}, nil
+	}
+
+	setOneSecretByName := func(smtc *secretManagerTestCase) {
+		enabledAtt := keyvault.SecretAttributes{
+			Enabled: &enabled,
+		}
+		secretItem := keyvault.SecretItem{
+			ID:         &secretName,
+			Attributes: &enabledAtt,
+		}
+
+		secretList := make([]keyvault.SecretItem, 0)
+		secretList = append(secretList, secretItem)
+
+		list := keyvault.SecretListResult{
+			Value: &secretList,
+		}
+
+		resultPage := keyvault.NewSecretListResultPage(list, getNextPage)
+		smtc.listOutput = keyvault.NewSecretListResultIterator(resultPage)
+
+		smtc.expectedSecret = secretString
+		smtc.secretOutput = keyvault.SecretBundle{
+			Value: &secretString,
+		}
+
+		smtc.expectedData[secretName] = []byte(secretString)
+	}
+
+	setTwoSecretsByName := func(smtc *secretManagerTestCase) {
+		enabledAtt := keyvault.SecretAttributes{
+			Enabled: &enabled,
+		}
+		secretItemOne := keyvault.SecretItem{
+			ID:         &secretName,
+			Attributes: &enabledAtt,
+		}
+
+		secretItemTwo := keyvault.SecretItem{
+			ID:         &wrongName,
+			Attributes: &enabledAtt,
+		}
+
+		secretList := make([]keyvault.SecretItem, 1)
+		secretList = append(secretList, secretItemOne, secretItemTwo)
+
+		list := keyvault.SecretListResult{
+			Value: &secretList,
+		}
+
+		resultPage := keyvault.NewSecretListResultPage(list, getNextPage)
+		smtc.listOutput = keyvault.NewSecretListResultIterator(resultPage)
+
+		smtc.expectedSecret = secretString
+		smtc.secretOutput = keyvault.SecretBundle{
+			Value: &secretString,
+		}
+
+		smtc.expectedData[secretName] = []byte(secretString)
+	}
+
+	setOneSecretByTag := func(smtc *secretManagerTestCase) {
+		enabledAtt := keyvault.SecretAttributes{
+			Enabled: &enabled,
+		}
+		secretItem := keyvault.SecretItem{
+			ID:         &secretName,
+			Attributes: &enabledAtt,
+			Tags:       map[string]*string{"environment": &environment},
+		}
+
+		secretList := make([]keyvault.SecretItem, 0)
+		secretList = append(secretList, secretItem)
+
+		list := keyvault.SecretListResult{
+			Value: &secretList,
+		}
+
+		resultPage := keyvault.NewSecretListResultPage(list, getNextPage)
+		smtc.listOutput = keyvault.NewSecretListResultIterator(resultPage)
+
+		smtc.expectedSecret = secretString
+		smtc.secretOutput = keyvault.SecretBundle{
+			Value: &secretString,
+		}
+		smtc.refFind.Tags = map[string]string{"environment": environment}
+
+		smtc.expectedData[secretName] = []byte(secretString)
+	}
+
+	setTwoSecretsByTag := func(smtc *secretManagerTestCase) {
+		enabled := true
+		enabledAtt := keyvault.SecretAttributes{
+			Enabled: &enabled,
+		}
+		secretItem := keyvault.SecretItem{
+			ID:         &secretName,
+			Attributes: &enabledAtt,
+			Tags:       map[string]*string{"environment": &environment, "author": &author},
+		}
+
+		secretList := make([]keyvault.SecretItem, 0)
+		secretList = append(secretList, secretItem)
+
+		list := keyvault.SecretListResult{
+			Value: &secretList,
+		}
+
+		resultPage := keyvault.NewSecretListResultPage(list, getNextPage)
+		smtc.listOutput = keyvault.NewSecretListResultIterator(resultPage)
+
+		smtc.expectedSecret = secretString
+		smtc.secretOutput = keyvault.SecretBundle{
+			Value: &secretString,
+		}
+		smtc.refFind.Tags = map[string]string{"environment": environment, "author": author}
+
+		smtc.expectedData[secretName] = []byte(secretString)
+	}
+
+	successCases := []*secretManagerTestCase{
+		makeValidSecretManagerTestCaseCustom(setOneSecretByName),
+		makeValidSecretManagerTestCaseCustom(setTwoSecretsByName),
+		makeValidSecretManagerTestCaseCustom(setOneSecretByTag),
+		makeValidSecretManagerTestCaseCustom(setTwoSecretsByTag),
+	}
+
+	sm := Azure{
+		provider: &esv1beta1.AzureKVProvider{VaultURL: pointer.StringPtr(fakeURL)},
+	}
+	for k, v := range successCases {
+		sm.baseClient = v.mockClient
+		out, err := sm.GetAllSecrets(context.Background(), *v.refFind)
+		if !utils.ErrorContains(err, v.expectError) {
+			t.Errorf(unexpectedError, k, err.Error(), v.expectError)
+		}
+		if err == nil && !reflect.DeepEqual(out, v.expectedData) {
+			t.Errorf(unexpectedSecretData, k, v.expectedData, out)
+		}
+	}
+}
+
 func makeValidRef() *esv1beta1.ExternalSecretDataRemoteRef {
 	return &esv1beta1.ExternalSecretDataRemoteRef{
-		Key:     "test-secret",
-		Version: "default",
+		Key:      "test-secret",
+		Version:  "default",
+		Property: "",
+	}
+}
+
+func makeValidFind() *esv1beta1.ExternalSecretFind {
+	return &esv1beta1.ExternalSecretFind{
+		Name: &esv1beta1.FindName{
+			RegExp: "^example",
+		},
+		Tags: map[string]string{},
 	}
 }