feat: implement aws/GetAllSecrets

Signed-off-by: Moritz Johner <beller.moritz@googlemail.com>

e2e/framework/framework.go

@@ -27,6 +27,7 @@ import (
 	crclient "sigs.k8s.io/controller-runtime/pkg/client"
 
 	esv1alpha1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1alpha1"
+	esv1beta1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1beta1"
 	"github.com/external-secrets/external-secrets/e2e/framework/addon"
 	"github.com/external-secrets/external-secrets/e2e/framework/log"
 	"github.com/external-secrets/external-secrets/e2e/framework/util"
@@ -35,6 +36,7 @@ import (
 func init() {
 	_ = kscheme.AddToScheme(util.Scheme)
 	_ = esv1alpha1.AddToScheme(util.Scheme)
+	_ = esv1beta1.AddToScheme(util.Scheme)
 }
 
 type Framework struct {

e2e/framework/testcase.go

@@ -22,17 +22,21 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	esv1alpha1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1alpha1"
+	esv1beta1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1beta1"
 	"github.com/external-secrets/external-secrets/e2e/framework/log"
 )
 
-var TargetSecretName = "target-secret"
+var DefaultTargetSecretName = "target-secret"
+
+const DefaultExternalSecretName = "e2e-es"
 
 // TestCase contains the test infra to run a table driven test.
 type TestCase struct {
-	Framework      *Framework
-	ExternalSecret *esv1alpha1.ExternalSecret
-	Secrets        map[string]string
-	ExpectedSecret *v1.Secret
+	Framework          *Framework
+	ExternalSecret     *esv1alpha1.ExternalSecret
+	BetaExternalSecret *esv1beta1.ExternalSecret
+	Secrets            map[string]string
+	ExpectedSecret     *v1.Secret
 }
 
 // SecretStoreProvider is a interface that must be implemented
@@ -64,16 +68,26 @@ func TableFunc(f *Framework, prov SecretStoreProvider) func(...func(*TestCase))
 		}
 
 		// create external secret
-		err = tc.Framework.CRClient.Create(context.Background(), tc.ExternalSecret)
-		Expect(err).ToNot(HaveOccurred())
+		if tc.ExternalSecret != nil {
+			err = tc.Framework.CRClient.Create(context.Background(), tc.ExternalSecret)
+			Expect(err).ToNot(HaveOccurred())
+		}
 
-		// in case target name is empty
-		if tc.ExternalSecret.Spec.Target.Name == "" {
-			TargetSecretName = tc.ExternalSecret.ObjectMeta.Name
+		if tc.BetaExternalSecret != nil {
+			err = tc.Framework.CRClient.Create(context.Background(), tc.BetaExternalSecret)
+			Expect(err).ToNot(HaveOccurred())
+		}
+
+		secName := DefaultExternalSecretName
+		if tc.ExternalSecret != nil && tc.ExternalSecret.Spec.Target.Name != "" {
+			secName = tc.ExternalSecret.Spec.Target.Name
+		}
+		if tc.BetaExternalSecret != nil && tc.BetaExternalSecret.Spec.Target.Name != "" {
+			secName = tc.BetaExternalSecret.Spec.Target.Name
 		}
 
 		// wait for Kind=Secret to have the expected data
-		secret, err := tc.Framework.WaitForSecretValue(tc.Framework.Namespace.Name, TargetSecretName, tc.ExpectedSecret)
+		secret, err := tc.Framework.WaitForSecretValue(tc.Framework.Namespace.Name, secName, tc.ExpectedSecret)
 		if err != nil {
 			log.Logf("Did not match. Expected: %+v, Got: %+v", tc.ExpectedSecret, secret)
 		}
@@ -87,7 +101,7 @@ func makeDefaultTestCase(f *Framework) *TestCase {
 		Framework: f,
 		ExternalSecret: &esv1alpha1.ExternalSecret{
 			ObjectMeta: metav1.ObjectMeta{
-				Name:      "e2e-es",
+				Name:      DefaultExternalSecretName,
 				Namespace: f.Namespace.Name,
 			},
 			Spec: esv1alpha1.ExternalSecretSpec{
@@ -95,9 +109,26 @@ func makeDefaultTestCase(f *Framework) *TestCase {
 					Name: f.Namespace.Name,
 				},
 				Target: esv1alpha1.ExternalSecretTarget{
-					Name: TargetSecretName,
+					Name: DefaultTargetSecretName,
 				},
 			},
 		},
 	}
 }
+
+func DefaultBetaExternalSecret(f *Framework) *esv1beta1.ExternalSecret {
+	return &esv1beta1.ExternalSecret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      DefaultExternalSecretName,
+			Namespace: f.Namespace.Name,
+		},
+		Spec: esv1beta1.ExternalSecretSpec{
+			SecretStoreRef: esv1beta1.SecretStoreRef{
+				Name: f.Namespace.Name,
+			},
+			Target: esv1beta1.ExternalSecretTarget{
+				Name: DefaultTargetSecretName,
+			},
+		},
+	}
+}

e2e/suite/aws/parameterstore/parameterstore.go

@@ -41,5 +41,6 @@ var _ = Describe("[aws] ", Label("aws", "parameterstore"), func() {
 		Entry(common.SSHKeySyncDataProperty(f)),
 		Entry(common.SyncWithoutTargetName(f)),
 		Entry(common.JSONDataWithoutTargetName(f)),
+		Entry(common.FindByName(f)),
 	)
 })

e2e/suite/aws/secretsmanager/secretsmanager.go

@@ -41,5 +41,6 @@ var _ = Describe("[aws] ", Label("aws", "secretsmanager"), func() {
 		Entry(common.SSHKeySyncDataProperty(f)),
 		Entry(common.SyncWithoutTargetName(f)),
 		Entry(common.JSONDataWithoutTargetName(f)),
+		Entry(common.FindByName(f)),
 	)
 })

e2e/suite/common/find_by_name.go

@@ -0,0 +1,57 @@
+/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+limitations under the License.
+*/
+package common
+
+import (
+	"fmt"
+
+	v1 "k8s.io/api/core/v1"
+
+	esapi "github.com/external-secrets/external-secrets/apis/externalsecrets/v1beta1"
+	"github.com/external-secrets/external-secrets/e2e/framework"
+)
+
+// This case creates multiple secrets with simple key/value pairs and syncs them using multiple .Spec.Data blocks.
+// Not supported by: vault.
+func FindByName(f *framework.Framework) (string, func(*framework.TestCase)) {
+	return "[common] should find secrets by name using .DataFrom[]", func(tc *framework.TestCase) {
+		secretKeyOne := fmt.Sprintf("e2e-find-name-%s-%s", f.Namespace.Name, "one")
+		secretKeyTwo := fmt.Sprintf("e2e-find-name-%s-%s", f.Namespace.Name, "two")
+		secretKeyThree := fmt.Sprintf("e2e-find-name-%s-%s", f.Namespace.Name, "three")
+		secretValue := "something"
+		tc.Secrets = map[string]string{
+			secretKeyOne:   secretValue,
+			secretKeyTwo:   secretValue,
+			secretKeyThree: secretValue,
+		}
+		tc.ExpectedSecret = &v1.Secret{
+			Type: v1.SecretTypeOpaque,
+			Data: map[string][]byte{
+				fmt.Sprintf("e2e-find-name-%s-one", f.Namespace.Name):   []byte(secretValue),
+				fmt.Sprintf("e2e-find-name-%s-two", f.Namespace.Name):   []byte(secretValue),
+				fmt.Sprintf("e2e-find-name-%s-three", f.Namespace.Name): []byte(secretValue),
+			},
+		}
+		tc.ExternalSecret = nil
+		tc.BetaExternalSecret = framework.DefaultBetaExternalSecret(f)
+		tc.BetaExternalSecret.Spec.DataFrom = []esapi.ExternalSecretDataFromRemoteRef{
+			{
+				Find: &esapi.ExternalSecretFind{
+					Name: &esapi.FindName{
+						RegExp: fmt.Sprintf("e2e-find-name-%s.+", f.Namespace.Name),
+					},
+				},
+			},
+		}
+	}
+}

pkg/find/find.go

@@ -0,0 +1,40 @@
+/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package find
+
+import (
+	"fmt"
+	"regexp"
+
+	esv1beta1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1beta1"
+)
+
+type Matcher struct {
+	re *regexp.Regexp
+}
+
+func New(findName esv1beta1.FindName) (*Matcher, error) {
+	cmp, err := regexp.Compile(findName.RegExp)
+	if err != nil {
+		return nil, fmt.Errorf("could not compile find.name.regexp [%s]: %w", findName.RegExp, err)
+	}
+	return &Matcher{
+		re: cmp,
+	}, nil
+}
+
+func (m *Matcher) MatchName(name string) bool {
+	return m.re.MatchString(name)
+}

pkg/provider/aws/parameterstore/fake/fake.go

@@ -29,6 +29,10 @@ func (sm *Client) GetParameter(in *ssm.GetParameterInput) (*ssm.GetParameterOutp
 	return sm.valFn(in)
 }
 
+func (sm *Client) DescribeParameters(*ssm.DescribeParametersInput) (*ssm.DescribeParametersOutput, error) {
+	return nil, nil
+}
+
 func (sm *Client) WithValue(in *ssm.GetParameterInput, val *ssm.GetParameterOutput, err error) {
 	sm.valFn = func(paramIn *ssm.GetParameterInput) (*ssm.GetParameterOutput, error) {
 		if !cmp.Equal(paramIn, in) {

pkg/provider/aws/parameterstore/parameterstore.go

@@ -16,7 +16,9 @@ package parameterstore
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
+	"strings"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/session"
@@ -25,7 +27,9 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 
 	esv1beta1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1beta1"
+	"github.com/external-secrets/external-secrets/pkg/find"
 	"github.com/external-secrets/external-secrets/pkg/provider/aws/util"
+	utilpointer "k8s.io/utils/pointer"
 )
 
 // ParameterStore is a provider for AWS ParameterStore.
@@ -38,8 +42,14 @@ type ParameterStore struct {
 // see: https://docs.aws.amazon.com/sdk-for-go/api/service/ssm/ssmiface/
 type PMInterface interface {
 	GetParameter(*ssm.GetParameterInput) (*ssm.GetParameterOutput, error)
+	DescribeParameters(*ssm.DescribeParametersInput) (*ssm.DescribeParametersOutput, error)
 }
 
+const (
+	errUnexpectedFindOperator = "unexpected find operator"
+	errDuplicateKey           = "duplicate key mapping at %s"
+)
+
 var log = ctrl.Log.WithName("provider").WithName("aws").WithName("parameterstore")
 
 // New constructs a ParameterStore Provider that is specific to a store.
@@ -52,8 +62,113 @@ func New(sess *session.Session) (*ParameterStore, error) {
 
 // Empty GetAllSecrets.
 func (pm *ParameterStore) GetAllSecrets(ctx context.Context, ref esv1beta1.ExternalSecretFind) (map[string][]byte, error) {
-	// TO be implemented
-	return nil, fmt.Errorf("GetAllSecrets not implemented")
+	if ref.Name != nil {
+		return pm.findByName(ref)
+	}
+	if len(ref.Tags) > 0 {
+		return pm.findByTags(ref)
+	}
+	return nil, errors.New(errUnexpectedFindOperator)
+}
+
+func (pm *ParameterStore) findByName(ref esv1beta1.ExternalSecretFind) (map[string][]byte, error) {
+	matcher, err := find.New(*ref.Name)
+	if err != nil {
+		return nil, err
+	}
+	data := make(map[string][]byte)
+	var nextToken *string
+	for {
+		it, err := pm.client.DescribeParameters(&ssm.DescribeParametersInput{
+			NextToken: nextToken,
+		})
+		if err != nil {
+			return nil, err
+		}
+		log.Info("aws pm findByName", "parameters", len(it.Parameters))
+		for _, param := range it.Parameters {
+			if !matcher.MatchName(*param.Name) {
+				continue
+			}
+			err = pm.fetchAndSet(data, *param.Name)
+			if err != nil {
+				return nil, err
+			}
+		}
+		nextToken = it.NextToken
+		if nextToken == nil {
+			break
+		}
+	}
+	return data, nil
+}
+
+func (pm *ParameterStore) findByTags(ref esv1beta1.ExternalSecretFind) (map[string][]byte, error) {
+	filters := make([]*ssm.ParameterStringFilter, len(ref.Tags))
+	for k, v := range ref.Tags {
+		filters = append(filters, &ssm.ParameterStringFilter{
+			Key:    utilpointer.StringPtr(fmt.Sprintf("tag:%s", k)),
+			Values: []*string{utilpointer.StringPtr(v)},
+			Option: utilpointer.StringPtr("Equals"),
+		})
+	}
+
+	data := make(map[string][]byte)
+	var nextToken *string
+	for {
+		it, err := pm.client.DescribeParameters(&ssm.DescribeParametersInput{
+			ParameterFilters: filters,
+			NextToken:        nextToken,
+		})
+		if err != nil {
+			return nil, err
+		}
+		log.V(1).Info("aws pm findByTags found", "parameters", len(it.Parameters))
+		for _, param := range it.Parameters {
+			err = pm.fetchAndSet(data, *param.Name)
+			if err != nil {
+				return nil, err
+			}
+		}
+		nextToken = it.NextToken
+		if nextToken == nil {
+			break
+		}
+	}
+	return data, nil
+}
+
+func (pm *ParameterStore) fetchAndSet(data map[string][]byte, name string) error {
+	out, err := pm.client.GetParameter(&ssm.GetParameterInput{
+		Name:           utilpointer.StringPtr(name),
+		WithDecryption: aws.Bool(true),
+	})
+	if err != nil {
+		return util.SanitizeErr(err)
+	}
+
+	// Note: multiple key names CAN collide: `/dev/my_db` and `/dev/my/db` would result
+	//       in the same key `dev_my_db` being mapped
+	key := mapSecretKey(name)
+	if _, exists := data[key]; exists {
+		return fmt.Errorf(errDuplicateKey, key)
+	}
+
+	// secret keys must consist of alphanumeric characters or `-`, `_` or `.`
+	data[mapSecretKey(name)] = []byte(*out.Parameter.Value)
+	return nil
+}
+
+// mapSecretKey maps the parameter key to a secret key. Example: `/foo/bar/baz` -> `foo_bar_baz`.
+// The secret keys must consist of alphanumeric characters or `-`, `_` or `.`
+// AWS Parameter Names use the same character set BUT in addition to that the slash character `/`
+// is used to delineate hierarchies in parameter names
+// see aws docs: https://docs.aws.amazon.com/systems-manager/latest/userguide/sysman-paramstore-su-create.html
+// Example Parameter:  /dev/myapp/password
+// Example Secret Key: dev_myapp_password.
+func mapSecretKey(str string) string {
+	str = strings.TrimLeft(str, "/")
+	return strings.ReplaceAll(str, "/", "_")
 }
 
 // GetSecret returns a single secret from the provider.

pkg/provider/aws/secretsmanager/secretsmanager.go

@@ -17,7 +17,11 @@ package secretsmanager
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
+	"regexp"
+
+	utilpointer "k8s.io/utils/pointer"
 
 	"github.com/aws/aws-sdk-go/aws/session"
 	awssm "github.com/aws/aws-sdk-go/service/secretsmanager"
@@ -25,6 +29,7 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 
 	esv1beta1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1beta1"
+	"github.com/external-secrets/external-secrets/pkg/find"
 	"github.com/external-secrets/external-secrets/pkg/provider/aws/util"
 )
 
@@ -39,8 +44,14 @@ type SecretsManager struct {
 // see: https://docs.aws.amazon.com/sdk-for-go/api/service/secretsmanager/secretsmanageriface/
 type SMInterface interface {
 	GetSecretValue(*awssm.GetSecretValueInput) (*awssm.GetSecretValueOutput, error)
+	ListSecrets(*awssm.ListSecretsInput) (*awssm.ListSecretsOutput, error)
 }
 
+const (
+	errUnexpectedFindOperator = "unexpected find operator"
+	errDuplicateKey           = "duplicate key mapping at %s"
+)
+
 var log = ctrl.Log.WithName("provider").WithName("aws").WithName("secretsmanager")
 
 // New creates a new SecretsManager client.
@@ -78,8 +89,126 @@ func (sm *SecretsManager) fetch(_ context.Context, ref esv1beta1.ExternalSecretD
 
 // Empty GetAllSecrets.
 func (sm *SecretsManager) GetAllSecrets(ctx context.Context, ref esv1beta1.ExternalSecretFind) (map[string][]byte, error) {
-	// TO be implemented
-	return nil, fmt.Errorf("GetAllSecrets not implemented")
+	if ref.Name != nil {
+		return sm.findByName(ctx, ref)
+	}
+	if len(ref.Tags) > 0 {
+		return sm.findByTags(ctx, ref)
+	}
+	return nil, errors.New(errUnexpectedFindOperator)
+}
+
+func (sm *SecretsManager) findByName(ctx context.Context, ref esv1beta1.ExternalSecretFind) (map[string][]byte, error) {
+	matcher, err := find.New(*ref.Name)
+	if err != nil {
+		return nil, err
+	}
+	data := make(map[string][]byte)
+	var nextToken *string
+
+	for {
+		ctrl.Log.Info("aws sm findByName", "nextToken", nextToken)
+		it, err := sm.client.ListSecrets(&awssm.ListSecretsInput{
+			NextToken: nextToken,
+		})
+		if err != nil {
+			return nil, err
+		}
+		ctrl.Log.Info("aws sm findByName found", "secrets", len(it.SecretList))
+		for _, secret := range it.SecretList {
+			if !matcher.MatchName(*secret.Name) {
+				continue
+			}
+			ctrl.Log.Info("aws sm findByName matches", "name", *secret.Name)
+			err = sm.fetchAndSet(ctx, data, *secret.Name)
+			if err != nil {
+				return nil, err
+			}
+		}
+		nextToken = it.NextToken
+		if nextToken == nil {
+			break
+		}
+	}
+	return data, nil
+}
+
+func (sm *SecretsManager) findByTags(ctx context.Context, ref esv1beta1.ExternalSecretFind) (map[string][]byte, error) {
+	filters := make([]*awssm.Filter, len(ref.Tags)*2)
+	for k, v := range ref.Tags {
+		filters = append(filters, &awssm.Filter{
+			Key: utilpointer.StringPtr(awssm.FilterNameStringTypeTagKey),
+			Values: []*string{
+				utilpointer.StringPtr(k),
+			},
+		}, &awssm.Filter{
+			Key: utilpointer.StringPtr(awssm.FilterNameStringTypeTagValue),
+			Values: []*string{
+				utilpointer.StringPtr(v),
+			},
+		})
+	}
+
+	data := make(map[string][]byte)
+	var nextToken *string
+	for {
+		ctrl.Log.Info("aws sm findByTag", "nextToken", nextToken)
+		it, err := sm.client.ListSecrets(&awssm.ListSecretsInput{
+			Filters:   filters,
+			NextToken: nextToken,
+		})
+		if err != nil {
+			return nil, err
+		}
+		ctrl.Log.Info("aws sm findByTag found", "secrets", len(it.SecretList))
+		for _, secret := range it.SecretList {
+			err = sm.fetchAndSet(ctx, data, *secret.Name)
+			if err != nil {
+				return nil, err
+			}
+		}
+		nextToken = it.NextToken
+		if nextToken == nil {
+			break
+		}
+	}
+	return data, nil
+}
+
+func (sm *SecretsManager) fetchAndSet(ctx context.Context, data map[string][]byte, name string) error {
+	ctrl.Log.Info("aws sm fetchAndSet fetch", "name", name)
+	sec, err := sm.fetch(ctx, esv1beta1.ExternalSecretDataRemoteRef{
+		Key: name,
+		// Right now we only support AWSCURRENT as version
+		// There is no intent to support specific versions
+		// or specific aliases like AWSPREVIOUS or AWSPENDING
+		Version: "AWSCURRENT",
+	})
+	if err != nil {
+		return err
+	}
+
+	// Note: multiple key names can collide:
+	//       foo/bar and foo$bar would result in the same key
+	//       foo_bar being mapped.
+	key := mapSecretKey(name)
+	if _, exist := data[key]; exist {
+		return fmt.Errorf(errDuplicateKey, key)
+	}
+
+	if sec.SecretString != nil {
+		data[key] = []byte(*sec.SecretString)
+	}
+	if sec.SecretBinary != nil {
+		data[key] = sec.SecretBinary
+	}
+	return nil
+}
+
+var keyChars = regexp.MustCompile(`[^A-Za-z0-9_\-.]+`)
+
+func mapSecretKey(key string) string {
+	return keyChars.ReplaceAllString(key, "_")
 }
 
 // GetSecret returns a single secret from the provider.

pkg/provider/aws/secretsmanager/secretsmanager_test.go

@@ -22,10 +22,9 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	awssm "github.com/aws/aws-sdk-go/service/secretsmanager"
-	"github.com/google/go-cmp/cmp"
-
 	esv1beta1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1beta1"
 	fakesm "github.com/external-secrets/external-secrets/pkg/provider/aws/secretsmanager/fake"
+	"github.com/google/go-cmp/cmp"
 )
 
 type secretsManagerTestCase struct {
@@ -298,3 +297,43 @@ func ErrorContains(out error, want string) bool {
 	}
 	return strings.Contains(out.Error(), want)
 }
+
+func Test_mapSecretKey(t *testing.T) {
+	type args struct {
+		key string
+	}
+	tests := []struct {
+		name string
+		args args
+		want string
+	}{
+		{
+			name: "replace special chars",
+			args: args{
+				key: "/foo/bar/baz",
+			},
+			want: "_foo_bar_baz",
+		},
+		{
+			name: "keep alphanumeric as-is",
+			args: args{
+				key: "my_special_value",
+			},
+			want: "my_special_value",
+		},
+		{
+			name: "keep alphanumeric as-is",
+			args: args{
+				key: `my_special_value_$1_$2`,
+			},
+			want: "my_special_value__1__2",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := mapSecretKey(tt.args.key); got != tt.want {
+				t.Errorf("mapSecretKey() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}