feat: add secret push format to AWS secrets manager (#3189)

docs/provider/aws-secrets-manager.md

@@ -100,6 +100,18 @@ Additional settings can be set at the `SecretStore` level to control the behavio
 {% include 'aws-sm-store-secretsmanager-config.yaml' %}
 ```
 
+#### Additional Metadata for PushSecret
+
+It's possible to configure AWS Secrets Manager to either push secrets in `binary` format or as plain `string`.
+
+To control this behaviour set the following provider metadata:
+
+```yaml
+{% include 'aws-sm-push-secret-with-metadata.yaml' %}
+```
+
+`secretPushFormat` takes two options. `binary` and `string`, where `binary` is the _default_.
+
 ### JSON Secret Values
 
 SecretsManager supports *simple* key/value pairs that are stored as json. If you use the API you can store more complex JSON objects. You can access nested values or arrays using [gjson syntax](https://github.com/tidwall/gjson/blob/master/SYNTAX.md):

docs/snippets/aws-sm-push-secret-with-metadata.yaml

@@ -0,0 +1,21 @@
+apiVersion: external-secrets.io/v1alpha1
+kind: PushSecret
+metadata:
+  name: pushsecret-example # Customisable
+  namespace: teamb # Same of the SecretStores
+spec:
+  deletionPolicy: Delete
+  refreshInterval: 10s # Refresh interval for which push secret will reconcile
+  secretStoreRefs: # A list of secret stores to push secrets to
+    - name: teamb-secret-store
+      kind: SecretStore
+  selector:
+    secret:
+      name: my-secret # Source Kubernetes secret to be pushed
+  data:
+    - match:
+        secretKey: key1 # Source Kubernetes secret key to be pushed
+        remoteRef:
+          remoteKey: teamb-my-first-parameter-3 # Remote reference (where the secret is going to be pushed)
+      metadata:
+        secretPushFormat: string

docs/snippets/aws-sm-store-secretsmanager-config.yaml

@@ -13,4 +13,4 @@ spec:
         # These parameters are only relevant when the deletionPolicy is set to Delete.
         # See: https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_DeleteSecret.html#API_DeleteSecret_RequestSyntax
         forceDeleteWithoutRecovery: true
-        # recoveryWindowInDays: 9 (conflicts with forceDeleteWithoutRecovery)
+        # recoveryWindowInDays: 9 (conflicts with forceDeleteWithoutRecovery)

pkg/provider/aws/secretsmanager/secretsmanager.go

@@ -40,6 +40,14 @@ import (
 	"github.com/external-secrets/external-secrets/pkg/find"
 	"github.com/external-secrets/external-secrets/pkg/metrics"
 	"github.com/external-secrets/external-secrets/pkg/provider/aws/util"
+	"github.com/external-secrets/external-secrets/pkg/utils"
+)
+
+// Declares metadata information for pushing secrets to AWS Secret Store.
+const (
+	SecretPushFormatKey    = "secretPushFormat"
+	SecretPushFormatString = "string"
+	SecretPushFormatBinary = "binary"
 )
 
 // https://github.com/external-secrets/external-secrets/issues/644
@@ -215,15 +223,6 @@ func (sm *SecretsManager) PushSecret(ctx context.Context, secret *corev1.Secret,
 
 	secretName := psd.GetRemoteKey()
 	value := secret.Data[psd.GetSecretKey()]
-	managedBy := managedBy
-	externalSecrets := externalSecrets
-	externalSecretsTag := []*awssm.Tag{
-		{
-			Key:   &managedBy,
-			Value: &externalSecrets,
-		},
-	}
-
 	secretValue := awssm.GetSecretValueInput{
 		SecretId: &secretName,
 	}
@@ -248,44 +247,15 @@ func (sm *SecretsManager) PushSecret(ctx context.Context, secret *corev1.Secret,
 		if ok := errors.As(err, &aerr); !ok {
 			return err
 		}
+
 		if aerr.Code() == awssm.ErrCodeResourceNotFoundException {
-			secretVersion := initialVersion
-			secretRequest := awssm.CreateSecretInput{
-				Name:               &secretName,
-				SecretBinary:       value,
-				Tags:               externalSecretsTag,
-				ClientRequestToken: &secretVersion,
-			}
-			_, err = sm.client.CreateSecretWithContext(ctx, &secretRequest)
-			metrics.ObserveAPICall(constants.ProviderAWSSM, constants.CallAWSSMCreateSecret, err)
-			return err
+			return sm.createSecretWithContext(ctx, secretName, psd, value)
 		}
-		return err
-	}
-	data, err := sm.client.DescribeSecretWithContext(ctx, &secretInput)
-	metrics.ObserveAPICall(constants.ProviderAWSSM, constants.CallAWSSMDescribeSecret, err)
-	if err != nil {
-		return err
-	}
-	if !isManagedByESO(data) {
-		return fmt.Errorf("secret not managed by external-secrets")
-	}
-	if awsSecret != nil && bytes.Equal(awsSecret.SecretBinary, value) {
-		return nil
-	}
 
-	newVersionNumber, err := bumpVersionNumber(awsSecret.VersionId)
-	if err != nil {
 		return err
 	}
-	input := &awssm.PutSecretValueInput{
-		SecretId:           awsSecret.ARN,
-		SecretBinary:       value,
-		ClientRequestToken: newVersionNumber,
-	}
-	_, err = sm.client.PutSecretValueWithContext(ctx, input)
-	metrics.ObserveAPICall(constants.ProviderAWSSM, constants.CallAWSSMPutSecretValue, err)
-	return err
+
+	return sm.putSecretValueWithContext(ctx, secretInput, awsSecret, psd, value)
 }
 
 func padOrTrim(b []byte) []byte {
@@ -562,3 +532,66 @@ func (sm *SecretsManager) Validate() (esv1beta1.ValidationResult, error) {
 func (sm *SecretsManager) Capabilities() esv1beta1.SecretStoreCapabilities {
 	return esv1beta1.SecretStoreReadWrite
 }
+
+func (sm *SecretsManager) createSecretWithContext(ctx context.Context, secretName string, psd esv1beta1.PushSecretData, value []byte) error {
+	secretPushFormat, err := utils.FetchValueFromMetadata(SecretPushFormatKey, psd.GetMetadata(), SecretPushFormatBinary)
+	if err != nil {
+		return fmt.Errorf("failed to parse metadata: %w", err)
+	}
+
+	input := &awssm.CreateSecretInput{
+		Name:         &secretName,
+		SecretBinary: value,
+		Tags: []*awssm.Tag{
+			{
+				Key:   utilpointer.To(managedBy),
+				Value: utilpointer.To(externalSecrets),
+			},
+		},
+		ClientRequestToken: utilpointer.To(initialVersion),
+	}
+	if secretPushFormat == SecretPushFormatString {
+		input.SetSecretBinary(nil).SetSecretString(string(value))
+	}
+
+	_, err = sm.client.CreateSecretWithContext(ctx, input)
+	metrics.ObserveAPICall(constants.ProviderAWSSM, constants.CallAWSSMCreateSecret, err)
+
+	return err
+}
+
+func (sm *SecretsManager) putSecretValueWithContext(ctx context.Context, secretInput awssm.DescribeSecretInput, awsSecret *awssm.GetSecretValueOutput, psd esv1beta1.PushSecretData, value []byte) error {
+	data, err := sm.client.DescribeSecretWithContext(ctx, &secretInput)
+	metrics.ObserveAPICall(constants.ProviderAWSSM, constants.CallAWSSMDescribeSecret, err)
+	if err != nil {
+		return err
+	}
+	if !isManagedByESO(data) {
+		return fmt.Errorf("secret not managed by external-secrets")
+	}
+	if awsSecret != nil && bytes.Equal(awsSecret.SecretBinary, value) {
+		return nil
+	}
+
+	newVersionNumber, err := bumpVersionNumber(awsSecret.VersionId)
+	if err != nil {
+		return err
+	}
+	input := &awssm.PutSecretValueInput{
+		SecretId:           awsSecret.ARN,
+		SecretBinary:       value,
+		ClientRequestToken: newVersionNumber,
+	}
+	secretPushFormat, err := utils.FetchValueFromMetadata(SecretPushFormatKey, psd.GetMetadata(), SecretPushFormatBinary)
+	if err != nil {
+		return fmt.Errorf("failed to parse metadata: %w", err)
+	}
+	if secretPushFormat == SecretPushFormatString {
+		input.SetSecretBinary(nil).SetSecretString(string(value))
+	}
+
+	_, err = sm.client.PutSecretValueWithContext(ctx, input)
+	metrics.ObserveAPICall(constants.ProviderAWSSM, constants.CallAWSSMPutSecretValue, err)
+
+	return err
+}

pkg/provider/aws/secretsmanager/secretsmanager_test.go

@@ -32,6 +32,7 @@ import (
 	"github.com/google/go-cmp/cmp"
 	"github.com/stretchr/testify/assert"
 	corev1 "k8s.io/api/core/v1"
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/utils/ptr"
 
@@ -464,6 +465,9 @@ func TestSetSecret(t *testing.T) {
 	}
 
 	pushSecretDataWithoutProperty := fake.PushSecretData{SecretKey: secretKey, RemoteKey: "fake-key", Property: ""}
+	pushSecretDataWithMetadata := fake.PushSecretData{SecretKey: secretKey, RemoteKey: "fake-key", Property: "", Metadata: &apiextensionsv1.JSON{
+		Raw: []byte(`{"secretPushFormat": "string"}`),
+	}}
 	pushSecretDataWithProperty := fake.PushSecretData{SecretKey: secretKey, RemoteKey: "fake-key", Property: "other-fake-property"}
 
 	type args struct {
@@ -496,6 +500,22 @@ func TestSetSecret(t *testing.T) {
 				err: nil,
 			},
 		},
+		"SetSecretSucceedsWithExistingSecretAndStringFormat": {
+			reason: "a secret can be pushed to aws secrets manager when it already exists",
+			args: args{
+				store: makeValidSecretStore().Spec.Provider.AWS,
+				client: fakesm.Client{
+					GetSecretValueWithContextFn: fakesm.NewGetSecretValueWithContextFn(secretValueOutput, nil),
+					CreateSecretWithContextFn:   fakesm.NewCreateSecretWithContextFn(secretOutput, nil),
+					PutSecretValueWithContextFn: fakesm.NewPutSecretValueWithContextFn(putSecretOutput, nil),
+					DescribeSecretWithContextFn: fakesm.NewDescribeSecretWithContextFn(tagSecretOutput, nil),
+				},
+				pushSecretData: pushSecretDataWithMetadata,
+			},
+			want: want{
+				err: nil,
+			},
+		},
 		"SetSecretSucceedsWithNewSecret": {
 			reason: "a secret can be pushed to aws secrets manager if it doesn't already exist",
 			args: args{

pkg/utils/utils.go

@@ -31,6 +31,8 @@ import (
 	"time"
 	"unicode"
 
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+
 	esv1beta1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1beta1"
 	esmeta "github.com/external-secrets/external-secrets/apis/meta/v1"
 	"github.com/external-secrets/external-secrets/pkg/template/v2"
@@ -41,6 +43,10 @@ const (
 	errExecute = "unable to execute transform template: %s"
 )
 
+var (
+	errKeyNotFound = errors.New("key not found")
+)
+
 // JSONMarshal takes an interface and returns a new escaped and encoded byte slice.
 func JSONMarshal(t interface{}) ([]byte, error) {
 	buffer := &bytes.Buffer{}
@@ -413,3 +419,45 @@ func ConvertToType[T any](obj interface{}) (T, error) {
 
 	return v, nil
 }
+
+// FetchValueFromMetadata fetches a key from a metadata if it exists. It will recursively look in
+// embedded values as well. Must be a unique key, otherwise it will just return the first
+// occurrence.
+func FetchValueFromMetadata[T any](key string, data *apiextensionsv1.JSON, def T) (t T, _ error) {
+	if data == nil {
+		return def, nil
+	}
+
+	m := map[string]any{}
+	if err := json.Unmarshal(data.Raw, &m); err != nil {
+		return t, fmt.Errorf("failed to parse JSON raw data: %w", err)
+	}
+
+	v, err := dig[T](key, m)
+	if err != nil {
+		if errors.Is(err, errKeyNotFound) {
+			return def, nil
+		}
+	}
+
+	return v, nil
+}
+
+func dig[T any](key string, data map[string]any) (t T, _ error) {
+	if v, ok := data[key]; ok {
+		c, k := v.(T)
+		if !k {
+			return t, fmt.Errorf("failed to convert value to the desired type; was: %T", v)
+		}
+
+		return c, nil
+	}
+
+	for _, v := range data {
+		if ty, ok := v.(map[string]any); ok {
+			return dig[T](key, ty)
+		}
+	}
+
+	return t, errKeyNotFound
+}

pkg/utils/utils_test.go

@@ -19,8 +19,9 @@ import (
 	"testing"
 	"time"
 
-	vault "github.com/oracle/oci-go-sdk/v65/vault"
+	"github.com/oracle/oci-go-sdk/v65/vault"
 	v1 "k8s.io/api/core/v1"
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 
 	esv1beta1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1beta1"
 )
@@ -539,3 +540,87 @@ func TestRewrite(t *testing.T) {
 		})
 	}
 }
+
+func TestFetchValueFromMetadata(t *testing.T) {
+	type args struct {
+		key  string
+		data *apiextensionsv1.JSON
+		def  any
+	}
+	type testCase struct {
+		name    string
+		args    args
+		wantT   any
+		wantErr bool
+	}
+	tests := []testCase{
+		{
+			name: "plain dig for an existing key",
+			args: args{
+				key: "key",
+				data: &apiextensionsv1.JSON{
+					Raw: []byte(
+						`{"key": "value"}`,
+					),
+				},
+				def: "def",
+			},
+			wantT:   "value",
+			wantErr: false,
+		},
+		{
+			name: "return default if key not found",
+			args: args{
+				key: "key2",
+				data: &apiextensionsv1.JSON{
+					Raw: []byte(
+						`{"key": "value"}`,
+					),
+				},
+				def: "def",
+			},
+			wantT:   "def",
+			wantErr: false,
+		},
+		{
+			name: "use a different type",
+			args: args{
+				key: "key",
+				data: &apiextensionsv1.JSON{
+					Raw: []byte(
+						`{"key": 123}`,
+					),
+				},
+				def: 1234,
+			},
+			wantT:   float64(123), // unmarshal is always float64
+			wantErr: false,
+		},
+		{
+			name: "digging deeper",
+			args: args{
+				key: "key2",
+				data: &apiextensionsv1.JSON{
+					Raw: []byte(
+						`{"key": {"key2": "value"}}`,
+					),
+				},
+				def: "",
+			},
+			wantT:   "value",
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotT, err := FetchValueFromMetadata(tt.args.key, tt.args.data, tt.args.def)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("FetchValueFromMetadata() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(gotT, tt.wantT) {
+				t.Errorf("FetchValueFromMetadata() gotT = %v, want %v", gotT, tt.wantT)
+			}
+		})
+	}
+}