Merge pull request #870 from haf-tech/ibmcloud-sm-kv2

Enhance IBM Secrets Manager support with kv secretType

docs/provider-ibm-secrets-manager.md

@@ -49,7 +49,16 @@ See here for a list of [publicly available endpoints](https://cloud.ibm.com/apid
 ![iam-create-success](./pictures/screenshot_service_url.png)
 
 ### Secret Types
-We support the following secret types of [IBM Secrets Manager](https://cloud.ibm.com/apidocs/secrets-manager): `arbitrary`, `username_password`, `iam_credentials`, `public_cert` and `imported_cert`. To define the type of secret you would like to sync you need to prefix the secret id with the desired type. If the secret type is not specified it is defaulted to `arbitrary`:
+We support the following secret types of [IBM Secrets Manager](https://cloud.ibm.com/apidocs/secrets-manager):
+
+* `arbitrary` 
+* `username_password`
+* `iam_credentials`
+* `imported_cert`
+* `public_cert`
+* `kv` 
+
+To define the type of secret you would like to sync you need to prefix the secret id with the desired type. If the secret type is not specified it is defaulted to `arbitrary`:
 
 ```yaml
 {% include 'ibm-es-types.yaml' %}
@@ -75,6 +84,55 @@ The behavior for the different secret types is as following:
 * `remoteRef` requires a `property` to be set for either `certificate`, `private_key` or `intermediate` to retrieve respective fields from the secrets manager secret and set in specified `secretKey`
 * `dataFrom` retrieves all `certificate`, `private_key` and `intermediate` fields from the secrets manager secret and sets appropriate key:value pairs in the resulting Kubernetes secret
 
+#### kv
+* An optional `property` field can be set to `remoteRef` to select requested key from the KV secret. If not set, the entire secret will be returned
+* `dataFrom` retrieves a string from secrets manager and tries to parse it as JSON object setting the key:values pairs in resulting Kubernetes secret if successful
+
+```json
+{
+  "key1": "val1",
+  "key2": "val2",
+  "key3": {
+    "keyA": "valA",
+    "keyB": "valB"
+  },
+  "special.key": "special-content"
+}
+```
+
+```yaml
+data:
+- secretKey: key3_keyB
+  remoteRef:
+    key: 'kv/aaaaa-bbbb-cccc-dddd-eeeeee'
+    property: 'key3.keyB'
+- secretKey: special_key
+  remoteRef:
+    key: 'kv/aaaaa-bbbb-cccc-dddd-eeeeee'
+    property: 'special.key'
+- secretKey: key_all
+  remoteRef:
+    key: 'kv/aaaaa-bbbb-cccc-dddd-eeeeee'
+
+dataFrom:
+  - key: 'kv/aaaaa-bbbb-cccc-dddd-eeeeee'
+    property: 'key3'
+```
+
+results in
+
+```yaml
+data:
+  # secrets from data
+  key3_keyB: ... #valB
+  special_key: ... #special-content
+  key_all: ... #{"key1":"val1","key2":"val2", ..."special.key":"special-content"}
+  
+  # secrets from dataFrom
+  keyA: ... #valA
+  keyB: ... #valB
+```
+
 
 ### Creating external secret
 

docs/snippets/ibm-es-types.yaml

@@ -21,6 +21,19 @@ spec:
       key: imported_cert/zzzzzzz-zzzz-zzzz-zzzz-zzzzzzzzzzzz
       property: certificate
   - secretKey: bap
-      remoteRef:
-        key: public_cert/zzzzzzz-zzzz-zzzz-zzzz-zzzzzzzzzzzz
-        property: certificate
+    remoteRef:
+      key: public_cert/zzzzzzz-zzzz-zzzz-zzzz-zzzzzzzzzzzz
+      property: certificate
+  - secretKey: kv_without_key
+    remoteRef:
+      key: kv/zzzzzzz-zzzz-zzzz-zzzz-zzzzzzzzzzzz
+  - secretKey: kv_key
+    remoteRef:
+      key: kv/zzzzzzz-zzzz-zzzz-zzzz-zzzzzzzzzzzz
+      property: 'keyid'
+  - secretKey: kv_key_with_path
+    remoteRef:
+      key: kv/zzzzzzz-zzzz-zzzz-zzzz-zzzzzzzzzzzz
+      property: 'key.path'
+  dataFrom:
+  

pkg/provider/ibm/provider.go

@@ -17,17 +17,20 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"strconv"
 	"strings"
 	"time"
 
-	"github.com/IBM/go-sdk-core/v5/core"
+	core "github.com/IBM/go-sdk-core/v5/core"
 	sm "github.com/IBM/secrets-manager-go-sdk/secretsmanagerv1"
+	gjson "github.com/tidwall/gjson"
 	corev1 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/types"
+	types "k8s.io/apimachinery/pkg/types"
+	ctrl "sigs.k8s.io/controller-runtime"
 	kclient "sigs.k8s.io/controller-runtime/pkg/client"
 
 	esv1beta1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1beta1"
-	"github.com/external-secrets/external-secrets/pkg/utils"
+	utils "github.com/external-secrets/external-secrets/pkg/utils"
 )
 
 const (
@@ -60,6 +63,8 @@ type client struct {
 	credentials []byte
 }
 
+var log = ctrl.Log.WithName("provider").WithName("ibm").WithName("secretsmanager")
+
 func (c *client) setAuth(ctx context.Context) error {
 	credentialsSecret := &corev1.Secret{}
 	credentialsSecretName := c.store.Auth.SecretRef.SecretAPIKey.Name
@@ -143,6 +148,10 @@ func (ibm *providerIBM) GetSecret(ctx context.Context, ref esv1beta1.ExternalSec
 
 		return getPublicCertSecret(ibm, &secretName, ref)
 
+	case sm.CreateSecretOptionsSecretTypeKvConst:
+
+		return getKVSecret(ibm, &secretName, ref)
+
 	default:
 		return nil, fmt.Errorf("unknown secret type %s", secretType)
 	}
@@ -237,6 +246,85 @@ func getUsernamePasswordSecret(ibm *providerIBM, secretName *string, ref esv1bet
 	return nil, fmt.Errorf("key %s does not exist in secret %s", ref.Property, ref.Key)
 }
 
+// Returns a secret of type kv and supports json path.
+func getKVSecret(ibm *providerIBM, secretName *string, ref esv1beta1.ExternalSecretDataRemoteRef) ([]byte, error) {
+	secret, err := getSecretByType(ibm, secretName, sm.CreateSecretOptionsSecretTypeKvConst)
+	if err != nil {
+		return nil, err
+	}
+
+	log.Info("fetching secret", "secretName", secretName, "key", ref.Key)
+
+	secretData := secret.SecretData.(map[string]interface{})
+
+	payload, ok := secretData["payload"]
+	if !ok {
+		return nil, fmt.Errorf("no payload returned for secret %s", ref.Key)
+	}
+
+	payloadJSON := payload
+
+	payloadJSONMap, ok := payloadJSON.(map[string]interface{})
+	if ok {
+		var payloadJSONByte []byte
+		payloadJSONByte, err = json.Marshal(payloadJSONMap)
+		if err != nil {
+			return nil, fmt.Errorf("marshaling payload from secret failed. %w", err)
+		}
+		payloadJSON = string(payloadJSONByte)
+	}
+
+	// no property requested, return the entire payload
+	if ref.Property == "" {
+		return []byte(payloadJSON.(string)), nil
+	}
+
+	// returns the requested key
+	// consider that the key contains a ".". this could be one of 2 options
+	// a) "." is part of the key name
+	// b) "." is symbole for JSON path
+	if ref.Property != "" {
+		refProperty := ref.Property
+
+		// a) "." is part the key name
+		// escape "."
+		idx := strings.Index(refProperty, ".")
+		if idx > 0 {
+			refProperty = strings.ReplaceAll(refProperty, ".", "\\.")
+
+			val := gjson.Get(payloadJSON.(string), refProperty)
+			if val.Exists() {
+				return []byte(val.String()), nil
+			}
+		}
+
+		// b) "." is symbole for JSON path
+		// try to get value for this path
+		val := gjson.Get(payloadJSON.(string), ref.Property)
+		if !val.Exists() {
+			return nil, fmt.Errorf("key %s does not exist in secret %s", ref.Property, ref.Key)
+		}
+		return []byte(val.String()), nil
+	}
+
+	return nil, fmt.Errorf("no property provided for secret %s", ref.Key)
+}
+
+func getSecretByType(ibm *providerIBM, secretName *string, secretType string) (*sm.SecretResource, error) {
+	response, _, err := ibm.IBMClient.GetSecret(
+		&sm.GetSecretOptions{
+			SecretType: core.StringPtr(secretType),
+			ID:         secretName,
+		})
+	if err != nil {
+		return nil, err
+	}
+
+	secret := response.Resources[0].(*sm.SecretResource)
+
+	return secret, nil
+}
+
 func (ibm *providerIBM) GetSecretMap(ctx context.Context, ref esv1beta1.ExternalSecretDataRemoteRef) (map[string][]byte, error) {
 	if utils.IsNil(ibm.IBMClient) {
 		return nil, fmt.Errorf(errUninitalizedIBMProvider)
@@ -345,19 +433,61 @@ func (ibm *providerIBM) GetSecretMap(ctx context.Context, ref esv1beta1.External
 
 		return secretMap, nil
 
+	case sm.CreateSecretOptionsSecretTypeKvConst:
+		secret, err := getKVSecret(ibm, &secretName, ref)
+		if err != nil {
+			return nil, err
+		}
+		m := make(map[string]interface{})
+		err = json.Unmarshal(secret, &m)
+		if err != nil {
+			return nil, err
+		}
+
+		secretMap := byteArrayMap(m)
+
+		return secretMap, nil
+
 	default:
 		return nil, fmt.Errorf("unknown secret type %s", secretType)
 	}
 }
 
 func byteArrayMap(secretData map[string]interface{}) map[string][]byte {
+	var err error
 	secretMap := make(map[string][]byte)
 	for k, v := range secretData {
-		secretMap[k] = []byte(v.(string))
+		secretMap[k], err = getTypedKey(v)
+		if err != nil {
+			return nil
+		}
 	}
 	return secretMap
 }
 
+// kudos Vault Provider - convert from various types.
+func getTypedKey(v interface{}) ([]byte, error) {
+	switch t := v.(type) {
+	case string:
+		return []byte(t), nil
+	case map[string]interface{}:
+		return json.Marshal(t)
+	case map[string]string:
+		return json.Marshal(t)
+	case []byte:
+		return t, nil
+		// also covers int and float32 due to json.Marshal
+	case float64:
+		return []byte(strconv.FormatFloat(t, 'f', -1, 64)), nil
+	case bool:
+		return []byte(strconv.FormatBool(t)), nil
+	case nil:
+		return []byte(nil), nil
+	default:
+		return nil, fmt.Errorf("secret not in expected format")
+	}
+}
+
 func (ibm *providerIBM) Close(ctx context.Context) error {
 	return nil
 }

pkg/provider/ibm/provider_test.go

@@ -303,6 +303,112 @@ func TestIBMSecretManagerGetSecret(t *testing.T) {
 		smtc.expectError = "remoteRef.property required for secret type public_cert"
 	}
 
+	secretDataKV := make(map[string]interface{})
+	secretKVPayload := make(map[string]interface{})
+	secretKVPayload["key1"] = "val1"
+	secretDataKV["payload"] = secretKVPayload
+
+	secretDataKVComplex := make(map[string]interface{})
+	secretKVComplex := `{"key1":"val1","key2":"val2","key3":"val3","keyC":{"keyC1":"valC1", "keyC2":"valC2"}, "special.log": "file-content"}`
+
+	secretDataKVComplex["payload"] = secretKVComplex
+
+	secretKV := "kv/test-secret"
+	// bad case: kv type with key which is not in payload
+	badSecretKV := func(smtc *secretManagerTestCase) {
+		resources := []sm.SecretResourceIntf{
+			&sm.SecretResource{
+				SecretType: utilpointer.StringPtr(sm.CreateSecretOptionsSecretTypeKvConst),
+				Name:       utilpointer.StringPtr("testyname"),
+				SecretData: secretDataKV,
+			}}
+
+		smtc.apiInput.SecretType = core.StringPtr(sm.CreateSecretOptionsSecretTypeKvConst)
+		smtc.apiOutput.Resources = resources
+		smtc.ref.Key = secretKV
+		smtc.ref.Property = "other-key"
+		smtc.expectError = "key other-key does not exist in secret kv/test-secret"
+	}
+
+	// good case: kv type with property
+	setSecretKV := func(smtc *secretManagerTestCase) {
+		resources := []sm.SecretResourceIntf{
+			&sm.SecretResource{
+				SecretType: utilpointer.StringPtr(sm.CreateSecretOptionsSecretTypeKvConst),
+				Name:       utilpointer.StringPtr("testyname"),
+				SecretData: secretDataKV,
+			}}
+
+		smtc.apiInput.SecretType = core.StringPtr(sm.CreateSecretOptionsSecretTypeKvConst)
+		smtc.apiOutput.Resources = resources
+		smtc.ref.Key = secretKV
+		smtc.ref.Property = "key1"
+		smtc.expectedSecret = "val1"
+	}
+
+	// good case: kv type with property, returns specific value
+	setSecretKVWithKey := func(smtc *secretManagerTestCase) {
+		resources := []sm.SecretResourceIntf{
+			&sm.SecretResource{
+				SecretType: utilpointer.StringPtr(sm.CreateSecretOptionsSecretTypeKvConst),
+				Name:       utilpointer.StringPtr("testyname"),
+				SecretData: secretDataKVComplex,
+			}}
+
+		smtc.apiInput.SecretType = core.StringPtr(sm.CreateSecretOptionsSecretTypeKvConst)
+		smtc.apiOutput.Resources = resources
+		smtc.ref.Key = secretKV
+		smtc.ref.Property = "key2"
+		smtc.expectedSecret = "val2"
+	}
+
+	// good case: kv type with property and path, returns specific value
+	setSecretKVWithKeyPath := func(smtc *secretManagerTestCase) {
+		resources := []sm.SecretResourceIntf{
+			&sm.SecretResource{
+				SecretType: utilpointer.StringPtr(sm.CreateSecretOptionsSecretTypeKvConst),
+				Name:       utilpointer.StringPtr("testyname"),
+				SecretData: secretDataKVComplex,
+			}}
+
+		smtc.apiInput.SecretType = core.StringPtr(sm.CreateSecretOptionsSecretTypeKvConst)
+		smtc.apiOutput.Resources = resources
+		smtc.ref.Key = secretKV
+		smtc.ref.Property = "keyC.keyC2"
+		smtc.expectedSecret = "valC2"
+	}
+
+	// good case: kv type with property and dot, returns specific value
+	setSecretKVWithKeyDot := func(smtc *secretManagerTestCase) {
+		resources := []sm.SecretResourceIntf{
+			&sm.SecretResource{
+				SecretType: utilpointer.StringPtr(sm.CreateSecretOptionsSecretTypeKvConst),
+				Name:       utilpointer.StringPtr("testyname"),
+				SecretData: secretDataKVComplex,
+			}}
+
+		smtc.apiInput.SecretType = core.StringPtr(sm.CreateSecretOptionsSecretTypeKvConst)
+		smtc.apiOutput.Resources = resources
+		smtc.ref.Key = secretKV
+		smtc.ref.Property = "special.log"
+		smtc.expectedSecret = "file-content"
+	}
+
+	// good case: kv type without property, returns all
+	setSecretKVWithOutKey := func(smtc *secretManagerTestCase) {
+		resources := []sm.SecretResourceIntf{
+			&sm.SecretResource{
+				SecretType: utilpointer.StringPtr(sm.CreateSecretOptionsSecretTypeKvConst),
+				Name:       utilpointer.StringPtr("testyname"),
+				SecretData: secretDataKVComplex,
+			}}
+
+		smtc.apiInput.SecretType = core.StringPtr(sm.CreateSecretOptionsSecretTypeKvConst)
+		smtc.apiOutput.Resources = resources
+		smtc.ref.Key = secretKV
+		smtc.expectedSecret = secretKVComplex
+	}
+
 	successCases := []*secretManagerTestCase{
 		makeValidSecretManagerTestCase(),
 		makeValidSecretManagerTestCaseCustom(setSecretString),
@@ -314,6 +420,12 @@ func TestIBMSecretManagerGetSecret(t *testing.T) {
 		makeValidSecretManagerTestCaseCustom(setSecretIam),
 		makeValidSecretManagerTestCaseCustom(setSecretCert),
 		makeValidSecretManagerTestCaseCustom(badSecretCert),
+		makeValidSecretManagerTestCaseCustom(setSecretKV),
+		makeValidSecretManagerTestCaseCustom(setSecretKVWithKey),
+		makeValidSecretManagerTestCaseCustom(setSecretKVWithKeyPath),
+		makeValidSecretManagerTestCaseCustom(setSecretKVWithKeyDot),
+		makeValidSecretManagerTestCaseCustom(setSecretKVWithOutKey),
+		makeValidSecretManagerTestCaseCustom(badSecretKV),
 		makeValidSecretManagerTestCaseCustom(setSecretPublicCert),
 		makeValidSecretManagerTestCaseCustom(badSecretPublicCert),
 	}
@@ -332,6 +444,7 @@ func TestIBMSecretManagerGetSecret(t *testing.T) {
 }
 
 func TestGetSecretMap(t *testing.T) {
+	secretKeyName := "kv/test-secret"
 	secretUsername := "user1"
 	secretPassword := "P@ssw0rd"
 	secretAPIKey := "01234567890"
@@ -339,6 +452,17 @@ func TestGetSecretMap(t *testing.T) {
 	secretPrivateKey := "private_key_value"
 	secretIntermediate := "intermediate_value"
 
+	secretComplex := map[string]interface{}{
+		"key1": "val1",
+		"key2": "val2",
+		"keyC": map[string]interface{}{
+			"keyC1": map[string]string{
+				"keyA": "valA",
+				"keyB": "valB",
+			},
+		},
+	}
+
 	// good case: default version & deserialization
 	setDeserialization := func(smtc *secretManagerTestCase) {
 		secretData := make(map[string]interface{})
@@ -449,6 +573,84 @@ func TestGetSecretMap(t *testing.T) {
 		smtc.expectedData["intermediate"] = []byte(secretIntermediate)
 	}
 
+	// good case: kv, no property, return entire payload as key:value pairs
+	setSecretKV := func(smtc *secretManagerTestCase) {
+		secretData := make(map[string]interface{})
+		secretData["payload"] = secretComplex
+
+		resources := []sm.SecretResourceIntf{
+			&sm.SecretResource{
+				SecretType: utilpointer.StringPtr(sm.CreateSecretOptionsSecretTypeKvConst),
+				Name:       utilpointer.StringPtr("testyname"),
+				SecretData: secretData,
+			}}
+
+		smtc.apiInput.SecretType = core.StringPtr(sm.CreateSecretOptionsSecretTypeKvConst)
+		smtc.apiOutput.Resources = resources
+		smtc.ref.Key = secretKeyName
+		smtc.expectedData["key1"] = []byte("val1")
+		smtc.expectedData["key2"] = []byte("val2")
+		smtc.expectedData["keyC"] = []byte(`{"keyC1":{"keyA":"valA","keyB":"valB"}}`)
+	}
+
+	// good case: kv, with property
+	setSecretKVWithProperty := func(smtc *secretManagerTestCase) {
+		secretData := make(map[string]interface{})
+		secretData["payload"] = secretComplex
+
+		resources := []sm.SecretResourceIntf{
+			&sm.SecretResource{
+				SecretType: utilpointer.StringPtr(sm.CreateSecretOptionsSecretTypeKvConst),
+				Name:       utilpointer.StringPtr("testyname"),
+				SecretData: secretData,
+			}}
+
+		smtc.apiInput.SecretType = core.StringPtr(sm.CreateSecretOptionsSecretTypeKvConst)
+		smtc.ref.Property = "keyC"
+		smtc.apiOutput.Resources = resources
+		smtc.ref.Key = secretKeyName
+		smtc.expectedData["keyC1"] = []byte(`{"keyA":"valA","keyB":"valB"}`)
+	}
+
+	// good case: kv, with property and path
+	setSecretKVWithPathAndProperty := func(smtc *secretManagerTestCase) {
+		secretData := make(map[string]interface{})
+		secretData["payload"] = secretComplex
+
+		resources := []sm.SecretResourceIntf{
+			&sm.SecretResource{
+				SecretType: utilpointer.StringPtr(sm.CreateSecretOptionsSecretTypeKvConst),
+				Name:       utilpointer.StringPtr("testyname"),
+				SecretData: secretData,
+			}}
+
+		smtc.apiInput.SecretType = core.StringPtr(sm.CreateSecretOptionsSecretTypeKvConst)
+		smtc.ref.Property = "keyC.keyC1"
+		smtc.apiOutput.Resources = resources
+		smtc.ref.Key = secretKeyName
+		smtc.expectedData["keyA"] = []byte("valA")
+		smtc.expectedData["keyB"] = []byte("valB")
+	}
+
+	// bad case: kv, with property and path
+	badSecretKVWithUnknownProperty := func(smtc *secretManagerTestCase) {
+		secretData := make(map[string]interface{})
+		secretData["payload"] = secretComplex
+
+		resources := []sm.SecretResourceIntf{
+			&sm.SecretResource{
+				SecretType: utilpointer.StringPtr(sm.CreateSecretOptionsSecretTypeKvConst),
+				Name:       utilpointer.StringPtr("testyname"),
+				SecretData: secretData,
+			}}
+
+		smtc.apiInput.SecretType = core.StringPtr(sm.CreateSecretOptionsSecretTypeKvConst)
+		smtc.ref.Property = "unknown.property"
+		smtc.apiOutput.Resources = resources
+		smtc.ref.Key = secretKeyName
+		smtc.expectError = "key unknown.property does not exist in secret kv/test-secret"
+	}
+
 	successCases := []*secretManagerTestCase{
 		makeValidSecretManagerTestCaseCustom(setDeserialization),
 		makeValidSecretManagerTestCaseCustom(setInvalidJSON),
@@ -457,6 +659,10 @@ func TestGetSecretMap(t *testing.T) {
 		makeValidSecretManagerTestCaseCustom(setSecretUserPass),
 		makeValidSecretManagerTestCaseCustom(setSecretIam),
 		makeValidSecretManagerTestCaseCustom(setSecretCert),
+		makeValidSecretManagerTestCaseCustom(setSecretKV),
+		makeValidSecretManagerTestCaseCustom(setSecretKVWithProperty),
+		makeValidSecretManagerTestCaseCustom(setSecretKVWithPathAndProperty),
+		makeValidSecretManagerTestCaseCustom(badSecretKVWithUnknownProperty),
 		makeValidSecretManagerTestCaseCustom(setSecretPublicCert),
 	}