WIP: GetAllSecrets for vault method

Signed-off-by: Gustavo Carvalho <gustavo.carvalho@container-solutions.com>

pkg/provider/vault/vault.go

@@ -24,6 +24,7 @@ import (
 	"io/ioutil"
 	"net/http"
 	"os"
+	"regexp"
 	"strconv"
 	"strings"
 
@@ -229,9 +230,129 @@ func (c *connector) ValidateStore(store esv1beta1.GenericStore) error {
 }
 
 // Empty GetAllSecrets.
+// GetAllSecrets
+// First load all secrets from secretStore path configuration
+// Then, gets secrets from a matching name or matching custom_metadata
 func (v *client) GetAllSecrets(ctx context.Context, ref esv1beta1.ExternalSecretFind) (map[string][]byte, error) {
-	// TO be implemented
-	return nil, fmt.Errorf("GetAllSecrets not implemented")
+	potentialSecrets, err := v.listSecrets(ctx, "")
+	if err != nil {
+		return nil, err
+	}
+	if ref.Name != nil {
+		return v.findSecretsFromName(ctx, potentialSecrets, *ref.Name)
+	}
+	return v.findSecretsFromTags(ctx, potentialSecrets, ref.Tags)
+}
+
+func (v *client) findSecretsFromTags(ctx context.Context, candidates []string, tags map[string]string) (map[string][]byte, error) {
+	secrets := make(map[string][]byte)
+	for _, name := range candidates {
+		match := true
+		metadata, err := v.readSecretMetadata(ctx, name)
+		if err != nil {
+			return nil, err
+		}
+		for tk, tv := range tags {
+			p, ok := metadata[tk]
+			if !ok || p != tv {
+				match = false
+				break
+			}
+		}
+		if match {
+			secret, err := v.GetSecret(ctx, esv1beta1.ExternalSecretDataRemoteRef{Key: name})
+			if err != nil {
+				return nil, err
+			}
+			newName := strings.ReplaceAll(name, "/", "-")
+			secrets[newName] = secret
+		}
+	}
+	return secrets, nil
+}
+
+func (v *client) findSecretsFromName(ctx context.Context, candidates []string, ref esv1beta1.FindName) (map[string][]byte, error) {
+	secrets := make(map[string][]byte)
+	for _, name := range candidates {
+		ok, err := regexp.MatchString(ref.RegExp, name)
+		if err != nil {
+			return nil, err
+		}
+		if ok {
+			secret, err := v.GetSecret(ctx, esv1beta1.ExternalSecretDataRemoteRef{Key: name})
+			if err != nil {
+				return nil, err
+			}
+			newName := strings.ReplaceAll(name, "/", "-")
+			secrets[newName] = secret
+		}
+	}
+	return secrets, nil
+}
+
+func (v *client) listSecrets(ctx context.Context, path string) ([]string, error) {
+	secrets := make([]string, 0)
+	url := "/v1/" + *v.store.Path + "/metadata/" + path
+	r := v.client.NewRequest(http.MethodGet, url)
+	r.Params.Set("list", "true")
+	resp, err := v.client.RawRequestWithContext(ctx, r)
+	if err != nil {
+		return nil, fmt.Errorf(errReadSecret, err)
+	}
+	secret, parseErr := vault.ParseSecret(resp.Body)
+	if parseErr != nil {
+		return nil, parseErr
+	}
+	t, ok := secret.Data["keys"]
+	if !ok {
+		return nil, nil
+	}
+	paths := t.([]interface{})
+	for _, p := range paths {
+		strPath := p.(string)
+		fullPath := path + strPath // because path always ends with a /
+		if path == "" {
+			fullPath = strPath
+		}
+		// Recurrently find secrets
+		if strings.HasSuffix(p.(string), "/") {
+			var partial = make([]string, 0)
+			partial, err = v.listSecrets(ctx, fullPath)
+			if err != nil {
+				return nil, err
+			}
+			secrets = append(secrets, partial...)
+		} else {
+			secrets = append(secrets, fullPath)
+		}
+	}
+	return secrets, nil
+}
+
+func (v *client) readSecretMetadata(ctx context.Context, path string) (map[string]string, error) {
+	metadata := make(map[string]string)
+	url := "/v1/" + *v.store.Path + "/metadata/" + path
+	r := v.client.NewRequest(http.MethodGet, url)
+	resp, err := v.client.RawRequestWithContext(ctx, r)
+	if err != nil {
+		return nil, fmt.Errorf(errReadSecret, err)
+	}
+	secret, parseErr := vault.ParseSecret(resp.Body)
+	if parseErr != nil {
+		return nil, parseErr
+	}
+	t, ok := secret.Data["custom_metadata"]
+	if !ok {
+		return nil, nil
+	}
+	d, ok := t.(map[string]interface{})
+	if !ok {
+		return metadata, nil
+	}
+	for k, v := range d {
+		metadata[k] = v.(string)
+	}
+	return metadata, nil
 }
 
 // GetSecret supports two types:

pkg/provider/vault/vault_test.go

@@ -983,6 +983,234 @@ func TestGetSecretMap(t *testing.T) {
 	}
 }
 
+func TestGetAllSecrets(t *testing.T) {
+	errBoom := errors.New("boom")
+	secret := map[string]interface{}{
+		"access_key":    "access_key",
+		"access_secret": "access_secret",
+	}
+	secondSecret := map[string]interface{}{
+		"access_key":    "access_key2",
+		"access_secret": "access_secret2",
+		"token":         nil,
+	}
+
+	type args struct {
+		store   *esv1beta1.VaultProvider
+		kube    kclient.Client
+		vClient Client
+		ns      string
+		data    esv1beta1.ExternalSecretDataRemoteRef
+	}
+
+	type want struct {
+		err error
+		val map[string][]byte
+	}
+
+	cases := map[string]struct {
+		reason string
+		args   args
+		want   want
+	}{
+		"ReadSecretKV2": {
+			reason: "Should Map the Secret with DataFrom.Find.Name",
+			args: args{
+				store: makeValidSecretStoreWithVersion(esv1beta1.VaultKVStoreV1).Spec.Provider.Vault,
+				vClient: &fake.VaultClient{
+					MockNewRequest: fake.NewMockNewRequestFn(&vault.Request{}),
+					MockRawRequestWithContext: fake.NewMockRawRequestWithContextFn(
+						newVaultResponseWithData(secret), nil,
+					),
+				},
+			},
+			want: want{
+				err: nil,
+				val: map[string][]byte{
+					"access_key":    []byte("access_key"),
+					"access_secret": []byte("access_secret"),
+				},
+			},
+		},
+		"ReadSecretKV2": {
+			reason: "Should map the secret even if it has a nil value",
+			args: args{
+				store: makeValidSecretStoreWithVersion(esv1beta1.VaultKVStoreV2).Spec.Provider.Vault,
+				vClient: &fake.VaultClient{
+					MockNewRequest: fake.NewMockNewRequestFn(&vault.Request{}),
+					MockRawRequestWithContext: fake.NewMockRawRequestWithContextFn(
+						newVaultResponseWithData(
+							map[string]interface{}{
+								"data": secret,
+							},
+						), nil,
+					),
+				},
+			},
+			want: want{
+				err: nil,
+				val: map[string][]byte{
+					"access_key":    []byte("access_key"),
+					"access_secret": []byte("access_secret"),
+				},
+			},
+		},
+		"ReadSecretWithNilValueKV1": {
+			reason: "Should map the secret even if it has a nil value",
+			args: args{
+				store: makeValidSecretStoreWithVersion(esv1beta1.VaultKVStoreV1).Spec.Provider.Vault,
+				vClient: &fake.VaultClient{
+					MockNewRequest: fake.NewMockNewRequestFn(&vault.Request{}),
+					MockRawRequestWithContext: fake.NewMockRawRequestWithContextFn(
+						newVaultResponseWithData(secretWithNilVal), nil,
+					),
+				},
+			},
+			want: want{
+				err: nil,
+				val: map[string][]byte{
+					"access_key":    []byte("access_key"),
+					"access_secret": []byte("access_secret"),
+					"token":         []byte(nil),
+				},
+			},
+		},
+		"ReadSecretWithNilValueKV2": {
+			reason: "Should map the secret even if it has a nil value",
+			args: args{
+				store: makeValidSecretStoreWithVersion(esv1beta1.VaultKVStoreV2).Spec.Provider.Vault,
+				vClient: &fake.VaultClient{
+					MockNewRequest: fake.NewMockNewRequestFn(&vault.Request{}),
+					MockRawRequestWithContext: fake.NewMockRawRequestWithContextFn(
+						newVaultResponseWithData(
+							map[string]interface{}{
+								"data": secretWithNilVal,
+							},
+						), nil,
+					),
+				},
+			},
+			want: want{
+				err: nil,
+				val: map[string][]byte{
+					"access_key":    []byte("access_key"),
+					"access_secret": []byte("access_secret"),
+					"token":         []byte(nil),
+				},
+			},
+		},
+		"ReadSecretWithTypesKV2": {
+			reason: "Should map the secret even if it has other types",
+			args: args{
+				store: makeValidSecretStoreWithVersion(esv1beta1.VaultKVStoreV2).Spec.Provider.Vault,
+				vClient: &fake.VaultClient{
+					MockNewRequest: fake.NewMockNewRequestFn(&vault.Request{}),
+					MockRawRequestWithContext: fake.NewMockRawRequestWithContextFn(
+						newVaultResponseWithData(
+							map[string]interface{}{
+								"data": secretWithTypes,
+							},
+						), nil,
+					),
+				},
+			},
+			want: want{
+				err: nil,
+				val: map[string][]byte{
+					"access_secret": []byte("access_secret"),
+					"f32":           []byte("2.12"),
+					"f64":           []byte("2.1234534153423423"),
+					"int":           []byte("42"),
+					"bool":          []byte("true"),
+					"bt":            []byte("Zm9vYmFy"), // base64
+				},
+			},
+		},
+		"ReadNestedSecret": {
+			reason: "Should map the secret for deeply nested property",
+			args: args{
+				store: makeValidSecretStoreWithVersion(esv1beta1.VaultKVStoreV2).Spec.Provider.Vault,
+				data: esv1beta1.ExternalSecretDataRemoteRef{
+					Property: "nested",
+				},
+				vClient: &fake.VaultClient{
+					MockNewRequest: fake.NewMockNewRequestFn(&vault.Request{}),
+					MockRawRequestWithContext: fake.NewMockRawRequestWithContextFn(
+						newVaultResponseWithData(
+							map[string]interface{}{
+								"data": secretWithNestedVal,
+							},
+						), nil,
+					),
+				},
+			},
+			want: want{
+				err: nil,
+				val: map[string][]byte{
+					"foo": []byte(`{"mhkeih":"yada yada","oke":"yup"}`),
+				},
+			},
+		},
+		"ReadDeeplyNestedSecret": {
+			reason: "Should map the secret for deeply nested property",
+			args: args{
+				store: makeValidSecretStoreWithVersion(esv1beta1.VaultKVStoreV2).Spec.Provider.Vault,
+				data: esv1beta1.ExternalSecretDataRemoteRef{
+					Property: "nested.foo",
+				},
+				vClient: &fake.VaultClient{
+					MockNewRequest: fake.NewMockNewRequestFn(&vault.Request{}),
+					MockRawRequestWithContext: fake.NewMockRawRequestWithContextFn(
+						newVaultResponseWithData(
+							map[string]interface{}{
+								"data": secretWithNestedVal,
+							},
+						), nil,
+					),
+				},
+			},
+			want: want{
+				err: nil,
+				val: map[string][]byte{
+					"oke":    []byte("yup"),
+					"mhkeih": []byte("yada yada"),
+				},
+			},
+		},
+		"ReadSecretError": {
+			reason: "Should return error if vault client fails to read secret.",
+			args: args{
+				store: makeSecretStore().Spec.Provider.Vault,
+				vClient: &fake.VaultClient{
+					MockNewRequest:            fake.NewMockNewRequestFn(&vault.Request{}),
+					MockRawRequestWithContext: fake.NewMockRawRequestWithContextFn(nil, errBoom),
+				},
+			},
+			want: want{
+				err: fmt.Errorf(errReadSecret, errBoom),
+			},
+		},
+	}
+
+	for name, tc := range cases {
+		t.Run(name, func(t *testing.T) {
+			vStore := &client{
+				kube:      tc.args.kube,
+				client:    tc.args.vClient,
+				store:     tc.args.store,
+				namespace: tc.args.ns,
+			}
+			val, err := vStore.GetSecretMap(context.Background(), tc.args.data)
+			if diff := cmp.Diff(tc.want.err, err, test.EquateErrors()); diff != "" {
+				t.Errorf("\n%s\nvault.GetSecretMap(...): -want error, +got error:\n%s", tc.reason, diff)
+			}
+			if diff := cmp.Diff(tc.want.val, val); diff != "" {
+				t.Errorf("\n%s\nvault.GetSecretMap(...): -want val, +got val:\n%s", tc.reason, diff)
+			}
+		})
+	}
+}
+
 func TestGetSecretPath(t *testing.T) {
 	storeV2 := makeValidSecretStore()
 	storeV2NoPath := storeV2.DeepCopy()