feat: allow accessing original Vault response from VaultDynamicSecret (#4358)

Signed-off-by: Michal Baumgartner <michal.baumgartner@ataccama.com>
Co-authored-by: Gergely Brautigam <182850+Skarlso@users.noreply.github.com>

apis/generators/v1alpha1/types_vault.go

@@ -38,6 +38,7 @@ type VaultDynamicSecretSpec struct {
 	// When using e.g. /auth/token/create the "data" section is empty but
 	// the "auth" section contains the generated token.
 	// Please refer to the vault docs regarding the result data structure.
+	// Additionally, accessing the raw response is possibly by using "Raw" result type.
 	// +kubebuilder:default=Data
 	ResultType VaultDynamicSecretResultType `json:"resultType,omitempty"`
 
@@ -57,12 +58,13 @@ type VaultDynamicSecretSpec struct {
 	AllowEmptyResponse bool `json:"allowEmptyResponse,omitempty"`
 }
 
-// +kubebuilder:validation:Enum=Data;Auth
+// +kubebuilder:validation:Enum=Data;Auth;Raw
 type VaultDynamicSecretResultType string
 
 const (
 	VaultDynamicSecretResultTypeData VaultDynamicSecretResultType = "Data"
 	VaultDynamicSecretResultTypeAuth VaultDynamicSecretResultType = "Auth"
+	VaultDynamicSecretResultTypeRaw  VaultDynamicSecretResultType = "Raw"
 )
 
 // +kubebuilder:object:root=true

config/crds/bases/generators.external-secrets.io_clustergenerators.yaml

@@ -1666,9 +1666,11 @@ spec:
                           When using e.g. /auth/token/create the "data" section is empty but
                           the "auth" section contains the generated token.
                           Please refer to the vault docs regarding the result data structure.
+                          Additionally, accessing the raw response is possibly by using "Raw" result type.
                         enum:
                         - Data
                         - Auth
+                        - Raw
                         type: string
                       retrySettings:
                         description: Used to configure http retries if failed

config/crds/bases/generators.external-secrets.io_vaultdynamicsecrets.yaml

@@ -862,9 +862,11 @@ spec:
                   When using e.g. /auth/token/create the "data" section is empty but
                   the "auth" section contains the generated token.
                   Please refer to the vault docs regarding the result data structure.
+                  Additionally, accessing the raw response is possibly by using "Raw" result type.
                 enum:
                 - Data
                 - Auth
+                - Raw
                 type: string
               retrySettings:
                 description: Used to configure http retries if failed

deploy/crds/bundle.yaml

@@ -15447,9 +15447,11 @@ spec:
                             When using e.g. /auth/token/create the "data" section is empty but
                             the "auth" section contains the generated token.
                             Please refer to the vault docs regarding the result data structure.
+                            Additionally, accessing the raw response is possibly by using "Raw" result type.
                           enum:
                             - Data
                             - Auth
+                            - Raw
                           type: string
                         retrySettings:
                           description: Used to configure http retries if failed
@@ -17680,9 +17682,11 @@ spec:
                     When using e.g. /auth/token/create the "data" section is empty but
                     the "auth" section contains the generated token.
                     Please refer to the vault docs regarding the result data structure.
+                    Additionally, accessing the raw response is possibly by using "Raw" result type.
                   enum:
                     - Data
                     - Auth
+                    - Raw
                   type: string
                 retrySettings:
                   description: Used to configure http retries if failed

docs/api/generator/vault.md

@@ -10,7 +10,9 @@ All secrets engines should be supported by providing matching `path`, `method`
 and `parameters` values to the Generator spec (see example below).
 
 Exact output keys and values depend on the Vault secret engine used; nested values
-are stored into the resulting Secret in JSON format.
+are stored into the resulting Secret in JSON format. The generator exposes `data`
+section of the response from Vault API by default. To adjust the behaviour, use
+`resultType` key.
 
 ## Example manifest
 

docs/snippets/generator-vault.yaml

@@ -10,6 +10,7 @@ spec:
   parameters:
     common_name: "localhost"
     ip_sans: "127.0.0.1,127.0.0.11"
+  resultType: "Data"  # "Auth" and "Raw" are also available
   provider:
     server: "http://vault.default.svc.cluster.local:8200"
     auth:

pkg/generator/vault/vault.go

@@ -135,6 +135,15 @@ func (g *Generator) prepareResponse(res *genv1alpha1.VaultDynamicSecret, result
 		if err != nil {
 			return nil, nil, err
 		}
+	} else if res.Spec.ResultType == genv1alpha1.VaultDynamicSecretResultTypeRaw {
+		rawJSON, err := json.Marshal(result)
+		if err != nil {
+			return nil, nil, err
+		}
+		err = json.Unmarshal(rawJSON, &data)
+		if err != nil {
+			return nil, nil, err
+		}
 	} else {
 		data = result.Data
 	}

pkg/generator/vault/vault_test.go

@@ -20,6 +20,7 @@ import (
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
+	vaultapi "github.com/hashicorp/vault/api"
 	corev1 "k8s.io/api/core/v1"
 	apiextensions "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -30,17 +31,20 @@ import (
 	utilfake "github.com/external-secrets/external-secrets/pkg/provider/util/fake"
 	provider "github.com/external-secrets/external-secrets/pkg/provider/vault"
 	"github.com/external-secrets/external-secrets/pkg/provider/vault/fake"
+	"github.com/external-secrets/external-secrets/pkg/provider/vault/util"
 )
 
 type args struct {
-	jsonSpec *apiextensions.JSON
-	kube     kclient.Client
-	corev1   typedcorev1.CoreV1Interface
+	jsonSpec      *apiextensions.JSON
+	kube          kclient.Client
+	corev1        typedcorev1.CoreV1Interface
+	vaultClientFn func(config *vaultapi.Config) (util.Client, error)
 }
 
 type want struct {
-	val map[string][]byte
-	err error
+	val        map[string][]byte
+	partialVal map[string][]byte
+	err        error
 }
 
 type testCase struct {
@@ -235,20 +239,173 @@ spec:
 				val: nil,
 			},
 		},
+		"DataResultType": {
+			reason: "Allow accessing the data section of the response from Vault API.",
+			args: args{
+				corev1: utilfake.NewCreateTokenMock().WithToken("ok"),
+				jsonSpec: &apiextensions.JSON{
+					Raw: []byte(`apiVersion: generators.external-secrets.io/v1alpha1
+kind: VaultDynamicSecret
+spec:
+  provider:
+    auth:
+      kubernetes:
+        role: test
+        serviceAccountRef:
+          name: "testing"
+  path: "github/token/example"`),
+				},
+				kube: clientfake.NewClientBuilder().WithObjects(&corev1.ServiceAccount{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "testing",
+						Namespace: "testing",
+					},
+					Secrets: []corev1.ObjectReference{
+						{
+							Name: "test",
+						},
+					},
+				}).Build(),
+				vaultClientFn: fake.ModifiableClientWithLoginMock(
+					func(cl *fake.VaultClient) {
+						cl.MockLogical.ReadWithDataWithContextFn = func(ctx context.Context, path string, data map[string][]string) (*vaultapi.Secret, error) {
+							return &vaultapi.Secret{
+								Data: map[string]interface{}{
+									"key": "value",
+								},
+							}, nil
+						}
+					},
+				),
+			},
+			want: want{
+				err: nil,
+				val: map[string][]byte{"key": []byte("value")},
+			},
+		},
+		"AuthResultType": {
+			reason: "Allow accessing auth section of the response from Vault API.",
+			args: args{
+				corev1: utilfake.NewCreateTokenMock().WithToken("ok"),
+				jsonSpec: &apiextensions.JSON{
+					Raw: []byte(`apiVersion: generators.external-secrets.io/v1alpha1
+kind: VaultDynamicSecret
+spec:
+  provider:
+    auth:
+      kubernetes:
+        role: test
+        serviceAccountRef:
+          name: "testing"
+  path: "github/token/example"
+  resultType: "Auth"`),
+				},
+				kube: clientfake.NewClientBuilder().WithObjects(&corev1.ServiceAccount{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "testing",
+						Namespace: "testing",
+					},
+					Secrets: []corev1.ObjectReference{
+						{
+							Name: "test",
+						},
+					},
+				}).Build(),
+				vaultClientFn: fake.ModifiableClientWithLoginMock(
+					func(cl *fake.VaultClient) {
+						cl.MockLogical.ReadWithDataWithContextFn = func(ctx context.Context, path string, data map[string][]string) (*vaultapi.Secret, error) {
+							return &vaultapi.Secret{
+								Auth: &vaultapi.SecretAuth{
+									EntityID: "123",
+								},
+							}, nil
+						}
+					},
+				),
+			},
+			want: want{
+				err:        nil,
+				partialVal: map[string][]byte{"entity_id": []byte("123")},
+			},
+		},
+		"RawResultType": {
+			reason: "Allow accessing auth section of the response from Vault API.",
+			args: args{
+				corev1: utilfake.NewCreateTokenMock().WithToken("ok"),
+				jsonSpec: &apiextensions.JSON{
+					Raw: []byte(`apiVersion: generators.external-secrets.io/v1alpha1
+kind: VaultDynamicSecret
+spec:
+  provider:
+    auth:
+      kubernetes:
+        role: test
+        serviceAccountRef:
+          name: "testing"
+  path: "github/token/example"
+  resultType: "Raw"`),
+				},
+				kube: clientfake.NewClientBuilder().WithObjects(&corev1.ServiceAccount{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "testing",
+						Namespace: "testing",
+					},
+					Secrets: []corev1.ObjectReference{
+						{
+							Name: "test",
+						},
+					},
+				}).Build(),
+				vaultClientFn: fake.ModifiableClientWithLoginMock(
+					func(cl *fake.VaultClient) {
+						cl.MockLogical.ReadWithDataWithContextFn = func(ctx context.Context, path string, data map[string][]string) (*vaultapi.Secret, error) {
+							return &vaultapi.Secret{
+								LeaseID: "123",
+								Data: map[string]interface{}{
+									"key": "value",
+								},
+							}, nil
+						}
+					},
+				),
+			},
+			want: want{
+				err: nil,
+				partialVal: map[string][]byte{
+					"lease_id": []byte("123"),
+					"data":     []byte(`{"key":"value"}`),
+				},
+			},
+		},
 	}
 
 	for name, tc := range cases {
 		t.Run(name, func(t *testing.T) {
-			c := &provider.Provider{NewVaultClient: fake.ClientWithLoginMock}
+			newClientFn := fake.ClientWithLoginMock
+			if tc.args.vaultClientFn != nil {
+				newClientFn = tc.args.vaultClientFn
+			}
+			c := &provider.Provider{NewVaultClient: newClientFn}
 			gen := &Generator{}
 			val, _, err := gen.generate(context.Background(), c, tc.args.jsonSpec, tc.args.kube, tc.args.corev1, "testing")
-			if err != nil || tc.want.err != nil {
-				if diff := cmp.Diff(tc.want.err.Error(), err.Error()); diff != "" {
-					t.Errorf("\n%s\nvault.GetSecret(...): -want error, +got error:\n%s", tc.reason, diff)
+			if tc.want.err != nil {
+				if err != nil {
+					if diff := cmp.Diff(tc.want.err.Error(), err.Error()); diff != "" {
+						t.Errorf("\n%s\nvault.GetSecret(...): -want error, +got error:\n%s", tc.reason, diff)
+					}
+				} else {
+					t.Errorf("\n%s\nvault.GetSecret(...): -want error, +got val:\n%s", tc.reason, val)
+				}
+			} else if tc.want.partialVal != nil {
+				for k, v := range tc.want.partialVal {
+					if diff := cmp.Diff(v, val[k]); diff != "" {
+						t.Errorf("\n%s\nvault.GetSecret(...) -> %s: -want partial, +got partial:\n%s", k, tc.reason, diff)
+					}
+				}
+			} else {
+				if diff := cmp.Diff(tc.want.val, val); diff != "" {
+					t.Errorf("\n%s\nvault.GetSecret(...): -want val, +got val:\n%s", tc.reason, diff)
 				}
-			}
-			if diff := cmp.Diff(tc.want.val, val); diff != "" {
-				t.Errorf("\n%s\nvault.GetSecret(...): -want val, +got val:\n%s", tc.reason, diff)
 			}
 		})
 	}

pkg/provider/vault/fake/vault.go

@@ -278,8 +278,18 @@ func (c *VaultClient) AddHeader(key, value string) {
 	c.MockAddHeader(key, value)
 }
 
-func ClientWithLoginMock(_ *vault.Config) (util.Client, error) {
-	cl := VaultClient{
+func ClientWithLoginMock(config *vault.Config) (util.Client, error) {
+	return clientWithLoginMockOptions(config)
+}
+
+func ModifiableClientWithLoginMock(opts ...func(cl *VaultClient)) func(config *vault.Config) (util.Client, error) {
+	return func(config *vault.Config) (util.Client, error) {
+		return clientWithLoginMockOptions(config, opts...)
+	}
+}
+
+func clientWithLoginMockOptions(_ *vault.Config, opts ...func(cl *VaultClient)) (util.Client, error) {
+	cl := &VaultClient{
 		MockAuthToken: NewAuthTokenFn(),
 		MockSetToken:  NewSetTokenFn(),
 		MockToken:     NewTokenFn(""),
@@ -287,6 +297,10 @@ func ClientWithLoginMock(_ *vault.Config) (util.Client, error) {
 		MockLogical:   NewVaultLogical(),
 	}
 
+	for _, opt := range opts {
+		opt(cl)
+	}
+
 	return &util.VaultClient{
 		SetTokenFunc:     cl.SetToken,
 		TokenFunc:        cl.Token,