infisical: support secrets within paths for `data` references (#4305)

* infisical: allow for retrieving secrets within paths

Signed-off-by: Gergely Brautigam <182850+Skarlso@users.noreply.github.com>

On-behalf-of: Joey Pereira

* fixed my faulty index path logic and test asserts

Signed-off-by: Gergely Brautigam <182850+Skarlso@users.noreply.github.com>

---------

Signed-off-by: Gergely Brautigam <182850+Skarlso@users.noreply.github.com>
Co-authored-by: Gergely Brautigam <182850+Skarlso@users.noreply.github.com>

docs/provider/infisical.md

@@ -3,6 +3,7 @@
 Sync secrets from [Infisical](https://www.infisical.com) to your Kubernetes cluster using External Secrets Operator.
 
 ## Authentication
+
 In order for the operator to fetch secrets from Infisical, it needs to first authenticate with Infisical.
 
 To authenticate, you can use [Universal Auth](https://infisical.com/docs/documentation/platform/identities/universal-auth) from [Machine identities](https://infisical.com/docs/documentation/platform/identities/machine-identities).
@@ -42,7 +43,22 @@ You will then need to create a generic `SecretStore`. An sample `SecretStore` ha
 !!! Note
     For `ClusterSecretStore`, be sure to set `namespace` in `universalAuthCredentials.clientId` and `universalAuthCredentials.clientSecret`.
 
-## Fetch Individual Secret(s)
+## Fetching secrets
+
+For the following examples, it assumes we have a secret structure in an Infisical project with the following structure:
+
+```plaintext
+/API_KEY
+/DB_PASSWORD
+/JSON_BLOB
+/my-app
+  /SERVICE_PASSWORD
+  /ADMIN_PASSWORD
+```
+
+Where `JSON_BLOB` is a JSON string like `{"key": "value"}`.
+
+### Fetch Individual Secret(s)
 
 To sync one or more secrets individually, use the following YAML:
 
@@ -50,7 +66,7 @@ To sync one or more secrets individually, use the following YAML:
 {% include 'infisical-fetch-secret.yaml' %}
 ```
 
-## Fetch All Secrets
+### Fetch All Secrets
 
 To sync all secrets from an Infisical , use the following YAML:
 
@@ -58,11 +74,10 @@ To sync all secrets from an Infisical , use the following YAML:
 {% include 'infisical-fetch-all-secrets.yaml' %}
 ```
 
-## Filter By Prefix/Name
+### Filtering secrets
 
 To filter secrets by `path` (path prefix) and `name` (regular expression).
 
 ``` yaml
 {% include 'infisical-filtered-secrets.yaml' %}
 ```
-

docs/snippets/infisical-fetch-all-secrets.yaml

@@ -10,6 +10,8 @@ spec:
   target:
     name: auth-api
 
+  # dataFrom will fetch all secrets that are inside the `secretsPath`. When `recursive` is
+  # enabled, it will also fetch all secrets recursively in sub-directories.
   dataFrom:
     - find:
         name:

docs/snippets/infisical-fetch-secret.yaml

@@ -11,6 +11,18 @@ spec:
     name: auth-api
 
   data:
+    # When referencing a secret within the `secretsPath`, the `key` can just be a secret
+    # name.
     - secretKey: API_KEY
       remoteRef:
         key: API_KEY
+    # Properties can be extracted from secrets that are JSON strings.
+    - secretKey: JSON_KEY
+      remoteRef:
+        key: JSON_BLOB
+        property: key
+    # When referencing secrets in paths (other than `secretsPath`), the `key` must be an
+    # absolute path to the secret.
+    - secretKey: PASSWORD
+      remoteRef:
+        key: /my-app/SERVICE_PASSWORD

docs/snippets/infisical-generic-secret-store.yaml

@@ -5,6 +5,10 @@ metadata:
 spec:
   provider:
     infisical:
+      # Optional (default: https://app.infisical.com).
+      #
+      # Override this if you are using a different Infisical instance.
+      hostAPI: https://app.infisical.com
       auth:
         universalAuthCredentials:
           clientId:
@@ -15,15 +19,22 @@ spec:
             key: clientSecret
             namespace: default
             name: universal-auth-credentials
-      # Details to pull secrets from
       secretsScope:
         projectSlug: first-project-fujo
-        environmentSlug: dev # "dev", "staging", "prod", etc..
-        # optional
-        secretsPath: / # Root is "/"
-        # optional
-        recursive: true # Default is false
+        # "dev", "staging", "prod", etc.
+        environmentSlug: dev
+        # Optional (default: `/`).
+        #
+        # Secrets will only be retrieved from this path for `data` and `dataFrom` rules. When a
+        # `data` `remoteRef` uses a path (e.g. `/foo/bar`), that reference will use an absolute
+        # reference and disregard this default.
+        #
+        # If you need to prevent access to secrets outside of this path, rely on instead setting
+        # Access Controls in Infisical.
+        secretsPath: /
+        # Optional (default: false).
+        #
+        # When recursive is enabled, secrets retrieved using `dataFrom` patterns will fetch all secrets recursive.
+        recursive: false
         # optional
         expandSecretReferences: false # Default is true
-      # optional
-      hostAPI: https://app.infisical.com

pkg/provider/infisical/client.go

@@ -43,14 +43,42 @@ func getPropertyValue(jsonData, propertyName, keyName string) ([]byte, error) {
 	return []byte(result.Str), nil
 }
 
-// if GetSecret returns an error with type NoSecretError.
-// then the secret entry will be deleted depending on the deletionPolicy.
+// getSecretAddress returns the path and key from the given key.
+//
+// Users can configure a root path, and when a SecretKey is provided with a slash we assume that it is
+// within a path appended to the root path.
+//
+// If the key is not addressing a path at all (i.e. has no `/`), simply return the original
+// path and key.
+func getSecretAddress(defaultPath, key string) (string, string, error) {
+	if !strings.Contains(key, "/") {
+		return defaultPath, key, nil
+	}
+
+	// Check if `key` starts with a `/`, and throw and error if it does not.
+	if !strings.HasPrefix(key, "/") {
+		return "", "", fmt.Errorf("a secret key referencing a folder must start with a '/' as it is an absolute path, key: %s", key)
+	}
+
+	// Otherwise, take the prefix from `key` and use that as the path. We intentionally discard
+	// `defaultPath`.
+	lastIndex := strings.LastIndex(key, "/")
+	return key[:lastIndex], key[lastIndex+1:], nil
+}
+
+// GetSecret if this returns an error with type NoSecretError then the secret entry will be deleted depending on the
+// deletionPolicy.
 func (p *Provider) GetSecret(ctx context.Context, ref esv1.ExternalSecretDataRemoteRef) ([]byte, error) {
+	path, key, err := getSecretAddress(p.apiScope.SecretPath, ref.Key)
+	if err != nil {
+		return nil, err
+	}
+
 	secret, err := p.apiClient.GetSecretByKeyV3(api.GetSecretByKeyV3Request{
 		EnvironmentSlug:        p.apiScope.EnvironmentSlug,
 		ProjectSlug:            p.apiScope.ProjectSlug,
-		SecretKey:              ref.Key,
-		SecretPath:             p.apiScope.SecretPath,
+		SecretKey:              key,
+		SecretPath:             path,
 		ExpandSecretReferences: p.apiScope.ExpandSecretReferences,
 	})
 

pkg/provider/infisical/client_test.go

@@ -0,0 +1,62 @@
+/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package infisical
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetSecretAddress(t *testing.T) {
+	t.Run("when the key is not addressing a path and uses the default path", func(t *testing.T) {
+		path, key, err := getSecretAddress("/", "foo")
+		assert.NoError(t, err)
+		assert.Equal(t, "/", path)
+		assert.Equal(t, "foo", key)
+
+		path, key, err = getSecretAddress("/foo", "bar")
+		assert.NoError(t, err)
+		assert.Equal(t, "/foo", path)
+		assert.Equal(t, "bar", key)
+	})
+
+	t.Run("when the key is addressing a path", func(t *testing.T) {
+		path, key, err := getSecretAddress("/", "/foo/bar")
+		assert.NoError(t, err)
+		assert.Equal(t, path, "/foo")
+		assert.Equal(t, key, "bar")
+	})
+
+	t.Run("when the key is addressing a path and ignores the default path", func(t *testing.T) {
+		path, key, err := getSecretAddress("/foo", "/bar/baz")
+		assert.NoError(t, err)
+		assert.Equal(t, "/bar", path)
+		assert.Equal(t, "baz", key)
+	})
+
+	t.Run("works with a nested directory", func(t *testing.T) {
+		path, key, err := getSecretAddress("/", "/foo/bar/baz")
+		assert.NoError(t, err)
+		assert.Equal(t, "/foo/bar", path)
+		assert.Equal(t, "baz", key, "baz")
+	})
+
+	t.Run("fails when the key is a folder but does not begin with a slash", func(t *testing.T) {
+		_, _, err := getSecretAddress("/", "bar/baz")
+		assert.Error(t, err)
+		assert.Equal(t, err.Error(), "a secret key referencing a folder must start with a '/' as it is an absolute path, key: bar/baz")
+	})
+}

pkg/provider/infisical/provider.go

@@ -21,22 +21,16 @@ import (
 	"net/http"
 	"time"
 
-	ctrl "sigs.k8s.io/controller-runtime"
 	kclient "sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
 
 	esv1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1"
 	esmeta "github.com/external-secrets/external-secrets/apis/meta/v1"
 	"github.com/external-secrets/external-secrets/pkg/provider/infisical/api"
-	"github.com/external-secrets/external-secrets/pkg/provider/infisical/constants"
 	"github.com/external-secrets/external-secrets/pkg/utils"
 	"github.com/external-secrets/external-secrets/pkg/utils/resolvers"
 )
 
-var (
-	Logger = ctrl.Log.WithName("provider").WithName(constants.ProviderName)
-)
-
 type Provider struct {
 	apiClient *api.InfisicalClient
 	apiScope  *InfisicalClientScope
@@ -96,13 +90,18 @@ func (p *Provider) NewClient(ctx context.Context, store esv1.GenericStore, kube
 			return nil, fmt.Errorf("failed to authenticate via universal auth %w", err)
 		}
 
+		secretPath := infisicalSpec.SecretsScope.SecretsPath
+		if secretPath == "" {
+			secretPath = "/"
+		}
+
 		return &Provider{
 			apiClient: apiClient,
 			apiScope: &InfisicalClientScope{
 				EnvironmentSlug:        infisicalSpec.SecretsScope.EnvironmentSlug,
 				ProjectSlug:            infisicalSpec.SecretsScope.ProjectSlug,
 				Recursive:              infisicalSpec.SecretsScope.Recursive,
-				SecretPath:             infisicalSpec.SecretsScope.SecretsPath,
+				SecretPath:             secretPath,
 				ExpandSecretReferences: infisicalSpec.SecretsScope.ExpandSecretReferences,
 			},
 		}, nil

pkg/provider/infisical/provider_test.go

@@ -7,7 +7,7 @@ You may obtain a copy of the License at
 
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or impliec.
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */
@@ -16,7 +16,11 @@ package infisical
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -39,6 +43,7 @@ type TestCases struct {
 	Name           string
 	MockStatusCode int
 	MockResponse   any
+	Key            string
 	Property       string
 	Error          error
 	Output         any
@@ -57,6 +62,7 @@ func TestGetSecret(t *testing.T) {
 					SecretValue: "bar",
 				},
 			},
+			Key:    key,
 			Output: []byte("bar"),
 		},
 		{
@@ -68,6 +74,7 @@ func TestGetSecret(t *testing.T) {
 					SecretValue: `{"bar": "value"}`,
 				},
 			},
+			Key:      key,
 			Property: "bar",
 			Output:   []byte("value"),
 		},
@@ -79,9 +86,22 @@ func TestGetSecret(t *testing.T) {
 				Err:        "Not Found",
 				Message:    "Secret not found",
 			},
-			Error:  esv1.NoSecretError{},
+			Key:    "key",
+			Error:  esv1.NoSecretErr,
 			Output: "",
 		},
+		{
+			Name:           "Key_with_slash",
+			MockStatusCode: 200,
+			MockResponse: api.GetSecretByKeyV3Response{
+				Secret: api.SecretsV3{
+					SecretKey:   "bar",
+					SecretValue: "value",
+				},
+			},
+			Key:    "/foo/bar",
+			Output: []byte("value"),
+		},
 	}
 
 	for _, tc := range testCases {
@@ -94,7 +114,7 @@ func TestGetSecret(t *testing.T) {
 			}
 
 			output, err := p.GetSecret(context.Background(), esv1.ExternalSecretDataRemoteRef{
-				Key:      key,
+				Key:      tc.Key,
 				Property: tc.Property,
 			})
 
@@ -108,6 +128,54 @@ func TestGetSecret(t *testing.T) {
 	}
 }
 
+// TestGetSecretWithPath verifies that request is translated from a key
+// `/foo/bar` to a secret `bar` with `secretPath` of `/foo`.
+func TestGetSecretWithPath(t *testing.T) {
+	requestedKey := "/foo/bar"
+	expectedSecretPath := "/foo"
+	expectedSecretKey := "bar"
+
+	// Prepare the mock response.
+	data := api.GetSecretByKeyV3Response{
+		Secret: api.SecretsV3{
+			SecretKey:   expectedSecretKey,
+			SecretValue: `value`,
+		},
+	}
+	body, err := json.Marshal(data)
+	if err != nil {
+		panic(err)
+	}
+
+	// Prepare the mock server, which asserts the request translation is correct.
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		assert.Equal(t, fmt.Sprintf("/api/v3/secrets/raw/%s", expectedSecretKey), r.URL.Path)
+		assert.Equal(t, expectedSecretPath, r.URL.Query().Get("secretPath"))
+		w.WriteHeader(200)
+		_, err := w.Write(body)
+		if err != nil {
+			panic(err)
+		}
+	}))
+	defer server.Close()
+
+	client, err := api.NewAPIClient(server.URL, server.Client())
+	require.NoError(t, err)
+	p := &Provider{
+		apiClient: client,
+		apiScope:  &apiScope,
+	}
+
+	// Retrieve the secret.
+	output, err := p.GetSecret(context.Background(), esv1.ExternalSecretDataRemoteRef{
+		Key:      requestedKey,
+		Property: "",
+	})
+	// And, we should get back the expected secret value.
+	require.NoError(t, err)
+	assert.Equal(t, []byte("value"), output)
+}
+
 func TestGetSecretMap(t *testing.T) {
 	key := "foo"
 	testCases := []TestCases{
@@ -120,6 +188,7 @@ func TestGetSecretMap(t *testing.T) {
 					SecretValue: `{"bar": "value"}`,
 				},
 			},
+			Key: key,
 			Output: map[string][]byte{
 				"bar": []byte("value"),
 			},
@@ -128,6 +197,7 @@ func TestGetSecretMap(t *testing.T) {
 			Name:           "Get_invalid_map",
 			MockStatusCode: 200,
 			MockResponse:   []byte(``),
+			Key:            key,
 			Error:          errors.New("unable to unmarshal secret foo"),
 		},
 	}
@@ -141,7 +211,8 @@ func TestGetSecretMap(t *testing.T) {
 				apiScope:  &apiScope,
 			}
 			output, err := p.GetSecretMap(context.Background(), esv1.ExternalSecretDataRemoteRef{
-				Key: key,
+				Key:      tc.Key,
+				Property: tc.Property,
 			})
 			if tc.Error == nil {
 				assert.NoError(t, err)
@@ -153,7 +224,7 @@ func TestGetSecretMap(t *testing.T) {
 	}
 }
 
-func makeSecretStore(projectSlug, environment, secretPath string, fn ...storeModifier) *esv1.SecretStore {
+func makeSecretStore(projectSlug, environment, secretsPath string, fn ...storeModifier) *esv1.SecretStore {
 	store := &esv1.SecretStore{
 		Spec: esv1.SecretStoreSpec{
 			Provider: &esv1.SecretStoreProvider{
@@ -162,7 +233,7 @@ func makeSecretStore(projectSlug, environment, secretPath string, fn ...storeMod
 						UniversalAuthCredentials: &esv1.UniversalAuthCredentials{},
 					},
 					SecretsScope: esv1.MachineIdentityScopeInWorkspace{
-						SecretsPath:     secretPath,
+						SecretsPath:     secretsPath,
 						EnvironmentSlug: environment,
 						ProjectSlug:     projectSlug,
 					},
@@ -199,6 +270,7 @@ func withClientSecret(name, key string, namespace *string) storeModifier {
 }
 
 type ValidateStoreTestCase struct {
+	name        string
 	store       *esv1.SecretStore
 	assertError func(t *testing.T, err error)
 }
@@ -211,31 +283,37 @@ func TestValidateStore(t *testing.T) {
 
 	testCases := []ValidateStoreTestCase{
 		{
+			name:  "Missing projectSlug",
 			store: makeSecretStore("", "", ""),
 			assertError: func(t *testing.T, err error) {
 				require.ErrorAs(t, err, &authScopeMissingErr)
 			},
 		},
 		{
+			name:  "Missing clientID",
 			store: makeSecretStore(apiScope.ProjectSlug, apiScope.EnvironmentSlug, apiScope.SecretPath, withClientID(authType, randomID, nil)),
 			assertError: func(t *testing.T, err error) {
 				require.ErrorAs(t, err, &authCredMissingErr)
 			},
 		},
 		{
+			name:  "Missing clientSecret",
 			store: makeSecretStore(apiScope.ProjectSlug, apiScope.EnvironmentSlug, apiScope.SecretPath, withClientSecret(authType, randomID, nil)),
 			assertError: func(t *testing.T, err error) {
 				require.ErrorAs(t, err, &authCredMissingErr)
 			},
 		},
 		{
+			name:        "Success",
 			store:       makeSecretStore(apiScope.ProjectSlug, apiScope.EnvironmentSlug, apiScope.SecretPath, withClientID(authType, randomID, nil), withClientSecret(authType, randomID, nil)),
 			assertError: func(t *testing.T, err error) { require.NoError(t, err) },
 		},
 	}
 	p := Provider{}
 	for _, tc := range testCases {
-		_, err := p.ValidateStore(tc.store)
-		tc.assertError(t, err)
+		t.Run(tc.name, func(t *testing.T) {
+			_, err := p.ValidateStore(tc.store)
+			tc.assertError(t, err)
+		})
 	}
 }