feat(ngrok): use API filtering for vault and secret lookups (#6376)

Co-authored-by: Gergely Bräutigam <gergely.brautigam@sap.com>
Signed-off-by: Jonathan Stacks <jonstacks@users.noreply.github.com>

go.mod

@@ -330,7 +330,7 @@ require (
 	github.com/muesli/cancelreader v0.2.2 // indirect
 	github.com/muesli/termenv v0.16.0 // indirect
 	github.com/nebius/gosdk v0.0.0-20260204094009-511fd4d4f7a1 // indirect
-	github.com/ngrok/ngrok-api-go/v7 v7.6.0 // indirect
+	github.com/ngrok/ngrok-api-go/v9 v9.0.0 // indirect
 	github.com/oapi-codegen/runtime v1.1.2 // indirect
 	github.com/openbao/openbao/api/v2 v2.5.1-0.20260603121413-a08669ff09ec // indirect
 	github.com/opentracing/basictracer-go v1.1.0 // indirect

go.sum

@@ -918,8 +918,8 @@ github.com/nats-io/nkeys v0.3.0/go.mod h1:gvUNGjVcM2IPr5rCsRsC6Wb3Hr2CQAm08dsxtV
 github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
 github.com/nebius/gosdk v0.0.0-20260204094009-511fd4d4f7a1 h1:Lt5HZDEeSNJJbppzFEtlOYMGlGIOVi6YoC9YJMHv60A=
 github.com/nebius/gosdk v0.0.0-20260204094009-511fd4d4f7a1/go.mod h1:8r4EhhGJ+RMUfdiVVpZ8pEb0b+O7hLG8JXDAgGyu89o=
-github.com/ngrok/ngrok-api-go/v7 v7.6.0 h1:DW9FqEgSN6+Dgl25O8ha1LS49CqX2c9vO0Z53CN8Vqs=
-github.com/ngrok/ngrok-api-go/v7 v7.6.0/go.mod h1:Si/pYAJmbCuo4Fb3xz0MF6N5ubRvPdUixETBwhFvBf0=
+github.com/ngrok/ngrok-api-go/v9 v9.0.0 h1:dBTOl+PK8oUN/WAores6V/CTlyPEqD8yH70ToDSuAWg=
+github.com/ngrok/ngrok-api-go/v9 v9.0.0/go.mod h1:Y6SGu3WzfTRGJiHVSk2REXp3cmi0cEsAvJDsb30dBuU=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
 github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=

providers/v1/ngrok/client.go

@@ -24,10 +24,11 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"strconv"
 	"sync"
 	"time"
 
-	"github.com/ngrok/ngrok-api-go/v7"
+	"github.com/ngrok/ngrok-api-go/v9"
 	corev1 "k8s.io/api/core/v1"
 	v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 
@@ -60,7 +61,7 @@ type VaultClient interface {
 	Create(context.Context, *ngrok.VaultCreate) (*ngrok.Vault, error)
 	Get(context.Context, string) (*ngrok.Vault, error)
 	GetSecretsByVault(string, *ngrok.Paging) ngrok.Iter[*ngrok.Secret]
-	List(*ngrok.Paging) ngrok.Iter[*ngrok.Vault]
+	List(*ngrok.FilteredPaging) ngrok.Iter[*ngrok.Vault]
 }
 
 // SecretsClient defines interface for interactions with ngrok secrets API.
@@ -68,7 +69,7 @@ type SecretsClient interface {
 	Create(context.Context, *ngrok.SecretCreate) (*ngrok.Secret, error)
 	Delete(context.Context, string) error
 	Get(context.Context, string) (*ngrok.Secret, error)
-	List(*ngrok.Paging) ngrok.Iter[*ngrok.Secret]
+	List(*ngrok.FilteredPaging) ngrok.Iter[*ngrok.Secret]
 	Update(context.Context, *ngrok.SecretUpdate) (*ngrok.Secret, error)
 }
 
@@ -273,7 +274,7 @@ func (c *client) getVaultByName(ctx context.Context, name string) (*ngrok.Vault,
 	listCtx, cancel := context.WithTimeout(ctx, defaultListTimeout)
 	defer cancel()
 
-	iter := c.vaultClient.List(nil)
+	iter := c.vaultClient.List(newObjectNameFilteredPaging(name))
 	for iter.Next(listCtx) {
 		vault := iter.Item()
 		if vault.Name == name {
@@ -290,10 +291,10 @@ func (c *client) getVaultByName(ctx context.Context, name string) (*ngrok.Vault,
 
 // getSecretByVaultIDAndName retrieves a secret by its vault ID and secret name.
 func (c *client) getSecretByVaultIDAndName(ctx context.Context, vaultID, name string) (*ngrok.Secret, error) {
-	iter := c.vaultClient.GetSecretsByVault(vaultID, nil)
+	iter := c.secretsClient.List(newObjectNameFilteredPaging(name))
 	for iter.Next(ctx) {
 		secret := iter.Item()
-		if secret.Name == name {
+		if secret.Name == name && secret.Vault.ID == vaultID {
 			return secret, nil
 		}
 	}
@@ -330,3 +331,10 @@ func parseAndDefaultMetadata(data *v1.JSON) (PushSecretMetadataSpec, error) {
 
 	return def, nil
 }
+
+func newObjectNameFilteredPaging(name string) *ngrok.FilteredPaging {
+	filter := fmt.Sprintf("obj.name == %s", strconv.Quote(name))
+	return &ngrok.FilteredPaging{
+		Filter: &filter,
+	}
+}

providers/v1/ngrok/client_test.go

@@ -20,7 +20,7 @@ import (
 	"encoding/json"
 	"errors"
 
-	"github.com/ngrok/ngrok-api-go/v7"
+	"github.com/ngrok/ngrok-api-go/v9"
 	corev1 "k8s.io/api/core/v1"
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -80,9 +80,11 @@ func WithVaultName(vaultName string) testClientOpt {
 
 var _ = Describe("client", func() {
 	var (
-		s         *fake.Store
-		c         *client
-		vaultName string
+		s          *fake.Store
+		c          *client
+		vaultsAPI  *fake.VaultClient
+		secretsAPI *fake.SecretsClient
+		vaultName  string
 
 		listVaultsErr  error
 		listSecretsErr error
@@ -96,9 +98,11 @@ var _ = Describe("client", func() {
 	})
 
 	JustBeforeEach(func() {
+		vaultsAPI = s.VaultClient().WithListError(listVaultsErr)
+		secretsAPI = s.SecretsClient().WithListError(listSecretsErr)
 		c = &client{
-			vaultClient:   s.VaultClient().WithListError(listVaultsErr),
-			secretsClient: s.SecretsClient().WithListError(listSecretsErr),
+			vaultClient:   vaultsAPI,
+			secretsClient: secretsAPI,
 			vaultName:     vaultName,
 		}
 	})
@@ -356,6 +360,119 @@ spec:
 		})
 	})
 
+	Describe("getVaultByName", func() {
+		var (
+			vault   *ngrok.Vault
+			fetched *ngrok.Vault
+			err     error
+		)
+
+		BeforeEach(func(ctx SpecContext) {
+			var createErr error
+			vault, createErr = s.CreateVault(&ngrok.VaultCreate{
+				Name: vaultName,
+			})
+			Expect(createErr).ToNot(HaveOccurred())
+
+			_, createErr = s.CreateVault(&ngrok.VaultCreate{
+				Name: "other-vault",
+			})
+			Expect(createErr).ToNot(HaveOccurred())
+		})
+
+		JustBeforeEach(func(ctx SpecContext) {
+			fetched, err = c.getVaultByName(ctx, vaultName)
+		})
+
+		It("should return the matching vault", func() {
+			Expect(err).ToNot(HaveOccurred())
+			Expect(fetched).ToNot(BeNil())
+			Expect(fetched.ID).To(Equal(vault.ID))
+		})
+
+		It("should filter vaults by name", func() {
+			Expect(vaultsAPI.LastListPaging()).ToNot(BeNil())
+			Expect(vaultsAPI.LastListPaging().Filter).ToNot(BeNil())
+			Expect(*vaultsAPI.LastListPaging().Filter).To(Equal(`obj.name == "test-vault"`))
+		})
+	})
+
+	Describe("getSecretByVaultIDAndName", func() {
+		var (
+			targetVault *ngrok.Vault
+			otherVault  *ngrok.Vault
+			found       *ngrok.Secret
+			err         error
+			secretName  string
+		)
+
+		BeforeEach(func(ctx SpecContext) {
+			secretName = "shared-name"
+
+			var createErr error
+			targetVault, createErr = s.CreateVault(&ngrok.VaultCreate{Name: vaultName})
+			Expect(createErr).ToNot(HaveOccurred())
+
+			otherVault, createErr = s.CreateVault(&ngrok.VaultCreate{Name: "other-vault"})
+			Expect(createErr).ToNot(HaveOccurred())
+		})
+
+		JustBeforeEach(func(ctx SpecContext) {
+			found, err = c.getSecretByVaultIDAndName(ctx, targetVault.ID, secretName)
+		})
+
+		When("a secret with the same name exists in multiple vaults", func() {
+			var targetSecret *ngrok.Secret
+
+			BeforeEach(func(ctx SpecContext) {
+				_, createErr := s.CreateSecret(&ngrok.SecretCreate{
+					VaultID: otherVault.ID,
+					Name:    secretName,
+					Value:   "other-value",
+				})
+				Expect(createErr).ToNot(HaveOccurred())
+
+				targetSecret, createErr = s.CreateSecret(&ngrok.SecretCreate{
+					VaultID: targetVault.ID,
+					Name:    secretName,
+					Value:   "target-value",
+				})
+				Expect(createErr).ToNot(HaveOccurred())
+			})
+
+			It("should return the secret from the target vault", func() {
+				Expect(err).ToNot(HaveOccurred())
+				Expect(found).ToNot(BeNil())
+				Expect(found.ID).To(Equal(targetSecret.ID))
+				Expect(found.Vault.ID).To(Equal(targetVault.ID))
+			})
+
+			It("should filter secrets by name before checking the vault", func() {
+				Expect(secretsAPI.LastListPaging()).ToNot(BeNil())
+				Expect(secretsAPI.LastListPaging().Filter).ToNot(BeNil())
+				Expect(*secretsAPI.LastListPaging().Filter).To(Equal(`obj.name == "shared-name"`))
+			})
+		})
+
+		When("only another vault has a secret with that name", func() {
+			BeforeEach(func(ctx SpecContext) {
+				_, createErr := s.CreateSecret(&ngrok.SecretCreate{
+					VaultID: otherVault.ID,
+					Name:    secretName,
+					Value:   "other-value",
+				})
+				Expect(createErr).ToNot(HaveOccurred())
+			})
+
+			It("should report the secret as missing from the target vault", func() {
+				Expect(found).To(BeNil())
+				Expect(err).To(HaveOccurred())
+				Expect(err).To(MatchError(ContainSubstring("secret 'shared-name' does not exist")))
+				Expect(err).To(MatchError(ContainSubstring(errVaultSecretDoesNotExist.Error())))
+			})
+		})
+	})
+
 	Describe("DeleteSecret", func() {
 		var (
 			secretName string

providers/v1/ngrok/fake/fake.go

@@ -23,11 +23,12 @@ import (
 	"math/rand"
 	"net/http"
 	"slices"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
 
-	"github.com/ngrok/ngrok-api-go/v7"
+	"github.com/ngrok/ngrok-api-go/v9"
 )
 
 func GenerateRandomString(length int) string {
@@ -450,6 +451,9 @@ type VaultClient struct {
 	store     *Store
 	createErr error
 	listErr   error
+
+	lastListPagingMu sync.Mutex
+	lastListPaging   *ngrok.FilteredPaging
 }
 
 // WithCreateError sets an error to be returned when Create is called.
@@ -483,10 +487,21 @@ func (m *VaultClient) WithListError(err error) *VaultClient {
 	return m
 }
 
+func (m *VaultClient) LastListPaging() *ngrok.FilteredPaging {
+	m.lastListPagingMu.Lock()
+	defer m.lastListPagingMu.Unlock()
+
+	return cloneFilteredPaging(m.lastListPaging)
+}
+
 // List returns an iterator over the vaults.
 // If an error is set, it will return that error instead of the vaults.
-func (m *VaultClient) List(paging *ngrok.Paging) ngrok.Iter[*ngrok.Vault] {
-	return NewIter(m.store.ListVaults(), m.listErr)
+func (m *VaultClient) List(paging *ngrok.FilteredPaging) ngrok.Iter[*ngrok.Vault] {
+	m.lastListPagingMu.Lock()
+	defer m.lastListPagingMu.Unlock()
+
+	m.lastListPaging = cloneFilteredPaging(paging)
+	return NewIter(filterVaults(m.store.ListVaults(), paging), m.listErr)
 }
 
 // SecretsClient is a mock implementation of the SecretsClient interface.
@@ -498,6 +513,9 @@ type SecretsClient struct {
 	updateErr error
 	deleteErr error
 	listErr   error
+
+	lastListPagingMu sync.Mutex
+	lastListPaging   *ngrok.FilteredPaging
 }
 
 // WithCreateError sets an error to be returned when Create is called.
@@ -561,10 +579,21 @@ func (m *SecretsClient) WithListError(err error) *SecretsClient {
 	return m
 }
 
+func (m *SecretsClient) LastListPaging() *ngrok.FilteredPaging {
+	m.lastListPagingMu.Lock()
+	defer m.lastListPagingMu.Unlock()
+
+	return cloneFilteredPaging(m.lastListPaging)
+}
+
 // List returns an iterator over the secrets.
 // If an error is set, it will return that error instead of the secrets.
-func (m *SecretsClient) List(paging *ngrok.Paging) ngrok.Iter[*ngrok.Secret] {
-	return NewIter(m.store.ListSecrets(), m.listErr)
+func (m *SecretsClient) List(paging *ngrok.FilteredPaging) ngrok.Iter[*ngrok.Secret] {
+	m.lastListPagingMu.Lock()
+	defer m.lastListPagingMu.Unlock()
+
+	m.lastListPaging = cloneFilteredPaging(paging)
+	return NewIter(filterSecrets(m.store.ListSecrets(), paging), m.listErr)
 }
 
 // Iter is a mock iterator that implements the ngrok.Iter[T] interface.
@@ -604,3 +633,75 @@ func NewIter[T any](items []T, err error) *Iter[T] {
 		n:     -1,
 	}
 }
+
+func cloneFilteredPaging(paging *ngrok.FilteredPaging) *ngrok.FilteredPaging {
+	if paging == nil {
+		return nil
+	}
+
+	clone := *paging
+	if paging.Filter != nil {
+		filter := *paging.Filter
+		clone.Filter = &filter
+	}
+	if paging.BeforeID != nil {
+		beforeID := *paging.BeforeID
+		clone.BeforeID = &beforeID
+	}
+	if paging.Limit != nil {
+		limit := *paging.Limit
+		clone.Limit = &limit
+	}
+	return &clone
+}
+
+func filterVaults(vaults []*ngrok.Vault, paging *ngrok.FilteredPaging) []*ngrok.Vault {
+	if paging == nil || paging.Filter == nil {
+		return vaults
+	}
+
+	name, ok := exactNameFilter(*paging.Filter)
+	if !ok {
+		return vaults
+	}
+
+	filtered := make([]*ngrok.Vault, 0, len(vaults))
+	for _, vault := range vaults {
+		if vault.Name == name {
+			filtered = append(filtered, vault)
+		}
+	}
+	return filtered
+}
+
+func filterSecrets(secrets []*ngrok.Secret, paging *ngrok.FilteredPaging) []*ngrok.Secret {
+	if paging == nil || paging.Filter == nil {
+		return secrets
+	}
+
+	name, ok := exactNameFilter(*paging.Filter)
+	if !ok {
+		return secrets
+	}
+
+	filtered := make([]*ngrok.Secret, 0, len(secrets))
+	for _, secret := range secrets {
+		if secret.Name == name {
+			filtered = append(filtered, secret)
+		}
+	}
+	return filtered
+}
+
+func exactNameFilter(filter string) (string, bool) {
+	rest, ok := strings.CutPrefix(filter, "obj.name == ")
+	if !ok {
+		return "", false
+	}
+
+	name, err := strconv.Unquote(rest)
+	if err != nil {
+		return "", false
+	}
+	return name, true
+}

providers/v1/ngrok/go.mod

@@ -5,7 +5,7 @@ go 1.26.4
 require (
 	github.com/external-secrets/external-secrets/apis v0.0.0
 	github.com/external-secrets/external-secrets/runtime v0.0.0
-	github.com/ngrok/ngrok-api-go/v7 v7.6.0
+	github.com/ngrok/ngrok-api-go/v9 v9.0.0
 	github.com/onsi/ginkgo/v2 v2.28.0
 	github.com/onsi/gomega v1.39.1
 	k8s.io/api v0.35.2

providers/v1/ngrok/go.sum

@@ -103,8 +103,8 @@ github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFd
 github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
-github.com/ngrok/ngrok-api-go/v7 v7.6.0 h1:DW9FqEgSN6+Dgl25O8ha1LS49CqX2c9vO0Z53CN8Vqs=
-github.com/ngrok/ngrok-api-go/v7 v7.6.0/go.mod h1:Si/pYAJmbCuo4Fb3xz0MF6N5ubRvPdUixETBwhFvBf0=
+github.com/ngrok/ngrok-api-go/v9 v9.0.0 h1:dBTOl+PK8oUN/WAores6V/CTlyPEqD8yH70ToDSuAWg=
+github.com/ngrok/ngrok-api-go/v9 v9.0.0/go.mod h1:Y6SGu3WzfTRGJiHVSk2REXp3cmi0cEsAvJDsb30dBuU=
 github.com/onsi/ginkgo/v2 v2.28.0 h1:Rrf+lVLmtlBIKv6KrIGJCjyY8N36vDVcutbGJkyqjJc=
 github.com/onsi/ginkgo/v2 v2.28.0/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo=
 github.com/onsi/gomega v1.39.1 h1:1IJLAad4zjPn2PsnhH70V4DKRFlrCzGBNrNaru+Vf28=

providers/v1/ngrok/provider.go

@@ -22,9 +22,9 @@ import (
 	"fmt"
 	"net/url"
 
-	"github.com/ngrok/ngrok-api-go/v7"
-	"github.com/ngrok/ngrok-api-go/v7/secrets"
-	"github.com/ngrok/ngrok-api-go/v7/vaults"
+	"github.com/ngrok/ngrok-api-go/v9"
+	"github.com/ngrok/ngrok-api-go/v9/secrets"
+	"github.com/ngrok/ngrok-api-go/v9/vaults"
 	kubeClient "sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
 
@@ -91,32 +91,22 @@ func (p *Provider) NewClient(ctx context.Context, store esv1.GenericStore, kubeC
 	vaultClient := getVaultsClient(clientConfig)
 	secretsClient := getSecretsClient(clientConfig)
 
-	listCtx, cancel := context.WithTimeout(ctx, defaultListTimeout)
-	defer cancel()
-
-	var vault *ngrok.Vault
-	vaultIter := vaultClient.List(nil)
-	for vaultIter.Next(listCtx) {
-		if vaultIter.Item().Name == cfg.Vault.Name {
-			vault = vaultIter.Item()
-			break
-		}
+	c := &client{
+		vaultClient:   vaultClient,
+		secretsClient: secretsClient,
+		vaultName:     cfg.Vault.Name,
 	}
 
-	if err := vaultIter.Err(); err != nil {
+	vault, err := c.getVaultByName(ctx, cfg.Vault.Name)
+	if err != nil {
+		if errors.Is(err, errVaultDoesNotExist) {
+			return nil, fmt.Errorf("vault %q not found", cfg.Vault.Name)
+		}
 		return nil, fmt.Errorf("error listing vaults: %w", err)
 	}
 
-	if vault == nil {
-		return nil, fmt.Errorf("vault %q not found", cfg.Vault.Name)
-	}
-
-	return &client{
-		vaultClient:   vaultClient,
-		secretsClient: secretsClient,
-		vaultName:     cfg.Vault.Name,
-		vaultID:       vault.ID,
-	}, nil
+	c.setVaultID(vault.ID)
+	return c, nil
 }
 
 // ValidateStore validates the store configuration.

providers/v1/ngrok/provider_test.go

@@ -20,7 +20,7 @@ import (
 	"fmt"
 	"testing"
 
-	"github.com/ngrok/ngrok-api-go/v7"
+	"github.com/ngrok/ngrok-api-go/v9"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	kubeClient "sigs.k8s.io/controller-runtime/pkg/client"
@@ -101,6 +101,7 @@ var _ = Describe("Provider", func() {
 		var (
 			store            esv1.GenericStore
 			ngrokStore       *fake.Store
+			vaultsClient     *fake.VaultClient
 			namespace        string
 			kubeClient       kubeClient.Client
 			ngrokCredentials *corev1.Secret
@@ -125,7 +126,8 @@ var _ = Describe("Provider", func() {
 
 		JustBeforeEach(func() {
 			getVaultsClient = func(_ *ngrok.ClientConfig) VaultClient {
-				return ngrokStore.VaultClient().WithListError(vaultListErr)
+				vaultsClient = ngrokStore.VaultClient().WithListError(vaultListErr)
+				return vaultsClient
 			}
 			getSecretsClient = func(_ *ngrok.ClientConfig) SecretsClient {
 				return ngrokStore.SecretsClient()
@@ -198,6 +200,13 @@ var _ = Describe("Provider", func() {
 					It("should return a non-nil client", func() {
 						Expect(client).NotTo(BeNil())
 					})
+
+					It("should filter vaults by name when listing", func() {
+						Expect(vaultsClient).NotTo(BeNil())
+						Expect(vaultsClient.LastListPaging()).NotTo(BeNil())
+						Expect(vaultsClient.LastListPaging().Filter).NotTo(BeNil())
+						Expect(*vaultsClient.LastListPaging().Filter).To(Equal(fmt.Sprintf("obj.name == %q", vaultName)))
+					})
 				})
 
 				When("there is an error listing vaults", func() {
@@ -304,6 +313,13 @@ var _ = Describe("Provider", func() {
 					It("should return a non-nil client", func() {
 						Expect(client).NotTo(BeNil())
 					})
+
+					It("should filter vaults by name when listing", func() {
+						Expect(vaultsClient).NotTo(BeNil())
+						Expect(vaultsClient.LastListPaging()).NotTo(BeNil())
+						Expect(vaultsClient.LastListPaging().Filter).NotTo(BeNil())
+						Expect(*vaultsClient.LastListPaging().Filter).To(Equal(fmt.Sprintf("obj.name == %q", vaultName)))
+					})
 				})
 			})
 		})