Merge pull request #946 from external-secrets/feat/gcp-getall

feat: gcp getall implementaion

e2e/suite/gcp/gcp.go

@@ -46,6 +46,10 @@ var _ = Describe("[gcp]", Label("gcp", "secretsmanager"), func() {
 		Entry(common.SSHKeySyncDataProperty(f)),
 		Entry(common.SyncWithoutTargetName(f)),
 		Entry(common.JSONDataWithoutTargetName(f)),
+		Entry(common.FindByName(f)),
+		Entry(common.FindByNameWithPath(f)),
+		Entry(common.FindByTag(f)),
+		Entry(common.FindByTagWithPath(f)),
 		Entry(common.SyncV1Alpha1(f)),
 		Entry("should sync p12 encoded cert secret", p12Cert),
 	)

e2e/suite/gcp/provider.go

@@ -113,6 +113,7 @@ func (s *GcpProvider) CreateSecret(key string, val framework.SecretEntry) {
 		Parent:   fmt.Sprintf("projects/%s", s.projectID),
 		SecretId: key,
 		Secret: &secretmanagerpb.Secret{
+			Labels: val.Tags,
 			Replication: &secretmanagerpb.Replication{
 				Replication: &secretmanagerpb.Replication_Automatic_{
 					Automatic: &secretmanagerpb.Replication_Automatic{},

pkg/provider/aws/secretsmanager/secretsmanager.go

@@ -89,7 +89,7 @@ func (sm *SecretsManager) fetch(_ context.Context, ref esv1beta1.ExternalSecretD
 	return secretOut, nil
 }
 
-// Empty GetAllSecrets.
+// GetAllSecrets syncs multiple secrets from aws provider into a single Kubernetes Secret.
 func (sm *SecretsManager) GetAllSecrets(ctx context.Context, ref esv1beta1.ExternalSecretFind) (map[string][]byte, error) {
 	if ref.Name != nil {
 		return sm.findByName(ctx, ref)

pkg/provider/gcp/secretmanager/fake/fake.go

@@ -17,21 +17,26 @@ import (
 	"context"
 	"fmt"
 
+	secretmanager "cloud.google.com/go/secretmanager/apiv1"
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
-	grpc "github.com/googleapis/gax-go/v2"
+	"github.com/googleapis/gax-go/v2"
 	secretmanagerpb "google.golang.org/genproto/googleapis/cloud/secretmanager/v1"
 )
 
 type MockSMClient struct {
-	accessSecretFn func(ctx context.Context, req *secretmanagerpb.AccessSecretVersionRequest, opts ...grpc.CallOption) (*secretmanagerpb.AccessSecretVersionResponse, error)
+	accessSecretFn func(ctx context.Context, req *secretmanagerpb.AccessSecretVersionRequest, opts ...gax.CallOption) (*secretmanagerpb.AccessSecretVersionResponse, error)
+	ListSecretsFn  func(ctx context.Context, req *secretmanagerpb.ListSecretsRequest, opts ...gax.CallOption) *secretmanager.SecretIterator
 	closeFn        func() error
 }
 
-func (mc *MockSMClient) AccessSecretVersion(ctx context.Context, req *secretmanagerpb.AccessSecretVersionRequest, opts ...grpc.CallOption) (*secretmanagerpb.AccessSecretVersionResponse, error) {
+func (mc *MockSMClient) AccessSecretVersion(ctx context.Context, req *secretmanagerpb.AccessSecretVersionRequest, opts ...gax.CallOption) (*secretmanagerpb.AccessSecretVersionResponse, error) {
 	return mc.accessSecretFn(ctx, req)
 }
 
+func (mc *MockSMClient) ListSecrets(ctx context.Context, req *secretmanagerpb.ListSecretsRequest, opts ...gax.CallOption) *secretmanager.SecretIterator {
+	return mc.ListSecretsFn(ctx, req)
+}
 func (mc *MockSMClient) Close() error {
 	return mc.closeFn()
 }
@@ -44,7 +49,7 @@ func (mc *MockSMClient) NilClose() {
 
 func (mc *MockSMClient) WithValue(ctx context.Context, req *secretmanagerpb.AccessSecretVersionRequest, val *secretmanagerpb.AccessSecretVersionResponse, err error) {
 	if mc != nil {
-		mc.accessSecretFn = func(paramCtx context.Context, paramReq *secretmanagerpb.AccessSecretVersionRequest, paramOpts ...grpc.CallOption) (*secretmanagerpb.AccessSecretVersionResponse, error) {
+		mc.accessSecretFn = func(paramCtx context.Context, paramReq *secretmanagerpb.AccessSecretVersionRequest, paramOpts ...gax.CallOption) (*secretmanagerpb.AccessSecretVersionResponse, error) {
 			// type secretmanagerpb.AccessSecretVersionRequest contains unexported fields
 			// use cmpopts.IgnoreUnexported to ignore all the unexported fields in the cmp.
 			if !cmp.Equal(paramReq, req, cmpopts.IgnoreUnexported(secretmanagerpb.AccessSecretVersionRequest{})) {

pkg/provider/gcp/secretmanager/secretsmanager.go

@@ -16,7 +16,9 @@ package secretmanager
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
+	"strconv"
 	"strings"
 	"sync"
 
@@ -25,13 +27,16 @@ import (
 	"github.com/tidwall/gjson"
 	"golang.org/x/oauth2"
 	"golang.org/x/oauth2/google"
+	"google.golang.org/api/iterator"
 	"google.golang.org/api/option"
 	secretmanagerpb "google.golang.org/genproto/googleapis/cloud/secretmanager/v1"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
+	ctrl "sigs.k8s.io/controller-runtime"
 	kclient "sigs.k8s.io/controller-runtime/pkg/client"
 
 	esv1beta1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1beta1"
+	"github.com/external-secrets/external-secrets/pkg/find"
 	"github.com/external-secrets/external-secrets/pkg/utils"
 )
 
@@ -53,16 +58,20 @@ const (
 	errClientGetSecretAccess                  = "unable to access Secret from SecretManager Client: %w"
 	errJSONSecretUnmarshal                    = "unable to unmarshal secret: %w"
 
-	errInvalidStore         = "invalid store"
-	errInvalidStoreSpec     = "invalid store spec"
-	errInvalidStoreProv     = "invalid store provider"
-	errInvalidGCPProv       = "invalid gcp secrets manager provider"
-	errInvalidAuthSecretRef = "invalid auth secret ref: %w"
-	errInvalidWISARef       = "invalid workload identity service account reference: %w"
+	errInvalidStore           = "invalid store"
+	errInvalidStoreSpec       = "invalid store spec"
+	errInvalidStoreProv       = "invalid store provider"
+	errInvalidGCPProv         = "invalid gcp secrets manager provider"
+	errInvalidAuthSecretRef   = "invalid auth secret ref: %w"
+	errInvalidWISARef         = "invalid workload identity service account reference: %w"
+	errUnexpectedFindOperator = "unexpected find operator"
 )
 
+var log = ctrl.Log.WithName("provider").WithName("gcp").WithName("secretsmanager")
+
 type GoogleSecretManagerClient interface {
 	AccessSecretVersion(ctx context.Context, req *secretmanagerpb.AccessSecretVersionRequest, opts ...gax.CallOption) (*secretmanagerpb.AccessSecretVersionResponse, error)
+	ListSecrets(ctx context.Context, req *secretmanagerpb.ListSecretsRequest, opts ...gax.CallOption) *secretmanager.SecretIterator
 	Close() error
 }
 
@@ -83,10 +92,11 @@ type ProviderGCP struct {
 }
 
 type gClient struct {
-	kube             kclient.Client
-	store            *esv1beta1.GCPSMProvider
-	namespace        string
-	storeKind        string
+	kube      kclient.Client
+	store     *esv1beta1.GCPSMProvider
+	namespace string
+	storeKind string
+
 	workloadIdentity *workloadIdentity
 }
 
@@ -201,10 +211,124 @@ func (sm *ProviderGCP) NewClient(ctx context.Context, store esv1beta1.GenericSto
 	return sm, nil
 }
 
-// Empty GetAllSecrets.
+// GetAllSecrets syncs multiple secrets from gcp provider into a single Kubernetes Secret.
 func (sm *ProviderGCP) GetAllSecrets(ctx context.Context, ref esv1beta1.ExternalSecretFind) (map[string][]byte, error) {
-	// TO be implemented
-	return nil, fmt.Errorf("GetAllSecrets not implemented")
+	if ref.Name != nil {
+		return sm.findByName(ctx, ref)
+	}
+	if len(ref.Tags) > 0 {
+		return sm.findByTags(ctx, ref)
+	}
+	return nil, errors.New(errUnexpectedFindOperator)
+}
+
+func (sm *ProviderGCP) findByName(ctx context.Context, ref esv1beta1.ExternalSecretFind) (map[string][]byte, error) {
+	// regex matcher
+	matcher, err := find.New(*ref.Name)
+	if err != nil {
+		return nil, err
+	}
+	req := &secretmanagerpb.ListSecretsRequest{
+		Parent: fmt.Sprintf("projects/%s", sm.projectID),
+	}
+	if ref.Path != nil {
+		req.Filter = fmt.Sprintf("name:%s", *ref.Path)
+	}
+	// Call the API.
+	it := sm.SecretManagerClient.ListSecrets(ctx, req)
+	secretMap := make(map[string][]byte)
+	for {
+		resp, err := it.Next()
+		if errors.Is(err, iterator.Done) {
+			break
+		}
+		if err != nil {
+			return nil, fmt.Errorf("failed to list secrets: %w", err)
+		}
+		log.V(1).Info("gcp sm findByName found", "secrets", strconv.Itoa(it.PageInfo().Remaining()))
+		key := sm.trimName(resp.Name)
+		// If we don't match we skip.
+		// Also, if we have path, and it is not at the beguining we skip.
+		// We have to check if path is at the beguining of the key because
+		// there is no way to create a `name:%s*` (starts with) filter
+		// At https://cloud.google.com/secret-manager/docs/filtering you can use `*`
+		// but not like that it seems.
+		if !matcher.MatchName(key) || (ref.Path != nil && !strings.HasPrefix(key, *ref.Path)) {
+			continue
+		}
+		log.V(1).Info("gcp sm findByName matches", "name", resp.Name)
+		secretMap[key], err = sm.getData(ctx, key)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return utils.ConvertKeys(ref.ConversionStrategy, secretMap)
+}
+
+func (sm *ProviderGCP) getData(ctx context.Context, key string) ([]byte, error) {
+	dataRef := esv1beta1.ExternalSecretDataRemoteRef{
+		Key: key,
+	}
+	data, err := sm.GetSecret(ctx, dataRef)
+	if err != nil {
+		return []byte(""), err
+	}
+	return data, nil
+}
+
+func (sm *ProviderGCP) findByTags(ctx context.Context, ref esv1beta1.ExternalSecretFind) (map[string][]byte, error) {
+	var tagFilter string
+	for k, v := range ref.Tags {
+		tagFilter = fmt.Sprintf("%slabels.%s=%s ", tagFilter, k, v)
+	}
+	tagFilter = strings.TrimSuffix(tagFilter, " ")
+	if ref.Path != nil {
+		tagFilter = fmt.Sprintf("%s name:%s", tagFilter, *ref.Path)
+	}
+	req := &secretmanagerpb.ListSecretsRequest{
+		Parent: fmt.Sprintf("projects/%s", sm.projectID),
+	}
+	log.V(1).Info("gcp sm findByTags", "tagFilter", tagFilter)
+	req.Filter = tagFilter
+	// Call the API.
+	it := sm.SecretManagerClient.ListSecrets(ctx, req)
+	secretMap := make(map[string][]byte)
+	for {
+		resp, err := it.Next()
+		if errors.Is(err, iterator.Done) {
+			break
+		}
+		if err != nil {
+			return nil, fmt.Errorf("failed to list secrets: %w", err)
+		}
+		key := sm.trimName(resp.Name)
+		if ref.Path != nil && !strings.HasPrefix(key, *ref.Path) {
+			continue
+		}
+		log.V(1).Info("gcp sm findByTags matches tags", "name", resp.Name)
+		secretMap[key], err = sm.getData(ctx, key)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return utils.ConvertKeys(ref.ConversionStrategy, secretMap)
+}
+
+func (sm *ProviderGCP) trimName(name string) string {
+	projectIDNumuber := sm.extractProjectIDNumber(name)
+	key := strings.TrimPrefix(name, fmt.Sprintf("projects/%s/secrets/", projectIDNumuber))
+	return key
+}
+
+// extractProjectIDNumber grabs the project id from the full name returned by gcp api
+// gcp api seems to always return the number and not the project name
+// (and users would always use the name, while requests accept both).
+func (sm *ProviderGCP) extractProjectIDNumber(secretFullName string) string {
+	s := strings.Split(secretFullName, "/")
+	projectIDNumuber := s[1]
+	return projectIDNumuber
 }
 
 // GetSecret returns a single secret from the provider.