✨gitlab: getAllSecrets (#1681)

* gitlab: getAllSecrets

Signed-off-by: Dominik Zeiger <dominik@zeiger.biz>

* Update pkg/provider/gitlab/gitlab.go

Co-authored-by: Gustavo Fernandes de Carvalho <gusfcarvalho@gmail.com>
Signed-off-by: Dominik Zeiger <domizei385@users.noreply.github.com>
Signed-off-by: Dominik Zeiger <dominik@zeiger.biz>

* gitlab: added some test coverage

Signed-off-by: Dominik Zeiger <dominik@zeiger.biz>

Signed-off-by: Dominik Zeiger <dominik@zeiger.biz>
Signed-off-by: Dominik Zeiger <domizei385@users.noreply.github.com>
Co-authored-by: Gustavo Fernandes de Carvalho <gusfcarvalho@gmail.com>

docs/contributing/devguide.md

@@ -80,7 +80,7 @@ To remove the CRDs run:
 make crds.uninstall
 ```
 
-If you need to test some other k8s integrations and need the operator to be deployed to the actuall cluster while developing, you can use the following workflow:
+If you need to test some other k8s integrations and need the operator to be deployed to the actual cluster while developing, you can use the following workflow:
 
 ```
 kind create cluster --name external-secrets

docs/guides/datafrom-rewrite.md

@@ -11,7 +11,7 @@ Rewrite operations are all applied before `ConversionStrategy` is applied.
 ### Regexp
 This method implements rewriting through the use of regular expressions. It needs a `source` and a `target` field. The source field is where the definition of the matching regular expression goes, where the `target` field is where the replacing expression goes.
 
-Some considerations about the impelementation of Regexp Rewrite:
+Some considerations about the implementation of Regexp Rewrite:
 
 1. The input of a subsequent rewrite operation are the outputs of the previous rewrite.
 2. If a given set of keys do not match any Rewrite operation, there will be no error. Rather, the original keys will be used.

pkg/provider/gitlab/gitlab.go

@@ -27,6 +27,7 @@ import (
 	kclient "sigs.k8s.io/controller-runtime/pkg/client"
 
 	esv1beta1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1beta1"
+	"github.com/external-secrets/external-secrets/pkg/find"
 	"github.com/external-secrets/external-secrets/pkg/utils"
 )
 
@@ -38,6 +39,9 @@ const (
 	errList                                   = "could not verify if the client is valid: %w"
 	errAuth                                   = "client is not allowed to get secrets"
 	errUninitializedGitlabProvider            = "provider gitlab is not initialized"
+	errNameNotDefined                         = "'find.name' is mandatory"
+	errTagsNotImplemented                     = "'find.tags' is not currently supported by Gitlab provider"
+	errPathNotImplemented                     = "'find.path' is not implemented in the Gitlab provider"
 	errJSONSecretUnmarshal                    = "unable to unmarshal secret: %w"
 )
 
@@ -98,7 +102,7 @@ func (c *gClient) setAuth(ctx context.Context) error {
 	}
 
 	c.credentials = credentialsSecret.Data[c.store.Auth.SecretRef.AccessToken.Key]
-	if (c.credentials == nil) || (len(c.credentials) == 0) {
+	if c.credentials == nil || len(c.credentials) == 0 {
 		return fmt.Errorf(errMissingSAK)
 	}
 	// I don't know where ProjectID is being set
@@ -155,10 +159,44 @@ func (g *Gitlab) NewClient(ctx context.Context, store esv1beta1.GenericStore, ku
 	return g, nil
 }
 
-// Empty GetAllSecrets.
+// GetAllSecrets syncs all gitlab project variables into a single Kubernetes Secret.
 func (g *Gitlab) GetAllSecrets(ctx context.Context, ref esv1beta1.ExternalSecretFind) (map[string][]byte, error) {
-	// TO be implemented
-	return nil, fmt.Errorf("GetAllSecrets not implemented")
+	if utils.IsNil(g.client) {
+		return nil, fmt.Errorf(errUninitializedGitlabProvider)
+	}
+	if ref.Tags != nil {
+		return nil, fmt.Errorf(errTagsNotImplemented)
+	}
+	if ref.Path != nil {
+		return nil, fmt.Errorf(errPathNotImplemented)
+	}
+	if ref.Name == nil {
+		return nil, fmt.Errorf(errNameNotDefined)
+	}
+
+	allData, _, err := g.client.ListVariables(g.projectID, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	var matcher *find.Matcher
+	if ref.Name != nil {
+		m, err := find.New(*ref.Name)
+		if err != nil {
+			return nil, err
+		}
+		matcher = m
+	}
+	secretData := make(map[string][]byte)
+	for _, data := range allData {
+		matching, key := matchesFilter(g.environment, data, matcher)
+		if !matching {
+			continue
+		}
+		secretData[key] = []byte(data.Value)
+	}
+
+	return secretData, nil
 }
 
 func (g *Gitlab) GetSecret(ctx context.Context, ref esv1beta1.ExternalSecretDataRemoteRef) ([]byte, error) {
@@ -176,6 +214,7 @@ func (g *Gitlab) GetSecret(ctx context.Context, ref esv1beta1.ExternalSecretData
 	// 	"masked": true,
 	// 	"environment_scope": "*"
 	// }
+
 	var vopts *gitlab.GetProjectVariableOptions
 	if g.environment != "" {
 		vopts = &gitlab.GetProjectVariableOptions{Filter: &gitlab.VariableFilter{EnvironmentScope: g.environment}}
@@ -226,6 +265,22 @@ func (g *Gitlab) GetSecretMap(ctx context.Context, ref esv1beta1.ExternalSecretD
 	return secretData, nil
 }
 
+func matchesFilter(environment string, data *gitlab.ProjectVariable, matcher *find.Matcher) (bool, string) {
+	if environment != "" && environment != "*" {
+		// as of now gitlab does not support filtering of EnvironmentScope through the api call
+		if data.EnvironmentScope != environment {
+			return false, ""
+		}
+	}
+
+	key := data.Key
+	if key == "" || (matcher != nil && !matcher.MatchName(key)) {
+		return false, ""
+	}
+
+	return true, key
+}
+
 func (g *Gitlab) Close(ctx context.Context) error {
 	return nil
 }

pkg/provider/gitlab/gitlab_test.go

@@ -15,24 +15,34 @@ package gitlab
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"net/http"
 	"reflect"
 	"strings"
 	"testing"
 
-	gitlab "github.com/xanzy/go-gitlab"
+	"github.com/google/uuid"
+	tassert "github.com/stretchr/testify/assert"
+	"github.com/xanzy/go-gitlab"
+	"github.com/yandex-cloud/go-sdk/iamkey"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	k8sclient "sigs.k8s.io/controller-runtime/pkg/client"
+	clientfake "sigs.k8s.io/controller-runtime/pkg/client/fake"
 
 	esv1beta1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1beta1"
-	v1 "github.com/external-secrets/external-secrets/apis/meta/v1"
+	esv1meta "github.com/external-secrets/external-secrets/apis/meta/v1"
 	fakegitlab "github.com/external-secrets/external-secrets/pkg/provider/gitlab/fake"
 )
 
 const (
-	project     = "my-Project"
-	username    = "user-name"
-	userkey     = "user-key"
-	environment = "prod"
+	project               = "my-Project"
+	username              = "user-name"
+	userkey               = "user-key"
+	environment           = "prod"
+	defaultErrorMessage   = "[%d] unexpected error: %s, expected: '%s'"
+	errMissingCredentials = "credentials are empty"
 )
 
 type secretManagerTestCase struct {
@@ -43,8 +53,8 @@ type secretManagerTestCase struct {
 	apiOutput                *gitlab.ProjectVariable
 	apiResponse              *gitlab.Response
 	ref                      *esv1beta1.ExternalSecretDataRemoteRef
+	refFind                  *esv1beta1.ExternalSecretFind
 	projectID                *string
-	environment              *string
 	apiErr                   error
 	expectError              string
 	expectedSecret           string
@@ -60,8 +70,8 @@ func makeValidSecretManagerTestCase() *secretManagerTestCase {
 		apiInputKey:              makeValidAPIInputKey(),
 		apiInputEnv:              makeValidEnvironment(),
 		ref:                      makeValidRef(),
+		refFind:                  makeValidFindRef(),
 		projectID:                nil,
-		environment:              nil,
 		apiOutput:                makeValidAPIOutput(),
 		apiResponse:              makeValidAPIResponse(),
 		apiErr:                   nil,
@@ -81,6 +91,16 @@ func makeValidRef() *esv1beta1.ExternalSecretDataRemoteRef {
 	}
 }
 
+func makeValidFindRef() *esv1beta1.ExternalSecretFind {
+	return &esv1beta1.ExternalSecretFind{}
+}
+
+func makeFindName(regexp string) *esv1beta1.FindName {
+	return &esv1beta1.FindName{
+		RegExp: regexp,
+	}
+}
+
 func makeValidAPIInputProjectID() string {
 	return "testID"
 }
@@ -117,6 +137,17 @@ func makeValidSecretManagerTestCaseCustom(tweaks ...func(smtc *secretManagerTest
 	return smtc
 }
 
+func makeValidSecretManagerGetAllTestCaseCustom(tweaks ...func(smtc *secretManagerTestCase)) *secretManagerTestCase {
+	smtc := makeValidSecretManagerTestCase()
+	smtc.ref = nil
+	smtc.refFind.Name = makeFindName(".*")
+	for _, fn := range tweaks {
+		fn(smtc)
+	}
+	smtc.mockClient.WithValue(smtc.apiInputProjectID, smtc.apiInputEnv, smtc.apiInputKey, smtc.apiOutput, smtc.apiResponse, smtc.apiErr)
+	return smtc
+}
+
 // This case can be shared by both GetSecret and GetSecretMap tests.
 // bad case: set apiErr.
 var setAPIErr = func(smtc *secretManagerTestCase) {
@@ -149,9 +180,91 @@ var setNilMockClient = func(smtc *secretManagerTestCase) {
 	smtc.expectError = errUninitializedGitlabProvider
 }
 
+func TestNewClient(t *testing.T) {
+	ctx := context.Background()
+	const namespace = "namespace"
+
+	store := &esv1beta1.SecretStore{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: namespace,
+		},
+		Spec: esv1beta1.SecretStoreSpec{
+			Provider: &esv1beta1.SecretStoreProvider{
+				Gitlab: &esv1beta1.GitlabProvider{},
+			},
+		},
+	}
+	provider, err := esv1beta1.GetProvider(store)
+	tassert.Nil(t, err)
+
+	k8sClient := clientfake.NewClientBuilder().Build()
+	secretClient, err := provider.NewClient(context.Background(), store, k8sClient, namespace)
+	tassert.EqualError(t, err, errMissingCredentials)
+	tassert.Nil(t, secretClient)
+
+	store.Spec.Provider.Gitlab.Auth = esv1beta1.GitlabAuth{}
+	secretClient, err = provider.NewClient(context.Background(), store, k8sClient, namespace)
+	tassert.EqualError(t, err, errMissingCredentials)
+	tassert.Nil(t, secretClient)
+
+	store.Spec.Provider.Gitlab.Auth.SecretRef = esv1beta1.GitlabSecretRef{}
+	secretClient, err = provider.NewClient(context.Background(), store, k8sClient, namespace)
+	tassert.EqualError(t, err, errMissingCredentials)
+	tassert.Nil(t, secretClient)
+
+	store.Spec.Provider.Gitlab.Auth.SecretRef.AccessToken = esv1meta.SecretKeySelector{}
+	secretClient, err = provider.NewClient(context.Background(), store, k8sClient, namespace)
+	tassert.EqualError(t, err, errMissingCredentials)
+	tassert.Nil(t, secretClient)
+
+	const authorizedKeySecretName = "authorizedKeySecretName"
+	const authorizedKeySecretKey = "authorizedKeySecretKey"
+	store.Spec.Provider.Gitlab.Auth.SecretRef.AccessToken.Name = authorizedKeySecretName
+	store.Spec.Provider.Gitlab.Auth.SecretRef.AccessToken.Key = authorizedKeySecretKey
+	secretClient, err = provider.NewClient(context.Background(), store, k8sClient, namespace)
+	tassert.EqualError(t, err, "couldn't find secret on cluster: secrets \"authorizedKeySecretName\" not found")
+	tassert.Nil(t, secretClient)
+
+	err = createK8sSecret(ctx, t, k8sClient, namespace, authorizedKeySecretName, authorizedKeySecretKey, toJSON(t, newFakeAuthorizedKey()))
+	tassert.Nil(t, err)
+
+	secretClient, err = provider.NewClient(context.Background(), store, k8sClient, namespace)
+	tassert.Nil(t, err)
+	tassert.NotNil(t, secretClient)
+}
+
+func toJSON(t *testing.T, v interface{}) []byte {
+	jsonBytes, err := json.Marshal(v)
+	tassert.Nil(t, err)
+	return jsonBytes
+}
+
+func createK8sSecret(ctx context.Context, t *testing.T, k8sClient k8sclient.Client, namespace, secretName, secretKey string, secretValue []byte) error {
+	err := k8sClient.Create(ctx, &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: namespace,
+			Name:      secretName,
+		},
+		Data: map[string][]byte{secretKey: secretValue},
+	})
+	tassert.Nil(t, err)
+	return nil
+}
+
+func newFakeAuthorizedKey() *iamkey.Key {
+	uniqueLabel := uuid.NewString()
+	return &iamkey.Key{
+		Id: uniqueLabel,
+		Subject: &iamkey.Key_ServiceAccountId{
+			ServiceAccountId: uniqueLabel,
+		},
+		PrivateKey: uniqueLabel,
+	}
+}
+
 // test the sm<->gcp interface
 // make sure correct values are passed and errors are handled accordingly.
-func TestGitlabSecretManagerGetSecret(t *testing.T) {
+func TestGetSecret(t *testing.T) {
 	secretValue := "changedvalue"
 	// good case: default version is set
 	// key is passed in, output is sent back
@@ -175,7 +288,7 @@ func TestGitlabSecretManagerGetSecret(t *testing.T) {
 		sm.client = v.mockClient
 		out, err := sm.GetSecret(context.Background(), *v.ref)
 		if !ErrorContains(err, v.expectError) {
-			t.Errorf("[%d] unexpected error: %s, expected: '%s'", k, err.Error(), v.expectError)
+			t.Errorf(defaultErrorMessage, k, err.Error(), v.expectError)
 		}
 		if string(out) != v.expectedSecret {
 			t.Errorf("[%d] unexpected secret: expected %s, got %s", k, v.expectedSecret, string(out))
@@ -183,6 +296,77 @@ func TestGitlabSecretManagerGetSecret(t *testing.T) {
 	}
 }
 
+func TestGetAllSecrets(t *testing.T) {
+	secretValue := "changedvalue"
+	// good case: default version is set
+	// key is passed in, output is sent back
+
+	setMissingFindRegex := func(smtc *secretManagerTestCase) {
+		smtc.refFind.Name = nil
+		smtc.expectError = "'find.name' is mandatory"
+	}
+	setUnsupportedFindTags := func(smtc *secretManagerTestCase) {
+		smtc.refFind.Tags = map[string]string{}
+		smtc.expectError = "'find.tags' is not currently supported by Gitlab provider"
+	}
+	setUnsupportedFindPath := func(smtc *secretManagerTestCase) {
+		path := "path"
+		smtc.refFind.Path = &path
+		smtc.expectError = "'find.path' is not implemented in the Gitlab provider"
+	}
+	setMatchingSecretFindString := func(smtc *secretManagerTestCase) {
+		smtc.apiOutput = &gitlab.ProjectVariable{
+			Key:              "testkey",
+			Value:            "changedvalue",
+			EnvironmentScope: "test",
+		}
+		smtc.expectedSecret = secretValue
+		smtc.refFind.Name = makeFindName("test.*")
+	}
+	setNoMatchingRegexpFindString := func(smtc *secretManagerTestCase) {
+		smtc.apiOutput = &gitlab.ProjectVariable{
+			Key:              "testkey",
+			Value:            "changedvalue",
+			EnvironmentScope: "test",
+		}
+		smtc.expectedSecret = ""
+		smtc.refFind.Name = makeFindName("foo.*")
+	}
+	setUnmatchedEnvironmentFindString := func(smtc *secretManagerTestCase) {
+		smtc.apiOutput = &gitlab.ProjectVariable{
+			Key:              "testkey",
+			Value:            "changedvalue",
+			EnvironmentScope: "prod",
+		}
+		smtc.expectedSecret = ""
+		smtc.refFind.Name = makeFindName("test.*")
+	}
+
+	cases := []*secretManagerTestCase{
+		makeValidSecretManagerGetAllTestCaseCustom(setMissingFindRegex),
+		makeValidSecretManagerGetAllTestCaseCustom(setUnsupportedFindTags),
+		makeValidSecretManagerGetAllTestCaseCustom(setUnsupportedFindPath),
+		makeValidSecretManagerGetAllTestCaseCustom(setMatchingSecretFindString),
+		makeValidSecretManagerGetAllTestCaseCustom(setNoMatchingRegexpFindString),
+		makeValidSecretManagerGetAllTestCaseCustom(setUnmatchedEnvironmentFindString),
+		makeValidSecretManagerGetAllTestCaseCustom(setAPIErr),
+		makeValidSecretManagerGetAllTestCaseCustom(setNilMockClient),
+	}
+
+	sm := Gitlab{}
+	sm.environment = "test"
+	for k, v := range cases {
+		sm.client = v.mockClient
+		out, err := sm.GetAllSecrets(context.Background(), *v.refFind)
+		if !ErrorContains(err, v.expectError) {
+			t.Errorf(defaultErrorMessage, k, err.Error(), v.expectError)
+		}
+		if v.expectError == "" && string(out[v.apiOutput.Key]) != v.expectedSecret {
+			t.Errorf("[%d] unexpected secret: expected %s, got %s", k, v.expectedSecret, string(out[v.apiOutput.Key]))
+		}
+	}
+}
+
 func TestValidate(t *testing.T) {
 	successCases := []*secretManagerTestCase{
 		makeValidSecretManagerTestCaseCustom(),
@@ -196,7 +380,7 @@ func TestValidate(t *testing.T) {
 		t.Logf("%+v", v)
 		validationResult, err := sm.Validate()
 		if !ErrorContains(err, v.expectError) {
-			t.Errorf("[%d], unexpected error: %s, expected: '%s'", k, err.Error(), v.expectError)
+			t.Errorf(defaultErrorMessage, k, err.Error(), v.expectError)
 		}
 		if validationResult != v.expectedValidationResult {
 			t.Errorf("[%d], unexpected validationResult: %s, expected: '%s'", k, validationResult, v.expectedValidationResult)
@@ -229,7 +413,7 @@ func TestGetSecretMap(t *testing.T) {
 		sm.client = v.mockClient
 		out, err := sm.GetSecretMap(context.Background(), *v.ref)
 		if !ErrorContains(err, v.expectError) {
-			t.Errorf("[%d] unexpected error: %s, expected: '%s'", k, err.Error(), v.expectError)
+			t.Errorf(defaultErrorMessage, k, err.Error(), v.expectError)
 		}
 		if err == nil && !reflect.DeepEqual(out, v.expectedData) {
 			t.Errorf("[%d] unexpected secret data: expected %#v, got %#v", k, v.expectedData, out)
@@ -269,7 +453,7 @@ func makeSecretStore(projectID, environment string, fn ...storeModifier) *esv1be
 
 func withAccessToken(name, key string, namespace *string) storeModifier {
 	return func(store *esv1beta1.SecretStore) *esv1beta1.SecretStore {
-		store.Spec.Provider.Gitlab.Auth.SecretRef.AccessToken = v1.SecretKeySelector{
+		store.Spec.Provider.Gitlab.Auth.SecretRef.AccessToken = esv1meta.SecretKeySelector{
 			Name:      name,
 			Key:       key,
 			Namespace: namespace,