feat: update Pulumi provider for GA (#3917)

Signed-off-by: Engin Diri <engin.diri@ediri.de>
Co-authored-by: Gergely Brautigam <182850+Skarlso@users.noreply.github.com>

Tiltfile

@@ -80,7 +80,7 @@ if settings.get('debug').get('enabled'):
 
 
 docker_build_with_restart(
-    'ghcr.io/external-secrets/external-secrets',
+    'oci.external-secrets.io/external-secrets/external-secrets',
     '.',
     dockerfile = dockerfile,
     entrypoint = entrypoint,

apis/externalsecrets/v1beta1/secretstore_pulumi_types.go

@@ -20,7 +20,7 @@ import (
 
 type PulumiProvider struct {
 	// APIURL is the URL of the Pulumi API.
-	// +kubebuilder:default="https://api.pulumi.com/api/preview"
+	// +kubebuilder:default="https://api.pulumi.com/api/esc"
 	APIURL string `json:"apiUrl,omitempty"`
 
 	// AccessToken is the access tokens to sign in to the Pulumi Cloud Console.
@@ -30,6 +30,8 @@ type PulumiProvider struct {
 	// To create a new organization, visit https://app.pulumi.com/ and click "New Organization".
 	Organization string `json:"organization"`
 
+	// Project is the name of the Pulumi ESC project the environment belongs to.
+	Project string `json:"project"`
 	// Environment are YAML documents composed of static key-value pairs, programmatic expressions,
 	// dynamically retrieved values from supported providers including all major clouds,
 	// and other Pulumi ESC environments.

config/crds/bases/external-secrets.io_clustersecretstores.yaml

@@ -3817,7 +3817,7 @@ spec:
                             type: object
                         type: object
                       apiUrl:
-                        default: https://api.pulumi.com/api/preview
+                        default: https://api.pulumi.com/api/esc
                         description: APIURL is the URL of the Pulumi API.
                         type: string
                       environment:
@@ -3832,10 +3832,15 @@ spec:
                           Organization are a space to collaborate on shared projects and stacks.
                           To create a new organization, visit https://app.pulumi.com/ and click "New Organization".
                         type: string
+                      project:
+                        description: Project is the name of the Pulumi ESC project
+                          the environment belongs to.
+                        type: string
                     required:
                     - accessToken
                     - environment
                     - organization
+                    - project
                     type: object
                   scaleway:
                     description: Scaleway

config/crds/bases/external-secrets.io_secretstores.yaml

@@ -3817,7 +3817,7 @@ spec:
                             type: object
                         type: object
                       apiUrl:
-                        default: https://api.pulumi.com/api/preview
+                        default: https://api.pulumi.com/api/esc
                         description: APIURL is the URL of the Pulumi API.
                         type: string
                       environment:
@@ -3832,10 +3832,15 @@ spec:
                           Organization are a space to collaborate on shared projects and stacks.
                           To create a new organization, visit https://app.pulumi.com/ and click "New Organization".
                         type: string
+                      project:
+                        description: Project is the name of the Pulumi ESC project
+                          the environment belongs to.
+                        type: string
                     required:
                     - accessToken
                     - environment
                     - organization
+                    - project
                     type: object
                   scaleway:
                     description: Scaleway

deploy/crds/bundle.yaml

@@ -4188,7 +4188,7 @@ spec:
                               type: object
                           type: object
                         apiUrl:
-                          default: https://api.pulumi.com/api/preview
+                          default: https://api.pulumi.com/api/esc
                           description: APIURL is the URL of the Pulumi API.
                           type: string
                         environment:
@@ -4203,10 +4203,14 @@ spec:
                             Organization are a space to collaborate on shared projects and stacks.
                             To create a new organization, visit https://app.pulumi.com/ and click "New Organization".
                           type: string
+                        project:
+                          description: Project is the name of the Pulumi ESC project the environment belongs to.
+                          type: string
                       required:
                         - accessToken
                         - environment
                         - organization
+                        - project
                       type: object
                     scaleway:
                       description: Scaleway
@@ -9968,7 +9972,7 @@ spec:
                               type: object
                           type: object
                         apiUrl:
-                          default: https://api.pulumi.com/api/preview
+                          default: https://api.pulumi.com/api/esc
                           description: APIURL is the URL of the Pulumi API.
                           type: string
                         environment:
@@ -9983,10 +9987,14 @@ spec:
                             Organization are a space to collaborate on shared projects and stacks.
                             To create a new organization, visit https://app.pulumi.com/ and click "New Organization".
                           type: string
+                        project:
+                          description: Project is the name of the Pulumi ESC project the environment belongs to.
+                          type: string
                       required:
                         - accessToken
                         - environment
                         - organization
+                        - project
                       type: object
                     scaleway:
                       description: Scaleway

docs/api/spec.md

@@ -6010,6 +6010,17 @@ To create a new organization, visit <a href="https://app.pulumi.com/">https://ap
 </tr>
 <tr>
 <td>
+<code>project</code></br>
+<em>
+string
+</em>
+</td>
+<td>
+<p>Project is the name of the Pulumi ESC project the environment belongs to.</p>
+</td>
+</tr>
+<tr>
+<td>
 <code>environment</code></br>
 <em>
 string

docs/provider/pulumi.md

@@ -2,13 +2,17 @@
 
 Sync environments, configs and secrets from [Pulumi ESC](https://www.pulumi.com/product/esc/) to Kubernetes using the External Secrets Operator.
 
+![Pulumi ESC](../pictures/pulumi-esc.png)
+
+More information about setting up [Pulumi](https://www.pulumi.com/) ESC can be found in the [Pulumi ESC documentation](https://www.pulumi.com/docs/esc/).
+
 ### Authentication
 
 Pulumi [Access Tokens](https://www.pulumi.com/docs/pulumi-cloud/access-management/access-tokens/) are recommended to access Pulumi ESC.
 
 ### Creating a SecretStore
 
-A Pulumi SecretStore can be created by specifying the `organization` and `environment` and referencing a Kubernetes secret containing the `accessToken`.
+A Pulumi `SecretStore` can be created by specifying the `organization`, `project` and `environment` and referencing a Kubernetes secret containing the `accessToken`.
 
 ```yaml
 apiVersion: external-secrets.io/v1beta1
@@ -19,6 +23,7 @@ spec:
   provider:
     pulumi:
       organization: <NAME_OF_THE_ORGANIZATION>
+      project: <NAME_OF_THE_PROJECT>
       environment: <NAME_OF_THE_ENVIRONMENT>
       accessToken:
         secretRef:
@@ -26,7 +31,29 @@ spec:
           key: <KEY_IN_KUBE_SECRET>
 ```
 
-If required, the API URL (`apiUrl`) can be customized as well. If not specified, the default value is `https://api.pulumi.com/api/preview`.
+If required, the API URL (`apiUrl`) can be customized as well. If not specified, the default value is `https://api.pulumi.com/api/esc`.
+
+### Creating a ClusterSecretStore
+
+Similarly, a `ClusterSecretStore` can be created by specifying the `namespace` and referencing a Kubernetes secret containing the `accessToken`.
+
+```yaml
+apiVersion: external-secrets.io/v1beta1
+kind: ClusterSecretStore
+metadata:
+  name: secret-store
+spec:
+  provider:
+    pulumi:
+      organization: <NAME_OF_THE_ORGANIZATION>
+      project: <NAME_OF_THE_PROJECT>
+      environment: <NAME_OF_THE_ENVIRONMENT>
+      accessToken:
+        secretRef:
+          name: <NAME_OF_KUBE_SECRET>
+          key: <KEY_IN_KUBE_SECRET>
+          namespace: <NAMESPACE>
+```
 
 ### Referencing Secrets
 

go.mod

@@ -89,7 +89,7 @@ require (
 	github.com/lestrrat-go/jwx/v2 v2.1.1
 	github.com/maxbrunsfeld/counterfeiter/v6 v6.9.0
 	github.com/passbolt/go-passbolt v0.7.1
-	github.com/pulumi/esc-sdk/sdk v0.9.2
+	github.com/pulumi/esc-sdk/sdk v0.10.0
 	github.com/scaleway/scaleway-sdk-go v1.0.0-beta.30
 	github.com/sethvargo/go-password v0.3.1
 	github.com/spf13/pflag v1.0.5

go.sum

@@ -596,8 +596,8 @@ github.com/prometheus/common v0.59.1 h1:LXb1quJHWm1P6wq/U824uxYi4Sg0oGvNeUm1z5dJ
 github.com/prometheus/common v0.59.1/go.mod h1:GpWM7dewqmVYcd7SmRaiWVe9SSqjf0UrwnYnpEZNuT0=
 github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
 github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
-github.com/pulumi/esc-sdk/sdk v0.9.2 h1:I+kKa7F/gY9lUiHEYuczHyrYB299CavG7rAB1yXybSw=
-github.com/pulumi/esc-sdk/sdk v0.9.2/go.mod h1:J6+8bCUJyLXvYOmTAc90/EhU1iUPr1Koo3NUnFzY78k=
+github.com/pulumi/esc-sdk/sdk v0.10.0 h1:tVZGVSVgSf/3UkKI3iC9E287eXw9VERvmdI4vN2BD4o=
+github.com/pulumi/esc-sdk/sdk v0.10.0/go.mod h1:J6+8bCUJyLXvYOmTAc90/EhU1iUPr1Koo3NUnFzY78k=
 github.com/r3labs/diff v0.0.0-20191120142937-b4ed99a31f5a h1:2v4Ipjxa3sh+xn6GvtgrMub2ci4ZLQMvTaYIba2lfdc=
 github.com/r3labs/diff v0.0.0-20191120142937-b4ed99a31f5a/go.mod h1:ozniNEFS3j1qCwHKdvraMn1WJOsUxHd7lYfukEIS4cs=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=

pkg/provider/pulumi/provider.go

@@ -39,6 +39,7 @@ const (
 	errNoStoreTypeOrWrongStoreType   = "no store type or wrong store type"
 	errOrganizationIsRequired        = "organization is required"
 	errEnvironmentIsRequired         = "environment is required"
+	errProjectIsRequired             = "project is required"
 	errSecretRefNameIsRequired       = "secretRef.name is required"
 	errSecretRefKeyIsRequired        = "secretRef.key is required"
 )
@@ -52,7 +53,6 @@ func (p *Provider) NewClient(ctx context.Context, store esv1beta1.GenericStore,
 	if storeKind == esv1beta1.ClusterSecretStoreKind && doesConfigDependOnNamespace(cfg) {
 		return nil, errors.New(errClusterStoreRequiresNamespace)
 	}
-
 	accessToken, err := loadAccessTokenSecret(ctx, cfg.AccessToken, kube, storeKind, namespace)
 	if err != nil {
 		return nil, err
@@ -69,6 +69,7 @@ func (p *Provider) NewClient(ctx context.Context, store esv1beta1.GenericStore,
 	return &client{
 		escClient:    *escClient,
 		authCtx:      authCtx,
+		project:      cfg.Project,
 		environment:  cfg.Environment,
 		organization: cfg.Organization,
 	}, nil
@@ -100,7 +101,7 @@ func getConfig(store esv1beta1.GenericStore) (*esv1beta1.PulumiProvider, error)
 	cfg := spec.Provider.Pulumi
 
 	if cfg.APIURL == "" {
-		cfg.APIURL = "https://api.pulumi.com/api/preview"
+		cfg.APIURL = "https://api.pulumi.com/api/esc"
 	}
 
 	if cfg.Organization == "" {
@@ -109,6 +110,9 @@ func getConfig(store esv1beta1.GenericStore) (*esv1beta1.PulumiProvider, error)
 	if cfg.Environment == "" {
 		return nil, errors.New(errEnvironmentIsRequired)
 	}
+	if cfg.Project == "" {
+		return nil, errors.New(errProjectIsRequired)
+	}
 	err := validateStoreSecretRef(store, cfg.AccessToken)
 	if err != nil {
 		return nil, err

pkg/provider/pulumi/pulumi.go

@@ -31,6 +31,7 @@ import (
 type client struct {
 	escClient    esc.EscClient
 	authCtx      context.Context
+	project      string
 	environment  string
 	organization string
 }
@@ -49,12 +50,11 @@ const (
 var _ esv1beta1.SecretsClient = &client{}
 
 func (c *client) GetSecret(_ context.Context, ref esv1beta1.ExternalSecretDataRemoteRef) ([]byte, error) {
-	env, err := c.escClient.OpenEnvironment(c.authCtx, c.organization, c.environment)
+	env, err := c.escClient.OpenEnvironment(c.authCtx, c.organization, c.project, c.environment)
 	if err != nil {
 		return nil, err
 	}
-
-	value, _, err := c.escClient.ReadEnvironmentProperty(c.authCtx, c.organization, c.environment, env.GetId(), ref.Key)
+	value, _, err := c.escClient.ReadEnvironmentProperty(c.authCtx, c.organization, c.project, c.environment, env.GetId(), ref.Key)
 	if err != nil {
 		return nil, err
 	}
@@ -97,7 +97,7 @@ func (c *client) PushSecret(_ context.Context, secret *corev1.Secret, data esv1b
 			},
 		},
 	}
-	_, oldValues, err := c.escClient.OpenAndReadEnvironment(c.authCtx, c.organization, c.environment)
+	_, oldValues, err := c.escClient.OpenAndReadEnvironment(c.authCtx, c.organization, c.project, c.environment)
 	if err != nil {
 		return fmt.Errorf(errReadEnvironment, err)
 	}
@@ -105,7 +105,7 @@ func (c *client) PushSecret(_ context.Context, secret *corev1.Secret, data esv1b
 	if err := mergo.Merge(&updatePayload.Values.AdditionalProperties, oldValues); err != nil {
 		return fmt.Errorf(errPushSecrets, err)
 	}
-	_, err = c.escClient.UpdateEnvironment(c.authCtx, c.organization, c.environment, updatePayload)
+	_, err = c.escClient.UpdateEnvironment(c.authCtx, c.organization, c.environment, c.project, updatePayload)
 	if err != nil {
 		return fmt.Errorf(errPushSecrets, err)
 	}
@@ -144,11 +144,11 @@ func GetMapFromInterface(i interface{}) (map[string][]byte, error) {
 }
 
 func (c *client) GetSecretMap(_ context.Context, ref esv1beta1.ExternalSecretDataRemoteRef) (map[string][]byte, error) {
-	env, err := c.escClient.OpenEnvironment(c.authCtx, c.organization, c.environment)
+	env, err := c.escClient.OpenEnvironment(c.authCtx, c.organization, c.project, c.environment)
 	if err != nil {
 		return nil, err
 	}
-	value, _, err := c.escClient.ReadEnvironmentProperty(c.authCtx, c.organization, c.environment, env.GetId(), ref.Key)
+	value, _, err := c.escClient.ReadEnvironmentProperty(c.authCtx, c.organization, c.project, c.environment, env.GetId(), ref.Key)
 	if err != nil {
 		return nil, err
 	}

pkg/provider/pulumi/pulumi_test.go

@@ -16,6 +16,7 @@ package pulumi
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"net/http"
 	"net/http/httptest"
 	"reflect"
@@ -38,7 +39,7 @@ func newTestClient(t *testing.T, _, pattern string, handler func(w http.Response
 	mux := http.NewServeMux()
 
 	mux.HandleFunc(pattern, handler)
-	mux.HandleFunc("/environments/foo/bar/open/", func(w http.ResponseWriter, r *http.Request) {
+	mux.HandleFunc("/environments/foo/default/bar/open/", func(w http.ResponseWriter, r *http.Request) {
 		r.Header.Add(contentType, contentTypeValue)
 		w.Header().Add(contentType, contentTypeValue)
 		w.WriteHeader(http.StatusOK)
@@ -65,6 +66,7 @@ func newTestClient(t *testing.T, _, pattern string, handler func(w http.Response
 		authCtx:      ctx,
 		organization: "foo",
 		environment:  "bar",
+		project:      "default",
 	}
 }
 
@@ -73,7 +75,7 @@ func TestGetSecret(t *testing.T) {
 		"b": "world",
 	}
 
-	client := newTestClient(t, http.MethodGet, "/environments/foo/bar/open/session-id", func(w http.ResponseWriter, r *http.Request) {
+	client := newTestClient(t, http.MethodGet, "/environments/foo/default/bar/open/session-id", func(w http.ResponseWriter, r *http.Request) {
 		r.Header.Add(contentType, contentTypeValue)
 		w.Header().Add(contentType, contentTypeValue)
 		err := json.NewEncoder(w).Encode(esc.NewValue(testmap, esc.Trace{}))
@@ -342,13 +344,14 @@ func TestGetSecretMap(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			p := newTestClient(t, http.MethodGet, "/environments/foo/bar/open/session-id", func(w http.ResponseWriter, r *http.Request) {
+			p := newTestClient(t, http.MethodGet, "/environments/foo/default/bar/open/session-id", func(w http.ResponseWriter, r *http.Request) {
 				r.Header.Add(contentType, contentTypeValue)
 				w.Header().Add(contentType, contentTypeValue)
 				err2 := json.NewEncoder(w).Encode(esc.NewValue(tt.input, esc.Trace{}))
 				require.NoError(t, err2)
 			})
 			got, err := p.GetSecretMap(context.TODO(), tt.ref)
+			fmt.Print(got)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("ProviderPulumi.GetSecretMap() error = %v, wantErr %v", err, tt.wantErr)
 				return