fix(template): fix templateFrom tpl execution

docs/api-externalsecret.md

@@ -9,7 +9,7 @@ be transformed and saved as a `Kind=Secret`:
 
 ## Template
 
-When the controller reconciles the `ExternalSecret` it will use the `spec.template` as a blueprint to construct a new `Kind=Secret`. You can use golang templates to define the blueprint and use template functions to transform secret values. See [advanced templating](guides-templating.md) for details.
+When the controller reconciles the `ExternalSecret` it will use the `spec.template` as a blueprint to construct a new `Kind=Secret`. You can use golang templates to define the blueprint and use template functions to transform secret values. You can also pull in `ConfigMaps` that contain golang-template data using `templateFrom`. See [advanced templating](guides-templating.md) for details.
 
 ## Update Behavior
 

docs/guides-templating.md

@@ -12,6 +12,14 @@ You can also use pre-defined functions to extract data from your secrets. Here:
 {% include 'pkcs12-template-external-secret.yaml' %}
 ```
 
+### TemplateFrom
+
+You do not have to define your templates inline in an ExternalSecret but you can pull `ConfigMaps` or other Secrets that contain a template. Consider the following example:
+
+``` yaml
+{% include 'template-from-secret.yaml' %}
+```
+
 ## Helper functions
 We provide a bunch of convenience functions that help you transform your secrets. A secret value is a `[]byte`.
 

docs/snippets/template-from-secret.yaml

@@ -0,0 +1,40 @@
+{% raw %}
+# define your tempalte in a config map
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: grafana-config-tpl
+data:
+  config.yaml: |
+    datasources:
+      - name: Graphite
+        type: graphite
+        access: proxy
+        url: http://localhost:8080
+        password: "{{ .password | toString }}" # <-- convert []byte to string
+        user: "{{ .user | toString }}"         # <-- convert []byte to string
+---
+apiVersion: external-secrets.io/v1alpha1
+kind: ExternalSecret
+metadata:
+  name: my-template-example
+spec:
+  # ...
+  target:
+    name: secret-to-be-created
+    template:
+      templateFrom:
+      - configMap:
+          # name of the configmap to pull in
+          name: grafana-config-tpl
+          # here you define the keys that should be used as template
+          items:
+          - key: config.yaml
+  data:
+  - secretKey: user
+    remoteRef:
+      key: /grafana/user
+  - secretKey: password
+    remoteRef:
+      key: /grafana/password
+{% endraw %}

pkg/controllers/externalsecret/externalsecret_controller.go

@@ -143,25 +143,36 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 		if err != nil {
 			return fmt.Errorf("could not set ExternalSecret controller reference: %w", err)
 		}
-		mergeTemplate(secret, externalSecret)
-		// prepare templateFrom data
-		templateData, err := r.getTemplateData(ctx, &externalSecret)
+		mergeMetadata(secret, externalSecret)
+		var tplMap map[string][]byte
+		var dataMap map[string][]byte
+
+		// get data
+		dataMap, err = r.getProviderSecretData(ctx, secretClient, &externalSecret)
 		if err != nil {
-			return fmt.Errorf("error fetching templateFrom data: %w", err)
+			return fmt.Errorf("could not get secret data from provider: %w", err)
 		}
-		log.Info("found template data", "tpl_data", templateData)
-		for k, v := range templateData {
-			secret.Data[k] = v
+
+		// no template: copy data and return
+		if externalSecret.Spec.Target.Template == nil {
+			for k, v := range dataMap {
+				secret.Data[k] = v
+			}
+			return nil
 		}
-		// overwrite provider data
-		data, err := r.getProviderSecretData(ctx, secretClient, &externalSecret)
+
+		// template: fetch & execute templates
+		tplMap, err = r.getTemplateData(ctx, &externalSecret)
 		if err != nil {
-			return fmt.Errorf("could not get secret data from provider: %w", err)
+			return fmt.Errorf("error fetching templateFrom data: %w", err)
 		}
-		for k, v := range data {
-			secret.Data[k] = v
+		// override templateFrom data with template data
+		for k, v := range externalSecret.Spec.Target.Template.Data {
+			tplMap[k] = []byte(v)
 		}
-		err = template.Execute(externalSecret.Spec.Target.Template, secret, data)
+
+		log.V(1).Info("found template data", "tpl_data", tplMap)
+		err = template.Execute(tplMap, dataMap, secret)
 		if err != nil {
 			return fmt.Errorf("could not execute template: %w", err)
 		}
@@ -230,7 +241,7 @@ func shouldRefresh(es esv1alpha1.ExternalSecret) bool {
 
 // we do not want to force-override the label/annotations
 // and only copy the necessary key/value pairs.
-func mergeTemplate(secret *v1.Secret, externalSecret esv1alpha1.ExternalSecret) {
+func mergeMetadata(secret *v1.Secret, externalSecret esv1alpha1.ExternalSecret) {
 	if secret.ObjectMeta.Labels == nil {
 		secret.ObjectMeta.Labels = make(map[string]string)
 	}

pkg/controllers/externalsecret/externalsecret_controller_test.go

@@ -233,7 +233,7 @@ var _ = Describe("ExternalSecret controller", func() {
 		const tplStaticVal = "tplstaticvalue"
 		const tplFromCMName = "template-cm"
 		const tplFromKey = "tpl-from-key"
-		const tplFromVal = "tpl-from-value"
+		const tplFromVal = "tpl-from-value: {{ .targetProperty | toString }} // {{ .bar | toString }}"
 		Expect(k8sClient.Create(context.Background(), &v1.ConfigMap{
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      "template-cm",
@@ -282,7 +282,7 @@ var _ = Describe("ExternalSecret controller", func() {
 			Expect(string(secret.Data[targetProp])).To(Equal(expectedSecretVal))
 			Expect(string(secret.Data[tplStaticKey])).To(Equal(tplStaticVal))
 			Expect(string(secret.Data["bar"])).To(Equal("value from map: map-bar-value"))
-			Expect(string(secret.Data[tplFromKey])).To(Equal(tplFromVal))
+			Expect(string(secret.Data[tplFromKey])).To(Equal("tpl-from-value: someValue // map-bar-value"))
 		}
 	}
 

pkg/template/template.go

@@ -27,8 +27,6 @@ import (
 	"github.com/youmark/pkcs8"
 	"golang.org/x/crypto/pkcs12"
 	corev1 "k8s.io/api/core/v1"
-
-	esv1alpha1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1alpha1"
 )
 
 var tplFuncs = tpl.FuncMap{
@@ -67,12 +65,12 @@ const (
 )
 
 // Execute renders the secret data as template. If an error occurs processing is stopped immediately.
-func Execute(template *esv1alpha1.ExternalSecretTemplate, secret *corev1.Secret, data map[string][]byte) error {
-	if template == nil {
+func Execute(tpl, data map[string][]byte, secret *corev1.Secret) error {
+	if tpl == nil {
 		return nil
 	}
-	for k, v := range template.Data {
-		val, err := execute(k, v, data)
+	for k, v := range tpl {
+		val, err := execute(k, string(v), data)
 		if err != nil {
 			return fmt.Errorf(errExecute, k, err)
 		}

pkg/template/template_test.go

@@ -19,8 +19,6 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	corev1 "k8s.io/api/core/v1"
-
-	esv1alpha1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1alpha1"
 )
 
 const (
@@ -136,7 +134,7 @@ KfMtQkBmCFTNk3fOtz3sgTiv0OHbokplsICEc4tUT5RWU0frwAjJT4Pk
 func TestExecute(t *testing.T) {
 	tbl := []struct {
 		name        string
-		tpl         *esv1alpha1.ExternalSecretTemplate
+		tpl         map[string][]byte
 		data        map[string][]byte
 		expetedData map[string][]byte
 		expErr      string
@@ -148,9 +146,9 @@ func TestExecute(t *testing.T) {
 		},
 		{
 			name: "base64decode func",
-			tpl: &esv1alpha1.ExternalSecretTemplate{Data: map[string]string{
-				"foo": "{{ .secret | base64decode | toString }}",
-			}},
+			tpl: map[string][]byte{
+				"foo": []byte("{{ .secret | base64decode | toString }}"),
+			},
 			data: map[string][]byte{
 				"secret": []byte("MTIzNA=="),
 			},
@@ -160,9 +158,9 @@ func TestExecute(t *testing.T) {
 		},
 		{
 			name: "fromJSON func",
-			tpl: &esv1alpha1.ExternalSecretTemplate{Data: map[string]string{
-				"foo": "{{ $var := .secret | fromJSON }}{{ $var.foo }}",
-			}},
+			tpl: map[string][]byte{
+				"foo": []byte("{{ $var := .secret | fromJSON }}{{ $var.foo }}"),
+			},
 			data: map[string][]byte{
 				"secret": []byte(`{"foo": "bar"}`),
 			},
@@ -172,9 +170,9 @@ func TestExecute(t *testing.T) {
 		},
 		{
 			name: "from & toJSON func",
-			tpl: &esv1alpha1.ExternalSecretTemplate{Data: map[string]string{
-				"foo": "{{ $var := .secret | fromJSON }}{{ $var.foo | toJSON }}",
-			}},
+			tpl: map[string][]byte{
+				"foo": []byte("{{ $var := .secret | fromJSON }}{{ $var.foo | toJSON }}"),
+			},
 			data: map[string][]byte{
 				"secret": []byte(`{"foo": {"baz":"bang"}}`),
 			},
@@ -184,16 +182,16 @@ func TestExecute(t *testing.T) {
 		},
 		{
 			name: "multiline template",
-			tpl: &esv1alpha1.ExternalSecretTemplate{Data: map[string]string{
-				"cfg": `
+			tpl: map[string][]byte{
+				"cfg": []byte(`
 		datasources:
 		- name: Graphite
 			type: graphite
 			access: proxy
 			url: http://localhost:8080
 			password: "{{ .password | toString }}"
-			user: "{{ .user | toString }}"`,
-			}},
+			user: "{{ .user | toString }}"`),
+			},
 			data: map[string][]byte{
 				"user":     []byte(`foobert`),
 				"password": []byte("harharhar"),
@@ -211,9 +209,9 @@ func TestExecute(t *testing.T) {
 		},
 		{
 			name: "base64 pipeline",
-			tpl: &esv1alpha1.ExternalSecretTemplate{Data: map[string]string{
-				"foo": `{{ "123412341234" | toBytes | base64encode | base64decode | toString }}`,
-			}},
+			tpl: map[string][]byte{
+				"foo": []byte(`{{ "123412341234" | toBytes | base64encode | base64decode | toString }}`),
+			},
 			data: map[string][]byte{},
 			expetedData: map[string][]byte{
 				"foo": []byte("123412341234"),
@@ -221,10 +219,10 @@ func TestExecute(t *testing.T) {
 		},
 		{
 			name: "base64 pkcs12 extract",
-			tpl: &esv1alpha1.ExternalSecretTemplate{Data: map[string]string{
-				"key":  `{{ .secret | base64decode | pkcs12key | pemPrivateKey }}`,
-				"cert": `{{ .secret | base64decode | pkcs12cert | pemCertificate }}`,
-			}},
+			tpl: map[string][]byte{
+				"key":  []byte(`{{ .secret | base64decode | pkcs12key | pemPrivateKey }}`),
+				"cert": []byte(`{{ .secret | base64decode | pkcs12cert | pemCertificate }}`),
+			},
 			data: map[string][]byte{
 				"secret": []byte(pkcs12ContentNoPass),
 			},
@@ -235,10 +233,10 @@ func TestExecute(t *testing.T) {
 		},
 		{
 			name: "base64 pkcs12 extract with password",
-			tpl: &esv1alpha1.ExternalSecretTemplate{Data: map[string]string{
-				"key":  `{{ .secret | base64decode | pkcs12keyPass "123456" | pemPrivateKey }}`,
-				"cert": `{{ .secret | base64decode | pkcs12certPass "123456" | pemCertificate }}`,
-			}},
+			tpl: map[string][]byte{
+				"key":  []byte(`{{ .secret | base64decode | pkcs12keyPass "123456" | pemPrivateKey }}`),
+				"cert": []byte(`{{ .secret | base64decode | pkcs12certPass "123456" | pemCertificate }}`),
+			},
 			data: map[string][]byte{
 				"secret": []byte(pkcs12ContentWithPass),
 			},
@@ -249,9 +247,9 @@ func TestExecute(t *testing.T) {
 		},
 		{
 			name: "base64 decode error",
-			tpl: &esv1alpha1.ExternalSecretTemplate{Data: map[string]string{
-				"key": `{{ .example | base64decode }}`,
-			}},
+			tpl: map[string][]byte{
+				"key": []byte(`{{ .example | base64decode }}`),
+			},
 			data: map[string][]byte{
 				"example": []byte("iam_no_base64"),
 			},
@@ -259,9 +257,9 @@ func TestExecute(t *testing.T) {
 		},
 		{
 			name: "pkcs12 key wrong password",
-			tpl: &esv1alpha1.ExternalSecretTemplate{Data: map[string]string{
-				"key": `{{ .secret | base64decode | pkcs12keyPass "wrong" | pemPrivateKey }}`,
-			}},
+			tpl: map[string][]byte{
+				"key": []byte(`{{ .secret | base64decode | pkcs12keyPass "wrong" | pemPrivateKey }}`),
+			},
 			data: map[string][]byte{
 				"secret": []byte(pkcs12ContentWithPass),
 			},
@@ -269,9 +267,9 @@ func TestExecute(t *testing.T) {
 		},
 		{
 			name: "pkcs12 cert wrong password",
-			tpl: &esv1alpha1.ExternalSecretTemplate{Data: map[string]string{
-				"cert": `{{ .secret | base64decode | pkcs12certPass "wrong" | pemCertificate }}`,
-			}},
+			tpl: map[string][]byte{
+				"cert": []byte(`{{ .secret | base64decode | pkcs12certPass "wrong" | pemCertificate }}`),
+			},
 			data: map[string][]byte{
 				"secret": []byte(pkcs12ContentWithPass),
 			},
@@ -279,25 +277,25 @@ func TestExecute(t *testing.T) {
 		},
 		{
 			name: "fromJSON error",
-			tpl: &esv1alpha1.ExternalSecretTemplate{Data: map[string]string{
-				"key": `{{ "{ # no json # }" | toBytes | fromJSON }}`,
-			}},
+			tpl: map[string][]byte{
+				"key": []byte(`{{ "{ # no json # }" | toBytes | fromJSON }}`),
+			},
 			data:   map[string][]byte{},
 			expErr: "unable to unmarshal json",
 		},
 		{
 			name: "template syntax error",
-			tpl: &esv1alpha1.ExternalSecretTemplate{Data: map[string]string{
-				"key": `{{ #xx }}`,
-			}},
+			tpl: map[string][]byte{
+				"key": []byte(`{{ #xx }}`),
+			},
 			data:   map[string][]byte{},
 			expErr: "unable to parse template",
 		},
 		{
 			name: "jwk rsa pub pem",
-			tpl: &esv1alpha1.ExternalSecretTemplate{Data: map[string]string{
-				"fn": `{{ .secret | jwkPublicKeyPem }}`,
-			}},
+			tpl: map[string][]byte{
+				"fn": []byte(`{{ .secret | jwkPublicKeyPem }}`),
+			},
 			data: map[string][]byte{
 				"secret": []byte(jwkPubRSA),
 			},
@@ -307,9 +305,9 @@ func TestExecute(t *testing.T) {
 		},
 		{
 			name: "jwk rsa priv pem",
-			tpl: &esv1alpha1.ExternalSecretTemplate{Data: map[string]string{
-				"fn": `{{ .secret | jwkPrivateKeyPem }}`,
-			}},
+			tpl: map[string][]byte{
+				"fn": []byte(`{{ .secret | jwkPrivateKeyPem }}`),
+			},
 			data: map[string][]byte{
 				"secret": []byte(jwkPrivRSA),
 			},
@@ -319,9 +317,9 @@ func TestExecute(t *testing.T) {
 		},
 		{
 			name: "jwk ecdsa pub pem",
-			tpl: &esv1alpha1.ExternalSecretTemplate{Data: map[string]string{
-				"fn": `{{ .secret | jwkPublicKeyPem }}`,
-			}},
+			tpl: map[string][]byte{
+				"fn": []byte(`{{ .secret | jwkPublicKeyPem }}`),
+			},
 			data: map[string][]byte{
 				"secret": []byte(jwkPubEC),
 			},
@@ -331,9 +329,9 @@ func TestExecute(t *testing.T) {
 		},
 		{
 			name: "jwk ecdsa priv pem",
-			tpl: &esv1alpha1.ExternalSecretTemplate{Data: map[string]string{
-				"fn": `{{ .secret | jwkPrivateKeyPem }}`,
-			}},
+			tpl: map[string][]byte{
+				"fn": []byte(`{{ .secret | jwkPrivateKeyPem }}`),
+			},
 			data: map[string][]byte{
 				"secret": []byte(jwkPrivEC),
 			},
@@ -349,7 +347,7 @@ func TestExecute(t *testing.T) {
 			sec := &corev1.Secret{
 				Data: make(map[string][]byte),
 			}
-			err := Execute(row.tpl, sec, row.data)
+			err := Execute(row.tpl, row.data, sec)
 			if !ErrorContains(err, row.expErr) {
 				t.Errorf("unexpected error: %s, expected: %s", err, row.expErr)
 			}