fix: template data should be string, fix update mechanics

apis/externalsecrets/v1alpha1/externalsecret_types.go

@@ -63,7 +63,7 @@ type ExternalSecretTemplate struct {
 	Metadata ExternalSecretTemplateMetadata `json:"metadata,omitempty"`
 
 	// +optional
-	Data map[string][]byte `json:"data,omitempty"`
+	Data map[string]string `json:"data,omitempty"`
 }
 
 // ExternalSecretTarget defines the Kubernetes Secret to be created

apis/externalsecrets/v1alpha1/zz_generated.deepcopy.go

@@ -322,17 +322,9 @@ func (in *ExternalSecretTemplate) DeepCopyInto(out *ExternalSecretTemplate) {
 	in.Metadata.DeepCopyInto(&out.Metadata)
 	if in.Data != nil {
 		in, out := &in.Data, &out.Data
-		*out = make(map[string][]byte, len(*in))
+		*out = make(map[string]string, len(*in))
 		for key, val := range *in {
-			var outVal []byte
-			if val == nil {
-				(*out)[key] = nil
-			} else {
-				in, out := &val, &outVal
-				*out = make([]byte, len(*in))
-				copy(*out, *in)
-			}
-			(*out)[key] = outVal
+			(*out)[key] = val
 		}
 	}
 }

deploy/crds/external-secrets.io_externalsecrets.yaml

@@ -139,7 +139,6 @@ spec:
                     properties:
                       data:
                         additionalProperties:
-                          format: byte
                           type: string
                         type: object
                       metadata:

hack/api-docs/Makefile

@@ -64,7 +64,7 @@ clean:
 # serve runs mkdocs as a local webserver for interactive development.
 # This will serve the live copy of the docs on 127.0.0.1:8000.
 .PHONY: serve
-serve:
+serve: build
 	$(DOCKER) run \
 		-it \
 		--sig-proxy=true \

pkg/controllers/externalsecret/externalsecret_controller.go

@@ -22,6 +22,7 @@ import (
 	"github.com/go-logr/logr"
 	"github.com/prometheus/client_golang/prometheus"
 	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
@@ -59,10 +60,13 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 	var externalSecret esv1alpha1.ExternalSecret
 
 	err := r.Get(ctx, req.NamespacedName, &externalSecret)
-	if err != nil {
+	if apierrors.IsNotFound(err) {
+		syncCallsTotal.With(syncCallsMetricLabels).Inc()
+		return ctrl.Result{}, nil
+	} else if err != nil {
 		log.Error(err, "could not get ExternalSecret")
 		syncCallsError.With(syncCallsMetricLabels).Inc()
-		return ctrl.Result{}, client.IgnoreNotFound(err)
+		return ctrl.Result{}, nil
 	}
 
 	store, err := r.getStore(ctx, &externalSecret)
@@ -100,12 +104,19 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 		syncCallsError.With(syncCallsMetricLabels).Inc()
 		return ctrl.Result{RequeueAfter: requeueAfter}, nil
 	}
-	secret := defaultSecret(externalSecret)
+	secret := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      externalSecret.Spec.Target.Name,
+			Namespace: externalSecret.Namespace,
+		},
+		Data: make(map[string][]byte),
+	}
 	_, err = ctrl.CreateOrUpdate(ctx, r.Client, secret, func() error {
 		err = controllerutil.SetControllerReference(&externalSecret, &secret.ObjectMeta, r.Scheme)
 		if err != nil {
 			return fmt.Errorf("could not set ExternalSecret controller reference: %w", err)
 		}
+		mergeTemplate(secret, externalSecret)
 		data, err := r.getProviderSecretData(ctx, secretClient, &externalSecret)
 		if err != nil {
 			return fmt.Errorf("could not get secret data from provider: %w", err)
@@ -114,7 +125,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 		for k, v := range data {
 			secret.Data[k] = v
 		}
-		err = template.Execute(secret, data)
+		err = template.Execute(externalSecret.Spec.Target.Template, secret, data)
 		if err != nil {
 			return fmt.Errorf("could not execute template: %w", err)
 		}
@@ -160,25 +171,30 @@ func shouldProcessStore(store esv1alpha1.GenericStore, class string) bool {
 	return false
 }
 
-func defaultSecret(es esv1alpha1.ExternalSecret) *corev1.Secret {
-	secret := &corev1.Secret{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:        es.Spec.Target.Name,
-			Namespace:   es.Namespace,
-			Labels:      es.Labels,
-			Annotations: es.Annotations,
-		},
-		Data: make(map[string][]byte),
+// we do not want to force-override the label/annotations
+// and only copy the necessary key/value pairs.
+func mergeTemplate(secret *corev1.Secret, externalSecret esv1alpha1.ExternalSecret) {
+	if secret.ObjectMeta.Labels == nil {
+		secret.ObjectMeta.Labels = make(map[string]string)
 	}
-
-	if es.Spec.Target.Template != nil {
-		secret.Type = es.Spec.Target.Template.Type
-		secret.Data = es.Spec.Target.Template.Data
-		secret.ObjectMeta.Labels = es.Spec.Target.Template.Metadata.Labels
-		secret.ObjectMeta.Annotations = es.Spec.Target.Template.Metadata.Annotations
+	if secret.ObjectMeta.Annotations == nil {
+		secret.ObjectMeta.Annotations = make(map[string]string)
+	}
+	if externalSecret.Spec.Target.Template == nil {
+		mergeMap(secret.ObjectMeta.Labels, externalSecret.ObjectMeta.Labels)
+		mergeMap(secret.ObjectMeta.Annotations, externalSecret.ObjectMeta.Annotations)
+		return
 	}
+	// if template is defined: use those labels/annotations
+	secret.Type = externalSecret.Spec.Target.Template.Type
+	mergeMap(secret.ObjectMeta.Labels, externalSecret.Spec.Target.Template.Metadata.Labels)
+	mergeMap(secret.ObjectMeta.Annotations, externalSecret.Spec.Target.Template.Metadata.Annotations)
+}
 
-	return secret
+func mergeMap(dest, src map[string]string) {
+	for k, v := range src {
+		dest[k] = v
+	}
 }
 
 func (r *Reconciler) getStore(ctx context.Context, externalSecret *esv1alpha1.ExternalSecret) (esv1alpha1.GenericStore, error) {

pkg/controllers/externalsecret/externalsecret_controller_test.go

@@ -264,8 +264,8 @@ var _ = Describe("ExternalSecret controller", func() {
 									"hihi": "ga",
 								},
 							},
-							Data: map[string][]byte{
-								templateSecretKey: []byte(templateSecretVal),
+							Data: map[string]string{
+								templateSecretKey: templateSecretVal,
 							},
 						},
 					},

pkg/template/template.go

@@ -25,6 +25,8 @@ import (
 	"github.com/youmark/pkcs8"
 	"golang.org/x/crypto/pkcs12"
 	corev1 "k8s.io/api/core/v1"
+
+	esv1alpha1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1alpha1"
 )
 
 var tplFuncs = tpl.FuncMap{
@@ -60,24 +62,35 @@ const (
 )
 
 // Execute renders the secret data as template. If an error occurs processing is stopped immediately.
-func Execute(secret *corev1.Secret, data map[string][]byte) error {
-	for k, v := range secret.Data {
-		t, err := tpl.New(k).
-			Funcs(tplFuncs).
-			Parse(string(v))
-		if err != nil {
-			return fmt.Errorf(errParse, k, err)
-		}
-		buf := bytes.NewBuffer(nil)
-		err = t.Execute(buf, data)
+func Execute(template *esv1alpha1.ExternalSecretTemplate, secret *corev1.Secret, data map[string][]byte) error {
+	if template == nil {
+		return nil
+	}
+	for k, v := range template.Data {
+		val, err := execute(k, v, data)
 		if err != nil {
 			return fmt.Errorf(errExecute, k, err)
 		}
-		secret.Data[k] = buf.Bytes()
+		secret.Data[k] = val
 	}
 	return nil
 }
 
+func execute(k, val string, data map[string][]byte) ([]byte, error) {
+	t, err := tpl.New(k).
+		Funcs(tplFuncs).
+		Parse(val)
+	if err != nil {
+		return nil, fmt.Errorf(errParse, k, err)
+	}
+	buf := bytes.NewBuffer(nil)
+	err = t.Execute(buf, data)
+	if err != nil {
+		return nil, fmt.Errorf(errExecute, k, err)
+	}
+	return buf.Bytes(), nil
+}
+
 func pkcs12keyPass(pass string, input []byte) ([]byte, error) {
 	key, _, err := pkcs12.Decode(input, pass)
 	if err != nil {

pkg/template/template_test.go

@@ -19,6 +19,8 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	corev1 "k8s.io/api/core/v1"
+
+	esv1alpha1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1alpha1"
 )
 
 const (
@@ -77,70 +79,70 @@ BixHvI/EJ8YK3ta5WdJWKC6hnA==
 
 func TestExecute(t *testing.T) {
 	tbl := []struct {
-		name      string
-		secret    map[string][]byte
-		data      map[string][]byte
-		outSecret map[string][]byte
-		expErr    string
+		name        string
+		tpl         *esv1alpha1.ExternalSecretTemplate
+		data        map[string][]byte
+		expetedData map[string][]byte
+		expErr      string
 	}{
 		{
-			name:   "test empty",
-			secret: nil,
-			data:   nil,
+			name: "test empty",
+			tpl:  nil,
+			data: nil,
 		},
 		{
 			name: "base64decode func",
-			secret: map[string][]byte{
-				"foo": []byte("{{ .secret | base64decode | toString }}"),
-			},
+			tpl: &esv1alpha1.ExternalSecretTemplate{Data: map[string]string{
+				"foo": "{{ .secret | base64decode | toString }}",
+			}},
 			data: map[string][]byte{
 				"secret": []byte("MTIzNA=="),
 			},
-			outSecret: map[string][]byte{
+			expetedData: map[string][]byte{
 				"foo": []byte("1234"),
 			},
 		},
 		{
 			name: "fromJSON func",
-			secret: map[string][]byte{
-				"foo": []byte("{{ $var := .secret | fromJSON }}{{ $var.foo }}"),
-			},
+			tpl: &esv1alpha1.ExternalSecretTemplate{Data: map[string]string{
+				"foo": "{{ $var := .secret | fromJSON }}{{ $var.foo }}",
+			}},
 			data: map[string][]byte{
 				"secret": []byte(`{"foo": "bar"}`),
 			},
-			outSecret: map[string][]byte{
+			expetedData: map[string][]byte{
 				"foo": []byte("bar"),
 			},
 		},
 		{
 			name: "from & toJSON func",
-			secret: map[string][]byte{
-				"foo": []byte("{{ $var := .secret | fromJSON }}{{ $var.foo | toJSON }}"),
-			},
+			tpl: &esv1alpha1.ExternalSecretTemplate{Data: map[string]string{
+				"foo": "{{ $var := .secret | fromJSON }}{{ $var.foo | toJSON }}",
+			}},
 			data: map[string][]byte{
 				"secret": []byte(`{"foo": {"baz":"bang"}}`),
 			},
-			outSecret: map[string][]byte{
+			expetedData: map[string][]byte{
 				"foo": []byte(`{"baz":"bang"}`),
 			},
 		},
 		{
 			name: "multiline template",
-			secret: map[string][]byte{
-				"cfg": []byte(`
+			tpl: &esv1alpha1.ExternalSecretTemplate{Data: map[string]string{
+				"cfg": `
 		datasources:
 		- name: Graphite
 			type: graphite
 			access: proxy
 			url: http://localhost:8080
 			password: "{{ .password | toString }}"
-			user: "{{ .user | toString }}"`),
-			},
+			user: "{{ .user | toString }}"`,
+			}},
 			data: map[string][]byte{
 				"user":     []byte(`foobert`),
 				"password": []byte("harharhar"),
 			},
-			outSecret: map[string][]byte{
+			expetedData: map[string][]byte{
 				"cfg": []byte(`
 		datasources:
 		- name: Graphite
@@ -153,47 +155,47 @@ func TestExecute(t *testing.T) {
 		},
 		{
 			name: "base64 pipeline",
-			secret: map[string][]byte{
-				"foo": []byte(`{{ "123412341234" | toBytes | base64encode | base64decode | toString }}`),
-			},
+			tpl: &esv1alpha1.ExternalSecretTemplate{Data: map[string]string{
+				"foo": `{{ "123412341234" | toBytes | base64encode | base64decode | toString }}`,
+			}},
 			data: map[string][]byte{},
-			outSecret: map[string][]byte{
+			expetedData: map[string][]byte{
 				"foo": []byte("123412341234"),
 			},
 		},
 		{
 			name: "base64 pkcs12 extract",
-			secret: map[string][]byte{
-				"key":  []byte(`{{ .secret | base64decode | pkcs12key | pemPrivateKey }}`),
-				"cert": []byte(`{{ .secret | base64decode | pkcs12cert | pemCertificate }}`),
-			},
+			tpl: &esv1alpha1.ExternalSecretTemplate{Data: map[string]string{
+				"key":  `{{ .secret | base64decode | pkcs12key | pemPrivateKey }}`,
+				"cert": `{{ .secret | base64decode | pkcs12cert | pemCertificate }}`,
+			}},
 			data: map[string][]byte{
 				"secret": []byte(pkcs12ContentNoPass),
 			},
-			outSecret: map[string][]byte{
+			expetedData: map[string][]byte{
 				"key":  []byte(pkcs12Key),
 				"cert": []byte(pkcs12Cert),
 			},
 		},
 		{
 			name: "base64 pkcs12 extract with password",
-			secret: map[string][]byte{
-				"key":  []byte(`{{ .secret | base64decode | pkcs12keyPass "123456" | pemPrivateKey }}`),
-				"cert": []byte(`{{ .secret | base64decode | pkcs12certPass "123456" | pemCertificate }}`),
-			},
+			tpl: &esv1alpha1.ExternalSecretTemplate{Data: map[string]string{
+				"key":  `{{ .secret | base64decode | pkcs12keyPass "123456" | pemPrivateKey }}`,
+				"cert": `{{ .secret | base64decode | pkcs12certPass "123456" | pemCertificate }}`,
+			}},
 			data: map[string][]byte{
 				"secret": []byte(pkcs12ContentWithPass),
 			},
-			outSecret: map[string][]byte{
+			expetedData: map[string][]byte{
 				"key":  []byte(pkcs12Key),
 				"cert": []byte(pkcs12Cert),
 			},
 		},
 		{
 			name: "base64 decode error",
-			secret: map[string][]byte{
-				"key": []byte(`{{ .example | base64decode }}`),
-			},
+			tpl: &esv1alpha1.ExternalSecretTemplate{Data: map[string]string{
+				"key": `{{ .example | base64decode }}`,
+			}},
 			data: map[string][]byte{
 				"example": []byte("iam_no_base64"),
 			},
@@ -201,9 +203,9 @@ func TestExecute(t *testing.T) {
 		},
 		{
 			name: "pkcs12 key wrong password",
-			secret: map[string][]byte{
-				"key": []byte(`{{ .secret | base64decode | pkcs12keyPass "wrong" | pemPrivateKey }}`),
-			},
+			tpl: &esv1alpha1.ExternalSecretTemplate{Data: map[string]string{
+				"key": `{{ .secret | base64decode | pkcs12keyPass "wrong" | pemPrivateKey }}`,
+			}},
 			data: map[string][]byte{
 				"secret": []byte(pkcs12ContentWithPass),
 			},
@@ -211,9 +213,9 @@ func TestExecute(t *testing.T) {
 		},
 		{
 			name: "pkcs12 cert wrong password",
-			secret: map[string][]byte{
-				"cert": []byte(`{{ .secret | base64decode | pkcs12certPass "wrong" | pemCertificate }}`),
-			},
+			tpl: &esv1alpha1.ExternalSecretTemplate{Data: map[string]string{
+				"cert": `{{ .secret | base64decode | pkcs12certPass "wrong" | pemCertificate }}`,
+			}},
 			data: map[string][]byte{
 				"secret": []byte(pkcs12ContentWithPass),
 			},
@@ -221,17 +223,17 @@ func TestExecute(t *testing.T) {
 		},
 		{
 			name: "fromJSON error",
-			secret: map[string][]byte{
-				"key": []byte(`{{ "{ # no json # }" | toBytes | fromJSON }}`),
-			},
+			tpl: &esv1alpha1.ExternalSecretTemplate{Data: map[string]string{
+				"key": `{{ "{ # no json # }" | toBytes | fromJSON }}`,
+			}},
 			data:   map[string][]byte{},
 			expErr: "unable to unmarshal json",
 		},
 		{
 			name: "template syntax error",
-			secret: map[string][]byte{
-				"key": []byte(`{{ #xx }}`),
-			},
+			tpl: &esv1alpha1.ExternalSecretTemplate{Data: map[string]string{
+				"key": `{{ #xx }}`,
+			}},
 			data:   map[string][]byte{},
 			expErr: "unable to parse template",
 		},
@@ -240,16 +242,17 @@ func TestExecute(t *testing.T) {
 	for i := range tbl {
 		row := tbl[i]
 		t.Run(row.name, func(t *testing.T) {
-			err := Execute(&corev1.Secret{
-				Data: row.secret,
-			}, row.data)
+			sec := &corev1.Secret{
+				Data: make(map[string][]byte),
+			}
+			err := Execute(row.tpl, sec, row.data)
 			if !ErrorContains(err, row.expErr) {
 				t.Errorf("unexpected error: %s, expected: %s", err, row.expErr)
 			}
-			if row.outSecret == nil {
+			if row.expetedData == nil {
 				return
 			}
-			assert.EqualValues(t, row.outSecret, row.secret)
+			assert.EqualValues(t, row.expetedData, sec.Data)
 		})
 	}
 }