Improve Grafana generator integration with in-cluster Grafana (#4519)

* Improve Grafana generator integration with in-cluster Grafana

Signed-off-by: solidDoWant <fred.heinecke@yahoo.com>

* Switch to URL parsing

Signed-off-by: solidDoWant <fred.heinecke@yahoo.com>

* rm unnecessary type conversion

Signed-off-by: solidDoWant <fred.heinecke@yahoo.com>

* `omitEmpty`  -> `omitempty`

Signed-off-by: solidDoWant <fred.heinecke@yahoo.com>

---------

Signed-off-by: solidDoWant <fred.heinecke@yahoo.com>
Co-authored-by: Moritz Johner <moolen@users.noreply.github.com>

apis/generators/v1alpha1/types_grafana.go

@@ -44,7 +44,21 @@ type GrafanaAuth struct {
 	// Note: you need a token which has elevated permissions to create service accounts.
 	// See here for the documentation on basic roles offered by Grafana:
 	// https://grafana.com/docs/grafana/latest/administration/roles-and-permissions/access-control/rbac-fixed-basic-role-definitions/
-	Token SecretKeySelector `json:"token"`
+	// +optional
+	Token *SecretKeySelector `json:"token,omitempty"`
+	// Basic auth credentials used to authenticate against the Grafana instance.
+	// Note: you need a token which has elevated permissions to create service accounts.
+	// See here for the documentation on basic roles offered by Grafana:
+	// https://grafana.com/docs/grafana/latest/administration/roles-and-permissions/access-control/rbac-fixed-basic-role-definitions/
+	// +optional
+	Basic *GrafanaBasicAuth `json:"basic,omitempty"`
+}
+
+type GrafanaBasicAuth struct {
+	// A basic auth username used to authenticate against the Grafana instance.
+	Username string `json:"username"`
+	// A basic auth password used to authenticate against the Grafana instance.
+	Password SecretKeySelector `json:"password"`
 }
 
 // GrafanaServiceAccountTokenState is the state type produced by the Grafana generator.

apis/generators/v1alpha1/zz_generated.deepcopy.go

@@ -701,7 +701,7 @@ func (in *GeneratorSpec) DeepCopyInto(out *GeneratorSpec) {
 	if in.GrafanaSpec != nil {
 		in, out := &in.GrafanaSpec, &out.GrafanaSpec
 		*out = new(GrafanaSpec)
-		**out = **in
+		(*in).DeepCopyInto(*out)
 	}
 }
 
@@ -964,7 +964,7 @@ func (in *Grafana) DeepCopyInto(out *Grafana) {
 	*out = *in
 	out.TypeMeta = in.TypeMeta
 	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
-	out.Spec = in.Spec
+	in.Spec.DeepCopyInto(&out.Spec)
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Grafana.
@@ -988,7 +988,16 @@ func (in *Grafana) DeepCopyObject() runtime.Object {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *GrafanaAuth) DeepCopyInto(out *GrafanaAuth) {
 	*out = *in
-	out.Token = in.Token
+	if in.Token != nil {
+		in, out := &in.Token, &out.Token
+		*out = new(SecretKeySelector)
+		**out = **in
+	}
+	if in.Basic != nil {
+		in, out := &in.Basic, &out.Basic
+		*out = new(GrafanaBasicAuth)
+		**out = **in
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GrafanaAuth.
@@ -1002,6 +1011,22 @@ func (in *GrafanaAuth) DeepCopy() *GrafanaAuth {
 }
 
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *GrafanaBasicAuth) DeepCopyInto(out *GrafanaBasicAuth) {
+	*out = *in
+	out.Password = in.Password
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GrafanaBasicAuth.
+func (in *GrafanaBasicAuth) DeepCopy() *GrafanaBasicAuth {
+	if in == nil {
+		return nil
+	}
+	out := new(GrafanaBasicAuth)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *GrafanaList) DeepCopyInto(out *GrafanaList) {
 	*out = *in
 	out.TypeMeta = in.TypeMeta
@@ -1067,7 +1092,7 @@ func (in *GrafanaServiceAccountTokenState) DeepCopy() *GrafanaServiceAccountToke
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *GrafanaSpec) DeepCopyInto(out *GrafanaSpec) {
 	*out = *in
-	out.Auth = in.Auth
+	in.Auth.DeepCopyInto(&out.Auth)
 	out.ServiceAccount = in.ServiceAccount
 }
 

config/crds/bases/generators.external-secrets.io_clustergenerators.yaml

@@ -537,6 +537,39 @@ spec:
                           Auth is the authentication configuration to authenticate
                           against the Grafana instance.
                         properties:
+                          basic:
+                            description: |-
+                              Basic auth credentials used to authenticate against the Grafana instance.
+                              Note: you need a token which has elevated permissions to create service accounts.
+                              See here for the documentation on basic roles offered by Grafana:
+                              https://grafana.com/docs/grafana/latest/administration/roles-and-permissions/access-control/rbac-fixed-basic-role-definitions/
+                            properties:
+                              password:
+                                description: A basic auth password used to authenticate
+                                  against the Grafana instance.
+                                properties:
+                                  key:
+                                    description: The key where the token is found.
+                                    maxLength: 253
+                                    minLength: 1
+                                    pattern: ^[-._a-zA-Z0-9]+$
+                                    type: string
+                                  name:
+                                    description: The name of the Secret resource being
+                                      referred to.
+                                    maxLength: 253
+                                    minLength: 1
+                                    pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$
+                                    type: string
+                                type: object
+                              username:
+                                description: A basic auth username used to authenticate
+                                  against the Grafana instance.
+                                type: string
+                            required:
+                            - password
+                            - username
+                            type: object
                           token:
                             description: |-
                               A service account token used to authenticate against the Grafana instance.
@@ -558,8 +591,6 @@ spec:
                                 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$
                                 type: string
                             type: object
-                        required:
-                        - token
                         type: object
                       serviceAccount:
                         description: |-

config/crds/bases/generators.external-secrets.io_grafanas.yaml

@@ -47,6 +47,39 @@ spec:
                   Auth is the authentication configuration to authenticate
                   against the Grafana instance.
                 properties:
+                  basic:
+                    description: |-
+                      Basic auth credentials used to authenticate against the Grafana instance.
+                      Note: you need a token which has elevated permissions to create service accounts.
+                      See here for the documentation on basic roles offered by Grafana:
+                      https://grafana.com/docs/grafana/latest/administration/roles-and-permissions/access-control/rbac-fixed-basic-role-definitions/
+                    properties:
+                      password:
+                        description: A basic auth password used to authenticate against
+                          the Grafana instance.
+                        properties:
+                          key:
+                            description: The key where the token is found.
+                            maxLength: 253
+                            minLength: 1
+                            pattern: ^[-._a-zA-Z0-9]+$
+                            type: string
+                          name:
+                            description: The name of the Secret resource being referred
+                              to.
+                            maxLength: 253
+                            minLength: 1
+                            pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$
+                            type: string
+                        type: object
+                      username:
+                        description: A basic auth username used to authenticate against
+                          the Grafana instance.
+                        type: string
+                    required:
+                    - password
+                    - username
+                    type: object
                   token:
                     description: |-
                       A service account token used to authenticate against the Grafana instance.
@@ -68,8 +101,6 @@ spec:
                         pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$
                         type: string
                     type: object
-                required:
-                - token
                 type: object
               serviceAccount:
                 description: |-

deploy/crds/bundle.yaml

@@ -14525,6 +14525,36 @@ spec:
                             Auth is the authentication configuration to authenticate
                             against the Grafana instance.
                           properties:
+                            basic:
+                              description: |-
+                                Basic auth credentials used to authenticate against the Grafana instance.
+                                Note: you need a token which has elevated permissions to create service accounts.
+                                See here for the documentation on basic roles offered by Grafana:
+                                https://grafana.com/docs/grafana/latest/administration/roles-and-permissions/access-control/rbac-fixed-basic-role-definitions/
+                              properties:
+                                password:
+                                  description: A basic auth password used to authenticate against the Grafana instance.
+                                  properties:
+                                    key:
+                                      description: The key where the token is found.
+                                      maxLength: 253
+                                      minLength: 1
+                                      pattern: ^[-._a-zA-Z0-9]+$
+                                      type: string
+                                    name:
+                                      description: The name of the Secret resource being referred to.
+                                      maxLength: 253
+                                      minLength: 1
+                                      pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$
+                                      type: string
+                                  type: object
+                                username:
+                                  description: A basic auth username used to authenticate against the Grafana instance.
+                                  type: string
+                              required:
+                                - password
+                                - username
+                              type: object
                             token:
                               description: |-
                                 A service account token used to authenticate against the Grafana instance.
@@ -14545,8 +14575,6 @@ spec:
                                   pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$
                                   type: string
                               type: object
-                          required:
-                            - token
                           type: object
                         serviceAccount:
                           description: |-
@@ -16464,6 +16492,36 @@ spec:
                     Auth is the authentication configuration to authenticate
                     against the Grafana instance.
                   properties:
+                    basic:
+                      description: |-
+                        Basic auth credentials used to authenticate against the Grafana instance.
+                        Note: you need a token which has elevated permissions to create service accounts.
+                        See here for the documentation on basic roles offered by Grafana:
+                        https://grafana.com/docs/grafana/latest/administration/roles-and-permissions/access-control/rbac-fixed-basic-role-definitions/
+                      properties:
+                        password:
+                          description: A basic auth password used to authenticate against the Grafana instance.
+                          properties:
+                            key:
+                              description: The key where the token is found.
+                              maxLength: 253
+                              minLength: 1
+                              pattern: ^[-._a-zA-Z0-9]+$
+                              type: string
+                            name:
+                              description: The name of the Secret resource being referred to.
+                              maxLength: 253
+                              minLength: 1
+                              pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$
+                              type: string
+                          type: object
+                        username:
+                          description: A basic auth username used to authenticate against the Grafana instance.
+                          type: string
+                      required:
+                        - password
+                        - username
+                      type: object
                     token:
                       description: |-
                         A service account token used to authenticate against the Grafana instance.
@@ -16484,8 +16542,6 @@ spec:
                           pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$
                           type: string
                       type: object
-                  required:
-                    - token
                   type: object
                 serviceAccount:
                   description: |-

e2e/suites/generator/grafana.go

@@ -98,7 +98,7 @@ var _ = Describe("grafana generator", Label("grafana"), func() {
 					Role: "Viewer",
 				},
 				Auth: genv1alpha1.GrafanaAuth{
-					Token: genv1alpha1.SecretKeySelector{
+					Token: &genv1alpha1.SecretKeySelector{
 						Name: grafanaCredsSecretName,
 						Key:  "grafana-token",
 					},

pkg/generator/grafana/grafana.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"net/url"
 	"strings"
 
 	"github.com/google/uuid"
@@ -40,23 +41,12 @@ func (w *Grafana) Generate(ctx context.Context, jsonSpec *apiextensions.JSON, kc
 	if err != nil {
 		return nil, nil, err
 	}
-	secret, err := resolvers.SecretKeyRef(ctx, kclient, resolvers.EmptyStoreKind, ns, &esmeta.SecretKeySelector{
-		Namespace: &ns,
-		Name:      gen.Spec.Auth.Token.Name,
-		Key:       gen.Spec.Auth.Token.Key,
-	})
+
+	cl, err := newClient(ctx, gen, kclient, ns)
 	if err != nil {
 		return nil, nil, err
 	}
-	url := strings.TrimPrefix(gen.Spec.URL, "https://")
-	cfg := &grafanaclient.TransportConfig{
-		Host:     url,
-		BasePath: "/api",
-		Schemes:  []string{"https"},
-		APIKey:   secret,
-	}
 
-	cl := grafanaclient.NewHTTPClientWithConfig(nil, cfg)
 	state, err := createOrGetServiceAccount(cl, gen)
 	if err != nil {
 		return nil, nil, err
@@ -100,24 +90,59 @@ func (w *Grafana) Cleanup(ctx context.Context, jsonSpec *apiextensions.JSON, pre
 }
 
 func newClient(ctx context.Context, gen *genv1alpha1.Grafana, kclient client.Client, ns string) (*grafanaclient.GrafanaHTTPAPI, error) {
-	secret, err := resolvers.SecretKeyRef(ctx, kclient, resolvers.EmptyStoreKind, ns, &esmeta.SecretKeySelector{
-		Namespace: &ns,
-		Name:      gen.Spec.Auth.Token.Name,
-		Key:       gen.Spec.Auth.Token.Key,
-	})
+	parsedURL, err := url.Parse(gen.Spec.URL)
 	if err != nil {
 		return nil, err
 	}
-	url := strings.TrimPrefix(gen.Spec.URL, "https://")
+
 	cfg := &grafanaclient.TransportConfig{
-		Host:     url,
-		BasePath: "/api",
-		Schemes:  []string{"https"},
-		APIKey:   secret,
+		Host:     parsedURL.Host,
+		BasePath: parsedURL.JoinPath("/api").Path,
+		Schemes:  []string{parsedURL.Scheme},
 	}
+
+	if err := setGrafanaClientCredentials(ctx, gen, kclient, ns, cfg); err != nil {
+		return nil, err
+	}
+
 	return grafanaclient.NewHTTPClientWithConfig(nil, cfg), nil
 }
 
+func setGrafanaClientCredentials(ctx context.Context, gen *genv1alpha1.Grafana, kclient client.Client, ns string, cfg *grafanaclient.TransportConfig) error {
+	// First try to use service account auth
+	if gen.Spec.Auth.Token != nil {
+		serviceAccountAPIKey, err := resolvers.SecretKeyRef(ctx, kclient, resolvers.EmptyStoreKind, ns, &esmeta.SecretKeySelector{
+			Namespace: &ns,
+			Name:      gen.Spec.Auth.Token.Name,
+			Key:       gen.Spec.Auth.Token.Key,
+		})
+		if err != nil {
+			return err
+		}
+
+		cfg.APIKey = serviceAccountAPIKey
+		return nil
+	}
+
+	// Next try to use basic auth
+	if gen.Spec.Auth.Basic != nil {
+		basicAuthPassword, err := resolvers.SecretKeyRef(ctx, kclient, resolvers.EmptyStoreKind, ns, &esmeta.SecretKeySelector{
+			Namespace: &ns,
+			Name:      gen.Spec.Auth.Basic.Password.Name,
+			Key:       gen.Spec.Auth.Basic.Password.Key,
+		})
+		if err != nil {
+			return err
+		}
+
+		cfg.BasicAuth = url.UserPassword(gen.Spec.Auth.Basic.Username, basicAuthPassword)
+		return nil
+	}
+
+	// No auth found, fail
+	return fmt.Errorf("no auth configuration found")
+}
+
 func createOrGetServiceAccount(cl *grafanaclient.GrafanaHTTPAPI, gen *genv1alpha1.Grafana) (*genv1alpha1.GrafanaServiceAccountTokenState, error) {
 	saList, err := cl.ServiceAccounts.SearchOrgServiceAccountsWithPaging(&grafanasa.SearchOrgServiceAccountsWithPagingParams{
 		Query: ptr.To(gen.Spec.ServiceAccount.Name),