Added support for standard K8s labels in metrics (#2064)

* Added support for standard K8s labels in metrics

Signed-off-by: KA <110458464+kallymsft@users.noreply.github.com>

* Added feature-flag for label metrics

Signed-off-by: KA <110458464+kallymsft@users.noreply.github.com>

---------

Signed-off-by: KA <110458464+kallymsft@users.noreply.github.com>

cmd/root.go

@@ -37,6 +37,7 @@ import (
 	genv1alpha1 "github.com/external-secrets/external-secrets/apis/generators/v1alpha1"
 	"github.com/external-secrets/external-secrets/pkg/controllers/clusterexternalsecret"
 	"github.com/external-secrets/external-secrets/pkg/controllers/externalsecret"
+	"github.com/external-secrets/external-secrets/pkg/controllers/externalsecret/esmetrics"
 	"github.com/external-secrets/external-secrets/pkg/controllers/pushsecret"
 	"github.com/external-secrets/external-secrets/pkg/controllers/secretstore"
 	"github.com/external-secrets/external-secrets/pkg/feature"
@@ -64,6 +65,7 @@ var (
 	enableClusterExternalSecretReconciler bool
 	enablePushSecretReconciler            bool
 	enableFloodGate                       bool
+	enableExtendedMetricLabels            bool
 	storeRequeueInterval                  time.Duration
 	serviceName, serviceNamespace         string
 	secretName, secretNamespace           string
@@ -123,6 +125,7 @@ var rootCmd = &cobra.Command{
 		}
 		logger := zap.New(zap.UseFlagOptions(&opts))
 		ctrl.SetLogger(logger)
+		esmetrics.SetUpMetrics(enableExtendedMetricLabels)
 		config := ctrl.GetConfigOrDie()
 		config.QPS = clientQPS
 		config.Burst = clientBurst
@@ -240,6 +243,7 @@ func init() {
 	rootCmd.Flags().BoolVar(&enableConfigMapsCache, "enable-configmaps-caching", false, "Enable secrets caching for external-secrets pod.")
 	rootCmd.Flags().DurationVar(&storeRequeueInterval, "store-requeue-interval", time.Minute*5, "Default Time duration between reconciling (Cluster)SecretStores")
 	rootCmd.Flags().BoolVar(&enableFloodGate, "enable-flood-gate", true, "Enable flood gate. External secret will be reconciled only if the ClusterStore or Store have an healthy or unknown state.")
+	rootCmd.Flags().BoolVar(&enableExtendedMetricLabels, "enable-extended-metric-labels", false, "Enable recommended kubernetes annotations as labels in metrics.")
 	fs := feature.Features()
 	for _, f := range fs {
 		rootCmd.Flags().AddFlagSet(f.Flags)

deploy/charts/external-secrets/README.md

@@ -85,6 +85,7 @@ The command removes all the Kubernetes components associated with the chart and
 | createOperator | bool | `true` | Specifies whether an external secret operator deployment be created. |
 | deploymentAnnotations | object | `{}` | Annotations to add to Deployment |
 | dnsConfig | object | `{}` | Specifies `dnsOptions` to deployment |
+| extendedMetricLabels | bool | `false` | If true external secrets will use recommended kubernetes annotations as prometheus metric labels. |
 | extraArgs | object | `{}` |  |
 | extraContainers | list | `[]` |  |
 | extraEnv | list | `[]` |  |

deploy/charts/external-secrets/templates/deployment.yaml

@@ -68,6 +68,9 @@ spec:
           {{- if .Values.controllerClass }}
           - --controller-class={{ .Values.controllerClass }}
           {{- end }}
+          {{- if .Values.extendedMetricLabels }}
+          - --enable-extended-metric-labels={{ .Values.extendedMetricLabels }}
+          {{- end }}
           {{- if .Values.concurrent }}
           - --concurrent={{ .Values.concurrent }}
           {{- end }}

deploy/charts/external-secrets/values.yaml

@@ -38,6 +38,10 @@ leaderElect: false
 # Secret Stores with the appropriate controller values.
 controllerClass: ""
 
+# -- If true external secrets will use recommended kubernetes
+# annotations as prometheus metric labels.
+extendedMetricLabels: false
+
 # -- If set external secrets are only reconciled in the
 # provided namespace
 scopedNamespace: ""

docs/api/controller-options.md

@@ -22,6 +22,7 @@ The core controller is invoked without a subcommand and can be configured with t
 | `--enable-secrets-caching`                    | boolean  | false                         | Enables the secrets caching for external-secrets pod.                                                                                                              |
 | `--enable-configmaps-caching`                 | boolean  | false                         | Enables the ConfigMap caching for external-secrets pod.                                                                                                            |
 | `--enable-flood-gate`                         | boolean  | true                          | Enable flood gate. External secret will be reconciled only if the ClusterStore or Store have an healthy or unknown state.                                          |
+| `--enable-extended-metric-labels`             | boolean  | true                          | Enable recommended kubernetes annotations as labels in metrics.                                          |
 | `--enable-leader-election`                    | boolean  | false                         | Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.                                              |
 | `--experimental-enable-aws-session-cache`     | boolean  | false                         | Enable experimental AWS session cache. External secret will reuse the AWS session without creating a new one on each request.                                      |
 | `--help`                                      |          |                               | help for external-secrets                                                                                                                                          |

pkg/controllers/externalsecret/esmetrics/esmetrics.go

@@ -0,0 +1,230 @@
+/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package esmetrics
+
+import (
+	"regexp"
+
+	"github.com/prometheus/client_golang/prometheus"
+	v1 "k8s.io/api/core/v1"
+	"sigs.k8s.io/controller-runtime/pkg/metrics"
+
+	esv1beta1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1beta1"
+)
+
+const (
+	ExternalSecretSubsystem            = "externalsecret"
+	SyncCallsKey                       = "sync_calls_total"
+	SyncCallsErrorKey                  = "sync_calls_error"
+	ExternalSecretStatusConditionKey   = "status_condition"
+	ExternalSecretReconcileDurationKey = "reconcile_duration"
+)
+
+var (
+	NonConditionMetricLabelNames = make([]string, 0)
+
+	ConditionMetricLabelNames = make([]string, 0)
+
+	NonConditionMetricLabels = make(map[string]string)
+
+	ConditionMetricLabels = make(map[string]string)
+)
+
+var counterVecMetrics map[string]*prometheus.CounterVec = map[string]*prometheus.CounterVec{}
+
+var gaugeVecMetrics map[string]*prometheus.GaugeVec = map[string]*prometheus.GaugeVec{}
+
+// Called at the root to set-up the metric logic using the
+// config flags provided.
+func SetUpMetrics(addKubeStandardLabels bool) {
+	// Figure out what the labels for the metrics are
+	if addKubeStandardLabels {
+		NonConditionMetricLabelNames = []string{
+			"name", "namespace",
+			"app_kubernetes_io_name", "app_kubernetes_io_instance",
+			"app_kubernetes_io_version", "app_kubernetes_io_component",
+			"app_kubernetes_io_part_of", "app_kubernetes_io_managed_by",
+		}
+
+		ConditionMetricLabelNames = []string{
+			"name", "namespace",
+			"condition", "status",
+			"app_kubernetes_io_name", "app_kubernetes_io_instance",
+			"app_kubernetes_io_version", "app_kubernetes_io_component",
+			"app_kubernetes_io_part_of", "app_kubernetes_io_managed_by",
+		}
+	} else {
+		NonConditionMetricLabelNames = []string{"name", "namespace"}
+
+		ConditionMetricLabelNames = []string{"name", "namespace", "condition", "status"}
+	}
+
+	// Set default values for each label
+	for _, k := range NonConditionMetricLabelNames {
+		NonConditionMetricLabels[k] = ""
+	}
+
+	for _, k := range ConditionMetricLabelNames {
+		ConditionMetricLabels[k] = ""
+	}
+
+	// Obtain the prometheus metrics and register
+	syncCallsTotal := prometheus.NewCounterVec(prometheus.CounterOpts{
+		Subsystem: ExternalSecretSubsystem,
+		Name:      SyncCallsKey,
+		Help:      "Total number of the External Secret sync calls",
+	}, NonConditionMetricLabelNames)
+
+	syncCallsError := prometheus.NewCounterVec(prometheus.CounterOpts{
+		Subsystem: ExternalSecretSubsystem,
+		Name:      SyncCallsErrorKey,
+		Help:      "Total number of the External Secret sync errors",
+	}, NonConditionMetricLabelNames)
+
+	externalSecretCondition := prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Subsystem: ExternalSecretSubsystem,
+		Name:      ExternalSecretStatusConditionKey,
+		Help:      "The status condition of a specific External Secret",
+	}, ConditionMetricLabelNames)
+
+	externalSecretReconcileDuration := prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Subsystem: ExternalSecretSubsystem,
+		Name:      ExternalSecretReconcileDurationKey,
+		Help:      "The duration time to reconcile the External Secret",
+	}, NonConditionMetricLabelNames)
+
+	metrics.Registry.MustRegister(syncCallsTotal, syncCallsError, externalSecretCondition, externalSecretReconcileDuration)
+
+	counterVecMetrics = map[string]*prometheus.CounterVec{
+		SyncCallsKey:      syncCallsTotal,
+		SyncCallsErrorKey: syncCallsError,
+	}
+
+	gaugeVecMetrics = map[string]*prometheus.GaugeVec{
+		ExternalSecretStatusConditionKey:   externalSecretCondition,
+		ExternalSecretReconcileDurationKey: externalSecretReconcileDuration,
+	}
+}
+
+func UpdateExternalSecretCondition(es *esv1beta1.ExternalSecret, condition *esv1beta1.ExternalSecretStatusCondition, value float64) {
+	esInfo := make(map[string]string)
+	esInfo["name"] = es.Name
+	esInfo["namespace"] = es.Namespace
+	for k, v := range es.Labels {
+		esInfo[k] = v
+	}
+	conditionLabels := RefineConditionMetricLabels(esInfo)
+	externalSecretCondition := GetGaugeVec(ExternalSecretStatusConditionKey)
+
+	switch condition.Type {
+	case esv1beta1.ExternalSecretDeleted:
+		// Remove condition=Ready metrics when the object gets deleted.
+		externalSecretCondition.Delete(RefineLabels(conditionLabels,
+			map[string]string{
+				"condition": string(esv1beta1.ExternalSecretReady),
+				"status":    string(v1.ConditionFalse),
+			}))
+
+		externalSecretCondition.Delete(RefineLabels(conditionLabels,
+			map[string]string{
+				"condition": string(esv1beta1.ExternalSecretReady),
+				"status":    string(v1.ConditionTrue),
+			}))
+
+	case esv1beta1.ExternalSecretReady:
+		// Remove condition=Deleted metrics when the object gets ready.
+		externalSecretCondition.Delete(RefineLabels(conditionLabels,
+			map[string]string{
+				"condition": string(esv1beta1.ExternalSecretDeleted),
+				"status":    string(v1.ConditionFalse),
+			}))
+
+		externalSecretCondition.Delete(RefineLabels(conditionLabels,
+			map[string]string{
+				"condition": string(esv1beta1.ExternalSecretDeleted),
+				"status":    string(v1.ConditionTrue),
+			}))
+
+		// Toggle opposite Status to 0
+		switch condition.Status {
+		case v1.ConditionFalse:
+			externalSecretCondition.With(RefineLabels(conditionLabels,
+				map[string]string{
+					"condition": string(esv1beta1.ExternalSecretReady),
+					"status":    string(v1.ConditionTrue),
+				})).Set(0)
+		case v1.ConditionTrue:
+			externalSecretCondition.With(RefineLabels(conditionLabels,
+				map[string]string{
+					"condition": string(esv1beta1.ExternalSecretReady),
+					"status":    string(v1.ConditionFalse),
+				})).Set(0)
+		case v1.ConditionUnknown:
+			break
+		default:
+			break
+		}
+
+	default:
+		break
+	}
+
+	externalSecretCondition.With(RefineLabels(conditionLabels,
+		map[string]string{
+			"condition": string(condition.Type),
+			"status":    string(condition.Status),
+		})).Set(value)
+}
+
+func GetCounterVec(key string) *prometheus.CounterVec {
+	return counterVecMetrics[key]
+}
+
+func GetGaugeVec(key string) *prometheus.GaugeVec {
+	return gaugeVecMetrics[key]
+}
+
+// Refine the given Prometheus Labels with values from a map `newLabels`
+// Only overwrite a value if the corresponding key is present in the
+// Prometheus' Labels already to avoid adding label names which are
+// not defined in a metric's description. Note that non-alphanumeric
+// characters from keys of `newLabels` are replaced by an underscore
+// because Prometheus does not accept non-alphanumeric, non-underscore
+// characters in label names.
+func RefineLabels(promLabels prometheus.Labels, newLabels map[string]string) prometheus.Labels {
+	nonAlphanumericRegex := regexp.MustCompile(`[^a-zA-Z0-9 ]+`)
+	var refinement = prometheus.Labels{}
+
+	for k, v := range promLabels {
+		refinement[k] = v
+	}
+
+	for k, v := range newLabels {
+		cleanKey := nonAlphanumericRegex.ReplaceAllString(k, "_")
+		if _, ok := refinement[cleanKey]; ok {
+			refinement[cleanKey] = v
+		}
+	}
+
+	return refinement
+}
+
+func RefineNonConditionMetricLabels(labels map[string]string) prometheus.Labels {
+	return RefineLabels(NonConditionMetricLabels, labels)
+}
+
+func RefineConditionMetricLabels(labels map[string]string) prometheus.Labels {
+	return RefineLabels(ConditionMetricLabels, labels)
+}

pkg/controllers/externalsecret/externalsecret_controller.go

@@ -22,7 +22,6 @@ import (
 	"time"
 
 	"github.com/go-logr/logr"
-	"github.com/prometheus/client_golang/prometheus"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/equality"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -38,6 +37,8 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 
 	esv1beta1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1beta1"
+	// Metrics.
+	"github.com/external-secrets/external-secrets/pkg/controllers/externalsecret/esmetrics"
 	// Loading registered generators.
 	_ "github.com/external-secrets/external-secrets/pkg/generator/register"
 	// Loading registered providers.
@@ -95,28 +96,47 @@ type Reconciler struct {
 func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
 	log := r.Log.WithValues("ExternalSecret", req.NamespacedName)
 
-	resourceLabels := prometheus.Labels{"name": req.Name, "namespace": req.Namespace}
+	resourceLabels := esmetrics.RefineNonConditionMetricLabels(map[string]string{"name": req.Name, "namespace": req.Namespace})
 	start := time.Now()
-	defer externalSecretReconcileDuration.With(resourceLabels).Set(float64(time.Since(start)))
-	defer syncCallsTotal.With(resourceLabels).Inc()
+
+	externalSecretReconcileDuration := esmetrics.GetGaugeVec(esmetrics.ExternalSecretReconcileDurationKey)
+	syncCallsTotal := esmetrics.GetCounterVec(esmetrics.SyncCallsKey)
+	syncCallsError := esmetrics.GetCounterVec(esmetrics.SyncCallsErrorKey)
 
 	var externalSecret esv1beta1.ExternalSecret
 	err := r.Get(ctx, req.NamespacedName, &externalSecret)
-	if apierrors.IsNotFound(err) {
-		conditionSynced := NewExternalSecretCondition(esv1beta1.ExternalSecretDeleted, v1.ConditionFalse, esv1beta1.ConditionReasonSecretDeleted, "Secret was deleted")
-		SetExternalSecretCondition(&esv1beta1.ExternalSecret{
-			ObjectMeta: metav1.ObjectMeta{
-				Name:      req.Name,
-				Namespace: req.Namespace,
-			},
-		}, *conditionSynced)
-		return ctrl.Result{}, nil
-	} else if err != nil {
+
+	if err != nil {
+		if apierrors.IsNotFound(err) {
+			conditionSynced := NewExternalSecretCondition(esv1beta1.ExternalSecretDeleted, v1.ConditionFalse, esv1beta1.ConditionReasonSecretDeleted, "Secret was deleted")
+			SetExternalSecretCondition(&esv1beta1.ExternalSecret{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      req.Name,
+					Namespace: req.Namespace,
+				},
+			}, *conditionSynced)
+
+			externalSecretReconcileDuration.With(resourceLabels).Set(float64(time.Since(start)))
+			syncCallsTotal.With(resourceLabels).Inc()
+
+			return ctrl.Result{}, nil
+		}
+
 		log.Error(err, errGetES)
 		syncCallsError.With(resourceLabels).Inc()
+
+		externalSecretReconcileDuration.With(resourceLabels).Set(float64(time.Since(start)))
+		syncCallsTotal.With(resourceLabels).Inc()
+
 		return ctrl.Result{}, nil
 	}
 
+	// if extended metrics is enabled, refine the time series vector
+	resourceLabels = esmetrics.RefineLabels(resourceLabels, externalSecret.Labels)
+
+	defer externalSecretReconcileDuration.With(resourceLabels).Set(float64(time.Since(start)))
+	defer syncCallsTotal.With(resourceLabels).Inc()
+
 	if shouldSkipClusterSecretStore(r, externalSecret) {
 		log.Info("skipping cluster secret store as it is disabled")
 		return ctrl.Result{}, nil

pkg/controllers/externalsecret/externalsecret_controller_test.go

@@ -23,6 +23,7 @@ import (
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	"github.com/prometheus/client_golang/prometheus"
 	dto "github.com/prometheus/client_model/go"
 	v1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -33,6 +34,7 @@ import (
 	esv1beta1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1beta1"
 	genv1alpha1 "github.com/external-secrets/external-secrets/apis/generators/v1alpha1"
 	ctest "github.com/external-secrets/external-secrets/pkg/controllers/commontest"
+	"github.com/external-secrets/external-secrets/pkg/controllers/externalsecret/esmetrics"
 	"github.com/external-secrets/external-secrets/pkg/provider/testing/fake"
 )
 
@@ -44,6 +46,14 @@ var (
 	interval       = time.Millisecond * 250
 )
 
+var (
+	testSyncCallsTotal *prometheus.CounterVec
+	testSyncCallsError *prometheus.CounterVec
+
+	testExternalSecretCondition         *prometheus.GaugeVec
+	testExternalSecretReconcileDuration *prometheus.GaugeVec
+)
+
 type testCase struct {
 	secretStore    esv1beta1.GenericStore
 	externalSecret *esv1beta1.ExternalSecret
@@ -160,10 +170,10 @@ var _ = Describe("ExternalSecret controller", func() {
 		ExternalSecretNamespace, err = ctest.CreateNamespaceWithLabels("test-ns", k8sClient, map[string]string{NamespaceLabelKey: NamespaceLabelValue})
 		Expect(err).ToNot(HaveOccurred())
 		metric.Reset()
-		syncCallsTotal.Reset()
-		syncCallsError.Reset()
-		externalSecretCondition.Reset()
-		externalSecretReconcileDuration.Reset()
+		testSyncCallsTotal.Reset()
+		testSyncCallsError.Reset()
+		testExternalSecretCondition.Reset()
+		testExternalSecretReconcileDuration.Reset()
 		fakeProvider.Reset()
 	})
 
@@ -294,8 +304,8 @@ var _ = Describe("ExternalSecret controller", func() {
 			Expect(externalSecretConditionShouldBe(ExternalSecretName, ExternalSecretNamespace, esv1beta1.ExternalSecretReady, v1.ConditionFalse, 0.0)).To(BeTrue())
 			Expect(externalSecretConditionShouldBe(ExternalSecretName, ExternalSecretNamespace, esv1beta1.ExternalSecretReady, v1.ConditionTrue, 1.0)).To(BeTrue())
 			Eventually(func() bool {
-				Expect(syncCallsTotal.WithLabelValues(ExternalSecretName, ExternalSecretNamespace).Write(&metric)).To(Succeed())
-				Expect(externalSecretReconcileDuration.WithLabelValues(ExternalSecretName, ExternalSecretNamespace).Write(&metricDuration)).To(Succeed())
+				Expect(testSyncCallsTotal.WithLabelValues(ExternalSecretName, ExternalSecretNamespace).Write(&metric)).To(Succeed())
+				Expect(testExternalSecretReconcileDuration.WithLabelValues(ExternalSecretName, ExternalSecretNamespace).Write(&metricDuration)).To(Succeed())
 				// three reconciliations: initial sync, status update, secret update
 				return metric.GetCounter().GetValue() >= 2.0 && metricDuration.GetGauge().GetValue() > 0.0
 			}, timeout, interval).Should(BeTrue())
@@ -400,8 +410,8 @@ var _ = Describe("ExternalSecret controller", func() {
 		}
 		tc.checkExternalSecret = func(es *esv1beta1.ExternalSecret) {
 			Eventually(func() bool {
-				Expect(syncCallsError.WithLabelValues(ExternalSecretName, ExternalSecretNamespace).Write(&metric)).To(Succeed())
-				Expect(externalSecretReconcileDuration.WithLabelValues(ExternalSecretName, ExternalSecretNamespace).Write(&metricDuration)).To(Succeed())
+				Expect(testSyncCallsError.WithLabelValues(ExternalSecretName, ExternalSecretNamespace).Write(&metric)).To(Succeed())
+				Expect(testExternalSecretReconcileDuration.WithLabelValues(ExternalSecretName, ExternalSecretNamespace).Write(&metricDuration)).To(Succeed())
 				return metric.GetCounter().GetValue() >= 2.0 && metricDuration.GetGauge().GetValue() > 0.0
 			}, timeout, interval).Should(BeTrue())
 			Expect(externalSecretConditionShouldBe(ExternalSecretName, ExternalSecretNamespace, esv1beta1.ExternalSecretReady, v1.ConditionFalse, 1.0)).To(BeTrue())
@@ -1286,8 +1296,8 @@ var _ = Describe("ExternalSecret controller", func() {
 		}
 		tc.checkExternalSecret = func(es *esv1beta1.ExternalSecret) {
 			Eventually(func() bool {
-				Expect(syncCallsError.WithLabelValues(ExternalSecretName, ExternalSecretNamespace).Write(&metric)).To(Succeed())
-				Expect(externalSecretReconcileDuration.WithLabelValues(ExternalSecretName, ExternalSecretNamespace).Write(&metricDuration)).To(Succeed())
+				Expect(testSyncCallsError.WithLabelValues(ExternalSecretName, ExternalSecretNamespace).Write(&metric)).To(Succeed())
+				Expect(testExternalSecretReconcileDuration.WithLabelValues(ExternalSecretName, ExternalSecretNamespace).Write(&metricDuration)).To(Succeed())
 				return metric.GetCounter().GetValue() >= 2.0 && metricDuration.GetGauge().GetValue() > 0.0
 			}, timeout, interval).Should(BeTrue())
 			Expect(externalSecretConditionShouldBe(ExternalSecretName, ExternalSecretNamespace, esv1beta1.ExternalSecretReady, v1.ConditionFalse, 1.0)).To(BeTrue())
@@ -1334,8 +1344,8 @@ var _ = Describe("ExternalSecret controller", func() {
 		}
 		tc.checkExternalSecret = func(es *esv1beta1.ExternalSecret) {
 			Eventually(func() bool {
-				Expect(syncCallsError.WithLabelValues(ExternalSecretName, ExternalSecretNamespace).Write(&metric)).To(Succeed())
-				Expect(externalSecretReconcileDuration.WithLabelValues(ExternalSecretName, ExternalSecretNamespace).Write(&metricDuration)).To(Succeed())
+				Expect(testSyncCallsError.WithLabelValues(ExternalSecretName, ExternalSecretNamespace).Write(&metric)).To(Succeed())
+				Expect(testExternalSecretReconcileDuration.WithLabelValues(ExternalSecretName, ExternalSecretNamespace).Write(&metricDuration)).To(Succeed())
 				return metric.GetCounter().GetValue() >= 2.0 && metricDuration.GetGauge().GetValue() > 0.0
 			}, timeout, interval).Should(BeTrue())
 			Expect(externalSecretConditionShouldBe(ExternalSecretName, ExternalSecretNamespace, esv1beta1.ExternalSecretReady, v1.ConditionFalse, 1.0)).To(BeTrue())
@@ -1466,8 +1476,8 @@ var _ = Describe("ExternalSecret controller", func() {
 		}
 		tc.checkExternalSecret = func(es *esv1beta1.ExternalSecret) {
 			Eventually(func() bool {
-				Expect(syncCallsError.WithLabelValues(ExternalSecretName, ExternalSecretNamespace).Write(&metric)).To(Succeed())
-				Expect(externalSecretReconcileDuration.WithLabelValues(ExternalSecretName, ExternalSecretNamespace).Write(&metricDuration)).To(Succeed())
+				Expect(testSyncCallsError.WithLabelValues(ExternalSecretName, ExternalSecretNamespace).Write(&metric)).To(Succeed())
+				Expect(testExternalSecretReconcileDuration.WithLabelValues(ExternalSecretName, ExternalSecretNamespace).Write(&metricDuration)).To(Succeed())
 				return metric.GetCounter().GetValue() >= 2.0 && metricDuration.GetGauge().GetValue() > 0.0
 			}, timeout, interval).Should(BeTrue())
 			Expect(externalSecretConditionShouldBe(ExternalSecretName, ExternalSecretNamespace, esv1beta1.ExternalSecretReady, v1.ConditionFalse, 1.0)).To(BeTrue())
@@ -1504,8 +1514,8 @@ var _ = Describe("ExternalSecret controller", func() {
 		}
 		tc.checkExternalSecret = func(es *esv1beta1.ExternalSecret) {
 			Eventually(func() bool {
-				Expect(syncCallsError.WithLabelValues(ExternalSecretName, ExternalSecretNamespace).Write(&metric)).To(Succeed())
-				Expect(externalSecretReconcileDuration.WithLabelValues(ExternalSecretName, ExternalSecretNamespace).Write(&metricDuration)).To(Succeed())
+				Expect(testSyncCallsError.WithLabelValues(ExternalSecretName, ExternalSecretNamespace).Write(&metric)).To(Succeed())
+				Expect(testExternalSecretReconcileDuration.WithLabelValues(ExternalSecretName, ExternalSecretNamespace).Write(&metricDuration)).To(Succeed())
 				return metric.GetCounter().GetValue() >= 2.0 && metricDuration.GetGauge().GetValue() > 0.0
 			}, timeout, interval).Should(BeTrue())
 			Expect(externalSecretConditionShouldBe(ExternalSecretName, ExternalSecretNamespace, esv1beta1.ExternalSecretReady, v1.ConditionFalse, 1.0)).To(BeTrue())
@@ -1530,8 +1540,8 @@ var _ = Describe("ExternalSecret controller", func() {
 		}
 		tc.checkExternalSecret = func(es *esv1beta1.ExternalSecret) {
 			Eventually(func() bool {
-				Expect(syncCallsError.WithLabelValues(ExternalSecretName, ExternalSecretNamespace).Write(&metric)).To(Succeed())
-				Expect(externalSecretReconcileDuration.WithLabelValues(ExternalSecretName, ExternalSecretNamespace).Write(&metricDuration)).To(Succeed())
+				Expect(testSyncCallsError.WithLabelValues(ExternalSecretName, ExternalSecretNamespace).Write(&metric)).To(Succeed())
+				Expect(testExternalSecretReconcileDuration.WithLabelValues(ExternalSecretName, ExternalSecretNamespace).Write(&metricDuration)).To(Succeed())
 				return metric.GetCounter().GetValue() >= 2.0 && metricDuration.GetGauge().GetValue() > 0.0
 			}, timeout, interval).Should(BeTrue())
 			Expect(externalSecretConditionShouldBe(ExternalSecretName, ExternalSecretNamespace, esv1beta1.ExternalSecretReady, v1.ConditionFalse, 1.0)).To(BeTrue())
@@ -1550,12 +1560,12 @@ var _ = Describe("ExternalSecret controller", func() {
 		tc.checkExternalSecret = func(es *esv1beta1.ExternalSecret) {
 			// Condition True and False should be 0, since the Condition was not created
 			Eventually(func() float64 {
-				Expect(externalSecretCondition.WithLabelValues(ExternalSecretName, ExternalSecretNamespace, string(esv1beta1.ExternalSecretReady), string(v1.ConditionTrue)).Write(&metric)).To(Succeed())
+				Expect(testExternalSecretCondition.WithLabelValues(ExternalSecretName, ExternalSecretNamespace, string(esv1beta1.ExternalSecretReady), string(v1.ConditionTrue)).Write(&metric)).To(Succeed())
 				return metric.GetGauge().GetValue()
 			}, timeout, interval).Should(Equal(0.0))
 
 			Eventually(func() float64 {
-				Expect(externalSecretCondition.WithLabelValues(ExternalSecretName, ExternalSecretNamespace, string(esv1beta1.ExternalSecretReady), string(v1.ConditionFalse)).Write(&metric)).To(Succeed())
+				Expect(testExternalSecretCondition.WithLabelValues(ExternalSecretName, ExternalSecretNamespace, string(esv1beta1.ExternalSecretReady), string(v1.ConditionFalse)).Write(&metric)).To(Succeed())
 				return metric.GetGauge().GetValue()
 			}, timeout, interval).Should(Equal(0.0))
 
@@ -2233,7 +2243,7 @@ var _ = Describe("Controller Reconcile logic", func() {
 
 func externalSecretConditionShouldBe(name, ns string, ct esv1beta1.ExternalSecretConditionType, cs v1.ConditionStatus, v float64) bool {
 	return Eventually(func() float64 {
-		Expect(externalSecretCondition.WithLabelValues(name, ns, string(ct), string(cs)).Write(&metric)).To(Succeed())
+		Expect(testExternalSecretCondition.WithLabelValues(name, ns, string(ct), string(cs)).Write(&metric)).To(Succeed())
 		return metric.GetGauge().GetValue()
 	}, timeout, interval).Should(Equal(v))
 }
@@ -2245,4 +2255,10 @@ func init() {
 			Service: esv1beta1.AWSServiceSecretsManager,
 		},
 	})
+
+	esmetrics.SetUpMetrics(false)
+	testSyncCallsTotal = esmetrics.GetCounterVec(esmetrics.SyncCallsKey)
+	testSyncCallsError = esmetrics.GetCounterVec(esmetrics.SyncCallsErrorKey)
+	testExternalSecretCondition = esmetrics.GetGaugeVec(esmetrics.ExternalSecretStatusConditionKey)
+	testExternalSecretReconcileDuration = esmetrics.GetGaugeVec(esmetrics.ExternalSecretReconcileDurationKey)
 }

pkg/controllers/externalsecret/metrics.go

@@ -1,127 +0,0 @@
-/*
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package externalsecret
-
-import (
-	"github.com/prometheus/client_golang/prometheus"
-	v1 "k8s.io/api/core/v1"
-	"sigs.k8s.io/controller-runtime/pkg/metrics"
-
-	esv1beta1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1beta1"
-)
-
-const (
-	ExternalSecretSubsystem            = "externalsecret"
-	SyncCallsKey                       = "sync_calls_total"
-	SyncCallsErrorKey                  = "sync_calls_error"
-	externalSecretStatusConditionKey   = "status_condition"
-	externalSecretReconcileDurationKey = "reconcile_duration"
-)
-
-var (
-	syncCallsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
-		Subsystem: ExternalSecretSubsystem,
-		Name:      SyncCallsKey,
-		Help:      "Total number of the External Secret sync calls",
-	}, []string{"name", "namespace"})
-
-	syncCallsError = prometheus.NewCounterVec(prometheus.CounterOpts{
-		Subsystem: ExternalSecretSubsystem,
-		Name:      SyncCallsErrorKey,
-		Help:      "Total number of the External Secret sync errors",
-	}, []string{"name", "namespace"})
-
-	externalSecretCondition = prometheus.NewGaugeVec(prometheus.GaugeOpts{
-		Subsystem: ExternalSecretSubsystem,
-		Name:      externalSecretStatusConditionKey,
-		Help:      "The status condition of a specific External Secret",
-	}, []string{"name", "namespace", "condition", "status"})
-
-	externalSecretReconcileDuration = prometheus.NewGaugeVec(prometheus.GaugeOpts{
-		Subsystem: ExternalSecretSubsystem,
-		Name:      externalSecretReconcileDurationKey,
-		Help:      "The duration time to reconcile the External Secret",
-	}, []string{"name", "namespace"})
-)
-
-// updateExternalSecretCondition updates the ExternalSecret conditions.
-func updateExternalSecretCondition(es *esv1beta1.ExternalSecret, condition *esv1beta1.ExternalSecretStatusCondition, value float64) {
-	switch condition.Type {
-	case esv1beta1.ExternalSecretDeleted:
-		// Remove condition=Ready metrics when the object gets deleted.
-		externalSecretCondition.Delete(prometheus.Labels{
-			"name":      es.Name,
-			"namespace": es.Namespace,
-			"condition": string(esv1beta1.ExternalSecretReady),
-			"status":    string(v1.ConditionFalse),
-		})
-		externalSecretCondition.Delete(prometheus.Labels{
-			"name":      es.Name,
-			"namespace": es.Namespace,
-			"condition": string(esv1beta1.ExternalSecretReady),
-			"status":    string(v1.ConditionTrue),
-		})
-
-	case esv1beta1.ExternalSecretReady:
-		// Remove condition=Deleted metrics when the object gets ready.
-		externalSecretCondition.Delete(prometheus.Labels{
-			"name":      es.Name,
-			"namespace": es.Namespace,
-			"condition": string(esv1beta1.ExternalSecretDeleted),
-			"status":    string(v1.ConditionFalse),
-		})
-		externalSecretCondition.Delete(prometheus.Labels{
-			"name":      es.Name,
-			"namespace": es.Namespace,
-			"condition": string(esv1beta1.ExternalSecretDeleted),
-			"status":    string(v1.ConditionTrue),
-		})
-		// Toggle opposite Status to 0
-		switch condition.Status {
-		case v1.ConditionFalse:
-			externalSecretCondition.With(prometheus.Labels{
-				"name":      es.Name,
-				"namespace": es.Namespace,
-				"condition": string(esv1beta1.ExternalSecretReady),
-				"status":    string(v1.ConditionTrue),
-			}).Set(0)
-		case v1.ConditionTrue:
-			externalSecretCondition.With(prometheus.Labels{
-				"name":      es.Name,
-				"namespace": es.Namespace,
-				"condition": string(esv1beta1.ExternalSecretReady),
-				"status":    string(v1.ConditionFalse),
-			}).Set(0)
-		case v1.ConditionUnknown:
-			break
-		default:
-			break
-		}
-
-	default:
-		break
-	}
-
-	externalSecretCondition.With(prometheus.Labels{
-		"name":      es.Name,
-		"namespace": es.Namespace,
-		"condition": string(condition.Type),
-		"status":    string(condition.Status),
-	}).Set(value)
-}
-
-func init() {
-	metrics.Registry.MustRegister(syncCallsTotal, syncCallsError, externalSecretCondition, externalSecretReconcileDuration)
-}

pkg/controllers/externalsecret/util.go

@@ -18,6 +18,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	esv1beta1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1beta1"
+	esmetrics "github.com/external-secrets/external-secrets/pkg/controllers/externalsecret/esmetrics"
 )
 
 // NewExternalSecretCondition a set of default options for creating an External Secret Condition.
@@ -49,7 +50,7 @@ func SetExternalSecretCondition(es *esv1beta1.ExternalSecret, condition esv1beta
 
 	if currentCond != nil && currentCond.Status == condition.Status &&
 		currentCond.Reason == condition.Reason && currentCond.Message == condition.Message {
-		updateExternalSecretCondition(es, &condition, 1.0)
+		esmetrics.UpdateExternalSecretCondition(es, &condition, 1.0)
 		return
 	}
 
@@ -61,10 +62,10 @@ func SetExternalSecretCondition(es *esv1beta1.ExternalSecret, condition esv1beta
 	es.Status.Conditions = append(filterOutCondition(es.Status.Conditions, condition.Type), condition)
 
 	if currentCond != nil {
-		updateExternalSecretCondition(es, currentCond, 0.0)
+		esmetrics.UpdateExternalSecretCondition(es, currentCond, 0.0)
 	}
 
-	updateExternalSecretCondition(es, &condition, 1.0)
+	esmetrics.UpdateExternalSecretCondition(es, &condition, 1.0)
 }
 
 // filterOutCondition returns an empty set of conditions with the provided type.