Add reconcile_duration metrics (#2382)

* Add reconcile_duration metrics

Signed-off-by: shuheiktgw <s-kitagawa@mercari.com>

* fix: increase dupl threshold

Signed-off-by: Moritz Johner <beller.moritz@googlemail.com>

---------

Signed-off-by: shuheiktgw <s-kitagawa@mercari.com>
Signed-off-by: Moritz Johner <beller.moritz@googlemail.com>
Co-authored-by: Moritz Johner <beller.moritz@googlemail.com>

.golangci.yaml

@@ -32,6 +32,8 @@ linters-settings:
     min-complexity: 16
   goheader:
     template-path: ./hack/boilerplate.go.txt
+  dupl:
+    threshold: 200
   govet:
     check-shadowing: false
   lll:

cmd/root.go

@@ -41,7 +41,10 @@ import (
 	"github.com/external-secrets/external-secrets/pkg/controllers/externalsecret/esmetrics"
 	ctrlmetrics "github.com/external-secrets/external-secrets/pkg/controllers/metrics"
 	"github.com/external-secrets/external-secrets/pkg/controllers/pushsecret"
+	"github.com/external-secrets/external-secrets/pkg/controllers/pushsecret/psmetrics"
 	"github.com/external-secrets/external-secrets/pkg/controllers/secretstore"
+	"github.com/external-secrets/external-secrets/pkg/controllers/secretstore/cssmetrics"
+	"github.com/external-secrets/external-secrets/pkg/controllers/secretstore/ssmetrics"
 	"github.com/external-secrets/external-secrets/pkg/feature"
 )
 
@@ -145,6 +148,8 @@ var rootCmd = &cobra.Command{
 			setupLog.Error(err, "unable to start manager")
 			os.Exit(1)
 		}
+
+		ssmetrics.SetUpMetrics()
 		if err = (&secretstore.StoreReconciler{
 			Client:          mgr.GetClient(),
 			Log:             ctrl.Log.WithName("controllers").WithName("SecretStore"),
@@ -156,6 +161,7 @@ var rootCmd = &cobra.Command{
 			os.Exit(1)
 		}
 		if enableClusterStoreReconciler {
+			cssmetrics.SetUpMetrics()
 			if err = (&secretstore.ClusterStoreReconciler{
 				Client:          mgr.GetClient(),
 				Log:             ctrl.Log.WithName("controllers").WithName("ClusterSecretStore"),
@@ -183,6 +189,7 @@ var rootCmd = &cobra.Command{
 			os.Exit(1)
 		}
 		if enablePushSecretReconciler {
+			psmetrics.SetUpMetrics()
 			if err = (&pushsecret.Reconciler{
 				Client:          mgr.GetClient(),
 				Log:             ctrl.Log.WithName("controllers").WithName("PushSecret"),

pkg/controllers/pushsecret/psmetrics/psmetrics.go

@@ -0,0 +1,49 @@
+/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package psmetrics
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"sigs.k8s.io/controller-runtime/pkg/metrics"
+
+	ctrlmetrics "github.com/external-secrets/external-secrets/pkg/controllers/metrics"
+)
+
+const (
+	PushSecretSubsystem            = "pushsecret"
+	PushSecretReconcileDurationKey = "reconcile_duration"
+)
+
+var gaugeVecMetrics = map[string]*prometheus.GaugeVec{}
+
+// SetUpMetrics is called at the root to set-up the metric logic using the
+// config flags provided.
+func SetUpMetrics() {
+	pushSecretReconcileDuration := prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Subsystem: PushSecretSubsystem,
+		Name:      PushSecretReconcileDurationKey,
+		Help:      "The duration time to reconcile the Push Secret",
+	}, ctrlmetrics.NonConditionMetricLabelNames)
+
+	metrics.Registry.MustRegister(pushSecretReconcileDuration)
+
+	gaugeVecMetrics = map[string]*prometheus.GaugeVec{
+		PushSecretReconcileDurationKey: pushSecretReconcileDuration,
+	}
+}
+
+func GetGaugeVec(key string) *prometheus.GaugeVec {
+	return gaugeVecMetrics[key]
+}

pkg/controllers/pushsecret/pushsecret_controller.go

@@ -32,7 +32,9 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 
 	esapi "github.com/external-secrets/external-secrets/apis/externalsecrets/v1alpha1"
-	v1beta1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1beta1"
+	"github.com/external-secrets/external-secrets/apis/externalsecrets/v1beta1"
+	ctrlmetrics "github.com/external-secrets/external-secrets/pkg/controllers/metrics"
+	"github.com/external-secrets/external-secrets/pkg/controllers/pushsecret/psmetrics"
 	"github.com/external-secrets/external-secrets/pkg/controllers/secretstore"
 )
 
@@ -60,6 +62,13 @@ type Reconciler struct {
 
 func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
 	log := r.Log.WithValues("pushsecret", req.NamespacedName)
+
+	resourceLabels := ctrlmetrics.RefineNonConditionMetricLabels(map[string]string{"name": req.Name, "namespace": req.Namespace})
+	start := time.Now()
+
+	pushSecretReconcileDuration := psmetrics.GetGaugeVec(psmetrics.PushSecretReconcileDurationKey)
+	defer func() { pushSecretReconcileDuration.With(resourceLabels).Set(float64(time.Since(start))) }()
+
 	var ps esapi.PushSecret
 	err := r.Get(ctx, req.NamespacedName, &ps)
 	mgr := secretstore.NewManager(r.Client, r.ControllerClass, false)

pkg/controllers/pushsecret/pushsecret_controller_test.go

@@ -29,9 +29,10 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
-	v1alpha1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1alpha1"
-	v1beta1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1beta1"
+	"github.com/external-secrets/external-secrets/apis/externalsecrets/v1alpha1"
+	"github.com/external-secrets/external-secrets/apis/externalsecrets/v1beta1"
 	ctest "github.com/external-secrets/external-secrets/pkg/controllers/commontest"
+	"github.com/external-secrets/external-secrets/pkg/controllers/pushsecret/psmetrics"
 	"github.com/external-secrets/external-secrets/pkg/provider/testing/fake"
 )
 
@@ -53,6 +54,7 @@ func init() {
 	v1beta1.ForceRegister(fakeProvider, &v1beta1.SecretStoreProvider{
 		Fake: &v1beta1.FakeProvider{},
 	})
+	psmetrics.SetUpMetrics()
 }
 
 func checkCondition(status v1alpha1.PushSecretStatus, cond v1alpha1.PushSecretStatusCondition) bool {

pkg/controllers/secretstore/clustersecretstore_controller.go

@@ -26,6 +26,8 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	esapi "github.com/external-secrets/external-secrets/apis/externalsecrets/v1beta1"
+	ctrlmetrics "github.com/external-secrets/external-secrets/pkg/controllers/metrics"
+	"github.com/external-secrets/external-secrets/pkg/controllers/secretstore/cssmetrics"
 	// Loading registered providers.
 	_ "github.com/external-secrets/external-secrets/pkg/provider/register"
 )
@@ -42,6 +44,13 @@ type ClusterStoreReconciler struct {
 
 func (r *ClusterStoreReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
 	log := r.Log.WithValues("clustersecretstore", req.NamespacedName)
+
+	resourceLabels := ctrlmetrics.RefineNonConditionMetricLabels(map[string]string{"name": req.Name, "namespace": req.Namespace})
+	start := time.Now()
+
+	clusterSecretStoreReconcileDuration := cssmetrics.GetGaugeVec(cssmetrics.ClusterSecretStoreReconcileDurationKey)
+	defer func() { clusterSecretStoreReconcileDuration.With(resourceLabels).Set(float64(time.Since(start))) }()
+
 	var css esapi.ClusterSecretStore
 	err := r.Get(ctx, req.NamespacedName, &css)
 	if apierrors.IsNotFound(err) {

pkg/controllers/secretstore/cssmetrics/cssmetrics.go

@@ -0,0 +1,49 @@
+/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cssmetrics
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"sigs.k8s.io/controller-runtime/pkg/metrics"
+
+	ctrlmetrics "github.com/external-secrets/external-secrets/pkg/controllers/metrics"
+)
+
+const (
+	ClusterSecretStoreSubsystem            = "clustersecretstore"
+	ClusterSecretStoreReconcileDurationKey = "reconcile_duration"
+)
+
+var gaugeVecMetrics = map[string]*prometheus.GaugeVec{}
+
+// SetUpMetrics is called at the root to set-up the metric logic using the
+// config flags provided.
+func SetUpMetrics() {
+	clusterSecretStoreReconcileDuration := prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Subsystem: ClusterSecretStoreSubsystem,
+		Name:      ClusterSecretStoreReconcileDurationKey,
+		Help:      "The duration time to reconcile the Cluster Secret Store",
+	}, ctrlmetrics.NonConditionMetricLabelNames)
+
+	metrics.Registry.MustRegister(clusterSecretStoreReconcileDuration)
+
+	gaugeVecMetrics = map[string]*prometheus.GaugeVec{
+		ClusterSecretStoreReconcileDurationKey: clusterSecretStoreReconcileDuration,
+	}
+}
+
+func GetGaugeVec(key string) *prometheus.GaugeVec {
+	return gaugeVecMetrics[key]
+}

pkg/controllers/secretstore/secretstore_controller.go

@@ -26,6 +26,8 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	esapi "github.com/external-secrets/external-secrets/apis/externalsecrets/v1beta1"
+	ctrlmetrics "github.com/external-secrets/external-secrets/pkg/controllers/metrics"
+	"github.com/external-secrets/external-secrets/pkg/controllers/secretstore/ssmetrics"
 	// Loading registered providers.
 	_ "github.com/external-secrets/external-secrets/pkg/provider/register"
 )
@@ -42,6 +44,13 @@ type StoreReconciler struct {
 
 func (r *StoreReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
 	log := r.Log.WithValues("secretstore", req.NamespacedName)
+
+	resourceLabels := ctrlmetrics.RefineNonConditionMetricLabels(map[string]string{"name": req.Name, "namespace": req.Namespace})
+	start := time.Now()
+
+	secretStoreReconcileDuration := ssmetrics.GetGaugeVec(ssmetrics.SecretStoreReconcileDurationKey)
+	defer func() { secretStoreReconcileDuration.With(resourceLabels).Set(float64(time.Since(start))) }()
+
 	var ss esapi.SecretStore
 	err := r.Get(ctx, req.NamespacedName, &ss)
 	if apierrors.IsNotFound(err) {

pkg/controllers/secretstore/ssmetrics/ssmetrics.go

@@ -0,0 +1,49 @@
+/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package ssmetrics
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"sigs.k8s.io/controller-runtime/pkg/metrics"
+
+	ctrlmetrics "github.com/external-secrets/external-secrets/pkg/controllers/metrics"
+)
+
+const (
+	SecretStoreSubsystem            = "secretstore"
+	SecretStoreReconcileDurationKey = "reconcile_duration"
+)
+
+var gaugeVecMetrics = map[string]*prometheus.GaugeVec{}
+
+// SetUpMetrics is called at the root to set-up the metric logic using the
+// config flags provided.
+func SetUpMetrics() {
+	secretStoreReconcileDuration := prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Subsystem: SecretStoreSubsystem,
+		Name:      SecretStoreReconcileDurationKey,
+		Help:      "The duration time to reconcile the Secret Store",
+	}, ctrlmetrics.NonConditionMetricLabelNames)
+
+	metrics.Registry.MustRegister(secretStoreReconcileDuration)
+
+	gaugeVecMetrics = map[string]*prometheus.GaugeVec{
+		SecretStoreReconcileDurationKey: secretStoreReconcileDuration,
+	}
+}
+
+func GetGaugeVec(key string) *prometheus.GaugeVec {
+	return gaugeVecMetrics[key]
+}

pkg/controllers/secretstore/suite_test.go

@@ -30,6 +30,9 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 
 	esapi "github.com/external-secrets/external-secrets/apis/externalsecrets/v1beta1"
+	ctrlmetrics "github.com/external-secrets/external-secrets/pkg/controllers/metrics"
+	"github.com/external-secrets/external-secrets/pkg/controllers/secretstore/cssmetrics"
+	"github.com/external-secrets/external-secrets/pkg/controllers/secretstore/ssmetrics"
 )
 
 var cfg *rest.Config
@@ -100,3 +103,9 @@ var _ = AfterSuite(func() {
 	err := testEnv.Stop()
 	Expect(err).ToNot(HaveOccurred())
 })
+
+func init() {
+	ctrlmetrics.SetUpLabelNames(false)
+	cssmetrics.SetUpMetrics()
+	ssmetrics.SetUpMetrics()
+}