Add status_condition metrics for the secret store controllers (#2404)

Signed-off-by: shuheiktgw <s-kitagawa@mercari.com>

pkg/controllers/secretstore/clustersecretstore_controller.go

@@ -60,7 +60,7 @@ func (r *ClusterStoreReconciler) Reconcile(ctx context.Context, req ctrl.Request
 		return ctrl.Result{}, err
 	}
 
-	return reconcile(ctx, req, &css, r.Client, log, r.ControllerClass, r.recorder, r.RequeueInterval)
+	return reconcile(ctx, req, &css, r.Client, log, r.ControllerClass, cssmetrics.GetGaugeVec, r.recorder, r.RequeueInterval)
 }
 
 // SetupWithManager returns a new controller builder that will be started by the provided Manager.

pkg/controllers/secretstore/common.go

@@ -25,6 +25,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	esapi "github.com/external-secrets/external-secrets/apis/externalsecrets/v1beta1"
+	"github.com/external-secrets/external-secrets/pkg/controllers/secretstore/metrics"
 )
 
 const (
@@ -39,8 +40,8 @@ const (
 	msgStoreValidated = "store validated"
 )
 
-func reconcile(ctx context.Context, req ctrl.Request, ss esapi.GenericStore, cl client.Client,
-	log logr.Logger, controllerClass string, recorder record.EventRecorder, requeueInterval time.Duration) (ctrl.Result, error) {
+func reconcile(ctx context.Context, req ctrl.Request, ss esapi.GenericStore, cl client.Client, log logr.Logger,
+	controllerClass string, gaugeVecGetter metrics.GaugeVevGetter, recorder record.EventRecorder, requeueInterval time.Duration) (ctrl.Result, error) {
 	if !ShouldProcessStore(ss, controllerClass) {
 		log.V(1).Info("skip store")
 		return ctrl.Result{}, nil
@@ -62,7 +63,7 @@ func reconcile(ctx context.Context, req ctrl.Request, ss esapi.GenericStore, cl
 	// validateStore modifies the store conditions
 	// we have to patch the status
 	log.V(1).Info("validating")
-	err := validateStore(ctx, req.Namespace, controllerClass, ss, cl, recorder)
+	err := validateStore(ctx, req.Namespace, controllerClass, ss, cl, gaugeVecGetter, recorder)
 	if err != nil {
 		log.Error(err, "unable to validate store")
 		return ctrl.Result{}, err
@@ -79,7 +80,7 @@ func reconcile(ctx context.Context, req ctrl.Request, ss esapi.GenericStore, cl
 
 	recorder.Event(ss, v1.EventTypeNormal, esapi.ReasonStoreValid, msgStoreValidated)
 	cond := NewSecretStoreCondition(esapi.SecretStoreReady, v1.ConditionTrue, esapi.ReasonStoreValid, msgStoreValidated)
-	SetExternalSecretCondition(ss, *cond)
+	SetExternalSecretCondition(ss, *cond, gaugeVecGetter)
 
 	return ctrl.Result{
 		RequeueAfter: requeueInterval,
@@ -89,20 +90,20 @@ func reconcile(ctx context.Context, req ctrl.Request, ss esapi.GenericStore, cl
 // validateStore tries to construct a new client
 // if it fails sets a condition and writes events.
 func validateStore(ctx context.Context, namespace, controllerClass string, store esapi.GenericStore,
-	client client.Client, recorder record.EventRecorder) error {
+	client client.Client, gaugeVecGetter metrics.GaugeVevGetter, recorder record.EventRecorder) error {
 	mgr := NewManager(client, controllerClass, false)
 	defer mgr.Close(ctx)
 	cl, err := mgr.GetFromStore(ctx, store, namespace)
 	if err != nil {
 		cond := NewSecretStoreCondition(esapi.SecretStoreReady, v1.ConditionFalse, esapi.ReasonInvalidProviderConfig, errUnableCreateClient)
-		SetExternalSecretCondition(store, *cond)
+		SetExternalSecretCondition(store, *cond, gaugeVecGetter)
 		recorder.Event(store, v1.EventTypeWarning, esapi.ReasonInvalidProviderConfig, err.Error())
 		return fmt.Errorf(errStoreClient, err)
 	}
 	validationResult, err := cl.Validate()
 	if err != nil && validationResult != esapi.ValidationResultUnknown {
 		cond := NewSecretStoreCondition(esapi.SecretStoreReady, v1.ConditionFalse, esapi.ReasonValidationFailed, errUnableValidateStore)
-		SetExternalSecretCondition(store, *cond)
+		SetExternalSecretCondition(store, *cond, gaugeVecGetter)
 		recorder.Event(store, v1.EventTypeWarning, esapi.ReasonValidationFailed, err.Error())
 		return fmt.Errorf(errValidationFailed, err)
 	}

pkg/controllers/secretstore/cssmetrics/cssmetrics.go

@@ -19,6 +19,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/metrics"
 
 	ctrlmetrics "github.com/external-secrets/external-secrets/pkg/controllers/metrics"
+	commonmetrics "github.com/external-secrets/external-secrets/pkg/controllers/secretstore/metrics"
 )
 
 const (
@@ -37,10 +38,17 @@ func SetUpMetrics() {
 		Help:      "The duration time to reconcile the Cluster Secret Store",
 	}, ctrlmetrics.NonConditionMetricLabelNames)
 
-	metrics.Registry.MustRegister(clusterSecretStoreReconcileDuration)
+	clusterSecretStoreCondition := prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Subsystem: ClusterSecretStoreSubsystem,
+		Name:      commonmetrics.StatusConditionKey,
+		Help:      "The status condition of a specific Cluster Secret Store",
+	}, ctrlmetrics.ConditionMetricLabelNames)
+
+	metrics.Registry.MustRegister(clusterSecretStoreReconcileDuration, clusterSecretStoreCondition)
 
 	gaugeVecMetrics = map[string]*prometheus.GaugeVec{
 		ClusterSecretStoreReconcileDurationKey: clusterSecretStoreReconcileDuration,
+		commonmetrics.StatusConditionKey:       clusterSecretStoreCondition,
 	}
 }
 

pkg/controllers/secretstore/metrics/metrics.go

@@ -0,0 +1,63 @@
+/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metrics
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	v1 "k8s.io/api/core/v1"
+
+	esapi "github.com/external-secrets/external-secrets/apis/externalsecrets/v1beta1"
+	ctrlmetrics "github.com/external-secrets/external-secrets/pkg/controllers/metrics"
+)
+
+const StatusConditionKey = "status_condition"
+
+type GaugeVevGetter func(key string) *prometheus.GaugeVec
+
+func UpdateStatusCondition(ss esapi.GenericStore, condition esapi.SecretStoreStatusCondition, gaugeVecGetter GaugeVevGetter) {
+	ssInfo := make(map[string]string)
+	ssInfo["name"] = ss.GetName()
+	ssInfo["namespace"] = ss.GetNamespace()
+	for k, v := range ss.GetLabels() {
+		ssInfo[k] = v
+	}
+	conditionLabels := ctrlmetrics.RefineConditionMetricLabels(ssInfo)
+	secretStoreCondition := gaugeVecGetter(StatusConditionKey)
+
+	if condition.Type == esapi.SecretStoreReady {
+		switch condition.Status {
+		case v1.ConditionFalse:
+			secretStoreCondition.With(ctrlmetrics.RefineLabels(conditionLabels,
+				map[string]string{
+					"condition": string(esapi.SecretStoreReady),
+					"status":    string(v1.ConditionTrue),
+				})).Set(0)
+		case v1.ConditionTrue:
+			secretStoreCondition.With(ctrlmetrics.RefineLabels(conditionLabels,
+				map[string]string{
+					"condition": string(esapi.SecretStoreReady),
+					"status":    string(v1.ConditionFalse),
+				})).Set(0)
+		case v1.ConditionUnknown:
+			break
+		}
+	}
+
+	secretStoreCondition.With(ctrlmetrics.RefineLabels(conditionLabels,
+		map[string]string{
+			"condition": string(condition.Type),
+			"status":    string(condition.Status),
+		})).Set(1)
+}

pkg/controllers/secretstore/metrics/metrics_test.go

@@ -0,0 +1,146 @@
+/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metrics
+
+import (
+	"testing"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/testutil"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	esapi "github.com/external-secrets/external-secrets/apis/externalsecrets/v1beta1"
+	"github.com/external-secrets/external-secrets/pkg/controllers/metrics"
+)
+
+func TestUpdateStatusCondition(t *testing.T) {
+	// Evacuate the original condition metric labels
+	tmpConditionMetricLabels := metrics.ConditionMetricLabels
+	defer func() {
+		metrics.ConditionMetricLabels = tmpConditionMetricLabels
+	}()
+	metrics.ConditionMetricLabels = map[string]string{"name": "", "namespace": "", "condition": "", "status": ""}
+
+	name := "test"
+	namespace := "test-namespace"
+
+	tests := []struct {
+		desc           string
+		condition      esapi.SecretStoreStatusCondition
+		expectedCount  int
+		expectedValues []struct {
+			labels        prometheus.Labels
+			expectedValue float64
+		}
+	}{
+		{
+			desc: "ConditionTrue",
+			condition: esapi.SecretStoreStatusCondition{
+				Type:   esapi.SecretStoreReady,
+				Status: v1.ConditionTrue,
+			},
+			expectedValues: []struct {
+				labels        prometheus.Labels
+				expectedValue float64
+			}{
+				{
+					labels: prometheus.Labels{
+						"namespace": namespace,
+						"name":      name,
+						"condition": "Ready",
+						"status":    "True",
+					},
+					expectedValue: 1,
+				},
+				{
+					labels: prometheus.Labels{
+						"namespace": namespace,
+						"name":      name,
+						"condition": "Ready",
+						"status":    "False",
+					},
+					expectedValue: 0,
+				},
+			},
+		},
+		{
+			desc: "ConditionFalse",
+			condition: esapi.SecretStoreStatusCondition{
+				Type:   esapi.SecretStoreReady,
+				Status: v1.ConditionFalse,
+			},
+			expectedValues: []struct {
+				labels        prometheus.Labels
+				expectedValue float64
+			}{
+				{
+					labels: prometheus.Labels{
+						"namespace": namespace,
+						"name":      name,
+						"condition": "Ready",
+						"status":    "True",
+					},
+					expectedValue: 0,
+				},
+				{
+					labels: prometheus.Labels{
+						"namespace": namespace,
+						"name":      name,
+						"condition": "Ready",
+						"status":    "False",
+					},
+					expectedValue: 1,
+				},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			ss := &esapi.SecretStore{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      name,
+					Namespace: namespace,
+				},
+			}
+
+			gaugeVec := prometheus.NewGaugeVec(prometheus.GaugeOpts{
+				Subsystem: "metrics",
+				Name:      "TestUpdateStatusCondition",
+			}, []string{"name", "namespace", "condition", "status"})
+
+			getter := func(key string) *prometheus.GaugeVec {
+				if key == StatusConditionKey {
+					return gaugeVec
+				}
+
+				return nil
+			}
+
+			UpdateStatusCondition(ss, test.condition, getter)
+
+			if got := testutil.CollectAndCount(gaugeVec); got != len(test.expectedValues) {
+				t.Fatalf("unexpected number of calls: got: %d, expected: %d", got, len(test.expectedValues))
+			}
+
+			for i, expected := range test.expectedValues {
+				if got := testutil.ToFloat64(gaugeVec.With(expected.labels)); got != expected.expectedValue {
+					t.Fatalf("#%d received unexpected gauge value: got: %v, expected: %v", i, got, expected.expectedValue)
+				}
+			}
+		})
+	}
+}

pkg/controllers/secretstore/secretstore_controller.go

@@ -61,7 +61,7 @@ func (r *StoreReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl
 		return ctrl.Result{}, err
 	}
 
-	return reconcile(ctx, req, &ss, r.Client, log, r.ControllerClass, r.recorder, r.RequeueInterval)
+	return reconcile(ctx, req, &ss, r.Client, log, r.ControllerClass, ssmetrics.GetGaugeVec, r.recorder, r.RequeueInterval)
 }
 
 // SetupWithManager returns a new controller builder that will be started by the provided Manager.

pkg/controllers/secretstore/ssmetrics/ssmetrics.go

@@ -19,6 +19,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/metrics"
 
 	ctrlmetrics "github.com/external-secrets/external-secrets/pkg/controllers/metrics"
+	commonmetrics "github.com/external-secrets/external-secrets/pkg/controllers/secretstore/metrics"
 )
 
 const (
@@ -37,10 +38,17 @@ func SetUpMetrics() {
 		Help:      "The duration time to reconcile the Secret Store",
 	}, ctrlmetrics.NonConditionMetricLabelNames)
 
-	metrics.Registry.MustRegister(secretStoreReconcileDuration)
+	secretStoreCondition := prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Subsystem: SecretStoreSubsystem,
+		Name:      commonmetrics.StatusConditionKey,
+		Help:      "The status condition of a specific Secret Store",
+	}, ctrlmetrics.ConditionMetricLabelNames)
+
+	metrics.Registry.MustRegister(secretStoreReconcileDuration, secretStoreCondition)
 
 	gaugeVecMetrics = map[string]*prometheus.GaugeVec{
-		SecretStoreReconcileDurationKey: secretStoreReconcileDuration,
+		SecretStoreReconcileDurationKey:  secretStoreReconcileDuration,
+		commonmetrics.StatusConditionKey: secretStoreCondition,
 	}
 }
 

pkg/controllers/secretstore/util.go

@@ -19,6 +19,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	esapi "github.com/external-secrets/external-secrets/apis/externalsecrets/v1beta1"
+	"github.com/external-secrets/external-secrets/pkg/controllers/secretstore/metrics"
 )
 
 // NewSecretStoreCondition a set of default options for creating an External Secret Condition.
@@ -45,7 +46,9 @@ func GetSecretStoreCondition(status esapi.SecretStoreStatus, condType esapi.Secr
 
 // SetExternalSecretCondition updates the external secret to include the provided
 // condition.
-func SetExternalSecretCondition(gs esapi.GenericStore, condition esapi.SecretStoreStatusCondition) {
+func SetExternalSecretCondition(gs esapi.GenericStore, condition esapi.SecretStoreStatusCondition, gaugeVecGetter metrics.GaugeVevGetter) {
+	metrics.UpdateStatusCondition(gs, condition, gaugeVecGetter)
+
 	status := gs.GetStatus()
 	currentCond := GetSecretStoreCondition(status, condition.Type)
 	if currentCond != nil && currentCond.Status == condition.Status &&