Add status_condition metric for ClusterExternalSecret (#2380)

* Add status_condition metric for ClusterExternalSecret

Signed-off-by: shuheiktgw <s-kitagawa@mercari.com>

* Register ClusterExternalSecretCondition metric

Signed-off-by: shuheiktgw <s-kitagawa@mercari.com>

* Stop setting namespace for ClusterExternalSecretStatusCondition

Signed-off-by: shuheiktgw <s-kitagawa@mercari.com>

---------

Signed-off-by: shuheiktgw <s-kitagawa@mercari.com>

pkg/controllers/clusterexternalsecret/cesmetrics/cesmetrics.go

@@ -16,14 +16,17 @@ package cesmetrics
 
 import (
 	"github.com/prometheus/client_golang/prometheus"
+	v1 "k8s.io/api/core/v1"
 	"sigs.k8s.io/controller-runtime/pkg/metrics"
 
+	esv1beta1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1beta1"
 	ctrlmetrics "github.com/external-secrets/external-secrets/pkg/controllers/metrics"
 )
 
 const (
 	ClusterExternalSecretSubsystem            = "clusterexternalsecret"
 	ClusterExternalSecretReconcileDurationKey = "reconcile_duration"
+	ClusterExternalSecretStatusConditionKey   = "status_condition"
 )
 
 var gaugeVecMetrics = map[string]*prometheus.GaugeVec{}
@@ -37,9 +40,16 @@ func SetUpMetrics() {
 		Help:      "The duration time to reconcile the Cluster External Secret",
 	}, ctrlmetrics.NonConditionMetricLabelNames)
 
-	metrics.Registry.MustRegister(clusterExternalSecretReconcileDuration)
+	clusterExternalSecretCondition := prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Subsystem: ClusterExternalSecretSubsystem,
+		Name:      ClusterExternalSecretStatusConditionKey,
+		Help:      "The status condition of a specific Cluster External Secret",
+	}, ctrlmetrics.ConditionMetricLabelNames)
+
+	metrics.Registry.MustRegister(clusterExternalSecretReconcileDuration, clusterExternalSecretCondition)
 
 	gaugeVecMetrics = map[string]*prometheus.GaugeVec{
+		ClusterExternalSecretStatusConditionKey:   clusterExternalSecretCondition,
 		ClusterExternalSecretReconcileDurationKey: clusterExternalSecretReconcileDuration,
 	}
 }
@@ -47,3 +57,56 @@ func SetUpMetrics() {
 func GetGaugeVec(key string) *prometheus.GaugeVec {
 	return gaugeVecMetrics[key]
 }
+
+func UpdateClusterExternalSecretCondition(ces *esv1beta1.ClusterExternalSecret, condition *esv1beta1.ClusterExternalSecretStatusCondition) {
+	if condition.Status != v1.ConditionTrue {
+		// This should not happen
+		return
+	}
+
+	cesInfo := make(map[string]string)
+	cesInfo["name"] = ces.Name
+	for k, v := range ces.Labels {
+		cesInfo[k] = v
+	}
+	conditionLabels := ctrlmetrics.RefineConditionMetricLabels(cesInfo)
+	clusterExternalSecretCondition := GetGaugeVec(ClusterExternalSecretStatusConditionKey)
+
+	targetConditionType := condition.Type
+	var theOtherConditionTypes []esv1beta1.ClusterExternalSecretConditionType
+	for _, ct := range []esv1beta1.ClusterExternalSecretConditionType{
+		esv1beta1.ClusterExternalSecretReady,
+		esv1beta1.ClusterExternalSecretPartiallyReady,
+		esv1beta1.ClusterExternalSecretNotReady,
+	} {
+		if ct != targetConditionType {
+			theOtherConditionTypes = append(theOtherConditionTypes, ct)
+		}
+	}
+
+	// Set the target condition metric
+	clusterExternalSecretCondition.With(ctrlmetrics.RefineLabels(conditionLabels,
+		map[string]string{
+			"condition": string(targetConditionType),
+			"status":    string(v1.ConditionTrue),
+		})).Set(1)
+	clusterExternalSecretCondition.With(ctrlmetrics.RefineLabels(conditionLabels,
+		map[string]string{
+			"condition": string(targetConditionType),
+			"status":    string(v1.ConditionFalse),
+		})).Set(0)
+
+	// Remove the other condition metrics
+	for _, ct := range theOtherConditionTypes {
+		clusterExternalSecretCondition.Delete(ctrlmetrics.RefineLabels(conditionLabels,
+			map[string]string{
+				"condition": string(ct),
+				"status":    string(v1.ConditionFalse),
+			}))
+		clusterExternalSecretCondition.Delete(ctrlmetrics.RefineLabels(conditionLabels,
+			map[string]string{
+				"condition": string(ct),
+				"status":    string(v1.ConditionTrue),
+			}))
+	}
+}

pkg/controllers/clusterexternalsecret/cesmetrics/cesmetrics_test.go

@@ -0,0 +1,120 @@
+/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cesmetrics
+
+import (
+	"testing"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/testutil"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	esv1beta1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1beta1"
+	"github.com/external-secrets/external-secrets/pkg/controllers/metrics"
+)
+
+func TestUpdateClusterExternalSecretCondition(t *testing.T) {
+	// Evacuate the original condition metric labels
+	tmpConditionMetricLabels := metrics.ConditionMetricLabels
+	defer func() {
+		metrics.ConditionMetricLabels = tmpConditionMetricLabels
+	}()
+	metrics.ConditionMetricLabels = map[string]string{"name": "", "namespace": "", "condition": "", "status": ""}
+
+	name := "test"
+
+	tests := []struct {
+		desc           string
+		condition      *esv1beta1.ClusterExternalSecretStatusCondition
+		expectedCount  int
+		expectedValues []struct {
+			labels        prometheus.Labels
+			expectedValue float64
+		}
+	}{
+		{
+			desc: "ConditionTrue",
+			condition: &esv1beta1.ClusterExternalSecretStatusCondition{
+				Type:   esv1beta1.ClusterExternalSecretReady,
+				Status: v1.ConditionTrue,
+			},
+			expectedValues: []struct {
+				labels        prometheus.Labels
+				expectedValue float64
+			}{
+				{
+					labels: prometheus.Labels{
+						"namespace": "",
+						"name":      name,
+						"condition": "Ready",
+						"status":    "True",
+					},
+					expectedValue: 1.0,
+				},
+				{
+					labels: prometheus.Labels{
+						"namespace": "",
+						"name":      name,
+						"condition": "Ready",
+						"status":    "False",
+					},
+					expectedValue: 0.0,
+				},
+			},
+		},
+		{
+			desc: "ConditionFalse",
+			condition: &esv1beta1.ClusterExternalSecretStatusCondition{
+				Type:   esv1beta1.ClusterExternalSecretReady,
+				Status: v1.ConditionFalse,
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			ces := &esv1beta1.ClusterExternalSecret{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: name,
+				},
+			}
+
+			// Evacuate the original gauge vec
+			tmpGaugeVec := GetGaugeVec(ClusterExternalSecretStatusConditionKey)
+			defer func() {
+				gaugeVecMetrics[ClusterExternalSecretStatusConditionKey] = tmpGaugeVec
+			}()
+
+			gaugeVec := prometheus.NewGaugeVec(prometheus.GaugeOpts{
+				Subsystem: "csmetrics",
+				Name:      "TestUpdateClusterExternalSecretCondition",
+			}, []string{"name", "namespace", "condition", "status"})
+
+			gaugeVecMetrics[ClusterExternalSecretStatusConditionKey] = gaugeVec
+			UpdateClusterExternalSecretCondition(ces, test.condition)
+
+			if got := testutil.CollectAndCount(gaugeVec); got != len(test.expectedValues) {
+				t.Fatalf("unexpected number of calls: got: %d, expected: %d", got, len(test.expectedValues))
+			}
+
+			for i, expected := range test.expectedValues {
+				if got := testutil.ToFloat64(gaugeVec.With(expected.labels)); got != expected.expectedValue {
+					t.Fatalf("#%d received unexpected gauge value: got: %v, expected: %v", i, got, expected.expectedValue)
+				}
+			}
+		})
+	}
+}

pkg/controllers/clusterexternalsecret/clusterexternalsecret_controller.go

@@ -127,14 +127,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 		provisionedNamespaces = append(provisionedNamespaces, namespace.ObjectMeta.Name)
 	}
 
-	conditionType := getCondition(failedNamespaces, &namespaceList)
-
-	condition := NewClusterExternalSecretCondition(conditionType, v1.ConditionTrue)
-
-	if conditionType != esv1beta1.ClusterExternalSecretReady {
-		condition.Message = errNamespacesFailed
-	}
-
+	condition := NewClusterExternalSecretCondition(failedNamespaces, &namespaceList)
 	SetClusterExternalSecretCondition(&clusterExternalSecret, *condition)
 	setFailedNamespaces(&clusterExternalSecret, failedNamespaces)
 
@@ -237,18 +230,6 @@ func checkForError(getError error, existingES *esv1beta1.ExternalSecret) string
 	return ""
 }
 
-func getCondition(namespaces map[string]string, namespaceList *v1.NamespaceList) esv1beta1.ClusterExternalSecretConditionType {
-	if len(namespaces) == 0 {
-		return esv1beta1.ClusterExternalSecretReady
-	}
-
-	if len(namespaces) < len(namespaceList.Items) {
-		return esv1beta1.ClusterExternalSecretPartiallyReady
-	}
-
-	return esv1beta1.ClusterExternalSecretNotReady
-}
-
 func getRemovedNamespaces(nsList v1.NamespaceList, provisionedNs []string) []string {
 	result := []string{}
 

pkg/controllers/clusterexternalsecret/util.go

@@ -18,16 +18,24 @@ import (
 	v1 "k8s.io/api/core/v1"
 
 	esv1beta1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1beta1"
+	"github.com/external-secrets/external-secrets/pkg/controllers/clusterexternalsecret/cesmetrics"
 )
 
-func NewClusterExternalSecretCondition(condType esv1beta1.ClusterExternalSecretConditionType, status v1.ConditionStatus) *esv1beta1.ClusterExternalSecretStatusCondition {
-	return &esv1beta1.ClusterExternalSecretStatusCondition{
-		Type:   condType,
-		Status: status,
+func NewClusterExternalSecretCondition(failedNamespaces map[string]string, namespaceList *v1.NamespaceList) *esv1beta1.ClusterExternalSecretStatusCondition {
+	conditionType := getConditionType(failedNamespaces, namespaceList)
+	condition := &esv1beta1.ClusterExternalSecretStatusCondition{
+		Type:   conditionType,
+		Status: v1.ConditionTrue,
 	}
+
+	if conditionType != esv1beta1.ClusterExternalSecretReady {
+		condition.Message = errNamespacesFailed
+	}
+
+	return condition
 }
 
-// GetExternalSecretCondition returns the condition with the provided type.
+// GetClusterExternalSecretCondition returns the condition with the provided type.
 func GetClusterExternalSecretCondition(status esv1beta1.ClusterExternalSecretStatus, condType esv1beta1.ClusterExternalSecretConditionType) *esv1beta1.ClusterExternalSecretStatusCondition {
 	for i := range status.Conditions {
 		c := status.Conditions[i]
@@ -40,6 +48,7 @@ func GetClusterExternalSecretCondition(status esv1beta1.ClusterExternalSecretSta
 
 func SetClusterExternalSecretCondition(ces *esv1beta1.ClusterExternalSecret, condition esv1beta1.ClusterExternalSecretStatusCondition) {
 	ces.Status.Conditions = append(filterOutCondition(ces.Status.Conditions, condition.Type), condition)
+	cesmetrics.UpdateClusterExternalSecretCondition(ces, &condition)
 }
 
 // filterOutCondition returns an empty set of conditions with the provided type.
@@ -63,3 +72,15 @@ func ContainsNamespace(namespaces v1.NamespaceList, namespace string) bool {
 
 	return false
 }
+
+func getConditionType(failedNamespaces map[string]string, namespaceList *v1.NamespaceList) esv1beta1.ClusterExternalSecretConditionType {
+	if len(failedNamespaces) == 0 {
+		return esv1beta1.ClusterExternalSecretReady
+	}
+
+	if len(failedNamespaces) < len(namespaceList.Items) {
+		return esv1beta1.ClusterExternalSecretPartiallyReady
+	}
+
+	return esv1beta1.ClusterExternalSecretNotReady
+}