fix: metrics not being correctly updated and deleted (#5714)

pkg/controllers/clusterexternalsecret/cesmetrics/cesmetrics.go

@@ -79,10 +79,13 @@ func UpdateClusterExternalSecretCondition(ces *esv1.ClusterExternalSecret, condi
 	conditionLabels := ctrlmetrics.RefineConditionMetricLabels(cesInfo)
 	clusterExternalSecretCondition := GetGaugeVec(ClusterExternalSecretStatusConditionKey)
 
-	theOtherStatus := v1.ConditionFalse
-	if condition.Status == v1.ConditionFalse {
-		theOtherStatus = v1.ConditionTrue
+	// This handles cases where labels may have changed
+	baseLabels := prometheus.Labels{
+		"name":      ces.Name,
+		"condition": string(condition.Type),
+		"status":    string(v1.ConditionFalse),
 	}
+	clusterExternalSecretCondition.DeletePartialMatch(baseLabels)
 
 	clusterExternalSecretCondition.With(ctrlmetrics.RefineLabels(conditionLabels,
 		map[string]string{
@@ -92,7 +95,7 @@ func UpdateClusterExternalSecretCondition(ces *esv1.ClusterExternalSecret, condi
 	clusterExternalSecretCondition.With(ctrlmetrics.RefineLabels(conditionLabels,
 		map[string]string{
 			"condition": string(condition.Type),
-			"status":    string(theOtherStatus),
+			"status":    string(v1.ConditionFalse),
 		})).Set(0)
 }
 

pkg/controllers/clusterpushsecret/cpsmetrics/cpsmetrics.go

@@ -78,10 +78,13 @@ func UpdateClusterPushSecretCondition(ces *v1alpha1.ClusterPushSecret, condition
 	conditionLabels := ctrlmetrics.RefineConditionMetricLabels(cesInfo)
 	ClusterPushSecretCondition := GetGaugeVec(ClusterPushSecretStatusConditionKey)
 
-	theOtherStatus := v1.ConditionFalse
-	if condition.Status == v1.ConditionFalse {
-		theOtherStatus = v1.ConditionTrue
+	// This handles cases where labels may have changed
+	baseLabels := prometheus.Labels{
+		"name":      ces.Name,
+		"condition": string(condition.Type),
+		"status":    string(v1.ConditionFalse),
 	}
+	ClusterPushSecretCondition.DeletePartialMatch(baseLabels)
 
 	ClusterPushSecretCondition.With(ctrlmetrics.RefineLabels(conditionLabels,
 		map[string]string{
@@ -91,7 +94,7 @@ func UpdateClusterPushSecretCondition(ces *v1alpha1.ClusterPushSecret, condition
 	ClusterPushSecretCondition.With(ctrlmetrics.RefineLabels(conditionLabels,
 		map[string]string{
 			"condition": string(condition.Type),
-			"status":    string(theOtherStatus),
+			"status":    string(v1.ConditionFalse),
 		})).Set(0)
 }
 

pkg/controllers/externalsecret/esmetrics/esmetrics.go

@@ -95,44 +95,61 @@ func UpdateExternalSecretCondition(es *esv1.ExternalSecret, condition *esv1.Exte
 	conditionLabels := ctrlmetrics.RefineConditionMetricLabels(esInfo)
 	externalSecretCondition := GetGaugeVec(ExternalSecretStatusConditionKey)
 
+	// this allows us to delete metrics even when other labels (like helm annotations) have changed
+	baseLabels := prometheus.Labels{
+		"name":      es.Name,
+		"namespace": es.Namespace,
+	}
+
 	switch condition.Type {
 	case esv1.ExternalSecretDeleted:
 		// Remove condition=Ready metrics when the object gets deleted.
-		externalSecretCondition.Delete(ctrlmetrics.RefineLabels(conditionLabels,
-			map[string]string{
-				"condition": string(esv1.ExternalSecretReady),
-				"status":    string(v1.ConditionFalse),
-			}))
-
-		externalSecretCondition.Delete(ctrlmetrics.RefineLabels(conditionLabels,
-			map[string]string{
-				"condition": string(esv1.ExternalSecretReady),
-				"status":    string(v1.ConditionTrue),
-			}))
+		baseLabels["condition"] = string(esv1.ExternalSecretReady)
+		baseLabels["status"] = string(v1.ConditionFalse)
+		externalSecretCondition.DeletePartialMatch(baseLabels)
+
+		baseLabels["status"] = string(v1.ConditionTrue)
+		externalSecretCondition.DeletePartialMatch(baseLabels)
+		delete(baseLabels, "condition")
+		delete(baseLabels, "status")
 
 	case esv1.ExternalSecretReady:
 		// Remove condition=Deleted metrics when the object gets ready.
-		externalSecretCondition.Delete(ctrlmetrics.RefineLabels(conditionLabels,
-			map[string]string{
-				"condition": string(esv1.ExternalSecretDeleted),
-				"status":    string(v1.ConditionFalse),
-			}))
-
-		externalSecretCondition.Delete(ctrlmetrics.RefineLabels(conditionLabels,
-			map[string]string{
-				"condition": string(esv1.ExternalSecretDeleted),
-				"status":    string(v1.ConditionTrue),
-			}))
-
-		// Toggle opposite Status to 0
+		baseLabels["condition"] = string(esv1.ExternalSecretDeleted)
+		baseLabels["status"] = string(v1.ConditionFalse)
+		externalSecretCondition.DeletePartialMatch(baseLabels)
+
+		baseLabels["status"] = string(v1.ConditionTrue)
+		externalSecretCondition.DeletePartialMatch(baseLabels)
+		delete(baseLabels, "condition")
+		delete(baseLabels, "status")
+
+		// Toggle opposite Status to 0, but first delete any stale metrics with old labels
 		switch condition.Status {
 		case v1.ConditionFalse:
+			// delete any existing metrics with status True (regardless of other labels)
+			// condition is fixed to ExternalSecretReady because other statuses were already handled above.
+			baseLabels["condition"] = string(esv1.ExternalSecretReady)
+			baseLabels["status"] = string(v1.ConditionTrue)
+			externalSecretCondition.DeletePartialMatch(baseLabels)
+			delete(baseLabels, "condition")
+			delete(baseLabels, "status")
+
+			// Set the metric with current labels
 			externalSecretCondition.With(ctrlmetrics.RefineLabels(conditionLabels,
 				map[string]string{
 					"condition": string(esv1.ExternalSecretReady),
 					"status":    string(v1.ConditionTrue),
 				})).Set(0)
 		case v1.ConditionTrue:
+			// delete any existing metrics with status False (regardless of other labels)
+			baseLabels["condition"] = string(esv1.ExternalSecretReady)
+			baseLabels["status"] = string(v1.ConditionFalse)
+			externalSecretCondition.DeletePartialMatch(baseLabels)
+			delete(baseLabels, "condition")
+			delete(baseLabels, "status")
+
+			// finally, set the metric with current labels
 			externalSecretCondition.With(ctrlmetrics.RefineLabels(conditionLabels,
 				map[string]string{
 					"condition": string(esv1.ExternalSecretReady),

pkg/controllers/pushsecret/psmetrics/psmetrics.go

@@ -73,17 +73,39 @@ func UpdatePushSecretCondition(ps *esapi.PushSecret, condition *esapi.PushSecret
 	conditionLabels := ctrlmetrics.RefineConditionMetricLabels(psInfo)
 	pushSecretCondition := GetGaugeVec(PushSecretStatusConditionKey)
 
+	// This allows us to delete metrics even when other labels (like helm annotations) have changed
+	baseLabels := prometheus.Labels{
+		"name":      ps.Name,
+		"namespace": ps.Namespace,
+	}
+
 	switch condition.Type {
 	case esapi.PushSecretReady:
-		// Toggle opposite Status to 0
+		// Toggle opposite Status to 0, but first delete any stale metrics with old labels
 		switch condition.Status {
 		case v1.ConditionFalse:
+			// delete any existing metrics with status True (regardless of other labels)
+			baseLabels["condition"] = string(esapi.PushSecretReady)
+			baseLabels["status"] = string(v1.ConditionTrue)
+			pushSecretCondition.DeletePartialMatch(baseLabels)
+			delete(baseLabels, "condition")
+			delete(baseLabels, "status")
+
+			// Set the metric with current labels
 			pushSecretCondition.With(ctrlmetrics.RefineLabels(conditionLabels,
 				map[string]string{
 					"condition": string(esapi.PushSecretReady),
 					"status":    string(v1.ConditionTrue),
 				})).Set(0)
 		case v1.ConditionTrue:
+			// delete any existing metrics with status False (regardless of other labels)
+			baseLabels["condition"] = string(esapi.PushSecretReady)
+			baseLabels["status"] = string(v1.ConditionFalse)
+			pushSecretCondition.DeletePartialMatch(baseLabels)
+			delete(baseLabels, "condition")
+			delete(baseLabels, "status")
+
+			// finally, set the metric with current labels
 			pushSecretCondition.With(ctrlmetrics.RefineLabels(conditionLabels,
 				map[string]string{
 					"condition": string(esapi.PushSecretReady),