feat: add provider metrics (#2024)

* feat: add provider metrics

This adds a counter metric `provider_api_calls_count` that observes
the results of upstream secret provider api calls.

(1) Observability
It allows an user to break down issues by provider and api call by
observing the status=error|success label. More details around the error
can be found in  the logs.

(2) Cost Management
Some providers charge by API calls issued. By providing observability
for the number of calls issued helps users to understand the impact of
deploying ESO and fine-tuning `spec.refreshInterval`.

(3) Rate Limiting
Some providers implement rate-limiting for their services. Having
metrics
for success/failure count helps to understand how many requests are
issued by a given ESO deployment per cluster.

Signed-off-by: Moritz Johner <beller.moritz@googlemail.com>

* fix: add service monitor for cert-controller and add SLIs

Signed-off-by: Moritz Johner <beller.moritz@googlemail.com>

---------

Signed-off-by: Moritz Johner <beller.moritz@googlemail.com>

deploy/charts/external-secrets/templates/_helpers.tpl

@@ -51,6 +51,11 @@ app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
 app.kubernetes.io/managed-by: {{ .Release.Service }}
 {{- end }}
 
+{{- define "external-secrets-webhook-metrics.labels" -}}
+{{ include "external-secrets-webhook.selectorLabels" . }}
+app.kubernetes.io/metrics: "webhook"
+{{- end }}
+
 {{- define "external-secrets-cert-controller.labels" -}}
 helm.sh/chart: {{ include "external-secrets.chart" . }}
 {{ include "external-secrets-cert-controller.selectorLabels" . }}
@@ -60,6 +65,11 @@ app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
 app.kubernetes.io/managed-by: {{ .Release.Service }}
 {{- end }}
 
+{{- define "external-secrets-cert-controller-metrics.labels" -}}
+{{ include "external-secrets-cert-controller.selectorLabels" . }}
+app.kubernetes.io/metrics: "cert-controller"
+{{- end }}
+
 {{/*
 Selector labels
 */}}

deploy/charts/external-secrets/templates/servicemonitor.yaml

@@ -45,4 +45,101 @@ spec:
     relabelings:
       {{- toYaml . | nindent 6 }}
     {{- end }}
+---
+{{- if .Values.webhook.create }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "external-secrets.fullname" . }}-webhook-metrics
+  namespace: {{ .Release.Namespace | quote }}
+  labels:
+    {{- include "external-secrets-webhook-metrics.labels" . | nindent 4 }}
+spec:
+  type: ClusterIP
+  ports:
+    - port: 8080
+      protocol: TCP
+      name: metrics
+  selector:
+    {{- include "external-secrets-webhook.selectorLabels" . | nindent 4 }}
+---
+apiVersion: "monitoring.coreos.com/v1"
+kind: ServiceMonitor
+metadata:
+  labels:
+    {{- include "external-secrets-webhook.labels" . | nindent 4 }}
+{{- if .Values.serviceMonitor.additionalLabels }}
+{{ toYaml .Values.serviceMonitor.additionalLabels | indent 4 }}
+{{- end }}
+  name: {{ include "external-secrets.fullname" . }}-webhook-metrics
+  namespace: {{ .Release.Namespace | quote }}
+spec:
+  selector:
+    matchLabels:
+      {{- include "external-secrets-webhook-metrics.labels" . | nindent 6 }}
+  namespaceSelector:
+    matchNames:
+    - {{ .Release.Namespace | quote }}
+  endpoints:
+  - port: metrics
+    interval: {{ .Values.serviceMonitor.interval }}
+    scrapeTimeout: {{ .Values.serviceMonitor.scrapeTimeout }}
+    honorLabels: {{ .Values.serviceMonitor.honorLabels }}
+    {{- with .Values.serviceMonitor.metricRelabelings }}
+    metricRelabelings:
+      {{- toYaml . | nindent 6 }}
+    {{- end }}
+    {{- with .Values.serviceMonitor.relabelings }}
+    relabelings:
+      {{- toYaml . | nindent 6 }}
+    {{- end }}
+{{- end }}
+{{- if .Values.certController.create }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "external-secrets.fullname" . }}-cert-controller-metrics
+  namespace: {{ .Release.Namespace | quote }}
+  labels:
+    {{- include "external-secrets-cert-controller-metrics.labels" . | nindent 4 }}
+spec:
+  type: ClusterIP
+  ports:
+    - port: 8080
+      protocol: TCP
+      name: metrics
+  selector:
+    {{- include "external-secrets-cert-controller.selectorLabels" . | nindent 4 }}
+---
+apiVersion: "monitoring.coreos.com/v1"
+kind: ServiceMonitor
+metadata:
+  labels:
+    {{- include "external-secrets-cert-controller.labels" . | nindent 4 }}
+{{- if .Values.serviceMonitor.additionalLabels }}
+{{ toYaml .Values.serviceMonitor.additionalLabels | indent 4 }}
+{{- end }}
+  name: {{ include "external-secrets.fullname" . }}-cert-controller-metrics
+  namespace: {{ .Release.Namespace | quote }}
+spec:
+  selector:
+    matchLabels:
+      {{- include "external-secrets-cert-controller-metrics.labels" . | nindent 6 }}
+  namespaceSelector:
+    matchNames:
+    - {{ .Release.Namespace | quote }}
+  endpoints:
+  - port: metrics
+    interval: {{ .Values.serviceMonitor.interval }}
+    scrapeTimeout: {{ .Values.serviceMonitor.scrapeTimeout }}
+    honorLabels: {{ .Values.serviceMonitor.honorLabels }}
+    {{- with .Values.serviceMonitor.metricRelabelings }}
+    metricRelabelings:
+      {{- toYaml . | nindent 6 }}
+    {{- end }}
+    {{- with .Values.serviceMonitor.relabelings }}
+    relabelings:
+      {{- toYaml . | nindent 6 }}
+    {{- end }}
+{{- end }}
 {{- end }}

docs/api/metrics.md

@@ -15,6 +15,7 @@ The Operator has the metrics inherited from Kubebuilder plus some custom metrics
 
 | Name                                           | Type      | Description                                                                                                                                                                                                            |
 | ---------------------------------------------- | --------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `externalsecret_provider_api_calls_count`      | Counter   | Number of API calls made to an upstream secret provider API. The metric provides a `provider`, `call` and `status` labels.                                                                                             |
 | `externalsecret_sync_calls_total`              | Counter   | Total number of the External Secret sync calls                                                                                                                                                                         |
 | `externalsecret_sync_calls_error`              | Counter   | Total number of the External Secret sync errors                                                                                                                                                                        |
 | `externalsecret_status_condition`              | Gauge     | The status condition of a specific External Secret                                                                                                                                                                     |
@@ -25,3 +26,67 @@ The Operator has the metrics inherited from Kubebuilder plus some custom metrics
 | `controller_runtime_reconcile_queue_length`    | Gauge     | Length of reconcile queue per controller                                                                                                                                                                               |
 | `controller_runtime_max_concurrent_reconciles` | Gauge     | Maximum number of concurrent reconciles per controller                                                                                                                                                                 |
 | `controller_runtime_active_workers`            | Gauge     | Number of currently used workers per controller                                                                                                                                                                        |
+
+## Dashboard
+
+We provide a [Grafana Dashboard](https://raw.githubusercontent.com/external-secrets/external-secrets/main/docs/snippets/dashboard.json) that gives you an overview of External Secrets Operator:
+
+![ESO Dashboard](../pictures/eso-dashboard-1.png)
+![ESO Dashboard](../pictures/eso-dashboard-2.png)
+
+
+## Service Level Indicators and Alerts
+
+We find the following Service Level Indicators (SLIs) useful when operating ESO. They should give you a good starting point and hints to develop your own Service Level Objectives (SLOs).
+
+#### Webhook HTTP Status Codes
+The webhook HTTP status code indicates that a HTTP Request was answered successfully or not.
+If the Webhook pod is not able to serve the requests properly then that failure may cascade down to the controller or any other user of `kube-apiserver`.
+
+SLI Example: request error percentage.
+```
+sum(increase(controller_runtime_webhook_requests_total{service=~"external-secrets.*",code="500"}[1m]))
+/
+sum(increase(controller_runtime_webhook_requests_total{service=~"external-secrets.*"}[1m]))
+```
+
+#### Webhook HTTP Request Latency
+If the webhook server is not able to respond in time then that may cause a timeout at the client.
+This failure may cascade down to the controller or any other user of `kube-apiserver`.
+
+SLI Example: p99 across all webhook requests.
+```
+histogram_quantile(0.99,
+  sum(rate(controller_runtime_webhook_latency_seconds_bucket{service=~"external-secrets.*"}[5m])) by (le)
+)
+```
+
+#### Controller Workqueue Depth
+If the workqueue depth is > 0 for a longer period of time then this is an indicator for the controller not being able to reconcile resources in time. I.e. delivery of secret updates is delayed.
+
+Note: when a controller is restarted, then `queue length = total number of resources`. Make sure to measure the time it takes for the controller to fully reconcile all secrets after a restart. In large clusters this may take a while, make sure to define an acceptable timeframe to fully reconcile all resources.
+
+```
+sum(
+  workqueue_depth{service=~"external-secrets.*"}
+) by (name)
+```
+
+#### Controller Reconcile Latency
+The controller should be able to reconcile resources within a reasonable timeframe. When latency is high secret delivery may impacted.
+
+SLI Example: p99 across all controllers.
+```
+histogram_quantile(0.99,
+  sum(rate(controller_runtime_reconcile_time_seconds_bucket{service=~"external-secrets.*"}[5m])) by (le)
+)
+```
+
+#### Controller Reconcile Error
+The controller should be able to reconcile resources without errors. When errors occurr secret delivery may be impacted which could cascade down to the secret consuming applications.
+
+```
+sum(increase(
+  controller_runtime_reconcile_total{service=~"external-secrets.*",controller=~"$controller"}[1m])
+) by (result)
+```

docs/snippets/dashboard.json

@@ -0,0 +1,984 @@
+{
+    "annotations": {
+      "list": [
+        {
+          "builtIn": 1,
+          "datasource": {
+            "type": "grafana",
+            "uid": "-- Grafana --"
+          },
+          "enable": true,
+          "hide": true,
+          "iconColor": "rgba(0, 211, 255, 1)",
+          "name": "Annotations & Alerts",
+          "target": {
+            "limit": 100,
+            "matchAny": false,
+            "tags": [],
+            "type": "dashboard"
+          },
+          "type": "dashboard"
+        }
+      ]
+    },
+    "editable": true,
+    "fiscalYearStartMonth": 0,
+    "graphTooltip": 0,
+    "id": 27,
+    "links": [],
+    "liveNow": false,
+    "panels": [
+      {
+        "gridPos": {
+          "h": 1,
+          "w": 24,
+          "x": 0,
+          "y": 0
+        },
+        "id": 27,
+        "title": "Admission Control Webhook",
+        "type": "row"
+      },
+      {
+        "datasource": {
+          "type": "prometheus",
+          "uid": "prometheus"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "palette-classic"
+            },
+            "custom": {
+              "axisCenteredZero": false,
+              "axisColorMode": "text",
+              "axisLabel": "",
+              "axisPlacement": "auto",
+              "barAlignment": 0,
+              "drawStyle": "line",
+              "fillOpacity": 0,
+              "gradientMode": "none",
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              },
+              "lineInterpolation": "linear",
+              "lineWidth": 1,
+              "pointSize": 5,
+              "scaleDistribution": {
+                "type": "linear"
+              },
+              "showPoints": "auto",
+              "spanNulls": false,
+              "stacking": {
+                "group": "A",
+                "mode": "none"
+              },
+              "thresholdsStyle": {
+                "mode": "off"
+              }
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            }
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 8,
+          "w": 12,
+          "x": 0,
+          "y": 1
+        },
+        "id": 53,
+        "options": {
+          "legend": {
+            "calcs": [],
+            "displayMode": "list",
+            "placement": "bottom",
+            "showLegend": true
+          },
+          "tooltip": {
+            "mode": "single",
+            "sort": "none"
+          }
+        },
+        "targets": [
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "prometheus"
+            },
+            "editorMode": "code",
+            "expr": "sum(increase(controller_runtime_webhook_requests_total{service=~\"external-secrets.*\"}[1m])) by (webhook)",
+            "legendFormat": "__auto",
+            "range": true,
+            "refId": "A"
+          }
+        ],
+        "title": "requests by path per minute",
+        "type": "timeseries"
+      },
+      {
+        "datasource": {
+          "type": "prometheus",
+          "uid": "prometheus"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "palette-classic"
+            },
+            "custom": {
+              "axisCenteredZero": false,
+              "axisColorMode": "text",
+              "axisLabel": "",
+              "axisPlacement": "auto",
+              "barAlignment": 0,
+              "drawStyle": "line",
+              "fillOpacity": 0,
+              "gradientMode": "none",
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              },
+              "lineInterpolation": "linear",
+              "lineWidth": 1,
+              "pointSize": 5,
+              "scaleDistribution": {
+                "type": "linear"
+              },
+              "showPoints": "auto",
+              "spanNulls": false,
+              "stacking": {
+                "group": "A",
+                "mode": "none"
+              },
+              "thresholdsStyle": {
+                "mode": "off"
+              }
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            }
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 8,
+          "w": 12,
+          "x": 12,
+          "y": 1
+        },
+        "id": 67,
+        "options": {
+          "legend": {
+            "calcs": [],
+            "displayMode": "list",
+            "placement": "bottom",
+            "showLegend": true
+          },
+          "tooltip": {
+            "mode": "single",
+            "sort": "none"
+          }
+        },
+        "targets": [
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "prometheus"
+            },
+            "editorMode": "code",
+            "expr": "sum(controller_runtime_webhook_requests_in_flight{service=~\"external-secrets.*\"}) by (webhook)",
+            "legendFormat": "__auto",
+            "range": true,
+            "refId": "A"
+          }
+        ],
+        "title": "requests in flight",
+        "type": "timeseries"
+      },
+      {
+        "datasource": {
+          "type": "prometheus",
+          "uid": "prometheus"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "palette-classic"
+            },
+            "custom": {
+              "axisCenteredZero": false,
+              "axisColorMode": "text",
+              "axisLabel": "",
+              "axisPlacement": "auto",
+              "barAlignment": 0,
+              "drawStyle": "line",
+              "fillOpacity": 0,
+              "gradientMode": "none",
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              },
+              "lineInterpolation": "linear",
+              "lineWidth": 1,
+              "pointSize": 5,
+              "scaleDistribution": {
+                "type": "linear"
+              },
+              "showPoints": "auto",
+              "spanNulls": false,
+              "stacking": {
+                "group": "A",
+                "mode": "none"
+              },
+              "thresholdsStyle": {
+                "mode": "off"
+              }
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            }
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 8,
+          "w": 12,
+          "x": 0,
+          "y": 9
+        },
+        "id": 80,
+        "options": {
+          "legend": {
+            "calcs": [],
+            "displayMode": "list",
+            "placement": "bottom",
+            "showLegend": true
+          },
+          "tooltip": {
+            "mode": "single",
+            "sort": "none"
+          }
+        },
+        "targets": [
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "prometheus"
+            },
+            "editorMode": "code",
+            "expr": "sum(increase(controller_runtime_webhook_requests_total{service=~\"external-secrets.*\"}[1m])) by (code)",
+            "legendFormat": "__auto",
+            "range": true,
+            "refId": "A"
+          }
+        ],
+        "title": "requests by code per minute",
+        "type": "timeseries"
+      },
+      {
+        "datasource": {
+          "type": "prometheus",
+          "uid": "prometheus"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "custom": {
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              },
+              "scaleDistribution": {
+                "type": "linear"
+              }
+            }
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 8,
+          "w": 12,
+          "x": 12,
+          "y": 9
+        },
+        "id": 54,
+        "options": {
+          "calculate": false,
+          "cellGap": 1,
+          "color": {
+            "exponent": 0.5,
+            "fill": "dark-orange",
+            "mode": "scheme",
+            "reverse": false,
+            "scale": "exponential",
+            "scheme": "Oranges",
+            "steps": 64
+          },
+          "exemplars": {
+            "color": "rgba(255,0,255,0.7)"
+          },
+          "filterValues": {
+            "le": 1e-9
+          },
+          "legend": {
+            "show": true
+          },
+          "rowsFrame": {
+            "layout": "auto"
+          },
+          "tooltip": {
+            "show": true,
+            "yHistogram": false
+          },
+          "yAxis": {
+            "axisPlacement": "left",
+            "reverse": false
+          }
+        },
+        "pluginVersion": "9.3.1",
+        "targets": [
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "prometheus"
+            },
+            "editorMode": "code",
+            "expr": "sum(rate(controller_runtime_webhook_latency_seconds_bucket{service=~\"external-secrets.*\"}[$__rate_interval])) by (le)",
+            "legendFormat": "{{le}}",
+            "range": true,
+            "refId": "A"
+          }
+        ],
+        "title": "webhook latency",
+        "type": "heatmap"
+      },
+      {
+        "collapsed": false,
+        "gridPos": {
+          "h": 1,
+          "w": 24,
+          "x": 0,
+          "y": 17
+        },
+        "id": 17,
+        "panels": [],
+        "title": "Controllers",
+        "type": "row"
+      },
+      {
+        "datasource": {
+          "type": "prometheus",
+          "uid": "prometheus"
+        },
+        "description": "",
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "thresholds"
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            }
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 6,
+          "w": 3.4285714285714284,
+          "x": 0,
+          "y": 18
+        },
+        "id": 5,
+        "maxPerRow": 12,
+        "options": {
+          "colorMode": "value",
+          "graphMode": "area",
+          "justifyMode": "auto",
+          "orientation": "auto",
+          "reduceOptions": {
+            "calcs": [
+              "lastNotNull"
+            ],
+            "fields": "",
+            "values": false
+          },
+          "textMode": "auto"
+        },
+        "pluginVersion": "9.3.1",
+        "repeat": "controller",
+        "repeatDirection": "h",
+        "targets": [
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "prometheus"
+            },
+            "editorMode": "code",
+            "expr": "sum(controller_runtime_max_concurrent_reconciles{service=~\"external-secrets.*\",controller=\"$controller\"}) by (controller)",
+            "legendFormat": "__auto",
+            "range": true,
+            "refId": "A"
+          }
+        ],
+        "title": "max concurrent: $controller",
+        "type": "stat"
+      },
+      {
+        "datasource": {
+          "type": "prometheus",
+          "uid": "prometheus"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "palette-classic"
+            },
+            "custom": {
+              "axisCenteredZero": false,
+              "axisColorMode": "text",
+              "axisLabel": "",
+              "axisPlacement": "auto",
+              "barAlignment": 0,
+              "drawStyle": "line",
+              "fillOpacity": 0,
+              "gradientMode": "none",
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              },
+              "lineInterpolation": "linear",
+              "lineWidth": 1,
+              "pointSize": 5,
+              "scaleDistribution": {
+                "type": "linear"
+              },
+              "showPoints": "auto",
+              "spanNulls": false,
+              "stacking": {
+                "group": "A",
+                "mode": "none"
+              },
+              "thresholdsStyle": {
+                "mode": "off"
+              }
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            }
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 8,
+          "w": 3.4285714285714284,
+          "x": 0,
+          "y": 24
+        },
+        "id": 3,
+        "maxPerRow": 8,
+        "options": {
+          "legend": {
+            "calcs": [],
+            "displayMode": "list",
+            "placement": "bottom",
+            "showLegend": true
+          },
+          "tooltip": {
+            "mode": "single",
+            "sort": "none"
+          }
+        },
+        "repeat": "controller",
+        "repeatDirection": "h",
+        "targets": [
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "prometheus"
+            },
+            "editorMode": "code",
+            "expr": "sum(increase(controller_runtime_reconcile_total{service=~\"external-secrets.*\",controller=~\"$controller\"}[1m])) by (result)",
+            "legendFormat": "__auto",
+            "range": true,
+            "refId": "A"
+          }
+        ],
+        "title": "reconcile rate per minute: $controller",
+        "type": "timeseries"
+      },
+      {
+        "datasource": {
+          "type": "prometheus",
+          "uid": "prometheus"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "palette-classic"
+            },
+            "custom": {
+              "axisCenteredZero": false,
+              "axisColorMode": "text",
+              "axisLabel": "",
+              "axisPlacement": "auto",
+              "barAlignment": 0,
+              "drawStyle": "line",
+              "fillOpacity": 0,
+              "gradientMode": "none",
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              },
+              "lineInterpolation": "linear",
+              "lineWidth": 1,
+              "pointSize": 5,
+              "scaleDistribution": {
+                "type": "linear"
+              },
+              "showPoints": "auto",
+              "spanNulls": false,
+              "stacking": {
+                "group": "A",
+                "mode": "none"
+              },
+              "thresholdsStyle": {
+                "mode": "off"
+              }
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            }
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 9,
+          "w": 8,
+          "x": 0,
+          "y": 32
+        },
+        "id": 2,
+        "options": {
+          "legend": {
+            "calcs": [],
+            "displayMode": "list",
+            "placement": "bottom",
+            "showLegend": true
+          },
+          "tooltip": {
+            "mode": "single",
+            "sort": "none"
+          }
+        },
+        "targets": [
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "prometheus"
+            },
+            "editorMode": "code",
+            "expr": "sum(controller_runtime_active_workers{service=~\"external-secrets.*\",controller=~\"$controller\"}) by (controller)",
+            "legendFormat": "__auto",
+            "range": true,
+            "refId": "A"
+          }
+        ],
+        "title": "active workers by controller",
+        "type": "timeseries"
+      },
+      {
+        "datasource": {
+          "type": "prometheus",
+          "uid": "prometheus"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "palette-classic"
+            },
+            "custom": {
+              "axisCenteredZero": false,
+              "axisColorMode": "text",
+              "axisLabel": "",
+              "axisPlacement": "auto",
+              "barAlignment": 0,
+              "drawStyle": "line",
+              "fillOpacity": 0,
+              "gradientMode": "none",
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              },
+              "lineInterpolation": "linear",
+              "lineWidth": 1,
+              "pointSize": 5,
+              "scaleDistribution": {
+                "type": "linear"
+              },
+              "showPoints": "auto",
+              "spanNulls": false,
+              "stacking": {
+                "group": "A",
+                "mode": "none"
+              },
+              "thresholdsStyle": {
+                "mode": "off"
+              }
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            }
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 9,
+          "w": 7,
+          "x": 8,
+          "y": 32
+        },
+        "id": 37,
+        "options": {
+          "legend": {
+            "calcs": [],
+            "displayMode": "list",
+            "placement": "bottom",
+            "showLegend": true
+          },
+          "tooltip": {
+            "mode": "single",
+            "sort": "none"
+          }
+        },
+        "targets": [
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "prometheus"
+            },
+            "editorMode": "code",
+            "expr": "sum(workqueue_depth{service=~\"external-secrets.*\"}) by (name)",
+            "legendFormat": "__auto",
+            "range": true,
+            "refId": "A"
+          }
+        ],
+        "title": "workqueue depth",
+        "type": "timeseries"
+      },
+      {
+        "datasource": {
+          "type": "prometheus",
+          "uid": "prometheus"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "palette-classic"
+            },
+            "custom": {
+              "axisCenteredZero": false,
+              "axisColorMode": "text",
+              "axisLabel": "",
+              "axisPlacement": "auto",
+              "barAlignment": 0,
+              "drawStyle": "line",
+              "fillOpacity": 0,
+              "gradientMode": "none",
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              },
+              "lineInterpolation": "linear",
+              "lineWidth": 1,
+              "pointSize": 5,
+              "scaleDistribution": {
+                "type": "linear"
+              },
+              "showPoints": "auto",
+              "spanNulls": false,
+              "stacking": {
+                "group": "A",
+                "mode": "none"
+              },
+              "thresholdsStyle": {
+                "mode": "off"
+              }
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            }
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 9,
+          "w": 9,
+          "x": 15,
+          "y": 32
+        },
+        "id": 15,
+        "options": {
+          "legend": {
+            "calcs": [],
+            "displayMode": "list",
+            "placement": "bottom",
+            "showLegend": true
+          },
+          "tooltip": {
+            "mode": "single",
+            "sort": "none"
+          }
+        },
+        "targets": [
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "prometheus"
+            },
+            "editorMode": "code",
+            "expr": "sum(increase(externalsecret_provider_api_calls_count{service=~\"external-secrets.*\"}[1m])) by(provider, call, status)",
+            "legendFormat": "{{provider}}/{{call}}={{status}}",
+            "range": true,
+            "refId": "A"
+          }
+        ],
+        "title": "API calls by provider",
+        "type": "timeseries"
+      },
+      {
+        "datasource": {
+          "type": "prometheus",
+          "uid": "prometheus"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "custom": {
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              },
+              "scaleDistribution": {
+                "type": "linear"
+              }
+            }
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 8,
+          "w": 3.4285714285714284,
+          "x": 0,
+          "y": 41
+        },
+        "id": 39,
+        "maxPerRow": 8,
+        "options": {
+          "calculate": false,
+          "cellGap": 1,
+          "cellValues": {
+            "unit": "short"
+          },
+          "color": {
+            "exponent": 0.5,
+            "fill": "dark-orange",
+            "mode": "scheme",
+            "reverse": false,
+            "scale": "exponential",
+            "scheme": "Oranges",
+            "steps": 10
+          },
+          "exemplars": {
+            "color": "rgba(255,0,255,0.7)"
+          },
+          "filterValues": {
+            "le": 1e-9
+          },
+          "legend": {
+            "show": true
+          },
+          "rowsFrame": {
+            "layout": "auto"
+          },
+          "tooltip": {
+            "show": true,
+            "yHistogram": false
+          },
+          "yAxis": {
+            "axisPlacement": "left",
+            "max": "5",
+            "min": 0,
+            "reverse": false,
+            "unit": "s"
+          }
+        },
+        "pluginVersion": "9.3.1",
+        "repeat": "controller",
+        "repeatDirection": "h",
+        "targets": [
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "prometheus"
+            },
+            "editorMode": "code",
+            "expr": "rate(controller_runtime_reconcile_time_seconds_bucket{service=~\"external-secrets.*\",controller=~\"$controller\"}[$__rate_interval])",
+            "legendFormat": "{{le}}",
+            "range": true,
+            "refId": "A"
+          }
+        ],
+        "title": "reconcile time latency: $controller",
+        "type": "heatmap"
+      }
+    ],
+    "refresh": false,
+    "schemaVersion": 37,
+    "style": "dark",
+    "tags": [],
+    "templating": {
+      "list": [
+        {
+          "allValue": ".*",
+          "current": {
+            "selected": false,
+            "text": [
+              "All"
+            ],
+            "value": [
+              "$__all"
+            ]
+          },
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "definition": "label_values(controller_runtime_active_workers{service=~\"external-secrets.*\"},  controller)",
+          "hide": 0,
+          "includeAll": true,
+          "multi": true,
+          "name": "controller",
+          "options": [],
+          "query": {
+            "query": "label_values(controller_runtime_active_workers{service=~\"external-secrets.*\"},  controller)",
+            "refId": "StandardVariableQuery"
+          },
+          "refresh": 1,
+          "regex": "",
+          "skipUrlSync": false,
+          "sort": 0,
+          "type": "query"
+        }
+      ]
+    },
+    "time": {
+      "from": "now-6h",
+      "to": "now"
+    },
+    "timepicker": {},
+    "timezone": "",
+    "title": "External Secrets Operator",
+    "uid": "n4IdKaJVk",
+    "version": 25,
+    "weekStart": ""
+  }

pkg/provider/aws/parameterstore/parameterstore.go

@@ -31,6 +31,7 @@ import (
 	esv1beta1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1beta1"
 	"github.com/external-secrets/external-secrets/pkg/find"
 	"github.com/external-secrets/external-secrets/pkg/provider/aws/util"
+	"github.com/external-secrets/external-secrets/pkg/provider/metrics"
 )
 
 // https://github.com/external-secrets/external-secrets/issues/644
@@ -79,7 +80,7 @@ func (pm *ParameterStore) getTagsByName(ctx aws.Context, ref *ssm.GetParameterOu
 	}
 
 	data, err := pm.client.ListTagsForResourceWithContext(ctx, &parameterTags)
-
+	metrics.ObserveAPICall(metrics.ProviderAWSPS, metrics.CallAWSPSListTagsForResource, err)
 	if err != nil {
 		return nil, fmt.Errorf("error listing tags %w", err)
 	}
@@ -93,6 +94,7 @@ func (pm *ParameterStore) DeleteSecret(ctx context.Context, remoteRef esv1beta1.
 		Name: &secretName,
 	}
 	existing, err := pm.client.GetParameterWithContext(ctx, &secretValue)
+	metrics.ObserveAPICall(metrics.ProviderAWSPS, metrics.CallAWSPSGetParameter, err)
 	var awsError awserr.Error
 	ok := errors.As(err, &awsError)
 	if err != nil && (!ok || awsError.Code() != ssm.ErrCodeParameterNotFound) {
@@ -115,6 +117,7 @@ func (pm *ParameterStore) DeleteSecret(ctx context.Context, remoteRef esv1beta1.
 			Name: &secretName,
 		}
 		_, err = pm.client.DeleteParameterWithContext(ctx, deleteInput)
+		metrics.ObserveAPICall(metrics.ProviderAWSPS, metrics.CallAWSPSDeleteParameter, err)
 		if err != nil {
 			return fmt.Errorf("could not delete parameter %v: %w", secretName, err)
 		}
@@ -141,6 +144,7 @@ func (pm *ParameterStore) PushSecret(ctx context.Context, value []byte, remoteRe
 	}
 
 	existing, err := pm.client.GetParameterWithContext(ctx, &secretValue)
+	metrics.ObserveAPICall(metrics.ProviderAWSPS, metrics.CallAWSPSGetParameter, err)
 	var awsError awserr.Error
 	ok := errors.As(err, &awsError)
 	if err != nil && (!ok || awsError.Code() != ssm.ErrCodeParameterNotFound) {
@@ -196,6 +200,7 @@ func (pm *ParameterStore) setManagedRemoteParameter(ctx context.Context, secretR
 	}
 
 	_, err := pm.client.PutParameterWithContext(ctx, &secretRequest)
+	metrics.ObserveAPICall(metrics.ProviderAWSPS, metrics.CallAWSPSPutParameter, err)
 	if err != nil {
 		return fmt.Errorf("unexpected error pushing parameter %v: %w", secretRequest.Name, err)
 	}
@@ -235,6 +240,7 @@ func (pm *ParameterStore) findByName(ctx context.Context, ref esv1beta1.External
 				NextToken:        nextToken,
 				ParameterFilters: pathFilter,
 			})
+		metrics.ObserveAPICall(metrics.ProviderAWSPS, metrics.CallAWSPSDescribeParameter, err)
 		if err != nil {
 			return nil, err
 		}
@@ -283,6 +289,7 @@ func (pm *ParameterStore) findByTags(ctx context.Context, ref esv1beta1.External
 				ParameterFilters: filters,
 				NextToken:        nextToken,
 			})
+		metrics.ObserveAPICall(metrics.ProviderAWSPS, metrics.CallAWSPSDescribeParameter, err)
 		if err != nil {
 			return nil, err
 		}
@@ -306,6 +313,7 @@ func (pm *ParameterStore) fetchAndSet(ctx context.Context, data map[string][]byt
 		Name:           utilpointer.String(name),
 		WithDecryption: aws.Bool(true),
 	})
+	metrics.ObserveAPICall(metrics.ProviderAWSPS, metrics.CallAWSPSGetParameter, err)
 	if err != nil {
 		return util.SanitizeErr(err)
 	}
@@ -320,7 +328,7 @@ func (pm *ParameterStore) GetSecret(ctx context.Context, ref esv1beta1.ExternalS
 		Name:           &ref.Key,
 		WithDecryption: aws.Bool(true),
 	})
-
+	metrics.ObserveAPICall(metrics.ProviderAWSPS, metrics.CallAWSPSGetParameter, err)
 	nsf := esv1beta1.NoSecretError{}
 	var nf *ssm.ParameterNotFound
 	if errors.As(err, &nf) || errors.As(err, &nsf) {

pkg/provider/aws/secretsmanager/secretsmanager.go

@@ -34,6 +34,7 @@ import (
 	esv1beta1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1beta1"
 	"github.com/external-secrets/external-secrets/pkg/find"
 	"github.com/external-secrets/external-secrets/pkg/provider/aws/util"
+	"github.com/external-secrets/external-secrets/pkg/provider/metrics"
 )
 
 // https://github.com/external-secrets/external-secrets/issues/644
@@ -135,6 +136,7 @@ func (sm *SecretsManager) fetch(ctx context.Context, ref esv1beta1.ExternalSecre
 			}
 		}
 		secretOut, err = sm.client.GetSecretValue(getSecretValueInput)
+		metrics.ObserveAPICall(metrics.ProviderAWSSM, metrics.CallAWSSMGetSecretValue, err)
 		var nf *awssm.ResourceNotFoundException
 		if errors.As(err, &nf) {
 			return nil, esv1beta1.NoSecretErr
@@ -171,6 +173,7 @@ func (sm *SecretsManager) DeleteSecret(ctx context.Context, remoteRef esv1beta1.
 		SecretId: &secretName,
 	}
 	awsSecret, err := sm.client.GetSecretValueWithContext(ctx, &secretValue)
+	metrics.ObserveAPICall(metrics.ProviderAWSSM, metrics.CallAWSSMGetSecretValue, err)
 	var aerr awserr.Error
 	if err != nil {
 		if ok := errors.As(err, &aerr); !ok {
@@ -182,6 +185,7 @@ func (sm *SecretsManager) DeleteSecret(ctx context.Context, remoteRef esv1beta1.
 		return err
 	}
 	data, err := sm.client.DescribeSecretWithContext(ctx, &secretInput)
+	metrics.ObserveAPICall(metrics.ProviderAWSSM, metrics.CallAWSSMDescribeSecret, err)
 	if err != nil {
 		return err
 	}
@@ -192,6 +196,7 @@ func (sm *SecretsManager) DeleteSecret(ctx context.Context, remoteRef esv1beta1.
 		SecretId: awsSecret.ARN,
 	}
 	_, err = sm.client.DeleteSecretWithContext(ctx, deleteInput)
+	metrics.ObserveAPICall(metrics.ProviderAWSSM, metrics.CallAWSSMDeleteSecret, err)
 	return err
 }
 
@@ -220,6 +225,7 @@ func (sm *SecretsManager) PushSecret(ctx context.Context, value []byte, remoteRe
 	}
 
 	awsSecret, err := sm.client.GetSecretValueWithContext(ctx, &secretValue)
+	metrics.ObserveAPICall(metrics.ProviderAWSSM, metrics.CallAWSSMGetSecretValue, err)
 	var aerr awserr.Error
 	if err != nil {
 		if ok := errors.As(err, &aerr); !ok {
@@ -227,11 +233,13 @@ func (sm *SecretsManager) PushSecret(ctx context.Context, value []byte, remoteRe
 		}
 		if aerr.Code() == awssm.ErrCodeResourceNotFoundException {
 			_, err = sm.client.CreateSecretWithContext(ctx, &secretRequest)
+			metrics.ObserveAPICall(metrics.ProviderAWSSM, metrics.CallAWSSMCreateSecret, err)
 			return err
 		}
 		return err
 	}
 	data, err := sm.client.DescribeSecretWithContext(ctx, &secretInput)
+	metrics.ObserveAPICall(metrics.ProviderAWSSM, metrics.CallAWSSMDescribeSecret, err)
 	if err != nil {
 		return err
 	}
@@ -246,6 +254,7 @@ func (sm *SecretsManager) PushSecret(ctx context.Context, value []byte, remoteRe
 		SecretBinary: value,
 	}
 	_, err = sm.client.PutSecretValueWithContext(ctx, input)
+	metrics.ObserveAPICall(metrics.ProviderAWSSM, metrics.CallAWSSMPutSecretValue, err)
 	return err
 }
 
@@ -295,6 +304,7 @@ func (sm *SecretsManager) findByName(ctx context.Context, ref esv1beta1.External
 			Filters:   filters,
 			NextToken: nextToken,
 		})
+		metrics.ObserveAPICall(metrics.ProviderAWSSM, metrics.CallAWSSMListSecrets, err)
 		if err != nil {
 			return nil, err
 		}
@@ -350,6 +360,7 @@ func (sm *SecretsManager) findByTags(ctx context.Context, ref esv1beta1.External
 			Filters:   filters,
 			NextToken: nextToken,
 		})
+		metrics.ObserveAPICall(metrics.ProviderAWSSM, metrics.CallAWSSMListSecrets, err)
 		if err != nil {
 			return nil, err
 		}

pkg/provider/azure/keyvault/keyvault.go

@@ -49,6 +49,7 @@ import (
 
 	esv1beta1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1beta1"
 	smmeta "github.com/external-secrets/external-secrets/apis/meta/v1"
+	"github.com/external-secrets/external-secrets/pkg/provider/metrics"
 	"github.com/external-secrets/external-secrets/pkg/utils"
 )
 
@@ -244,12 +245,14 @@ func canDelete(tags map[string]*string, err error) (bool, error) {
 
 func (a *Azure) deleteKeyVaultKey(ctx context.Context, keyName string) error {
 	value, err := a.baseClient.GetKey(ctx, *a.provider.VaultURL, keyName, "")
+	metrics.ObserveAPICall(metrics.ProviderAzureKV, metrics.CallAzureKVGetKey, err)
 	ok, err := canDelete(value.Tags, err)
 	if err != nil {
 		return fmt.Errorf("error getting key %v: %w", keyName, err)
 	}
 	if ok {
 		_, err = a.baseClient.DeleteKey(ctx, *a.provider.VaultURL, keyName)
+		metrics.ObserveAPICall(metrics.ProviderAzureKV, metrics.CallAzureKVDeleteKey, err)
 		if err != nil {
 			return fmt.Errorf("error deleting key %v: %w", keyName, err)
 		}
@@ -259,12 +262,14 @@ func (a *Azure) deleteKeyVaultKey(ctx context.Context, keyName string) error {
 
 func (a *Azure) deleteKeyVaultSecret(ctx context.Context, secretName string) error {
 	value, err := a.baseClient.GetSecret(ctx, *a.provider.VaultURL, secretName, "")
+	metrics.ObserveAPICall(metrics.ProviderAzureKV, metrics.CallAzureKVGetSecret, err)
 	ok, err := canDelete(value.Tags, err)
 	if err != nil {
 		return fmt.Errorf("error getting secret %v: %w", secretName, err)
 	}
 	if ok {
 		_, err = a.baseClient.DeleteSecret(ctx, *a.provider.VaultURL, secretName)
+		metrics.ObserveAPICall(metrics.ProviderAzureKV, metrics.CallAzureKVDeleteSecret, err)
 		if err != nil {
 			return fmt.Errorf("error deleting secret %v: %w", secretName, err)
 		}
@@ -274,12 +279,14 @@ func (a *Azure) deleteKeyVaultSecret(ctx context.Context, secretName string) err
 
 func (a *Azure) deleteKeyVaultCertificate(ctx context.Context, certName string) error {
 	value, err := a.baseClient.GetCertificate(ctx, *a.provider.VaultURL, certName, "")
+	metrics.ObserveAPICall(metrics.ProviderAzureKV, metrics.CallAzureKVGetCertificate, err)
 	ok, err := canDelete(value.Tags, err)
 	if err != nil {
 		return fmt.Errorf("error getting certificate %v: %w", certName, err)
 	}
 	if ok {
 		_, err = a.baseClient.DeleteCertificate(ctx, *a.provider.VaultURL, certName)
+		metrics.ObserveAPICall(metrics.ProviderAzureKV, metrics.CallAzureKVDeleteCertificate, err)
 		if err != nil {
 			return fmt.Errorf("error deleting certificate %v: %w", certName, err)
 		}
@@ -353,6 +360,7 @@ func canCreate(tags map[string]*string, err error) (bool, error) {
 
 func (a *Azure) setKeyVaultSecret(ctx context.Context, secretName string, value []byte) error {
 	secret, err := a.baseClient.GetSecret(ctx, *a.provider.VaultURL, secretName, "")
+	metrics.ObserveAPICall(metrics.ProviderAzureKV, metrics.CallAzureKVGetSecret, err)
 	ok, err := canCreate(secret.Tags, err)
 	if err != nil {
 		return fmt.Errorf("cannot get secret %v: %w", secretName, err)
@@ -374,6 +382,7 @@ func (a *Azure) setKeyVaultSecret(ctx context.Context, secretName string, value
 		},
 	}
 	_, err = a.baseClient.SetSecret(ctx, *a.provider.VaultURL, secretName, secretParams)
+	metrics.ObserveAPICall(metrics.ProviderAzureKV, metrics.CallAzureKVGetSecret, err)
 	if err != nil {
 		return fmt.Errorf("could not set secret %v: %w", secretName, err)
 	}
@@ -387,6 +396,7 @@ func (a *Azure) setKeyVaultCertificate(ctx context.Context, secretName string, v
 		return fmt.Errorf("value from secret is not a valid certificate: %w", err)
 	}
 	cert, err := a.baseClient.GetCertificate(ctx, *a.provider.VaultURL, secretName, "")
+	metrics.ObserveAPICall(metrics.ProviderAzureKV, metrics.CallAzureKVGetCertificate, err)
 	ok, err := canCreate(cert.Tags, err)
 	if err != nil {
 		return fmt.Errorf("cannot get certificate %v: %w", secretName, err)
@@ -405,6 +415,7 @@ func (a *Azure) setKeyVaultCertificate(ctx context.Context, secretName string, v
 		},
 	}
 	_, err = a.baseClient.ImportCertificate(ctx, *a.provider.VaultURL, secretName, params)
+	metrics.ObserveAPICall(metrics.ProviderAzureKV, metrics.CallAzureKVImportCertificate, err)
 	if err != nil {
 		return fmt.Errorf("could not import certificate %v: %w", secretName, err)
 	}
@@ -441,6 +452,7 @@ func (a *Azure) setKeyVaultKey(ctx context.Context, secretName string, value []b
 		return fmt.Errorf("error unmarshalling key: %w", err)
 	}
 	keyFromVault, err := a.baseClient.GetKey(ctx, *a.provider.VaultURL, secretName, "")
+	metrics.ObserveAPICall(metrics.ProviderAzureKV, metrics.CallAzureKVGetKey, err)
 	ok, err := canCreate(keyFromVault.Tags, err)
 	if err != nil {
 		return fmt.Errorf("cannot get key %v: %w", secretName, err)
@@ -459,6 +471,7 @@ func (a *Azure) setKeyVaultKey(ctx context.Context, secretName string, value []b
 		},
 	}
 	_, err = a.baseClient.ImportKey(ctx, *a.provider.VaultURL, secretName, params)
+	metrics.ObserveAPICall(metrics.ProviderAzureKV, metrics.CallAzureKVImportKey, err)
 	if err != nil {
 		return fmt.Errorf("could not import key %v: %w", secretName, err)
 	}
@@ -589,6 +602,7 @@ func (a *Azure) GetSecret(ctx context.Context, ref esv1beta1.ExternalSecretDataR
 		// returns a SecretBundle with the secret value
 		// https://pkg.go.dev/github.com/Azure/azure-sdk-for-go/services/keyvault/v7.0/keyvault#SecretBundle
 		secretResp, err := a.baseClient.GetSecret(context.Background(), *a.provider.VaultURL, secretName, ref.Version)
+		metrics.ObserveAPICall(metrics.ProviderAzureKV, metrics.CallAzureKVGetSecret, err)
 		err = parseError(err)
 		if err != nil {
 			return nil, err
@@ -601,6 +615,7 @@ func (a *Azure) GetSecret(ctx context.Context, ref esv1beta1.ExternalSecretDataR
 		// returns a CertBundle. We return CER contents of x509 certificate
 		// see: https://pkg.go.dev/github.com/Azure/azure-sdk-for-go/services/keyvault/v7.0/keyvault#CertificateBundle
 		certResp, err := a.baseClient.GetCertificate(context.Background(), *a.provider.VaultURL, secretName, ref.Version)
+		metrics.ObserveAPICall(metrics.ProviderAzureKV, metrics.CallAzureKVGetCertificate, err)
 		err = parseError(err)
 		if err != nil {
 			return nil, err
@@ -614,6 +629,7 @@ func (a *Azure) GetSecret(ctx context.Context, ref esv1beta1.ExternalSecretDataR
 		// azure kv returns only public keys
 		// see: https://pkg.go.dev/github.com/Azure/azure-sdk-for-go/services/keyvault/v7.0/keyvault#KeyBundle
 		keyResp, err := a.baseClient.GetKey(context.Background(), *a.provider.VaultURL, secretName, ref.Version)
+		metrics.ObserveAPICall(metrics.ProviderAzureKV, metrics.CallAzureKVGetKey, err)
 		err = parseError(err)
 		if err != nil {
 			return nil, err
@@ -631,6 +647,7 @@ func (a *Azure) GetSecret(ctx context.Context, ref esv1beta1.ExternalSecretDataR
 func (a *Azure) getSecretTags(ref esv1beta1.ExternalSecretDataRemoteRef) (map[string]*string, error) {
 	_, secretName := getObjType(ref)
 	secretResp, err := a.baseClient.GetSecret(context.Background(), *a.provider.VaultURL, secretName, ref.Version)
+	metrics.ObserveAPICall(metrics.ProviderAzureKV, metrics.CallAzureKVGetSecret, err)
 	err = parseError(err)
 	if err != nil {
 		return nil, err

pkg/provider/gcp/secretmanager/client.go

@@ -34,6 +34,7 @@ import (
 
 	esv1beta1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1beta1"
 	"github.com/external-secrets/external-secrets/pkg/find"
+	"github.com/external-secrets/external-secrets/pkg/provider/metrics"
 	"github.com/external-secrets/external-secrets/pkg/utils"
 )
 
@@ -91,6 +92,7 @@ func (c *Client) DeleteSecret(ctx context.Context, remoteRef esv1beta1.PushRemot
 	gcpSecret, err = c.smClient.GetSecret(ctx, &secretmanagerpb.GetSecretRequest{
 		Name: fmt.Sprintf("projects/%s/secrets/%s", c.store.ProjectID, remoteRef.GetRemoteKey()),
 	})
+	metrics.ObserveAPICall(metrics.ProviderGCPSM, metrics.CallGCPSMGetSecret, err)
 	var gErr *apierror.APIError
 
 	if errors.As(err, &gErr) {
@@ -111,7 +113,9 @@ func (c *Client) DeleteSecret(ctx context.Context, remoteRef esv1beta1.PushRemot
 	deleteSecretVersionReq := &secretmanagerpb.DeleteSecretRequest{
 		Name: fmt.Sprintf("projects/%s/secrets/%s", c.store.ProjectID, remoteRef.GetRemoteKey()),
 	}
-	return c.smClient.DeleteSecret(ctx, deleteSecretVersionReq)
+	err = c.smClient.DeleteSecret(ctx, deleteSecretVersionReq)
+	metrics.ObserveAPICall(metrics.ProviderGCPSM, metrics.CallGCPSMDeleteSecret, err)
+	return err
 }
 
 func parseError(err error) error {
@@ -145,12 +149,14 @@ func (c *Client) PushSecret(ctx context.Context, payload []byte, remoteRef esv1b
 	gcpSecret, err = c.smClient.GetSecret(ctx, &secretmanagerpb.GetSecretRequest{
 		Name: fmt.Sprintf("projects/%s/secrets/%s", c.store.ProjectID, remoteRef.GetRemoteKey()),
 	})
+	metrics.ObserveAPICall(metrics.ProviderGCPSM, metrics.CallGCPSMGetSecret, err)
 
 	var gErr *apierror.APIError
 
 	if err != nil && errors.As(err, &gErr) {
 		if gErr.GRPCStatus().Code() == codes.NotFound {
 			gcpSecret, err = c.smClient.CreateSecret(ctx, createSecretReq)
+			metrics.ObserveAPICall(metrics.ProviderGCPSM, metrics.CallGCPSMCreateSecret, err)
 			if err != nil {
 				return err
 			}
@@ -168,6 +174,7 @@ func (c *Client) PushSecret(ctx context.Context, payload []byte, remoteRef esv1b
 	gcpVersion, err := c.smClient.AccessSecretVersion(ctx, &secretmanagerpb.AccessSecretVersionRequest{
 		Name: fmt.Sprintf("projects/%s/secrets/%s/versions/latest", c.store.ProjectID, remoteRef.GetRemoteKey()),
 	})
+	metrics.ObserveAPICall(metrics.ProviderGCPSM, metrics.CallGCPSMAccessSecretVersion, err)
 
 	if errors.As(err, &gErr) {
 		if err != nil && gErr.GRPCStatus().Code() != codes.NotFound {
@@ -189,7 +196,7 @@ func (c *Client) PushSecret(ctx context.Context, payload []byte, remoteRef esv1b
 	}
 
 	_, err = c.smClient.AddSecretVersion(ctx, addSecretVersionReq)
-
+	metrics.ObserveAPICall(metrics.ProviderGCPSM, metrics.CallGCPSMAddSecretVersion, err)
 	if err != nil {
 		return err
 	}
@@ -224,8 +231,10 @@ func (c *Client) findByName(ctx context.Context, ref esv1beta1.ExternalSecretFin
 	// Call the API.
 	it := c.smClient.ListSecrets(ctx, req)
 	secretMap := make(map[string][]byte)
+	var resp *secretmanagerpb.Secret
+	defer metrics.ObserveAPICall(metrics.ProviderGCPSM, metrics.CallGCPSMListSecrets, err)
 	for {
-		resp, err := it.Next()
+		resp, err = it.Next()
 		if errors.Is(err, iterator.Done) {
 			break
 		}
@@ -280,9 +289,12 @@ func (c *Client) findByTags(ctx context.Context, ref esv1beta1.ExternalSecretFin
 	req.Filter = tagFilter
 	// Call the API.
 	it := c.smClient.ListSecrets(ctx, req)
+	var resp *secretmanagerpb.Secret
+	var err error
+	defer metrics.ObserveAPICall(metrics.ProviderGCPSM, metrics.CallGCPSMListSecrets, err)
 	secretMap := make(map[string][]byte)
 	for {
-		resp, err := it.Next()
+		resp, err = it.Next()
 		if errors.Is(err, iterator.Done) {
 			break
 		}
@@ -333,6 +345,7 @@ func (c *Client) GetSecret(ctx context.Context, ref esv1beta1.ExternalSecretData
 		Name: fmt.Sprintf("projects/%s/secrets/%s/versions/%s", c.store.ProjectID, ref.Key, version),
 	}
 	result, err := c.smClient.AccessSecretVersion(ctx, req)
+	metrics.ObserveAPICall(metrics.ProviderGCPSM, metrics.CallGCPSMAccessSecretVersion, err)
 	err = parseError(err)
 	if err != nil {
 		return nil, fmt.Errorf(errClientGetSecretAccess, err)

pkg/provider/gcp/secretmanager/workload_identity.go

@@ -41,6 +41,7 @@ import (
 	ctrlcfg "sigs.k8s.io/controller-runtime/pkg/client/config"
 
 	esv1beta1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1beta1"
+	"github.com/external-secrets/external-secrets/pkg/provider/metrics"
 )
 
 const (
@@ -127,11 +128,13 @@ func (w *workloadIdentity) TokenSource(ctx context.Context, auth esv1beta1.GCPSM
 	gcpSA := sa.Annotations[gcpSAAnnotation]
 
 	resp, err := w.saTokenGenerator.Generate(ctx, audiences, saKey.Name, saKey.Namespace)
+	metrics.ObserveAPICall(metrics.ProviderGCPSM, metrics.CallGCPSMGenerateSAToken, err)
 	if err != nil {
 		return nil, fmt.Errorf(errFetchPodToken, err)
 	}
 
 	idBindToken, err := w.idBindTokenGenerator.Generate(ctx, http.DefaultClient, resp.Status.Token, idPool, idProvider)
+	metrics.ObserveAPICall(metrics.ProviderGCPSM, metrics.CallGCPSMGenerateIDBindToken, err)
 	if err != nil {
 		return nil, fmt.Errorf(errFetchIBToken, err)
 	}
@@ -146,6 +149,7 @@ func (w *workloadIdentity) TokenSource(ctx context.Context, auth esv1beta1.GCPSM
 		Name:  fmt.Sprintf("projects/-/serviceAccounts/%s", gcpSA),
 		Scope: secretmanager.DefaultAuthScopes(),
 	}, gax.WithGRPCOptions(grpc.PerRPCCredentials(oauth.TokenSource{TokenSource: oauth2.StaticTokenSource(idBindToken)})))
+	metrics.ObserveAPICall(metrics.ProviderGCPSM, metrics.CallGCPSMGenerateAccessToken, err)
 	if err != nil {
 		return nil, fmt.Errorf(errGenAccessToken, err)
 	}

pkg/provider/gitlab/gitlab.go

@@ -31,6 +31,7 @@ import (
 
 	esv1beta1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1beta1"
 	"github.com/external-secrets/external-secrets/pkg/find"
+	"github.com/external-secrets/external-secrets/pkg/provider/metrics"
 	"github.com/external-secrets/external-secrets/pkg/utils"
 )
 
@@ -243,6 +244,7 @@ func (g *Gitlab) GetAllSecrets(ctx context.Context, ref esv1beta1.ExternalSecret
 		for groupPage := 1; ; groupPage++ {
 			gopts.Page = groupPage
 			groupVars, response, err := g.groupVariablesClient.ListVariables(groupID, gopts)
+			metrics.ObserveAPICall(metrics.ProviderGitLab, metrics.CallGitLabGroupListVariables, err)
 			if err != nil {
 				return nil, err
 			}
@@ -263,6 +265,7 @@ func (g *Gitlab) GetAllSecrets(ctx context.Context, ref esv1beta1.ExternalSecret
 	for projectPage := 1; ; projectPage++ {
 		popts.Page = projectPage
 		projectData, response, err := g.projectVariablesClient.ListVariables(g.projectID, popts)
+		metrics.ObserveAPICall(metrics.ProviderGitLab, metrics.CallGitLabProjectListVariables, err)
 		if err != nil {
 			return nil, err
 		}
@@ -320,9 +323,11 @@ func (g *Gitlab) GetSecret(ctx context.Context, ref esv1beta1.ExternalSecretData
 	}
 
 	data, resp, err := g.projectVariablesClient.GetVariable(g.projectID, ref.Key, vopts)
+	metrics.ObserveAPICall(metrics.ProviderGitLab, metrics.CallGitLabProjectVariableGet, err)
 	if !isEmptyOrWildcard(g.environment) && resp.StatusCode == http.StatusNotFound {
 		vopts.Filter.EnvironmentScope = "*"
 		data, resp, err = g.projectVariablesClient.GetVariable(g.projectID, ref.Key, vopts)
+		metrics.ObserveAPICall(metrics.ProviderGitLab, metrics.CallGitLabProjectVariableGet, err)
 	}
 
 	if resp.StatusCode >= 400 && resp.StatusCode != http.StatusNotFound && err != nil {
@@ -346,6 +351,7 @@ func (g *Gitlab) GetSecret(ctx context.Context, ref esv1beta1.ExternalSecretData
 		}
 
 		groupVar, resp, err := g.groupVariablesClient.GetVariable(groupID, ref.Key, nil)
+		metrics.ObserveAPICall(metrics.ProviderGitLab, metrics.CallGitLabGroupGetVariable, err)
 		if resp.StatusCode >= 400 && resp.StatusCode != http.StatusNotFound && err != nil {
 			return nil, err
 		}
@@ -430,6 +436,7 @@ func (g *Gitlab) Close(ctx context.Context) error {
 func (g *Gitlab) Validate() (esv1beta1.ValidationResult, error) {
 	if g.projectID != "" {
 		_, resp, err := g.projectVariablesClient.ListVariables(g.projectID, nil)
+		metrics.ObserveAPICall(metrics.ProviderGitLab, metrics.CallGitLabProjectListVariables, err)
 		if err != nil {
 			return esv1beta1.ValidationResultError, fmt.Errorf(errList, err)
 		} else if resp == nil || resp.StatusCode != http.StatusOK {
@@ -446,6 +453,7 @@ func (g *Gitlab) Validate() (esv1beta1.ValidationResult, error) {
 	if len(g.groupIDs) > 0 {
 		for _, groupID := range g.groupIDs {
 			_, resp, err := g.groupVariablesClient.ListVariables(groupID, nil)
+			metrics.ObserveAPICall(metrics.ProviderGitLab, metrics.CallGitLabGroupListVariables, err)
 			if err != nil {
 				return esv1beta1.ValidationResultError, fmt.Errorf(errList, err)
 			} else if resp == nil || resp.StatusCode != http.StatusOK {
@@ -460,6 +468,7 @@ func (g *Gitlab) Validate() (esv1beta1.ValidationResult, error) {
 func (g *Gitlab) ResolveGroupIds() error {
 	if g.inheritFromGroups {
 		projectGroups, resp, err := g.projectsClient.ListProjectsGroups(g.projectID, nil)
+		metrics.ObserveAPICall(metrics.ProviderGitLab, metrics.CallGitLabListProjectsGroups, err)
 		if resp.StatusCode >= 400 && err != nil {
 			return err
 		}

pkg/provider/ibm/provider.go

@@ -31,6 +31,7 @@ import (
 	kclient "sigs.k8s.io/controller-runtime/pkg/client"
 
 	esv1beta1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1beta1"
+	"github.com/external-secrets/external-secrets/pkg/provider/metrics"
 	utils "github.com/external-secrets/external-secrets/pkg/utils"
 )
 
@@ -185,6 +186,7 @@ func getArbitrarySecret(ibm *providerIBM, secretName *string) ([]byte, error) {
 			SecretType: core.StringPtr(sm.GetSecretOptionsSecretTypeArbitraryConst),
 			ID:         secretName,
 		})
+	metrics.ObserveAPICall(metrics.ProviderIBMSM, metrics.CallIBMSMGetSecret, err)
 	if err != nil {
 		return nil, err
 	}
@@ -201,6 +203,7 @@ func getImportCertSecret(ibm *providerIBM, secretName *string, ref esv1beta1.Ext
 			SecretType: core.StringPtr(sm.CreateSecretOptionsSecretTypeImportedCertConst),
 			ID:         secretName,
 		})
+	metrics.ObserveAPICall(metrics.ProviderIBMSM, metrics.CallIBMSMGetSecret, err)
 	if err != nil {
 		return nil, err
 	}
@@ -220,6 +223,7 @@ func getPublicCertSecret(ibm *providerIBM, secretName *string, ref esv1beta1.Ext
 			SecretType: core.StringPtr(sm.CreateSecretOptionsSecretTypePublicCertConst),
 			ID:         secretName,
 		})
+	metrics.ObserveAPICall(metrics.ProviderIBMSM, metrics.CallIBMSMGetSecret, err)
 	if err != nil {
 		return nil, err
 	}
@@ -239,6 +243,7 @@ func getPrivateCertSecret(ibm *providerIBM, secretName *string, ref esv1beta1.Ex
 			SecretType: core.StringPtr(sm.CreateSecretOptionsSecretTypePrivateCertConst),
 			ID:         secretName,
 		})
+	metrics.ObserveAPICall(metrics.ProviderIBMSM, metrics.CallIBMSMGetSecret, err)
 	if err != nil {
 		return nil, err
 	}
@@ -258,6 +263,7 @@ func getIamCredentialsSecret(ibm *providerIBM, secretName *string) ([]byte, erro
 			SecretType: core.StringPtr(sm.CreateSecretOptionsSecretTypeIamCredentialsConst),
 			ID:         secretName,
 		})
+	metrics.ObserveAPICall(metrics.ProviderIBMSM, metrics.CallIBMSMGetSecret, err)
 	if err != nil {
 		return nil, err
 	}
@@ -274,6 +280,7 @@ func getUsernamePasswordSecret(ibm *providerIBM, secretName *string, ref esv1bet
 			SecretType: core.StringPtr(sm.CreateSecretOptionsSecretTypeUsernamePasswordConst),
 			ID:         secretName,
 		})
+	metrics.ObserveAPICall(metrics.ProviderIBMSM, metrics.CallIBMSMGetSecret, err)
 	if err != nil {
 		return nil, err
 	}
@@ -357,6 +364,7 @@ func getSecretByType(ibm *providerIBM, secretName *string, secretType string) (*
 			SecretType: core.StringPtr(secretType),
 			ID:         secretName,
 		})
+	metrics.ObserveAPICall(metrics.ProviderIBMSM, metrics.CallIBMSMGetSecret, err)
 	if err != nil {
 		return nil, err
 	}
@@ -387,6 +395,7 @@ func (ibm *providerIBM) GetSecretMap(ctx context.Context, ref esv1beta1.External
 				SecretType: core.StringPtr(sm.GetSecretOptionsSecretTypeArbitraryConst),
 				ID:         &ref.Key,
 			})
+		metrics.ObserveAPICall(metrics.ProviderIBMSM, metrics.CallIBMSMGetSecret, err)
 		if err != nil {
 			return nil, err
 		}
@@ -411,6 +420,7 @@ func (ibm *providerIBM) GetSecretMap(ctx context.Context, ref esv1beta1.External
 				SecretType: core.StringPtr(sm.CreateSecretOptionsSecretTypeUsernamePasswordConst),
 				ID:         &secretName,
 			})
+		metrics.ObserveAPICall(metrics.ProviderIBMSM, metrics.CallIBMSMGetSecret, err)
 		if err != nil {
 			return nil, err
 		}
@@ -428,6 +438,7 @@ func (ibm *providerIBM) GetSecretMap(ctx context.Context, ref esv1beta1.External
 				SecretType: core.StringPtr(sm.CreateSecretOptionsSecretTypeIamCredentialsConst),
 				ID:         &secretName,
 			})
+		metrics.ObserveAPICall(metrics.ProviderIBMSM, metrics.CallIBMSMGetSecret, err)
 		if err != nil {
 			return nil, err
 		}
@@ -446,6 +457,7 @@ func (ibm *providerIBM) GetSecretMap(ctx context.Context, ref esv1beta1.External
 				SecretType: core.StringPtr(sm.CreateSecretOptionsSecretTypeImportedCertConst),
 				ID:         &secretName,
 			})
+		metrics.ObserveAPICall(metrics.ProviderIBMSM, metrics.CallIBMSMGetSecret, err)
 		if err != nil {
 			return nil, err
 		}
@@ -463,6 +475,7 @@ func (ibm *providerIBM) GetSecretMap(ctx context.Context, ref esv1beta1.External
 				SecretType: core.StringPtr(sm.CreateSecretOptionsSecretTypePublicCertConst),
 				ID:         &secretName,
 			})
+		metrics.ObserveAPICall(metrics.ProviderIBMSM, metrics.CallIBMSMGetSecret, err)
 		if err != nil {
 			return nil, err
 		}
@@ -480,6 +493,7 @@ func (ibm *providerIBM) GetSecretMap(ctx context.Context, ref esv1beta1.External
 				SecretType: core.StringPtr(sm.CreateSecretOptionsSecretTypePrivateCertConst),
 				ID:         &secretName,
 			})
+		metrics.ObserveAPICall(metrics.ProviderIBMSM, metrics.CallIBMSMGetSecret, err)
 		if err != nil {
 			return nil, err
 		}

pkg/provider/kubernetes/client.go

@@ -24,6 +24,7 @@ import (
 
 	esv1beta1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1beta1"
 	"github.com/external-secrets/external-secrets/pkg/find"
+	"github.com/external-secrets/external-secrets/pkg/provider/metrics"
 	"github.com/external-secrets/external-secrets/pkg/utils"
 )
 
@@ -61,6 +62,7 @@ func (c *Client) PushSecret(ctx context.Context, value []byte, remoteRef esv1bet
 
 func (c *Client) GetSecretMap(ctx context.Context, ref esv1beta1.ExternalSecretDataRemoteRef) (map[string][]byte, error) {
 	secret, err := c.userSecretClient.Get(ctx, ref.Key, metav1.GetOptions{})
+	metrics.ObserveAPICall(metrics.ProviderKubernetes, metrics.CallKubernetesGetSecret, err)
 	if apierrors.IsNotFound(err) {
 		return nil, esv1beta1.NoSecretError{}
 	}
@@ -87,6 +89,7 @@ func (c *Client) findByTags(ctx context.Context, ref esv1beta1.ExternalSecretFin
 		return nil, fmt.Errorf("unable to validate selector tags: %w", err)
 	}
 	secrets, err := c.userSecretClient.List(ctx, metav1.ListOptions{LabelSelector: sel.String()})
+	metrics.ObserveAPICall(metrics.ProviderKubernetes, metrics.CallKubernetesListSecrets, err)
 	if err != nil {
 		return nil, fmt.Errorf("unable to list secrets: %w", err)
 	}
@@ -103,6 +106,7 @@ func (c *Client) findByTags(ctx context.Context, ref esv1beta1.ExternalSecretFin
 
 func (c *Client) findByName(ctx context.Context, ref esv1beta1.ExternalSecretFind) (map[string][]byte, error) {
 	secrets, err := c.userSecretClient.List(ctx, metav1.ListOptions{})
+	metrics.ObserveAPICall(metrics.ProviderKubernetes, metrics.CallKubernetesListSecrets, err)
 	if err != nil {
 		return nil, fmt.Errorf("unable to list secrets: %w", err)
 	}

pkg/provider/kubernetes/validate.go

@@ -21,6 +21,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	esv1beta1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1beta1"
+	"github.com/external-secrets/external-secrets/pkg/provider/metrics"
 	"github.com/external-secrets/external-secrets/pkg/utils"
 )
 
@@ -79,6 +80,7 @@ func (c *Client) Validate() (esv1beta1.ValidationResult, error) {
 		},
 	}
 	authReview, err := c.userReviewClient.Create(ctx, &t, metav1.CreateOptions{})
+	metrics.ObserveAPICall(metrics.ProviderKubernetes, metrics.CallKubernetesCreateSelfSubjectRulesReview, err)
 	if err != nil {
 		return esv1beta1.ValidationResultUnknown, fmt.Errorf("could not verify if client is valid: %w", err)
 	}

pkg/provider/metrics/metrics.go

@@ -0,0 +1,116 @@
+/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metrics
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"sigs.k8s.io/controller-runtime/pkg/metrics"
+)
+
+const (
+	ExternalSecretSubsystem = "externalsecret"
+	providerAPICalls        = "provider_api_calls_count"
+)
+
+const (
+	ProviderAWSSM           = "AWS/SecretsManager"
+	CallAWSSMGetSecretValue = "GetSecretValue"
+	CallAWSSMDescribeSecret = "DescribeSecret"
+	CallAWSSMDeleteSecret   = "DeleteSecret"
+	CallAWSSMCreateSecret   = "CreateSecret"
+	CallAWSSMPutSecretValue = "PutSecretValue"
+	CallAWSSMListSecrets    = "ListSecrets"
+
+	ProviderAWSPS                = "AWS/ParameterStore"
+	CallAWSPSGetParameter        = "GetParameter"
+	CallAWSPSPutParameter        = "PutParameter"
+	CallAWSPSDeleteParameter     = "DeleteParameter"
+	CallAWSPSDescribeParameter   = "DescribeParameter"
+	CallAWSPSListTagsForResource = "ListTagsForResource"
+
+	ProviderAzureKV              = "Azure/KeyVault"
+	CallAzureKVGetKey            = "GetKey"
+	CallAzureKVDeleteKey         = "DeleteKey"
+	CallAzureKVImportKey         = "ImportKey"
+	CallAzureKVGetSecret         = "GetSecret"
+	CallAzureKVDeleteSecret      = "DeleteSecret"
+	CallAzureKVGetCertificate    = "GetCertificate"
+	CallAzureKVDeleteCertificate = "DeleteCertificate"
+	CallAzureKVImportCertificate = "ImportCertificate"
+
+	ProviderGCPSM                = "GCP/SecretManager"
+	CallGCPSMGetSecret           = "GetSecret"
+	CallGCPSMDeleteSecret        = "DeleteSecret"
+	CallGCPSMCreateSecret        = "CreateSecret"
+	CallGCPSMAccessSecretVersion = "AccessSecretVersion"
+	CallGCPSMAddSecretVersion    = "AddSecretVersion"
+	CallGCPSMListSecrets         = "ListSecrets"
+	CallGCPSMGenerateSAToken     = "GenerateServiceAccountToken"
+	CallGCPSMGenerateIDBindToken = "GenerateIDBindToken"
+	CallGCPSMGenerateAccessToken = "GenerateAccessToken"
+
+	ProviderHCVault            = "HashiCorp/Vault"
+	CallHCVaultLogin           = "Login"
+	CallHCVaultRevokeSelf      = "RevokeSelf"
+	CallHCVaultLookupSelf      = "LookupSelf"
+	CallHCVaultReadSecretData  = "ReadSecretData"
+	CallHCVaultWriteSecretData = "WriteSecretData"
+	CallHCVaultDeleteSecret    = "DeleteSecret"
+	CallHCVaultListSecrets     = "ListSecrets"
+
+	ProviderKubernetes                         = "Kubernetes"
+	CallKubernetesGetSecret                    = "GetSecret"
+	CallKubernetesListSecrets                  = "ListSecrets"
+	CallKubernetesCreateSelfSubjectRulesReview = "CreateSelfSubjectRulesReview"
+
+	ProviderIBMSM      = "IBM/SecretsManager"
+	CallIBMSMGetSecret = "GetSecret"
+
+	ProviderWebhook    = "Webhook"
+	CallWebhookHTTPReq = "HTTPRequest"
+
+	ProviderGitLab                 = "GitLab"
+	CallGitLabListProjectsGroups   = "ListProjectsGroups"
+	CallGitLabProjectVariableGet   = "ProjectVariableGet"
+	CallGitLabProjectListVariables = "ProjectVariablesList"
+	CallGitLabGroupGetVariable     = "GroupVariableGet"
+	CallGitLabGroupListVariables   = "GroupVariablesList"
+
+	StatusError   = "error"
+	StatusSuccess = "success"
+)
+
+var (
+	syncCallsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Subsystem: ExternalSecretSubsystem,
+		Name:      providerAPICalls,
+		Help:      "Number of API calls towards the secret provider",
+	}, []string{"provider", "call", "status"})
+)
+
+func ObserveAPICall(provider, call string, err error) {
+	syncCallsTotal.WithLabelValues(provider, call, deriveStatus(err)).Inc()
+}
+
+func deriveStatus(err error) string {
+	if err != nil {
+		return StatusError
+	}
+	return StatusSuccess
+}
+
+func init() {
+	metrics.Registry.MustRegister(syncCallsTotal)
+}

pkg/provider/vault/vault.go

@@ -49,6 +49,7 @@ import (
 	"github.com/external-secrets/external-secrets/pkg/cache"
 	"github.com/external-secrets/external-secrets/pkg/feature"
 	"github.com/external-secrets/external-secrets/pkg/find"
+	"github.com/external-secrets/external-secrets/pkg/provider/metrics"
 	"github.com/external-secrets/external-secrets/pkg/utils"
 )
 
@@ -415,10 +416,12 @@ func (v *client) DeleteSecret(ctx context.Context, remoteRef esv1beta1.PushRemot
 		return nil
 	}
 	_, err = v.logical.DeleteWithContext(ctx, path)
+	metrics.ObserveAPICall(metrics.ProviderHCVault, metrics.CallHCVaultDeleteSecret, err)
 	if err != nil {
 		return fmt.Errorf("could not delete secret %v: %w", remoteRef.GetRemoteKey(), err)
 	}
 	_, err = v.logical.DeleteWithContext(ctx, metaPath)
+	metrics.ObserveAPICall(metrics.ProviderHCVault, metrics.CallHCVaultDeleteSecret, err)
 	if err != nil {
 		return fmt.Errorf("could not delete secret metadata %v: %w", remoteRef.GetRemoteKey(), err)
 	}
@@ -470,11 +473,13 @@ func (v *client) PushSecret(ctx context.Context, value []byte, remoteRef esv1bet
 		return nil
 	}
 	_, err = v.logical.WriteWithContext(ctx, metaPath, label)
+	metrics.ObserveAPICall(metrics.ProviderHCVault, metrics.CallHCVaultWriteSecretData, err)
 	if err != nil {
 		return err
 	}
 	// Otherwise, create or update the version.
 	_, err = v.logical.WriteWithContext(ctx, path, secretToPush)
+	metrics.ObserveAPICall(metrics.ProviderHCVault, metrics.CallHCVaultWriteSecretData, err)
 	return err
 }
 
@@ -555,6 +560,7 @@ func (v *client) listSecrets(ctx context.Context, path string) ([]string, error)
 		return nil, err
 	}
 	secret, err := v.logical.ListWithContext(ctx, url)
+	metrics.ObserveAPICall(metrics.ProviderHCVault, metrics.CallHCVaultListSecrets, err)
 	if err != nil {
 		return nil, fmt.Errorf(errReadSecret, err)
 	}
@@ -593,6 +599,7 @@ func (v *client) readSecretMetadata(ctx context.Context, path string) (map[strin
 		return nil, err
 	}
 	secret, err := v.logical.ReadWithDataWithContext(ctx, url, nil)
+	metrics.ObserveAPICall(metrics.ProviderHCVault, metrics.CallHCVaultReadSecretData, err)
 	if err != nil {
 		return nil, fmt.Errorf(errReadSecret, err)
 	}
@@ -854,6 +861,7 @@ func (v *client) readSecret(ctx context.Context, path, version string) (map[stri
 		params["version"] = []string{version}
 	}
 	vaultSecret, err := v.logical.ReadWithDataWithContext(ctx, dataPath, params)
+	metrics.ObserveAPICall(metrics.ProviderHCVault, metrics.CallHCVaultReadSecretData, err)
 	if err != nil {
 		return nil, fmt.Errorf(errReadSecret, err)
 	}
@@ -1193,6 +1201,7 @@ func (v *client) serviceAccountToken(ctx context.Context, serviceAccountRef esme
 func checkToken(ctx context.Context, token Token) (bool, error) {
 	// https://www.vaultproject.io/api-docs/auth/token#lookup-a-token-self
 	resp, err := token.LookupSelfWithContext(ctx)
+	metrics.ObserveAPICall(metrics.ProviderHCVault, metrics.CallHCVaultLookupSelf, err)
 	if err != nil {
 		return false, err
 	}
@@ -1214,6 +1223,7 @@ func revokeTokenIfValid(ctx context.Context, client Client) error {
 	}
 	if valid {
 		err = client.AuthToken().RevokeSelfWithContext(ctx, client.Token())
+		metrics.ObserveAPICall(metrics.ProviderHCVault, metrics.CallHCVaultRevokeSelf, err)
 		if err != nil {
 			return fmt.Errorf(errVaultRevokeToken, err)
 		}
@@ -1235,6 +1245,7 @@ func (v *client) requestTokenWithAppRoleRef(ctx context.Context, appRole *esv1be
 		return err
 	}
 	_, err = v.auth.Login(ctx, appRoleClient)
+	metrics.ObserveAPICall(metrics.ProviderHCVault, metrics.CallHCVaultLogin, err)
 	if err != nil {
 		return err
 	}
@@ -1251,6 +1262,7 @@ func (v *client) requestTokenWithKubernetesAuth(ctx context.Context, kubernetesA
 		return err
 	}
 	_, err = v.auth.Login(ctx, k)
+	metrics.ObserveAPICall(metrics.ProviderHCVault, metrics.CallHCVaultLogin, err)
 	if err != nil {
 		return err
 	}
@@ -1315,6 +1327,7 @@ func (v *client) requestTokenWithLdapAuth(ctx context.Context, ldapAuth *esv1bet
 		return err
 	}
 	_, err = v.auth.Login(ctx, l)
+	metrics.ObserveAPICall(metrics.ProviderHCVault, metrics.CallHCVaultLogin, err)
 	if err != nil {
 		return err
 	}
@@ -1351,6 +1364,7 @@ func (v *client) requestTokenWithJwtAuth(ctx context.Context, jwtAuth *esv1beta1
 	}
 	url := strings.Join([]string{"auth", jwtAuth.Path, "login"}, "/")
 	vaultResult, err := v.logical.WriteWithContext(ctx, url, parameters)
+	metrics.ObserveAPICall(metrics.ProviderHCVault, metrics.CallHCVaultWriteSecretData, err)
 	if err != nil {
 		return err
 	}
@@ -1385,6 +1399,7 @@ func (v *client) requestTokenWithCertAuth(ctx context.Context, certAuth *esv1bet
 
 	url := strings.Join([]string{"auth", "cert", "login"}, "/")
 	vaultResult, err := v.logical.WriteWithContext(ctx, url, nil)
+	metrics.ObserveAPICall(metrics.ProviderHCVault, metrics.CallHCVaultWriteSecretData, err)
 	if err != nil {
 		return fmt.Errorf(errVaultRequest, err)
 	}

pkg/provider/webhook/webhook.go

@@ -34,6 +34,7 @@ import (
 
 	esv1beta1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1beta1"
 	esmeta "github.com/external-secrets/external-secrets/apis/meta/v1"
+	"github.com/external-secrets/external-secrets/pkg/provider/metrics"
 	"github.com/external-secrets/external-secrets/pkg/template/v2"
 	"github.com/external-secrets/external-secrets/pkg/utils"
 )
@@ -280,6 +281,7 @@ func (w *WebHook) getWebhookData(ctx context.Context, provider *esv1beta1.Webhoo
 	}
 
 	resp, err := w.http.Do(req)
+	metrics.ObserveAPICall(metrics.ProviderWebhook, metrics.CallWebhookHTTPReq, err)
 	if err != nil {
 		return nil, fmt.Errorf("failed to call endpoint: %w", err)
 	}