feat(helm): Add control of response to missing prometheus (#5087)

Signed-off-by: Pat Riehecky <riehecky@fnal.gov>
Co-authored-by: Gergely Brautigam <skarlso777@gmail.com>

deploy/charts/external-secrets/README.md

@@ -185,6 +185,7 @@ The command removes all the Kubernetes components associated with the chart and
 | serviceMonitor.metricRelabelings | list | `[]` | Metric relabel configs to apply to samples before ingestion. [Metric Relabeling](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs) |
 | serviceMonitor.namespace | string | `""` | namespace where you want to install ServiceMonitors |
 | serviceMonitor.relabelings | list | `[]` | Relabel configs to apply to samples before ingestion. [Relabeling](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config) |
+| serviceMonitor.renderMode | string | `"skipIfMissing"` | How should we react to missing CRD "`monitoring.coreos.com/v1/ServiceMonitor`" Possible values: - `skipIfMissing`: Only render ServiceMonitor resources if CRD is present, skip if missing. - `failIfMissing`: Fail Helm install if CRD is not present. - `alwaysRender` : Always render ServiceMonitor resources, do not check for CRD. @schema enum: - skipIfMissing - failIfMissing - alwaysRender @schema |
 | serviceMonitor.scrapeTimeout | string | `"25s"` | Timeout if metrics can't be retrieved in given time interval |
 | strategy | object | `{}` | Set deployment strategy |
 | tolerations | list | `[]` |  |

deploy/charts/external-secrets/templates/NOTES.txt

@@ -1,3 +1,4 @@
+{{- $shouldRenderStr := include "external-secrets.shouldRenderServiceMonitor" . | trim }}
 external-secrets has been deployed successfully in namespace {{ template "external-secrets.namespace" . }}!
 
 In order to begin using ExternalSecrets, you will need to set up a SecretStore
@@ -5,3 +6,7 @@ or ClusterSecretStore resource (for example, by creating a 'vault' SecretStore).
 
 More information on the different types of SecretStores and how to configure them
 can be found in our Github: {{ .Chart.Home }}
+
+{{- if and .Values.serviceMonitor.enabled (eq $shouldRenderStr "false") -}}
+WARNING: ServiceMonitors were not deployed due to missing CRD monitoring.coreos.com/v1/ServiceMonitor
+{{- end -}}

deploy/charts/external-secrets/templates/_helpers.tpl

@@ -57,6 +57,7 @@ app.kubernetes.io/managed-by: {{ .Release.Service }}
 {{- end }}
 
 {{- define "external-secrets-webhook.labels" -}}
+{{- $shouldRenderStr := include "external-secrets.shouldRenderServiceMonitor" . | trim }}
 helm.sh/chart: {{ include "external-secrets.chart" . }}
 {{ include "external-secrets-webhook.selectorLabels" . }}
 {{- if .Chart.AppVersion }}
@@ -66,7 +67,7 @@ app.kubernetes.io/managed-by: {{ .Release.Service }}
 {{- with .Values.commonLabels }}
 {{ toYaml . }}
 {{- end }}
-{{- if and ( .Capabilities.APIVersions.Has "monitoring.coreos.com/v1" ) .Values.serviceMonitor.enabled }}
+{{- if and .Values.serviceMonitor.enabled (eq $shouldRenderStr "true") }}
 app.kubernetes.io/metrics: "webhook"
 {{- with .Values.webhook.service.labels }}
 {{ toYaml . }}
@@ -97,6 +98,7 @@ app.kubernetes.io/metrics: "webhook"
 {{- end }}
 
 {{- define "external-secrets-cert-controller.labels" -}}
+{{- $shouldRenderStr := include "external-secrets.shouldRenderServiceMonitor" . | trim }}
 helm.sh/chart: {{ include "external-secrets.chart" . }}
 {{ include "external-secrets-cert-controller.selectorLabels" . }}
 {{- if .Chart.AppVersion }}
@@ -106,7 +108,7 @@ app.kubernetes.io/managed-by: {{ .Release.Service }}
 {{- with .Values.commonLabels }}
 {{ toYaml . }}
 {{- end }}
-{{- if and ( .Capabilities.APIVersions.Has "monitoring.coreos.com/v1" ) .Values.serviceMonitor.enabled }}
+{{- if and .Values.serviceMonitor.enabled (eq $shouldRenderStr "true") }}
 app.kubernetes.io/metrics: "cert-controller"
 {{- end }}
 {{- end }}
@@ -247,3 +249,26 @@ Fail the install if a cluster scoped reconciler is enabled while its namespace s
   {{- fail "You have disabled processing of PushSecrets but not ClusterPushSecrets. This is an invalid configuration. ClusterPushSecret processing depends on processing of PushSecrets. Please either enable processing of PushSecrets, or disable processing of ClusterPushSecrets." }}
 {{- end -}}
 {{- end -}}
+
+{{/*
+Decide whether to render the ServiceMonitor resource.
+*/}}
+{{- define "external-secrets.shouldRenderServiceMonitor" -}}
+  {{- $mode := .Values.serviceMonitor.renderMode | default "skipIfMissing" -}}
+  {{- if eq $mode "alwaysRender" -}}
+    true
+  {{- else if eq $mode "skipIfMissing" -}}
+    {{- if has "monitoring.coreos.com/v1/ServiceMonitor" .Capabilities.APIVersions -}}
+      true
+    {{- else -}}
+      false
+    {{- end -}}
+  {{- else if eq $mode "failIfMissing" -}}
+    {{- if not (has "monitoring.coreos.com/v1/ServiceMonitor" .Capabilities.APIVersions) -}}
+      {{- fail "ServiceMonitor CRD is required but not present in the cluster. See https://github.com/prometheus-operator/prometheus-operator/blob/main/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml or the settings for .Values.serviceMonitor.renderMode to suppress this error." -}}
+    {{- end -}}
+    true
+  {{- else -}}
+    {{- fail (printf "Invalid renderMode '%s'. Must be one of: skipIfMissing, failIfMissing, alwaysRender." $mode) -}}
+  {{- end -}}
+{{- end -}}

deploy/charts/external-secrets/templates/cert-controller-service.yaml

@@ -1,4 +1,9 @@
-{{- if and .Values.certController.create ( or .Values.certController.metrics.service.enabled ( and ( .Capabilities.APIVersions.Has "monitoring.coreos.com/v1" ) .Values.serviceMonitor.enabled )) (not .Values.webhook.certManager.enabled) }}
+{{- $shouldRenderStr := include "external-secrets.shouldRenderServiceMonitor" . | trim }}
+{{- if and .Values.certController.create
+         (or .Values.certController.metrics.service.enabled
+             (and (eq $shouldRenderStr "true")
+                  .Values.serviceMonitor.enabled))
+         (not .Values.webhook.certManager.enabled) }}
 apiVersion: v1
 kind: Service
 metadata:

deploy/charts/external-secrets/templates/service.yaml

@@ -1,4 +1,5 @@
-{{- if or (and ( .Capabilities.APIVersions.Has "monitoring.coreos.com/v1" ) .Values.serviceMonitor.enabled) .Values.metrics.service.enabled -}}
+{{- $shouldRenderStr := include "external-secrets.shouldRenderServiceMonitor" . | trim }}
+{{- if or .Values.metrics.service.enabled (and .Values.serviceMonitor.enabled (eq $shouldRenderStr "true")) -}}
 apiVersion: v1
 kind: Service
 metadata:

deploy/charts/external-secrets/templates/servicemonitor.yaml

@@ -1,4 +1,5 @@
-{{- if and ( .Capabilities.APIVersions.Has "monitoring.coreos.com/v1" ) .Values.serviceMonitor.enabled -}}
+{{- $shouldRenderStr := include "external-secrets.shouldRenderServiceMonitor" . | trim }}
+{{- if and .Values.serviceMonitor.enabled (eq $shouldRenderStr "true") }}
 apiVersion: "monitoring.coreos.com/v1"
 kind: ServiceMonitor
 metadata:

deploy/charts/external-secrets/templates/webhook-service.yaml

@@ -1,4 +1,5 @@
 {{- if and .Values.webhook.create .Values.webhook.service.enabled }}
+{{- $shouldRenderStr := include "external-secrets.shouldRenderServiceMonitor" . | trim }}
 apiVersion: v1
 kind: Service
 metadata:
@@ -24,12 +25,14 @@ spec:
     targetPort: webhook
     protocol: TCP
     name: webhook
-  {{- if or .Values.webhook.metrics.service.enabled ( and ( .Capabilities.APIVersions.Has "monitoring.coreos.com/v1" ) .Values.serviceMonitor.enabled ) }}
+{{- if or .Values.webhook.metrics.service.enabled 
+         (and .Values.serviceMonitor.enabled 
+              (eq $shouldRenderStr "true")) }}
   - port: {{ .Values.webhook.metrics.service.port }}
     protocol: TCP
     targetPort: metrics
     name: metrics
-  {{- end }}
+{{- end }}
   selector:
     {{- include "external-secrets-webhook.selectorLabels" . | nindent 4 }}
 {{- end }}

deploy/charts/external-secrets/tests/cert_controller_test.yaml

@@ -145,7 +145,7 @@ tests:
       serviceMonitor.enabled: true
     capabilities:
       apiVersions:
-        - "monitoring.coreos.com/v1"
+        - "monitoring.coreos.com/v1/ServiceMonitor"
     templates:
       - cert-controller-service.yaml
     asserts:
@@ -154,6 +154,35 @@ tests:
       - equal:
           path: metadata.labels["app.kubernetes.io/metrics"]
           value: "cert-controller"
+  - it: should render service with metrics label when APIVersions are not present, serviceMonitor is enabled, and serviceMonitor.renderMode is alwaysRender
+    set:
+      serviceMonitor.enabled: true
+      serviceMonitor.renderMode: alwaysRender
+    templates:
+      - cert-controller-service.yaml
+    asserts:
+      - hasDocuments:
+          count: 1
+      - equal:
+          path: metadata.labels["app.kubernetes.io/metrics"]
+          value: "cert-controller"
+  - it: should fail if APIVersions is missing, serviceMonitor is enabled, and serviceMonitor.renderMode is failIfMissing
+    set:
+      serviceMonitor.enabled: true
+      serviceMonitor.renderMode: failIfMissing
+    templates:
+      - cert-controller-service.yaml
+    asserts:
+      - failedTemplate: {}
+  - it: should not render service when APIVersions is not present, serviceMonitor is enabled, and and serviceMonitor.renderMode is skipIfMissing
+    set:
+      serviceMonitor.enabled: true
+      serviceMonitor.renderMode: skipIfMissing
+    templates:
+      - cert-controller-service.yaml
+    asserts:
+      - hasDocuments:
+          count: 0
   - it: should not render service when APIVersions is not present but serviceMonitor is enabled
     set:
       serviceMonitor.enabled: true
@@ -167,7 +196,7 @@ tests:
       serviceMonitor.enabled: false
     capabilities:
       apiVersions:
-        - "monitoring.coreos.com/v1"
+        - "monitoring.coreos.com/v1/ServiceMonitor"
     templates:
       - cert-controller-service.yaml
     asserts:

deploy/charts/external-secrets/tests/service_monitor_test.yaml

@@ -7,10 +7,30 @@ tests:
       serviceMonitor.enabled: true
     capabilities:
       apiVersions:
-        - "monitoring.coreos.com/v1"
+        - "monitoring.coreos.com/v1/ServiceMonitor"
     asserts:
       - hasDocuments:
           count: 3
+  - it: should render service monitor when APIVersions is not present, serviceMonitor is enabled, and shouldRenderServiceMonitor is alwaysRender
+    set:
+      serviceMonitor.enabled: true
+      serviceMonitor.renderMode: alwaysRender
+    asserts:
+      - hasDocuments:
+          count: 3
+  - it: should fail if APIVersions is missing, serviceMonitor is enabled, and serviceMonitor.renderMode is failIfMissing
+    set:
+      serviceMonitor.enabled: true
+      serviceMonitor.renderMode: failIfMissing
+    asserts:
+      - failedTemplate: {}
+  - it: should not render service monitor when APIVersions is not present, serviceMonitor is enabled, and serviceMonitor.renderMode is skipIfMissing
+    set:
+      serviceMonitor.enabled: true
+      serviceMonitor.renderMode: skipIfMissing
+    asserts:
+      - hasDocuments:
+          count: 0
   - it: should not render service monitor when APIVersions is not present but serviceMonitor is enabled
     set:
       serviceMonitor.enabled: true
@@ -22,7 +42,7 @@ tests:
       serviceMonitor.enabled: false
     capabilities:
       apiVersions:
-        - "monitoring.coreos.com/v1"
+        - "monitoring.coreos.com/v1/ServiceMonitor"
     asserts:
       - hasDocuments:
           count: 0

deploy/charts/external-secrets/tests/service_test.yaml

@@ -15,12 +15,29 @@ tests:
       serviceMonitor.enabled: true
     capabilities:
       apiVersions:
-        - "monitoring.coreos.com/v1"
+        - "monitoring.coreos.com/v1/ServiceMonitor"
     templates:
       - service.yaml
     asserts:
       - hasDocuments:
           count: 1
+  - it: should fail if APIVersions is missing, serviceMonitor is enabled, and serviceMonitor.renderMode is failIfMissing
+    set:
+      serviceMonitor.enabled: true
+      serviceMonitor.renderMode: failIfMissing
+    templates:
+      - service.yaml
+    asserts:
+      - failedTemplate: {}
+  - it: should not render service when APIVersions is not present, serviceMonitor is enabled, and serviceMonitor.renderMode is skipIfMissing
+    set:
+      serviceMonitor.enabled: true
+      serviceMonitor.renderMode: skipIfMissing
+    templates:
+      - service.yaml
+    asserts:
+      - hasDocuments:
+          count: 0
   - it: should not render service when APIVersions is not present but serviceMonitor is enabled
     set:
       serviceMonitor.enabled: true
@@ -34,7 +51,7 @@ tests:
       serviceMonitor.enabled: false
     capabilities:
       apiVersions:
-        - "monitoring.coreos.com/v1"
+        - "monitoring.coreos.com/v1/ServiceMonitor"
     templates:
       - service.yaml
     asserts:

deploy/charts/external-secrets/tests/webhook_test.yaml

@@ -219,7 +219,7 @@ tests:
       serviceMonitor.enabled: true
     capabilities:
       apiVersions:
-        - "monitoring.coreos.com/v1"
+        - "monitoring.coreos.com/v1/ServiceMonitor"
     templates:
       - webhook-service.yaml
     asserts:
@@ -229,6 +229,39 @@ tests:
       - equal:
           path: metadata.labels["app.kubernetes.io/metrics"]
           value: "webhook"
+  - it: should expose metrics port and metrics label when APIVersions is not present, serviceMonitor is enabled, and serviceMonitor.renderMode is alwaysRender
+    set:
+      serviceMonitor.enabled: true
+      serviceMonitor.renderMode: alwaysRender
+    templates:
+      - webhook-service.yaml
+    asserts:
+      - equal:
+          path: spec.ports[1].name
+          value: metrics
+      - equal:
+          path: metadata.labels["app.kubernetes.io/metrics"]
+          value: "webhook"
+  - it: should fail if APIVersions is missing, serviceMonitor is enabled, and serviceMonitor.renderMode is failIfMissing
+    set:
+      serviceMonitor.enabled: true
+      serviceMonitor.renderMode: failIfMissing
+    templates:
+      - webhook-service.yaml
+    asserts:
+      - failedTemplate: {}
+  - it: should not expose metrics port nor metrics label when APIVersions is not present, serviceMonitor is enabled, and serviceMonitor.renderMode is skipIfMissing
+    set:
+      serviceMonitor.enabled: true
+      serviceMonitor.renderMode: skipIfMissing
+    templates:
+      - webhook-service.yaml
+    asserts:
+      - lengthEqual:
+          path: spec.ports
+          count: 1
+      - isNull:
+          path: metadata.labels["app.kubernetes.io/metrics"]
   - it: should not expose metrics port nor metrics label when APIVersions is not present but serviceMonitor is enabled
     set:
       serviceMonitor.enabled: true
@@ -245,7 +278,7 @@ tests:
       serviceMonitor.enabled: false
     capabilities:
       apiVersions:
-        - "monitoring.coreos.com/v1"
+        - "monitoring.coreos.com/v1/ServiceMonitor"
     templates:
       - webhook-service.yaml
     asserts:

deploy/charts/external-secrets/values.schema.json

@@ -673,6 +673,14 @@
                 "relabelings": {
                     "type": "array"
                 },
+                "renderMode": {
+                    "type": "string",
+                    "enum": [
+                        "skipIfMissing",
+                        "failIfMissing",
+                        "alwaysRender"
+                    ]
+                },
                 "scrapeTimeout": {
                     "type": "string"
                 }

deploy/charts/external-secrets/values.yaml

@@ -197,6 +197,21 @@ serviceMonitor:
   # -- Specifies whether to create a ServiceMonitor resource for collecting Prometheus metrics
   enabled: false
 
+  # -- How should we react to missing CRD "`monitoring.coreos.com/v1/ServiceMonitor`"
+  #
+  # Possible values:
+  # - `skipIfMissing`: Only render ServiceMonitor resources if CRD is present, skip if missing.
+  # - `failIfMissing`: Fail Helm install if CRD is not present.
+  # - `alwaysRender` : Always render ServiceMonitor resources, do not check for CRD.
+
+  # @schema
+  # enum:
+  # - skipIfMissing
+  # - failIfMissing
+  # - alwaysRender
+  # @schema
+  renderMode: skipIfMissing   # @schema enum: [skipIfMissing, failIfMissing, alwaysRender]
+
   # -- namespace where you want to install ServiceMonitors
   namespace: ""