feat: add liveness probe to eso controller (#4930)

* feat: add liveness probe to eso controller

Signed-off-by: Gergely Brautigam <182850+Skarlso@users.noreply.github.com>

On-behalf-of: gergely.brautigam@sap.com

* fixed the linter warning

Signed-off-by: Gergely Brautigam <182850+Skarlso@users.noreply.github.com>

On-behalf-of: gergely.brautigam@sap.com

* fixed the helm schema generator as values and output was removed effectively

Signed-off-by: Gergely Brautigam <182850+Skarlso@users.noreply.github.com>

On-behalf-of: gergely.brautigam@sap.com

* add named port and add a helm test for the new value

Signed-off-by: Gergely Brautigam <182850+Skarlso@users.noreply.github.com>

On-behalf-of: gergely.brautigam@sap.com

Makefile

@@ -196,7 +196,7 @@ helm.schema.plugin:
 
 helm.schema.update: helm.schema.plugin
 	@$(INFO) Generating values.schema.json
-	@helm schema --values $(HELM_DIR)/values.yaml --output $(HELM_DIR)/values.schema.json
+	@helm schema -f $(HELM_DIR)/values.yaml -o $(HELM_DIR)/values.schema.json
 	@$(OK) Generated values.schema.json
 
 helm.generate:

cmd/controller/root.go

@@ -30,6 +30,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/cache"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
+	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/metrics/server"
 	"sigs.k8s.io/controller-runtime/pkg/webhook"
 
@@ -61,6 +62,7 @@ var (
 	setupLog                              = ctrl.Log.WithName("setup")
 	dnsName                               string
 	certDir                               string
+	liveAddr                              string
 	metricsAddr                           string
 	healthzAddr                           string
 	controllerClass                       string
@@ -141,6 +143,7 @@ var rootCmd = &cobra.Command{
 			Metrics: server.Options{
 				BindAddress: metricsAddr,
 			},
+			HealthProbeBindAddress: liveAddr,
 			WebhookServer: webhook.NewServer(webhook.Options{
 				Port: 9443,
 			}),
@@ -287,6 +290,11 @@ var rootCmd = &cobra.Command{
 			}
 		}
 
+		if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
+			setupLog.Error(err, "unable to add controller healthz check")
+			os.Exit(1)
+		}
+
 		fs := feature.Features()
 		for _, f := range fs {
 			if f.Initialize == nil {
@@ -315,6 +323,7 @@ func init() {
 	rootCmd.Flags().IntVar(&concurrent, "concurrent", 1, "The number of concurrent reconciles.")
 	rootCmd.Flags().Float32Var(&clientQPS, "client-qps", 50, "QPS configuration to be passed to rest.Client")
 	rootCmd.Flags().IntVar(&clientBurst, "client-burst", 100, "Maximum Burst allowed to be passed to rest.Client")
+	rootCmd.Flags().StringVar(&liveAddr, "live-addr", ":8082", "The address the live endpoint binds to.")
 	rootCmd.Flags().StringVar(&loglevel, "loglevel", "info", "loglevel to use, one of: debug, info, warn, error, dpanic, panic, fatal")
 	rootCmd.Flags().StringVar(&zapTimeEncoding, "zap-time-encoding", "epoch", "Zap time encoding (one of 'epoch', 'millis', 'nano', 'iso8601', 'rfc3339' or 'rfc3339nano')")
 	rootCmd.Flags().StringVar(&namespace, "namespace", "", "watch external secrets scoped in the provided namespace only. ClusterSecretStore can be used but only work if it doesn't reference resources from other namespaces")

deploy/charts/external-secrets/README.md

@@ -126,6 +126,15 @@ The command removes all the Kubernetes components associated with the chart and
 | imagePullSecrets | list | `[]` |  |
 | installCRDs | bool | `true` | If set, install and upgrade CRDs through helm chart. |
 | leaderElect | bool | `false` | If true, external-secrets will perform leader election between instances to ensure no more than one instance of external-secrets operates at a time. |
+| livenessProbe.address | string | `""` | Address for liveness probe. |
+| livenessProbe.enabled | bool | `false` | Enabled determines if the liveness probe should be used or not. By default it's disabled. |
+| livenessProbe.failureThreshold | int | `5` | Number of consecutive probe failures that should occur before considering the probe as failed. |
+| livenessProbe.httpGet | object | `{"path":"/healthz","port":"8082"}` | Handler for liveness probe. |
+| livenessProbe.httpGet.port | string | `"8082"` | Set this value to 8082 to active liveness probes. |
+| livenessProbe.initialDelaySeconds | int | `10` | Delay in seconds for the container to start before performing the initial probe. |
+| livenessProbe.periodSeconds | int | `10` | Period in seconds for K8s to start performing probes. |
+| livenessProbe.successThreshold | int | `1` | Number of successful probes to mark probe successful. |
+| livenessProbe.timeoutSeconds | int | `5` | Specify the maximum amount of time to wait for a probe to respond before considering it fails. |
 | log | object | `{"level":"info","timeEncoding":"epoch"}` | Specifies Log Params to the External Secrets Operator |
 | metrics.listen.port | int | `8080` |  |
 | metrics.service.annotations | object | `{}` | Additional service annotations |

deploy/charts/external-secrets/templates/deployment.yaml

@@ -102,10 +102,17 @@ spec:
           - --metrics-addr=:{{ .Values.metrics.listen.port }}
           - --loglevel={{ .Values.log.level }}
           - --zap-time-encoding={{ .Values.log.timeEncoding }}
+          {{- if .Values.livenessProbe.enabled }}
+          - --live-addr={{ .Values.livenessProbe.address }}:{{ .Values.livenessProbe.httpGet.port }}
+          {{- end }}
           ports:
             - containerPort: {{ .Values.metrics.listen.port }}
               protocol: TCP
               name: metrics
+          {{- if .Values.livenessProbe.enabled }}
+          livenessProbe:
+          {{- toYaml .Values.livenessProbe | nindent 12 }}
+          {{- end }}
           {{- with .Values.extraEnv }}
           env:
             {{- toYaml . | nindent 12 }}

deploy/charts/external-secrets/tests/controller_test.yaml

@@ -100,3 +100,20 @@ tests:
       - equal:
           path: spec.strategy.rollingUpdate.maxUnavailable
           value: 0
+  - it: should add livenessProbe if define
+    set:
+      livenessProbe:
+        enabled: true
+        initialDelaySeconds: 10
+        periodSeconds: 10
+        timeoutSeconds: 10
+        failureThreshold: 10
+        successThreshold: 10
+        httpGet:
+          path: /healthz
+          port: "8080"
+          scheme: HTTP
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.httpGet.port
+          value: "8080"

deploy/charts/external-secrets/values.schema.json

@@ -403,6 +403,43 @@
         "leaderElect": {
             "type": "boolean"
         },
+        "livenessProbe": {
+            "type": "object",
+            "properties": {
+                "address": {
+                    "type": "string"
+                },
+                "enabled": {
+                    "type": "boolean"
+                },
+                "failureThreshold": {
+                    "type": "integer"
+                },
+                "httpGet": {
+                    "type": "object",
+                    "properties": {
+                        "path": {
+                            "type": "string"
+                        },
+                        "port": {
+                            "type": "string"
+                        }
+                    }
+                },
+                "initialDelaySeconds": {
+                    "type": "integer"
+                },
+                "periodSeconds": {
+                    "type": "integer"
+                },
+                "successThreshold": {
+                    "type": "integer"
+                },
+                "timeoutSeconds": {
+                    "type": "integer"
+                }
+            }
+        },
         "log": {
             "type": "object",
             "properties": {

deploy/charts/external-secrets/values.yaml

@@ -257,6 +257,27 @@ grafanaDashboard:
   # https://github.com/grafana/helm-charts/tree/main/charts/grafana
   annotations: {}
 
+livenessProbe:
+  # -- Enabled determines if the liveness probe should be used or not. By default it's disabled.
+  enabled: false
+  # -- Address for liveness probe.
+  address: ""
+  # -- Specify the maximum amount of time to wait for a probe to respond before considering it fails.
+  timeoutSeconds: 5
+  # -- Number of consecutive probe failures that should occur before considering the probe as failed.
+  failureThreshold: 5
+  # -- Period in seconds for K8s to start performing probes.
+  periodSeconds: 10
+  # -- Number of successful probes to mark probe successful.
+  successThreshold: 1
+  # -- Delay in seconds for the container to start before performing the initial probe.
+  initialDelaySeconds: 10
+  # -- Handler for liveness probe.
+  httpGet:
+    # -- Set this value to 8082 to active liveness probes.
+    port: "8082"
+    path: /healthz
+
 nodeSelector: {}
 
 tolerations: []

docs/api/controller-options.md

@@ -31,20 +31,21 @@ The core controller is invoked without a subcommand and can be configured with t
 | `--help`                                      |          |         | help for external-secrets                                                                                                                                          |
 | `--loglevel`                                  | string   | info    | loglevel to use, one of: debug, info, warn, error, dpanic, panic, fatal                                                                                            |
 | `--zap-time-encoding`                         | string   | epoch   | loglevel to use, one of: epoch, millis, nano, iso8601, rfc3339, rfc3339nano                                                                                        |
+| `--live-addr`                                 | string   | :8082   | The address the live endpoint binds to                                                                                                                             |
 | `--metrics-addr`                              | string   | :8080   | The address the metric endpoint binds to.                                                                                                                          |
 | `--namespace`                                 | string   | -       | watch external secrets scoped in the provided namespace only. ClusterSecretStore can be used but only work if it doesn't reference resources from other namespaces |
 | `--store-requeue-interval`                    | duration | 5m0s    | Default Time duration between reconciling (Cluster)SecretStores                                                                                                    |
 
 ## Cert Controller Flags
 
-| Name                       | Type     | Default                  | Description                                                                                                            |
-| -------------------------- | -------- | ------------------------ | --------------------------------------------------------------------------------------------------------------------- |
+| Name                       | Type     | Default                  | Descripton                                                                                                            |
+|----------------------------|----------|--------------------------|-----------------------------------------------------------------------------------------------------------------------|
 | `--crd-requeue-interval`   | duration | 5m0s                     | Time duration between reconciling CRDs for new certs                                                                  |
 | `--enable-leader-election` | boolean  | false                    | Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager. |
 | `--healthz-addr`           | string   | :8081                    | The address the health endpoint binds to.                                                                             |
 | `--help`                   |          |                          | help for certcontroller                                                                                               |
 | `--loglevel`               | string   | info                     | loglevel to use, one of: debug, info, warn, error, dpanic, panic, fatal                                               |
-| `--zap-time-encoding`                                  | string   | epoch                          | time encoding to use, one of: epoch, millis, nano, iso8601, rfc3339, rfc3339nano                                                                                            |
+| `--zap-time-encoding`      | string   | epoch                    | time encoding to use, one of: epoch, millis, nano, iso8601, rfc3339, rfc3339nano                                      |
 | `--metrics-addr`           | string   | :8080                    | The address the metric endpoint binds to.                                                                             |
 | `--secret-name`            | string   | external-secrets-webhook | Secret to store certs for webhook                                                                                     |
 | `--secret-namespace`       | string   | default                  | namespace of the secret to store certs                                                                                |

tests/__snapshot__/clustersecretstore-v1.yaml

@@ -341,6 +341,11 @@ spec:
       serviceUrl: string
     infisical:
       auth:
+        awsAuthCredentials:
+          identityId:
+            key: string
+            name: string
+            namespace: string
         azureAuthCredentials:
           identityId:
             key: string
@@ -373,6 +378,15 @@ spec:
             key: string
             name: string
             namespace: string
+        kubernetesAuthCredentials:
+          identityId:
+            key: string
+            name: string
+            namespace: string
+          serviceAccountTokenPath:
+            key: string
+            name: string
+            namespace: string
         ldapAuthCredentials:
           identityId:
             key: string
@@ -415,6 +429,11 @@ spec:
             key: string
             name: string
             namespace: string
+        tokenAuthCredentials:
+          accessToken:
+            key: string
+            name: string
+            namespace: string
         universalAuthCredentials:
           clientId:
             key: string

tests/__snapshot__/secretstore-v1.yaml

@@ -341,6 +341,11 @@ spec:
       serviceUrl: string
     infisical:
       auth:
+        awsAuthCredentials:
+          identityId:
+            key: string
+            name: string
+            namespace: string
         azureAuthCredentials:
           identityId:
             key: string
@@ -373,6 +378,15 @@ spec:
             key: string
             name: string
             namespace: string
+        kubernetesAuthCredentials:
+          identityId:
+            key: string
+            name: string
+            namespace: string
+          serviceAccountTokenPath:
+            key: string
+            name: string
+            namespace: string
         ldapAuthCredentials:
           identityId:
             key: string
@@ -415,6 +429,11 @@ spec:
             key: string
             name: string
             namespace: string
+        tokenAuthCredentials:
+          accessToken:
+            key: string
+            name: string
+            namespace: string
         universalAuthCredentials:
           clientId:
             key: string