feat: configurable leader election lease timings (#6644)

* feat: configurable leader election lease timings

Signed-off-by: Izz Noland <izz@linux.com>

* test(charts): cover leader election timing args

Signed-off-by: Izz Noland <izz@linux.com>

* chore(charts): regenerate values.schema.json for leader election values

Signed-off-by: Izz Noland <izz@linux.com>

---------

Signed-off-by: Izz Noland <izz@linux.com>
Co-authored-by: Alexander Chernov <alexander@chernov.it>
Co-authored-by: Jean-Philippe Evrard <jean-philippe.evrard+rochepub@external.roche.com>

cmd/controller/root.go

@@ -74,6 +74,9 @@ var (
 	controllerClass                       string
 	enableLeaderElection                  bool
 	leaderElectionID                      string
+	leaderElectionLeaseDuration           time.Duration
+	leaderElectionRenewDeadline           time.Duration
+	leaderElectionRetryPeriod             time.Duration
 	enableSecretsCache                    bool
 	enableConfigMapsCache                 bool
 	enableManagedSecretsCache             bool
@@ -182,6 +185,9 @@ var rootCmd = &cobra.Command{
 			},
 			LeaderElection:   enableLeaderElection,
 			LeaderElectionID: leaderElectionID,
+			LeaseDuration:    &leaderElectionLeaseDuration,
+			RenewDeadline:    &leaderElectionRenewDeadline,
+			RetryPeriod:      &leaderElectionRetryPeriod,
 		}
 		if namespace != "" {
 			mgrOpts.Cache.DefaultNamespaces = map[string]cache.Config{
@@ -347,6 +353,12 @@ func init() {
 			"Enabling this will ensure there is only one active controller manager.")
 	rootCmd.Flags().StringVar(&leaderElectionID, "leader-election-id", "external-secrets-controller",
 		"The ID of the lease object used for leader election. Set this to a unique value when running multiple deployments in the same namespace.")
+	rootCmd.Flags().DurationVar(&leaderElectionLeaseDuration, "leader-election-lease-duration", 15*time.Second,
+		"The duration that non-leader candidates will wait to force acquire leadership. This is measured against time of last observed ack.")
+	rootCmd.Flags().DurationVar(&leaderElectionRenewDeadline, "leader-election-renew-deadline", 10*time.Second,
+		"The interval between attempts by the acting leader to renew its leadership before it stops leading. This must be less than the lease duration.")
+	rootCmd.Flags().DurationVar(&leaderElectionRetryPeriod, "leader-election-retry-period", 2*time.Second,
+		"The duration the clients should wait between attempting acquisition and renewal of a leadership.")
 	rootCmd.Flags().IntVar(&concurrent, "concurrent", 1, "The number of concurrent reconciles.")
 	rootCmd.Flags().Float32Var(&clientQPS, "client-qps", 50, "QPS configuration to be passed to rest.Client")
 	rootCmd.Flags().IntVar(&clientBurst, "client-burst", 100, "Maximum Burst allowed to be passed to rest.Client")

deploy/charts/external-secrets/README.md

@@ -167,6 +167,9 @@ The command removes all the Kubernetes components associated with the chart and
 | installCRDs | bool | `true` | If set, install and upgrade CRDs through helm chart. |
 | leaderElect | bool | `false` | If true, external-secrets will perform leader election between instances to ensure no more than one instance of external-secrets operates at a time. |
 | leaderElectionID | string | "external-secrets-controller" | ID of the lease object used for leader election. Leave empty to use the default ('external-secrets-controller'). Set to a unique value when running multiple independent ESO deployments in the same namespace. |
+| leaderElectionLeaseDuration | string | "15s" | Duration that non-leader candidates will wait to force acquire leadership. Increase this along with renewDeadline to tolerate a busy or briefly unavailable API server (for example during control plane maintenance) without churning leadership. Leave empty to use the controller default ('15s'). |
+| leaderElectionRenewDeadline | string | "10s" | Duration that the acting leader will retry refreshing leadership before giving up. Must be less than leaderElectionLeaseDuration. Leave empty to use the controller default ('10s'). |
+| leaderElectionRetryPeriod | string | "2s" | Duration the leader election client waits between tries of actions. Leave empty to use the controller default ('2s'). |
 | livenessProbe.enabled | bool | `false` | Enabled determines if the liveness probe should be used or not. By default it's disabled. |
 | livenessProbe.spec | object | `{"address":"","failureThreshold":5,"httpGet":{"path":"/healthz","port":"live"},"initialDelaySeconds":10,"periodSeconds":10,"port":8082,"successThreshold":1,"timeoutSeconds":5}` | The body of the liveness probe settings. |
 | livenessProbe.spec.address | string | `""` | Bind address for the health server used by both liveness and readiness probes (--live-addr flag). |

deploy/charts/external-secrets/templates/deployment.yaml

@@ -76,6 +76,15 @@ spec:
           {{- if .Values.leaderElectionID }}
           - --leader-election-id={{ .Values.leaderElectionID }}
           {{- end }}
+          {{- if .Values.leaderElectionLeaseDuration }}
+          - --leader-election-lease-duration={{ .Values.leaderElectionLeaseDuration }}
+          {{- end }}
+          {{- if .Values.leaderElectionRenewDeadline }}
+          - --leader-election-renew-deadline={{ .Values.leaderElectionRenewDeadline }}
+          {{- end }}
+          {{- if .Values.leaderElectionRetryPeriod }}
+          - --leader-election-retry-period={{ .Values.leaderElectionRetryPeriod }}
+          {{- end }}
           {{- if or .Values.scopedNamespace .Values.scopedRBAC }}
           - --namespace={{ .Values.scopedNamespace | default .Release.Namespace }}
           {{- end }}

deploy/charts/external-secrets/tests/controller_test.yaml

@@ -225,6 +225,32 @@ tests:
       - notContains:
           path: spec.template.spec.containers[0].args
           content: "--enable-http2"
+  - it: should render leader election timing args when set
+    set:
+      leaderElectionLeaseDuration: 60s
+      leaderElectionRenewDeadline: 40s
+      leaderElectionRetryPeriod: 5s
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].args
+          content: "--leader-election-lease-duration=60s"
+      - contains:
+          path: spec.template.spec.containers[0].args
+          content: "--leader-election-renew-deadline=40s"
+      - contains:
+          path: spec.template.spec.containers[0].args
+          content: "--leader-election-retry-period=5s"
+  - it: should not render leader election timing args by default
+    asserts:
+      - notContains:
+          path: spec.template.spec.containers[0].args
+          content: "--leader-election-lease-duration=15s"
+      - notContains:
+          path: spec.template.spec.containers[0].args
+          content: "--leader-election-renew-deadline=10s"
+      - notContains:
+          path: spec.template.spec.containers[0].args
+          content: "--leader-election-retry-period=2s"
   - it: should default to hostUsers absent
     capabilities:
       majorVersion: '1'

deploy/charts/external-secrets/values.schema.json

@@ -584,6 +584,15 @@
         "leaderElectionID": {
             "type": "string"
         },
+        "leaderElectionLeaseDuration": {
+            "type": "string"
+        },
+        "leaderElectionRenewDeadline": {
+            "type": "string"
+        },
+        "leaderElectionRetryPeriod": {
+            "type": "string"
+        },
         "livenessProbe": {
             "type": "object",
             "properties": {

deploy/charts/external-secrets/values.yaml

@@ -96,6 +96,24 @@ leaderElect: false
 # @default -- "external-secrets-controller"
 leaderElectionID: ""
 
+# -- Duration that non-leader candidates will wait to force acquire leadership.
+# Increase this along with renewDeadline to tolerate a busy or briefly unavailable API server
+# (for example during control plane maintenance) without churning leadership.
+# Leave empty to use the controller default ('15s').
+# @default -- "15s"
+leaderElectionLeaseDuration: ""
+
+# -- Duration that the acting leader will retry refreshing leadership before giving up.
+# Must be less than leaderElectionLeaseDuration.
+# Leave empty to use the controller default ('10s').
+# @default -- "10s"
+leaderElectionRenewDeadline: ""
+
+# -- Duration the leader election client waits between tries of actions.
+# Leave empty to use the controller default ('2s').
+# @default -- "2s"
+leaderElectionRetryPeriod: ""
+
 # -- If set external secrets will filter matching
 # Secret Stores with the appropriate controller values.
 controllerClass: ""