feat(certcontroller): Allow restricting CRDs and Webhook configs in Informer cache (#3588)

* feat: Add component labels to custom resource definitions

Prerequisite for restricting the CRDs cached by Informer

Signed-off-by: Tsubasa Nagasawa <toversus2357@gmail.com>

* feat(certcontroller): Allow restricting CRDs and Webhook configs in Informer cache

The certcontroller watches CRDs and Webhook configurations, and
manages CA certificates for conversion webhooks of CRDs and Webhook
configurations. Some clusters have a large number of CRDs and Webhook
configurations installed. Additionally, some CRDs have large object sizes.
Currently, the certcontroller holds all CRDs and Webhook configurations
in the Informer cache. Since this includes CRDs not managed by the
certcontroller for CA certificates, memory usage tends to be high.
This PR adds a label to the CRDs and configures the Informer cache to hold
only the CRDs and Webhook configurations restricted by the label selector.
It assumes that the CRDs have a label. Depending on how the External Secrets
Operator is managed, it may be possible to update the External Secrets
Operator without updating the CRDs, so as a precaution, it can be turned
on/off via a startup option. It is disabled by default.

Signed-off-by: Tsubasa Nagasawa <toversus2357@gmail.com>

---------

Signed-off-by: Tsubasa Nagasawa <toversus2357@gmail.com>
Signed-off-by: Bill Hamilton <bill.hamilton@delinea.com>

apis/externalsecrets/v1beta1/clusterexternalsecret_types.go

@@ -102,6 +102,7 @@ type ClusterExternalSecretStatus struct {
 // +kubebuilder:storageversion
 // +kubebuilder:resource:scope=Cluster,categories={externalsecrets},shortName=ces
 // +kubebuilder:subresource:status
+// +kubebuilder:metadata:labels="external-secrets.io/component=controller"
 // +kubebuilder:printcolumn:name="Store",type=string,JSONPath=`.spec.externalSecretSpec.secretStoreRef.name`
 // +kubebuilder:printcolumn:name="Refresh Interval",type=string,JSONPath=`.spec.refreshTime`
 // +kubebuilder:printcolumn:name="Ready",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].status`

apis/externalsecrets/v1beta1/externalsecret_types.go

@@ -453,6 +453,7 @@ type ExternalSecretStatus struct {
 // +kubebuilder:storageversion
 // ExternalSecret is the Schema for the external-secrets API.
 // +kubebuilder:subresource:status
+// +kubebuilder:metadata:labels="external-secrets.io/component=controller"
 // +kubebuilder:resource:scope=Namespaced,categories={externalsecrets},shortName=es
 // +kubebuilder:printcolumn:name="Store",type=string,JSONPath=`.spec.secretStoreRef.name`
 // +kubebuilder:printcolumn:name="Refresh Interval",type=string,JSONPath=`.spec.refreshInterval`

apis/externalsecrets/v1beta1/secretstore_types.go

@@ -262,6 +262,7 @@ type SecretStoreStatus struct {
 // +kubebuilder:printcolumn:name="Capabilities",type=string,JSONPath=`.status.capabilities`
 // +kubebuilder:printcolumn:name="Ready",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].status`
 // +kubebuilder:subresource:status
+// +kubebuilder:metadata:labels="external-secrets.io/component=controller"
 // +kubebuilder:resource:scope=Namespaced,categories={externalsecrets},shortName=ss
 type SecretStore struct {
 	metav1.TypeMeta   `json:",inline"`
@@ -289,6 +290,7 @@ type SecretStoreList struct {
 // +kubebuilder:printcolumn:name="Capabilities",type=string,JSONPath=`.status.capabilities`
 // +kubebuilder:printcolumn:name="Ready",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].status`
 // +kubebuilder:subresource:status
+// +kubebuilder:metadata:labels="external-secrets.io/component=controller"
 // +kubebuilder:resource:scope=Cluster,categories={externalsecrets},shortName=css
 type ClusterSecretStore struct {
 	metav1.TypeMeta   `json:",inline"`

apis/generators/v1alpha1/generator_acr.go

@@ -104,6 +104,7 @@ type AzureACRServicePrincipalAuthSecretRef struct {
 // +kubebuilder:object:root=true
 // +kubebuilder:storageversion
 // +kubebuilder:subresource:status
+// +kubebuilder:metadata:labels="external-secrets.io/component=controller"
 // +kubebuilder:resource:scope=Namespaced,categories={acraccesstoken},shortName=acraccesstoken
 type ACRAccessToken struct {
 	metav1.TypeMeta   `json:",inline"`

apis/generators/v1alpha1/generator_ecr.go

@@ -74,6 +74,7 @@ type AWSJWTAuth struct {
 // +kubebuilder:object:root=true
 // +kubebuilder:storageversion
 // +kubebuilder:subresource:status
+// +kubebuilder:metadata:labels="external-secrets.io/component=controller"
 // +kubebuilder:resource:scope=Namespaced,categories={ecrauthorizationtoken},shortName=ecrauthorizationtoken
 type ECRAuthorizationToken struct {
 	metav1.TypeMeta   `json:",inline"`

apis/generators/v1alpha1/generator_fake.go

@@ -35,6 +35,7 @@ type FakeSpec struct {
 // +kubebuilder:object:root=true
 // +kubebuilder:storageversion
 // +kubebuilder:subresource:status
+// +kubebuilder:metadata:labels="external-secrets.io/component=controller"
 // +kubebuilder:resource:scope=Namespaced,categories={fake},shortName=fake
 type Fake struct {
 	metav1.TypeMeta   `json:",inline"`

apis/generators/v1alpha1/generator_gcr.go

@@ -52,6 +52,7 @@ type GCPWorkloadIdentity struct {
 // +kubebuilder:object:root=true
 // +kubebuilder:storageversion
 // +kubebuilder:subresource:status
+// +kubebuilder:metadata:labels="external-secrets.io/component=controller"
 // +kubebuilder:resource:scope=Namespaced,categories={gcraccesstoken},shortName=gcraccesstoken
 type GCRAccessToken struct {
 	metav1.TypeMeta   `json:",inline"`

apis/generators/v1alpha1/generator_github.go

@@ -41,6 +41,7 @@ type GithubSecretRef struct {
 // +kubebuilder:object:root=true
 // +kubebuilder:storageversion
 // +kubebuilder:subresource:status
+// +kubebuilder:metadata:labels="external-secrets.io/component=controller"
 // +kubebuilder:resource:scope=Namespaced,categories={githubaccesstoken},shortName=githubaccesstoken
 type GithubAccessToken struct {
 	metav1.TypeMeta   `json:",inline"`

apis/generators/v1alpha1/generator_password.go

@@ -52,6 +52,7 @@ type PasswordSpec struct {
 // +kubebuilder:object:root=true
 // +kubebuilder:storageversion
 // +kubebuilder:subresource:status
+// +kubebuilder:metadata:labels="external-secrets.io/component=controller"
 // +kubebuilder:resource:scope=Namespaced,categories={password},shortName=password
 type Password struct {
 	metav1.TypeMeta   `json:",inline"`

apis/generators/v1alpha1/generator_vault.go

@@ -59,6 +59,7 @@ const (
 // +kubebuilder:object:root=true
 // +kubebuilder:storageversion
 // +kubebuilder:subresource:status
+// +kubebuilder:metadata:labels="external-secrets.io/component=controller"
 // +kubebuilder:resource:scope=Namespaced,categories={vaultdynamicsecret},shortName=vaultdynamicsecret
 type VaultDynamicSecret struct {
 	metav1.TypeMeta   `json:",inline"`

apis/generators/v1alpha1/generator_webhook.go

@@ -112,6 +112,7 @@ type SecretKeySelector struct {
 // +kubebuilder:object:root=true
 // +kubebuilder:storageversion
 // +kubebuilder:subresource:status
+// +kubebuilder:metadata:labels="external-secrets.io/component=controller"
 // +kubebuilder:resource:scope=Namespaced,categories={webhook},shortName=webhookl
 type Webhook struct {
 	metav1.TypeMeta   `json:",inline"`

cmd/certcontroller.go

@@ -22,14 +22,19 @@ import (
 
 	"github.com/spf13/cobra"
 	"go.uber.org/zap/zapcore"
+	admissionregistration "k8s.io/api/admissionregistration/v1"
 	v1 "k8s.io/api/core/v1"
+	apiextensions "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	"k8s.io/apimachinery/pkg/labels"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/cache"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 	"sigs.k8s.io/controller-runtime/pkg/metrics/server"
 	"sigs.k8s.io/controller-runtime/pkg/webhook"
 
+	"github.com/external-secrets/external-secrets/pkg/constants"
 	"github.com/external-secrets/external-secrets/pkg/controllers/crds"
 	"github.com/external-secrets/external-secrets/pkg/controllers/webhookconfig"
 )
@@ -59,6 +64,22 @@ var certcontrollerCmd = &cobra.Command{
 		logger := zap.New(zap.UseFlagOptions(&opts))
 		ctrl.SetLogger(logger)
 
+		cacheOptions := cache.Options{}
+		if enablePartialCache {
+			cacheOptions.ByObject = map[client.Object]cache.ByObject{
+				&admissionregistration.ValidatingWebhookConfiguration{}: {
+					Label: labels.SelectorFromSet(map[string]string{
+						constants.WellKnownLabelKey: constants.WellKnownLabelValueWebhook,
+					}),
+				},
+				&apiextensions.CustomResourceDefinition{}: {
+					Label: labels.SelectorFromSet(map[string]string{
+						constants.WellKnownLabelKey: constants.WellKnownLabelValueController,
+					}),
+				},
+			}
+		}
+
 		mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
 			Scheme: scheme,
 			Metrics: server.Options{
@@ -70,6 +91,7 @@ var certcontrollerCmd = &cobra.Command{
 			HealthProbeBindAddress: healthzAddr,
 			LeaderElection:         enableLeaderElection,
 			LeaderElectionID:       "crd-certs-controller",
+			Cache:                  cacheOptions,
 			Client: client.Options{
 				Cache: &client.CacheOptions{
 					DisableFor: []client.Object{
@@ -139,6 +161,8 @@ func init() {
 	certcontrollerCmd.Flags().StringVar(&secretName, "secret-name", "external-secrets-webhook", "Secret to store certs for webhook")
 	certcontrollerCmd.Flags().StringVar(&secretNamespace, "secret-namespace", "default", "namespace of the secret to store certs")
 	certcontrollerCmd.Flags().StringSliceVar(&crdNames, "crd-names", []string{"externalsecrets.external-secrets.io", "clustersecretstores.external-secrets.io", "secretstores.external-secrets.io"}, "CRD names reconciled by the controller")
+	certcontrollerCmd.Flags().BoolVar(&enablePartialCache, "enable-partial-cache", false,
+		"Enable caching of only the relevant CRDs and Webhook configurations in the Informer to improve memory efficiency")
 	certcontrollerCmd.Flags().BoolVar(&enableLeaderElection, "enable-leader-election", false,
 		"Enable leader election for controller manager. "+
 			"Enabling this will ensure there is only one active controller manager.")

cmd/root.go

@@ -64,6 +64,7 @@ var (
 	enableLeaderElection                  bool
 	enableSecretsCache                    bool
 	enableConfigMapsCache                 bool
+	enablePartialCache                    bool
 	concurrent                            int
 	port                                  int
 	clientQPS                             float32

config/crds/bases/external-secrets.io_clusterexternalsecrets.yaml

@@ -3,6 +3,8 @@ kind: CustomResourceDefinition
 metadata:
   annotations:
     controller-gen.kubebuilder.io/version: v0.15.0
+  labels:
+    external-secrets.io/component: controller
   name: clusterexternalsecrets.external-secrets.io
 spec:
   group: external-secrets.io

config/crds/bases/external-secrets.io_clustersecretstores.yaml

@@ -3,6 +3,8 @@ kind: CustomResourceDefinition
 metadata:
   annotations:
     controller-gen.kubebuilder.io/version: v0.15.0
+  labels:
+    external-secrets.io/component: controller
   name: clustersecretstores.external-secrets.io
 spec:
   group: external-secrets.io

config/crds/bases/external-secrets.io_externalsecrets.yaml

@@ -3,6 +3,8 @@ kind: CustomResourceDefinition
 metadata:
   annotations:
     controller-gen.kubebuilder.io/version: v0.15.0
+  labels:
+    external-secrets.io/component: controller
   name: externalsecrets.external-secrets.io
 spec:
   group: external-secrets.io

config/crds/bases/external-secrets.io_secretstores.yaml

@@ -3,6 +3,8 @@ kind: CustomResourceDefinition
 metadata:
   annotations:
     controller-gen.kubebuilder.io/version: v0.15.0
+  labels:
+    external-secrets.io/component: controller
   name: secretstores.external-secrets.io
 spec:
   group: external-secrets.io

config/crds/bases/generators.external-secrets.io_acraccesstokens.yaml

@@ -3,6 +3,8 @@ kind: CustomResourceDefinition
 metadata:
   annotations:
     controller-gen.kubebuilder.io/version: v0.15.0
+  labels:
+    external-secrets.io/component: controller
   name: acraccesstokens.generators.external-secrets.io
 spec:
   group: generators.external-secrets.io

config/crds/bases/generators.external-secrets.io_ecrauthorizationtokens.yaml

@@ -3,6 +3,8 @@ kind: CustomResourceDefinition
 metadata:
   annotations:
     controller-gen.kubebuilder.io/version: v0.15.0
+  labels:
+    external-secrets.io/component: controller
   name: ecrauthorizationtokens.generators.external-secrets.io
 spec:
   group: generators.external-secrets.io

config/crds/bases/generators.external-secrets.io_fakes.yaml

@@ -3,6 +3,8 @@ kind: CustomResourceDefinition
 metadata:
   annotations:
     controller-gen.kubebuilder.io/version: v0.15.0
+  labels:
+    external-secrets.io/component: controller
   name: fakes.generators.external-secrets.io
 spec:
   group: generators.external-secrets.io

config/crds/bases/generators.external-secrets.io_gcraccesstokens.yaml

@@ -3,6 +3,8 @@ kind: CustomResourceDefinition
 metadata:
   annotations:
     controller-gen.kubebuilder.io/version: v0.15.0
+  labels:
+    external-secrets.io/component: controller
   name: gcraccesstokens.generators.external-secrets.io
 spec:
   group: generators.external-secrets.io

config/crds/bases/generators.external-secrets.io_githubaccesstokens.yaml

@@ -3,6 +3,8 @@ kind: CustomResourceDefinition
 metadata:
   annotations:
     controller-gen.kubebuilder.io/version: v0.15.0
+  labels:
+    external-secrets.io/component: controller
   name: githubaccesstokens.generators.external-secrets.io
 spec:
   group: generators.external-secrets.io

config/crds/bases/generators.external-secrets.io_passwords.yaml

@@ -3,6 +3,8 @@ kind: CustomResourceDefinition
 metadata:
   annotations:
     controller-gen.kubebuilder.io/version: v0.15.0
+  labels:
+    external-secrets.io/component: controller
   name: passwords.generators.external-secrets.io
 spec:
   group: generators.external-secrets.io

config/crds/bases/generators.external-secrets.io_vaultdynamicsecrets.yaml

@@ -3,6 +3,8 @@ kind: CustomResourceDefinition
 metadata:
   annotations:
     controller-gen.kubebuilder.io/version: v0.15.0
+  labels:
+    external-secrets.io/component: controller
   name: vaultdynamicsecrets.generators.external-secrets.io
 spec:
   group: generators.external-secrets.io

config/crds/bases/generators.external-secrets.io_webhooks.yaml

@@ -3,6 +3,8 @@ kind: CustomResourceDefinition
 metadata:
   annotations:
     controller-gen.kubebuilder.io/version: v0.15.0
+  labels:
+    external-secrets.io/component: controller
   name: webhooks.generators.external-secrets.io
 spec:
   group: generators.external-secrets.io

deploy/crds/bundle.yaml

@@ -3,6 +3,8 @@ kind: CustomResourceDefinition
 metadata:
   annotations:
     controller-gen.kubebuilder.io/version: v0.15.0
+  labels:
+    external-secrets.io/component: controller
   name: clusterexternalsecrets.external-secrets.io
 spec:
   group: external-secrets.io
@@ -659,6 +661,8 @@ kind: CustomResourceDefinition
 metadata:
   annotations:
     controller-gen.kubebuilder.io/version: v0.15.0
+  labels:
+    external-secrets.io/component: controller
   name: clustersecretstores.external-secrets.io
 spec:
   group: external-secrets.io
@@ -5013,6 +5017,8 @@ kind: CustomResourceDefinition
 metadata:
   annotations:
     controller-gen.kubebuilder.io/version: v0.15.0
+  labels:
+    external-secrets.io/component: controller
   name: externalsecrets.external-secrets.io
 spec:
   group: external-secrets.io
@@ -6204,6 +6210,8 @@ kind: CustomResourceDefinition
 metadata:
   annotations:
     controller-gen.kubebuilder.io/version: v0.15.0
+  labels:
+    external-secrets.io/component: controller
   name: secretstores.external-secrets.io
 spec:
   group: external-secrets.io
@@ -10558,6 +10566,8 @@ kind: CustomResourceDefinition
 metadata:
   annotations:
     controller-gen.kubebuilder.io/version: v0.15.0
+  labels:
+    external-secrets.io/component: controller
   name: acraccesstokens.generators.external-secrets.io
 spec:
   group: generators.external-secrets.io
@@ -10753,6 +10763,8 @@ kind: CustomResourceDefinition
 metadata:
   annotations:
     controller-gen.kubebuilder.io/version: v0.15.0
+  labels:
+    external-secrets.io/component: controller
   name: ecrauthorizationtokens.generators.external-secrets.io
 spec:
   group: generators.external-secrets.io
@@ -10919,6 +10931,8 @@ kind: CustomResourceDefinition
 metadata:
   annotations:
     controller-gen.kubebuilder.io/version: v0.15.0
+  labels:
+    external-secrets.io/component: controller
   name: fakes.generators.external-secrets.io
 spec:
   group: generators.external-secrets.io
@@ -10994,6 +11008,8 @@ kind: CustomResourceDefinition
 metadata:
   annotations:
     controller-gen.kubebuilder.io/version: v0.15.0
+  labels:
+    external-secrets.io/component: controller
   name: gcraccesstokens.generators.external-secrets.io
 spec:
   group: generators.external-secrets.io
@@ -11121,6 +11137,8 @@ kind: CustomResourceDefinition
 metadata:
   annotations:
     controller-gen.kubebuilder.io/version: v0.15.0
+  labels:
+    external-secrets.io/component: controller
   name: githubaccesstokens.generators.external-secrets.io
 spec:
   group: generators.external-secrets.io
@@ -11222,6 +11240,8 @@ kind: CustomResourceDefinition
 metadata:
   annotations:
     controller-gen.kubebuilder.io/version: v0.15.0
+  labels:
+    external-secrets.io/component: controller
   name: passwords.generators.external-secrets.io
 spec:
   group: generators.external-secrets.io
@@ -11319,6 +11339,8 @@ kind: CustomResourceDefinition
 metadata:
   annotations:
     controller-gen.kubebuilder.io/version: v0.15.0
+  labels:
+    external-secrets.io/component: controller
   name: vaultdynamicsecrets.generators.external-secrets.io
 spec:
   group: generators.external-secrets.io
@@ -12010,6 +12032,8 @@ kind: CustomResourceDefinition
 metadata:
   annotations:
     controller-gen.kubebuilder.io/version: v0.15.0
+  labels:
+    external-secrets.io/component: controller
   name: webhooks.generators.external-secrets.io
 spec:
   group: generators.external-secrets.io

pkg/constants/constants.go

@@ -97,4 +97,8 @@ const (
 
 	StatusError   = "error"
 	StatusSuccess = "success"
+
+	WellKnownLabelKey             = "external-secrets.io/component"
+	WellKnownLabelValueController = "controller"
+	WellKnownLabelValueWebhook    = "webhook"
 )

pkg/controllers/webhookconfig/webhookconfig.go

@@ -33,6 +33,8 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
+
+	"github.com/external-secrets/external-secrets/pkg/constants"
 )
 
 type Reconciler struct {
@@ -75,9 +77,6 @@ func New(k8sClient client.Client, scheme *runtime.Scheme, leaderChan <-chan stru
 }
 
 const (
-	wellKnownLabelKey   = "external-secrets.io/component"
-	wellKnownLabelValue = "webhook"
-
 	ReasonUpdateFailed   = "UpdateFailed"
 	errWebhookNotReady   = "webhook not ready"
 	errSubsetsNotReady   = "subsets not ready"
@@ -98,8 +97,8 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 		return ctrl.Result{}, err
 	}
 
-	if cfg.Labels[wellKnownLabelKey] != wellKnownLabelValue {
-		log.Info("ignoring webhook due to missing labels", wellKnownLabelKey, wellKnownLabelValue)
+	if cfg.Labels[constants.WellKnownLabelKey] != constants.WellKnownLabelValueWebhook {
+		log.Info("ignoring webhook due to missing labels", constants.WellKnownLabelKey, constants.WellKnownLabelValueWebhook)
 		return ctrl.Result{}, nil
 	}
 

pkg/controllers/webhookconfig/webhookconfig_test.go

@@ -25,6 +25,8 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	pointer "k8s.io/utils/ptr"
 
+	"github.com/external-secrets/external-secrets/pkg/constants"
+
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 )
@@ -126,7 +128,7 @@ var _ = Describe("ValidatingWebhookConfig reconcile", Ordered, func() {
 	}
 
 	IgnoreNoMatch := func(tc *testCase) {
-		delete(tc.vwc.ObjectMeta.Labels, wellKnownLabelKey)
+		delete(tc.vwc.ObjectMeta.Labels, constants.WellKnownLabelKey)
 		tc.assert = func() {
 			Consistently(func() bool {
 				var vwc admissionregistration.ValidatingWebhookConfiguration
@@ -232,7 +234,7 @@ func makeValidatingWebhookConfig() *admissionregistration.ValidatingWebhookConfi
 		ObjectMeta: metav1.ObjectMeta{
 			Name: "name-shouldnt-matter",
 			Labels: map[string]string{
-				wellKnownLabelKey: wellKnownLabelValue,
+				constants.WellKnownLabelKey: constants.WellKnownLabelValueWebhook,
 			},
 		},
 		Webhooks: []admissionregistration.ValidatingWebhook{