feat: migration from endpoint to endpointslice (#5008)

* feat: migration from endpoint to endpointslice

Signed-off-by: Michal Virgovic <michal.virgovic1@gmail.com>

* feat: extract the duplicate code to utils

Signed-off-by: Michal Virgovic <michal.virgovic1@gmail.com>

* feat: rename error vars

Signed-off-by: Michal Virgovic <michal.virgovic1@gmail.com>

* feat: add back the endpoint access

Signed-off-by: Michal Virgovic <michal.virgovic1@gmail.com>

---------

Signed-off-by: Michal Virgovic <michal.virgovic1@gmail.com>
Co-authored-by: Gergely Brautigam <182850+Skarlso@users.noreply.github.com>

deploy/charts/external-secrets/templates/cert-controller-rbac.yaml

@@ -43,6 +43,14 @@ rules:
     - "get"
     - "watch"
   - apiGroups:
+    - "discovery.k8s.io"
+    resources:
+    - "endpointslices"
+    verbs:
+    - "list"
+    - "get"
+    - "watch"
+  - apiGroups:
     - ""
     resources:
     - "events"

pkg/controllers/crds/crds_controller.go

@@ -33,6 +33,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/external-secrets/external-secrets/pkg/utils"
 	"github.com/go-logr/logr"
 	corev1 "k8s.io/api/core/v1"
 	apiext "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
@@ -53,9 +54,7 @@ const (
 	certValidityDuration = 10 * 365 * 24 * time.Hour
 	LookaheadInterval    = 90 * 24 * time.Hour
 
-	errResNotReady       = "resource not ready: %s"
-	errSubsetsNotReady   = "subsets not ready"
-	errAddressesNotReady = "addresses not ready"
+	errResNotReady = "resource not ready: %s"
 )
 
 type Reconciler struct {
@@ -151,7 +150,7 @@ func (r *Reconciler) ReadyCheck(_ *http.Request) error {
 	if err := r.checkCRDs(); err != nil {
 		return err
 	}
-	return r.checkEndpoints()
+	return utils.CheckEndpointSlicesReady(context.TODO(), r.Client, r.SvcName, r.SvcNamespace)
 }
 
 func (r *Reconciler) checkCRDs() error {
@@ -166,24 +165,6 @@ func (r *Reconciler) checkCRDs() error {
 	return nil
 }
 
-func (r *Reconciler) checkEndpoints() error {
-	var eps corev1.Endpoints
-	err := r.Get(context.TODO(), types.NamespacedName{
-		Name:      r.SvcName,
-		Namespace: r.SvcNamespace,
-	}, &eps)
-	if err != nil {
-		return err
-	}
-	if len(eps.Subsets) == 0 {
-		return errors.New(errSubsetsNotReady)
-	}
-	if len(eps.Subsets[0].Addresses) == 0 {
-		return errors.New(errAddressesNotReady)
-	}
-	return nil
-}
-
 func (r *Reconciler) SetupWithManager(mgr ctrl.Manager, opts controller.Options) error {
 	r.recorder = mgr.GetEventRecorderFor("custom-resource-definition")
 	return ctrl.NewControllerManagedBy(mgr).

pkg/controllers/webhookconfig/webhookconfig.go

@@ -23,6 +23,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/external-secrets/external-secrets/pkg/utils"
 	"github.com/go-logr/logr"
 	admissionregistration "k8s.io/api/admissionregistration/v1"
 	v1 "k8s.io/api/core/v1"
@@ -84,11 +85,9 @@ func New(k8sClient client.Client, scheme *runtime.Scheme, leaderChan <-chan stru
 }
 
 const (
-	ReasonUpdateFailed   = "UpdateFailed"
-	errWebhookNotReady   = "webhook not ready"
-	errSubsetsNotReady   = "subsets not ready"
-	errAddressesNotReady = "addresses not ready"
-	errCACertNotReady    = "ca cert not yet ready"
+	ReasonUpdateFailed = "UpdateFailed"
+	errWebhookNotReady = "webhook not ready"
+	errCACertNotReady  = "ca cert not yet ready"
 
 	caCertName = "ca.crt"
 )
@@ -153,21 +152,8 @@ func (r *Reconciler) ReadyCheck(_ *http.Request) error {
 	if !r.webhookReady {
 		return errors.New(errWebhookNotReady)
 	}
-	var eps v1.Endpoints
-	err := r.Get(context.TODO(), types.NamespacedName{
-		Name:      r.SvcName,
-		Namespace: r.SvcNamespace,
-	}, &eps)
-	if err != nil {
-		return err
-	}
-	if len(eps.Subsets) == 0 {
-		return errors.New(errSubsetsNotReady)
-	}
-	if len(eps.Subsets[0].Addresses) == 0 {
-		return errors.New(errAddressesNotReady)
-	}
-	return nil
+
+	return utils.CheckEndpointSlicesReady(context.TODO(), r.Client, r.SvcName, r.SvcNamespace)
 }
 
 // reads the ca cert and updates the webhook config.

pkg/controllers/webhookconfig/webhookconfig_test.go

@@ -21,6 +21,7 @@ import (
 
 	admissionregistration "k8s.io/api/admissionregistration/v1"
 	corev1 "k8s.io/api/core/v1"
+	discoveryv1 "k8s.io/api/discovery/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	pointer "k8s.io/utils/ptr"
@@ -54,11 +55,11 @@ lNjvmHAXUfOE/cbD7EP++X17kWt41FjmePc=
 `
 
 type testCase struct {
-	vwc       *admissionregistration.ValidatingWebhookConfiguration
-	service   *corev1.Service
-	endpoints *corev1.Endpoints
-	secret    *corev1.Secret
-	assert    func()
+	vwc           *admissionregistration.ValidatingWebhookConfiguration
+	service       *corev1.Service
+	endpointSlice *discoveryv1.EndpointSlice
+	secret        *corev1.Secret
+	assert        func()
 }
 
 var _ = Describe("ValidatingWebhookConfig reconcile", Ordered, func() {
@@ -73,17 +74,15 @@ var _ = Describe("ValidatingWebhookConfig reconcile", Ordered, func() {
 		k8sClient.Delete(ctx, test.vwc)
 		k8sClient.Delete(ctx, test.secret)
 		k8sClient.Delete(ctx, test.service)
-		k8sClient.Delete(ctx, test.endpoints)
+		k8sClient.Delete(ctx, test.endpointSlice)
 	})
 
 	// Should patch VWC
 	PatchAndReady := func(tc *testCase) {
-		tc.endpoints.Subsets = nil
-
-		// endpoints become ready in a moment
+		// endpoint slice becomes ready in a moment
 		go func() {
 			<-time.After(time.Second * 4)
-			eps := makeEndpoints()
+			eps := makeEndpointSlice()
 			err := k8sClient.Update(context.Background(), eps)
 			Expect(err).ToNot(HaveOccurred())
 		}()
@@ -231,7 +230,7 @@ var _ = Describe("ValidatingWebhookConfig reconcile", Ordered, func() {
 		err = k8sClient.Create(ctx, test.service)
 		Expect(err).ToNot(HaveOccurred())
 
-		err = k8sClient.Create(ctx, test.endpoints)
+		err = k8sClient.Create(ctx, test.endpointSlice)
 		Expect(err).ToNot(HaveOccurred())
 
 		test.assert()
@@ -312,19 +311,20 @@ func makeService() *corev1.Service {
 	}
 }
 
-func makeEndpoints() *corev1.Endpoints {
-	return &corev1.Endpoints{
+func makeEndpointSlice() *discoveryv1.EndpointSlice {
+	return &discoveryv1.EndpointSlice{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      ctrlSvcName,
+			Name:      ctrlSvcName + "-slice",
 			Namespace: ctrlSvcNamespace,
+			Labels: map[string]string{
+				"kubernetes.io/service-name": ctrlSvcName,
+			},
 		},
-		Subsets: []corev1.EndpointSubset{
+		AddressType: discoveryv1.AddressTypeIPv4,
+		Endpoints: []discoveryv1.Endpoint{
 			{
-				Addresses: []corev1.EndpointAddress{
-					{
-						IP: "1.2.3.4",
-					},
-				},
+				Addresses:  []string{"1.2.3.4"},
+				Conditions: discoveryv1.EndpointConditions{Ready: pointer.To(true)},
 			},
 		},
 	}
@@ -335,9 +335,9 @@ func makeDefaultTestcase() *testCase {
 		assert: func() {
 			// this is a noop by default
 		},
-		vwc:       makeValidatingWebhookConfig(),
-		secret:    makeSecret(),
-		service:   makeService(),
-		endpoints: makeEndpoints(),
+		vwc:           makeValidatingWebhookConfig(),
+		secret:        makeSecret(),
+		service:       makeService(),
+		endpointSlice: makeEndpointSlice(),
 	}
 }

pkg/utils/utils.go

@@ -39,6 +39,7 @@ import (
 
 	"github.com/go-logr/logr"
 	corev1 "k8s.io/api/core/v1"
+	discoveryv1 "k8s.io/api/discovery/v1"
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -58,8 +59,10 @@ const (
 )
 
 var (
-	errKeyNotFound = errors.New("key not found")
-	unicodeRegex   = regexp.MustCompile(`_U([0-9a-fA-F]{4,5})_`)
+	errAddressesNotReady      = errors.New("addresses not ready")
+	errEndpointSlicesNotReady = errors.New("endpointSlice objects not ready")
+	errKeyNotFound            = errors.New("key not found")
+	unicodeRegex              = regexp.MustCompile(`_U([0-9a-fA-F]{4,5})_`)
 )
 
 // JSONMarshal takes an interface and returns a new escaped and encoded byte slice.
@@ -828,3 +831,29 @@ func getCertFromConfigMap(ctx context.Context, namespace string, c client.Client
 
 	return []byte(val), nil
 }
+
+func CheckEndpointSlicesReady(ctx context.Context, c client.Client, svcName, svcNamespace string) error {
+	var sliceList discoveryv1.EndpointSliceList
+	err := c.List(ctx, &sliceList,
+		client.InNamespace(svcNamespace),
+		client.MatchingLabels{"kubernetes.io/service-name": svcName},
+	)
+	if err != nil {
+		return err
+	}
+	if len(sliceList.Items) == 0 {
+		return errEndpointSlicesNotReady
+	}
+	readyAddresses := 0
+	for _, slice := range sliceList.Items {
+		for _, ep := range slice.Endpoints {
+			if ep.Conditions.Ready != nil && *ep.Conditions.Ready {
+				readyAddresses += len(ep.Addresses)
+			}
+		}
+	}
+	if readyAddresses == 0 {
+		return errAddressesNotReady
+	}
+	return nil
+}