helm-external-secrets/pkg/controllers/provider/controller_test.go

/*
Copyright © The ESO Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    https://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package provider

import (
	"context"
	"crypto/rand"
	"crypto/rsa"
	"crypto/tls"
	"crypto/x509"
	"crypto/x509/pkix"
	"encoding/pem"
	"math/big"
	"net"
	"testing"
	"time"

	"github.com/go-logr/logr"
	"github.com/prometheus/client_golang/prometheus"
	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/credentials"
	"google.golang.org/grpc/status"
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
	ctrl "sigs.k8s.io/controller-runtime"
	"sigs.k8s.io/controller-runtime/pkg/client"
	fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake"

	esv1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1"
	k8sv2alpha1 "github.com/external-secrets/external-secrets/apis/provider/kubernetes/v2alpha1"
	ctrlmetrics "github.com/external-secrets/external-secrets/pkg/controllers/metrics"
	pb "github.com/external-secrets/external-secrets/proto/provider"
)

type recordingProviderGRPCServer struct {
	pb.UnimplementedSecretStoreProviderServer

	validateRequest     *pb.ValidateRequest
	validateResponse    *pb.ValidateResponse
	capabilitiesRequest *pb.CapabilitiesRequest
	capabilitiesResp    *pb.CapabilitiesResponse
	capabilitiesErr     error
}

func (s *recordingProviderGRPCServer) Validate(_ context.Context, req *pb.ValidateRequest) (*pb.ValidateResponse, error) {
	s.validateRequest = req
	if s.validateResponse != nil {
		return s.validateResponse, nil
	}
	return &pb.ValidateResponse{Valid: true}, nil
}

func (s *recordingProviderGRPCServer) Capabilities(_ context.Context, req *pb.CapabilitiesRequest) (*pb.CapabilitiesResponse, error) {
	s.capabilitiesRequest = req
	if s.capabilitiesErr != nil {
		return nil, s.capabilitiesErr
	}
	if s.capabilitiesResp != nil {
		return s.capabilitiesResp, nil
	}
	return &pb.CapabilitiesResponse{Capabilities: pb.SecretStoreCapabilities_READ_WRITE}, nil
}

func TestValidateStoreAndGetCapabilitiesSendsProviderReferenceAndNamespace(t *testing.T) {
	scheme := runtime.NewScheme()
	utilruntime.Must(clientgoscheme.AddToScheme(scheme))
	utilruntime.Must(esv1.AddToScheme(scheme))

	server, address, tlsSecret := newProviderGRPCServer(t)
	store := &esv1.Provider{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "provider",
			Namespace: "tenant-a",
		},
		Spec: esv1.ProviderSpec{
			Config: esv1.ProviderConfig{
				Address: address,
				ProviderRef: esv1.ProviderReference{
					APIVersion: "provider.external-secrets.io/v2alpha1",
					Kind:       "Kubernetes",
					Name:       "backend",
					Namespace:  "config-ns",
				},
			},
		},
	}

	kubeClient := fakeclient.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(store, &corev1.Secret{
			ObjectMeta: metav1.ObjectMeta{
				Name:      "external-secrets-provider-tls",
				Namespace: "tenant-a",
			},
			Data: tlsSecret,
		}).
		Build()

	r := &Reconciler{Client: kubeClient, Log: logr.Discard()}

	caps, err := r.validateStoreAndGetCapabilities(context.Background(), store)
	if err != nil {
		t.Fatalf("validateStoreAndGetCapabilities() error = %v", err)
	}
	if caps != esv1.ProviderReadWrite {
		t.Fatalf("expected ProviderReadWrite, got %q", caps)
	}
	assertProviderReference(t, server.validateRequest.ProviderRef, store.Spec.Config.ProviderRef, esv1.ProviderKindStr)
	if server.validateRequest.SourceNamespace != "tenant-a" {
		t.Fatalf("unexpected validate source namespace: %q", server.validateRequest.SourceNamespace)
	}
	assertProviderReference(t, server.capabilitiesRequest.ProviderRef, store.Spec.Config.ProviderRef, esv1.ProviderKindStr)
	if server.capabilitiesRequest.SourceNamespace != "tenant-a" {
		t.Fatalf("unexpected capabilities source namespace: %q", server.capabilitiesRequest.SourceNamespace)
	}
}

func TestValidateStoreAndGetCapabilitiesFallsBackToReadOnlyOnCapabilitiesError(t *testing.T) {
	scheme := runtime.NewScheme()
	utilruntime.Must(clientgoscheme.AddToScheme(scheme))
	utilruntime.Must(esv1.AddToScheme(scheme))

	server, address, tlsSecret := newProviderGRPCServer(t)
	server.capabilitiesErr = status.Error(codes.Unavailable, "capabilities unavailable")

	store := &esv1.Provider{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "provider",
			Namespace: "tenant-a",
		},
		Spec: esv1.ProviderSpec{
			Config: esv1.ProviderConfig{
				Address: address,
				ProviderRef: esv1.ProviderReference{
					APIVersion: "provider.external-secrets.io/v2alpha1",
					Kind:       "Kubernetes",
					Name:       "backend",
				},
			},
		},
	}

	kubeClient := fakeclient.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(store, &corev1.Secret{
			ObjectMeta: metav1.ObjectMeta{
				Name:      "external-secrets-provider-tls",
				Namespace: "tenant-a",
			},
			Data: tlsSecret,
		}).
		Build()

	r := &Reconciler{Client: kubeClient, Log: logr.Discard()}

	caps, err := r.validateStoreAndGetCapabilities(context.Background(), store)
	if err != nil {
		t.Fatalf("expected fallback to read-only, got error %v", err)
	}
	if caps != esv1.ProviderReadOnly {
		t.Fatalf("expected ProviderReadOnly, got %q", caps)
	}
}

func TestValidateStoreAndGetCapabilitiesReturnsValidationError(t *testing.T) {
	scheme := runtime.NewScheme()
	utilruntime.Must(clientgoscheme.AddToScheme(scheme))
	utilruntime.Must(esv1.AddToScheme(scheme))

	server, address, tlsSecret := newProviderGRPCServer(t)
	server.validateResponse = &pb.ValidateResponse{
		Valid: false,
		Error: "invalid credentials",
	}

	store := &esv1.Provider{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "provider",
			Namespace: "tenant-a",
		},
		Spec: esv1.ProviderSpec{
			Config: esv1.ProviderConfig{
				Address: address,
				ProviderRef: esv1.ProviderReference{
					APIVersion: "provider.external-secrets.io/v2alpha1",
					Kind:       "Kubernetes",
					Name:       "backend",
				},
			},
		},
	}

	kubeClient := fakeclient.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(store, &corev1.Secret{
			ObjectMeta: metav1.ObjectMeta{
				Name:      "external-secrets-provider-tls",
				Namespace: "tenant-a",
			},
			Data: tlsSecret,
		}).
		Build()

	r := &Reconciler{Client: kubeClient, Log: logr.Discard()}

	_, err := r.validateStoreAndGetCapabilities(context.Background(), store)
	if err == nil || err.Error() != "provider validation failed: provider validation failed: invalid credentials" {
		t.Fatalf("unexpected error: %v", err)
	}
}

func TestReconcileValidationFailureClearsStaleCapabilitiesAndUpdatesCondition(t *testing.T) {
	ctrlmetrics.SetUpLabelNames(false)
	previousMetrics := gaugeVecMetrics
	gaugeVecMetrics = map[string]*prometheus.GaugeVec{
		ProviderReconcileDurationKey: prometheus.NewGaugeVec(prometheus.GaugeOpts{
			Subsystem: ProviderSubsystem,
			Name:      ProviderReconcileDurationKey,
		}, ctrlmetrics.NonConditionMetricLabelNames),
		StatusConditionKey: prometheus.NewGaugeVec(prometheus.GaugeOpts{
			Subsystem: ProviderSubsystem,
			Name:      StatusConditionKey,
		}, ctrlmetrics.ConditionMetricLabelNames),
	}
	t.Cleanup(func() {
		gaugeVecMetrics = previousMetrics
	})

	scheme := runtime.NewScheme()
	utilruntime.Must(clientgoscheme.AddToScheme(scheme))
	utilruntime.Must(esv1.AddToScheme(scheme))

	server, address, tlsSecret := newProviderGRPCServer(t)
	server.validateResponse = &pb.ValidateResponse{
		Valid: false,
		Error: "invalid credentials",
	}

	store := &esv1.Provider{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "provider",
			Namespace: "tenant-a",
		},
		Spec: esv1.ProviderSpec{
			Config: esv1.ProviderConfig{
				Address: address,
				ProviderRef: esv1.ProviderReference{
					APIVersion: "provider.external-secrets.io/v2alpha1",
					Kind:       "Kubernetes",
					Name:       "backend",
				},
			},
		},
		Status: esv1.ProviderStatus{
			Capabilities: esv1.ProviderReadWrite,
		},
	}

	kubeClient := fakeclient.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(store, &corev1.Secret{
			ObjectMeta: metav1.ObjectMeta{
				Name:      "external-secrets-provider-tls",
				Namespace: "tenant-a",
			},
			Data: tlsSecret,
		}).
		WithStatusSubresource(store).
		Build()

	r := &Reconciler{
		Client:          kubeClient,
		Log:             logr.Discard(),
		RequeueInterval: 37 * time.Second,
	}

	result, err := r.Reconcile(context.Background(), ctrl.Request{
		NamespacedName: client.ObjectKey{Name: "provider", Namespace: "tenant-a"},
	})
	if err != nil {
		t.Fatalf("Reconcile() error = %v", err)
	}
	if result.RequeueAfter != 37*time.Second {
		t.Fatalf("expected requeue interval, got %#v", result)
	}

	var updated esv1.Provider
	if err := kubeClient.Get(context.Background(), client.ObjectKey{Name: "provider", Namespace: "tenant-a"}, &updated); err != nil {
		t.Fatalf("Get() error = %v", err)
	}

	if updated.Status.Capabilities != "" {
		t.Fatalf("expected capabilities to be cleared, got %q", updated.Status.Capabilities)
	}
	if len(updated.Status.Conditions) != 1 {
		t.Fatalf("expected a single condition, got %#v", updated.Status.Conditions)
	}

	condition := updated.Status.Conditions[0]
	if condition.Type != esv1.ProviderReady || condition.Status != metav1.ConditionFalse {
		t.Fatalf("unexpected condition: %#v", condition)
	}
	if condition.Reason != "ValidationFailed" {
		t.Fatalf("unexpected condition reason: %q", condition.Reason)
	}
	if condition.Message != "provider validation failed: provider validation failed: invalid credentials" {
		t.Fatalf("unexpected condition message: %q", condition.Message)
	}
}

func TestSetNotReadyConditionUpdatesReasonAndMessageWithoutChangingTransitionTime(t *testing.T) {
	ctrlmetrics.SetUpLabelNames(false)
	previousMetrics := gaugeVecMetrics
	gaugeVecMetrics = map[string]*prometheus.GaugeVec{
		StatusConditionKey: prometheus.NewGaugeVec(prometheus.GaugeOpts{
			Subsystem: ProviderSubsystem,
			Name:      StatusConditionKey,
		}, ctrlmetrics.ConditionMetricLabelNames),
	}
	t.Cleanup(func() {
		gaugeVecMetrics = previousMetrics
	})

	previousTransition := metav1.NewTime(time.Unix(1700000000, 0))
	store := &esv1.Provider{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "provider",
			Namespace: "tenant-a",
		},
		Status: esv1.ProviderStatus{
			Conditions: []esv1.ProviderCondition{{
				Type:               esv1.ProviderReady,
				Status:             metav1.ConditionFalse,
				LastTransitionTime: previousTransition,
				Reason:             "OldReason",
				Message:            "old message",
			}},
		},
	}

	r := &Reconciler{Log: logr.Discard()}
	r.setNotReadyCondition(store, "ValidationFailed", "new message")

	if len(store.Status.Conditions) != 1 {
		t.Fatalf("expected a single condition, got %#v", store.Status.Conditions)
	}

	condition := store.Status.Conditions[0]
	if condition.Status != metav1.ConditionFalse {
		t.Fatalf("expected false status, got %q", condition.Status)
	}
	if condition.Reason != "ValidationFailed" {
		t.Fatalf("expected updated reason, got %q", condition.Reason)
	}
	if condition.Message != "new message" {
		t.Fatalf("expected updated message, got %q", condition.Message)
	}
	if !condition.LastTransitionTime.Equal(&previousTransition) {
		t.Fatalf("expected transition time to remain %v, got %v", previousTransition, condition.LastTransitionTime)
	}
}

func TestFindProvidersForKubernetesProviderEnqueuesMatchingProviders(t *testing.T) {
	scheme := runtime.NewScheme()
	utilruntime.Must(clientgoscheme.AddToScheme(scheme))
	utilruntime.Must(esv1.AddToScheme(scheme))

	kubeClient := fakeclient.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(
			&esv1.Provider{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "match",
					Namespace: "tenant-a",
				},
				Spec: esv1.ProviderSpec{
					Config: esv1.ProviderConfig{
						ProviderRef: esv1.ProviderReference{
							APIVersion: k8sv2alpha1.GroupVersion.String(),
							Kind:       k8sv2alpha1.Kind,
							Name:       "backend",
							Namespace:  "config-ns",
						},
					},
				},
			},
			&esv1.Provider{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "wrong-name",
					Namespace: "tenant-a",
				},
				Spec: esv1.ProviderSpec{
					Config: esv1.ProviderConfig{
						ProviderRef: esv1.ProviderReference{
							APIVersion: k8sv2alpha1.GroupVersion.String(),
							Kind:       k8sv2alpha1.Kind,
							Name:       "other",
							Namespace:  "config-ns",
						},
					},
				},
			},
			&esv1.Provider{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "wrong-kind",
					Namespace: "tenant-a",
				},
				Spec: esv1.ProviderSpec{
					Config: esv1.ProviderConfig{
						ProviderRef: esv1.ProviderReference{
							APIVersion: k8sv2alpha1.GroupVersion.String(),
							Kind:       "AWS",
							Name:       "backend",
							Namespace:  "config-ns",
						},
					},
				},
			},
		).
		Build()

	requests := findProvidersForKubernetesProvider(context.Background(), kubeClient, &k8sv2alpha1.Kubernetes{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "backend",
			Namespace: "config-ns",
		},
	})

	if len(requests) != 1 {
		t.Fatalf("expected one reconcile request, got %#v", requests)
	}
	if requests[0].NamespacedName != (client.ObjectKey{Name: "match", Namespace: "tenant-a"}) {
		t.Fatalf("unexpected reconcile request: %#v", requests[0])
	}
}

func newProviderGRPCServer(t *testing.T) (*recordingProviderGRPCServer, string, map[string][]byte) {
	t.Helper()

	serverCert, serverKey, clientCert, clientKey, caCert := newProviderTLSArtifacts(t, "127.0.0.1")

	caPool := x509.NewCertPool()
	if !caPool.AppendCertsFromPEM(caCert) {
		t.Fatal("failed to append CA cert")
	}
	tlsCert, err := tls.X509KeyPair(serverCert, serverKey)
	if err != nil {
		t.Fatalf("X509KeyPair() error = %v", err)
	}

	lis, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatalf("Listen() error = %v", err)
	}

	server := &recordingProviderGRPCServer{}
	grpcServer := grpc.NewServer(grpc.Creds(credentials.NewTLS(&tls.Config{
		MinVersion:   tls.VersionTLS12,
		Certificates: []tls.Certificate{tlsCert},
		ClientCAs:    caPool,
		ClientAuth:   tls.RequireAndVerifyClientCert,
	})))
	pb.RegisterSecretStoreProviderServer(grpcServer, server)
	go func() {
		_ = grpcServer.Serve(lis)
	}()

	t.Cleanup(func() {
		grpcServer.Stop()
		_ = lis.Close()
	})

	return server, lis.Addr().String(), map[string][]byte{
		"ca.crt":     caCert,
		"client.crt": clientCert,
		"client.key": clientKey,
	}
}

func assertProviderReference(t *testing.T, got *pb.ProviderReference, want esv1.ProviderReference, wantStoreRefKind string) {
	t.Helper()

	if got == nil {
		t.Fatal("expected provider reference to be set")
	}
	if got.ApiVersion != want.APIVersion || got.Kind != want.Kind || got.Name != want.Name || got.Namespace != want.Namespace {
		t.Fatalf("unexpected provider ref: got=%#v want=%#v", got, want)
	}
	if got.StoreRefKind != wantStoreRefKind {
		t.Fatalf("unexpected store ref kind: got=%q want=%q", got.StoreRefKind, wantStoreRefKind)
	}
}

func newProviderTLSArtifacts(t *testing.T, host string) (serverCertPEM, serverKeyPEM, clientCertPEM, clientKeyPEM, caCertPEM []byte) {
	t.Helper()

	caKey, err := rsa.GenerateKey(rand.Reader, 2048)
	if err != nil {
		t.Fatalf("GenerateKey() error = %v", err)
	}

	caTemplate := &x509.Certificate{
		SerialNumber:          big.NewInt(1),
		Subject:               pkix.Name{CommonName: "provider-controller-test-ca"},
		NotBefore:             time.Now().Add(-time.Hour),
		NotAfter:              time.Now().Add(24 * time.Hour),
		KeyUsage:              x509.KeyUsageCertSign | x509.KeyUsageCRLSign,
		BasicConstraintsValid: true,
		IsCA:                  true,
	}

	caDER, err := x509.CreateCertificate(rand.Reader, caTemplate, caTemplate, &caKey.PublicKey, caKey)
	if err != nil {
		t.Fatalf("CreateCertificate() error = %v", err)
	}
	caCert, err := x509.ParseCertificate(caDER)
	if err != nil {
		t.Fatalf("ParseCertificate() error = %v", err)
	}

	serverCertPEM, serverKeyPEM = newProviderSignedTLSCert(t, caCert, caKey, 2, host, []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth})
	clientCertPEM, clientKeyPEM = newProviderSignedTLSCert(t, caCert, caKey, 3, host, []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth})
	caCertPEM = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: caDER})
	return serverCertPEM, serverKeyPEM, clientCertPEM, clientKeyPEM, caCertPEM
}

func newProviderSignedTLSCert(t *testing.T, caCert *x509.Certificate, caKey *rsa.PrivateKey, serial int64, host string, usages []x509.ExtKeyUsage) ([]byte, []byte) {
	t.Helper()

	key, err := rsa.GenerateKey(rand.Reader, 2048)
	if err != nil {
		t.Fatalf("GenerateKey() error = %v", err)
	}

	template := &x509.Certificate{
		SerialNumber: big.NewInt(serial),
		Subject:      pkix.Name{CommonName: host},
		NotBefore:    time.Now().Add(-time.Hour),
		NotAfter:     time.Now().Add(24 * time.Hour),
		KeyUsage:     x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
		ExtKeyUsage:  usages,
	}

	if ip := net.ParseIP(host); ip != nil {
		template.IPAddresses = []net.IP{ip}
	} else {
		template.DNSNames = []string{host}
	}

	der, err := x509.CreateCertificate(rand.Reader, template, caCert, &key.PublicKey, caKey)
	if err != nil {
		t.Fatalf("CreateCertificate() error = %v", err)
	}

	return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}),
		pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)})
}