helm-external-secrets/e2e/framework/addon/vault.go

/*
Copyright © 2025 ESO Maintainer Team

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    https://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package addon

import (
	"crypto/rand"
	"crypto/rsa"
	"crypto/x509"
	"crypto/x509/pkix"
	"encoding/json"
	"encoding/pem"
	"errors"
	"fmt"
	"math/big"
	"net"
	"net/http"
	"os"
	"path/filepath"
	"time"

	rbacv1 "k8s.io/api/rbac/v1"

	"k8s.io/apimachinery/pkg/types"
	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"

	"github.com/golang-jwt/jwt/v4"
	vault "github.com/hashicorp/vault/api"
	. "github.com/onsi/ginkgo/v2"
	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	"github.com/external-secrets/external-secrets-e2e/framework/util"
)

type Vault struct {
	chart         *HelmChart
	Namespace     string
	PodName       string
	VaultClient   *vault.Client
	VaultURL      string
	VaultMtlsURL  string
	portForwarder *PortForward

	RootToken          string
	VaultServerCA      []byte
	ServerCert         []byte
	ServerKey          []byte
	VaultClientCA      []byte
	ClientCert         []byte
	ClientKey          []byte
	JWTPubkey          []byte
	JWTPrivKey         []byte
	JWTToken           string
	JWTRole            string
	JWTPath            string
	JWTK8sPath         string
	KubernetesAuthPath string
	KubernetesAuthRole string

	AppRoleSecret string
	AppRoleID     string
	AppRolePath   string
}

const privatePemType = "RSA PRIVATE KEY"

func NewVault() *Vault {
	repo := "hashicorp-vault"
	return &Vault{
		chart: &HelmChart{
			Namespace:    "vault",
			ReleaseName:  "vault",
			Chart:        fmt.Sprintf("%s/vault", repo),
			ChartVersion: "0.30.1",
			Repo: ChartRepo{
				Name: repo,
				URL:  "https://helm.releases.hashicorp.com",
			},
			Args: []string{
				"--create-namespace",
			},
			Values: []string{filepath.Join(AssetDir(), "vault.values.yaml")},
		},
		Namespace: "vault",
	}
}

type OperatorInitResponse struct {
	UnsealKeysB64 []string `json:"unseal_keys_b64"`
	RootToken     string   `json:"root_token"`
}

func (l *Vault) Install() error {
	// From Kubernetes 1.32+ on the oidc endpoint is not available to unauthenticated clients.
	// We create this clusterrole to allow vault to access the oidc endpoint.
	// see: https://github.com/ansible-collections/kubernetes.core/issues/868
	crb := &rbacv1.ClusterRoleBinding{
		ObjectMeta: metav1.ObjectMeta{
			Name: "allow-anon-oidc",
		},
	}
	_, err := controllerutil.CreateOrUpdate(GinkgoT().Context(), l.chart.config.CRClient, crb, func() error {
		crb.Subjects = []rbacv1.Subject{
			{
				APIGroup: "rbac.authorization.k8s.io",
				Kind:     "User",
				Name:     "system:anonymous",
			},
		}
		crb.RoleRef = rbacv1.RoleRef{
			APIGroup: "rbac.authorization.k8s.io",
			Kind:     "ClusterRole",
			Name:     "system:service-account-issuer-discovery",
		}
		return nil
	})
	if err != nil {
		return err
	}
	if err = l.chart.Install(); err != nil {
		return err
	}

	if err = l.patchVaultService(); err != nil {
		return err
	}

	if err = l.initVault(); err != nil {
		return err
	}

	if err = l.configureVault(); err != nil {
		return err
	}

	return nil
}

func (l *Vault) patchVaultService() error {
	serviceName := l.chart.ReleaseName
	servicePatch := []byte(`[{"op": "add", "path": "/spec/ports/-", "value": { "name": "https-mtls", "port": 8210, "protocol": "TCP", "targetPort": 8210 }}]`)
	clientSet := l.chart.config.KubeClientSet
	_, err := clientSet.CoreV1().Services(l.Namespace).
		Patch(GinkgoT().Context(), serviceName, types.JSONPatchType, servicePatch, metav1.PatchOptions{})
	return err
}

func (l *Vault) initVault() error {
	// gen certificates and put them into the secret
	serverRootPem, serverPem, serverKeyPem, clientRootPem, clientPem, clientKeyPem, err := genVaultCertificates(l.Namespace, l.chart.ReleaseName)
	if err != nil {
		return fmt.Errorf("unable to gen vault certs: %w", err)
	}
	jwtPrivkey, jwtPubkey, jwtToken, err := genVaultJWTKeys()
	if err != nil {
		return fmt.Errorf("unable to generate vault jwt keys: %w", err)
	}

	// make certs available to the struct
	// so it can be used by the provider
	l.VaultServerCA = serverRootPem
	l.ServerCert = serverPem
	l.ServerKey = serverKeyPem
	l.VaultClientCA = clientRootPem
	l.ClientCert = clientPem
	l.ClientKey = clientKeyPem
	l.JWTPrivKey = jwtPrivkey
	l.JWTPubkey = jwtPubkey
	l.JWTToken = jwtToken
	l.JWTPath = "myjwt"                                // see configure-vault.sh
	l.JWTK8sPath = "myjwtk8s"                          // see configure-vault.sh
	l.JWTRole = "external-secrets-operator"            // see configure-vault.sh
	l.KubernetesAuthPath = "mykubernetes"              // see configure-vault.sh
	l.KubernetesAuthRole = "external-secrets-operator" // see configure-vault.sh

	// vault-config contains vault init config and policies
	files, err := os.ReadDir(fmt.Sprintf("%s/vault-config", AssetDir()))
	if err != nil {
		return err
	}
	sec := &v1.Secret{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "vault-tls-config",
			Namespace: l.Namespace,
		},
		Data: map[string][]byte{},
	}
	_, err = controllerutil.CreateOrUpdate(GinkgoT().Context(), l.chart.config.CRClient, sec, func() error {
		sec.Data = map[string][]byte{}
		for _, f := range files {
			name := f.Name()
			data := mustReadFile(fmt.Sprintf("%s/vault-config/%s", AssetDir(), name))
			sec.Data[name] = data
		}
		sec.Data["vault-server-ca.pem"] = serverRootPem
		sec.Data["server-cert.pem"] = serverPem
		sec.Data["server-cert-key.pem"] = serverKeyPem
		sec.Data["vault-client-ca.pem"] = clientRootPem
		sec.Data["es-client.pem"] = clientPem
		sec.Data["es-client-key.pem"] = clientKeyPem
		sec.Data["jwt-pubkey.pem"] = jwtPubkey

		return nil
	})
	if err != nil {
		return err
	}

	pl, err := util.WaitForPodsRunning(l.chart.config.KubeClientSet, 1, l.Namespace, metav1.ListOptions{
		LabelSelector: "app.kubernetes.io/name=vault",
	})
	if err != nil {
		return fmt.Errorf("error waiting for vault to be running: %w", err)
	}
	l.PodName = pl.Items[0].Name

	out, err := util.ExecCmd(
		l.chart.config.KubeClientSet,
		l.chart.config.KubeConfig,
		l.PodName, l.Namespace, "vault operator init --format=json")
	if err != nil {
		return fmt.Errorf("error initializing vault: %w", err)
	}

	var res OperatorInitResponse
	err = json.Unmarshal([]byte(out), &res)
	if err != nil {
		return err
	}
	l.RootToken = res.RootToken

	for _, k := range res.UnsealKeysB64 {
		_, err = util.ExecCmd(
			l.chart.config.KubeClientSet,
			l.chart.config.KubeConfig,
			l.PodName, l.Namespace, "vault operator unseal "+k)
		if err != nil {
			return fmt.Errorf("unable to unseal vault: %w", err)
		}
	}

	// vault becomes ready after it has been unsealed
	err = util.WaitForPodsReady(l.chart.config.KubeClientSet, 1, l.Namespace, metav1.ListOptions{
		LabelSelector: "app.kubernetes.io/name=vault",
	})
	if err != nil {
		return fmt.Errorf("error waiting for vault to be ready: %w", err)
	}

	// This e2e test provider uses a local port-forwarded to talk to the vault API instead
	// of using the kubernetes service. This allows us to run the e2e test suite locally.
	l.portForwarder, err = NewPortForward(l.chart.config.KubeClientSet, l.chart.config.KubeConfig, "vault", l.chart.Namespace, 8200)
	if err != nil {
		return err
	}
	if err := l.portForwarder.Start(); err != nil {
		return err
	}

	serverCA := l.VaultServerCA
	caCertPool := x509.NewCertPool()
	ok := caCertPool.AppendCertsFromPEM(serverCA)
	if !ok {
		panic("unable to append server ca cert")
	}
	cfg := vault.DefaultConfig()
	l.VaultURL = fmt.Sprintf("https://%s.%s.svc.cluster.local:8200", l.chart.ReleaseName, l.Namespace)
	l.VaultMtlsURL = fmt.Sprintf("https://%s.%s.svc.cluster.local:8210", l.chart.ReleaseName, l.Namespace)
	cfg.Address = fmt.Sprintf("https://localhost:%d", l.portForwarder.localPort)
	cfg.HttpClient.Transport.(*http.Transport).TLSClientConfig.RootCAs = caCertPool
	l.VaultClient, err = vault.NewClient(cfg)
	if err != nil {
		return fmt.Errorf("unable to create vault client: %w", err)
	}
	l.VaultClient.SetToken(l.RootToken)

	return nil
}

func (l *Vault) configureVault() error {
	cmd := `sh /etc/vault-config/configure-vault.sh %s`
	_, err := util.ExecCmd(
		l.chart.config.KubeClientSet,
		l.chart.config.KubeConfig,
		l.PodName, l.Namespace, fmt.Sprintf(cmd, l.RootToken))
	if err != nil {
		return fmt.Errorf("unable to configure vault: %w", err)
	}

	// configure appRole
	l.AppRolePath = "myapprole"
	req := l.VaultClient.NewRequest(http.MethodGet, fmt.Sprintf("/v1/auth/%s/role/eso-e2e-role/role-id", l.AppRolePath))
	res, err := l.VaultClient.RawRequest(req) //nolint:staticcheck
	if err != nil {
		return err
	}
	defer func() {
		_ = res.Body.Close()
	}()
	sec, err := vault.ParseSecret(res.Body)
	if err != nil {
		return err
	}

	l.AppRoleID = sec.Data["role_id"].(string)

	// parse role id
	req = l.VaultClient.NewRequest(http.MethodPost, fmt.Sprintf("/v1/auth/%s/role/eso-e2e-role/secret-id", l.AppRolePath))
	res, err = l.VaultClient.RawRequest(req) //nolint:staticcheck
	if err != nil {
		return err
	}
	defer func() {
		_ = res.Body.Close()
	}()
	sec, err = vault.ParseSecret(res.Body)
	if err != nil {
		return err
	}
	l.AppRoleSecret = sec.Data["secret_id"].(string)
	return nil
}

func (l *Vault) Logs() error {
	return l.chart.Logs()
}

func (l *Vault) Uninstall() error {
	if l.portForwarder != nil {
		l.portForwarder.Close()
		l.portForwarder = nil
	}
	if err := l.chart.Uninstall(); err != nil {
		return err
	}
	return l.chart.config.KubeClientSet.CoreV1().Namespaces().Delete(GinkgoT().Context(), l.chart.Namespace, metav1.DeleteOptions{})
}

func (l *Vault) Setup(cfg *Config) error {
	return l.chart.Setup(cfg)
}

// nolint:gocritic
func genVaultCertificates(namespace, serviceName string) ([]byte, []byte, []byte, []byte, []byte, []byte, error) {
	// gen server ca + certs
	serverRootCert, serverRootPem, serverRootKey, err := genCARoot()
	if err != nil {
		return nil, nil, nil, nil, nil, nil, fmt.Errorf("unable to generate ca cert: %w", err)
	}
	serverPem, serverKey, err := genPeerCert(serverRootCert, serverRootKey, "vault", []string{
		"localhost",
		serviceName,
		fmt.Sprintf("%s.%s.svc.cluster.local", serviceName, namespace)})
	if err != nil {
		return nil, nil, nil, nil, nil, nil, errors.New("unable to generate vault server cert")
	}
	serverKeyPem := pem.EncodeToMemory(&pem.Block{
		Type:  privatePemType,
		Bytes: x509.MarshalPKCS1PrivateKey(serverKey)},
	)
	// gen client ca + certs
	clientRootCert, clientRootPem, clientRootKey, err := genCARoot()
	if err != nil {
		return nil, nil, nil, nil, nil, nil, fmt.Errorf("unable to generate ca cert: %w", err)
	}
	clientPem, clientKey, err := genPeerCert(clientRootCert, clientRootKey, "vault-client", nil)
	if err != nil {
		return nil, nil, nil, nil, nil, nil, errors.New("unable to generate vault server cert")
	}
	clientKeyPem := pem.EncodeToMemory(&pem.Block{
		Type:  privatePemType,
		Bytes: x509.MarshalPKCS1PrivateKey(clientKey)},
	)
	return serverRootPem, serverPem, serverKeyPem, clientRootPem, clientPem, clientKeyPem, err
}

func genVaultJWTKeys() ([]byte, []byte, string, error) {
	key, err := rsa.GenerateKey(rand.Reader, 2048)
	if err != nil {
		return nil, nil, "", err
	}
	privPem := pem.EncodeToMemory(&pem.Block{
		Type:  privatePemType,
		Bytes: x509.MarshalPKCS1PrivateKey(key),
	})
	pk, err := x509.MarshalPKIXPublicKey(&key.PublicKey)
	if err != nil {
		return nil, nil, "", err
	}
	pubPem := pem.EncodeToMemory(&pem.Block{
		Type:  "RSA PUBLIC KEY",
		Bytes: pk,
	})

	token := jwt.NewWithClaims(jwt.SigningMethodPS256, jwt.MapClaims{
		"aud":  "vault.client",
		"sub":  "vault@example",
		"iss":  "example.iss",
		"user": "eso",
		"exp":  time.Now().Add(time.Hour).Unix(),
		"nbf":  time.Date(2015, 10, 10, 12, 0, 0, 0, time.UTC).Unix(),
	})

	// Sign and get the complete encoded token as a string using the secret
	tokenString, err := token.SignedString(key)
	if err != nil {
		return nil, nil, "", err
	}

	return privPem, pubPem, tokenString, nil
}

func genCARoot() (*x509.Certificate, []byte, *rsa.PrivateKey, error) {
	tpl := x509.Certificate{
		SerialNumber: big.NewInt(1),
		Subject: pkix.Name{
			Country:      []string{"/dev/null"},
			Organization: []string{"External Secrets ACME"},
			CommonName:   "External Secrets Vault CA",
		},
		NotBefore:             time.Now(),
		NotAfter:              time.Now().Add(time.Hour),
		KeyUsage:              x509.KeyUsageCertSign | x509.KeyUsageCRLSign,
		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
		BasicConstraintsValid: true,
		IsCA:                  true,
		MaxPathLen:            2,
	}
	pkey, err := rsa.GenerateKey(rand.Reader, 2048)
	if err != nil {
		return nil, nil, nil, err
	}
	rootCert, rootPEM, err := genCert(&tpl, &tpl, &pkey.PublicKey, pkey)
	return rootCert, rootPEM, pkey, err
}

func genCert(template, parent *x509.Certificate, publicKey *rsa.PublicKey, privateKey *rsa.PrivateKey) (*x509.Certificate, []byte, error) {
	certBytes, err := x509.CreateCertificate(rand.Reader, template, parent, publicKey, privateKey)
	if err != nil {
		return nil, nil, fmt.Errorf("failed to create certificate: %w", err)
	}
	cert, err := x509.ParseCertificate(certBytes)
	if err != nil {
		return nil, nil, fmt.Errorf("failed to parse certificate: %w", err)
	}
	b := pem.Block{Type: "CERTIFICATE", Bytes: certBytes}
	certPEM := pem.EncodeToMemory(&b)

	return cert, certPEM, err
}

func genPeerCert(signingCert *x509.Certificate, signingKey *rsa.PrivateKey, cn string, dnsNames []string) ([]byte, *rsa.PrivateKey, error) {
	pkey, err := rsa.GenerateKey(rand.Reader, 2048)
	if err != nil {
		return nil, nil, err
	}
	tpl := x509.Certificate{
		Subject: pkix.Name{
			Country:      []string{"/dev/null"},
			Organization: []string{"External Secrets ACME"},
			CommonName:   cn,
		},
		SerialNumber:   big.NewInt(1),
		NotBefore:      time.Now(),
		NotAfter:       time.Now().Add(time.Hour),
		KeyUsage:       x509.KeyUsageCRLSign,
		ExtKeyUsage:    []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
		IsCA:           false,
		MaxPathLenZero: true,
		IPAddresses:    []net.IP{net.ParseIP("127.0.0.1")},
		DNSNames:       dnsNames,
	}
	_, serverPEM, err := genCert(&tpl, signingCert, &pkey.PublicKey, signingKey)
	return serverPEM, pkey, err
}

func mustReadFile(path string) []byte {
	b, err := os.ReadFile(path)
	if err != nil {
		panic(err)
	}
	return b
}