feat: add vault e2e tests

e2e/Makefile

@@ -4,7 +4,7 @@ SHELL       := /bin/bash
 
 IMG_TAG     = test
 IMG         = local/external-secrets-e2e:$(IMG_TAG)
-K8S_VERSION = "1.20.7"
+KIND_IMG    = "kindest/node:v1.20.7@sha256:cbeaf907fc78ac97ce7b625e4bf0de16e3ea725daf6b04f930bd14c67c671ff9"
 BUILD_ARGS  ?=
 export FOCUS := $(FOCUS)
 
@@ -13,7 +13,7 @@ start-kind: ## Start kind cluster
 	  --name external-secrets \
 	  --config kind.yaml \
 	  --retain \
-	  --image "kindest/node:v$(K8S_VERSION)"
+	  --image "$(KIND_IMG)"
 
 test: e2e-image ## Run e2e tests against current kube context
 	$(MAKE) -C ../ docker.build \

e2e/e2e_test.go

@@ -38,9 +38,6 @@ var _ = SynchronizedBeforeSuite(func() []byte {
 	err := util.WaitForURL("http://localstack.default/health")
 	Expect(err).ToNot(HaveOccurred())
 
-	By("installing vault")
-	addon.InstallGlobalAddon(addon.NewVault(), cfg)
-
 	By("installing eso")
 	addon.InstallGlobalAddon(addon.NewESO(), cfg)
 	return nil

e2e/framework/addon/chart.go

@@ -13,9 +13,9 @@ limitations under the License.
 package addon
 
 import (
+	"bytes"
 	"context"
 	"fmt"
-	"os"
 	"os/exec"
 
 	corev1 "k8s.io/api/core/v1"
@@ -79,22 +79,28 @@ func (c *HelmChart) Install() error {
 		args = append(args, "--set", fmt.Sprintf("%s=%s", s.Key, s.Value))
 	}
 
+	var sout, serr bytes.Buffer
 	log.Logf("installing chart %s", c.ReleaseName)
 	cmd := exec.Command("helm", args...)
-	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stderr
-	return cmd.Run()
+	cmd.Stdout = &sout
+	cmd.Stderr = &serr
+	err = cmd.Run()
+	if err != nil {
+		return fmt.Errorf("unable to run cmd: %w: %s %s", err, sout.String(), serr.String())
+	}
+	return nil
 }
 
 // Uninstall removes the chart aswell as the repo.
 func (c *HelmChart) Uninstall() error {
+	var sout, serr bytes.Buffer
 	args := []string{"delete", "--namespace", c.Namespace, c.ReleaseName}
 	cmd := exec.Command("helm", args...)
-	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stderr
+	cmd.Stdout = &sout
+	cmd.Stderr = &serr
 	err := cmd.Run()
 	if err != nil {
-		return err
+		return fmt.Errorf("unable to delete helm release: %w: %s, %s", err, sout.String(), serr.String())
 	}
 	return c.removeRepo()
 }
@@ -103,23 +109,32 @@ func (c *HelmChart) addRepo() error {
 	if c.Repo.Name == "" || c.Repo.URL == "" {
 		return nil
 	}
-	log.Logf("adding repo %s", c.Repo.Name)
+	var sout, serr bytes.Buffer
 	args := []string{"repo", "add", c.Repo.Name, c.Repo.URL}
 	cmd := exec.Command("helm", args...)
-	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stderr
-	return cmd.Run()
+	cmd.Stdout = &sout
+	cmd.Stderr = &serr
+	err := cmd.Run()
+	if err != nil {
+		return fmt.Errorf("unable to add helm repo: %w: %s, %s", err, sout.String(), serr.String())
+	}
+	return nil
 }
 
 func (c *HelmChart) removeRepo() error {
 	if c.Repo.Name == "" || c.Repo.URL == "" {
 		return nil
 	}
+	var sout, serr bytes.Buffer
 	args := []string{"repo", "remove", c.Repo.Name}
 	cmd := exec.Command("helm", args...)
-	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stderr
-	return cmd.Run()
+	cmd.Stdout = &sout
+	cmd.Stderr = &serr
+	err := cmd.Run()
+	if err != nil {
+		return fmt.Errorf("unable to remove repo: %w: %s, %s", err, sout.String(), serr.String())
+	}
+	return nil
 }
 
 // Logs fetches the logs from all pods managed by this release

e2e/framework/addon/eso.go

@@ -21,7 +21,7 @@ func NewESO() *ESO {
 	return &ESO{
 		&HelmChart{
 			Namespace:   "default",
-			ReleaseName: "eso-aws-sm",
+			ReleaseName: "eso",
 			Chart:       "/k8s/deploy/charts/external-secrets",
 			Values:      []string{"/k8s/eso.values.yaml"},
 		},

e2e/framework/addon/vault.go

@@ -13,41 +13,414 @@ limitations under the License.
 */
 package addon
 
-import "github.com/external-secrets/external-secrets/e2e/framework/util"
+import (
+	"context"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/json"
+	"encoding/pem"
+	"fmt"
+	"math/big"
+	"net"
+	"net/http"
+	"os"
+	"time"
+
+	"github.com/golang-jwt/jwt"
+	vault "github.com/hashicorp/vault/api"
+
+	// nolint
+	. "github.com/onsi/ginkgo"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/external-secrets/external-secrets/e2e/framework/util"
+)
 
 type Vault struct {
-	Addon
+	chart       *HelmChart
+	Namespace   string
+	PodName     string
+	VaultClient *vault.Client
+	VaultURL    string
+
+	RootToken          string
+	VaultServerCA      []byte
+	ServerCert         []byte
+	ServerKey          []byte
+	VaultClientCA      []byte
+	ClientCert         []byte
+	ClientKey          []byte
+	JWTPubkey          []byte
+	JWTPrivKey         []byte
+	JWTToken           string
+	JWTRole            string
+	KubernetesAuthPath string
+	KubernetesAuthRole string
+
+	AppRoleSecret string
+	AppRoleID     string
+	AppRolePath   string
 }
 
-func NewVault() *Vault {
+func NewVault(namespace string) *Vault {
+	repo := "hashicorp-" + namespace
 	return &Vault{
-		&HelmChart{
-			Namespace:    "default",
-			ReleaseName:  "vault",
-			Chart:        "hashicorp/vault",
+		chart: &HelmChart{
+			Namespace:    namespace,
+			ReleaseName:  fmt.Sprintf("vault-%s", namespace), // avoid cluster role collision
+			Chart:        fmt.Sprintf("%s/vault", repo),
 			ChartVersion: "0.11.0",
 			Repo: ChartRepo{
-				Name: "hashicorp",
+				Name: repo,
 				URL:  "https://helm.releases.hashicorp.com",
 			},
-			Vars: []StringTuple{
-				{
-					Key:   "server.dev.enabled",
-					Value: "true",
-				},
-				{
-					Key:   "injector.enabled",
-					Value: "false",
-				},
-			},
+			Values: []string{"/k8s/vault.values.yaml"},
 		},
+		Namespace: namespace,
 	}
 }
 
+type OperatorInitResponse struct {
+	UnsealKeysB64 []string `json:"unseal_keys_b64"`
+	RootToken     string   `json:"root_token"`
+}
+
 func (l *Vault) Install() error {
-	err := l.Addon.Install()
+	By("Installing vault in " + l.Namespace)
+	err := l.chart.Install()
+	if err != nil {
+		return err
+	}
+
+	err = l.initVault()
+	if err != nil {
+		return err
+	}
+
+	err = l.configureVault()
 	if err != nil {
 		return err
 	}
-	return util.WaitForURL("http://vault.default:8200/ui/")
+
+	return nil
+}
+
+func (l *Vault) initVault() error {
+	sec := &v1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "vault-tls-config",
+			Namespace: l.Namespace,
+		},
+		Data: map[string][]byte{},
+	}
+
+	// vault-config contains vault init config and policies
+	files, err := os.ReadDir("/k8s/vault-config")
+	if err != nil {
+		return err
+	}
+	for _, f := range files {
+		name := f.Name()
+		data := mustReadFile(fmt.Sprintf("/k8s/vault-config/%s", name))
+		sec.Data[name] = data
+	}
+
+	// gen certificates and put them into the secret
+	serverRootPem, serverPem, serverKeyPem, clientRootPem, clientPem, clientKeyPem, err := genVaultCertificates(l.Namespace)
+	if err != nil {
+		return fmt.Errorf("unable to gen vault certs: %w", err)
+	}
+	jwtPrivkey, jwtPubkey, jwtToken, err := genVaultJWTKeys()
+	if err != nil {
+		return fmt.Errorf("unable to generate vault jwt keys: %w", err)
+	}
+
+	// pass certs to secret
+	sec.Data["vault-server-ca.pem"] = serverRootPem
+	sec.Data["server-cert.pem"] = serverPem
+	sec.Data["server-cert-key.pem"] = serverKeyPem
+	sec.Data["vault-client-ca.pem"] = clientRootPem
+	sec.Data["es-client.pem"] = clientPem
+	sec.Data["es-client-key.pem"] = clientKeyPem
+	sec.Data["jwt-pubkey.pem"] = jwtPubkey
+
+	// make certs available to the struct
+	// so it can be used by the provider
+	l.VaultServerCA = serverRootPem
+	l.ServerCert = serverPem
+	l.ServerKey = serverKeyPem
+	l.VaultClientCA = clientRootPem
+	l.ClientCert = clientPem
+	l.ClientKey = clientKeyPem
+	l.JWTPrivKey = jwtPrivkey
+	l.JWTPubkey = jwtPubkey
+	l.JWTToken = jwtToken
+	l.JWTRole = "external-secrets-operator"            // see configure-vault.sh
+	l.KubernetesAuthPath = "mykubernetes"              // see configure-vault.sh
+	l.KubernetesAuthRole = "external-secrets-operator" // see configure-vault.sh
+
+	By("Creating vault TLS secret")
+	err = l.chart.config.CRClient.Create(context.Background(), sec)
+	if err != nil {
+		return err
+	}
+
+	By("Waiting for vault pods to be running")
+	pl, err := util.WaitForPodsRunning(l.chart.config.KubeClientSet, 1, l.Namespace, metav1.ListOptions{
+		LabelSelector: "app.kubernetes.io/name=vault",
+	})
+	if err != nil {
+		return fmt.Errorf("error waiting for vault to be running: %w", err)
+	}
+	l.PodName = pl.Items[0].Name
+
+	By("Initializing vault")
+	out, err := util.ExecCmd(
+		l.chart.config.KubeClientSet,
+		l.chart.config.KubeConfig,
+		l.PodName, l.Namespace, "vault operator init --format=json")
+	if err != nil {
+		return fmt.Errorf("error initializing vault: %w", err)
+	}
+
+	By("Parsing init response")
+	var res OperatorInitResponse
+	err = json.Unmarshal([]byte(out), &res)
+	if err != nil {
+		return err
+	}
+	l.RootToken = res.RootToken
+
+	By("Unsealing vault")
+	for _, k := range res.UnsealKeysB64 {
+		_, err = util.ExecCmd(
+			l.chart.config.KubeClientSet,
+			l.chart.config.KubeConfig,
+			l.PodName, l.Namespace, "vault operator unseal "+k)
+		if err != nil {
+			return fmt.Errorf("unable to unseal vault: %w", err)
+		}
+	}
+
+	// vault becomes ready after it has been unsealed
+	err = util.WaitForPodsReady(l.chart.config.KubeClientSet, 1, l.Namespace, metav1.ListOptions{
+		LabelSelector: "app.kubernetes.io/name=vault",
+	})
+	if err != nil {
+		return fmt.Errorf("error waiting for vault to be ready: %w", err)
+	}
+	serverCA := l.VaultServerCA
+	caCertPool := x509.NewCertPool()
+	ok := caCertPool.AppendCertsFromPEM(serverCA)
+	if !ok {
+		panic("unable to append server ca cert")
+	}
+	cfg := vault.DefaultConfig()
+	l.VaultURL = fmt.Sprintf("https://vault-%s.%s.svc.cluster.local:8200", l.Namespace, l.Namespace)
+	cfg.Address = l.VaultURL
+	cfg.HttpClient.Transport.(*http.Transport).TLSClientConfig.RootCAs = caCertPool
+	l.VaultClient, err = vault.NewClient(cfg)
+	if err != nil {
+		return fmt.Errorf("unable to create vault client: %w", err)
+	}
+	l.VaultClient.SetToken(l.RootToken)
+
+	return nil
+}
+
+func (l *Vault) configureVault() error {
+	By("configuring vault")
+	cmd := `sh /etc/vault-config/configure-vault.sh %s`
+	_, err := util.ExecCmd(
+		l.chart.config.KubeClientSet,
+		l.chart.config.KubeConfig,
+		l.PodName, l.Namespace, fmt.Sprintf(cmd, l.RootToken))
+	if err != nil {
+		return fmt.Errorf("unable to configure vault: %w", err)
+	}
+
+	// configure appRole
+	l.AppRolePath = "myapprole"
+	req := l.VaultClient.NewRequest(http.MethodGet, fmt.Sprintf("/v1/auth/%s/role/eso-e2e-role/role-id", l.AppRolePath))
+	res, err := l.VaultClient.RawRequest(req)
+	if err != nil {
+		return err
+	}
+	defer res.Body.Close()
+	sec, err := vault.ParseSecret(res.Body)
+	if err != nil {
+		return err
+	}
+
+	l.AppRoleID = sec.Data["role_id"].(string)
+
+	// parse role id
+	req = l.VaultClient.NewRequest(http.MethodPost, fmt.Sprintf("/v1/auth/%s/role/eso-e2e-role/secret-id", l.AppRolePath))
+	res, err = l.VaultClient.RawRequest(req)
+	if err != nil {
+		return err
+	}
+	defer res.Body.Close()
+	sec, err = vault.ParseSecret(res.Body)
+	if err != nil {
+		return err
+	}
+	l.AppRoleSecret = sec.Data["secret_id"].(string)
+	return nil
+}
+
+func (l *Vault) Logs() error {
+	return l.chart.Logs()
+}
+
+func (l *Vault) Uninstall() error {
+	return l.chart.Uninstall()
+}
+
+func (l *Vault) Setup(cfg *Config) error {
+	return l.chart.Setup(cfg)
+}
+
+func genVaultCertificates(namespace string) ([]byte, []byte, []byte, []byte, []byte, []byte, error) {
+	// gen server ca + certs
+	serverRootCert, serverRootPem, serverRootKey, err := genCARoot()
+	if err != nil {
+		return nil, nil, nil, nil, nil, nil, fmt.Errorf("unable to generate ca cert: %w", err)
+	}
+	serverPem, serverKey, err := genPeerCert(serverRootCert, serverRootKey, "vault", []string{
+		"localhost",
+		"vault-" + namespace,
+		fmt.Sprintf("vault-%s.%s.svc.cluster.local", namespace, namespace)})
+	if err != nil {
+		return nil, nil, nil, nil, nil, nil, fmt.Errorf("unable to generate vault server cert")
+	}
+	serverKeyPem := pem.EncodeToMemory(&pem.Block{
+		Type:  "RSA PRIVATE KEY",
+		Bytes: x509.MarshalPKCS1PrivateKey(serverKey)},
+	)
+	// gen client ca + certs
+	clientRootCert, clientRootPem, clientRootKey, err := genCARoot()
+	if err != nil {
+		return nil, nil, nil, nil, nil, nil, fmt.Errorf("unable to generate ca cert: %w", err)
+	}
+	clientPem, clientKey, err := genPeerCert(clientRootCert, clientRootKey, "vault-client", nil)
+	if err != nil {
+		return nil, nil, nil, nil, nil, nil, fmt.Errorf("unable to generate vault server cert")
+	}
+	clientKeyPem := pem.EncodeToMemory(&pem.Block{
+		Type:  "RSA PRIVATE KEY",
+		Bytes: x509.MarshalPKCS1PrivateKey(clientKey)},
+	)
+	return serverRootPem, serverPem, serverKeyPem, clientRootPem, clientPem, clientKeyPem, err
+}
+
+func genVaultJWTKeys() ([]byte, []byte, string, error) {
+	key, err := rsa.GenerateKey(rand.Reader, 2048)
+	if err != nil {
+		return nil, nil, "", err
+	}
+	privPem := pem.EncodeToMemory(&pem.Block{
+		Type:  "RSA PRIVATE KEY",
+		Bytes: x509.MarshalPKCS1PrivateKey(key),
+	})
+	pk, err := x509.MarshalPKIXPublicKey(&key.PublicKey)
+	if err != nil {
+		return nil, nil, "", err
+	}
+	pubPem := pem.EncodeToMemory(&pem.Block{
+		Type:  "RSA PUBLIC KEY",
+		Bytes: pk,
+	})
+
+	token := jwt.NewWithClaims(jwt.SigningMethodPS256, jwt.MapClaims{
+		"aud":  "vault.client",
+		"sub":  "vault@example",
+		"iss":  "example.iss",
+		"user": "eso",
+		"exp":  time.Now().Add(time.Hour).Unix(),
+		"nbf":  time.Date(2015, 10, 10, 12, 0, 0, 0, time.UTC).Unix(),
+	})
+
+	// Sign and get the complete encoded token as a string using the secret
+	tokenString, err := token.SignedString(key)
+	if err != nil {
+		return nil, nil, "", err
+	}
+
+	return privPem, pubPem, tokenString, nil
+}
+
+func genCARoot() (*x509.Certificate, []byte, *rsa.PrivateKey, error) {
+	tpl := x509.Certificate{
+		SerialNumber: big.NewInt(1),
+		Subject: pkix.Name{
+			Country:      []string{"/dev/null"},
+			Organization: []string{"External Secrets ACME"},
+			CommonName:   "External Secrets Vault CA",
+		},
+		NotBefore:             time.Now(),
+		NotAfter:              time.Now().Add(time.Hour),
+		KeyUsage:              x509.KeyUsageCertSign | x509.KeyUsageCRLSign,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
+		BasicConstraintsValid: true,
+		IsCA:                  true,
+		MaxPathLen:            2,
+	}
+	pkey, err := rsa.GenerateKey(rand.Reader, 2048)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+	rootCert, rootPEM, err := genCert(&tpl, &tpl, &pkey.PublicKey, pkey)
+	return rootCert, rootPEM, pkey, err
+}
+
+func genCert(template, parent *x509.Certificate, publicKey *rsa.PublicKey, privateKey *rsa.PrivateKey) (*x509.Certificate, []byte, error) {
+	certBytes, err := x509.CreateCertificate(rand.Reader, template, parent, publicKey, privateKey)
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to create certificate: %w", err)
+	}
+	cert, err := x509.ParseCertificate(certBytes)
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to parse certificate: %w", err)
+	}
+	b := pem.Block{Type: "CERTIFICATE", Bytes: certBytes}
+	certPEM := pem.EncodeToMemory(&b)
+
+	return cert, certPEM, err
+}
+
+func genPeerCert(signingCert *x509.Certificate, signingKey *rsa.PrivateKey, cn string, dnsNames []string) ([]byte, *rsa.PrivateKey, error) {
+	pkey, err := rsa.GenerateKey(rand.Reader, 2048)
+	if err != nil {
+		return nil, nil, err
+	}
+	tpl := x509.Certificate{
+		Subject: pkix.Name{
+			Country:      []string{"/dev/null"},
+			Organization: []string{"External Secrets ACME"},
+			CommonName:   cn,
+		},
+		SerialNumber:   big.NewInt(1),
+		NotBefore:      time.Now(),
+		NotAfter:       time.Now().Add(time.Hour),
+		KeyUsage:       x509.KeyUsageCRLSign,
+		ExtKeyUsage:    []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
+		IsCA:           false,
+		MaxPathLenZero: true,
+		IPAddresses:    []net.IP{net.ParseIP("127.0.0.1")},
+		DNSNames:       dnsNames,
+	}
+	_, serverPEM, err := genCert(&tpl, signingCert, &pkey.PublicKey, signingKey)
+	return serverPEM, pkey, err
+}
+
+func mustReadFile(path string) []byte {
+	b, err := os.ReadFile(path)
+	if err != nil {
+		panic(err)
+	}
+	return b
 }

e2e/framework/framework.go

@@ -18,6 +18,7 @@ import (
 
 	// nolint
 	. "github.com/onsi/ginkgo"
+
 	// nolint
 	. "github.com/onsi/gomega"
 	api "k8s.io/api/core/v1"
@@ -83,11 +84,32 @@ func (f *Framework) BeforeEach() {
 
 // AfterEach deletes the namespace and cleans up the registered addons.
 func (f *Framework) AfterEach() {
+	for _, a := range f.Addons {
+		err := a.Uninstall()
+		Expect(err).ToNot(HaveOccurred())
+	}
+	// reset addons to default once the run is done
+	f.Addons = []addon.Addon{}
 	By("deleting test namespace")
 	err := util.DeleteKubeNamespace(f.Namespace.Name, f.KubeClientSet)
 	Expect(err).NotTo(HaveOccurred())
 }
 
+func (f *Framework) Install(a addon.Addon) {
+	f.Addons = append(f.Addons, a)
+
+	By("installing addon")
+	err := a.Setup(&addon.Config{
+		KubeConfig:    f.KubeConfig,
+		KubeClientSet: f.KubeClientSet,
+		CRClient:      f.CRClient,
+	})
+	Expect(err).NotTo(HaveOccurred())
+
+	err = a.Install()
+	Expect(err).NotTo(HaveOccurred())
+}
+
 // NewConfig loads and returns the kubernetes credentials from the environment.
 // KUBECONFIG env var takes precedence and falls back to in-cluster config.
 func NewConfig() (*rest.Config, *kubernetes.Clientset, crclient.Client) {

e2e/framework/testcase.go

@@ -42,14 +42,16 @@ type SecretStoreProvider interface {
 }
 
 // TableFunc returns the main func that runs a TestCase in a table driven test.
-func TableFunc(f *Framework, prov SecretStoreProvider) func(func(*TestCase)) {
-	return func(customize func(*TestCase)) {
+func TableFunc(f *Framework, prov SecretStoreProvider) func(...func(*TestCase)) {
+	return func(tweaks ...func(*TestCase)) {
 		var err error
 
 		// make default test case
 		// and apply customization to it
 		tc := makeDefaultTestCase(f)
-		customize(tc)
+		for _, tweak := range tweaks {
+			tweak(tc)
+		}
 
 		// create secrets & defer delete
 		for k, v := range tc.Secrets {

e2e/framework/util/util.go

@@ -14,6 +14,7 @@ limitations under the License.
 package util
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"net/http"
@@ -24,6 +25,9 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/kubernetes/scheme"
+	restclient "k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/remotecommand"
 )
 
 const (
@@ -66,6 +70,123 @@ func namespaceNotExist(c kubernetes.Interface, namespace string) wait.ConditionF
 	}
 }
 
+// ExecCmd exec command on specific pod and wait the command's output.
+func ExecCmd(client kubernetes.Interface, config *restclient.Config, podName, namespace string,
+	command string) (string, error) {
+	cmd := []string{
+		"sh",
+		"-c",
+		command,
+	}
+
+	req := client.CoreV1().RESTClient().Post().Resource("pods").Name(podName).
+		Namespace(namespace).SubResource("exec")
+	option := &v1.PodExecOptions{
+		Command: cmd,
+		Stdin:   false,
+		Stdout:  true,
+		Stderr:  true,
+		TTY:     false,
+	}
+	req.VersionedParams(
+		option,
+		scheme.ParameterCodec,
+	)
+	exec, err := remotecommand.NewSPDYExecutor(config, "POST", req.URL())
+	if err != nil {
+		return "", err
+	}
+	var stdout, stderr bytes.Buffer
+	err = exec.Stream(remotecommand.StreamOptions{
+		Stdout: &stdout,
+		Stderr: &stderr,
+		Tty:    false,
+	})
+	if err != nil {
+		return "", fmt.Errorf("unable to exec stream: %w: \n%s\n%s", err, stdout.String(), stderr.String())
+	}
+
+	return stdout.String() + stderr.String(), nil
+}
+
+// WaitForPodsRunning waits for a given amount of time until a group of Pods is running in the given namespace.
+func WaitForPodsRunning(kubeClientSet kubernetes.Interface, expectedReplicas int, namespace string, opts metav1.ListOptions) (*v1.PodList, error) {
+	var pods *v1.PodList
+	err := wait.PollImmediate(1*time.Second, time.Minute*5, func() (bool, error) {
+		pl, err := kubeClientSet.CoreV1().Pods(namespace).List(context.TODO(), opts)
+		if err != nil {
+			return false, nil
+		}
+
+		r := 0
+		for i := range pl.Items {
+			if pl.Items[i].Status.Phase == v1.PodRunning {
+				r++
+			}
+		}
+
+		if r == expectedReplicas {
+			pods = pl
+			return true, nil
+		}
+
+		return false, nil
+	})
+	return pods, err
+}
+
+// WaitForPodsReady waits for a given amount of time until a group of Pods is running in the given namespace.
+func WaitForPodsReady(kubeClientSet kubernetes.Interface, expectedReplicas int, namespace string, opts metav1.ListOptions) error {
+	return wait.PollImmediate(1*time.Second, time.Minute*5, func() (bool, error) {
+		pl, err := kubeClientSet.CoreV1().Pods(namespace).List(context.TODO(), opts)
+		if err != nil {
+			return false, nil
+		}
+
+		r := 0
+		for i := range pl.Items {
+			if isRunning, _ := podRunningReady(&pl.Items[i]); isRunning {
+				r++
+			}
+		}
+
+		if r == expectedReplicas {
+			return true, nil
+		}
+
+		return false, nil
+	})
+}
+
+// podRunningReady checks whether pod p's phase is running and it has a ready
+// condition of status true.
+func podRunningReady(p *v1.Pod) (bool, error) {
+	// Check the phase is running.
+	if p.Status.Phase != v1.PodRunning {
+		return false, fmt.Errorf("want pod '%s' on '%s' to be '%v' but was '%v'",
+			p.ObjectMeta.Name, p.Spec.NodeName, v1.PodRunning, p.Status.Phase)
+	}
+	// Check the ready condition is true.
+
+	if !isPodReady(p) {
+		return false, fmt.Errorf("pod '%s' on '%s' didn't have condition {%v %v}; conditions: %v",
+			p.ObjectMeta.Name, p.Spec.NodeName, v1.PodReady, v1.ConditionTrue, p.Status.Conditions)
+	}
+	return true, nil
+}
+
+func isPodReady(p *v1.Pod) bool {
+	for _, condition := range p.Status.Conditions {
+		if condition.Type != v1.ContainersReady {
+			continue
+		}
+
+		return condition.Status == v1.ConditionTrue
+	}
+
+	return false
+}
+
 // WaitForURL tests the provided url. Once a http 200 is returned the func returns with no error.
 // Timeout is 5min.
 func WaitForURL(url string) error {

e2e/k8s/vault-config/configure-vault.sh

@@ -0,0 +1,85 @@
+#!/bin/sh
+set -euxo pipefail;
+
+export VAULT_TOKEN=${1}
+
+# ------------------
+#   SECRET BACKENDS
+# ------------------
+vault secrets enable -path=secret -version=2 kv
+vault secrets enable -path=secret_v1 -version=1 kv
+
+# ------------------
+#   CERT AUTH
+#   https://www.vaultproject.io/docs/auth/cert
+# ------------------
+vault auth enable cert
+vault policy write \
+    external-secrets-operator \
+    /etc/vault-config/vault-policy-es.hcl
+
+vault write auth/cert/certs/external-secrets-operator \
+    display_name=external-secrets-operator \
+    policies=external-secrets-operator \
+    certificate=@/etc/vault-config/es-client.pem \
+    ttl=3600
+
+# test certificate login
+unset VAULT_TOKEN
+vault login \
+    -client-cert=/etc/vault-config/es-client.pem \
+    -client-key=/etc/vault-config/es-client-key.pem \
+    -method=cert \
+    name=external-secrets-operator
+
+vault kv put secret/foo/bar baz=bang
+vault kv get secret/foo/bar
+
+# ------------------
+#   App Role AUTH
+#   https://www.vaultproject.io/docs/auth/approle
+# ------------------
+export VAULT_TOKEN=${1}
+vault auth enable -path=myapprole approle
+
+vault write auth/myapprole/role/eso-e2e-role \
+    secret_id_ttl=10m \
+    token_num_uses=10 \
+    token_policies=external-secrets-operator \
+    token_ttl=1h \
+    token_max_ttl=4h \
+    secret_id_num_uses=40
+
+# ------------------
+#   App Role AUTH
+#   https://www.vaultproject.io/docs/auth/jwt
+# ------------------
+vault auth enable jwt
+
+vault write auth/jwt/config \
+   jwt_validation_pubkeys=@/etc/vault-config/jwt-pubkey.pem \
+   bound_issuer="example.iss" \
+   default_role="external-secrets-operator"
+
+vault write auth/jwt/role/external-secrets-operator \
+    role_type="jwt" \
+    bound_subject="vault@example" \
+    bound_audiences="vault.client" \
+    user_claim="user" \
+    policies=external-secrets-operator \
+    ttl=1h
+
+# ------------------
+#   Kubernetes AUTH
+#   https://www.vaultproject.io/docs/auth/kubernetes
+# ------------------
+vault auth enable -path=mykubernetes kubernetes
+vault write auth/mykubernetes/config \
+    kubernetes_host=https://kubernetes.default.svc.cluster.local \
+    kubernetes_ca_cert=@/var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+
+vault write auth/mykubernetes/role/external-secrets-operator \
+    bound_service_account_names=* \
+    bound_service_account_namespaces=* \
+    policies=external-secrets-operator \
+    ttl=1h

e2e/k8s/vault-config/vault-policy-es.hcl

@@ -0,0 +1,7 @@
+path "secret/+/*" {
+  capabilities = ["create", "read", "update", "delete", "list"]
+}
+
+path "secret_v1/+/*" {
+  capabilities = ["create", "read", "update", "delete", "list"]
+}

e2e/k8s/vault.values.yaml

@@ -0,0 +1,27 @@
+injector:
+  enabled: false
+server:
+  extraEnvironmentVars:
+    VAULT_CACERT: /etc/vault-config/vault-server-ca.pem
+    VAULT_ADDR: https://127.0.0.1:8200
+  volumeMounts:
+    - name: tls-config
+      mountPath: /etc/vault-config
+      readOnly: true
+  volumes:
+    - name: tls-config
+      secret:
+        secretName: vault-tls-config
+  standalone:
+    config: |
+      ui = true
+      listener "tcp" {
+        address = "[::]:8200"
+        cluster_address = "[::]:8201"
+        tls_cert_file = "/etc/vault-config/server-cert.pem"
+        tls_key_file = "/etc/vault-config/server-cert-key.pem"
+        tls_client_ca_file = "/etc/vault-config/vault-client-ca.pem"
+      }
+      storage "file" {
+        path = "/vault/data"
+      }

e2e/kind.yaml

@@ -9,15 +9,6 @@ kubeadmConfigPatches:
       service-account-key-file: "/etc/kubernetes/pki/sa.pub"
       service-account-signing-key-file: "/etc/kubernetes/pki/sa.key"
       service-account-issuer: "https://s3-XXXXXXXXXX.amazonaws.com/XXXXXXXXXXXXXXXXXXXXX"
-- |
-  apiVersion: kubelet.config.k8s.io/v1beta1
-  kind: KubeletConfiguration
-  metadata:
-    name: config
-  # this is only relevant for btrfs uses
-  # https://github.com/kubernetes/kubernetes/issues/80633#issuecomment-550994513
-  featureGates:
-    LocalStorageCapacityIsolation: false
 nodes:
 - role: control-plane
 - role: worker

e2e/suite/vault/provider.go

@@ -31,81 +31,239 @@ import (
 	esv1alpha1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1alpha1"
 	esmeta "github.com/external-secrets/external-secrets/apis/meta/v1"
 	"github.com/external-secrets/external-secrets/e2e/framework"
+	"github.com/external-secrets/external-secrets/e2e/framework/addon"
 )
 
 type vaultProvider struct {
 	url       string
-	token     string
 	client    *vault.Client
 	framework *framework.Framework
 }
 
-func newVaultProvider(f *framework.Framework, url, token string) *vaultProvider {
-	vc, err := vault.NewClient(&vault.Config{
-		Address: url,
-	})
-	Expect(err).ToNot(HaveOccurred())
-	vc.SetToken(token)
+const (
+	certAuthProviderName    = "cert-auth-provider"
+	appRoleAuthProviderName = "app-role-provider"
+	kvv1ProviderName        = "kv-v1-provider"
+	jwtProviderName         = "jwt-provider"
+	kubernetesProviderName  = "kubernetes-provider"
+)
 
+func newVaultProvider(f *framework.Framework) *vaultProvider {
 	prov := &vaultProvider{
 		framework: f,
-		url:       url,
-		token:     token,
-		client:    vc,
 	}
 	BeforeEach(prov.BeforeEach)
 	return prov
 }
 
+// CreateSecret creates a secret in both kv v1 and v2 provider.
 func (s *vaultProvider) CreateSecret(key, val string) {
 	req := s.client.NewRequest(http.MethodPost, fmt.Sprintf("/v1/secret/data/%s", key))
 	req.BodyBytes = []byte(fmt.Sprintf(`{"data": %s}`, val))
 	_, err := s.client.RawRequestWithContext(context.Background(), req)
 	Expect(err).ToNot(HaveOccurred())
+
+	req = s.client.NewRequest(http.MethodPost, fmt.Sprintf("/v1/secret_v1/%s", key))
+	req.BodyBytes = []byte(val)
+	_, err = s.client.RawRequestWithContext(context.Background(), req)
+	Expect(err).ToNot(HaveOccurred())
 }
 
 func (s *vaultProvider) DeleteSecret(key string) {
 	req := s.client.NewRequest(http.MethodDelete, fmt.Sprintf("/v1/secret/data/%s", key))
 	_, err := s.client.RawRequestWithContext(context.Background(), req)
 	Expect(err).ToNot(HaveOccurred())
+
+	req = s.client.NewRequest(http.MethodDelete, fmt.Sprintf("/v1/secret_v1/%s", key))
+	_, err = s.client.RawRequestWithContext(context.Background(), req)
+	Expect(err).ToNot(HaveOccurred())
 }
 
 func (s *vaultProvider) BeforeEach() {
+	ns := s.framework.Namespace.Name
+	v := addon.NewVault(ns)
+	s.framework.Install(v)
+	s.client = v.VaultClient
+	s.url = v.VaultURL
+
+	s.CreateCertStore(v, ns)
+	s.CreateTokenStore(v, ns)
+	s.CreateAppRoleStore(v, ns)
+	s.CreateV1Store(v, ns)
+	s.CreateJWTStore(v, ns)
+	s.CreateKubernetesAuthStore(v, ns)
+}
+
+func makeStore(name, ns string, v *addon.Vault) *esv1alpha1.SecretStore {
+	return &esv1alpha1.SecretStore{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: ns,
+		},
+		Spec: esv1alpha1.SecretStoreSpec{
+			Provider: &esv1alpha1.SecretStoreProvider{
+				Vault: &esv1alpha1.VaultProvider{
+					Version:  esv1alpha1.VaultKVStoreV2,
+					Path:     "secret",
+					Server:   v.VaultURL,
+					CABundle: v.VaultServerCA,
+				},
+			},
+		},
+	}
+}
+
+func (s *vaultProvider) CreateCertStore(v *addon.Vault, ns string) {
 	By("creating a vault secret")
+	clientCert := v.ClientCert
+	clientKey := v.ClientKey
 	vaultCreds := &v1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      "provider-secret",
-			Namespace: s.framework.Namespace.Name,
+			Name:      certAuthProviderName,
+			Namespace: ns,
 		},
-		StringData: map[string]string{
-			"token": s.token, // vault dev-mode default token
+		Data: map[string][]byte{
+			"token":       []byte(v.RootToken),
+			"client_cert": clientCert,
+			"client_key":  clientKey,
 		},
 	}
 	err := s.framework.CRClient.Create(context.Background(), vaultCreds)
 	Expect(err).ToNot(HaveOccurred())
 
 	By("creating an secret store for vault")
-	secretStore := &esv1alpha1.SecretStore{
+	secretStore := makeStore(certAuthProviderName, ns, v)
+	secretStore.Spec.Provider.Vault.Auth = esv1alpha1.VaultAuth{
+		Cert: &esv1alpha1.VaultCertAuth{
+			ClientCert: esmeta.SecretKeySelector{
+				Name: certAuthProviderName,
+				Key:  "client_cert",
+			},
+			SecretRef: esmeta.SecretKeySelector{
+				Name: certAuthProviderName,
+				Key:  "client_key",
+			},
+		},
+	}
+	err = s.framework.CRClient.Create(context.Background(), secretStore)
+	Expect(err).ToNot(HaveOccurred())
+}
+
+func (s vaultProvider) CreateTokenStore(v *addon.Vault, ns string) {
+	vaultCreds := &v1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      s.framework.Namespace.Name,
-			Namespace: s.framework.Namespace.Name,
+			Name:      "token-provider",
+			Namespace: ns,
 		},
-		Spec: esv1alpha1.SecretStoreSpec{
-			Provider: &esv1alpha1.SecretStoreProvider{
-				Vault: &esv1alpha1.VaultProvider{
-					Version: esv1alpha1.VaultKVStoreV2,
-					Path:    "secret",
-					Server:  s.url,
-					Auth: esv1alpha1.VaultAuth{
-						TokenSecretRef: &esmeta.SecretKeySelector{
-							Name: "provider-secret",
-							Key:  "token",
-						},
-					},
-				},
+		Data: map[string][]byte{
+			"token": []byte(v.RootToken),
+		},
+	}
+	err := s.framework.CRClient.Create(context.Background(), vaultCreds)
+	Expect(err).ToNot(HaveOccurred())
+	secretStore := makeStore(s.framework.Namespace.Name, ns, v)
+	secretStore.Spec.Provider.Vault.Auth = esv1alpha1.VaultAuth{
+		TokenSecretRef: &esmeta.SecretKeySelector{
+			Name: "token-provider",
+			Key:  "token",
+		},
+	}
+	err = s.framework.CRClient.Create(context.Background(), secretStore)
+	Expect(err).ToNot(HaveOccurred())
+}
+
+func (s vaultProvider) CreateAppRoleStore(v *addon.Vault, ns string) {
+	By("creating a vault secret")
+	vaultCreds := &v1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      appRoleAuthProviderName,
+			Namespace: ns,
+		},
+		Data: map[string][]byte{
+			"approle_secret": []byte(v.AppRoleSecret),
+		},
+	}
+	err := s.framework.CRClient.Create(context.Background(), vaultCreds)
+	Expect(err).ToNot(HaveOccurred())
+
+	By("creating an secret store for vault")
+	secretStore := makeStore(appRoleAuthProviderName, ns, v)
+	secretStore.Spec.Provider.Vault.Auth = esv1alpha1.VaultAuth{
+		AppRole: &esv1alpha1.VaultAppRole{
+			Path:   v.AppRolePath,
+			RoleID: v.AppRoleID,
+			SecretRef: esmeta.SecretKeySelector{
+				Name: appRoleAuthProviderName,
+				Key:  "approle_secret",
 			},
 		},
 	}
 	err = s.framework.CRClient.Create(context.Background(), secretStore)
 	Expect(err).ToNot(HaveOccurred())
 }
+
+func (s vaultProvider) CreateV1Store(v *addon.Vault, ns string) {
+	vaultCreds := &v1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "v1-provider",
+			Namespace: ns,
+		},
+		Data: map[string][]byte{
+			"token": []byte(v.RootToken),
+		},
+	}
+	err := s.framework.CRClient.Create(context.Background(), vaultCreds)
+	Expect(err).ToNot(HaveOccurred())
+	secretStore := makeStore(kvv1ProviderName, ns, v)
+	secretStore.Spec.Provider.Vault.Version = esv1alpha1.VaultKVStoreV1
+	secretStore.Spec.Provider.Vault.Path = "secret_v1"
+	secretStore.Spec.Provider.Vault.Auth = esv1alpha1.VaultAuth{
+		TokenSecretRef: &esmeta.SecretKeySelector{
+			Name: "v1-provider",
+			Key:  "token",
+		},
+	}
+	err = s.framework.CRClient.Create(context.Background(), secretStore)
+	Expect(err).ToNot(HaveOccurred())
+}
+
+func (s vaultProvider) CreateJWTStore(v *addon.Vault, ns string) {
+	vaultCreds := &v1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "jwt-provider",
+			Namespace: ns,
+		},
+		Data: map[string][]byte{
+			"jwt": []byte(v.JWTToken),
+		},
+	}
+	err := s.framework.CRClient.Create(context.Background(), vaultCreds)
+	Expect(err).ToNot(HaveOccurred())
+	secretStore := makeStore(jwtProviderName, ns, v)
+	secretStore.Spec.Provider.Vault.Auth = esv1alpha1.VaultAuth{
+		Jwt: &esv1alpha1.VaultJwtAuth{
+			Role: v.JWTRole,
+			SecretRef: esmeta.SecretKeySelector{
+				Name: "jwt-provider",
+				Key:  "jwt",
+			},
+		},
+	}
+	err = s.framework.CRClient.Create(context.Background(), secretStore)
+	Expect(err).ToNot(HaveOccurred())
+}
+
+func (s vaultProvider) CreateKubernetesAuthStore(v *addon.Vault, ns string) {
+	secretStore := makeStore(kubernetesProviderName, ns, v)
+	secretStore.Spec.Provider.Vault.Auth = esv1alpha1.VaultAuth{
+		Kubernetes: &esv1alpha1.VaultKubernetesAuth{
+			Path: v.KubernetesAuthPath,
+			Role: v.KubernetesAuthRole,
+			ServiceAccountRef: &esmeta.ServiceAccountSelector{
+				Name: "default",
+			},
+		},
+	}
+	err := s.framework.CRClient.Create(context.Background(), secretStore)
+	Expect(err).ToNot(HaveOccurred())
+}

e2e/suite/vault/vault.go

@@ -28,10 +28,74 @@ var _ = Describe("[vault] ", func() {
 
 	DescribeTable("sync secrets",
 		framework.TableFunc(f,
-			newVaultProvider(f, "http://vault.default:8200", "root")),
-		Entry(common.JSONDataFromSync(f)),
-		Entry(common.JSONDataWithProperty(f)),
-		Entry(common.JSONDataWithTemplate(f)),
-		Entry(common.DataPropertyDockerconfigJSON(f)),
+			newVaultProvider(f)),
+		// uses token auth
+		compose("with token auth", f, common.JSONDataFromSync, useTokenAuth),
+		compose("with token auth", f, common.JSONDataWithProperty, useTokenAuth),
+		compose("with token auth", f, common.JSONDataWithTemplate, useTokenAuth),
+		compose("with token auth", f, common.DataPropertyDockerconfigJSON, useTokenAuth),
+		// use cert auth
+		compose("with cert auth", f, common.JSONDataFromSync, useCertAuth),
+		compose("with cert auth", f, common.JSONDataWithProperty, useCertAuth),
+		compose("with cert auth", f, common.JSONDataWithTemplate, useCertAuth),
+		compose("with cert auth", f, common.DataPropertyDockerconfigJSON, useCertAuth),
+		// use approle auth
+		compose("with appRole auth", f, common.JSONDataFromSync, useApproleAuth),
+		compose("with appRole auth", f, common.JSONDataWithProperty, useApproleAuth),
+		compose("with appRole auth", f, common.JSONDataWithTemplate, useApproleAuth),
+		compose("with appRole auth", f, common.DataPropertyDockerconfigJSON, useApproleAuth),
+		// use v1 provider
+		compose("with v1 kv provider", f, common.JSONDataFromSync, useV1Provider),
+		compose("with v1 kv provider", f, common.JSONDataWithProperty, useV1Provider),
+		compose("with v1 kv provider", f, common.JSONDataWithTemplate, useV1Provider),
+		compose("with v1 kv provider", f, common.DataPropertyDockerconfigJSON, useV1Provider),
+		// use jwt provider
+		compose("with jwt provider", f, common.JSONDataFromSync, useJWTProvider),
+		compose("with jwt provider", f, common.JSONDataWithProperty, useJWTProvider),
+		compose("with jwt provider", f, common.JSONDataWithTemplate, useJWTProvider),
+		compose("with jwt provider", f, common.DataPropertyDockerconfigJSON, useJWTProvider),
+		// use kubernetes provider
+		compose("with kubernetes provider", f, common.JSONDataFromSync, useKubernetesProvider),
+		compose("with kubernetes provider", f, common.JSONDataWithProperty, useKubernetesProvider),
+		compose("with kubernetes provider", f, common.JSONDataWithTemplate, useKubernetesProvider),
+		compose("with kubernetes provider", f, common.DataPropertyDockerconfigJSON, useKubernetesProvider),
 	)
 })
+
+func compose(descAppend string, f *framework.Framework, fn func(f *framework.Framework) (string, func(*framework.TestCase)), tweaks ...func(*framework.TestCase)) TableEntry {
+	desc, tfn := fn(f)
+	tweaks = append(tweaks, tfn)
+	te := Entry(desc + " " + descAppend)
+
+	// need to convert []func to []interface{}
+	ifs := make([]interface{}, len(tweaks))
+	for i := 0; i < len(tweaks); i++ {
+		ifs[i] = tweaks[i]
+	}
+	te.Parameters = ifs
+	return te
+}
+
+func useTokenAuth(tc *framework.TestCase) {
+	tc.ExternalSecret.Spec.SecretStoreRef.Name = tc.Framework.Namespace.Name
+}
+
+func useCertAuth(tc *framework.TestCase) {
+	tc.ExternalSecret.Spec.SecretStoreRef.Name = certAuthProviderName
+}
+
+func useApproleAuth(tc *framework.TestCase) {
+	tc.ExternalSecret.Spec.SecretStoreRef.Name = appRoleAuthProviderName
+}
+
+func useV1Provider(tc *framework.TestCase) {
+	tc.ExternalSecret.Spec.SecretStoreRef.Name = kvv1ProviderName
+}
+
+func useJWTProvider(tc *framework.TestCase) {
+	tc.ExternalSecret.Spec.SecretStoreRef.Name = jwtProviderName
+}
+
+func useKubernetesProvider(tc *framework.TestCase) {
+	tc.ExternalSecret.Spec.SecretStoreRef.Name = kubernetesProviderName
+}

go.mod

@@ -44,6 +44,7 @@ require (
 	github.com/fatih/color v1.10.0 // indirect
 	github.com/frankban/quicktest v1.10.0 // indirect
 	github.com/go-logr/logr v0.4.0
+	github.com/golang-jwt/jwt v3.2.2+incompatible
 	github.com/google/go-cmp v0.5.5
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/uuid v1.2.0

go.sum

@@ -143,6 +143,7 @@ github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
@@ -241,6 +242,8 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
+github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -486,6 +489,7 @@ github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR
 github.com/mitchellh/mapstructure v1.3.3 h1:SzB1nHZ2Xi+17FP0zVQBHIZqvwRN9408fJO8h+eeNA8=
 github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
+github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8=
 github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=
 github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=