Lookup cluster identity from instance metadata (#4575)

issues: #3540

This change adds the capability to the gcp secret manager provider to
lookup the clusters workload id pool and provider from the metadata
server.

Signed-off-by: Felix Ehrenpfort <felix@ehrenpfort.de>
Co-authored-by: Gustavo Fernandes de Carvalho <17139678+gusfcarvalho@users.noreply.github.com>

go.mod

@@ -61,6 +61,7 @@ require (
 require github.com/1Password/connect-sdk-go v1.5.3
 
 require (
+	cloud.google.com/go/compute/metadata v0.6.0
 	dario.cat/mergo v1.0.1
 	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.0
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.2
@@ -109,7 +110,6 @@ require (
 	al.essio.dev/pkg/shellescape v1.6.0 // indirect
 	cloud.google.com/go/auth v0.15.0 // indirect
 	cloud.google.com/go/auth/oauth2adapt v0.2.7 // indirect
-	cloud.google.com/go/compute/metadata v0.6.0 // indirect
 	github.com/ProtonMail/go-crypto v1.1.6 // indirect
 	github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f // indirect
 	github.com/ProtonMail/gopenpgp/v2 v2.8.3 // indirect

pkg/provider/gcp/secretmanager/workload_identity.go

@@ -23,6 +23,7 @@ import (
 	"net/http"
 	"time"
 
+	"cloud.google.com/go/compute/metadata"
 	iam "cloud.google.com/go/iam/credentials/apiv1"
 	"cloud.google.com/go/iam/credentials/apiv1/credentialspb"
 	secretmanager "cloud.google.com/go/secretmanager/apiv1"
@@ -52,6 +53,7 @@ const (
 	errFetchPodToken  = "unable to fetch pod token: %w"
 	errFetchIBToken   = "unable to fetch identitybindingtoken: %w"
 	errGenAccessToken = "unable to generate gcp access token: %w"
+	errLookupIdentity = "unable to lookup workload identity: %w"
 	errNoProjectID    = "unable to find ProjectID in storeSpec"
 )
 
@@ -59,6 +61,7 @@ const (
 // to create a gcp oauth token.
 type workloadIdentity struct {
 	iamClient            IamClient
+	metadataClient       MetadataClient
 	idBindTokenGenerator idBindTokenGenerator
 	saTokenGenerator     saTokenGenerator
 	clusterProjectID     string
@@ -70,6 +73,12 @@ type IamClient interface {
 	Close() error
 }
 
+// interface to GCP Metadata API.
+type MetadataClient interface {
+	InstanceAttributeValueWithContext(ctx context.Context, attr string) (string, error)
+	ProjectIDWithContext(ctx context.Context) (string, error)
+}
+
 // interface to securetoken/identitybindingtoken API.
 type idBindTokenGenerator interface {
 	Generate(context.Context, *http.Client, string, string, string) (*oauth2.Token, error)
@@ -91,12 +100,46 @@ func newWorkloadIdentity(ctx context.Context, projectID string) (*workloadIdenti
 	}
 	return &workloadIdentity{
 		iamClient:            iamc,
+		metadataClient:       newMetadataClient(),
 		idBindTokenGenerator: newIDBindTokenGenerator(),
 		saTokenGenerator:     satg,
 		clusterProjectID:     projectID,
 	}, nil
 }
 
+func (w *workloadIdentity) gcpWorkloadIdentity(ctx context.Context, id *esv1beta1.GCPWorkloadIdentity) (string, string, error) {
+	var err error
+
+	projectID := id.ClusterProjectID
+	if projectID == "" {
+		if projectID, err = w.metadataClient.ProjectIDWithContext(ctx); err != nil {
+			return "", "", fmt.Errorf("unable to get project id: %w", err)
+		}
+	}
+
+	clusterLocation := id.ClusterLocation
+	if clusterLocation == "" {
+		if clusterLocation, err = w.metadataClient.InstanceAttributeValueWithContext(ctx, "cluster-location"); err != nil {
+			return "", "", fmt.Errorf("unable to determine cluster location: %w", err)
+		}
+	}
+
+	clusterName := id.ClusterName
+	if clusterName == "" {
+		if clusterName, err = w.metadataClient.InstanceAttributeValueWithContext(ctx, "cluster-name"); err != nil {
+			return "", "", fmt.Errorf("unable to determine cluster name: %w", err)
+		}
+	}
+
+	idPool := fmt.Sprintf("%s.svc.id.goog", projectID)
+	idProvider := fmt.Sprintf("https://container.googleapis.com/v1/projects/%s/locations/%s/clusters/%s",
+		projectID,
+		clusterLocation,
+		clusterName,
+	)
+	return idPool, idProvider, nil
+}
+
 func (w *workloadIdentity) TokenSource(ctx context.Context, auth esv1beta1.GCPSMAuth, isClusterKind bool, kube kclient.Client, namespace string) (oauth2.TokenSource, error) {
 	wi := auth.WorkloadIdentity
 	if wi == nil {
@@ -118,11 +161,11 @@ func (w *workloadIdentity) TokenSource(ctx context.Context, auth esv1beta1.GCPSM
 		return nil, err
 	}
 
-	idProvider := fmt.Sprintf("https://container.googleapis.com/v1/projects/%s/locations/%s/clusters/%s",
-		w.clusterProjectID,
-		wi.ClusterLocation,
-		wi.ClusterName)
-	idPool := fmt.Sprintf("%s.svc.id.goog", w.clusterProjectID)
+	idPool, idProvider, err := w.gcpWorkloadIdentity(ctx, wi)
+	if err != nil {
+		return nil, fmt.Errorf(errLookupIdentity, err)
+	}
+
 	audiences := []string{idPool}
 	if len(wi.ServiceAccountRef.Audiences) > 0 {
 		audiences = append(audiences, wi.ServiceAccountRef.Audiences...)
@@ -181,6 +224,12 @@ func newIAMClient(ctx context.Context) (IamClient, error) {
 	return iam.NewIamCredentialsClient(ctx, iamOpts...)
 }
 
+func newMetadataClient() MetadataClient {
+	return metadata.NewClient(&http.Client{
+		Timeout: 5 * time.Second,
+	})
+}
+
 type k8sSATokenGenerator struct {
 	corev1 clientcorev1.CoreV1Interface
 }

pkg/provider/gcp/secretmanager/workload_identity_test.go

@@ -17,6 +17,8 @@ package secretmanager
 import (
 	"context"
 	"encoding/json"
+	"errors"
+	"fmt"
 	"io"
 	"net/http"
 	"net/http/httptest"
@@ -45,6 +47,7 @@ type workloadIdentityTest struct {
 	genAccessToken func(context.Context, *credentialspb.GenerateAccessTokenRequest, ...gax.CallOption) (*credentialspb.GenerateAccessTokenResponse, error)
 	genIDBindToken func(ctx context.Context, client *http.Client, k8sToken, idPool, idProvider string) (*oauth2.Token, error)
 	genSAToken     func(c context.Context, s1 []string, s2, s3 string) (*authv1.TokenRequest, error)
+	instMetadata   map[string]string
 	store          esv1beta1.GenericStore
 	kubeObjects    []client.Object
 }
@@ -134,15 +137,30 @@ func TestWorkloadIdentity(t *testing.T) {
 				},
 			}),
 		),
+		composeTestcase(
+			defaultTestCase("lookup cluster id from instance metadata"),
+			expTokenSource(),
+			expectToken(defaultGenAccessToken),
+			withStore(
+				composeStore(defaultStore(), withClusterID("", "", "")),
+			),
+			withInstMetadata(map[string]string{
+				"project-id":       "1234",
+				"cluster-location": "example",
+				"cluster-name":     "foobar",
+			}),
+		),
 	}
 
 	for _, row := range tbl {
 		t.Run(row.name, func(t *testing.T) {
 			fakeIam := &fakeIAMClient{generateAccessTokenFunc: row.genAccessToken}
+			fakeMeta := &fakeMetadataClient{metadata: row.instMetadata}
 			fakeIDBGen := &fakeIDBindTokenGen{generateFunc: row.genIDBindToken}
 			fakeSATG := &fakeSATokenGen{GenerateFunc: row.genSAToken}
 			w := &workloadIdentity{
 				iamClient:            fakeIam,
+				metadataClient:       fakeMeta,
 				idBindTokenGenerator: fakeIDBGen,
 				saTokenGenerator:     fakeSATG,
 			}
@@ -258,6 +276,12 @@ func withK8sResources(objs []client.Object) testCaseMutator {
 	}
 }
 
+func withInstMetadata(metadata map[string]string) testCaseMutator {
+	return func(tc *workloadIdentityTest) {
+		tc.instMetadata = metadata
+	}
+}
+
 var (
 	defaultGenAccessToken = "default-gen-access-token"
 	defaultIDBindToken    = "default-id-bind-token"
@@ -284,6 +308,9 @@ func defaultTestCase(name string) *workloadIdentityTest {
 				},
 			}, nil
 		},
+		instMetadata: map[string]string{
+			"project-id": "1234",
+		},
 		kubeObjects: []client.Object{
 			&v1.ServiceAccount{
 				ObjectMeta: metav1.ObjectMeta{
@@ -378,6 +405,15 @@ func composeStore(store esv1beta1.GenericStore, mutators ...storeMutator) esv1be
 	return store
 }
 
+func withClusterID(project, location, name string) storeMutator {
+	return func(store esv1beta1.GenericStore) {
+		spc := store.GetSpec()
+		spc.Provider.GCPSM.Auth.WorkloadIdentity.ClusterProjectID = project
+		spc.Provider.GCPSM.Auth.WorkloadIdentity.ClusterLocation = location
+		spc.Provider.GCPSM.Auth.WorkloadIdentity.ClusterName = name
+	}
+}
+
 func withSANamespace(namespace string) storeMutator {
 	return func(store esv1beta1.GenericStore) {
 		spc := store.GetSpec()
@@ -407,6 +443,25 @@ func (f *fakeIAMClient) Close() error {
 	return nil
 }
 
+// fake Metadata Client.
+type fakeMetadataClient struct {
+	metadata map[string]string
+}
+
+func (f *fakeMetadataClient) InstanceAttributeValueWithContext(ctx context.Context, attr string) (string, error) {
+	if val, ok := f.metadata[attr]; ok {
+		return val, nil
+	}
+	return "", fmt.Errorf("attr %s not found", attr)
+}
+
+func (f *fakeMetadataClient) ProjectIDWithContext(ctx context.Context) (string, error) {
+	if val, ok := f.metadata["project-id"]; ok {
+		return val, nil
+	}
+	return "", errors.New("attr project-id not found")
+}
+
 // fake SA Token Generator.
 type fakeSATokenGen struct {
 	GenerateFunc func(context.Context, []string, string, string) (*authv1.TokenRequest, error)