build: add provider build and generate targets

.dockerignore

@@ -1,6 +1,10 @@
 .git
 .github
 .gitignore
+.cache
+**/.cache
+.tmp
+**/.tmp
 .golangci.yaml
 ADOPTERS.md
 CNAME

.github/workflows/publish.yml

@@ -116,6 +116,7 @@ jobs:
         if: env.IS_FORK != ''
         shell: bash
         env:
+          DOCKER: docker buildx
           IMAGE_TAG: ${{ steps.container_info.outputs.image-tag }}
           BUILD_ARGS: ${{ inputs.build-args }}
           DOCKER_BUILD_ARGS: >-
@@ -128,6 +129,7 @@ jobs:
         if: env.IS_FORK == ''
         shell: bash
         env:
+          DOCKER: docker buildx
           IMAGE_TAG: ${{ steps.container_info.outputs.image-tag }}
           BUILD_ARGS: ${{ inputs.build-args }}
           DOCKER_BUILD_ARGS: --no-cache --load

Makefile

@@ -6,6 +6,21 @@ MAKEFLAGS     += --warn-undefined-variables
 .SHELLFLAGS   := -euo pipefail -c
 
 ARCH ?= amd64 arm64 ppc64le
+
+# Detect local architecture for e2e testing
+LOCAL_ARCH := $(shell uname -m)
+ifeq ($(LOCAL_ARCH),x86_64)
+	LOCAL_GOARCH := amd64
+else ifeq ($(LOCAL_ARCH),aarch64)
+	LOCAL_GOARCH := arm64
+else ifeq ($(LOCAL_ARCH),arm64)
+	LOCAL_GOARCH := arm64
+else ifeq ($(LOCAL_ARCH),ppc64le)
+	LOCAL_GOARCH := ppc64le
+else
+	LOCAL_GOARCH := amd64
+endif
+
 BUILD_ARGS ?= CGO_ENABLED=0
 DOCKER_BUILD_ARGS ?=
 DOCKERFILE ?= Dockerfile
@@ -69,16 +84,38 @@ ERR		= echo ${TIME} ${RED}[FAIL]${CNone}
 OK		= echo ${TIME} ${GREEN}[ OK ]${CNone}
 FAIL	= (echo ${TIME} ${RED}[FAIL]${CNone} && false)
 
+# ====================================================================================
+# Protobuf
+
+.PHONY: proto
+proto: ## Generate protobuf code
+	@$(INFO) generating protobuf code
+	@protoc --go_out=. --go_opt=paths=source_relative \
+		--go-grpc_out=. --go-grpc_opt=paths=source_relative \
+		-I. \
+		providers/v2/common/proto/provider/secretstore.proto
+	@protoc --go_out=. --go_opt=paths=source_relative \
+		--go-grpc_out=. --go-grpc_opt=paths=source_relative \
+		-I. \
+		providers/v2/common/proto/generator/generator.proto
+	@for file in \
+		providers/v2/common/proto/provider/secretstore.pb.go \
+		providers/v2/common/proto/provider/secretstore_grpc.pb.go \
+		providers/v2/common/proto/generator/generator.pb.go \
+		providers/v2/common/proto/generator/generator_grpc.pb.go; do \
+		tmp=$$(mktemp); \
+		cat hack/boilerplate.go.txt "$$file" > "$$tmp"; \
+		mv "$$tmp" "$$file"; \
+	done
+	@$(OK) protobuf code generated
+
 # ====================================================================================
 # Conformance
 
-reviewable: generate docs manifests helm.generate helm.schema.update helm.docs lint license.check helm.test.update test.crds.update tf.fmt ## Ensure a PR is ready for review.
-	@go mod tidy
-	@cd e2e/ && go mod tidy
-	@cd apis/ && go mod tidy
-	@cd runtime/ && go mod tidy
-	@for provider in providers/v1/*/; do (cd $$provider && go mod tidy); done
-	@for generator in generators/v1/*/; do (cd $$generator && go mod tidy); done
+reviewable: generate docs manifests helm.generate helm.schema.update helm.docs lint license.check helm.test.update test.crds.update tf.fmt generate-providers verify-providers ## Ensure a PR is ready for review.
+	@for module in . e2e apis runtime $$(find providers/v1 generators/v1 providers/v2 -name go.mod -not -path '*/vendor/*' -exec dirname {} \; | sort); do \
+		(cd "$$module" && GOWORK=off go mod tidy); \
+	done
 
 check-diff: reviewable ## Ensure branch is clean.
 	@$(INFO) checking that branch is clean
@@ -99,16 +136,16 @@ license.check:
 go-work:
 	@$(INFO) creating go workspace
 	@rm -rf go.work go.work.sum
-	@go work init
-	@go work use -r .
-	@go work edit -dropuse ./e2e
-	@go work sync
+	@GOWORK=off go work init
+	@GOWORK="$(shell pwd)/go.work" go work use -r .
+	@GOWORK="$(shell pwd)/go.work" go work edit -dropuse ./e2e
+	@GOWORK="$(shell pwd)/go.work" go work sync
 	@$(OK) created go workspace
 
 .PHONY: test
 test: generate envtest go-work ## Run tests
 	@$(INFO) go test unit-tests
-	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(KUBERNETES_VERSION) -p path --bin-dir $(LOCALBIN))" go test -tags $(PROVIDER) work -v -race -coverprofile cover.out
+	KUBEBUILDER_ASSETS="$(abspath $(shell $(ENVTEST) use $(KUBERNETES_VERSION) -p path --bin-dir $(LOCALBIN)))" go test -tags $(PROVIDER) work -v -race -coverprofile cover.out
 	@$(OK) go test unit-tests
 
 .PHONY: test.e2e
@@ -186,6 +223,21 @@ generate: ## Generate code and crds
 	@./hack/crd.generate.sh $(BUNDLE_DIR) $(CRD_DIR)
 	@$(OK) Finished generating deepcopy and crds
 
+generate-providers: ## Generate provider main.go and Dockerfile files from provider.yaml configs
+	@$(INFO) Generating provider files
+	@cd providers/v2/hack && go run generate-provider-main.go -providers-dir=..
+	@$(OK) Generated provider files
+
+verify-providers: ## Verify that provider files are up to date
+	@$(INFO) Verifying provider files are up to date
+	@cd providers/v2/hack && go run generate-provider-main.go -providers-dir=.. -dry-run
+	@if ! git diff --quiet providers/v2/*/main.go providers/v2/*/Dockerfile 2>/dev/null; then \
+		echo "Provider files are out of date. Run 'make generate-providers' to update them."; \
+		git diff providers/v2/*/main.go providers/v2/*/Dockerfile; \
+		exit 1; \
+	fi
+	@$(OK) Provider files are up to date
+
 # ====================================================================================
 # Local Utility
 
@@ -200,7 +252,7 @@ manifests: helm.generate ## Generate manifests from helm chart
 	helm template external-secrets $(HELM_DIR) -f deploy/manifests/helm-values.yaml > $(OUTPUT_DIR)/deploy/manifests/external-secrets.yaml
 
 crds.install: generate ## Install CRDs into a cluster. This is for convenience
-	kubectl apply -f $(BUNDLE_DIR) --server-side
+	kubectl apply -f $(BUNDLE_DIR) --server-side --force-conflicts
 
 crds.uninstall: ## Uninstall CRDs from a cluster. This is for convenience
 	kubectl delete -f $(BUNDLE_DIR)
@@ -323,17 +375,87 @@ docker.tag:  ## Emit IMAGE_TAG
 	@echo $(IMAGE_TAG)
 
 .PHONY: docker.build
-docker.build: $(addprefix build-,$(ARCH)) ## Build the docker image
-	@$(INFO) $(DOCKER) build
-	echo $(DOCKER) buildx build -f $(DOCKERFILE) . $(DOCKER_BUILD_ARGS) -t $(IMAGE_NAME):$(IMAGE_TAG)
-	$(DOCKER) buildx build -f $(DOCKERFILE) . $(DOCKER_BUILD_ARGS) -t $(IMAGE_NAME):$(IMAGE_TAG)
-	@$(OK) $(DOCKER) build
+docker.build: docker.build.controller docker.build.providers ## Build all docker images (controller + providers)
+
+.PHONY: docker.build.e2e
+docker.build.e2e: docker.build.controller.e2e ## Build docker images for local e2e testing (local arch only)
+
+.PHONY: docker.build.controller
+docker.build.controller: $(addprefix build-,$(ARCH)) ## Build the controller docker image
+	@$(INFO) $(DOCKER) build controller
+	@echo $(DOCKER) build -f $(DOCKERFILE) . $(DOCKER_BUILD_ARGS) -t $(IMAGE_NAME):$(IMAGE_TAG)
+	@DOCKER_BUILDKIT=1 $(DOCKER) build -f $(DOCKERFILE) . $(DOCKER_BUILD_ARGS) -t $(IMAGE_NAME):$(IMAGE_TAG)
+	@$(OK) $(DOCKER) build controller
+
+.PHONY: docker.build.controller.e2e
+docker.build.controller.e2e: build-$(LOCAL_GOARCH) ## Build the controller docker image for local arch only
+	@$(INFO) $(DOCKER) build controller for $(LOCAL_GOARCH)
+	@echo $(DOCKER) build -f $(DOCKERFILE) . $(DOCKER_BUILD_ARGS) -t $(IMAGE_NAME):$(IMAGE_TAG)
+	@DOCKER_BUILDKIT=1 $(DOCKER) build -f $(DOCKERFILE) . $(DOCKER_BUILD_ARGS) -t $(IMAGE_NAME):$(IMAGE_TAG)
+	@$(OK) $(DOCKER) build controller for $(LOCAL_GOARCH)
+
+.PHONY: docker.build.providers
+docker.build.providers: docker.build.provider.kubernetes docker.build.provider.aws docker.build.provider.fake ## Build all provider images
+
+.PHONY: docker.build.provider.kubernetes
+docker.build.provider.kubernetes: ## Build Kubernetes provider image
+	@$(INFO) $(DOCKER) build kubernetes provider
+	@DOCKER_BUILDKIT=1 $(DOCKER) build \
+		-f providers/v2/kubernetes/Dockerfile \
+		. \
+		$(DOCKER_BUILD_ARGS) \
+		-t $(IMAGE_REGISTRY)/external-secrets/provider-kubernetes:$(IMAGE_TAG)
+	@$(OK) $(DOCKER) build kubernetes provider
+
+.PHONY: docker.build.provider.aws
+docker.build.provider.aws: ## Build AWS provider image
+	@$(INFO) $(DOCKER) build AWS provider
+	@DOCKER_BUILDKIT=1 $(DOCKER) build \
+		-f providers/v2/aws/Dockerfile \
+		. \
+		$(DOCKER_BUILD_ARGS) \
+		-t $(IMAGE_REGISTRY)/external-secrets/provider-aws:$(IMAGE_TAG)
+	@$(OK) $(DOCKER) build AWS provider
+
+.PHONY: docker.build.provider.fake
+docker.build.provider.fake: ## Build Fake provider image
+	@$(INFO) $(DOCKER) build Fake provider
+	@DOCKER_BUILDKIT=1 $(DOCKER) build \
+		-f providers/v2/fake/Dockerfile \
+		. \
+		$(DOCKER_BUILD_ARGS) \
+		-t $(IMAGE_REGISTRY)/external-secrets/provider-fake:$(IMAGE_TAG)
+	@$(OK) $(DOCKER) build Fake provider
 
 .PHONY: docker.push
-docker.push: ## Push the docker image to the registry
-	@$(INFO) $(DOCKER) push
+docker.push: docker.push.controller docker.push.providers ## Push all docker images to the registry
+
+.PHONY: docker.push.controller
+docker.push.controller: ## Push the controller docker image to the registry
+	@$(INFO) $(DOCKER) push controller
 	@$(DOCKER) push $(IMAGE_NAME):$(IMAGE_TAG)
-	@$(OK) $(DOCKER) push
+	@$(OK) $(DOCKER) push controller
+
+.PHONY: docker.push.providers
+docker.push.providers: docker.push.provider.kubernetes docker.push.provider.aws docker.push.provider.fake ## Push all provider images
+
+.PHONY: docker.push.provider.kubernetes
+docker.push.provider.kubernetes: ## Push Kubernetes provider image
+	@$(INFO) $(DOCKER) push kubernetes provider
+	@$(DOCKER) push $(IMAGE_REGISTRY)/external-secrets/provider-kubernetes:$(IMAGE_TAG)
+	@$(OK) $(DOCKER) push kubernetes provider
+
+.PHONY: docker.push.provider.aws
+docker.push.provider.aws: ## Push AWS provider image
+	@$(INFO) $(DOCKER) push AWS provider
+	@$(DOCKER) push $(IMAGE_REGISTRY)/external-secrets/provider-aws:$(IMAGE_TAG)
+	@$(OK) $(DOCKER) push AWS provider
+
+.PHONY: docker.push.provider.fake
+docker.push.provider.fake: ## Push Fake provider image
+	@$(INFO) $(DOCKER) push Fake provider
+	@$(DOCKER) push $(IMAGE_REGISTRY)/external-secrets/provider-fake:$(IMAGE_TAG)
+	@$(OK) $(DOCKER) push Fake provider
 
 # RELEASE_TAG is tag to promote. Default is promoting to main branch, but can be overriden
 # to promote a tag to a specific version.