[internal] Update GitHub Actions workflow files

pulumi · Nov 22, 2024 · 925155d · 925155d
1 parent be16028
commit 925155d
Show file tree

Hide file tree

Showing 2 changed files with 136 additions and 90 deletions.
diff --git a/.gitignore b/.gitignore
@@ -32,4 +32,7 @@ sdk/java/gradlew.bat
 sdk/python/venv
 
 go.work
-go.work.sum
+go.work.sum
+
+# Ignore local build tracking directory
+.make
diff --git a/Makefile b/Makefile
@@ -27,148 +27,191 @@ LDFLAGS_UPSTREAM_VERSION=
 LDFLAGS_EXTRAS=
 LDFLAGS=$(LDFLAGS_PROJ_VERSION) $(LDFLAGS_UPSTREAM_VERSION) $(LDFLAGS_EXTRAS) $(LDFLAGS_STRIP_SYMBOLS)
 
-development: install_plugins provider build_sdks install_sdks
+# Create a `.make` directory for tracking targets which don't generate a single file output. This should be ignored by git.
+# For targets which either don't generate a single file output, or the output file is committed, we use a "sentinel"
+# file within `.make/` to track the staleness of the target and only rebuild when needed. 
+# For each phony target, we create an internal target with the same name, but prefixed with `.make/` where the work is performed.
+# At the end of each internal target we run `@touch $@` to update the file which is the name of the target.
 
-build: install_plugins provider build_sdks install_sdks
-
-build_sdks: build_nodejs build_python build_dotnet build_go build_java 
-
-install_go_sdk:
-
-install_java_sdk:
-
-install_python_sdk:
-
-install_sdks: install_dotnet_sdk install_python_sdk install_nodejs_sdk install_java_sdk
+# Ensure all directories exist before evaluating targets to avoid issues with `touch` creating directories.
+_ := $(shell mkdir -p .make bin .pulumi/bin)
 
+# Keep aliases for old targets to ensure backwards compatibility
+development: build
 only_build: build
-
-build_dotnet: export PULUMI_HOME := $(WORKING_DIR)/.pulumi
-build_dotnet: export PATH := $(WORKING_DIR)/.pulumi/bin:$(PATH)
-build_dotnet: export PULUMI_CONVERT_EXAMPLES_CACHE_DIR := $(WORKING_DIR)/.pulumi/examples-cache
-build_dotnet: upstream
-	PULUMI_CONVERT=$(PULUMI_CONVERT) PULUMI_DISABLE_AUTOMATIC_PLUGIN_ACQUISITION=$(PULUMI_CONVERT) $(WORKING_DIR)/bin/$(TFGEN) dotnet --out sdk/dotnet/
+# Build the provider and all SDKs and install ready for testing
+build: install_plugins provider build_sdks install_sdks
+# Creates all generated files which need to be committed
+generate: generate_sdks schema
+generate_sdks: generate_nodejs generate_python generate_dotnet generate_go generate_java
+build_sdks: build_nodejs build_python build_dotnet build_go build_java
+install_sdks: install_nodejs_sdk install_python_sdk install_dotnet_sdk install_go_sdk install_java_sdk
+.PHONY: development only_build build generate generate_sdks build_sdks install_sdks
+
+GEN_PULUMI_HOME := $(WORKING_DIR)/.pulumi
+GEN_PULUMI_CONVERT_EXAMPLES_CACHE_DIR := $(GEN_PULUMI_HOME)/examples-cache
+GEN_ENVS := PULUMI_HOME=$(GEN_PULUMI_HOME) PULUMI_CONVERT_EXAMPLES_CACHE_DIR=$(GEN_PULUMI_CONVERT_EXAMPLES_CACHE_DIR) PULUMI_CONVERT=$(PULUMI_CONVERT) PULUMI_DISABLE_AUTOMATIC_PLUGIN_ACQUISITION=$(PULUMI_CONVERT)
+
+generate_dotnet: .make/generate_dotnet
+build_dotnet: .make/build_dotnet
+.make/generate_dotnet: export PATH := $(WORKING_DIR)/.pulumi/bin:$(PATH)
+.make/generate_dotnet: bin/$(TFGEN)
+	$(GEN_ENVS) $(WORKING_DIR)/bin/$(TFGEN) dotnet --out sdk/dotnet/
 	cd sdk/dotnet/ && \
 		printf "module fake_dotnet_module // Exclude this directory from Go tools\n\ngo 1.17\n" > go.mod && \
-		echo "$(VERSION_GENERIC)" >version.txt && \
-		dotnet build
-
-build_go: export PULUMI_HOME := $(WORKING_DIR)/.pulumi
-build_go: export PATH := $(WORKING_DIR)/.pulumi/bin:$(PATH)
-build_go: export PULUMI_CONVERT_EXAMPLES_CACHE_DIR := $(WORKING_DIR)/.pulumi/examples-cache
-build_go: upstream
-	PULUMI_CONVERT=$(PULUMI_CONVERT) PULUMI_DISABLE_AUTOMATIC_PLUGIN_ACQUISITION=$(PULUMI_CONVERT) $(WORKING_DIR)/bin/$(TFGEN) go --out sdk/go/
+		echo "$(VERSION_GENERIC)" >version.txt
+	@touch $@
+.make/build_dotnet: .make/generate_dotnet
+	cd sdk/dotnet/ && dotnet build
+	@touch $@
+.PHONY: generate_dotnet build_dotnet
+
+generate_go: .make/generate_go
+build_go: .make/build_go
+.make/generate_go: export PATH := $(WORKING_DIR)/.pulumi/bin:$(PATH)
+.make/generate_go: bin/$(TFGEN)
+	$(GEN_ENVS) $(WORKING_DIR)/bin/$(TFGEN) go --out sdk/go/
+	@touch $@
+.make/build_go: .make/generate_go
 	cd sdk && go list "$$(grep -e "^module" go.mod | cut -d ' ' -f 2)/go/..." | xargs -I {} bash -c 'go build {} && go clean -i {}'
-
-build_java: PACKAGE_VERSION := $(VERSION_GENERIC)
-build_java: export PULUMI_HOME := $(WORKING_DIR)/.pulumi
-build_java: export PATH := $(WORKING_DIR)/.pulumi/bin:$(PATH)
-build_java: export PULUMI_CONVERT_EXAMPLES_CACHE_DIR := $(WORKING_DIR)/.pulumi/examples-cache
-build_java: bin/pulumi-java-gen upstream
-	$(WORKING_DIR)/bin/$(JAVA_GEN) generate --schema provider/cmd/$(PROVIDER)/schema.json --out sdk/java  --build gradle-nexus
+	@touch $@
+.PHONY: generate_go build_go
+
+generate_java: .make/generate_java
+build_java: .make/build_java
+.make/generate_java: export PATH := $(WORKING_DIR)/.pulumi/bin:$(PATH)
+.make/generate_java: PACKAGE_VERSION := $(VERSION_GENERIC)
+.make/generate_java: bin/pulumi-java-gen .make/schema
+	PULUMI_HOME=$(GEN_PULUMI_HOME) PULUMI_CONVERT_EXAMPLES_CACHE_DIR=$(GEN_PULUMI_CONVERT_EXAMPLES_CACHE_DIR) bin/$(JAVA_GEN) generate --schema provider/cmd/$(PROVIDER)/schema.json --out sdk/java  --build gradle-nexus
+	printf "module fake_java_module // Exclude this directory from Go tools\n\ngo 1.17\n" > sdk/java/go.mod
+	@touch $@
+.make/build_java: PACKAGE_VERSION := $(VERSION_GENERIC)
+.make/build_java: .make/generate_java
 	cd sdk/java/ && \
-		printf "module fake_java_module // Exclude this directory from Go tools\n\ngo 1.17\n" > go.mod && \
 		gradle --console=plain build && \
 		gradle --console=plain javadoc
-
-build_nodejs: export PULUMI_HOME := $(WORKING_DIR)/.pulumi
-build_nodejs: export PATH := $(WORKING_DIR)/.pulumi/bin:$(PATH)
-build_nodejs: export PULUMI_CONVERT_EXAMPLES_CACHE_DIR := $(WORKING_DIR)/.pulumi/examples-cache
-build_nodejs: upstream
-	PULUMI_CONVERT=$(PULUMI_CONVERT) PULUMI_DISABLE_AUTOMATIC_PLUGIN_ACQUISITION=$(PULUMI_CONVERT) $(WORKING_DIR)/bin/$(TFGEN) nodejs --out sdk/nodejs/
+	@touch $@
+.PHONY: generate_java build_java
+
+generate_nodejs: .make/generate_nodejs
+build_nodejs: .make/build_nodejs
+.make/generate_nodejs: export PATH := $(WORKING_DIR)/.pulumi/bin:$(PATH)
+.make/generate_nodejs: bin/$(TFGEN)
+	$(GEN_ENVS) $(WORKING_DIR)/bin/$(TFGEN) nodejs --out sdk/nodejs/
+	printf "module fake_nodejs_module // Exclude this directory from Go tools\n\ngo 1.17\n" > sdk/nodejs/go.mod
+	@touch $@
+.make/build_nodejs: .make/generate_nodejs
 	cd sdk/nodejs/ && \
-		printf "module fake_nodejs_module // Exclude this directory from Go tools\n\ngo 1.17\n" > go.mod && \
 		yarn install && \
 		yarn run tsc && \
 		cp ../../README.md ../../LICENSE* package.json yarn.lock ./bin/
-
-build_python: export PULUMI_HOME := $(WORKING_DIR)/.pulumi
-build_python: export PATH := $(WORKING_DIR)/.pulumi/bin:$(PATH)
-build_python: export PULUMI_CONVERT_EXAMPLES_CACHE_DIR := $(WORKING_DIR)/.pulumi/examples-cache
-build_python: upstream
-	rm -rf sdk/python/
-	PULUMI_CONVERT=$(PULUMI_CONVERT) PULUMI_DISABLE_AUTOMATIC_PLUGIN_ACQUISITION=$(PULUMI_CONVERT) $(WORKING_DIR)/bin/$(TFGEN) python --out sdk/python/
+	@touch $@
+.PHONY: generate_nodejs build_nodejs
+
+generate_python: .make/generate_python
+build_python: .make/build_python
+.make/generate_python: export PATH := $(WORKING_DIR)/.pulumi/bin:$(PATH)
+.make/generate_python: bin/$(TFGEN)
+	$(GEN_ENVS) $(WORKING_DIR)/bin/$(TFGEN) python --out sdk/python/
+	printf "module fake_python_module // Exclude this directory from Go tools\n\ngo 1.17\n" > sdk/python/go.mod
+	cp README.md sdk/python/
+	@touch $@
+.make/build_python: .make/generate_python
 	cd sdk/python/ && \
-		printf "module fake_python_module // Exclude this directory from Go tools\n\ngo 1.17\n" > go.mod && \
-		cp ../../README.md . && \
 		rm -rf ./bin/ ../python.bin/ && cp -R . ../python.bin && mv ../python.bin ./bin && \
 		rm ./bin/go.mod && \
 		python3 -m venv venv && \
 		./venv/bin/python -m pip install build==1.2.1 && \
 		cd ./bin && \
 		../venv/bin/python -m build .
+	@touch $@
+.PHONY: generate_python build_python
 
 clean:
 	rm -rf sdk/{dotnet,nodejs,go,python}
+	rm -rf bin/*
+	rm -rf .make/*
+.PHONY: clean
 
-cleanup:
-	rm -r $(WORKING_DIR)/bin
-	rm -f provider/cmd/$(PROVIDER)/schema.go
-
-help:
-	@grep '^[^.#]\+:\s\+.*#' Makefile | \
-	sed "s/\(.\+\):\s*\(.*\) #\s*\(.*\)/`printf "\033[93m"`\1`printf "\033[0m"`	\3 [\2]/" | \
-	expand -t20
-
-install_dotnet_sdk:
+install_dotnet_sdk: .make/install_dotnet_sdk
+.make/install_dotnet_sdk: .make/build_dotnet
 	mkdir -p $(WORKING_DIR)/nuget
 	find . -name '*.nupkg' -print -exec cp -p {} $(WORKING_DIR)/nuget \;
-
-install_nodejs_sdk:
+	@touch $@
+install_go_sdk:
+install_java_sdk:
+install_nodejs_sdk: .make/install_nodejs_sdk
+.make/install_nodejs_sdk: .make/build_nodejs
 	yarn link --cwd $(WORKING_DIR)/sdk/nodejs/bin
+	@touch $@
+install_python_sdk:
+.PHONY: install_dotnet_sdk install_go_sdk install_java_sdk install_nodejs_sdk install_python_sdk
 
-install_plugins: export PULUMI_HOME := $(WORKING_DIR)/.pulumi
-install_plugins: export PATH := $(WORKING_DIR)/.pulumi/bin:$(PATH)
-install_plugins: .pulumi/bin/pulumi
+install_plugins: .make/install_plugins
+.make/install_plugins: export PULUMI_HOME := $(WORKING_DIR)/.pulumi
+.make/install_plugins: export PATH := $(WORKING_DIR)/.pulumi/bin:$(PATH)
+.make/install_plugins: .pulumi/bin/pulumi
 	.pulumi/bin/pulumi plugin install resource random 4.8.2
 	.pulumi/bin/pulumi plugin install resource std 1.6.2
 	.pulumi/bin/pulumi plugin install converter terraform 1.0.15
+	@touch $@
+.PHONY: install_plugins
 
 lint_provider: provider
 	cd provider && golangci-lint run --path-prefix provider -c ../.golangci.yml
-
 # `lint_provider.fix` is a utility target meant to be run manually
 # that will run the linter and fix errors when possible.
 lint_provider.fix:
 	cd provider && golangci-lint run --path-prefix provider -c ../.golangci.yml --fix
+.PHONY: lint_provider lint_provider.fix
 
 # `make provider_no_deps` builds the provider binary directly, without ensuring that
 # `cmd/pulumi-resource-xyz/schema.json` is valid and up to date.
 # To create a release ready binary, you should use `make provider`.
+build_provider_cmd = cd provider && go build $(PULUMI_PROVIDER_BUILD_PARALLELISM) -o $(WORKING_DIR)/bin/$(PROVIDER) -ldflags "$(LDFLAGS)" $(PROJECT)/$(PROVIDER_PATH)/cmd/$(PROVIDER)
+provider: bin/$(PROVIDER)
 provider_no_deps:
-	(cd provider && go build $(PULUMI_PROVIDER_BUILD_PARALLELISM) -o $(WORKING_DIR)/bin/$(PROVIDER) -ldflags "$(LDFLAGS)" $(PROJECT)/$(PROVIDER_PATH)/cmd/$(PROVIDER))
-
-provider: tfgen provider_no_deps
+	$(call build_provider_cmd)
+bin/$(PROVIDER): .make/schema
+	$(call build_provider_cmd)
+.PHONY: provider provider_no_deps
 
 test: export PATH := $(WORKING_DIR)/bin:$(PATH)
 test:
 	cd examples && go test -v -tags=all -parallel $(TESTPARALLELISM) -timeout 2h
+.PHONY: test
 
 test_provider:
-	@echo ""
-	@echo "== test_provider ==================================================================="
-	@echo ""
 	cd provider && go test -v -short ./... -parallel $(TESTPARALLELISM)
-
-tfgen: install_plugins upstream tfgen_no_deps
-
-tfgen_no_deps: export PULUMI_HOME := $(WORKING_DIR)/.pulumi
-tfgen_no_deps: export PATH := $(WORKING_DIR)/.pulumi/bin:$(PATH)
-tfgen_no_deps: export PULUMI_CONVERT := $(PULUMI_CONVERT)
-tfgen_no_deps: export PULUMI_CONVERT_EXAMPLES_CACHE_DIR := $(WORKING_DIR)/.pulumi/examples-cache
-tfgen_no_deps: export PULUMI_DISABLE_AUTOMATIC_PLUGIN_ACQUISITION := $(PULUMI_CONVERT)
-tfgen_no_deps: export PULUMI_MISSING_DOCS_ERROR := $(PULUMI_MISSING_DOCS_ERROR)
-tfgen_no_deps: tfgen_build_only
+.PHONY: test_provider
+
+tfgen: schema
+schema: .make/schema 
+# This does actually have dependencies, but we're keeping it around for backwards compatibility for now
+tfgen_no_deps: .make/schema
+.make/schema: export PULUMI_HOME := $(WORKING_DIR)/.pulumi
+.make/schema: export PATH := $(WORKING_DIR)/.pulumi/bin:$(PATH)
+.make/schema: export PULUMI_CONVERT := $(PULUMI_CONVERT)
+.make/schema: export PULUMI_CONVERT_EXAMPLES_CACHE_DIR := $(WORKING_DIR)/.pulumi/examples-cache
+.make/schema: export PULUMI_DISABLE_AUTOMATIC_PLUGIN_ACQUISITION := $(PULUMI_CONVERT)
+.make/schema: export PULUMI_MISSING_DOCS_ERROR := $(PULUMI_MISSING_DOCS_ERROR)
+.make/schema: bin/$(TFGEN) provider/resources.go provider/go.mod .make/install_plugins .make/upstream
 	$(WORKING_DIR)/bin/$(TFGEN) schema --out provider/cmd/$(PROVIDER)
 	(cd provider && VERSION=$(VERSION_GENERIC) go generate cmd/$(PROVIDER)/main.go)
-
-tfgen_build_only:
+	@touch $@
+tfgen_build_only: bin/$(TFGEN)
+bin/$(TFGEN):
 	(cd provider && go build $(PULUMI_PROVIDER_BUILD_PARALLELISM) -o $(WORKING_DIR)/bin/$(TFGEN) -ldflags "$(LDFLAGS_PROJ_VERSION) $(LDFLAGS_EXTRAS)" $(PROJECT)/$(PROVIDER_PATH)/cmd/$(TFGEN))
+.PHONY: tfgen schema tfgen_no_deps tfgen_build_only
 
-upstream:
+upstream: .make/upstream
+# Re-run if the upstream commit or the patches change
+.make/upstream: $(wildcard patches/*) $(wildcard .git/modules/upstream/HEAD)
 ifneq ("$(wildcard upstream)","")
 	./upstream.sh init
 endif
+	@touch $@
+.PHONY: upstream
 
 bin/pulumi-java-gen: .pulumi-java-gen.version
 	pulumictl download-binary -n pulumi-language-java -v v$(shell cat .pulumi-java-gen.version) -r pulumi/pulumi-java
@@ -180,6 +223,7 @@ bin/pulumi-java-gen: .pulumi-java-gen.version
 #
 ci-mgmt: .ci-mgmt.yaml
 	go run github.com/pulumi/ci-mgmt/provider-ci@master generate
+.PHONY: ci-mgmt
 
 # Because some codegen depends on the version of the CLI used, we install a local CLI
 # version pinned to the same version as `provider/go.mod`.
@@ -190,21 +234,20 @@ ci-mgmt: .ci-mgmt.yaml
 .pulumi/bin/pulumi: .pulumi/version
 	@if [ -x .pulumi/bin/pulumi ] && [ "v$$(cat .pulumi/version)" = "$$(.pulumi/bin/pulumi version)" ]; then \
 		echo "pulumi/bin/pulumi version: v$$(cat .pulumi/version)"; \
+		touch $@; \
 	else \
 		curl -fsSL https://get.pulumi.com | \
 			HOME=$(WORKING_DIR) sh -s -- --version "$$(cat .pulumi/version)"; \
 	fi
 
 # Compute the version of Pulumi to use by inspecting the Go dependencies of the provider.
 .pulumi/version: provider/go.mod
-	@mkdir -p .pulumi
-	@cd provider && go list -f "{{slice .Version 1}}" -m github.com/pulumi/pulumi/pkg/v3 | tee ../$@
+	cd provider && go list -f "{{slice .Version 1}}" -m github.com/pulumi/pulumi/pkg/v3 | tee ../$@
 
 # Start debug server for tfgen
 debug_tfgen:
 	dlv  --listen=:2345 --headless=true --api-version=2  exec $(WORKING_DIR)/bin/$(TFGEN) -- schema --out provider/cmd/$(PROVIDER)
-
-.PHONY: development build build_sdks install_go_sdk install_java_sdk install_python_sdk install_sdks only_build build_dotnet build_go build_java build_nodejs build_python clean cleanup help install_dotnet_sdk install_nodejs_sdk install_plugins lint_provider provider provider_no_deps test tfgen upstream ci-mgmt test_provider debug_tfgen tfgen_build_only
+.PHONY: debug_tfgen
 
 # Provider cross-platform build & packaging