From 925155df06a6d3bf62467fee6f60646b3922a129 Mon Sep 17 00:00:00 2001 From: Pulumi Bot Date: Fri, 22 Nov 2024 12:51:53 +0000 Subject: [PATCH] [internal] Update GitHub Actions workflow files --- .gitignore | 5 +- Makefile | 221 ++++++++++++++++++++++++++++++++--------------------- 2 files changed, 136 insertions(+), 90 deletions(-) diff --git a/.gitignore b/.gitignore index e09e4368..4c359652 100644 --- a/.gitignore +++ b/.gitignore @@ -32,4 +32,7 @@ sdk/java/gradlew.bat sdk/python/venv go.work -go.work.sum \ No newline at end of file +go.work.sum + +# Ignore local build tracking directory +.make diff --git a/Makefile b/Makefile index 8ad87cd5..5c40efb1 100644 --- a/Makefile +++ b/Makefile @@ -27,148 +27,191 @@ LDFLAGS_UPSTREAM_VERSION= LDFLAGS_EXTRAS= LDFLAGS=$(LDFLAGS_PROJ_VERSION) $(LDFLAGS_UPSTREAM_VERSION) $(LDFLAGS_EXTRAS) $(LDFLAGS_STRIP_SYMBOLS) -development: install_plugins provider build_sdks install_sdks +# Create a `.make` directory for tracking targets which don't generate a single file output. This should be ignored by git. +# For targets which either don't generate a single file output, or the output file is committed, we use a "sentinel" +# file within `.make/` to track the staleness of the target and only rebuild when needed. +# For each phony target, we create an internal target with the same name, but prefixed with `.make/` where the work is performed. +# At the end of each internal target we run `@touch $@` to update the file which is the name of the target. -build: install_plugins provider build_sdks install_sdks - -build_sdks: build_nodejs build_python build_dotnet build_go build_java - -install_go_sdk: - -install_java_sdk: - -install_python_sdk: - -install_sdks: install_dotnet_sdk install_python_sdk install_nodejs_sdk install_java_sdk +# Ensure all directories exist before evaluating targets to avoid issues with `touch` creating directories. +_ := $(shell mkdir -p .make bin .pulumi/bin) +# Keep aliases for old targets to ensure backwards compatibility +development: build only_build: build - -build_dotnet: export PULUMI_HOME := $(WORKING_DIR)/.pulumi -build_dotnet: export PATH := $(WORKING_DIR)/.pulumi/bin:$(PATH) -build_dotnet: export PULUMI_CONVERT_EXAMPLES_CACHE_DIR := $(WORKING_DIR)/.pulumi/examples-cache -build_dotnet: upstream - PULUMI_CONVERT=$(PULUMI_CONVERT) PULUMI_DISABLE_AUTOMATIC_PLUGIN_ACQUISITION=$(PULUMI_CONVERT) $(WORKING_DIR)/bin/$(TFGEN) dotnet --out sdk/dotnet/ +# Build the provider and all SDKs and install ready for testing +build: install_plugins provider build_sdks install_sdks +# Creates all generated files which need to be committed +generate: generate_sdks schema +generate_sdks: generate_nodejs generate_python generate_dotnet generate_go generate_java +build_sdks: build_nodejs build_python build_dotnet build_go build_java +install_sdks: install_nodejs_sdk install_python_sdk install_dotnet_sdk install_go_sdk install_java_sdk +.PHONY: development only_build build generate generate_sdks build_sdks install_sdks + +GEN_PULUMI_HOME := $(WORKING_DIR)/.pulumi +GEN_PULUMI_CONVERT_EXAMPLES_CACHE_DIR := $(GEN_PULUMI_HOME)/examples-cache +GEN_ENVS := PULUMI_HOME=$(GEN_PULUMI_HOME) PULUMI_CONVERT_EXAMPLES_CACHE_DIR=$(GEN_PULUMI_CONVERT_EXAMPLES_CACHE_DIR) PULUMI_CONVERT=$(PULUMI_CONVERT) PULUMI_DISABLE_AUTOMATIC_PLUGIN_ACQUISITION=$(PULUMI_CONVERT) + +generate_dotnet: .make/generate_dotnet +build_dotnet: .make/build_dotnet +.make/generate_dotnet: export PATH := $(WORKING_DIR)/.pulumi/bin:$(PATH) +.make/generate_dotnet: bin/$(TFGEN) + $(GEN_ENVS) $(WORKING_DIR)/bin/$(TFGEN) dotnet --out sdk/dotnet/ cd sdk/dotnet/ && \ printf "module fake_dotnet_module // Exclude this directory from Go tools\n\ngo 1.17\n" > go.mod && \ - echo "$(VERSION_GENERIC)" >version.txt && \ - dotnet build - -build_go: export PULUMI_HOME := $(WORKING_DIR)/.pulumi -build_go: export PATH := $(WORKING_DIR)/.pulumi/bin:$(PATH) -build_go: export PULUMI_CONVERT_EXAMPLES_CACHE_DIR := $(WORKING_DIR)/.pulumi/examples-cache -build_go: upstream - PULUMI_CONVERT=$(PULUMI_CONVERT) PULUMI_DISABLE_AUTOMATIC_PLUGIN_ACQUISITION=$(PULUMI_CONVERT) $(WORKING_DIR)/bin/$(TFGEN) go --out sdk/go/ + echo "$(VERSION_GENERIC)" >version.txt + @touch $@ +.make/build_dotnet: .make/generate_dotnet + cd sdk/dotnet/ && dotnet build + @touch $@ +.PHONY: generate_dotnet build_dotnet + +generate_go: .make/generate_go +build_go: .make/build_go +.make/generate_go: export PATH := $(WORKING_DIR)/.pulumi/bin:$(PATH) +.make/generate_go: bin/$(TFGEN) + $(GEN_ENVS) $(WORKING_DIR)/bin/$(TFGEN) go --out sdk/go/ + @touch $@ +.make/build_go: .make/generate_go cd sdk && go list "$$(grep -e "^module" go.mod | cut -d ' ' -f 2)/go/..." | xargs -I {} bash -c 'go build {} && go clean -i {}' - -build_java: PACKAGE_VERSION := $(VERSION_GENERIC) -build_java: export PULUMI_HOME := $(WORKING_DIR)/.pulumi -build_java: export PATH := $(WORKING_DIR)/.pulumi/bin:$(PATH) -build_java: export PULUMI_CONVERT_EXAMPLES_CACHE_DIR := $(WORKING_DIR)/.pulumi/examples-cache -build_java: bin/pulumi-java-gen upstream - $(WORKING_DIR)/bin/$(JAVA_GEN) generate --schema provider/cmd/$(PROVIDER)/schema.json --out sdk/java --build gradle-nexus + @touch $@ +.PHONY: generate_go build_go + +generate_java: .make/generate_java +build_java: .make/build_java +.make/generate_java: export PATH := $(WORKING_DIR)/.pulumi/bin:$(PATH) +.make/generate_java: PACKAGE_VERSION := $(VERSION_GENERIC) +.make/generate_java: bin/pulumi-java-gen .make/schema + PULUMI_HOME=$(GEN_PULUMI_HOME) PULUMI_CONVERT_EXAMPLES_CACHE_DIR=$(GEN_PULUMI_CONVERT_EXAMPLES_CACHE_DIR) bin/$(JAVA_GEN) generate --schema provider/cmd/$(PROVIDER)/schema.json --out sdk/java --build gradle-nexus + printf "module fake_java_module // Exclude this directory from Go tools\n\ngo 1.17\n" > sdk/java/go.mod + @touch $@ +.make/build_java: PACKAGE_VERSION := $(VERSION_GENERIC) +.make/build_java: .make/generate_java cd sdk/java/ && \ - printf "module fake_java_module // Exclude this directory from Go tools\n\ngo 1.17\n" > go.mod && \ gradle --console=plain build && \ gradle --console=plain javadoc - -build_nodejs: export PULUMI_HOME := $(WORKING_DIR)/.pulumi -build_nodejs: export PATH := $(WORKING_DIR)/.pulumi/bin:$(PATH) -build_nodejs: export PULUMI_CONVERT_EXAMPLES_CACHE_DIR := $(WORKING_DIR)/.pulumi/examples-cache -build_nodejs: upstream - PULUMI_CONVERT=$(PULUMI_CONVERT) PULUMI_DISABLE_AUTOMATIC_PLUGIN_ACQUISITION=$(PULUMI_CONVERT) $(WORKING_DIR)/bin/$(TFGEN) nodejs --out sdk/nodejs/ + @touch $@ +.PHONY: generate_java build_java + +generate_nodejs: .make/generate_nodejs +build_nodejs: .make/build_nodejs +.make/generate_nodejs: export PATH := $(WORKING_DIR)/.pulumi/bin:$(PATH) +.make/generate_nodejs: bin/$(TFGEN) + $(GEN_ENVS) $(WORKING_DIR)/bin/$(TFGEN) nodejs --out sdk/nodejs/ + printf "module fake_nodejs_module // Exclude this directory from Go tools\n\ngo 1.17\n" > sdk/nodejs/go.mod + @touch $@ +.make/build_nodejs: .make/generate_nodejs cd sdk/nodejs/ && \ - printf "module fake_nodejs_module // Exclude this directory from Go tools\n\ngo 1.17\n" > go.mod && \ yarn install && \ yarn run tsc && \ cp ../../README.md ../../LICENSE* package.json yarn.lock ./bin/ - -build_python: export PULUMI_HOME := $(WORKING_DIR)/.pulumi -build_python: export PATH := $(WORKING_DIR)/.pulumi/bin:$(PATH) -build_python: export PULUMI_CONVERT_EXAMPLES_CACHE_DIR := $(WORKING_DIR)/.pulumi/examples-cache -build_python: upstream - rm -rf sdk/python/ - PULUMI_CONVERT=$(PULUMI_CONVERT) PULUMI_DISABLE_AUTOMATIC_PLUGIN_ACQUISITION=$(PULUMI_CONVERT) $(WORKING_DIR)/bin/$(TFGEN) python --out sdk/python/ + @touch $@ +.PHONY: generate_nodejs build_nodejs + +generate_python: .make/generate_python +build_python: .make/build_python +.make/generate_python: export PATH := $(WORKING_DIR)/.pulumi/bin:$(PATH) +.make/generate_python: bin/$(TFGEN) + $(GEN_ENVS) $(WORKING_DIR)/bin/$(TFGEN) python --out sdk/python/ + printf "module fake_python_module // Exclude this directory from Go tools\n\ngo 1.17\n" > sdk/python/go.mod + cp README.md sdk/python/ + @touch $@ +.make/build_python: .make/generate_python cd sdk/python/ && \ - printf "module fake_python_module // Exclude this directory from Go tools\n\ngo 1.17\n" > go.mod && \ - cp ../../README.md . && \ rm -rf ./bin/ ../python.bin/ && cp -R . ../python.bin && mv ../python.bin ./bin && \ rm ./bin/go.mod && \ python3 -m venv venv && \ ./venv/bin/python -m pip install build==1.2.1 && \ cd ./bin && \ ../venv/bin/python -m build . + @touch $@ +.PHONY: generate_python build_python clean: rm -rf sdk/{dotnet,nodejs,go,python} + rm -rf bin/* + rm -rf .make/* +.PHONY: clean -cleanup: - rm -r $(WORKING_DIR)/bin - rm -f provider/cmd/$(PROVIDER)/schema.go - -help: - @grep '^[^.#]\+:\s\+.*#' Makefile | \ - sed "s/\(.\+\):\s*\(.*\) #\s*\(.*\)/`printf "\033[93m"`\1`printf "\033[0m"` \3 [\2]/" | \ - expand -t20 - -install_dotnet_sdk: +install_dotnet_sdk: .make/install_dotnet_sdk +.make/install_dotnet_sdk: .make/build_dotnet mkdir -p $(WORKING_DIR)/nuget find . -name '*.nupkg' -print -exec cp -p {} $(WORKING_DIR)/nuget \; - -install_nodejs_sdk: + @touch $@ +install_go_sdk: +install_java_sdk: +install_nodejs_sdk: .make/install_nodejs_sdk +.make/install_nodejs_sdk: .make/build_nodejs yarn link --cwd $(WORKING_DIR)/sdk/nodejs/bin + @touch $@ +install_python_sdk: +.PHONY: install_dotnet_sdk install_go_sdk install_java_sdk install_nodejs_sdk install_python_sdk -install_plugins: export PULUMI_HOME := $(WORKING_DIR)/.pulumi -install_plugins: export PATH := $(WORKING_DIR)/.pulumi/bin:$(PATH) -install_plugins: .pulumi/bin/pulumi +install_plugins: .make/install_plugins +.make/install_plugins: export PULUMI_HOME := $(WORKING_DIR)/.pulumi +.make/install_plugins: export PATH := $(WORKING_DIR)/.pulumi/bin:$(PATH) +.make/install_plugins: .pulumi/bin/pulumi .pulumi/bin/pulumi plugin install resource random 4.8.2 .pulumi/bin/pulumi plugin install resource std 1.6.2 .pulumi/bin/pulumi plugin install converter terraform 1.0.15 + @touch $@ +.PHONY: install_plugins lint_provider: provider cd provider && golangci-lint run --path-prefix provider -c ../.golangci.yml - # `lint_provider.fix` is a utility target meant to be run manually # that will run the linter and fix errors when possible. lint_provider.fix: cd provider && golangci-lint run --path-prefix provider -c ../.golangci.yml --fix +.PHONY: lint_provider lint_provider.fix # `make provider_no_deps` builds the provider binary directly, without ensuring that # `cmd/pulumi-resource-xyz/schema.json` is valid and up to date. # To create a release ready binary, you should use `make provider`. +build_provider_cmd = cd provider && go build $(PULUMI_PROVIDER_BUILD_PARALLELISM) -o $(WORKING_DIR)/bin/$(PROVIDER) -ldflags "$(LDFLAGS)" $(PROJECT)/$(PROVIDER_PATH)/cmd/$(PROVIDER) +provider: bin/$(PROVIDER) provider_no_deps: - (cd provider && go build $(PULUMI_PROVIDER_BUILD_PARALLELISM) -o $(WORKING_DIR)/bin/$(PROVIDER) -ldflags "$(LDFLAGS)" $(PROJECT)/$(PROVIDER_PATH)/cmd/$(PROVIDER)) - -provider: tfgen provider_no_deps + $(call build_provider_cmd) +bin/$(PROVIDER): .make/schema + $(call build_provider_cmd) +.PHONY: provider provider_no_deps test: export PATH := $(WORKING_DIR)/bin:$(PATH) test: cd examples && go test -v -tags=all -parallel $(TESTPARALLELISM) -timeout 2h +.PHONY: test test_provider: - @echo "" - @echo "== test_provider ===================================================================" - @echo "" cd provider && go test -v -short ./... -parallel $(TESTPARALLELISM) - -tfgen: install_plugins upstream tfgen_no_deps - -tfgen_no_deps: export PULUMI_HOME := $(WORKING_DIR)/.pulumi -tfgen_no_deps: export PATH := $(WORKING_DIR)/.pulumi/bin:$(PATH) -tfgen_no_deps: export PULUMI_CONVERT := $(PULUMI_CONVERT) -tfgen_no_deps: export PULUMI_CONVERT_EXAMPLES_CACHE_DIR := $(WORKING_DIR)/.pulumi/examples-cache -tfgen_no_deps: export PULUMI_DISABLE_AUTOMATIC_PLUGIN_ACQUISITION := $(PULUMI_CONVERT) -tfgen_no_deps: export PULUMI_MISSING_DOCS_ERROR := $(PULUMI_MISSING_DOCS_ERROR) -tfgen_no_deps: tfgen_build_only +.PHONY: test_provider + +tfgen: schema +schema: .make/schema +# This does actually have dependencies, but we're keeping it around for backwards compatibility for now +tfgen_no_deps: .make/schema +.make/schema: export PULUMI_HOME := $(WORKING_DIR)/.pulumi +.make/schema: export PATH := $(WORKING_DIR)/.pulumi/bin:$(PATH) +.make/schema: export PULUMI_CONVERT := $(PULUMI_CONVERT) +.make/schema: export PULUMI_CONVERT_EXAMPLES_CACHE_DIR := $(WORKING_DIR)/.pulumi/examples-cache +.make/schema: export PULUMI_DISABLE_AUTOMATIC_PLUGIN_ACQUISITION := $(PULUMI_CONVERT) +.make/schema: export PULUMI_MISSING_DOCS_ERROR := $(PULUMI_MISSING_DOCS_ERROR) +.make/schema: bin/$(TFGEN) provider/resources.go provider/go.mod .make/install_plugins .make/upstream $(WORKING_DIR)/bin/$(TFGEN) schema --out provider/cmd/$(PROVIDER) (cd provider && VERSION=$(VERSION_GENERIC) go generate cmd/$(PROVIDER)/main.go) - -tfgen_build_only: + @touch $@ +tfgen_build_only: bin/$(TFGEN) +bin/$(TFGEN): (cd provider && go build $(PULUMI_PROVIDER_BUILD_PARALLELISM) -o $(WORKING_DIR)/bin/$(TFGEN) -ldflags "$(LDFLAGS_PROJ_VERSION) $(LDFLAGS_EXTRAS)" $(PROJECT)/$(PROVIDER_PATH)/cmd/$(TFGEN)) +.PHONY: tfgen schema tfgen_no_deps tfgen_build_only -upstream: +upstream: .make/upstream +# Re-run if the upstream commit or the patches change +.make/upstream: $(wildcard patches/*) $(wildcard .git/modules/upstream/HEAD) ifneq ("$(wildcard upstream)","") ./upstream.sh init endif + @touch $@ +.PHONY: upstream bin/pulumi-java-gen: .pulumi-java-gen.version pulumictl download-binary -n pulumi-language-java -v v$(shell cat .pulumi-java-gen.version) -r pulumi/pulumi-java @@ -180,6 +223,7 @@ bin/pulumi-java-gen: .pulumi-java-gen.version # ci-mgmt: .ci-mgmt.yaml go run github.com/pulumi/ci-mgmt/provider-ci@master generate +.PHONY: ci-mgmt # Because some codegen depends on the version of the CLI used, we install a local CLI # version pinned to the same version as `provider/go.mod`. @@ -190,6 +234,7 @@ ci-mgmt: .ci-mgmt.yaml .pulumi/bin/pulumi: .pulumi/version @if [ -x .pulumi/bin/pulumi ] && [ "v$$(cat .pulumi/version)" = "$$(.pulumi/bin/pulumi version)" ]; then \ echo "pulumi/bin/pulumi version: v$$(cat .pulumi/version)"; \ + touch $@; \ else \ curl -fsSL https://get.pulumi.com | \ HOME=$(WORKING_DIR) sh -s -- --version "$$(cat .pulumi/version)"; \ @@ -197,14 +242,12 @@ ci-mgmt: .ci-mgmt.yaml # Compute the version of Pulumi to use by inspecting the Go dependencies of the provider. .pulumi/version: provider/go.mod - @mkdir -p .pulumi - @cd provider && go list -f "{{slice .Version 1}}" -m github.com/pulumi/pulumi/pkg/v3 | tee ../$@ + cd provider && go list -f "{{slice .Version 1}}" -m github.com/pulumi/pulumi/pkg/v3 | tee ../$@ # Start debug server for tfgen debug_tfgen: dlv --listen=:2345 --headless=true --api-version=2 exec $(WORKING_DIR)/bin/$(TFGEN) -- schema --out provider/cmd/$(PROVIDER) - -.PHONY: development build build_sdks install_go_sdk install_java_sdk install_python_sdk install_sdks only_build build_dotnet build_go build_java build_nodejs build_python clean cleanup help install_dotnet_sdk install_nodejs_sdk install_plugins lint_provider provider provider_no_deps test tfgen upstream ci-mgmt test_provider debug_tfgen tfgen_build_only +.PHONY: debug_tfgen # Provider cross-platform build & packaging