Template PULUMI_CONVERT rollout (#655)

To be reviewed with pulumi/pulumi-azuread#484

Part of pulumi/pulumi-terraform-bridge#1280

As a pre-requisite to PULUMI_CONVERT being enabled, it's necessary to
pin Pulumi CLI so that examples continue to be deterministic. This PR
borrows targets from pulumi-azure-native that perform CLI pinning.

## Updates

### CLI 

Unlike azure-native, it assumes that the CLI version we want to use to
convert examples is the same as pulumi/pkg version we want to use to
build this provider. This removes a degree of freedom but buys automatic
upgrades - the existing mechanisms that update pulumi/pkg (in
upgrade-provider) should also update this Pulumi CLI version.

### Converter

This is not enforced here but
pulumi/pulumi-azuread#484 adds TF converter
pinning to the CI-mgmt plugins section. This PR makes changes to allow
opting into converter and not just resource plugins. There is currently
no mechanism to upgrade this, but presumably upgrade-provider can build
mechanisms to upgrade all plugins referenced in this config file. Any
such upgrade is likely to trigger the need to regenerate SDKs.

provider-ci/Makefile

@@ -50,8 +50,7 @@ lint-providers/%/repo: $(ACTIONLINT) providers/%/repo
 	cd providers/$*/repo &&	git init && ../../../$(ACTIONLINT) -config-file ../../../actionlint.yml && rm -rf .git
 	@scripts/shellcheck.sh providers/$*/repo
 
-LINT_RULES := $(addprefix lint-, $(PROVIDER_REPOS))
-lint-providers: $(LINT_RULES)
+lint-providers: lint-providers/aws/repo lint-providers/docker/repo lint-providers/cloudflare/repo
 
 test-workflows/%: providers/%/repo
 	rm -r test-workflows/$*

provider-ci/internal/pkg/templates/bridged-provider/Makefile

@@ -19,6 +19,11 @@ WORKING_DIR := $(shell pwd)
 #{{- if .Config.goBuildParallelism }}#
 PULUMI_PROVIDER_BUILD_PARALLELISM ?= -p #{{ .Config.goBuildParallelism }}#
 #{{- end }}#
+#{{- if .Config.pulumiConvert }}#
+PULUMI_CONVERT := 1
+#{{- else }}#
+PULUMI_CONVERT := 0
+#{{- end }}#
 
 development: install_plugins provider build_sdks install_sdks
 
@@ -106,10 +111,9 @@ install_dotnet_sdk:
 install_nodejs_sdk:
 	yarn link --cwd $(WORKING_DIR)/sdk/nodejs/bin
 
-install_plugins:
-	[ -x "$(shell command -v pulumi 2>/dev/null)" ] || curl -fsSL https://get.pulumi.com | sh
+install_plugins: .pulumi/bin/pulumi
 	#{{- range .Config.plugins }}#
-	pulumi plugin install resource #{{ .name }}# #{{ .version }}#
+	.pulumi/bin/pulumi plugin install #{{ or .kind "resource" }}# #{{ .name }}# #{{ .version }}#
 	#{{- end }}#
 
 lint_provider: provider
@@ -123,7 +127,7 @@ test:
 
 tfgen: install_plugins#{{ if eq .Config.team "ecosystem" }}# upstream#{{ end }}##{{ if .Config.docsCmd }}# docs#{{ end }}#
 	(cd provider && go build $(PULUMI_PROVIDER_BUILD_PARALLELISM) -o $(WORKING_DIR)/bin/$(TFGEN) -ldflags "-X $(PROJECT)/$(VERSION_PATH)=$(VERSION)" $(PROJECT)/$(PROVIDER_PATH)/cmd/$(TFGEN))
-	$(WORKING_DIR)/bin/$(TFGEN) schema --out provider/cmd/$(PROVIDER)
+	PATH=${PWD}/.pulumi/bin:$$PATH PULUMI_CONVERT=$(PULUMI_CONVERT) $(WORKING_DIR)/bin/$(TFGEN) schema --out provider/cmd/$(PROVIDER)
 	(cd provider && VERSION=$(VERSION) go generate cmd/$(PROVIDER)/main.go)
 
 #{{- if eq .Config.team "ecosystem" }}#
@@ -167,5 +171,13 @@ ci-mgmt: .ci-mgmt.yaml
 		--template bridged-provider \
 		--config $<
 
+.pulumi/bin/pulumi: HOME := $(WORKING_DIR)
+.pulumi/bin/pulumi: .pulumi/version
+	curl -fsSL https://get.pulumi.com | sh -s -- --version $(cat .pulumi/version)
+
+# Compute the version of Pulumi to use by inspecting the Go dependencies of the provider.
+.pulumi/version:
+	@mkdir -p .pulumi
+	@cd provider && go list -f "{{slice .Version 1}}" -m github.com/pulumi/pulumi/pkg/v3 | tee ../$@
 
 .PHONY: development build build_sdks install_go_sdk install_java_sdk install_python_sdk install_sdks only_build build_dotnet build_go build_java build_nodejs build_python clean cleanup#{{ if .Config.docsCmd }}# docs#{{end}}# help install_dotnet_sdk install_nodejs_sdk install_plugins lint_provider provider test tfgen#{{ if eq .Config.team "ecosystem" }}# upstream upstream.finalize upstream.rebase#{{end}}# ci-mgmt

provider-ci/test-workflows/aws/Makefile

@@ -13,6 +13,7 @@ JAVA_GEN_VERSION := v0.9.5
 TESTPARALLELISM := 10
 WORKING_DIR := $(shell pwd)
 PULUMI_PROVIDER_BUILD_PARALLELISM ?= -p 2
+PULUMI_CONVERT := 0
 
 development: install_plugins provider build_sdks install_sdks
 
@@ -94,14 +95,13 @@ install_dotnet_sdk:
 install_nodejs_sdk:
 	yarn link --cwd $(WORKING_DIR)/sdk/nodejs/bin
 
-install_plugins:
-	[ -x "$(shell command -v pulumi 2>/dev/null)" ] || curl -fsSL https://get.pulumi.com | sh
-	pulumi plugin install resource archive 0.0.1
-	pulumi plugin install resource tls 4.10.0
-	pulumi plugin install resource github 4.10.0
-	pulumi plugin install resource kubernetes 3.17.0
-	pulumi plugin install resource random 4.8.2
-	pulumi plugin install resource github 5.14.0
+install_plugins: .pulumi/bin/pulumi
+	.pulumi/bin/pulumi plugin install resource archive 0.0.1
+	.pulumi/bin/pulumi plugin install resource tls 4.10.0
+	.pulumi/bin/pulumi plugin install resource github 4.10.0
+	.pulumi/bin/pulumi plugin install resource kubernetes 3.17.0
+	.pulumi/bin/pulumi plugin install resource random 4.8.2
+	.pulumi/bin/pulumi plugin install resource github 5.14.0
 
 lint_provider: provider
 	cd provider && golangci-lint run -c ../.golangci.yml
@@ -114,7 +114,7 @@ test:
 
 tfgen: install_plugins upstream
 	(cd provider && go build $(PULUMI_PROVIDER_BUILD_PARALLELISM) -o $(WORKING_DIR)/bin/$(TFGEN) -ldflags "-X $(PROJECT)/$(VERSION_PATH)=$(VERSION)" $(PROJECT)/$(PROVIDER_PATH)/cmd/$(TFGEN))
-	$(WORKING_DIR)/bin/$(TFGEN) schema --out provider/cmd/$(PROVIDER)
+	PATH=${PWD}/.pulumi/bin:$$PATH PULUMI_CONVERT=$(PULUMI_CONVERT) $(WORKING_DIR)/bin/$(TFGEN) schema --out provider/cmd/$(PROVIDER)
 	(cd provider && VERSION=$(VERSION) go generate cmd/$(PROVIDER)/main.go)
 
 upstream:
@@ -151,5 +151,13 @@ ci-mgmt: .ci-mgmt.yaml
 		--template bridged-provider \
 		--config $<
 
+.pulumi/bin/pulumi: HOME := $(WORKING_DIR)
+.pulumi/bin/pulumi: .pulumi/version
+	curl -fsSL https://get.pulumi.com | sh -s -- --version $(cat .pulumi/version)
+
+# Compute the version of Pulumi to use by inspecting the Go dependencies of the provider.
+.pulumi/version:
+	@mkdir -p .pulumi
+	@cd provider && go list -f "{{slice .Version 1}}" -m github.com/pulumi/pulumi/pkg/v3 | tee ../$@
 
 .PHONY: development build build_sdks install_go_sdk install_java_sdk install_python_sdk install_sdks only_build build_dotnet build_go build_java build_nodejs build_python clean cleanup help install_dotnet_sdk install_nodejs_sdk install_plugins lint_provider provider test tfgen upstream upstream.finalize upstream.rebase ci-mgmt

provider-ci/test-workflows/cloudflare/Makefile

@@ -12,6 +12,7 @@ JAVA_GEN := pulumi-java-gen
 JAVA_GEN_VERSION := v0.9.3
 TESTPARALLELISM := 10
 WORKING_DIR := $(shell pwd)
+PULUMI_CONVERT := 0
 
 development: install_plugins provider build_sdks install_sdks
 
@@ -93,10 +94,9 @@ install_dotnet_sdk:
 install_nodejs_sdk:
 	yarn link --cwd $(WORKING_DIR)/sdk/nodejs/bin
 
-install_plugins:
-	[ -x "$(shell command -v pulumi 2>/dev/null)" ] || curl -fsSL https://get.pulumi.com | sh
-	pulumi plugin install resource gcp 5.0.0
-	pulumi plugin install resource tls 4.0.0
+install_plugins: .pulumi/bin/pulumi
+	.pulumi/bin/pulumi plugin install resource gcp 5.0.0
+	.pulumi/bin/pulumi plugin install resource tls 4.0.0
 
 lint_provider: provider
 	cd provider && golangci-lint run -c ../.golangci.yml
@@ -109,7 +109,7 @@ test:
 
 tfgen: install_plugins upstream
 	(cd provider && go build $(PULUMI_PROVIDER_BUILD_PARALLELISM) -o $(WORKING_DIR)/bin/$(TFGEN) -ldflags "-X $(PROJECT)/$(VERSION_PATH)=$(VERSION)" $(PROJECT)/$(PROVIDER_PATH)/cmd/$(TFGEN))
-	$(WORKING_DIR)/bin/$(TFGEN) schema --out provider/cmd/$(PROVIDER)
+	PATH=${PWD}/.pulumi/bin:$$PATH PULUMI_CONVERT=$(PULUMI_CONVERT) $(WORKING_DIR)/bin/$(TFGEN) schema --out provider/cmd/$(PROVIDER)
 	(cd provider && VERSION=$(VERSION) go generate cmd/$(PROVIDER)/main.go)
 
 upstream:
@@ -139,5 +139,13 @@ ci-mgmt: .ci-mgmt.yaml
 		--template bridged-provider \
 		--config $<
 
+.pulumi/bin/pulumi: HOME := $(WORKING_DIR)
+.pulumi/bin/pulumi: .pulumi/version
+	curl -fsSL https://get.pulumi.com | sh -s -- --version $(cat .pulumi/version)
+
+# Compute the version of Pulumi to use by inspecting the Go dependencies of the provider.
+.pulumi/version:
+	@mkdir -p .pulumi
+	@cd provider && go list -f "{{slice .Version 1}}" -m github.com/pulumi/pulumi/pkg/v3 | tee ../$@
 
 .PHONY: development build build_sdks install_go_sdk install_java_sdk install_python_sdk install_sdks only_build build_dotnet build_go build_java build_nodejs build_python clean cleanup help install_dotnet_sdk install_nodejs_sdk install_plugins lint_provider provider test tfgen upstream upstream.finalize upstream.rebase ci-mgmt

provider-ci/test-workflows/docker/Makefile

@@ -12,6 +12,7 @@ JAVA_GEN := pulumi-java-gen
 JAVA_GEN_VERSION := v0.9.7
 TESTPARALLELISM := 10
 WORKING_DIR := $(shell pwd)
+PULUMI_CONVERT := 0
 
 development: install_plugins provider build_sdks install_sdks
 
@@ -96,9 +97,8 @@ install_dotnet_sdk:
 install_nodejs_sdk:
 	yarn link --cwd $(WORKING_DIR)/sdk/nodejs/bin
 
-install_plugins:
-	[ -x "$(shell command -v pulumi 2>/dev/null)" ] || curl -fsSL https://get.pulumi.com | sh
-	pulumi plugin install resource aws 5.37.0
+install_plugins: .pulumi/bin/pulumi
+	.pulumi/bin/pulumi plugin install resource aws 5.37.0
 
 lint_provider: provider
 	cd provider && golangci-lint run -c ../.golangci.yml
@@ -111,7 +111,7 @@ test:
 
 tfgen: install_plugins upstream docs
 	(cd provider && go build $(PULUMI_PROVIDER_BUILD_PARALLELISM) -o $(WORKING_DIR)/bin/$(TFGEN) -ldflags "-X $(PROJECT)/$(VERSION_PATH)=$(VERSION)" $(PROJECT)/$(PROVIDER_PATH)/cmd/$(TFGEN))
-	$(WORKING_DIR)/bin/$(TFGEN) schema --out provider/cmd/$(PROVIDER)
+	PATH=${PWD}/.pulumi/bin:$$PATH PULUMI_CONVERT=$(PULUMI_CONVERT) $(WORKING_DIR)/bin/$(TFGEN) schema --out provider/cmd/$(PROVIDER)
 	(cd provider && VERSION=$(VERSION) go generate cmd/$(PROVIDER)/main.go)
 
 upstream:
@@ -141,5 +141,13 @@ ci-mgmt: .ci-mgmt.yaml
 		--template bridged-provider \
 		--config $<
 
+.pulumi/bin/pulumi: HOME := $(WORKING_DIR)
+.pulumi/bin/pulumi: .pulumi/version
+	curl -fsSL https://get.pulumi.com | sh -s -- --version $(cat .pulumi/version)
+
+# Compute the version of Pulumi to use by inspecting the Go dependencies of the provider.
+.pulumi/version:
+	@mkdir -p .pulumi
+	@cd provider && go list -f "{{slice .Version 1}}" -m github.com/pulumi/pulumi/pkg/v3 | tee ../$@
 
 .PHONY: development build build_sdks install_go_sdk install_java_sdk install_python_sdk install_sdks only_build build_dotnet build_go build_java build_nodejs build_python clean cleanup docs help install_dotnet_sdk install_nodejs_sdk install_plugins lint_provider provider test tfgen upstream upstream.finalize upstream.rebase ci-mgmt