From 9c3ba6e5ac5c73a59d44188a9a0ec53a8f0bf028 Mon Sep 17 00:00:00 2001 From: Manuel Vogel Date: Tue, 27 Jun 2023 12:07:22 +0200 Subject: [PATCH 1/6] test: for empty reponse on provider registry json Signed-off-by: Manuel Vogel --- internal/app/install.go | 9 ++++++--- internal/app/install_test.go | 12 ++++++++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/internal/app/install.go b/internal/app/install.go index eab13f8..d8e8daa 100644 --- a/internal/app/install.go +++ b/internal/app/install.go @@ -75,6 +75,7 @@ func (a *App) GetProviderData(providerName string) (Provider, error) { func getProviderData(providerName string, requestTimeoutInSeconds int, terraformRegistryURL string) (Provider, error) { url := terraformRegistryURL + providerName + fmt.Printf("Getting provider data from %s\n", url) client := &http.Client{Timeout: time.Second * time.Duration(float64(requestTimeoutInSeconds))} ctx := context.Background() @@ -130,13 +131,15 @@ func checkoutSourceCode(baseDir string, gitURL string, version string) string { var r *git.Repository repoDir := extractRepoNameFromURL(gitURL) + fmt.Printf("Extracted repo %s to %s", gitURL, repoDir) fullPath := baseDir + "/" + repoDir if !isDirExistent(fullPath) { - logrus.Infof("Cloning %s to %s", gitURL, fullPath) + log.Printf("Cloning %s to %s", gitURL, fullPath) cloneRepo(gitURL, fullPath) } + logrus.Infof("Plain open %s", fullPath) r, err := git.PlainOpen(fullPath) CheckIfError(err) @@ -144,7 +147,7 @@ func checkoutSourceCode(baseDir string, gitURL string, version string) string { CheckIfError(err) // Clean the repository - logrus.Infof("Resetting %s and pulling latest changes", fullPath) + log.Printf("Resetting %s and pulling latest changes", fullPath) err = w.Reset(&git.ResetOptions{Mode: git.HardReset}) CheckIfError(err) @@ -320,7 +323,7 @@ func (a *App) Install(providerName string, version string, customBuildCommand st logrus.Fatalf("Error while trying to get provider data from terraform registry: %v", err.Error()) } - logrus.Infof("Provider data: %v", providerData) + fmt.Printf("Provider data: %v\n", providerData) gitRepo := providerData.Repo diff --git a/internal/app/install_test.go b/internal/app/install_test.go index 7c2dd5f..6defee8 100644 --- a/internal/app/install_test.go +++ b/internal/app/install_test.go @@ -108,6 +108,18 @@ func TestGetProviderData(t *testing.T) { t.Error("Should run into JSON parse error") } }) + + t.Run("Should error with mismatched JSON due to non existing provider data", func(t *testing.T) { + provider := "hashicorp/vault" + httpmock.Activate() + defer httpmock.DeactivateAndReset() + httpmock.RegisterResponder("GET", "https://registry.terraform.io/v1/providers/"+provider, + httpmock.NewStringResponder(200, `{"errors":["Not Found"]}`)) + _, err := getProviderData(provider, 2, DefaultTerraformRegistryURL) + if err == nil { + t.Error("Should run into JSON parse error") + } + }) } func TestCheckoutSourceCode(t *testing.T) { From 735476e49bfa0f4b50d5851396856fe948f62ea0 Mon Sep 17 00:00:00 2001 From: Manuel Vogel Date: Tue, 27 Jun 2023 14:54:07 +0200 Subject: [PATCH 2/6] fix: test for error on querying repo url Signed-off-by: Manuel Vogel --- internal/app/install.go | 4 ++++ internal/app/install_test.go | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/internal/app/install.go b/internal/app/install.go index d8e8daa..7ea6f63 100644 --- a/internal/app/install.go +++ b/internal/app/install.go @@ -109,6 +109,10 @@ func getProviderData(providerName string, requestTimeoutInSeconds int, terraform return Provider{}, fmt.Errorf("could not parse JSON %w", err) } + if data.Repo == "" { + return Provider{}, fmt.Errorf("parsed JSON but repo could no be determined. Got json '%s' from url: '%s'", body, url) + } + return data, nil } diff --git a/internal/app/install_test.go b/internal/app/install_test.go index 6defee8..9adae43 100644 --- a/internal/app/install_test.go +++ b/internal/app/install_test.go @@ -117,7 +117,7 @@ func TestGetProviderData(t *testing.T) { httpmock.NewStringResponder(200, `{"errors":["Not Found"]}`)) _, err := getProviderData(provider, 2, DefaultTerraformRegistryURL) if err == nil { - t.Error("Should run into JSON parse error") + t.Error("Should run into empty repo string error") } }) } From e0989a42beda49b7eb6e36a81451169e3cd5b81a Mon Sep 17 00:00:00 2001 From: Manuel Vogel Date: Tue, 27 Jun 2023 15:01:35 +0200 Subject: [PATCH 3/6] fix: linter fmt for log Signed-off-by: Manuel Vogel --- internal/app/install.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/app/install.go b/internal/app/install.go index 7ea6f63..cd95435 100644 --- a/internal/app/install.go +++ b/internal/app/install.go @@ -75,7 +75,7 @@ func (a *App) GetProviderData(providerName string) (Provider, error) { func getProviderData(providerName string, requestTimeoutInSeconds int, terraformRegistryURL string) (Provider, error) { url := terraformRegistryURL + providerName - fmt.Printf("Getting provider data from %s\n", url) + log.Printf("Getting provider data from %s\n", url) client := &http.Client{Timeout: time.Second * time.Duration(float64(requestTimeoutInSeconds))} ctx := context.Background() @@ -135,7 +135,7 @@ func checkoutSourceCode(baseDir string, gitURL string, version string) string { var r *git.Repository repoDir := extractRepoNameFromURL(gitURL) - fmt.Printf("Extracted repo %s to %s", gitURL, repoDir) + log.Printf("Extracted repo %s to %s", gitURL, repoDir) fullPath := baseDir + "/" + repoDir if !isDirExistent(fullPath) { @@ -327,7 +327,7 @@ func (a *App) Install(providerName string, version string, customBuildCommand st logrus.Fatalf("Error while trying to get provider data from terraform registry: %v", err.Error()) } - fmt.Printf("Provider data: %v\n", providerData) + log.Printf("Provider data: %v\n", providerData) gitRepo := providerData.Repo From 21de9c99c373bdda2ce6f12b1013f9e6820e232c Mon Sep 17 00:00:00 2001 From: Manuel Vogel Date: Tue, 27 Jun 2023 16:20:50 +0200 Subject: [PATCH 4/6] feat: add new custom-provider-repository-url flag Signed-off-by: Manuel Vogel --- .github/workflows/test.yml | 1 + cmd/install.go | 7 +++++++ internal/app/app.go | 5 +++++ internal/app/install.go | 12 ++++++++++-- internal/app/install_test.go | 24 ++++++++++++++++++++---- 5 files changed, 43 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bdb9b7d..07bc4f7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -54,3 +54,4 @@ jobs: ./dist/release/m1-terraform-provider-helper install hashicorp/github -v v3.0.0 --custom-build-command "go fmt ./... && make fmt && make build" ./dist/release/m1-terraform-provider-helper install hashicorp/random -v v3.1.0 --custom-build-command "gofmt -s -w tools && make build" ./dist/release/m1-terraform-provider-helper install mongodb/mongodbatlas -v v0.8.2 --custom-build-command "go fmt ./... && make build" + ./dist/release/m1-terraform-provider-helper install hashicorp/terraform-provider-mysql -v v1.9.0 --custom-provider-repository-url "https://github.com/hashicorp/terraform-provider-mysql" diff --git a/cmd/install.go b/cmd/install.go index 1839d00..efae547 100644 --- a/cmd/install.go +++ b/cmd/install.go @@ -15,6 +15,8 @@ func installCmd() *cobra.Command { var customTerraformRegistryURL string + var customProviderRepositoryURL string + cmd := &cobra.Command{ Use: "install [providerName]", Args: cobra.ExactArgs(1), @@ -24,6 +26,10 @@ func installCmd() *cobra.Command { a := app.New() a.Init() + if customProviderRepositoryURL != "" { + a.SetCustomProviderRepositoryURL(customProviderRepositoryURL) + } + if customTerraformRegistryURL != "" { a.SetTerraformRegistryURL(customTerraformRegistryURL) } @@ -40,6 +46,7 @@ func installCmd() *cobra.Command { cmd.Flags().StringVarP(&versionString, "version", "v", "", "The version of the provider") cmd.Flags().StringVar(&customBuildCommand, "custom-build-command", "", "A custom build command to execute instead of the built-in commands") cmd.Flags().StringVarP(&customTerraformRegistryURL, "custom-terraform-registry-url", "u", "", "A custom URL of Terraform registry") + cmd.Flags().StringVarP(&customProviderRepositoryURL, "custom-provider-repository-url", "p", "", "A custom URL of the provider repository") return cmd } diff --git a/internal/app/app.go b/internal/app/app.go index 2bd760e..125782d 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -24,6 +24,7 @@ type Config struct { GoPath string ProvidersCacheDir string TerraformRegistryURL string + ProviderRepositoryURL string RequestTimeoutInSeconds int } @@ -184,3 +185,7 @@ func (a *App) ListProviders() { func (a *App) SetTerraformRegistryURL(url string) { a.Config.TerraformRegistryURL = url } + +func (a *App) SetCustomProviderRepositoryURL(url string) { + a.Config.ProviderRepositoryURL = url +} diff --git a/internal/app/install.go b/internal/app/install.go index cd95435..9091a06 100644 --- a/internal/app/install.go +++ b/internal/app/install.go @@ -70,10 +70,18 @@ func executeBashCommand(command string, baseDir string) string { } func (a *App) GetProviderData(providerName string) (Provider, error) { - return getProviderData(providerName, a.Config.RequestTimeoutInSeconds, a.Config.TerraformRegistryURL) + return getProviderData(providerName, a.Config.RequestTimeoutInSeconds, a.Config.TerraformRegistryURL, a.Config.ProviderRepositoryURL) } -func getProviderData(providerName string, requestTimeoutInSeconds int, terraformRegistryURL string) (Provider, error) { +func getProviderData(providerName string, requestTimeoutInSeconds int, terraformRegistryURL, providerRepositoryURL string) (Provider, error) { + if providerRepositoryURL != "" { + log.Printf("Using custom provider repository url: '%s'\n", providerRepositoryURL) + return Provider{ + Repo: providerRepositoryURL, + Description: "Custom provider url", + }, nil + } + url := terraformRegistryURL + providerName log.Printf("Getting provider data from %s\n", url) diff --git a/internal/app/install_test.go b/internal/app/install_test.go index 9adae43..30b8042 100644 --- a/internal/app/install_test.go +++ b/internal/app/install_test.go @@ -71,7 +71,7 @@ func TestGetProviderData(t *testing.T) { httpmock.RegisterResponder("GET", "https://registry.terraform.io/v1/providers/"+provider, httpmock.NewStringResponder(200, `{"description": "`+description+`", "source": "`+repo+`"}`)) - providerData, err := getProviderData(provider, 2, DefaultTerraformRegistryURL) + providerData, err := getProviderData(provider, 2, DefaultTerraformRegistryURL, "") if err != nil { t.Errorf("Should not have an error %s ", err) } @@ -89,7 +89,7 @@ func TestGetProviderData(t *testing.T) { httpmock.RegisterResponder("GET", "https://registry.terraform.io/v1/providers/"+provider, httpmock.NewStringResponder(200, `{"description": "`+description+`", "source": "`+repo+`"}`).Delay(3*time.Second)) - _, err := getProviderData(provider, 2, DefaultTerraformRegistryURL) + _, err := getProviderData(provider, 2, DefaultTerraformRegistryURL, "") if !strings.HasPrefix(err.Error(), "timeout error") { t.Errorf("Expected \"error timeout\" but got %#v", err.Error()) } @@ -103,7 +103,7 @@ func TestGetProviderData(t *testing.T) { defer httpmock.DeactivateAndReset() httpmock.RegisterResponder("GET", "https://registry.terraform.io/v1/providers/"+provider, httpmock.NewStringResponder(200, `{"test:"812}`)) - _, err := getProviderData(provider, 2, DefaultTerraformRegistryURL) + _, err := getProviderData(provider, 2, DefaultTerraformRegistryURL, "") if err == nil { t.Error("Should run into JSON parse error") } @@ -115,11 +115,27 @@ func TestGetProviderData(t *testing.T) { defer httpmock.DeactivateAndReset() httpmock.RegisterResponder("GET", "https://registry.terraform.io/v1/providers/"+provider, httpmock.NewStringResponder(200, `{"errors":["Not Found"]}`)) - _, err := getProviderData(provider, 2, DefaultTerraformRegistryURL) + _, err := getProviderData(provider, 2, DefaultTerraformRegistryURL, "") if err == nil { t.Error("Should run into empty repo string error") } }) + + t.Run("Should fallback to given provider repo url", func(t *testing.T) { + provider := "hashicorp/terraform-provider-mysql" + customRepo := "https://github.com/hashicorp/terraform-provider-mysql" + customdRepoDescription := "Custom provider url" + providerData, err := getProviderData(provider, 2, DefaultTerraformRegistryURL, customRepo) + if err != nil { + t.Errorf("Should not have an error %s ", err) + } + if providerData.Repo != customRepo { + t.Errorf("expected %#v, but got %#v", customRepo, providerData.Repo) + } + if providerData.Description != customdRepoDescription { + t.Errorf("expected %#v, but got %#v", customdRepoDescription, providerData.Description) + } + }) } func TestCheckoutSourceCode(t *testing.T) { From 4d104ee3523d32765bbae36a080a40153aa57608 Mon Sep 17 00:00:00 2001 From: Manuel Vogel Date: Tue, 27 Jun 2023 16:27:35 +0200 Subject: [PATCH 5/6] fix: linter Signed-off-by: Manuel Vogel --- internal/app/install.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/app/install.go b/internal/app/install.go index 9091a06..6341f34 100644 --- a/internal/app/install.go +++ b/internal/app/install.go @@ -76,6 +76,7 @@ func (a *App) GetProviderData(providerName string) (Provider, error) { func getProviderData(providerName string, requestTimeoutInSeconds int, terraformRegistryURL, providerRepositoryURL string) (Provider, error) { if providerRepositoryURL != "" { log.Printf("Using custom provider repository url: '%s'\n", providerRepositoryURL) + return Provider{ Repo: providerRepositoryURL, Description: "Custom provider url", From 4be10eac0e522f7d318f76c5a4cb19ead92de4cc Mon Sep 17 00:00:00 2001 From: Manuel Vogel Date: Thu, 29 Jun 2023 08:56:40 +0200 Subject: [PATCH 6/6] docs(readme): add explanation --- README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.md b/README.md index 7a8b1d7..33f17df 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ A CLI to manage the installation and compilation of Terraform providers for an A - [Debugging Installation Problems](#debugging-installation-problems) - [Terraform Lockfile handling](#terraform-lockfile-handling) - [Providing custom build commands](#providing-custom-build-commands) + - [Providing custom provider repository](#providing-custom-provider-repository) - [Logging](#logging) - [Timeouts](#timeouts) - [Plugin Directory](#plugin-directory) @@ -129,6 +130,19 @@ The `install` commands relies on an internal `buildCommands` map to find the cor Please refer to the documentation of the provider to find out the build command. +### Providing custom provider repository +You can override the built-in querying mechanism of the terraform registry by using the `--custom-provider-repository-url` flag. + +**Explanation**: +The `install` commands relies on an internal queries the default terraform registry url (which you can also override), to +determine the url of the git repository of the desired provider. However, for some providers there is no url +as they are, e.g. already *archived*. + +For example for the mysql provider the command would be +```sh +m1-terraform-provider-helper install hashicorp/terraform-provider-mysql -v v1.9.0 --custom-provider-repository-url "https://github.com/hashicorp/terraform-provider-mysql" +``` + ### Logging You can enable additional log output by setting the `TF_HELPER_LOG` environment variable to `info` or `debug` log level.