fix: empty reponse on provider registry json

.github/workflows/test.yml

@@ -54,3 +54,4 @@ jobs:
           ./dist/release/m1-terraform-provider-helper install hashicorp/github -v v3.0.0 --custom-build-command "go fmt ./... && make fmt && make build"
           ./dist/release/m1-terraform-provider-helper install hashicorp/random -v v3.1.0 --custom-build-command "gofmt -s -w tools && make build"
           ./dist/release/m1-terraform-provider-helper install mongodb/mongodbatlas -v v0.8.2 --custom-build-command "go fmt ./... && make build"
+          ./dist/release/m1-terraform-provider-helper install hashicorp/terraform-provider-mysql -v v1.9.0 --custom-provider-repository-url "https://github.com/hashicorp/terraform-provider-mysql"

README.md

@@ -26,6 +26,7 @@ A CLI to manage the installation and compilation of Terraform providers for an A
     - [Debugging Installation Problems](#debugging-installation-problems)
     - [Terraform Lockfile handling](#terraform-lockfile-handling)
     - [Providing custom build commands](#providing-custom-build-commands)
+    - [Providing custom provider repository](#providing-custom-provider-repository)
     - [Logging](#logging)
     - [Timeouts](#timeouts)
     - [Plugin Directory](#plugin-directory)
@@ -129,6 +130,19 @@ The `install` commands relies on an internal `buildCommands` map to find the cor
 
 Please refer to the documentation of the provider to find out the build command.
 
+### Providing custom provider repository
+You can override the built-in querying mechanism of the terraform registry by using the `--custom-provider-repository-url` flag.
+
+**Explanation**:
+The `install` commands relies on an internal queries the default terraform registry url (which you can also override), to 
+determine the url of the git repository of the desired provider. However, for some providers there is no url 
+as they are, e.g. already *archived*. 
+
+For example for the mysql provider the command would be
+```sh
+m1-terraform-provider-helper install hashicorp/terraform-provider-mysql -v v1.9.0 --custom-provider-repository-url "https://github.com/hashicorp/terraform-provider-mysql"
+```
+
 ### Logging
 
 You can enable additional log output by setting the `TF_HELPER_LOG` environment variable to `info` or `debug` log level.

cmd/install.go

@@ -15,6 +15,8 @@ func installCmd() *cobra.Command {
 
 	var customTerraformRegistryURL string
 
+	var customProviderRepositoryURL string
+
 	cmd := &cobra.Command{
 		Use:   "install [providerName]",
 		Args:  cobra.ExactArgs(1),
@@ -24,6 +26,10 @@ func installCmd() *cobra.Command {
 			a := app.New()
 			a.Init()
 
+			if customProviderRepositoryURL != "" {
+				a.SetCustomProviderRepositoryURL(customProviderRepositoryURL)
+			}
+
 			if customTerraformRegistryURL != "" {
 				a.SetTerraformRegistryURL(customTerraformRegistryURL)
 			}
@@ -40,6 +46,7 @@ func installCmd() *cobra.Command {
 	cmd.Flags().StringVarP(&versionString, "version", "v", "", "The version of the provider")
 	cmd.Flags().StringVar(&customBuildCommand, "custom-build-command", "", "A custom build command to execute instead of the built-in commands")
 	cmd.Flags().StringVarP(&customTerraformRegistryURL, "custom-terraform-registry-url", "u", "", "A custom URL of Terraform registry")
+	cmd.Flags().StringVarP(&customProviderRepositoryURL, "custom-provider-repository-url", "p", "", "A custom URL of the provider repository")
 
 	return cmd
 }

internal/app/app.go

@@ -24,6 +24,7 @@ type Config struct {
 	GoPath                   string
 	ProvidersCacheDir        string
 	TerraformRegistryURL     string
+	ProviderRepositoryURL    string
 	RequestTimeoutInSeconds  int
 }
 
@@ -184,3 +185,7 @@ func (a *App) ListProviders() {
 func (a *App) SetTerraformRegistryURL(url string) {
 	a.Config.TerraformRegistryURL = url
 }
+
+func (a *App) SetCustomProviderRepositoryURL(url string) {
+	a.Config.ProviderRepositoryURL = url
+}

internal/app/install.go

@@ -70,11 +70,21 @@ func executeBashCommand(command string, baseDir string) string {
 }
 
 func (a *App) GetProviderData(providerName string) (Provider, error) {
-	return getProviderData(providerName, a.Config.RequestTimeoutInSeconds, a.Config.TerraformRegistryURL)
+	return getProviderData(providerName, a.Config.RequestTimeoutInSeconds, a.Config.TerraformRegistryURL, a.Config.ProviderRepositoryURL)
 }
 
-func getProviderData(providerName string, requestTimeoutInSeconds int, terraformRegistryURL string) (Provider, error) {
+func getProviderData(providerName string, requestTimeoutInSeconds int, terraformRegistryURL, providerRepositoryURL string) (Provider, error) {
+	if providerRepositoryURL != "" {
+		log.Printf("Using custom provider repository url: '%s'\n", providerRepositoryURL)
+
+		return Provider{
+			Repo:        providerRepositoryURL,
+			Description: "Custom provider url",
+		}, nil
+	}
+
 	url := terraformRegistryURL + providerName
+	log.Printf("Getting provider data from %s\n", url)
 
 	client := &http.Client{Timeout: time.Second * time.Duration(float64(requestTimeoutInSeconds))}
 	ctx := context.Background()
@@ -108,6 +118,10 @@ func getProviderData(providerName string, requestTimeoutInSeconds int, terraform
 		return Provider{}, fmt.Errorf("could not parse JSON %w", err)
 	}
 
+	if data.Repo == "" {
+		return Provider{}, fmt.Errorf("parsed JSON but repo could no be determined. Got json '%s' from url: '%s'", body, url)
+	}
+
 	return data, nil
 }
 
@@ -130,21 +144,23 @@ func checkoutSourceCode(baseDir string, gitURL string, version string) string {
 	var r *git.Repository
 
 	repoDir := extractRepoNameFromURL(gitURL)
+	log.Printf("Extracted repo %s to %s", gitURL, repoDir)
 	fullPath := baseDir + "/" + repoDir
 
 	if !isDirExistent(fullPath) {
-		logrus.Infof("Cloning %s to %s", gitURL, fullPath)
+		log.Printf("Cloning %s to %s", gitURL, fullPath)
 		cloneRepo(gitURL, fullPath)
 	}
 
+	logrus.Infof("Plain open %s", fullPath)
 	r, err := git.PlainOpen(fullPath)
 	CheckIfError(err)
 
 	w, err := r.Worktree()
 	CheckIfError(err)
 
 	// Clean the repository
-	logrus.Infof("Resetting %s and pulling latest changes", fullPath)
+	log.Printf("Resetting %s and pulling latest changes", fullPath)
 
 	err = w.Reset(&git.ResetOptions{Mode: git.HardReset})
 	CheckIfError(err)
@@ -320,7 +336,7 @@ func (a *App) Install(providerName string, version string, customBuildCommand st
 		logrus.Fatalf("Error while trying to get provider data from terraform registry: %v", err.Error())
 	}
 
-	logrus.Infof("Provider data: %v", providerData)
+	log.Printf("Provider data: %v\n", providerData)
 
 	gitRepo := providerData.Repo
 

internal/app/install_test.go

@@ -71,7 +71,7 @@ func TestGetProviderData(t *testing.T) {
 		httpmock.RegisterResponder("GET", "https://registry.terraform.io/v1/providers/"+provider,
 			httpmock.NewStringResponder(200, `{"description": "`+description+`",
 			"source": "`+repo+`"}`))
-		providerData, err := getProviderData(provider, 2, DefaultTerraformRegistryURL)
+		providerData, err := getProviderData(provider, 2, DefaultTerraformRegistryURL, "")
 		if err != nil {
 			t.Errorf("Should not have an error %s ", err)
 		}
@@ -89,7 +89,7 @@ func TestGetProviderData(t *testing.T) {
 		httpmock.RegisterResponder("GET", "https://registry.terraform.io/v1/providers/"+provider,
 			httpmock.NewStringResponder(200, `{"description": "`+description+`",
 			"source": "`+repo+`"}`).Delay(3*time.Second))
-		_, err := getProviderData(provider, 2, DefaultTerraformRegistryURL)
+		_, err := getProviderData(provider, 2, DefaultTerraformRegistryURL, "")
 		if !strings.HasPrefix(err.Error(), "timeout error") {
 			t.Errorf("Expected \"error timeout\" but got %#v", err.Error())
 		}
@@ -103,11 +103,39 @@ func TestGetProviderData(t *testing.T) {
 		defer httpmock.DeactivateAndReset()
 		httpmock.RegisterResponder("GET", "https://registry.terraform.io/v1/providers/"+provider,
 			httpmock.NewStringResponder(200, `{"test:"812}`))
-		_, err := getProviderData(provider, 2, DefaultTerraformRegistryURL)
+		_, err := getProviderData(provider, 2, DefaultTerraformRegistryURL, "")
 		if err == nil {
 			t.Error("Should run into JSON parse error")
 		}
 	})
+
+	t.Run("Should error with mismatched JSON due to non existing provider data", func(t *testing.T) {
+		provider := "hashicorp/vault"
+		httpmock.Activate()
+		defer httpmock.DeactivateAndReset()
+		httpmock.RegisterResponder("GET", "https://registry.terraform.io/v1/providers/"+provider,
+			httpmock.NewStringResponder(200, `{"errors":["Not Found"]}`))
+		_, err := getProviderData(provider, 2, DefaultTerraformRegistryURL, "")
+		if err == nil {
+			t.Error("Should run into empty repo string error")
+		}
+	})
+
+	t.Run("Should fallback to given provider repo url", func(t *testing.T) {
+		provider := "hashicorp/terraform-provider-mysql"
+		customRepo := "https://github.com/hashicorp/terraform-provider-mysql"
+		customdRepoDescription := "Custom provider url"
+		providerData, err := getProviderData(provider, 2, DefaultTerraformRegistryURL, customRepo)
+		if err != nil {
+			t.Errorf("Should not have an error %s ", err)
+		}
+		if providerData.Repo != customRepo {
+			t.Errorf("expected %#v, but got %#v", customRepo, providerData.Repo)
+		}
+		if providerData.Description != customdRepoDescription {
+			t.Errorf("expected %#v, but got %#v", customdRepoDescription, providerData.Description)
+		}
+	})
 }
 
 func TestCheckoutSourceCode(t *testing.T) {