Add option to filter edit rules by location in docs conversion pipeline and apply for installation docs

pkg/tfbridge/info/info.go

@@ -233,6 +233,13 @@ type DocsPath struct {
 	TfToken string
 }
 
+type EditPhase int
+
+const (
+	PreCodeTranslation EditPhase = iota
+	PostCodeTranslation
+)
+
 type DocsEdit struct {
 	// The file name at which this rule applies. File names are matched via filepath.Match.
 	//
@@ -251,7 +258,8 @@ type DocsEdit struct {
 	// The function that performs the edit on the file bytes.
 	//
 	// Must not be nil.
-	Edit func(path string, content []byte) ([]byte, error)
+	Edit  func(path string, content []byte) ([]byte, error)
+	Phase EditPhase
 }
 
 // TFProviderLicense is a way to be able to pass a license type for the upstream Terraform provider.

pkg/tfgen/docs.go

@@ -44,6 +44,7 @@ import (
 
 	"github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tf2pulumi/convert"
 	"github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfbridge"
+	"github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfbridge/info"
 	"github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfgen/parse"
 )
 
@@ -536,7 +537,7 @@ func (p *tfMarkdownParser) parse(tfMarkdown []byte) (entityDocs, error) {
 		Attributes: make(map[string]string),
 	}
 	var err error
-	tfMarkdown, err = p.editRules.apply(p.markdownFileName, tfMarkdown)
+	tfMarkdown, err = p.editRules.apply(p.markdownFileName, tfMarkdown, info.PreCodeTranslation)
 	if err != nil {
 		return entityDocs{}, fmt.Errorf("file %s: %w", p.markdownFileName, err)
 	}

pkg/tfgen/edit_rules.go

@@ -21,6 +21,7 @@ import (
 	"regexp"
 
 	"github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfbridge"
+	"github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfbridge/info"
 )
 
 func defaultEditRules() editRules {
@@ -29,24 +30,52 @@ func defaultEditRules() editRules {
 		boundedReplace("[tT]erraform [pP]lan", "pulumi preview"),
 		// Replace content such as " Terraform Apply." with " pulumi up."
 		boundedReplace("[tT]erraform [aA]pply", "pulumi up"),
-		reReplace(`"([mM])ade (by|with) [tT]erraform"`, `"Made $2 Pulumi"`),
+		reReplace(`"([mM])ade (by|with) [tT]erraform"`, `"Made $2 Pulumi"`, info.PreCodeTranslation),
 		// A markdown link that has terraform in the link component.
-		reReplace(`\[([^\]]*)\]\([^\)]*terraform([^\)]*)\)`, "$1"),
+		reReplace(`\[([^\]]*)\]\([^\)]*terraform([^\)]*)\)`, "$1", info.PreCodeTranslation),
 		fixupImports(),
 		// Replace content such as "[email protected]" with "[email protected]"
-		reReplace("@hashicorp.com", "@example.com"),
+		reReplace("@hashicorp.com", "@example.com", info.PreCodeTranslation),
+
+		// The following edit rules may be applied after translating the code sections in a document.
+		// Its primary use case is for the docs translation approach spearheaded in installation_docs.go.
+		// These edit rules allow us to safely transform certain strings that we would otherwise need in the
+		// code translation or nested type discovery process.
+		// These rules are currently only called when generating installation docs.
+		skipSectionHeadersEdit(),
+		removeTfVersionMentions(),
+		//Replace "providers.tf" with "Pulumi.yaml"
+		reReplace(`providers.tf`, `Pulumi.yaml`, info.PostCodeTranslation),
+		reReplace(`terraform init`, `pulumi up`, info.PostCodeTranslation),
+		// Replace all " T/terraform" with " P/pulumi"
+		reReplace(`Terraform`, `Pulumi`, info.PostCodeTranslation),
+		reReplace(`terraform`, `pulumi`, info.PostCodeTranslation),
+		// Replace all "H/hashicorp" strings
+		reReplace(`Hashicorp`, `Pulumi`, info.PostCodeTranslation),
+		reReplace(`hashicorp`, `pulumi`, info.PostCodeTranslation),
+		// Reformat certain headers
+		reReplace(`The following arguments are supported`,
+			`The following configuration inputs are supported`, info.PostCodeTranslation),
+		reReplace(`Argument Reference`,
+			`Configuration Reference`, info.PostCodeTranslation),
+		reReplace(`Schema`,
+			`Configuration Reference`, info.PostCodeTranslation),
+		reReplace("### Optional\n", "", info.PostCodeTranslation),
+		reReplace(`block contains the following arguments`,
+			`input has the following nested fields`, info.PostCodeTranslation),
+		reReplace(`provider block`, `provider configuration`, info.PostCodeTranslation),
 	}
 }
 
 type editRules []tfbridge.DocsEdit
 
-func (rr editRules) apply(fileName string, contents []byte) ([]byte, error) {
+func (rr editRules) apply(fileName string, contents []byte, phase info.EditPhase) ([]byte, error) {
 	for _, rule := range rr {
 		match, err := filepath.Match(rule.Path, fileName)
 		if err != nil {
 			return nil, fmt.Errorf("invalid glob: %q: %w", rule.Path, err)
 		}
-		if !match {
+		if !match || (rule.Phase != phase) {
 			continue
 		}
 		contents, err = rule.Edit(fileName, contents)
@@ -84,14 +113,15 @@ func boundedReplace(from, to string) tfbridge.DocsEdit {
 }
 
 // reReplace creates a regex based replace.
-func reReplace(from, to string) tfbridge.DocsEdit {
+func reReplace(from, to string, phase info.EditPhase) tfbridge.DocsEdit {
 	r := regexp.MustCompile(from)
 	bTo := []byte(to)
 	return tfbridge.DocsEdit{
 		Path: "*",
 		Edit: func(_ string, content []byte) ([]byte, error) {
 			return r.ReplaceAll(content, bTo), nil
 		},
+		Phase: phase,
 	}
 }
 

pkg/tfgen/edit_rules_test.go

@@ -0,0 +1,245 @@
+package tfgen
+
+import (
+	"bytes"
+	"runtime"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfbridge"
+	"github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfbridge/info"
+)
+
+func TestApplyEditRules(t *testing.T) {
+	t.Parallel()
+
+	type testCase struct {
+		// The name of the test case.
+		name     string
+		docFile  DocFile
+		expected []byte
+		phase    info.EditPhase
+	}
+
+	tests := []testCase{
+		{
+			name: "Replaces t/Terraform plan With pulumi preview",
+			docFile: DocFile{
+				Content: []byte("Any mention of Terraform plan or terraform plan will be Pulumi preview or pulumi preview"),
+			},
+			expected: []byte("Any mention of pulumi preview or pulumi preview will be Pulumi preview or pulumi preview"),
+		},
+		{
+			name: "Strips Hashicorp links correctly",
+			docFile: DocFile{
+				Content: []byte(readfile(t, "test_data/replace-links/input.md")),
+			},
+			expected: []byte(readfile(t, "test_data/replace-links/expected.md")),
+		},
+		{
+			name: "Strips Terraform links correctly",
+			docFile: DocFile{
+				Content: []byte("This provider requires at least [Terraform 1.0](https://www.terraform.io/downloads.html)."),
+			},
+			expected: []byte("This provider requires at least Terraform 1.0."),
+		},
+		{
+			name: "Replaces h/Hashicorp With p/Pulumi",
+			docFile: DocFile{
+				Content: []byte("Any mention of Hashicorp or hashicorp will be Pulumi or pulumi"),
+			},
+			expected: []byte("Any mention of Pulumi or pulumi will be Pulumi or pulumi"),
+			phase:    info.PostCodeTranslation,
+		},
+		{
+			name: "Replaces t/Terraform With p/Pulumi",
+			docFile: DocFile{
+				Content: []byte("Any mention of Terraform or terraform will be Pulumi or pulumi"),
+			},
+			expected: []byte("Any mention of Pulumi or pulumi will be Pulumi or pulumi"),
+			phase:    info.PostCodeTranslation,
+		},
+		{
+			name: "Replaces argument headers with input headers",
+			docFile: DocFile{
+				Content: []byte("# Argument Reference\n" +
+					"The following arguments are supported:\n* `some_argument`\n\n" +
+					"block contains the following arguments"),
+			},
+			expected: []byte("# Configuration Reference\n" +
+				"The following configuration inputs are supported:\n* `some_argument`\n\n" +
+				"input has the following nested fields"),
+			phase: info.PostCodeTranslation,
+		},
+		{
+			name: "Skips sections about logging by default",
+			docFile: DocFile{
+				Content:  []byte("# I am a provider\n\n### Additional Logging\n This section should be skipped"),
+				FileName: "filename",
+			},
+			expected: []byte("# I am a provider\n"),
+			phase:    info.PostCodeTranslation,
+		},
+		{
+			name: "Strips mentions of Terraform version pattern 1",
+			docFile: DocFile{
+				Content: []byte("This is a provider. It requires terraform 0.12 or later."),
+			},
+			expected: []byte("This is a provider."),
+			phase:    info.PostCodeTranslation,
+		},
+		{
+			name: "Strips mentions of Terraform version pattern 2",
+			docFile: DocFile{
+				Content: []byte("This is a provider. It requires terraform v0.12 or later."),
+			},
+			expected: []byte("This is a provider."),
+			phase:    info.PostCodeTranslation,
+		},
+		{
+			name: "Strips mentions of Terraform version pattern 3",
+			docFile: DocFile{
+				Content: []byte("This is a provider with an example. For Terraform v1.5 and later:\n Use this code."),
+			},
+			expected: []byte("This is a provider with an example.\nUse this code."),
+			phase:    info.PostCodeTranslation,
+		},
+		{
+			name: "Strips mentions of Terraform version pattern 4",
+			docFile: DocFile{
+				Content: []byte("This is a provider with an example. Terraform 1.5 and later:\n Use this code."),
+			},
+			expected: []byte("This is a provider with an example.\nUse this code."),
+			phase:    info.PostCodeTranslation,
+		},
+		{
+			name: "Strips mentions of Terraform version pattern 5",
+			docFile: DocFile{
+				Content: []byte("This is a provider with an example. Terraform 1.5 and earlier:\n Use this code."),
+			},
+			expected: []byte("This is a provider with an example.\nUse this code."),
+			phase:    info.PostCodeTranslation,
+		},
+		{
+			name: "Strips mentions of Terraform version pattern 6",
+			docFile: DocFile{
+				Content: []byte("This provider requires at least Terraform 1.0."),
+			},
+			expected: []byte(""),
+			phase:    info.PostCodeTranslation,
+		},
+		{
+			name: "Strips mentions of Terraform version pattern 7",
+			docFile: DocFile{
+				Content: []byte("This provider requires Terraform 1.0."),
+			},
+			expected: []byte(""),
+			phase:    info.PostCodeTranslation,
+		},
+		{
+			name: "Strips mentions of Terraform version pattern 8",
+			docFile: DocFile{
+				Content: []byte("A minimum of Terraform 1.4.0 is recommended."),
+			},
+			expected: []byte(""),
+			phase:    info.PostCodeTranslation,
+		},
+		{
+			name: "Strips mentions of Terraform version With Surrounding Text",
+			docFile: DocFile{
+				Content: []byte(readfile(t, "test_data/replace-terraform-version/input.md")),
+			},
+			expected: []byte(readfile(t, "test_data/replace-terraform-version/expected.md")),
+			phase:    info.PostCodeTranslation,
+		},
+		{
+			// Found in linode
+			name: "Rewrites providers.tf to Pulumi.yaml",
+			docFile: DocFile{
+				Content: []byte(readfile(t, "test_data/rewrite-providers-tf-to-pulumi-yaml/input.md")),
+			},
+			expected: []byte(readfile(t, "test_data/rewrite-providers-tf-to-pulumi-yaml/expected.md")),
+			phase:    info.PostCodeTranslation,
+		},
+		{
+			name: "Rewrites terraform init to pulumi up",
+			docFile: DocFile{
+				Content: []byte(readfile(t, "test_data/rewrite-tf-init-to-pulumi-up/input.md")),
+			},
+			expected: []byte(readfile(t, "test_data/rewrite-tf-init-to-pulumi-up/expected.md")),
+			phase:    info.PostCodeTranslation,
+		},
+		{
+			// Found in linode
+			name: "Replaces provider block with provider configuration",
+			docFile: DocFile{
+				Content: []byte(readfile(t, "test_data/replace-provider-block/input.md")),
+			},
+			expected: []byte(readfile(t, "test_data/replace-provider-block/expected.md")),
+			phase:    info.PostCodeTranslation,
+		},
+	}
+	edits := defaultEditRules()
+
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+			if runtime.GOOS == "windows" {
+				t.Skipf("Skipping on Windows due to a newline handling issue")
+			}
+			actual, err := edits.apply("*", tt.docFile.Content, tt.phase)
+			require.NoError(t, err)
+			assertEqualHTML(t, string(tt.expected), string(actual))
+		})
+	}
+}
+
+func TestApplyCustomEditRules(t *testing.T) {
+	t.Parallel()
+
+	type testCase struct {
+		// The name of the test case.
+		name     string
+		docFile  DocFile
+		expected []byte
+		edits    editRules
+	}
+
+	tests := []testCase{
+		{
+			name: "Replaces specified text pre code translation",
+			docFile: DocFile{
+				Content: []byte("This provider has a hroffic unreadable typo"),
+			},
+			expected: []byte("This provider has a horrific unreadable typo, which is now fixed"),
+			edits: append(
+				defaultEditRules(),
+				tfbridge.DocsEdit{
+					Path: "testfile.md",
+					Edit: func(_ string, content []byte) ([]byte, error) {
+						return bytes.ReplaceAll(
+							content,
+							[]byte("hroffic unreadable typo"),
+							[]byte("horrific unreadable typo, which is now fixed"),
+						), nil
+					},
+				},
+			),
+		},
+	}
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+			if runtime.GOOS == "windows" {
+				t.Skipf("Skipping on Windows due to a newline handling issue")
+			}
+			actual, err := tt.edits.apply("testfile.md", tt.docFile.Content, info.PreCodeTranslation)
+			require.NoError(t, err)
+			assertEqualHTML(t, string(tt.expected), string(actual))
+		})
+	}
+
+}