fix: strip upstream frontmatter (#2361)

When generating Pulumi frontmatter, we do not need to obtain the `title`
content by parsing the upstream file. We can use the provider name
instead.

This fixes ~a bug~ behavior where the previous code was interpreting
comments inside of the upstream frontmatter as H1. Example:
https://github.com/kreuzwerker/terraform-provider-docker/blob/master/docs/index.md?plain=1#L2
- this never made it to actually get filed as a bug; it was discovered
during content generation.

Additionally, H1 title removal is now done via Goldmark transformer.

- **Strip upstream frontmatter by default and use provider name to
generate Pulumi frontmatter title. Add tests.**
- **Add capitalization**

pkg/tfgen/docs.go

@@ -383,7 +383,7 @@ func trimFrontMatter(text []byte) []byte {
 	if idx == -1 {
 		return text
 	}
-	return body[idx+3:]
+	return body[idx+4:]
 }
 
 func splitByMarkdownHeaders(text string, level int) [][]string {

pkg/tfgen/installation_docs.go

@@ -22,17 +22,17 @@ import (
 )
 
 func plainDocsParser(docFile *DocFile, g *Generator) ([]byte, error) {
-	// Get file content without front matter, and split title
-	contentStr, title := getBodyAndTitle(string(docFile.Content))
+	// Get file content without front matter
+	content := trimFrontMatter(docFile.Content)
 	// Add pulumi-specific front matter
 	// Generate pulumi-specific front matter
-	frontMatter := writeFrontMatter(title)
+	frontMatter := writeFrontMatter(g.info.Name)
 
 	// Generate pulumi-specific installation instructions
 	installationInstructions := writeInstallationInstructions(g.info.Golang.ImportBasePath, g.info.Name)
 
 	// Add instructions to top of file
-	contentStr = frontMatter + installationInstructions + contentStr
+	contentStr := frontMatter + installationInstructions + string(content)
 
 	//Translate code blocks to Pulumi
 	contentStr, err := translateCodeBlocks(contentStr, g)
@@ -46,6 +46,12 @@ func plainDocsParser(docFile *DocFile, g *Generator) ([]byte, error) {
 		return nil, err
 	}
 
+	// Remove the title. A title gets populated from Hugo frontmatter; we do not want two.
+	contentBytes, err = removeTitle(contentBytes)
+	if err != nil {
+		return nil, err
+	}
+
 	//Reformat field names. Configuration fields are camelCased like nodejs.
 	contentStr, _ = reformatText(infoContext{
 		language: "nodejs",
@@ -57,10 +63,10 @@ func plainDocsParser(docFile *DocFile, g *Generator) ([]byte, error) {
 }
 
 func writeFrontMatter(providerName string) string {
-	// Capitalize the provider name for the title
-
+	// Capitalize the package name
 	capitalize := cases.Title(language.English)
 	title := capitalize.String(providerName)
+
 	return fmt.Sprintf(delimiter+
 		"title: %[1]s Provider\n"+
 		"meta_desc: Provides an overview on how to configure the Pulumi %[1]s provider.\n"+
@@ -70,17 +76,6 @@ func writeFrontMatter(providerName string) string {
 		title)
 }
 
-func getBodyAndTitle(content string) (string, string) {
-	// The first header in `index.md` is the package name, of the format `# Foo Provider`.
-	titleIndex := strings.Index(content, "# ")
-	// Get the location fo the next newline
-	nextNewLine := strings.Index(content[titleIndex:], "\n") + titleIndex
-	// Get the title line, without the h1 anchor
-	title := content[titleIndex+2 : nextNewLine]
-	// strip the title and any front matter
-	return content[nextNewLine+1:], title
-}
-
 // writeInstallationInstructions renders the following for any provider:
 // ****
 // Installation
@@ -251,6 +246,46 @@ func convertExample(g *Generator, code string, exampleNumber int) (string, error
 	return exampleContent, nil
 }
 
+type titleRemover struct {
+}
+
+var _ parser.ASTTransformer = titleRemover{}
+
+func (tr titleRemover) Transform(node *ast.Document, reader text.Reader, pc parser.Context) {
+	err := ast.Walk(node, func(n ast.Node, entering bool) (ast.WalkStatus, error) {
+		// The first header we encounter should be the document title.
+		header, found := n.(*ast.Heading)
+		if !found || header.Level != 1 || !entering {
+			return ast.WalkContinue, nil
+		}
+
+		parent := n.Parent()
+		contract.Assertf(parent != nil, "parent cannot be nil")
+		// Removal here is safe, as we want to remove only the first header anyway.
+		n.Parent().RemoveChild(parent, header)
+		return ast.WalkStop, nil
+	})
+	contract.AssertNoErrorf(err, "impossible: ast.Walk should never error")
+}
+
+func removeTitle(
+	content []byte,
+) ([]byte, error) {
+	// Instantiate our transformer
+	titleRemover := titleRemover{}
+	gm := goldmark.New(
+		goldmark.WithExtensions(parse.TFRegistryExtension),
+		goldmark.WithParserOptions(parser.WithASTTransformers(
+			util.Prioritized(titleRemover, 1000),
+		)),
+		goldmark.WithRenderer(markdown.NewRenderer()),
+	)
+	var buf bytes.Buffer
+	// Convert parses the source, applies transformers, and renders output to buf
+	err := gm.Convert(content, &buf)
+	return buf.Bytes(), err
+}
+
 type sectionSkipper struct {
 	shouldSkipHeader func(headerText string) bool
 }

pkg/tfgen/installation_docs_test.go

@@ -56,6 +56,74 @@ func TestPlainDocsParser(t *testing.T) {
 	}
 }
 
+func TestTrimFrontmatter(t *testing.T) {
+	type testCase struct {
+		// The name of the test case.
+		name     string
+		input    string
+		expected string
+	}
+
+	tests := []testCase{
+		{
+			name:     "Strips Upstream Frontmatter",
+			input:    readfile(t, "test_data/strip-front-matter/openstack-input.md"),
+			expected: readfile(t, "test_data/strip-front-matter/openstack-expected.md"),
+		},
+		{
+			name:     "Returns Body If No Frontmatter",
+			input:    readfile(t, "test_data/strip-front-matter/artifactory-input.md"),
+			expected: readfile(t, "test_data/strip-front-matter/artifactory-expected.md"),
+		},
+	}
+
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			if runtime.GOOS == "windows" {
+				t.Skipf("Skipping on Windows due to a test setup issue")
+			}
+			t.Parallel()
+			actual := trimFrontMatter([]byte(tt.input))
+			assertEqualHTML(t, tt.expected, string(actual))
+		})
+	}
+}
+
+func TestRemoveTitle(t *testing.T) {
+	type testCase struct {
+		// The name of the test case.
+		name     string
+		input    string
+		expected string
+	}
+
+	tests := []testCase{
+		{
+			name:     "Strips Title Placed Anywhere",
+			input:    readfile(t, "test_data/remove-title/openstack-input.md"),
+			expected: readfile(t, "test_data/remove-title/openstack-expected.md"),
+		},
+		{
+			name:     "Strips Title On Top",
+			input:    readfile(t, "test_data/remove-title/artifactory-input.md"),
+			expected: readfile(t, "test_data/remove-title/artifactory-expected.md"),
+		},
+	}
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			if runtime.GOOS == "windows" {
+				t.Skipf("Skipping on Windows due to a test setup issue")
+			}
+			t.Parallel()
+			actual, err := removeTitle([]byte(tt.input))
+			assert.NoError(t, err)
+			assertEqualHTML(t, tt.expected, string(actual))
+		})
+	}
+}
+
 //nolint:lll
 func TestWriteInstallationInstructions(t *testing.T) {
 	t.Parallel()
@@ -101,7 +169,7 @@ func TestWriteFrontMatter(t *testing.T) {
 
 	tc := testCase{
 		name:         "Generates Front Matter for installation-configuration.md",
-		providerName: "Test",
+		providerName: "test",
 		expected: delimiter +
 			"title: Test Provider\n" +
 			"meta_desc: Provides an overview on how to configure the Pulumi Test provider.\n" +

pkg/tfgen/test_data/remove-title/artifactory-expected.md

@@ -0,0 +1,16 @@
+The [Artifactory](https://jfrog.com/artifactory/) provider is used to interact with the resources supported by Artifactory. The provider needs to be configured with the proper credentials before it can be used.
+
+Links to documentation for specific resources can be found in the table of contents to the left.
+
+This provider requires access to Artifactory APIs, which are only available in the _licensed_ pro and enterprise editions. You can determine which license you have by accessing the following the URL `${host}/artifactory/api/system/licenses/`.
+
+You can either access it via API, or web browser - it require admin level credentials.
+
+```sh
+curl -sL ${host}/artifactory/api/system/licenses/ | jq .
+{
+  "type" : "Enterprise Plus Trial",
+  "validThrough" : "Jan 29, 2022",
+  "licensedTo" : "JFrog Ltd"
+}
+```

pkg/tfgen/test_data/remove-title/artifactory-input.md

@@ -0,0 +1,18 @@
+# Artifactory Provider
+
+The [Artifactory](https://jfrog.com/artifactory/) provider is used to interact with the resources supported by Artifactory. The provider needs to be configured with the proper credentials before it can be used.
+
+Links to documentation for specific resources can be found in the table of contents to the left.
+
+This provider requires access to Artifactory APIs, which are only available in the _licensed_ pro and enterprise editions. You can determine which license you have by accessing the following the URL `${host}/artifactory/api/system/licenses/`.
+
+You can either access it via API, or web browser - it require admin level credentials.
+
+```sh
+curl -sL ${host}/artifactory/api/system/licenses/ | jq .
+{
+  "type" : "Enterprise Plus Trial",
+  "validThrough" : "Jan 29, 2022",
+  "licensedTo" : "JFrog Ltd"
+}
+```

pkg/tfgen/test_data/remove-title/openstack-expected.md

@@ -0,0 +1,28 @@
+---
+layout: "openstack"
+page_title: "Provider: OpenStack"
+sidebar_current: "docs-openstack-index"
+description: |-
+  The OpenStack provider is used to interact with the many resources supported by OpenStack. The provider needs to be configured with the proper credentials before it can be used.
+---
+
+The OpenStack provider is used to interact with the
+many resources supported by OpenStack. The provider needs to be configured
+with the proper credentials before it can be used.
+
+Use the navigation to the left to read about the available resources.
+
+## Example Usage
+
+```hcl
+# Define required providers
+terraform {
+  required_version = ">= 0.14.0"
+  required_providers {
+    openstack = {
+      source  = "terraform-provider-openstack/openstack"
+      version = "~> 1.53.0"
+    }
+  }
+}
+```

pkg/tfgen/test_data/remove-title/openstack-input.md

@@ -0,0 +1,30 @@
+---
+layout: "openstack"
+page_title: "Provider: OpenStack"
+sidebar_current: "docs-openstack-index"
+description: |-
+  The OpenStack provider is used to interact with the many resources supported by OpenStack. The provider needs to be configured with the proper credentials before it can be used.
+---
+
+# OpenStack Provider
+
+The OpenStack provider is used to interact with the
+many resources supported by OpenStack. The provider needs to be configured
+with the proper credentials before it can be used.
+
+Use the navigation to the left to read about the available resources.
+
+## Example Usage
+
+```hcl
+# Define required providers
+terraform {
+  required_version = ">= 0.14.0"
+  required_providers {
+    openstack = {
+      source  = "terraform-provider-openstack/openstack"
+      version = "~> 1.53.0"
+    }
+  }
+}
+```

pkg/tfgen/test_data/strip-front-matter/artifactory-expected.md

@@ -0,0 +1,18 @@
+# Artifactory Provider
+
+The [Artifactory](https://jfrog.com/artifactory/) provider is used to interact with the resources supported by Artifactory. The provider needs to be configured with the proper credentials before it can be used.
+
+Links to documentation for specific resources can be found in the table of contents to the left.
+
+This provider requires access to Artifactory APIs, which are only available in the _licensed_ pro and enterprise editions. You can determine which license you have by accessing the following the URL `${host}/artifactory/api/system/licenses/`.
+
+You can either access it via API, or web browser - it require admin level credentials.
+
+```sh
+curl -sL ${host}/artifactory/api/system/licenses/ | jq .
+{
+  "type" : "Enterprise Plus Trial",
+  "validThrough" : "Jan 29, 2022",
+  "licensedTo" : "JFrog Ltd"
+}
+```

pkg/tfgen/test_data/strip-front-matter/artifactory-input.md

@@ -0,0 +1,18 @@
+# Artifactory Provider
+
+The [Artifactory](https://jfrog.com/artifactory/) provider is used to interact with the resources supported by Artifactory. The provider needs to be configured with the proper credentials before it can be used.
+
+Links to documentation for specific resources can be found in the table of contents to the left.
+
+This provider requires access to Artifactory APIs, which are only available in the _licensed_ pro and enterprise editions. You can determine which license you have by accessing the following the URL `${host}/artifactory/api/system/licenses/`.
+
+You can either access it via API, or web browser - it require admin level credentials.
+
+```sh
+curl -sL ${host}/artifactory/api/system/licenses/ | jq .
+{
+  "type" : "Enterprise Plus Trial",
+  "validThrough" : "Jan 29, 2022",
+  "licensedTo" : "JFrog Ltd"
+}
+```

pkg/tfgen/test_data/strip-front-matter/openstack-expected.md

@@ -0,0 +1,22 @@
+# OpenStack Provider
+
+The OpenStack provider is used to interact with the
+many resources supported by OpenStack. The provider needs to be configured
+with the proper credentials before it can be used.
+
+Use the navigation to the left to read about the available resources.
+
+## Example Usage
+
+```hcl
+# Define required providers
+terraform {
+required_version = ">= 0.14.0"
+  required_providers {
+    openstack = {
+      source  = "terraform-provider-openstack/openstack"
+      version = "~> 1.53.0"
+    }
+  }
+}
+```