diff --git a/go.mod b/go.mod index 2c0ba5ad0..9a9ab9e16 100644 --- a/go.mod +++ b/go.mod @@ -208,7 +208,7 @@ require ( github.com/pulumi/pulumi/pkg/v3 v3.119.0 github.com/pulumi/pulumi/sdk/v3 v3.119.0 github.com/rivo/uniseg v0.4.4 // indirect - github.com/rogpeppe/go-internal v1.11.0 // indirect + github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/ryanuber/go-glob v1.0.0 // indirect github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 // indirect github.com/santhosh-tekuri/jsonschema/v5 v5.0.0 // indirect diff --git a/go.sum b/go.sum index abd94956d..533ed791f 100644 --- a/go.sum +++ b/go.sum @@ -1931,8 +1931,9 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= -github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58/go.mod h1:6lfFZQK844Gfx8o5WFuvpxWRwnSoipWe/p622j1v06w= diff --git a/pf/go.mod b/pf/go.mod index e90211404..61bc5b13d 100644 --- a/pf/go.mod +++ b/pf/go.mod @@ -198,7 +198,7 @@ require ( github.com/pulumi/pulumi/sdk/v3 v3.119.0 github.com/pulumi/schema-tools v0.1.2 // indirect github.com/rivo/uniseg v0.4.4 // indirect - github.com/rogpeppe/go-internal v1.11.0 // indirect + github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/ryanuber/go-glob v1.0.0 // indirect github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 // indirect diff --git a/pf/go.sum b/pf/go.sum index b0dd82e2d..67a5d6c03 100644 --- a/pf/go.sum +++ b/pf/go.sum @@ -1929,8 +1929,9 @@ github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYe github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58/go.mod h1:6lfFZQK844Gfx8o5WFuvpxWRwnSoipWe/p622j1v06w= diff --git a/pf/tests/go.mod b/pf/tests/go.mod index 77e5be808..6d7f3e14a 100644 --- a/pf/tests/go.mod +++ b/pf/tests/go.mod @@ -213,7 +213,7 @@ require ( github.com/pulumi/pulumi/sdk/v3 v3.119.0 github.com/pulumi/schema-tools v0.1.2 // indirect github.com/rivo/uniseg v0.4.4 // indirect - github.com/rogpeppe/go-internal v1.11.0 // indirect + github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/ryanuber/go-glob v1.0.0 // indirect github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 // indirect diff --git a/pf/tests/go.sum b/pf/tests/go.sum index 688b977e6..67581a4dc 100644 --- a/pf/tests/go.sum +++ b/pf/tests/go.sum @@ -1957,8 +1957,9 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= -github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58/go.mod h1:6lfFZQK844Gfx8o5WFuvpxWRwnSoipWe/p622j1v06w= diff --git a/pkg/tests/go.mod b/pkg/tests/go.mod index b5469a46b..1b49ddbe4 100644 --- a/pkg/tests/go.mod +++ b/pkg/tests/go.mod @@ -207,7 +207,7 @@ require ( github.com/pulumi/pulumi/sdk/v3 v3.119.0 github.com/pulumi/terraform-diff-reader v0.0.2 // indirect github.com/rivo/uniseg v0.4.4 // indirect - github.com/rogpeppe/go-internal v1.11.0 // indirect + github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/ryanuber/go-glob v1.0.0 // indirect github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 // indirect github.com/santhosh-tekuri/jsonschema/v5 v5.0.0 // indirect diff --git a/pkg/tests/go.sum b/pkg/tests/go.sum index 3ad9a5f5b..ccf4e864e 100644 --- a/pkg/tests/go.sum +++ b/pkg/tests/go.sum @@ -1941,8 +1941,9 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= -github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58/go.mod h1:6lfFZQK844Gfx8o5WFuvpxWRwnSoipWe/p622j1v06w= diff --git a/pkg/tfgen/docs.go b/pkg/tfgen/docs.go index 44a806bad..af007744a 100644 --- a/pkg/tfgen/docs.go +++ b/pkg/tfgen/docs.go @@ -47,6 +47,9 @@ import ( const ( startPulumiCodeChooser = "" endPulumiCodeChooser = "" + + // The Hugo front matter delimiter + delimiter = "---\n" ) // argumentDocs contains the documentation metadata for an argument of the resource. @@ -145,6 +148,8 @@ const ( ResourceDocs DocKind = "resources" // DataSourceDocs indicates documentation pertaining to data source entities. DataSourceDocs DocKind = "data-sources" + // InstallationDocs indicates documentation pertaining to provider configuration and installation. + InstallationDocs DocKind = "installation" ) func (k DocKind) String() string { @@ -1415,7 +1420,6 @@ func (g *Generator) convertExamples(docs string, path examplePath) string { strings.TrimRightFunc(docs[:exampleIndex], unicode.IsSpace), docs[exampleIndex:]) } - if cliConverterEnabled() { return g.cliConverter().StartConvertingExamples(docs, path) } @@ -1567,7 +1571,6 @@ func (g *Generator) convertExamplesInner( } langs := genLanguageToSlice(g.language) convertedBlock, err := convertHCL(e, hcl, path.String(), langs) - if err != nil { // We do not write this section, ever. // @@ -1949,7 +1952,7 @@ func genLanguageToSlice(input Language) []string { return []string{convert.LanguageGo} case PCL: return []string{convert.LanguagePulumi} - case Schema: + case Schema, RegistryDocs: return []string{ convert.LanguageTypescript, convert.LanguagePython, @@ -2266,3 +2269,47 @@ var ( func guessIsHCL(code string) bool { return guessIsHCLPattern.MatchString(code) } + +func plainDocsParser(docFile *DocFile, g *Generator) ([]byte, error) { + // Get file content without front matter, and split title + contentStr, title := getBodyAndTitle(string(docFile.Content)) + // Add pulumi-specific front matter + contentStr = writeFrontMatter(title) + contentStr + + //TODO: See https://github.com/pulumi/pulumi-terraform-bridge/issues/2078 + // - translate code blocks with code choosers + // - apply default edit rules + // - reformat TF names + // - Translation for certain headers such as "Arguments Reference" or "Configuration block" + // - Ability to omit irrelevant sections + return []byte(contentStr), nil +} + +func writeFrontMatter(title string) string { + return fmt.Sprintf(delimiter+ + "title: %s Installation & Configuration\n"+ + "meta_desc: Provides an overview on how to configure the Pulumi %s.\n"+ + "layout: package\n"+ + delimiter, + title, title) +} + +func writeIndexFrontMatter(displayName string) string { + return fmt.Sprintf(delimiter+ + "title: %s\n"+ + "meta_desc: The %s provider for Pulumi can be used to provision any of the cloud resources available in %s.\n"+ + "layout: package\n"+ + delimiter, + displayName, displayName, displayName) +} + +func getBodyAndTitle(content string) (string, string) { + // The first header in `index.md` is the package name, of the format `# Foo Provider`. + titleIndex := strings.Index(content, "# ") + // Get the location fo the next newline + nextNewLine := strings.Index(content[titleIndex:], "\n") + titleIndex + // Get the title line, without the h1 anchor + title := content[titleIndex+2 : nextNewLine] + // strip the title and any front matter + return content[nextNewLine+1:], title +} diff --git a/pkg/tfgen/docs_test.go b/pkg/tfgen/docs_test.go index c6174c846..61d3b3faa 100644 --- a/pkg/tfgen/docs_test.go +++ b/pkg/tfgen/docs_test.go @@ -1801,6 +1801,17 @@ func (m mockSource) getDatasource(rawname string, info *tfbridge.DocInfo) (*DocF return nil, nil } +func (m mockSource) getInstallation(info *tfbridge.DocInfo) (*DocFile, error) { + f, ok := m["index.md"] + if !ok { + return nil, nil + } + return &DocFile{ + Content: []byte(f), + FileName: "index.md", + }, nil +} + type mockSink struct{ t *testing.T } func (mockSink) warn(string, ...interface{}) {} @@ -2001,3 +2012,43 @@ resource "aws_ami" "example" { }) } } + +func TestPlainDocsParser(t *testing.T) { + t.Parallel() + + type testCase struct { + // The name of the test case. + name string + docFile DocFile + expected []byte + } + + tests := []testCase{ + { + name: "Replaces Upstream Front Matter With Pulumi Front Matter", + docFile: DocFile{ + Content: []byte("---\nlayout: \"openstack\"\npage_title: \"Provider: OpenStack\"\nsidebar_current: \"docs-openstack-index\"\ndescription: |-\n The OpenStack provider is used to interact with the many resources supported by OpenStack. The provider needs to be configured with the proper credentials before it can be used.\n---\n\n# OpenStack Provider\n\nThe OpenStack provider is used to interact with the\nmany resources supported by OpenStack. The provider needs to be configured\nwith the proper credentials before it can be used.\n\nUse the navigation to the left to read about the available resources."), + }, + expected: []byte("---\ntitle: OpenStack Provider Installation & Configuration\nmeta_desc: Provides an overview on how to configure the Pulumi OpenStack Provider.\nlayout: package\n---\n\nThe OpenStack provider is used to interact with the\nmany resources supported by OpenStack. The provider needs to be configured\nwith the proper credentials before it can be used.\n\nUse the navigation to the left to read about the available resources."), + }, + { + name: "Writes Pulumi Style Front Matter If Not Present", + docFile: DocFile{ + Content: []byte("# Artifactory Provider\n\nThe [Artifactory](https://jfrog.com/artifactory/) provider is used to interact with the resources supported by Artifactory. The provider needs to be configured with the proper credentials before it can be used.\n\nLinks to documentation for specific resources can be found in the table of contents to the left.\n\nThis provider requires access to Artifactory APIs, which are only available in the _licensed_ pro and enterprise editions. You can determine which license you have by accessing the following the URL `${host}/artifactory/api/system/licenses/`.\n\nYou can either access it via API, or web browser - it require admin level credentials."), + }, + expected: []byte("---\ntitle: Artifactory Provider Installation & Configuration\nmeta_desc: Provides an overview on how to configure the Pulumi Artifactory Provider.\nlayout: package\n---\n\nThe [Artifactory](https://jfrog.com/artifactory/) provider is used to interact with the resources supported by Artifactory. The provider needs to be configured with the proper credentials before it can be used.\n\nLinks to documentation for specific resources can be found in the table of contents to the left.\n\nThis provider requires access to Artifactory APIs, which are only available in the _licensed_ pro and enterprise editions. You can determine which license you have by accessing the following the URL `${host}/artifactory/api/system/licenses/`.\n\nYou can either access it via API, or web browser - it require admin level credentials."), + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + g := &Generator{ + sink: mockSink{t}, + } + actual, err := plainDocsParser(&tt.docFile, g) + require.NoError(t, err) + require.Equal(t, string(tt.expected), string(actual)) + }) + } +} diff --git a/pkg/tfgen/generate.go b/pkg/tfgen/generate.go index 07f263be5..7debd97c3 100644 --- a/pkg/tfgen/generate.go +++ b/pkg/tfgen/generate.go @@ -18,6 +18,8 @@ import ( "context" "encoding/json" "fmt" + "golang.org/x/text/cases" + "golang.org/x/text/language" "os" "path" "path/filepath" @@ -94,6 +96,14 @@ const ( CSharp Language = "dotnet" Schema Language = "schema" PCL Language = "pulumi" + // RegistryDocs + // Setting RegistryDocs as a separate bridge "language" in the bridge allows us to create custom logic specific to + // transforming and emitting upstream installation docs. + // When we generate registry docs, we want to: + //- be able to generate them via a separate command so we can enable it on a per-provider basis + //- be able to pass a separate output location from the schema location (in this case, `docs/`) + //- convert examples into all Pulumi-supported languages + RegistryDocs Language = "registry-docs" ) func (l Language) shouldConvertExamples() bool { @@ -787,7 +797,7 @@ func NewGenerator(opts GeneratorOptions) (*Generator, error) { // Ensure the language is valid. switch lang { - case Golang, NodeJS, Python, CSharp, Schema, PCL: + case Golang, NodeJS, Python, CSharp, Schema, PCL, RegistryDocs: // OK default: return nil, errors.Errorf("unrecognized language runtime: %s", lang) @@ -946,8 +956,29 @@ func (g *Generator) UnstableGenerateFromSchema(genSchemaResult *GenerateSchemaRe // Go ahead and let the language generator do its thing. If we're emitting the schema, just go ahead and serialize // it out. - var files map[string][]byte + files := make(map[string][]byte) + switch g.language { + case RegistryDocs: + source := NewGitRepoDocsSource(g) + installationFile, err := source.getInstallation(nil) + if err != nil { + return errors.Wrapf(err, "failed to obtain an index.md file for this provider") + } + content, err := plainDocsParser(installationFile, g) + if err != nil { + return errors.Wrapf(err, "failed to parse installation docs") + } + files["installation-configuration.md"] = content + // Populate minimal _index.md file + displayName := g.info.DisplayName + if displayName == "" { + // Capitalize the package name + capitalize := cases.Title(language.English) + displayName = capitalize.String(g.info.Name) + } + indexContent := writeIndexFrontMatter(displayName) + files["_index.md"] = []byte(indexContent) case Schema: // Omit the version so that the spec is stable if the version is e.g. derived from the current Git commit hash. pulumiPackageSpec.Version = "" @@ -1000,8 +1031,10 @@ func (g *Generator) UnstableGenerateFromSchema(genSchemaResult *GenerateSchemaRe } // Emit the Pulumi project information. - if err = g.emitProjectMetadata(g.pkg, g.language); err != nil { - return errors.Wrapf(err, "failed to create project file") + if g.language != RegistryDocs { + if err = g.emitProjectMetadata(g.pkg, g.language); err != nil { + return errors.Wrapf(err, "failed to create project file") + } } // Close the plugin host. @@ -1538,7 +1571,7 @@ func (g *Generator) gatherOverlays() (moduleMap, error) { if csharpinfo := g.info.CSharp; csharpinfo != nil { overlay = csharpinfo.Overlay } - case Schema, PCL: + case Schema, PCL, RegistryDocs: // N/A default: contract.Failf("unrecognized language: %s", g.language) diff --git a/pkg/tfgen/source.go b/pkg/tfgen/source.go index b17d9c2e5..485de6080 100644 --- a/pkg/tfgen/source.go +++ b/pkg/tfgen/source.go @@ -33,6 +33,9 @@ type DocsSource interface { // Get the bytes for a datasource with TF token rawname. getDatasource(rawname string, info *tfbridge.DocInfo) (*DocFile, error) + + // Get the bytes for the provider installation doc. + getInstallation(info *tfbridge.DocInfo) (*DocFile, error) } type DocFile struct { @@ -70,6 +73,11 @@ func (gh *gitRepoSource) getDatasource(rawname string, info *tfbridge.DocInfo) ( return gh.getFile(rawname, info, DataSourceDocs) } +func (gh *gitRepoSource) getInstallation(info *tfbridge.DocInfo) (*DocFile, error) { + // The installation docs do not have a rawname. + return gh.getFile("", info, InstallationDocs) +} + // getFile implements the private logic necessary to get a file from a TF Git repo's website section. func (gh *gitRepoSource) getFile( rawname string, info *tfbridge.DocInfo, kind DocKind, @@ -86,11 +94,17 @@ func (gh *gitRepoSource) getFile( return nil, fmt.Errorf("repo for token %q: %w", rawname, err) } } - - possibleMarkdownNames := getMarkdownNames(gh.resourcePrefix, rawname, gh.docRules) - - if info != nil && info.Source != "" { - possibleMarkdownNames = append(possibleMarkdownNames, info.Source) + var possibleMarkdownNames []string + switch kind { + case InstallationDocs: + possibleMarkdownNames = append(possibleMarkdownNames, "index.md", "index.html.markdown") + case ResourceDocs, DataSourceDocs: + possibleMarkdownNames = getMarkdownNames(gh.resourcePrefix, rawname, gh.docRules) + if info != nil && info.Source != "" { + possibleMarkdownNames = append(possibleMarkdownNames, info.Source) + } + default: + return nil, fmt.Errorf("unknown docs kind: %s", kind) } return readMarkdown(repoPath, kind, possibleMarkdownNames) @@ -197,7 +211,6 @@ func readMarkdown(repo string, kind DocKind, possibleLocations []string) (*DocFi if err != nil { return nil, fmt.Errorf("could not gather location prefix for %q: %w", repo, err) } - for _, prefix := range locationPrefix { for _, name := range possibleLocations { location := filepath.Join(prefix, name) @@ -236,6 +249,19 @@ func getDocsPath(repo string, kind DocKind) ([]string, error) { var paths []string + if kind == InstallationDocs { + // ${repo}/docs/ + if p := filepath.Join(repo, "docs"); exists(p) { + paths = append(paths, p) + } + // ${repo}/website/docs + // + // This is the legacy way to describe docs. + if p := filepath.Join(repo, "website", "docs"); exists(p) { + paths = append(paths, p) + } + return paths, err + } // ${repo}/website/docs/r // // This is the legacy way to describe docs.