diff --git a/themes/default/layouts/partials/registry/package/icon.html b/themes/default/layouts/partials/registry/package/icon.html index a41272a918..d12d270dc9 100644 --- a/themes/default/layouts/partials/registry/package/icon.html +++ b/themes/default/layouts/partials/registry/package/icon.html @@ -1,5 +1,5 @@ {{ $url := printf "/logos/pkg/%s.svg" .name }} -{{ if ne .logo_url "" }} +{{ if and (isset .logo_url) (ne .logo_url "") }} {{ $url = .logo_url }} {{ end }} diff --git a/tools/resourcedocsgen/cmd/docs/registry.go b/tools/resourcedocsgen/cmd/docs/registry.go index 175e7c39fd..23a0a4fef5 100644 --- a/tools/resourcedocsgen/cmd/docs/registry.go +++ b/tools/resourcedocsgen/cmd/docs/registry.go @@ -52,27 +52,13 @@ func genResourceDocsForPackageFromRegistryMetadata( metadata pkg.PackageMeta, docsOutDir, packageTreeJSONOutDir string, ) error { glog.Infoln("Generating docs for", metadata.Name) - if metadata.RepoURL == "" { - return errors.Errorf("metadata for package %q does not contain the repo_url", metadata.Name) - } - - schemaFilePath := fmt.Sprintf(defaultSchemaFilePathFormat, metadata.Name) - if metadata.SchemaFilePath != "" { - schemaFilePath = metadata.SchemaFilePath - } - // Make sure the schema file path does not have a leading slash. - // We'll add in the URL format below. It's easier to read that way. - schemaFilePath = strings.TrimPrefix(schemaFilePath, "/") - - repoSlug, err := getRepoSlug(metadata.RepoURL) + schemaFileURL, err := getSchemaFileURL(metadata) if err != nil { - return errors.WithMessage(err, "could not get repo slug") + return fmt.Errorf("failed to get schema_file_url: %w", err) } - glog.Infoln("Reading remote schema file from VCS") - // TODO: Support raw URLs for other VCS too. - schemaFileURL := fmt.Sprintf("https://raw.githubusercontent.com/%s/%s/%s", repoSlug, metadata.Version, schemaFilePath) + resp, err := http.Get(schemaFileURL) //nolint:gosec if err != nil { return errors.Wrapf(err, "reading schema file from VCS %s", schemaFileURL) @@ -85,18 +71,19 @@ func genResourceDocsForPackageFromRegistryMetadata( return errors.Wrapf(err, "reading response body from %s", schemaFileURL) } - // The source schema can be in YAML format. If that's the case - // convert it to JSON first. - if strings.HasSuffix(schemaFilePath, ".yaml") { - schemaBytes, err = yaml.YAMLToJSON(schemaBytes) - if err != nil { - return errors.Wrap(err, "reading YAML schema") - } - } - var mainSpec pschema.PackageSpec - if err := json.Unmarshal(schemaBytes, &mainSpec); err != nil { - return errors.Wrap(err, "unmarshalling schema into a PackageSpec") + + switch { + // The source schema can be in YAML format. + case strings.HasSuffix(schemaFileURL, ".yaml"): + if err := yaml.Unmarshal(schemaBytes, &mainSpec); err != nil { + return errors.Wrap(err, "unmarshalling YAML schema into PackageSpec") + } + // If we don't have another format, assume JSON. + default: + if err := json.Unmarshal(schemaBytes, &mainSpec); err != nil { + return errors.Wrap(err, "unmarshalling JSON schema into PackageSpec") + } } pulPkg, genctx, err := getPulumiPackageFromSchema(docsOutDir, mainSpec) @@ -117,6 +104,34 @@ func genResourceDocsForPackageFromRegistryMetadata( return nil } +func getSchemaFileURL(metadata pkg.PackageMeta) (string, error) { + if metadata.SchemaFileURL != "" { + return metadata.SchemaFileURL, nil + } + + // We don't have an explicit SchemaFileURL, so migrate from SchemaFilePath. + + if metadata.RepoURL == "" { + return "", errors.Errorf("metadata for package %q does not contain the repo_url", metadata.Name) + } + + schemaFilePath := fmt.Sprintf(defaultSchemaFilePathFormat, metadata.Name) + if p := metadata.SchemaFilePath; p != "" { //nolint:staticcheck + schemaFilePath = p + } + + // Make sure the schema file path does not have a leading slash. + // We'll add in the URL format below. It's easier to read that way. + schemaFilePath = strings.TrimPrefix(schemaFilePath, "/") + + repoSlug, err := getRepoSlug(metadata.RepoURL) + if err != nil { + return "", errors.WithMessage(err, "could not get repo slug") + } + + return fmt.Sprintf("https://raw.githubusercontent.com/%s/%s/%s", repoSlug, metadata.Version, schemaFilePath), nil +} + func getRegistryPackagesPath(repoPath string) string { return filepath.Join(repoPath, "themes", "default", "data", "registry", "packages") } diff --git a/tools/resourcedocsgen/cmd/docs/registry_test.go b/tools/resourcedocsgen/cmd/docs/registry_test.go index 65da4c79b2..e639949a93 100644 --- a/tools/resourcedocsgen/cmd/docs/registry_test.go +++ b/tools/resourcedocsgen/cmd/docs/registry_test.go @@ -22,13 +22,9 @@ import ( "github.com/stretchr/testify/require" ) -//nolint:paralleltest // resourceDocsFromRegistryCmd relies on global state. func TestGenerateDocsSinglePackage(t *testing.T) { - // Set up the correct directory structure: - registryDir := t.TempDir() - util.WriteFile(t, - filepath.Join(registryDir, "themes", "default", "data", "registry", "packages", "random.yaml"), - `category: Utility + t.Parallel() + const partialMetadata = `category: Utility component: false description: A Pulumi package to safely use randomness in Pulumi programs. featured: false @@ -38,10 +34,14 @@ native: false package_status: ga publisher: Pulumi repo_url: https://github.com/pulumi/pulumi-random -schema_file_path: provider/cmd/pulumi-resource-random/schema.json title: random updated_on: 1729058626 -version: v4.16.7`) +version: v4.16.7` + registryDir := t.TempDir() + util.WriteFile(t, + filepath.Join(registryDir, "themes", "default", "data", "registry", "packages", "random.yaml"), + partialMetadata+` +schema_file_path: provider/cmd/pulumi-resource-random/schema.json`) basePackageTreeJSONOutDir := t.TempDir() baseDocsOutDir := t.TempDir() @@ -53,12 +53,33 @@ version: v4.16.7`) "--registryDir", registryDir, }) require.NoError(t, cmd.Execute()) - t.Run("docs", func(t *testing.T) { util.AssertDirEqual(t, baseDocsOutDir) }) - t.Run("tree", func(t *testing.T) { util.AssertDirEqual(t, basePackageTreeJSONOutDir) }) + t.Run("docs", func(t *testing.T) { t.Parallel(); util.AssertDirEqual(t, baseDocsOutDir) }) + t.Run("tree", func(t *testing.T) { t.Parallel(); util.AssertDirEqual(t, basePackageTreeJSONOutDir) }) + t.Run("check-schema-file-url", func(t *testing.T) { + t.Parallel() + registryDirV2 := t.TempDir() + util.WriteFile(t, + filepath.Join(registryDirV2, "themes", "default", "data", "registry", "packages", "random.yaml"), + partialMetadata+` ++schema_file_url: https://raw.githubusercontent.com/pulumi/pulumi-random/v4.16.7/provider/cmd/pulumi-resource-random/schema.json`) //nolint:lll + basePackageTreeJSONOutDirV2 := t.TempDir() + baseDocsOutDirV2 := t.TempDir() + + cmd := resourceDocsFromRegistryCmd() + cmd.SetArgs([]string{ + "random", /* pkgName */ + "--baseDocsOutDir", baseDocsOutDirV2, + "--basePackageTreeJSONOutDir", basePackageTreeJSONOutDirV2, + "--registryDir", registryDirV2, + }) + require.NoError(t, cmd.Execute()) + util.AssertDirsEqual(t, baseDocsOutDir, baseDocsOutDirV2) + util.AssertDirsEqual(t, basePackageTreeJSONOutDir, basePackageTreeJSONOutDirV2) + }) } -//nolint:paralleltest // resourceDocsFromRegistryCmd relies on global state. func TestGenerateDocsAllPackage(t *testing.T) { + t.Parallel() // Set up the correct directory structure: registryDir := t.TempDir() util.WriteFile(t, @@ -106,6 +127,6 @@ version: v2.6.1 "--registryDir", registryDir, }) require.NoError(t, cmd.Execute()) - t.Run("docs", func(t *testing.T) { util.AssertDirEqual(t, baseDocsOutDir) }) - t.Run("tree", func(t *testing.T) { util.AssertDirEqual(t, basePackageTreeJSONOutDir) }) + t.Run("docs", func(t *testing.T) { t.Parallel(); util.AssertDirEqual(t, baseDocsOutDir) }) + t.Run("tree", func(t *testing.T) { t.Parallel(); util.AssertDirEqual(t, basePackageTreeJSONOutDir) }) } diff --git a/tools/resourcedocsgen/cmd/metadata.go b/tools/resourcedocsgen/cmd/metadata.go index ab342480df..e8bd5a162b 100644 --- a/tools/resourcedocsgen/cmd/metadata.go +++ b/tools/resourcedocsgen/cmd/metadata.go @@ -20,6 +20,7 @@ import ( "fmt" "io" "net/http" + "net/url" "slices" "strings" "time" @@ -31,6 +32,7 @@ import ( "github.com/pulumi/pulumi/sdk/v3/go/common/util/contract" "github.com/pulumi/registry/tools/resourcedocsgen/pkg" "github.com/spf13/cobra" + "github.com/spf13/pflag" "golang.org/x/text/cases" "golang.org/x/text/language" ) @@ -45,7 +47,7 @@ var featuredPackages = []string{ } func PackageMetadataCmd() *cobra.Command { - var repoSlug repoSlug + var repoSlug optional[repoSlug, *repoSlug] var providerName string var categoryStr string var component bool @@ -55,63 +57,85 @@ func PackageMetadataCmd() *cobra.Command { var version string var metadataDir string var packageDocsDir string + var schemaFileURL string + var indexFileURL string + var installationConfigurationURL string cmd := &cobra.Command{ Use: "metadata ", Short: "Generate package metadata from Pulumi schema", RunE: func(cmd *cobra.Command, args []string) error { - if schemaFile == "" && providerName == "" { - providerName = strings.Replace(repoSlug.name, "pulumi-", "", -1) + if repoSlug.isSet && schemaFile == "" && providerName == "" { + providerName = strings.Replace(repoSlug.v.name, "pulumi-", "", -1) } - mainSpec, err := readRemoteSchemaFile( - inferSchemaFileURL(providerName, schemaFile, repoSlug.String(), version), - repoSlug) + publishedDate := time.Now() + + if schemaFileURL == "" { + if !repoSlug.isSet { + return errors.New("repoSlug is required unless schemaFileURL is passed") + } + schemaFileURL = inferSchemaFileURL(providerName, schemaFile, repoSlug.String(), version) + // try and get the version release data using the github releases API + // + // We only attempt to infer the date when schemaFileURL is missing, implying we are + // sourcing this from a GH repo. + if pd, ok, err := inferPublishDate(repoSlug.v, version); err != nil { + return errors.WithMessage(err, "failed to infer publish date") + } else if ok { + publishedDate = pd + } + } else { + if schemaFile != "" { + glog.Warning("schemaFile was ignored - schemaFileURL takes precedence") + } + } + mainSpec, err := readRemoteSchemaFile(schemaFileURL, repoSlug.v) if err != nil { return errors.WithMessage(err, "unable to read remote schema file") } - // try and get the version release data using the github releases API - tags, err := getGitHubTags(repoSlug.String()) - if err != nil { - return errors.Wrap(err, "github tags") + // Unify the version passed in (if any) with the version on the schema (if any). + switch { + // If we don't have any version, then error. + case mainSpec.Version == "" && version == "": + return fmt.Errorf("No version available for package %s", providerName) + // Version has been specified, so use it. + // + // This can override mainSpec.Version. + case version != "": + mainSpec.Version = version + // Infer the version from the schema. + default: + contract.Assertf(mainSpec.Version != "", "impossible - version was checked earlier") + version = mainSpec.Version } - var commitDetails string - for _, tag := range tags { - if tag.Name == version { - commitDetails = tag.Commit.URL - break + // Unify the repo passed in (if any) with the version in the schema (if any). + switch { + case mainSpec.Repository == "" && !repoSlug.isSet: + return fmt.Errorf("No repository available for package %s", providerName) + case repoSlug.isSet: + mainSpec.Repository = "https://github.com/" + repoSlug.String() + default: + r, err := url.Parse(mainSpec.Repository) + if err == nil { + err = repoSlug.Set(r.Path) } - } - - publishedDate := time.Now() - if commitDetails != "" { - var commit pkg.GitHubCommit - // now let's make a request to the specific commit to get the date - commitResp, err := http.Get(commitDetails) //nolint:gosec if err != nil { - return fmt.Errorf("getting release info for %s: %w", repoSlug, err) + return errors.Wrapf(err, "parsing repo url %q from schema", mainSpec.Repository) } - - defer commitResp.Body.Close() - err = json.NewDecoder(commitResp.Body).Decode(&commit) - if err != nil { - return fmt.Errorf("constructing commit information for %s: %w", repoSlug, err) + if r.Hostname() != "github.com" { + return fmt.Errorf("mainSpec.Repository host must be from GH, found %q", r.Hostname()) } - publishedDate = commit.Commit.Author.Date - } - - mainSpec.Version = version - - if mainSpec.Repository == "" { - // we already know the repo slug so we can reconstruct the repository name using that - mainSpec.Repository = "https://github.com/" + repoSlug.String() + if err := repoSlug.Set(r.Path); err != nil { + return errors.Wrapf(err, "parsing repo slug from %q", mainSpec.Repository) + } } status := pkg.PackageStatusGA - if strings.HasPrefix(version, "v0.") { + if strings.HasPrefix(version, "v0.") || strings.HasPrefix(version, "0.") { status = pkg.PackageStatusPublicPreview } @@ -151,14 +175,8 @@ func PackageMetadataCmd() *cobra.Command { case mainSpec.Publisher != "": publisherName = mainSpec.Publisher default: - contract.Assertf(repoSlug.owner != "", "repoSlug.owner is non-empty by construction") - publisherName = cases.Title(language.Und, cases.NoLower).String(repoSlug.owner) - } - - cleanSchemaFilePath := func(s string) string { - s = strings.ReplaceAll(s, "../", "") - s = strings.ReplaceAll(s, "pulumi-"+mainSpec.Name, "") - return s + contract.Assertf(repoSlug.v.owner != "", "repoSlug.owner is non-empty by construction") + publisherName = cases.Title(language.Und, cases.NoLower).String(repoSlug.v.owner) } pm := pkg.PackageMeta{ @@ -168,9 +186,8 @@ func PackageMetadataCmd() *cobra.Command { Publisher: publisherName, Title: title, - RepoURL: mainSpec.Repository, - SchemaFilePath: cleanSchemaFilePath(schemaFile), - + RepoURL: mainSpec.Repository, + SchemaFileURL: schemaFileURL, PackageStatus: status, UpdatedOn: publishedDate.Unix(), Version: version, @@ -185,11 +202,6 @@ func PackageMetadataCmd() *cobra.Command { return errors.Wrap(err, "generating package metadata") } - if metadataDir == "" { - // if the user hasn't specified an metadataDir, we will default to - // the path within the registry folder. - metadataDir = "themes/default/data/registry/packages" - } metadataFileName := mainSpec.Name + ".yaml" if err := pkg.EmitFile(metadataDir, metadataFileName, b); err != nil { return errors.Wrap(err, "writing metadata file") @@ -201,14 +213,24 @@ func PackageMetadataCmd() *cobra.Command { packageDocsDir = "themes/default/content/registry/packages/" + mainSpec.Name } - requiredFiles := []string{ - "_index.md", - "installation-configuration.md", + requiredFiles := []struct { + name, url string + }{ + {"_index.md", indexFileURL}, + {"installation-configuration.md", installationConfigurationURL}, } + for _, requiredFile := range requiredFiles { - requiredFilePath := fmt.Sprintf("https://raw.githubusercontent.com/%s/%s/docs/%s", - repoSlug, version, requiredFile) - content, err := readRemoteFile(requiredFilePath, repoSlug.owner) + docsURLOrDefault := func(specified, name string) string { + if specified != "" { + return specified + } + return "https://raw.githubusercontent.com/" + repoSlug.String() + "/" + version + "/docs/" + name + } + + url := docsURLOrDefault(requiredFile.url, requiredFile.name) + + content, err := readRemoteFile(url, repoSlug.v.owner) if err != nil { return err } @@ -221,15 +243,15 @@ func PackageMetadataCmd() *cobra.Command { if rest, ok := bytes.CutPrefix(bytes.TrimLeft(content, "\n\t\r "), []byte("---\n")); ok { content = append([]byte(`--- -# WARNING: this file was fetched from `+requiredFilePath+` +# WARNING: this file was fetched from `+requiredFile.url+` # Do not edit by hand unless you're certain you know what you are doing! `), rest...) } else { return fmt.Errorf(`expected file %s to start with YAML front-matter ("---\n"), found leading %q`, - requiredFilePath, strings.Split(string(content), "\n")[0]) + requiredFile.url, strings.Split(string(content), "\n")[0]) } - if err := pkg.EmitFile(packageDocsDir, requiredFile, content); err != nil { + if err := pkg.EmitFile(packageDocsDir, requiredFile.name, content); err != nil { return errors.Wrap(err, fmt.Sprintf("writing %s file", requiredFile)) } } @@ -241,6 +263,12 @@ func PackageMetadataCmd() *cobra.Command { cmd.Flags().Var(&repoSlug, "repoSlug", "The repository slug e.g. pulumi/pulumi-provider") cmd.Flags().StringVar(&providerName, "providerName", "", "The name of the provider e.g. aws, aws-native. "+ "Required when there is no schemaFile flag specified.") + cmd.Flags().StringVar(&schemaFileURL, "schemaFileURL", "", + `The URL from which the schema can be retrieved. + +schemaFileURL takes precedence over schemaFile.`) + cmd.Flags().StringVar(&indexFileURL, "indexFileURL", "", + `The URL from which the docs/_index.md file can be retrieved.`) cmd.Flags().StringVarP(&schemaFile, "schemaFile", "s", "", "Relative path to the schema.json file from the root of the repository. If no schemaFile is specified,"+ " then providerName is required so the schemaFile path can "+ @@ -254,16 +282,13 @@ func PackageMetadataCmd() *cobra.Command { "The display name of the package. If omitted, the name of the package will be used") cmd.Flags().BoolVar(&component, "component", false, "Whether or not this package is a component and not a provider") - cmd.Flags().StringVar(&metadataDir, "metadataDir", "", + cmd.Flags().StringVar(&metadataDir, "metadataDir", "themes/default/data/registry/packages", "The location to save the metadata - this will default to the folder "+ "structure that the registry expects (themes/default/data/registry/packages)") cmd.Flags().StringVar(&packageDocsDir, "packageDocsDir", "", "The location to save the package docs - this will default to the folder "+ "structure that the registry expects (themes/default/content/registry/packages)") - contract.AssertNoErrorf(cmd.MarkFlagRequired("version"), "could not find version") - contract.AssertNoErrorf(cmd.MarkFlagRequired("repoSlug"), "could not find repoSlug") - return cmd } @@ -403,6 +428,27 @@ func getGitHubTags(repoSlug string) ([]pkg.GitHubTag, error) { return tags, nil } +// optional makes a pflag.Value optional +type optional[V any, T interface { + *V + pflag.Value +}] struct { + v V + isSet bool +} + +func (s *optional[U, T]) String() string { return T(&s.v).String() } + +func (s *optional[U, T]) Set(input string) error { + if input == "" { + return nil + } + s.isSet = true + return T(&s.v).Set(input) +} + +func (s *optional[U, T]) Type() string { return T(&s.v).Type() } + type repoSlug struct{ name, owner string } func (s repoSlug) String() string { return s.owner + "/" + s.name } @@ -475,3 +521,38 @@ func readRemoteSchemaFile(schemaFileURL string, repo repoSlug) (*pschema.Package } return spec, nil } + +func inferPublishDate(repo repoSlug, version string) (time.Time, bool, error) { + // try and get the version release data using the github releases API + tags, err := getGitHubTags(repo.String()) + if err != nil { + return time.Time{}, false, errors.Wrap(err, "github tags") + } + + var commitDetails string + for _, tag := range tags { + if tag.Name == version { + commitDetails = tag.Commit.URL + break + } + } + + if commitDetails == "" { + return time.Time{}, false, nil + } + + var commit pkg.GitHubCommit + // now let's make a request to the specific commit to get the date + commitResp, err := http.Get(commitDetails) //nolint:gosec + if err != nil { + return time.Time{}, false, fmt.Errorf("getting release info for %s: %w", repo, err) + } + + defer commitResp.Body.Close() + err = json.NewDecoder(commitResp.Body).Decode(&commit) + if err != nil { + return time.Time{}, false, fmt.Errorf("constructing commit information for %s: %w", repo, err) + } + + return commit.Commit.Author.Date, true, nil +} diff --git a/tools/resourcedocsgen/cmd/testdata/TestMetadataBridgedProvider/metadata/random.yaml.golden b/tools/resourcedocsgen/cmd/testdata/TestMetadataBridgedProvider/metadata/random.yaml.golden index f139839cc1..7029569ef2 100644 --- a/tools/resourcedocsgen/cmd/testdata/TestMetadataBridgedProvider/metadata/random.yaml.golden +++ b/tools/resourcedocsgen/cmd/testdata/TestMetadataBridgedProvider/metadata/random.yaml.golden @@ -2,13 +2,12 @@ category: Utility component: false description: A Pulumi package to safely use randomness in Pulumi programs. featured: false -logo_url: "" name: random native: false package_status: ga publisher: Pulumi repo_url: https://github.com/pulumi/pulumi-random -schema_file_path: provider/cmd/pulumi-resource-random/schema.json +schema_file_url: https://raw.githubusercontent.com/pulumi/pulumi-random/v4.16.7/provider/cmd/pulumi-resource-random/schema.json title: random updated_on: 1729058626 version: v4.16.7 diff --git a/tools/resourcedocsgen/cmd/testdata/TestMetadataComponentProvider/metadata/aws-apigateway.yaml.golden b/tools/resourcedocsgen/cmd/testdata/TestMetadataComponentProvider/metadata/aws-apigateway.yaml.golden index de7ed12087..19d1c4151b 100644 --- a/tools/resourcedocsgen/cmd/testdata/TestMetadataComponentProvider/metadata/aws-apigateway.yaml.golden +++ b/tools/resourcedocsgen/cmd/testdata/TestMetadataComponentProvider/metadata/aws-apigateway.yaml.golden @@ -8,7 +8,7 @@ native: false package_status: ga publisher: Pulumi repo_url: https://github.com/pulumi/pulumi-aws-apigateway -schema_file_path: schema.yaml +schema_file_url: https://raw.githubusercontent.com/pulumi/pulumi-aws-apigateway/v2.6.1/schema.yaml title: AWS API Gateway updated_on: 1727876192 version: v2.6.1 diff --git a/tools/resourcedocsgen/cmd/testdata/TestMetadataNativeProvider/metadata/command.yaml.golden b/tools/resourcedocsgen/cmd/testdata/TestMetadataNativeProvider/metadata/command.yaml.golden index 6aef763e7f..28db590675 100644 --- a/tools/resourcedocsgen/cmd/testdata/TestMetadataNativeProvider/metadata/command.yaml.golden +++ b/tools/resourcedocsgen/cmd/testdata/TestMetadataNativeProvider/metadata/command.yaml.golden @@ -9,7 +9,7 @@ native: true package_status: ga publisher: Pulumi repo_url: https://github.com/pulumi/pulumi-command -schema_file_path: provider/cmd/pulumi-resource-command/schema.json +schema_file_url: https://raw.githubusercontent.com/pulumi/pulumi-command/v1.0.0/provider/cmd/pulumi-resource-command/schema.json title: Command updated_on: 1719590084 version: v1.0.0 diff --git a/tools/resourcedocsgen/internal/tests/util/fs.go b/tools/resourcedocsgen/internal/tests/util/fs.go index 23ee44a4ca..7c833ff92d 100644 --- a/tools/resourcedocsgen/internal/tests/util/fs.go +++ b/tools/resourcedocsgen/internal/tests/util/fs.go @@ -15,6 +15,7 @@ package util import ( + "errors" "io/fs" "os" "path/filepath" @@ -22,6 +23,7 @@ import ( "testing" "github.com/hexops/autogold/v2" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -40,33 +42,75 @@ import ( func AssertDirEqual(t *testing.T, root string) { var structure []string require.NoError(t, filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { - t.Logf("examining %s", path) - if path == root { - return nil - } - if err != nil { + if path == root || err != nil { return err } pathName := strings.TrimLeft(strings.TrimPrefix(path, root), "/") require.NotEmpty(t, pathName, "internal error - pathName should not be empty") - t.Logf("named path - %s", pathName) structure = append(structure, pathName) if d.IsDir() { return nil } - t.Logf("reading file %s", pathName) content, err := os.ReadFile(path) - if err != nil { - return err - } + require.NoError(t, err, "could not read walked file") t.Run(pathName, func(t *testing.T) { autogold.ExpectFile(t, autogold.Raw(content)) }) return nil })) t.Run("directory-structure", (func(t *testing.T) { autogold.ExpectFile(t, structure) })) } +// AssertDirsEqual asserts that each file located under root is byte-for-byte identical +// with another directory, and that the directory structures match. +// +// If you just want to assert that a directory structure is unchanged or show updates, see +// [AssertDirEqual]. AssertDirsEqual is about showing that two already written out +// directories are equivalent. +func AssertDirsEqual(t *testing.T, expected, actual string) { + var expectedStructure, actualStructure []string + require.NoError(t, filepath.WalkDir(expected, func(path string, d fs.DirEntry, err error) error { + if path == expected || err != nil { + return err + } + + pathName := strings.TrimLeft(strings.TrimPrefix(path, expected), "/") + require.NotEmpty(t, pathName, "internal error - pathName should not be empty") + expectedStructure = append(expectedStructure, pathName) + if d.IsDir() { + return nil + } + + actualPath := filepath.Join(actual, pathName) + actualContent, err := os.ReadFile(actualPath) + if errors.Is(err, os.ErrNotExist) { + assert.Fail(t, "Missing expected file [%s/]%s", pathName, actual) + return nil + } + require.NoError(t, err, "Could not open existing file %q", actualPath) + + expectedContent, err := os.ReadFile(path) + require.NoError(t, err, "walked file should exist") + assert.Equalf(t, string(expectedContent), string(actualContent), "File %s doesn't match", pathName) + + return nil + })) + + require.NoError(t, filepath.WalkDir(actual, func(path string, d fs.DirEntry, err error) error { + if path == actual || err != nil { + return err + } + + pathName := strings.TrimLeft(strings.TrimPrefix(path, actual), "/") + require.NotEmpty(t, pathName, "internal error - pathName should not be empty") + actualStructure = append(actualStructure, pathName) + return nil + })) + + assert.ElementsMatch(t, expectedStructure, actualStructure, + "Directory structure does not match") +} + func WriteFile(t *testing.T, path, contents string) { require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o700)) require.NoError(t, os.WriteFile(path, []byte(contents), 0o600)) diff --git a/tools/resourcedocsgen/pkg/metadata.go b/tools/resourcedocsgen/pkg/metadata.go index a8869cb3ee..ad68361940 100644 --- a/tools/resourcedocsgen/pkg/metadata.go +++ b/tools/resourcedocsgen/pkg/metadata.go @@ -45,11 +45,13 @@ type PackageMeta struct { // Title is the package's display-friendly name. Title string `json:"title"` Description string `json:"description"` - LogoURL string `json:"logo_url"` - RepoURL string `json:"repo_url"` - // SchemaFilePath is the path to the package's schema file (json or yaml) - // relative to the root of that package's repo. - SchemaFilePath string `json:"schema_file_path"` + + // A publicly available URL to retrieve the schema from. + // + // LogoURL is derived from the provider schema. + LogoURL string `json:"logo_url,omitempty"` + RepoURL string `json:"repo_url"` + SchemaFileURL string `json:"schema_file_url"` // A publicly available URL to retrieve the schema from. UpdatedOn int64 `json:"updated_on"` Publisher string `json:"publisher"` @@ -65,4 +67,10 @@ type PackageMeta struct { // Component indicates if the package is a component and not // a provider. Component bool `json:"component"` + + // SchemaFilePath is the path to the package's schema file (json or yaml) + // relative to the root of that package's repo. + // + // Deprecated: Prefer to use SchemaFileURL. + SchemaFilePath string `json:"schema_file_path,omitempty"` }