diff --git a/pkg/tfgen/docs.go b/pkg/tfgen/docs.go index d11cbcb63..b23268051 100644 --- a/pkg/tfgen/docs.go +++ b/pkg/tfgen/docs.go @@ -250,6 +250,7 @@ func getDocsForResource(g *Generator, source DocsSource, kind DocKind, panic("unknown docs kind") } + entitiesCount++ if err != nil { return entityDocs{}, fmt.Errorf("get docs for token %s: %w", rawname, err) } @@ -1709,6 +1710,7 @@ func (g *Generator) convertHCL(hcl, path, exampleTitle string, languages []strin if isCompleteFailure { hclAllLangsConversionFailures++ + hclConversionAttempts++ if exampleTitle == "" { g.warn(fmt.Sprintf("unable to convert HCL example for Pulumi entity '%s': %v. The example will be dropped "+ "from any generated docs or SDKs.", path, err)) @@ -1752,6 +1754,8 @@ func (g *Generator) convertHCL(hcl, path, exampleTitle string, languages []strin err = nil } + hclConversionAttempts++ + return result.String(), nil } @@ -1828,6 +1832,7 @@ func cleanupDoc( g.debug("Cleaning up description text for [%v]", name) cleanupText, elided := reformatText(infoCtx, doc.Description, footerLinks) + totalDescriptions++ if elided { g.debug("Found in the description. Attempting to extract examples from the description and " + "reformat examples only.") diff --git a/pkg/tfgen/metrics.go b/pkg/tfgen/metrics.go index 94694d196..385930887 100644 --- a/pkg/tfgen/metrics.go +++ b/pkg/tfgen/metrics.go @@ -16,11 +16,13 @@ package tfgen import ( "fmt" + schemaTools "github.com/pulumi/schema-tools/pkg" ) var ( ignoredDocHeaders = make(map[string]int) + totalDescriptions int elidedDescriptions int // i.e., we discard the entire description, including examples elidedDescriptionsOnly int // we discarded the description proper, but were able to preserve the examples elidedArguments int @@ -28,6 +30,7 @@ var ( elidedAttributes int unexpectedSnippets int + hclConversionAttempts int // conversion attempts hclAllLangsConversionFailures int // examples that failed to convert in any language // examples that failed to convert in one, but not all, languages. This is less severe impact because users will @@ -43,11 +46,19 @@ var ( argumentDescriptionsFromAttributes int // General metrics: + entitiesCount int entitiesMissingDocs int schemaStats schemaTools.PulumiSchemaStats ) +func getPercentage(top int, bottom int) float64 { + if bottom == 0 { + return 100 + } + return float64(top) / float64(bottom) * 100 +} + // printDocStats outputs metrics relating to document parsing and conversion func printDocStats() { fmt.Println("") @@ -56,43 +67,88 @@ func printDocStats() { fmt.Printf("\t%d total resources containing %d total inputs.\n", schemaStats.Resources.TotalResources, schemaStats.Resources.TotalInputProperties) fmt.Printf("\t%d total functions.\n", schemaStats.Functions.TotalFunctions) - fmt.Printf("\t%d entities are missing docs entirely because they could not be found in the upstream provider.\n", - entitiesMissingDocs) + fmt.Printf( + "\t%d of %d (%.2f%%) entities are missing docs entirely because they could not be found in the upstream provider.\n", + entitiesMissingDocs, entitiesCount, getPercentage(entitiesMissingDocs, entitiesCount), + ) fmt.Println("") fmt.Println("Description metrics:") - fmt.Printf("\t%d entity descriptions contained an reference and were dropped, including examples.\n", - elidedDescriptions) - fmt.Printf("\t%d entity descriptions contained an reference and were dropped, but examples were preserved.\n", - elidedDescriptionsOnly) + fmt.Printf( + "\t%d of %d (%.2f%%) entity descriptions contained an reference and were dropped, including examples.\n", + elidedDescriptions, totalDescriptions, getPercentage(elidedDescriptions, totalDescriptions), + ) + fmt.Printf( + "\t%d of %d (%.2f%%) entity descriptions contained an reference and were dropped, "+ + "but examples were preserved.\n", + elidedDescriptionsOnly, + totalDescriptions, + getPercentage(elidedDescriptionsOnly, totalDescriptions), + ) fmt.Println("") fmt.Println("Example conversion metrics:") - fmt.Printf("\t%d HCL examples failed to convert in all languages\n", hclAllLangsConversionFailures) - fmt.Printf("\t%d HCL examples were converted in at least one language but failed to convert to TypeScript\n", - hclTypeScriptPartialConversionFailures) - fmt.Printf("\t%d HCL examples were converted in at least one language but failed to convert to Python\n", - hclPythonPartialConversionFailures) - fmt.Printf("\t%d HCL examples were converted in at least one language but failed to convert to Go\n", - hclGoPartialConversionFailures) - fmt.Printf("\t%d HCL examples were converted in at least one language but failed to convert to C#\n", - hclCSharpPartialConversionFailures) + fmt.Printf( + "\t%d of %d (%.2f%%) HCL examples failed to convert in all languages\n", + hclAllLangsConversionFailures, + hclConversionAttempts, + getPercentage(hclAllLangsConversionFailures, hclConversionAttempts), + ) + fmt.Printf( + "\t%d of %d (%.2f%%) HCL examples were converted in at least one language but failed to convert to TypeScript\n", + hclTypeScriptPartialConversionFailures, + hclConversionAttempts, + getPercentage(hclTypeScriptPartialConversionFailures, hclConversionAttempts), + ) + fmt.Printf( + "\t%d of %d (%.2f%%) HCL examples were converted in at least one language but failed to convert to Python\n", + hclPythonPartialConversionFailures, + hclConversionAttempts, + getPercentage(hclPythonPartialConversionFailures, hclConversionAttempts), + ) + fmt.Printf( + "\t%d of %d (%.2f%%) HCL examples were converted in at least one language but failed to convert to Go\n", + hclGoPartialConversionFailures, + hclConversionAttempts, + getPercentage(hclGoPartialConversionFailures, hclConversionAttempts), + ) + fmt.Printf( + "\t%d of %d (%.2f%%) HCL examples were converted in at least one language but failed to convert to C#\n", + hclCSharpPartialConversionFailures, + hclConversionAttempts, + getPercentage(hclCSharpPartialConversionFailures, hclConversionAttempts), + ) fmt.Printf("\t%d entity document sections contained unexpected HCL code snippets. Examples will be converted, "+ "but may not display correctly in the registry, e.g. lacking tabs.\n", unexpectedSnippets) fmt.Println("") fmt.Println("Argument metrics:") fmt.Printf("\t%d argument descriptions were parsed from the upstream docs\n", totalArgumentsFromDocs) - fmt.Printf("\t%d top-level input property descriptions came from an upstream attribute (as opposed to an argument). "+ - "Nested arguments are not included in this count.\n", argumentDescriptionsFromAttributes) - fmt.Printf("\t%d arguments contained an reference and had their descriptions dropped.\n", - elidedArguments) - fmt.Printf("\t%d nested arguments contained an reference and had their descriptions dropped.\n", - elidedNestedArguments) - //nolint:lll - fmt.Printf("\t%d of %d resource inputs (%.2f%%) are missing descriptions in the schema\n", - schemaStats.Resources.InputPropertiesMissingDescriptions, schemaStats.Resources.TotalInputProperties, - float64(schemaStats.Resources.InputPropertiesMissingDescriptions)/float64(schemaStats.Resources.TotalInputProperties)*100) + fmt.Printf( + "\t%d of %d (%.2f%%) top-level input property descriptions came from an upstream attribute "+ + "(as opposed to an argument). Nested arguments are not included in this count.\n", + argumentDescriptionsFromAttributes, + totalArgumentsFromDocs, + getPercentage(argumentDescriptionsFromAttributes, totalArgumentsFromDocs), + ) + fmt.Printf("\t%d of %d (%.2f%%) arguments contained an reference and had their descriptions dropped.\n", + elidedArguments, totalArgumentsFromDocs, getPercentage(elidedArguments, totalArgumentsFromDocs)) + fmt.Printf( + "\t%d of %d (%.2f%%) nested arguments contained an reference and had their descriptions dropped.\n", + elidedNestedArguments, + totalArgumentsFromDocs, + getPercentage( + elidedNestedArguments, totalArgumentsFromDocs), + ) + fmt.Printf( + "\t%d of %d (%.2f%%) resource inputs are missing descriptions in the schema\n", + schemaStats.Resources.InputPropertiesMissingDescriptions, + schemaStats.Resources.TotalInputProperties, + getPercentage( + schemaStats.Resources.InputPropertiesMissingDescriptions, + schemaStats.Resources.TotalInputProperties, + ), + ) fmt.Println("") fmt.Println("Attribute metrics:")