Extend resource configuration API to include resource metadata

go.mod

@@ -17,6 +17,7 @@ require (
 	github.com/spf13/afero v1.8.0
 	github.com/zclconf/go-cty v1.10.0
 	golang.org/x/tools v0.1.10-0.20220218145154-897bd77cd717
+	gopkg.in/yaml.v2 v2.4.0
 	k8s.io/api v0.24.0
 	k8s.io/apimachinery v0.24.0
 	k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9
@@ -120,7 +121,6 @@ require (
 	google.golang.org/grpc v1.46.0 // indirect
 	google.golang.org/protobuf v1.28.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
 	k8s.io/client-go v0.24.0 // indirect
 	k8s.io/component-base v0.24.0 // indirect

pkg/config/provider.go

@@ -11,6 +11,7 @@ import (
 	tfjson "github.com/hashicorp/terraform-json"
 	"github.com/pkg/errors"
 
+	"github.com/upbound/upjet/pkg/registry"
 	conversiontfjson "github.com/upbound/upjet/pkg/types/conversion/tfjson"
 )
 
@@ -95,10 +96,6 @@ type Provider struct {
 	// resource name.
 	Resources map[string]*Resource
 
-	// ProviderMetadataPath is the scraped provider metadata file path
-	// from Terraform registry
-	ProviderMetadataPath string
-
 	// resourceConfigurators is a map holding resource configurators where key
 	// is Terraform resource name.
 	resourceConfigurators map[string]ResourceConfiguratorChain
@@ -153,7 +150,7 @@ func WithDefaultResourceOptions(opts ...ResourceOption) ProviderOption {
 // NewProvider builds and returns a new Provider from provider
 // tfjson schema, that is generated using Terraform CLI with:
 // `terraform providers schema --json`
-func NewProvider(schema []byte, prefix string, modulePath string, metadataPath string, opts ...ProviderOption) *Provider {
+func NewProvider(schema []byte, prefix string, modulePath string, metadata []byte, opts ...ProviderOption) *Provider {
 	ps := tfjson.ProviderSchemas{}
 	if err := ps.UnmarshalJSON(schema); err != nil {
 		panic(err)
@@ -179,7 +176,6 @@ func NewProvider(schema []byte, prefix string, modulePath string, metadataPath s
 			".+",
 		},
 		Resources:             map[string]*Resource{},
-		ProviderMetadataPath:  metadataPath,
 		resourceConfigurators: map[string]ResourceConfiguratorChain{},
 	}
 
@@ -203,9 +199,26 @@ func NewProvider(schema []byte, prefix string, modulePath string, metadataPath s
 
 		p.Resources[name] = DefaultResource(name, terraformResource, p.DefaultResourceOptions...)
 	}
+	if err := p.loadMetadata(metadata); err != nil {
+		panic(errors.Wrap(err, "cannot load provider metadata"))
+	}
 	return p
 }
 
+func (p *Provider) loadMetadata(metadata []byte) error {
+	if len(metadata) == 0 {
+		return nil
+	}
+	providerMetadata, err := registry.NewProviderMetadataFromFile(metadata)
+	if err != nil {
+		return errors.Wrap(err, "cannot load provider metadata")
+	}
+	for name, r := range p.Resources {
+		r.MetaResource = providerMetadata.Resources[name]
+	}
+	return nil
+}
+
 // AddResourceConfigurator adds resource specific configurators.
 func (p *Provider) AddResourceConfigurator(resource string, c ResourceConfiguratorFn) { //nolint:interfacer
 	// Note(turkenh): nolint reasoning - easier to provide a function without

pkg/config/resource.go

@@ -18,6 +18,8 @@ import (
 	"github.com/crossplane/crossplane-runtime/pkg/fieldpath"
 	"github.com/crossplane/crossplane-runtime/pkg/reconciler/managed"
 	xpresource "github.com/crossplane/crossplane-runtime/pkg/resource"
+
+	"github.com/upbound/upjet/pkg/registry"
 )
 
 // SetIdentifierArgumentsFn sets the name of the resource in Terraform attributes map,
@@ -290,4 +292,8 @@ type Resource struct {
 
 	// LateInitializer configuration to control late-initialization behaviour
 	LateInitializer LateInitializer
+
+	// MetaResource is the metadata associated with the resource scraped from
+	// the Terraform registry.
+	MetaResource *registry.Resource
 }

pkg/pipeline/example.go

@@ -13,11 +13,11 @@ import (
 	"strings"
 
 	"github.com/crossplane/crossplane-runtime/pkg/fieldpath"
+	xpmeta "github.com/crossplane/crossplane-runtime/pkg/meta"
 	"github.com/pkg/errors"
 	"sigs.k8s.io/yaml"
 
 	"github.com/upbound/upjet/pkg/config"
-	"github.com/upbound/upjet/pkg/resource/json"
 	tjtypes "github.com/upbound/upjet/pkg/types"
 )
 
@@ -32,55 +32,20 @@ type pavedWithManifest struct {
 	refsResolved bool
 }
 
-// ResourceExample represents the scraped example HCL configuration
-// for a Terraform resource
-type ResourceExample struct {
-	Manifest   string            `yaml:"manifest"`
-	References map[string]string `yaml:"references,omitempty"`
-}
-
-// Resource represents the scraped metadata for a Terraform resource
-type Resource struct {
-	SubCategory      string            `yaml:"subCategory"`
-	Description      string            `yaml:"description,omitempty"`
-	Name             string            `yaml:"name"`
-	TitleName        string            `yaml:"titleName"`
-	Examples         []ResourceExample `yaml:"examples,omitempty"`
-	ArgumentDocs     map[string]string `yaml:"argumentDocs"`
-	ImportStatements []string          `yaml:"importStatements"`
-}
-
-// ProviderMetadata metadata for a Terraform native provider
-type ProviderMetadata struct {
-	Name      string               `yaml:"name"`
-	Resources map[string]*Resource `yaml:"resources"`
-}
-
-// NewProviderMetadataFromFile loads metadata from the specified YAML-formatted file
-func NewProviderMetadataFromFile(path string) (*ProviderMetadata, error) {
-	buff, err := ioutil.ReadFile(filepath.Clean(path))
-	if err != nil {
-		return nil, errors.Wrapf(err, "failed to read metadata file %q", path)
-	}
-
-	metadata := &ProviderMetadata{}
-	return metadata, errors.Wrap(yaml.Unmarshal(buff, metadata), "failed to unmarshal provider metadata")
-}
-
 // ExampleGenerator represents a pipeline for generating example manifests.
 // Generates example manifests for Terraform resources under examples-generated.
 type ExampleGenerator struct {
-	rootDir      string
-	resourceMeta map[string]*Resource
-	resources    map[string]*pavedWithManifest
+	rootDir        string
+	configResource map[string]*config.Resource
+	resources      map[string]*pavedWithManifest
 }
 
 // NewExampleGenerator returns a configured ExampleGenerator
-func NewExampleGenerator(rootDir string, resourceMeta map[string]*Resource) *ExampleGenerator {
+func NewExampleGenerator(rootDir string, configResource map[string]*config.Resource) *ExampleGenerator {
 	return &ExampleGenerator{
-		rootDir:      rootDir,
-		resourceMeta: resourceMeta,
-		resources:    make(map[string]*pavedWithManifest),
+		rootDir:        rootDir,
+		configResource: configResource,
+		resources:      make(map[string]*pavedWithManifest),
 	}
 }
 
@@ -169,27 +134,27 @@ func (eg *ExampleGenerator) resolveReferences(params map[string]interface{}) err
 
 // Generate generates an example manifest for the specified Terraform resource.
 func (eg *ExampleGenerator) Generate(group, version string, r *config.Resource, fieldTransformations map[string]tjtypes.Transformation) error {
-	rm := eg.resourceMeta[r.Name]
+	rm := eg.configResource[r.Name].MetaResource
 	if rm == nil || len(rm.Examples) == 0 {
 		return nil
 	}
-	var exampleParams map[string]interface{}
-	if err := json.TFParser.Unmarshal([]byte(rm.Examples[0].Manifest), &exampleParams); err != nil {
-		return errors.Wrapf(err, "cannot unmarshal example manifest for resource: %s", r.Name)
-	}
+	exampleParams := rm.Examples[0].Paved.UnstructuredContent()
 	transformFields(exampleParams, r.ExternalName.OmittedFields, fieldTransformations, "")
 
+	metadata := map[string]interface{}{
+		"name": "example",
+	}
+	if len(rm.ExternalName) != 0 {
+		metadata["annotations"] = map[string]string{
+			xpmeta.AnnotationKeyExternalName: rm.ExternalName,
+		}
+	}
 	example := map[string]interface{}{
 		"apiVersion": fmt.Sprintf("%s/%s", group, version),
 		"kind":       r.Kind,
-		"metadata": map[string]interface{}{
-			"name": "example",
-		},
+		"metadata":   metadata,
 		"spec": map[string]interface{}{
 			"forProvider": exampleParams,
-			"providerConfigRef": map[string]interface{}{
-				"name": "example",
-			},
 		},
 	}
 	manifestDir := filepath.Join(eg.rootDir, "examples-generated", strings.ToLower(strings.Split(group, ".")[0]))

pkg/pipeline/run.go

@@ -23,9 +23,9 @@ import (
 	"sort"
 	"strings"
 
-	"github.com/upbound/upjet/pkg/config"
-
 	"github.com/crossplane/crossplane-runtime/pkg/errors"
+
+	"github.com/upbound/upjet/pkg/config"
 )
 
 type terraformedInput struct {
@@ -57,15 +57,6 @@ func Run(pc *config.Provider, rootDir string) { // nolint:gocyclo
 		resourcesGroups[group][resource.Version][name] = resource
 	}
 
-	metaResources := make(map[string]*Resource)
-	if pc.ProviderMetadataPath != "" {
-		providerMetadata, err := NewProviderMetadataFromFile(filepath.Join(rootDir, pc.ProviderMetadataPath))
-		if err != nil {
-			panic(errors.Wrap(err, "cannot read Terraform provider metadata"))
-		}
-		metaResources = providerMetadata.Resources
-	}
-
 	// Add ProviderConfig API package to the list of API version packages.
 	apiVersionPkgList := make([]string, 0)
 	for _, p := range pc.BasePackages.APIVersion {
@@ -77,7 +68,7 @@ func Run(pc *config.Provider, rootDir string) { // nolint:gocyclo
 		controllerPkgList = append(controllerPkgList, filepath.Join(pc.ModulePath, p))
 	}
 	count := 0
-	exampleGen := NewExampleGenerator(rootDir, metaResources)
+	exampleGen := NewExampleGenerator(rootDir, pc.Resources)
 	for group, versions := range resourcesGroups {
 		for version, resources := range versions {
 			var tfResources []*terraformedInput

pkg/registry/resource.go

@@ -0,0 +1,94 @@
+/*
+Copyright 2022 Upbound Inc.
+*/
+
+package registry
+
+import (
+	"github.com/crossplane/crossplane-runtime/pkg/fieldpath"
+	"github.com/pkg/errors"
+	"gopkg.in/yaml.v2"
+)
+
+const (
+	// RandRFC1123Subdomain represents a template variable to be substituted
+	// by the test runner at runtime with a random RFC1123 subdomain string.
+	RandRFC1123Subdomain = "${Rand.RFC1123Subdomain}"
+)
+
+// Dependencies are the example manifests for the dependency resources.
+// Key is formatted as <Terraform resource>.<example name>
+type Dependencies map[string]string
+
+// ResourceExample represents the scraped example HCL configuration
+// for a Terraform resource
+type ResourceExample struct {
+	Name         string            `yaml:"name"`
+	Manifest     string            `yaml:"manifest"`
+	References   map[string]string `yaml:"references,omitempty"`
+	Dependencies Dependencies      `yaml:"dependencies,omitempty"`
+	Paved        fieldpath.Paved   `yaml:"-"`
+}
+
+// Resource represents the scraped metadata for a Terraform resource
+type Resource struct {
+	// SubCategory is the category name under which this Resource resides in
+	// Terraform registry docs. Example:"Key Vault" for Azure Vault resources.
+	// In Terraform docs, resources are grouped (categorized) using this field.
+	SubCategory string `yaml:"subCategory"`
+	// Description is a short description for the resource as it appears in
+	// Terraform registry. Example: "Manages a Key Vault Key." for the
+	// azurerm_key_vault_key resource.
+	// This field is suitable for use in generating CRD Kind documentation.
+	Description string `yaml:"description,omitempty"`
+	// Name is the Terraform name of the resource. Example: azurerm_key_vault_key
+	Name string `yaml:"name"`
+	// Title is the title name of the resource that appears in
+	// the Terraform registry doc page for a Terraform resource.
+	Title string `yaml:"title"`
+	// Examples are the example HCL configuration blocks for the resource
+	// that appear in the resource's registry page. They are in the same
+	// order as they appear on the registry page.
+	Examples []ResourceExample `yaml:"examples,omitempty"`
+	// ArgumentDocs maps resource attributes to their documentation in the
+	// resource's registry page.
+	ArgumentDocs map[string]string `yaml:"argumentDocs"`
+	// ImportStatements are the example Terraform import statements as they
+	// appear in the resource's registry page.
+	// Example: terraform import azurerm_key_vault.example /subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mygroup1/providers/Microsoft.KeyVault/vaults/vault1
+	ImportStatements []string `yaml:"importStatements"`
+	// ExternalName configured for this resource. This allows the
+	// external name used in the generated example manifests to be
+	// overridden for a specific resource via configuration.
+	ExternalName string `yaml:"-"`
+}
+
+// ProviderMetadata metadata for a Terraform native provider
+type ProviderMetadata struct {
+	Name      string               `yaml:"name"`
+	Resources map[string]*Resource `yaml:"resources"`
+}
+
+// NewProviderMetadataFromFile loads metadata from the specified YAML-formatted document
+func NewProviderMetadataFromFile(providerMetadata []byte) (*ProviderMetadata, error) {
+	metadata := &ProviderMetadata{}
+	if err := yaml.Unmarshal(providerMetadata, metadata); err != nil {
+		return nil, errors.Wrap(err, "failed to unmarshal provider metadata")
+	}
+	for name, rm := range metadata.Resources {
+		for j, re := range rm.Examples {
+			if err := re.Paved.UnmarshalJSON([]byte(re.Manifest)); err != nil {
+				return nil, errors.Wrapf(err, "cannot pave example manifest JSON: %s", re.Manifest)
+			}
+			rm.Examples[j] = re
+		}
+		metadata.Resources[name] = rm
+	}
+	return metadata, nil
+}
+
+// SetPathValue sets the field at the specified path to the given value
+// in the example manifest.
+func (re *ResourceExample) SetPathValue(fieldPath string, val interface{}) error {
+	return errors.Wrapf(re.Paved.SetValue(fieldPath, val), "cannot set example manifest path %q to value: %#v", fieldPath, val)
+}