Extend resource configuration API to include resource metadata

Signed-off-by: Alper Rifat Ulucinar <[email protected]>

go.mod

@@ -17,6 +17,7 @@ require (
 	github.com/spf13/afero v1.8.0
 	github.com/zclconf/go-cty v1.10.0
 	golang.org/x/tools v0.1.10-0.20220218145154-897bd77cd717
+	gopkg.in/yaml.v2 v2.4.0
 	k8s.io/api v0.24.0
 	k8s.io/apimachinery v0.24.0
 	k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9
@@ -120,7 +121,6 @@ require (
 	google.golang.org/grpc v1.46.0 // indirect
 	google.golang.org/protobuf v1.28.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
 	k8s.io/client-go v0.24.0 // indirect
 	k8s.io/component-base v0.24.0 // indirect

pkg/config/provider.go

@@ -6,11 +6,13 @@ package config
 
 import (
 	"fmt"
+	"path/filepath"
 	"regexp"
 
 	tfjson "github.com/hashicorp/terraform-json"
 	"github.com/pkg/errors"
 
+	"github.com/upbound/upjet/pkg/meta"
 	conversiontfjson "github.com/upbound/upjet/pkg/types/conversion/tfjson"
 )
 
@@ -99,6 +101,10 @@ type Provider struct {
 	// from Terraform registry
 	ProviderMetadataPath string
 
+	// RootDir of the Crossplane provider repo for code generation and
+	// configuration files.
+	RootDir string
+
 	// resourceConfigurators is a map holding resource configurators where key
 	// is Terraform resource name.
 	resourceConfigurators map[string]ResourceConfiguratorChain
@@ -153,7 +159,7 @@ func WithDefaultResourceOptions(opts ...ResourceOption) ProviderOption {
 // NewProvider builds and returns a new Provider from provider
 // tfjson schema, that is generated using Terraform CLI with:
 // `terraform providers schema --json`
-func NewProvider(schema []byte, prefix string, modulePath string, metadataPath string, opts ...ProviderOption) *Provider {
+func NewProvider(schema []byte, rootDir string, prefix string, modulePath string, metadataPath string, opts ...ProviderOption) *Provider {
 	ps := tfjson.ProviderSchemas{}
 	if err := ps.UnmarshalJSON(schema); err != nil {
 		panic(err)
@@ -180,6 +186,7 @@ func NewProvider(schema []byte, prefix string, modulePath string, metadataPath s
 		},
 		Resources:             map[string]*Resource{},
 		ProviderMetadataPath:  metadataPath,
+		RootDir:               rootDir,
 		resourceConfigurators: map[string]ResourceConfiguratorChain{},
 	}
 
@@ -203,9 +210,27 @@ func NewProvider(schema []byte, prefix string, modulePath string, metadataPath s
 
 		p.Resources[name] = DefaultResource(name, terraformResource, p.DefaultResourceOptions...)
 	}
+	if err := p.loadMetadata(); err != nil {
+		panic(errors.Wrap(err, "cannot load provider metadata"))
+	}
 	return p
 }
 
+func (p *Provider) loadMetadata() error {
+	if p.ProviderMetadataPath == "" {
+		return nil
+	}
+	metadataPath := filepath.Join(p.RootDir, p.ProviderMetadataPath)
+	providerMetadata, err := meta.NewProviderMetadataFromFile(metadataPath)
+	if err != nil {
+		return errors.Wrapf(err, "cannot load provider metadata from file: %s", metadataPath)
+	}
+	for name, r := range p.Resources {
+		r.MetaResource = providerMetadata.Resources[name]
+	}
+	return nil
+}
+
 // AddResourceConfigurator adds resource specific configurators.
 func (p *Provider) AddResourceConfigurator(resource string, c ResourceConfiguratorFn) { //nolint:interfacer
 	// Note(turkenh): nolint reasoning - easier to provide a function without

pkg/config/resource.go

@@ -18,6 +18,8 @@ import (
 	"github.com/crossplane/crossplane-runtime/pkg/fieldpath"
 	"github.com/crossplane/crossplane-runtime/pkg/reconciler/managed"
 	xpresource "github.com/crossplane/crossplane-runtime/pkg/resource"
+
+	"github.com/upbound/upjet/pkg/meta"
 )
 
 // SetIdentifierArgumentsFn sets the name of the resource in Terraform attributes map,
@@ -290,4 +292,8 @@ type Resource struct {
 
 	// LateInitializer configuration to control late-initialization behaviour
 	LateInitializer LateInitializer
+
+	// MetaResource is the metadata associated with the resource scraped from
+	// the Terraform registry.
+	MetaResource *meta.Resource
 }

pkg/meta/resource.go

@@ -0,0 +1,78 @@
+/*
+Copyright 2022 Upbound Inc.
+*/
+
+package meta
+
+import (
+	"io/ioutil"
+	"path/filepath"
+
+	"github.com/crossplane/crossplane-runtime/pkg/fieldpath"
+	"github.com/pkg/errors"
+	"gopkg.in/yaml.v2"
+)
+
+const (
+	RandRFC1123Subdomain = "{{ .Rand.RFC1123Subdomain }}"
+)
+
+// Dependencies are the example manifests for the dependency resources.
+// Key is formatted as <Terraform resource>.<example name>
+type Dependencies map[string]string
+
+// ResourceExample represents the scraped example HCL configuration
+// for a Terraform resource
+type ResourceExample struct {
+	Manifest     string            `yaml:"manifest"`
+	References   map[string]string `yaml:"references,omitempty"`
+	Dependencies Dependencies      `yaml:"dependencies,omitempty"`
+	Paved        fieldpath.Paved   `yaml:"-"`
+}
+
+// Resource represents the scraped metadata for a Terraform resource
+type Resource struct {
+	SubCategory      string            `yaml:"subCategory"`
+	Description      string            `yaml:"description,omitempty"`
+	Name             string            `yaml:"name"`
+	TitleName        string            `yaml:"titleName"`
+	Examples         []ResourceExample `yaml:"examples,omitempty"`
+	ArgumentDocs     map[string]string `yaml:"argumentDocs"`
+	ImportStatements []string          `yaml:"importStatements"`
+	ExternalName     string            `yaml:"-"`
+}
+
+// ProviderMetadata metadata for a Terraform native provider
+type ProviderMetadata struct {
+	Name      string               `yaml:"name"`
+	Resources map[string]*Resource `yaml:"resources"`
+}
+
+// NewProviderMetadataFromFile loads metadata from the specified YAML-formatted file
+func NewProviderMetadataFromFile(path string) (*ProviderMetadata, error) {
+	buff, err := ioutil.ReadFile(filepath.Clean(path))
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to read metadata file %q", path)
+	}
+
+	metadata := &ProviderMetadata{}
+	if err := yaml.Unmarshal(buff, metadata); err != nil {
+		return nil, errors.Wrap(err, "failed to unmarshal provider metadata")
+	}
+	for name, rm := range metadata.Resources {
+		for j, re := range rm.Examples {
+			if err := re.Paved.UnmarshalJSON([]byte(re.Manifest)); err != nil {
+				return nil, errors.Wrapf(err, "cannot pave example manifest JSON: %s", re.Manifest)
+			}
+			rm.Examples[j] = re
+		}
+		metadata.Resources[name] = rm
+	}
+	return metadata, nil
+}
+
+// SetPathValue sets the field at the specified path to the given value
+// in the example manifest.
+func (re *ResourceExample) SetPathValue(fieldPath string, val interface{}) error {
+	return errors.Wrapf(re.Paved.SetValue(fieldPath, val), "cannot set example manifest path %q to value: %#v", fieldPath, val)
+}

pkg/pipeline/example.go

@@ -13,11 +13,11 @@ import (
 	"strings"
 
 	"github.com/crossplane/crossplane-runtime/pkg/fieldpath"
+	xpmeta "github.com/crossplane/crossplane-runtime/pkg/meta"
 	"github.com/pkg/errors"
 	"sigs.k8s.io/yaml"
 
 	"github.com/upbound/upjet/pkg/config"
-	"github.com/upbound/upjet/pkg/resource/json"
 	tjtypes "github.com/upbound/upjet/pkg/types"
 )
 
@@ -32,55 +32,20 @@ type pavedWithManifest struct {
 	refsResolved bool
 }
 
-// ResourceExample represents the scraped example HCL configuration
-// for a Terraform resource
-type ResourceExample struct {
-	Manifest   string            `yaml:"manifest"`
-	References map[string]string `yaml:"references,omitempty"`
-}
-
-// Resource represents the scraped metadata for a Terraform resource
-type Resource struct {
-	SubCategory      string            `yaml:"subCategory"`
-	Description      string            `yaml:"description,omitempty"`
-	Name             string            `yaml:"name"`
-	TitleName        string            `yaml:"titleName"`
-	Examples         []ResourceExample `yaml:"examples,omitempty"`
-	ArgumentDocs     map[string]string `yaml:"argumentDocs"`
-	ImportStatements []string          `yaml:"importStatements"`
-}
-
-// ProviderMetadata metadata for a Terraform native provider
-type ProviderMetadata struct {
-	Name      string               `yaml:"name"`
-	Resources map[string]*Resource `yaml:"resources"`
-}
-
-// NewProviderMetadataFromFile loads metadata from the specified YAML-formatted file
-func NewProviderMetadataFromFile(path string) (*ProviderMetadata, error) {
-	buff, err := ioutil.ReadFile(filepath.Clean(path))
-	if err != nil {
-		return nil, errors.Wrapf(err, "failed to read metadata file %q", path)
-	}
-
-	metadata := &ProviderMetadata{}
-	return metadata, errors.Wrap(yaml.Unmarshal(buff, metadata), "failed to unmarshal provider metadata")
-}
-
 // ExampleGenerator represents a pipeline for generating example manifests.
 // Generates example manifests for Terraform resources under examples-generated.
 type ExampleGenerator struct {
-	rootDir      string
-	resourceMeta map[string]*Resource
-	resources    map[string]*pavedWithManifest
+	rootDir        string
+	configResource map[string]*config.Resource
+	resources      map[string]*pavedWithManifest
 }
 
 // NewExampleGenerator returns a configured ExampleGenerator
-func NewExampleGenerator(rootDir string, resourceMeta map[string]*Resource) *ExampleGenerator {
+func NewExampleGenerator(rootDir string, configResource map[string]*config.Resource) *ExampleGenerator {
 	return &ExampleGenerator{
-		rootDir:      rootDir,
-		resourceMeta: resourceMeta,
-		resources:    make(map[string]*pavedWithManifest),
+		rootDir:        rootDir,
+		configResource: configResource,
+		resources:      make(map[string]*pavedWithManifest),
 	}
 }
 
@@ -169,26 +134,29 @@ func (eg *ExampleGenerator) resolveReferences(params map[string]interface{}) err
 
 // Generate generates an example manifest for the specified Terraform resource.
 func (eg *ExampleGenerator) Generate(group, version string, r *config.Resource, fieldTransformations map[string]tjtypes.Transformation) error {
-	rm := eg.resourceMeta[r.Name]
+	rm := eg.configResource[r.Name].MetaResource
 	if rm == nil || len(rm.Examples) == 0 {
 		return nil
 	}
-	var exampleParams map[string]interface{}
-	if err := json.TFParser.Unmarshal([]byte(rm.Examples[0].Manifest), &exampleParams); err != nil {
-		return errors.Wrapf(err, "cannot unmarshal example manifest for resource: %s", r.Name)
-	}
+	exampleParams := rm.Examples[0].Paved.UnstructuredContent()
 	transformFields(exampleParams, r.ExternalName.OmittedFields, fieldTransformations, "")
 
+	metadata := map[string]interface{}{
+		"name": "example",
+	}
+	if len(rm.ExternalName) != 0 {
+		metadata["annotations"] = map[string]string{
+			xpmeta.AnnotationKeyExternalName: rm.ExternalName,
+		}
+	}
 	example := map[string]interface{}{
 		"apiVersion": fmt.Sprintf("%s/%s", group, version),
 		"kind":       r.Kind,
-		"metadata": map[string]interface{}{
-			"name": "example",
-		},
+		"metadata":   metadata,
 		"spec": map[string]interface{}{
 			"forProvider": exampleParams,
 			"providerConfigRef": map[string]interface{}{
-				"name": "example",
+				"name": "default",
 			},
 		},
 	}

pkg/pipeline/run.go

@@ -23,9 +23,9 @@ import (
 	"sort"
 	"strings"
 
-	"github.com/upbound/upjet/pkg/config"
-
 	"github.com/crossplane/crossplane-runtime/pkg/errors"
+
+	"github.com/upbound/upjet/pkg/config"
 )
 
 type terraformedInput struct {
@@ -34,7 +34,7 @@ type terraformedInput struct {
 }
 
 // Run runs the Terrajet code generation pipelines.
-func Run(pc *config.Provider, rootDir string) { // nolint:gocyclo
+func Run(pc *config.Provider) { // nolint:gocyclo
 	// Note(turkenh): nolint reasoning - this is the main function of the code
 	// generation pipeline. We didn't want to split it into multiple functions
 	// for better readability considering the straightforward logic here.
@@ -57,15 +57,6 @@ func Run(pc *config.Provider, rootDir string) { // nolint:gocyclo
 		resourcesGroups[group][resource.Version][name] = resource
 	}
 
-	metaResources := make(map[string]*Resource)
-	if pc.ProviderMetadataPath != "" {
-		providerMetadata, err := NewProviderMetadataFromFile(filepath.Join(rootDir, pc.ProviderMetadataPath))
-		if err != nil {
-			panic(errors.Wrap(err, "cannot read Terraform provider metadata"))
-		}
-		metaResources = providerMetadata.Resources
-	}
-
 	// Add ProviderConfig API package to the list of API version packages.
 	apiVersionPkgList := make([]string, 0)
 	for _, p := range pc.BasePackages.APIVersion {
@@ -77,14 +68,14 @@ func Run(pc *config.Provider, rootDir string) { // nolint:gocyclo
 		controllerPkgList = append(controllerPkgList, filepath.Join(pc.ModulePath, p))
 	}
 	count := 0
-	exampleGen := NewExampleGenerator(rootDir, metaResources)
+	exampleGen := NewExampleGenerator(pc.RootDir, pc.Resources)
 	for group, versions := range resourcesGroups {
 		for version, resources := range versions {
 			var tfResources []*terraformedInput
-			versionGen := NewVersionGenerator(rootDir, pc.ModulePath, group, version)
-			crdGen := NewCRDGenerator(versionGen.Package(), rootDir, pc.ShortName, group, version)
-			tfGen := NewTerraformedGenerator(versionGen.Package(), rootDir, group, version)
-			ctrlGen := NewControllerGenerator(rootDir, pc.ModulePath, group)
+			versionGen := NewVersionGenerator(pc.RootDir, pc.ModulePath, group, version)
+			crdGen := NewCRDGenerator(versionGen.Package(), pc.RootDir, pc.ShortName, group, version)
+			tfGen := NewTerraformedGenerator(versionGen.Package(), pc.RootDir, group, version)
+			ctrlGen := NewControllerGenerator(pc.RootDir, pc.ModulePath, group)
 
 			for _, name := range sortedResources(resources) {
 				paramTypeName, err := crdGen.Generate(resources[name])
@@ -121,24 +112,24 @@ func Run(pc *config.Provider, rootDir string) { // nolint:gocyclo
 		panic(errors.Wrapf(err, "cannot store examples"))
 	}
 
-	if err := NewRegisterGenerator(rootDir, pc.ModulePath).Generate(apiVersionPkgList); err != nil {
+	if err := NewRegisterGenerator(pc.RootDir, pc.ModulePath).Generate(apiVersionPkgList); err != nil {
 		panic(errors.Wrap(err, "cannot generate register file"))
 	}
-	if err := NewSetupGenerator(rootDir, pc.ModulePath).Generate(controllerPkgList); err != nil {
+	if err := NewSetupGenerator(pc.RootDir, pc.ModulePath).Generate(controllerPkgList); err != nil {
 		panic(errors.Wrap(err, "cannot generate setup file"))
 	}
 
 	// NOTE(muvaf): gosec linter requires that the whole command is hard-coded.
 	// So, we set the directory of the command instead of passing in the directory
 	// as an argument to "find".
 	apisCmd := exec.Command("bash", "-c", "goimports -w $(find . -iname 'zz_*')")
-	apisCmd.Dir = filepath.Clean(filepath.Join(rootDir, "apis"))
+	apisCmd.Dir = filepath.Clean(filepath.Join(pc.RootDir, "apis"))
 	if out, err := apisCmd.CombinedOutput(); err != nil {
 		panic(errors.Wrap(err, "cannot run goimports for apis folder: "+string(out)))
 	}
 
 	internalCmd := exec.Command("bash", "-c", "goimports -w $(find . -iname 'zz_*')")
-	internalCmd.Dir = filepath.Clean(filepath.Join(rootDir, "internal"))
+	internalCmd.Dir = filepath.Clean(filepath.Join(pc.RootDir, "internal"))
 	if out, err := internalCmd.CombinedOutput(); err != nil {
 		panic(errors.Wrap(err, "cannot run goimports for internal folder: "+string(out)))
 	}