Merge pull request #3463 from hashicorp/more-terraform-v2-fixes

dataapiv2: More terraform v2 fixes

tools/data-api/internal/endpoints/v1/terraform.go

@@ -19,7 +19,11 @@ func (api Api) terraform(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	resources := mapTerraformResources(service.TerraformDetails.Resources)
+	resources, err := mapTerraformResources(service.TerraformDetails.Resources)
+	if err != nil {
+		internalServerError(w, err)
+		return
+	}
 
 	payload := models.TerraformDetails{
 		DataSources: map[string]models.TerraformDataSourceDetails{},
@@ -29,11 +33,11 @@ func (api Api) terraform(w http.ResponseWriter, r *http.Request) {
 	render.JSON(w, r, payload)
 }
 
-func mapTerraformResources(input map[string]repositories.TerraformResourceDetails) map[string]models.TerraformResourceDetails {
+func mapTerraformResources(input map[string]repositories.TerraformResourceDetails) (map[string]models.TerraformResourceDetails, error) {
 	output := make(map[string]models.TerraformResourceDetails, 0)
 
 	for _, resource := range input {
-		output[resource.Label] = models.TerraformResourceDetails{
+		resourceDetails := models.TerraformResourceDetails{
 			ApiVersion: resource.ApiVersion,
 			CreateMethod: models.MethodDefinition{
 				Generate:         resource.CreateMethod.Generate,
@@ -65,7 +69,6 @@ func mapTerraformResources(input map[string]repositories.TerraformResourceDetail
 			ResourceIdName:  resource.ResourceIdName,
 			ResourceName:    resource.ResourceName,
 			SchemaModelName: resource.SchemaModelName,
-			SchemaModels:    mapSchemaModels(&resource.SchemaModels),
 			Tests: models.TerraformResourceTestsDefinition{
 				BasicConfiguration:          resource.Tests.BasicConfiguration,
 				RequiresImportConfiguration: resource.Tests.RequiresImportConfiguration,
@@ -77,15 +80,21 @@ func mapTerraformResources(input map[string]repositories.TerraformResourceDetail
 			UpdateMethod: mapUpdateMethod(resource.UpdateMethod),
 		}
 
+		schemaModels, err := mapSchemaModels(&resource.SchemaModels)
+		if err != nil {
+			return nil, fmt.Errorf("mapping schema models for %s: %+v", resource.ResourceName, err)
+		}
+		resourceDetails.SchemaModels = schemaModels
+
 		// todo remove this when https://github.com/hashicorp/pandora/issues/3352 is fixed
 		// tests won't be added unless Generate is true when writing this out in dataapigeneratorjson/helpers.go writeTestsHclToFile
 		// so we can set this to true if BasicConfiguration has been written out
-		if output[resource.Label].Tests.BasicConfiguration != "" {
-			currResource := output[resource.Label]
-			currResource.Tests.Generate = true
-			output[resource.Label] = currResource
+		if resourceDetails.Tests.BasicConfiguration != "" {
+			resourceDetails.Tests.Generate = true
 		}
+
+		output[resource.Label] = resourceDetails
 	}
 
-	return output
+	return output, nil
 }

tools/data-api/internal/endpoints/v1/terraform_mappings.go

@@ -1,58 +1,105 @@
 package v1
 
 import (
+	"fmt"
 	"github.com/hashicorp/go-azure-helpers/lang/pointer"
 	"github.com/hashicorp/pandora/tools/data-api/internal/repositories"
 	"github.com/hashicorp/pandora/tools/data-api/models"
 )
 
-func mapSchemaModels(input *map[string]repositories.TerraformSchemaModelDefinition) map[string]models.TerraformSchemaModelDefinition {
+func mapSchemaModels(input *map[string]repositories.TerraformSchemaModelDefinition) (map[string]models.TerraformSchemaModelDefinition, error) {
 	if input == nil {
-		return nil
+		return nil, nil
 	}
 
 	output := make(map[string]models.TerraformSchemaModelDefinition, 0)
 
 	for name, model := range *input {
-		output[name] = models.TerraformSchemaModelDefinition{
-			Fields: mapFields(model.Fields),
+		var modelDefinition models.TerraformSchemaModelDefinition
+
+		fields, err := mapFields(model.Fields)
+		if err != nil {
+			return nil, fmt.Errorf("mapping fields for %s: %+v", name, err)
 		}
+		modelDefinition.Fields = fields
+
+		output[name] = modelDefinition
 	}
 
-	return output
+	return output, nil
 }
 
-func mapFields(input map[string]repositories.TerraformSchemaFieldDefinition) map[string]models.TerraformSchemaFieldDefinition {
+func mapFields(input map[string]repositories.TerraformSchemaFieldDefinition) (map[string]models.TerraformSchemaFieldDefinition, error) {
 	output := make(map[string]models.TerraformSchemaFieldDefinition, 0)
 
 	for name, field := range input {
-		output[name] = models.TerraformSchemaFieldDefinition{
-			ObjectDefinition: mapTerraformObjectDefinition(field.ObjectDefinition),
-			Computed:         field.Computed,
-			ForceNew:         field.ForceNew,
-			HclName:          field.HclName,
-			Optional:         field.Optional,
-			Required:         field.Required,
+		fieldDefinition := models.TerraformSchemaFieldDefinition{
+			Computed: field.Computed,
+			ForceNew: field.ForceNew,
+			HclName:  field.HclName,
+			Optional: field.Optional,
+			Required: field.Required,
 			Documentation: models.TerraformSchemaDocumentationDefinition{
 				Markdown: field.Documentation.Markdown,
 			},
 			Validation: mapValidation(field.Validation),
 		}
+
+		objectDefinition, err := mapTerraformObjectDefinition(field.ObjectDefinition)
+		if err != nil {
+			return output, fmt.Errorf("mapping object definition for %s: %+v", name, err)
+		}
+		fieldDefinition.ObjectDefinition = objectDefinition
+
+		output[name] = fieldDefinition
 	}
 
-	return output
+	return output, nil
+}
+
+var terraformSchemaObjectDefinitionToTerraformFieldSchemaTypes = map[repositories.TerraformSchemaFieldType]models.TerraformSchemaFieldType{
+	repositories.BooleanTerraformSchemaObjectDefinitionType:                       models.TerraformSchemaFieldTypeBoolean,
+	repositories.DateTimeTerraformSchemaObjectDefinitionType:                      models.TerraformSchemaFieldTypeDateTime,
+	repositories.DictionaryTerraformSchemaObjectDefinitionType:                    models.TerraformSchemaFieldTypeDictionary,
+	repositories.EdgeZoneTerraformSchemaObjectDefinitionType:                      models.TerraformSchemaFieldTypeEdgeZone,
+	repositories.SystemAssignedIdentityTerraformSchemaObjectDefinitionType:        models.TerraformSchemaFieldTypeIdentitySystemAssigned,
+	repositories.SystemAndUserAssignedIdentityTerraformSchemaObjectDefinitionType: models.TerraformSchemaFieldTypeIdentitySystemAndUserAssigned,
+	repositories.SystemOrUserAssignedIdentityTerraformSchemaObjectDefinitionType:  models.TerraformSchemaFieldTypeIdentitySystemOrUserAssigned,
+	repositories.UserAssignedIdentityTerraformSchemaObjectDefinitionType:          models.TerraformSchemaFieldTypeIdentityUserAssigned,
+	repositories.LocationTerraformSchemaObjectDefinitionType:                      models.TerraformSchemaFieldTypeLocation,
+	repositories.FloatTerraformSchemaObjectDefinitionType:                         models.TerraformSchemaFieldTypeFloat,
+	repositories.IntegerTerraformSchemaObjectDefinitionType:                       models.TerraformSchemaFieldTypeInteger,
+	repositories.ListTerraformSchemaObjectDefinitionType:                          models.TerraformSchemaFieldTypeList,
+	repositories.ReferenceTerraformSchemaObjectDefinitionType:                     models.TerraformSchemaFieldTypeReference,
+	repositories.ResourceGroupTerraformSchemaObjectDefinitionType:                 models.TerraformSchemaFieldTypeResourceGroup,
+	repositories.SetTerraformSchemaObjectDefinitionType:                           models.TerraformSchemaFieldTypeSet,
+	repositories.StringTerraformSchemaFieldType:                                   models.TerraformSchemaFieldTypeString,
+	repositories.TagsTerraformSchemaObjectDefinitionType:                          models.TerraformSchemaFieldTypeTags,
+	repositories.SkuTerraformSchemaObjectDefinitionType:                           models.TerraformSchemaFieldTypeSku,
+	repositories.ZoneTerraformSchemaObjectDefinitionType:                          models.TerraformSchemaFieldTypeZone,
+	repositories.ZonesTerraformSchemaObjectDefinitionType:                         models.TerraformSchemaFieldTypeZones,
 }
 
-func mapTerraformObjectDefinition(input repositories.TerraformSchemaFieldObjectDefinition) models.TerraformSchemaFieldObjectDefinition {
+func mapTerraformObjectDefinition(input repositories.TerraformSchemaFieldObjectDefinition) (models.TerraformSchemaFieldObjectDefinition, error) {
 	output := models.TerraformSchemaFieldObjectDefinition{}
 
 	if input.NestedObject != nil {
-		output.NestedObject = pointer.To(mapTerraformObjectDefinition(*input.NestedObject))
+		nestedObject, err := mapTerraformObjectDefinition(*input.NestedObject)
+		if err != nil {
+			return output, err
+		}
+		output.NestedObject = pointer.To(nestedObject)
 	}
 	output.ReferenceName = input.ReferenceName
 	output.Type = models.TerraformSchemaFieldType(input.Type)
 
-	return output
+	mapped, ok := terraformSchemaObjectDefinitionToTerraformFieldSchemaTypes[input.Type]
+	if !ok {
+		return output, fmt.Errorf("internal-error: missing mapping for Terraform Schema Field Type %q", string(input.Type))
+	}
+	output.Type = mapped
+
+	return output, nil
 }
 
 func mapUpdateMethod(input *repositories.MethodDefinition) *models.MethodDefinition {

tools/data-api/internal/repositories/helpers.go

@@ -3,6 +3,7 @@ package repositories
 import (
 	"fmt"
 	"io"
+	"log"
 	"os"
 	"strconv"
 	"strings"
@@ -81,6 +82,13 @@ func getTerraformDefinitionInfo(fileName string) (string, string, error) {
 	definitionName := splitName[0]
 	definitionType := strings.Split(splitName[1], ".")[0]
 
+	// Resource-Schema Files can have multiple files with the model type appended to it (ie. KubernetesFleetManager-Resource-Schema-FleetHubProfile
+	// that final piece of the final name is not used and can be safely ignored
+	if strings.Contains(strings.ToLower(definitionType), "resource-schema") && len(strings.Split(definitionType, "-")) >= 3 {
+		log.Printf("FileName: %s", fileName)
+		definitionType = "Resource-Schema"
+	}
+
 	return definitionName, definitionType, nil
 
 }

tools/data-api/internal/repositories/services.go

@@ -588,7 +588,7 @@ func (s *ServicesRepositoryImpl) ProcessTerraformDefinitions(serviceName string)
 			}
 
 		case "resource-schema":
-			resource.SchemaModels, err = parseTerraformDefinitionResourceSchemaFromFilePath(terraformDefinitionsPath, file)
+			resource.SchemaModels, err = parseTerraformDefinitionResourceSchemaFromFilePath(terraformDefinitionsPath, file, resource.SchemaModels)
 			if err != nil {
 				return nil, err
 			}
@@ -832,17 +832,20 @@ func parseTerraformTestFromFilePath(resourcePath string, file os.DirEntry) (stri
 	return contents, nil
 }
 
-func parseTerraformDefinitionResourceSchemaFromFilePath(resourcePath string, file os.DirEntry) (map[string]TerraformSchemaModelDefinition, error) {
-	schemaModelDefinition := make(map[string]TerraformSchemaModelDefinition)
+func parseTerraformDefinitionResourceSchemaFromFilePath(resourcePath string, file os.DirEntry, input map[string]TerraformSchemaModelDefinition) (map[string]TerraformSchemaModelDefinition, error) {
+	if input == nil {
+		input = make(map[string]TerraformSchemaModelDefinition)
+	}
+
 	contents, err := loadJson(path.Join(resourcePath, file.Name()))
 	if err != nil {
-		return schemaModelDefinition, err
+		return input, err
 	}
 
 	var schemaModel dataapimodels.TerraformSchemaModel
 
 	if err := json.Unmarshal(*contents, &schemaModel); err != nil {
-		return schemaModelDefinition, fmt.Errorf("unmarshaling Terraform Resource Schema %+v", err)
+		return input, fmt.Errorf("unmarshaling Terraform Resource Schema %+v", err)
 	}
 
 	fields := make(map[string]TerraformSchemaFieldDefinition)
@@ -878,15 +881,11 @@ func parseTerraformDefinitionResourceSchemaFromFilePath(resourcePath string, fil
 		fields[field.Name] = fieldDefinition
 	}
 
-	// todo do we take the file name and strip it of these pieces or is this information somewhere else
-	// the v1 data api has this value as `LoadTestResourceSchema` which is the filename (LoadTest-Resource-Schema.json) with the following stripped
-	modelDefinitionName := strings.Replace(strings.Replace(file.Name(), "-", "", -1), ".json", "", -1)
-
-	schemaModelDefinition[modelDefinitionName] = TerraformSchemaModelDefinition{
+	input[schemaModel.Name] = TerraformSchemaModelDefinition{
 		Fields: fields,
 	}
 
-	return schemaModelDefinition, nil
+	return input, nil
 }
 
 func parseTerraformDefinitionResourceTestsFromFilePath(resourcePath string, file os.DirEntry) (TerraformResourceTestsDefinition, error) {

tools/importer-rest-api-specs/components/dataapigeneratorjson/templater_terraform_mappings.go

@@ -115,17 +115,17 @@ func mapTerraformSchemaMappings(input resourcemanager.MappingDefinition) (*dataa
 
 func mapTerraformSchemaResourceIdMappings(input []resourcemanager.ResourceIdMappingDefinition) []dataapimodels.TerraformResourceIdMappingDefinition {
 	// we need the ordering to be consistent else to avoid noisy regenerations, so let's order this on one of the keys
-	schemaFieldNames := make([]string, 0)
-	schemaFieldNamesToResourceIdMappings := make(map[string]resourcemanager.ResourceIdMappingDefinition)
+	segmentNames := make([]string, 0)
+	segmentNamesToResourceIdMappings := make(map[string]resourcemanager.ResourceIdMappingDefinition)
 	for _, item := range input {
-		schemaFieldNames = append(schemaFieldNames, item.SchemaFieldName)
-		schemaFieldNamesToResourceIdMappings[item.SchemaFieldName] = item
+		segmentNames = append(segmentNames, item.SegmentName)
+		segmentNamesToResourceIdMappings[item.SegmentName] = item
 	}
-	sort.Strings(schemaFieldNames)
+	sort.Strings(segmentNames)
 
 	output := make([]dataapimodels.TerraformResourceIdMappingDefinition, 0)
-	for _, schemaFieldName := range schemaFieldNames {
-		resourceIdMapping := schemaFieldNamesToResourceIdMappings[schemaFieldName]
+	for _, schemaFieldName := range segmentNames {
+		resourceIdMapping := segmentNamesToResourceIdMappings[schemaFieldName]
 		output = append(output, dataapimodels.TerraformResourceIdMappingDefinition{
 			SchemaFieldName:    resourceIdMapping.SchemaFieldName,
 			SegmentName:        resourceIdMapping.SegmentName,

tools/importer-rest-api-specs/components/dataapigeneratorjson/templater_terraform_schema.go

@@ -3,6 +3,7 @@ package dataapigeneratorjson
 import (
 	"fmt"
 	"sort"
+	"strings"
 
 	"github.com/hashicorp/go-azure-helpers/lang/pointer"
 	"github.com/hashicorp/pandora/tools/sdk/dataapimodels"
@@ -30,7 +31,8 @@ func mapTerraformSchemaModelDefinition(modelName string, schemaModel resourceman
 
 	return &dataapimodels.TerraformSchemaModel{
 		Fields: schemaFields,
-		Name:   modelName,
+		// todo remove Schema when https://github.com/hashicorp/pandora/issues/3346 is addressed
+		Name: fmt.Sprintf("%sSchema", modelName),
 	}, nil
 }
 
@@ -113,6 +115,12 @@ func mapTerraformSchemaObjectDefinition(input resourcemanager.TerraformSchemaFie
 		Type:          mapped,
 	}
 
+	if input.ReferenceName != nil && !strings.HasSuffix(*input.ReferenceName, "Schema") {
+		// todo remove Schema when https://github.com/hashicorp/pandora/issues/3346 is addressed
+		referenceName := fmt.Sprintf("%sSchema", *input.ReferenceName)
+		objectDefinition.ReferenceName = &referenceName
+	}
+
 	if input.NestedObject != nil {
 		nestedItem, err := mapTerraformSchemaObjectDefinition(*input.NestedObject)
 		if err != nil {