From 298bce80e2fd5959bd006eb342bae31da5b3090d Mon Sep 17 00:00:00 2001 From: Matthew Frahry Date: Tue, 14 Nov 2023 23:12:14 -0800 Subject: [PATCH] importer-rest-api-specs - Output Terraform Tests as HCL (#3341) * Write tests as HCl * Output tests to data api * Cleanup --- .../data-api/internal/repositories/helpers.go | 52 +++++++++- .../internal/repositories/services.go | 97 +++++++++++++++++++ .../generate_terraform.go | 3 + .../generate_terraform_resource.go | 9 +- .../dataapigeneratorjson/helpers.go | 60 ++++++++++++ .../dataapigeneratorjson/interface.go | 15 +-- .../dataapigeneratorjson/service.go | 16 +-- .../templater_terraform_resource_tests.go | 2 - 8 files changed, 230 insertions(+), 24 deletions(-) diff --git a/tools/data-api/internal/repositories/helpers.go b/tools/data-api/internal/repositories/helpers.go index 6179aa17eef..d708c700061 100644 --- a/tools/data-api/internal/repositories/helpers.go +++ b/tools/data-api/internal/repositories/helpers.go @@ -4,6 +4,7 @@ import ( "fmt" "io" "os" + "strconv" "strings" ) @@ -29,7 +30,6 @@ func loadJson(path string) (*[]byte, error) { if err != nil { return nil, fmt.Errorf("loading %q: %+v", path, err) } - defer contents.Close() byteValue, err := io.ReadAll(contents) @@ -40,6 +40,21 @@ func loadJson(path string) (*[]byte, error) { return &byteValue, nil } +func loadHcl(path string) (string, error) { + contents, err := os.Open(path) + if err != nil { + return "", fmt.Errorf("loading %q: %+v", path, err) + } + defer contents.Close() + + byteValue, err := io.ReadAll(contents) + if err != nil { + return "", fmt.Errorf("reading contents of %q: %+v", path, err) + } + + return string(byteValue), nil +} + // getDefinitionInfo transforms the file names in the api definitions directory into a definition type and a name e.g. // Model-KeyVaultProperties.json -> type = Model and name = KeyVaultProperties func getDefinitionInfo(fileName string) (string, string, error) { @@ -69,3 +84,38 @@ func getTerraformDefinitionInfo(fileName string) (string, string, error) { return definitionName, definitionType, nil } + +// getTerraformTestInfo transforms the file names in the Tests directory into a name, Terraform Definition type, Test Type e.g. +// LoadTest-Resource.json -> name = LoadTest and type = Resource +func getTerraformTestInfo(fileName string) (string, string, string, error) { + if !strings.HasSuffix(fileName, ".hcl") { + return "", "", "", fmt.Errorf("file %q has an extensions not supported by the data api", fileName) + } + splitName := strings.SplitN(fileName, "-", 3) + + definitionName := splitName[0] + definitionType := splitName[1] + testType := strings.Split(splitName[2], ".")[0] + + return definitionName, definitionType, testType, nil + +} + +// getTerraformOtherTestInfo transforms an otherTestType into `Other`, TestName, and a TestNum e.g. +// LoadTest-Resource-Other-Foo-2 -> testName = Foo and testNum = 2 +func getTerraformOtherTestInfo(otherTestType string) (string, int, error) { + splitName := strings.SplitN(otherTestType, "-", 4) + if len(splitName) != 4 { + return "", -1, fmt.Errorf("expected OtherTest to be split into format Other-Foo-2. Received: %+v", otherTestType) + } + + testName := splitName[1] + + testNum, err := strconv.Atoi(splitName[2]) + if err != nil { + return "", -1, fmt.Errorf("converting %s to int: %+v", splitName[2], err) + } + + return testName, testNum, nil + +} diff --git a/tools/data-api/internal/repositories/services.go b/tools/data-api/internal/repositories/services.go index f49ebe4a618..219f5b802dd 100644 --- a/tools/data-api/internal/repositories/services.go +++ b/tools/data-api/internal/repositories/services.go @@ -133,6 +133,10 @@ func (s *ServicesRepositoryImpl) ProcessServiceDefinitions(serviceName string) ( if versions != nil { for _, version := range *versions { + // The Terraform directory can be skipped as it only has a subdirectory for tests + if version == "Terraform" { + continue + } resources, err := listSubDirectories(fmt.Sprintf("%s/%s/%s", s.directory, serviceName, version)) if err != nil { return nil, fmt.Errorf("retrieving resources for %s: %+v", version, err) @@ -505,6 +509,90 @@ func (s *ServicesRepositoryImpl) ProcessTerraformDefinitions(serviceName string) terraformDetails.Resources[definitionName] = resource } + terraformTestsPath := path.Join(terraformDefinitionsPath, "Tests") + testFiles, err := os.ReadDir(terraformTestsPath) + if err != nil { + if strings.Contains(err.Error(), "no such file or directory") { + return nil, nil + } + return nil, fmt.Errorf("retrieving tests under %s: %+v", terraformTestsPath, err) + } + + for _, file := range testFiles { + if file.IsDir() { + continue + } + + // todo the `_` is defintionType (ie. Resource, Datasource?), we'll probably do something with that later but for now we'll ignore + definitionName, _, testType, err := getTerraformTestInfo(file.Name()) + if err != nil { + return nil, err + } + + if _, ok := terraformDetails.Resources[definitionName]; !ok { + break + } + resource := terraformDetails.Resources[definitionName] + tests := resource.Tests + + lowerCaseTestType := strings.ToLower(testType) + switch { + case lowerCaseTestType == "basic-test": + basicConfig, err := parseTerraformTestFromFilePath(terraformTestsPath, file) + if err != nil { + return nil, err + } + tests.BasicConfiguration = basicConfig + case lowerCaseTestType == "complete-test": + completeConfig, err := parseTerraformTestFromFilePath(terraformTestsPath, file) + if err != nil { + return nil, err + } + tests.CompleteConfiguration = &completeConfig + case lowerCaseTestType == "requires-import-test": + requiresImportConfig, err := parseTerraformTestFromFilePath(terraformTestsPath, file) + if err != nil { + return nil, err + } + tests.RequiresImportConfiguration = requiresImportConfig + case lowerCaseTestType == "template-test": + templateConfig, err := parseTerraformTestFromFilePath(terraformTestsPath, file) + if err != nil { + return nil, err + } + tests.TemplateConfiguration = &templateConfig + + case strings.HasPrefix(lowerCaseTestType, "other"): + // todo we're assuming that tests are read in order and we should instead confirm that order + testName, _, err := getTerraformOtherTestInfo(testType) + if err != nil { + return nil, err + } + + if tests.OtherTests == nil { + tests.OtherTests = pointer.To(make(map[string][]string)) + } + otherTests := *tests.OtherTests + + if otherTests[testName] == nil { + otherTests[testName] = make([]string, 0) + } + otherTest := otherTests[testName] + + otherTestConfig, err := parseTerraformTestFromFilePath(terraformTestsPath, file) + if err != nil { + return nil, err + } + + otherTest = append(otherTest, otherTestConfig) + otherTests[testName] = otherTest + tests.OtherTests = &otherTests + } + + resource.Tests = tests + terraformDetails.Resources[definitionName] = resource + } + return &terraformDetails, nil } @@ -645,6 +733,15 @@ func parseTerraformDefinitionResourceMappingsFromFilePath(resourcePath string, f return mappings, nil } +func parseTerraformTestFromFilePath(resourcePath string, file os.DirEntry) (string, error) { + contents, err := loadHcl(path.Join(resourcePath, file.Name())) + if err != nil { + return contents, err + } + + return contents, nil +} + func parseTerraformDefinitionResourceSchemaFromFilePath(resourcePath string, file os.DirEntry) (map[string]TerraformSchemaModelDefinition, error) { schemaModelDefinition := make(map[string]TerraformSchemaModelDefinition) contents, err := loadJson(path.Join(resourcePath, file.Name())) diff --git a/tools/importer-rest-api-specs/components/dataapigeneratorjson/generate_terraform.go b/tools/importer-rest-api-specs/components/dataapigeneratorjson/generate_terraform.go index 5062c16441d..b613e346efc 100644 --- a/tools/importer-rest-api-specs/components/dataapigeneratorjson/generate_terraform.go +++ b/tools/importer-rest-api-specs/components/dataapigeneratorjson/generate_terraform.go @@ -26,6 +26,9 @@ func (s Generator) generateTerraformDefinitions(apiVersion models.AzureApiDefini if err := ensureDirectoryExists(s.workingDirectoryForTerraform, s.logger); err != nil { return fmt.Errorf("ensuring the Terraform Directory at %q exists: %+v", s.workingDirectoryForTerraform, err) } + if err := ensureDirectoryExists(s.workingDirectoryForTerraformTests, s.logger); err != nil { + return fmt.Errorf("ensuring the Terraform Tests Directory at %q exists: %+v", s.workingDirectoryForTerraformTests, err) + } for resourceName, resource := range apiVersion.Resources { if resource.Terraform == nil { diff --git a/tools/importer-rest-api-specs/components/dataapigeneratorjson/generate_terraform_resource.go b/tools/importer-rest-api-specs/components/dataapigeneratorjson/generate_terraform_resource.go index 13c9d84266f..5a14efef4c6 100644 --- a/tools/importer-rest-api-specs/components/dataapigeneratorjson/generate_terraform_resource.go +++ b/tools/importer-rest-api-specs/components/dataapigeneratorjson/generate_terraform_resource.go @@ -49,15 +49,10 @@ func (s Generator) generateTerraformResourceDefinition(resourceLabel string, det } // output the Tests for this Terraform Resource - resourceTestsFileName := path.Join(s.workingDirectoryForTerraform, fmt.Sprintf("%s-Resource-Tests.json", details.ResourceName)) + resourceTestsFileName := path.Join(s.workingDirectoryForTerraformTests, fmt.Sprintf("%s-Resource-Tests.hcl", details.ResourceName)) s.logger.Trace(fmt.Sprintf("Generating Tests for the Terraform Resource into %q", resourceTestsFileName)) resourceTestsCode := mapTerraformResourceTestDefinition(details.Tests) - s.logger.Trace("Marshalling Tests for the Terraform Resource..") - testsCode, err := json.MarshalIndent(resourceTestsCode, "", "\t") - if err != nil { - return fmt.Errorf("marshaling Tests for the Terraform Resource %q: %+v", resourceLabel, err) - } - if err := writeJsonToFile(resourceTestsFileName, testsCode); err != nil { + if err := writeTestsHclToFile(s.workingDirectoryForTerraformTests, details.ResourceName, resourceTestsCode); err != nil { return fmt.Errorf("generating Tests for the Terraform Resource %q: %+v", resourceLabel, err) } diff --git a/tools/importer-rest-api-specs/components/dataapigeneratorjson/helpers.go b/tools/importer-rest-api-specs/components/dataapigeneratorjson/helpers.go index b36c389fc52..e7138813f20 100644 --- a/tools/importer-rest-api-specs/components/dataapigeneratorjson/helpers.go +++ b/tools/importer-rest-api-specs/components/dataapigeneratorjson/helpers.go @@ -6,6 +6,7 @@ import ( "path" "github.com/hashicorp/go-hclog" + dataApiModels "github.com/hashicorp/pandora/tools/importer-rest-api-specs/components/dataapigeneratorjson/models" ) func (s Generator) workingDirectoryForResource(resource string) string { @@ -55,3 +56,62 @@ func writeJsonToFile(fileName string, fileContents []byte) error { file.Close() return nil } + +func writeTestsHclToFile(directory, resourceName string, tests dataApiModels.TerraformResourceTestConfig) error { + if !tests.Generate { + return nil + } + if tests.TemplateConfig != nil { + templateTestFileName := path.Join(directory, fmt.Sprintf("%s-Resource-Template-Test.hcl", resourceName)) + if err := writeStringToFile(templateTestFileName, *tests.TemplateConfig); err != nil { + return fmt.Errorf("writing Template Test Config: %+v", err) + } + } + + basicTestFileName := path.Join(directory, fmt.Sprintf("%s-Resource-Basic-Test.hcl", resourceName)) + if err := writeStringToFile(basicTestFileName, tests.BasicConfig); err != nil { + return fmt.Errorf("writing Basic Test Config: %+v", err) + } + + requiresImportTestFileName := path.Join(directory, fmt.Sprintf("%s-Resource-Requires-Import-Test.hcl", resourceName)) + if err := writeStringToFile(requiresImportTestFileName, tests.RequiresImport); err != nil { + return fmt.Errorf("writing Requires Import Test Config: %+v", err) + } + + if tests.CompleteConfig != nil { + completeTestFileName := path.Join(directory, fmt.Sprintf("%s-Resource-Complete-Test.hcl", resourceName)) + if err := writeStringToFile(completeTestFileName, *tests.CompleteConfig); err != nil { + return fmt.Errorf("writing Complete Test Config: %+v", err) + } + } + + if tests.OtherTests != nil { + for otherTestName, v := range *tests.OtherTests { + for i, test := range v { + otherTestFileName := path.Join(directory, fmt.Sprintf("%s-Resource-Other-%s-%d-Test.hcl", resourceName, otherTestName, i)) + if err := writeStringToFile(otherTestFileName, test); err != nil { + return fmt.Errorf("writing %s Test Config: %+v", otherTestName, err) + } + } + } + } + + return nil +} + +func writeStringToFile(fileName string, fileContents string) error { + existing, err := os.Open(fileName) + if os.IsExist(err) { + return fmt.Errorf("existing file exists at %q", fileName) + } + existing.Close() + + file, err := os.Create(fileName) + if err != nil { + return fmt.Errorf("creating %q: %+v", fileName, err) + } + + file.WriteString(fileContents) + file.Close() + return nil +} diff --git a/tools/importer-rest-api-specs/components/dataapigeneratorjson/interface.go b/tools/importer-rest-api-specs/components/dataapigeneratorjson/interface.go index 6e64b97077f..ff802d72e5d 100644 --- a/tools/importer-rest-api-specs/components/dataapigeneratorjson/interface.go +++ b/tools/importer-rest-api-specs/components/dataapigeneratorjson/interface.go @@ -5,13 +5,14 @@ import ( ) type Generator struct { - outputDirectory string - resourceProvider *string - serviceName string - terraformPackageName *string - workingDirectoryForService string - workingDirectoryForApiVersion string - workingDirectoryForTerraform string + outputDirectory string + resourceProvider *string + serviceName string + terraformPackageName *string + workingDirectoryForService string + workingDirectoryForApiVersion string + workingDirectoryForTerraform string + workingDirectoryForTerraformTests string // TODO: pass this into methods as needed, so that we can ensure the logger is always named as required? logger hclog.Logger diff --git a/tools/importer-rest-api-specs/components/dataapigeneratorjson/service.go b/tools/importer-rest-api-specs/components/dataapigeneratorjson/service.go index b8348a6b8a9..68eba89ba1a 100644 --- a/tools/importer-rest-api-specs/components/dataapigeneratorjson/service.go +++ b/tools/importer-rest-api-specs/components/dataapigeneratorjson/service.go @@ -13,15 +13,17 @@ func NewForService(serviceName, outputDirectory string, resourceProvider, terraf normalisedServiceName := strings.ReplaceAll(serviceName, "-", "") serviceWorkingDirectory := path.Join(outputDirectory, strings.Title(normalisedServiceName)) terraformWorkingDirectory := path.Join(serviceWorkingDirectory, "Terraform") + terraformTestsWorkingDirectory := path.Join(terraformWorkingDirectory, "Tests") return &Generator{ - logger: logger, - outputDirectory: outputDirectory, - resourceProvider: resourceProvider, - serviceName: serviceName, - terraformPackageName: terraformPackageName, - workingDirectoryForService: serviceWorkingDirectory, - workingDirectoryForTerraform: terraformWorkingDirectory, + logger: logger, + outputDirectory: outputDirectory, + resourceProvider: resourceProvider, + serviceName: serviceName, + terraformPackageName: terraformPackageName, + workingDirectoryForService: serviceWorkingDirectory, + workingDirectoryForTerraform: terraformWorkingDirectory, + workingDirectoryForTerraformTests: terraformTestsWorkingDirectory, } } diff --git a/tools/importer-rest-api-specs/components/dataapigeneratorjson/templater_terraform_resource_tests.go b/tools/importer-rest-api-specs/components/dataapigeneratorjson/templater_terraform_resource_tests.go index c31abc20035..71073308e35 100644 --- a/tools/importer-rest-api-specs/components/dataapigeneratorjson/templater_terraform_resource_tests.go +++ b/tools/importer-rest-api-specs/components/dataapigeneratorjson/templater_terraform_resource_tests.go @@ -7,8 +7,6 @@ import ( ) func mapTerraformResourceTestDefinition(input resourcemanager.TerraformResourceTestsDefinition) dataApiModels.TerraformResourceTestConfig { - // TODO: looking at the data more and more, these probably want to become `*.hcl` files? - // Perhaps `Resource-Test-{Basic|Complete}.hcl` and `Resource-Test-{Other}{1|2|3}.hcl`? testConfig := dataApiModels.TerraformResourceTestConfig{ BasicConfig: input.BasicConfiguration, CompleteConfig: input.CompleteConfiguration,