From 0ab62a38a43f70435c4a0d77bc745e50c25381f7 Mon Sep 17 00:00:00 2001 From: Aastha Mahendru Date: Fri, 12 Jan 2024 19:20:23 +0000 Subject: [PATCH] fix merge --- CONTRIBUTING.md | 55 +++++++++++++++++++++++++++++++++++--- GNUmakefile | 12 +++++++-- go.mod | 1 - internal/config/client.go | 10 +++---- scripts/generate-doc.sh | 2 +- scripts/schema-scaffold.sh | 54 +++++++++++++++++++++++++++++++++++++ 6 files changed, 121 insertions(+), 13 deletions(-) create mode 100644 scripts/schema-scaffold.sh diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f80dafa4e2..d160a71eb2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -12,7 +12,10 @@ Thanks for your interest in contributing to MongoDB Atlas Terraform Provider, th - [Running Acceptance Tests](#running-acceptance-tests) - [Code and Test Best Practices](#code-and-test-best-practices) - [Creating New Resource and Data Sources](#creating-new-resources-and-data-sources) + - [Scaffolding Initial Code and File Structure](#scaffolding-initial-code-and-file-structure) + - [Scaffolding Schema and Model Definitions](#scaffolding-schema-and-model-definitions) - [Documentation Best Practices](#documentation-best-practices) + - [Creating Resource and Data source Documentation](#creating-resource-and-data-source-documentation) - [Discovering New API features](#discovering-new-api-features) @@ -309,21 +312,67 @@ To do this you can: ### Creating New Resource and Data Sources -A scaffolding command was defined with the intention of speeding up development process, while also preserving common conventions throughout our codebase. +A set of commands have been defined with the intention of speeding up development process, while also preserving common conventions throughout our codebase. + +#### Scaffolding Initial Code and File Structure This command can be used the following way: ```bash -make scaffold name=streamInstance type=resource +make scaffold resource_name=streamInstance type=resource ``` -- **name**: The name of the resource, which must be defined in camel case. +- **resource_name**: The name of the resource, which must be defined in camel case. - **type**: Describes the type of resource being created. There are 3 different types: `resource`, `data-source`, `plural-data-source`. This will generate resource/data source files and accompanying test files needed for starting the development, and will contain multiple comments with `TODO:` statements which give guidance for the development. +#### Scaffolding Schema and Model Definitions + +Complementary to the `scaffold` command, there is a command which generates the initial Terraform schema definition and associated Go types for a resource or data source. This processes leverages [Code Generation Tools](https://developer.hashicorp.com/terraform/plugin/code-generation) developed by HashiCorp, which in turn make use of the [Atlas Admin API](https://www.mongodb.com/docs/atlas/reference/api-resources-spec/v2/) OpenAPI Specification. + +##### Running the command + +Both `tfplugingen-openapi` and `tfplugingen-framework` must be installed. This can be done by running `make tools`. + +The command takes a single argument which specifies the resource or data source where the code generation is run, defined in camel case, e.g.: +```bash +make scaffold-schemas resource_name=streamInstance +``` + +As a pre-requiste, the relevant resource/data source directory must define a configuration file in the path `./internal/service//tfplugingen/generator_config.yml`. The content of this file will define which resource and/or data source schemas will be generated by providing the API endpoints they are mapped to. See the [Generator Config](https://developer.hashicorp.com/terraform/plugin/code-generation/openapi-generator#generator-config) documentation for more information on configuration options. An example defined in our repository can be found in [searchdeployment/tfplugingen/generator_config.yml](https://github.com/mongodb/terraform-provider-mongodbatlas/blob/master/internal/service/searchdeployment/tfplugingen/generator_config.yml). + +As a result of the execution, the schema definitions and associated model types will be defined in separate files depending on the resources and data sources that were configured in the generator_config.yml file: +- `data_source__schema.go` +- `resource__schema.go` + +Note: if the resulting file paths already exist the content will be stored in files with a `_gen.go` postfix, and in this case any content will be overwritten. This can be useful for comparing the latest autogenerated schema against the existing implementation. + +##### Considerations over generated schema and types + +- Generated Go type should include a TF prefix to follow the convention in our codebase, this will not be present in generated code. +- Some attribute names may need to be adjusted if there is a difference in how they are named in Terraform vs the API. An examples of this is `group_id` → `project_id`. +- Inferred characteristics of an attribute (computed, optional, required) may not always be an accurate representation and should be revised. Details of inference logic can be found in [OAS Types to Provider Attributes](https://github.com/hashicorp/terraform-plugin-codegen-openapi/blob/main/DESIGN.md#oas-types-to-provider-attributes). +- Missing [sensitive](https://developer.hashicorp.com/terraform/plugin/framework/handling-data/attributes/string#sensitive) field in attributes. +- Missing plan modifiers such as `RequiresReplace()` in attributes. +- Terraform specific attributes such as [timeouts](https://developer.hashicorp.com/terraform/plugin/framework/resources/timeouts#specifying-timeouts-in-configuration) need to be included manually. +- If nested attributes are defined a set of helper functions are generated for using the model. The usage of the generated functions can be considered optional as the current documentation is not very clear on the usage (More details in [terraform-plugin-codegen-framework/issues/80](https://github.com/hashicorp/terraform-plugin-codegen-framework/issues/80)). + + ## Documentation Best Practices - In our documentation, when a resource field allows a maximum of only one item, we do not format that field as an array. Instead, we create a subsection specifically for this field. Within this new subsection, we enumerate all the attributes of the field. Let's illustrate this with an example: [cloud_backup_schedule.html.markdown](https://github.com/mongodb/terraform-provider-mongodbatlas/blob/master/website/docs/r/cloud_backup_schedule.html.markdown?plain=1#L207) +### Creating Resource and Data source Documentation +We autogenerate the documentation of our provider resources and data sources via [tfplugindocs](https://github.com/hashicorp/terraform-plugin-docs). + +#### How to generate the documentation for a resource +- Make sure that the resource and data source schemas have defined the fields `MarkdownDescription` and `Description`. + - We recommend to use [Scaffolding Schema and Model Definitions](#scaffolding-schema-and-model-definitions) to autogenerate the schema via the Open API specification. +- Add the resource/data source templates to the [templates](templates) folder. See [README.md](templates/README.md) for more info. +- Run the Makefile command `generate-doc` +```bash +export resource_name=search_deployment && make generate-doc +``` + ## Discovering New API features Most of the new features of the provider are using [atlas-sdk](https://github.com/mongodb/atlas-sdk-go) diff --git a/GNUmakefile b/GNUmakefile index 7756a4cc73..a68b4d3f7f 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -115,12 +115,20 @@ link-git-hooks: ## Install git hooks update-atlas-sdk: ## Update the atlas-sdk dependency ./scripts/update-sdk.sh -# details on usage can be found in CONTRIBUTING.md under "Creating New Resource and Data Sources" +# e.g. run: make scaffold resource_name=streamInstance type=resource +# - type argument can have the values: `resource`, `data-source`, `plural-data-source`. +# details on usage can be found in CONTRIBUTING.md under "Scaffolding initial Code and File Structure" .PHONY: scaffold scaffold: - @go run ./tools/scaffold/*.go $(name) $(type) + @go run ./tools/scaffold/*.go $(resource_name) $(type) @echo "Reminder: configure the new $(type) in provider.go" +# e.g. run: make scaffold-schemas resource_name=streamInstance +# details on usage can be found in CONTRIBUTING.md under "Scaffolding Schema and Model Definitions" +.PHONY: scaffold-schemas +scaffold-schemas: + @scripts/schema-scaffold.sh $(resource_name) + .PHONY: generate-doc generate-doc: ## Generate the resource documentation via tfplugindocs diff --git a/go.mod b/go.mod index a0a594d924..84a44ac9fd 100644 --- a/go.mod +++ b/go.mod @@ -128,7 +128,6 @@ require ( github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect github.com/zclconf/go-cty-yaml v1.0.2 // indirect - go.mongodb.org/atlas-sdk/v20231115003 v20231115003.1.0 // indirect go.opencensus.io v0.24.0 // indirect golang.org/x/crypto v0.17.0 // indirect golang.org/x/mod v0.14.0 // indirect diff --git a/internal/config/client.go b/internal/config/client.go index 5fc1fb3ec6..4cffb723a4 100644 --- a/internal/config/client.go +++ b/internal/config/client.go @@ -8,17 +8,15 @@ import ( "net/url" "time" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/logging" + "github.com/mongodb-forks/digest" + "github.com/mongodb/terraform-provider-mongodbatlas/version" + "github.com/spf13/cast" oldAtlasSDK "go.mongodb.org/atlas-sdk/v20231001002/admin" atlasSDK "go.mongodb.org/atlas-sdk/v20231115003/admin" matlasClient "go.mongodb.org/atlas/mongodbatlas" realmAuth "go.mongodb.org/realm/auth" "go.mongodb.org/realm/realm" - - "github.com/hashicorp/terraform-plugin-sdk/v2/helper/logging" - "github.com/mongodb-forks/digest" - "github.com/spf13/cast" - - "github.com/mongodb/terraform-provider-mongodbatlas/version" ) const ( diff --git a/scripts/generate-doc.sh b/scripts/generate-doc.sh index 5b48017132..60a92a248f 100755 --- a/scripts/generate-doc.sh +++ b/scripts/generate-doc.sh @@ -32,7 +32,7 @@ set -euo pipefail -TF_VERSION="${TF_VERSION:-"1.6"}" # TF version to use when running tfplugindocs. Default: 1.6.6 +TF_VERSION="${TF_VERSION:-"1.6"}" # TF version to use when running tfplugindocs. Default: 1.6 TEMPLATE_FOLDER_PATH="${TEMPLATE_FOLDER_PATH:-"templates"}" # PATH to the templates folder. Default: templates diff --git a/scripts/schema-scaffold.sh b/scripts/schema-scaffold.sh new file mode 100644 index 0000000000..fb79fad5fc --- /dev/null +++ b/scripts/schema-scaffold.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +set -euo pipefail + +: "${1?"Name of resource or data source must be provided."}" + +# URL to download Atlas Admin API Spec +atlas_admin_api_spec="https://raw.githubusercontent.com/mongodb/atlas-sdk-go/main/openapi/atlas-api-transformed.yaml" + +echo "Downloading api spec" +curl -L "$atlas_admin_api_spec" -o "./api-spec.yml" + +resource_name=$1 +resource_name_lower_case="$(echo "$resource_name" | awk '{print tolower($0)}')" +resource_name_snake_case="$(echo "$resource_name" | perl -pe 's/([a-z0-9])([A-Z])/$1_\L$2/g')" + +pushd "./internal/service/$resource_name_lower_case" || exit + +# Running HashiCorp code generation tools + +echo "Generating provider code specification" +# Generate provider code specification using api spec and generator config +tfplugingen-openapi generate --config ./tfplugingen/generator_config.yml --output provider-code-spec.json ../../../api-spec.yml + +echo "Generating resource and data source schemas and models" +# Generate resource and data sources schemas using provider code specification +tfplugingen-framework generate data-sources --input provider-code-spec.json --output ./ --package "$resource_name_lower_case" +tfplugingen-framework generate resources --input provider-code-spec.json --output ./ --package "$resource_name_lower_case" + + +rm ../../../api-spec.yml +rm provider-code-spec.json + + +rename_file() { + local old_name=$1 + local new_name=$2 + + # Check if the original file exists + if [ -e "$old_name" ]; then + # If the target new name exists, use the alternative name with _gen + if [ -e "$new_name" ]; then + echo "File $new_name already exists, writing content in ${new_name%.*}_gen.go" + new_name="${new_name%.*}_gen.go" + fi + + echo "Created file in $new_name" + # Rename the file + mv "$old_name" "$new_name" + fi +} + +rename_file "${resource_name_snake_case}_data_source_gen.go" "data_source_${resource_name_snake_case}_schema.go" +rename_file "${resource_name_snake_case}s_data_source_gen.go" "data_source_${resource_name_snake_case}s_schema.go" +rename_file "${resource_name_snake_case}_resource_gen.go" "resource_${resource_name_snake_case}_schema.go"