Update new provider documentation

cmd/scraper/main.go

@@ -19,7 +19,7 @@ func main() {
 		outFile      = app.Flag("out", "Provider metadata output file path").Short('o').Default("provider-metadata.yaml").OpenFile(os.O_CREATE, 0644)
 		providerName = app.Flag("name", "Provider name").Short('n').Required().String()
 		codeXPath    = app.Flag("code-xpath", "Code XPath expression").Default(`//code[@class="language-terraform" or @class="language-hcl"]/text()`).String()
-		preludeXPath = app.Flag("prelude-xpath", "Prelude XPath expression").Default(`//text()[contains(., "description") and contains(., "subcategory")]`).String()
+		preludeXPath = app.Flag("prelude-xpath", "Prelude XPath expression").Default(`//text()[contains(., "description") and contains(., "page_title")]`).String()
 		fieldXPath   = app.Flag("field-xpath", "Field documentation XPath expression").Default(`//ul/li//code[1]/text()`).String()
 		importXPath  = app.Flag("import-xpath", "Import statements XPath expression").Default(`//code[@class="language-shell"]/text()`).String()
 		repoPath     = app.Flag("repo", "Terraform provider repo path").Short('r').Required().ExistingDir()

docs/generating-a-provider.md

@@ -4,7 +4,8 @@ In this guide, we will generate a Crossplane provider based on an existing
 Terraform provider using Upjet.
 
 We have chosen [Terraform GitHub provider] as an example, but the process will
-be quite similar for any other Terraform provider.
+be quite similar for any other Terraform provider. We will use `myorg` as the
+example organization name to be used.
 
 ## Generate
 
@@ -19,15 +20,9 @@ be quite similar for any other Terraform provider.
 
 3. Replace `template` with your provider name.
 
-    1. Export `ProviderName`:
-
-        ```bash
-        export ProviderNameLower=github
-        export ProviderNameUpper=GitHub
-        ```
-
-    2. Run the `./hack/prepare.sh` script from repo root to prepare the repo, e.g., to
-       replace all occurrences of `template` with your provider name:
+    1. Run the `./hack/prepare.sh` script from repo root to prepare the repo, e.g., to
+       replace all occurrences of `template` with your provider name and `upbound`
+       with your organization name:
 
         ```bash
        ./hack/prepare.sh
@@ -38,19 +33,22 @@ be quite similar for any other Terraform provider.
 
     ```makefile
     export TERRAFORM_PROVIDER_SOURCE := integrations/github
-    export TERRAFORM_PROVIDER_VERSION := 4.19.2
+    export TERRAFORM_PROVIDER_REPO := https://github.com/integrations/terraform-provider-github
+    export TERRAFORM_PROVIDER_VERSION := 5.5.0
     export TERRAFORM_PROVIDER_DOWNLOAD_NAME := terraform-provider-github
-    export TERRAFORM_PROVIDER_DOWNLOAD_URL_PREFIX := https://releases.hashicorp.com/terraform-provider-github/4.19.2
+    export TERRAFORM_NATIVE_PROVIDER_BINARY := terraform-provider-github_v5.5.0_x5
+    export TERRAFORM_DOCS_PATH := website/docs/r
     ```
 
    You can find `TERRAFORM_PROVIDER_SOURCE` and `TERRAFORM_PROVIDER_VERSION` in
    [Terraform GitHub provider] documentation by hitting the "**USE PROVIDER**"
    button. Check [this line in controller Dockerfile] to see how these
-   variables are used to build the provider plugin binary.
-
+   variables are used to build the provider plugin binary. `TERRAFORM_DOCS_PATH`
+   is the directory where resource documentation is stored in the repository of
+   the Terraform provider.
 
 5. Implement `ProviderConfig` logic. In `upjet-provider-template`, there is already
-   a boilerplate code in file `internal/clients/${ProviderNameLower}.go` which
+   a boilerplate code in file `internal/clients/github.go` which
    takes care of properly fetching secret data referenced from `ProviderConfig`
    resource.
 
@@ -62,24 +60,20 @@ be quite similar for any other Terraform provider.
      keyBaseURL = "base_url"
      keyOwner = "owner"
      keyToken = "token"
-
-     // GitHub credentials environment variable names
-     envToken = "GITHUB_TOKEN"
    )
 
    func TerraformSetupBuilder(version, providerSource, providerVersion string) terraform.SetupFn {
      ...
      // set provider configuration
      ps.Configuration = map[string]any{}
-     if v, ok := githubCreds[keyBaseURL]; ok {
-         ps.Configuration[keyBaseURL] = v
+     if v, ok := creds[keyBaseURL]; ok {
+       ps.Configuration[keyBaseURL] = v
      }
-     if v, ok := githubCreds[keyOwner]; ok {
-         ps.Configuration[keyOwner] = v
+     if v, ok := creds[keyOwner]; ok {
+       ps.Configuration[keyOwner] = v
      }
-     // set environment variables for sensitive provider configuration
-     ps.Env = []string{
-         fmt.Sprintf(fmtEnvVar, envToken, githubCreds[keyToken]),
+     if v, ok := creds[keyToken]; ok {
+       ps.Configuration[keyToken] = v
      }
      return ps, nil
    }
@@ -89,15 +83,20 @@ be quite similar for any other Terraform provider.
    and only start with generating CRDs for [github_repository] and
    [github_branch] Terraform resources.
 
-   To limit the resources to be generated, we need to provide an include list
-   option with `tjconfig.WithIncludeList` in file `config/provider.go`:
+   Only the resources with external name configuration should be generated.
+   Let's add external name configurations for these two resources in 
+   `config/external_name.go` as an entry to the map called `ExternalNameConfigs`:
 
    ```go
-   pc := tjconfig.NewProviderWithSchema([]byte(providerSchema), resourcePrefix, modulePath,
-       tjconfig.WithIncludeList([]string{
-           "github_repository$",
-           "github_branch$",
-       }))
+   // ExternalNameConfigs contains all external name configurations for this
+   // provider.
+   var ExternalNameConfigs = map[string]config.ExternalName{
+     // Name is a parameter and it is also used to import the resource.
+     "github_repository": config.NameAsIdentifier,
+     // The import ID consists of several parameters. We'll use branch name as
+     // the external name.
+     "github_branch": config.TemplatedStringAsIdentifier("branch", "{{ .parameters.repository }}:{{ .external_name }}:{{ .parameters.branch }}"),
+   }
    ```
 
 7. Finally, we would need to add some custom configurations for these two
@@ -119,8 +118,7 @@ be quite similar for any other Terraform provider.
    // Configure configures individual resources by adding custom ResourceConfigurators.
    func Configure(p *config.Provider) {
        p.AddResourceConfigurator("github_repository", func(r *config.Resource) {
-
-           // we need to override the default group that upjet generated for
+           // We need to override the default group that upjet generated for
            // this resource, which would be "github"
            r.ShortGroup = "repository"
        })
@@ -129,28 +127,24 @@ be quite similar for any other Terraform provider.
    ```
 
    ```bash
+   # Note that you need to change `myorg/provider-github`.
    cat <<EOF > config/branch/config.go
    package branch
 
    import "github.com/upbound/upjet/pkg/config"
 
    func Configure(p *config.Provider) {
        p.AddResourceConfigurator("github_branch", func(r *config.Resource) {
-
-           // we need to override the default group that upjet generated for
+           // We need to override the default group that upjet generated for
            // this resource, which would be "github"
            r.ShortGroup = "branch"
 
-           // Identifier for this resource is assigned by the provider. In other
-           // words it is not simply the name of the resource.
-           r.ExternalName = config.IdentifierFromProvider
-
            // This resource need the repository in which branch would be created
            // as an input. And by defining it as a reference to Repository
            // object, we can build cross resource referencing. See
            // repositoryRef in the example in the Testing section below.
            r.References["repository"] = config.Reference{
-               Type: "github.com/crossplane-contrib/provider-jet-github/apis/repository/v1alpha1.Repository",
+               Type: "github.com/myorg/provider-github/apis/repository/v1alpha1.Repository",
            }
        })
    }
@@ -167,14 +161,15 @@ be quite similar for any other Terraform provider.
        "github.com/upbound/upjet/pkg/types/conversion/cli"
        "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
 
-   +   "github.com/crossplane-contrib/provider-jet-github/config/branch"
-   +   "github.com/crossplane-contrib/provider-jet-github/config/repository"
+   +   "github.com/myorg/provider-github/config/branch"
+   +   "github.com/myorg/provider-github/config/repository"
     )
 
     func GetProvider() *tjconfig.Provider {
        ...
        for _, configure := range []func(provider *tjconfig.Provider){
                // add custom config functions
+   -           null.Configure,
    +           repository.Configure,
    +           branch.Configure,
        } {
@@ -192,13 +187,9 @@ be quite similar for any other Terraform provider.
    make generate
    ```
 
-### Adding New Resources
-
-To add more resources, please **follow the steps between 6-8 for each resource**.
+### Adding More Resources
 
-Alternatively, you can drop the `tjconfig.WithIncludeList` option in provider
-Configuration which would generate all resources, and you can add resource
-configurations as a next step.
+See the guide [here][new-resource-short] to add more resources.
 
 ## Test
 
@@ -213,7 +204,7 @@ Now let's test our generated resources.
    mkdir examples/branch
 
    # remove the sample directory which was an example in the template
-   rm -rf examples/sample
+   rm -rf examples/null
    ```
 
    Create a provider secret template:
@@ -240,7 +231,7 @@ Now let's test our generated resources.
 
    ```bash
    cat <<EOF > examples/repository/repository.yaml
-   apiVersion: repository.github.jet.crossplane.io/v1alpha1
+   apiVersion: repository.github.upbound.io/v1alpha1
    kind: Repository
    metadata:
      name: hello-crossplane
@@ -249,7 +240,7 @@ Now let's test our generated resources.
        description: "Managed with Crossplane Github Provider (generated with Upjet)"
        visibility: public
        template:
-         - owner: crossplane-contrib
+         - owner: upbound
            repository: upjet-provider-template
      providerConfigRef:
        name: default
@@ -261,20 +252,22 @@ Now let's test our generated resources.
 
    ```bash
    cat <<EOF > examples/branch/branch.yaml
-   apiVersion: branch.github.jet.crossplane.io/v1alpha1
+   apiVersion: branch.github.upbound.io/v1alpha1
    kind: Branch
    metadata:
      name: hello-upjet
    spec:
      forProvider:
-       branch: hello-upjet
        repositoryRef:
          name: hello-crossplane
      providerConfigRef:
        name: default
    EOF
    ```
 
+   In order to change the `apiVersion`, you can use `WithRootGroup` and `WithShortName`
+   options in `config/provider.go` as arguments to `ujconfig.NewProvider`.
+
 2. Generate a [Personal Access Token](https://github.com/settings/tokens) for
    your Github account with `repo/public_repo` and `delete_repo` scopes.
 
@@ -327,6 +320,8 @@ Now let's test our generated resources.
    Verify that repo `hello-crossplane` and branch `hello-upjet` created under
    your GitHub account.
 
+8. You can check the errors and events by calling `kubectl describe` for either
+   of the resources.
 
 9. Cleanup
 
@@ -347,3 +342,4 @@ Now let's test our generated resources.
 [github_branch]: https://registry.terraform.io/providers/integrations/github/latest/docs/resources/branch
 [this line in controller Dockerfile]: https://github.com/upbound/upjet-provider-template/blob/main/cluster/images/official-provider-template-controller/Dockerfile#L18-L26
 [terraform-plugin-sdk]: https://github.com/hashicorp/terraform-plugin-sdk
+[new-resource-short]: add-new-resource-short.md

pkg/config/provider.go

@@ -192,8 +192,8 @@ func NewProvider(schema []byte, prefix string, modulePath string, metadata []byt
 	p := &Provider{
 		ModulePath:              modulePath,
 		TerraformResourcePrefix: fmt.Sprintf("%s_", prefix),
-		RootGroup:               fmt.Sprintf("%s.jet.crossplane.io", prefix),
-		ShortName:               fmt.Sprintf("%sjet", prefix),
+		RootGroup:               fmt.Sprintf("%s.upbound.io", prefix),
+		ShortName:               fmt.Sprintf("%s", prefix),
 		BasePackages:            DefaultBasePackages,
 		IncludeList: []string{
 			// Include all Resources

pkg/registry/meta.go

@@ -230,8 +230,8 @@ func (r *Resource) scrapePrelude(doc *html.Node, preludeXPath string) error {
 	}
 	r.Description = strings.TrimSpace(strings.Replace(r.Description, "|-", "", 1))
 
-	if r.SubCategory == "" || r.Title == "" {
-		return errors.Errorf("failed to parse prelude. Description: %s, Subcategory: %s, Title name: %s. Raw data:%s\n",
+	if r.Title == "" {
+		return errors.Errorf("failed to find title. Description: %s, Subcategory: %s, Title name: %s. Raw data:%s\n",
 			r.Description, r.SubCategory, r.Title, rawData)
 	}
 	return nil