From ec3a607945bd87f788a7d60cdac1612c850d329b Mon Sep 17 00:00:00 2001 From: Alex Shcherbakov Date: Mon, 2 Oct 2023 18:07:52 +0300 Subject: [PATCH] feat: Use autogenerated JSON schema (#14111) Follow-up for #13876 Blocked by: - [x] https://github.com/invopop/jsonschema/pull/103 - [x] https://github.com/invopop/jsonschema/pull/105 - [x] https://github.com/cloudquery/codegen/pull/29 - [x] https://github.com/cloudquery/plugin-sdk/pull/1254 --- plugins/source/gcp/Makefile | 6 +- plugins/source/gcp/client/spec/gen/main.go | 24 +++ plugins/source/gcp/client/spec/schema.json | 162 ++++++++------------ plugins/source/gcp/client/spec/spec.go | 31 ++-- plugins/source/gcp/client/spec/spec_test.go | 9 +- plugins/source/gcp/go.mod | 3 +- plugins/source/gcp/go.sum | 2 + 7 files changed, 125 insertions(+), 112 deletions(-) create mode 100644 plugins/source/gcp/client/spec/gen/main.go diff --git a/plugins/source/gcp/Makefile b/plugins/source/gcp/Makefile index 2f13ba831b0aac..ecd111970abedb 100644 --- a/plugins/source/gcp/Makefile +++ b/plugins/source/gcp/Makefile @@ -40,6 +40,10 @@ gen-docs: build gen-services: go run codegen/main.go +.PHONY: gen-spec-schema +gen-spec-schema: + go run client/spec/gen/main.go + # All gen targets .PHONY: gen -gen: gen-docs +gen: gen-spec-schema gen-docs diff --git a/plugins/source/gcp/client/spec/gen/main.go b/plugins/source/gcp/client/spec/gen/main.go new file mode 100644 index 00000000000000..dde63caf815717 --- /dev/null +++ b/plugins/source/gcp/client/spec/gen/main.go @@ -0,0 +1,24 @@ +package main + +import ( + "fmt" + "log" + "path" + "runtime" + + "github.com/cloudquery/cloudquery/plugins/source/gcp/client/spec" + "github.com/cloudquery/codegen/jsonschema" +) + +func main() { + fmt.Println("Generating JSON schema for plugin spec") + jsonschema.GenerateIntoFile(new(spec.Spec), path.Join(currDir(), "..", "schema.json")) +} + +func currDir() string { + _, filename, _, ok := runtime.Caller(0) + if !ok { + log.Fatal("Failed to get caller information") + } + return path.Dir(filename) +} diff --git a/plugins/source/gcp/client/spec/schema.json b/plugins/source/gcp/client/spec/schema.json index 64c868e8023963..beb65779f78e47 100644 --- a/plugins/source/gcp/client/spec/schema.json +++ b/plugins/source/gcp/client/spec/schema.json @@ -1,159 +1,127 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/cloudquery/cloudquery/plugins/source/gcp/client/spec/spec", "$ref": "#/$defs/Spec", "$defs": { - "Spec": { - "title": "CloudQuery GCP Source Plugin Spec", + "CredentialsConfig": { + "properties": { + "target_principal": { + "type": "string", + "format": "email" + }, + "scopes": { + "items": { + "type": "string", + "pattern": "^https://www.googleapis.com/auth/(.)+$" + }, + "type": "array", + "default": [ + "https://www.googleapis.com/auth/cloud-platform" + ] + }, + "delegates": { + "items": { + "type": "string", + "format": "email" + }, + "type": "array" + }, + "subject": { + "type": "string", + "minLength": 1 + } + }, + "additionalProperties": false, "type": "object", + "required": [ + "target_principal" + ] + }, + "Spec": { + "not": { + "required": [ + "project_filter", + "folder_ids" + ] + }, "properties": { "project_ids": { - "title": "Project IDs", - "description": "GCP projects to connect to. By default will use all projects available to the current authenticated account", - "type": "array", "items": { "type": "string", "minLength": 1 - } + }, + "type": "array" }, "folder_ids": { - "title": "Folder IDs", - "description": "Location of the projects to sync from. Mutually exclusive with project_filter", - "type": "array", "items": { "type": "string", "pattern": "^(folders|organizations)/(.)+$" - } + }, + "type": "array" + }, + "folder_recursion_depth": { + "type": "integer", + "minimum": 0, + "default": 100 }, "organization_ids": { - "title": "Organization IDs", - "description": "List of GCP organization IDs to query. Defaults to all accessible projects", - "type": "array", "items": { "type": "string", "minLength": 1 - } + }, + "type": "array" }, "project_filter": { - "title": "Project Filter", - "description": "Filter to apply to projects. See https://cloud.google.com/resource-manager/docs/creating-managing-projects#listing_projects_with_a_filter for more information. Mutually exclusive with folder_ids", "type": "string" }, "organization_filter": { - "title": "Organization Filter", - "description": "Filter to apply to organizations. See https://cloud.google.com/resource-manager/docs/creating-managing-projects#listing_projects_with_a_filter for more information.", "type": "string" }, - "folder_recursion_depth": { - "title": "Folder Recursion Depth", - "description": "Number of levels of folders to recurse into. 0 means no recursion, 1 means only immediate children, etc.", - "type": "integer", - "minimum": 0, - "default": 100 - }, "service_account_key_json": { - "type": "string", - "description": "GCP service account key content. Using service accounts is not recommended, but if it is used it is better to use environment or file variable substitution" + "type": "string" }, "backoff_delay": { - "title": "Backoff Delay", - "description": "Number of seconds to wait before retrying a failed API call", "type": "integer", "minimum": 0, "default": 30 }, "backoff_retries": { - "title": "Backoff Retries", - "description": "Number of times to retry a failed API call", "type": "integer", "minimum": 0, "default": 0 }, "discovery_concurrency": { - "title": "Discovery Concurrency", - "description": "Number of concurrent API discovery requests to make", "type": "integer", "minimum": 1, "default": 100 }, "enabled_services_only": { - "title": "Enabled Services Only", - "description": "Whether only enabled services should be queried", "type": "boolean" }, "concurrency": { - "title": "Concurrency", - "description": "Number of concurrent API requests to make", "type": "integer", "minimum": 1, "default": 50000 }, "scheduler": { - "title": "Scheduler Strategy", - "description": "The scheduler strategy to use when determining the order of resources to sync", - "type": "string", - "enum": [ - "dfs", - "round-robin", - "shuffle" - ], - "default": "dfs" + "$ref": "#/$defs/Strategy" }, "service_account_impersonation": { - "title": "Service Account Impersonation", "$ref": "#/$defs/CredentialsConfig" } }, - "not": { - "description": "project_filter & folder_ids are mutually exclusive", - "required": [ - "project_filter", - "folder_ids" - ] - }, - "additionalProperties": false - }, - "CredentialsConfig": { - "type": "object", - "title": "Credentials Config", - "properties": { - "target_principal": { - "title": "Target Principal", - "type": "string", - "description": "Email address of the service account to impersonate", - "format": "email" - }, - "scopes": { - "title": "Scopes", - "description": "OAuth 2.0 scopes that the impersonated credential should have. See https://developers.google.com/identity/protocols/oauth2/scopes for more details.", - "type": "array", - "items": { - "type": "string", - "pattern": "^https://www.googleapis.com/auth/(.)+$" - }, - "default": [ - "https://www.googleapis.com/auth/cloud-platform" - ] - }, - "delegates": { - "title": "Delegates", - "description": "Service account email addresses in a delegation chain. Each service account must be granted roles/iam.serviceAccountTokenCreator on the next service account in the chain.", - "type": "array", - "items": { - "type": "string", - "format": "email" - } - }, - "subject": { - "title": "Subject", - "description": "The subject field of a JWT (sub)", - "type": "string", - "minLength": 1 - } - }, "additionalProperties": false, - "required": [ - "target_principal" - ] + "type": "object" + }, + "Strategy": { + "type": "string", + "enum": [ + "dfs", + "round-robin", + "shuffle" + ], + "title": "CloudQuery scheduling strategy", + "default": "dfs" } } } diff --git a/plugins/source/gcp/client/spec/spec.go b/plugins/source/gcp/client/spec/spec.go index 65463fd331de77..12830ae38efe4d 100644 --- a/plugins/source/gcp/client/spec/spec.go +++ b/plugins/source/gcp/client/spec/spec.go @@ -4,23 +4,24 @@ import ( _ "embed" "github.com/cloudquery/plugin-sdk/v4/scheduler" + "github.com/invopop/jsonschema" "google.golang.org/api/cloudresourcemanager/v1" ) // Spec defines GCP source plugin Spec type Spec struct { - ProjectIDs []string `json:"project_ids"` - FolderIDs []string `json:"folder_ids"` - FolderRecursionDepth *int `json:"folder_recursion_depth"` - OrganizationIDs []string `json:"organization_ids"` + ProjectIDs []string `json:"project_ids" jsonschema:"minLength=1"` + FolderIDs []string `json:"folder_ids" jsonschema:"pattern=^(folders|organizations)/(.)+$"` + FolderRecursionDepth *int `json:"folder_recursion_depth" jsonschema:"minimum=0,default=100"` + OrganizationIDs []string `json:"organization_ids" jsonschema:"minLength=1"` ProjectFilter string `json:"project_filter"` OrganizationFilter string `json:"organization_filter"` ServiceAccountKeyJSON string `json:"service_account_key_json"` - BackoffDelay int `json:"backoff_delay"` - BackoffRetries int `json:"backoff_retries"` - DiscoveryConcurrency int `json:"discovery_concurrency"` + BackoffDelay int `json:"backoff_delay" jsonschema:"minimum=0,default=30"` + BackoffRetries int `json:"backoff_retries" jsonschema:"minimum=0,default=0"` + DiscoveryConcurrency int `json:"discovery_concurrency" jsonschema:"minimum=1,default=100"` EnabledServicesOnly bool `json:"enabled_services_only"` - Concurrency int `json:"concurrency"` + Concurrency int `json:"concurrency" jsonschema:"minimum=1,default=50000"` Scheduler scheduler.Strategy `json:"scheduler,omitempty"` ServiceAccountImpersonation *CredentialsConfig `json:"service_account_impersonation"` } @@ -54,23 +55,29 @@ func (spec *Spec) SetDefaults() { spec.ServiceAccountImpersonation.SetDefaults() } +// JSONSchemaExtend is required to add `not` section for `project_filter` & `folder_ids` being mutually exclusive. +// We use value receiver because of https://github.com/invopop/jsonschema/issues/102 +func (Spec) JSONSchemaExtend(sc *jsonschema.Schema) { + sc.Not = &jsonschema.Schema{Required: []string{"project_filter", "folder_ids"}} +} + //go:embed schema.json var JSONSchema string type CredentialsConfig struct { // TargetPrincipal is the email address of the service account to // impersonate. Required. - TargetPrincipal string `json:"target_principal"` + TargetPrincipal string `json:"target_principal" jsonschema:"required,format=email"` // Scopes that the impersonated credential should have. Required. - Scopes []string `json:"scopes"` + Scopes []string `json:"scopes" jsonschema:"pattern=^https://www.googleapis.com/auth/(.)+$,default=https://www.googleapis.com/auth/cloud-platform"` // Delegates are the service account email addresses in a delegation chain. // Each service account must be granted roles/iam.serviceAccountTokenCreator // on the next service account in the chain. Optional. - Delegates []string `json:"delegates"` + Delegates []string `json:"delegates" jsonschema:"format=email"` // Subject is the subject field of a JWT (sub). This field should only be set if you // wish to impersonate as a user. This feature is useful when using domain // wide delegation. Optional. - Subject string `json:"subject"` + Subject string `json:"subject" jsonschema:"minLength=1"` } func (c *CredentialsConfig) SetDefaults() { diff --git a/plugins/source/gcp/client/spec/spec_test.go b/plugins/source/gcp/client/spec/spec_test.go index 51dc14fe0f7d83..3ff0374b212dff 100644 --- a/plugins/source/gcp/client/spec/spec_test.go +++ b/plugins/source/gcp/client/spec/spec_test.go @@ -4,11 +4,12 @@ import ( "encoding/json" "testing" + "github.com/cloudquery/codegen/jsonschema" "github.com/cloudquery/plugin-sdk/v4/plugin" "github.com/stretchr/testify/require" ) -func TestSpec(t *testing.T) { +func TestJSONSchema(t *testing.T) { validator, err := plugin.JSONSchemaValidator(JSONSchema) require.NoError(t, err) @@ -377,3 +378,9 @@ func TestSpec(t *testing.T) { }) } } + +func TestEnsureJSONSchema(t *testing.T) { + data, err := jsonschema.Generate(new(Spec)) + require.NoError(t, err) + require.JSONEqf(t, string(data), JSONSchema, "new schema should be:\n%s\n", string(data)) +} diff --git a/plugins/source/gcp/go.mod b/plugins/source/gcp/go.mod index 1cd682f2bb44de..62a2ad39f82f3b 100644 --- a/plugins/source/gcp/go.mod +++ b/plugins/source/gcp/go.mod @@ -45,12 +45,14 @@ require ( cloud.google.com/go/websecurityscanner v1.6.1 cloud.google.com/go/workflows v1.12.0 github.com/apache/arrow/go/v14 v14.0.0-20230929201650-00efb06dc0de + github.com/cloudquery/codegen v0.3.3 github.com/cloudquery/plugin-sdk/v4 v4.12.0 github.com/cockroachdb/cockroachdb-parser v0.0.0-20230705064001-302c9ad52e1a github.com/golang/mock v1.6.0 github.com/googleapis/gax-go/v2 v2.12.0 github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.1 github.com/iancoleman/strcase v0.2.0 + github.com/invopop/jsonschema v0.11.0 github.com/julienschmidt/httprouter v1.3.0 github.com/mjibson/sqlfmt v0.5.0 github.com/rs/zerolog v1.29.1 @@ -109,7 +111,6 @@ require ( github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/invopop/jsonschema v0.11.0 // indirect github.com/klauspost/compress v1.16.7 // indirect github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/kr/pretty v0.3.1 // indirect diff --git a/plugins/source/gcp/go.sum b/plugins/source/gcp/go.sum index 2445049ba25a0e..26a4ec769cb9fd 100644 --- a/plugins/source/gcp/go.sum +++ b/plugins/source/gcp/go.sum @@ -177,6 +177,8 @@ github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2u github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudquery/arrow/go/v14 v14.0.0-20231002001222-7ded38b478cd h1:LtWC4oaOh0kI5Xhgl0U4w04vm9qdux/H1E0XBv/Lyio= github.com/cloudquery/arrow/go/v14 v14.0.0-20231002001222-7ded38b478cd/go.mod h1:/SqmdO2dsWqFHqQQeupnsr0ollL8C91n3x0I72rArY8= +github.com/cloudquery/codegen v0.3.3 h1:riM+5GVXcjNJ8gmgRocSY+QiZedmmjvM9TwEBTfzBr4= +github.com/cloudquery/codegen v0.3.3/go.mod h1:+UAJNPydpAJfMwYKhJgzogwebKdY4sJolH/LuQoTDC0= github.com/cloudquery/plugin-pb-go v1.11.0 h1:fmuFFI0+R4gH4w/ehWOEZW1ajK6XFj3xGEyJEYoD3DI= github.com/cloudquery/plugin-pb-go v1.11.0/go.mod h1:QbDGHLlQ2+Gp9OltuKvhQ9Bmn7OaR1kE35voGFfvpXI= github.com/cloudquery/plugin-sdk/v2 v2.7.0 h1:hRXsdEiaOxJtsn/wZMFQC9/jPfU1MeMK3KF+gPGqm7U=