Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Add support for jsonschema #1214

Merged
merged 4 commits into from
Sep 20, 2023
Merged

feat: Add support for jsonschema #1214

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
2267334
feat: Add support for jsonschema
yevgenypats Sep 16, 2023
5e72629
tidy
candiduslynx Sep 20, 2023
36c445f
Merge branch 'main' into feat/json_schema
candiduslynx Sep 20, 2023
b6cd5c5
tidy x2
candiduslynx Sep 20, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions examples/simple_plugin/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ require (
github.com/mattn/go-isatty v0.0.19 // indirect
github.com/pierrec/lz4/v4 v4.1.18 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 // indirect
github.com/spf13/cobra v1.6.1 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/stretchr/testify v1.8.4 // indirect
Expand Down
2 changes: 2 additions & 0 deletions examples/simple_plugin/go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,8 @@ github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/rs/zerolog v1.30.0 h1:SymVODrcRsaRaSInD9yQtKbtWqwsfoPcRff/oRXLj4c=
github.com/rs/zerolog v1.30.0/go.mod h1:/tk+P47gFdPXq4QYjvCmT5/Gsug2nagsFWBWhAiSi1w=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4=
github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA=
github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY=
Expand Down
1 change: 1 addition & 0 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ require (
github.com/google/uuid v1.3.1
github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0
github.com/rs/zerolog v1.29.1
github.com/santhosh-tekuri/jsonschema/v5 v5.3.1
github.com/spf13/cobra v1.6.1
github.com/stretchr/testify v1.8.4
github.com/thoas/go-funk v0.9.3
Expand Down
2 changes: 2 additions & 0 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -202,6 +202,8 @@ github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/rs/zerolog v1.29.1 h1:cO+d60CHkknCbvzEWxP0S9K6KqyTjrCNUy1LdQLCGPc=
github.com/rs/zerolog v1.29.1/go.mod h1:Le6ESbR7hc+DP6Lt1THiV8CQSdkkNrd3R0XbEgp3ZBU=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4=
github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA=
github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY=
Expand Down
6 changes: 6 additions & 0 deletions plugin/options.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,12 @@ func WithBuildTargets(targets []BuildTarget) Option {
}
}

func WithJSONSchema(schema string) Option {
return func(p *Plugin) {
p.schema = schema
}
}

type TableOptions struct {
Tables []string
SkipTables []string
Expand Down
31 changes: 31 additions & 0 deletions plugin/plugin.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,16 @@ package plugin

import (
"context"
"encoding/json"
"fmt"
"strings"
"sync"

"github.com/apache/arrow/go/v14/arrow"
"github.com/cloudquery/plugin-sdk/v4/message"
"github.com/cloudquery/plugin-sdk/v4/schema"
"github.com/rs/zerolog"
"github.com/santhosh-tekuri/jsonschema/v5"
)

var ErrNotImplemented = fmt.Errorf("not implemented")
Expand Down Expand Up @@ -66,6 +69,10 @@ type Plugin struct {
// NoInternalColumns if set to true will not add internal columns to tables such as _cq_id and _cq_parent_id
// useful for sources such as PostgreSQL and other databases
internalColumns bool
// schema is the JSONSchema of the plugin spec
schema string
// validator object to validate specs
schemaValidator *jsonschema.Schema
}

// NewPlugin returns a new CloudQuery Plugin with the given name, version and implementation.
Expand All @@ -81,6 +88,19 @@ func NewPlugin(name string, version string, newClient NewClientFunc, options ...
for _, opt := range options {
opt(&p)
}
if p.schema != "" {
c := jsonschema.NewCompiler()
c.Draft = jsonschema.Draft2020
if err := c.AddResource("schema.json", strings.NewReader(p.schema)); err != nil {
panic(fmt.Errorf("failed add plugin JSONSchema: %w", err))
}
schemaValidator, err := c.Compile("schema.json")
if err != nil {
panic(fmt.Errorf("failed to compile plugin JSONSchema: %w", err))
}
p.schemaValidator = schemaValidator
}

return &p
}

Expand Down Expand Up @@ -120,6 +140,17 @@ func (p *Plugin) Init(ctx context.Context, spec []byte, options NewClientOptions
}
defer p.mu.Unlock()
var err error

if p.schemaValidator != nil {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we want to validate the top level schema too via JSON schema? Otherwise the validation error messages might be confusing (some in the JSON schema validator format and others in the JSON decoder format). We'll probably need to do this in the CLI

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Also maybe you can share how a validation error looks like from the console

var v any
if err := json.Unmarshal(spec, &v); err != nil {
return fmt.Errorf("failed to unmarshal plugin spec: %w", err)
}
if err := p.schemaValidator.Validate(v); err != nil {
return fmt.Errorf("plugin spec is invalid: %w", err)
}
}

p.client, err = p.newClient(ctx, p.logger, spec, options)
if err != nil {
return fmt.Errorf("failed to initialize client: %w", err)
Expand Down
Loading