feat!: Update to SDK V4 & fix infinite loop on transactions (#10)

.github/workflows/test.yml

@@ -21,9 +21,9 @@ jobs:
           go-version-file: go.mod
           cache: true
       - name: golangci-lint
-        uses: golangci/golangci-lint-action@v3
+        uses: golangci/golangci-lint-action@v6
         with:
-          version: v1.50.1
+          version: v1.59.1
       - name: Get dependencies
         run: go get -t -d ./...
       - name: Build

.golangci.yml

@@ -0,0 +1,122 @@
+run:
+  tests: true
+  skip-dirs:
+    - bin
+    - docs
+    - client/mocks
+    - resources/forks
+    - codegen
+    - internal/trello
+  timeout: 15m
+  build-tags:
+    - all
+
+linters-settings:
+  errcheck:
+    check-blank: false
+    ignore: fmt:.*,[rR]ead|[wW]rite|[cC]lose,io:Copy
+  gocritic:
+    disabled-checks:
+      - commentFormatting
+  dupl:
+    # tokens count to trigger issue, 150 by default
+    threshold: 500
+
+  misspell:
+    ignore-words:
+      - hdinsight
+
+  revive:
+    enable-all-rules: true
+    rules:
+      - name: cyclomatic
+        disabled: true
+      - name: argument-limit
+        disabled: true
+      - name: function-length
+        disabled: true
+      - name: function-result-limit
+        disabled: true
+      - name: line-length-limit
+        disabled: true
+      - name: file-header
+        disabled: true
+      - name: cognitive-complexity
+        disabled: true
+      - name: banned-characters
+        disabled: true
+      - name: max-public-structs
+        disabled: true
+      - name: add-constant
+        disabled: true
+      - name: unhandled-error
+        disabled: true
+      - name: var-naming
+        disabled: true
+      - name: deep-exit
+        disabled: true
+      - name: exported
+        disabled: false
+        arguments:
+          - 'disableStutteringCheck'
+      - name: unused-parameter
+        disabled: true
+      - name: confusing-naming
+        disabled: true
+      - name: confusing-results
+        disabled: true
+      - name: flag-parameter
+        disabled: true
+      - name: nested-structs
+        disabled: true
+      - name: unchecked-type-assertion
+        disabled: true
+      - name: import-alias-naming
+        disabled: true
+      - name: redundant-import-alias
+        disabled: true
+
+  gofmt:
+    rewrite-rules:
+    - pattern:     'interface{}'
+      replacement: 'any'
+    - pattern:     'a[b:len(a)]'
+      replacement: 'a[b:]'
+
+
+linters:
+  enable:
+    - asciicheck
+    - bodyclose
+    - dupl
+    - errcheck
+    - gocritic
+    - gofmt
+    - gosimple
+    - govet
+    - ineffassign
+    - importas
+    - misspell
+    - nakedret
+    - prealloc
+    - revive
+    - staticcheck
+    - unconvert
+    - unparam
+    - unused
+
+issues:
+  exclude-rules:
+    # Exclude some linters from running on tests files.
+    - path: _test\.go
+      linters:
+        - dupl
+        - gocritic
+      # Exclude some linters from running on resource files.
+    - path: resources(\\|\/).*\.go
+      linters:
+        - dupl
+    # Exclude some linters from running on services files.
+    - path: services\.go
+      linters:
+        - dupl

client/client.go

@@ -2,12 +2,10 @@ package client
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"strings"
 
-	"github.com/cloudquery/plugin-pb-go/specs"
-	"github.com/cloudquery/plugin-sdk/v3/plugins/source"
-	"github.com/cloudquery/plugin-sdk/v3/schema"
 	"github.com/hashicorp/go-retryablehttp"
 	"github.com/plaid/plaid-go/v10/plaid"
 	"github.com/rs/zerolog"
@@ -19,28 +17,29 @@ type Client struct {
 	ClientId    string
 	Secret      string
 	AccessToken string
+	Spec        Spec
 }
 
-func (c *Client) ID() string {
+func (*Client) ID() string {
 	return "plaid"
 }
 
 type httpLogger struct {
 	zerolog.Logger
 }
 
-func (l httpLogger) Printf(format string, v ...interface{}) {
+func (l httpLogger) Printf(format string, v ...any) {
 	if strings.Contains(format, "retrying") {
 		l.Logger.Info().Msgf(format, v...)
 	} else {
 		l.Logger.Debug().Msgf(format, v...)
 	}
 }
 
-func New(ctx context.Context, logger zerolog.Logger, s specs.Source, opts source.Options) (schema.ClientMeta, error) {
+func New(ctx context.Context, logger zerolog.Logger, s []byte) (*Client, error) {
 	var pluginSpec Spec
 
-	if err := s.UnmarshalSpec(&pluginSpec); err != nil {
+	if err := json.Unmarshal(s, &pluginSpec); err != nil {
 		return nil, fmt.Errorf("failed to unmarshal plugin spec: %w", err)
 	}
 

client/resolvers.go

@@ -4,7 +4,7 @@ import (
 	"context"
 	"fmt"
 
-	"github.com/cloudquery/plugin-sdk/v3/schema"
+	"github.com/cloudquery/plugin-sdk/v4/schema"
 	"github.com/plaid/plaid-go/v10/plaid"
 	"github.com/thoas/go-funk"
 )

client/testing.go

@@ -10,9 +10,10 @@ import (
 	"testing"
 	"time"
 
-	"github.com/cloudquery/plugin-pb-go/specs"
-	"github.com/cloudquery/plugin-sdk/v3/plugins/source"
-	"github.com/cloudquery/plugin-sdk/v3/schema"
+	"github.com/cloudquery/plugin-sdk/v4/plugin"
+	"github.com/cloudquery/plugin-sdk/v4/scheduler"
+	"github.com/cloudquery/plugin-sdk/v4/schema"
+	"github.com/cloudquery/plugin-sdk/v4/transformers"
 	"github.com/plaid/plaid-go/v10/plaid"
 	"github.com/rs/zerolog"
 )
@@ -28,49 +29,67 @@ func TestServer(t *testing.T, data any) *httptest.Server {
 	return ts
 }
 
+func remove(s schema.ColumnList, i string) schema.ColumnList {
+	for j, c := range s {
+		if c.Name == i {
+			return append(s[:j], s[j+1:]...)
+		}
+	}
+
+	return s
+}
+
 func TestHelper(t *testing.T, table *schema.Table, ts *httptest.Server) {
-	version := "vDev"
 	table.IgnoreInTests = false
 	t.Helper()
-	l := zerolog.New(zerolog.NewTestWriter(t)).Output(zerolog.ConsoleWriter{Out: os.Stderr, TimeFormat: time.StampMicro}).Level(zerolog.DebugLevel).With().Timestamp().Logger()
 
-	newTestExecutionClient := func(ctx context.Context, logger zerolog.Logger, spec specs.Source, opts source.Options) (schema.ClientMeta, error) {
-		configuration := plaid.NewConfiguration()
-		configuration.UseEnvironment(plaid.Development)
-		urlParts := strings.Split(ts.URL, "://")
-		configuration.Scheme = urlParts[0]
-		configuration.Host = urlParts[1]
-		client := plaid.NewAPIClient(configuration)
-		s := Spec{
-			ClientId:    "test",
-			Secret:      "test",
-			AccessToken: "test",
-		}
-		s.SetDefaults()
-		err := s.Validate()
-		if err != nil {
-			return nil, err
-		}
-		return &Client{
-			Logger:   l,
-			Services: client,
-			ClientId: s.ClientId,
-			Secret:   s.Secret,
-		}, nil
+	l := zerolog.New(zerolog.NewTestWriter(t)).Output(
+		zerolog.ConsoleWriter{Out: os.Stderr, TimeFormat: time.StampMicro},
+	).Level(zerolog.DebugLevel).With().Timestamp().Logger()
+	sched := scheduler.NewScheduler(scheduler.WithLogger(l))
+
+	configuration := plaid.NewConfiguration()
+	configuration.UseEnvironment(plaid.Development)
+	urlParts := strings.Split(ts.URL, "://")
+	configuration.Scheme = urlParts[0]
+	configuration.Host = urlParts[1]
+	client := plaid.NewAPIClient(configuration)
+	s := Spec{
+		ClientId:    "test",
+		Secret:      "test",
+		AccessToken: "test",
+	}
+	s.SetDefaults()
+	err := s.Validate()
+	if err != nil {
+		t.Fatal(err)
 	}
-	p := source.NewPlugin(
-		table.Name,
-		version,
-		[]*schema.Table{
-			table,
-		},
-		newTestExecutionClient)
-	p.SetLogger(l)
-	source.TestPluginSync(t, p, specs.Source{
-		Name:         "dev",
-		Path:         "cloudquery/dev",
-		Version:      version,
-		Tables:       []string{table.Name},
-		Destinations: []string{"mock-destination"},
+
+	c := &Client{
+		Logger:      l,
+		Services:    client,
+		ClientId:    s.ClientId,
+		Secret:      s.Secret,
+		AccessToken: s.AccessToken,
+	}
+
+	tables := schema.Tables{table}
+	if err := transformers.TransformTables(tables); err != nil {
+		t.Fatal(err)
+	}
+
+	// We need to remove additional_properties column from the table as faker cannot generate data with interface{} type
+	err = transformers.Apply(tables, func(table *schema.Table) error {
+		table.Columns = remove(table.Columns, "additional_properties")
+		return nil
 	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	messages, err := sched.SyncAll(context.Background(), c, tables)
+	if err != nil {
+		t.Fatalf("failed to sync: %v", err)
+	}
+	plugin.ValidateNoEmptyColumns(t, tables, messages)
 }

client/transformers.go

@@ -3,9 +3,9 @@ package client
 import (
 	"reflect"
 
-	"github.com/apache/arrow/go/v13/arrow"
-	"github.com/cloudquery/plugin-sdk/v3/schema"
-	"github.com/cloudquery/plugin-sdk/v3/transformers"
+	"github.com/apache/arrow/go/v16/arrow"
+	"github.com/cloudquery/plugin-sdk/v4/schema"
+	"github.com/cloudquery/plugin-sdk/v4/transformers"
 	"github.com/plaid/plaid-go/v10/plaid"
 )
 

docs/tables/plaid_account_balances.md

@@ -8,8 +8,6 @@ The primary key for this table is **item_id**.
 
 | Name          | Type          |
 | ------------- | ------------- |
-|_cq_source_name|`utf8`|
-|_cq_sync_time|`timestamp[us, tz=UTC]`|
 |_cq_id|`uuid`|
 |_cq_parent_id|`uuid`|
 |item_id (PK)|`utf8`|

docs/tables/plaid_auths.md

@@ -8,8 +8,6 @@ The primary key for this table is **item_id**.
 
 | Name          | Type          |
 | ------------- | ------------- |
-|_cq_source_name|`utf8`|
-|_cq_sync_time|`timestamp[us, tz=UTC]`|
 |_cq_id|`uuid`|
 |_cq_parent_id|`uuid`|
 |item_id (PK)|`utf8`|

docs/tables/plaid_identities.md

@@ -8,8 +8,6 @@ The primary key for this table is **item_id**.
 
 | Name          | Type          |
 | ------------- | ------------- |
-|_cq_source_name|`utf8`|
-|_cq_sync_time|`timestamp[us, tz=UTC]`|
 |_cq_id|`uuid`|
 |_cq_parent_id|`uuid`|
 |item_id (PK)|`utf8`|

docs/tables/plaid_institutions.md

@@ -8,8 +8,6 @@ The primary key for this table is **institution_id**.
 
 | Name          | Type          |
 | ------------- | ------------- |
-|_cq_source_name|`utf8`|
-|_cq_sync_time|`timestamp[us, tz=UTC]`|
 |_cq_id|`uuid`|
 |_cq_parent_id|`uuid`|
 |institution_id (PK)|`utf8`|

docs/tables/plaid_investments_holdings.md

@@ -8,8 +8,6 @@ The primary key for this table is **item_id**.
 
 | Name          | Type          |
 | ------------- | ------------- |
-|_cq_source_name|`utf8`|
-|_cq_sync_time|`timestamp[us, tz=UTC]`|
 |_cq_id|`uuid`|
 |_cq_parent_id|`uuid`|
 |item_id (PK)|`utf8`|

docs/tables/plaid_investments_transactions.md

@@ -8,8 +8,6 @@ The primary key for this table is **item_id**.
 
 | Name          | Type          |
 | ------------- | ------------- |
-|_cq_source_name|`utf8`|
-|_cq_sync_time|`timestamp[us, tz=UTC]`|
 |_cq_id|`uuid`|
 |_cq_parent_id|`uuid`|
 |item_id (PK)|`utf8`|

docs/tables/plaid_liabilities.md

@@ -8,8 +8,6 @@ The primary key for this table is **item_id**.
 
 | Name          | Type          |
 | ------------- | ------------- |
-|_cq_source_name|`utf8`|
-|_cq_sync_time|`timestamp[us, tz=UTC]`|
 |_cq_id|`uuid`|
 |_cq_parent_id|`uuid`|
 |item_id (PK)|`utf8`|

docs/tables/plaid_transactions.md

@@ -8,8 +8,6 @@ The composite primary key for this table is (**transaction_type**, **transaction
 
 | Name          | Type          |
 | ------------- | ------------- |
-|_cq_source_name|`utf8`|
-|_cq_sync_time|`timestamp[us, tz=UTC]`|
 |_cq_id|`uuid`|
 |_cq_parent_id|`uuid`|
 |transaction_type (PK)|`utf8`|