Skip to content

Commit

Permalink
data.json plugin (#30)
Browse files Browse the repository at this point in the history
  • Loading branch information
dobarx authored Jan 16, 2024
1 parent e323400 commit 1851f2d
Show file tree
Hide file tree
Showing 12 changed files with 598 additions and 6 deletions.
7 changes: 5 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,20 +3,22 @@ module github.com/blackstork-io/fabric
go 1.21.5

require (
github.com/Masterminds/semver/v3 v3.2.1
github.com/hashicorp/go-hclog v0.14.1
github.com/hashicorp/go-plugin v1.6.0
github.com/hashicorp/hcl/v2 v2.19.1
github.com/itchyny/gojq v0.12.14
github.com/stretchr/testify v1.8.4
github.com/zclconf/go-cty v1.13.0
golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611
golang.org/x/term v0.15.0
)

require (
github.com/Masterminds/semver/v3 v3.2.1 // indirect
github.com/agext/levenshtein v1.2.1 // indirect
github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect
github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/fatih/color v1.7.0 // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/google/go-cmp v0.5.9 // indirect
Expand All @@ -27,11 +29,12 @@ require (
github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77 // indirect
github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7 // indirect
github.com/oklog/run v1.0.0 // indirect
github.com/stretchr/testify v1.8.4 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
golang.org/x/net v0.17.0 // indirect
golang.org/x/sys v0.15.0 // indirect
golang.org/x/text v0.13.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect
google.golang.org/grpc v1.59.0 // indirect
google.golang.org/protobuf v1.31.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
6 changes: 6 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0
github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
github.com/agext/levenshtein v1.2.1 h1:QmvMAjj2aEICytGiWzmxoE0x2KZvE0fvmqMOfy2tjT8=
github.com/agext/levenshtein v1.2.1/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558=
github.com/apparentlymart/go-dump v0.0.0-20180507223929-23540a00eaa3 h1:ZSTrOEhiM5J5RFxEaFvMZVEAM1KvT1YzbEOwB2EAGjA=
github.com/apparentlymart/go-dump v0.0.0-20180507223929-23540a00eaa3/go.mod h1:oL81AME2rN47vu18xqj1S1jPIPuN7afo62yKTNn3XMM=
github.com/apparentlymart/go-textseg/v13 v13.0.0 h1:Y+KvPE1NYz0xl601PVImeQfFyEy6iT90AvPUL1NNfNw=
github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo=
github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY=
Expand Down Expand Up @@ -61,6 +63,8 @@ github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcU
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/zclconf/go-cty v1.13.0 h1:It5dfKTTZHe9aeppbNOda3mN7Ag7sg6QkBNm6TkyFa0=
github.com/zclconf/go-cty v1.13.0/go.mod h1:YKQzy/7pZ7iq2jNFzy5go57xdxdWoLLpaEp4u238AE0=
github.com/zclconf/go-cty-debug v0.0.0-20191215020915-b22d67c1ba0b h1:FosyBZYxY34Wul7O/MSKey3txpPYyCqVO5ZyceuQJEI=
github.com/zclconf/go-cty-debug v0.0.0-20191215020915-b22d67c1ba0b/go.mod h1:ZRKQfBXbGkpdV6QMzT3rU1kSTAnfu1dO8dPKjYprgj8=
golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611 h1:qCEDpW1G+vcj3Y7Fy52pEM1AWm3abj8WimGYejI3SC4=
golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI=
golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
Expand All @@ -83,5 +87,7 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
6 changes: 3 additions & 3 deletions pluginInterface/v1/plugin.go → plugininterface/v1/plugin.go
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package plugin
package plugininterface

import (
"github.com/hashicorp/hcl/v2"
Expand Down Expand Up @@ -49,6 +49,6 @@ type Result struct {
// `content` plugins return a markdown string
// `data` plugins return a map[string]any that would be put into the global config
// TODO: hard-code typecast based on the plugin kind while handling the result
result any
diags hcl.Diagnostics
Result any
Diags hcl.Diagnostics
}
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package plugin
package plugininterface

import "github.com/Masterminds/semver/v3"

Expand Down
71 changes: 71 additions & 0 deletions plugins/data/json/helpers_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
package json

import (
"encoding/json"
"fmt"
"io/fs"
"os"
"path/filepath"
"strings"
"testing"
)

type testFS struct {
FS fs.FS
path string
}

func makeTestFS(tb testing.TB) testFS {
tb.Helper()

path, err := filepath.EvalSymlinks(tb.TempDir())
if err != nil {
tb.Fatalf("failed to create testFS: %s", err)
}

path = filepath.ToSlash(path)

tb.Logf("creating testFS at %s", path)
return testFS{
FS: os.DirFS(path),
path: path,
}
}

func (t testFS) Open(name string) (fs.File, error) {
return t.FS.Open(filepath.ToSlash(name))
}

func (t testFS) Path() string {
return t.path
}

func (t testFS) WriteFile(name string, data []byte, perm os.FileMode) error {
name = filepath.ToSlash(name)
if filepath.IsAbs(name) {
if strings.HasPrefix(name, t.path) {
return os.WriteFile(name, data, perm)
}
return fmt.Errorf("path is outside test fs root folder")
}
return os.WriteFile(filepath.ToSlash(filepath.Join(t.path, name)), data, perm)
}

func (t testFS) MkdirAll(path string, perm os.FileMode) error {
path = filepath.ToSlash(path)
if filepath.IsAbs(path) {
if strings.HasPrefix(path, t.path) {
return os.MkdirAll(path, perm)
}
return fmt.Errorf("path is outside test fs root folder")
}
return os.MkdirAll(filepath.ToSlash(filepath.Join(t.path, path)), perm)
}

func testJSON(m any) json.RawMessage {
contents, err := json.Marshal(m)
if err != nil {
panic(err)
}
return contents
}
66 changes: 66 additions & 0 deletions plugins/data/json/plugin.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
package json

import (
"os"

"github.com/Masterminds/semver/v3"
"github.com/blackstork-io/fabric/plugininterface/v1"
"github.com/hashicorp/hcl/v2"
"github.com/hashicorp/hcl/v2/hcldec"
"github.com/zclconf/go-cty/cty"
)

var Version = semver.MustParse("0.1.0")

type Plugin struct{}

func (Plugin) GetPlugins() []plugininterface.Plugin {
return []plugininterface.Plugin{
{
Namespace: "blackstork",
Kind: "data",
Name: "json",
Version: plugininterface.Version(*Version),
ConfigSpec: nil,
InvocationSpec: &hcldec.ObjectSpec{
"glob": &hcldec.AttrSpec{
Name: "glob",
Type: cty.String,
Required: true,
},
},
},
}
}

func (Plugin) Call(args plugininterface.Args) plugininterface.Result {
glob := args.Args.GetAttr("glob").AsString()
wd, err := os.Getwd()
if err != nil {
return plugininterface.Result{
Diags: hcl.Diagnostics{{
Severity: hcl.DiagError,
Summary: "Failed to get current working directory",
Detail: err.Error(),
}},
}
}
filesystem := os.DirFS(wd)
docs, err := readFS(filesystem, glob)
if err != nil {
return plugininterface.Result{
Diags: hcl.Diagnostics{{
Severity: hcl.DiagError,
Summary: "Failed to read json files",
Detail: err.Error(),
}},
}
}
data := make([]any, len(docs))
for i, doc := range docs {
data[i] = doc.Map()
}
return plugininterface.Result{
Result: data,
}
}
103 changes: 103 additions & 0 deletions plugins/data/json/plugin_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
package json

import (
"testing"

"github.com/blackstork-io/fabric/plugininterface/v1"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/zclconf/go-cty/cty"
)

func TestPlugin_GetPlugins(t *testing.T) {
plugin := Plugin{}
plugins := plugin.GetPlugins()
require.Len(t, plugins, 1, "expected 1 plugin")
got := plugins[0]
assert.Equal(t, "json", got.Name)
assert.Equal(t, "data", got.Kind)
assert.Equal(t, "blackstork", got.Namespace)
assert.Equal(t, Version.String(), got.Version.Cast().String())
assert.Nil(t, got.ConfigSpec)
assert.NotNil(t, got.InvocationSpec)
}

func TestPlugin_Call(t *testing.T) {
tt := []struct {
name string
glob string
expected plugininterface.Result
}{
{
name: "empty_list",
glob: "unknown_dir/*.json",
expected: plugininterface.Result{
Result: []any{},
},
},
{
name: "one_file",
glob: "testdata/a.json",
expected: plugininterface.Result{
Result: []any{
map[string]any{
"filename": "testdata/a.json",
"contents": map[string]any{
"property_for": "a.json",
},
},
},
},
},
{
name: "dir",
glob: "testdata/dir/*.json",
expected: plugininterface.Result{
Result: []any{
map[string]any{
"filename": "testdata/dir/b.json",
"contents": []any{
map[string]any{
"id": float64(1),
"property_for": "dir/b.json",
},
map[string]any{
"id": float64(2),
"property_for": "dir/b.json",
},
},
},
map[string]any{
"filename": "testdata/dir/c.json",
"contents": []any{
map[string]any{
"id": float64(3),
"property_for": "dir/c.json",
},
map[string]any{
"id": float64(4),
"property_for": "dir/c.json",
},
},
},
},
},
},
}

for _, tc := range tt {
t.Run(tc.name, func(t *testing.T) {
plugin := Plugin{}
args := plugininterface.Args{
Kind: "json",
Name: "json",
Args: cty.ObjectVal(map[string]cty.Value{
"glob": cty.StringVal(tc.glob),
}),
}
got := plugin.Call(args)
assert.Equal(t, tc.expected, got)
})
}

}
47 changes: 47 additions & 0 deletions plugins/data/json/read.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
package json

import (
"encoding/json"
"io/fs"
)

// JSONDocument represents a JSON document that was read from the filesystem
type JSONDocument struct {
Filename string `json:"filename"`
Contents json.RawMessage `json:"contents"`
}

func (doc JSONDocument) Map() map[string]any {
var result any
_ = json.Unmarshal(doc.Contents, &result)
return map[string]any{
"filename": doc.Filename,
"contents": result,
}
}

// readFS reads all JSON documents from the filesystem that match the given glob pattern
// The pattern is relative to the root of the filesystem
func readFS(filesystem fs.FS, pattern string) ([]JSONDocument, error) {
matchers, err := fs.Glob(filesystem, pattern)
if err != nil {
return nil, err
}
result := []JSONDocument{}
for _, matcher := range matchers {
file, err := filesystem.Open(matcher)
if err != nil {
return nil, err
}
var contents json.RawMessage
err = json.NewDecoder(file).Decode(&contents)
if err != nil {
return nil, err
}
result = append(result, JSONDocument{
Filename: matcher,
Contents: contents,
})
}
return result, nil
}
Loading

0 comments on commit 1851f2d

Please sign in to comment.