From 1851f2dfa87775b63de0757ac16b6b64c89d5e8e Mon Sep 17 00:00:00 2001 From: dobarx <111326505+dobarx@users.noreply.github.com> Date: Tue, 16 Jan 2024 13:35:17 +0200 Subject: [PATCH] `data.json` plugin (#30) --- go.mod | 7 +- go.sum | 6 + .../v1/plugin.go | 6 +- .../v1/semver.go | 2 +- plugins/data/json/helpers_test.go | 71 +++++ plugins/data/json/plugin.go | 66 +++++ plugins/data/json/plugin_test.go | 103 +++++++ plugins/data/json/read.go | 47 +++ plugins/data/json/read_test.go | 273 ++++++++++++++++++ plugins/data/json/testdata/a.json | 3 + plugins/data/json/testdata/dir/b.json | 10 + plugins/data/json/testdata/dir/c.json | 10 + 12 files changed, 598 insertions(+), 6 deletions(-) rename {pluginInterface => plugininterface}/v1/plugin.go (96%) rename {pluginInterface => plugininterface}/v1/semver.go (96%) create mode 100644 plugins/data/json/helpers_test.go create mode 100644 plugins/data/json/plugin.go create mode 100644 plugins/data/json/plugin_test.go create mode 100644 plugins/data/json/read.go create mode 100644 plugins/data/json/read_test.go create mode 100644 plugins/data/json/testdata/a.json create mode 100644 plugins/data/json/testdata/dir/b.json create mode 100644 plugins/data/json/testdata/dir/c.json diff --git a/go.mod b/go.mod index 48319a3a..e1b67bc6 100644 --- a/go.mod +++ b/go.mod @@ -3,20 +3,22 @@ module github.com/blackstork-io/fabric go 1.21.5 require ( + github.com/Masterminds/semver/v3 v3.2.1 github.com/hashicorp/go-hclog v0.14.1 github.com/hashicorp/go-plugin v1.6.0 github.com/hashicorp/hcl/v2 v2.19.1 github.com/itchyny/gojq v0.12.14 + github.com/stretchr/testify v1.8.4 github.com/zclconf/go-cty v1.13.0 golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611 golang.org/x/term v0.15.0 ) require ( - github.com/Masterminds/semver/v3 v3.2.1 // indirect github.com/agext/levenshtein v1.2.1 // indirect github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/fatih/color v1.7.0 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/go-cmp v0.5.9 // indirect @@ -27,11 +29,12 @@ require ( github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77 // indirect github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7 // indirect github.com/oklog/run v1.0.0 // indirect - github.com/stretchr/testify v1.8.4 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect golang.org/x/net v0.17.0 // indirect golang.org/x/sys v0.15.0 // indirect golang.org/x/text v0.13.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect google.golang.org/grpc v1.59.0 // indirect google.golang.org/protobuf v1.31.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 82348a5e..cb6a780d 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,8 @@ github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0 github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/agext/levenshtein v1.2.1 h1:QmvMAjj2aEICytGiWzmxoE0x2KZvE0fvmqMOfy2tjT8= github.com/agext/levenshtein v1.2.1/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= +github.com/apparentlymart/go-dump v0.0.0-20180507223929-23540a00eaa3 h1:ZSTrOEhiM5J5RFxEaFvMZVEAM1KvT1YzbEOwB2EAGjA= +github.com/apparentlymart/go-dump v0.0.0-20180507223929-23540a00eaa3/go.mod h1:oL81AME2rN47vu18xqj1S1jPIPuN7afo62yKTNn3XMM= github.com/apparentlymart/go-textseg/v13 v13.0.0 h1:Y+KvPE1NYz0xl601PVImeQfFyEy6iT90AvPUL1NNfNw= github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo= github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY= @@ -61,6 +63,8 @@ github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcU github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/zclconf/go-cty v1.13.0 h1:It5dfKTTZHe9aeppbNOda3mN7Ag7sg6QkBNm6TkyFa0= github.com/zclconf/go-cty v1.13.0/go.mod h1:YKQzy/7pZ7iq2jNFzy5go57xdxdWoLLpaEp4u238AE0= +github.com/zclconf/go-cty-debug v0.0.0-20191215020915-b22d67c1ba0b h1:FosyBZYxY34Wul7O/MSKey3txpPYyCqVO5ZyceuQJEI= +github.com/zclconf/go-cty-debug v0.0.0-20191215020915-b22d67c1ba0b/go.mod h1:ZRKQfBXbGkpdV6QMzT3rU1kSTAnfu1dO8dPKjYprgj8= golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611 h1:qCEDpW1G+vcj3Y7Fy52pEM1AWm3abj8WimGYejI3SC4= golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= @@ -83,5 +87,7 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pluginInterface/v1/plugin.go b/plugininterface/v1/plugin.go similarity index 96% rename from pluginInterface/v1/plugin.go rename to plugininterface/v1/plugin.go index bed84829..a4cde5fe 100644 --- a/pluginInterface/v1/plugin.go +++ b/plugininterface/v1/plugin.go @@ -1,4 +1,4 @@ -package plugin +package plugininterface import ( "github.com/hashicorp/hcl/v2" @@ -49,6 +49,6 @@ type Result struct { // `content` plugins return a markdown string // `data` plugins return a map[string]any that would be put into the global config // TODO: hard-code typecast based on the plugin kind while handling the result - result any - diags hcl.Diagnostics + Result any + Diags hcl.Diagnostics } diff --git a/pluginInterface/v1/semver.go b/plugininterface/v1/semver.go similarity index 96% rename from pluginInterface/v1/semver.go rename to plugininterface/v1/semver.go index 7109847b..1a347952 100644 --- a/pluginInterface/v1/semver.go +++ b/plugininterface/v1/semver.go @@ -1,4 +1,4 @@ -package plugin +package plugininterface import "github.com/Masterminds/semver/v3" diff --git a/plugins/data/json/helpers_test.go b/plugins/data/json/helpers_test.go new file mode 100644 index 00000000..c7390390 --- /dev/null +++ b/plugins/data/json/helpers_test.go @@ -0,0 +1,71 @@ +package json + +import ( + "encoding/json" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + "testing" +) + +type testFS struct { + FS fs.FS + path string +} + +func makeTestFS(tb testing.TB) testFS { + tb.Helper() + + path, err := filepath.EvalSymlinks(tb.TempDir()) + if err != nil { + tb.Fatalf("failed to create testFS: %s", err) + } + + path = filepath.ToSlash(path) + + tb.Logf("creating testFS at %s", path) + return testFS{ + FS: os.DirFS(path), + path: path, + } +} + +func (t testFS) Open(name string) (fs.File, error) { + return t.FS.Open(filepath.ToSlash(name)) +} + +func (t testFS) Path() string { + return t.path +} + +func (t testFS) WriteFile(name string, data []byte, perm os.FileMode) error { + name = filepath.ToSlash(name) + if filepath.IsAbs(name) { + if strings.HasPrefix(name, t.path) { + return os.WriteFile(name, data, perm) + } + return fmt.Errorf("path is outside test fs root folder") + } + return os.WriteFile(filepath.ToSlash(filepath.Join(t.path, name)), data, perm) +} + +func (t testFS) MkdirAll(path string, perm os.FileMode) error { + path = filepath.ToSlash(path) + if filepath.IsAbs(path) { + if strings.HasPrefix(path, t.path) { + return os.MkdirAll(path, perm) + } + return fmt.Errorf("path is outside test fs root folder") + } + return os.MkdirAll(filepath.ToSlash(filepath.Join(t.path, path)), perm) +} + +func testJSON(m any) json.RawMessage { + contents, err := json.Marshal(m) + if err != nil { + panic(err) + } + return contents +} diff --git a/plugins/data/json/plugin.go b/plugins/data/json/plugin.go new file mode 100644 index 00000000..54ac36ff --- /dev/null +++ b/plugins/data/json/plugin.go @@ -0,0 +1,66 @@ +package json + +import ( + "os" + + "github.com/Masterminds/semver/v3" + "github.com/blackstork-io/fabric/plugininterface/v1" + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/hcldec" + "github.com/zclconf/go-cty/cty" +) + +var Version = semver.MustParse("0.1.0") + +type Plugin struct{} + +func (Plugin) GetPlugins() []plugininterface.Plugin { + return []plugininterface.Plugin{ + { + Namespace: "blackstork", + Kind: "data", + Name: "json", + Version: plugininterface.Version(*Version), + ConfigSpec: nil, + InvocationSpec: &hcldec.ObjectSpec{ + "glob": &hcldec.AttrSpec{ + Name: "glob", + Type: cty.String, + Required: true, + }, + }, + }, + } +} + +func (Plugin) Call(args plugininterface.Args) plugininterface.Result { + glob := args.Args.GetAttr("glob").AsString() + wd, err := os.Getwd() + if err != nil { + return plugininterface.Result{ + Diags: hcl.Diagnostics{{ + Severity: hcl.DiagError, + Summary: "Failed to get current working directory", + Detail: err.Error(), + }}, + } + } + filesystem := os.DirFS(wd) + docs, err := readFS(filesystem, glob) + if err != nil { + return plugininterface.Result{ + Diags: hcl.Diagnostics{{ + Severity: hcl.DiagError, + Summary: "Failed to read json files", + Detail: err.Error(), + }}, + } + } + data := make([]any, len(docs)) + for i, doc := range docs { + data[i] = doc.Map() + } + return plugininterface.Result{ + Result: data, + } +} diff --git a/plugins/data/json/plugin_test.go b/plugins/data/json/plugin_test.go new file mode 100644 index 00000000..e39336c3 --- /dev/null +++ b/plugins/data/json/plugin_test.go @@ -0,0 +1,103 @@ +package json + +import ( + "testing" + + "github.com/blackstork-io/fabric/plugininterface/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/zclconf/go-cty/cty" +) + +func TestPlugin_GetPlugins(t *testing.T) { + plugin := Plugin{} + plugins := plugin.GetPlugins() + require.Len(t, plugins, 1, "expected 1 plugin") + got := plugins[0] + assert.Equal(t, "json", got.Name) + assert.Equal(t, "data", got.Kind) + assert.Equal(t, "blackstork", got.Namespace) + assert.Equal(t, Version.String(), got.Version.Cast().String()) + assert.Nil(t, got.ConfigSpec) + assert.NotNil(t, got.InvocationSpec) +} + +func TestPlugin_Call(t *testing.T) { + tt := []struct { + name string + glob string + expected plugininterface.Result + }{ + { + name: "empty_list", + glob: "unknown_dir/*.json", + expected: plugininterface.Result{ + Result: []any{}, + }, + }, + { + name: "one_file", + glob: "testdata/a.json", + expected: plugininterface.Result{ + Result: []any{ + map[string]any{ + "filename": "testdata/a.json", + "contents": map[string]any{ + "property_for": "a.json", + }, + }, + }, + }, + }, + { + name: "dir", + glob: "testdata/dir/*.json", + expected: plugininterface.Result{ + Result: []any{ + map[string]any{ + "filename": "testdata/dir/b.json", + "contents": []any{ + map[string]any{ + "id": float64(1), + "property_for": "dir/b.json", + }, + map[string]any{ + "id": float64(2), + "property_for": "dir/b.json", + }, + }, + }, + map[string]any{ + "filename": "testdata/dir/c.json", + "contents": []any{ + map[string]any{ + "id": float64(3), + "property_for": "dir/c.json", + }, + map[string]any{ + "id": float64(4), + "property_for": "dir/c.json", + }, + }, + }, + }, + }, + }, + } + + for _, tc := range tt { + t.Run(tc.name, func(t *testing.T) { + plugin := Plugin{} + args := plugininterface.Args{ + Kind: "json", + Name: "json", + Args: cty.ObjectVal(map[string]cty.Value{ + "glob": cty.StringVal(tc.glob), + }), + } + got := plugin.Call(args) + assert.Equal(t, tc.expected, got) + }) + } + +} diff --git a/plugins/data/json/read.go b/plugins/data/json/read.go new file mode 100644 index 00000000..abe66e19 --- /dev/null +++ b/plugins/data/json/read.go @@ -0,0 +1,47 @@ +package json + +import ( + "encoding/json" + "io/fs" +) + +// JSONDocument represents a JSON document that was read from the filesystem +type JSONDocument struct { + Filename string `json:"filename"` + Contents json.RawMessage `json:"contents"` +} + +func (doc JSONDocument) Map() map[string]any { + var result any + _ = json.Unmarshal(doc.Contents, &result) + return map[string]any{ + "filename": doc.Filename, + "contents": result, + } +} + +// readFS reads all JSON documents from the filesystem that match the given glob pattern +// The pattern is relative to the root of the filesystem +func readFS(filesystem fs.FS, pattern string) ([]JSONDocument, error) { + matchers, err := fs.Glob(filesystem, pattern) + if err != nil { + return nil, err + } + result := []JSONDocument{} + for _, matcher := range matchers { + file, err := filesystem.Open(matcher) + if err != nil { + return nil, err + } + var contents json.RawMessage + err = json.NewDecoder(file).Decode(&contents) + if err != nil { + return nil, err + } + result = append(result, JSONDocument{ + Filename: matcher, + Contents: contents, + }) + } + return result, nil +} diff --git a/plugins/data/json/read_test.go b/plugins/data/json/read_test.go new file mode 100644 index 00000000..408a5268 --- /dev/null +++ b/plugins/data/json/read_test.go @@ -0,0 +1,273 @@ +package json + +import ( + "encoding/json" + "path/filepath" + "reflect" + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_readFS_Structures(t *testing.T) { + // test cases + tt := []struct { + name string + files []string + dirs []string + pattern string + expected []JSONDocument + }{ + { + name: "empty", + files: []string{}, + dirs: []string{}, + pattern: "*.json", + expected: []JSONDocument{}, + }, + { + name: "one_file", + files: []string{"a.json"}, + dirs: []string{}, + pattern: "*.json", + expected: []JSONDocument{ + { + Filename: "a.json", + Contents: testJSON(map[string]any{ + "property_for": "a.json", + }), + }, + }, + }, + { + name: "two_files", + files: []string{"a.json", "b.json"}, + dirs: []string{}, + pattern: "*.json", + expected: []JSONDocument{ + { + Filename: "a.json", + Contents: testJSON(map[string]any{ + "property_for": "a.json", + }), + }, + { + Filename: "b.json", + Contents: testJSON(map[string]any{ + "property_for": "b.json", + }), + }, + }, + }, + { + name: "one_file_in_one_dir", + files: []string{"dir/a.json"}, + dirs: []string{"dir"}, + pattern: "dir/*.json", + expected: []JSONDocument{ + { + Filename: "dir/a.json", + Contents: testJSON(map[string]any{ + "property_for": "dir/a.json", + }), + }, + }, + }, + { + name: "one_file_in_two_dirs", + files: []string{"dir1/a.json", "dir2/a.json"}, + dirs: []string{"dir1", "dir2"}, + pattern: "*/a.json", + expected: []JSONDocument{ + { + Filename: "dir1/a.json", + Contents: testJSON(map[string]any{ + "property_for": "dir1/a.json", + }), + }, + { + Filename: "dir2/a.json", + Contents: testJSON(map[string]any{ + "property_for": "dir2/a.json", + }), + }, + }, + }, + { + name: "two_files_in_one_dir", + files: []string{"dir/a.json", "dir/b.json"}, + dirs: []string{"dir"}, + pattern: "dir/*.json", + expected: []JSONDocument{ + { + Filename: "dir/a.json", + Contents: testJSON(map[string]any{ + "property_for": "dir/a.json", + }), + }, + { + Filename: "dir/b.json", + Contents: testJSON(map[string]any{ + "property_for": "dir/b.json", + }), + }, + }, + }, + } + + for _, tc := range tt { + t.Run(tc.name, func(t *testing.T) { + tmpfs := makeTestFS(t) + for _, file := range tc.files { + data := testJSON(map[string]any{ + "property_for": file, + }) + assert.NoError(t, tmpfs.MkdirAll(filepath.Dir(file), 0o764)) + assert.NoError(t, tmpfs.WriteFile(file, []byte(data), 0o654)) + } + for _, dir := range tc.dirs { + assert.NoError(t, tmpfs.MkdirAll(dir, 0o764)) + } + result, err := readFS(tmpfs, tc.pattern) + assert.NoError(t, err) + assert.Equal(t, tc.expected, result) + }) + } +} + +func Test_readFS_Errors(t *testing.T) { + tt := []struct { + name string + files []string + dirs []string + pattern string + preparefn func(filename string) json.RawMessage + }{ + { + name: "pattern_error", + files: []string{}, + dirs: []string{}, + pattern: "[", + }, + { + name: "file_error", + files: []string{"a.json"}, + dirs: []string{}, + pattern: "*.json", + preparefn: func(filename string) json.RawMessage { + return []byte("not json") + }, + }, + } + + for _, tc := range tt { + t.Run(tc.name, func(t *testing.T) { + tmpfs := makeTestFS(t) + for _, file := range tc.files { + data := tc.preparefn(file) + assert.NoError(t, tmpfs.MkdirAll(filepath.Dir(file), 0o764)) + assert.NoError(t, tmpfs.WriteFile(file, []byte(data), 0o654)) + } + for _, dir := range tc.dirs { + assert.NoError(t, tmpfs.MkdirAll(dir, 0o764)) + } + _, err := readFS(tmpfs, tc.pattern) + assert.Error(t, err) + }) + } +} + +func TestJSONDocument_Map(t *testing.T) { + type fields struct { + Filename string + Contents json.RawMessage + } + tt := []struct { + name string + fields fields + want map[string]any + }{ + { + name: "simple", + fields: fields{ + Filename: "a.json", + Contents: testJSON(map[string]any{ + "property_for": "a.json", + }), + }, + want: map[string]any{ + "filename": "a.json", + "contents": map[string]any{ + "property_for": "a.json", + }, + }, + }, + { + name: "complex", + fields: fields{ + Filename: "a.json", + Contents: testJSON(map[string]any{ + "property_for": "a.json", + "nested": map[string]any{ + "property_for": "a.json", + "nested": map[string]any{ + "property_for": "a.json", + }, + }, + }), + }, + want: map[string]any{ + "filename": "a.json", + "contents": map[string]any{ + "property_for": "a.json", + "nested": map[string]any{ + "property_for": "a.json", + "nested": map[string]any{ + "property_for": "a.json", + }, + }, + }, + }, + }, + { + name: "array", + fields: fields{ + Filename: "a.json", + Contents: testJSON([]any{ + map[string]any{ + "id": float64(0), + "property_for": "a.json", + }, + map[string]any{ + "id": float64(1), + "property_for": "a.json", + }, + }), + }, + want: map[string]any{ + "filename": "a.json", + "contents": []any{ + map[string]any{ + "id": float64(0), + "property_for": "a.json", + }, + map[string]any{ + "id": float64(1), + "property_for": "a.json", + }, + }, + }, + }, + } + for _, tt := range tt { + t.Run(tt.name, func(t *testing.T) { + doc := JSONDocument{ + Filename: tt.fields.Filename, + Contents: tt.fields.Contents, + } + if got := doc.Map(); !reflect.DeepEqual(got, tt.want) { + t.Errorf("JSONDocument.Map() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/plugins/data/json/testdata/a.json b/plugins/data/json/testdata/a.json new file mode 100644 index 00000000..371fe60e --- /dev/null +++ b/plugins/data/json/testdata/a.json @@ -0,0 +1,3 @@ +{ + "property_for": "a.json" +} \ No newline at end of file diff --git a/plugins/data/json/testdata/dir/b.json b/plugins/data/json/testdata/dir/b.json new file mode 100644 index 00000000..d997d020 --- /dev/null +++ b/plugins/data/json/testdata/dir/b.json @@ -0,0 +1,10 @@ +[ + { + "id": 1, + "property_for": "dir/b.json" + }, + { + "id": 2, + "property_for": "dir/b.json" + } +] \ No newline at end of file diff --git a/plugins/data/json/testdata/dir/c.json b/plugins/data/json/testdata/dir/c.json new file mode 100644 index 00000000..9b6e7e23 --- /dev/null +++ b/plugins/data/json/testdata/dir/c.json @@ -0,0 +1,10 @@ +[ + { + "id": 3, + "property_for": "dir/c.json" + }, + { + "id": 4, + "property_for": "dir/c.json" + } +] \ No newline at end of file