From 9f6f26bfa286209f25eef7661e964ce57008fca0 Mon Sep 17 00:00:00 2001 From: dobarx <111326505+dobarx@users.noreply.github.com> Date: Fri, 2 Feb 2024 11:45:47 +0200 Subject: [PATCH] Revert "Parser v2 (#14)" deleted plugins (#65) --- plugins/content/table/impl.go | 61 --- plugins/content/table/plugin.go | 166 +++++++ plugins/content/table/plugin_test.go | 331 ++++++++++++++ plugins/content/text/impl.go | 51 --- plugins/content/text/plugin.go | 21 +- plugins/content/text/plugin_test.go | 320 +++++++++++++ plugins/data/csv/plugin.go | 90 ++++ plugins/data/csv/plugin_test.go | 194 ++++++++ plugins/data/csv/read.go | 53 +++ plugins/data/csv/testdata/comma.csv | 4 + plugins/data/csv/testdata/empty.csv | 0 plugins/data/csv/testdata/invalid.csv | 4 + plugins/data/csv/testdata/semicolon.csv | 4 + plugins/data/elasticsearch/plugin.go | 251 +++++++++++ .../elasticsearch/plugin_integration_test.go | 426 ++++++++++++++++++ plugins/data/elasticsearch/plugin_test.go | 21 + plugins/data/elasticsearch/testdata/data.json | 23 + plugins/data/json/helpers_test.go | 71 +++ plugins/data/json/plugin.go | 66 +++ plugins/data/json/plugin_test.go | 103 +++++ plugins/data/json/read.go | 47 ++ plugins/data/json/read_test.go | 273 +++++++++++ plugins/data/json/testdata/a.json | 3 + plugins/data/json/testdata/dir/b.json | 10 + plugins/data/json/testdata/dir/c.json | 10 + 25 files changed, 2479 insertions(+), 124 deletions(-) delete mode 100644 plugins/content/table/impl.go create mode 100644 plugins/content/table/plugin.go create mode 100644 plugins/content/table/plugin_test.go delete mode 100644 plugins/content/text/impl.go create mode 100644 plugins/content/text/plugin_test.go create mode 100644 plugins/data/csv/plugin.go create mode 100644 plugins/data/csv/plugin_test.go create mode 100644 plugins/data/csv/read.go create mode 100644 plugins/data/csv/testdata/comma.csv create mode 100644 plugins/data/csv/testdata/empty.csv create mode 100644 plugins/data/csv/testdata/invalid.csv create mode 100644 plugins/data/csv/testdata/semicolon.csv create mode 100644 plugins/data/elasticsearch/plugin.go create mode 100644 plugins/data/elasticsearch/plugin_integration_test.go create mode 100644 plugins/data/elasticsearch/plugin_test.go create mode 100644 plugins/data/elasticsearch/testdata/data.json create mode 100644 plugins/data/json/helpers_test.go create mode 100644 plugins/data/json/plugin.go create mode 100644 plugins/data/json/plugin_test.go create mode 100644 plugins/data/json/read.go create mode 100644 plugins/data/json/read_test.go create mode 100644 plugins/data/json/testdata/a.json create mode 100644 plugins/data/json/testdata/dir/b.json create mode 100644 plugins/data/json/testdata/dir/c.json diff --git a/plugins/content/table/impl.go b/plugins/content/table/impl.go deleted file mode 100644 index 348aaa76..00000000 --- a/plugins/content/table/impl.go +++ /dev/null @@ -1,61 +0,0 @@ -package table - -import ( - "bytes" - "fmt" - "text/template" - - "github.com/blackstork-io/fabric/pkg/jsontools" - "github.com/blackstork-io/fabric/plugins/content" -) - -// Actual implementation of the plugin - -type Impl struct{} - -var _ content.Plugin = (*Impl)(nil) - -const PluginName = "content.table" - -func (Impl) Execute(attrsRaw, dictRaw any) (resp string, err error) { - var attrs struct { - Text string `json:"text"` - Columns []string `json:"columns"` - } - var dict any - err = jsontools.UnmarshalBytes(attrsRaw, &attrs) - if err != nil { - return - } - err = jsontools.UnmarshalBytes(dictRaw, &dict) - if err != nil { - return - } - - tmpl, err := template.New(PluginName).Parse(attrs.Text) - if err != nil { - err = fmt.Errorf("failed to parse the template: %w; template: `%s`", err, attrs.Text) - return - } - - var buf bytes.Buffer - buf.WriteString(PluginName) - buf.WriteByte(':') - - err = tmpl.Execute(&buf, dict) - if err != nil { - err = fmt.Errorf("failed to execute the template: %w; template: `%s`; dict: `%s`", err, attrs.Text, jsontools.Dump(dict)) - return - } - buf.WriteByte('.') - - if len(attrs.Columns) == 0 { - return buf.String(), nil - } - buf.WriteString(attrs.Columns[0]) - for _, col := range attrs.Columns[1:] { - buf.WriteByte(',') - buf.WriteString(col) - } - return buf.String(), nil -} diff --git a/plugins/content/table/plugin.go b/plugins/content/table/plugin.go new file mode 100644 index 00000000..9eef2bc1 --- /dev/null +++ b/plugins/content/table/plugin.go @@ -0,0 +1,166 @@ +package table + +import ( + "bytes" + "errors" + "fmt" + "strings" + "text/template" + + "github.com/Masterminds/semver/v3" + "github.com/blackstork-io/fabric/plugininterface/v1" + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/hcldec" + "github.com/zclconf/go-cty/cty" +) + +var Version = semver.MustParse("0.1.0") + +type Plugin struct{} + +type cellTmpl = *template.Template + +func (Plugin) GetPlugins() []plugininterface.Plugin { + return []plugininterface.Plugin{ + { + Namespace: "blackstork", + Kind: "content", + Name: "table", + Version: plugininterface.Version(*Version), + ConfigSpec: nil, + InvocationSpec: &hcldec.ObjectSpec{ + "columns": &hcldec.AttrSpec{ + Name: "columns", + Type: cty.List(cty.Object(map[string]cty.Type{ + "header": cty.String, + "value": cty.String, + })), + Required: true, + }, + }, + }, + } +} + +func (p Plugin) Call(args plugininterface.Args) plugininterface.Result { + headers, values, err := p.parseArgs(args) + if err != nil { + return plugininterface.Result{ + Diags: hcl.Diagnostics{{ + Severity: hcl.DiagError, + Summary: "Failed to parse arguments", + Detail: err.Error(), + }}, + } + } + result, err := p.render(headers, values, args.Context) + if err != nil { + return plugininterface.Result{ + Diags: hcl.Diagnostics{{ + Severity: hcl.DiagError, + Summary: "Failed to render table", + Detail: err.Error(), + }}, + } + } + return plugininterface.Result{ + Result: result, + } +} + +func (p Plugin) parseArgs(args plugininterface.Args) (headers []cellTmpl, values []cellTmpl, err error) { + arr := args.Args.GetAttr("columns") + if arr.IsNull() { + return nil, nil, errors.New("columns is required") + } + if len(arr.AsValueSlice()) == 0 { + return nil, nil, errors.New("columns must not be empty") + } + for _, val := range arr.AsValueSlice() { + obj := val.AsValueMap() + var ( + header cty.Value + value cty.Value + ok = false + ) + if header, ok = obj["header"]; !ok || header.IsNull() { + return nil, nil, errors.New("missing header in table cell") + } + if value, ok = obj["value"]; !ok || value.IsNull() { + return nil, nil, errors.New("missing value in table cell") + } + + headerTmpl, err := template.New("header").Parse(header.AsString()) + if err != nil { + return nil, nil, fmt.Errorf("failed to parse header template: %w", err) + } + valueTmpl, err := template.New("value").Parse(value.AsString()) + if err != nil { + return nil, nil, fmt.Errorf("failed to parse value template: %w", err) + } + headers = append(headers, headerTmpl) + values = append(values, valueTmpl) + } + return +} + +func (p Plugin) render(headers, values []cellTmpl, datactx map[string]any) (string, error) { + hstr := make([]string, len(headers)) + vstr := [][]string{} + for i, header := range headers { + var buf bytes.Buffer + err := header.Execute(&buf, datactx) + if err != nil { + return "", fmt.Errorf("failed to render header: %w", err) + } + hstr[i] = strings.TrimSpace( + strings.ReplaceAll(buf.String(), "\n", " "), + ) + } + if datactx == nil { + return "", errors.New("data context is nil") + } + if queryResult, ok := datactx["query_result"]; ok && queryResult != nil { + queryResult, ok := queryResult.([]any) + if !ok { + return "", errors.New("query_result is not an array") + } + for _, row := range queryResult { + rowstr := make([]string, len(values)) + for i, value := range values { + var buf bytes.Buffer + err := value.Execute(&buf, row) + if err != nil { + return "", fmt.Errorf("failed to render value: %w", err) + } + rowstr[i] = strings.TrimSpace( + strings.ReplaceAll(buf.String(), "\n", " "), + ) + } + vstr = append(vstr, rowstr) + } + } + var buf bytes.Buffer + buf.WriteByte('|') + for _, header := range hstr { + buf.WriteString(header) + buf.WriteByte('|') + } + buf.WriteByte('\n') + buf.WriteByte('|') + for range hstr { + buf.WriteString("-") + buf.WriteByte('|') + } + buf.WriteByte('\n') + for _, row := range vstr { + buf.WriteByte('|') + for _, value := range row { + buf.WriteString(value) + buf.WriteByte('|') + } + buf.WriteByte('\n') + } + return buf.String(), nil + +} diff --git a/plugins/content/table/plugin_test.go b/plugins/content/table/plugin_test.go new file mode 100644 index 00000000..7b830ad2 --- /dev/null +++ b/plugins/content/table/plugin_test.go @@ -0,0 +1,331 @@ +package table + +import ( + "testing" + + "github.com/blackstork-io/fabric/plugininterface/v1" + "github.com/hashicorp/hcl/v2" + "github.com/stretchr/testify/suite" + "github.com/zclconf/go-cty/cty" +) + +type PluginTestSuite struct { + suite.Suite + plugin plugininterface.PluginRPC +} + +func TestPluginSuite(t *testing.T) { + suite.Run(t, &PluginTestSuite{}) +} + +func (s *PluginTestSuite) SetupSuite() { + s.plugin = Plugin{} +} + +func (s *PluginTestSuite) TestGetPlugins() { + plugins := s.plugin.GetPlugins() + s.Require().Len(plugins, 1, "expected 1 plugin") + got := plugins[0] + s.Equal("table", got.Name) + s.Equal("content", got.Kind) + s.Equal("blackstork", got.Namespace) + s.Equal(Version.String(), got.Version.Cast().String()) + s.Nil(got.ConfigSpec) + s.NotNil(got.InvocationSpec) +} + +func (s *PluginTestSuite) TestCallNilQueryResult() { + args := plugininterface.Args{ + Kind: "content", + Name: "table", + Args: cty.ObjectVal(map[string]cty.Value{ + "columns": cty.ListVal([]cty.Value{ + cty.ObjectVal(map[string]cty.Value{ + "header": cty.StringVal("{{.col_prefix}} Name"), + "value": cty.StringVal("{{.name}}"), + }), + cty.ObjectVal(map[string]cty.Value{ + "header": cty.StringVal("{{.col_prefix}} Age"), + "value": cty.StringVal("{{.age}}"), + }), + }), + }), + Context: map[string]any{ + "col_prefix": "User", + "query_result": nil, + }, + } + result := s.plugin.Call(args) + expected := plugininterface.Result{ + Result: "|User Name|User Age|\n|-|-|\n", + } + s.Equal(expected, result) +} + +func (s *PluginTestSuite) TestCallEmptyQueryResult() { + args := plugininterface.Args{ + Kind: "content", + Name: "table", + Args: cty.ObjectVal(map[string]cty.Value{ + "columns": cty.ListVal([]cty.Value{ + cty.ObjectVal(map[string]cty.Value{ + "header": cty.StringVal("{{.col_prefix}} Name"), + "value": cty.StringVal("{{.name}}"), + }), + cty.ObjectVal(map[string]cty.Value{ + "header": cty.StringVal("{{.col_prefix}} Age"), + "value": cty.StringVal("{{.age}}"), + }), + }), + }), + Context: map[string]any{ + "col_prefix": "User", + "query_result": []any{}, + }, + } + result := s.plugin.Call(args) + expected := plugininterface.Result{ + Result: "|User Name|User Age|\n|-|-|\n", + } + s.Equal(expected, result) +} +func (s *PluginTestSuite) TestCallBasic() { + args := plugininterface.Args{ + Kind: "content", + Name: "table", + Args: cty.ObjectVal(map[string]cty.Value{ + "columns": cty.ListVal([]cty.Value{ + cty.ObjectVal(map[string]cty.Value{ + "header": cty.StringVal("{{.col_prefix}} Name"), + "value": cty.StringVal("{{.name}}"), + }), + cty.ObjectVal(map[string]cty.Value{ + "header": cty.StringVal("{{.col_prefix}} Age"), + "value": cty.StringVal("{{.age}}"), + }), + }), + }), + Context: map[string]any{ + "col_prefix": "User", + "query_result": []any{ + map[string]any{ + "name": "John", + "age": 42, + }, + map[string]any{ + "name": "Jane", + "age": 43, + }, + }, + }, + } + result := s.plugin.Call(args) + expected := plugininterface.Result{ + Result: "|User Name|User Age|\n|-|-|\n|John|42|\n|Jane|43|\n", + } + s.Equal(expected, result) +} + +func (s *PluginTestSuite) TestCallMissingHeader() { + args := plugininterface.Args{ + Kind: "content", + Name: "table", + Args: cty.ObjectVal(map[string]cty.Value{ + "columns": cty.ListVal([]cty.Value{ + cty.ObjectVal(map[string]cty.Value{ + "value": cty.StringVal("{{.name}}"), + }), + cty.ObjectVal(map[string]cty.Value{ + "value": cty.StringVal("{{.age}}"), + }), + }), + }), + Context: nil, + } + result := s.plugin.Call(args) + expected := plugininterface.Result{ + Diags: hcl.Diagnostics{{ + Severity: hcl.DiagError, + Summary: "Failed to parse arguments", + Detail: "missing header in table cell", + }}, + } + s.Equal(expected, result) +} +func (s *PluginTestSuite) TestCallMissingValue() { + args := plugininterface.Args{ + Kind: "content", + Name: "table", + Args: cty.ObjectVal(map[string]cty.Value{ + "columns": cty.ListVal([]cty.Value{ + cty.ObjectVal(map[string]cty.Value{ + "header": cty.StringVal("{{.col_prefix}} Name"), + }), + cty.ObjectVal(map[string]cty.Value{ + "header": cty.StringVal("{{.col_prefix}} Age"), + }), + }), + }), + Context: nil, + } + result := s.plugin.Call(args) + expected := plugininterface.Result{ + Diags: hcl.Diagnostics{{ + Severity: hcl.DiagError, + Summary: "Failed to parse arguments", + Detail: "missing value in table cell", + }}, + } + s.Equal(expected, result) +} + +func (s *PluginTestSuite) TestCallNilHeader() { + args := plugininterface.Args{ + Kind: "content", + Name: "table", + Args: cty.ObjectVal(map[string]cty.Value{ + "columns": cty.ListVal([]cty.Value{ + cty.ObjectVal(map[string]cty.Value{ + "header": cty.NullVal(cty.String), + "value": cty.StringVal("{{.name}}"), + }), + cty.ObjectVal(map[string]cty.Value{ + "header": cty.NullVal(cty.String), + "value": cty.StringVal("{{.age}}"), + }), + }), + }), + Context: nil, + } + result := s.plugin.Call(args) + expected := plugininterface.Result{ + Diags: hcl.Diagnostics{{ + Severity: hcl.DiagError, + Summary: "Failed to parse arguments", + Detail: "missing header in table cell", + }}, + } + s.Equal(expected, result) +} +func (s *PluginTestSuite) TestCallNilValue() { + args := plugininterface.Args{ + Kind: "content", + Name: "table", + Args: cty.ObjectVal(map[string]cty.Value{ + "columns": cty.ListVal([]cty.Value{ + cty.ObjectVal(map[string]cty.Value{ + "header": cty.StringVal("{{.col_prefix}} Name"), + "value": cty.NullVal(cty.String), + }), + cty.ObjectVal(map[string]cty.Value{ + "header": cty.StringVal("{{.col_prefix}} Age"), + "value": cty.NullVal(cty.String), + }), + }), + }), + Context: nil, + } + result := s.plugin.Call(args) + expected := plugininterface.Result{ + Diags: hcl.Diagnostics{{ + Severity: hcl.DiagError, + Summary: "Failed to parse arguments", + Detail: "missing value in table cell", + }}, + } + s.Equal(expected, result) +} + +func (s *PluginTestSuite) TestCallNilColumns() { + args := plugininterface.Args{ + Kind: "content", + Name: "table", + Args: cty.ObjectVal(map[string]cty.Value{ + "columns": cty.NullVal(cty.List(cty.Object(map[string]cty.Type{}))), + }), + } + result := s.plugin.Call(args) + expected := plugininterface.Result{ + Diags: hcl.Diagnostics{{ + Severity: hcl.DiagError, + Summary: "Failed to parse arguments", + Detail: "columns is required", + }}, + } + s.Equal(expected, result) +} + +func (s *PluginTestSuite) TestCallEmptyColumns() { + args := plugininterface.Args{ + Kind: "content", + Name: "table", + Args: cty.ObjectVal(map[string]cty.Value{ + "columns": cty.ListValEmpty(cty.Object(map[string]cty.Type{})), + }), + } + result := s.plugin.Call(args) + expected := plugininterface.Result{ + Diags: hcl.Diagnostics{{ + Severity: hcl.DiagError, + Summary: "Failed to parse arguments", + Detail: "columns must not be empty", + }}, + } + s.Equal(expected, result) +} + +func (s *PluginTestSuite) TestCallInvalidHeaderTemplate() { + args := plugininterface.Args{ + Kind: "content", + Name: "table", + Args: cty.ObjectVal(map[string]cty.Value{ + "columns": cty.ListVal([]cty.Value{ + cty.ObjectVal(map[string]cty.Value{ + "header": cty.StringVal("{{.col_prefix} Name"), + "value": cty.StringVal("{{.name}}"), + }), + cty.ObjectVal(map[string]cty.Value{ + "header": cty.StringVal("{{.col_prefix}} Age"), + "value": cty.StringVal("{{.age}}"), + }), + }), + }), + } + result := s.plugin.Call(args) + expected := plugininterface.Result{ + Diags: hcl.Diagnostics{{ + Severity: hcl.DiagError, + Summary: "Failed to parse arguments", + Detail: "failed to parse header template: template: header:1: bad character U+007D '}'", + }}, + } + s.Equal(expected, result) +} + +func (s *PluginTestSuite) TestCallInvalidValueTemplate() { + args := plugininterface.Args{ + Kind: "content", + Name: "table", + Args: cty.ObjectVal(map[string]cty.Value{ + "columns": cty.ListVal([]cty.Value{ + cty.ObjectVal(map[string]cty.Value{ + "header": cty.StringVal("{{.col_prefix}} Name"), + "value": cty.StringVal("{{.name}"), + }), + cty.ObjectVal(map[string]cty.Value{ + "header": cty.StringVal("{{.col_prefix}} Age"), + "value": cty.StringVal("{{.age}}"), + }), + }), + }), + } + result := s.plugin.Call(args) + expected := plugininterface.Result{ + Diags: hcl.Diagnostics{{ + Severity: hcl.DiagError, + Summary: "Failed to parse arguments", + Detail: "failed to parse value template: template: value:1: bad character U+007D '}'", + }}, + } + s.Equal(expected, result) +} diff --git a/plugins/content/text/impl.go b/plugins/content/text/impl.go deleted file mode 100644 index cbe49809..00000000 --- a/plugins/content/text/impl.go +++ /dev/null @@ -1,51 +0,0 @@ -package text - -import ( - "bytes" - "fmt" - "text/template" - - "github.com/blackstork-io/fabric/pkg/jsontools" - "github.com/blackstork-io/fabric/plugins/content" -) - -// Actual implementation of the plugin - -type Impl struct{} - -var _ content.Plugin = (*Impl)(nil) - -const PluginName = "content.text" - -func (Impl) Execute(attrsRaw, dictRaw any) (resp string, err error) { - var attrs struct { - Text string `json:"text"` - } - var dict any - err = jsontools.UnmarshalBytes(attrsRaw, &attrs) - if err != nil { - return - } - err = jsontools.UnmarshalBytes(dictRaw, &dict) - if err != nil { - return - } - - tmpl, err := template.New(PluginName).Parse(attrs.Text) - if err != nil { - err = fmt.Errorf("failed to parse the template: %w; template: `%s`", err, attrs.Text) - return - } - - var buf bytes.Buffer - buf.WriteString(PluginName) - buf.WriteByte(':') - - err = tmpl.Execute(&buf, dict) - if err != nil { - err = fmt.Errorf("failed to execute the template: %w; template: `%s`; dict: `%s`", err, attrs.Text, jsontools.Dump(dict)) - return - } - resp = buf.String() - return -} diff --git a/plugins/content/text/plugin.go b/plugins/content/text/plugin.go index dbabc63e..decccb70 100644 --- a/plugins/content/text/plugin.go +++ b/plugins/content/text/plugin.go @@ -9,17 +9,14 @@ import ( "text/template" "github.com/Masterminds/semver/v3" + "github.com/blackstork-io/fabric/plugininterface/v1" "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/hcldec" "github.com/zclconf/go-cty/cty" - - plugin "github.com/blackstork-io/fabric/pluginInterface/v1" ) -var ( - Version = semver.MustParse("0.1.0") - allowedFormats = []string{"text", "title", "code", "blockquote"} -) +var Version = semver.MustParse("0.1.0") +var allowedFormats = []string{"text", "title", "code", "blockquote"} const ( minAbsoluteTitleSize = int64(1) @@ -31,13 +28,13 @@ const ( type Plugin struct{} -func (Plugin) GetPlugins() []plugin.Plugin { - return []plugin.Plugin{ +func (Plugin) GetPlugins() []plugininterface.Plugin { + return []plugininterface.Plugin{ { Namespace: "blackstork", Kind: "content", Name: "text", - Version: plugin.Version(*Version), + Version: plugininterface.Version(*Version), ConfigSpec: nil, InvocationSpec: &hcldec.ObjectSpec{ "text": &hcldec.AttrSpec{ @@ -142,10 +139,10 @@ func (p Plugin) renderBlockquote(text string, datactx map[string]any) (string, e return "> " + strings.ReplaceAll(text, "\n", "\n> "), nil } -func (p Plugin) Call(args plugin.Args) plugin.Result { +func (p Plugin) Call(args plugininterface.Args) plugininterface.Result { result, err := p.render(args.Args, args.Context) if err != nil { - return plugin.Result{ + return plugininterface.Result{ Diags: hcl.Diagnostics{{ Severity: hcl.DiagError, Summary: "Failed to render text", @@ -153,7 +150,7 @@ func (p Plugin) Call(args plugin.Args) plugin.Result { }}, } } - return plugin.Result{ + return plugininterface.Result{ Result: result, } } diff --git a/plugins/content/text/plugin_test.go b/plugins/content/text/plugin_test.go new file mode 100644 index 00000000..51c4f2a0 --- /dev/null +++ b/plugins/content/text/plugin_test.go @@ -0,0 +1,320 @@ +package text + +import ( + "testing" + + "github.com/blackstork-io/fabric/plugininterface/v1" + "github.com/hashicorp/hcl/v2" + "github.com/stretchr/testify/suite" + "github.com/zclconf/go-cty/cty" +) + +type PluginTestSuite struct { + suite.Suite + plugin plugininterface.PluginRPC +} + +func TestPluginSuite(t *testing.T) { + suite.Run(t, &PluginTestSuite{}) +} + +func (s *PluginTestSuite) SetupSuite() { + s.plugin = Plugin{} +} + +func (s *PluginTestSuite) TestGetPlugins() { + plugins := s.plugin.GetPlugins() + s.Require().Len(plugins, 1, "expected 1 plugin") + got := plugins[0] + s.Equal("text", got.Name) + s.Equal("content", got.Kind) + s.Equal("blackstork", got.Namespace) + s.Equal(Version.String(), got.Version.Cast().String()) + s.Nil(got.ConfigSpec) + s.NotNil(got.InvocationSpec) +} + +func (s *PluginTestSuite) TestCallMissingText() { + args := plugininterface.Args{ + Args: cty.ObjectVal(map[string]cty.Value{ + "text": cty.NullVal(cty.String), + "format_as": cty.NullVal(cty.String), + "absolute_title_size": cty.NullVal(cty.Number), + "code_language": cty.NullVal(cty.String), + }), + Context: map[string]any{ + "name": "World", + }, + } + expected := plugininterface.Result{ + Diags: hcl.Diagnostics{{ + Severity: hcl.DiagError, + Summary: "Failed to render text", + Detail: "text is required", + }}, + } + s.Equal(expected, s.plugin.Call(args)) +} +func (s *PluginTestSuite) TestCallText() { + args := plugininterface.Args{ + Args: cty.ObjectVal(map[string]cty.Value{ + "text": cty.StringVal("Hello {{.name}}!"), + "format_as": cty.NullVal(cty.String), + "absolute_title_size": cty.NullVal(cty.Number), + "code_language": cty.NullVal(cty.String), + }), + Context: map[string]any{ + "name": "World", + }, + } + expected := plugininterface.Result{ + Result: "Hello World!", + } + s.Equal(expected, s.plugin.Call(args)) +} + +func (s *PluginTestSuite) TestCallTextNoTemplate() { + args := plugininterface.Args{ + Args: cty.ObjectVal(map[string]cty.Value{ + "text": cty.StringVal("Hello World!"), + "format_as": cty.NullVal(cty.String), + "absolute_title_size": cty.NullVal(cty.Number), + "code_language": cty.NullVal(cty.String), + }), + Context: nil, + } + expected := plugininterface.Result{ + Result: "Hello World!", + } + s.Equal(expected, s.plugin.Call(args)) +} + +func (s *PluginTestSuite) TestCallTitleDefault() { + args := plugininterface.Args{ + Args: cty.ObjectVal(map[string]cty.Value{ + "text": cty.StringVal("Hello {{.name}}!"), + "format_as": cty.StringVal("title"), + "absolute_title_size": cty.NullVal(cty.Number), + "code_language": cty.NullVal(cty.String), + }), + Context: map[string]any{ + "name": "World", + }, + } + expected := plugininterface.Result{ + Result: "# Hello World!", + } + s.Equal(expected, s.plugin.Call(args)) +} + +func (s *PluginTestSuite) TestCallTitleWithTextMultiline() { + args := plugininterface.Args{ + Args: cty.ObjectVal(map[string]cty.Value{ + "text": cty.StringVal("Hello\n{{.name}}\nfor you!"), + "format_as": cty.StringVal("title"), + "absolute_title_size": cty.NullVal(cty.Number), + "code_language": cty.NullVal(cty.String), + }), + Context: map[string]any{ + "name": "World", + }, + } + expected := plugininterface.Result{ + Result: "# Hello World for you!", + } + s.Equal(expected, s.plugin.Call(args)) +} + +func (s *PluginTestSuite) TestCallTitleWithSize() { + args := plugininterface.Args{ + Args: cty.ObjectVal(map[string]cty.Value{ + "text": cty.StringVal("Hello {{.name}}!"), + "format_as": cty.StringVal("title"), + "absolute_title_size": cty.NumberIntVal(3), + "code_language": cty.NullVal(cty.String), + }), + Context: map[string]any{ + "name": "World", + }, + } + expected := plugininterface.Result{ + Result: "### Hello World!", + } + s.Equal(expected, s.plugin.Call(args)) +} + +func (s *PluginTestSuite) TestCallTitleWithSizeTooSmall() { + args := plugininterface.Args{ + Args: cty.ObjectVal(map[string]cty.Value{ + "text": cty.StringVal("Hello {{.name}}!"), + "format_as": cty.StringVal("title"), + "absolute_title_size": cty.NumberIntVal(0), + "code_language": cty.NullVal(cty.String), + }), + Context: map[string]any{ + "name": "World", + }, + } + expected := plugininterface.Result{ + Diags: hcl.Diagnostics{{ + Severity: hcl.DiagError, + Summary: "Failed to render text", + Detail: "absolute_title_size must be between 1 and 6", + }}, + } + s.Equal(expected, s.plugin.Call(args)) +} + +func (s *PluginTestSuite) TestCallTitleWithSizeTooBig() { + args := plugininterface.Args{ + Args: cty.ObjectVal(map[string]cty.Value{ + "text": cty.StringVal("Hello {{.name}}!"), + "format_as": cty.StringVal("title"), + "absolute_title_size": cty.NumberIntVal(7), + "code_language": cty.NullVal(cty.String), + }), + Context: map[string]any{ + "name": "World", + }, + } + expected := plugininterface.Result{ + Diags: hcl.Diagnostics{{ + Severity: hcl.DiagError, + Summary: "Failed to render text", + Detail: "absolute_title_size must be between 1 and 6", + }}, + } + s.Equal(expected, s.plugin.Call(args)) +} + +func (s *PluginTestSuite) TestCallInvalidFormat() { + args := plugininterface.Args{ + Args: cty.ObjectVal(map[string]cty.Value{ + "text": cty.StringVal("Hello World!"), + "format_as": cty.StringVal("unknown"), + "absolute_title_size": cty.NullVal(cty.Number), + "code_language": cty.NullVal(cty.String), + }), + Context: nil, + } + expected := plugininterface.Result{ + Diags: hcl.Diagnostics{{ + Severity: hcl.DiagError, + Summary: "Failed to render text", + Detail: "format_as must be one of text, title, code, blockquote", + }}, + } + s.Equal(expected, s.plugin.Call(args)) +} + +func (s *PluginTestSuite) TestCallInvalidTemplate() { + args := plugininterface.Args{ + Args: cty.ObjectVal(map[string]cty.Value{ + "text": cty.StringVal("Hello {{.name}!"), + "format_as": cty.NullVal(cty.String), + "absolute_title_size": cty.NullVal(cty.Number), + "code_language": cty.NullVal(cty.String), + }), + Context: map[string]any{ + "name": "World", + }, + } + expected := plugininterface.Result{ + Diags: hcl.Diagnostics{{ + Severity: hcl.DiagError, + Summary: "Failed to render text", + Detail: "failed to parse text template: template: text:1: bad character U+007D '}'", + }}, + } + s.Equal(expected, s.plugin.Call(args)) +} + +func (s *PluginTestSuite) TestCallCodeDefault() { + args := plugininterface.Args{ + Args: cty.ObjectVal(map[string]cty.Value{ + "text": cty.StringVal(`Hello {{.name}}!`), + "format_as": cty.StringVal("code"), + "absolute_title_size": cty.NullVal(cty.Number), + "code_language": cty.NullVal(cty.String), + }), + Context: map[string]any{ + "name": "World", + }, + } + expected := plugininterface.Result{ + Result: "```\nHello World!\n```", + } + s.Equal(expected, s.plugin.Call(args)) +} + +func (s *PluginTestSuite) TestCallCodeNoLanguage() { + args := plugininterface.Args{ + Args: cty.ObjectVal(map[string]cty.Value{ + "text": cty.StringVal(`{"hello": "{{.name}}"}`), + "format_as": cty.StringVal("code"), + "absolute_title_size": cty.NullVal(cty.Number), + "code_language": cty.StringVal("json"), + }), + Context: map[string]any{ + "name": "world", + }, + } + expected := plugininterface.Result{ + Result: "```json\n{\"hello\": \"world\"}\n```", + } + s.Equal(expected, s.plugin.Call(args)) +} + +func (s *PluginTestSuite) TestCallBlockquote() { + args := plugininterface.Args{ + Args: cty.ObjectVal(map[string]cty.Value{ + "text": cty.StringVal(`Hello {{.name}}!`), + "format_as": cty.StringVal("blockquote"), + "absolute_title_size": cty.NullVal(cty.Number), + "code_language": cty.NullVal(cty.String), + }), + Context: map[string]any{ + "name": "World", + }, + } + expected := plugininterface.Result{ + Result: "> Hello World!", + } + s.Equal(expected, s.plugin.Call(args)) +} + +func (s *PluginTestSuite) TestCallBlockquoteMultiline() { + args := plugininterface.Args{ + Args: cty.ObjectVal(map[string]cty.Value{ + "text": cty.StringVal("Hello\n{{.name}}\nfor you!"), + "format_as": cty.StringVal("blockquote"), + "absolute_title_size": cty.NullVal(cty.Number), + "code_language": cty.NullVal(cty.String), + }), + Context: map[string]any{ + "name": "World", + }, + } + expected := plugininterface.Result{ + Result: "> Hello\n> World\n> for you!", + } + s.Equal(expected, s.plugin.Call(args)) +} + +func (s *PluginTestSuite) TestCallBlockquoteMultilineDoubleNewline() { + args := plugininterface.Args{ + Args: cty.ObjectVal(map[string]cty.Value{ + "text": cty.StringVal("Hello\n{{.name}}\n\nfor you!"), + "format_as": cty.StringVal("blockquote"), + "absolute_title_size": cty.NullVal(cty.Number), + "code_language": cty.NullVal(cty.String), + }), + Context: map[string]any{ + "name": "World", + }, + } + expected := plugininterface.Result{ + Result: "> Hello\n> World\n> \n> for you!", + } + s.Equal(expected, s.plugin.Call(args)) +} diff --git a/plugins/data/csv/plugin.go b/plugins/data/csv/plugin.go new file mode 100644 index 00000000..40fdb039 --- /dev/null +++ b/plugins/data/csv/plugin.go @@ -0,0 +1,90 @@ +package csv + +import ( + "os" + + "github.com/Masterminds/semver/v3" + "github.com/blackstork-io/fabric/plugininterface/v1" + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/hcldec" + "github.com/zclconf/go-cty/cty" +) + +var Version = semver.MustParse("0.1.0") + +const defaultDelimiter = ',' + +type Plugin struct{} + +func (Plugin) GetPlugins() []plugininterface.Plugin { + return []plugininterface.Plugin{ + { + Namespace: "blackstork", + Kind: "data", + Name: "csv", + Version: plugininterface.Version(*Version), + ConfigSpec: nil, + InvocationSpec: &hcldec.ObjectSpec{ + "path": &hcldec.AttrSpec{ + Name: "path", + Type: cty.String, + Required: true, + }, + "delimiter": &hcldec.AttrSpec{ + Name: "delimiter", + Type: cty.String, + Required: false, + }, + }, + }, + } +} + +func (Plugin) Call(args plugininterface.Args) plugininterface.Result { + path := args.Args.GetAttr("path") + if path.IsNull() || path.AsString() == "" { + return plugininterface.Result{ + Diags: hcl.Diagnostics{{ + Severity: hcl.DiagError, + Summary: "path is required", + }}, + } + } + delim := args.Args.GetAttr("delimiter") + if delim.IsNull() { + delim = cty.StringVal(string(defaultDelimiter)) + } + if len(delim.AsString()) != 1 { + return plugininterface.Result{ + Diags: hcl.Diagnostics{{ + Severity: hcl.DiagError, + Summary: "delimiter must be a single character", + }}, + } + } + delimRune := []rune(delim.AsString())[0] + wd, err := os.Getwd() + if err != nil { + return plugininterface.Result{ + Diags: hcl.Diagnostics{{ + Severity: hcl.DiagError, + Summary: "Failed to get current working directory", + Detail: err.Error(), + }}, + } + } + filesystem := os.DirFS(wd) + data, err := readFS(filesystem, path.AsString(), delimRune) + if err != nil { + return plugininterface.Result{ + Diags: hcl.Diagnostics{{ + Severity: hcl.DiagError, + Summary: "Failed to read csv file", + Detail: err.Error(), + }}, + } + } + return plugininterface.Result{ + Result: data, + } +} diff --git a/plugins/data/csv/plugin_test.go b/plugins/data/csv/plugin_test.go new file mode 100644 index 00000000..4a78d1d9 --- /dev/null +++ b/plugins/data/csv/plugin_test.go @@ -0,0 +1,194 @@ +package csv + +import ( + "testing" + + "github.com/blackstork-io/fabric/plugininterface/v1" + "github.com/hashicorp/hcl/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/zclconf/go-cty/cty" +) + +func TestPlugin_GetPlugins(t *testing.T) { + plugin := Plugin{} + plugins := plugin.GetPlugins() + require.Len(t, plugins, 1, "expected 1 plugin") + got := plugins[0] + assert.Equal(t, "csv", got.Name) + assert.Equal(t, "data", got.Kind) + assert.Equal(t, "blackstork", got.Namespace) + assert.Equal(t, Version.String(), got.Version.Cast().String()) + assert.Nil(t, got.ConfigSpec) + assert.NotNil(t, got.InvocationSpec) +} + +func TestPlugin_Call(t *testing.T) { + tt := []struct { + name string + path string + delimiter string + expected plugininterface.Result + }{ + { + name: "comma_delim", + path: "testdata/comma.csv", + delimiter: ",", + expected: plugininterface.Result{ + Result: []map[string]any{ + { + "id": "b8fa4bb0-6dd4-45ba-96e0-9a182b2b932e", + "active": true, + "name": "Stacey", + "age": int64(26), + "height": float64(1.98), + }, + { + "id": "b0086c49-bcd8-4aae-9f88-4f46b128e709", + "active": false, + "name": "Myriam", + "age": int64(33), + "height": float64(1.81), + }, + { + "id": "a12d2a8c-eebc-42b3-be52-1ab0a2969a81", + "active": true, + "name": "Oralee", + "age": int64(31), + "height": float64(2.23), + }, + }, + }, + }, + { + name: "semicolon_delim", + path: "testdata/semicolon.csv", + delimiter: ";", + expected: plugininterface.Result{ + Result: []map[string]any{ + { + "id": "b8fa4bb0-6dd4-45ba-96e0-9a182b2b932e", + "active": true, + "name": "Stacey", + "age": int64(26), + "height": float64(1.98), + }, + { + "id": "b0086c49-bcd8-4aae-9f88-4f46b128e709", + "active": false, + "name": "Myriam", + "age": int64(33), + "height": float64(1.81), + }, + { + "id": "a12d2a8c-eebc-42b3-be52-1ab0a2969a81", + "active": true, + "name": "Oralee", + "age": int64(31), + "height": float64(2.23), + }, + }, + }, + }, + { + name: "empty_path", + expected: plugininterface.Result{ + Diags: hcl.Diagnostics{{ + Severity: hcl.DiagError, + Summary: "path is required", + }}, + }, + }, + { + name: "invalid_delimiter", + path: "testdata/comma.csv", + delimiter: "abc", + expected: plugininterface.Result{ + Diags: hcl.Diagnostics{{ + Severity: hcl.DiagError, + Summary: "delimiter must be a single character", + }}, + }, + }, + { + name: "default_delimiter", + path: "testdata/comma.csv", + expected: plugininterface.Result{ + Result: []map[string]any{ + { + "id": "b8fa4bb0-6dd4-45ba-96e0-9a182b2b932e", + "active": true, + "name": "Stacey", + "age": int64(26), + "height": float64(1.98), + }, + { + "id": "b0086c49-bcd8-4aae-9f88-4f46b128e709", + "active": false, + "name": "Myriam", + "age": int64(33), + "height": float64(1.81), + }, + { + "id": "a12d2a8c-eebc-42b3-be52-1ab0a2969a81", + "active": true, + "name": "Oralee", + "age": int64(31), + "height": float64(2.23), + }, + }, + }, + }, + { + name: "invalid_path", + path: "testdata/does_not_exist.csv", + expected: plugininterface.Result{ + Diags: hcl.Diagnostics{{ + Severity: hcl.DiagError, + Summary: "Failed to read csv file", + Detail: "open testdata/does_not_exist.csv: no such file or directory", + }}, + }, + }, + + { + name: "invalid_csv", + path: "testdata/invalid.csv", + expected: plugininterface.Result{ + Diags: hcl.Diagnostics{{ + Severity: hcl.DiagError, + Summary: "Failed to read csv file", + Detail: "record on line 2: wrong number of fields", + }}, + }, + }, + { + name: "empty_csv", + path: "testdata/empty.csv", + delimiter: ",", + expected: plugininterface.Result{ + Result: []map[string]any{}, + }, + }, + } + + for _, tc := range tt { + t.Run(tc.name, func(t *testing.T) { + plugin := Plugin{} + delim := cty.StringVal(tc.delimiter) + if tc.delimiter == "" { + delim = cty.NullVal(cty.String) + } + args := plugininterface.Args{ + Kind: "data", + Name: "csv", + Args: cty.ObjectVal(map[string]cty.Value{ + "path": cty.StringVal(tc.path), + "delimiter": delim, + }), + } + got := plugin.Call(args) + assert.Equal(t, tc.expected, got) + }) + } +} diff --git a/plugins/data/csv/read.go b/plugins/data/csv/read.go new file mode 100644 index 00000000..823d6337 --- /dev/null +++ b/plugins/data/csv/read.go @@ -0,0 +1,53 @@ +package csv + +import ( + "encoding/csv" + "encoding/json" + "io/fs" +) + +func readFS(filesystem fs.FS, path string, sep rune) ([]map[string]any, error) { + f, err := filesystem.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + r := csv.NewReader(f) + r.Comma = sep + records, err := r.ReadAll() + if err != nil { + return nil, err + } + if len(records) == 0 { + return []map[string]any{}, nil + } + result := make([]map[string]any, len(records)-1) + headers := records[0] + for i, record := range records[1:] { + result[i] = make(map[string]any, len(headers)) + for j, header := range headers { + if header == "" { + continue + } + if j >= len(record) { + result[i][header] = nil + continue + } + if record[j] == "true" { + result[i][header] = true + } else if record[j] == "false" { + result[i][header] = false + } else { + n := json.Number(record[j]) + if e, err := n.Int64(); err == nil { + result[i][header] = e + } else if f, err := n.Float64(); err == nil { + result[i][header] = f + } else { + result[i][header] = record[j] + } + } + } + } + return result, nil +} diff --git a/plugins/data/csv/testdata/comma.csv b/plugins/data/csv/testdata/comma.csv new file mode 100644 index 00000000..94ecff74 --- /dev/null +++ b/plugins/data/csv/testdata/comma.csv @@ -0,0 +1,4 @@ +id,active,name,age,height +b8fa4bb0-6dd4-45ba-96e0-9a182b2b932e,true,Stacey,26,1.98 +b0086c49-bcd8-4aae-9f88-4f46b128e709,false,Myriam,33,1.81 +a12d2a8c-eebc-42b3-be52-1ab0a2969a81,true,Oralee,31,2.23 \ No newline at end of file diff --git a/plugins/data/csv/testdata/empty.csv b/plugins/data/csv/testdata/empty.csv new file mode 100644 index 00000000..e69de29b diff --git a/plugins/data/csv/testdata/invalid.csv b/plugins/data/csv/testdata/invalid.csv new file mode 100644 index 00000000..813f3b3c --- /dev/null +++ b/plugins/data/csv/testdata/invalid.csv @@ -0,0 +1,4 @@ +id,name,age,height +b8fa4bb0-6dd4-45ba-96e0-9a182b2b932e Stacey,26,1.98 +b0086c49-bcd8-4aae-9f88-4f46b128e709,Myriam,33,1.81, +a12d2a8c-eebc-42b3-be52-1ab0a2969a81,Oralee,31,2.23 \ No newline at end of file diff --git a/plugins/data/csv/testdata/semicolon.csv b/plugins/data/csv/testdata/semicolon.csv new file mode 100644 index 00000000..c19b9385 --- /dev/null +++ b/plugins/data/csv/testdata/semicolon.csv @@ -0,0 +1,4 @@ +id;active;name;age;height +b8fa4bb0-6dd4-45ba-96e0-9a182b2b932e;true;Stacey;26;1.98 +b0086c49-bcd8-4aae-9f88-4f46b128e709;false;Myriam;33;1.81 +a12d2a8c-eebc-42b3-be52-1ab0a2969a81;true;Oralee;31;2.23 \ No newline at end of file diff --git a/plugins/data/elasticsearch/plugin.go b/plugins/data/elasticsearch/plugin.go new file mode 100644 index 00000000..c8b78be9 --- /dev/null +++ b/plugins/data/elasticsearch/plugin.go @@ -0,0 +1,251 @@ +package elasticsearch + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + + "github.com/Masterminds/semver/v3" + "github.com/blackstork-io/fabric/plugininterface/v1" + es "github.com/elastic/go-elasticsearch/v8" + "github.com/elastic/go-elasticsearch/v8/esapi" + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/hcldec" + "github.com/zclconf/go-cty/cty" +) + +const ( + defaultBaseURL = "http://localhost:9200" + defaultUsername = "elastic" +) + +var Version = semver.MustParse("0.1.0") + +type Plugin struct{} + +func (Plugin) GetPlugins() []plugininterface.Plugin { + return []plugininterface.Plugin{ + { + Namespace: "blackstork", + Kind: "data", + Name: "elasticsearch", + Version: plugininterface.Version(*Version), + ConfigSpec: &hcldec.ObjectSpec{ + "base_url": &hcldec.AttrSpec{ + Name: "base_url", + Type: cty.String, + Required: true, + }, + "cloud_id": &hcldec.AttrSpec{ + Name: "cloud_id", + Type: cty.String, + Required: false, + }, + "api_key_str": &hcldec.AttrSpec{ + Name: "api_key_str", + Type: cty.String, + Required: false, + }, + "api_key": &hcldec.AttrSpec{ + Name: "api_key", + Type: cty.List(cty.String), + Required: false, + }, + "basic_auth_username": &hcldec.AttrSpec{ + Name: "basic_auth_username", + Type: cty.String, + Required: false, + }, + "basic_auth_password": &hcldec.AttrSpec{ + Name: "basic_auth_password", + Type: cty.String, + Required: false, + }, + "bearer_auth": &hcldec.AttrSpec{ + Name: "bearer_auth", + Type: cty.String, + Required: false, + }, + "ca_certs": &hcldec.AttrSpec{ + Name: "ca_certs", + Type: cty.String, + Required: false, + }, + }, + InvocationSpec: &hcldec.ObjectSpec{ + "index": &hcldec.AttrSpec{ + Name: "index", + Type: cty.String, + Required: true, + }, + "id": &hcldec.AttrSpec{ + Name: "index", + Type: cty.String, + Required: false, + }, + "query_string": &hcldec.AttrSpec{ + Name: "query_string", + Type: cty.String, + Required: false, + }, + "query": &hcldec.AttrSpec{ + Name: "query", + Type: cty.Map(cty.DynamicPseudoType), + Required: false, + }, + "fields": &hcldec.AttrSpec{ + Name: "fields", + Type: cty.List(cty.String), + Required: false, + }, + }, + }, + } +} + +func (p Plugin) makeClient(pcfg cty.Value) (*es.Client, error) { + cfg := &es.Config{ + Addresses: []string{defaultBaseURL}, + Username: defaultUsername, + } + if baseURL := pcfg.GetAttr("base_url"); !baseURL.IsNull() { + cfg.Addresses = []string{baseURL.AsString()} + } + if cloudID := pcfg.GetAttr("cloud_id"); !cloudID.IsNull() { + cfg.CloudID = cloudID.AsString() + } + if apiKeyStr := pcfg.GetAttr("api_key_str"); !apiKeyStr.IsNull() { + cfg.APIKey = apiKeyStr.AsString() + } + if apiKey := pcfg.GetAttr("api_key"); !apiKey.IsNull() { + list := apiKey.AsValueSlice() + if len(list) != 2 { + return nil, fmt.Errorf("api_key must be a list of 2 strings") + } + cfg.APIKey = base64.StdEncoding.EncodeToString([]byte( + fmt.Sprintf("%s:%s", list[0].AsString(), list[1].AsString())), + ) + } + if basicAuthUsername := pcfg.GetAttr("basic_auth_username"); !basicAuthUsername.IsNull() { + cfg.Username = basicAuthUsername.AsString() + } + if basicAuthPassword := pcfg.GetAttr("basic_auth_password"); !basicAuthPassword.IsNull() { + cfg.Password = basicAuthPassword.AsString() + } + if bearerAuth := pcfg.GetAttr("bearer_auth"); !bearerAuth.IsNull() { + cfg.ServiceToken = bearerAuth.AsString() + } + if caCerts := pcfg.GetAttr("ca_certs"); !caCerts.IsNull() { + cfg.CACert = []byte(caCerts.AsString()) + } + return es.NewClient(*cfg) +} + +func (p Plugin) Call(args plugininterface.Args) plugininterface.Result { + + client, err := p.makeClient(args.Config) + if err != nil { + return plugininterface.Result{ + Diags: hcl.Diagnostics{{ + Severity: hcl.DiagError, + Summary: "Failed to create elasticsearch client", + Detail: err.Error(), + }}, + } + } + id := args.Args.GetAttr("id") + var data map[string]any + if !id.IsNull() { + data, err = p.getByID(client.Get, args.Args) + } else { + data, err = p.search(client.Search, args.Args) + } + if err != nil { + return plugininterface.Result{ + Diags: hcl.Diagnostics{{ + Severity: hcl.DiagError, + Summary: "Failed to get data", + Detail: err.Error(), + }}, + } + } + return plugininterface.Result{ + Result: data, + } +} + +func (Plugin) getByID(fn esapi.Get, args cty.Value) (map[string]any, error) { + index := args.GetAttr("index") + if index.IsNull() { + return nil, errors.New("index is required") + } + id := args.GetAttr("id") + if id.IsNull() { + return nil, errors.New("id is required when id is specified") + } + opts := []func(*esapi.GetRequest){} + if fields := args.GetAttr("fields"); !fields.IsNull() { + fieldSlice := fields.AsValueSlice() + fieldStrings := make([]string, len(fieldSlice)) + for i, v := range fieldSlice { + fieldStrings[i] = v.AsString() + } + opts = append(opts, fn.WithSource(fieldStrings...)) + } + res, err := fn(index.AsString(), id.AsString(), opts...) + if err != nil { + return nil, err + } + if res.IsError() { + return nil, fmt.Errorf("failed to get document: %s", res.String()) + } + var data map[string]any + if err := json.NewDecoder(res.Body).Decode(&data); err != nil { + return nil, fmt.Errorf("failed to unmarshal document: %s", err) + } + return data, nil +} + +func (Plugin) search(fn esapi.Search, args cty.Value) (map[string]any, error) { + index := args.GetAttr("index") + if index.IsNull() { + return nil, errors.New("index is required") + } + opts := []func(*esapi.SearchRequest){ + fn.WithIndex(index.AsString()), + } + if queryString := args.GetAttr("query_string"); !queryString.IsNull() { + opts = append(opts, fn.WithQuery(queryString.AsString())) + } + if query := args.GetAttr("query"); !query.IsNull() { + queryRaw, err := json.Marshal(map[string]any{ + "query": query.AsValueMap(), + }) + if err != nil { + return nil, fmt.Errorf("failed to marshal query: %s", err) + } + opts = append(opts, fn.WithBody(bytes.NewReader(queryRaw))) + } + if fields := args.GetAttr("fields"); !fields.IsNull() { + fieldSlice := fields.AsValueSlice() + fieldStrings := make([]string, len(fieldSlice)) + for i, v := range fieldSlice { + fieldStrings[i] = v.AsString() + } + opts = append(opts, fn.WithSource(fieldStrings...)) + } + + res, err := fn(opts...) + if err != nil { + return nil, err + } else if res.IsError() { + return nil, fmt.Errorf("failed to search: %s", res.String()) + } + var data map[string]any + if err := json.NewDecoder(res.Body).Decode(&data); err != nil { + return nil, fmt.Errorf("failed to unmarshal search result: %s", err) + } + return data, nil +} diff --git a/plugins/data/elasticsearch/plugin_integration_test.go b/plugins/data/elasticsearch/plugin_integration_test.go new file mode 100644 index 00000000..e754aad3 --- /dev/null +++ b/plugins/data/elasticsearch/plugin_integration_test.go @@ -0,0 +1,426 @@ +package elasticsearch + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "os" + "testing" + + "github.com/blackstork-io/fabric/plugininterface/v1" + es "github.com/elastic/go-elasticsearch/v8" + "github.com/stretchr/testify/suite" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/modules/elasticsearch" + "github.com/zclconf/go-cty/cty" +) + +// IntegrationTestSuite is a test suite to test integration with real elasticsearch +type IntegrationTestSuite struct { + suite.Suite + container *elasticsearch.ElasticsearchContainer + client *es.Client + plugin Plugin + cfg cty.Value + ctx context.Context +} + +func TestIntegrationSuite(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration tests") + } + suite.Run(t, &IntegrationTestSuite{}) +} + +func (s *IntegrationTestSuite) SetupSuite() { + s.ctx = context.Background() + opts := []testcontainers.ContainerCustomizer{ + testcontainers.WithImage("docker.elastic.co/elasticsearch/elasticsearch:8.9.0"), + elasticsearch.WithPassword("password123"), + } + container, err := elasticsearch.RunContainer(s.ctx, opts...) + + s.Require().NoError(err, "failed to start elasticsearch container") + s.container = container + client, err := es.NewClient(es.Config{ + Addresses: []string{ + container.Settings.Address, + }, + Username: "elastic", + Password: container.Settings.Password, + CACert: container.Settings.CACert, + }) + s.Require().NoError(err, "failed to create elasticsearch client") + s.client = client + s.cfg = cty.ObjectVal(map[string]cty.Value{ + "base_url": cty.StringVal(s.container.Settings.Address), + "cloud_id": cty.NullVal(cty.String), + "api_key_str": cty.NullVal(cty.String), + "api_key": cty.NullVal(cty.List(cty.String)), + "basic_auth_username": cty.StringVal("elastic"), + "basic_auth_password": cty.StringVal("password123"), + "bearer_auth": cty.NullVal(cty.String), + "ca_certs": cty.StringVal(string(s.container.Settings.CACert)), + }) +} + +func (s *IntegrationTestSuite) TearDownSuite() { + s.Require().NoError(s.container.Terminate(s.ctx), "failed to stop elasticsearch container") +} + +type testDataObject struct { + ID string `json:"id"` + Type string `json:"type"` + Active bool `json:"active"` + Age int `json:"age"` + Name string `json:"name"` +} + +func (s *IntegrationTestSuite) SetupTest() { + file, err := os.ReadFile("testdata/data.json") + s.Require().NoError(err, "failed to read data.json") + dataList := []testDataObject{} + err = json.Unmarshal(file, &dataList) + s.Require().NoError(err, "failed to unmarshal data.json") + res, err := s.client.Indices.Create("test_index") + s.Require().NoError(err, "failed to create index test_index") + s.Require().Equal(http.StatusOK, res.StatusCode) + for _, data := range dataList { + dataBytes, err := json.Marshal(data) + s.Require().NoError(err, "failed to marshal data") + res, err := s.client.Create("test_index", data.ID, bytes.NewReader(dataBytes)) + s.Require().NoError(err, "failed to index data") + s.Require().False(res.IsError(), "failed to index data") + res, err = s.client.Index("test_index", bytes.NewReader(dataBytes), s.client.Index.WithDocumentID(data.ID)) + s.Require().NoError(err, "failed to index data") + s.Require().False(res.IsError(), "failed to index data") + } + res, err = s.client.Indices.Refresh() + s.Require().NoError(err, "failed to refresh indices") + s.Require().False(res.IsError(), "failed to refresh indices") +} + +func (s *IntegrationTestSuite) TearDownTest() { + res, err := s.client.Indices.Delete([]string{"test_index"}) + s.Require().NoError(err, "failed to delete indices") + s.Require().False(res.IsError(), "failed to delete indices: %s", res.String()) +} + +func (s *IntegrationTestSuite) TestSearchDefaults() { + res := s.plugin.Call(plugininterface.Args{ + Kind: "data", + Name: "elasticsearch", + Config: s.cfg, + Args: cty.ObjectVal(map[string]cty.Value{ + "id": cty.NullVal(cty.String), + "index": cty.StringVal("test_index"), + "query": cty.NullVal(cty.DynamicPseudoType), + "query_string": cty.NullVal(cty.String), + "fields": cty.NullVal(cty.String), + }), + }) + s.Require().False(res.Diags.HasErrors(), "failed to call plugin: %v", res.Diags.Error()) + data := res.Result.(map[string]any) + raw, err := json.MarshalIndent(data["hits"], "", " ") + s.Require().NoError(err, "failed to marshal data: %v", res.Diags.Error()) + s.JSONEq(`{ + "hits": [ + { + "_id": "54f7a815-eac5-4f7c-a339-5fefd0f54967", + "_index": "test_index", + "_score": 1, + "_source": { + "active": false, + "age": 39, + "id": "54f7a815-eac5-4f7c-a339-5fefd0f54967", + "name": "Davidson", + "type": "foo" + } + }, + { + "_id": "0c68e63d-daaa-4a62-92e6-e855bd144fb6", + "_index": "test_index", + "_score": 1, + "_source": { + "active": false, + "age": 20, + "id": "0c68e63d-daaa-4a62-92e6-e855bd144fb6", + "name": "Thompson", + "type": "bar" + } + }, + { + "_id": "a117a5e6-23d0-4daa-be3c-a70900ca4163", + "_index": "test_index", + "_score": 1, + "_source": { + "active": true, + "age": 21, + "id": "a117a5e6-23d0-4daa-be3c-a70900ca4163", + "name": "Armstrong", + "type": "foo" + } + } + ], + "max_score": 1, + "total": { + "relation": "eq", + "value": 3 + } + }`, string(raw)) +} + +func (s *IntegrationTestSuite) TestSearchFields() { + res := s.plugin.Call(plugininterface.Args{ + Kind: "data", + Name: "elasticsearch", + Config: s.cfg, + Args: cty.ObjectVal(map[string]cty.Value{ + "id": cty.NullVal(cty.String), + "index": cty.StringVal("test_index"), + "query": cty.NullVal(cty.DynamicPseudoType), + "query_string": cty.NullVal(cty.String), + "fields": cty.ListVal([]cty.Value{cty.StringVal("name"), cty.StringVal("age")}), + }), + }) + s.Require().False(res.Diags.HasErrors(), "failed to call plugin: %v", res.Diags.Error()) + data := res.Result.(map[string]any) + raw, err := json.MarshalIndent(data["hits"], "", " ") + s.Require().NoError(err, "failed to marshal data: %v", res.Diags.Error()) + s.JSONEq(`{ + "hits": [ + { + "_id": "54f7a815-eac5-4f7c-a339-5fefd0f54967", + "_index": "test_index", + "_score": 1, + "_source": { + "age": 39, + "name": "Davidson" + } + }, + { + "_id": "0c68e63d-daaa-4a62-92e6-e855bd144fb6", + "_index": "test_index", + "_score": 1, + "_source": { + "age": 20, + "name": "Thompson" + } + }, + { + "_id": "a117a5e6-23d0-4daa-be3c-a70900ca4163", + "_index": "test_index", + "_score": 1, + "_source": { + "age": 21, + "name": "Armstrong" + } + } + ], + "max_score": 1, + "total": { + "relation": "eq", + "value": 3 + } + }`, string(raw)) +} +func (s *IntegrationTestSuite) TestSearchQueryString() { + res := s.plugin.Call(plugininterface.Args{ + Kind: "data", + Name: "elasticsearch", + Config: s.cfg, + Args: cty.ObjectVal(map[string]cty.Value{ + "id": cty.NullVal(cty.String), + "index": cty.StringVal("test_index"), + "query": cty.NullVal(cty.DynamicPseudoType), + "query_string": cty.StringVal("type:foo"), + "fields": cty.NullVal(cty.String), + }), + }) + s.Require().False(res.Diags.HasErrors(), "failed to call plugin: %v", res.Diags.Error()) + data := res.Result.(map[string]any) + raw, err := json.MarshalIndent(data["hits"], "", " ") + s.Require().NoError(err, "failed to marshal data: %v", res.Diags.Error()) + s.JSONEq(`{ + "hits": [ + { + "_id": "54f7a815-eac5-4f7c-a339-5fefd0f54967", + "_index": "test_index", + "_score": 0.44183272, + "_source": { + "active": false, + "age": 39, + "id": "54f7a815-eac5-4f7c-a339-5fefd0f54967", + "name": "Davidson", + "type": "foo" + } + }, + { + "_id": "a117a5e6-23d0-4daa-be3c-a70900ca4163", + "_index": "test_index", + "_score": 0.44183272, + "_source": { + "active": true, + "age": 21, + "id": "a117a5e6-23d0-4daa-be3c-a70900ca4163", + "name": "Armstrong", + "type": "foo" + } + } + ], + "max_score": 0.44183272, + "total": { + "relation": "eq", + "value": 2 + } + }`, string(raw)) +} + +func (s *IntegrationTestSuite) TestSearchQuery() { + res := s.plugin.Call(plugininterface.Args{ + Kind: "data", + Name: "elasticsearch", + Config: s.cfg, + Args: cty.ObjectVal(map[string]cty.Value{ + "id": cty.NullVal(cty.String), + "index": cty.StringVal("test_index"), + "query": cty.MapVal(map[string]cty.Value{ + "match_all": cty.MapValEmpty(cty.DynamicPseudoType), + }), + "query_string": cty.NullVal(cty.String), + "fields": cty.NullVal(cty.String), + }), + }) + s.Require().False(res.Diags.HasErrors(), "failed to call plugin: %v", res.Diags.Error()) + data := res.Result.(map[string]any) + raw, err := json.MarshalIndent(data["hits"], "", " ") + s.Require().NoError(err, "failed to marshal data: %v", res.Diags.Error()) + s.JSONEq(`{ + "hits": [ + { + "_id": "54f7a815-eac5-4f7c-a339-5fefd0f54967", + "_index": "test_index", + "_score": 1, + "_source": { + "active": false, + "age": 39, + "id": "54f7a815-eac5-4f7c-a339-5fefd0f54967", + "name": "Davidson", + "type": "foo" + } + }, + { + "_id": "0c68e63d-daaa-4a62-92e6-e855bd144fb6", + "_index": "test_index", + "_score": 1, + "_source": { + "active": false, + "age": 20, + "id": "0c68e63d-daaa-4a62-92e6-e855bd144fb6", + "name": "Thompson", + "type": "bar" + } + }, + { + "_id": "a117a5e6-23d0-4daa-be3c-a70900ca4163", + "_index": "test_index", + "_score": 1, + "_source": { + "active": true, + "age": 21, + "id": "a117a5e6-23d0-4daa-be3c-a70900ca4163", + "name": "Armstrong", + "type": "foo" + } + } + ], + "max_score": 1, + "total": { + "relation": "eq", + "value": 3 + } + }`, string(raw)) +} + +func (s *IntegrationTestSuite) TestGetByID() { + res := s.plugin.Call(plugininterface.Args{ + Kind: "data", + Name: "elasticsearch", + Config: s.cfg, + Args: cty.ObjectVal(map[string]cty.Value{ + "id": cty.StringVal("0c68e63d-daaa-4a62-92e6-e855bd144fb6"), + "index": cty.StringVal("test_index"), + "query": cty.NullVal(cty.DynamicPseudoType), + "query_string": cty.NullVal(cty.String), + "fields": cty.NullVal(cty.String), + }), + }) + s.Require().False(res.Diags.HasErrors(), "failed to call plugin: %v", res.Diags.Error()) + data := res.Result.(map[string]any) + raw, err := json.MarshalIndent(data, "", " ") + + s.Require().NoError(err, "failed to marshal data: %v", res.Diags.Error()) + s.JSONEq(`{ + "_id": "0c68e63d-daaa-4a62-92e6-e855bd144fb6", + "_index": "test_index", + "_primary_term": 1, + "_seq_no": 3, + "_source": { + "active": false, + "age": 20, + "id": "0c68e63d-daaa-4a62-92e6-e855bd144fb6", + "name": "Thompson", + "type": "bar" + }, + "_version": 2, + "found": true + }`, string(raw)) +} +func (s *IntegrationTestSuite) TestGetByIDFields() { + res := s.plugin.Call(plugininterface.Args{ + Kind: "data", + Name: "elasticsearch", + Config: s.cfg, + Args: cty.ObjectVal(map[string]cty.Value{ + "id": cty.StringVal("0c68e63d-daaa-4a62-92e6-e855bd144fb6"), + "index": cty.StringVal("test_index"), + "query": cty.NullVal(cty.DynamicPseudoType), + "query_string": cty.NullVal(cty.String), + "fields": cty.ListVal([]cty.Value{cty.StringVal("name"), cty.StringVal("age")}), + }), + }) + s.Require().False(res.Diags.HasErrors(), "failed to call plugin: %v", res.Diags.Error()) + data := res.Result.(map[string]any) + raw, err := json.MarshalIndent(data, "", " ") + + s.Require().NoError(err, "failed to marshal data: %v", res.Diags.Error()) + s.JSONEq(`{ + "_id": "0c68e63d-daaa-4a62-92e6-e855bd144fb6", + "_index": "test_index", + "_primary_term": 1, + "_seq_no": 3, + "_source": { + "age": 20, + "name": "Thompson" + }, + "_version": 2, + "found": true + }`, string(raw)) +} + +func (s *IntegrationTestSuite) TestGetByIDNotFound() { + res := s.plugin.Call(plugininterface.Args{ + Kind: "data", + Name: "elasticsearch", + Config: s.cfg, + Args: cty.ObjectVal(map[string]cty.Value{ + "id": cty.StringVal("00000000-0000-0000-0000-000000000000"), + "index": cty.StringVal("test_index"), + "query": cty.NullVal(cty.DynamicPseudoType), + "query_string": cty.NullVal(cty.String), + "fields": cty.NullVal(cty.String), + }), + }) + s.Require().True(res.Diags.HasErrors(), "should have failed") +} diff --git a/plugins/data/elasticsearch/plugin_test.go b/plugins/data/elasticsearch/plugin_test.go new file mode 100644 index 00000000..24c41418 --- /dev/null +++ b/plugins/data/elasticsearch/plugin_test.go @@ -0,0 +1,21 @@ +package elasticsearch + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestPlugin_GetPlugins(t *testing.T) { + plugin := Plugin{} + plugins := plugin.GetPlugins() + require.Len(t, plugins, 1, "expected 1 plugin") + got := plugins[0] + assert.Equal(t, "elasticsearch", got.Name) + assert.Equal(t, "data", got.Kind) + assert.Equal(t, "blackstork", got.Namespace) + assert.Equal(t, Version.String(), got.Version.Cast().String()) + assert.NotNil(t, got.ConfigSpec) + assert.NotNil(t, got.InvocationSpec) +} diff --git a/plugins/data/elasticsearch/testdata/data.json b/plugins/data/elasticsearch/testdata/data.json new file mode 100644 index 00000000..f36863e4 --- /dev/null +++ b/plugins/data/elasticsearch/testdata/data.json @@ -0,0 +1,23 @@ +[ + { + "id": "54f7a815-eac5-4f7c-a339-5fefd0f54967", + "type": "foo", + "active": false, + "age": 39, + "name": "Davidson" + }, + { + "id": "0c68e63d-daaa-4a62-92e6-e855bd144fb6", + "type": "bar", + "active": false, + "age": 20, + "name": "Thompson" + }, + { + "id": "a117a5e6-23d0-4daa-be3c-a70900ca4163", + "type": "foo", + "active": true, + "age": 21, + "name": "Armstrong" + } +] \ No newline at end of file diff --git a/plugins/data/json/helpers_test.go b/plugins/data/json/helpers_test.go new file mode 100644 index 00000000..c7390390 --- /dev/null +++ b/plugins/data/json/helpers_test.go @@ -0,0 +1,71 @@ +package json + +import ( + "encoding/json" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + "testing" +) + +type testFS struct { + FS fs.FS + path string +} + +func makeTestFS(tb testing.TB) testFS { + tb.Helper() + + path, err := filepath.EvalSymlinks(tb.TempDir()) + if err != nil { + tb.Fatalf("failed to create testFS: %s", err) + } + + path = filepath.ToSlash(path) + + tb.Logf("creating testFS at %s", path) + return testFS{ + FS: os.DirFS(path), + path: path, + } +} + +func (t testFS) Open(name string) (fs.File, error) { + return t.FS.Open(filepath.ToSlash(name)) +} + +func (t testFS) Path() string { + return t.path +} + +func (t testFS) WriteFile(name string, data []byte, perm os.FileMode) error { + name = filepath.ToSlash(name) + if filepath.IsAbs(name) { + if strings.HasPrefix(name, t.path) { + return os.WriteFile(name, data, perm) + } + return fmt.Errorf("path is outside test fs root folder") + } + return os.WriteFile(filepath.ToSlash(filepath.Join(t.path, name)), data, perm) +} + +func (t testFS) MkdirAll(path string, perm os.FileMode) error { + path = filepath.ToSlash(path) + if filepath.IsAbs(path) { + if strings.HasPrefix(path, t.path) { + return os.MkdirAll(path, perm) + } + return fmt.Errorf("path is outside test fs root folder") + } + return os.MkdirAll(filepath.ToSlash(filepath.Join(t.path, path)), perm) +} + +func testJSON(m any) json.RawMessage { + contents, err := json.Marshal(m) + if err != nil { + panic(err) + } + return contents +} diff --git a/plugins/data/json/plugin.go b/plugins/data/json/plugin.go new file mode 100644 index 00000000..54ac36ff --- /dev/null +++ b/plugins/data/json/plugin.go @@ -0,0 +1,66 @@ +package json + +import ( + "os" + + "github.com/Masterminds/semver/v3" + "github.com/blackstork-io/fabric/plugininterface/v1" + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/hcldec" + "github.com/zclconf/go-cty/cty" +) + +var Version = semver.MustParse("0.1.0") + +type Plugin struct{} + +func (Plugin) GetPlugins() []plugininterface.Plugin { + return []plugininterface.Plugin{ + { + Namespace: "blackstork", + Kind: "data", + Name: "json", + Version: plugininterface.Version(*Version), + ConfigSpec: nil, + InvocationSpec: &hcldec.ObjectSpec{ + "glob": &hcldec.AttrSpec{ + Name: "glob", + Type: cty.String, + Required: true, + }, + }, + }, + } +} + +func (Plugin) Call(args plugininterface.Args) plugininterface.Result { + glob := args.Args.GetAttr("glob").AsString() + wd, err := os.Getwd() + if err != nil { + return plugininterface.Result{ + Diags: hcl.Diagnostics{{ + Severity: hcl.DiagError, + Summary: "Failed to get current working directory", + Detail: err.Error(), + }}, + } + } + filesystem := os.DirFS(wd) + docs, err := readFS(filesystem, glob) + if err != nil { + return plugininterface.Result{ + Diags: hcl.Diagnostics{{ + Severity: hcl.DiagError, + Summary: "Failed to read json files", + Detail: err.Error(), + }}, + } + } + data := make([]any, len(docs)) + for i, doc := range docs { + data[i] = doc.Map() + } + return plugininterface.Result{ + Result: data, + } +} diff --git a/plugins/data/json/plugin_test.go b/plugins/data/json/plugin_test.go new file mode 100644 index 00000000..75d80471 --- /dev/null +++ b/plugins/data/json/plugin_test.go @@ -0,0 +1,103 @@ +package json + +import ( + "testing" + + "github.com/blackstork-io/fabric/plugininterface/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/zclconf/go-cty/cty" +) + +func TestPlugin_GetPlugins(t *testing.T) { + plugin := Plugin{} + plugins := plugin.GetPlugins() + require.Len(t, plugins, 1, "expected 1 plugin") + got := plugins[0] + assert.Equal(t, "json", got.Name) + assert.Equal(t, "data", got.Kind) + assert.Equal(t, "blackstork", got.Namespace) + assert.Equal(t, Version.String(), got.Version.Cast().String()) + assert.Nil(t, got.ConfigSpec) + assert.NotNil(t, got.InvocationSpec) +} + +func TestPlugin_Call(t *testing.T) { + tt := []struct { + name string + glob string + expected plugininterface.Result + }{ + { + name: "empty_list", + glob: "unknown_dir/*.json", + expected: plugininterface.Result{ + Result: []any{}, + }, + }, + { + name: "one_file", + glob: "testdata/a.json", + expected: plugininterface.Result{ + Result: []any{ + map[string]any{ + "filename": "testdata/a.json", + "contents": map[string]any{ + "property_for": "a.json", + }, + }, + }, + }, + }, + { + name: "dir", + glob: "testdata/dir/*.json", + expected: plugininterface.Result{ + Result: []any{ + map[string]any{ + "filename": "testdata/dir/b.json", + "contents": []any{ + map[string]any{ + "id": float64(1), + "property_for": "dir/b.json", + }, + map[string]any{ + "id": float64(2), + "property_for": "dir/b.json", + }, + }, + }, + map[string]any{ + "filename": "testdata/dir/c.json", + "contents": []any{ + map[string]any{ + "id": float64(3), + "property_for": "dir/c.json", + }, + map[string]any{ + "id": float64(4), + "property_for": "dir/c.json", + }, + }, + }, + }, + }, + }, + } + + for _, tc := range tt { + t.Run(tc.name, func(t *testing.T) { + plugin := Plugin{} + args := plugininterface.Args{ + Kind: "data", + Name: "json", + Args: cty.ObjectVal(map[string]cty.Value{ + "glob": cty.StringVal(tc.glob), + }), + } + got := plugin.Call(args) + assert.Equal(t, tc.expected, got) + }) + } + +} diff --git a/plugins/data/json/read.go b/plugins/data/json/read.go new file mode 100644 index 00000000..abe66e19 --- /dev/null +++ b/plugins/data/json/read.go @@ -0,0 +1,47 @@ +package json + +import ( + "encoding/json" + "io/fs" +) + +// JSONDocument represents a JSON document that was read from the filesystem +type JSONDocument struct { + Filename string `json:"filename"` + Contents json.RawMessage `json:"contents"` +} + +func (doc JSONDocument) Map() map[string]any { + var result any + _ = json.Unmarshal(doc.Contents, &result) + return map[string]any{ + "filename": doc.Filename, + "contents": result, + } +} + +// readFS reads all JSON documents from the filesystem that match the given glob pattern +// The pattern is relative to the root of the filesystem +func readFS(filesystem fs.FS, pattern string) ([]JSONDocument, error) { + matchers, err := fs.Glob(filesystem, pattern) + if err != nil { + return nil, err + } + result := []JSONDocument{} + for _, matcher := range matchers { + file, err := filesystem.Open(matcher) + if err != nil { + return nil, err + } + var contents json.RawMessage + err = json.NewDecoder(file).Decode(&contents) + if err != nil { + return nil, err + } + result = append(result, JSONDocument{ + Filename: matcher, + Contents: contents, + }) + } + return result, nil +} diff --git a/plugins/data/json/read_test.go b/plugins/data/json/read_test.go new file mode 100644 index 00000000..408a5268 --- /dev/null +++ b/plugins/data/json/read_test.go @@ -0,0 +1,273 @@ +package json + +import ( + "encoding/json" + "path/filepath" + "reflect" + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_readFS_Structures(t *testing.T) { + // test cases + tt := []struct { + name string + files []string + dirs []string + pattern string + expected []JSONDocument + }{ + { + name: "empty", + files: []string{}, + dirs: []string{}, + pattern: "*.json", + expected: []JSONDocument{}, + }, + { + name: "one_file", + files: []string{"a.json"}, + dirs: []string{}, + pattern: "*.json", + expected: []JSONDocument{ + { + Filename: "a.json", + Contents: testJSON(map[string]any{ + "property_for": "a.json", + }), + }, + }, + }, + { + name: "two_files", + files: []string{"a.json", "b.json"}, + dirs: []string{}, + pattern: "*.json", + expected: []JSONDocument{ + { + Filename: "a.json", + Contents: testJSON(map[string]any{ + "property_for": "a.json", + }), + }, + { + Filename: "b.json", + Contents: testJSON(map[string]any{ + "property_for": "b.json", + }), + }, + }, + }, + { + name: "one_file_in_one_dir", + files: []string{"dir/a.json"}, + dirs: []string{"dir"}, + pattern: "dir/*.json", + expected: []JSONDocument{ + { + Filename: "dir/a.json", + Contents: testJSON(map[string]any{ + "property_for": "dir/a.json", + }), + }, + }, + }, + { + name: "one_file_in_two_dirs", + files: []string{"dir1/a.json", "dir2/a.json"}, + dirs: []string{"dir1", "dir2"}, + pattern: "*/a.json", + expected: []JSONDocument{ + { + Filename: "dir1/a.json", + Contents: testJSON(map[string]any{ + "property_for": "dir1/a.json", + }), + }, + { + Filename: "dir2/a.json", + Contents: testJSON(map[string]any{ + "property_for": "dir2/a.json", + }), + }, + }, + }, + { + name: "two_files_in_one_dir", + files: []string{"dir/a.json", "dir/b.json"}, + dirs: []string{"dir"}, + pattern: "dir/*.json", + expected: []JSONDocument{ + { + Filename: "dir/a.json", + Contents: testJSON(map[string]any{ + "property_for": "dir/a.json", + }), + }, + { + Filename: "dir/b.json", + Contents: testJSON(map[string]any{ + "property_for": "dir/b.json", + }), + }, + }, + }, + } + + for _, tc := range tt { + t.Run(tc.name, func(t *testing.T) { + tmpfs := makeTestFS(t) + for _, file := range tc.files { + data := testJSON(map[string]any{ + "property_for": file, + }) + assert.NoError(t, tmpfs.MkdirAll(filepath.Dir(file), 0o764)) + assert.NoError(t, tmpfs.WriteFile(file, []byte(data), 0o654)) + } + for _, dir := range tc.dirs { + assert.NoError(t, tmpfs.MkdirAll(dir, 0o764)) + } + result, err := readFS(tmpfs, tc.pattern) + assert.NoError(t, err) + assert.Equal(t, tc.expected, result) + }) + } +} + +func Test_readFS_Errors(t *testing.T) { + tt := []struct { + name string + files []string + dirs []string + pattern string + preparefn func(filename string) json.RawMessage + }{ + { + name: "pattern_error", + files: []string{}, + dirs: []string{}, + pattern: "[", + }, + { + name: "file_error", + files: []string{"a.json"}, + dirs: []string{}, + pattern: "*.json", + preparefn: func(filename string) json.RawMessage { + return []byte("not json") + }, + }, + } + + for _, tc := range tt { + t.Run(tc.name, func(t *testing.T) { + tmpfs := makeTestFS(t) + for _, file := range tc.files { + data := tc.preparefn(file) + assert.NoError(t, tmpfs.MkdirAll(filepath.Dir(file), 0o764)) + assert.NoError(t, tmpfs.WriteFile(file, []byte(data), 0o654)) + } + for _, dir := range tc.dirs { + assert.NoError(t, tmpfs.MkdirAll(dir, 0o764)) + } + _, err := readFS(tmpfs, tc.pattern) + assert.Error(t, err) + }) + } +} + +func TestJSONDocument_Map(t *testing.T) { + type fields struct { + Filename string + Contents json.RawMessage + } + tt := []struct { + name string + fields fields + want map[string]any + }{ + { + name: "simple", + fields: fields{ + Filename: "a.json", + Contents: testJSON(map[string]any{ + "property_for": "a.json", + }), + }, + want: map[string]any{ + "filename": "a.json", + "contents": map[string]any{ + "property_for": "a.json", + }, + }, + }, + { + name: "complex", + fields: fields{ + Filename: "a.json", + Contents: testJSON(map[string]any{ + "property_for": "a.json", + "nested": map[string]any{ + "property_for": "a.json", + "nested": map[string]any{ + "property_for": "a.json", + }, + }, + }), + }, + want: map[string]any{ + "filename": "a.json", + "contents": map[string]any{ + "property_for": "a.json", + "nested": map[string]any{ + "property_for": "a.json", + "nested": map[string]any{ + "property_for": "a.json", + }, + }, + }, + }, + }, + { + name: "array", + fields: fields{ + Filename: "a.json", + Contents: testJSON([]any{ + map[string]any{ + "id": float64(0), + "property_for": "a.json", + }, + map[string]any{ + "id": float64(1), + "property_for": "a.json", + }, + }), + }, + want: map[string]any{ + "filename": "a.json", + "contents": []any{ + map[string]any{ + "id": float64(0), + "property_for": "a.json", + }, + map[string]any{ + "id": float64(1), + "property_for": "a.json", + }, + }, + }, + }, + } + for _, tt := range tt { + t.Run(tt.name, func(t *testing.T) { + doc := JSONDocument{ + Filename: tt.fields.Filename, + Contents: tt.fields.Contents, + } + if got := doc.Map(); !reflect.DeepEqual(got, tt.want) { + t.Errorf("JSONDocument.Map() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/plugins/data/json/testdata/a.json b/plugins/data/json/testdata/a.json new file mode 100644 index 00000000..371fe60e --- /dev/null +++ b/plugins/data/json/testdata/a.json @@ -0,0 +1,3 @@ +{ + "property_for": "a.json" +} \ No newline at end of file diff --git a/plugins/data/json/testdata/dir/b.json b/plugins/data/json/testdata/dir/b.json new file mode 100644 index 00000000..d997d020 --- /dev/null +++ b/plugins/data/json/testdata/dir/b.json @@ -0,0 +1,10 @@ +[ + { + "id": 1, + "property_for": "dir/b.json" + }, + { + "id": 2, + "property_for": "dir/b.json" + } +] \ No newline at end of file diff --git a/plugins/data/json/testdata/dir/c.json b/plugins/data/json/testdata/dir/c.json new file mode 100644 index 00000000..9b6e7e23 --- /dev/null +++ b/plugins/data/json/testdata/dir/c.json @@ -0,0 +1,10 @@ +[ + { + "id": 3, + "property_for": "dir/c.json" + }, + { + "id": 4, + "property_for": "dir/c.json" + } +] \ No newline at end of file