From 0113664f9b7d62767ef6c70ff5591c0ba6333ad8 Mon Sep 17 00:00:00 2001 From: dobarx <111326505+dobarx@users.noreply.github.com> Date: Wed, 17 Jan 2024 14:07:00 +0200 Subject: [PATCH] `data.csv` plugin (#45) --- plugins/data/csv/plugin.go | 90 +++++++++++ plugins/data/csv/plugin_test.go | 194 ++++++++++++++++++++++++ plugins/data/csv/read.go | 53 +++++++ plugins/data/csv/testdata/comma.csv | 4 + plugins/data/csv/testdata/empty.csv | 0 plugins/data/csv/testdata/invalid.csv | 4 + plugins/data/csv/testdata/semicolon.csv | 4 + 7 files changed, 349 insertions(+) create mode 100644 plugins/data/csv/plugin.go create mode 100644 plugins/data/csv/plugin_test.go create mode 100644 plugins/data/csv/read.go create mode 100644 plugins/data/csv/testdata/comma.csv create mode 100644 plugins/data/csv/testdata/empty.csv create mode 100644 plugins/data/csv/testdata/invalid.csv create mode 100644 plugins/data/csv/testdata/semicolon.csv diff --git a/plugins/data/csv/plugin.go b/plugins/data/csv/plugin.go new file mode 100644 index 00000000..40fdb039 --- /dev/null +++ b/plugins/data/csv/plugin.go @@ -0,0 +1,90 @@ +package csv + +import ( + "os" + + "github.com/Masterminds/semver/v3" + "github.com/blackstork-io/fabric/plugininterface/v1" + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/hcldec" + "github.com/zclconf/go-cty/cty" +) + +var Version = semver.MustParse("0.1.0") + +const defaultDelimiter = ',' + +type Plugin struct{} + +func (Plugin) GetPlugins() []plugininterface.Plugin { + return []plugininterface.Plugin{ + { + Namespace: "blackstork", + Kind: "data", + Name: "csv", + Version: plugininterface.Version(*Version), + ConfigSpec: nil, + InvocationSpec: &hcldec.ObjectSpec{ + "path": &hcldec.AttrSpec{ + Name: "path", + Type: cty.String, + Required: true, + }, + "delimiter": &hcldec.AttrSpec{ + Name: "delimiter", + Type: cty.String, + Required: false, + }, + }, + }, + } +} + +func (Plugin) Call(args plugininterface.Args) plugininterface.Result { + path := args.Args.GetAttr("path") + if path.IsNull() || path.AsString() == "" { + return plugininterface.Result{ + Diags: hcl.Diagnostics{{ + Severity: hcl.DiagError, + Summary: "path is required", + }}, + } + } + delim := args.Args.GetAttr("delimiter") + if delim.IsNull() { + delim = cty.StringVal(string(defaultDelimiter)) + } + if len(delim.AsString()) != 1 { + return plugininterface.Result{ + Diags: hcl.Diagnostics{{ + Severity: hcl.DiagError, + Summary: "delimiter must be a single character", + }}, + } + } + delimRune := []rune(delim.AsString())[0] + wd, err := os.Getwd() + if err != nil { + return plugininterface.Result{ + Diags: hcl.Diagnostics{{ + Severity: hcl.DiagError, + Summary: "Failed to get current working directory", + Detail: err.Error(), + }}, + } + } + filesystem := os.DirFS(wd) + data, err := readFS(filesystem, path.AsString(), delimRune) + if err != nil { + return plugininterface.Result{ + Diags: hcl.Diagnostics{{ + Severity: hcl.DiagError, + Summary: "Failed to read csv file", + Detail: err.Error(), + }}, + } + } + return plugininterface.Result{ + Result: data, + } +} diff --git a/plugins/data/csv/plugin_test.go b/plugins/data/csv/plugin_test.go new file mode 100644 index 00000000..4a78d1d9 --- /dev/null +++ b/plugins/data/csv/plugin_test.go @@ -0,0 +1,194 @@ +package csv + +import ( + "testing" + + "github.com/blackstork-io/fabric/plugininterface/v1" + "github.com/hashicorp/hcl/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/zclconf/go-cty/cty" +) + +func TestPlugin_GetPlugins(t *testing.T) { + plugin := Plugin{} + plugins := plugin.GetPlugins() + require.Len(t, plugins, 1, "expected 1 plugin") + got := plugins[0] + assert.Equal(t, "csv", got.Name) + assert.Equal(t, "data", got.Kind) + assert.Equal(t, "blackstork", got.Namespace) + assert.Equal(t, Version.String(), got.Version.Cast().String()) + assert.Nil(t, got.ConfigSpec) + assert.NotNil(t, got.InvocationSpec) +} + +func TestPlugin_Call(t *testing.T) { + tt := []struct { + name string + path string + delimiter string + expected plugininterface.Result + }{ + { + name: "comma_delim", + path: "testdata/comma.csv", + delimiter: ",", + expected: plugininterface.Result{ + Result: []map[string]any{ + { + "id": "b8fa4bb0-6dd4-45ba-96e0-9a182b2b932e", + "active": true, + "name": "Stacey", + "age": int64(26), + "height": float64(1.98), + }, + { + "id": "b0086c49-bcd8-4aae-9f88-4f46b128e709", + "active": false, + "name": "Myriam", + "age": int64(33), + "height": float64(1.81), + }, + { + "id": "a12d2a8c-eebc-42b3-be52-1ab0a2969a81", + "active": true, + "name": "Oralee", + "age": int64(31), + "height": float64(2.23), + }, + }, + }, + }, + { + name: "semicolon_delim", + path: "testdata/semicolon.csv", + delimiter: ";", + expected: plugininterface.Result{ + Result: []map[string]any{ + { + "id": "b8fa4bb0-6dd4-45ba-96e0-9a182b2b932e", + "active": true, + "name": "Stacey", + "age": int64(26), + "height": float64(1.98), + }, + { + "id": "b0086c49-bcd8-4aae-9f88-4f46b128e709", + "active": false, + "name": "Myriam", + "age": int64(33), + "height": float64(1.81), + }, + { + "id": "a12d2a8c-eebc-42b3-be52-1ab0a2969a81", + "active": true, + "name": "Oralee", + "age": int64(31), + "height": float64(2.23), + }, + }, + }, + }, + { + name: "empty_path", + expected: plugininterface.Result{ + Diags: hcl.Diagnostics{{ + Severity: hcl.DiagError, + Summary: "path is required", + }}, + }, + }, + { + name: "invalid_delimiter", + path: "testdata/comma.csv", + delimiter: "abc", + expected: plugininterface.Result{ + Diags: hcl.Diagnostics{{ + Severity: hcl.DiagError, + Summary: "delimiter must be a single character", + }}, + }, + }, + { + name: "default_delimiter", + path: "testdata/comma.csv", + expected: plugininterface.Result{ + Result: []map[string]any{ + { + "id": "b8fa4bb0-6dd4-45ba-96e0-9a182b2b932e", + "active": true, + "name": "Stacey", + "age": int64(26), + "height": float64(1.98), + }, + { + "id": "b0086c49-bcd8-4aae-9f88-4f46b128e709", + "active": false, + "name": "Myriam", + "age": int64(33), + "height": float64(1.81), + }, + { + "id": "a12d2a8c-eebc-42b3-be52-1ab0a2969a81", + "active": true, + "name": "Oralee", + "age": int64(31), + "height": float64(2.23), + }, + }, + }, + }, + { + name: "invalid_path", + path: "testdata/does_not_exist.csv", + expected: plugininterface.Result{ + Diags: hcl.Diagnostics{{ + Severity: hcl.DiagError, + Summary: "Failed to read csv file", + Detail: "open testdata/does_not_exist.csv: no such file or directory", + }}, + }, + }, + + { + name: "invalid_csv", + path: "testdata/invalid.csv", + expected: plugininterface.Result{ + Diags: hcl.Diagnostics{{ + Severity: hcl.DiagError, + Summary: "Failed to read csv file", + Detail: "record on line 2: wrong number of fields", + }}, + }, + }, + { + name: "empty_csv", + path: "testdata/empty.csv", + delimiter: ",", + expected: plugininterface.Result{ + Result: []map[string]any{}, + }, + }, + } + + for _, tc := range tt { + t.Run(tc.name, func(t *testing.T) { + plugin := Plugin{} + delim := cty.StringVal(tc.delimiter) + if tc.delimiter == "" { + delim = cty.NullVal(cty.String) + } + args := plugininterface.Args{ + Kind: "data", + Name: "csv", + Args: cty.ObjectVal(map[string]cty.Value{ + "path": cty.StringVal(tc.path), + "delimiter": delim, + }), + } + got := plugin.Call(args) + assert.Equal(t, tc.expected, got) + }) + } +} diff --git a/plugins/data/csv/read.go b/plugins/data/csv/read.go new file mode 100644 index 00000000..823d6337 --- /dev/null +++ b/plugins/data/csv/read.go @@ -0,0 +1,53 @@ +package csv + +import ( + "encoding/csv" + "encoding/json" + "io/fs" +) + +func readFS(filesystem fs.FS, path string, sep rune) ([]map[string]any, error) { + f, err := filesystem.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + r := csv.NewReader(f) + r.Comma = sep + records, err := r.ReadAll() + if err != nil { + return nil, err + } + if len(records) == 0 { + return []map[string]any{}, nil + } + result := make([]map[string]any, len(records)-1) + headers := records[0] + for i, record := range records[1:] { + result[i] = make(map[string]any, len(headers)) + for j, header := range headers { + if header == "" { + continue + } + if j >= len(record) { + result[i][header] = nil + continue + } + if record[j] == "true" { + result[i][header] = true + } else if record[j] == "false" { + result[i][header] = false + } else { + n := json.Number(record[j]) + if e, err := n.Int64(); err == nil { + result[i][header] = e + } else if f, err := n.Float64(); err == nil { + result[i][header] = f + } else { + result[i][header] = record[j] + } + } + } + } + return result, nil +} diff --git a/plugins/data/csv/testdata/comma.csv b/plugins/data/csv/testdata/comma.csv new file mode 100644 index 00000000..94ecff74 --- /dev/null +++ b/plugins/data/csv/testdata/comma.csv @@ -0,0 +1,4 @@ +id,active,name,age,height +b8fa4bb0-6dd4-45ba-96e0-9a182b2b932e,true,Stacey,26,1.98 +b0086c49-bcd8-4aae-9f88-4f46b128e709,false,Myriam,33,1.81 +a12d2a8c-eebc-42b3-be52-1ab0a2969a81,true,Oralee,31,2.23 \ No newline at end of file diff --git a/plugins/data/csv/testdata/empty.csv b/plugins/data/csv/testdata/empty.csv new file mode 100644 index 00000000..e69de29b diff --git a/plugins/data/csv/testdata/invalid.csv b/plugins/data/csv/testdata/invalid.csv new file mode 100644 index 00000000..813f3b3c --- /dev/null +++ b/plugins/data/csv/testdata/invalid.csv @@ -0,0 +1,4 @@ +id,name,age,height +b8fa4bb0-6dd4-45ba-96e0-9a182b2b932e Stacey,26,1.98 +b0086c49-bcd8-4aae-9f88-4f46b128e709,Myriam,33,1.81, +a12d2a8c-eebc-42b3-be52-1ab0a2969a81,Oralee,31,2.23 \ No newline at end of file diff --git a/plugins/data/csv/testdata/semicolon.csv b/plugins/data/csv/testdata/semicolon.csv new file mode 100644 index 00000000..c19b9385 --- /dev/null +++ b/plugins/data/csv/testdata/semicolon.csv @@ -0,0 +1,4 @@ +id;active;name;age;height +b8fa4bb0-6dd4-45ba-96e0-9a182b2b932e;true;Stacey;26;1.98 +b0086c49-bcd8-4aae-9f88-4f46b128e709;false;Myriam;33;1.81 +a12d2a8c-eebc-42b3-be52-1ab0a2969a81;true;Oralee;31;2.23 \ No newline at end of file