data.csv plugin (#45)

plugins/data/csv/plugin.go

@@ -0,0 +1,90 @@
+package csv
+
+import (
+	"os"
+
+	"github.com/Masterminds/semver/v3"
+	"github.com/blackstork-io/fabric/plugininterface/v1"
+	"github.com/hashicorp/hcl/v2"
+	"github.com/hashicorp/hcl/v2/hcldec"
+	"github.com/zclconf/go-cty/cty"
+)
+
+var Version = semver.MustParse("0.1.0")
+
+const defaultDelimiter = ','
+
+type Plugin struct{}
+
+func (Plugin) GetPlugins() []plugininterface.Plugin {
+	return []plugininterface.Plugin{
+		{
+			Namespace:  "blackstork",
+			Kind:       "data",
+			Name:       "csv",
+			Version:    plugininterface.Version(*Version),
+			ConfigSpec: nil,
+			InvocationSpec: &hcldec.ObjectSpec{
+				"path": &hcldec.AttrSpec{
+					Name:     "path",
+					Type:     cty.String,
+					Required: true,
+				},
+				"delimiter": &hcldec.AttrSpec{
+					Name:     "delimiter",
+					Type:     cty.String,
+					Required: false,
+				},
+			},
+		},
+	}
+}
+
+func (Plugin) Call(args plugininterface.Args) plugininterface.Result {
+	path := args.Args.GetAttr("path")
+	if path.IsNull() || path.AsString() == "" {
+		return plugininterface.Result{
+			Diags: hcl.Diagnostics{{
+				Severity: hcl.DiagError,
+				Summary:  "path is required",
+			}},
+		}
+	}
+	delim := args.Args.GetAttr("delimiter")
+	if delim.IsNull() {
+		delim = cty.StringVal(string(defaultDelimiter))
+	}
+	if len(delim.AsString()) != 1 {
+		return plugininterface.Result{
+			Diags: hcl.Diagnostics{{
+				Severity: hcl.DiagError,
+				Summary:  "delimiter must be a single character",
+			}},
+		}
+	}
+	delimRune := []rune(delim.AsString())[0]
+	wd, err := os.Getwd()
+	if err != nil {
+		return plugininterface.Result{
+			Diags: hcl.Diagnostics{{
+				Severity: hcl.DiagError,
+				Summary:  "Failed to get current working directory",
+				Detail:   err.Error(),
+			}},
+		}
+	}
+	filesystem := os.DirFS(wd)
+	data, err := readFS(filesystem, path.AsString(), delimRune)
+	if err != nil {
+		return plugininterface.Result{
+			Diags: hcl.Diagnostics{{
+				Severity: hcl.DiagError,
+				Summary:  "Failed to read csv file",
+				Detail:   err.Error(),
+			}},
+		}
+	}
+	return plugininterface.Result{
+		Result: data,
+	}
+}

plugins/data/csv/plugin_test.go

@@ -0,0 +1,194 @@
+package csv
+
+import (
+	"testing"
+
+	"github.com/blackstork-io/fabric/plugininterface/v1"
+	"github.com/hashicorp/hcl/v2"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/zclconf/go-cty/cty"
+)
+
+func TestPlugin_GetPlugins(t *testing.T) {
+	plugin := Plugin{}
+	plugins := plugin.GetPlugins()
+	require.Len(t, plugins, 1, "expected 1 plugin")
+	got := plugins[0]
+	assert.Equal(t, "csv", got.Name)
+	assert.Equal(t, "data", got.Kind)
+	assert.Equal(t, "blackstork", got.Namespace)
+	assert.Equal(t, Version.String(), got.Version.Cast().String())
+	assert.Nil(t, got.ConfigSpec)
+	assert.NotNil(t, got.InvocationSpec)
+}
+
+func TestPlugin_Call(t *testing.T) {
+	tt := []struct {
+		name      string
+		path      string
+		delimiter string
+		expected  plugininterface.Result
+	}{
+		{
+			name:      "comma_delim",
+			path:      "testdata/comma.csv",
+			delimiter: ",",
+			expected: plugininterface.Result{
+				Result: []map[string]any{
+					{
+						"id":     "b8fa4bb0-6dd4-45ba-96e0-9a182b2b932e",
+						"active": true,
+						"name":   "Stacey",
+						"age":    int64(26),
+						"height": float64(1.98),
+					},
+					{
+						"id":     "b0086c49-bcd8-4aae-9f88-4f46b128e709",
+						"active": false,
+						"name":   "Myriam",
+						"age":    int64(33),
+						"height": float64(1.81),
+					},
+					{
+						"id":     "a12d2a8c-eebc-42b3-be52-1ab0a2969a81",
+						"active": true,
+						"name":   "Oralee",
+						"age":    int64(31),
+						"height": float64(2.23),
+					},
+				},
+			},
+		},
+		{
+			name:      "semicolon_delim",
+			path:      "testdata/semicolon.csv",
+			delimiter: ";",
+			expected: plugininterface.Result{
+				Result: []map[string]any{
+					{
+						"id":     "b8fa4bb0-6dd4-45ba-96e0-9a182b2b932e",
+						"active": true,
+						"name":   "Stacey",
+						"age":    int64(26),
+						"height": float64(1.98),
+					},
+					{
+						"id":     "b0086c49-bcd8-4aae-9f88-4f46b128e709",
+						"active": false,
+						"name":   "Myriam",
+						"age":    int64(33),
+						"height": float64(1.81),
+					},
+					{
+						"id":     "a12d2a8c-eebc-42b3-be52-1ab0a2969a81",
+						"active": true,
+						"name":   "Oralee",
+						"age":    int64(31),
+						"height": float64(2.23),
+					},
+				},
+			},
+		},
+		{
+			name: "empty_path",
+			expected: plugininterface.Result{
+				Diags: hcl.Diagnostics{{
+					Severity: hcl.DiagError,
+					Summary:  "path is required",
+				}},
+			},
+		},
+		{
+			name:      "invalid_delimiter",
+			path:      "testdata/comma.csv",
+			delimiter: "abc",
+			expected: plugininterface.Result{
+				Diags: hcl.Diagnostics{{
+					Severity: hcl.DiagError,
+					Summary:  "delimiter must be a single character",
+				}},
+			},
+		},
+		{
+			name: "default_delimiter",
+			path: "testdata/comma.csv",
+			expected: plugininterface.Result{
+				Result: []map[string]any{
+					{
+						"id":     "b8fa4bb0-6dd4-45ba-96e0-9a182b2b932e",
+						"active": true,
+						"name":   "Stacey",
+						"age":    int64(26),
+						"height": float64(1.98),
+					},
+					{
+						"id":     "b0086c49-bcd8-4aae-9f88-4f46b128e709",
+						"active": false,
+						"name":   "Myriam",
+						"age":    int64(33),
+						"height": float64(1.81),
+					},
+					{
+						"id":     "a12d2a8c-eebc-42b3-be52-1ab0a2969a81",
+						"active": true,
+						"name":   "Oralee",
+						"age":    int64(31),
+						"height": float64(2.23),
+					},
+				},
+			},
+		},
+		{
+			name: "invalid_path",
+			path: "testdata/does_not_exist.csv",
+			expected: plugininterface.Result{
+				Diags: hcl.Diagnostics{{
+					Severity: hcl.DiagError,
+					Summary:  "Failed to read csv file",
+					Detail:   "open testdata/does_not_exist.csv: no such file or directory",
+				}},
+			},
+		},
+
+		{
+			name: "invalid_csv",
+			path: "testdata/invalid.csv",
+			expected: plugininterface.Result{
+				Diags: hcl.Diagnostics{{
+					Severity: hcl.DiagError,
+					Summary:  "Failed to read csv file",
+					Detail:   "record on line 2: wrong number of fields",
+				}},
+			},
+		},
+		{
+			name:      "empty_csv",
+			path:      "testdata/empty.csv",
+			delimiter: ",",
+			expected: plugininterface.Result{
+				Result: []map[string]any{},
+			},
+		},
+	}
+
+	for _, tc := range tt {
+		t.Run(tc.name, func(t *testing.T) {
+			plugin := Plugin{}
+			delim := cty.StringVal(tc.delimiter)
+			if tc.delimiter == "" {
+				delim = cty.NullVal(cty.String)
+			}
+			args := plugininterface.Args{
+				Kind: "data",
+				Name: "csv",
+				Args: cty.ObjectVal(map[string]cty.Value{
+					"path":      cty.StringVal(tc.path),
+					"delimiter": delim,
+				}),
+			}
+			got := plugin.Call(args)
+			assert.Equal(t, tc.expected, got)
+		})
+	}
+}

plugins/data/csv/read.go

@@ -0,0 +1,53 @@
+package csv
+
+import (
+	"encoding/csv"
+	"encoding/json"
+	"io/fs"
+)
+
+func readFS(filesystem fs.FS, path string, sep rune) ([]map[string]any, error) {
+	f, err := filesystem.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	r := csv.NewReader(f)
+	r.Comma = sep
+	records, err := r.ReadAll()
+	if err != nil {
+		return nil, err
+	}
+	if len(records) == 0 {
+		return []map[string]any{}, nil
+	}
+	result := make([]map[string]any, len(records)-1)
+	headers := records[0]
+	for i, record := range records[1:] {
+		result[i] = make(map[string]any, len(headers))
+		for j, header := range headers {
+			if header == "" {
+				continue
+			}
+			if j >= len(record) {
+				result[i][header] = nil
+				continue
+			}
+			if record[j] == "true" {
+				result[i][header] = true
+			} else if record[j] == "false" {
+				result[i][header] = false
+			} else {
+				n := json.Number(record[j])
+				if e, err := n.Int64(); err == nil {
+					result[i][header] = e
+				} else if f, err := n.Float64(); err == nil {
+					result[i][header] = f
+				} else {
+					result[i][header] = record[j]
+				}
+			}
+		}
+	}
+	return result, nil
+}

plugins/data/csv/testdata/comma.csv

@@ -0,0 +1,4 @@
+id,active,name,age,height
+b8fa4bb0-6dd4-45ba-96e0-9a182b2b932e,true,Stacey,26,1.98
+b0086c49-bcd8-4aae-9f88-4f46b128e709,false,Myriam,33,1.81
+a12d2a8c-eebc-42b3-be52-1ab0a2969a81,true,Oralee,31,2.23

plugins/data/csv/testdata/invalid.csv

@@ -0,0 +1,4 @@
+id,name,age,height
+b8fa4bb0-6dd4-45ba-96e0-9a182b2b932e Stacey,26,1.98
+b0086c49-bcd8-4aae-9f88-4f46b128e709,Myriam,33,1.81,
+a12d2a8c-eebc-42b3-be52-1ab0a2969a81,Oralee,31,2.23

plugins/data/csv/testdata/semicolon.csv

@@ -0,0 +1,4 @@
+id;active;name;age;height
+b8fa4bb0-6dd4-45ba-96e0-9a182b2b932e;true;Stacey;26;1.98
+b0086c49-bcd8-4aae-9f88-4f46b128e709;false;Myriam;33;1.81
+a12d2a8c-eebc-42b3-be52-1ab0a2969a81;true;Oralee;31;2.23