new datatype: xml + improvements to pretty

lmorg · Sep 23, 2024 · 9c719eb · 9c719eb
1 parent 65d870c
commit 9c719eb
Show file tree

Hide file tree

Showing 53 changed files with 714 additions and 91 deletions.
diff --git a/app/app.go b/app/app.go
@@ -16,9 +16,9 @@ const Name = "murex"
 const (
 	Major     = 6
 	Minor     = 3
-	Revision  = 4247
-	Branch    = "website"
-	BuildDate = "2024-09-23  16:17:32"
+	Revision  = 4277
+	Branch    = "xml-parser"
+	BuildDate = "2024-09-23  23:03:48"
 )
 
 // Copyright is the copyright owner string

diff --git a/builtins/core.go b/builtins/core.go
@@ -70,6 +70,7 @@ import (
 	_ "github.com/lmorg/murex/builtins/types/querystring" // encoding values as URL query strings
 	_ "github.com/lmorg/murex/builtins/types/string"      // string data type
 	_ "github.com/lmorg/murex/builtins/types/toml"        // TOML data type
+	_ "github.com/lmorg/murex/builtins/types/xml"         // XML data type
 	_ "github.com/lmorg/murex/builtins/types/yaml"        // YAML data type
 
 	_ "github.com/lmorg/murex/builtins/optional" // optional plugins

diff --git a/builtins/core/pretty/method_test.go b/builtins/core/pretty/method_test.go
@@ -0,0 +1,26 @@
+package pretty
+
+import (
+	"testing"
+
+	_ "github.com/lmorg/murex/builtins/types/generic"
+	_ "github.com/lmorg/murex/builtins/types/json"
+	"github.com/lmorg/murex/lang/types"
+	"github.com/lmorg/murex/test"
+)
+
+func TestPrettyDefault(t *testing.T) {
+	input := `{"foo":"bar"}`
+	output := "{\n    \"foo\": \"bar\"\n}"
+
+	test.RunMethodTest(t, cmdPretty, "pretty", input, types.Json, []string{}, output, nil)
+	test.RunMethodTest(t, cmdPretty, "pretty", input, types.Generic, []string{}, output, nil)
+}
+
+func TestPrettyStrict(t *testing.T) {
+	input := `{"foo":"bar"}`
+	output := "{\n    \"foo\": \"bar\"\n}"
+
+	test.RunMethodTest(t, cmdPretty, "pretty", input, types.Json, []string{"--strict"}, output, nil)
+	test.RunMethodTest(t, cmdPretty, "pretty", input, types.Generic, []string{"--strict"}, input, nil)
+}
diff --git a/builtins/core/pretty/pretty.go b/builtins/core/pretty/pretty.go
@@ -3,8 +3,10 @@ package pretty
 import (
 	"bytes"
 	"encoding/json"
+	"fmt"
 	"io"
 
+	"github.com/lmorg/murex/builtins/types/xml"
 	"github.com/lmorg/murex/config/defaults"
 	"github.com/lmorg/murex/lang"
 	"github.com/lmorg/murex/lang/parameters"
@@ -15,40 +17,50 @@ func init() {
 	//lang.GoFunctions["pretty"] = cmdPretty
 	lang.DefineMethod("pretty", cmdPretty, types.Json, types.Json)
 
-	defaults.AppendProfile(`
-		autocomplete: set pretty { [{
-			"Flags": [ "--strict" ]
-		}] }
-	`)
+	defaults.AppendProfile(fmt.Sprintf(`
+		autocomplete set pretty %%[{
+			Flags: [ %[1]s %[2]s ]
+			FlagValues: {
+				%[2]s: [{ Flags: [ json xml ] }]
+			}
+		}]
+	`, fStrict, fDataType))
 }
 
+const (
+	fStrict   = "--strict"
+	fDataType = "--type"
+)
+
 func cmdPretty(p *lang.Process) error {
 	if err := p.ErrIfNotAMethod(); err != nil {
 		return err
 	}
 
 	flags, _, err := p.Parameters.ParseFlags(&parameters.Arguments{
 		Flags: map[string]string{
-			"--strict": "bool",
+			fStrict:   types.Boolean,
+			fDataType: types.String,
 		},
 	})
 	if err != nil {
 		return err
 	}
 
 	switch {
-	case flags["--strict"] == types.TrueString:
-		return prettyStrict(p)
+	case flags[fDataType] != "":
+		return prettyType(p, flags[fDataType], flags[fStrict] == types.TrueString)
+
 	default:
-		return prettyDefault(p)
+		return prettyType(p, p.Stdin.GetDataType(), flags[fStrict] == types.TrueString)
 	}
 }
 
-func prettyStrict(p *lang.Process) error {
-	dt := p.Stdin.GetDataType()
+func prettyType(p *lang.Process, dt string, strict bool) error {
 	p.Stdout.SetDataType(dt)
 
-	if dt == types.Json {
+	switch dt {
+	case types.Json:
 		b, err := p.Stdin.ReadAll()
 		if err != nil {
 			return err
@@ -62,26 +74,36 @@ func prettyStrict(p *lang.Process) error {
 
 		_, err = p.Stdout.Write(prettyJSON.Bytes())
 		return err
-	}
 
-	_, err := io.Copy(p.Stdout, p.Stdin)
-	return err
-}
+	case "xml":
+		v, err := xml.UnmarshalFromProcess(p)
+		if err != nil {
+			return err
+		}
+		b, err := xml.MarshalTTY(v, true)
+		if err != nil {
+			return err
+		}
+		_, err = p.Stdout.Write(b)
+		return err
 
-func prettyDefault(p *lang.Process) error {
-	p.Stdout.SetDataType(types.Json)
+	case types.Generic:
+		if !strict {
+			var err error
+			err = prettyType(p, types.Json, false)
+			if err == nil {
+				return nil
+			}
+			err = prettyType(p, "xml", false)
+			if err == nil {
+				return nil
+			}
+		}
 
-	b, err := p.Stdin.ReadAll()
-	if err != nil {
-		return err
-	}
+		fallthrough
 
-	var prettyJSON bytes.Buffer
-	err = json.Indent(&prettyJSON, b, "", "    ")
-	if err != nil {
+	default:
+		_, err := io.Copy(p.Stdout, p.Stdin)
 		return err
 	}
-
-	_, err = p.Stdout.Write(prettyJSON.Bytes())
-	return err
 }
diff --git a/builtins/core/pretty/pretty_doc.yaml b/builtins/core/pretty/pretty_doc.yaml
@@ -5,30 +5,34 @@
   SubCategoryIDs:
     - commands.str
   Summary: >-
-    Prettifies JSON to make it human readable
+    Prettifies data documents to make it human readable
   Description: |-
-    Takes JSON from the stdin and reformats it to make it human readable, then
+    Takes JSON or XML from the stdin and reformats it to make it human readable, then
     outputs that to stdout.
   Usage: |-
     ```
-    <stdin> -> pretty -> <stdout>
+    <stdin> -> [ --strict | --type (XML|JSON) ] -> <stdout>
     ```
   Examples: |-
     ```
-    » tout json {"Array":[1,2,3],"Map":{"String": "Foobar","Number":123.456}} -> pretty 
+    » %{Array:[1,2,3],Map:{String:Foobar,Number:123.456}} -> pretty 
     {
         "Array": [
             1,
             2,
             3
         ],
         "Map": {
-            "String": "Foobar",
-            "Number": 123.456
+            "Number": 123.456,
+            "String": "Foobar"
         }
     }
     ```
   Flags:
+    "--strict": >-
+      If data type doesn't have a pretty parser, then just output stdin (default behaviour is to try every parser until one works)
+    "--type": >-
+      Specify a pretty parser (supported values: "json", "xml")
   Detail:
   Synonyms:
     - pretty

diff --git a/builtins/core/pretty/pretty_test.go b/builtins/core/pretty/pretty_test.go
@@ -1,26 +1,52 @@
-package pretty
+package pretty_test
 
 import (
+	"fmt"
+	"strings"
 	"testing"
 
-	_ "github.com/lmorg/murex/builtins/types/generic"
-	_ "github.com/lmorg/murex/builtins/types/json"
-	"github.com/lmorg/murex/lang/types"
+	_ "github.com/lmorg/murex/builtins"
 	"github.com/lmorg/murex/test"
 )
 
-func TestPrettyDefault(t *testing.T) {
-	input := `{"foo":"bar"}`
-	output := "{\n    \"foo\": \"bar\"\n}"
+const (
+	svgPretty = `
+<svg aria-label="Version: 6.3.4247" height="20" role="img" width="122" xlink="http://www.w3.org/1999/xlink" xmlns="http://www.w3.org/2000/svg">
+    <clipPath id="r">
+        <rect fill="#fff" height="20" rx="3" width="122"/>
+    </clipPath>
+    <g clip-path="url(#r)">
+        <rect fill="#555" height="20" width="51"/>
+        <rect fill="#2ea44f" height="20" width="71" x="51"/>
+        <rect fill="url(#s)" height="20" width="122"/>
+    </g>
+    <g fill="#fff" font-family="Verdana,Geneva,DejaVu Sans,sans-serif" font-size="110" text-anchor="middle" text-rendering="geometricPrecision">
+        <text aria-hidden="true" fill="#010101" fill-opacity="0.3" textLength="410" transform="scale(.1)" x="265" y="150">Version</text>
+        <text fill="#fff" textLength="410" transform="scale(.1)" x="265" y="140">Version</text>
+        <text aria-hidden="true" fill="#010101" fill-opacity="0.3" textLength="610" transform="scale(.1)" x="855" y="150">6.3.4247</text>
+        <text fill="#fff" textLength="610" transform="scale(.1)" x="855" y="140">6.3.4247</text>
+    </g>
+    <linearGradient id="s" x2="0" y2="100%">
+        <stop offset="0" stop-color="#bbb" stop-opacity="0.1"/>
+        <stop offset="1" stop-opacity="0.1"/>
+    </linearGradient>
+    <title>Version: 6.3.4247</title>
+</svg>`
 
-	test.RunMethodTest(t, cmdPretty, "pretty", input, types.Json, []string{}, output, nil)
-	test.RunMethodTest(t, cmdPretty, "pretty", input, types.Generic, []string{}, output, nil)
-}
+	svgMinified = `<svg aria-label="Version: 6.3.4247" height="20" role="img" width="122" xlink="http://www.w3.org/1999/xlink" xmlns="http://www.w3.org/2000/svg"><clipPath id="r"><rect fill="#fff" height="20" rx="3" width="122"/></clipPath><g clip-path="url(#r)"><rect fill="#555" height="20" width="51"/><rect fill="#2ea44f" height="20" width="71" x="51"/><rect fill="url(#s)" height="20" width="122"/></g><g fill="#fff" font-family="Verdana,Geneva,DejaVu Sans,sans-serif" font-size="110" text-anchor="middle" text-rendering="geometricPrecision"><text aria-hidden="true" fill="#010101" fill-opacity="0.3" textLength="410" transform="scale(.1)" x="265" y="150">Version</text><text fill="#fff" textLength="410" transform="scale(.1)" x="265" y="140">Version</text><text aria-hidden="true" fill="#010101" fill-opacity="0.3" textLength="610" transform="scale(.1)" x="855" y="150">6.3.4247</text><text fill="#fff" textLength="610" transform="scale(.1)" x="855" y="140">6.3.4247</text></g><linearGradient id="s" x2="0" y2="100%"><stop offset="0" stop-color="#bbb" stop-opacity="0.1"/><stop offset="1" stop-opacity="0.1"/></linearGradient><title>Version: 6.3.4247</title></svg>`
+)
 
-func TestPrettyStrict(t *testing.T) {
-	input := `{"foo":"bar"}`
-	output := "{\n    \"foo\": \"bar\"\n}"
+func TestMarshallers(t *testing.T) {
+	tests := []test.MurexTest{
+		{
+			Block:  fmt.Sprintf(`%%(%s) -> :xml: format json -> format xml`, svgMinified),
+			Stdout: strings.TrimSpace(svgMinified),
+		},
+		{
+			Block:  fmt.Sprintf(`%%(%s) -> :xml: format json -> format xml -> pretty`, svgMinified),
+			Stdout: strings.TrimSpace(svgPretty),
+		},
+	}
 
-	test.RunMethodTest(t, cmdPretty, "pretty", input, types.Json, []string{"--strict"}, output, nil)
-	test.RunMethodTest(t, cmdPretty, "pretty", input, types.Generic, []string{"--strict"}, input, nil)
+	test.RunMurexTests(tests, t)
 }
diff --git a/builtins/docs/summaries.go b/builtins/docs/summaries.go
@@ -93,7 +93,7 @@ func init() {
 		"trypipe":               "Checks for non-zero exits of each function in a pipeline",
 		"post":                  "HTTP POST request with a JSON-parsable return",
 		"prepend":               "Add data to the start of an array",
-		"pretty":                "Prettifies JSON to make it human readable",
+		"pretty":                "Prettifies data documents to make it human readable",
 		"struct-keys":           "Outputs all the keys in a structure as a file path",
 		"private":               "Define a private function block",
 		"time":                  "Returns the execution run time of a command or block",
@@ -222,7 +222,7 @@ func init() {
 		"commands/trypipe":                "Checks for non-zero exits of each function in a pipeline",
 		"commands/post":                   "HTTP POST request with a JSON-parsable return",
 		"commands/prepend":                "Add data to the start of an array",
-		"commands/pretty":                 "Prettifies JSON to make it human readable",
+		"commands/pretty":                 "Prettifies data documents to make it human readable",
 		"commands/struct-keys":            "Outputs all the keys in a structure as a file path",
 		"commands/private":                "Define a private function block",
 		"commands/time":                   "Returns the execution run time of a command or block",
@@ -327,6 +327,7 @@ func init() {
 		"types/paths":                     "Structured array for working with `$PATH` style data",
 		"types/str":                       "string (primitive)",
 		"types/toml":                      "Tom's Obvious, Minimal Language (TOML)",
+		"types/xml":                       "Extensible Markup Language (XML) (experimental)",
 		"types/yaml":                      "YAML Ain't Markup Language (YAML)",
 		"types/mxjson":                    "Murex-flavoured JSON (deprecated)",
 		"variables/numeric":               "Variables who's name is a positive integer, eg `0`, `1`, `2`, `3` and above",
@@ -941,6 +942,7 @@ func init() {
 		"types/str":                       "types/str",
 		"types/string":                    "types/str",
 		"types/toml":                      "types/toml",
+		"types/xml":                       "types/xml",
 		"types/yaml":                      "types/yaml",
 		"types/mxjson":                    "types/mxjson",
 		"variables/numeric":               "variables/numeric",

diff --git a/builtins/types/csv/csv.go b/builtins/types/csv/csv.go
@@ -26,6 +26,7 @@ func init() {
 		"application/x-csv",
 		"text/csv",
 		"text/x-csv",
+		"+csv",
 	)
 
 	lang.SetFileExtensions(typeName, "csv")

diff --git a/builtins/types/csv/csv_doc.yaml b/builtins/types/csv/csv_doc.yaml
@@ -49,6 +49,7 @@
       - application/x-csv
       - text/csv
       - text/x-csv
+      - +csv
     Extensions:
       - csv
   Synonyms:

diff --git a/builtins/types/json/json.go b/builtins/types/json/json.go
@@ -23,6 +23,7 @@ func init() {
 		"application/x-json",
 		"text/json",
 		"text/x-json",
+		"+json",
 	)
 	lang.SetFileExtensions(types.Json, "json", "tfstate")
 }
diff --git a/builtins/types/json/json_doc.yaml b/builtins/types/json/json_doc.yaml
@@ -59,6 +59,7 @@
       - application/x-json
       - text/json
       - text/x-json
+      - +json
     Extensions:
       - json
   Related:

diff --git a/builtins/types/sexp/sexp.go b/builtins/types/sexp/sexp.go
@@ -38,7 +38,7 @@ func init() {
 		"text/x-sexp",
 	)
 
-	lang.SetFileExtensions(sexpr, "sexp")
+	lang.SetFileExtensions(sexpr, "sexp", "lisp")
 }
 
 func readIndexC(p *lang.Process, params []string) error { return readIndex(p, params, true) }

diff --git a/builtins/types/toml/toml.go b/builtins/types/toml/toml.go
@@ -30,6 +30,7 @@ func init() {
 		"application/x-toml",
 		"text/toml",
 		"text/x-toml",
+		"+toml",
 	)
 
 	lang.SetFileExtensions(typeName, "toml")

diff --git a/builtins/types/toml/toml_doc.yaml b/builtins/types/toml/toml_doc.yaml
@@ -64,6 +64,7 @@
       - application/x-toml
       - text/toml
       - text/x-toml
+      - +toml
     Extensions:
       - toml
   Related:

diff --git a/builtins/types/xml/array_read.go b/builtins/types/xml/array_read.go
@@ -0,0 +1,17 @@
+package xml
+
+import (
+	"context"
+
+	"github.com/lmorg/murex/lang"
+	"github.com/lmorg/murex/lang/stdio"
+)
+
+func readArray(ctx context.Context, read stdio.Io, callback func([]byte)) error {
+	// Create a marshaller function to pass to ArrayTemplate
+	marshaller := func(v interface{}) ([]byte, error) {
+		return MarshalTTY(v, read.IsTTY())
+	}
+
+	return lang.ArrayTemplate(ctx, marshaller, unmarshaller, read, callback)
+}