Extend transform to include nested JSON objects and arrays

build.sh

@@ -1,7 +1,6 @@
 #!/usr/bin/env bash
 set -e
 
-debug='false'
 fast='false'
 json=''
 runtests='false'
@@ -16,8 +15,7 @@ usage() {
     echo 'Builds the "metadb" executable in the bin directory'
     echo ''
     echo 'Flags:'
-    echo '-d  Build with debug tag'
-    echo '-f  Do not remove executable before compiling'
+    echo '-f  Fast build (do not remove executable before compiling)'
     echo '-h  Help'
     echo '-t  Run tests'
     echo '-T  Run tests and other checks; requires'
@@ -26,15 +24,13 @@ usage() {
     echo '    go install github.com/kisielk/errcheck@latest'
     echo '-v  Enable verbose output'
     # echo '-D  Enable "-tags dynamic" compiler option'
-    # echo '-X  Include experimental code'
+    echo '-X  Build with experimental code included'
 }
 
-while getopts 'cdfhJtvTXD' flag; do
+while getopts 'fhJtvTXD' flag; do
     case "${flag}" in
         t) runtests='true' ;;
         T) runalltests='true' ;;
-        c) ;;
-        d) debug='true' ;;
         f) fast='true' ;;
         J) echo "build.sh: -J option is deprecated" 1>&2 ;;
         h) usage
@@ -66,26 +62,7 @@ if [[ -v METADB_FOLIO ]]; then
     echo "build.sh: using folio reference \"$METADB_FOLIO\"" 1>&2
 fi
 
-if $experiment; then
-    echo "The \"include experimental code\" option (-X) has been selected."
-    read -p "This may prevent later upgrades.  Are you sure? " yn
-    case $yn in
-        [Yy] ) break ;;
-        [Yy][Ee][Ss] ) break ;;
-        * ) echo "Exiting" 1>&2
-            exit 1 ;;
-    esac
-    # json='-X main.rewriteJSON=1'
-    echo "build.sh: experimental code will be included" 1>&2
-fi
-
-bindir=bin
-
-if ! $fast; then
-    rm -f ./$bindir/metadb ./cmd/metadb/parser/gram.go ./cmd/metadb/parser/scan.go ./cmd/metadb/parser/y.output
-fi
-
-version=`git describe --tags --always`
+tags=''
 
 # Check which operating system is running.
 case "$(uname -s)" in
@@ -99,13 +76,37 @@ esac
 #     tags='-tags dynamic'
 # fi
 
-if $debug; then
-    tags='-tags debug'
-    echo "build.sh: building with debug tag" 1>&2
+if $experiment; then
+    # echo "The \"build with experimental code\" option (-X) has been selected."
+    # read -p "This may prevent later upgrades.  Are you sure? " yn
+    # case $yn in
+    #     [Yy] ) break ;;
+    #     [Yy][Ee][Ss] ) break ;;
+    #     * ) echo "Exiting" 1>&2
+    #         exit 1 ;;
+    # esac
+    # json='-X main.rewriteJSON=1'
+    if [ -n "$tags" ]; then
+        tags="${tags},"
+    fi
+    tags="${tags}experimental"
+    echo "build.sh: building with experimental code" 1>&2
+fi
+
+if [ -n "$tags" ]; then
+    tags="-tags $tags"
+fi
+
+bindir=bin
+
+if ! $fast; then
+    rm -f ./$bindir/metadb ./cmd/metadb/parser/gram.go ./cmd/metadb/parser/scan.go ./cmd/metadb/parser/y.output
 fi
 
 mkdir -p $bindir
 
+version=`git describe --tags --always`
+
 go generate $v ./...
 
 go build -o $bindir $v $tags -ldflags "-X github.com/metadb-project/metadb/cmd/metadb/util.MetadbVersion=$version -X github.com/metadb-project/metadb/cmd/metadb/util.FolioVersion=$METADB_FOLIO $json" ./cmd/metadb

cmd/metadb/command/command.go

@@ -176,6 +176,11 @@ func DataTypeToSQL(dtype DataType, typeSize int64) string {
 	}
 }
 
+// CommandGraph is a list of commands in which each command may itself contain a
+// list of sub-commands.  A command that is not a sub-command, in other words not
+// reachable from another command, is called a "root command."  Root commands
+// represent basic change events; sub-commands generally represent transformed
+// records.
 type CommandGraph struct {
 	Commands *list.List
 }
@@ -184,6 +189,47 @@ func NewCommandGraph() *CommandGraph {
 	return &CommandGraph{Commands: list.New()}
 }
 
+func (g *CommandGraph) String() string {
+	var b strings.Builder
+	g.writeCommands(&b, g.Commands, 0)
+	return b.String()
+}
+
+func (g *CommandGraph) writeCommands(b *strings.Builder, commands *list.List, indent int) {
+	if commands == nil {
+		return
+	}
+	for e := commands.Front(); e != nil; e = e.Next() {
+		for i := 0; i < indent; i++ {
+			b.WriteRune(' ')
+		}
+		cmd := *(e.Value.(*Command))
+		fmt.Fprintf(b, "> %s %s.%s\n", cmd.Op, cmd.SchemaName, cmd.TableName)
+		for i := range cmd.Column {
+			for j := 0; j < indent+8; j++ {
+				b.WriteRune(' ')
+			}
+			if cmd.Column[i].PrimaryKey == 0 {
+				b.WriteRune('-')
+			} else {
+				fmt.Fprintf(b, "= [%d]", cmd.Column[i].PrimaryKey)
+			}
+			fmt.Fprintf(b, " %s (%s): ", cmd.Column[i].Name, DataTypeToSQL(cmd.Column[i].DType, cmd.Column[i].DTypeSize))
+			if cmd.Column[i].SQLData == nil {
+				b.WriteString("null")
+			} else {
+				s := *(cmd.Column[i].SQLData)
+				if len(s) > 40 {
+					s = s[:40] + "..."
+				}
+				fmt.Fprintf(b, "%s", s)
+			}
+			b.WriteRune('\n')
+		}
+		g.writeCommands(b, (e.Value.(*Command)).Subcommands, indent+4)
+	}
+}
+
 type Command struct {
 	Op              Operation
 	SchemaName      string

cmd/metadb/config/expoff.go

@@ -0,0 +1,5 @@
+//go:build !experimental
+
+package config
+
+const Experimental bool = false

cmd/metadb/config/expon.go

@@ -0,0 +1,5 @@
+//go:build experimental
+
+package config
+
+const Experimental bool = true