diff --git a/cmd/sizes/main.go b/cmd/sizes/main.go index bddb090821..20ff09e272 100644 --- a/cmd/sizes/main.go +++ b/cmd/sizes/main.go @@ -7,6 +7,8 @@ package main import ( + "fmt" + "github.com/johnkerl/miller/internal/pkg/mlrval" ) @@ -15,5 +17,6 @@ func main() { mvs[0] = *mlrval.FromString("hello") mvs[1] = *mlrval.FromString("world") mvs[0].ShowSizes() + fmt.Println() mvs[1].ShowSizes() } diff --git a/internal/pkg/mlrval/mlrmap_accessors.go b/internal/pkg/mlrval/mlrmap_accessors.go index cd37dd1b08..7e5fa99783 100644 --- a/internal/pkg/mlrval/mlrmap_accessors.go +++ b/internal/pkg/mlrval/mlrmap_accessors.go @@ -339,7 +339,7 @@ func (mlrmap *Mlrmap) getWithMlrvalArrayIndex(index *Mlrval) (*Mlrval, error) { current := mlrmap var retval *Mlrval = nil lib.InternalCodingErrorIf(!index.IsArray()) - array := index.arrayval + array := index.x.arrayval n := len(array) for i, piece := range array { next, err := current.GetWithMlrvalIndex(piece) @@ -350,7 +350,7 @@ func (mlrmap *Mlrmap) getWithMlrvalArrayIndex(index *Mlrval) (*Mlrval, error) { if !next.IsMap() { return nil, fmt.Errorf("mlr: cannot multi-index non-map.") } - current = next.mapval + current = next.x.mapval } else { retval = next.Copy() } @@ -784,7 +784,7 @@ func (mlrmap *Mlrmap) SortByKeyRecursively() { // Old record will be GC'ed: just move pointers value := mlrmap.Get(key) if value.IsMap() { - value.mapval.SortByKeyRecursively() + value.x.mapval.SortByKeyRecursively() } other.PutReference(key, value) } diff --git a/internal/pkg/mlrval/mlrmap_flatten_unflatten.go b/internal/pkg/mlrval/mlrmap_flatten_unflatten.go index 307bcc26a4..140e672864 100644 --- a/internal/pkg/mlrval/mlrmap_flatten_unflatten.go +++ b/internal/pkg/mlrval/mlrmap_flatten_unflatten.go @@ -137,13 +137,13 @@ func (mlrmap *Mlrmap) CopyUnflattened( // Is the field name something dot something? if strings.Contains(pe.Key, separator) { arrayOfIndices := SplitAXHelper(pe.Key, separator) - lib.InternalCodingErrorIf(len(arrayOfIndices.arrayval) < 1) + lib.InternalCodingErrorIf(len(arrayOfIndices.x.arrayval) < 1) // If the input field name was "x.a" then remember the "x". - baseIndex := arrayOfIndices.arrayval[0].String() + baseIndex := arrayOfIndices.x.arrayval[0].String() affectedBaseIndices[baseIndex] = true // Use PutIndexed to assign $x["a"] = 7, or $x["b"] = 8, etc. other.PutIndexed( - CopyMlrvalArray(arrayOfIndices.arrayval), + CopyMlrvalArray(arrayOfIndices.x.arrayval), unflattenTerminal(pe.Value).Copy(), ) } else { @@ -187,13 +187,13 @@ func (mlrmap *Mlrmap) CopyUnflattenFields( // Is the field name something dot something? if strings.Contains(pe.Key, separator) { arrayOfIndices := SplitAXHelper(pe.Key, separator) - lib.InternalCodingErrorIf(len(arrayOfIndices.arrayval) < 1) + lib.InternalCodingErrorIf(len(arrayOfIndices.x.arrayval) < 1) // If the input field name was "x.a" then remember the "x". - baseIndex := arrayOfIndices.arrayval[0].String() + baseIndex := arrayOfIndices.x.arrayval[0].String() if fieldNameSet[baseIndex] { // Use PutIndexed to assign $x["a"] = 7, or $x["b"] = 8, etc. other.PutIndexed( - CopyMlrvalArray(arrayOfIndices.arrayval), + CopyMlrvalArray(arrayOfIndices.x.arrayval), unflattenTerminal(pe.Value).Copy(), ) affectedBaseIndices[baseIndex] = true @@ -247,7 +247,7 @@ func SplitAXHelper(input string, separator string) *Mlrval { output := FromArray(make([]*Mlrval, len(fields))) for i, field := range fields { - output.arrayval[i] = FromString(field) + output.x.arrayval[i] = FromString(field) } return output diff --git a/internal/pkg/mlrval/mlrval_accessors.go b/internal/pkg/mlrval/mlrval_accessors.go index d0ce1a8b03..18ffb77cc0 100644 --- a/internal/pkg/mlrval/mlrval_accessors.go +++ b/internal/pkg/mlrval/mlrval_accessors.go @@ -8,7 +8,7 @@ import ( func (mv *Mlrval) GetArrayLength() (int, bool) { if mv.IsArray() { - return len(mv.arrayval), true + return len(mv.x.arrayval), true } else { return -999, false } @@ -35,13 +35,13 @@ func (mv *Mlrval) FlattenToMap(prefix string, delimiter string) Mlrval { if mv.IsMap() { // Without this, the for-loop below is zero-pass and fields with "{}" // values would disappear entirely in a JSON-to-CSV conversion. - if mv.mapval.IsEmpty() { + if mv.x.mapval.IsEmpty() { if prefix != "" { retval.PutCopy(prefix, FromString("{}")) } } - for pe := mv.mapval.Head; pe != nil; pe = pe.Next { + for pe := mv.x.mapval.Head; pe != nil; pe = pe.Next { nextPrefix := pe.Key if prefix != "" { nextPrefix = prefix + delimiter + nextPrefix @@ -49,7 +49,7 @@ func (mv *Mlrval) FlattenToMap(prefix string, delimiter string) Mlrval { if pe.Value.IsMap() || pe.Value.IsArray() { nextResult := pe.Value.FlattenToMap(nextPrefix, delimiter) lib.InternalCodingErrorIf(nextResult.mvtype != MT_MAP) - for pf := nextResult.mapval.Head; pf != nil; pf = pf.Next { + for pf := nextResult.x.mapval.Head; pf != nil; pf = pf.Next { retval.PutCopy(pf.Key, pf.Value.Copy()) } } else { @@ -60,13 +60,13 @@ func (mv *Mlrval) FlattenToMap(prefix string, delimiter string) Mlrval { } else if mv.IsArray() { // Without this, the for-loop below is zero-pass and fields with "[]" // values would disappear entirely in a JSON-to-CSV conversion. - if len(mv.arrayval) == 0 { + if len(mv.x.arrayval) == 0 { if prefix != "" { retval.PutCopy(prefix, FromString("[]")) } } - for zindex, value := range mv.arrayval { + for zindex, value := range mv.x.arrayval { nextPrefix := strconv.Itoa(zindex + 1) // Miller user-space indices are 1-up if prefix != "" { nextPrefix = prefix + delimiter + nextPrefix @@ -74,7 +74,7 @@ func (mv *Mlrval) FlattenToMap(prefix string, delimiter string) Mlrval { if value.IsMap() || value.IsArray() { nextResult := value.FlattenToMap(nextPrefix, delimiter) lib.InternalCodingErrorIf(nextResult.mvtype != MT_MAP) - for pf := nextResult.mapval.Head; pf != nil; pf = pf.Next { + for pf := nextResult.x.mapval.Head; pf != nil; pf = pf.Next { retval.PutCopy(pf.Key, pf.Value.Copy()) } } else { diff --git a/internal/pkg/mlrval/mlrval_collections.go b/internal/pkg/mlrval/mlrval_collections.go index eaab23b5f5..f76d35ff37 100644 --- a/internal/pkg/mlrval/mlrval_collections.go +++ b/internal/pkg/mlrval/mlrval_collections.go @@ -86,7 +86,7 @@ func (mv *Mlrval) ArrayGet(mindex *Mlrval) Mlrval { if !mindex.IsInt() { return *ERROR } - value := arrayGetAliased(&mv.arrayval, int(mindex.intval)) + value := arrayGetAliased(&mv.x.arrayval, int(mindex.intval)) if value == nil { return *ABSENT } else { @@ -116,12 +116,12 @@ func (mv *Mlrval) ArrayPut(mindex *Mlrval, value *Mlrval) { os.Exit(1) } - ok := arrayPutAliased(&mv.arrayval, int(mindex.intval), value) + ok := arrayPutAliased(&mv.x.arrayval, int(mindex.intval), value) if !ok { fmt.Fprintf( os.Stderr, "mlr: array index %d out of bounds %d..%d\n", - mindex.intval, 1, len(mv.arrayval), + mindex.intval, 1, len(mv.x.arrayval), ) os.Exit(1) } @@ -213,7 +213,7 @@ func (mv *Mlrval) ArrayAppend(value *Mlrval) { // Silent no-ops are not good UX ... return } - mv.arrayval = append(mv.arrayval, value) + mv.x.arrayval = append(mv.x.arrayval, value) } // ================================================================ @@ -222,7 +222,7 @@ func (mv *Mlrval) MapGet(key *Mlrval) Mlrval { return *ERROR } - mval, err := mv.mapval.GetWithMlrvalIndex(key) + mval, err := mv.x.mapval.GetWithMlrvalIndex(key) if err != nil { // xxx maybe error-return in the API return *ERROR } @@ -243,9 +243,9 @@ func (mv *Mlrval) MapPut(key *Mlrval, value *Mlrval) { } if key.IsString() { - mv.mapval.PutCopy(key.printrep, value) + mv.x.mapval.PutCopy(key.printrep, value) } else if key.IsInt() { - mv.mapval.PutCopy(key.String(), value) + mv.x.mapval.PutCopy(key.String(), value) } // TODO: need to be careful about semantics here. // Silent no-ops are not good UX ... @@ -291,19 +291,19 @@ func (mv *Mlrval) PutIndexed(indices []*Mlrval, rvalue *Mlrval) error { lib.InternalCodingErrorIf(len(indices) < 1) if mv.IsMap() { - return putIndexedOnMap(mv.mapval, indices, rvalue) + return putIndexedOnMap(mv.x.mapval, indices, rvalue) } else if mv.IsArray() { - return putIndexedOnArray(&mv.arrayval, indices, rvalue) + return putIndexedOnArray(&mv.x.arrayval, indices, rvalue) } else { baseIndex := indices[0] if baseIndex.IsString() { *mv = *FromEmptyMap() - return putIndexedOnMap(mv.mapval, indices, rvalue) + return putIndexedOnMap(mv.x.mapval, indices, rvalue) } else if baseIndex.IsInt() { *mv = *FromEmptyArray() - return putIndexedOnArray(&mv.arrayval, indices, rvalue) + return putIndexedOnArray(&mv.x.arrayval, indices, rvalue) } else { return errors.New( "mlr: only maps and arrays are indexable; got " + mv.GetTypeName(), @@ -326,7 +326,7 @@ func putIndexedOnMap(baseMap *Mlrmap, indices []*Mlrval, rvalue *Mlrval) error { ".", ) } - *baseMap = *rvalue.mapval.Copy() + *baseMap = *rvalue.x.mapval.Copy() return nil } @@ -438,10 +438,10 @@ func (mv *Mlrval) RemoveIndexed(indices []*Mlrval) error { lib.InternalCodingErrorIf(len(indices) < 1) if mv.IsMap() { - return removeIndexedOnMap(mv.mapval, indices) + return removeIndexedOnMap(mv.x.mapval, indices) } else if mv.IsArray() { - return removeIndexedOnArray(&mv.arrayval, indices) + return removeIndexedOnArray(&mv.x.arrayval, indices) } else { return errors.New( @@ -659,13 +659,13 @@ func NewMlrvalForAutoDeepen(mvtype MVType) (*Mlrval, error) { func (mv *Mlrval) Arrayify() *Mlrval { if mv.IsMap() { - if mv.mapval.IsEmpty() { + if mv.x.mapval.IsEmpty() { return mv } convertible := true i := 0 - for pe := mv.mapval.Head; pe != nil; pe = pe.Next { + for pe := mv.x.mapval.Head; pe != nil; pe = pe.Next { sval := strconv.Itoa(i + 1) // Miller user-space indices are 1-up i++ if pe.Key != sval { @@ -675,9 +675,9 @@ func (mv *Mlrval) Arrayify() *Mlrval { } if convertible { - arrayval := make([]*Mlrval, mv.mapval.FieldCount) + arrayval := make([]*Mlrval, mv.x.mapval.FieldCount) i := 0 - for pe := mv.mapval.Head; pe != nil; pe = pe.Next { + for pe := mv.x.mapval.Head; pe != nil; pe = pe.Next { arrayval[i] = pe.Value.Copy() i++ } @@ -690,8 +690,8 @@ func (mv *Mlrval) Arrayify() *Mlrval { } else if mv.IsArray() { // TODO: comment (or rethink) that this modifies its inputs!! output := mv.Copy() - for i := range mv.arrayval { - output.arrayval[i] = output.arrayval[i].Arrayify() + for i := range mv.x.arrayval { + output.x.arrayval[i] = output.x.arrayval[i].Arrayify() } return output diff --git a/internal/pkg/mlrval/mlrval_copy.go b/internal/pkg/mlrval/mlrval_copy.go index 115882c4ed..7d1067e893 100644 --- a/internal/pkg/mlrval/mlrval_copy.go +++ b/internal/pkg/mlrval/mlrval_copy.go @@ -4,9 +4,17 @@ package mlrval func (mv *Mlrval) Copy() *Mlrval { other := *mv if mv.mvtype == MT_MAP { - other.mapval = mv.mapval.Copy() + other.x = &mlrvalExtended{ + mapval: mv.x.mapval.Copy(), + } } else if mv.mvtype == MT_ARRAY { - other.arrayval = CopyMlrvalArray(mv.arrayval) + other.x = &mlrvalExtended{ + arrayval: CopyMlrvalArray(mv.x.arrayval), + } + } else if mv.mvtype == MT_FUNC { + other.x = &mlrvalExtended{ + funcval: mv.x.funcval, + } } return &other } diff --git a/internal/pkg/mlrval/mlrval_get.go b/internal/pkg/mlrval/mlrval_get.go index 038e8933cd..9cd2fd4db9 100644 --- a/internal/pkg/mlrval/mlrval_get.go +++ b/internal/pkg/mlrval/mlrval_get.go @@ -65,7 +65,7 @@ func (mv *Mlrval) GetBoolValue() (boolValue bool, isBool bool) { func (mv *Mlrval) GetArray() []*Mlrval { if mv.IsArray() { - return mv.arrayval + return mv.x.arrayval } else { return nil } @@ -73,7 +73,7 @@ func (mv *Mlrval) GetArray() []*Mlrval { func (mv *Mlrval) GetMap() *Mlrmap { if mv.IsMap() { - return mv.mapval + return mv.x.mapval } else { return nil } @@ -81,7 +81,7 @@ func (mv *Mlrval) GetMap() *Mlrmap { func (mv *Mlrval) GetFunction() interface{} { if mv.Type() == MT_FUNC { - return mv.funcval + return mv.x.funcval } else { return nil } @@ -122,12 +122,12 @@ func (mv *Mlrval) AcquireBoolValue() bool { func (mv *Mlrval) AcquireArrayValue() []*Mlrval { lib.InternalCodingErrorIf(mv.mvtype != MT_ARRAY) - return mv.arrayval + return mv.x.arrayval } func (mv *Mlrval) AcquireMapValue() *Mlrmap { lib.InternalCodingErrorIf(mv.mvtype != MT_MAP) - return mv.mapval + return mv.x.mapval } func (mv *Mlrval) GetNumericToFloatValueOrDie() (floatValue float64) { diff --git a/internal/pkg/mlrval/mlrval_json.go b/internal/pkg/mlrval/mlrval_json.go index f3982edccd..9395057f45 100644 --- a/internal/pkg/mlrval/mlrval_json.go +++ b/internal/pkg/mlrval/mlrval_json.go @@ -411,7 +411,7 @@ func (mv *Mlrval) marshalJSONArray( // TODO: libify allTerminal := true - for _, element := range mv.arrayval { + for _, element := range mv.x.arrayval { if element.IsArrayOrMap() { allTerminal = false break @@ -429,11 +429,11 @@ func (mv *Mlrval) marshalJSONArraySingleLine( elementNestingDepth int, outputIsStdout bool, ) (string, error) { - n := len(mv.arrayval) + n := len(mv.x.arrayval) var buffer bytes.Buffer buffer.WriteByte('[') - for i, element := range mv.arrayval { + for i, element := range mv.x.arrayval { elementString, err := element.marshalJSONAux(JSON_SINGLE_LINE, elementNestingDepth+1, outputIsStdout) if err != nil { return "", err @@ -466,7 +466,7 @@ func (mv *Mlrval) marshalJSONArrayMultipleLines( elementNestingDepth int, outputIsStdout bool, ) (string, error) { - n := len(mv.arrayval) + n := len(mv.x.arrayval) var buffer bytes.Buffer // Write empty array as '[]' @@ -475,7 +475,7 @@ func (mv *Mlrval) marshalJSONArrayMultipleLines( buffer.WriteByte('\n') } - for i, element := range mv.arrayval { + for i, element := range mv.x.arrayval { elementString, err := element.marshalJSONAux(jsonFormatting, elementNestingDepth+1, outputIsStdout) if err != nil { return "", err @@ -508,7 +508,7 @@ func (mv *Mlrval) marshalJSONMap( outputIsStdout bool, ) (string, error) { lib.InternalCodingErrorIf(mv.mvtype != MT_MAP) - s, err := mv.mapval.marshalJSONAux(jsonFormatting, elementNestingDepth, outputIsStdout) + s, err := mv.x.mapval.marshalJSONAux(jsonFormatting, elementNestingDepth, outputIsStdout) if err != nil { return "", err } diff --git a/internal/pkg/mlrval/mlrval_new.go b/internal/pkg/mlrval/mlrval_new.go index d1c12fb611..d5a4a932a2 100644 --- a/internal/pkg/mlrval/mlrval_new.go +++ b/internal/pkg/mlrval/mlrval_new.go @@ -209,7 +209,9 @@ func FromFunction(funcval interface{}, name string) *Mlrval { mvtype: MT_FUNC, printrep: name, printrepValid: true, - funcval: funcval, + x: &mlrvalExtended{ + funcval: funcval, + }, } } @@ -218,7 +220,9 @@ func FromArray(arrayval []*Mlrval) *Mlrval { mvtype: MT_ARRAY, printrep: "(bug-if-you-see-this:case-4)", // INVALID_PRINTREP, printrepValid: false, - arrayval: CopyMlrvalArray(arrayval), + x: &mlrvalExtended{ + arrayval: CopyMlrvalArray(arrayval), + }, } } @@ -231,7 +235,9 @@ func FromMap(mapval *Mlrmap) *Mlrval { mvtype: MT_MAP, printrep: "(bug-if-you-see-this:case-5)", // INVALID_PRINTREP, printrepValid: false, - mapval: mapval.Copy(), + x: &mlrvalExtended{ + mapval: mapval.Copy(), + }, } } diff --git a/internal/pkg/mlrval/mlrval_new_test.go b/internal/pkg/mlrval/mlrval_new_test.go index 05aebada46..b7f21a7dbc 100644 --- a/internal/pkg/mlrval/mlrval_new_test.go +++ b/internal/pkg/mlrval/mlrval_new_test.go @@ -124,17 +124,17 @@ func TestFromFunction(t *testing.T) { mv := FromFunction("test data", "f001") assert.Equal(t, MT_FUNC, mv.mvtype) assert.True(t, mv.printrepValid) - assert.Equal(t, "test data", mv.funcval.(string)) + assert.Equal(t, "test data", mv.x.funcval.(string)) } func TestFromArray(t *testing.T) { mv := FromArray([]*Mlrval{FromInt(10)}) assert.Equal(t, MT_ARRAY, mv.mvtype) - assert.Equal(t, 1, len(mv.arrayval)) + assert.Equal(t, 1, len(mv.x.arrayval)) } func TestFromMap(t *testing.T) { mv := FromMap(NewMlrmap()) assert.Equal(t, MT_MAP, mv.mvtype) - assert.True(t, mv.mapval.IsEmpty()) + assert.True(t, mv.x.mapval.IsEmpty()) } diff --git a/internal/pkg/mlrval/mlrval_output.go b/internal/pkg/mlrval/mlrval_output.go index 4d67573401..078b61f44d 100644 --- a/internal/pkg/mlrval/mlrval_output.go +++ b/internal/pkg/mlrval/mlrval_output.go @@ -110,12 +110,12 @@ func (mv *Mlrval) StringifyValuesRecursively() { switch mv.mvtype { case MT_ARRAY: - for i, _ := range mv.arrayval { - mv.arrayval[i].StringifyValuesRecursively() + for i, _ := range mv.x.arrayval { + mv.x.arrayval[i].StringifyValuesRecursively() } case MT_MAP: - for pe := mv.mapval.Head; pe != nil; pe = pe.Next { + for pe := mv.x.mapval.Head; pe != nil; pe = pe.Next { pe.Value.StringifyValuesRecursively() } @@ -129,13 +129,17 @@ func (mv *Mlrval) ShowSizes() { fmt.Printf("mv.intval %p %d\n", &mv.intval, reflect.TypeOf(mv.intval).Size()) fmt.Printf("mv.floatval %p %d\n", &mv.floatval, reflect.TypeOf(mv.floatval).Size()) fmt.Printf("mv.printrep %p %d\n", &mv.printrep, reflect.TypeOf(mv.printrep).Size()) + + fmt.Printf("mv.x %p %d\n", &mv.mvtype, reflect.TypeOf(mv.x).Size()) + if mv.x != nil { + fmt.Printf("mv.x.arrayval %p %d\n", &mv.x.arrayval, reflect.TypeOf(mv.x.arrayval).Size()) + fmt.Printf("mv.x.mapval %p %d\n", &mv.x.mapval, reflect.TypeOf(mv.x.mapval).Size()) + if mv.x.funcval != nil { + fmt.Printf("mv.x.funcval %p %d\n", &mv.x.funcval, reflect.TypeOf(mv.x.funcval).Size()) + } + } + fmt.Printf("mv.printrepValid %p %d\n", &mv.printrepValid, reflect.TypeOf(mv.printrepValid).Size()) fmt.Printf("mv.boolval %p %d\n", &mv.boolval, reflect.TypeOf(mv.boolval).Size()) fmt.Printf("mv.mvtype %p %d\n", &mv.mvtype, reflect.TypeOf(mv.mvtype).Size()) - - fmt.Printf("mv.arrayval %p %d\n", &mv.arrayval, reflect.TypeOf(mv.arrayval).Size()) - fmt.Printf("mv.mapval %p %d\n", &mv.mapval, reflect.TypeOf(mv.mapval).Size()) - if mv.funcval != nil { - fmt.Printf("mv.funcval %p %d\n", &mv.funcval, reflect.TypeOf(mv.funcval).Size()) - } } diff --git a/internal/pkg/mlrval/mlrval_type.go b/internal/pkg/mlrval/mlrval_type.go index a752360a07..84a3f33aef 100644 --- a/internal/pkg/mlrval/mlrval_type.go +++ b/internal/pkg/mlrval/mlrval_type.go @@ -54,16 +54,23 @@ package mlrval type Mlrval struct { - intval int64 - floatval float64 - printrep string + intval int64 + floatval float64 + printrep string + + x *mlrvalExtended + printrepValid bool boolval bool // Enumeration for string / int / float / boolean / etc. // I would call this "type" not "mvtype" but "type" is a keyword in Go. mvtype MVType +} +// The Mlrval type is a (non-union) compound type where arrayval, mapval, and funcval are (a) +// largish, and (b) not usually used. +type mlrvalExtended struct { arrayval []*Mlrval mapval *Mlrmap // First-class-function literals from internal/pkg/dsl/cst.