Merge branch 'main' of github.com:johnkerl/miller

johnkerl · Mar 6, 2023 · f528e65 · f528e65
2 parents e984631 + 3b1280d
commit f528e65
Show file tree

Hide file tree

Showing 51 changed files with 667 additions and 57 deletions.
diff --git a/create-release-tarball b/create-release-tarball
@@ -92,6 +92,7 @@ $tar \
   ./internal \
   ./regression_test.go \
   ./man \
-  ./test
+  ./test \
+  ./tools
 
 echo "Wrote   $TGZ_NAME"
diff --git a/docs/src/manpage.md b/docs/src/manpage.md
@@ -191,13 +191,14 @@ MILLER(1)                                                            MILLER(1)
 
 1mVERB LIST0m
        altkv bar bootstrap cat check clean-whitespace count-distinct count
-       count-similar cut decimate fill-down fill-empty filter flatten format-values
-       fraction gap grep group-by group-like having-fields head histogram json-parse
-       json-stringify join label latin1-to-utf8 least-frequent merge-fields
-       most-frequent nest nothing put regularize remove-empty-columns rename reorder
-       repeat reshape sample sec2gmtdate sec2gmt seqgen shuffle skip-trivial-records
-       sort sort-within-records split stats1 stats2 step summary tac tail tee
-       template top utf8-to-latin1 unflatten uniq unspace unsparsify
+       count-similar cut decimate downcase fill-down fill-empty filter flatten
+       format-values fraction gap grep group-by group-like having-fields head
+       histogram json-parse json-stringify join label latin1-to-utf8 least-frequent
+       merge-fields most-frequent nest nothing put regularize remove-empty-columns
+       rename reorder repeat reshape sample sec2gmtdate sec2gmt seqgen shuffle
+       skip-trivial-records sort sort-within-records split stats1 stats2 step summary
+       tac tail tee template top utf8-to-latin1 unflatten uniq unspace unsparsify
+       upcase
 
 1mFUNCTION LIST0m
        abs acos acosh any append apply arrayify asin asinh asserting_absent
@@ -1004,6 +1005,14 @@ MILLER(1)                                                            MILLER(1)
         -n {n} Decimation factor (default 10).
        -h|--help Show this message.
 
+   1mdowncase0m
+       Usage: mlr downcase [options]
+       Lowercases strings in record keys and/or values.
+       Options:
+       -k        Downcase only keys, not keys and values.
+       -v        Downcase only values, not keys and values.
+       -h|--help Show this message.
+
    1mfill-down0m
        Usage: mlr fill-down [options]
        If a given record has a missing value for a given field, fill that from
@@ -2113,6 +2122,14 @@ MILLER(1)                                                            MILLER(1)
        being 'b=3,c=4', then the output is the two records 'a=1,b=2,c=' and
        'a=,b=3,c=4'.
 
+   1mupcase0m
+       Usage: mlr upcase [options]
+       Uppercases strings in record keys and/or values.
+       Options:
+       -k        Upcase only keys, not keys and values.
+       -v        Upcase only values, not keys and values.
+       -h|--help Show this message.
+
 1mFUNCTIONS FOR FILTER/PUT0m
    1mabs0m
         (class=math #args=1) Absolute value.

diff --git a/docs/src/manpage.txt b/docs/src/manpage.txt
@@ -170,13 +170,14 @@ MILLER(1)                                                            MILLER(1)
 
 1mVERB LIST0m
        altkv bar bootstrap cat check clean-whitespace count-distinct count
-       count-similar cut decimate fill-down fill-empty filter flatten format-values
-       fraction gap grep group-by group-like having-fields head histogram json-parse
-       json-stringify join label latin1-to-utf8 least-frequent merge-fields
-       most-frequent nest nothing put regularize remove-empty-columns rename reorder
-       repeat reshape sample sec2gmtdate sec2gmt seqgen shuffle skip-trivial-records
-       sort sort-within-records split stats1 stats2 step summary tac tail tee
-       template top utf8-to-latin1 unflatten uniq unspace unsparsify
+       count-similar cut decimate downcase fill-down fill-empty filter flatten
+       format-values fraction gap grep group-by group-like having-fields head
+       histogram json-parse json-stringify join label latin1-to-utf8 least-frequent
+       merge-fields most-frequent nest nothing put regularize remove-empty-columns
+       rename reorder repeat reshape sample sec2gmtdate sec2gmt seqgen shuffle
+       skip-trivial-records sort sort-within-records split stats1 stats2 step summary
+       tac tail tee template top utf8-to-latin1 unflatten uniq unspace unsparsify
+       upcase
 
 1mFUNCTION LIST0m
        abs acos acosh any append apply arrayify asin asinh asserting_absent
@@ -983,6 +984,14 @@ MILLER(1)                                                            MILLER(1)
         -n {n} Decimation factor (default 10).
        -h|--help Show this message.
 
+   1mdowncase0m
+       Usage: mlr downcase [options]
+       Lowercases strings in record keys and/or values.
+       Options:
+       -k        Downcase only keys, not keys and values.
+       -v        Downcase only values, not keys and values.
+       -h|--help Show this message.
+
    1mfill-down0m
        Usage: mlr fill-down [options]
        If a given record has a missing value for a given field, fill that from
@@ -2092,6 +2101,14 @@ MILLER(1)                                                            MILLER(1)
        being 'b=3,c=4', then the output is the two records 'a=1,b=2,c=' and
        'a=,b=3,c=4'.
 
+   1mupcase0m
+       Usage: mlr upcase [options]
+       Uppercases strings in record keys and/or values.
+       Options:
+       -k        Upcase only keys, not keys and values.
+       -v        Upcase only values, not keys and values.
+       -h|--help Show this message.
+
 1mFUNCTIONS FOR FILTER/PUT0m
    1mabs0m
         (class=math #args=1) Absolute value.

diff --git a/docs/src/reference-verbs.md b/docs/src/reference-verbs.md
@@ -824,6 +824,20 @@ Options:
 -h|--help Show this message.
 </pre>
 
+## downcase
+
+<pre class="pre-highlight-in-pair">
+<b>mlr downcase --help</b>
+</pre>
+<pre class="pre-non-highlight-in-pair">
+Usage: mlr downcase [options]
+Lowercases strings in record keys and/or values.
+Options:
+-k        Downcase only keys, not keys and values.
+-v        Downcase only values, not keys and values.
+-h|--help Show this message.
+</pre>
+
 ## fill-down
 
 <pre class="pre-highlight-in-pair">
@@ -4271,3 +4285,16 @@ a b v u w x
 - - 1 - 2 -
 </pre>
 
+## upcase
+
+<pre class="pre-highlight-in-pair">
+<b>mlr upcase --help</b>
+</pre>
+<pre class="pre-non-highlight-in-pair">
+Usage: mlr upcase [options]
+Uppercases strings in record keys and/or values.
+Options:
+-k        Upcase only keys, not keys and values.
+-v        Upcase only values, not keys and values.
+-h|--help Show this message.
+</pre>
diff --git a/docs/src/reference-verbs.md.in b/docs/src/reference-verbs.md.in
@@ -297,6 +297,12 @@ GENMD-RUN-COMMAND
 mlr decimate --help
 GENMD-EOF
 
+## downcase
+
+GENMD-RUN-COMMAND
+mlr downcase --help
+GENMD-EOF
+
 ## fill-down
 
 GENMD-RUN-COMMAND
@@ -1289,3 +1295,8 @@ GENMD-RUN-COMMAND
 mlr --ijson --opprint unsparsify -f a,b,u,v,w,x then regularize data/sparse.json
 GENMD-EOF
 
+## upcase
+
+GENMD-RUN-COMMAND
+mlr upcase --help
+GENMD-EOF
diff --git a/go.mod b/go.mod
@@ -18,7 +18,6 @@ go 1.15
 
 require (
 	github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb
-	github.com/goccmack/gocc v0.0.0-20230228185258-2292f9e40198 // indirect
 	github.com/johnkerl/lumin v1.0.0
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
 	github.com/lestrrat-go/strftime v1.0.6

diff --git a/go.sum b/go.sum
@@ -8,8 +8,6 @@ github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb h1:IT4JYU7k4ikYg1S
 github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb/go.mod h1:bH6Xx7IW64qjjJq8M2u4dxNaBiDfKK+z/3eGDpXEQhc=
 github.com/felixge/fgprof v0.9.3 h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g=
 github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNun8eiPw=
-github.com/goccmack/gocc v0.0.0-20230228185258-2292f9e40198 h1:FSii2UQeSLngl3jFoR4tUKZLprO7qUlh/TKKticc0BM=
-github.com/goccmack/gocc v0.0.0-20230228185258-2292f9e40198/go.mod h1:DTh/Y2+NbnOVVoypCCQrovMPDKUGp4yZpSbWg5D0XIM=
 github.com/google/pprof v0.0.0-20211214055906-6f57359322fd h1:1FjCyPC+syAzJ5/2S8fqdZK1R22vvA0J7JZKcuOIQ7Y=
 github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg=
 github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
@@ -39,37 +37,12 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
 github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
-golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8=
-golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab h1:2QkjZIsXupsJbJIdSjjUOgWK3aEtzyuh2mPt3l/CkeU=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf h1:MZ2shdL+ZM/XzY3ZGOnh4Nlpnxz5GSOhOmtHo3iPU6M=
-golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
-golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
-golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

diff --git a/internal/pkg/transformers/aaa_transformer_table.go b/internal/pkg/transformers/aaa_transformer_table.go
@@ -22,6 +22,7 @@ var TRANSFORMER_LOOKUP_TABLE = []TransformerSetup{
 	CountSimilarSetup,
 	CutSetup,
 	DecimateSetup,
+	DowncaseSetup,
 	FillDownSetup,
 	FillEmptySetup,
 	FilterSetup,
@@ -75,6 +76,7 @@ var TRANSFORMER_LOOKUP_TABLE = []TransformerSetup{
 	UniqSetup,
 	UnspaceSetup,
 	UnsparsifySetup,
+	UpcaseSetup,
 }
 
 func ShowHelpForTransformer(verb string) bool {