cmd/flux-kvs: add [--json|--raw] options to "put"

Problem:  flux-kvs put presumes values should be stored as JSON,
but the KVS no longer requires this.

Add type options to allow the user to choose how values are stored.

If no options, value is stored as a NULL terminated string.

If --raw, value is stored as with no options, but without a
NULL terminator.  For --raw mode only, key=- may be used to
take read value from stdin.

If --json, value is stored as a NULL terminated string if it
is valid encoded JSON; otherwise it is first encoded as a JSON
string.  This mimics the old default behavior of flux-kvs put
that is expected by many tests in t1000-kvs.t and t1002-kvs-extra.t.

Add --json to flux kvs put where used in various sharness tests.

Partial fix to flux-framework#1159.

src/cmd/flux-kvs.c

@@ -56,6 +56,16 @@ static void dump_kvs_dir (const flux_kvsdir_t *dir, bool Ropt, bool dopt);
 
 #define min(a,b) ((a)<(b)?(a):(b))
 
+static struct optparse_option put_opts[] =  {
+    { .name = "json", .key = 'j', .has_arg = 0,
+      .usage = "Store value(s) as encoded JSON",
+    },
+    { .name = "raw", .key = 'r', .has_arg = 0,
+      .usage = "Store value(s) as-is without adding NULL termination",
+    },
+    OPTPARSE_TABLE_END
+};
+
 static struct optparse_option dir_opts[] =  {
     { .name = "recursive", .key = 'R', .has_arg = 0,
       .usage = "Recursively display keys under subdirectories",
@@ -127,11 +137,11 @@ static struct optparse_subcommand subcommands[] = {
       NULL
     },
     { "put",
-      "key=value [key=value...]",
+      "[-j|-r] key=value [key=value...]",
       "Store value under key",
       cmd_put,
       0,
-      NULL
+      put_opts
     },
     { "dir",
       "[-R] [-d] [key]",
@@ -380,14 +390,33 @@ int cmd_put (optparse_t *p, int argc, char **argv)
             log_msg_exit ("put: you must specify a value as key=value");
         *val++ = '\0';
 
-        json_t *obj;
-        if ((obj = json_loads (val, JSON_DECODE_ANY, NULL))) {
-            if (flux_kvs_txn_put (txn, 0, key, val) < 0)
+        if (optparse_hasopt (p, "json")) {
+            json_t *obj;
+            if ((obj = json_loads (val, JSON_DECODE_ANY, NULL))) {
+                if (flux_kvs_txn_put (txn, 0, key, val) < 0)
+                    log_err_exit ("%s", key);
+                json_decref (obj);
+            }
+            else { // encode as JSON string if not already valid encoded JSON
+                if (flux_kvs_txn_pack (txn, 0, key, "s", val) < 0)
+                    log_err_exit ("%s", key);
+            }
+        }
+        else if (optparse_hasopt (p, "raw")) {
+            int len;
+            uint8_t *buf = NULL;
+            if (!strcmp (val, "-")) { // special handling for "--raw key=-"
+                if ((len = read_all (STDIN_FILENO, &buf)) < 0)
+                    log_err_exit ("stdin");
+                val = (char *)buf;
+            } else
+                len = strlen (val);
+            if (flux_kvs_txn_put_raw (txn, 0, key, val, len) < 0)
                 log_err_exit ("%s", key);
-            json_decref (obj);
+            free (buf);
         }
-        else { // encode as JSON string if not already valid encoded JSON
-            if (flux_kvs_txn_pack (txn, 0, key, "s", val) < 0)
+        else {
+            if (flux_kvs_txn_put (txn, 0, key, val) < 0)
                 log_err_exit ("%s", key);
         }
         free (key);

t/issues/t0441-kvs-put-get.sh

@@ -3,7 +3,7 @@
 
 TEST=issue441
 
-flux kvs put ${TEST}.x=foo
+flux kvs put --json ${TEST}.x=foo
 
 flux kvs get ${TEST}.x.y && test $? -eq 1
 

t/issues/t0821-kvs-segfault.sh

@@ -2,6 +2,6 @@
 # kvs put test="large string", get test.x fails without panic
 
 TEST=issue0821
-flux kvs put ${TEST}="xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+flux kvs put --json ${TEST}="xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
 flux kvs get ${TEST}.x && test $? -eq 1
 flux kvs get ${TEST}   # fails if broker died

t/lua/t1002-kvs.t

@@ -196,7 +196,7 @@ type_ok (kw, 'userdata', "f:kvswatcher returns kvswatcher object")
 kw.testkey = "foo"
 is (kw.testkey, 'foo', "Can set arbitrary members of kvswatcher object")
 
-os.execute (string.format ("flux kvs put %s=%s", data.key, data.value))
+os.execute (string.format ("flux kvs put --json %s=%s", data.key, data.value))
 
 local to = f:timer {
     timeout = 1500,
@@ -247,15 +247,15 @@ local t, err = f:timer {
 }
 
 -- Excute on rank 3 via flux-exec:
-os.execute (string.format ("sleep 0.25 && flux exec -r 3 flux kvs put %s=%s",
+os.execute (string.format ("sleep 0.25 && flux exec -r 3 flux kvs put --json %s=%s",
     data.key, data.value))
 local r, err = f:reactor()
 isnt (r, -1, "reactor exited normally with ".. (r and r or err))
 is (ncount, 2, "kvswatch callback invoked exactly twice")
 
 note ("Ensure kvs watch callback not invoked after kvswatcher removal")
 ok (kw:remove(), "Can remove kvswatcher without error")
-os.execute (string.format ("sleep 0.25 && flux exec -r 3 flux kvs put %s=%s",
+os.execute (string.format ("sleep 0.25 && flux exec -r 3 flux kvs put --json %s=%s",
     data.key, 'test3'))
 
 local t, err = f:timer {

t/t0017-security.t

@@ -279,7 +279,7 @@ test_expect_success 'connector delivers kvs.setroot event to owner connection' '
 	    $SHARNESS_TEST_SRCDIR/scripts/event-trace-bypass.lua \
 		kvs kvs.test.end \
                 "flux event pub kvs.test.a; \
-                 flux kvs put ev9=42; \
+                 flux kvs put --json ev9=42; \
                  flux event pub kvs.test.end" >ev9.out &&
 	grep -q kvs.setroot ev9.out
 '
@@ -289,7 +289,7 @@ test_expect_success 'dispatcher delivers kvs.setroot event to owner connection'
 	    $SHARNESS_TEST_SRCDIR/scripts/event-trace.lua \
 		kvs kvs.test.end \
                 "flux event pub kvs.test.a; \
-                 flux kvs put ev10=42; \
+                 flux kvs put --json ev10=42; \
                  flux event pub kvs.test.end" >ev10.out &&
 	grep -q kvs.setroot ev10.out
 '
@@ -300,7 +300,7 @@ test_expect_success 'connector suppresses kvs.setroot event to guest connection'
 	    $SHARNESS_TEST_SRCDIR/scripts/event-trace-bypass.lua \
 		kvs kvs.test.end \
                 "flux event pub kvs.test.a; \
-                 flux kvs put ev11=42; \
+                 flux kvs put --json ev11=42; \
                  flux event pub kvs.test.end" >ev11.out &&
 	! grep -q kvs.setroot ev11.out
 '