From 9a234079db7c0b66cab8b567c201ffbcf43ca9a5 Mon Sep 17 00:00:00 2001
From: Albert Chu <chu11@llnl.gov>
Date: Thu, 25 Jan 2018 17:49:12 -0800
Subject: [PATCH 1/5] modules/kvs: Add 'owner' to namespace create rpc

In addition, store namespace owner in kvsroot.
---
 src/common/libkvs/kvs.c        |  6 +++++-
 src/modules/kvs/kvs.c          | 23 ++++++++++++++++++-----
 src/modules/kvs/kvsroot.c      |  2 ++
 src/modules/kvs/kvsroot.h      |  2 ++
 src/modules/kvs/test/kvsroot.c |  5 +++++
 5 files changed, 32 insertions(+), 6 deletions(-)

diff --git a/src/common/libkvs/kvs.c b/src/common/libkvs/kvs.c
index 85e178ad6fa6..b5ed0ec38959 100644
--- a/src/common/libkvs/kvs.c
+++ b/src/common/libkvs/kvs.c
@@ -25,6 +25,8 @@
 #if HAVE_CONFIG_H
 #include "config.h"
 #endif
+#include <unistd.h>
+
 #include <flux/core.h>
 
 const char *get_kvs_namespace (void)
@@ -42,9 +44,11 @@ flux_future_t *flux_kvs_namespace_create (flux_t *h, const char *namespace,
         return NULL;
     }
 
+    /* N.B. uid cast to int */
     return flux_rpc_pack (h, "kvs.namespace.create", 0, 0,
-                          "{ s:s s:i }",
+                          "{ s:s s:i s:i }",
                           "namespace", namespace,
+                          "owner", geteuid (),
                           "flags", flags);
 }
 
diff --git a/src/modules/kvs/kvs.c b/src/modules/kvs/kvs.c
index af35594c45a3..1fea881775db 100644
--- a/src/modules/kvs/kvs.c
+++ b/src/modules/kvs/kvs.c
@@ -311,6 +311,7 @@ static void getroot_completion (flux_future_t *f, void *arg)
     const flux_msg_t *msg;
     const char *namespace;
     int rootseq, flags;
+    uint32_t owner;
     const char *ref;
     struct kvsroot *root;
     int save_errno;
@@ -327,7 +328,9 @@ static void getroot_completion (flux_future_t *f, void *arg)
         goto error;
     }
 
-    if (flux_rpc_get_unpack (f, "{ s:i s:s s:i }",
+    /* N.B. owner read into uint32_t */
+    if (flux_rpc_get_unpack (f, "{ s:i s:i s:s s:i }",
+                             "owner", &owner,
                              "rootseq", &rootseq,
                              "rootref", &ref,
                              "flags", &flags) < 0) {
@@ -343,6 +346,7 @@ static void getroot_completion (flux_future_t *f, void *arg)
                                               ctx->cache,
                                               ctx->hash_name,
                                               namespace,
+                                              owner,
                                               flags))) {
             flux_log_error (ctx->h, "%s: kvsroot_mgr_create_root", __FUNCTION__);
             goto error;
@@ -1879,7 +1883,9 @@ static void getroot_request_cb (flux_t *h, flux_msg_handler_t *mh,
         }
     }
 
-    if (flux_respond_pack (h, msg, "{ s:i s:s s:i }",
+    /* N.B. owner cast into int */
+    if (flux_respond_pack (h, msg, "{ s:i s:i s:s s:i }",
+                           "owner", root->owner,
                            "rootseq", root->seq,
                            "rootref", root->ref,
                            "flags", root->flags) < 0) {
@@ -2219,7 +2225,8 @@ static void stats_clear_request_cb (flux_t *h, flux_msg_handler_t *mh,
         flux_log_error (h, "%s: flux_respond", __FUNCTION__);
 }
 
-static int namespace_create (kvs_ctx_t *ctx, const char *namespace, int flags)
+static int namespace_create (kvs_ctx_t *ctx, const char *namespace,
+                             uint32_t owner, int flags)
 {
     struct kvsroot *root;
     json_t *rootdir = NULL;
@@ -2238,6 +2245,7 @@ static int namespace_create (kvs_ctx_t *ctx, const char *namespace, int flags)
                                           ctx->cache,
                                           ctx->hash_name,
                                           namespace,
+                                          owner,
                                           flags))) {
         flux_log_error (ctx->h, "%s: kvsroot_mgr_create_root", __FUNCTION__);
         goto cleanup;
@@ -2281,18 +2289,21 @@ static void namespace_create_request_cb (flux_t *h, flux_msg_handler_t *mh,
 {
     kvs_ctx_t *ctx = arg;
     const char *namespace;
+    uint32_t owner;
     int flags;
 
     assert (ctx->rank == 0);
 
-    if (flux_request_unpack (msg, NULL, "{ s:s s:i }",
+    /* N.B. owner read into uint32_t */
+    if (flux_request_unpack (msg, NULL, "{ s:s s:i s:i }",
                              "namespace", &namespace,
+                             "owner", &owner,
                              "flags", &flags) < 0) {
         flux_log_error (h, "%s: flux_request_unpack", __FUNCTION__);
         goto error;
     }
 
-    if (namespace_create (ctx, namespace, flags) < 0) {
+    if (namespace_create (ctx, namespace, owner, flags) < 0) {
         flux_log_error (h, "%s: namespace_create", __FUNCTION__);
         goto error;
     }
@@ -2553,6 +2564,7 @@ int mod_main (flux_t *h, int argc, char **argv)
     if (ctx->rank == 0) {
         struct kvsroot *root;
         blobref_t rootref;
+        uint32_t owner = geteuid ();
 
         if (store_initial_rootdir (ctx, rootref) < 0) {
             flux_log_error (h, "storing initial root object");
@@ -2569,6 +2581,7 @@ int mod_main (flux_t *h, int argc, char **argv)
                                                   ctx->cache,
                                                   ctx->hash_name,
                                                   KVS_PRIMARY_NAMESPACE,
+                                                  owner,
                                                   0))) {
                 flux_log_error (h, "kvsroot_mgr_create_root");
                 goto done;
diff --git a/src/modules/kvs/kvsroot.c b/src/modules/kvs/kvsroot.c
index c76488a60d14..d8fdbdbd836e 100644
--- a/src/modules/kvs/kvsroot.c
+++ b/src/modules/kvs/kvsroot.c
@@ -108,6 +108,7 @@ struct kvsroot *kvsroot_mgr_create_root (kvsroot_mgr_t *km,
                                          struct cache *cache,
                                          const char *hash_name,
                                          const char *namespace,
+                                         uint32_t owner,
                                          int flags)
 {
     struct kvsroot *root;
@@ -143,6 +144,7 @@ struct kvsroot *kvsroot_mgr_create_root (kvsroot_mgr_t *km,
         goto error;
     }
 
+    root->owner = owner;
     root->flags = flags;
     root->remove = false;
 
diff --git a/src/modules/kvs/kvsroot.h b/src/modules/kvs/kvsroot.h
index e98e91f1303a..3436f5ce053a 100644
--- a/src/modules/kvs/kvsroot.h
+++ b/src/modules/kvs/kvsroot.h
@@ -13,6 +13,7 @@ typedef struct kvsroot_mgr kvsroot_mgr_t;
 
 struct kvsroot {
     char *namespace;
+    uint32_t owner;
     int seq;
     blobref_t ref;
     commit_mgr_t *cm;
@@ -37,6 +38,7 @@ struct kvsroot *kvsroot_mgr_create_root (kvsroot_mgr_t *km,
                                          struct cache *cache,
                                          const char *hash_name,
                                          const char *namespace,
+                                         uint32_t owner,
                                          int flags);
 
 int kvsroot_mgr_remove_root (kvsroot_mgr_t *km, const char *namespace);
diff --git a/src/modules/kvs/test/kvsroot.c b/src/modules/kvs/test/kvsroot.c
index 14f0218490d1..b97ce6601663 100644
--- a/src/modules/kvs/test/kvsroot.c
+++ b/src/modules/kvs/test/kvsroot.c
@@ -2,6 +2,7 @@
 #include "config.h"
 #endif
 #include <stdbool.h>
+#include <unistd.h>
 #include <jansson.h>
 
 #include "src/common/libtap/tap.h"
@@ -30,6 +31,7 @@ void basic_api_tests (void)
                                          cache,
                                          "sha1",
                                          KVS_PRIMARY_NAMESPACE,
+                                         geteuid (),
                                          0)) != NULL,
          "kvsroot_mgr_create_root works");
 
@@ -119,6 +121,7 @@ void basic_iter_tests (void)
                                          cache,
                                          "sha1",
                                          "foo",
+                                         geteuid (),
                                          0)) != NULL,
          "kvsroot_mgr_create_root works");
 
@@ -126,6 +129,7 @@ void basic_iter_tests (void)
                                          cache,
                                          "sha1",
                                          "bar",
+                                         geteuid (),
                                          0)) != NULL,
          "kvsroot_mgr_create_root works");
 
@@ -187,6 +191,7 @@ void basic_commit_mgr_tests (void)
                                          cache,
                                          "sha1",
                                          KVS_PRIMARY_NAMESPACE,
+                                         geteuid (),
                                          0)) != NULL,
          "kvsroot_mgr_create_root works");
 

From 9b38c6b17a025764198010cf210e4697088edc5f Mon Sep 17 00:00:00 2001
From: Albert Chu <chu11@llnl.gov>
Date: Fri, 26 Jan 2018 11:08:35 -0800
Subject: [PATCH 2/5] common/libkvs: Add owner to namespace create

Modify flux_kvs_namespace_create() to allow caller to specify
an alternate namespace owner.
---
 doc/man3/flux_kvs_namespace_create.adoc | 5 ++++-
 src/cmd/flux-kvs.c                      | 3 ++-
 src/common/libkvs/kvs.c                 | 6 +++---
 src/common/libkvs/kvs.h                 | 2 +-
 src/common/libkvs/test/kvs.c            | 2 +-
 5 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/doc/man3/flux_kvs_namespace_create.adoc b/doc/man3/flux_kvs_namespace_create.adoc
index 5626192cd895..4d93fcecde6e 100644
--- a/doc/man3/flux_kvs_namespace_create.adoc
+++ b/doc/man3/flux_kvs_namespace_create.adoc
@@ -14,6 +14,7 @@ SYNOPSIS
 
  flux_future_t *flux_kvs_namespace_create (flux_t *h,
                                            const char *namespace,
+                                           uint32_t owner,
                                            int flags);
 
  flux_future_t *flux_kvs_namespace_remove (flux_t *h,
@@ -24,7 +25,9 @@ DESCRIPTION
 
 `flux_kvs_namespace_create()` creates a KVS namespace.  Within a
 namespace, users can get/put KVS values completely independent of
-other KVS namespaces.
+other KVS namespaces.  An owner of the namespace other than the
+instance owner can be chosen by setting _owner_.  Otherwise, _owner_
+can be set to FLUX_USERID_UNKNOWN.
 
 `flux_kvs_namespace_remove()` removes a KVS namespace.
 
diff --git a/src/cmd/flux-kvs.c b/src/cmd/flux-kvs.c
index c2801411aaba..38b01b114f0c 100644
--- a/src/cmd/flux-kvs.c
+++ b/src/cmd/flux-kvs.c
@@ -364,7 +364,8 @@ int cmd_namespace_create (optparse_t *p, int argc, char **argv)
     for (i = optindex; i < argc; i++) {
         const char *name = argv[i];
         int flags = 0;
-        if (!(f = flux_kvs_namespace_create (h, name, flags))
+        if (!(f = flux_kvs_namespace_create (h, name, FLUX_USERID_UNKNOWN,
+                                             flags))
             || flux_future_get (f, NULL) < 0)
             log_err_exit ("%s", name);
         flux_future_destroy (f);
diff --git a/src/common/libkvs/kvs.c b/src/common/libkvs/kvs.c
index b5ed0ec38959..9e70336c942a 100644
--- a/src/common/libkvs/kvs.c
+++ b/src/common/libkvs/kvs.c
@@ -37,18 +37,18 @@ const char *get_kvs_namespace (void)
 }
 
 flux_future_t *flux_kvs_namespace_create (flux_t *h, const char *namespace,
-                                          int flags)
+                                          uint32_t owner, int flags)
 {
     if (!namespace || flags) {
         errno = EINVAL;
         return NULL;
     }
 
-    /* N.B. uid cast to int */
+    /* N.B. owner cast to int */
     return flux_rpc_pack (h, "kvs.namespace.create", 0, 0,
                           "{ s:s s:i s:i }",
                           "namespace", namespace,
-                          "owner", geteuid (),
+                          "owner", owner,
                           "flags", flags);
 }
 
diff --git a/src/common/libkvs/kvs.h b/src/common/libkvs/kvs.h
index a96e059e41a4..78ea23207292 100644
--- a/src/common/libkvs/kvs.h
+++ b/src/common/libkvs/kvs.h
@@ -33,7 +33,7 @@ enum {
  *   consistent".
  */
 flux_future_t *flux_kvs_namespace_create (flux_t *h, const char *namespace,
-                                          int flags);
+                                          uint32_t owner, int flags);
 flux_future_t *flux_kvs_namespace_remove (flux_t *h, const char *namespace);
 
 /* Synchronization:
diff --git a/src/common/libkvs/test/kvs.c b/src/common/libkvs/test/kvs.c
index a255efbc3182..2df71b85f2d6 100644
--- a/src/common/libkvs/test/kvs.c
+++ b/src/common/libkvs/test/kvs.c
@@ -15,7 +15,7 @@ void errors (void)
     /* check simple error cases */
 
     errno = 0;
-    ok (flux_kvs_namespace_create (NULL, NULL, 5) == NULL && errno == EINVAL,
+    ok (flux_kvs_namespace_create (NULL, NULL, 0, 5) == NULL && errno == EINVAL,
         "flux_kvs_namespace_create fails on bad input");
 
     errno = 0;

From a4d2764262ce36b430133efb85a2f672b3fd06af Mon Sep 17 00:00:00 2001
From: Albert Chu <chu11@llnl.gov>
Date: Fri, 26 Jan 2018 11:16:54 -0800
Subject: [PATCH 3/5] cmd/flux-kvs: Support namespace-create owner

Allow user to specify an alternate owner of a namespace when it
is created.
---
 doc/man1/flux-kvs.adoc |  6 ++++--
 src/cmd/builtin/user.c |  1 +
 src/cmd/flux-kvs.c     | 22 +++++++++++++++++++---
 3 files changed, 24 insertions(+), 5 deletions(-)

diff --git a/doc/man1/flux-kvs.adoc b/doc/man1/flux-kvs.adoc
index 7c1f3acb0ef4..49f9ba196909 100644
--- a/doc/man1/flux-kvs.adoc
+++ b/doc/man1/flux-kvs.adoc
@@ -55,8 +55,10 @@ arguments are described below.
 
 COMMANDS
 --------
-*namespace-create* 'name' ['name...']::
-Create a new kvs namespace.
+*namespace-create* [-o owner] 'name' ['name...']::
+Create a new kvs namespace.  User may specify an alternate userid of a
+user that owns the namespace via '-o'.  Specifying an alternate owner
+would allow a non-instance owner to read/write to a namespace.
 
 *namespace-remove* 'name' ['name...']::
 Remove a kvs namespace.
diff --git a/src/cmd/builtin/user.c b/src/cmd/builtin/user.c
index 78788354b55a..b1b1dbb9d245 100644
--- a/src/cmd/builtin/user.c
+++ b/src/cmd/builtin/user.c
@@ -225,6 +225,7 @@ static int internal_user_lookup (optparse_t *p, int ac, char *av[])
         exit (1);
     }
     userid = strtoul (av[n], &endptr, 10);
+
     if (*endptr != '\0')
         userid = lookup_user (av[n]);
     if (userid == FLUX_USERID_UNKNOWN)
diff --git a/src/cmd/flux-kvs.c b/src/cmd/flux-kvs.c
index 38b01b114f0c..9c27a5c32e33 100644
--- a/src/cmd/flux-kvs.c
+++ b/src/cmd/flux-kvs.c
@@ -60,6 +60,13 @@ static void dump_kvs_dir (const flux_kvsdir_t *dir, int maxcol,
 
 #define min(a,b) ((a)<(b)?(a):(b))
 
+static struct optparse_option namespace_create_opts[] =  {
+    { .name = "owner", .key = 'o', .has_arg = 1,
+      .usage = "Specify alternate namespace owner via userid",
+    },
+    OPTPARSE_TABLE_END
+};
+
 static struct optparse_option readlink_opts[] =  {
     { .name = "at", .key = 'a', .has_arg = 1,
       .usage = "Lookup relative to RFC 11 snapshot reference",
@@ -176,7 +183,7 @@ static struct optparse_subcommand subcommands[] = {
       "Create a KVS namespace",
       cmd_namespace_create,
       0,
-      NULL
+      namespace_create_opts
     },
     { "namespace-remove",
       "name [name...]",
@@ -355,17 +362,26 @@ int cmd_namespace_create (optparse_t *p, int argc, char **argv)
     flux_t *h = (flux_t *)optparse_get_data (p, "flux_handle");
     flux_future_t *f;
     int optindex, i;
+    uint32_t owner = FLUX_USERID_UNKNOWN;
+    const char *str;
 
     optindex = optparse_option_index (p);
     if ((optindex - argc) == 0) {
         optparse_print_usage (p);
         exit (1);
     }
+
+    if ((str = optparse_get_str (p, "owner", NULL))) {
+        char *endptr;
+        owner = strtoul (str, &endptr, 10);
+        if (*endptr != '\0')
+            log_err_exit ("--owner requires an unsigned integer argument");
+    }
+
     for (i = optindex; i < argc; i++) {
         const char *name = argv[i];
         int flags = 0;
-        if (!(f = flux_kvs_namespace_create (h, name, FLUX_USERID_UNKNOWN,
-                                             flags))
+        if (!(f = flux_kvs_namespace_create (h, name, owner, flags))
             || flux_future_get (f, NULL) < 0)
             log_err_exit ("%s", name);
         flux_future_destroy (f);

From 384d87ab0ee9b1c419aafdb392de747772ab0f06 Mon Sep 17 00:00:00 2001
From: Albert Chu <chu11@llnl.gov>
Date: Fri, 26 Jan 2018 16:42:15 -0800
Subject: [PATCH 4/5] modules/kvs: Add namespace message authentication

For a number of kvs callback functions, allow FLUX_ROLE_USER
messages to be received by the callback functions.  In those callbacks,
ensure that the message is coming specifically from the instance
owner or the owner of the namespace.
---
 src/modules/kvs/kvs.c | 63 ++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 57 insertions(+), 6 deletions(-)

diff --git a/src/modules/kvs/kvs.c b/src/modules/kvs/kvs.c
index 1fea881775db..f6f4c6523bff 100644
--- a/src/modules/kvs/kvs.c
+++ b/src/modules/kvs/kvs.c
@@ -284,6 +284,39 @@ static int event_unsubscribe (kvs_ctx_t *ctx, const char *namespace)
     return rc;
 }
 
+/*
+ * security
+ */
+
+static int check_user (kvs_ctx_t *ctx, struct kvsroot *root,
+                       const flux_msg_t *msg)
+{
+    uint32_t rolemask;
+
+    if (flux_msg_get_rolemask (msg, &rolemask) < 0) {
+        flux_log_error (ctx->h, "flux_msg_get_rolemask");
+        return -1;
+    }
+
+    if (rolemask & FLUX_ROLE_OWNER)
+        return 0;
+
+    if (rolemask & FLUX_ROLE_USER) {
+        uint32_t userid;
+
+        if (flux_msg_get_userid (msg, &userid) < 0) {
+            flux_log_error (ctx->h, "flux_msg_get_userid");
+            return -1;
+        }
+
+        if (userid == root->owner)
+            return 0;
+    }
+
+    errno = EPERM;
+    return -1;
+}
+
 /*
  * set/get root
  */
@@ -446,8 +479,13 @@ static struct kvsroot *getroot (kvs_ctx_t *ctx, const char *namespace,
                 return NULL;
             }
             (*stall) = true;
+            return NULL;
         }
     }
+
+    if (check_user (ctx, root, msg) < 0)
+        return NULL;
+
     return root;
 }
 
@@ -1575,6 +1613,9 @@ static void unwatch_request_cb (flux_t *h, flux_msg_handler_t *mh,
     if (!(root = kvsroot_mgr_lookup_root_safe (ctx->km, namespace)))
         goto done;
 
+    if (check_user (ctx, root, msg) < 0)
+        goto done;
+
     if (!(p.key = kvs_util_normalize_key (key, NULL))) {
         errnum = errno;
         goto done;
@@ -1773,6 +1814,7 @@ static void fence_request_cb (flux_t *h, flux_msg_handler_t *mh,
     else {
         flux_future_t *f;
 
+        /* route to rank 0 as instance owner */
         if (!(f = flux_rpc_pack (h, "kvs.relayfence", 0, FLUX_RPC_NORESPONSE,
                                  "{ s:O s:s s:s s:i s:i }",
                                  "ops", ops,
@@ -1869,6 +1911,9 @@ static void getroot_request_cb (flux_t *h, flux_msg_handler_t *mh,
             errno = ENOTSUP;
             goto error;
         }
+
+        if (check_user (ctx, root, msg) < 0)
+            goto error;
     }
     else {
         /* If root is not initialized, we have to intialize ourselves
@@ -2450,16 +2495,22 @@ static const struct flux_msg_handler_spec htab[] = {
     { FLUX_MSGTYPE_EVENT,   "kvs.stats.clear",stats_clear_event_cb, 0 },
     { FLUX_MSGTYPE_EVENT,   "kvs.setroot.*",  setroot_event_cb, 0 },
     { FLUX_MSGTYPE_EVENT,   "kvs.error.*",    error_event_cb, 0 },
-    { FLUX_MSGTYPE_REQUEST, "kvs.getroot",    getroot_request_cb, 0 },
+    { FLUX_MSGTYPE_REQUEST, "kvs.getroot",
+                            getroot_request_cb, FLUX_ROLE_USER },
     { FLUX_MSGTYPE_REQUEST, "kvs.dropcache",  dropcache_request_cb, 0 },
     { FLUX_MSGTYPE_EVENT,   "kvs.dropcache",  dropcache_event_cb, 0 },
     { FLUX_MSGTYPE_EVENT,   "hb",             heartbeat_cb, 0 },
     { FLUX_MSGTYPE_REQUEST, "kvs.disconnect", disconnect_request_cb, 0 },
-    { FLUX_MSGTYPE_REQUEST, "kvs.unwatch",    unwatch_request_cb, 0 },
-    { FLUX_MSGTYPE_REQUEST, "kvs.sync",       sync_request_cb, 0 },
-    { FLUX_MSGTYPE_REQUEST, "kvs.get",        get_request_cb, 0 },
-    { FLUX_MSGTYPE_REQUEST, "kvs.watch",      watch_request_cb, 0 },
-    { FLUX_MSGTYPE_REQUEST, "kvs.fence",      fence_request_cb, 0 },
+    { FLUX_MSGTYPE_REQUEST, "kvs.unwatch",
+                            unwatch_request_cb, FLUX_ROLE_USER },
+    { FLUX_MSGTYPE_REQUEST, "kvs.sync",
+                            sync_request_cb, FLUX_ROLE_USER },
+    { FLUX_MSGTYPE_REQUEST, "kvs.get",
+                            get_request_cb, FLUX_ROLE_USER },
+    { FLUX_MSGTYPE_REQUEST, "kvs.watch",
+                            watch_request_cb, FLUX_ROLE_USER },
+    { FLUX_MSGTYPE_REQUEST, "kvs.fence",
+                            fence_request_cb, FLUX_ROLE_USER },
     { FLUX_MSGTYPE_REQUEST, "kvs.relayfence", relayfence_request_cb, 0 },
     { FLUX_MSGTYPE_REQUEST, "kvs.namespace.create",
                             namespace_create_request_cb, 0 },

From b0b2768dd50c1ef5b080dfdd5554d8d3ebe265e5 Mon Sep 17 00:00:00 2001
From: Albert Chu <chu11@llnl.gov>
Date: Mon, 29 Jan 2018 16:47:48 -0800
Subject: [PATCH 5/5] t/kvs: Add t1005-kvs-security.t

Add security tests to ensure namespaces operate properly for
owner/user access.
---
 t/Makefile.am          |   2 +
 t/t1005-kvs-security.t | 398 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 400 insertions(+)
 create mode 100755 t/t1005-kvs-security.t

diff --git a/t/Makefile.am b/t/Makefile.am
index 9c86f13849aa..ad68a10be4e9 100644
--- a/t/Makefile.am
+++ b/t/Makefile.am
@@ -53,6 +53,7 @@ TESTS = \
 	t1002-kvs-watch.t \
 	t1003-kvs-stress.t \
 	t1004-kvs-namespace.t \
+	t1005-kvs-security.t \
 	t1101-barrier-basic.t \
 	t1102-cmddriver.t \
 	t1103-apidisconnect.t \
@@ -127,6 +128,7 @@ check_SCRIPTS = \
 	t1002-kvs-watch.t \
 	t1003-kvs-stress.t \
 	t1004-kvs-namespace.t \
+	t1005-kvs-security.t \
 	t1101-barrier-basic.t \
 	t1102-cmddriver.t \
 	t1103-apidisconnect.t \
diff --git a/t/t1005-kvs-security.t b/t/t1005-kvs-security.t
new file mode 100755
index 000000000000..20c39f0e9d67
--- /dev/null
+++ b/t/t1005-kvs-security.t
@@ -0,0 +1,398 @@
+#!/bin/sh
+
+test_description='Test flux-kvs and kvs in flux session
+
+These are tests for ensuring multiple namespaces work.
+'
+
+. `dirname $0`/kvs/kvs-helper.sh
+
+. `dirname $0`/sharness.sh
+
+if test "$TEST_LONG" = "t"; then
+    test_set_prereq LONGTEST
+fi
+
+# Size the session to one more than the number of cores, minimum of 4
+SIZE=$(test_size_large)
+test_under_flux ${SIZE} kvs
+echo "# $0: flux session size will be ${SIZE}"
+
+DIR=test.a.b
+
+# Just in case its set in the environment
+unset FLUX_KVS_NAMESPACE
+
+NAMESPACETMP=namespacetmp
+
+test_kvs_key_namespace() {
+        export FLUX_KVS_NAMESPACE=$1
+	flux kvs get --json "$2" >output
+	echo "$3" >expected
+        unset FLUX_KVS_NAMESPACE
+	test_cmp expected output
+}
+
+put_kvs_key_namespace() {
+        export FLUX_KVS_NAMESPACE=$1
+	flux kvs put --json "$2=$3"
+        unset FLUX_KVS_NAMESPACE
+}
+
+unlink_kvs_dir_namespace() {
+        export FLUX_KVS_NAMESPACE=$1
+        flux kvs unlink -Rf $2
+        unset FLUX_KVS_NAMESPACE
+}
+
+version_kvs_namespace() {
+        export FLUX_KVS_NAMESPACE=$1
+        version=`flux kvs version`
+        eval $2=$version
+        unset FLUX_KVS_NAMESPACE
+}
+
+put_kvs_namespace_exitvalue() {
+        export FLUX_KVS_NAMESPACE=$1
+	flux kvs put --json "$2=$3"
+        eval $4="$?"
+        unset FLUX_KVS_NAMESPACE
+}
+
+get_kvs_namespace_exitvalue() {
+        export FLUX_KVS_NAMESPACE=$1
+	flux kvs get --json "$2"
+        eval $3="$?"
+        unset FLUX_KVS_NAMESPACE
+}
+
+watch_kvs_namespace_exitvalue() {
+        export FLUX_KVS_NAMESPACE=$1
+	flux kvs watch -o -c 1 "$2"
+        eval $3="$?"
+        unset FLUX_KVS_NAMESPACE
+}
+
+version_kvs_namespace_exitvalue() {
+        export FLUX_KVS_NAMESPACE=$1
+	flux kvs version
+        eval $2="$?"
+        unset FLUX_KVS_NAMESPACE
+}
+
+wait_kvs_namespace_exitvalue() {
+        export FLUX_KVS_NAMESPACE=$1
+	flux kvs wait $2
+        eval $3="$?"
+        unset FLUX_KVS_NAMESPACE
+}
+
+wait_watch_put_namespace() {
+        export FLUX_KVS_NAMESPACE=$1
+        wait_watch_put $2 $3
+        exitvalue=$?
+        unset FLUX_KVS_NAMESPACE
+        return $exitvalue
+}
+
+set_userid() {
+        export FLUX_HANDLE_USERID=$1
+        export FLUX_HANDLE_ROLEMASK=0x2
+}
+
+unset_userid() {
+        unset FLUX_HANDLE_USERID
+        unset FLUX_HANDLE_ROLEMASK
+}
+
+#
+# Basic tests, make sure only instance owner can perform operations
+# and non-namespace owner can't perform operations
+#
+
+test_expect_success 'kvs: namespace create fails (user)' '
+        set_userid 9999 &&
+	! flux kvs namespace-create $NAMESPACETMP-OWNER &&
+        unset_userid
+'
+
+test_expect_success 'kvs: namespace create works (owner)' '
+	flux kvs namespace-create $NAMESPACETMP-OWNER
+'
+
+test_expect_success 'kvs: put fails (user)' '
+        set_userid 9999 &&
+        put_kvs_namespace_exitvalue $NAMESPACETMP-OWNER $DIR.test 1 exitvalue &&
+        unset_userid &&
+        test $exitvalue -ne 0
+'
+
+test_expect_success 'kvs: put works (owner)' '
+        put_kvs_key_namespace $NAMESPACETMP-OWNER $DIR.test 1 &&
+        test_kvs_key_namespace $NAMESPACETMP-OWNER $DIR.test 1
+'
+
+test_expect_success 'kvs: get fails (user)' '
+        set_userid 9999 &&
+        get_kvs_namespace_exitvalue $NAMESPACETMP-OWNER $DIR.test exitvalue &&
+        unset_userid &&
+        test $exitvalue -ne 0
+'
+
+test_expect_success 'kvs: get fails on other ranks (user)' '
+        ! flux exec -r 1 sh -c "FLUX_KVS_NAMESPACE=$NAMESPACETMP-OWNER \
+                                FLUX_HANDLE_USERID=9999 \
+                                FLUX_HANDLE_ROLEMASK=0x2 \
+                                flux kvs get $DIR.test"
+'
+
+test_expect_success 'kvs: get works on other ranks (owner)' '
+        flux exec -r 1 sh -c "FLUX_KVS_NAMESPACE=$NAMESPACETMP-OWNER \
+                              flux kvs get $DIR.test"
+'
+
+
+test_expect_success NO_CHAIN_LINT 'kvs: watch works (owner)'  '
+        unlink_kvs_dir_namespace $NAMESPACETMP-OWNER $DIR &&
+        put_kvs_key_namespace $NAMESPACETMP-OWNER $DIR.watch 0 &&
+        wait_watch_put_namespace $NAMESPACETMP-OWNER "$DIR.watch" "0"
+        rm -f watch_out
+        export FLUX_KVS_NAMESPACE=$NAMESPACETMP-OWNER
+        stdbuf -oL flux kvs watch -o -c 1 $DIR.watch >watch_out &
+        watchpid=$! &&
+        wait_watch_file watch_out "0"
+        flux kvs put --json $DIR.watch=1 &&
+        wait $watchpid
+        unset FLUX_KVS_NAMESPACE
+cat >expected <<-EOF &&
+0
+1
+EOF
+        test_cmp watch_out expected
+'
+
+test_expect_success 'kvs: watch fails (user)' '
+        set_userid 9999 &&
+        watch_kvs_namespace_exitvalue $NAMESPACETMP-OWNER $DIR.test exitvalue &&
+        unset_userid &&
+        test $exitvalue -ne 0
+'
+
+test_expect_success 'kvs: version fails (user)' '
+        set_userid 9999 &&
+        version_kvs_namespace_exitvalue $NAMESPACETMP-OWNER exitvalue &&
+        unset_userid &&
+        test $exitvalue -ne 0
+'
+
+test_expect_success 'kvs: version fails on other ranks (user)' '
+        ! flux exec -r 1 sh -c "FLUX_KVS_NAMESPACE=$NAMESPACETMP-OWNER \
+                                FLUX_HANDLE_USERID=9999 \
+                                FLUX_HANDLE_ROLEMASK=0x2 \
+                                flux kvs version"
+'
+
+test_expect_success 'kvs: wait fails (user)' '
+        set_userid 9999 &&
+        wait_kvs_namespace_exitvalue $NAMESPACETMP-OWNER $DIR.test exitvalue &&
+        unset_userid &&
+        test $exitvalue -ne 0
+'
+
+test_expect_success 'kvs: namespace remove fails (user)' '
+        set_userid 9999 &&
+	! flux kvs namespace-remove $NAMESPACETMP-OWNER &&
+        unset_userid
+'
+
+test_expect_success 'kvs: namespace remove works (owner)' '
+        put_kvs_key_namespace $NAMESPACETMP-OWNER $DIR.tmp 1 &&
+        test_kvs_key_namespace $NAMESPACETMP-OWNER $DIR.tmp 1 &&
+        flux kvs namespace-remove $NAMESPACETMP-OWNER &&
+        get_kvs_namespace_exitvalue $NAMESPACETMP-OWNER $DIR.tmp exitvalue &&
+        test $exitvalue -ne 0
+'
+
+#
+# Namespace tests owned by user
+#
+
+test_expect_success 'kvs: namespace create works (owner, for user)' '
+	flux kvs namespace-create -o 9999 $NAMESPACETMP-USER
+'
+
+test_expect_success 'kvs: namespace put/get works (user)' '
+        set_userid 9999 &&
+        put_kvs_key_namespace $NAMESPACETMP-USER $DIR.test 1 &&
+        test_kvs_key_namespace $NAMESPACETMP-USER $DIR.test 1 &&
+        unset_userid
+'
+
+test_expect_success 'kvs: put/get works (owner)' '
+        put_kvs_key_namespace $NAMESPACETMP-USER $DIR.test 2 &&
+        test_kvs_key_namespace $NAMESPACETMP-USER $DIR.test 2
+'
+
+test_expect_success 'kvs: put fails (wrong user)' '
+        set_userid 9000 &&
+        put_kvs_namespace_exitvalue $NAMESPACETMP-USER $DIR.test 1 exitvalue &&
+        unset_userid &&
+        test $exitvalue -ne 0
+'
+
+test_expect_success 'kvs: get works on other ranks (user)' '
+        flux exec -r 1 sh -c "FLUX_KVS_NAMESPACE=$NAMESPACETMP-USER \
+                              FLUX_HANDLE_USERID=9999 \
+                              FLUX_HANDLE_ROLEMASK=0x2 \
+                              flux kvs get $DIR.test"
+'
+
+test_expect_success 'kvs: get works on other ranks (owner)' '
+        flux exec -r 1 sh -c "FLUX_KVS_NAMESPACE=$NAMESPACETMP-USER \
+                              flux kvs get $DIR.test"
+'
+
+test_expect_success 'kvs: get fails (wrong user)' '
+        set_userid 9000 &&
+        get_kvs_namespace_exitvalue $NAMESPACETMP-USER $DIR.test exitvalue &&
+        unset_userid &&
+        test $exitvalue -ne 0
+'
+
+test_expect_success 'kvs: get fails on other ranks (wrong user)' '
+        ! flux exec -r 1 sh -c "FLUX_KVS_NAMESPACE=$NAMESPACETMP-USER \
+                                FLUX_HANDLE_USERID=9000 \
+                                FLUX_HANDLE_ROLEMASK=0x2 \
+                                flux kvs get $DIR.test"
+'
+
+test_expect_success NO_CHAIN_LINT 'kvs: watch works (user)'  '
+        set_userid 9999 &&
+        unlink_kvs_dir_namespace $NAMESPACETMP-USER $DIR &&
+        put_kvs_key_namespace $NAMESPACETMP-USER $DIR.watch 0 &&
+        wait_watch_put_namespace $NAMESPACETMP-USER "$DIR.watch" "0"
+        rm -f watch_out
+        export FLUX_KVS_NAMESPACE=$NAMESPACETMP-USER
+        stdbuf -oL flux kvs watch -o -c 1 $DIR.watch >watch_out &
+        watchpid=$! &&
+        wait_watch_file watch_out "0"
+        flux kvs put --json $DIR.watch=1 &&
+        wait $watchpid
+        unset FLUX_KVS_NAMESPACE
+        unset_userid
+cat >expected <<-EOF &&
+0
+1
+EOF
+        test_cmp watch_out expected
+'
+
+test_expect_success NO_CHAIN_LINT 'kvs: watch works (owner)'  '
+        unlink_kvs_dir_namespace $NAMESPACETMP-USER $DIR &&
+        put_kvs_key_namespace $NAMESPACETMP-USER $DIR.watch 0 &&
+        wait_watch_put_namespace $NAMESPACETMP-USER "$DIR.watch" "0"
+        rm -f watch_out
+        export FLUX_KVS_NAMESPACE=$NAMESPACETMP-USER
+        stdbuf -oL flux kvs watch -o -c 1 $DIR.watch >watch_out &
+        watchpid=$! &&
+        wait_watch_file watch_out "0"
+        flux kvs put --json $DIR.watch=1 &&
+        wait $watchpid
+        unset FLUX_KVS_NAMESPACE
+cat >expected <<-EOF &&
+0
+1
+EOF
+        test_cmp watch_out expected
+'
+
+test_expect_success 'kvs: watch fails (wrong user)' '
+        set_userid 9000 &&
+        watch_kvs_namespace_exitvalue $NAMESPACETMP-USER $DIR.test exitvalue &&
+        unset_userid &&
+        test $exitvalue -ne 0
+'
+
+test_expect_success NO_CHAIN_LINT 'kvs: version & wait works (user)' '
+        set_userid 9999
+        version_kvs_namespace $NAMESPACETMP-USER VERS
+        VERS=$((VERS + 1))
+        export FLUX_KVS_NAMESPACE=$NAMESPACETMP-USER
+        flux kvs wait $VERS &
+        kvswaitpid=$!
+        flux kvs put --json $DIR.xxx=99
+        unset FLUX_KVS_NAMESPACE
+        unset_userid
+        test_expect_code 0 wait $kvswaitpid
+'
+
+test_expect_success 'kvs: version works on other ranks (user)' '
+        flux exec -r 1 sh -c "FLUX_KVS_NAMESPACE=$NAMESPACETMP-USER \
+                              FLUX_HANDLE_USERID=9999 \
+                              FLUX_HANDLE_ROLEMASK=0x2 \
+                              flux kvs version"
+'
+
+test_expect_success 'kvs: version works on other ranks (owner)' '
+        flux exec -r 1 sh -c "FLUX_KVS_NAMESPACE=$NAMESPACETMP-USER \
+                              flux kvs version"
+'
+
+test_expect_success 'kvs: version fails (wrong user)' '
+        set_userid 9000 &&
+        version_kvs_namespace_exitvalue $NAMESPACETMP-USER exitvalue &&
+        unset_userid &&
+        test $exitvalue -ne 0
+'
+
+test_expect_success 'kvs: version fails on other ranks (wrong user)' '
+        ! flux exec -r 1 sh -c "FLUX_KVS_NAMESPACE=$NAMESPACETMP-USER \
+                                FLUX_HANDLE_USERID=9000 \
+                                FLUX_HANDLE_ROLEMASK=0x2 \
+                                flux kvs version"
+'
+
+test_expect_success 'kvs: wait fails (wrong user)' '
+        set_userid 9000 &&
+        wait_kvs_namespace_exitvalue $NAMESPACETMP-USER $DIR.test exitvalue &&
+        unset_userid &&
+        test $exitvalue -ne 0
+'
+
+test_expect_success 'kvs: namespace remove still fails (user)' '
+        set_userid 9999 &&
+        ! flux kvs namespace-remove $NAMESPACETMP-USER &&
+        unset_userid
+'
+
+test_expect_success 'kvs: namespace remove still works (owner)' '
+        put_kvs_key_namespace $NAMESPACETMP-USER $DIR.tmp 1 &&
+        test_kvs_key_namespace $NAMESPACETMP-USER $DIR.tmp 1 &&
+        flux kvs namespace-remove $NAMESPACETMP-USER &&
+        get_kvs_namespace_exitvalue $NAMESPACETMP-USER $DIR.tmp exitvalue &&
+        test $exitvalue -ne 0
+'
+
+#
+# Basic tests, user can't perform non-namespace operations
+#
+
+test_expect_success 'kvs: dropcache fails (user)' '
+        set_userid 9999 &&
+        ! flux kvs dropcache &&
+        unset_userid
+'
+
+test_expect_success 'kvs: stats fails (user)' '
+        set_userid 9999 &&
+        ! flux module stats kvs &&
+        unset_userid
+'
+
+test_expect_success 'kvs: stats clear fails (user)' '
+        set_userid 9999 &&
+        ! flux module stats -c kvs &&
+        unset_userid
+'
+
+test_done