From dcb86d6329040828940a3a380c95bffc8b2ee97e Mon Sep 17 00:00:00 2001 From: Jim Garlick Date: Thu, 29 Jun 2017 10:13:35 -0700 Subject: [PATCH 01/19] modules/kvs: validate "get" input Problem: the client API implementation of "getat", validates that the root treeobj, but the server accepts anything. This leaves the KVS module somewhat vulnerable malformed input. Move validation to the KVS module. --- src/modules/kvs/kvs.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/modules/kvs/kvs.c b/src/modules/kvs/kvs.c index ffe81f293a2d..2a51ba662906 100644 --- a/src/modules/kvs/kvs.c +++ b/src/modules/kvs/kvs.c @@ -755,7 +755,8 @@ static void get_request_cb (flux_t *h, flux_msg_handler_t *w, * Otherwise, use the current root. */ if (root_dirent) { - if (!Jget_str (root_dirent, "DIRREF", &root_ref)) { + if (dirent_validate (root_dirent) < 0 + || !Jget_str (root_dirent, "DIRREF", &root_ref)) { errno = EINVAL; goto done; } From 1493b438c176179d50dfd76104fa2bc0e287f78f Mon Sep 17 00:00:00 2001 From: Jim Garlick Date: Thu, 29 Jun 2017 14:30:43 -0700 Subject: [PATCH 02/19] test/t1002-kvs-cmd.t: cleanup This test runs --verbose, these errors are omitted: ./t1002-kvs-cmd.t: 352: [: 0: unexpected operator ./t1002-kvs-cmd.t: 352: [: 0: unexpected operator ./t1002-kvs-cmd.t: 380: [: 1: unexpected operator Use [ $i -eq 50 ] to compare numbers rather than [ $i == "50" ]. I'm not sure what, if any, effect this actually had on the tests. I seemed to have some intermediate failures that went away after fixing this, so possibly the while loop was exiting on the first iteration, and that worked most of the time. Also, tidy up here documents by replacing << with <<- and indending them. --- t/t1002-kvs-cmd.t | 138 +++++++++++++++++++++++----------------------- 1 file changed, 69 insertions(+), 69 deletions(-) diff --git a/t/t1002-kvs-cmd.t b/t/t1002-kvs-cmd.t index 1b284e5cf347..47fa3571dcf5 100755 --- a/t/t1002-kvs-cmd.t +++ b/t/t1002-kvs-cmd.t @@ -344,12 +344,12 @@ test_expect_success 'kvs: version and wait' ' wait_watch_put() { i=0 - while [ "$(flux kvs get $1 2> /dev/null)" != "$2" ] && [ $i -lt "50" ] + while [ "$(flux kvs get $1 2> /dev/null)" != "$2" ] && [ $i -lt 50 ] do sleep 0.1 i=$((i + 1)) done - if [ $i == "50" ] + if [ $i -eq 50 ] then return 1 fi @@ -358,12 +358,12 @@ wait_watch_put() { wait_watch_empty() { i=0 - while flux kvs get $1 2> /dev/null && [ $i -lt "50" ] + while flux kvs get $1 2> /dev/null && [ $i -lt 50 ] do sleep 0.1 i=$((i + 1)) done - if [ $i == "50" ] + if [ $i -eq 50 ] then return 1 fi @@ -372,12 +372,12 @@ wait_watch_empty() { wait_watch_current() { i=0 - while [ "$(tail -n 1 watch_out 2> /dev/null)" != "$1" ] && [ $i -lt "50" ] + while [ "$(tail -n 1 watch_out 2> /dev/null)" != "$1" ] && [ $i -lt 50 ] do sleep 0.1 i=$((i + 1)) done - if [ $i == "50" ] + if [ $i -eq 50 ] then return 1 fi @@ -401,10 +401,10 @@ test_expect_success 'kvs: watch a key' ' wait_watch_current "0" flux kvs put $DIR.foo=1 && wait $watchpid - cat >expected <expected <<-EOF + 0 + 1 + EOF test_cmp watch_out expected ' @@ -417,10 +417,10 @@ test_expect_success 'kvs: watch a key that at first doesnt exist' ' wait_watch_current "nil" && flux kvs put $DIR.foo=1 && wait $watchpid - cat >expected <expected <<-EOF + nil + 1 + EOF test_cmp watch_out expected ' @@ -434,10 +434,10 @@ test_expect_success 'kvs: watch a key that gets removed' ' wait_watch_current "0" && flux kvs unlink $DIR.foo && wait $watchpid - cat >expected <expected <<-EOF + 0 + nil + EOF test_cmp watch_out expected ' @@ -451,12 +451,12 @@ test_expect_success 'kvs: watch a key that becomes a dir' ' wait_watch_current "0" && flux kvs put $DIR.foo.bar.baz=1 && wait $watchpid - cat >expected <expected <<-EOF + 0 + ====================== + $DIR.foo.bar. + ====================== + EOF test_cmp watch_out expected ' @@ -471,12 +471,12 @@ test_expect_success 'kvs: watch a dir' ' wait_watch_current "======================" && flux kvs put $DIR.a.a=1 && wait $watchpid - cat >expected <expected <<-EOF + $DIR.a. + ====================== + $DIR.a. + ====================== + EOF test_cmp watch_out expected ' @@ -489,12 +489,12 @@ test_expect_success 'kvs: watch a dir that at first doesnt exist' ' wait_watch_current "nil" && flux kvs put $DIR.a.a=1 && wait $watchpid - cat >expected <expected <<-EOF + nil + ====================== + $DIR.a. + ====================== + EOF test_cmp watch_out expected ' @@ -509,12 +509,12 @@ test_expect_success 'kvs: watch a dir that gets removed' ' wait_watch_current "======================" && flux kvs unlink -R $DIR.a && wait $watchpid - cat >expected <expected <<-EOF + $DIR.a.a. + ====================== + nil + ====================== + EOF test_cmp watch_out expected ' @@ -529,11 +529,11 @@ test_expect_success 'kvs: watch a dir, converted into a key' ' wait_watch_current "======================" && flux kvs put $DIR.a=1 && wait $watchpid - cat >expected <expected <<-EOF + $DIR.a.a. + ====================== + 1 + EOF test_cmp watch_out expected ' @@ -552,12 +552,12 @@ test_expect_success 'kvs: watch a dir, prefix path converted into a key' ' wait_watch_current "======================" && flux kvs put $DIR=1 && wait $watchpid - cat >expected <expected <<-EOF + $DIR.a.a. + ====================== + nil + ====================== + EOF test_cmp watch_out expected ' @@ -572,14 +572,14 @@ test_expect_success 'kvs: watch a dir with -R' ' wait_watch_current "======================" && flux kvs put $DIR.a.a=1 && wait $watchpid - cat >expected <expected <<-EOF + $DIR.a.a = 0 + $DIR.a.b = 0 + ====================== + $DIR.a.a = 1 + $DIR.a.b = 0 + ====================== + EOF test_cmp watch_out expected ' @@ -594,14 +594,14 @@ test_expect_success 'kvs: watch a dir with -R and -d' ' wait_watch_current "======================" && flux kvs put $DIR.a.a=1 && wait $watchpid - cat >expected <expected <<-EOF + $DIR.a.a + $DIR.a.b + ====================== + $DIR.a.a + $DIR.a.b + ====================== + EOF test_cmp watch_out expected ' From b071c81006483efe7756db6821c9563c5945e4bf Mon Sep 17 00:00:00 2001 From: Jim Garlick Date: Wed, 28 Jun 2017 12:10:10 -0700 Subject: [PATCH 03/19] libkvs/lookup: add future-based lookup API --- src/common/libkvs/Makefile.am | 3 +- src/common/libkvs/kvs.h | 2 + src/common/libkvs/kvs_lookup.c | 110 +++++++++++++++++++++++++++++++++ src/common/libkvs/kvs_lookup.h | 21 +++++++ 4 files changed, 135 insertions(+), 1 deletion(-) create mode 100644 src/common/libkvs/kvs_lookup.c create mode 100644 src/common/libkvs/kvs_lookup.h diff --git a/src/common/libkvs/Makefile.am b/src/common/libkvs/Makefile.am index 4e148a7fd598..b8f822d22792 100644 --- a/src/common/libkvs/Makefile.am +++ b/src/common/libkvs/Makefile.am @@ -13,6 +13,7 @@ noinst_LTLIBRARIES = libkvs.la libkvs_la_SOURCES = \ kvs.c \ + kvs_lookup.c \ proto.c \ proto.h \ json_dirent.c \ @@ -20,7 +21,7 @@ libkvs_la_SOURCES = \ kvs_deprecated.h fluxinclude_HEADERS = \ - kvs.h + kvs.h kvs_lookup.h TESTS = \ test_proto.t \ diff --git a/src/common/libkvs/kvs.h b/src/common/libkvs/kvs.h index 0f186352ebbc..fdf2101423db 100644 --- a/src/common/libkvs/kvs.h +++ b/src/common/libkvs/kvs.h @@ -5,6 +5,8 @@ #include #include +#include "kvs_lookup.h" + /* Flags for commit and fence operations */ enum flux_kvs_flags { diff --git a/src/common/libkvs/kvs_lookup.c b/src/common/libkvs/kvs_lookup.c new file mode 100644 index 000000000000..9c3af190dffc --- /dev/null +++ b/src/common/libkvs/kvs_lookup.c @@ -0,0 +1,110 @@ +/*****************************************************************************\ + * Copyright (c) 2017 Lawrence Livermore National Security, LLC. Produced at + * the Lawrence Livermore National Laboratory (cf, AUTHORS, DISCLAIMER.LLNS). + * LLNL-CODE-658032 All rights reserved. + * + * This file is part of the Flux resource manager framework. + * For details, see https://github.com/flux-framework. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the license, or (at your option) + * any later version. + * + * Flux is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the IMPLIED WARRANTY OF MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the terms and conditions of the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. + * See also: http://www.gnu.org/licenses/ +\*****************************************************************************/ + +#if HAVE_CONFIG_H +#include "config.h" +#endif +#include +#include +#include +#include +#include +#include + +#include "kvs_lookup.h" + +flux_future_t *flux_kvs_lookup (flux_t *h, int flags, const char *key) +{ + return flux_rpcf (h, "kvs.get", FLUX_NODEID_ANY, 0, "{s:s s:i}", + "key", key, + "flags", flags); +} + +flux_future_t *flux_kvs_lookupat (flux_t *h, int flags, const char *key, + const char *treeobj) +{ + flux_future_t *f; + json_t *obj = NULL; + + if (!treeobj) { + f = flux_kvs_lookup (h, flags, key); + } + else { + if (!(obj = json_loads (treeobj, 0, NULL))) { + errno = EINVAL; + return NULL; + } + f = flux_rpcf (h, "kvs.get", FLUX_NODEID_ANY, 0, "{s:s s:i s:O}", + "key", key, + "flags", flags, + "rootdir", obj); + } + json_decref (obj); + return f; +} + +int flux_kvs_lookup_get (flux_future_t *f, const char **json_str) +{ + const char *auxkey = "flux::kvs_valstr"; + json_t *obj; + char *s; + + if (!(s = flux_future_aux_get (f, auxkey))) { + if (flux_rpc_getf (f, "{s:o}", "val", &obj) < 0) + return -1; + if (!(s = json_dumps (obj, JSON_COMPACT|JSON_ENCODE_ANY))) { + errno = EINVAL; + return -1; + } + if (flux_future_aux_set (f, auxkey, s, free) < 0) { + free (s); + errno = ENOMEM; + return -1; + } + } + if (json_str) + *json_str = s; + return 0; +} + +int flux_kvs_lookup_getf (flux_future_t *f, const char *fmt, ...) +{ + va_list ap; + json_t *obj; + int rc; + + if (flux_rpc_getf (f, "{s:o}", "val", &obj) < 0) + return -1; + va_start (ap, fmt); + if ((rc = json_vunpack_ex (obj, NULL, 0, fmt, ap) < 0)) + errno = EPROTO; + va_end (ap); + + return rc; +} + + +/* + * vi:tabstop=4 shiftwidth=4 expandtab + */ diff --git a/src/common/libkvs/kvs_lookup.h b/src/common/libkvs/kvs_lookup.h new file mode 100644 index 000000000000..c046e857d0c5 --- /dev/null +++ b/src/common/libkvs/kvs_lookup.h @@ -0,0 +1,21 @@ +#ifndef _FLUX_CORE_KVS_LOOKUP_H +#define _FLUX_CORE_KVS_LOOKUP_H + +enum kvs_lookup_flags { + FLUX_KVS_READDIR = 1, + FLUX_KVS_READLINK = 2, + FLUX_KVS_TREEOBJ = 16, +}; + +flux_future_t *flux_kvs_lookup (flux_t *h, int flags, const char *key); +flux_future_t *flux_kvs_lookupat (flux_t *h, int flags, const char *key, + const char *treeobj); + +int flux_kvs_lookup_get (flux_future_t *f, const char **json_str); +int flux_kvs_lookup_getf (flux_future_t *f, const char *fmt, ...); + +#endif /* !_FLUX_CORE_KVS_LOOKUP_H */ + +/* + * vi:tabstop=4 shiftwidth=4 expandtab + */ From ef28c054bedcc2264608c6b901951823fa861769 Mon Sep 17 00:00:00 2001 From: Jim Garlick Date: Wed, 28 Jun 2017 13:01:40 -0700 Subject: [PATCH 04/19] libkvs/deprecated: reimplement deprecated functions Reimplement deprecated json-c based API functions in a more standalone way, in terms of newer public interfaces. --- src/common/libkvs/Makefile.am | 1 + src/common/libkvs/kvs.c | 72 +----------- src/common/libkvs/kvs_deprecated.c | 179 +++++++++++++++++++++++++++++ 3 files changed, 181 insertions(+), 71 deletions(-) create mode 100644 src/common/libkvs/kvs_deprecated.c diff --git a/src/common/libkvs/Makefile.am b/src/common/libkvs/Makefile.am index b8f822d22792..2d8ef3ce59c8 100644 --- a/src/common/libkvs/Makefile.am +++ b/src/common/libkvs/Makefile.am @@ -18,6 +18,7 @@ libkvs_la_SOURCES = \ proto.h \ json_dirent.c \ json_dirent.h \ + kvs_deprecated.c \ kvs_deprecated.h fluxinclude_HEADERS = \ diff --git a/src/common/libkvs/kvs.c b/src/common/libkvs/kvs.c index e59a3a541ec4..16b9fdf03aca 100644 --- a/src/common/libkvs/kvs.c +++ b/src/common/libkvs/kvs.c @@ -71,7 +71,7 @@ struct kvsdir_iterator_struct { typedef enum { WATCH_STRING, WATCH_INT, WATCH_INT64, WATCH_DOUBLE, - WATCH_BOOLEAN, WATCH_OBJECT, WATCH_JSONSTR, WATCH_DIR, + WATCH_BOOLEAN, WATCH_JSONSTR, WATCH_DIR, } watch_type_t; typedef struct { @@ -376,12 +376,6 @@ int kvs_get_symlinkat (flux_t *h, const char *treeobj, return rc; } -/* deprecated */ -int kvs_get_obj (flux_t *h, const char *key, json_object **val) -{ - return getobj (h, NULL, key, 0, val); -} - int kvs_get_dir (flux_t *h, kvsdir_t **dir, const char *fmt, ...) { va_list ap; @@ -668,11 +662,6 @@ static int dispatch_watch (flux_t *h, kvs_watcher_t *wp, json_object *val) kvsdir_destroy (dir); break; } - case WATCH_OBJECT: { - kvs_set_obj_f set = wp->set; - rc = set (wp->key, val, wp->arg, errnum); - break; - } case WATCH_JSONSTR: { kvs_set_f set = wp->set; rc = set (wp->key, Jtostr (val), wp->arg, errnum); @@ -772,9 +761,6 @@ static int watch_rpc (flux_t *h, const char *key, json_object **val, return ret; } -/* *valp is IN/OUT parameter. - * IN *valp is freed internally. Caller must free OUT *val. - */ static int watch_once_obj (flux_t *h, const char *key, json_object **valp) { int rc = -1; @@ -790,12 +776,6 @@ static int watch_once_obj (flux_t *h, const char *key, json_object **valp) return rc; } -/* deprecated */ -int kvs_watch_once_obj (flux_t *h, const char *key, json_object **valp) -{ - return watch_once_obj (h, key, valp); -} - int kvs_watch_once (flux_t *h, const char *key, char **valp) { json_object *val = NULL; @@ -877,24 +857,6 @@ int kvs_watch_once_dir (flux_t *h, kvsdir_t **dirp, const char *fmt, ...) return rc; } -int kvs_watch_obj (flux_t *h, const char *key, kvs_set_obj_f set, void *arg) -{ - uint32_t matchtag; - kvs_watcher_t *wp; - json_object *val = NULL; - int rc = -1; - - if (watch_rpc (h, key, &val, 0, &matchtag) < 0) - goto done; - wp = add_watcher (h, key, WATCH_OBJECT, matchtag, set, arg); - dispatch_watch (h, wp, val); - rc = 0; -done: - if (val) - json_object_put (val); - return rc; -} - int kvs_watch (flux_t *h, const char *key, kvs_set_f set, void *arg) { uint32_t matchtag; @@ -1088,17 +1050,6 @@ int kvs_put (flux_t *h, const char *key, const char *json_str) return rc; } -int kvs_put_obj (flux_t *h, const char *key, json_object *val) -{ - int rc = -1; - - if (kvs_put (h, key, Jtostr (val)) < 0) - goto done; - rc = 0; -done: - return rc; -} - int kvs_put_string (flux_t *h, const char *key, const char *val) { json_object *o = NULL; @@ -1405,11 +1356,6 @@ static int dirgetobj (kvsdir_t *dir, const char *name, return rc; } -int kvsdir_get_obj (kvsdir_t *dir, const char *name, json_object **valp) -{ - return dirgetobj (dir, name, 0, valp); -} - int kvsdir_get (kvsdir_t *dir, const char *name, char **valp) { json_object *v = NULL; @@ -1560,22 +1506,6 @@ int kvsdir_get_boolean (kvsdir_t *dir, const char *name, bool *valp) return rc; } -int kvsdir_put_obj (kvsdir_t *dir, const char *name, json_object *val) -{ - int rc; - char *key; - - if (dir->rootref) { - errno = EROFS; - return -1; - } - key = kvsdir_key_at (dir, name); - rc = kvs_put (dir->handle, key, Jtostr (val)); - free (key); - - return (rc); -} - int kvsdir_put (kvsdir_t *dir, const char *name, const char *val) { int rc; diff --git a/src/common/libkvs/kvs_deprecated.c b/src/common/libkvs/kvs_deprecated.c new file mode 100644 index 000000000000..245b9d50aede --- /dev/null +++ b/src/common/libkvs/kvs_deprecated.c @@ -0,0 +1,179 @@ +/*****************************************************************************\ + * Copyright (c) 2017 Lawrence Livermore National Security, LLC. Produced at + * the Lawrence Livermore National Laboratory (cf, AUTHORS, DISCLAIMER.LLNS). + * LLNL-CODE-658032 All rights reserved. + * + * This file is part of the Flux resource manager framework. + * For details, see https://github.com/flux-framework. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the license, or (at your option) + * any later version. + * + * Flux is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the IMPLIED WARRANTY OF MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the terms and conditions of the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. + * See also: http://www.gnu.org/licenses/ +\*****************************************************************************/ + +#if HAVE_CONFIG_H +#include "config.h" +#endif +#include +#include +#include +#include +#include + +#include "kvs_deprecated.h" + +/* Get + */ + +static int common_get_obj (flux_t *h, const char *key, json_object **valp) +{ + flux_future_t *f; + const char *json_str; + int rc = -1; + + if (!(f = flux_kvs_lookup (h, 0, key))) + goto done; + if (flux_kvs_lookup_get (f, &json_str) < 0) + goto done; + if (valp) { + if (!(*valp = json_tokener_parse (json_str))) { + errno = ENOMEM; + goto done; + } + } + rc = 0; +done: + flux_future_destroy (f); + return rc; +} + +int kvs_get_obj (flux_t *h, const char *key, json_object **valp) +{ + return common_get_obj (h, key, valp); +} + +int kvsdir_get_obj (kvsdir_t *dir, const char *name, json_object **valp) +{ + flux_t *h = kvsdir_handle (dir); + char *key = kvsdir_key_at (dir, name); + int rc; + + rc = common_get_obj (h, key, valp); + free (key); + return rc; +} + +/* Put + */ + +static int common_put_obj (flux_t *h, const char *key, json_object *val) +{ + int rc = -1; + const char *json_str = NULL; + + if (val) + json_str = json_object_to_json_string (val); + if (kvs_put (h, key, json_str) < 0) + goto done; + rc = 0; +done: + return rc; +} + +int kvs_put_obj (flux_t *h, const char *key, json_object *val) +{ + return common_put_obj (h, key, val); +} + +int kvsdir_put_obj (kvsdir_t *dir, const char *name, json_object *val) +{ + flux_t *h = kvsdir_handle (dir); + char *key = kvsdir_key_at (dir, name); + int rc = -1; + + /* N.B. dropped EROFS check for dir->rootdir != NULL */ + + rc = common_put_obj (h, key, val); + free (key); + return rc; +} + +/* Watch + */ + +struct watch_state { + kvs_set_obj_f cb; + void *arg; +}; + +static int watch_cb (const char *key, const char *val, void *arg, int errnum) +{ + struct watch_state *ws = arg; + json_object *obj = NULL; + int rc; + + if (errnum == 0 && val != NULL) { + if (!(obj = json_tokener_parse (val))) + errnum = ENOMEM; + } + rc = ws->cb (key, obj, arg, errnum); + if (obj) + json_object_put (obj); + return rc; +} + +int kvs_watch_obj (flux_t *h, const char *key, kvs_set_obj_f set, void *arg) +{ + struct watch_state *ws = calloc (1, sizeof (*ws)); + if (!ws) { + errno = ENOMEM; + return -1; + } + ws->cb = set; + ws->arg = arg; + + if (kvs_watch (h, key, watch_cb, ws) < 0) { + free (ws); + return -1; + } + + /* Free the little wrapper struct when handle is destroyed + */ + char auxkey[64]; + snprintf (auxkey, sizeof (auxkey), "flux::kvs_watch_%p", ws); + flux_aux_set (h, auxkey, ws, free); + + return 0; +} + +int kvs_watch_once_obj (flux_t *h, const char *key, json_object **valp) +{ + char *inout = NULL; + + if (*valp) + inout = (char *)json_object_to_json_string (*valp); + if (kvs_watch_once (h, key, &inout) < 0) { + free (inout); + return -1; + } + if (*valp) + json_object_put (*valp); + *valp = inout ? json_tokener_parse (inout) : NULL; + return 0; +} + + +/* + * vi:tabstop=4 shiftwidth=4 expandtab + */ From 52185a7dd6c9bfabd1fd5935492793bf67afc62b Mon Sep 17 00:00:00 2001 From: Jim Garlick Date: Wed, 28 Jun 2017 14:50:19 -0700 Subject: [PATCH 05/19] libkvs/classic: reimplement classic get functions Reimplement kvs_get() and type-specific convenience functions in terms of new lookup API. We will phase these into deprecated status later on. --- src/common/libkvs/Makefile.am | 3 +- src/common/libkvs/kvs.c | 177 ------------------------- src/common/libkvs/kvs.h | 12 +- src/common/libkvs/kvs_classic.c | 220 ++++++++++++++++++++++++++++++++ src/common/libkvs/kvs_classic.h | 23 ++++ 5 files changed, 246 insertions(+), 189 deletions(-) create mode 100644 src/common/libkvs/kvs_classic.c create mode 100644 src/common/libkvs/kvs_classic.h diff --git a/src/common/libkvs/Makefile.am b/src/common/libkvs/Makefile.am index 2d8ef3ce59c8..e195474f8d0e 100644 --- a/src/common/libkvs/Makefile.am +++ b/src/common/libkvs/Makefile.am @@ -18,11 +18,12 @@ libkvs_la_SOURCES = \ proto.h \ json_dirent.c \ json_dirent.h \ + kvs_classic.c \ kvs_deprecated.c \ kvs_deprecated.h fluxinclude_HEADERS = \ - kvs.h kvs_lookup.h + kvs.h kvs_lookup.h kvs_classic.h TESTS = \ test_proto.t \ diff --git a/src/common/libkvs/kvs.c b/src/common/libkvs/kvs.c index 16b9fdf03aca..c1ec97222f17 100644 --- a/src/common/libkvs/kvs.c +++ b/src/common/libkvs/kvs.c @@ -292,41 +292,6 @@ static int getobj (flux_t *h, json_object *rootdir, const char *key, return rc; } -int kvs_get (flux_t *h, const char *key, char **val) -{ - json_object *v = NULL; - - if (getobj (h, NULL, key, 0, &v) < 0) - return -1; - if (val) - *val = xstrdup (Jtostr (v)); - Jput (v); - return 0; -} - -int kvs_getat (flux_t *h, const char *treeobj, - const char *key, char **val) -{ - json_object *v = NULL; - json_object *dirent = NULL; - - if (!treeobj || !key || !(dirent = Jfromstr (treeobj)) - || dirent_validate (dirent) < 0) { - errno = EINVAL; - goto error; - } - if (getobj (h, dirent, key, 0, &v) < 0) - goto error; - if (val) - *val = xstrdup (Jtostr (v)); - Jput (dirent); - return 0; -error: - Jput (v); - Jput (dirent); - return -1; -} - int kvs_get_dirat (flux_t *h, const char *treeobj, const char *key, kvsdir_t **dir) { @@ -349,33 +314,6 @@ int kvs_get_dirat (flux_t *h, const char *treeobj, return rc; } -int kvs_get_symlinkat (flux_t *h, const char *treeobj, - const char *key, char **val) -{ - json_object *v = NULL; - json_object *dirent = NULL; - int rc = -1; - - if (!treeobj || !key || !(dirent = Jfromstr (treeobj)) - || dirent_validate (dirent) < 0) { - errno = EINVAL; - goto done; - } - if (getobj (h, dirent, key, KVS_PROTO_READLINK, &v) < 0) - goto done; - if (json_object_get_type (v) != json_type_string) { - errno = EPROTO; - goto done; - } - if (val) - *val = xstrdup (json_object_get_string (v)); - rc = 0; -done: - Jput (v); - Jput (dirent); - return rc; -} - int kvs_get_dir (flux_t *h, kvsdir_t **dir, const char *fmt, ...) { va_list ap; @@ -406,25 +344,6 @@ int kvs_get_dir (flux_t *h, kvsdir_t **dir, const char *fmt, ...) return rc; } -int kvs_get_symlink (flux_t *h, const char *key, char **val) -{ - json_object *v = NULL; - int rc = -1; - - if (getobj (h, NULL, key, KVS_PROTO_READLINK, &v) < 0) - goto done; - if (json_object_get_type (v) != json_type_string) { - errno = EPROTO; - goto done; - } - if (val) - *val = xstrdup (json_object_get_string (v)); - rc = 0; -done: - Jput (v); - return rc; -} - int kvs_get_treeobj (flux_t *h, const char *key, char **val) { json_object *v = NULL; @@ -443,102 +362,6 @@ int kvs_get_treeobj (flux_t *h, const char *key, char **val) return rc; } -int kvs_get_string (flux_t *h, const char *key, char **val) -{ - json_object *v = NULL; - int rc = -1; - - if (getobj (h, NULL, key, 0, &v) < 0) - goto done; - if (json_object_get_type (v) != json_type_string) { - errno = EPROTO; - goto done; - } - if (val) - *val = xstrdup (json_object_get_string (v)); - rc = 0; -done: - Jput (v); - return rc; -} - -int kvs_get_int (flux_t *h, const char *key, int *val) -{ - json_object *v = NULL; - int rc = -1; - - if (getobj (h, NULL, key, 0, &v) < 0) - goto done; - if (json_object_get_type (v) != json_type_int) { - errno = EPROTO; - goto done; - } - if (val) - *val = json_object_get_int (v); - rc = 0; -done: - Jput (v); - return rc; -} - -int kvs_get_int64 (flux_t *h, const char *key, int64_t *val) -{ - json_object *v = NULL; - int rc = -1; - - if (getobj (h, NULL, key, 0, &v) < 0) - goto done; - if (json_object_get_type (v) != json_type_int) { - errno = EPROTO; - goto done; - } - if (val) - *val = json_object_get_int64 (v); - rc = 0; -done: - Jput (v); - return rc; -} - -int kvs_get_double (flux_t *h, const char *key, double *val) -{ - json_object *v = NULL; - int rc = -1; - - if (getobj (h, NULL, key, 0, &v) < 0) - goto done; - if (json_object_get_type (v) != json_type_double - && json_object_get_type (v) != json_type_int) { - errno = EPROTO; - goto done; - } - if (val) - *val = json_object_get_double (v); - rc = 0; -done: - Jput (v); - return rc; -} - -int kvs_get_boolean (flux_t *h, const char *key, bool *val) -{ - json_object *v = NULL; - int rc = -1; - - if (getobj (h, NULL, key, 0, &v) < 0) - goto done; - if (json_object_get_type (v) != json_type_boolean) { - errno = EPROTO; - goto done; - } - if (val) - *val = json_object_get_boolean (v); - rc = 0; -done: - Jput (v); - return rc; -} - /** ** WATCH **/ diff --git a/src/common/libkvs/kvs.h b/src/common/libkvs/kvs.h index fdf2101423db..3cbba1cb6d9b 100644 --- a/src/common/libkvs/kvs.h +++ b/src/common/libkvs/kvs.h @@ -6,6 +6,7 @@ #include #include "kvs_lookup.h" +#include "kvs_classic.h" /* Flags for commit and fence operations */ @@ -42,15 +43,8 @@ void kvsdir_incref (kvsdir_t *dir); * values must be freed with kvsdir_destroy(). These functions return * -1 on error (errno set), 0 on success. */ -int kvs_get (flux_t *h, const char *key, char **json_str); int kvs_get_dir (flux_t *h, kvsdir_t **dirp, const char *fmt, ...) __attribute__ ((format (printf, 3, 4))); -int kvs_get_string (flux_t *h, const char *key, char **valp); -int kvs_get_int (flux_t *h, const char *key, int *valp); -int kvs_get_int64 (flux_t *h, const char *key, int64_t *valp); -int kvs_get_double (flux_t *h, const char *key, double *valp); -int kvs_get_boolean (flux_t *h, const char *key, bool *valp); -int kvs_get_symlink (flux_t *h, const char *key, char **valp); /* Get treeobj associated with a key. Caller must free. */ @@ -58,12 +52,8 @@ int kvs_get_treeobj (flux_t *h, const char *key, char **treeobj); /* Like kvs_get() but lookup is relative to 'treeobj'. */ -int kvs_getat (flux_t *h, const char *treeobj, - const char *key, char **json_str); int kvs_get_dirat (flux_t *h, const char *treeobj, const char *key, kvsdir_t **dirp); -int kvs_get_symlinkat (flux_t *h, const char *treeobj, - const char *key, char **val); /* kvs_watch* is like kvs_get* except the registered callback is called diff --git a/src/common/libkvs/kvs_classic.c b/src/common/libkvs/kvs_classic.c new file mode 100644 index 000000000000..d3225e0a7202 --- /dev/null +++ b/src/common/libkvs/kvs_classic.c @@ -0,0 +1,220 @@ +/*****************************************************************************\ + * Copyright (c) 2017 Lawrence Livermore National Security, LLC. Produced at + * the Lawrence Livermore National Laboratory (cf, AUTHORS, DISCLAIMER.LLNS). + * LLNL-CODE-658032 All rights reserved. + * + * This file is part of the Flux resource manager framework. + * For details, see https://github.com/flux-framework. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the license, or (at your option) + * any later version. + * + * Flux is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the IMPLIED WARRANTY OF MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the terms and conditions of the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. + * See also: http://www.gnu.org/licenses/ +\*****************************************************************************/ + +#if HAVE_CONFIG_H +#include "config.h" +#endif +#include +#include +#include +#include +#include + +int kvs_get (flux_t *h, const char *key, char **valp) +{ + flux_future_t *f; + const char *json_str; + int rc = -1; + + if (!(f = flux_kvs_lookup (h, 0, key))) + goto done; + if (flux_kvs_lookup_get (f, &json_str) < 0) + goto done; + if (valp) { + if (!(*valp = strdup (json_str))) { + errno = ENOMEM; + goto done; + } + } + rc = 0; +done: + flux_future_destroy (f); + return rc; +} + +int kvs_get_string (flux_t *h, const char *key, char **valp) +{ + flux_future_t *f; + const char *s; + int rc = -1; + + if (!(f = flux_kvs_lookup (h, 0, key))) + goto done; + if (flux_kvs_lookup_getf (f, "s", &s) < 0) + goto done; + if (valp) { + if (!(*valp = strdup (s))) { + errno = ENOMEM; + goto done; + } + } + rc = 0; +done: + flux_future_destroy (f); + return rc; +} + +int kvs_get_int (flux_t *h, const char *key, int *valp) +{ + flux_future_t *f; + int i; + int rc = -1; + + if (!(f = flux_kvs_lookup (h, 0, key))) + goto done; + if (flux_kvs_lookup_getf (f, "i", &i) < 0) + goto done; + if (valp) + *valp = i; + rc = 0; +done: + flux_future_destroy (f); + return rc; +} + +int kvs_get_int64 (flux_t *h, const char *key, int64_t *valp) +{ + flux_future_t *f; + int64_t i; + int rc = -1; + + if (!(f = flux_kvs_lookup (h, 0, key))) + goto done; + if (flux_kvs_lookup_getf (f, "I", &i) < 0) + goto done; + if (valp) + *valp = i; + rc = 0; +done: + flux_future_destroy (f); + return rc; +} + +int kvs_get_double (flux_t *h, const char *key, double *valp) +{ + flux_future_t *f; + double d; + int rc = -1; + + if (!(f = flux_kvs_lookup (h, 0, key))) + goto done; + if (flux_kvs_lookup_getf (f, "F", &d) < 0) + goto done; + if (valp) + *valp = d; + rc = 0; +done: + flux_future_destroy (f); + return rc; +} + +int kvs_get_boolean (flux_t *h, const char *key, bool *valp) +{ + flux_future_t *f; + int b; + int rc = -1; + + if (!(f = flux_kvs_lookup (h, 0, key))) + goto done; + if (flux_kvs_lookup_getf (f, "b", &b) < 0) + goto done; + if (valp) + *valp = b; + rc = 0; +done: + flux_future_destroy (f); + return rc; +} + +int kvs_get_symlink (flux_t *h, const char *key, char **valp) +{ + flux_future_t *f; + const char *s; + int rc = -1; + + if (!(f = flux_kvs_lookup (h, FLUX_KVS_READLINK, key))) + goto done; + if (flux_kvs_lookup_getf (f, "s", &s) < 0) + goto done; + if (valp) { + if (!(*valp = strdup (s))) { + errno = ENOMEM; + goto done; + } + } + rc = 0; +done: + flux_future_destroy (f); + return rc; +} + +int kvs_getat (flux_t *h, const char *treeobj, + const char *key, char **valp) +{ + flux_future_t *f; + const char *json_str; + int rc = -1; + + if (!(f = flux_kvs_lookupat (h, 0, key, treeobj))) + goto done; + if (flux_kvs_lookup_get (f, &json_str) < 0) + goto done; + if (valp) { + if (!(*valp = strdup (json_str))) { + errno = ENOMEM; + goto done; + } + } + rc = 0; +done: + flux_future_destroy (f); + return rc; +} + +int kvs_get_symlinkat (flux_t *h, const char *treeobj, + const char *key, char **valp) +{ + flux_future_t *f; + const char *s; + int rc = -1; + + if (!(f = flux_kvs_lookupat (h, FLUX_KVS_READLINK, key, treeobj))) + goto done; + if (flux_kvs_lookup_getf (f, "s", &s) < 0) + goto done; + if (valp) { + if (!(*valp = strdup (s))) { + errno = ENOMEM; + goto done; + } + } + rc = 0; +done: + flux_future_destroy (f); + return rc; +} + +/* + * vi:tabstop=4 shiftwidth=4 expandtab + */ diff --git a/src/common/libkvs/kvs_classic.h b/src/common/libkvs/kvs_classic.h new file mode 100644 index 000000000000..e67ab8d911ec --- /dev/null +++ b/src/common/libkvs/kvs_classic.h @@ -0,0 +1,23 @@ +#ifndef _FLUX_KVS_CLASSIC_H +#define _FLUX_KVS_CLASSIC_H + +/* These interfaces are on their way to being deprecated */ + +int kvs_get (flux_t *h, const char *key, char **json_str); +int kvs_get_string (flux_t *h, const char *key, char **valp); +int kvs_get_int (flux_t *h, const char *key, int *valp); +int kvs_get_int64 (flux_t *h, const char *key, int64_t *valp); +int kvs_get_double (flux_t *h, const char *key, double *valp); +int kvs_get_boolean (flux_t *h, const char *key, bool *valp); +int kvs_get_symlink (flux_t *h, const char *key, char **valp); + +int kvs_getat (flux_t *h, const char *treeobj, + const char *key, char **json_str); +int kvs_get_symlinkat (flux_t *h, const char *treeobj, + const char *key, char **val); + +#endif /* !_FLUX_KVS_CLASSIC_H */ + +/* + * vi:tabstop=4 shiftwidth=4 expandtab + */ From a5b9a94fa2610a1602a446a42ebbaf737d6f7005 Mon Sep 17 00:00:00 2001 From: Jim Garlick Date: Thu, 29 Jun 2017 10:37:11 -0700 Subject: [PATCH 06/19] libkvs/dir: reimplement kvsdir_t operations Unwind kvsdir_t details from kvs.c. Make it an opaque type to the rest of the KVS client API implementation. Reimplement using jansson. --- src/common/libkvs/Makefile.am | 7 +- src/common/libkvs/kvs.c | 245 +++++++------------------------- src/common/libkvs/kvs.h | 42 +----- src/common/libkvs/kvs_classic.c | 23 +++ src/common/libkvs/kvs_classic.h | 1 + src/common/libkvs/kvs_dir.c | 216 ++++++++++++++++++++++++++++ src/common/libkvs/kvs_dir.h | 49 +++++++ 7 files changed, 350 insertions(+), 233 deletions(-) create mode 100644 src/common/libkvs/kvs_dir.c create mode 100644 src/common/libkvs/kvs_dir.h diff --git a/src/common/libkvs/Makefile.am b/src/common/libkvs/Makefile.am index e195474f8d0e..bee8728e705e 100644 --- a/src/common/libkvs/Makefile.am +++ b/src/common/libkvs/Makefile.am @@ -18,12 +18,17 @@ libkvs_la_SOURCES = \ proto.h \ json_dirent.c \ json_dirent.h \ + kvs_dir.c \ kvs_classic.c \ kvs_deprecated.c \ kvs_deprecated.h + fluxinclude_HEADERS = \ - kvs.h kvs_lookup.h kvs_classic.h + kvs.h \ + kvs_lookup.h \ + kvs_dir.h \ + kvs_classic.h TESTS = \ test_proto.t \ diff --git a/src/common/libkvs/kvs.c b/src/common/libkvs/kvs.c index c1ec97222f17..a433046adaed 100644 --- a/src/common/libkvs/kvs.c +++ b/src/common/libkvs/kvs.c @@ -54,21 +54,6 @@ #include "json_dirent.h" -struct kvsdir_struct { - flux_t *handle; - json_object *rootref; /* optional snapshot reference */ - char *key; - json_object *o; - int count; - int usecount; -}; - -struct kvsdir_iterator_struct { - kvsdir_t *dir; - struct json_object_iterator next; - struct json_object_iterator end; -}; - typedef enum { WATCH_STRING, WATCH_INT, WATCH_INT64, WATCH_DOUBLE, WATCH_BOOLEAN, WATCH_JSONSTR, WATCH_DIR, @@ -125,139 +110,16 @@ static kvsctx_t *getctx (flux_t *h) return ctx; } -/** - ** kvsdir_t primary functions. - ** A kvsdir_t is analagous to posix (DIR *). - **/ - -void kvsdir_incref (kvsdir_t *dir) -{ - dir->usecount++; -} - -void kvsdir_destroy (kvsdir_t *dir) -{ - if (--dir->usecount == 0) { - Jput (dir->rootref); - free (dir->key); - json_object_put (dir->o); - free (dir); - } -} - -static kvsdir_t *kvsdir_alloc (flux_t *handle, json_object *rootref, - const char *key, json_object *o) -{ - kvsdir_t *dir = xzmalloc (sizeof (*dir)); - - dir->handle = handle; - dir->rootref = Jget (rootref); /* may be NULL */ - dir->key = xstrdup (key); - dir->o = o; - json_object_get (dir->o); - dir->count = -1; /* uninitialized */ - dir->usecount = 1; - - return dir; -} - -int kvsdir_get_size (kvsdir_t *dir) -{ - json_object_iter iter; - - if (dir->count == -1) { - dir->count = 0; - json_object_object_foreachC (dir->o, iter) { - dir->count++; - } - } - return dir->count; -} - -const char *kvsdir_key (kvsdir_t *dir) -{ - return dir->key; -} - -void *kvsdir_handle (kvsdir_t *dir) -{ - return dir->handle; -} - -void kvsitr_rewind (kvsitr_t *itr) -{ - itr->next = json_object_iter_begin (itr->dir->o); -} - -kvsitr_t *kvsitr_create (kvsdir_t *dir) -{ - kvsitr_t *itr = xzmalloc (sizeof (*itr)); - - itr->dir = dir; - itr->next = json_object_iter_begin (itr->dir->o); - itr->end = json_object_iter_end (itr->dir->o); - - return itr; -} - -void kvsitr_destroy (kvsitr_t *itr) -{ - free (itr); -} - -const char *kvsitr_next (kvsitr_t *itr) -{ - const char *name = NULL; - - if (!json_object_iter_equal (&itr->end, &itr->next)) { - name = json_object_iter_peek_name (&itr->next); - (void)json_object_iter_next (&itr->next); - } - - return name; -} - -bool kvsdir_exists (kvsdir_t *dir, const char *name) -{ - json_object *dirent; - return (json_object_object_get_ex (dir->o, name, &dirent) && dirent); -} - -bool kvsdir_isdir (kvsdir_t *dir, const char *name) -{ - json_object *dirent; - return (json_object_object_get_ex (dir->o, name, &dirent) - && dirent - && (json_object_object_get_ex (dirent, "DIRREF", NULL) - || json_object_object_get_ex (dirent, "DIRVAL", NULL)) - ); -} - -bool kvsdir_issymlink (kvsdir_t *dir, const char *name) -{ - json_object *dirent; - return (json_object_object_get_ex (dir->o, name, &dirent) - && dirent - && json_object_object_get_ex (dirent, "LINKVAL", NULL)); -} - - -char *kvsdir_key_at (kvsdir_t *dir, const char *name) -{ - if (!strcmp (dir->key, ".")) - return xstrdup (name); - return xasprintf ("%s.%s", dir->key, name); -} - /** ** GET **/ -static int getobj (flux_t *h, json_object *rootdir, const char *key, +static int getobj (flux_t *h, const char *rootref, const char *key, int flags, json_object **val) { flux_future_t *f = NULL; const char *json_str; + json_object *rootref_obj = NULL; json_object *in = NULL; json_object *out = NULL; json_object *v = NULL; @@ -268,7 +130,13 @@ static int getobj (flux_t *h, json_object *rootdir, const char *key, errno = EINVAL; goto done; } - if (!(in = kp_tget_enc (rootdir, key, flags))) + if (rootref) { + if (!(rootref_obj = Jfromstr (rootref))) { + errno = ENOMEM; + goto done; + } + } + if (!(in = kp_tget_enc (rootref_obj, key, flags))) goto done; if (!(f = flux_rpc (h, "kvs.get", Jtostr (in), FLUX_NODEID_ANY, 0))) goto done; @@ -287,30 +155,28 @@ static int getobj (flux_t *h, json_object *rootdir, const char *key, saved_errno = errno; Jput (in); Jput (out); + Jput (rootref_obj); flux_future_destroy (f); errno = saved_errno; return rc; } -int kvs_get_dirat (flux_t *h, const char *treeobj, +int kvs_get_dirat (flux_t *h, const char *rootref, const char *key, kvsdir_t **dir) { json_object *v = NULL; - json_object *rootref = NULL; int rc = -1; - if (!treeobj || !key || !dir || !(rootref = Jfromstr (treeobj)) - || dirent_validate (rootref) < 0) { + if (!rootref || !key || !dir) { errno = EINVAL; goto done; } if (getobj (h, rootref, key, KVS_PROTO_READDIR, &v) < 0) goto done; - *dir = kvsdir_alloc (h, rootref, key, v); + *dir = kvsdir_create (h, rootref, key, Jtostr (v)); rc = 0; done: Jput (v); - Jput (rootref); return rc; } @@ -335,7 +201,7 @@ int kvs_get_dir (flux_t *h, kvsdir_t **dir, const char *fmt, ...) const char *k = strlen (key) > 0 ? key : "."; if (getobj (h, NULL, k, KVS_PROTO_READDIR, &v) < 0) goto done; - *dir = kvsdir_alloc (h, NULL, k, v); + *dir = kvsdir_create (h, NULL, k, Jtostr (v)); rc = 0; done: Jput (v); @@ -344,24 +210,6 @@ int kvs_get_dir (flux_t *h, kvsdir_t **dir, const char *fmt, ...) return rc; } -int kvs_get_treeobj (flux_t *h, const char *key, char **val) -{ - json_object *v = NULL; - const char *s; - int rc = -1; - - if (getobj (h, NULL, key, KVS_PROTO_TREEOBJ, &v) < 0) - goto done; - if (val) { - s = json_object_to_json_string_ext (v, JSON_C_TO_STRING_PLAIN); - *val = xstrdup (s); - } - rc = 0; -done: - Jput (v); - return rc; -} - /** ** WATCH **/ @@ -479,7 +327,8 @@ static int dispatch_watch (flux_t *h, kvs_watcher_t *wp, json_object *val) } case WATCH_DIR: { kvs_set_dir_f set = wp->set; - kvsdir_t *dir = val ? kvsdir_alloc (h, NULL, wp->key, val) : NULL; + kvsdir_t *dir = val ? kvsdir_create (h, NULL, wp->key, Jtostr (val)) + : NULL; rc = set (wp->key, dir, wp->arg, errnum); if (dir) kvsdir_destroy (dir); @@ -659,8 +508,11 @@ int kvs_watch_once_dir (flux_t *h, kvsdir_t **dirp, const char *fmt, ...) va_end (ap); if (*dirp) { - val = (*dirp)->o; - json_object_get (val); + const char *s; + if (!(s = kvsdir_tostring (*dirp))) + goto done; + if (!(val = Jfromstr (s))) + goto done; } if (watch_rpc (h, key, &val, KVS_PROTO_ONCE | KVS_PROTO_READDIR, NULL) < 0) goto done; @@ -670,7 +522,7 @@ int kvs_watch_once_dir (flux_t *h, kvsdir_t **dirp, const char *fmt, ...) } if (*dirp) kvsdir_destroy (*dirp); - *dirp = kvsdir_alloc (h, NULL, key, val); + *dirp = kvsdir_create (h, NULL, key, Jtostr (val)); rc = 0; done: if (val) @@ -1170,11 +1022,12 @@ int kvs_dropcache (flux_t *h) static int dirgetobj (kvsdir_t *dir, const char *name, int flags, json_object **val) { + flux_t *h = kvsdir_handle (dir); char *key; int rc; key = kvsdir_key_at (dir, name); - rc = getobj (dir->handle, dir->rootref, key, flags, val); + rc = getobj (h, kvsdir_rootref (dir), key, flags, val); free (key); return rc; } @@ -1192,6 +1045,7 @@ int kvsdir_get (kvsdir_t *dir, const char *name, char **valp) int kvsdir_get_dir (kvsdir_t *dir, kvsdir_t **dirp, const char *fmt, ...) { + flux_t *h = kvsdir_handle (dir); int rc = -1; char *name, *key; va_list ap; @@ -1203,9 +1057,9 @@ int kvsdir_get_dir (kvsdir_t *dir, kvsdir_t **dirp, const char *fmt, ...) va_end (ap); key = kvsdir_key_at (dir, name); - if (getobj (dir->handle, dir->rootref, key, KVS_PROTO_READDIR, &v) < 0) + if (getobj (h, kvsdir_rootref (dir), key, KVS_PROTO_READDIR, &v) < 0) goto done; - *dirp = kvsdir_alloc (dir->handle, dir->rootref, key, v); + *dirp = kvsdir_create (h, kvsdir_rootref (dir), key, Jtostr (v)); rc = 0; done: Jput (v); @@ -1331,15 +1185,16 @@ int kvsdir_get_boolean (kvsdir_t *dir, const char *name, bool *valp) int kvsdir_put (kvsdir_t *dir, const char *name, const char *val) { + flux_t *h = kvsdir_handle (dir); int rc; char *key; - if (dir->rootref) { + if (kvsdir_rootref (dir) != NULL) { errno = EROFS; return -1; } key = kvsdir_key_at (dir, name); - rc = kvs_put (dir->handle, key, val); + rc = kvs_put (h, key, val); free (key); return (rc); @@ -1347,15 +1202,16 @@ int kvsdir_put (kvsdir_t *dir, const char *name, const char *val) int kvsdir_put_string (kvsdir_t *dir, const char *name, const char *val) { + flux_t *h = kvsdir_handle (dir); int rc; char *key; - if (dir->rootref) { + if (kvsdir_rootref (dir) != NULL) { errno = EROFS; return -1; } key = kvsdir_key_at (dir, name); - rc = kvs_put_string (dir->handle, key, val); + rc = kvs_put_string (h, key, val); free (key); return (rc); @@ -1363,15 +1219,16 @@ int kvsdir_put_string (kvsdir_t *dir, const char *name, const char *val) int kvsdir_put_int (kvsdir_t *dir, const char *name, int val) { + flux_t *h = kvsdir_handle (dir); int rc; char *key; - if (dir->rootref) { + if (kvsdir_rootref (dir) != NULL) { errno = EROFS; return -1; } key = kvsdir_key_at (dir, name); - rc = kvs_put_int (dir->handle, key, val); + rc = kvs_put_int (h, key, val); free (key); return (rc); @@ -1379,15 +1236,16 @@ int kvsdir_put_int (kvsdir_t *dir, const char *name, int val) int kvsdir_put_int64 (kvsdir_t *dir, const char *name, int64_t val) { + flux_t *h = kvsdir_handle (dir); int rc; char *key; - if (dir->rootref) { + if (kvsdir_rootref (dir) != NULL) { errno = EROFS; return -1; } key = kvsdir_key_at (dir, name); - rc = kvs_put_int64 (dir->handle, key, val); + rc = kvs_put_int64 (h, key, val); free (key); return (rc); @@ -1395,15 +1253,16 @@ int kvsdir_put_int64 (kvsdir_t *dir, const char *name, int64_t val) int kvsdir_put_double (kvsdir_t *dir, const char *name, double val) { + flux_t *h = kvsdir_handle (dir); int rc; char *key; - if (dir->rootref) { + if (kvsdir_rootref (dir) != NULL) { errno = EROFS; return -1; } key = kvsdir_key_at (dir, name); - rc = kvs_put_double (dir->handle, key, val); + rc = kvs_put_double (h, key, val); free (key); return (rc); @@ -1411,15 +1270,16 @@ int kvsdir_put_double (kvsdir_t *dir, const char *name, double val) int kvsdir_put_boolean (kvsdir_t *dir, const char *name, bool val) { + flux_t *h = kvsdir_handle (dir); int rc; char *key; - if (dir->rootref) { + if (kvsdir_rootref (dir) != NULL) { errno = EROFS; return -1; } key = kvsdir_key_at (dir, name); - rc = kvs_put_boolean (dir->handle, key, val); + rc = kvs_put_boolean (h, key, val); free (key); return (rc); @@ -1427,15 +1287,16 @@ int kvsdir_put_boolean (kvsdir_t *dir, const char *name, bool val) int kvsdir_mkdir (kvsdir_t *dir, const char *name) { + flux_t *h = kvsdir_handle (dir); int rc; char *key; - if (dir->rootref) { + if (kvsdir_rootref (dir) != NULL) { errno = EROFS; return -1; } key = kvsdir_key_at (dir, name); - rc = kvs_mkdir (dir->handle, key); + rc = kvs_mkdir (h, key); free (key); return (rc); @@ -1443,15 +1304,16 @@ int kvsdir_mkdir (kvsdir_t *dir, const char *name) int kvsdir_symlink (kvsdir_t *dir, const char *name, const char *target) { + flux_t *h = kvsdir_handle (dir); int rc; char *key; - if (dir->rootref) { + if (kvsdir_rootref (dir) != NULL) { errno = EROFS; return -1; } key = kvsdir_key_at (dir, name); - rc = kvs_symlink (dir->handle, key, target); + rc = kvs_symlink (h, key, target); free (key); return (rc); @@ -1459,15 +1321,16 @@ int kvsdir_symlink (kvsdir_t *dir, const char *name, const char *target) int kvsdir_unlink (kvsdir_t *dir, const char *name) { + flux_t *h = kvsdir_handle (dir); int rc; char *key; - if (dir->rootref) { + if (kvsdir_rootref (dir) != NULL) { errno = EROFS; return -1; } key = kvsdir_key_at (dir, name); - rc = kvs_unlink (dir->handle, key); + rc = kvs_unlink (h, key); free (key); return (rc); diff --git a/src/common/libkvs/kvs.h b/src/common/libkvs/kvs.h index 3cbba1cb6d9b..79064b093c02 100644 --- a/src/common/libkvs/kvs.h +++ b/src/common/libkvs/kvs.h @@ -6,6 +6,7 @@ #include #include "kvs_lookup.h" +#include "kvs_dir.h" #include "kvs_classic.h" /* Flags for commit and fence operations @@ -14,8 +15,6 @@ enum flux_kvs_flags { KVS_NO_MERGE = 1, /* disallow commits to be mergeable with others */ }; -typedef struct kvsdir_struct kvsdir_t; - typedef int (*kvs_set_f)(const char *key, const char *json_str, void *arg, int errnum); typedef int (*kvs_set_dir_f)(const char *key, kvsdir_t *dir, void *arg, @@ -30,11 +29,6 @@ typedef int (*kvs_set_double_f)(const char *key, double val, void *arg, typedef int (*kvs_set_boolean_f)(const char *key, bool val, void *arg, int errnum); -/* Destroy a kvsdir object returned from kvs_get_dir() or kvsdir_get_dir() - */ -void kvsdir_destroy (kvsdir_t *dir); -void kvsdir_incref (kvsdir_t *dir); - /* The basic get and put operations, with convenience functions * for simple types. You will get an error if you call kvs_get() * on a directory (return -1, errno = EISDIR). Use kvs_get_dir() which @@ -46,10 +40,6 @@ void kvsdir_incref (kvsdir_t *dir); int kvs_get_dir (flux_t *h, kvsdir_t **dirp, const char *fmt, ...) __attribute__ ((format (printf, 3, 4))); -/* Get treeobj associated with a key. Caller must free. - */ -int kvs_get_treeobj (flux_t *h, const char *key, char **treeobj); - /* Like kvs_get() but lookup is relative to 'treeobj'. */ int kvs_get_dirat (flux_t *h, const char *treeobj, @@ -112,36 +102,6 @@ int kvs_put_boolean (flux_t *h, const char *key, bool val); */ int kvs_put_treeobj (flux_t *h, const char *key, const char *treeobj); -/* An iterator interface for walking the list of names in a kvsdir_t - * returned by kvs_get_dir(). kvsitr_create() always succeeds. - * kvsitr_next() returns NULL when the last item is reached. - */ -typedef struct kvsdir_iterator_struct kvsitr_t; -kvsitr_t *kvsitr_create (kvsdir_t *dir); -void kvsitr_destroy (kvsitr_t *itr); -const char *kvsitr_next (kvsitr_t *itr); -void kvsitr_rewind (kvsitr_t *itr); - -/* Test attributes of 'name', relative to kvsdir object. - * This is intended for testing names returned by kvsitr_next (no recursion). - * Symlinks are not dereferenced, i.e. symlink pointing to dir will read - * issymlink=true, isdir=false. - */ -bool kvsdir_exists (kvsdir_t *dir, const char *name); -bool kvsdir_isdir (kvsdir_t *dir, const char *name); -bool kvsdir_issymlink (kvsdir_t *dir, const char *name); - -/* Get key associated with a directory or directory entry. - * Both functions always succeed. - */ -const char *kvsdir_key (kvsdir_t *dir); -char *kvsdir_key_at (kvsdir_t *dir, const char *key); /* caller frees result */ -void *kvsdir_handle (kvsdir_t *dir); - -/* Get the number of keys in a directory. - */ -int kvsdir_get_size (kvsdir_t *dir); - /* Remove a key from the namespace. If it represents a directory, * its contents are also removed. kvsdir_unlink removes it relative to 'dir'. * Since ordering of put/mkdir/unlink requests within a commit is not defined, diff --git a/src/common/libkvs/kvs_classic.c b/src/common/libkvs/kvs_classic.c index d3225e0a7202..478da193c4bb 100644 --- a/src/common/libkvs/kvs_classic.c +++ b/src/common/libkvs/kvs_classic.c @@ -169,6 +169,28 @@ int kvs_get_symlink (flux_t *h, const char *key, char **valp) return rc; } +int kvs_get_treeobj (flux_t *h, const char *key, char **valp) +{ + flux_future_t *f; + const char *json_str; + int rc = -1; + + if (!(f = flux_kvs_lookup (h, FLUX_KVS_TREEOBJ, key))) + goto done; + if (flux_kvs_lookup_get (f, &json_str) < 0) + goto done; + if (valp) { + if (!(*valp = strdup (json_str))) { + errno = ENOMEM; + goto done; + } + } + rc = 0; +done: + flux_future_destroy (f); + return rc; +} + int kvs_getat (flux_t *h, const char *treeobj, const char *key, char **valp) { @@ -215,6 +237,7 @@ int kvs_get_symlinkat (flux_t *h, const char *treeobj, return rc; } + /* * vi:tabstop=4 shiftwidth=4 expandtab */ diff --git a/src/common/libkvs/kvs_classic.h b/src/common/libkvs/kvs_classic.h index e67ab8d911ec..17f21b57556e 100644 --- a/src/common/libkvs/kvs_classic.h +++ b/src/common/libkvs/kvs_classic.h @@ -10,6 +10,7 @@ int kvs_get_int64 (flux_t *h, const char *key, int64_t *valp); int kvs_get_double (flux_t *h, const char *key, double *valp); int kvs_get_boolean (flux_t *h, const char *key, bool *valp); int kvs_get_symlink (flux_t *h, const char *key, char **valp); +int kvs_get_treeobj (flux_t *h, const char *key, char **valp); int kvs_getat (flux_t *h, const char *treeobj, const char *key, char **json_str); diff --git a/src/common/libkvs/kvs_dir.c b/src/common/libkvs/kvs_dir.c new file mode 100644 index 000000000000..a6eea14be063 --- /dev/null +++ b/src/common/libkvs/kvs_dir.c @@ -0,0 +1,216 @@ +/*****************************************************************************\ + * Copyright (c) 2017 Lawrence Livermore National Security, LLC. Produced at + * the Lawrence Livermore National Laboratory (cf, AUTHORS, DISCLAIMER.LLNS). + * LLNL-CODE-658032 All rights reserved. + * + * This file is part of the Flux resource manager framework. + * For details, see https://github.com/flux-framework. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the license, or (at your option) + * any later version. + * + * Flux is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the IMPLIED WARRANTY OF MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the terms and conditions of the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. + * See also: http://www.gnu.org/licenses/ +\*****************************************************************************/ + +#if HAVE_CONFIG_H +#include "config.h" +#endif +#include +#include +#include +#include +#include +#include + + +struct kvsdir_struct { + flux_t *handle; + char *rootref; /* optional snapshot reference */ + char *key; + json_t *dirobj; + char *dirobj_string; + int usecount; +}; + +struct kvsdir_iterator_struct { + json_t *dirobj; + void *iter; +}; + +void kvsdir_incref (kvsdir_t *dir) +{ + dir->usecount++; +} + +void kvsdir_destroy (kvsdir_t *dir) +{ + if (--dir->usecount == 0) { + free (dir->rootref); + free (dir->key); + json_decref (dir->dirobj); + free (dir); + } +} + +/* If rootref is non-NULL, the kvsdir records the root reference + * so that subsequent kvsdir_get_* accesses can be relative to that + * snapshot. Otherwise, they are relative to the current root. + */ +kvsdir_t *kvsdir_create (flux_t *handle, const char *rootref, + const char *key, const char *json_str) +{ + kvsdir_t *dir; + + if (!(dir = calloc (1, sizeof (*dir)))) + goto nomem; + + dir->handle = handle; + if (rootref) { + if (!(dir->rootref = strdup (rootref))) + goto nomem; + } + if (!(dir->key = strdup (key))) + goto nomem; + if (!(dir->dirobj = json_loads (json_str, 0, NULL))) + goto nomem; + dir->usecount = 1; + + return dir; +nomem: + kvsdir_destroy (dir); + errno = ENOMEM; + return NULL; +} + +const char *kvsdir_tostring (kvsdir_t *dir) +{ + if (!dir->dirobj_string) { + if (!(dir->dirobj_string = json_dumps (dir->dirobj, JSON_COMPACT))) { + errno = ENOMEM; + return NULL; + } + } + return dir->dirobj_string; +} + +int kvsdir_get_size (kvsdir_t *dir) +{ + return json_object_size (dir->dirobj); +} + +const char *kvsdir_key (kvsdir_t *dir) +{ + return dir->key; +} + +void *kvsdir_handle (kvsdir_t *dir) +{ + return dir->handle; +} + +const char *kvsdir_rootref (kvsdir_t *dir) +{ + return dir->rootref; +} + +void kvsitr_destroy (kvsitr_t *itr) +{ + if (itr) { + free (itr); + } +} + +kvsitr_t *kvsitr_create (kvsdir_t *dir) +{ + kvsitr_t *itr; + + if (!(itr = calloc (1, sizeof (*itr)))) + goto nomem; + itr->dirobj = dir->dirobj; + itr->iter = json_object_iter (itr->dirobj); + + return itr; +nomem: + kvsitr_destroy (itr); + errno = ENOMEM; + return NULL; +} + +void kvsitr_rewind (kvsitr_t *itr) +{ + itr->iter = json_object_iter (itr->dirobj); +} + +const char *kvsitr_next (kvsitr_t *itr) +{ + const char *name = NULL; + + if (itr->iter) { + name = json_object_iter_key (itr->iter); + itr->iter = json_object_iter_next (itr->dirobj, itr->iter); + } + return name; +} + +bool kvsdir_exists (kvsdir_t *dir, const char *name) +{ + if (json_object_get (dir->dirobj, name)) + return true; + return false; +} + +bool kvsdir_isdir (kvsdir_t *dir, const char *name) +{ + json_t *obj = json_object_get (dir->dirobj, name); + + if (obj) { + if (json_object_get (obj, "DIRREF") || json_object_get (obj, "DIRVAL")) + return true; + } + return false; +} + +bool kvsdir_issymlink (kvsdir_t *dir, const char *name) +{ + json_t *obj = json_object_get (dir->dirobj, name); + + if (obj) { + if (json_object_get (obj, "LINKVAL")) + return true; + } + return false; +} + + +char *kvsdir_key_at (kvsdir_t *dir, const char *name) +{ + char *s; + + if (!strcmp (dir->key, ".")) { + if (!(s = strdup (name))) + goto nomem; + } + else { + if (asprintf (&s, "%s.%s", dir->key, name) < 0) + goto nomem; + } + return s; +nomem: + errno = ENOMEM; + return NULL; +} + + +/* + * vi:tabstop=4 shiftwidth=4 expandtab + */ diff --git a/src/common/libkvs/kvs_dir.h b/src/common/libkvs/kvs_dir.h new file mode 100644 index 000000000000..23ffa39f89e9 --- /dev/null +++ b/src/common/libkvs/kvs_dir.h @@ -0,0 +1,49 @@ +#ifndef _FLUX_CORE_KVS_DIR_H +#define _FLUX_CORE_KVS_DIR_H + +typedef struct kvsdir_struct kvsdir_t; +typedef struct kvsdir_iterator_struct kvsitr_t; + +/* Destroy a kvsdir object returned from kvs_get_dir() or kvsdir_get_dir() + */ +kvsdir_t *kvsdir_create (flux_t *handle, const char *rootref, + const char *key, const char *json_str); +void kvsdir_destroy (kvsdir_t *dir); +void kvsdir_incref (kvsdir_t *dir); +const char *kvsdir_tostring (kvsdir_t *dir); + +/* An iterator interface for walking the list of names in a kvsdir_t + * returned by kvs_get_dir(). kvsitr_create() always succeeds. + * kvsitr_next() returns NULL when the last item is reached. + */ +kvsitr_t *kvsitr_create (kvsdir_t *dir); +void kvsitr_destroy (kvsitr_t *itr); +const char *kvsitr_next (kvsitr_t *itr); +void kvsitr_rewind (kvsitr_t *itr); + +/* Test attributes of 'name', relative to kvsdir object. + * This is intended for testing names returned by kvsitr_next (no recursion). + * Symlinks are not dereferenced, i.e. symlink pointing to dir will read + * issymlink=true, isdir=false. + */ +bool kvsdir_exists (kvsdir_t *dir, const char *name); +bool kvsdir_isdir (kvsdir_t *dir, const char *name); +bool kvsdir_issymlink (kvsdir_t *dir, const char *name); + +/* Get key associated with a directory or directory entry. + * Both functions always succeed. + */ +const char *kvsdir_key (kvsdir_t *dir); +char *kvsdir_key_at (kvsdir_t *dir, const char *key); /* caller frees result */ +void *kvsdir_handle (kvsdir_t *dir); +const char *kvsdir_rootref (kvsdir_t *dir); + +/* Get the number of keys in a directory. + */ +int kvsdir_get_size (kvsdir_t *dir); + +#endif /* !_FLUX_CORE_KVS_DIR_H */ + +/* + * vi:tabstop=4 shiftwidth=4 expandtab + */ From 60a7789cb773227471099ca0d9bf78e44286c587 Mon Sep 17 00:00:00 2001 From: Jim Garlick Date: Thu, 29 Jun 2017 12:49:44 -0700 Subject: [PATCH 07/19] libkvs/classic: reimplement kvsdir_get_*() --- src/common/libkvs/kvs.c | 214 ------------------------- src/common/libkvs/kvs.h | 27 ---- src/common/libkvs/kvs_classic.c | 271 ++++++++++++++++++++++++++++++++ src/common/libkvs/kvs_classic.h | 15 ++ 4 files changed, 286 insertions(+), 241 deletions(-) diff --git a/src/common/libkvs/kvs.c b/src/common/libkvs/kvs.c index a433046adaed..ec0c5bc478ff 100644 --- a/src/common/libkvs/kvs.c +++ b/src/common/libkvs/kvs.c @@ -49,7 +49,6 @@ #include "src/common/libutil/xzmalloc.h" #include "src/common/libutil/blobref.h" -#include "kvs_deprecated.h" #include "proto.h" #include "json_dirent.h" @@ -161,55 +160,6 @@ static int getobj (flux_t *h, const char *rootref, const char *key, return rc; } -int kvs_get_dirat (flux_t *h, const char *rootref, - const char *key, kvsdir_t **dir) -{ - json_object *v = NULL; - int rc = -1; - - if (!rootref || !key || !dir) { - errno = EINVAL; - goto done; - } - if (getobj (h, rootref, key, KVS_PROTO_READDIR, &v) < 0) - goto done; - *dir = kvsdir_create (h, rootref, key, Jtostr (v)); - rc = 0; -done: - Jput (v); - return rc; -} - -int kvs_get_dir (flux_t *h, kvsdir_t **dir, const char *fmt, ...) -{ - va_list ap; - char *key = NULL; - json_object *v = NULL; - int rc = -1; - - if (!h || !dir || !fmt) { - errno = EINVAL; - goto done; - } - va_start (ap, fmt); - key = xvasprintf (fmt, ap); - va_end (ap); - - /* N.B. python kvs tests use empty string key for some reason. - * Don't break them for now. - */ - const char *k = strlen (key) > 0 ? key : "."; - if (getobj (h, NULL, k, KVS_PROTO_READDIR, &v) < 0) - goto done; - *dir = kvsdir_create (h, NULL, k, Jtostr (v)); - rc = 0; -done: - Jput (v); - if (key) - free (key); - return rc; -} - /** ** WATCH **/ @@ -1019,170 +969,6 @@ int kvs_dropcache (flux_t *h) ** kvsdir_t convenience functions **/ -static int dirgetobj (kvsdir_t *dir, const char *name, - int flags, json_object **val) -{ - flux_t *h = kvsdir_handle (dir); - char *key; - int rc; - - key = kvsdir_key_at (dir, name); - rc = getobj (h, kvsdir_rootref (dir), key, flags, val); - free (key); - return rc; -} - -int kvsdir_get (kvsdir_t *dir, const char *name, char **valp) -{ - json_object *v = NULL; - if (dirgetobj (dir, name, 0, &v) < 0) - return -1; - if (valp) - *valp = xstrdup (Jtostr (v)); - Jput (v); - return 0; -} - -int kvsdir_get_dir (kvsdir_t *dir, kvsdir_t **dirp, const char *fmt, ...) -{ - flux_t *h = kvsdir_handle (dir); - int rc = -1; - char *name, *key; - va_list ap; - json_object *v = NULL; - - va_start (ap, fmt); - if (vasprintf (&name, fmt, ap) < 0) - oom (); - va_end (ap); - - key = kvsdir_key_at (dir, name); - if (getobj (h, kvsdir_rootref (dir), key, KVS_PROTO_READDIR, &v) < 0) - goto done; - *dirp = kvsdir_create (h, kvsdir_rootref (dir), key, Jtostr (v)); - rc = 0; -done: - Jput (v); - free (key); - free (name); - return rc; -} - -int kvsdir_get_symlink (kvsdir_t *dir, const char *name, char **valp) -{ - int rc = -1; - json_object *v = NULL; - - if (dirgetobj (dir, name, KVS_PROTO_READLINK, &v) < 0) - goto done; - if (json_object_get_type (v) != json_type_string) { - errno = EPROTO; - goto done; - } - if (valp) - *valp = xstrdup (json_object_get_string (v)); - rc = 0; -done: - Jput (v); - return rc; -} - -int kvsdir_get_string (kvsdir_t *dir, const char *name, char **valp) -{ - json_object *v = NULL; - int rc = -1; - - if (dirgetobj (dir, name, 0, &v) < 0) - goto done; - if (json_object_get_type (v) != json_type_string) { - errno = EPROTO; - goto done; - } - if (valp) - *valp = xstrdup (json_object_get_string (v)); - rc = 0; -done: - Jput (v); - return rc; -} - -int kvsdir_get_int (kvsdir_t *dir, const char *name, int *valp) -{ - json_object *v = NULL; - int rc = -1; - - if (dirgetobj (dir, name, 0, &v) < 0) - goto done; - if (json_object_get_type (v) != json_type_int) { - errno = EPROTO; - goto done; - } - if (valp) - *valp = json_object_get_int (v); - rc = 0; -done: - Jput (v); - return rc; -} - -int kvsdir_get_int64 (kvsdir_t *dir, const char *name, int64_t *valp) -{ - json_object *v = NULL; - int rc = -1; - - if (dirgetobj (dir, name, 0, &v) < 0) - goto done; - if (json_object_get_type (v) != json_type_int) { - errno = EPROTO; - goto done; - } - if (valp) - *valp = json_object_get_int64 (v); - rc = 0; -done: - Jput (v); - return rc; -} - -int kvsdir_get_double (kvsdir_t *dir, const char *name, double *valp) -{ - json_object *v = NULL; - int rc = -1; - - if (dirgetobj (dir, name, 0, &v) < 0) - goto done; - if (json_object_get_type (v) != json_type_double - && json_object_get_type (v) != json_type_int) { - errno = EPROTO; - goto done; - } - if (valp) - *valp = json_object_get_double (v); - rc = 0; -done: - Jput (v); - return rc; -} - -int kvsdir_get_boolean (kvsdir_t *dir, const char *name, bool *valp) -{ - json_object *v = NULL; - int rc = -1; - - if (dirgetobj (dir, name, 0, &v) < 0) - goto done; - if (json_object_get_type (v) != json_type_boolean) { - errno = EPROTO; - goto done; - } - if (valp) - *valp = json_object_get_boolean (v); - rc = 0; -done: - Jput (v); - return rc; -} - int kvsdir_put (kvsdir_t *dir, const char *name, const char *val) { flux_t *h = kvsdir_handle (dir); diff --git a/src/common/libkvs/kvs.h b/src/common/libkvs/kvs.h index 79064b093c02..787275c59ba7 100644 --- a/src/common/libkvs/kvs.h +++ b/src/common/libkvs/kvs.h @@ -29,23 +29,6 @@ typedef int (*kvs_set_double_f)(const char *key, double val, void *arg, typedef int (*kvs_set_boolean_f)(const char *key, bool val, void *arg, int errnum); -/* The basic get and put operations, with convenience functions - * for simple types. You will get an error if you call kvs_get() - * on a directory (return -1, errno = EISDIR). Use kvs_get_dir() which - * returns the opaque kvsdir_t type. kvs_get() and kvs_get_string() - * return values that must be freed with free(). kvs_get_dir() return - * values must be freed with kvsdir_destroy(). These functions return - * -1 on error (errno set), 0 on success. - */ -int kvs_get_dir (flux_t *h, kvsdir_t **dirp, const char *fmt, ...) - __attribute__ ((format (printf, 3, 4))); - -/* Like kvs_get() but lookup is relative to 'treeobj'. - */ -int kvs_get_dirat (flux_t *h, const char *treeobj, - const char *key, kvsdir_t **dirp); - - /* kvs_watch* is like kvs_get* except the registered callback is called * to set the value. It will be called immediately to set the initial * value and again each time the value changes. @@ -185,16 +168,6 @@ int kvs_dropcache (flux_t *h); * They behave exactly like their kvs_ counterparts, except the 'key' path * is resolved relative to the directory. */ -int kvsdir_get (kvsdir_t *dir, const char *key, char **json_str); -int kvsdir_get_dir (kvsdir_t *dir, kvsdir_t **dirp, const char *fmt, ...) - __attribute__ ((format (printf, 3, 4))); -int kvsdir_get_string (kvsdir_t *dir, const char *key, char **valp); -int kvsdir_get_int (kvsdir_t *dir, const char *key, int *valp); -int kvsdir_get_int64 (kvsdir_t *dir, const char *key, int64_t *valp); -int kvsdir_get_double (kvsdir_t *dir, const char *key, double *valp); -int kvsdir_get_boolean (kvsdir_t *dir, const char *key, bool *valp); -int kvsdir_get_symlink (kvsdir_t *dir, const char *key, char **valp); - int kvsdir_put (kvsdir_t *dir, const char *key, const char *json_str); int kvsdir_put_string (kvsdir_t *dir, const char *key, const char *val); int kvsdir_put_int (kvsdir_t *dir, const char *key, int val); diff --git a/src/common/libkvs/kvs_classic.c b/src/common/libkvs/kvs_classic.c index 478da193c4bb..cd5fbf443215 100644 --- a/src/common/libkvs/kvs_classic.c +++ b/src/common/libkvs/kvs_classic.c @@ -147,6 +147,42 @@ int kvs_get_boolean (flux_t *h, const char *key, bool *valp) return rc; } +int kvs_get_dir (flux_t *h, kvsdir_t **dir, const char *fmt, ...) +{ + flux_future_t *f = NULL; + const char *json_str; + va_list ap; + char *key = NULL; + int rc = -1; + + if (!h || !dir || !fmt) { + errno = EINVAL; + goto done; + } + va_start (ap, fmt); + if (vasprintf (&key, fmt, ap) < 0) + errno = ENOMEM; + va_end (ap); + if (!key) + goto done; + /* N.B. python kvs tests use empty string key for some reason. + * Don't break them for now. + */ + const char *k = strlen (key) > 0 ? key : "."; + + if (!(f = flux_kvs_lookup (h, FLUX_KVS_READDIR, k))) + goto done; + if (flux_kvs_lookup_get (f, &json_str) < 0) + goto done; + if (!(*dir = kvsdir_create (h, NULL, k, json_str))) + goto done; + rc = 0; +done: + free (key); + flux_future_destroy (f); + return rc; +} + int kvs_get_symlink (flux_t *h, const char *key, char **valp) { flux_future_t *f; @@ -214,6 +250,29 @@ int kvs_getat (flux_t *h, const char *treeobj, return rc; } +int kvs_get_dirat (flux_t *h, const char *rootref, + const char *key, kvsdir_t **dir) +{ + flux_future_t *f = NULL; + const char *json_str; + int rc = -1; + + if (!rootref || !key || !dir) { + errno = EINVAL; + goto done; + } + if (!(f = flux_kvs_lookupat (h, FLUX_KVS_READDIR, key, rootref))) + goto done; + if (flux_kvs_lookup_get (f, &json_str) < 0) + goto done; + if (!(*dir = kvsdir_create (h, rootref, key, json_str))) + goto done; + rc = 0; +done: + flux_future_destroy (f); + return rc; +} + int kvs_get_symlinkat (flux_t *h, const char *treeobj, const char *key, char **valp) { @@ -237,6 +296,218 @@ int kvs_get_symlinkat (flux_t *h, const char *treeobj, return rc; } +int kvsdir_get (kvsdir_t *dir, const char *name, char **valp) +{ + flux_t *h = kvsdir_handle (dir); + const char *rootref = kvsdir_rootref (dir); + flux_future_t *f = NULL; + const char *json_str; + char *key; + int rc = -1; + + if (!(key = kvsdir_key_at (dir, name))) + goto done; + if (!(f = flux_kvs_lookupat (h, 0, key, rootref))) + goto done; + if (flux_kvs_lookup_get (f, &json_str) < 0) + goto done; + if (valp) { + if (!(*valp = strdup (json_str))) { + errno = ENOMEM; + goto done; + } + } + rc = 0; +done: + free (key); + flux_future_destroy (f); + return rc; +} + +int kvsdir_get_dir (kvsdir_t *dir, kvsdir_t **dirp, const char *fmt, ...) +{ + flux_t *h = kvsdir_handle (dir); + const char *rootref = kvsdir_rootref (dir); + flux_future_t *f = NULL; + va_list ap; + const char *json_str; + char *name = NULL; + char *key = NULL; + int rc = -1; + + va_start (ap, fmt); + if (vasprintf (&name, fmt, ap) < 0) + errno = ENOMEM; + va_end (ap); + if (!name) + goto done; + if (!(key = kvsdir_key_at (dir, name))) + goto done; + if (!(f = flux_kvs_lookupat (h, FLUX_KVS_READDIR, key, rootref))) + goto done; + if (flux_kvs_lookup_get (f, &json_str) < 0) + goto done; + if (!(*dirp = kvsdir_create (h, rootref, key, json_str))) + goto done; + rc = 0; +done: + free (key); + free (name); + flux_future_destroy (f); + return rc; +} + +int kvsdir_get_symlink (kvsdir_t *dir, const char *name, char **valp) +{ + flux_t *h = kvsdir_handle (dir); + const char *rootref = kvsdir_rootref (dir); + flux_future_t *f = NULL; + const char *s; + char *key; + int rc = -1; + + if (!(key = kvsdir_key_at (dir, name))) + goto done; + if (!(f = flux_kvs_lookupat (h, FLUX_KVS_READLINK, key, rootref))) + goto done; + if (flux_kvs_lookup_getf (f, "s", &s) < 0) + goto done; + if (valp) { + if (!(*valp = strdup (s))) { + errno = ENOMEM; + goto done; + } + } + rc = 0; +done: + free (key); + flux_future_destroy (f); + return rc; +} + +int kvsdir_get_string (kvsdir_t *dir, const char *name, char **valp) +{ + flux_t *h = kvsdir_handle (dir); + const char *rootref = kvsdir_rootref (dir); + flux_future_t *f = NULL; + const char *s; + char *key; + int rc = -1; + + if (!(key = kvsdir_key_at (dir, name))) + goto done; + if (!(f = flux_kvs_lookupat (h, 0, key, rootref))) + goto done; + if (flux_kvs_lookup_getf (f, "s", &s) < 0) + goto done; + if (valp) { + if (!(*valp = strdup (s))) { + errno = ENOMEM; + goto done; + } + } + rc = 0; +done: + free (key); + flux_future_destroy (f); + return rc; +} + +int kvsdir_get_int (kvsdir_t *dir, const char *name, int *valp) +{ + flux_t *h = kvsdir_handle (dir); + const char *rootref = kvsdir_rootref (dir); + flux_future_t *f = NULL; + int i; + char *key; + int rc = -1; + + if (!(key = kvsdir_key_at (dir, name))) + goto done; + if (!(f = flux_kvs_lookupat (h, 0, key, rootref))) + goto done; + if (flux_kvs_lookup_getf (f, "i", &i) < 0) + goto done; + if (valp) + *valp = i; + rc = 0; +done: + free (key); + flux_future_destroy (f); + return rc; +} + +int kvsdir_get_int64 (kvsdir_t *dir, const char *name, int64_t *valp) +{ + flux_t *h = kvsdir_handle (dir); + const char *rootref = kvsdir_rootref (dir); + flux_future_t *f = NULL; + int64_t i; + char *key; + int rc = -1; + + if (!(key = kvsdir_key_at (dir, name))) + goto done; + if (!(f = flux_kvs_lookupat (h, 0, key, rootref))) + goto done; + if (flux_kvs_lookup_getf (f, "I", &i) < 0) + goto done; + if (valp) + *valp = i; + rc = 0; +done: + free (key); + flux_future_destroy (f); + return rc; +} + +int kvsdir_get_double (kvsdir_t *dir, const char *name, double *valp) +{ + flux_t *h = kvsdir_handle (dir); + const char *rootref = kvsdir_rootref (dir); + flux_future_t *f = NULL; + double d; + char *key; + int rc = -1; + + if (!(key = kvsdir_key_at (dir, name))) + goto done; + if (!(f = flux_kvs_lookupat (h, 0, key, rootref))) + goto done; + if (flux_kvs_lookup_getf (f, "F", &d) < 0) + goto done; + if (valp) + *valp = d; + rc = 0; +done: + free (key); + flux_future_destroy (f); + return rc; +} + +int kvsdir_get_boolean (kvsdir_t *dir, const char *name, bool *valp) +{ + flux_t *h = kvsdir_handle (dir); + const char *rootref = kvsdir_rootref (dir); + flux_future_t *f = NULL; + int i; + char *key; + int rc = -1; + + if (!(key = kvsdir_key_at (dir, name))) + goto done; + if (!(f = flux_kvs_lookupat (h, 0, key, rootref))) + goto done; + if (flux_kvs_lookup_getf (f, "b", &i) < 0) + goto done; + if (valp) + *valp = i; + rc = 0; +done: + free (key); + flux_future_destroy (f); + return rc; +} /* * vi:tabstop=4 shiftwidth=4 expandtab diff --git a/src/common/libkvs/kvs_classic.h b/src/common/libkvs/kvs_classic.h index 17f21b57556e..b909b1c34b8c 100644 --- a/src/common/libkvs/kvs_classic.h +++ b/src/common/libkvs/kvs_classic.h @@ -11,12 +11,27 @@ int kvs_get_double (flux_t *h, const char *key, double *valp); int kvs_get_boolean (flux_t *h, const char *key, bool *valp); int kvs_get_symlink (flux_t *h, const char *key, char **valp); int kvs_get_treeobj (flux_t *h, const char *key, char **valp); +int kvs_get_dir (flux_t *h, kvsdir_t **dirp, const char *fmt, ...) + __attribute__ ((format (printf, 3, 4))); int kvs_getat (flux_t *h, const char *treeobj, const char *key, char **json_str); +int kvs_get_dirat (flux_t *h, const char *treeobj, + const char *key, kvsdir_t **dirp); int kvs_get_symlinkat (flux_t *h, const char *treeobj, const char *key, char **val); +int kvsdir_get (kvsdir_t *dir, const char *key, char **json_str); +int kvsdir_get_dir (kvsdir_t *dir, kvsdir_t **dirp, const char *fmt, ...) + __attribute__ ((format (printf, 3, 4))); +int kvsdir_get_string (kvsdir_t *dir, const char *key, char **valp); +int kvsdir_get_int (kvsdir_t *dir, const char *key, int *valp); +int kvsdir_get_int64 (kvsdir_t *dir, const char *key, int64_t *valp); +int kvsdir_get_double (kvsdir_t *dir, const char *key, double *valp); +int kvsdir_get_boolean (kvsdir_t *dir, const char *key, bool *valp); +int kvsdir_get_symlink (kvsdir_t *dir, const char *key, char **valp); + + #endif /* !_FLUX_KVS_CLASSIC_H */ /* From 90c3877af71f71de981e81f1af26bf9d895398dc Mon Sep 17 00:00:00 2001 From: Jim Garlick Date: Thu, 29 Jun 2017 16:31:39 -0700 Subject: [PATCH 08/19] libkvs: simplify kvs_copy() Now that all the other "get" functions have been reimplemented, the last user of the internal function getobj() is ks kvs_copy(). Implement it using the new lookup functions and get rid of getobj(). --- src/common/libkvs/kvs.c | 76 ++++++++++------------------------------- 1 file changed, 18 insertions(+), 58 deletions(-) diff --git a/src/common/libkvs/kvs.c b/src/common/libkvs/kvs.c index ec0c5bc478ff..0cdf0444480f 100644 --- a/src/common/libkvs/kvs.c +++ b/src/common/libkvs/kvs.c @@ -109,57 +109,6 @@ static kvsctx_t *getctx (flux_t *h) return ctx; } -/** - ** GET - **/ - -static int getobj (flux_t *h, const char *rootref, const char *key, - int flags, json_object **val) -{ - flux_future_t *f = NULL; - const char *json_str; - json_object *rootref_obj = NULL; - json_object *in = NULL; - json_object *out = NULL; - json_object *v = NULL; - int saved_errno; - int rc = -1; - - if (!h || !key) { - errno = EINVAL; - goto done; - } - if (rootref) { - if (!(rootref_obj = Jfromstr (rootref))) { - errno = ENOMEM; - goto done; - } - } - if (!(in = kp_tget_enc (rootref_obj, key, flags))) - goto done; - if (!(f = flux_rpc (h, "kvs.get", Jtostr (in), FLUX_NODEID_ANY, 0))) - goto done; - if (flux_rpc_get (f, &json_str) < 0) - goto done; - if (!json_str || !(out = Jfromstr (json_str))) { - errno = EPROTO; - goto done; - } - if (kp_rget_dec (out, NULL, &v) < 0) - goto done; - if (val) - *val = Jget (v); - rc = 0; -done: - saved_errno = errno; - Jput (in); - Jput (out); - Jput (rootref_obj); - flux_future_destroy (f); - errno = saved_errno; - return rc; -} - /** ** WATCH **/ @@ -1124,14 +1073,25 @@ int kvsdir_unlink (kvsdir_t *dir, const char *name) int kvs_copy (flux_t *h, const char *from, const char *to) { - json_object *dirent; - if (getobj (h, NULL, from, KVS_PROTO_TREEOBJ, &dirent) < 0) - return -1; - if (kvs_put_dirent (h, to, dirent) < 0) { - Jput (dirent); - return -1; + flux_future_t *f; + const char *json_str; + json_object *dirent = NULL; + int rc = -1; + + if (!(f = flux_kvs_lookup (h, FLUX_KVS_TREEOBJ, from))) + goto done; + if (flux_kvs_lookup_get (f, &json_str) < 0) + goto done; + if (!(dirent = Jfromstr (json_str))) { + errno = EPROTO; + goto done; } - return 0; + if (kvs_put_dirent (h, to, dirent) < 0) // steals dirent reference + goto done; + rc = 0; +done: + flux_future_destroy (f); + return rc; } int kvs_move (flux_t *h, const char *from, const char *to) From 45022c6a2c9e1fdb2eac7f909a63558a505cbcdd Mon Sep 17 00:00:00 2001 From: Jim Garlick Date: Thu, 29 Jun 2017 16:43:04 -0700 Subject: [PATCH 09/19] libkvs/watch: split watch out to its own module --- src/common/libkvs/Makefile.am | 4 +- src/common/libkvs/kvs.c | 494 +---------------------------- src/common/libkvs/kvs.h | 56 +--- src/common/libkvs/kvs_watch.c | 569 ++++++++++++++++++++++++++++++++++ src/common/libkvs/kvs_watch.h | 63 ++++ 5 files changed, 639 insertions(+), 547 deletions(-) create mode 100644 src/common/libkvs/kvs_watch.c create mode 100644 src/common/libkvs/kvs_watch.h diff --git a/src/common/libkvs/Makefile.am b/src/common/libkvs/Makefile.am index bee8728e705e..23156bdca8d3 100644 --- a/src/common/libkvs/Makefile.am +++ b/src/common/libkvs/Makefile.am @@ -21,13 +21,15 @@ libkvs_la_SOURCES = \ kvs_dir.c \ kvs_classic.c \ kvs_deprecated.c \ - kvs_deprecated.h + kvs_deprecated.h \ + kvs_watch.c fluxinclude_HEADERS = \ kvs.h \ kvs_lookup.h \ kvs_dir.h \ + kvs_watch.h \ kvs_classic.h TESTS = \ diff --git a/src/common/libkvs/kvs.c b/src/common/libkvs/kvs.c index 0cdf0444480f..ffbc1ab8824c 100644 --- a/src/common/libkvs/kvs.c +++ b/src/common/libkvs/kvs.c @@ -53,39 +53,17 @@ #include "json_dirent.h" -typedef enum { - WATCH_STRING, WATCH_INT, WATCH_INT64, WATCH_DOUBLE, - WATCH_BOOLEAN, WATCH_JSONSTR, WATCH_DIR, -} watch_type_t; - -typedef struct { - watch_type_t type; - void *set; - void *arg; - flux_t *h; - char *key; - uint32_t matchtag; -} kvs_watcher_t; - typedef struct { - zhash_t *watchers; /* kvs_watch_t hashed by stringified matchtag */ - flux_msg_handler_t *w; json_object *ops; /* JSON array of put, unlink, etc operations */ zhash_t *fence_ops; json_object *fence_context; } kvsctx_t; -static void watch_response_cb (flux_t *h, flux_msg_handler_t *w, - const flux_msg_t *msg, void *arg); - static void freectx (void *arg) { kvsctx_t *ctx = arg; if (ctx) { - zhash_destroy (&ctx->watchers); zhash_destroy (&ctx->fence_ops); - if (ctx->w) - flux_msg_handler_destroy (ctx->w); Jput (ctx->ops); free (ctx); } @@ -93,482 +71,16 @@ static void freectx (void *arg) static kvsctx_t *getctx (flux_t *h) { - kvsctx_t *ctx = (kvsctx_t *)flux_aux_get (h, "kvscli"); - struct flux_match match = FLUX_MATCH_RESPONSE; + const char *auxkey = "flux::kvs_client"; + kvsctx_t *ctx = (kvsctx_t *)flux_aux_get (h, auxkey); if (!ctx) { ctx = xzmalloc (sizeof (*ctx)); - if (!(ctx->watchers = zhash_new ())) - oom (); - match.topic_glob = "kvs.watch"; - if (!(ctx->w = flux_msg_handler_create (h, match, watch_response_cb, - ctx))) - oom (); - flux_aux_set (h, "kvscli", ctx, freectx); + flux_aux_set (h, auxkey, ctx, freectx); } return ctx; } -/** - ** WATCH - **/ - -static void destroy_watcher (void *arg) -{ - kvs_watcher_t *wp = arg; - free (wp->key); - flux_matchtag_free (wp->h, wp->matchtag); - free (wp); -} - -static kvs_watcher_t *add_watcher (flux_t *h, const char *key, watch_type_t type, - uint32_t matchtag, void *fun, void *arg) -{ - kvsctx_t *ctx = getctx (h); - kvs_watcher_t *wp = xzmalloc (sizeof (*wp)); - int lastcount = zhash_size (ctx->watchers); - - assert (matchtag != FLUX_MATCHTAG_NONE); - wp->h = h; - wp->key = xstrdup (key); - wp->matchtag = matchtag; - wp->set = fun; - wp->type = type; - wp->arg = arg; - - char *k = xasprintf ("%"PRIu32, matchtag); - zhash_update (ctx->watchers, k, wp); - zhash_freefn (ctx->watchers, k, destroy_watcher); - free (k); - - if (lastcount == 0) - flux_msg_handler_start (ctx->w); - return wp; -} - -static kvs_watcher_t *lookup_watcher (flux_t *h, uint32_t matchtag) -{ - kvsctx_t *ctx = getctx (h); - char *k = xasprintf ("%"PRIu32, matchtag); - kvs_watcher_t *wp = zhash_lookup (ctx->watchers, k); - free (k); - return wp; -} - -int kvs_unwatch (flux_t *h, const char *key) -{ - kvsctx_t *ctx = getctx (h); - flux_future_t *f = NULL; - json_object *in = NULL; - int rc = -1; - - if (!(in = kp_tunwatch_enc (key))) - goto done; - if (!(f = flux_rpc (h, "kvs.unwatch", Jtostr (in), FLUX_NODEID_ANY, 0))) - goto done; - if (flux_future_get (f, NULL) < 0) - goto done; - /* Delete all watchers for the specified key. - */ - zlist_t *hashkeys = zhash_keys (ctx->watchers); - char *k = zlist_first (hashkeys); - while (k) { - kvs_watcher_t *wp = zhash_lookup (ctx->watchers, k); - if (wp && !strcmp (wp->key, key)) - zhash_delete (ctx->watchers, k); - k = zlist_next (hashkeys); - } - zlist_destroy (&hashkeys); - if (zhash_size (ctx->watchers) == 0) - flux_msg_handler_stop (ctx->w); - rc = 0; -done: - Jput (in); - flux_future_destroy (f); - return rc; -} - -static int dispatch_watch (flux_t *h, kvs_watcher_t *wp, json_object *val) -{ - int errnum = val ? 0 : ENOENT; - int rc = -1; - - switch (wp->type) { - case WATCH_STRING: { - kvs_set_string_f set = wp->set; - const char *s = val ? json_object_get_string (val) : NULL; - rc = set (wp->key, s, wp->arg, errnum); - break; - } - case WATCH_INT: { - kvs_set_int_f set = wp->set; - int i = val ? json_object_get_int (val) : 0; - rc = set (wp->key, i, wp->arg, errnum); - break; - } - case WATCH_INT64: { - kvs_set_int64_f set = wp->set; - int64_t i = val ? json_object_get_int64 (val) : 0; - rc = set (wp->key, i, wp->arg, errnum); - break; - } - case WATCH_DOUBLE: { - kvs_set_double_f set = wp->set; - double d = val ? json_object_get_double (val) : 0; - rc = set (wp->key, d, wp->arg, errnum); - break; - } - case WATCH_BOOLEAN: { - kvs_set_boolean_f set = wp->set; - bool b = val ? json_object_get_boolean (val) : false; - rc = set (wp->key, b, wp->arg, errnum); - break; - } - case WATCH_DIR: { - kvs_set_dir_f set = wp->set; - kvsdir_t *dir = val ? kvsdir_create (h, NULL, wp->key, Jtostr (val)) - : NULL; - rc = set (wp->key, dir, wp->arg, errnum); - if (dir) - kvsdir_destroy (dir); - break; - } - case WATCH_JSONSTR: { - kvs_set_f set = wp->set; - rc = set (wp->key, Jtostr (val), wp->arg, errnum); - break; - } - } - return rc; -} - -static void watch_response_cb (flux_t *h, flux_msg_handler_t *w, - const flux_msg_t *msg, void *arg) -{ - const char *json_str; - json_object *out = NULL; - json_object *val; - uint32_t matchtag; - kvs_watcher_t *wp; - - if (flux_response_decode (msg, NULL, &json_str) < 0) - goto done; - if (flux_msg_get_matchtag (msg, &matchtag) < 0) - goto done; - if (!json_str || !(out = Jfromstr (json_str))) { - errno = EPROTO; - goto done; - } - if (kp_rwatch_dec (out, &val) < 0) - goto done; - if ((wp = lookup_watcher (h, matchtag))) - if (dispatch_watch (h, wp, val) < 0) - flux_reactor_stop_error (flux_get_reactor (h)); -done: - Jput (out); -} - -/* Not strictly an RPC since multiple replies are possible. - * Send the kvs.watch request and receive the first reply, synchronously. - * If 'matchtag' is non-NULL return the request's matchtag in it for - * adding to the watcher state; else retire the matchtag. - */ -static int watch_rpc (flux_t *h, const char *key, json_object **val, - int flags, uint32_t *matchtag) -{ - struct flux_match match = { .typemask = FLUX_MSGTYPE_RESPONSE, - .topic_glob = NULL }; - json_object *in = NULL; - json_object *out = NULL; - const char *json_str; - flux_msg_t *request_msg = NULL; - flux_msg_t *response_msg = NULL; - json_object *v = NULL; - int ret = -1; - - /* Send the request. - */ - assert ((flags & KVS_PROTO_ONCE) || matchtag != NULL); - match.matchtag = flux_matchtag_alloc (h, 0); - if (match.matchtag == FLUX_MATCHTAG_NONE) { - errno = EAGAIN; - goto done; - } - if (!(flags & KVS_PROTO_ONCE)) - flags |= KVS_PROTO_FIRST; - if (!(in = kp_twatch_enc (key, *val, flags))) - goto done; - if (!(request_msg = flux_request_encode ("kvs.watch", Jtostr (in)))) - goto done; - if (flux_msg_set_matchtag (request_msg, match.matchtag) < 0) - goto done; - if (flux_send (h, request_msg, 0) < 0) - goto done; - /* Receive the (first) response. - */ - if (!(response_msg = flux_recv (h, match, 0))) - goto done; - if (flux_response_decode (response_msg, NULL, &json_str) < 0) - goto done; - if (!json_str || !(out = Jfromstr (json_str))) { - errno = EPROTO; - goto done; - } - if (kp_rwatch_dec (out, &v) < 0) /* v may be NULL (no ENOENT here) */ - goto done; - *val = Jget (v); - if (matchtag) - *matchtag = match.matchtag; - ret = 0; -done: - if (match.matchtag != FLUX_MATCHTAG_NONE) { - if (!matchtag || ret == -1) - flux_matchtag_free (h, match.matchtag); - } - Jput (in); - Jput (out); - flux_msg_destroy (request_msg); - flux_msg_destroy (response_msg); - return ret; -} - -static int watch_once_obj (flux_t *h, const char *key, json_object **valp) -{ - int rc = -1; - - if (watch_rpc (h, key, valp, KVS_PROTO_ONCE, NULL) < 0) - goto done; - if (*valp == NULL) { - errno = ENOENT; - goto done; - } - rc = 0; -done: - return rc; -} - -int kvs_watch_once (flux_t *h, const char *key, char **valp) -{ - json_object *val = NULL; - int rc = -1; - - if (!h || !key || !valp) { - errno = EINVAL; - goto done; - } - if (*valp) { - if (!(val = Jfromstr (*valp))) { - errno = EINVAL; - goto done; - } - } - if (watch_once_obj (h, key, &val) < 0) - goto done; - if (*valp) - free (*valp); - *valp = val ? xstrdup (Jtostr (val)) : NULL; - rc = 0; -done: - Jput (val); - return rc; -} - -int kvs_watch_once_int (flux_t *h, const char *key, int *valp) -{ - json_object *val; - int rc = -1; - - if (!(val = json_object_new_int (*valp))) - oom (); - if (watch_rpc (h, key, &val, KVS_PROTO_ONCE, NULL) < 0) - goto done; - if (!val) { - errno = ENOENT; - goto done; - } - *valp = json_object_get_int (val); - rc = 0; -done: - if (val) - json_object_put (val); - return rc; -} - -int kvs_watch_once_dir (flux_t *h, kvsdir_t **dirp, const char *fmt, ...) -{ - json_object *val = NULL; - char *key; - va_list ap; - int rc = -1; - - va_start (ap, fmt); - if (vasprintf (&key, fmt, ap) < 0) - oom (); - va_end (ap); - - if (*dirp) { - const char *s; - if (!(s = kvsdir_tostring (*dirp))) - goto done; - if (!(val = Jfromstr (s))) - goto done; - } - if (watch_rpc (h, key, &val, KVS_PROTO_ONCE | KVS_PROTO_READDIR, NULL) < 0) - goto done; - if (val == NULL) { - errno = ENOENT; - goto done; - } - if (*dirp) - kvsdir_destroy (*dirp); - *dirp = kvsdir_create (h, NULL, key, Jtostr (val)); - rc = 0; -done: - if (val) - json_object_put (val); - if (key) - free (key); - return rc; -} - -int kvs_watch (flux_t *h, const char *key, kvs_set_f set, void *arg) -{ - uint32_t matchtag; - kvs_watcher_t *wp; - json_object *val = NULL; - int rc = -1; - - if (watch_rpc (h, key, &val, 0, &matchtag) < 0) - goto done; - wp = add_watcher (h, key, WATCH_JSONSTR, matchtag, set, arg); - dispatch_watch (h, wp, val); - rc = 0; -done: - if (val) - json_object_put (val); - return rc; -} - -int kvs_watch_dir (flux_t *h, kvs_set_dir_f set, void *arg, const char *fmt, ...) -{ - uint32_t matchtag; - kvs_watcher_t *wp; - json_object *val = NULL; - char *key; - int rc = -1; - va_list ap; - - va_start (ap, fmt); - if (vasprintf (&key, fmt, ap) < 0) - oom (); - va_end (ap); - - if (watch_rpc (h, key, &val, KVS_PROTO_READDIR, &matchtag) < 0) - goto done; - wp = add_watcher (h, key, WATCH_DIR, matchtag, set, arg); - dispatch_watch (h, wp, val); - rc = 0; -done: - if (val) - json_object_put (val); - if (key) - free (key); - return rc; -} - -int kvs_watch_string (flux_t *h, const char *key, kvs_set_string_f set, - void *arg) -{ - uint32_t matchtag; - kvs_watcher_t *wp; - json_object *val = NULL; - int rc = -1; - - if (watch_rpc (h, key, &val, 0, &matchtag) < 0) - goto done; - wp = add_watcher (h, key, WATCH_STRING, matchtag, set, arg); - dispatch_watch (h, wp, val); - rc = 0; -done: - if (val) - json_object_put (val); - return rc; -} - -int kvs_watch_int (flux_t *h, const char *key, kvs_set_int_f set, void *arg) -{ - uint32_t matchtag; - kvs_watcher_t *wp; - json_object *val = NULL; - int rc = -1; - - if (watch_rpc (h, key, &val, 0, &matchtag) < 0) - goto done; - wp = add_watcher (h, key, WATCH_INT, matchtag, set, arg); - dispatch_watch (h, wp, val); - rc = 0; -done: - if (val) - json_object_put (val); - return rc; -} - -int kvs_watch_int64 (flux_t *h, const char *key, kvs_set_int64_f set, void *arg) -{ - uint32_t matchtag; - kvs_watcher_t *wp; - json_object *val = NULL; - int rc = -1; - - if (watch_rpc (h, key, &val, 0, &matchtag) < 0) - goto done; - wp = add_watcher (h, key, WATCH_INT64, matchtag, set, arg); - dispatch_watch (h, wp, val); - rc = 0; -done: - if (val) - json_object_put (val); - return rc; -} - -int kvs_watch_double (flux_t *h, const char *key, kvs_set_double_f set, - void *arg) -{ - uint32_t matchtag; - kvs_watcher_t *wp; - json_object *val = NULL; - int rc = -1; - - if (watch_rpc (h, key, &val, 0, &matchtag) < 0) - goto done; - wp = add_watcher (h, key, WATCH_DOUBLE, matchtag, set, arg); - dispatch_watch (h, wp, val); - rc = 0; -done: - if (val) - json_object_put (val); - return rc; -} - -int kvs_watch_boolean (flux_t *h, const char *key, kvs_set_boolean_f set, - void *arg) -{ - uint32_t matchtag; - kvs_watcher_t *wp; - json_object *val = NULL; - int rc = -1; - - if (watch_rpc (h, key, &val, 0, &matchtag) < 0) - goto done; - wp = add_watcher (h, key, WATCH_BOOLEAN, matchtag, set, arg); - dispatch_watch (h, wp, val); - rc = 0; -done: - if (val) - json_object_put (val); - return rc; -} - /** ** PUT **/ diff --git a/src/common/libkvs/kvs.h b/src/common/libkvs/kvs.h index 787275c59ba7..b35b414590b1 100644 --- a/src/common/libkvs/kvs.h +++ b/src/common/libkvs/kvs.h @@ -8,6 +8,7 @@ #include "kvs_lookup.h" #include "kvs_dir.h" #include "kvs_classic.h" +#include "kvs_watch.h" /* Flags for commit and fence operations */ @@ -15,61 +16,6 @@ enum flux_kvs_flags { KVS_NO_MERGE = 1, /* disallow commits to be mergeable with others */ }; -typedef int (*kvs_set_f)(const char *key, const char *json_str, void *arg, - int errnum); -typedef int (*kvs_set_dir_f)(const char *key, kvsdir_t *dir, void *arg, - int errnum); -typedef int (*kvs_set_string_f)(const char *key, const char *val, void *arg, - int errnum); -typedef int (*kvs_set_int_f)(const char *key, int val, void *arg, int errnum); -typedef int (*kvs_set_int64_f)(const char *key, int64_t val, void *arg, - int errnum); -typedef int (*kvs_set_double_f)(const char *key, double val, void *arg, - int errnum); -typedef int (*kvs_set_boolean_f)(const char *key, bool val, void *arg, - int errnum); - -/* kvs_watch* is like kvs_get* except the registered callback is called - * to set the value. It will be called immediately to set the initial - * value and again each time the value changes. - * Any storage associated with the value given the - * callback is freed when the callback returns. If a value is unset, the - * callback gets errnum = ENOENT. - */ -int kvs_watch (flux_t *h, const char *key, kvs_set_f set, void *arg); -int kvs_watch_dir (flux_t *h, kvs_set_dir_f set, void *arg, - const char *fmt, ...) - __attribute__ ((format (printf, 4, 5))); -int kvs_watch_string (flux_t *h, const char *key, kvs_set_string_f set, - void *arg); -int kvs_watch_int (flux_t *h, const char *key, kvs_set_int_f set, void *arg); -int kvs_watch_int64 (flux_t *h, const char *key, kvs_set_int64_f set, void *arg); -int kvs_watch_double (flux_t *h, const char *key, kvs_set_double_f set, - void *arg); -int kvs_watch_boolean (flux_t *h, const char *key, kvs_set_boolean_f set, - void *arg); - -/* Cancel a kvs_watch, freeing server-side state, and unregistering any - * callback. Returns 0 on success, or -1 with errno set on error. - */ -int kvs_unwatch (flux_t *h, const char *key); - -/* While the above callback interface makes sense in plugin context, - * the following is better for API context. 'json_str', 'dirp', and - * 'valp' are IN/OUT parameters. You should first read the current - * value, then pass it into the respective kvs_watch_once call, which - * will return with a new value when it changes. (The original value - * is freed inside the function; the new one must be freed by the - * caller). *json_str, *dirp, and *valp may be passed in with a NULL - * value. If the key is not set, ENOENT is returned without affecting - * *valp. - * FIXME: add more types. - */ -int kvs_watch_once (flux_t *h, const char *key, char **json_str); -int kvs_watch_once_dir (flux_t *h, kvsdir_t **dirp, const char *fmt, ...) - __attribute__ ((format (printf, 3, 4))); -int kvs_watch_once_int (flux_t *h, const char *key, int *valp); - /* kvs_put() and kvs_put_string() both make copies of the value argument * The caller retains ownership of the original. * These functions return -1 on error (errno set), 0 on success. diff --git a/src/common/libkvs/kvs_watch.c b/src/common/libkvs/kvs_watch.c new file mode 100644 index 000000000000..da27b49caddb --- /dev/null +++ b/src/common/libkvs/kvs_watch.c @@ -0,0 +1,569 @@ +/*****************************************************************************\ + * Copyright (c) 2014 Lawrence Livermore National Security, LLC. Produced at + * the Lawrence Livermore National Laboratory (cf, AUTHORS, DISCLAIMER.LLNS). + * LLNL-CODE-658032 All rights reserved. + * + * This file is part of the Flux resource manager framework. + * For details, see https://github.com/flux-framework. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the license, or (at your option) + * any later version. + * + * Flux is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the IMPLIED WARRANTY OF MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the terms and conditions of the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. + * See also: http://www.gnu.org/licenses/ +\*****************************************************************************/ + +#if HAVE_CONFIG_H +#include "config.h" +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "src/common/libutil/shortjson.h" +#include "src/common/libutil/log.h" +#include "src/common/libutil/xzmalloc.h" +#include "src/common/libutil/blobref.h" + +#include "proto.h" +#include "json_dirent.h" + + +typedef enum { + WATCH_STRING, WATCH_INT, WATCH_INT64, WATCH_DOUBLE, + WATCH_BOOLEAN, WATCH_JSONSTR, WATCH_DIR, +} watch_type_t; + +typedef struct { + watch_type_t type; + void *set; + void *arg; + flux_t *h; + char *key; + uint32_t matchtag; +} kvs_watcher_t; + +typedef struct { + zhash_t *watchers; /* kvs_watch_t hashed by stringified matchtag */ + flux_msg_handler_t *w; +} kvs_watch_ctx_t; + +static void watch_response_cb (flux_t *h, flux_msg_handler_t *w, + const flux_msg_t *msg, void *arg); + +static void freectx (void *arg) +{ + kvs_watch_ctx_t *ctx = arg; + if (ctx) { + zhash_destroy (&ctx->watchers); + flux_msg_handler_destroy (ctx->w); + free (ctx); + } +} + +static kvs_watch_ctx_t *getctx (flux_t *h) +{ + const char *auxkey = "flux::kvs_watch"; + kvs_watch_ctx_t *ctx = (kvs_watch_ctx_t *)flux_aux_get (h, auxkey); + struct flux_match match = FLUX_MATCH_RESPONSE; + + if (!ctx) { + ctx = xzmalloc (sizeof (*ctx)); + if (!(ctx->watchers = zhash_new ())) + oom (); + match.topic_glob = "kvs.watch"; + if (!(ctx->w = flux_msg_handler_create (h, match, watch_response_cb, + ctx))) + oom (); + flux_aux_set (h, auxkey, ctx, freectx); + } + return ctx; +} + +/** + ** WATCH + **/ + +static void destroy_watcher (void *arg) +{ + kvs_watcher_t *wp = arg; + free (wp->key); + flux_matchtag_free (wp->h, wp->matchtag); + free (wp); +} + +static kvs_watcher_t *add_watcher (flux_t *h, const char *key, watch_type_t type, + uint32_t matchtag, void *fun, void *arg) +{ + kvs_watch_ctx_t *ctx = getctx (h); + kvs_watcher_t *wp = xzmalloc (sizeof (*wp)); + int lastcount = zhash_size (ctx->watchers); + + assert (matchtag != FLUX_MATCHTAG_NONE); + wp->h = h; + wp->key = xstrdup (key); + wp->matchtag = matchtag; + wp->set = fun; + wp->type = type; + wp->arg = arg; + + char *k = xasprintf ("%"PRIu32, matchtag); + zhash_update (ctx->watchers, k, wp); + zhash_freefn (ctx->watchers, k, destroy_watcher); + free (k); + + if (lastcount == 0) + flux_msg_handler_start (ctx->w); + return wp; +} + +static kvs_watcher_t *lookup_watcher (flux_t *h, uint32_t matchtag) +{ + kvs_watch_ctx_t *ctx = getctx (h); + char *k = xasprintf ("%"PRIu32, matchtag); + kvs_watcher_t *wp = zhash_lookup (ctx->watchers, k); + free (k); + return wp; +} + +int kvs_unwatch (flux_t *h, const char *key) +{ + kvs_watch_ctx_t *ctx = getctx (h); + flux_future_t *f = NULL; + json_object *in = NULL; + int rc = -1; + + if (!(in = kp_tunwatch_enc (key))) + goto done; + if (!(f = flux_rpc (h, "kvs.unwatch", Jtostr (in), FLUX_NODEID_ANY, 0))) + goto done; + if (flux_future_get (f, NULL) < 0) + goto done; + /* Delete all watchers for the specified key. + */ + zlist_t *hashkeys = zhash_keys (ctx->watchers); + char *k = zlist_first (hashkeys); + while (k) { + kvs_watcher_t *wp = zhash_lookup (ctx->watchers, k); + if (wp && !strcmp (wp->key, key)) + zhash_delete (ctx->watchers, k); + k = zlist_next (hashkeys); + } + zlist_destroy (&hashkeys); + if (zhash_size (ctx->watchers) == 0) + flux_msg_handler_stop (ctx->w); + rc = 0; +done: + Jput (in); + flux_future_destroy (f); + return rc; +} + +static int dispatch_watch (flux_t *h, kvs_watcher_t *wp, json_object *val) +{ + int errnum = val ? 0 : ENOENT; + int rc = -1; + + switch (wp->type) { + case WATCH_STRING: { + kvs_set_string_f set = wp->set; + const char *s = val ? json_object_get_string (val) : NULL; + rc = set (wp->key, s, wp->arg, errnum); + break; + } + case WATCH_INT: { + kvs_set_int_f set = wp->set; + int i = val ? json_object_get_int (val) : 0; + rc = set (wp->key, i, wp->arg, errnum); + break; + } + case WATCH_INT64: { + kvs_set_int64_f set = wp->set; + int64_t i = val ? json_object_get_int64 (val) : 0; + rc = set (wp->key, i, wp->arg, errnum); + break; + } + case WATCH_DOUBLE: { + kvs_set_double_f set = wp->set; + double d = val ? json_object_get_double (val) : 0; + rc = set (wp->key, d, wp->arg, errnum); + break; + } + case WATCH_BOOLEAN: { + kvs_set_boolean_f set = wp->set; + bool b = val ? json_object_get_boolean (val) : false; + rc = set (wp->key, b, wp->arg, errnum); + break; + } + case WATCH_DIR: { + kvs_set_dir_f set = wp->set; + kvsdir_t *dir = val ? kvsdir_create (h, NULL, wp->key, Jtostr (val)) + : NULL; + rc = set (wp->key, dir, wp->arg, errnum); + if (dir) + kvsdir_destroy (dir); + break; + } + case WATCH_JSONSTR: { + kvs_set_f set = wp->set; + rc = set (wp->key, Jtostr (val), wp->arg, errnum); + break; + } + } + return rc; +} + +static void watch_response_cb (flux_t *h, flux_msg_handler_t *w, + const flux_msg_t *msg, void *arg) +{ + const char *json_str; + json_object *out = NULL; + json_object *val; + uint32_t matchtag; + kvs_watcher_t *wp; + + if (flux_response_decode (msg, NULL, &json_str) < 0) + goto done; + if (flux_msg_get_matchtag (msg, &matchtag) < 0) + goto done; + if (!json_str || !(out = Jfromstr (json_str))) { + errno = EPROTO; + goto done; + } + if (kp_rwatch_dec (out, &val) < 0) + goto done; + if ((wp = lookup_watcher (h, matchtag))) + if (dispatch_watch (h, wp, val) < 0) + flux_reactor_stop_error (flux_get_reactor (h)); +done: + Jput (out); +} + +/* Not strictly an RPC since multiple replies are possible. + * Send the kvs.watch request and receive the first reply, synchronously. + * If 'matchtag' is non-NULL return the request's matchtag in it for + * adding to the watcher state; else retire the matchtag. + */ +static int watch_rpc (flux_t *h, const char *key, json_object **val, + int flags, uint32_t *matchtag) +{ + struct flux_match match = { .typemask = FLUX_MSGTYPE_RESPONSE, + .topic_glob = NULL }; + json_object *in = NULL; + json_object *out = NULL; + const char *json_str; + flux_msg_t *request_msg = NULL; + flux_msg_t *response_msg = NULL; + json_object *v = NULL; + int ret = -1; + + /* Send the request. + */ + assert ((flags & KVS_PROTO_ONCE) || matchtag != NULL); + match.matchtag = flux_matchtag_alloc (h, 0); + if (match.matchtag == FLUX_MATCHTAG_NONE) { + errno = EAGAIN; + goto done; + } + if (!(flags & KVS_PROTO_ONCE)) + flags |= KVS_PROTO_FIRST; + if (!(in = kp_twatch_enc (key, *val, flags))) + goto done; + if (!(request_msg = flux_request_encode ("kvs.watch", Jtostr (in)))) + goto done; + if (flux_msg_set_matchtag (request_msg, match.matchtag) < 0) + goto done; + if (flux_send (h, request_msg, 0) < 0) + goto done; + /* Receive the (first) response. + */ + if (!(response_msg = flux_recv (h, match, 0))) + goto done; + if (flux_response_decode (response_msg, NULL, &json_str) < 0) + goto done; + if (!json_str || !(out = Jfromstr (json_str))) { + errno = EPROTO; + goto done; + } + if (kp_rwatch_dec (out, &v) < 0) /* v may be NULL (no ENOENT here) */ + goto done; + *val = Jget (v); + if (matchtag) + *matchtag = match.matchtag; + ret = 0; +done: + if (match.matchtag != FLUX_MATCHTAG_NONE) { + if (!matchtag || ret == -1) + flux_matchtag_free (h, match.matchtag); + } + Jput (in); + Jput (out); + flux_msg_destroy (request_msg); + flux_msg_destroy (response_msg); + return ret; +} + +static int watch_once_obj (flux_t *h, const char *key, json_object **valp) +{ + int rc = -1; + + if (watch_rpc (h, key, valp, KVS_PROTO_ONCE, NULL) < 0) + goto done; + if (*valp == NULL) { + errno = ENOENT; + goto done; + } + rc = 0; +done: + return rc; +} + +int kvs_watch_once (flux_t *h, const char *key, char **valp) +{ + json_object *val = NULL; + int rc = -1; + + if (!h || !key || !valp) { + errno = EINVAL; + goto done; + } + if (*valp) { + if (!(val = Jfromstr (*valp))) { + errno = EINVAL; + goto done; + } + } + if (watch_once_obj (h, key, &val) < 0) + goto done; + if (*valp) + free (*valp); + *valp = val ? xstrdup (Jtostr (val)) : NULL; + rc = 0; +done: + Jput (val); + return rc; +} + +int kvs_watch_once_int (flux_t *h, const char *key, int *valp) +{ + json_object *val; + int rc = -1; + + if (!(val = json_object_new_int (*valp))) + oom (); + if (watch_rpc (h, key, &val, KVS_PROTO_ONCE, NULL) < 0) + goto done; + if (!val) { + errno = ENOENT; + goto done; + } + *valp = json_object_get_int (val); + rc = 0; +done: + if (val) + json_object_put (val); + return rc; +} + +int kvs_watch_once_dir (flux_t *h, kvsdir_t **dirp, const char *fmt, ...) +{ + json_object *val = NULL; + char *key; + va_list ap; + int rc = -1; + + va_start (ap, fmt); + if (vasprintf (&key, fmt, ap) < 0) + oom (); + va_end (ap); + + if (*dirp) { + const char *s; + if (!(s = kvsdir_tostring (*dirp))) + goto done; + if (!(val = Jfromstr (s))) + goto done; + } + if (watch_rpc (h, key, &val, KVS_PROTO_ONCE | KVS_PROTO_READDIR, NULL) < 0) + goto done; + if (val == NULL) { + errno = ENOENT; + goto done; + } + if (*dirp) + kvsdir_destroy (*dirp); + *dirp = kvsdir_create (h, NULL, key, Jtostr (val)); + rc = 0; +done: + if (val) + json_object_put (val); + if (key) + free (key); + return rc; +} + +int kvs_watch (flux_t *h, const char *key, kvs_set_f set, void *arg) +{ + uint32_t matchtag; + kvs_watcher_t *wp; + json_object *val = NULL; + int rc = -1; + + if (watch_rpc (h, key, &val, 0, &matchtag) < 0) + goto done; + wp = add_watcher (h, key, WATCH_JSONSTR, matchtag, set, arg); + dispatch_watch (h, wp, val); + rc = 0; +done: + if (val) + json_object_put (val); + return rc; +} + +int kvs_watch_dir (flux_t *h, kvs_set_dir_f set, void *arg, const char *fmt, ...) +{ + uint32_t matchtag; + kvs_watcher_t *wp; + json_object *val = NULL; + char *key; + int rc = -1; + va_list ap; + + va_start (ap, fmt); + if (vasprintf (&key, fmt, ap) < 0) + oom (); + va_end (ap); + + if (watch_rpc (h, key, &val, KVS_PROTO_READDIR, &matchtag) < 0) + goto done; + wp = add_watcher (h, key, WATCH_DIR, matchtag, set, arg); + dispatch_watch (h, wp, val); + rc = 0; +done: + if (val) + json_object_put (val); + if (key) + free (key); + return rc; +} + +int kvs_watch_string (flux_t *h, const char *key, kvs_set_string_f set, + void *arg) +{ + uint32_t matchtag; + kvs_watcher_t *wp; + json_object *val = NULL; + int rc = -1; + + if (watch_rpc (h, key, &val, 0, &matchtag) < 0) + goto done; + wp = add_watcher (h, key, WATCH_STRING, matchtag, set, arg); + dispatch_watch (h, wp, val); + rc = 0; +done: + if (val) + json_object_put (val); + return rc; +} + +int kvs_watch_int (flux_t *h, const char *key, kvs_set_int_f set, void *arg) +{ + uint32_t matchtag; + kvs_watcher_t *wp; + json_object *val = NULL; + int rc = -1; + + if (watch_rpc (h, key, &val, 0, &matchtag) < 0) + goto done; + wp = add_watcher (h, key, WATCH_INT, matchtag, set, arg); + dispatch_watch (h, wp, val); + rc = 0; +done: + if (val) + json_object_put (val); + return rc; +} + +int kvs_watch_int64 (flux_t *h, const char *key, kvs_set_int64_f set, void *arg) +{ + uint32_t matchtag; + kvs_watcher_t *wp; + json_object *val = NULL; + int rc = -1; + + if (watch_rpc (h, key, &val, 0, &matchtag) < 0) + goto done; + wp = add_watcher (h, key, WATCH_INT64, matchtag, set, arg); + dispatch_watch (h, wp, val); + rc = 0; +done: + if (val) + json_object_put (val); + return rc; +} + +int kvs_watch_double (flux_t *h, const char *key, kvs_set_double_f set, + void *arg) +{ + uint32_t matchtag; + kvs_watcher_t *wp; + json_object *val = NULL; + int rc = -1; + + if (watch_rpc (h, key, &val, 0, &matchtag) < 0) + goto done; + wp = add_watcher (h, key, WATCH_DOUBLE, matchtag, set, arg); + dispatch_watch (h, wp, val); + rc = 0; +done: + if (val) + json_object_put (val); + return rc; +} + +int kvs_watch_boolean (flux_t *h, const char *key, kvs_set_boolean_f set, + void *arg) +{ + uint32_t matchtag; + kvs_watcher_t *wp; + json_object *val = NULL; + int rc = -1; + + if (watch_rpc (h, key, &val, 0, &matchtag) < 0) + goto done; + wp = add_watcher (h, key, WATCH_BOOLEAN, matchtag, set, arg); + dispatch_watch (h, wp, val); + rc = 0; +done: + if (val) + json_object_put (val); + return rc; +} + +/* + * vi:tabstop=4 shiftwidth=4 expandtab + */ diff --git a/src/common/libkvs/kvs_watch.h b/src/common/libkvs/kvs_watch.h new file mode 100644 index 000000000000..bda5467a39d7 --- /dev/null +++ b/src/common/libkvs/kvs_watch.h @@ -0,0 +1,63 @@ +#ifndef _FLUX_CORE_KVS_WATCH_H +#define _FLUX_CORE_KVS_WATCH_H + +typedef int (*kvs_set_f)(const char *key, const char *json_str, void *arg, + int errnum); +typedef int (*kvs_set_dir_f)(const char *key, kvsdir_t *dir, void *arg, + int errnum); +typedef int (*kvs_set_string_f)(const char *key, const char *val, void *arg, + int errnum); +typedef int (*kvs_set_int_f)(const char *key, int val, void *arg, int errnum); +typedef int (*kvs_set_int64_f)(const char *key, int64_t val, void *arg, + int errnum); +typedef int (*kvs_set_double_f)(const char *key, double val, void *arg, + int errnum); +typedef int (*kvs_set_boolean_f)(const char *key, bool val, void *arg, + int errnum); + +/* kvs_watch* is like kvs_get* except the registered callback is called + * to set the value. It will be called immediately to set the initial + * value and again each time the value changes. + * Any storage associated with the value given the + * callback is freed when the callback returns. If a value is unset, the + * callback gets errnum = ENOENT. + */ +int kvs_watch (flux_t *h, const char *key, kvs_set_f set, void *arg); +int kvs_watch_dir (flux_t *h, kvs_set_dir_f set, void *arg, + const char *fmt, ...) + __attribute__ ((format (printf, 4, 5))); +int kvs_watch_string (flux_t *h, const char *key, kvs_set_string_f set, + void *arg); +int kvs_watch_int (flux_t *h, const char *key, kvs_set_int_f set, void *arg); +int kvs_watch_int64 (flux_t *h, const char *key, kvs_set_int64_f set, void *arg); +int kvs_watch_double (flux_t *h, const char *key, kvs_set_double_f set, + void *arg); +int kvs_watch_boolean (flux_t *h, const char *key, kvs_set_boolean_f set, + void *arg); + +/* Cancel a kvs_watch, freeing server-side state, and unregistering any + * callback. Returns 0 on success, or -1 with errno set on error. + */ +int kvs_unwatch (flux_t *h, const char *key); + +/* While the above callback interface makes sense in plugin context, + * the following is better for API context. 'json_str', 'dirp', and + * 'valp' are IN/OUT parameters. You should first read the current + * value, then pass it into the respective kvs_watch_once call, which + * will return with a new value when it changes. (The original value + * is freed inside the function; the new one must be freed by the + * caller). *json_str, *dirp, and *valp may be passed in with a NULL + * value. If the key is not set, ENOENT is returned without affecting + * *valp. + * FIXME: add more types. + */ +int kvs_watch_once (flux_t *h, const char *key, char **json_str); +int kvs_watch_once_dir (flux_t *h, kvsdir_t **dirp, const char *fmt, ...) + __attribute__ ((format (printf, 3, 4))); +int kvs_watch_once_int (flux_t *h, const char *key, int *valp); + +#endif /* !_FLUX_CORE_KVS_WATCH_H */ + +/* + * vi:tabstop=4 shiftwidth=4 expandtab + */ From 69ccaff8b0e4f472f25584ded52eed024ff1a0a0 Mon Sep 17 00:00:00 2001 From: Jim Garlick Date: Sat, 1 Jul 2017 14:45:46 +0000 Subject: [PATCH 10/19] libkvs/classic: drop kvs[dir]_get_boolean Drop these functions that have no users: kvs_get_boolean() kvsdir_get_boolean() Update "getas" test driver to drop boolean support --- src/common/libkvs/kvs_classic.c | 42 --------------------------------- src/common/libkvs/kvs_classic.h | 2 -- t/kvs/getas.c | 16 ++----------- 3 files changed, 2 insertions(+), 58 deletions(-) diff --git a/src/common/libkvs/kvs_classic.c b/src/common/libkvs/kvs_classic.c index cd5fbf443215..9d9b782bc043 100644 --- a/src/common/libkvs/kvs_classic.c +++ b/src/common/libkvs/kvs_classic.c @@ -129,24 +129,6 @@ int kvs_get_double (flux_t *h, const char *key, double *valp) return rc; } -int kvs_get_boolean (flux_t *h, const char *key, bool *valp) -{ - flux_future_t *f; - int b; - int rc = -1; - - if (!(f = flux_kvs_lookup (h, 0, key))) - goto done; - if (flux_kvs_lookup_getf (f, "b", &b) < 0) - goto done; - if (valp) - *valp = b; - rc = 0; -done: - flux_future_destroy (f); - return rc; -} - int kvs_get_dir (flux_t *h, kvsdir_t **dir, const char *fmt, ...) { flux_future_t *f = NULL; @@ -485,30 +467,6 @@ int kvsdir_get_double (kvsdir_t *dir, const char *name, double *valp) return rc; } -int kvsdir_get_boolean (kvsdir_t *dir, const char *name, bool *valp) -{ - flux_t *h = kvsdir_handle (dir); - const char *rootref = kvsdir_rootref (dir); - flux_future_t *f = NULL; - int i; - char *key; - int rc = -1; - - if (!(key = kvsdir_key_at (dir, name))) - goto done; - if (!(f = flux_kvs_lookupat (h, 0, key, rootref))) - goto done; - if (flux_kvs_lookup_getf (f, "b", &i) < 0) - goto done; - if (valp) - *valp = i; - rc = 0; -done: - free (key); - flux_future_destroy (f); - return rc; -} - /* * vi:tabstop=4 shiftwidth=4 expandtab */ diff --git a/src/common/libkvs/kvs_classic.h b/src/common/libkvs/kvs_classic.h index b909b1c34b8c..928b4a8a638b 100644 --- a/src/common/libkvs/kvs_classic.h +++ b/src/common/libkvs/kvs_classic.h @@ -8,7 +8,6 @@ int kvs_get_string (flux_t *h, const char *key, char **valp); int kvs_get_int (flux_t *h, const char *key, int *valp); int kvs_get_int64 (flux_t *h, const char *key, int64_t *valp); int kvs_get_double (flux_t *h, const char *key, double *valp); -int kvs_get_boolean (flux_t *h, const char *key, bool *valp); int kvs_get_symlink (flux_t *h, const char *key, char **valp); int kvs_get_treeobj (flux_t *h, const char *key, char **valp); int kvs_get_dir (flux_t *h, kvsdir_t **dirp, const char *fmt, ...) @@ -28,7 +27,6 @@ int kvsdir_get_string (kvsdir_t *dir, const char *key, char **valp); int kvsdir_get_int (kvsdir_t *dir, const char *key, int *valp); int kvsdir_get_int64 (kvsdir_t *dir, const char *key, int64_t *valp); int kvsdir_get_double (kvsdir_t *dir, const char *key, double *valp); -int kvsdir_get_boolean (kvsdir_t *dir, const char *key, bool *valp); int kvsdir_get_symlink (kvsdir_t *dir, const char *key, char **valp); diff --git a/t/kvs/getas.c b/t/kvs/getas.c index 6f40b85574d8..c77933d2299b 100644 --- a/t/kvs/getas.c +++ b/t/kvs/getas.c @@ -115,12 +115,6 @@ void dirgetas (flux_t *h, const char *dir, const char *key, const char *type) log_err_exit ("kvsdir_get_int64 %s", key); printf ("%" PRIi64 "\n", value); } - else if (!strcmp (type, "boolean")) { - bool value; - if (kvsdir_get_boolean (d, key, &value) < 0) - log_err_exit ("kvsdir_get_int64 %s", key); - printf ("%s\n", value ? "true" : "false"); - } else if (!strcmp (type, "double")) { double value; if (kvsdir_get_double (d, key, &value) < 0) @@ -135,7 +129,7 @@ void dirgetas (flux_t *h, const char *dir, const char *key, const char *type) free (s); } else { - log_msg_exit ("unknown type (use int/int64/boolean/double/string)"); + log_msg_exit ("unknown type (use int/int64/double/string)"); } kvsdir_destroy (d); @@ -162,12 +156,6 @@ void getas (flux_t *h, const char *key, const char *type) log_err_exit ("kvs_get_int64 %s", key); printf ("%" PRIi64 "\n", value); } - else if (!strcmp (type, "boolean")) { - bool value; - if (kvs_get_boolean (h, key, &value) < 0) - log_err_exit ("kvs_get_int64 %s", key); - printf ("%s\n", value ? "true" : "false"); - } else if (!strcmp (type, "double")) { double value; if (kvs_get_double (h, key, &value) < 0) @@ -182,7 +170,7 @@ void getas (flux_t *h, const char *key, const char *type) free (s); } else { - log_msg_exit ("unknown type (use int/int64/boolean/double/string)"); + log_msg_exit ("unknown type (use int/int64/double/string)"); } } From d0e4f6c24027e883c734bf31a559b6d2aa8e5733 Mon Sep 17 00:00:00 2001 From: Jim Garlick Date: Sat, 1 Jul 2017 18:30:51 +0000 Subject: [PATCH 11/19] libkvs/watch: drop kvs_watch_boolean --- src/common/libkvs/kvs_watch.c | 27 +-------------------------- src/common/libkvs/kvs_watch.h | 4 ---- 2 files changed, 1 insertion(+), 30 deletions(-) diff --git a/src/common/libkvs/kvs_watch.c b/src/common/libkvs/kvs_watch.c index da27b49caddb..a93ae056d283 100644 --- a/src/common/libkvs/kvs_watch.c +++ b/src/common/libkvs/kvs_watch.c @@ -55,7 +55,7 @@ typedef enum { WATCH_STRING, WATCH_INT, WATCH_INT64, WATCH_DOUBLE, - WATCH_BOOLEAN, WATCH_JSONSTR, WATCH_DIR, + WATCH_JSONSTR, WATCH_DIR, } watch_type_t; typedef struct { @@ -213,12 +213,6 @@ static int dispatch_watch (flux_t *h, kvs_watcher_t *wp, json_object *val) rc = set (wp->key, d, wp->arg, errnum); break; } - case WATCH_BOOLEAN: { - kvs_set_boolean_f set = wp->set; - bool b = val ? json_object_get_boolean (val) : false; - rc = set (wp->key, b, wp->arg, errnum); - break; - } case WATCH_DIR: { kvs_set_dir_f set = wp->set; kvsdir_t *dir = val ? kvsdir_create (h, NULL, wp->key, Jtostr (val)) @@ -545,25 +539,6 @@ int kvs_watch_double (flux_t *h, const char *key, kvs_set_double_f set, return rc; } -int kvs_watch_boolean (flux_t *h, const char *key, kvs_set_boolean_f set, - void *arg) -{ - uint32_t matchtag; - kvs_watcher_t *wp; - json_object *val = NULL; - int rc = -1; - - if (watch_rpc (h, key, &val, 0, &matchtag) < 0) - goto done; - wp = add_watcher (h, key, WATCH_BOOLEAN, matchtag, set, arg); - dispatch_watch (h, wp, val); - rc = 0; -done: - if (val) - json_object_put (val); - return rc; -} - /* * vi:tabstop=4 shiftwidth=4 expandtab */ diff --git a/src/common/libkvs/kvs_watch.h b/src/common/libkvs/kvs_watch.h index bda5467a39d7..c359693af748 100644 --- a/src/common/libkvs/kvs_watch.h +++ b/src/common/libkvs/kvs_watch.h @@ -12,8 +12,6 @@ typedef int (*kvs_set_int64_f)(const char *key, int64_t val, void *arg, int errnum); typedef int (*kvs_set_double_f)(const char *key, double val, void *arg, int errnum); -typedef int (*kvs_set_boolean_f)(const char *key, bool val, void *arg, - int errnum); /* kvs_watch* is like kvs_get* except the registered callback is called * to set the value. It will be called immediately to set the initial @@ -32,8 +30,6 @@ int kvs_watch_int (flux_t *h, const char *key, kvs_set_int_f set, void *arg); int kvs_watch_int64 (flux_t *h, const char *key, kvs_set_int64_f set, void *arg); int kvs_watch_double (flux_t *h, const char *key, kvs_set_double_f set, void *arg); -int kvs_watch_boolean (flux_t *h, const char *key, kvs_set_boolean_f set, - void *arg); /* Cancel a kvs_watch, freeing server-side state, and unregistering any * callback. Returns 0 on success, or -1 with errno set on error. From 8e0b534c41679729eb6b3ff4bc600bdce5c041c6 Mon Sep 17 00:00:00 2001 From: Jim Garlick Date: Sat, 1 Jul 2017 19:24:34 +0000 Subject: [PATCH 12/19] libkvs/watch: drop kvs_watch_double() --- src/common/libkvs/kvs_watch.c | 27 +-------------------------- src/common/libkvs/kvs_watch.h | 4 ---- 2 files changed, 1 insertion(+), 30 deletions(-) diff --git a/src/common/libkvs/kvs_watch.c b/src/common/libkvs/kvs_watch.c index a93ae056d283..8ab5ce33fa91 100644 --- a/src/common/libkvs/kvs_watch.c +++ b/src/common/libkvs/kvs_watch.c @@ -54,7 +54,7 @@ typedef enum { - WATCH_STRING, WATCH_INT, WATCH_INT64, WATCH_DOUBLE, + WATCH_STRING, WATCH_INT, WATCH_INT64, WATCH_JSONSTR, WATCH_DIR, } watch_type_t; @@ -207,12 +207,6 @@ static int dispatch_watch (flux_t *h, kvs_watcher_t *wp, json_object *val) rc = set (wp->key, i, wp->arg, errnum); break; } - case WATCH_DOUBLE: { - kvs_set_double_f set = wp->set; - double d = val ? json_object_get_double (val) : 0; - rc = set (wp->key, d, wp->arg, errnum); - break; - } case WATCH_DIR: { kvs_set_dir_f set = wp->set; kvsdir_t *dir = val ? kvsdir_create (h, NULL, wp->key, Jtostr (val)) @@ -520,25 +514,6 @@ int kvs_watch_int64 (flux_t *h, const char *key, kvs_set_int64_f set, void *arg) return rc; } -int kvs_watch_double (flux_t *h, const char *key, kvs_set_double_f set, - void *arg) -{ - uint32_t matchtag; - kvs_watcher_t *wp; - json_object *val = NULL; - int rc = -1; - - if (watch_rpc (h, key, &val, 0, &matchtag) < 0) - goto done; - wp = add_watcher (h, key, WATCH_DOUBLE, matchtag, set, arg); - dispatch_watch (h, wp, val); - rc = 0; -done: - if (val) - json_object_put (val); - return rc; -} - /* * vi:tabstop=4 shiftwidth=4 expandtab */ diff --git a/src/common/libkvs/kvs_watch.h b/src/common/libkvs/kvs_watch.h index c359693af748..b46dad406ab9 100644 --- a/src/common/libkvs/kvs_watch.h +++ b/src/common/libkvs/kvs_watch.h @@ -10,8 +10,6 @@ typedef int (*kvs_set_string_f)(const char *key, const char *val, void *arg, typedef int (*kvs_set_int_f)(const char *key, int val, void *arg, int errnum); typedef int (*kvs_set_int64_f)(const char *key, int64_t val, void *arg, int errnum); -typedef int (*kvs_set_double_f)(const char *key, double val, void *arg, - int errnum); /* kvs_watch* is like kvs_get* except the registered callback is called * to set the value. It will be called immediately to set the initial @@ -28,8 +26,6 @@ int kvs_watch_string (flux_t *h, const char *key, kvs_set_string_f set, void *arg); int kvs_watch_int (flux_t *h, const char *key, kvs_set_int_f set, void *arg); int kvs_watch_int64 (flux_t *h, const char *key, kvs_set_int64_f set, void *arg); -int kvs_watch_double (flux_t *h, const char *key, kvs_set_double_f set, - void *arg); /* Cancel a kvs_watch, freeing server-side state, and unregistering any * callback. Returns 0 on success, or -1 with errno set on error. From 76cde4c9ab28dfa59fae5175574458201474c955 Mon Sep 17 00:00:00 2001 From: Jim Garlick Date: Sat, 1 Jul 2017 21:17:33 +0000 Subject: [PATCH 13/19] libkvs/watch: drop kvs_watch_int64 --- src/common/libkvs/kvs_watch.c | 26 +------------------------- src/common/libkvs/kvs_watch.h | 3 --- 2 files changed, 1 insertion(+), 28 deletions(-) diff --git a/src/common/libkvs/kvs_watch.c b/src/common/libkvs/kvs_watch.c index 8ab5ce33fa91..c5949bfa051e 100644 --- a/src/common/libkvs/kvs_watch.c +++ b/src/common/libkvs/kvs_watch.c @@ -54,7 +54,7 @@ typedef enum { - WATCH_STRING, WATCH_INT, WATCH_INT64, + WATCH_STRING, WATCH_INT, WATCH_JSONSTR, WATCH_DIR, } watch_type_t; @@ -201,12 +201,6 @@ static int dispatch_watch (flux_t *h, kvs_watcher_t *wp, json_object *val) rc = set (wp->key, i, wp->arg, errnum); break; } - case WATCH_INT64: { - kvs_set_int64_f set = wp->set; - int64_t i = val ? json_object_get_int64 (val) : 0; - rc = set (wp->key, i, wp->arg, errnum); - break; - } case WATCH_DIR: { kvs_set_dir_f set = wp->set; kvsdir_t *dir = val ? kvsdir_create (h, NULL, wp->key, Jtostr (val)) @@ -496,24 +490,6 @@ int kvs_watch_int (flux_t *h, const char *key, kvs_set_int_f set, void *arg) return rc; } -int kvs_watch_int64 (flux_t *h, const char *key, kvs_set_int64_f set, void *arg) -{ - uint32_t matchtag; - kvs_watcher_t *wp; - json_object *val = NULL; - int rc = -1; - - if (watch_rpc (h, key, &val, 0, &matchtag) < 0) - goto done; - wp = add_watcher (h, key, WATCH_INT64, matchtag, set, arg); - dispatch_watch (h, wp, val); - rc = 0; -done: - if (val) - json_object_put (val); - return rc; -} - /* * vi:tabstop=4 shiftwidth=4 expandtab */ diff --git a/src/common/libkvs/kvs_watch.h b/src/common/libkvs/kvs_watch.h index b46dad406ab9..fe803a30d5ac 100644 --- a/src/common/libkvs/kvs_watch.h +++ b/src/common/libkvs/kvs_watch.h @@ -8,8 +8,6 @@ typedef int (*kvs_set_dir_f)(const char *key, kvsdir_t *dir, void *arg, typedef int (*kvs_set_string_f)(const char *key, const char *val, void *arg, int errnum); typedef int (*kvs_set_int_f)(const char *key, int val, void *arg, int errnum); -typedef int (*kvs_set_int64_f)(const char *key, int64_t val, void *arg, - int errnum); /* kvs_watch* is like kvs_get* except the registered callback is called * to set the value. It will be called immediately to set the initial @@ -25,7 +23,6 @@ int kvs_watch_dir (flux_t *h, kvs_set_dir_f set, void *arg, int kvs_watch_string (flux_t *h, const char *key, kvs_set_string_f set, void *arg); int kvs_watch_int (flux_t *h, const char *key, kvs_set_int_f set, void *arg); -int kvs_watch_int64 (flux_t *h, const char *key, kvs_set_int64_f set, void *arg); /* Cancel a kvs_watch, freeing server-side state, and unregistering any * callback. Returns 0 on success, or -1 with errno set on error. From be1e23d5c685fb3ad578f02ddab302afe63c78a4 Mon Sep 17 00:00:00 2001 From: Jim Garlick Date: Sat, 1 Jul 2017 21:30:00 +0000 Subject: [PATCH 14/19] libkvs/watch: drop kvs_watch_string --- src/common/libkvs/kvs_watch.c | 28 +--------------------------- src/common/libkvs/kvs_watch.h | 4 ---- 2 files changed, 1 insertion(+), 31 deletions(-) diff --git a/src/common/libkvs/kvs_watch.c b/src/common/libkvs/kvs_watch.c index c5949bfa051e..b900a8114363 100644 --- a/src/common/libkvs/kvs_watch.c +++ b/src/common/libkvs/kvs_watch.c @@ -54,8 +54,7 @@ typedef enum { - WATCH_STRING, WATCH_INT, - WATCH_JSONSTR, WATCH_DIR, + WATCH_INT, WATCH_JSONSTR, WATCH_DIR, } watch_type_t; typedef struct { @@ -189,12 +188,6 @@ static int dispatch_watch (flux_t *h, kvs_watcher_t *wp, json_object *val) int rc = -1; switch (wp->type) { - case WATCH_STRING: { - kvs_set_string_f set = wp->set; - const char *s = val ? json_object_get_string (val) : NULL; - rc = set (wp->key, s, wp->arg, errnum); - break; - } case WATCH_INT: { kvs_set_int_f set = wp->set; int i = val ? json_object_get_int (val) : 0; @@ -453,25 +446,6 @@ int kvs_watch_dir (flux_t *h, kvs_set_dir_f set, void *arg, const char *fmt, ... return rc; } -int kvs_watch_string (flux_t *h, const char *key, kvs_set_string_f set, - void *arg) -{ - uint32_t matchtag; - kvs_watcher_t *wp; - json_object *val = NULL; - int rc = -1; - - if (watch_rpc (h, key, &val, 0, &matchtag) < 0) - goto done; - wp = add_watcher (h, key, WATCH_STRING, matchtag, set, arg); - dispatch_watch (h, wp, val); - rc = 0; -done: - if (val) - json_object_put (val); - return rc; -} - int kvs_watch_int (flux_t *h, const char *key, kvs_set_int_f set, void *arg) { uint32_t matchtag; diff --git a/src/common/libkvs/kvs_watch.h b/src/common/libkvs/kvs_watch.h index fe803a30d5ac..fbbc5eaf9db9 100644 --- a/src/common/libkvs/kvs_watch.h +++ b/src/common/libkvs/kvs_watch.h @@ -5,8 +5,6 @@ typedef int (*kvs_set_f)(const char *key, const char *json_str, void *arg, int errnum); typedef int (*kvs_set_dir_f)(const char *key, kvsdir_t *dir, void *arg, int errnum); -typedef int (*kvs_set_string_f)(const char *key, const char *val, void *arg, - int errnum); typedef int (*kvs_set_int_f)(const char *key, int val, void *arg, int errnum); /* kvs_watch* is like kvs_get* except the registered callback is called @@ -20,8 +18,6 @@ int kvs_watch (flux_t *h, const char *key, kvs_set_f set, void *arg); int kvs_watch_dir (flux_t *h, kvs_set_dir_f set, void *arg, const char *fmt, ...) __attribute__ ((format (printf, 4, 5))); -int kvs_watch_string (flux_t *h, const char *key, kvs_set_string_f set, - void *arg); int kvs_watch_int (flux_t *h, const char *key, kvs_set_int_f set, void *arg); /* Cancel a kvs_watch, freeing server-side state, and unregistering any From 879962a0a5d0558ce46ccdf6441d5cc39dfb256a Mon Sep 17 00:00:00 2001 From: Jim Garlick Date: Sat, 1 Jul 2017 22:27:38 +0000 Subject: [PATCH 15/19] libkvs/watch: drop kvs_watch_int Move test users over to kvs_watch, then drop kvs_watch_int which has no non-test users. --- src/common/libkvs/kvs_watch.c | 26 +-------------- src/common/libkvs/kvs_watch.h | 2 -- t/kvs/commitmerge.c | 6 ++-- t/kvs/watch.c | 61 ++++++++++++++++++++++------------- 4 files changed, 43 insertions(+), 52 deletions(-) diff --git a/src/common/libkvs/kvs_watch.c b/src/common/libkvs/kvs_watch.c index b900a8114363..c87976791e0e 100644 --- a/src/common/libkvs/kvs_watch.c +++ b/src/common/libkvs/kvs_watch.c @@ -54,7 +54,7 @@ typedef enum { - WATCH_INT, WATCH_JSONSTR, WATCH_DIR, + WATCH_JSONSTR, WATCH_DIR, } watch_type_t; typedef struct { @@ -188,12 +188,6 @@ static int dispatch_watch (flux_t *h, kvs_watcher_t *wp, json_object *val) int rc = -1; switch (wp->type) { - case WATCH_INT: { - kvs_set_int_f set = wp->set; - int i = val ? json_object_get_int (val) : 0; - rc = set (wp->key, i, wp->arg, errnum); - break; - } case WATCH_DIR: { kvs_set_dir_f set = wp->set; kvsdir_t *dir = val ? kvsdir_create (h, NULL, wp->key, Jtostr (val)) @@ -446,24 +440,6 @@ int kvs_watch_dir (flux_t *h, kvs_set_dir_f set, void *arg, const char *fmt, ... return rc; } -int kvs_watch_int (flux_t *h, const char *key, kvs_set_int_f set, void *arg) -{ - uint32_t matchtag; - kvs_watcher_t *wp; - json_object *val = NULL; - int rc = -1; - - if (watch_rpc (h, key, &val, 0, &matchtag) < 0) - goto done; - wp = add_watcher (h, key, WATCH_INT, matchtag, set, arg); - dispatch_watch (h, wp, val); - rc = 0; -done: - if (val) - json_object_put (val); - return rc; -} - /* * vi:tabstop=4 shiftwidth=4 expandtab */ diff --git a/src/common/libkvs/kvs_watch.h b/src/common/libkvs/kvs_watch.h index fbbc5eaf9db9..1bbe5dbfe62e 100644 --- a/src/common/libkvs/kvs_watch.h +++ b/src/common/libkvs/kvs_watch.h @@ -5,7 +5,6 @@ typedef int (*kvs_set_f)(const char *key, const char *json_str, void *arg, int errnum); typedef int (*kvs_set_dir_f)(const char *key, kvsdir_t *dir, void *arg, int errnum); -typedef int (*kvs_set_int_f)(const char *key, int val, void *arg, int errnum); /* kvs_watch* is like kvs_get* except the registered callback is called * to set the value. It will be called immediately to set the initial @@ -18,7 +17,6 @@ int kvs_watch (flux_t *h, const char *key, kvs_set_f set, void *arg); int kvs_watch_dir (flux_t *h, kvs_set_dir_f set, void *arg, const char *fmt, ...) __attribute__ ((format (printf, 4, 5))); -int kvs_watch_int (flux_t *h, const char *key, kvs_set_int_f set, void *arg); /* Cancel a kvs_watch, freeing server-side state, and unregistering any * callback. Returns 0 on success, or -1 with errno set on error. diff --git a/t/kvs/commitmerge.c b/t/kvs/commitmerge.c index a58b56f62b39..fd2995a917b0 100644 --- a/t/kvs/commitmerge.c +++ b/t/kvs/commitmerge.c @@ -101,7 +101,7 @@ void watch_prepare_cb (flux_reactor_t *r, flux_watcher_t *w, flux_reactor_stop (r); } -static int watch_count_cb (const char *key, int val, void *arg, int errnum) +static int watch_count_cb (const char *key, const char *json_str, void *arg, int errnum) { thd_t *t = arg; @@ -161,8 +161,8 @@ void *watchthread (void *arg) r = flux_get_reactor (t->h); - if (kvs_watch_int (t->h, key, watch_count_cb, t) < 0) - log_err_exit ("kvs_watch_int %s", key); + if (kvs_watch (t->h, key, watch_count_cb, t) < 0) + log_err_exit ("kvs_watch %s", key); pw = flux_prepare_watcher_create (r, watch_prepare_cb, NULL); diff --git a/t/kvs/watch.c b/t/kvs/watch.c index f892672288f6..0cb8cea03ae8 100644 --- a/t/kvs/watch.c +++ b/t/kvs/watch.c @@ -33,6 +33,7 @@ #if HAVE_CONFIG_H #include "config.h" #endif +#include #include #include @@ -92,13 +93,21 @@ static void wait_ready (void) /* expect val: {-1,0,1,...,(changes - 1)} * count will therefore run 0...changes. */ -static int mt_watch_cb (const char *k, int val, void *arg, int errnum) +static int mt_watch_cb (const char *k, const char *json_str, void *arg, int errnum) { thd_t *t = arg; + json_t *obj; + int val; + if (errnum != 0) { log_errn (errnum, "%d: %s", t->n, __FUNCTION__); return -1; } + if (!(obj = json_loads (json_str, JSON_DECODE_ANY, NULL))) { + log_msg ("%d: %s failed to decode value", t->n, __FUNCTION__); + return -1; + } + val = json_integer_value (obj); if (val == t->last_val) { log_msg ("%d: %s: called with same value as last time: %d", t->n, __FUNCTION__, val); @@ -110,12 +119,13 @@ static int mt_watch_cb (const char *k, int val, void *arg, int errnum) if (val + 1 == changes) flux_reactor_stop (flux_get_reactor (t->h)); t->change_count++; + json_decref (obj); return 0; } /* Watch a key that does not exist throughout the test. */ -static int mt_watchnil_cb (const char *k, int val, void *arg, int errnum) +static int mt_watchnil_cb (const char *k, const char *json_str, void *arg, int errnum) { thd_t *t = arg; if (errnum != ENOENT) { @@ -128,7 +138,7 @@ static int mt_watchnil_cb (const char *k, int val, void *arg, int errnum) /* Watch a key that exists but does not change throughout the test. */ -static int mt_watchstable_cb (const char *k, int val, void *arg, int errnum) +static int mt_watchstable_cb (const char *k, const char *json_ttr, void *arg, int errnum) { thd_t *t = arg; @@ -152,16 +162,16 @@ void *thread (void *arg) /* The first kvs.watch reply is handled synchronously, then other kvs.watch * replies will arrive asynchronously and be handled by the reactor. */ - if (kvs_watch_int (t->h, key, mt_watch_cb, t) < 0) { - log_err ("%d: kvs_watch_int", t->n); + if (kvs_watch (t->h, key, mt_watch_cb, t) < 0) { + log_err ("%d: kvs_watch", t->n); goto done; } - if (kvs_watch_int (t->h, "nonexistent-key", mt_watchnil_cb, t) < 0) { - log_err ("%d: kvs_watch_int", t->n); + if (kvs_watch (t->h, "nonexistent-key", mt_watchnil_cb, t) < 0) { + log_err ("%d: kvs_watch", t->n); goto done; } - if (kvs_watch_int (t->h, key_stable, mt_watchstable_cb, t) < 0) { - log_err ("%d: kvs_watch_int", t->n); + if (kvs_watch (t->h, key_stable, mt_watchstable_cb, t) < 0) { + log_err ("%d: kvs_watch", t->n); goto done; } if (flux_reactor_run (flux_get_reactor (t->h), 0) < 0) { @@ -269,15 +279,22 @@ void test_mt (int argc, char **argv) flux_close (h); } -static int selfmod_watch_cb (const char *key, int val, void *arg, int errnum) +static int selfmod_watch_cb (const char *key, const char *json_str, void *arg, int errnum) { - log_msg ("%s: value = %d errnum = %d", __FUNCTION__, val, errnum); + int val; + json_t *obj; + + log_msg ("%s: value = %s errnum = %d", __FUNCTION__, json_str, errnum); + if (!(obj = json_loads (json_str, JSON_DECODE_ANY, NULL))) + log_msg_exit ("%s: failed to decode json value", __FUNCTION__); + val = json_integer_value (obj); flux_t *h = arg; if (kvs_put_int (h, key, val + 1) < 0) log_err_exit ("%s: kvs_put_int", __FUNCTION__); if (kvs_commit (h, 0) < 0) log_err_exit ("%s: kvs_commit", __FUNCTION__); + json_decref (obj); return (val == 0 ? -1 : 0); } @@ -298,8 +315,8 @@ void test_selfmod (int argc, char **argv) log_err_exit ("kvs_put_int"); if (kvs_commit (h, 0) < 0) log_err_exit ("kvs_commit"); - if (kvs_watch_int (h, key, selfmod_watch_cb, h) < 0) - log_err_exit ("kvs_watch_int"); + if (kvs_watch (h, key, selfmod_watch_cb, h) < 0) + log_err_exit ("kvs_watch"); log_msg ("reactor: start"); flux_reactor_run (flux_get_reactor (h), 0); @@ -308,7 +325,7 @@ void test_selfmod (int argc, char **argv) flux_close (h); } -static int unwatch_watch_cb (const char *key, int val, void *arg, int errnum) +static int unwatch_watch_cb (const char *key, const char *json_str, void *arg, int errnum) { int *count = arg; (*count)++; @@ -357,8 +374,8 @@ void test_unwatch (int argc, char **argv) if (!(ctx.h = flux_open (NULL, 0))) log_err_exit ("flux_open"); r = flux_get_reactor (ctx.h); - if (kvs_watch_int (ctx.h, ctx.key, unwatch_watch_cb, &count) < 0) - log_err_exit ("kvs_watch_int %s", ctx.key); + if (kvs_watch (ctx.h, ctx.key, unwatch_watch_cb, &count) < 0) + log_err_exit ("kvs_watch %s", ctx.key); if (!(timer = flux_timer_watcher_create (r, 0.001, 0.001, unwatch_timer_cb, &ctx))) log_err_exit ("flux_timer_watcher_create"); @@ -371,7 +388,7 @@ void test_unwatch (int argc, char **argv) flux_close (ctx.h); } -static int unwatchloop_cb (const char *key, int val, void *arg, int errnum) +static int unwatchloop_cb (const char *key, const char *json_str, void *arg, int errnum) { return 0; } @@ -394,8 +411,8 @@ void test_unwatchloop (int argc, char **argv) log_err_exit ("flux_open"); uint32_t avail = flux_matchtag_avail (h, 0); for (i = 0; i < 1000; i++) { - if (kvs_watch_int (h, key, unwatchloop_cb, NULL) < 0) - log_err_exit ("kvs_watch_int[%d] %s", i, key); + if (kvs_watch (h, key, unwatchloop_cb, NULL) < 0) + log_err_exit ("kvs_watch[%d] %s", i, key); if (kvs_unwatch (h, key) < 0) log_err_exit ("kvs_unwatch[%d] %s", i, key); } @@ -406,7 +423,7 @@ void test_unwatchloop (int argc, char **argv) flux_close (h); } -static int simulwatch_cb (const char *key, int val, void *arg, int errnum) +static int simulwatch_cb (const char *key, const char *json_str, void *arg, int errnum) { int *count = arg; (*count)++; @@ -447,8 +464,8 @@ void test_simulwatch (int argc, char **argv) if (get_watch_stats (h, &start) < 0) log_err_exit ("kvs.stats.get"); for (i = 0; i < max; i++) { - if (kvs_watch_int (h, key, simulwatch_cb, &count) < 0) - log_err_exit ("kvs_watch_int[%d] %s", i, key); + if (kvs_watch (h, key, simulwatch_cb, &count) < 0) + log_err_exit ("kvs_watch[%d] %s", i, key); if ((i % 4096 == 0 && i > 0 && i + 4096 < max)) log_msg ("%d of %d watchers registered (continuing)", i, max); } From 34cd81618156ae4bf3052f20e614c8c8e8c44d7e Mon Sep 17 00:00:00 2001 From: Jim Garlick Date: Sun, 2 Jul 2017 13:27:17 +0000 Subject: [PATCH 16/19] libkvs/watch: drop kvs_watch_once_int --- src/common/libkvs/kvs_watch.c | 21 --------------------- src/common/libkvs/kvs_watch.h | 1 - 2 files changed, 22 deletions(-) diff --git a/src/common/libkvs/kvs_watch.c b/src/common/libkvs/kvs_watch.c index c87976791e0e..8d160338aef0 100644 --- a/src/common/libkvs/kvs_watch.c +++ b/src/common/libkvs/kvs_watch.c @@ -337,27 +337,6 @@ int kvs_watch_once (flux_t *h, const char *key, char **valp) return rc; } -int kvs_watch_once_int (flux_t *h, const char *key, int *valp) -{ - json_object *val; - int rc = -1; - - if (!(val = json_object_new_int (*valp))) - oom (); - if (watch_rpc (h, key, &val, KVS_PROTO_ONCE, NULL) < 0) - goto done; - if (!val) { - errno = ENOENT; - goto done; - } - *valp = json_object_get_int (val); - rc = 0; -done: - if (val) - json_object_put (val); - return rc; -} - int kvs_watch_once_dir (flux_t *h, kvsdir_t **dirp, const char *fmt, ...) { json_object *val = NULL; diff --git a/src/common/libkvs/kvs_watch.h b/src/common/libkvs/kvs_watch.h index 1bbe5dbfe62e..9325bd6de742 100644 --- a/src/common/libkvs/kvs_watch.h +++ b/src/common/libkvs/kvs_watch.h @@ -37,7 +37,6 @@ int kvs_unwatch (flux_t *h, const char *key); int kvs_watch_once (flux_t *h, const char *key, char **json_str); int kvs_watch_once_dir (flux_t *h, kvsdir_t **dirp, const char *fmt, ...) __attribute__ ((format (printf, 3, 4))); -int kvs_watch_once_int (flux_t *h, const char *key, int *valp); #endif /* !_FLUX_CORE_KVS_WATCH_H */ From 5ac3ba1db8274756c521f9271df099e4e7fbf423 Mon Sep 17 00:00:00 2001 From: Jim Garlick Date: Sun, 2 Jul 2017 13:35:58 +0000 Subject: [PATCH 17/19] libkvs/deprecated: drop kvsdir_get_obj --- src/common/libkvs/kvs_deprecated.c | 11 ----------- src/common/libkvs/kvs_deprecated.h | 3 --- 2 files changed, 14 deletions(-) diff --git a/src/common/libkvs/kvs_deprecated.c b/src/common/libkvs/kvs_deprecated.c index 245b9d50aede..0a4c8403597e 100644 --- a/src/common/libkvs/kvs_deprecated.c +++ b/src/common/libkvs/kvs_deprecated.c @@ -63,17 +63,6 @@ int kvs_get_obj (flux_t *h, const char *key, json_object **valp) return common_get_obj (h, key, valp); } -int kvsdir_get_obj (kvsdir_t *dir, const char *name, json_object **valp) -{ - flux_t *h = kvsdir_handle (dir); - char *key = kvsdir_key_at (dir, name); - int rc; - - rc = common_get_obj (h, key, valp); - free (key); - return rc; -} - /* Put */ diff --git a/src/common/libkvs/kvs_deprecated.h b/src/common/libkvs/kvs_deprecated.h index c69df7864ab9..ae218c0a1f5c 100644 --- a/src/common/libkvs/kvs_deprecated.h +++ b/src/common/libkvs/kvs_deprecated.h @@ -23,9 +23,6 @@ int kvs_watch_once_obj (flux_t *h, const char *key, json_object **valp) int kvs_put_obj (flux_t *h, const char *key, json_object *val) __attribute__ ((deprecated)); -int kvsdir_get_obj (kvsdir_t *dir, const char *key, json_object **valp) - __attribute__ ((deprecated)); - int kvsdir_put_obj (kvsdir_t *dir, const char *key, json_object *val) __attribute__ ((deprecated)); From 2a1c19403a0f616518127deb62396d2d67c7f762 Mon Sep 17 00:00:00 2001 From: Jim Garlick Date: Sun, 2 Jul 2017 13:37:36 +0000 Subject: [PATCH 18/19] libkvs/deprecated: drop kvs_put_obj --- src/common/libkvs/kvs_deprecated.c | 5 ----- src/common/libkvs/kvs_deprecated.h | 3 --- 2 files changed, 8 deletions(-) diff --git a/src/common/libkvs/kvs_deprecated.c b/src/common/libkvs/kvs_deprecated.c index 0a4c8403597e..0a78eba1a0ad 100644 --- a/src/common/libkvs/kvs_deprecated.c +++ b/src/common/libkvs/kvs_deprecated.c @@ -80,11 +80,6 @@ static int common_put_obj (flux_t *h, const char *key, json_object *val) return rc; } -int kvs_put_obj (flux_t *h, const char *key, json_object *val) -{ - return common_put_obj (h, key, val); -} - int kvsdir_put_obj (kvsdir_t *dir, const char *name, json_object *val) { flux_t *h = kvsdir_handle (dir); diff --git a/src/common/libkvs/kvs_deprecated.h b/src/common/libkvs/kvs_deprecated.h index ae218c0a1f5c..81666a95629c 100644 --- a/src/common/libkvs/kvs_deprecated.h +++ b/src/common/libkvs/kvs_deprecated.h @@ -20,9 +20,6 @@ int kvs_watch_obj (flux_t *h, const char *key, kvs_set_obj_f set, void *arg) int kvs_watch_once_obj (flux_t *h, const char *key, json_object **valp) __attribute__ ((deprecated)); -int kvs_put_obj (flux_t *h, const char *key, json_object *val) - __attribute__ ((deprecated)); - int kvsdir_put_obj (kvsdir_t *dir, const char *key, json_object *val) __attribute__ ((deprecated)); From 3718a7af063c6dfd3a00ec552d69b9f5361dff3b Mon Sep 17 00:00:00 2001 From: Jim Garlick Date: Mon, 3 Jul 2017 16:02:04 +0000 Subject: [PATCH 19/19] libkvs: install kvs.h into core subdirectory Fixes flux-framework/flux-sched#251 --- src/common/libkvs/Makefile.am | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/libkvs/Makefile.am b/src/common/libkvs/Makefile.am index 23156bdca8d3..09dbb6b66c74 100644 --- a/src/common/libkvs/Makefile.am +++ b/src/common/libkvs/Makefile.am @@ -25,7 +25,7 @@ libkvs_la_SOURCES = \ kvs_watch.c -fluxinclude_HEADERS = \ +fluxcoreinclude_HEADERS = \ kvs.h \ kvs_lookup.h \ kvs_dir.h \