diff --git a/etc/rc1 b/etc/rc1 index dfb6b615d37c..a24e53bc2c3e 100755 --- a/etc/rc1 +++ b/etc/rc1 @@ -11,11 +11,15 @@ modload() { fi } +backing_module() { + local backingmod=$(flux getattr content.backing-module 2>/dev/null) || : + echo ${backingmod:-content-sqlite} +} + modload all barrier if test $RANK -eq 0; then - backingmod=$(flux getattr content.backing-module 2>/dev/null) || : - backingmod=${backingmod:-content-sqlite} + backingmod=$(backing_module) dumpfile=$(flux getattr content.restore 2>/dev/null) || : if test -n "${dumpfile}"; then if test "${dumpfile}" = "auto"; then @@ -30,13 +34,15 @@ if test $RANK -eq 0; then fi fi if test -n "${dumpfile}"; then - flux module load ${backingmod} truncate + if test "${backingmod}" != "none"; then + flux module load ${backingmod} truncate + fi echo "restoring content from ${dumpfile}" flux restore --quiet --checkpoint ${dumpfile} if test -n "${dumplink}"; then rm -f ${dumplink} fi - else + elif test "${backingmod}" != "none"; then flux module load ${backingmod} fi fi @@ -45,7 +51,9 @@ modload all kvs modload all kvs-watch if test $RANK -eq 0; then - flux startlog --post-start-event + if test "$(backing_module)" != "none"; then + flux startlog --post-start-event + fi fi modload all resource @@ -58,9 +66,13 @@ if test $RANK -eq 0 -a -n "${period}"; then flux module load job-archive fi -if test $RANK -eq 0 && ! flux startlog --check --quiet; then - flux queue stop - flux queue disable "Flux is in safe mode due to an incomplete shutdown." +if test $RANK -eq 0; then + if test "$(backing_module)" != "none"; then + if ! flux startlog --check --quiet; then + flux queue stop + flux queue disable "Flux is in safe mode due to an incomplete shutdown." + fi + fi fi modload all job-ingest diff --git a/etc/rc3 b/etc/rc3 index 97bc61e2f6cb..4fce37658ad1 100755 --- a/etc/rc3 +++ b/etc/rc3 @@ -11,6 +11,11 @@ modrm() { fi } +backing_module() { + local backingmod=$(flux getattr content.backing-module 2>/dev/null) || : + echo ${backingmod:-content-sqlite} +} + core_dir=$(cd ${0%/*} && pwd -P) all_dirs=$core_dir${FLUX_RC_EXTRA:+":$FLUX_RC_EXTRA"} IFS=: @@ -37,17 +42,20 @@ modrm 0 cron modrm all barrier if test $RANK -eq 0; then - flux startlog --post-finish-event || exit_rc=1 + if test "$(backing_module)" != "none"; then + flux startlog --post-finish-event || exit_rc=1 + fi fi modrm all kvs-watch modrm all kvs -flux content flush || exit_rc=1 +if test "$(backing_module)" != "none"; then + flux content flush || exit_rc=1 +fi if test $RANK -eq 0; then - backingmod=$(flux getattr content.backing-module 2>/dev/null) - backingmod=${backingmod:-content-sqlite} + backingmod=$(backing_module) dumpfile=$(flux getattr content.dump 2>/dev/null) if test $exit_rc -eq 0 -a -n "${dumpfile}"; then if test "${dumpfile}" = "auto"; then @@ -63,7 +71,9 @@ if test $RANK -eq 0; then exit_rc=1 fi fi - flux module remove ${backingmod} || exit_rc=1 + if test "${backingmod}" != "none"; then + flux module remove ${backingmod} || exit_rc=1 + fi fi exit $exit_rc diff --git a/src/broker/content-cache.c b/src/broker/content-cache.c index 2d12fad71ae0..ab21020d6a77 100644 --- a/src/broker/content-cache.c +++ b/src/broker/content-cache.c @@ -692,6 +692,7 @@ static void content_register_backing_request (flux_t *h, if (flux_respond (h, msg, NULL) < 0) flux_log_error (h, "error responding to register-backing request"); (void)cache_flush (cache); + (void)checkpoints_flush (cache->checkpoint); return; error: if (flux_respond_error (h, msg, errno, errstr) < 0) diff --git a/src/broker/content-checkpoint.c b/src/broker/content-checkpoint.c index 769ec9ebc08d..36a8bbcd277c 100644 --- a/src/broker/content-checkpoint.c +++ b/src/broker/content-checkpoint.c @@ -15,6 +15,7 @@ #endif #include #include +#include #include #include "src/common/libczmqcontainers/czmq_containers.h" @@ -27,21 +28,95 @@ struct content_checkpoint { flux_msg_handler_t **handlers; uint32_t rank; struct content_cache *cache; + zhashx_t *hash; + unsigned int hash_dirty; }; +struct checkpoint_data { + struct content_checkpoint *checkpoint; + json_t *value; + uint8_t dirty:1; + bool in_progress; + int refcount; +}; + +static struct checkpoint_data * +checkpoint_data_incref (struct checkpoint_data *data) +{ + if (data) + data->refcount++; + return data; +} + +static void checkpoint_data_decref (struct checkpoint_data *data) +{ + if (data && --data->refcount == 0) { + if (data->dirty) + data->checkpoint->hash_dirty--; + json_decref (data->value); + free (data); + } +} + +/* zhashx_destructor_fn */ +static void checkpoint_data_decref_wrapper (void **arg) +{ + if (arg) { + struct checkpoint_data *data = *arg; + checkpoint_data_decref (data); + } +} + +static struct checkpoint_data * +checkpoint_data_create (struct content_checkpoint *checkpoint, + json_t *value) +{ + struct checkpoint_data *data = NULL; + + if (!(data = calloc (1, sizeof (*data)))) + return NULL; + data->checkpoint = checkpoint; + data->value = json_incref (value); + data->refcount = 1; + return data; +} + +static int checkpoint_data_update (struct content_checkpoint *checkpoint, + const char *key, + json_t *value) +{ + struct checkpoint_data *data = NULL; + + if (!(data = checkpoint_data_create (checkpoint, value))) + return -1; + + zhashx_update (checkpoint->hash, key, data); + data->dirty = 1; + checkpoint->hash_dirty++; + return 0; +} + static void checkpoint_get_continuation (flux_future_t *f, void *arg) { struct content_checkpoint *checkpoint = arg; const flux_msg_t *msg = flux_future_aux_get (f, "msg"); - const char *s; + const char *key; + json_t *value = NULL; assert (msg); - if (flux_rpc_get (f, &s) < 0) + if (flux_request_unpack (msg, NULL, "{s:s}", "key", &key) < 0) goto error; - if (flux_respond (checkpoint->h, msg, s) < 0) + if (flux_rpc_get_unpack (f, "{s:o}", "value", &value) < 0) + goto error; + + if (checkpoint_data_update (checkpoint, key, value) < 0) + goto error; + + if (flux_respond_pack (checkpoint->h, msg, "{s:O}", "value", value) < 0) flux_log_error (checkpoint->h, "error responding to checkpoint-get"); + flux_future_destroy (f); return; @@ -53,7 +128,7 @@ static void checkpoint_get_continuation (flux_future_t *f, void *arg) static int checkpoint_get_forward (struct content_checkpoint *checkpoint, const flux_msg_t *msg, - const char *s, + const char *key, const char **errstr) { const char *topic = "content.checkpoint-get"; @@ -66,7 +141,12 @@ static int checkpoint_get_forward (struct content_checkpoint *checkpoint, rank = 0; } - if (!(f = flux_rpc (checkpoint->h, topic, s, rank, 0)) + if (!(f = flux_rpc_pack (checkpoint->h, + topic, + rank, + 0, + "{s:s}", + "key", key)) || flux_future_then (f, -1, checkpoint_get_continuation, @@ -93,20 +173,31 @@ void content_checkpoint_get_request (flux_t *h, flux_msg_handler_t *mh, const flux_msg_t *msg, void *arg) { struct content_checkpoint *checkpoint = arg; - const char *s = NULL; + const char *key; const char *errstr = NULL; - if (checkpoint->rank == 0) { - if (!content_cache_backing_loaded (checkpoint->cache)) { - errno = ENOSYS; + if (flux_request_unpack (msg, NULL, "{s:s}", "key", &key) < 0) + goto error; + + if (checkpoint->rank == 0 + && !content_cache_backing_loaded (checkpoint->cache)) { + struct checkpoint_data *data = zhashx_lookup (checkpoint->hash, key); + if (!data) { + errstr = "checkpoint key unavailable"; + errno = ENOENT; goto error; } + if (flux_respond_pack (h, msg, + "{s:O}", + "value", data->value) < 0) + flux_log_error (h, "error responding to checkpoint-get"); + return; } - if (flux_request_decode (msg, NULL, &s) < 0) - goto error; - - if (checkpoint_get_forward (checkpoint, msg, s, &errstr) < 0) + if (checkpoint_get_forward (checkpoint, + msg, + key, + &errstr) < 0) goto error; return; @@ -140,7 +231,8 @@ static void checkpoint_put_continuation (flux_future_t *f, void *arg) static int checkpoint_put_forward (struct content_checkpoint *checkpoint, const flux_msg_t *msg, - const char *s, + const char *key, + json_t *value, const char **errstr) { const char *topic = "content.checkpoint-put"; @@ -153,7 +245,10 @@ static int checkpoint_put_forward (struct content_checkpoint *checkpoint, rank = 0; } - if (!(f = flux_rpc (checkpoint->h, topic, s, rank, 0)) + if (!(f = flux_rpc_pack (checkpoint->h, topic, rank, 0, + "{s:s s:O}", + "key", key, + "value", value)) || flux_future_then (f, -1, checkpoint_put_continuation, @@ -180,20 +275,35 @@ void content_checkpoint_put_request (flux_t *h, flux_msg_handler_t *mh, const flux_msg_t *msg, void *arg) { struct content_checkpoint *checkpoint = arg; - const char *s = NULL; + const char *key; + json_t *value; const char *errstr = NULL; + if (flux_request_unpack (msg, + NULL, + "{s:s s:o}", + "key", + &key, + "value", + &value) < 0) + goto error; + if (checkpoint->rank == 0) { - if (!content_cache_backing_loaded (checkpoint->cache)) { - errno = ENOSYS; + if (checkpoint_data_update (checkpoint, key, value) < 0) goto error; + + if (!content_cache_backing_loaded (checkpoint->cache)) { + if (flux_respond (h, msg, NULL) < 0) + flux_log_error (checkpoint->h, "error responding to checkpoint-put"); + return; } } - if (flux_request_decode (msg, NULL, &s) < 0) - goto error; - - if (checkpoint_put_forward (checkpoint, msg, s, &errstr) < 0) + if (checkpoint_put_forward (checkpoint, + msg, + key, + value, + &errstr) < 0) goto error; return; @@ -203,6 +313,72 @@ void content_checkpoint_put_request (flux_t *h, flux_msg_handler_t *mh, flux_log_error (h, "error responding to checkpoint-put request"); } +static void checkpoint_flush_continuation (flux_future_t *f, void *arg) +{ + struct checkpoint_data *data = arg; + int rv; + + assert (data); + if ((rv = flux_rpc_get (f, NULL)) < 0) + flux_log_error (data->checkpoint->h, "checkpoint flush rpc"); + if (!rv) { + data->dirty = 0; + data->checkpoint->hash_dirty--; + } + data->in_progress = false; + checkpoint_data_decref (data); + flux_future_destroy (f); +} + +static int checkpoint_flush (struct content_checkpoint *checkpoint, + struct checkpoint_data *data) +{ + if (data->dirty && !data->in_progress) { + const char *key = zhashx_cursor (checkpoint->hash); + const char *topic = "content-backing.checkpoint-put"; + flux_future_t *f; + if (!(f = flux_rpc_pack (checkpoint->h, topic, 0, 0, + "{s:s s:O}", + "key", key, + "value", data->value)) + || flux_future_then (f, + -1, + checkpoint_flush_continuation, + (void *)checkpoint_data_incref (data)) < 0) { + flux_log_error (checkpoint->h, "%s: checkpoint flush", __FUNCTION__); + flux_future_destroy (f); + return -1; + } + data->in_progress = true; + } + return 0; +} + +int checkpoints_flush (struct content_checkpoint *checkpoint) +{ + int last_errno = 0; + int rc = 0; + + if (checkpoint->hash_dirty > 0) { + struct checkpoint_data *data = zhashx_first (checkpoint->hash); + while (data) { + if (checkpoint_flush (checkpoint, data) < 0) { + last_errno = errno; + rc = -1; + /* A few errors we will consider "unrecoverable", so + * break out */ + if (errno == ENOSYS + || errno == ENOMEM) + break; + } + data = zhashx_next (checkpoint->hash); + } + } + if (rc < 0) + errno = last_errno; + return rc; +} + static const struct flux_msg_handler_spec htab[] = { { FLUX_MSGTYPE_REQUEST, @@ -224,6 +400,7 @@ void content_checkpoint_destroy (struct content_checkpoint *checkpoint) if (checkpoint) { int saved_errno = errno; flux_msg_handler_delvec (checkpoint->handlers); + zhashx_destroy (&checkpoint->hash); free (checkpoint); errno = saved_errno; } @@ -241,10 +418,17 @@ struct content_checkpoint *content_checkpoint_create ( checkpoint->h = h; checkpoint->rank = rank; checkpoint->cache = cache; + + if (!(checkpoint->hash = zhashx_new ())) + goto nomem; + zhashx_set_destructor (checkpoint->hash, checkpoint_data_decref_wrapper); + if (flux_msg_handler_addvec (h, htab, checkpoint, &checkpoint->handlers) < 0) goto error; return checkpoint; +nomem: + errno = ENOMEM; error: content_checkpoint_destroy (checkpoint); return NULL; diff --git a/src/broker/content-checkpoint.h b/src/broker/content-checkpoint.h index 3a2593354fbe..853bd9cbcd6d 100644 --- a/src/broker/content-checkpoint.h +++ b/src/broker/content-checkpoint.h @@ -19,6 +19,8 @@ struct content_checkpoint *content_checkpoint_create ( struct content_cache *cache); void content_checkpoint_destroy (struct content_checkpoint *checkpoint); +int checkpoints_flush (struct content_checkpoint *checkpoint); + #endif /* !HAVE_BROKER_CONTENT_CHECKPOINT_H */ /* diff --git a/t/Makefile.am b/t/Makefile.am index 9028ec5c18e4..e06dd594dfc1 100644 --- a/t/Makefile.am +++ b/t/Makefile.am @@ -69,6 +69,7 @@ TESTSCRIPTS = \ t0011-content-cache.t \ t0012-content-sqlite.t \ t0024-content-s3.t \ + t0028-content-backing-none.t \ t0025-broker-state-machine.t \ t0027-broker-groups.t \ t0013-config-file.t \ diff --git a/t/content/content-helper.sh b/t/content/content-helper.sh index 9e72aead8f75..6530a73ec266 100755 --- a/t/content/content-helper.sh +++ b/t/content/content-helper.sh @@ -30,3 +30,17 @@ checkpoint_get() { o=$(checkpoint_get_msg $1) jq -j -c -n ${o} | $RPC content.checkpoint-get } + +# Identical to checkpoint_put(), but go directly to backing store +# Usage: checkpoint_put key rootref +checkpoint_backing_put() { + o=$(checkpoint_put_msg $1 $2) + jq -j -c -n ${o} | $RPC content-backing.checkpoint-put +} + +# Identical to checkpoint_get(), but go directly to backing store +# Usage: checkpoint_get key +checkpoint_backing_get() { + o=$(checkpoint_get_msg $1) + jq -j -c -n ${o} | $RPC content-backing.checkpoint-get +} diff --git a/t/t0012-content-sqlite.t b/t/t0012-content-sqlite.t index 3099f1c05147..4cfcbc39230d 100755 --- a/t/t0012-content-sqlite.t +++ b/t/t0012-content-sqlite.t @@ -174,6 +174,24 @@ test_expect_success HAVE_JQ 'checkpoint-get foo still returns rootref baz' ' test_cmp rootref3.exp rootref3.out ' +test_expect_success HAVE_JQ 'checkpoint-backing-get foo returns rootref baz' ' + echo baz >rootref_backing.exp && + checkpoint_backing_get foo \ + | jq -r .value \ + | jq -r .rootref >rootref_backing.out && + test_cmp rootref_backing.exp rootref_backing.out +' + +test_expect_success HAVE_JQ 'checkpoint-backing-put foo w/ rootref boof' ' + checkpoint_backing_put foo boof +' + +test_expect_success HAVE_JQ 'checkpoint-get foo returned rootref boof' ' + echo boof >rootref4.exp && + checkpoint_get foo | jq -r .value | jq -r .rootref >rootref4.out && + test_cmp rootref4.exp rootref4.out +' + test_expect_success HAVE_JQ 'checkpoint-get noexist fails with No such...' ' test_must_fail checkpoint_get noexist 2>badkey.err && grep "No such file or directory" badkey.err @@ -206,6 +224,49 @@ test_expect_success 'remove content-sqlite module on rank 0' ' flux module remove content-sqlite ' +test_expect_success HAVE_JQ 'checkpoint-put foo w/ rootref spoon' ' + checkpoint_put foo spoon +' + +test_expect_success HAVE_JQ 'checkpoint-get foo returned rootref spoon' ' + echo spoon >rootref5.exp && + checkpoint_get foo | jq -r .value | jq -r .rootref >rootref5.out && + test_cmp rootref5.exp rootref5.out +' + +test_expect_success 'load content-sqlite module on rank 0' ' + flux module load content-sqlite +' + +# arg1 - expected reference +wait_checkpoint_flush() { + local expected=$1 + local i=0 + while checkpoint_backing_get foo \ + | jq -r .value \ + | jq -r .rootref > checkpointflush.out \ + && [ $i -lt 50 ] + do + checkpoint=$(cat checkpointflush.out) + if [ "${checkpoint}" = "${expected}" ] + then + return 0 + fi + sleep 0.1 + i=$((i + 1)) + done + return 1 +} + +test_expect_success HAVE_JQ 'checkpoint-backing-get foo returns spoon' ' + wait_checkpoint_flush spoon +' + +test_expect_success 'remove content-sqlite module on rank 0' ' + flux content flush && + flux module remove content-sqlite +' + test_expect_success 'remove heartbeat module' ' flux module remove heartbeat ' diff --git a/t/t0018-content-files.t b/t/t0018-content-files.t index 67430f7a9c67..f661357e5409 100755 --- a/t/t0018-content-files.t +++ b/t/t0018-content-files.t @@ -214,6 +214,28 @@ test_expect_success HAVE_JQ 'checkpoint-get foo returned rootref with shorter ro test_cmp rootref5.exp rootref5.out ' +test_expect_success HAVE_JQ 'checkpoint-put updates foo rootref to boof' ' + checkpoint_put foo boof +' + +test_expect_success HAVE_JQ 'checkpoint-backing-get foo returns rootref boof' ' + echo boof >rootref_backing.exp && + checkpoint_backing_get foo \ + | jq -r .value \ + | jq -r .rootref >rootref_backing.out && + test_cmp rootref_backing.exp rootref_backing.out +' + +test_expect_success HAVE_JQ 'checkpoint-backing-put foo w/ rootref poof' ' + checkpoint_backing_put foo poof +' + +test_expect_success HAVE_JQ 'checkpoint-get foo returned rootref boof' ' + echo poof >rootref6.exp && + checkpoint_get foo | jq -r .value | jq -r .rootref >rootref6.out && + test_cmp rootref6.exp rootref6.out +' + test_expect_success 'checkpoint-get bad request fails with EPROTO' ' test_must_fail $RPC content.checkpoint-get badget.err && grep "Protocol error" badget.err @@ -227,5 +249,47 @@ test_expect_success 'remove content-files module' ' flux module remove content-files ' +test_expect_success HAVE_JQ 'checkpoint-put foo w/ rootref spoon' ' + checkpoint_put foo spoon +' + +test_expect_success HAVE_JQ 'checkpoint-get foo returned rootref spoon' ' + echo spoon >rootref7.exp && + checkpoint_get foo | jq -r .value | jq -r .rootref >rootref7.out && + test_cmp rootref7.exp rootref7.out +' + +test_expect_success 'load content-files module on rank 0' ' + flux module load content-files +' + +# arg1 - expected reference +wait_checkpoint_flush() { + local expected=$1 + local i=0 + while checkpoint_backing_get foo \ + | jq -r .value \ + | jq -r .rootref > checkpointflush.out \ + && [ $i -lt 50 ] + do + checkpoint=$(cat checkpointflush.out) + if [ "${checkpoint}" = "${expected}" ] + then + return 0 + fi + sleep 0.1 + i=$((i + 1)) + done + return 1 +} + +test_expect_success HAVE_JQ 'checkpoint-backing-get foo returns spoon' ' + wait_checkpoint_flush spoon +' + +test_expect_success 'remove content-files module on rank 0' ' + flux content flush && + flux module remove content-files +' test_done diff --git a/t/t0024-content-s3.t b/t/t0024-content-s3.t index 9c8799061754..60e38960231f 100755 --- a/t/t0024-content-s3.t +++ b/t/t0024-content-s3.t @@ -169,6 +169,24 @@ test_expect_success HAVE_JQ 'checkpoint-get foo still returns rootref baz' ' test_cmp rootref3.exp rootref3.out ' +test_expect_success HAVE_JQ 'checkpoint-backing-get foo returns rootref baz' ' + echo baz >rootref_backing.exp && + checkpoint_backing_get foo \ + | jq -r .value \ + | jq -r .rootref >rootref_backing.out && + test_cmp rootref_backing.exp rootref_backing.out +' + +test_expect_success HAVE_JQ 'checkpoint-backing-put foo w/ rootref boof' ' + checkpoint_backing_put foo boof +' + +test_expect_success HAVE_JQ 'checkpoint-get foo returned rootref boof' ' + echo boof >rootref4.exp && + checkpoint_get foo | jq -r .value | jq -r .rootref >rootref4.out && + test_cmp rootref4.exp rootref4.out +' + test_expect_success 'config: reload config does not take effect immediately' ' flux config reload 2>/dev/null && flux dmesg | grep content-s3 | tail -1 >reload.log && @@ -204,6 +222,51 @@ test_expect_success 'config: unload module' ' flux module remove content-s3 ' +test_expect_success HAVE_JQ 'checkpoint-put foo w/ rootref spoon' ' + checkpoint_put foo spoon +' + +test_expect_success HAVE_JQ 'checkpoint-get foo returned rootref spoon' ' + echo spoon >rootref5.exp && + checkpoint_get foo | jq -r .value | jq -r .rootref >rootref5.out && + test_cmp rootref5.exp rootref5.out +' + +test_expect_success 'load content-s3 module on rank 0' ' + cp content-s3.save content-s3.toml && + cp creds/creds.save creds/creds.toml && + flux config reload && + flux module load content-s3 +' + +# arg1 - expected reference +wait_checkpoint_flush() { + local expected=$1 + local i=0 + while checkpoint_backing_get foo \ + | jq -r .value \ + | jq -r .rootref > checkpointflush.out \ + && [ $i -lt 50 ] + do + checkpoint=$(cat checkpointflush.out) + if [ "${checkpoint}" = "${expected}" ] + then + return 0 + fi + sleep 0.1 + i=$((i + 1)) + done + return 1 +} + +test_expect_success HAVE_JQ 'checkpoint-backing-get foo returns spoon' ' + wait_checkpoint_flush spoon +' + +test_expect_success 'config: unload module' ' + flux module remove content-s3 +' + test_expect_success 'config: module fails to load without config file' ' rm -f content-s3.toml && cp creds/creds.save creds/creds.toml && diff --git a/t/t0028-content-backing-none.t b/t/t0028-content-backing-none.t new file mode 100755 index 000000000000..5febc7f1cb3e --- /dev/null +++ b/t/t0028-content-backing-none.t @@ -0,0 +1,87 @@ +#!/bin/sh + +test_description='Test broker content checkpoint w/o backing module' + +. `dirname $0`/content/content-helper.sh + +. `dirname $0`/sharness.sh + +test_under_flux 2 minimal +echo "# $0: flux session size will be ${SIZE}" + +RPC=${FLUX_BUILD_DIR}/t/request/rpc + +test_expect_success HAVE_JQ 'checkpoint-get fails, no checkpoints yet' ' + checkpoint_put foo bar +' + +test_expect_success HAVE_JQ 'checkpoint-put foo w/ rootref bar' ' + checkpoint_put foo bar +' + +test_expect_success HAVE_JQ 'checkpoint-get foo returned rootref bar' ' + echo bar >rootref.exp && + checkpoint_get foo | jq -r .value | jq -r .rootref >rootref.out && + test_cmp rootref.exp rootref.out +' + +test_expect_success HAVE_JQ 'checkpoint-put on rank 1 forwards to rank 0' ' + o=$(checkpoint_put_msg rankone rankref) && + jq -j -c -n ${o} | flux exec -r 1 ${RPC} content.checkpoint-put +' + +test_expect_success HAVE_JQ 'checkpoint-get on rank 1 forwards to rank 0' ' + echo rankref >rankref.exp && + o=$(checkpoint_get_msg rankone) && + jq -j -c -n ${o} \ + | flux exec -r 1 ${RPC} content.checkpoint-get \ + | jq -r .value | jq -r .rootref > rankref.out && + test_cmp rankref.exp rankref.out +' + +test_expect_success 'flux-dump --checkpoint with missing checkpoint fails' ' + test_must_fail flux dump --checkpoint foo.tar +' + +test_expect_success 'load kvs and create some kvs data' ' + flux module load kvs && + flux kvs put a=1 && + flux kvs put b=foo +' + +test_expect_success 'reload kvs' ' + flux module reload kvs && + test $(flux kvs get a) = "1" && + test $(flux kvs get b) = "foo" +' + +test_expect_success 'unload kvs' ' + flux module remove kvs +' + +test_expect_success 'dump default=kvs-primary checkpoint works' ' + flux dump --checkpoint foo.tar +' + +test_expect_success 'restore content' ' + flux restore --checkpoint foo.tar +' + +test_expect_success 'reload kvs' ' + flux module load kvs +' + +test_expect_success 'verify KVS content restored' ' + test $(flux kvs get a) = "1" && + test $(flux kvs get b) = "foo" +' + +test_expect_success 'unload kvs' ' + flux module remove kvs +' + +test_expect_success 'content.backing-module input of none works' ' + flux start -o,-Scontent.backing-module=none /bin/true +' + +test_done diff --git a/t/t2807-dump-cmd.t b/t/t2807-dump-cmd.t index 0181b7ddb4bc..922ad8b74e5a 100755 --- a/t/t2807-dump-cmd.t +++ b/t/t2807-dump-cmd.t @@ -23,7 +23,7 @@ test_expect_success 'flux-restore with no args prints Usage message' ' ' test_expect_success 'flux-dump with no backing store fails' ' test_must_fail flux dump --checkpoint foo.tar 2>nostore.err && - grep "Function not implemented" nostore.err + grep "checkpoint key unavailable" nostore.err ' test_expect_success 'flux-dump with bad archive file fails' ' test_must_fail flux dump /badfile.tar 2>badfile.err && @@ -177,9 +177,9 @@ test_expect_success 'restore to key fails when kvs is not loaded' ' test_expect_success 'unload content-sqlite' ' flux module remove content-sqlite ' -test_expect_success 'restore --checkpoint with no backing store fails' ' - test_must_fail flux restore --checkpoint foo.tar 2>noback.err && - grep "error updating checkpoint" noback.err +test_expect_success 'restore --checkpoint with no backing store cant flush' ' + flux restore --checkpoint foo.tar 2>noback.err && + grep "error flushing content cache" noback.err ' test_expect_success 'dump --no-cache with no backing store fails' ' test_must_fail flux dump --no-cache --checkpoint x.tar