diff --git a/src/cmd/builtin/startlog.c b/src/cmd/builtin/startlog.c index 82639fb237e7..3895eb69aa12 100644 --- a/src/cmd/builtin/startlog.c +++ b/src/cmd/builtin/startlog.c @@ -46,26 +46,45 @@ static void content_flush (flux_t *h) static void kvs_checkpoint_put (flux_t *h, const char *treeobj) { json_t *o; + json_t *value; const char *rootref; flux_future_t *f; + double timestamp; if (!(o = treeobj_decode (treeobj)) || !(rootref = treeobj_get_blobref (o, 0))) log_err_exit ("Error decoding treeobj from eventlog commit"); + + timestamp = flux_reactor_now (flux_get_reactor (h)); + + /* version 0 checkpoint + * - blobref string only + * version 1 checkpoint object + * - {"version":1 "rootref":s "timestamp":f} + */ + if (!(value = json_pack ("{s:i s:s s:f}", + "version", 1, + "rootref", rootref, + "timestamp", timestamp))) { + errno = ENOMEM; + log_err_exit ("Error encoding checkpoint object"); + } + if (!(f = flux_rpc_pack (h, "kvs-checkpoint.put", 0, 0, - "{s:s s:s}", + "{s:s s:O}", "key", "kvs-primary", "value", - rootref)) + value)) || flux_rpc_get (f, NULL) < 0) log_msg_exit ("Error writing kvs checkpoint: %s", future_strerror (f, errno)); flux_future_destroy (f); json_decref (o); + json_decref (value); } static void post_startlog_event (flux_t *h, diff --git a/src/modules/content-files/content-files.c b/src/modules/content-files/content-files.c index 0879384d53f9..43e6d89fe24c 100644 --- a/src/modules/content-files/content-files.c +++ b/src/modules/content-files/content-files.c @@ -52,6 +52,7 @@ #include "config.h" #endif #include +#include #include "src/common/libutil/blobref.h" #include "src/common/libutil/log.h" @@ -157,24 +158,36 @@ void checkpoint_get_cb (flux_t *h, const char *key; void *data = NULL; size_t size; + json_t *o = NULL; const char *errstr = NULL; if (flux_request_unpack (msg, NULL, "{s:s}", "key", &key) < 0) goto error; if (filedb_get (ctx->dbpath, key, &data, &size, &errstr) < 0) goto error; + if (size > 0) { + /* recovery from version 0 checkpoint blobref not supported */ + if (!(o = json_loads (data, 0, NULL))) { + errno = EINVAL; + goto error; + } + } + else + o = json_null (); if (flux_respond_pack (h, msg, - "{s:s}", + "{s:O}", "value", - size > 0 ? data : "") < 0) + o) < 0) flux_log_error (h, "error responding to kvs-checkpoint.get request"); free (data); + json_decref (o); return; error: if (flux_respond_error (h, msg, errno, errstr) < 0) flux_log_error (h, "error responding to kvs-checkpoint.get request"); free (data); + json_decref (o); } /* Handle a kvs-checkpoint.put request from the rank 0 kvs module. @@ -187,25 +200,32 @@ void checkpoint_put_cb (flux_t *h, { struct content_files *ctx = arg; const char *key; - const char *value; + json_t *o; + char *value = NULL; const char *errstr = NULL; if (flux_request_unpack (msg, NULL, - "{s:s s:s}", + "{s:s s:o}", "key", &key, "value", - &value) < 0) + &o) < 0) + goto error; + if (!(value = json_dumps (o, JSON_COMPACT))) { + errno = EINVAL; goto error; + } if (filedb_put (ctx->dbpath, key, value, strlen (value), &errstr) < 0) goto error; if (flux_respond (h, msg, NULL) < 0) flux_log_error (h, "error responding to kvs-checkpoint.put request"); + free (value); return; error: if (flux_respond_error (h, msg, errno, errstr) < 0) flux_log_error (h, "error responding to kvs-checkpoint.put request"); + free (value); } /* Destroy module context. diff --git a/src/modules/content-s3/content-s3.c b/src/modules/content-s3/content-s3.c index c63bd7a92da4..763ae33bad5f 100644 --- a/src/modules/content-s3/content-s3.c +++ b/src/modules/content-s3/content-s3.c @@ -326,8 +326,8 @@ void checkpoint_get_cb (flux_t *h, flux_msg_handler_t *mh, const flux_msg_t *msg struct content_s3 *ctx = arg; const char *key; void *data = NULL; - char *dup = NULL; size_t size; + json_t *o = NULL; if (flux_request_unpack (msg, NULL, "{s:s}", "key", &key) < 0) goto error; @@ -335,26 +335,33 @@ void checkpoint_get_cb (flux_t *h, flux_msg_handler_t *mh, const flux_msg_t *msg if (s3_get (ctx->cfg, key, &data, &size, &errstr) < 0) goto error; - if (!(dup = strndup (data, size))) - goto error; + if (size > 0) { + if (!(o = json_loadb (data, size, 0, NULL))) { + /* recovery from version 0 checkpoint blobref not supported */ + errno = EINVAL; + goto error; + } + } + else + o = json_null (); if (flux_respond_pack (h, msg, - "{s:s}", + "{s:O}", "value", - size > 0 ? dup : "") < 0) { + o) < 0) { errno = EIO; flux_log_error (h, "error responding to kvs-checkpoint.get request (pack)"); } free (data); - free (dup); + json_decref (o); return; error: if (flux_respond_error (h, msg, errno, errstr) < 0) flux_log_error (h, "error responding to kvs-checkpoint.get request"); free (data); - free (dup); + json_decref (o); } /* Handle a kvs-checkpoint.put request from the rank 0 kvs module. @@ -364,26 +371,33 @@ void checkpoint_put_cb (flux_t *h, flux_msg_handler_t *mh, const flux_msg_t *msg { struct content_s3 *ctx = arg; const char *key; - const char *value; + json_t *o; + char *value = NULL; const char *errstr = NULL; if (flux_request_unpack (msg, NULL, - "{s:s s:s}", + "{s:s s:o}", "key", &key, "value", - &value) < 0) + &o) < 0) goto error; + if (!(value = json_dumps (o, JSON_COMPACT))) { + errno = EINVAL; + goto error; + } if (s3_put (ctx->cfg, key, value, strlen (value), &errstr) < 0) goto error; if (flux_respond (h, msg, NULL) < 0) flux_log_error (h, "error responding to kvs-checkpoint.put request (pack)"); + free (value); return; error: if (flux_respond_error (h, msg, errno, errstr) < 0) flux_log_error (h, "error responding to kvs-checkpoint.put request"); + free (value); } /* Table of message handler callbacks registered below. diff --git a/src/modules/content-sqlite/content-sqlite.c b/src/modules/content-sqlite/content-sqlite.c index 83983bc4c604..3ae4d9717da2 100644 --- a/src/modules/content-sqlite/content-sqlite.c +++ b/src/modules/content-sqlite/content-sqlite.c @@ -16,6 +16,7 @@ #include #include #include +#include #include "src/common/libutil/blobref.h" #include "src/common/libutil/log.h" @@ -337,6 +338,8 @@ void checkpoint_get_cb (flux_t *h, { struct content_sqlite *ctx = arg; const char *key; + char *s; + json_t *o = NULL; if (flux_request_unpack (msg, NULL, "{s:s}", "key", &key) < 0) goto error; @@ -353,18 +356,33 @@ void checkpoint_get_cb (flux_t *h, errno = ENOENT; goto error; } + s = (char *)sqlite3_column_text (ctx->checkpt_get_stmt, 0); + if (!(o = json_loads (s, 0, NULL))) { + /* version 0 checkpoint is a blobref string, used before v0.36 + * reply with updated object representation. + */ + if (!(o = json_pack ("{s:i s:s s:f}", + "version", 0, + "rootref", s, + "timestamp", 0.))) { + errno = EINVAL; + goto error; + } + } if (flux_respond_pack (h, msg, - "{s:s}", + "{s:O}", "value", - sqlite3_column_text (ctx->checkpt_get_stmt, 0)) < 0) + o) < 0) flux_log_error (h, "flux_respond_pack"); (void )sqlite3_reset (ctx->checkpt_get_stmt); + json_decref (o); return; error: if (flux_respond_error (h, msg, errno, NULL) < 0) flux_log_error (h, "flux_respond_error"); (void )sqlite3_reset (ctx->checkpt_get_stmt); + json_decref (o); } void checkpoint_put_cb (flux_t *h, @@ -374,16 +392,21 @@ void checkpoint_put_cb (flux_t *h, { struct content_sqlite *ctx = arg; const char *key; - const char *value; + json_t *o; + char *value = NULL; if (flux_request_unpack (msg, NULL, - "{s:s s:s}", + "{s:s s:o}", "key", &key, "value", - &value) < 0) + &o) < 0) goto error; + if (!(value = json_dumps (o, JSON_COMPACT))) { + errno = EINVAL; + goto error; + } if (sqlite3_bind_text (ctx->checkpt_put_stmt, 1, (char *)key, @@ -395,7 +418,7 @@ void checkpoint_put_cb (flux_t *h, } if (sqlite3_bind_text (ctx->checkpt_put_stmt, 2, - (char *)value, + value, strlen (value), SQLITE_STATIC) != SQLITE_OK) { log_sqlite_error (ctx, "checkpt_put: binding value"); @@ -411,11 +434,13 @@ void checkpoint_put_cb (flux_t *h, if (flux_respond (h, msg, NULL) < 0) flux_log_error (h, "flux_respond"); (void )sqlite3_reset (ctx->checkpt_put_stmt); + free (value); return; error: if (flux_respond_error (h, msg, errno, NULL) < 0) flux_log_error (h, "flux_respond_error"); (void )sqlite3_reset (ctx->checkpt_put_stmt); + free (value); } static void content_sqlite_closedb (struct content_sqlite *ctx) diff --git a/src/modules/kvs/kvs.c b/src/modules/kvs/kvs.c index 95fe4feb354c..ab985ec3a732 100644 --- a/src/modules/kvs/kvs.c +++ b/src/modules/kvs/kvs.c @@ -2707,62 +2707,18 @@ static void process_args (struct kvs_ctx *ctx, int ac, char **av) } } -static int checkpoint_get_version0 (const char *value, char *buf, size_t len) -{ - /* if value is a blobref, its version 0 checkpoint */ - if (blobref_validate (value) == 0) { - if (strlen (value) >= len) - return -1; - strcpy (buf, value); - return 0; - } - return -1; -} - -static int checkpoint_get_version1 (const char *value, - char *buf, size_t len, - double *timestamp) -{ - const char *rootref = NULL; - json_t *o = NULL; - int version; - int rv = -1; - - if (!(o = json_loads (value, 0, NULL))) - goto error; - - if (json_unpack (o, "{s:i}", "version", &version) < 0) - goto error; - /* only can handle version 1 right now */ - if (version == 1) { - if (json_unpack (o, "{s:s s:f}", - "rootref", &rootref, - "timestamp", timestamp) < 0) - goto error; - if (strlen (rootref) >= len) - goto error; - strcpy (buf, rootref); - } - else - goto error; - - rv = 0; -error: - json_decref (o); - return rv; -} - -/* Synchronously get string value by key from checkpoint service. - * Copy value to buf with '\0' termination. +/* Synchronously get checkpoint databy key from checkpoint service. + * Copy rootref buf with '\0' termination. * Return 0 on success, -1 on failure, */ static int checkpoint_get (flux_t *h, const char *key, char *buf, size_t len) { flux_future_t *f = NULL; - const char *value = NULL; - double timestamp = 0.; + json_t *o; char datestr[128]; - int rv = -1; + int version; + const char *rootref = NULL; + double timestamp = 0.; if (!(f = flux_rpc_pack (h, "kvs-checkpoint.get", @@ -2772,14 +2728,28 @@ static int checkpoint_get (flux_t *h, const char *key, char *buf, size_t len) "key", key))) return -1; - if (flux_rpc_get_unpack (f, "{s:s}", "value", &value) < 0) + if (flux_rpc_get_unpack (f, "{s:o}", "value", &o) < 0) goto error; - if (checkpoint_get_version1 (value, buf, len, ×tamp) < 0 - && checkpoint_get_version0 (value, buf, len) < 0) { - errno = EINVAL; - goto error; + if (json_unpack (o, "{s:i}", "version", &version) < 0) + goto error_einval; + + if (version == 0) { + if (json_unpack (o, "{s:s}", "rootref", &rootref) < 0) + goto error_einval; } + else if (version == 1) { + if (json_unpack (o, "{s:s s:f}", + "rootref", &rootref, + "timestamp", ×tamp) < 0) + goto error_einval; + } + else + goto error_einval; + + if (strlen (rootref) >= len) + goto error_einval; + strcpy (buf, rootref); if (timestamp > 0.) { time_t sec = timestamp; @@ -2792,10 +2762,14 @@ static int checkpoint_get (flux_t *h, const char *key, char *buf, size_t len) flux_log (h, LOG_INFO, "restored kvs-primary from checkpoint on %s", datestr); - rv = 0; + flux_future_destroy (f); + return 0; + +error_einval: + errno = EINVAL; error: flux_future_destroy (f); - return rv; + return -1; } /* Synchronously store key-value pair to checkpoint service. @@ -2806,7 +2780,6 @@ static int checkpoint_put (flux_t *h, const char *key, const char *rootref) flux_future_t *f = NULL; double timestamp; json_t *o = NULL; - char *value = NULL; int save_errno, rv = -1; timestamp = flux_reactor_now (flux_get_reactor (h)); @@ -2823,20 +2796,16 @@ static int checkpoint_put (flux_t *h, const char *key, const char *rootref) errno = ENOMEM; goto error; } - if (!(value = json_dumps (o, JSON_COMPACT))) { - errno = ENOMEM; - goto error; - } if (!(f = flux_rpc_pack (h, "kvs-checkpoint.put", 0, 0, - "{s:s s:s}", + "{s:s s:O}", "key", key, "value", - value))) + o))) goto error; if (flux_rpc_get (f, NULL) < 0) goto error; @@ -2845,7 +2814,6 @@ static int checkpoint_put (flux_t *h, const char *key, const char *rootref) save_errno = errno; flux_future_destroy (f); json_decref (o); - free (value); errno = save_errno; return rv; } diff --git a/t/t0012-content-sqlite.t b/t/t0012-content-sqlite.t index 682ffc6cd45d..5159bcefe357 100755 --- a/t/t0012-content-sqlite.t +++ b/t/t0012-content-sqlite.t @@ -119,30 +119,37 @@ test_expect_success 'fill the cache with more data for later purging' ' ' kvs_checkpoint_put() { - jq -j -c -n "{key:\"$1\",value:\"$2\"}" | $RPC kvs-checkpoint.put + o="{key:\"$1\",value:{version:1,rootref:\"$2\",timestamp:2.2}}" + jq -j -c -n ${o} | $RPC kvs-checkpoint.put } kvs_checkpoint_get() { jq -j -c -n "{key:\"$1\"}" | $RPC kvs-checkpoint.get } -test_expect_success HAVE_JQ 'kvs-checkpoint.put foo=bar' ' +test_expect_success HAVE_JQ 'kvs-checkpoint.put foo w/ rootref bar' ' kvs_checkpoint_put foo bar ' -test_expect_success HAVE_JQ 'kvs-checkpoint.get foo returned bar' ' - echo bar >value.exp && - kvs_checkpoint_get foo | jq -r .value >value.out && - test_cmp value.exp value.out +test_expect_success HAVE_JQ 'kvs-checkpoint.get foo returned rootref bar' ' + echo bar >rootref.exp && + kvs_checkpoint_get foo | jq -r .value | jq -r .rootref >rootref.out && + test_cmp rootref.exp rootref.out ' -test_expect_success HAVE_JQ 'kvs-checkpoint.put updates foo=baz' ' +# use grep instead of compare, incase of floating point rounding +test_expect_success HAVE_JQ 'kvs-checkpoint.get foo returned correct timestamp' ' + kvs_checkpoint_get foo | jq -r .value | jq -r .timestamp >timestamp.out && + grep 2.2 timestamp.out +' + +test_expect_success HAVE_JQ 'kvs-checkpoint.put updates foo rooref to baz' ' kvs_checkpoint_put foo baz ' -test_expect_success HAVE_JQ 'kvs-checkpoint.get foo returned baz' ' - echo baz >value2.exp && - kvs_checkpoint_get foo | jq -r .value >value2.out && - test_cmp value2.exp value2.out +test_expect_success HAVE_JQ 'kvs-checkpoint.get foo returned rootref baz' ' + echo baz >rootref2.exp && + kvs_checkpoint_get foo | jq -r .value | jq -r .rootref >rootref2.out && + test_cmp rootref2.exp rootref2.out ' test_expect_success 'flush + reload content-sqlite module on rank 0' ' @@ -150,10 +157,10 @@ test_expect_success 'flush + reload content-sqlite module on rank 0' ' flux module reload content-sqlite ' -test_expect_success HAVE_JQ 'kvs-checkpoint.get foo still returns baz' ' - echo baz >value3.exp && - kvs_checkpoint_get foo | jq -r .value >value3.out && - test_cmp value3.exp value3.out +test_expect_success HAVE_JQ 'kvs-checkpoint.get foo still returns rootref baz' ' + echo baz >rootref3.exp && + kvs_checkpoint_get foo | jq -r .value | jq -r .rootref >rootref3.out && + test_cmp rootref3.exp rootref3.out ' test_expect_success HAVE_JQ 'kvs-checkpoint.get noexist fails with No such...' ' diff --git a/t/t0018-content-files.t b/t/t0018-content-files.t index 39dfe9ac2722..2c6691afba37 100755 --- a/t/t0018-content-files.t +++ b/t/t0018-content-files.t @@ -58,9 +58,10 @@ recheck_cache_blob() { flux content load $(cat blobref.$1) >blob.$1.cachecheck && test_cmp blob.$1 blob.$1.cachecheck } -# Usage: kvs_checkpoint_put key value +# Usage: kvs_checkpoint_put key rootref kvs_checkpoint_put() { - jq -j -c -n "{key:\"$1\",value:\"$2\"}" | $RPC kvs-checkpoint.put + o="{key:\"$1\",value:{version:1,rootref:\"$2\",timestamp:2.2}}" + jq -j -c -n ${o} | $RPC kvs-checkpoint.put } # Usage: kvs_checkpoint_get key >value kvs_checkpoint_get() { @@ -103,24 +104,30 @@ test_expect_success LONGTEST 'store/load/verify various size large blobs' ' test $err -eq 0 ' -test_expect_success HAVE_JQ 'kvs-checkpoint.put foo=bar' ' - kvs_checkpoint_put foo bar +test_expect_success HAVE_JQ 'kvs-checkpoint.put foo w/ rootref bar' ' + kvs_checkpoint_put foo bar ' -test_expect_success HAVE_JQ 'kvs-checkpoint.get foo returned bar' ' - echo bar >value.exp && - kvs_checkpoint_get foo | jq -r .value >value.out && - test_cmp value.exp value.out +test_expect_success HAVE_JQ 'kvs-checkpoint.get foo returned rootref bar' ' + echo bar >rootref.exp && + kvs_checkpoint_get foo | jq -r .value | jq -r .rootref >rootref.out && + test_cmp rootref.exp rootref.out ' -test_expect_success HAVE_JQ 'kvs-checkpoint.put updates foo=baz' ' +# use grep instead of compare, incase of floating point rounding +test_expect_success HAVE_JQ 'kvs-checkpoint.get foo returned correct timestamp' ' + kvs_checkpoint_get foo | jq -r .value | jq -r .timestamp >timestamp.out && + grep 2.2 timestamp.out +' + +test_expect_success HAVE_JQ 'kvs-checkpoint.put updates foo rooref to baz' ' kvs_checkpoint_put foo baz ' -test_expect_success HAVE_JQ 'kvs-checkpoint.get foo returned baz' ' - echo baz >value2.exp && - kvs_checkpoint_get foo | jq -r .value >value2.out && - test_cmp value2.exp value2.out +test_expect_success HAVE_JQ 'kvs-checkpoint.get foo returned rootref baz' ' + echo baz >rootref2.exp && + kvs_checkpoint_get foo | jq -r .value | jq -r .rootref >rootref2.out && + test_cmp rootref2.exp rootref2.out ' test_expect_success 'reload content-files module' ' @@ -143,9 +150,10 @@ test_expect_success LONGTEST 'reload/verify various size large blobs' ' test $err -eq 0 ' -test_expect_success HAVE_JQ 'kvs-checkpoint.get foo returns same value' ' - kvs_checkpoint_get foo | jq -r .value >value3.out && - test_cmp value2.exp value3.out +test_expect_success HAVE_JQ 'kvs-checkpoint.get foo still returns rootref baz' ' + echo baz >rootref3.exp && + kvs_checkpoint_get foo | jq -r .value | jq -r .rootref >rootref3.out && + test_cmp rootref3.exp rootref3.out ' test_expect_success 'load with invalid blobref fails' ' diff --git a/t/t0024-content-s3.t b/t/t0024-content-s3.t index 0bc2bd29993e..6b806071257b 100755 --- a/t/t0024-content-s3.t +++ b/t/t0024-content-s3.t @@ -63,9 +63,10 @@ recheck_cache_blob() { flux content load $(cat blobref.$1) >blob.$1.cachecheck && test_cmp blob.$1 blob.$1.cachecheck } -# Usage: kvs_checkpoint_put key value +# Usage: kvs_checkpoint_put key rootref kvs_checkpoint_put() { - jq -j -c -n "{key:\"$1\",value:\"$2\"}" | $RPC kvs-checkpoint.put + o="{key:\"$1\",value:{version:1,rootref:\"$2\",timestamp:2.2}}" + jq -j -c -n ${o} | $RPC kvs-checkpoint.put } # Usage: kvs_checkpoint_get key >value kvs_checkpoint_get() { @@ -118,24 +119,31 @@ test_expect_success LONGTEST 'store/load/verify various size large blobs' ' test $err -eq 0 ' -test_expect_success HAVE_JQ 'kvs-checkpoint.put foo=bar' ' - kvs_checkpoint_put foo bar + +test_expect_success HAVE_JQ 'kvs-checkpoint.put foo w/ rootref bar' ' + kvs_checkpoint_put foo bar +' + +test_expect_success HAVE_JQ 'kvs-checkpoint.get foo returned rootref bar' ' + echo bar >rootref.exp && + kvs_checkpoint_get foo | jq -r .value | jq -r .rootref >rootref.out && + test_cmp rootref.exp rootref.out ' -test_expect_success HAVE_JQ 'kvs-checkpoint.get foo returned bar' ' - echo bar >value.exp && - kvs_checkpoint_get foo | jq -r .value >value.out && - test_cmp value.exp value.out +# use grep instead of compare, incase of floating point rounding +test_expect_success HAVE_JQ 'kvs-checkpoint.get foo returned correct timestamp' ' + kvs_checkpoint_get foo | jq -r .value | jq -r .timestamp >timestamp.out && + grep 2.2 timestamp.out ' -test_expect_success HAVE_JQ 'kvs-checkpoint.put updates foo=baz' ' +test_expect_success HAVE_JQ 'kvs-checkpoint.put updates foo rooref to baz' ' kvs_checkpoint_put foo baz ' -test_expect_success HAVE_JQ 'kvs-checkpoint.get foo returned baz' ' - echo baz >value2.exp && - kvs_checkpoint_get foo | jq -r .value >value2.out && - test_cmp value2.exp value2.out +test_expect_success HAVE_JQ 'kvs-checkpoint.get foo returned rootref baz' ' + echo baz >rootref2.exp && + kvs_checkpoint_get foo | jq -r .value | jq -r .rootref >rootref2.out && + test_cmp rootref2.exp rootref2.out ' test_expect_success 'reload content-s3 module' ' @@ -158,9 +166,10 @@ test_expect_success LONGTEST 'reload/verify various size large blobs' ' test $err -eq 0 ' -test_expect_success HAVE_JQ 'kvs-checkpoint.get foo returns same value' ' - kvs_checkpoint_get foo | jq -r .value >value3.out && - test_cmp value2.exp value3.out +test_expect_success HAVE_JQ 'kvs-checkpoint.get foo still returns rootref baz' ' + echo baz >rootref3.exp && + kvs_checkpoint_get foo | jq -r .value | jq -r .rootref >rootref3.out && + test_cmp rootref3.exp rootref3.out ' test_expect_success 'config: reload config does not take effect immediately' ' @@ -252,4 +261,52 @@ test_expect_success 'remove content-s3 module' ' flux module remove content-s3 ' +## +# Tests of kvs checkpointing +## + +test_expect_success 'generate rc1/rc3 for content-s3 backing' ' + cat >rc1-content-s3 <rc3-content-s3 <gets3.out +' + +test_expect_success 'content from previous instance survived (s3)' ' + echo 43 >gets3.exp && + test_cmp gets3.exp gets3.out +' + +test_expect_success 're-run instance, verify checkpoint date saved (s3)' ' + flux start -o,--setattr=broker.rc1_path=$(pwd)/rc1-content-s3 \ + -o,--setattr=broker.rc3_path=$(pwd)/rc3-content-s3 \ + flux dmesg >dmesgs3.out +' + +# just check for todays date, not time for obvious reasons +test_expect_success 'verify date in flux logs (s3)' ' + today=`date --iso-8601` && + grep checkpoint dmesgs3.out | grep ${today} +' + test_done diff --git a/t/t2010-kvs-snapshot-restore.t b/t/t2010-kvs-snapshot-restore.t index e95fcbb780cb..b635274086c8 100755 --- a/t/t2010-kvs-snapshot-restore.t +++ b/t/t2010-kvs-snapshot-restore.t @@ -11,7 +11,11 @@ test_under_flux 1 CHANGECHECKPOINT=${FLUX_SOURCE_DIR}/t/kvs/change-checkpoint.py -test_expect_success 'run instance with content.backing-path set' ' +# +# test content-sqlite backing +# + +test_expect_success 'run instance with content.backing-path set (sqlite)' ' flux start -o,--setattr=content.backing-path=$(pwd)/content.sqlite \ flux kvs put testkey=42 ' @@ -21,44 +25,100 @@ test_expect_success 'content.sqlite file exists after instance exited' ' echo Size in bytes: $(stat --format "%s" content.sqlite) ' -test_expect_success 're-run instance with content.backing-path set' ' +test_expect_success 're-run instance with content.backing-path set (sqlite)' ' flux start -o,--setattr=content.backing-path=$(pwd)/content.sqlite \ - flux kvs get testkey >get.out + flux kvs get testkey >getsqlite.out ' -test_expect_success 'content from previous instance survived' ' - echo 42 >get.exp && - test_cmp get.exp get.out +test_expect_success 'content from previous instance survived (sqlite)' ' + echo 42 >getsqlite.exp && + test_cmp getsqlite.exp getsqlite.out ' -test_expect_success 're-run instance, verify checkpoint date saved' ' +test_expect_success 're-run instance, verify checkpoint date saved (sqlite)' ' flux start -o,--setattr=content.backing-path=$(pwd)/content.sqlite \ - flux dmesg >dmesg1.out + flux dmesg >dmesgsqlite1.out ' # just check for todays date, not time for obvious reasons -test_expect_success 'verify date in flux logs' ' +test_expect_success 'verify date in flux logs (sqlite)' ' today=`date --iso-8601` && - grep checkpoint dmesg1.out | grep ${today} + grep checkpoint dmesgsqlite1.out | grep ${today} ' -test_expect_success 're-run instance, get rootref' ' +test_expect_success 're-run instance, get rootref (sqlite)' ' flux start -o,--setattr=content.backing-path=$(pwd)/content.sqlite \ - flux kvs getroot -b > getroot.out + flux kvs getroot -b > getrootsqlite.out ' -test_expect_success 'write rootref to checkpoint path, emulating original checkpoint' ' - rootref=$(cat getroot.out) && +test_expect_success 'write rootref to checkpoint path, emulating original checkpoint (sqlite)' ' + rootref=$(cat getrootsqlite.out) && ${CHANGECHECKPOINT} $(pwd)/content.sqlite "kvs-primary" ${rootref} ' -test_expect_success 're-run instance, verify checkpoint correctly loaded' ' +test_expect_success 're-run instance, verify checkpoint correctly loaded (sqlite)' ' flux start -o,--setattr=content.backing-path=$(pwd)/content.sqlite \ - flux dmesg >dmesg2.out + flux dmesg >dmesgsqlite2.out +' + +test_expect_success 'verify checkpoint loaded with no date (sqlite)' ' + grep checkpoint dmesgsqlite2.out | grep "N\/A" +' + +# +# test content-files backing +# + +test_expect_success 'generate rc1/rc3 for content-files backing' ' + cat >rc1-content-files <rc3-content-files <getfiles.out +' + +test_expect_success 'content from previous instance survived (files)' ' + echo 43 >getfiles.exp && + test_cmp getfiles.exp getfiles.out +' + +test_expect_success 're-run instance, verify checkpoint date saved (files)' ' + flux start -o,--setattr=content.backing-path=$(pwd)/content.files \ + -o,--setattr=broker.rc1_path=$(pwd)/rc1-content-files \ + -o,--setattr=broker.rc3_path=$(pwd)/rc3-content-files \ + flux dmesg >dmesgfiles.out +' + +# just check for todays date, not time for obvious reasons +test_expect_success 'verify date in flux logs (files)' ' + today=`date --iso-8601` && + grep checkpoint dmesgfiles.out | grep ${today} ' test_done