Skip to content

Commit

Permalink
cmd/kvs: Add --file option to kvs put & get
Browse files Browse the repository at this point in the history 
Allow user to specify a file with put & get, so that raw data can be
written to / read from the KVS.
  • Loading branch information
@chu11
chu11 committed Aug 16, 2017
1 parent 0cb8154 commit c5b36e5
Show file tree
Hide file tree
Showing 3 changed files with 160 additions and 31 deletions.
14 changes: 9 additions & 5 deletions doc/man1/flux-kvs.adoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,13 +42,17 @@ arguments are described below.

COMMANDS
--------
*get* 'key' ['key...']::
Retrieve the value stored under 'key'. If nothing has been stored under
'key', display an error message.
*get* [-f file] 'key' ['key...']::
Retrieve the value stored under 'key'. If nothing has been stored
under 'key', display an error message. If user specifies a file via
'-f', output raw data to the file instead. If file is equal to '-',
data will be outupt to stdout.

*put* 'key=value' ['key=value...']::
*put* [-f file] 'key=value' ['key=value...']::
Store 'value' under 'key' and commit it. If it already has a value,
overwrite it.
overwrite it. Instead of specifying a value, user may specify a file
via '-f' to write raw data to KVS. If file is equal to '-', data will
be read from stdin.

*dir* [-R] [-d] ['key']::
Display all keys and their values under the directory 'key'.
Expand Down
128 changes: 102 additions & 26 deletions src/cmd/flux-kvs.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,20 @@ int cmd_dir (optparse_t *p, int argc, char **argv);

static void dump_kvs_dir (kvsdir_t *dir, bool Ropt, bool dopt);

static struct optparse_option get_opts[] = {
{ .name = "file", .key = 'f', .has_arg = 1,
.usage = "Specify file to write value to, or '-' for stdout"
},
OPTPARSE_TABLE_END
};

static struct optparse_option put_opts[] = {
{ .name = "file", .key = 'f', .has_arg = 1,
.usage = "Specify file which contains key value, or '-' for stdin"
},
OPTPARSE_TABLE_END
};

static struct optparse_option dir_opts[] = {
{ .name = "recursive", .key = 'R', .has_arg = 0,
.usage = "Recursively display keys under subdirectories",
Expand Down Expand Up @@ -96,18 +110,18 @@ static struct optparse_option unlink_opts[] = {

static struct optparse_subcommand subcommands[] = {
{ "get",
"key [key...]",
"[-f file] key [key...]",
"Get value stored under key",
cmd_get,
0,
NULL
get_opts
},
{ "put",
"key=value [key=value...]",
"[-f file] key=value [key=value...]",
"Store value under key",
cmd_put,
0,
NULL
put_opts
},
{ "dir",
"[-R] [-d] [key]",
Expand Down Expand Up @@ -310,57 +324,119 @@ int cmd_get (optparse_t *p, int argc, char **argv)
flux_t *h = (flux_t *)optparse_get_data (p, "flux_handle");
const char *key, *json_str;
flux_future_t *f;
int optindex, i;
int fd, len, optindex, i;
const char *file = NULL;
uint8_t *buf = NULL;

optindex = optparse_option_index (p);
if ((optindex - argc) == 0) {
optparse_print_usage (p);
exit (1);
}
for (i = optindex; i < argc; i++) {
key = argv[i];
if (!(f = flux_kvs_lookup (h, 0, key))

file = optparse_get_str (p, "file", NULL);
if (file) {
if (strcmp (file, "-")) {
if ((fd = creat (file, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH)) < 0)
log_err_exit ("%s", file);
}
else
fd = STDOUT_FILENO;

for (i = optindex; i < argc; i++) {
key = argv[i];
if (!(f = flux_kvs_lookup (h, 0, key))
|| flux_kvs_lookup_get_raw (f, (void **)&buf, &len) < 0)
log_err_exit ("%s", key);
if (write_all (fd, buf, len) < 0)
log_err_exit ("%s", file);
flux_future_destroy (f);
}

if (strcmp (file, "-"))
close (fd);
} else {
for (i = optindex; i < argc; i++) {
key = argv[i];
if (!(f = flux_kvs_lookup (h, 0, key))
|| flux_kvs_lookup_get (f, &json_str) < 0)
log_err_exit ("%s", key);
output_key_json_str (NULL, json_str, key);
flux_future_destroy (f);
log_err_exit ("%s", key);
output_key_json_str (NULL, json_str, key);
flux_future_destroy (f);
}
}
return (0);
}

int cmd_put (optparse_t *p, int argc, char **argv)
{
flux_t *h = (flux_t *)optparse_get_data (p, "flux_handle");
int optindex, i;
int fd, len, optindex, i;
flux_future_t *f;
flux_kvs_txn_t *txn;
uint8_t *buf = NULL;
const char *file;

optindex = optparse_option_index (p);
if ((optindex - argc) == 0) {
optparse_print_usage (p);
exit (1);
}
if (!(txn = flux_kvs_txn_create ()))
log_err_exit ("flux_kvs_txn_create");
for (i = optindex; i < argc; i++) {
char *key = xstrdup (argv[i]);
char *val = strchr (key, '=');
if (!val)
log_msg_exit ("put: you must specify a value as key=value");
*val++ = '\0';

if (flux_kvs_txn_put (txn, 0, key, val) < 0) {
if (errno != EINVAL)
log_err_exit ("%s", key);
if (flux_kvs_txn_pack (txn, 0, key, "s", val) < 0)

if ((file = optparse_get_str (p, "file", NULL))) {
if (!strcmp (file, "-")) {
if ((len = read_all (STDIN_FILENO, &buf)) < 0)
log_err_exit ("stdin");
} else {
if ((fd = open (file, O_RDONLY)) < 0)
log_err_exit ("%s", file);
if ((len = read_all (fd, &buf)) < 0)
log_err_exit ("%s", file);
(void)close (fd);
}
if (!len)
log_msg_exit ("put: no data found via --file");

if (!(txn = flux_kvs_txn_create ()))
log_err_exit ("flux_kvs_txn_create");
for (i = optindex; i < argc; i++) {
char *key = xstrdup (argv[i]);
char *val = strchr (key, '=');

if (val)
log_msg_exit ("put: do not specify value if specifying --file");

if (flux_kvs_txn_put_raw (txn, 0, key, buf, len) < 0)
log_err_exit ("%s", key);
free (key);
}
free (key);
}
else {
if (!(txn = flux_kvs_txn_create ()))
log_err_exit ("flux_kvs_txn_create");
for (i = optindex; i < argc; i++) {
char *key = xstrdup (argv[i]);
char *val = strchr (key, '=');

if (!val)
log_msg_exit ("put: you must specify a value as key=value");
*val++ = '\0';

if (flux_kvs_txn_put (txn, 0, key, val) < 0) {
if (errno != EINVAL)
log_err_exit ("%s", key);
if (flux_kvs_txn_pack (txn, 0, key, "s", val) < 0)
log_err_exit ("%s", key);
}
free (key);
}
}

if (!(f = flux_kvs_commit (h, 0, txn)) || flux_future_get (f, NULL) < 0)
log_err_exit ("flux_kvs_commit");
flux_future_destroy (f);
flux_kvs_txn_destroy (txn);
free (buf);
return (0);
}

Expand Down
49 changes: 49 additions & 0 deletions t/t1000-kvs.t
View file
Original file line number Diff line number Diff line change
Expand Up @@ -301,6 +301,55 @@ test_expect_success 'kvs: put key of . fails' '
test_must_fail flux kvs put .=1
'

#
# get & put with --file tests
#

test_expect_success 'kvs: put with --file and key=val' '
flux kvs unlink -Rf $DIR &&
echo "foobar" > test.data &&
test_must_fail flux kvs put -f test.data $DIR.data=value
'

test_expect_success 'kvs: put with --file and bad filename' '
flux kvs unlink -Rf $DIR &&
test_must_fail flux kvs put -f foobar.file $DIR.data
'

test_expect_success 'kvs: put with --file and empty data' '
flux kvs unlink -Rf $DIR &&
rm -f test.data &&
touch test.data &&
test_must_fail flux kvs put -f test.data $DIR.data
'

test_expect_success 'kvs: put and get with --file' '
flux kvs unlink -Rf $DIR &&
rm -f test.data &&
rm -f read.data &&
echo "foobar" > test.data &&
flux kvs put -f test.data $DIR.data &&
flux kvs get -f read.data $DIR.data &&
test_cmp test.data read.data
'

test_expect_success 'kvs: put and get with --file, use stdin/stdout' '
flux kvs unlink -Rf $DIR &&
rm -f read.data &&
echo "foobar" | flux kvs put -f "-" $DIR.data &&
flux kvs get -f "-" $DIR.data > outputX &&
echo "foobar" > expectedX &&
test_cmp outputX expectedX
'

test_expect_success 'kvs: large put and get with --file' '
flux kvs unlink -Rf $DIR &&
dd if=/dev/urandom bs=4096 count=1 >test.data &&
flux kvs put -f test.data $DIR.data &&
flux kvs get -f read.data $DIR.data &&
test_cmp test.data read.data
'

#
# dir corner case tests
#
Expand Down

0 comments on commit c5b36e5

Please sign in to comment.