testsuite: cover offline KVS garbage collection

Problem: there is no test coverage for offline KVS garbage
collection.

Add a sharness script that exercises this functionality.

t/Makefile.am

@@ -187,6 +187,7 @@ TESTSCRIPTS = \
 	t2807-dump-cmd.t \
 	t2808-shutdown-cmd.t \
 	t2809-job-purge.t \
+	t2810-kvs-garbage-collect.t \
 	t2900-job-timelimits.t \
 	t3000-mpi-basic.t \
 	t3001-mpi-personalities.t \

t/t2810-kvs-garbage-collect.t

@@ -0,0 +1,92 @@
+#!/bin/sh
+
+test_description='Test offline KVS garbage collection'
+
+. $(dirname $0)/sharness.sh
+
+test_expect_success 'create test script' '
+	cat >runjobs.sh <<-EOT &&
+	#!/bin/bash -e
+	trap "" SIGHUP
+	flux mini submit --cc=1-10 /bin/true >/dev/null
+	flux queue drain
+	backingmod=\$(flux getattr content.backing-module)
+	flux module stats --type int --parse object_count \$backingmod
+	EOT
+	chmod +x runjobs.sh
+'
+test_expect_success 'run instance that leaves an auto dump' '
+	mkdir -p state &&
+	flux start -o,-Sstatedir=state \
+	    -o,-Scontent.dump=auto \
+	    -o,-Slog-filename=dmesg.log \
+	    ./runjobs.sh >object_count
+'
+test_expect_success 'broker logs report dump activity' '
+	grep "dumping content to" dmesg.log
+'
+test_expect_success 'dump exists and RESTORE symlink is valid' '
+	test -h state/RESTORE &&
+	readlink -f state/RESTORE >archive &&
+	test -f $(cat archive)
+'
+test_expect_success 'restart instance with auto restore' '
+	flux start -o,-Sstatedir=state \
+	    -o,-Scontent.restore=auto \
+	    -o,-Slog-filename=dmesg2.log \
+	    flux module stats \
+	        --type int --parse object_count content-sqlite >object_count2
+'
+test_expect_success 'broker logs report restore activity' '
+	grep "restoring content from" dmesg2.log
+'
+test_expect_success 'number of stored objects was reduced by GC' '
+	before=$(cat object_count) &&
+	after=$(cat object_count2) &&
+	test $before -gt $after
+'
+test_expect_success 'RESTORE symlink is gone' '
+	test_must_fail test -h state/RESTORE
+'
+test_expect_success 'archive file remains' '
+	test -f $(cat archive)
+'
+
+#
+# Now repeat the above test with
+# - content-files backend
+# - no statedir
+# - explicitly named dump file (not auto)
+#
+test_expect_success 'run instance that leaves a named dump' '
+	flux start -o,-Slog-filename=dmesg3.log \
+	    -o,-Scontent.dump=foo.tgz \
+	    -o,-Scontent.backing-module=content-files \
+	    ./runjobs.sh >object_count3
+'
+test_expect_success 'broker logs report dump activity' '
+	grep "dumping content to" dmesg3.log
+'
+test_expect_success 'dump exists in current directory' '
+	test -f foo.tgz
+'
+test_expect_success 'no RESTORE link was created because path is explicit' '
+	test_must_fail test -h RESTORE
+'
+test_expect_success 'restart instance and restore' '
+	flux start -o,-Slog-filename=dmesg4.log \
+	    -o,-Scontent.restore=foo.tgz \
+	    -o,-Scontent.backing-module=content-files \
+	    flux module stats \
+	        --type int --parse object_count content-files >object_count4
+'
+test_expect_success 'broker logs report restore activity' '
+	grep "restoring content from" dmesg4.log
+'
+test_expect_success 'number of stored objects was reduced by GC' '
+	before=$(cat object_count3) &&
+	after=$(cat object_count4) &&
+	test $before -gt $after
+'
+
+test_done