Add example output files.

README.md

@@ -58,11 +58,19 @@ various NVBench features and usecases:
 
 - [Runtime and compile-time parameter sweeps](examples/axes.cu)
 - [Enums and compile-time-constant-integral parameter axes](examples/enums.cu)
-- [Reporting item/sec and byte/sec throughput statistics](examples/throughput.cu)
+- [Reporting simple item/sec and byte/sec throughput statistics](examples/throughput.cu)
+- [Gathering and reporting CUPTI metrics](examples/auto_throughput.cu)
 - [Skipping benchmark configurations](examples/skip.cu)
 - [Benchmarks that sync CUDA devices: `nvbench::exec_tag::sync`](examples/exec_tag_sync.cu)
 - [Manual timing: `nvbench::exec_tag::timer`](examples/exec_tag_timer.cu)
 
+### Example Output Samples
+
+Sample outputs for `--list`, `--markdown`, `--json`, and `--csv` are provided
+for each example. These are located in the [examples/outputs](examples/outputs/)
+directory. See the associated [README](examples/outputs/README.md) for more
+information.
+
 ### Building Examples
 
 To build the examples:

examples/CMakeLists.txt

@@ -11,6 +11,7 @@ set(example_srcs
 # Metatarget for all examples:
 add_custom_target(nvbench.example.all)
 add_dependencies(nvbench.all nvbench.example.all)
+set(examples)
 
 foreach(example_src IN LISTS example_srcs)
   get_filename_component(example_name "${example_src}" NAME_WLE)
@@ -25,4 +26,7 @@ foreach(example_src IN LISTS example_srcs)
   )
 
   add_dependencies(nvbench.example.all ${example_name})
+  list(APPEND examples ${example_name})
 endforeach()
+
+add_subdirectory(outputs)

examples/outputs/CMakeLists.txt

@@ -0,0 +1,52 @@
+set(NVBench_EXAMPLE_OUTPUT_DEVICES "all" CACHE STRING
+  "--devices argument used when generating example outputs."
+)
+set(NVBench_EXAMPLE_OUTPUT_TIMEOUT "15.0" CACHE STRING
+  "--timeout argument used when generating example outputs."
+)
+mark_as_advanced(
+  NVBench_EXAMPLE_OUTPUT_DEVICES
+  NVBench_EXAMPLE_OUTPUT_TIMEOUT
+)
+
+set_property(GLOBAL PROPERTY JOB_POOLS exclusive=1)
+
+add_custom_target(nvbench.regenerate_example_outputs)
+add_custom_target(nvbench.example.all.list)
+
+set(results_dir "${CMAKE_CURRENT_BINARY_DIR}/results")
+
+foreach (example IN LISTS examples)
+  # Print --list
+  add_custom_target(${example}.list
+    COMMAND "$<TARGET_FILE:${example}>"
+      --md "${results_dir}/${example}.list.md"
+      --list
+    BYPRODUCTS "${results_dir}/${example}.list.md"
+    COMMENT "Generating ${example}.list.md..."
+    VERBATIM
+  )
+  add_dependencies(${example}.list nvbench.example.all)
+  add_dependencies(nvbench.regenerate_example_outputs ${example}.list)
+  add_dependencies(nvbench.example.all.list ${example}.list)
+
+  # Run and output all formats
+  add_custom_target(${example}.outputs
+    COMMAND "$<TARGET_FILE:${example}>"
+      --quiet
+      --devices ${NVBench_EXAMPLE_OUTPUT_DEVICES}
+      --timeout ${NVBench_EXAMPLE_OUTPUT_TIMEOUT}
+      --md "${results_dir}/${example}.md"
+      --csv "${results_dir}/${example}.csv"
+      --json "${results_dir}/${example}.json"
+    BYPRODUCTS
+      "${results_dir}/${example}.md"
+      "${results_dir}/${example}.csv"
+      "${results_dir}/${example}.json"
+    COMMENT "Generating ${example} outputs (json, csv, md)..."
+    JOB_POOL exclusive
+    VERBATIM
+  )
+  add_dependencies(${example}.outputs nvbench.example.all)
+  add_dependencies(nvbench.regenerate_example_outputs ${example}.outputs)
+endforeach()

examples/outputs/README.md

@@ -0,0 +1,32 @@
+# Example Outputs
+
+This directory contains sample outputs for each of the NVBench examples:
+
+- `${example}.list.md`: The output of invoking the example with `--list`.
+- `${example}.md`: The output of invoking the example with `--md stdout`.
+- `${example}.json`: The output of invoking the example with `--json stdout`.
+- `${example}.csv`: The output of invoking the example with `--csv stdout`.
+
+These files are only meant to provide samples of NVBench output formats. The
+results and measurements in these files are not intended to demonstrate peak
+performance of the devices and algorithms measured. The results may
+intentionally include errors (noisy results, timeouts, etc) to show how these
+are reported.
+
+# Generating
+
+The outputs are generated by building the `nvbench.regenerate_example_outputs`
+target. Ideally, CUPTI metrics should be enabled and the GPU clocks locked
+to `base`:
+
+```bash
+# Enable non-root users to collect CUPTI metrics:
+sudo rmmod nvidia
+sudo modprobe nvidia NVreg_RestrictProfilingToAdminUsers=0
+# Enable persistence mode and lock GPU clocks to base:
+sudo <any nvbench example> --pm 1 --lgc base
+
+ninja nvbench.regenerate_example_outputs
+```
+
+The files will be written to `<nvbench_build_dir>/outputs/results/`.

examples/outputs/nvbench.example.auto_throughput.csv

@@ -0,0 +1,9 @@
+Benchmark,Device,Device Name,T,Stride,Skipped,Elements,HBWPeak,LoadEff,StoreEff,L1HitRate,L2HitRate,Samples,CPU Time (sec),Noise,GPU Time (sec),Noise,Elem/s (elem/sec),Batch GPU (sec),Batch
+throughput_bench,0,Quadro GV100,1,1,No,33554432,0.6585438475423353,1,1,0,0.49999904648868265,1078,0.00047011594805194853,0.0014244727929907238,0.0004639086524829754,0.0015463138823336802,72329825754.28767,0.0004611097332451499,1134
+throughput_bench,0,Quadro GV100,1,4,No,33554432,0.6947096407739456,0.25,1,0,0.2000340992245478,452,0.0011125579800884962,0.0006852662282240376,0.0011062891657373588,0.0006886487256687341,30330616116.660107,0.001102912886702233,473
+throughput_bench,0,Quadro GV100,2,1,No,33554432,0.671727291393325,0.5069332663594991,1,0.2682065963745117,0.5437664402009483,548,0.0009194173795620432,0.0006406757855676574,0.0009131468997819581,0.0006974456101610215,36745929935.27345,0.0009104959699842665,576
+throughput_bench,0,Quadro GV100,2,4,No,33554432,0.5825672478320796,0.12669523778540573,1,0.20691384209526908,0.3812073432135215,175,0.0028746560114285722,0.002486847758258855,0.002868418363843645,0.0024933453791821615,11697886341.460135,0.002855257117229959,184
+throughput_bench,1,Quadro GP100,1,1,Yes,,,,,,,,,,,,,,
+throughput_bench,1,Quadro GP100,1,4,Yes,,,,,,,,,,,,,,
+throughput_bench,1,Quadro GP100,2,1,Yes,,,,,,,,,,,,,,
+throughput_bench,1,Quadro GP100,2,4,Yes,,,,,,,,,,,,,,