Don't Mesh Around public code release

.gitignore

@@ -0,0 +1,22 @@
+# Binary and object files
+bin
+obj
+out
+*.o
+
+# Python
+__pycache__
+venv
+*.pyc
+
+# Experiment outputs
+data
+models
+plot
+*.pdf
+*.out
+*.log
+
+# 03-side-channel libgcrypt repos
+03-side-channel/victim/libgcrypt-1.5.2
+03-side-channel/victim/libgcrypt-1.6.3

01-noc-reverse-engineering/Makefile

@@ -0,0 +1,46 @@
+CC:= gcc
+HOSTNAME := $(shell hostname|awk '{print toupper($$0)'})
+CFLAGS:= -O3 -D_POSIX_SOURCE -D_GNU_SOURCE -m64 -D$(HOSTNAME)
+CFLAGSO1:= -O1 -D_POSIX_SOURCE -D_GNU_SOURCE -m64 -D$(HOSTNAME)
+LIBS:= -lpthread -lrt
+
+all: obj bin out plot transmitter transmitter-no-loads receiver setup-sem cleanup-sem
+
+transmitter: obj/transmitter.o ../util/util.o ../util/pmon_utils.o ../util/machine_const.o ../util/skx_hash_utils.o ../util/pfn_util.o
+	$(CC) -o bin/$@ $^ $(LIBS)
+
+transmitter-no-loads: obj/transmitter-no-loads.o ../util/util.o ../util/pmon_utils.o ../util/machine_const.o ../util/skx_hash_utils.o ../util/pfn_util.o
+	$(CC) -o bin/$@ $^ $(LIBS)
+
+receiver: obj/receiver.o ../util/util.o ../util/pmon_utils.o ../util/machine_const.o ../util/skx_hash_utils.o ../util/pfn_util.o
+	$(CC) -o bin/$@ $^ $(LIBS)
+
+setup-sem: obj/setup-sem.o
+	$(CC) -o bin/$@ $^ $(LIBS)
+
+cleanup-sem: obj/cleanup-sem.o
+	$(CC) -o bin/$@ $^ $(LIBS)
+
+# pmon_utils needs to be compiled with -O1 for the get_corresponding_cha function to work
+../util/pmon_utils.o: ../util/pmon_utils.c
+	$(CC) -c $(CFLAGSO1) -o $@  $^
+
+obj/%.o: %.c
+	$(CC) -c $(CFLAGS) -o $@ $<
+
+obj:
+	mkdir -p $@
+
+bin:
+	mkdir -p $@
+
+out:
+	mkdir -p $@
+
+plot:
+	mkdir -p $@
+
+clean:
+	rm -rf bin obj
+
+.PHONY: all clean

01-noc-reverse-engineering/README.md

@@ -0,0 +1,38 @@
+# Mesh Interconnect Reverse Engineering
+
+These scripts facilitate running the transmitter and receiver in different configurations.
+We use experiments to confirm our lane scheduling policy and priority arbitration policy.
+These scripts generate data for Figure 6.
+
+## Prerequisites
+
+- Make all artifacts with `make`
+- Ensure that the Python virtual environment has been installed in the parent directory
+- Run `./setup.sh` to prepare the machine
+
+## Running the Case Studies
+
+**Expected Runtime: 3 min**
+
+Running `sudo ../venv/bin/python placement-experiments.py` will produce data that aligns with Figure 6.
+Run `./cleanup.sh` to restore the machine settings.
+
+## Troubleshooting
+
+The following are some commonly-observed issues with this script.
+
+- **I see many large negative latency difference values.**
+
+    There are probably too many L1/L2 cache hits that are not being filtered out correctly.
+    You can examine the output latency values in `data/{placement-config}/tx_on.out` and `data/{placement-config}/tx_off.out`
+    `placement-config` is a 6-number string of the following form: `{tx_core}-{tx_slice_a}-{tx_slice_b}-{monitor_core}-{monitor_ms_slice}-{monitor_ev_slice}`
+
+    Adjust the values in the `filter_trace` function in `placement-experiments.py` to filter out the high and low outliers.
+    On our machine, the expected LLC access latency is around 70 cycles.
+
+- **A few reported values do not match Figure 6.**
+
+    These experiments are sensitive to noise and may require running a few re-runs to collect all the data accurately.
+    Additionally, values with magnitude less than or equal to 0.5 are considered 0 contention difference.
+    Configurations that incur slice port contention (indicated by the hatch-shaded boxes) can have higher variability than other squares.
+    These configurations may occasionally see contention differences slightly below 5 or above 10.

01-noc-reverse-engineering/cleanup-sem.c

@@ -0,0 +1,22 @@
+#include <fcntl.h>           /* For O_* constants */
+#include <sys/stat.h>        /* For mode constants */
+#include <semaphore.h>
+#include <errno.h>
+#include <stdio.h>
+
+int main(int argc, char const *argv[])
+{
+	if (sem_unlink("setup_sem") != 0) {
+		perror("Unlink setup_sem");
+	}
+
+	if (sem_unlink("tx_ready") != 0) {
+		perror("Unlink tx_ready");
+	}
+
+	if (sem_unlink("rx_ready") != 0) {
+		perror("Unlink rx_ready");
+	}
+
+	return 0;
+}

01-noc-reverse-engineering/cleanup.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+# Kill any stray transmitters in case run.sh did not terminate correctly
+sudo killall -9 transmitter
+sudo killall -9 transmitter-no-loads
+
+# Delete semaphores
+sudo bin/cleanup-sem
+
+# Restore the various frequency settings, if they were changed
+echo powersave | sudo tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor 2> /dev/null		# set powersave governor
+echo 0 | sudo tee /sys/devices/system/cpu/intel_pstate/no_turbo		# re-enables turbo boost
+sudo wrmsr 0x620 0xc18		# re-sets the uncore frequency range
+
+# Restore environment after running experiments
+../util/cleanup.sh

01-noc-reverse-engineering/placement-experiments.py

@@ -0,0 +1,128 @@
+import subprocess
+from collections import namedtuple
+
+import numpy as np
+
+Placement = namedtuple('Placement', 'tx_core tx_slice_a tx_slice_b rx_core rx_ms_slice rx_ev_slice')
+
+DIVIDER = '=' * 40
+
+DIE_LAYOUT = [
+    [0, 4, 9, 13, 17, 22],
+    [-1, 5, 10, 14, 18, -1],
+    [1, 6, 11, 15, 19, 23],
+    [2, 7, 12, -1, 20, 24],
+    [3, 8, -1, 16, 21, 25]
+]
+
+
+def print_coord(slice_id):
+    """Return a string that represents slice_id using the notation from the paper."""
+    coord = None
+    for r, row in enumerate(DIE_LAYOUT):
+        if slice_id in row:
+            coord = (row.index(slice_id), r)
+    if coord is None:
+        print(f'Error: could not find Slice ID {slice_id} in DIE_LAYOUT')
+    return f'({coord[1]},{coord[0]})'
+
+
+def get_placement_path(p):
+    """Return the path for the data of the experiment with placment p."""
+    return f'data/{p.tx_core}-{p.tx_slice_a}-{p.tx_slice_b}-{p.rx_core}-{p.rx_ms_slice}-{p.rx_ev_slice}'
+
+
+def test_placement(p):
+    """Run a single test with a specific tx/rx placement.
+
+    p: Placement consisting of tx_core, tx_slice_a, tx_slice_b, rx_core, rx_ms_slice rx_ev_slice
+    Output traces are stored in data/{tx_core}-{tx_slice_a}-{tx_slice_b}-{rx_core}-{rx_ms_slice}-{rx_ev_slice}/
+    The output traces should be named tx_on.log and tx_off.log.
+    """
+    output_path = get_placement_path(p)
+    cmd = f'./run-single.sh {p.tx_core} {p.tx_slice_a} {p.tx_slice_b} {p.rx_core} {p.rx_ms_slice} {p.rx_ev_slice} {output_path}'.split(' ')
+    subprocess.run(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+
+
+def load_trace(filepath):
+    return np.genfromtxt(filepath, delimiter=' ')
+
+
+def filter_trace(trace, percentile=10):
+    upper_thresh = 100
+    lower_thresh = 40
+    return trace[np.logical_and(trace > lower_thresh, trace < upper_thresh)]
+
+
+def get_latency_diff(p):
+    """Post-process a set of tx_on/tx_off traces to get the latency difference."""
+    exp_path = get_placement_path(p)
+    tx_on_trace = load_trace(f'{exp_path}/tx_on.log')
+    tx_off_trace = load_trace(f'{exp_path}/tx_off.log')
+    tx_on_mean = np.mean(filter_trace(tx_on_trace[:, 1]))
+    tx_off_mean = np.mean(filter_trace(tx_off_trace[:, 1]))
+    return round(tx_on_mean - tx_off_mean, 1)
+
+
+def lane_scheduling_case_study():
+    """Reproduce Figure 6a from the paper.
+
+    This case study demonstrates the lane scheduling policy.
+    The receiver monitors Core(0,2) -> Slice(0,3).
+    The transmitter is varied across all possible positions within the row.
+    """
+    print(f'{DIVIDER}\nLane Scheduling Policy Case Study')
+    print(f'Receiver: {print_coord(9)}->{print_coord(13)}')
+    print('Transmitter\tLatency Difference')
+    row_0 = [0, 4, 9, 13, 17, 22]
+    rx_core = 9
+    rx_ms_slice = 13
+    rx_ev_slice = rx_core  # use a local EV slice
+    for tx_core in row_0:
+        if tx_core == rx_core:
+            continue
+        tx_slice_b = tx_core  # use a local EV slice
+        for tx_slice_a in row_0:
+            p = Placement(tx_core, tx_slice_a, tx_slice_b, rx_core, rx_ms_slice, rx_ev_slice)
+            test_placement(p)
+            diff = get_latency_diff(p)
+            print(f'{print_coord(p.tx_core)}->{print_coord(p.tx_slice_a)}:\t{diff:4.1f}')
+    print(f'{DIVIDER}\n')
+
+
+def priority_arbitration_case_study():
+    """Reproduce Figure 6b from the paper.
+
+    This case study demonstrates the priority arbitration policy.
+    The receiver monitors Core(0,0) -> Slice(0,5).
+    The transmitter is varied across all possible positions within the row.
+    """
+
+    print(f'{DIVIDER}\nLane Scheduling Policy Case Study')
+    print(f'Receiver: {print_coord(0)}->{print_coord(22)}')
+    print('Transmitter\tLatency Difference')
+    row_0 = [0, 4, 9, 13, 17, 22]
+    rx_core = 0
+    rx_ms_slice = 22
+    rx_ev_slice = rx_core  # use a local EV slice
+    for tx_core in row_0:
+        # Do not pin the tx and rx to the same core
+        if tx_core == rx_core:
+            continue
+
+        tx_slice_b = tx_core  # Use the local slice
+        for tx_slice_a in row_0:
+            p = Placement(tx_core, tx_slice_a, tx_slice_b, rx_core, rx_ms_slice, rx_ev_slice)
+            test_placement(p)
+            diff = get_latency_diff(p)
+            print(f'{print_coord(p.tx_core)}->{print_coord(p.tx_slice_a)}:\t{diff:4.1f}')
+    print(f'{DIVIDER}\n')
+
+
+def main():
+    lane_scheduling_case_study()
+    priority_arbitration_case_study()
+
+
+if __name__ == '__main__':
+    main()