*Figure 1: Throughput-latency curve and effective throughput of Llama 2 7b using A6000. Runs the client with 60 generation steps and input prompt length of 2600.*
- \ No newline at end of file + diff --git a/benchmarks/inference/mii/plot_config.yaml b/benchmarks/inference/mii/plot_config.yaml new file mode 100644 index 000000000..48a5a3171 --- /dev/null +++ b/benchmarks/inference/mii/plot_config.yaml @@ -0,0 +1,7 @@ +label: "vLLM" +color: "purple" +marker: "o" +linestyle: "--" +polyfit_degree: 0 +x_max : 30 +y_max : 10 diff --git a/benchmarks/inference/mii/plot_effective_throughput.py b/benchmarks/inference/mii/plot_effective_throughput.py deleted file mode 100644 index 350c269c3..000000000 --- a/benchmarks/inference/mii/plot_effective_throughput.py +++ /dev/null @@ -1,176 +0,0 @@ -import argparse -from pathlib import Path -import glob -import matplotlib.pyplot as plt -import numpy as np -import pandas as pd - -from postprocess_results import read_json, get_tokenizer - -RAGGED_BATCH_SIZE = 768 -SLA_PROMPT_TOKENS_PER_SEC = 512 -SLA_GEN_TOKENS_PER_SEC = [1, 2, 3, 4, 6, 8] -EMA_SPAN = 16 - -tp_sizes_all = { - "7b": [1], - "70b": [4, 8] -} - -tp_sizes_test = { - "7b": [1] -} - -prompt_gen_pairs_all = [ - (1200, 60), - (1200, 128), - (2600, 60), - (2600, 128), -] - -prompt_gen_pairs_test = [ - (2600, 60) -] - -def get_args(): - parser = argparse.ArgumentParser() - parser.add_argument("--test", action="store_true") - parser.add_argument("--no_vllm", action="store_true") - parser.add_argument("--log_dir", type=Path, default=".") - parser.add_argument("--out_dir", type=Path, default="charts/goodtput") - args = parser.parse_args() - return args - - -def check_token_latency_step(response_details, token_index): - P50_token_latency = np.percentile([r.token_gen_time[token_index] for r in response_details if len(r.token_gen_time) > token_index], 50) - P90_token_latency = np.percentile([r.token_gen_time[token_index] for r in response_details if len(r.token_gen_time) > token_index], 90) - P99_token_latency = np.percentile([r.token_gen_time[token_index] for r in response_details if len(r.token_gen_time) > token_index], 99) - - return P50_token_latency, P90_token_latency, P99_token_latency - - -def validate_token_cum_latency_SLA(response_detail, sla_token_gen): - cumsum_latencies = np.cumsum(np.array(response_detail.token_gen_time[1:])) - return all([cumsum_latencies[i] <= (1 / sla_token_gen) * (i + 1) for i in range(len(cumsum_latencies))]) - - -def validate_token_ema_latency_SLA(response_detail, sla_token_gen, ema_span): - ema_latency = pd.Series(response_detail.token_gen_time[1:]).ewm(span=ema_span).mean().values.tolist() - return all([t < 1. / sla_token_gen for t in ema_latency]) - - -def validate_prompt_latency_SLA(response_detail, sla_token_gen, f): - tokenizer = get_tokenizer() - prompt_length = len(tokenizer.tokenize(response_detail.prompt)) - prompt_latency_SLA = prompt_length / SLA_PROMPT_TOKENS_PER_SEC - if prompt_latency_SLA < response_detail.token_gen_time[0]: - return False - - if len(response_detail.token_gen_time) == 1: - return True - - return f[0](response_detail, sla_token_gen, *f[1]) - - -def calc_throughput(response_details): - start_time = min([r.start_time for r in response_details]) - end_time = max([r.end_time for r in response_details]) - return len(response_details) / (end_time - start_time) - - -def extract_values(file_pattern, sla_token_gen, validate_func): - files = glob.glob(file_pattern) - print(f"Found {len(files)} files") - goodputs = {} - good_ratios = {} - for f in files: - prof_args, response_details = read_json(f) - client_num = prof_args["client_num"] - num_req_ok = len([r for r in response_details if validate_prompt_latency_SLA(r, sla_token_gen, validate_func)]) - goodputs[client_num] = calc_throughput(response_details) * (num_req_ok / len(response_details)) - good_ratios[client_num] = num_req_ok / len(response_details) - - return goodputs, good_ratios - - -def display_results(model_size, tp, bs, sla_token_gen, prompt, gen, log_dir, out_dir): - if not log_dir.exists(): - print(f"Log directory {log_dir} does not exist") - return - - if not out_dir.exists(): - out_dir.mkdir(parents=True, exist_ok=True) - - print(f"model: {model_size} Prompt: {prompt}, Generation: {gen}, TP: {tp} sla_token_gen: {sla_token_gen}") - - mii_file_pattern = f"{log_dir}/logs.llama2-{model_size}-tp{tp}-b{bs}/llama2-{model_size}-tp{tp}-b{bs}_c*_p{prompt}_g{gen}.json" - if not args.no_vllm: - vllm_file_pattern = f"{log_dir}/logs.vllm-llama2-{model_size}-tp{tp}/vllm-llama2-{model_size}-tp{tp}_c*_p{prompt}_g{gen}.json" - - validate_funcs = [ - (validate_token_cum_latency_SLA, (), "cum"), - (validate_token_ema_latency_SLA, (EMA_SPAN, ), f"ema{EMA_SPAN}"), - ] - - for f in validate_funcs: - - mii_goodputs, mii_good_ratios = extract_values(mii_file_pattern, sla_token_gen, f) - client_num_list = sorted(list(mii_goodputs.keys())) - mii_goodputs_list = [mii_goodputs[client_num] for client_num in client_num_list] - - if not args.no_vllm: - vllm_goodputs, vllm_good_ratios = extract_values(vllm_file_pattern, sla_token_gen, f) - vllm_goodputs_list = [vllm_goodputs[client_num] for client_num in client_num_list] - - # print(f"MII {mii_goodputs_list} ratio={mii_good_ratios}") - # print(f"vLLM {vllm_goodputs_list} ratio={vllm_good_ratios}") - - # Plotting the scatter plot - plt.figure(figsize=(7, 4)) - plt.scatter(client_num_list, mii_goodputs_list, label=f"DeepSpeed-FastGen", marker="o", color="blue") - if not args.no_vllm: - plt.scatter(client_num_list, vllm_goodputs_list, label=f"vLLM", marker="x", color="orange") - - fit_x_list = np.arange(min(client_num_list), max(client_num_list), 0.1) - mii_fit_model = np.polyfit(client_num_list, mii_goodputs_list, 4) - mii_model_fn = np.poly1d(mii_fit_model) - plt.plot(fit_x_list, mii_model_fn(fit_x_list), color="blue", alpha=0.5, linestyle="--") - - if not args.no_vllm: - vllm_fit_model = np.polyfit(client_num_list, vllm_goodputs_list, 4) - vllm_model_fn = np.poly1d(vllm_fit_model) - plt.plot(fit_x_list, vllm_model_fn(fit_x_list), color="orange", alpha=0.5, linestyle="--") - - title = f"Effective throughput (SLA prompt: {SLA_PROMPT_TOKENS_PER_SEC} tokens/s, generation: {sla_token_gen} tokens/s)\n" \ - + f'Llama 2 {model_size.upper()} Prompt: {prompt}, Generation: {gen}, TP: {tp}' - plt.title(title, fontsize=10) - plt.xlabel('Number of clients', fontsize=10) - plt.ylabel('Effective throughput (queries/s)', fontsize=10) - # plt.rcParams['figure.subplot.bottom'] = 0.30 - plt.ylim(bottom=-0.05) - plt.legend() - plt.grid(True) - # plt.show() - out_file = out_dir / f"goodput_llama{model_size}_SLAp{SLA_PROMPT_TOKENS_PER_SEC}g{sla_token_gen}_tp{tp}_b{bs}_p{prompt}g{gen}_{f[2]}.png" - plt.savefig(out_file) - plt.clf() - print(f"Saved {out_file}") - - -if __name__ == "__main__": - args = get_args() - - if args.test: - tp_sizes = tp_sizes_test - prompt_gen_pairs = prompt_gen_pairs_test - else: - tp_sizes = tp_sizes_all - prompt_gen_pairs = prompt_gen_pairs_all - - for model_size, tps in tp_sizes.items(): - for tp in tps: - for prompt, gen in prompt_gen_pairs: - for sla_token_gen in SLA_GEN_TOKENS_PER_SEC: - display_results(model_size, tp, RAGGED_BATCH_SIZE, sla_token_gen, prompt, gen, args.log_dir, args.out_dir) - diff --git a/benchmarks/inference/mii/plot_latency_percentile.py b/benchmarks/inference/mii/plot_latency_percentile.py deleted file mode 100644 index c91c78bf1..000000000 --- a/benchmarks/inference/mii/plot_latency_percentile.py +++ /dev/null @@ -1,110 +0,0 @@ -import argparse -import glob -from pathlib import Path -import matplotlib.pyplot as plt -import numpy as np -import itertools - -from postprocess_results import read_json, get_token_latency - -bs = 768 -SKIP_HEAD_TOKEN_NUM = 2 -SKIP_REQUEST_NUM = 100 - -tp_sizes = { - "70b": [4], -} - -prompt_gen_pairs = [ - (2600, 128), -] - - -def get_args(): - parser = argparse.ArgumentParser() - parser.add_argument("--log_dir", type=Path, default=".") - parser.add_argument("--out_dir", type=Path, default="charts/percentile_token_latency") - args = parser.parse_args() - return args - - -def extract_values(file_pattern): - files = glob.glob(file_pattern) - - latencies = {} - for f in files: - prof_args, response_details = read_json(f) - client_num = prof_args["client_num"] - - response_details.sort(key=lambda r: r.start_time) - response_details = response_details[SKIP_REQUEST_NUM:-SKIP_REQUEST_NUM] - token_latencies = [r.token_gen_time[SKIP_HEAD_TOKEN_NUM:-1] for r in response_details] - - flat_latency_list = list(itertools.chain(*token_latencies)) - latencies[client_num] = flat_latency_list - return latencies - - -def output_charts(model_size, tp, bs, prompt, gen, log_dir, out_dir): - if not log_dir.exists(): - print(f"Log directory {log_dir} does not exist") - return - - if not out_dir.exists(): - out_dir.mkdir(parents=True, exist_ok=True) - - mii_file_pattern = f"{log_dir}/logs.llama2-{model_size}-tp{tp}-b{bs}/llama2-{model_size}-tp{tp}-b{bs}_c*_p{prompt}_g{gen}.json" - vllm_file_pattern = f"{log_dir}/logs.vllm-llama2-{model_size}-tp{tp}/vllm-llama2-{model_size}-tp{tp}_c*_p{prompt}_g{gen}.json" - - mii_latencies = extract_values(mii_file_pattern) - vllm_latencies = extract_values(vllm_file_pattern) - client_num_list = sorted(list(mii_latencies.keys())) - - for client_num in client_num_list: - plt.figure(figsize=(6, 4)) - - percentile = 95 - - P50_vllm_val = np.percentile(vllm_latencies[client_num], 50) - P50_mii_val = np.percentile(mii_latencies[client_num], 50) - P90_vllm_val = np.percentile(vllm_latencies[client_num], 90) - P90_mii_val = np.percentile(mii_latencies[client_num], 90) - P95_vllm_val = np.percentile(vllm_latencies[client_num], 95) - P95_mii_val = np.percentile(mii_latencies[client_num], 95) - - # print(f"P50_vllm_val={P50_vllm_val}") - # print(f"P50_mii_val={P50_mii_val}") - # print(f"P90_vllm_val={P90_vllm_val}") - # print(f"P90_mii_val={P90_mii_val}") - # print(f"P95_vllm_val={P95_vllm_val}") - # print(f"P95_mii_val={P95_mii_val}") - - out_file = out_dir / f"p{percentile}_token_latency_llama{model_size}_c{client_num}_tp{tp}_p{prompt}g{gen}.png" - - x1 = [1, 2, 3] - y1 = [P50_vllm_val, P90_vllm_val, P95_vllm_val] - - x2 = [1.3, 2.3, 3.3] - y2 = [P50_mii_val, P90_mii_val, P95_mii_val] - - label_x = ['P50', 'P90', 'P95'] - - plt.bar(x1, y1, width=0.3, label='vLLM', align="center", color="orange") - plt.bar(x2, y2, width=0.3, label="DeepSpeed-FastGen", align="center", color="blue") - plt.ylabel('Latency', fontsize=14) - plt.legend(loc=2) - - plt.xticks([1.15, 2.15, 3.15], label_x) - - plt.savefig(out_file) - print(f"Saved {out_file}") - - -if __name__ == "__main__": - args = get_args() - - for model_size, tps in tp_sizes.items(): - for tp in tps: - for prompt, gen in prompt_gen_pairs: - output_charts(model_size, tp, bs, prompt, gen, args.log_dir, args.out_dir) - diff --git a/benchmarks/inference/mii/plot_repl_scale.py b/benchmarks/inference/mii/plot_repl_scale.py deleted file mode 100644 index 394c54588..000000000 --- a/benchmarks/inference/mii/plot_repl_scale.py +++ /dev/null @@ -1,95 +0,0 @@ -import glob -import matplotlib.pyplot as plt -import argparse -from pathlib import Path -import numpy as np - -from postprocess_results import read_json, get_summary - -bs = 768 - -REPLICA_NUMS = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] - -tp_sizes = { - "70b": [4], -} - -prompt_gen_pairs = [ - (2600, 60), -] - -def get_args(): - parser = argparse.ArgumentParser() - parser.add_argument("--log_dir", type=Path, default=".") - parser.add_argument("--out_dir", type=Path, default="charts/repl_scale") - args = parser.parse_args() - return args - - -def extract_values(file_pattern): - files = glob.glob(file_pattern) - - clients = [] - throughputs = [] - latencies = [] - for f in files: - prof_args, response_details = read_json(f) - summary = get_summary(prof_args, response_details) - clients.append(prof_args["client_num"]) - throughputs.append(summary.throughput) - latencies.append(summary.latency) - - return clients, throughputs, latencies - - -def output_charts(model_size, tp, bs, prompt, gen, log_dir, out_dir): - if not log_dir.exists(): - print(f"Log directory {log_dir} does not exist") - return - - if not out_dir.exists(): - out_dir.mkdir(parents=True, exist_ok=True) - - throughputs = {} - for repl in REPLICA_NUMS: - mii_file_pattern = f"{log_dir}/logs.llama2-{model_size}-tp{tp}-b{bs}_repl{repl}/llama2-{model_size}-tp{tp}-b{bs}_repl{repl}_c*_p{prompt}_g{gen}.json" - print(f"Looking for {mii_file_pattern}") - clients, mii_throughputs, mii_latencies = extract_values(mii_file_pattern) - - for c, th in zip(clients, mii_throughputs): - client_per_repl = c // repl - if client_per_repl not in throughputs: - throughputs[client_per_repl] = [] - print(f"Throughput for {client_per_repl} clients: {th}") - throughputs[client_per_repl].append(th) - - for c in throughputs: - - # Plotting the scatter plot - plt.figure(figsize=(6, 4)) - - plt.bar(REPLICA_NUMS, throughputs[c], color="blue", alpha=0.9) - - fit_x_list = np.arange(min(REPLICA_NUMS), max(REPLICA_NUMS), 0.1) - mii_fit_model = np.polyfit(REPLICA_NUMS, throughputs[c], 1) - mii_model_fn = np.poly1d(mii_fit_model) - plt.plot(fit_x_list, mii_model_fn(fit_x_list), color="blue", linestyle="--") - - plt.title(f'Model Llama 2 {model_size.upper()}, Prompt: {prompt}, Generation: {gen}, TP: {tp}') - plt.xlabel('Number of replicas', fontsize=14) - plt.ylabel('Throughput (queries/s)', fontsize=14) - plt.grid(True) - plt.tight_layout() - # plt.show() - out_file = out_dir / f"repl_scale_llama{model_size}_tp{tp}_p{prompt}g{gen}.png" - plt.savefig(out_file) - - -if __name__ == "__main__": - args = get_args() - - for model_size, tps in tp_sizes.items(): - for tp in tps: - for prompt, gen in prompt_gen_pairs: - output_charts(model_size, tp, bs, prompt, gen, args.log_dir, args.out_dir) - diff --git a/benchmarks/inference/mii/plot_th_lat.py b/benchmarks/inference/mii/plot_th_lat.py deleted file mode 100644 index e99dc5a3e..000000000 --- a/benchmarks/inference/mii/plot_th_lat.py +++ /dev/null @@ -1,116 +0,0 @@ -import glob -import matplotlib.pyplot as plt -import argparse -from pathlib import Path -import numpy as np -import pdb -from postprocess_results import read_json, get_summary - -bs = 768 - -tp_sizes_test = { - "7b": [1] -} - -tp_sizes_all = { - "7b": [1], - "70b": [4, 8], -} - -prompt_gen_pairs_test = [ - (2600, 60) -] - -prompt_gen_pairs_all = [ - (1200, 60), - (1200, 128), - (2600, 60), - (2600, 128), -] - -def get_args(): - parser = argparse.ArgumentParser() - parser.add_argument("--test", action="store_true") - parser.add_argument("--no_vllm", action="store_true") - parser.add_argument("--log_dir", type=Path, default=".") - parser.add_argument("--out_dir", type=Path, default="charts/throughput_latency") - args = parser.parse_args() - return args - - -def extract_values(file_pattern): - files = glob.glob(file_pattern) - - print(f"Found {len(files)}") - print('\n'.join(files)) - - clients = [] - throughputs = [] - latencies = [] - for f in files: - prof_args, response_details = read_json(f) - summary = get_summary(prof_args, response_details) - clients.append(prof_args["client_num"]) - throughputs.append(summary.throughput) - latencies.append(summary.latency) - - return clients, throughputs, latencies - - -def output_charts(model_size, tp, bs, prompt, gen, log_dir, out_dir): - if not log_dir.exists(): - print(f"Log directory {log_dir} does not exist") - return - - if not out_dir.exists(): - out_dir.mkdir(parents=True, exist_ok=True) - - mii_file_pattern = f"{log_dir}/logs.llama2-{model_size}-tp{tp}-b{bs}/llama2-{model_size}-tp{tp}-b{bs}_c*_p{prompt}_g{gen}.json" - if not args.no_vllm: - vllm_file_pattern = f"{log_dir}/logs.vllm-llama2-{model_size}-tp{tp}/vllm-llama2-{model_size}-tp{tp}_c*_p{prompt}_g{gen}.json" - - _, mii_throughputs, mii_latencies = extract_values(mii_file_pattern) - if not args.no_vllm: - _, vllm_throughputs, vllm_latencies = extract_values(vllm_file_pattern) - - # Plotting the scatter plot - plt.figure(figsize=(6, 4)) - - if not args.no_vllm: - plt.scatter(vllm_throughputs, vllm_latencies, label=f"vLLM", marker="x", color="orange") - fit_vllm_x_list = np.arange(min(vllm_throughputs), max(vllm_throughputs), 0.01) - vllm_vllm_model = np.polyfit(vllm_throughputs, vllm_latencies, 3) - vllm_model_fn = np.poly1d(vllm_vllm_model) - plt.plot(fit_vllm_x_list, vllm_model_fn(fit_vllm_x_list), color="orange", alpha=0.5, linestyle="--") - - plt.scatter(mii_throughputs, mii_latencies, label=f"DeepSpeed FastGen", marker="o", color="blue") - fit_mii_x_list = np.arange(min(mii_throughputs), max(mii_throughputs), 0.01) - mii_fit_model = np.polyfit(mii_throughputs, mii_latencies, 3) - mii_model_fn = np.poly1d(mii_fit_model) - plt.plot(fit_mii_x_list, mii_model_fn(fit_mii_x_list), color="blue", alpha=0.5, linestyle="--") - - plt.title(f'Model Llama 2 {model_size.upper()}, Prompt: {prompt}, Generation: {gen}, TP: {tp}') - plt.xlabel('Throughput (queries/s)', fontsize=14) - plt.ylabel('Latency', fontsize=14) - plt.legend() - plt.grid(True) - plt.tight_layout() - out_file = out_dir / f"th_lat_curve_llama{model_size}_tp{tp}_p{prompt}g{gen}.png" - print(f"Saving {out_file}") - plt.savefig(out_file) - - -if __name__ == "__main__": - args = get_args() - if args.test: - tp_sizes = tp_sizes_test - prompt_gen_pairs = prompt_gen_pairs_test - else: - tp_sizes = tp_sizes_all - prompt_gen_pairs = prompt_gen_pairs_test_all - - for model_size, tps in tp_sizes.items(): - for tp in tps: - for prompt, gen in prompt_gen_pairs: - output_charts(model_size, tp, bs, prompt, gen, args.log_dir, args.out_dir) - diff --git a/benchmarks/inference/mii/plot_tp_sizes.py b/benchmarks/inference/mii/plot_tp_sizes.py deleted file mode 100644 index 546310258..000000000 --- a/benchmarks/inference/mii/plot_tp_sizes.py +++ /dev/null @@ -1,98 +0,0 @@ -import glob -import matplotlib.pyplot as plt -import argparse -from pathlib import Path -import numpy as np - -from postprocess_results import read_json, get_summary - -bs = 768 - -tp_sizes = { - # "7b": [1], - "13b": [1, 2, 4], - # "70b": [4, 8], -} - -prompt_gen_pairs = [ - (1200, 60), - (1200, 128), - (2600, 60), - (2600, 128), - (2600, 256), -] - -def get_args(): - parser = argparse.ArgumentParser() - parser.add_argument("--log_dir", type=Path, default="logs.release") - parser.add_argument("--out_dir", type=Path, default="charts/tp_sizes") - args = parser.parse_args() - return args - - -def extract_values(file_pattern): - files = glob.glob(file_pattern) - - print(f"Found {len(files)}") - print('\n'.join(files)) - - clients = [] - throughputs = [] - latencies = [] - for f in files: - prof_args, response_details = read_json(f) - summary = get_summary(prof_args, response_details) - clients.append(prof_args["client_num"]) - throughputs.append(summary.throughput) - latencies.append(summary.latency) - - return clients, throughputs, latencies - - -def output_charts(model_size, tps, bs, prompt, gen, log_dir, out_dir): - if not log_dir.exists(): - print(f"Log directory {log_dir} does not exist") - return - - if not out_dir.exists(): - out_dir.mkdir(parents=True, exist_ok=True) - - # Plotting the scatter plot - plt.figure(figsize=(6, 4)) - - colors = ["orange", "green", "brown"] - - for tp, color in zip(tps, colors): - mii_file_pattern = f"{log_dir}/logs.llama2-{model_size}-tp{tp}-b{bs}/llama2-{model_size}-tp{tp}-b{bs}_c*_p{prompt}_g{gen}.json" - _, mii_throughputs, mii_latencies = extract_values(mii_file_pattern) - - if len(mii_throughputs) == 0: - continue - - n_params = int(model_size[:-1]) - tflops_per_query = n_params * (prompt + gen) * 2 * 1e-3 - mii_tflops = [th * tflops_per_query / tp for th in mii_throughputs] - - plt.scatter(mii_tflops, mii_latencies, label=f"TP={tp}", marker="o", color=color) - fit_mii_x_list = np.arange(min(mii_tflops), max(mii_tflops), 0.01) - mii_fit_model = np.polyfit(mii_tflops, mii_latencies, 3) - mii_model_fn = np.poly1d(mii_fit_model) - plt.plot(fit_mii_x_list, mii_model_fn(fit_mii_x_list), color=color, alpha=0.5, linestyle="--") - - plt.title(f'Model Llama 2 {model_size.upper()}, Prompt: {prompt}, Generation: {gen}, TP: {tps}') - plt.xlabel('TFLOPs (per GPU)', fontsize=14) - plt.ylabel('Latency', fontsize=14) - plt.legend() - plt.grid(True) - # plt.show() - out_file = out_dir / f"tp_sizes_llama{model_size}_tp{'_'.join([str(tp) for tp in tps])}_p{prompt}g{gen}.png" - plt.savefig(out_file) - - -if __name__ == "__main__": - args = get_args() - - for model_size, tps in tp_sizes.items(): - for prompt, gen in prompt_gen_pairs: - output_charts(model_size, tps, bs, prompt, gen, args.log_dir, args.out_dir) - diff --git a/benchmarks/inference/mii/postprocess_results.py b/benchmarks/inference/mii/postprocess_results.py deleted file mode 100644 index cb2000d5f..000000000 --- a/benchmarks/inference/mii/postprocess_results.py +++ /dev/null @@ -1,112 +0,0 @@ -import argparse -from pathlib import Path -import json -import numpy as np -from statistics import mean -from functools import reduce -from dataclasses import dataclass -from typing import List - -from transformers import AutoTokenizer - - -tokenizer = None - - -@dataclass -class ResponseDetails: - generated_tokens: List[str] - prompt: str - start_time: float - end_time: float - model_time: float - token_gen_time: List[float] - - -@dataclass -class ProfilingSummary: - throughput: float - latency: float - token_gen_latency: float - first_token_latency: float - tokens_per_sec: float - - -def parse_args(): - parser = argparse.ArgumentParser(description="Postprocess results") - parser.add_argument('-i', '--input_path', type=Path, default="results.json") - - args = parser.parse_args() - return args - - -def get_tokenizer(): - global tokenizer - if tokenizer is None: - tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-2-7b-hf") - return tokenizer - - -def read_json(file_path): - with open(file_path, 'r') as f: - data = json.load(f) - - args = data["args"] - - response_details = [] - for response in data["response_details"]: - response_details.append(ResponseDetails(**response)) - - return args, response_details - - -def get_summary(args, response_details): - client_num = args["client_num"] - - # Calculate latency and throughput using P95 latency - latency = mean([r.end_time - r.start_time for r in response_details]) - throughput = client_num / latency - - tokens_per_sec = mean([(len(get_tokenizer().tokenize(r.prompt)) + len(r.generated_tokens)) / (r.end_time - r.start_time) for r in response_details]) - first_token_latency = mean([r.token_gen_time[0] for r in response_details]) - - token_gen_latency_flat = reduce(list.__add__, [r.token_gen_time[1:-1] for r in response_details if len(r.token_gen_time) > 2]) - token_gen_latency = mean([t for t in token_gen_latency_flat]) - - return ProfilingSummary(throughput, latency, token_gen_latency, first_token_latency, tokens_per_sec) - - -def get_token_latency(response_details, percentile=None, variance=False, cumulative=False): - req_latencies = [r.token_gen_time for r in response_details] - if cumulative: - req_latencies = [np.cumsum(np.array(r.token_gen_time)).tolist() for r in response_details] - max_gen_length = max([len(r.generated_tokens) for r in response_details]) - latency = [] - for i in range(max_gen_length): - if variance: - token_latency_step = np.var([latency[i] for latency in req_latencies if len(latency) > i]) - if percentile is None: - token_latency_step = [latency[i] for latency in req_latencies if len(latency) > i] - else: - token_latency_step = np.percentile([latency[i] for latency in req_latencies if len(latency) > i], percentile) - - latency.append(token_latency_step) - - return latency - - -def get_token_acc_latency(response_details, percentile=99): - return get_token_latency(response_details, percentile, cumulative=True) - - -if __name__ == "__main__": - args = parse_args() - prof_args, response_details = read_json(args.input_path) - - ps = get_summary(prof_args, response_details) - print(f"Deployment: {prof_args['deployment_name']} Clients: {prof_args['client_num']}, " - + f"Query throughput: {ps.throughput:.3f} queries/s, " - + f"Token throughput (total): {ps.tokens_per_sec:.3f} tokens/s, " - + f"Query latency: {ps.latency:.3f} s, " - + f"Token generation latency: {ps.token_gen_latency:.3f} s/token, " - + f"First token received: {ps.first_token_latency:.3f} s") diff --git a/benchmarks/inference/mii/requirements.txt b/benchmarks/inference/mii/requirements.txt new file mode 100644 index 000000000..9f338ace5 --- /dev/null +++ b/benchmarks/inference/mii/requirements.txt @@ -0,0 +1,6 @@ +transformers +matplotlib +deepspeed-mii>=0.2.0 +vllm>=0.2.7 +numpy +tabulate diff --git a/benchmarks/inference/mii/run_all.sh b/benchmarks/inference/mii/run_all.sh index ca504a6c9..7c9311aea 100644 --- a/benchmarks/inference/mii/run_all.sh +++ b/benchmarks/inference/mii/run_all.sh @@ -1,25 +1,15 @@ -RAGGED_BATCH_SIZE=768 -PARAM_SIZES=(7b 13b 70b) +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 -declare -A TP_SIZES -TP_SIZES["7b"]="1" -TP_SIZES["13b"]="1:2:4" -TP_SIZES["70b"]="4:8" +# DeepSpeed Team -for PARAM_SIZE in ${PARAM_SIZES[@]}; do - - IFS=':' read -ra TP_VALUES <<< ${TP_SIZES[${PARAM_SIZE}]} - for TP in ${TP_VALUES[@]}; do - DEPLOYMENT_NAME=llama2-${PARAM_SIZE}-tp${TP}-b${RAGGED_BATCH_SIZE} - python server.py --model_name meta-llama/Llama-2-${PARAM_SIZE}-hf -d ${DEPLOYMENT_NAME} -m ${TP} -b ${RAGGED_BATCH_SIZE} start +MODELS=(meta-llama/Llama-2-7b-hf meta-llama/Llama-2-13b-hf meta-llama/Llama-2-70b-hf tiiuae/falcon-40B tiiuae/falcon-180B microsoft/phi-2 mistralai/Mixtral-8x7B-v0.1) - DEPLOYMENT_NAME=${DEPLOYMENT_NAME} PROMPT_LENGTH=2600 MAX_NEW_TOKENS=60 bash ./run_benchmark_client.sh - DEPLOYMENT_NAME=${DEPLOYMENT_NAME} PROMPT_LENGTH=2600 MAX_NEW_TOKENS=128 bash ./run_benchmark_client.sh - DEPLOYMENT_NAME=${DEPLOYMENT_NAME} PROMPT_LENGTH=1200 MAX_NEW_TOKENS=60 bash ./run_benchmark_client.sh - DEPLOYMENT_NAME=${DEPLOYMENT_NAME} PROMPT_LENGTH=1200 MAX_NEW_TOKENS=128 bash ./run_benchmark_client.sh - - echo "Stopping server" - python server.py -d ${DEPLOYMENT_NAME} stop - sleep 120 - done +for MODEL in ${MODELS[@]}; do + python ./run_benchmark.py --model ${MODEL} --stream --backend fastgen + python ./run_benchmark.py --model ${MODEL} --stream --backend vllm done + +# Extra runs for Mixtral with non-default settings +python ./run_benchmark.py --model mistralai/Mixtral-8x7B-v0.1 --stream --tp_size 4 --mean_prompt_length 500 --mean_max_new_tokens 150 500 1024 --backend fastgen +python ./run_benchmark.py --model mistralai/Mixtral-8x7B-v0.1 --stream --tp_size 4 --mean_prompt_length 500 --mean_max_new_tokens 150 500 1024 --backend vllm \ No newline at end of file diff --git a/benchmarks/inference/mii/run_all_replica.sh b/benchmarks/inference/mii/run_all_replica.sh deleted file mode 100644 index b3fba0408..000000000 --- a/benchmarks/inference/mii/run_all_replica.sh +++ /dev/null @@ -1,25 +0,0 @@ -RAGGED_BATCH_SIZE=768 -PARAM_SIZES=(7b) -REPLICA_NUMS=(1) - -declare -A TP_SIZES -TP_SIZES["7b"]="4" -TP_SIZES["13b"]="1" -TP_SIZES["70b"]="4" - -for PARAM_SIZE in ${PARAM_SIZES[@]}; do - IFS=':' read -ra TP_VALUES <<< ${TP_SIZES[${PARAM_SIZE}]} - for TP in ${TP_VALUES[@]}; do - for REPL in ${REPLICA_NUMS[@]}; do - DEPLOYMENT_NAME=llama2-${PARAM_SIZE}-tp${TP}-b${RAGGED_BATCH_SIZE}_repl${REPL} - python server.py --model_name meta-llama/Llama-2-${PARAM_SIZE}-hf -d ${DEPLOYMENT_NAME} -m ${TP} -r ${REPL} -b ${RAGGED_BATCH_SIZE} start - - REQUEST_NUM=$((256 * ${REPL})) - DEPLOYMENT_NAME=${DEPLOYMENT_NAME} PROMPT_LENGTH=2600 MAX_NEW_TOKENS=60 CLIENT_NUMS=$((16 * ${REPL})) REQUEST_NUM=$((256 * ${REPL})) bash ./run_bench_client_num.sh - - echo "Stopping server" - python server.py -d ${DEPLOYMENT_NAME} stop - sleep 120 - done - done -done diff --git a/benchmarks/inference/mii/run_all_vllm.sh b/benchmarks/inference/mii/run_all_vllm.sh deleted file mode 100644 index 572377f13..000000000 --- a/benchmarks/inference/mii/run_all_vllm.sh +++ /dev/null @@ -1,26 +0,0 @@ -RAGGED_BATCH_SIZE=768 -PARAM_SIZES=(7b 13b 70b) - -declare -A TP_SIZES -TP_SIZES["7b"]="1" -TP_SIZES["13b"]="1:2:4" -TP_SIZES["70b"]="4:8" - -for PARAM_SIZE in ${PARAM_SIZES[@]}; do - - IFS=':' read -ra TP_VALUES <<< ${TP_SIZES[${PARAM_SIZE}]} - for TP in ${TP_VALUES[@]}; do - DEPLOYMENT_NAME=vllm-llama2-${PARAM_SIZE}-tp${TP} - python -m vllm.entrypoints.api_server --host 127.0.0.1 --port 26500 --tensor-parallel-size ${TP} --model meta-llama/Llama-2-${PARAM_SIZE}-hf & - sleep 60 - - DEPLOYMENT_NAME=${DEPLOYMENT_NAME} PROMPT_LENGTH=2600 MAX_NEW_TOKENS=60 VLLM="--vllm" bash ./run_benchmark_client.sh - DEPLOYMENT_NAME=${DEPLOYMENT_NAME} PROMPT_LENGTH=2600 MAX_NEW_TOKENS=128 VLLM="--vllm" bash ./run_benchmark_client.sh - DEPLOYMENT_NAME=${DEPLOYMENT_NAME} PROMPT_LENGTH=1200 MAX_NEW_TOKENS=60 VLLM="--vllm" bash ./run_benchmark_client.sh - DEPLOYMENT_NAME=${DEPLOYMENT_NAME} PROMPT_LENGTH=1200 MAX_NEW_TOKENS=128 VLLM="--vllm" bash ./run_benchmark_client.sh - - echo "Stopping server" - pkill -u ${USER} -f vllm.entrypoints.api_server - sleep 30 - done -done diff --git a/benchmarks/inference/mii/run_aml.sh b/benchmarks/inference/mii/run_aml.sh new file mode 100644 index 000000000..90ad50e2c --- /dev/null +++ b/benchmarks/inference/mii/run_aml.sh @@ -0,0 +1,20 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# Run benchmark against AML endpoint +python ./run_benchmark.py \ + --model
+ 
+
+
+
+ ds_report output showing availability of DeepNVMe operators (async_io and gds) in a DeepSpeed installation.
+
+
+
+If `async_io` opertator is unavailable, you will need to install the appropriate `libaio` library binaries for your Linux flavor. For example, Ubuntu users will need to run `apt install libaio-dev`. In general, you should carefully inspect `ds_report` output for helpful tips such as the following:
+
+```bash
+[WARNING] async_io requires the dev libaio .so object and headers but these were not found.
+[WARNING] async_io: please install the libaio-dev package with apt
+[WARNING] If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found.
+```
+
+To enable `gds` operator, you will need to install NVIDIA GDS by consulting the appropriate guide for [bare-metal systems](https://docs.nvidia.com/gpudirect-storage/troubleshooting-guide/index.html) or Azure VMs (coming soon).
+
+## Tensor Load Examples
+The tensor load example scripts share a common command-line interface, which is illustrated below using `py_read_load_cpu_tensor.py`.
+```bash
+$ python py_load_cpu_tensor.py --help
+usage: py_load_cpu_tensor.py [-h] --input_file INPUT_FILE [--loop LOOP] [--validate]
+
+options:
+ -h, --help show this help message and exit
+ --input_file INPUT_FILE
+ File on NVMe device that will read as input.
+ --loop LOOP The number of times to repeat the operation (default 3).
+ --validate Run validation step that compares tensor value against Python file read
+```
+Before running these example scripts ensure that the input file exists on an NVMe device. The `--validate` option is relevant only to the DeepNVme implementations. This option provides minimal correctness checking by comparing against a tensor loaded using Python. We also provide a bash script `run_load_tensor.sh`, which runs all the example tensor load scripts.
+
+
+## Tensor Store Examples
+The tensor store examples share a command-line interface, which is illustrated below using `py_store_cpu_tensor.py`
+```bash
+$ python py_store_cpu_tensor.py --help
+usage: py_store_cpu_tensor.py [-h] --nvme_folder NVME_FOLDER [--mb_size MB_SIZE] [--loop LOOP] [--validate]
+
+options:
+ -h, --help show this help message and exit
+ --nvme_folder NVME_FOLDER
+ NVMe folder for file write.
+ --mb_size MB_SIZE Size of tensor to save in MB (default 1024).
+ --loop LOOP The number of times to repeat the operation (default 3).
+ --validate Run validation step that compares tensor value against Python file read
+
+```
+Before running these examples ensure that the output folder exists on an NVMe device and that you have write permission. The `--validate` option is relevant only to the DeepNVMe implementations. This option provides minimal correctness checking by comparing the output file against that created using Python. We also provide a bash script `run_store_tensor.sh`, which runs all the example tensor store scripts.
+
+
+## Performance Advisory
+Although this folder is primarily meant to help with integrating DeepNVMe into your Deep Learning applications, the example scripts also print out performance numbers of read and write throughput. So, we expect you will observe some performance advantage of DeepNVMe compared to Python. However, do note that it is likely that better performance can be realized by tuning DeepNVMe for your environment. Such tuning efforts will ideally generate more optimal values for configuring DeepNVMe.
+
+For reference, DeepNVMe configuration using hard-coded constants for `aio_` implementations is as follows:
+
+```python
+ aio_handle = AsyncIOBuilder().load().aio_handle(1024**2, 128, True, True, 1)
+```
+
+The corresponding DeepNVMe configuration for `gds_` implementations is as follows:
+
+```python
+ gds_handle = GDSBuilder().load().gds_handle(1024**2, 128, True, True, 1)
+```
+
+Despite the above caveat, it seems that some performance numbers would be useful here to help set the right expectations. The experiments were conducted on an Azure [NC80adis_H100_v5](https://learn.microsoft.com/en-us/azure/virtual-machines/ncads-h100-v5) series virtual machine (VM). This VM includes two 3.5TB local NVMe devices (labelled Microsoft NVMe Direct Disk v2) that we combined into a single RAID-0 volume. The software environment included Ubuntu 22.04.4 LTS, Linux kernel 6.5.0-26-generic, Pytorch 2.4, and CUDA 12.4. We ran experiments of 1GB data transfers using the unmodified scripts, i.e., without DeepNVMe tuning, and present the throughput results in the tables below. In summary, we observed that DeepNVMe significantly accelerates I/O operations compared to Python. DeepNVMe is 8-16X faster for loading tensor data, and 11X-119X faster for writing tensor data.
+
+Load 1GB CPU tensor (1GB file read) | GB/sec | Speedup over Python |
+|---|---|---|
+py_load_cpu_tensor.py | 1.5 | - |
+aio_load_cpu_tensor.py | 12.3 | 8X |
+
+Load 1GB GPU tensor (1GB file read) | GB/sec | Speedup over Python |
+|---|---|---|
+py_load_gpu_tensor.py | 0.7| - |
+aio_load_gpu_tensor.py | 9.9 | 14X |
+gds_load_gpu_tensor.py | 11.1 | 16X |
+
+
+Store 1GB CPU tensor (1GB file write) | GB/sec | Speedup over Python |
+|---|---|---|
+py_store_cpu_tensor.py | 0.7 | - |
+aio_store_cpu_tensor.py | 8.1 | 11X |
+
+
+Store 1GB GPU tensor (1GB file write) | GB/sec | Speedup over Python |
+|---|---|---|
+py_store_gpu_tensor.py | 0.5 | - |
+aio_store_gpu_tensor.py | 8.3 | 18X |
+gds_store_gpu_tensor.py | 8.6 | 19X |
+
+
+
+# Conclusion
+We hope you find this document and example scripts useful for integrating DeepNVMe into your applications.
diff --git a/deepnvme/file_access/aio_load_cpu_tensor.py b/deepnvme/file_access/aio_load_cpu_tensor.py
new file mode 100644
index 000000000..27a1e61c5
--- /dev/null
+++ b/deepnvme/file_access/aio_load_cpu_tensor.py
@@ -0,0 +1,31 @@
+import torch
+import os, timeit, functools
+from deepspeed.ops.op_builder import AsyncIOBuilder
+from utils import parse_read_arguments, GIGA_UNIT
+
+def file_read(inp_f, handle, bounce_buffer):
+ handle.sync_pread(bounce_buffer, inp_f)
+ return bounce_buffer.cpu()
+
+def main():
+ args = parse_read_arguments()
+ input_file = args.input_file
+ file_sz = os.path.getsize(input_file)
+ cnt = args.loop
+
+ aio_handle = AsyncIOBuilder().load().aio_handle()
+ bounce_buffer = torch.empty(os.path.getsize(input_file), dtype=torch.uint8).pin_memory()
+
+ t = timeit.Timer(functools.partial(file_read, input_file, aio_handle, bounce_buffer))
+ aio_t = t.timeit(cnt)
+ aio_gbs = (cnt*file_sz)/GIGA_UNIT/aio_t
+ print(f'aio load_cpu: {file_sz/GIGA_UNIT} GB, {aio_t/cnt} secs, {aio_gbs:5.2f} GB/sec')
+
+ if args.validate:
+ from py_load_cpu_tensor import file_read as py_file_read
+ aio_tensor = file_read(input_file, aio_handle, bounce_buffer)
+ py_tensor = py_file_read(input_file)
+ print(f'Validation success = {aio_tensor.equal(py_tensor)}')
+
+if __name__ == "__main__":
+ main()
diff --git a/deepnvme/file_access/aio_load_gpu_tensor.py b/deepnvme/file_access/aio_load_gpu_tensor.py
new file mode 100644
index 000000000..aeecc6e5d
--- /dev/null
+++ b/deepnvme/file_access/aio_load_gpu_tensor.py
@@ -0,0 +1,32 @@
+import torch
+import os, timeit, functools
+from deepspeed.ops.op_builder import AsyncIOBuilder
+from utils import parse_read_arguments, GIGA_UNIT
+
+def file_read(inp_f, handle, bounce_buffer):
+ handle.sync_pread(bounce_buffer, inp_f)
+ return bounce_buffer.cuda()
+
+
+def main():
+ args = parse_read_arguments()
+ input_file = args.input_file
+ file_sz = os.path.getsize(input_file)
+ cnt = args.loop
+
+ aio_handle = AsyncIOBuilder().load().aio_handle()
+ bounce_buffer = torch.empty(os.path.getsize(input_file), dtype=torch.uint8).pin_memory()
+
+ t = timeit.Timer(functools.partial(file_read, input_file, aio_handle, bounce_buffer))
+ aio_t = t.timeit(cnt)
+ aio_gbs = (cnt*file_sz)/GIGA_UNIT/aio_t
+ print(f'aio load_gpu: {file_sz/GIGA_UNIT} GB, {aio_t/cnt} secs, {aio_gbs:5.2f} GB/sec')
+
+ if args.validate:
+ from py_load_cpu_tensor import file_read as py_file_read
+ aio_tensor = file_read(input_file, aio_handle, bounce_buffer).cpu()
+ py_tensor = py_file_read(input_file)
+ print(f'Validation success = {aio_tensor.equal(py_tensor)}')
+
+if __name__ == "__main__":
+ main()
diff --git a/deepnvme/file_access/aio_store_cpu_tensor.py b/deepnvme/file_access/aio_store_cpu_tensor.py
new file mode 100644
index 000000000..20c03792b
--- /dev/null
+++ b/deepnvme/file_access/aio_store_cpu_tensor.py
@@ -0,0 +1,40 @@
+import torch
+import os, timeit, functools, pathlib
+from deepspeed.ops.op_builder import AsyncIOBuilder
+from utils import parse_write_arguments, GIGA_UNIT
+
+def file_write(out_f, tensor, handle, bounce_buffer):
+ bounce_buffer.copy_(tensor)
+ handle.sync_pwrite(bounce_buffer, out_f)
+
+def main():
+ args = parse_write_arguments()
+ cnt = args.loop
+ output_file = os.path.join(args.nvme_folder, f'test_ouput_{args.mb_size}MB.pt')
+ pathlib.Path(output_file).unlink(missing_ok=True)
+ file_sz = args.mb_size*(1024**2)
+ app_tensor = torch.empty(file_sz, dtype=torch.uint8, device='cpu', requires_grad=False)
+
+ aio_handle = AsyncIOBuilder().load().aio_handle()
+ bounce_buffer = torch.empty(file_sz, dtype=torch.uint8, requires_grad=False).pin_memory()
+
+
+ t = timeit.Timer(functools.partial(file_write, output_file, app_tensor, aio_handle, bounce_buffer))
+
+ aio_t = t.timeit(cnt)
+ aio_gbs = (cnt*file_sz)/GIGA_UNIT/aio_t
+ print(f'aio store_cpu: {file_sz/GIGA_UNIT} GB, {aio_t/cnt} secs, {aio_gbs:5.2f} GB/sec')
+
+ if args.validate:
+ import tempfile, filecmp
+ from py_store_cpu_tensor import file_write as py_file_write
+ py_ref_file = os.path.join(tempfile.gettempdir(), os.path.basename(output_file))
+ py_file_write(py_ref_file, app_tensor)
+ filecmp.clear_cache()
+ print(f'Validation success = {filecmp.cmp(py_ref_file, output_file, shallow=False) }')
+
+ pathlib.Path(output_file).unlink(missing_ok=True)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/deepnvme/file_access/aio_store_gpu_tensor.py b/deepnvme/file_access/aio_store_gpu_tensor.py
new file mode 100644
index 000000000..71a4aa7bb
--- /dev/null
+++ b/deepnvme/file_access/aio_store_gpu_tensor.py
@@ -0,0 +1,40 @@
+import torch
+import os, timeit, functools, pathlib
+from deepspeed.ops.op_builder import AsyncIOBuilder
+from utils import parse_write_arguments, GIGA_UNIT
+
+def file_write(out_f, tensor, handle, bounce_buffer):
+ bounce_buffer.copy_(tensor)
+ handle.sync_pwrite(bounce_buffer, out_f)
+
+def main():
+ args = parse_write_arguments()
+ cnt = args.loop
+ output_file = os.path.join(args.nvme_folder, f'test_ouput_{args.mb_size}MB.pt')
+ pathlib.Path(output_file).unlink(missing_ok=True)
+ file_sz = args.mb_size*(1024**2)
+ app_tensor = torch.empty(file_sz, dtype=torch.uint8, device='cuda', requires_grad=False)
+
+ aio_handle = AsyncIOBuilder().load().aio_handle()
+ bounce_buffer = torch.empty(file_sz, dtype=torch.uint8, requires_grad=False).pin_memory()
+
+
+ t = timeit.Timer(functools.partial(file_write, output_file, app_tensor, aio_handle, bounce_buffer))
+
+ aio_t = t.timeit(cnt)
+ aio_gbs = (cnt*file_sz)/GIGA_UNIT/aio_t
+ print(f'aio store_gpu: {file_sz/GIGA_UNIT} GB, {aio_t/cnt} secs, {aio_gbs:5.2f} GB/sec')
+
+ if args.validate:
+ import tempfile, filecmp
+ from py_store_cpu_tensor import file_write as py_file_write
+ py_ref_file = os.path.join(tempfile.gettempdir(), os.path.basename(output_file))
+ py_file_write(py_ref_file, app_tensor)
+ filecmp.clear_cache()
+ print(f'Validation success = {filecmp.cmp(py_ref_file, output_file, shallow=False) }')
+
+ pathlib.Path(output_file).unlink(missing_ok=True)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/deepnvme/file_access/gds_load_gpu_tensor.py b/deepnvme/file_access/gds_load_gpu_tensor.py
new file mode 100644
index 000000000..dd6273707
--- /dev/null
+++ b/deepnvme/file_access/gds_load_gpu_tensor.py
@@ -0,0 +1,33 @@
+import torch
+import os, timeit, functools
+from utils import parse_read_arguments, GIGA_UNIT
+from deepspeed.ops.op_builder import GDSBuilder
+
+def file_read(inp_f, handle, gpu_buffer):
+ handle.sync_pread(gpu_buffer, inp_f)
+ return gpu_buffer.cuda()
+
+def main():
+ args = parse_read_arguments()
+ input_file = args.input_file
+ file_sz = os.path.getsize(input_file)
+ cnt = args.loop
+
+ gds_handle = GDSBuilder().load().gds_handle()
+ gds_buffer = gds_handle.new_pinned_device_tensor(file_sz, torch.empty(0, dtype=torch.uint8, device='cuda', requires_grad=False))
+
+ t = timeit.Timer(functools.partial(file_read, input_file, gds_handle, gds_buffer))
+ gds_t = t.timeit(cnt)
+ gds_gbs = (cnt*file_sz)/GIGA_UNIT/gds_t
+ print(f'gds load_gpu: {file_sz/GIGA_UNIT} GB, {gds_t/cnt} secs, {gds_gbs:5.2f} GB/sec')
+
+ if args.validate:
+ from py_load_cpu_tensor import file_read as py_file_read
+ aio_tensor = file_read(input_file, gds_handle, gds_buffer).cpu()
+ py_tensor = py_file_read(input_file)
+ print(f'Validation success = {aio_tensor.equal(py_tensor)}')
+
+ gds_handle.free_pinned_device_tensor(gds_buffer)
+
+if __name__ == "__main__":
+ main()
diff --git a/deepnvme/file_access/gds_store_gpu_tensor.py b/deepnvme/file_access/gds_store_gpu_tensor.py
new file mode 100644
index 000000000..06ba508ba
--- /dev/null
+++ b/deepnvme/file_access/gds_store_gpu_tensor.py
@@ -0,0 +1,39 @@
+import torch
+import os, timeit, functools, pathlib
+from deepspeed.ops.op_builder import GDSBuilder
+from utils import parse_write_arguments, GIGA_UNIT
+
+def file_write(out_f, tensor, handle, gpu_buffer):
+ gpu_buffer.copy_(tensor)
+ handle.sync_pwrite(gpu_buffer, out_f)
+
+def main():
+ args = parse_write_arguments()
+ cnt = args.loop
+ output_file = os.path.join(args.nvme_folder, f'test_ouput_{args.mb_size}MB.pt')
+ pathlib.Path(output_file).unlink(missing_ok=True)
+ file_sz = args.mb_size*(1024**2)
+ app_tensor = torch.empty(file_sz, dtype=torch.uint8, device='cuda', requires_grad=False)
+
+ gds_handle = GDSBuilder().load().gds_handle()
+ gds_buffer = gds_handle.new_pinned_device_tensor(file_sz, torch.empty(0, dtype=torch.uint8, device='cuda', requires_grad=False))
+
+ t = timeit.Timer(functools.partial(file_write, output_file, app_tensor, gds_handle, gds_buffer))
+
+ gds_t = t.timeit(cnt)
+ gds_gbs = (cnt*file_sz)/GIGA_UNIT/gds_t
+ print(f'gds store_gpu: {file_sz/GIGA_UNIT} GB, {gds_t/cnt} secs, {gds_gbs:5.2f} GB/sec')
+
+ if args.validate:
+ import tempfile, filecmp
+ from py_store_cpu_tensor import file_write as py_file_write
+ py_ref_file = os.path.join(tempfile.gettempdir(), os.path.basename(output_file))
+ py_file_write(py_ref_file, app_tensor)
+ filecmp.clear_cache()
+ print(f'Validation success = {filecmp.cmp(py_ref_file, output_file, shallow=False) }')
+
+ gds_handle.free_pinned_device_tensor(gds_buffer)
+ pathlib.Path(output_file).unlink(missing_ok=True)
+
+if __name__ == "__main__":
+ main()
diff --git a/deepnvme/file_access/media/deepnvme_ops_report.png b/deepnvme/file_access/media/deepnvme_ops_report.png
new file mode 100644
index 000000000..c05e9b863
Binary files /dev/null and b/deepnvme/file_access/media/deepnvme_ops_report.png differ
diff --git a/deepnvme/file_access/py_load_cpu_tensor.py b/deepnvme/file_access/py_load_cpu_tensor.py
new file mode 100644
index 000000000..0650848f0
--- /dev/null
+++ b/deepnvme/file_access/py_load_cpu_tensor.py
@@ -0,0 +1,22 @@
+import torch
+import os, timeit, functools
+from utils import parse_read_arguments, GIGA_UNIT
+
+def file_read(inp_f):
+ with open(inp_f, 'rb') as f:
+ tensor = torch.frombuffer(f.read(), dtype=torch.uint8)
+ return tensor
+
+def main():
+ args = parse_read_arguments()
+ input_file = args.input_file
+ file_sz = os.path.getsize(input_file)
+ cnt = args.loop
+
+ t = timeit.Timer(functools.partial(file_read, input_file))
+ py_t = t.timeit(cnt)
+ py_gbs = (cnt*file_sz)/GIGA_UNIT/py_t
+ print(f'py load_cpu: {file_sz/GIGA_UNIT} GB, {py_t/cnt} secs, {py_gbs:5.2f} GB/sec')
+
+if __name__ == "__main__":
+ main()
diff --git a/deepnvme/file_access/py_load_gpu_tensor.py b/deepnvme/file_access/py_load_gpu_tensor.py
new file mode 100644
index 000000000..976967dca
--- /dev/null
+++ b/deepnvme/file_access/py_load_gpu_tensor.py
@@ -0,0 +1,22 @@
+import torch
+import os, timeit, functools
+from utils import parse_read_arguments, GIGA_UNIT
+
+def file_read(inp_f):
+ with open(inp_f, 'rb') as f:
+ tensor = torch.frombuffer(f.read(), dtype=torch.uint8)
+ return tensor.cuda()
+
+def main():
+ args = parse_read_arguments()
+ input_file = args.input_file
+ file_sz = os.path.getsize(input_file)
+ cnt = args.loop
+
+ t = timeit.Timer(functools.partial(file_read, input_file))
+ py_t = t.timeit(cnt)
+ py_gbs = (cnt*file_sz)/GIGA_UNIT/py_t
+ print(f'py load_gpu: {file_sz/GIGA_UNIT} GB, {py_t/cnt} secs, {py_gbs:5.2f} GB/sec')
+
+if __name__ == "__main__":
+ main()
diff --git a/deepnvme/file_access/py_store_cpu_tensor.py b/deepnvme/file_access/py_store_cpu_tensor.py
new file mode 100644
index 000000000..50e477186
--- /dev/null
+++ b/deepnvme/file_access/py_store_cpu_tensor.py
@@ -0,0 +1,26 @@
+import torch
+import os, timeit, functools
+import pathlib
+from utils import parse_write_arguments, GIGA_UNIT
+
+def file_write(out_f, tensor):
+ with open(out_f, 'wb') as f:
+ f.write(tensor.numpy(force=True))
+
+def main():
+ args = parse_write_arguments()
+ cnt = args.loop
+ output_file = os.path.join(args.nvme_folder, f'test_ouput_{args.mb_size}MB.pt')
+ pathlib.Path(output_file).unlink(missing_ok=True)
+ file_sz = args.mb_size*(1024**2)
+ cpu_tensor = torch.empty(file_sz, dtype=torch.uint8, device='cpu', requires_grad=False)
+
+ t = timeit.Timer(functools.partial(file_write, output_file, cpu_tensor))
+
+ py_t = t.timeit(cnt)
+ py_gbs = (cnt*file_sz)/GIGA_UNIT/py_t
+ print(f'py store_cpu: {file_sz/GIGA_UNIT} GB, {py_t/cnt} secs, {py_gbs:5.2f} GB/sec')
+ pathlib.Path(output_file).unlink(missing_ok=True)
+
+if __name__ == "__main__":
+ main()
diff --git a/deepnvme/file_access/py_store_gpu_tensor.py b/deepnvme/file_access/py_store_gpu_tensor.py
new file mode 100644
index 000000000..a64209a12
--- /dev/null
+++ b/deepnvme/file_access/py_store_gpu_tensor.py
@@ -0,0 +1,27 @@
+import torch
+import os, timeit, functools
+import pathlib
+from utils import parse_write_arguments, GIGA_UNIT
+
+def file_write(out_f, tensor):
+ with open(out_f, 'wb') as f:
+ f.write(tensor.numpy(force=True))
+
+def main():
+ args = parse_write_arguments()
+ cnt = args.loop
+ output_file = os.path.join(args.nvme_folder, f'test_ouput_{args.mb_size}MB.pt')
+ pathlib.Path(output_file).unlink(missing_ok=True)
+ file_sz = args.mb_size*(1024**2)
+ gpu_tensor = torch.empty(file_sz, dtype=torch.uint8, device='cuda', requires_grad=False)
+
+ t = timeit.Timer(functools.partial(file_write, output_file, gpu_tensor))
+
+ py_t = t.timeit(cnt)
+ py_gbs = (cnt*file_sz)/GIGA_UNIT/py_t
+ print(f'py store_gpu: {file_sz/GIGA_UNIT} GB, {py_t/cnt} secs, {py_gbs:5.2f} GB/sec')
+ pathlib.Path(output_file).unlink(missing_ok=True)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/deepnvme/file_access/run_load_tensor.sh b/deepnvme/file_access/run_load_tensor.sh
new file mode 100644
index 000000000..e410c98b9
--- /dev/null
+++ b/deepnvme/file_access/run_load_tensor.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+
+if [[ $# -ne 1 ]]; then
+ echo "Usage: $0 "
+ exit 1
+fi
+
+input_file=$1
+if ! [[ -f "$input_file" ]]; then
+ echo "Error: $input_file does not exist"
+ exit 1
+fi
+
+
+echo "Running load tensor examples using $input_file"
+for f in aio_load_cpu_tensor.py aio_load_gpu_tensor.py \
+ gds_load_gpu_tensor.py \
+ py_load_cpu_tensor.py py_load_gpu_tensor.py; do
+ cmd="python $f --input_file $input_file"
+ sync
+ echo $cmd
+ eval $cmd
+ sleep 2
+done
+
+
diff --git a/deepnvme/file_access/run_store_tensor.sh b/deepnvme/file_access/run_store_tensor.sh
new file mode 100644
index 000000000..a10b3c219
--- /dev/null
+++ b/deepnvme/file_access/run_store_tensor.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+
+if [[ $# -ne 1 ]]; then
+ echo "Usage: $0