From b74b0a439bab332f07e8982c58b92c520614edff Mon Sep 17 00:00:00 2001 From: Wang XiangYu Date: Mon, 2 Mar 2020 19:10:13 +0800 Subject: [PATCH] 0.8+R@10 (#5) * u Signed-off-by: Nicky * 0.8+R@10 Co-authored-by: Tinkerrr Co-authored-by: Erik Bernhardsson --- .gitignore | 2 ++ ann_benchmarks/plotting/metrics.py | 18 ++++++++++++++++++ ann_benchmarks/plotting/plot_variants.py | 11 ++++++++++- ann_benchmarks/plotting/utils.py | 4 ++++ 4 files changed, 34 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 3161ecdbf..4ef712aa0 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,5 @@ install/*.yaml install/lib-*/ data/* *.class + +.idea/ diff --git a/ann_benchmarks/plotting/metrics.py b/ann_benchmarks/plotting/metrics.py index d16d74250..53e3c30e1 100644 --- a/ann_benchmarks/plotting/metrics.py +++ b/ann_benchmarks/plotting/metrics.py @@ -104,16 +104,34 @@ def dist_computations(queries, attrs): "worst": float("-inf"), "lim": [0.0, 1.03] }, + "k-nn_0.8": { + "description": "Recall 0.8+", + "function": lambda true_distances, run_distances, metrics, run_attrs: knn(true_distances, run_distances, run_attrs["count"], metrics).attrs['mean'], # noqa + "worst": float("-inf"), + "lim": [0.80, 1.03] + }, "epsilon": { "description": "Epsilon 0.01 Recall", "function": lambda true_distances, run_distances, metrics, run_attrs: epsilon(true_distances, run_distances, run_attrs["count"], metrics).attrs['mean'], # noqa "worst": float("-inf") }, + "epsilon_0.8": { + "description": "Epsilon 0.01 Recall 0.8+", + "function": lambda true_distances, run_distances, metrics, run_attrs: epsilon(true_distances, run_distances, run_attrs["count"], metrics).attrs['mean'], # noqa + "worst": float("-inf"), + "lim": [0.80, 1.03] + }, "largeepsilon": { "description": "Epsilon 0.1 Recall", "function": lambda true_distances, run_distances, metrics, run_attrs: epsilon(true_distances, run_distances, run_attrs["count"], metrics, 0.1).attrs['mean'], # noqa "worst": float("-inf") }, + "largeepsilon_0.8": { + "description": "Epsilon 0.1 Recall 0.8+", + "function": lambda true_distances, run_distances, metrics, run_attrs: epsilon(true_distances, run_distances, run_attrs["count"], metrics, 0.1).attrs['mean'], # noqa + "worst": float("-inf"), + "lim": [0.80, 1.03] + }, "rel": { "description": "Relative Error", "function": lambda true_distances, run_distances, metrics, run_attrs: rel(true_distances, run_distances, metrics), # noqa diff --git a/ann_benchmarks/plotting/plot_variants.py b/ann_benchmarks/plotting/plot_variants.py index f3632c00d..2e85a4820 100644 --- a/ann_benchmarks/plotting/plot_variants.py +++ b/ann_benchmarks/plotting/plot_variants.py @@ -9,5 +9,14 @@ "recall/candidates": ("k-nn", "candidates"), "recall/qpssize": ("k-nn", "queriessize"), "eps/time": ("epsilon", "qps"), - "largeeps/time": ("largeepsilon", "qps") + "largeeps/time": ("largeepsilon", "qps"), + + "recall_0.8/time": ("k-nn_0.8", "qps"), + "recall_0.8/buildtime": ("k-nn_0.8", "build"), + "recall_0.8/indexsize": ("k-nn_0.8", "indexsize"), + "recall_0.8/distcomps": ("k-nn_0.8", "distcomps"), + "recall_0.8/candidates": ("k-nn_0.8", "candidates"), + "recall_0.8/qpssize": ("k-nn_0.8", "queriessize"), + "eps_0.8/time": ("epsilon_0.8", "qps"), + "largeeps_0.8/time": ("largeepsilon_0.8", "qps") } diff --git a/ann_benchmarks/plotting/utils.py b/ann_benchmarks/plotting/utils.py index 31a1c73da..987c448f6 100644 --- a/ann_benchmarks/plotting/utils.py +++ b/ann_benchmarks/plotting/utils.py @@ -26,6 +26,10 @@ def create_pointset(data, xn, yn): for algo, algo_name, xv, yv in data: if not xv or not yv: continue + if "lim" in xm and not (xm["lim"][0] <= xv <= xm["lim"][1]): + continue + if "lim" in ym and not (ym["lim"][0] <= yv <= ym["lim"][1]): + continue axs.append(xv) ays.append(yv) als.append(algo_name)