Add alaya pyglass

ann_benchmarks/algorithms/alaya/pyglass/.gitignore

@@ -0,0 +1,2 @@
+build
+.cache

ann_benchmarks/algorithms/alaya/pyglass/CMakeLists.txt

@@ -0,0 +1,26 @@
+cmake_minimum_required (VERSION 3.17)
+project(glass LANGUAGES CXX)
+
+
+add_library(glass INTERFACE)
+target_link_libraries(glass
+    INTERFACE
+)
+target_include_directories(glass INTERFACE .) 
+
+set(CXX_STANDARD 17)
+
+set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+
+# SET(CMAKE_CXX_FLAGS  "-Wall -Wextra -O0 -lrt -std=c++20 -march=native -fpic -fopenmp -ftree-vectorize -fno-exceptions -fno-rtti" )
+
+SET(CMAKE_CXX_FLAGS  "-Wextra -O3 -lrt -std=c++17 -march=native -fPIC -fopenmp -ftree-vectorize -fno-exceptions -fno-rtti" )
+
+add_executable(main main/main.cpp)
+target_link_libraries(main glass fmt::fmt)
+
+# add_executable(main examples/main.cc)
+# target_link_libraries(main glass)
+
+add_library(AlayaDB SHARED bind.cpp)
+target_link_libraries(AlayaDB glass)

ann_benchmarks/algorithms/alaya/pyglass/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 zh Wang
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

ann_benchmarks/algorithms/alaya/pyglass/README.md

@@ -0,0 +1,92 @@
+# Graph Library for Approximate Similarity Search
+
+pyglass is a library for fast inference of graph index for approximate similarity search.
+
+## Features
+
+- Supports multiple graph algorithms, like [**HNSW**](https://github.com/nmslib/hnswlib) and [**NSG**](https://github.com/ZJULearning/nsg).
+- Supports multiple hardware platforms, like **X86** and **ARM**. Support for **GPU** is on the way
+- No third-party library dependencies, does not rely on OpenBLAS / MKL or any other computing framework.
+- Sophisticated memory management and data structure design, very low memory footprint.
+- It's high performant.
+
+## Installation
+### Installation from Wheel
+pyglass can be installed using pip as follows:
+```bash
+pip3 install glassppy
+```
+
+### Installation from Source
+If there's some problem when installing from wheel, you can try to build from source.
+``` bash
+sudo apt-get update && sudo apt-get install -y build-essential git python3 python3-distutils python3-venv
+```
+``` bash
+pip3 install numpy
+pip3 install pybind11
+```
+``` bash
+bash build.sh
+```
+
+## Quick Tour
+A runnable demo is at [examples/demo.ipynb](https://github.com/zilliztech/pyglass/blob/master/examples/demo.ipynb). It's highly recommended to try it.
+
+## Usage
+**Import library**
+```python
+>>> import glassppy as glass
+```
+**Load Data**
+```python
+>>> n, d = 10000, 128
+>>> X = np.random.randn(n, d)
+>>> Y = np.random.randn(d)
+```
+**Create Index**
+pyglass supports **HNSW** and **NSG** index currently
+```python
+>>> index = glass.Index(index_type="HNSW", dim=d, metric="L2", R=32, L=50)
+>>> index = glass.Index(index_type="NSG", dim=d, metric="L2", R=32, L=50)
+```
+**Build Graph**
+```python
+>>> graph = index.build(X)
+```
+**Create Searcher**
+Searcher accepts `level` parameter as the optimization level. You can set `level` as `0` or `1` or `2`. The higher the level, the faster the searching, but it may cause unstable recall.
+```python
+>>> optimize_level = 2
+>>> searcher = glass.Searcher(graph=graph, data=X, metric="L2", level=optimize_level)
+>>> searcher.set_ef(32)
+```
+**(Optional) Optimize Searcher**
+```python
+>>> searcher.optimize()
+```
+**Searching**
+```python
+>>> ret = searcher.search(query=Y, k=10)
+>>> print(ret)
+```
+
+## Performance
+
+Glass is among one of the top performant ann algorithms on [ann-benchmarks](https://ann-benchmarks.com/)
+
+### fashion-mnist-784-euclidean
+![](docs/figures/fashion-mnist-784-euclidean_10_euclidean.png)
+### gist-960-euclidean
+![](docs/figures/gist-960-euclidean_10_euclidean.png)
+### sift-128-euclidean
+![](docs/figures/sift-128-euclidean_10_euclidean.png)
+
+### Quick Benchmark
+
+1. Change configuration file `examples/config.json`
+2. Run benchmark
+```
+python3 examples/main.py
+```
+3. You could check plots on `results` folder

ann_benchmarks/algorithms/alaya/pyglass/bind.cpp

@@ -0,0 +1,72 @@
+#include <glass/common.hpp>
+#include <glass/graph.hpp>
+#include <glass/hnsw/hnsw.hpp>
+#include <glass/neighbor.hpp>
+#include <glass/quant/quant.hpp>
+#include <glass/searcher.hpp>
+
+extern "C" {
+
+void *searcher;
+float *data;
+
+void fit(float *vec_data, int data_num, int data_dim) {
+  data = vec_data;
+  auto hnsw = (glass::HNSW *)new glass::HNSW(data_dim, "L2");
+  hnsw->Build(data, data_num);
+
+  // const std::string metric = "L2";
+  // int level = 3;
+
+  searcher = (glass::Searcher<glass::SQ8SymmetricQuantizer<glass::Metric::L2>>
+                  *)(new glass::Searcher<glass::SQ8SymmetricQuantizer<glass::Metric::L2>>(hnsw->GetGraph()));
+
+  auto hnsw_s = (glass::Searcher<glass::SQ8SymmetricQuantizer<glass::Metric::L2>> *)searcher;
+
+  // hnsw_s->SetEf(ef);
+
+  hnsw_s->SetData(data, data_num, data_dim);
+
+  hnsw_s->Optimize(1);
+}
+
+void batch_query(float *query, int query_num, int dim, int k, int ef, int rerank_k, unsigned *res) {
+  printf("init search\n");
+  std::vector<std::vector<int>> tmp_id(query_num, std::vector<int>(ef));
+  std::vector<glass::searcher::LPool<float>> res_pool(query_num, glass::searcher::LPool<float>(rerank_k));
+
+  printf("before search\n");
+
+  auto hnsw_s = (glass::Searcher<glass::SQ8SymmetricQuantizer<glass::Metric::L2>> *)searcher;
+  hnsw_s->SetEf(ef);
+  for (size_t q = 0; q < query_num; ++q) {
+    // printf("query id: %d\n", q);
+    auto &ids = tmp_id[q];
+    auto cur_query = query + q * dim;
+    hnsw_s->Search(cur_query, ef, ids.data());
+
+    for (size_t i = 0; i < k; ++i) {
+      res[q * k + i] = ids[i];
+    }
+
+    // printf("a search");
+
+    // printf("rerank: %d, k: %d, ef: %d\n", rerank_k, k, ef);
+    // for (size_t i = 0; i < rerank_k; ++i) {
+    //   res_pool[q].insert_back(ids[i], glass::L2SqrRef(cur_query, data + ids[i] * dim, dim));
+    // }
+    // res_pool[q].sorted();
+    // for (size_t i = rerank_k; i < ef; ++i) {
+    //   res_pool[q].emplace_insert(ids[i], glass::L2SqrRef(cur_query, data + ids[i] * dim, dim));
+    // }
+
+    // printf("res\n");
+    // for (size_t i = 0; i < k; ++i) {
+    //   res[q * k + i] = res_pool[q].data_[i].id;
+    //   printf("%d, ", res[q * k + i]);
+    // }
+    // printf("\n");
+  }
+}
+
+}  // extern C

ann_benchmarks/algorithms/alaya/pyglass/build.sh

@@ -0,0 +1,6 @@
+cd python
+rm -rf build
+python setup.py bdist_wheel
+pip uninstall glassppy -y
+cd dist
+ls | xargs pip install

ann_benchmarks/algorithms/alaya/pyglass/examples/config.json

@@ -0,0 +1,38 @@
+{
+  "datasets": [
+    "glove-100-angular",
+    "sift-128-euclidean",
+    "fashion-mnist-784-euclidean",
+    "nytimes-256-angular",
+    "glove-25-angular"
+  ],
+  "index_types": [
+    "HNSW",
+    "NSG"
+  ],
+  "R": 32,
+  "L": 200,
+  "levels": [
+    0,
+    1,
+    2
+  ],
+  "topk": 10,
+  "efs": [
+    10,
+    15,
+    20,
+    30,
+    40,
+    60,
+    80,
+    100,
+    150,
+    200,
+    300
+  ],
+  "runs": 5,
+  "batch": true,
+  "optimize": true,
+  "plot": true
+}

ann_benchmarks/algorithms/alaya/pyglass/examples/demo.ipynb

@@ -0,0 +1,127 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import glassppy as glass\n",
+    "from ann_dataset import dataset_dict"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "topk = 10\n",
+    "dataset = dataset_dict['sift-128-euclidean']()\n",
+    "X_train = dataset.get_database()\n",
+    "X_test = dataset.get_queries()\n",
+    "Y = dataset.get_groundtruth(topk)\n",
+    "n, d = X_train.shape\n",
+    "nq, d = X_test.shape\n",
+    "metric = dataset.metric\n",
+    "print(f\"n = {n}, d = {d}, nq = {nq}, metric = {metric}\")\n",
+    "print(f\"dataset size = {n * d * 4 / 1024 / 1024:.2f}MB\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "index = glass.Index(index_type=\"HNSW\", dim=d, metric=metric, R=32, L=100)\n",
+    "g = index.build(X_train)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "s = glass.Searcher(graph=g, data=X_train, metric=metric, level=2)\n",
+    "s.set_ef(36)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from os import cpu_count\n",
+    "from time import time\n",
+    "num_threads = cpu_count()\n",
+    "\n",
+    "pred = s.batch_search(query=X_test, k=topk, num_threads=num_threads).reshape(-1, topk)\n",
+    "recall = dataset.evaluate(pred, topk)\n",
+    "print(f\"Recall = {recall * 100:.2f}%\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "s.batch_search(query=X_test, k=topk, num_threads=num_threads)  # warmup\n",
+    "for iter in range(10):\n",
+    "    t1 = time()\n",
+    "    pred = s.batch_search(query=X_test, k=topk, num_threads=num_threads)\n",
+    "    t2 = time()\n",
+    "    print(f\"QPS = {nq / (t2 - t1):.2f}\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "s.optimize()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "s.batch_search(query=X_test, k=topk, num_threads=num_threads)  # warmup\n",
+    "for iter in range(10):\n",
+    "    t1 = time()\n",
+    "    pred = s.batch_search(query=X_test, k=topk,\n",
+    "                          num_threads=num_threads).reshape(-1, topk)\n",
+    "    t2 = time()\n",
+    "    print(f\"QPS = {nq / (t2 - t1)}\")\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "hy",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.3"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}