diff --git a/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/dbms/index/diskann/graph/AbstractDynamicExplorationGraph.kt b/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/dbms/index/diskann/graph/AbstractDynamicExplorationGraph.kt index f0f0940c6..b9aaadd3a 100644 --- a/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/dbms/index/diskann/graph/AbstractDynamicExplorationGraph.kt +++ b/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/dbms/index/diskann/graph/AbstractDynamicExplorationGraph.kt @@ -1,16 +1,16 @@ package org.vitrivr.cottontail.dbms.index.diskann.graph -import it.unimi.dsi.fastutil.longs.Long2ObjectArrayMap -import jetbrains.exodus.core.dataStructures.hash.LongHashSet +import it.unimi.dsi.fastutil.objects.Object2FloatLinkedOpenHashMap +import org.apache.lucene.search.Weight import org.vitrivr.cottontail.core.database.TupleId import org.vitrivr.cottontail.core.types.VectorValue +import org.vitrivr.cottontail.utilities.graph.Graph +import java.lang.Math.floorDiv import java.util.* +import kotlin.collections.HashMap +import kotlin.collections.HashSet import kotlin.math.max -typealias NodeId = Long - -typealias Weight = Double - /** * This class implements a Dynamic Exploration Graph (DEG) as proposed in [1]. It can be used to perform approximate nearest neighbour search (ANNS). * @@ -20,7 +20,7 @@ typealias Weight = Double * @author Ralph Gasser * @version 1.0.0 */ -abstract class AbstractDynamicExplorationGraph(val degree: Int): Iterable.Node>> { +abstract class AbstractDynamicExplorationGraph(private val degree: Int, val graph: Graph.Node>) { init { @@ -35,53 +35,60 @@ abstract class AbstractDynamicExplorationGraph(val degree: Int): Iterable

() var skipRng = false - /* Start insert procedure (. */ - while (nearest.size < this.degree) { - val nodesToExplore = nearest.entries.filter { !nearest.containsKey(it.key) }.associate { it.key to it.value.first }.toMutableMap() - while (nearest.size < this.degree && nodesToExplore.isNotEmpty()) { - var closestNodeId = nodesToExplore.keys.first() - var closestNode = nodesToExplore.values.first() + /* Start insert procedure. */ + while (connect.size < this.degree) { + val nodesToExplore = search.entries.filter { !connect.contains(it.key) }.associate { it.key to it.value }.toMutableMap() + while (connect.size < this.degree && nodesToExplore.isNotEmpty()) { + var closestNode = nodesToExplore.keys.first() var closestDistance = Double.MAX_VALUE - for ((nodeId, node) in nodesToExplore.entries) { + for ((node, _) in nodesToExplore.entries) { val distance = this.distance(vector, node.vector) if (distance < closestDistance) { closestDistance = distance - closestNodeId = nodeId closestNode = node } - nodesToExplore.remove(closestNodeId) + } + nodesToExplore.remove(closestNode) - /* Identify the best vertex to connect to existing vertex. */ - if (skipRng || checkMrng(newNode, closestNode)) { - val longestEdge = closestNode.neighbours.entries.filter { newNode.neighbours.containsKey(it.key) }.maxBy { it.value } - newNode.addEdge(longestEdge.key, distance) - newNode.addEdge(closestNodeId, closestDistance) + /* Identify the best vertex to connect to existing vertex. */ + if (skipRng || checkMrng(newNode, connect, closestNode)) { + val farthestNodeFromClosest = this.graph.edges(closestNode).filter { !connect.contains(it.key) }.maxBy { it.value }.key + connect[closestNode] = this.distance(closestNode.vector, newNode.vector).toFloat() + connect[farthestNodeFromClosest] = this.distance(farthestNodeFromClosest.vector, newNode.vector).toFloat() - /* Update receiving node. */ - closestNode.removeEdge(longestEdge.key) - storeNode(closestNodeId, closestNode) - } + /* Update receiving node. */ + this.graph.removeEdge(farthestNodeFromClosest, closestNode) } } skipRng = true } - } - /* Store new node. */ - this.storeNode(newNodeId, newNode) + /* */ + this.graph.addVertex(newNode) + for ((node, weight) in connect) { + this.graph.addEdge(newNode, node, weight) + } + } } /** @@ -92,29 +99,27 @@ abstract class AbstractDynamicExplorationGraph(val degree: Int): Iterable

> { - val seed = this.getSeedNodes() - val checked = LongHashSet() - var r = Double.MAX_VALUE + fun search(query: V, k: Int, epsilon: Double): Map { + val seed = this.getSeedNodes(this.degree) + val checked = HashSet() + var r = Float.MAX_VALUE /* Results. */ - val results = Long2ObjectArrayMap>(k + 1) + val results = Object2FloatLinkedOpenHashMap(k + 1) /* Perform search. */ while (seed.isNotEmpty()) { /* Find seed node closest to query. */ - var closestNodeId = seed.keys.first() - var closestNode: Node = seed.values.first() + var closestNode: Node = seed.first() var closestDistance = Double.MAX_VALUE - for ((id, node) in seed) { + for (node in seed) { val distance = this.distance(query, node.vector) if (distance < closestDistance) { closestDistance = distance - closestNodeId = id closestNode = node } } - seed.remove(closestNodeId) + seed.remove(closestNode) /* Abort condition. */ if (closestDistance > r * (1 + epsilon)) { @@ -122,24 +127,23 @@ abstract class AbstractDynamicExplorationGraph(val degree: Int): Iterable

k) { - val largest = results.long2ObjectEntrySet().maxBy { it.value.second } - results.remove(largest.longKey) - r = largest.value.second + val largest = results.maxBy { it.value } + results.removeFloat(largest.key) + r = largest.value } } } /* Add node ID to set of checked nodes. */ - checked.add(nodeId) + checked.add(node) } } } @@ -147,33 +151,6 @@ abstract class AbstractDynamicExplorationGraph(val degree: Int): Iterable

> = NodeIterator() - - /** - * Stores the [Node] with the given [NodeId] - * - * @param nodeId The [NodeId] of the [Node] to return. - * @param node The [Node] to store. - * @throws NoSuchElementException If [Node] with [NodeId] doesn't exist. - */ - protected abstract fun storeNode(nodeId: NodeId, node: Node) - - /** - * Returns the [Node] with the given [NodeId] - * - * @param nodeId The [NodeId] of the [Node] to return. - * @return [Node] - * @throws NoSuchElementException If [Node] with [NodeId] doesn't exist. - */ - protected abstract fun getNode(nodeId: NodeId): Node - /** * Returns the size of this [AbstractDynamicExplorationGraph]. * @@ -189,6 +166,8 @@ abstract class AbstractDynamicExplorationGraph(val degree: Int): Iterable

(val degree: Int): Iterable

{ - val map = Long2ObjectArrayMap() - val random = SplittableRandom() - (0 until size).map { - while (true) { - val nextNodeId = random.nextLong(0L, this.size()) - val nextNode = this.getNode(nextNodeId) - if (map.putIfAbsent(nextNodeId, nextNode) != null) { - break - } + private fun getSeedNodes(size: Int): MutableSet { + require(size <= this.size()) { "Negative size of $size" } + val set = HashSet() + for ((i, node) in this.graph.withIndex()) { + if (i % floorDiv(this.graph.size(), size.toLong()) == 0L) { + set.add(node) } + if (set.size >= size) break } - return map + return set } /** @@ -226,11 +202,12 @@ abstract class AbstractDynamicExplorationGraph(val degree: Int): Iterable

, v2: Node): Boolean { + val v2N = this.graph.edges(v2) + val neighbours = v1N.keys intersect v2N.keys val distance = this.distance(v1.vector, v2.vector) - for (nodeId in neighbours) { - if (distance > max(v2.neighbours[nodeId] ?: 0.0, v1.neighbours[nodeId] ?: 0.0)) { + for (node in neighbours) { + if (distance > max(v2N[node] ?: 0.0f, v1N[node] ?: 0.0f)) { return false } } @@ -243,43 +220,10 @@ abstract class AbstractDynamicExplorationGraph(val degree: Int): Iterable

) { + inner class Node(val identifier: I) { /** The [VectorValue]; this value is loaded lazily. */ val vector: V by lazy { loadVector(this.identifier) } - - /** The neighbours of this [Node]. */ - val neighbours: Map - get() = this._edges.toMap() - - /** - * Adds a new edge to this [Node]. - * - * @param nodeId The [NodeId] of the - */ - fun addEdge(nodeId: NodeId, weight: Weight) { - require(this._edges.size < this@AbstractDynamicExplorationGraph.degree) { "Node contains to many edges (maximum degree is ${this@AbstractDynamicExplorationGraph.degree})." } - require(nodeId > 0 && nodeId < this@AbstractDynamicExplorationGraph.size()) { "NodeId $nodeId is out-of-bounds (maximum size = ${size()})." } - this._edges[nodeId] = weight - } - - /** - * Removes an edge from this [Node]. - * - * @param nodeId The [NodeId] of the edge to remove. - */ - fun removeEdge(nodeId: NodeId) { - this._edges.remove(nodeId) - } - } - - /** - * Returns an [Iterator] over the [Node]s in this [AbstractDynamicExplorationGraph]. - * - * Important: This is a fairly naive implementation that could be improved in concrete implementations. - */ - inner class NodeIterator: Iterator> { - private var current: NodeId = 0L - override fun hasNext(): Boolean = this.current < this@AbstractDynamicExplorationGraph.size() - override fun next(): Pair = this.current to this@AbstractDynamicExplorationGraph.getNode(this.current++) + override fun equals(other: Any?): Boolean = other is AbstractDynamicExplorationGraph<*,*>.Node && other.identifier == this.identifier + override fun hashCode(): Int = this.identifier.hashCode() } } \ No newline at end of file diff --git a/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/dbms/index/diskann/graph/InMemoryDynamicExplorationGraph.kt b/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/dbms/index/diskann/graph/InMemoryDynamicExplorationGraph.kt new file mode 100644 index 000000000..9e23d8d0c --- /dev/null +++ b/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/dbms/index/diskann/graph/InMemoryDynamicExplorationGraph.kt @@ -0,0 +1,19 @@ +package org.vitrivr.cottontail.dbms.index.diskann.graph + +import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap +import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap +import org.vitrivr.cottontail.utilities.graph.Graph +import org.vitrivr.cottontail.utilities.graph.memory.InMemoryGraph + +/** + * + */ +class InMemoryDynamicExplorationGraph(degree: Int, private val df: (V, V) -> Double): AbstractDynamicExplorationGraph(degree, InMemoryGraph(degree)) { + private val vectors = Object2ObjectOpenHashMap() + override fun size(): Long = this.graph.size() + override fun distance(a: V, b: V): Double = this.df(a, b) + override fun loadVector(identifier: I): V = this.vectors[identifier] ?: throw NoSuchElementException("Could not find vector for identifier $identifier") + override fun storeVector(identifier: I, vector: V) { + this.vectors[identifier] = vector + } +} \ No newline at end of file diff --git a/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/utilities/graph/Graph.kt b/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/utilities/graph/Graph.kt new file mode 100644 index 000000000..8b30b5dcc --- /dev/null +++ b/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/utilities/graph/Graph.kt @@ -0,0 +1,65 @@ +package org.vitrivr.cottontail.utilities.graph + +/** + * A [Graph] data structure on elements of type [V] and weighted edges. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +interface Graph: Iterable { + /** + * Returns the number of vertexes in this [Graph]. + * + * @return Number of vertexes in this [Graph]. + */ + fun size(): Long + + /** + * + */ + fun edges(v: V): Map + + /** + * Adds a new vertex of type [V] to this [Graph]. + * + * @param v The vertex [V] to add. + * @return True on success, false otherwise. + */ + fun addVertex(v: V): Boolean + + /** + * Removes a vertex of type [V] from this [Graph]. + * + * @param v The vertex [V] to remove. + * @return True on success, false otherwise. + */ + fun removeVertex(v: V): Boolean + + /** + * Adds an edge between two vertices to this [Graph] + * + * @param from The vertex [V] to start the edge at. + * @param to The vertex [V] to end the edg at. + * @return True on success, false otherwise. + */ + fun addEdge(from: V, to: V): Boolean = addEdge(from, to, 0.0f) + + /** + * Adds an edge between two vertices to this [Graph] + * + * @param from The vertex [V] to start the edge at. + * @param to The vertex [V] to end the edg at. + * @param weight The weight of the edge. + * @return True on success, false otherwise. + */ + fun addEdge(from: V, to: V, weight: Float): Boolean + + /** + * Removes an edge between two vertices to this [Graph] + * + * @param from The start vertex [V]. + * @param to The end vertex [V]. + * @return True on success, false otherwise. + */ + fun removeEdge(from: V, to: V): Boolean +} \ No newline at end of file diff --git a/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/utilities/graph/memory/InMemoryGraph.kt b/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/utilities/graph/memory/InMemoryGraph.kt new file mode 100644 index 000000000..5a65ce674 --- /dev/null +++ b/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/utilities/graph/memory/InMemoryGraph.kt @@ -0,0 +1,94 @@ +package org.vitrivr.cottontail.utilities.graph.memory + +import it.unimi.dsi.fastutil.objects.Object2FloatOpenHashMap +import it.unimi.dsi.fastutil.objects.Object2ObjectLinkedOpenHashMap +import org.vitrivr.cottontail.utilities.graph.Graph +import kotlin.math.max +import kotlin.math.sign + +/** + * An in memory implementation of the [Graph] interface. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +class InMemoryGraph(val maxDegree: Int = Int.MAX_VALUE): Graph { + + /** An [Object2ObjectLinkedOpenHashMap] used as adjacency list for this [InMemoryGraph]. */ + private val map = Object2ObjectLinkedOpenHashMap>() + + /** + * Adds a new vertex of type [V] to this [InMemoryGraph]. + * + * @param v The vertex [V] to add. + * @return True on success, false otherwise. + */ + override fun addVertex(v: V): Boolean { + if (!this.map.containsKey(v)) { + this.map[v] = Object2FloatOpenHashMap() + return true + } + return false + } + + /** + * Removes a vertex of type [V] from this [InMemoryGraph]. + * + * @param v The vertex [V] to remove. + * @return True on success, false otherwise. + */ + override fun removeVertex(v: V): Boolean = (this.map.remove(v) != null) + + + /** + * Adds an edge between two vertices to this [Graph] + * + * @param from The vertex [V] to start the edge at. + * @param to The vertex [V] to end the edg at. + * @param weight The weight of the edge. + * @return True on success, false otherwise. + */ + override fun addEdge(from: V, to: V, weight: Float): Boolean { + val e1 = this.map[from] ?: throw NoSuchElementException("The vertex $from does not exist in the graph." ) + val e2 = this.map[to] ?: throw NoSuchElementException("The vertex $to does not exist in the graph." ) + if (!e1.containsKey(to) && !e2.containsKey(from)) { + check(e1.size <= this.maxDegree) { "The vertex $from already has too many edges (maxDegree = ${this.maxDegree})." } + check(e2.size <= this.maxDegree) { "The vertex $from already has too many edges (maxDegree = ${this.maxDegree})." } + e1[to] = weight + e2[from] = weight + return true + } + return false + } + + /** + * Removes an edge between two vertices to this [Graph] + * + * @param from The start vertex [V]. + * @param to The end vertex [V]. + * @return True on success, false otherwise. + */ + override fun removeEdge(from: V, to: V): Boolean { + val e1 = this.map[from] ?: throw NoSuchElementException("The vertex $from does not exist in the graph." ) + val e2 = this.map[to] ?: throw NoSuchElementException("The vertex $to does not exist in the graph." ) + if (e1.containsKey(to) && e2.containsKey(from)) { + e1.removeFloat(to) + e2.removeFloat(from) + return true + } + return false + } + + /** + * Returns the number of vertexes in this [Graph]. + * + * @return Number of vertexes in this [Graph]. + */ + override fun size(): Long = this.map.size.toLong() + override fun edges(from: V): Map = this.map[from] ?: throw NoSuchElementException("The vertex $from does not exist in the graph.") + + /** + * + */ + override fun iterator(): Iterator = this.map.keys.iterator() +} \ No newline at end of file