From cae2da7491cb6cba3442a3b02284956d63107da6 Mon Sep 17 00:00:00 2001 From: Zhangmei Li Date: Wed, 26 Jan 2022 21:11:08 +0800 Subject: [PATCH] improve CachedGraphTransaction perf Change-Id: I9f60f68d2faedb099e89adc2b1c1f4948d177a40 --- .../hugegraph/backend/cache/CacheManager.java | 6 ++ .../backend/cache/CachedGraphTransaction.java | 85 ++++++++++++++----- .../hugegraph/backend/query/QueryResults.java | 1 - 3 files changed, 69 insertions(+), 23 deletions(-) diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/CacheManager.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/CacheManager.java index f3e0f69444..7a88459b2b 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/CacheManager.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/CacheManager.java @@ -111,6 +111,8 @@ public Cache cache(String name) { public Cache cache(String name, long capacity) { if (!this.caches.containsKey(name)) { this.caches.putIfAbsent(name, new RamCache(capacity)); + LOG.info("Init RamCache for '{}' with capacity {}", + name, capacity); } @SuppressWarnings("unchecked") Cache cache = (Cache) this.caches.get(name); @@ -124,6 +126,8 @@ public Cache offheapCache(HugeGraph graph, String name, if (!this.caches.containsKey(name)) { OffheapCache cache = new OffheapCache(graph, capacity, avgElemSize); this.caches.putIfAbsent(name, cache); + LOG.info("Init OffheapCache for '{}' with capacity {}", + name, capacity); } @SuppressWarnings("unchecked") Cache cache = (Cache) this.caches.get(name); @@ -140,6 +144,8 @@ public Cache levelCache(HugeGraph graph, String name, OffheapCache cache2 = new OffheapCache(graph, capacity2, avgElemSize); this.caches.putIfAbsent(name, new LevelCache(cache1, cache2)); + LOG.info("Init LevelCache for '{}' with capacity {}:{}", + name, capacity1, capacity2); } @SuppressWarnings("unchecked") Cache cache = (Cache) this.caches.get(name); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/CachedGraphTransaction.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/CachedGraphTransaction.java index 5889e56b88..7462ead734 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/CachedGraphTransaction.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/CachedGraphTransaction.java @@ -216,16 +216,51 @@ private void clearCache(HugeType type, boolean notify) { } } + private boolean enableCacheVertex() { + return this.verticesCache.capacity() > 0L; + } + + private boolean enableCacheEdge() { + return this.edgesCache.capacity() > 0L; + } + + private boolean needCacheVertex(HugeVertex vertex) { + return vertex.sizeOfSubProperties() <= MAX_CACHE_PROPS_PER_VERTEX; + } + @Override + @Watched(prefix = "graphcache") protected final Iterator queryVerticesFromBackend(Query query) { - if (!query.ids().isEmpty() && query.conditions().isEmpty()) { + if (this.enableCacheVertex() && + query.idsSize() > 0 && query.conditionsSize() == 0) { return this.queryVerticesByIds((IdQuery) query); } else { return super.queryVerticesFromBackend(query); } } + @Watched(prefix = "graphcache") private Iterator queryVerticesByIds(IdQuery query) { + if (query.idsSize() == 1) { + Id vertexId = query.ids().iterator().next(); + HugeVertex vertex = (HugeVertex) this.verticesCache.get(vertexId); + if (vertex != null) { + if (!vertex.expired()) { + return QueryResults.iterator(vertex); + } + this.verticesCache.invalidate(vertexId); + } + Iterator rs = super.queryVerticesFromBackend(query); + vertex = QueryResults.one(rs); + if (vertex == null) { + return QueryResults.emptyIterator(); + } + if (needCacheVertex(vertex)) { + this.verticesCache.update(vertex.id(), vertex); + } + return QueryResults.iterator(vertex); + } + IdQuery newQuery = new IdQuery(HugeType.VERTEX, query); List vertices = new ArrayList<>(); for (Id vertexId : query.ids()) { @@ -254,11 +289,10 @@ private Iterator queryVerticesByIds(IdQuery query) { // Generally there are not too much data with id query ListIterator listIterator = QueryResults.toList(rs); for (HugeVertex vertex : listIterator.list()) { - if (vertex.sizeOfSubProperties() > MAX_CACHE_PROPS_PER_VERTEX) { - // Skip large vertex - continue; + // Skip large vertex + if (needCacheVertex(vertex)) { + this.verticesCache.update(vertex.id(), vertex); } - this.verticesCache.update(vertex.id(), vertex); } results.extend(listIterator); } @@ -267,14 +301,15 @@ private Iterator queryVerticesByIds(IdQuery query) { } @Override - @Watched + @Watched(prefix = "graphcache") protected final Iterator queryEdgesFromBackend(Query query) { RamTable ramtable = this.params().ramtable(); if (ramtable != null && ramtable.matched(query)) { return ramtable.query(query); } - if (query.empty() || query.paging() || query.bigCapacity()) { + if (!this.enableCacheEdge() || query.empty() || + query.paging() || query.bigCapacity()) { // Query all edges or query edges in paging, don't cache it return super.queryEdgesFromBackend(query); } @@ -321,6 +356,7 @@ protected final Iterator queryEdgesFromBackend(Query query) { } @Override + @Watched(prefix = "graphcache") protected final void commitMutation2Backend(BackendMutation... mutations) { // Collect changes before commit Collection updates = this.verticesInTxUpdated(); @@ -333,28 +369,33 @@ protected final void commitMutation2Backend(BackendMutation... mutations) { try { super.commitMutation2Backend(mutations); // Update vertex cache - for (HugeVertex vertex : updates) { - vertexIds[vertexOffset++] = vertex.id(); - if (vertex.sizeOfSubProperties() > MAX_CACHE_PROPS_PER_VERTEX) { - // Skip large vertex - this.verticesCache.invalidate(vertex.id()); - continue; + if (this.enableCacheVertex()) { + for (HugeVertex vertex : updates) { + vertexIds[vertexOffset++] = vertex.id(); + if (needCacheVertex(vertex)) { + // Update cache + this.verticesCache.updateIfPresent(vertex.id(), vertex); + } else { + // Skip large vertex + this.verticesCache.invalidate(vertex.id()); + } } - this.verticesCache.updateIfPresent(vertex.id(), vertex); } } finally { // Update removed vertex in cache whatever success or fail - for (HugeVertex vertex : deletions) { - vertexIds[vertexOffset++] = vertex.id(); - this.verticesCache.invalidate(vertex.id()); - } - if (vertexOffset > 0) { - this.notifyChanges(Cache.ACTION_INVALIDED, - HugeType.VERTEX, vertexIds); + if (this.enableCacheVertex()) { + for (HugeVertex vertex : deletions) { + vertexIds[vertexOffset++] = vertex.id(); + this.verticesCache.invalidate(vertex.id()); + } + if (vertexOffset > 0) { + this.notifyChanges(Cache.ACTION_INVALIDED, + HugeType.VERTEX, vertexIds); + } } // Update edge cache if any edges change - if (edgesInTxSize > 0) { + if (edgesInTxSize > 0 && this.enableCacheEdge()) { // TODO: Use a more precise strategy to update the edge cache this.edgesCache.clear(); this.notifyChanges(Cache.ACTION_CLEARED, HugeType.EDGE, null); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/QueryResults.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/QueryResults.java index 10f47f6623..5e70f6ae31 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/QueryResults.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/QueryResults.java @@ -22,7 +22,6 @@ import java.util.Collection; import java.util.Collections; import java.util.Iterator; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.NoSuchElementException;