From beee53e161f19a117d8d2b0a88ae5246eb0719a3 Mon Sep 17 00:00:00 2001 From: Zhangmei Li Date: Sun, 19 Aug 2018 10:40:29 +0800 Subject: [PATCH] speed up tinkerpop test how to: 1.speed up rocksdb backend by truncating tables 2.don't need to clear if the database is empty 3.don't need to clear variables if it's empty 4.don't need to commit hbase each len-prefix for delete index-label potential problems: 1.mysql may block if create task vertex-label when truncating table 2.rocksdb may miss CF if create task vertex-label when truncating table 3.hbase may block if truncate hbase with version<2.0 improve #14 Change-Id: I4b2393aea8b0fc63c1886e984a576a1f5808b25c --- .../com/baidu/hugegraph/api/GraphsAPI.java | 25 +---- .../hugegraph/auth/HugeGraphAuthProxy.java | 5 + .../store/cassandra/CassandraSessionPool.java | 4 + .../store/cassandra/CassandraStore.java | 68 ++++++++---- .../store/cassandra/CassandraTable.java | 17 ++- .../java/com/baidu/hugegraph/HugeGraph.java | 41 ++++++- .../baidu/hugegraph/backend/LocalCounter.java | 4 + .../backend/cache/CachedBackendStore.java | 5 + .../cache/CachedSchemaTransaction.java | 88 ++++++++++----- .../backend/serializer/BinarySerializer.java | 1 + .../store/AbstractBackendStoreProvider.java | 38 ++++++- .../hugegraph/backend/store/BackendStore.java | 3 + .../backend/store/BackendStoreProvider.java | 4 + .../backend/store/memory/InMemoryDBStore.java | 35 ++++-- .../store/memory/InMemoryDBStoreProvider.java | 13 +++ .../store/memory/InMemoryDBTables.java | 9 +- .../backend/tx/GraphTransaction.java | 3 +- .../baidu/hugegraph/config/CoreOptions.java | 24 ++-- .../hugegraph/job/schema/SchemaCallable.java | 1 + .../baidu/hugegraph/task/TaskScheduler.java | 105 +++++++++++++----- .../optimize/HugeGraphStepStrategy.java | 2 +- .../optimize/HugeVertexStepStrategy.java | 2 +- .../traversal/optimize/TraversalUtil.java | 20 ++-- .../java/com/baidu/hugegraph/util/Events.java | 1 + .../backend/store/hbase/HbaseSessions.java | 14 +++ .../backend/store/hbase/HbaseStore.java | 22 ++++ .../backend/store/hbase/HbaseTables.java | 2 +- .../backend/store/mysql/MysqlStore.java | 51 +++++---- .../store/mysql/MysqlStoreProvider.java | 2 +- .../backend/store/mysql/MysqlTable.java | 35 ++++-- .../backend/store/palo/PaloStore.java | 2 +- .../backend/store/rocksdb/RocksDBStore.java | 14 ++- .../backend/store/rocksdb/RocksDBTables.java | 38 +++---- .../store/scylladb/ScyllaDBTables.java | 85 ++++++++++---- .../store/scylladb/ScyllaDBTablesWithMV.java | 16 ++- .../baidu/hugegraph/tinkerpop/TestGraph.java | 36 ++++++ .../tinkerpop/TestGraphProvider.java | 63 ++++++----- 37 files changed, 645 insertions(+), 253 deletions(-) diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/GraphsAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/GraphsAPI.java index 8c1f2e3c9e..989b5b1e9b 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/GraphsAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/GraphsAPI.java @@ -36,13 +36,10 @@ import javax.ws.rs.core.Context; import javax.ws.rs.core.SecurityContext; -import org.apache.tinkerpop.gremlin.structure.Edge; -import org.apache.tinkerpop.gremlin.structure.Vertex; import org.slf4j.Logger; import com.baidu.hugegraph.HugeGraph; import com.baidu.hugegraph.core.GraphManager; -import com.baidu.hugegraph.schema.SchemaManager; import com.baidu.hugegraph.server.RestServer; import com.baidu.hugegraph.util.Log; import com.codahale.metrics.annotation.Timed; @@ -127,27 +124,7 @@ public void clear(@Context GraphManager manager, throw new IllegalArgumentException(String.format( "Please take the message: %s", CONFIRM_CLEAR)); } - - // Clear vertex and edge - commit(g, () -> { - g.traversal().E().toStream().forEach(Edge::remove); - g.traversal().V().toStream().forEach(Vertex::remove); - }); - - // Schema operation will auto commit - SchemaManager schema = g.schema(); - schema.getIndexLabels().forEach(elem -> { - schema.indexLabel(elem.name()).remove(); - }); - schema.getEdgeLabels().forEach(elem -> { - schema.edgeLabel(elem.name()).remove(); - }); - schema.getVertexLabels().forEach(elem -> { - schema.vertexLabel(elem.name()).remove(); - }); - schema.getPropertyKeys().forEach(elem -> { - schema.propertyKey(elem.name()).remove(); - }); + g.truncateBackend(); } @PUT diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/auth/HugeGraphAuthProxy.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/auth/HugeGraphAuthProxy.java index d8fb44e20d..931a8e860b 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/auth/HugeGraphAuthProxy.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/auth/HugeGraphAuthProxy.java @@ -168,6 +168,11 @@ public void clearBackend() { this.hugegraph.clearBackend(); } + public void truncateBackend() { + this.verifyPermission(ROLE_ADMIN); + this.hugegraph.truncateBackend(); + } + public void restoring(boolean restoring) { this.verifyPermission(ROLE_ADMIN); this.hugegraph.restoring(restoring); diff --git a/hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraSessionPool.java b/hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraSessionPool.java index b80c1481a4..5a2d8457de 100644 --- a/hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraSessionPool.java +++ b/hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraSessionPool.java @@ -222,6 +222,10 @@ public void open() { this.session = cluster().connect(keyspace()); } + public boolean opened() { + return this.session != null; + } + @Override public boolean closed() { if (this.session == null) { diff --git a/hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraStore.java b/hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraStore.java index 669f52be1d..42dce44a45 100644 --- a/hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraStore.java +++ b/hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraStore.java @@ -19,8 +19,11 @@ package com.baidu.hugegraph.backend.store.cassandra; +import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -84,7 +87,7 @@ public CassandraStore(final BackendStoreProvider provider, private void registerMetaHandlers() { this.registerMetaHandler("metrics", (session, meta, args) -> { - CassandraMetrics metrics = new CassandraMetrics(cluster(), conf); + CassandraMetrics metrics = new CassandraMetrics(cluster(), this.conf); return metrics.getMetrics(); }); } @@ -243,12 +246,25 @@ public BackendFeatures features() { @Override public void init() { this.checkClusterConnected(); - this.initKeyspace(); + if (this.sessions.session().opened()) { + // Session has ever been opened. + LOG.warn("Session has ever been opened(exist keyspace '{}' before)", + this.keyspace); + } else { + // Create keyspace if needed + if (!this.existsKeyspace()) { + this.initKeyspace(); + } + // Open session explicitly to get the exception when it fails + this.sessions.session().open(); + } + + // Create tables this.checkSessionConnected(); this.initTables(); - LOG.info("Store initialized: {}", this.store); + LOG.debug("Store initialized: {}", this.store); } @Override @@ -261,7 +277,15 @@ public void clear() { this.clearKeyspace(); } - LOG.info("Store cleared: {}", this.store); + LOG.debug("Store cleared: {}", this.store); + } + + @Override + public void truncate() { + this.checkSessionConnected(); + + this.truncateTables(); + LOG.debug("Store truncated: {}", this.store); } @Override @@ -359,7 +383,8 @@ protected void initKeyspace() { replication.putIfAbsent("replication_factor", factor); Statement stmt = SchemaBuilder.createKeyspace(this.keyspace) - .ifNotExists().with().replication(replication); + .ifNotExists().with() + .replication(replication); // Create keyspace with non-keyspace-session LOG.debug("Create keyspace: {}", stmt); @@ -371,7 +396,6 @@ protected void initKeyspace() { session.close(); } } - this.sessions.session().open(); } protected void clearKeyspace() { @@ -395,18 +419,29 @@ protected boolean existsKeyspace() { protected void initTables() { CassandraSessionPool.Session session = this.sessions.session(); - for (CassandraTable table : this.tables.values()) { + for (CassandraTable table : this.tables()) { table.init(session); } } protected void clearTables() { CassandraSessionPool.Session session = this.sessions.session(); - for (CassandraTable table : this.tables.values()) { + for (CassandraTable table : this.tables()) { table.clear(session); } } + protected void truncateTables() { + CassandraSessionPool.Session session = this.sessions.session(); + for (CassandraTable table : this.tables()) { + table.truncate(session); + } + } + + protected Collection tables() { + return this.tables.values(); + } + @Override protected final CassandraTable table(HugeType type) { assert type != null; @@ -472,19 +507,10 @@ public CassandraSchemaStore(BackendStoreProvider provider, } @Override - protected void initTables() { - super.initTables(); - - CassandraSessionPool.Session session = super.sessions.session(); - this.counters.init(session); - } - - @Override - protected void clearTables() { - super.clearTables(); - - CassandraSessionPool.Session session = super.sessions.session(); - this.counters.clear(session); + protected Collection tables() { + List tables = new ArrayList<>(super.tables()); + tables.add(this.counters); + return tables; } @Override diff --git a/hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraTable.java b/hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraTable.java index fe9f010849..db55235042 100644 --- a/hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraTable.java +++ b/hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraTable.java @@ -119,13 +119,13 @@ protected List