diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/core/GraphManager.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/core/GraphManager.java index 90ef5426d4..090eb4c591 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/core/GraphManager.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/core/GraphManager.java @@ -42,7 +42,6 @@ import com.baidu.hugegraph.backend.cache.Cache; import com.baidu.hugegraph.backend.cache.CacheManager; import com.baidu.hugegraph.backend.store.BackendStoreSystemInfo; -import com.baidu.hugegraph.backend.store.memory.InMemoryDBStoreProvider; import com.baidu.hugegraph.config.HugeConfig; import com.baidu.hugegraph.config.ServerOptions; import com.baidu.hugegraph.exception.NotSupportException; @@ -189,7 +188,9 @@ private void loadGraph(String name, String path) { private void checkBackendVersionOrExit() { for (String graph : this.graphs()) { HugeGraph hugegraph = this.graph(graph); - if (InMemoryDBStoreProvider.matchType(hugegraph.backend())) { + boolean persistence = hugegraph.graphTransaction().store() + .features().supportsPersistence(); + if (!persistence) { hugegraph.initBackend(); } BackendStoreSystemInfo info = new BackendStoreSystemInfo(hugegraph); diff --git a/hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraSessionPool.java b/hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraSessionPool.java index 5a2d8457de..c4351b3657 100644 --- a/hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraSessionPool.java +++ b/hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraSessionPool.java @@ -104,12 +104,12 @@ public final synchronized Cluster cluster() { } @Override - public final synchronized Session session() { + public final Session session() { return (Session) super.getOrNewSession(); } @Override - protected final synchronized Session newSession() { + protected Session newSession() { E.checkState(this.cluster != null, "Cassandra cluster has not been initialized"); return new Session(); @@ -157,7 +157,7 @@ public BatchStatement add(Statement statement) { } @Override - public void clear() { + public void rollback() { this.batch.clear(); } diff --git a/hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraStore.java b/hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraStore.java index fdb8e86725..e7af77dfe8 100644 --- a/hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraStore.java +++ b/hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraStore.java @@ -356,7 +356,7 @@ public void rollbackTx() { session.txState(TxState.ROLLBACKING); try { - session.clear(); + session.rollback(); } finally { // Assume batch commit would auto rollback session.txState(TxState.CLEAN); diff --git a/hugegraph-core/pom.xml b/hugegraph-core/pom.xml index 01ec1f1520..a1d53b1847 100644 --- a/hugegraph-core/pom.xml +++ b/hugegraph-core/pom.xml @@ -19,7 +19,7 @@ com.baidu.hugegraph hugegraph-common - 1.5.6 + 1.5.8 diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/HugeGraph.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/HugeGraph.java index 7a4d96ada5..7143cc452e 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/HugeGraph.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/HugeGraph.java @@ -385,10 +385,6 @@ public Iterator vertices(Query query) { return this.graphTransaction().queryVertices(query); } - public Iterator adjacentVertices(Iterator edges) { - return this.graphTransaction().queryAdjacentVertices(edges); - } - @Override public Iterator edges(Object... objects) { if (objects.length == 0) { @@ -401,6 +397,14 @@ public Iterator edges(Query query) { return this.graphTransaction().queryEdges(query); } + public Iterator adjacentVertices(Iterator edges) { + return this.graphTransaction().queryAdjacentVertices(edges); + } + + public Iterator adjacentEdges(Id vertexId) { + return this.graphTransaction().queryEdgesByVertex(vertexId); + } + public PropertyKey propertyKey(Id id) { PropertyKey pk = this.schemaTransaction().getPropertyKey(id); E.checkArgument(pk != null, "Undefined property key id: '%s'", id); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/CachedGraphTransaction.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/CachedGraphTransaction.java index 1cca6bc2a3..e8e21942a3 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/CachedGraphTransaction.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/CachedGraphTransaction.java @@ -40,7 +40,7 @@ import com.baidu.hugegraph.type.HugeType; import com.google.common.collect.ImmutableList; -public class CachedGraphTransaction extends GraphTransaction { +public final class CachedGraphTransaction extends GraphTransaction { private final static int MAX_CACHE_EDGES_PER_QUERY = 100; diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/CachedSchemaTransaction.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/CachedSchemaTransaction.java index 9dc6467487..357f6b8f5f 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/CachedSchemaTransaction.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/CachedSchemaTransaction.java @@ -24,7 +24,6 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import java.util.function.Function; import com.baidu.hugegraph.HugeGraph; import com.baidu.hugegraph.backend.id.Id; @@ -40,7 +39,7 @@ import com.baidu.hugegraph.util.Events; import com.google.common.collect.ImmutableSet; -public class CachedSchemaTransaction extends SchemaTransaction { +public final class CachedSchemaTransaction extends SchemaTransaction { private final Cache idCache; private final Cache nameCache; @@ -153,12 +152,26 @@ private static Id generateId(HugeType type, String name) { return IdGenerator.of(type.string() + "-" + name); } - private Object getOrFetch(HugeType type, Id id, - Function fetcher) { + @Override + protected void addSchema(SchemaElement schema) { + super.addSchema(schema); + + this.resetCachedAllIfReachedCapacity(); + + Id prefixedId = generateId(schema.type(), schema.id()); + this.idCache.update(prefixedId, schema); + + Id prefixedName = generateId(schema.type(), schema.name()); + this.nameCache.update(prefixedName, schema); + } + + @Override + @SuppressWarnings("unchecked") + protected T getSchema(HugeType type, Id id) { Id prefixedId = generateId(type, id); Object value = this.idCache.get(prefixedId); if (value == null) { - value = fetcher.apply(id); + value = super.getSchema(type, id); if (value != null) { this.resetCachedAllIfReachedCapacity(); @@ -169,15 +182,17 @@ private Object getOrFetch(HugeType type, Id id, this.nameCache.update(prefixedName, schema); } } - return value; + return (T) value; } - private Object getOrFetch(HugeType type, String name, - Function fetcher) { + @Override + @SuppressWarnings("unchecked") + protected T getSchema(HugeType type, + String name) { Id prefixedName = generateId(type, name); Object value = this.nameCache.get(prefixedName); if (value == null) { - value = fetcher.apply(name); + value = super.getSchema(type, name); if (value != null) { this.resetCachedAllIfReachedCapacity(); @@ -188,36 +203,6 @@ private Object getOrFetch(HugeType type, String name, this.idCache.update(prefixedId, schema); } } - return value; - } - - @Override - protected void addSchema(SchemaElement schema) { - super.addSchema(schema); - - this.resetCachedAllIfReachedCapacity(); - - Id prefixedId = generateId(schema.type(), schema.id()); - this.idCache.update(prefixedId, schema); - - Id prefixedName = generateId(schema.type(), schema.name()); - this.nameCache.update(prefixedName, schema); - } - - @Override - @SuppressWarnings("unchecked") - protected T getSchema(HugeType type, Id id) { - Object value = this.getOrFetch(type, id, - k -> super.getSchema(type, id)); - return (T) value; - } - - @Override - @SuppressWarnings("unchecked") - protected T getSchema(HugeType type, - String name) { - Object value = this.getOrFetch(type, name, - k -> super.getSchema(type, name)); return (T) value; } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/RamCache.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/RamCache.java index 9b86d3327d..02dac63271 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/RamCache.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/RamCache.java @@ -51,6 +51,7 @@ public class RamCache implements Cache { // NOTE: the count in number of items, not in bytes private final int capacity; + private final int halfCapacity; // Implement LRU cache private final ConcurrentMap> map; @@ -63,14 +64,14 @@ public RamCache() { } public RamCache(int capacity) { - this.keyLock = new KeyLock(); - - if (capacity < 1) { - capacity = 1; + if (capacity < 0) { + capacity = 0; } + this.keyLock = new KeyLock(); this.capacity = capacity; + this.halfCapacity = this.capacity >> 1; - int initialCapacity = capacity >> 3; + int initialCapacity = capacity >= MB ? capacity >> 10 : 256; if (initialCapacity > MAX_INIT_CAP) { initialCapacity = MAX_INIT_CAP; } @@ -80,9 +81,18 @@ public RamCache(int capacity) { } @Watched(prefix = "ramcache") - private Object access(Id id) { + private final Object access(Id id) { assert id != null; + if (this.map.size() <= this.halfCapacity) { + LinkNode node = this.map.get(id); + if (node == null) { + return null; + } + assert id.equals(node.key()); + return node.value(); + } + final Lock lock = this.keyLock.lock(id); try { LinkNode node = this.map.get(id); @@ -91,7 +101,7 @@ private Object access(Id id) { } // NOTE: update the queue only if the size > capacity/2 - if (this.map.size() > this.capacity >> 1) { + if (this.map.size() > this.halfCapacity) { // Move the node from mid to tail if (this.queue.remove(node) == null) { // The node may be removed by others through dequeue() @@ -100,13 +110,6 @@ private Object access(Id id) { this.queue.enqueue(node); } - // Ignore concurrent write for hits - ++this.hits; - if (LOG.isDebugEnabled()) { - LOG.debug("RamCache cached '{}' (hits={}, miss={})", - id, this.hits, this.miss); - } - assert id.equals(node.key()); return node.value(); } finally { @@ -115,7 +118,7 @@ private Object access(Id id) { } @Watched(prefix = "ramcache") - private void write(Id id, Object value) { + private final void write(Id id, Object value) { assert id != null; assert this.capacity > 0; @@ -163,14 +166,13 @@ private void write(Id id, Object value) { // Add the new item to tail of the queue, then map it this.map.put(id, this.queue.enqueue(id, value)); - } finally { lock.unlock(); } } @Watched(prefix = "ramcache") - private void remove(Id id) { + private final void remove(Id id) { assert id != null; final Lock lock = this.keyLock.lock(id); @@ -192,16 +194,23 @@ private void remove(Id id) { @Override public Object get(Id id) { Object value = null; - if (this.map.containsKey(id)) { + if (this.map.size() <= this.halfCapacity || this.map.containsKey(id)) { // Maybe the id removed by other threads and returned null value value = this.access(id); } + if (value == null) { ++this.miss; if (LOG.isDebugEnabled()) { LOG.debug("RamCache missed '{}' (miss={}, hits={})", id, this.miss, this.hits); } + } else { + ++this.hits; + if (LOG.isDebugEnabled()) { + LOG.debug("RamCache cached '{}' (hits={}, miss={})", + id, this.hits, this.miss); + } } return value; } @@ -210,10 +219,11 @@ public Object get(Id id) { @Override public Object getOrFetch(Id id, Function fetcher) { Object value = null; - if (this.map.containsKey(id)) { + if (this.map.size() <= this.halfCapacity || this.map.containsKey(id)) { // Maybe the id removed by other threads and returned null value value = this.access(id); } + if (value == null) { ++this.miss; if (LOG.isDebugEnabled()) { @@ -223,6 +233,12 @@ public Object getOrFetch(Id id, Function fetcher) { // Do fetch and update the cache value = fetcher.apply(id); this.update(id, value); + } else { + ++this.hits; + if (LOG.isDebugEnabled()) { + LOG.debug("RamCache cached '{}' (hits={}, miss={})", + id, this.hits, this.miss); + } } return value; } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/id/IdGenerator.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/id/IdGenerator.java index 20ac48c1a5..1f1fa042e7 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/id/IdGenerator.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/id/IdGenerator.java @@ -28,15 +28,15 @@ public abstract class IdGenerator { public abstract Id generate(HugeVertex vertex); - public static Id of(String id) { + public final static Id of(String id) { return new StringId(id); } - public static Id of(long id) { + public final static Id of(long id) { return new LongId(id); } - public static Id of(byte[] bytes, boolean number) { + public final static Id of(byte[] bytes, boolean number) { return number ? new LongId(bytes) : new StringId(bytes); } @@ -45,7 +45,7 @@ public static Id of(byte[] bytes, boolean number) { * @param id original string id value * @return wrapped id object */ - public Id generate(String id) { + public final Id generate(String id) { return of(id); } @@ -54,7 +54,7 @@ public Id generate(String id) { * @param id original long id value * @return wrapped id object */ - public Id generate(long id) { + public final Id generate(long id) { return of(id); } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/Condition.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/Condition.java index 06edca2924..f1456bcd59 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/Condition.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/Condition.java @@ -397,12 +397,6 @@ public boolean test(HugeElement element) { public Condition copy() { return new And(this.left().copy(), this.right().copy()); } - - @Override - public boolean isFlattened() { - // If this is flattened, its sub-condition should not be nested - return this.left().isRelation() && this.right().isRelation(); - } } public static class Or extends BinCondition { @@ -442,6 +436,10 @@ public abstract static class Relation extends Condition { // The value serialized(code/string) by backend store. protected Object serialValue; + protected Set UNFLATTEN_RELATION_TYPES = ImmutableSet.of( + RelationType.IN, RelationType.NOT_IN, + RelationType.TEXT_CONTAINS_ANY); + @Override public ConditionType type() { return ConditionType.RELATION; @@ -476,6 +474,11 @@ public boolean test(Object value) { return this.relation.test(value, this.value); } + @Override + public boolean isFlattened() { + return !this.UNFLATTEN_RELATION_TYPES.contains(this.relation); + } + @Override public List relations() { return ImmutableList.of(this); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/ConditionQuery.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/ConditionQuery.java index a83a755659..8d7600eb7a 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/ConditionQuery.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/ConditionQuery.java @@ -38,7 +38,7 @@ import com.baidu.hugegraph.util.E; import com.google.common.base.Function; -public class ConditionQuery extends IdQuery { +public final class ConditionQuery extends IdQuery { // Conditions will be concated with `and` by default private Set conditions = new LinkedHashSet<>(); @@ -136,8 +136,7 @@ public List relations() { return relations; } - public Object condition(Object key) { - this.checkFlattened(); + public T condition(Object key) { List values = new ArrayList<>(); for (Condition c : this.conditions) { if (c.isRelation()) { @@ -152,15 +151,16 @@ public Object condition(Object key) { } E.checkState(values.size() == 1, "Illegal key '%s' with more than one value", key); - return values.get(0); + @SuppressWarnings("unchecked") + T value = (T) values.get(0); + return value; } public void unsetCondition(Object key) { - this.checkFlattened(); for (Iterator iter = this.conditions.iterator(); iter.hasNext();) { Condition c = iter.next(); - assert c.isRelation(); + E.checkState(c.isRelation(), "Can't unset condition '%s'", c); if (((Condition.Relation) c).key().equals(key)) { iter.remove(); } @@ -203,6 +203,15 @@ public boolean allSysprop() { return true; } + public boolean allRelation() { + for (Condition c : this.conditions) { + if (!c.isRelation()) { + return false; + } + } + return true; + } + public List syspropConditions() { this.checkFlattened(); List conds = new ArrayList<>(); @@ -388,11 +397,17 @@ public boolean test(HugeElement element) { } public void checkFlattened() { + E.checkState(this.isFlattened(), + "Query has none-flatten condition: %s", this); + } + + public boolean isFlattened() { for (Condition condition : this.conditions) { - E.checkState(condition.isFlattened(), - "Condition Query has none-flatten condition '%s'", - condition); + if (!condition.isFlattened()) { + return false; + } } + return true; } public void optimized(int optimizedType) { diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/ConditionQueryFlatten.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/ConditionQueryFlatten.java index 0f7773ee90..2c228fe238 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/ConditionQueryFlatten.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/ConditionQueryFlatten.java @@ -34,15 +34,15 @@ import com.baidu.hugegraph.util.NumericUtil; import com.google.common.collect.ImmutableList; -public class ConditionQueryFlatten { +public final class ConditionQueryFlatten { public static List flatten(ConditionQuery query) { - List queries = new ArrayList<>(); - if (query.conditions().isEmpty()) { - queries.add(query.copy()); - return queries; + if (query.isFlattened()) { + return Arrays.asList(query); } + List queries = new ArrayList<>(); + // Flatten IN/NOT_IN if needed Set conditions = new HashSet<>(); for (Condition condition : query.conditions()) { @@ -65,6 +65,8 @@ public static List flatten(ConditionQuery query) { results = and(results, flattenAndOr(condition)); } } + + // Optimize useless condition assert results != null; for (Relations relations : results) { relations = optimizeRelations(relations); @@ -80,6 +82,7 @@ public static List flatten(ConditionQuery query) { queries.add(cq); } } + return queries; } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/IdPrefixQuery.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/IdPrefixQuery.java index 48cf77a9f9..a221f3ae77 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/IdPrefixQuery.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/IdPrefixQuery.java @@ -25,7 +25,7 @@ import com.baidu.hugegraph.util.Bytes; import com.baidu.hugegraph.util.E; -public class IdPrefixQuery extends Query { +public final class IdPrefixQuery extends Query { private final Id start; private final boolean inclusiveStart; diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/IdRangeQuery.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/IdRangeQuery.java index 3c643282c8..7547254ea8 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/IdRangeQuery.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/IdRangeQuery.java @@ -25,7 +25,7 @@ import com.baidu.hugegraph.util.Bytes; import com.baidu.hugegraph.util.E; -public class IdRangeQuery extends Query { +public final class IdRangeQuery extends Query { private final Id start; private final Id end; diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/BinaryBackendEntry.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/BinaryBackendEntry.java index 703691c477..da421156e9 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/BinaryBackendEntry.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/BinaryBackendEntry.java @@ -78,7 +78,7 @@ public String toString() { public BackendColumn column(byte[] name) { for (BackendColumn col : this.columns) { - if (Arrays.equals(col.name, name)) { + if (Bytes.equals(col.name, name)) { return col; } } @@ -157,7 +157,7 @@ public boolean equals(Object obj) { return true; } - protected static class BinaryId implements Id { + protected static final class BinaryId implements Id { private final byte[] bytes; private final Id id; diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/BinaryInlineSerializer.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/BinaryInlineSerializer.java new file mode 100644 index 0000000000..aebfe1e6fd --- /dev/null +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/BinaryInlineSerializer.java @@ -0,0 +1,102 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.backend.serializer; + +import org.apache.commons.lang.NotImplementedException; + +import com.baidu.hugegraph.HugeGraph; +import com.baidu.hugegraph.backend.id.Id; +import com.baidu.hugegraph.backend.store.BackendEntry; +import com.baidu.hugegraph.backend.store.BackendEntry.BackendColumn; +import com.baidu.hugegraph.schema.VertexLabel; +import com.baidu.hugegraph.structure.HugeVertex; +import com.baidu.hugegraph.structure.HugeVertexProperty; +import com.baidu.hugegraph.util.Bytes; + +public class BinaryInlineSerializer extends BinarySerializer { + + public BinaryInlineSerializer() { + super(true); + } + + @Override + public BackendEntry writeVertex(HugeVertex vertex) { + BinaryBackendEntry entry = newBackendEntry(vertex); + + if (vertex.removed()) { + return entry; + } + + int propsCount = vertex.getProperties().size(); + BytesBuffer buffer = BytesBuffer.allocate(8 + 16 * propsCount); + + // Write vertex label + buffer.writeId(vertex.schemaLabel().id()); + + // Write all properties of the vertex + this.formatProperties(vertex.getProperties().values(), buffer); + + entry.column(entry.id().asBytes(), buffer.bytes()); + + return entry; + } + + @Override + public HugeVertex readVertex(HugeGraph graph, BackendEntry bytesEntry) { + if (bytesEntry == null) { + return null; + } + BinaryBackendEntry entry = this.convertEntry(bytesEntry); + + // Parse id + Id id = entry.id().origin(); + HugeVertex vertex = new HugeVertex(graph, id, VertexLabel.NONE); + + // Parse all properties and edges of a Vertex + for (BackendColumn col : entry.columns()) { + if (Bytes.equals(entry.id().asBytes(), col.name)) { + // Parse vertex properties + assert entry.columnsSize() == 1 : entry.columnsSize(); + this.parseVertex(col.value, vertex); + } else { + // Parse vertex edges + this.parseColumn(col, vertex); + } + } + + return vertex; + } + + protected void parseVertex(byte[] value, HugeVertex vertex) { + BytesBuffer buffer = BytesBuffer.wrap(value); + + // Parse vertex label + VertexLabel label = vertex.graph().vertexLabel(buffer.readId()); + vertex.vertexLabel(label); + + // Parse properties + this.parseProperties(buffer, vertex); + } + + @Override + public BackendEntry writeVertexProperty(HugeVertexProperty prop) { + throw new NotImplementedException("Unsupported writeVertexProperty()"); + } +} diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/BinarySerializer.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/BinarySerializer.java index cbfc54c036..7d6f342fef 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/BinarySerializer.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/BinarySerializer.java @@ -94,18 +94,17 @@ public BinaryBackendEntry newBackendEntry(HugeType type, Id id) { return new BinaryBackendEntry(type, bid); } - private BinaryBackendEntry newBackendEntry(HugeVertex vertex) { + protected final BinaryBackendEntry newBackendEntry(HugeVertex vertex) { return newBackendEntry(vertex.type(), vertex.id()); } - private BinaryBackendEntry newBackendEntry(HugeEdge edge) { + protected final BinaryBackendEntry newBackendEntry(HugeEdge edge) { BinaryId id = new BinaryId(formatEdgeName(edge), edge.idWithDirection()); return new BinaryBackendEntry(edge.type(), id); } - @SuppressWarnings("unused") - private BinaryBackendEntry newBackendEntry(SchemaElement elem) { + protected final BinaryBackendEntry newBackendEntry(SchemaElement elem) { return newBackendEntry(elem.type(), elem.id()); } @@ -182,6 +181,25 @@ protected void parseProperty(Id pkeyId, byte[] val, HugeElement owner) { } } + protected void formatProperties(Collection> props, + BytesBuffer buffer) { + // Write properties size + buffer.writeInt(props.size()); + + // Write properties data + for (HugeProperty property : props) { + buffer.writeId(property.propertyKey().id()); + buffer.writeBytes(KryoUtil.toKryo(property.value())); + } + } + + protected void parseProperties(BytesBuffer buffer, HugeElement owner) { + int size = buffer.readInt(); + for (int i = 0; i < size; i++) { + this.parseProperty(buffer.readId(), buffer.readBytes(), owner); + } + } + protected byte[] formatEdgeName(HugeEdge edge) { // owner-vertex + dir + edge-label + sort-values + other-vertex @@ -203,14 +221,8 @@ protected byte[] formatEdgeValue(HugeEdge edge) { // Write edge id //buffer.writeId(edge.id()); - // Write edge properties size - buffer.writeInt(propsCount); - - // Write edge properties data - for (HugeProperty property : edge.getProperties().values()) { - buffer.writeId(property.propertyKey().id()); - buffer.writeBytes(KryoUtil.toKryo(property.value())); - } + // Write edge properties + this.formatProperties(edge.getProperties().values(), buffer); return buffer.bytes(); } @@ -271,16 +283,13 @@ protected void parseEdge(BackendColumn col, HugeVertex vertex, vertex.propNotLoaded(); otherVertex.propNotLoaded(); - // Write edge-id + edge-properties + // Parse edge-id + edge-properties buffer = BytesBuffer.wrap(col.value); //Id id = buffer.readId(); - // Write edge properties - int size = buffer.readInt(); - for (int i = 0; i < size; i++) { - this.parseProperty(buffer.readId(), buffer.readBytes(), edge); - } + // Parse edge properties + this.parseProperties(buffer, edge); } protected void parseColumn(BackendColumn col, HugeVertex vertex) { @@ -502,19 +511,20 @@ private Query writeQueryEdgeRangeCondition(ConditionQuery cq) { E.checkArgument(sortValues.size() >= 1 && sortValues.size() <= 2, "Edge range query must be with sort-values range"); // Would ignore target vertex - Object vertex = cq.condition(HugeKeys.OWNER_VERTEX); - Object direction = cq.condition(HugeKeys.DIRECTION); + Id vertex = cq.condition(HugeKeys.OWNER_VERTEX); + Directions direction = cq.condition(HugeKeys.DIRECTION); if (direction == null) { direction = Directions.OUT; } - Object label = cq.condition(HugeKeys.LABEL); + Id label = cq.condition(HugeKeys.LABEL); - BytesBuffer start = BytesBuffer.allocate(256); - start.writeId(HugeVertex.getIdValue(vertex)); - start.write(((Directions) direction).type().code()); - start.writeId((Id) label); + int size = 1 + vertex.length() + 1 + label.length() + 16; + BytesBuffer start = BytesBuffer.allocate(size); + start.writeId(vertex); + start.write(direction.type().code()); + start.writeId(label); - BytesBuffer end = BytesBuffer.allocate(256); + BytesBuffer end = BytesBuffer.allocate(size); end.copyFrom(start); int minEq = -1; @@ -556,7 +566,7 @@ private Query writeQueryEdgeRangeCondition(ConditionQuery cq) { private Query writeQueryEdgePrefixCondition(ConditionQuery cq) { int count = 0; - BytesBuffer buffer = BytesBuffer.allocate(256); + BytesBuffer buffer = BytesBuffer.allocate(64); for (HugeKeys key : EdgeId.KEYS) { Object value = cq.condition(key); @@ -573,8 +583,7 @@ private Query writeQueryEdgePrefixCondition(ConditionQuery cq) { if (key == HugeKeys.OWNER_VERTEX || key == HugeKeys.OTHER_VERTEX) { - Id id = HugeVertex.getIdValue(value); - buffer.writeId(id); + buffer.writeId((Id) value); } else if (key == HugeKeys.DIRECTION) { byte t = ((Directions) value).type().code(); buffer.write(t); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/BytesBuffer.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/BytesBuffer.java index 8f96bdc108..90cde540f6 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/BytesBuffer.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/BytesBuffer.java @@ -48,6 +48,10 @@ public final class BytesBuffer { public static final int ID_MAX_LEN = UINT8_MAX & 0x7f + 1; // 128 public static final int BIG_ID_MAX_LEN = UINT16_MAX & 0x7fff + 1; // 32768 + public static final long ID_MIN = Long.MIN_VALUE >> 3; + public static final long ID_MAX = Long.MAX_VALUE >> 3; + public static final long ID_MASK = 0x0fffffffffffffffL; + // The value must be in range [8, 128(ID_MAX_LEN)] public static final int INDEX_ID_MAX_LENGTH = 32; @@ -194,6 +198,10 @@ public BytesBuffer writeString(String val) { return this; } + public byte peek() { + return this.buffer.get(this.buffer.position()); + } + public byte read() { return this.buffer.get(); } @@ -323,12 +331,13 @@ public Id readId() { } public Id readId(boolean big) { - int b = this.readUInt8(); + int b = this.peek(); boolean number = (b & 0x80) == 0; - int len = b & 0x7f; if (number) { - return IdGenerator.of(this.readNumber(len)); + return IdGenerator.of(this.readNumber(b)); } else { + this.readUInt8(); + int len = b & 0x7f; if (big) { int high = len << 8; int low = this.readUInt8(); @@ -355,31 +364,50 @@ public BinaryId parseId() { } private void writeNumber(long val) { + int positive = val >= 0 ? 0x10 : 0x00; if (Byte.MIN_VALUE <= val && val <= Byte.MAX_VALUE) { - this.writeUInt8(1); + this.writeUInt8(0x00 | positive); this.write((byte) val); } else if (Short.MIN_VALUE <= val && val <= Short.MAX_VALUE) { - this.writeUInt8(2); + this.writeUInt8(0x20 | positive); this.writeShort((short) val); } else if (Integer.MIN_VALUE <= val && val <= Integer.MAX_VALUE) { - this.writeUInt8(4); + this.writeUInt8(0x40 | positive); this.writeInt((int) val); } else { - this.writeUInt8(8); - this.writeLong(val); + E.checkArgument(ID_MIN < val && val < ID_MAX, + "Id value must be in [%s, %s], but got %s", + ID_MIN, ID_MAX, val); + this.writeLong((val & ID_MASK) | ((0x60L | positive) << 56)); } } - private long readNumber(int len) { - if (len <= 1) { - return this.read(); - } else if (len <= 2) { - return this.readShort(); - } else if (len <= 4) { - return this.readInt(); - } else { - assert len == 8 : len; - return this.readLong(); + private long readNumber(int b) { + // Parse length from byte 0b0llsnnnn: bits `ll` is the number length + E.checkArgument((b & 0x80) == 0, + "Not a number type with prefix byte '0x%s'", + Integer.toHexString(b)); + int length = b >> 5; + boolean positive = (b & 0x10) > 0; + switch (length) { + case 0: + this.read(); + return this.read(); + case 1: + this.read(); + return this.readShort(); + case 2: + this.read(); + return this.readInt(); + case 3: + long value = this.readLong(); + value &= ID_MASK; + if (!positive) { + value |= Long.MIN_VALUE; + } + return value; + default: + throw new AssertionError("Invalid length of number: " + length); } } } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/SerializerFactory.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/SerializerFactory.java index fccef08864..ebfbcbc4a3 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/SerializerFactory.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/SerializerFactory.java @@ -35,6 +35,8 @@ public class SerializerFactory { public static AbstractSerializer serializer(String name) { if (name.equalsIgnoreCase("binary")) { return new BinarySerializer(); + } else if (name.equalsIgnoreCase("binaryinline")) { + return new BinaryInlineSerializer(); } else if (name.equalsIgnoreCase("text")) { return new TextSerializer(); } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendFeatures.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendFeatures.java index 24d68437a3..b9cb3f136b 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendFeatures.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendFeatures.java @@ -21,6 +21,10 @@ public interface BackendFeatures { + public default boolean supportsPersistence() { + return true; + } + public boolean supportsScanToken(); public boolean supportsScanKeyPrefix(); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendSession.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendSession.java index 017caa247e..1f905c7195 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendSession.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendSession.java @@ -38,10 +38,10 @@ public BackendSession() { public abstract boolean closed(); - public abstract void clear(); - public abstract Object commit(); + public abstract void rollback(); + public abstract boolean hasChanges(); protected int attach() { diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/memory/InMemoryDBStore.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/memory/InMemoryDBStore.java index 7946722505..8b025be06d 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/memory/InMemoryDBStore.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/memory/InMemoryDBStore.java @@ -314,6 +314,11 @@ public long getCounter(HugeType type) { */ private static final BackendFeatures FEATURES = new BackendFeatures() { + @Override + public boolean supportsPersistence() { + return false; + } + @Override public boolean supportsScanToken() { return false; diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/GraphIndexTransaction.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/GraphIndexTransaction.java index 5a35c517d4..3973536be4 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/GraphIndexTransaction.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/GraphIndexTransaction.java @@ -374,8 +374,8 @@ private Set queryByUserpropWithSearchIndex(ConditionQuery query, // Do query Set ids = InsertionOrderUtil.newSet(); - for (ConditionQuery q : ConditionQueryFlatten.flatten(query)) { - IndexQueries queries = index.constructIndexQueries(q); + for (ConditionQuery cq : ConditionQueryFlatten.flatten(query)) { + IndexQueries queries = index.constructIndexQueries(cq); ids.addAll(this.intersectIndexQueries(queries)); } return ids; diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/GraphTransaction.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/GraphTransaction.java index f4b3e4c173..8120fa5e33 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/GraphTransaction.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/GraphTransaction.java @@ -311,28 +311,36 @@ public Iterator query(Query query) { return super.query(query); } - List queries = new ArrayList<>(); - IdQuery ids = new IdQuery(query.resultType(), query); - for (ConditionQuery cq: ConditionQueryFlatten.flatten( - (ConditionQuery) query)) { - Query q = this.optimizeQuery(cq); + ConditionQuery cq = (ConditionQuery) query; + if (cq.isFlattened()) { /* * NOTE: There are two possibilities for this query: * 1.sysprop-query, which would not be empty. * 2.index-query result(ids after optimization), which may be empty. */ + Query q = this.optimizeQuery(cq); + // Return empty if there is no result after index-query + if (q.empty()) { + return Collections.emptyIterator(); + } + return super.query(q); + } + + // Flatten and optimize the query + List queries = new ArrayList<>(); + IdQuery idQuery = new IdQuery(query.resultType(), query); + for (ConditionQuery fcq: ConditionQueryFlatten.flatten(cq)) { + Query q = this.optimizeQuery(fcq); if (q.getClass() == IdQuery.class && !q.ids().isEmpty()) { - ids.query(q.ids()); + idQuery.query(q.ids()); } else if (!q.empty()) { - // Return empty if there is no result after index-query queries.add(q); } } - - ExtendableIterator rs = new ExtendableIterator<>(); - if (!ids.empty()) { - queries.add(ids); + if (!idQuery.empty()) { + queries.add(idQuery); } + ExtendableIterator rs = new ExtendableIterator<>(); for (Query q : queries) { rs.extend(super.query(q)); } @@ -604,7 +612,9 @@ public Iterator queryEdges() { public Iterator queryEdges(Query query) { Iterator results = this.queryEdgesFromBackend(query); - Set returnedEdges = new HashSet<>(); + // TODO: any unconsidered case, maybe the query with OR condition? + boolean withDuplicatedEdge = false; + Set returnedEdges = withDuplicatedEdge ? new HashSet<>() : null; results = new FilterIterator<>(results, edge -> { // Filter hidden results if (!query.showHidden() && Graph.Hidden.isHidden(edge.label())) { @@ -614,12 +624,15 @@ public Iterator queryEdges(Query query) { if (!this.filterResultFromIndexQuery(query, edge)) { return false; } - - // Filter repeat edges (TODO: split edges table into OUT&IN table) + // Without repeated edges if not querying by BOTH all edges + if (!withDuplicatedEdge) { + return true; + } + // Filter duplicated edges (edge may be repeated query both) if (!returnedEdges.contains(edge.id())) { /* - * NOTE: Maybe some edges are IN and others are OUT if - * querying edges both directions, perhaps it would look + * NOTE: Maybe some edges are IN and others are OUT + * if querying edges both directions, perhaps it would look * better if we convert all edges in results to OUT, but * that would break the logic when querying IN edges. */ diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/config/CoreOptions.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/config/CoreOptions.java index 1843e3a53f..52f4103cd0 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/config/CoreOptions.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/config/CoreOptions.java @@ -162,7 +162,7 @@ public static synchronized CoreOptions instance() { new ConfigOption<>( "schema.cache_capacity", "The max cache size(items) of schema cache.", - rangeInt(1, Integer.MAX_VALUE), + rangeInt(0, Integer.MAX_VALUE), 100000 ); @@ -178,7 +178,7 @@ public static synchronized CoreOptions instance() { new ConfigOption<>( "vertex.cache_capacity", "The max cache size(items) of vertex cache.", - rangeInt(1, Integer.MAX_VALUE), + rangeInt(0, Integer.MAX_VALUE), (1000 * 1000 * 10) ); @@ -194,7 +194,7 @@ public static synchronized CoreOptions instance() { new ConfigOption<>( "edge.cache_capacity", "The max cache size(items) of edge cache.", - rangeInt(1, Integer.MAX_VALUE), + rangeInt(0, Integer.MAX_VALUE), (1000 * 1000 * 1) ); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeEdge.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeEdge.java index bb7e5f13c9..73fec1af1b 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeEdge.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeEdge.java @@ -447,7 +447,7 @@ public String toString() { return StringFactory.edgeString(this); } - public static Id getIdValue(Object idValue) { + public static final Id getIdValue(Object idValue) { Id id = HugeElement.getIdValue(idValue); if (id == null || id instanceof EdgeId) { return id; diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeElement.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeElement.java index 756a710e15..9992b7ee90 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeElement.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeElement.java @@ -47,9 +47,12 @@ import com.baidu.hugegraph.type.define.HugeKeys; import com.baidu.hugegraph.util.CollectionUtil; import com.baidu.hugegraph.util.E; +import com.google.common.collect.ImmutableMap; public abstract class HugeElement implements Element, GraphType { + private static final Map> EMPTY = ImmutableMap.of(); + private final HugeGraph graph; protected Id id; @@ -62,7 +65,7 @@ public HugeElement(final HugeGraph graph, Id id) { E.checkArgument(graph != null, "HugeElement graph can't be null"); this.graph = graph; this.id = id; - this.properties = new HashMap<>(); + this.properties = EMPTY; this.removed = false; this.fresh = false; this.propLoaded = true; @@ -149,6 +152,9 @@ public int sizeOfProperties() { @Watched(prefix = "element") public HugeProperty setProperty(HugeProperty prop) { PropertyKey pkey = prop.propertyKey(); + if (this.properties == EMPTY) { + this.properties = new HashMap<>(); + } return this.properties.put(pkey.id(), prop); } @@ -276,7 +282,7 @@ public int hashCode() { * @return Key-value pairs that are classified and processed */ @Watched(prefix = "element") - public static ElementKeys classifyKeys(Object... keyValues) { + public static final ElementKeys classifyKeys(Object... keyValues) { ElementKeys elemKeys = new ElementKeys(); if ((keyValues.length & 1) == 1) { @@ -305,7 +311,7 @@ public static ElementKeys classifyKeys(Object... keyValues) { return elemKeys; } - public static Id getIdValue(HugeType type, Object idValue) { + public static final Id getIdValue(HugeType type, Object idValue) { assert type.isGraph(); Id id = getIdValue(idValue); if (type.isVertex()) { @@ -342,7 +348,7 @@ protected static Id getIdValue(Object idValue) { } @Watched(prefix = "element") - public static Object getLabelValue(Object... keyValues) { + public static final Object getLabelValue(Object... keyValues) { Object labelValue = null; for (int i = 0; i < keyValues.length; i = i + 2) { if (keyValues[i].equals(T.label)) { @@ -351,7 +357,7 @@ public static Object getLabelValue(Object... keyValues) { labelValue instanceof VertexLabel, "Expect a string or a VertexLabel object " + "as the vertex label argument, but got: '%s'", - labelValue); + labelValue); if (labelValue instanceof String) { ElementHelper.validateLabel((String) labelValue); } @@ -361,7 +367,7 @@ public static Object getLabelValue(Object... keyValues) { return labelValue; } - public static class ElementKeys { + public static final class ElementKeys { private Object label = null; private Object id = null; diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeVertex.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeVertex.java index 4740494d29..6cb8aaab38 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeVertex.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeVertex.java @@ -539,7 +539,7 @@ public String toString() { return StringFactory.vertexString(this); } - public static Id getIdValue(Object idValue) { + public static final Id getIdValue(Object idValue) { return HugeElement.getIdValue(idValue); } } diff --git a/hugegraph-hbase/src/main/java/com/baidu/hugegraph/backend/store/hbase/HbaseSessions.java b/hugegraph-hbase/src/main/java/com/baidu/hugegraph/backend/store/hbase/HbaseSessions.java index f93555d279..8ee05f3a7e 100644 --- a/hugegraph-hbase/src/main/java/com/baidu/hugegraph/backend/store/hbase/HbaseSessions.java +++ b/hugegraph-hbase/src/main/java/com/baidu/hugegraph/backend/store/hbase/HbaseSessions.java @@ -108,12 +108,12 @@ public synchronized void open(HugeConfig conf) throws IOException { } @Override - protected boolean opened() { + protected synchronized boolean opened() { return this.hbase != null && !this.hbase.isClosed(); } @Override - public final synchronized Session session() { + public final Session session() { return (Session) super.getOrNewSession(); } @@ -123,7 +123,7 @@ protected Session newSession() { } @Override - protected void doClose() { + protected synchronized void doClose() { if (this.hbase == null || this.hbase.isClosed()) { return; } @@ -262,14 +262,6 @@ public boolean closed() { return this.closed; } - /** - * Clear updates not committed in the session - */ - @Override - public void clear() { - this.batch.clear(); - } - /** * Any change in the session */ @@ -309,6 +301,14 @@ public Integer commit() { return count; } + /** + * Rollback all updates(put/delete) not committed + */ + @Override + public void rollback() { + this.batch.clear(); + } + /** * Add a row record to a table */ diff --git a/hugegraph-hbase/src/main/java/com/baidu/hugegraph/backend/store/hbase/HbaseStore.java b/hugegraph-hbase/src/main/java/com/baidu/hugegraph/backend/store/hbase/HbaseStore.java index a766d5346e..7768c31f9b 100644 --- a/hugegraph-hbase/src/main/java/com/baidu/hugegraph/backend/store/hbase/HbaseStore.java +++ b/hugegraph-hbase/src/main/java/com/baidu/hugegraph/backend/store/hbase/HbaseStore.java @@ -300,11 +300,7 @@ public void commitTx() { this.checkOpened(); Session session = this.sessions.session(); - try { - session.commit(); - } finally { - session.clear(); - } + session.commit(); } @Override diff --git a/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlSessions.java b/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlSessions.java index 5b1c1256b8..6749596efa 100644 --- a/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlSessions.java +++ b/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlSessions.java @@ -47,7 +47,7 @@ public class MysqlSessions extends BackendSessionPool { private HugeConfig config; private String database; - private boolean opened; + private volatile boolean opened; public MysqlSessions(HugeConfig config, String database, String store) { super(database + "/" + store); @@ -69,7 +69,7 @@ public String database() { * @throws SQLException if a database access error occurs */ @Override - public void open(HugeConfig config) throws Exception { + public synchronized void open(HugeConfig config) throws Exception { try (Connection conn = this.open(false)) { this.opened = true; } @@ -118,21 +118,21 @@ private Connection connect(String url) throws SQLException { return DriverManager.getConnection(url, username, password); } - @Override - protected synchronized Session newSession() { - return new Session(); - } - @Override protected void doClose() { // pass } @Override - public synchronized Session session() { + public Session session() { return (Session) super.getOrNewSession(); } + @Override + protected Session newSession() { + return new Session(); + } + public void checkSessionConnected() { Session session = this.session(); E.checkState(session != null, "MySQL session has not been initialized"); @@ -269,7 +269,6 @@ public boolean closed() { return !this.opened; } - @Override public void clear() { this.count = 0; SQLException exception = null; @@ -316,6 +315,7 @@ public Integer commit() { return updated; } + @Override public void rollback() { this.clear(); try { diff --git a/hugegraph-palo/src/main/java/com/baidu/hugegraph/backend/store/palo/PaloSessions.java b/hugegraph-palo/src/main/java/com/baidu/hugegraph/backend/store/palo/PaloSessions.java index bb5454c808..58454c6b12 100644 --- a/hugegraph-palo/src/main/java/com/baidu/hugegraph/backend/store/palo/PaloSessions.java +++ b/hugegraph-palo/src/main/java/com/baidu/hugegraph/backend/store/palo/PaloSessions.java @@ -65,7 +65,7 @@ public PaloSessions(HugeConfig config, String database, String store, // Scan disk files and restore session information this.restoreSessionInfo(config, tableDirs); - this.timer = new Timer(); + this.timer = new Timer(true); long interval = config.get(PaloOptions.PALO_POLL_INTERVAL); this.loadTask = new PaloLoadTask(tableDirs); this.timer.schedule(this.loadTask, 0, interval * 1000); @@ -95,25 +95,21 @@ protected String buildCreateDatabase(String database) { } @Override - protected final synchronized Session newSession() { - int id = this.counter.incrementAndGet(); - this.locks.put(id, new ReentrantReadWriteLock()); - return new Session(id); + public final Session session() { + return (Session) super.getOrNewSession(); } @Override - public final synchronized Session session() { - return (Session) super.getOrNewSession(); + protected final Session newSession() { + int id = this.counter.incrementAndGet(); + this.locks.put(id, new ReentrantReadWriteLock()); + return new Session(id); } @Override public void close() { - if (this.loadTask != null) { - this.loadTask.join(); - } - if (this.timer != null) { - this.timer.cancel(); - } + this.loadTask.join(); + this.timer.cancel(); super.close(); } diff --git a/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBSessions.java b/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBSessions.java index 01db11cf4a..bfbe332273 100644 --- a/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBSessions.java +++ b/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBSessions.java @@ -49,8 +49,8 @@ public RocksDBSessions(String database, String store) { public static abstract class Session extends BackendSession { public static final int SCAN_ANY = 0x80; - public static final int SCAN_PREFIX_WITH_BEGIN = 0x01; - public static final int SCAN_PREFIX_WITH_END = 0x02; + public static final int SCAN_PREFIX_BEGIN = 0x01; + public static final int SCAN_PREFIX_END = 0x02; public static final int SCAN_GT_BEGIN = 0x04; public static final int SCAN_GTE_BEGIN = 0x0c; public static final int SCAN_LT_END = 0x10; @@ -81,5 +81,9 @@ public BackendColumnIterator scan(String table, byte[] keyTo) { return this.scan(table, keyFrom, keyTo, SCAN_LT_END); } + + public static boolean matchScanType(int expected, int actual) { + return (expected & actual) == expected; + } } } diff --git a/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBStdSessions.java b/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBStdSessions.java index d77485e433..37d055d73e 100644 --- a/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBStdSessions.java +++ b/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBStdSessions.java @@ -138,7 +138,7 @@ public void open(HugeConfig config) throws Exception { @Override protected boolean opened() { - return this.rocksdb != null; + return this.rocksdb != null && this.rocksdb.isOwningHandle(); } @Override @@ -186,13 +186,13 @@ public String property(String property) { } @Override - public final synchronized Session session() { + public final Session session() { return (Session) super.getOrNewSession(); } @Override - protected final synchronized Session newSession() { - E.checkState(this.rocksdb != null, + protected final Session newSession() { + E.checkState(this.rocksdb.isOwningHandle(), "RocksDB has not been initialized"); return new StdSession(this.conf); } @@ -215,7 +215,7 @@ private void checkValid() { } private RocksDB rocksdb() { - assert this.rocksdb.isOwningHandle(); + this.checkValid(); return this.rocksdb; } @@ -408,14 +408,6 @@ public boolean closed() { return this.closed; } - /** - * Clear updates not committed in the session - */ - @Override - public void clear() { - this.batch.clear(); - } - /** * Any change in the session */ @@ -459,6 +451,14 @@ public Integer commit() { return count; } + /** + * Rollback all updates(put/delete) not committed + */ + @Override + public void rollback() { + this.batch.clear(); + } + /** * Add a KV record to a table */ @@ -571,7 +571,7 @@ public BackendColumnIterator scan(String table, byte[] prefix) { options.setPrefixSameAsStart(true); RocksIterator itor = rocksdb().newIterator(cf(table), options); return new ColumnIterator(table, itor, prefix, null, - SCAN_PREFIX_WITH_BEGIN); + SCAN_PREFIX_BEGIN); } /** @@ -623,22 +623,22 @@ public ColumnIterator(String table, RocksIterator itor, } private void checkArguments() { - E.checkArgument(!(this.match(Session.SCAN_PREFIX_WITH_BEGIN) && - this.match(Session.SCAN_PREFIX_WITH_END)), + E.checkArgument(!(this.match(Session.SCAN_PREFIX_BEGIN) && + this.match(Session.SCAN_PREFIX_END)), "Can't set SCAN_PREFIX_WITH_BEGIN and " + "SCAN_PREFIX_WITH_END at the same time"); - E.checkArgument(!(this.match(Session.SCAN_PREFIX_WITH_BEGIN) && + E.checkArgument(!(this.match(Session.SCAN_PREFIX_BEGIN) && this.match(Session.SCAN_GT_BEGIN)), "Can't set SCAN_PREFIX_WITH_BEGIN and " + "SCAN_GT_BEGIN/SCAN_GTE_BEGIN at the same time"); - E.checkArgument(!(this.match(Session.SCAN_PREFIX_WITH_END) && + E.checkArgument(!(this.match(Session.SCAN_PREFIX_END) && this.match(Session.SCAN_LT_END)), "Can't set SCAN_PREFIX_WITH_END and " + "SCAN_LT_END/SCAN_LTE_END at the same time"); - if (this.match(Session.SCAN_PREFIX_WITH_BEGIN)) { + if (this.match(Session.SCAN_PREFIX_BEGIN)) { E.checkArgument(this.keyBegin != null, "Parameter `keyBegin` can't be null " + "if set SCAN_PREFIX_WITH_BEGIN"); @@ -647,7 +647,7 @@ private void checkArguments() { "if set SCAN_PREFIX_WITH_BEGIN"); } - if (this.match(Session.SCAN_PREFIX_WITH_END)) { + if (this.match(Session.SCAN_PREFIX_END)) { E.checkArgument(this.keyEnd != null, "Parameter `keyEnd` can't be null " + "if set SCAN_PREFIX_WITH_END"); @@ -667,7 +667,7 @@ private void checkArguments() { } private boolean match(int expected) { - return (expected & this.scanType) == expected; + return Session.matchScanType(expected, this.scanType); } /** @@ -737,13 +737,13 @@ private void seek() { } private boolean filter(byte[] key) { - if (this.match(Session.SCAN_PREFIX_WITH_BEGIN)) { + if (this.match(Session.SCAN_PREFIX_BEGIN)) { /* * Prefix with `keyBegin`? * TODO: use custom prefix_extractor instead */ return Bytes.prefixWith(key, this.keyBegin); - } else if (this.match(Session.SCAN_PREFIX_WITH_END)) { + } else if (this.match(Session.SCAN_PREFIX_END)) { /* * Prefix with `keyEnd`? * like the following query for range index: diff --git a/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBStore.java b/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBStore.java index 8db5eb4bc7..5fc8620adf 100644 --- a/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBStore.java +++ b/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBStore.java @@ -30,6 +30,7 @@ import java.util.Map.Entry; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import java.util.stream.Collectors; import org.apache.commons.io.FileUtils; @@ -69,8 +70,11 @@ public abstract class RocksDBStore extends AbstractBackendStore { private final Map tableDiskMapping; // DataPath:RocksDB mapping - private static final Map dbs = - new ConcurrentHashMap<>(); + protected static final ConcurrentMap dbs; + + static { + dbs = new ConcurrentHashMap<>(); + } public RocksDBStore(final BackendStoreProvider provider, final String database, final String store) { @@ -148,10 +152,7 @@ public synchronized void open(HugeConfig config) { } // Open base disk - String dataPath = this.wrapPath(config.get(RocksDBOptions.DATA_PATH)); - String walPath = this.wrapPath(config.get(RocksDBOptions.WAL_PATH)); - - this.sessions = this.open(config, dataPath, walPath, this.tableNames()); + this.sessions = this.open(config, this.tableNames()); // Open tables with optimized disk List disks = config.get(RocksDBOptions.DATA_DISKS); @@ -165,6 +166,12 @@ public synchronized void open(HugeConfig config) { } } + protected RocksDBSessions open(HugeConfig config, List tableNames) { + String dataPath = this.wrapPath(config.get(RocksDBOptions.DATA_PATH)); + String walPath = this.wrapPath(config.get(RocksDBOptions.WAL_PATH)); + return this.open(config, dataPath, walPath, tableNames); + } + protected RocksDBSessions open(HugeConfig config, String dataPath, String walPath, List tableNames) { LOG.info("Opening RocksDB with data path: {}", dataPath); @@ -400,7 +407,7 @@ public void rollbackTx() { this.checkOpened(); for (Session session : this.session()) { - session.clear(); + session.rollback(); } } @@ -435,7 +442,8 @@ private List session() { private void checkOpened() { E.checkState(this.sessions != null && !this.sessions.closed(), - "RocksDB store has not been opened"); + "The '%s' store of %s has not been opened", + this.database, this.provider.type()); } private void parseTableDiskMapping(List disks) { diff --git a/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBTable.java b/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBTable.java index 795dfe9d87..d580e06a8d 100644 --- a/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBTable.java +++ b/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBTable.java @@ -168,7 +168,7 @@ protected BackendColumnIterator queryByPrefix(Session session, IdPrefixQuery query) { int type = query.inclusiveStart() ? Session.SCAN_GTE_BEGIN : Session.SCAN_GT_BEGIN; - type |= Session.SCAN_PREFIX_WITH_END; + type |= Session.SCAN_PREFIX_END; return session.scan(this.table(), query.start().asBytes(), query.prefix().asBytes(), type); } diff --git a/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBTables.java b/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBTables.java index b1edac6105..1370d554ff 100644 --- a/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBTables.java +++ b/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBTables.java @@ -247,7 +247,7 @@ protected BackendColumnIterator queryByCond(Session session, if (max == null) { E.checkArgumentNotNull(prefix, "Range index prefix is missing"); return session.scan(this.table(), begin, prefix.asBytes(), - Session.SCAN_PREFIX_WITH_END); + Session.SCAN_PREFIX_END); } else { byte[] end = max.asBytes(); int type = maxEq ? Session.SCAN_LTE_END : Session.SCAN_LT_END; diff --git a/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdbsst/RocksDBSstSessions.java b/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdbsst/RocksDBSstSessions.java index d3968da316..38bcbbf3b6 100644 --- a/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdbsst/RocksDBSstSessions.java +++ b/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdbsst/RocksDBSstSessions.java @@ -120,12 +120,12 @@ private SstFileWriter table(String table) { } @Override - public final synchronized Session session() { + public final Session session() { return (Session) super.getOrNewSession(); } @Override - protected final synchronized Session newSession() { + protected Session newSession() { return new SstSession(); } @@ -172,14 +172,6 @@ public boolean closed() { return this.closed; } - /** - * Clear updates not committed in the session - */ - @Override - public void clear() { - this.batch.clear(); - } - /** * Any change in the session */ @@ -222,6 +214,14 @@ public Integer commit() { return count; } + /** + * Rollback updates not committed in the session + */ + @Override + public void rollback() { + this.batch.clear(); + } + /** * Get property value by name from specified table */ diff --git a/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdbsst/RocksDBSstStoreProvider.java b/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdbsst/RocksDBSstStoreProvider.java index 08a2807ba2..ea09b4e83f 100644 --- a/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdbsst/RocksDBSstStoreProvider.java +++ b/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdbsst/RocksDBSstStoreProvider.java @@ -19,33 +19,15 @@ package com.baidu.hugegraph.backend.store.rocksdbsst; -import org.slf4j.Logger; - import com.baidu.hugegraph.backend.store.BackendStore; import com.baidu.hugegraph.backend.store.rocksdb.RocksDBStoreProvider; import com.baidu.hugegraph.backend.store.rocksdbsst.RocksDBSstStore.RocksDBSstGraphStore; -import com.baidu.hugegraph.util.E; -import com.baidu.hugegraph.util.Log; public class RocksDBSstStoreProvider extends RocksDBStoreProvider { - private static final Logger LOG = Log.logger(RocksDBSstStore.class); - @Override - public BackendStore loadGraphStore(String name) { - LOG.debug("RocksDBSstStoreProvider load GraphStore '{}'", name); - - this.checkOpened(); - if (!this.stores.containsKey(name)) { - BackendStore s = new RocksDBSstGraphStore(this, database(), name); - this.stores.putIfAbsent(name, s); - } - - BackendStore store = this.stores.get(name); - E.checkNotNull(store, "store"); - E.checkState(store instanceof RocksDBSstGraphStore, - "GraphStore must be an instance of RocksDBSstGraphStore"); - return store; + protected BackendStore newGraphStore(String store) { + return new RocksDBSstGraphStore(this, this.database(), store); } @Override diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/core/EdgeCoreTest.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/core/EdgeCoreTest.java index e144713340..4ba026887f 100644 --- a/hugegraph-test/src/main/java/com/baidu/hugegraph/core/EdgeCoreTest.java +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/core/EdgeCoreTest.java @@ -42,6 +42,7 @@ import com.baidu.hugegraph.HugeGraph; import com.baidu.hugegraph.backend.BackendException; +import com.baidu.hugegraph.backend.id.Id; import com.baidu.hugegraph.backend.query.ConditionQuery; import com.baidu.hugegraph.backend.query.Query; import com.baidu.hugegraph.backend.serializer.BytesBuffer; @@ -557,6 +558,33 @@ public void testQueryAllEdges() { assertContains(edges, "authored", james, java3); } + @Test + public void testQueryAllEdgesWithGraphAPI() { + HugeGraph graph = graph(); + init18Edges(); + + // All edges + List edges = ImmutableList.copyOf(graph.edges()); + + Assert.assertEquals(18, edges.size()); + + Vertex james = vertex("author", "id", 1); + Vertex guido = vertex("author", "id", 2); + + Vertex java = vertex("language", "name", "java"); + Vertex python = vertex("language", "name", "python"); + + Vertex java1 = vertex("book", "name", "java-1"); + Vertex java2 = vertex("book", "name", "java-2"); + Vertex java3 = vertex("book", "name", "java-3"); + + assertContains(edges, "created", james, java); + assertContains(edges, "created", guido, python); + assertContains(edges, "authored", james, java1); + assertContains(edges, "authored", james, java2); + assertContains(edges, "authored", james, java3); + } + @Test public void testQueryEdgesWithOrderBy() { HugeGraph graph = graph(); @@ -843,6 +871,48 @@ public void testQueryEdgesById() { Assert.assertEquals(1, edges.size()); } + @Test + public void testQueryEdgesByIdWithGraphAPI() { + HugeGraph graph = graph(); + init18Edges(); + + Object id = graph.traversal().E().toList().get(0).id(); + List edges = ImmutableList.copyOf(graph.edges(id)); + Assert.assertEquals(1, edges.size()); + } + + @Test + public void testQueryEdgesByIdWithGraphAPIAndNotCommitedUpdate() { + HugeGraph graph = graph(); + init18Edges(); + + Edge edge = graph.traversal().E().hasLabel("look").toList().get(0); + Object id = edge.id(); + Assert.assertTrue(graph.edges(id).hasNext()); + + edge.property("score", 101); + + List edges = ImmutableList.copyOf(graph.edges(id)); + Assert.assertEquals(1, edges.size()); + Assert.assertEquals(101, (int) edges.get(0).value("score")); + } + + @Test + public void testQueryEdgesByIdWithGraphAPIAndNotCommitedRemoved() { + HugeGraph graph = graph(); + init18Edges(); + + Edge edge = graph.traversal().E().toList().get(0); + Object id = edge.id(); + Assert.assertTrue(graph.edges(id).hasNext()); + + edge.remove(); + Assert.assertFalse(graph.edges(id).hasNext()); + + graph.tx().rollback(); + Assert.assertTrue(graph.edges(id).hasNext()); + } + @Test public void testQueryEdgesByIdNotFound() { HugeGraph graph = graph(); @@ -888,6 +958,9 @@ public void testQueryEdgesByLabel() { edges = graph.traversal().E().hasLabel("know").toList(); Assert.assertEquals(1, edges.size()); + + edges = graph.traversal().E().hasLabel("created", "authored").toList(); + Assert.assertEquals(5, edges.size()); } @Test @@ -918,6 +991,21 @@ public void testQueryBothEdgesOfVertex() { Assert.assertEquals(6, edges.size()); } + @Test + public void testQueryBothEdgesOfVertexWithGraphAPI() { + HugeGraph graph = graph(); + init18Edges(); + + // Query edges of a vertex + Vertex james = vertex("author", "id", 1); + List edges = ImmutableList.copyOf( + graph.adjacentEdges((Id) james.id())); + Assert.assertEquals(6, edges.size()); + + edges = ImmutableList.copyOf(james.edges(Direction.BOTH)); + Assert.assertEquals(6, edges.size()); + } + @Test public void testQueryBothVerticesOfVertex() { HugeGraph graph = graph(); diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/core/VertexCoreTest.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/core/VertexCoreTest.java index 9fad0b6d3a..0f8130fa9a 100644 --- a/hugegraph-test/src/main/java/com/baidu/hugegraph/core/VertexCoreTest.java +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/core/VertexCoreTest.java @@ -44,6 +44,7 @@ import com.baidu.hugegraph.backend.BackendException; import com.baidu.hugegraph.backend.id.Id; import com.baidu.hugegraph.backend.id.IdGenerator; +import com.baidu.hugegraph.backend.id.SnowflakeIdGenerator; import com.baidu.hugegraph.backend.id.SplicingIdGenerator; import com.baidu.hugegraph.backend.query.ConditionQuery; import com.baidu.hugegraph.backend.query.Query; @@ -57,6 +58,7 @@ import com.baidu.hugegraph.testutil.Assert; import com.baidu.hugegraph.testutil.FakeObjects.FakeVertex; import com.baidu.hugegraph.testutil.Utils; +import com.baidu.hugegraph.testutil.Whitebox; import com.baidu.hugegraph.traversal.optimize.Text; import com.baidu.hugegraph.traversal.optimize.TraversalUtil; import com.baidu.hugegraph.type.HugeType; @@ -392,17 +394,21 @@ public void testAddVertexWithAutomaticIdStrategyButPassedId() { } @Test - public void testAddVertexWithAutomaticIdStrategyAndNotPassedId() { + public void testAddVertexWithAutomaticIdStrategy() { HugeGraph graph = graph(); SchemaManager schema = graph.schema(); + // May be set forceString=true by config + Whitebox.setInternalState(SnowflakeIdGenerator.instance(graph), + "forceString", false); + schema.vertexLabel("programmer") .useAutomaticId() .properties("name", "age", "city") .create(); - graph.addVertex(T.label, "programmer", "name", "marko", - "age", 18, "city", "Beijing"); + Vertex v1 = graph.addVertex(T.label, "programmer", "name", "marko", + "age", 18, "city", "Beijing"); graph.tx().commit(); List vertices = graph.traversal().V().toList(); @@ -410,6 +416,61 @@ public void testAddVertexWithAutomaticIdStrategyAndNotPassedId() { assertContains(vertices, T.label, "programmer", "name", "marko", "age", 18, "city", "Beijing"); + + Vertex v2 = graph.addVertex(T.label, "programmer", "name", "marko", + "age", 18, "city", "Beijing"); + graph.tx().commit(); + + Assert.assertNotEquals(v1.id(), v2.id()); + + vertices = graph.traversal().V().toList(); + Assert.assertEquals(2, vertices.size()); + + vertices = graph.traversal().V(v2.id()).toList(); + Assert.assertEquals(1, vertices.size()); + assertContains(vertices, + T.label, "programmer", "name", "marko", + "age", 18, "city", "Beijing"); + } + + @Test + public void testAddVertexWithAutomaticIdStrategyAndForceStringId() { + HugeGraph graph = graph(); + SchemaManager schema = graph.schema(); + + // May be set forceString=false by config + Whitebox.setInternalState(SnowflakeIdGenerator.instance(graph), + "forceString", true); + + schema.vertexLabel("programmer") + .useAutomaticId() + .properties("name", "age", "city") + .create(); + + Vertex v1 = graph.addVertex(T.label, "programmer", "name", "marko", + "age", 18, "city", "Beijing"); + graph.tx().commit(); + + List vertices = graph.traversal().V().toList(); + Assert.assertEquals(1, vertices.size()); + assertContains(vertices, + T.label, "programmer", "name", "marko", + "age", 18, "city", "Beijing"); + + Vertex v2 = graph.addVertex(T.label, "programmer", "name", "marko", + "age", 18, "city", "Beijing"); + graph.tx().commit(); + + Assert.assertNotEquals(v1.id(), v2.id()); + + vertices = graph.traversal().V().toList(); + Assert.assertEquals(2, vertices.size()); + + vertices = graph.traversal().V(v2.id()).toList(); + Assert.assertEquals(1, vertices.size()); + assertContains(vertices, + T.label, "programmer", "name", "marko", + "age", 18, "city", "Beijing"); } @Test @@ -590,6 +651,25 @@ public void testQueryAll() { assertContains(vertexes, T.label, "book", "name", "java-1"); } + @Test + public void testQueryAllWithGraphAPI() { + HugeGraph graph = graph(); + init10Vertices(); + + // Query all + List vertexes = ImmutableList.copyOf(graph.vertices()); + + Assert.assertEquals(10, vertexes.size()); + + assertContains(vertexes, + T.label, "author", "id", 1, "name", "James Gosling", + "age", 62, "lived", "Canadian"); + + assertContains(vertexes, T.label, "language", "name", "java"); + + assertContains(vertexes, T.label, "book", "name", "java-1"); + } + @Test public void testQueryAllWithLimit() { HugeGraph graph = graph(); @@ -1727,12 +1807,33 @@ public void testQueryWithTxNotCommittedByIdInOtherThread() Assert.assertEquals(0, size.get()); } + @Test + public void testQueryWithTxNotCommittedUpdatedProp() { + HugeGraph graph = graph(); + + graph.addVertex(T.label, "person", "name", "marko", + "age", 18, "city", "Beijing"); + Vertex v = graph.addVertex(T.label, "person", "name", "james", + "age", 19, "city", "Hongkong"); + graph().tx().commit(); + + v.property("age", 20); + + List vertices = graph.traversal().V() + .where(__.values("age").is(20)) + .toList(); + Assert.assertEquals(1, vertices.size()); + Assert.assertEquals(v.id(), vertices.get(0).id()); + } + @Test public void testQueryByJointIndexes() { initPersonIndex(true); graph().addVertex(T.label, "person", "name", "Baby", "city", "Hongkong", "age", 3); + graph().tx().commit(); + List vertices; vertices = graph().traversal().V().has("age", 3).toList(); Assert.assertEquals(1, vertices.size()); @@ -1757,6 +1858,7 @@ public void testQueryByJointIndexesAndCompositeIndexForOneLabel() { graph().addVertex(T.label, "person", "name", "Tom", "city", "Hongkong", "age", 3); graph().tx().commit(); + List vertices; vertices = graph().traversal().V().has("age", 3).toList(); Assert.assertEquals(1, vertices.size()); diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/cache/CacheManagerTest.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/cache/CacheManagerTest.java index 47be3bcd78..e5a6246ff9 100644 --- a/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/cache/CacheManagerTest.java +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/cache/CacheManagerTest.java @@ -19,6 +19,7 @@ package com.baidu.hugegraph.unit.cache; +import java.lang.reflect.Proxy; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -171,12 +172,22 @@ public void testCacheExpire() { waitTillNext(40); - // Would call tick() per 10s + // Would call tick() per 30s Mockito.verify(mockCache1, Mockito.times(1)).tick(); Mockito.verify(mockCache2, Mockito.times(1)).tick(); Assert.assertEquals(0, cache1.size()); Assert.assertEquals(1, cache2.size()); } + + @SuppressWarnings("unused") + private static Cache newCacheProxy(Cache cache) { + Object p = Proxy.newProxyInstance(Cache.class.getClassLoader(), + new Class[]{Cache.class}, + (proxy, method, args) -> { + return method.invoke(cache, args); + }); + return (Cache) p; + } } diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/cache/RamCacheTest.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/cache/RamCacheTest.java index d9aa2d06dd..ae7e99924b 100644 --- a/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/cache/RamCacheTest.java +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/cache/RamCacheTest.java @@ -45,11 +45,46 @@ public void teardown() throws Exception { } @Test - public void testUpdateGet() { + public void testUpdateAndGet() { RamCache cache = new RamCache(); Id id = IdGenerator.of("1"); + Assert.assertNull(cache.get(id)); + cache.update(id, "value-1"); Assert.assertEquals("value-1", cache.get(id)); + + cache.update(id, "value-2"); + Assert.assertEquals("value-2", cache.get(id)); + } + + @Test + public void testUpdateAndGetWithSizeEqualCapacity() { + RamCache cache = new RamCache(4); + cache.update(IdGenerator.of("1"), "value-1"); + cache.update(IdGenerator.of("2"), "value-2"); + cache.update(IdGenerator.of("3"), "value-3"); + cache.update(IdGenerator.of("4"), "value-4"); + + Assert.assertEquals("value-1", cache.get(IdGenerator.of("1"))); + Assert.assertEquals("value-2", cache.get(IdGenerator.of("2"))); + Assert.assertEquals("value-3", cache.get(IdGenerator.of("3"))); + Assert.assertEquals("value-4", cache.get(IdGenerator.of("4"))); + } + + @Test + public void testGetOrFetch() { + RamCache cache = new RamCache(); + Id id = IdGenerator.of("1"); + Assert.assertNull(cache.get(id)); + + Assert.assertEquals("value-1", cache.getOrFetch(id, key -> { + return "value-1"; + })); + + cache.update(id, "value-2"); + Assert.assertEquals("value-2", cache.getOrFetch(id, key -> { + return "value-1"; + })); } @Test @@ -69,6 +104,19 @@ public void testUpdateIfAbsentWithExistKey() { Assert.assertEquals("value-1", cache.get(id)); } + @Test + public void testUpdateIfPresent() { + RamCache cache = new RamCache(); + Id id = IdGenerator.of("1"); + cache.updateIfPresent(id, "value-1"); + Assert.assertEquals(null, cache.get(id)); + + cache.update(id, "value-1"); + Assert.assertEquals("value-1", cache.get(id)); + cache.updateIfPresent(id, "value-2"); + Assert.assertEquals("value-2", cache.get(id)); + } + @Test public void testInvalidate() { RamCache cache = new RamCache(); @@ -106,9 +154,12 @@ public void testCapacity() { cache = new RamCache(1); Assert.assertEquals(1, cache.capacity()); - // The min capacity is 1 cache = new RamCache(0); - Assert.assertEquals(1, cache.capacity()); + Assert.assertEquals(0, cache.capacity()); + + // The min capacity is 0 + cache = new RamCache(-1); + Assert.assertEquals(0, cache.capacity()); } @Test diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/rocksdb/BaseRocksDBUnitTest.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/rocksdb/BaseRocksDBUnitTest.java index ec600af08a..919e9f2312 100644 --- a/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/rocksdb/BaseRocksDBUnitTest.java +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/rocksdb/BaseRocksDBUnitTest.java @@ -89,7 +89,7 @@ protected void commit() { try { this.rocks.session().commit(); } finally { - this.rocks.session().clear(); + this.rocks.session().rollback(); } }