From 670102229c9a4891d2764564562f6f582d97b3d2 Mon Sep 17 00:00:00 2001 From: zhoney Date: Thu, 29 Oct 2020 11:12:55 +0800 Subject: [PATCH] set indexlabel invalid if create or rebuild failed (#1226) * set indexlabel invalid if create or rebuild failed * also check if indexlabel invalid when query by index * set vl/el to UNDELETED if fail to delete Change-Id: If890b3bd8c441b63642a2006479cf4ccca9ddf67 --- .../backend/tx/GraphIndexTransaction.java | 12 +++++++ .../backend/tx/GraphTransaction.java | 2 +- .../job/schema/EdgeLabelRemoveCallable.java | 22 ++++++++---- .../job/schema/IndexLabelRemoveCallable.java | 24 ++++++++----- .../job/schema/RebuildIndexCallable.java | 35 ++++++++++++------- .../job/schema/VertexLabelRemoveCallable.java | 25 ++++++++----- .../schema/builder/AbstractBuilder.java | 7 ++++ .../schema/builder/IndexLabelBuilder.java | 15 +++++--- .../hugegraph/type/define/SchemaStatus.java | 14 +++++++- 9 files changed, 113 insertions(+), 43 deletions(-) diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/GraphIndexTransaction.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/GraphIndexTransaction.java index 2ab6a07ea5..0bb19acb4c 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/GraphIndexTransaction.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/GraphIndexTransaction.java @@ -86,6 +86,7 @@ import com.baidu.hugegraph.type.define.Action; import com.baidu.hugegraph.type.define.HugeKeys; import com.baidu.hugegraph.type.define.IndexType; +import com.baidu.hugegraph.type.define.SchemaStatus; import com.baidu.hugegraph.util.CollectionUtil; import com.baidu.hugegraph.util.E; import com.baidu.hugegraph.util.InsertionOrderUtil; @@ -369,6 +370,7 @@ public IdHolderList queryIndex(ConditionQuery query) { private IdHolderList queryByLabel(ConditionQuery query) { HugeType queryType = query.resultType(); IndexLabel il = IndexLabel.label(queryType); + validateIndexLabel(il); Id label = query.condition(HugeKeys.LABEL); assert label != null; @@ -426,6 +428,9 @@ private IdHolderList queryByUserprop(ConditionQuery query) { // Do index query IdHolderList holders = new IdHolderList(paging); for (MatchedIndex index : indexes) { + for (IndexLabel il : index.indexLabels()) { + validateIndexLabel(il); + } if (paging && index.indexLabels().size() > 1) { throw new NotSupportException("joint index query in paging"); } @@ -1349,6 +1354,13 @@ private static NoIndexException noIndexException(HugeGraph graph, name, String.join("/", mismatched)); } + private static void validateIndexLabel(IndexLabel indexLabel) { + E.checkArgument(indexLabel.status().ok(), + "Can't query by label index '%s' due to " + + "it's in status %s(CREATED expected)", + indexLabel, indexLabel.status()); + } + private static boolean hasNullableProp(HugeElement element, Id key) { return element.schemaLabel().nullableKeys().contains(key); } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/GraphTransaction.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/GraphTransaction.java index b0351016e8..971e4efec9 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/GraphTransaction.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/GraphTransaction.java @@ -1594,7 +1594,7 @@ private Iterator filterUnmatchedRecords( return false; } // Filter vertices/edges of deleting label - if (elem.schemaLabel().status() == SchemaStatus.DELETING && + if (elem.schemaLabel().status().deleting() && !query.showDeleting()) { return false; } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/schema/EdgeLabelRemoveCallable.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/schema/EdgeLabelRemoveCallable.java index 4697c98f97..2af51a57dc 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/schema/EdgeLabelRemoveCallable.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/schema/EdgeLabelRemoveCallable.java @@ -58,14 +58,22 @@ protected static void removeEdgeLabel(HugeGraphParams graph, Id id) { try { locks.lockWrites(LockUtil.EDGE_LABEL_DELETE, id); schemaTx.updateSchemaStatus(edgeLabel, SchemaStatus.DELETING); - for (Id indexId : indexIds) { - IndexLabelRemoveCallable.removeIndexLabel(graph, indexId); + try { + for (Id indexId : indexIds) { + IndexLabelRemoveCallable.removeIndexLabel(graph, indexId); + } + // Remove all edges which has matched label + graphTx.removeEdges(edgeLabel); + removeSchema(schemaTx, edgeLabel); + /* + * Should commit changes to backend store before release + * delete lock + */ + graph.graph().tx().commit(); + } catch (Throwable e) { + schemaTx.updateSchemaStatus(edgeLabel, SchemaStatus.UNDELETED); + throw e; } - // Remove all edges which has matched label - graphTx.removeEdges(edgeLabel); - removeSchema(schemaTx, edgeLabel); - // Should commit changes to backend store before release delete lock - graph.graph().tx().commit(); } finally { locks.unlock(); } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/schema/IndexLabelRemoveCallable.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/schema/IndexLabelRemoveCallable.java index f94b7ee368..f1eb3683c3 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/schema/IndexLabelRemoveCallable.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/schema/IndexLabelRemoveCallable.java @@ -54,14 +54,22 @@ protected static void removeIndexLabel(HugeGraphParams graph, Id id) { // TODO add update lock // Set index label to "deleting" status schemaTx.updateSchemaStatus(indexLabel, SchemaStatus.DELETING); - // Remove index data - // TODO: use event to replace direct call - graphTx.removeIndex(indexLabel); - // Remove label from indexLabels of vertex or edge label - removeIndexLabelFromBaseLabel(schemaTx, indexLabel); - removeSchema(schemaTx, indexLabel); - // Should commit changes to backend store before release delete lock - graph.graph().tx().commit(); + try { + // Remove index data + // TODO: use event to replace direct call + graphTx.removeIndex(indexLabel); + // Remove label from indexLabels of vertex or edge label + removeIndexLabelFromBaseLabel(schemaTx, indexLabel); + removeSchema(schemaTx, indexLabel); + /* + * Should commit changes to backend store before release + * delete lock + */ + graph.graph().tx().commit(); + } catch (Throwable e) { + schemaTx.updateSchemaStatus(indexLabel, SchemaStatus.INVALID); + throw e; + } } finally { locks.unlock(); } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/schema/RebuildIndexCallable.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/schema/RebuildIndexCallable.java index cfb931ef73..da6477170f 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/schema/RebuildIndexCallable.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/schema/RebuildIndexCallable.java @@ -119,19 +119,27 @@ private void rebuildIndex(SchemaLabel label, Collection indexLabelIds) { * They have different id lead to it can't compare and optimize */ graphTx.commit(); - if (label.type() == HugeType.VERTEX_LABEL) { - @SuppressWarnings("unchecked") - Consumer consumer = (Consumer) indexUpdater; - graphTx.traverseVerticesByLabel((VertexLabel) label, - consumer, false); - } else { - assert label.type() == HugeType.EDGE_LABEL; - @SuppressWarnings("unchecked") - Consumer consumer = (Consumer) indexUpdater; - graphTx.traverseEdgesByLabel((EdgeLabel) label, - consumer, false); + + try { + if (label.type() == HugeType.VERTEX_LABEL) { + @SuppressWarnings("unchecked") + Consumer consumer = (Consumer) indexUpdater; + graphTx.traverseVerticesByLabel((VertexLabel) label, + consumer, false); + } else { + assert label.type() == HugeType.EDGE_LABEL; + @SuppressWarnings("unchecked") + Consumer consumer = (Consumer) indexUpdater; + graphTx.traverseEdgesByLabel((EdgeLabel) label, + consumer, false); + } + graphTx.commit(); + } catch (Throwable e) { + for (IndexLabel il : ils) { + schemaTx.updateSchemaStatus(il, SchemaStatus.INVALID); + } + throw e; } - graphTx.commit(); for (IndexLabel il : ils) { schemaTx.updateSchemaStatus(il, SchemaStatus.CREATED); @@ -158,6 +166,9 @@ private void removeIndex(Collection indexLabelIds) { try { locks.lockWrites(LockUtil.INDEX_LABEL_DELETE, indexLabelIds); graphTx.removeIndex(il); + } catch (Throwable e) { + schemaTx.updateSchemaStatus(il, SchemaStatus.INVALID); + throw e; } finally { locks.unlock(); } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/schema/VertexLabelRemoveCallable.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/schema/VertexLabelRemoveCallable.java index 713cb797a9..7d3cf80205 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/schema/VertexLabelRemoveCallable.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/schema/VertexLabelRemoveCallable.java @@ -74,16 +74,23 @@ protected static void removeVertexLabel(HugeGraphParams graph, Id id) { try { locks.lockWrites(LockUtil.VERTEX_LABEL_DELETE, id); schemaTx.updateSchemaStatus(vertexLabel, SchemaStatus.DELETING); - for (Id indexLabelId : indexLabelIds) { - IndexLabelRemoveCallable.removeIndexLabel(graph, indexLabelId); + try { + for (Id ilId : indexLabelIds) { + IndexLabelRemoveCallable.removeIndexLabel(graph, ilId); + } + // TODO: use event to replace direct call + // Deleting a vertex will automatically deletes the held edge + graphTx.removeVertices(vertexLabel); + removeSchema(schemaTx, vertexLabel); + /* + * Should commit changes to backend store before release + * delete lock + */ + graph.graph().tx().commit(); + } catch (Throwable e) { + schemaTx.updateSchemaStatus(vertexLabel, SchemaStatus.UNDELETED); + throw e; } - - // TODO: use event to replace direct call - // Deleting a vertex will automatically deletes the held edge - graphTx.removeVertices(vertexLabel); - removeSchema(schemaTx, vertexLabel); - // Should commit changes to backend store before release delete lock - graph.graph().tx().commit(); } finally { locks.unlock(); } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/schema/builder/AbstractBuilder.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/schema/builder/AbstractBuilder.java index 9674a05096..a6b49e03e4 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/schema/builder/AbstractBuilder.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/schema/builder/AbstractBuilder.java @@ -30,9 +30,11 @@ import com.baidu.hugegraph.schema.EdgeLabel; import com.baidu.hugegraph.schema.IndexLabel; import com.baidu.hugegraph.schema.PropertyKey; +import com.baidu.hugegraph.schema.SchemaElement; import com.baidu.hugegraph.schema.VertexLabel; import com.baidu.hugegraph.type.HugeType; import com.baidu.hugegraph.type.define.GraphMode; +import com.baidu.hugegraph.type.define.SchemaStatus; import com.baidu.hugegraph.util.E; import com.baidu.hugegraph.util.LockUtil; @@ -77,6 +79,11 @@ protected V lockCheckAndCreateSchema(HugeType type, String name, } } + protected void updateSchemaStatus(SchemaElement element, + SchemaStatus status) { + this.transaction.updateSchemaStatus(element, status); + } + protected void checkSchemaIdIfRestoringMode(HugeType type, Id id) { if (this.transaction.graphMode() == GraphMode.RESTORING) { E.checkArgument(id != null, diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/schema/builder/IndexLabelBuilder.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/schema/builder/IndexLabelBuilder.java index 5c20056a3c..86393efa4b 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/schema/builder/IndexLabelBuilder.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/schema/builder/IndexLabelBuilder.java @@ -223,12 +223,17 @@ public IndexLabel.CreatedIndexLabel createWithTask() { // Create index label (just schema) indexLabel.status(SchemaStatus.CREATING); this.graph().addIndexLabel(schemaLabel, indexLabel); + try { + // Async rebuild index + Id rebuildTask = this.rebuildIndex(indexLabel, removeTasks); + E.checkNotNull(rebuildTask, "rebuild-index task"); - // Async rebuild index - Id rebuildTask = this.rebuildIndex(indexLabel, removeTasks); - E.checkNotNull(rebuildTask, "rebuild-index task"); - - return new IndexLabel.CreatedIndexLabel(indexLabel, rebuildTask); + return new IndexLabel.CreatedIndexLabel(indexLabel, + rebuildTask); + } catch (Throwable e) { + this.updateSchemaStatus(indexLabel, SchemaStatus.INVALID); + throw e; + } }); } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/type/define/SchemaStatus.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/type/define/SchemaStatus.java index f6cf218456..be906dfede 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/type/define/SchemaStatus.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/type/define/SchemaStatus.java @@ -27,7 +27,11 @@ public enum SchemaStatus implements SerialEnum { REBUILDING(3, "rebuilding"), - DELETING(4, "deleting"); + DELETING(4, "deleting"), + + UNDELETED(5, "undeleted"), + + INVALID(6, "invalid"); private byte code = 0; private String name = null; @@ -42,6 +46,14 @@ public enum SchemaStatus implements SerialEnum { this.name = name; } + public boolean ok() { + return this == CREATED; + } + + public boolean deleting() { + return this == DELETING || this == UNDELETED; + } + @Override public byte code() { return this.code;