feat(server): support new backend Hstore (#2560)

subtask of #2483
apache · Jul 10, 2024 · 9f89afc · 9f89afc
1 parent 37e71c4
commit 9f89afc
Show file tree

Hide file tree

Showing 108 changed files with 10,558 additions and 330 deletions.
diff --git a/.github/workflows/pd-store-ci.yml b/.github/workflows/pd-store-ci.yml
@@ -116,3 +116,73 @@ jobs:
         uses: codecov/[email protected]
         with:
           file: ${{ env.REPORT_DIR }}/*.xml
+
+  hstore:
+    # TODO: avoid duplicated env setup
+    runs-on: ubuntu-latest
+    env:
+      USE_STAGE: 'true' # Whether to include the stage repository.
+      TRAVIS_DIR: hugegraph-server/hugegraph-dist/src/assembly/travis
+      REPORT_DIR: target/site/jacoco
+      BACKEND: hstore
+      RELEASE_BRANCH: ${{ startsWith(github.ref_name, 'release-') || startsWith(github.ref_name, 'test-') || startsWith(github.base_ref, 'release-') }}
+
+    steps:
+      - name: Install JDK 11
+        uses: actions/setup-java@v3
+        with:
+          java-version: '11'
+          distribution: 'zulu'
+
+      - name: Cache Maven packages
+        uses: actions/cache@v3
+        with:
+          path: ~/.m2
+          key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
+          restore-keys: ${{ runner.os }}-m2
+
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 2
+
+      - name: use staged maven repo settings
+        if: ${{ env.USE_STAGE == 'true' }}
+        run: |
+          cp $HOME/.m2/settings.xml /tmp/settings.xml
+          mv -vf .github/configs/settings.xml $HOME/.m2/settings.xml
+
+      - name: Package
+        run: |
+          mvn clean package -U -Dmaven.javadoc.skip=true -Dmaven.test.skip=true -ntp
+
+      - name: Prepare env and service
+        run: |
+          $TRAVIS_DIR/install-backend.sh $BACKEND
+
+      - name: Run unit test
+        run: |
+          $TRAVIS_DIR/run-unit-test.sh $BACKEND
+
+      - name: Run core test
+        run: |
+          $TRAVIS_DIR/run-core-test.sh $BACKEND
+
+      - name: Run api test
+        run: |
+          $TRAVIS_DIR/run-api-test.sh $BACKEND $REPORT_DIR
+
+      - name: Run raft test
+        if: ${{ env.BACKEND == 'rocksdb' }}
+        run: |
+          $TRAVIS_DIR/run-api-test-for-raft.sh $BACKEND $REPORT_DIR
+
+      - name: Run TinkerPop test
+        if: ${{ env.RELEASE_BRANCH == 'true' }}
+        run: |
+          $TRAVIS_DIR/run-tinkerpop-test.sh $BACKEND tinkerpop
+
+      - name: Upload coverage to Codecov
+        uses: codecov/[email protected]
+        with:
+          file: ${{ env.REPORT_DIR }}/*.xml
diff --git a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/job/TaskAPI.java b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/job/TaskAPI.java
@@ -136,11 +136,12 @@ public Map<String, Object> get(@Context GraphManager manager,
     @RedirectFilter.RedirectMasterRole
     public void delete(@Context GraphManager manager,
                        @PathParam("graph") String graph,
-                       @PathParam("id") long id) {
+                       @PathParam("id") long id,
+                       @DefaultValue("false") @QueryParam("force") boolean force) {
         LOG.debug("Graph [{}] delete task: {}", graph, id);
 
         TaskScheduler scheduler = graph(manager, graph).taskScheduler();
-        HugeTask<?> task = scheduler.delete(IdGenerator.of(id));
+        HugeTask<?> task = scheduler.delete(IdGenerator.of(id), force);
         E.checkArgument(task != null, "There is no task with id '%s'", id);
     }
 

diff --git a/...raph-server/hugegraph-api/src/main/java/org/apache/hugegraph/auth/HugeGraphAuthProxy.java b/...raph-server/hugegraph-api/src/main/java/org/apache/hugegraph/auth/HugeGraphAuthProxy.java
@@ -26,6 +26,7 @@
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
+import java.util.concurrent.Callable;
 import java.util.concurrent.Future;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.ThreadFactory;
@@ -71,6 +72,7 @@
 import org.apache.hugegraph.structure.HugeFeatures;
 import org.apache.hugegraph.structure.HugeVertex;
 import org.apache.hugegraph.task.HugeTask;
+import org.apache.hugegraph.task.ServerInfoManager;
 import org.apache.hugegraph.task.TaskManager;
 import org.apache.hugegraph.task.TaskScheduler;
 import org.apache.hugegraph.task.TaskStatus;
@@ -1085,10 +1087,10 @@ public <V> Iterator<HugeTask<V>> tasks(TaskStatus status,
         }
 
         @Override
-        public <V> HugeTask<V> delete(Id id) {
+        public <V> HugeTask<V> delete(Id id, boolean force) {
             verifyTaskPermission(HugePermission.DELETE,
                                  this.taskScheduler.task(id));
-            return this.taskScheduler.delete(id);
+            return this.taskScheduler.delete(id, force);
         }
 
         @Override
@@ -1124,6 +1126,36 @@ public void checkRequirement(String op) {
             this.taskScheduler.checkRequirement(op);
         }
 
+        @Override
+        public <V> V call(Callable<V> callable) {
+            verifyAnyPermission();
+            return this.taskScheduler.call(callable);
+        }
+
+        @Override
+        public <V> V call(Runnable runnable) {
+            verifyAnyPermission();
+            return this.taskScheduler.call(runnable);
+        }
+
+        @Override
+        public ServerInfoManager serverManager() {
+            verifyAnyPermission();
+            return this.taskScheduler.serverManager();
+        }
+
+        @Override
+        public String graphName() {
+            verifyAnyPermission();
+            return this.taskScheduler.graphName();
+        }
+
+        @Override
+        public void taskDone(HugeTask<?> task) {
+            verifyAnyPermission();
+            this.taskScheduler.taskDone(task);
+        }
+
         private void verifyTaskPermission(HugePermission actionPerm) {
             verifyPermission(actionPerm, ResourceType.TASK);
         }

diff --git a/...-cassandra/src/main/java/org/apache/hugegraph/backend/store/cassandra/CassandraStore.java b/...-cassandra/src/main/java/org/apache/hugegraph/backend/store/cassandra/CassandraStore.java
@@ -592,7 +592,7 @@ protected Collection<CassandraTable> tables() {
 
     @Override
     protected final CassandraTable table(HugeType type) {
-        return this.table(type.string());
+        return this.table(convertTaskOrServerToVertex(type).string());
     }
 
     protected final CassandraTable table(String name) {

diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/HugeGraphParams.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/HugeGraphParams.java
@@ -24,6 +24,7 @@
 import org.apache.hugegraph.backend.store.BackendStore;
 import org.apache.hugegraph.backend.store.ram.RamTable;
 import org.apache.hugegraph.backend.tx.GraphTransaction;
+import org.apache.hugegraph.backend.tx.ISchemaTransaction;
 import org.apache.hugegraph.backend.tx.SchemaTransaction;
 import org.apache.hugegraph.config.HugeConfig;
 import org.apache.hugegraph.event.EventHub;
@@ -47,7 +48,7 @@ public interface HugeGraphParams {
 
     GraphReadMode readMode();
 
-    SchemaTransaction schemaTransaction();
+    ISchemaTransaction schemaTransaction();
 
     GraphTransaction systemTransaction();
 
@@ -94,4 +95,6 @@ public interface HugeGraphParams {
     RamTable ramtable();
 
     <T> void submitEphemeralJob(EphemeralJob<T> job);
+
+    String schedulerType();
 }
diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/StandardHugeGraph.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/StandardHugeGraph.java
@@ -37,6 +37,7 @@
 import org.apache.hugegraph.backend.cache.CacheNotifier.SchemaCacheNotifier;
 import org.apache.hugegraph.backend.cache.CachedGraphTransaction;
 import org.apache.hugegraph.backend.cache.CachedSchemaTransaction;
+import org.apache.hugegraph.backend.cache.CachedSchemaTransactionV2;
 import org.apache.hugegraph.backend.id.Id;
 import org.apache.hugegraph.backend.id.IdGenerator;
 import org.apache.hugegraph.backend.id.SnowflakeIdGenerator;
@@ -52,7 +53,7 @@
 import org.apache.hugegraph.backend.store.raft.RaftGroupManager;
 import org.apache.hugegraph.backend.store.ram.RamTable;
 import org.apache.hugegraph.backend.tx.GraphTransaction;
-import org.apache.hugegraph.backend.tx.SchemaTransaction;
+import org.apache.hugegraph.backend.tx.ISchemaTransaction;
 import org.apache.hugegraph.config.CoreOptions;
 import org.apache.hugegraph.config.HugeConfig;
 import org.apache.hugegraph.config.TypedOption;
@@ -69,6 +70,7 @@
 import org.apache.hugegraph.masterelection.RoleElectionStateMachine;
 import org.apache.hugegraph.masterelection.StandardClusterRoleStore;
 import org.apache.hugegraph.masterelection.StandardRoleElectionStateMachine;
+import org.apache.hugegraph.meta.MetaManager;
 import org.apache.hugegraph.perf.PerfUtil.Watched;
 import org.apache.hugegraph.rpc.RpcServiceConfig4Client;
 import org.apache.hugegraph.rpc.RpcServiceConfig4Server;
@@ -176,6 +178,8 @@ public class StandardHugeGraph implements HugeGraph {
 
     private final RamTable ramtable;
 
+    private final String schedulerType;
+
     public StandardHugeGraph(HugeConfig config) {
         this.params = new StandardHugeGraphParams();
         this.configuration = config;
@@ -209,6 +213,7 @@ public StandardHugeGraph(HugeConfig config) {
         this.closed = false;
         this.mode = GraphMode.NONE;
         this.readMode = GraphReadMode.OLTP_ONLY;
+        this.schedulerType = config.get(CoreOptions.SCHEDULER_TYPE);
 
         LockUtil.init(this.name);
 
@@ -221,6 +226,13 @@ public StandardHugeGraph(HugeConfig config) {
             throw new HugeException(message, e);
         }
 
+        if (isHstore()) {
+            // TODO: parameterize the remaining configurations
+            MetaManager.instance().connect("hg", MetaManager.MetaDriverType.PD,
+                                           "ca", "ca", "ca",
+                                           config.get(CoreOptions.PD_PEERS));
+        }
+
         try {
             this.tx = new TinkerPopTransaction(this);
             boolean supportsPersistence = this.backendStoreFeatures().supportsPersistence();
@@ -457,9 +469,18 @@ private void clearVertexCache() {
         }
     }
 
-    private SchemaTransaction openSchemaTransaction() throws HugeException {
+    private boolean isHstore() {
+        return this.storeProvider.isHstore();
+    }
+
+    private ISchemaTransaction openSchemaTransaction() throws HugeException {
         this.checkGraphNotClosed();
         try {
+            if (isHstore()) {
+                return new CachedSchemaTransactionV2(
+                    MetaManager.instance().metaDriver(),
+                    MetaManager.instance().cluster(), this.params);
+            }
             return new CachedSchemaTransaction(this.params, loadSchemaStore());
         } catch (BackendException e) {
             String message = "Failed to open schema transaction";
@@ -504,11 +525,14 @@ private BackendStore loadGraphStore() {
     }
 
     private BackendStore loadSystemStore() {
+        if (isHstore()) {
+            return this.storeProvider.loadGraphStore(this.configuration);
+        }
         return this.storeProvider.loadSystemStore(this.configuration);
     }
 
     @Watched
-    private SchemaTransaction schemaTransaction() {
+    private ISchemaTransaction schemaTransaction() {
         this.checkGraphNotClosed();
         /*
          * NOTE: each schema operation will be auto committed,
@@ -1196,7 +1220,7 @@ public GraphReadMode readMode() {
         }
 
         @Override
-        public SchemaTransaction schemaTransaction() {
+        public ISchemaTransaction schemaTransaction() {
             return StandardHugeGraph.this.schemaTransaction();
         }
 
@@ -1316,6 +1340,11 @@ public RamTable ramtable() {
         public <T> void submitEphemeralJob(EphemeralJob<T> job) {
             this.ephemeralJobQueue.add(job);
         }
+
+        @Override
+        public String schedulerType() {
+            return StandardHugeGraph.this.schedulerType;
+        }
     }
 
     private class TinkerPopTransaction extends AbstractThreadLocalTransaction {
@@ -1447,7 +1476,7 @@ private void setClosed() {
             }
         }
 
-        private SchemaTransaction schemaTransaction() {
+        private ISchemaTransaction schemaTransaction() {
             return this.getOrNewTransaction().schemaTx;
         }
 
@@ -1468,7 +1497,7 @@ private Txs getOrNewTransaction() {
 
             Txs txs = this.transactions.get();
             if (txs == null) {
-                SchemaTransaction schemaTransaction = null;
+                ISchemaTransaction schemaTransaction = null;
                 SysTransaction sysTransaction = null;
                 GraphTransaction graphTransaction = null;
                 try {
@@ -1511,12 +1540,12 @@ private void destroyTransaction() {
 
     private static final class Txs {
 
-        private final SchemaTransaction schemaTx;
+        private final ISchemaTransaction schemaTx;
         private final SysTransaction systemTx;
         private final GraphTransaction graphTx;
         private long openedTime;
 
-        public Txs(SchemaTransaction schemaTx, SysTransaction systemTx,
+        public Txs(ISchemaTransaction schemaTx, SysTransaction systemTx,
                    GraphTransaction graphTx) {
             assert schemaTx != null && systemTx != null && graphTx != null;
             this.schemaTx = schemaTx;

diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/auth/HugeAccess.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/auth/HugeAccess.java
@@ -228,4 +228,9 @@ private String[] initProperties() {
             return super.initProperties(props);
         }
     }
+
+    public static HugeAccess fromMap(Map<String, Object> map) {
+        HugeAccess access = new HugeAccess(null, null, null);
+        return fromMap(map, access);
+    }
 }
diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/auth/HugeBelong.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/auth/HugeBelong.java
@@ -32,10 +32,15 @@
 
 public class HugeBelong extends Relationship {
 
+    public static final String UG = "ug";
+    public static final String UR = "ur";
+    public static final String GR = "gr";
+    public static final String ALL = "*";
     private static final long serialVersionUID = -7242751631755533423L;
 
     private final Id user;
     private final Id group;
+    private String link;
     private String description;
 
     public HugeBelong(Id user, Id group) {
@@ -74,6 +79,10 @@ public Id target() {
         return this.group;
     }
 
+    public String link() {
+        return this.link;
+    }
+
     public String description() {
         return this.description;
     }
@@ -193,4 +202,9 @@ private String[] initProperties() {
             return super.initProperties(props);
         }
     }
+
+    public static HugeBelong fromMap(Map<String, Object> map) {
+        HugeBelong belong = new HugeBelong(null, null);
+        return fromMap(map, belong);
+    }
 }