Some bug fix for load task and schema manage (hugegraph#246)

OshotOkill · Nov 3, 2020 · 3f14a56 · 3f14a56
1 parent 1d74cbc
commit 3f14a56
Show file tree

Hide file tree

Showing 45 changed files with 871 additions and 648 deletions.
diff --git a/hubble-be/pom.xml b/hubble-be/pom.xml
@@ -32,10 +32,6 @@
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-cache</artifactId>
         </dependency>
-        <dependency>
-            <groupId>org.springframework.boot</groupId>
-            <artifactId>spring-boot-starter-aop</artifactId>
-        </dependency>
         <dependency>
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-test</artifactId>
@@ -98,38 +94,11 @@
                 </exclusion>
             </exclusions>
         </dependency>
-        <dependency>
-            <groupId>com.baidu.hugegraph</groupId>
-            <artifactId>hugegraph-client</artifactId>
-            <version>1.8.10</version>
-            <exclusions>
-                <exclusion>
-                    <groupId>org.apache.logging.log4j</groupId>
-                    <artifactId>log4j-api</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>org.apache.logging.log4j</groupId>
-                    <artifactId>log4j-core</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>org.apache.logging.log4j</groupId>
-                    <artifactId>log4j-slf4j-impl</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>com.baidu.hugegraph</groupId>
-                    <artifactId>hugegraph-common</artifactId>
-                </exclusion>
-            </exclusions>
-        </dependency>
         <dependency>
             <groupId>com.baidu.hugegraph</groupId>
             <artifactId>hugegraph-loader</artifactId>
-            <version>0.10.4</version>
+            <version>0.10.5</version>
             <exclusions>
-                <exclusion>
-                    <groupId>com.baidu.hugegraph</groupId>
-                    <artifactId>hugegraph-client</artifactId>
-                </exclusion>
                 <exclusion>
                     <groupId>org.apache.logging.log4j</groupId>
                     <artifactId>log4j-api</artifactId>

diff --git a/hubble-be/src/main/java/com/baidu/hugegraph/config/AsyncConfig.java b/hubble-be/src/main/java/com/baidu/hugegraph/config/AsyncConfig.java
@@ -33,9 +33,9 @@ public class AsyncConfig implements AsyncConfigurer {
     @Override
     public Executor getAsyncExecutor() {
         ThreadPoolTaskExecutor taskExecutor = new ThreadPoolTaskExecutor();
-        taskExecutor.setCorePoolSize(4);
-        taskExecutor.setMaxPoolSize(8);
-        taskExecutor.setQueueCapacity(16);
+        taskExecutor.setCorePoolSize(8);
+        taskExecutor.setMaxPoolSize(16);
+        taskExecutor.setQueueCapacity(1024);
         taskExecutor.initialize();
         return taskExecutor;
     }

diff --git a/hubble-be/src/main/java/com/baidu/hugegraph/controller/GraphConnectionController.java b/hubble-be/src/main/java/com/baidu/hugegraph/controller/GraphConnectionController.java
@@ -42,7 +42,6 @@
 import com.baidu.hugegraph.driver.HugeClient;
 import com.baidu.hugegraph.entity.GraphConnection;
 import com.baidu.hugegraph.exception.ExternalException;
-import com.baidu.hugegraph.exception.InternalException;
 import com.baidu.hugegraph.options.HubbleOptions;
 import com.baidu.hugegraph.service.GraphConnectionService;
 import com.baidu.hugegraph.service.HugeClientPoolService;
@@ -149,9 +148,7 @@ public GraphConnection create(@RequestBody GraphConnection newEntity) {
         Ex.check(verifyResult.isEnabled(), Constant.STATUS_UNAUTHORIZED,
                  verifyResult.getMessage());
 
-        if (this.connService.save(newEntity) != 1) {
-            throw new InternalException("entity.insert.failed", newEntity);
-        }
+        this.connService.save(newEntity);
         this.poolService.put(newEntity, client);
         return newEntity;
     }
@@ -180,9 +177,7 @@ public GraphConnection update(@PathVariable("id") int id,
         Ex.check(verifyResult.isEnabled(), Constant.STATUS_UNAUTHORIZED,
                  verifyResult.getMessage());
 
-        if (this.connService.update(entity) != 1) {
-            throw new InternalException("entity.update.failed", entity);
-        }
+        this.connService.update(entity);
         this.poolService.put(entity, client);
         return entity;
     }
@@ -193,10 +188,7 @@ public GraphConnection delete(@PathVariable("id") int id) {
         if (oldEntity == null) {
             throw new ExternalException("graph-connection.not-exist.id", id);
         }
-        int rows = this.connService.remove(id);
-        if (rows != 1) {
-            throw new InternalException("entity.delete.failed", oldEntity);
-        }
+        this.connService.remove(id);
         this.poolService.remove(oldEntity);
         this.licenseService.updateAllGraphStatus();
         return oldEntity;

diff --git a/hubble-be/src/main/java/com/baidu/hugegraph/controller/SettingController.java b/hubble-be/src/main/java/com/baidu/hugegraph/controller/SettingController.java
@@ -33,7 +33,6 @@
 
 import com.baidu.hugegraph.common.Constant;
 import com.baidu.hugegraph.entity.UserInfo;
-import com.baidu.hugegraph.exception.InternalException;
 import com.baidu.hugegraph.service.UserInfoService;
 import com.baidu.hugegraph.util.E;
 
@@ -62,16 +61,10 @@ public UserInfo config(@RequestParam(value = "locale",
                                .username(username)
                                .locale(locale)
                                .build();
-            int rows = this.service.save(userInfo);
-            if (rows == 0) {
-                throw new InternalException("entity.insert.failed");
-            }
+            this.service.save(userInfo);
         } else {
             userInfo.setLocale(locale);
-            int rows = this.service.update(userInfo);
-            if (rows == 0) {
-                throw new InternalException("entity.update.failed");
-            }
+            this.service.update(userInfo);
         }
 
         Cookie cookie = new Cookie(Constant.COOKIE_USER, userInfo.getUsername());

diff --git a/hubble-be/src/main/java/com/baidu/hugegraph/controller/load/FileMappingController.java b/hubble-be/src/main/java/com/baidu/hugegraph/controller/load/FileMappingController.java
@@ -37,17 +37,19 @@
 
 import com.baidu.hugegraph.common.Constant;
 import com.baidu.hugegraph.controller.BaseController;
+import com.baidu.hugegraph.entity.enums.JobStatus;
 import com.baidu.hugegraph.entity.load.EdgeMapping;
 import com.baidu.hugegraph.entity.load.ElementMapping;
 import com.baidu.hugegraph.entity.load.FileMapping;
 import com.baidu.hugegraph.entity.load.FileSetting;
+import com.baidu.hugegraph.entity.load.JobManager;
 import com.baidu.hugegraph.entity.load.LoadParameter;
 import com.baidu.hugegraph.entity.load.VertexMapping;
 import com.baidu.hugegraph.entity.schema.EdgeLabelEntity;
 import com.baidu.hugegraph.entity.schema.VertexLabelEntity;
 import com.baidu.hugegraph.exception.ExternalException;
-import com.baidu.hugegraph.exception.InternalException;
 import com.baidu.hugegraph.service.load.FileMappingService;
+import com.baidu.hugegraph.service.load.JobManagerService;
 import com.baidu.hugegraph.service.schema.EdgeLabelService;
 import com.baidu.hugegraph.service.schema.VertexLabelService;
 import com.baidu.hugegraph.util.Ex;
@@ -67,6 +69,8 @@ public class FileMappingController extends BaseController {
     private EdgeLabelService elService;
     @Autowired
     private FileMappingService service;
+    @Autowired
+    private JobManagerService jobService;
 
     @GetMapping
     public IPage<FileMapping> list(@PathVariable("connId") int connId,
@@ -99,9 +103,7 @@ public void delete(@PathVariable("id") int id) {
         }
 
         this.service.deleteDiskFile(mapping);
-        if (this.service.remove(id) != 1) {
-            throw new InternalException("entity.delete.failed", mapping);
-        }
+        this.service.remove(id);
     }
 
     @DeleteMapping
@@ -130,18 +132,14 @@ public FileMapping fileSetting(@PathVariable("id") int id,
         if (mapping == null) {
             throw new ExternalException("load.file-mapping.not-exist.id", id);
         }
-        // unescape \\t to \t
-        newEntity.unescapeDelimiterIfNeeded();
+        // Change format to TEXT if needed
+        newEntity.changeFormatIfNeeded();
         FileSetting oldEntity = mapping.getFileSetting();
         FileSetting entity = this.mergeEntity(oldEntity, newEntity);
         mapping.setFileSetting(entity);
         // Read column names and values then fill it
         this.service.extractColumns(mapping);
-        if (this.service.update(mapping) != 1) {
-            throw new InternalException("entity.update.failed", mapping);
-        }
-        // escape \t to \\t
-        mapping.getFileSetting().escapeDelimiterIfNeeded();
+        this.service.update(mapping);
         return mapping;
     }
 
@@ -157,9 +155,7 @@ public FileMapping addVertexMapping(@PathVariable("connId") int connId,
 
         newEntity.setId(HubbleUtil.generateSimpleId());
         mapping.getVertexMappings().add(newEntity);
-        if (this.service.update(mapping) != 1) {
-            throw new InternalException("entity.update.failed", mapping);
-        }
+        this.service.update(mapping);
         return mapping;
     }
 
@@ -182,9 +178,7 @@ public FileMapping updateVertexMapping(@PathVariable("connId") int connId,
         Set<VertexMapping> vertexMappings = mapping.getVertexMappings();
         vertexMappings.remove(vertexMapping);
         vertexMappings.add(newEntity);
-        if (this.service.update(mapping) != 1) {
-            throw new InternalException("entity.update.failed", mapping);
-        }
+        this.service.update(mapping);
         return mapping;
     }
 
@@ -202,9 +196,7 @@ public FileMapping deleteVertexMapping(@PathVariable("id") int id,
             throw new ExternalException(
                       "load.file-mapping.vertex-mapping.not-exist.id", vmid);
         }
-        if (this.service.update(mapping) != 1) {
-            throw new InternalException("entity.update.failed", mapping);
-        }
+        this.service.update(mapping);
         return mapping;
     }
 
@@ -220,9 +212,7 @@ public FileMapping addEdgeMapping(@PathVariable("connId") int connId,
 
         newEntity.setId(HubbleUtil.generateSimpleId());
         mapping.getEdgeMappings().add(newEntity);
-        if (this.service.update(mapping) != 1) {
-            throw new InternalException("entity.update.failed", mapping);
-        }
+        this.service.update(mapping);
         return mapping;
     }
 
@@ -245,9 +235,7 @@ public FileMapping updateEdgeMapping(@PathVariable("connId") int connId,
         Set<EdgeMapping> edgeMappings = mapping.getEdgeMappings();
         edgeMappings.remove(edgeMapping);
         edgeMappings.add(newEntity);
-        if (this.service.update(mapping) != 1) {
-            throw new InternalException("entity.update.failed", mapping);
-        }
+        this.service.update(mapping);
         return mapping;
     }
 
@@ -265,9 +253,7 @@ public FileMapping deleteEdgeMapping(@PathVariable("id") int id,
             throw new ExternalException(
                       "load.file-mapping.edge-mapping.not-exist.id", emid);
         }
-        if (this.service.update(mapping) != 1) {
-            throw new InternalException("entity.update.failed", mapping);
-        }
+        this.service.update(mapping);
         return mapping;
     }
 
@@ -282,12 +268,22 @@ public void loadParameter(@RequestBody LoadParameter newEntity) {
             LoadParameter oldEntity = mapping.getLoadParameter();
             LoadParameter entity = this.mergeEntity(oldEntity, newEntity);
             mapping.setLoadParameter(entity);
-            if (this.service.update(mapping) != 1) {
-                throw new InternalException("entity.update.failed", mapping);
-            }
+            this.service.update(mapping);
         }
     }
 
+    @PutMapping("next-step")
+    public JobManager nextStep(@PathVariable("jobId") int jobId) {
+        JobManager jobEntity = this.jobService.get(jobId);
+        Ex.check(jobEntity != null, "job-manager.not-exist.id", jobId);
+        Ex.check(jobEntity.getJobStatus() == JobStatus.MAPPING,
+                 "job.manager.status.unexpected",
+                 JobStatus.MAPPING, jobEntity.getJobStatus());
+        jobEntity.setJobStatus(JobStatus.SETTING);
+        this.jobService.update(jobEntity);
+        return jobEntity;
+    }
+
     private void checkVertexMappingValid(int connId, VertexMapping vertexMapping,
                                          FileMapping fileMapping) {
         VertexLabelEntity vl = this.vlService.get(vertexMapping.getLabel(),