Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Some bug fix for load task and schema manage #246

Merged
merged 7 commits into from
Oct 30, 2020
Merged

Some bug fix for load task and schema manage #246

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
f92fac4
Some bug fix for load task and schema manage
Linary Sep 29, 2020
abf84a3
tiny improve
Linary Oct 19, 2020
653896d
add API for JobManager status controling
Linary Oct 23, 2020
405b75c
Use background thread to delete unfinished files
Linary Oct 26, 2020
eaf4494
wrap save/update/remove method, and rename finish API to next-step
Linary Oct 27, 2020
2ae54a7
tiny improve
Linary Oct 28, 2020
236472f
tiny improve
Linary Oct 30, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 1 addition & 32 deletions hubble-be/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,6 @@
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-cache</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-aop</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
Expand Down Expand Up @@ -98,38 +94,11 @@
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.baidu.hugegraph</groupId>
<artifactId>hugegraph-client</artifactId>
<version>1.8.10</version>
<exclusions>
<exclusion>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j-impl</artifactId>
</exclusion>
<exclusion>
<groupId>com.baidu.hugegraph</groupId>
<artifactId>hugegraph-common</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.baidu.hugegraph</groupId>
<artifactId>hugegraph-loader</artifactId>
<version>0.10.4</version>
<version>0.10.5</version>
<exclusions>
<exclusion>
<groupId>com.baidu.hugegraph</groupId>
<artifactId>hugegraph-client</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
Expand Down
6 changes: 3 additions & 3 deletions hubble-be/src/main/java/com/baidu/hugegraph/config/AsyncConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,9 @@ public class AsyncConfig implements AsyncConfigurer {
@Override
public Executor getAsyncExecutor() {
ThreadPoolTaskExecutor taskExecutor = new ThreadPoolTaskExecutor();
taskExecutor.setCorePoolSize(4);
taskExecutor.setMaxPoolSize(8);
taskExecutor.setQueueCapacity(16);
taskExecutor.setCorePoolSize(8);
taskExecutor.setMaxPoolSize(16);
taskExecutor.setQueueCapacity(1024);
taskExecutor.initialize();
return taskExecutor;
}
Expand Down
14 changes: 3 additions & 11 deletions hubble-be/src/main/java/com/baidu/hugegraph/controller/GraphConnectionController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,6 @@
import com.baidu.hugegraph.driver.HugeClient;
import com.baidu.hugegraph.entity.GraphConnection;
import com.baidu.hugegraph.exception.ExternalException;
import com.baidu.hugegraph.exception.InternalException;
import com.baidu.hugegraph.options.HubbleOptions;
import com.baidu.hugegraph.service.GraphConnectionService;
import com.baidu.hugegraph.service.HugeClientPoolService;
Expand Down Expand Up @@ -149,9 +148,7 @@ public GraphConnection create(@RequestBody GraphConnection newEntity) {
Ex.check(verifyResult.isEnabled(), Constant.STATUS_UNAUTHORIZED,
verifyResult.getMessage());

if (this.connService.save(newEntity) != 1) {
throw new InternalException("entity.insert.failed", newEntity);
}
this.connService.save(newEntity);
this.poolService.put(newEntity, client);
return newEntity;
}
Expand Down Expand Up @@ -180,9 +177,7 @@ public GraphConnection update(@PathVariable("id") int id,
Ex.check(verifyResult.isEnabled(), Constant.STATUS_UNAUTHORIZED,
verifyResult.getMessage());

if (this.connService.update(entity) != 1) {
throw new InternalException("entity.update.failed", entity);
}
this.connService.update(entity);
this.poolService.put(entity, client);
return entity;
}
Expand All @@ -193,10 +188,7 @@ public GraphConnection delete(@PathVariable("id") int id) {
if (oldEntity == null) {
throw new ExternalException("graph-connection.not-exist.id", id);
}
int rows = this.connService.remove(id);
if (rows != 1) {
throw new InternalException("entity.delete.failed", oldEntity);
}
this.connService.remove(id);
this.poolService.remove(oldEntity);
this.licenseService.updateAllGraphStatus();
return oldEntity;
Expand Down
11 changes: 2 additions & 9 deletions hubble-be/src/main/java/com/baidu/hugegraph/controller/SettingController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@

import com.baidu.hugegraph.common.Constant;
import com.baidu.hugegraph.entity.UserInfo;
import com.baidu.hugegraph.exception.InternalException;
import com.baidu.hugegraph.service.UserInfoService;
import com.baidu.hugegraph.util.E;

Expand Down Expand Up @@ -62,16 +61,10 @@ public UserInfo config(@RequestParam(value = "locale",
.username(username)
.locale(locale)
.build();
int rows = this.service.save(userInfo);
if (rows == 0) {
throw new InternalException("entity.insert.failed");
}
this.service.save(userInfo);
} else {
userInfo.setLocale(locale);
int rows = this.service.update(userInfo);
if (rows == 0) {
throw new InternalException("entity.update.failed");
}
this.service.update(userInfo);
}

Cookie cookie = new Cookie(Constant.COOKIE_USER, userInfo.getUsername());
Expand Down
60 changes: 28 additions & 32 deletions hubble-be/src/main/java/com/baidu/hugegraph/controller/load/FileMappingController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,17 +37,19 @@

import com.baidu.hugegraph.common.Constant;
import com.baidu.hugegraph.controller.BaseController;
import com.baidu.hugegraph.entity.enums.JobStatus;
import com.baidu.hugegraph.entity.load.EdgeMapping;
import com.baidu.hugegraph.entity.load.ElementMapping;
import com.baidu.hugegraph.entity.load.FileMapping;
import com.baidu.hugegraph.entity.load.FileSetting;
import com.baidu.hugegraph.entity.load.JobManager;
import com.baidu.hugegraph.entity.load.LoadParameter;
import com.baidu.hugegraph.entity.load.VertexMapping;
import com.baidu.hugegraph.entity.schema.EdgeLabelEntity;
import com.baidu.hugegraph.entity.schema.VertexLabelEntity;
import com.baidu.hugegraph.exception.ExternalException;
import com.baidu.hugegraph.exception.InternalException;
import com.baidu.hugegraph.service.load.FileMappingService;
import com.baidu.hugegraph.service.load.JobManagerService;
import com.baidu.hugegraph.service.schema.EdgeLabelService;
import com.baidu.hugegraph.service.schema.VertexLabelService;
import com.baidu.hugegraph.util.Ex;
Expand All @@ -67,6 +69,8 @@ public class FileMappingController extends BaseController {
private EdgeLabelService elService;
@Autowired
private FileMappingService service;
@Autowired
private JobManagerService jobService;

@GetMapping
public IPage<FileMapping> list(@PathVariable("connId") int connId,
Expand Down Expand Up @@ -99,9 +103,7 @@ public void delete(@PathVariable("id") int id) {
}

this.service.deleteDiskFile(mapping);
if (this.service.remove(id) != 1) {
throw new InternalException("entity.delete.failed", mapping);
}
this.service.remove(id);
}

@DeleteMapping
Expand Down Expand Up @@ -130,18 +132,14 @@ public FileMapping fileSetting(@PathVariable("id") int id,
if (mapping == null) {
throw new ExternalException("load.file-mapping.not-exist.id", id);
}
// unescape \\t to \t
newEntity.unescapeDelimiterIfNeeded();
// Change format to TEXT if needed
newEntity.changeFormatIfNeeded();
FileSetting oldEntity = mapping.getFileSetting();
FileSetting entity = this.mergeEntity(oldEntity, newEntity);
mapping.setFileSetting(entity);
// Read column names and values then fill it
this.service.extractColumns(mapping);
if (this.service.update(mapping) != 1) {
throw new InternalException("entity.update.failed", mapping);
}
// escape \t to \\t
mapping.getFileSetting().escapeDelimiterIfNeeded();
this.service.update(mapping);
return mapping;
}

Expand All @@ -157,9 +155,7 @@ public FileMapping addVertexMapping(@PathVariable("connId") int connId,

newEntity.setId(HubbleUtil.generateSimpleId());
mapping.getVertexMappings().add(newEntity);
if (this.service.update(mapping) != 1) {
throw new InternalException("entity.update.failed", mapping);
}
this.service.update(mapping);
return mapping;
}

Expand All @@ -182,9 +178,7 @@ public FileMapping updateVertexMapping(@PathVariable("connId") int connId,
Set<VertexMapping> vertexMappings = mapping.getVertexMappings();
vertexMappings.remove(vertexMapping);
vertexMappings.add(newEntity);
if (this.service.update(mapping) != 1) {
throw new InternalException("entity.update.failed", mapping);
}
this.service.update(mapping);
return mapping;
}

Expand All @@ -202,9 +196,7 @@ public FileMapping deleteVertexMapping(@PathVariable("id") int id,
throw new ExternalException(
"load.file-mapping.vertex-mapping.not-exist.id", vmid);
}
if (this.service.update(mapping) != 1) {
throw new InternalException("entity.update.failed", mapping);
}
this.service.update(mapping);
return mapping;
}

Expand All @@ -220,9 +212,7 @@ public FileMapping addEdgeMapping(@PathVariable("connId") int connId,

newEntity.setId(HubbleUtil.generateSimpleId());
mapping.getEdgeMappings().add(newEntity);
if (this.service.update(mapping) != 1) {
throw new InternalException("entity.update.failed", mapping);
}
this.service.update(mapping);
return mapping;
}

Expand All @@ -245,9 +235,7 @@ public FileMapping updateEdgeMapping(@PathVariable("connId") int connId,
Set<EdgeMapping> edgeMappings = mapping.getEdgeMappings();
edgeMappings.remove(edgeMapping);
edgeMappings.add(newEntity);
if (this.service.update(mapping) != 1) {
throw new InternalException("entity.update.failed", mapping);
}
this.service.update(mapping);
return mapping;
}

Expand All @@ -265,9 +253,7 @@ public FileMapping deleteEdgeMapping(@PathVariable("id") int id,
throw new ExternalException(
"load.file-mapping.edge-mapping.not-exist.id", emid);
}
if (this.service.update(mapping) != 1) {
throw new InternalException("entity.update.failed", mapping);
}
this.service.update(mapping);
return mapping;
}

Expand All @@ -282,12 +268,22 @@ public void loadParameter(@RequestBody LoadParameter newEntity) {
LoadParameter oldEntity = mapping.getLoadParameter();
LoadParameter entity = this.mergeEntity(oldEntity, newEntity);
mapping.setLoadParameter(entity);
if (this.service.update(mapping) != 1) {
throw new InternalException("entity.update.failed", mapping);
}
this.service.update(mapping);
}
}

@PutMapping("next-step")
public JobManager nextStep(@PathVariable("jobId") int jobId) {
JobManager jobEntity = this.jobService.get(jobId);
Ex.check(jobEntity != null, "job-manager.not-exist.id", jobId);
Ex.check(jobEntity.getJobStatus() == JobStatus.MAPPING,
"job.manager.status.unexpected",
JobStatus.MAPPING, jobEntity.getJobStatus());
jobEntity.setJobStatus(JobStatus.SETTING);
this.jobService.update(jobEntity);
return jobEntity;
}

private void checkVertexMappingValid(int connId, VertexMapping vertexMapping,
FileMapping fileMapping) {
VertexLabelEntity vl = this.vlService.get(vertexMapping.getLabel(),
Expand Down
Loading