Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Some bug fix for load task and schema manage #246

Merged
merged 7 commits into from
Oct 30, 2020
Merged

Some bug fix for load task and schema manage #246

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
f92fac4
Some bug fix for load task and schema manage
Linary Sep 29, 2020
abf84a3
tiny improve
Linary Oct 19, 2020
653896d
add API for JobManager status controling
Linary Oct 23, 2020
405b75c
Use background thread to delete unfinished files
Linary Oct 26, 2020
eaf4494
wrap save/update/remove method, and rename finish API to next-step
Linary Oct 27, 2020
2ae54a7
tiny improve
Linary Oct 28, 2020
236472f
tiny improve
Linary Oct 30, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 1 addition & 32 deletions hubble-be/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,6 @@
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-cache</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-aop</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
Expand Down Expand Up @@ -98,38 +94,11 @@
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.baidu.hugegraph</groupId>
<artifactId>hugegraph-client</artifactId>
<version>1.8.10</version>
<exclusions>
<exclusion>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j-impl</artifactId>
</exclusion>
<exclusion>
<groupId>com.baidu.hugegraph</groupId>
<artifactId>hugegraph-common</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.baidu.hugegraph</groupId>
<artifactId>hugegraph-loader</artifactId>
<version>0.10.4</version>
<version>0.10.5</version>
<exclusions>
<exclusion>
<groupId>com.baidu.hugegraph</groupId>
<artifactId>hugegraph-client</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
Expand Down
4 changes: 2 additions & 2 deletions hubble-be/src/main/java/com/baidu/hugegraph/config/AsyncConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,8 @@ public class AsyncConfig implements AsyncConfigurer {
@Override
public Executor getAsyncExecutor() {
ThreadPoolTaskExecutor taskExecutor = new ThreadPoolTaskExecutor();
taskExecutor.setCorePoolSize(4);
taskExecutor.setMaxPoolSize(8);
taskExecutor.setCorePoolSize(8);
taskExecutor.setMaxPoolSize(16);
taskExecutor.setQueueCapacity(16);
taskExecutor.initialize();
return taskExecutor;
Expand Down
22 changes: 18 additions & 4 deletions hubble-be/src/main/java/com/baidu/hugegraph/controller/load/FileMappingController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,17 +37,20 @@

import com.baidu.hugegraph.common.Constant;
import com.baidu.hugegraph.controller.BaseController;
import com.baidu.hugegraph.entity.enums.JobStatus;
import com.baidu.hugegraph.entity.load.EdgeMapping;
import com.baidu.hugegraph.entity.load.ElementMapping;
import com.baidu.hugegraph.entity.load.FileMapping;
import com.baidu.hugegraph.entity.load.FileSetting;
import com.baidu.hugegraph.entity.load.JobManager;
import com.baidu.hugegraph.entity.load.LoadParameter;
import com.baidu.hugegraph.entity.load.VertexMapping;
import com.baidu.hugegraph.entity.schema.EdgeLabelEntity;
import com.baidu.hugegraph.entity.schema.VertexLabelEntity;
import com.baidu.hugegraph.exception.ExternalException;
import com.baidu.hugegraph.exception.InternalException;
import com.baidu.hugegraph.service.load.FileMappingService;
import com.baidu.hugegraph.service.load.JobManagerService;
import com.baidu.hugegraph.service.schema.EdgeLabelService;
import com.baidu.hugegraph.service.schema.VertexLabelService;
import com.baidu.hugegraph.util.Ex;
Expand All @@ -67,6 +70,8 @@ public class FileMappingController extends BaseController {
private EdgeLabelService elService;
@Autowired
private FileMappingService service;
@Autowired
private JobManagerService jobService;

@GetMapping
public IPage<FileMapping> list(@PathVariable("connId") int connId,
Expand Down Expand Up @@ -130,8 +135,8 @@ public FileMapping fileSetting(@PathVariable("id") int id,
if (mapping == null) {
throw new ExternalException("load.file-mapping.not-exist.id", id);
}
// unescape \\t to \t
newEntity.unescapeDelimiterIfNeeded();
// change format to TEXT if needed
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

// Change format to TEXT if needed

newEntity.changeFormatIfNeeded();
FileSetting oldEntity = mapping.getFileSetting();
FileSetting entity = this.mergeEntity(oldEntity, newEntity);
mapping.setFileSetting(entity);
Expand All @@ -140,8 +145,6 @@ public FileMapping fileSetting(@PathVariable("id") int id,
if (this.service.update(mapping) != 1) {
throw new InternalException("entity.update.failed", mapping);
}
// escape \t to \\t
mapping.getFileSetting().escapeDelimiterIfNeeded();
return mapping;
}

Expand Down Expand Up @@ -288,6 +291,17 @@ public void loadParameter(@RequestBody LoadParameter newEntity) {
}
}

@PutMapping("finish")
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

prefer rename to another name, like ready-loading

public JobManager finish(@PathVariable("jobId") int jobId) {
JobManager jobEntity = this.jobService.get(jobId);
Ex.check(jobEntity != null, "job-manager.not-exist.id", jobId);
Ex.check(jobEntity.getJobStatus() == JobStatus.MAPPING,
"job.manager.status.unexpected",
JobStatus.MAPPING, jobEntity.getJobStatus());
jobEntity.setJobStatus(JobStatus.SETTING);
return jobEntity;
}

private void checkVertexMappingValid(int connId, VertexMapping vertexMapping,
FileMapping fileMapping) {
VertexLabelEntity vl = this.vlService.get(vertexMapping.getLabel(),
Expand Down
Loading