Skip to content

Commit

Permalink
Merge pull request #66 from devgateway/pre-release/v1.0
Browse files Browse the repository at this point in the history
IATIIMPORT-162 1.0 release
  • Loading branch information
jdeanquin-dg authored Sep 28, 2018
2 parents c5fac30 + 8a8da51 commit 5500384
Show file tree
Hide file tree
Showing 215 changed files with 56,410 additions and 1,243 deletions.
12 changes: 12 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
rebel.xml
import-core/import-scheduler/target
import-core/import-ui/src/main/resources/static/images
import-core/import-ui/src/main/resources/static/scripts/global.js
import-core/import-ui/src/main/resources/static/scripts/lt-ie9.js
import-core/import-ui/src/main/resources/static/scripts/main.js
import-core/import-ui/src/main/resources/static/styles/global.css
import-core/import-ui/src/main/resources/static/styles/main.css
import-core/import-ui/src/main/webapp/package-lock.json
modernizr.js
package-lock.json
index.html
2 changes: 1 addition & 1 deletion import-core/import-rest/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
<parent>
<groupId>org.devgateway.importtool</groupId>
<artifactId>importtool</artifactId>
<version>0.0.19-SNAPSHOT</version>
<version>1.0</version>
</parent>
<artifactId>import-rest</artifactId>
<dependencies>
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
package org.devgateway.importtool.rest;

import java.util.ArrayList;
import java.util.List;
import java.util.Optional;

import javax.servlet.http.HttpServletRequest;

import org.devgateway.importtool.dao.ReportingOrgRepository;
import org.devgateway.importtool.model.DataSource;
import org.devgateway.importtool.model.ReportingOrganization;
import org.devgateway.importtool.services.DataSourceService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RestController;

@RestController
@RequestMapping(value = "/data-source")
class DataSourceController {
@Autowired
private DataSourceService dataSourceService;
@Autowired
private ReportingOrgRepository reportingOrgRepository;

@RequestMapping(method = RequestMethod.GET)
ResponseEntity<DataSource> getDataSource(HttpServletRequest request) {
return new ResponseEntity<>(dataSourceService.getDataSource(), HttpStatus.OK);
}

@RequestMapping(method = RequestMethod.POST)
ResponseEntity<DataSource> saveDataSource(@RequestBody DataSource ds, HttpServletRequest request) {
return new ResponseEntity<>(dataSourceService.save(ds), HttpStatus.OK);
}

@RequestMapping(method = RequestMethod.GET, value={"/reporting-orgs", "/reporting-orgs/{orgName}"})
ResponseEntity<List<ReportingOrganization>> getOrganizations(HttpServletRequest request,
@PathVariable Optional<String> orgName) {
List<ReportingOrganization> reportingOrganizations;
if (orgName.isPresent()) {
reportingOrganizations = reportingOrgRepository.findByNameContainsIgnoreCase(orgName.get());
} else {
reportingOrganizations = reportingOrgRepository.findAll();
}
return new ResponseEntity<>(reportingOrganizations, HttpStatus.OK);
}
@RequestMapping(method = RequestMethod.GET, value="reporting-orgs/with-updates")
ResponseEntity<List<ReportingOrganization>> getOrganizationsWithUpdates(HttpServletRequest request) {
return new ResponseEntity<>(reportingOrgRepository.groupingCriteriaWithUpdates(), HttpStatus.OK);
}

}
Original file line number Diff line number Diff line change
@@ -1,37 +1,32 @@
package org.devgateway.importtool.rest;

import static org.devgateway.importtool.services.processor.helper.Constants.CURRENT_FILE_ID;
import static org.devgateway.importtool.services.processor.helper.Constants.DESTINATION_PROCESSOR;
import static org.devgateway.importtool.services.processor.helper.Constants.DOCUMENT_MAPPER;
import static org.devgateway.importtool.services.processor.helper.Constants.SESSION_TOKEN;
import static org.devgateway.importtool.services.processor.helper.Constants.SOURCE_PROCESSOR;

import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;

import javax.servlet.http.HttpServletRequest;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.devgateway.importtool.dao.FileRepository;
import org.devgateway.importtool.dao.ProjectRepository;
import org.devgateway.importtool.endpoint.EPConstants;
import org.devgateway.importtool.dao.ReportingOrgRepository;
import org.devgateway.importtool.endpoint.DataFetchServiceConstants;
import org.devgateway.importtool.endpoint.EPMessages;
import org.devgateway.importtool.endpoint.Param;
import org.devgateway.importtool.model.FetchResult;
import org.devgateway.importtool.model.File;
import org.devgateway.importtool.model.ImportSummary;
import org.devgateway.importtool.model.ReportingOrganization;
import org.devgateway.importtool.rest.dto.FetchOrganizationDetails;
import org.devgateway.importtool.security.ImportSessionToken;
import org.devgateway.importtool.services.ActivityFetchService;
import org.devgateway.importtool.services.ImportService;
import org.devgateway.importtool.services.processor.IATIProcessor;
import org.devgateway.importtool.services.processor.helper.DocumentMapper;
import org.devgateway.importtool.services.processor.helper.IDestinationProcessor;
import org.devgateway.importtool.services.processor.helper.IDocumentMapper;
import org.devgateway.importtool.services.processor.helper.ISourceProcessor;
import org.devgateway.importtool.services.processor.helper.JsonBean;
import org.devgateway.importtool.services.processor.helper.Status;
import org.devgateway.importtool.services.request.ImportRequest;
import org.devgateway.importtool.services.response.DocumentMappingResponse;
import org.devgateway.importtool.services.response.ImportExecuteResponse;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
Expand All @@ -44,6 +39,24 @@
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.multipart.MultipartFile;

import javax.servlet.http.HttpServletRequest;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import static org.devgateway.importtool.services.processor.helper.Constants.CURRENT_FILE_ID;
import static org.devgateway.importtool.services.processor.helper.Constants.DESTINATION_PROCESSOR;
import static org.devgateway.importtool.services.processor.helper.Constants.DOCUMENT_MAPPER;
import static org.devgateway.importtool.services.processor.helper.Constants.IATI_STORE_ACTIVITIES;
import static org.devgateway.importtool.services.processor.helper.Constants.SESSION_TOKEN;
import static org.devgateway.importtool.services.processor.helper.Constants.SOURCE_PROCESSOR;
import static org.devgateway.importtool.services.processor.helper.Constants.REPORTING_ORG;


@RestController
@RequestMapping(value = "/import")
Expand All @@ -53,37 +66,58 @@ class ImportController {
private FileRepository repository;
@Autowired
private ProjectRepository projectRepository;

@Autowired
ReportingOrgRepository reportingOrgRepository;

@Autowired
private ImportService importService;


@Autowired
private ActivityFetchService activityFetchService;

private Log log = LogFactory.getLog(getClass());

@Value("${IATIProcessor.default_country}")
private String defaultCountry;

@RequestMapping(method = RequestMethod.GET, value = "/new/{sourceProcessorName}/{destinationProcessorName}/{authenticationToken}/{userName}")
ResponseEntity<ImportSessionToken> initiateImport(@PathVariable String sourceProcessorName, @PathVariable String destinationProcessorName, @PathVariable String authenticationToken, @PathVariable String userName,
HttpServletRequest request) {
boolean isAutomaticProcessor = false;
if(request.getSession().getAttribute(IATI_STORE_ACTIVITIES) != null) {
isAutomaticProcessor = true;
}
log.debug("Initialized import");
request.getSession().removeAttribute(SOURCE_PROCESSOR);
request.getSession().removeAttribute(DESTINATION_PROCESSOR);
request.getSession().removeAttribute(SESSION_TOKEN);
request.getSession().removeAttribute(DOCUMENT_MAPPER);
ISourceProcessor srcProcessor = importService.getSourceProcessor(sourceProcessorName);
request.getSession().setAttribute(SOURCE_PROCESSOR, srcProcessor);
IDestinationProcessor destProcessor = importService.getDestinationProcessor(destinationProcessorName, authenticationToken);
IDestinationProcessor destProcessor = importService.getDestinationProcessor(destinationProcessorName,
authenticationToken, isAutomaticProcessor);
request.getSession().setAttribute(DESTINATION_PROCESSOR, destProcessor);
ImportSessionToken importSessionToken = new ImportSessionToken(authenticationToken, userName, new Date(), srcProcessor.getDescriptiveName(), destProcessor.getDescriptiveName());
request.getSession().setAttribute(SESSION_TOKEN, importSessionToken);
return new ResponseEntity<>(importSessionToken, HttpStatus.OK);

if(isAutomaticProcessor) {
FetchResult fr = (FetchResult) request.getSession().getAttribute(IATI_STORE_ACTIVITIES);
srcProcessor.setFromDataStore(true);
srcProcessor.setInput(fr.getActivities());
logAutomatedImport(request, srcProcessor);
}
request.getSession().setAttribute(SOURCE_PROCESSOR, srcProcessor);
return new ResponseEntity<>(importSessionToken, HttpStatus.OK);
}

@RequestMapping(method = RequestMethod.GET, value = "/refresh/{authenticationToken}")
public ResponseEntity<List<File>> refreshToken(@PathVariable String authenticationToken, HttpServletRequest request) {
ImportSessionToken authToken = (ImportSessionToken) request.getSession().getAttribute(SESSION_TOKEN);
authToken.setAuthenticationToken(authenticationToken);
IDestinationProcessor destProcessor = (IDestinationProcessor) request.getSession().getAttribute(DESTINATION_PROCESSOR);
destProcessor.setAuthenticationToken(authenticationToken);

return new ResponseEntity<>(null, HttpStatus.OK);
return new ResponseEntity<>( HttpStatus.OK);
}

@RequestMapping(method = RequestMethod.GET, value = "/uploaded")
Expand All @@ -106,28 +140,58 @@ public ResponseEntity<String> wipe(HttpServletRequest request) {
request.getSession().removeAttribute(DESTINATION_PROCESSOR);
request.getSession().removeAttribute(SESSION_TOKEN);
request.getSession().removeAttribute(DOCUMENT_MAPPER);
request.getSession().removeAttribute(IATI_STORE_ACTIVITIES);
request.getSession().removeAttribute(REPORTING_ORG);
request.getSession().removeAttribute(CURRENT_FILE_ID);
return new ResponseEntity<>("{'error': ''}", HttpStatus.OK);
}

@RequestMapping(method = RequestMethod.POST, value = "/upload")
public ResponseEntity<String> handleFileUpload(@RequestParam("file_data") MultipartFile file, HttpServletRequest request) {
if (!file.isEmpty()) {
try {
try {
ISourceProcessor srcProcessor = (ISourceProcessor) request.getSession().getAttribute(SOURCE_PROCESSOR);
ImportSessionToken authToken = (ImportSessionToken) request.getSession().getAttribute(SESSION_TOKEN);
File uploadedFile = importService.uploadFile(file, srcProcessor, authToken);
request.getSession().setAttribute(CURRENT_FILE_ID, uploadedFile.getId());
InputStream is = new ByteArrayInputStream(file.getBytes());
srcProcessor.setInput(is);
processFile(file.getOriginalFilename(), request,srcProcessor);
return new ResponseEntity<>("{}", HttpStatus.OK);
} catch (Exception e) {
e.printStackTrace();
return new ResponseEntity<>(EPMessages.ERROR_UPLOADING_FILE_CHECK_INITIAL_STEPS.toString(), HttpStatus.OK);
log.error(e);
return new ResponseEntity<>(EPMessages.ERROR_UPLOADING_FILE_CHECK_INITIAL_STEPS.toString(),
HttpStatus.OK);
}
} else {
return new ResponseEntity<>(EPMessages.ERROR_UPLOADING_FILE.toString(), HttpStatus.OK);
}
}

@RequestMapping(method = RequestMethod.POST, value = "/filter")


private void processFile(String fileName, HttpServletRequest request, ISourceProcessor srcProcessor)
throws IOException {

ImportSessionToken authToken = (ImportSessionToken) request.getSession().getAttribute(SESSION_TOKEN);
File uploadedFile = importService.logImportSession(fileName ,srcProcessor, authToken);
request.getSession().setAttribute(CURRENT_FILE_ID, uploadedFile.getId());
}

private void logAutomatedImport (HttpServletRequest request, ISourceProcessor srcProcessor) {
try {
String reportingOrgId = (String)request.getSession().getAttribute(REPORTING_ORG);
List<ReportingOrganization> orgs = reportingOrgRepository.findByOrgIdIgnoreCase(reportingOrgId);
String orgName = orgs.size() > 0 ? orgs.get(0).getName() : "";

ImportSessionToken authToken = (ImportSessionToken) request.getSession().getAttribute(SESSION_TOKEN);
File uploadedFile;

uploadedFile = importService.logImportSession(orgName + " - " + srcProcessor.getDescriptiveName() , srcProcessor, authToken);
request.getSession().setAttribute(CURRENT_FILE_ID, uploadedFile.getId());
} catch (IOException e) {
e.printStackTrace();
}
}


@RequestMapping(method = RequestMethod.POST, value = "/fetch")
ResponseEntity<ImportSessionToken> filter(@PathVariable String authenticationToken) {
ImportSessionToken authObject = new ImportSessionToken(authenticationToken, "", new Date(), "", null);
// TODO: Execute the filters
Expand Down Expand Up @@ -198,6 +262,72 @@ ResponseEntity<ImportSummary> getSummary(HttpServletRequest request) {
return new ResponseEntity<>(HttpStatus.INTERNAL_SERVER_ERROR);
}
return new ResponseEntity<>(importService.getSummary(documentMapper, importSessionToken, processor), HttpStatus.OK);
}
}


@RequestMapping(method = RequestMethod.GET, value = "/fetch/results")
ResponseEntity<FetchOrganizationDetails> fetchResult(HttpServletRequest request) {
FetchOrganizationDetails organizationDetails = new FetchOrganizationDetails();

FetchResult activitiesFromDataStore = (FetchResult) request.getSession().getAttribute(IATI_STORE_ACTIVITIES);
if (activitiesFromDataStore != null) {
if (activitiesFromDataStore.getStatus().equals(Status.FAILED_WITH_ERROR)
|| activitiesFromDataStore.getStatus().equals(Status.COMPLETED)) { //activities have
if (activitiesFromDataStore.getStatus().equals(Status.COMPLETED)) {
if (activitiesFromDataStore.getStatus().equals(Status.COMPLETED)) {
activitiesFromDataStore.getVersions().retainAll(IATIProcessor.IMPLEMENTED_VERSIONS);
organizationDetails.setVersions(activitiesFromDataStore.getVersions());
organizationDetails.setProjectWithUpdates(projectRepository.findProjectUpdated());
request.getSession().setAttribute(IATI_STORE_ACTIVITIES, activitiesFromDataStore);
organizationDetails.setStatus(Status.COMPLETED);
} else {
organizationDetails.setStatus(Status.FAILED_WITH_ERROR);
}
} else {
organizationDetails.setStatus(Status.IN_PROGRESS);
}
}
}else{
organizationDetails.setStatus(Status.FAILED_WITH_ERROR);
}
return new ResponseEntity<>(organizationDetails, HttpStatus.OK);

}

@RequestMapping(method = RequestMethod.GET, value = "/fetch/initialize/{reportingOrgId}")
ResponseEntity<FetchOrganizationDetails> fetch(HttpServletRequest request, @PathVariable("reportingOrgId") String
reportingOrgId) {
FetchOrganizationDetails organizationDetails = new FetchOrganizationDetails();

try {
FetchResult activitiesFromDataStore = new FetchResult();
activitiesFromDataStore.setStatus(Status.NOT_STARTED);
List<Param> params = DataFetchServiceConstants.getCommonParams(reportingOrgId, defaultCountry);
activityFetchService.fetchResult(reportingOrgId, params, activitiesFromDataStore);
request.getSession().setAttribute(IATI_STORE_ACTIVITIES,activitiesFromDataStore);
request.getSession().setAttribute(REPORTING_ORG, reportingOrgId);
return new ResponseEntity<>(organizationDetails, HttpStatus.OK);

} catch (Exception e) {
log.error(e);
return new ResponseEntity<>(HttpStatus.INTERNAL_SERVER_ERROR);
}
}

@RequestMapping(method = RequestMethod.GET, value = "/translation-files")
ResponseEntity<Map<String,List<String>>> generateTranslationFiles(HttpServletRequest request) {
Map<String,List<String>> missingVersions = new HashMap<String,List<String>>();

IATIProcessor.IMPLEMENTED_VERSIONS.stream().forEach(version->{
String versionName = "IATI" + version.replace(".", "");
try {
ISourceProcessor srcProcessor = importService.getSourceProcessor(versionName);
missingVersions.put(versionName, srcProcessor.buildTooltipsFields());
}catch (Exception ex){
log.error("cannot process for version " + versionName,ex);
}
});
return new ResponseEntity<>(missingVersions, HttpStatus.OK);
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
package org.devgateway.importtool.rest.dto;

import org.devgateway.importtool.model.Project;
import org.devgateway.importtool.services.processor.helper.Status;

import java.util.List;
import java.util.Set;

public class FetchOrganizationDetails {
private Status status;
private Set<String> versions;
private List<Project> projectWithUpdates;

public FetchOrganizationDetails(){
status = Status.NOT_STARTED;
}


public Set<String> getVersions() {
return versions;
}

public void setVersions(Set<String> versions) {
this.versions = versions;
}

public List<Project> getProjectWithUpdates() {
return projectWithUpdates;
}

public void setProjectWithUpdates(List<Project> projectWithUpdates) {
this.projectWithUpdates = projectWithUpdates;
}

public Status getStatus() {
return status;
}

public void setStatus(Status status) {
this.status = status;
}
}
Loading

0 comments on commit 5500384

Please sign in to comment.