Merge pull request #215 from devgateway/develop

AMP-29984 2.2 release
devgateway · Mar 26, 2021 · 5957656 · 5957656
2 parents dafaac0 + e9657cf
commit 5957656
Show file tree

Hide file tree

Showing 68 changed files with 286 additions and 2,115 deletions.
diff --git a/import-core/import-rest/pom.xml b/import-core/import-rest/pom.xml
@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>org.devgateway.importtool</groupId>
 		<artifactId>importtool</artifactId>
-		<version>2.1</version>
+		<version>2.2</version>
 	</parent>
 	<artifactId>import-rest</artifactId>
 	<dependencies>

diff --git a/import-core/import-rest/src/main/java/org/devgateway/importtool/rest/ImportController.java b/import-core/import-rest/src/main/java/org/devgateway/importtool/rest/ImportController.java
@@ -1,26 +1,5 @@
 package org.devgateway.importtool.rest;
 
-import static org.devgateway.importtool.services.processor.destination.AmpStaticProcessorConstants.SESSION_COOKIE_NAME;
-import static org.devgateway.importtool.services.processor.helper.Constants.CURRENT_FILE_ID;
-import static org.devgateway.importtool.services.processor.helper.Constants.DESTINATION_PROCESSOR;
-import static org.devgateway.importtool.services.processor.helper.Constants.DOCUMENT_MAPPER;
-import static org.devgateway.importtool.services.processor.helper.Constants.IATI_STORE_ACTIVITIES;
-import static org.devgateway.importtool.services.processor.helper.Constants.REPORTING_ORG;
-import static org.devgateway.importtool.services.processor.helper.Constants.SESSION_TOKEN;
-import static org.devgateway.importtool.services.processor.helper.Constants.SOURCE_PROCESSOR;
-
-import java.io.ByteArrayInputStream;
-import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import javax.servlet.http.Cookie;
-import javax.servlet.http.HttpServletRequest;
-
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.devgateway.importtool.dao.FileRepository;
@@ -39,11 +18,7 @@
 import org.devgateway.importtool.services.ImportService;
 import org.devgateway.importtool.services.ProjectTranslator;
 import org.devgateway.importtool.services.processor.IATIProcessor;
-import org.devgateway.importtool.services.processor.helper.DocumentMapper;
-import org.devgateway.importtool.services.processor.helper.IDestinationProcessor;
-import org.devgateway.importtool.services.processor.helper.IDocumentMapper;
-import org.devgateway.importtool.services.processor.helper.ISourceProcessor;
-import org.devgateway.importtool.services.processor.helper.Status;
+import org.devgateway.importtool.services.processor.helper.*;
 import org.devgateway.importtool.services.request.ImportRequest;
 import org.devgateway.importtool.services.response.DocumentMappingResponse;
 import org.devgateway.importtool.services.response.ImportExecuteResponse;
@@ -53,14 +28,18 @@
 import org.springframework.http.MediaType;
 import org.springframework.http.ResponseEntity;
 import org.springframework.transaction.annotation.Transactional;
-import org.springframework.web.bind.annotation.PathVariable;
-import org.springframework.web.bind.annotation.RequestBody;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RequestMethod;
-import org.springframework.web.bind.annotation.RequestParam;
-import org.springframework.web.bind.annotation.RestController;
+import org.springframework.web.bind.annotation.*;
 import org.springframework.web.multipart.MultipartFile;
 
+import javax.servlet.http.Cookie;
+import javax.servlet.http.HttpServletRequest;
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.util.*;
+
+import static org.devgateway.importtool.services.processor.destination.AmpStaticProcessorConstants.SESSION_COOKIE_NAME;
+import static org.devgateway.importtool.services.processor.helper.Constants.*;
+
 
 @RestController
 @RequestMapping(value = "/import")
@@ -281,22 +260,22 @@ ResponseEntity<FetchOrganizationDetails> fetchResult(HttpServletRequest request)
 			if (activitiesFromDataStore.getStatus().equals(Status.FAILED_WITH_ERROR)
 					|| activitiesFromDataStore.getStatus().equals(Status.COMPLETED)) { //activities  have
 				if (activitiesFromDataStore.getStatus().equals(Status.COMPLETED)) {
-					if (activitiesFromDataStore.getStatus().equals(Status.COMPLETED)) {
-						activitiesFromDataStore.getVersions().retainAll(IATIProcessor.IMPLEMENTED_VERSIONS);
-						organizationDetails.setVersions(activitiesFromDataStore.getVersions());
-						organizationDetails.setProjectWithUpdates(projectRepository.findProjectUpdated());
-						request.getSession().setAttribute(IATI_STORE_ACTIVITIES, activitiesFromDataStore);
-						organizationDetails.setStatus(Status.COMPLETED);
-					} else {
-						organizationDetails.setStatus(Status.FAILED_WITH_ERROR);
-					}
+					activitiesFromDataStore.getVersions().retainAll(IATIProcessor.IMPLEMENTED_VERSIONS);
+					organizationDetails.setVersions(activitiesFromDataStore.getVersions());
+					organizationDetails.setProjectWithUpdates(projectRepository.findProjectUpdated());
+					request.getSession().setAttribute(IATI_STORE_ACTIVITIES, activitiesFromDataStore);
+					organizationDetails.setStatus(Status.COMPLETED);
 				} else {
-					organizationDetails.setStatus(Status.IN_PROGRESS);
+					organizationDetails.setStatus(Status.FAILED_WITH_ERROR);
+					organizationDetails.setMessage(activitiesFromDataStore.getMessage());
 				}
+			} else {
+				organizationDetails.setStatus(Status.IN_PROGRESS);
 			}
-		}else{
-			 organizationDetails.setStatus(Status.FAILED_WITH_ERROR);
+		} else {
+			organizationDetails.setStatus(Status.FAILED_WITH_ERROR);
 		}
+
 		return new ResponseEntity<>(organizationDetails, HttpStatus.OK);
 
 	}

diff --git a/...mport-rest/src/main/java/org/devgateway/importtool/rest/dto/FetchOrganizationDetails.java b/...mport-rest/src/main/java/org/devgateway/importtool/rest/dto/FetchOrganizationDetails.java
@@ -11,6 +11,8 @@ public class FetchOrganizationDetails {
     private Set<String> versions;
     private List<Project> projectWithUpdates;
 
+    private String message;
+
     public FetchOrganizationDetails(){
         status = Status.NOT_STARTED;
     }
@@ -39,4 +41,12 @@ public Status getStatus() {
     public void setStatus(Status status) {
         this.status = status;
     }
+
+    public String getMessage() {
+        return message;
+    }
+
+    public void setMessage(String message) {
+        this.message = message;
+    }
 }
diff --git a/import-core/import-scheduler/pom.xml b/import-core/import-scheduler/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <artifactId>importtool</artifactId>
         <groupId>org.devgateway.importtool</groupId>
-        <version>2.1</version>
+        <version>2.2</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>

diff --git a/import-core/import-services/pom.xml b/import-core/import-services/pom.xml
@@ -5,7 +5,7 @@
 
 	<parent>
 		<groupId>org.devgateway.importtool</groupId>
-		<version>2.1</version>
+		<version>2.2</version>
 		<artifactId>importtool</artifactId>
 	</parent>
 	<artifactId>import-services</artifactId>

diff --git a/import-core/import-services/src/main/java/org/devgateway/importtool/model/FetchResult.java b/import-core/import-services/src/main/java/org/devgateway/importtool/model/FetchResult.java
@@ -1,17 +1,17 @@
 package org.devgateway.importtool.model;
 
-import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.Set;
 
 import org.devgateway.importtool.services.processor.helper.Status;
 import org.w3c.dom.Document;
-import org.w3c.dom.NodeList;
 
 public class FetchResult {
 	private Set<String> versions;
 	private Document activities;
 	private Status status;
+
+	private String message;
 
 	public Set<String> getVersions() {
 		return versions;
@@ -34,4 +34,12 @@ public Status getStatus() {
 	public void setStatus(Status status) {
 		this.status = status;
 	}
+
+	public String getMessage() {
+		return message;
+	}
+
+	public void setMessage(String message) {
+		this.message = message;
+	}
 }
diff --git a/...mport-services/src/main/java/org/devgateway/importtool/services/ActivityFetchService.java b/...mport-services/src/main/java/org/devgateway/importtool/services/ActivityFetchService.java
@@ -1,24 +1,5 @@
 package org.devgateway.importtool.services;
 
-import java.io.*;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Paths;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.time.LocalDateTime;
-import java.time.ZoneId;
-import java.util.*;
-import java.util.stream.Collectors;
-
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
-import javax.xml.parsers.ParserConfigurationException;
-import javax.xml.xpath.XPath;
-import javax.xml.xpath.XPathConstants;
-import javax.xml.xpath.XPathExpressionException;
-import javax.xml.xpath.XPathFactory;
-
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.http.conn.ssl.NoopHostnameVerifier;
@@ -36,6 +17,7 @@
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.http.client.HttpComponentsClientHttpRequestFactory;
 import org.springframework.scheduling.annotation.Async;
+import org.springframework.web.client.RestClientException;
 import org.springframework.web.client.RestTemplate;
 import org.w3c.dom.Document;
 import org.w3c.dom.Element;
@@ -44,6 +26,27 @@
 import org.w3c.dom.ls.DOMImplementationLS;
 import org.w3c.dom.ls.LSSerializer;
 
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.xpath.XPath;
+import javax.xml.xpath.XPathConstants;
+import javax.xml.xpath.XPathExpressionException;
+import javax.xml.xpath.XPathFactory;
+import java.io.*;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
 import static java.util.stream.Collectors.groupingBy;
 import static java.util.stream.Collectors.mapping;
 import static org.devgateway.importtool.endpoint.DataFetchServiceConstants.PARAMETER_REPLACE_VALUE;
@@ -106,9 +109,10 @@ private void fetchResultsAsync(String reportingOrg, List<Param> parameters, Fetc
         } catch (RuntimeException ex) {
             //we should probably not save it until is valid. leaving it here
             //for simplicity
-            //Files.delete(Paths.get(fileName));
-            //result.setStatus(Status.FAILED_WITH_ERROR);
-            //throw ex;
+            Files.deleteIfExists(Paths.get(fileName));
+            result.setStatus(Status.FAILED_WITH_ERROR);
+            result.setMessage("Error fetching activities from the datastore: " + ex.getMessage());
+            throw ex;
         }
     }
 
@@ -175,8 +179,12 @@ public String fetchFetchFromDataStoreAsString(String reportingOrg, List<Param> q
 
 
         String url = getUrlForReportingOrg(reportingOrg) + getParameters(queryParameters, defaultParameters);
-        log.debug(url);
-        return restTemplate.getForObject(url, String.class);
+        try {
+            return restTemplate.getForObject(url, String.class);
+        } catch (RestClientException ex) {
+            log.error("Cannot get activities from datastore", ex);
+            throw new RuntimeException(ex.getLocalizedMessage(), ex);
+        }
     }
 
     private String getParameters(List<Param> queryParam, List<Param> parameters) {

diff --git a/...rt-core/import-services/src/main/java/org/devgateway/importtool/services/DataService.java b/...rt-core/import-services/src/main/java/org/devgateway/importtool/services/DataService.java
@@ -29,25 +29,55 @@ public List<FieldValueMapping> getValueMapping(IDocumentMapper documentMapper){
 
         if (documentMapper.getFieldMappingObject() != null && documentMapper.getFieldMappingObject().size() > 0 && (documentMapper.getValueMappingObject() == null || documentMapper.getValueMappingObject().size() == 0)) {
             if (documentMapper.getValueMappingObject() == null) {
-                documentMapper.setValueMappingObject(new ArrayList<FieldValueMapping>());
+                documentMapper.setValueMappingObject(new ArrayList<>());
             }
             documentMapper.getFieldMappingObject().stream().forEach(fieldMapping -> {
                 addFieldValueMapping(documentMapper, fieldMapping, valuesInSelectedSourceProjects);
             });
-
-        } else if (documentMapper.getValueMappingObject().size() != documentMapper.getFieldMappingObject().size()) {
+        } else {
             documentMapper.getFieldMappingObject().stream().forEach(fieldMapping -> {
-                Boolean alreadyInserted = documentMapper.getValueMappingObject().stream().anyMatch(n -> { return n.getSourceField().getUniqueFieldName().equals(fieldMapping.getSourceField().getUniqueFieldName());});
-                if(!alreadyInserted) {
+                documentMapper.getValueMappingObject()
+                        .removeIf(fv -> !fieldValueMappingMatchWithFieldMapping(fv, fieldMapping));
+
+                Boolean alreadyInserted = documentMapper.getValueMappingObject().stream()
+                        .anyMatch(n -> isFieldValueOfFieldMapping(n, fieldMapping));
+
+                if (!alreadyInserted) {
                     addFieldValueMapping(documentMapper, fieldMapping, valuesInSelectedSourceProjects);
                 }
             });
-
         }
 
         return documentMapper.getValueMappingObject();
     }
-
+
+    /**
+     * Detect if the field value's source field and destination field matches with the field mapping's field
+     *
+     * @param fv
+     * @param fieldMapping
+     * @return
+     */
+    private boolean fieldValueMappingMatchWithFieldMapping(FieldValueMapping fv, FieldMapping fieldMapping) {
+        if (fv.getSourceField().getUniqueFieldName().equals(fieldMapping.getSourceField().getUniqueFieldName())) {
+            return fv.getDestinationField().getUniqueFieldName().equals(fieldMapping.getDestinationField().getUniqueFieldName());
+        }
+
+        return true;
+    }
+
+    /**
+     * Detect if the value mapping's fields are the same of field mapping's fields
+     *
+     * @param fv
+     * @param fieldMapping
+     * @return
+     */
+    private boolean isFieldValueOfFieldMapping(FieldValueMapping fv, FieldMapping fieldMapping) {
+        return fv.getSourceField().getUniqueFieldName().equals(fieldMapping.getSourceField().getUniqueFieldName())
+                && fv.getDestinationField().getUniqueFieldName().equals(fieldMapping.getDestinationField().getUniqueFieldName());
+    }
+
     private void addFieldValueMapping(IDocumentMapper documentMapper, FieldMapping fieldMapping, Map<String, Set<String>> valuesInSelectedSourceProjects) {
         FieldValueMapping fvm = new FieldValueMapping();
         fvm.setSourceField(fieldMapping.getSourceField());
@@ -84,7 +114,7 @@ private void addFieldValueMapping(IDocumentMapper documentMapper, FieldMapping f
 
         documentMapper.getValueMappingObject().add(fvm);
     }
-    
+
     public List<FieldMapping> setFieldMapping(IDocumentMapper documentMapper, List<FieldMapping> fieldMapping){
         documentMapper.setFieldMappingObject(fieldMapping);     
         //update field value mappings

diff --git a/...t-services/src/main/java/org/devgateway/importtool/services/OrganizationFetchService.java b/...t-services/src/main/java/org/devgateway/importtool/services/OrganizationFetchService.java
@@ -45,7 +45,7 @@ public void fetch() {
             } while (response.getResult().size() == PAGE_SIZE);
             log.debug("Organizations fetched");
         } catch (RestClientException ex) {
-            log.error("Cannot get organizations from datastore", ex);
+            log.error("Cannot get organizations from the datastore", ex);
             throw new RuntimeException(ex);
         }
     }

diff --git a/...rvices/src/main/java/org/devgateway/importtool/services/processor/AMPStaticProcessor.java b/...rvices/src/main/java/org/devgateway/importtool/services/processor/AMPStaticProcessor.java
@@ -1231,7 +1231,10 @@ private Integer getIdByValue(String fieldValue, Optional<FieldValueMapping> optV
             } else {
                 return n.getValue().equals(fieldValue);
             }
-        }).findFirst().get();
+        }).findFirst().orElse(null);
+		if (fvs == null) {
+			return null;
+		} ;
         Integer sourceValueIndex = fvs.getIndex();
         Integer destinationValueIndex = vm.getValueIndexMapping().get(sourceValueIndex);
         FieldValue fvd = vm.getDestinationField().getPossibleValues().stream().filter(n -> {