Merge pull request #27 from devgateway/v0.0.15-pre-release

- Import activities with multiple donors - Refactor AMPStaticProcessor
devgateway · Nov 22, 2017 · 9f8be72 · 9f8be72
2 parents 7e36913 + 3cc2320
commit 9f8be72
Show file tree

Hide file tree

Showing 8 changed files with 557 additions and 473 deletions.
diff --git a/import-core/import-rest/pom.xml b/import-core/import-rest/pom.xml
@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>org.devgateway.importtool</groupId>
 		<artifactId>importtool</artifactId>
-		<version>0.0.14-SNAPSHOT</version>
+		<version>0.0.15-SNAPSHOT</version>
 	</parent>
 	<artifactId>import-rest</artifactId>
 	<dependencies>

diff --git a/import-core/import-services/pom.xml b/import-core/import-services/pom.xml
@@ -5,7 +5,7 @@
 
 	<parent>
 		<groupId>org.devgateway.importtool</groupId>
-		<version>0.0.14-SNAPSHOT</version>
+		<version>0.0.15-SNAPSHOT</version>
 		<artifactId>importtool</artifactId>
 	</parent>
 	<artifactId>import-services</artifactId>

diff --git a/...rvices/src/main/java/org/devgateway/importtool/services/processor/AMPStaticProcessor.java b/...rvices/src/main/java/org/devgateway/importtool/services/processor/AMPStaticProcessor.java
diff --git a/...-services/src/main/java/org/devgateway/importtool/services/processor/IATI1XProcessor.java b/...-services/src/main/java/org/devgateway/importtool/services/processor/IATI1XProcessor.java
@@ -417,6 +417,8 @@ private List<InternalDocument> extractDocuments(Document doc) throws Exception {
 			String currency = !("".equals(element.getAttribute("default-currency"))) ? element.getAttribute("default-currency") : this.defaultCurrency;			
 			document.addStringField("default-currency", currency);	
 			String defaultLanguageCode = !("".equals(element.getAttribute("xml:lang"))) ? element.getAttribute("xml:lang") : this.defaultLanguage;
+
+
 			NodeList fieldNodeList;			
 			for (Field field : getFields()) {
 				switch (field.getType()) {
@@ -554,10 +556,13 @@ private List<InternalDocument> extractDocuments(Document doc) throws Exception {
 								if(!field.getPossibleValues().stream().anyMatch(n->{ return n.getCode().equals(stringOrgValue);})) {									
 									field.getPossibleValues().add(fv);
 								}
+
+
 								document.addOrganizationField(field.getFieldName() + "_" + field.getSubType() + "_" + index, orgFields);
 							}
 						}
 					}
+
 					break;
 				case MULTILANG_STRING:					
 					Map<String, String> mlv = new HashMap<String, String>();
@@ -590,8 +595,7 @@ private List<InternalDocument> extractDocuments(Document doc) throws Exception {
 						nodes = (NodeList) xPath.evaluate("transaction/transaction-type[@code='" + field.getSubType() + "' or @code= '" + field.getSubTypeCode() + "']/parent::*", element, XPathConstants.NODESET);
 						for (int j = 0; j < nodes.getLength(); ++j) {
 							String reference = "";
-							String receivingOrganization = "";
-
+
 							Element e = (Element) nodes.item(j);
 							// Reference
 							reference = e.getAttribute("ref");
@@ -605,38 +609,53 @@ private List<InternalDocument> extractDocuments(Document doc) throws Exception {
 							if (!isValidDate(localDate)){
 								localDate = (e.getElementsByTagName("transaction-date").item(0) != null && e.getElementsByTagName("transaction-date").item(0).getAttributes() != null) ? e.getElementsByTagName("transaction-date").item(0).getAttributes().getNamedItem("iso-date").getNodeValue() : "";
 							}
-
-							// Providing Org
-							final String providingOrganization = (e.getElementsByTagName("provider-org").item(0) != null && e.getElementsByTagName("provider-org").item(0).getChildNodes().getLength() > 0) ? e.getElementsByTagName("provider-org").item(0).getChildNodes().item(0).getNodeValue() : null;
-
+
+							Element receiverNode = e.getElementsByTagName("receiver-org").item(0) != null
+									? (Element) e.getElementsByTagName("receiver-org").item(0) : null;
+
+							final String receivingOrganization = (receiverNode != null
+									&& receiverNode.getChildNodes().item(0) != null)
+											? receiverNode.getChildNodes().item(0).getNodeValue() : "";
+
+							Element providerNode = e.getElementsByTagName("provider-org").item(0) != null
+									? (Element) e.getElementsByTagName("provider-org").item(0) : null;
+
+							final String providingOrganization = (providerNode != null
+									&& providerNode.getChildNodes().getLength() > 0)
+											? providerNode.getChildNodes().item(0).getNodeValue() : "";
+							final String providerRef = (providerNode != null) ? providerNode.getAttribute("ref") : "";
+
 							// Get the field for provider org
 							Optional<Field> fieldValue = filterFieldList.stream().filter(n -> {
 								return "provider-org".equals(n.getFieldName());
 							}).findFirst();
 
-							// If it has filters set, check if this transaction complies
-							if(fieldValue.isPresent() && fieldValue.get().getFilters().size() > 0) {
-								// See if the current transaction has the correct provider organization
+							// If it has filters set, check if this transaction
+							// complies
+							if (fieldValue.isPresent() && fieldValue.get().getFilters().size() > 0) {
+								// See if the current transaction has the
+								// correct provider organization
 								Optional<String> optField = fieldValue.get().getFilters().stream().filter(n -> {
 									return n.equals(providingOrganization);
 								}).findAny();
-
-								if(!optField.isPresent()) { // If it's not there, then move to the next transaction
+
+								if (!optField.isPresent()) {
+									// If it's not there, then move to the next
+									// transaction
 									continue;
 								}
 							}
-
-							// Receiving Org
-							receivingOrganization = (e.getElementsByTagName("receiver-org").item(0) != null && e.getElementsByTagName("receiver-org").item(0).getChildNodes().getLength() > 0) ? e.getElementsByTagName("receiver-org").item(0).getChildNodes().item(0).getNodeValue() : null;
 
 							Map<String, String> transactionFields = new HashMap<String, String>();
 							transactionFields.put("date", localDate);
 							transactionFields.put("receiving-org", receivingOrganization);
+							transactionFields.put("providing-org", providingOrganization);
+							transactionFields.put("provider-org-ref", providerRef);
 							transactionFields.put("reference", reference);
 							transactionFields.put("value", localValue);
 							transactionFields.put("subtype", field.getSubType());
-
-							document.addTransactionField("transaction" + field.getSubType() + "_" + j, transactionFields);
+							document.addTransactionField("transaction" + field.getSubType() + "_" + j,
+									transactionFields);
 						}
 
 					} catch (XPathExpressionException e1) {

diff --git a/...-services/src/main/java/org/devgateway/importtool/services/processor/IATI2XProcessor.java b/...-services/src/main/java/org/devgateway/importtool/services/processor/IATI2XProcessor.java
@@ -644,9 +644,7 @@ private List<InternalDocument> extractDocuments(Document doc) throws Exception {
 						nodes = (NodeList) xPath.evaluate("transaction/transaction-type[@code='" + field.getSubType() + "' or @code= '" + field.getSubTypeCode() + "']/parent::*", element, XPathConstants.NODESET);
 						for (int j = 0; j < nodes.getLength(); ++j) {
 							String reference = "";
-							String receivingOrganization = "";
-
-							Element e = (Element) nodes.item(j);
+													Element e = (Element) nodes.item(j);
 							// Reference
 							reference = e.getAttribute("ref");
 							// Amount
@@ -664,39 +662,50 @@ private List<InternalDocument> extractDocuments(Document doc) throws Exception {
 															// defensive
 							{
 								localDate = e.getElementsByTagName("transaction-date").item(0).getAttributes().getNamedItem("iso-date").getNodeValue();
-							}
+							}				
+
+							final String receivingOrganization = extractNarrative(e, "receiver-org");
+							Element providerNode = e.getElementsByTagName("provider-org").item(0) != null
+									? (Element) e.getElementsByTagName("provider-org").item(0) : null;
+
+							final String providingOrganization = (providerNode != null
+									&& providerNode.getElementsByTagName("narrative").item(0) != null)
+											? providerNode.getElementsByTagName("narrative").item(0).getTextContent()
+											: "";
+							final String providerRef = providerNode != null ? providerNode.getAttribute("ref") : "";
 
-							// Providing Org
-							final String providingOrganization = extractNarrative(e, "provider-org");
-
 							// Get the field for provider org
 							Optional<Field> fieldValue = filterFieldList.stream().filter(n -> {
 								return "provider-org".equals(n.getFieldName());
 							}).findFirst();
 
-							// If it has filters set, check if this transaction complies
-							if(fieldValue.isPresent() && fieldValue.get().getFilters().size() > 0) {
-								// See if the current transaction has the correct provider organization
+							// If it has filters set, check if this transaction
+							// complies
+							if (fieldValue.isPresent() && fieldValue.get().getFilters().size() > 0) {
+								// See if the current transaction has the
+								// correct provider organization
 								Optional<String> optField = fieldValue.get().getFilters().stream().filter(n -> {
 									return n.equals(providingOrganization);
 								}).findAny();
-
-								if(!optField.isPresent()) { // If it's not there, then move to the next transaction
+
+								if (!optField.isPresent()) {
+									// If it's not there, then move to the next
+									// transaction
 									continue;
 								}
 							}
 							// Receiving Org
-							receivingOrganization = extractNarrative(e, "receiver-org");
-
+
 							Map<String, String> transactionFields = new HashMap<String, String>();
 							transactionFields.put("date", localDate);
 							transactionFields.put("providing-org", providingOrganization);
 							transactionFields.put("receiving-org", receivingOrganization);
+							transactionFields.put("provider-org-ref", providerRef);
 							transactionFields.put("reference", reference);
 							transactionFields.put("value", localValue);
 							transactionFields.put("subtype", field.getSubType());
-
-							document.addTransactionField("transaction" + field.getSubType() + "_" + j, transactionFields);
+							document.addTransactionField("transaction" + field.getSubType() + "_" + j,
+									transactionFields);
 						}
 
 					} catch (XPathExpressionException e1) {

diff --git a/...ort-services/src/main/java/org/devgateway/importtool/services/processor/helper/Field.java b/...ort-services/src/main/java/org/devgateway/importtool/services/processor/helper/Field.java
@@ -22,6 +22,7 @@ public class Field {
 	private List<Field> dependencies = new ArrayList<Field>();
 	private boolean isRequired;
 	private boolean isExclusive;
+	private int length;
 
 	private boolean filterRequired;
 	// These types are silly but needed for now.
@@ -209,4 +210,12 @@ public void setMultiLangDisplayName(Map<String, String> multiLangDisplayName) {
 		this.multiLangDisplayName = multiLangDisplayName;
 	}
 
+	public int getLength() {
+		return length;
+	}
+
+	public void setLength(int length) {
+		this.length = length;
+	}
+
 }
diff --git a/import-core/import-ui/pom.xml b/import-core/import-ui/pom.xml
@@ -12,7 +12,7 @@
 	<parent>
 		<groupId>org.devgateway.importtool</groupId>
 		<artifactId>importtool</artifactId>
-		<version>0.0.14-SNAPSHOT</version>
+		<version>0.0.15-SNAPSHOT</version>
 	</parent>
 
 	<dependencies>

diff --git a/import-core/pom.xml b/import-core/pom.xml
@@ -5,7 +5,7 @@
 	<artifactId>importtool</artifactId>
 	<name>IMPORT TOOL Parent Package</name>
 	<url>http://maven.apache.org</url>
-	<version>0.0.14-SNAPSHOT</version>
+	<version>0.0.15-SNAPSHOT</version>
 
 	<parent>
 		<groupId>org.springframework.boot</groupId>