fix apache#2256 Kettle import: Formula step not migrated correctly

bamaer · Jul 18, 2023 · 12d3bdd · 12d3bdd
1 parent b4f147f
commit 12d3bdd
Show file tree

Hide file tree

Showing 2 changed files with 91 additions and 13 deletions.
diff --git a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/formula.adoc b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/formula.adoc
@@ -83,6 +83,10 @@ TIP: Check the Apache POI docs page for a https://poi.apache.org/components/spre
 
 toc::[]
 
+== Notes
+
+The kettle import process is able to migrate the existing Formula steps contained in kettle processes in the new Hop Formula transform. Even if the migration will go without any problem, the migration process is unable to enter in the inner details of the Formula's transform macros and perform all the necessary checks to guarantee the correct behavior and the expected correct results. Therefore, even if the majority of the macros are the same as the existing macros you can find in Kettle Formula step, we strongly suggest to perfom a review of all of your fomulas transforms to verify that they behave as expected after the migration to the new Hop format.
+
 == Functions
 
 === Basic Computation

diff --git a/plugins/misc/import/src/main/java/org/apache/hop/imports/kettle/KettleImport.java b/plugins/misc/import/src/main/java/org/apache/hop/imports/kettle/KettleImport.java
@@ -32,6 +32,7 @@
 import org.apache.hop.core.extension.ExtensionPointHandler;
 import org.apache.hop.core.plugins.IPlugin;
 import org.apache.hop.core.plugins.PluginRegistry;
+import org.apache.hop.core.row.value.ValueMetaFactory;
 import org.apache.hop.core.util.StringUtil;
 import org.apache.hop.core.util.Utils;
 import org.apache.hop.core.vfs.HopVfs;
@@ -89,6 +90,7 @@ private enum EntryType {
     JOB,
     START,
     DUMMY,
+    FORMULA,
     OTHER
   };
 
@@ -182,7 +184,7 @@ private void handleHopFile(FileObject kettleFile) throws HopException {
       documentElement.insertBefore(nameSync, XmlHandler.getSubNode(documentElement, "description"));
     }
 
-    processNode(doc, documentElement, EntryType.OTHER);
+    processNode(doc, documentElement, EntryType.OTHER, 0);
 
     // Align x/y locations with a grid size...
     //
@@ -492,10 +494,13 @@ private void renameNode(Document doc, Element element, String newElementName) {
     doc.renameNode(element, null, newElementName);
   }
 
-  private void processNode(Document doc, Node node, EntryType entryType) {
+  private void processNode(Document doc, Node node, EntryType entryType, int depth) {
     Node nodeToProcess = node;
     NodeList nodeList = nodeToProcess.getChildNodes();
 
+    // Set depth level
+    depth += 1;
+
     // do a first pass to remove repository definitions
     for (int i = 0; i < nodeList.getLength(); i++) {
       Node repositoryNode = nodeList.item(i);
@@ -516,8 +521,11 @@ private void processNode(Document doc, Node node, EntryType entryType) {
       }
     }
 
+    Node firstFormulaNode = null;
+
     for (int i = 0; i < nodeList.getLength(); i++) {
       Node currentNode = nodeList.item(i);
+
       if (currentNode.getNodeType() == Node.ELEMENT_NODE) {
         // Identify if an entry is of type START or DUMMY type because they must be managed properly
         if (currentNode.getNodeName().equals("entry")) {
@@ -553,6 +561,20 @@ private void processNode(Document doc, Node node, EntryType entryType) {
           }
         }
 
+        if (currentNode.getNodeName().equals("step")) {
+          entryType = EntryType.OTHER;
+          NodeList currentNodeChildNodes = currentNode.getChildNodes();
+          for (int i1 = 0; i1 < currentNodeChildNodes.getLength(); i1++) {
+            Node childNode = currentNodeChildNodes.item(i1);
+            if (childNode.getNodeType() == Node.ELEMENT_NODE) {
+              if (childNode.getNodeName().equals("type")
+                  && childNode.getChildNodes().item(0).getNodeValue().equals("Formula")) {
+                entryType = EntryType.FORMULA;
+              }
+            }
+          }
+        }
+
         // remove superfluous elements
         if (entryType == EntryType.OTHER) {
           if (KettleConst.kettleElementsToRemove.containsKey(currentNode.getNodeName())) {
@@ -590,6 +612,36 @@ private void processNode(Document doc, Node node, EntryType entryType) {
           }
         }
 
+        if (entryType == EntryType.FORMULA && currentNode.getNodeName().equals("formula")) {
+          if (firstFormulaNode == null) {
+            Element formulasElement = doc.createElement("formulas");
+            formulasElement.appendChild(currentNode);
+            firstFormulaNode = formulasElement;
+          } else {
+            // Append new one to collection of new formula nodes
+            firstFormulaNode.appendChild(currentNode);
+          }
+        } else if (entryType == EntryType.FORMULA
+            && !currentNode.getNodeName().equals("formula")
+            && firstFormulaNode != null
+            && depth == 2) {
+          // Always in same steps' nodeset, same depth level and we finished formula definition
+          // nodes
+          // Add new collections to formlua nodeset
+          node.appendChild(firstFormulaNode);
+          // reset the entry type to OTHER because we left the formula nodeset
+          entryType = EntryType.OTHER;
+          firstFormulaNode = null;
+        } else if (entryType == EntryType.FORMULA
+            && currentNode.getNodeName().equals("formula_string")) {
+          String formulaValue = currentNode.getFirstChild().getNodeValue();
+          currentNode.getFirstChild().setNodeValue(formulaValue.replaceAll(";", ","));
+        } else if (entryType == EntryType.FORMULA
+            && currentNode.getNodeName().equals("value_type")) {
+          String formulaType = currentNode.getFirstChild().getNodeValue();
+          currentNode.getFirstChild().setNodeValue(Integer.toString(ValueMetaFactory.getIdForValueMeta(formulaType)));
+        }
+
         if (entryType == EntryType.JOB || entryType == EntryType.TRANS) {
           if (currentNode.getNodeName().equals("run_configuration")) {
             if (entryType == EntryType.JOB)
@@ -599,7 +651,6 @@ else if (entryType == EntryType.TRANS)
           }
         }
 
-
         // rename Kettle elements to Hop elements
         if (KettleConst.kettleElementReplacements.containsKey(currentNode.getNodeName())) {
           renameNode(
@@ -614,7 +665,7 @@ else if (entryType == EntryType.TRANS)
               KettleConst.kettleReplaceContent.get(currentNode.getTextContent()));
         }
 
-        processNode(doc, currentNode, entryType);
+        processNode(doc, currentNode, entryType, depth);
       }
 
       // partial node content replacement
@@ -710,15 +761,28 @@ private void alignLocations(Node parentNode, int gridSize) {
   @Override
   public String getImportReport() {
     String eol = System.getProperty("line.separator");
-    String messageString = BaseMessages.getString(PKG, "KettleImportDialog.ImportSummary.Imported.Label") + eol;
+    String messageString =
+        BaseMessages.getString(PKG, "KettleImportDialog.ImportSummary.Imported.Label") + eol;
     if (getKjbCounter() > 0) {
-      messageString += getKjbCounter() + " " + BaseMessages.getString(PKG, "KettleImportDialog.ImportSummary.ImportedJobs.Label") + eol;
+      messageString +=
+          getKjbCounter()
+              + " "
+              + BaseMessages.getString(PKG, "KettleImportDialog.ImportSummary.ImportedJobs.Label")
+              + eol;
     }
     if (getKtrCounter() > 0) {
-      messageString += getKtrCounter() + " " + BaseMessages.getString(PKG, "KettleImportDialog.ImportSummary.ImportedTransf.Label") + eol;
+      messageString +=
+          getKtrCounter()
+              + " "
+              + BaseMessages.getString(PKG, "KettleImportDialog.ImportSummary.ImportedTransf.Label")
+              + eol;
     }
     if (getOtherCounter() > 0) {
-      messageString += getOtherCounter() + " " + BaseMessages.getString(PKG, "KettleImportDialog.ImportSummary.ImportedOther.Label") + eol;
+      messageString +=
+          getOtherCounter()
+              + " "
+              + BaseMessages.getString(PKG, "KettleImportDialog.ImportSummary.ImportedOther.Label")
+              + eol;
     }
     if (getVariableCounter() > 0) {
       messageString +=
@@ -756,7 +820,9 @@ public int getKjbCounter() {
     return kjbCounter;
   }
 
-  /** @param kjbCounter The kjbCounter to set */
+  /**
+   * @param kjbCounter The kjbCounter to set
+   */
   public void setKjbCounter(int kjbCounter) {
     this.kjbCounter = kjbCounter;
   }
@@ -770,7 +836,9 @@ public int getKtrCounter() {
     return ktrCounter;
   }
 
-  /** @param ktrCounter The ktrCounter to set */
+  /**
+   * @param ktrCounter The ktrCounter to set
+   */
   public void setKtrCounter(int ktrCounter) {
     this.ktrCounter = ktrCounter;
   }
@@ -784,7 +852,9 @@ public int getOtherCounter() {
     return otherCounter;
   }
 
-  /** @param otherCounter The otherCounter to set */
+  /**
+   * @param otherCounter The otherCounter to set
+   */
   public void setOtherCounter(int otherCounter) {
     this.otherCounter = otherCounter;
   }
@@ -798,7 +868,9 @@ public String getVariablesTargetConfigFile() {
     return variablesTargetConfigFile;
   }
 
-  /** @param variablesTargetConfigFile The variablesTargetConfigFile to set */
+  /**
+   * @param variablesTargetConfigFile The variablesTargetConfigFile to set
+   */
   public void setVariablesTargetConfigFile(String variablesTargetConfigFile) {
     this.variablesTargetConfigFile = variablesTargetConfigFile;
   }
@@ -812,7 +884,9 @@ public String getConnectionsReportFileName() {
     return connectionsReportFileName;
   }
 
-  /** @param connectionsReportFileName The connectionsReportFileName to set */
+  /**
+   * @param connectionsReportFileName The connectionsReportFileName to set
+   */
   public void setConnectionsReportFileName(String connectionsReportFileName) {
     this.connectionsReportFileName = connectionsReportFileName;
   }