From 72ab5790343a652ed895b1e51788e68a076fb84c Mon Sep 17 00:00:00 2001 From: Nicolas Adment <39568358+nadment@users.noreply.github.com> Date: Mon, 5 Dec 2022 22:26:49 +0100 Subject: [PATCH 1/2] Fix #1886 : Cleanup XML of transform Row Normaliser --- .../transforms/normaliser/Normaliser.java | 28 +- .../normaliser/NormaliserDialog.java | 32 +-- .../normaliser/NormaliserField.java | 97 +++++++ .../transforms/normaliser/NormaliserMeta.java | 249 +++--------------- .../messages/messages_de_DE.properties | 1 - .../messages/messages_en_US.properties | 2 +- .../messages/messages_es_AR.properties | 1 - .../messages/messages_es_ES.properties | 1 - .../messages/messages_fr_FR.properties | 1 - .../messages/messages_it_IT.properties | 1 - .../messages/messages_ja_JP.properties | 1 - .../messages/messages_zh_CN.properties | 1 - .../NormaliserMetaInjectionTest.java | 7 +- .../normaliser/NormaliserMetaTest.java | 86 +----- .../transforms/normaliser/NormaliserTest.java | 16 +- .../test/resources/normaliser-transform.xml | 55 ++++ 16 files changed, 239 insertions(+), 340 deletions(-) create mode 100644 plugins/transforms/normaliser/src/main/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserField.java create mode 100644 plugins/transforms/normaliser/src/test/resources/normaliser-transform.xml diff --git a/plugins/transforms/normaliser/src/main/java/org/apache/hop/pipeline/transforms/normaliser/Normaliser.java b/plugins/transforms/normaliser/src/main/java/org/apache/hop/pipeline/transforms/normaliser/Normaliser.java index 4ad9c2735c1..6236ca04dd2 100644 --- a/plugins/transforms/normaliser/src/main/java/org/apache/hop/pipeline/transforms/normaliser/Normaliser.java +++ b/plugins/transforms/normaliser/src/main/java/org/apache/hop/pipeline/transforms/normaliser/Normaliser.java @@ -55,8 +55,6 @@ public boolean processRow() throws HopException { } List normFieldList; - int i; - int e; if (first) { // INITIALISE @@ -65,7 +63,6 @@ public boolean processRow() throws HopException { data.inputRowMeta = getInputRowMeta(); data.outputRowMeta = data.inputRowMeta.clone(); meta.getFields(data.outputRowMeta, getTransformName(), null, null, this, metadataProvider); - int normFieldsLength = meta.getNormaliserFields().length; data.typeToFieldIndex = new HashMap<>(); String typeValue; int dataFieldNr; @@ -75,8 +72,8 @@ public boolean processRow() throws HopException { data.type_occ = new ArrayList<>(); data.maxlen = 0; - for (i = 0; i < normFieldsLength; i++) { - typeValue = meta.getNormaliserFields()[i].getValue(); + for (NormaliserField field : meta.getNormaliserFields()) { + typeValue = field.getValue(); if (!data.type_occ.contains(typeValue)) { data.type_occ.add(typeValue); } @@ -93,13 +90,13 @@ public boolean processRow() throws HopException { // On a test data set with 2500 fields and about 36000 input rows (outputting over 22m // rows), the time went from // 12min to about 1min 35sec. - dataFieldNr = data.inputRowMeta.indexOfValue(meta.getNormaliserFields()[i].getName()); + dataFieldNr = data.inputRowMeta.indexOfValue(field.getName()); if (dataFieldNr < 0) { logError( BaseMessages.getString( PKG, "Normaliser.Log.CouldNotFindFieldInRow", - meta.getNormaliserFields()[i].getName())); + field.getName())); setErrors(1); stopAll(); return false; @@ -119,7 +116,7 @@ public boolean processRow() throws HopException { Set normaliserFields = meta.getFieldNames(); int irmSize = data.inputRowMeta.size(); - for (i = 0; i < irmSize; i++) { + for (int i = 0; i < irmSize; i++) { IValueMeta v = data.inputRowMeta.getValueMeta(Integer.valueOf(i)); // Backwards compatibility - old loop called Const.indexofstring which uses equalsIgnoreCase if (!normaliserFields.contains(v.getName().toLowerCase())) { @@ -144,7 +141,7 @@ public boolean processRow() throws HopException { // Now do the normalization // Loop over the unique occurrences of the different types. // - for (e = 0; e < typeOccSize; e++) { + for (int e = 0; e < typeOccSize; e++) { typeValue = data.type_occ.get(e); // Create an output row per type @@ -154,7 +151,7 @@ public boolean processRow() throws HopException { // Copy the input row data, excluding the fields that are normalized... // - for (i = 0; i < copyFldNrsSz; i++) { + for (int i = 0; i < copyFldNrsSz; i++) { nr = data.copyFieldnrs.get(i); outputRowData[outputIndex++] = r[nr]; } @@ -167,7 +164,7 @@ public boolean processRow() throws HopException { // normFieldList = data.typeToFieldIndex.get(typeValue); int normFieldListSz = normFieldList.size(); - for (i = 0; i < normFieldListSz; i++) { + for (int i = 0; i < normFieldListSz; i++) { value = r[normFieldList.get(i)]; outputRowData[outputIndex++] = value; } @@ -185,13 +182,4 @@ public boolean processRow() throws HopException { return true; } - - @Override - public boolean init() { - if (super.init()) { - // Add init code here. - return true; - } - return false; - } } diff --git a/plugins/transforms/normaliser/src/main/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserDialog.java b/plugins/transforms/normaliser/src/main/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserDialog.java index b901c2feb1c..9af48ade9bc 100644 --- a/plugins/transforms/normaliser/src/main/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserDialog.java +++ b/plugins/transforms/normaliser/src/main/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserDialog.java @@ -95,7 +95,7 @@ public String open() { shell.setText(BaseMessages.getString(PKG, "NormaliserDialog.Shell.Title")); int middle = props.getMiddlePct(); - int margin = props.getMargin(); + int margin = PropsUi.getMargin(); // Buttons at the bottom wOk = new Button(shell, SWT.PUSH); @@ -156,8 +156,7 @@ public String open() { wlFields.setLayoutData(fdlFields); final int fieldsCols = 3; - final int fieldsRows = input.getNormaliserFields().length; - + final int fieldsRows = input.getNormaliserFields().size(); colinf = new ColumnInfo[fieldsCols]; colinf[0] = new ColumnInfo( @@ -244,16 +243,17 @@ public void getData() { wTypefield.setText(input.getTypeField()); } - for (int i = 0; i < input.getNormaliserFields().length; i++) { + for (int i = 0; i < input.getNormaliserFields().size(); i++) { TableItem item = wFields.table.getItem(i); - if (input.getNormaliserFields()[i].getName() != null) { - item.setText(NAME_INDEX, input.getNormaliserFields()[i].getName()); + NormaliserField field = input.getNormaliserFields().get(i); + if (field.getName() != null) { + item.setText(NAME_INDEX, field.getName()); } - if (input.getNormaliserFields()[i].getValue() != null) { - item.setText(VALUE_INDEX, input.getNormaliserFields()[i].getValue()); + if (field.getValue() != null) { + item.setText(VALUE_INDEX,field.getValue()); } - if (input.getNormaliserFields()[i].getNorm() != null) { - item.setText(NORM_INDEX, input.getNormaliserFields()[i].getNorm()); + if (field.getNorm() != null) { + item.setText(NORM_INDEX, field.getNorm()); } } @@ -281,15 +281,17 @@ private void ok() { int i; - int nrFields = wFields.nrNonEmpty(); - input.allocate(nrFields); + int nrFields = wFields.nrNonEmpty(); + input.getNormaliserFields().clear(); // CHECKSTYLE:Indentation:OFF for (i = 0; i < nrFields; i++) { TableItem item = wFields.getNonEmpty(i); - input.getNormaliserFields()[i].setName(item.getText(NAME_INDEX)); - input.getNormaliserFields()[i].setValue(item.getText(VALUE_INDEX)); - input.getNormaliserFields()[i].setNorm(item.getText(NORM_INDEX)); + NormaliserField field = new NormaliserField(); + field.setName(item.getText(NAME_INDEX)); + field.setValue(item.getText(VALUE_INDEX)); + field.setNorm(item.getText(NORM_INDEX)); + input.getNormaliserFields().add(field); } dispose(); diff --git a/plugins/transforms/normaliser/src/main/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserField.java b/plugins/transforms/normaliser/src/main/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserField.java new file mode 100644 index 00000000000..800ab36ff6c --- /dev/null +++ b/plugins/transforms/normaliser/src/main/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserField.java @@ -0,0 +1,97 @@ +package org.apache.hop.pipeline.transforms.normaliser; + +import org.apache.hop.metadata.api.HopMetadataProperty; +import java.util.Objects; + +public class NormaliserField { + + @HopMetadataProperty(key="name", injectionKey = "NAME", injectionKeyDescription = "NormaliserMeta.Injection.NAME") + private String name; + + @HopMetadataProperty(key="value", injectionKey = "VALUE", injectionKeyDescription = "NormaliserMeta.Injection.VALUE") + private String value; + + @HopMetadataProperty(key="norm", injectionKey = "NORMALISED", injectionKeyDescription = "NormaliserMeta.Injection.NORMALISED") + private String norm; + + public NormaliserField() { + // Do nothing + } + + public NormaliserField(NormaliserField field) { + this.name = field.name; + this.value = field.value; + this.norm = field.norm; + } + + /** @return the name */ + public String getName() { + return name; + } + + /** @param name the name to set */ + public void setName(String name) { + this.name = name; + } + + /** @return the value */ + public String getValue() { + return value; + } + + /** @param value the value to set */ + public void setValue(String value) { + this.value = value; + } + + /** @return the norm */ + public String getNorm() { + return norm; + } + + /** @param norm the norm to set */ + public void setNorm(String norm) { + this.norm = norm; + } + + @Override + public int hashCode() { + return Objects.hash(name, norm, value); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + NormaliserField other = (NormaliserField) obj; + if (name == null) { + if (other.name != null) { + return false; + } + } else if (!name.equals(other.name)) { + return false; + } + if (norm == null) { + if (other.norm != null) { + return false; + } + } else if (!norm.equals(other.norm)) { + return false; + } + if (value == null) { + if (other.value != null) { + return false; + } + } else if (!value.equals(other.value)) { + return false; + } + return true; + } +} \ No newline at end of file diff --git a/plugins/transforms/normaliser/src/main/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserMeta.java b/plugins/transforms/normaliser/src/main/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserMeta.java index fd07a75965a..51c1c67cbd6 100644 --- a/plugins/transforms/normaliser/src/main/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserMeta.java +++ b/plugins/transforms/normaliser/src/main/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserMeta.java @@ -22,21 +22,16 @@ import org.apache.hop.core.ICheckResult; import org.apache.hop.core.annotations.Transform; import org.apache.hop.core.exception.HopTransformException; -import org.apache.hop.core.exception.HopXmlException; -import org.apache.hop.core.injection.Injection; -import org.apache.hop.core.injection.InjectionDeep; -import org.apache.hop.core.injection.InjectionSupported; import org.apache.hop.core.row.IRowMeta; import org.apache.hop.core.row.IValueMeta; import org.apache.hop.core.row.value.ValueMetaString; import org.apache.hop.core.variables.IVariables; -import org.apache.hop.core.xml.XmlHandler; import org.apache.hop.i18n.BaseMessages; +import org.apache.hop.metadata.api.HopMetadataProperty; import org.apache.hop.metadata.api.IHopMetadataProvider; import org.apache.hop.pipeline.PipelineMeta; import org.apache.hop.pipeline.transform.BaseTransformMeta; import org.apache.hop.pipeline.transform.TransformMeta; -import org.w3c.dom.Node; import java.util.ArrayList; import java.util.HashSet; @@ -75,18 +70,27 @@ categoryDescription = "i18n:org.apache.hop.pipeline.transform:BaseTransform.Category.Transform", keywords = "i18n::NormaliserMeta.keyword", documentationUrl = "/pipeline/transforms/rownormaliser.html") -@InjectionSupported( - localizationPrefix = "NormaliserMeta.Injection.", - groups = {"FIELDS"}) public class NormaliserMeta extends BaseTransformMeta { private static final Class PKG = NormaliserMeta.class; // For Translator - private String typeField; // Name of the new type-field. + /** Name of the new type-field */ + @HopMetadataProperty(key="typefield", injectionKey = "TYPEFIELD", injectionKeyDescription = "NormaliserMeta.Injection.TYPEFIELD") + private String typeField; - @InjectionDeep private NormaliserField[] normaliserFields = {}; + @HopMetadataProperty(groupKey = "fields", key = "field", injectionGroupKey = "FIELDS", + injectionGroupDescription = "NormaliserMeta.Injection.FIELDS") + private List normaliserFields; public NormaliserMeta() { - super(); // allocate BaseTransformMeta + super(); + this.normaliserFields = new ArrayList<>(); + } + + public NormaliserMeta(NormaliserMeta meta) { + this(); + for (NormaliserField field : meta.normaliserFields) { + normaliserFields.add(new NormaliserField(field)); + } } /** @return Returns the typeField. */ @@ -99,90 +103,35 @@ public void setTypeField(String typeField) { this.typeField = typeField; } - public NormaliserField[] getNormaliserFields() { + public List getNormaliserFields() { return normaliserFields; } - public void setNormaliserFields(NormaliserField[] normaliserFields) { + public void setNormaliserFields(List normaliserFields) { this.normaliserFields = normaliserFields; } public Set getFieldNames() { Set fieldNames = new HashSet<>(); - String s; - for (int i = 0; i < normaliserFields.length; i++) { - s = normaliserFields[i].getName(); - if (s != null) { - fieldNames.add(s.toLowerCase()); + + for (NormaliserField field : normaliserFields) { + if (field.getName() != null) { + fieldNames.add(field.getName().toLowerCase()); } } return fieldNames; } - @Override - public void loadXml(Node transformNode, IHopMetadataProvider metadataProvider) - throws HopXmlException { - readData(transformNode); - } - - public void allocate(int nrFields) { - normaliserFields = new NormaliserField[nrFields]; - for (int i = 0; i < nrFields; i++) { - normaliserFields[i] = new NormaliserField(); - } - } @Override public Object clone() { - NormaliserMeta retval = (NormaliserMeta) super.clone(); - - int nrFields = normaliserFields.length; - - retval.allocate(nrFields); - - for (int i = 0; i < nrFields; i++) { - retval.normaliserFields[i] = (NormaliserField) normaliserFields[i].clone(); - } - - return retval; - } - - private void readData(Node transformNode) throws HopXmlException { - try { - typeField = XmlHandler.getTagValue(transformNode, "typefield"); - - Node fields = XmlHandler.getSubNode(transformNode, "fields"); - int nrFields = XmlHandler.countNodes(fields, "field"); - - allocate(nrFields); - - for (int i = 0; i < nrFields; i++) { - Node fnode = XmlHandler.getSubNodeByNr(fields, "field", i); - - normaliserFields[i].setName(XmlHandler.getTagValue(fnode, "name")); - normaliserFields[i].setValue(XmlHandler.getTagValue(fnode, "value")); - normaliserFields[i].setNorm(XmlHandler.getTagValue(fnode, "norm")); - } - } catch (Exception e) { - throw new HopXmlException( - BaseMessages.getString(PKG, "NormaliserMeta.Exception.UnableToLoadTransformMetaFromXML"), - e); - } + return new NormaliserMeta(this); } @Override public void setDefault() { - typeField = "typefield"; - - int nrFields = 0; - - allocate(nrFields); - - for (int i = 0; i < nrFields; i++) { - normaliserFields[i].setName("field" + i); - normaliserFields[i].setValue("value" + i); - normaliserFields[i].setNorm("value" + i); - } + this.typeField = "typefield"; + this.normaliserFields = new ArrayList<>(); } @Override @@ -200,14 +149,15 @@ public void getFields( List normOcc = new ArrayList<>(); List fieldOcc = new ArrayList<>(); int maxlen = 0; - for (int i = 0; i < normaliserFields.length; i++) { - if (!normOcc.contains(normaliserFields[i].getNorm())) { - normOcc.add(normaliserFields[i].getNorm()); - fieldOcc.add(normaliserFields[i].getName()); + //for (int i = 0; i < normaliserFields.length; i++) { + for (NormaliserField field : normaliserFields) { + if (!normOcc.contains(field.getNorm())) { + normOcc.add(field.getNorm()); + fieldOcc.add(field.getName()); } - if (normaliserFields[i].getValue().length() > maxlen) { - maxlen = normaliserFields[i].getValue().length(); + if (field.getValue().length() > maxlen) { + maxlen = field.getValue().length(); } } @@ -239,33 +189,14 @@ public void getFields( // Now remove all the normalized fields... // - for (int i = 0; i < normaliserFields.length; i++) { - int idx = row.indexOfValue(normaliserFields[i].getName()); + for (NormaliserField field : normaliserFields) { + int idx = row.indexOfValue(field.getName()); if (idx >= 0) { row.removeValueMeta(idx); } } } - @Override - public String getXml() { - StringBuilder retval = new StringBuilder(); - - retval.append(" " + XmlHandler.addTagValue("typefield", typeField)); - - retval.append(" "); - for (int i = 0; i < normaliserFields.length; i++) { - retval.append(" "); - retval.append(" " + XmlHandler.addTagValue("name", normaliserFields[i].getName())); - retval.append(" " + XmlHandler.addTagValue("value", normaliserFields[i].getValue())); - retval.append(" " + XmlHandler.addTagValue("norm", normaliserFields[i].getNorm())); - retval.append(" "); - } - retval.append(" "); - - return retval.toString(); - } - @Override public void check( List remarks, @@ -295,18 +226,16 @@ public void check( errorMessage = ""; boolean errorFound = false; - for (int i = 0; i < normaliserFields.length; i++) { - String lufield = normaliserFields[i].getName(); - - IValueMeta v = prev.searchValueMeta(lufield); - if (v == null) { + for (NormaliserField field : normaliserFields) { + IValueMeta valueMeta = prev.searchValueMeta(field.getName()); + if (valueMeta == null) { if (first) { first = false; errorMessage += BaseMessages.getString(PKG, "NormaliserMeta.CheckResult.FieldsNotFound") + Const.CR; } errorFound = true; - errorMessage += "\t\t" + lufield + Const.CR; + errorMessage += "\t\t" + field.getName() + Const.CR; } } if (errorFound) { @@ -345,106 +274,4 @@ public void check( remarks.add(cr); } } - - public static class NormaliserField implements Cloneable { - - @Injection(name = "NAME", group = "FIELDS") - private String name; - - @Injection(name = "VALUE", group = "FIELDS") - private String value; - - @Injection(name = "NORMALISED", group = "FIELDS") - private String norm; - - public NormaliserField() { - // Do nothing - } - - /** @return the name */ - public String getName() { - return name; - } - - /** @param name the name to set */ - public void setName(String name) { - this.name = name; - } - - /** @return the value */ - public String getValue() { - return value; - } - - /** @param value the value to set */ - public void setValue(String value) { - this.value = value; - } - - /** @return the norm */ - public String getNorm() { - return norm; - } - - /** @param norm the norm to set */ - public void setNorm(String norm) { - this.norm = norm; - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((name == null) ? 0 : name.hashCode()); - result = prime * result + ((norm == null) ? 0 : norm.hashCode()); - result = prime * result + ((value == null) ? 0 : value.hashCode()); - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null) { - return false; - } - if (getClass() != obj.getClass()) { - return false; - } - NormaliserField other = (NormaliserField) obj; - if (name == null) { - if (other.name != null) { - return false; - } - } else if (!name.equals(other.name)) { - return false; - } - if (norm == null) { - if (other.norm != null) { - return false; - } - } else if (!norm.equals(other.norm)) { - return false; - } - if (value == null) { - if (other.value != null) { - return false; - } - } else if (!value.equals(other.value)) { - return false; - } - return true; - } - - @Override - public Object clone() { - try { - NormaliserField retval = (NormaliserField) super.clone(); - return retval; - } catch (CloneNotSupportedException e) { - throw new RuntimeException(e); - } - } - } } diff --git a/plugins/transforms/normaliser/src/main/resources/org/apache/hop/pipeline/transforms/normaliser/messages/messages_de_DE.properties b/plugins/transforms/normaliser/src/main/resources/org/apache/hop/pipeline/transforms/normaliser/messages/messages_de_DE.properties index 8f85b6d1008..f545a0a3bd3 100644 --- a/plugins/transforms/normaliser/src/main/resources/org/apache/hop/pipeline/transforms/normaliser/messages/messages_de_DE.properties +++ b/plugins/transforms/normaliser/src/main/resources/org/apache/hop/pipeline/transforms/normaliser/messages/messages_de_DE.properties @@ -31,7 +31,6 @@ NormaliserDialog.FailedToGetFields.DialogTitle=Das Laden der Felder ist fehlgesc NormaliserDialog.FailedToGetFields.DialogMessage=Konnte Felder von vorherigen Schritten wegen einem Fehler nicht laden Normaliser.Log.CouldNotFindFieldInRow=Konnte das Feld [{0} in der Zeile nicht finden! Normaliser.Log.LineNumber=Zeilennr -NormaliserMeta.Exception.UnableToLoadTransformMetaFromXML=Konnte die Schrittinformationen nicht von der XML-Datei laden NormaliserMeta.CheckResult.TransformReceivingFieldsOK=Schritt ist zu einem vorherignm verbunden und empfängt {0} Felder NormaliserMeta.CheckResult.FieldsNotFound=Felder zum Normalisieren wurden nicht im Input Datenstrom gefunden: NormaliserMeta.CheckResult.AllFieldsFound=Alle Felder zum Normalisieren wurden im Input Datenstrom gefunden. diff --git a/plugins/transforms/normaliser/src/main/resources/org/apache/hop/pipeline/transforms/normaliser/messages/messages_en_US.properties b/plugins/transforms/normaliser/src/main/resources/org/apache/hop/pipeline/transforms/normaliser/messages/messages_en_US.properties index b31e0a46565..d2d9c6670cc 100644 --- a/plugins/transforms/normaliser/src/main/resources/org/apache/hop/pipeline/transforms/normaliser/messages/messages_en_US.properties +++ b/plugins/transforms/normaliser/src/main/resources/org/apache/hop/pipeline/transforms/normaliser/messages/messages_en_US.properties @@ -28,7 +28,6 @@ NormaliserDialog.FailedToGetFields.DialogTitle=Get fields failed NormaliserDialog.FailedToGetFields.DialogMessage=Unable to get fields from previous transforms because of an error Normaliser.Log.CouldNotFindFieldInRow=Couldn''t find field [{0} in row! Normaliser.Log.LineNumber=linenr -NormaliserMeta.Exception.UnableToLoadTransformMetaFromXML=Unable to load transform info from XML NormaliserMeta.Exception.UnableToFindField=Unable to find field from upstream transform: {0} NormaliserMeta.CheckResult.TransformReceivingFieldsOK=Transform is connected to previous one, receiving {0} fields NormaliserMeta.CheckResult.FieldsNotFound=Fields to normalise, not found in input stream: @@ -36,6 +35,7 @@ NormaliserMeta.CheckResult.AllFieldsFound=All fields to normalise are found in t NormaliserMeta.CheckResult.CouldNotReadFieldsFromPreviousTransform=Couldn''t read fields from the previous transform. NormaliserMeta.CheckResult.TransformReceivingInfoOK=Transform is receiving info from other transforms. NormaliserMeta.CheckResult.NoInputReceivedError=No input received from other transforms\! +NormaliserMeta.Injection.TYPEFIELD=The name of the type field NormaliserMeta.Injection.FIELDS=All the fields NormaliserMeta.Injection.NAME=Input field name NormaliserMeta.Injection.VALUE=Type field value diff --git a/plugins/transforms/normaliser/src/main/resources/org/apache/hop/pipeline/transforms/normaliser/messages/messages_es_AR.properties b/plugins/transforms/normaliser/src/main/resources/org/apache/hop/pipeline/transforms/normaliser/messages/messages_es_AR.properties index 83b6e1d3ac7..a60e9a90c09 100644 --- a/plugins/transforms/normaliser/src/main/resources/org/apache/hop/pipeline/transforms/normaliser/messages/messages_es_AR.properties +++ b/plugins/transforms/normaliser/src/main/resources/org/apache/hop/pipeline/transforms/normaliser/messages/messages_es_AR.properties @@ -26,7 +26,6 @@ Normaliser.Log.LineNumber=N\u00FAmero de l\u00EDnea NormaliserDialog.GetFields.Button=\ &Obtener campos NormaliserMeta.CheckResult.TransformReceivingInfoOK=El paso est\u00E1 recibiendo informaci\u00F3n de otros pasos. NormaliserDialog.ColumnInfo.Type=Tipo -NormaliserMeta.Exception.UnableToLoadTransformMetaFromXML=No se ha podido cargar informaci\u00F3n del paso desde XML NormaliserDialog.TypeField.Label=Campo de tipo NormaliserDialog.ColumnInfo.Fieldname=Campo NormaliserDialog.Shell.Title=Normalizador de filas diff --git a/plugins/transforms/normaliser/src/main/resources/org/apache/hop/pipeline/transforms/normaliser/messages/messages_es_ES.properties b/plugins/transforms/normaliser/src/main/resources/org/apache/hop/pipeline/transforms/normaliser/messages/messages_es_ES.properties index 40acabe744d..1264d3dfe07 100644 --- a/plugins/transforms/normaliser/src/main/resources/org/apache/hop/pipeline/transforms/normaliser/messages/messages_es_ES.properties +++ b/plugins/transforms/normaliser/src/main/resources/org/apache/hop/pipeline/transforms/normaliser/messages/messages_es_ES.properties @@ -31,7 +31,6 @@ NormaliserDialog.FailedToGetFields.DialogTitle=No se ha podido obtener los campo NormaliserDialog.FailedToGetFields.DialogMessage=No se ha podido obtener los campos de pasos anteriores debido a un error Normaliser.Log.CouldNotFindFieldInRow=No se ha podido encontrar el campo [{0}] en la fila! Normaliser.Log.LineNumber=Número de línea -NormaliserMeta.Exception.UnableToLoadTransformMetaFromXML=No se ha podido cargar información del paso desde XML NormaliserMeta.CheckResult.TransformReceivingFieldsOK=Paso esta conectado al anterior, recibiendo {0} campos NormaliserMeta.CheckResult.FieldsNotFound=Campos a normalizar que no se encuentran en el flujo de entrada: NormaliserMeta.CheckResult.AllFieldsFound=Todos los campos a normalizar se encuentran en el flujo de entrada. diff --git a/plugins/transforms/normaliser/src/main/resources/org/apache/hop/pipeline/transforms/normaliser/messages/messages_fr_FR.properties b/plugins/transforms/normaliser/src/main/resources/org/apache/hop/pipeline/transforms/normaliser/messages/messages_fr_FR.properties index e2716d56d47..6e58c3d4330 100644 --- a/plugins/transforms/normaliser/src/main/resources/org/apache/hop/pipeline/transforms/normaliser/messages/messages_fr_FR.properties +++ b/plugins/transforms/normaliser/src/main/resources/org/apache/hop/pipeline/transforms/normaliser/messages/messages_fr_FR.properties @@ -31,7 +31,6 @@ NormaliserDialog.FailedToGetFields.DialogTitle=R\u00E9cup\u00E9rer les champs en NormaliserDialog.FailedToGetFields.DialogMessage=Impossible de r\u00E9cup\u00E9rer les champs depuis les transformations pr\u00E9c\u00E9dentes Normaliser.Log.CouldNotFindFieldInRow=Impossible de trouver le champ [{0} dans la ligne! Normaliser.Log.LineNumber=N\u00B0Ligne -NormaliserMeta.Exception.UnableToLoadTransformMetaFromXML=Impossible de charger la transformation depuis le fichier XML NormaliserMeta.CheckResult.TransformReceivingFieldsOK=La transformation est connect\u00E9 aux transformations pr\u00E9c\u00E9dentes et r\u00E9cup\u00E8re {0} champs NormaliserMeta.CheckResult.FieldsNotFound=Les champs \u00E0 normaliser sont introuvables dans le flux d''entr\u00E9e: NormaliserMeta.CheckResult.AllFieldsFound=Tous les champs \u00E0 normaliser ont \u00E9t\u00E9 trouv\u00E9s dans le flux d''entr\u00E9e. diff --git a/plugins/transforms/normaliser/src/main/resources/org/apache/hop/pipeline/transforms/normaliser/messages/messages_it_IT.properties b/plugins/transforms/normaliser/src/main/resources/org/apache/hop/pipeline/transforms/normaliser/messages/messages_it_IT.properties index 5abde2f7cdb..0b9254a66f2 100644 --- a/plugins/transforms/normaliser/src/main/resources/org/apache/hop/pipeline/transforms/normaliser/messages/messages_it_IT.properties +++ b/plugins/transforms/normaliser/src/main/resources/org/apache/hop/pipeline/transforms/normaliser/messages/messages_it_IT.properties @@ -24,7 +24,6 @@ Normaliser.Log.LineNumber=linea n\u00B0 NormaliserDialog.GetFields.Button=&Preleva campi NormaliserMeta.CheckResult.TransformReceivingInfoOK=Il transform sta ricevendo informazioni dai precedente transforms. NormaliserDialog.ColumnInfo.Type=Tipo -NormaliserMeta.Exception.UnableToLoadTransformMetaFromXML=Impossibile caricare le informazioni del transform da XML NormaliserDialog.TypeField.Label=Tipo campo NormaliserDialog.ColumnInfo.Fieldname=Nome campo NormaliserDialog.Shell.Title=Normalizzatore righe diff --git a/plugins/transforms/normaliser/src/main/resources/org/apache/hop/pipeline/transforms/normaliser/messages/messages_ja_JP.properties b/plugins/transforms/normaliser/src/main/resources/org/apache/hop/pipeline/transforms/normaliser/messages/messages_ja_JP.properties index d1d5bed29ff..0df520fd345 100644 --- a/plugins/transforms/normaliser/src/main/resources/org/apache/hop/pipeline/transforms/normaliser/messages/messages_ja_JP.properties +++ b/plugins/transforms/normaliser/src/main/resources/org/apache/hop/pipeline/transforms/normaliser/messages/messages_ja_JP.properties @@ -31,7 +31,6 @@ NormaliserDialog.FailedToGetFields.DialogTitle=\u30d5\u30a3\u30fc\u30eb\u30c9\u3 NormaliserDialog.FailedToGetFields.DialogMessage=\u5148\u884c\u306e\u30b9\u30c6\u30c3\u30d7\u304b\u3089\u30d5\u30a3\u30fc\u30eb\u30c9\u60c5\u5831\u3092\u53d6\u5f97\u3067\u304d\u307e\u305b\u3093\u3002 Normaliser.Log.CouldNotFindFieldInRow=Couldn''t find field [{0} in row! Normaliser.Log.LineNumber=linenr -NormaliserMeta.Exception.UnableToLoadTransformMetaFromXML=Unable to load transform info from XML NormaliserMeta.CheckResult.TransformReceivingFieldsOK=Transform is connected to previous one, receiving {0} fields NormaliserMeta.CheckResult.FieldsNotFound=Fields to normalise, not found in input stream: NormaliserMeta.CheckResult.AllFieldsFound=All fields to normalise are found in the input stream. diff --git a/plugins/transforms/normaliser/src/main/resources/org/apache/hop/pipeline/transforms/normaliser/messages/messages_zh_CN.properties b/plugins/transforms/normaliser/src/main/resources/org/apache/hop/pipeline/transforms/normaliser/messages/messages_zh_CN.properties index 7dd287e34e9..7f2dd574845 100644 --- a/plugins/transforms/normaliser/src/main/resources/org/apache/hop/pipeline/transforms/normaliser/messages/messages_zh_CN.properties +++ b/plugins/transforms/normaliser/src/main/resources/org/apache/hop/pipeline/transforms/normaliser/messages/messages_zh_CN.properties @@ -37,5 +37,4 @@ NormaliserMeta.CheckResult.NoInputReceivedError=No input received from other tra NormaliserMeta.CheckResult.TransformReceivingFieldsOK=Transform is connected to previous one, receiving {0} fields NormaliserMeta.CheckResult.TransformReceivingInfoOK=Transform is receiving info from other transforms. NormaliserMeta.Exception.UnableToFindField=\u65E0\u6CD5\u4ECE\u4E0A\u6E38 Transform \u4E2D\u627E\u5230\u5B57\u6BB5 {0} -NormaliserMeta.Exception.UnableToLoadTransformMetaFromXML=Unable to load transform info from XML NormaliserMeta.keyword=normaliser diff --git a/plugins/transforms/normaliser/src/test/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserMetaInjectionTest.java b/plugins/transforms/normaliser/src/test/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserMetaInjectionTest.java index 68137136056..b2cef4a7ee2 100644 --- a/plugins/transforms/normaliser/src/test/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserMetaInjectionTest.java +++ b/plugins/transforms/normaliser/src/test/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserMetaInjectionTest.java @@ -33,8 +33,9 @@ public void setup() throws Exception { @Test public void test() throws Exception { - check("NAME", () -> meta.getNormaliserFields()[0].getName()); - check("VALUE", () -> meta.getNormaliserFields()[0].getValue()); - check("NORMALISED", () -> meta.getNormaliserFields()[0].getNorm()); + check("TYPEFIELD", () -> meta.getTypeField()); + check("NAME", () -> meta.getNormaliserFields().get(0).getName()); + check("VALUE", () -> meta.getNormaliserFields().get(0).getValue()); + check("NORMALISED", () -> meta.getNormaliserFields().get(0).getNorm()); } } diff --git a/plugins/transforms/normaliser/src/test/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserMetaTest.java b/plugins/transforms/normaliser/src/test/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserMetaTest.java index 9d96c8ea018..ab23e8f864e 100644 --- a/plugins/transforms/normaliser/src/test/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserMetaTest.java +++ b/plugins/transforms/normaliser/src/test/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserMetaTest.java @@ -16,96 +16,32 @@ */ package org.apache.hop.pipeline.transforms.normaliser; -import org.apache.commons.lang.builder.EqualsBuilder; import org.apache.hop.core.HopEnvironment; -import org.apache.hop.core.exception.HopException; import org.apache.hop.core.plugins.PluginRegistry; import org.apache.hop.junit.rules.RestoreHopEngineEnvironment; -import org.apache.hop.pipeline.transforms.loadsave.LoadSaveTester; -import org.apache.hop.pipeline.transforms.loadsave.initializer.IInitializer; -import org.apache.hop.pipeline.transforms.loadsave.validator.ArrayLoadSaveValidator; -import org.apache.hop.pipeline.transforms.loadsave.validator.IFieldLoadSaveValidator; +import org.apache.hop.pipeline.transform.TransformSerializationTestUtil; import org.junit.Before; import org.junit.ClassRule; import org.junit.Test; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.UUID; - -public class NormaliserMetaTest implements IInitializer { - LoadSaveTester loadSaveTester; - Class testMetaClass = NormaliserMeta.class; +public class NormaliserMetaTest { + @ClassRule public static RestoreHopEngineEnvironment env = new RestoreHopEngineEnvironment(); @Before public void setUpLoadSave() throws Exception { HopEnvironment.init(); PluginRegistry.init(); - List attributes = Arrays.asList("typeField", "normaliserFields"); - - Map getterMap = new HashMap<>(); - Map setterMap = new HashMap<>(); - - Map> attrValidatorMap = new HashMap<>(); - attrValidatorMap.put( - "normaliserFields", - new ArrayLoadSaveValidator<>(new NormaliserFieldLoadSaveValidator(), 5)); - - Map> typeValidatorMap = new HashMap<>(); - - loadSaveTester = - new LoadSaveTester<>( - testMetaClass, - attributes, - getterMap, - setterMap, - attrValidatorMap, - typeValidatorMap, - this); - } - - // Call the allocate method on the LoadSaveTester meta class - @Override - public void modify(NormaliserMeta someMeta) { - if (someMeta instanceof NormaliserMeta) { - ((NormaliserMeta) someMeta).allocate(5); - } } @Test - public void testSerialization() throws HopException { - loadSaveTester.testSerialization(); - } - - // NormaliserFieldLoadSaveValidator - public class NormaliserFieldLoadSaveValidator - implements IFieldLoadSaveValidator { - final Random rand = new Random(); - - @Override - public NormaliserMeta.NormaliserField getTestObject() { - NormaliserMeta.NormaliserField rtn = new NormaliserMeta.NormaliserField(); - rtn.setName(UUID.randomUUID().toString()); - rtn.setNorm(UUID.randomUUID().toString()); - rtn.setValue(UUID.randomUUID().toString()); - return rtn; - } - - @Override - public boolean validateTestObject(NormaliserMeta.NormaliserField testObject, Object actual) { - if (!(actual instanceof NormaliserMeta.NormaliserField)) { - return false; - } - NormaliserMeta.NormaliserField another = (NormaliserMeta.NormaliserField) actual; - return new EqualsBuilder() - .append(testObject.getName(), another.getName()) - .append(testObject.getNorm(), another.getNorm()) - .append(testObject.getValue(), another.getValue()) - .isEquals(); - } + public void testSerialization() throws Exception { + NormaliserMeta meta = + TransformSerializationTestUtil.testSerialization( + "/normaliser-transform.xml", NormaliserMeta.class); + + //assertEquals(2, meta.getFields().size()); + //assertEquals("fieldName", meta.getFields().get(0).getName()); + //assertEquals("two", meta.getFields().get(0).getValue()); } } diff --git a/plugins/transforms/normaliser/src/test/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserTest.java b/plugins/transforms/normaliser/src/test/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserTest.java index dd23afc9950..bbfa68aad26 100644 --- a/plugins/transforms/normaliser/src/test/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserTest.java +++ b/plugins/transforms/normaliser/src/test/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserTest.java @@ -43,34 +43,34 @@ public static void before() throws HopException { HopEnvironment.init(); } - private NormaliserMeta.NormaliserField[] getTestNormaliserFieldsWiki() { - NormaliserMeta.NormaliserField[] rtn = new NormaliserMeta.NormaliserField[6]; - rtn[0] = new NormaliserMeta.NormaliserField(); + private NormaliserField[] getTestNormaliserFieldsWiki() { + NormaliserField[] rtn = new NormaliserField[6]; + rtn[0] = new NormaliserField(); rtn[0].setName("pr_sl"); rtn[0].setNorm("Product Sales"); rtn[0].setValue("Product1"); // Type - rtn[1] = new NormaliserMeta.NormaliserField(); + rtn[1] = new NormaliserField(); rtn[1].setName("pr1_nr"); rtn[1].setNorm("Product Number"); rtn[1].setValue("Product1"); - rtn[2] = new NormaliserMeta.NormaliserField(); + rtn[2] = new NormaliserField(); rtn[2].setName("pr2_sl"); rtn[2].setNorm("Product Sales"); rtn[2].setValue("Product2"); - rtn[3] = new NormaliserMeta.NormaliserField(); + rtn[3] = new NormaliserField(); rtn[3].setName("pr2_nr"); rtn[3].setNorm("Product Number"); rtn[3].setValue("Product2"); - rtn[4] = new NormaliserMeta.NormaliserField(); + rtn[4] = new NormaliserField(); rtn[4].setName("pr3_sl"); rtn[4].setNorm("Product Sales"); rtn[4].setValue("Product3"); - rtn[5] = new NormaliserMeta.NormaliserField(); + rtn[5] = new NormaliserField(); rtn[5].setName("pr3_nr"); rtn[5].setNorm("Product Number"); rtn[5].setValue("Product3"); diff --git a/plugins/transforms/normaliser/src/test/resources/normaliser-transform.xml b/plugins/transforms/normaliser/src/test/resources/normaliser-transform.xml new file mode 100644 index 00000000000..89733e17dfd --- /dev/null +++ b/plugins/transforms/normaliser/src/test/resources/normaliser-transform.xml @@ -0,0 +1,55 @@ + + + + + + Normalise fieldA-D + Normaliser + + Y + + 1 + + none + + + type + + + fieldA + A + field + + + fieldB + B + field + + + fieldC + C + field + + + fieldD + D + field + + + + + 272 + 96 + + + From b91caa30b0d07a7a082be7240b1d7bc894a8fee0 Mon Sep 17 00:00:00 2001 From: Nicolas Adment <39568358+nadment@users.noreply.github.com> Date: Mon, 5 Dec 2022 23:43:25 +0100 Subject: [PATCH 2/2] Fix #1886 : Fix RAT --- .../transforms/normaliser/NormaliserField.java | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/plugins/transforms/normaliser/src/main/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserField.java b/plugins/transforms/normaliser/src/main/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserField.java index 800ab36ff6c..286dc41c45f 100644 --- a/plugins/transforms/normaliser/src/main/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserField.java +++ b/plugins/transforms/normaliser/src/main/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserField.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.hop.pipeline.transforms.normaliser; import org.apache.hop.metadata.api.HopMetadataProperty;