Normalize strings transformation

configurations/default/server.yml.tmp

@@ -34,6 +34,28 @@ modules:
     load_on_fetch: false
     # use_extension: mtc
     # update_frequency: 30 # in seconds
+  manager:
+    normalizeFieldTransformation:
+      # Enter capitalization exceptions (e.g. acronyms), in the desired case, and separated by commas.
+      defaultCapitalizationExceptions:
+        - ACE
+        - BART
+      # Enter substitutions (e.g. substitute '@' with 'at'), one dashed entry for each substitution, with:
+      # - pattern: the regex string pattern that will be replaced,
+      # - replacement: the replacement string for that pattern,
+      # - normalizeSpace: if true, the resulting field value will include one space before and after the replacement string.
+      #       Note: if the replacement must be blank, then normalizeSpace should be set to false
+      #       and whitespace management should be handled in pattern instead.
+      # Substitutions are executed in order they appear in the list.
+      defaultSubstitutions:
+        - description: "Replace '@' with 'at', and normalize space."
+          pattern: "@"
+          replacement: at
+          normalizeSpace: true
+        - description: "Replace '+' (\\+ in regex) and '&' with 'and', and normalize space."
+          pattern: "[\\+&]"
+          replacement: and
+          normalizeSpace: true
 extensions:
   transitland:
     enabled: true

configurations/test/server.yml.tmp

@@ -35,6 +35,39 @@ modules:
     load_on_fetch: false
     # use_extension: mtc
     # update_frequency: 30 # in seconds
+  manager:
+    normalizeFieldTransformation:
+      # Enter capitalization exceptions (e.g. acronyms), in the desired case, and separated by commas.
+      defaultCapitalizationExceptions:
+        - ACE
+        - BART
+        - SMART
+        - EB
+        - WB
+        - SB
+        - NB
+      # Enter substitutions (e.g. substitute '@' with 'at'), one dashed entry for each substitution, with:
+      # - pattern: the regex string pattern that will be replaced,
+      # - replacement: the replacement string for that pattern,
+      # - normalizeSpace: if true, the resulting field value will include one space before and after the replacement string.
+      #       Note: if the replacement must be blank, then normalizeSpace should be set to false
+      #       and whitespace management should be handled in pattern instead.
+      # Substitutions are executed in order they appear in the list.
+      defaultSubstitutions:
+        - description: "Replace '@' with 'at', and normalize space."
+          pattern: "@"
+          replacement: at
+          normalizeSpace: true
+        - description: "Replace '+' (\\+ in regex) and '&' with 'and, and normalize space."
+          pattern: "[\\+&]"
+          replacement: and
+          normalizeSpace: true
+        - description: "Remove content in parentheses and adjacent space outside the parentheses."
+          pattern: "\\s*\\(.+\\)\\s*"
+          replacement: ""
+        - description: "Remove content in square brackets and adjacent space outside the brackets."
+          pattern: "\\s*\\[.+\\]\\s*"
+          replacement: ""
 extensions:
   # Enable MTC extension so MTC-specific feed merge tests
   mtc:

pom.xml

@@ -39,7 +39,7 @@
         <url>https://github.com/ibi-group/datatools-server.git</url>
     </scm>
     <properties>
-        <jackson.version>2.10.1</jackson.version>
+        <jackson.version>2.12.1</jackson.version>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <!-- Using the latest version of geotools (e.g, 20) seems to cause issues with the shapefile
         plugin where the_geom for each feature is null. -->
@@ -255,6 +255,13 @@
             <version>5.7.0</version>
             <scope>test</scope>
         </dependency>
+        <!-- Provides JUnit 5.x parametrized tests -->
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-params</artifactId>
+            <version>5.5.2</version>
+            <scope>test</scope>
+        </dependency>
 
         <!-- Used for loading/fetching/validating/writing GTFS entities. gtfs-lib also provides access to:
          - commons-io - generic utilities
@@ -435,7 +442,6 @@
             <artifactId>aws-java-sdk-sts</artifactId>
             <version>${awsjavasdk.version}</version>
         </dependency>
-
     </dependencies>
 
 </project>

src/main/java/com/conveyal/datatools/common/status/MonitorableJob.java

@@ -288,15 +288,16 @@ private String getCallingMethodTrace() {
          * Shorthand method to update status object on successful job completion.
          */
         public void completeSuccessfully(String message) {
-            this.complete(false, message);
+            // Do not overwrite the message (and other fields), if the job has already been completed.
+            if (!this.completed) this.complete(false, message);
         }
 
         /**
          * Set job status to completed with error and message information.
          */
         private void complete(boolean isError, String message) {
             this.error = isError;
-            // Skip message update if null.
+            // Skip message update if the job message is null or the message has already been defined.
             if (message != null) this.message = message;
             this.percentComplete = 100;
             this.completed = true;

src/main/java/com/conveyal/datatools/manager/jobs/ArbitraryTransformJob.java

@@ -12,7 +12,7 @@
 /**
  * This job will apply a {@link ZipTransformation} or {@link DbTransformation} to a GTFS zip file or database namespace,
  * respectively, and generate the required FeedTransformTarget object from those inputs, which is passed into the
- * {@link FeedTransformation#transform} method.
+ * {@link FeedTransformation#doTransform} method.
  */
 public class ArbitraryTransformJob extends MonitorableJob {
 
@@ -43,6 +43,6 @@ public void jobLogic() {
         target.validate(status);
         if (status.error) return;
         // If target is valid, perform transformation.
-        transformation.transform(target, status);
+        transformation.doTransform(target, status);
     }
 }

src/main/java/com/conveyal/datatools/manager/models/transform/DbTransformation.java

@@ -1,13 +1,33 @@
 package com.conveyal.datatools.manager.models.transform;
 
+import com.conveyal.datatools.common.status.MonitorableJob;
+import com.conveyal.gtfs.loader.JdbcGtfsLoader;
+
 import java.util.List;
 
 /**
  * This is an abstract class that represents a transformation that should apply to a GTFS in database form. In other
  * words, subclasses will provide a transform override method that acts on a database namespace. Sample fields
  * matchField and matchValues can be used to construct a WHERE clause for applying updates to a filtered set of records.
  */
-public abstract class DbTransformation extends FeedTransformation {
+public abstract class DbTransformation extends FeedTransformation<FeedTransformDbTarget> {
     public String matchField;
     public List<String> matchValues;
+
+    protected String getTransformationTypeName() {
+        return FeedTransformDbTarget.class.getSimpleName();
+    }
+
+    protected void validateFieldNames(MonitorableJob.Status status) {
+        // Validate fields before running transform.
+
+        if (matchField == null) {
+            status.fail("Must provide valid match field");
+            return;
+        }
+        String cleanField = JdbcGtfsLoader.sanitize(matchField, null);
+        if (!matchField.equals(cleanField)) {
+            status.fail("Input match field contained disallowed special characters (only alphanumeric and underscores are permitted).");
+        }
+    }
 }

src/main/java/com/conveyal/datatools/manager/models/transform/DeleteRecordsTransformation.java

@@ -1,24 +1,18 @@
 package com.conveyal.datatools.manager.models.transform;
 
 import com.conveyal.datatools.common.status.MonitorableJob;
-import com.conveyal.datatools.manager.DataManager;
-import com.conveyal.datatools.manager.gtfsplus.tables.GtfsPlusTable;
 import com.conveyal.datatools.manager.models.Snapshot;
 import com.conveyal.datatools.manager.models.TableTransformResult;
 import com.conveyal.datatools.manager.persistence.Persistence;
-import com.conveyal.gtfs.loader.JdbcGtfsLoader;
-import com.conveyal.gtfs.loader.Table;
 import com.conveyal.gtfs.util.InvalidNamespaceException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.sql.Connection;
 import java.sql.PreparedStatement;
 import java.sql.SQLException;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
-import java.util.stream.Collectors;
 
 import static com.conveyal.datatools.manager.DataManager.GTFS_DATA_SOURCE;
 import static com.conveyal.gtfs.util.Util.ensureValidNamespace;
@@ -43,50 +37,14 @@ public static DeleteRecordsTransformation create(String table, String matchField
     }
 
     @Override
-    public void transform(FeedTransformTarget target, MonitorableJob.Status status) {
-        if (!(target instanceof FeedTransformDbTarget)) {
-            status.fail("Target must be FeedTransformDbTarget.");
-            return;
-        }
-        // Cast transform target to DB flavor.
-        FeedTransformDbTarget dbTarget = (FeedTransformDbTarget)target;
+    public void transform(FeedTransformDbTarget dbTarget, MonitorableJob.Status status) {
         // Fetch the referenced snapshot to transform.
         Snapshot snapshot = Persistence.snapshots.getById(dbTarget.snapshotId);
         if (snapshot == null) {
             status.fail(String.format("Cannot find snapshot to transform (id=%s)", dbTarget.snapshotId));
             return;
         }
 
-        // TODO: Move validation code into its own method?
-        final List<Table> tables =
-            Arrays.stream(Table.tablesInOrder)
-                .filter(Table::isSpecTable)
-                .collect(Collectors.toList());
-        if (DataManager.isModuleEnabled("gtfsplus")) {
-            // Add GTFS+ tables only if MTC extension is enabled.
-            tables.addAll(Arrays.asList(GtfsPlusTable.tables));
-        }
-        // Check that table name is valid (to prevent SQL injection).
-        Table matchingTable = null;
-        for (Table specTable : tables) {
-            if (specTable.name.equals(table)) {
-                matchingTable = specTable;
-                break;
-            }
-        }
-        if (matchingTable == null) {
-            status.fail("Table must be valid GTFS spec table name (without .txt).");
-            return;
-        }
-        if (matchField == null) {
-            status.fail("Must provide valid match field");
-            return;
-        }
-        String cleanField = JdbcGtfsLoader.sanitize(matchField, null);
-        if (!matchField.equals(cleanField)) {
-            status.fail("Input match field contained disallowed special characters (only alphanumeric and underscores are permitted).");
-            return;
-        }
         try {
             ensureValidNamespace(snapshot.namespace);
         } catch (InvalidNamespaceException e) {
@@ -111,9 +69,14 @@ public void transform(FeedTransformTarget target, MonitorableJob.Status status)
             int deleted = preparedStatement.executeUpdate();
             LOG.info("{} deleted {} records", this.getClass().getSimpleName(), deleted);
             connection.commit();
-            target.feedTransformResult.tableTransformResults.add(new TableTransformResult(table, deleted, 0, 0));
+            dbTarget.feedTransformResult.tableTransformResults.add(new TableTransformResult(table, deleted, 0, 0));
         } catch (SQLException e) {
             throw new RuntimeException(e);
         }
     }
+
+    @Override
+    public void validateParameters(MonitorableJob.Status status) {
+        // Does nothing.
+    }
 }

src/main/java/com/conveyal/datatools/manager/models/transform/FeedTransformation.java

@@ -1,14 +1,14 @@
 package com.conveyal.datatools.manager.models.transform;
 
 import com.conveyal.datatools.common.status.MonitorableJob;
+import com.conveyal.datatools.manager.utils.GtfsUtils;
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.fasterxml.jackson.annotation.JsonSubTypes;
 import com.fasterxml.jackson.annotation.JsonTypeInfo;
 import org.bson.codecs.pojo.annotations.BsonDiscriminator;
 
 import java.io.Serializable;
 
-
 /**
  * This abstract class is the base for arbitrary feed transformations.
  *
@@ -28,10 +28,11 @@
 @JsonTypeInfo(use = JsonTypeInfo.Id.NAME)
 @JsonSubTypes({
     @JsonSubTypes.Type(value = DeleteRecordsTransformation.class, name = "DeleteRecordsTransformation"),
+    @JsonSubTypes.Type(value = NormalizeFieldTransformation.class, name = "NormalizeFieldTransformation"),
     @JsonSubTypes.Type(value = ReplaceFileFromVersionTransformation.class, name = "ReplaceFileFromVersionTransformation"),
     @JsonSubTypes.Type(value = ReplaceFileFromStringTransformation.class, name = "ReplaceFileFromStringTransformation")
 })
-public abstract class FeedTransformation implements Serializable {
+public abstract class FeedTransformation<Target extends FeedTransformTarget> implements Serializable {
     private static final long serialVersionUID = 1L;
     public String table;
     public boolean active = true;
@@ -46,5 +47,58 @@ public boolean isActive() {
     //  it could return something object that contains a bool + message.
     // boolean isValid();
 
-    public abstract void transform(FeedTransformTarget target, MonitorableJob.Status status);
+    public void doTransform(FeedTransformTarget target, MonitorableJob.Status status) {
+        try {
+            // Attempt to cast transform target to correct flavor
+            // (fails the job if types mismatch.)
+            Target typedTarget = (Target)target;
+
+            // Validate parameters before running transform.
+            validateTableName(status);
+            validateFieldNames(status);
+            // Let subclasses check parameters.
+            validateParameters(status);
+            if (status.error) {
+                return;
+            }
+
+            // Pass the typed transform target to subclasses to transform.
+            transform(typedTarget, status);
+        } catch (ClassCastException classCastException) {
+            status.fail(
+                String.format("Transformation must be of type '%s'.", getTransformationTypeName())
+            );
+        }
+    }
+
+    protected abstract String getTransformationTypeName();
+
+    /**
+     * Contains the logic for this database-bound transformation.
+     * @param target The database-bound or ZIP-file-bound target the transformation will operate on.
+     * @param status Used to report success or failure status and details.
+     */
+    public abstract void transform(Target target, MonitorableJob.Status status);
+
+    /**
+     * At the moment, used by DbTransformation to validate field names.
+     */
+    protected abstract void validateFieldNames(MonitorableJob.Status status);
+
+    /**
+     * Handles validation logic prior to performing the transformation.
+     * Calling status.fail prevents the transform logic from running.
+     */
+    public abstract void validateParameters(MonitorableJob.Status status);
+
+    /**
+     * Checks that the table name for the transform is valid.
+     */
+    protected void validateTableName(MonitorableJob.Status status) {
+        // Validate fields before running transform.
+        if (GtfsUtils.getGtfsTable(table) == null) {
+            status.fail("Table must be valid GTFS spec table name (without .txt).");
+            return;
+        }
+    }
 }