feat(Associations): Enabled wildcard searching for associations (#269)

dataloader.properties

@@ -162,6 +162,15 @@ clientUserIDColumn=clientContact.id
 # processEmptyAssociations -- If set to true then all To-Many association cells that are empty will remove any
 #                             existing associations. Default value is false, which will ignore the empty cells.
 #
+# wildcardMatching -- If set to true then wildcards (*) can be used within cell to search for multiple associations
+#                     without having to list all of them explicitly. For example, "Java*" can be used to match
+#                     skill entries like "Java" and "Javascript".
+#                     For Search entities, the literal matching is removed, and the full lucene syntax is supported:
+#                     See all available search options here: https://lucene.apache.org/core/2_9_4/queryparsersyntax.html
+#                     For Query entities, (*) is is replaced with (%) and the full MSSQL 'like' syntax is supported.
+#                     See all available query options here:
+#                     https://docs.microsoft.com/en-us/sql/t-sql/language-elements/like-transact-sql?view=sql-server-2017#arguments
+#
 # singleByteEncoding -- If set to true then CSV files will be read in using the ISO-8859-1 (single-byte) encoding.
 #                       If set to false then CVS files will be read in using the UTF-8 (multi-byte) encoding.
 #                       The single byte encoding covers only latin characters and some accented characters while
@@ -171,6 +180,7 @@ clientUserIDColumn=clientContact.id
 listDelimiter=;
 dateFormat=MM/dd/yy HH:mm
 processEmptyAssociations=false
+wildcardMatching=true
 singleByteEncoding=false
 
 # ---------------------------------------------------------------------------------------------------------------------

src/main/java/com/bullhorn/dataloader/enums/Property.java

@@ -23,7 +23,8 @@ public enum Property {
     TOKEN_URL("tokenUrl"),
     USERNAME("username"),
     VERBOSE("verbose"),
-    WAIT_SECONDS_BETWEEN_FILES_IN_DIRECTORY("waitSecondsBetweenFilesInDirectory");
+    WAIT_SECONDS_BETWEEN_FILES_IN_DIRECTORY("waitSecondsBetweenFilesInDirectory"),
+    WILDCARD_MATCHING("wildcardMatching");
 
     private final String propertyName;
 

src/main/java/com/bullhorn/dataloader/rest/RestApi.java

@@ -109,6 +109,7 @@ public <T extends QueryEntity> List<T> queryForList(Class<T> type,
         return list;
     }
 
+    // TODO: Remove this now that the regular queryForList is recursive
     <T extends QueryEntity & AllRecordsEntity> List<T> queryForAllRecordsList(Class<T> type,
                                                                               String where,
                                                                               Set<String> fieldSet,
@@ -162,7 +163,8 @@ public <T extends AssociationEntity, E extends BullhornEntity> List<E> getAllAss
     public <C extends CrudResponse, T extends AssociationEntity> C associateWithEntity(
         Class<T> type, Integer entityId, AssociationField<T, ? extends BullhornEntity> associationName,
         Set<Integer> associationIds) {
-        printUtil.log(Level.DEBUG, "Associate(" + type.getSimpleName() + "): #" + entityId + " - " + associationName.getAssociationFieldName());
+        printUtil.log(Level.DEBUG, "Associate(" + type.getSimpleName() + "): #" + entityId + " - " + associationName.getAssociationFieldName()
+            + " (" + associationIds.size() + " associations)");
         C crudResponse = bullhornData.associateWithEntity(type, entityId, associationName, associationIds);
         restApiExtension.checkForRestSdkErrorMessages(crudResponse);
         return crudResponse;
@@ -171,7 +173,8 @@ public <C extends CrudResponse, T extends AssociationEntity> C associateWithEnti
     public <C extends CrudResponse, T extends AssociationEntity> C disassociateWithEntity(
         Class<T> type, Integer entityId, AssociationField<T, ? extends BullhornEntity> associationName,
         Set<Integer> associationIds) {
-        printUtil.log(Level.DEBUG, "Disassociate(" + type.getSimpleName() + "): #" + entityId + " - " + associationName.getAssociationFieldName());
+        printUtil.log(Level.DEBUG, "Disassociate(" + type.getSimpleName() + "): #" + entityId + " - " + associationName.getAssociationFieldName()
+            + " (" + associationIds.size() + " associations)");
         C crudResponse = bullhornData.disassociateWithEntity(type, entityId, associationName, associationIds);
         restApiExtension.checkForRestSdkErrorMessages(crudResponse);
         return crudResponse;

src/main/java/com/bullhorn/dataloader/task/AbstractTask.java

@@ -162,28 +162,46 @@ private <Q extends QueryEntity> List<B> queryForEntity(List<Field> entityExistFi
             Sets.newHashSet("id"), ParamFactory.queryParams());
     }
 
+    // TODO: Move to utility
     String getQueryStatement(String field, String value, Class fieldType, EntityInfo fieldEntityInfo) {
+        // Fix for the Note entity doing it's own thing when it comes to the 'id' field
         if (fieldEntityInfo == EntityInfo.NOTE && field.equals(StringConsts.ID)) {
             field = StringConsts.NOTE_ID;
         }
 
         if (Integer.class.equals(fieldType) || BigDecimal.class.equals(fieldType) || Boolean.class.equals(fieldType)) {
             return field + ":" + value;
-        } else if (DateTime.class.equals(fieldType) || String.class.equals(fieldType)) {
+        } else if (DateTime.class.equals(fieldType)) {
             return field + ":\"" + value + "\"";
+        } else if (String.class.equals(fieldType)) {
+            if (propertyFileUtil.getWildcardMatching()) {
+                return field + ": " + value; // Flexible match - non quoted string (falls back to whatever text exists in the cell)
+            } else {
+                return field + ":\"" + value + "\""; // Literal match - equals quoted string
+            }
         } else {
             throw new RestApiException("Failed to create lucene search string for: '" + field
                 + "' with unsupported field type: " + fieldType);
         }
     }
 
+    // TODO: Move to utility
     String getWhereStatement(String field, String value, Class fieldType) {
         if (Integer.class.equals(fieldType) || BigDecimal.class.equals(fieldType) || Double.class.equals(fieldType)) {
             return field + "=" + value;
         } else if (Boolean.class.equals(fieldType)) {
             return field + "=" + getBooleanWhereStatement(value);
         } else if (String.class.equals(fieldType)) {
-            return field + "='" + value + "'";
+            if (propertyFileUtil.getWildcardMatching()) {
+                // Not all string fields in query entities support the like syntax. Only use it if there is a non-escaped asterisk.
+                if (value.matches(".*[^\\\\][*].*")) {
+                    return field + " like '" + value.replaceAll("[*]", "%") + "'"; // Flexible match - using like syntax
+                } else {
+                    return field + "='" + value.replaceAll("[\\\\][*]", "*") + "'"; // Literal match after unescaping asterisks
+                }
+            } else {
+                return field + "='" + value + "'"; // Literal match - equals quoted string
+            }
         } else if (DateTime.class.equals(fieldType)) {
             // TODO: This needs to be of the format: `dateOfBirth:[20170808 TO 20170808235959]` for dates
             // Format: [yyyyMMdd TO yyyyMMddHHmmss] - a date range of one day

src/main/java/com/bullhorn/dataloader/task/LoadTask.java

@@ -259,7 +259,7 @@ private List<B> findAssociations(Field field) throws InvocationTargetException,
         if (!field.getStringValue().isEmpty()) {
             Set<String> values = Sets.newHashSet(field.getStringValue().split(propertyFileUtil.getListDelimiter()));
             associations = doFindAssociations(field);
-            if (associations.size() != values.size()) {
+            if (!propertyFileUtil.getWildcardMatching() && associations.size() != values.size()) {
                 Set<String> existingAssociationValues = getFieldValueSet(field, associations);
                 if (associations.size() > values.size()) {
                     String duplicates = existingAssociationValues.stream().map(n -> "\t" + n)
@@ -280,8 +280,7 @@ private List<B> findAssociations(Field field) throws InvocationTargetException,
 
     /**
      * Makes the lookup call to check that all associated values are present, and there are no duplicates. This will
-     * work with up to 500 associated records, such as candidates or businessSectors. It will perform the lookup using
-     * the field given after the period, like: 'businessSector.name' or 'candidate.id'
+     * perform the lookup using the field given after the period, like 'name' from 'businessSector.name'.
      *
      * @param field the To-Many association field to lookup records for
      */

src/main/java/com/bullhorn/dataloader/util/PropertyFileUtil.java

@@ -45,6 +45,7 @@ public class PropertyFileUtil {
     private Integer numThreads;
     private Integer waitSecondsBetweenFilesInDirectory;
     private Boolean processEmptyAssociations;
+    private Boolean wildcardMatching;
     private Boolean singleByteEncoding;
     private Boolean resultsFileEnabled;
     private String resultsFilePath;
@@ -191,6 +192,10 @@ public Boolean getProcessEmptyAssociations() {
         return processEmptyAssociations;
     }
 
+    public Boolean getWildcardMatching() {
+        return wildcardMatching;
+    }
+
     public Boolean getSingleByteEncoding() {
         return singleByteEncoding;
     }
@@ -342,6 +347,8 @@ private void processProperties(Properties properties, PropertyValidationUtil pro
             Property.WAIT_SECONDS_BETWEEN_FILES_IN_DIRECTORY.getName()));
         processEmptyAssociations = propertyValidationUtil.validateBooleanProperty(
             Boolean.valueOf(properties.getProperty(Property.PROCESS_EMPTY_ASSOCIATIONS.getName())));
+        wildcardMatching = propertyValidationUtil.validateBooleanProperty(
+            Boolean.valueOf(properties.getProperty(Property.WILDCARD_MATCHING.getName())));
         singleByteEncoding = propertyValidationUtil.validateBooleanProperty(
             Boolean.valueOf(properties.getProperty(Property.SINGLE_BYTE_ENCODING.getName())));
         resultsFileEnabled = propertyValidationUtil.validateBooleanProperty(

src/test/java/com/bullhorn/dataloader/integration/IntegrationTest.java

@@ -64,10 +64,10 @@ public void testIntegration() throws IOException {
         // Test using more than 100,000 characters in a field
         insertUpdateDeleteFromDirectory(TestUtils.getResourceFilePath("longFields"), false);
 
-        // Test using more than 500 associations in a To-Many field
-        // TODO: This is broken right now - can't use 1000 OR statements in where clause
-        // TODO: This will be fixed when wildcard searching for multiple records is enabled
-        //insertUpdateDeleteFromDirectory(TestUtils.getResourceFilePath("associationsOver500"), false);
+        // Test using more than 500 associations in a To-Many field - requires that wildcard matching is enabled
+        System.setProperty("wildcardMatching", "true");
+        insertUpdateDeleteFromDirectory(TestUtils.getResourceFilePath("associationsOver500"), false);
+        System.setProperty("wildcardMatching", "false");
 
         // Test for ignoring soft deleted entities
         insertUpdateDeleteFromDirectory(TestUtils.getResourceFilePath("softDeletes"), true);
@@ -78,6 +78,11 @@ public void testIntegration() throws IOException {
         // Test that the byte order mark is ignored when it's present in the input file as the first (hidden) character
         insertUpdateDeleteFromDirectory(TestUtils.getResourceFilePath("byteOrderMark"), false);
 
+        // Test for wildcard associations for candidates in a note
+        System.setProperty("wildcardMatching", "true");
+        insertUpdateDeleteFromDirectory(TestUtils.getResourceFilePath("wildcardMatching"), false);
+        System.setProperty("wildcardMatching", "false");
+
         // Run a test for processing empty association fields (with the setting turned on)
         System.setProperty("processEmptyAssociations", "true");
         insertUpdateDeleteFromDirectory(TestUtils.getResourceFilePath("processEmptyFields"), false);

src/test/java/com/bullhorn/dataloader/task/LoadTaskTest.java

@@ -1128,6 +1128,25 @@ public void testRunLuceneSearchStatementAllFieldTypes() throws Exception {
         TestUtils.verifyActionTotals(actionTotalsMock, Result.Action.INSERT, 1);
     }
 
+    @Test
+    public void testRunLuceneSearchStatementWildcardMatching() throws Exception {
+        String[] headerArray = new String[]{"action", "candidates.companyName", "comments"};
+        String[] valueArray = new String[]{"Email", "Boeing*", "Candidates generated from the companyName field"};
+        Row row = TestUtils.createRow(headerArray, valueArray);
+        when(propertyFileUtilMock.getWildcardMatching()).thenReturn(true);
+        when(restApiMock.insertEntity(any())).thenReturn(TestUtils.getResponse(ChangeType.INSERT, 90));
+
+        LoadTask task = new LoadTask(EntityInfo.NOTE, row, csvFileWriterMock,
+            propertyFileUtilMock, restApiMock, printUtilMock, actionTotalsMock, completeUtilMock);
+        task.run();
+
+        String expectedQuery = "(companyName: Boeing*) AND isDeleted:0";
+        verify(restApiMock, times(1)).searchForList(eq(Candidate.class), eq(expectedQuery), any(), any());
+        Result expectedResult = new Result(Result.Status.SUCCESS, Result.Action.INSERT, 90, "");
+        verify(csvFileWriterMock, times(1)).writeRow(any(), eq(expectedResult));
+        TestUtils.verifyActionTotals(actionTotalsMock, Result.Action.INSERT, 1);
+    }
+
     @Test
     public void testRunLuceneSearchStatementNullFieldDefaults() throws Exception {
         String[] headerArray = new String[]{"dayRate", "isLockedOut", "customInt1", "customFloat1", "customDate1"};
@@ -1228,6 +1247,99 @@ public void testRunDatabaseQueryWhereStatementNullFieldDefaults() throws Excepti
         TestUtils.verifyActionTotals(actionTotalsMock, Result.Action.INSERT, 1);
     }
 
+    @Test
+    public void testRunDatabaseQueryWhereStatementToManyWildcardMatching() throws Exception {
+        String[] headerArray = new String[]{"firstName", "lastName", "primarySkills.name"};
+        String[] valueArray = new String[]{"Stephanie", "Scribbles", "Sales*;Market*;IT"};
+        Row row = TestUtils.createRow(headerArray, valueArray);
+        when(propertyFileUtilMock.getWildcardMatching()).thenReturn(true);
+        when(restApiMock.queryForList(eq(Skill.class), any(), any(), any())).thenReturn(TestUtils.getList
+            (Skill.class, 1, 2, 3, 4, 5, 6));
+        when(restApiMock.getAllAssociationsList(eq(Candidate.class), any(),
+            eq(CandidateAssociations.getInstance().primarySkills()), any(), any()))
+            .thenReturn(TestUtils.getList(Skill.class, 1, 2));
+        when(restApiMock.insertEntity(any())).thenReturn(TestUtils.getResponse(ChangeType.INSERT, 100));
+
+        LoadTask task = new LoadTask(EntityInfo.CANDIDATE, row, csvFileWriterMock,
+            propertyFileUtilMock, restApiMock, printUtilMock, actionTotalsMock, completeUtilMock);
+        task.run();
+
+        String expectedQuery = "name like 'Sales%' OR name like 'Market%' OR name='IT'";
+        verify(restApiMock, times(1)).queryForList(eq(Skill.class), eq(expectedQuery),
+            any(), any());
+        verify(restApiMock, times(1)).associateWithEntity(eq(Candidate.class),
+            eq(100), eq(CandidateAssociations.getInstance().primarySkills()), eq(Sets.newHashSet(3, 4, 5, 6)));
+        Result expectedResult = new Result(Result.Status.SUCCESS, Result.Action.INSERT, 100, "");
+        verify(csvFileWriterMock, times(1)).writeRow(any(), eq(expectedResult));
+        TestUtils.verifyActionTotals(actionTotalsMock, Result.Action.INSERT, 1);
+    }
+
+    @Test
+    public void testRunDatabaseQueryWhereStatementToOneWildcardMatching() throws Exception {
+        String[] headerArray = new String[]{"firstName", "lastName", "category.name"};
+        String[] valueArray = new String[]{"Stephanie", "Scribbles", "Sales*"};
+        Row row = TestUtils.createRow(headerArray, valueArray);
+        when(propertyFileUtilMock.getWildcardMatching()).thenReturn(true);
+        when(restApiMock.insertEntity(any())).thenReturn(TestUtils.getResponse(ChangeType.INSERT, 100));
+        when(restApiMock.queryForList(eq(Category.class), any(), any(), any())).thenReturn(TestUtils.getList
+            (Category.class, 1));
+
+        LoadTask task = new LoadTask(EntityInfo.CANDIDATE, row, csvFileWriterMock,
+            propertyFileUtilMock, restApiMock, printUtilMock, actionTotalsMock, completeUtilMock);
+        task.run();
+
+        String expectedQuery = "name like 'Sales%'";
+        verify(restApiMock, times(1)).queryForList(eq(Category.class), eq(expectedQuery),
+            any(), any());
+        Result expectedResult = new Result(Result.Status.SUCCESS, Result.Action.INSERT, 100, "");
+        verify(csvFileWriterMock, times(1)).writeRow(any(), eq(expectedResult));
+        TestUtils.verifyActionTotals(actionTotalsMock, Result.Action.INSERT, 1);
+    }
+
+    @Test
+    public void testRunDatabaseQueryWhereStatementLimitWildcardMatching() throws Exception {
+        String[] headerArray = new String[]{"firstName", "lastName", "owner.name"};
+        String[] valueArray = new String[]{"Stephanie", "Scribbles", "Bob Smiley"};
+        Row row = TestUtils.createRow(headerArray, valueArray);
+        when(propertyFileUtilMock.getWildcardMatching()).thenReturn(true);
+        when(restApiMock.insertEntity(any())).thenReturn(TestUtils.getResponse(ChangeType.INSERT, 80));
+        when(restApiMock.queryForList(eq(CorporateUser.class), any(), any(), any())).thenReturn(TestUtils.getList
+            (CorporateUser.class, 1));
+
+        LoadTask task = new LoadTask(EntityInfo.CANDIDATE, row, csvFileWriterMock,
+            propertyFileUtilMock, restApiMock, printUtilMock, actionTotalsMock, completeUtilMock);
+        task.run();
+
+        String expectedQuery = "name='Bob Smiley'";
+        verify(restApiMock, times(1)).queryForList(eq(CorporateUser.class), eq(expectedQuery),
+            any(), any());
+        Result expectedResult = new Result(Result.Status.SUCCESS, Result.Action.INSERT, 80, "");
+        verify(csvFileWriterMock, times(1)).writeRow(any(), eq(expectedResult));
+        TestUtils.verifyActionTotals(actionTotalsMock, Result.Action.INSERT, 1);
+    }
+
+    @Test
+    public void testRunDatabaseQueryWhereStatementLimitWildcardMatchingWhenEscaped() throws Exception {
+        String[] headerArray = new String[]{"firstName", "lastName", "owner.name"};
+        String[] valueArray = new String[]{"Stephanie", "Scribbles", "\\*Bob Smiley\\*"};
+        Row row = TestUtils.createRow(headerArray, valueArray);
+        when(propertyFileUtilMock.getWildcardMatching()).thenReturn(true);
+        when(restApiMock.insertEntity(any())).thenReturn(TestUtils.getResponse(ChangeType.INSERT, 80));
+        when(restApiMock.queryForList(eq(CorporateUser.class), any(), any(), any())).thenReturn(TestUtils.getList
+            (CorporateUser.class, 1));
+
+        LoadTask task = new LoadTask(EntityInfo.CANDIDATE, row, csvFileWriterMock,
+            propertyFileUtilMock, restApiMock, printUtilMock, actionTotalsMock, completeUtilMock);
+        task.run();
+
+        String expectedQuery = "name='*Bob Smiley*'";
+        verify(restApiMock, times(1)).queryForList(eq(CorporateUser.class), eq(expectedQuery),
+            any(), any());
+        Result expectedResult = new Result(Result.Status.SUCCESS, Result.Action.INSERT, 80, "");
+        verify(csvFileWriterMock, times(1)).writeRow(any(), eq(expectedResult));
+        TestUtils.verifyActionTotals(actionTotalsMock, Result.Action.INSERT, 1);
+    }
+
     @Test
     public void testRunDatabaseQueryWhereStatementInvalidValues() throws Exception {
         Row row = TestUtils.createRow("salary1,isLastJob,customInt1,companyName,startDate",