Skip to content

Commit

Permalink
Merge pull request #6243 from deutschebank/db-contrib/waltz-6221-add-…
Browse files Browse the repository at this point in the history 
…string-filters-to-filter-group-population

Db contrib/waltz 6221 add string filters to filter group population
  • Loading branch information
@davidwatkins73
davidwatkins73 authored Oct 5, 2022
2 parents e3f877f + 09a1cf8 commit 9a6f226
Show file tree
Hide file tree
Showing 5 changed files with 152 additions and 76 deletions.
18 changes: 18 additions & 0 deletions waltz-model/src/main/java/org/finos/waltz/model/report_grid/FilterOperator.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
package org.finos.waltz.model.report_grid;

import java.util.Optional;

public enum FilterOperator {
CONTAINS_ANY_OPTION,
CONTAINS_ANY_STRING;

public static Optional<FilterOperator> parseString(String operator) {
if (operator.equalsIgnoreCase(CONTAINS_ANY_OPTION.name())) {
return Optional.of(CONTAINS_ANY_OPTION);
} else if (operator.equalsIgnoreCase(CONTAINS_ANY_STRING.name())) {
return Optional.of(CONTAINS_ANY_STRING);
} else {
return Optional.empty();
}
}
}
4 changes: 3 additions & 1 deletion waltz-model/src/main/java/org/finos/waltz/model/report_grid/GridFilter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ public abstract class GridFilter {

public abstract Long columnDefinitionId();

public abstract Set<String> optionCodes();
public abstract FilterOperator filterOperator();

public abstract Set<String> filterValues();

}
97 changes: 68 additions & 29 deletions ...ervice/src/main/java/org/finos/waltz/service/report_grid/ReportGridFilterViewService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

package org.finos.waltz.service.report_grid;

import org.finos.waltz.common.CollectionUtilities;
import org.finos.waltz.common.SetUtilities;
import org.finos.waltz.data.GenericSelectorFactory;
import org.finos.waltz.data.report_grid.ReportGridDao;
Expand All @@ -43,11 +44,13 @@
import static java.lang.String.format;
import static java.util.Collections.emptySet;
import static org.finos.waltz.common.Checks.checkNotNull;
import static org.finos.waltz.common.CollectionUtilities.*;
import static org.finos.waltz.common.CollectionUtilities.first;
import static org.finos.waltz.common.CollectionUtilities.isEmpty;
import static org.finos.waltz.common.ListUtilities.map;
import static org.finos.waltz.common.MapUtilities.groupBy;
import static org.finos.waltz.common.MapUtilities.indexBy;
import static org.finos.waltz.common.SetUtilities.*;
import static org.finos.waltz.common.StringUtilities.notEmpty;
import static org.finos.waltz.model.EntityReference.mkRef;
import static org.finos.waltz.model.IdSelectionOptions.mkOpts;
import static org.finos.waltz.service.report_grid.ReportGridUtilities.*;
Expand Down Expand Up @@ -187,32 +190,15 @@ private Set<Long> applyFilters(Set<ReportGridCell> cellData,

Set<Set<Long>> appIdsPassingFilters = gridFilters
.stream()
.map(d -> {
Collection<ReportGridCell> cellDataForColumn = dataByCol.getOrDefault(d.columnDefinitionId(), emptySet());

Set<Long> appsPassingFilter = cellDataForColumn
.stream()
.filter(c -> {
// rating cells may want to look up on rating id / code / external id
if (c.ratingIdValue() != null) {
RatingSchemeItem rating = ratingSchemeItemByIdMap.get(c.ratingIdValue());
Set<String> ratingIdentifiers = asSet(c.optionCode(), String.valueOf(rating.rating()), rating.name(), rating.externalId().orElse(null));
return notEmpty(intersection(d.optionCodes(), ratingIdentifiers));
} else {
return d.optionCodes().contains(c.optionCode());
}
})
.map(ReportGridCell::subjectId)
.collect(Collectors.toSet());


if (d.optionCodes().contains(NOT_PROVIDED_OPTION_CODE)) {
Set<Long> subjectIdsWithValues = SetUtilities.map(cellDataForColumn, ReportGridCell::subjectId);
Set<Long> subjectIdsWithoutValue = minus(subjectIds, subjectIdsWithValues);

return union(appsPassingFilter, subjectIdsWithoutValue);
.map(filter -> {
Collection<ReportGridCell> cellDataForColumn = dataByCol.getOrDefault(filter.columnDefinitionId(), emptySet());

if (filter.filterOperator().equals(FilterOperator.CONTAINS_ANY_OPTION)) {
return determineAppsPassingContainsOperatorFilter(subjectIds, ratingSchemeItemByIdMap, filter, cellDataForColumn);
} else if (filter.filterOperator().equals(FilterOperator.CONTAINS_ANY_STRING)) {
return determineAppsPassingContainsStringFilter(filter, cellDataForColumn);
} else {
return appsPassingFilter;
return subjectIds; // return all apps if filter operator not supported to support intersection
}
})
.collect(Collectors.toSet());
Expand All @@ -224,6 +210,59 @@ private Set<Long> applyFilters(Set<ReportGridCell> cellData,
}


private Set<Long> determineAppsPassingContainsStringFilter(GridFilter filter,
Collection<ReportGridCell> cellDataForColumn) {
Set<Long> cellsPassingFilters = cellDataForColumn
.stream()
.filter(c -> notEmpty(c.textValue()) && containsAny(filter.filterValues(), c.textValue()))
.map(ReportGridCell::subjectId)
.collect(Collectors.toSet());

return cellsPassingFilters;
}


private boolean containsAny(Set<String> searchStrings, String lookupString) {
for (String text : searchStrings) {
if (lookupString.contains(text)) {
return true;
}
}
return false;
}


private Set<Long> determineAppsPassingContainsOperatorFilter(Set<Long> subjectIds,
Map<Long, RatingSchemeItem> ratingSchemeItemByIdMap,
GridFilter filter,
Collection<ReportGridCell> cellDataForColumn) {
Set<Long> appsPassingFilter = cellDataForColumn
.stream()
.filter(c -> {
// rating cells may want to look up on rating id / code / external id
if (c.ratingIdValue() != null) {
RatingSchemeItem rating = ratingSchemeItemByIdMap.get(c.ratingIdValue());
Set<String> ratingIdentifiers = asSet(c.optionCode(), String.valueOf(rating.rating()), rating.name(), rating.externalId().orElse(null));
return CollectionUtilities.notEmpty(intersection(filter.filterValues(), ratingIdentifiers));
} else {
return filter.filterValues().contains(c.optionCode());
}
})
.map(ReportGridCell::subjectId)
.collect(Collectors.toSet());


if (filter.filterValues().contains(NOT_PROVIDED_OPTION_CODE)) {
Set<Long> subjectIdsWithValues = SetUtilities.map(cellDataForColumn, ReportGridCell::subjectId);
Set<Long> subjectIdsWithoutValue = minus(subjectIds, subjectIdsWithValues);

return union(appsPassingFilter, subjectIdsWithoutValue);
} else {
return appsPassingFilter;
}
}


private Set<ReportGridFilterInfo> findGridInfoWithFilters() {

Set<EntityNamedNote> filterPresetNotes = entityNamedNoteService.findByNoteTypeExtId(REPORT_GRID_APP_GROUP_CREATION_NOTE_TYPE_EXT_ID);
Expand All @@ -242,9 +281,9 @@ private Set<ReportGridFilterInfo> findGridInfoWithFilters() {
}


private ImmutableReportGridFilterInfo getGridFilterInfo(Map<String, ReportGridDefinition> gridsByExternalId,
Long appGroupId,
String noteText) {
private ReportGridFilterInfo getGridFilterInfo(Map<String, ReportGridDefinition> gridsByExternalId,
Long appGroupId,
String noteText) {

Tuple2<List<String>, List<List<String>>> gridInfoAndFilters;

Expand Down
48 changes: 31 additions & 17 deletions waltz-service/src/main/java/org/finos/waltz/service/report_grid/ReportGridUtilities.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,7 @@
import org.finos.waltz.model.HierarchyQueryScope;
import org.finos.waltz.model.IdSelectionOptions;
import org.finos.waltz.model.ImmutableIdSelectionOptions;
import org.finos.waltz.model.report_grid.GridFilter;
import org.finos.waltz.model.report_grid.ImmutableGridFilter;
import org.finos.waltz.model.report_grid.ReportGridDefinition;
import org.finos.waltz.model.report_grid.ReportGridFixedColumnDefinition;
import org.finos.waltz.model.report_grid.*;
import org.jooq.lambda.tuple.Tuple2;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand All @@ -28,7 +25,11 @@ public class ReportGridUtilities {

private static final Logger LOG = LoggerFactory.getLogger(ReportGridUtilities.class);
private static final int HEADER_COLUMN_COUNT = 4;
private static final int FILTER_OPTIONS_COLUMN_COUNT = 2;
private static final int FILTER_OPTIONS_COLUMN_COUNT = 3;
private static final String TABLE_HEADER = "| Grid Name | Grid Identifier | Vantage Point Kind | Vantage Point Id |";
private static final String FILTER_HEADER = "| Filter Column | Filter Operator | Value/s |";
private static final String FILTER_DELIMETER = ";";


public static Tuple2<List<String>, List<List<String>>> parseGridFilterNoteText(String noteText) {

Expand All @@ -38,11 +39,9 @@ public static Tuple2<List<String>, List<List<String>>> parseGridFilterNoteText(S

String[] lines = noteText.split("\\r?\\n");

String tableHeader = "| Grid Name | Grid Identifier | Vantage Point Kind | Vantage Point Id |";
String filterHeader = "| Filter Column | Column Option Codes |";

List<List<String>> headerRows = parseTableData(lines, tableHeader);
List<List<String>> filterRows = parseTableData(lines, filterHeader);
List<List<String>> headerRows = parseTableData(lines, TABLE_HEADER);
List<List<String>> filterRows = parseTableData(lines, FILTER_HEADER);

if (headerRows.size() != 1) {
throw new IllegalArgumentException(format(
Expand All @@ -68,7 +67,7 @@ public static Tuple2<List<String>, List<List<String>>> parseGridFilterNoteText(S

if (filterRow.size() != FILTER_OPTIONS_COLUMN_COUNT) {
throw new IllegalArgumentException(format(
"Incorrect number of filter columns found [%d], should follow : [Filter Column, Column Option Codes]",
"Incorrect number of filter columns found [%d], should follow : [Filter Column, Filter Operator, Value/s]",
filterRow.size()));
}

Expand All @@ -78,7 +77,7 @@ public static Tuple2<List<String>, List<List<String>>> parseGridFilterNoteText(S

public static Set<String> getFilterValues(String string) {
return Arrays
.stream(string.split(";"))
.stream(string.split(FILTER_DELIMETER))
.map(String::trim)
.collect(Collectors.toSet());
}
Expand Down Expand Up @@ -115,7 +114,7 @@ public static List<List<String>> parseTableData(String[] lines, String tableHead

List<String> cellData = Arrays
.stream(line.split("\\|"))
.map(String::trim)
.map(s -> s.replaceAll("`", "").trim())
.filter(s -> !StringUtilities.isEmpty(s))
.collect(Collectors.toList());

Expand Down Expand Up @@ -150,17 +149,32 @@ public static Set<GridFilter> getGridFilters(List<List<String>> filterRows, Repo
return filterRows
.stream()
.map(r -> {
String columnName = sanitizeString(r.get(0));

String columnString = r.get(0);
String filterOperator = r.get(1);
String values = r.get(2);

String columnName = sanitizeString(columnString);
Long columnDefnId = columnsDefinitionIdByName.get(columnName);

if (columnDefnId == null) {
LOG.info(format("Cannot find column '%s' on grid. Skipping this filter", columnName));
return null;
} else {
return ImmutableGridFilter.builder()
.columnDefinitionId(columnDefnId)
.optionCodes(getFilterValues(r.get(1)))
.build();

Optional<FilterOperator> operator = FilterOperator.parseString(filterOperator);

return operator
.map(op -> ImmutableGridFilter.builder()
.columnDefinitionId(columnDefnId)
.filterOperator(op)
.filterValues(getFilterValues(values))
.build())
.orElseGet(() -> {
LOG.info(format("Cannot parse filter operator: '%s'. Skipping this filter", filterOperator));
return null;
});

}
})
.filter(Objects::nonNull)
Expand Down
61 changes: 32 additions & 29 deletions waltz-service/src/test/java/org/finos/waltz/service/report_grid/ReportGridUtilitiesTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,55 +10,58 @@

public class ReportGridUtilitiesTest {

private static final String HEADER_TEXT = "| Grid Name | Grid Identifier | Vantage Point Kind | Vantage Point Id |";
private static final String FILTER_INFO_TEXT = "| Filter Column | Filter Operator | Value/s |";

private final String TEST_STRING = "| Grid Name | Grid Identifier | Vantage Point Kind | Vantage Point Id |\n" +
"| --- | --- | --- | --- |\n" +
"| Grid Name | {GRIDEXTID} | ORG_UNIT | 1 |\n" +
"| `Grid Name` | `{GRIDEXTID}` | `ORG_UNIT` | `1` |\n" +
"\n" +
"\n" +
"| Filter Column | Column Option Codes |\n" +
"| Filter Column | Filter Operator | Value/s |\n" +
"| --- | --- |\n" +
"| Asset Kind/Application | PROVIDED |\n" +
"| Developer | PROVIDED |";
"| `Asset Kind/Application` | `CONTAINS_ANY_OPTION` | `PROVIDED` |\n" +
"| `Developer` | `CONTAINS_ANY_OPTION` | `PROVIDED` |";

private final String NO_SPACED_TABLES = "| Grid Name | Grid Identifier | Vantage Point Kind | Vantage Point Id |\n" +
"| --- | --- | --- | --- |\n" +
"| Grid Name | {GRIDEXTID} | ORG_UNIT | 1 |\n" +
"| Filter Column | Column Option Codes |\n" +
"| `Grid Name` | `{GRIDEXTID}` | `ORG_UNIT` | `1` |\n" +
"| Filter Column | Filter Operator | Value/s |\n" +
"| --- | --- |\n" +
"| Asset Kind/Application | PROVIDED |\n" +
"| Developer | PROVIDED |";
"| `Asset Kind/Application` | `CONTAINS_ANY_OPTION` | `PROVIDED` |\n" +
"| `Developer` | `CONTAINS_ANY_OPTION` | `PROVIDED` |";

private final String TOO_MANY_HEADERS = "| Grid Name | Grid Identifier | Vantage Point Kind | Vantage Point Id |\n" +
"| --- | --- | --- | --- |\n" +
"| Grid Name | {GRIDEXTID} | ORG_UNIT | 1 | 12 |\n" +
"| Filter Column | Column Option Codes |\n" +
"| `Grid Name` | `{GRIDEXTID}` | `ORG_UNIT` | `1` | `12` |\n" +
"| Filter Column | Filter Operator | Value/s |\n" +
"| --- | --- |\n" +
"| Asset Kind/Application | PROVIDED |\n" +
"| Developer | PROVIDED |";
"| `Asset Kind/Application` | `CONTAINS_ANY_OPTION` | `PROVIDED` |\n" +
"| `Developer` | `CONTAINS_ANY_OPTION` | `PROVIDED` |";

private final String NOT_ENOUGH_HEADERS = "| Grid Name | Grid Identifier | Vantage Point Kind | Vantage Point Id |\n" +
"| --- | --- | --- | --- |\n" +
"| Grid Name | {GRIDEXTID} |\n" +
"| Filter Column | Column Option Codes |\n" +
"| `Grid Name` | `{GRIDEXTID}` |\n" +
"| Filter Column | Filter Operator | Value/s |\n" +
"| --- | --- |\n" +
"| Asset Kind/Application | PROVIDED |\n" +
"| Developer | PROVIDED |";
"| `Asset Kind/Application` | `CONTAINS_ANY_OPTION` | `PROVIDED` |\n" +
"| `Developer` | `CONTAINS_ANY_OPTION` | `PROVIDED` |";

private final String TOO_MANY_FILTER_COLS = "| Grid Name | Grid Identifier | Vantage Point Kind | Vantage Point Id |\n" +
"| --- | --- | --- | --- |\n" +
"| Grid Name | {GRIDEXTID} | ORG_UNIT | 1 |\n" +
"| Filter Column | Column Option Codes |\n" +
"| `Grid Name` | `{GRIDEXTID}` | `ORG_UNIT` | `1` |\n" +
"| Filter Column | Filter Operator | Value/s |\n" +
"| --- | --- |\n" +
"| Asset Kind/Application | PROVIDED |\n" +
"| Developer | PROVIDED | TESTING | TESTING |";
"| `Asset Kind/Application` | `CONTAINS_ANY_OPTION` | `PROVIDED` |\n" +
"| `Developer` | `CONTAINS_ANY_OPTION` | `PROVIDED` | `PROVIDED`";

private final String NOT_ENOUGH_FILTER_COLS = "| Grid Name | Grid Identifier | Vantage Point Kind | Vantage Point Id |\n" +
"| --- | --- | --- | --- |\n" +
"| Grid Name | {GRIDEXTID} | ORG_UNIT | 1 |\n" +
"| Filter Column | Column Option Codes |\n" +
"| `Grid Name` | `{GRIDEXTID}` | `ORG_UNIT` | `1` |\n" +
"| Filter Column | Filter Operator | Value/s |\n" +
"| --- | --- |\n" +
"| Asset Kind/Application | PROVIDED |\n" +
"| Developer |";
"|` Asset Kind/Application` | `PROVIDED` |\n" +
"| `Developer` |";

private final String[] TEST_STRING_ARRAY = TEST_STRING.split("\\r?\\n");

Expand All @@ -76,25 +79,25 @@ public void nullStringReturnsNull() {

@Test
public void parseEmptyTextShouldReturnEmptyList() {
List<List<String>> cellList = parseTableData(null, "| Grid Name | Grid Identifier | Vantage Point Kind | Vantage Point Id |");
List<List<String>> cellList = parseTableData(null, HEADER_TEXT);
assertEquals(0, cellList.size());
}

@Test
public void parseNullTextShouldReturnEmptyList() {
List<List<String>> cellList = parseTableData(null, "| Grid Name | Grid Identifier | Vantage Point Kind | Vantage Point Id |");
List<List<String>> cellList = parseTableData(null, HEADER_TEXT);
assertEquals(0, cellList.size());
}

@Test
public void parseNoteTextForHeaderShouldReturnOnlyOneRow() {
List<List<String>> cellList = parseTableData(TEST_STRING_ARRAY, "| Grid Name | Grid Identifier | Vantage Point Kind | Vantage Point Id |");
List<List<String>> cellList = parseTableData(TEST_STRING_ARRAY, HEADER_TEXT);
assertEquals(1, cellList.size());
}

@Test
public void parseNoteTextForFiltersShouldReturnTwoRows() {
List<List<String>> cellList = parseTableData(TEST_STRING_ARRAY, "| Filter Column | Column Option Codes |");
List<List<String>> cellList = parseTableData(TEST_STRING_ARRAY, FILTER_INFO_TEXT);
assertEquals(2, cellList.size());
}

Expand All @@ -107,7 +110,7 @@ public void parseNoteTextShouldHaveFourFieldsInGridInfoRow() {
@Test
public void parseNoteTextShouldHaveTwoFieldsInGridInfoRow() {
Tuple2<List<String>, List<List<String>>> noteText = ReportGridUtilities.parseGridFilterNoteText(TEST_STRING);
assertEquals(2, noteText.v2.get(0).size(), "Should have two fields in each filter info row");
assertEquals(3, noteText.v2.get(0).size(), "Should have two fields in each filter info row");
}

@Test
Expand Down

0 comments on commit 9a6f226

Please sign in to comment.