Skip to content

Commit

Permalink
AggregationStore: Templated filter column arguments (#2297)
Browse files Browse the repository at this point in the history 
* Added filter template support for columns

* Build passes

* Added more tests

* Added column argument validator unit tests

* Build passed

Co-authored-by: Aaron Klish <[email protected]>
  • Loading branch information
@aklish
aklish and Aaron Klish authored Sep 13, 2021
1 parent 65eaaa1 commit cc0dffc
Show file tree
Hide file tree
Showing 21 changed files with 399 additions and 55 deletions.
90 changes: 71 additions & 19 deletions ...src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStoreTransaction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@
import com.yahoo.elide.datastores.aggregation.core.QueryResponse;
import com.yahoo.elide.datastores.aggregation.filter.visitor.MatchesTemplateVisitor;
import com.yahoo.elide.datastores.aggregation.metadata.MetaDataStore;
import com.yahoo.elide.datastores.aggregation.metadata.models.Column;
import com.yahoo.elide.datastores.aggregation.metadata.models.RequiresFilter;
import com.yahoo.elide.datastores.aggregation.metadata.models.Table;
import com.yahoo.elide.datastores.aggregation.query.Query;
import com.yahoo.elide.datastores.aggregation.query.QueryResult;
Expand All @@ -33,7 +35,6 @@
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import java.util.stream.Stream;

/**
* Transaction handler for {@link AggregationDataStore}.
Expand Down Expand Up @@ -154,33 +155,84 @@ Query buildQuery(EntityProjection entityProjection, RequestScope scope) {

Query query = translator.getQuery();

FilterExpression filterTemplate = table.getRequiredFilter(scope.getDictionary());
if (filterTemplate != null) {
Query modifiedQuery = addTableFilterArguments(table, query, scope.getDictionary());
modifiedQuery = addColumnFilterArguments(table, modifiedQuery, scope.getDictionary());

Map<String, Argument> templateFilterArguments = new HashMap<>();
if (!MatchesTemplateVisitor.isValid(filterTemplate, query.getWhereFilter(), templateFilterArguments)
&& (!MatchesTemplateVisitor.isValid(filterTemplate, query.getHavingFilter(),
templateFilterArguments))) {
String message = String.format("Querying %s requires a mandatory filter: %s",
table.getName(), table.getRequiredFilter());
return modifiedQuery;
}

throw new BadRequestException(message);
}
@VisibleForTesting
Query addTableFilterArguments(Table table, Query query, EntityDictionary dictionary) {
FilterExpression filterTemplate = table.getRequiredFilter(dictionary);

Query modifiedQuery = query;
if (filterTemplate != null) {
Map<String, Argument> allArguments = validateRequiredFilter(filterTemplate, query, table);

if (! templateFilterArguments.isEmpty()) {
Map<String, Argument> tableArguments = Stream.concat(
templateFilterArguments.entrySet().stream(),
query.getArguments().entrySet().stream()
).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
if (!allArguments.isEmpty()) {
if (query.getArguments() != null) {
allArguments.putAll(query.getArguments());
}

return Query.builder()
modifiedQuery = Query.builder()
.query(query)
.arguments(tableArguments)
.arguments(allArguments)
.build();
}
}

return query;
return modifiedQuery;
}

@VisibleForTesting
Query addColumnFilterArguments(Table table, Query query, EntityDictionary dictionary) {

Query.QueryBuilder queryBuilder = Query.builder();

query.getColumnProjections().stream().forEach(projection -> {
Column column = table.getColumn(Column.class, projection.getName());

FilterExpression requiredFilter = column.getRequiredFilter(dictionary);

if (requiredFilter != null) {
Map<String, Argument> allArguments = validateRequiredFilter(requiredFilter, query, column);
if (projection.getArguments() != null) {
allArguments.putAll(projection.getArguments());
}
queryBuilder.column(projection.withArguments(allArguments));
} else {
queryBuilder.column(projection);
}
});

return queryBuilder
.arguments(query.getArguments())
.havingFilter(query.getHavingFilter())
.whereFilter(query.getWhereFilter())
.sorting(query.getSorting())
.pagination(query.getPagination())
.bypassingCache(query.isBypassingCache())
.source(query.getSource())
.scope(query.getScope())
.build();
}

private Map<String, Argument> validateRequiredFilter(
FilterExpression filterTemplate,
Query query,
RequiresFilter requiresFilter
) {
Map<String, Argument> templateFilterArguments = new HashMap<>();
if (!MatchesTemplateVisitor.isValid(filterTemplate, query.getWhereFilter(), templateFilterArguments)
&& (!MatchesTemplateVisitor.isValid(filterTemplate, query.getHavingFilter(),
templateFilterArguments))) {
String message = String.format("Querying %s requires a mandatory filter: %s",
requiresFilter.getName(), requiresFilter.getRequiredFilter());

throw new BadRequestException(message);
}

return templateFilterArguments;
}

@Override
Expand Down
6 changes: 6 additions & 0 deletions ...gregation/src/main/java/com/yahoo/elide/datastores/aggregation/annotation/ColumnMeta.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,12 @@
String [] tags() default {};
String [] values() default {};

/**
* Whether or not querying this column requires a client provided filter.
* @return The required filter template.
*/
String filterTemplate() default "";

/**
* Indicates the cardinality for the column.
* @return size
Expand Down
15 changes: 15 additions & 0 deletions ...e-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/dynamic/TableType.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -469,6 +469,11 @@ public String[] values() {
return new String[0];
}

@Override
public String filterTemplate() {
return measure.getFilterTemplate();
}

@Override
public CardinalitySize size() {
return CardinalitySize.UNKNOWN;
Expand Down Expand Up @@ -605,6 +610,11 @@ public String[] values() {
return dimension.getValues().toArray(new String[0]);
}

@Override
public String filterTemplate() {
return dimension.getFilterTemplate();
}

@Override
public CardinalitySize size() {
if (dimension.getCardinality() == null || dimension.getCardinality().isEmpty()) {
Expand Down Expand Up @@ -780,6 +790,11 @@ public String[] values() {
return new String[0];
}

@Override
public String filterTemplate() {
return "";
}

@Override
public CardinalitySize size() {
return CardinalitySize.UNKNOWN;
Expand Down
6 changes: 5 additions & 1 deletion ...regation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/Column.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@
@Getter
@EqualsAndHashCode
@ToString
public abstract class Column implements Versioned, Named {
public abstract class Column implements Versioned, Named, RequiresFilter {

@Id
private final String id;
Expand Down Expand Up @@ -73,6 +73,8 @@ public abstract class Column implements Versioned, Named {

private final Set<String> values;

private String requiredFilter;

@OneToOne
@Setter
private TableSource tableSource = null;
Expand Down Expand Up @@ -111,6 +113,7 @@ protected Column(Table table, String fieldName, EntityDictionary dictionary) {
this.tableSourceDefinition = meta.tableSource();
this.valueSourceType = ValueSourceType.getValueSourceType(this.values, this.tableSourceDefinition);
this.cardinality = meta.size();
this.requiredFilter = meta.filterTemplate();
} else {
this.friendlyName = name;
this.description = null;
Expand All @@ -120,6 +123,7 @@ protected Column(Table table, String fieldName, EntityDictionary dictionary) {
this.tableSourceDefinition = null;
this.valueSourceType = ValueSourceType.NONE;
this.cardinality = CardinalitySize.UNKNOWN;
this.requiredFilter = null;
}

if (dictionary.attributeOrRelationAnnotationExists(tableClass, fieldName, MetricFormula.class)) {
Expand Down
37 changes: 37 additions & 0 deletions .../src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/RequiresFilter.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
/*
* Copyright 2021, Yahoo Inc.
* Licensed under the Apache License, Version 2.0
* See LICENSE file in project root for terms.
*/
package com.yahoo.elide.datastores.aggregation.metadata.models;

import com.yahoo.elide.core.dictionary.EntityDictionary;
import com.yahoo.elide.core.filter.dialect.ParseException;
import com.yahoo.elide.core.filter.dialect.RSQLFilterDialect;
import com.yahoo.elide.core.filter.expression.FilterExpression;
import com.yahoo.elide.core.type.Type;
import com.yahoo.elide.modelconfig.model.Named;
import org.apache.commons.lang3.StringUtils;

/**
* Metadata models that require a client filter.
*/
public interface RequiresFilter extends Named {
Table getTable();

String getRequiredFilter();

default FilterExpression getRequiredFilter(EntityDictionary dictionary) {
Type<?> cls = dictionary.getEntityClass(getTable().getName(), getTable().getVersion());
RSQLFilterDialect filterDialect = new RSQLFilterDialect(dictionary);

if (StringUtils.isNotEmpty(getRequiredFilter())) {
try {
return filterDialect.parseFilterExpression(getRequiredFilter(), cls, false, true);
} catch (ParseException e) {
throw new IllegalStateException(e);
}
}
return null;
}
}
21 changes: 4 additions & 17 deletions ...gregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/Table.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,6 @@
import com.yahoo.elide.annotation.Exclude;
import com.yahoo.elide.annotation.Include;
import com.yahoo.elide.core.dictionary.EntityDictionary;
import com.yahoo.elide.core.filter.dialect.ParseException;
import com.yahoo.elide.core.filter.dialect.RSQLFilterDialect;
import com.yahoo.elide.core.filter.expression.FilterExpression;
import com.yahoo.elide.core.type.Type;
import com.yahoo.elide.core.utils.TypeHelper;
import com.yahoo.elide.datastores.aggregation.AggregationDataStore;
Expand All @@ -25,7 +22,6 @@
import com.yahoo.elide.datastores.aggregation.queryengines.sql.annotation.FromSubquery;
import com.yahoo.elide.datastores.aggregation.queryengines.sql.annotation.FromTable;
import com.yahoo.elide.modelconfig.model.Named;
import org.apache.commons.lang3.StringUtils;
import lombok.AccessLevel;
import lombok.EqualsAndHashCode;
import lombok.Getter;
Expand All @@ -49,7 +45,7 @@
@Getter
@EqualsAndHashCode
@ToString
public abstract class Table implements Versioned, Named {
public abstract class Table implements Versioned, Named, RequiresFilter {

@Id
private final String id;
Expand Down Expand Up @@ -321,18 +317,9 @@ public boolean isMetric(String fieldName) {
return getMetric(fieldName) != null;
}

public FilterExpression getRequiredFilter(EntityDictionary dictionary) {
Type<?> cls = dictionary.getEntityClass(name, version);
RSQLFilterDialect filterDialect = new RSQLFilterDialect(dictionary);

if (StringUtils.isNotEmpty(requiredFilter)) {
try {
return filterDialect.parseFilterExpression(requiredFilter, cls, false, true);
} catch (ParseException e) {
throw new IllegalStateException(e);
}
}
return null;
@Override
public Table getTable() {
return this;
}

public boolean hasArgumentDefinition(String argName) {
Expand Down
6 changes: 5 additions & 1 deletion ...c/main/java/com/yahoo/elide/datastores/aggregation/validator/ColumnArgumentValidator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ public void validate() {
.flatMap(Set::stream)
.map(ColumnArgReference::getArgName)
.forEach(argName -> {
if (!column.hasArgumentDefinition(argName)) {
if (!column.hasArgumentDefinition(argName) && !hasTemplateFilterArgument(argName)) {
throw new IllegalStateException(String.format(errorMsgPrefix
+ "Argument '%s' is not defined but found '{{$$column.args.%s}}'.",
argName, argName));
Expand Down Expand Up @@ -141,4 +141,8 @@ private void verifyPinnedArguments(Map<String, Argument> mergedArguments, Column
}
});
}

private boolean hasTemplateFilterArgument(String argName) {
return column.getRequiredFilter() != null && column.getRequiredFilter().contains("{{" + argName + "}}");
}
}
Loading

0 comments on commit cc0dffc

Please sign in to comment.