Elide 5 Agg Store: Multiple time grains (#1806)

* Prototype code to add multiple time grains back * Revised test HJSON files * Fixed a bunch of bugs * Revamped all analytic time types * Month Serde tests pass * Added time serde tests * YearSerde tests pass * QuarterSerde tests pass * Second, Minute, Hour, and Day serdes now pass * Week and ISOWeek serde tests pass * Fixed bug in grain validation logic * Fixed more bugs * Fixed most aggregation store tests * Fixed last bug in Aggregation store * Build complete * Added support for conversions to java.sql.Date * Added one multi-grain time dimension model and test * Added multiple time grain fetch unit test * Added failing filter by alias tests * All tests pass but one * Filter by month alias test working * Added sort by alias test * Added new IT test for parameterized time dimensions * Added graphql test DSL unit test * Added dynamic IT tests with multiple time grains * Added tests for ParameterizedModel * Added test for invalid grain * Added GraphQL entity projection maker unit test * Added unit tests for entity hydration * Inspection rework * More inspection rework Co-authored-by: Aaron Klish <[email protected]>
yahoo · Feb 4, 2021 · ba73bc2 · ba73bc2
1 parent c3a735c
commit ba73bc2
Show file tree

Hide file tree

Showing 79 changed files with 1,790 additions and 828 deletions.
diff --git a/elide-core/src/main/java/com/yahoo/elide/core/dictionary/ArgumentType.java b/elide-core/src/main/java/com/yahoo/elide/core/dictionary/ArgumentType.java
@@ -7,19 +7,24 @@
 
 import com.yahoo.elide.core.type.Type;
 
-import lombok.Getter;
+import lombok.Value;
 
 /**
  * Argument Type wraps an argument to the type of value it accepts.
  */
+@Value
 public class ArgumentType {
-    @Getter
     private String name;
-    @Getter
     private Type<?> type;
+    private Object defaultValue;
 
     public ArgumentType(String name, Type<?> type) {
+        this(name, type, null);
+    }
+
+    public ArgumentType(String name, Type<?> type, Object defaultValue) {
         this.name = name;
         this.type = type;
+        this.defaultValue = defaultValue;
     }
 }
diff --git a/...src/main/java/com/yahoo/elide/core/exceptions/InvalidParameterizedAttributeException.java b/...src/main/java/com/yahoo/elide/core/exceptions/InvalidParameterizedAttributeException.java
@@ -5,6 +5,7 @@
  */
 package com.yahoo.elide.core.exceptions;
 
+import com.yahoo.elide.core.request.Argument;
 import com.yahoo.elide.core.request.Attribute;
 
 /**
@@ -15,4 +16,9 @@ public InvalidParameterizedAttributeException(Attribute attribute) {
         super(HttpStatus.SC_BAD_REQUEST, "No attribute found with matching parameters for attribute: "
                 + attribute.toString());
     }
+
+    public InvalidParameterizedAttributeException(String attributeName, Argument argument) {
+        super(HttpStatus.SC_BAD_REQUEST, String.format("Invalid argument : %s for attribute: %s",
+                argument, attributeName));
+    }
 }
diff --git a/elide-core/src/main/java/com/yahoo/elide/core/filter/predicates/FilterPredicate.java b/elide-core/src/main/java/com/yahoo/elide/core/filter/predicates/FilterPredicate.java
@@ -43,6 +43,7 @@ public class FilterPredicate implements FilterExpression, Function<RequestScope,
     @Getter @NonNull private List<Object> values;
     @Getter @NonNull private String field;
     @Getter @NonNull private String fieldPath;
+    @Getter @NonNull private Type fieldType;
 
     public static boolean toManyInPath(EntityDictionary dictionary, Path path) {
         return path.getPathElements().stream()
@@ -85,6 +86,9 @@ public FilterPredicate(Path path, Operator op, List<Object> values) {
         this.fieldPath = path.getPathElements().stream()
                 .map(PathElement::getFieldName)
                 .collect(Collectors.joining(PERIOD));
+        this.fieldType = path.lastElement()
+                .map(PathElement::getType)
+                .orElse(null);
     }
 
     /**

diff --git a/elide-core/src/main/java/com/yahoo/elide/core/type/ParameterizedModel.java b/elide-core/src/main/java/com/yahoo/elide/core/type/ParameterizedModel.java
@@ -6,19 +6,24 @@
 
 package com.yahoo.elide.core.type;
 
+import com.yahoo.elide.annotation.Exclude;
 import com.yahoo.elide.core.exceptions.InvalidParameterizedAttributeException;
 import com.yahoo.elide.core.request.Argument;
 import com.yahoo.elide.core.request.Attribute;
 
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 
 /**
  * Base class that contains one or more parameterized attributes.
  */
 public abstract class ParameterizedModel {
-    private Map<Attribute, ParameterizedAttribute> parameterizedAttributes;
+
+    @Exclude
+    protected Map<Attribute, ParameterizedAttribute> parameterizedAttributes;
 
     public ParameterizedModel() {
         this(new HashMap<>());
@@ -45,9 +50,37 @@ public <T> T invoke(Set<Argument> arguments) {
      * @return The attribute value.
      */
     public <T> T invoke(Attribute attribute) {
-        if (! parameterizedAttributes.containsKey(attribute)) {
+        Optional<Attribute> match = parameterizedAttributes.keySet().stream()
+
+                //Only filter by alias required.  (Filtering by type may not work with inheritance).
+                .filter((modelAttribute) -> attribute.getAlias().equals(modelAttribute.getAlias()))
+                .findFirst();
+
+        if (! match.isPresent()) {
             throw new InvalidParameterizedAttributeException(attribute);
         }
-        return parameterizedAttributes.get(attribute).invoke(attribute.getArguments());
+
+        return parameterizedAttributes.get(match.get()).invoke(attribute.getArguments());
+    }
+
+    /**
+     * Fetch the attribute value by name.
+     * @param alias The field name to fetch.
+     * @param defaultValue Returned if the field name is not found
+     * @param <T> The return type of the attribute.
+     * @return The attribute value or the provided default value.
+     */
+    public <T> T fetch(String alias, T defaultValue) {
+        Optional<Attribute> match = parameterizedAttributes.keySet().stream()
+
+                //Only filter by alias required.  (Filtering by type may not work with inheritance).
+                .filter((modelAttribute) -> alias.equals(modelAttribute.getAlias()))
+                .findFirst();
+
+        if (! match.isPresent()) {
+            return defaultValue;
+        }
+
+        return parameterizedAttributes.get(match.get()).invoke(new HashSet<>());
     }
 }
diff --git a/elide-core/src/test/java/com/yahoo/elide/core/type/ParameterizedModelTest.java b/elide-core/src/test/java/com/yahoo/elide/core/type/ParameterizedModelTest.java
@@ -0,0 +1,61 @@
+/*
+ * Copyright 2020, Yahoo Inc.
+ * Licensed under the Apache License, Version 2.0
+ * See LICENSE file in project root for terms.
+ */
+
+package com.yahoo.elide.core.type;
+
+import static com.yahoo.elide.core.type.ClassType.STRING_TYPE;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.mockito.Mockito.spy;
+import com.yahoo.elide.core.exceptions.InvalidParameterizedAttributeException;
+import com.yahoo.elide.core.request.Attribute;
+import org.junit.jupiter.api.Test;
+
+public class ParameterizedModelTest {
+
+    @Test
+    public void testInvoke() {
+        ParameterizedModel testModel = spy(ParameterizedModel.class);
+        Attribute testAttribute = Attribute.builder().type(STRING_TYPE).name("foo").build();
+        String testValue = "bar";
+
+        testModel.addAttributeValue(testAttribute, testValue);
+
+        assertEquals(testValue, testModel.invoke(testAttribute));
+    }
+
+    @Test
+    public void testInvokeException() {
+        ParameterizedModel testModel = spy(ParameterizedModel.class);
+        Attribute testAttribute = Attribute.builder().type(STRING_TYPE).name("foo").build();
+
+        Exception exception = assertThrows(InvalidParameterizedAttributeException.class,
+                () -> testModel.invoke(testAttribute));
+
+        assertEquals("No attribute found with matching parameters for attribute: Attribute(name=foo)",
+                exception.getMessage());
+    }
+
+    @Test
+    public void testFetch() {
+        ParameterizedModel testModel = spy(ParameterizedModel.class);
+        Attribute testAttribute = Attribute.builder().type(STRING_TYPE).name("foo").build();
+        String testValue = "bar";
+
+        testModel.addAttributeValue(testAttribute, testValue);
+
+        assertEquals(testValue, testModel.fetch(testAttribute.getAlias(), "blah"));
+    }
+
+    @Test
+    public void testFetchDefault() {
+        ParameterizedModel testModel = spy(ParameterizedModel.class);
+        Attribute testAttribute = Attribute.builder().type(STRING_TYPE).name("foo").build();
+        String testValue = "blah";
+
+        assertEquals(testValue, testModel.fetch(testAttribute.getAlias(), testValue));
+    }
+}
diff --git a/...ggregation/src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStore.java b/...ggregation/src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStore.java
@@ -7,12 +7,16 @@
 
 import com.yahoo.elide.core.datastore.DataStore;
 import com.yahoo.elide.core.datastore.DataStoreTransaction;
+import com.yahoo.elide.core.dictionary.ArgumentType;
 import com.yahoo.elide.core.dictionary.EntityDictionary;
+import com.yahoo.elide.core.type.ClassType;
 import com.yahoo.elide.core.type.Type;
 import com.yahoo.elide.core.utils.ClassScanner;
 import com.yahoo.elide.datastores.aggregation.annotation.Join;
 import com.yahoo.elide.datastores.aggregation.cache.Cache;
 import com.yahoo.elide.datastores.aggregation.core.QueryLogger;
+import com.yahoo.elide.datastores.aggregation.metadata.models.Table;
+import com.yahoo.elide.datastores.aggregation.metadata.models.TimeDimension;
 import com.yahoo.elide.datastores.aggregation.queryengines.sql.annotation.FromSubquery;
 import com.yahoo.elide.datastores.aggregation.queryengines.sql.annotation.FromTable;
 import lombok.Builder;
@@ -57,6 +61,16 @@ public void populateEntityDictionary(EntityDictionary dictionary) {
         ClassScanner.getAnnotatedClasses(AGGREGATION_STORE_CLASSES).forEach(
                 cls -> dictionary.bindEntity(cls, Collections.singleton(Join.class))
         );
+
+        /* Add 'grain' argument to each TimeDimensionColumn */
+        for (Table table : queryEngine.getMetaDataStore().getMetaData(new ClassType<>(Table.class))) {
+            for (TimeDimension timeDim : table.getTimeDimensions()) {
+                dictionary.addArgumentToAttribute(
+                        dictionary.getEntityClass(table.getName(), table.getVersion()),
+                        timeDim.getName(),
+                        new ArgumentType("grain", ClassType.STRING_TYPE, timeDim.getDefaultGrain().getGrain()));
+            }
+        }
     }
 
     @Override

diff --git a/...aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/annotation/Temporal.java b/...aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/annotation/Temporal.java
@@ -28,7 +28,7 @@
      *
      * @return time grain.
      */
-    TimeGrainDefinition grain() default @TimeGrainDefinition(grain = TimeGrain.DAY, expression = "{{}}");
+    TimeGrainDefinition[] grains() default { @TimeGrainDefinition(grain = TimeGrain.DAY, expression = "{{}}") };
 
     /**
      * The timezone in {@link String} of the column.

diff --git a/...regation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/MetaDataStore.java b/...regation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/MetaDataStore.java
@@ -231,7 +231,10 @@ private void addColumn(Column column) {
         addMetaData(column, version);
 
         if (column instanceof TimeDimension) {
-            addTimeDimensionGrain(((TimeDimension) column).getSupportedGrain(), version);
+            TimeDimension timeDimension = (TimeDimension) column;
+            for (TimeDimensionGrain grain : timeDimension.getSupportedGrains()) {
+                addTimeDimensionGrain(grain, version);
+            }
         } else if (column instanceof Metric) {
             addMetricFunction(((Metric) column).getMetricFunction(), version);
         }

diff --git a/...n/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/TimeDimension.java b/...n/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/TimeDimension.java
@@ -8,11 +8,15 @@
 import com.yahoo.elide.annotation.Include;
 import com.yahoo.elide.core.dictionary.EntityDictionary;
 import com.yahoo.elide.datastores.aggregation.annotation.Temporal;
+import com.yahoo.elide.datastores.aggregation.metadata.enums.TimeGrain;
 import lombok.EqualsAndHashCode;
 import lombok.ToString;
 import lombok.Value;
 
+import java.util.Arrays;
+import java.util.LinkedHashSet;
 import java.util.TimeZone;
+import java.util.stream.Collectors;
 import javax.persistence.ManyToMany;
 
 /**
@@ -25,7 +29,7 @@
 public class TimeDimension extends Column {
     @ManyToMany
     @ToString.Exclude
-    TimeDimensionGrain supportedGrain;
+    LinkedHashSet<TimeDimensionGrain> supportedGrains;
 
     TimeZone timezone;
 
@@ -38,6 +42,16 @@ public TimeDimension(Table table, String fieldName, EntityDictionary dictionary)
                 Temporal.class,
                 fieldName);
 
-        this.supportedGrain = new TimeDimensionGrain(getId(), temporal.grain());
+        if (temporal.grains().length == 0) {
+            this.supportedGrains = new LinkedHashSet<>(Arrays.asList(new TimeDimensionGrain(getId(), TimeGrain.DAY)));
+        } else {
+            this.supportedGrains = Arrays.stream(temporal.grains())
+                    .map(grain -> new TimeDimensionGrain(getId(), grain))
+                    .collect(Collectors.toCollection(LinkedHashSet::new));
+        }
+    }
+
+    public TimeDimensionGrain getDefaultGrain() {
+        return supportedGrains.iterator().next();
     }
 }
diff --git a/.../main/java/com/yahoo/elide/datastores/aggregation/metadata/models/TimeDimensionGrain.java b/.../main/java/com/yahoo/elide/datastores/aggregation/metadata/models/TimeDimensionGrain.java
@@ -25,9 +25,20 @@ public class TimeDimensionGrain {
     private final String format;
 
     public TimeDimensionGrain(String fieldName, TimeGrainDefinition definition) {
-        this.id = fieldName + "." + definition.grain().name().toLowerCase(Locale.ENGLISH);
+        this.id = getId(fieldName, definition.grain());
         this.grain = definition.grain();
         this.expression = definition.expression();
         this.format = definition.grain().getFormat();
     }
+
+    public TimeDimensionGrain(String fieldName, TimeGrain grain) {
+        this.id = getId(fieldName, grain);
+        this.grain = grain;
+        this.expression = "{{}}";
+        this.format = grain.getFormat();
+    }
+
+    private static String getId(String fieldName, TimeGrain grain) {
+        return fieldName + "." + grain.name().toLowerCase(Locale.ENGLISH);
+    }
 }
diff --git a/...ore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/query/Queryable.java b/...ore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/query/Queryable.java
@@ -62,12 +62,12 @@ default String getAlias(String columnName) {
 
     /**
      * Retrieves a column by name.
-     * @param name The name of the column.
+     * @param name The alias of the column.
      * @return The column.
      */
     default ColumnProjection getColumnProjection(String name) {
         return getColumnProjections().stream()
-                .filter(dim -> dim.getName().equals(name))
+                .filter(dim -> dim.getAlias().equals(name))
                 .findFirst()
                 .orElse(null);
     }
@@ -79,7 +79,7 @@ default ColumnProjection getColumnProjection(String name) {
      */
     default ColumnProjection getDimensionProjection(String name) {
         return getDimensionProjections().stream()
-                .filter(dim -> dim.getName().equals(name))
+                .filter(dim -> dim.getAlias().equals(name))
                 .findFirst()
                 .orElse(null);
     }
@@ -97,7 +97,7 @@ default ColumnProjection getDimensionProjection(String name) {
      */
     default MetricProjection getMetricProjection(String name) {
         return getMetricProjections().stream()
-                .filter(metric -> metric.getName().equals(name))
+                .filter(metric -> metric.getAlias().equals(name))
                 .findFirst()
                 .orElse(null);
     }
@@ -115,7 +115,7 @@ default MetricProjection getMetricProjection(String name) {
      */
     default TimeDimensionProjection getTimeDimensionProjection(String name) {
         return getTimeDimensionProjections().stream()
-                .filter(dim -> dim.getName().equals(name))
+                .filter(dim -> dim.getAlias().equals(name))
                 .findFirst()
                 .orElse(null);
     }