SQL Query Engine (#878)

* AggregationDataStore: Schema (#846)

* AggregationDataStore: Static Attribute Aggregation

* Address comments

* Implement TimeDimension and all its supporting components

* refactor

* Address comments from @aklish

* Address comments from @aklish && Tests & Javadoc

* Address comments from @aklish

* Address comments from @aklish and @hellohanchen

* Address comments from Aaron

* ToMany is not supported

* Address comments from Aaron

Added calcite as a dependency.  Merged in changes for QueryEngine interface

Fixed checkstyle issues

Added basic H2 DB test harness

Started breaking out projections

Moved getValue and setValue from PersistentResource to EntityDictionary

Added basic logic to hydrate entities

Added FromTable and FromSubquery annotations.  Add explicit exclusion of entity relationship hydration

Minor cleanup

Refactored HQLFilterOperation to take an alias generator

Added test support for RSQL filter generation. Some cleanup

Added basic support for WHERE clause filtering on the fact table

Added working test for subquery SQL

Added basic join logic for filters

Added a test with a subquery and a filter join

Refactored Schema classes and Query to support metric aggregation SQL expansion

Added group by support

Added logic for ID generation

Added sorting logic and test

Added pagination support and testing

All column references use proper name now for SQL

Removed calcite as a query engine

Refactored HQLFilterOperation so it can be used for Having and Where clause generaiton

Added HAVING clause support

Changed Query to take schema instead of entityClass

First pass at cleanup

Fixed checkstyles

Cleanup

Cleanup

Added a complex SQL expression test and fixed bugs

Fixed merge issues.  Added another test.  Added better logging

Fixed bug in pagination SQL generation

* Build is working

* Inspection rework

elide-core/src/main/java/com/yahoo/elide/core/EntityDictionary.java

@@ -1360,18 +1360,6 @@ private Map coerceMap(Object target, Map<?, ?> values, String fieldName) {
         return result;
     }
 
-    private boolean isValidParameterizedMap(Map<?, ?> values, Class<?> keyType, Class<?> valueType) {
-        for (Map.Entry<?, ?> entry : values.entrySet()) {
-            Object key = entry.getKey();
-            Object value = entry.getValue();
-            if ((key != null && !keyType.isAssignableFrom(key.getClass()))
-                    || (value != null && !valueType.isAssignableFrom(value.getClass()))) {
-                return false;
-            }
-        }
-        return true;
-    }
-
     /**
      * Returns whether or not a specified annotation is present on an entity field or its corresponding method.
      *
@@ -1402,6 +1390,18 @@ public boolean isValidField(Class<?> cls, String fieldName) {
         return getAllFields(cls).contains(fieldName);
     }
 
+    private boolean isValidParameterizedMap(Map<?, ?> values, Class<?> keyType, Class<?> valueType) {
+        for (Map.Entry<?, ?> entry : values.entrySet()) {
+            Object key = entry.getKey();
+            Object value = entry.getValue();
+            if ((key != null && !keyType.isAssignableFrom(key.getClass()))
+                    || (value != null && !valueType.isAssignableFrom(value.getClass()))) {
+                return false;
+            }
+        }
+        return true;
+    }
+
     /**
      * Binds the entity class if not yet bound.
      * @param entityClass the class to bind.

elide-core/src/main/java/com/yahoo/elide/core/PersistentResource.java

@@ -1416,7 +1416,9 @@ protected void delFromCollection(
      */
     protected void setValue(String fieldName, Object value) {
         final Object original = getValueUnchecked(fieldName);
+
         dictionary.setValue(obj, fieldName, value);
+
         triggerUpdate(fieldName, original, value);
     }
 
@@ -1428,8 +1430,7 @@ protected void setValue(String fieldName, Object value) {
      * @return the value
      */
     public static Object getValue(Object target, String fieldName, RequestScope requestScope) {
-        EntityDictionary dictionary = requestScope.getDictionary();
-        return dictionary.getValue(target, fieldName, requestScope);
+        return requestScope.getDictionary().getValue(target, fieldName, requestScope);
     }
 
     /**

elide-core/src/main/java/com/yahoo/elide/core/TimedFunction.java

@@ -0,0 +1,40 @@
+/*
+ * Copyright 2019, Yahoo Inc.
+ * Licensed under the Apache License, Version 2.0
+ * See LICENSE file in project root for terms.
+ */
+
+package com.yahoo.elide.core;
+
+import lombok.Data;
+import lombok.extern.slf4j.Slf4j;
+
+import java.util.function.Supplier;
+
+/**
+ * Wraps a function and logs how long it took to run (in millis).
+ * @param <R> The function return type.
+ */
+@Slf4j
+@Data
+public class TimedFunction<R> implements Supplier<R> {
+
+    public TimedFunction(Supplier<R> toRun, String logMessage) {
+        this.toRun = toRun;
+        this.logMessage = logMessage;
+    }
+
+    private Supplier<R> toRun;
+    private String logMessage;
+
+    @Override
+    public R get() {
+        long start = System.currentTimeMillis();
+        R ret = toRun.get();
+        long end = System.currentTimeMillis();
+
+        log.debug(logMessage + "\tTime spent: {}", end - start);
+
+        return ret;
+    }
+}

elide-core/src/main/java/com/yahoo/elide/core/filter/dialect/RSQLFilterDialect.java

@@ -161,7 +161,6 @@ public Map<String, FilterExpression> parseTypedExpression(String path, Multivalu
         return expressionByType;
     }
 
-
     /**
      * Parses a RSQL string into an Elide FilterExpression.
      * @param expressionText the RSQL string

elide-core/src/main/java/com/yahoo/elide/core/pagination/Pagination.java

@@ -99,8 +99,8 @@ private Pagination(Map<PaginationKey, Integer> pageData, int defaultMaxPageSize,
     public static Pagination fromOffsetAndLimit(int limit, int offset, boolean generatePageTotals) {
 
         ImmutableMap.Builder<PaginationKey, Integer> pageData = ImmutableMap.<PaginationKey, Integer>builder()
-                    .put(PAGE_KEYS.get(PAGE_OFFSET_KEY), offset)
-                    .put(PAGE_KEYS.get(PAGE_LIMIT_KEY), limit);
+                .put(PAGE_KEYS.get(PAGE_OFFSET_KEY), offset)
+                .put(PAGE_KEYS.get(PAGE_LIMIT_KEY), limit);
 
         if (generatePageTotals) {
             pageData.put(PAGE_KEYS.get(PAGE_TOTALS_KEY), 1);

elide-datastore/elide-datastore-aggregation/pom.xml

@@ -42,6 +42,18 @@
         <dependency>
             <groupId>com.yahoo.elide</groupId>
             <artifactId>elide-core</artifactId>
+            <version>4.5.2-SNAPSHOT</version>
+        </dependency>
+
+        <dependency>
+            <groupId>com.yahoo.elide</groupId>
+            <artifactId>elide-datastore-hibernate</artifactId>
+            <version>4.5.2-SNAPSHOT</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.projectlombok</groupId>
+            <artifactId>lombok</artifactId>
         </dependency>
 
         <!-- JPA -->
@@ -51,12 +63,36 @@
             <version>1.0.0.Final</version>
         </dependency>
 
+        <!-- Hibernate Entity Manager -->
+        <dependency>
+            <groupId>org.hibernate</groupId>
+            <artifactId>hibernate-entitymanager</artifactId>
+            <version>${hibernate5.version}</version>
+        </dependency>
+
         <!-- Test -->
         <dependency>
             <groupId>org.testng</groupId>
             <artifactId>testng</artifactId>
             <scope>test</scope>
         </dependency>
+
+        <!-- H2 -->
+        <dependency>
+            <groupId>com.h2database</groupId>
+            <artifactId>h2</artifactId>
+            <version>1.4.197</version>
+            <scope>test</scope>
+        </dependency>
+
+        <!-- Mockito -->
+        <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-core</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+
     </dependencies>
 
     <build>

elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/Column.java

@@ -9,6 +9,7 @@
 import com.yahoo.elide.datastores.aggregation.dimension.Dimension;
 import com.yahoo.elide.datastores.aggregation.metric.Metric;
 
+import com.yahoo.elide.datastores.aggregation.schema.Schema;
 import lombok.Getter;
 
 import java.util.Objects;
@@ -26,21 +27,25 @@ public abstract class Column {
     protected final String description;
     @Getter
     protected final Class<?> dataType;
+    @Getter
+    protected final Schema schema;
 
     /**
      * Constructor.
      *
+     * @param schema The schema this {@link Column} belongs to.
      * @param field  The entity field or relation that this {@link Column} represents
      * @param annotation  Provides static meta data about this {@link Column}
      * @param fieldType  The Java type for this entity field or relation
      *
      * @throws NullPointerException if {@code field} or {@code fieldType} is {@code null}
      */
-    public Column(String field, Meta annotation, Class<?> fieldType) {
+    public Column(Schema schema, String field, Meta annotation, Class<?> fieldType) {
         this.name = Objects.requireNonNull(field, "field");
         this.longName = annotation == null || annotation.longName().isEmpty() ? field : annotation.longName();
         this.description = annotation == null || annotation.description().isEmpty() ? field : annotation.description();
         this.dataType = Objects.requireNonNull(fieldType, "fieldType");
+        this.schema = schema;
     }
 
     @Override

elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/Query.java

@@ -11,28 +11,51 @@
 import com.yahoo.elide.core.sort.Sorting;
 import com.yahoo.elide.datastores.aggregation.dimension.Dimension;
 import com.yahoo.elide.datastores.aggregation.dimension.TimeDimension;
+import com.yahoo.elide.datastores.aggregation.metric.Aggregation;
 import com.yahoo.elide.datastores.aggregation.metric.Metric;
 
+import com.yahoo.elide.datastores.aggregation.schema.Schema;
 import lombok.Builder;
 import lombok.Data;
+import lombok.Singular;
 
-import java.util.Optional;
+import java.util.LinkedHashSet;
+import java.util.Map;
 import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 /**
  * A {@link Query} is an object representing a query executed by {@link QueryEngine}.
  */
 @Data
 @Builder
 public class Query {
+    private final Schema schema;
 
-    private final Class<?> entityClass;
-    private final Set<Metric> metrics;
+    @Singular
+    private final Map<Metric, Class<? extends Aggregation>> metrics;
+
+    @Singular
     private final Set<Dimension> groupDimensions;
+
+    @Singular
     private final Set<TimeDimension> timeDimensions;
-    private final Optional<FilterExpression> whereFilter;
-    private final Optional<FilterExpression> havingFilter;
-    private final Optional<Sorting> sorting;
-    private final Optional<Pagination> pagination;
+
+    private final FilterExpression whereFilter;
+    private final FilterExpression havingFilter;
+    private final Sorting sorting;
+    private final Pagination pagination;
     private final RequestScope scope;
+
+    /**
+     * Returns all the dimensions regardless of type.
+     * @return All the dimensions.
+     */
+    public Set<Dimension> getDimensions() {
+        return Stream.concat(getGroupDimensions().stream(), getTimeDimensions().stream())
+                .collect(
+                        Collectors.toCollection(LinkedHashSet::new)
+                );
+    }
 }

elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/annotation/Temporal.java

@@ -24,7 +24,7 @@
 public @interface Temporal {
 
     /**
-     * The finest unit into which temporal column can be divided
+     * The finest unit into which temporal column can be divided.
      *
      * @return The finest supported time grain of a persistent storage column
      */

elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/dimension/DegenerateDimension.java

@@ -9,6 +9,7 @@
 import com.yahoo.elide.datastores.aggregation.annotation.CardinalitySize;
 import com.yahoo.elide.datastores.aggregation.annotation.Meta;
 
+import com.yahoo.elide.datastores.aggregation.schema.Schema;
 import lombok.Getter;
 
 import java.util.Objects;
@@ -46,6 +47,7 @@ public static ColumnType parseColumnType(String dimensionField, Class<?> cls, En
     /**
      * Constructor.
      *
+     * @param schema The schema this {@link Dimension} belongs to.
      * @param dimensionField  The entity field or relation that this {@link Dimension} represents
      * @param annotation  Provides static meta data about this {@link Dimension}
      * @param fieldType  The Java type for this entity field or relation
@@ -56,6 +58,7 @@ public static ColumnType parseColumnType(String dimensionField, Class<?> cls, En
      * @throws NullPointerException any argument, except for {@code annotation}, is {@code null}
      */
     public DegenerateDimension(
+            Schema schema,
             String dimensionField,
             Meta annotation,
             Class<?> fieldType,
@@ -64,6 +67,7 @@ public DegenerateDimension(
             ColumnType columnType
     ) {
         super(
+                schema,
                 dimensionField,
                 annotation,
                 fieldType,

elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/dimension/EntityDimension.java

@@ -12,6 +12,7 @@
 import com.yahoo.elide.datastores.aggregation.annotation.FriendlyName;
 import com.yahoo.elide.datastores.aggregation.annotation.Meta;
 
+import com.yahoo.elide.datastores.aggregation.schema.Schema;
 import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
 
@@ -116,6 +117,7 @@ protected static CardinalitySize getDefaultCardinality() {
     /**
      * Constructor.
      *
+     * @param schema The schema this {@link Dimension} belongs to.
      * @param dimensionField  The entity field or relation that this {@link Dimension} represents
      * @param annotation  Provides static meta data about this {@link Dimension}
      * @param fieldType  The Java type for this entity field or relation
@@ -125,13 +127,14 @@ protected static CardinalitySize getDefaultCardinality() {
      * @throws NullPointerException any argument, except for {@code annotation}, is {@code null}
      */
     public EntityDimension(
+            Schema schema,
             String dimensionField,
             Meta annotation,
             Class<?> fieldType,
             CardinalitySize cardinality,
             String friendlyName
     ) {
-        this(dimensionField, annotation, fieldType, DimensionType.ENTITY, cardinality, friendlyName);
+        this(schema, dimensionField, annotation, fieldType, DimensionType.ENTITY, cardinality, friendlyName);
     }
 
     /**
@@ -147,14 +150,15 @@ public EntityDimension(
      * @throws NullPointerException any argument, except for {@code annotation}, is {@code null}
      */
     protected EntityDimension(
+            Schema schema,
             String dimensionField,
             Meta annotation,
             Class<?> fieldType,
             DimensionType dimensionType,
             CardinalitySize cardinality,
             String friendlyName
     ) {
-        super(dimensionField, annotation, fieldType);
+        super(schema, dimensionField, annotation, fieldType);
 
         this.dimensionType = Objects.requireNonNull(dimensionType, "dimensionType");
         this.cardinality = Objects.requireNonNull(cardinality, "cardinality");

elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/dimension/TimeDimension.java

@@ -7,6 +7,7 @@
 
 import com.yahoo.elide.datastores.aggregation.annotation.CardinalitySize;
 import com.yahoo.elide.datastores.aggregation.annotation.Meta;
+import com.yahoo.elide.datastores.aggregation.schema.Schema;
 import com.yahoo.elide.datastores.aggregation.time.TimeGrain;
 
 import lombok.Getter;
@@ -32,6 +33,7 @@ public class TimeDimension extends DegenerateDimension {
     /**
      * Constructor.
      *
+     * @param schema The schema this dimension belongs to
      * @param dimensionField  The entity field or relation that this {@link Dimension} represents
      * @param annotation  Provides static meta data about this {@link Dimension}
      * @param fieldType  The Java type for this entity field or relation
@@ -43,6 +45,7 @@ public class TimeDimension extends DegenerateDimension {
      * @throws NullPointerException any argument, except for {@code annotation}, is {@code null}
      */
     public TimeDimension(
+            Schema schema,
             String dimensionField,
             Meta annotation,
             Class<?> fieldType,
@@ -51,7 +54,7 @@ public TimeDimension(
             TimeZone timeZone,
             TimeGrain timeGrain
     ) {
-        super(dimensionField, annotation, fieldType, cardinality, friendlyName, ColumnType.TEMPORAL);
+        super(schema, dimensionField, annotation, fieldType, cardinality, friendlyName, ColumnType.TEMPORAL);
         this.timeZone = Objects.requireNonNull(timeZone, "timeZone");
         this.timeGrain = Objects.requireNonNull(timeGrain, "timeGrain");
     }