metadata refactor (#1179)

* metadata refactor

* merge table and analyticView

* fix reflection package

* Make Table constrcut its own columns

* table json type alias

* add comment

elide-core/src/main/java/com/yahoo/elide/utils/ClassScanner.java

@@ -10,7 +10,9 @@
 import io.github.classgraph.ScanResult;
 
 import java.lang.annotation.Annotation;
+import java.util.Arrays;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -50,7 +52,7 @@ static public Set<Class<?>> getAnnotatedClasses(String packageName, Class<? exte
      * @param annotations  One or more annotation to search for
      * @return The classes
      */
-    static public Set<Class<?>> getAnnotatedClasses(Class<? extends Annotation> ...annotations) {
+    static public Set<Class<?>> getAnnotatedClasses(List<Class<? extends Annotation>> annotations) {
         Set<Class<?>> result = new HashSet<>();
         try (ScanResult scanResult = new ClassGraph().enableAllInfo().scan()) {
             for (Class<? extends Annotation> annotation : annotations) {
@@ -62,6 +64,11 @@ static public Set<Class<?>> getAnnotatedClasses(Class<? extends Annotation> ...a
         return result;
     }
 
+    @SafeVarargs
+    static public Set<Class<?>> getAnnotatedClasses(Class<? extends Annotation> ...annotations) {
+        return getAnnotatedClasses(Arrays.asList(annotations));
+    }
+
     /**
      * Returns all classes within a package.
      * @param packageName The root package to search.

elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStore.java

@@ -9,36 +9,30 @@
 import com.yahoo.elide.core.DataStore;
 import com.yahoo.elide.core.DataStoreTransaction;
 import com.yahoo.elide.core.EntityDictionary;
-import com.yahoo.elide.datastores.aggregation.metadata.MetaDataStore;
-import com.yahoo.elide.datastores.aggregation.metadata.models.AnalyticView;
+import com.yahoo.elide.datastores.aggregation.metadata.models.Table;
 import com.yahoo.elide.datastores.aggregation.metadata.models.TimeDimension;
 import com.yahoo.elide.datastores.aggregation.queryengines.sql.annotation.FromSubquery;
 import com.yahoo.elide.datastores.aggregation.queryengines.sql.annotation.FromTable;
 import com.yahoo.elide.utils.ClassScanner;
 
 import java.lang.annotation.Annotation;
+import java.util.Arrays;
+import java.util.List;
 
 /**
  * DataStore that supports Aggregation. Uses {@link QueryEngine} to return results.
  */
 public class AggregationDataStore implements DataStore {
-
-    private final QueryEngineFactory queryEngineFactory;
-
-    private final MetaDataStore metaDataStore;
-
     private QueryEngine queryEngine;
 
     /**
      * These are the classes the Aggregation Store manages.
      */
-    private static final Class[] AGGREGATION_STORE_CLASSES = {
-            FromTable.class, FromSubquery.class };
+    private static final List<Class<? extends Annotation>> AGGREGATION_STORE_CLASSES =
+            Arrays.asList(FromTable.class, FromSubquery.class);
 
-    public AggregationDataStore(QueryEngineFactory queryEngineFactory,
-                                MetaDataStore metaDataStore) {
-        this.queryEngineFactory = queryEngineFactory;
-        this.metaDataStore = metaDataStore;
+    public AggregationDataStore(QueryEngine queryEngine) {
+        this.queryEngine = queryEngine;
     }
 
     /**
@@ -48,17 +42,15 @@ public AggregationDataStore(QueryEngineFactory queryEngineFactory,
     @Override
     public void populateEntityDictionary(EntityDictionary dictionary) {
         for (Class<? extends Annotation> cls : AGGREGATION_STORE_CLASSES) {
-            // bind non-jpa entities, including analyticViews and views
+            // bind non-jpa entity tables
             ClassScanner.getAnnotatedClasses(cls).forEach(dictionary::bindEntity);
         }
 
-        queryEngine = queryEngineFactory.buildQueryEngine(metaDataStore);
-
         /* Add 'grain' argument to each TimeDimensionColumn */
-        for (AnalyticView table : metaDataStore.getMetaData(AnalyticView.class)) {
+        for (Table table : queryEngine.getMetaDataStore().getMetaData(Table.class)) {
             for (TimeDimension timeDim : table.getColumns(TimeDimension.class)) {
                 dictionary.addArgumentToAttribute(
-                        table.getCls(),
+                        dictionary.getEntityClass(table.getName()),
                         timeDim.getName(),
                         new ArgumentType("grain", String.class));
             }

elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStoreTransaction.java

@@ -63,9 +63,11 @@ public void close() throws IOException {
 
     @VisibleForTesting
     private Query buildQuery(EntityProjection entityProjection, RequestScope scope) {
-        Table table = queryEngine.getTable(entityProjection.getType());
-        EntityProjectionTranslator translator = new EntityProjectionTranslator(table,
-                entityProjection, scope.getDictionary());
+        Table table = queryEngine.getTable(scope.getDictionary().getJsonAliasFor(entityProjection.getType()));
+        EntityProjectionTranslator translator = new EntityProjectionTranslator(
+                table,
+                entityProjection,
+                scope.getDictionary());
         return translator.getQuery();
     }
 }

elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/EntityProjectionTranslator.java

@@ -11,7 +11,6 @@
 import com.yahoo.elide.datastores.aggregation.filter.visitor.FilterConstraints;
 import com.yahoo.elide.datastores.aggregation.filter.visitor.SplitFilterExpressionVisitor;
 import com.yahoo.elide.datastores.aggregation.metadata.metric.MetricFunctionInvocation;
-import com.yahoo.elide.datastores.aggregation.metadata.models.AnalyticView;
 import com.yahoo.elide.datastores.aggregation.metadata.models.Dimension;
 import com.yahoo.elide.datastores.aggregation.metadata.models.Table;
 import com.yahoo.elide.datastores.aggregation.metadata.models.TimeDimension;
@@ -37,7 +36,7 @@
  * Helper for Aggregation Data Store which does the work associated with extracting {@link Query}.
  */
 public class EntityProjectionTranslator {
-    private AnalyticView queriedTable;
+    private Table queriedTable;
 
     private EntityProjection entityProjection;
     private Set<ColumnProjection> dimensionProjections;
@@ -48,11 +47,7 @@ public class EntityProjectionTranslator {
     private EntityDictionary dictionary;
 
     public EntityProjectionTranslator(Table table, EntityProjection entityProjection, EntityDictionary dictionary) {
-        if (!(table instanceof AnalyticView)) {
-            throw new InvalidOperationException("Queried table is not analyticView: " + table.getName());
-        }
-
-        this.queriedTable = (AnalyticView) table;
+        this.queriedTable = table;
         this.entityProjection = entityProjection;
         this.dictionary = dictionary;
         dimensionProjections = resolveNonTimeDimensions();
@@ -67,7 +62,7 @@ public EntityProjectionTranslator(Table table, EntityProjection entityProjection
      */
     public Query getQuery() {
         Query query = Query.builder()
-                .analyticView(queriedTable)
+                .table(queriedTable)
                 .metrics(metrics)
                 .groupByDimensions(dimensionProjections)
                 .timeDimensions(timeDimensions)

elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/QueryEngine.java

@@ -7,9 +7,18 @@
 
 import com.yahoo.elide.core.DataStore;
 import com.yahoo.elide.core.DataStoreTransaction;
+import com.yahoo.elide.core.EntityDictionary;
+import com.yahoo.elide.datastores.aggregation.metadata.MetaDataStore;
 import com.yahoo.elide.datastores.aggregation.metadata.models.Table;
 import com.yahoo.elide.datastores.aggregation.query.Query;
 
+import com.google.common.base.Functions;
+
+import lombok.Getter;
+
+import java.util.Map;
+import java.util.stream.Collectors;
+
 /**
  * A {@link QueryEngine} is an abstraction that an AggregationDataStore leverages to run analytic queries (OLAP style)
  * against an underlying persistence layer.
@@ -51,7 +60,49 @@
  * <p>
  * This is a {@link java.util.function functional interface} whose functional method is {@link #executeQuery(Query)}.
  */
-public interface QueryEngine {
+public abstract class QueryEngine {
+    @Getter
+    private final MetaDataStore metaDataStore;
+
+    @Getter
+    private final EntityDictionary metadataDictionary;
+
+    @Getter
+    private final Map<String, Table> tables;
+
+    /**
+     * QueryEngine is constructed with a metadata store and is responsible for constructing all Tables and Entities
+     * metadata in this metadata store.
+     *
+     * @param metaDataStore a metadata store
+     */
+    public QueryEngine(MetaDataStore metaDataStore) {
+        this.metaDataStore = metaDataStore;
+        this.metadataDictionary = metaDataStore.getDictionary();
+        populateMetaData(metaDataStore);
+        this.tables = metaDataStore.getMetaData(Table.class).stream()
+                .collect(Collectors.toMap(Table::getName, Functions.identity()));
+    }
+
+    /**
+     * Construct Table metadata for an entity.
+     *
+     * @param entityClass entity class
+     * @param metaDataDictionary metadata dictionary
+     * @return constructed Table
+     */
+    protected abstract Table constructTable(Class<?> entityClass, EntityDictionary metaDataDictionary);
+
+    /**
+     * Query is responsible for constructing all Tables and Entities metadata in this metadata store.
+     *
+     * @param metaDataStore metadata store to populate
+     */
+    private void populateMetaData(MetaDataStore metaDataStore) {
+        metaDataStore.getModelsToBind().stream()
+                .map(model -> constructTable(model, metadataDictionary))
+                .forEach(metaDataStore::addTable);
+    }
 
     /**
      * Executes the specified {@link Query} against a specific persistent storage, which understand the provided
@@ -61,12 +112,14 @@ public interface QueryEngine {
      *
      * @return query results
      */
-    Iterable<Object> executeQuery(Query query);
+    public abstract Iterable<Object> executeQuery(Query query);
 
     /**
      * Returns the schema for a given entity class.
-     * @param entityClass The class to map to a schema.
+     * @param classAlias json type alias for that class
      * @return The schema that represents the provided entity.
      */
-    Table getTable(Class<?> entityClass);
+    public Table getTable(String classAlias) {
+        return tables.get(classAlias);
+    }
 }

elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/QueryValidator.java

@@ -14,7 +14,7 @@
 import com.yahoo.elide.core.filter.expression.NotFilterExpression;
 import com.yahoo.elide.core.filter.expression.OrFilterExpression;
 import com.yahoo.elide.datastores.aggregation.metadata.metric.MetricFunctionInvocation;
-import com.yahoo.elide.datastores.aggregation.metadata.models.AnalyticView;
+import com.yahoo.elide.datastores.aggregation.metadata.models.Table;
 import com.yahoo.elide.datastores.aggregation.query.ColumnProjection;
 import com.yahoo.elide.datastores.aggregation.query.Query;
 import com.yahoo.elide.request.Sorting;
@@ -31,15 +31,15 @@ public class QueryValidator {
     private Query query;
     private Set<String> allFields;
     private EntityDictionary dictionary;
-    private AnalyticView queriedTable;
+    private Table queriedTable;
     private List<MetricFunctionInvocation> metrics;
     private Set<ColumnProjection> dimensionProjections;
 
     public QueryValidator(Query query, Set<String> allFields, EntityDictionary dictionary) {
         this.query = query;
         this.allFields = allFields;
         this.dictionary = dictionary;
-        this.queriedTable = query.getAnalyticView();
+        this.queriedTable = query.getTable();
         this.metrics = query.getMetrics();
         this.dimensionProjections = query.getDimensions();
     }
@@ -71,12 +71,14 @@ private void validateHavingClause(FilterExpression havingClause) {
             Class<?> cls = last.getType();
             String fieldName = last.getFieldName();
 
-            if (cls != queriedTable.getCls()) {
+            Class<?> tableClass = dictionary.getEntityClass(queriedTable.getName());
+
+            if (cls != tableClass) {
                 throw new InvalidOperationException(
                         String.format(
                                 "Can't filter on relationship field %s in HAVING clause when querying table %s.",
                                 path.toString(),
-                                queriedTable.getCls().getSimpleName()));
+                                tableClass.getSimpleName()));
             }
 
             if (queriedTable.isMetric(fieldName)) {

elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/MetaDataStore.java

@@ -10,7 +10,6 @@
 import com.yahoo.elide.core.exceptions.DuplicateMappingException;
 import com.yahoo.elide.datastores.aggregation.AggregationDataStore;
 import com.yahoo.elide.datastores.aggregation.annotation.MetricAggregation;
-import com.yahoo.elide.datastores.aggregation.metadata.models.AnalyticView;
 import com.yahoo.elide.datastores.aggregation.metadata.models.Column;
 import com.yahoo.elide.datastores.aggregation.metadata.models.DataType;
 import com.yahoo.elide.datastores.aggregation.metadata.models.FunctionArgument;
@@ -25,54 +24,51 @@
 
 import org.hibernate.annotations.Subselect;
 
+import lombok.Getter;
+
+import java.lang.annotation.Annotation;
+import java.util.Arrays;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Set;
 import java.util.stream.Collectors;
 
 /**
  * MetaDataStore is a in-memory data store that manage data models for an {@link AggregationDataStore}.
  */
 public class MetaDataStore extends HashMapDataStore {
-    public static final Package META_DATA_PACKAGE = Table.class.getPackage();
+    private static final Package META_DATA_PACKAGE = Table.class.getPackage();
+
+    private static final List<Class<? extends Annotation>> METADATA_STORE_ANNOTATIONS =
+            Arrays.asList(FromTable.class, FromSubquery.class, Subselect.class, javax.persistence.Table.class);
 
-    private static final Class[] METADATA_STORE_ANNOTATIONS = {
-            FromTable.class, FromSubquery.class, Subselect.class, javax.persistence.Table.class};
+    @Getter
+    private final Set<Class<?>> modelsToBind;
 
     public MetaDataStore() {
         super(META_DATA_PACKAGE);
 
         this.dictionary = new EntityDictionary(new HashMap<>());
 
-        ClassScanner.getAllClasses(Table.class.getPackage().getName()).forEach(cls -> dictionary.bindEntity(cls));
-
-        Set<Class<?>> modelsToBind = ClassScanner.getAnnotatedClasses(METADATA_STORE_ANNOTATIONS);
+        // bind meta data models to dictionary
+        ClassScanner.getAllClasses(Table.class.getPackage().getName()).forEach(dictionary::bindEntity);
 
-        // bind data models in the package
-        modelsToBind.forEach(modelClass -> {
-                dictionary.bindEntity(modelClass);
-        });
-
-        // resolve meta data from the bound models
-        modelsToBind.forEach(modelClass -> {
-            addTable(isAnalyticView(modelClass)
-                ? new AnalyticView(modelClass, dictionary)
-                : new Table(modelClass, dictionary));
-        });
+        // bind external data models in the package
+        this.modelsToBind = ClassScanner.getAnnotatedClasses(METADATA_STORE_ANNOTATIONS);
+        modelsToBind.forEach(dictionary::bindEntity);
     }
 
     @Override
     public void populateEntityDictionary(EntityDictionary dictionary) {
-        ClassScanner.getAllClasses(META_DATA_PACKAGE.getName()).stream().forEach(cls -> {
-            dictionary.bindEntity(cls);
-        });
+        ClassScanner.getAllClasses(META_DATA_PACKAGE.getName()).forEach(dictionary::bindEntity);
     }
 
     /**
      * Add a table metadata object.
      *
      * @param table table metadata
      */
-    private void addTable(Table table) {
+    public void addTable(Table table) {
         addMetaData(table);
         table.getColumns().forEach(this::addColumn);
     }
@@ -169,14 +165,4 @@ public <T> Set<T> getMetaData(Class<T> cls) {
     public static boolean isMetricField(EntityDictionary dictionary, Class<?> cls, String fieldName) {
         return dictionary.attributeOrRelationAnnotationExists(cls, fieldName, MetricAggregation.class);
     }
-
-    /**
-     * Returns whether an entity class is analytic view.
-     *
-     * @param cls entity class
-     * @return True if {@link FromTable} or {@link FromSubquery} is presented.
-     */
-    private static boolean isAnalyticView(Class<?> cls) {
-        return cls.isAnnotationPresent(FromTable.class) || cls.isAnnotationPresent(FromSubquery.class);
-    }
 }