-
Notifications
You must be signed in to change notification settings - Fork 228
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* AggregationDataStore: Schema (#846) * AggregationDataStore: Static Attribute Aggregation * Address comments * Implement TimeDimension and all its supporting components * refactor * Address comments from @aklish * Address comments from @aklish && Tests & Javadoc * Address comments from @aklish * Address comments from @aklish and @hellohanchen * Address comments from Aaron * ToMany is not supported * Address comments from Aaron * Added basic H2 DB test harness * Started breaking out projections * Moved getValue and setValue from PersistentResource to EntityDictionary * Added basic logic to hydrate entities * Added FromTable and FromSubquery annotations. Add explicit exclusion of entity relationship hydration * Refactored HQLFilterOperation to take an alias generator * Added test support for RSQL filter generation. Some cleanup * Added basic support for WHERE clause filtering on the fact table * Added working test for subquery SQL * Added basic join logic for filters * Added a test with a subquery and a filter join * Refactored Schema classes and Query to support metric aggregation SQL expansion * Added group by support * Added logic for ID generation * Added sorting logic and test * Added pagination support and testing * All column references use proper name now for SQL * Removed calcite as a query engine * Refactored HQLFilterOperation so it can be used for Having and Where clause generaiton * Added HAVING clause support * Changed Query to take schema instead of entityClass * First pass at cleanup * Fixed checkstyles * Cleanup * Hydrate Relationship * Cleanup * Added a complex SQL expression test and fixed bugs * Fixed merge issues. Added another test. Added better logging * Self-review * Self-review * Self-review * Self-review * Self-review * Address comments from @aklish * Refactor EntityHydrator (#893) * rebase * keep Jiaqi's changes * fix id * fix maven verify * Remove HQLFilterOperation * fix dictionary * fix SqlEngineTest * remove unused part * make codacy happy * should use getParametrizedType * address comments
- Loading branch information
1 parent
91192ee
commit 2bc77c3
Showing
21 changed files
with
890 additions
and
344 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
204 changes: 204 additions & 0 deletions
204
...n/src/main/java/com/yahoo/elide/datastores/aggregation/engine/AbstractEntityHydrator.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,204 @@ | ||
| /* | ||
| * Copyright 2019, Yahoo Inc. | ||
| * Licensed under the Apache License, Version 2.0 | ||
| * See LICENSE file in project root for terms. | ||
| */ | ||
| package com.yahoo.elide.datastores.aggregation.engine; | ||
|
|
||
| import com.yahoo.elide.core.EntityDictionary; | ||
| import com.yahoo.elide.datastores.aggregation.Query; | ||
| import com.yahoo.elide.datastores.aggregation.QueryEngine; | ||
| import com.yahoo.elide.datastores.aggregation.dimension.Dimension; | ||
| import com.yahoo.elide.datastores.aggregation.dimension.DimensionType; | ||
| import com.yahoo.elide.datastores.aggregation.metric.Metric; | ||
|
|
||
| import com.google.common.base.Preconditions; | ||
|
|
||
| import org.apache.commons.lang3.mutable.MutableInt; | ||
|
|
||
| import lombok.AccessLevel; | ||
| import lombok.Getter; | ||
|
|
||
| import java.util.ArrayList; | ||
| import java.util.HashMap; | ||
| import java.util.List; | ||
| import java.util.Map; | ||
| import java.util.stream.Collectors; | ||
|
|
||
| /** | ||
| * {@link AbstractEntityHydrator} hydrates the entity loaded by {@link QueryEngine#executeQuery(Query)}. | ||
| * <p> | ||
| * {@link AbstractEntityHydrator} is not thread-safe and should be accessed by only 1 thread in this application, | ||
| * because it uses {@link StitchList}. See {@link StitchList} for more details. | ||
| */ | ||
| public abstract class AbstractEntityHydrator { | ||
|
|
||
| @Getter(AccessLevel.PROTECTED) | ||
| private final EntityDictionary entityDictionary; | ||
|
|
||
| @Getter(AccessLevel.PRIVATE) | ||
| private final StitchList stitchList; | ||
|
|
||
| @Getter(AccessLevel.PROTECTED) | ||
| private final List<Map<String, Object>> results = new ArrayList<>(); | ||
|
|
||
| @Getter(AccessLevel.PRIVATE) | ||
| private final Query query; | ||
|
|
||
| /** | ||
| * Constructor. | ||
| * | ||
| * @param results The loaded objects from {@link QueryEngine#executeQuery(Query)} | ||
| * @param query The query passed to {@link QueryEngine#executeQuery(Query)} to load the objects | ||
| * @param entityDictionary An object that sets entity instance values and provides entity metadata info | ||
| */ | ||
| public AbstractEntityHydrator(List<Object> results, Query query, EntityDictionary entityDictionary) { | ||
| this.stitchList = new StitchList(entityDictionary); | ||
| this.query = query; | ||
| this.entityDictionary = entityDictionary; | ||
|
|
||
| //Get all the projections from the client query. | ||
| List<String> projections = this.query.getMetrics().keySet().stream() | ||
| .map(Metric::getName) | ||
| .collect(Collectors.toList()); | ||
|
|
||
| projections.addAll(this.query.getDimensions().stream() | ||
| .map(Dimension::getName) | ||
| .collect(Collectors.toList())); | ||
|
|
||
|
|
||
| results.forEach(result -> { | ||
| Map<String, Object> row = new HashMap<>(); | ||
|
|
||
| Object[] resultValues = result instanceof Object[] ? (Object[]) result : new Object[] { result }; | ||
|
|
||
| Preconditions.checkArgument(projections.size() == resultValues.length); | ||
|
|
||
| for (int idx = 0; idx < resultValues.length; idx++) { | ||
| Object value = resultValues[idx]; | ||
| String fieldName = projections.get(idx); | ||
| row.put(fieldName, value); | ||
| } | ||
|
|
||
| this.results.add(row); | ||
| }); | ||
| } | ||
|
|
||
| /** | ||
| * Loads a map of relationship object ID to relationship object instance. | ||
| * <p> | ||
| * Note the relationship cannot be toMany. This method will be invoked for every relationship field of the | ||
| * requested entity. Its implementation should return the result of the following query | ||
| * <p> | ||
| * <b>Given a relationship with type {@code relationshipType} in an entity, loads all relationship | ||
| * objects whose foreign keys are one of the specified list, {@code joinFieldIds}</b>. | ||
| * <p> | ||
| * For example, when the relationship is loaded from SQL and we have the following example identity: | ||
| * <pre> | ||
| * public class PlayerStats { | ||
| * private String id; | ||
| * private Country country; | ||
| * | ||
| * @OneToOne | ||
| * @JoinColumn(name = "country_id") | ||
| * public Country getCountry() { | ||
| * return country; | ||
| * } | ||
| * } | ||
| * </pre> | ||
| * In this case {@code relationshipType = Country.class}. If {@code country} is | ||
| * requested in {@code PlayerStats} query and 3 stats, for example, are found in database whose country ID's are | ||
| * {@code joinFieldIds = [840, 344, 840]}, then this method should effectively run the following query (JPQL as | ||
| * example) | ||
| * <pre> | ||
| * {@code | ||
| * SELECT e FROM country_table e WHERE country_id IN (840, 344); | ||
| * } | ||
| * </pre> | ||
| * and returns the map of [840: Country(id:840), 344: Country(id:344)] | ||
| * | ||
| * @param relationshipType The type of relationship | ||
| * @param joinFieldIds The specified list of join ID's against the relationship | ||
| * | ||
| * @return a list of hydrating values | ||
| */ | ||
| protected abstract Map<Object, Object> getRelationshipValues( | ||
| Class<?> relationshipType, | ||
| List<Object> joinFieldIds | ||
| ); | ||
|
|
||
| public Iterable<Object> hydrate() { | ||
| //Coerce the results into entity objects. | ||
| MutableInt counter = new MutableInt(0); | ||
|
|
||
| List<Object> queryResults = getResults().stream() | ||
| .map((result) -> coerceObjectToEntity(result, counter)) | ||
| .collect(Collectors.toList()); | ||
|
|
||
| if (getStitchList().shouldStitch()) { | ||
| // relationship is requested, stitch relationship then | ||
| populateObjectLookupTable(); | ||
| getStitchList().stitch(); | ||
| } | ||
|
|
||
| return queryResults; | ||
| } | ||
|
|
||
| /** | ||
| * Coerces results from a {@link Query} into an Object. | ||
| * | ||
| * @param result a fieldName-value map | ||
| * @param counter Monotonically increasing number to generate IDs. | ||
| * @return A hydrated entity object. | ||
| */ | ||
| protected Object coerceObjectToEntity(Map<String, Object> result, MutableInt counter) { | ||
| Class<?> entityClass = query.getSchema().getEntityClass(); | ||
|
|
||
| //Construct the object. | ||
| Object entityInstance; | ||
| try { | ||
| entityInstance = entityClass.newInstance(); | ||
| } catch (InstantiationException | IllegalAccessException e) { | ||
| throw new IllegalStateException(e); | ||
| } | ||
|
|
||
| result.forEach((fieldName, value) -> { | ||
| Dimension dim = query.getSchema().getDimension(fieldName); | ||
|
|
||
| if (dim != null && dim.getDimensionType() == DimensionType.ENTITY) { | ||
| getStitchList().todo(entityInstance, fieldName, value); // We don't hydrate relationships here. | ||
| } else { | ||
| getEntityDictionary().setValue(entityInstance, fieldName, value); | ||
| } | ||
| }); | ||
|
|
||
| //Set the ID (it must be coerced from an integer) | ||
| getEntityDictionary().setValue( | ||
| entityInstance, | ||
| getEntityDictionary().getIdFieldName(entityClass), | ||
| counter.getAndIncrement() | ||
| ); | ||
|
|
||
| return entityInstance; | ||
| } | ||
|
|
||
| /** | ||
| * Foe each requested relationship, run a single query to load all relationship objects whose ID's are involved in | ||
| * the request. | ||
| */ | ||
| private void populateObjectLookupTable() { | ||
| // mapping: relationship field name -> join ID's | ||
| Map<String, List<Object>> hydrationIdsByRelationship = getStitchList().getHydrationMapping(); | ||
|
|
||
| // hydrate each relationship | ||
| for (Map.Entry<String, List<Object>> entry : hydrationIdsByRelationship.entrySet()) { | ||
| String joinField = entry.getKey(); | ||
| List<Object> joinFieldIds = entry.getValue(); | ||
| Class<?> relationshipType = getEntityDictionary().getParameterizedType( | ||
| getQuery().getSchema().getEntityClass(), | ||
| joinField); | ||
|
|
||
| getStitchList().populateLookup(relationshipType, getRelationshipValues(relationshipType, joinFieldIds)); | ||
| } | ||
| } | ||
| } |
74 changes: 74 additions & 0 deletions
74
...gation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLEntityHydrator.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,74 @@ | ||
| /* | ||
| * Copyright 2019, Yahoo Inc. | ||
| * Licensed under the Apache License, Version 2.0 | ||
| * See LICENSE file in project root for terms. | ||
| */ | ||
| package com.yahoo.elide.datastores.aggregation.engine; | ||
|
|
||
| import com.yahoo.elide.core.EntityDictionary; | ||
| import com.yahoo.elide.datastores.aggregation.Query; | ||
| import lombok.AccessLevel; | ||
| import lombok.Getter; | ||
|
|
||
| import java.util.AbstractMap; | ||
| import java.util.Collections; | ||
| import java.util.LinkedList; | ||
| import java.util.List; | ||
| import java.util.Map; | ||
| import java.util.stream.Collectors; | ||
|
|
||
| import javax.persistence.EntityManager; | ||
|
|
||
| /** | ||
| * {@link SQLEntityHydrator} hydrates the entity loaded by {@link SQLQueryEngine#executeQuery(Query)}. | ||
| */ | ||
| public class SQLEntityHydrator extends AbstractEntityHydrator { | ||
|
|
||
| @Getter(AccessLevel.PRIVATE) | ||
| private final EntityManager entityManager; | ||
|
|
||
| /** | ||
| * Constructor. | ||
| * | ||
| * @param results The loaded objects from {@link SQLQueryEngine#executeQuery(Query)} | ||
| * @param query The query passed to {@link SQLQueryEngine#executeQuery(Query)} to load the objects | ||
| * @param entityDictionary An object that sets entity instance values and provides entity metadata info | ||
| * @param entityManager An service that issues JPQL queries to load relationship objects | ||
| */ | ||
| public SQLEntityHydrator( | ||
| List<Object> results, | ||
| Query query, | ||
| EntityDictionary entityDictionary, | ||
| EntityManager entityManager | ||
| ) { | ||
| super(results, query, entityDictionary); | ||
| this.entityManager = entityManager; | ||
| } | ||
|
|
||
| @Override | ||
| protected Map<Object, Object> getRelationshipValues( | ||
| Class<?> relationshipType, | ||
| List<Object> joinFieldIds | ||
| ) { | ||
| if (joinFieldIds.isEmpty()) { | ||
| return Collections.emptyMap(); | ||
| } | ||
|
|
||
| List<Object> uniqueIds = joinFieldIds.stream().distinct().collect(Collectors.toCollection(LinkedList::new)); | ||
|
|
||
| List<Object> loaded = getEntityManager() | ||
| .createQuery( | ||
| String.format( | ||
| "SELECT e FROM %s e WHERE %s IN (:idList)", | ||
| relationshipType.getCanonicalName(), | ||
| getEntityDictionary().getIdFieldName(relationshipType) | ||
| ) | ||
| ) | ||
| .setParameter("idList", uniqueIds) | ||
| .getResultList(); | ||
|
|
||
| return loaded.stream() | ||
| .map(obj -> new AbstractMap.SimpleImmutableEntry<>((Object) getEntityDictionary().getId(obj), obj)) | ||
| .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); | ||
| } | ||
| } |
Oops, something went wrong.