From f04055be667cfa422e651f9500153195f063fbf3 Mon Sep 17 00:00:00 2001 From: John Karp Date: Mon, 11 May 2020 13:39:41 -0500 Subject: [PATCH 1/6] Make QueryEngine cache bypass flag part of Query --- .../AggregationDataStoreTransaction.java | 2 +- .../datastores/aggregation/QueryEngine.java | 5 +-- .../datastores/aggregation/query/Query.java | 5 +++ .../queryengines/AbstractEntityHydrator.java | 6 +-- .../queryengines/sql/SQLEntityHydrator.java | 7 ++-- .../queryengines/sql/SQLQueryEngine.java | 2 +- .../queryengines/sql/QueryEngineTest.java | 38 +++++++++---------- .../queryengines/sql/SubselectTest.java | 6 +-- .../queryengines/sql/ViewTest.java | 12 +++--- 9 files changed, 44 insertions(+), 39 deletions(-) diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStoreTransaction.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStoreTransaction.java index ae2e6cdc09..f3a5cb6bda 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStoreTransaction.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStoreTransaction.java @@ -53,7 +53,7 @@ public void createObject(Object entity, RequestScope scope) { @Override public Iterable loadObjects(EntityProjection entityProjection, RequestScope scope) { Query query = buildQuery(entityProjection, scope); - return queryEngine.executeQuery(query, true); + return queryEngine.executeQuery(query); } @Override diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/QueryEngine.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/QueryEngine.java index 6c57bce21a..d05ecec561 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/QueryEngine.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/QueryEngine.java @@ -163,11 +163,10 @@ private void populateMetaData(MetaDataStore metaDataStore) { * Executes the specified {@link Query} against a specific persistent storage, which understand the provided * {@link Query}. Results may be taken from a cache, if configured. * - * @param query The query customized for a particular persistent storage or storage client - * @param useCache Whether to use the cache, if configured + * @param query The query customized for a particular persistent storage or storage client * @return query results */ - public abstract Iterable executeQuery(Query query, boolean useCache); + public abstract Iterable executeQuery(Query query); /** * Returns the schema for a given entity class. diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/query/Query.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/query/Query.java index 21233ca448..564deb24af 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/query/Query.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/query/Query.java @@ -44,6 +44,11 @@ public class Query { Pagination pagination; RequestScope scope; + /** + * Whether to bypass the {@link QueryEngine} cache for this query. + */ + boolean bypassingCache; + /** * Returns all the dimensions regardless of type. * @return All the dimensions. diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/AbstractEntityHydrator.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/AbstractEntityHydrator.java index 5ca65349ca..6b5c174896 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/AbstractEntityHydrator.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/AbstractEntityHydrator.java @@ -27,7 +27,7 @@ import java.util.stream.Collectors; /** - * {@link AbstractEntityHydrator} hydrates the entity loaded by {@link QueryEngine#executeQuery(Query, boolean)}. + * {@link AbstractEntityHydrator} hydrates the entity loaded by {@link QueryEngine#executeQuery(Query)}. *

* {@link AbstractEntityHydrator} is not thread-safe and should be accessed by only 1 thread in this application, * because it uses {@link StitchList}. See {@link StitchList} for more details. @@ -49,8 +49,8 @@ public abstract class AbstractEntityHydrator { /** * Constructor. * - * @param results The loaded objects from {@link QueryEngine#executeQuery(Query, boolean)} - * @param query The query passed to {@link QueryEngine#executeQuery(Query, boolean)} to load the objects + * @param results The loaded objects from {@link QueryEngine#executeQuery(Query)} + * @param query The query passed to {@link QueryEngine#executeQuery(Query)} to load the objects * @param entityDictionary An object that sets entity instance values and provides entity metadata info */ public AbstractEntityHydrator(List results, Query query, EntityDictionary entityDictionary) { diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SQLEntityHydrator.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SQLEntityHydrator.java index 7ae9d8b0e1..5d158c76e4 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SQLEntityHydrator.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SQLEntityHydrator.java @@ -6,6 +6,7 @@ package com.yahoo.elide.datastores.aggregation.queryengines.sql; import com.yahoo.elide.core.EntityDictionary; +import com.yahoo.elide.datastores.aggregation.QueryEngine; import com.yahoo.elide.datastores.aggregation.query.Query; import com.yahoo.elide.datastores.aggregation.queryengines.AbstractEntityHydrator; import com.yahoo.elide.utils.coerce.CoerceUtil; @@ -22,7 +23,7 @@ import javax.persistence.EntityManager; /** - * {@link SQLEntityHydrator} hydrates the entity loaded by {@link SQLQueryEngine#executeQuery(Query, boolean)}. + * {@link SQLEntityHydrator} hydrates the entity loaded by {@link QueryEngine#executeQuery(Query)}. */ public class SQLEntityHydrator extends AbstractEntityHydrator { @@ -32,8 +33,8 @@ public class SQLEntityHydrator extends AbstractEntityHydrator { /** * Constructor. * - * @param results The loaded objects from {@link SQLQueryEngine#executeQuery(Query, boolean)} - * @param query The query passed to {@link SQLQueryEngine#executeQuery(Query, boolean)} to load the objects + * @param results The loaded objects from {@link QueryEngine#executeQuery(Query)} + * @param query The query passed to {@link QueryEngine#executeQuery(Query)} to load the objects * @param entityDictionary An object that sets entity instance values and provides entity metadata info * @param entityManager An service that issues JPQL queries to load relationship objects */ diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SQLQueryEngine.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SQLQueryEngine.java index 86c37d5e37..bccd96f072 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SQLQueryEngine.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SQLQueryEngine.java @@ -115,7 +115,7 @@ public MetricProjection constructMetricProjection(Metric metric, } @Override - public Iterable executeQuery(Query query, boolean useCache) { + public Iterable executeQuery(Query query) { EntityManager entityManager = null; EntityTransaction transaction = null; try { diff --git a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/QueryEngineTest.java b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/QueryEngineTest.java index a25350efe3..543b2c7e3f 100644 --- a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/QueryEngineTest.java +++ b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/QueryEngineTest.java @@ -54,7 +54,7 @@ public void testFullTableLoad() { .timeDimension(toProjection(playerStatsTable.getTimeDimension("recordedDate"), TimeGrain.DAY)) .build(); - List results = StreamSupport.stream(engine.executeQuery(query, true).spliterator(), false) + List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) .collect(Collectors.toList()); PlayerStats stats0 = new PlayerStats(); @@ -91,7 +91,7 @@ public void testFromSubQuery() { .metric(invoke(playerStatsViewTable.getMetric("highScore"))) .build(); - List results = StreamSupport.stream(engine.executeQuery(query, true).spliterator(), false) + List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) .collect(Collectors.toList()); PlayerStatsView stats2 = new PlayerStatsView(); @@ -123,7 +123,7 @@ public void testAllArgumentQuery() throws Exception { .sorting(new SortingImpl(sortMap, PlayerStatsView.class, dictionary)) .build(); - List results = StreamSupport.stream(engine.executeQuery(query, true).spliterator(), false) + List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) .collect(Collectors.toList()); PlayerStatsView stats2 = new PlayerStatsView(); @@ -151,7 +151,7 @@ public void testDegenerateDimensionFilter() throws Exception { PlayerStats.class, false)) .build(); - List results = StreamSupport.stream(engine.executeQuery(query, true).spliterator(), false) + List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) .collect(Collectors.toList()); PlayerStats stats1 = new PlayerStats(); @@ -178,7 +178,7 @@ public void testNotProjectedFilter() throws Exception { PlayerStatsView.class, false)) .build(); - List results = StreamSupport.stream(engine.executeQuery(query, true).spliterator(), false) + List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) .collect(Collectors.toList()); PlayerStatsView stats2 = new PlayerStatsView(); @@ -201,7 +201,7 @@ public void testSortAggregatedMetric() { .sorting(new SortingImpl(sortMap, PlayerStats.class, dictionary)) .build(); - List results = StreamSupport.stream(engine.executeQuery(query, true).spliterator(), false) + List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) .collect(Collectors.toList()); PlayerStats stats0 = new PlayerStats(); @@ -235,7 +235,7 @@ public void testSortJoin() { .sorting(new SortingImpl(sortMap, PlayerStats.class, dictionary)) .build(); - List results = StreamSupport.stream(engine.executeQuery(query, true).spliterator(), false) + List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) .collect(Collectors.toList()); PlayerStats stats0 = new PlayerStats(); @@ -285,7 +285,7 @@ public void testPagination() { .pagination(pagination) .build(); - List results = StreamSupport.stream(engine.executeQuery(query, true).spliterator(), false) + List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) .collect(Collectors.toList()); //Jon Doe,1234,72,Good,840,2019-07-12 00:00:00 @@ -315,7 +315,7 @@ public void testHavingClause() throws Exception { PlayerStats.class, false)) .build(); - List results = StreamSupport.stream(engine.executeQuery(query, true).spliterator(), false) + List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) .collect(Collectors.toList()); // Only "Good" rating would have total high score less than 2400 @@ -344,7 +344,7 @@ public void testHavingClauseJoin() throws Exception { PlayerStats.class, false)) .build(); - List results = StreamSupport.stream(engine.executeQuery(query, true).spliterator(), false) + List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) .collect(Collectors.toList()); PlayerStats stats0 = new PlayerStats(); @@ -381,7 +381,7 @@ public void testSortByMultipleColumns() { .sorting(new SortingImpl(sortMap, PlayerStats.class, dictionary)) .build(); - List results = StreamSupport.stream(engine.executeQuery(query, true).spliterator(), false) + List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) .collect(Collectors.toList()); PlayerStats stats0 = new PlayerStats(); @@ -419,7 +419,7 @@ public void testJoinToGroupBy() { .groupByDimension(toProjection(playerStatsTable.getDimension("countryIsoCode"))) .build(); - List results = StreamSupport.stream(engine.executeQuery(query, true).spliterator(), false) + List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) .collect(Collectors.toList()); PlayerStats stats1 = new PlayerStats(); @@ -452,7 +452,7 @@ public void testJoinToFilter() throws Exception { PlayerStats.class, false)) .build(); - List results = StreamSupport.stream(engine.executeQuery(query, true).spliterator(), false) + List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) .collect(Collectors.toList()); PlayerStats stats1 = new PlayerStats(); @@ -487,7 +487,7 @@ public void testJoinToSort() { .sorting(new SortingImpl(sortMap, PlayerStats.class, dictionary)) .build(); - List results = StreamSupport.stream(engine.executeQuery(query, true).spliterator(), false) + List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) .collect(Collectors.toList()); PlayerStats stats1 = new PlayerStats(); @@ -525,7 +525,7 @@ public void testTotalScoreByMonth() { .timeDimension(toProjection(playerStatsTable.getTimeDimension("recordedDate"), TimeGrain.MONTH)) .build(); - List results = StreamSupport.stream(engine.executeQuery(query, true).spliterator(), false) + List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) .collect(Collectors.toList()); PlayerStats stats0 = new PlayerStats(); @@ -554,7 +554,7 @@ public void testFilterByTemporalDimension() { .whereFilter(predicate) .build(); - List results = StreamSupport.stream(engine.executeQuery(query, true).spliterator(), false) + List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) .collect(Collectors.toList()); PlayerStats stats0 = new PlayerStats(); @@ -579,7 +579,7 @@ public void testAmbiguousFields() { .sorting(new SortingImpl(sortMap, PlayerStats.class, dictionary)) .build(); - List results = StreamSupport.stream(engine.executeQuery(query, true).spliterator(), false) + List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) .collect(Collectors.toList()); PlayerStats stats0 = new PlayerStats(); @@ -614,7 +614,7 @@ public void testNullJoinToStringValue() { .groupByDimension(toProjection(playerStatsTable.getDimension("countryNickName"))) .build(); - List results = StreamSupport.stream(engine.executeQuery(query, true).spliterator(), false) + List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) .collect(Collectors.toList()); PlayerStats stats1 = new PlayerStats(); @@ -640,7 +640,7 @@ public void testNullJoinToIntValue() { .groupByDimension(toProjection(playerStatsTable.getDimension("countryUnSeats"))) .build(); - List results = StreamSupport.stream(engine.executeQuery(query, true).spliterator(), false) + List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) .collect(Collectors.toList()); PlayerStats stats1 = new PlayerStats(); diff --git a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SubselectTest.java b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SubselectTest.java index be307fc11c..d6fc403633 100644 --- a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SubselectTest.java +++ b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SubselectTest.java @@ -54,7 +54,7 @@ public void testJoinToGroupBy() throws Exception { .groupByDimension(toProjection(playerStatsTable.getDimension("subCountryIsoCode"))) .build(); - List results = StreamSupport.stream(engine.executeQuery(query, true).spliterator(), false) + List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) .collect(Collectors.toList()); PlayerStats stats1 = new PlayerStats(); @@ -87,7 +87,7 @@ public void testJoinToFilter() throws Exception { PlayerStats.class, false)) .build(); - List results = StreamSupport.stream(engine.executeQuery(query, true).spliterator(), false) + List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) .collect(Collectors.toList()); PlayerStats stats1 = new PlayerStats(); @@ -124,7 +124,7 @@ public void testJoinToSort() throws Exception { .sorting(new SortingImpl(sortMap, PlayerStats.class, dictionary)) .build(); - List results = StreamSupport.stream(engine.executeQuery(query, true).spliterator(), false) + List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) .collect(Collectors.toList()); PlayerStats stats1 = new PlayerStats(); diff --git a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/ViewTest.java b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/ViewTest.java index eda281bd52..e4c44fa14f 100644 --- a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/ViewTest.java +++ b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/ViewTest.java @@ -46,7 +46,7 @@ public void testViewAttribute() { .sorting(new SortingImpl(sortMap, PlayerStatsWithView.class, dictionary)) .build(); - List results = StreamSupport.stream(engine.executeQuery(query, true).spliterator(), false) + List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) .collect(Collectors.toList()); PlayerStatsWithView usa0 = new PlayerStatsWithView(); @@ -80,7 +80,7 @@ public void testNestedViewAttribute() throws Exception { .sorting(new SortingImpl(sortMap, PlayerStatsWithView.class, dictionary)) .build(); - List results = StreamSupport.stream(engine.executeQuery(query, true).spliterator(), false) + List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) .collect(Collectors.toList()); PlayerStatsWithView usa0 = new PlayerStatsWithView(); @@ -115,7 +115,7 @@ public void testNestedRelationshipAttribute() throws Exception { .sorting(new SortingImpl(sortMap, PlayerStatsWithView.class, dictionary)) .build(); - List results = StreamSupport.stream(engine.executeQuery(query, true).spliterator(), false) + List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) .collect(Collectors.toList()); PlayerStatsWithView usa0 = new PlayerStatsWithView(); @@ -150,7 +150,7 @@ public void testSortingViewAttribute() throws Exception { .sorting(new SortingImpl(sortMap, PlayerStatsWithView.class, dictionary)) .build(); - List results = StreamSupport.stream(engine.executeQuery(query, true).spliterator(), false) + List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) .collect(Collectors.toList()); PlayerStatsWithView usa0 = new PlayerStatsWithView(); @@ -185,7 +185,7 @@ public void testSortingNestedViewAttribute() throws Exception { .sorting(new SortingImpl(sortMap, PlayerStatsWithView.class, dictionary)) .build(); - List results = StreamSupport.stream(engine.executeQuery(query, true).spliterator(), false) + List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) .collect(Collectors.toList()); PlayerStatsWithView usa0 = new PlayerStatsWithView(); @@ -220,7 +220,7 @@ public void testSortingNestedRelationshipAttribute() throws Exception { .sorting(new SortingImpl(sortMap, PlayerStatsWithView.class, dictionary)) .build(); - List results = StreamSupport.stream(engine.executeQuery(query, true).spliterator(), false) + List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) .collect(Collectors.toList()); PlayerStatsWithView usa0 = new PlayerStatsWithView(); From 9118f44b225754277f95b9f47ce087da1c127523 Mon Sep 17 00:00:00 2001 From: John Karp Date: Mon, 11 May 2020 17:47:07 -0500 Subject: [PATCH 2/6] Forming the cache key is not free, so don't a use stub cache --- .../datastores/aggregation/QueryEngine.java | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/QueryEngine.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/QueryEngine.java index d05ecec561..ae56424da6 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/QueryEngine.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/QueryEngine.java @@ -91,11 +91,7 @@ public QueryEngine(MetaDataStore metaDataStore, Cache cache) { populateMetaData(metaDataStore); this.tables = metaDataStore.getMetaData(Table.class).stream() .collect(Collectors.toMap(Table::getId, Functions.identity())); - if (cache != null) { - this.cache = cache; - } else { - this.cache = new NoCache(); - } + this.cache = cache; } /** @@ -176,16 +172,4 @@ private void populateMetaData(MetaDataStore metaDataStore) { public Table getTable(String classAlias) { return tables.get(classAlias); } - - private static class NoCache implements Cache { - @Override - public Iterable get(Object key) { - return null; - } - - @Override - public void put(Object key, Iterable result) { - // Do nothing - } - } } From e1744371e6701549bcbb402f953ce667aeb43580 Mon Sep 17 00:00:00 2001 From: John Karp Date: Thu, 14 May 2020 15:24:20 -0500 Subject: [PATCH 3/6] Add QueryResult class --- .../AggregationDataStoreTransaction.java | 2 +- .../datastores/aggregation/QueryEngine.java | 3 +- .../datastores/aggregation/query/Cache.java | 8 ++-- .../aggregation/query/QueryResult.java | 22 +++++++++++ .../queryengines/sql/SQLQueryEngine.java | 7 +++- .../queryengines/sql/QueryEngineTest.java | 38 +++++++++---------- .../queryengines/sql/SubselectTest.java | 6 +-- .../queryengines/sql/ViewTest.java | 12 +++--- 8 files changed, 62 insertions(+), 36 deletions(-) create mode 100644 elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/query/QueryResult.java diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStoreTransaction.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStoreTransaction.java index f3a5cb6bda..d475961dde 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStoreTransaction.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStoreTransaction.java @@ -53,7 +53,7 @@ public void createObject(Object entity, RequestScope scope) { @Override public Iterable loadObjects(EntityProjection entityProjection, RequestScope scope) { Query query = buildQuery(entityProjection, scope); - return queryEngine.executeQuery(query); + return queryEngine.executeQuery(query).getData(); } @Override diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/QueryEngine.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/QueryEngine.java index ae56424da6..4308e4db10 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/QueryEngine.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/QueryEngine.java @@ -18,6 +18,7 @@ import com.yahoo.elide.datastores.aggregation.query.ColumnProjection; import com.yahoo.elide.datastores.aggregation.query.MetricProjection; import com.yahoo.elide.datastores.aggregation.query.Query; +import com.yahoo.elide.datastores.aggregation.query.QueryResult; import com.yahoo.elide.datastores.aggregation.query.TimeDimensionProjection; import com.yahoo.elide.request.Argument; @@ -162,7 +163,7 @@ private void populateMetaData(MetaDataStore metaDataStore) { * @param query The query customized for a particular persistent storage or storage client * @return query results */ - public abstract Iterable executeQuery(Query query); + public abstract QueryResult executeQuery(Query query); /** * Returns the schema for a given entity class. diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/query/Cache.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/query/Cache.java index 841a00a198..733f6d573b 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/query/Cache.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/query/Cache.java @@ -6,16 +6,16 @@ package com.yahoo.elide.datastores.aggregation.query; /** - * A cache for Query results. + * A cache for {@link QueryResult}s. */ public interface Cache { /** - * Load Query result from cache. Exceptions should be passed through. + * Load QueryResult from cache. Exceptions should be passed through. * * @param key a key to look up in the cache. * @return query results from cache, or null if not found. */ - Iterable get(Object key); + QueryResult get(Object key); /** * Insert results into cache. @@ -23,5 +23,5 @@ public interface Cache { * @param key the key to associate with the query * @param result the result to cache with the key */ - void put(Object key, Iterable result); + void put(Object key, QueryResult result); } diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/query/QueryResult.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/query/QueryResult.java new file mode 100644 index 0000000000..80460e69da --- /dev/null +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/query/QueryResult.java @@ -0,0 +1,22 @@ +/* + * Copyright 2019, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ +package com.yahoo.elide.datastores.aggregation.query; + +import com.yahoo.elide.datastores.aggregation.QueryEngine; + +import lombok.Builder; +import lombok.NonNull; +import lombok.Value; + +/** + * A {@link QueryResult} contains the results from {@link QueryEngine#executeQuery(Query)}. + */ +@Value +@Builder +public class QueryResult { + @NonNull + Iterable data; +} diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SQLQueryEngine.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SQLQueryEngine.java index bccd96f072..3876ecaa6e 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SQLQueryEngine.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SQLQueryEngine.java @@ -23,6 +23,7 @@ import com.yahoo.elide.datastores.aggregation.query.ColumnProjection; import com.yahoo.elide.datastores.aggregation.query.MetricProjection; import com.yahoo.elide.datastores.aggregation.query.Query; +import com.yahoo.elide.datastores.aggregation.query.QueryResult; import com.yahoo.elide.datastores.aggregation.query.TimeDimensionProjection; import com.yahoo.elide.datastores.aggregation.queryengines.sql.metadata.SQLMetric; import com.yahoo.elide.datastores.aggregation.queryengines.sql.metadata.SQLReferenceTable; @@ -115,7 +116,7 @@ public MetricProjection constructMetricProjection(Metric metric, } @Override - public Iterable executeQuery(Query query) { + public QueryResult executeQuery(Query query) { EntityManager entityManager = null; EntityTransaction transaction = null; try { @@ -133,6 +134,7 @@ public Iterable executeQuery(Query query) { javax.persistence.Query jpaQuery = entityManager.createNativeQuery(sql.toString()); + QueryResult.QueryResultBuilder resultBuilder = QueryResult.builder(); Pagination pagination = query.getPagination(); if (pagination != null) { jpaQuery.setFirstResult(pagination.getOffset()); @@ -166,7 +168,8 @@ public Iterable executeQuery(Query query) { () -> jpaQuery.setHint(QueryHints.HINT_READONLY, true).getResultList(), "Running Query: " + sql).get(); - return new SQLEntityHydrator(results, query, getMetadataDictionary(), entityManager).hydrate(); + resultBuilder.data(new SQLEntityHydrator(results, query, getMetadataDictionary(), entityManager).hydrate()); + return resultBuilder.build(); } finally { if (transaction != null && transaction.isActive()) { transaction.commit(); diff --git a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/QueryEngineTest.java b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/QueryEngineTest.java index 543b2c7e3f..880fd85423 100644 --- a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/QueryEngineTest.java +++ b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/QueryEngineTest.java @@ -54,7 +54,7 @@ public void testFullTableLoad() { .timeDimension(toProjection(playerStatsTable.getTimeDimension("recordedDate"), TimeGrain.DAY)) .build(); - List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) + List results = StreamSupport.stream(engine.executeQuery(query).getData().spliterator(), false) .collect(Collectors.toList()); PlayerStats stats0 = new PlayerStats(); @@ -91,7 +91,7 @@ public void testFromSubQuery() { .metric(invoke(playerStatsViewTable.getMetric("highScore"))) .build(); - List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) + List results = StreamSupport.stream(engine.executeQuery(query).getData().spliterator(), false) .collect(Collectors.toList()); PlayerStatsView stats2 = new PlayerStatsView(); @@ -123,7 +123,7 @@ public void testAllArgumentQuery() throws Exception { .sorting(new SortingImpl(sortMap, PlayerStatsView.class, dictionary)) .build(); - List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) + List results = StreamSupport.stream(engine.executeQuery(query).getData().spliterator(), false) .collect(Collectors.toList()); PlayerStatsView stats2 = new PlayerStatsView(); @@ -151,7 +151,7 @@ public void testDegenerateDimensionFilter() throws Exception { PlayerStats.class, false)) .build(); - List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) + List results = StreamSupport.stream(engine.executeQuery(query).getData().spliterator(), false) .collect(Collectors.toList()); PlayerStats stats1 = new PlayerStats(); @@ -178,7 +178,7 @@ public void testNotProjectedFilter() throws Exception { PlayerStatsView.class, false)) .build(); - List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) + List results = StreamSupport.stream(engine.executeQuery(query).getData().spliterator(), false) .collect(Collectors.toList()); PlayerStatsView stats2 = new PlayerStatsView(); @@ -201,7 +201,7 @@ public void testSortAggregatedMetric() { .sorting(new SortingImpl(sortMap, PlayerStats.class, dictionary)) .build(); - List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) + List results = StreamSupport.stream(engine.executeQuery(query).getData().spliterator(), false) .collect(Collectors.toList()); PlayerStats stats0 = new PlayerStats(); @@ -235,7 +235,7 @@ public void testSortJoin() { .sorting(new SortingImpl(sortMap, PlayerStats.class, dictionary)) .build(); - List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) + List results = StreamSupport.stream(engine.executeQuery(query).getData().spliterator(), false) .collect(Collectors.toList()); PlayerStats stats0 = new PlayerStats(); @@ -285,7 +285,7 @@ public void testPagination() { .pagination(pagination) .build(); - List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) + List results = StreamSupport.stream(engine.executeQuery(query).getData().spliterator(), false) .collect(Collectors.toList()); //Jon Doe,1234,72,Good,840,2019-07-12 00:00:00 @@ -315,7 +315,7 @@ public void testHavingClause() throws Exception { PlayerStats.class, false)) .build(); - List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) + List results = StreamSupport.stream(engine.executeQuery(query).getData().spliterator(), false) .collect(Collectors.toList()); // Only "Good" rating would have total high score less than 2400 @@ -344,7 +344,7 @@ public void testHavingClauseJoin() throws Exception { PlayerStats.class, false)) .build(); - List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) + List results = StreamSupport.stream(engine.executeQuery(query).getData().spliterator(), false) .collect(Collectors.toList()); PlayerStats stats0 = new PlayerStats(); @@ -381,7 +381,7 @@ public void testSortByMultipleColumns() { .sorting(new SortingImpl(sortMap, PlayerStats.class, dictionary)) .build(); - List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) + List results = StreamSupport.stream(engine.executeQuery(query).getData().spliterator(), false) .collect(Collectors.toList()); PlayerStats stats0 = new PlayerStats(); @@ -419,7 +419,7 @@ public void testJoinToGroupBy() { .groupByDimension(toProjection(playerStatsTable.getDimension("countryIsoCode"))) .build(); - List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) + List results = StreamSupport.stream(engine.executeQuery(query).getData().spliterator(), false) .collect(Collectors.toList()); PlayerStats stats1 = new PlayerStats(); @@ -452,7 +452,7 @@ public void testJoinToFilter() throws Exception { PlayerStats.class, false)) .build(); - List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) + List results = StreamSupport.stream(engine.executeQuery(query).getData().spliterator(), false) .collect(Collectors.toList()); PlayerStats stats1 = new PlayerStats(); @@ -487,7 +487,7 @@ public void testJoinToSort() { .sorting(new SortingImpl(sortMap, PlayerStats.class, dictionary)) .build(); - List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) + List results = StreamSupport.stream(engine.executeQuery(query).getData().spliterator(), false) .collect(Collectors.toList()); PlayerStats stats1 = new PlayerStats(); @@ -525,7 +525,7 @@ public void testTotalScoreByMonth() { .timeDimension(toProjection(playerStatsTable.getTimeDimension("recordedDate"), TimeGrain.MONTH)) .build(); - List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) + List results = StreamSupport.stream(engine.executeQuery(query).getData().spliterator(), false) .collect(Collectors.toList()); PlayerStats stats0 = new PlayerStats(); @@ -554,7 +554,7 @@ public void testFilterByTemporalDimension() { .whereFilter(predicate) .build(); - List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) + List results = StreamSupport.stream(engine.executeQuery(query).getData().spliterator(), false) .collect(Collectors.toList()); PlayerStats stats0 = new PlayerStats(); @@ -579,7 +579,7 @@ public void testAmbiguousFields() { .sorting(new SortingImpl(sortMap, PlayerStats.class, dictionary)) .build(); - List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) + List results = StreamSupport.stream(engine.executeQuery(query).getData().spliterator(), false) .collect(Collectors.toList()); PlayerStats stats0 = new PlayerStats(); @@ -614,7 +614,7 @@ public void testNullJoinToStringValue() { .groupByDimension(toProjection(playerStatsTable.getDimension("countryNickName"))) .build(); - List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) + List results = StreamSupport.stream(engine.executeQuery(query).getData().spliterator(), false) .collect(Collectors.toList()); PlayerStats stats1 = new PlayerStats(); @@ -640,7 +640,7 @@ public void testNullJoinToIntValue() { .groupByDimension(toProjection(playerStatsTable.getDimension("countryUnSeats"))) .build(); - List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) + List results = StreamSupport.stream(engine.executeQuery(query).getData().spliterator(), false) .collect(Collectors.toList()); PlayerStats stats1 = new PlayerStats(); diff --git a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SubselectTest.java b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SubselectTest.java index d6fc403633..661ce3f2d8 100644 --- a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SubselectTest.java +++ b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SubselectTest.java @@ -54,7 +54,7 @@ public void testJoinToGroupBy() throws Exception { .groupByDimension(toProjection(playerStatsTable.getDimension("subCountryIsoCode"))) .build(); - List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) + List results = StreamSupport.stream(engine.executeQuery(query).getData().spliterator(), false) .collect(Collectors.toList()); PlayerStats stats1 = new PlayerStats(); @@ -87,7 +87,7 @@ public void testJoinToFilter() throws Exception { PlayerStats.class, false)) .build(); - List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) + List results = StreamSupport.stream(engine.executeQuery(query).getData().spliterator(), false) .collect(Collectors.toList()); PlayerStats stats1 = new PlayerStats(); @@ -124,7 +124,7 @@ public void testJoinToSort() throws Exception { .sorting(new SortingImpl(sortMap, PlayerStats.class, dictionary)) .build(); - List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) + List results = StreamSupport.stream(engine.executeQuery(query).getData().spliterator(), false) .collect(Collectors.toList()); PlayerStats stats1 = new PlayerStats(); diff --git a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/ViewTest.java b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/ViewTest.java index e4c44fa14f..7c53da04b2 100644 --- a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/ViewTest.java +++ b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/ViewTest.java @@ -46,7 +46,7 @@ public void testViewAttribute() { .sorting(new SortingImpl(sortMap, PlayerStatsWithView.class, dictionary)) .build(); - List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) + List results = StreamSupport.stream(engine.executeQuery(query).getData().spliterator(), false) .collect(Collectors.toList()); PlayerStatsWithView usa0 = new PlayerStatsWithView(); @@ -80,7 +80,7 @@ public void testNestedViewAttribute() throws Exception { .sorting(new SortingImpl(sortMap, PlayerStatsWithView.class, dictionary)) .build(); - List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) + List results = StreamSupport.stream(engine.executeQuery(query).getData().spliterator(), false) .collect(Collectors.toList()); PlayerStatsWithView usa0 = new PlayerStatsWithView(); @@ -115,7 +115,7 @@ public void testNestedRelationshipAttribute() throws Exception { .sorting(new SortingImpl(sortMap, PlayerStatsWithView.class, dictionary)) .build(); - List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) + List results = StreamSupport.stream(engine.executeQuery(query).getData().spliterator(), false) .collect(Collectors.toList()); PlayerStatsWithView usa0 = new PlayerStatsWithView(); @@ -150,7 +150,7 @@ public void testSortingViewAttribute() throws Exception { .sorting(new SortingImpl(sortMap, PlayerStatsWithView.class, dictionary)) .build(); - List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) + List results = StreamSupport.stream(engine.executeQuery(query).getData().spliterator(), false) .collect(Collectors.toList()); PlayerStatsWithView usa0 = new PlayerStatsWithView(); @@ -185,7 +185,7 @@ public void testSortingNestedViewAttribute() throws Exception { .sorting(new SortingImpl(sortMap, PlayerStatsWithView.class, dictionary)) .build(); - List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) + List results = StreamSupport.stream(engine.executeQuery(query).getData().spliterator(), false) .collect(Collectors.toList()); PlayerStatsWithView usa0 = new PlayerStatsWithView(); @@ -220,7 +220,7 @@ public void testSortingNestedRelationshipAttribute() throws Exception { .sorting(new SortingImpl(sortMap, PlayerStatsWithView.class, dictionary)) .build(); - List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) + List results = StreamSupport.stream(engine.executeQuery(query).getData().spliterator(), false) .collect(Collectors.toList()); PlayerStatsWithView usa0 = new PlayerStatsWithView(); From 1b50cc00fb4af70f08cd9de194c23c0496c17ca4 Mon Sep 17 00:00:00 2001 From: John Karp Date: Thu, 14 May 2020 16:02:44 -0500 Subject: [PATCH 4/6] Add pageTotals to QueryResult --- .../elide/datastores/aggregation/query/QueryResult.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/query/QueryResult.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/query/QueryResult.java index 80460e69da..020fa8bfac 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/query/QueryResult.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/query/QueryResult.java @@ -6,6 +6,7 @@ package com.yahoo.elide.datastores.aggregation.query; import com.yahoo.elide.datastores.aggregation.QueryEngine; +import com.yahoo.elide.request.Pagination; import lombok.Builder; import lombok.NonNull; @@ -19,4 +20,9 @@ public class QueryResult { @NonNull Iterable data; + + /** + * Total record count. Null unless Query had Pagination with {@link Pagination#returnPageTotals()} set. + */ + Long pageTotals; } From 5fd31724c242be3d2d289e2918a89ddfb5382906 Mon Sep 17 00:00:00 2001 From: John Karp Date: Fri, 15 May 2020 18:49:24 -0500 Subject: [PATCH 5/6] Pass page totals through QueryResult instead of Pagination --- .../elide/core/pagination/PaginationImpl.java | 26 ++++------ .../com/yahoo/elide/request/Pagination.java | 14 +++--- .../AggregationDataStoreTransaction.java | 7 ++- .../EntityProjectionTranslator.java | 3 +- .../query/ImmutablePagination.java | 49 +++++++++++++++++++ .../datastores/aggregation/query/Query.java | 5 +- .../queryengines/sql/SQLQueryEngine.java | 2 +- .../queryengines/sql/QueryEngineTest.java | 24 +++------ 8 files changed, 85 insertions(+), 45 deletions(-) create mode 100644 elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/query/ImmutablePagination.java diff --git a/elide-core/src/main/java/com/yahoo/elide/core/pagination/PaginationImpl.java b/elide-core/src/main/java/com/yahoo/elide/core/pagination/PaginationImpl.java index d419a8786b..1c6315cb67 100644 --- a/elide-core/src/main/java/com/yahoo/elide/core/pagination/PaginationImpl.java +++ b/elide-core/src/main/java/com/yahoo/elide/core/pagination/PaginationImpl.java @@ -64,17 +64,18 @@ public enum PaginationKey { offset, number, size, limit, totals } private static final String PAGE_KEYS_CSV = PAGE_KEYS.keySet().stream().collect(Collectors.joining(", ")); @Getter - private Integer offset; + private final int offset; @Getter - private Integer limit; + private final int limit; - private Boolean generateTotals; + private final boolean generateTotals; - private Boolean isDefault; + @Getter + private final boolean defaultInstance; @Getter - private Class entityClass; + private final Class entityClass; /** * Constructor. @@ -95,7 +96,7 @@ public PaginationImpl(Class entityClass, Boolean pageByPages) { this.entityClass = entityClass; - this.isDefault = (clientOffset == null && clientLimit == null && generateTotals == null); + this.defaultInstance = (clientOffset == null && clientLimit == null && generateTotals == null); Paginate paginate = entityClass != null ? (Paginate) entityClass.getAnnotation(Paginate.class) : null; @@ -107,7 +108,7 @@ public PaginationImpl(Class entityClass, String pageSizeLabel = pageByPages ? "size" : "limit"; - if (limit > maxLimit && !isDefault) { + if (limit > maxLimit && !defaultInstance) { throw new InvalidValueException("Pagination " + pageSizeLabel + " must be less than or equal to " + maxLimit); } @@ -138,19 +139,10 @@ public PaginationImpl(Class entityClass, * @return true if page totals should be returned. */ @Override - public Boolean returnPageTotals() { + public boolean returnPageTotals() { return generateTotals; } - /** - * Whether or not the client requested pagination or the system defaults are in effect. - * @return True if the system defaults are in effect. - */ - @Override - public Boolean isDefaultInstance() { - return isDefault; - } - /** * Given json-api paging params, generate page and pageSize values from query params. * diff --git a/elide-core/src/main/java/com/yahoo/elide/request/Pagination.java b/elide-core/src/main/java/com/yahoo/elide/request/Pagination.java index 22c8220685..49cd6836ec 100644 --- a/elide-core/src/main/java/com/yahoo/elide/request/Pagination.java +++ b/elide-core/src/main/java/com/yahoo/elide/request/Pagination.java @@ -14,35 +14,35 @@ public interface Pagination { /** * Default offset (in records) it client does not provide one. */ - public static final int DEFAULT_OFFSET = 0; + int DEFAULT_OFFSET = 0; /** * Default page limit (in records) it client does not provide one. */ - public static final int DEFAULT_PAGE_LIMIT = 500; + int DEFAULT_PAGE_LIMIT = 500; /** * Maximum allowable page limit (in records). */ - public static final int MAX_PAGE_LIMIT = 10000; + int MAX_PAGE_LIMIT = 10000; /** * Get the page offset. * @return record offset. */ - Integer getOffset(); + int getOffset(); /** * Get the page limit. * @return record limit. */ - Integer getLimit(); + int getLimit(); /** * Whether or not to fetch the collection size or not. * @return true if the client wants the total size of the collection. */ - Boolean returnPageTotals(); + boolean returnPageTotals(); /** * Get the total size of the collection @@ -60,5 +60,5 @@ public interface Pagination { * Is this the default instance (not present). * @return true if pagination wasn't requested. False otherwise. */ - public Boolean isDefaultInstance(); + boolean isDefaultInstance(); } diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStoreTransaction.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStoreTransaction.java index d475961dde..c994305a0f 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStoreTransaction.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStoreTransaction.java @@ -9,6 +9,7 @@ import com.yahoo.elide.core.RequestScope; import com.yahoo.elide.datastores.aggregation.metadata.models.Table; import com.yahoo.elide.datastores.aggregation.query.Query; +import com.yahoo.elide.datastores.aggregation.query.QueryResult; import com.yahoo.elide.request.EntityProjection; import com.google.common.annotations.VisibleForTesting; @@ -53,7 +54,11 @@ public void createObject(Object entity, RequestScope scope) { @Override public Iterable loadObjects(EntityProjection entityProjection, RequestScope scope) { Query query = buildQuery(entityProjection, scope); - return queryEngine.executeQuery(query).getData(); + QueryResult result = queryEngine.executeQuery(query); + if (entityProjection.getPagination() != null && entityProjection.getPagination().returnPageTotals()) { + entityProjection.getPagination().setPageTotals(result.getPageTotals()); + } + return result.getData(); } @Override diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/EntityProjectionTranslator.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/EntityProjectionTranslator.java index 00db729699..d37861b29f 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/EntityProjectionTranslator.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/EntityProjectionTranslator.java @@ -14,6 +14,7 @@ import com.yahoo.elide.datastores.aggregation.metadata.models.Table; import com.yahoo.elide.datastores.aggregation.metadata.models.TimeDimension; import com.yahoo.elide.datastores.aggregation.query.ColumnProjection; +import com.yahoo.elide.datastores.aggregation.query.ImmutablePagination; import com.yahoo.elide.datastores.aggregation.query.MetricProjection; import com.yahoo.elide.datastores.aggregation.query.Query; import com.yahoo.elide.datastores.aggregation.query.TimeDimensionProjection; @@ -72,7 +73,7 @@ public Query getQuery() { .whereFilter(whereFilter) .havingFilter(havingFilter) .sorting(entityProjection.getSorting()) - .pagination(entityProjection.getPagination()) + .pagination(ImmutablePagination.from(entityProjection.getPagination())) .build(); QueryValidator validator = new QueryValidator(query, getAllFields(), dictionary); validator.validate(); diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/query/ImmutablePagination.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/query/ImmutablePagination.java new file mode 100644 index 0000000000..46a4b07c29 --- /dev/null +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/query/ImmutablePagination.java @@ -0,0 +1,49 @@ +/* + * Copyright 2020, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ + +package com.yahoo.elide.datastores.aggregation.query; + +import com.yahoo.elide.request.Pagination; + +import lombok.Value; + +/** + * An immutable Pagination. Doesn't support getPageTotals/setPageTotals; page totals must be returned via QueryResult. + */ +@Value +public class ImmutablePagination implements Pagination { + + int offset; + int limit; + boolean defaultInstance; + boolean returnPageTotals; + + public static ImmutablePagination from(Pagination src) { + if (src instanceof ImmutablePagination) { + return (ImmutablePagination) src; + } else if (src != null) { + return new ImmutablePagination( + src.getOffset(), src.getLimit(), src.isDefaultInstance(), src.returnPageTotals()); + } else { + return null; + } + } + + @Override + public boolean returnPageTotals() { + return returnPageTotals; + } + + @Override + public Long getPageTotals() { + return null; + } + + @Override + public void setPageTotals(Long pageTotals) { + throw new UnsupportedOperationException("ImmutablePagination does not support setPageTotals"); + } +} diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/query/Query.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/query/Query.java index 564deb24af..308c16ea23 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/query/Query.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/query/Query.java @@ -9,9 +9,9 @@ import com.yahoo.elide.core.filter.expression.FilterExpression; import com.yahoo.elide.datastores.aggregation.QueryEngine; import com.yahoo.elide.datastores.aggregation.metadata.models.Table; -import com.yahoo.elide.request.Pagination; import com.yahoo.elide.request.Sorting; import lombok.Builder; +import lombok.NonNull; import lombok.Singular; import lombok.Value; @@ -27,6 +27,7 @@ @Value @Builder public class Query { + @NonNull Table table; @Singular @@ -41,7 +42,7 @@ public class Query { FilterExpression whereFilter; FilterExpression havingFilter; Sorting sorting; - Pagination pagination; + ImmutablePagination pagination; RequestScope scope; /** diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SQLQueryEngine.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SQLQueryEngine.java index 3876ecaa6e..712695a339 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SQLQueryEngine.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SQLQueryEngine.java @@ -156,7 +156,7 @@ public QueryResult executeQuery(Query query) { "Running Query: " + paginationSQL ).get(); - pagination.setPageTotals(total); + resultBuilder.pageTotals(total); } } diff --git a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/QueryEngineTest.java b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/QueryEngineTest.java index 880fd85423..7f46957873 100644 --- a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/QueryEngineTest.java +++ b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/QueryEngineTest.java @@ -10,14 +10,15 @@ import com.yahoo.elide.core.Path; import com.yahoo.elide.core.filter.FilterPredicate; import com.yahoo.elide.core.filter.Operator; -import com.yahoo.elide.core.pagination.PaginationImpl; import com.yahoo.elide.core.sort.SortingImpl; import com.yahoo.elide.datastores.aggregation.example.PlayerStats; import com.yahoo.elide.datastores.aggregation.example.PlayerStatsView; import com.yahoo.elide.datastores.aggregation.framework.SQLUnitTest; import com.yahoo.elide.datastores.aggregation.metadata.enums.TimeGrain; import com.yahoo.elide.datastores.aggregation.metadata.models.Table; +import com.yahoo.elide.datastores.aggregation.query.ImmutablePagination; import com.yahoo.elide.datastores.aggregation.query.Query; +import com.yahoo.elide.datastores.aggregation.query.QueryResult; import com.yahoo.elide.datastores.aggregation.queryengines.sql.annotation.FromSubquery; import com.yahoo.elide.request.Sorting; @@ -267,25 +268,16 @@ public void testSortJoin() { */ @Test public void testPagination() { - PaginationImpl pagination = new PaginationImpl( - PlayerStats.class, - 0, - 1, - PaginationImpl.DEFAULT_PAGE_LIMIT, - PaginationImpl.MAX_PAGE_LIMIT, - true, - false - ); - Query query = Query.builder() .table(playerStatsTable) .metric(invoke(playerStatsTable.getMetric("lowScore"))) .groupByDimension(toProjection(playerStatsTable.getDimension("overallRating"))) .timeDimension(toProjection(playerStatsTable.getTimeDimension("recordedDate"), TimeGrain.DAY)) - .pagination(pagination) + .pagination(new ImmutablePagination(0, 1, false, true)) .build(); - List results = StreamSupport.stream(engine.executeQuery(query).getData().spliterator(), false) + QueryResult result = engine.executeQuery(query); + List data = StreamSupport.stream(result.getData().spliterator(), false) .collect(Collectors.toList()); //Jon Doe,1234,72,Good,840,2019-07-12 00:00:00 @@ -295,9 +287,9 @@ public void testPagination() { stats1.setOverallRating("Good"); stats1.setRecordedDate(Timestamp.valueOf("2019-07-12 00:00:00")); - assertEquals(results.size(), 1, "Number of records returned does not match"); - assertEquals(results.get(0), stats1, "Returned record does not match"); - assertEquals(pagination.getPageTotals(), 3, "Page totals does not match"); + assertEquals(data.size(), 1, "Number of records returned does not match"); + assertEquals(data.get(0), stats1, "Returned record does not match"); + assertEquals(result.getPageTotals(), 3, "Page totals does not match"); } /** From e89f059a8442bf943d9517441edb8adf1f763abb Mon Sep 17 00:00:00 2001 From: John Karp Date: Wed, 20 May 2020 13:21:15 -0500 Subject: [PATCH 6/6] Codacy doesn't know @Value makes fields private --- .../query/ImmutablePagination.java | 8 ++++---- .../datastores/aggregation/query/Query.java | 20 +++++++++---------- .../aggregation/query/QueryResult.java | 4 ++-- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/query/ImmutablePagination.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/query/ImmutablePagination.java index 46a4b07c29..2d3e84d611 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/query/ImmutablePagination.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/query/ImmutablePagination.java @@ -16,10 +16,10 @@ @Value public class ImmutablePagination implements Pagination { - int offset; - int limit; - boolean defaultInstance; - boolean returnPageTotals; + private int offset; + private int limit; + private boolean defaultInstance; + private boolean returnPageTotals; public static ImmutablePagination from(Pagination src) { if (src instanceof ImmutablePagination) { diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/query/Query.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/query/Query.java index 308c16ea23..39e72d7811 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/query/Query.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/query/Query.java @@ -28,27 +28,27 @@ @Builder public class Query { @NonNull - Table table; + private Table table; @Singular - List metrics; + private List metrics; @Singular - Set groupByDimensions; + private Set groupByDimensions; @Singular - Set timeDimensions; + private Set timeDimensions; - FilterExpression whereFilter; - FilterExpression havingFilter; - Sorting sorting; - ImmutablePagination pagination; - RequestScope scope; + private FilterExpression whereFilter; + private FilterExpression havingFilter; + private Sorting sorting; + private ImmutablePagination pagination; + private RequestScope scope; /** * Whether to bypass the {@link QueryEngine} cache for this query. */ - boolean bypassingCache; + private boolean bypassingCache; /** * Returns all the dimensions regardless of type. diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/query/QueryResult.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/query/QueryResult.java index 020fa8bfac..7458077cb9 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/query/QueryResult.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/query/QueryResult.java @@ -19,10 +19,10 @@ @Builder public class QueryResult { @NonNull - Iterable data; + private Iterable data; /** * Total record count. Null unless Query had Pagination with {@link Pagination#returnPageTotals()} set. */ - Long pageTotals; + private Long pageTotals; }