diff --git a/elide-core/src/main/java/com/yahoo/elide/core/DataStoreTransaction.java b/elide-core/src/main/java/com/yahoo/elide/core/DataStoreTransaction.java index 2e3be0cc11..f931f57761 100644 --- a/elide-core/src/main/java/com/yahoo/elide/core/DataStoreTransaction.java +++ b/elide-core/src/main/java/com/yahoo/elide/core/DataStoreTransaction.java @@ -266,6 +266,7 @@ default boolean supportsPagination(Class entityClass, FilterExpression expres /** * Cancel running transaction. * Implementation must be thread-safe. + * @param scope contains request level metadata. */ - void cancel(); + void cancel(RequestScope scope); } diff --git a/elide-core/src/main/java/com/yahoo/elide/core/RequestScope.java b/elide-core/src/main/java/com/yahoo/elide/core/RequestScope.java index 5b4cf9ae69..a57584e276 100644 --- a/elide-core/src/main/java/com/yahoo/elide/core/RequestScope.java +++ b/elide-core/src/main/java/com/yahoo/elide/core/RequestScope.java @@ -65,10 +65,11 @@ public class RequestScope implements com.yahoo.elide.security.RequestScope { @Getter private final int updateStatusCode; @Getter private final MultipleFilterDialect filterDialect; @Getter private final String apiVersion; + @Getter @Setter private Map headers; //TODO - this ought to be read only and set in the constructor. @Getter @Setter private EntityProjection entityProjection; - private final UUID requestId; + @Getter private final UUID requestId; private final Map expressionsByType; private PublishSubject lifecycleEvents; @@ -143,6 +144,7 @@ public RequestScope(String baseUrlEndPoint, this.dirtyResources = new LinkedHashSet<>(); this.deletedResources = new LinkedHashSet<>(); this.requestId = requestId; + this.headers = new HashMap<>(); Function permissionExecutorGenerator = elideSettings.getPermissionExecutor(); this.permissionExecutor = (permissionExecutorGenerator == null) @@ -231,6 +233,7 @@ protected RequestScope(String path, String apiVersion, this.updateStatusCode = outerRequestScope.updateStatusCode; this.queuedLifecycleEvents = outerRequestScope.queuedLifecycleEvents; this.requestId = outerRequestScope.requestId; + this.headers = outerRequestScope.headers; } public Set getNewResources() { diff --git a/elide-core/src/main/java/com/yahoo/elide/core/datastore/inmemory/HashMapStoreTransaction.java b/elide-core/src/main/java/com/yahoo/elide/core/datastore/inmemory/HashMapStoreTransaction.java index cc3ccbcf73..d782e09fe7 100644 --- a/elide-core/src/main/java/com/yahoo/elide/core/datastore/inmemory/HashMapStoreTransaction.java +++ b/elide-core/src/main/java/com/yahoo/elide/core/datastore/inmemory/HashMapStoreTransaction.java @@ -189,7 +189,7 @@ private boolean containsObject(Object obj) { } @Override - public void cancel() { + public void cancel(RequestScope scope) { //nothing to cancel in HashMap store transaction } } diff --git a/elide-core/src/main/java/com/yahoo/elide/core/datastore/inmemory/InMemoryStoreTransaction.java b/elide-core/src/main/java/com/yahoo/elide/core/datastore/inmemory/InMemoryStoreTransaction.java index 4752764968..061e3de6f9 100644 --- a/elide-core/src/main/java/com/yahoo/elide/core/datastore/inmemory/InMemoryStoreTransaction.java +++ b/elide-core/src/main/java/com/yahoo/elide/core/datastore/inmemory/InMemoryStoreTransaction.java @@ -469,7 +469,7 @@ private Pair, Optional> splitPagination( } @Override - public void cancel() { - tx.cancel(); + public void cancel(RequestScope scope) { + tx.cancel(scope); } } diff --git a/elide-core/src/main/java/com/yahoo/elide/core/datastore/wrapped/TransactionWrapper.java b/elide-core/src/main/java/com/yahoo/elide/core/datastore/wrapped/TransactionWrapper.java index 69f4472207..28ec39074e 100644 --- a/elide-core/src/main/java/com/yahoo/elide/core/datastore/wrapped/TransactionWrapper.java +++ b/elide-core/src/main/java/com/yahoo/elide/core/datastore/wrapped/TransactionWrapper.java @@ -126,7 +126,7 @@ public void close() throws IOException { } @Override - public void cancel() { - tx.cancel(); + public void cancel(RequestScope scope) { + tx.cancel(scope); } } diff --git a/elide-core/src/test/java/com/yahoo/elide/core/DataStoreTransactionTest.java b/elide-core/src/test/java/com/yahoo/elide/core/DataStoreTransactionTest.java index a431da77cb..d45df17508 100644 --- a/elide-core/src/test/java/com/yahoo/elide/core/DataStoreTransactionTest.java +++ b/elide-core/src/test/java/com/yahoo/elide/core/DataStoreTransactionTest.java @@ -162,7 +162,7 @@ public void createObject(Object entity, RequestScope scope) { } @Override - public void cancel() { + public void cancel(RequestScope scope) { //nothing } } diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStore.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStore.java index e5d492e8e1..098b52525b 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStore.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStore.java @@ -11,6 +11,7 @@ import com.yahoo.elide.core.EntityDictionary; import com.yahoo.elide.datastores.aggregation.annotation.Join; import com.yahoo.elide.datastores.aggregation.cache.Cache; +import com.yahoo.elide.datastores.aggregation.core.QueryLogger; import com.yahoo.elide.datastores.aggregation.metadata.models.Table; import com.yahoo.elide.datastores.aggregation.metadata.models.TimeDimension; import com.yahoo.elide.datastores.aggregation.queryengines.sql.annotation.FromSubquery; @@ -36,6 +37,7 @@ public class AggregationDataStore implements DataStore { @NonNull private final QueryEngine queryEngine; private final Cache cache; private final Set> dynamicCompiledClasses; + private final QueryLogger queryLogger; /** * These are the classes the Aggregation Store manages. @@ -74,6 +76,6 @@ public void populateEntityDictionary(EntityDictionary dictionary) { @Override public DataStoreTransaction beginTransaction() { - return new AggregationDataStoreTransaction(queryEngine, cache); + return new AggregationDataStoreTransaction(queryEngine, cache, queryLogger); } } diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStoreTransaction.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStoreTransaction.java index 21db3f81d4..e8a8f5d73c 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStoreTransaction.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStoreTransaction.java @@ -6,9 +6,13 @@ package com.yahoo.elide.datastores.aggregation; import com.yahoo.elide.core.DataStoreTransaction; +import com.yahoo.elide.core.HttpStatus; import com.yahoo.elide.core.RequestScope; +import com.yahoo.elide.core.exceptions.HttpStatusException; import com.yahoo.elide.datastores.aggregation.cache.Cache; import com.yahoo.elide.datastores.aggregation.cache.QueryKeyExtractor; +import com.yahoo.elide.datastores.aggregation.core.QueryLogger; +import com.yahoo.elide.datastores.aggregation.core.QueryResponse; import com.yahoo.elide.datastores.aggregation.metadata.models.Table; import com.yahoo.elide.datastores.aggregation.query.Query; import com.yahoo.elide.datastores.aggregation.query.QueryResult; @@ -19,6 +23,7 @@ import lombok.ToString; import java.io.IOException; + /** * Transaction handler for {@link AggregationDataStore}. */ @@ -27,11 +32,14 @@ public class AggregationDataStoreTransaction implements DataStoreTransaction { private final QueryEngine queryEngine; private final Cache cache; private final QueryEngine.Transaction queryEngineTransaction; + private final QueryLogger queryLogger; - public AggregationDataStoreTransaction(QueryEngine queryEngine, Cache cache) { + public AggregationDataStoreTransaction(QueryEngine queryEngine, Cache cache, + QueryLogger queryLogger) { this.queryEngine = queryEngine; this.cache = cache; this.queryEngineTransaction = queryEngine.beginTransaction(); + this.queryLogger = queryLogger; } @Override @@ -61,27 +69,43 @@ public void createObject(Object entity, RequestScope scope) { @Override public Iterable loadObjects(EntityProjection entityProjection, RequestScope scope) { - Query query = buildQuery(entityProjection, scope); QueryResult result = null; - + QueryResponse response = null; String cacheKey = null; - if (cache != null && !query.isBypassingCache()) { - String tableVersion = queryEngine.getTableVersion(query.getTable(), queryEngineTransaction); - if (tableVersion != null) { - cacheKey = tableVersion + ';' + QueryKeyExtractor.extractKey(query); - result = cache.get(cacheKey); + try { + queryLogger.acceptQuery(scope.getRequestId(), scope.getUser(), scope.getHeaders(), + scope.getApiVersion(), scope.getQueryParams(), scope.getPath()); + Query query = buildQuery(entityProjection, scope); + if (cache != null && !query.isBypassingCache()) { + String tableVersion = queryEngine.getTableVersion(query.getTable(), queryEngineTransaction); + if (tableVersion != null) { + cacheKey = tableVersion + ';' + QueryKeyExtractor.extractKey(query); + result = cache.get(cacheKey); + } } - } - if (result == null) { - result = queryEngine.executeQuery(query, queryEngineTransaction); - if (cacheKey != null) { - cache.put(cacheKey, result); + boolean isCached = result == null ? false : true; + String queryText = queryEngine.explain(query); + queryLogger.processQuery(scope.getRequestId(), query, queryText, isCached); + if (result == null) { + result = queryEngine.executeQuery(query, queryEngineTransaction); + if (cacheKey != null) { + cache.put(cacheKey, result); + } } + if (entityProjection.getPagination() != null && entityProjection.getPagination().returnPageTotals()) { + entityProjection.getPagination().setPageTotals(result.getPageTotals()); + } + response = new QueryResponse(HttpStatus.SC_OK, result.getData(), null); + return result.getData(); + } catch (HttpStatusException e) { + response = new QueryResponse(e.getStatus(), null, e.getMessage()); + throw e; + } catch (Exception e) { + response = new QueryResponse(HttpStatus.SC_INTERNAL_SERVER_ERROR, null, e.getMessage()); + throw e; + } finally { + queryLogger.completeQuery(scope.getRequestId(), response); } - if (entityProjection.getPagination() != null && entityProjection.getPagination().returnPageTotals()) { - entityProjection.getPagination().setPageTotals(result.getPageTotals()); - } - return result.getData(); } @Override @@ -101,7 +125,8 @@ Query buildQuery(EntityProjection entityProjection, RequestScope scope) { } @Override - public void cancel() { + public void cancel(RequestScope scope) { + queryLogger.cancelQuery(scope.getRequestId()); queryEngineTransaction.cancel(); } } diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/QueryEngine.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/QueryEngine.java index 74c41318f6..79d644cb30 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/QueryEngine.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/QueryEngine.java @@ -193,4 +193,11 @@ public interface Transaction extends AutoCloseable { public Table getTable(String classAlias) { return tables.get(classAlias); } + + /** + * Explains the specified query passed in + * @param query The query customized for a particular persistent storage or storage client + * @return SQL string corresponding to the given query + */ + public abstract String explain(Query query); } diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/core/NoopQueryLogger.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/core/NoopQueryLogger.java new file mode 100644 index 0000000000..2a5ef3083a --- /dev/null +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/core/NoopQueryLogger.java @@ -0,0 +1,41 @@ +/* + * Copyright 2020, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ +package com.yahoo.elide.datastores.aggregation.core; + +import com.yahoo.elide.datastores.aggregation.query.Query; +import com.yahoo.elide.security.User; + +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import javax.ws.rs.core.MultivaluedMap; + +/** + * Default NoopQuery Logger Implementation for Elide + */ +public class NoopQueryLogger implements QueryLogger { + + @Override + public void acceptQuery(UUID queryId, User user, Map headers, String apiVer, + Optional> queryParams, String path) { + //does nothing + } + + @Override + public void processQuery(UUID queryId, Query query, String apiQuery, boolean isCached) { + //does nothing + } + + @Override + public void cancelQuery(UUID queryId) { + //does nothing + } + + @Override + public void completeQuery(UUID queryId, QueryResponse response) { + //does nothing + } +} diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/core/QueryLogger.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/core/QueryLogger.java new file mode 100644 index 0000000000..7a2be19d75 --- /dev/null +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/core/QueryLogger.java @@ -0,0 +1,55 @@ +/* + * Copyright 2020, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ +package com.yahoo.elide.datastores.aggregation.core; + +import com.yahoo.elide.datastores.aggregation.query.Query; +import com.yahoo.elide.security.User; + +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import javax.ws.rs.core.MultivaluedMap; + +/** + * Query Logger Interface for Elide + */ +public interface QueryLogger { + + /** + * Accepts the incoming JSON API query and notes the start time for the query + * @param queryId The RequestScope requestId. + * @param user The Principal user + * @param headers Http Request Headers + * @param apiVer API Version + * @param queryParams QueryParams for the incoming JSON API query + * @param path The apiQuery endpoint path for the incoming query + */ + void acceptQuery(UUID queryId, User user, Map headers, String apiVer, + Optional> queryParams, String path); + + /** + * Processes and logs all the queries from QueryDetail + * @param queryId The RequestScope requestId. + * @param query The underlying Query + * @param apiQuery The output querytext + * @param isCached Whether the result came from a cache or not + */ + void processQuery(UUID queryId, Query query, String apiQuery, boolean isCached); + + /** + * Cancels all queries currently running for a particular requestId + * Implementation must be thread-safe. + * @param queryId The RequestScope requestId. + */ + void cancelQuery(UUID queryId); + + /** + * Registers the endtime for a query and logs it out + * @param queryId The RequestScope requestId. + * @param response The ElideResponse object + */ + void completeQuery(UUID queryId, QueryResponse response); +} diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/core/QueryResponse.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/core/QueryResponse.java new file mode 100644 index 0000000000..d08cb6100d --- /dev/null +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/core/QueryResponse.java @@ -0,0 +1,19 @@ +/* + * Copyright 2020, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ +package com.yahoo.elide.datastores.aggregation.core; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +/** + * QueryLogger Response Class + */ +@AllArgsConstructor +public class QueryResponse { + @Getter private final int responseCode; + @Getter private final Iterable data; + @Getter private final String errorMessage; +} diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SQLQueryEngine.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SQLQueryEngine.java index 869de2d9c5..c392cb2717 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SQLQueryEngine.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SQLQueryEngine.java @@ -228,6 +228,11 @@ public String getTableVersion(Table table, Transaction transaction) { return tableVersion; } + @Override + public String explain(Query query) { + return toSQL(query).toString(); + } + /** * Translates the client query into SQL. * diff --git a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/AggregationDataStoreTransactionTest.java b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/AggregationDataStoreTransactionTest.java index 2e83131659..e25e3dcf32 100644 --- a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/AggregationDataStoreTransactionTest.java +++ b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/AggregationDataStoreTransactionTest.java @@ -11,17 +11,23 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.when; import com.yahoo.elide.core.RequestScope; import com.yahoo.elide.core.pagination.PaginationImpl; import com.yahoo.elide.datastores.aggregation.cache.Cache; import com.yahoo.elide.datastores.aggregation.cache.QueryKeyExtractor; +import com.yahoo.elide.datastores.aggregation.core.QueryLogger; +import com.yahoo.elide.datastores.aggregation.core.QueryResponse; import com.yahoo.elide.datastores.aggregation.example.PlayerStats; import com.yahoo.elide.datastores.aggregation.framework.SQLUnitTest; import com.yahoo.elide.datastores.aggregation.query.Query; import com.yahoo.elide.datastores.aggregation.query.QueryResult; +import com.yahoo.elide.datastores.aggregation.queryengines.sql.query.SQLQuery; import com.yahoo.elide.request.EntityProjection; import com.yahoo.elide.request.Pagination; @@ -42,6 +48,7 @@ class AggregationDataStoreTransactionTest extends SQLUnitTest { @Mock private QueryEngine.Transaction qeTransaction; @Mock private RequestScope scope; @Mock private Cache cache; + @Mock private QueryLogger queryLogger; private Query query = Query.builder().table(playerStatsTable).build(); private final String queryKey = QueryKeyExtractor.extractKey(query); @@ -50,8 +57,8 @@ class AggregationDataStoreTransactionTest extends SQLUnitTest { // inject our own query instead of using buildQuery impl private class MyAggregationDataStoreTransaction extends AggregationDataStoreTransaction { - public MyAggregationDataStoreTransaction(QueryEngine queryEngine, Cache cache) { - super(queryEngine, cache); + public MyAggregationDataStoreTransaction(QueryEngine queryEngine, Cache cache, QueryLogger queryLogger) { + super(queryEngine, cache, queryLogger); } @Override @@ -72,10 +79,17 @@ public void setUp() { @Test public void loadObjectsPopulatesCache() { + Mockito.reset(queryLogger); + QueryResult queryResult = QueryResult.builder().data(DATA).build(); + SQLQuery myQuery = SQLQuery.builder().clientQuery(query) + .fromClause(query.getTable().getName()) + .projectionClause(" ").build(); when(queryEngine.getTableVersion(playerStatsTable, qeTransaction)).thenReturn("foo"); when(queryEngine.executeQuery(query, qeTransaction)).thenReturn(queryResult); - AggregationDataStoreTransaction transaction = new MyAggregationDataStoreTransaction(queryEngine, cache); + when(queryEngine.explain(query)).thenReturn(myQuery.toString()); + AggregationDataStoreTransaction transaction = + new MyAggregationDataStoreTransaction(queryEngine, cache, queryLogger); EntityProjection entityProjection = EntityProjection.builder().type(PlayerStats.class).build(); assertEquals(DATA, transaction.loadObjects(entityProjection, scope)); @@ -84,15 +98,29 @@ public void loadObjectsPopulatesCache() { Mockito.verify(cache).get(cacheKey); Mockito.verify(cache).put(cacheKey, queryResult); Mockito.verifyNoMoreInteractions(cache); + Mockito.verify(queryLogger, times(1)).acceptQuery( + Mockito.eq(scope.getRequestId()), + any(), any(), any(), any(), any()); + Mockito.verify(queryLogger, times(1)).processQuery( + Mockito.eq(scope.getRequestId()), any(), any(), Mockito.eq(false)); + Mockito.verify(queryLogger, times(1)).completeQuery( + Mockito.eq(scope.getRequestId()), any()); } @Test public void loadObjectsUsesCache() { + Mockito.reset(queryLogger); + String cacheKey = "foo;" + queryKey; QueryResult queryResult = QueryResult.builder().data(DATA).build(); + SQLQuery myQuery = SQLQuery.builder().clientQuery(query) + .fromClause(query.getTable().getName()) + .projectionClause(" ").build(); when(cache.get(cacheKey)).thenReturn(queryResult); when(queryEngine.getTableVersion(playerStatsTable, qeTransaction)).thenReturn("foo"); - AggregationDataStoreTransaction transaction = new MyAggregationDataStoreTransaction(queryEngine, cache); + when(queryEngine.explain(query)).thenReturn(myQuery.toString()); + AggregationDataStoreTransaction transaction = + new MyAggregationDataStoreTransaction(queryEngine, cache, queryLogger); EntityProjection entityProjection = EntityProjection.builder().type(PlayerStats.class).build(); assertEquals(DATA, transaction.loadObjects(entityProjection, scope)); @@ -100,14 +128,28 @@ public void loadObjectsUsesCache() { Mockito.verify(queryEngine, never()).executeQuery(any(), any()); Mockito.verify(cache).get(cacheKey); Mockito.verifyNoMoreInteractions(cache); + Mockito.verify(queryLogger, times(1)).acceptQuery( + Mockito.eq(scope.getRequestId()), + any(), any(), any(), any(), any()); + Mockito.verify(queryLogger, times(1)).processQuery( + Mockito.eq(scope.getRequestId()), any(), any(), Mockito.eq(true)); + Mockito.verify(queryLogger, times(1)).completeQuery( + Mockito.eq(scope.getRequestId()), any()); } @Test public void loadObjectsPassesPagination() { + Mockito.reset(queryLogger); + QueryResult queryResult = QueryResult.builder().data(DATA).pageTotals(314L).build(); + SQLQuery myQuery = SQLQuery.builder().clientQuery(query) + .fromClause(query.getTable().getName()) + .projectionClause(" ").build(); when(cache.get(anyString())).thenReturn(queryResult); when(queryEngine.getTableVersion(playerStatsTable, qeTransaction)).thenReturn("foo"); - AggregationDataStoreTransaction transaction = new MyAggregationDataStoreTransaction(queryEngine, cache); + when(queryEngine.explain(query)).thenReturn(myQuery.toString()); + AggregationDataStoreTransaction transaction = + new MyAggregationDataStoreTransaction(queryEngine, cache, queryLogger); Pagination pagination = new PaginationImpl( String.class, null, null, DEFAULT_PAGE_LIMIT, MAX_PAGE_LIMIT, true, false); EntityProjection entityProjection = EntityProjection.builder() @@ -120,32 +162,103 @@ public void loadObjectsPassesPagination() { Mockito.verify(queryEngine, never()).executeQuery(any(), any()); Mockito.verify(cache).get(cacheKey); Mockito.verifyNoMoreInteractions(cache); + Mockito.verify(queryLogger, times(1)).acceptQuery( + Mockito.eq(scope.getRequestId()), + any(), any(), any(), any(), any()); + Mockito.verify(queryLogger, times(1)).processQuery( + Mockito.eq(scope.getRequestId()), any(), any(), Mockito.eq(true)); + Mockito.verify(queryLogger, times(1)).completeQuery( + Mockito.eq(scope.getRequestId()), any()); } @Test public void loadObjectsNoTableVersion() { + Mockito.reset(queryLogger); + + SQLQuery myQuery = SQLQuery.builder().clientQuery(query) + .fromClause(query.getTable().getName()) + .projectionClause(" ").build(); when(queryEngine.executeQuery(query, qeTransaction)) .thenReturn(QueryResult.builder().data(Collections.emptyList()).build()); - AggregationDataStoreTransaction transaction = new MyAggregationDataStoreTransaction(queryEngine, cache); + when(queryEngine.explain(query)).thenReturn(myQuery.toString()); + AggregationDataStoreTransaction transaction = + new MyAggregationDataStoreTransaction(queryEngine, cache, queryLogger); EntityProjection entityProjection = EntityProjection.builder().type(PlayerStats.class).build(); transaction.loadObjects(entityProjection, scope); Mockito.verifyNoInteractions(cache); + Mockito.verify(queryLogger, times(1)).acceptQuery( + Mockito.eq(scope.getRequestId()), + any(), any(), any(), any(), any()); + Mockito.verify(queryLogger, times(1)).processQuery( + Mockito.eq(scope.getRequestId()), any(), any(), Mockito.eq(false)); + Mockito.verify(queryLogger, times(1)).completeQuery( + Mockito.eq(scope.getRequestId()), any()); } @Test public void loadObjectsBypassCache() { + Mockito.reset(queryLogger); + query = Query.builder().table(playerStatsTable).bypassingCache(true).build(); + SQLQuery myQuery = SQLQuery.builder().clientQuery(query) + .fromClause(query.getTable().getName()) + .projectionClause(" ").build(); QueryResult queryResult = QueryResult.builder().data(DATA).build(); when(queryEngine.executeQuery(query, qeTransaction)).thenReturn(queryResult); - AggregationDataStoreTransaction transaction = new MyAggregationDataStoreTransaction(queryEngine, cache); + when(queryEngine.explain(query)).thenReturn(myQuery.toString()); + AggregationDataStoreTransaction transaction = + new MyAggregationDataStoreTransaction(queryEngine, cache, queryLogger); EntityProjection entityProjection = EntityProjection.builder().type(PlayerStats.class).build(); assertEquals(DATA, transaction.loadObjects(entityProjection, scope)); Mockito.verify(queryEngine, never()).getTableVersion(any(), any()); Mockito.verifyNoInteractions(cache); + Mockito.verify(queryLogger, times(1)).acceptQuery( + Mockito.eq(scope.getRequestId()), + any(), any(), any(), any(), any()); + Mockito.verify(queryLogger, times(1)).processQuery( + Mockito.eq(scope.getRequestId()), any(), any(), Mockito.eq(false)); + Mockito.verify(queryLogger, times(1)).completeQuery( + Mockito.eq(scope.getRequestId()), any()); + } + + @Test + public void loadObjectsExceptionThrownTest() throws Exception { + Mockito.reset(queryLogger); + String nullPointerExceptionMessage = "Cannot dereference an object with value Null"; + try { + query = Query.builder().table(playerStatsTable).bypassingCache(true).build(); + doThrow(new NullPointerException(nullPointerExceptionMessage)) + .when(queryEngine).executeQuery(query, qeTransaction); + AggregationDataStoreTransaction transaction = + new MyAggregationDataStoreTransaction(queryEngine, cache, queryLogger); + EntityProjection entityProjection = EntityProjection.builder().type(PlayerStats.class).build(); + transaction.loadObjects(entityProjection, scope); + } catch (Exception e) { + assertEquals(nullPointerExceptionMessage, e.getMessage()); + Mockito.verify(queryLogger).completeQuery(Mockito.eq(scope.getRequestId()), + argThat((QueryResponse qResponse) -> qResponse.getErrorMessage() == e.getMessage())); + } + + Mockito.verify(queryLogger, times(1)).acceptQuery( + Mockito.eq(scope.getRequestId()), + any(), any(), any(), any(), any()); + Mockito.verify(queryLogger, times(1)).processQuery( + Mockito.eq(scope.getRequestId()), any(), any(), Mockito.eq(false)); + Mockito.verify(queryLogger, times(1)).completeQuery( + Mockito.eq(scope.getRequestId()), any()); + } + + @Test + public void aggregationQueryLoggerCancelQueryTest() { + Mockito.reset(queryLogger); + AggregationDataStoreTransaction transaction = + new MyAggregationDataStoreTransaction(queryEngine, cache, queryLogger); + transaction.cancel(scope); + Mockito.verify(queryLogger, times(1)).cancelQuery(Mockito.eq(scope.getRequestId())); } } diff --git a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/framework/AggregationDataStoreTestHarness.java b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/framework/AggregationDataStoreTestHarness.java index 44ca681891..51af2d1105 100644 --- a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/framework/AggregationDataStoreTestHarness.java +++ b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/framework/AggregationDataStoreTestHarness.java @@ -8,6 +8,7 @@ import com.yahoo.elide.core.DataStore; import com.yahoo.elide.core.datastore.test.DataStoreTestHarness; import com.yahoo.elide.datastores.aggregation.AggregationDataStore; +import com.yahoo.elide.datastores.aggregation.core.NoopQueryLogger; import com.yahoo.elide.datastores.aggregation.metadata.MetaDataStore; import com.yahoo.elide.datastores.aggregation.queryengines.sql.SQLQueryEngine; import com.yahoo.elide.datastores.jpa.JpaDataStore; @@ -35,6 +36,7 @@ public DataStore getDataStore() { AggregationDataStore aggregationDataStore = AggregationDataStore.builder() .queryEngine(new SQLQueryEngine(metaDataStore, entityManagerFactory, txCancel)) + .queryLogger(new NoopQueryLogger()) .build(); DataStore jpaStore = new JpaDataStore( diff --git a/elide-datastore/elide-datastore-hibernate3/src/main/java/com/yahoo/elide/datastores/hibernate3/HibernateTransaction.java b/elide-datastore/elide-datastore-hibernate3/src/main/java/com/yahoo/elide/datastores/hibernate3/HibernateTransaction.java index a507fffb9e..5d7c7c9636 100644 --- a/elide-datastore/elide-datastore-hibernate3/src/main/java/com/yahoo/elide/datastores/hibernate3/HibernateTransaction.java +++ b/elide-datastore/elide-datastore-hibernate3/src/main/java/com/yahoo/elide/datastores/hibernate3/HibernateTransaction.java @@ -306,7 +306,7 @@ public Integer getQueryLimit() { } @Override - public void cancel() { + public void cancel(RequestScope scope) { session.cancelQuery(); } } diff --git a/elide-datastore/elide-datastore-hibernate5/src/main/java/com/yahoo/elide/datastores/hibernate5/HibernateTransaction.java b/elide-datastore/elide-datastore-hibernate5/src/main/java/com/yahoo/elide/datastores/hibernate5/HibernateTransaction.java index 51c9024af9..5b59f2a18c 100644 --- a/elide-datastore/elide-datastore-hibernate5/src/main/java/com/yahoo/elide/datastores/hibernate5/HibernateTransaction.java +++ b/elide-datastore/elide-datastore-hibernate5/src/main/java/com/yahoo/elide/datastores/hibernate5/HibernateTransaction.java @@ -302,7 +302,7 @@ public void close() throws IOException { } @Override - public void cancel() { + public void cancel(RequestScope scope) { session.cancelQuery(); } } diff --git a/elide-datastore/elide-datastore-jpa/src/main/java/com/yahoo/elide/datastores/jpa/transaction/AbstractJpaTransaction.java b/elide-datastore/elide-datastore-jpa/src/main/java/com/yahoo/elide/datastores/jpa/transaction/AbstractJpaTransaction.java index ca3c3f5a0b..b56804c8b4 100644 --- a/elide-datastore/elide-datastore-jpa/src/main/java/com/yahoo/elide/datastores/jpa/transaction/AbstractJpaTransaction.java +++ b/elide-datastore/elide-datastore-jpa/src/main/java/com/yahoo/elide/datastores/jpa/transaction/AbstractJpaTransaction.java @@ -311,7 +311,7 @@ private Long getTotalRecords(AbstractHQLQueryBuilder.Relationship relationsh } @Override - public void cancel() { + public void cancel(RequestScope scope) { jpaTransactionCancel.accept(em); } } diff --git a/elide-datastore/elide-datastore-multiplex/src/main/java/com/yahoo/elide/datastores/multiplex/MultiplexTransaction.java b/elide-datastore/elide-datastore-multiplex/src/main/java/com/yahoo/elide/datastores/multiplex/MultiplexTransaction.java index a3f2048e72..3209f6d6bf 100644 --- a/elide-datastore/elide-datastore-multiplex/src/main/java/com/yahoo/elide/datastores/multiplex/MultiplexTransaction.java +++ b/elide-datastore/elide-datastore-multiplex/src/main/java/com/yahoo/elide/datastores/multiplex/MultiplexTransaction.java @@ -245,7 +245,7 @@ private Serializable extractId(FilterExpression filterExpression, } @Override - public void cancel() { - transactions.values().forEach(dataStoreTransaction -> dataStoreTransaction.cancel()); + public void cancel(RequestScope scope) { + transactions.values().forEach(dataStoreTransaction -> dataStoreTransaction.cancel(scope)); } } diff --git a/elide-datastore/elide-datastore-multiplex/src/test/java/com/yahoo/elide/datastores/multiplex/TestDataStore.java b/elide-datastore/elide-datastore-multiplex/src/test/java/com/yahoo/elide/datastores/multiplex/TestDataStore.java index d861eaeb18..063fab3841 100644 --- a/elide-datastore/elide-datastore-multiplex/src/test/java/com/yahoo/elide/datastores/multiplex/TestDataStore.java +++ b/elide-datastore/elide-datastore-multiplex/src/test/java/com/yahoo/elide/datastores/multiplex/TestDataStore.java @@ -79,7 +79,7 @@ public Iterable loadObjects( } @Override - public void cancel() { + public void cancel(RequestScope scope) { // Nothing } } diff --git a/elide-datastore/elide-datastore-multiplex/src/test/java/com/yahoo/elide/datastores/multiplex/bridgeable/BridgeableRedisStore.java b/elide-datastore/elide-datastore-multiplex/src/test/java/com/yahoo/elide/datastores/multiplex/bridgeable/BridgeableRedisStore.java index 57bb8141aa..3d0d20e330 100644 --- a/elide-datastore/elide-datastore-multiplex/src/test/java/com/yahoo/elide/datastores/multiplex/bridgeable/BridgeableRedisStore.java +++ b/elide-datastore/elide-datastore-multiplex/src/test/java/com/yahoo/elide/datastores/multiplex/bridgeable/BridgeableRedisStore.java @@ -244,7 +244,7 @@ public T createNewObject(Class entityClass) { } @Override - public void cancel() { + public void cancel(RequestScope scope) { // Nothing } } diff --git a/elide-datastore/elide-datastore-noop/src/main/java/com/yahoo/elide/datastores/noop/NoopTransaction.java b/elide-datastore/elide-datastore-noop/src/main/java/com/yahoo/elide/datastores/noop/NoopTransaction.java index 59ba8c2431..9769e25eba 100644 --- a/elide-datastore/elide-datastore-noop/src/main/java/com/yahoo/elide/datastores/noop/NoopTransaction.java +++ b/elide-datastore/elide-datastore-noop/src/main/java/com/yahoo/elide/datastores/noop/NoopTransaction.java @@ -123,7 +123,7 @@ public void close() throws IOException { * No-op transaction, do nothing. */ @Override - public void cancel() { + public void cancel(RequestScope scope) { // No-op transaction, do nothing. } } diff --git a/elide-spring/elide-spring-boot-autoconfigure/src/main/java/com/yahoo/elide/spring/config/ElideAutoConfiguration.java b/elide-spring/elide-spring-boot-autoconfigure/src/main/java/com/yahoo/elide/spring/config/ElideAutoConfiguration.java index c8ff2a8ec3..e498edf4b9 100644 --- a/elide-spring/elide-spring-boot-autoconfigure/src/main/java/com/yahoo/elide/spring/config/ElideAutoConfiguration.java +++ b/elide-spring/elide-spring-boot-autoconfigure/src/main/java/com/yahoo/elide/spring/config/ElideAutoConfiguration.java @@ -19,6 +19,8 @@ import com.yahoo.elide.datastores.aggregation.QueryEngine; import com.yahoo.elide.datastores.aggregation.cache.Cache; import com.yahoo.elide.datastores.aggregation.cache.CaffeineCache; +import com.yahoo.elide.datastores.aggregation.core.NoopQueryLogger; +import com.yahoo.elide.datastores.aggregation.core.QueryLogger; import com.yahoo.elide.datastores.aggregation.metadata.MetaDataStore; import com.yahoo.elide.datastores.aggregation.queryengines.sql.SQLQueryEngine; import com.yahoo.elide.datastores.aggregation.queryengines.sql.annotation.FromSubquery; @@ -35,6 +37,7 @@ import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.DependsOn; import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.binder.cache.CaffeineCacheMetrics; @@ -180,9 +183,13 @@ public QueryEngine buildQueryEngine(EntityManagerFactory entityManagerFactory, */ @Bean @ConditionalOnMissingBean - public DataStore buildDataStore(EntityManagerFactory entityManagerFactory, QueryEngine queryEngine, - ObjectProvider dynamicCompiler, ElideConfigProperties settings, - @Autowired(required = false) Cache cache) + @DependsOn({"buildQueryLogger"}) + public DataStore buildDataStore(EntityManagerFactory entityManagerFactory, + QueryEngine queryEngine, + ObjectProvider dynamicCompiler, + ElideConfigProperties settings, + @Autowired(required = false) Cache cache, + @Autowired(required = false) QueryLogger querylogger) throws ClassNotFoundException { AggregationDataStore.AggregationDataStoreBuilder aggregationDataStoreBuilder = AggregationDataStore.builder() .queryEngine(queryEngine); @@ -193,6 +200,7 @@ public DataStore buildDataStore(EntityManagerFactory entityManagerFactory, Query aggregationDataStoreBuilder.dynamicCompiledClasses(annotatedClass); } aggregationDataStoreBuilder.cache(cache); + aggregationDataStoreBuilder.queryLogger(querylogger); AggregationDataStore aggregationDataStore = aggregationDataStoreBuilder.build(); JpaDataStore jpaDataStore = new JpaDataStore(entityManagerFactory::createEntityManager, @@ -220,6 +228,16 @@ public Cache buildQueryCache(ElideConfigProperties settings) { return cache; } + /** + * Creates a querylogger to be used by {@link #buildDataStore} for aggregation + * @return The default Noop QueryLogger. + */ + @Bean + @ConditionalOnMissingBean + public QueryLogger buildQueryLogger() { + return new NoopQueryLogger(); + } + /** * Creates a singular swagger document for JSON-API. * @param dictionary Contains the static metadata about Elide models. diff --git a/elide-standalone/src/main/java/com/yahoo/elide/standalone/config/ElideStandaloneSettings.java b/elide-standalone/src/main/java/com/yahoo/elide/standalone/config/ElideStandaloneSettings.java index b108253a78..2ab89deadf 100644 --- a/elide-standalone/src/main/java/com/yahoo/elide/standalone/config/ElideStandaloneSettings.java +++ b/elide-standalone/src/main/java/com/yahoo/elide/standalone/config/ElideStandaloneSettings.java @@ -24,6 +24,7 @@ import com.yahoo.elide.datastores.aggregation.QueryEngine; import com.yahoo.elide.datastores.aggregation.cache.Cache; import com.yahoo.elide.datastores.aggregation.cache.CaffeineCache; +import com.yahoo.elide.datastores.aggregation.core.NoopQueryLogger; import com.yahoo.elide.datastores.aggregation.metadata.MetaDataStore; import com.yahoo.elide.datastores.aggregation.queryengines.sql.SQLQueryEngine; import com.yahoo.elide.datastores.aggregation.queryengines.sql.annotation.FromSubquery; @@ -419,7 +420,7 @@ default DataStore getDataStore(MetaDataStore metaDataStore, AggregationDataStore default AggregationDataStore getAggregationDataStore(QueryEngine queryEngine, Optional optionalCompiler) { AggregationDataStore.AggregationDataStoreBuilder aggregationDataStoreBuilder = AggregationDataStore.builder() - .queryEngine(queryEngine); + .queryEngine(queryEngine).queryLogger(new NoopQueryLogger()); if (enableDynamicModelConfig()) { Set> annotatedClasses = getDynamicClassesIfAvailable(optionalCompiler, FromTable.class);