From 34fbcd282798c6668f61ff35202c3c2aa783857f Mon Sep 17 00:00:00 2001 From: Moiz Arafat Date: Fri, 5 Feb 2021 13:13:44 -0500 Subject: [PATCH] TableExport Spring Controller (#1811) * Export spring controller * review comments * review comments * review comments * review comments * review comments Co-authored-by: Aaron Klish --- .../export/formatter/JSONExportFormatter.java | 27 ++- .../async/operation/TableExportOperation.java | 5 +- .../FileResultStorageEngine.java | 13 +- .../storageengine/ResultStorageEngine.java | 2 + .../formatter/CSVExportFormatterTest.java | 39 +++- .../formatter/JSONExportFormatterTest.java | 42 +++- .../yahoo/elide/core/PersistentResource.java | 15 +- .../elide/core/exceptions/HttpStatus.java | 1 + .../utils/coerce/converters/URLSerde.java | 31 +++ .../elide/core/utils/ClassScannerTest.java | 2 +- .../utils/coerce/converters/URLSerdeTest.java | 35 ++++ .../config/ElideAsyncConfiguration.java | 8 +- .../config/ExportControllerProperties.java | 5 + .../spring/controllers/ExportController.java | 107 ++++++++++ .../main/resources/META-INF/spring.factories | 3 +- .../example/models/aggregation/Stats.java | 3 + .../example/tests/AggregationStoreTest.java | 22 +- .../test/java/example/tests/AsyncTest.java | 191 +++++++++++++++++- .../java/example/tests/ControllerTest.java | 26 ++- .../tests/DisableAggStoreAsyncTest.java | 110 +++++++++- .../tests/DisableAggStoreControllerTest.java | 8 +- .../DisableMetaDataStoreControllerTest.java | 9 +- .../java/example/tests/DynamicConfigTest.java | 23 +-- .../java/example/tests/IntegrationTest.java | 1 + .../application-disableAggStore.yaml | 43 ---- .../application-disableMetaDataStore.yaml | 45 ----- .../src/test/resources/application.yaml | 3 + .../src/test/resources/db/test_init.sql | 31 +++ 28 files changed, 680 insertions(+), 170 deletions(-) create mode 100644 elide-core/src/main/java/com/yahoo/elide/core/utils/coerce/converters/URLSerde.java create mode 100644 elide-core/src/test/java/com/yahoo/elide/core/utils/coerce/converters/URLSerdeTest.java create mode 100644 elide-spring/elide-spring-boot-autoconfigure/src/main/java/com/yahoo/elide/spring/controllers/ExportController.java delete mode 100644 elide-spring/elide-spring-boot-autoconfigure/src/test/resources/application-disableAggStore.yaml delete mode 100644 elide-spring/elide-spring-boot-autoconfigure/src/test/resources/application-disableMetaDataStore.yaml create mode 100644 elide-spring/elide-spring-boot-autoconfigure/src/test/resources/db/test_init.sql diff --git a/elide-async/src/main/java/com/yahoo/elide/async/export/formatter/JSONExportFormatter.java b/elide-async/src/main/java/com/yahoo/elide/async/export/formatter/JSONExportFormatter.java index a1da667032..09c632ca51 100644 --- a/elide-async/src/main/java/com/yahoo/elide/async/export/formatter/JSONExportFormatter.java +++ b/elide-async/src/main/java/com/yahoo/elide/async/export/formatter/JSONExportFormatter.java @@ -8,12 +8,20 @@ import com.yahoo.elide.Elide; import com.yahoo.elide.async.models.TableExport; import com.yahoo.elide.core.PersistentResource; +import com.yahoo.elide.core.request.Attribute; import com.yahoo.elide.core.request.EntityProjection; +import com.yahoo.elide.jsonapi.models.Resource; + import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import lombok.extern.slf4j.Slf4j; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; + /** * JSON output format implementation. */ @@ -48,9 +56,10 @@ public static String resourceToJSON(ObjectMapper mapper, PersistentResource reso } StringBuilder str = new StringBuilder(); - try { - str.append(mapper.writeValueAsString(resource.getObject())); + Resource jsonResource = resource.toResource(getRelationships(resource), getAttributes(resource)); + + str.append(mapper.writeValueAsString(jsonResource.getAttributes())); } catch (JsonProcessingException e) { log.error("Exception when converting to JSON {}", e.getMessage()); throw new IllegalStateException(e); @@ -58,6 +67,20 @@ public static String resourceToJSON(ObjectMapper mapper, PersistentResource reso return str.toString(); } + protected static Map getAttributes(PersistentResource resource) { + final Map attributes = new LinkedHashMap<>(); + final Set attrFields = resource.getRequestScope().getEntityProjection().getAttributes(); + + for (Attribute field : attrFields) { + attributes.put(field.getName(), resource.getAttribute(field)); + } + return attributes; + } + + protected static Map getRelationships(PersistentResource resource) { + return Collections.emptyMap(); + } + @Override public String preFormat(EntityProjection projection, TableExport query) { return "["; diff --git a/elide-async/src/main/java/com/yahoo/elide/async/operation/TableExportOperation.java b/elide-async/src/main/java/com/yahoo/elide/async/operation/TableExportOperation.java index cead914f5b..1f271cb851 100644 --- a/elide-async/src/main/java/com/yahoo/elide/async/operation/TableExportOperation.java +++ b/elide-async/src/main/java/com/yahoo/elide/async/operation/TableExportOperation.java @@ -81,9 +81,9 @@ public AsyncAPIResult call() { exportResult.setUrl(new URL(generateDownloadURL(exportObj, (RequestScope) scope))); exportResult.setRecordCount(recordNumber); } catch (BadRequestException e) { - exportResult.setMessage("Download url generation failure."); - } catch (MalformedURLException e) { exportResult.setMessage("EntityProjection generation failure."); + } catch (MalformedURLException e) { + exportResult.setMessage("Download url generation failure."); } catch (Exception e) { exportResult.setMessage(e.getMessage()); } finally { @@ -125,6 +125,7 @@ public Observable export(TableExport exportObj, RequestScope //TODO - Can we have projectionInfo as null? RequestScope exportRequestScope = getRequestScope(exportObj, prevScope.getUser(), prevScope.getApiVersion(), tx); + exportRequestScope.setEntityProjection(projection); if (projection != null) { results = PersistentResource.loadRecords(projection, Collections.emptyList(), exportRequestScope); diff --git a/elide-async/src/main/java/com/yahoo/elide/async/service/storageengine/FileResultStorageEngine.java b/elide-async/src/main/java/com/yahoo/elide/async/service/storageengine/FileResultStorageEngine.java index de16c32bed..67523516ba 100644 --- a/elide-async/src/main/java/com/yahoo/elide/async/service/storageengine/FileResultStorageEngine.java +++ b/elide-async/src/main/java/com/yahoo/elide/async/service/storageengine/FileResultStorageEngine.java @@ -31,9 +31,6 @@ public class FileResultStorageEngine implements ResultStorageEngine { @Setter private String basePath; - public FileResultStorageEngine() { - } - /** * Constructor. * @param basePath basePath for storing the files. Can be absolute or relative. @@ -55,14 +52,14 @@ public TableExport storeResults(TableExport tableExport, Observable resu writer.flush(); }, throwable -> { - throw new IllegalStateException(throwable); + throw new IllegalStateException(STORE_ERROR, throwable); }, () -> { writer.flush(); } ); } catch (IOException e) { - throw new IllegalStateException(e); + throw new IllegalStateException(STORE_ERROR, e); } return tableExport; @@ -85,7 +82,7 @@ public boolean hasNext() { record = reader.readLine(); return record != null; } catch (IOException e) { - throw new IllegalStateException(e); + throw new IllegalStateException(RETRIEVE_ERROR, e); } } @@ -106,7 +103,7 @@ private BufferedReader getReader(String asyncQueryID) { return Files.newBufferedReader(Paths.get(basePath + File.separator + asyncQueryID)); } catch (IOException e) { log.debug(e.getMessage()); - throw new IllegalStateException("Unable to retrieve results."); + throw new IllegalStateException(RETRIEVE_ERROR, e); } } @@ -115,7 +112,7 @@ private BufferedWriter getWriter(String asyncQueryID) { return Files.newBufferedWriter(Paths.get(basePath + File.separator + asyncQueryID)); } catch (IOException e) { log.debug(e.getMessage()); - throw new IllegalStateException("Unable to store results."); + throw new IllegalStateException(STORE_ERROR, e); } } } diff --git a/elide-async/src/main/java/com/yahoo/elide/async/service/storageengine/ResultStorageEngine.java b/elide-async/src/main/java/com/yahoo/elide/async/service/storageengine/ResultStorageEngine.java index 121e5c1613..df2595ce28 100644 --- a/elide-async/src/main/java/com/yahoo/elide/async/service/storageengine/ResultStorageEngine.java +++ b/elide-async/src/main/java/com/yahoo/elide/async/service/storageengine/ResultStorageEngine.java @@ -13,6 +13,8 @@ * Utility interface used for storing the results of AsyncQuery for downloads. */ public interface ResultStorageEngine { + public static final String RETRIEVE_ERROR = "Unable to retrieve results."; + public static final String STORE_ERROR = "Unable to store results."; /** * Stores the result of the query. diff --git a/elide-async/src/test/java/com/yahoo/elide/async/export/formatter/CSVExportFormatterTest.java b/elide-async/src/test/java/com/yahoo/elide/async/export/formatter/CSVExportFormatterTest.java index 50f23ef747..34391c96f7 100644 --- a/elide-async/src/test/java/com/yahoo/elide/async/export/formatter/CSVExportFormatterTest.java +++ b/elide-async/src/test/java/com/yahoo/elide/async/export/formatter/CSVExportFormatterTest.java @@ -6,6 +6,7 @@ package com.yahoo.elide.async.export.formatter; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -15,11 +16,13 @@ import com.yahoo.elide.async.models.ResultType; import com.yahoo.elide.async.models.TableExport; import com.yahoo.elide.core.PersistentResource; +import com.yahoo.elide.core.RequestScope; import com.yahoo.elide.core.datastore.inmemory.HashMapDataStore; import com.yahoo.elide.core.dictionary.EntityDictionary; import com.yahoo.elide.core.request.Attribute; import com.yahoo.elide.core.request.EntityProjection; import com.yahoo.elide.core.security.checks.Check; +import com.yahoo.elide.jsonapi.models.Resource; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -28,6 +31,7 @@ import java.nio.file.Path; import java.text.SimpleDateFormat; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.Map; import java.util.Set; @@ -38,6 +42,7 @@ public class CSVExportFormatterTest { private static final SimpleDateFormat FORMATTER = new SimpleDateFormat(FORMAT); private HashMapDataStore dataStore; private Elide elide; + private RequestScope scope; @BeforeEach public void setupMocks(@TempDir Path tempDir) { @@ -49,28 +54,44 @@ public void setupMocks(@TempDir Path tempDir) { .withISO8601Dates("yyyy-MM-dd'T'HH:mm'Z'", TimeZone.getTimeZone("UTC")) .build()); FORMATTER.setTimeZone(TimeZone.getTimeZone("GMT")); + scope = mock(RequestScope.class); } @Test public void testResourceToCSV() { CSVExportFormatter formatter = new CSVExportFormatter(elide, false); TableExport queryObj = new TableExport(); - String query = "{ tableExport { edges { node { id query queryType requestId principalName status createdOn updatedOn" - + " asyncAfterSeconds resultType result} } } }"; + String query = "{ tableExport { edges { node { query queryType createdOn} } } }"; String id = "edc4a871-dff2-4054-804e-d80075cf827d"; queryObj.setId(id); queryObj.setQuery(query); queryObj.setQueryType(QueryType.GRAPHQL_V1_0); queryObj.setResultType(ResultType.CSV); - String row = "\"edc4a871-dff2-4054-804e-d80075cf827d\", \"{ tableExport { edges { node { id query queryType requestId" - + " principalName status createdOn updatedOn asyncAfterSeconds resultType result} } } }\", \"GRAPHQL_V1_0\"" - + ", \"" + queryObj.getRequestId() + "\", " + "null" + ", \"" + queryObj.getStatus() + "\", \"" + FORMATTER.format(queryObj.getCreatedOn()) - + "\", \"" + FORMATTER.format(queryObj.getUpdatedOn()) + "\", " + "10.0, \"CSV\", null"; + String row = "\"{ tableExport { edges { node { query queryType createdOn} } } }\", \"GRAPHQL_V1_0\"" + + ", \"" + FORMATTER.format(queryObj.getCreatedOn()); - PersistentResource resource = mock(PersistentResource.class); - when(resource.getObject()).thenReturn(queryObj); - String output = formatter.format(resource, 1); + // Prepare EntityProjection + Set attributes = new LinkedHashSet(); + attributes.add(Attribute.builder().type(TableExport.class).name("query").alias("query").build()); + attributes.add(Attribute.builder().type(TableExport.class).name("queryType").build()); + attributes.add(Attribute.builder().type(TableExport.class).name("createdOn").build()); + EntityProjection projection = EntityProjection.builder().type(TableExport.class).attributes(attributes).build(); + + Map resourceAttributes = new LinkedHashMap<>(); + resourceAttributes.put("query", query); + resourceAttributes.put("queryType", queryObj.getQueryType()); + resourceAttributes.put("createdOn", queryObj.getCreatedOn()); + + Resource resource = new Resource("tableExport", "0", resourceAttributes, null, null, null); + //Resource(type=stats, id=0, attributes={dimension=Bar, measure=150}, relationships=null, links=null, meta=null) + PersistentResource persistentResource = mock(PersistentResource.class); + when(persistentResource.getObject()).thenReturn(queryObj); + when(persistentResource.getRequestScope()).thenReturn(scope); + when(persistentResource.toResource(any(), any())).thenReturn(resource); + when(scope.getEntityProjection()).thenReturn(projection); + + String output = formatter.format(persistentResource, 1); assertEquals(true, output.contains(row)); } diff --git a/elide-async/src/test/java/com/yahoo/elide/async/export/formatter/JSONExportFormatterTest.java b/elide-async/src/test/java/com/yahoo/elide/async/export/formatter/JSONExportFormatterTest.java index 973123d7e8..8d1b8d7cdc 100644 --- a/elide-async/src/test/java/com/yahoo/elide/async/export/formatter/JSONExportFormatterTest.java +++ b/elide-async/src/test/java/com/yahoo/elide/async/export/formatter/JSONExportFormatterTest.java @@ -6,6 +6,7 @@ package com.yahoo.elide.async.export.formatter; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -15,9 +16,13 @@ import com.yahoo.elide.async.models.ResultType; import com.yahoo.elide.async.models.TableExport; import com.yahoo.elide.core.PersistentResource; +import com.yahoo.elide.core.RequestScope; import com.yahoo.elide.core.datastore.inmemory.HashMapDataStore; import com.yahoo.elide.core.dictionary.EntityDictionary; +import com.yahoo.elide.core.request.Attribute; +import com.yahoo.elide.core.request.EntityProjection; import com.yahoo.elide.core.security.checks.Check; +import com.yahoo.elide.jsonapi.models.Resource; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -26,7 +31,10 @@ import java.nio.file.Path; import java.text.SimpleDateFormat; import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.Map; +import java.util.Set; import java.util.TimeZone; public class JSONExportFormatterTest { @@ -34,6 +42,7 @@ public class JSONExportFormatterTest { private static final SimpleDateFormat FORMATTER = new SimpleDateFormat(FORMAT); private HashMapDataStore dataStore; private Elide elide; + private RequestScope scope; @BeforeEach public void setupMocks(@TempDir Path tempDir) { @@ -45,28 +54,43 @@ public void setupMocks(@TempDir Path tempDir) { .withISO8601Dates("yyyy-MM-dd'T'HH:mm'Z'", TimeZone.getTimeZone("UTC")) .build()); FORMATTER.setTimeZone(TimeZone.getTimeZone("GMT")); + scope = mock(RequestScope.class); } @Test public void testResourceToJSON() { JSONExportFormatter formatter = new JSONExportFormatter(elide); TableExport queryObj = new TableExport(); - String query = "/tableExport"; + String query = "{ tableExport { edges { node { query queryType createdOn} } } }"; String id = "edc4a871-dff2-4054-804e-d80075cf827d"; queryObj.setId(id); queryObj.setQuery(query); queryObj.setQueryType(QueryType.GRAPHQL_V1_0); queryObj.setResultType(ResultType.CSV); - String start = "{\"id\":\"edc4a871-dff2-4054-804e-d80075cf827d\",\"query\":\"/tableExport\"," - + "\"queryType\":\"GRAPHQL_V1_0\",\"requestId\":\"" + queryObj.getRequestId() + "\",\"principalName\":null" - + ",\"status\":\"" + queryObj.getStatus() + "\",\"createdOn\":\"" + FORMATTER.format(queryObj.getCreatedOn()) - + "\",\"updatedOn\":\"" + FORMATTER.format(queryObj.getUpdatedOn()) + "\",\"asyncAfterSeconds\":10,\"resultType\":\"CSV\"," - + "\"result\":null}"; + String start = "{\"query\":\"{ tableExport { edges { node { query queryType createdOn} } } }\"," + + "\"queryType\":\"GRAPHQL_V1_0\",\"createdOn\":\"" + FORMATTER.format(queryObj.getCreatedOn()) + "\"}"; - PersistentResource resource = mock(PersistentResource.class); - when(resource.getObject()).thenReturn(queryObj); - String output = formatter.format(resource, 1); + // Prepare EntityProjection + Set attributes = new LinkedHashSet(); + attributes.add(Attribute.builder().type(TableExport.class).name("query").alias("query").build()); + attributes.add(Attribute.builder().type(TableExport.class).name("queryType").build()); + attributes.add(Attribute.builder().type(TableExport.class).name("createdOn").build()); + EntityProjection projection = EntityProjection.builder().type(TableExport.class).attributes(attributes).build(); + + Map resourceAttributes = new LinkedHashMap<>(); + resourceAttributes.put("query", query); + resourceAttributes.put("queryType", queryObj.getQueryType()); + resourceAttributes.put("createdOn", queryObj.getCreatedOn()); + + Resource resource = new Resource("tableExport", "0", resourceAttributes, null, null, null); + PersistentResource persistentResource = mock(PersistentResource.class); + when(persistentResource.getObject()).thenReturn(queryObj); + when(persistentResource.getRequestScope()).thenReturn(scope); + when(persistentResource.toResource(any(), any())).thenReturn(resource); + when(scope.getEntityProjection()).thenReturn(projection); + + String output = formatter.format(persistentResource, 1); assertEquals(true, output.contains(start)); } } diff --git a/elide-core/src/main/java/com/yahoo/elide/core/PersistentResource.java b/elide-core/src/main/java/com/yahoo/elide/core/PersistentResource.java index b050127727..5f32d1e63b 100644 --- a/elide-core/src/main/java/com/yahoo/elide/core/PersistentResource.java +++ b/elide-core/src/main/java/com/yahoo/elide/core/PersistentResource.java @@ -1342,12 +1342,23 @@ public Resource toResource(EntityProjection projection) { */ private Resource toResource(final Supplier> relationshipSupplier, final Supplier> attributeSupplier) { + return toResource(relationshipSupplier.get(), attributeSupplier.get()); + } + + /** + * Convert a persistent resource to a resource. + * @param relationships The relationships + * @param attributes The attributes + * @return The Resource + */ + public Resource toResource(final Map relationships, + final Map attributes) { final Resource resource = new Resource(typeName, (obj == null) ? uuid.orElseThrow( () -> new InvalidEntityBodyException("No id found on object")) : dictionary.getId(obj)); - resource.setRelationships(relationshipSupplier.get()); - resource.setAttributes(attributeSupplier.get()); + resource.setRelationships(relationships); + resource.setAttributes(attributes); if (requestScope.getElideSettings().isEnableJsonLinks()) { resource.setLinks(requestScope.getElideSettings().getJsonApiLinks().getResourceLevelLinks(this)); } diff --git a/elide-core/src/main/java/com/yahoo/elide/core/exceptions/HttpStatus.java b/elide-core/src/main/java/com/yahoo/elide/core/exceptions/HttpStatus.java index b92cb33744..7d94083993 100644 --- a/elide-core/src/main/java/com/yahoo/elide/core/exceptions/HttpStatus.java +++ b/elide-core/src/main/java/com/yahoo/elide/core/exceptions/HttpStatus.java @@ -16,6 +16,7 @@ public class HttpStatus { public static final int SC_BAD_REQUEST = 400; public static final int SC_FORBIDDEN = 403; public static final int SC_NOT_FOUND = 404; + public static final int SC_METHOD_NOT_ALLOWED = 405; public static final int SC_TIMEOUT = 408; public static final int SC_LOCKED = 423; public static final int SC_INTERNAL_SERVER_ERROR = 500; diff --git a/elide-core/src/main/java/com/yahoo/elide/core/utils/coerce/converters/URLSerde.java b/elide-core/src/main/java/com/yahoo/elide/core/utils/coerce/converters/URLSerde.java new file mode 100644 index 0000000000..127986f926 --- /dev/null +++ b/elide-core/src/main/java/com/yahoo/elide/core/utils/coerce/converters/URLSerde.java @@ -0,0 +1,31 @@ +/* + * Copyright 2021, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ +package com.yahoo.elide.core.utils.coerce.converters; + +import com.yahoo.elide.core.exceptions.InvalidValueException; + +import java.net.MalformedURLException; +import java.net.URL; + +@ElideTypeConverter(type = URL.class, name = "URL") +public class URLSerde implements Serde { + + @Override + public URL deserialize(String val) { + URL url; + try { + url = new URL(val); + } catch (MalformedURLException e) { + throw new InvalidValueException("Invalid URL " + val); + } + return url; + } + + @Override + public String serialize(URL val) { + return val.toString(); + } +} diff --git a/elide-core/src/test/java/com/yahoo/elide/core/utils/ClassScannerTest.java b/elide-core/src/test/java/com/yahoo/elide/core/utils/ClassScannerTest.java index 80882eb182..767c545d47 100644 --- a/elide-core/src/test/java/com/yahoo/elide/core/utils/ClassScannerTest.java +++ b/elide-core/src/test/java/com/yahoo/elide/core/utils/ClassScannerTest.java @@ -18,7 +18,7 @@ public class ClassScannerTest { @Test public void testGetAllClasses() { Set> classes = ClassScanner.getAllClasses("com.yahoo.elide.core.utils"); - assertEquals(29, classes.size()); + assertEquals(31, classes.size()); assertTrue(classes.contains(ClassScannerTest.class)); } diff --git a/elide-core/src/test/java/com/yahoo/elide/core/utils/coerce/converters/URLSerdeTest.java b/elide-core/src/test/java/com/yahoo/elide/core/utils/coerce/converters/URLSerdeTest.java new file mode 100644 index 0000000000..1e1de9b294 --- /dev/null +++ b/elide-core/src/test/java/com/yahoo/elide/core/utils/coerce/converters/URLSerdeTest.java @@ -0,0 +1,35 @@ +/* + * Copyright 2021, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ +package com.yahoo.elide.core.utils.coerce.converters; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +import java.net.MalformedURLException; +import java.net.URL; + +public class URLSerdeTest { + + @Test + public void testURLSerialize() throws MalformedURLException { + + URL url = new URL("https://elide.io"); + String expected = "https://elide.io"; + URLSerde urlSerde = new URLSerde(); + Object actual = urlSerde.serialize(url); + assertEquals(expected, actual); + } + + @Test + public void testURLDeserialize() throws MalformedURLException { + URL expectedURL = new URL("https://elide.io"); + String actual = "https://elide.io"; + URLSerde urlSerde = new URLSerde(); + Object actualURL = urlSerde.deserialize(actual); + assertEquals(expectedURL, actualURL); + } +} diff --git a/elide-spring/elide-spring-boot-autoconfigure/src/main/java/com/yahoo/elide/spring/config/ElideAsyncConfiguration.java b/elide-spring/elide-spring-boot-autoconfigure/src/main/java/com/yahoo/elide/spring/config/ElideAsyncConfiguration.java index 9bc1006f55..d5da6c9e5f 100644 --- a/elide-spring/elide-spring-boot-autoconfigure/src/main/java/com/yahoo/elide/spring/config/ElideAsyncConfiguration.java +++ b/elide-spring/elide-spring-boot-autoconfigure/src/main/java/com/yahoo/elide/spring/config/ElideAsyncConfiguration.java @@ -24,6 +24,7 @@ import com.yahoo.elide.async.service.AsyncExecutorService; import com.yahoo.elide.async.service.dao.AsyncAPIDAO; import com.yahoo.elide.async.service.dao.DefaultAsyncAPIDAO; +import com.yahoo.elide.async.service.storageengine.FileResultStorageEngine; import com.yahoo.elide.async.service.storageengine.ResultStorageEngine; import com.yahoo.elide.core.dictionary.EntityDictionary; import com.yahoo.elide.core.exceptions.InvalidOperationException; @@ -153,10 +154,11 @@ public AsyncAPIDAO buildAsyncAPIDAO(Elide elide) { */ @Bean @ConditionalOnMissingBean - @ConditionalOnProperty(prefix = "elide.async.download", name = "enabled", matchIfMissing = false) + @ConditionalOnProperty(prefix = "elide.async.export", name = "enabled", matchIfMissing = false) public ResultStorageEngine buildResultStorageEngine(Elide elide, ElideConfigProperties settings, AsyncAPIDAO asyncQueryDAO) { - // TODO: Initialize with FileResultStorageEngine - return null; + FileResultStorageEngine resultStorageEngine = new FileResultStorageEngine(settings.getAsync().getExport() + .getStorageDestination()); + return resultStorageEngine; } } diff --git a/elide-spring/elide-spring-boot-autoconfigure/src/main/java/com/yahoo/elide/spring/config/ExportControllerProperties.java b/elide-spring/elide-spring-boot-autoconfigure/src/main/java/com/yahoo/elide/spring/config/ExportControllerProperties.java index 7505eb0ba8..c8065a523e 100644 --- a/elide-spring/elide-spring-boot-autoconfigure/src/main/java/com/yahoo/elide/spring/config/ExportControllerProperties.java +++ b/elide-spring/elide-spring-boot-autoconfigure/src/main/java/com/yahoo/elide/spring/config/ExportControllerProperties.java @@ -22,4 +22,9 @@ public class ExportControllerProperties extends ControllerProperties { * The URL path prefix for the controller. */ private String path = "/export"; + + /** + * Storage engine destination . + */ + private String storageDestination = "/tmp"; } diff --git a/elide-spring/elide-spring-boot-autoconfigure/src/main/java/com/yahoo/elide/spring/controllers/ExportController.java b/elide-spring/elide-spring-boot-autoconfigure/src/main/java/com/yahoo/elide/spring/controllers/ExportController.java new file mode 100644 index 0000000000..0df152cb15 --- /dev/null +++ b/elide-spring/elide-spring-boot-autoconfigure/src/main/java/com/yahoo/elide/spring/controllers/ExportController.java @@ -0,0 +1,107 @@ +/* + * Copyright 2020, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ +package com.yahoo.elide.spring.controllers; + +import com.yahoo.elide.async.service.storageengine.ResultStorageEngine; +import com.yahoo.elide.core.exceptions.HttpStatus; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.context.annotation.Configuration; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody; + +import io.reactivex.Observable; +import lombok.extern.slf4j.Slf4j; + +import java.io.IOException; + +import javax.servlet.http.HttpServletResponse; + +/** + * Spring rest controller for Elide Export. + * When enabled it is highly recommended to + * configure explicitly the TaskExecutor used in Spring MVC for executing + * asynchronous requests using StreamingResponseBody. Refer + * {@link org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody}. + */ +@Slf4j +@Configuration +@RestController +@RequestMapping(value = "${elide.async.export.path:/export}") +@ConditionalOnExpression("${elide.async.export.enabled:false}") +public class ExportController { + + private ResultStorageEngine resultStorageEngine; + + @Autowired + public ExportController(ResultStorageEngine resultStorageEngine) { + log.debug("Started ~~"); + this.resultStorageEngine = resultStorageEngine; + } + + /** + * Single entry point for export requests. + * @param asyncQueryId Id of results to download + * @param response HttpServletResponse instance + * @return ResponseEntity + */ + @GetMapping(path = "/{asyncQueryId}") + public ResponseEntity export(@PathVariable String asyncQueryId, + HttpServletResponse response) { + + Observable observableResults = resultStorageEngine.getResultsByID(asyncQueryId); + StreamingResponseBody streamingOutput = outputStream -> { + observableResults + .subscribe( + resultString -> { + outputStream.write(resultString.concat(System.getProperty("line.separator")).getBytes()); + }, + error -> { + String message = error.getMessage(); + try { + log.debug(message); + if (message != null && message.equals(ResultStorageEngine.RETRIEVE_ERROR)) { + response.sendError(HttpStatus.SC_NOT_FOUND, asyncQueryId + "not found"); + } else { + response.sendError(HttpStatus.SC_INTERNAL_SERVER_ERROR); + } + } catch (IOException | IllegalStateException e) { + // If stream was flushed, Attachment download has already started. + // response.sendError causes java.lang.IllegalStateException: + // Cannot call sendError() after the response has been committed. + // This will return 200 status. + // Add error message in the attachment as a way to signal errors. + outputStream.write( + "Error Occured...." + .concat(System.getProperty("line.separator")) + .getBytes() + ); + log.debug(e.getMessage()); + } finally { + outputStream.flush(); + outputStream.close(); + } + }, + () -> { + outputStream.flush(); + outputStream.close(); + } + ); + }; + + return ResponseEntity + .ok() + .header("Content-Disposition", "attachment; filename=" + asyncQueryId) + .contentType(MediaType.APPLICATION_OCTET_STREAM) + .body(streamingOutput); + } +} diff --git a/elide-spring/elide-spring-boot-autoconfigure/src/main/resources/META-INF/spring.factories b/elide-spring/elide-spring-boot-autoconfigure/src/main/resources/META-INF/spring.factories index 89e2683d18..ea711bf7a4 100644 --- a/elide-spring/elide-spring-boot-autoconfigure/src/main/resources/META-INF/spring.factories +++ b/elide-spring/elide-spring-boot-autoconfigure/src/main/resources/META-INF/spring.factories @@ -4,4 +4,5 @@ org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ com.yahoo.elide.spring.config.ElideDynamicConfiguration, \ com.yahoo.elide.spring.controllers.JsonApiController, \ com.yahoo.elide.spring.controllers.GraphqlController, \ - com.yahoo.elide.spring.controllers.SwaggerController + com.yahoo.elide.spring.controllers.SwaggerController, \ + com.yahoo.elide.spring.controllers.ExportController diff --git a/elide-spring/elide-spring-boot-autoconfigure/src/test/java/example/models/aggregation/Stats.java b/elide-spring/elide-spring-boot-autoconfigure/src/test/java/example/models/aggregation/Stats.java index 85bcc5da58..2d72a77253 100644 --- a/elide-spring/elide-spring-boot-autoconfigure/src/test/java/example/models/aggregation/Stats.java +++ b/elide-spring/elide-spring-boot-autoconfigure/src/test/java/example/models/aggregation/Stats.java @@ -9,6 +9,8 @@ import com.yahoo.elide.datastores.aggregation.annotation.MetricFormula; import com.yahoo.elide.datastores.aggregation.queryengines.sql.annotation.FromTable; import com.yahoo.elide.datastores.aggregation.queryengines.sql.annotation.VersionQuery; + +import lombok.Data; import lombok.EqualsAndHashCode; import lombok.ToString; @@ -19,6 +21,7 @@ @ToString @FromTable(name = "stats") @VersionQuery(sql = "SELECT COUNT(*) FROM stats") +@Data public class Stats { /** diff --git a/elide-spring/elide-spring-boot-autoconfigure/src/test/java/example/tests/AggregationStoreTest.java b/elide-spring/elide-spring-boot-autoconfigure/src/test/java/example/tests/AggregationStoreTest.java index 7f32471373..6d292f93c0 100644 --- a/elide-spring/elide-spring-boot-autoconfigure/src/test/java/example/tests/AggregationStoreTest.java +++ b/elide-spring/elide-spring-boot-autoconfigure/src/test/java/example/tests/AggregationStoreTest.java @@ -29,18 +29,19 @@ /** * Example functional tests for Aggregation Store. */ +@Sql( + executionPhase = Sql.ExecutionPhase.BEFORE_TEST_METHOD, + scripts = "classpath:db/test_init.sql", + statements = { + "INSERT INTO Stats (id, measure, dimension) VALUES\n" + + "\t\t(1,100,'Foo')," + + "\t\t(2,150,'Bar');" +}) public class AggregationStoreTest extends IntegrationTest { /** * This test demonstrates an example test using the aggregation store. */ @Test - @Sql(statements = { - "DROP TABLE Stats IF EXISTS;", - "CREATE TABLE Stats(id int, measure int, dimension VARCHAR(255));", - "INSERT INTO Stats (id, measure, dimension) VALUES\n" - + "\t\t(1,100,'Foo')," - + "\t\t(2,150,'Bar');" - }) public void jsonApiGetTestNoHeader(@Autowired MeterRegistry metrics) { when() .get("/json/stats?fields[stats]=measure") @@ -82,13 +83,6 @@ public void jsonApiGetTestNoHeader(@Autowired MeterRegistry metrics) { * This test demonstrates an example test using the aggregation store. */ @Test - @Sql(statements = { - "DROP TABLE Stats IF EXISTS;", - "CREATE TABLE Stats(id int, measure int, dimension VARCHAR(255));", - "INSERT INTO Stats (id, measure, dimension) VALUES\n" - + "\t\t(1,100,'Foo')," - + "\t\t(2,150,'Bar');" - }) public void jsonApiGetTest(@Autowired MeterRegistry metrics) { Map requestHeaders = new HashMap<>(); requestHeaders.put("bypassCache", "true"); diff --git a/elide-spring/elide-spring-boot-autoconfigure/src/test/java/example/tests/AsyncTest.java b/elide-spring/elide-spring-boot-autoconfigure/src/test/java/example/tests/AsyncTest.java index ac76038c7f..98760730e1 100644 --- a/elide-spring/elide-spring-boot-autoconfigure/src/test/java/example/tests/AsyncTest.java +++ b/elide-spring/elide-spring-boot-autoconfigure/src/test/java/example/tests/AsyncTest.java @@ -13,14 +13,17 @@ import static com.yahoo.elide.test.jsonapi.JsonApiDSL.resource; import static com.yahoo.elide.test.jsonapi.JsonApiDSL.type; import static io.restassured.RestAssured.given; +import static io.restassured.RestAssured.when; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.notNullValue; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.fail; import com.yahoo.elide.core.exceptions.HttpStatus; + import org.junit.jupiter.api.Test; import org.springframework.test.context.jdbc.Sql; import org.springframework.test.context.jdbc.SqlMergeMode; + import io.restassured.response.Response; import javax.ws.rs.core.MediaType; @@ -29,11 +32,20 @@ * Basic functional tests to test Async service setup, JSONAPI and GRAPHQL endpoints. */ @SqlMergeMode(SqlMergeMode.MergeMode.MERGE) -@Sql(executionPhase = Sql.ExecutionPhase.BEFORE_TEST_METHOD, - statements = "INSERT INTO ArtifactGroup (name, commonName, description, deprecated) VALUES\n" - + "\t\t('com.example.repository','Example Repository','The code for this project', false);") -@Sql(executionPhase = Sql.ExecutionPhase.AFTER_TEST_METHOD, - statements = "DELETE FROM ArtifactVersion; DELETE FROM ArtifactProduct; DELETE FROM ArtifactGroup;") +@Sql( + executionPhase = Sql.ExecutionPhase.BEFORE_TEST_METHOD, + scripts = "classpath:db/test_init.sql", + statements = { + "INSERT INTO ArtifactGroup (name, commonName, description, deprecated) VALUES\n" + + "\t\t('com.example.repository','Example Repository','The code for this project', false);", + "INSERT INTO Stats (id, measure, dimension) VALUES\n" + + "\t\t(1,100,'Foo')," + + "\t\t(2,150,'Bar');", + "INSERT INTO PlayerStats (name, highScore, countryId, createdOn, updatedOn) VALUES\n" + + "\t\t('Sachin',100, 1, '2020-01-01', now());", + "INSERT INTO PlayerCountry (id, isoCode) VALUES\n" + + "\t\t(1, 'IND');" +}) public class AsyncTest extends IntegrationTest { @Test @@ -106,4 +118,173 @@ public void testAsyncApiEndpoint() throws InterruptedException { } } } +// TODO Uncomment once AsyncExecutorService Singleton is removed. https://github.com/yahoo/elide/issues/1798 +/* + @Test + public void testExportDynamicModel() throws InterruptedException { + //Create Table Export + given() + .contentType(JSONAPI_CONTENT_TYPE) + .body( + data( + resource( + type("tableExport"), + id("ba31ca4e-ed8f-4be0-a0f3-12088fa9265d"), + attributes( + attr("query", "{\"query\":\"{playerStats(filter:\\\"createdOn>=2020-01-01;createdOn<2020-01-02\\\"){ edges{node{countryCode highScore}}}}\",\"variables\":null}"), + attr("queryType", "GRAPHQL_V1_0"), + attr("status", "QUEUED"), + attr("asyncAfterSeconds", "10"), + attr("resultType", "CSV") + ) + ) + ).toJSON()) + .when() + .post("/json/tableExport") + .then() + .statusCode(org.apache.http.HttpStatus.SC_CREATED); + + int i = 0; + while (i < 1000) { + Thread.sleep(10); + Response response = given() + .accept("application/vnd.api+json") + .get("/json/tableExport/ba31ca4e-ed8f-4be0-a0f3-12088fa9265d"); + + String outputResponse = response.jsonPath().getString("data.attributes.status"); + + //If Async Query is created and completed then validate results + if (outputResponse.equals("COMPLETE")) { + + // Validate AsyncQuery Response + response + .then() + .statusCode(HttpStatus.SC_OK) + .body("data.id", equalTo("ba31ca4e-ed8f-4be0-a0f3-12088fa9265d")) + .body("data.type", equalTo("tableExport")) + .body("data.attributes.queryType", equalTo("GRAPHQL_V1_0")) + .body("data.attributes.status", equalTo("COMPLETE")) + .body("data.attributes.result.message", equalTo(null)) + .body("data.attributes.result.url", + equalTo("https://elide.io" + "/export/ba31ca4e-ed8f-4be0-a0f3-12088fa9265d")); + + // Validate GraphQL Response + String responseGraphQL = given() + .contentType(MediaType.APPLICATION_JSON) + .accept(MediaType.APPLICATION_JSON) + .body("{\"query\":\"{ tableExport(ids: [\\\"ba31ca4e-ed8f-4be0-a0f3-12088fa9265d\\\"]) " + + "{ edges { node { id queryType status resultType result " + + "{ url httpStatus recordCount } } } } }\"," + + "\"variables\":null }") + .post("/graphql") + .asString(); + + String expectedResponse = "{\"data\":{\"tableExport\":{\"edges\":[{\"node\":{\"id\":\"ba31ca4e-ed8f-4be0-a0f3-12088fa9265d\"," + + "\"queryType\":\"GRAPHQL_V1_0\",\"status\":\"COMPLETE\",\"resultType\":\"CSV\"," + + "\"result\":{\"url\":\"https://elide.io/export/ba31ca4e-ed8f-4be0-a0f3-12088fa9265d\",\"httpStatus\":200,\"recordCount\":1}}}]}}}"; + + assertEquals(expectedResponse, responseGraphQL); + break; + } else if (!(outputResponse.equals("PROCESSING"))) { + fail("Async Query has failed."); + break; + } + } + when() + .get("/export/ba31ca4e-ed8f-4be0-a0f3-12088fa9265d") + .then() + .statusCode(HttpStatus.SC_OK); + } + + @Test + public void testExportStaticModel() throws InterruptedException { + //Create Table Export + given() + .contentType(JSONAPI_CONTENT_TYPE) + .body( + data( + resource( + type("tableExport"), + id("ba31ca4e-ed8f-4be0-a0f3-12088fa9264d"), + attributes( + attr("query", "{\"query\":\"{ stats { edges { node { dimension measure } } } }\",\"variables\":null}"), + attr("queryType", "GRAPHQL_V1_0"), + attr("status", "QUEUED"), + attr("asyncAfterSeconds", "10"), + attr("resultType", "CSV") + ) + ) + ).toJSON()) + .when() + .post("/json/tableExport") + .then() + .statusCode(org.apache.http.HttpStatus.SC_CREATED); + + int i = 0; + while (i < 1000) { + Thread.sleep(10); + Response response = given() + .accept("application/vnd.api+json") + .get("/json/tableExport/ba31ca4e-ed8f-4be0-a0f3-12088fa9264d"); + + String outputResponse = response.jsonPath().getString("data.attributes.status"); + + //If Async Query is created and completed then validate results + if (outputResponse.equals("COMPLETE")) { + + // Validate AsyncQuery Response + response + .then() + .statusCode(HttpStatus.SC_OK) + .body("data.id", equalTo("ba31ca4e-ed8f-4be0-a0f3-12088fa9264d")) + .body("data.type", equalTo("tableExport")) + .body("data.attributes.queryType", equalTo("GRAPHQL_V1_0")) + .body("data.attributes.status", equalTo("COMPLETE")) + .body("data.attributes.result.message", equalTo(null)) + .body("data.attributes.result.url", + equalTo("https://elide.io" + "/export/ba31ca4e-ed8f-4be0-a0f3-12088fa9264d")); + + // Validate GraphQL Response + String responseGraphQL = given() + .contentType(MediaType.APPLICATION_JSON) + .accept(MediaType.APPLICATION_JSON) + .body("{\"query\":\"{ tableExport(ids: [\\\"ba31ca4e-ed8f-4be0-a0f3-12088fa9264d\\\"]) " + + "{ edges { node { id queryType status resultType result " + + "{ url httpStatus recordCount } } } } }\"," + + "\"variables\":null }") + .post("/graphql") + .asString(); + + String expectedResponse = "{\"data\":{\"tableExport\":{\"edges\":[{\"node\":{\"id\":\"ba31ca4e-ed8f-4be0-a0f3-12088fa9264d\"," + + "\"queryType\":\"GRAPHQL_V1_0\",\"status\":\"COMPLETE\",\"resultType\":\"CSV\"," + + "\"result\":{\"url\":\"https://elide.io/export/ba31ca4e-ed8f-4be0-a0f3-12088fa9264d\",\"httpStatus\":200,\"recordCount\":2}}}]}}}"; + + assertEquals(expectedResponse, responseGraphQL); + break; + } else if (!(outputResponse.equals("PROCESSING"))) { + fail("Async Query has failed."); + break; + } + } + when() + .get("/export/ba31ca4e-ed8f-4be0-a0f3-12088fa9264d") + .then() + .statusCode(HttpStatus.SC_OK); + } +*/ + @Test + public void exportControllerTest() { + when() + .get("/export/asyncQueryId") + .then() + .statusCode(HttpStatus.SC_NOT_FOUND); + } + + @Test + public void postExportControllerTest() { + when() + .post("/export/asyncQueryId") + .then() + .statusCode(HttpStatus.SC_METHOD_NOT_ALLOWED); + } } diff --git a/elide-spring/elide-spring-boot-autoconfigure/src/test/java/example/tests/ControllerTest.java b/elide-spring/elide-spring-boot-autoconfigure/src/test/java/example/tests/ControllerTest.java index 36a72f4c88..2af0813ec4 100644 --- a/elide-spring/elide-spring-boot-autoconfigure/src/test/java/example/tests/ControllerTest.java +++ b/elide-spring/elide-spring-boot-autoconfigure/src/test/java/example/tests/ControllerTest.java @@ -36,6 +36,7 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.springframework.context.annotation.Import; +import org.springframework.test.context.ActiveProfiles; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.jdbc.Sql; import org.springframework.test.context.jdbc.SqlMergeMode; @@ -47,17 +48,20 @@ * Example functional test. */ @SqlMergeMode(SqlMergeMode.MergeMode.MERGE) -@Sql(executionPhase = Sql.ExecutionPhase.BEFORE_TEST_METHOD, +@Sql( + executionPhase = Sql.ExecutionPhase.BEFORE_TEST_METHOD, + scripts = "classpath:db/test_init.sql", statements = "INSERT INTO ArtifactGroup (name, commonName, description, deprecated) VALUES\n" - + "\t\t('com.example.repository','Example Repository','The code for this project', false);") -@Sql(executionPhase = Sql.ExecutionPhase.AFTER_TEST_METHOD, - statements = "DELETE FROM ArtifactVersion; DELETE FROM ArtifactProduct; DELETE FROM ArtifactGroup;") + + "\t\t('com.example.repository','Example Repository','The code for this project', false);" +) @Import(IntegrationTestSetup.class) @TestPropertySource( properties = { - "elide.json-api.enableLinks=true" + "elide.json-api.enableLinks=true", + "elide.async.export.enabled=false" } ) +@ActiveProfiles("default") public class ControllerTest extends IntegrationTest { private String baseUrl; @@ -463,4 +467,16 @@ public void graphqlTestForbiddenCreate() { .body("errors.message", contains("CreatePermission Denied")) .statusCode(200); } + + // Controller disabled by default. + @Test + public void exportControllerDisabledTest() { + // Though post is not supported for export we can use it to test if controller is disabled. + // post returns with 404 if controller is disabled and 405 when enabled. + when() + .post("/export/asyncQueryId") + .then() + .body("error", equalTo("Not Found")) + .statusCode(HttpStatus.SC_NOT_FOUND); + } } diff --git a/elide-spring/elide-spring-boot-autoconfigure/src/test/java/example/tests/DisableAggStoreAsyncTest.java b/elide-spring/elide-spring-boot-autoconfigure/src/test/java/example/tests/DisableAggStoreAsyncTest.java index 27a18ad781..2b0c283281 100644 --- a/elide-spring/elide-spring-boot-autoconfigure/src/test/java/example/tests/DisableAggStoreAsyncTest.java +++ b/elide-spring/elide-spring-boot-autoconfigure/src/test/java/example/tests/DisableAggStoreAsyncTest.java @@ -5,12 +5,116 @@ */ package example.tests; -import org.springframework.test.context.ActiveProfiles; +import static com.yahoo.elide.Elide.JSONAPI_CONTENT_TYPE; +import static com.yahoo.elide.test.jsonapi.JsonApiDSL.attr; +import static com.yahoo.elide.test.jsonapi.JsonApiDSL.attributes; +import static com.yahoo.elide.test.jsonapi.JsonApiDSL.data; +import static com.yahoo.elide.test.jsonapi.JsonApiDSL.id; +import static com.yahoo.elide.test.jsonapi.JsonApiDSL.resource; +import static com.yahoo.elide.test.jsonapi.JsonApiDSL.type; +import static io.restassured.RestAssured.given; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; + +import com.yahoo.elide.core.exceptions.HttpStatus; + +import org.junit.jupiter.api.Test; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.jdbc.Sql; +import org.springframework.test.context.jdbc.SqlMergeMode; + +import io.restassured.response.Response; + +import javax.ws.rs.core.MediaType; /** * Executes Async tests with Aggregation Store disabled. */ -@ActiveProfiles("disableAggStore") -public class DisableAggStoreAsyncTest extends AsyncTest { +@SqlMergeMode(SqlMergeMode.MergeMode.MERGE) +@Sql( + executionPhase = Sql.ExecutionPhase.BEFORE_TEST_METHOD, + scripts = "classpath:db/test_init.sql", + statements = "INSERT INTO ArtifactGroup (name, commonName, description, deprecated) VALUES\n" + + "\t\t('com.example.repository','Example Repository','The code for this project', false);" +) +@TestPropertySource( + properties = { + "elide.aggregation-store.enabled=false" + } +) +public class DisableAggStoreAsyncTest extends IntegrationTest { + + // Test if AsyncQuery is functional with AggregationStore disabled. + @Test + public void testAsyncApiFunctionDisabledAggStore() throws InterruptedException { + //Create Async Request + given() + .contentType(JSONAPI_CONTENT_TYPE) + .body( + data( + resource( + type("asyncQuery"), + id("ba31ca4e-ed8f-4be0-a0f3-12088fa9263d"), + attributes( + attr("query", "/group"), + attr("queryType", "JSONAPI_V1_0"), + attr("status", "QUEUED"), + attr("asyncAfterSeconds", "10") + ) + ) + ).toJSON()) + .when() + .post("/json/asyncQuery") + .then() + .statusCode(org.apache.http.HttpStatus.SC_CREATED); + + int i = 0; + while (i < 1000) { + Thread.sleep(10); + Response response = given() + .accept("application/vnd.api+json") + .get("/json/asyncQuery/ba31ca4e-ed8f-4be0-a0f3-12088fa9263d"); + + String outputResponse = response.jsonPath().getString("data.attributes.status"); + + //If Async Query is created and completed then validate results + if (outputResponse.equals("COMPLETE")) { + + // Validate AsyncQuery Response + response + .then() + .statusCode(HttpStatus.SC_OK) + .body("data.id", equalTo("ba31ca4e-ed8f-4be0-a0f3-12088fa9263d")) + .body("data.type", equalTo("asyncQuery")) + .body("data.attributes.queryType", equalTo("JSONAPI_V1_0")) + .body("data.attributes.status", equalTo("COMPLETE")) + .body("data.attributes.result.contentLength", notNullValue()) + .body("data.attributes.result.responseBody", equalTo("{\"data\":" + + "[{\"type\":\"group\",\"id\":\"com.example.repository\",\"attributes\":" + + "{\"commonName\":\"Example Repository\",\"deprecated\":false,\"description\":\"The code for this project\"}," + + "\"relationships\":{\"products\":{\"data\":[]}}}]}")); + + // Validate GraphQL Response + String responseGraphQL = given() + .contentType(MediaType.APPLICATION_JSON) + .accept(MediaType.APPLICATION_JSON) + .body("{\"query\":\"{ asyncQuery(ids: [\\\"ba31ca4e-ed8f-4be0-a0f3-12088fa9263d\\\"]) " + + "{ edges { node { id queryType status result " + + "{ responseBody httpStatus contentLength } } } } }\"," + + "\"variables\":null }") + .post("/graphql") + .asString(); + + String expectedResponse = "{\"data\":{\"asyncQuery\":{\"edges\":[{\"node\":{\"id\":\"ba31ca4e-ed8f-4be0-a0f3-12088fa9263d\",\"queryType\":\"JSONAPI_V1_0\",\"status\":\"COMPLETE\",\"result\":{\"responseBody\":\"{\\\"data\\\":[{\\\"type\\\":\\\"group\\\",\\\"id\\\":\\\"com.example.repository\\\",\\\"attributes\\\":{\\\"commonName\\\":\\\"Example Repository\\\",\\\"deprecated\\\":false,\\\"description\\\":\\\"The code for this project\\\"},\\\"relationships\\\":{\\\"products\\\":{\\\"data\\\":[]}}}]}\",\"httpStatus\":200,\"contentLength\":208}}}]}}}"; + assertEquals(expectedResponse, responseGraphQL); + break; + } else if (!(outputResponse.equals("PROCESSING"))) { + fail("Async Query has failed."); + break; + } + } + } } diff --git a/elide-spring/elide-spring-boot-autoconfigure/src/test/java/example/tests/DisableAggStoreControllerTest.java b/elide-spring/elide-spring-boot-autoconfigure/src/test/java/example/tests/DisableAggStoreControllerTest.java index 11ff0b74e9..73e433e947 100644 --- a/elide-spring/elide-spring-boot-autoconfigure/src/test/java/example/tests/DisableAggStoreControllerTest.java +++ b/elide-spring/elide-spring-boot-autoconfigure/src/test/java/example/tests/DisableAggStoreControllerTest.java @@ -9,12 +9,16 @@ import static org.hamcrest.Matchers.containsInAnyOrder; import com.yahoo.elide.core.exceptions.HttpStatus; import org.junit.jupiter.api.Test; -import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.TestPropertySource; /** * Executes Controller tests with Aggregation Store disabled. */ -@ActiveProfiles("disableAggStore") +@TestPropertySource( + properties = { + "elide.aggregation-store.enabled=false" + } +) public class DisableAggStoreControllerTest extends ControllerTest { @Override diff --git a/elide-spring/elide-spring-boot-autoconfigure/src/test/java/example/tests/DisableMetaDataStoreControllerTest.java b/elide-spring/elide-spring-boot-autoconfigure/src/test/java/example/tests/DisableMetaDataStoreControllerTest.java index f02e094c16..a97d126457 100644 --- a/elide-spring/elide-spring-boot-autoconfigure/src/test/java/example/tests/DisableMetaDataStoreControllerTest.java +++ b/elide-spring/elide-spring-boot-autoconfigure/src/test/java/example/tests/DisableMetaDataStoreControllerTest.java @@ -9,12 +9,17 @@ import static org.hamcrest.Matchers.containsInAnyOrder; import com.yahoo.elide.core.exceptions.HttpStatus; import org.junit.jupiter.api.Test; -import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.TestPropertySource; /** * Executes Controller tests with Aggregation Store disabled. */ -@ActiveProfiles("disableMetaDataStore") +@TestPropertySource( + properties = { + "elide.aggregation-store.enabled=true", + "elide.aggregation-store.enableMetaDataStore=false" + } +) public class DisableMetaDataStoreControllerTest extends ControllerTest { @Override diff --git a/elide-spring/elide-spring-boot-autoconfigure/src/test/java/example/tests/DynamicConfigTest.java b/elide-spring/elide-spring-boot-autoconfigure/src/test/java/example/tests/DynamicConfigTest.java index 98f2cf34c2..2072e74875 100644 --- a/elide-spring/elide-spring-boot-autoconfigure/src/test/java/example/tests/DynamicConfigTest.java +++ b/elide-spring/elide-spring-boot-autoconfigure/src/test/java/example/tests/DynamicConfigTest.java @@ -24,20 +24,15 @@ * Dynamic Configuration functional test. */ @SqlMergeMode(SqlMergeMode.MergeMode.MERGE) -@Sql(executionPhase = Sql.ExecutionPhase.BEFORE_TEST_METHOD, - statements = "CREATE TABLE PlayerStats (name varchar(255) not null," - + "\t\t countryId varchar(255), createdOn timestamp, updatedOn timestamp," - + "\t\t highScore bigint, primary key (name));" - + "CREATE TABLE PlayerCountry (id varchar(255) not null," - + "\t\t isoCode varchar(255), primary key (id));" - + "INSERT INTO PlayerStats (name,countryId,createdOn,updatedOn) VALUES\n" - + "\t\t('SerenaWilliams','1','2000-10-10','2001-10-10');" - + "INSERT INTO PlayerCountry (id,isoCode) VALUES\n" - + "\t\t('2','IND');" - + "INSERT INTO PlayerCountry (id,isoCode) VALUES\n" - + "\t\t('1','USA');") -@Sql(executionPhase = Sql.ExecutionPhase.AFTER_TEST_METHOD, -statements = "DROP TABLE PlayerStats; DROP TABLE PlayerCountry;") +@Sql( + executionPhase = Sql.ExecutionPhase.BEFORE_TEST_METHOD, + scripts = "classpath:db/test_init.sql", + statements = "INSERT INTO PlayerStats (name,countryId,createdOn,updatedOn) VALUES\n" + + "\t\t('SerenaWilliams','1','2000-10-10','2001-10-10');" + + "INSERT INTO PlayerCountry (id,isoCode) VALUES\n" + + "\t\t('2','IND');" + + "INSERT INTO PlayerCountry (id,isoCode) VALUES\n" + + "\t\t('1','USA');") public class DynamicConfigTest extends IntegrationTest { /** * This test demonstrates an example test using the JSON-API DSL. diff --git a/elide-spring/elide-spring-boot-autoconfigure/src/test/java/example/tests/IntegrationTest.java b/elide-spring/elide-spring-boot-autoconfigure/src/test/java/example/tests/IntegrationTest.java index 13a2c741d6..c37a13575b 100644 --- a/elide-spring/elide-spring-boot-autoconfigure/src/test/java/example/tests/IntegrationTest.java +++ b/elide-spring/elide-spring-boot-autoconfigure/src/test/java/example/tests/IntegrationTest.java @@ -9,6 +9,7 @@ import org.junit.jupiter.api.TestInstance; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.web.server.LocalServerPort; + import io.restassured.RestAssured; import java.util.TimeZone; diff --git a/elide-spring/elide-spring-boot-autoconfigure/src/test/resources/application-disableAggStore.yaml b/elide-spring/elide-spring-boot-autoconfigure/src/test/resources/application-disableAggStore.yaml deleted file mode 100644 index a73bc7c30d..0000000000 --- a/elide-spring/elide-spring-boot-autoconfigure/src/test/resources/application-disableAggStore.yaml +++ /dev/null @@ -1,43 +0,0 @@ -server: - port: 4001 - -elide: - modelPackage: 'example.models' - json-api: - path: /json - enabled: true - graphql: - path: /graphql - enabled: true - swagger: - path: /doc - enabled: true - async: - enabled: true - threadPoolSize: 7 - maxRunTimeMinutes: 65 - cleanupEnabled: true - queryCleanupDays: 7 - defaultAsyncAPIDAO: true - dynamic-config: - path: src/test/resources/configs - enabled: true - aggregation-store: - enabled: false -spring: - jpa: - show-sql: true - properties: - hibernate: - dialect: 'org.hibernate.dialect.H2Dialect' - jdbc: - use_scrollable_resultset: true - hibernate: - naming: - physical-strategy: 'org.hibernate.boot.model.naming.PhysicalNamingStrategyStandardImpl' - ddl-auto: 'create' - datasource: - url: 'jdbc:h2:mem:db1;DB_CLOSE_DELAY=-1' - username: 'sa' - password: '' - driver-class-name: 'org.h2.Driver' diff --git a/elide-spring/elide-spring-boot-autoconfigure/src/test/resources/application-disableMetaDataStore.yaml b/elide-spring/elide-spring-boot-autoconfigure/src/test/resources/application-disableMetaDataStore.yaml deleted file mode 100644 index 35ad513ca5..0000000000 --- a/elide-spring/elide-spring-boot-autoconfigure/src/test/resources/application-disableMetaDataStore.yaml +++ /dev/null @@ -1,45 +0,0 @@ -server: - port: 4001 - -elide: - modelPackage: 'example.models' - json-api: - path: /json - enabled: true - graphql: - path: /graphql - enabled: true - swagger: - path: /doc - enabled: true - async: - enabled: true - threadPoolSize: 7 - maxRunTimeMinutes: 65 - cleanupEnabled: true - queryCleanupDays: 7 - defaultAsyncAPIDAO: true - dynamic-config: - path: src/test/resources/configs - enabled: true - aggregation-store: - enabled: true - default-dialect: h2 - enableMetaDataStore: false -spring: - jpa: - show-sql: true - properties: - hibernate: - dialect: 'org.hibernate.dialect.H2Dialect' - jdbc: - use_scrollable_resultset: true - hibernate: - naming: - physical-strategy: 'org.hibernate.boot.model.naming.PhysicalNamingStrategyStandardImpl' - ddl-auto: 'create' - datasource: - url: 'jdbc:h2:mem:db1;DB_CLOSE_DELAY=-1' - username: 'sa' - password: '' - driver-class-name: 'org.h2.Driver' diff --git a/elide-spring/elide-spring-boot-autoconfigure/src/test/resources/application.yaml b/elide-spring/elide-spring-boot-autoconfigure/src/test/resources/application.yaml index 9d1906a8f5..6eead13406 100644 --- a/elide-spring/elide-spring-boot-autoconfigure/src/test/resources/application.yaml +++ b/elide-spring/elide-spring-boot-autoconfigure/src/test/resources/application.yaml @@ -19,6 +19,9 @@ elide: cleanupEnabled: true queryCleanupDays: 7 defaultAsyncAPIDAO: true + export: + enabled: true + path: /export dynamic-config: path: src/test/resources/configs enabled: true diff --git a/elide-spring/elide-spring-boot-autoconfigure/src/test/resources/db/test_init.sql b/elide-spring/elide-spring-boot-autoconfigure/src/test/resources/db/test_init.sql new file mode 100644 index 0000000000..5985374a30 --- /dev/null +++ b/elide-spring/elide-spring-boot-autoconfigure/src/test/resources/db/test_init.sql @@ -0,0 +1,31 @@ +-- Initialization SQL for the Tests. +-- They are designed to be executed before each test. + +-- Create Tables if not present. +CREATE TABLE IF NOT EXISTS Stats ( + id int, + measure int, + dimension VARCHAR(255) +); + +CREATE TABLE IF NOT EXISTS PlayerStats ( + name varchar(255) not null, + countryId varchar(255), + createdOn timestamp, + updatedOn timestamp, + highScore bigint, + primary key (name) +); +CREATE TABLE IF NOT EXISTS PlayerCountry ( + id varchar(255) not null, + isoCode varchar(255), + primary key (id) +); + +-- Cleanup tables. +DELETE FROM ArtifactVersion; +DELETE FROM ArtifactProduct; +DELETE FROM ArtifactGroup; +DELETE FROM Stats; +DELETE FROM PlayerStats; +DELETE FROM PlayerCountry;