From 4caaae234214edf6a61bea57531de79520604d54 Mon Sep 17 00:00:00 2001 From: Moiz Arafat Date: Thu, 6 Jan 2022 10:20:20 -0500 Subject: [PATCH] File Extension Support for Export Attachments (#2475) --- .../elide/async/models/FileExtensionType.java | 26 +++++++++++++++ .../yahoo/elide/async/models/ResultType.java | 14 ++++++-- .../async/operation/TableExportOperation.java | 6 +++- .../FileResultStorageEngine.java | 33 ++++++++++++------- .../storageengine/ResultStorageEngine.java | 12 +++++-- .../GraphQLTableExportOperationTest.java | 2 +- .../JsonAPITableExportOperationTest.java | 4 +-- .../FileResultStorageEngineTest.java | 4 +-- ...egrationTestApplicationResourceConfig.java | 2 +- .../validator/DynamicConfigValidator.java | 2 +- .../config/ElideAsyncConfiguration.java | 2 +- .../config/ExportControllerProperties.java | 5 +++ .../test/java/example/tests/AsyncTest.java | 12 +++---- .../src/test/resources/application.yaml | 1 + .../config/ElideResourceConfig.java | 3 +- .../config/ElideStandaloneAsyncSettings.java | 10 ++++++ 16 files changed, 106 insertions(+), 32 deletions(-) create mode 100644 elide-async/src/main/java/com/yahoo/elide/async/models/FileExtensionType.java diff --git a/elide-async/src/main/java/com/yahoo/elide/async/models/FileExtensionType.java b/elide-async/src/main/java/com/yahoo/elide/async/models/FileExtensionType.java new file mode 100644 index 0000000000..409d2e7fa6 --- /dev/null +++ b/elide-async/src/main/java/com/yahoo/elide/async/models/FileExtensionType.java @@ -0,0 +1,26 @@ +/* + * Copyright 2022, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ + +package com.yahoo.elide.async.models; + +/** + * ENUM of supported file extension types. + */ +public enum FileExtensionType { + JSON(".json"), + CSV(".csv"), + NONE(""); + + private final String extension; + + FileExtensionType(String extension) { + this.extension = extension; + } + + public String getExtension() { + return extension; + } +} diff --git a/elide-async/src/main/java/com/yahoo/elide/async/models/ResultType.java b/elide-async/src/main/java/com/yahoo/elide/async/models/ResultType.java index edaff10fb0..35e7e79904 100644 --- a/elide-async/src/main/java/com/yahoo/elide/async/models/ResultType.java +++ b/elide-async/src/main/java/com/yahoo/elide/async/models/ResultType.java @@ -9,6 +9,16 @@ * ENUM of supported result types. */ public enum ResultType { - JSON, - CSV + JSON(FileExtensionType.JSON), + CSV(FileExtensionType.CSV); + + private final FileExtensionType fileExtensionType; + + ResultType(FileExtensionType fileExtensionType) { + this.fileExtensionType = fileExtensionType; + } + + public FileExtensionType getFileExtensionType() { + return fileExtensionType; + } } diff --git a/elide-async/src/main/java/com/yahoo/elide/async/operation/TableExportOperation.java b/elide-async/src/main/java/com/yahoo/elide/async/operation/TableExportOperation.java index bb86221132..48f7517bcc 100644 --- a/elide-async/src/main/java/com/yahoo/elide/async/operation/TableExportOperation.java +++ b/elide-async/src/main/java/com/yahoo/elide/async/operation/TableExportOperation.java @@ -11,6 +11,7 @@ import com.yahoo.elide.async.export.validator.Validator; import com.yahoo.elide.async.models.AsyncAPI; import com.yahoo.elide.async.models.AsyncAPIResult; +import com.yahoo.elide.async.models.FileExtensionType; import com.yahoo.elide.async.models.TableExport; import com.yahoo.elide.async.models.TableExportResult; import com.yahoo.elide.async.service.AsyncExecutorService; @@ -159,7 +160,10 @@ public abstract RequestScope getRequestScope(TableExport exportObj, RequestScope public String generateDownloadURL(TableExport exportObj, RequestScope scope) { String downloadPath = scope.getElideSettings().getExportApiPath(); String baseURL = scope.getBaseUrlEndPoint(); - return baseURL + downloadPath + "/" + exportObj.getId(); + String extension = this.engine.isExtensionEnabled() + ? exportObj.getResultType().getFileExtensionType().getExtension() + : FileExtensionType.NONE.getExtension(); + return baseURL + downloadPath + "/" + exportObj.getId() + extension; } /** diff --git a/elide-async/src/main/java/com/yahoo/elide/async/service/storageengine/FileResultStorageEngine.java b/elide-async/src/main/java/com/yahoo/elide/async/service/storageengine/FileResultStorageEngine.java index 72c03c2366..065e42cb22 100644 --- a/elide-async/src/main/java/com/yahoo/elide/async/service/storageengine/FileResultStorageEngine.java +++ b/elide-async/src/main/java/com/yahoo/elide/async/service/storageengine/FileResultStorageEngine.java @@ -6,6 +6,7 @@ package com.yahoo.elide.async.service.storageengine; +import com.yahoo.elide.async.models.FileExtensionType; import com.yahoo.elide.async.models.TableExport; import io.reactivex.Observable; import lombok.Getter; @@ -23,27 +24,32 @@ /** * Default implementation of ResultStorageEngine that stores results on local filesystem. - * It supports Async Module to store results with async query. + * It supports Async Module to store results with Table Export query. */ @Singleton @Slf4j @Getter public class FileResultStorageEngine implements ResultStorageEngine { @Setter private String basePath; + @Setter private boolean enableExtension; /** * Constructor. * @param basePath basePath for storing the files. Can be absolute or relative. */ - public FileResultStorageEngine(String basePath) { + public FileResultStorageEngine(String basePath, boolean enableExtension) { this.basePath = basePath; + this.enableExtension = enableExtension; } @Override public TableExport storeResults(TableExport tableExport, Observable result) { - log.debug("store AsyncResults for Download"); + log.debug("store TableExportResults for Download"); + String extension = this.isExtensionEnabled() + ? tableExport.getResultType().getFileExtensionType().getExtension() + : FileExtensionType.NONE.getExtension(); - try (BufferedWriter writer = getWriter(tableExport.getId())) { + try (BufferedWriter writer = getWriter(tableExport.getId(), extension)) { result .map(record -> record.concat(System.lineSeparator())) .subscribe( @@ -64,11 +70,11 @@ public TableExport storeResults(TableExport tableExport, Observable resu } @Override - public Observable getResultsByID(String asyncQueryID) { - log.debug("getAsyncResultsByID"); + public Observable getResultsByID(String tableExportID) { + log.debug("getTableExportResultsByID"); return Observable.using( - () -> getReader(asyncQueryID), + () -> getReader(tableExportID), reader -> Observable.fromIterable(() -> new Iterator() { private String record = null; @@ -93,21 +99,26 @@ public String next() { BufferedReader::close); } - private BufferedReader getReader(String asyncQueryID) { + private BufferedReader getReader(String tableExportID) { try { - return Files.newBufferedReader(Paths.get(basePath + File.separator + asyncQueryID)); + return Files.newBufferedReader(Paths.get(basePath + File.separator + tableExportID)); } catch (IOException e) { log.debug(e.getMessage()); throw new IllegalStateException(RETRIEVE_ERROR, e); } } - private BufferedWriter getWriter(String asyncQueryID) { + private BufferedWriter getWriter(String tableExportID, String extension) { try { - return Files.newBufferedWriter(Paths.get(basePath + File.separator + asyncQueryID)); + return Files.newBufferedWriter(Paths.get(basePath + File.separator + tableExportID + extension)); } catch (IOException e) { log.debug(e.getMessage()); throw new IllegalStateException(STORE_ERROR, e); } } + + @Override + public boolean isExtensionEnabled() { + return this.enableExtension; + } } diff --git a/elide-async/src/main/java/com/yahoo/elide/async/service/storageengine/ResultStorageEngine.java b/elide-async/src/main/java/com/yahoo/elide/async/service/storageengine/ResultStorageEngine.java index df2595ce28..91d5cd28e8 100644 --- a/elide-async/src/main/java/com/yahoo/elide/async/service/storageengine/ResultStorageEngine.java +++ b/elide-async/src/main/java/com/yahoo/elide/async/service/storageengine/ResultStorageEngine.java @@ -26,8 +26,14 @@ public interface ResultStorageEngine { /** * Searches for the async query results by ID and returns the record. - * @param asyncQueryID is the query ID of the AsyncQuery - * @return returns the result associated with the AsyncQueryID + * @param tableExportID is the ID of the TableExport. It may include extension too if enabled. + * @return returns the result associated with the tableExportID */ - public Observable getResultsByID(String asyncQueryID); + public Observable getResultsByID(String tableExportID); + + /** + * Whether the result storage engine has enabled extensions for attachments. + * @return returns whether the file extensions are enabled + */ + public boolean isExtensionEnabled(); } diff --git a/elide-async/src/test/java/com/yahoo/elide/async/operation/GraphQLTableExportOperationTest.java b/elide-async/src/test/java/com/yahoo/elide/async/operation/GraphQLTableExportOperationTest.java index d43cbf3d98..45340897fb 100644 --- a/elide-async/src/test/java/com/yahoo/elide/async/operation/GraphQLTableExportOperationTest.java +++ b/elide-async/src/test/java/com/yahoo/elide/async/operation/GraphQLTableExportOperationTest.java @@ -77,7 +77,7 @@ public void setupMocks(@TempDir Path tempDir) { user = mock(User.class); requestScope = mock(RequestScope.class); asyncExecutorService = mock(AsyncExecutorService.class); - engine = new FileResultStorageEngine(tempDir.toString()); + engine = new FileResultStorageEngine(tempDir.toString(), false); when(asyncExecutorService.getElide()).thenReturn(elide); when(requestScope.getApiVersion()).thenReturn(NO_VERSION); when(requestScope.getUser()).thenReturn(user); diff --git a/elide-async/src/test/java/com/yahoo/elide/async/operation/JsonAPITableExportOperationTest.java b/elide-async/src/test/java/com/yahoo/elide/async/operation/JsonAPITableExportOperationTest.java index 4cf3d809ac..84db453729 100644 --- a/elide-async/src/test/java/com/yahoo/elide/async/operation/JsonAPITableExportOperationTest.java +++ b/elide-async/src/test/java/com/yahoo/elide/async/operation/JsonAPITableExportOperationTest.java @@ -78,7 +78,7 @@ public void setupMocks(@TempDir Path tempDir) { user = mock(User.class); requestScope = mock(RequestScope.class); asyncExecutorService = mock(AsyncExecutorService.class); - engine = new FileResultStorageEngine(tempDir.toString()); + engine = new FileResultStorageEngine(tempDir.toString(), true); when(asyncExecutorService.getElide()).thenReturn(elide); when(requestScope.getApiVersion()).thenReturn(NO_VERSION); when(requestScope.getUser()).thenReturn(user); @@ -102,7 +102,7 @@ public void testProcessQuery() throws URISyntaxException, IOException { TableExportResult queryResultObj = (TableExportResult) jsonAPIOperation.call(); assertEquals(200, queryResultObj.getHttpStatus()); - assertEquals("https://elide.io/export/edc4a871-dff2-4054-804e-d80075cf827d", queryResultObj.getUrl().toString()); + assertEquals("https://elide.io/export/edc4a871-dff2-4054-804e-d80075cf827d.csv", queryResultObj.getUrl().toString()); assertEquals(1, queryResultObj.getRecordCount()); assertNull(queryResultObj.getMessage()); } diff --git a/elide-async/src/test/java/com/yahoo/elide/async/service/storageengine/FileResultStorageEngineTest.java b/elide-async/src/test/java/com/yahoo/elide/async/service/storageengine/FileResultStorageEngineTest.java index d6fb9d7ab5..4f0fb2a45c 100644 --- a/elide-async/src/test/java/com/yahoo/elide/async/service/storageengine/FileResultStorageEngineTest.java +++ b/elide-async/src/test/java/com/yahoo/elide/async/service/storageengine/FileResultStorageEngineTest.java @@ -67,7 +67,7 @@ public void testStoreResultsFail(@TempDir File tempDir) { } private String readResultsFile(String path, String queryId) { - FileResultStorageEngine engine = new FileResultStorageEngine(path); + FileResultStorageEngine engine = new FileResultStorageEngine(path, false); return engine.getResultsByID(queryId).collect(() -> new StringBuilder(), (resultBuilder, tempResult) -> { @@ -80,7 +80,7 @@ private String readResultsFile(String path, String queryId) { } private void storeResultsFile(String path, String queryId, Observable storable) { - FileResultStorageEngine engine = new FileResultStorageEngine(path); + FileResultStorageEngine engine = new FileResultStorageEngine(path, false); TableExport query = new TableExport(); query.setId(queryId); diff --git a/elide-integration-tests/src/test/java/com/yahoo/elide/async/integration/tests/framework/AsyncIntegrationTestApplicationResourceConfig.java b/elide-integration-tests/src/test/java/com/yahoo/elide/async/integration/tests/framework/AsyncIntegrationTestApplicationResourceConfig.java index c8414802b5..c3e25663b3 100644 --- a/elide-integration-tests/src/test/java/com/yahoo/elide/async/integration/tests/framework/AsyncIntegrationTestApplicationResourceConfig.java +++ b/elide-integration-tests/src/test/java/com/yahoo/elide/async/integration/tests/framework/AsyncIntegrationTestApplicationResourceConfig.java @@ -116,7 +116,7 @@ protected void configure() { // Create ResultStorageEngine Path storageDestination = (Path) servletContext.getAttribute(STORAGE_DESTINATION_ATTR); if (storageDestination != null) { // TableExport is enabled - ResultStorageEngine resultStorageEngine = new FileResultStorageEngine(storageDestination.toAbsolutePath().toString()); + ResultStorageEngine resultStorageEngine = new FileResultStorageEngine(storageDestination.toAbsolutePath().toString(), false); bind(resultStorageEngine).to(ResultStorageEngine.class).named("resultStorageEngine"); Map supportedFormatters = new HashMap<>(); diff --git a/elide-model-config/src/main/java/com/yahoo/elide/modelconfig/validator/DynamicConfigValidator.java b/elide-model-config/src/main/java/com/yahoo/elide/modelconfig/validator/DynamicConfigValidator.java index 1701cc526c..caec5731ff 100644 --- a/elide-model-config/src/main/java/com/yahoo/elide/modelconfig/validator/DynamicConfigValidator.java +++ b/elide-model-config/src/main/java/com/yahoo/elide/modelconfig/validator/DynamicConfigValidator.java @@ -194,7 +194,7 @@ public void readConfigs() throws IOException { readConfigs(fileLoader.loadResources()); } - public void readConfigs(Map resourceMap) throws IOException { + public void readConfigs(Map resourceMap) { this.modelVariables = readVariableConfig(Config.MODELVARIABLE, resourceMap); this.elideSecurityConfig = readSecurityConfig(resourceMap); this.dbVariables = readVariableConfig(Config.DBVARIABLE, resourceMap); diff --git a/elide-spring/elide-spring-boot-autoconfigure/src/main/java/com/yahoo/elide/spring/config/ElideAsyncConfiguration.java b/elide-spring/elide-spring-boot-autoconfigure/src/main/java/com/yahoo/elide/spring/config/ElideAsyncConfiguration.java index 7dc4ccf3e2..0039aaa4cd 100644 --- a/elide-spring/elide-spring-boot-autoconfigure/src/main/java/com/yahoo/elide/spring/config/ElideAsyncConfiguration.java +++ b/elide-spring/elide-spring-boot-autoconfigure/src/main/java/com/yahoo/elide/spring/config/ElideAsyncConfiguration.java @@ -165,7 +165,7 @@ public AsyncAPIDAO buildAsyncAPIDAO(Elide elide) { public ResultStorageEngine buildResultStorageEngine(Elide elide, ElideConfigProperties settings, AsyncAPIDAO asyncQueryDAO) { FileResultStorageEngine resultStorageEngine = new FileResultStorageEngine(settings.getAsync().getExport() - .getStorageDestination()); + .getStorageDestination(), settings.getAsync().getExport().isExtensionEnabled()); return resultStorageEngine; } } diff --git a/elide-spring/elide-spring-boot-autoconfigure/src/main/java/com/yahoo/elide/spring/config/ExportControllerProperties.java b/elide-spring/elide-spring-boot-autoconfigure/src/main/java/com/yahoo/elide/spring/config/ExportControllerProperties.java index bc15f21e07..96a61db2dd 100644 --- a/elide-spring/elide-spring-boot-autoconfigure/src/main/java/com/yahoo/elide/spring/config/ExportControllerProperties.java +++ b/elide-spring/elide-spring-boot-autoconfigure/src/main/java/com/yahoo/elide/spring/config/ExportControllerProperties.java @@ -20,6 +20,11 @@ public class ExportControllerProperties extends ControllerProperties { */ private boolean skipCSVHeader = false; + /** + * Enable Adding Extension to table export attachments. + */ + private boolean extensionEnabled = false; + /** * The URL path prefix for the controller. */ diff --git a/elide-spring/elide-spring-boot-autoconfigure/src/test/java/example/tests/AsyncTest.java b/elide-spring/elide-spring-boot-autoconfigure/src/test/java/example/tests/AsyncTest.java index 7e6e66aa70..d95841566c 100644 --- a/elide-spring/elide-spring-boot-autoconfigure/src/test/java/example/tests/AsyncTest.java +++ b/elide-spring/elide-spring-boot-autoconfigure/src/test/java/example/tests/AsyncTest.java @@ -161,7 +161,7 @@ public void testExportDynamicModel() throws InterruptedException { .body("data.attributes.status", equalTo("COMPLETE")) .body("data.attributes.result.message", equalTo(null)) .body("data.attributes.result.url", - equalTo("https://elide.io" + "/export/ba31ca4e-ed8f-4be0-a0f3-12088fa9265d")); + equalTo("https://elide.io" + "/export/ba31ca4e-ed8f-4be0-a0f3-12088fa9265d.csv")); // Validate GraphQL Response String responseGraphQL = given() @@ -176,7 +176,7 @@ public void testExportDynamicModel() throws InterruptedException { String expectedResponse = "{\"data\":{\"tableExport\":{\"edges\":[{\"node\":{\"id\":\"ba31ca4e-ed8f-4be0-a0f3-12088fa9265d\"," + "\"queryType\":\"GRAPHQL_V1_0\",\"status\":\"COMPLETE\",\"resultType\":\"CSV\"," - + "\"result\":{\"url\":\"https://elide.io/export/ba31ca4e-ed8f-4be0-a0f3-12088fa9265d\",\"httpStatus\":200,\"recordCount\":1}}}]}}}"; + + "\"result\":{\"url\":\"https://elide.io/export/ba31ca4e-ed8f-4be0-a0f3-12088fa9265d.csv\",\"httpStatus\":200,\"recordCount\":1}}}]}}}"; assertEquals(expectedResponse, responseGraphQL); break; @@ -184,7 +184,7 @@ public void testExportDynamicModel() throws InterruptedException { assertEquals("PROCESSING", outputResponse, "Async Query has failed."); } when() - .get("/export/ba31ca4e-ed8f-4be0-a0f3-12088fa9265d") + .get("/export/ba31ca4e-ed8f-4be0-a0f3-12088fa9265d.csv") .then() .statusCode(HttpStatus.SC_OK); } @@ -235,7 +235,7 @@ public void testExportStaticModel() throws InterruptedException { .body("data.attributes.status", equalTo("COMPLETE")) .body("data.attributes.result.message", equalTo(null)) .body("data.attributes.result.url", - equalTo("https://elide.io" + "/export/ba31ca4e-ed8f-4be0-a0f3-12088fa9264d")); + equalTo("https://elide.io" + "/export/ba31ca4e-ed8f-4be0-a0f3-12088fa9264d.csv")); // Validate GraphQL Response String responseGraphQL = given() @@ -250,7 +250,7 @@ public void testExportStaticModel() throws InterruptedException { String expectedResponse = "{\"data\":{\"tableExport\":{\"edges\":[{\"node\":{\"id\":\"ba31ca4e-ed8f-4be0-a0f3-12088fa9264d\"," + "\"queryType\":\"GRAPHQL_V1_0\",\"status\":\"COMPLETE\",\"resultType\":\"CSV\"," - + "\"result\":{\"url\":\"https://elide.io/export/ba31ca4e-ed8f-4be0-a0f3-12088fa9264d\",\"httpStatus\":200,\"recordCount\":2}}}]}}}"; + + "\"result\":{\"url\":\"https://elide.io/export/ba31ca4e-ed8f-4be0-a0f3-12088fa9264d.csv\",\"httpStatus\":200,\"recordCount\":2}}}]}}}"; assertEquals(expectedResponse, responseGraphQL); break; @@ -258,7 +258,7 @@ public void testExportStaticModel() throws InterruptedException { assertEquals("PROCESSING", outputResponse, "Async Query has failed."); } when() - .get("/export/ba31ca4e-ed8f-4be0-a0f3-12088fa9264d") + .get("/export/ba31ca4e-ed8f-4be0-a0f3-12088fa9264d.csv") .then() .statusCode(HttpStatus.SC_OK); } diff --git a/elide-spring/elide-spring-boot-autoconfigure/src/test/resources/application.yaml b/elide-spring/elide-spring-boot-autoconfigure/src/test/resources/application.yaml index 6c7c0f4dd5..3df45b0160 100644 --- a/elide-spring/elide-spring-boot-autoconfigure/src/test/resources/application.yaml +++ b/elide-spring/elide-spring-boot-autoconfigure/src/test/resources/application.yaml @@ -26,6 +26,7 @@ elide: export: enabled: true path: /export + extensionEnabled: true dynamic-config: path: src/test/resources/configs enabled: true diff --git a/elide-standalone/src/main/java/com/yahoo/elide/standalone/config/ElideResourceConfig.java b/elide-standalone/src/main/java/com/yahoo/elide/standalone/config/ElideResourceConfig.java index f7962c1744..0139ae7f71 100644 --- a/elide-standalone/src/main/java/com/yahoo/elide/standalone/config/ElideResourceConfig.java +++ b/elide-standalone/src/main/java/com/yahoo/elide/standalone/config/ElideResourceConfig.java @@ -165,7 +165,8 @@ protected void bindAsync( ResultStorageEngine resultStorageEngine = asyncProperties.getResultStorageEngine(); if (resultStorageEngine == null) { - resultStorageEngine = new FileResultStorageEngine(asyncProperties.getStorageDestination()); + resultStorageEngine = new FileResultStorageEngine(asyncProperties.getStorageDestination(), + asyncProperties.enableExtension()); } bind(resultStorageEngine).to(ResultStorageEngine.class).named("resultStorageEngine"); diff --git a/elide-standalone/src/main/java/com/yahoo/elide/standalone/config/ElideStandaloneAsyncSettings.java b/elide-standalone/src/main/java/com/yahoo/elide/standalone/config/ElideStandaloneAsyncSettings.java index e6627033e6..3cfd683430 100644 --- a/elide-standalone/src/main/java/com/yahoo/elide/standalone/config/ElideStandaloneAsyncSettings.java +++ b/elide-standalone/src/main/java/com/yahoo/elide/standalone/config/ElideStandaloneAsyncSettings.java @@ -117,6 +117,16 @@ default boolean enableExport() { return false; } + /** + * Enable the addition of extensions to Export attachments. + * If false, the attachments will be downloaded without extensions. + * + * @return Default: False + */ + default boolean enableExtension() { + return false; + } + /** * Skip generating Header when exporting in CSV format. *