File Extension Support for Export Attachments (#2475)

yahoo · Jan 6, 2022 · 4caaae2 · 4caaae2
1 parent 3385fb0
commit 4caaae2
Show file tree

Hide file tree

Showing 16 changed files with 106 additions and 32 deletions.
diff --git a/elide-async/src/main/java/com/yahoo/elide/async/models/FileExtensionType.java b/elide-async/src/main/java/com/yahoo/elide/async/models/FileExtensionType.java
@@ -0,0 +1,26 @@
+/*
+ * Copyright 2022, Yahoo Inc.
+ * Licensed under the Apache License, Version 2.0
+ * See LICENSE file in project root for terms.
+ */
+
+package com.yahoo.elide.async.models;
+
+/**
+ * ENUM of supported file extension types.
+ */
+public enum FileExtensionType {
+    JSON(".json"),
+    CSV(".csv"),
+    NONE("");
+
+    private final String extension;
+
+    FileExtensionType(String extension) {
+        this.extension = extension;
+    }
+
+    public String getExtension() {
+        return extension;
+    }
+}
diff --git a/elide-async/src/main/java/com/yahoo/elide/async/models/ResultType.java b/elide-async/src/main/java/com/yahoo/elide/async/models/ResultType.java
@@ -9,6 +9,16 @@
  * ENUM of supported result types.
  */
 public enum ResultType {
-    JSON,
-    CSV
+    JSON(FileExtensionType.JSON),
+    CSV(FileExtensionType.CSV);
+
+    private final FileExtensionType fileExtensionType;
+
+    ResultType(FileExtensionType fileExtensionType) {
+        this.fileExtensionType = fileExtensionType;
+    }
+
+    public FileExtensionType getFileExtensionType() {
+        return fileExtensionType;
+    }
 }
diff --git a/elide-async/src/main/java/com/yahoo/elide/async/operation/TableExportOperation.java b/elide-async/src/main/java/com/yahoo/elide/async/operation/TableExportOperation.java
@@ -11,6 +11,7 @@
 import com.yahoo.elide.async.export.validator.Validator;
 import com.yahoo.elide.async.models.AsyncAPI;
 import com.yahoo.elide.async.models.AsyncAPIResult;
+import com.yahoo.elide.async.models.FileExtensionType;
 import com.yahoo.elide.async.models.TableExport;
 import com.yahoo.elide.async.models.TableExportResult;
 import com.yahoo.elide.async.service.AsyncExecutorService;
@@ -159,7 +160,10 @@ public abstract RequestScope getRequestScope(TableExport exportObj, RequestScope
     public String generateDownloadURL(TableExport exportObj, RequestScope scope) {
         String downloadPath =  scope.getElideSettings().getExportApiPath();
         String baseURL = scope.getBaseUrlEndPoint();
-        return baseURL + downloadPath + "/" + exportObj.getId();
+        String extension = this.engine.isExtensionEnabled()
+                ? exportObj.getResultType().getFileExtensionType().getExtension()
+                : FileExtensionType.NONE.getExtension();
+        return baseURL + downloadPath + "/" + exportObj.getId() + extension;
     }
 
     /**

diff --git a/...nc/src/main/java/com/yahoo/elide/async/service/storageengine/FileResultStorageEngine.java b/...nc/src/main/java/com/yahoo/elide/async/service/storageengine/FileResultStorageEngine.java
@@ -6,6 +6,7 @@
 
 package com.yahoo.elide.async.service.storageengine;
 
+import com.yahoo.elide.async.models.FileExtensionType;
 import com.yahoo.elide.async.models.TableExport;
 import io.reactivex.Observable;
 import lombok.Getter;
@@ -23,27 +24,32 @@
 
 /**
  * Default implementation of ResultStorageEngine that stores results on local filesystem.
- * It supports Async Module to store results with async query.
+ * It supports Async Module to store results with Table Export query.
  */
 @Singleton
 @Slf4j
 @Getter
 public class FileResultStorageEngine implements ResultStorageEngine {
     @Setter private String basePath;
+    @Setter private boolean enableExtension;
 
     /**
      * Constructor.
      * @param basePath basePath for storing the files. Can be absolute or relative.
      */
-    public FileResultStorageEngine(String basePath) {
+    public FileResultStorageEngine(String basePath, boolean enableExtension) {
         this.basePath = basePath;
+        this.enableExtension = enableExtension;
     }
 
     @Override
     public TableExport storeResults(TableExport tableExport, Observable<String> result) {
-        log.debug("store AsyncResults for Download");
+        log.debug("store TableExportResults for Download");
+        String extension = this.isExtensionEnabled()
+                ? tableExport.getResultType().getFileExtensionType().getExtension()
+                : FileExtensionType.NONE.getExtension();
 
-        try (BufferedWriter writer = getWriter(tableExport.getId())) {
+        try (BufferedWriter writer = getWriter(tableExport.getId(), extension)) {
             result
                 .map(record -> record.concat(System.lineSeparator()))
                 .subscribe(
@@ -64,11 +70,11 @@ public TableExport storeResults(TableExport tableExport, Observable<String> resu
     }
 
     @Override
-    public Observable<String> getResultsByID(String asyncQueryID) {
-        log.debug("getAsyncResultsByID");
+    public Observable<String> getResultsByID(String tableExportID) {
+        log.debug("getTableExportResultsByID");
 
         return Observable.using(
-                () -> getReader(asyncQueryID),
+                () -> getReader(tableExportID),
                 reader -> Observable.fromIterable(() -> new Iterator<String>() {
                     private String record = null;
 
@@ -93,21 +99,26 @@ public String next() {
                 BufferedReader::close);
     }
 
-    private BufferedReader getReader(String asyncQueryID) {
+    private BufferedReader getReader(String tableExportID) {
         try {
-            return Files.newBufferedReader(Paths.get(basePath + File.separator + asyncQueryID));
+            return Files.newBufferedReader(Paths.get(basePath + File.separator + tableExportID));
         } catch (IOException e) {
             log.debug(e.getMessage());
             throw new IllegalStateException(RETRIEVE_ERROR, e);
         }
     }
 
-    private BufferedWriter getWriter(String asyncQueryID) {
+    private BufferedWriter getWriter(String tableExportID, String extension) {
         try {
-            return Files.newBufferedWriter(Paths.get(basePath + File.separator + asyncQueryID));
+            return Files.newBufferedWriter(Paths.get(basePath + File.separator + tableExportID + extension));
         } catch (IOException e) {
             log.debug(e.getMessage());
             throw new IllegalStateException(STORE_ERROR, e);
         }
     }
+
+    @Override
+    public boolean isExtensionEnabled() {
+        return this.enableExtension;
+    }
 }
diff --git a/...-async/src/main/java/com/yahoo/elide/async/service/storageengine/ResultStorageEngine.java b/...-async/src/main/java/com/yahoo/elide/async/service/storageengine/ResultStorageEngine.java
@@ -26,8 +26,14 @@ public interface ResultStorageEngine {
 
     /**
      * Searches for the async query results by ID and returns the record.
-     * @param asyncQueryID is the query ID of the AsyncQuery
-     * @return returns the result associated with the AsyncQueryID
+     * @param tableExportID is the ID of the TableExport. It may include extension too if enabled.
+     * @return returns the result associated with the tableExportID
      */
-    public Observable<String> getResultsByID(String asyncQueryID);
+    public Observable<String> getResultsByID(String tableExportID);
+
+    /**
+     * Whether the result storage engine has enabled extensions for attachments.
+     * @return returns whether the file extensions are enabled
+     */
+    public boolean isExtensionEnabled();
 }
diff --git a/...-async/src/test/java/com/yahoo/elide/async/operation/GraphQLTableExportOperationTest.java b/...-async/src/test/java/com/yahoo/elide/async/operation/GraphQLTableExportOperationTest.java
@@ -77,7 +77,7 @@ public void setupMocks(@TempDir Path tempDir) {
         user = mock(User.class);
         requestScope = mock(RequestScope.class);
         asyncExecutorService = mock(AsyncExecutorService.class);
-        engine = new FileResultStorageEngine(tempDir.toString());
+        engine = new FileResultStorageEngine(tempDir.toString(), false);
         when(asyncExecutorService.getElide()).thenReturn(elide);
         when(requestScope.getApiVersion()).thenReturn(NO_VERSION);
         when(requestScope.getUser()).thenReturn(user);

diff --git a/...-async/src/test/java/com/yahoo/elide/async/operation/JsonAPITableExportOperationTest.java b/...-async/src/test/java/com/yahoo/elide/async/operation/JsonAPITableExportOperationTest.java
@@ -78,7 +78,7 @@ public void setupMocks(@TempDir Path tempDir) {
         user = mock(User.class);
         requestScope = mock(RequestScope.class);
         asyncExecutorService = mock(AsyncExecutorService.class);
-        engine = new FileResultStorageEngine(tempDir.toString());
+        engine = new FileResultStorageEngine(tempDir.toString(), true);
         when(asyncExecutorService.getElide()).thenReturn(elide);
         when(requestScope.getApiVersion()).thenReturn(NO_VERSION);
         when(requestScope.getUser()).thenReturn(user);
@@ -102,7 +102,7 @@ public void testProcessQuery() throws URISyntaxException, IOException  {
         TableExportResult queryResultObj = (TableExportResult) jsonAPIOperation.call();
 
         assertEquals(200, queryResultObj.getHttpStatus());
-        assertEquals("https://elide.io/export/edc4a871-dff2-4054-804e-d80075cf827d", queryResultObj.getUrl().toString());
+        assertEquals("https://elide.io/export/edc4a871-dff2-4054-804e-d80075cf827d.csv", queryResultObj.getUrl().toString());
         assertEquals(1, queryResultObj.getRecordCount());
         assertNull(queryResultObj.getMessage());
     }

diff --git a/...rc/test/java/com/yahoo/elide/async/service/storageengine/FileResultStorageEngineTest.java b/...rc/test/java/com/yahoo/elide/async/service/storageengine/FileResultStorageEngineTest.java
@@ -67,7 +67,7 @@ public void testStoreResultsFail(@TempDir File tempDir) {
     }
 
     private String readResultsFile(String path, String queryId) {
-        FileResultStorageEngine engine = new FileResultStorageEngine(path);
+        FileResultStorageEngine engine = new FileResultStorageEngine(path, false);
 
         return engine.getResultsByID(queryId).collect(() -> new StringBuilder(),
                 (resultBuilder, tempResult) -> {
@@ -80,7 +80,7 @@ private String readResultsFile(String path, String queryId) {
     }
 
     private void storeResultsFile(String path, String queryId, Observable<String> storable) {
-        FileResultStorageEngine engine = new FileResultStorageEngine(path);
+        FileResultStorageEngine engine = new FileResultStorageEngine(path, false);
         TableExport query = new TableExport();
         query.setId(queryId);
 

diff --git a/...lide/async/integration/tests/framework/AsyncIntegrationTestApplicationResourceConfig.java b/...lide/async/integration/tests/framework/AsyncIntegrationTestApplicationResourceConfig.java
@@ -116,7 +116,7 @@ protected void configure() {
                 // Create ResultStorageEngine
                 Path storageDestination = (Path) servletContext.getAttribute(STORAGE_DESTINATION_ATTR);
                 if (storageDestination != null) { // TableExport is enabled
-                    ResultStorageEngine resultStorageEngine = new FileResultStorageEngine(storageDestination.toAbsolutePath().toString());
+                    ResultStorageEngine resultStorageEngine = new FileResultStorageEngine(storageDestination.toAbsolutePath().toString(), false);
                     bind(resultStorageEngine).to(ResultStorageEngine.class).named("resultStorageEngine");
 
                     Map<ResultType, TableExportFormatter> supportedFormatters = new HashMap<>();

diff --git a/...el-config/src/main/java/com/yahoo/elide/modelconfig/validator/DynamicConfigValidator.java b/...el-config/src/main/java/com/yahoo/elide/modelconfig/validator/DynamicConfigValidator.java
@@ -194,7 +194,7 @@ public void readConfigs() throws IOException {
         readConfigs(fileLoader.loadResources());
     }
 
-    public void readConfigs(Map<String, ConfigFile> resourceMap) throws IOException {
+    public void readConfigs(Map<String, ConfigFile> resourceMap) {
         this.modelVariables = readVariableConfig(Config.MODELVARIABLE, resourceMap);
         this.elideSecurityConfig = readSecurityConfig(resourceMap);
         this.dbVariables = readVariableConfig(Config.DBVARIABLE, resourceMap);

diff --git a/...ot-autoconfigure/src/main/java/com/yahoo/elide/spring/config/ElideAsyncConfiguration.java b/...ot-autoconfigure/src/main/java/com/yahoo/elide/spring/config/ElideAsyncConfiguration.java
@@ -165,7 +165,7 @@ public AsyncAPIDAO buildAsyncAPIDAO(Elide elide) {
     public ResultStorageEngine buildResultStorageEngine(Elide elide, ElideConfigProperties settings,
             AsyncAPIDAO asyncQueryDAO) {
         FileResultStorageEngine resultStorageEngine = new FileResultStorageEngine(settings.getAsync().getExport()
-                .getStorageDestination());
+                .getStorageDestination(), settings.getAsync().getExport().isExtensionEnabled());
         return resultStorageEngine;
     }
 }
diff --git a/...autoconfigure/src/main/java/com/yahoo/elide/spring/config/ExportControllerProperties.java b/...autoconfigure/src/main/java/com/yahoo/elide/spring/config/ExportControllerProperties.java
@@ -20,6 +20,11 @@ public class ExportControllerProperties extends ControllerProperties {
      */
     private boolean skipCSVHeader = false;
 
+    /**
+     * Enable Adding Extension to table export attachments.
+     */
+    private boolean extensionEnabled = false;
+
     /**
      * The URL path prefix for the controller.
      */

diff --git a/elide-spring/elide-spring-boot-autoconfigure/src/test/java/example/tests/AsyncTest.java b/elide-spring/elide-spring-boot-autoconfigure/src/test/java/example/tests/AsyncTest.java
@@ -161,7 +161,7 @@ public void testExportDynamicModel() throws InterruptedException {
                         .body("data.attributes.status", equalTo("COMPLETE"))
                         .body("data.attributes.result.message", equalTo(null))
                         .body("data.attributes.result.url",
-                                equalTo("https://elide.io" + "/export/ba31ca4e-ed8f-4be0-a0f3-12088fa9265d"));
+                                equalTo("https://elide.io" + "/export/ba31ca4e-ed8f-4be0-a0f3-12088fa9265d.csv"));
 
                 // Validate GraphQL Response
                 String responseGraphQL = given()
@@ -176,15 +176,15 @@ public void testExportDynamicModel() throws InterruptedException {
 
                 String expectedResponse = "{\"data\":{\"tableExport\":{\"edges\":[{\"node\":{\"id\":\"ba31ca4e-ed8f-4be0-a0f3-12088fa9265d\","
                         + "\"queryType\":\"GRAPHQL_V1_0\",\"status\":\"COMPLETE\",\"resultType\":\"CSV\","
-                        + "\"result\":{\"url\":\"https://elide.io/export/ba31ca4e-ed8f-4be0-a0f3-12088fa9265d\",\"httpStatus\":200,\"recordCount\":1}}}]}}}";
+                        + "\"result\":{\"url\":\"https://elide.io/export/ba31ca4e-ed8f-4be0-a0f3-12088fa9265d.csv\",\"httpStatus\":200,\"recordCount\":1}}}]}}}";
 
                 assertEquals(expectedResponse, responseGraphQL);
                 break;
             }
             assertEquals("PROCESSING", outputResponse, "Async Query has failed.");
         }
         when()
-                .get("/export/ba31ca4e-ed8f-4be0-a0f3-12088fa9265d")
+                .get("/export/ba31ca4e-ed8f-4be0-a0f3-12088fa9265d.csv")
                 .then()
                 .statusCode(HttpStatus.SC_OK);
     }
@@ -235,7 +235,7 @@ public void testExportStaticModel() throws InterruptedException {
                         .body("data.attributes.status", equalTo("COMPLETE"))
                         .body("data.attributes.result.message", equalTo(null))
                         .body("data.attributes.result.url",
-                                equalTo("https://elide.io" + "/export/ba31ca4e-ed8f-4be0-a0f3-12088fa9264d"));
+                                equalTo("https://elide.io" + "/export/ba31ca4e-ed8f-4be0-a0f3-12088fa9264d.csv"));
 
                 // Validate GraphQL Response
                 String responseGraphQL = given()
@@ -250,15 +250,15 @@ public void testExportStaticModel() throws InterruptedException {
 
                 String expectedResponse = "{\"data\":{\"tableExport\":{\"edges\":[{\"node\":{\"id\":\"ba31ca4e-ed8f-4be0-a0f3-12088fa9264d\","
                         + "\"queryType\":\"GRAPHQL_V1_0\",\"status\":\"COMPLETE\",\"resultType\":\"CSV\","
-                        + "\"result\":{\"url\":\"https://elide.io/export/ba31ca4e-ed8f-4be0-a0f3-12088fa9264d\",\"httpStatus\":200,\"recordCount\":2}}}]}}}";
+                        + "\"result\":{\"url\":\"https://elide.io/export/ba31ca4e-ed8f-4be0-a0f3-12088fa9264d.csv\",\"httpStatus\":200,\"recordCount\":2}}}]}}}";
 
                 assertEquals(expectedResponse, responseGraphQL);
                 break;
             }
             assertEquals("PROCESSING", outputResponse, "Async Query has failed.");
         }
         when()
-                .get("/export/ba31ca4e-ed8f-4be0-a0f3-12088fa9264d")
+                .get("/export/ba31ca4e-ed8f-4be0-a0f3-12088fa9264d.csv")
                 .then()
                 .statusCode(HttpStatus.SC_OK);
     }

diff --git a/elide-spring/elide-spring-boot-autoconfigure/src/test/resources/application.yaml b/elide-spring/elide-spring-boot-autoconfigure/src/test/resources/application.yaml
@@ -26,6 +26,7 @@ elide:
     export:
       enabled: true
       path: /export
+      extensionEnabled: true
   dynamic-config:
     path: src/test/resources/configs
     enabled: true

diff --git a/elide-standalone/src/main/java/com/yahoo/elide/standalone/config/ElideResourceConfig.java b/elide-standalone/src/main/java/com/yahoo/elide/standalone/config/ElideResourceConfig.java
@@ -165,7 +165,8 @@ protected void bindAsync(
 
                 ResultStorageEngine resultStorageEngine = asyncProperties.getResultStorageEngine();
                 if (resultStorageEngine == null) {
-                    resultStorageEngine = new FileResultStorageEngine(asyncProperties.getStorageDestination());
+                    resultStorageEngine = new FileResultStorageEngine(asyncProperties.getStorageDestination(),
+                            asyncProperties.enableExtension());
                 }
                 bind(resultStorageEngine).to(ResultStorageEngine.class).named("resultStorageEngine");
 

diff --git a/...ndalone/src/main/java/com/yahoo/elide/standalone/config/ElideStandaloneAsyncSettings.java b/...ndalone/src/main/java/com/yahoo/elide/standalone/config/ElideStandaloneAsyncSettings.java
@@ -117,6 +117,16 @@ default boolean enableExport() {
         return false;
     }
 
+    /**
+     * Enable the addition of extensions to Export attachments.
+     * If false, the attachments will be downloaded without extensions.
+     *
+     * @return Default: False
+     */
+    default boolean enableExtension() {
+        return false;
+    }
+
     /**
      * Skip generating Header when exporting in CSV format.
      *