TableExportHook Binding (#1802)

elide-spring/elide-spring-boot-autoconfigure/src/main/java/com/yahoo/elide/spring/config/AsyncProperties.java

@@ -54,4 +54,9 @@ public class AsyncProperties {
      * Whether or not the async feature is enabled.
      */
     private boolean enabled = false;
+
+    /**
+     * Settings for the export controller.
+     */
+    private ExportControllerProperties export;
 }

elide-spring/elide-spring-boot-autoconfigure/src/main/java/com/yahoo/elide/spring/config/ElideAsyncConfiguration.java

@@ -9,15 +9,26 @@
 import static com.yahoo.elide.annotation.LifeCycleHookBinding.Operation.READ;
 import static com.yahoo.elide.annotation.LifeCycleHookBinding.TransactionPhase.POSTCOMMIT;
 import static com.yahoo.elide.annotation.LifeCycleHookBinding.TransactionPhase.PRESECURITY;
+
 import com.yahoo.elide.Elide;
+import com.yahoo.elide.async.export.formatter.CSVExportFormatter;
+import com.yahoo.elide.async.export.formatter.JSONExportFormatter;
+import com.yahoo.elide.async.export.formatter.TableExportFormatter;
 import com.yahoo.elide.async.hooks.AsyncQueryHook;
+import com.yahoo.elide.async.hooks.TableExportHook;
+import com.yahoo.elide.async.models.AsyncAPI;
 import com.yahoo.elide.async.models.AsyncQuery;
+import com.yahoo.elide.async.models.ResultType;
+import com.yahoo.elide.async.models.TableExport;
 import com.yahoo.elide.async.service.AsyncCleanerService;
 import com.yahoo.elide.async.service.AsyncExecutorService;
 import com.yahoo.elide.async.service.dao.AsyncAPIDAO;
 import com.yahoo.elide.async.service.dao.DefaultAsyncAPIDAO;
 import com.yahoo.elide.async.service.storageengine.ResultStorageEngine;
 import com.yahoo.elide.core.dictionary.EntityDictionary;
+import com.yahoo.elide.core.exceptions.InvalidOperationException;
+import com.yahoo.elide.core.security.RequestScope;
+
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
@@ -27,6 +38,9 @@
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 
+import java.util.HashMap;
+import java.util.Map;
+
 /**
  * Async Configuration For Elide Services.  Override any of the beans (by defining your own)
  * and setting flags to disable in properties to change the default behavior.
@@ -50,20 +64,58 @@ public class ElideAsyncConfiguration {
     public AsyncExecutorService buildAsyncExecutorService(Elide elide, ElideConfigProperties settings,
             AsyncAPIDAO asyncQueryDao, EntityDictionary dictionary,
             @Autowired(required = false) ResultStorageEngine resultStorageEngine) {
-        AsyncExecutorService.init(elide, settings.getAsync().getThreadPoolSize(),
+        AsyncProperties asyncProperties = settings.getAsync();
+        AsyncExecutorService.init(elide, asyncProperties.getThreadPoolSize(),
                 asyncQueryDao, resultStorageEngine);
         AsyncExecutorService asyncExecutorService = AsyncExecutorService.getInstance();
 
         // Binding AsyncQuery LifeCycleHook
         AsyncQueryHook asyncQueryHook = new AsyncQueryHook(asyncExecutorService,
-                settings.getAsync().getMaxAsyncAfterSeconds());
+                asyncProperties.getMaxAsyncAfterSeconds());
         dictionary.bindTrigger(AsyncQuery.class, READ, PRESECURITY, asyncQueryHook, false);
         dictionary.bindTrigger(AsyncQuery.class, CREATE, POSTCOMMIT, asyncQueryHook, false);
         dictionary.bindTrigger(AsyncQuery.class, CREATE, PRESECURITY, asyncQueryHook, false);
 
+        // Initialize the Formatters.
+        boolean skipCSVHeader = asyncProperties.getExport() != null ? asyncProperties.getExport().isSkipCSVHeader()
+            : false;
+        Map<ResultType, TableExportFormatter> supportedFormatters = new HashMap<ResultType, TableExportFormatter>();
+        supportedFormatters.put(ResultType.CSV, new CSVExportFormatter(skipCSVHeader));
+        supportedFormatters.put(ResultType.JSON, new JSONExportFormatter());
+
+        // Binding TableExport LifeCycleHook
+        TableExportHook tableExportHook = getTableExportHook(asyncExecutorService, settings, supportedFormatters);
+        dictionary.bindTrigger(TableExport.class, READ, PRESECURITY, tableExportHook, false);
+        dictionary.bindTrigger(TableExport.class, CREATE, POSTCOMMIT, tableExportHook, false);
+        dictionary.bindTrigger(TableExport.class, CREATE, PRESECURITY, tableExportHook, false);
+
         return AsyncExecutorService.getInstance();
     }
 
+    // TODO Remove this method when ElideSettings has all the settings.
+    // Then the check can be done in TableExportHook.
+    // Trying to avoid adding too many individual properties to ElideSettings for now.
+    // https://github.com/yahoo/elide/issues/1803
+    private TableExportHook getTableExportHook(AsyncExecutorService asyncExecutorService,
+            ElideConfigProperties settings, Map<ResultType, TableExportFormatter> supportedFormatters) {
+        boolean exportEnabled = settings.getAsync().getExport() != null ? settings.getAsync().getExport().isEnabled()
+                : false;
+        TableExportHook tableExportHook = null;
+        if (exportEnabled) {
+            tableExportHook = new TableExportHook(asyncExecutorService, settings.getAsync().getMaxAsyncAfterSeconds(),
+                    supportedFormatters);
+        } else {
+            tableExportHook = new TableExportHook(asyncExecutorService, settings.getAsync().getMaxAsyncAfterSeconds(),
+                    supportedFormatters) {
+                @Override
+                public void validateOptions(AsyncAPI export, RequestScope requestScope) {
+                    throw new InvalidOperationException("TableExport is not supported.");
+                }
+            };
+        }
+        return tableExportHook;
+    }
+
     /**
      * Configure the AsyncCleanerService used for cleaning up async query requests.
      * @param elide elideObject.

elide-spring/elide-spring-boot-autoconfigure/src/main/java/com/yahoo/elide/spring/config/ElideAutoConfiguration.java

@@ -148,6 +148,12 @@ public Elide initializeElide(EntityDictionary dictionary,
                 .withJsonApiPath(settings.getJsonApi().getPath())
                 .withGraphQLApiPath(settings.getGraphql().getPath());
 
+        if (settings.getAsync() != null
+                && settings.getAsync().getExport() != null
+                && settings.getAsync().getExport().isEnabled()) {
+            builder.withDownloadApiPath(settings.getAsync().getExport().getPath());
+        }
+
         if (settings.getJsonApi() != null
                 && settings.getJsonApi().isEnabled()
                 && settings.getJsonApi().isEnableLinks()) {

elide-spring/elide-spring-boot-autoconfigure/src/main/java/com/yahoo/elide/spring/config/ExportControllerProperties.java

@@ -0,0 +1,25 @@
+/*
+ * Copyright 2019, Yahoo Inc.
+ * Licensed under the Apache License, Version 2.0
+ * See LICENSE file in project root for terms.
+ */
+package com.yahoo.elide.spring.config;
+
+import lombok.Data;
+
+/**
+ * Extra controller properties for the export endpoint.
+ */
+@Data
+public class ExportControllerProperties extends ControllerProperties {
+
+    /**
+     * Skip including Header in CSV formatted export.
+     */
+    private boolean skipCSVHeader = false;
+
+    /**
+     * The URL path prefix for the controller.
+     */
+    private String path = "/export";
+}

elide-standalone/src/main/java/com/yahoo/elide/standalone/config/ElideResourceConfig.java

@@ -11,15 +11,24 @@
 import static com.yahoo.elide.annotation.LifeCycleHookBinding.TransactionPhase.PRESECURITY;
 import com.yahoo.elide.Elide;
 import com.yahoo.elide.ElideSettings;
+import com.yahoo.elide.async.export.formatter.CSVExportFormatter;
+import com.yahoo.elide.async.export.formatter.JSONExportFormatter;
+import com.yahoo.elide.async.export.formatter.TableExportFormatter;
 import com.yahoo.elide.async.hooks.AsyncQueryHook;
+import com.yahoo.elide.async.hooks.TableExportHook;
+import com.yahoo.elide.async.models.AsyncAPI;
 import com.yahoo.elide.async.models.AsyncQuery;
+import com.yahoo.elide.async.models.ResultType;
+import com.yahoo.elide.async.models.TableExport;
 import com.yahoo.elide.async.service.AsyncCleanerService;
 import com.yahoo.elide.async.service.AsyncExecutorService;
 import com.yahoo.elide.async.service.dao.AsyncAPIDAO;
 import com.yahoo.elide.async.service.dao.DefaultAsyncAPIDAO;
 import com.yahoo.elide.async.service.storageengine.ResultStorageEngine;
 import com.yahoo.elide.core.datastore.DataStore;
 import com.yahoo.elide.core.dictionary.EntityDictionary;
+import com.yahoo.elide.core.exceptions.InvalidOperationException;
+import com.yahoo.elide.core.security.RequestScope;
 import com.yahoo.elide.datastores.aggregation.AggregationDataStore;
 import com.yahoo.elide.datastores.aggregation.QueryEngine;
 import com.yahoo.elide.datastores.aggregation.metadata.MetaDataStore;
@@ -36,7 +45,9 @@
 import org.glassfish.jersey.server.ResourceConfig;
 import lombok.extern.slf4j.Slf4j;
 
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 import java.util.function.Consumer;
@@ -154,6 +165,19 @@ protected void configure() {
                     dictionary.bindTrigger(AsyncQuery.class, CREATE, POSTCOMMIT, asyncQueryHook, false);
                     dictionary.bindTrigger(AsyncQuery.class, CREATE, PRESECURITY, asyncQueryHook, false);
 
+                    // Initialize the Formatters.
+                    Map<ResultType, TableExportFormatter> supportedFormatters = new HashMap<ResultType,
+                        TableExportFormatter>();
+                    supportedFormatters.put(ResultType.CSV, new CSVExportFormatter(asyncProperties.skipCSVHeader()));
+                    supportedFormatters.put(ResultType.JSON, new JSONExportFormatter());
+
+                    // Binding TableExport LifeCycleHook
+                    TableExportHook tableExportHook = getTableExportHook(AsyncExecutorService.getInstance(),
+                            asyncProperties, supportedFormatters);
+                    dictionary.bindTrigger(TableExport.class, READ, PRESECURITY, tableExportHook, false);
+                    dictionary.bindTrigger(TableExport.class, CREATE, POSTCOMMIT, tableExportHook, false);
+                    dictionary.bindTrigger(TableExport.class, CREATE, PRESECURITY, tableExportHook, false);
+
                     // Binding async cleanup service
                     if (asyncProperties.enableCleanup()) {
                         AsyncCleanerService.init(elide, asyncProperties.getMaxRunTimeSeconds(),
@@ -216,4 +240,26 @@ public static HealthCheckRegistry getHealthCheckRegistry() {
 
         return healthCheckRegistry;
     }
+
+    // TODO Remove this method when ElideSettings has all the settings.
+    // Then the check can be done in TableExportHook.
+    // Trying to avoid adding too many individual properties to ElideSettings for now.
+    // https://github.com/yahoo/elide/issues/1803
+    private TableExportHook getTableExportHook(AsyncExecutorService asyncExecutorService,
+            ElideStandaloneAsyncSettings asyncProperties, Map<ResultType, TableExportFormatter> supportedFormatters) {
+        TableExportHook tableExportHook = null;
+        if (asyncProperties.enableExport()) {
+            tableExportHook = new TableExportHook(asyncExecutorService, asyncProperties.getMaxAsyncAfterSeconds(),
+                    supportedFormatters);
+        } else {
+            tableExportHook = new TableExportHook(asyncExecutorService, asyncProperties.getMaxAsyncAfterSeconds(),
+                    supportedFormatters) {
+                @Override
+                public void validateOptions(AsyncAPI export, RequestScope requestScope) {
+                    throw new InvalidOperationException("TableExport is not supported.");
+                }
+            };
+        }
+        return tableExportHook;
+    }
 }

elide-standalone/src/main/java/com/yahoo/elide/standalone/config/ElideStandaloneAsyncSettings.java

@@ -95,4 +95,31 @@ default AsyncAPIDAO getAPIDAO() {
     default ResultStorageEngine getResultStorageEngine() {
         return null;
     }
+
+    /**
+     * API root path specification for the export endpoint
+     *
+     * @return Default: /export
+     */
+    default String getExportApiPathSpec() {
+        return "/export/*";
+    }
+
+    /**
+     * Enable the Export endpoint. If false, the endpoint and export support will be disabled.
+     *
+     * @return Default: False
+     */
+    default boolean enableExport() {
+        return false;
+    }
+
+    /**
+     * Skip generating Header when exporting in CSV format.
+     *
+     * @return Default: False
+     */
+    default boolean skipCSVHeader() {
+        return false;
+    }
 }

elide-standalone/src/main/java/com/yahoo/elide/standalone/config/ElideStandaloneSettings.java

@@ -100,8 +100,12 @@ default ElideSettings getElideSettings(EntityDictionary dictionary, DataStore da
                 .withGraphQLApiPath(getGraphQLApiPathSpec().replaceAll("/\\*", ""))
                 .withAuditLogger(getAuditLogger());
 
+        if (getAsyncProperties().enableExport()) {
+            builder.withDownloadApiPath(getAsyncProperties().getExportApiPathSpec().replaceAll("/\\*", ""));
+        }
+
         if (enableISO8601Dates()) {
-            builder = builder.withISO8601Dates("yyyy-MM-dd'T'HH:mm'Z'", TimeZone.getTimeZone("UTC"));
+            builder.withISO8601Dates("yyyy-MM-dd'T'HH:mm'Z'", TimeZone.getTimeZone("UTC"));
         }
 
         return builder.build();