From 5dac19974e76b1788af7bc8302534af752aca032 Mon Sep 17 00:00:00 2001 From: Tomoyuki Morita Date: Fri, 28 Jun 2024 11:45:27 -0700 Subject: [PATCH] Fix comments Signed-off-by: Tomoyuki Morita --- .../DataSourceSparkParameterComposer.java | 7 ++++++- .../GeneralSparkParameterComposer.java | 16 +++++++++++++++- .../SparkParameterComposerCollection.java | 17 ++++++++++++++--- ...ecutionEngineConfigClusterSettingLoader.java | 1 + 4 files changed, 36 insertions(+), 5 deletions(-) diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/parameter/DataSourceSparkParameterComposer.java b/async-query-core/src/main/java/org/opensearch/sql/spark/parameter/DataSourceSparkParameterComposer.java index 4411bfe22d..324889b6e0 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/parameter/DataSourceSparkParameterComposer.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/parameter/DataSourceSparkParameterComposer.java @@ -9,7 +9,12 @@ import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext; import org.opensearch.sql.spark.dispatcher.model.DispatchQueryRequest; -/** Compose Spark parameter based on DataSourceMetadata */ +/** + * Compose Spark parameters specific to the {@link + * org.opensearch.sql.datasource.model.DataSourceType} based on the {@link DataSourceMetadata}. For + * the parameters not specific to {@link org.opensearch.sql.datasource.model.DataSourceType}, please + * use {@link GeneralSparkParameterComposer}. + */ public interface DataSourceSparkParameterComposer { void compose( DataSourceMetadata dataSourceMetadata, diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/parameter/GeneralSparkParameterComposer.java b/async-query-core/src/main/java/org/opensearch/sql/spark/parameter/GeneralSparkParameterComposer.java index b5200d550c..c3d46ba5c6 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/parameter/GeneralSparkParameterComposer.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/parameter/GeneralSparkParameterComposer.java @@ -8,8 +8,22 @@ import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext; import org.opensearch.sql.spark.dispatcher.model.DispatchQueryRequest; -/** Compose spark submit parameters. See {@link SparkParameterComposerCollection}. */ +/** + * Compose spark submit parameters based on the request and context. For {@link + * org.opensearch.sql.datasource.model.DataSourceType} specific parameters, please use {@link + * DataSourceSparkParameterComposer}. See {@link SparkParameterComposerCollection}. + */ public interface GeneralSparkParameterComposer { + + /** + * Modify sparkSubmitParameters based on dispatchQueryRequest and context. + * + * @param sparkSubmitParameters Implementation of this method will modify this. + * @param dispatchQueryRequest Request. Implementation can refer it to compose + * sparkSubmitParameters. + * @param context Context of the request. Implementation can refer it to compose + * sparkSubmitParameters. + */ void compose( SparkSubmitParameters sparkSubmitParameters, DispatchQueryRequest dispatchQueryRequest, diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/parameter/SparkParameterComposerCollection.java b/async-query-core/src/main/java/org/opensearch/sql/spark/parameter/SparkParameterComposerCollection.java index 281759afd2..a6a88738bf 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/parameter/SparkParameterComposerCollection.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/parameter/SparkParameterComposerCollection.java @@ -16,12 +16,19 @@ import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext; import org.opensearch.sql.spark.dispatcher.model.DispatchQueryRequest; -/** Stores Spark parameter composers and dispatch compose request to each composer */ +/** + * Stores Spark parameter composers and dispatch compose request to each composer. Composers should + * be registered during initialization such as in Guice Module. + */ public class SparkParameterComposerCollection { - Collection generalComposers = new ArrayList<>(); - Map> datasourceComposers = + private Collection generalComposers = new ArrayList<>(); + private Map> datasourceComposers = new HashMap<>(); + /** + * Register composers for specific DataSourceType. The registered composer is called only if the + * request is for the dataSourceType. + */ public void register(DataSourceType dataSourceType, DataSourceSparkParameterComposer composer) { if (!datasourceComposers.containsKey(dataSourceType)) { datasourceComposers.put(dataSourceType, new LinkedList<>()); @@ -29,6 +36,10 @@ public void register(DataSourceType dataSourceType, DataSourceSparkParameterComp datasourceComposers.get(dataSourceType).add(composer); } + /** + * Register general composer. The composer is called when spark parameter is generated regardless + * of datasource type. + */ public void register(GeneralSparkParameterComposer composer) { generalComposers.add(composer); } diff --git a/async-query/src/main/java/org/opensearch/sql/spark/config/SparkExecutionEngineConfigClusterSettingLoader.java b/async-query/src/main/java/org/opensearch/sql/spark/config/SparkExecutionEngineConfigClusterSettingLoader.java index 561a4653a5..73b057ca5c 100644 --- a/async-query/src/main/java/org/opensearch/sql/spark/config/SparkExecutionEngineConfigClusterSettingLoader.java +++ b/async-query/src/main/java/org/opensearch/sql/spark/config/SparkExecutionEngineConfigClusterSettingLoader.java @@ -14,6 +14,7 @@ import org.apache.commons.lang3.StringUtils; import org.opensearch.sql.common.setting.Settings; +/** Load SparkExecutionEngineConfigClusterSetting from settings with privilege check. */ @RequiredArgsConstructor public class SparkExecutionEngineConfigClusterSettingLoader { private final Settings settings;