Skip to content

Commit

Permalink
Fix comments
Browse files Browse the repository at this point in the history
Signed-off-by: Tomoyuki Morita <[email protected]>
  • Loading branch information
ykmr1224 committed Jun 28, 2024
1 parent 5bb363c commit 5dac199
Show file tree
Hide file tree
Showing 4 changed files with 36 additions and 5 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,12 @@
import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext;
import org.opensearch.sql.spark.dispatcher.model.DispatchQueryRequest;

/** Compose Spark parameter based on DataSourceMetadata */
/**
* Compose Spark parameters specific to the {@link
* org.opensearch.sql.datasource.model.DataSourceType} based on the {@link DataSourceMetadata}. For
* the parameters not specific to {@link org.opensearch.sql.datasource.model.DataSourceType}, please
* use {@link GeneralSparkParameterComposer}.
*/
public interface DataSourceSparkParameterComposer {
void compose(
DataSourceMetadata dataSourceMetadata,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,22 @@
import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext;
import org.opensearch.sql.spark.dispatcher.model.DispatchQueryRequest;

/** Compose spark submit parameters. See {@link SparkParameterComposerCollection}. */
/**
* Compose spark submit parameters based on the request and context. For {@link
* org.opensearch.sql.datasource.model.DataSourceType} specific parameters, please use {@link
* DataSourceSparkParameterComposer}. See {@link SparkParameterComposerCollection}.
*/
public interface GeneralSparkParameterComposer {

/**
* Modify sparkSubmitParameters based on dispatchQueryRequest and context.
*
* @param sparkSubmitParameters Implementation of this method will modify this.
* @param dispatchQueryRequest Request. Implementation can refer it to compose
* sparkSubmitParameters.
* @param context Context of the request. Implementation can refer it to compose
* sparkSubmitParameters.
*/
void compose(
SparkSubmitParameters sparkSubmitParameters,
DispatchQueryRequest dispatchQueryRequest,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,19 +16,30 @@
import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext;
import org.opensearch.sql.spark.dispatcher.model.DispatchQueryRequest;

/** Stores Spark parameter composers and dispatch compose request to each composer */
/**
* Stores Spark parameter composers and dispatch compose request to each composer. Composers should
* be registered during initialization such as in Guice Module.
*/
public class SparkParameterComposerCollection {
Collection<GeneralSparkParameterComposer> generalComposers = new ArrayList<>();
Map<DataSourceType, Collection<DataSourceSparkParameterComposer>> datasourceComposers =
private Collection<GeneralSparkParameterComposer> generalComposers = new ArrayList<>();
private Map<DataSourceType, Collection<DataSourceSparkParameterComposer>> datasourceComposers =
new HashMap<>();

/**
* Register composers for specific DataSourceType. The registered composer is called only if the
* request is for the dataSourceType.
*/
public void register(DataSourceType dataSourceType, DataSourceSparkParameterComposer composer) {
if (!datasourceComposers.containsKey(dataSourceType)) {
datasourceComposers.put(dataSourceType, new LinkedList<>());
}
datasourceComposers.get(dataSourceType).add(composer);
}

/**
* Register general composer. The composer is called when spark parameter is generated regardless
* of datasource type.
*/
public void register(GeneralSparkParameterComposer composer) {
generalComposers.add(composer);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
import org.apache.commons.lang3.StringUtils;
import org.opensearch.sql.common.setting.Settings;

/** Load SparkExecutionEngineConfigClusterSetting from settings with privilege check. */
@RequiredArgsConstructor
public class SparkExecutionEngineConfigClusterSettingLoader {
private final Settings settings;
Expand Down

0 comments on commit 5dac199

Please sign in to comment.