-
Notifications
You must be signed in to change notification settings - Fork 141
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Read extra Spark submit parameters from cluster settings (#2219)
* Add default setting for Spark execution engine Signed-off-by: Chen Dai <[email protected]> * Pass extra parameters to Spark dispatcher Signed-off-by: Chen Dai <[email protected]> * Wrap read default setting file with previlege action Signed-off-by: Chen Dai <[email protected]> * Fix spotless format Signed-off-by: Chen Dai <[email protected]> * Use input stream to read default config file Signed-off-by: Chen Dai <[email protected]> * Add UT for dispatcher Signed-off-by: Chen Dai <[email protected]> * Add more UT Signed-off-by: Chen Dai <[email protected]> * Remove default config setting Signed-off-by: Chen Dai <[email protected]> * Fix spotless check in spark module Signed-off-by: Chen Dai <[email protected]> * Refactor test code Signed-off-by: Chen Dai <[email protected]> * Add more UT on config class Signed-off-by: Chen Dai <[email protected]> --------- Signed-off-by: Chen Dai <[email protected]>
- Loading branch information
Showing
10 changed files
with
251 additions
and
123 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
30 changes: 30 additions & 0 deletions
30
spark/src/test/java/org/opensearch/sql/spark/asyncquery/model/SparkSubmitParametersTest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,30 @@ | ||
/* | ||
* Copyright OpenSearch Contributors | ||
* SPDX-License-Identifier: Apache-2.0 | ||
*/ | ||
|
||
package org.opensearch.sql.spark.asyncquery.model; | ||
|
||
import static org.junit.jupiter.api.Assertions.assertNotNull; | ||
import static org.junit.jupiter.api.Assertions.assertTrue; | ||
|
||
import org.junit.jupiter.api.Test; | ||
|
||
public class SparkSubmitParametersTest { | ||
|
||
@Test | ||
public void testBuildWithoutExtraParameters() { | ||
String params = SparkSubmitParameters.Builder.builder().build().toString(); | ||
|
||
assertNotNull(params); | ||
} | ||
|
||
@Test | ||
public void testBuildWithExtraParameters() { | ||
String params = | ||
SparkSubmitParameters.Builder.builder().extraParameters("--conf A=1").build().toString(); | ||
|
||
// Assert the conf is included with a space | ||
assertTrue(params.endsWith(" --conf A=1")); | ||
} | ||
} |
Oops, something went wrong.