diff --git a/cli/dcos-spark/main.go b/cli/dcos-spark/main.go index 690be026d8f07..a57a98ed30f36 100644 --- a/cli/dcos-spark/main.go +++ b/cli/dcos-spark/main.go @@ -24,6 +24,7 @@ type SparkCommand struct { submitDockerImage string submitDcosSpace string submitEnv map[string]string + secretPath string statusSkipMessage bool @@ -145,6 +146,24 @@ func (cmd *SparkCommand) runWebui(a *kingpin.Application, e *kingpin.ParseElemen return nil } +func (cmd *SparkCommand) runGenerateSecret(a *kingpin.Application, e *kingpin.ParseElement, c *kingpin.ParseContext) error { + secret, err := GetRandomStringSecret() + + if err != nil { + return err + } + + _, err = client.RunCLICommand( + "security", "secrets", "create", cmd.secretPath, fmt.Sprintf("-v %s", secret)) + + if err != nil { + log.Fatalf("Unable to create secret, %s", err) + return err + } + + return err +} + func handleCommands(app *kingpin.Application) { cmd := &SparkCommand{submitEnv: make(map[string]string)} run := app.Command("run", "Submit a job to the Spark Mesos Dispatcher").Action(cmd.runSubmit) @@ -181,6 +200,10 @@ func handleCommands(app *kingpin.Application) { kill := app.Command("kill", "Aborts a submitted Spark job").Action(cmd.runKill) kill.Arg("submission-id", "The ID of the Spark job").Required().StringVar(&cmd.submissionId) + secret := app.Command("secret", "Make a shared secret, used for RPC authentication"). + Action(cmd.runGenerateSecret) + secret.Arg("secret_path", "path and name for the secret").Required().StringVar(&cmd.secretPath) + app.Command("webui", "Returns the Spark Web UI URL").Action(cmd.runWebui) } diff --git a/cli/dcos-spark/secretGenerator.go b/cli/dcos-spark/secretGenerator.go new file mode 100644 index 0000000000000..c53eb3bae1454 --- /dev/null +++ b/cli/dcos-spark/secretGenerator.go @@ -0,0 +1,24 @@ +package main + +import ( + "crypto/rand" + "encoding/base64" +) + +const KEYLENGTH = 128 + +func generateRandomBytes(n int) ([]byte, error) { + // https://elithrar.github.io/article/generating-secure-random-numbers-crypto-rand/ + b := make([]byte, n) + _, err := rand.Read(b) + if err != nil { + return nil, err + } + return b, nil +} + +func GetRandomStringSecret() (string, error) { + b, err := generateRandomBytes(KEYLENGTH) + return base64.URLEncoding.EncodeToString(b), err +} + diff --git a/cli/dcos-spark/submit_builder.go b/cli/dcos-spark/submit_builder.go index 1c711da20f43b..fca22a2ec533e 100644 --- a/cli/dcos-spark/submit_builder.go +++ b/cli/dcos-spark/submit_builder.go @@ -3,14 +3,12 @@ package main import ( "bufio" "bytes" - "encoding/base64" "encoding/json" "errors" "fmt" "github.com/mesosphere/dcos-commons/cli/client" "github.com/mesosphere/dcos-commons/cli/config" "gopkg.in/alecthomas/kingpin.v3-unstable" - "io/ioutil" "log" "net/url" "os" @@ -22,6 +20,10 @@ var keyWhitespaceValPattern = regexp.MustCompile("(.+)\\s+(.+)") var backslashNewlinePattern = regexp.MustCompile("\\s*\\\\s*\\n\\s+") var collapseSpacesPattern = regexp.MustCompile(`[\s\p{Zs}]{2,}`) +const SECRET_REFERENCE_TEMPLATE = "spark.mesos.%s.secret.names" +const SECRET_FILENAME_TEMPLATE = "spark.mesos.%s.secret.filenames" +const SECRET_ENVKEY_TEMPLATE = "spark.mesos.%s.secret.envkeys" + type sparkVal struct { flagName string propName string @@ -241,8 +243,7 @@ func sparkSubmitHelp() string { func prepareBase64Secret(secretPath string, isEncoded bool) string { ss := strings.Split(secretPath, "/") s := ss[len(ss) - 1] // The secret file without any slashes - // TODO document how secret formatting works w.r.t decoding - // secrets with __dcos_base64__ will be decoded by mesos or spark + // secrets with __dcos_base64__ will be decoded by Mesos if strings.HasPrefix(s, "__dcos_base64__") || strings.HasSuffix(s, "base64") { // if we have the .base64, maintain the whole thing spark-env will decode it return strings.TrimPrefix(s, "__dcos_base64__") @@ -255,9 +256,11 @@ func prepareBase64Secret(secretPath string, isEncoded bool) string { } func addArgsForFileBasedSecret(args *sparkArgs, secretPath, property string) { - args.properties["spark.mesos.driver.secret.names"] = secretPath + secretRefProp := fmt.Sprintf(SECRET_REFERENCE_TEMPLATE, "driver") + secretFileProp := fmt.Sprintf(SECRET_FILENAME_TEMPLATE, "driver") + appendToProperty(secretRefProp, secretPath, args) + appendToProperty(secretFileProp, prepareBase64Secret(secretPath, true), args) args.properties[property] = prepareBase64Secret(secretPath, false) - args.properties["spark.mesos.driver.secret.filenames"] = prepareBase64Secret(secretPath, true) } func setupKerberosAuthArgs(args *sparkArgs) error { @@ -271,9 +274,9 @@ func setupKerberosAuthArgs(args *sparkArgs) error { return nil } if args.tgtSecretValue != "" { // using secret by value - args.properties["spark.mesos.driver.secret.values"] = args.tgtSecretValue + appendToProperty("spark.mesos.driver.secret.values", args.tgtSecretValue, args) args.properties["spark.mesos.driverEnv.KRB5CCNAME"] = "tgt" - args.properties["spark.mesos.driver.secret.filenames"] = "tgt.base64" + appendToProperty(fmt.Sprintf(SECRET_FILENAME_TEMPLATE, "driver"), "tgt.base64", args) return nil } return errors.New(fmt.Sprintf("Unable to add Kerberos args, got args %s", args)) @@ -306,9 +309,9 @@ func setupTLSArgs(args *sparkArgs) { taskTypes :=[]string{"driver", "executor"} for _, taskType := range taskTypes { - appendToProperty(fmt.Sprintf("spark.mesos.%s.secret.names", taskType), joinedPaths, args) - appendToProperty(fmt.Sprintf("spark.mesos.%s.secret.filenames", taskType), joinedFilenames, args) - appendToPropertyIfSet(fmt.Sprintf("spark.mesos.%s.secret.envkeys", taskType), joinedEnvkeys, args) + appendToProperty(fmt.Sprintf(SECRET_REFERENCE_TEMPLATE, taskType), joinedPaths, args) + appendToProperty(fmt.Sprintf(SECRET_FILENAME_TEMPLATE, taskType), joinedFilenames, args) + appendToPropertyIfSet(fmt.Sprintf(SECRET_ENVKEY_TEMPLATE, taskType), joinedEnvkeys, args) } // Passwords @@ -510,20 +513,6 @@ func appendToPropertyIfSet(propValue, toAppend string, args *sparkArgs) { } } -func getBase64Content(path string) string { - log.Printf("Opening file %s", path) - data, err := ioutil.ReadFile(path) - if err != nil { - log.Fatal(err) - } - - var encodebuf bytes.Buffer - encoder := base64.NewEncoder(base64.StdEncoding, &encodebuf) - encoder.Write(data) - encoder.Close() // must be called before returning string to ensure flush - return encodebuf.String() -} - func buildSubmitJson(cmd *SparkCommand) (string, error) { // first, import any values in the provided properties file (space separated "key val") // then map applicable envvars diff --git a/docs/security.md b/docs/security.md index 5cc2367c0c2f4..bfb165dc7b529 100644 --- a/docs/security.md +++ b/docs/security.md @@ -86,7 +86,34 @@ See the [Secrets Documentation about spaces][13] for more details about spaces. the keystore and truststore secrets will also show up as environment-based secrets, due to the way secrets are implemented. You can ignore these extra environment variables. +# Spark SASL (RPC endpoint authentication) +Spark uses Simple Authentication Security Layer (SASL) to authenticate Executors with the Driver and for encrypting messages sent between components. This functionality relies on a shared secret between all components you expect to communicate with each other. A secret can be generated with the DC/OS Spark CLI +```bash +dcos spark secret +# for example +dcos spark secret /sparkAuthSecret +``` +This will generate a random secret and upload it to the DC/OS secrets store [14] at the designated path. To use this secret for RPC authentication add the following configutations to your CLI command: +```bash +dcos spark run --submit-args="\ +... +--conf spark.mesos.containerizer=mesos \ # Mesos UCR is required for secrets +--conf spark.authenticate=true \ # tell Spark to use authentication +--conf spark.authenticate.enableSaslEncryption=true \ # tell Spark to encrypt with Sasl +--conf spark.authenticate.secret=sparkauthsecret.secret \ # name of file-based secret for Driver, you may change the name +--conf spark.executorEnv._SPARK_AUTH_SECRET=sparkauthsecret.secret \ # name of file-based secret for the Executors +--conf spark.mesos.driver.secret.names= \ # secret path generated in the previous step, for Driver +--conf spark.mesos.driver.secret.filenames=sparkauthsecret.secret \ # tell Mesos to put the secret in this file in the Driver +--conf spark.mesos.executor.secret.names= \ # secret path generated in previous step for Executor +--conf spark.mesos.executor.secret.filenames=sparkauthsecret.secret \ # tell Mesos to put the secret in this File for the Executors +... +" + +``` + + [11]: https://docs.mesosphere.com/1.9/overview/architecture/components/ [12]: http://docs.oracle.com/javase/8/docs/technotes/tools/unix/keytool.html [13]: https://docs.mesosphere.com/1.10/security/#spaces + [14]: https://docs.mesosphere.com/latest/security/secrets/ diff --git a/tests/test_spark.py b/tests/test_spark.py index b4a0daf84f902..9e09bfec14139 100644 --- a/tests/test_spark.py +++ b/tests/test_spark.py @@ -14,6 +14,7 @@ import shakedown import sdk_utils +import sdk_cmd from tests import s3 from tests import utils @@ -51,6 +52,31 @@ def test_jar(app_name="/spark"): args=["--class", 'com.typesafe.spark.test.mesos.framework.runners.SparkJobRunner']) +@pytest.mark.sanity +def test_rpc_auth(): + secret_name = "sparkauth" + + rc, stdout, stderr = sdk_cmd.run_raw_cli("spark secret /{}".format(secret_name)) + assert rc == 0, "Failed to generate Spark auth secret, stderr {err} stdout {out}".format(err=stderr, out=stdout) + + args = ["--conf", "spark.mesos.containerizer=mesos", + "--conf", "spark.authenticate=true", + "--conf", "spark.authenticate.secret=sparkauthsecret.secret", + "--conf", "spark.authenticate.enableSaslEncryption=true", + "--conf", "spark.executorEnv._SPARK_AUTH_SECRET=sparkauthsecret.secret", + "--conf", "spark.mesos.driver.secret.names={}".format(secret_name), + "--conf", "spark.mesos.driver.secret.filenames=sparkauthsecret.secret", + "--conf", "spark.mesos.executor.secret.names={}".format(secret_name), + "--conf", "spark.mesos.executor.secret.filenames=sparkauthsecret.secret", + "--class", "org.apache.spark.examples.SparkPi"] + + utils.run_tests(app_url=utils.SPARK_EXAMPLES, + app_args="100", + expected_output="Pi is roughly 3", + app_name="/spark", + args=args) + + @pytest.mark.sanity def test_sparkPi(): utils.run_tests(app_url=utils.SPARK_EXAMPLES,