[DCOS-60858] [Spark Operator] Test mounting secrets

tests/security_test.go

@@ -3,11 +3,13 @@ package tests
 import (
 	"errors"
 	"fmt"
+	"strings"
+	"testing"
+
 	"github.com/mesosphere/kudo-spark-operator/tests/utils"
 	log "github.com/sirupsen/logrus"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
-	"testing"
 )
 
 type securityTestCase interface {
@@ -295,3 +297,96 @@ func runTestCase(tc securityTestCase) error {
 
 	return err
 }
+
+func TestEnvBasedSecret(t *testing.T) {
+	secretName := "env-based-secret"
+	secretKey := "secretKey"
+	jobDescription, err := runSecretTest(secretName, "", secretKey)
+
+	if err != nil {
+		t.Error(err.Error())
+	}
+
+	if strings.Contains(jobDescription, fmt.Sprintf("set to the key '%s' in secret '%s'", secretKey, secretName)) {
+		log.Infof("Successfully set environment variable to the key '%s' in secret '%s'", secretKey, secretName)
+	} else {
+		t.Errorf("Unnable to set environment variable to the key '%s' in secret '%s'", secretKey, secretName)
+	}
+}
+
+func TestFileBasedSecrets(t *testing.T) {
+	secretName := "file-based-secret"
+	secretPath := "/mnt/secrets"
+	jobDescription, err := runSecretTest(secretName, secretPath, "")
+
+	if err != nil {
+		t.Error(err.Error())
+	}
+
+	if strings.Contains(jobDescription, fmt.Sprintf("%s from %s-volume", secretPath, secretName)) {
+		log.Infof("Successfully mounted secret path '%s' from '%s-volume'", secretPath, secretName)
+	} else {
+		t.Errorf("Unnable to mount secret path '%s' from '%s-volume'", secretPath, secretName)
+	}
+}
+
+func runSecretTest(secretName string, secretPath string, secretKey string) (string, error) {
+	spark := utils.SparkOperatorInstallation{}
+	err := spark.InstallSparkOperator()
+	defer spark.CleanUp()
+
+	if err != nil {
+		return "", err
+	}
+
+	client, err := utils.GetK8sClientSet()
+	if err != nil {
+		return "", err
+	}
+
+	secretData := make(map[string]string)
+	if secretKey != "" {
+		secretData[secretKey] = "secretValue"
+	} else {
+		secretData["secretKey"] = "secretValue"
+	}
+
+	err = utils.CreateSecret(client, secretName, spark.Namespace, secretData)
+	if err != nil {
+		return "", err
+	}
+
+	jobName := "mock-task-runner"
+	job := utils.SparkJob{
+		Name:     jobName,
+		Template: "spark-mock-task-runner-job.yaml",
+		Params: map[string]interface{}{
+			"args":       []string{"1", "15"},
+			"SecretName": secretName,
+			"SecretPath": secretPath,
+			"SecretKey":  secretKey,
+		},
+	}
+
+	err = spark.SubmitJob(&job)
+	if err != nil {
+		return "", err
+	}
+
+	err = spark.WaitUntilSucceeded(job)
+	if err != nil {
+		return "", err
+	}
+
+	jobDescription, err := utils.Kubectl(
+		"describe",
+		"pod",
+		"--namespace="+spark.Namespace,
+		jobName+"-driver",
+	)
+	if err != nil {
+		return "", err
+	}
+
+	return jobDescription, nil
+}

tests/templates/spark-mock-task-runner-job.yaml

@@ -25,10 +25,32 @@ spec:
       version: {{ .SparkVersion }}
       metrics-exposed: "true"
     serviceAccount: {{ .ServiceAccount }}
+    {{- if and .Params.SecretName .Params.SecretPath }}
+    secrets:
+    - name: {{ .Params.SecretName }}
+      path: {{ .Params.SecretPath }}
+      secretType: Opaque
+    {{- else if and .Params.SecretName .Params.SecretKey }}
+    envSecretKeyRefs:
+      SECRET_ENV:
+        name: {{ .Params.SecretName }}
+        key: {{ .Params.SecretKey }}
+    {{- end }}
   executor:
     cores: 1
     instances: 1
     memory: "512m"
     labels:
       version: {{ .SparkVersion }}
       metrics-exposed: "true"
+    {{- if and .Params.SecretName .Params.SecretPath }}
+    secrets:
+    - name: {{ .Params.SecretName }}
+      path: {{ .Params.SecretPath }}
+      secretType: Opaque
+    {{- else if and .Params.SecretName .Params.SecretKey }}
+    envSecretKeyRefs:
+      SECRET_ENV:
+        name: {{ .Params.SecretName }}
+        key: {{ .Params.SecretKey }}
+    {{- end }}

tests/utils/k8s.go

@@ -5,16 +5,17 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
-	log "github.com/sirupsen/logrus"
 	"io"
+	"os"
+	"os/exec"
+	"strings"
+
+	log "github.com/sirupsen/logrus"
 	v1 "k8s.io/api/core/v1"
 	apiErrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/clientcmd"
-	"os"
-	"os/exec"
-	"strings"
 )
 
 /* client-go util methods */
@@ -79,6 +80,19 @@ func CreateServiceAccount(clientSet *kubernetes.Clientset, name string, namespac
 	return err
 }
 
+func CreateSecret(clientSet *kubernetes.Clientset, name string, namespace string, secretData map[string]string) error {
+	log.Infof("Creating a secret %s/%s with Secret Data: %q", namespace, name, secretData)
+	secret := v1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: name,
+		},
+		StringData: secretData,
+	}
+
+	_, err := clientSet.CoreV1().Secrets(namespace).Create(&secret)
+	return err
+}
+
 func getPodLog(clientSet *kubernetes.Clientset, namespace string, pod string, tailLines int64) (string, error) {
 	opts := v1.PodLogOptions{}
 	if tailLines > 0 {