[DCOS-60858] [Spark Operator] Test mounting secrets (#59)

tests/security_test.go

@@ -3,11 +3,13 @@ package tests
 import (
 	"errors"
 	"fmt"
+	"strings"
+	"testing"
+
 	"github.com/mesosphere/kudo-spark-operator/tests/utils"
 	log "github.com/sirupsen/logrus"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
-	"testing"
 )
 
 type securityTestCase interface {
@@ -295,3 +297,88 @@ func runTestCase(tc securityTestCase) error {
 
 	return err
 }
+
+func TestEnvBasedSecrets(t *testing.T) {
+	err := runSecretTest("env-based-secret", "", "secretKey", "set to the key 'secretKey' in secret 'env-based-secret'")
+
+	if err != nil {
+		t.Error(err.Error())
+	}
+}
+
+func TestFileBasedSecrets(t *testing.T) {
+	err := runSecretTest("file-based-secret", "/mnt/secrets", "", "/mnt/secrets from file-based-secret-volume")
+
+	if err != nil {
+		t.Error(err.Error())
+	}
+}
+
+func runSecretTest(secretName string, secretPath string, secretKey string, expectedSecret string) error {
+	spark := utils.SparkOperatorInstallation{}
+	err := spark.InstallSparkOperator()
+	defer spark.CleanUp()
+
+	if err != nil {
+		return err
+	}
+
+	secretData := make(map[string]string)
+	if secretKey != "" {
+		secretData[secretKey] = "secretValue"
+	} else {
+		secretData["secretKey"] = "secretValue"
+	}
+
+	err = utils.CreateSecret(spark.K8sClients, secretName, spark.Namespace, secretData)
+	if err != nil {
+		return err
+	}
+
+	jobName := "mock-task-runner"
+	job := utils.SparkJob{
+		Name:     jobName,
+		Template: "spark-mock-task-runner-job.yaml",
+		Params: map[string]interface{}{
+			"args":       []string{"1", "15"},
+			"SecretName": secretName,
+			"SecretPath": secretPath,
+			"SecretKey":  secretKey,
+		},
+	}
+
+	err = spark.SubmitJob(&job)
+	if err != nil {
+		return err
+	}
+
+	err = spark.WaitUntilSucceeded(job)
+	if err != nil {
+		return err
+	}
+
+	jobDescription, err := utils.Kubectl(
+		"describe",
+		"pod",
+		"--namespace="+spark.Namespace,
+		utils.DriverPodName(jobName),
+	)
+	if err != nil {
+		return err
+	}
+
+	if strings.Contains(jobDescription, expectedSecret) {
+		if secretKey != "" {
+			log.Infof("Successfully set environment variable to the key '%s' in secret '%s'", secretKey, secretName)
+		} else {
+			log.Infof("Successfully mounted secret path '%s' from '%s-volume'", secretPath, secretName)
+		}
+	} else {
+		if secretKey != "" {
+			return fmt.Errorf("Unnable to set environment variable to the key '%s' in secret '%s'", secretKey, secretName)
+		}
+		return fmt.Errorf("Unnable to mount secret path '%s' from '%s-volume'", secretPath, secretName)
+	}
+
+	return nil
+}

tests/templates/spark-mock-task-runner-job.yaml

@@ -25,10 +25,32 @@ spec:
       version: {{ .SparkVersion }}
       metrics-exposed: "true"
     serviceAccount: {{ .ServiceAccount }}
+    {{- if and .Params.SecretName .Params.SecretPath }}
+    secrets:
+    - name: {{ .Params.SecretName }}
+      path: {{ .Params.SecretPath }}
+      secretType: Opaque
+    {{- else if and .Params.SecretName .Params.SecretKey }}
+    envSecretKeyRefs:
+      SECRET_ENV:
+        name: {{ .Params.SecretName }}
+        key: {{ .Params.SecretKey }}
+    {{- end }}
   executor:
     cores: 1
     instances: 1
     memory: "512m"
     labels:
       version: {{ .SparkVersion }}
       metrics-exposed: "true"
+    {{- if and .Params.SecretName .Params.SecretPath }}
+    secrets:
+    - name: {{ .Params.SecretName }}
+      path: {{ .Params.SecretPath }}
+      secretType: Opaque
+    {{- else if and .Params.SecretName .Params.SecretKey }}
+    envSecretKeyRefs:
+      SECRET_ENV:
+        name: {{ .Params.SecretName }}
+        key: {{ .Params.SecretKey }}
+    {{- end }}

tests/utils/job.go

@@ -2,8 +2,9 @@ package utils
 
 import (
 	"errors"
-	log "github.com/sirupsen/logrus"
 	"os"
+
+	log "github.com/sirupsen/logrus"
 )
 
 type SparkJob struct {
@@ -41,12 +42,12 @@ func (spark *SparkOperatorInstallation) SubmitJob(job *SparkJob) error {
 }
 
 func (spark *SparkOperatorInstallation) DriverLog(job SparkJob) (string, error) {
-	driverPodName := driverPodName(job.Name)
+	driverPodName := DriverPodName(job.Name)
 	return getPodLog(spark.K8sClients, job.Namespace, driverPodName, 0)
 }
 
 func (spark *SparkOperatorInstallation) DriverLogContains(job SparkJob, text string) (bool, error) {
-	driverPodName := driverPodName(job.Name)
+	driverPodName := DriverPodName(job.Name)
 	return podLogContains(spark.K8sClients, job.Namespace, driverPodName, text)
 }
 
@@ -67,16 +68,16 @@ func (spark *SparkOperatorInstallation) WaitForOutput(job SparkJob, text string)
 
 	if err != nil {
 		log.Errorf("The text '%s' haven't appeared in the log in %s", text, defaultRetryTimeout.String())
-		logPodLogTail(spark.K8sClients, job.Namespace, driverPodName(job.Name), 10)
+		logPodLogTail(spark.K8sClients, job.Namespace, DriverPodName(job.Name), 10)
 	}
 	return err
 }
 
 func (spark *SparkOperatorInstallation) WaitUntilSucceeded(job SparkJob) error {
-	driverPodName := driverPodName(job.Name)
+	driverPodName := DriverPodName(job.Name)
 	return waitForPodStatusPhase(spark.K8sClients, driverPodName, job.Namespace, "Succeeded")
 }
 
-func driverPodName(jobName string) string {
+func DriverPodName(jobName string) string {
 	return jobName + "-driver"
 }

tests/utils/k8s.go

@@ -5,16 +5,17 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
-	log "github.com/sirupsen/logrus"
 	"io"
+	"os"
+	"os/exec"
+	"strings"
+
+	log "github.com/sirupsen/logrus"
 	v1 "k8s.io/api/core/v1"
 	apiErrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/clientcmd"
-	"os"
-	"os/exec"
-	"strings"
 )
 
 /* client-go util methods */
@@ -79,6 +80,19 @@ func CreateServiceAccount(clientSet *kubernetes.Clientset, name string, namespac
 	return err
 }
 
+func CreateSecret(clientSet *kubernetes.Clientset, name string, namespace string, secretData map[string]string) error {
+	log.Infof("Creating a secret %s/%s with Secret Data: %q", namespace, name, secretData)
+	secret := v1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: name,
+		},
+		StringData: secretData,
+	}
+
+	_, err := clientSet.CoreV1().Secrets(namespace).Create(&secret)
+	return err
+}
+
 func getPodLog(clientSet *kubernetes.Clientset, namespace string, pod string, tailLines int64) (string, error) {
 	opts := v1.PodLogOptions{}
 	if tailLines > 0 {