From 475380c2a140adf8ab8ff3a9d43124e4d275296a Mon Sep 17 00:00:00 2001 From: farhan5900 Date: Mon, 11 Nov 2019 12:38:30 -0300 Subject: [PATCH 1/5] Adds File based secret test --- tests/security_test.go | 96 ++++++++++++++++++- .../templates/spark-mock-task-runner-job.yaml | 22 +++++ tests/utils/k8s.go | 20 +++- 3 files changed, 134 insertions(+), 4 deletions(-) diff --git a/tests/security_test.go b/tests/security_test.go index ca262ec6..24906324 100644 --- a/tests/security_test.go +++ b/tests/security_test.go @@ -1,10 +1,13 @@ package tests import ( + "fmt" + "strings" + "testing" + "github.com/mesosphere/kudo-spark-operator/tests/utils" log "github.com/sirupsen/logrus" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "testing" ) type serviceAccountTestCase struct { @@ -141,3 +144,94 @@ func runServiceAccountTestCase(tc serviceAccountTestCase) error { return err } + +func TestEnvBasedSecret(t *testing.T) { + secretName := "env-based-secret" + secretKey := "secretKey" + secretEnv := "SECRET_ENV" + jobDescription, err := runSecretTest(secretName, "", secretKey) + + if err != nil { + t.Error(err.Error()) + } + + if strings.Contains(jobDescription, fmt.Sprintf("%s from %s-volume", secretKey, secretName)) { + log.Infof("Successfully exported environment variable '%s' from secret '%s'", secretEnv, secretName) + } else { + t.Errorf("Unnable to export environment variable '%s' from secret '%s'", secretEnv, secretName) + } +} + +func TestFileBasedSecrets(t *testing.T) { + secretName := "file-based-secret" + secretPath := "/mnt/secrets" + jobDescription, err := runSecretTest(secretName, secretPath, "") + + if err != nil { + t.Error(err.Error()) + } + + if strings.Contains(jobDescription, fmt.Sprintf("%s from %s-volume", secretPath, secretName)) { + log.Infof("Successfully mounted secret path '%s' from '%s-volume'", secretPath, secretName) + } else { + t.Errorf("Unnable to mount secret path '%s' from '%s-volume'", secretPath, secretName) + } +} + +func runSecretTest(secretName string, secretPath string, secretKey string) (string, error) { + spark := utils.SparkOperatorInstallation{} + err := spark.InstallSparkOperator() + defer spark.CleanUp() + + if err != nil { + return "", err + } + + client, err := utils.GetK8sClientSet() + if err != nil { + return "", err + } + + secretData := map[string]string{ + secretKey: "secretValue", + } + + err = utils.CreateSecret(client, secretName, spark.Namespace, secretData) + if err != nil { + return "", err + } + + jobName := "mock-task-runner" + job := utils.SparkJob{ + Name: jobName, + Template: "spark-mock-task-runner-job.yaml", + Params: map[string]interface{}{ + "args": []string{"1", "15"}, + "SecretName": secretName, + "SecretPath": secretPath, + "SecretKey": secretKey, + }, + } + + err = spark.SubmitJob(&job) + if err != nil { + return "", err + } + + err = spark.WaitUntilSucceeded(job) + if err != nil { + return "", err + } + + jobDescription, err := utils.Kubectl( + "describe", + "pod", + "--namespace="+spark.Namespace, + jobName+"-driver", + ) + if err != nil { + return "", err + } + + return jobDescription, nil +} diff --git a/tests/templates/spark-mock-task-runner-job.yaml b/tests/templates/spark-mock-task-runner-job.yaml index e2df5a8e..717dba58 100644 --- a/tests/templates/spark-mock-task-runner-job.yaml +++ b/tests/templates/spark-mock-task-runner-job.yaml @@ -25,6 +25,17 @@ spec: version: {{ .SparkVersion }} metrics-exposed: "true" serviceAccount: {{ .ServiceAccount }} + {{- if and .Params.SecretName .Params.SecretPath }} + secrets: + - name: {{ .Params.SecretName }} + path: {{ .Params.SecretPath }} + secretType: Opaque + {{- else if and .Params.SecretName .Params.SecretKey }} + envSecretKeyRefs: + SECRET_ENV: + name: {{ .Params.SecretName }} + key: {{ .Params.SecretKey }} + {{- end }} executor: cores: 1 instances: 1 @@ -32,3 +43,14 @@ spec: labels: version: {{ .SparkVersion }} metrics-exposed: "true" + {{- if and .Params.SecretName .Params.SecretPath }} + secrets: + - name: {{ .Params.SecretName }} + path: {{ .Params.SecretPath }} + secretType: Opaque + {{- else if and .Params.SecretName .Params.SecretKey }} + envSecretKeyRefs: + SECRET_ENV: + name: {{ .Params.SecretName }} + key: {{ .Params.SecretKey }} + {{- end }} diff --git a/tests/utils/k8s.go b/tests/utils/k8s.go index 83c1d185..a9c805b8 100644 --- a/tests/utils/k8s.go +++ b/tests/utils/k8s.go @@ -5,15 +5,16 @@ import ( "bytes" "errors" "fmt" - log "github.com/sirupsen/logrus" "io" + "os/exec" + "strings" + + log "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" apiErrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/clientcmd" - "os/exec" - "strings" ) /* client-go util methods */ @@ -78,6 +79,19 @@ func CreateServiceAccount(clientSet *kubernetes.Clientset, name string, namespac return err } +func CreateSecret(clientSet *kubernetes.Clientset, name string, namespace string, secretData map[string]string) error { + log.Infof("Creating a secret %s/%s with Secret Data: %q", namespace, name, secretData) + secret := v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + StringData: secretData, + } + + _, err := clientSet.CoreV1().Secrets(namespace).Create(&secret) + return err +} + func getPodLog(clientSet *kubernetes.Clientset, namespace string, pod string, tailLines int64) (string, error) { opts := v1.PodLogOptions{} if tailLines > 0 { From 2b0f65984adb0901dfe7f72ea1a9a9eb7828d253 Mon Sep 17 00:00:00 2001 From: farhan5900 Date: Mon, 11 Nov 2019 13:12:18 -0300 Subject: [PATCH 2/5] Adds test for env based secret --- tests/security_test.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/security_test.go b/tests/security_test.go index 6752eb01..cbbfecfa 100644 --- a/tests/security_test.go +++ b/tests/security_test.go @@ -301,17 +301,16 @@ func runTestCase(tc securityTestCase) error { func TestEnvBasedSecret(t *testing.T) { secretName := "env-based-secret" secretKey := "secretKey" - secretEnv := "SECRET_ENV" jobDescription, err := runSecretTest(secretName, "", secretKey) if err != nil { t.Error(err.Error()) } - if strings.Contains(jobDescription, fmt.Sprintf("%s from %s-volume", secretKey, secretName)) { - log.Infof("Successfully exported environment variable '%s' from secret '%s'", secretEnv, secretName) + if strings.Contains(jobDescription, fmt.Sprintf("set to the key '%s' in secret '%s'", secretKey, secretName)) { + log.Infof("Successfully set environment variable to the key '%s' in secret '%s'", secretKey, secretName) } else { - t.Errorf("Unnable to export environment variable '%s' from secret '%s'", secretEnv, secretName) + t.Errorf("Unnable to set environment variable to the key '%s' in secret '%s'", secretKey, secretName) } } From ab6cc47dca59f3f09699921fed066d1e6664b8ca Mon Sep 17 00:00:00 2001 From: farhan5900 Date: Mon, 11 Nov 2019 14:13:55 -0300 Subject: [PATCH 3/5] Fixes missing key from secret data in file based secret test --- tests/security_test.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/security_test.go b/tests/security_test.go index cbbfecfa..0c003526 100644 --- a/tests/security_test.go +++ b/tests/security_test.go @@ -344,8 +344,11 @@ func runSecretTest(secretName string, secretPath string, secretKey string) (stri return "", err } - secretData := map[string]string{ - secretKey: "secretValue", + secretData := make(map[string]string) + if secretKey != "" { + secretData[secretKey] = "secretValue" + } else { + secretData["secretKey"] = "secretValue" } err = utils.CreateSecret(client, secretName, spark.Namespace, secretData) From ac69995410810bd83afcff3bf3b3f4c7b4b2021a Mon Sep 17 00:00:00 2001 From: farhan5900 Date: Tue, 12 Nov 2019 11:43:26 -0300 Subject: [PATCH 4/5] Cleanups secret tests --- tests/security_test.go | 51 ++++++++++++++++++++---------------------- 1 file changed, 24 insertions(+), 27 deletions(-) diff --git a/tests/security_test.go b/tests/security_test.go index 0c003526..885a7277 100644 --- a/tests/security_test.go +++ b/tests/security_test.go @@ -298,50 +298,34 @@ func runTestCase(tc securityTestCase) error { return err } -func TestEnvBasedSecret(t *testing.T) { - secretName := "env-based-secret" - secretKey := "secretKey" - jobDescription, err := runSecretTest(secretName, "", secretKey) +func TestEnvBasedSecrets(t *testing.T) { + err := runSecretTest("env-based-secret", "", "secretKey", "set to the key 'secretKey' in secret 'env-based-secret'") if err != nil { t.Error(err.Error()) } - - if strings.Contains(jobDescription, fmt.Sprintf("set to the key '%s' in secret '%s'", secretKey, secretName)) { - log.Infof("Successfully set environment variable to the key '%s' in secret '%s'", secretKey, secretName) - } else { - t.Errorf("Unnable to set environment variable to the key '%s' in secret '%s'", secretKey, secretName) - } } func TestFileBasedSecrets(t *testing.T) { - secretName := "file-based-secret" - secretPath := "/mnt/secrets" - jobDescription, err := runSecretTest(secretName, secretPath, "") + err := runSecretTest("file-based-secret", "/mnt/secrets", "", "/mnt/secrets from file-based-secret-volume") if err != nil { t.Error(err.Error()) } - - if strings.Contains(jobDescription, fmt.Sprintf("%s from %s-volume", secretPath, secretName)) { - log.Infof("Successfully mounted secret path '%s' from '%s-volume'", secretPath, secretName) - } else { - t.Errorf("Unnable to mount secret path '%s' from '%s-volume'", secretPath, secretName) - } } -func runSecretTest(secretName string, secretPath string, secretKey string) (string, error) { +func runSecretTest(secretName string, secretPath string, secretKey string, expectedSecret string) error { spark := utils.SparkOperatorInstallation{} err := spark.InstallSparkOperator() defer spark.CleanUp() if err != nil { - return "", err + return err } client, err := utils.GetK8sClientSet() if err != nil { - return "", err + return err } secretData := make(map[string]string) @@ -353,7 +337,7 @@ func runSecretTest(secretName string, secretPath string, secretKey string) (stri err = utils.CreateSecret(client, secretName, spark.Namespace, secretData) if err != nil { - return "", err + return err } jobName := "mock-task-runner" @@ -370,12 +354,12 @@ func runSecretTest(secretName string, secretPath string, secretKey string) (stri err = spark.SubmitJob(&job) if err != nil { - return "", err + return err } err = spark.WaitUntilSucceeded(job) if err != nil { - return "", err + return err } jobDescription, err := utils.Kubectl( @@ -385,8 +369,21 @@ func runSecretTest(secretName string, secretPath string, secretKey string) (stri jobName+"-driver", ) if err != nil { - return "", err + return err } - return jobDescription, nil + if strings.Contains(jobDescription, expectedSecret) { + if secretKey != "" { + log.Infof("Successfully set environment variable to the key '%s' in secret '%s'", secretKey, secretName) + } else { + log.Infof("Successfully mounted secret path '%s' from '%s-volume'", secretPath, secretName) + } + } else { + if secretKey != "" { + return fmt.Errorf("Unnable to set environment variable to the key '%s' in secret '%s'", secretKey, secretName) + } + return fmt.Errorf("Unnable to mount secret path '%s' from '%s-volume'", secretPath, secretName) + } + + return nil } From 66327e50fec120fed1d4e943ebd8133783a9d7ef Mon Sep 17 00:00:00 2001 From: farhan5900 Date: Wed, 13 Nov 2019 13:54:40 -0300 Subject: [PATCH 5/5] Exported driverPodName from utils and used spark K8sClients --- tests/security_test.go | 9 ++------- tests/utils/job.go | 13 +++++++------ 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/tests/security_test.go b/tests/security_test.go index 885a7277..0a061c4e 100644 --- a/tests/security_test.go +++ b/tests/security_test.go @@ -323,11 +323,6 @@ func runSecretTest(secretName string, secretPath string, secretKey string, expec return err } - client, err := utils.GetK8sClientSet() - if err != nil { - return err - } - secretData := make(map[string]string) if secretKey != "" { secretData[secretKey] = "secretValue" @@ -335,7 +330,7 @@ func runSecretTest(secretName string, secretPath string, secretKey string, expec secretData["secretKey"] = "secretValue" } - err = utils.CreateSecret(client, secretName, spark.Namespace, secretData) + err = utils.CreateSecret(spark.K8sClients, secretName, spark.Namespace, secretData) if err != nil { return err } @@ -366,7 +361,7 @@ func runSecretTest(secretName string, secretPath string, secretKey string, expec "describe", "pod", "--namespace="+spark.Namespace, - jobName+"-driver", + utils.DriverPodName(jobName), ) if err != nil { return err diff --git a/tests/utils/job.go b/tests/utils/job.go index ca6a807a..56ec6504 100644 --- a/tests/utils/job.go +++ b/tests/utils/job.go @@ -2,8 +2,9 @@ package utils import ( "errors" - log "github.com/sirupsen/logrus" "os" + + log "github.com/sirupsen/logrus" ) type SparkJob struct { @@ -41,12 +42,12 @@ func (spark *SparkOperatorInstallation) SubmitJob(job *SparkJob) error { } func (spark *SparkOperatorInstallation) DriverLog(job SparkJob) (string, error) { - driverPodName := driverPodName(job.Name) + driverPodName := DriverPodName(job.Name) return getPodLog(spark.K8sClients, job.Namespace, driverPodName, 0) } func (spark *SparkOperatorInstallation) DriverLogContains(job SparkJob, text string) (bool, error) { - driverPodName := driverPodName(job.Name) + driverPodName := DriverPodName(job.Name) return podLogContains(spark.K8sClients, job.Namespace, driverPodName, text) } @@ -67,16 +68,16 @@ func (spark *SparkOperatorInstallation) WaitForOutput(job SparkJob, text string) if err != nil { log.Errorf("The text '%s' haven't appeared in the log in %s", text, defaultRetryTimeout.String()) - logPodLogTail(spark.K8sClients, job.Namespace, driverPodName(job.Name), 10) + logPodLogTail(spark.K8sClients, job.Namespace, DriverPodName(job.Name), 10) } return err } func (spark *SparkOperatorInstallation) WaitUntilSucceeded(job SparkJob) error { - driverPodName := driverPodName(job.Name) + driverPodName := DriverPodName(job.Name) return waitForPodStatusPhase(spark.K8sClients, driverPodName, job.Namespace, "Succeeded") } -func driverPodName(jobName string) string { +func DriverPodName(jobName string) string { return jobName + "-driver" }