improve kubeconfig/gcp handling of credentials in e2e tests

docs/cloud-provider.md

@@ -152,7 +152,8 @@ tags:
 ### machine.spec.providerConfig.cloudProviderSpec
 
 ```yaml
-serviceAccount: "<< GOOGLE_SERVICE_ACCOUNT >>"
+# The service account needs to be base64-encoded.
+serviceAccount: "<< GOOGLE_SERVICE_ACCOUNT_BASE64 >>"
 # See https://cloud.google.com/compute/docs/regions-zones/
 zone: "europe-west3-a"
 # See https://cloud.google.com/compute/docs/machine-types
@@ -302,8 +303,8 @@ tags:
 
 ### machine.spec.providerConfig.cloudProviderSpec
 ```yaml
-# kubeconfig to access KubeVirt cluster
-kubeconfig: '<< KUBECONFIG >>'
+# base64-encoded kubeconfig to access KubeVirt cluster
+kubeconfig: '<< KUBECONFIG_BASE64 >>'
 # KubeVirt namespace
 namespace: kube-system
 # kubernetes storage class

examples/gce-machinedeployment.yaml

@@ -6,8 +6,13 @@ metadata:
   name: machine-controller-gce
   namespace: kube-system
 type: Opaque
-stringData:
-  serviceAccount: "<< GOOGLE_SERVICE_ACCOUNT >>"
+data:
+  # The base64 encoding here is only to satisfy Kubernetes'
+  # Secret storage and to prevent multiline string replacement
+  # issues if we used stringData here (because the GCP SA is
+  # a multiline JSON string).
+  serviceAccount: "<< GOOGLE_SERVICE_ACCOUNT_BASE64 >>"
+
 ---
 apiVersion: "cluster.k8s.io/v1alpha1"
 kind: MachineDeployment

examples/kubevirt-machinedeployment.yaml

@@ -28,9 +28,9 @@ spec:
           cloudProviderSpec:
             auth:
               kubeconfig:
-                # Can also be set via the env var 'KUBEVIRT_KUBECONFIG' on the machine-controller
-                # If specified directly, this value should be a base64 encoded kubeconfig in either yaml or json format.
-                value: "<< KUBECONFIG >>"
+                # Can also be set via the env var 'KUBEVIRT_KUBECONFIG' on the machine-controller.
+                # If instead specified directly, this value should be a base64 encoded kubeconfig.
+                value: "<< KUBECONFIG_BASE64 >>"
             virtualMachine:
               template:
                 cpus: "1"

pkg/cloudprovider/provider/gce/config.go

@@ -226,17 +226,23 @@ func newConfig(resolver *providerconfig.ConfigVarResolver, spec v1alpha1.Provide
 // postprocessServiceAccount processes the service account and creates a JWT configuration
 // out of it.
 func (cfg *config) postprocessServiceAccount() error {
-	sa, err := base64.StdEncoding.DecodeString(cfg.serviceAccount)
-	if err != nil {
-		return fmt.Errorf("failed to decode base64 service account: %w", err)
+	sa := cfg.serviceAccount
+
+	// safely decode the service account, in case we did not read the value
+	// from a "known-safe" location (like the MachineDeployment), but from
+	// an environment variable.
+	decoded, err := base64.StdEncoding.DecodeString(cfg.serviceAccount)
+	if err == nil {
+		sa = string(decoded)
 	}
+
 	sam := map[string]string{}
-	err = json.Unmarshal(sa, &sam)
+	err = json.Unmarshal([]byte(sa), &sam)
 	if err != nil {
 		return fmt.Errorf("failed unmarshalling service account: %w", err)
 	}
 	cfg.projectID = sam["project_id"]
-	cfg.jwtConfig, err = google.JWTConfigFromJSON(sa, compute.ComputeScope)
+	cfg.jwtConfig, err = google.JWTConfigFromJSON([]byte(sa), compute.ComputeScope)
 	if err != nil {
 		return fmt.Errorf("failed preparing JWT: %w", err)
 	}

pkg/cloudprovider/provider/gce/types/types.go

@@ -30,6 +30,7 @@ import (
 // CloudProviderSpec contains the specification of the cloud provider taken
 // from the provider configuration.
 type CloudProviderSpec struct {
+	// ServiceAccount must be base64-encoded.
 	ServiceAccount               providerconfigtypes.ConfigVarString  `json:"serviceAccount,omitempty"`
 	Zone                         providerconfigtypes.ConfigVarString  `json:"zone"`
 	MachineType                  providerconfigtypes.ConfigVarString  `json:"machineType"`

test/e2e/provisioning/all_e2e_test.go

@@ -20,6 +20,7 @@ package provisioning
 
 import (
 	"context"
+	"encoding/base64"
 	"flag"
 	"fmt"
 	"os"
@@ -293,12 +294,23 @@ func TestKubevirtProvisioningE2E(t *testing.T) {
 	selector := OsSelector("ubuntu", "centos", "flatcar", "rockylinux")
 
 	params := []string{
-		fmt.Sprintf("<< KUBECONFIG >>=%s", kubevirtKubeconfig),
+		fmt.Sprintf("<< KUBECONFIG_BASE64 >>=%s", safeBase64Encoding(kubevirtKubeconfig)),
 	}
 
 	runScenarios(t, selector, params, kubevirtManifest, fmt.Sprintf("kubevirt-%s", *testRunIdentifier))
 }
 
+// safeBase64Encoding takes a value and encodes it with base64
+// if it is not already encoded.
+func safeBase64Encoding(value string) string {
+	// If there was no error, the original value was already encoded.
+	if _, err := base64.StdEncoding.DecodeString(value); err == nil {
+		return value
+	}
+
+	return base64.StdEncoding.EncodeToString([]byte(value))
+}
+
 func TestOpenstackProvisioningE2E(t *testing.T) {
 	t.Parallel()
 
@@ -708,8 +720,9 @@ func TestGCEProvisioningE2E(t *testing.T) {
 	// Act. GCE does not support CentOS.
 	selector := OsSelector("ubuntu")
 	params := []string{
-		fmt.Sprintf("<< GOOGLE_SERVICE_ACCOUNT >>=%s", googleServiceAccount),
+		fmt.Sprintf("<< GOOGLE_SERVICE_ACCOUNT_BASE64 >>=%s", safeBase64Encoding(googleServiceAccount)),
 	}
+
 	runScenarios(t, selector, params, GCEManifest, fmt.Sprintf("gce-%s", *testRunIdentifier))
 }
 

test/e2e/provisioning/testdata/machinedeployment-gce.yaml

@@ -24,8 +24,9 @@ spec:
             - "<< YOUR_PUBLIC_KEY >>"
           cloudProvider: "gce"
           cloudProviderSpec:
-            # If empty, can be set via GOOGLE_SERVICE_ACCOUNT env var
-            serviceAccount: "<< GOOGLE_SERVICE_ACCOUNT >>"
+            # If empty, can be set via GOOGLE_SERVICE_ACCOUNT env var. The environment variable
+            # should be plaintext. The value in the cloudProviderSpec however must be base64-encoded.
+            serviceAccount: "<< GOOGLE_SERVICE_ACCOUNT_BASE64 >>"
             # See https://cloud.google.com/compute/docs/regions-zones/
             zone: "europe-west3-a"
             # See https://cloud.google.com/compute/docs/machine-types

test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml

@@ -28,7 +28,7 @@ spec:
           cloudProviderSpec:
             auth:
               kubeconfig:
-                value: '<< KUBECONFIG >>'
+                value: '<< KUBECONFIG_BASE64 >>'
             virtualMachine:
               template:
                 cpus: "1"