From 74433d8680a5cd50553c268a686ac98013f1a7ba Mon Sep 17 00:00:00 2001 From: Christoph Mewes Date: Mon, 15 Aug 2022 12:05:18 +0200 Subject: [PATCH 1/6] make it clear where and how to base64 encode the kubevirt kubeconfig --- docs/cloud-provider.md | 4 ++-- examples/kubevirt-machinedeployment.yaml | 6 +++--- test/e2e/provisioning/all_e2e_test.go | 14 +++++++++++++- .../testdata/machinedeployment-kubevirt.yaml | 2 +- 4 files changed, 19 insertions(+), 7 deletions(-) diff --git a/docs/cloud-provider.md b/docs/cloud-provider.md index c18284f43..34c386cd9 100644 --- a/docs/cloud-provider.md +++ b/docs/cloud-provider.md @@ -302,8 +302,8 @@ tags: ### machine.spec.providerConfig.cloudProviderSpec ```yaml -# kubeconfig to access KubeVirt cluster -kubeconfig: '<< KUBECONFIG >>' +# base64-encoded kubeconfig to access KubeVirt cluster +kubeconfig: '<< KUBECONFIG_BASE64 >>' # KubeVirt namespace namespace: kube-system # kubernetes storage class diff --git a/examples/kubevirt-machinedeployment.yaml b/examples/kubevirt-machinedeployment.yaml index a40b95703..c7d959a25 100644 --- a/examples/kubevirt-machinedeployment.yaml +++ b/examples/kubevirt-machinedeployment.yaml @@ -28,9 +28,9 @@ spec: cloudProviderSpec: auth: kubeconfig: - # Can also be set via the env var 'KUBEVIRT_KUBECONFIG' on the machine-controller - # If specified directly, this value should be a base64 encoded kubeconfig in either yaml or json format. - value: "<< KUBECONFIG >>" + # Can also be set via the env var 'KUBEVIRT_KUBECONFIG' on the machine-controller. + # If instead specified directly, this value should be a base64 encoded kubeconfig. + value: "<< KUBECONFIG_BASE64 >>" virtualMachine: template: cpus: "1" diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index c19c979c6..ac90d4a14 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -20,6 +20,7 @@ package provisioning import ( "context" + "encoding/base64" "flag" "fmt" "os" @@ -293,12 +294,23 @@ func TestKubevirtProvisioningE2E(t *testing.T) { selector := OsSelector("ubuntu", "centos", "flatcar", "rockylinux") params := []string{ - fmt.Sprintf("<< KUBECONFIG >>=%s", kubevirtKubeconfig), + fmt.Sprintf("<< KUBECONFIG_BASE64 >>=%s", safeBase64Encoding(kubevirtKubeconfig)), } runScenarios(t, selector, params, kubevirtManifest, fmt.Sprintf("kubevirt-%s", *testRunIdentifier)) } +// safeBase64Encoding takes a value and encodes it with base64 +// if it is not already encoded. +func safeBase64Encoding(value string) string { + // If there was no error, the original value was already encoded. + if _, err := base64.StdEncoding.DecodeString(value); err == nil { + return value + } + + return base64.StdEncoding.EncodeToString([]byte(value)) +} + func TestOpenstackProvisioningE2E(t *testing.T) { t.Parallel() diff --git a/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml b/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml index 29a42da25..5c66b5622 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml @@ -28,7 +28,7 @@ spec: cloudProviderSpec: auth: kubeconfig: - value: '<< KUBECONFIG >>' + value: '<< KUBECONFIG_BASE64 >>' virtualMachine: template: cpus: "1" From 831b8c974d3c69c23eca609106f31b4ed49b8916 Mon Sep 17 00:00:00 2001 From: Christoph Mewes Date: Mon, 15 Aug 2022 13:07:39 +0200 Subject: [PATCH 2/6] same goodness for GCP --- docs/cloud-provider.md | 3 ++- examples/gce-machinedeployment.yaml | 5 ++++- pkg/cloudprovider/provider/gce/config.go | 12 +++++++++--- pkg/cloudprovider/provider/gce/types/types.go | 1 + test/e2e/provisioning/all_e2e_test.go | 3 ++- .../provisioning/testdata/machinedeployment-gce.yaml | 5 +++-- 6 files changed, 21 insertions(+), 8 deletions(-) diff --git a/docs/cloud-provider.md b/docs/cloud-provider.md index 34c386cd9..75a065640 100644 --- a/docs/cloud-provider.md +++ b/docs/cloud-provider.md @@ -152,7 +152,8 @@ tags: ### machine.spec.providerConfig.cloudProviderSpec ```yaml -serviceAccount: "<< GOOGLE_SERVICE_ACCOUNT >>" +# The service account needs to be base64-encoded. +serviceAccount: "<< GOOGLE_SERVICE_ACCOUNT_BASE64 >>" # See https://cloud.google.com/compute/docs/regions-zones/ zone: "europe-west3-a" # See https://cloud.google.com/compute/docs/machine-types diff --git a/examples/gce-machinedeployment.yaml b/examples/gce-machinedeployment.yaml index 37c8eecb2..eb90a1616 100644 --- a/examples/gce-machinedeployment.yaml +++ b/examples/gce-machinedeployment.yaml @@ -7,7 +7,10 @@ metadata: namespace: kube-system type: Opaque stringData: - serviceAccount: "<< GOOGLE_SERVICE_ACCOUNT >>" + # The secret should contain the plaintext service account, but since + # it's a secret, for this YAML file it needs to be base64-encoded. + serviceAccount: "<< GOOGLE_SERVICE_ACCOUNT_BASE64 >>" + --- apiVersion: "cluster.k8s.io/v1alpha1" kind: MachineDeployment diff --git a/pkg/cloudprovider/provider/gce/config.go b/pkg/cloudprovider/provider/gce/config.go index cacea48e6..25108db29 100644 --- a/pkg/cloudprovider/provider/gce/config.go +++ b/pkg/cloudprovider/provider/gce/config.go @@ -226,10 +226,16 @@ func newConfig(resolver *providerconfig.ConfigVarResolver, spec v1alpha1.Provide // postprocessServiceAccount processes the service account and creates a JWT configuration // out of it. func (cfg *config) postprocessServiceAccount() error { - sa, err := base64.StdEncoding.DecodeString(cfg.serviceAccount) - if err != nil { - return fmt.Errorf("failed to decode base64 service account: %w", err) + sa := cfg.serviceAccount + + // safely decode the service account, in case we did not read the value + // from a "known-safe" location (like the MachineDeployment), but from + // an environment variable. + decoded, err := base64.StdEncoding.DecodeString(cfg.serviceAccount) + if err == nil { + sa = string(decoded) } + sam := map[string]string{} err = json.Unmarshal(sa, &sam) if err != nil { diff --git a/pkg/cloudprovider/provider/gce/types/types.go b/pkg/cloudprovider/provider/gce/types/types.go index c1b059d8d..0ac0f3335 100644 --- a/pkg/cloudprovider/provider/gce/types/types.go +++ b/pkg/cloudprovider/provider/gce/types/types.go @@ -30,6 +30,7 @@ import ( // CloudProviderSpec contains the specification of the cloud provider taken // from the provider configuration. type CloudProviderSpec struct { + // ServiceAccount must be base64-encoded. ServiceAccount providerconfigtypes.ConfigVarString `json:"serviceAccount,omitempty"` Zone providerconfigtypes.ConfigVarString `json:"zone"` MachineType providerconfigtypes.ConfigVarString `json:"machineType"` diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index ac90d4a14..eb13c126d 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -720,8 +720,9 @@ func TestGCEProvisioningE2E(t *testing.T) { // Act. GCE does not support CentOS. selector := OsSelector("ubuntu") params := []string{ - fmt.Sprintf("<< GOOGLE_SERVICE_ACCOUNT >>=%s", googleServiceAccount), + fmt.Sprintf("<< GOOGLE_SERVICE_ACCOUNT_BASE64 >>=%s", safeBase64Encoding(googleServiceAccount)), } + runScenarios(t, selector, params, GCEManifest, fmt.Sprintf("gce-%s", *testRunIdentifier)) } diff --git a/test/e2e/provisioning/testdata/machinedeployment-gce.yaml b/test/e2e/provisioning/testdata/machinedeployment-gce.yaml index a2d9eb4d3..5fb0a6c82 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-gce.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-gce.yaml @@ -24,8 +24,9 @@ spec: - "<< YOUR_PUBLIC_KEY >>" cloudProvider: "gce" cloudProviderSpec: - # If empty, can be set via GOOGLE_SERVICE_ACCOUNT env var - serviceAccount: "<< GOOGLE_SERVICE_ACCOUNT >>" + # If empty, can be set via GOOGLE_SERVICE_ACCOUNT env var. The environment variable + # should be plaintext. The value in the cloudProviderSpec however must be base64-encoded. + serviceAccount: "<< GOOGLE_SERVICE_ACCOUNT_BASE64 >>" # See https://cloud.google.com/compute/docs/regions-zones/ zone: "europe-west3-a" # See https://cloud.google.com/compute/docs/machine-types From 43f2da0338cb0fa22641690c008ad4cc5207162b Mon Sep 17 00:00:00 2001 From: Christoph Mewes Date: Mon, 15 Aug 2022 13:13:14 +0200 Subject: [PATCH 3/6] fix weirdness --- examples/gce-machinedeployment.yaml | 4 +--- pkg/cloudprovider/provider/gce/config.go | 2 +- test/e2e/provisioning/all_e2e_test.go | 11 +++++++++++ 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/examples/gce-machinedeployment.yaml b/examples/gce-machinedeployment.yaml index eb90a1616..d4e10ec49 100644 --- a/examples/gce-machinedeployment.yaml +++ b/examples/gce-machinedeployment.yaml @@ -7,9 +7,7 @@ metadata: namespace: kube-system type: Opaque stringData: - # The secret should contain the plaintext service account, but since - # it's a secret, for this YAML file it needs to be base64-encoded. - serviceAccount: "<< GOOGLE_SERVICE_ACCOUNT_BASE64 >>" + serviceAccount: "<< GOOGLE_SERVICE_ACCOUNT >>" --- apiVersion: "cluster.k8s.io/v1alpha1" diff --git a/pkg/cloudprovider/provider/gce/config.go b/pkg/cloudprovider/provider/gce/config.go index 25108db29..cc838a863 100644 --- a/pkg/cloudprovider/provider/gce/config.go +++ b/pkg/cloudprovider/provider/gce/config.go @@ -237,7 +237,7 @@ func (cfg *config) postprocessServiceAccount() error { } sam := map[string]string{} - err = json.Unmarshal(sa, &sam) + err = json.Unmarshal([]byte(sa), &sam) if err != nil { return fmt.Errorf("failed unmarshalling service account: %w", err) } diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index eb13c126d..de13147f5 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -311,6 +311,16 @@ func safeBase64Encoding(value string) string { return base64.StdEncoding.EncodeToString([]byte(value)) } +// safeBase64Decoding takes a value and decodes it with base64 if it was encoded. +func safeBase64Decoding(value string) string { + // If there was no error, the original value was already encoded. + if decoded, err := base64.StdEncoding.DecodeString(value); err == nil { + return string(decoded) + } + + return value +} + func TestOpenstackProvisioningE2E(t *testing.T) { t.Parallel() @@ -720,6 +730,7 @@ func TestGCEProvisioningE2E(t *testing.T) { // Act. GCE does not support CentOS. selector := OsSelector("ubuntu") params := []string{ + fmt.Sprintf("<< GOOGLE_SERVICE_ACCOUNT >>=%s", safeBase64Decoding(googleServiceAccount)), fmt.Sprintf("<< GOOGLE_SERVICE_ACCOUNT_BASE64 >>=%s", safeBase64Encoding(googleServiceAccount)), } From 576fe07322bfdfcc0dc091e721c4d673e8743646 Mon Sep 17 00:00:00 2001 From: Christoph Mewes Date: Mon, 15 Aug 2022 13:16:14 +0200 Subject: [PATCH 4/6] bring back weirdness --- examples/gce-machinedeployment.yaml | 8 ++++++-- test/e2e/provisioning/all_e2e_test.go | 12 +----------- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/examples/gce-machinedeployment.yaml b/examples/gce-machinedeployment.yaml index d4e10ec49..c7ab56ffa 100644 --- a/examples/gce-machinedeployment.yaml +++ b/examples/gce-machinedeployment.yaml @@ -6,8 +6,12 @@ metadata: name: machine-controller-gce namespace: kube-system type: Opaque -stringData: - serviceAccount: "<< GOOGLE_SERVICE_ACCOUNT >>" +data: + # The base64 encoding here is only to satisfy Kubernetes' + # Secret storage and to prevent multiline string replacement + # issues if we used stringData here (because the GCP SA is + # a multiline JSON string). + serviceAccount: "<< GOOGLE_SERVICE_ACCOUNT_BASE64 >>" --- apiVersion: "cluster.k8s.io/v1alpha1" diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index de13147f5..1ca1b09bb 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -311,16 +311,6 @@ func safeBase64Encoding(value string) string { return base64.StdEncoding.EncodeToString([]byte(value)) } -// safeBase64Decoding takes a value and decodes it with base64 if it was encoded. -func safeBase64Decoding(value string) string { - // If there was no error, the original value was already encoded. - if decoded, err := base64.StdEncoding.DecodeString(value); err == nil { - return string(decoded) - } - - return value -} - func TestOpenstackProvisioningE2E(t *testing.T) { t.Parallel() @@ -730,7 +720,7 @@ func TestGCEProvisioningE2E(t *testing.T) { // Act. GCE does not support CentOS. selector := OsSelector("ubuntu") params := []string{ - fmt.Sprintf("<< GOOGLE_SERVICE_ACCOUNT >>=%s", safeBase64Decoding(googleServiceAccount)), + fmt.Sprintf("<< GOOGLE_SERVICE_ACCOUNT >>=%s", googleServiceAccount), fmt.Sprintf("<< GOOGLE_SERVICE_ACCOUNT_BASE64 >>=%s", safeBase64Encoding(googleServiceAccount)), } From ed389ba98f8ddc9a30e46156214d93281faa0892 Mon Sep 17 00:00:00 2001 From: Christoph Mewes Date: Mon, 15 Aug 2022 13:21:12 +0200 Subject: [PATCH 5/6] things --- test/e2e/provisioning/all_e2e_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index 1ca1b09bb..eb13c126d 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -720,7 +720,6 @@ func TestGCEProvisioningE2E(t *testing.T) { // Act. GCE does not support CentOS. selector := OsSelector("ubuntu") params := []string{ - fmt.Sprintf("<< GOOGLE_SERVICE_ACCOUNT >>=%s", googleServiceAccount), fmt.Sprintf("<< GOOGLE_SERVICE_ACCOUNT_BASE64 >>=%s", safeBase64Encoding(googleServiceAccount)), } From 38cfb3c64f8ad0223786b770b9064d00ec0b348e Mon Sep 17 00:00:00 2001 From: Christoph Mewes Date: Mon, 15 Aug 2022 13:26:09 +0200 Subject: [PATCH 6/6] ffs vscode --- pkg/cloudprovider/provider/gce/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/cloudprovider/provider/gce/config.go b/pkg/cloudprovider/provider/gce/config.go index cc838a863..934adca06 100644 --- a/pkg/cloudprovider/provider/gce/config.go +++ b/pkg/cloudprovider/provider/gce/config.go @@ -242,7 +242,7 @@ func (cfg *config) postprocessServiceAccount() error { return fmt.Errorf("failed unmarshalling service account: %w", err) } cfg.projectID = sam["project_id"] - cfg.jwtConfig, err = google.JWTConfigFromJSON(sa, compute.ComputeScope) + cfg.jwtConfig, err = google.JWTConfigFromJSON([]byte(sa), compute.ComputeScope) if err != nil { return fmt.Errorf("failed preparing JWT: %w", err) }