From 3e7f51b599b66ac1be798db706ffcde895229b5e Mon Sep 17 00:00:00 2001 From: ahreehong <46465244+ahreehong@users.noreply.github.com> Date: Fri, 13 Jan 2023 13:02:44 -0800 Subject: [PATCH] registry mirror credentials for etcd machines (#15) --- controllers/etcdadmconfig_controller.go | 46 ++++++++++++++ pkg/userdata/bottlerocket/node_userdata.go | 25 +++++++- .../bottlerocket/node_userdata_test.go | 61 ++++++++++++++++++- pkg/userdata/bottlerocket/template.go | 2 +- pkg/userdata/input.go | 6 ++ 5 files changed, 137 insertions(+), 3 deletions(-) diff --git a/controllers/etcdadmconfig_controller.go b/controllers/etcdadmconfig_controller.go index 1b922d6..f9a430e 100644 --- a/controllers/etcdadmconfig_controller.go +++ b/controllers/etcdadmconfig_controller.go @@ -19,6 +19,7 @@ package controllers import ( "context" "fmt" + "k8s.io/apimachinery/pkg/types" "path/filepath" "time" @@ -49,6 +50,9 @@ import ( ) const stopKubeletCommand = "systemctl stop kubelet" +const registrySecretName = "registry-credentials" +const registryUsernameKey = "username" +const registryPasswordKey = "password" // InitLocker is a lock that is used around etcdadm init type InitLocker interface { @@ -240,6 +244,17 @@ func (r *EtcdadmConfigReconciler) initializeEtcd(ctx context.Context, scope *Sco Certificates: CACertKeyPair, } + // grab user pass for registry mirror + if &scope.Config.Spec.RegistryMirror != nil { + username, password, err := r.resolveRegistryCredentials(ctx, scope.Config) + if err != nil { + log.Info("Cannot find secret for registry credentials, proceeding without registry credentials") + } else { + initInput.RegistryMirrorCredentials.Username = string(username) + initInput.RegistryMirrorCredentials.Password = string(password) + } + } + // only do this if etcdadm not baked in image if !scope.Config.Spec.EtcdadmBuiltin { if len(scope.Config.Spec.EtcdadmInstallCommands) > 0 { @@ -312,6 +327,17 @@ func (r *EtcdadmConfigReconciler) joinEtcd(ctx context.Context, scope *Scope) (_ Certificates: etcdCerts, } + // grab user pass for registry mirror + if &scope.Config.Spec.RegistryMirror != nil { + username, password, err := r.resolveRegistryCredentials(ctx, scope.Config) + if err != nil { + log.Info("Cannot find secret for registry credentials, proceeding without registry credentials") + } else { + joinInput.RegistryMirrorCredentials.Username = string(username) + joinInput.RegistryMirrorCredentials.Password = string(password) + } + } + if !scope.Config.Spec.EtcdadmBuiltin { if len(scope.Config.Spec.EtcdadmInstallCommands) > 0 { joinInput.PreEtcdadmCommands = append(joinInput.PreEtcdadmCommands, scope.Config.Spec.EtcdadmInstallCommands...) @@ -399,3 +425,23 @@ func (r *EtcdadmConfigReconciler) storeBootstrapData(ctx context.Context, config conditions.MarkTrue(config, bootstrapv1.DataSecretAvailableCondition) return nil } + +func (r *EtcdadmConfigReconciler) resolveRegistryCredentials(ctx context.Context, config *etcdbootstrapv1.EtcdadmConfig) ([]byte, []byte, error) { + secret := &corev1.Secret{} + key := types.NamespacedName{Namespace: config.Namespace, Name: registrySecretName} + if err := r.Client.Get(ctx, key, secret); err != nil { + if apierrors.IsNotFound(err) { + return nil, nil, errors.Wrapf(err, "secret not found: %s", key) + } + return nil, nil, errors.Wrapf(err, "failed to retrieve Secret %q", key) + } + username, ok := secret.Data[registryUsernameKey] + if !ok { + return nil, nil, errors.Errorf("secret references non-existent secret key: %q", "username") + } + password, ok := secret.Data[registryPasswordKey] + if !ok { + return nil, nil, errors.Errorf("secret references non-existent secret key: %q", "password") + } + return username, password, nil +} diff --git a/pkg/userdata/bottlerocket/node_userdata.go b/pkg/userdata/bottlerocket/node_userdata.go index f781cc8..3640489 100644 --- a/pkg/userdata/bottlerocket/node_userdata.go +++ b/pkg/userdata/bottlerocket/node_userdata.go @@ -8,6 +8,7 @@ import ( "text/template" etcdbootstrapv1 "github.com/aws/etcdadm-bootstrap-provider/api/v1beta1" + "github.com/aws/etcdadm-bootstrap-provider/pkg/userdata" "github.com/go-logr/logr" "github.com/pkg/errors" bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" @@ -70,6 +71,17 @@ no-proxy = [{{stringsJoin .NoProxyEndpoints "," }}] data = "{{.RegistryMirrorCACert}}" trusted=true {{- end -}} +` + registryMirrorCredentialsTemplate = `{{ define "registryMirrorCredentialsSettings" -}} +[[settings.container-registry.credentials]] +registry = "public.ecr.aws" +username = "{{.RegistryMirrorUsername}}" +password = "{{.RegistryMirrorPassword}}" +[[settings.container-registry.credentials]] +registry = "{{.RegistryMirrorEndpoint}}" +username = "{{.RegistryMirrorUsername}}" +password = "{{.RegistryMirrorPassword}}" +{{- end -}} ` bottlerocketNodeInitSettingsTemplate = `{{template "hostContainersSettings" .}} @@ -90,6 +102,10 @@ trusted=true {{- if (ne .RegistryMirrorCACert "")}} {{template "registryMirrorCACertSettings" .}} {{- end -}} + +{{- if and (ne .RegistryMirrorUsername "") (ne .RegistryMirrorPassword "")}} +{{template "registryMirrorCredentialsSettings" .}} +{{- end -}} ` ) @@ -99,12 +115,14 @@ type bottlerocketSettingsInput struct { NoProxyEndpoints []string RegistryMirrorEndpoint string RegistryMirrorCACert string + RegistryMirrorUsername string + RegistryMirrorPassword string HostContainers []etcdbootstrapv1.BottlerocketHostContainer BootstrapContainers []etcdbootstrapv1.BottlerocketBootstrapContainer } // generateBottlerocketNodeUserData returns the userdata for the host bottlerocket in toml format -func generateBottlerocketNodeUserData(kubeadmBootstrapContainerUserData []byte, users []bootstrapv1.User, config etcdbootstrapv1.EtcdadmConfigSpec, log logr.Logger) ([]byte, error) { +func generateBottlerocketNodeUserData(kubeadmBootstrapContainerUserData []byte, users []bootstrapv1.User, registryMirrorCredentials userdata.RegistryMirrorCredentials, config etcdbootstrapv1.EtcdadmConfigSpec, log logr.Logger) ([]byte, error) { // base64 encode the kubeadm bootstrapContainer's user data b64KubeadmBootstrapContainerUserData := base64.StdEncoding.EncodeToString(kubeadmBootstrapContainerUserData) @@ -159,6 +177,8 @@ func generateBottlerocketNodeUserData(kubeadmBootstrapContainerUserData []byte, if config.RegistryMirror.CACert != "" { bottlerocketInput.RegistryMirrorCACert = base64.StdEncoding.EncodeToString([]byte(config.RegistryMirror.CACert)) } + bottlerocketInput.RegistryMirrorUsername = registryMirrorCredentials.Username + bottlerocketInput.RegistryMirrorPassword = registryMirrorCredentials.Password } bottlerocketNodeUserData, err := generateNodeUserData("InitBottlerocketNode", bottlerocketNodeInitSettingsTemplate, bottlerocketInput) @@ -219,6 +239,9 @@ func generateNodeUserData(kind string, tpl string, data interface{}) ([]byte, er if _, err := tm.Parse(registryMirrorCACertTemplate); err != nil { return nil, errors.Wrapf(err, "failed to parse registry mirror ca cert %s template", kind) } + if _, err := tm.Parse(registryMirrorCredentialsTemplate); err != nil { + return nil, errors.Wrapf(err, "failed to parse registry mirror credentials %s template", kind) + } t, err := tm.Parse(tpl) if err != nil { diff --git a/pkg/userdata/bottlerocket/node_userdata_test.go b/pkg/userdata/bottlerocket/node_userdata_test.go index 3d3afc8..dc87696 100644 --- a/pkg/userdata/bottlerocket/node_userdata_test.go +++ b/pkg/userdata/bottlerocket/node_userdata_test.go @@ -9,6 +9,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" "github.com/aws/etcdadm-bootstrap-provider/api/v1beta1" + "github.com/aws/etcdadm-bootstrap-provider/pkg/userdata" ) const userDataMinimum = ` @@ -105,6 +106,37 @@ mode = "once" source = "custom-bootstrap-image-2" user-data = "xyz"` +const userDataWithRegistryAuth = ` +[settings.host-containers.admin] +enabled = true +superpowered = true +user-data = "CnsKCSJzc2giOiB7CgkJImF1dGhvcml6ZWQta2V5cyI6IFsic3NoLWtleSJdCgl9Cn0=" +[settings.host-containers.kubeadm-bootstrap] +enabled = true +superpowered = true +source = "kubeadm-bootstrap-image" +user-data = "a3ViZWFkbUJvb3RzdHJhcFVzZXJEYXRh" + +[settings.kubernetes] +cluster-domain = "cluster.local" +standalone-mode = true +authentication-mode = "tls" +server-tls-bootstrap = false +pod-infra-container-image = "pause-image" +[settings.container-registry.mirrors] +"public.ecr.aws" = ["https://registry-endpoint"] +[settings.pki.registry-mirror-ca] +data = "Y2FjZXJ0" +trusted=true +[[settings.container-registry.credentials]] +registry = "public.ecr.aws" +username = "username" +password = "password" +[[settings.container-registry.credentials]] +registry = "registry-endpoint" +username = "username" +password = "password"` + func TestGenerateBottlerocketNodeUserData(t *testing.T) { g := NewWithT(t) @@ -112,6 +144,7 @@ func TestGenerateBottlerocketNodeUserData(t *testing.T) { name string kubeadmBootstrapUserData string users []bootstrapv1.User + registryCredentials userdata.RegistryMirrorCredentials etcdConfig v1beta1.EtcdadmConfigSpec output string }{ @@ -217,10 +250,36 @@ func TestGenerateBottlerocketNodeUserData(t *testing.T) { }, output: userDataWithProxyRegistryBootstrapContainers, }, + { + name: "with registry with authentication", + kubeadmBootstrapUserData: "kubeadmBootstrapUserData", + users: []bootstrapv1.User{ + { + SSHAuthorizedKeys: []string{ + "ssh-key", + }, + }, + }, + registryCredentials: userdata.RegistryMirrorCredentials{ + Username: "username", + Password: "password", + }, + etcdConfig: v1beta1.EtcdadmConfigSpec{ + BottlerocketConfig: &v1beta1.BottlerocketConfig{ + BootstrapImage: "kubeadm-bootstrap-image", + PauseImage: "pause-image", + }, + RegistryMirror: &v1beta1.RegistryMirrorConfiguration{ + Endpoint: "registry-endpoint", + CACert: "cacert", + }, + }, + output: userDataWithRegistryAuth, + }, } for _, testcase := range testcases { t.Run(testcase.name, func(t *testing.T) { - b, err := generateBottlerocketNodeUserData([]byte(testcase.kubeadmBootstrapUserData), testcase.users, testcase.etcdConfig, logr.New(log.NullLogSink{})) + b, err := generateBottlerocketNodeUserData([]byte(testcase.kubeadmBootstrapUserData), testcase.users, testcase.registryCredentials, testcase.etcdConfig, logr.New(log.NullLogSink{})) g.Expect(err).NotTo(HaveOccurred()) g.Expect(string(b)).To(Equal(testcase.output)) }) diff --git a/pkg/userdata/bottlerocket/template.go b/pkg/userdata/bottlerocket/template.go index 2374215..f440fba 100644 --- a/pkg/userdata/bottlerocket/template.go +++ b/pkg/userdata/bottlerocket/template.go @@ -29,7 +29,7 @@ func generateUserData(kind string, tpl string, data interface{}, input *userdata return nil, err } - return generateBottlerocketNodeUserData(bootstrapContainerUserData, input.Users, config, log) + return generateBottlerocketNodeUserData(bootstrapContainerUserData, input.Users, input.RegistryMirrorCredentials, config, log) } func generateBootstrapContainerUserData(kind string, tpl string, data interface{}) ([]byte, error) { diff --git a/pkg/userdata/input.go b/pkg/userdata/input.go index f6a12b2..2215e90 100644 --- a/pkg/userdata/input.go +++ b/pkg/userdata/input.go @@ -40,6 +40,7 @@ type BaseUserData struct { Mounts []bootstrapv1.MountPoints ControlPlane bool SentinelFileCommand string + RegistryMirrorCredentials } type EtcdadmArgs struct { @@ -50,6 +51,11 @@ type EtcdadmArgs struct { CipherSuites string } +type RegistryMirrorCredentials struct { + Username string + Password string +} + func (args *EtcdadmArgs) SystemdFlags() []string { flags := make([]string, 0, 3) flags = append(flags, "--init-system systemd")