registry mirror credentials for etcd machines (mrajashree#15)

controllers/etcdadmconfig_controller.go

@@ -19,6 +19,7 @@ package controllers
 import (
 	"context"
 	"fmt"
+	"k8s.io/apimachinery/pkg/types"
 	"path/filepath"
 	"time"
 
@@ -49,6 +50,9 @@ import (
 )
 
 const stopKubeletCommand = "systemctl stop kubelet"
+const registrySecretName = "registry-credentials"
+const registryUsernameKey = "username"
+const registryPasswordKey = "password"
 
 // InitLocker is a lock that is used around etcdadm init
 type InitLocker interface {
@@ -240,6 +244,17 @@ func (r *EtcdadmConfigReconciler) initializeEtcd(ctx context.Context, scope *Sco
 		Certificates: CACertKeyPair,
 	}
 
+	// grab user pass for registry mirror
+	if &scope.Config.Spec.RegistryMirror != nil {
+		username, password, err := r.resolveRegistryCredentials(ctx, scope.Config)
+		if err != nil {
+			log.Info("Cannot find secret for registry credentials, proceeding without registry credentials")
+		} else {
+			initInput.RegistryMirrorCredentials.Username = string(username)
+			initInput.RegistryMirrorCredentials.Password = string(password)
+		}
+	}
+
 	// only do this if etcdadm not baked in image
 	if !scope.Config.Spec.EtcdadmBuiltin {
 		if len(scope.Config.Spec.EtcdadmInstallCommands) > 0 {
@@ -312,6 +327,17 @@ func (r *EtcdadmConfigReconciler) joinEtcd(ctx context.Context, scope *Scope) (_
 		Certificates: etcdCerts,
 	}
 
+	// grab user pass for registry mirror
+	if &scope.Config.Spec.RegistryMirror != nil {
+		username, password, err := r.resolveRegistryCredentials(ctx, scope.Config)
+		if err != nil {
+			log.Info("Cannot find secret for registry credentials, proceeding without registry credentials")
+		} else {
+			joinInput.RegistryMirrorCredentials.Username = string(username)
+			joinInput.RegistryMirrorCredentials.Password = string(password)
+		}
+	}
+
 	if !scope.Config.Spec.EtcdadmBuiltin {
 		if len(scope.Config.Spec.EtcdadmInstallCommands) > 0 {
 			joinInput.PreEtcdadmCommands = append(joinInput.PreEtcdadmCommands, scope.Config.Spec.EtcdadmInstallCommands...)
@@ -399,3 +425,23 @@ func (r *EtcdadmConfigReconciler) storeBootstrapData(ctx context.Context, config
 	conditions.MarkTrue(config, bootstrapv1.DataSecretAvailableCondition)
 	return nil
 }
+
+func (r *EtcdadmConfigReconciler) resolveRegistryCredentials(ctx context.Context, config *etcdbootstrapv1.EtcdadmConfig) ([]byte, []byte, error) {
+	secret := &corev1.Secret{}
+	key := types.NamespacedName{Namespace: config.Namespace, Name: registrySecretName}
+	if err := r.Client.Get(ctx, key, secret); err != nil {
+		if apierrors.IsNotFound(err) {
+			return nil, nil, errors.Wrapf(err, "secret not found: %s", key)
+		}
+		return nil, nil, errors.Wrapf(err, "failed to retrieve Secret %q", key)
+	}
+	username, ok := secret.Data[registryUsernameKey]
+	if !ok {
+		return nil, nil, errors.Errorf("secret references non-existent secret key: %q", "username")
+	}
+	password, ok := secret.Data[registryPasswordKey]
+	if !ok {
+		return nil, nil, errors.Errorf("secret references non-existent secret key: %q", "password")
+	}
+	return username, password, nil
+}

pkg/userdata/bottlerocket/node_userdata.go

@@ -8,6 +8,7 @@ import (
 	"text/template"
 
 	etcdbootstrapv1 "github.com/aws/etcdadm-bootstrap-provider/api/v1beta1"
+	"github.com/aws/etcdadm-bootstrap-provider/pkg/userdata"
 	"github.com/go-logr/logr"
 	"github.com/pkg/errors"
 	bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
@@ -70,6 +71,17 @@ no-proxy = [{{stringsJoin .NoProxyEndpoints "," }}]
 data = "{{.RegistryMirrorCACert}}"
 trusted=true
 {{- end -}}
+`
+	registryMirrorCredentialsTemplate = `{{ define "registryMirrorCredentialsSettings" -}}
+[[settings.container-registry.credentials]]
+registry = "public.ecr.aws"
+username = "{{.RegistryMirrorUsername}}"
+password = "{{.RegistryMirrorPassword}}"
+[[settings.container-registry.credentials]]
+registry = "{{.RegistryMirrorEndpoint}}"
+username = "{{.RegistryMirrorUsername}}"
+password = "{{.RegistryMirrorPassword}}"
+{{- end -}}
 `
 	bottlerocketNodeInitSettingsTemplate = `{{template "hostContainersSettings" .}}
 
@@ -90,6 +102,10 @@ trusted=true
 {{- if (ne .RegistryMirrorCACert "")}}
 {{template "registryMirrorCACertSettings" .}}
 {{- end -}}
+
+{{- if and (ne .RegistryMirrorUsername "") (ne .RegistryMirrorPassword "")}}
+{{template "registryMirrorCredentialsSettings" .}}
+{{- end -}}
 `
 )
 
@@ -99,12 +115,14 @@ type bottlerocketSettingsInput struct {
 	NoProxyEndpoints       []string
 	RegistryMirrorEndpoint string
 	RegistryMirrorCACert   string
+	RegistryMirrorUsername string
+	RegistryMirrorPassword string
 	HostContainers         []etcdbootstrapv1.BottlerocketHostContainer
 	BootstrapContainers    []etcdbootstrapv1.BottlerocketBootstrapContainer
 }
 
 // generateBottlerocketNodeUserData returns the userdata for the host bottlerocket in toml format
-func generateBottlerocketNodeUserData(kubeadmBootstrapContainerUserData []byte, users []bootstrapv1.User, config etcdbootstrapv1.EtcdadmConfigSpec, log logr.Logger) ([]byte, error) {
+func generateBottlerocketNodeUserData(kubeadmBootstrapContainerUserData []byte, users []bootstrapv1.User, registryMirrorCredentials userdata.RegistryMirrorCredentials, config etcdbootstrapv1.EtcdadmConfigSpec, log logr.Logger) ([]byte, error) {
 	// base64 encode the kubeadm bootstrapContainer's user data
 	b64KubeadmBootstrapContainerUserData := base64.StdEncoding.EncodeToString(kubeadmBootstrapContainerUserData)
 
@@ -159,6 +177,8 @@ func generateBottlerocketNodeUserData(kubeadmBootstrapContainerUserData []byte,
 		if config.RegistryMirror.CACert != "" {
 			bottlerocketInput.RegistryMirrorCACert = base64.StdEncoding.EncodeToString([]byte(config.RegistryMirror.CACert))
 		}
+		bottlerocketInput.RegistryMirrorUsername = registryMirrorCredentials.Username
+		bottlerocketInput.RegistryMirrorPassword = registryMirrorCredentials.Password
 	}
 
 	bottlerocketNodeUserData, err := generateNodeUserData("InitBottlerocketNode", bottlerocketNodeInitSettingsTemplate, bottlerocketInput)
@@ -219,6 +239,9 @@ func generateNodeUserData(kind string, tpl string, data interface{}) ([]byte, er
 	if _, err := tm.Parse(registryMirrorCACertTemplate); err != nil {
 		return nil, errors.Wrapf(err, "failed to parse registry mirror ca cert %s template", kind)
 	}
+	if _, err := tm.Parse(registryMirrorCredentialsTemplate); err != nil {
+		return nil, errors.Wrapf(err, "failed to parse registry mirror credentials %s template", kind)
+	}
 
 	t, err := tm.Parse(tpl)
 	if err != nil {

pkg/userdata/bottlerocket/node_userdata_test.go

@@ -9,6 +9,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/log"
 
 	"github.com/aws/etcdadm-bootstrap-provider/api/v1beta1"
+	"github.com/aws/etcdadm-bootstrap-provider/pkg/userdata"
 )
 
 const userDataMinimum = `
@@ -105,13 +106,45 @@ mode = "once"
 source = "custom-bootstrap-image-2"
 user-data = "xyz"`
 
+const userDataWithRegistryAuth = `
+[settings.host-containers.admin]
+enabled = true
+superpowered = true
+user-data = "CnsKCSJzc2giOiB7CgkJImF1dGhvcml6ZWQta2V5cyI6IFsic3NoLWtleSJdCgl9Cn0="
+[settings.host-containers.kubeadm-bootstrap]
+enabled = true
+superpowered = true
+source = "kubeadm-bootstrap-image"
+user-data = "a3ViZWFkbUJvb3RzdHJhcFVzZXJEYXRh"
+
+[settings.kubernetes]
+cluster-domain = "cluster.local"
+standalone-mode = true
+authentication-mode = "tls"
+server-tls-bootstrap = false
+pod-infra-container-image = "pause-image"
+[settings.container-registry.mirrors]
+"public.ecr.aws" = ["https://registry-endpoint"]
+[settings.pki.registry-mirror-ca]
+data = "Y2FjZXJ0"
+trusted=true
+[[settings.container-registry.credentials]]
+registry = "public.ecr.aws"
+username = "username"
+password = "password"
+[[settings.container-registry.credentials]]
+registry = "registry-endpoint"
+username = "username"
+password = "password"`
+
 func TestGenerateBottlerocketNodeUserData(t *testing.T) {
 	g := NewWithT(t)
 
 	testcases := []struct {
 		name                     string
 		kubeadmBootstrapUserData string
 		users                    []bootstrapv1.User
+		registryCredentials      userdata.RegistryMirrorCredentials
 		etcdConfig               v1beta1.EtcdadmConfigSpec
 		output                   string
 	}{
@@ -217,10 +250,36 @@ func TestGenerateBottlerocketNodeUserData(t *testing.T) {
 			},
 			output: userDataWithProxyRegistryBootstrapContainers,
 		},
+		{
+			name:                     "with registry with authentication",
+			kubeadmBootstrapUserData: "kubeadmBootstrapUserData",
+			users: []bootstrapv1.User{
+				{
+					SSHAuthorizedKeys: []string{
+						"ssh-key",
+					},
+				},
+			},
+			registryCredentials: userdata.RegistryMirrorCredentials{
+				Username: "username",
+				Password: "password",
+			},
+			etcdConfig: v1beta1.EtcdadmConfigSpec{
+				BottlerocketConfig: &v1beta1.BottlerocketConfig{
+					BootstrapImage: "kubeadm-bootstrap-image",
+					PauseImage:     "pause-image",
+				},
+				RegistryMirror: &v1beta1.RegistryMirrorConfiguration{
+					Endpoint: "registry-endpoint",
+					CACert:   "cacert",
+				},
+			},
+			output: userDataWithRegistryAuth,
+		},
 	}
 	for _, testcase := range testcases {
 		t.Run(testcase.name, func(t *testing.T) {
-			b, err := generateBottlerocketNodeUserData([]byte(testcase.kubeadmBootstrapUserData), testcase.users, testcase.etcdConfig, logr.New(log.NullLogSink{}))
+			b, err := generateBottlerocketNodeUserData([]byte(testcase.kubeadmBootstrapUserData), testcase.users, testcase.registryCredentials, testcase.etcdConfig, logr.New(log.NullLogSink{}))
 			g.Expect(err).NotTo(HaveOccurred())
 			g.Expect(string(b)).To(Equal(testcase.output))
 		})

pkg/userdata/bottlerocket/template.go

@@ -29,7 +29,7 @@ func generateUserData(kind string, tpl string, data interface{}, input *userdata
 		return nil, err
 	}
 
-	return generateBottlerocketNodeUserData(bootstrapContainerUserData, input.Users, config, log)
+	return generateBottlerocketNodeUserData(bootstrapContainerUserData, input.Users, input.RegistryMirrorCredentials, config, log)
 }
 
 func generateBootstrapContainerUserData(kind string, tpl string, data interface{}) ([]byte, error) {

pkg/userdata/input.go

@@ -40,6 +40,7 @@ type BaseUserData struct {
 	Mounts              []bootstrapv1.MountPoints
 	ControlPlane        bool
 	SentinelFileCommand string
+	RegistryMirrorCredentials
 }
 
 type EtcdadmArgs struct {
@@ -50,6 +51,11 @@ type EtcdadmArgs struct {
 	CipherSuites    string
 }
 
+type RegistryMirrorCredentials struct {
+	Username string
+	Password string
+}
+
 func (args *EtcdadmArgs) SystemdFlags() []string {
 	flags := make([]string, 0, 3)
 	flags = append(flags, "--init-system systemd")