WIP: ✨ Etcd EBS Volumes

api/v1alpha2/awsmachine_conversion.go

@@ -50,6 +50,10 @@ func restoreAWSMachineSpec(restored *infrav1alpha3.AWSMachineSpec, dst *infrav1a
 		restored.RootVolume.DeepCopyInto(dst.RootVolume)
 	}
 
+	if restored.EtcdVolume != nil {
+		restored.EtcdVolume.DeepCopyInto(dst.EtcdVolume)
+	}
+
 	// override the SSHKeyName conversion if we are roundtripping from v1alpha3 and the v1alpha3 value is nil
 	if dst.SSHKeyName != nil && *dst.SSHKeyName == "" && restored.SSHKeyName == nil {
 		dst.SSHKeyName = nil

api/v1alpha3/awsmachine_types.go

@@ -88,6 +88,10 @@ type AWSMachineSpec struct {
 	// +optional
 	RootVolume *RootVolume `json:"rootVolume,omitempty"`
 
+	// EtcdVolume encapsulates the configuration options for the etcd data-store volume
+	// +optional
+	EtcdVolume *EtcdVolume `json:"etcdVolume,omitempty"`
+
 	// NetworkInterfaces is a list of ENIs to associate with the instance.
 	// A maximum of 2 may be specified.
 	// +optional

api/v1alpha3/types.go

@@ -542,6 +542,10 @@ type Instance struct {
 	// +optional
 	RootVolume *RootVolume `json:"rootVolume,omitempty"`
 
+	// Configuration options for the etcd storage volume.
+	// +optional
+	EtcdVolume *EtcdVolume `json:"etcdVolume,omitempty"`
+
 	// Specifies ENIs attached to instance
 	NetworkInterfaces []string `json:"networkInterfaces,omitempty"`
 
@@ -574,3 +578,29 @@ type RootVolume struct {
 	// +optional
 	EncryptionKey string `json:"encryptionKey,omitempty"`
 }
+
+// EtcdVolume encapsulates the configuration options for the etcd volume
+type EtcdVolume struct {
+	// Size specifies size (in Gi) of the root storage device.
+	// Must be greater than the image root snapshot size or 8 (whichever is greater).
+	// +kubebuilder:validation:Minimum=8
+	Size int64 `json:"size"`
+
+	// Type is the type of the root volume (e.g. gp2, io1, etc...).
+	// +optional
+	Type string `json:"type,omitempty"`
+
+	// IOPS is the number of IOPS requested for the disk. Not applicable to all types.
+	// +optional
+	IOPS int64 `json:"iops,omitempty"`
+
+	// Encrypted is whether the volume should be encrypted or not.
+	// +optional
+	Encrypted bool `json:"encrypted,omitempty"`
+
+	// EncryptionKey is the KMS key to use to encrypt the volume. Can be either a KMS key ID or ARN.
+	// If Encrypted is set and this is omitted, the default AWS key will be used.
+	// The key must already exist and be accessible by the controller.
+	// +optional
+	EncryptionKey string `json:"encryptionKey,omitempty"`
+}

config/crd/bases/infrastructure.cluster.x-k8s.io_awsclusters.yaml

@@ -632,6 +632,39 @@ spec:
                     description: Specifies whether enhanced networking with ENA is
                       enabled.
                     type: boolean
+                  etcdVolume:
+                    description: Configuration options for the etcd storage volume.
+                    properties:
+                      encrypted:
+                        description: Encrypted is whether the volume should be encrypted
+                          or not.
+                        type: boolean
+                      encryptionKey:
+                        description: EncryptionKey is the KMS key to use to encrypt
+                          the volume. Can be either a KMS key ID or ARN. If Encrypted
+                          is set and this is omitted, the default AWS key will be
+                          used. The key must already exist and be accessible by the
+                          controller.
+                        type: string
+                      iops:
+                        description: IOPS is the number of IOPS requested for the
+                          disk. Not applicable to all types.
+                        format: int64
+                        type: integer
+                      size:
+                        description: Size specifies size (in Gi) of the root storage
+                          device. Must be greater than the image root snapshot size
+                          or 8 (whichever is greater).
+                        format: int64
+                        minimum: 8
+                        type: integer
+                      type:
+                        description: Type is the type of the root volume (e.g. gp2,
+                          io1, etc...).
+                        type: string
+                    required:
+                    - size
+                    type: object
                   iamProfile:
                     description: The name of the IAM instance profile associated with
                       the instance, if applicable.

config/crd/bases/infrastructure.cluster.x-k8s.io_awsmachines.yaml

@@ -419,6 +419,39 @@ spec:
                       as a node against the workload cluster.
                     type: string
                 type: object
+              etcdVolume:
+                description: EtcdVolume encapsulates the configuration options for
+                  the etcd data-store volume
+                properties:
+                  encrypted:
+                    description: Encrypted is whether the volume should be encrypted
+                      or not.
+                    type: boolean
+                  encryptionKey:
+                    description: EncryptionKey is the KMS key to use to encrypt the
+                      volume. Can be either a KMS key ID or ARN. If Encrypted is set
+                      and this is omitted, the default AWS key will be used. The key
+                      must already exist and be accessible by the controller.
+                    type: string
+                  iops:
+                    description: IOPS is the number of IOPS requested for the disk.
+                      Not applicable to all types.
+                    format: int64
+                    type: integer
+                  size:
+                    description: Size specifies size (in Gi) of the root storage device.
+                      Must be greater than the image root snapshot size or 8 (whichever
+                      is greater).
+                    format: int64
+                    minimum: 8
+                    type: integer
+                  type:
+                    description: Type is the type of the root volume (e.g. gp2, io1,
+                      etc...).
+                    type: string
+                required:
+                - size
+                type: object
               failureDomain:
                 description: FailureDomain is the failure domain unique identifier
                   this Machine should be attached to, as defined in Cluster API. For

config/crd/bases/infrastructure.cluster.x-k8s.io_awsmachinetemplates.yaml

@@ -378,6 +378,40 @@ spec:
                               machine registers as a node against the workload cluster.
                             type: string
                         type: object
+                      etcdVolume:
+                        description: EtcdVolume encapsulates the configuration options
+                          for the etcd data-store volume
+                        properties:
+                          encrypted:
+                            description: Encrypted is whether the volume should be
+                              encrypted or not.
+                            type: boolean
+                          encryptionKey:
+                            description: EncryptionKey is the KMS key to use to encrypt
+                              the volume. Can be either a KMS key ID or ARN. If Encrypted
+                              is set and this is omitted, the default AWS key will
+                              be used. The key must already exist and be accessible
+                              by the controller.
+                            type: string
+                          iops:
+                            description: IOPS is the number of IOPS requested for
+                              the disk. Not applicable to all types.
+                            format: int64
+                            type: integer
+                          size:
+                            description: Size specifies size (in Gi) of the root storage
+                              device. Must be greater than the image root snapshot
+                              size or 8 (whichever is greater).
+                            format: int64
+                            minimum: 8
+                            type: integer
+                          type:
+                            description: Type is the type of the root volume (e.g.
+                              gp2, io1, etc...).
+                            type: string
+                        required:
+                        - size
+                        type: object
                       failureDomain:
                         description: FailureDomain is the failure domain unique identifier
                           this Machine should be attached to, as defined in Cluster

pkg/cloud/services/ec2/instances.go

@@ -108,6 +108,7 @@ func (s *Service) CreateInstance(scope *scope.MachineScope, userData []byte) (*i
 		Type:              scope.AWSMachine.Spec.InstanceType,
 		IAMProfile:        scope.AWSMachine.Spec.IAMInstanceProfile,
 		RootVolume:        scope.AWSMachine.Spec.RootVolume,
+		EtcdVolume:        scope.AWSMachine.Spec.EtcdVolume,
 		NetworkInterfaces: scope.AWSMachine.Spec.NetworkInterfaces,
 	}
 
@@ -395,6 +396,37 @@ func (s *Service) runInstance(role string, i *infrav1.Instance) (*infrav1.Instan
 		}
 	}
 
+	if i.EtcdVolume != nil {
+
+		etcdDeviceName := aws.String("/dev/sdb")
+
+		ebsEtcdDevice := &ec2.EbsBlockDevice{
+			DeleteOnTermination: aws.Bool(true),
+			VolumeSize:          aws.Int64(i.EtcdVolume.Size),
+			Encrypted:           aws.Bool(i.DeepCopy().EtcdVolume.Encrypted),
+		}
+
+		if i.EtcdVolume.IOPS != 0 {
+			ebsEtcdDevice.Iops = aws.Int64(i.EtcdVolume.IOPS)
+		}
+
+		if i.EtcdVolume.EncryptionKey != "" {
+			ebsEtcdDevice.Encrypted = aws.Bool(true)
+			ebsEtcdDevice.KmsKeyId = aws.String(i.EtcdVolume.EncryptionKey)
+		}
+
+		if i.EtcdVolume.Type != "" {
+			ebsEtcdDevice.VolumeType = aws.String(i.EtcdVolume.Type)
+		}
+
+		input.BlockDeviceMappings = append(input.BlockDeviceMappings, []*ec2.BlockDeviceMapping{
+			{
+				DeviceName: etcdDeviceName,
+				Ebs:        ebsEtcdDevice,
+			},
+		}...)
+	}
+
 	if len(i.Tags) > 0 {
 		spec := &ec2.TagSpecification{ResourceType: aws.String(ec2.ResourceTypeInstance)}
 		for key, value := range i.Tags {

templates/cluster-template.yaml

@@ -23,6 +23,8 @@ metadata:
 spec:
   region: "${AWS_REGION}"
   sshKeyName: "${AWS_SSH_KEY_NAME}"
+  bastion:
+    enabled: true
 ---
 kind: KubeadmControlPlane
 apiVersion: controlplane.cluster.x-k8s.io/v1alpha3
@@ -35,6 +37,18 @@ spec:
     apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3
     name: "${CLUSTER_NAME}-control-plane"
   kubeadmConfigSpec:
+    preKubeadmCommands:
+    - if [ -d "/var/lib/etcd/lost+found" ]; then rm -Rf /var/lib/etcd/lost+found; fi
+    diskSetup:
+      filesystems:
+      - device: /dev/nvme1n1p1
+        filesystem: ext4
+        label: etcd_disk
+      partitions:
+      - device: /dev/nvme1n1
+        layout: true
+        overwrite: false
+        tableType: mbr
     initConfiguration:
       nodeRegistration:
         name: '{{ ds.meta_data.local_hostname }}'
@@ -106,6 +120,16 @@ metadata:
 spec:
   template:
     spec:
+    diskSetup:
+      filesystems:
+      - device: /dev/nvme1n1p1
+        filesystem: ext4
+        label: etcd_disk
+      partitions:
+      - device: /dev/nvme1n1
+        layout: true
+        overwrite: false
+        tableType: mbr
       joinConfiguration:
         nodeRegistration:
           name: '{{ ds.meta_data.local_hostname }}'