Merge pull request #4 from newrelic-forks/kubernetes-sigs-main

Kubernetes sigs main

.gitattributes

@@ -1,2 +1,5 @@
 # Specify generated cluster templates as generated files
 **/cluster-template-*.yaml linguist-generated
+
+*.sh text eol=lf
+*.yaml text eol=lf

Makefile

@@ -116,6 +116,10 @@ YQ_VER := v4.14.2
 YQ_BIN := yq
 YQ :=  $(TOOLS_BIN_DIR)/$(YQ_BIN)-$(YQ_VER)
 
+KIND_VER := v0.14.0
+KIND_BIN := kind
+KIND :=  $(TOOLS_BIN_DIR)/$(KIND_BIN)-$(KIND_VER)
+
 KUBE_APISERVER=$(TOOLS_BIN_DIR)/kube-apiserver
 ETCD=$(TOOLS_BIN_DIR)/etcd
 
@@ -242,10 +246,10 @@ verify-tiltfile: ## Verify Tiltfile format.
 ##@ Development:
 
 .PHONY: install-tools # populate hack/tools/bin
-install-tools: $(ENVSUBST) $(KUSTOMIZE) $(KUBECTL) $(HELM) $(GINKGO)
+install-tools: $(ENVSUBST) $(KUSTOMIZE) $(KUBECTL) $(HELM) $(GINKGO) $(KIND)
 
 .PHONY: create-management-cluster
-create-management-cluster: $(KUSTOMIZE) $(ENVSUBST) ## Create a management cluster.
+create-management-cluster: $(KUSTOMIZE) $(ENVSUBST) $(KUBECTL) $(KIND) ## Create a management cluster.
 	# Create kind management cluster.
 	$(MAKE) kind-create
 
@@ -255,61 +259,64 @@ create-management-cluster: $(KUSTOMIZE) $(ENVSUBST) ## Create a management clust
 	# Create secret for AzureClusterIdentity
 	./hack/create-identity-secret.sh
 
+	# Create customized cloud provider configs
+	./hack/create-custom-cloud-provider-config.sh
+
 	# Deploy CAPI
-	curl --retry $(CURL_RETRIES) -sSL https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.1.4/cluster-api-components.yaml | $(ENVSUBST) | kubectl apply -f -
+	curl --retry $(CURL_RETRIES) -sSL https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.1.4/cluster-api-components.yaml | $(ENVSUBST) | $(KUBECTL) apply -f -
 
 	# Deploy CAPZ
-	kind load docker-image $(CONTROLLER_IMG)-$(ARCH):$(TAG) --name=capz
-	$(KUSTOMIZE) build config/default | $(ENVSUBST) | kubectl apply -f -
+	$(KIND) load docker-image $(CONTROLLER_IMG)-$(ARCH):$(TAG) --name=capz
+	$(KUSTOMIZE) build config/default | $(ENVSUBST) | $(KUBECTL) apply -f -
 
 	# Wait for CAPI deployments
-	kubectl wait --for=condition=Available --timeout=5m -n capi-system deployment -l cluster.x-k8s.io/provider=cluster-api
-	kubectl wait --for=condition=Available --timeout=5m -n capi-kubeadm-bootstrap-system deployment -l cluster.x-k8s.io/provider=bootstrap-kubeadm
-	kubectl wait --for=condition=Available --timeout=5m -n capi-kubeadm-control-plane-system deployment -l cluster.x-k8s.io/provider=control-plane-kubeadm
+	$(KUBECTL) wait --for=condition=Available --timeout=5m -n capi-system deployment -l cluster.x-k8s.io/provider=cluster-api
+	$(KUBECTL) wait --for=condition=Available --timeout=5m -n capi-kubeadm-bootstrap-system deployment -l cluster.x-k8s.io/provider=bootstrap-kubeadm
+	$(KUBECTL) wait --for=condition=Available --timeout=5m -n capi-kubeadm-control-plane-system deployment -l cluster.x-k8s.io/provider=control-plane-kubeadm
 
 	# apply CNI ClusterResourceSets
 	source ./scripts/ci-configmap.sh
 
-	kubectl apply -f templates/addons/calico-resource-set.yaml
+	$(KUBECTL) apply -f templates/addons/calico-resource-set.yaml
 
 	# Wait for CAPZ deployments
-	kubectl wait --for=condition=Available --timeout=5m -n capz-system deployment -l cluster.x-k8s.io/provider=infrastructure-azure
+	$(KUBECTL) wait --for=condition=Available --timeout=5m -n capz-system deployment -l cluster.x-k8s.io/provider=infrastructure-azure
 
 	# required sleep for when creating management and workload cluster simultaneously
 	sleep 10
-	@echo 'Set kubectl context to the kind management cluster by running "kubectl config set-context kind-capz"'
+	@echo 'Set kubectl context to the kind management cluster by running "$(KUBECTL) config set-context kind-capz"'
 
 .PHONY: create-workload-cluster
-create-workload-cluster: $(ENVSUBST) ## Create a workload cluster.
+create-workload-cluster: $(ENVSUBST) $(KUBECTL) ## Create a workload cluster.
 	# Create workload Cluster.
 	@if [ -f "$(TEMPLATES_DIR)/$(CLUSTER_TEMPLATE)" ]; then \
-		$(ENVSUBST) < "$(TEMPLATES_DIR)/$(CLUSTER_TEMPLATE)" | kubectl apply -f -; \
+		$(ENVSUBST) < "$(TEMPLATES_DIR)/$(CLUSTER_TEMPLATE)" | $(KUBECTL) apply -f -; \
 	elif [ -f "$(CLUSTER_TEMPLATE)" ]; then \
-		$(ENVSUBST) < "$(CLUSTER_TEMPLATE)" | kubectl apply -f -; \
+		$(ENVSUBST) < "$(CLUSTER_TEMPLATE)" | $(KUBECTL) apply -f -; \
 	else \
-		curl --retry "$(CURL_RETRIES)" "$(CLUSTER_TEMPLATE)" | "$(ENVSUBST)" | kubectl apply -f -; \
+		curl --retry "$(CURL_RETRIES)" "$(CLUSTER_TEMPLATE)" | "$(ENVSUBST)" | $(KUBECTL) apply -f -; \
 	fi
 
 	# Wait for the kubeconfig to become available.
-	timeout --foreground 300 bash -c "while ! kubectl get secrets | grep $(CLUSTER_NAME)-kubeconfig; do sleep 1; done"
+	timeout --foreground 300 bash -c "while ! $(KUBECTL) get secrets | grep $(CLUSTER_NAME)-kubeconfig; do sleep 1; done"
 	# Get kubeconfig and store it locally.
-	kubectl get secrets $(CLUSTER_NAME)-kubeconfig -o json | jq -r .data.value | base64 --decode > ./kubeconfig
-	timeout --foreground 600 bash -c "while ! kubectl --kubeconfig=./kubeconfig get nodes | grep control-plane; do sleep 1; done"
+	$(KUBECTL) get secrets $(CLUSTER_NAME)-kubeconfig -o json | jq -r .data.value | base64 --decode > ./kubeconfig
+	timeout --foreground 600 bash -c "while ! $(KUBECTL) --kubeconfig=./kubeconfig get nodes | grep control-plane; do sleep 1; done"
 
-	@echo 'run "kubectl --kubeconfig=./kubeconfig ..." to work with the new target cluster'
+	@echo 'run "$(KUBECTL) --kubeconfig=./kubeconfig ..." to work with the new target cluster'
 
 .PHONY: create-aks-cluster
-create-aks-cluster: $(KUSTOMIZE) $(ENVSUBST) ## Create a aks cluster.
+create-aks-cluster: $(KUSTOMIZE) $(ENVSUBST) $(KUBECTL) ## Create a aks cluster.
 	# Create managed Cluster.
-	$(ENVSUBST) < $(TEMPLATES_DIR)/$(MANAGED_CLUSTER_TEMPLATE) | kubectl apply -f -
+	$(ENVSUBST) < $(TEMPLATES_DIR)/$(MANAGED_CLUSTER_TEMPLATE) | $(KUBECTL) apply -f -
 
 	# Wait for the kubeconfig to become available.
-	timeout --foreground 300 bash -c "while ! kubectl get secrets | grep $(CLUSTER_NAME)-kubeconfig; do sleep 1; done"
+	timeout --foreground 300 bash -c "while ! $(KUBECTL) get secrets | grep $(CLUSTER_NAME)-kubeconfig; do sleep 1; done"
 	# Get kubeconfig and store it locally.
-	kubectl get secrets $(CLUSTER_NAME)-kubeconfig -o json | jq -r .data.value | base64 --decode > ./kubeconfig
-	timeout --foreground 600 bash -c "while ! kubectl --kubeconfig=./kubeconfig get nodes | grep control-plane; do sleep 1; done"
+	$(KUBECTL) get secrets $(CLUSTER_NAME)-kubeconfig -o json | jq -r .data.value | base64 --decode > ./kubeconfig
+	timeout --foreground 600 bash -c "while ! $(KUBECTL) --kubeconfig=./kubeconfig get nodes | grep control-plane; do sleep 1; done"
 
-	@echo 'run "kubectl --kubeconfig=./kubeconfig ..." to work with the new target cluster'
+	@echo 'run "$(KUBECTL) --kubeconfig=./kubeconfig ..." to work with the new target cluster'
 
 
 .PHONY: create-cluster
@@ -321,9 +328,9 @@ create-cluster: ## Create a workload development Kubernetes cluster on Azure in
 	create-workload-cluster
 
 .PHONY: delete-workload-cluster
-delete-workload-cluster: ## Deletes the example workload Kubernetes cluster.
+delete-workload-cluster: $(KUBECTL) ## Deletes the example workload Kubernetes cluster.
 	@echo 'Your Azure resources will now be deleted, this can take up to 20 minutes'
-	kubectl delete cluster $(CLUSTER_NAME)
+	$(KUBECTL) delete cluster $(CLUSTER_NAME)
 
 ## --------------------------------------
 ## Docker
@@ -682,12 +689,12 @@ tilt-up: install-tools kind-create ## Start tilt and build kind cluster if neede
 
 .PHONY: delete-cluster
 delete-cluster: delete-workload-cluster  ## Deletes the example kind cluster "capz".
-	kind delete cluster --name=capz
+	$(KIND) delete cluster --name=capz
 
 .PHONY: kind-reset
 kind-reset: ## Destroys the "capz" and "capz-e2e" kind clusters.
-	kind delete cluster --name=capz || true
-	kind delete cluster --name=capz-e2e || true
+	$(KIND) delete cluster --name=capz || true
+	$(KIND) delete cluster --name=capz-e2e || true
 
 ## --------------------------------------
 ## Tooling Binaries
@@ -709,6 +716,7 @@ ginkgo: $(GINKGO) ## Build a local copy of ginkgo.
 kubectl: $(KUBECTL) ## Build a local copy of kubectl.
 helm: $(HELM) ## Build a local copy of helm.
 yq: $(YQ) ## Build a local copy of yq.
+kind: $(KIND) ## Build a local copy of kind.
 
 $(CONVERSION_VERIFIER): go.mod
 	cd $(TOOLS_DIR); go build -tags=tools -o $@ sigs.k8s.io/cluster-api/hack/tools/conversion-verifier
@@ -759,6 +767,9 @@ $(HELM): ## Put helm into tools folder.
 	ln -sf $(HELM) $(TOOLS_BIN_DIR)/$(HELM_BIN)
 	rm -f $(TOOLS_BIN_DIR)/get_helm.sh
 
+$(KIND): ## Build kind into tools folder.
+	GOBIN=$(TOOLS_BIN_DIR) $(GO_INSTALL) sigs.k8s.io/kind $(KIND_BIN) $(KIND_VER)
+
 .PHONY: $(ENVSUBST_BIN)
 $(ENVSUBST_BIN): $(ENVSUBST)
 
@@ -776,3 +787,6 @@ $(YQ): ## Build yq from tools folder.
 
 .PHONY: $(YQ_BIN)
 $(YQ_BIN): $(YQ) ## Building yq from the tools folder.
+
+.PHONY: $(KIND_BIN)
+$(KIND_BIN): $(KIND)

Tiltfile

@@ -3,6 +3,7 @@
 envsubst_cmd = "./hack/tools/bin/envsubst"
 kubectl_cmd = "./hack/tools/bin/kubectl"
 helm_cmd = "./hack/tools/bin/helm"
+kind_cmd = "./hack/tools/bin/kind"
 tools_bin = "./hack/tools/bin"
 
 #Add tools to path
@@ -145,7 +146,7 @@ def observability():
         ],
     ))
 
-    internal_kubeconfig = str(local("kind get kubeconfig --name ${KIND_CLUSTER_NAME:-capz} --internal"))
+    internal_kubeconfig = str(local(kind_cmd + " get kubeconfig --name ${KIND_CLUSTER_NAME:-capz} --internal"))
     k8s_yaml(helm(
         "./hack/observability/cluster-api-visualizer/chart",
         name = "visualize-cluster",

api/v1alpha3/azuremachine_conversion.go

@@ -59,6 +59,10 @@ func (src *AzureMachine) ConvertTo(dstRaw conversion.Hub) error {
 		dst.Spec.Image.ComputeGallery = restored.Spec.Image.ComputeGallery
 	}
 
+	if restored.Spec.AdditionalCapabilities != nil {
+		dst.Spec.AdditionalCapabilities = restored.Spec.AdditionalCapabilities
+	}
+
 	dst.Spec.SubnetName = restored.Spec.SubnetName
 
 	dst.Status.LongRunningOperationStates = restored.Status.LongRunningOperationStates

api/v1alpha3/azuremachinetemplate_conversion.go

@@ -57,6 +57,10 @@ func (src *AzureMachineTemplate) ConvertTo(dstRaw conversion.Hub) error {
 		dst.Spec.Template.Spec.Image.ComputeGallery = restored.Spec.Template.Spec.Image.ComputeGallery
 	}
 
+	if restored.Spec.Template.Spec.AdditionalCapabilities != nil {
+		dst.Spec.Template.Spec.AdditionalCapabilities = restored.Spec.Template.Spec.AdditionalCapabilities
+	}
+
 	dst.Spec.Template.Spec.SubnetName = restored.Spec.Template.Spec.SubnetName
 	dst.Spec.Template.ObjectMeta = restored.Spec.Template.ObjectMeta
 

api/v1alpha4/azuremachine_conversion.go

@@ -36,14 +36,18 @@ func (src *AzureMachine) ConvertTo(dstRaw conversion.Hub) error {
 		return err
 	}
 
-
 	if restored.Spec.NetworkInterfaces != nil {
 		dst.Spec.NetworkInterfaces = restored.Spec.NetworkInterfaces
 	}
 
 	if restored.Spec.Image != nil && restored.Spec.Image.ComputeGallery != nil {
 		dst.Spec.Image.ComputeGallery = restored.Spec.Image.ComputeGallery
 	}
+
+	if restored.Spec.AdditionalCapabilities != nil {
+		dst.Spec.AdditionalCapabilities = restored.Spec.AdditionalCapabilities
+	}
+
 	return nil
 }
 

api/v1alpha4/azuremachinetemplate_conversion.go

@@ -45,6 +45,10 @@ func (src *AzureMachineTemplate) ConvertTo(dstRaw conversion.Hub) error {
 		dst.Spec.Template.Spec.Image.ComputeGallery = restored.Spec.Template.Spec.Image.ComputeGallery
 	}
 
+	if restored.Spec.Template.Spec.AdditionalCapabilities != nil {
+		dst.Spec.Template.Spec.AdditionalCapabilities = restored.Spec.Template.Spec.AdditionalCapabilities
+	}
+
 	dst.Spec.Template.ObjectMeta = restored.Spec.Template.ObjectMeta
 
 	return nil

api/v1beta1/azureclusteridentity_types.go

@@ -44,10 +44,10 @@ type AllowedNamespaces struct {
 // AzureClusterIdentitySpec defines the parameters that are used to create an AzureIdentity.
 type AzureClusterIdentitySpec struct {
 	// Type is the type of Azure Identity used.
-	// ServicePrincipal, ServicePrincipalCertificate, or ManualServicePrincipal.
+	// ServicePrincipal, ServicePrincipalCertificate, UserAssignedMSI or ManualServicePrincipal.
 	Type IdentityType `json:"type"`
 	// ResourceID is the Azure resource ID for the User Assigned MSI resource.
-	// Not currently supported.
+	// Only applicable when type is UserAssignedMSI.
 	// +optional
 	ResourceID string `json:"resourceID,omitempty"`
 	// ClientID is the service principal client ID.

api/v1beta1/azuremachine_default.go

@@ -19,6 +19,7 @@ package v1beta1
 import (
 	"encoding/base64"
 
+	"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2021-11-01/compute"
 	"golang.org/x/crypto/ssh"
 	"k8s.io/apimachinery/pkg/util/uuid"
 	utilSSH "sigs.k8s.io/cluster-api-provider-azure/util/ssh"
@@ -67,7 +68,12 @@ func (s *AzureMachineSpec) SetDataDisksDefaults() {
 			}
 		}
 		if disk.CachingType == "" {
-			s.DataDisks[i].CachingType = "ReadWrite"
+			if s.DataDisks[i].ManagedDisk != nil &&
+				s.DataDisks[i].ManagedDisk.StorageAccountType == string(compute.StorageAccountTypesUltraSSDLRS) {
+				s.DataDisks[i].CachingType = string(compute.CachingTypesNone)
+			} else {
+				s.DataDisks[i].CachingType = string(compute.CachingTypesReadWrite)
+			}
 		}
 	}
 }

api/v1beta1/azuremachine_default_test.go

@@ -215,6 +215,14 @@ func TestAzureMachineSpec_SetDataDisksDefaults(t *testing.T) {
 					DiskSizeGB: 30,
 					Lun:        to.Int32Ptr(2),
 				},
+				{
+					NameSuffix: "testdisk3",
+					DiskSizeGB: 30,
+					ManagedDisk: &ManagedDiskParameters{
+						StorageAccountType: "UltraSSD_LRS",
+					},
+					Lun: to.Int32Ptr(3),
+				},
 			},
 			output: []DataDisk{
 				{
@@ -229,6 +237,15 @@ func TestAzureMachineSpec_SetDataDisksDefaults(t *testing.T) {
 					Lun:         to.Int32Ptr(2),
 					CachingType: "ReadWrite",
 				},
+				{
+					NameSuffix: "testdisk3",
+					DiskSizeGB: 30,
+					Lun:        to.Int32Ptr(3),
+					ManagedDisk: &ManagedDiskParameters{
+						StorageAccountType: "UltraSSD_LRS",
+					},
+					CachingType: "None",
+				},
 			},
 		},
 	}

api/v1beta1/azuremachine_types.go

@@ -86,6 +86,10 @@ type AzureMachineSpec struct {
 	// +optional
 	AdditionalTags Tags `json:"additionalTags,omitempty"`
 
+	// AdditionalCapabilities specifies additional capabilities enabled or disabled on the virtual machine.
+	// +optional
+	AdditionalCapabilities *AdditionalCapabilities `json:"additionalCapabilities,omitempty"`
+
 	// AllocatePublicIP allows the ability to create dynamic public ips for machines where this value is true.
 	// +optional
 	AllocatePublicIP bool `json:"allocatePublicIP,omitempty"`
@@ -187,6 +191,15 @@ type AzureMachineStatus struct {
 	LongRunningOperationStates Futures `json:"longRunningOperationStates,omitempty"`
 }
 
+// AdditionalCapabilities enables or disables a capability on the virtual machine.
+type AdditionalCapabilities struct {
+	// UltraSSDEnabled enables or disables Azure UltraSSD capability for the virtual machine.
+	// Defaults to true if Ultra SSD data disks are specified,
+	// otherwise it doesn't set the capability on the VM.
+	// +optional
+	UltraSSDEnabled *bool `json:"ultraSSDEnabled,omitempty"`
+}
+
 // +kubebuilder:object:root=true
 // +kubebuilder:printcolumn:name="Ready",type="string",JSONPath=".status.conditions[?(@.type=='Ready')].status"
 // +kubebuilder:printcolumn:name="Reason",type="string",JSONPath=".status.conditions[?(@.type=='Ready')].reason"

api/v1beta1/azuremachine_validation.go

@@ -154,7 +154,7 @@ func ValidateDataDisks(dataDisks []DataDisk, fieldPath *field.Path) field.ErrorL
 		}
 
 		// validate cachingType
-		allErrs = append(allErrs, validateCachingType(disk.CachingType, fieldPath)...)
+		allErrs = append(allErrs, validateCachingType(disk.CachingType, fieldPath, disk.ManagedDisk)...)
 	}
 	return allErrs
 }
@@ -173,7 +173,7 @@ func ValidateOSDisk(osDisk OSDisk, fieldPath *field.Path) field.ErrorList {
 		allErrs = append(allErrs, field.Required(fieldPath.Child("OSType"), "the OS type cannot be empty"))
 	}
 
-	allErrs = append(allErrs, validateCachingType(osDisk.CachingType, fieldPath)...)
+	allErrs = append(allErrs, validateCachingType(osDisk.CachingType, fieldPath, osDisk.ManagedDisk)...)
 
 	if osDisk.ManagedDisk != nil {
 		if errs := validateManagedDisk(osDisk.ManagedDisk, fieldPath.Child("managedDisk"), true); len(errs) > 0 {
@@ -289,10 +289,16 @@ func validateStorageAccountType(storageAccountType string, fieldPath *field.Path
 	return allErrs
 }
 
-func validateCachingType(cachingType string, fieldPath *field.Path) field.ErrorList {
+func validateCachingType(cachingType string, fieldPath *field.Path, managedDisk *ManagedDiskParameters) field.ErrorList {
 	allErrs := field.ErrorList{}
 	cachingTypeChildPath := fieldPath.Child("CachingType")
 
+	if managedDisk != nil && managedDisk.StorageAccountType == string(compute.StorageAccountTypesUltraSSDLRS) {
+		if cachingType != string(compute.CachingTypesNone) {
+			allErrs = append(allErrs, field.Invalid(cachingTypeChildPath, cachingType, fmt.Sprintf("cachingType '%s' is not supported when storageAccountType is '%s'. Allowed values are: '%s'", cachingType, compute.StorageAccountTypesUltraSSDLRS, compute.CachingTypesNone)))
+		}
+	}
+
 	for _, possibleCachingType := range compute.PossibleCachingTypesValues() {
 		if string(possibleCachingType) == cachingType {
 			return allErrs