diff --git a/Makefile b/Makefile index 7f56e4e63d6..5b25f24dc11 100644 --- a/Makefile +++ b/Makefile @@ -387,6 +387,7 @@ create-cluster: ## Create a workload development Kubernetes cluster on Azure in EXP_MACHINE_POOL=true \ EXP_EDGEZONE=true \ EXP_ASO_API=true \ + EXP_APISERVER_ILB=true \ $(MAKE) create-management-cluster \ create-workload-cluster @@ -751,7 +752,7 @@ tilt-up: install-tools ## Start tilt and build kind cluster if needed. @if [ -z "${AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY}" ]; then \ export AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY=$(shell cat $(AZURE_IDENTITY_ID_FILEPATH)); \ fi; \ - CLUSTER_TOPOLOGY=true EXP_ASO_API=true EXP_CLUSTER_RESOURCE_SET=true EXP_MACHINE_POOL=true EXP_KUBEADM_BOOTSTRAP_FORMAT_IGNITION=true EXP_EDGEZONE=true tilt up + CLUSTER_TOPOLOGY=true EXP_ASO_API=true EXP_CLUSTER_RESOURCE_SET=true EXP_MACHINE_POOL=true EXP_KUBEADM_BOOTSTRAP_FORMAT_IGNITION=true EXP_EDGEZONE=true EXP_APISERVER_ILB=true tilt up .PHONY: delete-cluster delete-cluster: delete-workload-cluster ## Deletes the example kind cluster "capz". diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index dddc46d6a47..cbc1152fbbc 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -24,7 +24,7 @@ spec: - --leader-elect - "--diagnostics-address=${CAPZ_DIAGNOSTICS_ADDRESS:=:8443}" - "--insecure-diagnostics=${CAPZ_INSECURE_DIAGNOSTICS:=false}" - - "--feature-gates=MachinePool=${EXP_MACHINE_POOL:=true},AKSResourceHealth=${EXP_AKS_RESOURCE_HEALTH:=false},EdgeZone=${EXP_EDGEZONE:=false},ASOAPI=${EXP_ASO_API:=true}" + - "--feature-gates=MachinePool=${EXP_MACHINE_POOL:=true},AKSResourceHealth=${EXP_AKS_RESOURCE_HEALTH:=false},EdgeZone=${EXP_EDGEZONE:=false},ASOAPI=${EXP_ASO_API:=true},APIServerILB=${EXP_APISERVER_ILB:=false}" - "--v=0" image: controller:latest imagePullPolicy: Always diff --git a/docs/book/src/developers/development.md b/docs/book/src/developers/development.md index a99b7be31da..df2a43055e9 100644 --- a/docs/book/src/developers/development.md +++ b/docs/book/src/developers/development.md @@ -21,6 +21,8 @@ - [Tilt for dev in CAPZ](#tilt-for-dev-in-capz) - [Tilt for dev in both CAPZ and CAPI](#tilt-for-dev-in-both-capz-and-capi) - [Deploying a workload cluster](#deploying-a-workload-cluster) + - [Tilt for dev using internal load balancer (ILB) for intra-cluster node-apiserver traffic](#tilt-for-dev-using-internal-load-balancer-ilb-for-intra-cluster-node-apiserver-traffic) + - [Flavors for dev using internal load balancer (ILB) for intra-cluster node-apiserver traffic](#flavors-for-dev-using-internal-load-balancer-ilb-for-intra-cluster-node-apiserver-traffic) - [Viewing Telemetry](#viewing-telemetry) - [Debugging](#debugging) - [Manual Testing](#manual-testing) @@ -151,6 +153,17 @@ development will span both CAPZ and CAPI, then follow the [CAPI and CAPZ instruc #### Tilt for dev in CAPZ + + If you want to develop in CAPZ and get a local development cluster working quickly, this is the path for you. Create a file named `tilt-settings.yaml` in the root of the CAPZ repository with the following contents: @@ -247,6 +260,36 @@ make delete-workload-cluster > Check out the [self-managed](../self-managed/troubleshooting.md) and [managed](../managed/troubleshooting.md) troubleshooting guides for common errors you might run into. +#### Tilt for dev using internal load balancer (ILB) for intra-cluster node-apiserver traffic + +This flow is for developers who want to leverage the internal load balancer for intra-cluster node-apiserver traffic. +You can achieve this by setting the `EXP_APISERVER_ILB` environment variable to `true` in your shell (run `export EXP_APISERVER_ILB=true`) and then create the CAPZ management cluster. + +We also encourage you to use AKS cluster as your management cluster. + +Outline of the steps: +- `make clean` +- `make generate` +- Set `REGISTRY` in your env. Preferably an Azure Container Registry. +- Run `docker-build-all` to build all the images. +- Run `make acr-login` to login to your ACR. +- Run `docker-push-all` to push all the images to your ACR. +- Run `make aks-create` to create an AKS cluster. _Notice the changes that get applied to the `tilt-settings.yaml` file._ +- Run `export EXP_APISERVER_ILB=true` in your shell. +- Run `make tilt-up` to start Tilt. + +``` +TODO: +1. Come up with a shorter make target to do all the above steps for MS Tenants. +2. VNet peering should be exported out as a shell script for users to run. +``` + +##### Flavors for dev using internal load balancer (ILB) for intra-cluster node-apiserver traffic + +There are two flavors available for development in CAPZ for MSFT Tenant: +- [apiserver-ilb](../../../../templates/cluster-template-apiserver-ilb.yaml): VM based default flavor that brings up native K8s clusters with Linux nodes. +- [apiserver-ilb-windows](../../../../templates/cluster-template-windows-apiserver-ilb.yaml): VM based flavor that brings up native K8s clusters with Linux and Windows nodes. + #### Viewing Telemetry The CAPZ controller emits tracing and metrics data. When run in Tilt, the KinD management cluster is diff --git a/feature/feature.go b/feature/feature.go index b4af07d23fd..81eb0faa572 100644 --- a/feature/feature.go +++ b/feature/feature.go @@ -52,6 +52,12 @@ const ( // owner: @nojnhuh // alpha: v1.15 ASOAPI featuregate.Feature = "ASOAPI" + + // APIServerILB is a CAPZ feature gate to create an internal LB for the API Server of the workload cluster. + // Defaults to false. + // owner: @nawazkh + // alpha: v1.18 + APIServerILB featuregate.Feature = "APIServerILB" ) func init() { @@ -66,4 +72,5 @@ var defaultCAPZFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{ AKSResourceHealth: {Default: false, PreRelease: featuregate.Alpha}, EdgeZone: {Default: false, PreRelease: featuregate.Alpha}, ASOAPI: {Default: true, PreRelease: featuregate.Alpha}, + APIServerILB: {Default: false, PreRelease: featuregate.Alpha}, } diff --git a/hack/observability/opentelemetry/controller-manager-patch.yaml b/hack/observability/opentelemetry/controller-manager-patch.yaml index 2ba355b841d..a952b4cbd05 100644 --- a/hack/observability/opentelemetry/controller-manager-patch.yaml +++ b/hack/observability/opentelemetry/controller-manager-patch.yaml @@ -12,5 +12,5 @@ spec: - "--diagnostics-address=:8080" - "--insecure-diagnostics" - "--leader-elect" - - "--feature-gates=MachinePool=${EXP_MACHINE_POOL:=true},AKSResourceHealth=${EXP_AKS_RESOURCE_HEALTH:=false},EdgeZone=${EXP_EDGEZONE:=false},ASOAPI=${EXP_ASO_API:=true}" + - "--feature-gates=MachinePool=${EXP_MACHINE_POOL:=true},AKSResourceHealth=${EXP_AKS_RESOURCE_HEALTH:=false},EdgeZone=${EXP_EDGEZONE:=false},ASOAPI=${EXP_ASO_API:=true},APIServerILB=${EXP_APISERVER_ILB:=false}" - "--enable-tracing"