From 0fea5e56d5a52ad3d479ea1f4eb0860d3786b51a Mon Sep 17 00:00:00 2001 From: Bastian Ulke Date: Thu, 28 Apr 2022 21:56:45 +0200 Subject: [PATCH] Minor fixes & usability improvements (#305) * Allow communication with API server via udp/1194. References: https://github.com/mspnp/aks-baseline/issues/223 https://docs.microsoft.com/en-us/azure/firewall/protect-azure-kubernetes-service * Return IP address instead of res. ID (acc to doc) * Minimal user feedback: echo variables to console. * ifconfig.io to return IPv4 addr for access policy * Notes for macOS users, having BSD sed. * Improvement to comment. Co-authored-by: Chad Kittel * Comment out firewall rule, but add hints. * Enable FW rule in bicep; remove warning. Co-authored-by: Chad Kittel --- 02-ca-certificates.md | 2 ++ 03-aad.md | 7 +++++++ 04-networking.md | 2 ++ 05-bootstrap-prep.md | 10 +++++++++ 06-aks-cluster.md | 2 ++ 07-bootstrap-validation.md | 1 + 08-workload-prerequisites.md | 5 ++++- ...ecret-management-and-ingress-controller.md | 2 ++ 10-workload.md | 5 +++++ 11-validation.md | 1 + networking/hub-regionA.bicep | 21 +++++++++++++++++++ networking/spoke-BU0001A0008.bicep | 2 +- 12 files changed, 58 insertions(+), 2 deletions(-) diff --git a/02-ca-certificates.md b/02-ca-certificates.md index 17878969..762caec9 100644 --- a/02-ca-certificates.md +++ b/02-ca-certificates.md @@ -29,6 +29,7 @@ Now that you have the [prerequisites](./01-prerequisites.md) met, follow the ste ```bash export APP_GATEWAY_LISTENER_CERTIFICATE_AKS_BASELINE=$(cat appgw.pfx | base64 | tr -d '\n') + echo APP_GATEWAY_LISTENER_CERTIFICATE_AKS_BASELINE: $APP_GATEWAY_LISTENER_CERTIFICATE_AKS_BASELINE ``` 1. Generate the wildcard certificate for the AKS Ingress Controller @@ -45,6 +46,7 @@ Now that you have the [prerequisites](./01-prerequisites.md) met, follow the ste ```bash export AKS_INGRESS_CONTROLLER_CERTIFICATE_BASE64_AKS_BASELINE=$(cat traefik-ingress-internal-aks-ingress-tls.crt | base64 | tr -d '\n') + echo AKS_INGRESS_CONTROLLER_CERTIFICATE_BASE64_AKS_BASELINE: $AKS_INGRESS_CONTROLLER_CERTIFICATE_BASE64_AKS_BASELINE ``` ### Save your work in-progress diff --git a/03-aad.md b/03-aad.md index 38063b1a..87949e8a 100644 --- a/03-aad.md +++ b/03-aad.md @@ -22,6 +22,7 @@ Following the steps below you will result in an Azure AD configuration that will ```bash export TENANTID_AZURERBAC_AKS_BASELINE=$(az account show --query tenantId -o tsv) + echo TENANTID_AZURERBAC_AKS_BASELINE: $TENANTID_AZURERBAC_AKS_BASELINE ``` 1. Playing the role as the Contoso Bicycle Azure AD team, login into the tenant where Kubernetes Cluster API authorization will be associated with. @@ -29,6 +30,7 @@ Following the steps below you will result in an Azure AD configuration that will ```bash az login -t --allow-no-subscriptions export TENANTID_K8SRBAC_AKS_BASELINE=$(az account show --query tenantId -o tsv) + echo TENANTID_K8SRBAC_AKS_BASELINE: $TENANTID_K8SRBAC_AKS_BASELINE ``` 1. Create/identify the Azure AD security group that is going to map to the [Kubernetes Cluster Admin](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles) role `cluster-admin`. @@ -37,6 +39,7 @@ Following the steps below you will result in an Azure AD configuration that will ```bash export AADOBJECTID_GROUP_CLUSTERADMIN_AKS_BASELINE=$(az ad group create --display-name 'cluster-admins-bu0001a000800' --mail-nickname 'cluster-admins-bu0001a000800' --description "Principals in this group are cluster admins in the bu0001a000800 cluster." --query objectId -o tsv) + echo AADOBJECTID_GROUP_CLUSTERADMIN_AKS_BASELINE: $AADOBJECTID_GROUP_CLUSTERADMIN_AKS_BASELINE ``` This Azure AD group object ID will be used later while creating the cluster. This way, once the cluster gets deployed the new group will get the proper Cluster Role bindings in Kubernetes. @@ -49,6 +52,9 @@ Following the steps below you will result in an Azure AD configuration that will TENANTDOMAIN_K8SRBAC=$(az ad signed-in-user show --query 'userPrincipalName' -o tsv | cut -d '@' -f 2 | sed 's/\"//') AADOBJECTNAME_USER_CLUSTERADMIN=bu0001a000800-admin AADOBJECTID_USER_CLUSTERADMIN=$(az ad user create --display-name=${AADOBJECTNAME_USER_CLUSTERADMIN} --user-principal-name ${AADOBJECTNAME_USER_CLUSTERADMIN}@${TENANTDOMAIN_K8SRBAC} --force-change-password-next-login --password ChangeMebu0001a0008AdminChangeMe --query objectId -o tsv) + echo TENANTDOMAIN_K8SRBAC: $TENANTDOMAIN_K8SRBAC + echo AADOBJECTNAME_USER_CLUSTERADMIN: $AADOBJECTNAME_USER_CLUSTERADMIN + echo AADOBJECTID_USER_CLUSTERADMIN: $AADOBJECTID_USER_CLUSTERADMIN ``` 1. Add the cluster admin user(s) to the cluster admin security group. @@ -63,6 +69,7 @@ Following the steps below you will result in an Azure AD configuration that will ```bash export AADOBJECTID_GROUP_A0008_READER_AKS_BASELINE=$(az ad group create --display-name 'cluster-ns-a0008-readers-bu0001a000800' --mail-nickname 'cluster-ns-a0008-readers-bu0001a000800' --description "Principals in this group are readers of namespace a0008 in the bu0001a000800 cluster." --query objectId -o tsv) + echo AADOBJECTID_GROUP_A0008_READER_AKS_BASELINE: $AADOBJECTID_GROUP_A0008_READER_AKS_BASELINE ``` ## Kubernetes RBAC backing store diff --git a/04-networking.md b/04-networking.md index d04a6402..0036247d 100644 --- a/04-networking.md +++ b/04-networking.md @@ -76,6 +76,7 @@ The following two resource groups will be created and populated with networking ```bash RESOURCEID_VNET_HUB=$(az deployment group show -g rg-enterprise-networking-hubs -n hub-default --query properties.outputs.hubVnetId.value -o tsv) + echo RESOURCEID_VNET_HUB: $RESOURCEID_VNET_HUB # [This takes about four minutes to run.] az deployment group create -g rg-enterprise-networking-spokes -f networking/spoke-BU0001A0008.bicep -p location=eastus2 hubVnetResourceId="${RESOURCEID_VNET_HUB}" @@ -93,6 +94,7 @@ The following two resource groups will be created and populated with networking ```bash RESOURCEID_SUBNET_NODEPOOLS=$(az deployment group show -g rg-enterprise-networking-spokes -n spoke-BU0001A0008 --query properties.outputs.nodepoolSubnetResourceIds.value -o json) + echo RESOURCEID_VNET_HUB: $RESOURCEID_SUBNET_NODEPOOLS # [This takes about ten minutes to run.] az deployment group create -g rg-enterprise-networking-hubs -f networking/hub-regionA.bicep -p location=eastus2 nodepoolSubnetResourceIds="${RESOURCEID_SUBNET_NODEPOOLS}" diff --git a/05-bootstrap-prep.md b/05-bootstrap-prep.md index dc744c78..5c9f86e8 100644 --- a/05-bootstrap-prep.md +++ b/05-bootstrap-prep.md @@ -35,6 +35,7 @@ We'll be bootstrapping this cluster with the Flux GitOps agent as installed as a ```bash export RESOURCEID_VNET_CLUSTERSPOKE_AKS_BASELINE=$(az deployment group show -g rg-enterprise-networking-spokes -n spoke-BU0001A0008 --query properties.outputs.clusterVnetResourceId.value -o tsv) + echo RESOURCEID_VNET_CLUSTERSPOKE_AKS_BASELINE: $RESOURCEID_VNET_CLUSTERSPOKE_AKS_BASELINE ``` 1. Deploy the container registry template. @@ -51,6 +52,7 @@ We'll be bootstrapping this cluster with the Flux GitOps agent as installed as a ```bash # Get your ACR instance name export ACR_NAME_AKS_BASELINE=$(az deployment group show -g rg-bu0001a0008 -n acr-stamp --query properties.outputs.containerRegistryName.value -o tsv) + echo ACR_NAME_AKS_BASELINE: $ACR_NAME_AKS_BASELINE # Import core image(s) hosted in public container registries to be used during bootstrapping az acr import --source docker.io/weaveworks/kured:1.9.0 -n $ACR_NAME_AKS_BASELINE @@ -68,7 +70,15 @@ We'll be bootstrapping this cluster with the Flux GitOps agent as installed as a ```bash sed -i "s:docker.io:${ACR_NAME_AKS_BASELINE}.azurecr.io:" ./cluster-manifests/cluster-baseline-settings/kured.yaml + ``` + Note, that if you are on macOS, you might need to use the following command instead: + ```bash + sed -i '' 's:docker.io:'"${ACR_NAME_AKS_BASELINE}"'.azurecr.io:g' ./cluster-manifests/cluster-baseline-settings/kured.yaml + ``` + Now commit changes to repository. + + ```bash git commit -a -m "Update image source to use my ACR instance instead of a public container registry." git push ``` diff --git a/06-aks-cluster.md b/06-aks-cluster.md index 14498870..b0cb7ef8 100644 --- a/06-aks-cluster.md +++ b/06-aks-cluster.md @@ -10,6 +10,7 @@ Now that your [ACR instance is deployed and ready to support cluster bootstrappi ```bash GITOPS_REPOURL=$(git config --get remote.origin.url) + echo GITOPS_REPOURL: $GITOPS_REPOURL ``` 1. Deploy the cluster ARM template. @@ -36,6 +37,7 @@ Now that your [ACR instance is deployed and ready to support cluster bootstrappi # Federated Identity, see https://github.com/Azure/login#configure-deployment-credentials. az ad sp create-for-rbac --name "github-workflow-aks-cluster" --sdk-auth --skip-assignment > sp.json export APP_ID=$(grep -oP '(?<="clientId": ").*?[^\\](?=",)' sp.json) + echo APP_ID: $APP_ID # Wait for propagation until az ad sp show --id ${APP_ID} &> /dev/null ; do echo "Waiting for Azure AD propagation" && sleep 5; done diff --git a/07-bootstrap-validation.md b/07-bootstrap-validation.md index fd3930de..b1f63ebc 100644 --- a/07-bootstrap-validation.md +++ b/07-bootstrap-validation.md @@ -21,6 +21,7 @@ GitOps allows a team to author Kubernetes manifest files, persist them in their ```bash AKS_CLUSTER_NAME=$(az aks list -g rg-bu0001a0008 --query '[0].name' -o tsv) + echo AKS_CLUSTER_NAME: $AKS_CLUSTER_NAME ``` 1. Get AKS `kubectl` credentials. diff --git a/08-workload-prerequisites.md b/08-workload-prerequisites.md index 1ec07818..07a85e93 100644 --- a/08-workload-prerequisites.md +++ b/08-workload-prerequisites.md @@ -14,11 +14,14 @@ The AKS Cluster has been [bootstrapped](./07-bootstrap-validation.md), wrapping ```bash export KEYVAULT_NAME_AKS_BASELINE=$(az deployment group show --resource-group rg-bu0001a0008 -n cluster-stamp --query properties.outputs.keyVaultName.value -o tsv) + echo KEYVAULT_NAME_AKS_BASELINE: $KEYVAULT_NAME_AKS_BASELINE TEMP_ROLEASSIGNMENT_TO_UPLOAD_CERT=$(az role assignment create --role a4417e6f-fecd-4de8-b567-7b0420556985 --assignee-principal-type user --assignee-object-id $(az ad signed-in-user show --query 'objectId' -o tsv) --scope $(az keyvault show --name $KEYVAULT_NAME_AKS_BASELINE --query 'id' -o tsv) --query 'id' -o tsv) + echo TEMP_ROLEASSIGNMENT_TO_UPLOAD_CERT: $TEMP_ROLEASSIGNMENT_TO_UPLOAD_CERT # If you are behind a proxy or some other egress that does not provide a consistent IP, you'll need to manually adjust the # Azure Key Vault firewall to allow this traffic. - CURRENT_IP_ADDRESS=$(curl -s https://ifconfig.io) + CURRENT_IP_ADDRESS=$(curl -s -4 https://ifconfig.io) + echo CURRENT_IP_ADDRESS: $CURRENT_IP_ADDRESS az keyvault network-rule add -n $KEYVAULT_NAME_AKS_BASELINE --ip-address ${CURRENT_IP_ADDRESS} ``` diff --git a/09-secret-management-and-ingress-controller.md b/09-secret-management-and-ingress-controller.md index 16ada01b..f579e036 100644 --- a/09-secret-management-and-ingress-controller.md +++ b/09-secret-management-and-ingress-controller.md @@ -9,6 +9,8 @@ Previously you have configured [workload prerequisites](./08-workload-prerequisi ```bash TRAEFIK_USER_ASSIGNED_IDENTITY_RESOURCE_ID=$(az deployment group show --resource-group rg-bu0001a0008 -n cluster-stamp --query properties.outputs.aksIngressControllerPodManagedIdentityResourceId.value -o tsv) TRAEFIK_USER_ASSIGNED_IDENTITY_CLIENT_ID=$(az deployment group show --resource-group rg-bu0001a0008 -n cluster-stamp --query properties.outputs.aksIngressControllerPodManagedIdentityClientId.value -o tsv) + echo TRAEFIK_USER_ASSIGNED_IDENTITY_RESOURCE_ID: $TRAEFIK_USER_ASSIGNED_IDENTITY_RESOURCE_ID + echo TRAEFIK_USER_ASSIGNED_IDENTITY_CLIENT_ID: $TRAEFIK_USER_ASSIGNED_IDENTITY_CLIENT_ID ``` 1. Ensure your bootstrapping process has created the following namespace. diff --git a/10-workload.md b/10-workload.md index 089ec8ac..395bd25e 100644 --- a/10-workload.md +++ b/10-workload.md @@ -12,6 +12,11 @@ The cluster now has an [Traefik configured with a TLS certificate](./08-secret-m sed -i "s/contoso.com/${DOMAIN_NAME_AKS_BASELINE}/" workload/aspnetapp-ingress-patch.yaml ``` + Note, that if you are on macOS, you might need to use the following command instead: + ```bash + sed -i '' 's/contoso.com/'"${DOMAIN_NAME_AKS_BASELINE}"'/g' workload/aspnetapp-ingress-patch.yaml + ``` + 1. Deploy the ASP.NET Core Docker sample web app > The workload definition demonstrates the inclusion of a Pod Disruption Budget rule, ingress configuration, and pod (anti-)affinity rules for your reference. diff --git a/11-validation.md b/11-validation.md index d12046ab..45cf0ba7 100644 --- a/11-validation.md +++ b/11-validation.md @@ -15,6 +15,7 @@ This section will help you to validate the workload is exposed correctly and res ```bash # query the Azure Application Gateway Public Ip APPGW_PUBLIC_IP=$(az deployment group show --resource-group rg-enterprise-networking-spokes -n spoke-BU0001A0008 --query properties.outputs.appGwPublicIpAddress.value -o tsv) + echo APPGW_PUBLIC_IP: $APPGW_PUBLIC_IP ``` 1. Create `A` Record for DNS diff --git a/networking/hub-regionA.bicep b/networking/hub-regionA.bicep index c30ab3f3..cefeb5bc 100644 --- a/networking/hub-regionA.bicep +++ b/networking/hub-regionA.bicep @@ -482,6 +482,27 @@ resource fwPolicy 'Microsoft.Network/firewallPolicies@2021-05-01' = { '443' ] } + // NOTE: This rule is only required for for clusters not yet running in konnectivity mode and can be removed once it has been fully rolled out. + { + ruleType: 'NetworkRule' + name: 'pod-to-api-server_udp-1194' + description: 'This allows pods to communicate with the API server. Only needed if your cluster is not yet using konnectivity.' + ipProtocols: [ + 'UDP' + ] + sourceAddresses: [] + sourceIpGroups: [ + ipgNodepoolSubnet.id + ] + destinationAddresses: [ + 'AzureCloud.${location}' // Ideally you'd list your AKS server endpoints in appliction rules, instead of this wide-ranged rule + ] + destinationIpGroups: [] + destinationFqdns: [] + destinationPorts: [ + '1194' + ] + } ] } ] diff --git a/networking/spoke-BU0001A0008.bicep b/networking/spoke-BU0001A0008.bicep index 74b52b04..315d6a31 100644 --- a/networking/spoke-BU0001A0008.bicep +++ b/networking/spoke-BU0001A0008.bicep @@ -434,4 +434,4 @@ output clusterVnetResourceId string = vnetSpoke.id output nodepoolSubnetResourceIds array = [ vnetSpoke::snetClusterNodes.id ] -output appGwPublicIpAddress string = pipPrimaryClusterIp.id +output appGwPublicIpAddress string = pipPrimaryClusterIp.properties.ipAddress