From beff19125aa8cc7da603e90cab459ad0e87d3782 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 8 Apr 2022 16:52:23 +0000 Subject: [PATCH 01/54] initial cluster-stamp.bicep - add params and variables --- cluster-stamp.bicep | 101 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 cluster-stamp.bicep diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep new file mode 100644 index 00000000..0255e518 --- /dev/null +++ b/cluster-stamp.bicep @@ -0,0 +1,101 @@ +@description('The regional network spoke VNet Resource ID that the cluster will be joined to') +@minLength(79) +param targetVnetResourceId string + +@description('Azure AD Group in the identified tenant that will be granted the highly privileged cluster-admin role. If Azure RBAC is used, then this group will get a role assignment to Azure RBAC, else it will be assigned directly to the cluster\'s admin group.') +param clusterAdminAadGroupObjectId string + +@description('Azure AD Group in the identified tenant that will be granted the read only privileges in the a0008 namespace that exists in the cluster. This is only used when Azure RBAC is used for Kubernetes RBAC.') +param a0008NamespaceReaderAadGroupObjectId string + +@description('Your AKS control plane Cluster API authentication tenant') +param k8sControlPlaneAuthorizationTenantId string + +@description('The certificate data for app gateway TLS termination. It is base64') +param appGatewayListenerCertificate string + +@description('The Base64 encoded AKS Ingress Controller public certificate (as .crt or .cer) to be stored in Azure Key Vault as secret and referenced by Azure Application Gateway as a trusted root certificate.') +param aksIngressControllerCertificate string + +@description('IP ranges authorized to contact the Kubernetes API server. Passing an empty array will result in no IP restrictions. If any are provided, remember to also provide the public IP of the egress Azure Firewall otherwise your nodes will not be able to talk to the API server (e.g. Flux).') +param clusterAuthorizedIPRanges array = [] + +@description('AKS Service, Node Pool, and supporting services (KeyVault, App Gateway, etc) region. This needs to be the same region as the vnet provided in these parameters.') +@allowed([ + 'australiaeast' + 'canadacentral' + 'centralus' + 'eastus' + 'eastus2' + 'westus2' + 'francecentral' + 'germanywestcentral' + 'northeurope' + 'southafricanorth' + 'southcentralus' + 'uksouth' + 'westeurope' + 'japaneast' + 'southeastasia' +]) +param location string = 'eastus2' +param kubernetesVersion string = '1.22.4' + +@description('Domain name to use for App Gateway and AKS ingress.') +param domainName string = 'contoso.com' + +@description('Your cluster will be bootstrapped from this git repo.') +@minLength(9) +param gitOpsBootstrappingRepoHttpsUrl string = 'https://github.com/mspnp/aks-baseline' + +@description('You cluster will be bootstrapped from this branch in the identifed git repo.') +@minLength(1) +param gitOpsBootstrappingRepoBranch string = 'main' + +var networkContributorRole = '${subscription().id}/providers/Microsoft.Authorization/roleDefinitions/4d97b98b-1d4f-4787-a291-c67834d212e7' +var monitoringMetricsPublisherRole = '${subscription().id}/providers/Microsoft.Authorization/roleDefinitions/3913510d-42f4-4e42-8a64-420c390055eb' +var acrPullRole = '${subscription().id}/providers/Microsoft.Authorization/roleDefinitions/7f951dda-4ed3-4680-a7ca-43fe172d538d' +var managedIdentityOperatorRole = '${subscription().id}/providers/Microsoft.Authorization/roleDefinitions/f1a07417-d97a-45cb-824c-7a7467783830' +var virtualMachineContributorRole = '${subscription().id}/providers/Microsoft.Authorization/roleDefinitions/9980e02c-c2be-4d73-94e8-173b1dc7cf3c' +var keyVaultReader = '${subscription().id}/providers/Microsoft.Authorization/roleDefinitions/21090545-7ca7-4776-b22c-e363652d74d2' +var keyVaultSecretsUserRole = '${subscription().id}/providers/Microsoft.Authorization/roleDefinitions/4633458b-17de-408a-b874-0445c86b69e6' +var clusterAdminRoleId = 'b1ff04bb-8a4e-4dc4-8eb5-8693973ce19b' +var clusterReaderRoleId = '7f6c6a51-bcf8-42ba-9220-52d62157d7db' +var serviceClusterUserRoleId = '4abbcc35-e782-43d8-92c5-2d3f1bd2253f' +var subRgUniqueString = uniqueString('aks', subscription().subscriptionId, resourceGroup().id) + +var clusterName = 'aks-${subRgUniqueString}' +var nodeResourceGroupName = 'rg-${clusterName}-nodepools' +var logAnalyticsWorkspaceName = 'la-${clusterName}' +var containerInsightsSolutionName_var = 'ContainerInsights(${logAnalyticsWorkspaceName})' +var defaultAcrName = 'acraks${subRgUniqueString}' + +var vNetResourceGroup = split(targetVnetResourceId, '/')[4] +var vnetName = split(targetVnetResourceId, '/')[8] +var vnetNodePoolSubnetResourceId = '${targetVnetResourceId}/subnets/snet-clusternodes' +var vnetPrivateLinkEndpointsSubnetResourceId = '${targetVnetResourceId}/subnets/snet-privatelinkendpoints' +var vnetIngressServicesSubnetResourceId = '${targetVnetResourceId}/subnets/snet-cluster-ingressservices' + +var agwName = 'apw-${clusterName}' + +var akvPrivateDnsZonesName = 'privatelink.vaultcore.azure.net' + +var aksIngressDomainName = 'aks-ingress.${domainName}' +var aksBackendDomainName = 'bu0001a0008-00.${aksIngressDomainName}' +var policyResourceIdAKSLinuxRestrictive = '/providers/Microsoft.Authorization/policySetDefinitions/42b8ef37-b724-4e24-bbc8-7a7708edfe00' +var policyResourceIdEnforceHttpsIngress = '/providers/Microsoft.Authorization/policyDefinitions/1a5b4dca-0b6f-4cf5-907c-56316bc1bf3d' +var policyResourceIdEnforceInternalLoadBalancers = '/providers/Microsoft.Authorization/policyDefinitions/3fc4dc25-5baf-40d8-9b05-7fe74c1bc64e' +var policyResourceIdRoRootFilesystem = '/providers/Microsoft.Authorization/policyDefinitions/df49d893-a74c-421d-bc95-c663042e5b80' +var policyResourceIdEnforceResourceLimits = '/providers/Microsoft.Authorization/policyDefinitions/e345eecc-fa47-480f-9e88-67dcc122b164' +var policyResourceIdEnforceImageSource = '/providers/Microsoft.Authorization/policyDefinitions/febd0533-8e55-448f-b837-bd0e06f16469' +var policyResourceIdEnforceDefenderInCluster = '/providers/Microsoft.Authorization/policyDefinitions/a1840de2-8088-4ea8-b153-b4c723e9cb01' +var policyAssignmentNameAKSLinuxRestrictive = guid(policyResourceIdAKSLinuxRestrictive, resourceGroup().name, clusterName) +var policyAssignmentNameEnforceHttpsIngress = guid(policyResourceIdEnforceHttpsIngress, resourceGroup().name, clusterName) +var policyAssignmentNameEnforceInternalLoadBalancers = guid(policyResourceIdEnforceInternalLoadBalancers, resourceGroup().name, clusterName) +var policyAssignmentNameRoRootFilesystem = guid(policyResourceIdRoRootFilesystem, resourceGroup().name, clusterName) +var policyAssignmentNameEnforceResourceLimits = guid(policyResourceIdEnforceResourceLimits, resourceGroup().name, clusterName) +var policyAssignmentNameEnforceImageSource = guid(policyResourceIdEnforceImageSource, resourceGroup().name, clusterName) +var policyAssignmentNameEnforceDefenderInCluster = guid(policyResourceIdEnforceDefenderInCluster, resourceGroup().name, clusterName) +var isUsingAzureRBACasKubernetesRBAC = (subscription().tenantId == k8sControlPlaneAuthorizationTenantId) + +output aksClusterName string = clusterName From ecbe2c2b3e8eef8da12329e1e50d3e765420ef69 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 8 Apr 2022 16:53:39 +0000 Subject: [PATCH 02/54] add mis --- cluster-stamp.bicep | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 0255e518..91c5cb69 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -98,4 +98,21 @@ var policyAssignmentNameEnforceImageSource = guid(policyResourceIdEnforceImageSo var policyAssignmentNameEnforceDefenderInCluster = guid(policyResourceIdEnforceDefenderInCluster, resourceGroup().name, clusterName) var isUsingAzureRBACasKubernetesRBAC = (subscription().tenantId == k8sControlPlaneAuthorizationTenantId) +resource clusterControlPlaneIdentityName 'Microsoft.ManagedIdentity/userAssignedIdentities@2018-11-30' = { + name: 'mi-${clusterName}-controlplane' + location: location +} + +resource miAppGatewayFrontend 'Microsoft.ManagedIdentity/userAssignedIdentities@2018-11-30' = { + name: 'mi-appgateway-frontend' + location: location +} + +resource podmiIngressController 'Microsoft.ManagedIdentity/userAssignedIdentities@2018-11-30' = { + name: 'podmi-ingress-controller' + location: location +} + output aksClusterName string = clusterName +output aksIngressControllerPodManagedIdentityResourceId string = podmiIngressController.id +output aksIngressControllerPodManagedIdentityClientId string = reference(podmiIngressController.id, '2018-11-30').clientId From e83242b2e0aeeeac5c6c0a4c754ea3706f115949 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 8 Apr 2022 16:54:21 +0000 Subject: [PATCH 03/54] add akv --- cluster-stamp.bicep | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 91c5cb69..1aa1ceb8 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -113,6 +113,36 @@ resource podmiIngressController 'Microsoft.ManagedIdentity/userAssignedIdentitie location: location } +resource kv 'Microsoft.KeyVault/vaults@2021-11-01-preview' = { + name: 'kv-${clusterName}' + location: location + properties: { + accessPolicies: [] + sku: { + family: 'A' + name: 'standard' + } + tenantId: subscription().tenantId + networkAcls: { + bypass: 'AzureServices' + defaultAction: 'Deny' + ipRules: [] + virtualNetworkRules: [] + } + enableRbacAuthorization: true + enabledForDeployment: false + enabledForDiskEncryption: false + enabledForTemplateDeployment: false + enableSoftDelete: true + } + dependsOn: [ + miAppGatewayFrontend + podmiIngressController + ] + +} + output aksClusterName string = clusterName output aksIngressControllerPodManagedIdentityResourceId string = podmiIngressController.id output aksIngressControllerPodManagedIdentityClientId string = reference(podmiIngressController.id, '2018-11-30').clientId +output keyVaultName string = kv.name From ea0f77b1b3b8af89c9cb81b0a7dbe793f6c63bd4 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 8 Apr 2022 16:56:47 +0000 Subject: [PATCH 04/54] add akv secrets --- cluster-stamp.bicep | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 1aa1ceb8..6aab4b9c 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -140,6 +140,19 @@ resource kv 'Microsoft.KeyVault/vaults@2021-11-01-preview' = { podmiIngressController ] + resource kvsGatewayPublicCert 'secrets' = { + name: 'gateway-public-cert' + properties: { + value: appGatewayListenerCertificate + } + } + + resource kvsAppGwIngressInternalAksIngressTls 'secrets' = { + name: 'appgw-ingress-internal-aks-ingress-tls' + properties: { + value: aksIngressControllerCertificate + } + } } output aksClusterName string = clusterName From bc0cc72e36278e4d0c1af0327b12048e0be8d4e4 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 8 Apr 2022 16:57:38 +0000 Subject: [PATCH 05/54] add akv diagnostic settings --- cluster-stamp.bicep | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 6aab4b9c..88af5c36 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -155,6 +155,25 @@ resource kv 'Microsoft.KeyVault/vaults@2021-11-01-preview' = { } } +resource kv_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { + scope: kv + name: 'default' + properties: { + workspaceId: resourceId('Microsoft.OperationalInsights/workspaces', logAnalyticsWorkspaceName) + logs: [ + { + category: 'AuditEvent' + enabled: true + } + ] + metrics: [ + { + category: 'AllMetrics' + enabled: true + } + ] + } +} output aksClusterName string = clusterName output aksIngressControllerPodManagedIdentityResourceId string = podmiIngressController.id output aksIngressControllerPodManagedIdentityClientId string = reference(podmiIngressController.id, '2018-11-30').clientId From 6c50a2f54426e4644f158e82012935cab0ab4b10 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 8 Apr 2022 16:58:36 +0000 Subject: [PATCH 06/54] add role assignments --- cluster-stamp.bicep | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 88af5c36..fa008870 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -174,6 +174,47 @@ resource kv_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-0 ] } } + +resource miAppGatewayFrontendSecretsUserRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { + scope: kv + name: '${guid(resourceGroup().id, 'mi-appgateway-frontend', keyVaultSecretsUserRole)}' + properties: { + roleDefinitionId: keyVaultSecretsUserRole + principalId: miAppGatewayFrontend.properties.principalId + principalType: 'ServicePrincipal' + } +} + +resource miAppGatewayFrontendKeyVaultReader_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { + scope: kv + name: '${guid(resourceGroup().id, 'mi-appgateway-frontend', keyVaultReader)}' + properties: { + roleDefinitionId: keyVaultReader + principalId: miAppGatewayFrontend.properties.principalId + principalType: 'ServicePrincipal' + } +} + +resource podmiIngressControllerSecretsUserRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { + scope: kv + name: '${guid(resourceGroup().id, 'podmi-ingress-controller', keyVaultSecretsUserRole)}' + properties: { + roleDefinitionId: keyVaultSecretsUserRole + principalId: podmiIngressController.properties.principalId + principalType: 'ServicePrincipal' + } +} + +resource podmiIngressControllerKeyVaultReader_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { + scope: kv + name: '${guid(resourceGroup().id, 'podmi-ingress-controller', keyVaultReader)}' + properties: { + roleDefinitionId: keyVaultReader + principalId: podmiIngressController.properties.principalId + principalType: 'ServicePrincipal' + } +} + output aksClusterName string = clusterName output aksIngressControllerPodManagedIdentityResourceId string = podmiIngressController.id output aksIngressControllerPodManagedIdentityClientId string = reference(podmiIngressController.id, '2018-11-30').clientId From bb69f33c1c0c45edaebc8dfe80c660b219ebb424 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 8 Apr 2022 18:08:30 +0000 Subject: [PATCH 07/54] add private dns and private endpoint for akv --- cluster-stamp.bicep | 53 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 51 insertions(+), 2 deletions(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index fa008870..ee6d1ac6 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -78,8 +78,6 @@ var vnetIngressServicesSubnetResourceId = '${targetVnetResourceId}/subnets/snet- var agwName = 'apw-${clusterName}' -var akvPrivateDnsZonesName = 'privatelink.vaultcore.azure.net' - var aksIngressDomainName = 'aks-ingress.${domainName}' var aksBackendDomainName = 'bu0001a0008-00.${aksIngressDomainName}' var policyResourceIdAKSLinuxRestrictive = '/providers/Microsoft.Authorization/policySetDefinitions/42b8ef37-b724-4e24-bbc8-7a7708edfe00' @@ -215,6 +213,57 @@ resource podmiIngressControllerKeyVaultReader_roleAssignment 'Microsoft.Authoriz } } +resource pdzKv 'Microsoft.Network/privateDnsZones@2020-06-01' = { + name: 'privatelink.vaultcore.azure.net' + location: 'global' + + resource vnetlnk 'virtualNetworkLinks' = { + name: 'to_${vnetName}' + location: 'global' + properties: { + virtualNetwork: { + id: targetVnetResourceId + } + registrationEnabled: false + } + } +} + +resource peKv 'Microsoft.Network/privateEndpoints@2021-05-01' = { + name: 'pe-${kv.name}' + location: location + properties: { + subnet: { + id: vnetPrivateLinkEndpointsSubnetResourceId + } + privateLinkServiceConnections: [ + { + name: 'to_${vnetName}' + properties: { + privateLinkServiceId: kv.id + groupIds: [ + 'vault' + ] + } + } + ] + } + + resource pdnszg 'privateDnsZoneGroups' = { + name: 'default' + properties: { + privateDnsZoneConfigs: [ + { + name: 'privatelink-akv-net' + properties: { + privateDnsZoneId: pdzKv.id + } + } + ] + } + } +} + output aksClusterName string = clusterName output aksIngressControllerPodManagedIdentityResourceId string = podmiIngressController.id output aksIngressControllerPodManagedIdentityClientId string = reference(podmiIngressController.id, '2018-11-30').clientId From 32f0cd1b96b1ad036b8cdd64c3d0734701db9425 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 8 Apr 2022 18:26:16 +0000 Subject: [PATCH 08/54] add private dns and private endpoint for the in-cluster ingress --- cluster-stamp.bicep | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index ee6d1ac6..f30c4ab1 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -264,6 +264,34 @@ resource peKv 'Microsoft.Network/privateEndpoints@2021-05-01' = { } } +resource pdzAksIngress 'Microsoft.Network/privateDnsZones@2020-06-01' = { + name: aksIngressDomainName + location: 'global' + + resource aksIngressDomainName_bu0001a0008_00 'A' = { + name: 'bu0001a0008-00' + properties: { + ttl: 3600 + aRecords: [ + { + ipv4Address: '10.240.4.4' + } + ] + } + } + + resource vnetlnk 'virtualNetworkLinks' = { + name: 'to_${vnetName}' + location: 'global' + properties: { + virtualNetwork: { + id: targetVnetResourceId + } + registrationEnabled: false + } + } +} + output aksClusterName string = clusterName output aksIngressControllerPodManagedIdentityResourceId string = podmiIngressController.id output aksIngressControllerPodManagedIdentityClientId string = reference(podmiIngressController.id, '2018-11-30').clientId From f8173729ea32c545b1c4c853213b2450536bb299 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 8 Apr 2022 18:54:36 +0000 Subject: [PATCH 09/54] add activity log alerts --- cluster-stamp.bicep | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index f30c4ab1..090c75de 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -96,6 +96,33 @@ var policyAssignmentNameEnforceImageSource = guid(policyResourceIdEnforceImageSo var policyAssignmentNameEnforceDefenderInCluster = guid(policyResourceIdEnforceDefenderInCluster, resourceGroup().name, clusterName) var isUsingAzureRBACasKubernetesRBAC = (subscription().tenantId == k8sControlPlaneAuthorizationTenantId) +resource alaRgRecommendations 'Microsoft.Insights/activityLogAlerts@2020-10-01' = { + name: 'AllAzureAdvisorAlert' + location: 'Global' + properties: { + scopes: [ + resourceGroup().id + ] + condition: { + allOf: [ + { + field: 'category' + equals: 'Recommendation' + } + { + field: 'operationName' + equals: 'Microsoft.Advisor/recommendations/available/action' + } + ] + } + actions: { + actionGroups: [] + } + enabled: true + description: 'All azure advisor alerts' + } +} + resource clusterControlPlaneIdentityName 'Microsoft.ManagedIdentity/userAssignedIdentities@2018-11-30' = { name: 'mi-${clusterName}-controlplane' location: location From 39f2b2a9407c1b3378daec21d0052e51259999b5 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 8 Apr 2022 19:02:10 +0000 Subject: [PATCH 10/54] add ws prometheus saved searches --- cluster-stamp.bicep | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 090c75de..ef601396 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -123,6 +123,28 @@ resource alaRgRecommendations 'Microsoft.Insights/activityLogAlerts@2020-10-01' } } +resource ssPrometheusAll 'Microsoft.OperationalInsights/workspaces/savedSearches@2020-08-01' = { + name: '${logAnalyticsWorkspaceName}/AllPrometheus' + properties: { + etag: '*' + category: 'Prometheus' + displayName: 'All collected Prometheus information' + query: 'InsightsMetrics | where Namespace == "prometheus"' + version: 1 + } +} + +resource ssPrometheusKuredRequestedReeboot 'Microsoft.OperationalInsights/workspaces/savedSearches@2020-08-01' = { + name: '${logAnalyticsWorkspaceName}/NodeRebootRequested' + properties: { + etag: '*' + category: 'Prometheus' + displayName: 'Nodes reboot required by kured' + query: 'InsightsMetrics | where Namespace == "prometheus" and Name == "kured_reboot_required" | where Val > 0' + version: 1 + } +} + resource clusterControlPlaneIdentityName 'Microsoft.ManagedIdentity/userAssignedIdentities@2018-11-30' = { name: 'mi-${clusterName}-controlplane' location: location From d1966874eb91e8df8d98ec96f59fb4242979517a Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 8 Apr 2022 19:14:21 +0000 Subject: [PATCH 11/54] add solutions --- cluster-stamp.bicep | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index ef601396..bee688d9 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -67,7 +67,6 @@ var subRgUniqueString = uniqueString('aks', subscription().subscriptionId, resou var clusterName = 'aks-${subRgUniqueString}' var nodeResourceGroupName = 'rg-${clusterName}-nodepools' var logAnalyticsWorkspaceName = 'la-${clusterName}' -var containerInsightsSolutionName_var = 'ContainerInsights(${logAnalyticsWorkspaceName})' var defaultAcrName = 'acraks${subRgUniqueString}' var vNetResourceGroup = split(targetVnetResourceId, '/')[4] @@ -145,6 +144,34 @@ resource ssPrometheusKuredRequestedReeboot 'Microsoft.OperationalInsights/worksp } } +resource sci 'Microsoft.OperationsManagement/solutions@2015-11-01-preview' = { + name: 'ContainerInsights(${logAnalyticsWorkspaceName})' + location: location + properties: { + workspaceResourceId: resourceId('Microsoft.OperationalInsights/workspaces', logAnalyticsWorkspaceName) + } + plan: { + name: 'ContainerInsights(${logAnalyticsWorkspaceName})' + product: 'OMSGallery/ContainerInsights' + promotionCode: '' + publisher: 'Microsoft' + } +} + +resource skva 'Microsoft.OperationsManagement/solutions@2015-11-01-preview' = { + name: 'KeyVaultAnalytics(${logAnalyticsWorkspaceName})' + location: location + properties: { + workspaceResourceId: resourceId('Microsoft.OperationalInsights/workspaces', logAnalyticsWorkspaceName) + } + plan: { + name: 'KeyVaultAnalytics(${logAnalyticsWorkspaceName})' + product: 'OMSGallery/KeyVaultAnalytics' + promotionCode: '' + publisher: 'Microsoft' + } +} + resource clusterControlPlaneIdentityName 'Microsoft.ManagedIdentity/userAssignedIdentities@2018-11-30' = { name: 'mi-${clusterName}-controlplane' location: location From 668029fb678c9f454c8d938420c7f24f0e06a95d Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 8 Apr 2022 19:20:35 +0000 Subject: [PATCH 12/54] add failed pods scheduled query --- cluster-stamp.bicep | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index bee688d9..f09adab0 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -172,6 +172,37 @@ resource skva 'Microsoft.OperationsManagement/solutions@2015-11-01-preview' = { } } +resource sqrPodFailed 'Microsoft.Insights/scheduledQueryRules@2018-04-16' = { + name: 'PodFailedScheduledQuery' + location: location + properties: { + description: 'Alert on pod Failed phase.' + enabled: 'true' + source: { + query: '//https://docs.microsoft.com/azure/azure-monitor/insights/container-insights-alerts \r\n let endDateTime = now(); let startDateTime = ago(1h); let trendBinSize = 1m; let clusterName = "${clusterName}"; KubePodInventory | where TimeGenerated < endDateTime | where TimeGenerated >= startDateTime | where ClusterName == clusterName | distinct ClusterName, TimeGenerated | summarize ClusterSnapshotCount = count() by bin(TimeGenerated, trendBinSize), ClusterName | join hint.strategy=broadcast ( KubePodInventory | where TimeGenerated < endDateTime | where TimeGenerated >= startDateTime | distinct ClusterName, Computer, PodUid, TimeGenerated, PodStatus | summarize TotalCount = count(), PendingCount = sumif(1, PodStatus =~ "Pending"), RunningCount = sumif(1, PodStatus =~ "Running"), SucceededCount = sumif(1, PodStatus =~ "Succeeded"), FailedCount = sumif(1, PodStatus =~ "Failed") by ClusterName, bin(TimeGenerated, trendBinSize) ) on ClusterName, TimeGenerated | extend UnknownCount = TotalCount - PendingCount - RunningCount - SucceededCount - FailedCount | project TimeGenerated, TotalCount = todouble(TotalCount) / ClusterSnapshotCount, PendingCount = todouble(PendingCount) / ClusterSnapshotCount, RunningCount = todouble(RunningCount) / ClusterSnapshotCount, SucceededCount = todouble(SucceededCount) / ClusterSnapshotCount, FailedCount = todouble(FailedCount) / ClusterSnapshotCount, UnknownCount = todouble(UnknownCount) / ClusterSnapshotCount| summarize AggregatedValue = avg(FailedCount) by bin(TimeGenerated, trendBinSize)' + dataSourceId: resourceId('Microsoft.OperationalInsights/workspaces', logAnalyticsWorkspaceName) + queryType: 'ResultCount' + } + schedule: { + frequencyInMinutes: 5 + timeWindowInMinutes: 10 + } + action: { + 'odata.type': 'Microsoft.WindowsAzure.Management.Monitoring.Alerts.Models.Microsoft.AppInsights.Nexus.DataContracts.Resources.ScheduledQueryRules.AlertingAction' + severity: 3 + trigger: { + thresholdOperator: 'GreaterThan' + threshold: 3 + metricTrigger: { + thresholdOperator: 'GreaterThan' + threshold: 2 + metricTriggerType: 'Consecutive' + } + } + } + } +} + resource clusterControlPlaneIdentityName 'Microsoft.ManagedIdentity/userAssignedIdentities@2018-11-30' = { name: 'mi-${clusterName}-controlplane' location: location From 2fdba24112ef4c0733a6c42a44c11816e26c1b42 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 8 Apr 2022 19:42:14 +0000 Subject: [PATCH 13/54] add policy assignments --- cluster-stamp.bicep | 145 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 145 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index f09adab0..b954cac1 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -203,6 +203,151 @@ resource sqrPodFailed 'Microsoft.Insights/scheduledQueryRules@2018-04-16' = { } } +resource paAKSLinuxRestrictive 'Microsoft.Authorization/policyAssignments@2021-06-01' = { + name: policyAssignmentNameAKSLinuxRestrictive + properties: { + displayName: '[${clusterName}] ${reference(policyResourceIdAKSLinuxRestrictive, '2020-09-01').displayName}' + scope: subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name) + policyDefinitionId: policyResourceIdAKSLinuxRestrictive + parameters: { + excludedNamespaces: { + value: [ + 'kube-system' + 'gatekeeper-system' + 'azure-arc' + 'cluster-baseline-settings' + ] + } + effect: { + value: 'audit' + } + } + } +} + +resource paEnforceHttpsIngress 'Microsoft.Authorization/policyAssignments@2021-06-01' = { + name: policyAssignmentNameEnforceHttpsIngress + properties: { + displayName: '[${clusterName}] ${reference(policyResourceIdEnforceHttpsIngress, '2020-09-01').displayName}' + scope: subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name) + policyDefinitionId: policyResourceIdEnforceHttpsIngress + parameters: { + excludedNamespaces: { + value: [] + } + effect: { + value: 'deny' + } + } + } +} + +resource paEnforceInternalLoadBalancers 'Microsoft.Authorization/policyAssignments@2021-06-01' = { + name: policyAssignmentNameEnforceInternalLoadBalancers + properties: { + displayName: '[${clusterName}] ${reference(policyResourceIdEnforceInternalLoadBalancers, '2020-09-01').displayName}' + scope: subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name) + policyDefinitionId: policyResourceIdEnforceInternalLoadBalancers + parameters: { + excludedNamespaces: { + value: [] + } + effect: { + value: 'deny' + } + } + } +} + +resource paRoRootFilesystem 'Microsoft.Authorization/policyAssignments@2021-06-01' = { + name: policyAssignmentNameRoRootFilesystem + properties: { + displayName: '[${clusterName}] ${reference(policyResourceIdRoRootFilesystem, '2020-09-01').displayName}' + scope: subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name) + policyDefinitionId: policyResourceIdRoRootFilesystem + parameters: { + excludedNamespaces: { + value: [ + 'kube-system' + 'gatekeeper-system' + 'azure-arc' + ] + } + effect: { + value: 'audit' + } + } + } +} + +resource paEnforceResourceLimits 'Microsoft.Authorization/policyAssignments@2021-06-01' = { + name: policyAssignmentNameEnforceResourceLimits + properties: { + displayName: '[${clusterName}] ${reference(policyResourceIdEnforceResourceLimits, '2020-09-01').displayName}' + scope: subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name) + policyDefinitionId: policyResourceIdEnforceResourceLimits + parameters: { + cpuLimit: { + value: '1000m' + } + memoryLimit: { + value: '512Mi' + } + excludedNamespaces: { + value: [ + 'kube-system' + 'gatekeeper-system' + 'azure-arc' + 'cluster-baseline-settings' + 'flux-system' + ] + } + effect: { + value: 'deny' + } + } + } +} + +resource paEnforceImageSource 'Microsoft.Authorization/policyAssignments@2021-06-01' = { + name: policyAssignmentNameEnforceImageSource + properties: { + displayName: '[${clusterName}] ${reference(policyResourceIdEnforceImageSource, '2020-09-01').displayName}' + scope: subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name) + policyDefinitionId: policyResourceIdEnforceImageSource + parameters: { + allowedContainerImagesRegex: { + value: '${defaultAcrName}.azurecr.io/.+$|mcr.microsoft.com/.+$|azurearcfork8s.azurecr.io/azurearcflux/images/stable/.+$|docker.io/weaveworks/kured.+$|docker.io/library/.+$' + } + excludedNamespaces: { + value: [ + 'kube-system' + 'gatekeeper-system' + 'azure-arc' + ] + } + effect: { + value: 'deny' + } + } + } +} + +resource paEnforceDefenderInCluster 'Microsoft.Authorization/policyAssignments@2021-06-01' = { + name: policyAssignmentNameEnforceDefenderInCluster + properties: { + displayName: '[${clusterName}] ${reference(policyResourceIdEnforceDefenderInCluster, '2020-09-01').displayName}' + description: 'Microsoft Defender for Containers should be enabled in the cluster.' + scope: subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name) + policyDefinitionId: policyResourceIdEnforceDefenderInCluster + parameters: { + effect: { + value: 'Audit' + } + } + } +} + resource clusterControlPlaneIdentityName 'Microsoft.ManagedIdentity/userAssignedIdentities@2018-11-30' = { name: 'mi-${clusterName}-controlplane' location: location From 2ce9de0b07425cac8736e29642e28d14785f0a99 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 8 Apr 2022 19:52:28 +0000 Subject: [PATCH 14/54] fix user managed identity resource naming convention --- cluster-stamp.bicep | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index b954cac1..58a456ae 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -348,7 +348,7 @@ resource paEnforceDefenderInCluster 'Microsoft.Authorization/policyAssignments@2 } } -resource clusterControlPlaneIdentityName 'Microsoft.ManagedIdentity/userAssignedIdentities@2018-11-30' = { +resource miClusterControlPlane 'Microsoft.ManagedIdentity/userAssignedIdentities@2018-11-30' = { name: 'mi-${clusterName}-controlplane' location: location } @@ -425,7 +425,7 @@ resource kv_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-0 } } -resource miAppGatewayFrontendSecretsUserRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { +resource kvMiAppGatewayFrontendSecretsUserRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { scope: kv name: '${guid(resourceGroup().id, 'mi-appgateway-frontend', keyVaultSecretsUserRole)}' properties: { @@ -435,7 +435,7 @@ resource miAppGatewayFrontendSecretsUserRole_roleAssignment 'Microsoft.Authoriza } } -resource miAppGatewayFrontendKeyVaultReader_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { +resource kvMiAppGatewayFrontendKeyVaultReader_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { scope: kv name: '${guid(resourceGroup().id, 'mi-appgateway-frontend', keyVaultReader)}' properties: { @@ -445,7 +445,7 @@ resource miAppGatewayFrontendKeyVaultReader_roleAssignment 'Microsoft.Authorizat } } -resource podmiIngressControllerSecretsUserRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { +resource kvPodMiIngressControllerSecretsUserRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { scope: kv name: '${guid(resourceGroup().id, 'podmi-ingress-controller', keyVaultSecretsUserRole)}' properties: { @@ -455,7 +455,7 @@ resource podmiIngressControllerSecretsUserRole_roleAssignment 'Microsoft.Authori } } -resource podmiIngressControllerKeyVaultReader_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { +resource kvPodMiIngressControllerKeyVaultReader_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { scope: kv name: '${guid(resourceGroup().id, 'podmi-ingress-controller', keyVaultReader)}' properties: { From e54138cdbd4d2d469a8f77271e1996cba86f09f4 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 8 Apr 2022 20:48:46 +0000 Subject: [PATCH 15/54] add network contributor role assignments for cluster control plane identity highlights: - resource group: networking - spoke - assign roles at the spoke ingress and cluster nodes subnets level --- cluster-stamp.bicep | 10 ++++++ ...dentityHasRbacToSelfManagedResources.bicep | 35 +++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 nested_EnsureClusterIdentityHasRbacToSelfManagedResources.bicep diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 58a456ae..0e57f031 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -465,6 +465,16 @@ resource kvPodMiIngressControllerKeyVaultReader_roleAssignment 'Microsoft.Author } } +module ndEnsureClusterIdentityHasRbacToSelfManagedResources 'nested_EnsureClusterIdentityHasRbacToSelfManagedResources.bicep' = { + name: 'EnsureClusterIdentityHasRbacToSelfManagedResources' + scope: resourceGroup(vNetResourceGroup) + params: { + miClusterControlPlanePrincipalId: miClusterControlPlane.properties.principalId + clusterControlPlaneIdentityName: miClusterControlPlane.name + vnetName: vnetName + } +} + resource pdzKv 'Microsoft.Network/privateDnsZones@2020-06-01' = { name: 'privatelink.vaultcore.azure.net' location: 'global' diff --git a/nested_EnsureClusterIdentityHasRbacToSelfManagedResources.bicep b/nested_EnsureClusterIdentityHasRbacToSelfManagedResources.bicep new file mode 100644 index 00000000..d61c6ad4 --- /dev/null +++ b/nested_EnsureClusterIdentityHasRbacToSelfManagedResources.bicep @@ -0,0 +1,35 @@ +param miClusterControlPlanePrincipalId string +param clusterControlPlaneIdentityName string +param vnetName string + +var networkContributorRole = '${subscription().id}/providers/Microsoft.Authorization/roleDefinitions/4d97b98b-1d4f-4787-a291-c67834d212e7' + +resource snetClusterNodes 'Microsoft.Network/virtualNetworks/subnets@2021-05-01' existing = { + name: '${vnetName}/snet-clusternodes' +} + +resource snetClusterIngress 'Microsoft.Network/virtualNetworks/subnets@2021-05-01' existing = { + name: '${vnetName}/snet-clusteringressservices' +} + +resource snetClusterNodesMiClusterControlPlaneNetworkContributorRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { + scope: snetClusterNodes + name: guid(snetClusterNodes.id, networkContributorRole, clusterControlPlaneIdentityName) + properties: { + roleDefinitionId: networkContributorRole + description: 'Allows cluster identity to join the nodepool vmss resources to this subnet.' + principalId: miClusterControlPlanePrincipalId + principalType: 'ServicePrincipal' + } +} + +resource snetClusterIngressServicesMiClusterControlPlaneSecretsUserRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { + scope: snetClusterIngress + name: guid(snetClusterIngress.id, networkContributorRole, clusterControlPlaneIdentityName) + properties: { + roleDefinitionId: networkContributorRole + description: 'Allows cluster identity to join load balancers (ingress resources) to this subnet.' + principalId: miClusterControlPlanePrincipalId + principalType: 'ServicePrincipal' + } +} From 8662f5e5dba1be3e982312679e290cf2d4ba1a3d Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 8 Apr 2022 21:10:29 +0000 Subject: [PATCH 16/54] add cluster --- cluster-stamp.bicep | 186 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 186 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 0e57f031..eb507078 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -554,6 +554,192 @@ resource pdzAksIngress 'Microsoft.Network/privateDnsZones@2020-06-01' = { } } +resource mc 'Microsoft.ContainerService/managedClusters@2022-01-02-preview' = { + name: clusterName + location: location + tags: { + 'Business unit': 'BU0001' + 'Application identifier': 'a0008' + } + properties: { + kubernetesVersion: kubernetesVersion + dnsPrefix: uniqueString(subscription().subscriptionId, resourceGroup().id, clusterName) + agentPoolProfiles: [ + { + name: 'npsystem' + count: 3 + vmSize: 'Standard_DS2_v2' + osDiskSizeGB: 80 + osDiskType: 'Ephemeral' + osType: 'Linux' + minCount: 3 + maxCount: 4 + vnetSubnetID: vnetNodePoolSubnetResourceId + enableAutoScaling: true + type: 'VirtualMachineScaleSets' + mode: 'System' + scaleSetPriority: 'Regular' + scaleSetEvictionPolicy: 'Delete' + orchestratorVersion: kubernetesVersion + enableNodePublicIP: false + maxPods: 30 + availabilityZones: [ + '1' + '2' + '3' + ] + upgradeSettings: { + maxSurge: '33%' + } + nodeTaints: [ + 'CriticalAddonsOnly=true:NoSchedule' + ] + } + { + name: 'npuser01' + count: 2 + vmSize: 'Standard_DS3_v2' + osDiskSizeGB: 120 + osDiskType: 'Ephemeral' + osType: 'Linux' + minCount: 2 + maxCount: 5 + vnetSubnetID: vnetNodePoolSubnetResourceId + enableAutoScaling: true + type: 'VirtualMachineScaleSets' + mode: 'User' + scaleSetPriority: 'Regular' + scaleSetEvictionPolicy: 'Delete' + orchestratorVersion: kubernetesVersion + enableNodePublicIP: false + maxPods: 30 + availabilityZones: [ + '1' + '2' + '3' + ] + upgradeSettings: { + maxSurge: '33%' + } + } + ] + servicePrincipalProfile: { + clientId: 'msi' + } + addonProfiles: { + httpApplicationRouting: { + enabled: false + } + omsagent: { + enabled: true + config: { + logAnalyticsWorkspaceResourceId: resourceId('Microsoft.OperationalInsights/workspaces', logAnalyticsWorkspaceName) + } + } + aciConnectorLinux: { + enabled: false + } + azurepolicy: { + enabled: true + config: { + version: 'v2' + } + } + azureKeyvaultSecretsProvider: { + enabled: true + config: { + enableSecretRotation: 'false' + } + } + } + nodeResourceGroup: nodeResourceGroupName + enableRBAC: true + enablePodSecurityPolicy: false + maxAgentPools: 2 + networkProfile: { + networkPlugin: 'azure' + networkPolicy: 'azure' + outboundType: 'userDefinedRouting' + loadBalancerSku: 'standard' + loadBalancerProfile: json('null') + serviceCidr: '172.16.0.0/16' + dnsServiceIP: '172.16.0.10' + dockerBridgeCidr: '172.18.0.1/16' + } + aadProfile: { + managed: true + enableAzureRBAC: isUsingAzureRBACasKubernetesRBAC + adminGroupObjectIDs: ((!isUsingAzureRBACasKubernetesRBAC) ? array(clusterAdminAadGroupObjectId) : []) + tenantID: k8sControlPlaneAuthorizationTenantId + } + autoScalerProfile: { + 'balance-similar-node-groups': 'false' + expander: 'random' + 'max-empty-bulk-delete': '10' + 'max-graceful-termination-sec': '600' + 'max-node-provision-time': '15m' + 'max-total-unready-percentage': '45' + 'new-pod-scale-up-delay': '0s' + 'ok-total-unready-count': '3' + 'scale-down-delay-after-add': '10m' + 'scale-down-delay-after-delete': '20s' + 'scale-down-delay-after-failure': '3m' + 'scale-down-unneeded-time': '10m' + 'scale-down-unready-time': '20m' + 'scale-down-utilization-threshold': '0.5' + 'scan-interval': '10s' + 'skip-nodes-with-local-storage': 'true' + 'skip-nodes-with-system-pods': 'true' + } + apiServerAccessProfile: { + authorizedIPRanges: clusterAuthorizedIPRanges + enablePrivateCluster: false + } + podIdentityProfile: { + enabled: false + userAssignedIdentities: [] + userAssignedIdentityExceptions: [] + } + disableLocalAccounts: true + securityProfile: { + azureDefender: { + enabled: true + logAnalyticsWorkspaceResourceId: resourceId('Microsoft.OperationalInsights/workspaces', logAnalyticsWorkspaceName) + } + } + oidcIssuerProfile: { + enabled: true + } + } + identity: { + type: 'UserAssigned' + userAssignedIdentities: { + '${miClusterControlPlane.id}': {} + } + } + sku: { + name: 'Basic' + tier: 'Paid' + } + dependsOn: [ + sci + + ndEnsureClusterIdentityHasRbacToSelfManagedResources + + paAKSLinuxRestrictive + paEnforceHttpsIngress + paEnforceInternalLoadBalancers + paEnforceResourceLimits + paRoRootFilesystem + paEnforceImageSource + paEnforceDefenderInCluster + + peKv + kvPodMiIngressControllerKeyVaultReader_roleAssignment + kvMiAppGatewayFrontendSecretsUserRole_roleAssignment + ] +} + output aksClusterName string = clusterName output aksIngressControllerPodManagedIdentityResourceId string = podmiIngressController.id output aksIngressControllerPodManagedIdentityClientId string = reference(podmiIngressController.id, '2018-11-30').clientId From 1536a5788caca8c666124c2ecaca8dd7f5aec692 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 8 Apr 2022 21:44:40 +0000 Subject: [PATCH 17/54] add network contributor role assignments for cluster kubelet identity at the subscription level highlights: - resource group: aks cluster --- cluster-stamp.bicep | 9 ++++++++- ...nsureClusterUserAssignedHasRbacToManageVMSS.bicep | 12 ++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 nested_EnsureClusterUserAssignedHasRbacToManageVMSS.bicep diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index eb507078..14a3439b 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -56,7 +56,6 @@ var networkContributorRole = '${subscription().id}/providers/Microsoft.Authoriza var monitoringMetricsPublisherRole = '${subscription().id}/providers/Microsoft.Authorization/roleDefinitions/3913510d-42f4-4e42-8a64-420c390055eb' var acrPullRole = '${subscription().id}/providers/Microsoft.Authorization/roleDefinitions/7f951dda-4ed3-4680-a7ca-43fe172d538d' var managedIdentityOperatorRole = '${subscription().id}/providers/Microsoft.Authorization/roleDefinitions/f1a07417-d97a-45cb-824c-7a7467783830' -var virtualMachineContributorRole = '${subscription().id}/providers/Microsoft.Authorization/roleDefinitions/9980e02c-c2be-4d73-94e8-173b1dc7cf3c' var keyVaultReader = '${subscription().id}/providers/Microsoft.Authorization/roleDefinitions/21090545-7ca7-4776-b22c-e363652d74d2' var keyVaultSecretsUserRole = '${subscription().id}/providers/Microsoft.Authorization/roleDefinitions/4633458b-17de-408a-b874-0445c86b69e6' var clusterAdminRoleId = 'b1ff04bb-8a4e-4dc4-8eb5-8693973ce19b' @@ -740,6 +739,14 @@ resource mc 'Microsoft.ContainerService/managedClusters@2022-01-02-preview' = { ] } +module ndEnsureClusterUserAssignedHasRbacToManageVMSS 'nested_EnsureClusterUserAssignedHasRbacToManageVMSS.bicep' = { + name: 'EnsureClusterUserAssignedHasRbacToManageVMSS' + scope: resourceGroup(nodeResourceGroupName) + params: { + kubeletidentityObjectId: reference(mc.id, '2020-03-01').identityProfile.kubeletidentity.objectId + } +} + output aksClusterName string = clusterName output aksIngressControllerPodManagedIdentityResourceId string = podmiIngressController.id output aksIngressControllerPodManagedIdentityClientId string = reference(podmiIngressController.id, '2018-11-30').clientId diff --git a/nested_EnsureClusterUserAssignedHasRbacToManageVMSS.bicep b/nested_EnsureClusterUserAssignedHasRbacToManageVMSS.bicep new file mode 100644 index 00000000..53d382cd --- /dev/null +++ b/nested_EnsureClusterUserAssignedHasRbacToManageVMSS.bicep @@ -0,0 +1,12 @@ +param kubeletidentityObjectId string + +var virtualMachineContributorRole = '${subscription().id}/providers/Microsoft.Authorization/roleDefinitions/9980e02c-c2be-4d73-94e8-173b1dc7cf3c' + +resource id 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { + name: guid(resourceGroup().id) + properties: { + roleDefinitionId: virtualMachineContributorRole + principalId: kubeletidentityObjectId + principalType: 'ServicePrincipal' + } +} From e90f92d0c89f70a24f1314a2d6991db814ffee2f Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 8 Apr 2022 22:10:30 +0000 Subject: [PATCH 18/54] add acr pull role assignments for cluster kubelet identity --- cluster-stamp.bicep | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 14a3439b..deb24836 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -739,6 +739,21 @@ resource mc 'Microsoft.ContainerService/managedClusters@2022-01-02-preview' = { ] } +resource acr 'Microsoft.ContainerRegistry/registries@2021-12-01-preview' existing = { + name: defaultAcrName +} + +resource acrKubeletAcrPullRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { + scope: acr + name: guid(mc.id, acrPullRole) + properties: { + roleDefinitionId: acrPullRole + description: 'Allows AKS to pull container images from this ACR instance.' + principalId: reference(mc.id, '2020-12-01').identityProfile.kubeletidentity.objectId + principalType: 'ServicePrincipal' + } +} + module ndEnsureClusterUserAssignedHasRbacToManageVMSS 'nested_EnsureClusterUserAssignedHasRbacToManageVMSS.bicep' = { name: 'EnsureClusterUserAssignedHasRbacToManageVMSS' scope: resourceGroup(nodeResourceGroupName) From af8cc7b47e775f87a46ca0eaae77d5f27cbb939f Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 8 Apr 2022 22:12:04 +0000 Subject: [PATCH 19/54] [aks extension] add flux --- cluster-stamp.bicep | 67 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index deb24836..1a6b2629 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -754,6 +754,73 @@ resource acrKubeletAcrPullRole_roleAssignment 'Microsoft.Authorization/roleAssig } } +resource mcFlux_extension 'Microsoft.KubernetesConfiguration/extensions@2021-09-01' = { + scope: mc + name: 'flux' + properties: { + extensionType: 'microsoft.flux' + autoUpgradeMinorVersion: true + releaseTrain: 'Stable' + scope: { + cluster: { + releaseNamespace: 'flux-system' + } + } + configurationSettings: { + 'helm-controller.enabled': 'false' + 'source-controller.enabled': 'true' + 'kustomize-controller.enabled': 'true' + 'notification-controller.enabled': 'false' + 'image-automation-controller.enabled': 'false' + 'image-reflector-controller.enabled': 'false' + } + configurationProtectedSettings: {} + } + dependsOn: [ + acrKubeletAcrPullRole_roleAssignment + ] +} + +resource mc_fluxConfiguration 'Microsoft.KubernetesConfiguration/fluxConfigurations@2022-03-01' = { + scope: mc + name: 'bootstrap' + properties: { + scope: 'cluster' + namespace: 'flux-system' + sourceKind: 'GitRepository' + gitRepository: { + url: gitOpsBootstrappingRepoHttpsUrl + timeoutInSeconds: 180 + syncIntervalInSeconds: 300 + repositoryRef: { + branch: gitOpsBootstrappingRepoBranch + tag: null + semver: null + commit: null + } + sshKnownHosts: '' + httpsUser: null + httpsCACert: null + localAuthRef: null + } + kustomizations: { + unified: { + path: './cluster-manifests' + dependsOn: [] + timeoutInSeconds: 300 + syncIntervalInSeconds: 300 + retryIntervalInSeconds: null + prune: true + force: false + } + } + } + dependsOn: [ + mcFlux_extension + acrKubeletAcrPullRole_roleAssignment + ] +} + module ndEnsureClusterUserAssignedHasRbacToManageVMSS 'nested_EnsureClusterUserAssignedHasRbacToManageVMSS.bicep' = { name: 'EnsureClusterUserAssignedHasRbacToManageVMSS' scope: resourceGroup(nodeResourceGroupName) From e0079052e0fcd82936955d9f2cd19fd67b095d71 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Tue, 19 Apr 2022 14:11:56 +0000 Subject: [PATCH 20/54] add monitoring metrics role assignments for cluster omsagent identity --- cluster-stamp.bicep | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 1a6b2629..4ffd22fb 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -754,6 +754,16 @@ resource acrKubeletAcrPullRole_roleAssignment 'Microsoft.Authorization/roleAssig } } +resource mcOmsAgentMonitoringMetricsPublisherRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { + scope: mc + name: guid(mc.id, 'omsagent', monitoringMetricsPublisherRole) + properties: { + roleDefinitionId: monitoringMetricsPublisherRole + principalId: reference(mc.id, '2020-12-01').addonProfiles.omsagent.identity.objectId + principalType: 'ServicePrincipal' + } +} + resource mcFlux_extension 'Microsoft.KubernetesConfiguration/extensions@2021-09-01' = { scope: mc name: 'flux' From c68b0dfb19f0eac8cdd9e8ead58cbb264bbc06d2 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Tue, 19 Apr 2022 14:15:44 +0000 Subject: [PATCH 21/54] add mc diagnostic settings --- cluster-stamp.bicep | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 4ffd22fb..cf957e3e 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -764,6 +764,32 @@ resource mcOmsAgentMonitoringMetricsPublisherRole_roleAssignment 'Microsoft.Auth } } +resource mc_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { + scope: mc + name: 'default' + properties: { + workspaceId: resourceId('Microsoft.OperationalInsights/workspaces', logAnalyticsWorkspaceName) + logs: [ + { + category: 'cluster-autoscaler' + enabled: true + } + { + category: 'kube-controller-manager' + enabled: true + } + { + category: 'kube-audit-admin' + enabled: true + } + { + category: 'guard' + enabled: true + } + ] + } +} + resource mcFlux_extension 'Microsoft.KubernetesConfiguration/extensions@2021-09-01' = { scope: mc name: 'flux' From a0ffae34b755a4e94a931185ded83de7b620c3dd Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Tue, 19 Apr 2022 14:18:17 +0000 Subject: [PATCH 22/54] add event grid system topic --- cluster-stamp.bicep | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index cf957e3e..3727c7e0 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -865,6 +865,15 @@ module ndEnsureClusterUserAssignedHasRbacToManageVMSS 'nested_EnsureClusterUserA } } +resource st 'Microsoft.EventGrid/systemTopics@2021-12-01' = { + name: clusterName + location: location + properties: { + source: mc.id + topicType: 'Microsoft.ContainerService.ManagedClusters' + } +} + output aksClusterName string = clusterName output aksIngressControllerPodManagedIdentityResourceId string = podmiIngressController.id output aksIngressControllerPodManagedIdentityClientId string = reference(podmiIngressController.id, '2018-11-30').clientId From 157edb52da00b14e0d4c16c08943be4ae2338dff Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Tue, 19 Apr 2022 14:20:59 +0000 Subject: [PATCH 23/54] add st diagnostic settings --- cluster-stamp.bicep | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 3727c7e0..f7b4c662 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -874,6 +874,26 @@ resource st 'Microsoft.EventGrid/systemTopics@2021-12-01' = { } } +resource st_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { + scope: st + name: 'default' + properties: { + workspaceId: resourceId('Microsoft.OperationalInsights/workspaces', logAnalyticsWorkspaceName) + logs: [ + { + category: 'DeliveryFailures' + enabled: true + } + ] + metrics: [ + { + category: 'AllMetrics' + enabled: true + } + ] + } +} + output aksClusterName string = clusterName output aksIngressControllerPodManagedIdentityResourceId string = podmiIngressController.id output aksIngressControllerPodManagedIdentityClientId string = reference(podmiIngressController.id, '2018-11-30').clientId From 9c2f101de23c2b8972267171973445e110e5bff1 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Tue, 19 Apr 2022 14:33:01 +0000 Subject: [PATCH 24/54] add managed identity operator role assignments for cluster kubelet identity --- cluster-stamp.bicep | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index f7b4c662..beb37df9 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -764,6 +764,16 @@ resource mcOmsAgentMonitoringMetricsPublisherRole_roleAssignment 'Microsoft.Auth } } +resource miKubeletManagedIdentityOperatorRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { + scope: podmiIngressController + name: guid(resourceGroup().id, 'podmi-ingress-controller', managedIdentityOperatorRole) + properties: { + roleDefinitionId: managedIdentityOperatorRole + principalId: reference(mc.id, '2020-12-01').identityProfile.kubeletidentity.objectId + principalType: 'ServicePrincipal' + } +} + resource mc_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { scope: mc name: 'default' From 935134926f70a0b0aa17610dae94a99653c92894 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Tue, 19 Apr 2022 15:08:31 +0000 Subject: [PATCH 25/54] add cluster metrics alerts --- cluster-stamp.bicep | 584 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 584 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index beb37df9..703c5eb9 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -157,6 +157,590 @@ resource sci 'Microsoft.OperationsManagement/solutions@2015-11-01-preview' = { } } +resource maHighNodeCPUUtilization 'Microsoft.Insights/metricAlerts@2018-03-01' = { + name: 'Node CPU utilization high for ${clusterName} CI-1' + location: 'global' + properties: { + scopes: [ + mc.id + ] + actions: [] + criteria: { + allOf: [ + { + criterionType: 'StaticThresholdCriterion' + dimensions: [ + { + name: 'host' + operator: 'Include' + values: [ + '*' + ] + } + ] + metricName: 'cpuUsagePercentage' + metricNamespace: 'Insights.Container/nodes' + name: 'Metric1' + operator: 'GreaterThan' + threshold: '80' + timeAggregation: 'Average' + skipMetricValidation: true + } + ] + 'odata.type': 'Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria' + } + description: 'Node CPU utilization across the cluster.' + enabled: true + evaluationFrequency: 'PT1M' + severity: 3 + targetResourceType: 'microsoft.containerservice/managedclusters' + windowSize: 'PT5M' + } + dependsOn: [ + sci + ] +} + +resource maHighNodeWorkingSetMemoryUtilization 'Microsoft.Insights/metricAlerts@2018-03-01' = { + name: 'Node working set memory utilization high for ${clusterName} CI-2' + location: 'global' + properties: { + actions: [] + criteria: { + allOf: [ + { + criterionType: 'StaticThresholdCriterion' + dimensions: [ + { + name: 'host' + operator: 'Include' + values: [ + '*' + ] + } + ] + metricName: 'memoryWorkingSetPercentage' + metricNamespace: 'Insights.Container/nodes' + name: 'Metric1' + operator: 'GreaterThan' + threshold: '80' + timeAggregation: 'Average' + skipMetricValidation: true + } + ] + 'odata.type': 'Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria' + } + description: 'Node working set memory utilization across the cluster.' + enabled: true + evaluationFrequency: 'PT1M' + scopes: [ + mc.id + ] + severity: 3 + targetResourceType: 'microsoft.containerservice/managedclusters' + windowSize: 'PT5M' + } + dependsOn: [ + sci + ] +} + +resource maJobsCompletedMoreThan6HoursAgo 'Microsoft.Insights/metricAlerts@2018-03-01' = { + name: 'Jobs completed more than 6 hours ago for ${clusterName} CI-11' + location: 'global' + properties: { + actions: [] + criteria: { + allOf: [ + { + criterionType: 'StaticThresholdCriterion' + dimensions: [ + { + name: 'controllerName' + operator: 'Include' + values: [ + '*' + ] + } + { + name: 'kubernetes namespace' + operator: 'Include' + values: [ + '*' + ] + } + ] + metricName: 'completedJobsCount' + metricNamespace: 'Insights.Container/pods' + name: 'Metric1' + operator: 'GreaterThan' + threshold: '0' + timeAggregation: 'Average' + skipMetricValidation: true + } + ] + 'odata.type': 'Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria' + } + description: 'This alert monitors completed jobs (more than 6 hours ago).' + enabled: true + evaluationFrequency: 'PT1M' + scopes: [ + mc.id + ] + severity: 3 + targetResourceType: 'microsoft.containerservice/managedclusters' + windowSize: 'PT1M' + } + dependsOn: [ + sci + ] +} + +resource maHighContainerCPUUsage 'Microsoft.Insights/metricAlerts@2018-03-01' = { + name: 'Container CPU usage high for ${clusterName} CI-9' + location: 'global' + properties: { + actions: [] + criteria: { + allOf: [ + { + criterionType: 'StaticThresholdCriterion' + dimensions: [ + { + name: 'controllerName' + operator: 'Include' + values: [ + '*' + ] + } + { + name: 'kubernetes namespace' + operator: 'Include' + values: [ + '*' + ] + } + ] + metricName: 'cpuExceededPercentage' + metricNamespace: 'Insights.Container/containers' + name: 'Metric1' + operator: 'GreaterThan' + threshold: '90' + timeAggregation: 'Average' + skipMetricValidation: true + } + ] + 'odata.type': 'Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria' + } + description: 'This alert monitors container CPU utilization.' + enabled: true + evaluationFrequency: 'PT1M' + scopes: [ + mc.id + ] + severity: 3 + targetResourceType: 'microsoft.containerservice/managedclusters' + windowSize: 'PT5M' + } + dependsOn: [ + sci + ] +} + +resource maHighContainerWorkingSetMemoryUsage 'Microsoft.Insights/metricAlerts@2018-03-01' = { + name: 'Container working set memory usage high for ${clusterName} CI-10' + location: 'global' + properties: { + actions: [] + criteria: { + allOf: [ + { + criterionType: 'StaticThresholdCriterion' + dimensions: [ + { + name: 'controllerName' + operator: 'Include' + values: [ + '*' + ] + } + { + name: 'kubernetes namespace' + operator: 'Include' + values: [ + '*' + ] + } + ] + metricName: 'memoryWorkingSetExceededPercentage' + metricNamespace: 'Insights.Container/containers' + name: 'Metric1' + operator: 'GreaterThan' + threshold: '90' + timeAggregation: 'Average' + skipMetricValidation: true + } + ] + 'odata.type': 'Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria' + } + description: 'This alert monitors container working set memory utilization.' + enabled: true + evaluationFrequency: 'PT1M' + scopes: [ + mc.id + ] + severity: 3 + targetResourceType: 'microsoft.containerservice/managedclusters' + windowSize: 'PT5M' + } + dependsOn: [ + sci + ] +} + +resource maPodsInFailedState 'Microsoft.Insights/metricAlerts@2018-03-01' = { + name: 'Pods in failed state for ${clusterName} CI-4' + location: 'global' + properties: { + actions: [] + criteria: { + allOf: [ + { + criterionType: 'StaticThresholdCriterion' + dimensions: [ + { + name: 'phase' + operator: 'Include' + values: [ + 'Failed' + ] + } + ] + metricName: 'podCount' + metricNamespace: 'Insights.Container/pods' + name: 'Metric1' + operator: 'GreaterThan' + threshold: '0' + timeAggregation: 'Average' + skipMetricValidation: true + } + ] + 'odata.type': 'Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria' + } + description: 'Pod status monitoring.' + enabled: true + evaluationFrequency: 'PT1M' + scopes: [ + mc.id + ] + severity: 3 + targetResourceType: 'microsoft.containerservice/managedclusters' + windowSize: 'PT5M' + } + dependsOn: [ + sci + ] +} + +resource maHighDiskUsage 'Microsoft.Insights/metricAlerts@2018-03-01' = { + name: 'Disk usage high for ${clusterName} CI-5' + location: 'global' + properties: { + actions: [] + criteria: { + allOf: [ + { + criterionType: 'StaticThresholdCriterion' + dimensions: [ + { + name: 'host' + operator: 'Include' + values: [ + '*' + ] + } + { + name: 'device' + operator: 'Include' + values: [ + '*' + ] + } + ] + metricName: 'DiskUsedPercentage' + metricNamespace: 'Insights.Container/nodes' + name: 'Metric1' + operator: 'GreaterThan' + threshold: '80' + timeAggregation: 'Average' + skipMetricValidation: true + } + ] + 'odata.type': 'Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria' + } + description: 'This alert monitors disk usage for all nodes and storage devices.' + enabled: true + evaluationFrequency: 'PT1M' + scopes: [ + mc.id + ] + severity: 3 + targetResourceType: 'microsoft.containerservice/managedclusters' + windowSize: 'PT5M' + } + dependsOn: [ + sci + ] +} + +resource maNodesInNotReadyStatus 'Microsoft.Insights/metricAlerts@2018-03-01' = { + name: 'Nodes in not ready status for ${clusterName} CI-3' + location: 'global' + properties: { + actions: [] + criteria: { + allOf: [ + { + criterionType: 'StaticThresholdCriterion' + dimensions: [ + { + name: 'status' + operator: 'Include' + values: [ + 'NotReady' + ] + } + ] + metricName: 'nodesCount' + metricNamespace: 'Insights.Container/nodes' + name: 'Metric1' + operator: 'GreaterThan' + threshold: '0' + timeAggregation: 'Average' + skipMetricValidation: true + } + ] + 'odata.type': 'Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria' + } + description: 'Node status monitoring.' + enabled: true + evaluationFrequency: 'PT1M' + scopes: [ + mc.id + ] + severity: 3 + targetResourceType: 'microsoft.containerservice/managedclusters' + windowSize: 'PT5M' + } + dependsOn: [ + sci + ] +} + +resource maContainersGettingKilledOOM 'Microsoft.Insights/metricAlerts@2018-03-01' = { + name: 'Containers getting OOM killed for ${clusterName} CI-6' + location: 'global' + properties: { + actions: [] + criteria: { + allOf: [ + { + criterionType: 'StaticThresholdCriterion' + dimensions: [ + { + name: 'kubernetes namespace' + operator: 'Include' + values: [ + '*' + ] + } + { + name: 'controllerName' + operator: 'Include' + values: [ + '*' + ] + } + ] + metricName: 'oomKilledContainerCount' + metricNamespace: 'Insights.Container/pods' + name: 'Metric1' + operator: 'GreaterThan' + threshold: '0' + timeAggregation: 'Average' + skipMetricValidation: true + } + ] + 'odata.type': 'Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria' + } + description: 'This alert monitors number of containers killed due to out of memory (OOM) error.' + enabled: true + evaluationFrequency: 'PT1M' + scopes: [ + mc.id + ] + severity: 3 + targetResourceType: 'microsoft.containerservice/managedclusters' + windowSize: 'PT1M' + } + dependsOn: [ + sci + ] +} + +resource maHighPersistentVolumeUsage 'Microsoft.Insights/metricAlerts@2018-03-01' = { + name: 'Persistent volume usage high for ${clusterName} CI-18' + location: 'global' + properties: { + actions: [] + criteria: { + allOf: [ + { + criterionType: 'StaticThresholdCriterion' + dimensions: [ + { + name: 'podName' + operator: 'Include' + values: [ + '*' + ] + } + { + name: 'kubernetesNamespace' + operator: 'Include' + values: [ + '*' + ] + } + ] + metricName: 'pvUsageExceededPercentage' + metricNamespace: 'Insights.Container/persistentvolumes' + name: 'Metric1' + operator: 'GreaterThan' + threshold: '80' + timeAggregation: 'Average' + skipMetricValidation: true + } + ] + 'odata.type': 'Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria' + } + description: 'This alert monitors persistent volume utilization.' + enabled: false + evaluationFrequency: 'PT1M' + scopes: [ + mc.id + ] + severity: 3 + targetResourceType: 'microsoft.containerservice/managedclusters' + windowSize: 'PT5M' + } + dependsOn: [ + sci + ] +} + +resource maPodsNotInReadyState 'Microsoft.Insights/metricAlerts@2018-03-01' = { + name: 'Pods not in ready state for ${clusterName} CI-8' + location: 'global' + properties: { + actions: [] + criteria: { + allOf: [ + { + criterionType: 'StaticThresholdCriterion' + dimensions: [ + { + name: 'controllerName' + operator: 'Include' + values: [ + '*' + ] + } + { + name: 'kubernetes namespace' + operator: 'Include' + values: [ + '*' + ] + } + ] + metricName: 'PodReadyPercentage' + metricNamespace: 'Insights.Container/pods' + name: 'Metric1' + operator: 'LessThan' + threshold: '80' + timeAggregation: 'Average' + skipMetricValidation: true + } + ] + 'odata.type': 'Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria' + } + description: 'This alert monitors for excessive pods not in the ready state.' + enabled: true + evaluationFrequency: 'PT1M' + scopes: [ + mc.id + ] + severity: 3 + targetResourceType: 'microsoft.containerservice/managedclusters' + windowSize: 'PT5M' + } + dependsOn: [ + sci + ] +} + +resource maRestartingContainerCount 'Microsoft.Insights/metricAlerts@2018-03-01' = { + name: 'Restarting container count for ${clusterName} CI-7' + location: 'global' + properties: { + actions: [] + criteria: { + allOf: [ + { + criterionType: 'StaticThresholdCriterion' + dimensions: [ + { + name: 'kubernetes namespace' + operator: 'Include' + values: [ + '*' + ] + } + { + name: 'controllerName' + operator: 'Include' + values: [ + '*' + ] + } + ] + metricName: 'restartingContainerCount' + metricNamespace: 'Insights.Container/pods' + name: 'Metric1' + operator: 'GreaterThan' + threshold: '0' + timeAggregation: 'Average' + skipMetricValidation: true + } + ] + 'odata.type': 'Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria' + } + description: 'This alert monitors number of containers restarting across the cluster.' + enabled: true + evaluationFrequency: 'PT1M' + scopes: [ + mc.id + ] + severity: 3 + targetResourceType: 'Microsoft.ContainerService/managedClusters' + windowSize: 'PT1M' + } + dependsOn: [ + sci + ] +} + resource skva 'Microsoft.OperationsManagement/solutions@2015-11-01-preview' = { name: 'KeyVaultAnalytics(${logAnalyticsWorkspaceName})' location: location From 8f6e0f7a7253843897cbee14e096f81de7471ea9 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Tue, 19 Apr 2022 15:36:45 +0000 Subject: [PATCH 26/54] add mc role assignments for aad groups --- cluster-stamp.bicep | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 703c5eb9..14437624 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -1358,6 +1358,50 @@ resource miKubeletManagedIdentityOperatorRole_roleAssignment 'Microsoft.Authoriz } } +resource mcAadAdminGroupClusterAdminRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = if (isUsingAzureRBACasKubernetesRBAC) { + scope: mc + name: guid('aad-admin-group', mc.id, clusterAdminAadGroupObjectId) + properties: { + roleDefinitionId: '/subscriptions/${subscription().subscriptionId}/providers/Microsoft.Authorization/roleDefinitions/${clusterAdminRoleId}' + description: 'Members of this group are cluster admins of this cluster.' + principalId: clusterAdminAadGroupObjectId + principalType: 'Group' + } +} + +resource mcAadAdminGroupServiceClusterUserRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = if (isUsingAzureRBACasKubernetesRBAC) { + scope: mc + name: guid('aad-admin-group-sc', mc.id, clusterAdminAadGroupObjectId) + properties: { + roleDefinitionId: '/subscriptions/${subscription().subscriptionId}/providers/Microsoft.Authorization/roleDefinitions/${serviceClusterUserRoleId}' + description: 'Members of this group are cluster users of this cluster.' + principalId: clusterAdminAadGroupObjectId + principalType: 'Group' + } +} + +resource maAadA0008ReaderGroupClusterReaderRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = if (isUsingAzureRBACasKubernetesRBAC && (!(a0008NamespaceReaderAadGroupObjectId == clusterAdminAadGroupObjectId))) { + scope: mc // TODO: reference namespace instead + name: guid('aad-a0008-reader-group', mc.id, a0008NamespaceReaderAadGroupObjectId) + properties: { + roleDefinitionId: '/subscriptions/${subscription().subscriptionId}/providers/Microsoft.Authorization/roleDefinitions/${clusterReaderRoleId}' + principalId: a0008NamespaceReaderAadGroupObjectId + description: 'Members of this group are cluster admins of the a0008 namespace in this cluster.' + principalType: 'Group' + } +} + +resource maAadA0008ReaderGroupServiceClusterUserRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = if (isUsingAzureRBACasKubernetesRBAC && (!(a0008NamespaceReaderAadGroupObjectId == clusterAdminAadGroupObjectId))) { + scope: mc + name: guid('aad-a0008-reader-group-sc', mc.id, a0008NamespaceReaderAadGroupObjectId) + properties: { + roleDefinitionId: '/subscriptions/${subscription().subscriptionId}/providers/Microsoft.Authorization/roleDefinitions/${serviceClusterUserRoleId}' + principalId: a0008NamespaceReaderAadGroupObjectId + description: 'Members of this group are cluster users of this cluster.' + principalType: 'Group' + } +} + resource mc_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { scope: mc name: 'default' From 0f9bfd0106b0101a761d6b2a321ba7fe13b62405 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Tue, 19 Apr 2022 15:59:17 +0000 Subject: [PATCH 27/54] add appgw --- cluster-stamp.bicep | 195 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 195 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 14437624..5e7c2f2e 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -1532,6 +1532,201 @@ resource st_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-0 } } +resource agw 'Microsoft.Network/applicationGateways@2021-05-01' = { + name: agwName + location: location + identity: { + type: 'UserAssigned' + userAssignedIdentities: { + '${miAppGatewayFrontend.id}': {} + } + } + zones: pickZones('Microsoft.Network', 'applicationGateways', location, 3) + properties: { + sku: { + name: 'WAF_v2' + tier: 'WAF_v2' + } + sslPolicy: { + policyType: 'Custom' + cipherSuites: [ + 'TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384' + 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256' + ] + minProtocolVersion: 'TLSv1_2' + } + trustedRootCertificates: [ + { + name: 'root-cert-wildcard-aks-ingress' + properties: { + keyVaultSecretId: '${reference(kv.name).vaultUri}secrets/appgw-ingress-internal-aks-ingress-tls' + } + } + ] + gatewayIPConfigurations: [ + { + name: 'apw-ip-configuration' + properties: { + subnet: { + id: '${targetVnetResourceId}/subnets/snet-applicationgateway' + } + } + } + ] + frontendIPConfigurations: [ + { + name: 'apw-frontend-ip-configuration' + properties: { + publicIPAddress: { + id: resourceId(subscription().subscriptionId, vNetResourceGroup, 'Microsoft.Network/publicIpAddresses', 'pip-BU0001A0008-00') + } + } + } + ] + frontendPorts: [ + { + name: 'port-443' + properties: { + port: 443 + } + } + ] + autoscaleConfiguration: { + minCapacity: 0 + maxCapacity: 10 + } + webApplicationFirewallConfiguration: { + enabled: true + firewallMode: 'Prevention' + ruleSetType: 'OWASP' + ruleSetVersion: '3.2' + exclusions: [] + fileUploadLimitInMb: 10 + disabledRuleGroups: [] + } + enableHttp2: false + sslCertificates: [ + { + name: '${agwName}-ssl-certificate' + properties: { + keyVaultSecretId: '${reference(kv.name).vaultUri}secrets/gateway-public-cert' + } + } + ] + probes: [ + { + name: 'probe-${aksBackendDomainName}' + properties: { + protocol: 'Https' + path: '/favicon.ico' + interval: 30 + timeout: 30 + unhealthyThreshold: 3 + pickHostNameFromBackendHttpSettings: true + minServers: 0 + match: {} + } + } + ] + backendAddressPools: [ + { + name: aksBackendDomainName + properties: { + backendAddresses: [ + { + fqdn: aksBackendDomainName + } + ] + } + } + ] + backendHttpSettingsCollection: [ + { + name: 'aks-ingress-backendpool-httpsettings' + properties: { + port: 443 + protocol: 'Https' + cookieBasedAffinity: 'Disabled' + pickHostNameFromBackendAddress: true + requestTimeout: 20 + probe: { + id: resourceId('Microsoft.Network/applicationGateways/probes', agwName, 'probe-${aksBackendDomainName}') + } + trustedRootCertificates: [ + { + id: resourceId('Microsoft.Network/applicationGateways/trustedRootCertificates', agwName, 'root-cert-wildcard-aks-ingress') + } + ] + } + } + ] + httpListeners: [ + { + name: 'listener-https' + properties: { + frontendIPConfiguration: { + id: resourceId('Microsoft.Network/applicationGateways/frontendIPConfigurations', agwName, 'apw-frontend-ip-configuration') + } + frontendPort: { + id: resourceId('Microsoft.Network/applicationGateways/frontendPorts', agwName, 'port-443') + } + protocol: 'Https' + sslCertificate: { + id: resourceId('Microsoft.Network/applicationGateways/sslCertificates', agwName, '${agwName}-ssl-certificate') + } + hostName: 'bicycle.${domainName}' + hostNames: [] + requireServerNameIndication: true + } + } + ] + requestRoutingRules: [ + { + name: 'apw-routing-rules' + properties: { + ruleType: 'Basic' + httpListener: { + id: resourceId('Microsoft.Network/applicationGateways/httpListeners', agwName, 'listener-https') + } + backendAddressPool: { + id: resourceId('Microsoft.Network/applicationGateways/backendAddressPools', agwName, aksBackendDomainName) + } + backendHttpSettings: { + id: resourceId('Microsoft.Network/applicationGateways/backendHttpSettingsCollection', agwName, 'aks-ingress-backendpool-httpsettings') + } + } + } + ] + } + dependsOn: [ + peKv + kvMiAppGatewayFrontendKeyVaultReader_roleAssignment + kvMiAppGatewayFrontendSecretsUserRole_roleAssignment + ] +} + +resource agwdiagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { + scope: agw + name: 'default' + properties: { + workspaceId: resourceId('Microsoft.OperationalInsights/workspaces', logAnalyticsWorkspaceName) + logs: [ + { + category: 'ApplicationGatewayAccessLog' + enabled: true + } + { + category: 'ApplicationGatewayPerformanceLog' + enabled: true + } + { + category: 'ApplicationGatewayFirewallLog' + enabled: true + } + ] + } +} + output aksClusterName string = clusterName output aksIngressControllerPodManagedIdentityResourceId string = podmiIngressController.id output aksIngressControllerPodManagedIdentityClientId string = reference(podmiIngressController.id, '2018-11-30').clientId From cc4459468d127bd59062e402d8610a1c2dd3ec01 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Wed, 20 Apr 2022 12:25:57 +0000 Subject: [PATCH 28/54] add default values to props from resources --- cluster-stamp.bicep | 85 +++++++++++++++++++++++++++++++++++++-------- 1 file changed, 70 insertions(+), 15 deletions(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 5e7c2f2e..4a4c70c0 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -119,6 +119,7 @@ resource alaRgRecommendations 'Microsoft.Insights/activityLogAlerts@2020-10-01' enabled: true description: 'All azure advisor alerts' } + dependsOn: [] } resource ssPrometheusAll 'Microsoft.OperationalInsights/workspaces/savedSearches@2020-08-01' = { @@ -130,6 +131,7 @@ resource ssPrometheusAll 'Microsoft.OperationalInsights/workspaces/savedSearches query: 'InsightsMetrics | where Namespace == "prometheus"' version: 1 } + dependsOn: [] } resource ssPrometheusKuredRequestedReeboot 'Microsoft.OperationalInsights/workspaces/savedSearches@2020-08-01' = { @@ -147,6 +149,8 @@ resource sci 'Microsoft.OperationsManagement/solutions@2015-11-01-preview' = { name: 'ContainerInsights(${logAnalyticsWorkspaceName})' location: location properties: { + containedResources: [] + referencedResources: [] workspaceResourceId: resourceId('Microsoft.OperationalInsights/workspaces', logAnalyticsWorkspaceName) } plan: { @@ -155,12 +159,14 @@ resource sci 'Microsoft.OperationsManagement/solutions@2015-11-01-preview' = { promotionCode: '' publisher: 'Microsoft' } + dependsOn: [] } resource maHighNodeCPUUtilization 'Microsoft.Insights/metricAlerts@2018-03-01' = { name: 'Node CPU utilization high for ${clusterName} CI-1' location: 'global' properties: { + autoMitigate: true scopes: [ mc.id ] @@ -182,7 +188,7 @@ resource maHighNodeCPUUtilization 'Microsoft.Insights/metricAlerts@2018-03-01' = metricNamespace: 'Insights.Container/nodes' name: 'Metric1' operator: 'GreaterThan' - threshold: '80' + threshold: 80 timeAggregation: 'Average' skipMetricValidation: true } @@ -205,6 +211,7 @@ resource maHighNodeWorkingSetMemoryUtilization 'Microsoft.Insights/metricAlerts@ name: 'Node working set memory utilization high for ${clusterName} CI-2' location: 'global' properties: { + autoMitigate: true actions: [] criteria: { allOf: [ @@ -223,7 +230,7 @@ resource maHighNodeWorkingSetMemoryUtilization 'Microsoft.Insights/metricAlerts@ metricNamespace: 'Insights.Container/nodes' name: 'Metric1' operator: 'GreaterThan' - threshold: '80' + threshold: 80 timeAggregation: 'Average' skipMetricValidation: true } @@ -249,6 +256,7 @@ resource maJobsCompletedMoreThan6HoursAgo 'Microsoft.Insights/metricAlerts@2018- name: 'Jobs completed more than 6 hours ago for ${clusterName} CI-11' location: 'global' properties: { + autoMitigate: true actions: [] criteria: { allOf: [ @@ -274,7 +282,7 @@ resource maJobsCompletedMoreThan6HoursAgo 'Microsoft.Insights/metricAlerts@2018- metricNamespace: 'Insights.Container/pods' name: 'Metric1' operator: 'GreaterThan' - threshold: '0' + threshold: 0 timeAggregation: 'Average' skipMetricValidation: true } @@ -300,6 +308,7 @@ resource maHighContainerCPUUsage 'Microsoft.Insights/metricAlerts@2018-03-01' = name: 'Container CPU usage high for ${clusterName} CI-9' location: 'global' properties: { + autoMitigate: true actions: [] criteria: { allOf: [ @@ -325,7 +334,7 @@ resource maHighContainerCPUUsage 'Microsoft.Insights/metricAlerts@2018-03-01' = metricNamespace: 'Insights.Container/containers' name: 'Metric1' operator: 'GreaterThan' - threshold: '90' + threshold: 90 timeAggregation: 'Average' skipMetricValidation: true } @@ -351,6 +360,7 @@ resource maHighContainerWorkingSetMemoryUsage 'Microsoft.Insights/metricAlerts@2 name: 'Container working set memory usage high for ${clusterName} CI-10' location: 'global' properties: { + autoMitigate: true actions: [] criteria: { allOf: [ @@ -376,7 +386,7 @@ resource maHighContainerWorkingSetMemoryUsage 'Microsoft.Insights/metricAlerts@2 metricNamespace: 'Insights.Container/containers' name: 'Metric1' operator: 'GreaterThan' - threshold: '90' + threshold: 90 timeAggregation: 'Average' skipMetricValidation: true } @@ -402,6 +412,7 @@ resource maPodsInFailedState 'Microsoft.Insights/metricAlerts@2018-03-01' = { name: 'Pods in failed state for ${clusterName} CI-4' location: 'global' properties: { + autoMitigate: true actions: [] criteria: { allOf: [ @@ -420,7 +431,7 @@ resource maPodsInFailedState 'Microsoft.Insights/metricAlerts@2018-03-01' = { metricNamespace: 'Insights.Container/pods' name: 'Metric1' operator: 'GreaterThan' - threshold: '0' + threshold: 0 timeAggregation: 'Average' skipMetricValidation: true } @@ -446,6 +457,7 @@ resource maHighDiskUsage 'Microsoft.Insights/metricAlerts@2018-03-01' = { name: 'Disk usage high for ${clusterName} CI-5' location: 'global' properties: { + autoMitigate: true actions: [] criteria: { allOf: [ @@ -471,7 +483,7 @@ resource maHighDiskUsage 'Microsoft.Insights/metricAlerts@2018-03-01' = { metricNamespace: 'Insights.Container/nodes' name: 'Metric1' operator: 'GreaterThan' - threshold: '80' + threshold: 80 timeAggregation: 'Average' skipMetricValidation: true } @@ -497,6 +509,7 @@ resource maNodesInNotReadyStatus 'Microsoft.Insights/metricAlerts@2018-03-01' = name: 'Nodes in not ready status for ${clusterName} CI-3' location: 'global' properties: { + autoMitigate: true actions: [] criteria: { allOf: [ @@ -515,7 +528,7 @@ resource maNodesInNotReadyStatus 'Microsoft.Insights/metricAlerts@2018-03-01' = metricNamespace: 'Insights.Container/nodes' name: 'Metric1' operator: 'GreaterThan' - threshold: '0' + threshold: 0 timeAggregation: 'Average' skipMetricValidation: true } @@ -541,6 +554,7 @@ resource maContainersGettingKilledOOM 'Microsoft.Insights/metricAlerts@2018-03-0 name: 'Containers getting OOM killed for ${clusterName} CI-6' location: 'global' properties: { + autoMitigate: true actions: [] criteria: { allOf: [ @@ -566,7 +580,7 @@ resource maContainersGettingKilledOOM 'Microsoft.Insights/metricAlerts@2018-03-0 metricNamespace: 'Insights.Container/pods' name: 'Metric1' operator: 'GreaterThan' - threshold: '0' + threshold: 0 timeAggregation: 'Average' skipMetricValidation: true } @@ -592,6 +606,7 @@ resource maHighPersistentVolumeUsage 'Microsoft.Insights/metricAlerts@2018-03-01 name: 'Persistent volume usage high for ${clusterName} CI-18' location: 'global' properties: { + autoMitigate: true actions: [] criteria: { allOf: [ @@ -617,7 +632,7 @@ resource maHighPersistentVolumeUsage 'Microsoft.Insights/metricAlerts@2018-03-01 metricNamespace: 'Insights.Container/persistentvolumes' name: 'Metric1' operator: 'GreaterThan' - threshold: '80' + threshold: 80 timeAggregation: 'Average' skipMetricValidation: true } @@ -643,6 +658,7 @@ resource maPodsNotInReadyState 'Microsoft.Insights/metricAlerts@2018-03-01' = { name: 'Pods not in ready state for ${clusterName} CI-8' location: 'global' properties: { + autoMitigate: true actions: [] criteria: { allOf: [ @@ -668,7 +684,7 @@ resource maPodsNotInReadyState 'Microsoft.Insights/metricAlerts@2018-03-01' = { metricNamespace: 'Insights.Container/pods' name: 'Metric1' operator: 'LessThan' - threshold: '80' + threshold: 80 timeAggregation: 'Average' skipMetricValidation: true } @@ -694,6 +710,7 @@ resource maRestartingContainerCount 'Microsoft.Insights/metricAlerts@2018-03-01' name: 'Restarting container count for ${clusterName} CI-7' location: 'global' properties: { + autoMitigate: true actions: [] criteria: { allOf: [ @@ -719,7 +736,7 @@ resource maRestartingContainerCount 'Microsoft.Insights/metricAlerts@2018-03-01' metricNamespace: 'Insights.Container/pods' name: 'Metric1' operator: 'GreaterThan' - threshold: '0' + threshold: 0 timeAggregation: 'Average' skipMetricValidation: true } @@ -745,6 +762,8 @@ resource skva 'Microsoft.OperationsManagement/solutions@2015-11-01-preview' = { name: 'KeyVaultAnalytics(${logAnalyticsWorkspaceName})' location: location properties: { + containedResources: [] + referencedResources: [] workspaceResourceId: resourceId('Microsoft.OperationalInsights/workspaces', logAnalyticsWorkspaceName) } plan: { @@ -753,12 +772,15 @@ resource skva 'Microsoft.OperationsManagement/solutions@2015-11-01-preview' = { promotionCode: '' publisher: 'Microsoft' } + dependsOn: [] } resource sqrPodFailed 'Microsoft.Insights/scheduledQueryRules@2018-04-16' = { name: 'PodFailedScheduledQuery' location: location properties: { + autoMitigate: true + displayName: '[${clusterName}] Scheduled Query for Pod Failed Alert' description: 'Alert on pod Failed phase.' enabled: 'true' source: { @@ -772,7 +794,7 @@ resource sqrPodFailed 'Microsoft.Insights/scheduledQueryRules@2018-04-16' = { } action: { 'odata.type': 'Microsoft.WindowsAzure.Management.Monitoring.Alerts.Models.Microsoft.AppInsights.Nexus.DataContracts.Resources.ScheduledQueryRules.AlertingAction' - severity: 3 + severity: '3' trigger: { thresholdOperator: 'GreaterThan' threshold: 3 @@ -784,6 +806,7 @@ resource sqrPodFailed 'Microsoft.Insights/scheduledQueryRules@2018-04-16' = { } } } + dependsOn: [] } resource paAKSLinuxRestrictive 'Microsoft.Authorization/policyAssignments@2021-06-01' = { @@ -806,10 +829,12 @@ resource paAKSLinuxRestrictive 'Microsoft.Authorization/policyAssignments@2021-0 } } } + dependsOn: [] } resource paEnforceHttpsIngress 'Microsoft.Authorization/policyAssignments@2021-06-01' = { name: policyAssignmentNameEnforceHttpsIngress + location: 'global' properties: { displayName: '[${clusterName}] ${reference(policyResourceIdEnforceHttpsIngress, '2020-09-01').displayName}' scope: subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name) @@ -823,6 +848,7 @@ resource paEnforceHttpsIngress 'Microsoft.Authorization/policyAssignments@2021-0 } } } + dependsOn: [] } resource paEnforceInternalLoadBalancers 'Microsoft.Authorization/policyAssignments@2021-06-01' = { @@ -840,6 +866,7 @@ resource paEnforceInternalLoadBalancers 'Microsoft.Authorization/policyAssignmen } } } + dependsOn: [] } resource paRoRootFilesystem 'Microsoft.Authorization/policyAssignments@2021-06-01' = { @@ -861,10 +888,12 @@ resource paRoRootFilesystem 'Microsoft.Authorization/policyAssignments@2021-06-0 } } } + dependsOn: [] } resource paEnforceResourceLimits 'Microsoft.Authorization/policyAssignments@2021-06-01' = { name: policyAssignmentNameEnforceResourceLimits + dependsOn: [] properties: { displayName: '[${clusterName}] ${reference(policyResourceIdEnforceResourceLimits, '2020-09-01').displayName}' scope: subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name) @@ -914,6 +943,7 @@ resource paEnforceImageSource 'Microsoft.Authorization/policyAssignments@2021-06 } } } + dependsOn: [] } resource paEnforceDefenderInCluster 'Microsoft.Authorization/policyAssignments@2021-06-01' = { @@ -929,6 +959,7 @@ resource paEnforceDefenderInCluster 'Microsoft.Authorization/policyAssignments@2 } } } + dependsOn: [] } resource miClusterControlPlane 'Microsoft.ManagedIdentity/userAssignedIdentities@2018-11-30' = { @@ -967,7 +998,11 @@ resource kv 'Microsoft.KeyVault/vaults@2021-11-01-preview' = { enabledForDiskEncryption: false enabledForTemplateDeployment: false enableSoftDelete: true - } + softDeleteRetentionInDays: 7 + enablePurgeProtection: false + createMode: 'default' + publicNetworkAccess: 'disabled' + } dependsOn: [ miAppGatewayFrontend podmiIngressController @@ -1006,6 +1041,7 @@ resource kv_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-0 } ] } + dependsOn: [] } resource kvMiAppGatewayFrontendSecretsUserRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { @@ -1016,6 +1052,7 @@ resource kvMiAppGatewayFrontendSecretsUserRole_roleAssignment 'Microsoft.Authori principalId: miAppGatewayFrontend.properties.principalId principalType: 'ServicePrincipal' } + dependsOn: [] } resource kvMiAppGatewayFrontendKeyVaultReader_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { @@ -1026,6 +1063,7 @@ resource kvMiAppGatewayFrontendKeyVaultReader_roleAssignment 'Microsoft.Authoriz principalId: miAppGatewayFrontend.properties.principalId principalType: 'ServicePrincipal' } + dependsOn: [] } resource kvPodMiIngressControllerSecretsUserRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { @@ -1036,6 +1074,7 @@ resource kvPodMiIngressControllerSecretsUserRole_roleAssignment 'Microsoft.Autho principalId: podmiIngressController.properties.principalId principalType: 'ServicePrincipal' } + dependsOn: [] } resource kvPodMiIngressControllerKeyVaultReader_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { @@ -1046,6 +1085,7 @@ resource kvPodMiIngressControllerKeyVaultReader_roleAssignment 'Microsoft.Author principalId: podmiIngressController.properties.principalId principalType: 'ServicePrincipal' } + dependsOn: [] } module ndEnsureClusterIdentityHasRbacToSelfManagedResources 'nested_EnsureClusterIdentityHasRbacToSelfManagedResources.bicep' = { @@ -1061,7 +1101,6 @@ module ndEnsureClusterIdentityHasRbacToSelfManagedResources 'nested_EnsureCluste resource pdzKv 'Microsoft.Network/privateDnsZones@2020-06-01' = { name: 'privatelink.vaultcore.azure.net' location: 'global' - resource vnetlnk 'virtualNetworkLinks' = { name: 'to_${vnetName}' location: 'global' @@ -1072,6 +1111,7 @@ resource pdzKv 'Microsoft.Network/privateDnsZones@2020-06-01' = { registrationEnabled: false } } + dependsOn: [] } resource peKv 'Microsoft.Network/privateEndpoints@2021-05-01' = { @@ -1093,6 +1133,7 @@ resource peKv 'Microsoft.Network/privateEndpoints@2021-05-01' = { } ] } + dependsOn: [] resource pdnszg 'privateDnsZoneGroups' = { name: 'default' @@ -1112,6 +1153,7 @@ resource peKv 'Microsoft.Network/privateEndpoints@2021-05-01' = { resource pdzAksIngress 'Microsoft.Network/privateDnsZones@2020-06-01' = { name: aksIngressDomainName location: 'global' + dependsOn: [] resource aksIngressDomainName_bu0001a0008_00 'A' = { name: 'bu0001a0008-00' @@ -1293,6 +1335,7 @@ resource mc 'Microsoft.ContainerService/managedClusters@2022-01-02-preview' = { oidcIssuerProfile: { enabled: true } + enableNamespaceResources: false } identity: { type: 'UserAssigned' @@ -1336,6 +1379,7 @@ resource acrKubeletAcrPullRole_roleAssignment 'Microsoft.Authorization/roleAssig principalId: reference(mc.id, '2020-12-01').identityProfile.kubeletidentity.objectId principalType: 'ServicePrincipal' } + dependsOn: [] } resource mcOmsAgentMonitoringMetricsPublisherRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { @@ -1346,6 +1390,7 @@ resource mcOmsAgentMonitoringMetricsPublisherRole_roleAssignment 'Microsoft.Auth principalId: reference(mc.id, '2020-12-01').addonProfiles.omsagent.identity.objectId principalType: 'ServicePrincipal' } + dependsOn: [] } resource miKubeletManagedIdentityOperatorRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { @@ -1356,6 +1401,7 @@ resource miKubeletManagedIdentityOperatorRole_roleAssignment 'Microsoft.Authoriz principalId: reference(mc.id, '2020-12-01').identityProfile.kubeletidentity.objectId principalType: 'ServicePrincipal' } + dependsOn: [] } resource mcAadAdminGroupClusterAdminRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = if (isUsingAzureRBACasKubernetesRBAC) { @@ -1367,6 +1413,7 @@ resource mcAadAdminGroupClusterAdminRole_roleAssignment 'Microsoft.Authorization principalId: clusterAdminAadGroupObjectId principalType: 'Group' } + dependsOn: [] } resource mcAadAdminGroupServiceClusterUserRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = if (isUsingAzureRBACasKubernetesRBAC) { @@ -1378,6 +1425,7 @@ resource mcAadAdminGroupServiceClusterUserRole_roleAssignment 'Microsoft.Authori principalId: clusterAdminAadGroupObjectId principalType: 'Group' } + dependsOn: [] } resource maAadA0008ReaderGroupClusterReaderRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = if (isUsingAzureRBACasKubernetesRBAC && (!(a0008NamespaceReaderAadGroupObjectId == clusterAdminAadGroupObjectId))) { @@ -1389,6 +1437,7 @@ resource maAadA0008ReaderGroupClusterReaderRole_roleAssignment 'Microsoft.Author description: 'Members of this group are cluster admins of the a0008 namespace in this cluster.' principalType: 'Group' } + dependsOn: [] } resource maAadA0008ReaderGroupServiceClusterUserRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = if (isUsingAzureRBACasKubernetesRBAC && (!(a0008NamespaceReaderAadGroupObjectId == clusterAdminAadGroupObjectId))) { @@ -1400,6 +1449,7 @@ resource maAadA0008ReaderGroupServiceClusterUserRole_roleAssignment 'Microsoft.A description: 'Members of this group are cluster users of this cluster.' principalType: 'Group' } + dependsOn: [] } resource mc_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { @@ -1426,6 +1476,7 @@ resource mc_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-0 } ] } + dependsOn: [] } resource mcFlux_extension 'Microsoft.KubernetesConfiguration/extensions@2021-09-01' = { @@ -1501,6 +1552,7 @@ module ndEnsureClusterUserAssignedHasRbacToManageVMSS 'nested_EnsureClusterUserA params: { kubeletidentityObjectId: reference(mc.id, '2020-03-01').identityProfile.kubeletidentity.objectId } + dependsOn: [] } resource st 'Microsoft.EventGrid/systemTopics@2021-12-01' = { @@ -1510,6 +1562,7 @@ resource st 'Microsoft.EventGrid/systemTopics@2021-12-01' = { source: mc.id topicType: 'Microsoft.ContainerService.ManagedClusters' } + dependsOn: [] } resource st_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { @@ -1530,6 +1583,7 @@ resource st_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-0 } ] } + dependsOn: [] } resource agw 'Microsoft.Network/applicationGateways@2021-05-01' = { @@ -1725,6 +1779,7 @@ resource agwdiagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01 } ] } + dependsOn: [] } output aksClusterName string = clusterName From de0dd6cef8c3ed15f553fd615bb1bc2a69a15b31 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Tue, 19 Apr 2022 17:39:05 +0000 Subject: [PATCH 29/54] final touches --- 06-aks-cluster.md | 4 +- cluster-stamp.bicep | 2 - cluster-stamp.json | 2035 ------------------------------- github-workflow/aks-deploy.yaml | 8 +- 4 files changed, 6 insertions(+), 2043 deletions(-) delete mode 100644 cluster-stamp.json diff --git a/06-aks-cluster.md b/06-aks-cluster.md index 98779958..14498870 100644 --- a/06-aks-cluster.md +++ b/06-aks-cluster.md @@ -19,7 +19,7 @@ Now that your [ACR instance is deployed and ready to support cluster bootstrappi ```bash # [This takes about 18 minutes.] - az deployment group create -g rg-bu0001a0008 -f cluster-stamp.json -p targetVnetResourceId=${RESOURCEID_VNET_CLUSTERSPOKE_AKS_BASELINE} clusterAdminAadGroupObjectId=${AADOBJECTID_GROUP_CLUSTERADMIN_AKS_BASELINE} a0008NamespaceReaderAadGroupObjectId=${AADOBJECTID_GROUP_A0008_READER_AKS_BASELINE} k8sControlPlaneAuthorizationTenantId=${TENANTID_K8SRBAC_AKS_BASELINE} appGatewayListenerCertificate=${APP_GATEWAY_LISTENER_CERTIFICATE_AKS_BASELINE} aksIngressControllerCertificate=${AKS_INGRESS_CONTROLLER_CERTIFICATE_BASE64_AKS_BASELINE} domainName=${DOMAIN_NAME_AKS_BASELINE} gitOpsBootstrappingRepoHttpsUrl=${GITOPS_REPOURL} + az deployment group create -g rg-bu0001a0008 -f cluster-stamp.bicep -p targetVnetResourceId=${RESOURCEID_VNET_CLUSTERSPOKE_AKS_BASELINE} clusterAdminAadGroupObjectId=${AADOBJECTID_GROUP_CLUSTERADMIN_AKS_BASELINE} a0008NamespaceReaderAadGroupObjectId=${AADOBJECTID_GROUP_A0008_READER_AKS_BASELINE} k8sControlPlaneAuthorizationTenantId=${TENANTID_K8SRBAC_AKS_BASELINE} appGatewayListenerCertificate=${APP_GATEWAY_LISTENER_CERTIFICATE_AKS_BASELINE} aksIngressControllerCertificate=${AKS_INGRESS_CONTROLLER_CERTIFICATE_BASE64_AKS_BASELINE} domainName=${DOMAIN_NAME_AKS_BASELINE} gitOpsBootstrappingRepoHttpsUrl=${GITOPS_REPOURL} ``` > Alteratively, you could have updated the [`azuredeploy.parameters.prod.json`](./azuredeploy.parameters.prod.json) file and deployed as above, using `-p "@azuredeploy.parameters.prod.json"` instead of providing the individual key-value pairs. @@ -117,7 +117,7 @@ Now that your [ACR instance is deployed and ready to support cluster bootstrappi ## Container registry note -:warning: To aid in ease of deployment of this cluster and your experimentation with workloads, Azure Policy and Azure Firewall are currently configured to allow your cluster to pull images from _public container registries_ such as Docker Hub. For a production system, you'll want to update Azure Policy parameter named `allowedContainerImagesRegex` in your `cluster-stamp.json` file to only list those container registries that you are willing to take a dependency on and what namespaces those policies apply to, and make Azure Firewall allowances for the same. This will protect your cluster from unapproved registries being used, which may prevent issues while trying to pull images from a registry which doesn't provide SLA guarantees for your deployment. +:warning: To aid in ease of deployment of this cluster and your experimentation with workloads, Azure Policy and Azure Firewall are currently configured to allow your cluster to pull images from _public container registries_ such as Docker Hub. For a production system, you'll want to update Azure Policy parameter named `allowedContainerImagesRegex` in your `cluster-stamp.bicep` file to only list those container registries that you are willing to take a dependency on and what namespaces those policies apply to, and make Azure Firewall allowances for the same. This will protect your cluster from unapproved registries being used, which may prevent issues while trying to pull images from a registry which doesn't provide SLA guarantees for your deployment. This deployment creates an SLA-backed Azure Container Registry for your cluster's needs. Your organization may have a central container registry for you to use, or your registry may be tied specifically to your application's infrastructure (as demonstrated in this implementation). **Only use container registries that satisfy the security and availability needs of your application.** diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 4a4c70c0..c5fd0573 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -52,7 +52,6 @@ param gitOpsBootstrappingRepoHttpsUrl string = 'https://github.com/mspnp/aks-bas @minLength(1) param gitOpsBootstrappingRepoBranch string = 'main' -var networkContributorRole = '${subscription().id}/providers/Microsoft.Authorization/roleDefinitions/4d97b98b-1d4f-4787-a291-c67834d212e7' var monitoringMetricsPublisherRole = '${subscription().id}/providers/Microsoft.Authorization/roleDefinitions/3913510d-42f4-4e42-8a64-420c390055eb' var acrPullRole = '${subscription().id}/providers/Microsoft.Authorization/roleDefinitions/7f951dda-4ed3-4680-a7ca-43fe172d538d' var managedIdentityOperatorRole = '${subscription().id}/providers/Microsoft.Authorization/roleDefinitions/f1a07417-d97a-45cb-824c-7a7467783830' @@ -72,7 +71,6 @@ var vNetResourceGroup = split(targetVnetResourceId, '/')[4] var vnetName = split(targetVnetResourceId, '/')[8] var vnetNodePoolSubnetResourceId = '${targetVnetResourceId}/subnets/snet-clusternodes' var vnetPrivateLinkEndpointsSubnetResourceId = '${targetVnetResourceId}/subnets/snet-privatelinkendpoints' -var vnetIngressServicesSubnetResourceId = '${targetVnetResourceId}/subnets/snet-cluster-ingressservices' var agwName = 'apw-${clusterName}' diff --git a/cluster-stamp.json b/cluster-stamp.json deleted file mode 100644 index 43a9d448..00000000 --- a/cluster-stamp.json +++ /dev/null @@ -1,2035 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "0.0.0.5", - "parameters": { - "targetVnetResourceId": { - "type": "string", - "minLength": 79, - "metadata": { - "description": "The regional network spoke VNet Resource ID that the cluster will be joined to" - } - }, - "clusterAdminAadGroupObjectId": { - "type": "string", - "metadata": { - "description": "Azure AD Group in the identified tenant that will be granted the highly privileged cluster-admin role. If Azure RBAC is used, then this group will get a role assignment to Azure RBAC, else it will be assigned directly to the cluster's admin group." - } - }, - "a0008NamespaceReaderAadGroupObjectId": { - "type": "string", - "metadata": { - "description": "Azure AD Group in the identified tenant that will be granted the read only privileges in the a0008 namespace that exists in the cluster. This is only used when Azure RBAC is used for Kubernetes RBAC." - } - }, - "k8sControlPlaneAuthorizationTenantId": { - "type": "string", - "metadata": { - "description": "Your AKS control plane Cluster API authentication tenant" - } - }, - "appGatewayListenerCertificate": { - "type": "string", - "metadata": { - "description": "The certificate data for app gateway TLS termination. It is base64" - } - }, - "aksIngressControllerCertificate": { - "type": "string", - "metadata": { - "description": "The Base64 encoded AKS Ingress Controller public certificate (as .crt or .cer) to be stored in Azure Key Vault as secret and referenced by Azure Application Gateway as a trusted root certificate." - } - }, - "clusterAuthorizedIPRanges": { - "type": "array", - "defaultValue": [], - "metadata": { - "description": "IP ranges authorized to contact the Kubernetes API server. Passing an empty array will result in no IP restrictions. If any are provided, remember to also provide the public IP of the egress Azure Firewall otherwise your nodes will not be able to talk to the API server (e.g. Flux)." - } - }, - "location": { - "defaultValue": "eastus2", - "type": "string", - "allowedValues": [ - "australiaeast", - "canadacentral", - "centralus", - "eastus", - "eastus2", - "westus2", - "francecentral", - "germanywestcentral", - "northeurope", - "southafricanorth", - "southcentralus", - "uksouth", - "westeurope", - "japaneast", - "southeastasia" - ], - "metadata": { - "description": "AKS Service, Node Pool, and supporting services (KeyVault, App Gateway, etc) region. This needs to be the same region as the vnet provided in these parameters." - } - }, - "kubernetesVersion": { - "defaultValue": "1.22.4", - "type": "string" - }, - "domainName": { - "type": "string", - "defaultValue": "contoso.com", - "metadata": { - "description": "Domain name to use for App Gateway and AKS ingress." - } - }, - "gitOpsBootstrappingRepoHttpsUrl": { - "type": "string", - "defaultValue": "https://github.com/mspnp/aks-baseline", - "minLength": 9, - "metadata": { - "description": "Your cluster will be bootstrapped from this git repo." - } - }, - "gitOpsBootstrappingRepoBranch": { - "type": "string", - "defaultValue": "main", - "minLength": 1, - "metadata": { - "description": "You cluster will be bootstrapped from this branch in the identifed git repo." - } - } - }, - "variables": { - "networkContributorRole": "[concat(subscription().Id, '/providers/Microsoft.Authorization/roleDefinitions/4d97b98b-1d4f-4787-a291-c67834d212e7')]", - "monitoringMetricsPublisherRole": "[concat(subscription().Id, '/providers/Microsoft.Authorization/roleDefinitions/3913510d-42f4-4e42-8a64-420c390055eb')]", - "acrPullRole": "[concat(subscription().Id, '/providers/Microsoft.Authorization/roleDefinitions/7f951dda-4ed3-4680-a7ca-43fe172d538d')]", - "managedIdentityOperatorRole": "[concat(subscription().Id, '/providers/Microsoft.Authorization/roleDefinitions/f1a07417-d97a-45cb-824c-7a7467783830')]", - "virtualMachineContributorRole": "[concat(subscription().Id, '/providers/Microsoft.Authorization/roleDefinitions/9980e02c-c2be-4d73-94e8-173b1dc7cf3c')]", - "keyVaultReader": "[concat(subscription().Id, '/providers/Microsoft.Authorization/roleDefinitions/21090545-7ca7-4776-b22c-e363652d74d2')]", - "keyVaultSecretsUserRole": "[concat(subscription().Id, '/providers/Microsoft.Authorization/roleDefinitions/4633458b-17de-408a-b874-0445c86b69e6')]", - "clusterAdminRoleId": "b1ff04bb-8a4e-4dc4-8eb5-8693973ce19b", - "clusterReaderRoleId": "7f6c6a51-bcf8-42ba-9220-52d62157d7db", - "serviceClusterUserRoleId": "4abbcc35-e782-43d8-92c5-2d3f1bd2253f", - "subRgUniqueString": "[uniqueString('aks', subscription().subscriptionId, resourceGroup().id)]", - - "nodeResourceGroupName": "[concat('rg-', variables('clusterName'), '-nodepools')]", - "clusterName": "[concat('aks-', variables('subRgUniqueString'))]", - "logAnalyticsWorkspaceName": "[concat('la-', variables('clusterName'))]", - "containerInsightsSolutionName": "[concat('ContainerInsights(', variables('logAnalyticsWorkspaceName'),')')]", - "defaultAcrName": "[concat('acraks', variables('subRgUniqueString'))]", - - "vNetResourceGroup": "[split(parameters('targetVnetResourceId'),'/')[4]]", - "vnetName": "[split(parameters('targetVnetResourceId'),'/')[8]]", - "vnetNodePoolSubnetResourceId": "[concat(parameters('targetVnetResourceId'), '/subnets/snet-clusternodes')]", - "vnetPrivateLinkEndpointsSubnetResourceId": "[concat(parameters('targetVnetResourceId'), '/subnets/snet-privatelinkendpoints')]", - "vnetIngressServicesSubnetResourceId": "[concat(parameters('targetVnetResourceId'), '/subnets/snet-cluster-ingressservices')]", - - "agwName": "[concat('apw-', variables('clusterName'))]", - - "akvPrivateDnsZonesName": "privatelink.vaultcore.azure.net", - - "clusterControlPlaneIdentityName": "[concat('mi-', variables('clusterName'), '-controlplane')]", - - "keyVaultName": "[concat('kv-', variables('clusterName'))]", - - "aksIngressDomainName": "[concat('aks-ingress.', parameters('domainName'))]", - "aksBackendDomainName": "[concat('bu0001a0008-00.', variables('aksIngressDomainName'))]", - - "policyResourceIdAKSLinuxRestrictive": "/providers/Microsoft.Authorization/policySetDefinitions/42b8ef37-b724-4e24-bbc8-7a7708edfe00", - "policyResourceIdEnforceHttpsIngress": "/providers/Microsoft.Authorization/policyDefinitions/1a5b4dca-0b6f-4cf5-907c-56316bc1bf3d", - "policyResourceIdEnforceInternalLoadBalancers": "/providers/Microsoft.Authorization/policyDefinitions/3fc4dc25-5baf-40d8-9b05-7fe74c1bc64e", - "policyResourceIdRoRootFilesystem": "/providers/Microsoft.Authorization/policyDefinitions/df49d893-a74c-421d-bc95-c663042e5b80", - "policyResourceIdEnforceResourceLimits": "/providers/Microsoft.Authorization/policyDefinitions/e345eecc-fa47-480f-9e88-67dcc122b164", - "policyResourceIdEnforceImageSource": "/providers/Microsoft.Authorization/policyDefinitions/febd0533-8e55-448f-b837-bd0e06f16469", - "policyResourceIdEnforceDefenderInCluster": "/providers/Microsoft.Authorization/policyDefinitions/a1840de2-8088-4ea8-b153-b4c723e9cb01", - "policyAssignmentNameAKSLinuxRestrictive": "[guid(variables('policyResourceIdAKSLinuxRestrictive'), resourceGroup().name, variables('clusterName'))]", - "policyAssignmentNameEnforceHttpsIngress": "[guid(variables('policyResourceIdEnforceHttpsIngress'), resourceGroup().name, variables('clusterName'))]", - "policyAssignmentNameEnforceInternalLoadBalancers": "[guid(variables('policyResourceIdEnforceInternalLoadBalancers'), resourceGroup().name, variables('clusterName'))]", - "policyAssignmentNameRoRootFilesystem": "[guid(variables('policyResourceIdRoRootFilesystem'), resourceGroup().name, variables('clusterName'))]", - "policyAssignmentNameEnforceResourceLimits": "[guid(variables('policyResourceIdEnforceResourceLimits'), resourceGroup().name, variables('clusterName'))]", - "policyAssignmentNameEnforceImageSource": "[guid(variables('policyResourceIdEnforceImageSource'), resourceGroup().name, variables('clusterName'))]", - "policyAssignmentNameEnforceDefenderInCluster": "[guid(variables('policyResourceIdEnforceDefenderInCluster'), resourceGroup().name, variables('clusterName'))]", - "isUsingAzureRBACasKubernetesRBAC": "[equals(subscription().tenantId, parameters('k8sControlPlaneAuthorizationTenantId'))]" - }, - "resources": [ - { - "name": "[variables('clusterControlPlaneIdentityName')]", - "type": "Microsoft.ManagedIdentity/userAssignedIdentities", - "apiVersion": "2018-11-30", - "location": "[parameters('location')]", - "comments": "The control plane identity used by the cluster. Used for networking access (VNET joining and DNS updating)" - }, - { - "type": "Microsoft.ManagedIdentity/userAssignedIdentities", - "apiVersion": "2018-11-30", - "name": "mi-appgateway-frontend", - "location": "[parameters('location')]", - "comments": "User Managed Identity that App Gateway is assigned. Used for Azure Key Vault Access." - }, - { - "type": "Microsoft.ManagedIdentity/userAssignedIdentities", - "apiVersion": "2018-11-30", - "name": "podmi-ingress-controller", - "location": "[parameters('location')]", - "comments": "User Managed Identity for the cluster's ingress controller pods. Used for Azure Key Vault Access." - }, - { - "type": "Microsoft.KeyVault/vaults", - "apiVersion": "2021-06-01-preview", - "name": "[variables('keyVaultName')]", - "location": "[parameters('location')]", - "dependsOn": [ - "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', 'mi-appgateway-frontend')]", - "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', 'podmi-ingress-controller')]" - ], - "properties": { - "accessPolicies": [], // Azure RBAC is used instead - "sku": { - "family": "A", - "name": "standard" - }, - "tenantId": "[subscription().tenantId]", - "networkAcls": { - "bypass": "AzureServices", // Required for AppGW communication - "defaultAction": "Deny", - "ipRules": [], - "virtualNetworkRules": [] - }, - "enableRbacAuthorization": true, - "enabledForDeployment": false, - "enabledForDiskEncryption": false, - "enabledForTemplateDeployment": false, - "enableSoftDelete": true - }, - "resources": [ - { - "type": "secrets", - "apiVersion": "2021-06-01-preview", - "name": "gateway-public-cert", - "dependsOn": [ - "[resourceId('Microsoft.KeyVault/vaults', variables('keyVaultName') )]" - ], - "properties": { - "value": "[parameters('appGatewayListenerCertificate')]" - } - }, - { - "type": "secrets", - "apiVersion": "2021-06-01-preview", - "name": "appgw-ingress-internal-aks-ingress-tls", - "dependsOn": [ - "[resourceId('Microsoft.KeyVault/vaults', variables('keyVaultName'))]" - ], - "properties": { - "value": "[parameters('aksIngressControllerCertificate')]" - } - }, - { - "type": "providers/diagnosticSettings", - "apiVersion": "2021-05-01-preview", - "name": "Microsoft.Insights/default", - "dependsOn": [ - "[resourceId('Microsoft.KeyVault/vaults', variables('keyVaultName'))]" - ], - "properties": { - "workspaceId": "[resourceId('Microsoft.OperationalInsights/workspaces', variables('logAnalyticsWorkspaceName'))]", - "logs": [ - { - "category": "AuditEvent", - "enabled": true - } - ], - "metrics": [ - { - "category": "AllMetrics", - "enabled": true - } - ] - } - }, - { - "type": "providers/roleAssignments", - "apiVersion": "2020-04-01-preview", - "name": "[concat('Microsoft.Authorization/', guid(resourceGroup().id, 'mi-appgateway-frontend', variables('keyVaultSecretsUserRole')))]", - "dependsOn": [ - "[resourceId('Microsoft.KeyVault/vaults', variables('keyVaultName'))]", - "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', 'mi-appgateway-frontend')]" - ], - "comments": "Grant the Azure Application Gateway managed identity with key vault reader role permissions; this allows pulling frontend and backend certificates.", - "properties": { - "roleDefinitionId": "[variables('keyVaultSecretsUserRole')]", - "principalId": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', 'mi-appgateway-frontend')).principalId]", - "principalType": "ServicePrincipal" - } - }, - { - "type": "providers/roleAssignments", - "apiVersion": "2020-04-01-preview", - "name": "[concat('Microsoft.Authorization/', guid(resourceGroup().id, 'mi-appgateway-frontend', variables('keyVaultReader')))]", - "dependsOn": [ - "[resourceId('Microsoft.KeyVault/vaults', variables('keyVaultName'))]", - "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', 'mi-appgateway-frontend')]" - ], - "comments": "Grant the Azure Application Gateway managed identity with key vault reader role permissions; this allows pulling frontend and backend certificates.", - "properties": { - "roleDefinitionId": "[variables('keyVaultReader')]", - "principalId": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', 'mi-appgateway-frontend')).principalId]", - "principalType": "ServicePrincipal" - } - }, - { - "type": "providers/roleAssignments", - "apiVersion": "2020-04-01-preview", - "name": "[concat('Microsoft.Authorization/', guid(resourceGroup().id, 'podmi-ingress-controller', variables('keyVaultSecretsUserRole')))]", - "dependsOn": [ - "[resourceId('Microsoft.KeyVault/vaults', variables('keyVaultName'))]", - "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', 'podmi-ingress-controller')]" - ], - "comments": "Grant the AKS cluster ingress controller pod managed identity with key vault reader role permissions; this allows our ingress controller to pull certificates.", - "properties": { - "roleDefinitionId": "[variables('keyVaultSecretsUserRole')]", - "principalId": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', 'podmi-ingress-controller')).principalId]", - "principalType": "ServicePrincipal" - } - }, - { - "type": "providers/roleAssignments", - "apiVersion": "2020-04-01-preview", - "name": "[concat('Microsoft.Authorization/', guid(resourceGroup().id, 'podmi-ingress-controller', variables('keyVaultReader')))]", - "dependsOn": [ - "[resourceId('Microsoft.KeyVault/vaults', variables('keyVaultName'))]", - "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', 'podmi-ingress-controller')]" - ], - "comments": "Grant the AKS cluster ingress controller pod managed identity with key vault reader role permissions; this allows our ingress controller to pull certificates.", - "properties": { - "roleDefinitionId": "[variables('keyVaultReader')]", - "principalId": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', 'podmi-ingress-controller')).principalId]", - "principalType": "ServicePrincipal" - } - } - ] - }, - { - "type": "Microsoft.Network/privateEndpoints", - "apiVersion": "2020-05-01", - "name": "[concat('pe-', variables('keyVaultName'))]", - "location": "[parameters('location')]", - "dependsOn": [ - "[resourceId('Microsoft.KeyVault/vaults', variables('keyVaultName'))]" - ], - "properties": { - "subnet": { - "id": "[variables('vnetPrivateLinkEndpointsSubnetResourceId')]" - }, - "privateLinkServiceConnections": [ - { - "name": "[concat('to_', variables('vnetName'))]", - "properties": { - "privateLinkServiceId": "[resourceId('Microsoft.KeyVault/vaults', variables('keyVaultName'))]", - "groupIds": [ - "vault" - ] - } - } - ] - }, - "resources": [ - { - "type": "privateDnsZoneGroups", - "apiVersion": "2020-05-01", - "name": "default", - "location": "[parameters('location')]", - "dependsOn": [ - "[resourceId('Microsoft.Network/privateEndpoints', concat('pe-', variables('keyVaultName')))]", - "[resourceId('Microsoft.Network/privateDnsZones', variables('akvPrivateDnsZonesName'))]" - ], - "properties": { - "privateDnsZoneConfigs": [ - { - "name": "privatelink-akv-net", - "properties": { - "privateDnsZoneId": "[resourceId('Microsoft.Network/privateDnsZones', variables('akvPrivateDnsZonesName'))]" - } - } - ] - } - } - ] - }, - { - "type": "Microsoft.Network/privateDnsZones", - "apiVersion": "2018-09-01", - "name": "[variables('akvPrivateDnsZonesName')]", - "location": "global", - "comments": "Enabling Azure Key Vault Private Link support.", - "properties": {}, - "resources": [ - { - "type": "virtualNetworkLinks", - "apiVersion": "2020-06-01", - "name": "[concat('to_', variables('vnetName'))]", - "location": "global", - "comments": "Enabling Azure Key Vault Private Link on cluster vnet.", - "dependsOn": [ - "[resourceId('Microsoft.Network/privateDnsZones', variables('akvPrivateDnsZonesName'))]" - ], - "properties": { - "virtualNetwork": { - "id": "[parameters('targetVnetResourceId')]" - }, - "registrationEnabled": false - } - } - ] - }, - { - "type": "Microsoft.Network/privateDnsZones", - "apiVersion": "2018-09-01", - "name": "[variables('aksIngressDomainName')]", - "location": "global", - "properties": {}, - "resources": [ - { - "type": "A", - "apiVersion": "2018-09-01", - "name": "bu0001a0008-00", - "dependsOn": [ - "[resourceId('Microsoft.Network/privateDnsZones', variables('aksIngressDomainName'))]" - ], - "properties": { - "ttl": 3600, - "aRecords": [ - { - "ipv4Address": "10.240.4.4" - } - ] - } - }, - { - "type": "virtualNetworkLinks", - "apiVersion": "2020-06-01", - "name": "[concat('to_', variables('vnetName'))]", - "location": "global", - "dependsOn": [ - "[resourceId('Microsoft.Network/privateDnsZones', variables('aksIngressDomainName'))]" - ], - "properties": { - "virtualNetwork": { - "id": "[parameters('targetVnetResourceId')]" - }, - "registrationEnabled": false - } - } - ] - }, - { - "type": "Microsoft.Network/applicationGateways", - "apiVersion": "2020-05-01", - "name": "[variables('agwName')]", - "location": "[parameters('location')]", - "dependsOn": [ - "[resourceId('Microsoft.KeyVault/vaults', variables('keyVaultName'))]", - "[resourceId('Microsoft.Network/privateEndpoints', concat('pe-', variables('keyVaultName')))]", - "[resourceId('Microsoft.KeyVault/vaults/providers/roleAssignments', variables('keyVaultName'), 'Microsoft.Authorization', guid(resourceGroup().id, 'mi-appgateway-frontend', variables('keyVaultReader')))]", - "[resourceId('Microsoft.KeyVault/vaults/providers/roleAssignments', variables('keyVaultName'), 'Microsoft.Authorization', guid(resourceGroup().id, 'mi-appgateway-frontend', variables('keyVaultSecretsUserRole')))]" - ], - "identity": { - "type": "UserAssigned", - "userAssignedIdentities": { - "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', 'mi-appgateway-frontend')]": {} - } - }, - "zones": "[pickZones('Microsoft.Network', 'applicationGateways', parameters('location'), 3)]", - "properties": { - "sku": { - "name": "WAF_v2", - "tier": "WAF_v2" - }, - "sslPolicy": { - "policyType": "Custom", - "cipherSuites": [ - "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", - "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256" - ], - "minProtocolVersion": "TLSv1_2" - }, - "trustedRootCertificates": [ - { - "name": "root-cert-wildcard-aks-ingress", - "properties": { - "keyVaultSecretId": "[concat(reference(variables('keyVaultName')).vaultUri,'secrets/appgw-ingress-internal-aks-ingress-tls')]" - } - } - ], - "gatewayIPConfigurations": [ - { - "name": "apw-ip-configuration", - "properties": { - "subnet": { - "id": "[concat(parameters('targetVnetResourceId'), '/subnets/snet-applicationgateway')]" - } - } - } - ], - "frontendIPConfigurations": [ - { - "name": "apw-frontend-ip-configuration", - "properties": { - "PublicIPAddress": { - "id": "[resourceId(subscription().subscriptionId, variables('vNetResourceGroup'), 'Microsoft.Network/publicIpAddresses', 'pip-BU0001A0008-00')]" - } - } - } - ], - "frontendPorts": [ - { - "name": "port-443", - "properties": { - "port": 443 - } - } - ], - "autoscaleConfiguration": { - "minCapacity": 0, - "maxCapacity": 10 - }, - "webApplicationFirewallConfiguration": { - "enabled": true, - "firewallMode": "Prevention", - "ruleSetType": "OWASP", - "ruleSetVersion": "3.2", - "exclusions": [], - "fileUploadLimitInMb": 10, - "disabledRuleGroups": [] - }, - "enableHttp2": false, - "sslCertificates": [ - { - "name": "[concat(variables('agwName'), '-ssl-certificate')]", - "properties": { - "keyVaultSecretId": "[concat(reference(variables('keyVaultName')).vaultUri,'secrets/gateway-public-cert')]" - } - } - ], - "probes": [ - { - "name": "[concat('probe-', variables('aksBackendDomainName'))]", - "properties": { - "protocol": "Https", - "path": "/favicon.ico", - "interval": 30, - "timeout": 30, - "unhealthyThreshold": 3, - "pickHostNameFromBackendHttpSettings": true, - "minServers": 0, - "match": { - } - } - } - ], - "backendAddressPools": [ - { - "name": "[variables('aksBackendDomainName')]", - "properties": { - "backendAddresses": [ - { - "fqdn": "[variables('aksBackendDomainName')]" - } - ] - } - } - ], - "backendHttpSettingsCollection": [ - { - "name": "aks-ingress-backendpool-httpsettings", - "properties": { - "port": 443, - "protocol": "Https", - "cookieBasedAffinity": "Disabled", - "pickHostNameFromBackendAddress": true, - "requestTimeout": 20, - "probe": { - "id": "[resourceId('Microsoft.Network/applicationGateways/probes', variables('agwName'), concat('probe-', variables('aksBackendDomainName')))]" - }, - "trustedRootCertificates": [ - { - "id": "[resourceId('Microsoft.Network/applicationGateways/trustedRootCertificates', variables('agwName'), 'root-cert-wildcard-aks-ingress')]" - } - ] - } - } - ], - "httpListeners": [ - { - "name": "listener-https", - "properties": { - "frontendIPConfiguration": { - "id": "[resourceId('Microsoft.Network/applicationGateways/frontendIPConfigurations', variables('agwName'), 'apw-frontend-ip-configuration')]" - }, - "frontendPort": { - "id": "[resourceId('Microsoft.Network/applicationGateways/frontendPorts', variables('agwName'), 'port-443')]" - }, - "protocol": "Https", - "sslCertificate": { - "id": "[resourceId('Microsoft.Network/applicationGateways/sslCertificates', variables('agwName'), concat(variables('agwName'), '-ssl-certificate'))]" - }, - "hostName": "[concat('bicycle.', parameters('domainName'))]", - "hostNames": [], - "requireServerNameIndication": true - } - } - ], - "requestRoutingRules": [ - { - "Name": "apw-routing-rules", - "properties": { - "RuleType": "Basic", - "httpListener": { - "id": "[resourceId('Microsoft.Network/applicationGateways/httpListeners', variables('agwName'), 'listener-https')]" - }, - "backendAddressPool": { - "id": "[resourceId('Microsoft.Network/applicationGateways/backendAddressPools', variables('agwName'), variables('aksBackendDomainName'))]" - }, - "backendHttpSettings": { - "id": "[resourceId('Microsoft.Network/applicationGateways/backendHttpSettingsCollection', variables('agwName'), 'aks-ingress-backendpool-httpsettings')]" - } - } - } - ] - }, - "resources": [ - { - "type": "providers/diagnosticSettings", - "apiVersion": "2021-05-01-preview", - "name": "Microsoft.Insights/default", - "dependsOn": [ - "[resourceId('Microsoft.Network/applicationGateways', variables('agwName'))]" - ], - "properties": { - "workspaceId": "[resourceId('Microsoft.OperationalInsights/workspaces', variables('logAnalyticsWorkspaceName'))]", - "logs": [ - { - "category": "ApplicationGatewayAccessLog", - "enabled": true - }, - { - "category": "ApplicationGatewayPerformanceLog", - "enabled": true - }, - { - "category": "ApplicationGatewayFirewallLog", - "enabled": true - } - ] - } - } - ] - }, - { - "type": "Microsoft.Resources/deployments", - "name": "EnsureClusterIdentityHasRbacToSelfManagedResources", - "apiVersion": "2020-06-01", - "dependsOn": [ - "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('clusterControlPlaneIdentityName'))]" - ], - "resourceGroup": "[variables('vNetResourceGroup')]", - "properties": { - "mode": "Incremental", - "template": { - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "resources": [ - { - "type": "Microsoft.Authorization/roleAssignments", - "apiVersion": "2020-04-01-preview", - "name": "[guid(variables('vnetNodePoolSubnetResourceId'), variables('networkContributorRole'), variables('clusterControlPlaneIdentityName'))]", - "scope": "[concat('Microsoft.Network/virtualNetworks/', variables('vnetName'), '/subnets/', 'snet-clusternodes')]", - "properties": { - "roleDefinitionId": "[variables('networkContributorRole')]", - "description": "Allows cluster identity to join the nodepool vmss resources to this subnet.", - "principalId": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('clusterControlPlaneIdentityName'))).principalId]", - "principalType": "ServicePrincipal" - } - }, - { - "type": "Microsoft.Authorization/roleAssignments", - "apiVersion": "2020-04-01-preview", - "name": "[guid(variables('vnetIngressServicesSubnetResourceId'), variables('networkContributorRole'), variables('clusterControlPlaneIdentityName'))]", - "scope": "[concat('Microsoft.Network/virtualNetworks/', variables('vnetName'), '/subnets/', 'snet-clusteringressservices')]", - "properties": { - "roleDefinitionId": "[variables('networkContributorRole')]", - "description": "Allows cluster identity to join load balancers (ingress resources) to this subnet.", - "principalId": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('clusterControlPlaneIdentityName'))).principalId]", - "principalType": "ServicePrincipal" - } - } - ] - } - } - }, - { - "type": "Microsoft.Resources/deployments", - "name": "EnsureClusterUserAssignedHasRbacToManageVMSS", - "apiVersion": "2017-05-10", - "dependsOn": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]" - ], - "resourceGroup": "[variables('nodeResourceGroupName')]", - "properties": { - "mode": "Incremental", - "template": { - "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "resources": [ - { - "type": "Microsoft.Authorization/roleAssignments", - "apiVersion": "2020-04-01-preview", - "name": "[guid(resourceGroup().id)]", - "comments": "It is required to grant the AKS cluster with Virtual Machine Contributor role permissions over the cluster infrastructure resource group to work with Managed Identities and aad-pod-identity. Otherwise MIC component fails while attempting to update MSI on VMSS cluster nodes", - "properties": { - "roleDefinitionId": "[variables('virtualMachineContributorRole')]", - "principalId": "[reference(resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName')), '2020-03-01').identityProfile.kubeletidentity.objectId]", - "principalType": "ServicePrincipal" - } - } - ] - } - } - }, - { - "type": "Microsoft.OperationalInsights/workspaces/savedSearches", - "apiVersion": "2020-08-01", - "name": "[concat(variables('logAnalyticsWorkspaceName'), '/AllPrometheus')]", - "properties": { - "eTag": "*", - "category": "Prometheus", - "displayName": "All collected Prometheus information", - "query": "InsightsMetrics | where Namespace == \"prometheus\"", - "version": 1 - } - }, - { - "type": "Microsoft.OperationalInsights/workspaces/savedSearches", - "apiVersion": "2020-08-01", - "name": "[concat(variables('logAnalyticsWorkspaceName'), '/NodeRebootRequested')]", - "properties": { - "eTag": "*", - "category": "Prometheus", - "displayName": "Nodes reboot required by kured", - "query": "InsightsMetrics | where Namespace == \"prometheus\" and Name == \"kured_reboot_required\" | where Val > 0", - "version": 1 - } - }, - { - "name": "PodFailedScheduledQuery", - "type": "Microsoft.Insights/scheduledQueryRules", - "apiVersion": "2018-04-16", - "location": "[parameters('location')]", - "dependsOn": [ - "[resourceId('Microsoft.OperationsManagement/solutions',variables('containerInsightsSolutionName'))]" - ], - "properties": { - "description": "Alert on pod Failed phase.", - "enabled": "true", - "source": { - "query": "[concat('//https://docs.microsoft.com/azure/azure-monitor/insights/container-insights-alerts \r\n let endDateTime = now(); let startDateTime = ago(1h); let trendBinSize = 1m; let clusterName = \"',variables('clusterName'),'\"; KubePodInventory | where TimeGenerated < endDateTime | where TimeGenerated >= startDateTime | where ClusterName == clusterName | distinct ClusterName, TimeGenerated | summarize ClusterSnapshotCount = count() by bin(TimeGenerated, trendBinSize), ClusterName | join hint.strategy=broadcast ( KubePodInventory | where TimeGenerated < endDateTime | where TimeGenerated >= startDateTime | distinct ClusterName, Computer, PodUid, TimeGenerated, PodStatus | summarize TotalCount = count(), PendingCount = sumif(1, PodStatus =~ \"Pending\"), RunningCount = sumif(1, PodStatus =~ \"Running\"), SucceededCount = sumif(1, PodStatus =~ \"Succeeded\"), FailedCount = sumif(1, PodStatus =~ \"Failed\") by ClusterName, bin(TimeGenerated, trendBinSize) ) on ClusterName, TimeGenerated | extend UnknownCount = TotalCount - PendingCount - RunningCount - SucceededCount - FailedCount | project TimeGenerated, TotalCount = todouble(TotalCount) / ClusterSnapshotCount, PendingCount = todouble(PendingCount) / ClusterSnapshotCount, RunningCount = todouble(RunningCount) / ClusterSnapshotCount, SucceededCount = todouble(SucceededCount) / ClusterSnapshotCount, FailedCount = todouble(FailedCount) / ClusterSnapshotCount, UnknownCount = todouble(UnknownCount) / ClusterSnapshotCount| summarize AggregatedValue = avg(FailedCount) by bin(TimeGenerated, trendBinSize)')]", - "dataSourceId": "[resourceId('Microsoft.OperationalInsights/workspaces', variables('logAnalyticsWorkspaceName'))]", - "queryType": "ResultCount" - }, - "schedule": { - "frequencyInMinutes": 5, - "timeWindowInMinutes": 10 - }, - "action": { - "odata.type": "Microsoft.WindowsAzure.Management.Monitoring.Alerts.Models.Microsoft.AppInsights.Nexus.DataContracts.Resources.ScheduledQueryRules.AlertingAction", - "severity": 3, - "trigger": { - "thresholdOperator": "GreaterThan", - "threshold": 3, - "metricTrigger": { - "thresholdOperator": "GreaterThan", - "threshold": 2, - "metricTriggerType": "Consecutive" - } - } - } - } - }, - { - "type": "microsoft.insights/activityLogAlerts", - "apiVersion": "2017-04-01", - "name": "AllAzureAdvisorAlert", - "location": "Global", - "properties": { - "scopes": [ - "[resourceGroup().id]" - ], - "condition": { - "allOf": [ - { - "field": "category", - "equals": "Recommendation" - }, - { - "field": "operationName", - "equals": "Microsoft.Advisor/recommendations/available/action" - } - ] - }, - "actions": { - "actionGroups": [ - ] - }, - "enabled": true, - "description": "All azure advisor alerts" - } - }, - { - "type": "Microsoft.OperationsManagement/solutions", - "apiVersion": "2015-11-01-preview", - "name": "[variables('containerInsightsSolutionName')]", - "location": "[parameters('location')]", - "properties": { - "workspaceResourceId": "[resourceId('Microsoft.OperationalInsights/workspaces', variables('logAnalyticsWorkspaceName'))]" - }, - "plan": { - "name": "[variables('containerInsightsSolutionName')]", - "product": "OMSGallery/ContainerInsights", - "promotionCode": "", - "publisher": "Microsoft" - } - }, - { - "type": "Microsoft.OperationsManagement/solutions", - "apiVersion": "2015-11-01-preview", - "name": "[concat('KeyVaultAnalytics(', variables('logAnalyticsWorkspaceName'),')')]", - "location": "[parameters('location')]", - "properties": { - "workspaceResourceId": "[resourceId('Microsoft.OperationalInsights/workspaces', variables('logAnalyticsWorkspaceName'))]" - }, - "plan": { - "name": "[concat('KeyVaultAnalytics(', variables('logAnalyticsWorkspaceName'),')')]", - "product": "OMSGallery/KeyVaultAnalytics", - "promotionCode": "", - "publisher": "Microsoft" - } - }, - { - "type": "Microsoft.Authorization/roleAssignments", - "apiVersion": "2020-04-01-preview", - "name": "[guid(resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName')), variables('acrPullRole'))]", - "dependsOn": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]" - ], - "scope": "[resourceId('Microsoft.ContainerRegistry/registries', variables('defaultAcrName'))]", - "properties": { - "roleDefinitionId": "[variables('acrPullRole')]", - "description": "Allows AKS to pull container images from this ACR instance.", - "principalId": "[reference(resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName')), '2020-12-01').identityProfile.kubeletidentity.objectId]", - "principalType": "ServicePrincipal" - } - }, - { - "type": "Microsoft.ContainerService/managedClusters", - "apiVersion": "2021-11-01-preview", - "name": "[variables('clusterName')]", - "location": "[parameters('location')]", - "tags": { - "Business unit": "BU0001", - "Application identifier": "a0008" - }, - "dependsOn": [ - "[resourceId('Microsoft.OperationsManagement/solutions', variables('containerInsightsSolutionName'))]", - "[resourceId(variables('vNetResourceGroup'), 'Microsoft.Resources/deployments', 'EnsureClusterIdentityHasRbacToSelfManagedResources')]", - "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('clusterControlPlaneIdentityName'))]", - "[resourceId('Microsoft.Authorization/policyAssignments', variables('policyAssignmentNameAKSLinuxRestrictive'))]", - "[resourceId('Microsoft.Authorization/policyAssignments', variables('policyAssignmentNameEnforceHttpsIngress'))]", - "[resourceId('Microsoft.Authorization/policyAssignments', variables('policyAssignmentNameEnforceImageSource'))]", - "[resourceId('Microsoft.Authorization/policyAssignments', variables('policyAssignmentNameEnforceInternalLoadBalancers'))]", - "[resourceId('Microsoft.Authorization/policyAssignments', variables('policyAssignmentNameEnforceResourceLimits'))]", - "[resourceId('Microsoft.Authorization/policyAssignments', variables('policyAssignmentNameRoRootFilesystem'))]", - "[resourceId('Microsoft.Authorization/policyAssignments', variables('policyAssignmentNameEnforceDefenderInCluster'))]", - "[resourceId('Microsoft.Network/privateEndpoints', concat('pe-', variables('keyVaultName')))]", - "[resourceId('Microsoft.KeyVault/vaults/providers/roleAssignments', variables('keyVaultName'), 'Microsoft.Authorization', guid(resourceGroup().id, 'podmi-ingress-controller', variables('keyVaultReader')))]", - "[resourceId('Microsoft.KeyVault/vaults/providers/roleAssignments', variables('keyVaultName'), 'Microsoft.Authorization', guid(resourceGroup().id, 'podmi-ingress-controller', variables('keyVaultSecretsUserRole')))]" - ], - "properties": { - "kubernetesVersion": "[parameters('kubernetesVersion')]", - "dnsPrefix": "[uniqueString(subscription().subscriptionId, resourceGroup().id, variables('clusterName'))]", - "agentPoolProfiles": [ - { - "name": "npsystem", - "count": 3, - "vmSize": "Standard_DS2_v2", - "osDiskSizeGB": 80, - "osDiskType": "Ephemeral", - "osType": "Linux", - "minCount": 3, - "maxCount": 4, - "vnetSubnetID": "[variables('vnetNodePoolSubnetResourceId')]", - "enableAutoScaling": true, - "type": "VirtualMachineScaleSets", - "mode": "System", - "scaleSetPriority": "Regular", - "scaleSetEvictionPolicy": "Delete", - "orchestratorVersion": "[parameters('kubernetesVersion')]", - "enableNodePublicIP": false, - "maxPods": 30, - "availabilityZones": [ - "1", - "2", - "3" - ], - "upgradeSettings": { - "maxSurge": "33%" - }, - "nodeTaints": [ - "CriticalAddonsOnly=true:NoSchedule" - ] - }, - { - "name": "npuser01", - "count": 2, - "vmSize": "Standard_DS3_v2", - "osDiskSizeGB": 120, - "osDiskType": "Ephemeral", - "osType": "Linux", - "minCount": 2, - "maxCount": 5, - "vnetSubnetID": "[variables('vnetNodePoolSubnetResourceId')]", - "enableAutoScaling": true, - "type": "VirtualMachineScaleSets", - "mode": "User", - "scaleSetPriority": "Regular", - "scaleSetEvictionPolicy": "Delete", - "orchestratorVersion": "[parameters('kubernetesVersion')]", - "enableNodePublicIP": false, - "maxPods": 30, - "availabilityZones": [ - "1", - "2", - "3" - ], - "upgradeSettings": { - "maxSurge": "33%" - } - } - ], - "servicePrincipalProfile": { - "clientId": "msi" - }, - "addonProfiles": { - "httpApplicationRouting": { - "enabled": false - }, - "omsagent": { - "enabled": true, - "config": { - "logAnalyticsWorkspaceResourceId": "[resourceId('Microsoft.OperationalInsights/workspaces', variables('logAnalyticsWorkspaceName'))]" - } - }, - "aciConnectorLinux": { - "enabled": false - }, - "azurepolicy": { - "enabled": true, - "config": { - "version": "v2" - } - }, - "azureKeyvaultSecretsProvider": { - "enabled": true, - "config": { - "enableSecretRotation": "false" - } - } - }, - "nodeResourceGroup": "[variables('nodeResourceGroupName')]", - "enableRBAC": true, - "enablePodSecurityPolicy": false, - "maxAgentPools": 2, - "networkProfile": { - "networkPlugin": "azure", - "networkPolicy": "azure", - "outboundType": "userDefinedRouting", - "loadBalancerSku": "standard", - "loadBalancerProfile": "[json('null')]", - "serviceCidr": "172.16.0.0/16", - "dnsServiceIP": "172.16.0.10", - "dockerBridgeCidr": "172.18.0.1/16" - }, - "aadProfile": { - "managed": true, - "enableAzureRBAC": "[variables('isUsingAzureRBACasKubernetesRBAC')]", - "adminGroupObjectIDs": "[if(not(variables('isUsingAzureRBACasKubernetesRBAC')), array(parameters('clusterAdminAadGroupObjectId')), createArray())]", - "tenantID": "[parameters('k8sControlPlaneAuthorizationTenantId')]" - }, - "autoScalerProfile": { - "balance-similar-node-groups": "false", - "expander": "random", - "max-empty-bulk-delete": "10", - "max-graceful-termination-sec": "600", - "max-node-provision-time": "15m", - "max-total-unready-percentage": "45", - "new-pod-scale-up-delay": "0s", - "ok-total-unready-count": "3", - "scale-down-delay-after-add": "10m", - "scale-down-delay-after-delete": "20s", - "scale-down-delay-after-failure": "3m", - "scale-down-unneeded-time": "10m", - "scale-down-unready-time": "20m", - "scale-down-utilization-threshold": "0.5", - "scan-interval": "10s", - "skip-nodes-with-local-storage": "true", - "skip-nodes-with-system-pods": "true" - }, - "apiServerAccessProfile": { - "authorizedIPRanges": "[parameters('clusterAuthorizedIPRanges')]", - "enablePrivateCluster": false - }, - "podIdentityProfile": { - "enabled": false, - "userAssignedIdentities": [], - "userAssignedIdentityExceptions": [] - }, - "disableLocalAccounts": true, - "securityProfile": { - "azureDefender": { - "enabled": true, - "logAnalyticsWorkspaceResourceId": "[resourceId('Microsoft.OperationalInsights/workspaces', variables('logAnalyticsWorkspaceName'))]" - } - }, - "oidcIssuerProfile": { - "enabled": true - } - }, - "identity": { - "type": "UserAssigned", - "userAssignedIdentities": { - "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('clusterControlPlaneIdentityName'))]": {} - } - }, - "sku": { - "name": "Basic", - "tier": "Paid" - }, - "resources": [ - { - "type": "providers/extensions", - "apiVersion": "2021-09-01", - "name": "Microsoft.KubernetesConfiguration/flux", - "comments": "Ensures that flux add-on (extension) is installed.", - "dependsOn": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]", - "[resourceId('Microsoft.ContainerRegistry/registries/providers/roleAssignments', variables('defaultAcrName'), 'Microsoft.Authorization', guid(resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName')), variables('acrPullRole')))]" - ], - "properties": { - "extensionType": "microsoft.flux", - "autoUpgradeMinorVersion": true, - "releaseTrain": "Stable", - "scope": { - "cluster": { - "releaseNamespace": "flux-system" - } - }, - "configurationSettings": { - "helm-controller.enabled": "false", - "source-controller.enabled": "true", - "kustomize-controller.enabled": "true", - "notification-controller.enabled": "false", - "image-automation-controller.enabled": "false", - "image-reflector-controller.enabled": "false" - }, - "configurationProtectedSettings": {} - } - }, - { - "type": "providers/fluxConfigurations", - "apiVersion": "2022-01-01-preview", - "name": "Microsoft.KubernetesConfiguration/bootstrap", - "comments": "Bootstraps your cluster using content from your repo.", - "dependsOn": [ - "[resourceId('Microsoft.ContainerService/managedClusters/providers/extensions', variables('clusterName'), 'Microsoft.KubernetesConfiguration', 'flux')]", - "[resourceId('Microsoft.ContainerRegistry/registries/providers/roleAssignments', variables('defaultAcrName'), 'Microsoft.Authorization', guid(resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName')), variables('acrPullRole')))]", - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]" - ], - "properties": { - "scope": "cluster", - "namespace": "flux-system", - "sourceKind": "GitRepository", - "gitRepository": { - "url": "[parameters('gitOpsBootstrappingRepoHttpsUrl')]", - "timeoutInSeconds": 180, - "syncIntervalInSeconds": 300, - "repositoryRef": { - "branch": "[parameters('gitOpsBootstrappingRepoBranch')]", - "tag": "[null()]", - "semver": "[null()]", - "commit": "[null()]" - }, - "sshKnownHosts": "", - "httpsUser": "[null()]", - "httpsCACert": "[null()]", - "localAuthRef": "[null()]" - }, - "kustomizations": { - "unified": { - "path": "./cluster-manifests", - "dependsOn": [], - "timeoutInSeconds": 300, - "syncIntervalInSeconds": 300, - "retryIntervalInSeconds": "[null()]", - "prune": true, - "force": false - } - } - } - }, - { - "type": "providers/roleAssignments", - "apiVersion": "2020-04-01-preview", - "name": "[concat('Microsoft.Authorization/', guid(resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName')), 'omsagent', variables('monitoringMetricsPublisherRole')))]", - "comments": "Grant the OMS Agent's Managed Identity the metrics publisher role to push alerts", - "dependsOn": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]" - ], - "properties": { - "roleDefinitionId": "[variables('monitoringMetricsPublisherRole')]", - "principalId": "[reference(resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName')), '2020-12-01').addonProfiles.omsagent.identity.objectId]", - "principalType": "ServicePrincipal" - } - }, - { - "type": "providers/diagnosticSettings", - "apiVersion": "2021-05-01-preview", - "name": "Microsoft.Insights/default", - "dependsOn": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]" - ], - "properties": { - "workspaceId": "[resourceId('Microsoft.OperationalInsights/workspaces', variables('logAnalyticsWorkspaceName'))]", - "logs": [ - { - "category": "cluster-autoscaler", - "enabled": true - }, - { - "category": "kube-controller-manager", - "enabled": true - }, - { - "category": "kube-audit-admin", - "enabled": true - }, - { - "category": "guard", - "enabled": true - } - ] - } - } - ] - }, - { - "type": "Microsoft.EventGrid/systemTopics", - "apiVersion": "2020-10-15-preview", - "name": "[variables('clusterName')]", - "location": "[parameters('location')]", - "dependsOn": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]" - ], - "properties": { - - "source": "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]", - "topicType": "Microsoft.ContainerService.ManagedClusters" - }, - "resources": [ - { - "type": "providers/diagnosticSettings", - "apiVersion": "2021-05-01-preview", - "name": "Microsoft.Insights/default", - "dependsOn": [ - "[resourceId('Microsoft.EventGrid/systemTopics', variables('clusterName'))]" - ], - "properties": { - "workspaceId": "[resourceId('Microsoft.OperationalInsights/workspaces', variables('logAnalyticsWorkspaceName'))]", - "logs": [ - { - "category": "DeliveryFailures", - "enabled": true - } - ], - "metrics": [ - { - "category": "AllMetrics", - "enabled": true - } - ] - } - } - ] - }, - { - "type": "Microsoft.Insights/metricAlerts", - "apiVersion": "2018-03-01", - "name": "[concat('Node CPU utilization high for ', variables('clusterName'), ' CI-1')]", - "location": "global", - "dependsOn": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]", - "[resourceId('Microsoft.OperationsManagement/solutions',variables('containerInsightsSolutionName'))]" - ], - "properties": { - "actions": [], - "criteria": { - "allOf": [ - { - "criterionType": "StaticThresholdCriterion", - "dimensions": [ - { - "name": "host", - "operator": "Include", - "values": [ - "*" - ] - } - ], - "metricName": "cpuUsagePercentage", - "metricNamespace": "Insights.Container/nodes", - "name": "Metric1", - "operator": "GreaterThan", - "threshold": 80.0, - "timeAggregation": "Average", - "skipMetricValidation": true - } - ], - "odata.type": "Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria" - }, - "description": "Node CPU utilization across the cluster.", - "enabled": true, - "evaluationFrequency": "PT1M", - "scopes": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]" - ], - "severity": 3, - "targetResourceType": "microsoft.containerservice/managedclusters", - "windowSize": "PT5M" - } - }, - { - "type": "Microsoft.Insights/metricAlerts", - "apiVersion": "2018-03-01", - "name": "[concat('Node working set memory utilization high for ', variables('clusterName'), ' CI-2')]", - "location": "global", - "dependsOn": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]", - "[resourceId('Microsoft.OperationsManagement/solutions',variables('containerInsightsSolutionName'))]" - ], - "properties": { - "actions": [], - "criteria": { - "allOf": [ - { - "criterionType": "StaticThresholdCriterion", - "dimensions": [ - { - "name": "host", - "operator": "Include", - "values": [ - "*" - ] - } - ], - "metricName": "memoryWorkingSetPercentage", - "metricNamespace": "Insights.Container/nodes", - "name": "Metric1", - "operator": "GreaterThan", - "threshold": 80.0, - "timeAggregation": "Average", - "skipMetricValidation": true - } - ], - "odata.type": "Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria" - }, - "description": "Node working set memory utilization across the cluster.", - "enabled": true, - "evaluationFrequency": "PT1M", - "scopes": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]" - ], - "severity": 3, - "targetResourceType": "microsoft.containerservice/managedclusters", - "windowSize": "PT5M" - } - }, - { - "type": "Microsoft.Insights/metricAlerts", - "apiVersion": "2018-03-01", - "name": "[concat('Jobs completed more than 6 hours ago for ', variables('clusterName'), ' CI-11')]", - "location": "global", - "dependsOn": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]", - "[resourceId('Microsoft.OperationsManagement/solutions',variables('containerInsightsSolutionName'))]" - ], - "properties": { - "actions": [], - "criteria": { - "allOf": [ - { - "criterionType": "StaticThresholdCriterion", - "dimensions": [ - { - "name": "controllerName", - "operator": "Include", - "values": [ - "*" - ] - }, - { - "name": "kubernetes namespace", - "operator": "Include", - "values": [ - "*" - ] - } - ], - "metricName": "completedJobsCount", - "metricNamespace": "Insights.Container/pods", - "name": "Metric1", - "operator": "GreaterThan", - "threshold": 0.0, - "timeAggregation": "Average", - "skipMetricValidation": true - } - ], - "odata.type": "Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria" - }, - "description": "This alert monitors completed jobs (more than 6 hours ago).", - "enabled": true, - "evaluationFrequency": "PT1M", - "scopes": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]" - ], - "severity": 3, - "targetResourceType": "microsoft.containerservice/managedclusters", - "windowSize": "PT1M" - } - }, - { - "type": "Microsoft.Insights/metricAlerts", - "apiVersion": "2018-03-01", - "name": "[concat('Container CPU usage high for ', variables('clusterName'), ' CI-9')]", - "location": "global", - "dependsOn": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]", - "[resourceId('Microsoft.OperationsManagement/solutions',variables('containerInsightsSolutionName'))]" - ], - "properties": { - "actions": [], - "criteria": { - "allOf": [ - { - "criterionType": "StaticThresholdCriterion", - "dimensions": [ - { - "name": "controllerName", - "operator": "Include", - "values": [ - "*" - ] - }, - { - "name": "kubernetes namespace", - "operator": "Include", - "values": [ - "*" - ] - } - ], - "metricName": "cpuExceededPercentage", - "metricNamespace": "Insights.Container/containers", - "name": "Metric1", - "operator": "GreaterThan", - "threshold": 90.0, - "timeAggregation": "Average", - "skipMetricValidation": true - } - ], - "odata.type": "Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria" - }, - "description": "This alert monitors container CPU utilization.", - "enabled": true, - "evaluationFrequency": "PT1M", - "scopes": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]" - ], - "severity": 3, - "targetResourceType": "microsoft.containerservice/managedclusters", - "windowSize": "PT5M" - } - }, - { - "type": "Microsoft.Insights/metricAlerts", - "apiVersion": "2018-03-01", - "name": "[concat('Container working set memory usage high for ', variables('clusterName'), ' CI-10')]", - "location": "global", - "dependsOn": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]", - "[resourceId('Microsoft.OperationsManagement/solutions',variables('containerInsightsSolutionName'))]" - ], - "properties": { - "actions": [], - "criteria": { - "allOf": [ - { - "criterionType": "StaticThresholdCriterion", - "dimensions": [ - { - "name": "controllerName", - "operator": "Include", - "values": [ - "*" - ] - }, - { - "name": "kubernetes namespace", - "operator": "Include", - "values": [ - "*" - ] - } - ], - "metricName": "memoryWorkingSetExceededPercentage", - "metricNamespace": "Insights.Container/containers", - "name": "Metric1", - "operator": "GreaterThan", - "threshold": 90.0, - "timeAggregation": "Average", - "skipMetricValidation": true - } - ], - "odata.type": "Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria" - }, - "description": "This alert monitors container working set memory utilization.", - "enabled": true, - "evaluationFrequency": "PT1M", - "scopes": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]" - ], - "severity": 3, - "targetResourceType": "microsoft.containerservice/managedclusters", - "windowSize": "PT5M" - } - }, - { - "type": "Microsoft.Insights/metricAlerts", - "apiVersion": "2018-03-01", - "name": "[concat('Pods in failed state for ', variables('clusterName'), ' CI-4')]", - "location": "global", - "dependsOn": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]", - "[resourceId('Microsoft.OperationsManagement/solutions',variables('containerInsightsSolutionName'))]" - ], - "properties": { - "actions": [], - "criteria": { - "allOf": [ - { - "criterionType": "StaticThresholdCriterion", - "dimensions": [ - { - "name": "phase", - "operator": "Include", - "values": [ - "Failed" - ] - } - ], - "metricName": "podCount", - "metricNamespace": "Insights.Container/pods", - "name": "Metric1", - "operator": "GreaterThan", - "threshold": 0.0, - "timeAggregation": "Average", - "skipMetricValidation": true - } - ], - "odata.type": "Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria" - }, - "description": "Pod status monitoring.", - "enabled": true, - "evaluationFrequency": "PT1M", - "scopes": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]" - ], - "severity": 3, - "targetResourceType": "microsoft.containerservice/managedclusters", - "windowSize": "PT5M" - } - }, - { - "type": "Microsoft.Insights/metricAlerts", - "apiVersion": "2018-03-01", - "name": "[concat('Disk usage high for ', variables('clusterName'), ' CI-5')]", - "location": "global", - "dependsOn": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]", - "[resourceId('Microsoft.OperationsManagement/solutions',variables('containerInsightsSolutionName'))]" - ], - "properties": { - "actions": [], - "criteria": { - "allOf": [ - { - "criterionType": "StaticThresholdCriterion", - "dimensions": [ - { - "name": "host", - "operator": "Include", - "values": [ - "*" - ] - }, - { - "name": "device", - "operator": "Include", - "values": [ - "*" - ] - } - ], - "metricName": "DiskUsedPercentage", - "metricNamespace": "Insights.Container/nodes", - "name": "Metric1", - "operator": "GreaterThan", - "threshold": 80.0, - "timeAggregation": "Average", - "skipMetricValidation": true - } - ], - "odata.type": "Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria" - }, - "description": "This alert monitors disk usage for all nodes and storage devices.", - "enabled": true, - "evaluationFrequency": "PT1M", - "scopes": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]" - ], - "severity": 3, - "targetResourceType": "microsoft.containerservice/managedclusters", - "windowSize": "PT5M" - } - }, - { - "type": "Microsoft.Insights/metricAlerts", - "apiVersion": "2018-03-01", - "name": "[concat('Nodes in not ready status for ', variables('clusterName'), ' CI-3')]", - "location": "global", - "dependsOn": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]", - "[resourceId('Microsoft.OperationsManagement/solutions',variables('containerInsightsSolutionName'))]" - ], - "properties": { - "actions": [], - "criteria": { - "allOf": [ - { - "criterionType": "StaticThresholdCriterion", - "dimensions": [ - { - "name": "status", - "operator": "Include", - "values": [ - "NotReady" - ] - } - ], - "metricName": "nodesCount", - "metricNamespace": "Insights.Container/nodes", - "name": "Metric1", - "operator": "GreaterThan", - "threshold": 0.0, - "timeAggregation": "Average", - "skipMetricValidation": true - } - ], - "odata.type": "Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria" - }, - "description": "Node status monitoring.", - "enabled": true, - "evaluationFrequency": "PT1M", - "scopes": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]" - ], - "severity": 3, - "targetResourceType": "microsoft.containerservice/managedclusters", - "windowSize": "PT5M" - } - }, - { - "type": "Microsoft.Insights/metricAlerts", - "apiVersion": "2018-03-01", - "name": "[concat('Containers getting OOM killed for ', variables('clusterName'), ' CI-6')]", - "location": "global", - "dependsOn": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]", - "[resourceId('Microsoft.OperationsManagement/solutions',variables('containerInsightsSolutionName'))]" - ], - "properties": { - "actions": [], - "criteria": { - "allOf": [ - { - "criterionType": "StaticThresholdCriterion", - "dimensions": [ - { - "name": "kubernetes namespace", - "operator": "Include", - "values": [ - "*" - ] - }, - { - "name": "controllerName", - "operator": "Include", - "values": [ - "*" - ] - } - ], - "metricName": "oomKilledContainerCount", - "metricNamespace": "Insights.Container/pods", - "name": "Metric1", - "operator": "GreaterThan", - "threshold": 0.0, - "timeAggregation": "Average", - "skipMetricValidation": true - } - ], - "odata.type": "Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria" - }, - "description": "This alert monitors number of containers killed due to out of memory (OOM) error.", - "enabled": true, - "evaluationFrequency": "PT1M", - "scopes": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]" - ], - "severity": 3, - "targetResourceType": "microsoft.containerservice/managedclusters", - "windowSize": "PT1M" - } - }, - { - "type": "Microsoft.Insights/metricAlerts", - "apiVersion": "2018-03-01", - "name": "[concat('Persistent volume usage high for ', variables('clusterName'), ' CI-18')]", - "location": "global", - "dependsOn": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]", - "[resourceId('Microsoft.OperationsManagement/solutions',variables('containerInsightsSolutionName'))]" - ], - "properties": { - "actions": [], - "criteria": { - "allOf": [ - { - "criterionType": "StaticThresholdCriterion", - "dimensions": [ - { - "name": "podName", - "operator": "Include", - "values": [ - "*" - ] - }, - { - "name": "kubernetesNamespace", - "operator": "Include", - "values": [ - "*" - ] - } - ], - "metricName": "pvUsageExceededPercentage", - "metricNamespace": "Insights.Container/persistentvolumes", - "name": "Metric1", - "operator": "GreaterThan", - "threshold": 80.0, - "timeAggregation": "Average", - "skipMetricValidation": true - } - ], - "odata.type": "Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria" - }, - "description": "This alert monitors persistent volume utilization.", - "enabled": false, - "evaluationFrequency": "PT1M", - "scopes": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]" - ], - "severity": 3, - "targetResourceType": "microsoft.containerservice/managedclusters", - "windowSize": "PT5M" - } - }, - { - "type": "Microsoft.Insights/metricAlerts", - "apiVersion": "2018-03-01", - "name": "[concat('Pods not in ready state for ', variables('clusterName'), ' CI-8')]", - "location": "global", - "dependsOn": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]", - "[resourceId('Microsoft.OperationsManagement/solutions',variables('containerInsightsSolutionName'))]" - ], - "properties": { - "actions": [], - "criteria": { - "allOf": [ - { - "criterionType": "StaticThresholdCriterion", - "dimensions": [ - { - "name": "controllerName", - "operator": "Include", - "values": [ - "*" - ] - }, - { - "name": "kubernetes namespace", - "operator": "Include", - "values": [ - "*" - ] - } - ], - "metricName": "PodReadyPercentage", - "metricNamespace": "Insights.Container/pods", - "name": "Metric1", - "operator": "LessThan", - "threshold": 80.0, - "timeAggregation": "Average", - "skipMetricValidation": true - } - ], - "odata.type": "Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria" - }, - "description": "This alert monitors for excessive pods not in the ready state.", - "enabled": true, - "evaluationFrequency": "PT1M", - "scopes": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]" - ], - "severity": 3, - "targetResourceType": "microsoft.containerservice/managedclusters", - "windowSize": "PT5M" - } - }, - { - "type": "Microsoft.Insights/metricAlerts", - "apiVersion": "2018-03-01", - "name": "[concat('Restarting container count for ', variables('clusterName'), ' CI-7')]", - "location": "global", - "dependsOn": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]", - "[resourceId('Microsoft.OperationsManagement/solutions',variables('containerInsightsSolutionName'))]" - ], - "properties": { - "actions": [], - "criteria": { - "allOf": [ - { - "criterionType": "StaticThresholdCriterion", - "dimensions": [ - { - "name": "kubernetes namespace", - "operator": "Include", - "values": [ - "*" - ] - }, - { - "name": "controllerName", - "operator": "Include", - "values": [ - "*" - ] - } - ], - "metricName": "restartingContainerCount", - "metricNamespace": "Insights.Container/pods", - "name": "Metric1", - "operator": "GreaterThan", - "threshold": 0.0, - "timeAggregation": "Average", - "skipMetricValidation": true - } - ], - "odata.type": "Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria" - }, - "description": "This alert monitors number of containers restarting across the cluster.", - "enabled": true, - "evaluationFrequency": "PT1M", - "scopes": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]" - ], - "severity": 3, - "targetResourceType": "Microsoft.ContainerService/managedClusters", - "windowSize": "PT1M" - } - }, - { - "condition": "[variables('isUsingAzureRBACasKubernetesRBAC')]", - "type": "Microsoft.Authorization/roleAssignments", - "apiVersion": "2020-04-01-preview", - "name": "[guid('aad-admin-group', resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName')), parameters('clusterAdminAadGroupObjectId'))]", - "dependsOn": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]" - ], - "scope": "[concat('Microsoft.ContainerService/managedClusters/', variables('clusterName'))]", - "properties": { - "roleDefinitionId": "[concat('/subscriptions/', subscription().subscriptionId, '/providers/Microsoft.Authorization/roleDefinitions/', variables('clusterAdminRoleId'))]", - "description": "Members of this group are cluster admins of this cluster.", - "principalId": "[parameters('clusterAdminAadGroupObjectId')]", - "principalType": "Group" - } - }, - { - "condition": "[variables('isUsingAzureRBACasKubernetesRBAC')]", - "type": "Microsoft.Authorization/roleAssignments", - "apiVersion": "2020-04-01-preview", - "name": "[guid('aad-admin-group-sc', resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName')), parameters('clusterAdminAadGroupObjectId'))]", - "dependsOn": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]" - ], - "scope": "[concat('Microsoft.ContainerService/managedClusters/', variables('clusterName'))]", - "properties": { - "roleDefinitionId": "[concat('/subscriptions/', subscription().subscriptionId, '/providers/Microsoft.Authorization/roleDefinitions/', variables('serviceClusterUserRoleId'))]", - "description": "Members of this group are cluster users of this cluster.", - "principalId": "[parameters('clusterAdminAadGroupObjectId')]", - "principalType": "Group" - } - }, - { - "condition": "[and(variables('isUsingAzureRBACasKubernetesRBAC'), not(equals(parameters('a0008NamespaceReaderAadGroupObjectId'), parameters('clusterAdminAadGroupObjectId'))))]", - "type": "Microsoft.Authorization/roleAssignments", - "apiVersion": "2020-04-01-preview", - "name": "[guid('aad-a0008-reader-group', resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName')), parameters('a0008NamespaceReaderAadGroupObjectId'))]", - "dependsOn": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]" - ], - "scope": "[concat('/subscriptions/', subscription().subscriptionId, '/resourcegroups/', resourceGroup().name, '/providers/Microsoft.ContainerService/managedClusters/', variables('clusterName'), '/namespaces/a0008')]", - "properties": { - "roleDefinitionId": "[concat('/subscriptions/', subscription().subscriptionId, '/providers/Microsoft.Authorization/roleDefinitions/', variables('clusterReaderRoleId'))]", - "principalId": "[parameters('a0008NamespaceReaderAadGroupObjectId')]", - "description": "Members of this group are cluster admins of the a0008 namespace in this cluster.", - "principalType": "Group" - } - }, - { - "condition": "[and(variables('isUsingAzureRBACasKubernetesRBAC'), not(equals(parameters('a0008NamespaceReaderAadGroupObjectId'), parameters('clusterAdminAadGroupObjectId'))))]", - "type": "Microsoft.Authorization/roleAssignments", - "apiVersion": "2020-04-01-preview", - "name": "[guid('aad-a0008-reader-group-sc', resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName')), parameters('a0008NamespaceReaderAadGroupObjectId'))]", - "dependsOn": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]" - ], - "scope": "[concat('Microsoft.ContainerService/managedClusters/', variables('clusterName'))]", - "properties": { - "roleDefinitionId": "[concat('/subscriptions/', subscription().subscriptionId, '/providers/Microsoft.Authorization/roleDefinitions/', variables('serviceClusterUserRoleId'))]", - "principalId": "[parameters('a0008NamespaceReaderAadGroupObjectId')]", - "description": "Members of this group are cluster users of this cluster.", - "principalType": "Group" - } - }, - { - "type": "Microsoft.ManagedIdentity/userAssignedIdentities/providers/roleAssignments", - "apiVersion": "2020-04-01-preview", - "name": "[concat('podmi-ingress-controller', '/Microsoft.Authorization/', guid(resourceGroup().id, 'podmi-ingress-controller', variables('managedIdentityOperatorRole')))]", - "comments": "Grant the AKS cluster with Managed Identity Operator role permissions over the managed identity used for the ingress controller. Allows it to be assigned to the underlying VMSS.", - "dependsOn": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]" - ], - "properties": { - "roleDefinitionId": "[variables('managedIdentityOperatorRole')]", - "principalId": "[reference(resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName')), '2020-11-01').identityProfile.kubeletidentity.objectId]", - "principalType": "ServicePrincipal" - } - }, - { - "type": "Microsoft.Authorization/policyAssignments", - "apiVersion": "2020-09-01", - "name": "[variables('policyAssignmentNameAKSLinuxRestrictive')]", - "comments": "Applying the 'AKS Linux Restrictive' policy to the resource group", - "properties": { - "displayName": "[concat('[', variables('clusterName'),'] ', reference(variables('policyResourceIdAKSLinuxRestrictive'), '2020-09-01').displayName)]", - "scope": "[subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name)]", - "policyDefinitionId": "[variables('policyResourceIdAKSLinuxRestrictive')]", - "parameters": { - "excludedNamespaces": { - "value": [ - "kube-system", - "gatekeeper-system", - "azure-arc", - "cluster-baseline-settings" - ] - }, - "effect": { - "value": "audit" - } - } - } - }, - { - "type": "Microsoft.Authorization/policyAssignments", - "apiVersion": "2020-09-01", - "name": "[variables('policyAssignmentNameEnforceHttpsIngress')]", - "comments": "Applying the 'Enforce HTTPS ingress in Kubernetes cluster' policy to the resource group.", - "properties": { - "displayName": "[concat('[', variables('clusterName'),'] ', reference(variables('policyResourceIdEnforceHttpsIngress'), '2020-09-01').displayName)]", - "scope": "[subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name)]", - "policyDefinitionId": "[variables('policyResourceIdEnforceHttpsIngress')]", - "parameters": { - "excludedNamespaces": { - "value": [] - }, - "effect": { - "value": "deny" - } - } - } - }, - { - "type": "Microsoft.Authorization/policyAssignments", - "apiVersion": "2020-09-01", - "name": "[variables('policyAssignmentNameEnforceInternalLoadBalancers')]", - "comments": "Applying the 'Enforce internal load balancers in Kubernetes cluster' policy to the resource group.", - "properties": { - "displayName": "[concat('[', variables('clusterName'),'] ', reference(variables('policyResourceIdEnforceInternalLoadBalancers'), '2020-09-01').displayName)]", - "scope": "[subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name)]", - "policyDefinitionId": "[variables('policyResourceIdEnforceInternalLoadBalancers')]", - "parameters": { - "excludedNamespaces": { - "value": [] - }, - "effect": { - "value": "deny" - } - } - } - }, - { - "type": "Microsoft.Authorization/policyAssignments", - "apiVersion": "2020-09-01", - "name": "[variables('policyAssignmentNameRoRootFilesystem')]", - "comments": "Applying the 'Kubernetes cluster containers should run with a read only root file system' policy to the resource group.", - "properties": { - "displayName": "[concat('[', variables('clusterName'),'] ', reference(variables('policyResourceIdRoRootFilesystem'), '2020-09-01').displayName)]", - "scope": "[subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name)]", - "policyDefinitionId": "[variables('policyResourceIdRoRootFilesystem')]", - "parameters": { - "excludedNamespaces": { - "value": [ - "kube-system", - "gatekeeper-system", - "azure-arc" - ] - }, - "effect": { - "value": "audit" - } - } - } - }, - { - "type": "Microsoft.Authorization/policyAssignments", - "apiVersion": "2020-09-01", - "name": "[variables('policyAssignmentNameEnforceResourceLimits')]", - "comments": "Applying the 'Container Images Resource Limits' policy at the resource group level.", - "properties": { - "displayName": "[concat('[', variables('clusterName'),'] ', reference(variables('policyResourceIdEnforceResourceLimits'), '2020-09-01').displayName)]", - "scope": "[subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name)]", - "policyDefinitionId": "[variables('policyResourceIdEnforceResourceLimits')]", - "parameters": { - "cpuLimit": { - "value": "1000m" - }, - "memoryLimit": { - "value": "512Mi" - }, - "excludedNamespaces": { - "value": [ - "kube-system", - "gatekeeper-system", - "azure-arc", - "cluster-baseline-settings", - "flux-system" /* Flux add-on, not all containers have limits defined by Microsoft */ - ] - }, - "effect": { - "value": "deny" - } - } - } - }, - { - "type": "Microsoft.Authorization/policyAssignments", - "apiVersion": "2020-09-01", - "name": "[variables('policyAssignmentNameEnforceImageSource')]", - "comments": "Applying the 'Allowed Container Images' regex policy at the resource group level. If all images are pull into your ARC instance as described in these instructions you can remove the docker.io entries.", - "properties": { - "displayName": "[concat('[', variables('clusterName'),'] ', reference(variables('policyResourceIdEnforceImageSource'), '2020-09-01').displayName)]", - "scope": "[subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name)]", - "policyDefinitionId": "[variables('policyResourceIdEnforceImageSource')]", - "parameters": { - "allowedContainerImagesRegex": { - "value": "[concat(variables('defaultAcrName'),'.azurecr.io/.+$|mcr.microsoft.com/.+$|azurearcfork8s.azurecr.io/azurearcflux/images/stable/.+$|docker.io/weaveworks/kured.+$|docker.io/library/.+$')]" - }, - "excludedNamespaces": { - "value": [ - "kube-system", - "gatekeeper-system", - "azure-arc" - ] - }, - "effect": { - "value": "deny" - } - } - } - }, - { - "type": "Microsoft.Authorization/policyAssignments", - "apiVersion": "2020-09-01", - "name": "[variables('policyAssignmentNameEnforceDefenderInCluster')]", - "comments": "Applying the 'Azure Kubernetes Service clusters should have Defender profile enabled' policy at the resource group level.", - "properties": { - "displayName": "[concat('[', variables('clusterName'),'] ', reference(variables('policyResourceIdEnforceDefenderInCluster'), '2020-09-01').displayName)]", - "description": "Microsoft Defender for Containers should be enabled in the cluster.", - "scope": "[subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name)]", - "policyDefinitionId": "[variables('policyResourceIdEnforceDefenderInCluster')]", - "parameters": { - "effect": { - "value": "Audit" - } - } - } - } - ], - "outputs": { - "aksClusterName": { - "type": "string", - "value": "[variables('clusterName')]" - }, - "aksIngressControllerPodManagedIdentityResourceId": { - "type": "string", - "value": "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', 'podmi-ingress-controller')]" - }, - "aksIngressControllerPodManagedIdentityClientId": { - "type": "string", - "value": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities','podmi-ingress-controller'), '2018-11-30').clientId]" - }, - "keyVaultName": { - "type": "string", - "value": "[variables('keyVaultName')]" - } - } -} diff --git a/github-workflow/aks-deploy.yaml b/github-workflow/aks-deploy.yaml index ba6029e6..0598a04c 100644 --- a/github-workflow/aks-deploy.yaml +++ b/github-workflow/aks-deploy.yaml @@ -9,7 +9,7 @@ # │   │   └── aks-deploy.yaml # ├── cluster-manifests # │   ├── cluster-baseline-settings/* -# └── cluster-stamp.json +# └── cluster-stamp.bicep # # 2. Ensure you have followed the prior sections before deploying this AKS cluster. This way, you will be capable of setting: # - the secrets values as detailed in the next step. @@ -25,12 +25,12 @@ name: Deploy AKS Baseline cluster stamp on: push: paths: - - 'cluster-stamp.json' + - 'cluster-stamp.bicep' - '.github/workflows/aks-deploy.yaml' branches: [ main ] pull_request: paths: - - 'cluster-stamp.json' + - 'cluster-stamp.bicep' - '.github/workflows/aks-deploy.yaml' branches: [ main ] @@ -69,7 +69,7 @@ jobs: az group create --name ${{ env.RESOURCE_GROUP }} --location ${{ env.RESOURCE_GROUP_LOCATION }} az deployment group $([[ ${{ github.event_name }} = pull_request ]] && echo what-if --no-pretty-print || echo create) \ --resource-group ${{ env.RESOURCE_GROUP }} \ - --template-file "cluster-stamp.json" \ + --template-file "cluster-stamp.bicep" \ --name "cluster-stamp" \ --parameters \ location=${{ env.AKS_LOCATION }} \ From eff5cb97ca71f299010018546dd02011a6bbf039 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Wed, 20 Apr 2022 14:13:57 +0000 Subject: [PATCH 30/54] bug fix: remove default prop since ARM is telling this cant be set to false --- cluster-stamp.bicep | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index c5fd0573..5c631218 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -997,10 +997,9 @@ resource kv 'Microsoft.KeyVault/vaults@2021-11-01-preview' = { enabledForTemplateDeployment: false enableSoftDelete: true softDeleteRetentionInDays: 7 - enablePurgeProtection: false createMode: 'default' publicNetworkAccess: 'disabled' - } + } dependsOn: [ miAppGatewayFrontend podmiIngressController From a1d83e837b0c9cc0ee26cca665470e33ba574193 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 22 Apr 2022 12:51:10 +0000 Subject: [PATCH 31/54] add warning while usin gforks --- 06-aks-cluster.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/06-aks-cluster.md b/06-aks-cluster.md index 14498870..39cee73d 100644 --- a/06-aks-cluster.md +++ b/06-aks-cluster.md @@ -12,6 +12,8 @@ Now that your [ACR instance is deployed and ready to support cluster bootstrappi GITOPS_REPOURL=$(git config --get remote.origin.url) ``` + > :warning: If you are using your own fork, please ensure this repo is reacheable as well as public accessible from your spoke vnet, and check you are assigning its web URL (`HTTPS`). + 1. Deploy the cluster ARM template. :exclamation: By default, this deployment will allow unrestricted access to your cluster's API Server. You can limit access to the API Server to a set of well-known IP addresses (i.,e. a jump box subnet (connected to by Azure Bastion), build agents, or any other networks you'll administer the cluster from) by setting the `clusterAuthorizedIPRanges` parameter in all deployment options. This setting will also impact traffic originating from within the cluster trying to use the API server, so you will also need to include _all_ of the public IPs used by your egress Azure Firewall. For more information, see [Secure access to the API server using authorized IP address ranges](https://docs.microsoft.com/azure/aks/api-server-authorized-ip-ranges#create-an-aks-cluster-with-api-server-authorized-ip-ranges-enabled). From e3ef03f44dc6282be826056fb504aa150ea448e2 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 22 Apr 2022 13:04:11 +0000 Subject: [PATCH 32/54] revert public ntw access disabled for ri --- cluster-stamp.bicep | 1 - 1 file changed, 1 deletion(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 5c631218..52381b1b 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -998,7 +998,6 @@ resource kv 'Microsoft.KeyVault/vaults@2021-11-01-preview' = { enableSoftDelete: true softDeleteRetentionInDays: 7 createMode: 'default' - publicNetworkAccess: 'disabled' } dependsOn: [ miAppGatewayFrontend From b2625ba777411e12b0b71ce8428a5361223897e7 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 22 Apr 2022 10:34:59 -0300 Subject: [PATCH 33/54] minor fix about known issue note --- 10-workload.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/10-workload.md b/10-workload.md index 089ec8ac..73d09c2b 100644 --- a/10-workload.md +++ b/10-workload.md @@ -49,7 +49,7 @@ The cluster now has an [Traefik configured with a TLS certificate](./08-secret-m exit ``` - > :beetle: If you are running a version of kubectl less than 1.23, you'll receive an error from the run command above as the method to provide container limits has changed between 1.22 and 1.23. On kubectl versions less than 1.23, you'll need to run the following command instead and you can [safely ignore the message about `--limits` being deprecated](https://github.com/kubernetes/kubectl/issues/1101). + > :beetle: If you are running a version of kubectl less than 1.23, you'll receive an error from the run command below as the method to provide container limits has changed between 1.22 and 1.23. On kubectl versions less than 1.23, you'll need to run the following command instead and you can [safely ignore the message about `--limits` being deprecated](https://github.com/kubernetes/kubectl/issues/1101). ```bash kubectl run curl -n a0008 -i --tty --rm --image=mcr.microsoft.com/azure-cli --limits='cpu=200m,memory=128Mi' From 3a9c88e6c60eac9033298568662653c97686128c Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 22 Apr 2022 15:18:29 +0000 Subject: [PATCH 34/54] Revert "minor fix about known issue note" This reverts commit b2625ba777411e12b0b71ce8428a5361223897e7. --- 10-workload.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/10-workload.md b/10-workload.md index 73d09c2b..089ec8ac 100644 --- a/10-workload.md +++ b/10-workload.md @@ -49,7 +49,7 @@ The cluster now has an [Traefik configured with a TLS certificate](./08-secret-m exit ``` - > :beetle: If you are running a version of kubectl less than 1.23, you'll receive an error from the run command below as the method to provide container limits has changed between 1.22 and 1.23. On kubectl versions less than 1.23, you'll need to run the following command instead and you can [safely ignore the message about `--limits` being deprecated](https://github.com/kubernetes/kubectl/issues/1101). + > :beetle: If you are running a version of kubectl less than 1.23, you'll receive an error from the run command above as the method to provide container limits has changed between 1.22 and 1.23. On kubectl versions less than 1.23, you'll need to run the following command instead and you can [safely ignore the message about `--limits` being deprecated](https://github.com/kubernetes/kubectl/issues/1101). ```bash kubectl run curl -n a0008 -i --tty --rm --image=mcr.microsoft.com/azure-cli --limits='cpu=200m,memory=128Mi' From 5f7b0b49bedd4dc5ebde6767e26471ccc04318a5 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Tue, 26 Apr 2022 17:41:43 +0000 Subject: [PATCH 35/54] Address PR Feedback: add missing comments to resources --- cluster-stamp.bicep | 23 ++++++++++++++++++- ...usterUserAssignedHasRbacToManageVMSS.bicep | 1 + 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 52381b1b..7fbda45d 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -807,6 +807,7 @@ resource sqrPodFailed 'Microsoft.Insights/scheduledQueryRules@2018-04-16' = { dependsOn: [] } +// Applying the 'AKS Linux Restrictive' policy to the resource group resource paAKSLinuxRestrictive 'Microsoft.Authorization/policyAssignments@2021-06-01' = { name: policyAssignmentNameAKSLinuxRestrictive properties: { @@ -830,6 +831,7 @@ resource paAKSLinuxRestrictive 'Microsoft.Authorization/policyAssignments@2021-0 dependsOn: [] } +// Applying the 'Enforce HTTPS ingress in Kubernetes cluster' policy to the resource group. resource paEnforceHttpsIngress 'Microsoft.Authorization/policyAssignments@2021-06-01' = { name: policyAssignmentNameEnforceHttpsIngress location: 'global' @@ -849,6 +851,7 @@ resource paEnforceHttpsIngress 'Microsoft.Authorization/policyAssignments@2021-0 dependsOn: [] } +// Applying the 'Enforce internal load balancers in Kubernetes cluster' policy to the resource group. resource paEnforceInternalLoadBalancers 'Microsoft.Authorization/policyAssignments@2021-06-01' = { name: policyAssignmentNameEnforceInternalLoadBalancers properties: { @@ -867,6 +870,7 @@ resource paEnforceInternalLoadBalancers 'Microsoft.Authorization/policyAssignmen dependsOn: [] } +// Applying the 'Kubernetes cluster containers should run with a read only root file system' policy to the resource group. resource paRoRootFilesystem 'Microsoft.Authorization/policyAssignments@2021-06-01' = { name: policyAssignmentNameRoRootFilesystem properties: { @@ -889,6 +893,7 @@ resource paRoRootFilesystem 'Microsoft.Authorization/policyAssignments@2021-06-0 dependsOn: [] } +// Applying the 'Container Images Resource Limits' policy at the resource group level. resource paEnforceResourceLimits 'Microsoft.Authorization/policyAssignments@2021-06-01' = { name: policyAssignmentNameEnforceResourceLimits dependsOn: [] @@ -919,6 +924,7 @@ resource paEnforceResourceLimits 'Microsoft.Authorization/policyAssignments@2021 } } +// Applying the 'Allowed Container Images' regex policy at the resource group level. If all images are pull into your ARC instance as described in these instructions you can remove the docker.io entries. resource paEnforceImageSource 'Microsoft.Authorization/policyAssignments@2021-06-01' = { name: policyAssignmentNameEnforceImageSource properties: { @@ -944,6 +950,7 @@ resource paEnforceImageSource 'Microsoft.Authorization/policyAssignments@2021-06 dependsOn: [] } +// Applying the 'Azure Kubernetes Service clusters should have Defender profile enabled' policy at the resource group level. resource paEnforceDefenderInCluster 'Microsoft.Authorization/policyAssignments@2021-06-01' = { name: policyAssignmentNameEnforceDefenderInCluster properties: { @@ -960,16 +967,19 @@ resource paEnforceDefenderInCluster 'Microsoft.Authorization/policyAssignments@2 dependsOn: [] } +// The control plane identity used by the cluster. Used for networking access (VNET joining and DNS updating) resource miClusterControlPlane 'Microsoft.ManagedIdentity/userAssignedIdentities@2018-11-30' = { name: 'mi-${clusterName}-controlplane' location: location } +// User Managed Identity that App Gateway is assigned. Used for Azure Key Vault Access. resource miAppGatewayFrontend 'Microsoft.ManagedIdentity/userAssignedIdentities@2018-11-30' = { name: 'mi-appgateway-frontend' location: location } +// User Managed Identity for the cluster's ingress controller pods. Used for Azure Key Vault Access resource podmiIngressController 'Microsoft.ManagedIdentity/userAssignedIdentities@2018-11-30' = { name: 'podmi-ingress-controller' location: location @@ -998,7 +1008,7 @@ resource kv 'Microsoft.KeyVault/vaults@2021-11-01-preview' = { enableSoftDelete: true softDeleteRetentionInDays: 7 createMode: 'default' - } + } dependsOn: [ miAppGatewayFrontend podmiIngressController @@ -1040,6 +1050,7 @@ resource kv_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-0 dependsOn: [] } +// Grant the Azure Application Gateway managed identity with key vault reader role permissions; this allows pulling frontend and backend certificates. resource kvMiAppGatewayFrontendSecretsUserRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { scope: kv name: '${guid(resourceGroup().id, 'mi-appgateway-frontend', keyVaultSecretsUserRole)}' @@ -1051,6 +1062,7 @@ resource kvMiAppGatewayFrontendSecretsUserRole_roleAssignment 'Microsoft.Authori dependsOn: [] } +// Grant the Azure Application Gateway managed identity with key vault reader role permissions; this allows pulling frontend and backend certificates. resource kvMiAppGatewayFrontendKeyVaultReader_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { scope: kv name: '${guid(resourceGroup().id, 'mi-appgateway-frontend', keyVaultReader)}' @@ -1062,6 +1074,7 @@ resource kvMiAppGatewayFrontendKeyVaultReader_roleAssignment 'Microsoft.Authoriz dependsOn: [] } +// Grant the AKS cluster ingress controller pod managed identity with key vault reader role permissions; this allows our ingress controller to pull certificates. resource kvPodMiIngressControllerSecretsUserRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { scope: kv name: '${guid(resourceGroup().id, 'podmi-ingress-controller', keyVaultSecretsUserRole)}' @@ -1073,6 +1086,7 @@ resource kvPodMiIngressControllerSecretsUserRole_roleAssignment 'Microsoft.Autho dependsOn: [] } +// Grant the AKS cluster ingress controller pod managed identity with key vault reader role permissions; this allows our ingress controller to pull certificates resource kvPodMiIngressControllerKeyVaultReader_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { scope: kv name: '${guid(resourceGroup().id, 'podmi-ingress-controller', keyVaultReader)}' @@ -1094,9 +1108,12 @@ module ndEnsureClusterIdentityHasRbacToSelfManagedResources 'nested_EnsureCluste } } +// Enabling Azure Key Vault Private Link support. resource pdzKv 'Microsoft.Network/privateDnsZones@2020-06-01' = { name: 'privatelink.vaultcore.azure.net' location: 'global' + + // Enabling Azure Key Vault Private Link on cluster vnet. resource vnetlnk 'virtualNetworkLinks' = { name: 'to_${vnetName}' location: 'global' @@ -1378,6 +1395,7 @@ resource acrKubeletAcrPullRole_roleAssignment 'Microsoft.Authorization/roleAssig dependsOn: [] } +// Grant the OMS Agent's Managed Identity the metrics publisher role to push alerts resource mcOmsAgentMonitoringMetricsPublisherRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { scope: mc name: guid(mc.id, 'omsagent', monitoringMetricsPublisherRole) @@ -1389,6 +1407,7 @@ resource mcOmsAgentMonitoringMetricsPublisherRole_roleAssignment 'Microsoft.Auth dependsOn: [] } +// Grant the AKS cluster with Managed Identity Operator role permissions over the managed identity used for the ingress controller. Allows it to be assigned to the underlying VMSS. resource miKubeletManagedIdentityOperatorRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { scope: podmiIngressController name: guid(resourceGroup().id, 'podmi-ingress-controller', managedIdentityOperatorRole) @@ -1475,6 +1494,7 @@ resource mc_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-0 dependsOn: [] } +// Ensures that flux add-on (extension) is installed. resource mcFlux_extension 'Microsoft.KubernetesConfiguration/extensions@2021-09-01' = { scope: mc name: 'flux' @@ -1502,6 +1522,7 @@ resource mcFlux_extension 'Microsoft.KubernetesConfiguration/extensions@2021-09- ] } +// Bootstraps your cluster using content from your repo. resource mc_fluxConfiguration 'Microsoft.KubernetesConfiguration/fluxConfigurations@2022-03-01' = { scope: mc name: 'bootstrap' diff --git a/nested_EnsureClusterUserAssignedHasRbacToManageVMSS.bicep b/nested_EnsureClusterUserAssignedHasRbacToManageVMSS.bicep index 53d382cd..0e4201f5 100644 --- a/nested_EnsureClusterUserAssignedHasRbacToManageVMSS.bicep +++ b/nested_EnsureClusterUserAssignedHasRbacToManageVMSS.bicep @@ -2,6 +2,7 @@ param kubeletidentityObjectId string var virtualMachineContributorRole = '${subscription().id}/providers/Microsoft.Authorization/roleDefinitions/9980e02c-c2be-4d73-94e8-173b1dc7cf3c' +// It is required to grant the AKS cluster with Virtual Machine Contributor role permissions over the cluster infrastructure resource group to work with Managed Identities and aad-pod-identity. Otherwise MIC component fails while attempting to update MSI on VMSS cluster nodes resource id 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { name: guid(resourceGroup().id) properties: { From fcf99a9a62dd8a2f7f6bc4108ebea52ec8b63ef2 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Tue, 26 Apr 2022 17:51:02 +0000 Subject: [PATCH 36/54] Address PR Feedback: add lost comments for akv --- cluster-stamp.bicep | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 7fbda45d..0acd7261 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -989,14 +989,14 @@ resource kv 'Microsoft.KeyVault/vaults@2021-11-01-preview' = { name: 'kv-${clusterName}' location: location properties: { - accessPolicies: [] + accessPolicies: [] // Azure RBAC is used instead sku: { family: 'A' name: 'standard' } tenantId: subscription().tenantId networkAcls: { - bypass: 'AzureServices' + bypass: 'AzureServices' // Required for AppGW communication defaultAction: 'Deny' ipRules: [] virtualNetworkRules: [] From 7e2c4e51d151c6dc93ea73751cda45a5205a9606 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Tue, 26 Apr 2022 17:54:33 +0000 Subject: [PATCH 37/54] Address PR Feedback: use native last method instead --- cluster-stamp.bicep | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 0acd7261..250bebdb 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -68,7 +68,7 @@ var logAnalyticsWorkspaceName = 'la-${clusterName}' var defaultAcrName = 'acraks${subRgUniqueString}' var vNetResourceGroup = split(targetVnetResourceId, '/')[4] -var vnetName = split(targetVnetResourceId, '/')[8] +var vnetName = last(split(targetVnetResourceId, '/')) var vnetNodePoolSubnetResourceId = '${targetVnetResourceId}/subnets/snet-clusternodes' var vnetPrivateLinkEndpointsSubnetResourceId = '${targetVnetResourceId}/subnets/snet-privatelinkendpoints' From fdc20225f59d88f3f7d82651cf978bef71884b61 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Tue, 26 Apr 2022 20:52:03 +0000 Subject: [PATCH 38/54] Address PR Feedback: use existing resources when possible, add regions and other minors --- cluster-stamp.bicep | 100 ++++++++++++------ ...dentityHasRbacToSelfManagedResources.bicep | 20 +++- ...usterUserAssignedHasRbacToManageVMSS.bicep | 6 ++ 3 files changed, 89 insertions(+), 37 deletions(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 250bebdb..4d3d1fe5 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -1,3 +1,7 @@ +targetScope = 'resourceGroup' + +/*** PARAMETERS ***/ + @description('The regional network spoke VNet Resource ID that the cluster will be joined to') @minLength(79) param targetVnetResourceId string @@ -52,6 +56,8 @@ param gitOpsBootstrappingRepoHttpsUrl string = 'https://github.com/mspnp/aks-bas @minLength(1) param gitOpsBootstrappingRepoBranch string = 'main' +/*** VARIABLES ***/ + var monitoringMetricsPublisherRole = '${subscription().id}/providers/Microsoft.Authorization/roleDefinitions/3913510d-42f4-4e42-8a64-420c390055eb' var acrPullRole = '${subscription().id}/providers/Microsoft.Authorization/roleDefinitions/7f951dda-4ed3-4680-a7ca-43fe172d538d' var managedIdentityOperatorRole = '${subscription().id}/providers/Microsoft.Authorization/roleDefinitions/f1a07417-d97a-45cb-824c-7a7467783830' @@ -64,14 +70,8 @@ var subRgUniqueString = uniqueString('aks', subscription().subscriptionId, resou var clusterName = 'aks-${subRgUniqueString}' var nodeResourceGroupName = 'rg-${clusterName}-nodepools' -var logAnalyticsWorkspaceName = 'la-${clusterName}' var defaultAcrName = 'acraks${subRgUniqueString}' -var vNetResourceGroup = split(targetVnetResourceId, '/')[4] -var vnetName = last(split(targetVnetResourceId, '/')) -var vnetNodePoolSubnetResourceId = '${targetVnetResourceId}/subnets/snet-clusternodes' -var vnetPrivateLinkEndpointsSubnetResourceId = '${targetVnetResourceId}/subnets/snet-privatelinkendpoints' - var agwName = 'apw-${clusterName}' var aksIngressDomainName = 'aks-ingress.${domainName}' @@ -92,6 +92,38 @@ var policyAssignmentNameEnforceImageSource = guid(policyResourceIdEnforceImageSo var policyAssignmentNameEnforceDefenderInCluster = guid(policyResourceIdEnforceDefenderInCluster, resourceGroup().name, clusterName) var isUsingAzureRBACasKubernetesRBAC = (subscription().tenantId == k8sControlPlaneAuthorizationTenantId) +/*** EXISTING HUB RESOURCES ***/ + +resource acr 'Microsoft.ContainerRegistry/registries@2021-12-01-preview' existing = { + name: defaultAcrName +} + +resource targetResourceGroup 'Microsoft.Resources/resourceGroups@2021-04-01' existing = { + scope: subscription() + name: '${split(targetVnetResourceId,'/')[4]}' +} + +resource targetVirtualNetwork 'Microsoft.Network/virtualNetworks@2021-05-01' existing = { + scope: targetResourceGroup + name: '${last(split(targetVnetResourceId,'/'))}' +} + +resource snetClusterNodes 'Microsoft.Network/virtualNetworks/subnets@2021-05-01' existing = { + parent: targetVirtualNetwork + name: 'snet-clusternodes' +} + +resource snetPrivatelinkendpoints 'Microsoft.Network/virtualNetworks/subnets@2021-05-01' existing = { + parent: targetVirtualNetwork + name: 'snet-privatelinkendpoints' +} + +resource la 'Microsoft.OperationalInsights/workspaces@2021-12-01-preview' existing = { + name: 'la-${clusterName}' +} + +/*** RESOURCES ***/ + resource alaRgRecommendations 'Microsoft.Insights/activityLogAlerts@2020-10-01' = { name: 'AllAzureAdvisorAlert' location: 'Global' @@ -121,7 +153,8 @@ resource alaRgRecommendations 'Microsoft.Insights/activityLogAlerts@2020-10-01' } resource ssPrometheusAll 'Microsoft.OperationalInsights/workspaces/savedSearches@2020-08-01' = { - name: '${logAnalyticsWorkspaceName}/AllPrometheus' + parent: la + name: 'AllPrometheus' properties: { etag: '*' category: 'Prometheus' @@ -133,7 +166,8 @@ resource ssPrometheusAll 'Microsoft.OperationalInsights/workspaces/savedSearches } resource ssPrometheusKuredRequestedReeboot 'Microsoft.OperationalInsights/workspaces/savedSearches@2020-08-01' = { - name: '${logAnalyticsWorkspaceName}/NodeRebootRequested' + parent: la + name: 'NodeRebootRequested' properties: { etag: '*' category: 'Prometheus' @@ -144,15 +178,15 @@ resource ssPrometheusKuredRequestedReeboot 'Microsoft.OperationalInsights/worksp } resource sci 'Microsoft.OperationsManagement/solutions@2015-11-01-preview' = { - name: 'ContainerInsights(${logAnalyticsWorkspaceName})' + name: 'ContainerInsights(${la.name})' location: location properties: { containedResources: [] referencedResources: [] - workspaceResourceId: resourceId('Microsoft.OperationalInsights/workspaces', logAnalyticsWorkspaceName) + workspaceResourceId: la.id } plan: { - name: 'ContainerInsights(${logAnalyticsWorkspaceName})' + name: 'ContainerInsights(${la.name})' product: 'OMSGallery/ContainerInsights' promotionCode: '' publisher: 'Microsoft' @@ -757,15 +791,15 @@ resource maRestartingContainerCount 'Microsoft.Insights/metricAlerts@2018-03-01' } resource skva 'Microsoft.OperationsManagement/solutions@2015-11-01-preview' = { - name: 'KeyVaultAnalytics(${logAnalyticsWorkspaceName})' + name: 'KeyVaultAnalytics(${la.name})' location: location properties: { containedResources: [] referencedResources: [] - workspaceResourceId: resourceId('Microsoft.OperationalInsights/workspaces', logAnalyticsWorkspaceName) + workspaceResourceId: la.id } plan: { - name: 'KeyVaultAnalytics(${logAnalyticsWorkspaceName})' + name: 'KeyVaultAnalytics(${la.name})' product: 'OMSGallery/KeyVaultAnalytics' promotionCode: '' publisher: 'Microsoft' @@ -783,7 +817,7 @@ resource sqrPodFailed 'Microsoft.Insights/scheduledQueryRules@2018-04-16' = { enabled: 'true' source: { query: '//https://docs.microsoft.com/azure/azure-monitor/insights/container-insights-alerts \r\n let endDateTime = now(); let startDateTime = ago(1h); let trendBinSize = 1m; let clusterName = "${clusterName}"; KubePodInventory | where TimeGenerated < endDateTime | where TimeGenerated >= startDateTime | where ClusterName == clusterName | distinct ClusterName, TimeGenerated | summarize ClusterSnapshotCount = count() by bin(TimeGenerated, trendBinSize), ClusterName | join hint.strategy=broadcast ( KubePodInventory | where TimeGenerated < endDateTime | where TimeGenerated >= startDateTime | distinct ClusterName, Computer, PodUid, TimeGenerated, PodStatus | summarize TotalCount = count(), PendingCount = sumif(1, PodStatus =~ "Pending"), RunningCount = sumif(1, PodStatus =~ "Running"), SucceededCount = sumif(1, PodStatus =~ "Succeeded"), FailedCount = sumif(1, PodStatus =~ "Failed") by ClusterName, bin(TimeGenerated, trendBinSize) ) on ClusterName, TimeGenerated | extend UnknownCount = TotalCount - PendingCount - RunningCount - SucceededCount - FailedCount | project TimeGenerated, TotalCount = todouble(TotalCount) / ClusterSnapshotCount, PendingCount = todouble(PendingCount) / ClusterSnapshotCount, RunningCount = todouble(RunningCount) / ClusterSnapshotCount, SucceededCount = todouble(SucceededCount) / ClusterSnapshotCount, FailedCount = todouble(FailedCount) / ClusterSnapshotCount, UnknownCount = todouble(UnknownCount) / ClusterSnapshotCount| summarize AggregatedValue = avg(FailedCount) by bin(TimeGenerated, trendBinSize)' - dataSourceId: resourceId('Microsoft.OperationalInsights/workspaces', logAnalyticsWorkspaceName) + dataSourceId: la.id queryType: 'ResultCount' } schedule: { @@ -1033,7 +1067,7 @@ resource kv_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-0 scope: kv name: 'default' properties: { - workspaceId: resourceId('Microsoft.OperationalInsights/workspaces', logAnalyticsWorkspaceName) + workspaceId: la.id logs: [ { category: 'AuditEvent' @@ -1100,11 +1134,11 @@ resource kvPodMiIngressControllerKeyVaultReader_roleAssignment 'Microsoft.Author module ndEnsureClusterIdentityHasRbacToSelfManagedResources 'nested_EnsureClusterIdentityHasRbacToSelfManagedResources.bicep' = { name: 'EnsureClusterIdentityHasRbacToSelfManagedResources' - scope: resourceGroup(vNetResourceGroup) + scope: targetResourceGroup params: { miClusterControlPlanePrincipalId: miClusterControlPlane.properties.principalId clusterControlPlaneIdentityName: miClusterControlPlane.name - vnetName: vnetName + targetVirtualNetworkName: targetVirtualNetwork.name } } @@ -1115,7 +1149,7 @@ resource pdzKv 'Microsoft.Network/privateDnsZones@2020-06-01' = { // Enabling Azure Key Vault Private Link on cluster vnet. resource vnetlnk 'virtualNetworkLinks' = { - name: 'to_${vnetName}' + name: 'to_${targetVirtualNetwork.name}' location: 'global' properties: { virtualNetwork: { @@ -1132,11 +1166,11 @@ resource peKv 'Microsoft.Network/privateEndpoints@2021-05-01' = { location: location properties: { subnet: { - id: vnetPrivateLinkEndpointsSubnetResourceId + id: snetPrivatelinkendpoints.id } privateLinkServiceConnections: [ { - name: 'to_${vnetName}' + name: 'to_${targetVirtualNetwork.name}' properties: { privateLinkServiceId: kv.id groupIds: [ @@ -1181,7 +1215,7 @@ resource pdzAksIngress 'Microsoft.Network/privateDnsZones@2020-06-01' = { } resource vnetlnk 'virtualNetworkLinks' = { - name: 'to_${vnetName}' + name: 'to_${targetVirtualNetwork.name}' location: 'global' properties: { virtualNetwork: { @@ -1212,7 +1246,7 @@ resource mc 'Microsoft.ContainerService/managedClusters@2022-01-02-preview' = { osType: 'Linux' minCount: 3 maxCount: 4 - vnetSubnetID: vnetNodePoolSubnetResourceId + vnetSubnetID: snetClusterNodes.id enableAutoScaling: true type: 'VirtualMachineScaleSets' mode: 'System' @@ -1242,7 +1276,7 @@ resource mc 'Microsoft.ContainerService/managedClusters@2022-01-02-preview' = { osType: 'Linux' minCount: 2 maxCount: 5 - vnetSubnetID: vnetNodePoolSubnetResourceId + vnetSubnetID: snetClusterNodes.id enableAutoScaling: true type: 'VirtualMachineScaleSets' mode: 'User' @@ -1271,7 +1305,7 @@ resource mc 'Microsoft.ContainerService/managedClusters@2022-01-02-preview' = { omsagent: { enabled: true config: { - logAnalyticsWorkspaceResourceId: resourceId('Microsoft.OperationalInsights/workspaces', logAnalyticsWorkspaceName) + logAnalyticsWorkspaceResourceId: la.id } } aciConnectorLinux: { @@ -1342,7 +1376,7 @@ resource mc 'Microsoft.ContainerService/managedClusters@2022-01-02-preview' = { securityProfile: { azureDefender: { enabled: true - logAnalyticsWorkspaceResourceId: resourceId('Microsoft.OperationalInsights/workspaces', logAnalyticsWorkspaceName) + logAnalyticsWorkspaceResourceId: la.id } } oidcIssuerProfile: { @@ -1379,10 +1413,6 @@ resource mc 'Microsoft.ContainerService/managedClusters@2022-01-02-preview' = { ] } -resource acr 'Microsoft.ContainerRegistry/registries@2021-12-01-preview' existing = { - name: defaultAcrName -} - resource acrKubeletAcrPullRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { scope: acr name: guid(mc.id, acrPullRole) @@ -1471,7 +1501,7 @@ resource mc_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-0 scope: mc name: 'default' properties: { - workspaceId: resourceId('Microsoft.OperationalInsights/workspaces', logAnalyticsWorkspaceName) + workspaceId: la.id logs: [ { category: 'cluster-autoscaler' @@ -1586,7 +1616,7 @@ resource st_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-0 scope: st name: 'default' properties: { - workspaceId: resourceId('Microsoft.OperationalInsights/workspaces', logAnalyticsWorkspaceName) + workspaceId: la.id logs: [ { category: 'DeliveryFailures' @@ -1649,7 +1679,7 @@ resource agw 'Microsoft.Network/applicationGateways@2021-05-01' = { name: 'apw-frontend-ip-configuration' properties: { publicIPAddress: { - id: resourceId(subscription().subscriptionId, vNetResourceGroup, 'Microsoft.Network/publicIpAddresses', 'pip-BU0001A0008-00') + id: resourceId(subscription().subscriptionId, targetResourceGroup.name, 'Microsoft.Network/publicIpAddresses', 'pip-BU0001A0008-00') } } } @@ -1780,7 +1810,7 @@ resource agwdiagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01 scope: agw name: 'default' properties: { - workspaceId: resourceId('Microsoft.OperationalInsights/workspaces', logAnalyticsWorkspaceName) + workspaceId: la.id logs: [ { category: 'ApplicationGatewayAccessLog' @@ -1799,6 +1829,8 @@ resource agwdiagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01 dependsOn: [] } +/*** OUTPUTS ***/ + output aksClusterName string = clusterName output aksIngressControllerPodManagedIdentityResourceId string = podmiIngressController.id output aksIngressControllerPodManagedIdentityClientId string = reference(podmiIngressController.id, '2018-11-30').clientId diff --git a/nested_EnsureClusterIdentityHasRbacToSelfManagedResources.bicep b/nested_EnsureClusterIdentityHasRbacToSelfManagedResources.bicep index d61c6ad4..a60be16e 100644 --- a/nested_EnsureClusterIdentityHasRbacToSelfManagedResources.bicep +++ b/nested_EnsureClusterIdentityHasRbacToSelfManagedResources.bicep @@ -1,17 +1,31 @@ +/*** PARAMETERS ***/ + param miClusterControlPlanePrincipalId string param clusterControlPlaneIdentityName string -param vnetName string +param targetVirtualNetworkName string + +/*** VARIABLES ***/ var networkContributorRole = '${subscription().id}/providers/Microsoft.Authorization/roleDefinitions/4d97b98b-1d4f-4787-a291-c67834d212e7' +/*** EXISTING HUB RESOURCES ***/ + +resource targetVirtualNetwork 'Microsoft.Network/virtualNetworks@2021-05-01' existing = { + name: targetVirtualNetworkName +} + resource snetClusterNodes 'Microsoft.Network/virtualNetworks/subnets@2021-05-01' existing = { - name: '${vnetName}/snet-clusternodes' + parent: targetVirtualNetwork + name: 'snet-clusternodes' } resource snetClusterIngress 'Microsoft.Network/virtualNetworks/subnets@2021-05-01' existing = { - name: '${vnetName}/snet-clusteringressservices' + parent: targetVirtualNetwork + name: 'snet-clusteringressservices' } +/*** RESOURCES ***/ + resource snetClusterNodesMiClusterControlPlaneNetworkContributorRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { scope: snetClusterNodes name: guid(snetClusterNodes.id, networkContributorRole, clusterControlPlaneIdentityName) diff --git a/nested_EnsureClusterUserAssignedHasRbacToManageVMSS.bicep b/nested_EnsureClusterUserAssignedHasRbacToManageVMSS.bicep index 0e4201f5..5db6d0d4 100644 --- a/nested_EnsureClusterUserAssignedHasRbacToManageVMSS.bicep +++ b/nested_EnsureClusterUserAssignedHasRbacToManageVMSS.bicep @@ -1,7 +1,13 @@ +/*** PARAMETERS ***/ + param kubeletidentityObjectId string +/*** VARIABLES ***/ + var virtualMachineContributorRole = '${subscription().id}/providers/Microsoft.Authorization/roleDefinitions/9980e02c-c2be-4d73-94e8-173b1dc7cf3c' +/*** RESOURCES ***/ + // It is required to grant the AKS cluster with Virtual Machine Contributor role permissions over the cluster infrastructure resource group to work with Managed Identities and aad-pod-identity. Otherwise MIC component fails while attempting to update MSI on VMSS cluster nodes resource id 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { name: guid(resourceGroup().id) From 1c205ac8079f9db78c7810f86e9c4c6fafb79cb6 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Tue, 26 Apr 2022 21:54:55 +0000 Subject: [PATCH 39/54] Address PR Feedback: use secretUri instead of interpolation --- cluster-stamp.bicep | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 4d3d1fe5..5e7c7e9f 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -1047,19 +1047,21 @@ resource kv 'Microsoft.KeyVault/vaults@2021-11-01-preview' = { miAppGatewayFrontend podmiIngressController ] +} - resource kvsGatewayPublicCert 'secrets' = { - name: 'gateway-public-cert' - properties: { - value: appGatewayListenerCertificate - } +resource kvsAppGwIngressInternalAksIngressTls 'Microsoft.KeyVault/vaults/secrets@2021-11-01-preview' = { + parent: kv + name: 'appgw-ingress-internal-aks-ingress-tls' + properties: { + value: aksIngressControllerCertificate } +} - resource kvsAppGwIngressInternalAksIngressTls 'secrets' = { - name: 'appgw-ingress-internal-aks-ingress-tls' - properties: { - value: aksIngressControllerCertificate - } +resource kvsGatewayPublicCert 'Microsoft.KeyVault/vaults/secrets@2021-11-01-preview' = { + parent: kv + name: 'gateway-public-cert' + properties: { + value: appGatewayListenerCertificate } } @@ -1660,7 +1662,7 @@ resource agw 'Microsoft.Network/applicationGateways@2021-05-01' = { { name: 'root-cert-wildcard-aks-ingress' properties: { - keyVaultSecretId: '${reference(kv.name).vaultUri}secrets/appgw-ingress-internal-aks-ingress-tls' + keyVaultSecretId: kvsAppGwIngressInternalAksIngressTls.properties.secretUri } } ] @@ -1710,7 +1712,7 @@ resource agw 'Microsoft.Network/applicationGateways@2021-05-01' = { { name: '${agwName}-ssl-certificate' properties: { - keyVaultSecretId: '${reference(kv.name).vaultUri}secrets/gateway-public-cert' + keyVaultSecretId: kvsGatewayPublicCert.properties.secretUri } } ] From 56efddef751b0a9c5036f296ab2f65083019dbeb Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Tue, 26 Apr 2022 22:05:24 +0000 Subject: [PATCH 40/54] Address PR Feedback: fix wrong dependency --- cluster-stamp.bicep | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 5e7c7e9f..eeffada8 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -1411,7 +1411,7 @@ resource mc 'Microsoft.ContainerService/managedClusters@2022-01-02-preview' = { peKv kvPodMiIngressControllerKeyVaultReader_roleAssignment - kvMiAppGatewayFrontendSecretsUserRole_roleAssignment + kvPodMiIngressControllerSecretsUserRole_roleAssignment ] } From b5574ecbfb8250804ba063928c35a3a57ea0af66 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Tue, 26 Apr 2022 22:12:07 +0000 Subject: [PATCH 41/54] Address PR Feedback: add targetScope to modules --- nested_EnsureClusterIdentityHasRbacToSelfManagedResources.bicep | 2 ++ nested_EnsureClusterUserAssignedHasRbacToManageVMSS.bicep | 2 ++ 2 files changed, 4 insertions(+) diff --git a/nested_EnsureClusterIdentityHasRbacToSelfManagedResources.bicep b/nested_EnsureClusterIdentityHasRbacToSelfManagedResources.bicep index a60be16e..c1a42761 100644 --- a/nested_EnsureClusterIdentityHasRbacToSelfManagedResources.bicep +++ b/nested_EnsureClusterIdentityHasRbacToSelfManagedResources.bicep @@ -1,3 +1,5 @@ +targetScope = 'resourceGroup' + /*** PARAMETERS ***/ param miClusterControlPlanePrincipalId string diff --git a/nested_EnsureClusterUserAssignedHasRbacToManageVMSS.bicep b/nested_EnsureClusterUserAssignedHasRbacToManageVMSS.bicep index 5db6d0d4..51e20dbf 100644 --- a/nested_EnsureClusterUserAssignedHasRbacToManageVMSS.bicep +++ b/nested_EnsureClusterUserAssignedHasRbacToManageVMSS.bicep @@ -1,3 +1,5 @@ +targetScope = 'resourceGroup' + /*** PARAMETERS ***/ param kubeletidentityObjectId string From e1bfd7223e7becc1c9e6c2d4d7231154caedcd9e Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Tue, 26 Apr 2022 22:16:38 +0000 Subject: [PATCH 42/54] Revert "add warning while usin gforks" This reverts commit a1d83e837b0c9cc0ee26cca665470e33ba574193. --- 06-aks-cluster.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/06-aks-cluster.md b/06-aks-cluster.md index 39cee73d..14498870 100644 --- a/06-aks-cluster.md +++ b/06-aks-cluster.md @@ -12,8 +12,6 @@ Now that your [ACR instance is deployed and ready to support cluster bootstrappi GITOPS_REPOURL=$(git config --get remote.origin.url) ``` - > :warning: If you are using your own fork, please ensure this repo is reacheable as well as public accessible from your spoke vnet, and check you are assigning its web URL (`HTTPS`). - 1. Deploy the cluster ARM template. :exclamation: By default, this deployment will allow unrestricted access to your cluster's API Server. You can limit access to the API Server to a set of well-known IP addresses (i.,e. a jump box subnet (connected to by Azure Bastion), build agents, or any other networks you'll administer the cluster from) by setting the `clusterAuthorizedIPRanges` parameter in all deployment options. This setting will also impact traffic originating from within the cluster trying to use the API server, so you will also need to include _all_ of the public IPs used by your egress Azure Firewall. For more information, see [Secure access to the API server using authorized IP address ranges](https://docs.microsoft.com/azure/aks/api-server-authorized-ip-ranges#create-an-aks-cluster-with-api-server-authorized-ip-ranges-enabled). From 09b875bd7d8810ceec234132b62aa733796e1e4e Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Tue, 26 Apr 2022 22:19:53 +0000 Subject: [PATCH 43/54] Aaddress PR Request: make more unique the role assignment name --- nested_EnsureClusterUserAssignedHasRbacToManageVMSS.bicep | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nested_EnsureClusterUserAssignedHasRbacToManageVMSS.bicep b/nested_EnsureClusterUserAssignedHasRbacToManageVMSS.bicep index 51e20dbf..8f9ea236 100644 --- a/nested_EnsureClusterUserAssignedHasRbacToManageVMSS.bicep +++ b/nested_EnsureClusterUserAssignedHasRbacToManageVMSS.bicep @@ -12,7 +12,7 @@ var virtualMachineContributorRole = '${subscription().id}/providers/Microsoft.Au // It is required to grant the AKS cluster with Virtual Machine Contributor role permissions over the cluster infrastructure resource group to work with Managed Identities and aad-pod-identity. Otherwise MIC component fails while attempting to update MSI on VMSS cluster nodes resource id 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { - name: guid(resourceGroup().id) + name: guid(resourceGroup().id, virtualMachineContributorRole) properties: { roleDefinitionId: virtualMachineContributorRole principalId: kubeletidentityObjectId From d50def270923aaea9edd260f825d37f44c9fc3ec Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Tue, 26 Apr 2022 22:42:14 +0000 Subject: [PATCH 44/54] Address PR Request: add descs to params --- ..._EnsureClusterIdentityHasRbacToSelfManagedResources.bicep | 5 +++++ nested_EnsureClusterUserAssignedHasRbacToManageVMSS.bicep | 1 + 2 files changed, 6 insertions(+) diff --git a/nested_EnsureClusterIdentityHasRbacToSelfManagedResources.bicep b/nested_EnsureClusterIdentityHasRbacToSelfManagedResources.bicep index c1a42761..17483153 100644 --- a/nested_EnsureClusterIdentityHasRbacToSelfManagedResources.bicep +++ b/nested_EnsureClusterIdentityHasRbacToSelfManagedResources.bicep @@ -2,8 +2,13 @@ targetScope = 'resourceGroup' /*** PARAMETERS ***/ +@description('The AKS Control Plane Principal Id to be given with Network Contributor Role in different spoke subnets, so it can join VMSS and load balancers resources to them.') param miClusterControlPlanePrincipalId string + +@description('The AKS Control Plane Principal Name to be used to create unique role assignments names.') param clusterControlPlaneIdentityName string + +@description('The regional network spoke VNet Resource name that the cluster is being joined to, so it can be used to discover subnets during role assignments.') param targetVirtualNetworkName string /*** VARIABLES ***/ diff --git a/nested_EnsureClusterUserAssignedHasRbacToManageVMSS.bicep b/nested_EnsureClusterUserAssignedHasRbacToManageVMSS.bicep index 8f9ea236..a23e6b62 100644 --- a/nested_EnsureClusterUserAssignedHasRbacToManageVMSS.bicep +++ b/nested_EnsureClusterUserAssignedHasRbacToManageVMSS.bicep @@ -2,6 +2,7 @@ targetScope = 'resourceGroup' /*** PARAMETERS ***/ +@description('The AKS Kubelet identity Object Id to be given with Virtual Machine Contributor Role to work with Managed Identities and aad-pod-identity') param kubeletidentityObjectId string /*** VARIABLES ***/ From df564606d842da331322be6bf090af1c79623788 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Wed, 27 Apr 2022 15:40:44 +0000 Subject: [PATCH 45/54] remove not required interpolation --- cluster-stamp.bicep | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index eeffada8..ead26654 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -1089,7 +1089,7 @@ resource kv_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-0 // Grant the Azure Application Gateway managed identity with key vault reader role permissions; this allows pulling frontend and backend certificates. resource kvMiAppGatewayFrontendSecretsUserRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { scope: kv - name: '${guid(resourceGroup().id, 'mi-appgateway-frontend', keyVaultSecretsUserRole)}' + name: guid(resourceGroup().id, 'mi-appgateway-frontend', keyVaultSecretsUserRole) properties: { roleDefinitionId: keyVaultSecretsUserRole principalId: miAppGatewayFrontend.properties.principalId @@ -1101,7 +1101,7 @@ resource kvMiAppGatewayFrontendSecretsUserRole_roleAssignment 'Microsoft.Authori // Grant the Azure Application Gateway managed identity with key vault reader role permissions; this allows pulling frontend and backend certificates. resource kvMiAppGatewayFrontendKeyVaultReader_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { scope: kv - name: '${guid(resourceGroup().id, 'mi-appgateway-frontend', keyVaultReader)}' + name: guid(resourceGroup().id, 'mi-appgateway-frontend', keyVaultReader) properties: { roleDefinitionId: keyVaultReader principalId: miAppGatewayFrontend.properties.principalId @@ -1113,7 +1113,7 @@ resource kvMiAppGatewayFrontendKeyVaultReader_roleAssignment 'Microsoft.Authoriz // Grant the AKS cluster ingress controller pod managed identity with key vault reader role permissions; this allows our ingress controller to pull certificates. resource kvPodMiIngressControllerSecretsUserRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { scope: kv - name: '${guid(resourceGroup().id, 'podmi-ingress-controller', keyVaultSecretsUserRole)}' + name: guid(resourceGroup().id, 'podmi-ingress-controller', keyVaultSecretsUserRole) properties: { roleDefinitionId: keyVaultSecretsUserRole principalId: podmiIngressController.properties.principalId @@ -1125,7 +1125,7 @@ resource kvPodMiIngressControllerSecretsUserRole_roleAssignment 'Microsoft.Autho // Grant the AKS cluster ingress controller pod managed identity with key vault reader role permissions; this allows our ingress controller to pull certificates resource kvPodMiIngressControllerKeyVaultReader_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { scope: kv - name: '${guid(resourceGroup().id, 'podmi-ingress-controller', keyVaultReader)}' + name: guid(resourceGroup().id, 'podmi-ingress-controller', keyVaultReader) properties: { roleDefinitionId: keyVaultReader principalId: podmiIngressController.properties.principalId From f7c59c2015fc5ad3e844802f75a99c5a2cf3e24f Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Wed, 27 Apr 2022 15:51:46 +0000 Subject: [PATCH 46/54] keep it nested and use accessor --- cluster-stamp.bicep | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index ead26654..0422f78e 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -1047,21 +1047,19 @@ resource kv 'Microsoft.KeyVault/vaults@2021-11-01-preview' = { miAppGatewayFrontend podmiIngressController ] -} -resource kvsAppGwIngressInternalAksIngressTls 'Microsoft.KeyVault/vaults/secrets@2021-11-01-preview' = { - parent: kv - name: 'appgw-ingress-internal-aks-ingress-tls' - properties: { - value: aksIngressControllerCertificate + resource kvsAppGwIngressInternalAksIngressTls 'secrets' = { + name: 'appgw-ingress-internal-aks-ingress-tls' + properties: { + value: aksIngressControllerCertificate + } } -} - -resource kvsGatewayPublicCert 'Microsoft.KeyVault/vaults/secrets@2021-11-01-preview' = { - parent: kv - name: 'gateway-public-cert' - properties: { - value: appGatewayListenerCertificate + + resource kvsGatewayPublicCert 'secrets' = { + name: 'gateway-public-cert' + properties: { + value: appGatewayListenerCertificate + } } } @@ -1662,7 +1660,7 @@ resource agw 'Microsoft.Network/applicationGateways@2021-05-01' = { { name: 'root-cert-wildcard-aks-ingress' properties: { - keyVaultSecretId: kvsAppGwIngressInternalAksIngressTls.properties.secretUri + keyVaultSecretId: kv::kvsAppGwIngressInternalAksIngressTls.properties.secretUri } } ] @@ -1712,7 +1710,7 @@ resource agw 'Microsoft.Network/applicationGateways@2021-05-01' = { { name: '${agwName}-ssl-certificate' properties: { - keyVaultSecretId: kvsGatewayPublicCert.properties.secretUri + keyVaultSecretId: kv::kvsGatewayPublicCert.properties.secretUri } } ] From 22e2afae6c0f57375a846c446ed08c53e34912ba Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Wed, 27 Apr 2022 16:42:34 +0000 Subject: [PATCH 47/54] Address PR Feedback: replace // by @description() as that gives a better UX --- cluster-stamp.bicep | 38 +++++++++---------- ...usterUserAssignedHasRbacToManageVMSS.bicep | 2 +- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 0422f78e..d060cf0d 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -841,7 +841,7 @@ resource sqrPodFailed 'Microsoft.Insights/scheduledQueryRules@2018-04-16' = { dependsOn: [] } -// Applying the 'AKS Linux Restrictive' policy to the resource group +@description('Applying the "AKS Linux Restrictive" policy to the resource group') resource paAKSLinuxRestrictive 'Microsoft.Authorization/policyAssignments@2021-06-01' = { name: policyAssignmentNameAKSLinuxRestrictive properties: { @@ -865,7 +865,7 @@ resource paAKSLinuxRestrictive 'Microsoft.Authorization/policyAssignments@2021-0 dependsOn: [] } -// Applying the 'Enforce HTTPS ingress in Kubernetes cluster' policy to the resource group. +@description('Applying the "Enforce HTTPS ingress in Kubernetes cluster" policy to the resource group.') resource paEnforceHttpsIngress 'Microsoft.Authorization/policyAssignments@2021-06-01' = { name: policyAssignmentNameEnforceHttpsIngress location: 'global' @@ -885,7 +885,7 @@ resource paEnforceHttpsIngress 'Microsoft.Authorization/policyAssignments@2021-0 dependsOn: [] } -// Applying the 'Enforce internal load balancers in Kubernetes cluster' policy to the resource group. +@description('Applying the "Enforce internal load balancers in Kubernetes cluster" policy to the resource group.') resource paEnforceInternalLoadBalancers 'Microsoft.Authorization/policyAssignments@2021-06-01' = { name: policyAssignmentNameEnforceInternalLoadBalancers properties: { @@ -904,7 +904,7 @@ resource paEnforceInternalLoadBalancers 'Microsoft.Authorization/policyAssignmen dependsOn: [] } -// Applying the 'Kubernetes cluster containers should run with a read only root file system' policy to the resource group. +@description('// Applying the "Kubernetes cluster containers should run with a read only root file system" policy to the resource group.') resource paRoRootFilesystem 'Microsoft.Authorization/policyAssignments@2021-06-01' = { name: policyAssignmentNameRoRootFilesystem properties: { @@ -927,7 +927,7 @@ resource paRoRootFilesystem 'Microsoft.Authorization/policyAssignments@2021-06-0 dependsOn: [] } -// Applying the 'Container Images Resource Limits' policy at the resource group level. +@description('Applying the "Container Images Resource Limits" policy at the resource group level.') resource paEnforceResourceLimits 'Microsoft.Authorization/policyAssignments@2021-06-01' = { name: policyAssignmentNameEnforceResourceLimits dependsOn: [] @@ -958,7 +958,7 @@ resource paEnforceResourceLimits 'Microsoft.Authorization/policyAssignments@2021 } } -// Applying the 'Allowed Container Images' regex policy at the resource group level. If all images are pull into your ARC instance as described in these instructions you can remove the docker.io entries. +@description('Applying the "Allowed Container Images" regex policy at the resource group level. If all images are pull into your ARC instance as described in these instructions you can remove the docker.io entries.') resource paEnforceImageSource 'Microsoft.Authorization/policyAssignments@2021-06-01' = { name: policyAssignmentNameEnforceImageSource properties: { @@ -984,7 +984,7 @@ resource paEnforceImageSource 'Microsoft.Authorization/policyAssignments@2021-06 dependsOn: [] } -// Applying the 'Azure Kubernetes Service clusters should have Defender profile enabled' policy at the resource group level. +@description('Applying the "Azure Kubernetes Service clusters should have Defender profile enabled" policy at the resource group level.') resource paEnforceDefenderInCluster 'Microsoft.Authorization/policyAssignments@2021-06-01' = { name: policyAssignmentNameEnforceDefenderInCluster properties: { @@ -1001,19 +1001,19 @@ resource paEnforceDefenderInCluster 'Microsoft.Authorization/policyAssignments@2 dependsOn: [] } -// The control plane identity used by the cluster. Used for networking access (VNET joining and DNS updating) +@description('The control plane identity used by the cluster. Used for networking access (VNET joining and DNS updating)') resource miClusterControlPlane 'Microsoft.ManagedIdentity/userAssignedIdentities@2018-11-30' = { name: 'mi-${clusterName}-controlplane' location: location } -// User Managed Identity that App Gateway is assigned. Used for Azure Key Vault Access. +@description('User Managed Identity that App Gateway is assigned. Used for Azure Key Vault Access.') resource miAppGatewayFrontend 'Microsoft.ManagedIdentity/userAssignedIdentities@2018-11-30' = { name: 'mi-appgateway-frontend' location: location } -// User Managed Identity for the cluster's ingress controller pods. Used for Azure Key Vault Access +@description('User Managed Identity for the cluster\'s ingress controller pods. Used for Azure Key Vault Access') resource podmiIngressController 'Microsoft.ManagedIdentity/userAssignedIdentities@2018-11-30' = { name: 'podmi-ingress-controller' location: location @@ -1084,7 +1084,7 @@ resource kv_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-0 dependsOn: [] } -// Grant the Azure Application Gateway managed identity with key vault reader role permissions; this allows pulling frontend and backend certificates. +@description('Grant the Azure Application Gateway managed identity with key vault reader role permissions; this allows pulling frontend and backend certificates.') resource kvMiAppGatewayFrontendSecretsUserRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { scope: kv name: guid(resourceGroup().id, 'mi-appgateway-frontend', keyVaultSecretsUserRole) @@ -1096,7 +1096,7 @@ resource kvMiAppGatewayFrontendSecretsUserRole_roleAssignment 'Microsoft.Authori dependsOn: [] } -// Grant the Azure Application Gateway managed identity with key vault reader role permissions; this allows pulling frontend and backend certificates. +@description('Grant the Azure Application Gateway managed identity with key vault reader role permissions; this allows pulling frontend and backend certificates.') resource kvMiAppGatewayFrontendKeyVaultReader_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { scope: kv name: guid(resourceGroup().id, 'mi-appgateway-frontend', keyVaultReader) @@ -1108,7 +1108,7 @@ resource kvMiAppGatewayFrontendKeyVaultReader_roleAssignment 'Microsoft.Authoriz dependsOn: [] } -// Grant the AKS cluster ingress controller pod managed identity with key vault reader role permissions; this allows our ingress controller to pull certificates. +@description('Grant the AKS cluster ingress controller pod managed identity with key vault reader role permissions; this allows our ingress controller to pull certificates.') resource kvPodMiIngressControllerSecretsUserRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { scope: kv name: guid(resourceGroup().id, 'podmi-ingress-controller', keyVaultSecretsUserRole) @@ -1120,7 +1120,7 @@ resource kvPodMiIngressControllerSecretsUserRole_roleAssignment 'Microsoft.Autho dependsOn: [] } -// Grant the AKS cluster ingress controller pod managed identity with key vault reader role permissions; this allows our ingress controller to pull certificates +@description('Grant the AKS cluster ingress controller pod managed identity with key vault reader role permissions; this allows our ingress controller to pull certificates') resource kvPodMiIngressControllerKeyVaultReader_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { scope: kv name: guid(resourceGroup().id, 'podmi-ingress-controller', keyVaultReader) @@ -1142,7 +1142,7 @@ module ndEnsureClusterIdentityHasRbacToSelfManagedResources 'nested_EnsureCluste } } -// Enabling Azure Key Vault Private Link support. +@description('Enabling Azure Key Vault Private Link support.') resource pdzKv 'Microsoft.Network/privateDnsZones@2020-06-01' = { name: 'privatelink.vaultcore.azure.net' location: 'global' @@ -1425,7 +1425,7 @@ resource acrKubeletAcrPullRole_roleAssignment 'Microsoft.Authorization/roleAssig dependsOn: [] } -// Grant the OMS Agent's Managed Identity the metrics publisher role to push alerts +@description('Grant the OMS Agent\'s Managed Identity the metrics publisher role to push alerts') resource mcOmsAgentMonitoringMetricsPublisherRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { scope: mc name: guid(mc.id, 'omsagent', monitoringMetricsPublisherRole) @@ -1437,7 +1437,7 @@ resource mcOmsAgentMonitoringMetricsPublisherRole_roleAssignment 'Microsoft.Auth dependsOn: [] } -// Grant the AKS cluster with Managed Identity Operator role permissions over the managed identity used for the ingress controller. Allows it to be assigned to the underlying VMSS. +@description('Grant the AKS cluster with Managed Identity Operator role permissions over the managed identity used for the ingress controller. Allows it to be assigned to the underlying VMSS.') resource miKubeletManagedIdentityOperatorRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { scope: podmiIngressController name: guid(resourceGroup().id, 'podmi-ingress-controller', managedIdentityOperatorRole) @@ -1524,7 +1524,7 @@ resource mc_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-0 dependsOn: [] } -// Ensures that flux add-on (extension) is installed. +@description('Ensures that flux add-on (extension) is installed.') resource mcFlux_extension 'Microsoft.KubernetesConfiguration/extensions@2021-09-01' = { scope: mc name: 'flux' @@ -1552,7 +1552,7 @@ resource mcFlux_extension 'Microsoft.KubernetesConfiguration/extensions@2021-09- ] } -// Bootstraps your cluster using content from your repo. +@description('Bootstraps your cluster using content from your repo.') resource mc_fluxConfiguration 'Microsoft.KubernetesConfiguration/fluxConfigurations@2022-03-01' = { scope: mc name: 'bootstrap' diff --git a/nested_EnsureClusterUserAssignedHasRbacToManageVMSS.bicep b/nested_EnsureClusterUserAssignedHasRbacToManageVMSS.bicep index a23e6b62..b4083682 100644 --- a/nested_EnsureClusterUserAssignedHasRbacToManageVMSS.bicep +++ b/nested_EnsureClusterUserAssignedHasRbacToManageVMSS.bicep @@ -11,7 +11,7 @@ var virtualMachineContributorRole = '${subscription().id}/providers/Microsoft.Au /*** RESOURCES ***/ -// It is required to grant the AKS cluster with Virtual Machine Contributor role permissions over the cluster infrastructure resource group to work with Managed Identities and aad-pod-identity. Otherwise MIC component fails while attempting to update MSI on VMSS cluster nodes +@description('It is required to grant the AKS cluster with Virtual Machine Contributor role permissions over the cluster infrastructure resource group to work with Managed Identities and aad-pod-identity. Otherwise MIC component fails while attempting to update MSI on VMSS cluster nodes') resource id 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { name: guid(resourceGroup().id, virtualMachineContributorRole) properties: { From 13df09aaef9510be8cb31380395d719f6703e682 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Wed, 27 Apr 2022 16:58:17 +0000 Subject: [PATCH 48/54] Address PR Feedback: min/max lenghts are applicable in our example --- ..._EnsureClusterIdentityHasRbacToSelfManagedResources.bicep | 5 +++++ nested_EnsureClusterUserAssignedHasRbacToManageVMSS.bicep | 2 ++ 2 files changed, 7 insertions(+) diff --git a/nested_EnsureClusterIdentityHasRbacToSelfManagedResources.bicep b/nested_EnsureClusterIdentityHasRbacToSelfManagedResources.bicep index 17483153..8377c84b 100644 --- a/nested_EnsureClusterIdentityHasRbacToSelfManagedResources.bicep +++ b/nested_EnsureClusterIdentityHasRbacToSelfManagedResources.bicep @@ -3,12 +3,17 @@ targetScope = 'resourceGroup' /*** PARAMETERS ***/ @description('The AKS Control Plane Principal Id to be given with Network Contributor Role in different spoke subnets, so it can join VMSS and load balancers resources to them.') +@minLength(36) +@maxLength(36) param miClusterControlPlanePrincipalId string @description('The AKS Control Plane Principal Name to be used to create unique role assignments names.') +@minLength(3) +@maxLength(128) param clusterControlPlaneIdentityName string @description('The regional network spoke VNet Resource name that the cluster is being joined to, so it can be used to discover subnets during role assignments.') +@minLength(1) param targetVirtualNetworkName string /*** VARIABLES ***/ diff --git a/nested_EnsureClusterUserAssignedHasRbacToManageVMSS.bicep b/nested_EnsureClusterUserAssignedHasRbacToManageVMSS.bicep index b4083682..0d983bc6 100644 --- a/nested_EnsureClusterUserAssignedHasRbacToManageVMSS.bicep +++ b/nested_EnsureClusterUserAssignedHasRbacToManageVMSS.bicep @@ -3,6 +3,8 @@ targetScope = 'resourceGroup' /*** PARAMETERS ***/ @description('The AKS Kubelet identity Object Id to be given with Virtual Machine Contributor Role to work with Managed Identities and aad-pod-identity') +@minLength(36) +@maxLength(36) param kubeletidentityObjectId string /*** VARIABLES ***/ From e1c9e6739cbfb85369a6a623b2ac17c7ac1eb3b7 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Wed, 27 Apr 2022 19:07:24 +0000 Subject: [PATCH 49/54] use cluster resource props instead of reference to get the identity --- cluster-stamp.bicep | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index d060cf0d..2ec6baae 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -1419,7 +1419,7 @@ resource acrKubeletAcrPullRole_roleAssignment 'Microsoft.Authorization/roleAssig properties: { roleDefinitionId: acrPullRole description: 'Allows AKS to pull container images from this ACR instance.' - principalId: reference(mc.id, '2020-12-01').identityProfile.kubeletidentity.objectId + principalId: mc.properties.identityProfile.kubeletidentity.objectId principalType: 'ServicePrincipal' } dependsOn: [] @@ -1431,7 +1431,7 @@ resource mcOmsAgentMonitoringMetricsPublisherRole_roleAssignment 'Microsoft.Auth name: guid(mc.id, 'omsagent', monitoringMetricsPublisherRole) properties: { roleDefinitionId: monitoringMetricsPublisherRole - principalId: reference(mc.id, '2020-12-01').addonProfiles.omsagent.identity.objectId + principalId: mc.properties.addonProfiles.omsagent.identity.objectId principalType: 'ServicePrincipal' } dependsOn: [] @@ -1443,7 +1443,7 @@ resource miKubeletManagedIdentityOperatorRole_roleAssignment 'Microsoft.Authoriz name: guid(resourceGroup().id, 'podmi-ingress-controller', managedIdentityOperatorRole) properties: { roleDefinitionId: managedIdentityOperatorRole - principalId: reference(mc.id, '2020-12-01').identityProfile.kubeletidentity.objectId + principalId: mc.properties.identityProfile.kubeletidentity.objectId principalType: 'ServicePrincipal' } dependsOn: [] @@ -1597,7 +1597,7 @@ module ndEnsureClusterUserAssignedHasRbacToManageVMSS 'nested_EnsureClusterUserA name: 'EnsureClusterUserAssignedHasRbacToManageVMSS' scope: resourceGroup(nodeResourceGroupName) params: { - kubeletidentityObjectId: reference(mc.id, '2020-03-01').identityProfile.kubeletidentity.objectId + kubeletidentityObjectId: mc.properties.identityProfile.kubeletidentity.objectId } dependsOn: [] } From 6610084f8b43f7b0758b8fe07e2bcbec04d4b74f Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Wed, 27 Apr 2022 19:27:41 +0000 Subject: [PATCH 50/54] role assign rbac k8s usgin a new ARM json module --- cluster-stamp.bicep | 17 +++--- ...d_EnsureAadA0008ReaderGroupHasK8sRbac.json | 55 +++++++++++++++++++ 2 files changed, 63 insertions(+), 9 deletions(-) create mode 100644 nested_EnsureAadA0008ReaderGroupHasK8sRbac.json diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 2ec6baae..f1ef3687 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -64,7 +64,6 @@ var managedIdentityOperatorRole = '${subscription().id}/providers/Microsoft.Auth var keyVaultReader = '${subscription().id}/providers/Microsoft.Authorization/roleDefinitions/21090545-7ca7-4776-b22c-e363652d74d2' var keyVaultSecretsUserRole = '${subscription().id}/providers/Microsoft.Authorization/roleDefinitions/4633458b-17de-408a-b874-0445c86b69e6' var clusterAdminRoleId = 'b1ff04bb-8a4e-4dc4-8eb5-8693973ce19b' -var clusterReaderRoleId = '7f6c6a51-bcf8-42ba-9220-52d62157d7db' var serviceClusterUserRoleId = '4abbcc35-e782-43d8-92c5-2d3f1bd2253f' var subRgUniqueString = uniqueString('aks', subscription().subscriptionId, resourceGroup().id) @@ -1473,14 +1472,14 @@ resource mcAadAdminGroupServiceClusterUserRole_roleAssignment 'Microsoft.Authori dependsOn: [] } -resource maAadA0008ReaderGroupClusterReaderRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = if (isUsingAzureRBACasKubernetesRBAC && (!(a0008NamespaceReaderAadGroupObjectId == clusterAdminAadGroupObjectId))) { - scope: mc // TODO: reference namespace instead - name: guid('aad-a0008-reader-group', mc.id, a0008NamespaceReaderAadGroupObjectId) - properties: { - roleDefinitionId: '/subscriptions/${subscription().subscriptionId}/providers/Microsoft.Authorization/roleDefinitions/${clusterReaderRoleId}' - principalId: a0008NamespaceReaderAadGroupObjectId - description: 'Members of this group are cluster admins of the a0008 namespace in this cluster.' - principalType: 'Group' +module ndEnsureAadA0008ReaderGroupHasK8sRbac 'nested_EnsureAadA0008ReaderGroupHasK8sRbac.json' = { + name: 'EnsureAadA0008ReaderGroupHasK8sRbac' + scope: targetResourceGroup + params: { + clusterAdminAadGroupObjectId: clusterAdminAadGroupObjectId + a0008NamespaceReaderAadGroupObjectId: a0008NamespaceReaderAadGroupObjectId + k8sControlPlaneAuthorizationTenantId: k8sControlPlaneAuthorizationTenantId + clusterName: clusterName } dependsOn: [] } diff --git a/nested_EnsureAadA0008ReaderGroupHasK8sRbac.json b/nested_EnsureAadA0008ReaderGroupHasK8sRbac.json new file mode 100644 index 00000000..a42dc080 --- /dev/null +++ b/nested_EnsureAadA0008ReaderGroupHasK8sRbac.json @@ -0,0 +1,55 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "0.0.0.5", + "parameters": { + "clusterAdminAadGroupObjectId": { + "type": "string", + "metadata": { + "description": "Azure AD Group in the identified tenant that will be granted the highly privileged cluster-admin role. If Azure RBAC is used, then this group will get a role assignment to Azure RBAC, else it will be assigned directly to the cluster's admin group." + } + }, + "a0008NamespaceReaderAadGroupObjectId": { + "type": "string", + "metadata": { + "description": "Azure AD Group in the identified tenant that will be granted the read only privileges in the a0008 namespace that exists in the cluster. This is only used when Azure RBAC is used for Kubernetes RBAC." + } + }, + "k8sControlPlaneAuthorizationTenantId": { + "type": "string", + "metadata": { + "description": "Your AKS control plane Cluster API authentication tenant" + } + }, + "clusterName": { + "type": "string", + "metadata": { + "description": "Your AKS Cluster name" + } + } + }, + "variables": { + "clusterReaderRoleId": "7f6c6a51-bcf8-42ba-9220-52d62157d7db", + + "isUsingAzureRBACasKubernetesRBAC": "[equals(subscription().tenantId, parameters('k8sControlPlaneAuthorizationTenantId'))]" + }, + "resources": [ + { + "condition": "[and(variables('isUsingAzureRBACasKubernetesRBAC'), not(equals(parameters('a0008NamespaceReaderAadGroupObjectId'), parameters('clusterAdminAadGroupObjectId'))))]", + "type": "Microsoft.Authorization/roleAssignments", + "apiVersion": "2020-04-01-preview", + "name": "[guid('aad-a0008-reader-group', resourceId('Microsoft.ContainerService/managedClusters', parameters('clusterName')), parameters('a0008NamespaceReaderAadGroupObjectId'))]", + "dependsOn": [ + "[resourceId('Microsoft.ContainerService/managedClusters', parameters('clusterName'))]" + ], + "scope": "[concat('/subscriptions/', subscription().subscriptionId, '/resourcegroups/', resourceGroup().name, '/providers/Microsoft.ContainerService/managedClusters/', parameters('clusterName'), '/namespaces/a0008')]", + "properties": { + "roleDefinitionId": "[concat('/subscriptions/', subscription().subscriptionId, '/providers/Microsoft.Authorization/roleDefinitions/', variables('clusterReaderRoleId'))]", + "principalId": "[parameters('a0008NamespaceReaderAadGroupObjectId')]", + "description": "Members of this group are cluster admins of the a0008 namespace in this cluster.", + "principalType": "Group" + } + } + ], + "outputs": { + } +} From c5f614cd193c66d53bf582c806c227f29ef6e5e3 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Wed, 27 Apr 2022 23:38:08 +0000 Subject: [PATCH 51/54] Revert "role assign rbac k8s usgin a new ARM json module" This reverts commit 6610084f8b43f7b0758b8fe07e2bcbec04d4b74f. --- cluster-stamp.bicep | 17 +++--- ...d_EnsureAadA0008ReaderGroupHasK8sRbac.json | 55 ------------------- 2 files changed, 9 insertions(+), 63 deletions(-) delete mode 100644 nested_EnsureAadA0008ReaderGroupHasK8sRbac.json diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index f1ef3687..2ec6baae 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -64,6 +64,7 @@ var managedIdentityOperatorRole = '${subscription().id}/providers/Microsoft.Auth var keyVaultReader = '${subscription().id}/providers/Microsoft.Authorization/roleDefinitions/21090545-7ca7-4776-b22c-e363652d74d2' var keyVaultSecretsUserRole = '${subscription().id}/providers/Microsoft.Authorization/roleDefinitions/4633458b-17de-408a-b874-0445c86b69e6' var clusterAdminRoleId = 'b1ff04bb-8a4e-4dc4-8eb5-8693973ce19b' +var clusterReaderRoleId = '7f6c6a51-bcf8-42ba-9220-52d62157d7db' var serviceClusterUserRoleId = '4abbcc35-e782-43d8-92c5-2d3f1bd2253f' var subRgUniqueString = uniqueString('aks', subscription().subscriptionId, resourceGroup().id) @@ -1472,14 +1473,14 @@ resource mcAadAdminGroupServiceClusterUserRole_roleAssignment 'Microsoft.Authori dependsOn: [] } -module ndEnsureAadA0008ReaderGroupHasK8sRbac 'nested_EnsureAadA0008ReaderGroupHasK8sRbac.json' = { - name: 'EnsureAadA0008ReaderGroupHasK8sRbac' - scope: targetResourceGroup - params: { - clusterAdminAadGroupObjectId: clusterAdminAadGroupObjectId - a0008NamespaceReaderAadGroupObjectId: a0008NamespaceReaderAadGroupObjectId - k8sControlPlaneAuthorizationTenantId: k8sControlPlaneAuthorizationTenantId - clusterName: clusterName +resource maAadA0008ReaderGroupClusterReaderRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = if (isUsingAzureRBACasKubernetesRBAC && (!(a0008NamespaceReaderAadGroupObjectId == clusterAdminAadGroupObjectId))) { + scope: mc // TODO: reference namespace instead + name: guid('aad-a0008-reader-group', mc.id, a0008NamespaceReaderAadGroupObjectId) + properties: { + roleDefinitionId: '/subscriptions/${subscription().subscriptionId}/providers/Microsoft.Authorization/roleDefinitions/${clusterReaderRoleId}' + principalId: a0008NamespaceReaderAadGroupObjectId + description: 'Members of this group are cluster admins of the a0008 namespace in this cluster.' + principalType: 'Group' } dependsOn: [] } diff --git a/nested_EnsureAadA0008ReaderGroupHasK8sRbac.json b/nested_EnsureAadA0008ReaderGroupHasK8sRbac.json deleted file mode 100644 index a42dc080..00000000 --- a/nested_EnsureAadA0008ReaderGroupHasK8sRbac.json +++ /dev/null @@ -1,55 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "0.0.0.5", - "parameters": { - "clusterAdminAadGroupObjectId": { - "type": "string", - "metadata": { - "description": "Azure AD Group in the identified tenant that will be granted the highly privileged cluster-admin role. If Azure RBAC is used, then this group will get a role assignment to Azure RBAC, else it will be assigned directly to the cluster's admin group." - } - }, - "a0008NamespaceReaderAadGroupObjectId": { - "type": "string", - "metadata": { - "description": "Azure AD Group in the identified tenant that will be granted the read only privileges in the a0008 namespace that exists in the cluster. This is only used when Azure RBAC is used for Kubernetes RBAC." - } - }, - "k8sControlPlaneAuthorizationTenantId": { - "type": "string", - "metadata": { - "description": "Your AKS control plane Cluster API authentication tenant" - } - }, - "clusterName": { - "type": "string", - "metadata": { - "description": "Your AKS Cluster name" - } - } - }, - "variables": { - "clusterReaderRoleId": "7f6c6a51-bcf8-42ba-9220-52d62157d7db", - - "isUsingAzureRBACasKubernetesRBAC": "[equals(subscription().tenantId, parameters('k8sControlPlaneAuthorizationTenantId'))]" - }, - "resources": [ - { - "condition": "[and(variables('isUsingAzureRBACasKubernetesRBAC'), not(equals(parameters('a0008NamespaceReaderAadGroupObjectId'), parameters('clusterAdminAadGroupObjectId'))))]", - "type": "Microsoft.Authorization/roleAssignments", - "apiVersion": "2020-04-01-preview", - "name": "[guid('aad-a0008-reader-group', resourceId('Microsoft.ContainerService/managedClusters', parameters('clusterName')), parameters('a0008NamespaceReaderAadGroupObjectId'))]", - "dependsOn": [ - "[resourceId('Microsoft.ContainerService/managedClusters', parameters('clusterName'))]" - ], - "scope": "[concat('/subscriptions/', subscription().subscriptionId, '/resourcegroups/', resourceGroup().name, '/providers/Microsoft.ContainerService/managedClusters/', parameters('clusterName'), '/namespaces/a0008')]", - "properties": { - "roleDefinitionId": "[concat('/subscriptions/', subscription().subscriptionId, '/providers/Microsoft.Authorization/roleDefinitions/', variables('clusterReaderRoleId'))]", - "principalId": "[parameters('a0008NamespaceReaderAadGroupObjectId')]", - "description": "Members of this group are cluster admins of the a0008 namespace in this cluster.", - "principalType": "Group" - } - } - ], - "outputs": { - } -} From a4d3e7f73d65327e22d1e53e863b326c7ac3bbbc Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Thu, 28 Apr 2022 01:11:19 +0000 Subject: [PATCH 52/54] reference namespaces from bicep and use this in roleassignments --- cluster-stamp.bicep | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 2ec6baae..535a711e 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -122,6 +122,11 @@ resource la 'Microsoft.OperationalInsights/workspaces@2021-12-01-preview' existi name: 'la-${clusterName}' } +resource nsA0008 'Microsoft.ContainerService/managedClusters/namespaces@2022-01-02-preview' existing = { + parent: mc + name: 'a0008' +} + /*** RESOURCES ***/ resource alaRgRecommendations 'Microsoft.Insights/activityLogAlerts@2020-10-01' = { @@ -1042,7 +1047,7 @@ resource kv 'Microsoft.KeyVault/vaults@2021-11-01-preview' = { enableSoftDelete: true softDeleteRetentionInDays: 7 createMode: 'default' - } + } dependsOn: [ miAppGatewayFrontend podmiIngressController @@ -1054,7 +1059,7 @@ resource kv 'Microsoft.KeyVault/vaults@2021-11-01-preview' = { value: aksIngressControllerCertificate } } - + resource kvsGatewayPublicCert 'secrets' = { name: 'gateway-public-cert' properties: { @@ -1474,7 +1479,7 @@ resource mcAadAdminGroupServiceClusterUserRole_roleAssignment 'Microsoft.Authori } resource maAadA0008ReaderGroupClusterReaderRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = if (isUsingAzureRBACasKubernetesRBAC && (!(a0008NamespaceReaderAadGroupObjectId == clusterAdminAadGroupObjectId))) { - scope: mc // TODO: reference namespace instead + scope: nsA0008 name: guid('aad-a0008-reader-group', mc.id, a0008NamespaceReaderAadGroupObjectId) properties: { roleDefinitionId: '/subscriptions/${subscription().subscriptionId}/providers/Microsoft.Authorization/roleDefinitions/${clusterReaderRoleId}' From dad6fe32ad1b4b125ff86b01a429133f066cb720 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Thu, 28 Apr 2022 01:14:52 +0000 Subject: [PATCH 53/54] Revert "Address PR Feedback: replace // by @description() as that gives a better UX" This reverts commit 22e2afae6c0f57375a846c446ed08c53e34912ba. --- cluster-stamp.bicep | 38 +++++++++---------- ...usterUserAssignedHasRbacToManageVMSS.bicep | 2 +- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 535a711e..31c49d28 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -846,7 +846,7 @@ resource sqrPodFailed 'Microsoft.Insights/scheduledQueryRules@2018-04-16' = { dependsOn: [] } -@description('Applying the "AKS Linux Restrictive" policy to the resource group') +// Applying the 'AKS Linux Restrictive' policy to the resource group resource paAKSLinuxRestrictive 'Microsoft.Authorization/policyAssignments@2021-06-01' = { name: policyAssignmentNameAKSLinuxRestrictive properties: { @@ -870,7 +870,7 @@ resource paAKSLinuxRestrictive 'Microsoft.Authorization/policyAssignments@2021-0 dependsOn: [] } -@description('Applying the "Enforce HTTPS ingress in Kubernetes cluster" policy to the resource group.') +// Applying the 'Enforce HTTPS ingress in Kubernetes cluster' policy to the resource group. resource paEnforceHttpsIngress 'Microsoft.Authorization/policyAssignments@2021-06-01' = { name: policyAssignmentNameEnforceHttpsIngress location: 'global' @@ -890,7 +890,7 @@ resource paEnforceHttpsIngress 'Microsoft.Authorization/policyAssignments@2021-0 dependsOn: [] } -@description('Applying the "Enforce internal load balancers in Kubernetes cluster" policy to the resource group.') +// Applying the 'Enforce internal load balancers in Kubernetes cluster' policy to the resource group. resource paEnforceInternalLoadBalancers 'Microsoft.Authorization/policyAssignments@2021-06-01' = { name: policyAssignmentNameEnforceInternalLoadBalancers properties: { @@ -909,7 +909,7 @@ resource paEnforceInternalLoadBalancers 'Microsoft.Authorization/policyAssignmen dependsOn: [] } -@description('// Applying the "Kubernetes cluster containers should run with a read only root file system" policy to the resource group.') +// Applying the 'Kubernetes cluster containers should run with a read only root file system' policy to the resource group. resource paRoRootFilesystem 'Microsoft.Authorization/policyAssignments@2021-06-01' = { name: policyAssignmentNameRoRootFilesystem properties: { @@ -932,7 +932,7 @@ resource paRoRootFilesystem 'Microsoft.Authorization/policyAssignments@2021-06-0 dependsOn: [] } -@description('Applying the "Container Images Resource Limits" policy at the resource group level.') +// Applying the 'Container Images Resource Limits' policy at the resource group level. resource paEnforceResourceLimits 'Microsoft.Authorization/policyAssignments@2021-06-01' = { name: policyAssignmentNameEnforceResourceLimits dependsOn: [] @@ -963,7 +963,7 @@ resource paEnforceResourceLimits 'Microsoft.Authorization/policyAssignments@2021 } } -@description('Applying the "Allowed Container Images" regex policy at the resource group level. If all images are pull into your ARC instance as described in these instructions you can remove the docker.io entries.') +// Applying the 'Allowed Container Images' regex policy at the resource group level. If all images are pull into your ARC instance as described in these instructions you can remove the docker.io entries. resource paEnforceImageSource 'Microsoft.Authorization/policyAssignments@2021-06-01' = { name: policyAssignmentNameEnforceImageSource properties: { @@ -989,7 +989,7 @@ resource paEnforceImageSource 'Microsoft.Authorization/policyAssignments@2021-06 dependsOn: [] } -@description('Applying the "Azure Kubernetes Service clusters should have Defender profile enabled" policy at the resource group level.') +// Applying the 'Azure Kubernetes Service clusters should have Defender profile enabled' policy at the resource group level. resource paEnforceDefenderInCluster 'Microsoft.Authorization/policyAssignments@2021-06-01' = { name: policyAssignmentNameEnforceDefenderInCluster properties: { @@ -1006,19 +1006,19 @@ resource paEnforceDefenderInCluster 'Microsoft.Authorization/policyAssignments@2 dependsOn: [] } -@description('The control plane identity used by the cluster. Used for networking access (VNET joining and DNS updating)') +// The control plane identity used by the cluster. Used for networking access (VNET joining and DNS updating) resource miClusterControlPlane 'Microsoft.ManagedIdentity/userAssignedIdentities@2018-11-30' = { name: 'mi-${clusterName}-controlplane' location: location } -@description('User Managed Identity that App Gateway is assigned. Used for Azure Key Vault Access.') +// User Managed Identity that App Gateway is assigned. Used for Azure Key Vault Access. resource miAppGatewayFrontend 'Microsoft.ManagedIdentity/userAssignedIdentities@2018-11-30' = { name: 'mi-appgateway-frontend' location: location } -@description('User Managed Identity for the cluster\'s ingress controller pods. Used for Azure Key Vault Access') +// User Managed Identity for the cluster's ingress controller pods. Used for Azure Key Vault Access resource podmiIngressController 'Microsoft.ManagedIdentity/userAssignedIdentities@2018-11-30' = { name: 'podmi-ingress-controller' location: location @@ -1089,7 +1089,7 @@ resource kv_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-0 dependsOn: [] } -@description('Grant the Azure Application Gateway managed identity with key vault reader role permissions; this allows pulling frontend and backend certificates.') +// Grant the Azure Application Gateway managed identity with key vault reader role permissions; this allows pulling frontend and backend certificates. resource kvMiAppGatewayFrontendSecretsUserRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { scope: kv name: guid(resourceGroup().id, 'mi-appgateway-frontend', keyVaultSecretsUserRole) @@ -1101,7 +1101,7 @@ resource kvMiAppGatewayFrontendSecretsUserRole_roleAssignment 'Microsoft.Authori dependsOn: [] } -@description('Grant the Azure Application Gateway managed identity with key vault reader role permissions; this allows pulling frontend and backend certificates.') +// Grant the Azure Application Gateway managed identity with key vault reader role permissions; this allows pulling frontend and backend certificates. resource kvMiAppGatewayFrontendKeyVaultReader_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { scope: kv name: guid(resourceGroup().id, 'mi-appgateway-frontend', keyVaultReader) @@ -1113,7 +1113,7 @@ resource kvMiAppGatewayFrontendKeyVaultReader_roleAssignment 'Microsoft.Authoriz dependsOn: [] } -@description('Grant the AKS cluster ingress controller pod managed identity with key vault reader role permissions; this allows our ingress controller to pull certificates.') +// Grant the AKS cluster ingress controller pod managed identity with key vault reader role permissions; this allows our ingress controller to pull certificates. resource kvPodMiIngressControllerSecretsUserRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { scope: kv name: guid(resourceGroup().id, 'podmi-ingress-controller', keyVaultSecretsUserRole) @@ -1125,7 +1125,7 @@ resource kvPodMiIngressControllerSecretsUserRole_roleAssignment 'Microsoft.Autho dependsOn: [] } -@description('Grant the AKS cluster ingress controller pod managed identity with key vault reader role permissions; this allows our ingress controller to pull certificates') +// Grant the AKS cluster ingress controller pod managed identity with key vault reader role permissions; this allows our ingress controller to pull certificates resource kvPodMiIngressControllerKeyVaultReader_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { scope: kv name: guid(resourceGroup().id, 'podmi-ingress-controller', keyVaultReader) @@ -1147,7 +1147,7 @@ module ndEnsureClusterIdentityHasRbacToSelfManagedResources 'nested_EnsureCluste } } -@description('Enabling Azure Key Vault Private Link support.') +// Enabling Azure Key Vault Private Link support. resource pdzKv 'Microsoft.Network/privateDnsZones@2020-06-01' = { name: 'privatelink.vaultcore.azure.net' location: 'global' @@ -1430,7 +1430,7 @@ resource acrKubeletAcrPullRole_roleAssignment 'Microsoft.Authorization/roleAssig dependsOn: [] } -@description('Grant the OMS Agent\'s Managed Identity the metrics publisher role to push alerts') +// Grant the OMS Agent's Managed Identity the metrics publisher role to push alerts resource mcOmsAgentMonitoringMetricsPublisherRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { scope: mc name: guid(mc.id, 'omsagent', monitoringMetricsPublisherRole) @@ -1442,7 +1442,7 @@ resource mcOmsAgentMonitoringMetricsPublisherRole_roleAssignment 'Microsoft.Auth dependsOn: [] } -@description('Grant the AKS cluster with Managed Identity Operator role permissions over the managed identity used for the ingress controller. Allows it to be assigned to the underlying VMSS.') +// Grant the AKS cluster with Managed Identity Operator role permissions over the managed identity used for the ingress controller. Allows it to be assigned to the underlying VMSS. resource miKubeletManagedIdentityOperatorRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { scope: podmiIngressController name: guid(resourceGroup().id, 'podmi-ingress-controller', managedIdentityOperatorRole) @@ -1529,7 +1529,7 @@ resource mc_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-0 dependsOn: [] } -@description('Ensures that flux add-on (extension) is installed.') +// Ensures that flux add-on (extension) is installed. resource mcFlux_extension 'Microsoft.KubernetesConfiguration/extensions@2021-09-01' = { scope: mc name: 'flux' @@ -1557,7 +1557,7 @@ resource mcFlux_extension 'Microsoft.KubernetesConfiguration/extensions@2021-09- ] } -@description('Bootstraps your cluster using content from your repo.') +// Bootstraps your cluster using content from your repo. resource mc_fluxConfiguration 'Microsoft.KubernetesConfiguration/fluxConfigurations@2022-03-01' = { scope: mc name: 'bootstrap' diff --git a/nested_EnsureClusterUserAssignedHasRbacToManageVMSS.bicep b/nested_EnsureClusterUserAssignedHasRbacToManageVMSS.bicep index 0d983bc6..43d4225a 100644 --- a/nested_EnsureClusterUserAssignedHasRbacToManageVMSS.bicep +++ b/nested_EnsureClusterUserAssignedHasRbacToManageVMSS.bicep @@ -13,7 +13,7 @@ var virtualMachineContributorRole = '${subscription().id}/providers/Microsoft.Au /*** RESOURCES ***/ -@description('It is required to grant the AKS cluster with Virtual Machine Contributor role permissions over the cluster infrastructure resource group to work with Managed Identities and aad-pod-identity. Otherwise MIC component fails while attempting to update MSI on VMSS cluster nodes') +// It is required to grant the AKS cluster with Virtual Machine Contributor role permissions over the cluster infrastructure resource group to work with Managed Identities and aad-pod-identity. Otherwise MIC component fails while attempting to update MSI on VMSS cluster nodes resource id 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { name: guid(resourceGroup().id, virtualMachineContributorRole) properties: { From c0dac32f88bc0b318fb97ecd09710b003ba23a0e Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Thu, 28 Apr 2022 15:25:58 +0000 Subject: [PATCH 54/54] Address PR Feedback: prevent from concatenating resource ids in bicep --- cluster-stamp.bicep | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 31c49d28..d6a3cd42 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -63,9 +63,9 @@ var acrPullRole = '${subscription().id}/providers/Microsoft.Authorization/roleDe var managedIdentityOperatorRole = '${subscription().id}/providers/Microsoft.Authorization/roleDefinitions/f1a07417-d97a-45cb-824c-7a7467783830' var keyVaultReader = '${subscription().id}/providers/Microsoft.Authorization/roleDefinitions/21090545-7ca7-4776-b22c-e363652d74d2' var keyVaultSecretsUserRole = '${subscription().id}/providers/Microsoft.Authorization/roleDefinitions/4633458b-17de-408a-b874-0445c86b69e6' -var clusterAdminRoleId = 'b1ff04bb-8a4e-4dc4-8eb5-8693973ce19b' -var clusterReaderRoleId = '7f6c6a51-bcf8-42ba-9220-52d62157d7db' -var serviceClusterUserRoleId = '4abbcc35-e782-43d8-92c5-2d3f1bd2253f' +var clusterAdminRole = '${subscription().subscriptionId}/providers/Microsoft.Authorization/roleDefinitions/b1ff04bb-8a4e-4dc4-8eb5-8693973ce19b' +var serviceClusterUserRole = '${subscription().subscriptionId}/providers/Microsoft.Authorization/roleDefinitions/4abbcc35-e782-43d8-92c5-2d3f1bd2253f' +var clusterReaderRole = '${subscription().subscriptionId}/providers/Microsoft.Authorization/roleDefinitions/7f6c6a51-bcf8-42ba-9220-52d62157d7db' var subRgUniqueString = uniqueString('aks', subscription().subscriptionId, resourceGroup().id) var clusterName = 'aks-${subRgUniqueString}' @@ -1458,7 +1458,7 @@ resource mcAadAdminGroupClusterAdminRole_roleAssignment 'Microsoft.Authorization scope: mc name: guid('aad-admin-group', mc.id, clusterAdminAadGroupObjectId) properties: { - roleDefinitionId: '/subscriptions/${subscription().subscriptionId}/providers/Microsoft.Authorization/roleDefinitions/${clusterAdminRoleId}' + roleDefinitionId: clusterAdminRole description: 'Members of this group are cluster admins of this cluster.' principalId: clusterAdminAadGroupObjectId principalType: 'Group' @@ -1470,7 +1470,7 @@ resource mcAadAdminGroupServiceClusterUserRole_roleAssignment 'Microsoft.Authori scope: mc name: guid('aad-admin-group-sc', mc.id, clusterAdminAadGroupObjectId) properties: { - roleDefinitionId: '/subscriptions/${subscription().subscriptionId}/providers/Microsoft.Authorization/roleDefinitions/${serviceClusterUserRoleId}' + roleDefinitionId: serviceClusterUserRole description: 'Members of this group are cluster users of this cluster.' principalId: clusterAdminAadGroupObjectId principalType: 'Group' @@ -1482,7 +1482,7 @@ resource maAadA0008ReaderGroupClusterReaderRole_roleAssignment 'Microsoft.Author scope: nsA0008 name: guid('aad-a0008-reader-group', mc.id, a0008NamespaceReaderAadGroupObjectId) properties: { - roleDefinitionId: '/subscriptions/${subscription().subscriptionId}/providers/Microsoft.Authorization/roleDefinitions/${clusterReaderRoleId}' + roleDefinitionId: clusterReaderRole principalId: a0008NamespaceReaderAadGroupObjectId description: 'Members of this group are cluster admins of the a0008 namespace in this cluster.' principalType: 'Group' @@ -1494,7 +1494,7 @@ resource maAadA0008ReaderGroupServiceClusterUserRole_roleAssignment 'Microsoft.A scope: mc name: guid('aad-a0008-reader-group-sc', mc.id, a0008NamespaceReaderAadGroupObjectId) properties: { - roleDefinitionId: '/subscriptions/${subscription().subscriptionId}/providers/Microsoft.Authorization/roleDefinitions/${serviceClusterUserRoleId}' + roleDefinitionId: serviceClusterUserRole principalId: a0008NamespaceReaderAadGroupObjectId description: 'Members of this group are cluster users of this cluster.' principalType: 'Group'