diff --git a/05-bootstrap-prep.md b/05-bootstrap-prep.md index 342c3dd0..dc744c78 100644 --- a/05-bootstrap-prep.md +++ b/05-bootstrap-prep.md @@ -41,7 +41,7 @@ We'll be bootstrapping this cluster with the Flux GitOps agent as installed as a ```bash # [This takes about four minutes.] - az deployment group create -g rg-bu0001a0008 -f acr-stamp.json -p targetVnetResourceId=${RESOURCEID_VNET_CLUSTERSPOKE_AKS_BASELINE} + az deployment group create -g rg-bu0001a0008 -f acr-stamp.bicep -p targetVnetResourceId=${RESOURCEID_VNET_CLUSTERSPOKE_AKS_BASELINE} ``` 1. Import cluster management images to your container registry. diff --git a/acr-stamp.bicep b/acr-stamp.bicep new file mode 100644 index 00000000..b0d99376 --- /dev/null +++ b/acr-stamp.bicep @@ -0,0 +1,211 @@ +targetScope = 'resourceGroup' + +/*** PARAMETERS ***/ + +@description('The regional network spoke VNet Resource ID that the cluster will be joined to.') +@minLength(79) +param targetVnetResourceId string + +@allowed([ + 'australiaeast' + 'canadacentral' + 'centralus' + 'eastus' + 'eastus2' + 'westus2' + 'francecentral' + 'germanywestcentral' + 'northeurope' + 'southafricanorth' + 'southcentralus' + 'uksouth' + 'westeurope' + 'japaneast' + 'southeastasia' +]) +@description('AKS Service, Node Pool, and supporting services (KeyVault, App Gateway, etc) region. This needs to be the same region as the vnet provided in these parameters.') +param location string = 'eastus2' + +@allowed([ + 'australiasoutheast' + 'canadaeast' + 'eastus2' + 'westus' + 'centralus' + 'westcentralus' + 'francesouth' + 'germanynorth' + 'westeurope' + 'ukwest' + 'northeurope' + 'japanwest' + 'southafricawest' + 'northcentralus' + 'eastasia' + 'eastus' + 'westus2' + 'francecentral' + 'uksouth' + 'japaneast' + 'southeastasia' +]) +@description('For Azure resources that support native geo-redunancy, provide the location the redundant service will have its secondary. Should be different than the location parameter and ideally should be a paired region - https://docs.microsoft.com/azure/best-practices-availability-paired-regions. This region does not need to support availability zones.') +param geoRedundancyLocation string = 'centralus' + +/*** VARIABLES ***/ + +var subRgUniqueString = uniqueString('aks', subscription().subscriptionId, resourceGroup().id) + +/*** EXISTING RESOURCES ***/ + +resource spokeResourceGroup 'Microsoft.Resources/resourceGroups@2021-04-01' existing = { + scope: subscription() + name: '${split(targetVnetResourceId,'/')[4]}' +} + +resource spokeVirtualNetwork 'Microsoft.Network/virtualNetworks@2021-05-01' existing = { + scope: spokeResourceGroup + name: '${last(split(targetVnetResourceId,'/'))}' + + resource snetClusterNodes 'subnets@2021-05-01' existing = { + name: 'snet-clusternodes' + } +} + +/*** RESOURCES ***/ + +// This Log Analytics workspace will be the log sink for all resources in the cluster resource group. This includes ACR, the AKS cluster, Key Vault, etc. It also is the Container Insights log sink for the AKS cluster. +resource laAks 'Microsoft.OperationalInsights/workspaces@2021-06-01' = { + name: 'la-aks-${subRgUniqueString}' + location: location + properties: { + sku: { + name: 'PerGB2018' + } + retentionInDays: 30 + } +} + +// Azure Container Registry will be exposed via Private Link, set up the related Private DNS zone and virtual network link to the spoke. +resource dnsPrivateZoneAcr 'Microsoft.Network/privateDnsZones@2020-06-01' = { + name: 'privatelink.azurecr.io' + location: 'global' + properties: {} + + resource dnsVnetLinkAcrToSpoke 'virtualNetworkLinks@2020-06-01' = { + name: 'to_${spokeVirtualNetwork.name}' + location: 'global' + properties: { + virtualNetwork: { + id: targetVnetResourceId + } + registrationEnabled: false + } + } +} + +// The Container Registry that the AKS cluster will be authorized to use to pull images. +resource acrAks 'Microsoft.ContainerRegistry/registries@2021-09-01' = { + name: 'acraks${subRgUniqueString}' + location: location + sku: { + name: 'Premium' + } + properties: { + adminUserEnabled: false + networkRuleSet: { + defaultAction: 'Deny' + ipRules: [] + } + policies: { + quarantinePolicy: { + status: 'disabled' + } + trustPolicy: { + type: 'Notary' + status: 'disabled' + } + retentionPolicy: { + days: 15 + status: 'enabled' + } + } + publicNetworkAccess: 'Disabled' + encryption: { + status: 'disabled' + } + dataEndpointEnabled: true + networkRuleBypassOptions: 'AzureServices' + zoneRedundancy: 'Disabled' // This Preview feature only supports three regions at this time, and eastus2's paired region (centralus), does not support this. So disabling for now. + } + + resource acrReplication 'replications@2021-09-01' = { + name: geoRedundancyLocation + location: geoRedundancyLocation + properties: {} + } +} + +resource acrAks_diagnosticsSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { + name: 'default' + scope: acrAks + properties: { + workspaceId: laAks.id + metrics: [ + { + timeGrain: 'PT1M' + category: 'AllMetrics' + enabled: true + } + ] + logs: [ + { + categoryGroup: 'allLogs' + enabled: true + } + ] + } +} + +// Expose Azure Container Registry via Private Link, into the cluster nodes subnet. +resource privateEndpointAcrToVnet 'Microsoft.Network/privateEndpoints@2021-05-01' = { + name: 'acr_to_${spokeVirtualNetwork.name}' + location: location + dependsOn: [ + acrAks::acrReplication + ] + properties: { + subnet: { + id: spokeVirtualNetwork::snetClusterNodes.id + } + privateLinkServiceConnections: [ + { + name: 'nodepools' + properties: { + privateLinkServiceId: acrAks.id + groupIds: [ + 'registry' + ] + } + } + ] + } + + resource privateDnsZoneGroupAcr 'privateDnsZoneGroups@2021-05-01' = { + name: 'default' + properties: { + privateDnsZoneConfigs: [ + { + name: 'privatelink-azurecr-io' + properties: { + privateDnsZoneId: dnsPrivateZoneAcr.id + } + } + ] + } + } +} + +/*** OUTPUTS ***/ + +output containerRegistryName string = acrAks.name diff --git a/acr-stamp.json b/acr-stamp.json deleted file mode 100644 index 1455b838..00000000 --- a/acr-stamp.json +++ /dev/null @@ -1,249 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "0.0.0.1", - "parameters": { - "targetVnetResourceId": { - "type": "string", - "minLength": 79, - "metadata": { - "description": "The regional network spoke VNet Resource ID that the cluster will be joined to." - } - }, - "location": { - "defaultValue": "eastus2", - "type": "string", - "allowedValues": [ - "australiaeast", - "canadacentral", - "centralus", - "eastus", - "eastus2", - "westus2", - "francecentral", - "germanywestcentral", - "northeurope", - "southafricanorth", - "southcentralus", - "uksouth", - "westeurope", - "japaneast", - "southeastasia" - ], - "metadata": { - "description": "AKS Service, Node Pool, and supporting services (KeyVault, App Gateway, etc) region. This needs to be the same region as the vnet provided in these parameters." - } - }, - "geoRedundancyLocation": { - "defaultValue": "centralus", - "type": "string", - "allowedValues": [ - "australiasoutheast", - "canadaeast", - "eastus2", - "westus", - "centralus", - "westcentralus", - "francesouth", - "germanynorth", - "westeurope", - "ukwest", - "northeurope", - "japanwest", - "southafricawest", - "northcentralus", - "eastasia", - "eastus", - "westus2", - "francecentral", - "uksouth", - "japaneast", - "southeastasia" - ], - "metadata": { - "description": "For Azure resources that support native geo-redunancy, provide the location the redundant service will have its secondary. Should be different than the location parameter and ideally should be a paired region - https://docs.microsoft.com/azure/best-practices-availability-paired-regions. This region does not need to support availability zones." - } - } - }, - "variables": { - "subRgUniqueString": "[uniqueString('aks', subscription().subscriptionId, resourceGroup().id)]", - - "clusterName": "[concat('aks-', variables('subRgUniqueString'))]", - "logAnalyticsWorkspaceName": "[concat('la-', variables('clusterName'))]", - "defaultAcrName": "[concat('acraks', variables('subRgUniqueString'))]", - - "vnetName": "[split(parameters('targetVnetResourceId'),'/')[8]]", - "vnetAcrPrivateEndpointSubnetResourceId": "[concat(parameters('targetVnetResourceId'), '/subnets/snet-clusternodes')]", - "acrPrivateDnsZonesName": "privatelink.azurecr.io" - }, - "resources": [ - { - "type": "Microsoft.OperationalInsights/workspaces", - "apiVersion": "2020-10-01", - "name": "[variables('logAnalyticsWorkspaceName')]", - "location": "[parameters('location')]", - "properties": { - "sku": { - "name": "PerGB2018" - }, - "retentionInDays": 30 - } - }, - { - "type": "Microsoft.Network/privateDnsZones", - "apiVersion": "2020-06-01", - "name": "[variables('acrPrivateDnsZonesName')]", - "location": "global", - "comments": "Enabling Azure Container Registry Private Link on vnet.", - "properties": {}, - "resources": [ - { - "type": "virtualNetworkLinks", - "apiVersion": "2020-06-01", - "name": "[concat('to_', variables('vnetName'))]", - "location": "global", - "comments": "Enabling cluster vnet private zone DNS lookup - used by cluster vnet for direct DNS queries (ones not proxied via the hub).", - "dependsOn": [ - "[resourceId('Microsoft.Network/privateDnsZones', variables('acrPrivateDnsZonesName'))]" - ], - "properties": { - "virtualNetwork": { - "id": "[parameters('targetVnetResourceId')]" - }, - "registrationEnabled": false - } - } - ] - }, - { - "type": "Microsoft.ContainerRegistry/registries", - "apiVersion": "2020-11-01-preview", - "name": "[variables('defaultAcrName')]", - "location": "[parameters('location')]", - "sku": { - "name": "Premium" - }, - "properties": { - "adminUserEnabled": false, - "networkRuleSet": { - "defaultAction": "Deny", - "virtualNetworkRules": [], - "ipRules": [] - }, - "policies": { - "quarantinePolicy": { - "status": "disabled" - }, - "trustPolicy": { - "type": "Notary", - "status": "disabled" - }, - "retentionPolicy": { - "days": 15, - "status": "enabled" - } - }, - "publicNetworkAccess": "Disabled", - "encryption": { - "status": "disabled" - }, - "dataEndpointEnabled": true, - "networkRuleBypassOptions": "AzureServices", - "zoneRedundancy": "Disabled" // This Preview feature only supports three regions at this time, and eastus2's paired region (centralus), does not support this. So disabling for now. - }, - "resources": [ - { - "type": "replications", - "apiVersion": "2020-11-01-preview", - "name": "[parameters('geoRedundancyLocation')]", - "location": "[parameters('geoRedundancyLocation')]", - "dependsOn": [ - "[variables('defaultAcrName')]" - ], - "properties": {} - }, - { - "type": "providers/diagnosticSettings", - "apiVersion": "2017-05-01-preview", - "name": "Microsoft.Insights/default", - "dependsOn": [ - "[resourceId('Microsoft.ContainerRegistry/registries', variables('defaultAcrName'))]", - "[resourceId('Microsoft.OperationalInsights/workspaces', variables('logAnalyticsWorkspaceName'))]" - ], - "properties": { - "workspaceId": "[resourceId('Microsoft.OperationalInsights/workspaces', variables('logAnalyticsWorkspaceName'))]", - "metrics": [ - { - "timeGrain": "PT1M", - "category": "AllMetrics", - "enabled": true - } - ], - "logs": [ - { - "category": "ContainerRegistryRepositoryEvents", - "enabled": true - }, - { - "category": "ContainerRegistryLoginEvents", - "enabled": true - } - ] - } - } - ] - }, - { - "type": "Microsoft.Network/privateEndpoints", - "apiVersion": "2020-11-01", - "name": "[concat('acr_to_', variables('vnetName'))]", - "location": "[parameters('location')]", - "dependsOn": [ - "[resourceId('Microsoft.ContainerRegistry/registries/replications', variables('defaultAcrName'), parameters('geoRedundancyLocation'))]" - ], - "properties": { - "subnet": { - "id": "[variables('vnetAcrPrivateEndpointSubnetResourceId')]" - }, - "privateLinkServiceConnections": [ - { - "name": "nodepools", - "properties": { - "privateLinkServiceId": "[resourceId('Microsoft.ContainerRegistry/registries', variables('defaultAcrName'))]", - "groupIds": [ - "registry" - ] - } - } - ] - }, - "resources": [ - { - "type": "privateDnsZoneGroups", - "apiVersion": "2020-11-01", - "name": "default", - "location": "[parameters('location')]", - "dependsOn": [ - "[resourceId('Microsoft.Network/privateEndpoints', concat('acr_to_', variables('vnetName')))]", - "[resourceId('Microsoft.Network/privateDnsZones', variables('acrPrivateDnsZonesName'))]" - ], - "properties": { - "privateDnsZoneConfigs": [ - { - "name": "privatelink-azurecr-io", - "properties": { - "privateDnsZoneId": "[resourceId('Microsoft.Network/privateDnsZones', variables('acrPrivateDnsZonesName'))]" - } - } - ] - } - } - ] - } - ], - "outputs": { - "containerRegistryName": { - "type": "string", - "value": "[variables('defaultAcrName')]" - } - } -}