diff --git a/src/powershell/Public/Deploy-FinOpsHub.ps1 b/src/powershell/Public/Deploy-FinOpsHub.ps1 index 0d67d9714..ab79eadef 100644 --- a/src/powershell/Public/Deploy-FinOpsHub.ps1 +++ b/src/powershell/Public/Deploy-FinOpsHub.ps1 @@ -11,13 +11,13 @@ Deploy-FinOpsHub calls Initialize-FinOpsHubDeployment before deploying the template. .PARAMETER Name - Required. Name of the FinOps hub instance. + Required. Name of the hub. Used to ensure unique resource names. .PARAMETER ResourceGroupName Required. Name of the resource group to deploy to. Will be created if it doesn't exist. .PARAMETER Location - Required. Azure location to execute the deployment from. + Required. Azure location where all resources should be created. See https://aka.ms/azureregions. .PARAMETER Version Optional. Version of the FinOps hub template to use. Default = "latest". @@ -26,15 +26,57 @@ Optional. Indicates that preview releases should also be included. Default = false. .PARAMETER StorageSku - Optional. Storage account SKU. Premium_LRS = Lowest cost, Premium_ZRS = High availability. Note Standard SKUs are not available for Data Lake gen2 storage. Default = "Premium_LRS". - + Optional. Storage SKU to use. LRS = Lowest cost, ZRS = High availability. Note Standard SKUs are not available for Data Lake gen2 storage. Allowed: Premium_LRS, Premium_ZRS. Default: Premium_LRS. + + .PARAMETER EnableInfrastructureEncryption + Optional. Enable infrastructure encryption on the storage account. Default = false. + + .PARAMETER RemoteHubStorageUri + Optional. Storage account to push data to for ingestion into a remote hub. + + .PARAMETER RemoteHubStorageKey + Optional. Storage account key to use when pushing data to a remote hub. + + .PARAMETER DataExplorerName + Optional. Name of the Azure Data Explorer cluster to use for advanced analytics. If empty, Azure Data Explorer will not be deployed. Required to use with Power BI if you have more than $2-5M/mo in costs being monitored. Default: "" (do not use). + + .PARAMETER DataExplorerSku + Optional. Name of the Azure Data Explorer SKU. Default: "Dev(No SLA)_Standard_E2a_v4". + + .PARAMETER DataExplorerCapacity + Optional. Number of nodes to use in the cluster. Allowed values: 1 for the Basic SKU tier and 2-1000 for Standard. Default: 1 for dev/test SKUs, 2 for standard SKUs. + .PARAMETER Tags - Optional. Tags for all resources. + Optional. Tags to apply to all resources. We will also add the cm-resource-parent tag for improved cost roll-ups in Cost Management. + + .PARAMETER TagsByResource + Optional. Tags to apply to resources based on their resource type. Resource type specific tags will be merged with tags for all resources. + + # .PARAMETER ScopesToMonitor + # Optional. List of scope IDs to monitor and ingest cost for. + + # .PARAMETER ExportRetentionInDays + # Optional. Number of days of data to retain in the msexports container. Default: 0. + + # .PARAMETER IngestionRetentionInMonths + # Optional. Number of months of data to retain in the ingestion container. Default: 13. + + .PARAMETER DataExplorerRawRetentionInDays int = 0 + Optional. Number of days of data to retain in the Data Explorer *_raw tables. Default: 0. + + .PARAMETER DataExplorerFinalRetentionInMonths + Optional. Number of months of data to retain in the Data Explorer *_final_v* tables. Default: 13. + + .PARAMETER EnablePublicAccess + Optional. Enable public access to the data lake. Default: true. + + .PARAMETER VirtualNetworkAddressPrefix + Optional. Address space for the workload. A /26 is required for the workload. Default: "10.20.30.0/26". .EXAMPLE - Deploy-FinOpsHub -Name MyHub -ResourceGroupName MyNewResourceGroup -Location westus + Deploy-FinOpsHub -Name MyHub -ResourceGroupName MyNewResourceGroup -Location westus -DataExplorerName MyFinOpsHubCluster - Deploys a FinOps hub instance named MyHub to the MyNewResourceGroup resource group. If the resource group does not exist, it will be created. If the hub already exists, it will be updated to the latest version. + Deploys a FinOps hub instance named MyHub to the MyNewResourceGroup resource group with a new MyFinOpsHubCluster Data Explorer cluster. If the resource group does not exist, it will be created. If the hub already exists, it will be updated to the latest version. .EXAMPLE Deploy-FinOpsHub -Name MyHub -ResourceGroupName MyExistingResourceGroup -Location westus -Version 0.1.1 @@ -75,6 +117,50 @@ function Deploy-FinOpsHub [string] $StorageSku = 'Premium_LRS', + [Parameter()] + [switch] + $EnableInfrastructureEncryption, + + [Parameter()] + [string] + $RemoteHubStorageUri, + + [Parameter()] + [string] + $RemoteHubStorageKey, + + [Parameter()] + [string] + $DataExplorerName, + + [Parameter()] + [ValidateSet('Dev(No SLA)_Standard_E2a_v4', 'Dev(No SLA)_Standard_D11_v2', 'Standard_D11_v2', 'Standard_D12_v2', 'Standard_D13_v2', 'Standard_D14_v2', 'Standard_D16d_v5', 'Standard_D32d_v4', 'Standard_D32d_v5', 'Standard_DS13_v2+1TB_PS', 'Standard_DS13_v2+2TB_PS', 'Standard_DS14_v2+3TB_PS', 'Standard_DS14_v2+4TB_PS', 'Standard_E2a_v4', 'Standard_E2ads_v5', 'Standard_E2d_v4', 'Standard_E2d_v5', 'Standard_E4a_v4', 'Standard_E4ads_v5', 'Standard_E4d_v4', 'Standard_E4d_v5', 'Standard_E8a_v4', 'Standard_E8ads_v5', 'Standard_E8as_v4+1TB_PS', 'Standard_E8as_v4+2TB_PS', 'Standard_E8as_v5+1TB_PS', 'Standard_E8as_v5+2TB_PS', 'Standard_E8d_v4', 'Standard_E8d_v5', 'Standard_E8s_v4+1TB_PS', 'Standard_E8s_v4+2TB_PS', 'Standard_E8s_v5+1TB_PS', 'Standard_E8s_v5+2TB_PS', 'Standard_E16a_v4', 'Standard_E16ads_v5', 'Standard_E16as_v4+3TB_PS', 'Standard_E16as_v4+4TB_PS', 'Standard_E16as_v5+3TB_PS', 'Standard_E16as_v5+4TB_PS', 'Standard_E16d_v4', 'Standard_E16d_v5', 'Standard_E16s_v4+3TB_PS', 'Standard_E16s_v4+4TB_PS', 'Standard_E16s_v5+3TB_PS', 'Standard_E16s_v5+4TB_PS', 'Standard_E64i_v3', 'Standard_E80ids_v4', 'Standard_EC8ads_v5', 'Standard_EC8as_v5+1TB_PS', 'Standard_EC8as_v5+2TB_PS', 'Standard_EC16ads_v5', 'Standard_EC16as_v5+3TB_PS', 'Standard_EC16as_v5+4TB_PS', 'Standard_L4s', 'Standard_L8as_v3', 'Standard_L8s', 'Standard_L8s_v2', 'Standard_L8s_v3', 'Standard_L16as_v3', 'Standard_L16s', 'Standard_L16s_v2', 'Standard_L16s_v3', 'Standard_L32as_v3', 'Standard_L32s_v3')] + [string] + $DataExplorerSku = 'Dev(No SLA)_Standard_D11_v2', + + [Parameter()] + [ValidateRange(1, 1000)] + [int] + $DataExplorerCapacity = 1, + + [Parameter()] + [ValidateRange(0, 9999)] + [int] + $DataExplorerRawRetentionInDays = 0, + + [Parameter()] + [ValidateRange(0, 999)] + [int] + $DataExplorerFinalRetentionInMonths = 13, + + [Parameter()] + [switch] + $DisablePublicAccess, + + [Parameter()] + [string] + $VirtualNetworkAddressPrefix = '10.20.30.0/26', + [Parameter()] [hashtable] $Tags @@ -82,22 +168,24 @@ function Deploy-FinOpsHub try { + # Create resource group if it doesn't exist $resourceGroupObject = Get-AzResourceGroup -Name $ResourceGroupName -ErrorAction 'SilentlyContinue' - if (-not $resourceGroupObject) + if (-not $resourceGroupObject -and (Test-ShouldProcess $PSCmdlet $ResourceGroupName 'CreateResourceGroup')) { - if (Test-ShouldProcess $PSCmdlet $ResourceGroupName 'CreateResourceGroup') - { - $resourceGroupObject = New-AzResourceGroup -Name $ResourceGroupName -Location $Location - } + $resourceGroupObject = New-AzResourceGroup -Name $ResourceGroupName -Location $Location } + # Create folder for download $toolkitPath = Join-Path $env:temp -ChildPath 'FinOpsToolkit' if (Test-ShouldProcess $PSCmdlet $toolkitPath 'CreateTempDirectory') { New-Directory -Path $toolkitPath } + + # Init deployment (register providers) Initialize-FinOpsHubDeployment -WhatIf:$WhatIfPreference + # Download template if (Test-ShouldProcess $PSCmdlet $Version 'DownloadTemplate') { Save-FinOpsHubTemplate -Version $Version -Preview:$Preview -Destination $toolkitPath @@ -114,6 +202,24 @@ function Deploy-FinOpsHub storageSku = $StorageSku } } + + if ($Version -eq 'latest' -or [version]$Version -ge '0.4') + { + $parameterSplat.TemplateParameterObject.Add('remoteHubStorageUri', $RemoteHubStorageUri) + $parameterSplat.TemplateParameterObject.Add('remoteHubStorageKey', $RemoteHubStorageKey) + } + + if ($Version -eq 'latest' -or [version]$Version -ge '0.7') + { + $parameterSplat.TemplateParameterObject.Add('enableInfrastructureEncryption', $EnableInfrastructureEncryption) + $parameterSplat.TemplateParameterObject.Add('dataExplorerName', $DataExplorerName) + $parameterSplat.TemplateParameterObject.Add('dataExplorerSku', $DataExplorerSku) + $parameterSplat.TemplateParameterObject.Add('dataExplorerCapacity', $DataExplorerCapacity) + $parameterSplat.TemplateParameterObject.Add('dataExplorerRawRetentionInDays', $DataExplorerRawRetentionInDays) + $parameterSplat.TemplateParameterObject.Add('dataExplorerFinalRetentionInMonths', $DataExplorerFinalRetentionInMonths) + $parameterSplat.TemplateParameterObject.Add('enablePublicAccess', -not $DisablePublicAccess) + $parameterSplat.TemplateParameterObject.Add('virtualNetworkAddressPrefix', $VirtualNetworkAddressPrefix) + } if ($Tags -and $Tags.Keys.Count -gt 0) { @@ -121,6 +227,7 @@ function Deploy-FinOpsHub } } + # Run the deployment if (Test-ShouldProcess $PSCmdlet $ResourceGroupName 'DeployFinOpsHub') { Write-Verbose -Message ($LocalizedData.Hub_Deploy_Deploy -f $bicepFile.FullName, $resourceGroupObject.ResourceGroupName) @@ -133,6 +240,7 @@ function Deploy-FinOpsHub } finally { + # Clean up downloaded files Remove-Item -Path $toolkitPath -Recurse -Force -ErrorAction 'SilentlyContinue' } } diff --git a/src/powershell/Public/Get-FinOpsHub.ps1 b/src/powershell/Public/Get-FinOpsHub.ps1 index 1d3b4f698..8450f7899 100644 --- a/src/powershell/Public/Get-FinOpsHub.ps1 +++ b/src/powershell/Public/Get-FinOpsHub.ps1 @@ -127,6 +127,7 @@ function Get-FinOpsHub } else { + # TODO: Read version from storage $status = 'Unknown' } diff --git a/src/powershell/Tests/Integration/Toolkit.Tests.ps1 b/src/powershell/Tests/Integration/Toolkit.Tests.ps1 index 4a12a775c..1810009bd 100644 --- a/src/powershell/Tests/Integration/Toolkit.Tests.ps1 +++ b/src/powershell/Tests/Integration/Toolkit.Tests.ps1 @@ -6,8 +6,8 @@ Describe 'Get-FinOpsToolkitVersion' { It 'Should return all known releases' { # Arrange - $plannedRelease = '0.5' - $expected = @('0.5', '0.4', '0.3', '0.2', '0.1.1', '0.1', '0.0.1') + $plannedRelease = '0.6' + $expected = @('0.6', '0.5', '0.4', '0.3', '0.2', '0.1.1', '0.1', '0.0.1') # Act $result = Get-FinOpsToolkitVersion diff --git a/src/scripts/Deploy-Toolkit.ps1 b/src/scripts/Deploy-Toolkit.ps1 index fe62d68b9..e6c9bf3de 100644 --- a/src/scripts/Deploy-Toolkit.ps1 +++ b/src/scripts/Deploy-Toolkit.ps1 @@ -144,16 +144,18 @@ if (Test-Path "$PSScriptRoot/../workbooks/$Template") { # Create resource group if it doesn't exist $rg = Get-AzResourceGroup $ResourceGroup -ErrorAction SilentlyContinue - If ($null -eq $rg) + if ($null -eq $rg) { New-AzResourceGroup ` -Name $ResourceGroup ` -Location $Location ` | Out-Null } - + # Start deployment + Write-Verbose "Deploying $templateFile..." $global:ftkDeployment = New-AzResourceGroupDeployment ` + -DeploymentName "ftk-$templateName".Replace('/', '-') ` -TemplateFile $templateFile ` -TemplateParameterObject $Parameters ` -ResourceGroupName $ResourceGroup ` @@ -176,7 +178,9 @@ if (Test-Path "$PSScriptRoot/../workbooks/$Template") } else { + Write-Verbose "Deploying $templateFile..." $global:ftkDeployment = New-AzSubscriptionDeployment ` + -DeploymentName "ftk-$templateName".Replace('/', '-') ` -TemplateFile $templateFile ` -TemplateParameterObject $Parameters ` -Location $Location ` @@ -193,7 +197,6 @@ if (Test-Path "$PSScriptRoot/../workbooks/$Template") } "tenant" { - $azContext = (Get-AzContext).Tenant Write-Host " → [tenant] $(iff ([string]::IsNullOrWhitespace($azContext.Name)) $azContext.Id $azContext.Name)..." $Parameters.Keys | ForEach-Object { Write-Host " $($_) = $($Parameters[$_])" } @@ -204,7 +207,9 @@ if (Test-Path "$PSScriptRoot/../workbooks/$Template") } else { + Write-Verbose "Deploying $templateFile..." $global:ftkDeployment = New-AzTenantDeployment ` + -DeploymentName "ftk-$templateName".Replace('/', '-') ` -TemplateFile $templateFile ` -TemplateParameterObject $Parameters ` -Location $Location ` diff --git a/src/templates/finops-hub/main.bicep b/src/templates/finops-hub/main.bicep index 285c6ff17..9820fb1ac 100644 --- a/src/templates/finops-hub/main.bicep +++ b/src/templates/finops-hub/main.bicep @@ -23,11 +23,21 @@ param location string = resourceGroup().location @description('Optional. Storage SKU to use. LRS = Lowest cost, ZRS = High availability. Note Standard SKUs are not available for Data Lake gen2 storage. Allowed: Premium_LRS, Premium_ZRS. Default: Premium_LRS.') param storageSku string = 'Premium_LRS' +@description('Optional. Enable infrastructure encryption on the storage account. Default = false.') +param enableInfrastructureEncryption bool = false + +@description('Optional. Storage account to push data to for ingestion into a remote hub.') +param remoteHubStorageUri string = '' + +@description('Optional. Storage account key to use when pushing data to a remote hub.') +@secure() +param remoteHubStorageKey string = '' + @description('Optional. Name of the Azure Data Explorer cluster to use for advanced analytics. If empty, Azure Data Explorer will not be deployed. Required to use with Power BI if you have more than $2-5M/mo in costs being monitored. Default: "" (do not use).') param dataExplorerName string = '' // https://learn.microsoft.com/azure/templates/microsoft.kusto/clusters?pivots=deployment-language-bicep#azuresku -@description('Optional. Name of the Azure Data Explorer SKU. Default: "Dev(No SLA)_Standard_E2a_v4".') +@description('Optional. Name of the Azure Data Explorer SKU. Default: "Dev(No SLA)_Standard_D11_v2".') @allowed([ 'Dev(No SLA)_Standard_E2a_v4' // 2 CPU, 16GB RAM, 24GB cache, $110/mo 'Dev(No SLA)_Standard_D11_v2' // 2 CPU, 14GB RAM, 78GB cache, $121/mo @@ -93,8 +103,8 @@ param dataExplorerName string = '' 'Standard_L16s_v3' 'Standard_L32as_v3' 'Standard_L32s_v3' -]) -param dataExplorerSku string = 'Dev(No SLA)_Standard_E2a_v4' +]) +param dataExplorerSku string = 'Dev(No SLA)_Standard_D11_v2' @description('Optional. Number of nodes to use in the cluster. Allowed values: 1 for the Basic SKU tier and 2-1000 for Standard. Default: 1 for dev/test SKUs, 2 for standard SKUs.') @minValue(1) @@ -122,16 +132,6 @@ param dataExplorerRawRetentionInDays int = 0 @description('Optional. Number of months of data to retain in the Data Explorer *_final_v* tables. Default: 13.') param dataExplorerFinalRetentionInMonths int = 13 -@description('Optional. Storage account to push data to for ingestion into a remote hub.') -param remoteHubStorageUri string = '' - -@description('Optional. Storage account key to use when pushing data to a remote hub.') -@secure() -param remoteHubStorageKey string = '' - -@description('Optional. Enable infrastructure encryption on the storage account. Default = false.') -param enableInfrastructureEncryption bool = false - @description('Optional. Enable public access to the data lake. Default: true.') param enablePublicAccess bool = true diff --git a/src/templates/finops-hub/modules/hub.bicep b/src/templates/finops-hub/modules/hub.bicep index fff7983cd..39ed1511e 100644 --- a/src/templates/finops-hub/modules/hub.bicep +++ b/src/templates/finops-hub/modules/hub.bicep @@ -21,11 +21,21 @@ param location string = resourceGroup().location @description('Optional. Storage SKU to use. LRS = Lowest cost, ZRS = High availability. Note Standard SKUs are not available for Data Lake gen2 storage. Allowed: Premium_LRS, Premium_ZRS. Default: Premium_LRS.') param storageSku string = 'Premium_LRS' +@description('Optional. Enable infrastructure encryption on the storage account. Default = false.') +param enableInfrastructureEncryption bool = false + +@description('Optional. Remote storage account for ingestion dataset.') +param remoteHubStorageUri string = '' + +@description('Optional. Storage account key for remote storage account.') +@secure() +param remoteHubStorageKey string = '' + @description('Optional. Name of the Azure Data Explorer cluster to use for advanced analytics. If empty, Azure Data Explorer will not be deployed. Required to use with Power BI if you have more than $2-5M/mo in costs being monitored. Default: "" (do not use).') param dataExplorerName string = '' // https://learn.microsoft.com/azure/templates/microsoft.kusto/clusters?pivots=deployment-language-bicep#azuresku -@description('Optional. Name of the Azure Data Explorer SKU. Default: "Dev(No SLA)_Standard_E2a_v4".') +@description('Optional. Name of the Azure Data Explorer SKU. Default: "Dev(No SLA)_Standard_D11_v2".') @allowed([ 'Dev(No SLA)_Standard_E2a_v4' // 2 CPU, 16GB RAM, 24GB cache, $110/mo 'Dev(No SLA)_Standard_D11_v2' // 2 CPU, 14GB RAM, 78GB cache, $121/mo @@ -92,7 +102,7 @@ param dataExplorerName string = '' 'Standard_L32as_v3' 'Standard_L32s_v3' ]) -param dataExplorerSku string = 'Dev(No SLA)_Standard_E2a_v4' +param dataExplorerSku string = 'Dev(No SLA)_Standard_D11_v2' @description('Optional. Number of nodes to use in the cluster. Allowed values: 1 for the Basic SKU tier and 2-1000 for Standard. Default: 1 for dev/test SKUs, 2 for standard SKUs.') @minValue(1) @@ -120,25 +130,15 @@ param dataExplorerRawRetentionInDays int = 0 @description('Optional. Number of months of data to retain in the Data Explorer *_final_v* tables. Default: 13.') param dataExplorerFinalRetentionInMonths int = 13 -@description('Optional. Remote storage account for ingestion dataset.') -param remoteHubStorageUri string = '' - -@description('Optional. Storage account key for remote storage account.') -@secure() -param remoteHubStorageKey string = '' +@description('Optional. Enable public access to the data lake. Default: true.') +param enablePublicAccess bool = true @description('Optional. Address space for the workload. A /26 is required for the workload. Default: "10.20.30.0/26".') param virtualNetworkAddressPrefix string = '10.20.30.0/26' -@description('Optional. Enable public access to the data lake. Default: true.') -param enablePublicAccess bool = true - @description('Optional. Enable telemetry to track anonymous module usage trends, monitor for bugs, and improve future releases.') param enableDefaultTelemetry bool = true -@description('Optional. Enable infrastructure encryption on the storage account. Default = false.') -param enableInfrastructureEncryption bool = false - //------------------------------------------------------------------------------ // Variables //------------------------------------------------------------------------------ diff --git a/src/templates/finops-hub/modules/storage.bicep b/src/templates/finops-hub/modules/storage.bicep index 0221503fd..30d10f0ba 100644 --- a/src/templates/finops-hub/modules/storage.bicep +++ b/src/templates/finops-hub/modules/storage.bicep @@ -112,7 +112,6 @@ resource storageAccount 'Microsoft.Storage/storageAccounts@2022-09-01' = { isHnsEnabled: true minimumTlsVersion: 'TLS1_2' allowBlobPublicAccess: false - }) publicNetworkAccess: 'Enabled' networkAcls: { bypass: 'AzureServices'