Bicep (#390)

* see change log file

.gitignore

@@ -225,3 +225,4 @@ Command_scratch.txt
 /src/SqlBuildManager.Console/ContainerApp/parms.json
 /src/SqlBuildManager.Console/ContainerApp/parms2.json
 src/env.txt
+/scripts/templates/test.ps1

CHANGELOG.md

@@ -1,5 +1,12 @@
 # SQL Build Manager Change Log
 
+### Version 15.4.1
+- *UPDATED:* Changed the test environment creation from az CLI commands to Bicep templates/modules
+- *UPDATED:* Modified/refactored internal handling of Manged Identity Client ID
+- *ADDED:* New EventHub logging type of `ScriptErrors` which will event out any script error messages that occur during execution
+- *UPDATED:* Optional feature to have Eventhub monitoring to attempt to create a custom consumer group to avoid any event read conflicts. The running identity must have "Event Hub Owner" RBAC priviledges and there are also two new arguments `--ehrg`/`--eventhubresourcegroup` and `--ehsub`/`--eventhubsubscriptionid` which would need to be provided. If these are not all met, it will continue to use the existing $Default. The custom consumer group will be deleted after run is complete. 
+- *UPDATED:* `sbm utility eventhub` now accepts a `--jobname=all` to query all events, `--timeout` in seconds for how long to continue to monitor after the last event is received and `--eventhubresourcegroup`/`--ehrg`,  `--eventhubsubscriptionid`/`--ehsub`, `--storageaccountname` and `--storageaccountkey` arguments to support the new optional feature to create a custom consumer group. The storage account is used for the consumer checkpointing 
+
 ### Version 15.4.0
 - *FIXED:* Corrected bug #386 - `sbm batch *` were not properly handling the `--authtype ManagedIdentity` argument ()
 - *FIXED:* Corrected bug #387 - The path for the `--targetdacpac` was not getting trimmed to just the file name when getting sent to the Batch nodes

scripts/templates/Batch/create_batch_account.ps1

@@ -1,15 +1,28 @@
 param
 (
+    [string] $prefix,
     [string] $resourceGroupName,
     [string] $batchAccountName,
     [string] $storageAccountName,
     [string] $userAssignedIdentity,
-    [string] $location
+    [string] $location,
+    [string] $path = "..\..\..\src\TestConfig"
 )
 Write-Host "Create Batch Account: $batchAccountName"  -ForegroundColor Cyan
-# $identity = az identity show --name $userAssignedIdentity --resource-group $resourceGroupName -o tsv --query id
+if("" -ne $prefix)
+{
+    . ./../prefix_resource_names.ps1 -prefix $prefix
+    . ./../key_file_names.ps1 -prefix $prefix -path $path
+}
+
+$params = "{ ""namePrefix"":{""value"":""$prefix""},"
+if("" -ne $batchAccountName) { $params += """batchAccountName"":{""value"":""$batchAccountName""}," }
+if("" -ne $storageAccountName) { $params += """storageAccountName"":{""value"":""$storageAccountName""}," }
+if("" -ne $userAssignedIdentity) { $params += """identityName"":{""value"":""$userAssignedIdentity""}," }
+$params = $params.TrimEnd(",")
+$params += "}"
+$params = $params | ConvertTo-Json
+Write-Host $params 
 
-# Write-Host "Using User Assigned Identity ID : $identity"  -ForegroundColor Green
-# az batch account create --name $batchAccountName --resource-group $resourceGroupName --storage-account $storageAccountName --location $location -o table --identity-type UserAssigned --ids $identity 
 $scriptDir = Split-Path $script:MyInvocation.MyCommand.Path
-az deployment group create --resource-group $resourceGroupName --template-file "$($scriptDir)/azuredeploy_batch.bicep" --parameters batchAccountName="$batchAccountName" identityName="$userAssignedIdentity" storageAccountName="$storageAccountName" -o table
+az deployment group create --resource-group $resourceGroupName --template-file "$($scriptDir)/../Modules/batch.bicep" --parameters $params -o table

scripts/templates/Batch/create_batch_settingsfiles.ps1

@@ -95,6 +95,9 @@ $params += @("--idrg",$identity.resourceGroup)
 $params += @("--tenantid", $tenantId)
 $params += @("--subscriptionid", $subscriptionId)
 $params += @("--silent")
+$params += @("--eventhublogging", "ScriptErrors")
+$params += @("--ehrg", $resourceGroupName)
+$params += @("--ehsub", $subscriptionId)
 
 if($vnetName -ne "" -and $subnetName -ne "")
 {
@@ -122,12 +125,13 @@ $keyVaultParam = @("--keyvaultname", $keyVaultName)
 # With Password auth (default)
 Write-Host "Saving settings file to $settingsJsonWindows" -ForegroundColor DarkGreen
 $tmpPath = @("--settingsfile", $settingsJsonWindows)
-Write-Host $params   $winParams   $tmpPath -ForegroundColor DarkYellow
+Write-Host $params   $winParams   $tmpPath $tmpPath  $ehConnParam -ForegroundColor DarkYellow
 Start-Process $sbmExe -ArgumentList ($params + $winParams + $tmpPath + $ehConnParam ) -Wait -NoNewWindow
 
 
 Write-Host "Saving settings file to $settingsJsonLinux" -ForegroundColor DarkGreen 
 $tmpPath = @("--settingsfile",$settingsJsonLinux)
+Write-Host $params  $linuxParams  $tmpPath  $ehConnParam -ForegroundColor DarkYellow
 Start-Process $sbmExe -ArgumentList ($params + $linuxParams + $tmpPath  + $ehConnParam )  -Wait -NoNewWindow
 
 
@@ -138,12 +142,13 @@ $authMi += @("--principalid",$identity.principalId)
 
 Write-Host "Saving settings file to $settingsJsonWindowsMi" -ForegroundColor DarkGreen
 $tmpPath = @("--settingsfile", $settingsJsonWindowsMi)
-Write-Host $params  $winParams   $tmpPath -ForegroundColor DarkYellow
+Write-Host $params  $winParams   $tmpPath $authMi  $ehNameParam -ForegroundColor DarkYellow
 Start-Process $sbmExe -ArgumentList ($params + $winParams + $tmpPath + $authMi + $ehNameParam) -Wait -NoNewWindow
 
 
 Write-Host "Saving settings file to $settingsJsonLinuxMi" -ForegroundColor DarkGreen 
 $tmpPath = @("--settingsfile",$settingsJsonLinuxMi)
+Write-Host $params  $linuxParams $tmpPath  $authMi  $ehNameParam -ForegroundColor DarkYellow
 Start-Process $sbmExe -ArgumentList ($params + $linuxParams + $tmpPath + $authMi + $ehNameParam ) -Wait -NoNewWindow
 
 
@@ -152,33 +157,41 @@ Start-Process $sbmExe -ArgumentList ($params + $linuxParams + $tmpPath + $authMi
 
 Write-Host "Saving settings file to $settingsJsonWindowsQueue" -ForegroundColor DarkGreen
 $tmpPath = @("--settingsfile",$settingsJsonWindowsQueue)
+Write-Host $params  $winParams  $tmpPath  $sbConnParam   $ehConnParam  -ForegroundColor DarkYellow
 Start-Process $sbmExe -ArgumentList ($params + $winParams + $tmpPath + $sbConnParam  + $ehConnParam ) -Wait -NoNewWindow
 
 Write-Host "Saving settings file to $settingsJsonWindowsQueueKv" -ForegroundColor DarkGreen 
 $tmpPath = @("--settingsfile",$settingsJsonWindowsQueueKv)
+Write-Host $params  $winParams  $tmpPath  $sbConnParam  $ehConnParam  $keyVaultParam  -ForegroundColor DarkYellow
 Start-Process $sbmExe -ArgumentList ($params + $winParams + $tmpPath + $sbConnParam + $ehConnParam + $keyVaultParam )   -Wait -NoNewWindow
 
 Write-Host "Saving settings file to $settingsJsonLinuxQueue" -ForegroundColor DarkGreen 
 $tmpPath = @("--settingsfile",$settingsJsonLinuxQueue)
+Write-Host $params  $linuxParams  $tmpPath $sbConnParam  $ehConnParam  -ForegroundColor DarkYellow
 Start-Process $sbmExe -ArgumentList ($params + $linuxParams + $tmpPath + $sbConnParam + $ehConnParam )  -Wait -NoNewWindow
 
 Write-Host "Saving settings file to $settingsJsonLinuxQueueKv" -ForegroundColor DarkGreen 
 $tmpPath = @("--settingsfile",$settingsJsonLinuxQueueKv)
+Write-Host $params  $linuxParams  $tmpPath  $sbConnParam   $ehConnParam  $keyVaultParam  -ForegroundColor DarkYellow
 Start-Process $sbmExe -ArgumentList ($params + $linuxParams + $tmpPath + $sbConnParam  + $ehConnParam + $keyVaultParam )  -Wait -NoNewWindow
 
 #With Managed Identity auth
 Write-Host "Saving settings file to $settingsJsonWindowsQueueMi" -ForegroundColor DarkGreen
 $tmpPath = @("--settingsfile",$settingsJsonWindowsQueueMi)
+Write-Host $params  $winParams  $tmpPath  $authMi  $sbNamespaceParam  $ehNameParam  -ForegroundColor DarkYellow
 Start-Process $sbmExe -ArgumentList ($params + $winParams + $tmpPath + $authMi + $sbNamespaceParam  + $ehNameParam) -Wait -NoNewWindow
 
 Write-Host "Saving settings file to $settingsJsonWindowsQueueKvMi" -ForegroundColor DarkGreen 
 $tmpPath = @("--settingsfile",$settingsJsonWindowsQueueKvMi)
+Write-Host $params  $winParams  $tmpPath  $authMi  $sbNamespaceParam   $ehNameParam  $keyVaultParam -ForegroundColor DarkYellow
 Start-Process $sbmExe -ArgumentList ($params + $winParams + $tmpPath + $authMi + $sbNamespaceParam  + $ehNameParam + $keyVaultParam)  -Wait -NoNewWindow
 
 Write-Host "Saving settings file to $settingsJsonLinuxQueueMi" -ForegroundColor DarkGreen 
 $tmpPath = @("--settingsfile",$settingsJsonLinuxQueueMi)
+Write-Host $params  $linuxParams  $tmpPath  $authMi $sbNamespaceParam $ehNameParam -ForegroundColor DarkYellow
 Start-Process $sbmExe -ArgumentList ($params + $linuxParams + $tmpPath + $authMi + $sbNamespaceParam  + $ehNameParam) -Wait -NoNewWindow
 
 Write-Host "Saving settings file to $settingsJsonLinuxQueueKvMi" -ForegroundColor DarkGreen 
 $tmpPath = @("--settingsfile",$settingsJsonLinuxQueueKvMi)
+Write-Host $params $linuxParams  $tmpPath $authMi $sbNamespaceParam  $ehNameParam $keyVaultParam -ForegroundColor DarkYellow
 Start-Process $sbmExe -ArgumentList ($params + $linuxParams + $tmpPath + $authMi + $sbNamespaceParam  + $ehNameParam + $keyVaultParam) -Wait -NoNewWindow

scripts/templates/ContainerApp/create_containerapp_env.ps1

@@ -1,25 +1,27 @@
 param
 (
+    [string] $prefix,
+    [string] $resourceGroupName,
     [string] $containerAppEnvName,
     [string] $logAnalyticsWorkspace,
     [string] $containerRegistryName,
     [string] $location,
-    [string] $resourceGroupName,
-    [string] $subnetId
+    [string] $subnetId,
+    [string] $path = "..\..\..\src\TestConfig"
 )
-Write-Host "Create Container App Environment"  -ForegroundColor Cyan
-
-$logAnalyticsClientId = az monitor log-analytics workspace show --query customerId -g $resourceGroupName -n $logAnalyticsWorkspace
-$logAnalyticsKey = az monitor log-analytics workspace get-shared-keys --query primarySharedKey -g $resourceGroupName -n $logAnalyticsWorkspace
 
-Write-Host "Creating Container App Environment: $containerAppEnvName" -ForegroundColor DarkGreen
-if("" -eq $subnetId)
+if("" -ne $prefix)
 {
- az containerapp env create -n "$containerAppEnvName" -g "$resourceGroupName" --logs-workspace-id "$logAnalyticsClientId" --logs-workspace-key "$logAnalyticsKey" --location "$location" -o table 
+    . ./../prefix_resource_names.ps1 -prefix $prefix
+    . ./../key_file_names.ps1 -prefix $prefix -path $path
 }
-else
+Write-Host "Creating Container App Environment: $containerAppEnvName" -ForegroundColor DarkGreen
+if("" -eq $subnetId)
 {
-    az containerapp env create -n "$containerAppEnvName" -g "$resourceGroupName" --logs-workspace-id "$logAnalyticsClientId" --logs-workspace-key "$logAnalyticsKey" --location "$location" --infrastructure-subnet-resource-id "$subnetId" -o table
+    $subnetId = az network vnet subnet show --resource-group $resourceGroupName --vnet-name $vnet --name $containerAppSubnet --query id -o tsv
 }
+$logAnalyticsClientId = az monitor log-analytics workspace show --query customerId -g $resourceGroupName -n $logAnalyticsWorkspace
+$logAnalyticsKey = az monitor log-analytics workspace get-shared-keys --query primarySharedKey -g $resourceGroupName -n $logAnalyticsWorkspace
 
-
+$scriptDir = Split-Path $script:MyInvocation.MyCommand.Path
+az deployment group create --resource-group $resourceGroupName --template-file "$($scriptDir)/../Modules/containerappenv.bicep" --parameters containerAppEnvName=$containerAppEnvName logAnalyticsClientId=$logAnalyticsClientId logAnalyticsKey=$logAnalyticsKey subnetId=$subnetId -o table

scripts/templates/ContainerApp/create_containerapp_settingsfile.ps1

@@ -126,17 +126,20 @@ foreach($auth in $authTypes)
 
     $params += @("--tenantid", $tenantId)
     $params +=("--environmentname",$containerAppEnvironmentName)
-    $params +=("--location",$location)
+    $params +=("--location","""$location""")
     $params +=("--resourcegroup", $resourceGroupName)
     $params +=("--imagetag",$imageTag)
     $params +=("--settingsfile",$tmpPath)
     $params +=("--settingsfilekey",$keyFile)
     $params +=("--storageaccountname",$storageAccountName)
+    $params += ("--ehrg", $resourceGroupName)
+    $params += ("--ehsub", $subscriptionId)
 
 
     $params +=("--defaultscripttimeout",500)
     $params +=("--subscriptionid",$subscriptionId)
     $params +=("--force","true")
+    $params += ("--eventhublogging", "ScriptErrors")
     if($haveSqlInfo)
     {
         $params +=("--username",$sqlUserName)

scripts/templates/ContainerApp/create_containerapp_settingsfile_fromprefix_all.ps1

@@ -3,8 +3,13 @@ param
     [string] $prefix
 )
 $scriptDir = Split-Path $script:MyInvocation.MyCommand.Path
-
 .$scriptDir\create_containerapp_settingsfile_fromprefix.ps1 -prefix $prefix -withContainerRegistry $true -withKeyVault $true
+
+$scriptDir = Split-Path $script:MyInvocation.MyCommand.Path
 .$scriptDir\create_containerapp_settingsfile_fromprefix.ps1 -prefix $prefix -withContainerRegistry $false -withKeyVault $true
+
+$scriptDir = Split-Path $script:MyInvocation.MyCommand.Path
 .$scriptDir\create_containerapp_settingsfile_fromprefix.ps1 -prefix $prefix -withContainerRegistry $true -withKeyVault $false
+
+$scriptDir = Split-Path $script:MyInvocation.MyCommand.Path
 .$scriptDir\create_containerapp_settingsfile_fromprefix.ps1 -prefix $prefix -withContainerRegistry $false -withKeyVault $false

scripts/templates/ContainerRegistry/create_container_registry.ps1

@@ -1,10 +1,14 @@
 param
 (
+    [string] $prefix,
     [string] $resourceGroupName,
     [string] $containerRegistryName,
     [string] $logAnalyticsWorkspace
 )
-
+if("" -ne $prefix)
+{
+    . ./../prefix_resource_names.ps1 -prefix $prefix
+}
 
 Write-Host "Creating Container Registry: $containerRegistryName" -ForegroundColor DarkGreen
 az acr create --resource-group "$resourceGroupName" --name $containerRegistryName --sku Standard --workspace $logAnalyticsWorkspace --admin-enabled true -o table