Version 14.2.0

.github/workflows/codeql-analysis.yml

@@ -50,12 +50,8 @@ jobs:
     - name: Setup .NET 6
       uses: actions/setup-dotnet@v1
       with:
-        dotnet-version: '6.0.100-rc.1.21458.32'
+        dotnet-version: '6.0.x'
 
-    - name: Setup .NET 5
-      uses: actions/setup-dotnet@v1
-      with:
-        dotnet-version: 5.0.401
 
     - name: Clean
       run: dotnet clean ./src/SqlBuildManager.Console/sbm.csproj --configuration Release && dotnet nuget locals all --clear
@@ -64,10 +60,10 @@ jobs:
       run: dotnet clean ./src/SqlSync/SQLSync.csproj --configuration Release && dotnet nuget locals all --clear
 
     - name: Restore and build console app
-      run: dotnet build ./src/SqlBuildManager.Console/sbm.csproj --configuration Release -f net5.0
+      run: dotnet build ./src/SqlBuildManager.Console/sbm.csproj --configuration Release -f net6.0
 
     - name: Restore and build desktop app
-      run: dotnet build ./src/SqlSync/SQLSync.csproj --configuration Release -f net5.0-windows
+      run: dotnet build ./src/SqlSync/SQLSync.csproj --configuration Release -f net6.0-windows
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl

.github/workflows/dotnetcore-build.yml

@@ -20,12 +20,12 @@ jobs:
     - name: Setup .NET 6
       uses: actions/setup-dotnet@v1
       with:
-        dotnet-version: '6.0.100-rc.1.21458.32'
+        dotnet-version: '6.0.x'
 
-    - name: Setup .NET 5
-      uses: actions/setup-dotnet@v1
-      with:
-        dotnet-version: 5.0.401
+    # - name: Setup .NET 5
+    #   uses: actions/setup-dotnet@v1
+    #   with:
+    #     dotnet-version: 5.0.x
 
     - name: Clean
       run: dotnet clean ./src/SqlBuildManager.Console/sbm.csproj --configuration Release && dotnet nuget locals all --clear
@@ -34,10 +34,10 @@ jobs:
       run: dotnet clean ./src/SqlSync/SQLSync.csproj --configuration Release && dotnet nuget locals all --clear  
 
     - name: Restore and build console app
-      run: dotnet build ./src/SqlBuildManager.Console/sbm.csproj --configuration Release -f net5.0
+      run: dotnet build ./src/SqlBuildManager.Console/sbm.csproj --configuration Release -f net6.0
 
     - name: Restore and build desktop app
-      run: dotnet build ./src/SqlSync/SQLSync.csproj --configuration Release -f net5.0-windows
+      run: dotnet build ./src/SqlSync/SQLSync.csproj --configuration Release -f net6.0-windows
 
     # Run tests
     - name: Run SqlBuildManager.Console.UnitTest
@@ -59,14 +59,14 @@ jobs:
       run: dotnet test ./src/SqlSync.SqlBuild.UnitTest/SqlSync.SqlBuild.UnitTest.csproj --configuration Release
 
     # .NET 5.0 pubish
-    - name: Publish the Windows desktop app
-      run: dotnet publish ./src/SqlSync/SqlSync.csproj -r win-x64 --configuration Release -f net5.0-windows
+    # - name: Publish the Windows desktop app
+    #   run: dotnet publish ./src/SqlSync/SqlSync.csproj -r win-x64 --configuration Release -f net5.0-windows
 
-    - name: Publish the Windows CLI app 
-      run: dotnet publish ./src/SqlBuildManager.Console/sbm.csproj -r win-x64 --configuration Release -f net5.0
+    # - name: Publish the Windows CLI app 
+    #   run: dotnet publish ./src/SqlBuildManager.Console/sbm.csproj -r win-x64 --configuration Release -f net5.0
 
-    - name: Publish the Linux CLI app
-      run: dotnet publish ./src/SqlBuildManager.Console/sbm.csproj -r linux-x64 --configuration Release -f net5.0
+    # - name: Publish the Linux CLI app
+    #   run: dotnet publish ./src/SqlBuildManager.Console/sbm.csproj -r linux-x64 --configuration Release -f net5.0
 
     # .NET Core 6.0 pubish
     - name: Publish the Windows desktop app
@@ -78,24 +78,24 @@ jobs:
     - name: Publish the Linux CLI app
       run: dotnet publish ./src/SqlBuildManager.Console/sbm.csproj -r linux-x64 --configuration Release -f net6.0
 
-    # .NET 5.0 artifacts
-    - name: Upload a Build Artifact sbm-windows
-      uses: actions/upload-artifact@v2
-      with:
-        name: sbm-windows
-        path: "./src/SqlBuildManager.Console/bin/Release/net5.0/win-x64/publish"
-
-    - name: Upload a Build Artifact sbm-linux
-      uses: actions/upload-artifact@v2
-      with:
-        name: sbm-linux
-        path: "./src/SqlBuildManager.Console/bin/Release/net5.0/linux-x64/publish"
-
-    - name: Upload a Build Artifact
-      uses: actions/upload-artifact@v2
-      with:
-        name: SqlBuildManager Desktop
-        path: "./src/SqlSync/bin/Release/net5.0-windows/win-x64/publish"     
+    # # .NET 5.0 artifacts
+    # - name: Upload a Build Artifact sbm-windows
+    #   uses: actions/upload-artifact@v2
+    #   with:
+    #     name: sbm-windows
+    #     path: "./src/SqlBuildManager.Console/bin/Release/net5.0/win-x64/publish"
+
+    # - name: Upload a Build Artifact sbm-linux
+    #   uses: actions/upload-artifact@v2
+    #   with:
+    #     name: sbm-linux
+    #     path: "./src/SqlBuildManager.Console/bin/Release/net5.0/linux-x64/publish"
+
+    # - name: Upload a Build Artifact
+    #   uses: actions/upload-artifact@v2
+    #   with:
+    #     name: SqlBuildManager Desktop
+    #     path: "./src/SqlSync/bin/Release/net5.0-windows/win-x64/publish"     
 
     # .NET 6 artifacts
     - name: Upload a Build Artifact sbm-windows

.gitignore

@@ -221,3 +221,7 @@ pip-log.txt
 /src/TestConfig
 Command_scratch.txt
 /scripts/templates/runme.ps1
+/src/SqlBuildManager.Console/WorkerApp/workerapp_arm_template.parameters.json
+/src/SqlBuildManager.Console/ContainerApp/parms.json
+/src/SqlBuildManager.Console/ContainerApp/parms2.json
+src/env.txt

CHANGELOG.md

@@ -1,5 +1,13 @@
-
-# SQL Build Manager Change Log
+# SQL Build Manager Change Log
+
+### Version 14.2.0
+
+- *ADDED:* You can now use Azure Container Apps as a compute platform for leveraging database builds with the new `sbm containerapps` commands. See the [Azure Container Apps documentation](containerapps.md) for background, information and how-to examples
+- *ADDED:* New options for container image tags and container registries for all container options (Azure Container Apps, Kubernetes, Azure Container Instance)
+- *UPDATED:* Updates to deployment scripts and unit tests
+- *ADDED:* New command `sbm utility eventhub` to scan EventHub for job event counts
+- *UPDATED:* Reorganized template scripts by service
+- *UPDATED:* Optimized EventHub client to checkpoint by job time so events are not missed when running multiple concurrent jobs
 
 ### Version 14.1.0
 

README.md

@@ -4,9 +4,9 @@ SQL Build Manager is a multi-faceted tool to allow you to manage the life-cycle
 
 ![.NET Core Build](https://github.com/mmckechney/SqlBuildManager/workflows/.NET%20Core%20Build/badge.svg)
 
-### **Important change in Version 14+:**
+### **Important changes in Version 14+:**
 
-There are two new options to massively parallel processing: [Kubernetes](docs/kubernetes.md) and [Azure Container Instance](docs/aci.md)!
+There are three new options to massively parallel processing: [Azure Container Apps](docs/containerapps.md), [Kubernetes](docs/kubernetes.md) and [Azure Container Instance](docs/aci.md)!
 
  [Batch node pools](docs/massively_parallel.md) are now created with assigned Managed Identities. Because of this, the workstation running `sbm` _needs to have a valid Azure authentication token_. This can be done via Azure CLI `az login`, Azure PowerShell `Connect-AzAccount`, or if running from an automation box, ensure that the machine itself has a Managed Identity that has permissions to create Azure resources. Alternatively, you can pre-create the batch pools manually via the Azure portal, being sure to assign the correct Managed Identity to the pool.
 
@@ -30,6 +30,7 @@ You will also need to be logged into Azure if you are leveraging Azure Key Vault
 - [Command Line Reference/ Quickstart](docs/commandline.md)
 - [Running Locally](docs/local_build.md)
 - [Massively Parallel Database Builds](docs/massively_parallel.md)
+  - [Azure Container Apps](docs/containerapp.md)
   - [Azure Batch](docs/azure_batch.md)
   - [Kubernetes](docs/kubernetes.md)
   - [Azure Container Instances (ACI)](docs/aci.md)
@@ -144,6 +145,10 @@ Using the `sbm threaded run` command will allow for updating multiple databases
 Using the `sbm batch run` command leverages Azure Batch to permit massively parallel updates across thousands of databases. To leverage Azure Batch, you will first need to set up your Batch account. The instructions for this can be found [here](docs/azure_batch.md).
 An excellent tool for viewing and monitoring your Azure batch accounts and jobs can be found here [https://azure.github.io/BatchExplorer/](https://azure.github.io/BatchExplorer/)
 
+### **Azure Container Apps**
+
+Using the `sbm containerapp` commands leverages Azure Container Apps to permit massively parallel updates across thousands of databases. Learn how to use Container Apps [here](docs/containerapp.md).
+
 ### **Kubernetes**
 
 Using the `sbm k8s` commands leverages Kubernetes to permit massively parallel updates across thousands of databases. To leverage Kubernetes, you will first need to set up a Kubernetes Cluster. The instructions for this can be found [here](docs/kubernetes.md#).

devops/azure-pipelines.yml

@@ -167,7 +167,7 @@ steps:
 - task: PublishBuildArtifacts@1
   displayName: Copying batch ARM template
   inputs:
-    PathtoPublish: './scripts/templates/azuredeploy.json'
+    PathtoPublish: './scripts/templates/azuredeploy_base.bicep'
     ArtifactName: 'drop'
     publishLocation: 'Container'
 

docs/aci.md

@@ -44,7 +44,7 @@ sbm aci savesettings --aciname "<ACI Name>" --acirg "<ACI resource group>" --ide
 
 You can automate they collection and saving of secrets with the included PowerShell script:
 
-- [create_aci_settingsfile.ps1](../scripts/templates/create_aci_settingsfile.ps1) - saves secrets to Key Vault and creates the settings JSON file for you.
+- [create_aci_settingsfile.ps1](../scripts/templates/aci/create_aci_settingsfile.ps1) - saves secrets to Key Vault and creates the settings JSON file for you.
 
 ``` PowerShell
 #Collects resource keys, saves them to Key Vault and creates settings file

docs/azure_batch.md

@@ -70,7 +70,7 @@ Azure Batch builds are started locally via `sbm.exe`. This process communicates
 
 ### Settings File
 
-While all of the values can be provided as arguments in the command line, it is strongly suggested you leverage `--settingsfile` and `--settingsfilekey` (the `--settingsfilekey` parameter is not required if you are leveraging `--keyvaultname`). A settings JSON file was created for you via [`create_azure_resources.ps1`](../scripts/templates/create_azure_resources.ps1) but can also be re-created for any set of Azure resources via [`create_batch_settingsfile.ps1`](../scripts/templates/create_batch_settingsfiles.ps1)
+While all of the values can be provided as arguments in the command line, it is strongly suggested you leverage `--settingsfile` and `--settingsfilekey` (the `--settingsfilekey` parameter is not required if you are leveraging `--keyvaultname`). A settings JSON file was created for you via [`create_azure_resources.ps1`](../scripts/templates/create_azure_resources.ps1) but can also be re-created for any set of Azure resources via [`create_batch_settingsfile.ps1`](../scripts/templates/Batch/create_batch_settingsfiles.ps1)
 
 You can also build it manually by executing `sbm batch savesettings` command and providing the appropriate arguments. See the argument details [here](azure_batch_commands.md#azure-batch-save-settings)
 

docs/azure_batch_example.md

@@ -8,7 +8,7 @@ az login
 ```
 
 ## 1. Save your batch settings to a `--settingsfile`
-This only needs to be run once and this file can be reused for future builds as long as you don't change any keys or passwords. By creating this file now, you greatly simplify the subsequent commands. If you include the `--keyvault` option, the secrets will be saved in Azure Key vault and not the settings file. You can also simplify the collection of the various keys and connection strings and saving by using the PowerShell script [create_batch_settingsfiles.ps1](../scripts/templates/create_batch_settingsfiles.ps1)
+This only needs to be run once and this file can be reused for future builds as long as you don't change any keys or passwords. By creating this file now, you greatly simplify the subsequent commands. If you include the `--keyvault` option, the secrets will be saved in Azure Key vault and not the settings file. You can also simplify the collection of the various keys and connection strings and saving by using the PowerShell script [create_batch_settingsfiles.ps1](../scripts/templates/Batch/create_batch_settingsfiles.ps1)
 
 Replace the sample values with your own account values and keys
 

docs/containerapp.md

@@ -0,0 +1,87 @@
+# Leveraging Azure Container Apps for database builds
+
+- [Why use Container Apps?](#why-use-container-apps)
+  - [Process Flow](massively_parallel.md#azure-container-instance-process-flow)
+- [Getting Started](#getting-started)
+  - [Container Image](#container-image)
+  - [Environment Setup](#environment-setup)
+- [Example and How To](#example-and-how-to)
+
+## Why use Container Apps?
+
+If you have a fleet of databases to update, it could take a very long time to run your build on a single machine, even if you leverage the [threaded](threaded_build.md) model. Similar to leveraging [Azure Batch](azure_batch.md) or [Azure Container Instance](aci.md), to ensure you can complete your updates in a timely fashion, SQL Build Manager can target Azure Container Apps to distribute you build across multiple compute nodes and pods - each leveraging their own set of concurrent tasks. You can control the level of concurrency to maximize throughput while not overloading your SQL Servers (see [details on concurrency management](concurrency_options.md))
+
+In this implementation, you will need a Container App Environment ant the database targeting and logging leverage [Azure Service Bus](https://azure.microsoft.com/en-us/services/service-bus/) and [Azure Event Hub](https://azure.microsoft.com/en-us/services/event-hubs) respectively. To leverage Container Apps, you will need an [Azure subscription](https://azure.microsoft.com/) with several Azure resources deployed.
+
+## Getting Started
+
+### Container Image
+
+The default container image can be found on Docker Hub at https://hub.docker.com/repository/docker/blueskydevus/sqlbuildmanager/general, or you could build your own from source using the following command from the `/src/` folder
+
+``` bash
+docker build -f Dockerfile .. -t sqlbuildmanager:latest
+```
+
+### Environment Setup
+
+While the Container App execution process will create the app for you, you will need a Container App Environment to deploy the app into. It also leverages [Azure Service Bus](https://azure.microsoft.com/en-us/services/service-bus/), [Azure Event Hub](https://azure.microsoft.com/en-us/services/event-hubs) and [Azure Blob Storage](https://azure.microsoft.com/en-us/services/storage/blobs/). You can create your own resources either through the [Azure portal](https://portal.azure.com), [az cli](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli) or [Azure PowerShell](https://docs.microsoft.com/en-us/powershell/azure/). The only special configuration is with Azure Service Bus which requires a Topic named `sqlbuildmanager`.
+
+It is recommended that you can create the resources via the included PowerShell [create_azure_resources.ps1](../scripts/templates/create_azure_resources.ps1). This script will create all of the resources you need and an option for 2 SQL servers and 20 databases in elastic pools. It will also create a new folder and pre-configured settings files in a folder `./src/TestConfig`. The settings files are needed for running integration tests but also serve as excellent references for you to create your own settings files.
+
+----
+
+## Example and How To
+
+### 1. Save the common settings to the config files
+
+The recommended way to run an Container App deployment is to first save the settings that you will leverage in a `--settingsfile` with a `--settingsfilekey`
+
+``` bash
+sbm containerapp savesettings -settingsfile "<settings file name>" --settingsfilekey "<settings file key name>" --environmentname "<container app env>" --location "<azure location of env>" --image "<container image name>" --imagetag "<image tag>" g "<environment resource group>" -sb "<service bus topic connection string>"  -kv "<Key Vault Name>" --storageaccountname "<storage acct name>" --storageaccountkey "<storage acct key>" -eh "<event hub connection string>" --defaultscripttimeout 500 --subscriptionid "<azure subscription id>" --force 
+```
+
+You can automate they collection and saving of secrets with the included PowerShell script:
+
+- [create_containerapp_settingsfile.ps1](../scripts/templates/ContainerApp/create_containerapp_settingsfile.ps1) - saves secrets to Key Vault and creates the settings JSON file for you.
+
+``` PowerShell
+# Collects resource keys and creates settings encrypted file
+create_containerapp_settingsfile.ps1 -path "<path to save the files>" -resourceGroupName "<resource group with the KV and identity>" -containerAppEnvironmentName "<env name>" -containerRegistryName "<if using Azure Container registry>" -storageAccountName "<Name of storage account>" -eventHubNamespaceName "<Name of event hub namespace>" -serviceBusNamespaceName "<Name of service bus namespace>" -sqlUserName "<SQL user name" -sqlPassword "<SQL Password>" -withContainerRegistry ($true|$false) 
+```
+
+### 2. Upload your SBM Package file to your storage account and create customized ARM template
+
+The Container App containers retrieve the build package from Azure storage, this command will create a storage container with the name of the `--jobname` (it will be lower cased and any invalid characters removed) and upload the SBM file to the new container. It will also create a customized ARM template which will be used to deploy the Container App containers in the `deploy` step.
+
+``` bash
+sbm containerapp prep --settingsfile "<settings file name>" --tag "<container version tag>" --jobname "<job name>" -P "<sbm package name>"
+```
+
+### 3. Queue up the override targets in Service Bus
+
+
+**IMPORTANT:** If using arguments, the `jobname` value _MUST_ match the value used in the `prep` steps otherwise the messages will not get processed.
+
+``` bash
+sbm containerapp enqueue --settingsfile "<settings file name>" --jobname "<job name>" --concurrencytype "<concurrency type>" --override "<override file name>"
+```
+
+### 4. Deploy Container and Monitor progress
+
+Next is to deploy the Container App to create the containers. By default, once the deployment is complete, it will start monitoring progress against the Service Bus and Event Hub. You can change this behavior by setting the `--monitor` argument to `false`. The `--override` argument is not required, but it will allow the monitor to track the target database count and stop monitoring when all targets have been processed.
+
+``` bash
+sbm containerapp deploy --settingsfile "<settings file name>" --jobname "<job name>" -P "<sbm package name>" --override "<override file name>"  --concurrencytype "<concurrency type>" --concurrency '<int value>' --monitor 
+```
+
+if you would rather run an extra step (for whatever reason), you can run a separate `monitor` command:
+
+``` bash
+sbm containerapp monitor --settingsfile "<settings file name>" --jobname "<job name>" --concurrencytype "<concurrency type>" --override "<override file name>"
+```
+
+ All of the run logs will be transferred from the pods to the storage container specified in the `jobname` argument. When monitoring is complete, it will output a Blob container SAS token that you can use in [Azure Storage Explorer](https://azure.microsoft.com/en-us/features/storage-explorer/) to easily view the logs.
+
+ **IMPORTANT:** After the `sbm containerapp deploy` (with monitoring) or `sbm containerapp monitor` completes, as part of the clean-up, it will remove the Service Bus Topic associated with the build. This will deactivate the running containers.
+