From 752fea7300b745a11261d92c696a2422d63478af Mon Sep 17 00:00:00 2001 From: Michael McKechney Date: Wed, 6 Jul 2022 13:10:30 -0400 Subject: [PATCH] Version 14.4.0 - *ADDED:* Added new `ManagedIdentity` authentication type to eliminate the need for a UserName and Password to authenticate to Azure SQL databases that have Azure AD authentication enabled and identity assigned - *ADDED:* Ability to use Managed Identity for Service Bus, Event Hub and Blob storage connections with most services (see [managed_identity.md](/docs/managed_identity.md) for details and limitations) - *ADDED:* New `--monitor` argument for `sbm batch run` to get running count of datbase activity (commits, error, in queue) - *ADDED:* New `--stream` argument for `sbm batch run` (used in conjunction with `--monitor`) to also stream specific database completion messages as the occur - *UPDATED:* Monitoring of remaining queue messages only when Service Bus is used, but no Event Hub connection is provided - *UPDATED:* Reorganized Unit Test settings file creation scripts to group by execution compute type --- CHANGELOG.md | 5 +- README.md | 13 +- devops/azure-docker-pipeline.yaml | 4 +- docs/managed_identity.md | 89 ++++++++++ .../Batch/create_batch_settingsfiles.ps1 | 24 +-- .../create_containerapp_settingsfile.ps1 | 15 +- .../templates/aci/create_aci_settingsfile.ps1 | 7 +- .../kubernetes/create_aks_job_yaml.ps1 | 8 +- .../kubernetes/create_aks_keyvault_config.ps1 | 4 +- .../create_aks_secrets_and_runtime_files.ps1 | 4 +- scripts/templates/kubernetes/sample_job.yaml | 1 + .../kubernetes/sample_job_keyvault.yaml | 1 + scripts/utility/Clean-ObjAndBin.ps1 | 1 + scripts/utility/Clear_UnitTest_Batch_Jobs.ps1 | 2 +- scripts/utility/Clear_UnitTest_Remnants.ps1 | 11 +- .../Validate_UnitTest_SettingsFiles.ps1 | 2 +- .../AciTests.cs | 6 +- .../BatchTests.cs | 156 ++++++++--------- .../ContainerAppTests.cs | 8 +- .../KubernetesTests.cs | 82 ++++----- .../LocalTests.cs | 4 +- src/SqlBuildManager.Console/Aad/AadHelper.cs | 8 +- .../Batch/BatchExecution.cs | 22 +-- .../CloudStorage/StorageManager.cs | 136 ++++++++++++--- .../ContainerApp/ContainerAppHelper.cs | 2 +- .../containerapp_arm_template.json | 2 +- .../containerapp_env_arm_template.json | 4 +- .../containerapp_identity_arm_template.json | 4 +- .../Events/EventManager.cs | 16 +- .../Kubernetes/ContainerManager.cs | 103 ++++++++--- .../Kubernetes/RuntimeYaml.cs | 2 + .../Kubernetes/SecretYaml.cs | 1 - .../Threaded/ThreadedRunner.cs | 24 --- src/SqlBuildManager.Console/Validation.cs | 22 +-- src/SqlBuildManager.Console/Worker.cs | 161 ++++++------------ src/SqlBuildManager.Console/sbm.csproj | 4 +- .../SqlBuildManager.Logging.csproj | 2 +- 37 files changed, 557 insertions(+), 403 deletions(-) create mode 100644 docs/managed_identity.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 31312a34..5e0a4f8c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,12 @@ # SQL Build Manager Change Log ### Version 14.4.0 - - *ADDED:* Added new `ManagedIdentity` authentication type to eliminate the need for a UserName and Password to authenticate to Azure SQL databases that have Azure AD authenication enabled and identity assigned - - *ADDED:* Ability to use Managed Identity for Service Bus and EventHub connections (see [README.md](README.md) for details) + - *ADDED:* Added new `ManagedIdentity` authentication type to eliminate the need for a UserName and Password to authenticate to Azure SQL databases that have Azure AD authentication enabled and identity assigned + - *ADDED:* Ability to use Managed Identity for Service Bus, Event Hub and Blob storage connections with most services (see [managed_identity.md](/docs/managed_identity.md) for details and limitations) - *ADDED:* New `--monitor` argument for `sbm batch run` to get running count of datbase activity (commits, error, in queue) - *ADDED:* New `--stream` argument for `sbm batch run` (used in conjunction with `--monitor`) to also stream specific database completion messages as the occur - *UPDATED:* Monitoring of remaining queue messages only when Service Bus is used, but no Event Hub connection is provided + - *UPDATED:* Reorganized Unit Test settings file creation scripts to group by execution compute type ### Version 14.3.0 - *ADDED:* Managed Identity and Key Vault support for Container Apps diff --git a/README.md b/README.md index b7c18a76..172823d8 100644 --- a/README.md +++ b/README.md @@ -4,18 +4,11 @@ SQL Build Manager is a multi-faceted tool to allow you to manage the life-cycle ![.NET Core Build](https://github.com/mmckechney/SqlBuildManager/workflows/.NET%20Core%20Build/badge.svg) -### _Be sure to review the [change log](CHANGELOG.md) for the latest updates, enhancements and bug fixes_ +#### _Be sure to review the [change log](CHANGELOG.md) for the latest updates, enhancements and bug fixes_ -### **Key feature enhancement with Version 14.4+: Expanded use of Azure Managed Identity to connect to resources** +### **Key feature enhancement with Version 14.4+: Expanded use of Azure User Assigned Managed Identity** -With this update, it significantly reduces the the need to save and manage secrets and connection strings - -- _Database Connections:_ [Kubernetes](docs/kubernetes.md), [Azure Container Instance](docs/aci.md) and [Batch](docs/massively_parallel.md) methods now allow for the use of Azure Managed Identity to authenticate to SQL Databases (with new `--authtype ManagedIdentity`). - -- _Service Bus:_ For any deployment type other than [Azure Container Apps](docs/containerapp.md), use the name of the Service Bus namespace as the value for `--servicebustopicconnection` in place of the full connection string (see below for explanation) -- _Event Hub:_ For any deployment type, use the Event Hub Namespace and Event Hub name as values for `--eventhubconnection` in the pipe delimited format `|` - -_NOTE:_ [Azure Container Apps](docs/containerapp.md) does not currently allow for Managed Identity authentication to Azure SQL database. Also, the Service Bus connection string is required for the KEDA scaler so you can not use just the namespace. You _can_ use the `|` convention for Event Hubs however! +With this update, it significantly reduces the the need to save and manage secrets and connection strings. For full details on leveraging Managed Identity to connect to the other Azure resources such as SQL Database, Blob storage, Service Bus, Event Hub, Key Vault and Azure Container registry, see the [Managed Identity documentation here](/docs/managed_identity.md). ### **Important changes in Version 14+:** diff --git a/devops/azure-docker-pipeline.yaml b/devops/azure-docker-pipeline.yaml index 18c2d7fb..537bbd42 100644 --- a/devops/azure-docker-pipeline.yaml +++ b/devops/azure-docker-pipeline.yaml @@ -33,8 +33,8 @@ steps: command: 'buildAndPush' Dockerfile: './src/Dockerfile' tags: | - 'latest-$(Build.SourceBranchName)' - 'latest-vNext' + latest-$(Build.SourceBranchName) + latest-vNext - task: Docker@2 displayName: Logout of DockerHub diff --git a/docs/managed_identity.md b/docs/managed_identity.md new file mode 100644 index 00000000..b975eef6 --- /dev/null +++ b/docs/managed_identity.md @@ -0,0 +1,89 @@ +# Using User Assigned Managed Identity + +Each of the remote execution options allows for varying use of Azure Managed Identities for authentication to supporting resources. This document shows: +- Which [execution option is capable of leveraging the identity](#managed-identity-support-between-resources-and-execution-type) to connect to which resource +- [How the identity gets associated](#managed-identity-to-compute-assignment) with the execution compute type +- The [configuration and/or arguments needed](#runtime-configuration-to-leverage-managed-identity) to instruct the use of the identity +- The [Azure RBAC roles requred](#azure-rbac-role-assignment-requirements) for the identity + + +## Managed Identity support between resources and execution type + + | | Azure Batch | Kubernetes (AKS) | Container Apps | Container Instance (ACI) | + | ------------------------ | :-----------: | :----------------------------------------------------------------------------------------------------------------------------:| :---------------: | :-----------------------: | + | Azure Key Vault | Yes | Yes, with [Key Vault Identity Provider](https://docs.microsoft.com/en-us/azure/aks/csi-secrets-store-driver) | Yes | Yes | + | SQL databases | Yes | Yes | No, see [note](#sql-database) | Yes | + | Blob Storage | No, see [note](#blob-storage)| Yes | Yes | Yes | + | Service Bus | Yes | Yes | No, see [note](#service-bus) | Yes | + | Event Hub | Yes | Yes | Yes | Yes | + | Azure Container Registry | N/A | Yes, with [`--attach-acr`](https://docs.microsoft.com/en-us/azure/aks/cluster-container-registry-integration?tabs=azure-cli)| No | No | + | | | | | | + + ## Managed Identity to Compute Assignment + +Examples of each of these can be generated for you by running [create_azure_resources.ps1](../scripts/templates/create_azure_resources.ps1) which will create samples Azure resources (including a user assigned Managed Identity) as well as sample settings files with various options that are used in the test caes. You can also review the test methods in [SqlBuildManager.Console.ExternalTest](../src/SqlBuildManager.Console.ExternalTest/) to see working examples of various compute options and settings. + + ### Azure Batch + +The identity is assigned at the creation of the Azure Batch account. For an example, see [azuredeploy_batch.bicep](../scripts/templates/Batch/azuredeploy_batch.bicep) + +### Kubernetes + +The identity first must be associated with the AKS VM Scale set. You can find an example of this assignment near the bottom of [this srcipt](../scripts/templates//kubernetes//create_aks_cluster.ps1) + + +The identity is assigned to the app at runtime in two steps. First `AzureIdentity` and `AzureIdentityBinding` resources are created in the cluster (see [podIdentityAndBinding_template.yaml](../scripts//templates/kubernetes/podIdentityAndBinding_template.yaml)). Then when the SQL Build Manger job is deployed, an `aadpodidbinding` label is added to the job spec (see [sample_job.yaml](../scripts/templates/kubernetes/sample_job.yaml)). This will tell Kubernetes to assign the identity to the job. + +### Container Apps + +The Container Apps workload that is deployed via `sbm containerapp deploy` leverages ARM templates to deploy the workload. This ARM template contains the Managed Identity assignment (see [containerapp_identity_arm_template.json](../src/SqlBuildManager.Console/ContainerApp/containerapp_identity_arm_template.json). The identity information needs to be passed via the Identity options in command line or be saved in to a settings file via `sbm containerapp savesettings` + +### Container Instance (ACI) + +ACI also uses an ARM template to deploy the workload when running `sbm aci deploy`. This ARM template contains the Managed Identity assignment(see [aci_arm_template.json](../src/SqlBuildManager.Console/Aci/aci_arm_template.json)). The identity information needs to be passed via the Identity options in command line or be saved in to a settings file via `sbm aci savesettings` + +## Runtime Configuration to leverage Managed Identity + +### Azure Key Vault + +To instruct the app to pull secrets from Azure Key Vault, you need to provide the Key Vault name in the `--keyvault` argument of the appropriate `savesettings`, `prep` and/or `deploy`. Alternatively, you can simply save it to the settings file .json with the `savesetting` command and leverage the settings file for all subsequent commands + +### SQL Database + +By default, the app uses username/password database authentication. To enable Managed Identity authentication, you will first need to add the [Manged Identity as a user to the database](https://docs.microsoft.com/en-us/azure/azure-sql/database/authentication-azure-ad-user-assigned-managed-identity?view=azuresql#managing-a-managed-identity-for-a-server-or-instance). Once this has been done, you can direct the app to use the identity to authenticate with the `--authtype ManagedIdentity` flag. (As always, this can be saved in a settings file with `savesettings` for easier execution and reuse). + +**NOTE:** Azre Container Apps does not currently allow for Managed Identity authentication to Azure SQL Databases + + +### Blob Storage + +To use Managed Identity to access blob storage, simply don't provide a value for `--storageaccountkey`. If this is not provided, the app will default to connecting with the identity. + +**NOTE:** Azure Batch requires the storage account key to manage storage and create SAS token URLs. The key must be provided as a settings file value, command line argument or be saved in Azure Key Vault. + +### Service Bus + +To use Managed Identity to connect to Azure Service Bus, use the Service Bus namespace as the value (\.servicebus.windows.net) for `--servicebustopicconnection`. + +**NOTE:** Azure Container Apps uses a [KEDA](https://keda.sh/) [service bus scaler](https://keda.sh/docs/2.7/scalers/azure-service-bus/) to manage scaling. This does not currently allow for Managed Identity authentication, so a full Service Bus connection string is still needed. + +### Event Hub + +To use Managed Identity to connect to Azure Event Hub, use the Event Hub namespace and Event Hub name as a pipe delimited value (.servicebus.windows.net) for `--eventhubconnection`. For example "\.servicebus.windows.net|\" + +### Azure Container Registry + +Only Kubernetes is able to natively connect to the container registry without an admin username and password. This is assigned at creation or update of the cluster using the [`--assign-acr`](https://docs.microsoft.com/en-us/azure/aks/cluster-container-registry-integration?tabs=azure-cli) flag + +## Azure RBAC Role Assignment Requirements + + +The Managed Identity assigned to the runtime compute will need the following Azure RBAC roles assigned to the resource group or the specific services. _NOTE:_ You can also use an identity to authenticate for the `prep` and `enqueue` steps. The identity of that user or machine will just also need these roles. +- `Storage Blob Data Contributor` - to read the build package and save log files +- `Key Vault Secrets User` - to pull secrets from the Key Vault +- `Azure Service Bus Data Owner` - to read messages and delete compelted Service Bus Topic Subscription +- `Azure Event Hubs Data Receiver` - to read events from Event Hub +- `Azure Event Hubs Data Sender` - to send events to Event Hub +- `AcrPull` - to pull images from Azure Container Registry + +You can see an example of the assignments in [`set_managedidentity_rbac.ps1`](../scripts/templates/ManagedIdentity/set_managedidentity_rbac.ps1) diff --git a/scripts/templates/Batch/create_batch_settingsfiles.ps1 b/scripts/templates/Batch/create_batch_settingsfiles.ps1 index 4f869e62..35adff6c 100644 --- a/scripts/templates/Batch/create_batch_settingsfiles.ps1 +++ b/scripts/templates/Batch/create_batch_settingsfiles.ps1 @@ -38,23 +38,23 @@ $serviceBusConnectionString = az servicebus topic authorization-rule keys list - $identity = az identity show --resource-group $resourceGroupName --name $identityName | ConvertFrom-Json -AsHashtable $subscriptionId = az account show -o tsv --query id -$settingsJsonWindows = Join-Path $path "settingsfile-windows.json" -$settingsJsonWindowsMi = Join-Path $path "settingsfile-windows-mi.json" +$settingsJsonWindows = Join-Path $path "settingsfile-batch-windows.json" +$settingsJsonWindowsMi = Join-Path $path "settingsfile-batch-windows-mi.json" -$settingsJsonLinux = Join-Path $path "settingsfile-linux.json" -$settingsJsonLinuxMi = Join-Path $path "settingsfile-linux-mi.json" +$settingsJsonLinux = Join-Path $path "settingsfile-batch-linux.json" +$settingsJsonLinuxMi = Join-Path $path "settingsfile-batch-linux-mi.json" -$settingsJsonWindowsQueue = Join-Path $path "settingsfile-windows-queue.json" -$settingsJsonWindowsQueueMi = Join-Path $path "settingsfile-windows-queue-mi.json" +$settingsJsonWindowsQueue = Join-Path $path "settingsfile-batch-windows-queue.json" +$settingsJsonWindowsQueueMi = Join-Path $path "settingsfile-batch-windows-queue-mi.json" -$settingsJsonLinuxQueue = Join-Path $path "settingsfile-linux-queue.json" -$settingsJsonLinuxQueueMi = Join-Path $path "settingsfile-linux-queue-mi.json" +$settingsJsonLinuxQueue = Join-Path $path "settingsfile-batch-linux-queue.json" +$settingsJsonLinuxQueueMi = Join-Path $path "settingsfile-batch-linux-queue-mi.json" -$settingsJsonWindowsQueueKv = Join-Path $path "settingsfile-windows-queue-keyvault.json" -$settingsJsonWindowsQueueKvMi = Join-Path $path "settingsfile-windows-queue-keyvault-mi.json" +$settingsJsonWindowsQueueKv = Join-Path $path "settingsfile-batch-windows-queue-keyvault.json" +$settingsJsonWindowsQueueKvMi = Join-Path $path "settingsfile-batch-windows-queue-keyvault-mi.json" -$settingsJsonLinuxQueueKv = Join-Path $path "settingsfile-linux-queue-keyvault.json" -$settingsJsonLinuxQueueKvMi = Join-Path $path "settingsfile-linux-queue-keyvault-mi.json" +$settingsJsonLinuxQueueKv = Join-Path $path "settingsfile-batch-linux-queue-keyvault.json" +$settingsJsonLinuxQueueKvMi = Join-Path $path "settingsfile-batch-linux-queue-keyvault-mi.json" $keyFile = Join-Path $path "settingsfilekey.txt" if($false -eq (Test-Path $keyFile)) diff --git a/scripts/templates/ContainerApp/create_containerapp_settingsfile.ps1 b/scripts/templates/ContainerApp/create_containerapp_settingsfile.ps1 index ca55c34e..3a811eff 100644 --- a/scripts/templates/ContainerApp/create_containerapp_settingsfile.ps1 +++ b/scripts/templates/ContainerApp/create_containerapp_settingsfile.ps1 @@ -15,7 +15,7 @@ param [string] $identityClientId, [string] $imageTag, [bool] $withContainerRegistry, - ## TODO: Enable Managed Identity. For now, ManagedIdentity for SQL Auth is not available on Container Apps, but SB and EH are OK. + ## TODO: Enable Managed Identity. For now, ManagedIdentity for SQL Auth is not available on Container Apps. SB KEDA also requires a connection string. But EH can fully use MI [ValidateSet("Password", "ManagedIdentity", "Both")] [string] $authType = "Both" ) @@ -96,20 +96,23 @@ if($false -eq (Test-Path $keyFile)) foreach($auth in $authTypes) { + $params = @("containerapp", "savesettings") + if($auth -eq "ManagedIdentity" ) { - $settingsContainerApp =$settingsContainerApp + "-mi" + $settingsContainerApp = $settingsContainerApp + "-mi" $sbAndEhArgs = @("--eventhubconnection","$($eventHubNamespaceName)|$($eventHubName)") } else { $sbAndEhArgs = @("--eventhubconnection",$eventHubConnectionString) + $params +=("--storageaccountkey",$storageAcctKey) } $sbAndEhArgs += @("--servicebustopicconnection",$serviceBusConnectionString) $tmpPath = "$($settingsContainerApp).json" Write-Host "Saving settings file to $tmpPath" -ForegroundColor DarkGreen - $params = @("containerapp", "savesettings") + $params +=("--environmentname",$containerAppEnvironmentName) $params +=("--location",$location) $params +=("--resourcegroup", $resourceGroupName) @@ -117,7 +120,7 @@ foreach($auth in $authTypes) $params +=("--settingsfile",$tmpPath) $params +=("--settingsfilekey",$keyFile) $params +=("--storageaccountname",$storageAccountName) - $params +=("--storageaccountkey",$storageAcctKey) + $params +=("--defaultscripttimeout",500) $params +=("--subscriptionid",$subscriptionId) @@ -150,10 +153,10 @@ foreach($auth in $authTypes) $params += ("--clientid", $identityClientId) } # Container apps don't take this yet! - #$params += ("--authtype", $auth) + #$params += ("--authtype", "Password") - Write-Host $params $sbAndEhArgs -ForegroundColor DarkYellow + #Write-Host $params $sbAndEhArgs -ForegroundColor DarkYellow Start-Process $sbmExe -ArgumentList ($params + $sbAndEhArgs) } diff --git a/scripts/templates/aci/create_aci_settingsfile.ps1 b/scripts/templates/aci/create_aci_settingsfile.ps1 index 60bccce6..7d9bc3ea 100644 --- a/scripts/templates/aci/create_aci_settingsfile.ps1 +++ b/scripts/templates/aci/create_aci_settingsfile.ps1 @@ -80,6 +80,8 @@ if($false -eq (Test-Path $keyFile)) $sbAndEhArgs = foreach($auth in $authTypes) { + $params = @("aci", "savesettings") + if($auth -eq "ManagedIdentity" ) { $settingsAci =$baseFileName + "-mi.json" @@ -91,10 +93,11 @@ foreach($auth in $authTypes) $settingsAci = $baseFileName + ".json" $sbAndEhArgs = @("-sb", """$serviceBusConnectionString""") $sbAndEhArgs += ("-eh","""$eventHubConnectionString""") + $params += ("--storageaccountkey",$storageAcctKey) } Write-Host "Saving settings file to $settingsAci" -ForegroundColor DarkGreen - $params = @("aci", "savesettings") + $params += ("--settingsfile", "$($settingsAci)") $params += ("--aciname", $aciName) $params += ("--identityname", $identityName) @@ -103,7 +106,7 @@ foreach($auth in $authTypes) $params += ("--acirg", $resourceGroupName) $params += ("-kv", $keyVaultName) $params += ("--storageaccountname", $storageAccountName) - $params += ("--storageaccountkey",$storageAcctKey) + $params += ("--defaultscripttimeout", "500") $params += ("--subscriptionid",$subscriptionId) $params += ("--force") diff --git a/scripts/templates/kubernetes/create_aks_job_yaml.ps1 b/scripts/templates/kubernetes/create_aks_job_yaml.ps1 index 3e95f2cb..b81694af 100644 --- a/scripts/templates/kubernetes/create_aks_job_yaml.ps1 +++ b/scripts/templates/kubernetes/create_aks_job_yaml.ps1 @@ -14,14 +14,14 @@ Write-Host "Saving AKS Job YAML files to :'$outputPath'" -ForegroundColor DarkGr # Copy sample K8s YAML files for test configs ############################################# $scriptDir = Split-Path $script:MyInvocation.MyCommand.Path -Copy-Item (Join-Path $scriptDir sample_job.yaml) (Join-Path $outputPath basic_job.yaml) -Copy-Item (Join-Path $scriptDir sample_job_keyvault.yaml) (Join-Path $outputPath basic_job_keyvault.yaml) +Copy-Item (Join-Path $scriptDir sample_job.yaml) (Join-Path $outputPath k8s-basic_job.yaml) +Copy-Item (Join-Path $scriptDir sample_job_keyvault.yaml) (Join-Path $outputPath k8s-basic_job_keyvault.yaml) if([string]::IsNullOrWhiteSpace($acrName) -eq $false) { $acrLoginServer = az acr show --resource-group $resourceGroupName --name $acrName -o tsv --query loginServer Write-Host "Using ACR login server name:'$acrLoginServer'" -ForegroundColor DarkGreen - ((Get-Content -Path (Join-Path $scriptDir sample_job.yaml)) -replace "blueskydevus", $acrLoginServer) | Out-File (Join-Path $outputPath acr_basic_job.yaml) - ((Get-Content -Path (Join-Path $scriptDir sample_job_keyvault.yaml)) -replace "blueskydevus", $acrLoginServer) | Out-File (Join-Path $outputPath acr_basic_job_keyvault.yaml) + ((Get-Content -Path (Join-Path $scriptDir sample_job.yaml)) -replace "blueskydevus", $acrLoginServer) | Out-File (Join-Path $outputPath k8s-acr_basic_job.yaml) + ((Get-Content -Path (Join-Path $scriptDir sample_job_keyvault.yaml)) -replace "blueskydevus", $acrLoginServer) | Out-File (Join-Path $outputPath k8s-acr_basic_job_keyvault.yaml) } \ No newline at end of file diff --git a/scripts/templates/kubernetes/create_aks_keyvault_config.ps1 b/scripts/templates/kubernetes/create_aks_keyvault_config.ps1 index 21fe478d..047f1fd4 100644 --- a/scripts/templates/kubernetes/create_aks_keyvault_config.ps1 +++ b/scripts/templates/kubernetes/create_aks_keyvault_config.ps1 @@ -24,7 +24,7 @@ $azureIdentityBinding = $azureIdentityBinding.Replace("{{subscriptionId}}", $sub $azureIdentityBinding = $azureIdentityBinding.Replace("{{userAssignedIdentityName}}", $userAssignedIdentityName) $azureIdentityBinding = $azureIdentityBinding.Replace("{{resourceGroupName}}", $resourceGroupName) $azureIdentityBinding = $azureIdentityBinding.Replace("{{userAssignedClientId}}", $userAssignedClientId) -$azureIdentityBinding | Out-File -FilePath (Join-Path $path "podIdentityAndBinding.yaml") +$azureIdentityBinding | Out-File -FilePath (Join-Path $path "k8s-podIdentityAndBinding.yaml") $fileName = Join-Path $path "secretProviderClass.yaml" Write-Host("Writing $fileName"); @@ -32,5 +32,5 @@ $providerClass= Get-Content "$($scriptDir)/secretProviderClass_template.yaml" $providerClass= $providerClass.Replace("{{userAssignedClientId}}", $userAssignedClientId) $providerClass= $providerClass.Replace("{{keyVaultName}}", $keyVaultName) $providerClass= $providerClass.Replace("{{tenantId}}", $tenantId) -$providerClass= $providerClass | Out-File -FilePath (Join-Path $path "secretProviderClass.yaml") +$providerClass= $providerClass | Out-File -FilePath (Join-Path $path "k8s-secretProviderClass.yaml") diff --git a/scripts/templates/kubernetes/create_aks_secrets_and_runtime_files.ps1 b/scripts/templates/kubernetes/create_aks_secrets_and_runtime_files.ps1 index cf05ad3d..68e90ad3 100644 --- a/scripts/templates/kubernetes/create_aks_secrets_and_runtime_files.ps1 +++ b/scripts/templates/kubernetes/create_aks_secrets_and_runtime_files.ps1 @@ -44,6 +44,7 @@ $params += @("-eh","""$eventHubConnectionString""") $params += @("-sb","""$serviceBusConnectionString""") $params += @("--concurrency", "5") $params += @("--concurrencytype", "Count") +$params += @("--prefix", "k8s") if($authTypes -contains "Password") @@ -61,13 +62,12 @@ if($authTypes -contains "ManagedIdentity") $params = @("k8s", "savesettings") $params += @("--path", $path) $params += @("--storageaccountname",$storageAccountName) - $params += @("--storageaccountkey","""$storageAcctKey""") $params += @("-eh","$($eventhubNamespaceName).servicebus.windows.net|$($eventHubName)") $params += @("-sb","$($serviceBusNamespaceName).servicebus.windows.net") $params += @("--concurrency", "5") $params += @("--concurrencytype", "Count") $params += @("--authtype", "ManagedIdentity") - $params += @("--prefix", "mi-full") + $params += @("--prefix", "k8s-mi") Start-Process $sbmExe -ArgumentList $params } diff --git a/scripts/templates/kubernetes/sample_job.yaml b/scripts/templates/kubernetes/sample_job.yaml index f3e988ec..6768caa1 100644 --- a/scripts/templates/kubernetes/sample_job.yaml +++ b/scripts/templates/kubernetes/sample_job.yaml @@ -10,6 +10,7 @@ spec: metadata: labels: jobgroup: sqlbuildmanager + aadpodidbinding: azure-pod-identity-binding-selector spec: containers: - name: sqlbuildmanager diff --git a/scripts/templates/kubernetes/sample_job_keyvault.yaml b/scripts/templates/kubernetes/sample_job_keyvault.yaml index bba3e72e..25aac965 100644 --- a/scripts/templates/kubernetes/sample_job_keyvault.yaml +++ b/scripts/templates/kubernetes/sample_job_keyvault.yaml @@ -10,6 +10,7 @@ spec: metadata: labels: jobgroup: sqlbuildmanager + aadpodidbinding: azure-pod-identity-binding-selector spec: containers: - name: sqlbuildmanager diff --git a/scripts/utility/Clean-ObjAndBin.ps1 b/scripts/utility/Clean-ObjAndBin.ps1 index 7524f159..4a06c2a7 100644 --- a/scripts/utility/Clean-ObjAndBin.ps1 +++ b/scripts/utility/Clean-ObjAndBin.ps1 @@ -1,2 +1,3 @@ Get-ChildItem ../../src -Include bin -Recurse -Force | Remove-Item -Recurse -Force Get-ChildItem ../../src -Include obj -Recurse -Force | Remove-Item -Recurse -Force +Remove-Item ../../src/.vs -Force diff --git a/scripts/utility/Clear_UnitTest_Batch_Jobs.ps1 b/scripts/utility/Clear_UnitTest_Batch_Jobs.ps1 index 036a73ab..72d6a937 100644 --- a/scripts/utility/Clear_UnitTest_Batch_Jobs.ps1 +++ b/scripts/utility/Clear_UnitTest_Batch_Jobs.ps1 @@ -25,7 +25,7 @@ else { foreach ($job in $jobs) { - if($job.StartsWith("SqlBuild") -or $job.StartsWith("batch-sbm")) + if($job.StartsWith("SqlBuild") -or $job.StartsWith("batch-") -or $job.StartsWith("bat-")) { Write-Host "Removing job: $($job)" -ForegroundColor Green az batch job delete --account-name $batchAccountName --account-endpoint $batchAcctEndpoint --account-key $batchAcctKey --job-id $job --yes diff --git a/scripts/utility/Clear_UnitTest_Remnants.ps1 b/scripts/utility/Clear_UnitTest_Remnants.ps1 index e15ac457..cfbcc824 100644 --- a/scripts/utility/Clear_UnitTest_Remnants.ps1 +++ b/scripts/utility/Clear_UnitTest_Remnants.ps1 @@ -2,13 +2,18 @@ param ( [string] $prefix, [string] $resourceGroupName, - [bool] $includeActive = $false + [bool] $includeActiveBatchJobs = $true, + [bool] $removeAllContainerApps = $false ) if("" -eq $resourceGroupName) { $resourceGroupName = "$prefix-rg" } -.\Clear_UnitTest_Batch_Jobs.ps1 -prefix $prefix -includeActive $includeActive +.\Clear_UnitTest_Batch_Jobs.ps1 -prefix $prefix -includeActive $includeActiveBatchJobs .\Clear_UnitTest_ServiceBus_Topics.ps1 -prefix $prefix -.\Clear_UnitTest_StorageAcct_Containers.ps1 -prefix $prefix \ No newline at end of file +.\Clear_UnitTest_StorageAcct_Containers.ps1 -prefix $prefix +if($removeAllContainerApps) +{ + .\Clean-ContainerApps.ps1 -resourceGroupName $resourceGroupName +} \ No newline at end of file diff --git a/scripts/utility/Validate_UnitTest_SettingsFiles.ps1 b/scripts/utility/Validate_UnitTest_SettingsFiles.ps1 index f14978e3..fdbbba81 100644 --- a/scripts/utility/Validate_UnitTest_SettingsFiles.ps1 +++ b/scripts/utility/Validate_UnitTest_SettingsFiles.ps1 @@ -12,7 +12,7 @@ foreach($file in $testSettingsFiles) if($name.EndsWith("yaml") -or $name.EndsWith("json")) { Write-Host "Looking for refrences to: $name" -ForegroundColor Cyan - $results = Get-ChildItem $testPath -File | Select-String -pattern $name + $results = Get-ChildItem $testPath -File | Select-String -pattern "/$name" if($null -ne $results) { Write-Host "$($results.Length) references found" -ForegroundColor Green diff --git a/src/SqlBuildManager.Console.ExternalTest/AciTests.cs b/src/SqlBuildManager.Console.ExternalTest/AciTests.cs index 5e496efa..b1e272d6 100644 --- a/src/SqlBuildManager.Console.ExternalTest/AciTests.cs +++ b/src/SqlBuildManager.Console.ExternalTest/AciTests.cs @@ -198,7 +198,7 @@ public void ACI_Queue_DacpacSource_KeyVault_Secrets_Success(string settingsFile, File.WriteAllLines(minusFirst, DatabaseHelper.ModifyTargetList(overrideFileContents, removeCount)); //Get the creds locally from the K8s file - var secretsFile = Path.GetFullPath("TestConfig/secrets.yaml"); + var secretsFile = Path.GetFullPath("TestConfig/k8s-secrets.yaml"); var ymlD = new ydn.Deserializer(); var obj = ymlD.Deserialize(File.ReadAllText(secretsFile)); var pw = Encoding.UTF8.GetString(Convert.FromBase64String(obj["data"]["Password"])); @@ -263,7 +263,7 @@ public void ACI_Queue_DacpacSource_KeyVault_Secrets_Success(string settingsFile, } [DataRow("TestConfig/settingsfile-aci.json", "latest-vNext", 3, 2, ConcurrencyType.Count)] [DataTestMethod] - public void ACI_Queue_DacpacSource_ForceApplyCustom_eyVault_Secrets_Success(string settingsFile, string imageTag, int containerCount, int concurrency, ConcurrencyType concurrencyType) + public void ACI_Queue_DacpacSource_ForceApplyCustom_KeyVault_Secrets_Success(string settingsFile, string imageTag, int containerCount, int concurrency, ConcurrencyType concurrencyType) { settingsFile = Path.GetFullPath(settingsFile); var overrideFile = Path.GetFullPath("TestConfig/databasetargets.cfg"); @@ -284,7 +284,7 @@ public void ACI_Queue_DacpacSource_ForceApplyCustom_eyVault_Secrets_Success(stri File.WriteAllLines(minusFirst, DatabaseHelper.ModifyTargetList(overrideFileContents, removeCount)); //Get the creds locally from the K8s file - var secretsFile = Path.GetFullPath("TestConfig/secrets.yaml"); + var secretsFile = Path.GetFullPath("TestConfig/k8s-secrets.yaml"); var ymlD = new ydn.Deserializer(); var obj = ymlD.Deserialize(File.ReadAllText(secretsFile)); var pw = Encoding.UTF8.GetString(Convert.FromBase64String(obj["data"]["Password"])); diff --git a/src/SqlBuildManager.Console.ExternalTest/BatchTests.cs b/src/SqlBuildManager.Console.ExternalTest/BatchTests.cs index 336d83d5..183fdd25 100644 --- a/src/SqlBuildManager.Console.ExternalTest/BatchTests.cs +++ b/src/SqlBuildManager.Console.ExternalTest/BatchTests.cs @@ -33,9 +33,9 @@ public void ConfigureProcessInfo() { SqlBuildManager.Logging.ApplicationLogging.CreateLogger("SqlBuildManager.Console.log", @"C:\temp"); - this.settingsFilePath = Path.GetFullPath("TestConfig/settingsfile-windows.json"); + this.settingsFilePath = Path.GetFullPath("TestConfig/settingsfile-batch-windows.json"); this.settingsFileKeyPath = Path.GetFullPath("TestConfig/settingsfilekey.txt"); - this.linuxSettingsFilePath = Path.GetFullPath("TestConfig/settingsfile-linux.json"); + this.linuxSettingsFilePath = Path.GetFullPath("TestConfig/settingsfile-batch-linux.json"); this.overrideFilePath = Path.GetFullPath("TestConfig/databasetargets.cfg"); this.cmdLine = new CommandLineArgs(); @@ -106,17 +106,17 @@ public static string ReleventLogFileContents(int startingLine) - [DataRow("runthreaded", "TestConfig/settingsfile-windows.json", ConcurrencyType.Count,10)] - [DataRow("run", "TestConfig/settingsfile-windows.json", ConcurrencyType.Count, 10)] - [DataRow("run", "TestConfig/settingsfile-linux.json", ConcurrencyType.Count, 10)] + [DataRow("runthreaded", "TestConfig/settingsfile-batch-windows.json", ConcurrencyType.Count,10)] + [DataRow("run", "TestConfig/settingsfile-batch-windows.json", ConcurrencyType.Count, 10)] + [DataRow("run", "TestConfig/settingsfile-batch-linux.json", ConcurrencyType.Count, 10)] - [DataRow("runthreaded", "TestConfig/settingsfile-windows.json", ConcurrencyType.Server, 2)] - [DataRow("run", "TestConfig/settingsfile-windows.json", ConcurrencyType.Server, 2)] - [DataRow("run", "TestConfig/settingsfile-linux.json", ConcurrencyType.Server, 2)] + [DataRow("runthreaded", "TestConfig/settingsfile-batch-windows.json", ConcurrencyType.Server, 2)] + [DataRow("run", "TestConfig/settingsfile-batch-windows.json", ConcurrencyType.Server, 2)] + [DataRow("run", "TestConfig/settingsfile-batch-linux.json", ConcurrencyType.Server, 2)] - [DataRow("runthreaded", "TestConfig/settingsfile-windows.json", ConcurrencyType.MaxPerServer, 2)] - [DataRow("run", "TestConfig/settingsfile-windows.json", ConcurrencyType.MaxPerServer, 2)] - [DataRow("run", "TestConfig/settingsfile-linux.json", ConcurrencyType.MaxPerServer, 2)] + [DataRow("runthreaded", "TestConfig/settingsfile-batch-windows.json", ConcurrencyType.MaxPerServer, 2)] + [DataRow("run", "TestConfig/settingsfile-batch-windows.json", ConcurrencyType.MaxPerServer, 2)] + [DataRow("run", "TestConfig/settingsfile-batch-linux.json", ConcurrencyType.MaxPerServer, 2)] [DataTestMethod] public void Batch_Override_SBMSource_ByConcurrencyType_Success(string batchMethod, string settingsFile, ConcurrencyType concurType, int concurrency) { @@ -159,8 +159,8 @@ public void Batch_Override_SBMSource_ByConcurrencyType_Success(string batchMetho } } - [DataRow("run", "TestConfig/settingsfile-windows-mi.json", ConcurrencyType.Count, 10)] - [DataRow("run", "TestConfig/settingsfile-linux-mi.json", ConcurrencyType.Count, 10)] + [DataRow("run", "TestConfig/settingsfile-batch-windows-mi.json", ConcurrencyType.Count, 10)] + [DataRow("run", "TestConfig/settingsfile-batch-linux-mi.json", ConcurrencyType.Count, 10)] [DataTestMethod] public void Batch_Override_SBMSource_ManagedIdentity_ByConcurrencyType_Success(string batchMethod, string settingsFile, ConcurrencyType concurType, int concurrency) { @@ -206,9 +206,9 @@ public void Batch_Override_SBMSource_ManagedIdentity_ByConcurrencyType_Success(s - [DataRow("runthreaded", "TestConfig/settingsfile-windows.json")] - [DataRow("run", "TestConfig/settingsfile-windows.json")] - [DataRow("run", "TestConfig/settingsfile-linux.json")] + [DataRow("runthreaded", "TestConfig/settingsfile-batch-windows.json")] + [DataRow("run", "TestConfig/settingsfile-batch-windows.json")] + [DataRow("run", "TestConfig/settingsfile-batch-linux.json")] [DataTestMethod] public void Batch_Override_SBMSource_RunWithError_MissingPackage(string batchMethod, string settingsFile) { @@ -235,9 +235,9 @@ public void Batch_Override_SBMSource_RunWithError_MissingPackage(string batchMet } - [DataRow("runthreaded", "TestConfig/settingsfile-windows.json")] - [DataRow("run", "TestConfig/settingsfile-windows.json")] - [DataRow("run", "TestConfig/settingsfile-linux.json")] + [DataRow("runthreaded", "TestConfig/settingsfile-batch-windows.json")] + [DataRow("run", "TestConfig/settingsfile-batch-windows.json")] + [DataRow("run", "TestConfig/settingsfile-batch-linux.json")] [DataTestMethod] public void Batch_Override_PlatinumDbSource_Succes(string batchMethod, string settingsFile) { @@ -278,9 +278,9 @@ public void Batch_Override_PlatinumDbSource_Succes(string batchMethod, string se } } - [DataRow("runthreaded", "TestConfig/settingsfile-windows.json")] - [DataRow("run", "TestConfig/settingsfile-windows.json")] - [DataRow("run", "TestConfig/settingsfile-linux.json")] + [DataRow("runthreaded", "TestConfig/settingsfile-batch-windows.json")] + [DataRow("run", "TestConfig/settingsfile-batch-windows.json")] + [DataRow("run", "TestConfig/settingsfile-batch-linux.json")] [DataTestMethod] public void Batch_Override_PlatinumDbSource_FirstDbAlreadyInSync(string batchMethod, string settingsFile) { @@ -328,9 +328,9 @@ public void Batch_Override_PlatinumDbSource_FirstDbAlreadyInSync(string batchMet } } - [DataRow("runthreaded", "TestConfig/settingsfile-windows.json")] - [DataRow("run", "TestConfig/settingsfile-windows.json")] - [DataRow("run", "TestConfig/settingsfile-linux.json")] + [DataRow("runthreaded", "TestConfig/settingsfile-batch-windows.json")] + [DataRow("run", "TestConfig/settingsfile-batch-windows.json")] + [DataRow("run", "TestConfig/settingsfile-batch-linux.json")] [DataTestMethod] public void Batch_Override_PlatinumDbSource_ADbAlreadyInSync(string batchMethod, string settingsFile) { @@ -381,9 +381,9 @@ public void Batch_Override_PlatinumDbSource_ADbAlreadyInSync(string batchMethod, } - [DataRow("runthreaded", "TestConfig/settingsfile-windows.json")] - [DataRow("run", "TestConfig/settingsfile-windows.json")] - [DataRow("run", "TestConfig/settingsfile-linux.json")] + [DataRow("runthreaded", "TestConfig/settingsfile-batch-windows.json")] + [DataRow("run", "TestConfig/settingsfile-batch-windows.json")] + [DataRow("run", "TestConfig/settingsfile-batch-linux.json")] [DataTestMethod] public void Batch_Override_DacpacSource_Success(string batchMethod, string settingsFile) { @@ -429,9 +429,9 @@ public void Batch_Override_DacpacSource_Success(string batchMethod, string setti } - [DataRow("runthreaded", "TestConfig/settingsfile-windows.json")] - [DataRow("run", "TestConfig/settingsfile-windows.json")] - [DataRow("run", "TestConfig/settingsfile-linux.json")] + [DataRow("runthreaded", "TestConfig/settingsfile-batch-windows.json")] + [DataRow("run", "TestConfig/settingsfile-batch-windows.json")] + [DataRow("run", "TestConfig/settingsfile-batch-linux.json")] [DataTestMethod] public void Batch_Override_DacpacSource_FirstDbAlreadyInSync(string batchMethod, string settingsFile) { @@ -482,9 +482,9 @@ public void Batch_Override_DacpacSource_FirstDbAlreadyInSync(string batchMethod, } - [DataRow("querythreaded", "TestConfig/settingsfile-windows.json")] - [DataRow("query", "TestConfig/settingsfile-windows.json")] - [DataRow("query", "TestConfig/settingsfile-linux.json")] + [DataRow("querythreaded", "TestConfig/settingsfile-batch-windows.json")] + [DataRow("query", "TestConfig/settingsfile-batch-windows.json")] + [DataRow("query", "TestConfig/settingsfile-batch-linux.json")] [DataTestMethod] public void Batch_Query_SelectSuccess(string batchMethod, string settingsFile) { @@ -546,9 +546,9 @@ public void Batch_Query_SelectSuccess(string batchMethod, string settingsFile) } - [DataRow("querythreaded", "TestConfig/settingsfile-windows.json")] - [DataRow("query", "TestConfig/settingsfile-windows.json")] - [DataRow("query", "TestConfig/settingsfile-linux.json")] + [DataRow("querythreaded", "TestConfig/settingsfile-batch-windows.json")] + [DataRow("query", "TestConfig/settingsfile-batch-windows.json")] + [DataRow("query", "TestConfig/settingsfile-batch-linux.json")] [DataTestMethod] public void Batch_Query_InsertFail(string batchMethod, string settingsFile) { @@ -585,9 +585,9 @@ public void Batch_Query_InsertFail(string batchMethod, string settingsFile) Assert.IsTrue(logFileContents.Contains("An INSERT, UPDATE or DELETE keyword was found"), "An INSERT statement should have been found"); } - [DataRow("querythreaded", "TestConfig/settingsfile-windows.json")] - [DataRow("query", "TestConfig/settingsfile-windows.json")] - [DataRow("query", "TestConfig/settingsfile-linux.json")] + [DataRow("querythreaded", "TestConfig/settingsfile-batch-windows.json")] + [DataRow("query", "TestConfig/settingsfile-batch-windows.json")] + [DataRow("query", "TestConfig/settingsfile-batch-linux.json")] [DataTestMethod] public void Batch_Query_DeleteFail(string batchMethod, string settingsFile) { @@ -624,9 +624,9 @@ public void Batch_Query_DeleteFail(string batchMethod, string settingsFile) Assert.IsTrue(logFileContents.Contains("An INSERT, UPDATE or DELETE keyword was found"), "A DELETE statement should have been found"); } - [DataRow("querythreaded", "TestConfig/settingsfile-windows.json")] - [DataRow("query", "TestConfig/settingsfile-windows.json")] - [DataRow("query", "TestConfig/settingsfile-linux.json")] + [DataRow("querythreaded", "TestConfig/settingsfile-batch-windows.json")] + [DataRow("query", "TestConfig/settingsfile-batch-windows.json")] + [DataRow("query", "TestConfig/settingsfile-batch-linux.json")] [DataTestMethod] public void Batch_Query_UpdateFail(string batchMethod, string settingsFile) { @@ -662,21 +662,21 @@ public void Batch_Query_UpdateFail(string batchMethod, string settingsFile) //Assert.IsTrue(logFileContents.Contains("An INSERT, UPDATE or DELETE keyword was found"), "An UPDATE statement should have been found"); } - [DataRow("runthreaded", "TestConfig/settingsfile-windows-queue.json", ConcurrencyType.Server, 2)] - [DataRow("run", "TestConfig/settingsfile-windows-queue.json", ConcurrencyType.Server, 2)] - [DataRow("run", "TestConfig/settingsfile-linux-queue.json",ConcurrencyType.Server, 2)] - [DataRow("run", "TestConfig/settingsfile-windows-queue-keyvault.json",ConcurrencyType.Server, 2)] - [DataRow("run", "TestConfig/settingsfile-linux-queue-keyvault.json",ConcurrencyType.Server, 2)] - [DataRow("runthreaded", "TestConfig/settingsfile-windows-queue.json", ConcurrencyType.MaxPerServer, 5)] - [DataRow("run", "TestConfig/settingsfile-windows-queue.json", ConcurrencyType.MaxPerServer, 5)] - [DataRow("run", "TestConfig/settingsfile-linux-queue.json", ConcurrencyType.MaxPerServer, 5)] - [DataRow("run", "TestConfig/settingsfile-windows-queue-keyvault.json", ConcurrencyType.MaxPerServer, 5)] - [DataRow("run", "TestConfig/settingsfile-linux-queue-keyvault.json", ConcurrencyType.MaxPerServer, 5)] - [DataRow("runthreaded", "TestConfig/settingsfile-windows-queue.json", ConcurrencyType.Count, 5)] - [DataRow("run", "TestConfig/settingsfile-windows-queue.json", ConcurrencyType.Count, 5)] - [DataRow("run", "TestConfig/settingsfile-linux-queue.json", ConcurrencyType.Count, 5)] - [DataRow("run", "TestConfig/settingsfile-windows-queue-keyvault.json", ConcurrencyType.Count, 5)] - [DataRow("run", "TestConfig/settingsfile-linux-queue-keyvault.json", ConcurrencyType.Count, 5)] + [DataRow("runthreaded", "TestConfig/settingsfile-batch-windows-queue.json", ConcurrencyType.Server, 2)] + [DataRow("run", "TestConfig/settingsfile-batch-windows-queue.json", ConcurrencyType.Server, 2)] + [DataRow("run", "TestConfig/settingsfile-batch-linux-queue.json",ConcurrencyType.Server, 2)] + [DataRow("run", "TestConfig/settingsfile-batch-windows-queue-keyvault.json",ConcurrencyType.Server, 2)] + [DataRow("run", "TestConfig/settingsfile-batch-linux-queue-keyvault.json",ConcurrencyType.Server, 2)] + [DataRow("runthreaded", "TestConfig/settingsfile-batch-windows-queue.json", ConcurrencyType.MaxPerServer, 5)] + [DataRow("run", "TestConfig/settingsfile-batch-windows-queue.json", ConcurrencyType.MaxPerServer, 5)] + [DataRow("run", "TestConfig/settingsfile-batch-linux-queue.json", ConcurrencyType.MaxPerServer, 5)] + [DataRow("run", "TestConfig/settingsfile-batch-windows-queue-keyvault.json", ConcurrencyType.MaxPerServer, 5)] + [DataRow("run", "TestConfig/settingsfile-batch-linux-queue-keyvault.json", ConcurrencyType.MaxPerServer, 5)] + [DataRow("runthreaded", "TestConfig/settingsfile-batch-windows-queue.json", ConcurrencyType.Count, 5)] + [DataRow("run", "TestConfig/settingsfile-batch-windows-queue.json", ConcurrencyType.Count, 5)] + [DataRow("run", "TestConfig/settingsfile-batch-linux-queue.json", ConcurrencyType.Count, 5)] + [DataRow("run", "TestConfig/settingsfile-batch-windows-queue-keyvault.json", ConcurrencyType.Count, 5)] + [DataRow("run", "TestConfig/settingsfile-batch-linux-queue-keyvault.json", ConcurrencyType.Count, 5)] [DataTestMethod] public void Batch_Queue_SBMSource_ByConcurrencyType_Success(string batchMethod, string settingsFile, ConcurrencyType concurType, int concurrency) { @@ -727,10 +727,10 @@ public void Batch_Queue_SBMSource_ByConcurrencyType_Success(string batchMethod, Assert.AreEqual(0, result, StandardExecutionErrorMessage(logFileContents)); } - [DataRow("run", "TestConfig/settingsfile-windows-queue-mi.json", ConcurrencyType.Count, 10)] - [DataRow("run", "TestConfig/settingsfile-windows-queue-keyvault-mi.json", ConcurrencyType.Count, 10)] - [DataRow("run", "TestConfig/settingsfile-linux-queue-mi.json", ConcurrencyType.Count, 10)] - [DataRow("run", "TestConfig/settingsfile-linux-queue-keyvault-mi.json", ConcurrencyType.Count, 10)] + [DataRow("run", "TestConfig/settingsfile-batch-windows-queue-mi.json", ConcurrencyType.Count, 10)] + [DataRow("run", "TestConfig/settingsfile-batch-windows-queue-keyvault-mi.json", ConcurrencyType.Count, 10)] + [DataRow("run", "TestConfig/settingsfile-batch-linux-queue-mi.json", ConcurrencyType.Count, 10)] + [DataRow("run", "TestConfig/settingsfile-batch-linux-queue-keyvault-mi.json", ConcurrencyType.Count, 10)] [DataTestMethod] public void Batch_Queue_SBMSource_ManagedIdentity_ByConcurrencyType_Success(string batchMethod, string settingsFile, ConcurrencyType concurType, int concurrency) { @@ -796,8 +796,8 @@ public void Batch_Queue_SBMSource_ManagedIdentity_ByConcurrencyType_Success(stri } } - [DataRow("run", "TestConfig/settingsfile-linux-queue.json", ConcurrencyType.MaxPerServer, 5)] - [DataRow("run", "TestConfig/settingsfile-windows-queue.json", ConcurrencyType.Count, 5)] + [DataRow("run", "TestConfig/settingsfile-batch-linux-queue.json", ConcurrencyType.MaxPerServer, 5)] + [DataRow("run", "TestConfig/settingsfile-batch-windows-queue.json", ConcurrencyType.Count, 5)] [DataTestMethod] public void Batch_Queue_SBMSource_MissingEventHubConnection_Success(string batchMethod, string settingsFile, ConcurrencyType concurType, int concurrency) { @@ -859,11 +859,11 @@ public void Batch_Queue_SBMSource_MissingEventHubConnection_Success(string batch Assert.AreEqual(0, result, StandardExecutionErrorMessage(logFileContents)); } - [DataRow("runthreaded", "TestConfig/settingsfile-windows-queue.json")] - [DataRow("run", "TestConfig/settingsfile-windows-queue.json")] - [DataRow("run", "TestConfig/settingsfile-linux-queue.json")] - [DataRow("run", "TestConfig/settingsfile-windows-queue-keyvault.json")] - [DataRow("run", "TestConfig/settingsfile-linux-queue-keyvault.json")] + [DataRow("runthreaded", "TestConfig/settingsfile-batch-windows-queue.json")] + [DataRow("run", "TestConfig/settingsfile-batch-windows-queue.json")] + [DataRow("run", "TestConfig/settingsfile-batch-linux-queue.json")] + [DataRow("run", "TestConfig/settingsfile-batch-windows-queue-keyvault.json")] + [DataRow("run", "TestConfig/settingsfile-batch-linux-queue-keyvault.json")] [DataTestMethod] public void Batch_Queue_PlatinumDbSource_Success(string batchMethod, string settingsFile) { @@ -920,11 +920,11 @@ public void Batch_Queue_PlatinumDbSource_Success(string batchMethod, string sett Assert.AreEqual(0, result, StandardExecutionErrorMessage(logFileContents)); } - [DataRow("runthreaded", "TestConfig/settingsfile-windows-queue.json")] - [DataRow("run", "TestConfig/settingsfile-windows-queue.json")] - [DataRow("run", "TestConfig/settingsfile-linux-queue.json")] - [DataRow("run", "TestConfig/settingsfile-windows-queue-keyvault.json")] - [DataRow("run", "TestConfig/settingsfile-linux-queue-keyvault.json")] + [DataRow("runthreaded", "TestConfig/settingsfile-batch-windows-queue.json")] + [DataRow("run", "TestConfig/settingsfile-batch-windows-queue.json")] + [DataRow("run", "TestConfig/settingsfile-batch-linux-queue.json")] + [DataRow("run", "TestConfig/settingsfile-batch-windows-queue-keyvault.json")] + [DataRow("run", "TestConfig/settingsfile-batch-linux-queue-keyvault.json")] [DataTestMethod] public void Batch_Queue_DacpacSource_Success(string batchMethod, string settingsFile) { @@ -984,8 +984,8 @@ public void Batch_Queue_DacpacSource_Success(string batchMethod, string settings } - [DataRow("runthreaded", "TestConfig/settingsfile-windows.json")] - [DataRow("run", "TestConfig/settingsfile-linux.json")] + [DataRow("runthreaded", "TestConfig/settingsfile-batch-windows.json")] + [DataRow("run", "TestConfig/settingsfile-batch-linux.json")] [DataTestMethod] public void Batch_Override_DacpacSource_FocceApplyCustom_Success(string batchMethod, string settingsFile) { @@ -1039,8 +1039,8 @@ public void Batch_Override_DacpacSource_FocceApplyCustom_Success(string batchMet } - [DataRow("runthreaded", "TestConfig/settingsfile-windows-queue.json")] - [DataRow("run", "TestConfig/settingsfile-linux-queue.json")] + [DataRow("runthreaded", "TestConfig/settingsfile-batch-windows-queue.json")] + [DataRow("run", "TestConfig/settingsfile-batch-linux-queue.json")] [DataTestMethod] public void Batch_Queue_DacpacSource_FocceApplyCustom_Success(string batchMethod, string settingsFile) { diff --git a/src/SqlBuildManager.Console.ExternalTest/ContainerAppTests.cs b/src/SqlBuildManager.Console.ExternalTest/ContainerAppTests.cs index be451301..722cc01d 100644 --- a/src/SqlBuildManager.Console.ExternalTest/ContainerAppTests.cs +++ b/src/SqlBuildManager.Console.ExternalTest/ContainerAppTests.cs @@ -125,10 +125,10 @@ public void ContainerApp_Queue_SBMSource_Success(string settingsFile, string ima } //TODO: Enable Managed Identity****** Managed Identity for SQL Authentication is not available for Container Apps currently, only SB and EH - [DataRow("TestConfig/settingsfile-containerapp-kv-mi.json", "latest-vNext", 3, 2, ConcurrencyType.Count)] + //[DataRow("TestConfig/settingsfile-containerapp-kv-mi.json", "latest-vNext", 3, 2, ConcurrencyType.Count)] [DataRow("TestConfig/settingsfile-containerapp-mi.json", "latest-vNext", 3, 2, ConcurrencyType.Count)] - [DataRow("TestConfig/settingsfile-containerapp-no-registry-mi.json", "latest-vNext", 3, 2, ConcurrencyType.Count)] - [DataRow("TestConfig/settingsfile-containerapp-no-registry-kv-mi.json", "latest-vNext", 3, 2, ConcurrencyType.Count)] + //[DataRow("TestConfig/settingsfile-containerapp-no-registry-mi.json", "latest-vNext", 3, 2, ConcurrencyType.Count)] + //[DataRow("TestConfig/settingsfile-containerapp-no-registry-kv-mi.json", "latest-vNext", 3, 2, ConcurrencyType.Count)] [DataTestMethod] public void ContainerApp_Queue_ManagedIdentity_SBMSource_Success(string settingsFile, string imageTag, int containerCount, int concurrency, ConcurrencyType concurrencyType) @@ -224,7 +224,7 @@ public void ContainerApp_EnvOnly_Queue_SBMSource_Success(string settingsFile, st int startingLine = TestHelper.LogFileCurrentLineCount(); RootCommand rootCommand = CommandLineBuilder.SetUp(); - string jobName = TestHelper.GetUniqueJobName("c"); + string jobName = TestHelper.GetUniqueJobName("ca"); string outputFile = Path.Combine(Directory.GetCurrentDirectory(), jobName + ".json"); //Prep the build diff --git a/src/SqlBuildManager.Console.ExternalTest/KubernetesTests.cs b/src/SqlBuildManager.Console.ExternalTest/KubernetesTests.cs index 154bf087..87c60948 100644 --- a/src/SqlBuildManager.Console.ExternalTest/KubernetesTests.cs +++ b/src/SqlBuildManager.Console.ExternalTest/KubernetesTests.cs @@ -37,14 +37,16 @@ public void CleanUp() } - [DataRow("TestConfig/runtime.yaml", "TestConfig/secrets.yaml", "TestConfig/basic_job.yaml")] - [DataRow("TestConfig/runtime.yaml", "TestConfig/secrets.yaml", "TestConfig/acr_basic_job.yaml")] + [DataRow("TestConfig/k8s-runtime.yaml", "TestConfig/k8s-secrets.yaml", "TestConfig/k8s-basic_job.yaml")] + [DataRow("TestConfig/k8s-runtime.yaml", "TestConfig/k8s-secrets.yaml", "TestConfig/k8s-acr_basic_job.yaml")] [DataTestMethod] public void Kubernetes_Queue_SBMSource_Local_Secrets_Success(string runtimeFile, string secretsFile, string deployFile) { var prc = new ProcessHelper(); secretsFile = Path.GetFullPath(secretsFile); runtimeFile = Path.GetFullPath(runtimeFile); + string testRunTimeFile = Path.Combine(Path.GetDirectoryName(runtimeFile), "k8s-runtime-test.yaml"); + File.Copy(runtimeFile, testRunTimeFile, true); deployFile = Path.GetFullPath(deployFile); var overrideFile = Path.GetFullPath("TestConfig/databasetargets.cfg"); var sbmFileName = Path.GetFullPath("SimpleSelect.sbm"); @@ -66,7 +68,7 @@ public void Kubernetes_Queue_SBMSource_Local_Secrets_Success(string runtimeFile, var args = new string[]{ "k8s", "prep", "--secretsfile", secretsFile, - "--runtimefile", runtimeFile, + "--runtimefile", testRunTimeFile, "--jobname", TestHelper.GetUniqueJobName("k8s"), "--packagename", sbmFileName}; @@ -79,7 +81,7 @@ public void Kubernetes_Queue_SBMSource_Local_Secrets_Success(string runtimeFile, args = new string[]{ "k8s", "enqueue", "--secretsfile", secretsFile, - "--runtimefile", runtimeFile, + "--runtimefile", testRunTimeFile, "--override", overrideFile}; val = rootCommand.InvokeAsync(args); val.Wait(); @@ -89,7 +91,7 @@ public void Kubernetes_Queue_SBMSource_Local_Secrets_Success(string runtimeFile, result = prc.ExecuteProcess("kubectl", $"apply -f {secretsFile}"); Assert.AreEqual(0, result); - result = prc.ExecuteProcess("kubectl", $"apply -f {runtimeFile}"); + result = prc.ExecuteProcess("kubectl", $"apply -f {testRunTimeFile}"); Assert.AreEqual(0, result); result = prc.ExecuteProcess("kubectl", $"apply -f {deployFile}"); @@ -102,7 +104,7 @@ public void Kubernetes_Queue_SBMSource_Local_Secrets_Success(string runtimeFile, args = new string[]{ "k8s", "monitor", "--secretsfile", secretsFile, - "--runtimefile", runtimeFile, + "--runtimefile", testRunTimeFile, "--override", overrideFile, "--unittest", "true"}; val = rootCommand.InvokeAsync(args); @@ -113,14 +115,16 @@ public void Kubernetes_Queue_SBMSource_Local_Secrets_Success(string runtimeFile, } - [DataRow("TestConfig/mi-full-runtime.yaml", "TestConfig/mi-full-secrets.yaml", "TestConfig/secretProviderClass.yaml", "TestConfig/podIdentityAndBinding.yaml", "TestConfig/acr_basic_job.yaml")] + [DataRow("TestConfig/k8s-mi-runtime.yaml", "TestConfig/k8s-secretProviderClass.yaml", "TestConfig/k8s-podIdentityAndBinding.yaml", "TestConfig/k8s-acr_basic_job.yaml")] [DataTestMethod] - public void Kubernetes_Queue_SBMSource_ManagedIdentity_Success(string runtimeFile, string secretsFile, string secretsProviderFile, string podIdentityFile, string deployFile) + public void Kubernetes_Queue_SBMSource_ManagedIdentity_Success(string runtimeFile, string secretsProviderFile, string podIdentityFile, string deployFile) { var prc = new ProcessHelper(); - secretsFile = Path.GetFullPath(secretsFile); + runtimeFile = Path.GetFullPath(runtimeFile); + string testRunTimeFile = Path.Combine(Path.GetDirectoryName(runtimeFile), "k8s-runtime-test.yaml"); + File.Copy(runtimeFile, testRunTimeFile, true); deployFile = Path.GetFullPath(deployFile); var overrideFile = Path.GetFullPath("TestConfig/databasetargets.cfg"); var sbmFileName = Path.GetFullPath("SimpleSelect.sbm"); @@ -141,8 +145,7 @@ public void Kubernetes_Queue_SBMSource_ManagedIdentity_Success(string runtimeFil //Prep the build var args = new string[]{ "k8s", "prep", - "--secretsfile", secretsFile, - "--runtimefile", runtimeFile, + "--runtimefile", testRunTimeFile, "--jobname", TestHelper.GetUniqueJobName("k8s"), "--packagename", sbmFileName}; @@ -154,8 +157,7 @@ public void Kubernetes_Queue_SBMSource_ManagedIdentity_Success(string runtimeFil //enqueue the topic messages args = new string[]{ "k8s", "enqueue", - "--secretsfile", secretsFile, - "--runtimefile", runtimeFile, + "--runtimefile", testRunTimeFile, "--override", overrideFile}; val = rootCommand.InvokeAsync(args); val.Wait(); @@ -168,10 +170,8 @@ public void Kubernetes_Queue_SBMSource_ManagedIdentity_Success(string runtimeFil result = prc.ExecuteProcess("kubectl", $"apply -f {podIdentityFile}"); Assert.AreEqual(0, result, "Failed to apply pod identity file"); - result = prc.ExecuteProcess("kubectl", $"apply -f {secretsFile}"); - Assert.AreEqual(0, result, "Failed to apply secrets file"); - - result = prc.ExecuteProcess("kubectl", $"apply -f {runtimeFile}"); + + result = prc.ExecuteProcess("kubectl", $"apply -f {testRunTimeFile}"); Assert.AreEqual(0, result, "Failed to apply runtime configmap file"); result = prc.ExecuteProcess("kubectl", $"apply -f {deployFile}"); @@ -183,8 +183,7 @@ public void Kubernetes_Queue_SBMSource_ManagedIdentity_Success(string runtimeFil //monitor for completion args = new string[]{ "k8s", "monitor", - "--secretsfile", secretsFile, - "--runtimefile", runtimeFile, + "--runtimefile", testRunTimeFile, "--override", overrideFile, "--unittest", "true"}; val = rootCommand.InvokeAsync(args); @@ -195,8 +194,8 @@ public void Kubernetes_Queue_SBMSource_ManagedIdentity_Success(string runtimeFil } - [DataRow("TestConfig/runtime.yaml", "TestConfig/secrets.yaml", "TestConfig/secretProviderClass.yaml", "TestConfig/podIdentityAndBinding.yaml", "TestConfig/basic_job_keyvault.yaml")] - [DataRow("TestConfig/runtime.yaml", "TestConfig/secrets.yaml", "TestConfig/secretProviderClass.yaml", "TestConfig/podIdentityAndBinding.yaml", "TestConfig/acr_basic_job_keyvault.yaml")] + [DataRow("TestConfig/k8s-runtime.yaml", "TestConfig/k8s-secrets.yaml", "TestConfig/k8s-secretProviderClass.yaml", "TestConfig/k8s-podIdentityAndBinding.yaml", "TestConfig/k8s-basic_job_keyvault.yaml")] + [DataRow("TestConfig/k8s-runtime.yaml", "TestConfig/k8s-secrets.yaml", "TestConfig/k8s-secretProviderClass.yaml", "TestConfig/k8s-podIdentityAndBinding.yaml", "TestConfig/k8s-acr_basic_job_keyvault.yaml")] [DataTestMethod] public void Kubernetes_Queue_SBMSource_KeyVault_Secrets_Success(string runtimeFile, string secretsFile, string secretsProviderFile, string podIdentityFile, string deployFile) { @@ -204,6 +203,8 @@ public void Kubernetes_Queue_SBMSource_KeyVault_Secrets_Success(string runtimeFi secretsProviderFile = Path.GetFullPath(secretsProviderFile); podIdentityFile = Path.GetFullPath(podIdentityFile); runtimeFile = Path.GetFullPath(runtimeFile); + string testRunTimeFile = Path.Combine(Path.GetDirectoryName(runtimeFile), "k8s-runtime-test.yaml"); + File.Copy(runtimeFile, testRunTimeFile, true); secretsFile = Path.GetFullPath(secretsFile); deployFile = Path.GetFullPath(deployFile); var overrideFile = Path.GetFullPath("TestConfig/databasetargets.cfg"); @@ -226,7 +227,7 @@ public void Kubernetes_Queue_SBMSource_KeyVault_Secrets_Success(string runtimeFi var args = new string[]{ "k8s", "prep", "--secretsfile", secretsFile, - "--runtimefile", runtimeFile, + "--runtimefile", testRunTimeFile, "--jobname", TestHelper.GetUniqueJobName("k8s-kv"), "--packagename", sbmFileName}; @@ -239,7 +240,7 @@ public void Kubernetes_Queue_SBMSource_KeyVault_Secrets_Success(string runtimeFi args = new string[]{ "k8s", "enqueue", "--secretsfile", secretsFile, - "--runtimefile", runtimeFile, + "--runtimefile", testRunTimeFile, "--override", overrideFile}; val = rootCommand.InvokeAsync(args); val.Wait(); @@ -252,7 +253,7 @@ public void Kubernetes_Queue_SBMSource_KeyVault_Secrets_Success(string runtimeFi result = prc.ExecuteProcess("kubectl", $"apply -f {podIdentityFile}"); Assert.AreEqual(0, result, "Failed to apply pod identity file"); - result = prc.ExecuteProcess("kubectl", $"apply -f {runtimeFile}"); + result = prc.ExecuteProcess("kubectl", $"apply -f {testRunTimeFile}"); Assert.AreEqual(0, result, "Failed to apply runtime file"); result = prc.ExecuteProcess("kubectl", $"apply -f {deployFile}"); @@ -261,12 +262,12 @@ public void Kubernetes_Queue_SBMSource_KeyVault_Secrets_Success(string runtimeFi result = prc.ExecuteProcess("kubectl", $"get pods"); Assert.AreEqual(0, result); - secretsFile = Path.GetFullPath("TestConfig/mi-full-secrets.yaml"); + //monitor for completion args = new string[]{ "k8s", "monitor", "--secretsfile", secretsFile, - "--runtimefile", runtimeFile, + "--runtimefile", testRunTimeFile, "--override", overrideFile, "--unittest", "true"}; val = rootCommand.InvokeAsync(args); @@ -277,8 +278,8 @@ public void Kubernetes_Queue_SBMSource_KeyVault_Secrets_Success(string runtimeFi } - [DataRow("TestConfig/runtime.yaml", "TestConfig/secrets.yaml", "TestConfig/secretProviderClass.yaml", "TestConfig/podIdentityAndBinding.yaml", "TestConfig/basic_job_keyvault.yaml")] - [DataRow("TestConfig/runtime.yaml", "TestConfig/secrets.yaml", "TestConfig/secretProviderClass.yaml", "TestConfig/podIdentityAndBinding.yaml", "TestConfig/acr_basic_job_keyvault.yaml")] + [DataRow("TestConfig/k8s-runtime.yaml", "TestConfig/k8s-secrets.yaml", "TestConfig/k8s-secretProviderClass.yaml", "TestConfig/k8s-podIdentityAndBinding.yaml", "TestConfig/k8s-basic_job_keyvault.yaml")] + [DataRow("TestConfig/k8s-runtime.yaml", "TestConfig/k8s-secrets.yaml", "TestConfig/k8s-secretProviderClass.yaml", "TestConfig/k8s-podIdentityAndBinding.yaml", "TestConfig/k8s-acr_basic_job_keyvault.yaml")] [DataTestMethod] public void Kubernetes_Queue_DacpacSource_KeyVault_Secrets_Success(string runtimeFile, string secretsFile, string secretsProviderFile, string podIdentityFile, string deployFile) { @@ -286,6 +287,8 @@ public void Kubernetes_Queue_DacpacSource_KeyVault_Secrets_Success(string runtim secretsProviderFile = Path.GetFullPath(secretsProviderFile); podIdentityFile = Path.GetFullPath(podIdentityFile); runtimeFile = Path.GetFullPath(runtimeFile); + string testRunTimeFile = Path.Combine(Path.GetDirectoryName(runtimeFile), "k8s-runtime-test.yaml"); + File.Copy(runtimeFile, testRunTimeFile, true); deployFile = Path.GetFullPath(deployFile); secretsFile = Path.GetFullPath(secretsFile); var overrideFile = Path.GetFullPath("TestConfig/databasetargets.cfg"); @@ -321,7 +324,7 @@ public void Kubernetes_Queue_DacpacSource_KeyVault_Secrets_Success(string runtim var args = new string[]{ "k8s", "prep", "--secretsfile", secretsFile, - "--runtimefile", runtimeFile, + "--runtimefile", testRunTimeFile, "--jobname", TestHelper.GetUniqueJobName("k8s-kv"), "--platinumdacpac", dacpacName, "--override", minusFirst @@ -336,7 +339,7 @@ public void Kubernetes_Queue_DacpacSource_KeyVault_Secrets_Success(string runtim args = new string[]{ "k8s", "enqueue", "--secretsfile", secretsFile, - "--runtimefile", runtimeFile, + "--runtimefile", testRunTimeFile, "--override", minusFirst }; val = rootCommand.InvokeAsync(args); @@ -350,7 +353,7 @@ public void Kubernetes_Queue_DacpacSource_KeyVault_Secrets_Success(string runtim result = prc.ExecuteProcess("kubectl", $"apply -f {podIdentityFile}"); Assert.AreEqual(0, result, "Failed to apply pod identity file"); - result = prc.ExecuteProcess("kubectl", $"apply -f {runtimeFile}"); + result = prc.ExecuteProcess("kubectl", $"apply -f {testRunTimeFile}"); Assert.AreEqual(0, result, "Failed to apply runtime file"); result = prc.ExecuteProcess("kubectl", $"apply -f {deployFile}"); @@ -363,7 +366,7 @@ public void Kubernetes_Queue_DacpacSource_KeyVault_Secrets_Success(string runtim args = new string[]{ "k8s", "monitor", "--secretsfile", secretsFile, - "--runtimefile", runtimeFile, + "--runtimefile", testRunTimeFile, "--override", minusFirst , "--unittest", "true"}; val = rootCommand.InvokeAsync(args); @@ -372,12 +375,12 @@ public void Kubernetes_Queue_DacpacSource_KeyVault_Secrets_Success(string runtim Assert.AreEqual(0, result); var logFileContents = TestHelper.ReleventLogFileContents(startingLine); - Assert.IsTrue(logFileContents.Contains("DACPAC created"), "A DACPAC should have been used for the build"); + Assert.IsTrue(logFileContents.Contains("DACPAC created") || logFileContents.Contains("Dacpac Databases In Sync"), "A DACPAC should have been used for the build"); } - [DataRow("TestConfig/runtime.yaml", "TestConfig/secrets.yaml", "TestConfig/secretProviderClass.yaml", "TestConfig/podIdentityAndBinding.yaml", "TestConfig/acr_basic_job_keyvault.yaml")] + [DataRow("TestConfig/k8s-runtime.yaml", "TestConfig/k8s-secrets.yaml", "TestConfig/k8s-secretProviderClass.yaml", "TestConfig/k8s-podIdentityAndBinding.yaml", "TestConfig/k8s-acr_basic_job_keyvault.yaml")] [DataTestMethod] public void Kubernetes_Queue_DacpacSource_ForceApplyCustom_KeyVault_Secrets_Success(string runtimeFile, string secretsFile, string secretsProviderFile, string podIdentityFile, string deployFile) { @@ -385,6 +388,8 @@ public void Kubernetes_Queue_DacpacSource_ForceApplyCustom_KeyVault_Secrets_Succ secretsProviderFile = Path.GetFullPath(secretsProviderFile); podIdentityFile = Path.GetFullPath(podIdentityFile); runtimeFile = Path.GetFullPath(runtimeFile); + string testRunTimeFile = Path.Combine(Path.GetDirectoryName(runtimeFile), "k8s-runtime-test.yaml"); + File.Copy(runtimeFile, testRunTimeFile, true); deployFile = Path.GetFullPath(deployFile); secretsFile = Path.GetFullPath(secretsFile); var overrideFile = Path.GetFullPath("TestConfig/databasetargets.cfg"); @@ -426,7 +431,7 @@ public void Kubernetes_Queue_DacpacSource_ForceApplyCustom_KeyVault_Secrets_Succ var args = new string[]{ "k8s", "prep", "--secretsfile", secretsFile, - "--runtimefile", runtimeFile, + "--runtimefile", testRunTimeFile, "--jobname", TestHelper.GetUniqueJobName("k8s-kv"), "--platinumdacpac", dacpacName, "--override", minusFirst @@ -439,12 +444,13 @@ public void Kubernetes_Queue_DacpacSource_ForceApplyCustom_KeyVault_Secrets_Succ //Create another table in the first that will be applied when the custom DACPAC is created DatabaseHelper.CreateRandomTable(cmdLine, firstOverride); + DatabaseHelper.CreateRandomTable(cmdLine, thirdOverride); //enqueue the topic messages args = new string[]{ "k8s", "enqueue", "--secretsfile", secretsFile, - "--runtimefile", runtimeFile, + "--runtimefile", testRunTimeFile, "--override", minusFirst }; val = rootCommand.InvokeAsync(args); @@ -458,7 +464,7 @@ public void Kubernetes_Queue_DacpacSource_ForceApplyCustom_KeyVault_Secrets_Succ result = prc.ExecuteProcess("kubectl", $"apply -f {podIdentityFile}"); Assert.AreEqual(0, result, "Failed to apply pod identity file"); - result = prc.ExecuteProcess("kubectl", $"apply -f {runtimeFile}"); + result = prc.ExecuteProcess("kubectl", $"apply -f {testRunTimeFile}"); Assert.AreEqual(0, result, "Failed to apply runtime file"); result = prc.ExecuteProcess("kubectl", $"apply -f {deployFile}"); @@ -471,7 +477,7 @@ public void Kubernetes_Queue_DacpacSource_ForceApplyCustom_KeyVault_Secrets_Succ args = new string[]{ "k8s", "monitor", "--secretsfile", secretsFile, - "--runtimefile", runtimeFile, + "--runtimefile", testRunTimeFile, "--override", minusFirst, "--unittest", "true", "--stream", "true" diff --git a/src/SqlBuildManager.Console.ExternalTest/LocalTests.cs b/src/SqlBuildManager.Console.ExternalTest/LocalTests.cs index 1f3397da..949c4b1b 100644 --- a/src/SqlBuildManager.Console.ExternalTest/LocalTests.cs +++ b/src/SqlBuildManager.Console.ExternalTest/LocalTests.cs @@ -28,9 +28,9 @@ public void ConfigureProcessInfo() { SqlBuildManager.Logging.ApplicationLogging.CreateLogger("SqlBuildManager.Console.log", @"C:\temp"); - this.settingsFilePath = Path.GetFullPath("TestConfig/settingsfile-windows.json"); + this.settingsFilePath = Path.GetFullPath("TestConfig/settingsfile-batch-windows.json"); this.settingsFileKeyPath = Path.GetFullPath("TestConfig/settingsfilekey.txt"); - this.linuxSettingsFilePath = Path.GetFullPath("TestConfig/settingsfile-linux.json"); + this.linuxSettingsFilePath = Path.GetFullPath("TestConfig/settingsfile-batch-linux.json"); this.overrideFilePath = Path.GetFullPath("TestConfig/databasetargets.cfg"); this.cmdLine = new CommandLineArgs(); diff --git a/src/SqlBuildManager.Console/Aad/AadHelper.cs b/src/SqlBuildManager.Console/Aad/AadHelper.cs index 8e40c5b4..9c1d13b8 100644 --- a/src/SqlBuildManager.Console/Aad/AadHelper.cs +++ b/src/SqlBuildManager.Console/Aad/AadHelper.cs @@ -43,12 +43,8 @@ internal static TokenCredential TokenCredential } else { - _tokenCred = new DefaultAzureCredential( - new DefaultAzureCredentialOptions() { - ManagedIdentityClientId = AadHelper.ManagedIdentityClientId, - ExcludeAzureCliCredential = false - }); - log.LogInformation($"Creating DefaultAzureCredential with ManagedIdentityClientId of: '{AadHelper.ManagedIdentityClientId}'"); + _tokenCred = new ChainedTokenCredential(new AzureCliCredential(), new ManagedIdentityCredential(ManagedIdentityClientId = AadHelper.ManagedIdentityClientId), new AzurePowerShellCredential()); + log.LogInformation($"Creating ChainedTokenCredential with ManagedIdentityClientId of: '{AadHelper.ManagedIdentityClientId}'"); } } return _tokenCred; diff --git a/src/SqlBuildManager.Console/Batch/BatchExecution.cs b/src/SqlBuildManager.Console/Batch/BatchExecution.cs index 7f37e1a2..08a0259d 100644 --- a/src/SqlBuildManager.Console/Batch/BatchExecution.cs +++ b/src/SqlBuildManager.Console/Batch/BatchExecution.cs @@ -1,12 +1,8 @@ -using Azure.Storage; -using Azure.Storage.Blobs; -using Microsoft.Azure.Batch; +using Microsoft.Azure.Batch; using Microsoft.Azure.Batch.Auth; using Microsoft.Azure.Batch.Common; using Microsoft.Azure.Management.Batch; -using Microsoft.Azure.Management.ContainerInstance.Fluent.Models; using Microsoft.Extensions.Logging; -using Microsoft.Rest.Azure; using SqlBuildManager.Console.Aad; using SqlBuildManager.Console.CloudStorage; using SqlBuildManager.Console.CommandLine; @@ -153,9 +149,9 @@ public Execution(CommandLineArgs cmdLine, string queryFile, string outputFile) : timer.Start(); //Get storage ready - BlobServiceClient storageSvcClient = StorageManager.CreateStorageClient(cmdLine.ConnectionArgs.StorageAccountName, cmdLine.ConnectionArgs.StorageAccountKey); - StorageSharedKeyCredential storageCreds = new StorageSharedKeyCredential(cmdLine.ConnectionArgs.StorageAccountName, cmdLine.ConnectionArgs.StorageAccountKey); - string containerSasToken = StorageManager.GetOutputContainerSasUrl(cmdLine.ConnectionArgs.StorageAccountName, storageContainerName, storageCreds, false); + var storageSvcClient = StorageManager.CreateStorageClient(cmdLine.ConnectionArgs.StorageAccountName, cmdLine.ConnectionArgs.StorageAccountKey); + var storageCreds = StorageManager.GetStorageSharedKeyCredential(cmdLine.ConnectionArgs.StorageAccountName, cmdLine.ConnectionArgs.StorageAccountKey); + string containerSasToken = StorageManager.GetOutputContainerSasUrl(cmdLine.ConnectionArgs.StorageAccountName, cmdLine.ConnectionArgs.StorageAccountKey, storageContainerName, false); log.LogDebug($"Output write SAS token: {containerSasToken}"); @@ -406,12 +402,10 @@ public Execution(CommandLineArgs cmdLine, string queryFile, string outputFile) : log.LogInformation($"Setting job {jobId} status to exit code: {myExitCode}"); CloudJob j = batchClient.JobOperations.GetJob(jobId); j.Terminate("Error"); - } - + } - readOnlySasToken = StorageManager.GetOutputContainerSasUrl(cmdLine.ConnectionArgs.StorageAccountName, storageContainerName, storageCreds, true); - log.LogInformation($"Log files can be found here: {readOnlySasToken}"); - log.LogInformation("The read-only SAS token URL is valid for 7 days."); + readOnlySasToken = StorageManager.GetOutputContainerSasUrl(cmdLine.ConnectionArgs.StorageAccountName, cmdLine.ConnectionArgs.StorageAccountKey, storageContainerName, true); + log.LogInformation($"The consolidated log files can be found in the Azure storage account '{cmdLine.ConnectionArgs.StorageAccountName}' in blob container '{storageContainerName}'"); log.LogInformation("You can download \"Azure Storage Explorer\" from here: https://azure.microsoft.com/en-us/features/storage-explorer/"); log.LogInformation("You can also get details on your Azure Batch execution from the \"Azure Batch Explorer\" found here: https://azure.github.io/BatchExplorer/"); @@ -621,7 +615,7 @@ private async Task CreateBatchPool(CommandLineArgs cmdLine, string poolId) case OsType.Windows: default: - imageReference = new bm.ImageReference(publisher: "MicrosoftWindowsServer", offer: "WindowsServer", sku: "2016-Datacenter-with-containers", version: "latest"); + imageReference = new bm.ImageReference(publisher: "MicrosoftWindowsServer", offer: "WindowsServer", sku: "2022-datacenter", version: "latest"); virtualMachineConfiguration = new bm.VirtualMachineConfiguration(imageReference: imageReference, nodeAgentSkuId: "batch.node.windows amd64"); break; diff --git a/src/SqlBuildManager.Console/CloudStorage/StorageManager.cs b/src/SqlBuildManager.Console/CloudStorage/StorageManager.cs index 7a6d663e..e32f3871 100644 --- a/src/SqlBuildManager.Console/CloudStorage/StorageManager.cs +++ b/src/SqlBuildManager.Console/CloudStorage/StorageManager.cs @@ -13,6 +13,8 @@ using SqlBuildManager.Console.CommandLine; using System.Text.RegularExpressions; using System.Linq; +using SqlBuildManager.Console.ContainerApp.Internal; +using static System.Net.WebRequestMethods; namespace SqlBuildManager.Console.CloudStorage { @@ -21,12 +23,26 @@ public class StorageManager private static ILogger log = SqlBuildManager.Logging.ApplicationLogging.CreateLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType); internal static BlobServiceClient CreateStorageClient(string storageAccountName, string storageAccountKey) { - StorageSharedKeyCredential creds = new StorageSharedKeyCredential(storageAccountName, storageAccountKey); - var serviceClient = new BlobServiceClient(new Uri($"https://{storageAccountName}.blob.core.windows.net"), creds); + BlobServiceClient serviceClient = null; + if (string.IsNullOrWhiteSpace(storageAccountKey)) + { + serviceClient = new BlobServiceClient(new Uri($"https://{storageAccountName}.blob.core.windows.net"),Aad.AadHelper.TokenCredential); + } + else + { + StorageSharedKeyCredential creds = new StorageSharedKeyCredential(storageAccountName, storageAccountKey); + serviceClient = new BlobServiceClient(new Uri($"https://{storageAccountName}.blob.core.windows.net"), creds); + } + + return serviceClient; } - internal static string GetStorageConnectionString(string storageAccountName, string storageAccountKey) + internal static StorageSharedKeyCredential GetStorageSharedKeyCredential(string storageAccountName, string storageAccountKey) + { + return new StorageSharedKeyCredential(storageAccountName, storageAccountKey); + } + private static string GetStorageConnectionString(string storageAccountName, string storageAccountKey) { return $"DefaultEndpointsProtocol=https;AccountName={storageAccountName};AccountKey={storageAccountKey};EndpointSuffix=core.windows.net"; } @@ -125,7 +141,7 @@ internal static async Task WriteFileToLocalStorage(BlobServiceClient sto log.LogInformation($"Downloading {blob.Uri} to local working file {localPath}"); BlobDownloadInfo blobDownload = await blob.DownloadAsync(); - using (FileStream fileStream = File.OpenWrite(localPath)) + using (FileStream fileStream = System.IO.File.OpenWrite(localPath)) { await blobDownload.Content.CopyToAsync(fileStream); } @@ -137,12 +153,17 @@ internal static async Task WriteFileToLocalStorage(BlobServiceClient sto return string.Empty; } } + + internal static string GetContainerRawUrl(string storageAccountName, string outputContainerName) + { + return $"https://{storageAccountName}.blob.core.windows.net/{outputContainerName}"; + } internal static string GetOutputContainerSasUrl(string storageAccountName, string storageAccountKey, string outputContainerName, bool forRead) { var cred = new StorageSharedKeyCredential(storageAccountName, storageAccountKey); return GetOutputContainerSasUrl(storageAccountName, outputContainerName,cred,forRead); } - internal static string GetOutputContainerSasUrl(string storageAccountName, string outputContainerName, StorageSharedKeyCredential storageCreds, bool forRead) + private static string GetOutputContainerSasUrl(string storageAccountName, string outputContainerName, StorageSharedKeyCredential storageCreds, bool forRead) { log.LogDebug($"Ensuring presence of output blob container '{outputContainerName}'"); var container = new BlobContainerClient(new Uri($"https://{storageAccountName}.blob.core.windows.net/{outputContainerName}"), storageCreds); @@ -191,8 +212,7 @@ internal static (string jobId, string storageContainerName) GetJobAndStorageName } internal static async Task StorageContainerExists(string storageAccountName, string storageAccountKey, string containerName) { - var connstr = GetStorageConnectionString(storageAccountName, storageAccountKey); - BlobContainerClient container = new BlobContainerClient(connstr, containerName); + var container = GetBlobContainerClient(storageAccountName, storageAccountKey, containerName); var r = await container.ExistsAsync(); return r.Value; @@ -201,8 +221,7 @@ internal static async Task DeleteStorageContainer(string storageAccountNam { try { - var connstr = GetStorageConnectionString(storageAccountName, storageAccountKey); - BlobContainerClient container = new BlobContainerClient(connstr, containerName); + var container = GetBlobContainerClient(storageAccountName, storageAccountKey, containerName); var result = await container.DeleteAsync(); if(result.Status == 202) { @@ -232,12 +251,10 @@ internal static async Task UploadFilesToContainer(string storageAccountNam } string blobName = Path.GetFileName(filePath); - var connstr = GetStorageConnectionString(storageAccountName, storageAccountKey); - BlobContainerClient container = new BlobContainerClient(connstr, containerName); + var container = GetBlobContainerClient(storageAccountName, storageAccountKey, containerName); await container.CreateIfNotExistsAsync(); - var storageCreds = new StorageSharedKeyCredential(storageAccountName, storageAccountKey); - BlockBlobClient blobData = new BlockBlobClient(new Uri($"https://{storageAccountName}.blob.core.windows.net/{containerName}/{blobName}"), storageCreds); //container.GetBlockBlobClient(blobName); + var blobData = GetBlockBlobClient(storageAccountName, storageAccountKey, containerName, blobName); using (var fs = new FileStream(filePath, FileMode.Open)) { blobData.Upload(fs); @@ -265,6 +282,31 @@ internal static async Task UploadFilesToContainer(string storageAccountNam return false; } } + internal static async Task DownloadBlobToLocal(string sasUrl, string localFileName) + { + try + { + var cloudBlobContainer = new BlobContainerClient(new Uri(sasUrl)); + var blob = cloudBlobContainer.GetBlobClient(Path.GetFileName(localFileName)); + var resp = await blob.DownloadToAsync(localFileName); + if (resp.Status < 300) + { + return true; + } + else + { + log.LogError($"Unable to download file {Path.GetFileName(localFileName)} to {localFileName}: {resp.ReasonPhrase}"); + return false; + } + } + catch (Exception ex) + { + log.LogError($"Unable to download file {Path.GetFileName(localFileName)} to {localFileName}: {ex.Message}"); + return false; + } + } + + internal static ResourceFile UploadFileToBatchContainer(string storageAcctName, string containerName, StorageSharedKeyCredential storageCreds, string filePath) { log.LogInformation($"Uploading file {filePath} to container [{containerName}]..."); @@ -333,9 +375,9 @@ internal static bool CombineQueryOutputfiles(BlobServiceClient storageSvcClient, } - if (File.Exists(tmp)) + if (System.IO.File.Exists(tmp)) { - File.Delete(tmp); + System.IO.File.Delete(tmp); } } counter++; @@ -347,16 +389,33 @@ internal static bool CombineQueryOutputfiles(BlobServiceClient storageSvcClient, } internal static async Task WriteLogsToBlobContainer(string storageAccountName, string storageAccountKey, string outputContainerName, string rootLoggingPath) + { + var container = GetBlobContainerClient(storageAccountName, storageAccountKey, outputContainerName); + var res = await WriteLogsToBlobContainer(container, rootLoggingPath); + return res; + } + internal static async Task WriteLogsToBlobContainer(string outputContainerSasUrl, string rootLoggingPath) + { + BlobContainerClient container = new BlobContainerClient(new Uri(outputContainerSasUrl)); + var res = await WriteLogsToBlobContainer(container, rootLoggingPath); + return res; + + } + private static async Task WriteLogsToBlobContainer(BlobContainerClient containerClient, string rootLoggingPath) { try { //var writeTasks = new List(); - log.LogInformation($"Writing log files to blob storage at {storageAccountName}"); + log.LogInformation($"Writing log files to blob storage at {containerClient.AccountName}"); var renameLogFiles = new string[] { "sqlbuildmanager", "csv" }; - var connstr = GetStorageConnectionString(storageAccountName, storageAccountKey); - BlobContainerClient container = new BlobContainerClient(connstr, outputContainerName); + + log.LogInformation($"Getting file list from {rootLoggingPath}"); var fileList = Directory.GetFiles(rootLoggingPath, "*.*", SearchOption.AllDirectories); - var taskId = Environment.GetEnvironmentVariable("HOSTNAME"); + var taskId = Environment.GetEnvironmentVariable("AZ_BATCH_TASK_ID"); + if(string.IsNullOrEmpty(taskId)) + { + taskId = Environment.GetEnvironmentVariable("HOSTNAME"); + } string machine = Environment.MachineName; foreach (var f in fileList) @@ -370,7 +429,7 @@ internal static async Task WriteLogsToBlobContainer(string storageAccountN tmp = $"{taskId}/{tmp}"; log.LogInformation($"Saving File '{f}' as '{tmp}'"); - var rename = container.GetBlockBlobClient(tmp); + var rename = containerClient.GetBlockBlobClient(tmp); using (var fs = new FileStream(f, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) { await rename.UploadAsync(fs); @@ -379,12 +438,9 @@ internal static async Task WriteLogsToBlobContainer(string storageAccountN } else if (renameLogFiles.Any(a => tmp.ToLower().IndexOf(a) > -1)) { - - //var localTemp = Path.Combine(Path.GetDirectoryName(f), tmp); - //File.Copy(f, localTemp); tmp = $"{taskId}/{tmp}"; log.LogInformation($"Saving File '{f}' as '{tmp}'"); - var rename = container.GetBlockBlobClient(tmp); + var rename = containerClient.GetBlockBlobClient(tmp); using (var fs = new FileStream(f, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) { await rename.UploadAsync(fs); @@ -394,7 +450,7 @@ internal static async Task WriteLogsToBlobContainer(string storageAccountN else { log.LogInformation($"Saving File '{f}' as '{tmp}'"); - var b = container.GetBlockBlobClient(tmp); + var b = containerClient.GetBlockBlobClient(tmp); using (var fs = new FileStream(f, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) { await b.UploadAsync(fs); @@ -415,6 +471,36 @@ internal static async Task WriteLogsToBlobContainer(string storageAccountN return false; } } + internal static BlobContainerClient GetBlobContainerClient(string storageAccountName, string storageAccountKey, string containerName) + { + BlobContainerClient containerClient = null; + if (string.IsNullOrWhiteSpace(storageAccountKey)) +{ + var url = $"https://{storageAccountName}.blob.core.windows.net/{containerName}"; + containerClient = new BlobContainerClient(new Uri(url), Aad.AadHelper.TokenCredential); + } + else + { + var connstr = GetStorageConnectionString(storageAccountName, storageAccountKey); + containerClient = new BlobContainerClient(connstr, containerName); + } + return containerClient; + } + private static BlockBlobClient GetBlockBlobClient(string storageAccountName, string storageAccountKey, string containerName, string blobName) + { + BlockBlobClient containerClient = null; + if (string.IsNullOrWhiteSpace(storageAccountKey)) + { + var url = $"https://{storageAccountName}.blob.core.windows.net/{containerName}/{blobName}"; + containerClient = new BlockBlobClient(new Uri(url), Aad.AadHelper.TokenCredential); + } + else + { + var connstr = GetStorageConnectionString(storageAccountName, storageAccountKey); + containerClient = new BlockBlobClient(connstr, containerName, blobName); + } + return containerClient; + } } diff --git a/src/SqlBuildManager.Console/ContainerApp/ContainerAppHelper.cs b/src/SqlBuildManager.Console/ContainerApp/ContainerAppHelper.cs index 0f3a3c6b..2340e666 100644 --- a/src/SqlBuildManager.Console/ContainerApp/ContainerAppHelper.cs +++ b/src/SqlBuildManager.Console/ContainerApp/ContainerAppHelper.cs @@ -53,7 +53,7 @@ internal static async Task DeployContainerApp(CommandLineArgs cmdLine) } else { - log.LogError("Container App deployment failed. Unablet to proceed."); + log.LogError("Container App deployment failed. Unable to proceed."); } return success; } diff --git a/src/SqlBuildManager.Console/ContainerApp/containerapp_arm_template.json b/src/SqlBuildManager.Console/ContainerApp/containerapp_arm_template.json index 64c5201d..cbf8746d 100644 --- a/src/SqlBuildManager.Console/ContainerApp/containerapp_arm_template.json +++ b/src/SqlBuildManager.Console/ContainerApp/containerapp_arm_template.json @@ -227,7 +227,7 @@ }, { "name": "Sbm_IdentityName", - "value": "[parameters('identityClientId')]" + "value": "[parameters('identityName')]" }, { "name": "Sbm_AllowObjectDelete", diff --git a/src/SqlBuildManager.Console/ContainerApp/containerapp_env_arm_template.json b/src/SqlBuildManager.Console/ContainerApp/containerapp_env_arm_template.json index 862c674c..bf00da98 100644 --- a/src/SqlBuildManager.Console/ContainerApp/containerapp_env_arm_template.json +++ b/src/SqlBuildManager.Console/ContainerApp/containerapp_env_arm_template.json @@ -107,7 +107,7 @@ "variables": { "environment_id": "[concat(subscription().id, '/resourceGroups/', resourceGroup().name, '/providers/Microsoft.App/managedEnvironments/', parameters('environmentName'))]", "var_registryName": "[if(empty(parameters('registryServer')),'blueskydevus/',concat(parameters('registryServer'),'/'))]", - "var_registry_secret_name": "[if(empty(parameters('registrypassword')),'registryPassword',concat(replace(parameters('registryServer'),'.',''),'-',parameters('registryUserName')))]" + "var_registry_secret_name": "[if(empty(parameters('registryServer')),'registryPassword',concat(replace(parameters('registryServer'),'.',''),'-',parameters('registryUserName')))]" }, "resources": [ { @@ -226,7 +226,7 @@ }, { "name": "Sbm_IdentityName", - "value": "[parameters('identityClientId')]" + "value": "[parameters('identityName')]" }, { "name": "Sbm_AllowObjectDelete", diff --git a/src/SqlBuildManager.Console/ContainerApp/containerapp_identity_arm_template.json b/src/SqlBuildManager.Console/ContainerApp/containerapp_identity_arm_template.json index 6129b7db..2dcb5732 100644 --- a/src/SqlBuildManager.Console/ContainerApp/containerapp_identity_arm_template.json +++ b/src/SqlBuildManager.Console/ContainerApp/containerapp_identity_arm_template.json @@ -106,7 +106,7 @@ "variables": { "environment_id": "[concat(subscription().id, '/resourceGroups/', resourceGroup().name, '/providers/Microsoft.App/managedEnvironments/', parameters('environmentName'))]", "var_registryName": "[if(empty(parameters('registryServer')),'blueskydevus/',concat(parameters('registryServer'),'/'))]", - "var_registry_secret_name": "[if(empty(parameters('registryPassword')),'registrypassword',concat(replace(parameters('registryServer'),'.',''),'-',parameters('registryUserName')))]" + "var_registry_secret_name": "[if(empty(parameters('registryServer')),'registrypassword',concat(replace(parameters('registryServer'),'.',''),'-',parameters('registryUserName')))]" }, "resources": [ { @@ -231,7 +231,7 @@ }, { "name": "Sbm_IdentityName", - "value": "[parameters('identityClientId')]" + "value": "[parameters('identityName')]" }, { "name": "Sbm_AllowObjectDelete", diff --git a/src/SqlBuildManager.Console/Events/EventManager.cs b/src/SqlBuildManager.Console/Events/EventManager.cs index c9ad3ec3..4b3fa46e 100644 --- a/src/SqlBuildManager.Console/Events/EventManager.cs +++ b/src/SqlBuildManager.Console/Events/EventManager.cs @@ -2,7 +2,9 @@ using Azure.Messaging.EventHubs.Consumer; using Azure.Messaging.EventHubs.Processor; using Azure.Storage.Blobs; +using Microsoft.Azure.Management.Storage.Fluent; using Microsoft.Extensions.Logging; +using SqlBuildManager.Console.CloudStorage; using SqlBuildManager.Console.Shared; using SqlBuildManager.Interfaces.Console; using System; @@ -12,6 +14,7 @@ using System.Text.Json; using System.Threading; using System.Threading.Tasks; +using cs = SqlBuildManager.Console.CloudStorage; namespace SqlBuildManager.Console.Events { @@ -20,10 +23,12 @@ public class EventManager : IDisposable private static ILogger log = SqlBuildManager.Logging.ApplicationLogging.CreateLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType); private string eventHubconnectionString = ""; private string storageContainerName = ""; - private string storageConnectionString = ""; + private string storageAccountName = ""; + private string storageAccountKey= ""; private string jobName = ""; private string consumerGroup = EventHubConsumerClient.DefaultConsumerGroupName; private DateTime utcMonitorStart = DateTime.UtcNow; + private int databaseCommitMessages = 0; private int databaseErrorMessages = 0; @@ -44,12 +49,13 @@ private void IncrementEventsScanned() } public bool StreamEvents { get;set; } = false; - public EventManager(string eventHubconnectionString, string storageConnectionString, string storageContainerName, string jobName) + public EventManager(string eventHubconnectionString, string storageAccountName, string storageAccountKey, string storageContainerName, string jobName) { this.eventHubconnectionString = eventHubconnectionString; this.jobName = jobName; - this.storageConnectionString = storageConnectionString; + this.storageAccountName = storageAccountName; this.storageContainerName = storageContainerName; + this.storageAccountKey = storageAccountKey; } private BlobContainerClient _blobClient = null; @@ -57,9 +63,9 @@ private BlobContainerClient BlobClient { get { - if(_blobClient == null) + if (_blobClient == null) { - _blobClient = new BlobContainerClient(storageConnectionString, "eventhubcheckpoint"); + _blobClient = cs.StorageManager.GetBlobContainerClient(this.storageAccountName, this.storageAccountKey, "eventhubcheckpoint"); _blobClient.CreateIfNotExistsAsync().Wait(); } return _blobClient; diff --git a/src/SqlBuildManager.Console/Kubernetes/ContainerManager.cs b/src/SqlBuildManager.Console/Kubernetes/ContainerManager.cs index d1244c2f..53a8dc1b 100644 --- a/src/SqlBuildManager.Console/Kubernetes/ContainerManager.cs +++ b/src/SqlBuildManager.Console/Kubernetes/ContainerManager.cs @@ -33,9 +33,6 @@ internal static (bool, CommandLineArgs) ReadSecrets(CommandLineArgs args) log.LogDebug($"serviceBus= {args.ConnectionArgs.ServiceBusTopicConnectionString}"); } - args.StorageAccountName = File.ReadAllText("/etc/sbm/StorageAccountName"); - log.LogDebug($"storageaccountname= {args.ConnectionArgs.StorageAccountName}"); - args.StorageAccountKey = File.ReadAllText("/etc/sbm/StorageAccountKey"); log.LogDebug($"storageaccountkey= {args.ConnectionArgs.StorageAccountKey}"); @@ -119,7 +116,6 @@ internal static (bool, CommandLineArgs) ReadRuntimeParameters(CommandLineArgs ar log.LogDebug($"authType= {args.AllowObjectDelete}"); } - if (File.Exists("/etc/runtime/EventHubConnectionString")) { args.EventHubConnection = File.ReadAllText("/etc/runtime/EventHubConnectionString"); @@ -132,32 +128,32 @@ internal static (bool, CommandLineArgs) ReadRuntimeParameters(CommandLineArgs ar log.LogDebug($"ServiceBusTopicConnectionString= {args.ConnectionArgs.ServiceBusTopicConnectionString}"); } + if (File.Exists("/etc/runtime/StorageAccountName")) + { + args.ConnectionArgs.StorageAccountName = File.ReadAllText("/etc/runtime/StorageAccountName"); + log.LogDebug($"StorageAccountName= {args.ConnectionArgs.StorageAccountName}"); + } + return (true, args); } internal static string GenerateSecretsYaml(CommandLineArgs args) { - + bool secretAdded = false; var yml = new SecretYaml(); - if (!string.IsNullOrWhiteSpace(args.ConnectionArgs.StorageAccountName)) - { - yml.data.StorageAccountName = args.ConnectionArgs.StorageAccountName.EncodeBase64(); - } - - if (!string.IsNullOrWhiteSpace(args.ConnectionArgs.StorageAccountKey)) - { - yml.data.StorageAccountKey = args.ConnectionArgs.StorageAccountKey.EncodeBase64(); - } + if (!string.IsNullOrWhiteSpace(args.ConnectionArgs.EventHubConnectionString) && ConnectionValidator.IsServiceBusConnectionString(args.ConnectionArgs.EventHubConnectionString)) { yml.data.EventHubConnectionString = args.ConnectionArgs.EventHubConnectionString.EncodeBase64(); + secretAdded = true; } if (!string.IsNullOrWhiteSpace(args.ConnectionArgs.ServiceBusTopicConnectionString) && ConnectionValidator.IsServiceBusConnectionString(args.ConnectionArgs.ServiceBusTopicConnectionString)) { yml.data.ServiceBusTopicConnectionString = args.ConnectionArgs.ServiceBusTopicConnectionString.EncodeBase64(); + secretAdded = true; } if (args.AuthenticationArgs.AuthenticationType != AuthenticationType.ManagedIdentity) @@ -165,18 +161,35 @@ internal static string GenerateSecretsYaml(CommandLineArgs args) if (!string.IsNullOrWhiteSpace(args.AuthenticationArgs.UserName)) { yml.data.UserName = args.AuthenticationArgs.UserName.EncodeBase64(); + secretAdded = true; } if (!string.IsNullOrWhiteSpace(args.AuthenticationArgs.Password)) { yml.data.Password = args.AuthenticationArgs.Password.EncodeBase64(); + secretAdded = true; + } + + if (!string.IsNullOrWhiteSpace(args.ConnectionArgs.StorageAccountKey)) + { + yml.data.StorageAccountKey = args.ConnectionArgs.StorageAccountKey.EncodeBase64(); + secretAdded = true; } } - var serializer = new SerializerBuilder().ConfigureDefaultValuesHandling(DefaultValuesHandling.OmitNull).Build(); - var yamlString = serializer.Serialize(yml); + if (secretAdded) + { + var serializer = new SerializerBuilder().ConfigureDefaultValuesHandling(DefaultValuesHandling.OmitNull).Build(); + var yamlString = serializer.Serialize(yml); + return yamlString; + } + else + { + log.LogInformation("No secrets were provided or needed. No secrets yaml is being generated"); + return string.Empty; + } - return yamlString; + } @@ -201,6 +214,12 @@ internal static string GenerateRuntimeYaml(CommandLineArgs args) { yml.data.EventHubConnectionString = args.ConnectionArgs.EventHubConnectionString; } + + if (!string.IsNullOrWhiteSpace(args.ConnectionArgs.StorageAccountName)) + { + yml.data.StorageAccountName = args.ConnectionArgs.StorageAccountName; + } + var serializer = new SerializerBuilder().ConfigureDefaultValuesHandling(DefaultValuesHandling.OmitNull).Build(); var yamlString = serializer.Serialize(yml); @@ -243,15 +262,28 @@ internal static CommandLineArgs GetArgumentsFromSecretsFile(string filename, Com { args = new CommandLineArgs(); } + if (args.AuthenticationArgs.AuthenticationType != AuthenticationType.ManagedIdentity) { - args.UserName = GetValueFromSecrets(filename, "UserName"); - args.Password = GetValueFromSecrets(filename, "Password"); - args.EventHubConnection = GetValueFromSecrets(filename, "EventHubConnectionString"); - args.ServiceBusTopicConnection = GetValueFromSecrets(filename, "ServiceBusTopicConnectionString"); + var tmp = GetValueFromSecrets(filename, "UserName"); + if (!string.IsNullOrWhiteSpace(tmp)) args.UserName = tmp; + + tmp = GetValueFromSecrets(filename, "Password"); + if (!string.IsNullOrWhiteSpace(tmp)) args.Password = tmp; + + tmp = GetValueFromSecrets(filename, "EventHubConnectionString"); + if (!string.IsNullOrWhiteSpace(tmp)) args.EventHubConnection = tmp; + + tmp = GetValueFromSecrets(filename, "ServiceBusTopicConnectionString"); + if (!string.IsNullOrWhiteSpace(tmp)) args.ServiceBusTopicConnection = tmp; + + tmp = GetValueFromSecrets(filename, "StorageAccountKey"); + if (!string.IsNullOrWhiteSpace(tmp)) args.StorageAccountKey = tmp; + + tmp = GetValueFromSecrets(filename, "StorageAccountName"); + if (!string.IsNullOrWhiteSpace(tmp)) args.StorageAccountName = tmp; } - args.StorageAccountKey = GetValueFromSecrets(filename, "StorageAccountKey"); - args.StorageAccountName = GetValueFromSecrets(filename, "StorageAccountName"); + return args; @@ -262,9 +294,17 @@ internal static CommandLineArgs GetArgumentsFromRuntimeFile(string filename, Com { args = new CommandLineArgs(); } - args.BuildFileName = GetValueFromRuntimestring(filename, "PackageName"); - args.JobName = GetValueFromRuntimestring(filename, "JobName"); - args.PlatinumDacpac = GetValueFromRuntimestring(filename, "DacpacName"); + + var tmp = GetValueFromRuntimestring(filename, "PackageName"); + if (!string.IsNullOrWhiteSpace(tmp)) args.BuildFileName = tmp; + + + tmp = GetValueFromRuntimestring(filename, "JobName"); + if (!string.IsNullOrWhiteSpace(tmp)) args.JobName = tmp; + + + tmp = GetValueFromRuntimestring(filename, "DacpacName"); + if (!string.IsNullOrWhiteSpace(tmp)) args.PlatinumDacpac = tmp; if (Enum.TryParse(GetValueFromRuntimestring(filename, "ConcurrencyType"), out ConcurrencyType con)) { @@ -283,10 +323,17 @@ internal static CommandLineArgs GetArgumentsFromRuntimeFile(string filename, Com if (args.AuthenticationArgs.AuthenticationType == AuthenticationType.ManagedIdentity) { - args.EventHubConnection = GetValueFromRuntimestring(filename, "EventHubConnectionString"); - args.ServiceBusTopicConnection = GetValueFromRuntimestring(filename, "ServiceBusTopicConnectionString"); + + tmp = GetValueFromRuntimestring(filename, "EventHubConnectionString"); + if (!string.IsNullOrWhiteSpace(tmp)) args.EventHubConnection = tmp; + + tmp = GetValueFromRuntimestring(filename, "ServiceBusTopicConnectionString"); + if (!string.IsNullOrWhiteSpace(tmp)) args.ServiceBusTopicConnection = tmp; } + tmp = GetValueFromRuntimestring(filename, "StorageAccountName"); + if (!string.IsNullOrWhiteSpace(tmp)) args.ConnectionArgs.StorageAccountName = tmp; + return args; } diff --git a/src/SqlBuildManager.Console/Kubernetes/RuntimeYaml.cs b/src/SqlBuildManager.Console/Kubernetes/RuntimeYaml.cs index 46ba71c5..39c34c7c 100644 --- a/src/SqlBuildManager.Console/Kubernetes/RuntimeYaml.cs +++ b/src/SqlBuildManager.Console/Kubernetes/RuntimeYaml.cs @@ -46,5 +46,7 @@ public class RuntimeData [YamlMember(ScalarStyle = ScalarStyle.SingleQuoted)] public string EventHubConnectionString { get; set; } = null; + public string StorageAccountName { get; set; } = null; + } } diff --git a/src/SqlBuildManager.Console/Kubernetes/SecretYaml.cs b/src/SqlBuildManager.Console/Kubernetes/SecretYaml.cs index 4a374cc9..e7ba68a1 100644 --- a/src/SqlBuildManager.Console/Kubernetes/SecretYaml.cs +++ b/src/SqlBuildManager.Console/Kubernetes/SecretYaml.cs @@ -25,7 +25,6 @@ public class SecretsData public string ServiceBusTopicConnectionString { get; set; } = null; public string UserName { get; set; } = null; public string Password { get; set; } = null; - public string StorageAccountName { get; set; } = null; public string StorageAccountKey { get; set; } = null; } } diff --git a/src/SqlBuildManager.Console/Threaded/ThreadedRunner.cs b/src/SqlBuildManager.Console/Threaded/ThreadedRunner.cs index b48b4101..1ed8d90c 100644 --- a/src/SqlBuildManager.Console/Threaded/ThreadedRunner.cs +++ b/src/SqlBuildManager.Console/Threaded/ThreadedRunner.cs @@ -257,30 +257,6 @@ await Task.Run(() => } - //private void Bg_ProgressChanged(object sender, ProgressChangedEventArgs e) - //{ - // if(e.UserState is GeneralStatusEventArgs) - // { - // log.LogInformation(((GeneralStatusEventArgs)e.UserState).StatusMessage); - // } - // else if(e.UserState is BuildScriptEventArgs) - // { - // } - // else if(e.UserState is ScriptRunStatusEventArgs) - // { - // } - // else if (e.UserState is ScriptRunProjectFileSavedEventArgs) - // { - // } - // else if (e.UserState is ScriptRunProjectFileSavedEventArgs) - // { - // } - // else - // { - // log.LogInformation(e.UserState.ToString()); - // } - //} - void helper_BuildSuccessTrialRolledBackEvent(object sender, EventArgs e) { log.LogDebug(this.TargetTag + " BuildSuccessTrialRolledBackEvent status: " + Enum.GetName(typeof(RunnerReturn), RunnerReturn.SuccessWithTrialRolledBack)); diff --git a/src/SqlBuildManager.Console/Validation.cs b/src/SqlBuildManager.Console/Validation.cs index 43abb53c..0a7168ed 100644 --- a/src/SqlBuildManager.Console/Validation.cs +++ b/src/SqlBuildManager.Console/Validation.cs @@ -336,39 +336,39 @@ public static int ValidateBatchArguments(CommandLineArgs cmdLine, out string[] e { int returnVal = 0; List messages = new List(); - if (String.IsNullOrEmpty(cmdLine.ConnectionArgs.BatchAccountName)) + if (String.IsNullOrWhiteSpace(cmdLine.ConnectionArgs.BatchAccountName)) { messages.Add("--batchaccountname is required in command line or --settingsfile Json"); returnVal = -888; } - if (String.IsNullOrEmpty(cmdLine.ConnectionArgs.BatchAccountKey)) + if (String.IsNullOrWhiteSpace(cmdLine.ConnectionArgs.BatchAccountKey)) { messages.Add("--batchaccountkey is required in command line or --settingsfile Json"); returnVal = -888; } - if (String.IsNullOrEmpty(cmdLine.ConnectionArgs.BatchAccountUrl)) + if (String.IsNullOrWhiteSpace(cmdLine.ConnectionArgs.BatchAccountUrl)) { messages.Add("--batchaccounturl is required in command line or --settingsfile Json"); returnVal = -888; } - if (String.IsNullOrEmpty(cmdLine.ConnectionArgs.StorageAccountName)) + if (String.IsNullOrWhiteSpace(cmdLine.ConnectionArgs.StorageAccountName)) { messages.Add("--storageaccountname is required in command line or --settingsfile Json"); returnVal = -888; } - if (String.IsNullOrEmpty(cmdLine.ConnectionArgs.StorageAccountKey)) + if (String.IsNullOrWhiteSpace(cmdLine.ConnectionArgs.StorageAccountKey) && string.IsNullOrWhiteSpace(cmdLine.IdentityArgs.ClientId)) { - messages.Add("--storageaccountkey is required in command line or --settingsfile Json"); + messages.Add("--storageaccountkey is required in command line or --settingsfile json if a Managed Identity is not included"); returnVal = -888; } - if (String.IsNullOrEmpty(cmdLine.BatchArgs.BatchVmSize)) + if (String.IsNullOrWhiteSpace(cmdLine.BatchArgs.BatchVmSize)) { messages.Add("--batchvmsize, is required in command line or --settingsfile Json"); returnVal = -888; } - if (!String.IsNullOrEmpty(cmdLine.ConnectionArgs.ServiceBusTopicConnectionString) && string.IsNullOrEmpty(cmdLine.BatchArgs.BatchJobName)) + if (!String.IsNullOrWhiteSpace(cmdLine.ConnectionArgs.ServiceBusTopicConnectionString) && string.IsNullOrEmpty(cmdLine.BatchArgs.BatchJobName)) { messages.Add("When --servicebusconnection is provided in command line or --settingsfile Json, then --batchjobname is required"); returnVal = -888; @@ -555,11 +555,11 @@ public static List ValidateStorageAccountArgs(CommandLineArgs cmdLine) if (string.IsNullOrWhiteSpace(cmdLine.ConnectionArgs.StorageAccountName)) { messages.Add("--storageaccountname is required in command line or --settingsfile"); - } + } - if(string.IsNullOrWhiteSpace(cmdLine.ConnectionArgs.StorageAccountKey)) + if (String.IsNullOrWhiteSpace(cmdLine.ConnectionArgs.StorageAccountKey) && string.IsNullOrWhiteSpace(cmdLine.IdentityArgs.ClientId)) { - messages.Add("--storageaccountkey is required in command line or --settingsfile"); + messages.Add("--storageaccountkey is required in command line or --settingsfile if a Managed Identity is not included"); } return messages; diff --git a/src/SqlBuildManager.Console/Worker.cs b/src/SqlBuildManager.Console/Worker.cs index 108f3f85..081cebfe 100644 --- a/src/SqlBuildManager.Console/Worker.cs +++ b/src/SqlBuildManager.Console/Worker.cs @@ -1,16 +1,16 @@ -using Azure.Storage.Blobs; -using Azure.Storage.Blobs.Specialized; -using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Newtonsoft.Json; using Spectre.Console; using SqlBuildManager.Console.Aad; +using SqlBuildManager.Console.Batch; using SqlBuildManager.Console.CloudStorage; using SqlBuildManager.Console.CommandLine; using SqlBuildManager.Console.ContainerApp; using SqlBuildManager.Console.KeyVault; using SqlBuildManager.Console.Kubernetes; using SqlBuildManager.Console.Queue; +using SqlBuildManager.Console.Shared; using SqlBuildManager.Console.Threaded; using SqlBuildManager.Enterprise.Policy; using SqlBuildManager.Interfaces.Console; @@ -20,8 +20,6 @@ using System; using System.Collections.Generic; using System.CommandLine; -using System.CommandLine.Builder; -using System.CommandLine.Help; using System.CommandLine.Parsing; using System.ComponentModel; using System.IO; @@ -29,10 +27,8 @@ using System.Text; using System.Threading; using System.Threading.Tasks; -using sb = SqlSync.SqlBuild; using clb = SqlBuildManager.Console.CommandLine.CommandLineBuilder; -using SqlBuildManager.Console.Batch; -using SqlBuildManager.Console.Shared; +using sb = SqlSync.SqlBuild; namespace SqlBuildManager.Console { @@ -159,7 +155,7 @@ internal static int QueryDatabases(CommandLineArgs cmdLine) if (!String.IsNullOrEmpty(cmdLine.BatchArgs.OutputContainerSasUrl)) { log.LogInformation("Writing log files to storage..."); - var blobTask = WriteLogsToBlobContainer(cmdLine.BatchArgs.OutputContainerSasUrl, cmdLine.RootLoggingPath); + var blobTask = StorageManager.WriteLogsToBlobContainer(cmdLine.ConnectionArgs.StorageAccountName, cmdLine.ConnectionArgs.StorageAccountKey, cmdLine.JobName, cmdLine.RootLoggingPath); blobTask.Wait(); } @@ -312,7 +308,7 @@ private static void Batch_MonitorEnd(object sender, EventArgs e) Worker.activeServiceBusMonitoring = false; } - internal static int RunBatchQuery(CommandLineArgs cmdLine) + internal static async Task RunBatchQuery(CommandLineArgs cmdLine) { SqlBuildManager.Logging.ApplicationLogging.SetLogLevel(cmdLine.LogLevel); log = SqlBuildManager.Logging.ApplicationLogging.CreateLogger(Program.applicationLogFileName, cmdLine.RootLoggingPath); @@ -357,10 +353,10 @@ internal static int RunBatchQuery(CommandLineArgs cmdLine) log.LogInformation("Downloading the consolidated output file..."); try { - var cloudBlobContainer = new BlobContainerClient(new Uri(readOnlySas)); - var blob = cloudBlobContainer.GetBlobClient(cmdLine.OutputFile.Name); - blob.DownloadTo(cmdLine.OutputFile.FullName); - log.LogInformation($"Output file copied locally to {cmdLine.OutputFile.FullName}"); + if(await StorageManager.DownloadBlobToLocal(readOnlySas, cmdLine.OutputFile.FullName)) + { + log.LogInformation($"Output file copied locally to {cmdLine.OutputFile.FullName}"); + } } catch (Exception exe) { @@ -733,7 +729,7 @@ internal static int RunThreadedExecution(CommandLineArgs cmdLine, bool unittest if (!String.IsNullOrEmpty(cmdLine.BatchArgs.OutputContainerSasUrl)) { log.LogInformation("Writing log files to storage..."); - var blobTask = WriteLogsToBlobContainer(cmdLine.BatchArgs.OutputContainerSasUrl, cmdLine.RootLoggingPath); + var blobTask = StorageManager.WriteLogsToBlobContainer(cmdLine.ConnectionArgs.StorageAccountName, cmdLine.ConnectionArgs.StorageAccountKey, cmdLine.JobName, cmdLine.RootLoggingPath); blobTask.Wait(); } @@ -742,78 +738,6 @@ internal static int RunThreadedExecution(CommandLineArgs cmdLine, bool unittest } - - private static async Task WriteLogsToBlobContainer(string outputContainerSasUrl, string rootLoggingPath) - { - try - { - //var writeTasks = new List(); - log.LogInformation($"Writing log files to blob storage at {outputContainerSasUrl}"); - var renameLogFiles = new string[] { "sqlbuildmanager", "csv" }; - - BlobContainerClient container = new BlobContainerClient(new Uri(outputContainerSasUrl)); - log.LogInformation($"Getting file list from {rootLoggingPath}"); - var fileList = Directory.GetFiles(rootLoggingPath, "*.*", SearchOption.AllDirectories); - var taskId = Environment.GetEnvironmentVariable("AZ_BATCH_TASK_ID"); - string machine = Environment.MachineName; - - foreach (var f in fileList) - { - try - { - var tmp = Path.GetRelativePath(rootLoggingPath, f); - - if (Program.AppendLogFiles.Any(a => tmp.ToLower().IndexOf(a) > -1)) - { - - tmp = $"{taskId}/{tmp}"; - log.LogInformation($"Saving File '{f}' as '{tmp}'"); - var rename = container.GetBlockBlobClient(tmp); - using (var fs = new FileStream(f, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) - { - await rename.UploadAsync(fs); - - } - } - else if (renameLogFiles.Any(a => tmp.ToLower().IndexOf(a) > -1)) - { - - //var localTemp = Path.Combine(Path.GetDirectoryName(f), tmp); - //File.Copy(f, localTemp); - tmp = $"{taskId}/{tmp}"; - log.LogInformation($"Saving File '{f}' as '{tmp}'"); - var rename = container.GetBlockBlobClient(tmp); - using (var fs = new FileStream(f, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) - { - await rename.UploadAsync(fs); - - } - } - else - { - log.LogInformation($"Saving File '{f}' as '{tmp}'"); - var b = container.GetBlockBlobClient(tmp); - using (var fs = new FileStream(f, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) - { - await b.UploadAsync(fs); - - } - } - } - catch (Exception e) - { - log.LogError($"Unable to upload log file '{f}' to blob storage: {e.Message}"); - } - } - return true; - } - catch (Exception exe) - { - log.LogError($"Unable to upload log files to blob storage.{Environment.NewLine}{exe.Message}"); - return false; - } - } - private static async Task RunGenericContainerQueueWorker(CommandLineArgs cmdLine) { int result = 1; @@ -842,7 +766,16 @@ private static async Task RunGenericContainerQueueWorker(CommandLineArgs cm keepGoing = false; } } - cmdLine.BatchArgs.OutputContainerSasUrl = CloudStorage.StorageManager.GetOutputContainerSasUrl(cmdLine.ConnectionArgs.StorageAccountName, cmdLine.ConnectionArgs.StorageAccountKey, jobName, false); + + if(string.IsNullOrWhiteSpace(cmdLine.ConnectionArgs.StorageAccountKey)) + { + cmdLine.BatchArgs.OutputContainerSasUrl = CloudStorage.StorageManager.GetContainerRawUrl(cmdLine.ConnectionArgs.StorageAccountName, jobName); + } + else + { + cmdLine.BatchArgs.OutputContainerSasUrl = CloudStorage.StorageManager.GetOutputContainerSasUrl(cmdLine.ConnectionArgs.StorageAccountName, cmdLine.ConnectionArgs.StorageAccountKey, jobName, false); + } + if (keepGoing) { @@ -926,9 +859,8 @@ internal static async Task MonitorServiceBusRuntimeProgress(CommandLineArgs var qManager = new Queue.QueueManager(cmdLine.ConnectionArgs.ServiceBusTopicConnectionString, cmdLine.JobName, cmdLine.ConcurrencyType); //set up event handler - var storageString = CloudStorage.StorageManager.GetStorageConnectionString(cmdLine.ConnectionArgs.StorageAccountName, cmdLine.ConnectionArgs.StorageAccountKey); (string jobName, string discard) = CloudStorage.StorageManager.GetJobAndStorageNames(cmdLine); - var ehandler = new Events.EventManager(cmdLine.ConnectionArgs.EventHubConnectionString, storageString, jobName, jobName); + var ehandler = new Events.EventManager(cmdLine.ConnectionArgs.EventHubConnectionString,cmdLine.ConnectionArgs.StorageAccountName, cmdLine.ConnectionArgs.StorageAccountKey,jobName, jobName); Task eventHubMonitorTask = null; if (!string.IsNullOrWhiteSpace(cmdLine.ConnectionArgs.EventHubConnectionString)) @@ -1061,14 +993,11 @@ internal static async Task MonitorServiceBusRuntimeProgress(CommandLineArgs await qManager.DeleteSubscription(); //Batch jobs have their own consolidation.... - if (cmdLine.BatchArgs == null || string.IsNullOrWhiteSpace(cmdLine.BatchArgs.BatchJobName)) + if (cmdLine.BatchArgs == null || string.IsNullOrWhiteSpace(cmdLine.BatchArgs.BatchPoolName)) { log.LogInformation("Consolidating log files"); StorageManager.ConsolidateLogFiles(cmdLine.ConnectionArgs.StorageAccountName, cmdLine.ConnectionArgs.StorageAccountKey, cmdLine.JobName, new List()); - string sas = StorageManager.GetOutputContainerSasUrl(cmdLine.ConnectionArgs.StorageAccountName, cmdLine.ConnectionArgs.StorageAccountKey, cmdLine.JobName, true); - - log.LogInformation($"Log files can be found here: {sas}"); - log.LogInformation("The read-only SAS token URL is valid for 7 days."); + log.LogInformation($"The consolidated log files can be found in the Azure storage account '{cmdLine.ConnectionArgs.StorageAccountName}' in blob container '{cmdLine.JobName}'"); log.LogInformation("You can download \"Azure Storage Explorer\" from here: https://azure.microsoft.com/en-us/features/storage-explorer/"); } if (error > 0) @@ -1193,9 +1122,14 @@ internal static async Task PrepAndUploadContainerAppBuildPackage(CommandLin { cmdLine.BuildFileName = packageName.FullName; } - if (string.IsNullOrWhiteSpace(cmdLine.ConnectionArgs.StorageAccountName) || string.IsNullOrWhiteSpace(cmdLine.ConnectionArgs.StorageAccountKey)) + if (string.IsNullOrWhiteSpace(cmdLine.ConnectionArgs.StorageAccountName)) { - log.LogError("--storageaccountname and --storageaccountkey are required"); + log.LogError("--storageaccountname is required"); + return -1; + } + if(string.IsNullOrWhiteSpace(cmdLine.ConnectionArgs.StorageAccountKey) && string.IsNullOrWhiteSpace(cmdLine.IdentityArgs.ClientId)) + { + log.LogError("--storageaccountkey is required if a Managed Identity is not included"); return -1; } @@ -1337,12 +1271,18 @@ internal static async Task UploadKubernetesBuildPackage(FileInfo secretsFil if (!string.IsNullOrWhiteSpace(storageAccountKey)) cmdLine.StorageAccountKey = storageAccountKey; if (!string.IsNullOrWhiteSpace(storageAccountName)) cmdLine.StorageAccountName = storageAccountName; - if (string.IsNullOrWhiteSpace(cmdLine.JobName) || string.IsNullOrWhiteSpace(cmdLine.ConnectionArgs.StorageAccountName) || string.IsNullOrWhiteSpace(cmdLine.ConnectionArgs.StorageAccountKey)) + if (string.IsNullOrWhiteSpace(cmdLine.JobName) || string.IsNullOrWhiteSpace(cmdLine.ConnectionArgs.StorageAccountName)) { - log.LogError("Values for --jobname, --storageaccountname and --storageaccountkey are required as prameters or included in the --secretsfile and --runtimefile"); + log.LogError("Values for --jobname, and --storageaccountname are required as prameters or included in the --secretsfile and --runtimefile"); return 1; } + //if (string.IsNullOrWhiteSpace(cmdLine.ConnectionArgs.StorageAccountKey) && string.IsNullOrWhiteSpace(cmdLine.IdentityArgs.ClientId)) + //{ + // log.LogError("A value for --storageaccountkey are required as prameters or included in the --secretsfile and --runtimefile"); + // return 1; + + //} (bool retVal, string sbmName) = await ValidateAndUploadContainerBuildFilesToStorage(cmdLine, packageName, platinumDacpac, force); if (!retVal) { @@ -1565,10 +1505,9 @@ internal static async Task PrepAndUploadAciBuildPackage(CommandLineArgs cmd } (bool na, cmdLine) = Cryptography.DecryptSensitiveFields(cmdLine); - if ((string.IsNullOrWhiteSpace(cmdLine.ConnectionArgs.StorageAccountName) || string.IsNullOrWhiteSpace(cmdLine.ConnectionArgs.StorageAccountKey)) - && string.IsNullOrWhiteSpace(cmdLine.ConnectionArgs.KeyVaultName)) + if (string.IsNullOrWhiteSpace(cmdLine.ConnectionArgs.StorageAccountName) && string.IsNullOrWhiteSpace(cmdLine.ConnectionArgs.KeyVaultName)) { - log.LogError("If --keyvaultname is not provided as an argument or in the --settingsfile, then --storageaccountname and --storageaccountkey are required"); + log.LogError("If --keyvaultname is not provided as an argument or in the --settingsfile, then --storageaccountname is required"); return -1; } (bool retVal, string sbmName) = await ValidateAndUploadContainerBuildFilesToStorage(cmdLine, packageName, platinumDacpac, force); @@ -1605,10 +1544,9 @@ internal static int GetEventHubEvents(CommandLineArgs cmdLine, DateTime? startDa bool firstLoop = true; (junk, cmdLine) = Init(cmdLine); - var storageString = CloudStorage.StorageManager.GetStorageConnectionString(cmdLine.ConnectionArgs.StorageAccountName, cmdLine.ConnectionArgs.StorageAccountKey); (string jobName, string discard) = CloudStorage.StorageManager.GetJobAndStorageNames(cmdLine); - var ehandler = new Events.EventManager(cmdLine.ConnectionArgs.EventHubConnectionString, storageString, jobName, jobName); - if(!startDate.HasValue) + var ehandler = new Events.EventManager(cmdLine.ConnectionArgs.EventHubConnectionString, cmdLine.ConnectionArgs.StorageAccountName, cmdLine.ConnectionArgs.StorageAccountKey, jobName, jobName); + if (!startDate.HasValue) { startDate = DateTime.UtcNow.AddDays(-14); } @@ -1688,10 +1626,17 @@ internal static void SaveKubernetesSettings(CommandLineArgs cmdLine, string pref } else { - string secrets = ContainerManager.GenerateSecretsYaml(cmdLine); var secretsName = Path.Combine(dir, string.IsNullOrWhiteSpace(prefix) ? "secrets.yaml" : $"{prefix}-secrets.yaml"); - File.WriteAllText(secretsName, secrets); - log.LogInformation($"Secrets file written to: {secretsName}"); + string secrets = ContainerManager.GenerateSecretsYaml(cmdLine); + if (!string.IsNullOrWhiteSpace(secrets)) + { + File.WriteAllText(secretsName, secrets); + log.LogInformation($"Secrets file written to: {secretsName}"); + } + else + { + log.LogInformation($"NO secrets are needed, NOT saving secrets yaml to : {secretsName}"); + } } string runtime = ContainerManager.GenerateRuntimeYaml(cmdLine); diff --git a/src/SqlBuildManager.Console/sbm.csproj b/src/SqlBuildManager.Console/sbm.csproj index 730b5cf8..6fece3bc 100644 --- a/src/SqlBuildManager.Console/sbm.csproj +++ b/src/SqlBuildManager.Console/sbm.csproj @@ -22,8 +22,8 @@ - - + + diff --git a/src/SqlBuildManager.Logging/SqlBuildManager.Logging.csproj b/src/SqlBuildManager.Logging/SqlBuildManager.Logging.csproj index 70aa7ea0..ce9608e2 100644 --- a/src/SqlBuildManager.Logging/SqlBuildManager.Logging.csproj +++ b/src/SqlBuildManager.Logging/SqlBuildManager.Logging.csproj @@ -15,7 +15,7 @@ - +