diff --git a/CHANGELOG.md b/CHANGELOG.md index d2c547ae..903a41d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # SQL Build Manager Change Log +### Version 15.5.0 +- *NEW:* For muti-database target builds, you can now specify custom concurrency tag. Previously, the only concurrency differentitor was by SQL Server Name. Please see the docs on [Concurrency](/docs/concurrency_options.md) and [Database targeting options](docs/override_options.md) to understand how to use this new feature. +- *UPDATED:* DACPAC creation timeouts now set to the value of `--defaultscripttimeout`. Previously, it was using the default settings. +- *UPDATED:* If a script package is failing after retries due to a timeout, the build will now immediately fail. The prior behavior was to create a custom DACPAC (if configured) and continue trying the build. This was just delaying the inevitable failure and wasting time. + ### Version 15.4.2 - *UPDATED:* Converted Batch Node Pool creation to the new `Azure.ResourceManager.Batch` SDK. **NOTE:** This may require you to add batch specific NSG rules if you deploy into a subnet. See [network.bicep's](./scripts/templates/Modules/network.bicep) `nsgBatchResource` resource to see the rules that are needed. diff --git a/docs/concurrency_options.md b/docs/concurrency_options.md index bc251d4b..6c285463 100644 --- a/docs/concurrency_options.md +++ b/docs/concurrency_options.md @@ -1,4 +1,4 @@ -# Concurrency Options for Threaded, Batch and Kubernetes executions +# Concurrency Options for Threaded, Batch, Kubernetes, Container App and ACI executions You can control the level of parallel execution with the combination of two arguments: `--concurrency` and `--concurrencytype`. While their meaning for threaded and batch/kubernetes are similar, there are some distinctions and subtleties when used together @@ -19,7 +19,7 @@ You can control the level of parallel execution with the combination of two argu - `Count` - (default) will use the value for `concurrency` as the maximum number of concurrent tasks allowed - `Server` - When using this value, the `concurrency` value is ignored. Instead, the app will interrogate the database targets and allow one task per SQL Server at a time - _For example:_ if there are 5 unuque SQL Server targets in the database targets config, there will 1 task per server, up to 5 concurrent tasks total + _For example:_ if there are 5 unique SQL Server targets in the database targets config, there will 1 task per server, up to 5 concurrent tasks total - `MaxPerServer` - Will interrogate the database targets and allow multiple tasks per server, up to the `concurrency` value. @@ -29,6 +29,9 @@ You can control the level of parallel execution with the combination of two argu - `MaxPerTag` - - Same behavior as `MaxPerServer` but allows you to set a concurrency value per tag value instead of server. See [database targets config file](override_options.md) for details on setting the concurrency tag value +*NOTE:* If a concurrency typf of `Tag` or `MaxPerTag` is set but there are database targets that are missing a tag value, the build will not start. + + ### --concurrency This argument takes an integer number (default is 8) defining the maximum number of concurrent threads per the set `concurrencytype` diff --git a/scripts/templates/Database/create_database_override_files.ps1 b/scripts/templates/Database/create_database_override_files.ps1 index 19321d2d..04adea48 100644 --- a/scripts/templates/Database/create_database_override_files.ps1 +++ b/scripts/templates/Database/create_database_override_files.ps1 @@ -1,8 +1,7 @@ param ( [string] $path, - [string] $prefix, - [string] $resourceGroupName + [string] $prefix ) . ./../prefix_resource_names.ps1 -prefix $prefix @@ -17,6 +16,7 @@ $databaseDbWithBadTargetConfigFile = Join-Path $path "databasetargets-badtargets $clientDbConfigFile = Join-Path $path "clientdbtargets.cfg" $doubleClientDbConfigFile = Join-Path $path "clientdbtargets-doubledb.cfg" $taggedDbConfigFile = Join-Path $path "databasetargets-tag.cfg" +$taggedClientDbConfigFile = Join-Path $path "clientdbtargets-tag.cfg" $serverTextFile = Join-Path $path "server.txt" $tag = @("TagA","TagB","TagC") $counter = 0 @@ -37,8 +37,9 @@ foreach($server in $sqlServers) } $outputDbConfig += ,@($server.fullyQualifiedDomainName + ":SqlBuildTest,"+$db) $databaseDbWithBadTargetConfig += ,@($server.fullyQualifiedDomainName + ":SqlBuildTest,"+$db) - $clientDbConfig += ,@($server.fullyQualifiedDomainName + ":client,"+$db) $taggedDbConfig += ,@($server.fullyQualifiedDomainName + ":SqlBuildTest,"+$db +"#" + $tag[$counter]) + $taggedClientDbConfig += ,@($server.fullyQualifiedDomainName + ":client,"+$db +"#" + $tag[$counter]) + $clientDbConfig += ,@($server.fullyQualifiedDomainName + ":client,"+$db) $counter = $counter +1; } @@ -63,19 +64,22 @@ foreach($server in $sqlServers) } <# Action that will repeat until the condition is met #> -Write-Host "Writing test database config to path set to $outputDbConfigFile" -ForegroundColor DarkGreen +Write-Host "Writing test database config to $outputDbConfigFile" -ForegroundColor DarkGreen $outputDbConfig | Set-Content -Path $outputDbConfigFile -Write-Host "Writing test database config to path set to $clientDbConfigFile" -ForegroundColor DarkGreen +Write-Host "Writing test database config to path $clientDbConfigFile" -ForegroundColor DarkGreen $clientDbConfig | Set-Content -Path $clientDbConfigFile -Write-Host "Writing test database config to path set to $doubleClientDbConfigFile" -ForegroundColor DarkGreen +Write-Host "Writing test database config with tags to path $taggedClientDbConfigFile" -ForegroundColor DarkGreen +$taggedClientDbConfigFile | Set-Content -Path $taggedClientDbConfigFile + +Write-Host "Writing test database config to path $doubleClientDbConfigFile" -ForegroundColor DarkGreen $doubleClientDbConfig | Set-Content -Path $doubleClientDbConfigFile -Write-Host "Writing test database config to path set to $databaseDbWithBadTargetConfigFile" -ForegroundColor DarkGreen +Write-Host "Writing test database config to path $databaseDbWithBadTargetConfigFile" -ForegroundColor DarkGreen $databaseDbWithBadTargetConfig | Set-Content -Path $databaseDbWithBadTargetConfigFile -Write-Host "Writing test database config to path set to $taggedDbConfigFile" -ForegroundColor DarkGreen +Write-Host "Writing test database config with tags to path $taggedDbConfigFile" -ForegroundColor DarkGreen $taggedDbConfig | Set-Content -Path $taggedDbConfigFile Write-Host "Creating server.txt file for SQL Query override config tests" -ForegroundColor DarkGreen diff --git a/scripts/templates/create_all_settingsfiles_fromprefix.ps1 b/scripts/templates/create_all_settingsfiles_fromprefix.ps1 index 42fd5fc5..189932af 100644 --- a/scripts/templates/create_all_settingsfiles_fromprefix.ps1 +++ b/scripts/templates/create_all_settingsfiles_fromprefix.ps1 @@ -23,4 +23,4 @@ param ( ./aci/create_aci_settingsfile_fromprefix.ps1 -sbmExe $sbmExe -path $outputPath -resourceGroupName $resourceGroupName -prefix $prefix . ./prefix_resource_names.ps1 -prefix $prefix -./Database/create_database_override_files.ps1 -sbmExe $sbmExe -path $outputPath -resourceGroupName $resourceGroupName \ No newline at end of file +./Database/create_database_override_files.ps1 -path $outputPath -prefix $prefix \ No newline at end of file diff --git a/src/AssemblyVersioning.cs b/src/AssemblyVersioning.cs index 37544637..2b4ed24c 100644 --- a/src/AssemblyVersioning.cs +++ b/src/AssemblyVersioning.cs @@ -24,5 +24,5 @@ // 2) Update the installer version to match the AssemblyVersion below. // These can be found in SqlBuildManager.Setup -> Organize Your Setup -> General Information -[assembly: AssemblyVersion("15.4.3")] -[assembly: AssemblyFileVersion("15.4.3")] +[assembly: AssemblyVersion("15.5.0")] +[assembly: AssemblyFileVersion("15.5.0")] diff --git a/src/SqlBuildManager.Console.ExternalTest/BatchTests.cs b/src/SqlBuildManager.Console.ExternalTest/BatchTests.cs index d02d1497..e8c55bec 100644 --- a/src/SqlBuildManager.Console.ExternalTest/BatchTests.cs +++ b/src/SqlBuildManager.Console.ExternalTest/BatchTests.cs @@ -12,6 +12,7 @@ using SqlBuildManager.Console.Batch; using Microsoft.Azure.Batch.Auth; using Microsoft.Azure.Batch; +using System.Threading; namespace SqlBuildManager.Console.ExternalTest { @@ -151,7 +152,11 @@ public void Batch_Override_SBMSource_ByTag_ConcurrencyType_Success(string batchM "--packagename", sbmFileName, "--concurrency", concurrency.ToString(), "--concurrencytype",concurType.ToString(), - "--jobname", jobName }; + "--jobname", jobName, + "--unittest", + "--monitor", + "--stream", + "--eventhublogging", EventHubLogging.IndividualScriptResults.ToString()}; RootCommand rootCommand = CommandLineBuilder.SetUp(); var val = rootCommand.InvokeAsync(args); @@ -172,6 +177,144 @@ public void Batch_Override_SBMSource_ByTag_ConcurrencyType_Success(string batchM } } + [DataRow("runthreaded", "TestConfig/settingsfile-batch-windows.json", ConcurrencyType.Tag, 10)] + [DataRow("run", "TestConfig/settingsfile-batch-windows.json", ConcurrencyType.Tag, 10)] + [DataRow("run", "TestConfig/settingsfile-batch-linux.json", ConcurrencyType.Tag, 10)] + + [DataRow("runthreaded", "TestConfig/settingsfile-batch-windows.json", ConcurrencyType.MaxPerTag, 2)] + [DataRow("run", "TestConfig/settingsfile-batch-windows.json", ConcurrencyType.MaxPerTag, 2)] + [DataRow("run", "TestConfig/settingsfile-batch-linux.json", ConcurrencyType.MaxPerTag, 2)] + [DataTestMethod] + public void Batch_Override_SBMSource_ByTag_ConcurrencyType_MissingTag_Fail(string batchMethod, string settingsFile, ConcurrencyType concurType, int concurrency) + { + string sbmFileName = Path.GetFullPath("SimpleSelect.sbm"); + if (!File.Exists(sbmFileName)) + { + File.WriteAllBytes(sbmFileName, Properties.Resources.SimpleSelect); + } + + settingsFile = Path.GetFullPath(settingsFile); + string jobName = GetUniqueBatchJobName("batch-sbm-tag"); + + //get the size of the log file before we start + int startingLine = LogFileCurrentLineCount(); + + var args = new string[]{ + "--loglevel", "Debug", + "batch", batchMethod, + "--settingsfile", settingsFile, + "--settingsfilekey", settingsFileKeyPath, + "--override", overrideFilePath, + "--packagename", sbmFileName, + "--concurrency", concurrency.ToString(), + "--concurrencytype",concurType.ToString(), + "--jobname", jobName, + "--unittest", + "--monitor", + "--stream", + "--eventhublogging", EventHubLogging.IndividualScriptResults.ToString()}; + + RootCommand rootCommand = CommandLineBuilder.SetUp(); + var val = rootCommand.InvokeAsync(args); + val.Wait(); + var result = val.Result; + Assert.IsTrue(val.Result != 0); + } + + [DataRow("runthreaded", "TestConfig/settingsfile-batch-windows-queue.json", ConcurrencyType.Tag, 2)] + [DataRow("run", "TestConfig/settingsfile-batch-windows-queue.json", ConcurrencyType.Tag, 2)] + [DataRow("run", "TestConfig/settingsfile-batch-linux-queue.json", ConcurrencyType.Tag, 2)] + [DataRow("run", "TestConfig/settingsfile-batch-windows-queue-keyvault.json", ConcurrencyType.Tag, 2)] + [DataRow("run", "TestConfig/settingsfile-batch-linux-queue-keyvault.json", ConcurrencyType.Tag, 2)] + [DataRow("runthreaded", "TestConfig/settingsfile-batch-windows-queue.json", ConcurrencyType.MaxPerTag, 5)] + [DataRow("run", "TestConfig/settingsfile-batch-windows-queue.json", ConcurrencyType.MaxPerTag, 5)] + [DataRow("run", "TestConfig/settingsfile-batch-linux-queue.json", ConcurrencyType.MaxPerTag, 5)] + [DataRow("run", "TestConfig/settingsfile-batch-windows-queue-keyvault.json", ConcurrencyType.MaxPerTag, 5)] + [DataRow("run", "TestConfig/settingsfile-batch-linux-queue-keyvault.json", ConcurrencyType.MaxPerTag, 5)] + [DataTestMethod] + public void Batch_Queue_SBMSource_ByTag_ConcurrencyType_Success(string batchMethod, string settingsFile, ConcurrencyType concurType, int concurrency) + { + settingsFile = Path.GetFullPath(settingsFile); + string sbmFileName = Path.GetFullPath("SimpleSelect.sbm"); + if (!File.Exists(sbmFileName)) + { + File.WriteAllBytes(sbmFileName, Properties.Resources.SimpleSelect); + } + string jobName = GetUniqueBatchJobName("batch-sbm-tag"); + int startingLine = LogFileCurrentLineCount(); + + var args = new string[]{ + "batch", "enqueue", + "--settingsfile", settingsFile, + "--settingsfilekey", settingsFileKeyPath, + "--override" , overrideWithTagFilePath, + "--concurrencytype", concurType.ToString(), + "--jobname", jobName}; + + RootCommand rootCommand = CommandLineBuilder.SetUp(); + Task val = rootCommand.InvokeAsync(args); + val.Wait(); + var result = val.Result; + + var logFileContents = ReleventLogFileContents(startingLine); + Assert.AreEqual(0, result, StandardExecutionErrorMessage(logFileContents)); + + args = new string[]{ + "--loglevel", "debug", + "batch", batchMethod, + "--settingsfile", settingsFile, + "--settingsfilekey", settingsFileKeyPath, + "--override", overrideWithTagFilePath, + "--packagename", sbmFileName, + "--concurrencytype", concurType.ToString(), + "--concurrency", concurrency.ToString(), + "--jobname", jobName, + "--unittest", + "--monitor", + "--stream" }; + + + val = rootCommand.InvokeAsync(args); + val.Wait(); + result = val.Result; + + logFileContents = ReleventLogFileContents(startingLine); + Assert.AreEqual(0, result, StandardExecutionErrorMessage(logFileContents)); + } + + [DataRow("runthreaded", "TestConfig/settingsfile-batch-windows-queue.json", ConcurrencyType.Tag, 2)] + [DataRow("run", "TestConfig/settingsfile-batch-linux-queue.json", ConcurrencyType.Tag, 2)] + [DataRow("run", "TestConfig/settingsfile-batch-windows-queue-keyvault.json", ConcurrencyType.Tag, 2)] + [DataRow("runthreaded", "TestConfig/settingsfile-batch-windows-queue.json", ConcurrencyType.MaxPerTag, 5)] + [DataRow("run", "TestConfig/settingsfile-batch-linux-queue.json", ConcurrencyType.MaxPerTag, 5)] + [DataRow("run", "TestConfig/settingsfile-batch-windows-queue-keyvault.json", ConcurrencyType.MaxPerTag, 5)] + [DataTestMethod] + public void Batch_Queue_SBMSource_ByTag_ConcurrencyType_MissingTag_Fail(string batchMethod, string settingsFile, ConcurrencyType concurType, int concurrency) + { + settingsFile = Path.GetFullPath(settingsFile); + string sbmFileName = Path.GetFullPath("SimpleSelect.sbm"); + if (!File.Exists(sbmFileName)) + { + File.WriteAllBytes(sbmFileName, Properties.Resources.SimpleSelect); + } + string jobName = GetUniqueBatchJobName("batch-sbm-tag"); + int startingLine = LogFileCurrentLineCount(); + + var args = new string[]{ + "batch", "enqueue", + "--settingsfile", settingsFile, + "--settingsfilekey", settingsFileKeyPath, + "--override" , overrideFilePath, + "--concurrencytype", concurType.ToString(), + "--jobname", jobName}; + + RootCommand rootCommand = CommandLineBuilder.SetUp(); + Task val = rootCommand.InvokeAsync(args); + val.Wait(); + Assert.IsTrue(val.Result != 0); + + } + [DataRow("runthreaded", "TestConfig/settingsfile-batch-windows.json", ConcurrencyType.Count, 10)] [DataRow("run", "TestConfig/settingsfile-batch-windows.json", ConcurrencyType.Count, 10)] [DataRow("run", "TestConfig/settingsfile-batch-linux.json", ConcurrencyType.Count, 10)] @@ -207,7 +350,11 @@ public void Batch_Override_SBMSource_ByConcurrencyType_Success(string batchMetho "--packagename", sbmFileName, "--concurrency", concurrency.ToString(), "--concurrencytype",concurType.ToString(), - "--jobname", jobName }; + "--jobname", jobName, + "--unittest", + "--monitor", + "--stream", + "--eventhublogging", EventHubLogging.IndividualScriptResults.ToString()}; RootCommand rootCommand = CommandLineBuilder.SetUp(); var val = rootCommand.InvokeAsync(args); @@ -297,8 +444,6 @@ public void Batch_SqlScriptOverride_SBMSource_Success(string batchMethod, string "--username", un, "--password", pw, "--outputfile", tmpOverride, - "--concurrency", concurrency.ToString(), - "--concurrencytype",concurType.ToString(), "--force"}; RootCommand rootCommand = CommandLineBuilder.SetUp(); @@ -317,8 +462,8 @@ public void Batch_SqlScriptOverride_SBMSource_Success(string batchMethod, string "--settingsfilekey", settingsFileKeyPath, "--override", tmpOverride, "--packagename", sbmFileName, - "--concurrency", "2", - "--concurrencytype","Server", + "--concurrency", concurrency.ToString(), + "--concurrencytype",concurType.ToString(), "--jobname", jobName}; rootCommand = CommandLineBuilder.SetUp(); @@ -346,8 +491,6 @@ public void Batch_SqlScriptOverride_SBMSource_Success(string batchMethod, string catch { } } - - [DataRow("run", "TestConfig/settingsfile-batch-windows-mi.json", ConcurrencyType.Count, 10)] [DataRow("run", "TestConfig/settingsfile-batch-linux-mi.json", ConcurrencyType.Count, 10)] [DataTestMethod] @@ -396,8 +539,6 @@ public void Batch_Override_SBMSource_ManagedIdentity_ByConcurrencyType_Success(s } } - - [DataRow("runthreaded", "TestConfig/settingsfile-batch-windows.json")] [DataRow("run", "TestConfig/settingsfile-batch-windows.json")] [DataRow("run", "TestConfig/settingsfile-batch-linux.json")] @@ -428,7 +569,6 @@ public void Batch_Override_SBMSource_RunWithError_MissingPackage(string batchMet Assert.IsTrue(logFileContents.Contains("Invalid command line set") && logFileContents.ToLower().Contains("packagename"), "This test should report a missing commandline"); } - [DataRow("runthreaded", "TestConfig/settingsfile-batch-windows.json")] [DataRow("run", "TestConfig/settingsfile-batch-windows.json")] [DataRow("run", "TestConfig/settingsfile-batch-linux.json")] @@ -526,6 +666,7 @@ public void Batch_Override_PlatinumDbSource_FirstDbAlreadyInSync(string batchMet } } + [DataRow("runthreaded", "TestConfig/settingsfile-batch-windows.json")] [DataRow("run", "TestConfig/settingsfile-batch-windows.json")] [DataRow("run", "TestConfig/settingsfile-batch-linux.json")] @@ -1789,33 +1930,38 @@ public void Batch_Queue_LongRunning_SBMSource_ByConcurrencyType_Success(string b [DataTestMethod] public async Task CreateBatchPool_Success(string settingsFile, string settingsFileKeyPath) { - var poolId = "TestPool1"; - settingsFile = Path.GetFullPath(settingsFile); - CommandLineArgs cmdLine = new CommandLineArgs(); - cmdLine.SettingsFileKey = settingsFileKeyPath; - cmdLine.FileInfoSettingsFile = new FileInfo(settingsFile); - if (cmdLine.IdentityArgs != null) SqlBuildManager.Console.Aad.AadHelper.ManagedIdentityClientId = cmdLine.IdentityArgs.ClientId; - if (cmdLine.IdentityArgs != null) SqlBuildManager.Console.Aad.AadHelper.TenantId = cmdLine.IdentityArgs.TenantId; + var poolId = "TestPool1"; + settingsFile = Path.GetFullPath(settingsFile); - - BatchManager mgr = new BatchManager(cmdLine); - var result = await mgr.CreateBatchPool(cmdLine, poolId); - - Assert.IsTrue(result); + CommandLineArgs cmdLine = new CommandLineArgs(); + cmdLine.SettingsFileKey = settingsFileKeyPath; + cmdLine.FileInfoSettingsFile = new FileInfo(settingsFile); + if (cmdLine.IdentityArgs != null) SqlBuildManager.Console.Aad.AadHelper.ManagedIdentityClientId = cmdLine.IdentityArgs.ClientId; + if (cmdLine.IdentityArgs != null) SqlBuildManager.Console.Aad.AadHelper.TenantId = cmdLine.IdentityArgs.TenantId; - - var batchToken = await SqlBuildManager.Console.Aad.AadHelper.GetBatchTokenString(); - BatchTokenCredentials batchTokenCredentials = new BatchTokenCredentials(cmdLine.ConnectionArgs.BatchAccountUrl, batchToken); - BatchClient batchClient = BatchClient.Open(batchTokenCredentials); - var existingPool = await batchClient.PoolOperations.GetPoolAsync(poolId); + BatchManager mgr = new BatchManager(cmdLine); + var result = await mgr.CreateBatchPool(cmdLine, poolId); + + Assert.IsTrue(result); - Assert.IsNotNull(existingPool); - Assert.IsNotNull(existingPool.Identity); + var batchToken = await SqlBuildManager.Console.Aad.AadHelper.GetBatchTokenString(); + BatchTokenCredentials batchTokenCredentials = new BatchTokenCredentials(cmdLine.ConnectionArgs.BatchAccountUrl, batchToken); + BatchClient batchClient = BatchClient.Open(batchTokenCredentials); + try + { + var existingPool = await batchClient.PoolOperations.GetPoolAsync(poolId); + Assert.IsNotNull(existingPool); + Assert.IsNotNull(existingPool.Identity); + } + finally + { + batchClient.PoolOperations.DeletePool(poolId); + } } } diff --git a/src/SqlBuildManager.Console.ExternalTest/KubernetesTests.cs b/src/SqlBuildManager.Console.ExternalTest/KubernetesTests.cs index 3f67db09..ca2adf6e 100644 --- a/src/SqlBuildManager.Console.ExternalTest/KubernetesTests.cs +++ b/src/SqlBuildManager.Console.ExternalTest/KubernetesTests.cs @@ -364,6 +364,76 @@ public void Kubernetes_Run_Queue_DoubleDbConfig_SBMSource_Success(string setting } + [DataRow("TestConfig/settingsfile-k8s-kv.json", ConcurrencyType.Count, 5)] + [DataRow("TestConfig/settingsfile-k8s-kv.json", ConcurrencyType.Server, 5)] + [DataRow("TestConfig/settingsfile-k8s-kv.json", ConcurrencyType.MaxPerServer, 5)] + [DataRow("TestConfig/settingsfile-k8s-kv.json", ConcurrencyType.Tag, 5)] + [DataRow("TestConfig/settingsfile-k8s-kv.json", ConcurrencyType.MaxPerTag, 5)] + [DataRow("TestConfig/settingsfile-k8s-kv-mi.json", ConcurrencyType.MaxPerTag, 5)] + [DataRow("TestConfig/settingsfile-k8s-sec-mi.json", ConcurrencyType.MaxPerTag, 3)] + [DataRow("TestConfig/settingsfile-k8s-sec.json", ConcurrencyType.MaxPerServer, 5)] + [DataTestMethod] + public void Kubernetes_Run_Queue_Concurrency_SBMSource_Success(string settingsFile, ConcurrencyType concurType, int concurrencyCount) + { + try + { + var prc = new ProcessHelper(); + settingsFile = Path.GetFullPath(settingsFile); + var overrideFilePlain = Path.GetFullPath("TestConfig/databasetargets.cfg"); + var overrideFileTags = Path.GetFullPath("TestConfig/databasetargets-tag.cfg"); + var overrideFile = concurType == ConcurrencyType.Tag || concurType == ConcurrencyType.MaxPerTag ? overrideFileTags : overrideFilePlain; + + var sbmFileName = Path.GetFullPath("SimpleSelect.sbm"); + if (!File.Exists(sbmFileName)) + { + File.WriteAllBytes(sbmFileName, Properties.Resources.SimpleSelect_DoubleClient); + } + string jobName = TestHelper.GetUniqueJobName("k8s"); + + + //get the size of the log file before we start + int startingLine = TestHelper.LogFileCurrentLineCount(); + + RootCommand rootCommand = CommandLineBuilder.SetUp(); + + //Clear any exiting pods + var result = prc.ExecuteProcess("kubectl", $"delete job sqlbuildmanager "); + + var args = new string[] + { + "--loglevel", "debug", + "k8s", "run", + "--settingsfile", settingsFile, + "--settingsfilekey", settingsFileKeyPath, + "--jobname", jobName, + "--override", overrideFile, + "--packagename", sbmFileName, + "--force", + "--unittest", + "--stream", + "--concurrencytype", concurType.ToString(), + "--concurrency", concurrencyCount.ToString() + }; + + + var val = rootCommand.InvokeAsync(args); + val.Wait(); + result = val.Result; + + Assert.AreEqual(0, result); + + var logFileContents = TestHelper.ReleventLogFileContents(startingLine); + + var dbCount = File.ReadAllText(overrideFile).Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries).Length; + Assert.IsTrue(ConsoleOutput.ToString().Contains($"Database Commits: {dbCount.ToString().PadLeft(5, '0')}")); + } + finally + { + Debug.WriteLine(ConsoleOutput.ToString()); + } + + } + //Can't run local unit tests with MI since an SBM package needs to be created :-) [DataRow("TestConfig/settingsfile-k8s-kv.json")] [DataRow("TestConfig/settingsfile-k8s-sec.json")] diff --git a/src/SqlBuildManager.Console.UnitTest/ConcurrencyTest.cs b/src/SqlBuildManager.Console.UnitTest/ConcurrencyTest.cs index 4f4f1ac8..037413a5 100644 --- a/src/SqlBuildManager.Console.UnitTest/ConcurrencyTest.cs +++ b/src/SqlBuildManager.Console.UnitTest/ConcurrencyTest.cs @@ -583,7 +583,7 @@ public void QueueMessage_Creation_Test() var output = Concurrency.ConcurrencyByInt(multiData, 10); var qMgr = new QueueManager("", "testing", CommandLine.ConcurrencyType.Count, true); - var messages = qMgr.CreateMessages(output, "testing"); + var messages = qMgr.CreateMessages(output, "testing", CommandLine.ConcurrencyType.Count); var msg = messages.First().As(); Assert.AreEqual(2, msg.DbOverrideSequence.Count()); } diff --git a/src/SqlBuildManager.Console.UnitTest/QueueMessageTest.cs b/src/SqlBuildManager.Console.UnitTest/QueueMessageTest.cs index b22b0adf..2459ebaa 100644 --- a/src/SqlBuildManager.Console.UnitTest/QueueMessageTest.cs +++ b/src/SqlBuildManager.Console.UnitTest/QueueMessageTest.cs @@ -20,7 +20,7 @@ public void QueueMessage_Creation_Test() var output = Concurrency.ConcurrencyByInt(multiData, 10); var qMgr = new QueueManager("", "testing", CommandLine.ConcurrencyType.Count,true); - var messages = qMgr.CreateMessages(output, "testing"); + var messages = qMgr.CreateMessages(output, "testing", CommandLine.ConcurrencyType.Count); } finally { @@ -43,7 +43,7 @@ public void QueueMessage_Creation_FromTag_Test() var output = Concurrency.ConcurrencyByType(multiData, 10, CommandLine.ConcurrencyType.Tag); var qMgr = new QueueManager("", "testing", CommandLine.ConcurrencyType.Count,true); - var messages = qMgr.CreateMessages(output, "testing"); + var messages = qMgr.CreateMessages(output, "testing", CommandLine.ConcurrencyType.Count); Assert.AreEqual(2367, messages.Count); } finally @@ -54,5 +54,29 @@ public void QueueMessage_Creation_FromTag_Test() } } } + + [TestMethod] + public void QueueMessage_Creation_FromTag_MissingTag_Test() + { + string tmpFile = string.Empty; + MultiDbData multiData; + + try + { + (tmpFile, multiData) = ConcurrencyTest.GetMultiDbData(ConcurrencyTest.MultiDbType.SingleTarget); + var output = Concurrency.ConcurrencyByType(multiData, 10, CommandLine.ConcurrencyType.Count); + var qMgr = new QueueManager("", "testing", CommandLine.ConcurrencyType.Tag, true); + + var messages = qMgr.CreateMessages(output, "testing", CommandLine.ConcurrencyType.Tag); + Assert.IsNull(messages); + } + finally + { + if (File.Exists(tmpFile)) + { + File.Delete(tmpFile); + } + } + } } } diff --git a/src/SqlBuildManager.Console/Batch/BatchManager.cs b/src/SqlBuildManager.Console/Batch/BatchManager.cs index ec6bf933..e25fa98f 100644 --- a/src/SqlBuildManager.Console/Batch/BatchManager.cs +++ b/src/SqlBuildManager.Console/Batch/BatchManager.cs @@ -99,7 +99,7 @@ public BatchManager(CommandLineArgs cmdLine, string queryFile, string outputFile log.LogInformation($"Extracting Platinum Dacpac from {cmdLine.DacPacArgs.PlatinumServerSource} : {cmdLine.DacPacArgs.PlatinumDbSource}"); string dacpacName = Path.Combine(cmdLine.RootLoggingPath, cmdLine.DacPacArgs.PlatinumDbSource + ".dacpac"); - if (!DacPacHelper.ExtractDacPac(cmdLine.DacPacArgs.PlatinumDbSource, cmdLine.DacPacArgs.PlatinumServerSource, cmdLine.AuthenticationArgs.AuthenticationType, cmdLine.AuthenticationArgs.UserName, cmdLine.AuthenticationArgs.Password, dacpacName)) + if (!DacPacHelper.ExtractDacPac(cmdLine.DacPacArgs.PlatinumDbSource, cmdLine.DacPacArgs.PlatinumServerSource, cmdLine.AuthenticationArgs.AuthenticationType, cmdLine.AuthenticationArgs.UserName, cmdLine.AuthenticationArgs.Password, dacpacName, cmdLine.TimeoutRetryCount)) { log.LogError($"Error creating the Platinum dacpac from {cmdLine.DacPacArgs.PlatinumServerSource} : {cmdLine.DacPacArgs.PlatinumDbSource}"); } diff --git a/src/SqlBuildManager.Console/Queue/QueueManager.cs b/src/SqlBuildManager.Console/Queue/QueueManager.cs index 5f6bfd97..0e50a557 100644 --- a/src/SqlBuildManager.Console/Queue/QueueManager.cs +++ b/src/SqlBuildManager.Console/Queue/QueueManager.cs @@ -115,6 +115,11 @@ public async Task SendTargetsToQueue(MultiDbData multiDb, ConcurrencyType c { try { + if ( (cType == ConcurrencyType.Tag || cType == ConcurrencyType.MaxPerTag) && multiDb.AsQueryable().Where(m => m.Overrides.Where(o => o.ConcurrencyTag.Length == 0).Any()).Any()) + { + log.LogError($"There are database targets that do not have a concurrency tag. This is required when the Concurrency Type is '{cType.ToString()}'. Please add a concurrency tag to all database targets before sending to the queue."); + return 0; + } log.LogInformation($"Setting up Topic Subscription with Job filter name '{jobName}'"); await RemoveDefaultFilters(); await CleanUpCustomFilters(); @@ -124,7 +129,11 @@ public async Task SendTargetsToQueue(MultiDbData multiDb, ConcurrencyType c //Use bucketing to 1 bucket to get flattened list of targest var concurrencyBuckets = Concurrency.ConcurrencyByType(multiDb, 1, ConcurrencyType.Count); - var messages = CreateMessages(concurrencyBuckets, jobName); + var messages = CreateMessages(concurrencyBuckets, jobName, cType); + if(messages == null) + { + return 0; + } int count = messages.Count(); int sentCount = 0; @@ -185,7 +194,7 @@ public async Task SendTargetsToQueue(MultiDbData multiDb, ConcurrencyType c } } - public List CreateMessages(List)>> buckets, string jobName) + public List CreateMessages(List)>> buckets, string jobName, ConcurrencyType cType) { try { @@ -200,14 +209,39 @@ public List CreateMessages(List 0) + { + msg.SessionId = target.Item2[0].ConcurrencyTag; + + } + else + { + msg.SessionId = target.Item1; + } msg.MessageId = Guid.NewGuid().ToString(); msgs.Add(msg); } @@ -227,9 +261,11 @@ public async Task> GetDatabaseTargetFromQueue(in switch (cType) { case ConcurrencyType.Server: + case ConcurrencyType.Tag: return await GetSessionBasedTargetsFromQueue(1, false); case ConcurrencyType.MaxPerServer: + case ConcurrencyType.MaxPerTag: return await GetSessionBasedTargetsFromQueue(maxMessages, false); case ConcurrencyType.Count: @@ -402,9 +438,12 @@ private CancellationToken GetCancellationToken(int waitMs = 5000) public async Task CompleteMessage(ServiceBusReceivedMessage message) { var t = message.As(); + string concurrency = t.ConcurrencyTag.Length == 0 ? "" : $" [ConcurrencyTag: {t.ConcurrencyTag}]"; try { - log.LogInformation($"Completing {t.ServerName}.{t.DbOverrideSequence[0].OverrideDbTarget} message ID '{message.MessageId}'"); + + log.LogInformation($"Completing {t.ServerName}.{t.DbOverrideSequence[0].OverrideDbTarget}{concurrency} message ID '{message.MessageId}'"); + if (_sessionReceiver != null) { await _sessionReceiver.CompleteMessageAsync(message); @@ -416,20 +455,21 @@ public async Task CompleteMessage(ServiceBusReceivedMessage message) } catch (ServiceBusException sbE) { - log.LogError($"Unable to complete message for {t.ServerName}.{t.DbOverrideSequence[0].OverrideDbTarget}. This may result in duplicate processing: {sbE.Message}"); + log.LogError($"Unable to complete message for {t.ServerName}.{t.DbOverrideSequence[0].OverrideDbTarget}{concurrency}. This may result in duplicate processing: {sbE.Message}"); } catch (Exception exe) { - log.LogError($"Unable to complete message for {t.ServerName}.{t.DbOverrideSequence[0].OverrideDbTarget}. This may result in duplicate processing: {exe.Message}"); + log.LogError($"Unable to complete message for {t.ServerName}.{t.DbOverrideSequence[0].OverrideDbTarget}{concurrency}. This may result in duplicate processing: {exe.Message}"); } return; } public async Task AbandonMessage(ServiceBusReceivedMessage message) { var t = message.As(); + string concurrency = t.ConcurrencyTag.Length == 0 ? "" : $" [ConcurrencyTag: {t.ConcurrencyTag}]"; try { - log.LogInformation($"Abandoning {t.ServerName}.{t.DbOverrideSequence[0].OverrideDbTarget} message ID '{message.MessageId}'"); + log.LogInformation($"Abandoning {t.ServerName}.{t.DbOverrideSequence[0].OverrideDbTarget}{concurrency} message ID '{message.MessageId}'"); if (_sessionReceiver != null) { await _sessionReceiver.AbandonMessageAsync(message); @@ -441,16 +481,17 @@ public async Task AbandonMessage(ServiceBusReceivedMessage message) } catch (Exception exe) { - log.LogError($"Failed to Abandon message for {t.ServerName}.{t.DbOverrideSequence[0].OverrideDbTarget}: {exe.Message}"); + log.LogError($"Failed to Abandon message for {t.ServerName}.{t.DbOverrideSequence[0].OverrideDbTarget}{concurrency}: {exe.Message}"); } return; } public async Task DeadletterMessage(ServiceBusReceivedMessage message) { var t = message.As(); + string concurrency = t.ConcurrencyTag.Length == 0 ? "" : $" [ConcurrencyTag: {t.ConcurrencyTag}]"; try { - log.LogInformation($"Deadlettering {t.ServerName}.{t.DbOverrideSequence[0].OverrideDbTarget} message ID '{message.MessageId}'"); + log.LogInformation($"Deadlettering {t.ServerName}.{t.DbOverrideSequence[0].OverrideDbTarget}{concurrency} message ID '{message.MessageId}'"); if (_sessionReceiver != null) { await _sessionReceiver.DeadLetterMessageAsync(message); @@ -462,16 +503,17 @@ public async Task DeadletterMessage(ServiceBusReceivedMessage message) } catch (Exception exe) { - log.LogError($"Failed to Deadletter message for {t.ServerName}.{t.DbOverrideSequence[0].OverrideDbTarget}: {exe.Message}"); + log.LogError($"Failed to Deadletter message for {t.ServerName}.{t.DbOverrideSequence[0].OverrideDbTarget}{concurrency}: {exe.Message}"); } return; } public async Task RenewMessageLock(ServiceBusReceivedMessage message) { var t = message.As(); + string concurrency = t.ConcurrencyTag.Length == 0 ? "" : $" [ConcurrencyTag: {t.ConcurrencyTag}]"; try { - log.LogDebug($"Renewing message lock on {t.ServerName}.{t.DbOverrideSequence[0].OverrideDbTarget} message ID '{message.MessageId}'"); + log.LogDebug($"Renewing message lock on {t.ServerName}.{t.DbOverrideSequence[0].OverrideDbTarget}{concurrency} message ID '{message.MessageId}'"); if (_sessionReceiver != null) { await _sessionReceiver.RenewSessionLockAsync(); @@ -483,7 +525,7 @@ public async Task RenewMessageLock(ServiceBusReceivedMessage message) } catch (Exception exe) { - log.LogError($"Failed to renew message lock for {t.ServerName}.{t.DbOverrideSequence[0].OverrideDbTarget}: {exe.Message}"); + log.LogError($"Failed to renew message lock for {t.ServerName}.{t.DbOverrideSequence[0].OverrideDbTarget}{concurrency}: {exe.Message}"); } return; } @@ -578,6 +620,8 @@ public async Task SubscriptionIsPreExisting() { case ConcurrencyType.MaxPerServer: case ConcurrencyType.Server: + case ConcurrencyType.MaxPerTag: + case ConcurrencyType.Tag: return await AdminClient.SubscriptionExistsAsync(topicName, topicSessionSubscriptionName); case ConcurrencyType.Count: @@ -611,6 +655,8 @@ private async Task CreateSubscriptions() case ConcurrencyType.MaxPerServer: case ConcurrencyType.Server: + case ConcurrencyType.Tag: + case ConcurrencyType.MaxPerTag: if (!await AdminClient.SubscriptionExistsAsync(topicName, topicSessionSubscriptionName)) { log.LogInformation($"Creating session enabled topic subscripton for `{jobName}'"); @@ -717,16 +763,22 @@ await pollyRetryPolicyForClean.ExecuteAsync(async () => public async Task MonitorServiceBustopic(ConcurrencyType concurrencyType) { SubscriptionRuntimeProperties props; - if (concurrencyType == ConcurrencyType.Count) - { - props = await AdminClient.GetSubscriptionRuntimePropertiesAsync(topicName, topicSubscriptionName, new CancellationToken()); - } - else + switch (concurrencyType) { - props = await AdminClient.GetSubscriptionRuntimePropertiesAsync(topicName, topicSessionSubscriptionName, new CancellationToken()); + case ConcurrencyType.Count: + + props = await AdminClient.GetSubscriptionRuntimePropertiesAsync(topicName, topicSubscriptionName, new CancellationToken()); + break; + case ConcurrencyType.MaxPerTag: + case ConcurrencyType.MaxPerServer: + case ConcurrencyType.Server: + case ConcurrencyType.Tag: + props = await AdminClient.GetSubscriptionRuntimePropertiesAsync(topicName, topicSessionSubscriptionName, new CancellationToken()); + break; + default: + throw new ArgumentException($"Unknow concurrency type of {concurrencyType}"); } return props.ActiveMessageCount; - } public void Dispose() diff --git a/src/SqlBuildManager.Console/Queue/TargetMessage.cs b/src/SqlBuildManager.Console/Queue/TargetMessage.cs index 6cfaae60..d639ddd5 100644 --- a/src/SqlBuildManager.Console/Queue/TargetMessage.cs +++ b/src/SqlBuildManager.Console/Queue/TargetMessage.cs @@ -1,4 +1,5 @@ -using SqlSync.Connection; +using Azure.ResourceManager.Network.Models; +using SqlSync.Connection; using System.Collections.Generic; namespace SqlBuildManager.Console.Queue @@ -7,5 +8,34 @@ public class TargetMessage { public string ServerName { get; set; } public List DbOverrideSequence { get; set; } + + private string concurrencyTag = string.Empty; + public string ConcurrencyTag + { + get + { + if (this.concurrencyTag.Length > 0) + { + return this.concurrencyTag; + } + else + { + if (this.DbOverrideSequence != null && this.DbOverrideSequence.Count > 0) + { + if (this.DbOverrideSequence[0].ConcurrencyTag.Length > 0) + { + return this.DbOverrideSequence[0].ConcurrencyTag; + } + + } + } + return string.Empty; + } + set + { + this.concurrencyTag = value; + } + } + } } diff --git a/src/SqlBuildManager.Console/Threaded/Concurrency.cs b/src/SqlBuildManager.Console/Threaded/Concurrency.cs index bfeedc17..38a182f9 100644 --- a/src/SqlBuildManager.Console/Threaded/Concurrency.cs +++ b/src/SqlBuildManager.Console/Threaded/Concurrency.cs @@ -265,7 +265,14 @@ public static List ConvertBucketsToConfigLines(List(); foreach (var sub in bucket) { - tmp.Add($"{sub.Item1}:{sub.Item2.ToList().Select(d => $"{d.DefaultDbTarget},{d.OverrideDbTarget}").Aggregate((a, b) => $"{a};{b}")}"); + if(sub.Item1.StartsWith("#")) + { + tmp.Add($"{sub.Item2[0].Server}:{sub.Item2.ToList().Select(d => $"{d.DefaultDbTarget},{d.OverrideDbTarget}#{d.ConcurrencyTag}").Aggregate((a, b) => $"{a};{b}")}"); + } + else + { + tmp.Add($"{sub.Item1}:{sub.Item2.ToList().Select(d => $"{d.DefaultDbTarget},{d.OverrideDbTarget}#{d.ConcurrencyTag}").Aggregate((a, b) => $"{a};{b}")}"); + } } bucketStrings.Add(tmp.ToArray()); } diff --git a/src/SqlBuildManager.Console/Threaded/LogMsg.cs b/src/SqlBuildManager.Console/Threaded/LogMsg.cs index 4e7551cf..53c8fa44 100644 --- a/src/SqlBuildManager.Console/Threaded/LogMsg.cs +++ b/src/SqlBuildManager.Console/Threaded/LogMsg.cs @@ -79,6 +79,9 @@ public string SourceDacPac [JsonPropertyName("ComputeHostName")] public string ComputeHostName { get; set; } = string.Empty; + [JsonPropertyName("ConcurrencyTag")] + public string ConcurrencyTag { get; set; } = string.Empty; + [JsonPropertyName("ScriptLog")] public ScriptLogData ScriptLog { get; set; } = null; diff --git a/src/SqlBuildManager.Console/Threaded/ThreadedManager.cs b/src/SqlBuildManager.Console/Threaded/ThreadedManager.cs index 73048ad0..9050739e 100644 --- a/src/SqlBuildManager.Console/Threaded/ThreadedManager.cs +++ b/src/SqlBuildManager.Console/Threaded/ThreadedManager.cs @@ -321,8 +321,10 @@ private async Task ExecuteFromQueue(CommandLineArgs cmdLine, string buildRe foreach (var message in messages) { var target = message.As(); + target.DbOverrideSequence[0].ConcurrencyTag = target.ConcurrencyTag; + ThreadedRunner runner = new ThreadedRunner(target.ServerName, target.DbOverrideSequence, cmdLine, buildRequestedBy, cmdLine.DacPacArgs.ForceCustomDacPac); - var msg = new LogMsg() { DatabaseName = runner.TargetDatabases, ServerName = runner.Server, RunId = ThreadedManager.RunID, Message = "Queuing up thread", LogType = LogType.Message }; + var msg = new LogMsg() { DatabaseName = runner.TargetDatabases, ServerName = runner.Server, RunId = ThreadedManager.RunID, Message = "Queuing up thread", LogType = LogType.Message, ConcurrencyTag = runner.ConcurrencyTag }; threadedLog.WriteToLog(msg); tasks.Add(ProcessThreadedBuildWithQueue(runner, message)); } @@ -367,7 +369,7 @@ private async Task ExecuteFromQueue(CommandLineArgs cmdLine, string buildRe log.LogInformation($"Extracting Platinum Dacpac from {cmdLine.DacPacArgs.PlatinumServerSource} : {cmdLine.DacPacArgs.PlatinumDbSource}"); string dacpacName = Path.Combine(ThreadedManager.rootLoggingPath, cmdLine.DacPacArgs.PlatinumDbSource + ".dacpac"); - if (!DacPacHelper.ExtractDacPac(cmdLine.DacPacArgs.PlatinumDbSource, cmdLine.DacPacArgs.PlatinumServerSource, cmdLine.AuthenticationArgs.AuthenticationType, cmdLine.AuthenticationArgs.UserName, cmdLine.AuthenticationArgs.Password, dacpacName)) + if (!DacPacHelper.ExtractDacPac(cmdLine.DacPacArgs.PlatinumDbSource, cmdLine.DacPacArgs.PlatinumServerSource, cmdLine.AuthenticationArgs.AuthenticationType, cmdLine.AuthenticationArgs.UserName, cmdLine.AuthenticationArgs.Password, dacpacName, cmdLine.DefaultScriptTimeout)) { var m = new LogMsg() { @@ -438,7 +440,7 @@ private async Task ProcessConcurrencyBucket(IEnumerable<(string, List ovr) in bucket) { ThreadedRunner runner = new ThreadedRunner(server, ovr, cmdLine, buildRequestedBy, forceCustomDacpac); - var msg = new LogMsg() { DatabaseName = runner.TargetDatabases, ServerName = runner.Server, RunId = ThreadedManager.RunID, Message = "Queuing up thread", LogType = LogType.Message }; + var msg = new LogMsg() { DatabaseName = runner.TargetDatabases, ServerName = runner.Server, RunId = ThreadedManager.RunID, Message = "Queuing up thread", LogType = LogType.Message, ConcurrencyTag = runner.ConcurrencyTag }; threadedLog.WriteToLog(msg); await ProcessThreadedBuild(runner); } @@ -493,7 +495,7 @@ private async Task ProcessThreadedBuild(ThreadedRunner runner) //SERVER:defaultDb,override string cfgString = String.Format("{0}:{1},{2}", runner.Server, runner.DefaultDatabaseName, runner.TargetDatabases); - msg = new LogMsg() { DatabaseName = runner.TargetDatabases, ServerName = runner.Server, SourceDacPac = runner.DacpacName, RunId = ThreadedManager.RunID, Message = "Starting up thread", LogType = LogType.Message }; + msg = new LogMsg() { DatabaseName = runner.TargetDatabases, ServerName = runner.Server, SourceDacPac = runner.DacpacName, RunId = ThreadedManager.RunID, Message = "Starting up thread", LogType = LogType.Message, ConcurrencyTag = runner.ConcurrencyTag}; threadedLog.WriteToLog(msg); //Run the scripts!! @@ -509,7 +511,9 @@ private async Task ProcessThreadedBuild(ThreadedRunner runner) case (int)RunnerReturn.SuccessWithTrialRolledBack: msg.LogType = LogType.Commit; threadedLog.WriteToLog(msg); - threadedLog.WriteToLog(new LogMsg() { LogType = LogType.SuccessDatabases, Message = cfgString }); + msg.LogType = LogType.SuccessDatabases; + msg.Message = cfgString; + threadedLog.WriteToLog(msg); returnVal = 0; break; @@ -518,7 +522,9 @@ private async Task ProcessThreadedBuild(ThreadedRunner runner) default: msg.LogType = LogType.Error; threadedLog.WriteToLog(msg); - threadedLog.WriteToLog(new LogMsg() { LogType = LogType.FailureDatabases, Message = cfgString }); + msg.LogType = LogType.FailureDatabases; + msg.Message = cfgString; + threadedLog.WriteToLog(msg); hasError = true; break; } diff --git a/src/SqlBuildManager.Console/Threaded/ThreadedQuery.cs b/src/SqlBuildManager.Console/Threaded/ThreadedQuery.cs index 60fd0b2a..1a9b77d1 100644 --- a/src/SqlBuildManager.Console/Threaded/ThreadedQuery.cs +++ b/src/SqlBuildManager.Console/Threaded/ThreadedQuery.cs @@ -15,6 +15,7 @@ using SqlBuildManager.Interfaces.Console; using SqlSync.SqlBuild.Status; using Azure.Messaging.ServiceBus; +using Azure.ResourceManager.Resources.Models; namespace SqlBuildManager.Console.Threaded { @@ -145,8 +146,13 @@ private async Task ExecuteQueryFromQueue(CommandLineArgs cmdLine, Connectio { var target = message.As(); var targetDb = target.DbOverrideSequence[0].OverrideDbTarget; + + if(target.ServerName.StartsWith("#")) + { + target.ServerName = target.DbOverrideSequence[0].Server; + } var runner = new QueryCollectionRunner(target.ServerName, targetDb, query, new List(), ReportType.CSV, tmpOutput,cmdLine.DefaultScriptTimeout, connData); - var msg = new LogMsg() { DatabaseName = targetDb, ServerName = target.ServerName, RunId = this.runId, Message = "Queuing up thread", LogType = LogType.Message }; + var msg = new LogMsg() { DatabaseName = targetDb, ServerName = target.ServerName, RunId = this.runId, Message = "Queuing up thread", LogType = LogType.Message, ConcurrencyTag = target.ConcurrencyTag }; threadLogger.WriteToLog(msg); tasks.Add(ProcessThreadedQueryWithQueue(runner, message)); } @@ -164,19 +170,20 @@ private async Task ExecuteQueryFromQueue(CommandLineArgs cmdLine, Connectio private async Task<(int, string)> ProcessThreadedQueryWithQueue(QueryCollectionRunner runner, ServiceBusReceivedMessage message) { var target = message.As(); - threadLogger.WriteToLog(new LogMsg() { DatabaseName = target.DbOverrideSequence[0].OverrideDbTarget, ServerName = target.ServerName, RunId = this.runId, Message = "Starting up thread", LogType = LogType.Message }); + target.DbOverrideSequence[0].ConcurrencyTag = target.ConcurrencyTag; + threadLogger.WriteToLog(new LogMsg() { DatabaseName = target.DbOverrideSequence[0].OverrideDbTarget, ServerName = target.ServerName, RunId = this.runId, Message = "Starting up thread", LogType = LogType.Message, ConcurrencyTag = target.DbOverrideSequence[0].ConcurrencyTag }); var retVal = await runner.CollectQueryData(); - threadLogger.WriteToLog(new LogMsg() { DatabaseName = target.DbOverrideSequence[0].OverrideDbTarget, ServerName = target.ServerName, RunId = this.runId, Message = "Thread complete", LogType = LogType.Message }); + threadLogger.WriteToLog(new LogMsg() { DatabaseName = target.DbOverrideSequence[0].OverrideDbTarget, ServerName = target.ServerName, RunId = this.runId, Message = "Thread complete", LogType = LogType.Message, ConcurrencyTag = target.DbOverrideSequence[0].ConcurrencyTag }); if (retVal.Item1 == 0) { await qManager.CompleteMessage(message); - threadLogger.WriteToLog(new LogMsg() { DatabaseName = target.DbOverrideSequence[0].OverrideDbTarget, ServerName = target.ServerName, RunId = this.runId, Message = "Thread complete", LogType = LogType.Commit }); + threadLogger.WriteToLog(new LogMsg() { DatabaseName = target.DbOverrideSequence[0].OverrideDbTarget, ServerName = target.ServerName, RunId = this.runId, Message = "Thread complete", LogType = LogType.Commit, ConcurrencyTag = target.DbOverrideSequence[0].ConcurrencyTag }); } else { await qManager.DeadletterMessage(message); - threadLogger.WriteToLog(new LogMsg() { DatabaseName = target.DbOverrideSequence[0].OverrideDbTarget, ServerName = target.ServerName, RunId = this.runId, Message = "Thread complete", LogType = LogType.Commit }); + threadLogger.WriteToLog(new LogMsg() { DatabaseName = target.DbOverrideSequence[0].OverrideDbTarget, ServerName = target.ServerName, RunId = this.runId, Message = "Thread complete", LogType = LogType.Commit, ConcurrencyTag = target.DbOverrideSequence[0].ConcurrencyTag }); } return retVal; } diff --git a/src/SqlBuildManager.Console/Threaded/ThreadedRunner.cs b/src/SqlBuildManager.Console/Threaded/ThreadedRunner.cs index 19eb61da..550412cb 100644 --- a/src/SqlBuildManager.Console/Threaded/ThreadedRunner.cs +++ b/src/SqlBuildManager.Console/Threaded/ThreadedRunner.cs @@ -85,6 +85,19 @@ public bool ForceCustomDacpac get { return forceCustomDacpac; } set { forceCustomDacpac = value; } } + + private string concurrencyTag = string.Empty; + public string ConcurrencyTag + { + get + { + return concurrencyTag; + } + set + { + concurrencyTag = value; + } + } public string DacpacName { get; set; } = string.Empty; private string username = string.Empty; private string password = string.Empty; @@ -95,7 +108,15 @@ public bool ForceCustomDacpac public ThreadedRunner(string serverName, List overrides, CommandLineArgs cmdArgs, string buildRequestedBy, bool forceCustomDacpac) { - server = serverName; + if (serverName.StartsWith("#")) + { + this.server = overrides[0].Server; + } + else + { + this.server = serverName; + } + this.concurrencyTag = overrides[0].ConcurrencyTag; this.overrides = overrides; isTrial = cmdArgs.Trial; this.cmdArgs = cmdArgs; diff --git a/src/SqlBuildManager.Console/Validation.cs b/src/SqlBuildManager.Console/Validation.cs index ffe55e10..716eafef 100644 --- a/src/SqlBuildManager.Console/Validation.cs +++ b/src/SqlBuildManager.Console/Validation.cs @@ -260,8 +260,32 @@ public static int ValidateAndLoadMultiDbData(string multiDbOverrideSettingFileNa log.LogError(error); return (int)ExecutionReturn.MissingTargetDbOverrideSetting; } + + if (!ValidateMultiDatabaseTags(multiData, cmdLine == null? ConcurrencyType.Count : cmdLine.ConcurrencyType)) + { + + error = $"There are database targets that do not have a concurrency tag. This is required when the Concurrency Type is '{cmdLine.ConcurrencyType}'. Please add a concurrency tag to all database targets."; + errorMessages = new string[] { error, "Returning error code: " + (int)ExecutionReturn.MissingOverrideTags }; + log.LogError(error); + return (int)ExecutionReturn.MissingOverrideTags; + } return 0; } + + public static bool ValidateMultiDatabaseTags(MultiDbData multiData, ConcurrencyType concurrencyType) + { + switch(concurrencyType) + { + case ConcurrencyType.Tag: + case ConcurrencyType.MaxPerTag: + if(multiData.AsQueryable().Where(m => m.Overrides.Where(o => o.ConcurrencyTag.Length == 0).Any()).Any()) + { + return false; + } + break; + } + return true; + } private static ConnectionData GetConnDataFromCommandLine(CommandLineArgs cmdLine) { ConnectionData connData = new ConnectionData(cmdLine.Server, cmdLine.Database); diff --git a/src/SqlBuildManager.Console/Worker/Worker.Utility.cs b/src/SqlBuildManager.Console/Worker/Worker.Utility.cs index 629281d1..20de7662 100644 --- a/src/SqlBuildManager.Console/Worker/Worker.Utility.cs +++ b/src/SqlBuildManager.Console/Worker/Worker.Utility.cs @@ -41,7 +41,7 @@ internal static int CreateDacpac(CommandLineArgs cmdLine) Directory.CreateDirectory(path); } - if (!sb.DacPacHelper.ExtractDacPac(cmdLine.Database, cmdLine.Server, cmdLine.AuthenticationArgs.AuthenticationType, cmdLine.AuthenticationArgs.UserName, cmdLine.AuthenticationArgs.Password, fullName)) + if (!sb.DacPacHelper.ExtractDacPac(cmdLine.Database, cmdLine.Server, cmdLine.AuthenticationArgs.AuthenticationType, cmdLine.AuthenticationArgs.UserName, cmdLine.AuthenticationArgs.Password, fullName, cmdLine.DefaultScriptTimeout)) { log.LogError($"Error creating the dacpac from {cmdLine.Server} : {cmdLine.Database}"); return (int)ExecutionReturn.BuildFileExtractionError; @@ -270,7 +270,7 @@ internal static int CreatePackageFromDiff(CommandLineArgs cmdLine) string id = Guid.NewGuid().ToString(); string goldTmp = Path.Combine(path, $"gold-{id}.dacpac"); string targetTmp = Path.Combine(path, $"target-{id}.dacpac"); - if (!sb.DacPacHelper.ExtractDacPac(cmdLine.SynchronizeArgs.GoldDatabase, cmdLine.SynchronizeArgs.GoldServer, cmdLine.AuthenticationArgs.AuthenticationType, cmdLine.AuthenticationArgs.UserName, cmdLine.AuthenticationArgs.Password, goldTmp)) + if (!sb.DacPacHelper.ExtractDacPac(cmdLine.SynchronizeArgs.GoldDatabase, cmdLine.SynchronizeArgs.GoldServer, cmdLine.AuthenticationArgs.AuthenticationType, cmdLine.AuthenticationArgs.UserName, cmdLine.AuthenticationArgs.Password, goldTmp, cmdLine.DefaultScriptTimeout)) { log.LogError($"Error creating the tempprary dacpac from {cmdLine.SynchronizeArgs.GoldServer} : {cmdLine.SynchronizeArgs.GoldDatabase}"); return (int)ExecutionReturn.BuildFileExtractionError; @@ -280,7 +280,7 @@ internal static int CreatePackageFromDiff(CommandLineArgs cmdLine) log.LogInformation($"Temporary DACPAC created from {cmdLine.SynchronizeArgs.GoldServer} : {cmdLine.SynchronizeArgs.GoldDatabase} saved to -- {goldTmp}"); } - if (!sb.DacPacHelper.ExtractDacPac(cmdLine.Database, cmdLine.Server, cmdLine.AuthenticationArgs.AuthenticationType, cmdLine.AuthenticationArgs.UserName, cmdLine.AuthenticationArgs.Password, targetTmp)) + if (!sb.DacPacHelper.ExtractDacPac(cmdLine.Database, cmdLine.Server, cmdLine.AuthenticationArgs.AuthenticationType, cmdLine.AuthenticationArgs.UserName, cmdLine.AuthenticationArgs.Password, targetTmp, cmdLine.DefaultScriptTimeout)) { log.LogError($"Error creating the tempprary dacpac from {cmdLine.Server} : {cmdLine.Database}"); return (int)ExecutionReturn.BuildFileExtractionError; diff --git a/src/SqlBuildManager.Interfaces/Console/Enumerations.cs b/src/SqlBuildManager.Interfaces/Console/Enumerations.cs index b992456d..9d33b1b1 100644 --- a/src/SqlBuildManager.Interfaces/Console/Enumerations.cs +++ b/src/SqlBuildManager.Interfaces/Console/Enumerations.cs @@ -74,7 +74,9 @@ public enum ExecutionReturn [EnumMember] CheckingConnections = 7000, [EnumMember] - DacpacDatabasesInSync = 87598 + DacpacDatabasesInSync = 87598, + [EnumMember] + MissingOverrideTags = 100323 } diff --git a/src/SqlBuildManager.Logging/Threaded/EventHubLogging.cs b/src/SqlBuildManager.Logging/Threaded/EventHubLogging.cs index 5f142d84..148a7f40 100644 --- a/src/SqlBuildManager.Logging/Threaded/EventHubLogging.cs +++ b/src/SqlBuildManager.Logging/Threaded/EventHubLogging.cs @@ -43,7 +43,7 @@ public static void ConfigureEventHubLogger(ILoggerFactory factory) } catch(Exception exe) { - Console.WriteLine("Failed to create EventHub Logger: " + exe.ToString()); + Console.WriteLine("Failed to create EventHub Logger: " + exe.Message); } } private static TokenCredential GetAadTokenCredential() diff --git a/src/SqlSync.Connection.UnitTest/ConnectionHelperTest.cs b/src/SqlSync.Connection.UnitTest/ConnectionHelperTest.cs index c5b9daa7..4ba7df8e 100644 --- a/src/SqlSync.Connection.UnitTest/ConnectionHelperTest.cs +++ b/src/SqlSync.Connection.UnitTest/ConnectionHelperTest.cs @@ -184,10 +184,10 @@ public void GetConnectionTest_FromConnectionDataObj() public void GetTargetDatabaseTest() { List overrides = new List(); - overrides.Add(new DatabaseOverride("default1", "override1")); - overrides.Add(new DatabaseOverride("default2", "override2")); - overrides.Add(new DatabaseOverride("default4", "default4")); - overrides.Add(new DatabaseOverride("MixedCASE", "override5")); + overrides.Add(new DatabaseOverride("server1", "default1", "override1")); + overrides.Add(new DatabaseOverride("server1", "default2", "override2")); + overrides.Add(new DatabaseOverride("server1", "default4", "default4")); + overrides.Add(new DatabaseOverride("server1", "MixedCASE", "override5")); string actual = ConnectionHelper.GetTargetDatabase("default2", overrides); Assert.AreEqual("override2", actual); @@ -206,8 +206,8 @@ public void GetTargetDatabaseTest() public void GetTargetDatabaseTest_EmptyDefault() { List overrides = new List(); - overrides.Add(new DatabaseOverride("default1", "override1")); - overrides.Add(new DatabaseOverride("default2", "override2")); + overrides.Add(new DatabaseOverride("server1", "default1", "override1")); + overrides.Add(new DatabaseOverride("server1", "default2", "override2")); string actual = ConnectionHelper.GetTargetDatabase("", overrides); Assert.AreEqual("override1", actual); @@ -221,8 +221,8 @@ public void GetTargetDatabaseTest_EmptyDefault() public void GetTargetDatabaseTest_NullDefault() { List overrides = new List(); - overrides.Add(new DatabaseOverride("default1", "override1")); - overrides.Add(new DatabaseOverride("default2", "override2")); + overrides.Add(new DatabaseOverride("server1", "default1", "override1")); + overrides.Add(new DatabaseOverride("server1", "default2", "override2")); string actual = ConnectionHelper.GetTargetDatabase(null, overrides); Assert.AreEqual("override1", actual); @@ -236,8 +236,8 @@ public void GetTargetDatabaseTest_NullDefault() public void GetTargetDatabaseTest_NoOverrideFound() { List overrides = new List(); - overrides.Add(new DatabaseOverride("default1", "override1")); - overrides.Add(new DatabaseOverride("default2", "override2")); + overrides.Add(new DatabaseOverride("server1", "default1", "override1")); + overrides.Add(new DatabaseOverride("server1", "default2", "override2")); string actual = ConnectionHelper.GetTargetDatabase("default3", overrides); Assert.AreEqual("default3", actual); @@ -251,10 +251,10 @@ public void GetTargetDatabaseTest_NoOverrideFound() public void GetTargetDatabaseTest_CaseInsensitive() { List overrides = new List(); - overrides.Add(new DatabaseOverride("default1", "override1")); - overrides.Add(new DatabaseOverride("default2", "override2")); - overrides.Add(new DatabaseOverride("default4", "default4")); - overrides.Add(new DatabaseOverride("MixedCASE", "override5")); + overrides.Add(new DatabaseOverride("server1", "default1", "override1")); + overrides.Add(new DatabaseOverride("server1", "default2", "override2")); + overrides.Add(new DatabaseOverride("server1", "default4", "default4")); + overrides.Add(new DatabaseOverride("server1", "MixedCASE", "override5")); string actual = ConnectionHelper.GetTargetDatabase("mixedCaSe", overrides); Assert.AreEqual("override5", actual); @@ -269,10 +269,10 @@ public void GetTargetDatabaseTest_CaseInsensitive() public void ValidateDatabaseOverridesTest_Good() { List overrides = new List(); - overrides.Add(new DatabaseOverride("default1", "override1")); - overrides.Add(new DatabaseOverride("default2", "override2")); - overrides.Add(new DatabaseOverride("default4", "default4")); - overrides.Add(new DatabaseOverride("MixedCASE", "override5")); + overrides.Add(new DatabaseOverride("server1", "default1", "override1")); + overrides.Add(new DatabaseOverride("server1", "default2", "override2")); + overrides.Add(new DatabaseOverride("server1", "default4", "default4")); + overrides.Add(new DatabaseOverride("server1", "MixedCASE", "override5")); bool expected = true; bool actual; @@ -287,10 +287,10 @@ public void ValidateDatabaseOverridesTest_Good() public void ValidateDatabaseOverridesTest_MissingSetting() { List overrides = new List(); - overrides.Add(new DatabaseOverride("default1", "override1")); - overrides.Add(new DatabaseOverride("default2", "override2")); - overrides.Add(new DatabaseOverride("", "")); - overrides.Add(new DatabaseOverride("MixedCASE", "override5")); + overrides.Add(new DatabaseOverride("server1", "default1", "override1")); + overrides.Add(new DatabaseOverride("server1", "default2", "override2")); + overrides.Add(new DatabaseOverride("server1", "", "")); + overrides.Add(new DatabaseOverride("server1", "MixedCASE", "override5")); bool expected = false; bool actual; @@ -318,10 +318,10 @@ public void ValidateDatabaseOverridesTest_NullList() public void ValidateDatabaseOverridesTest_NullOverride() { List overrides = new List(); - overrides.Add(new DatabaseOverride("default1", "override1")); - overrides.Add(new DatabaseOverride("default2", "override2")); + overrides.Add(new DatabaseOverride("server1", "default1", "override1")); + overrides.Add(new DatabaseOverride("server1", "default2", "override2")); overrides.Add(null); - overrides.Add(new DatabaseOverride("MixedCASE", "override5")); + overrides.Add(new DatabaseOverride("server1", "MixedCASE", "override5")); bool expected = false; bool actual; diff --git a/src/SqlSync.SqlBuild.UnitTest/DbOverrideSequenceTest.cs b/src/SqlSync.SqlBuild.UnitTest/DbOverrideSequenceTest.cs index 40779522..30354625 100644 --- a/src/SqlSync.SqlBuild.UnitTest/DbOverrideSequenceTest.cs +++ b/src/SqlSync.SqlBuild.UnitTest/DbOverrideSequenceTest.cs @@ -199,12 +199,12 @@ public void GetOverrideDatabaseNameListTest() { DbOverrides target = new DbOverrides(); List tmp1 = new List(); - tmp1.Add(new DatabaseOverride("default1a", "override1a")); - tmp1.Add(new DatabaseOverride("default1b", "override1b")); + tmp1.Add(new DatabaseOverride("server1", "default1a", "override1a")); + tmp1.Add(new DatabaseOverride("server1", "default1b", "override1b")); List tmp2 = new List(); - tmp2.Add(new DatabaseOverride("default2a", "override2a")); - tmp2.Add(new DatabaseOverride("default2b", "override2b")); + tmp2.Add(new DatabaseOverride("server1", "default2a", "override2a")); + tmp2.Add(new DatabaseOverride("server1", "default2b", "override2b")); target.AddRange(tmp1); target.AddRange(tmp2); @@ -223,8 +223,8 @@ public void AddTest() { DbOverrides target = new DbOverrides(); List tmp1 = new List(); - tmp1.Add(new DatabaseOverride("default1a", "override1a")); - tmp1.Add(new DatabaseOverride("default1b", "override1b")); + tmp1.Add(new DatabaseOverride("server1", "default1a", "override1a")); + tmp1.Add(new DatabaseOverride("server1", "default1b", "override1b")); target.AddRange(tmp1); Assert.AreEqual(2, target.Count); diff --git a/src/SqlSync.SqlBuild.UnitTest/MultiDbHelperTest.cs b/src/SqlSync.SqlBuild.UnitTest/MultiDbHelperTest.cs index 416eb8e7..1ebe3854 100644 --- a/src/SqlSync.SqlBuild.UnitTest/MultiDbHelperTest.cs +++ b/src/SqlSync.SqlBuild.UnitTest/MultiDbHelperTest.cs @@ -416,6 +416,29 @@ public void SerializeMultiDbAsJson_Test() } + [TestMethod()] + [DeploymentItem("SqlSync.SqlBuild.dll")] + public void SerializeMultiDbWithTagAsJson_Test() + { + MultiDbData cfg = new MultiDbData + { + new ServerData() { ServerName = "ServerA", Overrides = new DbOverrides(new DatabaseOverride("ServerA","default1", "override1", "TagA")) }, + new ServerData() { ServerName = "ServerA", Overrides = new DbOverrides(new DatabaseOverride("ServerA","default2", "override2", "TagA")) }, + new ServerData() { ServerName = "ServerA", Overrides = new DbOverrides(new DatabaseOverride("ServerA","default0", "override0", "TagA")) }, + new ServerData() { ServerName = "ServerA", Overrides = new DbOverrides(new DatabaseOverride("ServerA","defaultX", "overrideX", "TagB"), new DatabaseOverride("ServerA","defaultY", "overrideY", "TagB")) }, + new ServerData() { ServerName = "ServerB", Overrides = new DbOverrides( new DatabaseOverride("ServerB","default6", "override6", "TagB")) }, + new ServerData() { ServerName = "ServerB", Overrides = new DbOverrides( new DatabaseOverride("ServerB","default7", "override7", "TagB")) }, + new ServerData() { ServerName = "ServerB", Overrides = new DbOverrides( new DatabaseOverride("ServerB","default5", "override5", "TagB")) }, + }; + + + string actual; + actual = MultiDbHelper.SerializeMultiDbConfigurationToJson(cfg); + var expected = Properties.Resources.serialized_multidb_json_withtag_json; + Assert.AreEqual(expected, actual); + + } + [TestMethod()] [DeploymentItem("SqlSync.SqlBuild.dll")] public void SerializeAndDeserializeMultiDbAsJson_Test() diff --git a/src/SqlSync.SqlBuild.UnitTest/OverrideDataTest.cs b/src/SqlSync.SqlBuild.UnitTest/OverrideDataTest.cs index fc7fd5ca..23828a00 100644 --- a/src/SqlSync.SqlBuild.UnitTest/OverrideDataTest.cs +++ b/src/SqlSync.SqlBuild.UnitTest/OverrideDataTest.cs @@ -70,8 +70,8 @@ public TestContext TestContext public void TargetDatabaseOverridesTest() { List expected = new List(); - expected.Add(new DatabaseOverride("default1", "override1")); - expected.Add(new DatabaseOverride("default2", "override2")); + expected.Add(new DatabaseOverride("server1", "default1", "override1")); + expected.Add(new DatabaseOverride("server1", "default2", "override2")); List actual; OverrideData.TargetDatabaseOverrides = expected; actual = OverrideData.TargetDatabaseOverrides; diff --git a/src/SqlSync.SqlBuild.UnitTest/Properties/Resources.Designer.cs b/src/SqlSync.SqlBuild.UnitTest/Properties/Resources.Designer.cs index 0d9f0793..cbd640b0 100644 --- a/src/SqlSync.SqlBuild.UnitTest/Properties/Resources.Designer.cs +++ b/src/SqlSync.SqlBuild.UnitTest/Properties/Resources.Designer.cs @@ -114,6 +114,14 @@ internal static string serialized_multidb_json } } + internal static string serialized_multidb_json_withtag_json + { + get + { + return ResourceManager.GetString("serialized_multidb_json_withtag_json", resourceCulture); + } + } + /// /// Looks up a localized string similar to <?xml version="1.0" encoding="utf-8"?> diff --git a/src/SqlSync.SqlBuild.UnitTest/Properties/Resources.resx b/src/SqlSync.SqlBuild.UnitTest/Properties/Resources.resx index c2dbbdd9..8dd0af66 100644 --- a/src/SqlSync.SqlBuild.UnitTest/Properties/Resources.resx +++ b/src/SqlSync.SqlBuild.UnitTest/Properties/Resources.resx @@ -133,6 +133,9 @@ ..\Resources\serialized_multidb_json.txt;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089;Windows-1252 + + ..\Resources\serialized_multidb_json_withtag_json.txt;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089;utf-8 + ..\Resources\serialized_multidb_xml.txt;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089;utf-16 diff --git a/src/SqlSync.SqlBuild.UnitTest/Resources/serialized_multidb_json.txt b/src/SqlSync.SqlBuild.UnitTest/Resources/serialized_multidb_json.txt index a74fc7b3..ee67beca 100644 --- a/src/SqlSync.SqlBuild.UnitTest/Resources/serialized_multidb_json.txt +++ b/src/SqlSync.SqlBuild.UnitTest/Resources/serialized_multidb_json.txt @@ -3,6 +3,8 @@ "ServerName": "ServerA", "Overrides": [ { + "ConcurrencyTag": "", + "Server": "ServerA", "DefaultDbTarget": "default1", "OverrideDbTarget": "override1" } @@ -12,6 +14,8 @@ "ServerName": "ServerA", "Overrides": [ { + "ConcurrencyTag": "", + "Server": "ServerA", "DefaultDbTarget": "default2", "OverrideDbTarget": "override2" } @@ -21,6 +25,8 @@ "ServerName": "ServerA", "Overrides": [ { + "ConcurrencyTag": "", + "Server": "ServerA", "DefaultDbTarget": "default0", "OverrideDbTarget": "override0" } @@ -30,10 +36,14 @@ "ServerName": "ServerA", "Overrides": [ { + "ConcurrencyTag": "", + "Server": "ServerA", "DefaultDbTarget": "defaultX", "OverrideDbTarget": "overrideX" }, { + "ConcurrencyTag": "", + "Server": "ServerA", "DefaultDbTarget": "defaultY", "OverrideDbTarget": "overrideY" } @@ -43,6 +53,8 @@ "ServerName": "ServerB", "Overrides": [ { + "ConcurrencyTag": "", + "Server": "ServerB", "DefaultDbTarget": "default6", "OverrideDbTarget": "override6" } @@ -52,6 +64,8 @@ "ServerName": "ServerB", "Overrides": [ { + "ConcurrencyTag": "", + "Server": "ServerB", "DefaultDbTarget": "default7", "OverrideDbTarget": "override7" } @@ -61,6 +75,8 @@ "ServerName": "ServerB", "Overrides": [ { + "ConcurrencyTag": "", + "Server": "ServerB", "DefaultDbTarget": "default5", "OverrideDbTarget": "override5" } diff --git a/src/SqlSync.SqlBuild.UnitTest/Resources/serialized_multidb_json_withtag_json.txt b/src/SqlSync.SqlBuild.UnitTest/Resources/serialized_multidb_json_withtag_json.txt new file mode 100644 index 00000000..641f4cbf --- /dev/null +++ b/src/SqlSync.SqlBuild.UnitTest/Resources/serialized_multidb_json_withtag_json.txt @@ -0,0 +1,85 @@ +[ + { + "ServerName": "ServerA", + "Overrides": [ + { + "ConcurrencyTag": "TagA", + "Server": "ServerA", + "DefaultDbTarget": "default1", + "OverrideDbTarget": "override1" + } + ] + }, + { + "ServerName": "ServerA", + "Overrides": [ + { + "ConcurrencyTag": "TagA", + "Server": "ServerA", + "DefaultDbTarget": "default2", + "OverrideDbTarget": "override2" + } + ] + }, + { + "ServerName": "ServerA", + "Overrides": [ + { + "ConcurrencyTag": "TagA", + "Server": "ServerA", + "DefaultDbTarget": "default0", + "OverrideDbTarget": "override0" + } + ] + }, + { + "ServerName": "ServerA", + "Overrides": [ + { + "ConcurrencyTag": "TagB", + "Server": "ServerA", + "DefaultDbTarget": "defaultX", + "OverrideDbTarget": "overrideX" + }, + { + "ConcurrencyTag": "TagB", + "Server": "ServerA", + "DefaultDbTarget": "defaultY", + "OverrideDbTarget": "overrideY" + } + ] + }, + { + "ServerName": "ServerB", + "Overrides": [ + { + "ConcurrencyTag": "TagB", + "Server": "ServerB", + "DefaultDbTarget": "default6", + "OverrideDbTarget": "override6" + } + ] + }, + { + "ServerName": "ServerB", + "Overrides": [ + { + "ConcurrencyTag": "TagB", + "Server": "ServerB", + "DefaultDbTarget": "default7", + "OverrideDbTarget": "override7" + } + ] + }, + { + "ServerName": "ServerB", + "Overrides": [ + { + "ConcurrencyTag": "TagB", + "Server": "ServerB", + "DefaultDbTarget": "default5", + "OverrideDbTarget": "override5" + } + ] + } +] \ No newline at end of file diff --git a/src/SqlSync.SqlBuild.UnitTest/Resources/serialized_multidb_xml.txt b/src/SqlSync.SqlBuild.UnitTest/Resources/serialized_multidb_xml.txt index 4b6848fa..cd8d5bdc 100644 Binary files a/src/SqlSync.SqlBuild.UnitTest/Resources/serialized_multidb_xml.txt and b/src/SqlSync.SqlBuild.UnitTest/Resources/serialized_multidb_xml.txt differ diff --git a/src/SqlSync.SqlBuild.UnitTest/SqlSync.SqlBuild.UnitTest.csproj b/src/SqlSync.SqlBuild.UnitTest/SqlSync.SqlBuild.UnitTest.csproj index 7e4cb4fc..7ee32987 100644 --- a/src/SqlSync.SqlBuild.UnitTest/SqlSync.SqlBuild.UnitTest.csproj +++ b/src/SqlSync.SqlBuild.UnitTest/SqlSync.SqlBuild.UnitTest.csproj @@ -8,6 +8,7 @@ + @@ -16,6 +17,7 @@ + diff --git a/src/SqlSync.SqlBuild/AdHocQuery/QueryCollector.cs b/src/SqlSync.SqlBuild/AdHocQuery/QueryCollector.cs index 81947807..a4e11587 100644 --- a/src/SqlSync.SqlBuild/AdHocQuery/QueryCollector.cs +++ b/src/SqlSync.SqlBuild/AdHocQuery/QueryCollector.cs @@ -99,6 +99,7 @@ public bool GetQueryResults(string fileName, ReportType reportType, string query foreach (DatabaseOverride ovr in srv.Overrides) { + if (srv.ServerName.StartsWith("#")) srv.ServerName = ovr.Server; db = srv.ServerName + "." + ovr.OverrideDbTarget; QueryCollectionRunner runner = new QueryCollectionRunner(srv.ServerName, ovr.OverrideDbTarget, query, ovr.QueryRowData, reportType, resultsFilePath, scriptTimeout, connData); diff --git a/src/SqlSync.SqlBuild/DacPacHelper.cs b/src/SqlSync.SqlBuild/DacPacHelper.cs index c043f44f..150abfff 100644 --- a/src/SqlSync.SqlBuild/DacPacHelper.cs +++ b/src/SqlSync.SqlBuild/DacPacHelper.cs @@ -15,7 +15,7 @@ public class DacPacHelper { private static ILogger log = SqlBuildManager.Logging.ApplicationLogging.CreateLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType); - public static bool ExtractDacPac(string sourceDatabase, string sourceServer, AuthenticationType authType, string userName, string password, string dacPacFileName) + public static bool ExtractDacPac(string sourceDatabase, string sourceServer, AuthenticationType authType, string userName, string password, string dacPacFileName, int timeouts) { try @@ -25,7 +25,9 @@ public static bool ExtractDacPac(string sourceDatabase, string sourceServer, Aut DacExtractOptions opts = new DacExtractOptions(); opts.IgnoreExtendedProperties = true; opts.IgnoreUserLoginMappings = true; - opts.LongRunningCommandTimeout = 120; + opts.LongRunningCommandTimeout = timeouts; + opts.CommandTimeout = timeouts; + opts.DatabaseLockTimeout = timeouts; ConnectionData connData = new ConnectionData(sourceServer, sourceDatabase); connData.AuthenticationType = authType; @@ -280,7 +282,7 @@ internal static DacpacDeltasStatus CleanDacPacScript(string dacPacGeneratedScrip public static DacpacDeltasStatus UpdateBuildRunDataForDacPacSync(ref SqlBuildRunData runData, string targetServerName, string targetDatabase, AuthenticationType authType, string userName, string password, string workingDirectory, string buildRevision, int defaultScriptTimeout, bool allowObjectDelete) { string tmpDacPacName = Path.Combine(workingDirectory, targetDatabase + ".dacpac"); - if (!ExtractDacPac(targetDatabase, targetServerName, authType, userName, password, tmpDacPacName)) + if (!ExtractDacPac(targetDatabase, targetServerName, authType, userName, password, tmpDacPacName, runData.DefaultScriptTimeout)) { return DacpacDeltasStatus.ExtractionFailure; } @@ -339,7 +341,7 @@ public static DacpacDeltasStatus GetSbmFromDacPac(string rootLoggingPath, string else if (!string.IsNullOrEmpty(database) && !string.IsNullOrEmpty(server)) { string targetDacPac = Path.Combine(workingFolder, database + ".dacpac"); - if (!DacPacHelper.ExtractDacPac(database, server, authType, username, password, targetDacPac)) + if (!DacPacHelper.ExtractDacPac(database, server, authType, username, password, targetDacPac, defaultScriptTimeout)) { log.LogError($"Error extracting dacpac from {database} : {server}"); return DacpacDeltasStatus.ExtractionFailure; @@ -357,7 +359,7 @@ public static DacpacDeltasStatus GetSbmFromDacPac(string rootLoggingPath, string database = serv.Overrides.ElementAt(i).OverrideDbTarget; string targetDacPac = Path.Combine(workingFolder, database + ".dacpac"); - if (!DacPacHelper.ExtractDacPac(database, server, authType, username, password, targetDacPac)) + if (!DacPacHelper.ExtractDacPac(database, server, authType, username, password, targetDacPac, defaultScriptTimeout)) { log.LogError($"Error extracting dacpac from {server} : {database}"); return DacpacDeltasStatus.ExtractionFailure; diff --git a/src/SqlSync.SqlBuild/MultiDb/MultiDbHelper.cs b/src/SqlSync.SqlBuild/MultiDb/MultiDbHelper.cs index 6bd5ee81..aed2342d 100644 --- a/src/SqlSync.SqlBuild/MultiDb/MultiDbHelper.cs +++ b/src/SqlSync.SqlBuild/MultiDb/MultiDbHelper.cs @@ -116,7 +116,10 @@ public static MultiDbData ImportMultiDbTextConfig(string[] fileContents) { var tmp = dbs.Split('#', StringSplitOptions.RemoveEmptyEntries); dbs = tmp[0]; - tag = tmp[1]; + if (tmp.Length > 1) //just in case there is a # with no actual tag value. + { + tag = tmp[1]; + } } var sData = new ServerData(); @@ -321,7 +324,7 @@ public static bool ValidateMultiDatabaseData(MultiDbData dbData) return true; } - + } public class MultiDbConfigurationException : Exception { diff --git a/src/SqlSync.SqlBuild/SqlBuildEnums.cs b/src/SqlSync.SqlBuild/SqlBuildEnums.cs index fdacb532..1e53140b 100644 --- a/src/SqlSync.SqlBuild/SqlBuildEnums.cs +++ b/src/SqlSync.SqlBuild/SqlBuildEnums.cs @@ -128,6 +128,8 @@ public class BuildItemStatus /// Database was updated but required the creation and use of a custom dacpac /// public const string CommittedWithCustomDacpac = "CommittedWithCustomDacpac"; + + public const string FailedWithCustomDacpac = "FailedWithCustomDacpac"; } public class BatchParsing { diff --git a/src/SqlSync.SqlBuild/SqlBuildHelper.cs b/src/SqlSync.SqlBuild/SqlBuildHelper.cs index 75fa7c40..863dbd14 100644 --- a/src/SqlSync.SqlBuild/SqlBuildHelper.cs +++ b/src/SqlSync.SqlBuild/SqlBuildHelper.cs @@ -273,10 +273,36 @@ internal SqlSyncBuildData.BuildRow ProcessBuild(SqlBuildRunData runData, Backgro log.LogWarning($"Timeout encountered. Incrementing retries to {buildRetries}"); } } - + + bool candidateForCustomDacPac = false; + switch (buildResults.FinalStatus) + { + case BuildItemStatus.Committed: + case BuildItemStatus.CommittedWithTimeoutRetries: + case BuildItemStatus.AlreadyInSync: + case BuildItemStatus.TrialRolledBack: + case BuildItemStatus.CommittedWithCustomDacpac: + case BuildItemStatus.Pending: + candidateForCustomDacPac = false; + break; + case BuildItemStatus.FailedDueToScriptTimeout: + case BuildItemStatus.FailedWithCustomDacpac: + candidateForCustomDacPac = false; + log.LogWarning($"Build was not successful. Status is {buildResults.FinalStatus} and Platinum DACPAC name is '{runData.PlatinumDacPacFileName}', and this file exists '{File.Exists(runData.PlatinumDacPacFileName)}' "); + break; + case BuildItemStatus.RolledBack: + case BuildItemStatus.PendingRollBack: + case BuildItemStatus.FailedNoTransaction: + case BuildItemStatus.RolledBackAfterRetries: + candidateForCustomDacPac = true; + break; + default: + log.LogWarning($"Unrecognized Build Item status of {buildResults.FinalStatus}"); + candidateForCustomDacPac = true; + break; + } //Do we need to try to update the target using the Platinum Dacpac? - if (buildResults.FinalStatus != BuildItemStatus.Committed && buildResults.FinalStatus != BuildItemStatus.TrialRolledBack && - !string.IsNullOrEmpty(runData.PlatinumDacPacFileName) && File.Exists(runData.PlatinumDacPacFileName) && !runData.ForceCustomDacpac) + if (candidateForCustomDacPac && !string.IsNullOrEmpty(runData.PlatinumDacPacFileName) && File.Exists(runData.PlatinumDacPacFileName) && !runData.ForceCustomDacpac) { var database = ((SqlSyncBuildData.ScriptRow)filteredScripts[0].Row).Database; string targetDatabase = GetTargetDatabase(database); @@ -291,8 +317,13 @@ internal SqlSyncBuildData.BuildRow ProcessBuild(SqlBuildRunData runData, Backgro if (dacStat.FinalStatus == BuildItemStatus.Committed || dacStat.FinalStatus == BuildItemStatus.CommittedWithTimeoutRetries) { dacStat.FinalStatus = BuildItemStatus.CommittedWithCustomDacpac; + buildResults.FinalStatus = dacStat.FinalStatus; if (BuildCommittedEvent != null) BuildCommittedEvent(this, RunnerReturn.CommittedWithCustomDacpac); + }else + { + dacStat.FinalStatus = BuildItemStatus.FailedWithCustomDacpac; + buildResults.FinalStatus = dacStat.FinalStatus; } } else if (stat == DacpacDeltasStatus.InSync || stat == DacpacDeltasStatus.OnlyPostDeployment) @@ -303,18 +334,31 @@ internal SqlSyncBuildData.BuildRow ProcessBuild(SqlBuildRunData runData, Backgro } } - else if (buildResults.FinalStatus != BuildItemStatus.Committed && buildResults.FinalStatus != BuildItemStatus.Pending) - { - log.LogWarning($"Build was not successful. Status is {buildResults.FinalStatus.ToString()} and Platinum DACPAC name is '{runData.PlatinumDacPacFileName}', and this file exists '{File.Exists(runData.PlatinumDacPacFileName)}' "); - } - - //If a timeout gets here.. need to decide how to label the rollback if (buildResults.FinalStatus == BuildItemStatus.FailedDueToScriptTimeout && buildRetries > 1) //will always be at least 1.. + { buildResults.FinalStatus = BuildItemStatus.RolledBackAfterRetries; + } else if (buildResults.FinalStatus == BuildItemStatus.FailedDueToScriptTimeout) + { buildResults.FinalStatus = BuildItemStatus.RolledBack; + } + + switch (buildResults.FinalStatus) + { + case BuildItemStatus.Committed: + case BuildItemStatus.Pending: + case BuildItemStatus.CommittedWithTimeoutRetries: + case BuildItemStatus.TrialRolledBack: + case BuildItemStatus.AlreadyInSync: + case BuildItemStatus.CommittedWithCustomDacpac: + break; + default: + log.LogWarning($"Build was not successful. Status is {buildResults.FinalStatus} and Platinum DACPAC name is '{runData.PlatinumDacPacFileName}', and this file exists '{File.Exists(runData.PlatinumDacPacFileName)}' "); + break; + + } return buildResults; } diff --git a/src/SqlSync/SqlBuild/MultiDb/MultiDbPage.cs b/src/SqlSync/SqlBuild/MultiDb/MultiDbPage.cs index 58a85bc5..8f113b98 100644 --- a/src/SqlSync/SqlBuild/MultiDb/MultiDbPage.cs +++ b/src/SqlSync/SqlBuild/MultiDb/MultiDbPage.cs @@ -181,7 +181,7 @@ internal List GetServerData() } if (!found) { - DatabaseOverride tmp = new DatabaseOverride(defaultDb.DatabaseName, defaultDb.DatabaseName); + DatabaseOverride tmp = new DatabaseOverride(srvData.ServerName,defaultDb.DatabaseName, defaultDb.DatabaseName); srvData.Overrides.Add(tmp); } diff --git a/src/SqlSync/SqlBuild/MultiDb/MultiDbRunForm.cs b/src/SqlSync/SqlBuild/MultiDb/MultiDbRunForm.cs index 4d919b38..57710b4a 100644 --- a/src/SqlSync/SqlBuild/MultiDb/MultiDbRunForm.cs +++ b/src/SqlSync/SqlBuild/MultiDb/MultiDbRunForm.cs @@ -546,7 +546,7 @@ private void AddNewServerItem(string serverName, DatabaseList dbList) lstSrv.Add(new ServerData() { ServerName = serverName, - Overrides = new DbOverrides() { new DatabaseOverride(defaultDatabases[0], db.DatabaseName) } + Overrides = new DbOverrides() { new DatabaseOverride(serverName, defaultDatabases[0], db.DatabaseName) } }); } ServerData dat = new ServerData();