Message retrieval update (#108)

* Better deadletter handling
mmckechney · Apr 28, 2021 · 1d102c8 · 1d102c8
1 parent 31fa8c6
commit 1d102c8
Show file tree

Hide file tree

Showing 12 changed files with 324 additions and 39 deletions.
diff --git a/docs/change_notes.md b/docs/change_notes.md
@@ -5,6 +5,7 @@
 ### Version 13.0.1
 
 - *FIXED:* Updated distribution algorithm for `--concurrencytype` of `Server` and `MaxPerServer` when number of Batch nodes is very close to the number of SQL Server targets. Was yielding less than the number of nodes.  
+- *FIXED:* Updated Service Bus message retrieval to better manage when messages not matching the job name are in large quantity
 
 ### Version 13.0.0
 

diff --git a/src/SqlBuildManager.Console.ExternalTest/CliTests.cs b/src/SqlBuildManager.Console.ExternalTest/CliTests.cs
@@ -63,6 +63,7 @@ private void StdErrorReader()
         private string linuxSettingsFilePath;
         private string settingsFileKeyPath;
 
+        private string logFile;
         [TestInitialize]
         public void ConfigureProcessInfo()
         {
@@ -77,6 +78,12 @@ public void ConfigureProcessInfo()
             bool ds;
             (ds, this.cmdLine)= Cryptography.DecryptSensitiveFields(cmdLine);
             this.overrideFileContents = File.ReadAllLines(this.overrideFilePath).ToList();
+
+        }
+        [TestCleanup]
+        public void CleanUp()
+        {
+
         }
 
         #region Helpers
@@ -138,7 +145,7 @@ private string CreateDacpac(CommandLineArgs cmdLine, string server, string datab
         }
         private static string GetUniqueBatchJobName()
         {
-            string name = DateTime.Now.ToString("yyyy-MM-dd-HH-mm-ss") + "-" + Guid.NewGuid().ToString().Replace("-", "").Substring(0, 6);
+            string name = DateTime.Now.ToString("yyyy-MM-dd-HH-mm-ss") + "-" + Guid.NewGuid().ToString().ToLower().Replace("-", "").Substring(0, 6);
             return name;
         }
         #endregion
@@ -669,8 +676,9 @@ public void BatchQuery_UpdateFail(string batchMethod, string settingsFile)
         [DataRow("run", "TestConfig/settingsfile-windows-queue.json")]
         [DataRow("run", "TestConfig/settingsfile-linux-queue.json")]
         [DataTestMethod]
-        public void Batch_Queue_SBMSource_Success(string batchMethod, string settingsFile)
+        public void Batch_Queue_SBMSource_ByServer_Success(string batchMethod, string settingsFile)
         {
+            settingsFile = Path.GetFullPath(settingsFile);
             string sbmFileName = Path.GetFullPath("SimpleSelect.sbm");
             if (!File.Exists(sbmFileName))
             {
@@ -708,16 +716,205 @@ public void Batch_Queue_SBMSource_Success(string batchMethod, string settingsFil
             result = val.Result;
 
             Assert.AreEqual(0, result, StandardExecutionErrorMessage());
-            //TODO: Get output to examine
-            //Assert.IsTrue(this.output.Contains("Completed Successfully"), "This test was should have worked");
-            //if (batchMethod == "run")
-            //{
-            //    Assert.IsTrue(this.output.Contains($"Batch complete"), $"Should indicate that this was run as a batch job");
-            //}
-            //if (batchMethod == "runthreaded")
-            //{
-            //    Assert.IsTrue(this.output.Contains($"Total number of targets: {this.overrideFileContents.Count()}"), $"Should have run against a {this.overrideFileContents.Count()} databases");
-            //}
+
+        }
+
+        [DataRow("runthreaded", "TestConfig/settingsfile-windows-queue.json")]
+        [DataRow("run", "TestConfig/settingsfile-windows-queue.json")]
+        [DataRow("run", "TestConfig/settingsfile-linux-queue.json")]
+        [DataTestMethod]
+        public void Batch_Queue_SBMSource_MaxPerserver_Success(string batchMethod, string settingsFile)
+        {
+            settingsFile = Path.GetFullPath(settingsFile);
+            string sbmFileName = Path.GetFullPath("SimpleSelect.sbm");
+            if (!File.Exists(sbmFileName))
+            {
+                File.WriteAllBytes(sbmFileName, Properties.Resources.SimpleSelect);
+            }
+            string jobName = GetUniqueBatchJobName();
+
+            var args = new string[]{
+                "batch", "enqueue",
+                "--settingsfile", settingsFile,
+                "--settingsfilekey", this.settingsFileKeyPath,
+                "--override" , this.overrideFilePath,
+                "--concurrencytype",  ConcurrencyType.MaxPerServer.ToString(),
+                "--jobname", jobName};
+
+            RootCommand rootCommand = CommandLineConfig.SetUp();
+            Task<int> val = rootCommand.InvokeAsync(args);
+            val.Wait();
+            var result = val.Result;
+
+            Assert.AreEqual(0, result, StandardExecutionErrorMessage());
+
+            args = new string[]{
+            "batch",  batchMethod,
+            "--settingsfile", settingsFile,
+            "--settingsfilekey", this.settingsFileKeyPath,
+            "--override", this.overrideFilePath,
+            "--packagename", sbmFileName,
+            "--concurrencytype", ConcurrencyType.Server.ToString(),
+            "--concurrency", "5",
+            "--jobname", jobName };
+
+            val = rootCommand.InvokeAsync(args);
+            val.Wait();
+            result = val.Result;
+
+            Assert.AreEqual(0, result, StandardExecutionErrorMessage());
+
+        }
+
+        [DataRow("runthreaded", "TestConfig/settingsfile-windows-queue.json")]
+        [DataRow("run", "TestConfig/settingsfile-windows-queue.json")]
+        [DataRow("run", "TestConfig/settingsfile-linux-queue.json")]
+        [DataTestMethod]
+        public void Batch_Queue_SBMSource_Count_Success(string batchMethod, string settingsFile)
+        {
+            settingsFile = Path.GetFullPath(settingsFile);
+            string sbmFileName = Path.GetFullPath("SimpleSelect.sbm");
+            if (!File.Exists(sbmFileName))
+            {
+                File.WriteAllBytes(sbmFileName, Properties.Resources.SimpleSelect);
+            }
+            string jobName = GetUniqueBatchJobName();
+
+            var args = new string[]{
+                "batch", "enqueue",
+                "--settingsfile", settingsFile,
+                "--settingsfilekey", this.settingsFileKeyPath,
+                "--override" , this.overrideFilePath,
+                "--concurrencytype",  ConcurrencyType.Count.ToString(),
+                "--jobname", jobName};
+
+            RootCommand rootCommand = CommandLineConfig.SetUp();
+            Task<int> val = rootCommand.InvokeAsync(args);
+            val.Wait();
+            var result = val.Result;
+
+            Assert.AreEqual(0, result, StandardExecutionErrorMessage());
+
+            args = new string[]{
+            "batch",  batchMethod,
+            "--settingsfile", settingsFile,
+            "--settingsfilekey", this.settingsFileKeyPath,
+            "--override", this.overrideFilePath,
+            "--packagename", sbmFileName,
+            "--concurrencytype", ConcurrencyType.Server.ToString(),
+            "--concurrency", "5",
+            "--jobname", jobName };
+
+            val = rootCommand.InvokeAsync(args);
+            val.Wait();
+            result = val.Result;
+
+            Assert.AreEqual(0, result, StandardExecutionErrorMessage());
+
+        }
+
+        [DataRow("runthreaded", "TestConfig/settingsfile-windows-queue.json")]
+        [DataRow("run", "TestConfig/settingsfile-windows-queue.json")]
+        [DataRow("run", "TestConfig/settingsfile-linux-queue.json")]
+        [DataTestMethod]
+        public void Batch_Queue_PlatinumDbSource_Success(string batchMethod, string settingsFile)
+        {
+            settingsFile = Path.GetFullPath(settingsFile);
+            int removeCount = 1;
+            string server, database;
+            string firstOverride = this.overrideFileContents.First();
+            (server, database) = DatabaseHelper.ExtractServerAndDbFromLine(firstOverride);
+
+            string minusFirst = Path.GetFullPath("TestConfig/minusFirst.cfg");
+            File.WriteAllLines(minusFirst, DatabaseHelper.ModifyTargetList(this.overrideFileContents, removeCount));
+
+            string jobName = GetUniqueBatchJobName();
+
+            var args = new string[]{
+                "batch", "enqueue",
+                "--settingsfile", settingsFile,
+                "--settingsfilekey", this.settingsFileKeyPath,
+                "--override" , minusFirst,
+                "--concurrencytype",  ConcurrencyType.Count.ToString(),
+                "--jobname", jobName};
+
+            RootCommand rootCommand = CommandLineConfig.SetUp();
+            Task<int> val = rootCommand.InvokeAsync(args);
+            val.Wait();
+            var result = val.Result;
+
+            Assert.AreEqual(0, result, StandardExecutionErrorMessage());
+
+            args = new string[]{
+                "batch",  batchMethod,
+                "--settingsfile", settingsFile,
+                "--settingsfilekey", this.settingsFileKeyPath,
+                "--override", minusFirst,
+                "--platinumdbsource", database,
+                "--platinumserversource", server,
+                "--concurrencytype", ConcurrencyType.Server.ToString(),
+                "--concurrency", "5",
+                "--jobname", jobName };
+
+            val = rootCommand.InvokeAsync(args);
+            val.Wait();
+            result = val.Result;
+
+            Assert.AreEqual(0, result, StandardExecutionErrorMessage());
+        }
+
+        [DataRow("runthreaded", "TestConfig/settingsfile-windows-queue.json")]
+        [DataRow("run", "TestConfig/settingsfile-windows-queue.json")]
+        [DataRow("run", "TestConfig/settingsfile-linux-queue.json")]
+        [DataTestMethod]
+        public void Batch_Queue_DacpacSource_Success(string batchMethod, string settingsFile)
+        {
+            settingsFile = Path.GetFullPath(settingsFile);
+            int removeCount = 1;
+            string server, database;
+            string firstOverride = this.overrideFileContents.First();
+            (server, database) = DatabaseHelper.ExtractServerAndDbFromLine(firstOverride);
+
+            string minusFirst = Path.GetFullPath("TestConfig/minusFirst.cfg");
+            File.WriteAllLines(minusFirst, DatabaseHelper.ModifyTargetList(this.overrideFileContents, removeCount));
+
+            DatabaseHelper.CreateRandomTable(this.cmdLine, firstOverride);
+
+            string dacpacName = CreateDacpac(this.cmdLine, server, database);
+            Assert.IsNotNull(dacpacName, $"There was a problem creating the dacpac for this test\r\n{StandardExecutionErrorMessage()}");
+
+           string jobName = GetUniqueBatchJobName();
+
+            var args = new string[]{
+                "batch", "enqueue",
+                "--settingsfile", settingsFile,
+                "--settingsfilekey", this.settingsFileKeyPath,
+                "--override" , minusFirst,
+                "--concurrencytype",  ConcurrencyType.Count.ToString(),
+                "--jobname", jobName};
+
+            RootCommand rootCommand = CommandLineConfig.SetUp();
+            Task<int> val = rootCommand.InvokeAsync(args);
+            val.Wait();
+            var result = val.Result;
+
+            Assert.AreEqual(0, result, StandardExecutionErrorMessage());
+
+            args = new string[]{
+                "batch",  batchMethod,
+                "--settingsfile", settingsFile,
+                "--settingsfilekey", this.settingsFileKeyPath,
+                "--override", minusFirst,
+                "--platinumdacpac", dacpacName,
+                "--concurrencytype", ConcurrencyType.Server.ToString(),
+                "--concurrency", "5",
+                "--jobname", jobName };
+
+            val = rootCommand.InvokeAsync(args);
+            val.Wait();
+            result = val.Result;
+
+            Assert.AreEqual(0, result, StandardExecutionErrorMessage());
         }
 
 

diff --git a/src/SqlBuildManager.Console.ExternalTest/SqlBuildManager.Console.ExternalTest.csproj b/src/SqlBuildManager.Console.ExternalTest/SqlBuildManager.Console.ExternalTest.csproj
@@ -24,6 +24,14 @@
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </Content>
   </ItemGroup>
+  <ItemGroup>
+    <Content Remove="..\TestConfig\sbm-linux-11.4.0.zip" />
+    <Content Remove="..\TestConfig\sbm-linux-12.1.0.zip" />
+    <Content Remove="..\TestConfig\sbm-linux-12.2.0.zip" />
+    <Content Remove="..\TestConfig\sbm-windows-11.4.0.zip" />
+    <Content Remove="..\TestConfig\sbm-windows-12.1.0.zip" />
+    <Content Remove="..\TestConfig\sbm-windows-12.2.0.zip" />
+  </ItemGroup>
   <ItemGroup>
     <Compile Update="Properties\Resources.Designer.cs">
       <DesignTime>True</DesignTime>

diff --git a/src/SqlBuildManager.Console.UnitTest/ConcurrencyTest.cs b/src/SqlBuildManager.Console.UnitTest/ConcurrencyTest.cs
@@ -40,7 +40,7 @@ internal static (string, MultiDbData) GetMultiDbData()
         }
         internal static (string, MultiDbData) CreateDefinedMultiDbData(int serverCount, int[] dbCount)
         {
-            if(serverCount != dbCount.Length)
+            if (serverCount != dbCount.Length)
             {
                 return ("", null);
             }
@@ -78,11 +78,11 @@ internal static (string, MultiDbData) CreateRandomizedMultiDbData(int serverCoun
             Random rnd = new Random();
             StringBuilder sb = new StringBuilder();
             matrix = new int[serverCount];
-            for (int s=0;s< serverCount;s++)
+            for (int s = 0; s < serverCount; s++)
             {
                 var dbCount = rnd.Next(minDbCount, maxDbCount + 1);
                 matrix[s] = dbCount;
-                for(int d = 0;d<dbCount;d++)
+                for (int d = 0; d < dbCount; d++)
                 {
                     sb.AppendLine($"server{s}:default,database{d}");
                 }
@@ -155,14 +155,16 @@ public void MatchServersToFixedBucket()
                     }
                 }
             }
-            catch(OutOfMemoryException)
+            catch (OutOfMemoryException)
             {
                 //GitHub actions sometimes will run out of memory running this test!
             }
-            
+
         }
 
-        [DataRow(3, 8, new int[] { 92, 225, 126, 135, 266, 186, 280, 115 })]
+
+       // [DataRow(39, 47, new int[] { 175, 179, 203, 92, 153, 230, 154, 166, 247, 213, 39, 207, 192, 208, 42, 211, 212, 35, 282, 66, 82, 45, 94, 72, 124, 212, 118, 235, 263, 138, 30, 239, 99, 271, 114, 189, 25, 80, 31, 217, 255, 192, 81, 40, 84, 244, 178 })] //Actual:<38>
+        [DataRow( 3,  8, new int[] { 92, 225, 126, 135, 266, 186, 280, 115 })]
         [DataRow(26, 27, new int[] { 554, 436, 194, 441, 382, 440, 337, 242, 85, 449, 513, 426, 475, 151, 507, 460, 138, 425, 529, 120, 262, 117, 123, 391, 344, 260, 119 })] //Actual:<23>
         [DataRow(32, 38, new int[] { 218, 532, 396, 63, 227, 207, 185, 106, 556, 453, 528, 476, 512, 395, 73, 487, 121, 75, 450, 560, 456, 199, 488, 413, 311, 439, 132, 405, 448, 238, 266, 101, 368, 84, 133, 171, 31, 276 })] //Actual:<30>
         [DataRow(48, 52, new int[] { 155, 365, 406, 341, 92, 116, 294, 268, 495, 239, 260, 250, 214, 101, 190, 212, 319, 277, 137, 316, 199, 428, 198, 353, 166, 408, 239, 45, 71, 458, 231, 140, 129, 117, 451, 211, 168, 320, 378, 448, 337, 161, 149, 99, 178, 198, 43, 151, 131, 211, 407, 361 })] // Actual:<46>.

diff --git a/src/SqlBuildManager.Console/Batch/BatchExecution.cs b/src/SqlBuildManager.Console/Batch/BatchExecution.cs
@@ -529,15 +529,14 @@ private int ValidateBatchArgs(CommandLineArgs cmdLine, BatchType batchType)
             string jobToken = DateTime.Now.ToString("yyyy-MM-dd-HHmm-ss-fff");
             if (!string.IsNullOrWhiteSpace(cmdLine.BatchArgs.BatchJobName))
             {
-                cmdLine.BatchArgs.BatchJobName = Regex.Replace(cmdLine.BatchArgs.BatchJobName, "[^a-zA-Z0-9]", "");
-                cmdLine.BatchArgs.BatchJobName = cmdLine.BatchArgs.BatchJobName.ToLower();
-                if (cmdLine.BatchArgs.BatchJobName.Length > 47)
+                if (cmdLine.BatchArgs.BatchJobName.Length < 3 || cmdLine.BatchArgs.BatchJobName.Length > 41 || !Regex.IsMatch(cmdLine.BatchArgs.BatchJobName, @"^[a-z0-9]+(-[a-z0-9]+)*$"))
                 {
-                    cmdLine.BatchArgs.BatchJobName = cmdLine.BatchArgs.BatchJobName.Substring(0, 47);
+                    throw new ArgumentException("The batch job name must be  lower case, between 3 and 41 characters in length, and the only special character allowed are dashes '-'");
                 }
-                jobId = cmdLine.BatchArgs.BatchJobName + "-" + jobToken;
+
+                jobId = cmdLine.BatchArgs.BatchJobName;
                 poolId = PoolName;
-                storageContainerName = cmdLine.BatchArgs.BatchJobName;
+                storageContainerName = cmdLine.BatchArgs.BatchJobName + "-" + jobToken; ;
             }
             else
             {

diff --git a/src/SqlBuildManager.Console/Program.cs b/src/SqlBuildManager.Console/Program.cs
@@ -805,6 +805,12 @@ internal async static Task<int> QueueOverrideTargets(CommandLineArgs cmdLine)
                 log.LogError("A --servicebusconnection value is required. Please include this in either the settings file content or as a specific command option");
                 return 9839;
             }
+            (int ret, string msg) = Validation.ValidateBatchjobName(cmdLine.BatchArgs.BatchJobName);
+            if(ret != 0)
+            {
+                log.LogError(msg);
+                return ret;
+            }
 
             int tmpValReturn = Validation.ValidateAndLoadMultiDbData(cmdLine.MultiDbRunConfigFileName, cmdLine, out MultiDbData multiData, out string[] errorMessages);
             if (tmpValReturn != 0)