Skip to content

Commit

Permalink
Added BatchPreStage and BatchCleanup actions to allow for pre-configu…
Browse files Browse the repository at this point in the history 
…ring the batch pool and removing the pool nodes at a later time
  • Loading branch information
@mmckechney
mmckechney committed Apr 26, 2019
1 parent 2f3a6da commit d1af445
Show file tree
Hide file tree
Showing 14 changed files with 4,419 additions and 4,155 deletions.
8 changes: 4 additions & 4 deletions AssemblyVersioning.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@
// associated with an assembly.
//
[assembly: AssemblyCompany("Michael McKechney - www.mckechney.com")]
[assembly: AssemblyCopyright("Copyright © Michael McKechney 2004-2015")]
[assembly: AssemblyTrademark("Copyright © Michael McKechney 2004-2015")]
[assembly: AssemblyCopyright("Copyright © Michael McKechney 2004-2019")]
[assembly: AssemblyTrademark("Copyright © Michael McKechney 2004-2019")]

//
// Version information for an assembly consists of the following four values:
Expand All @@ -27,5 +27,5 @@
// 2) Update the installer version to match the AssemblyVersion below.
// These can be found in SqlBuildManager.Setup -> Organize Your Setup -> General Information
// ** Also, don't forget to update the change_notes.xml and .html files!
[assembly: AssemblyVersion("10.4.1.*")]
[assembly: AssemblyFileVersion("10.4.1")]
[assembly: AssemblyVersion("10.4.2.*")]
[assembly: AssemblyFileVersion("10.4.2")]
1 change: 0 additions & 1 deletion SQLSync.sln
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1346,7 +1346,6 @@ Global
{CC74AAB6-527F-4D03-A32C-E29DE9185B97}.Release|Any CPU.ActiveCfg = Release
{CC74AAB6-527F-4D03-A32C-E29DE9185B97}.Release|Any CPU.Build.0 = Release
{CC74AAB6-527F-4D03-A32C-E29DE9185B97}.Release|Mixed Platforms.ActiveCfg = Release
{CC74AAB6-527F-4D03-A32C-E29DE9185B97}.Release|Mixed Platforms.Build.0 = Release
{CC74AAB6-527F-4D03-A32C-E29DE9185B97}.Release|x64.ActiveCfg = Release
{CC74AAB6-527F-4D03-A32C-E29DE9185B97}.Release|x64.Build.0 = Release
{CC74AAB6-527F-4D03-A32C-E29DE9185B97}.Release|x86.ActiveCfg = Release
Expand Down
2 changes: 1 addition & 1 deletion SqlBuildManager.Console/App.config
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@
</layout>
</appender>
<root>
<level value="DEBUG" />
<level value="INFO" />
<!--<appender-ref ref="EventLogAppender"/>-->
<appender-ref ref="StandardRollingLogFileAppender" />
<appender-ref ref="ColoredConsoleAppender" />
Expand Down
202 changes: 184 additions & 18 deletions SqlBuildManager.Console/Batch/BatchExecution.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,19 +22,24 @@ public class Execution
private CommandLineArgs cmdLine;

// Batch resource settings
private const string PoolIdFormat = "SqlBuildManagerPool";
private string PoolName = "SqlBuildManagerPool";
private const string JobIdFormat = "SqlBuildManagerJob_{0}";

private const string baseTargetFormat = "target_{0}.cfg";

public Execution(CommandLineArgs cmdLine)
{
this.cmdLine = cmdLine;

if (!string.IsNullOrWhiteSpace(cmdLine.BatchArgs.BatchPoolName))
{
this.PoolName = cmdLine.BatchArgs.BatchPoolName;
}
}

public int StartBatch()
{
string jobId, poolId, inputContainerName, outputContainerName;
string jobId, poolId, storageContainerName;
string jobToken = DateTime.Now.ToString("yyyy-MM-dd-HHmm");
if (!string.IsNullOrWhiteSpace(cmdLine.BatchArgs.BatchJobName))
{
Expand All @@ -45,16 +50,14 @@ public int StartBatch()
cmdLine.BatchArgs.BatchJobName = cmdLine.BatchArgs.BatchJobName.Substring(0, 47);
}
jobId = cmdLine.BatchArgs.BatchJobName + "-" + jobToken;
poolId = PoolIdFormat;
inputContainerName = cmdLine.BatchArgs.BatchJobName;
outputContainerName = cmdLine.BatchArgs.BatchJobName;
poolId = PoolName;
storageContainerName = cmdLine.BatchArgs.BatchJobName;
}
else
{
jobId = string.Format(JobIdFormat, jobToken);
poolId = PoolIdFormat;
inputContainerName = jobToken;
outputContainerName = jobToken;
poolId = PoolName;
storageContainerName = jobToken;
}

int? myExitCode = 0;
Expand Down Expand Up @@ -116,8 +119,8 @@ public int StartBatch()
timer.Start();

//Get storage ready
CloudBlobClient blobClient = CreateStorageAndGetBlobClient(cmdLine.BatchArgs.StorageAccountName, cmdLine.BatchArgs.StorageAccountKey, inputContainerName);
string containerSasToken = GetOutputContainerSasUrl(blobClient, outputContainerName, false);
CloudBlobClient blobClient = CreateStorageAndGetBlobClient(cmdLine.BatchArgs.StorageAccountName, cmdLine.BatchArgs.StorageAccountKey, storageContainerName);
string containerSasToken = GetOutputContainerSasUrl(blobClient, storageContainerName, false);
if (log.IsDebugEnabled)
{
log.DebugFormat($"Output write SAS token: {containerSasToken}");
Expand Down Expand Up @@ -172,7 +175,7 @@ public int StartBatch()

foreach (string filePath in inputFilePaths)
{
inputFiles.Add(UploadFileToContainer(blobClient, inputContainerName, filePath));
inputFiles.Add(UploadFileToContainer(blobClient, storageContainerName, filePath));
}

//Create the individual command lines for each node
Expand Down Expand Up @@ -282,9 +285,9 @@ public int StartBatch()
}

log.Info("Consolidating log files");
ConsolidateLogFiles(blobClient, outputContainerName, inputFilePaths);
ConsolidateLogFiles(blobClient, storageContainerName, inputFilePaths);

string readOnlySasToken = GetOutputContainerSasUrl(blobClient, outputContainerName, true);
string readOnlySasToken = GetOutputContainerSasUrl(blobClient, storageContainerName, true);
log.InfoFormat("Log files can be found here: {0}", readOnlySasToken);
log.Info("The read-only SAS token URL is valid for 7 days.");
log.Info("You can download \"Azure Storage Explorer\" from here: https://azure.microsoft.com/en-us/features/storage-explorer/");
Expand Down Expand Up @@ -317,6 +320,8 @@ public int StartBatch()

}



private void ConsolidateLogFiles(CloudBlobClient blobClient, string outputContainerName, List<string> workerFiles)
{
workerFiles.AddRange(new string[] { "dacpac", "sbm", "sql","execution.log" });
Expand Down Expand Up @@ -409,12 +414,14 @@ private bool CreateBatchPool(BatchClient batchClient, string poolId, int nodeCou
}
catch (Exception exe)
{
log.Warn("Unable to get information on existing pool", exe);
log.WarnFormat($"Unable to get information on existing pool. {exe.ToString()}");
return false;
}
}
else
{
log.Error($"Received unexpected pool status: {be.RequestInformation?.BatchError.Code}");
log.Error("Unable to proceed!");
throw; // Any other exception is unexpected
}
}
Expand Down Expand Up @@ -549,10 +556,7 @@ private static string GetOutputContainerSasUrl(CloudBlobClient blobClient, strin
string containerSasToken = container.GetSharedAccessSignature(sasConstraints);
return String.Format("{0}{1}", container.Uri, containerSasToken);
}
private static bool CreateBatchPool(CommandLineArgs cmdLine)
{
return true;
}

/// <summary>
/// Gets a string array for all of the target DB override settings
/// </summary>
Expand Down Expand Up @@ -593,7 +597,169 @@ internal IList<string[]> SplitLoadEvenly(string[] allTargetConfig, int batchNode
}


public int PreStageBatchNodes()
{
string[] errorMessages;
log.Info("Validating batch pre-stage command parameters");
int tmpReturn = Validation.ValidateBatchPreStageArguments(ref cmdLine, out errorMessages);
if (tmpReturn != 0)
{
foreach (var msg in errorMessages)
{
log.Error(msg);
}
return tmpReturn;
}

log.Info("Creating Batch pool nodes ");

// Get a Batch client using account creds, and create the pool
BatchSharedKeyCredentials cred = new BatchSharedKeyCredentials(cmdLine.BatchArgs.BatchAccountUrl, cmdLine.BatchArgs.BatchAccountName, cmdLine.BatchArgs.BatchAccountKey);
var batchClient = BatchClient.Open(cred);

// Create a Batch pool, VM configuration, Windows Server image
bool success = CreateBatchPool(batchClient, PoolName, cmdLine.BatchArgs.BatchNodeCount, cmdLine.BatchArgs.BatchVmSize);

if (cmdLine.BatchArgs.PollBatchPoolStatus)
{
log.Info($"Waiting for pool {this.PoolName} to be created");
while (true)
{
var status = batchClient.PoolOperations.GetPool(PoolName, null, null);
if (status.AllocationState != AllocationState.Steady)
{
log.Info($"Pool status: {status.AllocationState}");
System.Threading.Thread.Sleep(10000);
}
else
{
break;
}
}

log.Info("Waiting for all nodes to complete creation");
while (true)
{
var status = batchClient.PoolOperations.GetPool(PoolName, null, null);
var nodes = status.ListComputeNodes(null, null);
if (nodes.Where(n => n.State != ComputeNodeState.Idle).Any())
{
if (log.IsDebugEnabled)
{
nodes.ForEachAsync(n =>
{
log.Info($"Node '{n.Id}' state = '{n.State}'");
});
}
else
{
var grp = nodes.GroupBy(n => n.State);
grp.ToList().ForEach(g =>
{
var cnt = g.First().State;
log.Info($"State: {g.Count().ToString().PadLeft(2, '0')} nodes at {g.First().State}");
});
}

System.Threading.Thread.Sleep(15000);
}
else
{

nodes.ToList().ForEach(n =>
{
log.Info($"Node '{n.Id}' state = '{n.State}'");
});
log.Info("All nodes ready for work!");
break;
}
}
}
else
{
log.Info($"PollBatchPoolStatus set to 'false'. Pool is being created, but you will not get updates on the status. If you want to attach to pool to get status, you rerun the same command with /PollBatchPoolStatus=true at any time.");
}

if(success)
{
log.Info($"Batch pool of {cmdLine.BatchArgs.BatchNodeCount} nodes created for account {cmdLine.BatchArgs.BatchAccountName} ");
return 0;
}
else
{
log.Error("There was a problem creating the Batch pool. Please see prior log messages");
return -65643;
}

}

public int CleanUpBatchNodes()
{
string[] errorMessages;
log.Info("Validating batch pre-stage command parameters");
int tmpReturn = Validation.ValidateBatchCleanUpArguments(ref cmdLine, out errorMessages);
if (tmpReturn != 0)
{
foreach (var msg in errorMessages)
{
log.Error(msg);
}
return tmpReturn;
}

log.Info("Cleaning up (deleting) Batch pool nodes ");

try
{
// Get a Batch client using account creds, and create the pool
BatchSharedKeyCredentials cred = new BatchSharedKeyCredentials(cmdLine.BatchArgs.BatchAccountUrl, cmdLine.BatchArgs.BatchAccountName, cmdLine.BatchArgs.BatchAccountKey);
var batchClient = BatchClient.Open(cred);


log.Info($"Deleting batch pool {this.PoolName} from Batch account {cmdLine.BatchArgs.BatchAccountName}");

if (cmdLine.BatchArgs.PollBatchPoolStatus)
{
//Delete the pool
batchClient.PoolOperations.DeletePool(this.PoolName);

if (cmdLine.BatchArgs.PollBatchPoolStatus)
{
var status = batchClient.PoolOperations.GetPool(PoolName, null, null);
var count = batchClient.PoolOperations.ListComputeNodes(PoolName, null, null).Count();
while (status != null && status.State == PoolState.Deleting && count > 0)
{
count = batchClient.PoolOperations.ListComputeNodes(PoolName, null, null).Count();
log.Info($"Pool delete in progress. Current node count: {count}");
System.Threading.Thread.Sleep(15000);

}

log.Info($"Pool {this.PoolName} successfully deleted");
}
return 0;
}
else
{
log.Info($"PollBatchPoolStatus set to 'false'. Pool is being delted, but you will not get updates on the status. If you want to attach to pool to get status, you rerun the same command with /PollBatchPoolStatus=true at any time.");
return 0;
}

}
catch (Exception exe)
{
if (exe.Message.ToLower().IndexOf("notfound") > -1)
{
log.Info($"The {this.PoolName} pool was not found. Was it already deleted?");
return 0;
}
else
{
log.Error($"Error encountered trying to delete pool {this.PoolName} from Batch account {cmdLine.BatchArgs.BatchAccountName}.\r\n{exe.ToString()}");
return 42345346;
}
}
}

}
}
17 changes: 16 additions & 1 deletion SqlBuildManager.Console/ConsoleHelp2.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ Action value options (/Action=<value>)
Threaded For updating multiple databases simultaneously from the current machine
Remote For updating multiple databases simultaneously using remote execution servers to spread the processing
Batch For updating multiple databases simultaneously using Azure batch services
BatchPreStage Pre-stage the Azure Batch VM nodes
Package Creates an SBM package from an SBX configuraiton file and scripts
PolicyCheck Performs a script policy check on the specified SBM package
GetHash Calculates the SHA-1 hash fingerprint value for the SBM package (scripts + run order)
Expand Down Expand Up @@ -41,7 +42,7 @@ Azure Batch Execution (/Action=Batch)
/PlatinumDacpac="<filename>" Name of the dacpac containing the platinum schema
/PackageName="<filename>" Name of the .sbm or .sbx file to execute
/RootLoggingPath="<directory>" Directory to save execution logs
/DeleteBatchPool=(true|false) Whether or not to delete the batch pool servers after an execution (default is true)
/DeleteBatchPool=(true|false) Whether or not to delete the batch pool servers after an execution (default is false)
/DeleteBatchJob=(true|false) Whether or not to delete the batch job after an execution (default is true)
/BatchNodeCount="##" Number of nodes to provision to run the batch job (default is 10)
/BatchVmSize="<size>" Size key for VM size required (see https://docs.microsoft.com/en-us/azure/virtual-machines/windows/sizes-general) [can also be set via BatchVmSize app settings key]
Expand All @@ -52,6 +53,20 @@ Azure Batch Execution (/Action=Batch)
/StorageAccountKey="<storage acct key>" Account Key for the storage account [can also be set via StorageAccountKey app settings key]
/BatchJobName="<name>" [Optional] User friendly name for the job. This will also be the container name for the stored logs. Any disallowed URL characters will be removed

Azure Batch Pre-Stage Batch nodes (/Action=BatchPreStage)
/BatchNodeCount="##" Number of nodes to provision to run the batch job (default is 10)
/BatchVmSize="<size>" Size key for VM size required (see https://docs.microsoft.com/en-us/azure/virtual-machines/windows/sizes-general) [can also be set via BatchVmSize app settings key]
/BatchAccountName="<batch acct name>" String name of the Azure Batch account [can also be set via BatchAccountName app settings key]
/BatchAccountKey="<batch acct key>" Account Key for the Azure Batch account [can also be set via BatchAccountKey app settings key]
/BatchAccountUrl="<batch acct url>" URL for the Azure Batch account [can also be set via BatchAccountUrl app settings key]
/PollBatchPoolStatus=(true|false) Whether or not you want to get updated status (true, default) or fire and forget (false)

Azure Batch Clean Up (delete) nodes (/Action=BatchCleanUp)
/BatchAccountName="<batch acct name>" String name of the Azure Batch account [can also be set via BatchAccountName app settings key]
/BatchAccountKey="<batch acct key>" Account Key for the Azure Batch account [can also be set via BatchAccountKey app settings key]
/BatchAccountUrl="<batch acct url>" URL for the Azure Batch account [can also be set via BatchAccountUrl app settings key]
/PollBatchPoolStatus=(true|false) Whether or not you want to get updated status (true, default) or fire and forget (false)

Remote Execution settings (/Action=Remote)
/RemoteServers=("<filename>"|derive|azure) Pointer to file that contains the list of remote execution servers,
"derive" to parse servers from DB list, azure to use Azure PaaS instances
Expand Down
Loading

0 comments on commit d1af445

Please sign in to comment.