Skip to content

Commit

Permalink
Merge pull request #251 from Lombiq/issue/WALMA-99
Browse files Browse the repository at this point in the history 
WALMA-99: Flaky UI tests
  • Loading branch information
@dministro
dministro authored Jan 19, 2023
2 parents b7d846b + 2899ef6 commit d8761f7
Show file tree
Hide file tree
Showing 2 changed files with 196 additions and 90 deletions.
20 changes: 20 additions & 0 deletions Lombiq.Tests.UI/Exceptions/DockerFileCopyException.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
using System;

namespace Lombiq.Tests.UI.Exceptions;

public class DockerFileCopyException : Exception
{
public DockerFileCopyException(string message)
: base(message)
{
}

public DockerFileCopyException(string message, Exception innerException)
: base(message, innerException)
{
}

public DockerFileCopyException()
{
}
}
266 changes: 176 additions & 90 deletions Lombiq.Tests.UI/Services/SqlServerManager.cs
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
using CliWrap;
using Lombiq.HelpfulLibraries.Cli;
using Lombiq.HelpfulLibraries.Common.Utilities;
using Lombiq.Tests.UI.Exceptions;
using Lombiq.Tests.UI.Helpers;
using Microsoft.Data.SqlClient;
using Microsoft.IdentityModel.Tokens;
using Microsoft.SqlServer.Management.Common;
using Microsoft.SqlServer.Management.Smo;
using System;
Expand Down Expand Up @@ -41,9 +44,11 @@ public sealed class SqlServerManager : IAsyncDisposable
private const string DbSnapshotName = "Database.bak";

private static readonly PortLeaseManager _portLeaseManager;
private static readonly SemaphoreSlim _semaphore = new(1, 1);

private readonly CliProgram _docker = new("docker");
private readonly SqlServerConfiguration _configuration;

private int _databaseId;
private string _serverName;
private string _databaseName;
Expand Down Expand Up @@ -114,131 +119,167 @@ public async Task TakeSnapshotAsync(
string containerName = null,
bool useCompressionIfAvailable = false)
{
var filePathRemote = GetSnapshotFilePath(snapshotDirectoryPathRemote);
var filePathLocal = GetSnapshotFilePath(snapshotDirectoryPathLocal ?? snapshotDirectoryPathRemote);
var directoryPathLocal =
Path.GetDirectoryName(filePathLocal) ??
throw new InvalidOperationException($"Failed to get the directory path for local path \"{filePathLocal}\".");
DebugHelper.WriteLineTimestamped($"Entering SqlServerManager semaphore in TakeSnapshotAsync().");

FileSystemHelper.EnsureDirectoryExists(directoryPathLocal);
if (File.Exists(filePathLocal)) File.Delete(filePathLocal);
await _semaphore.WaitAsync();
try
{
var filePathRemote = GetSnapshotFilePath(snapshotDirectoryPathRemote);
var filePathLocal = GetSnapshotFilePath(snapshotDirectoryPathLocal ?? snapshotDirectoryPathRemote);
var directoryPathLocal =
Path.GetDirectoryName(filePathLocal) ??
throw new InvalidOperationException($"Failed to get the directory path for local path \"{filePathLocal}\".");

var server = CreateServer();
FileSystemHelper.EnsureDirectoryExists(directoryPathLocal);
if (File.Exists(filePathLocal)) File.Delete(filePathLocal);

KillDatabaseProcesses(server);
var server = CreateServer();

var useCompression =
useCompressionIfAvailable &&
(server.EngineEdition == Edition.EnterpriseOrDeveloper || server.EngineEdition == Edition.Standard);
KillDatabaseProcesses(server);

var backup = new Backup
{
Action = BackupActionType.Database,
CopyOnly = true,
Checksum = true,
Incremental = false,
ContinueAfterError = false,
// We don't need compression for setup snapshots as those backups will be only short-lived and we want them
// to be fast.
CompressionOption = useCompression ? BackupCompressionOptions.On : BackupCompressionOptions.Off,
SkipTapeHeader = true,
UnloadTapeAfter = false,
NoRewind = true,
FormatMedia = true,
Initialize = true,
Database = _databaseName,
};
var useCompression =
useCompressionIfAvailable &&
(server.EngineEdition == Edition.EnterpriseOrDeveloper || server.EngineEdition == Edition.Standard);

var destination = new BackupDeviceItem(filePathRemote, DeviceType.File);
backup.Devices.Add(destination);
// We could use SqlBackupAsync() too but that's not Task-based async, we'd need to subscribe to an event which
// is messy.
backup.SqlBackup(server);
var backup = new Backup
{
Action = BackupActionType.Database,
CopyOnly = true,
Checksum = true,
Incremental = false,
ContinueAfterError = false,
// We don't need compression for setup snapshots as those backups will be only short-lived and we want
// them to be fast.
CompressionOption = useCompression ? BackupCompressionOptions.On : BackupCompressionOptions.Off,
SkipTapeHeader = true,
UnloadTapeAfter = false,
NoRewind = true,
FormatMedia = true,
Initialize = true,
Database = _databaseName,
};

var destination = new BackupDeviceItem(filePathRemote, DeviceType.File);
backup.Devices.Add(destination);
// We could use SqlBackupAsync() too but that's not Task-based async, we'd need to subscribe to an event
// which is messy.
backup.SqlBackup(server);

if (!string.IsNullOrEmpty(containerName))
{
if (File.Exists(filePathLocal)) File.Delete(filePathLocal);

if (!string.IsNullOrEmpty(containerName))
{
if (File.Exists(filePathLocal)) File.Delete(filePathLocal);
await Cli.Wrap("docker")
.WithArguments(new[] { "cp", $"{containerName}:{filePathRemote}", filePathLocal })
.ExecuteAsync();
}

await Cli.Wrap("docker")
.WithArguments(new[] { "cp", $"{containerName}:{filePathRemote}", filePathLocal })
.ExecuteAsync();
if (!File.Exists(filePathLocal))
{
throw filePathLocal == filePathRemote
? new InvalidOperationException($"A file wasn't created at \"{filePathLocal}\".")
: new FileNotFoundException(
$"A file was created at \"{filePathRemote}\" but it doesn't appear at \"{filePathLocal}\". " +
$"Are the two bound together? If you are using Docker, did you set up the local volume?");
}
}

if (!File.Exists(filePathLocal))
finally
{
throw filePathLocal == filePathRemote
? new InvalidOperationException($"A file wasn't created at \"{filePathLocal}\".")
: new FileNotFoundException(
$"A file was created at \"{filePathRemote}\" but it doesn't appear at \"{filePathLocal}\". " +
$"Are the two bound together? If you are using Docker, did you set up the local volume?");
DebugHelper.WriteLineTimestamped($"Exiting SqlServerManager semaphore in TakeSnapshotAsync().");
_semaphore.Release();
}
}

public async Task RestoreSnapshotAsync(
string snapshotDirectoryPathRemote,
string snapshotDirectoryPathLocal,
string containerName)
string containerName,
int maxRetries = 3)
{
if (_isDisposed)
DebugHelper.WriteLineTimestamped($"Entering SqlServerManager semaphore in RestoreSnapshotAsync().");

await _semaphore.WaitAsync();
try
{
throw new InvalidOperationException("This instance was already disposed.");
}
if (_isDisposed)
{
throw new InvalidOperationException("This instance was already disposed.");
}

var server = CreateServer();
var server = CreateServer();

if (!server.Databases.Contains(_databaseName))
{
throw new InvalidOperationException($"The database {_databaseName} doesn't exist. Something may have dropped it.");
}
if (!server.Databases.Contains(_databaseName))
{
throw new InvalidOperationException(
$"The database {_databaseName} doesn't exist. Something may have dropped it.");
}

if (!string.IsNullOrEmpty(containerName))
{
var remote = GetSnapshotFilePath(snapshotDirectoryPathRemote);
var local = GetSnapshotFilePath(snapshotDirectoryPathLocal);
if (!string.IsNullOrEmpty(containerName))
{
var remote = GetSnapshotFilePath(snapshotDirectoryPathRemote);
var local = GetSnapshotFilePath(snapshotDirectoryPathLocal);

// Clean up leftovers.
await DockerExecuteAsync(containerName, "rm", "-f", remote);
// Clean up leftovers.
await DockerExecuteAsync(containerName, "rm", "-f", remote);

// Copy back snapshot.
await _docker.ExecuteAsync(CancellationToken.None, "cp", Path.Combine(local), $"{containerName}:{remote}");
await EnsureDockerFileExistsAsync(local, containerName, remote, maxRetries);

// Reset ownership.
await DockerExecuteAsync(containerName, "bash", "-c", $"chown mssql:root '{remote}'");
}
// Reset ownership.
await DockerExecuteAsync(containerName, "bash", "-c", $"chown mssql:root '{remote}'");
}

KillDatabaseProcesses(server);
KillDatabaseProcesses(server);

var restore = new Restore();
restore.Devices.AddDevice(GetSnapshotFilePath(snapshotDirectoryPathRemote), DeviceType.File);
restore.Database = _databaseName;
restore.ReplaceDatabase = true;
var restore = new Restore();
restore.Devices.AddDevice(GetSnapshotFilePath(snapshotDirectoryPathRemote), DeviceType.File);
restore.Database = _databaseName;
restore.ReplaceDatabase = true;

// Since the DB is restored under a different name this relocation magic needs to happen. Taken from:
// https://stackoverflow.com/a/17547737/220230.
var dataFile = new RelocateFile
{
LogicalFileName = restore.ReadFileList(server).Rows[0][0].ToString(),
PhysicalFileName = server.Databases[_databaseName].FileGroups[0].Files[0].FileName,
};
// Since the DB is restored under a different name this relocation magic needs to happen. Taken from:
// https://stackoverflow.com/a/17547737/220230.
var dataFile = new RelocateFile
{
LogicalFileName = restore.ReadFileList(server).Rows[0][0].ToString(),
PhysicalFileName = server.Databases[_databaseName].FileGroups[0].Files[0].FileName,
};

var logFile = new RelocateFile
{
LogicalFileName = restore.ReadFileList(server).Rows[1][0].ToString(),
PhysicalFileName = server.Databases[_databaseName].LogFiles[0].FileName,
};
var logFile = new RelocateFile
{
LogicalFileName = restore.ReadFileList(server).Rows[1][0].ToString(),
PhysicalFileName = server.Databases[_databaseName].LogFiles[0].FileName,
};

restore.RelocateFiles.Add(dataFile);
restore.RelocateFiles.Add(logFile);
restore.RelocateFiles.Add(dataFile);
restore.RelocateFiles.Add(logFile);

// We're not using SqlRestoreAsync() due to the same reason we're not using SqlBackupAsync().
restore.SqlRestore(server);
// We're not using SqlRestoreAsync() due to the same reason we're not using SqlBackupAsync().
restore.SqlRestore(server);
}
finally
{
DebugHelper.WriteLineTimestamped($"Exiting SqlServerManager semaphore in RestoreSnapshotAsync().");
_semaphore.Release();
}
}

private Task DockerExecuteAsync(string containerName, params object[] command)
private Task DockerExecuteAsync(string containerName, params object[] command) =>
_docker.ExecuteAsync(
CreateArguments(containerName, command),
additionalExceptionText: null,
CancellationToken.None);

private Task<string> DockerExecuteAndGetOutputAsync(string containerName, params object[] command) =>
_docker.ExecuteAndGetOutputAsync(
CreateArguments(containerName, command),
additionalExceptionText: null,
CancellationToken.None);

private static List<object> CreateArguments(string containerName, params object[] command)
{
var arguments = new List<object> { "exec", "-u", 0, containerName };
arguments.AddRange(command);
return _docker.ExecuteAsync(arguments, additionalExceptionText: null, CancellationToken.None);

return arguments;
}

public async ValueTask DisposeAsync()
Expand Down Expand Up @@ -318,4 +359,49 @@ private static string GetSnapshotFilePath(string snapshotDirectoryPath) =>
'/' => $"{snapshotDirectoryPath.TrimEnd('/')}/{DbSnapshotName}",
_ => Path.Combine(Path.GetFullPath(snapshotDirectoryPath), DbSnapshotName),
};

private async Task EnsureDockerFileExistsAsync(string local, string containerName, string remote, int maxRetries)
{
var retryCount = 0;

string result;

do
{
try
{
// Copy back snapshot.
await _docker.ExecuteAsync(
CancellationToken.None,
"cp",
Path.Combine(local),
$"{containerName}:{remote}");

result = await DockerExecuteAndGetOutputAsync(containerName, "ls", Path.GetDirectoryName(remote));

if (result.IsNullOrEmpty())
{
DebugHelper.WriteLineTimestamped($"Attempt {(retryCount + 1).ToTechnicalString()} of copying " +
"snapshot failed. Retrying...");
}
}
catch (InvalidOperationException invalidOperationException)
{
result = string.Empty;
DebugHelper.WriteLineTimestamped($"Attempt {(retryCount + 1).ToTechnicalString()} of copying snapshot" +
$" failed with InvalidOperationException: {invalidOperationException.Message}. Retrying...");
}

retryCount++;

await Task.Delay(1000);
}
while (result.IsNullOrEmpty() && retryCount < maxRetries + 1);

if (result.IsNullOrEmpty())
{
throw new DockerFileCopyException(
$"Failed to copy snapshot file to \"{remote}\" after {maxRetries.ToTechnicalString()} retries.");
}
}
}

0 comments on commit d8761f7

Please sign in to comment.