Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

OSOE-497: Better exception message during visual verification testing on baseline-actual dimension mismatch #248

Merged
merged 10 commits into from
Jan 6, 2023
Merged

OSOE-497: Better exception message during visual verification testing on baseline-actual dimension mismatch #248

Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
f6265f9
Better assertion failure message on mismatch with the baseline image'…
Piedone Jan 3, 2023
50ee58a
Typos
Piedone Jan 3, 2023
f9852d4
Details exception message on why a a mismatch between the dimensions …
Piedone Jan 3, 2023
b43d44e
Using a word that Check Spelling also understands
Piedone Jan 3, 2023
83f98df
Refactoring how verification images are saved
Piedone Jan 5, 2023
07a47cc
Code styling
Piedone Jan 5, 2023
598af4e
Refactoring how images are saved to the failure dump
Piedone Jan 5, 2023
ba1b6ae
Merge remote-tracking branch 'origin/dev' into issue/OSOE-497
Piedone Jan 6, 2023
2b32ccd
Removing unneeded string concat
Piedone Jan 6, 2023
be7b87c
Removing unneeded line break
Piedone Jan 6, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
196 changes: 89 additions & 107 deletions Lombiq.Tests.UI/Extensions/VisualVerificationUITestContextExtensions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ public static class VisualVerificationUITestContextExtensions
/// </summary>
/// <param name="context">The <see cref="UITestContext"/> in which the extension is executed on.</param>
/// <param name="pixelErrorPercentageThreshold">Maximum acceptable pixel error in percentage.</param>
/// <param name="regionOfInterest">Region of interest. Can be null.</param>
/// <param name="regionOfInterest">Region of interest. Can be null.</param>
/// <param name="configurator">Action callback to configure the behavior. Can be null.</param>
/// <exception cref="VisualVerificationBaselineImageNotFoundException">
/// If no baseline image found under project path.
Expand All @@ -59,7 +59,7 @@ public static void AssertVisualVerificationApproved(
/// <param name="context">The <see cref="UITestContext"/> in which the extension is executed on.</param>
/// <param name="elementSelector">Selector for the target element.</param>
/// <param name="pixelErrorPercentageThreshold">Maximum acceptable pixel error in percentage.</param>
/// <param name="regionOfInterest">Region of interest. Can be null.</param>
/// <param name="regionOfInterest">Region of interest. Can be null.</param>
/// <param name="configurator">Action callback to configure the behavior. Can be null.</param>
/// <exception cref="VisualVerificationBaselineImageNotFoundException">
/// If no baseline image found under project path.
Expand Down Expand Up @@ -108,7 +108,7 @@ public static void AssertVisualVerificationApproved(
/// <param name="context">The <see cref="UITestContext"/> in which the extension is executed on.</param>
/// <param name="element">Target element.</param>
/// <param name="pixelErrorPercentageThreshold">Maximum acceptable pixel error in percentage.</param>
/// <param name="regionOfInterest">Region of interest. Can be null.</param>
/// <param name="regionOfInterest">Region of interest. Can be null.</param>
/// <param name="configurator">Action callback to configure the behavior. Can be null.</param>
/// <exception cref="VisualVerificationBaselineImageNotFoundException">
/// If no baseline image found under project path.
Expand Down Expand Up @@ -140,7 +140,7 @@ public static void AssertVisualVerificationApproved(
/// <param name="context">The <see cref="UITestContext"/> in which the extension is executed on.</param>
/// <param name="baseline">The baseline image.</param>
/// <param name="pixelErrorPercentageThreshold">Maximum acceptable pixel error in percentage.</param>
/// <param name="regionOfInterest">Region of interest. Can be null.</param>
/// <param name="regionOfInterest">Region of interest. Can be null.</param>
/// <param name="configurator">Action callback to configure the behavior. Can be null.</param>
public static void AssertVisualVerification(
this UITestContext context,
Expand All @@ -163,7 +163,7 @@ public static void AssertVisualVerification(
/// <param name="elementSelector">Selector for the target element.</param>
/// <param name="baseline">The baseline image.</param>
/// <param name="pixelErrorPercentageThreshold">Maximum acceptable pixel error in percentage.</param>
/// <param name="regionOfInterest">Region of interest. Can be null.</param>
/// <param name="regionOfInterest">Region of interest. Can be null.</param>
/// <param name="configurator">Action callback to configure the behavior. Can be null.</param>
public static void AssertVisualVerification(
this UITestContext context,
Expand Down Expand Up @@ -211,7 +211,7 @@ public static void AssertVisualVerification(
/// <param name="element">The target element.</param>
/// <param name="baseline">The baseline image.</param>
/// <param name="pixelErrorPercentageThreshold">Maximum acceptable pixel error in percentage.</param>
/// <param name="regionOfInterest">Region of interest. Can be null.</param>
/// <param name="regionOfInterest">Region of interest. Can be null.</param>
/// <param name="configurator">Action callback to configure the behavior. Can be null.</param>
public static void AssertVisualVerification(
this UITestContext context,
Expand Down Expand Up @@ -337,8 +337,7 @@ private static void AssertVisualVerificationApproved(
diff => comparator(approvedContext, diff),
regionOfInterest,
cfg => cfg.WithFileNamePrefix(approvedContext.BaselineFileName)
.WithFileNameSuffix(string.Empty),
approvedContext);
.WithFileNameSuffix(string.Empty));
}
finally
{
Expand All @@ -353,17 +352,18 @@ private static void SaveSuggestedImage(
string baselineFileName)
{
using var suggestedImage = context.TakeElementScreenshot(element);

suggestedImage.Save(baselineImagePath, new PngEncoder());
context.AddImageToFailureDump(baselineFileName + ".png", suggestedImage);
}

// Appending suggested baseline image to failure dump too.
private static void AddImageToFailureDump(
this UITestContext context,
string fileName,
Image image) =>
context.AppendFailureDump(
Path.Combine(
VisualVerificationMatchNames.DumpFolderName,
$"{baselineFileName}.png"),
suggestedImage.Clone(),
Path.Combine(VisualVerificationMatchNames.DumpFolderName, fileName),
image.Clone(),
messageIfExists: HintFailureDumpItemAlreadyExists);
}

private static void AssertVisualVerification(
this UITestContext context,
Expand All @@ -385,8 +385,7 @@ private static void AssertVisualVerification(
Image baseline,
Action<ICompareResult> comparator,
Rectangle? regionOfInterest = null,
Action<VisualMatchConfiguration> configurator = null,
VisualVerificationMatchApprovedContext approvedContext = null)
Action<VisualMatchConfiguration> configurator = null)
{
var configuration = new VisualMatchConfiguration();
configurator?.Invoke(configuration);
Expand All @@ -400,22 +399,32 @@ private static void AssertVisualVerification(
// We take a screenshot of the element area. This will be compared to a baseline image.
using var elementImageOriginal = context.TakeElementScreenshot(element).ShouldNotBeNull();
dministro marked this conversation as resolved.
Show resolved Hide resolved

// Checking the size of captured image.
try
{
elementImageOriginal.Width
.ShouldBeGreaterThanOrEqualTo(cropRegion.Left + cropRegion.Width);
elementImageOriginal.Height
.ShouldBeGreaterThanOrEqualTo(cropRegion.Top + cropRegion.Height);
}
catch
{
if (approvedContext != null)
var originalElementScreenshotFileName =
new[]
{
context.SaveSuggestedImage(element, approvedContext.BaselineImagePath, approvedContext.BaselineFileName);
configuration.FileNamePrefix,
VisualVerificationMatchNames.ElementImageFileName,
configuration.FileNameSuffix,
}
.JoinNotNullOrEmpty("-");

throw;
// Checking the dimensions of captured image. This needs to happen before any other comparisons, because that
// can only be done on images with the same dimensions.
var cropWidth = cropRegion.Left + cropRegion.Width;
var cropHeight = cropRegion.Top + cropRegion.Height;
if (elementImageOriginal.Width < cropWidth || elementImageOriginal.Height < cropHeight)
{
var cropRegionName = regionOfInterest == null ? "baseline image" : "selected region of interest";
var message = $"The dimensions of the captured element ({elementImageOriginal.Width.ToTechnicalString()}" +
$"px x {elementImageOriginal.Height.ToTechnicalString()}px) are smaller than the dimensions of the " +
$"{cropRegionName} ({cropWidth.ToTechnicalString()}px x {cropHeight.ToTechnicalString()}px). This " +
"can happen if due to a change in the app the captured element got smaller than before, or if the " +
$"{cropRegionName} is mistakenly too large. The suggested baseline image with a screenshot of the " +
"captured element was saved to the failure dump. Compare this with the original image used by the " +
"test and if suitable, use it as the baseline going forward.";
context.AddImageToFailureDump(originalElementScreenshotFileName, elementImageOriginal);

throw new VisualVerificationAssertionException(message);
}

using var baselineImageOriginal = baseline.Clone();
Expand All @@ -436,12 +445,11 @@ private static void AssertVisualVerification(
.CalcDiffImage(elementImageCropped)
.ShouldNotBeNull();

// Now we are one step away from the end. Here we create a statistical summary of the differences
// between the captured and the baseline image. In the end, the lower values are better.
// You can read more about how these statistical calculations are created here:
// Now we are one step away from the end. Here we create a statistical summary of the differences between the
// captured and the baseline image. In the end, the lower values are better. You can read more about how these
// statistical calculations are created here:
// https://github.com/Codeuctivity/ImageSharp.Compare/blob/2.0.46/ImageSharpCompare/ImageSharpCompare.cs#L218.
var diff = baselineImageCropped
.CompareTo(elementImageCropped);
var diff = baselineImageCropped.CompareTo(elementImageCropped);

try
{
Expand All @@ -451,88 +459,62 @@ private static void AssertVisualVerification(
{
// Here we append all the relevant items to the failure dump to help the investigation.
// The full-page screenshot
context.AppendFailureDump(
Path.Combine(
VisualVerificationMatchNames.DumpFolderName,
new[]
{
configuration.FileNamePrefix,
VisualVerificationMatchNames.FullScreenImageFileName,
configuration.FileNameSuffix,
}
.JoinNotNullOrEmpty("-")),
fullScreenImage.Clone(),
messageIfExists: HintFailureDumpItemAlreadyExists);
context.AddImageToFailureDump(
new[]
{
configuration.FileNamePrefix,
VisualVerificationMatchNames.FullScreenImageFileName,
configuration.FileNameSuffix,
}
.JoinNotNullOrEmpty("-"),
fullScreenImage);

// The original element screenshot
context.AppendFailureDump(
Path.Combine(
VisualVerificationMatchNames.DumpFolderName,
new[]
{
configuration.FileNamePrefix,
VisualVerificationMatchNames.ElementImageFileName,
configuration.FileNameSuffix,
}
.JoinNotNullOrEmpty("-")),
elementImageOriginal.Clone(),
messageIfExists: HintFailureDumpItemAlreadyExists);
context.AddImageToFailureDump(originalElementScreenshotFileName, elementImageOriginal);

// The original baseline image
context.AppendFailureDump(
Path.Combine(
VisualVerificationMatchNames.DumpFolderName,
new[]
{
configuration.FileNamePrefix,
VisualVerificationMatchNames.BaselineImageFileName,
configuration.FileNameSuffix,
}
.JoinNotNullOrEmpty("-")),
baselineImageOriginal.Clone(),
messageIfExists: HintFailureDumpItemAlreadyExists);
context.AddImageToFailureDump(
new[]
{
configuration.FileNamePrefix,
VisualVerificationMatchNames.BaselineImageFileName,
configuration.FileNameSuffix,
}
.JoinNotNullOrEmpty("-"),
baselineImageOriginal);

// The cropped baseline image
context.AppendFailureDump(
Path.Combine(
VisualVerificationMatchNames.DumpFolderName,
new[]
{
configuration.FileNamePrefix,
VisualVerificationMatchNames.CroppedBaselineImageFileName,
configuration.FileNameSuffix,
}
.JoinNotNullOrEmpty("-")),
baselineImageCropped.Clone(),
messageIfExists: HintFailureDumpItemAlreadyExists);
context.AddImageToFailureDump(
new[]
{
configuration.FileNamePrefix,
VisualVerificationMatchNames.CroppedBaselineImageFileName,
configuration.FileNameSuffix,
}
.JoinNotNullOrEmpty("-"),
baselineImageCropped);

// The cropped element image
context.AppendFailureDump(
Path.Combine(
VisualVerificationMatchNames.DumpFolderName,
new[]
{
configuration.FileNamePrefix,
VisualVerificationMatchNames.CroppedElementImageFileName,
configuration.FileNameSuffix,
}
.JoinNotNullOrEmpty("-")),
elementImageCropped.Clone(),
messageIfExists: HintFailureDumpItemAlreadyExists);
context.AddImageToFailureDump(
new[]
{
configuration.FileNamePrefix,
VisualVerificationMatchNames.CroppedElementImageFileName,
configuration.FileNameSuffix,
}
.JoinNotNullOrEmpty("-"),
elementImageCropped);

// The diff image
context.AppendFailureDump(
Path.Combine(
VisualVerificationMatchNames.DumpFolderName,
new[]
{
configuration.FileNamePrefix,
VisualVerificationMatchNames.DiffImageFileName,
configuration.FileNameSuffix,
}
.JoinNotNullOrEmpty("-")),
diffImage.Clone(),
messageIfExists: HintFailureDumpItemAlreadyExists);
context.AddImageToFailureDump(
new[]
{
configuration.FileNamePrefix,
VisualVerificationMatchNames.DiffImageFileName,
configuration.FileNameSuffix,
}
.JoinNotNullOrEmpty("-"),
diffImage);

// The diff stats
context.AppendFailureDump(
Expand Down