Skip to content

Commit

Permalink
Screenshot tests (#130)
Browse files Browse the repository at this point in the history 
* #9 Add snapshot testing library

* #9 Create script to boot test simulators

* #9 Create the UI test plan

* #9 Create shared schemes for test targets

* #9 Disable split view for UI tests

* #9 Fix fastlane dependencies

* #9 Add snapshot testing to the application

* #9 assert screenshots

* #9 fix swipe gestures on iPad

* #9 Fix accessing items in session verification screen

* #9 Workaround for flaky unit test

* #9 Specify scheme for alpha build

* #9 Add reference screenshots

* Update python script path and check assets for png check

* Update script path

* Use static timezone for simulator time

* Fix build after SwiftFormat

* Add changelog

* Upload failed screenshots artifact

* Always upload artifacts

* Update boot simulator script

* Update simulator overridden time

* Install pytz before tests

* Get time from Ruby script

* Disable SwiftUI animation when running UI tests

* Update screenshots after animation setting

* Include reference images in the artifact

* Update matching precision

* Update image matching precision & revert artifact content

* Include Xcode result in the artifact

* Update test output directory

* Disable gradient on splash screen for tests

* Tap next button explicitly

* Wait a bit before checking alert

* Wait 1 second

* Run SwiftFormat on project

* Ignore temporary screenshots

* Fix most of the PR remarks

* Fix conflicts

* Bump Python version to 3

* Update reference screenshots for authentication screens

* Update SwiftFormat

* Fix flakey session verification test.

* Update scheme.

Co-authored-by: Doug <[email protected]>
  • Loading branch information
@ismailgulek @pixlwave
ismailgulek and pixlwave authored Aug 11, 2022
1 parent 207cbde commit 2cb6dc1
Show file tree
Hide file tree
Showing 127 changed files with 944 additions and 83 deletions.
1 change: 1 addition & 0 deletions .gitattributes
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
UITests/Sources/__Snapshots__/** filter=lfs diff=lfs merge=lfs -text
16 changes: 16 additions & 0 deletions .github/workflows/tests.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,14 @@ jobs:
cancel-in-progress: true

steps:
- name: Setup Xcode version
uses: maxim-lobanov/[email protected]
with:
xcode-version: '13.4'

- uses: actions/checkout@v3
with:
lfs: true

- uses: actions/cache@v3
with:
Expand All @@ -33,6 +40,15 @@ jobs:
- name: Run tests
run: bundle exec fastlane tests

- name: Archive artifacts
uses: actions/upload-artifact@v3
if: always()
with:
name: test-output
path: fastlane/test_output
retention-days: 7
if-no-files-found: ignore

- name: Upload coverage to Codecov
uses: codecov/codecov-action@v3
11 changes: 11 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,3 +38,14 @@ Tools/Scripts/element-android
## macOS Files
.DS_Store
._*

## Temporary Screenshots
# ignore all
/UITests/Sources/__Snapshots__/Application/*
# but keep the references
!/UITests/Sources/__Snapshots__/Application/15-5-de-DE-iPad-9th-generation.*.png
!/UITests/Sources/__Snapshots__/Application/15-5-fr-FR-iPad-9th-generation.*.png
!/UITests/Sources/__Snapshots__/Application/15-5-en-GB-iPad-9th-generation.*.png
!/UITests/Sources/__Snapshots__/Application/15-5-de-DE-iPhone-13-Pro-Max.*.png
!/UITests/Sources/__Snapshots__/Application/15-5-fr-FR-iPhone-13-Pro-Max.*.png
!/UITests/Sources/__Snapshots__/Application/15-5-en-GB-iPhone-13-Pro-Max.*.png
4 changes: 2 additions & 2 deletions Dangerfile.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ if hasChangedViews {
}

// Check for pngs on resources
let hasPngs = !editedFiles.filter { $0.lowercased().hasSuffix(".png") }.isEmpty
let hasPngs = !editedFiles.filter { $0.lowercased().contains(".xcassets") && $0.lowercased().hasSuffix(".png") }.isEmpty
if hasPngs {
warn("You seem to have made changes to some images. Please consider using an SVG or PDF.")
warn("You seem to have made changes to some resource images. Please consider using an SVG or PDF.")
}
62 changes: 54 additions & 8 deletions ElementX.xcodeproj/project.pbxproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,7 @@
46562110EE202E580A5FFD9C /* RoomScreenViewModelTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 93CF7B19FFCF8EFBE0A8696A /* RoomScreenViewModelTests.swift */; };
4669804D0369FBED4E8625D1 /* ToastViewPresenter.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4470B8CB654B097D807AA713 /* ToastViewPresenter.swift */; };
490E606044B18985055FF690 /* SettingsUITests.swift in Sources */ = {isa = PBXBuildFile; fileRef = E3E29F98CF0E960689A410E3 /* SettingsUITests.swift */; };
492274DA6691EE985C2FCCAA /* SnapshotTesting in Frameworks */ = {isa = PBXBuildFile; productRef = 21C83087604B154AA30E9A8F /* SnapshotTesting */; };
499A26EB06C97E48C27A2DB9 /* BuildSettings.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3F87116470221880017CF522 /* BuildSettings.swift */; };
49E9B99CB6A275C7744351F0 /* LoginViewModel.swift in Sources */ = {isa = PBXBuildFile; fileRef = F2D58333B377888012740101 /* LoginViewModel.swift */; };
49F2E7DD8CAACE09CEECE3E6 /* SeparatorRoomTimelineView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6390A6DC140CA3D6865A66FF /* SeparatorRoomTimelineView.swift */; };
Expand Down Expand Up @@ -265,6 +266,7 @@
D6417E5A799C3C7F14F9EC0A /* SessionVerificationViewModelProtocol.swift in Sources */ = {isa = PBXBuildFile; fileRef = B3069ADED46D063202FE7698 /* SessionVerificationViewModelProtocol.swift */; };
D826154612415D2A3BB6EBF3 /* ListTableViewAdapter.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4E854E7CF531DAC5CBEBDC75 /* ListTableViewAdapter.swift */; };
D8359F67AF3A83516E9083C1 /* MockUserSession.swift in Sources */ = {isa = PBXBuildFile; fileRef = A4756C5A8C8649AD6C10C615 /* MockUserSession.swift */; };
D85D4FA590305180B4A41795 /* Tests.swift in Sources */ = {isa = PBXBuildFile; fileRef = BB3073CCD77D906B330BC1D6 /* Tests.swift */; };
D8CFF02C2730EE5BC4F17ABF /* ElementToggleStyle.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0960A7F5C1B0B6679BDF26F9 /* ElementToggleStyle.swift */; };
D94F664677C380A3CAB8D7F6 /* ActivityIndicatorPresenter.swift in Sources */ = {isa = PBXBuildFile; fileRef = 68706A66BBA04268F7747A2F /* ActivityIndicatorPresenter.swift */; };
DCB781BD227CA958809AFADF /* Coordinator.swift in Sources */ = {isa = PBXBuildFile; fileRef = 95CC95CD75B688E946438165 /* Coordinator.swift */; };
Expand Down Expand Up @@ -295,6 +297,7 @@
FA9C427FFB11B1AA2DCC5602 /* RoomProxyProtocol.swift in Sources */ = {isa = PBXBuildFile; fileRef = 47111410B6E659A697D472B5 /* RoomProxyProtocol.swift */; };
FC6B7436C3A5B3D0565227D5 /* ActivityIndicatorView.swift in Sources */ = {isa = PBXBuildFile; fileRef = AF05352F28D4E7336228E9F4 /* ActivityIndicatorView.swift */; };
FCB640C576292BEAF7FA3B2E /* SplashViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = C9F395A2E917115C7AAF7F34 /* SplashViewController.swift */; };
FE4593FC2A02AAF92E089565 /* ElementAnimations.swift in Sources */ = {isa = PBXBuildFile; fileRef = EF1593DD87F974F8509BB619 /* ElementAnimations.swift */; };
FE79E2BCCF69E8BF4D21E15A /* RoomMessageFactory.swift in Sources */ = {isa = PBXBuildFile; fileRef = FA154570F693D93513E584C1 /* RoomMessageFactory.swift */; };
FFD3E4FF948E06C7585317FC /* TimelineStyler.swift in Sources */ = {isa = PBXBuildFile; fileRef = 892E29C98C4E8182C9037F84 /* TimelineStyler.swift */; };
/* End PBXBuildFile section */
Expand Down Expand Up @@ -536,6 +539,7 @@
8C37FB986891D90BEAA93EAE /* UserSessionStore.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = UserSessionStore.swift; sourceTree = "<group>"; };
8D8169443E5AC5FF71BFB3DB /* cs */ = {isa = PBXFileReference; lastKnownFileType = text.plist.strings; name = cs; path = cs.lproj/Localizable.strings; sourceTree = "<group>"; };
8DC2C9E0E15C79BBDA80F0A2 /* TimelineStyle.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TimelineStyle.swift; sourceTree = "<group>"; };
8E088F2A1B9EC529D3221931 /* UITests.xctestplan */ = {isa = PBXFileReference; path = UITests.xctestplan; sourceTree = "<group>"; };
9010EE0CC913D095887EF36E /* OIDCService.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = OIDCService.swift; sourceTree = "<group>"; };
90733775209F4D4D366A268F /* RootRouterType.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RootRouterType.swift; sourceTree = "<group>"; };
92B61C243325DC76D3086494 /* EventBriefFactoryProtocol.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = EventBriefFactoryProtocol.swift; sourceTree = "<group>"; };
Expand Down Expand Up @@ -613,6 +617,7 @@
B83CB897B183BF3C33715F55 /* bn-IN */ = {isa = PBXFileReference; lastKnownFileType = text.plist.stringsdict; name = "bn-IN"; path = "bn-IN.lproj/Localizable.stringsdict"; sourceTree = "<group>"; };
B8A56EA2A5AE726F445CB2E3 /* eo */ = {isa = PBXFileReference; lastKnownFileType = text.plist.stringsdict; name = eo; path = eo.lproj/Localizable.stringsdict; sourceTree = "<group>"; };
B902EA6CD3296B0E10EE432B /* HomeScreen.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = HomeScreen.swift; sourceTree = "<group>"; };
BB3073CCD77D906B330BC1D6 /* Tests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Tests.swift; sourceTree = "<group>"; };
BC9B05D6B293A039EB963CA7 /* az */ = {isa = PBXFileReference; lastKnownFileType = text.plist.strings; name = az; path = az.lproj/Localizable.strings; sourceTree = "<group>"; };
BE6C10032A77AE7DC5AA4C50 /* MessageComposerTextField.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MessageComposerTextField.swift; sourceTree = "<group>"; };
BEE6BF9BA63FF42F8AF6EEEA /* sr */ = {isa = PBXFileReference; lastKnownFileType = text.plist.stringsdict; name = sr; path = sr.lproj/Localizable.stringsdict; sourceTree = "<group>"; };
Expand Down Expand Up @@ -684,6 +689,7 @@
EDB6E40BAD4504D899FAAC9A /* TemplateViewModel.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TemplateViewModel.swift; sourceTree = "<group>"; };
EE8BCD14EFED23459A43FDFF /* ja */ = {isa = PBXFileReference; lastKnownFileType = text.plist.strings; name = ja; path = ja.lproj/Localizable.strings; sourceTree = "<group>"; };
EEE384418EB1FEDFA62C9CD0 /* RoomTimelineViewFactoryProtocol.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RoomTimelineViewFactoryProtocol.swift; sourceTree = "<group>"; };
EF1593DD87F974F8509BB619 /* ElementAnimations.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ElementAnimations.swift; sourceTree = "<group>"; };
EF188681D6B6068CFAEAFC3F /* MXLogger.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = MXLogger.m; sourceTree = "<group>"; };
EFF7BF82A950B91BC5469E91 /* ViewFrameReader.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ViewFrameReader.swift; sourceTree = "<group>"; };
EFFA5FD06AAAC4AF544B594E /* AppDelegate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppDelegate.swift; sourceTree = "<group>"; };
Expand Down Expand Up @@ -720,6 +726,7 @@
A4E885358D7DD5A072A06824 /* SwiftState in Frameworks */,
29EE1791E0AFA1ABB7F23D2F /* GZIP in Frameworks */,
33CAC1226DFB8B5D8447D286 /* Sentry in Frameworks */,
492274DA6691EE985C2FCCAA /* SnapshotTesting in Frameworks */,
);
runOnlyForDeploymentPostprocessing = 0;
};
Expand Down Expand Up @@ -756,6 +763,7 @@
052CC920F473C10B509F9FC1 /* SwiftUI */ = {
isa = PBXGroup;
children = (
E2DA161C142B7AB8CC40F752 /* Animation */,
595B8797ED6A7489ABDCE384 /* ErrorHandling */,
CE2FBFD64A89F5DBE4EB30DB /* Layout */,
10578D9852BA78D309A1CBDF /* ViewModel */,
Expand Down Expand Up @@ -850,13 +858,6 @@
path = Resources;
sourceTree = "<group>";
};
3180C73BA7B8F5F7447C99B0 /* React */ = {
isa = PBXGroup;
children = (
);
path = React;
sourceTree = "<group>";
};
328DD5DA1281F758B72006C7 /* Views */ = {
isa = PBXGroup;
children = (
Expand Down Expand Up @@ -1143,6 +1144,7 @@
children = (
49D2C8E66E83EA578A7F318A /* Info.plist */,
D4DA544B2520BFA65D6DB4BB /* target.yml */,
8E088F2A1B9EC529D3221931 /* UITests.xctestplan */,
);
path = SupportingFiles;
sourceTree = "<group>";
Expand Down Expand Up @@ -1454,6 +1456,7 @@
1027BB9A852F445B7623897F /* ElementSettings.swift */,
12A626D74BBE9F4A60763B45 /* ImageAnonymizer.swift */,
6AD1A853D605C2146B0DC028 /* MatrixEntityRegex.swift */,
BB3073CCD77D906B330BC1D6 /* Tests.swift */,
44BBB96FAA2F0D53C507396B /* Extensions */,
8F9A844EB44B6AD7CA18FD96 /* HTMLParsing */,
06501F0E978B2D5C92771DC7 /* Logging */,
Expand Down Expand Up @@ -1497,13 +1500,20 @@
path = SessionVerification;
sourceTree = "<group>";
};
E2DA161C142B7AB8CC40F752 /* Animation */ = {
isa = PBXGroup;
children = (
EF1593DD87F974F8509BB619 /* ElementAnimations.swift */,
);
path = Animation;
sourceTree = "<group>";
};
E59565F441830B19DBAE567C /* Screens */ = {
isa = PBXGroup;
children = (
E74CD7681375AD2EAA34D66B /* Authentication */,
4009BE2E791C16AC6EE39A7E /* BugReport */,
B53CA9BECD3F97805E1432D0 /* HomeScreen */,
3180C73BA7B8F5F7447C99B0 /* React */,
679E9837ECA8D6776079D16E /* RoomScreen */,
D958761758AA1110476DE6A3 /* SessionVerification */,
70B74A432C241E56A7ACE610 /* Settings */,
Expand Down Expand Up @@ -1623,6 +1633,7 @@
isa = PBXNativeTarget;
buildConfigurationList = F1B67CF63C1231AEB14D70E6 /* Build configuration list for PBXNativeTarget "UITests" */;
buildPhases = (
17364E8B37FC780E07DDEAF7 /* Override Simulator Status Bars */,
BAD5CD7BE53A7C832569B67A /* Sources */,
86982BD498105258F3778110 /* Resources */,
CD30252A70288BD4BF476ED7 /* Frameworks */,
Expand All @@ -1644,6 +1655,7 @@
3853B78FB8531B83936C5DA6 /* SwiftState */,
1BCD21310B997A6837B854D6 /* GZIP */,
67E7A6F388D3BF85767609D9 /* Sentry */,
21C83087604B154AA30E9A8F /* SnapshotTesting */,
);
productName = UITests;
productReference = F506C6ADB1E1DA6638078E11 /* UITests.xctest */;
Expand Down Expand Up @@ -1805,6 +1817,7 @@
D283517192CAC3E2E6920765 /* XCRemoteSwiftPackageReference "Kingfisher" */,
80B898A3AD2AC63F3ABFC218 /* XCRemoteSwiftPackageReference "matrix-rust-components-swift" */,
A08925A9D5E3770DEB9D8509 /* XCRemoteSwiftPackageReference "sentry-cocoa" */,
F34594223DB1E7DCE48F92B8 /* XCRemoteSwiftPackageReference "swift-snapshot-testing" */,
6582B5AF3F104B0F7E031E7D /* XCRemoteSwiftPackageReference "SwiftState" */,
25B4484A6A20B9F1705DEEDA /* XCRemoteSwiftPackageReference "SwiftyBeaver" */,
);
Expand Down Expand Up @@ -1864,6 +1877,24 @@
/* End PBXResourcesBuildPhase section */

/* Begin PBXShellScriptBuildPhase section */
17364E8B37FC780E07DDEAF7 /* Override Simulator Status Bars */ = {
isa = PBXShellScriptBuildPhase;
buildActionMask = 2147483647;
files = (
);
inputFileListPaths = (
);
inputPaths = (
);
name = "Override Simulator Status Bars";
outputFileListPaths = (
);
outputPaths = (
);
runOnlyForDeploymentPostprocessing = 0;
shellPath = /bin/sh;
shellScript = "python3 $PROJECT_DIR/Tools/Scripts/bootTestSimulator.py --name 'iPhone 13 Pro Max' --version 'iOS.15.5'\npython3 $PROJECT_DIR/Tools/Scripts/bootTestSimulator.py --name 'iPad (9th generation)' --version 'iOS.15.5'\n";
};
98CA896D84BFD53B2554E891 /* ⚠️ SwiftLint */ = {
isa = PBXShellScriptBuildPhase;
buildActionMask = 2147483647;
Expand Down Expand Up @@ -1991,6 +2022,7 @@
DCB781BD227CA958809AFADF /* Coordinator.swift in Sources */,
C4F69156C31A447FEFF2A47C /* DTHTMLElement+AttributedStringBuilder.swift in Sources */,
EE8491AD81F47DF3C192497B /* DecorationTimelineItemProtocol.swift in Sources */,
FE4593FC2A02AAF92E089565 /* ElementAnimations.swift in Sources */,
06E93B2E3B32740B40F47CC5 /* ElementNavigationController.swift in Sources */,
9738F894DB1BD383BE05767A /* ElementSettings.swift in Sources */,
D8CFF02C2730EE5BC4F17ABF /* ElementToggleStyle.swift in Sources */,
Expand Down Expand Up @@ -2126,6 +2158,7 @@
1555A7643D85187D4851040C /* TemplateScreen.swift in Sources */,
75EA4ABBFAA810AFF289D6F4 /* TemplateViewModel.swift in Sources */,
5F1FDE49DFD0C680386E48F9 /* TemplateViewModelProtocol.swift in Sources */,
D85D4FA590305180B4A41795 /* Tests.swift in Sources */,
D013E70C8E28E43497820444 /* TextRoomMessage.swift in Sources */,
7963F98CDFDEAC75E072BD81 /* TextRoomTimelineItem.swift in Sources */,
5E0F2E612718BB4397A6D40A /* TextRoomTimelineView.swift in Sources */,
Expand Down Expand Up @@ -2742,6 +2775,14 @@
minimumVersion = 7.2.0;
};
};
F34594223DB1E7DCE48F92B8 /* XCRemoteSwiftPackageReference "swift-snapshot-testing" */ = {
isa = XCRemoteSwiftPackageReference;
repositoryURL = "https://github.com/pointfreeco/swift-snapshot-testing.git";
requirement = {
kind = upToNextMajorVersion;
minimumVersion = 1.9.0;
};
};
/* End XCRemoteSwiftPackageReference section */

/* Begin XCSwiftPackageProductDependency section */
Expand All @@ -2765,6 +2806,11 @@
package = 701C7BEF8F70F7A83E852DCC /* XCRemoteSwiftPackageReference "GZIP" */;
productName = GZIP;
};
21C83087604B154AA30E9A8F /* SnapshotTesting */ = {
isa = XCSwiftPackageProductDependency;
package = F34594223DB1E7DCE48F92B8 /* XCRemoteSwiftPackageReference "swift-snapshot-testing" */;
productName = SnapshotTesting;
};
36B7FC232711031AA2B0D188 /* DTCoreText */ = {
isa = XCSwiftPackageProductDependency;
package = C13F55E4518415CB4C278E73 /* XCRemoteSwiftPackageReference "DTCoreText" */;
Expand Down
11 changes: 10 additions & 1 deletion ElementX.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@
{
"identity" : "matrix-rust-components-swift",
"kind" : "remoteSourceControl",
"location" : "https://github.com/matrix-org/matrix-rust-components-swift",
"location" : "https://github.com/matrix-org/matrix-rust-components-swift.git",
"state" : {
"revision" : "1358c9c2a85cfb5fc1bfadce13565e02618725d4",
"version" : "1.0.13-alpha"
Expand All @@ -81,6 +81,15 @@
"version" : "7.18.1"
}
},
{
"identity" : "swift-snapshot-testing",
"kind" : "remoteSourceControl",
"location" : "https://github.com/pointfreeco/swift-snapshot-testing.git",
"state" : {
"revision" : "f8a9c997c3c1dab4e216a8ec9014e23144cbab37",
"version" : "1.9.0"
}
},
{
"identity" : "swiftstate",
"kind" : "remoteSourceControl",
Expand Down
7 changes: 0 additions & 7 deletions ElementX.xcodeproj/xcshareddata/xcschemes/ElementX.xcscheme
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,13 +63,6 @@
</Testables>
<CommandLineArguments>
</CommandLineArguments>
<EnvironmentVariables>
<EnvironmentVariable
key = "IS_RUNNING_UNIT_TESTS"
value = "1"
isEnabled = "YES">
</EnvironmentVariable>
</EnvironmentVariables>
<CodeCoverageTargets>
<BuildableReference
BuildableIdentifier = "primary"
Expand Down
Loading

0 comments on commit 2cb6dc1

Please sign in to comment.