From 4aa611bcf67a9383d7d771b0e0edd4e26bb8a5b4 Mon Sep 17 00:00:00 2001 From: Sean Olszewski Date: Thu, 9 Sep 2021 11:38:33 -0700 Subject: [PATCH 01/19] Add async setup/teardown variants --- Sources/XCTest/Public/XCAbstractTest.swift | 7 ++ Sources/XCTest/Public/XCTestCase.swift | 92 ++++++++++++++++++++-- 2 files changed, 92 insertions(+), 7 deletions(-) diff --git a/Sources/XCTest/Public/XCAbstractTest.swift b/Sources/XCTest/Public/XCAbstractTest.swift index c67c2b58a..191490e0a 100644 --- a/Sources/XCTest/Public/XCAbstractTest.swift +++ b/Sources/XCTest/Public/XCAbstractTest.swift @@ -52,6 +52,9 @@ open class XCTest { perform(testRun!) } + /// Async setup method called before the invocation of `setUp` for each test method in the class. + open func setUp() async throws {} + /// Setup method called before the invocation of `setUp` and the test method /// for each test method in the class. open func setUpWithError() throws {} @@ -68,6 +71,10 @@ open class XCTest { /// for each test method in the class. open func tearDownWithError() throws {} + /// Async teardown method which is called after the invocation of `tearDownWithError` + /// for each test method in the class. + open func tearDown() async throws {} + // FIXME: This initializer is required due to a Swift compiler bug on Linux. // It should be removed once the bug is fixed. public init() {} diff --git a/Sources/XCTest/Public/XCTestCase.swift b/Sources/XCTest/Public/XCTestCase.swift index 947a35db7..b7a9e4014 100644 --- a/Sources/XCTest/Public/XCTestCase.swift +++ b/Sources/XCTest/Public/XCTestCase.swift @@ -209,11 +209,9 @@ open class XCTestCase: XCTest { } private func performSetUpSequence() { - do { - try setUpWithError() - } catch { + func handleErrorDuringSetUp(_ error: Error) { if error.xct_shouldRecordAsTestFailure { - recordFailure(for: error) + self.recordFailure(for: error) } if error.xct_shouldSkipTestInvocation { @@ -225,10 +223,30 @@ open class XCTestCase: XCTest { } } - setUp() + do { + try awaitUsingExpectation { + try await self.setUp() + } + } catch { + handleErrorDuringSetUp(error) + } + + do { + try self.setUpWithError() + } catch { + handleErrorDuringSetUp(error) + } + + self.setUp() } private func performTearDownSequence() { + func handleErrorDuringTearDown(_ error: Error) { + if error.xct_shouldRecordAsTestFailure { + recordFailure(for: error) + } + } + runTeardownBlocks() tearDown() @@ -236,9 +254,15 @@ open class XCTestCase: XCTest { do { try tearDownWithError() } catch { - if error.xct_shouldRecordAsTestFailure { - recordFailure(for: error) + handleErrorDuringTearDown(error) + } + + do { + try awaitUsingExpectation { + try await self.tearDown() } + } catch { + handleErrorDuringTearDown(error) } } @@ -292,3 +316,57 @@ private func test(_ testFunc: @escaping (T) -> () throws -> Void) try testFunc(testCase)() } } + +public func asyncTest( + _ testClosureGenerator: @escaping (T) -> () async throws -> Void +) -> (T) -> () throws -> Void { + return { (testType: T) in + let testClosure = testClosureGenerator(testType) + return { + try awaitUsingExpectation(testClosure) + } + } +} + +private func awaitUsingExpectation( + _ closure: @escaping () async throws -> Void +) throws -> Void { + let expectation = XCTestExpectation(description: "async test completion") + let thrownErrorWrapper = ThrownErrorWrapper() + + Task { + defer { expectation.fulfill() } + + do { + try await closure() + } catch { + thrownErrorWrapper.error = error + } + } + + _ = XCTWaiter.wait(for: [expectation], timeout: asyncTestTimeout) + + if let error = thrownErrorWrapper.error { + throw error + } +} + +private final class ThrownErrorWrapper: @unchecked Sendable { + + private var _error: Error? + + var error: Error? { + get { + XCTWaiter.subsystemQueue.sync { _error } + } + set { + XCTWaiter.subsystemQueue.sync { _error = newValue } + } + } +} + + +// This time interval is set to a very large value due to their being no real native timeout functionality within corelibs-xctest. +// With the introduction of async/await support, the framework now relies on XCTestExpectations internally to coordinate the addition async portions of setup and tear down. +// This time interval is the timeout corelibs-xctest uses with XCTestExpectations. +private let asyncTestTimeout: TimeInterval = 60 * 60 * 24 * 30 From b61c8bc09dead6e2d37a9ca4c0cbacc7f04ea83c Mon Sep 17 00:00:00 2001 From: Sean Olszewski Date: Thu, 9 Sep 2021 11:44:22 -0700 Subject: [PATCH 02/19] Add availability checks --- Sources/XCTest/Public/XCAbstractTest.swift | 3 + Sources/XCTest/Public/XCTestCase.swift | 14 +- Tests/Functional/Asynchronous/Use/main.swift | 181 +++++++++++++++++++ 3 files changed, 194 insertions(+), 4 deletions(-) create mode 100644 Tests/Functional/Asynchronous/Use/main.swift diff --git a/Sources/XCTest/Public/XCAbstractTest.swift b/Sources/XCTest/Public/XCAbstractTest.swift index 191490e0a..8577043b9 100644 --- a/Sources/XCTest/Public/XCAbstractTest.swift +++ b/Sources/XCTest/Public/XCAbstractTest.swift @@ -53,6 +53,8 @@ open class XCTest { } /// Async setup method called before the invocation of `setUp` for each test method in the class. + /// + @available(macOS 12.0, *) open func setUp() async throws {} /// Setup method called before the invocation of `setUp` and the test method @@ -73,6 +75,7 @@ open class XCTest { /// Async teardown method which is called after the invocation of `tearDownWithError` /// for each test method in the class. + @available(macOS 12.0, *) open func tearDown() async throws {} // FIXME: This initializer is required due to a Swift compiler bug on Linux. diff --git a/Sources/XCTest/Public/XCTestCase.swift b/Sources/XCTest/Public/XCTestCase.swift index b7a9e4014..486fbddf4 100644 --- a/Sources/XCTest/Public/XCTestCase.swift +++ b/Sources/XCTest/Public/XCTestCase.swift @@ -224,8 +224,10 @@ open class XCTestCase: XCTest { } do { - try awaitUsingExpectation { - try await self.setUp() + if #available(macOS 12.0, *) { + try awaitUsingExpectation { + try await self.setUp() + } } } catch { handleErrorDuringSetUp(error) @@ -258,8 +260,10 @@ open class XCTestCase: XCTest { } do { - try awaitUsingExpectation { - try await self.tearDown() + if #available(macOS 12.0, *) { + try awaitUsingExpectation { + try await self.tearDown() + } } } catch { handleErrorDuringTearDown(error) @@ -317,6 +321,7 @@ private func test(_ testFunc: @escaping (T) -> () throws -> Void) } } +@available(macOS 12.0, *) public func asyncTest( _ testClosureGenerator: @escaping (T) -> () async throws -> Void ) -> (T) -> () throws -> Void { @@ -328,6 +333,7 @@ public func asyncTest( } } +@available(macOS 12.0, *) private func awaitUsingExpectation( _ closure: @escaping () async throws -> Void ) throws -> Void { diff --git a/Tests/Functional/Asynchronous/Use/main.swift b/Tests/Functional/Asynchronous/Use/main.swift new file mode 100644 index 000000000..4048a1a35 --- /dev/null +++ b/Tests/Functional/Asynchronous/Use/main.swift @@ -0,0 +1,181 @@ +// RUN: %{swiftc} %s -o %T/Use +// RUN: %T/Use > %t || true +// RUN: %{xctest_checker} %t %s + +#if os(macOS) + import SwiftXCTest +#else + import XCTest +#endif + +actor TestActor { + + enum Errors: String, Error { + case example + } + + private(set) var counter: Int = 0 + + func increment() async { + counter += 1 + } + + func decrement() async { + counter -= 1 + } + + func alwaysThrows() async throws { + throw TestActor.Errors.example + } + + func neverThrows() async throws {} +} + +// CHECK: Test Suite 'All tests' started at \d+-\d+-\d+ \d+:\d+:\d+\.\d+ +// CHECK: Test Suite '.*\.xctest' started at \d+-\d+-\d+ \d+:\d+:\d+\.\d+ + +// CHECK: Test Suite 'AsyncAwaitTests' started at \d+-\d+-\d+ \d+:\d+:\d+\.\d+ + +class AsyncAwaitTests: XCTestCase { + + lazy var subject = TestActor() + + static let allTests = { + return [ + ("test_explicitFailures_withinAsyncTests_areReported", adapt(test_explicitFailures_withinAsyncTests_areReported)), + ("test_asyncAnnotatedFunctionsCanPass", adapt(test_asyncAnnotatedFunctionsCanPass)), + ("test_actorsAreSupported", adapt(test_actorsAreSupported)), + ("test_asyncErrors_withinTestMethods_areReported", adapt(test_asyncErrors_withinTestMethods_areReported)), + ("test_asyncAwaitCalls_withinTeardownBlocks_areSupported", adapt(test_asyncAwaitCalls_withinTeardownBlocks_areSupported)), + ("test_asyncErrors_withinTeardownBlocks_areReported", adapt(test_asyncErrors_withinTeardownBlocks_areReported)), + ("test_somethingAsyncWithDelay", adapt(test_somethingAsyncWithDelay)), + ("test_syncWithinClassWithAsyncTestMethods", adapt(test_syncWithinClassWithAsyncTestMethods)), + ] + }() + + override func setUp() async throws {} + + override func tearDown() async throws {} + + // CHECK: Test Case 'AsyncAwaitTests.test_explicitFailures_withinAsyncTests_areReported' started at \d+-\d+-\d+ \d+:\d+:\d+\.\d+ + // CHECK: .*[/\\]Asynchronous[/\\]Use[/\\]main.swift:[[@LINE+3]]: error: AsyncAwaitTests.test_explicitFailures_withinAsyncTests_areReported : XCTAssertTrue failed - + // CHECK: Test Case 'AsyncAwaitTests.test_explicitFailures_withinAsyncTests_areReported' failed \(\d+\.\d+ seconds\) + func test_explicitFailures_withinAsyncTests_areReported() async throws { + XCTAssert(false) + } + + // CHECK: Test Case 'AsyncAwaitTests.test_asyncAnnotatedFunctionsCanPass' started at \d+-\d+-\d+ \d+:\d+:\d+\.\d+ + // CHECK: Test Case 'AsyncAwaitTests.test_asyncAnnotatedFunctionsCanPass' passed \(\d+\.\d+ seconds\) + func test_asyncAnnotatedFunctionsCanPass() async throws { + let value = await makeString() + XCTAssertNotEqual(value, "") + } + + // CHECK: Test Case 'AsyncAwaitTests.test_actorsAreSupported' started at \d+-\d+-\d+ \d+:\d+:\d+\.\d+ + // CHECK: Test Case 'AsyncAwaitTests.test_actorsAreSupported' passed \(\d+\.\d+ seconds\) + func test_actorsAreSupported() async throws { + let initialCounterValue = await subject.counter + XCTAssertEqual(initialCounterValue, 0) + + await subject.increment() + await subject.increment() + + let secondCounterValue = await subject.counter + XCTAssertEqual(secondCounterValue, 2) + + await subject.decrement() + let thirdCounterValue = await subject.counter + XCTAssertEqual(thirdCounterValue, 1) + } + + // CHECK: Test Case 'AsyncAwaitTests.test_asyncErrors_withinTestMethods_areReported' started at \d+-\d+-\d+ \d+:\d+:\d+\.\d+ + // CHECK: \:0: error: AsyncAwaitTests.test_asyncErrors_withinTestMethods_areReported : threw error "example" + // CHECK: Test Case 'AsyncAwaitTests.test_asyncErrors_withinTestMethods_areReported' failed \(\d+\.\d+ seconds\) + func test_asyncErrors_withinTestMethods_areReported() async throws { + try await subject.alwaysThrows() + } + + // CHECK: Test Case 'AsyncAwaitTests.test_asyncAwaitCalls_withinTeardownBlocks_areSupported' started at \d+-\d+-\d+ \d+:\d+:\d+\.\d+ + // CHECK: In teardown block\n + // CHECK: Test Case 'AsyncAwaitTests.test_asyncAwaitCalls_withinTeardownBlocks_areSupported' passed \(\d+\.\d+ seconds\) + func test_asyncAwaitCalls_withinTeardownBlocks_areSupported() async throws { + addTeardownBlock { + print("In teardown block") + try await self.subject.alwaysThrows() + } + } + + // CHECK: Test Case 'AsyncAwaitTests.test_asyncErrors_withinTeardownBlocks_areReported' started at \d+-\d+-\d+ \d+:\d+:\d+\.\d+\ + // CHECK: :0: error: AsyncAwaitTests.test_asyncErrors_withinTeardownBlocks_areReported : threw error "example"\n + // CHECK: Test Case 'AsyncAwaitTests.test_asyncErrors_withinTeardownBlocks_areReported' failed \(\d+\.\d+ seconds\) + func test_asyncErrors_withinTeardownBlocks_areReported() throws { + let issueRecordedExpectation = XCTestExpectation(description: "Asynchronous error recorded in: \(#function)") + + addTeardownBlock { + // Use addTeardownBlock here because the `addTeardownBlock` below intentionally throws an error so we can't `wait` after that in the same scope + self.wait(for: [issueRecordedExpectation], timeout: 1) + } + + addTeardownBlock { + do { + try await self.subject.alwaysThrows() + } catch { + issueRecordedExpectation.fulfill() + throw error + } + } + } + + // CHECK: Test Case 'AsyncAwaitTests.test_somethingAsyncWithDelay' started at \d+-\d+-\d+ \d+:\d+:\d+\.\d+ + // CHECK: Test Case 'AsyncAwaitTests.test_somethingAsyncWithDelay' passed \(\d+\.\d+ seconds\) + func test_somethingAsyncWithDelay() async throws { + try await doSomethingWithDelay() + } + + // CHECK: Test Case 'AsyncAwaitTests.test_syncWithinClassWithAsyncTestMethods' started at \d+-\d+-\d+ \d+:\d+:\d+\.\d+ + // CHECK: Test Case 'AsyncAwaitTests.test_syncWithinClassWithAsyncTestMethods' passed \(\d+\.\d+ seconds\) + func test_syncWithinClassWithAsyncTestMethods() /* intentionally non-async */ throws { + XCTAssert(Thread.isMainThread, "Expected to be ran on the main thread, but wasn't.") + } +} + +private extension AsyncAwaitTests { + + func makeString() async -> String { + """ + Some arbitrary text. + Nothing to see here, folx. + """ + } + + func doSomethingWithDelay() async throws { + func doSomethingWithDelay(completion: @escaping (Error?) -> Void) { + DispatchQueue.global().asyncAfter(deadline: .now() + .milliseconds(10)) { + completion(nil) + } + } + + try await withUnsafeThrowingContinuation { (continuation: UnsafeContinuation) in + doSomethingWithDelay { error in + if let error = error { + continuation.resume(throwing: error) + } else { + continuation.resume() + } + } + } + } +} + +// CHECK: Test Suite 'AsyncAwaitTests' failed at \d+-\d+-\d+ \d+:\d+:\d+\.\d+ +// CHECK: \t Executed 8 tests, with 3 failures \(2 unexpected\) in \d+\.\d+ \(\d+\.\d+\) seconds +// CHECK: Test Suite '.*\.xctest' failed at \d+-\d+-\d+ \d+:\d+:\d+\.\d+ +// CHECK: \t Executed 8 tests, with 3 failures \(2 unexpected\) in \d+\.\d+ \(\d+\.\d+\) seconds + +let tests = [testCase(AsyncAwaitTests.self, + AsyncAwaitTests.allTests)] +XCTMain(tests) + +// CHECK: Test Suite 'All tests' failed at \d+-\d+-\d+ \d+:\d+:\d+\.\d+ +// CHECK: \t Executed 8 tests, with 3 failures \(2 unexpected\) in \d+\.\d+ \(\d+\.\d+\) seconds + From a94963cd5653af98106d20a226ce100c00b6d9ea Mon Sep 17 00:00:00 2001 From: Sean Olszewski Date: Thu, 9 Sep 2021 11:47:46 -0700 Subject: [PATCH 03/19] Update tests; Update gitignore --- .gitignore | 1 + Tests/Functional/Asynchronous/Use/main.swift | 20 +++++++++----------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/.gitignore b/.gitignore index 1f07c20f9..43ba54d5c 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ xcuserdata *.xcscmblueprint .build/ Output/ +Tests/Functional/.lit_test_times.txt diff --git a/Tests/Functional/Asynchronous/Use/main.swift b/Tests/Functional/Asynchronous/Use/main.swift index 4048a1a35..f5f056a76 100644 --- a/Tests/Functional/Asynchronous/Use/main.swift +++ b/Tests/Functional/Asynchronous/Use/main.swift @@ -42,14 +42,14 @@ class AsyncAwaitTests: XCTestCase { static let allTests = { return [ - ("test_explicitFailures_withinAsyncTests_areReported", adapt(test_explicitFailures_withinAsyncTests_areReported)), - ("test_asyncAnnotatedFunctionsCanPass", adapt(test_asyncAnnotatedFunctionsCanPass)), - ("test_actorsAreSupported", adapt(test_actorsAreSupported)), - ("test_asyncErrors_withinTestMethods_areReported", adapt(test_asyncErrors_withinTestMethods_areReported)), - ("test_asyncAwaitCalls_withinTeardownBlocks_areSupported", adapt(test_asyncAwaitCalls_withinTeardownBlocks_areSupported)), - ("test_asyncErrors_withinTeardownBlocks_areReported", adapt(test_asyncErrors_withinTeardownBlocks_areReported)), - ("test_somethingAsyncWithDelay", adapt(test_somethingAsyncWithDelay)), - ("test_syncWithinClassWithAsyncTestMethods", adapt(test_syncWithinClassWithAsyncTestMethods)), + ("test_explicitFailures_withinAsyncTests_areReported", asyncTest(test_explicitFailures_withinAsyncTests_areReported)), + ("test_asyncAnnotatedFunctionsCanPass", asyncTest(test_asyncAnnotatedFunctionsCanPass)), + ("test_actorsAreSupported", asyncTest(test_actorsAreSupported)), + ("test_asyncErrors_withinTestMethods_areReported", asyncTest(test_asyncErrors_withinTestMethods_areReported)), + ("test_asyncAwaitCalls_withinTeardownBlocks_areSupported", asyncTest(test_asyncAwaitCalls_withinTeardownBlocks_areSupported)), + ("test_asyncErrors_withinTeardownBlocks_areReported", asyncTest(test_asyncErrors_withinTeardownBlocks_areReported)), + ("test_somethingAsyncWithDelay", asyncTest(test_somethingAsyncWithDelay)), + ("test_syncWithinClassWithAsyncTestMethods", asyncTest(test_syncWithinClassWithAsyncTestMethods)), ] }() @@ -172,9 +172,7 @@ private extension AsyncAwaitTests { // CHECK: Test Suite '.*\.xctest' failed at \d+-\d+-\d+ \d+:\d+:\d+\.\d+ // CHECK: \t Executed 8 tests, with 3 failures \(2 unexpected\) in \d+\.\d+ \(\d+\.\d+\) seconds -let tests = [testCase(AsyncAwaitTests.self, - AsyncAwaitTests.allTests)] -XCTMain(tests) +XCTMain([testCase(AsyncAwaitTests.allTests)]) // CHECK: Test Suite 'All tests' failed at \d+-\d+-\d+ \d+:\d+:\d+\.\d+ // CHECK: \t Executed 8 tests, with 3 failures \(2 unexpected\) in \d+\.\d+ \(\d+\.\d+\) seconds From c1d8e0fca69aec84bf45c18ae1d1cad9348d2fae Mon Sep 17 00:00:00 2001 From: Sean Olszewski Date: Thu, 9 Sep 2021 12:07:07 -0700 Subject: [PATCH 04/19] Add back TearDownBlocksState --- .../XCTestCase.TearDownBlocksState.swift | 48 +++++++++++++++++++ Sources/XCTest/Public/XCTestCase.swift | 41 +++++++++++----- XCTest.xcodeproj/project.pbxproj | 4 ++ 3 files changed, 80 insertions(+), 13 deletions(-) create mode 100644 Sources/XCTest/Public/XCTestCase.TearDownBlocksState.swift diff --git a/Sources/XCTest/Public/XCTestCase.TearDownBlocksState.swift b/Sources/XCTest/Public/XCTestCase.TearDownBlocksState.swift new file mode 100644 index 000000000..b0fbfd3ac --- /dev/null +++ b/Sources/XCTest/Public/XCTestCase.TearDownBlocksState.swift @@ -0,0 +1,48 @@ +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2021 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See http://swift.org/LICENSE.txt for license information +// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +// +// XCTestCase.ClosureType.swift +// Extension on XCTestCase which encapsulates ClosureType +// + +/// XCTestCase.TeardownBlocksState +/// A class which encapsulates teardown blocks which are registered via the `addTearDownBlock(_block:)` method. +/// Supports async and sync throwing methods + +extension XCTestCase { + final class TeardownBlocksState { + + private var wasFinalized = false + private var blocks: [() throws -> Void] = [] + + @available(macOS 12, *) + func append(_ block: @escaping () async throws -> Void) { + XCTWaiter.subsystemQueue.sync { + precondition(wasFinalized == false, "API violation -- attempting to add a teardown block after teardown blocks have been dequeued") + blocks.append { + try awaitUsingExpectation { try await block() } + } + } + } + + func append(_ block: @escaping () throws -> Void) { + XCTWaiter.subsystemQueue.sync { + precondition(wasFinalized == false, "API violation -- attempting to add a teardown block after teardown blocks have been dequeued") + blocks.append(block) } + } + + func finalize() -> [() throws -> Void] { + XCTWaiter.subsystemQueue.sync { + precondition(wasFinalized == false, "API violation -- attempting to run teardown blocks after they've already run") + wasFinalized = true + return blocks + } + } + } +} diff --git a/Sources/XCTest/Public/XCTestCase.swift b/Sources/XCTest/Public/XCTestCase.swift index 486fbddf4..badd6114b 100644 --- a/Sources/XCTest/Public/XCTestCase.swift +++ b/Sources/XCTest/Public/XCTestCase.swift @@ -35,6 +35,7 @@ open class XCTestCase: XCTest { private let testClosure: XCTestCaseClosure private var skip: XCTSkip? + private let teardownBlocksState: XCTestCase.TeardownBlocksState /// The name of the test case, consisting of its class name and the method /// name it will run. @@ -194,18 +195,11 @@ open class XCTestCase: XCTest { /// Teardown method called after the invocation of every test method in the /// class. open class func tearDown() {} - - private var teardownBlocks: [() -> Void] = [] - private var teardownBlocksDequeued: Bool = false - private let teardownBlocksQueue: DispatchQueue = DispatchQueue(label: "org.swift.XCTest.XCTestCase.teardownBlocks") - + /// Registers a block of teardown code to be run after the current test /// method ends. open func addTeardownBlock(_ block: @escaping () -> Void) { - teardownBlocksQueue.sync { - precondition(!self.teardownBlocksDequeued, "API violation -- attempting to add a teardown block after teardown blocks have been dequeued") - self.teardownBlocks.append(block) - } + teardownBlocksState.append(block) } private func performSetUpSequence() { @@ -241,14 +235,35 @@ open class XCTestCase: XCTest { self.setUp() } - + + static let asyncTestTimeout = 5 + + @available(macOS 12.0.0, *) private func performTearDownSequence() { - func handleErrorDuringTearDown(_ error: Error) { + @Sendable func handleErrorDuringTearDown(_ error: Error) { if error.xct_shouldRecordAsTestFailure { recordFailure(for: error) } } - + + @Sendable func runTeardownBlocks(errorHandler: @escaping (Error) -> Void) async { + for block in teardownBlocks.finalize().reversed() { + do { + try await block() + } catch { + errorHandler(error) + } + } + } + + let runTeardownBlocksExpectation = XCTestExpectation(description: "runTeardownBlocksExpectation") + Task { + defer { runTeardownBlocksExpectation.fulfill() } + await runTeardownBlocks(errorHandler: handleErrorDuringTearDown(_:)) + } + _ = XCTWaiter.wait(for: [runTeardownBlocksExpectation], timeout: Self.asyncTestTimeout) + + runTeardownBlocks() tearDown() @@ -334,7 +349,7 @@ public func asyncTest( } @available(macOS 12.0, *) -private func awaitUsingExpectation( +func awaitUsingExpectation( _ closure: @escaping () async throws -> Void ) throws -> Void { let expectation = XCTestExpectation(description: "async test completion") diff --git a/XCTest.xcodeproj/project.pbxproj b/XCTest.xcodeproj/project.pbxproj index 8edeb65fe..a51c4ac16 100644 --- a/XCTest.xcodeproj/project.pbxproj +++ b/XCTest.xcodeproj/project.pbxproj @@ -13,6 +13,7 @@ 17B6C3ED210F53BE00A11ECC /* WaiterManager.swift in Sources */ = {isa = PBXBuildFile; fileRef = 17B6C3EC210F53BE00A11ECC /* WaiterManager.swift */; }; 17B6C3EF210F990100A11ECC /* XCTWaiter+Validation.swift in Sources */ = {isa = PBXBuildFile; fileRef = 17B6C3EE210F990100A11ECC /* XCTWaiter+Validation.swift */; }; 17D5B680211216EF00D93239 /* SourceLocation.swift in Sources */ = {isa = PBXBuildFile; fileRef = 17D5B67F211216EF00D93239 /* SourceLocation.swift */; }; + A53D646726EA8F0E00F48361 /* XCTestCase.TearDownBlocksState.swift in Sources */ = {isa = PBXBuildFile; fileRef = A53D646626EA8F0E00F48361 /* XCTestCase.TearDownBlocksState.swift */; }; AE2FE0FB1CFE86DB003EF0D7 /* XCAbstractTest.swift in Sources */ = {isa = PBXBuildFile; fileRef = AE2FE0EA1CFE86DB003EF0D7 /* XCAbstractTest.swift */; }; AE2FE0FE1CFE86DB003EF0D7 /* XCTAssert.swift in Sources */ = {isa = PBXBuildFile; fileRef = AE2FE0ED1CFE86DB003EF0D7 /* XCTAssert.swift */; }; AE2FE0FF1CFE86DB003EF0D7 /* XCTestCase.swift in Sources */ = {isa = PBXBuildFile; fileRef = AE2FE0EE1CFE86DB003EF0D7 /* XCTestCase.swift */; }; @@ -58,6 +59,7 @@ 17B6C3EE210F990100A11ECC /* XCTWaiter+Validation.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "XCTWaiter+Validation.swift"; sourceTree = ""; }; 17D5B67F211216EF00D93239 /* SourceLocation.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SourceLocation.swift; sourceTree = ""; }; 5B5D86DB1BBC74AD00234F36 /* SwiftXCTest.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = SwiftXCTest.framework; sourceTree = BUILT_PRODUCTS_DIR; }; + A53D646626EA8F0E00F48361 /* XCTestCase.TearDownBlocksState.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = XCTestCase.TearDownBlocksState.swift; sourceTree = ""; }; AE2FE0EA1CFE86DB003EF0D7 /* XCAbstractTest.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = XCAbstractTest.swift; sourceTree = ""; }; AE2FE0ED1CFE86DB003EF0D7 /* XCTAssert.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = XCTAssert.swift; sourceTree = ""; }; AE2FE0EE1CFE86DB003EF0D7 /* XCTestCase.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = XCTestCase.swift; sourceTree = ""; }; @@ -160,6 +162,7 @@ AE2FE0F71CFE86DB003EF0D7 /* XCTestRun.swift */, AE2FE0F81CFE86DB003EF0D7 /* XCTestSuite.swift */, AE2FE0F91CFE86DB003EF0D7 /* XCTestSuiteRun.swift */, + A53D646626EA8F0E00F48361 /* XCTestCase.TearDownBlocksState.swift */, ); path = Public; sourceTree = ""; @@ -355,6 +358,7 @@ AE2FE1081CFE86DB003EF0D7 /* XCTestRun.swift in Sources */, AE2FE10A1CFE86DB003EF0D7 /* XCTestSuiteRun.swift in Sources */, AE2FE1161CFE86E6003EF0D7 /* ObjectWrapper.swift in Sources */, + A53D646726EA8F0E00F48361 /* XCTestCase.TearDownBlocksState.swift in Sources */, 172FF8A72117B74D0059CBC5 /* XCTNSNotificationExpectation.swift in Sources */, AE2FE11A1CFE86E6003EF0D7 /* WallClockTimeMetric.swift in Sources */, AE2FE1191CFE86E6003EF0D7 /* TestFiltering.swift in Sources */, From cb41e5585facca0ec69686b4768ed73a818a5ca2 Mon Sep 17 00:00:00 2001 From: Sean Olszewski Date: Thu, 9 Sep 2021 12:19:25 -0700 Subject: [PATCH 05/19] Add back the missing tests --- Sources/XCTest/Public/XCTestCase.swift | 42 ++++++------------- Tests/Functional/Asynchronous/Use/main.swift | 11 ++--- Tests/Functional/TestCaseLifecycle/main.swift | 14 ++++++- 3 files changed, 32 insertions(+), 35 deletions(-) diff --git a/Sources/XCTest/Public/XCTestCase.swift b/Sources/XCTest/Public/XCTestCase.swift index badd6114b..9535791bc 100644 --- a/Sources/XCTest/Public/XCTestCase.swift +++ b/Sources/XCTest/Public/XCTestCase.swift @@ -35,7 +35,7 @@ open class XCTestCase: XCTest { private let testClosure: XCTestCaseClosure private var skip: XCTSkip? - private let teardownBlocksState: XCTestCase.TeardownBlocksState + private let teardownBlocksState = XCTestCase.TeardownBlocksState() /// The name of the test case, consisting of its class name and the method /// name it will run. @@ -201,6 +201,13 @@ open class XCTestCase: XCTest { open func addTeardownBlock(_ block: @escaping () -> Void) { teardownBlocksState.append(block) } + + /// Registers a block of teardown code to be run after the current test + /// method ends. + @available(macOS 12.0, *) + open func addTeardownBlock(_ block: @escaping () async throws -> Void) { + teardownBlocksState.append(block) + } private func performSetUpSequence() { func handleErrorDuringSetUp(_ error: Error) { @@ -236,35 +243,25 @@ open class XCTestCase: XCTest { self.setUp() } - static let asyncTestTimeout = 5 - @available(macOS 12.0.0, *) private func performTearDownSequence() { - @Sendable func handleErrorDuringTearDown(_ error: Error) { + func handleErrorDuringTearDown(_ error: Error) { if error.xct_shouldRecordAsTestFailure { recordFailure(for: error) } } - @Sendable func runTeardownBlocks(errorHandler: @escaping (Error) -> Void) async { - for block in teardownBlocks.finalize().reversed() { + func runTeardownBlocks(errorHandler: @escaping (Error) -> Void) { + for block in self.teardownBlocksState.finalize().reversed() { do { - try await block() + try block() } catch { errorHandler(error) } } } - let runTeardownBlocksExpectation = XCTestExpectation(description: "runTeardownBlocksExpectation") - Task { - defer { runTeardownBlocksExpectation.fulfill() } - await runTeardownBlocks(errorHandler: handleErrorDuringTearDown(_:)) - } - _ = XCTWaiter.wait(for: [runTeardownBlocksExpectation], timeout: Self.asyncTestTimeout) - - - runTeardownBlocks() + runTeardownBlocks(errorHandler: handleErrorDuringTearDown(_:)) tearDown() @@ -285,19 +282,6 @@ open class XCTestCase: XCTest { } } - private func runTeardownBlocks() { - let blocks = teardownBlocksQueue.sync { () -> [() -> Void] in - self.teardownBlocksDequeued = true - let blocks = self.teardownBlocks - self.teardownBlocks = [] - return blocks - } - - for block in blocks.reversed() { - block() - } - } - open var continueAfterFailure: Bool { get { return true diff --git a/Tests/Functional/Asynchronous/Use/main.swift b/Tests/Functional/Asynchronous/Use/main.swift index f5f056a76..9286c7375 100644 --- a/Tests/Functional/Asynchronous/Use/main.swift +++ b/Tests/Functional/Asynchronous/Use/main.swift @@ -49,7 +49,7 @@ class AsyncAwaitTests: XCTestCase { ("test_asyncAwaitCalls_withinTeardownBlocks_areSupported", asyncTest(test_asyncAwaitCalls_withinTeardownBlocks_areSupported)), ("test_asyncErrors_withinTeardownBlocks_areReported", asyncTest(test_asyncErrors_withinTeardownBlocks_areReported)), ("test_somethingAsyncWithDelay", asyncTest(test_somethingAsyncWithDelay)), - ("test_syncWithinClassWithAsyncTestMethods", asyncTest(test_syncWithinClassWithAsyncTestMethods)), + ("test_syncWithinClassWithAsyncTestMethods", test_syncWithinClassWithAsyncTestMethods), ] }() @@ -97,7 +97,8 @@ class AsyncAwaitTests: XCTestCase { // CHECK: Test Case 'AsyncAwaitTests.test_asyncAwaitCalls_withinTeardownBlocks_areSupported' started at \d+-\d+-\d+ \d+:\d+:\d+\.\d+ // CHECK: In teardown block\n - // CHECK: Test Case 'AsyncAwaitTests.test_asyncAwaitCalls_withinTeardownBlocks_areSupported' passed \(\d+\.\d+ seconds\) + // CHECK: \:0: error: AsyncAwaitTests.test_asyncAwaitCalls_withinTeardownBlocks_areSupported : threw error "example" + // CHECK: Test Case 'AsyncAwaitTests.test_asyncAwaitCalls_withinTeardownBlocks_areSupported' failed \(\d+\.\d+ seconds\) func test_asyncAwaitCalls_withinTeardownBlocks_areSupported() async throws { addTeardownBlock { print("In teardown block") @@ -168,12 +169,12 @@ private extension AsyncAwaitTests { } // CHECK: Test Suite 'AsyncAwaitTests' failed at \d+-\d+-\d+ \d+:\d+:\d+\.\d+ -// CHECK: \t Executed 8 tests, with 3 failures \(2 unexpected\) in \d+\.\d+ \(\d+\.\d+\) seconds +// CHECK: \t Executed 8 tests, with 4 failures \(3 unexpected\) in \d+\.\d+ \(\d+\.\d+\) seconds // CHECK: Test Suite '.*\.xctest' failed at \d+-\d+-\d+ \d+:\d+:\d+\.\d+ -// CHECK: \t Executed 8 tests, with 3 failures \(2 unexpected\) in \d+\.\d+ \(\d+\.\d+\) seconds +// CHECK: \t Executed 8 tests, with 4 failures \(3 unexpected\) in \d+\.\d+ \(\d+\.\d+\) seconds XCTMain([testCase(AsyncAwaitTests.allTests)]) // CHECK: Test Suite 'All tests' failed at \d+-\d+-\d+ \d+:\d+:\d+\.\d+ -// CHECK: \t Executed 8 tests, with 3 failures \(2 unexpected\) in \d+\.\d+ \(\d+\.\d+\) seconds +// CHECK: \t Executed 8 tests, with 4 failures \(3 unexpected\) in \d+\.\d+ \(\d+\.\d+\) seconds diff --git a/Tests/Functional/TestCaseLifecycle/main.swift b/Tests/Functional/TestCaseLifecycle/main.swift index 1a75526a7..a81c33c2c 100644 --- a/Tests/Functional/TestCaseLifecycle/main.swift +++ b/Tests/Functional/TestCaseLifecycle/main.swift @@ -25,17 +25,20 @@ class SetUpTearDownTestCase: XCTestCase { override class func setUp() { super.setUp() print("In class \(#function)") + XCTAssert(Thread.isMainThread, "Expected to be ran on the main thread, but wasn't.") } override func setUp() { super.setUp() print("In \(#function)") value = 42 + XCTAssert(Thread.isMainThread, "Expected to be ran on the main thread, but wasn't.") } override func tearDown() { super.tearDown() print("In \(#function)") + XCTAssert(Thread.isMainThread, "Expected to be ran on the main thread, but wasn't.") } // CHECK: Test Case 'SetUpTearDownTestCase.test_hasValueFromSetUp' started at \d+-\d+-\d+ \d+:\d+:\d+\.\d+ @@ -52,6 +55,7 @@ class SetUpTearDownTestCase: XCTestCase { override class func tearDown() { super.tearDown() print("In class \(#function)") + XCTAssert(Thread.isMainThread, "Expected to be ran on the main thread, but wasn't.") } } // CHECK: Test Suite 'SetUpTearDownTestCase' passed at \d+-\d+-\d+ \d+:\d+:\d+\.\d+ @@ -95,16 +99,21 @@ class TeardownBlocksTestCase: XCTestCase { ("test_withSeveralTeardownBlocks", test_withSeveralTeardownBlocks), ] }() + + override func setUp() { + XCTAssert(Thread.isMainThread, "Expected to be ran on the main thread, but wasn't.") + } override func tearDown() { print("In tearDown function") + XCTAssert(Thread.isMainThread, "Expected to be ran on the main thread, but wasn't.") } // CHECK: Test Case 'TeardownBlocksTestCase.test_withoutTeardownBlocks' started at \d+-\d+-\d+ \d+:\d+:\d+\.\d+ // CHECK: In tearDown function // CHECK: Test Case 'TeardownBlocksTestCase.test_withoutTeardownBlocks' passed \(\d+\.\d+ seconds\) func test_withoutTeardownBlocks() { - // Intentionally blank + XCTAssert(Thread.isMainThread, "Expected to be ran on the main thread, but wasn't.") } // CHECK: Test Case 'TeardownBlocksTestCase.test_withATeardownBlock' started at \d+-\d+-\d+ \d+:\d+:\d+\.\d+ @@ -114,6 +123,7 @@ class TeardownBlocksTestCase: XCTestCase { func test_withATeardownBlock() { addTeardownBlock { print("In teardown block A") + XCTAssert(Thread.isMainThread, "Expected to be ran on the main thread, but wasn't.") } } @@ -125,9 +135,11 @@ class TeardownBlocksTestCase: XCTestCase { func test_withSeveralTeardownBlocks() { addTeardownBlock { print("In teardown block B") + XCTAssert(Thread.isMainThread, "Expected to be ran on the main thread, but wasn't.") } addTeardownBlock { print("In teardown block C") + XCTAssert(Thread.isMainThread, "Expected to be ran on the main thread, but wasn't.") } } } From 5e1ca37eb4a11671846dcd7a4339a60ba33ab013 Mon Sep 17 00:00:00 2001 From: Sean Olszewski Date: Thu, 9 Sep 2021 15:19:30 -0700 Subject: [PATCH 06/19] Update Sources/XCTest/Public/XCTestCase.swift Co-authored-by: Stuart Montgomery --- Sources/XCTest/Public/XCTestCase.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/XCTest/Public/XCTestCase.swift b/Sources/XCTest/Public/XCTestCase.swift index 9535791bc..3b940dc08 100644 --- a/Sources/XCTest/Public/XCTestCase.swift +++ b/Sources/XCTest/Public/XCTestCase.swift @@ -205,7 +205,7 @@ open class XCTestCase: XCTest { /// Registers a block of teardown code to be run after the current test /// method ends. @available(macOS 12.0, *) - open func addTeardownBlock(_ block: @escaping () async throws -> Void) { + open func addTeardownBlock(_ block: @Sendable @escaping () async throws -> Void) { teardownBlocksState.append(block) } From 55f88c5dffc96286267f06834081fee85cb0f506 Mon Sep 17 00:00:00 2001 From: Sean Olszewski Date: Thu, 9 Sep 2021 15:20:04 -0700 Subject: [PATCH 07/19] Update Sources/XCTest/Public/XCTestCase.TearDownBlocksState.swift Co-authored-by: Stuart Montgomery --- Sources/XCTest/Public/XCTestCase.TearDownBlocksState.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/XCTest/Public/XCTestCase.TearDownBlocksState.swift b/Sources/XCTest/Public/XCTestCase.TearDownBlocksState.swift index b0fbfd3ac..a60ed383d 100644 --- a/Sources/XCTest/Public/XCTestCase.TearDownBlocksState.swift +++ b/Sources/XCTest/Public/XCTestCase.TearDownBlocksState.swift @@ -22,7 +22,7 @@ extension XCTestCase { private var blocks: [() throws -> Void] = [] @available(macOS 12, *) - func append(_ block: @escaping () async throws -> Void) { + func append(_ block: @Sendable @escaping () async throws -> Void) { XCTWaiter.subsystemQueue.sync { precondition(wasFinalized == false, "API violation -- attempting to add a teardown block after teardown blocks have been dequeued") blocks.append { From eaf8439eca365d7d876771556420982c5dae0b70 Mon Sep 17 00:00:00 2001 From: Sean Olszewski Date: Thu, 9 Sep 2021 15:20:21 -0700 Subject: [PATCH 08/19] Update Sources/XCTest/Public/XCTestCase.TearDownBlocksState.swift Co-authored-by: Stuart Montgomery --- Sources/XCTest/Public/XCTestCase.TearDownBlocksState.swift | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Sources/XCTest/Public/XCTestCase.TearDownBlocksState.swift b/Sources/XCTest/Public/XCTestCase.TearDownBlocksState.swift index a60ed383d..a1a3a5d0a 100644 --- a/Sources/XCTest/Public/XCTestCase.TearDownBlocksState.swift +++ b/Sources/XCTest/Public/XCTestCase.TearDownBlocksState.swift @@ -23,11 +23,8 @@ extension XCTestCase { @available(macOS 12, *) func append(_ block: @Sendable @escaping () async throws -> Void) { - XCTWaiter.subsystemQueue.sync { - precondition(wasFinalized == false, "API violation -- attempting to add a teardown block after teardown blocks have been dequeued") - blocks.append { - try awaitUsingExpectation { try await block() } - } + self.append { + try awaitUsingExpectation { try await block() } } } From b8926d492105654d7a53f0b678359d37b9c3c7c3 Mon Sep 17 00:00:00 2001 From: Sean Olszewski Date: Thu, 9 Sep 2021 15:38:53 -0700 Subject: [PATCH 09/19] Address PR comments --- .../XCTestCase.TearDownBlocksState.swift | 27 ++++++++++--------- Sources/XCTest/Public/XCTestCase.swift | 2 +- XCTest.xcodeproj/project.pbxproj | 2 +- 3 files changed, 16 insertions(+), 15 deletions(-) rename Sources/XCTest/{Public => Private}/XCTestCase.TearDownBlocksState.swift (62%) diff --git a/Sources/XCTest/Public/XCTestCase.TearDownBlocksState.swift b/Sources/XCTest/Private/XCTestCase.TearDownBlocksState.swift similarity index 62% rename from Sources/XCTest/Public/XCTestCase.TearDownBlocksState.swift rename to Sources/XCTest/Private/XCTestCase.TearDownBlocksState.swift index a1a3a5d0a..4e6ebb560 100644 --- a/Sources/XCTest/Public/XCTestCase.TearDownBlocksState.swift +++ b/Sources/XCTest/Private/XCTestCase.TearDownBlocksState.swift @@ -6,32 +6,33 @@ // See http://swift.org/LICENSE.txt for license information // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors // -// -// XCTestCase.ClosureType.swift -// Extension on XCTestCase which encapsulates ClosureType -// - -/// XCTestCase.TeardownBlocksState -/// A class which encapsulates teardown blocks which are registered via the `addTearDownBlock(_block:)` method. -/// Supports async and sync throwing methods +// XCTestCase.TeardownBlocksState +// A class which encapsulates teardown blocks which are registered via the `addTearDownBlock(_block:)` method. +// Supports async and sync throwing methods extension XCTestCase { final class TeardownBlocksState { - + private var wasFinalized = false private var blocks: [() throws -> Void] = [] - + @available(macOS 12, *) - func append(_ block: @Sendable @escaping () async throws -> Void) { + + // We don't want to overload append(_:) below because of how Swift will implicitly promote sync closures to async closures, + // which can unexpectedly change their semantics in difficult to track down ways. + // + // Because of this, we chose the unusual decision to forgo overloading (which is a super sweet language feature <3) to prevent this issue from surprising any contributors to corelibs-xctest + func appendAsync(_ block: @Sendable @escaping () async throws -> Void) { self.append { try awaitUsingExpectation { try await block() } } } - + func append(_ block: @escaping () throws -> Void) { XCTWaiter.subsystemQueue.sync { precondition(wasFinalized == false, "API violation -- attempting to add a teardown block after teardown blocks have been dequeued") - blocks.append(block) } + blocks.append(block) + } } func finalize() -> [() throws -> Void] { diff --git a/Sources/XCTest/Public/XCTestCase.swift b/Sources/XCTest/Public/XCTestCase.swift index 3b940dc08..285a34f6a 100644 --- a/Sources/XCTest/Public/XCTestCase.swift +++ b/Sources/XCTest/Public/XCTestCase.swift @@ -206,7 +206,7 @@ open class XCTestCase: XCTest { /// method ends. @available(macOS 12.0, *) open func addTeardownBlock(_ block: @Sendable @escaping () async throws -> Void) { - teardownBlocksState.append(block) + teardownBlocksState.appendAsync(block) } private func performSetUpSequence() { diff --git a/XCTest.xcodeproj/project.pbxproj b/XCTest.xcodeproj/project.pbxproj index a51c4ac16..2473cee6a 100644 --- a/XCTest.xcodeproj/project.pbxproj +++ b/XCTest.xcodeproj/project.pbxproj @@ -138,6 +138,7 @@ AE63767D1D01ED17002C0EA8 /* TestListing.swift */, AE2FE1111CFE86E6003EF0D7 /* WallClockTimeMetric.swift */, AE2FE1131CFE86E6003EF0D7 /* XCTestCaseSuite.swift */, + A53D646626EA8F0E00F48361 /* XCTestCase.TearDownBlocksState.swift */, AE2FE1141CFE86E6003EF0D7 /* XCTestInternalObservation.swift */, 17B6C3EC210F53BE00A11ECC /* WaiterManager.swift */, 17D5B67F211216EF00D93239 /* SourceLocation.swift */, @@ -162,7 +163,6 @@ AE2FE0F71CFE86DB003EF0D7 /* XCTestRun.swift */, AE2FE0F81CFE86DB003EF0D7 /* XCTestSuite.swift */, AE2FE0F91CFE86DB003EF0D7 /* XCTestSuiteRun.swift */, - A53D646626EA8F0E00F48361 /* XCTestCase.TearDownBlocksState.swift */, ); path = Public; sourceTree = ""; From c385668fa0c95965e76a5eaa2c9c9a760cedf66b Mon Sep 17 00:00:00 2001 From: Sean Olszewski Date: Thu, 9 Sep 2021 15:42:03 -0700 Subject: [PATCH 10/19] Address PR comments --- Sources/XCTest/Public/XCAbstractTest.swift | 1 - Sources/XCTest/Public/XCTestCase.swift | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Sources/XCTest/Public/XCAbstractTest.swift b/Sources/XCTest/Public/XCAbstractTest.swift index 8577043b9..b2cbfee71 100644 --- a/Sources/XCTest/Public/XCAbstractTest.swift +++ b/Sources/XCTest/Public/XCAbstractTest.swift @@ -53,7 +53,6 @@ open class XCTest { } /// Async setup method called before the invocation of `setUp` for each test method in the class. - /// @available(macOS 12.0, *) open func setUp() async throws {} diff --git a/Sources/XCTest/Public/XCTestCase.swift b/Sources/XCTest/Public/XCTestCase.swift index 285a34f6a..0d4ee1829 100644 --- a/Sources/XCTest/Public/XCTestCase.swift +++ b/Sources/XCTest/Public/XCTestCase.swift @@ -35,7 +35,7 @@ open class XCTestCase: XCTest { private let testClosure: XCTestCaseClosure private var skip: XCTSkip? - private let teardownBlocksState = XCTestCase.TeardownBlocksState() + private let teardownBlocksState = TeardownBlocksState() /// The name of the test case, consisting of its class name and the method /// name it will run. From 9c2ca2bd30ff411be4cd5abc918d4f5af827c625 Mon Sep 17 00:00:00 2001 From: Sean Olszewski Date: Thu, 9 Sep 2021 15:56:06 -0700 Subject: [PATCH 11/19] Address last PR comment --- Sources/XCTest/Public/XCTestCase.swift | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Sources/XCTest/Public/XCTestCase.swift b/Sources/XCTest/Public/XCTestCase.swift index 0d4ee1829..8f0b0c1e6 100644 --- a/Sources/XCTest/Public/XCTestCase.swift +++ b/Sources/XCTest/Public/XCTestCase.swift @@ -251,17 +251,17 @@ open class XCTestCase: XCTest { } } - func runTeardownBlocks(errorHandler: @escaping (Error) -> Void) { + func runTeardownBlocks() { for block in self.teardownBlocksState.finalize().reversed() { do { try block() } catch { - errorHandler(error) + handleErrorDuringTearDown(error) } } } - runTeardownBlocks(errorHandler: handleErrorDuringTearDown(_:)) + runTeardownBlocks() tearDown() From ae1e84384ab30782cb216dd47455499d7fb619c7 Mon Sep 17 00:00:00 2001 From: Brian Croom Date: Fri, 10 Sep 2021 09:23:41 -0700 Subject: [PATCH 12/19] Reduce the new async addTeardownBlock to `public` Co-authored-by: Stuart Montgomery --- Sources/XCTest/Public/XCTestCase.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/XCTest/Public/XCTestCase.swift b/Sources/XCTest/Public/XCTestCase.swift index 8f0b0c1e6..ec6cddd04 100644 --- a/Sources/XCTest/Public/XCTestCase.swift +++ b/Sources/XCTest/Public/XCTestCase.swift @@ -205,7 +205,7 @@ open class XCTestCase: XCTest { /// Registers a block of teardown code to be run after the current test /// method ends. @available(macOS 12.0, *) - open func addTeardownBlock(_ block: @Sendable @escaping () async throws -> Void) { + public func addTeardownBlock(_ block: @Sendable @escaping () async throws -> Void) { teardownBlocksState.appendAsync(block) } From bdafb491280b5fdf4e677cb7ffbb109c9cb9499f Mon Sep 17 00:00:00 2001 From: Brian Croom Date: Mon, 13 Sep 2021 11:00:58 -0700 Subject: [PATCH 13/19] Add new source file to CMakeLists --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index e36355813..1b99588d4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -29,6 +29,7 @@ add_library(XCTest Sources/XCTest/Private/SourceLocation.swift Sources/XCTest/Private/WaiterManager.swift Sources/XCTest/Private/IgnoredErrors.swift + Sources/XCTest/Private/XCTestCase.TearDownBlocksState.swift Sources/XCTest/Public/XCTestRun.swift Sources/XCTest/Public/XCTestMain.swift Sources/XCTest/Public/XCTestCase.swift From a11d359c020436149a4ad3fea653771d20a75323 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Mon, 13 Sep 2021 13:30:01 -0500 Subject: [PATCH 14/19] A few minor edits for style, whitespace, and documentation cleanup --- .../XCTestCase.TearDownBlocksState.swift | 10 ++++------ Sources/XCTest/Public/XCAbstractTest.swift | 2 +- Sources/XCTest/Public/XCTestCase.swift | 20 +++++++++---------- 3 files changed, 15 insertions(+), 17 deletions(-) diff --git a/Sources/XCTest/Private/XCTestCase.TearDownBlocksState.swift b/Sources/XCTest/Private/XCTestCase.TearDownBlocksState.swift index 4e6ebb560..83f43fe47 100644 --- a/Sources/XCTest/Private/XCTestCase.TearDownBlocksState.swift +++ b/Sources/XCTest/Private/XCTestCase.TearDownBlocksState.swift @@ -5,23 +5,21 @@ // // See http://swift.org/LICENSE.txt for license information // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -// -// XCTestCase.TeardownBlocksState -// A class which encapsulates teardown blocks which are registered via the `addTearDownBlock(_block:)` method. -// Supports async and sync throwing methods extension XCTestCase { + + /// A class which encapsulates teardown blocks which are registered via the `addTeardownBlock(_:)` method. + /// Supports async and sync throwing methods. final class TeardownBlocksState { private var wasFinalized = false private var blocks: [() throws -> Void] = [] - @available(macOS 12, *) - // We don't want to overload append(_:) below because of how Swift will implicitly promote sync closures to async closures, // which can unexpectedly change their semantics in difficult to track down ways. // // Because of this, we chose the unusual decision to forgo overloading (which is a super sweet language feature <3) to prevent this issue from surprising any contributors to corelibs-xctest + @available(macOS 12.0, *) func appendAsync(_ block: @Sendable @escaping () async throws -> Void) { self.append { try awaitUsingExpectation { try await block() } diff --git a/Sources/XCTest/Public/XCAbstractTest.swift b/Sources/XCTest/Public/XCAbstractTest.swift index b2cbfee71..b286ea058 100644 --- a/Sources/XCTest/Public/XCAbstractTest.swift +++ b/Sources/XCTest/Public/XCAbstractTest.swift @@ -52,7 +52,7 @@ open class XCTest { perform(testRun!) } - /// Async setup method called before the invocation of `setUp` for each test method in the class. + /// Async setup method called before the invocation of `setUpWithError` for each test method in the class. @available(macOS 12.0, *) open func setUp() async throws {} diff --git a/Sources/XCTest/Public/XCTestCase.swift b/Sources/XCTest/Public/XCTestCase.swift index ec6cddd04..4a6ff7609 100644 --- a/Sources/XCTest/Public/XCTestCase.swift +++ b/Sources/XCTest/Public/XCTestCase.swift @@ -35,7 +35,6 @@ open class XCTestCase: XCTest { private let testClosure: XCTestCaseClosure private var skip: XCTSkip? - private let teardownBlocksState = TeardownBlocksState() /// The name of the test case, consisting of its class name and the method /// name it will run. @@ -195,13 +194,15 @@ open class XCTestCase: XCTest { /// Teardown method called after the invocation of every test method in the /// class. open class func tearDown() {} - + + private let teardownBlocksState = TeardownBlocksState() + /// Registers a block of teardown code to be run after the current test /// method ends. open func addTeardownBlock(_ block: @escaping () -> Void) { teardownBlocksState.append(block) } - + /// Registers a block of teardown code to be run after the current test /// method ends. @available(macOS 12.0, *) @@ -212,7 +213,7 @@ open class XCTestCase: XCTest { private func performSetUpSequence() { func handleErrorDuringSetUp(_ error: Error) { if error.xct_shouldRecordAsTestFailure { - self.recordFailure(for: error) + recordFailure(for: error) } if error.xct_shouldSkipTestInvocation { @@ -235,22 +236,21 @@ open class XCTestCase: XCTest { } do { - try self.setUpWithError() + try setUpWithError() } catch { handleErrorDuringSetUp(error) } - self.setUp() + setUp() } - - + private func performTearDownSequence() { func handleErrorDuringTearDown(_ error: Error) { if error.xct_shouldRecordAsTestFailure { recordFailure(for: error) } } - + func runTeardownBlocks() { for block in self.teardownBlocksState.finalize().reversed() { do { @@ -260,7 +260,7 @@ open class XCTestCase: XCTest { } } } - + runTeardownBlocks() tearDown() From ff0974cfd4a5e4007fe90064aaaae4cea8e399e6 Mon Sep 17 00:00:00 2001 From: Sean Olszewski Date: Wed, 22 Sep 2021 10:06:19 -0700 Subject: [PATCH 15/19] Add local lit config to temporarily disable test on the macOS in CI * This is due to the latest macOS SDK being unavailable at the moment. --- Tests/Functional/Asynchronous/Use/lit.local.cfg | 2 ++ Tests/Functional/Asynchronous/Use/main.swift | 1 + 2 files changed, 3 insertions(+) create mode 100644 Tests/Functional/Asynchronous/Use/lit.local.cfg diff --git a/Tests/Functional/Asynchronous/Use/lit.local.cfg b/Tests/Functional/Asynchronous/Use/lit.local.cfg new file mode 100644 index 000000000..4c3bcfd22 --- /dev/null +++ b/Tests/Functional/Asynchronous/Use/lit.local.cfg @@ -0,0 +1,2 @@ +if 'OS=macosx' not in config.available_features: + config.unsupported = True diff --git a/Tests/Functional/Asynchronous/Use/main.swift b/Tests/Functional/Asynchronous/Use/main.swift index 9286c7375..9e39d41a3 100644 --- a/Tests/Functional/Asynchronous/Use/main.swift +++ b/Tests/Functional/Asynchronous/Use/main.swift @@ -1,6 +1,7 @@ // RUN: %{swiftc} %s -o %T/Use // RUN: %T/Use > %t || true // RUN: %{xctest_checker} %t %s +// REQUIRES: OS=macosx #if os(macOS) import SwiftXCTest From 8b5e9f33ef36dec8998b330c7b0fcf3792ff373e Mon Sep 17 00:00:00 2001 From: Sean Olszewski Date: Wed, 22 Sep 2021 10:50:59 -0700 Subject: [PATCH 16/19] Lit config attempt #2 --- Tests/Functional/Asynchronous/Use/lit.local.cfg | 2 -- Tests/Functional/Asynchronous/Use/main.swift | 2 +- Tests/lit.cfg | 4 ++++ 3 files changed, 5 insertions(+), 3 deletions(-) delete mode 100644 Tests/Functional/Asynchronous/Use/lit.local.cfg create mode 100644 Tests/lit.cfg diff --git a/Tests/Functional/Asynchronous/Use/lit.local.cfg b/Tests/Functional/Asynchronous/Use/lit.local.cfg deleted file mode 100644 index 4c3bcfd22..000000000 --- a/Tests/Functional/Asynchronous/Use/lit.local.cfg +++ /dev/null @@ -1,2 +0,0 @@ -if 'OS=macosx' not in config.available_features: - config.unsupported = True diff --git a/Tests/Functional/Asynchronous/Use/main.swift b/Tests/Functional/Asynchronous/Use/main.swift index 9e39d41a3..249a635f3 100644 --- a/Tests/Functional/Asynchronous/Use/main.swift +++ b/Tests/Functional/Asynchronous/Use/main.swift @@ -1,7 +1,7 @@ // RUN: %{swiftc} %s -o %T/Use // RUN: %T/Use > %t || true // RUN: %{xctest_checker} %t %s -// REQUIRES: OS=macosx +// REQUIRES: concurrency_runtime #if os(macOS) import SwiftXCTest diff --git a/Tests/lit.cfg b/Tests/lit.cfg new file mode 100644 index 000000000..c0597ed8b --- /dev/null +++ b/Tests/lit.cfg @@ -0,0 +1,4 @@ +(run_cpu, run_vendor, run_os, run_vers) = re.match('([^-]+)-([^-]+)-([^0-9]+)(.*)', config.variant_triple).groups() + +if run_os != 'macosx' or run_vers >= 12: + config.available_features.add('concurrency_runtime') From 8794e0c6e5bcf6150ce60da3b2d9044eba7ce948 Mon Sep 17 00:00:00 2001 From: Sean Olszewski Date: Wed, 22 Sep 2021 11:01:53 -0700 Subject: [PATCH 17/19] Lit config attempt #3 --- Tests/Functional/lit.cfg | 5 +++++ Tests/lit.cfg | 4 ---- 2 files changed, 5 insertions(+), 4 deletions(-) delete mode 100644 Tests/lit.cfg diff --git a/Tests/Functional/lit.cfg b/Tests/Functional/lit.cfg index 07c8a8218..6e526ea0e 100644 --- a/Tests/Functional/lit.cfg +++ b/Tests/Functional/lit.cfg @@ -139,3 +139,8 @@ config.substitutions.append(('%{xctest_checker}', '%%{python} %s' % xctest_check # Add Python to run xctest_checker.py tests as part of XCTest tests config.substitutions.append( ('%{python}', pipes.quote(sys.executable)) ) + +(run_cpu, run_vendor, run_os, run_vers) = re.match('([^-]+)-([^-]+)-([^0-9]+)(.*)', config.variant_triple).groups() + +if run_os != 'macosx' or run_vers >= 12: + config.available_features.add('concurrency_runtime') diff --git a/Tests/lit.cfg b/Tests/lit.cfg deleted file mode 100644 index c0597ed8b..000000000 --- a/Tests/lit.cfg +++ /dev/null @@ -1,4 +0,0 @@ -(run_cpu, run_vendor, run_os, run_vers) = re.match('([^-]+)-([^-]+)-([^0-9]+)(.*)', config.variant_triple).groups() - -if run_os != 'macosx' or run_vers >= 12: - config.available_features.add('concurrency_runtime') From 1f92e0e95476ce7d38a143962fd8fbabb62b68f5 Mon Sep 17 00:00:00 2001 From: Sean Olszewski Date: Wed, 22 Sep 2021 13:06:03 -0700 Subject: [PATCH 18/19] Lit config attempt # --- Tests/Functional/lit.cfg | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Tests/Functional/lit.cfg b/Tests/Functional/lit.cfg index 6e526ea0e..49a5bf3b2 100644 --- a/Tests/Functional/lit.cfg +++ b/Tests/Functional/lit.cfg @@ -7,16 +7,18 @@ # # See http://swift.org/LICENSE.txt for license information # See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors - +from pkg_resources import parse_version import os import platform import tempfile import sys import lit import pipes +import re # Set up lit config. config.name = 'SwiftXCTestFunctionalTests' +config.os_info = (platform.system(), platform.mac_ver()[0]) config.test_format = lit.formats.ShTest(execute_external=False) config.suffixes = ['.swift'] @@ -140,7 +142,9 @@ config.substitutions.append(('%{xctest_checker}', '%%{python} %s' % xctest_check # Add Python to run xctest_checker.py tests as part of XCTest tests config.substitutions.append( ('%{python}', pipes.quote(sys.executable)) ) -(run_cpu, run_vendor, run_os, run_vers) = re.match('([^-]+)-([^-]+)-([^0-9]+)(.*)', config.variant_triple).groups() - -if run_os != 'macosx' or run_vers >= 12: +# Conditionally report the Swift 5.5 Concurrency runtime as available depending on the OS and version. +(run_os, run_vers) = config.os_info +os_is_not_macOS = run_os != 'Darwin' +macOS_version_is_recent_enough = parse_version(run_vers) >= parse_version('12.0') +if os_is_not_macOS or macOS_version_is_recent_enough: config.available_features.add('concurrency_runtime') From fe41749b3e19ddfd3f7ec7613821d7fa7aefe253 Mon Sep 17 00:00:00 2001 From: Brian Croom Date: Thu, 23 Sep 2021 09:50:45 -0700 Subject: [PATCH 19/19] Explicitly link the concurrency runtime weakly on macOS for now --- XCTest.xcodeproj/project.pbxproj | 2 ++ 1 file changed, 2 insertions(+) diff --git a/XCTest.xcodeproj/project.pbxproj b/XCTest.xcodeproj/project.pbxproj index 2473cee6a..db4f11578 100644 --- a/XCTest.xcodeproj/project.pbxproj +++ b/XCTest.xcodeproj/project.pbxproj @@ -514,6 +514,7 @@ "OTHER_LDFLAGS[sdk=macosx*]" = ( "-framework", SwiftFoundation, + "-weak-lswift_Concurrency", ); PRODUCT_BUNDLE_PACKAGE_TYPE = FMWK; SKIP_INSTALL = YES; @@ -538,6 +539,7 @@ "OTHER_LDFLAGS[sdk=macosx*]" = ( "-framework", SwiftFoundation, + "-weak-lswift_Concurrency", ); PRODUCT_BUNDLE_PACKAGE_TYPE = FMWK; SKIP_INSTALL = YES;