From 439cec16eb1a9b78a8c63f8ac5e083babca4f4e9 Mon Sep 17 00:00:00 2001 From: Tony Li Date: Mon, 5 Sep 2022 11:58:13 +1200 Subject: [PATCH 1/4] Make sure test results are archived when test suites fail --- .buildkite/commands/run-ui-tests.sh | 5 +++++ .buildkite/commands/run-unit-tests.sh | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/.buildkite/commands/run-ui-tests.sh b/.buildkite/commands/run-ui-tests.sh index e3f72603595..2e8b3c69963 100755 --- a/.buildkite/commands/run-ui-tests.sh +++ b/.buildkite/commands/run-ui-tests.sh @@ -35,7 +35,12 @@ install_cocoapods echo "--- ๐Ÿงช Testing" xcrun simctl list >> /dev/null rake mocks & +set +e bundle exec fastlane test_without_building name:"$TEST_NAME" device:"$DEVICE" ios_version:"$IOS_VERSION" +TESTS_EXIT_STATUS=$? +set -e echo "--- ๐Ÿ“ฆ Zipping test results" cd fastlane/test_output/ && zip -rq WooCommerce.xcresult.zip WooCommerce.xcresult && cd - + +exit $TESTS_EXIT_STATUS diff --git a/.buildkite/commands/run-unit-tests.sh b/.buildkite/commands/run-unit-tests.sh index 2a76feb75e3..14ec6c763b8 100755 --- a/.buildkite/commands/run-unit-tests.sh +++ b/.buildkite/commands/run-unit-tests.sh @@ -16,7 +16,12 @@ echo "--- :rubygems: Setting up Gems" install_gems echo "--- ๐Ÿงช Testing" +set +e bundle exec fastlane test_without_building name:UnitTests +TESTS_EXIT_STATUS=$? +set -e echo "--- ๐Ÿ“ฆ Zipping test results" cd fastlane/test_output/ && zip -rq WooCommerce.xcresult.zip WooCommerce.xcresult && cd - + +exit $TESTS_EXIT_STATUS From 6fcc82f97d245fcc87d2884b00762c5e9d9f5b0b Mon Sep 17 00:00:00 2001 From: Tony Li Date: Mon, 5 Sep 2022 12:07:30 +1200 Subject: [PATCH 2/4] Force a couple of test case failures --- WooCommerce/WooCommerceTests/AppCoordinatorTests.swift | 2 +- WooCommerce/WooCommerceUITests/Tests/OrdersTests.swift | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/WooCommerce/WooCommerceTests/AppCoordinatorTests.swift b/WooCommerce/WooCommerceTests/AppCoordinatorTests.swift index 978a5368140..6b872fb78e4 100644 --- a/WooCommerce/WooCommerceTests/AppCoordinatorTests.swift +++ b/WooCommerce/WooCommerceTests/AppCoordinatorTests.swift @@ -112,7 +112,7 @@ final class AppCoordinatorTests: XCTestCase { // so we have to wait a bit loginNavigationController.viewControllers.count > 1 } - XCTAssertTrue(loginNavigationController.topViewController is ULAccountMismatchViewController) + XCTAssertFalse(loginNavigationController.topViewController is ULAccountMismatchViewController) } func test_starting_app_logged_in_without_selected_site_presents_error_if_the_error_site_address_does_not_have_woo() throws { diff --git a/WooCommerce/WooCommerceUITests/Tests/OrdersTests.swift b/WooCommerce/WooCommerceUITests/Tests/OrdersTests.swift index d7b150d2477..028c52c70fe 100644 --- a/WooCommerce/WooCommerceUITests/Tests/OrdersTests.swift +++ b/WooCommerce/WooCommerceUITests/Tests/OrdersTests.swift @@ -23,6 +23,8 @@ final class OrdersTests: XCTestCase { .verifySingleOrder(order: orders[0]) .goBackToOrdersScreen() .verifyOrdersScreenLoaded() + + XCTAssertEqual(0, 1) } func test_create_new_order() throws { From 9dcb75c2c42dba43f64397187be1c9c84319bc4d Mon Sep 17 00:00:00 2001 From: Tony Li Date: Mon, 5 Sep 2022 13:10:02 +1200 Subject: [PATCH 3/4] Revert "Force a couple of test case failures" This reverts commit 6fcc82f97d245fcc87d2884b00762c5e9d9f5b0b. --- WooCommerce/WooCommerceTests/AppCoordinatorTests.swift | 2 +- WooCommerce/WooCommerceUITests/Tests/OrdersTests.swift | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/WooCommerce/WooCommerceTests/AppCoordinatorTests.swift b/WooCommerce/WooCommerceTests/AppCoordinatorTests.swift index 6b872fb78e4..978a5368140 100644 --- a/WooCommerce/WooCommerceTests/AppCoordinatorTests.swift +++ b/WooCommerce/WooCommerceTests/AppCoordinatorTests.swift @@ -112,7 +112,7 @@ final class AppCoordinatorTests: XCTestCase { // so we have to wait a bit loginNavigationController.viewControllers.count > 1 } - XCTAssertFalse(loginNavigationController.topViewController is ULAccountMismatchViewController) + XCTAssertTrue(loginNavigationController.topViewController is ULAccountMismatchViewController) } func test_starting_app_logged_in_without_selected_site_presents_error_if_the_error_site_address_does_not_have_woo() throws { diff --git a/WooCommerce/WooCommerceUITests/Tests/OrdersTests.swift b/WooCommerce/WooCommerceUITests/Tests/OrdersTests.swift index 028c52c70fe..d7b150d2477 100644 --- a/WooCommerce/WooCommerceUITests/Tests/OrdersTests.swift +++ b/WooCommerce/WooCommerceUITests/Tests/OrdersTests.swift @@ -23,8 +23,6 @@ final class OrdersTests: XCTestCase { .verifySingleOrder(order: orders[0]) .goBackToOrdersScreen() .verifyOrdersScreenLoaded() - - XCTAssertEqual(0, 1) } func test_create_new_order() throws { From 0dcfec050ed42aa860fbf5ed6af69d94654549db Mon Sep 17 00:00:00 2001 From: Tony Li Date: Tue, 6 Sep 2022 09:26:59 +1200 Subject: [PATCH 4/4] Expand test group log when tests failed --- .buildkite/commands/run-ui-tests.sh | 6 ++++++ .buildkite/commands/run-unit-tests.sh | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/.buildkite/commands/run-ui-tests.sh b/.buildkite/commands/run-ui-tests.sh index 2e8b3c69963..4f7876fb6aa 100755 --- a/.buildkite/commands/run-ui-tests.sh +++ b/.buildkite/commands/run-ui-tests.sh @@ -40,6 +40,12 @@ bundle exec fastlane test_without_building name:"$TEST_NAME" device:"$DEVICE" io TESTS_EXIT_STATUS=$? set -e +if [[ "$TESTS_EXIT_STATUS" -ne 0 ]]; then + # Keep the (otherwise collapsed) current "Testing" section open in Buildkite logs on error. See https://buildkite.com/docs/pipelines/managing-log-output#collapsing-output + echo "^^^ +++" + echo "UI Tests failed!" +fi + echo "--- ๐Ÿ“ฆ Zipping test results" cd fastlane/test_output/ && zip -rq WooCommerce.xcresult.zip WooCommerce.xcresult && cd - diff --git a/.buildkite/commands/run-unit-tests.sh b/.buildkite/commands/run-unit-tests.sh index 14ec6c763b8..2331bb4383d 100755 --- a/.buildkite/commands/run-unit-tests.sh +++ b/.buildkite/commands/run-unit-tests.sh @@ -21,6 +21,12 @@ bundle exec fastlane test_without_building name:UnitTests TESTS_EXIT_STATUS=$? set -e +if [[ "$TESTS_EXIT_STATUS" -ne 0 ]]; then + # Keep the (otherwise collapsed) current "Testing" section open in Buildkite logs on error. See https://buildkite.com/docs/pipelines/managing-log-output#collapsing-output + echo "^^^ +++" + echo "Unit Tests failed!" +fi + echo "--- ๐Ÿ“ฆ Zipping test results" cd fastlane/test_output/ && zip -rq WooCommerce.xcresult.zip WooCommerce.xcresult && cd -