From d2fd056876bb4222a08d454180d7df60e1df2135 Mon Sep 17 00:00:00 2001 From: Hector Ramos Date: Thu, 7 Jun 2018 23:08:49 -0700 Subject: [PATCH] Fix end-to-end tests Bump Babel preset to ^5 in new projects Output end-to-end test results to JUnit format Bump to Xcode 9.4.0 Bump to iOS and Apple TV 11.4 simulators Initialize simulators without instruments Downgrade to Node 8 when running end-to-end tests Rename e2e to end-to-end for clarity Fix lint failures in ContainerShip scripts --- .circleci/config.yml | 54 ++++++++++++------- .../run-android-ci-instrumentation-tests.js | 8 +-- local-cli/init/init.js | 2 +- scripts/.tests.env | 2 +- scripts/objc-test-ios.sh | 3 +- scripts/objc-test-tvos.sh | 3 +- scripts/objc-test.sh | 2 +- scripts/run-ci-e2e-tests.js | 40 +++++++++----- 8 files changed, 71 insertions(+), 43 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 31ea483c70c86d..185be9135eb451 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -198,7 +198,7 @@ aliases: # eslint sometimes runs into trouble generating the reports - &run-lint-checks name: Lint code - command: scripts/circleci/exec_swallow_error.sh yarn lint --format junit -o ~/react-native/reports/junit/js-lint-results.xml + command: scripts/circleci/exec_swallow_error.sh yarn lint --format junit -o ~/react-native/reports/junit/eslint/results.xml when: always - &run-flow-checks @@ -274,6 +274,14 @@ aliases: mkdir -p ~/react-native/reports/junit/ mkdir -p ~/react-native/reports/outputs/ + - &boot-simulator-iphone + name: Boot iPhone Simulator + command: xcrun simctl boot "iPhone 5s" || true + + - &boot-simulator-appletv + name: Boot Apple TV Simulator + command: xcrun simctl boot "Apple TV" || true + - &run-objc-ios-tests name: iOS Test Suite command: ./scripts/objc-test-ios.sh test @@ -282,9 +290,13 @@ aliases: name: tvOS Test Suite command: ./scripts/objc-test-tvos.sh test + - &run-e2e-tests + name: End-to-End Test Suite + command: node ./scripts/run-ci-e2e-tests.js --ios --tvos --js --retries 3; + - &run-objc-ios-e2e-tests name: iOS End-to-End Test Suite - command: node ./scripts/run-ci-e2e-tests.js --ios --js --retries 3; + command: node ./scripts/run-ci-e2e-tests.js --ios --retries 3; - &run-objc-tvos-e2e-tests name: tvOS End-to-End Test Suite @@ -316,7 +328,7 @@ android_defaults: &android_defaults macos_defaults: &macos_defaults <<: *defaults macos: - xcode: "9.2.0" + xcode: "9.4.0" version: 2 jobs: @@ -374,9 +386,7 @@ jobs: - attach_workspace: at: ~/react-native - - run: xcrun instruments -w "iPhone 5s (11.2)" || true - # See https://github.com/Homebrew/homebrew-core/issues/26358. - - run: brew upgrade python > /dev/null + - run: *boot-simulator-iphone - run: brew install watchman - run: *run-objc-ios-tests @@ -390,9 +400,7 @@ jobs: - attach_workspace: at: ~/react-native - - run: xcrun instruments -w "Apple TV 1080p (11.2)" || true - # See https://github.com/Homebrew/homebrew-core/issues/26358. - - run: brew upgrade python > /dev/null + - run: *boot-simulator-appletv - run: brew install watchman - run: *run-objc-tvos-tests @@ -400,13 +408,27 @@ jobs: path: ~/react-native/reports/junit # Runs end to end tests - test_ios_e2e: + test_end_to_end: <<: *macos_defaults steps: - attach_workspace: at: ~/react-native - - run: xcrun instruments -w "iPhone 5s (11.2)" || true + - run: *boot-simulator-iphone + + - run: + name: Configure Environment Variables + command: | + echo 'export PATH=/usr/local/opt/node@8/bin:$PATH' >> $BASH_ENV + source $BASH_ENV + + - run: + name: Install Node 8 + command: | + brew install node@8 + brew link node@8 + node -v + - run: *run-objc-ios-e2e-tests - store_test_results: @@ -625,7 +647,7 @@ workflows: - checkout_code # End-to-end tests - - test_ios_e2e: + - test_end_to_end: filters: *filter-ignore-gh-pages requires: - checkout_code @@ -672,11 +694,3 @@ workflows: # filters: *filter-ignore-gh-pages # requires: # - checkout_code - -experimental: - notify: - webhooks: - - url: https://code.facebook.com/circle/webhook/ - branches: - only: - - master diff --git a/ContainerShip/scripts/run-android-ci-instrumentation-tests.js b/ContainerShip/scripts/run-android-ci-instrumentation-tests.js index 706159c4902bc4..c16c7990ed317d 100644 --- a/ContainerShip/scripts/run-android-ci-instrumentation-tests.js +++ b/ContainerShip/scripts/run-android-ci-instrumentation-tests.js @@ -28,7 +28,7 @@ const path = require('path'); const colors = { GREEN: '\x1b[32m', RED: '\x1b[31m', - RESET: '\x1b[0m' + RESET: '\x1b[0m', }; const test_opts = { @@ -41,7 +41,7 @@ const test_opts = { TEST_TIMEOUT: parseInt(argv['test-timeout'] || 1000 * 60 * 10), OFFSET: argv.offset, - COUNT: argv.count + COUNT: argv.count, }; let max_test_class_length = Number.NEGATIVE_INFINITY; @@ -88,7 +88,7 @@ return async.mapSeries(testClasses, (clazz, callback) => { return async.retry(test_opts.RETRIES, (retryCb) => { const test_process = child_process.spawn('./ContainerShip/scripts/run-instrumentation-tests-via-adb-shell.sh', [test_opts.PACKAGE, clazz], { - stdio: 'inherit' + stdio: 'inherit', }); const timeout = setTimeout(() => { @@ -112,7 +112,7 @@ return async.mapSeries(testClasses, (clazz, callback) => { }, (err) => { return callback(null, { name: clazz, - status: err ? 'failure' : 'success' + status: err ? 'failure' : 'success', }); }); }, (err, results) => { diff --git a/local-cli/init/init.js b/local-cli/init/init.js index 17273f9881d847..d7497a4a828476 100644 --- a/local-cli/init/init.js +++ b/local-cli/init/init.js @@ -99,7 +99,7 @@ function generateProject(destinationRoot, newProjectName, options) { }); } if (!options['skip-jest']) { - const jestDeps = `jest babel-jest babel-preset-react-native react-test-renderer@${reactVersion}`; + const jestDeps = `jest babel-jest babel-preset-react-native@^5 react-test-renderer@${reactVersion}`; if (yarnVersion) { console.log('Adding Jest...'); execSync(`yarn add ${jestDeps} --dev --exact`, {stdio: 'inherit'}); diff --git a/scripts/.tests.env b/scripts/.tests.env index a03f487abe4d73..285578295970c9 100644 --- a/scripts/.tests.env +++ b/scripts/.tests.env @@ -17,7 +17,7 @@ export AVD_NAME="testAVD" export AVD_ABI=x86 ## IOS ## -export IOS_TARGET_OS="11.3" +export IOS_TARGET_OS="11.4" export IOS_DEVICE="iPhone 5s" export SCHEME="RNTester" diff --git a/scripts/objc-test-ios.sh b/scripts/objc-test-ios.sh index eaa4f7a8686511..56f6c5d91fe5f2 100755 --- a/scripts/objc-test-ios.sh +++ b/scripts/objc-test-ios.sh @@ -13,9 +13,10 @@ ROOT=$(dirname "$SCRIPTS") cd "$ROOT" +TEST_NAME="iOS" SCHEME="RNTester" SDK="iphonesimulator" -DESTINATION="platform=iOS Simulator,name=iPhone 5s,OS=10.3.1" +DESTINATION="platform=iOS Simulator,name=iPhone 5s,OS=11.4" # If there's a "test" argument, pass it to the test script. . ./scripts/objc-test.sh $1 diff --git a/scripts/objc-test-tvos.sh b/scripts/objc-test-tvos.sh index 9d17f7defeb6c2..fb7b8ed9bcca07 100755 --- a/scripts/objc-test-tvos.sh +++ b/scripts/objc-test-tvos.sh @@ -13,9 +13,10 @@ ROOT=$(dirname "$SCRIPTS") cd "$ROOT" +TEST_NAME="tvOS" SCHEME="RNTester-tvOS" SDK="appletvsimulator" -DESTINATION="platform=tvOS Simulator,name=Apple TV 1080p,OS=10.2" +DESTINATION="platform=tvOS Simulator,name=Apple TV,OS=11.4" # If there's a "test" argument, pass it to the test script. . ./scripts/objc-test.sh $1 diff --git a/scripts/objc-test.sh b/scripts/objc-test.sh index f2c0459989daaa..c799cb1c8f698a 100755 --- a/scripts/objc-test.sh +++ b/scripts/objc-test.sh @@ -79,7 +79,7 @@ xcodebuild \ -sdk $SDK \ -destination "$DESTINATION" \ build test \ - | xcpretty --report junit --output ~/react-native/reports/junit/objc-xcodebuild-results.xml + | xcpretty --report junit --output ~/react-native/reports/junit/$TEST_NAME/results.xml else diff --git a/scripts/run-ci-e2e-tests.js b/scripts/run-ci-e2e-tests.js index f3096c083a2821..facccf3878f297 100644 --- a/scripts/run-ci-e2e-tests.js +++ b/scripts/run-ci-e2e-tests.js @@ -104,8 +104,8 @@ try { cd('EndToEndTest'); if (argv.android) { - echo('Running an Android e2e test'); - echo('Installing e2e framework'); + echo('Running an Android end-to-end test'); + echo('Installing end-to-end framework'); if ( tryExecNTimes( () => @@ -155,14 +155,14 @@ try { SERVER_PID = packagerProcess.pid; // wait a bit to allow packager to startup exec('sleep 15s'); - echo('Executing android e2e test'); + echo('Executing android end-to-end test'); if ( tryExecNTimes(() => { exec('sleep 10s'); return exec('node node_modules/.bin/_mocha android-e2e-test.js').code; }, numberOfRetries) ) { - echo('Failed to run Android e2e tests'); + echo('Failed to run Android end-to-end tests'); echo('Most likely the code is broken'); exitCode = 1; throw Error(exitCode); @@ -171,7 +171,7 @@ try { if (argv.ios || argv.tvos) { var iosTestType = argv.tvos ? 'tvOS' : 'iOS'; - echo('Running the ' + iosTestType + 'app'); + echo('Running the ' + iosTestType + ' app'); cd('ios'); // Make sure we installed local version of react-native if (!test('-e', path.join('EndToEndTest', path.basename(MARKER_IOS)))) { @@ -193,22 +193,34 @@ try { 'response=$(curl --write-out %{http_code} --silent --output /dev/null localhost:8081/index.bundle?platform=ios&dev=true)', ); echo(`Starting packager server, ${SERVER_PID}`); - echo('Executing ' + iosTestType + ' e2e test'); + echo('Executing ' + iosTestType + ' end-to-end test'); if ( tryExecNTimes(() => { exec('sleep 10s'); + let destination = 'platform=iOS Simulator,name=iPhone 5s,OS=11.4'; + let sdk = 'iphonesimulator'; + let scheme = 'EndToEndTest'; + if (argv.tvos) { - return exec( - 'xcodebuild -destination "platform=tvOS Simulator,name=Apple TV 1080p,OS=10.0" -scheme EndToEndTest-tvOS -sdk appletvsimulator test | xcpretty && exit ${PIPESTATUS[0]}', - ).code; - } else { - return exec( - 'xcodebuild -destination "platform=iOS Simulator,name=iPhone 5s,OS=10.3.1" -scheme EndToEndTest -sdk iphonesimulator test | xcpretty && exit ${PIPESTATUS[0]}', - ).code; + destination = 'platform=tvOS Simulator,name=Apple TV,OS=11.4'; + sdk = 'appletvsimulator'; + scheme = 'EndToEndTest-tvOS'; } + + return exec( + 'xcodebuild -destination "' + + destination + + '" -scheme ' + + scheme + + ' -sdk ' + + sdk + + ' test | xcpretty --report junit --output ~/react-native/reports/junit/' + + iosTestType + + '-e2e/results.xml && exit ${PIPESTATUS[0]}', + ).code; }, numberOfRetries) ) { - echo('Failed to run ' + iosTestType + ' e2e tests'); + echo('Failed to run ' + iosTestType + ' end-to-end tests'); echo('Most likely the code is broken'); exitCode = 1; throw Error(exitCode);