diff --git a/scripts/circle-ci-artifacts-utils.js b/scripts/circle-ci-artifacts-utils.js new file mode 100644 index 00000000000000..3a9d9d2a0c83f1 --- /dev/null +++ b/scripts/circle-ci-artifacts-utils.js @@ -0,0 +1,198 @@ +#!/usr/bin/env node +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + */ + +'use strict'; + +const {exec} = require('shelljs'); + +const util = require('util'); +const asyncRequest = require('request'); +const request = util.promisify(asyncRequest); + +let circleCIHeaders; +let jobs; +let baseTemporaryPath; + +async function initialize(circleCIToken, baseTempPath, branchName) { + console.info('Getting CircleCI information'); + circleCIHeaders = {'Circle-Token': circleCIToken}; + baseTemporaryPath = baseTempPath; + exec(`mkdir -p ${baseTemporaryPath}`); + const pipeline = await _getLastCircleCIPipelineID(branchName); + const packageAndReleaseWorkflow = await _getPackageAndReleaseWorkflow( + pipeline.id, + ); + const testsWorkflow = await _getTestsWorkflow(pipeline.id); + const jobsPromises = [ + _getCircleCIJobs(packageAndReleaseWorkflow.id), + _getCircleCIJobs(testsWorkflow.id), + ]; + + const jobsResults = await Promise.all(jobsPromises); + + jobs = jobsResults.flatMap(j => j); +} + +function baseTmpPath() { + return baseTemporaryPath; +} + +async function _getLastCircleCIPipelineID(branchName) { + const options = { + method: 'GET', + url: 'https://circleci.com/api/v2/project/gh/facebook/react-native/pipeline', + qs: { + branch: branchName, + }, + headers: circleCIHeaders, + }; + + const response = await request(options); + if (response.error) { + throw new Error(error); + } + + const items = JSON.parse(response.body).items; + + if (!items || items.length === 0) { + throw new Error( + 'No pipelines found on this branch. Make sure that the CI has run at least once, successfully', + ); + } + + const lastPipeline = items[0]; + return {id: lastPipeline.id, number: lastPipeline.number}; +} + +async function _getSpecificWorkflow(pipelineId, workflowName) { + const options = { + method: 'GET', + url: `https://circleci.com/api/v2/pipeline/${pipelineId}/workflow`, + headers: circleCIHeaders, + }; + const response = await request(options); + if (response.error) { + throw new Error(error); + } + + const body = JSON.parse(response.body); + let workflow = body.items.find(w => w.name === workflowName); + _throwIfWorkflowNotFound(workflow, workflowName); + return workflow; +} + +function _throwIfWorkflowNotFound(workflow, name) { + if (!workflow) { + throw new Error( + `Can't find a workflow named ${name}. Please check whether that workflow has started.`, + ); + } +} + +async function _getPackageAndReleaseWorkflow(pipelineId) { + return _getSpecificWorkflow(pipelineId, 'package_and_publish_release_dryrun'); +} + +async function _getTestsWorkflow(pipelineId) { + return _getSpecificWorkflow(pipelineId, 'tests'); +} + +async function _getCircleCIJobs(workflowId) { + const options = { + method: 'GET', + url: `https://circleci.com/api/v2/workflow/${workflowId}/job`, + headers: circleCIHeaders, + }; + const response = await request(options); + if (response.error) { + throw new Error(error); + } + + const body = JSON.parse(response.body); + return body.items; +} + +async function _getJobsArtifacts(jobNumber) { + const options = { + method: 'GET', + url: `https://circleci.com/api/v2/project/gh/facebook/react-native/${jobNumber}/artifacts`, + headers: circleCIHeaders, + }; + const response = await request(options); + if (response.error) { + throw new Error(error); + } + + const body = JSON.parse(response.body); + return body.items; +} + +async function _findUrlForJob(jobName, artifactPath) { + const job = jobs.find(j => j.name === jobName); + _throwIfJobIsNull(job); + _throwIfJobIsUnsuccessful(job); + + const artifacts = await _getJobsArtifacts(job.job_number); + return artifacts.find(artifact => artifact.path.indexOf(artifactPath) > -1) + .url; +} + +function _throwIfJobIsNull(job) { + if (!job) { + throw new Error( + `Can't find a job with name ${job.name}. Please verify that it has been executed and that all its dependencies completed successfully.`, + ); + } +} + +function _throwIfJobIsUnsuccessful(job) { + if (job.status !== 'success') { + throw new Error( + `The job ${job.name} status is ${job.status}. We need a 'success' status to proceed with the testing.`, + ); + } +} + +async function artifactURLHermesDebug() { + return _findUrlForJob('build_hermes_macos-Debug', 'hermes-ios-debug.tar.gz'); +} + +async function artifactURLForMavenLocal() { + return _findUrlForJob('build_and_publish_npm_package-2', 'maven-local.zip'); +} + +async function artifactURLForHermesRNTesterAPK(emulatorArch) { + return _findUrlForJob( + 'test_android', + `rntester-apk/hermes/debug/app-hermes-${emulatorArch}-debug.apk`, + ); +} + +async function artifactURLForJSCRNTesterAPK(emulatorArch) { + return _findUrlForJob( + 'test_android', + `rntester-apk/jsc/debug/app-jsc-${emulatorArch}-debug.apk`, + ); +} + +function downloadArtifact(artifactURL, destination) { + exec(`rm -rf ${destination}`); + exec(`curl ${artifactURL} -Lo ${destination}`); +} + +module.exports = { + initialize, + downloadArtifact, + artifactURLForJSCRNTesterAPK, + artifactURLForHermesRNTesterAPK, + artifactURLForMavenLocal, + artifactURLHermesDebug, + baseTmpPath, +}; diff --git a/scripts/release-utils.js b/scripts/release-utils.js index 45d936df73e684..ab0cb4f8058c68 100644 --- a/scripts/release-utils.js +++ b/scripts/release-utils.js @@ -91,11 +91,6 @@ function generateiOSArtifacts( ) { pushd(`${hermesCoreSourceFolder}`); - //Need to generate hermesc - exec( - `${hermesCoreSourceFolder}/utils/build-hermesc-xcode.sh ${hermesCoreSourceFolder}/build_host_hermesc`, - ); - //Generating iOS Artifacts exec( `JSI_PATH=${jsiFolder} BUILD_TYPE=${buildType} ${hermesCoreSourceFolder}/utils/build-mac-framework.sh`, diff --git a/scripts/test-e2e-local-clean.js b/scripts/test-e2e-local-clean.js index ae79e228934d2f..30e94bf34d177a 100644 --- a/scripts/test-e2e-local-clean.js +++ b/scripts/test-e2e-local-clean.js @@ -44,6 +44,7 @@ if (isPackagerRunning() === 'running') { console.info('\n** Cleaning Gradle build artifacts **\n'); exec('./gradlew clean'); exec('rm -rf /tmp/maven-local'); +exec('rm -rf /tmp/react-native-tmp'); // iOS console.info('\n** Nuking the derived data folder **\n'); @@ -56,9 +57,6 @@ exec('rm -rf ~/Library/Caches/CocoaPods/Pods/External/hermes-engine'); console.info('\n** Removing the RNTester Pods **\n'); exec('rm -rf packages/rn-tester/Pods'); -// I'm not sure we want to also remove the lock file -// exec('rm -rf packages/rn-tester/Podfile.lock'); - // RNTestProject console.info('\n** Removing the RNTestProject folder **\n'); exec('rm -rf /tmp/RNTestProject'); diff --git a/scripts/test-e2e-local.js b/scripts/test-e2e-local.js index 6a8f679f29d249..0be3c7bb6a4950 100644 --- a/scripts/test-e2e-local.js +++ b/scripts/test-e2e-local.js @@ -16,27 +16,20 @@ * and to make it more accessible for other devs to play around with. */ -const {exec, exit, pushd, popd, pwd, cd, cp} = require('shelljs'); -const updateTemplatePackage = require('../scripts/update-template-package'); +const {exec, pushd, popd, pwd, cd} = require('shelljs'); +const updateTemplatePackage = require('./update-template-package'); const yargs = require('yargs'); +const path = require('path'); const fs = require('fs'); const { + checkPackagerRunning, maybeLaunchAndroidEmulator, - isPackagerRunning, launchPackagerInSeparateWindow, + setupCircleCIArtifacts, + prepareArtifacts, } = require('./testing-utils'); -const { - generateAndroidArtifacts, - generateiOSArtifacts, -} = require('./release-utils'); - -const { - downloadHermesSourceTarball, - expandHermesSourceTarball, -} = require('react-native/scripts/hermes/hermes-utils.js'); - const argv = yargs .option('t', { alias: 'target', @@ -52,105 +45,152 @@ const argv = yargs alias: 'hermes', type: 'boolean', default: true, + }) + .option('c', { + alias: 'circleciToken', + type: 'string', }).argv; -/* - * see the test-local-e2e.js script for clean up process - */ - -// command order: we ask the user to select if they want to test RN tester -// or RNTestProject - -// if they select RN tester, we ask if iOS or Android, and then we run the tests -// if they select RNTestProject, we run the RNTestProject test - -// let's check if Metro is already running, if it is let's kill it and start fresh -if (isPackagerRunning() === 'running') { - exec("lsof -i :8081 | grep LISTEN | /usr/bin/awk '{print $2}' | xargs kill"); -} - -const onReleaseBranch = exec('git rev-parse --abbrev-ref HEAD', { - silent: true, -}) - .stdout.trim() - .endsWith('-stable'); +// === RNTester === // -if (argv.target === 'RNTester') { - // FIXME: make sure that the commands retains colors - // (--ansi) doesn't always work - // see also https://github.com/shelljs/shelljs/issues/86 - pushd('packages/rn-tester'); +/** + * Start the test for RNTester on iOS. + * + * Parameters: + * - @circleCIArtifacts manager object to manage all the download of CircleCIArtifacts. If null, it will fallback not to use them. + * - @onReleaseBranch whether we are on a release branch or not + */ +async function testRNTesterIOS(circleCIArtifacts, onReleaseBranch) { + console.info( + `We're going to test the ${ + argv.hermes ? 'Hermes' : 'JSC' + } version of RNTester iOS with the new Architecture enabled`, + ); - if (argv.platform === 'iOS') { - console.info( - `We're going to test the ${ - argv.hermes ? 'Hermes' : 'JSC' - } version of RNTester iOS with the new Architecture enabled`, + // remember that for this to be successful + // you should have run bundle install once + // in your local setup + if (argv.hermes && circleCIArtifacts != null) { + const hermesURL = await circleCIArtifacts.artifactURLHermesDebug(); + const hermesPath = path.join( + circleCIArtifacts.baseTmpPath(), + 'hermes-ios-debug.tar.gz', ); - - // remember that for this to be successful - // you should have run bundle install once - // in your local setup - also: if I'm on release branch, I pick the - // hermes ref from the hermes ref file (see hermes-engine.podspec) + // download hermes source code from manifold + circleCIArtifacts.downloadArtifact(hermesURL, hermesPath); + console.info(`Downloaded Hermes in ${hermesPath}`); + exec( + `HERMES_ENGINE_TARBALL_PATH=${hermesPath} RCT_NEW_ARCH_ENABLED=1 bundle exec pod install --ansi`, + ); + } else { exec( `USE_HERMES=${ argv.hermes ? 1 : 0 } CI=${onReleaseBranch} RCT_NEW_ARCH_ENABLED=1 bundle exec pod install --ansi`, ); + } - // if everything succeeded so far, we can launch Metro and the app - // start the Metro server in a separate window - launchPackagerInSeparateWindow(pwd()); + // if everything succeeded so far, we can launch Metro and the app + // start the Metro server in a separate window + launchPackagerInSeparateWindow(pwd()); - // launch the app on iOS simulator - exec('npx react-native run-ios --scheme RNTester --simulator "iPhone 14"'); - } else { - // we do the android path here + // launch the app on iOS simulator + exec('npx react-native run-ios --scheme RNTester --simulator "iPhone 14"'); +} + +/** + * Start the test for RNTester on Android. + * + * Parameters: + * - @circleCIArtifacts manager object to manage all the download of CircleCIArtifacts. If null, it will fallback not to use them. + */ +async function testRNTesterAndroid(circleCIArtifacts) { + maybeLaunchAndroidEmulator(); + + console.info( + `We're going to test the ${ + argv.hermes ? 'Hermes' : 'JSC' + } version of RNTester Android with the new Architecture enabled`, + ); - maybeLaunchAndroidEmulator(); + // Start the Metro server so it will be ready if the app can be built and installed successfully. + launchPackagerInSeparateWindow(pwd()); - console.info( - `We're going to test the ${ - argv.hermes ? 'Hermes' : 'JSC' - } version of RNTester Android with the new Architecture enabled`, + // Wait for the Android Emulator to be properly loaded and bootstrapped + exec( + "adb wait-for-device shell 'while [[ -z $(getprop sys.boot_completed) ]]; do sleep 1; done; input keyevent 82'", + ); + + if (circleCIArtifacts != null) { + const downloadPath = path.join( + circleCIArtifacts.baseTmpPath(), + 'rntester.apk', ); + + const emulatorArch = exec('adb shell getprop ro.product.cpu.abi').trim(); + const rntesterAPKURL = argv.hermes + ? await circleCIArtifacts.artifactURLForHermesRNTesterAPK(emulatorArch) + : await circleCIArtifacts.artifactURLForJSCRNTesterAPK(emulatorArch); + + console.info('Start Downloading APK'); + circleCIArtifacts.downloadArtifact(rntesterAPKURL, downloadPath); + + exec(`adb install ${downloadPath}`); + } else { exec( `../../gradlew :packages:rn-tester:android:app:${ argv.hermes ? 'installHermesDebug' : 'installJscDebug' } --quiet`, ); + } - // launch the app on Android simulator - // TODO: we should find a way to make it work like for iOS, via npx react-native run-android - // currently, that fails with an error. + // launch the app + // TODO: we should find a way to make it work like for iOS, via npx react-native run-android + // currently, that fails with an error. + exec( + 'adb shell am start -n com.facebook.react.uiapp/com.facebook.react.uiapp.RNTesterActivity', + ); - // if everything succeeded so far, we can launch Metro and the app - // start the Metro server in a separate window - launchPackagerInSeparateWindow(pwd()); + // just to make sure that the Android up won't have troubles finding the Metro server + exec('adb reverse tcp:8081 tcp:8081'); +} - // launch the app - exec( - 'adb shell am start -n com.facebook.react.uiapp/com.facebook.react.uiapp.RNTesterActivity', - ); +/** + * Function that start testing on RNTester. + * + * Parameters: + * - @circleCIArtifacts manager object to manage all the download of CircleCIArtifacts. If null, it will fallback not to use them. + * - @onReleaseBranch whether we are on a release branch or not + */ +async function testRNTester(circleCIArtifacts, onReleaseBranch) { + // FIXME: make sure that the commands retains colors + // (--ansi) doesn't always work + // see also https://github.com/shelljs/shelljs/issues/86 + pushd('packages/rn-tester'); - // just to make sure that the Android up won't have troubles finding the Metro server - exec('adb reverse tcp:8081 tcp:8081'); + if (argv.platform === 'iOS') { + await testRNTesterIOS(circleCIArtifacts, onReleaseBranch); + } else { + await testRNTesterAndroid(circleCIArtifacts); } popd(); -} else { +} + +// === RNTestProject === // + +async function testRNTestProject(circleCIArtifacts) { console.info("We're going to test a fresh new RN project"); // create the local npm package to feed the CLI // base setup required (specular to publish-npm.js) - - // we need to add the unique timestamp to avoid npm/yarn to use some local caches const baseVersion = require('../packages/react-native/package.json').version; // in local testing, 1000.0.0 mean we are on main, every other case means we are // working on a release version const buildType = baseVersion !== '1000.0.0' ? 'release' : 'dry-run'; + // we need to add the unique timestamp to avoid npm/yarn to use some local caches const dateIdentifier = new Date() .toISOString() .slice(0, -8) @@ -159,62 +199,44 @@ if (argv.target === 'RNTester') { const releaseVersion = `${baseVersion}-${dateIdentifier}`; - // this is needed to generate the Android artifacts correctly - const exitCode = exec( - `node scripts/set-rn-version.js --to-version ${releaseVersion} --build-type ${buildType}`, - ).code; - - if (exitCode !== 0) { - console.error( - `Failed to set the RN version. Version ${releaseVersion} is not valid for ${buildType}`, - ); - process.exit(exitCode); - } - - // Generate native files for Android - generateAndroidArtifacts(releaseVersion); - - // Setting up generating native iOS (will be done later) + // Prepare some variables for later use const repoRoot = pwd(); const reactNativePackagePath = `${repoRoot}/packages/react-native`; - const jsiFolder = `${reactNativePackagePath}/ReactCommon/jsi`; - const hermesCoreSourceFolder = `${reactNativePackagePath}/sdks/hermes`; - - if (!fs.existsSync(hermesCoreSourceFolder)) { - console.info('The Hermes source folder is missing. Downloading...'); - downloadHermesSourceTarball(); - expandHermesSourceTarball(); - } - - // need to move the scripts inside the local hermes cloned folder - // cp sdks/hermes-engine/utils/*.sh /utils/. - cp( - `${reactNativePackagePath}/sdks/hermes-engine/utils/*.sh`, - `${reactNativePackagePath}/sdks/hermes/utils/.`, - ); - - // for this scenario, we only need to create the debug build - // (env variable PRODUCTION defines that podspec side) - const buildTypeiOSArtifacts = 'Debug'; - - // the android ones get set into /private/tmp/maven-local - const localMavenPath = '/private/tmp/maven-local'; + const localNodeTGZPath = `${reactNativePackagePath}/react-native-${releaseVersion}.tgz`; - // Generate native files for iOS - const tarballOutputPath = generateiOSArtifacts( - jsiFolder, - hermesCoreSourceFolder, - buildTypeiOSArtifacts, - localMavenPath, + const mavenLocalPath = + circleCIArtifacts != null + ? path.join(circleCIArtifacts.baseTmpPath(), 'maven-local.zip') + : '/private/tmp/maven-local'; + const hermesPath = await prepareArtifacts( + circleCIArtifacts, + mavenLocalPath, + localNodeTGZPath, + releaseVersion, + buildType, + reactNativePackagePath, ); - const localNodeTGZPath = `${reactNativePackagePath}/react-native-${releaseVersion}.tgz`; updateTemplatePackage({ 'react-native': `file:${localNodeTGZPath}`, }); // create locally the node module - exec('npm pack', {cwd: reactNativePackagePath}); + exec('npm pack --pack-destination ', {cwd: reactNativePackagePath}); + + // node pack does not creates a version of React Native with the right name on main. + // Let's add some defensive programming checks: + if (!fs.existsSync(localNodeTGZPath)) { + const tarfile = fs + .readdirSync(reactNativePackagePath) + .find(name => name.startsWith('react-native-') && name.endsWith('.tgz')); + if (!tarfile) { + throw new Error("Couldn't find a zipped version of react-native"); + } + exec( + `cp ${path.join(reactNativePackagePath, tarfile)} ${localNodeTGZPath}`, + ); + } pushd('/tmp/'); // need to avoid the pod install step - we'll do it later @@ -227,14 +249,14 @@ if (argv.target === 'RNTester') { // need to do this here so that Android will be properly setup either way exec( - 'echo "REACT_NATIVE_MAVEN_LOCAL_REPO=/private/tmp/maven-local" >> android/gradle.properties', + `echo "REACT_NATIVE_MAVEN_LOCAL_REPO=${mavenLocalPath}" >> android/gradle.properties`, ); // doing the pod install here so that it's easier to play around RNTestProject cd('ios'); exec('bundle install'); exec( - `HERMES_ENGINE_TARBALL_PATH=${tarballOutputPath} USE_HERMES=${ + `HERMES_ENGINE_TARBALL_PATH=${hermesPath} USE_HERMES=${ argv.hermes ? 1 : 0 } bundle exec pod install --ansi`, ); @@ -250,4 +272,34 @@ if (argv.target === 'RNTester') { popd(); } -exit(0); +async function main() { + /* + * see the test-local-e2e.js script for clean up process + */ + + // command order: we ask the user to select if they want to test RN tester + // or RNTestProject + + // if they select RN tester, we ask if iOS or Android, and then we run the tests + // if they select RNTestProject, we run the RNTestProject test + + checkPackagerRunning(); + + const branchName = exec('git rev-parse --abbrev-ref HEAD', { + silent: true, + }).stdout.trim(); + const onReleaseBranch = branchName.endsWith('-stable'); + + let circleCIArtifacts = await setupCircleCIArtifacts( + argv.circleciToken, + branchName, + ); + + if (argv.target === 'RNTester') { + await testRNTester(circleCIArtifacts, onReleaseBranch); + } else { + await testRNTestProject(circleCIArtifacts); + } +} + +main(); diff --git a/scripts/testing-utils.js b/scripts/testing-utils.js index c3ce4f78dde413..be4bc78b9c7f61 100644 --- a/scripts/testing-utils.js +++ b/scripts/testing-utils.js @@ -9,9 +9,23 @@ 'use strict'; -const {exec} = require('shelljs'); +const {exec, cp} = require('shelljs'); +const fs = require('fs'); const os = require('os'); const {spawn} = require('node:child_process'); +const path = require('path'); + +const circleCIArtifactsUtils = require('./circle-ci-artifacts-utils.js'); + +const { + generateAndroidArtifacts, + generateiOSArtifacts, +} = require('./release-utils'); + +const { + downloadHermesSourceTarball, + expandHermesSourceTarball, +} = require('../packages/react-native/scripts/hermes/hermes-utils.js'); /* * Android related utils - leverages android tooling @@ -35,12 +49,12 @@ const launchEmulator = emulatorName => { // from docs: "When using the detached option to start a long-running process, the process will not stay running in the background after the parent exits unless it is provided with a stdio configuration that is not connected to the parent. If the parent's stdio is inherited, the child will remain attached to the controlling terminal." // here: https://nodejs.org/api/child_process.html#optionsdetached - const cp = spawn(emulatorCommand, [`@${emulatorName}`], { + const child_process = spawn(emulatorCommand, [`@${emulatorName}`], { detached: true, stdio: 'ignore', }); - cp.unref(); + child_process.unref(); }; function tryLaunchEmulator() { @@ -69,7 +83,7 @@ function hasConnectedDevice() { } function maybeLaunchAndroidEmulator() { - if (hasConnectedDevice) { + if (hasConnectedDevice()) { console.info('Already have a device connected. Skip launching emulator.'); return; } @@ -112,11 +126,155 @@ function isPackagerRunning( // this is a very limited implementation of how this should work function launchPackagerInSeparateWindow(folderPath) { const command = `tell application "Terminal" to do script "cd ${folderPath} && yarn start"`; - exec(`osascript -e '${command}'`); + exec(`osascript -e '${command}' >/dev/null </utils/. + cp( + `${reactNativePackagePath}/sdks/hermes-engine/utils/*.sh`, + `${reactNativePackagePath}/sdks/hermes/utils/.`, + ); + + // for this scenario, we only need to create the debug build + // (env variable PRODUCTION defines that podspec side) + const buildTypeiOSArtifacts = 'Debug'; + + // the android ones get set into /private/tmp/maven-local + const localMavenPath = '/private/tmp/maven-local'; + + // Generate native files for iOS + const hermesPath = generateiOSArtifacts( + jsiFolder, + hermesCoreSourceFolder, + buildTypeiOSArtifacts, + localMavenPath, + ); + + return hermesPath; +} + +/** + * It prepares the artifacts required to run a new project created from the template + * + * Parameters: + * - @circleCIArtifacts manager object to manage all the download of CircleCIArtifacts. If null, it will fallback not to use them. + * - @mavenLocalPath path to the local maven repo that is needed by Android. + * - @localNodeTGZPath path where we want to store the react-native tgz. + * - @releaseVersion the version that is about to be released. + * - @buildType the type of build we want to execute if we build locally. + * - @reactNativePackagePath the path to the react native package within the repo. + * + * Returns: + * - @hermesPath the path to hermes for iOS + */ +async function prepareArtifacts( + circleCIArtifacts, + mavenLocalPath, + localNodeTGZPath, + releaseVersion, + buildType, + reactNativePackagePath, +) { + return circleCIArtifacts != null + ? await downloadArtifactsFromCircleCI( + circleCIArtifacts, + mavenLocalPath, + localNodeTGZPath, + ) + : buildArtifactsLocally(releaseVersion, buildType, reactNativePackagePath); } module.exports = { + checkPackagerRunning, maybeLaunchAndroidEmulator, isPackagerRunning, launchPackagerInSeparateWindow, + setupCircleCIArtifacts, + prepareArtifacts, };