chore(ci): E2E RN Version Matrix (#2735)

.github/workflows/e2e.yml

@@ -21,7 +21,7 @@ jobs:
         with:
           access_token: ${{ github.token }}
 
-  device-test:
+  sample:
     # Android emulator said to perform best with macos HAXM
     runs-on: macos-latest
     strategy:
@@ -33,10 +33,6 @@ jobs:
           - platform: ios
             runtime: '16.0'
             device: 'iPhone 14'
-    env:
-      PLATFORM: ${{ matrix.platform }}
-      RUNTIME: ${{ matrix.runtime }}
-      DEVICE: ${{ matrix.device }}
     steps:
       - uses: actions/checkout@v3
 
@@ -45,9 +41,6 @@ jobs:
           java-version: '11'
           distribution: 'adopt'
 
-      - name: Install Global Dependencies
-        run: yarn global add @sentry/cli yalc
-
       - name: NPM cache
         uses: actions/cache@v3
         id: deps-cache
@@ -74,28 +67,16 @@ jobs:
       - name: Build SDK
         run: yarn build
 
-      - name: Package SDK
-        run: yalc publish
-
-      - name: Prepare sample for testing
-        working-directory: ./sample
-        run: sh ./scripts/prepareConfigsForTesting.sh
-
-      - name: Install SDK in sample
-        working-directory: ./sample
-        run: yalc add @sentry/react-native
-
       - name: Install Sample Dependencies
         if: steps.deps-cache.outputs['cache-hit'] != 'true'
         working-directory: ./sample
         run: yarn install
 
       - run: pod install
         if: ${{ matrix.platform == 'ios' }}
-        working-directory: ./sample/ios
         env:
-          # TEST env var is used in podfile to determine whether to include the sentry SDK from relative path or node_modules.
-          TEST: true
+          PRODUCTION: 1
+        working-directory: ./sample/ios
 
       - name: Build ${{ matrix.platform }} sample app
         if: env.SENTRY_AUTH_TOKEN != null
@@ -129,46 +110,6 @@ jobs:
               build
           fi
 
-      - name: Start Appium Server
-        working-directory: ./sample
-        run: yarn run appium --log-timestamp --log-no-colors --log appium.${{ matrix.platform }}.log &
-
-        # Wait until the Appium server starts.
-      - name: Check Appium Server
-        uses: nick-fields/retry@943e742917ac94714d2f408a0e8320f2d1fcafcd # pin@v2
-        with:
-          timeout_seconds: 60
-          max_attempts: 10
-          command: curl --output /dev/null --silent --head --fail http://127.0.0.1:4723/sessions
-
-      - name: Run tests on Android
-        if: ${{ matrix.platform == 'android' }}
-        uses: reactivecircus/android-emulator-runner@50986b1464923454c95e261820bc626f38490ec0 # pin@v2
-        with:
-          api-level: 29
-          emulator-options: -accel on -no-snapshot -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none -camera-front none -timezone US/Pacific
-          working-directory: sample
-          script: |
-            # Collect logs
-            adb logcat '*:D' 2>&1 >adb.log &
-            adb devices -l
-
-            yarn test --verbose
-
-      - name: Run tests on iOS
-        if: ${{ matrix.platform == 'ios' }}
-        working-directory: ./sample
-        run: yarn test --verbose
-
-      - name: Upload logs
-        if: ${{ always() }}
-        uses: actions/upload-artifact@v3
-        with:
-          name: ${{ matrix.platform }}-logs
-          path: |
-            ./sample/*.log
-            ./sample/*.png
-
   metrics:
     runs-on: macos-latest
     strategy:
@@ -325,13 +266,29 @@ jobs:
           path: node_modules
           key: ${{ github.workflow }}-${{ github.job }}-npm-${{ hashFiles('yarn.lock') }}
 
+      - name: NPM cache E2E Tests Library
+        uses: actions/cache@v3
+        id: deps-cache-e2e-library
+        with:
+          path: test/e2e/node_modules
+          key: ${{ github.workflow }}-${{ github.job }}-npm-${{ hashFiles('test/e2e/yarn.lock') }}
+
       - name: Install SDK JS Dependencies
         if: steps.deps-cache.outputs['cache-hit'] != 'true'
         run: yarn install
 
+      - name: Install E2E Tests Library JS Dependencies
+        if: steps.deps-cache.outputs['deps-cache-e2e-library'] != 'true'
+        working-directory: test/e2e
+        run: yarn install
+
       - name: Build SDK
         run: yarn build
 
+      - name: Build E2E Tests Library
+        working-directory: test/e2e
+        run: yarn build
+
       - name: Package SDK
         run: yalc publish
 
@@ -347,6 +304,10 @@ jobs:
         working-directory: test/react-native/versions/${{ matrix.rn-version }}/RnDiffApp
         run: yarn install
 
+      - name: Add E2E Tests Library to App
+        working-directory: test/react-native/versions/${{ matrix.rn-version }}/RnDiffApp
+        run: yarn add ../../../../e2e
+
       - name: Install App Pods
         if: matrix.platform == 'ios'
         working-directory: test/react-native/versions/${{ matrix.rn-version }}/RnDiffApp/ios
@@ -387,8 +348,8 @@ jobs:
         run: |
           [[ "${{ matrix.build-type }}" == "production" ]] && CONFIG='Release' || CONFIG='Debug'
           echo "Building $CONFIG"
+          mkdir -p "DerivedData"
           derivedData="$(cd "DerivedData" ; pwd -P)"
-          mkdir -p "$derivedData"
           xcodebuild \
             -workspace RnDiffApp.xcworkspace \
             -configuration "$CONFIG" \
@@ -398,3 +359,157 @@ jobs:
             -sdk 'iphonesimulator${{ matrix.runtime }}' \
             -derivedDataPath "$derivedData" \
             build
+
+      - name: Archive Android APK
+        if: matrix.platform == 'android' && matrix.build-type == 'production'
+        run: |
+          BUILD_PATH=test/react-native/versions/${{ matrix.rn-version }}/RnDiffApp/android/app/build/outputs/apk/release
+          BUILD_NAME=app-release.apk
+          tar -cvf apk.tar -C $BUILD_PATH $BUILD_NAME
+
+      - name: Archive iOS APP
+        if: matrix.platform == 'ios' && matrix.build-type == 'production'
+        run: |
+          BUILD_PATH=test/react-native/versions/${{ matrix.rn-version }}/RnDiffApp/ios/DerivedData/Build/Products/Release-iphonesimulator
+          BUILD_NAME=RnDiffApp.app
+          tar -cvf app.tar -C $BUILD_PATH $BUILD_NAME
+
+      - name: Upload App APK
+        if: matrix.platform == 'android' && matrix.build-type == 'production'
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ matrix.rn-version }}-${{ matrix.rn-architecture }}-${{ matrix.platform }}-${{ matrix.build-type }}-app-package
+          path: apk.tar
+          retention-days: 1
+
+      - name: Upload App APP
+        if: matrix.platform == 'ios' && matrix.build-type == 'production'
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ matrix.rn-version }}-${{ matrix.rn-architecture }}-${{ matrix.platform }}-${{ matrix.build-type }}-app-package
+          path: app.tar
+          retention-days: 1
+
+  react-native-test:
+    name: Test RN ${{ matrix.rn-version }} ${{ matrix.rn-architecture }} ${{ matrix.platform }} ${{ matrix.build-type }}
+    needs: react-native-build
+    runs-on: macos-latest
+    strategy:
+      fail-fast: false # keeps matrix running if one fails
+      matrix:
+        rn-version: ['0.65.3', '0.70.6']
+        rn-architecture: ['legacy', 'new']
+        platform: ['android', 'ios']
+        build-type: ['production']
+        include:
+          - platform: ios
+            runtime: '16.0'
+            device: 'iPhone 14'
+        # exclude all rn versions lower than 0.70.0 for new architecture
+        exclude:
+          - rn-version: '0.65.3'
+            rn-architecture: 'new'
+    env:
+      PLATFORM: ${{ matrix.platform }}
+      RUNTIME: ${{ matrix.runtime }}
+      DEVICE: ${{ matrix.device }}
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Download App Package
+        if: matrix.build-type == 'production'
+        uses: actions/download-artifact@v3
+        with:
+          name: ${{ matrix.rn-version }}-${{ matrix.rn-architecture }}-${{ matrix.platform }}-${{ matrix.build-type }}-app-package
+          path: test/e2e
+
+      - name: Extract App Package
+        working-directory: test/e2e
+        run: tar -xvf *.tar
+
+      - uses: actions/setup-java@v3
+        with:
+          java-version: '11'
+          distribution: 'adopt'
+
+      - name: NPM cache E2E Tests Library
+        uses: actions/cache@v3
+        id: deps-cache-e2e-library
+        with:
+          path: test/e2e/node_modules
+          key: ${{ github.workflow }}-${{ github.job }}-npm-${{ hashFiles('test/e2e/yarn.lock') }}
+
+      - name: Install E2E Tests Library JS Dependencies
+        if: steps.deps-cache.outputs['deps-cache-e2e-library'] != 'true'
+        working-directory: test/e2e
+        run: yarn install
+
+      - name: Build iOS WebDriverAgent
+        if: matrix.platform == 'ios'
+        working-directory: test/e2e
+        run: |
+          mkdir -p "DerivedData"
+          derivedData="$(cd "DerivedData" ; pwd -P)"
+          xcodebuild \
+            -project node_modules/appium-webdriveragent/WebDriverAgent.xcodeproj \
+            -scheme WebDriverAgentRunner \
+            GCC_TREAT_WARNINGS_AS_ERRORS=0 \
+            COMPILER_INDEX_STORE_ENABLE=NO \
+            -destination 'platform=iOS Simulator,name=${{ matrix.device }}' \
+            ONLY_ACTIVE_ARCH=yes \
+            -sdk 'iphonesimulator${{ matrix.runtime }}' \
+            -derivedDataPath "$derivedData" \
+            build
+
+      - name: Start Appium Server
+        working-directory: test/e2e
+        run: yarn run appium --log-timestamp --log-no-colors --log appium.${{ matrix.platform }}.log &
+
+        # Wait until the Appium server starts.
+      - name: Check Appium Server
+        uses: nick-fields/retry@943e742917ac94714d2f408a0e8320f2d1fcafcd # pin@v2
+        with:
+          timeout_seconds: 60
+          max_attempts: 10
+          command: curl --output /dev/null --silent --head --fail http://127.0.0.1:4723/sessions
+
+      - name: Run tests on Android
+        if: ${{ matrix.platform == 'android' }}
+        env:
+          APPIUM_APP: ./app-release.apk
+        uses: reactivecircus/android-emulator-runner@50986b1464923454c95e261820bc626f38490ec0 # pin@v2
+        with:
+          working-directory: test/e2e
+          api-level: 29
+          emulator-options: >
+            -accel on
+            -no-snapshot
+            -gpu swiftshader_indirect
+            -noaudio
+            -no-boot-anim
+            -camera-back none
+            -camera-front none
+            -timezone US/Pacific
+          script: |
+            # Collect logs
+            adb logcat '*:D' 2>&1 >adb.log &
+            adb devices -l
+
+            yarn test --verbose
+
+      - name: Run tests on iOS
+        if: ${{ matrix.platform == 'ios' }}
+        working-directory: test/e2e
+        env:
+          APPIUM_APP: ./RnDiffApp.app
+          APPIUM_DERIVED_DATA: DerivedData
+        run: yarn test --verbose
+
+      - name: Upload logs
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ matrix.rn-version }}-${{ matrix.rn-architecture }}-${{ matrix.platform }}-${{ matrix.build-type }}-logs
+          path: |
+            test/e2e/*.log
+            test/e2e/*.png

package.json

@@ -97,7 +97,8 @@
       "js"
     ],
     "testPathIgnorePatterns": [
-      "<rootDir>/sample/"
+      "<rootDir>/sample/",
+      "<rootDir>/test/e2e/"
     ],
     "testEnvironment": "node",
     "testMatch": [

sample/ios/Podfile

@@ -18,9 +18,7 @@ target 'sample' do
     :app_path => "#{Pod::Config.instance.installation_root}/.."
   )
 
-  if !ENV['TEST']
-    pod 'RNSentry', :path => '../../RNSentry.podspec'
-  end
+  pod 'RNSentry', :path => '../../RNSentry.podspec'
 
   target 'sampleTests' do
     inherit! :complete

sample/src/App.tsx

@@ -13,7 +13,6 @@ import HomeScreen from './screens/HomeScreen';
 import TrackerScreen from './screens/TrackerScreen';
 import ManualTrackerScreen from './screens/ManualTrackerScreen';
 import PerformanceTimingScreen from './screens/PerformanceTimingScreen';
-import EndToEndTestsScreen from './screens/EndToEndTestsScreen';
 import ReduxScreen from './screens/ReduxScreen';
 
 import {store} from './reduxApp';
@@ -95,7 +94,6 @@ const App = () => {
             component={PerformanceTimingScreen}
           />
           <Stack.Screen name="Redux" component={ReduxScreen} />
-          <Stack.Screen name="EndToEndTests" component={EndToEndTestsScreen} />
         </Stack.Navigator>
       </NavigationContainer>
     </Provider>

src/js/tools/ModulesCollector.ts

@@ -4,6 +4,7 @@ import { posix, sep } from 'path';
 
 logger.enable();
 
+// eslint-disable-next-line @typescript-eslint/unbound-method
 const { dirname, join, resolve, sep: posixSep } = posix;
 
 interface Package {

test/e2e/.gitignore

@@ -0,0 +1,2 @@
+*.png
+*.log