feat: Reanimated v3 support in RNVisionCamera v2

.eslintrc.js

@@ -10,9 +10,9 @@ module.exports = {
     ecmaVersion: 2018,
     sourceType: 'module',
   },
-  ignorePatterns: ['scripts', 'lib', 'docs', 'example', 'app.plugin.js'],
+  ignorePatterns: ['scripts', 'lib', 'docs', 'example', 'app.plugin.js', '.eslintrc.js'],
   plugins: ['@typescript-eslint'],
-  extends: ['plugin:@typescript-eslint/recommended', '@react-native-community'],
+  extends: ['plugin:@typescript-eslint/recommended', '@react-native'],
   rules: {
     // eslint
     semi: 'off',

.github/workflows/validate-cpp.yml

@@ -24,7 +24,7 @@ jobs:
         with:
           github_token: ${{ secrets.github_token }}
           reporter: github-pr-review
-          flags: --linelength=230 --exclude "android/src/main/cpp/reanimated-headers"
+          flags: --linelength=230
           targets: --recursive cpp android/src/main/cpp
           filter: "-legal/copyright\
             ,-readability/todo\

.gitignore

@@ -20,6 +20,8 @@ build/
 *.perspectivev3
 !default.perspectivev3
 xcuserdata
+ios/.xcode.env.local
+
 *.xccheckout
 *.moved-aside
 DerivedData
@@ -39,6 +41,10 @@ android.iml
 # Cocoapods
 #
 example/ios/Pods
+/vendor/bundle/
+
+# Temporary files created by Metro to check the health of the file watcher
+.metro-health-check*
 
 # node.js
 #
@@ -68,3 +74,7 @@ docs/docs/api
 # External native build folder generated in Android Studio 2.2 and later
 .externalNativeBuild
 .cxx/
+
+.cxx/
+34	*.keystore
+35	!debug.keystore

android/CMakeLists.txt

@@ -60,15 +60,6 @@ if(${REACT_NATIVE_VERSION} GREATER_EQUAL 71)
                 "${NODE_MODULES_DIR}/react-native/ReactCommon/react/renderer/graphics/platform/cxx"
                 "${NODE_MODULES_DIR}/react-native/ReactCommon/runtimeexecutor"
                 "${NODE_MODULES_DIR}/react-native/ReactCommon/yoga"
-                # --- Reanimated ---
-                # New
-                "${NODE_MODULES_DIR}/react-native-reanimated/Common/cpp/AnimatedSensor"
-                "${NODE_MODULES_DIR}/react-native-reanimated/Common/cpp/Tools"
-                "${NODE_MODULES_DIR}/react-native-reanimated/Common/cpp/SpecTools"
-                "${NODE_MODULES_DIR}/react-native-reanimated/Common/cpp/SharedItems"
-                "${NODE_MODULES_DIR}/react-native-reanimated/Common/cpp/Registries"
-                "${NODE_MODULES_DIR}/react-native-reanimated/Common/cpp/LayoutAnimations"
-                "${NODE_MODULES_DIR}/react-native-reanimated/Common/cpp/hidden_headers"
                 "src/main/cpp"
         )
 else()
@@ -95,22 +86,6 @@ else()
                 "${NODE_MODULES_DIR}/hermes-engine/android/include/"
                 ${INCLUDE_JSI_CPP} # only on older RN versions
                 ${INCLUDE_JSIDYNAMIC_CPP} # only on older RN versions
-                # --- Reanimated ---
-                # Old
-                "${NODE_MODULES_DIR}/react-native-reanimated/Common/cpp/headers/AnimatedSensor"
-                "${NODE_MODULES_DIR}/react-native-reanimated/Common/cpp/headers/Tools"
-                "${NODE_MODULES_DIR}/react-native-reanimated/Common/cpp/headers/SpecTools"
-                "${NODE_MODULES_DIR}/react-native-reanimated/Common/cpp/headers/SharedItems"
-                "${NODE_MODULES_DIR}/react-native-reanimated/Common/cpp/headers/Registries"
-                "${NODE_MODULES_DIR}/react-native-reanimated/Common/cpp/headers/LayoutAnimations"
-                # New
-                "${NODE_MODULES_DIR}/react-native-reanimated/Common/cpp/AnimatedSensor"
-                "${NODE_MODULES_DIR}/react-native-reanimated/Common/cpp/Tools"
-                "${NODE_MODULES_DIR}/react-native-reanimated/Common/cpp/SpecTools"
-                "${NODE_MODULES_DIR}/react-native-reanimated/Common/cpp/SharedItems"
-                "${NODE_MODULES_DIR}/react-native-reanimated/Common/cpp/Registries"
-                "${NODE_MODULES_DIR}/react-native-reanimated/Common/cpp/LayoutAnimations"
-                "${NODE_MODULES_DIR}/react-native-reanimated/Common/cpp/hidden_headers"
                 "src/main/cpp"
         )
 endif()
@@ -154,7 +129,6 @@ if(${FOR_HERMES})
                         "${BUILD_DIR}/third-party-ndk/hermes/jni/${ANDROID_ABI}/libhermes.so"
                 )
         endif()
-        file (GLOB LIBREANIMATED_DIR "${BUILD_DIR}/react-native-reanimated-*-hermes.aar/jni/${ANDROID_ABI}")
 else()
         file (GLOB LIBJSC_DIR "${BUILD_DIR}/android-jsc*.aar/jni/${ANDROID_ABI}")
 
@@ -173,9 +147,6 @@ else()
                 ${PACKAGE_NAME}
                 ${JS_ENGINE_LIB}
         )
-
-        # Use Reanimated JSC
-        file (GLOB LIBREANIMATED_DIR "${BUILD_DIR}/react-native-reanimated-*-jsc.aar/jni/${ANDROID_ABI}")
 endif()
 
 if(${REACT_NATIVE_VERSION} LESS 71)
@@ -234,12 +205,7 @@ else()
         )
 endif()
 
-find_library(
-        REANIMATED_LIB
-        reanimated
-        PATHS ${LIBREANIMATED_DIR}
-        NO_CMAKE_FIND_ROOT_PATH
-)
+find_package(react-native-reanimated REQUIRED CONFIG)
 
 find_library(
         LOG_LIB
@@ -252,7 +218,7 @@ target_link_libraries(
         ${PACKAGE_NAME}
         ${LOG_LIB}
         ${JSI_LIB}
-        ${REANIMATED_LIB}
+        react-native-reanimated::reanimated
         ${REACT_NATIVE_JNI_LIB}
         ${FBJNI_LIB}
         ${FOLLY_LIB}

android/build.gradle

@@ -111,9 +111,10 @@ def resolveBuildType() {
 }
 
 // plugin.js file only exists since REA v2.
-def hasReanimated2 = file("${nodeModules}/react-native-reanimated/plugin.js").exists()
-def disableFrameProcessors = rootProject.ext.has("disableFrameProcessors") ? rootProject.ext.get("disableFrameProcessors").asBoolean() : false
-def ENABLE_FRAME_PROCESSORS = hasReanimated2 && !disableFrameProcessors
+// def hasReanimated2 = file("${nodeModules}/react-native-reanimated/plugin.js").exists()
+// def disableFrameProcessors = rootProject.ext.has("disableFrameProcessors") ? rootProject.ext.get("disableFrameProcessors").asBoolean() : false
+// def ENABLE_FRAME_PROCESSORS = hasReanimated2 && !disableFrameProcessors
+def ENABLE_FRAME_PROCESSORS = true
 
 if (ENABLE_FRAME_PROCESSORS) {
   logger.warn("VisionCamera: Frame Processors are enabled! Building C++ part...")
@@ -303,10 +304,6 @@ dependencies {
       def jscAAR = fileTree("${nodeModules}/jsc-android/dist/org/webkit/android-jsc").matching({ it.include "**/**/*.aar" }).singleFile
       extractJNI(files(rnAAR, jscAAR))
     }
-
-    def jsEngine = FOR_HERMES ? "hermes" : "jsc"
-    def reaAAR = "${nodeModules}/react-native-reanimated/android/react-native-reanimated-${REACT_NATIVE_VERSION}-${jsEngine}.aar"
-    extractJNI(files(reaAAR))
   }
 
   implementation "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version"

android/gradle.properties

@@ -13,7 +13,7 @@
 #Fri Feb 19 20:46:14 CET 2021
 VisionCamera_buildToolsVersion=30.0.0
 VisionCamera_compileSdkVersion=31
-VisionCamera_kotlinVersion=1.5.30
+VisionCamera_kotlinVersion=1.6.20
 VisionCamera_targetSdkVersion=31
 VisionCamera_ndkVersion=21.4.7075529
 android.enableJetifier=true

android/gradle/wrapper/gradle-wrapper.properties

@@ -1,5 +1,5 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-6.9-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-7.5.1-all.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

android/src/main/cpp/FrameProcessorRuntimeManager.cpp

@@ -8,12 +8,6 @@
 #include <utility>
 #include <string>
 
-#include "RuntimeDecorator.h"
-#include "RuntimeManager.h"
-#include "reanimated-headers/AndroidScheduler.h"
-#include "reanimated-headers/AndroidErrorHandler.h"
-
-#include "MakeJSIRuntime.h"
 #include "CameraView.h"
 #include "FrameHostObject.h"
 #include "JSIJNIConversion.h"
@@ -55,7 +49,6 @@ TSelf vision::FrameProcessorRuntimeManager::initHybrid(
   auto runtime = reinterpret_cast<jsi::Runtime*>(jsRuntimePointer);
   auto jsCallInvoker = jsCallInvokerHolder->cthis()->getCallInvoker();
   auto scheduler = std::shared_ptr<VisionCameraScheduler>(androidScheduler->cthis());
-  scheduler->setJSCallInvoker(jsCallInvoker);
 
   return makeCxxInstance(jThis, runtime, jsCallInvoker, scheduler);
 }
@@ -65,16 +58,8 @@ void vision::FrameProcessorRuntimeManager::initializeRuntime() {
                       "Initializing Vision JS-Runtime...");
 
   // create JSI runtime and decorate it
-  auto runtime = makeJSIRuntime();
-  reanimated::RuntimeDecorator::decorateRuntime(*runtime, "FRAME_PROCESSOR");
-  runtime->global().setProperty(*runtime, "_FRAME_PROCESSOR",
-                                jsi::Value(true));
 
   // create REA runtime manager
-  auto errorHandler = std::make_shared<reanimated::AndroidErrorHandler>(scheduler_);
-  _runtimeManager = std::make_unique<reanimated::RuntimeManager>(std::move(runtime),
-                                                                 errorHandler,
-                                                                 scheduler_);
 
   __android_log_write(ANDROID_LOG_INFO, TAG,
                       "Initialized Vision JS-Runtime!");
@@ -86,6 +71,11 @@ global_ref<CameraView::javaobject> FrameProcessorRuntimeManager::findCameraViewB
   return make_global(weakCameraView);
 }
 
+void FrameProcessorRuntimeManager::registerPlugins() {
+  static const auto registerPluginsMethod = javaPart_->getClass()->getMethod<void()>("registerPlugins");
+  registerPluginsMethod(javaPart_.get());
+}
+
 void FrameProcessorRuntimeManager::logErrorToJS(const std::string& message) {
   if (!this->jsCallInvoker_) {
     return;
@@ -105,16 +95,18 @@ void FrameProcessorRuntimeManager::logErrorToJS(const std::string& message) {
   });
 }
 
-void FrameProcessorRuntimeManager::setFrameProcessor(jsi::Runtime& runtime,
+void FrameProcessorRuntimeManager::setFrameProcessor(jsi::Runtime& rnRuntime,
                                                      int viewTag,
-                                                     const jsi::Value& frameProcessor) {
+                                                     const jsi::Value& frameProcessor,
+                                                     const jsi::Value& workletRuntimeValue) {
   __android_log_write(ANDROID_LOG_INFO, TAG,
                       "Setting new Frame Processor...");
 
-  if (!_runtimeManager || !_runtimeManager->runtime) {
-    throw jsi::JSError(runtime,
-                       "setFrameProcessor(..): VisionCamera's RuntimeManager is not yet initialized!");
-  }
+  workletRuntime_ = reanimated::extractWorkletRuntime(rnRuntime, workletRuntimeValue);
+  jsi::Runtime &visionRuntime = workletRuntime_->getJSIRuntime();
+  visionRuntime.global().setProperty(visionRuntime, "_FRAME_PROCESSOR", jsi::Value(true));
+
+  registerPlugins();
 
   // find camera view
   auto cameraView = findCameraViewById(viewTag);
@@ -123,27 +115,18 @@ void FrameProcessorRuntimeManager::setFrameProcessor(jsi::Runtime& runtime,
   // convert jsi::Function to a ShareableValue (can be shared across runtimes)
   __android_log_write(ANDROID_LOG_INFO, TAG,
                       "Adapting Shareable value from function (conversion to worklet)...");
-  auto worklet = reanimated::ShareableValue::adapt(runtime,
-                                                   frameProcessor,
-                                                   _runtimeManager.get());
+  auto shareableWorklet = reanimated::extractShareableOrThrow<reanimated::ShareableWorklet>(rnRuntime, frameProcessor);
   __android_log_write(ANDROID_LOG_INFO, TAG, "Successfully created worklet!");
 
   scheduler_->scheduleOnUI([=]() {
       // cast worklet to a jsi::Function for the new runtime
-      auto& rt = *_runtimeManager->runtime;
-      auto function = std::make_shared<jsi::Function>(worklet->getValue(rt).asObject(rt).asFunction(rt));
-
       // assign lambda to frame processor
-      cameraView->cthis()->setFrameProcessor([this, &rt, function](jni::alias_ref<JImageProxy::javaobject> frame) {
-          try {
-            // create HostObject which holds the Frame (JImageProxy)
-            auto hostObject = std::make_shared<FrameHostObject>(frame);
-            function->callWithThis(rt, *function, jsi::Object::createFromHostObject(rt, hostObject));
-          } catch (jsi::JSError& jsError) {
-            auto message = "Frame Processor threw an error: " + jsError.getMessage();
-            __android_log_write(ANDROID_LOG_ERROR, TAG, message.c_str());
-            this->logErrorToJS(message);
-          }
+      cameraView->cthis()->setFrameProcessor([=](jni::alias_ref<JImageProxy::javaobject> frame) {
+          // create HostObject which holds the Frame (JImageProxy)
+          auto frameHostObject = std::make_shared<FrameHostObject>(frame);
+          jsi::Runtime &runtime = workletRuntime_->getJSIRuntime();
+          auto hostObject = jsi::Object::createFromHostObject(runtime, frameHostObject);
+          workletRuntime_->runGuarded(shareableWorklet, hostObject);
       });
 
       __android_log_write(ANDROID_LOG_INFO, TAG, "Frame Processor set!");
@@ -189,10 +172,15 @@ void FrameProcessorRuntimeManager::installJSIBindings() {
       throw jsi::JSError(runtime,
                          "Camera::setFrameProcessor: Second argument ('frameProcessor') must be a function!");
     }
+    if (!arguments[2].isObject()) {
+      throw jsi::JSError(runtime,
+                         "Camera::setFrameProcessor: Third argument ('workletRuntime') must be an object!");
+    }
 
     double viewTag = arguments[0].asNumber();
     const jsi::Value& frameProcessor = arguments[1];
-    this->setFrameProcessor(runtime, static_cast<int>(viewTag), frameProcessor);
+    const jsi::Value& workletRuntimeValue = arguments[2];
+    this->setFrameProcessor(runtime, static_cast<int>(viewTag), frameProcessor, workletRuntimeValue);
 
     return jsi::Value::undefined();
   };
@@ -235,11 +223,11 @@ void FrameProcessorRuntimeManager::installJSIBindings() {
 
 void FrameProcessorRuntimeManager::registerPlugin(alias_ref<JFrameProcessorPlugin::javaobject> plugin) {
   // _runtimeManager might never be null, but we can never be too sure.
-  if (!_runtimeManager || !_runtimeManager->runtime) {
+  if (!workletRuntime_) {
     throw std::runtime_error("Tried to register plugin before initializing JS runtime! Call `initializeRuntime()` first.");
   }
 
-  auto& runtime = *_runtimeManager->runtime;
+  auto& runtime = workletRuntime_->getJSIRuntime();
 
   // we need a strong reference on the plugin, make_global does that.
   auto pluginGlobal = make_global(plugin);

android/src/main/cpp/FrameProcessorRuntimeManager.h

@@ -10,8 +10,7 @@
 #include <memory>
 #include <string>
 
-#include "RuntimeManager.h"
-#include "reanimated-headers/AndroidScheduler.h"
+#include "WorkletRuntime.h"
 
 #include "CameraView.h"
 #include "VisionCameraScheduler.h"
@@ -46,18 +45,20 @@ class FrameProcessorRuntimeManager : public jni::HybridClass<FrameProcessorRunti
   jni::global_ref<FrameProcessorRuntimeManager::javaobject> javaPart_;
   jsi::Runtime* runtime_;
   std::shared_ptr<facebook::react::CallInvoker> jsCallInvoker_;
-  std::shared_ptr<reanimated::RuntimeManager> _runtimeManager;
+  std::shared_ptr<reanimated::WorkletRuntime> workletRuntime_;
   std::shared_ptr<vision::VisionCameraScheduler> scheduler_;
 
   jni::global_ref<CameraView::javaobject> findCameraViewById(int viewId);
+  void registerPlugins();
   void initializeRuntime();
   void installJSIBindings();
   void registerPlugin(alias_ref<JFrameProcessorPlugin::javaobject> plugin);
   void logErrorToJS(const std::string& message);
 
   void setFrameProcessor(jsi::Runtime& runtime,                 // NOLINT(runtime/references)
                          int viewTag,
-                         const jsi::Value& frameProcessor);
+                         const jsi::Value& frameProcessor,
+                         const jsi::Value& workletRuntimeValue);
   void unsetFrameProcessor(int viewTag);
 };