feat: Create C++/OpenGL-based Video Pipeline for more efficient Recording and Frame Processing

android/CMakeLists.txt

@@ -7,10 +7,6 @@ set(BUILD_DIR ${CMAKE_SOURCE_DIR}/build)
 set(CMAKE_VERBOSE_MAKEFILE ON)
 set(CMAKE_CXX_STANDARD 17)
 
-# Folly
-include("${NODE_MODULES_DIR}/react-native/ReactAndroid/cmake-utils/folly-flags.cmake")
-add_compile_options(${folly_FLAGS})
-
 # Third party libraries (Prefabs)
 find_package(ReactAndroid REQUIRED CONFIG)
 find_package(fbjni REQUIRED CONFIG)
@@ -25,6 +21,7 @@ add_library(
         SHARED
         ../cpp/JSITypedArray.cpp
         src/main/cpp/VisionCamera.cpp
+        src/main/cpp/VideoPipeline.cpp
         # Frame Processor
         src/main/cpp/frameprocessor/FrameHostObject.cpp
         src/main/cpp/frameprocessor/FrameProcessorPluginHostObject.cpp
@@ -60,7 +57,6 @@ target_link_libraries(
         android                             # <-- Android JNI core
         ReactAndroid::jsi                   # <-- RN: JSI
         ReactAndroid::reactnativejni        # <-- RN: React Native JNI bindings
-        ReactAndroid::folly_runtime         # <-- RN: For casting JSI <> Java objects
         fbjni::fbjni                        # <-- fbjni
 )
 

android/build.gradle

@@ -93,7 +93,7 @@ android {
   }
 
   defaultConfig {
-    minSdkVersion safeExtGet('minSdkVersion', 21)
+    minSdkVersion safeExtGet('minSdkVersion', 26)
     compileSdkVersion safeExtGet('compileSdkVersion', 33)
     targetSdkVersion safeExtGet('targetSdkVersion', 33)
     versionCode 1

android/gradle.properties

@@ -14,10 +14,6 @@ org.gradle.configureondemand=true
 # http://www.gradle.org/docs/current/userguide/multi_project_builds.html#sec:decoupled_projects
 # org.gradle.parallel=true
 #Fri Feb 19 20:46:14 CET 2021
-VisionCamera_buildToolsVersion=30.0.0
-VisionCamera_compileSdkVersion=31
 VisionCamera_kotlinVersion=1.7.20
-VisionCamera_targetSdkVersion=31
-VisionCamera_ndkVersion=21.4.7075529
 android.enableJetifier=true
 android.useAndroidX=true

android/src/main/cpp/VideoPipeline.cpp

@@ -0,0 +1,131 @@
+//
+// Created by Marc Rousavy on 25.08.23.
+//
+
+#include "VideoPipeline.h"
+#include "OpenGLError.h"
+
+namespace vision {
+
+jni::local_ref<VideoPipeline::jhybriddata> VideoPipeline::initHybrid(jni::alias_ref<jhybridobject> jThis) {
+  return makeCxxInstance(jThis);
+}
+
+VideoPipeline::VideoPipeline(jni::alias_ref<jhybridobject> jThis): _javaPart(jni::make_global(jThis)) { }
+
+VideoPipeline::~VideoPipeline() {
+  if (_context.display != EGL_NO_DISPLAY) {
+    eglMakeCurrent(_context.display, EGL_NO_SURFACE, EGL_NO_SURFACE, EGL_NO_CONTEXT);
+    if (_context.surface != EGL_NO_SURFACE) {
+      __android_log_print(ANDROID_LOG_INFO, TAG, "Destroying OpenGL Surface...");
+      eglDestroySurface(_context.display, _context.surface);
+      _context.surface = EGL_NO_SURFACE;
+    }
+    if (_context.context != EGL_NO_CONTEXT) {
+      __android_log_print(ANDROID_LOG_INFO, TAG, "Destroying OpenGL Context...");
+      eglDestroyContext(_context.display, _context.context);
+      _context.context = EGL_NO_CONTEXT;
+    }
+    __android_log_print(ANDROID_LOG_INFO, TAG, "Destroying OpenGL Display...");
+    eglTerminate(_context.display);
+    _context.display = EGL_NO_DISPLAY;
+  }
+}
+
+void VideoPipeline::setSize(int width, int height) {
+  _width = width;
+  _height = height;
+}
+
+GLContext& VideoPipeline::getGLContext() {
+  bool successful;
+  // EGLDisplay
+  if (_context.display == EGL_NO_DISPLAY) {
+    __android_log_print(ANDROID_LOG_INFO, TAG, "Initializing EGLDisplay..");
+    _context.display = eglGetDisplay(EGL_DEFAULT_DISPLAY);
+    if (_context.display == EGL_NO_DISPLAY) throw OpenGLError("Failed to get default OpenGL Display!");
+
+    EGLint major;
+    EGLint minor;
+    successful = eglInitialize(_context.display, &major, &minor);
+    if (!successful) throw OpenGLError("Failed to initialize OpenGL!");
+  }
+
+  // EGLConfig
+  if (_context.config == nullptr) {
+    __android_log_print(ANDROID_LOG_INFO, TAG, "Initializing EGLConfig..");
+    EGLint attributes[] = {EGL_RENDERABLE_TYPE, EGL_OPENGL_ES2_BIT,
+                           EGL_SURFACE_TYPE, EGL_PBUFFER_BIT,
+                           EGL_ALPHA_SIZE, 8,
+                           EGL_BLUE_SIZE, 8,
+                           EGL_GREEN_SIZE, 8,
+                           EGL_RED_SIZE, 8,
+                           EGL_DEPTH_SIZE, 0,
+                           EGL_STENCIL_SIZE, 0,
+                           EGL_NONE};
+    EGLint numConfigs;
+    successful = eglChooseConfig(_context.display, attributes, &_context.config, 1, &numConfigs);
+    if (!successful || numConfigs == 0) throw OpenGLError("Failed to choose OpenGL config!");
+  }
+
+  // EGLContext
+  if (_context.context == EGL_NO_CONTEXT) {
+    __android_log_print(ANDROID_LOG_INFO, TAG, "Initializing EGLContext..");
+    EGLint contextAttributes[] = {EGL_CONTEXT_CLIENT_VERSION, 2, EGL_NONE};
+    _context.context = eglCreateContext(_context.display, _context.config, nullptr, contextAttributes);
+    if (_context.context == EGL_NO_CONTEXT) throw OpenGLError("Failed to create OpenGL context!");
+  }
+
+  // EGLSurface
+  if (_context.surface == EGL_NO_SURFACE) {
+    // If we don't have a surface at all
+    __android_log_print(ANDROID_LOG_INFO, TAG, "Initializing EGLSurface..");
+    EGLint attributes[] = {EGL_WIDTH, _width,
+                           EGL_HEIGHT, _height,
+                           EGL_NONE};
+    _context.surface = eglCreatePbufferSurface(_context.display, _context.config, attributes);
+  } else {
+    // If the surface size has changed we need to resize the surface
+    int currentWidth = 0, currentHeight = 0;
+    eglQuerySurface(_context.display, _context.surface, EGL_WIDTH, &currentWidth);
+    eglQuerySurface(_context.display, _context.surface, EGL_HEIGHT, &currentHeight);
+    if (currentWidth != _width || currentHeight != _height) {
+      __android_log_print(ANDROID_LOG_INFO, TAG, "Resizing EGLSurface from (%i x %i) to (%i x %i)..",
+                          currentWidth, currentHeight, _width, _height);
+      // Destroy current surface
+      eglDestroySurface(_context.display, _context.surface);
+      _context.surface = EGL_NO_SURFACE;
+      // Create new one with new dimensions
+      EGLint attributes[] = {EGL_WIDTH, _width,
+                             EGL_HEIGHT, _height,
+                             EGL_NONE};
+      _context.surface = eglCreatePbufferSurface(_context.display, _context.config, attributes);
+    }
+  }
+  if (_context.surface == EGL_NO_SURFACE) {
+    throw OpenGLError("Failed to create OpenGL Surface!");
+  }
+
+  successful = eglMakeCurrent(_context.display, _context.surface, _context.surface, _context.context);
+  if (!successful || eglGetError() != EGL_SUCCESS) throw OpenGLError("Failed to use current OpenGL context!");
+
+  return _context;
+}
+
+void VideoPipeline::setFrameProcessorOutputSurface(jobject surface) {
+
+}
+
+void VideoPipeline::setRecordingSessionOutputSurface(jobject surface, jint width, jint height) {
+
+}
+
+void VideoPipeline::registerNatives() {
+  registerHybrid({
+    makeNativeMethod("initHybrid", VideoPipeline::initHybrid),
+    makeNativeMethod("setFrameProcessorOutputSurface", VideoPipeline::setFrameProcessorOutputSurface),
+    makeNativeMethod("setRecordingSessionOutputSurface", VideoPipeline::setRecordingSessionOutputSurface),
+  });
+}
+
+} // vision

android/src/main/cpp/VideoPipeline.h

@@ -0,0 +1,53 @@
+//
+// Created by Marc Rousavy on 25.08.23.
+//
+
+#pragma once
+
+#include <jni.h>
+#include <fbjni/fbjni.h>
+#include <EGL/egl.h>
+
+namespace vision {
+
+using namespace facebook;
+
+struct GLContext {
+  EGLDisplay display = EGL_NO_DISPLAY;
+  EGLSurface surface = EGL_NO_SURFACE;
+  EGLContext context = EGL_NO_CONTEXT;
+  EGLConfig config = nullptr;
+};
+
+class VideoPipeline: public jni::HybridClass<VideoPipeline> {
+ public:
+  static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/utils/VideoPipeline;";
+  static jni::local_ref<jhybriddata> initHybrid(jni::alias_ref<jhybridobject> jThis);
+  static void registerNatives();
+
+ public:
+  ~VideoPipeline();
+  void setFrameProcessorOutputSurface(jobject surface);
+  void setRecordingSessionOutputSurface(jobject surface, jint width, jint height);
+
+ private:
+  // Private constructor. Use `create(..)` to create new instances.
+  explicit VideoPipeline(jni::alias_ref<jhybridobject> jThis);
+
+ private:
+  GLContext& getGLContext();
+  void setSize(int width, int height);
+
+ private:
+  GLContext _context;
+  int _width = 0;
+  int _height = 0;
+
+ private:
+  friend HybridBase;
+  jni::global_ref<javaobject> _javaPart;
+  static constexpr auto TAG = "VideoPipeline";
+};
+
+} // vision
+

android/src/main/cpp/VisionCamera.cpp

@@ -5,12 +5,14 @@
 #include "JVisionCameraProxy.h"
 #include "VisionCameraProxy.h"
 #include "SkiaRenderer.h"
+#include "VideoPipeline.h"
 
 JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM *vm, void *) {
   return facebook::jni::initialize(vm, [] {
     vision::VisionCameraInstaller::registerNatives();
     vision::JVisionCameraProxy::registerNatives();
     vision::JVisionCameraScheduler::registerNatives();
+    vision::VideoPipeline::registerNatives();
 #if VISION_CAMERA_ENABLE_FRAME_PROCESSORS
     vision::JFrameProcessor::registerNatives();
 #endif

android/src/main/cpp/frameprocessor/FrameHostObject.cpp

@@ -4,7 +4,6 @@
 
 #include "FrameHostObject.h"
 
-#include <android/log.h>
 #include <fbjni/fbjni.h>
 #include <jni.h>
 

android/src/main/cpp/frameprocessor/FrameHostObject.h

@@ -27,9 +27,6 @@ class JSI_EXPORT FrameHostObject : public jsi::HostObject {
 
  public:
   jni::global_ref<JFrame> frame;
-
- private:
-  static auto constexpr TAG = "VisionCamera";
 };
 
 } // namespace vision

android/src/main/cpp/frameprocessor/java-bindings/JVisionCameraScheduler.cpp

@@ -22,8 +22,8 @@ void JVisionCameraScheduler::dispatchAsync(const std::function<void()>& job) {
 
 void JVisionCameraScheduler::scheduleTrigger() {
   // 2. schedule `triggerUI` to be called on the java thread
-  static auto method = javaPart_->getClass()->getMethod<void()>("scheduleTrigger");
-  method(javaPart_.get());
+  static auto method = _javaPart->getClass()->getMethod<void()>("scheduleTrigger");
+  method(_javaPart.get());
 }
 
 void JVisionCameraScheduler::trigger() {

android/src/main/cpp/frameprocessor/java-bindings/JVisionCameraScheduler.h

@@ -33,12 +33,12 @@ class JVisionCameraScheduler : public jni::HybridClass<JVisionCameraScheduler> {
 
  private:
   friend HybridBase;
-  jni::global_ref<JVisionCameraScheduler::javaobject> javaPart_;
+  jni::global_ref<JVisionCameraScheduler::javaobject> _javaPart;
   std::queue<std::function<void()>> _jobs;
   std::mutex _mutex;
 
   explicit JVisionCameraScheduler(jni::alias_ref<JVisionCameraScheduler::jhybridobject> jThis):
-    javaPart_(jni::make_global(jThis)) {}
+    _javaPart(jni::make_global(jThis)) {}
 
   // Schedules a call to `trigger` on the VisionCamera FP Thread
   void scheduleTrigger();

android/src/main/java/com/mrousavy/camera/CameraSession.kt

@@ -253,7 +253,7 @@ class CameraSession(private val context: Context,
       val outputs = outputs ?: throw CameraNotReadyError()
       val videoOutput = outputs.videoOutput ?: throw VideoNotEnabledError()
 
-      val recording = RecordingSession(context, enableAudio, videoOutput.size, fps, codec, orientation, fileType, callback, onError)
+      val recording = RecordingSession(context, videoOutput.size, enableAudio, fps, codec, orientation, fileType, callback, onError)
       recording.start()
       this.recording = recording
     }

android/src/main/java/com/mrousavy/camera/CameraViewModule.kt

@@ -180,11 +180,6 @@ class CameraViewModule(reactContext: ReactApplicationContext): ReactContextBaseJ
 
   @ReactMethod
   fun requestCameraPermission(promise: Promise) {
-    if (Build.VERSION.SDK_INT < Build.VERSION_CODES.M) {
-      // API 21 and below always grants permission on app install
-      return promise.resolve(PermissionStatus.GRANTED.unionValue)
-    }
-
     val activity = reactApplicationContext.currentActivity
     if (activity is PermissionAwareActivity) {
       val currentRequestCode = RequestCode++
@@ -205,11 +200,6 @@ class CameraViewModule(reactContext: ReactApplicationContext): ReactContextBaseJ
 
   @ReactMethod
   fun requestMicrophonePermission(promise: Promise) {
-    if (Build.VERSION.SDK_INT < Build.VERSION_CODES.M) {
-      // API 21 and below always grants permission on app install
-      return promise.resolve(PermissionStatus.GRANTED.unionValue)
-    }
-
     val activity = reactApplicationContext.currentActivity
     if (activity is PermissionAwareActivity) {
       val currentRequestCode = RequestCode++

android/src/main/java/com/mrousavy/camera/extensions/CameraCharacteristics+getOutputSizes.kt

@@ -63,10 +63,6 @@ fun CameraCharacteristics.getVideoSizes(cameraId: String, format: Int): List<Siz
 fun CameraCharacteristics.getPhotoSizes(format: Int): List<Size> {
   val config = this.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP)!!
   val sizes = config.getOutputSizes(format) ?: emptyArray()
-  val highResSizes = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) {
-    config.getHighResolutionOutputSizes(format)
-  } else {
-    null
-  } ?: emptyArray()
+  val highResSizes = config.getHighResolutionOutputSizes(format)
   return sizes.plus(highResSizes).toList()
 }