Fix iOS NativeCamera texture pixel format

Plugins/NativeCamera/CMakeLists.txt

@@ -24,6 +24,9 @@ if(ANDROID)
     set(EXTENSIONS AndroidExtensions)
 elseif(APPLE)
     set(EXTENSIONS "-framework CoreMedia -framework AVFoundation")
+    set_target_properties(NativeCamera PROPERTIES
+        XCODE_ATTRIBUTE_CLANG_ENABLE_OBJC_ARC YES
+    )
 endif()
 
 target_link_to_dependencies(NativeCamera

Plugins/NativeCamera/Source/Apple/NativeCameraImpl.mm

@@ -1,3 +1,7 @@
+#if ! __has_feature(objc_arc)
+#error "ARC is off"
+#endif
+
 #import <MetalKit/MetalKit.h>
 #include <bgfx/bgfx.h>
 #include <bgfx/platform.h>
@@ -28,15 +32,98 @@ - (id)init:(std::shared_ptr<Babylon::Plugins::Camera::Impl::ImplData>)implData;
 
 @end
 
+namespace {
+    static bool isPixelFormatSupported(uint32_t pixelFormat)
+    {
+        switch (pixelFormat) {
+            case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange:
+            case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange:
+                return true;
+        }
+        return false;
+    }
+
+    typedef struct {
+        vector_float2 position;
+        vector_float2 uv;
+    } XRVertex;
+
+    static XRVertex vertices[] = {
+        // 2D positions, UV
+        { { -1, -1 },   { 0, 1 } },
+        { { -1, 1 },    { 0, 0 } },
+        { { 1, -1 },    { 1, 1 } },
+        { { 1, 1 },     { 1, 0 } },
+    };
+
+    constexpr char shaderSource[] = R"(
+        #include <metal_stdlib>
+        #include <simd/simd.h>
+        using namespace metal;
+        #include <simd/simd.h>
+        typedef struct
+        {
+            vector_float2 position;
+            vector_float2 uv;
+        } XRVertex;
+        typedef struct
+        {
+            float4 position [[position]];
+            float2 uv;
+        } RasterizerData;
+        vertex RasterizerData
+        vertexShader(uint vertexID [[vertex_id]],
+                        constant XRVertex *vertices [[buffer(0)]])
+        {
+            RasterizerData out;
+            out.position = vector_float4(vertices[vertexID].position.xy, 0.0, 1.0);
+            out.uv = vertices[vertexID].uv;
+            return out;
+        }
+        fragment float4 fragmentShader(RasterizerData in [[stage_in]],
+            texture2d<float, access::sample> cameraTextureY [[ texture(1) ]],
+            texture2d<float, access::sample> cameraTextureCbCr [[ texture(2) ]])
+        {
+            constexpr sampler linearSampler(mip_filter::linear, mag_filter::linear, min_filter::linear);
+            if (!is_null_texture(cameraTextureY) && !is_null_texture(cameraTextureCbCr))
+            {
+                const float4 cameraSampleY = cameraTextureY.sample(linearSampler, in.uv);
+                const float4 cameraSampleCbCr = cameraTextureCbCr.sample(linearSampler, in.uv);
+                const float4x4 ycbcrToRGBTransform = float4x4(
+                    float4(+1.0000f, +1.0000f, +1.0000f, +0.0000f),
+                    float4(+0.0000f, -0.3441f, +1.7720f, +0.0000f),
+                    float4(+1.4020f, -0.7141f, +0.0000f, +0.0000f),
+                    float4(-0.7010f, +0.5291f, -0.8860f, +1.0000f)
+                );
+                float4 ycbcr = float4(cameraSampleY.r, cameraSampleCbCr.rg, 1.0);
+                float4 cameraSample = ycbcrToRGBTransform * ycbcr;
+                cameraSample.a = 1.0;
+                return cameraSample;
+            }
+            else
+            {
+                return 0;
+            }
+        }
+    )";
+
+    id<MTLLibrary> CompileShader(id<MTLDevice> metalDevice, const char* source) {
+        NSError* error;
+        id<MTLLibrary> lib = [metalDevice newLibraryWithSource:@(source) options:nil error:&error];
+        if(nil != error) {
+            throw std::runtime_error{[error.localizedDescription cStringUsingEncoding:NSASCIIStringEncoding]};
+        }
+        return lib;
+    }
+}
+
 namespace Babylon::Plugins
 {
     struct Camera::Impl::ImplData
     {
         ~ImplData()
         {
             [avCaptureSession stopRunning];
-            [avCaptureSession release];
-            [cameraTextureDelegate release];
             if (textureCache)
             {
                 CVMetalTextureCacheFlush(textureCache, 0);
@@ -47,7 +134,16 @@ - (id)init:(std::shared_ptr<Babylon::Plugins::Camera::Impl::ImplData>)implData;
         CameraTextureDelegate* cameraTextureDelegate{};
         AVCaptureSession* avCaptureSession{};
         CVMetalTextureCacheRef textureCache{};
-        id <MTLTexture> textureBGRA{};
+        id<MTLTexture> textureY{};
+        id<MTLTexture> textureCbCr{};
+        id<MTLTexture> textureRGBA{};
+        id<MTLRenderPipelineState> cameraPipelineState{};
+        size_t width = 0;
+        size_t height = 0;
+        id<MTLDevice> metalDevice{};
+        id<MTLCommandQueue> commandQueue{};
+        id<MTLCommandBuffer> currentCommandBuffer{};
+        bool overrideBgfxTexture = true;
     };
     Camera::Impl::Impl(Napi::Env env, bool overrideCameraTexture)
         : m_deviceContext{nullptr}
@@ -63,14 +159,17 @@ - (id)init:(std::shared_ptr<Babylon::Plugins::Camera::Impl::ImplData>)implData;
 
     arcana::task<Camera::Impl::CameraDimensions, std::exception_ptr> Camera::Impl::Open(uint32_t maxWidth, uint32_t maxHeight, bool frontCamera)
     {
+        m_implData->commandQueue = (__bridge id<MTLCommandQueue>)bgfx::getInternalData()->commandQueue;
+        m_implData->metalDevice = (__bridge id<MTLDevice>)bgfx::getInternalData()->context;
+
         if (maxWidth == 0 || maxWidth > std::numeric_limits<int32_t>::max()) {
             maxWidth = std::numeric_limits<int32_t>::max();
         }
         if (maxHeight == 0 || maxHeight > std::numeric_limits<int32_t>::max()) {
             maxHeight = std::numeric_limits<int32_t>::max();
         }
 
-        auto metalDevice = (id<MTLDevice>)bgfx::getInternalData()->context;
+        auto metalDevice = m_implData->metalDevice;
 
         if (!m_deviceContext)
         {
@@ -89,6 +188,7 @@ - (id)init:(std::shared_ptr<Babylon::Plugins::Camera::Impl::ImplData>)implData;
             AVCaptureDevice* bestDevice{nullptr};
             AVCaptureDeviceFormat* bestFormat{nullptr};
             uint32_t bestPixelCount{0};
+            uint32_t bestPixelFormat{0};
             uint32_t bestDimDiff{0};
             NSArray* deviceTypes{nullptr};
             bool foundExactMatch{false};
@@ -123,6 +223,13 @@ - (id)init:(std::shared_ptr<Babylon::Plugins::Camera::Impl::ImplData>)implData;
                 {
                     CMVideoFormatDescriptionRef videoFormatRef{static_cast<CMVideoFormatDescriptionRef>(format.formatDescription)};
                     CMVideoDimensions dimensions{CMVideoFormatDescriptionGetDimensions(videoFormatRef)};
+                    uint32_t pixelFormat{static_cast<uint32_t>(CMFormatDescriptionGetMediaSubType(videoFormatRef))};
+
+                    // Reject unsupporeted pixel formats.
+                    if (!isPixelFormatSupported(pixelFormat))
+                    {
+                        continue;
+                    }
 
                     // Reject any resolution that doesn't qualify for the constraint.
                     if (static_cast<uint32_t>(dimensions.width) > maxWidth || static_cast<uint32_t>(dimensions.height) > maxHeight)
@@ -136,6 +243,7 @@ - (id)init:(std::shared_ptr<Babylon::Plugins::Camera::Impl::ImplData>)implData;
                     if (bestDevice == nullptr || pixelCount > bestPixelCount || (pixelCount == bestPixelCount && dimDiff < bestDimDiff))
                     {
                         bestPixelCount = pixelCount;
+                        bestPixelFormat = pixelFormat;
                         bestDevice = device;
                         bestFormat = format;
                         bestDimDiff = dimDiff;
@@ -207,14 +315,32 @@ - (id)init:(std::shared_ptr<Babylon::Plugins::Camera::Impl::ImplData>)implData;
             dispatch_queue_t sampleBufferQueue{dispatch_queue_create("CameraMulticaster", DISPATCH_QUEUE_SERIAL)};
             AVCaptureVideoDataOutput * dataOutput{[[AVCaptureVideoDataOutput alloc] init]};
             [dataOutput setAlwaysDiscardsLateVideoFrames:YES];
-            [dataOutput setVideoSettings:@{(id)kCVPixelBufferPixelFormatTypeKey: @(kCVPixelFormatType_32BGRA)}];
+            [dataOutput setVideoSettings:@{(id)kCVPixelBufferPixelFormatTypeKey: @(bestPixelFormat)}];
             [dataOutput setSampleBufferDelegate:m_implData->cameraTextureDelegate queue:sampleBufferQueue];
 
             // Actually start the camera session.
             [m_implData->avCaptureSession addOutput:dataOutput];
             [m_implData->avCaptureSession commitConfiguration];
             [m_implData->avCaptureSession startRunning];
 
+            // Create a pipeline state for converting the camera output to RGBA.
+            id<MTLLibrary> lib = CompileShader(metalDevice, shaderSource);
+            id<MTLFunction> vertexFunction = [lib newFunctionWithName:@"vertexShader"];
+            id<MTLFunction> fragmentFunction = [lib newFunctionWithName:@"fragmentShader"];
+
+            MTLRenderPipelineDescriptor *pipelineStateDescriptor = [[MTLRenderPipelineDescriptor alloc] init];
+            pipelineStateDescriptor.label = @"Native Camera YCbCr to RGBA Pipeline";
+            pipelineStateDescriptor.vertexFunction = vertexFunction;
+            pipelineStateDescriptor.fragmentFunction = fragmentFunction;
+            pipelineStateDescriptor.colorAttachments[0].pixelFormat = MTLPixelFormatBGRA8Unorm;
+            m_implData->cameraPipelineState = [metalDevice newRenderPipelineStateWithDescriptor:pipelineStateDescriptor error:&error];
+
+            if (!m_implData->cameraPipelineState) {
+                taskCompletionSource.complete(arcana::make_unexpected(std::make_exception_ptr(std::runtime_error{
+                    std::string("Failed to create camera pipeline state: ") + [error.localizedDescription cStringUsingEncoding:NSASCIIStringEncoding]})));
+                return;
+            }
+
             taskCompletionSource.complete(cameraDimensions);
         });
 
@@ -233,9 +359,12 @@ - (id)init:(std::shared_ptr<Babylon::Plugins::Camera::Impl::ImplData>)implData;
     void Camera::Impl::UpdateCameraTexture(bgfx::TextureHandle textureHandle)
     {
         arcana::make_task(m_deviceContext->BeforeRenderScheduler(), arcana::cancellation::none(), [this, textureHandle] {
-            if (m_implData->textureBGRA)
-            {
-                bgfx::overrideInternal(textureHandle, reinterpret_cast<uintptr_t>(m_implData->textureBGRA));
+            @synchronized(m_implData->cameraTextureDelegate) {
+                if (m_implData->overrideBgfxTexture && m_implData->textureRGBA != nil)
+                {
+                    bgfx::overrideInternal(textureHandle, reinterpret_cast<uintptr_t>(m_implData->textureRGBA));
+                    m_implData->overrideBgfxTexture = false;
+                }
             }
         });
     }
@@ -327,25 +456,107 @@ - (void)captureOutput:(AVCaptureOutput *)__unused captureOutput didOutputSampleB
     }
 
     CVPixelBufferRef pixelBuffer{CMSampleBufferGetImageBuffer(sampleBuffer)};
-    id<MTLTexture> textureBGRA{nil};
-
     size_t width = CVPixelBufferGetWidthOfPlane(pixelBuffer, 0);
     size_t height = CVPixelBufferGetHeightOfPlane(pixelBuffer, 0);
-    MTLPixelFormat pixelFormat{MTLPixelFormatBGRA8Unorm};
-
-    CVMetalTextureRef texture{nullptr};
-    CVReturn status{CVMetalTextureCacheCreateTextureFromImage(nullptr, implData->textureCache, pixelBuffer, nullptr, pixelFormat, width, height, 0, &texture)};
-    if (status == kCVReturnSuccess)
+
+    // Update both metal textures used by the renderer to display the camera image.
+    id<MTLTexture> textureY = [self getCameraTexture:pixelBuffer plane:0];
+    id<MTLTexture> textureCbCr = [self getCameraTexture:pixelBuffer plane:1];
+
+    if(textureY != nil && textureCbCr != nil)
     {
-        textureBGRA = CVMetalTextureGetTexture(texture);
-        CFRelease(texture);
+        @synchronized(self) {
+            implData->textureY = textureY;
+            implData->textureCbCr = textureCbCr;
+
+            if (implData->width != width || implData->height != height) {
+                implData->width = width;
+                implData->height = height;
+                MTLTextureDescriptor *textureDescriptor = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatRGBA8Unorm width:width height:height mipmapped:NO];
+                textureDescriptor.usage = MTLTextureUsageRenderTarget | MTLTextureUsageShaderRead;
+                implData->textureRGBA = [implData->metalDevice newTextureWithDescriptor:textureDescriptor];
+                implData->overrideBgfxTexture = true;
+            }
+        };
+
+        if (implData->textureRGBA != nil)
+        {
+            implData->currentCommandBuffer = [implData->commandQueue commandBuffer];
+            implData->currentCommandBuffer.label = @"NativeCameraCommandBuffer";
+            MTLRenderPassDescriptor *renderPassDescriptor = [MTLRenderPassDescriptor renderPassDescriptor];
+
+            if (renderPassDescriptor != nil) {
+                // Attach the color texture, on which we'll draw the camera texture (so no need to clear on load).
+                renderPassDescriptor.colorAttachments[0].texture = implData->textureRGBA;
+                renderPassDescriptor.colorAttachments[0].loadAction = MTLLoadActionDontCare;
+                renderPassDescriptor.colorAttachments[0].storeAction = MTLStoreActionStore;
+
+                // Create and end the render encoder.
+                id<MTLRenderCommandEncoder> renderEncoder = [implData->currentCommandBuffer renderCommandEncoderWithDescriptor:renderPassDescriptor];
+                renderEncoder.label = @"NativeCameraEncoder";
+
+                // Set the shader pipeline.
+                [renderEncoder setRenderPipelineState:implData->cameraPipelineState];
+
+                // Set the vertex data.
+                [renderEncoder setVertexBytes:vertices length:sizeof(vertices) atIndex:0];
+
+                // Set the textures.
+                [renderEncoder setFragmentTexture:textureY atIndex:1];
+                [renderEncoder setFragmentTexture:textureCbCr atIndex:2];
+
+                // Draw the triangles.
+                [renderEncoder drawPrimitives:MTLPrimitiveTypeTriangleStrip vertexStart:0 vertexCount:4];
+
+                [renderEncoder endEncoding];
+
+                [implData->currentCommandBuffer addCompletedHandler:^(id<MTLCommandBuffer>) {
+                    if (textureY != nil) {
+                        [textureY setPurgeableState:MTLPurgeableStateEmpty];
+                    }
+
+                    if (textureCbCr != nil) {
+                        [textureCbCr setPurgeableState:MTLPurgeableStateEmpty];
+                    }
+                }];
+            }
+
+            // Finalize rendering here & push the command buffer to the GPU.
+            [implData->currentCommandBuffer commit];
+        }
     }
+}
 
-    if (textureBGRA != nil)
-    {
-        dispatch_async(dispatch_get_main_queue(), ^{
-            implData->textureBGRA = textureBGRA;
-        });
+/**
+ Updates the captured texture with the current pixel buffer.
+*/
+- (id<MTLTexture>)getCameraTexture:(CVPixelBufferRef)pixelBuffer plane:(int)planeIndex {
+    CVReturn ret = CVPixelBufferLockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);
+    if (ret != kCVReturnSuccess) {
+        return {};
+    }
+
+    @try {
+        size_t planeWidth = CVPixelBufferGetWidthOfPlane(pixelBuffer, planeIndex);
+        size_t planeHeight = CVPixelBufferGetHeightOfPlane(pixelBuffer, planeIndex);
+
+        // Plane 0 is the Y plane, which is in R8Unorm format, and the second plane is the CBCR plane which is RG8Unorm format.
+        auto pixelFormat = planeIndex ? MTLPixelFormatRG8Unorm : MTLPixelFormatR8Unorm;
+        CVMetalTextureRef textureRef;
+
+        // Create a texture from the corresponding plane.
+        auto status = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, implData->textureCache, pixelBuffer, nil, pixelFormat, planeWidth, planeHeight, planeIndex, &textureRef);
+        if (status != kCVReturnSuccess) {
+            return nil;
+        }
+
+        id<MTLTexture> texture = CVMetalTextureGetTexture(textureRef);
+        CFRelease(textureRef);
+
+        return texture;
+    }
+    @finally {
+        CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);
     }
 }