From f0df31e182ec681e11eb449e251ad597ba18a2dc Mon Sep 17 00:00:00 2001 From: Ethan Cordray <16182181+BenjaminZehowlt@users.noreply.github.com> Date: Thu, 24 Oct 2024 05:09:49 -0400 Subject: [PATCH] Implement GPU acceleration for VR overlays (#121) * Implement GPU acceleration for VR overlays * Updated changelog & added credits to about view --------- Co-authored-by: Raphiiko --- CHANGELOG.md | 4 + src-overlay-sidecar/Managers/OvrManager.cs | 61 +++++++ .../Overlays/BaseWebOverlay.cs | 9 +- .../Overlays/Helpers/OffScreenBrowser.cs | 149 ++++++------------ .../oyasumivr-overlay-sidecar.csproj | 2 +- .../about-view/about-view.component.html | 6 + 6 files changed, 125 insertions(+), 106 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e76b750e..89c18809 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- GPU acceleration for SteamVR overlays (Community contribution by [BenjaminZehowlt](https://github.com/BenjaminZehowlt)) + ## [1.14.3] ### Changed diff --git a/src-overlay-sidecar/Managers/OvrManager.cs b/src-overlay-sidecar/Managers/OvrManager.cs index 77699e68..3e7dfe92 100644 --- a/src-overlay-sidecar/Managers/OvrManager.cs +++ b/src-overlay-sidecar/Managers/OvrManager.cs @@ -6,6 +6,11 @@ using SharpDX.DXGI; using Valve.VR; using Device = SharpDX.Direct3D11.Device; +using Device1 = SharpDX.Direct3D11.Device1; +using Device2 = SharpDX.Direct3D11.Device2; +using Device3 = SharpDX.Direct3D11.Device3; +using Device4 = SharpDX.Direct3D11.Device4; +using Device5 = SharpDX.Direct3D11.Device5; namespace overlay_sidecar; @@ -24,6 +29,9 @@ public class OvrManager { private MicMuteIndicatorOverlay? _micMuteIndicatorOverlay; private bool _active; private Device? _device; + private Device1? _device1; + private DeviceMultithread? _deviceMultithread; + private Query? _query; private Dictionary> inputActions = new(); public event EventHandler>> OnInputActionsChanged; @@ -35,6 +43,8 @@ public class OvrManager { public OverlayPointer? OverlayPointer => _overlayPointer; public Device D3D11Device => _device!; + public Device1? D3D11Device1 => _device1; + public Query D3D11Query => _query!; private OvrManager() { @@ -65,6 +75,18 @@ private async Task InitializeDevice() DeviceCreationFlags.BgraSupport) : new Device(DriverType.Hardware, DeviceCreationFlags.BgraSupport); + UpgradeDevice(); + + _device1 = _device.QueryInterface(); + + _deviceMultithread = _device.QueryInterfaceOrNull(); + _deviceMultithread?.SetMultithreadProtected(true); + + _query = new Query(_device, new QueryDescription + { + Type = QueryType.Event, + Flags = QueryFlags.None + }); } catch (SharpDXException err) { @@ -82,6 +104,45 @@ private async Task InitializeDevice() } } + // Upgrades the device to the newest supported interface. + private void UpgradeDevice() + { + Device5 device5 = _device!.QueryInterfaceOrNull(); + if (device5 != null) + { + _device.Dispose(); + _device = device5; + return; + } + Device4 device4 = _device.QueryInterfaceOrNull(); + if (device4 != null) + { + _device.Dispose(); + _device = device4; + return; + } + Device3 device3 = _device.QueryInterfaceOrNull(); + if (device3 != null) + { + _device.Dispose(); + _device = device3; + return; + } + Device2 device2 = _device.QueryInterfaceOrNull(); + if (device2 != null) + { + _device.Dispose(); + _device = device2; + return; + } + Device1 device1 = _device.QueryInterfaceOrNull(); + if (device1 != null) + { + _device.Dispose(); + _device = device1; + } + } + private void OverlayRenderLoop() { var timer = new RefreshRateTimer(); diff --git a/src-overlay-sidecar/Overlays/BaseWebOverlay.cs b/src-overlay-sidecar/Overlays/BaseWebOverlay.cs index ca8aa65b..3c115646 100644 --- a/src-overlay-sidecar/Overlays/BaseWebOverlay.cs +++ b/src-overlay-sidecar/Overlays/BaseWebOverlay.cs @@ -71,11 +71,11 @@ private async void Init(int resolution) ArraySize = 1, Format = Format.B8G8R8A8_UNorm, SampleDescription = new SampleDescription(1, 0), - Usage = ResourceUsage.Dynamic, - BindFlags = BindFlags.ShaderResource, - CpuAccessFlags = CpuAccessFlags.Write + BindFlags = BindFlags.ShaderResource } ); + + Browser!.UpdateTexture(_texture); } catch (SharpDXException err) { @@ -210,8 +210,7 @@ public void UpdateFrame() // Stop here if we are not ready, already disposed, or if the browser hasn't painted anything new for the past second or so. if (_texture == null || Disposed || Browser == null || DateTimeOffset.UtcNow.ToUnixTimeMilliseconds() - Browser.LastPaint >= 1000) return; - // Render the browser to the texture - Browser.RenderToTexture(_texture); + var texture = new Texture_t { handle = _texture.NativePointer diff --git a/src-overlay-sidecar/Overlays/Helpers/OffScreenBrowser.cs b/src-overlay-sidecar/Overlays/Helpers/OffScreenBrowser.cs index 5b883132..7e76b1e4 100644 --- a/src-overlay-sidecar/Overlays/Helpers/OffScreenBrowser.cs +++ b/src-overlay-sidecar/Overlays/Helpers/OffScreenBrowser.cs @@ -1,5 +1,7 @@ // Source: https://github.com/vrcx-team/VRCX/blob/master/OffScreenBrowser.cs +using SharpDX.Mathematics.Interop; + namespace overlay_sidecar; using CefSharp; @@ -12,25 +14,32 @@ namespace overlay_sidecar; using System.Threading; public class OffScreenBrowser : ChromiumWebBrowser, IRenderHandler { - private readonly ReaderWriterLockSlim _paintBufferLock; - private GCHandle _paintBuffer; - private int _width; - private int _height; + private Texture2D? _texture; private long _lastPaint; public long LastPaint => _lastPaint; public OffScreenBrowser(string address, int width, int height) : base( address, - new BrowserSettings() - { - WindowlessFrameRate = 60, - WebGl = CefState.Enabled, - DefaultEncoding = "UTF-8" - } + automaticallyCreateBrowser: false ) { - _paintBufferLock = new ReaderWriterLockSlim(); + var windowInfo = new WindowInfo(); + windowInfo.SetAsWindowless(IntPtr.Zero); + windowInfo.WindowlessRenderingEnabled = true; + windowInfo.SharedTextureEnabled = true; + windowInfo.Width = width; + windowInfo.Height = height; + + var browserSettings = new BrowserSettings() + { + WindowlessFrameRate = 60, + WebGl = CefState.Enabled, + DefaultEncoding = "UTF-8" + }; + + CreateBrowser(windowInfo, browserSettings); + Size = new System.Drawing.Size(width, height); RenderHandler = this; } @@ -40,72 +49,19 @@ public OffScreenBrowser(string address, int width, int height) RenderHandler = null; if (IsDisposed) return; base.Dispose(); - - _paintBufferLock.EnterWriteLock(); - try - { - if (_paintBuffer.IsAllocated) _paintBuffer.Free(); - } - finally - { - _paintBufferLock.ExitWriteLock(); - } - - _paintBufferLock.Dispose(); } - public void RenderToTexture(Texture2D texture) + public void UpdateTexture(Texture2D texture) { - _paintBufferLock.EnterReadLock(); - try - { - if (_width > 0 && - _height > 0) - { - var context = texture.Device.ImmediateContext; - var dataBox = context.MapSubresource( - texture, - 0, - MapMode.WriteDiscard, - MapFlags.None - ); - if (dataBox.IsEmpty == false) - { - var sourcePtr = _paintBuffer.AddrOfPinnedObject(); - var destinationPtr = dataBox.DataPointer; - var pitch = _width * 4; - var rowPitch = dataBox.RowPitch; - if (pitch == rowPitch) - WinApi.CopyMemory( - destinationPtr, - sourcePtr, - (uint)(_width * _height * 4) - ); - else - for (var y = _height; y > 0; --y) - { - WinApi.CopyMemory( - destinationPtr, - sourcePtr, - (uint)pitch - ); - sourcePtr += pitch; - destinationPtr += rowPitch; - } - } - context.UnmapSubresource(texture, 0); - } - } - finally - - { - _paintBufferLock.ExitReadLock(); - } + _texture = texture; } ScreenInfo? IRenderHandler.GetScreenInfo() { - return null; + return new ScreenInfo + { + DeviceScaleFactor = 1.0F + }; } bool IRenderHandler.GetScreenPoint(int viewX, int viewY, out int screenX, out int screenY) @@ -120,8 +76,30 @@ Rect IRenderHandler.GetViewRect() return new Rect(0, 0, Size.Width, Size.Height); } - void IRenderHandler.OnAcceleratedPaint(PaintElementType type, Rect dirtyRect, IntPtr sharedHandle) + void IRenderHandler.OnAcceleratedPaint(PaintElementType type, Rect dirtyRect, AcceleratedPaintInfo paintInfo) { + if (type != PaintElementType.View) return; + if (OvrManager.Instance.D3D11Device1 == null) return; + if (_texture == null) return; + + using var cefTexture = + OvrManager.Instance.D3D11Device1.OpenSharedResource1(paintInfo.SharedTextureHandle); + var context = OvrManager.Instance.D3D11Device.ImmediateContext; + context.CopyResource(cefTexture, _texture); + + Query query = OvrManager.Instance.D3D11Query; + context.End(query); + context.Flush(); + + RawBool q = context.GetData(query, AsynchronousFlags.DoNotFlush); + + while (!q) + { + Thread.Yield(); + q = context.GetData(query, AsynchronousFlags.DoNotFlush); + } + + _lastPaint = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); } void IRenderHandler.OnCursorChange(IntPtr cursor, CursorType type, CursorInfo customCursorInfo) @@ -134,35 +112,6 @@ void IRenderHandler.OnImeCompositionRangeChanged(CefSharp.Structs.Range selected void IRenderHandler.OnPaint(PaintElementType type, Rect dirtyRect, IntPtr buffer, int width, int height) { - if (type != PaintElementType.View) return; - _paintBufferLock.EnterWriteLock(); - try - { - if (_width != width || - _height != height) - { - _width = width; - _height = height; - if (_paintBuffer.IsAllocated) _paintBuffer.Free(); - - _paintBuffer = GCHandle.Alloc( - new byte[_width * _height * 4], - GCHandleType.Pinned - ); - } - - WinApi.CopyMemory( - _paintBuffer.AddrOfPinnedObject(), - buffer, - (uint)(width * height * 4) - ); - - _lastPaint = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); - } - finally - { - _paintBufferLock.ExitWriteLock(); - } } void IRenderHandler.OnPopupShow(bool show) diff --git a/src-overlay-sidecar/oyasumivr-overlay-sidecar.csproj b/src-overlay-sidecar/oyasumivr-overlay-sidecar.csproj index 200cf63b..79eb9bcf 100644 --- a/src-overlay-sidecar/oyasumivr-overlay-sidecar.csproj +++ b/src-overlay-sidecar/oyasumivr-overlay-sidecar.csproj @@ -15,7 +15,7 @@ - + diff --git a/src-ui/app/views/dashboard-view/views/about-view/about-view.component.html b/src-ui/app/views/dashboard-view/views/about-view/about-view.component.html index 693556d2..caeb0dbc 100644 --- a/src-ui/app/views/dashboard-view/views/about-view/about-view.component.html +++ b/src-ui/app/views/dashboard-view/views/about-view/about-view.component.html @@ -59,6 +59,12 @@ about.contributionType.feature +
+ BenjaminZehowlt | + about.contributionType.feature +
spaecd | about.contributionType.soundFx