diff --git a/.gitignore b/.gitignore index aa60cc9..a7291a5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ bin/ obj/ -.vs/VRCFTVarjoModule/v15 \ No newline at end of file +.vs +VRCFaceTracking.Core.dll \ No newline at end of file diff --git a/Assets/Aero.png b/Assets/Aero.png new file mode 100644 index 0000000..a293b5f Binary files /dev/null and b/Assets/Aero.png differ diff --git a/Assets/VR-1.png b/Assets/VR-1.png new file mode 100644 index 0000000..789f6ca Binary files /dev/null and b/Assets/VR-1.png differ diff --git a/Assets/VR-2.png b/Assets/VR-2.png new file mode 100644 index 0000000..6888a47 Binary files /dev/null and b/Assets/VR-2.png differ diff --git a/Assets/VR-3.png b/Assets/VR-3.png new file mode 100644 index 0000000..3983977 Binary files /dev/null and b/Assets/VR-3.png differ diff --git a/Assets/XR-1.png b/Assets/XR-1.png new file mode 100644 index 0000000..d237cc4 Binary files /dev/null and b/Assets/XR-1.png differ diff --git a/Assets/XR-3.png b/Assets/XR-3.png new file mode 100644 index 0000000..75402fa Binary files /dev/null and b/Assets/XR-3.png differ diff --git a/Assets/unknown.png b/Assets/unknown.png new file mode 100644 index 0000000..27da739 Binary files /dev/null and b/Assets/unknown.png differ diff --git a/TrackingLibs/Varjo/VarjoLib.dll b/TrackingLibs/VarjoLib.dll similarity index 100% rename from TrackingLibs/Varjo/VarjoLib.dll rename to TrackingLibs/VarjoLib.dll diff --git a/VRCFTVarjoModule.csproj b/VRCFTVarjoModule.csproj index ee774a6..66b5333 100644 --- a/VRCFTVarjoModule.csproj +++ b/VRCFTVarjoModule.csproj @@ -1,132 +1,27 @@ - + - Debug - AnyCPU - {381EDD16-E4AF-4971-B70B-77E3F17A8204} - Library - false - VRCFTVarjoModule - v4.7.2 - 512 - - - - true - full - false - bin\Debug\ - DEBUG;TRACE - prompt - 4 - false - - - pdbonly - true - bin\Release\ - TRACE - prompt - 4 - false - true - - - VRCFTVarjoModule - - - true - bin\x64\Debug\ - DEBUG;TRACE - full - x64 - prompt - MinimumRecommendedRules.ruleset - - - bin\x64\Release\ - TRACE - true - none - x64 - prompt - MinimumRecommendedRules.ruleset - true - - - true - bin\x86\Debug\ - DEBUG;TRACE - full - x86 - prompt - MinimumRecommendedRules.ruleset - - - bin\x86\Release\ - TRACE - true - pdbonly - x86 - prompt - MinimumRecommendedRules.ruleset - - - bin\Release %283rd Gen HMD%29\ - TRACE;GEN3HMD - true - true - pdbonly - AnyCPU - 7.3 - prompt - - - bin\x64\Release %283rd Gen HMD%29\ - TRACE;GEN3HMD - true - true - x64 - 7.3 - prompt - MinimumRecommendedRules.ruleset - - - bin\x86\Release %283rd Gen HMD%29\ - TRACE;GEN3HMD - true - pdbonly - x86 - 7.3 - prompt - MinimumRecommendedRules.ruleset + net7.0 + false - - - - - - - - - False - ..\..\osc\VRCFaceTracking\VRCFaceTracking.exe + + + + + + + VRCFaceTracking.Core.dll - - - - + + PreserveNewest + - + + + + - - - - - - xcopy "$(TargetDir)$(TargetName)$(TargetExt)" "%APPDATA%\VRCFaceTracking\CustomLibs" /Y - \ No newline at end of file diff --git a/VRCFTVarjoModule.sln b/VRCFTVarjoModule.sln index 12fc8a7..7bc872d 100644 --- a/VRCFTVarjoModule.sln +++ b/VRCFTVarjoModule.sln @@ -8,34 +8,13 @@ EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU - Debug|x64 = Debug|x64 - Debug|x86 = Debug|x86 - Release (3rd Gen HMD)|Any CPU = Release (3rd Gen HMD)|Any CPU - Release (3rd Gen HMD)|x64 = Release (3rd Gen HMD)|x64 - Release (3rd Gen HMD)|x86 = Release (3rd Gen HMD)|x86 Release|Any CPU = Release|Any CPU - Release|x64 = Release|x64 - Release|x86 = Release|x86 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution {381EDD16-E4AF-4971-B70B-77E3F17A8204}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {381EDD16-E4AF-4971-B70B-77E3F17A8204}.Debug|Any CPU.Build.0 = Debug|Any CPU - {381EDD16-E4AF-4971-B70B-77E3F17A8204}.Debug|x64.ActiveCfg = Debug|x64 - {381EDD16-E4AF-4971-B70B-77E3F17A8204}.Debug|x64.Build.0 = Debug|x64 - {381EDD16-E4AF-4971-B70B-77E3F17A8204}.Debug|x86.ActiveCfg = Debug|x86 - {381EDD16-E4AF-4971-B70B-77E3F17A8204}.Debug|x86.Build.0 = Debug|x86 - {381EDD16-E4AF-4971-B70B-77E3F17A8204}.Release (3rd Gen HMD)|Any CPU.ActiveCfg = Release (3rd Gen HMD)|Any CPU - {381EDD16-E4AF-4971-B70B-77E3F17A8204}.Release (3rd Gen HMD)|Any CPU.Build.0 = Release (3rd Gen HMD)|Any CPU - {381EDD16-E4AF-4971-B70B-77E3F17A8204}.Release (3rd Gen HMD)|x64.ActiveCfg = Release (3rd Gen HMD)|x64 - {381EDD16-E4AF-4971-B70B-77E3F17A8204}.Release (3rd Gen HMD)|x64.Build.0 = Release (3rd Gen HMD)|x64 - {381EDD16-E4AF-4971-B70B-77E3F17A8204}.Release (3rd Gen HMD)|x86.ActiveCfg = Release (3rd Gen HMD)|x86 - {381EDD16-E4AF-4971-B70B-77E3F17A8204}.Release (3rd Gen HMD)|x86.Build.0 = Release (3rd Gen HMD)|x86 {381EDD16-E4AF-4971-B70B-77E3F17A8204}.Release|Any CPU.ActiveCfg = Release|Any CPU {381EDD16-E4AF-4971-B70B-77E3F17A8204}.Release|Any CPU.Build.0 = Release|Any CPU - {381EDD16-E4AF-4971-B70B-77E3F17A8204}.Release|x64.ActiveCfg = Release|x64 - {381EDD16-E4AF-4971-B70B-77E3F17A8204}.Release|x64.Build.0 = Release|x64 - {381EDD16-E4AF-4971-B70B-77E3F17A8204}.Release|x86.ActiveCfg = Release|x86 - {381EDD16-E4AF-4971-B70B-77E3F17A8204}.Release|x86.Build.0 = Release|x86 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/VarjoCompanionInterface.cs b/VarjoCompanionInterface.cs deleted file mode 100644 index f0e27ce..0000000 --- a/VarjoCompanionInterface.cs +++ /dev/null @@ -1,80 +0,0 @@ -using System; -using System.Diagnostics; -using System.IO; -using System.IO.MemoryMappedFiles; -using System.Runtime.InteropServices; -using System.Threading; -using VRCFaceTracking; - -namespace VRCFTVarjoModule -{ - - public class VarjoCompanionInterface : VarjoInterface - { - private MemoryMappedFile MemMapFile; - private MemoryMappedViewAccessor ViewAccessor; - private Process CompanionProcess; - - public override bool Initialize() - { - if (!VarjoAvailable()) - { - Logger.Error("Varjo headset isn't detected"); - return false; - } - string modDir = GetModuleDir(); - string exePath = Path.Combine(modDir, "VarjoCompanion.exe"); - if (!File.Exists(exePath)) - { - Logger.Error("VarjoCompanion executable wasn't found!"); - return false; - } - CompanionProcess = new Process(); - CompanionProcess.StartInfo.WorkingDirectory = modDir; - CompanionProcess.StartInfo.FileName = exePath; - CompanionProcess.Start(); - - for (int i = 0; i < 5; i++) - { - try - { - MemMapFile = MemoryMappedFile.OpenExisting("VarjoEyeTracking"); - ViewAccessor = MemMapFile.CreateViewAccessor(); - return true; - } - catch (FileNotFoundException) - { - Console.WriteLine("VarjoEyeTracking mapped file doesn't exist; the companion app probably isn't running"); - } - catch (Exception ex) - { - Console.WriteLine("Could not open the mapped file: " + ex); - return false; - } - Thread.Sleep(500); - } - - return false; - } - - public override void Update() - { - if (MemMapFile == null) return; - ViewAccessor.Read(0, out gazeData); - } - - public override void Teardown() - { - if (MemMapFile == null) return; - //memoryGazeData.shutdown = true; // tell the companion app to shut down gracefully but it doesn't work anyway - ViewAccessor.Write(0, ref gazeData); - MemMapFile.Dispose(); - CompanionProcess.Kill(); - } - - public override string GetName() - { - return "companion"; - } - } -} \ No newline at end of file diff --git a/VarjoNativeInterface.cs b/VarjoNativeInterface.cs index 6955e99..0c755ee 100644 --- a/VarjoNativeInterface.cs +++ b/VarjoNativeInterface.cs @@ -1,132 +1,120 @@ -using System; +using Microsoft.Extensions.Logging; +using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Reflection; using System.Runtime.InteropServices; using System.Text; -using System.Threading.Tasks; using VRCFaceTracking; namespace VRCFTVarjoModule { - class VarjoNativeInterface : VarjoInterface + class VarjoNativeInterface { private IntPtr _session; + protected GazeData gazeData; + protected EyeMeasurements eyeMeasurements; + protected ILogger Logger; - public override bool Initialize() + #region Lifetime methods (Init, Update, Teardown) + public bool Initialize(ILogger loggerInstance) { + Logger = loggerInstance; + if (!VarjoAvailable()) { - Logger.Error("Varjo headset isn't detected"); - return false; - } - LoadLibrary(); - _session = varjo_SessionInit(); - if (_session == IntPtr.Zero) - { + Logger.LogError("Varjo headset isn't detected"); return false; } - if (!varjo_IsGazeAllowed(_session)) + if (LoadLibrary()) { - Logger.Error("Gaze tracking is not allowed! Please enable it in the Varjo Base!"); - return false; + _session = varjo_SessionInit(); + if (_session == IntPtr.Zero) + { + return false; + } + if (!varjo_IsGazeAllowed(_session)) + { + Logger.LogError("Gaze tracking is not allowed! Please enable it in the Varjo Base!"); + return false; + } + varjo_GazeInit(_session); + varjo_SyncProperties(_session); + return true; } - varjo_GazeInit(_session); - varjo_SyncProperties(_session); - return true; + return false; } - public override void Teardown() + public void Teardown() { - //no need to tear down anything right? + varjo_SessionShutDown(_session); } - public override void Update() + // Get's the newest Data from the SDK and stores it internally + public bool Update() { if (_session == IntPtr.Zero) - return; + return false; // Get's GazeData and EyeMeasurements from the Varjo SDK // Return value states whether or not the request was successful (true = has Data; false = Error occured) bool hasData = varjo_GetGazeData(_session, out gazeData, out eyeMeasurements); if (!hasData) - Logger.Msg("Error while getting Gaze Data"); + Logger.LogWarning("Error while getting Gaze Data"); + + return hasData; } - public override string GetName() + #endregion + + #region Public Getters + public GazeData GetGazeData() { - return "native DLL"; + return gazeData; } + public EyeMeasurements GetEyeMeasurements() + { + return eyeMeasurements; + } + + public string GetHMDName() + { + int bufferSize = varjo_GetPropertyStringSize(_session, VarjoPropertyKey.HMDProductName); + StringBuilder buffer = new StringBuilder(bufferSize); + varjo_GetPropertyString(_session, VarjoPropertyKey.HMDProductName, buffer, bufferSize); + + return buffer.ToString(); + } + #endregion + + #region Internal helper methods private bool LoadLibrary() { - IEnumerable dllPaths = ExtractAssemblies(new string[] { "Varjo.VarjoLib.dll" }); - var path = dllPaths.First(); + var path = System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location) + "\\TrackingLibs\\VarjoLib.dll"; if (path == null) { - Logger.Error(string.Concat("Couldn't extract the library ", path)); + Logger.LogError(string.Concat("Couldn't extract the library ", path)); return false; } if (LoadLibrary(path) == IntPtr.Zero) { - Logger.Error(string.Concat("Unable to load library ", path)); + Logger.LogError(string.Concat("Unable to load library ", path)); return false; } - Logger.Msg(string.Concat("Loaded library ", path)); + Logger.LogInformation(string.Concat("Loaded library ", path)); return true; } - - // I can't ask nicely to add my DLL into the dependency list so I had to steal code from the main repo :( - private static IEnumerable ExtractAssemblies(IEnumerable resourceNames) + private static bool VarjoAvailable() { - var extractedPaths = new List(); - - var dirName = Path.Combine(Utils.PersistentDataDirectory, "StockLibs"); - if (!Directory.Exists(dirName)) - Directory.CreateDirectory(dirName); - - foreach (var dll in resourceNames) - { - var dllPath = Path.Combine(dirName, GetAssemblyNameFromPath(dll)); - - using (var stm = Assembly.GetAssembly(typeof(VarjoNativeInterface)).GetManifestResourceStream("VRCFTVarjoModule.TrackingLibs." + dll)) - { - try - { - using (Stream outFile = File.Create(dllPath)) - { - const int sz = 4096; - var buf = new byte[sz]; - while (true) - { - if (stm == null) continue; - var nRead = stm.Read(buf, 0, sz); - if (nRead < 1) - break; - outFile.Write(buf, 0, nRead); - } - } - - extractedPaths.Add(dllPath); - } - catch (Exception e) - { - Logger.Error($"Failed to get DLL: " + e.Message); - } - } - } - return extractedPaths; - } - - private static string GetAssemblyNameFromPath(string path) - { - var splitPath = path.Split('.').ToList(); - splitPath.Reverse(); - return splitPath[1] + ".dll"; + // totally not how the official Varjo library works under the hood + return File.Exists("\\\\.\\pipe\\Varjo\\InfoService"); } + #endregion + #region DllImports [DllImport("kernel32", CharSet = CharSet.Unicode, ExactSpelling = false, SetLastError = true)] private static extern IntPtr LoadLibrary(string lpFileName); @@ -164,13 +152,20 @@ private static string GetAssemblyNameFromPath(string path) private static extern void varjo_RequestGazeCalibration(IntPtr session); [DllImport("VarjoLib", CharSet = CharSet.Auto)] - private static extern bool varjo_GetPropertyBool(IntPtr session, int propertyKey); + private static extern bool varjo_GetPropertyBool(IntPtr session, VarjoPropertyKey propertyKey); + + [DllImport("VarjoLib", CharSet = CharSet.Auto)] + private static extern int varjo_GetPropertyInt(IntPtr session, VarjoPropertyKey propertyKey); + + [DllImport("VarjoLib", CharSet = CharSet.Ansi)] + private static extern void varjo_GetPropertyString(IntPtr session, VarjoPropertyKey propertyKey, StringBuilder buffer, int bufferSize); [DllImport("VarjoLib", CharSet = CharSet.Auto)] - private static extern int varjo_GetPropertyInt(IntPtr session, int propertyKey); + private static extern int varjo_GetPropertyStringSize(IntPtr session, VarjoPropertyKey propertyKey); [DllImport("VarjoLib", CharSet = CharSet.Auto)] private static extern void varjo_SyncProperties(IntPtr session); + #endregion } } diff --git a/VarjoTrackingModule.cs b/VarjoTrackingModule.cs index 76e5196..47f08ea 100644 --- a/VarjoTrackingModule.cs +++ b/VarjoTrackingModule.cs @@ -1,11 +1,12 @@ -using System; +using Microsoft.Extensions.Logging; +using System.Collections.Generic; using System.IO; -using System.IO.MemoryMappedFiles; -using System.Runtime.InteropServices; using System.Threading; using VRCFaceTracking; -using VRCFaceTracking.Params; -using Vector2 = VRCFaceTracking.Params.Vector2; +using VRCFaceTracking.Core.Library; +using VRCFaceTracking.Core.Params.Data; +using VRCFaceTracking.Core.Params.Expressions; +using VRCFaceTracking.Core.Types; namespace VRCFTVarjoModule { @@ -26,243 +27,177 @@ enum VarjoOpennessMode : byte // Threshold of the maximum opening in Eye Openness that will be tracked as long as the eye status is "invalid" private static readonly float MAX_OPENNESS_DEVIATION = 0.1f; - // 999 for min and -1 for max, to ensure these Values get overwritten the first runthrough - private static double _minPupilSize = 999, _maxPupilSize = -1; - - // This function parses the external module's single-eye data into a VRCFT-Parseable format - public static void Update(ref Eye data, GazeRay external, float openness, GazeEyeStatus eyeStatus) - { - if ((int)eyeStatus >= 2) - { - data.Look = new Vector2((float)external.forward.x, (float)external.forward.y); - } - - parseOpenness(ref data, openness, eyeStatus >= GazeEyeStatus.Compensated); - } - - public static void Update(ref Eye data, GazeRay external, float openness, GazeStatus combinedStatus) + // Function to map a Varjo GazeRay to a Vector2 Ray for use in VRCFT + private static Vector2 GetGazeVector(GazeRay varjoGaze) { - if (combinedStatus == GazeStatus.Valid) - { - data.Look = new Vector2((float)external.forward.x, (float)external.forward.y); - } - - parseOpenness(ref data, openness, combinedStatus != GazeStatus.Invalid); + return new Vector2((float)varjoGaze.forward.x, (float)varjoGaze.forward.y); } - // This function parses the external module's full-data data into multiple VRCFT-Parseable single-eye structs - public static void Update(ref EyeTrackingData data, GazeData external, EyeMeasurements externalMeasurements) + // This function is used to disect the single Varjo Openness Float into a more managable Openness, Squeeze and Widen Value + // As the three Parameters are exclusive to one another (if one is between 0 or 1, the others have to be either 0 or 1), we only need to do maths for one parameter + // `openness` is returned as `-1` when VRCFT values should remain unchanged + private static (float openness, float squeeze, float widen) ParseOpenness(float currentOpenness, float externalOpenness, GazeEyeStatus eyeStatus) { - Update(ref data.Right, external.rightEye, externalMeasurements.rightEyeOpenness, external.rightStatus); - Update(ref data.Left, external.leftEye, externalMeasurements.leftEyeOpenness, external.leftStatus); - Update(ref data.Combined, external.gaze, (externalMeasurements.leftEyeOpenness + externalMeasurements.rightEyeOpenness) / 2, external.status); + float parsedOpenness; + float widen = 0; + float squeeze = 0; + VarjoOpennessMode mode; - // Determines whether the pupil Size/Eye dilation - // If one is open and the other closed, we don't want the closed one to pull down the Values of the open one. - double pupilSize = 0; - // Casting the status as ints allows for easier comparison; as we need Compensated (2) or Tracked (3), that means >= 2 - if ((int)external.leftStatus >= 2 && (int)external.rightStatus >= 2) - { - pupilSize = (externalMeasurements.leftPupilDiameterInMM + externalMeasurements.rightPupilDiameterInMM) / 2; - } - else if ((int)external.rightStatus >= 2) + // Check what range the Varjo Openness falls into and calculate the new "Openness" + if (externalOpenness <= EYE_SQUEEZE_THRESHOLD) { - pupilSize = externalMeasurements.rightPupilDiameterInMM; + parsedOpenness = 0; + mode = VarjoOpennessMode.Squeeze; } - else if ((int)external.leftStatus >= 2) + else if (externalOpenness >= EYE_WIDEN_THRESHOLD) { - pupilSize = externalMeasurements.leftPupilDiameterInMM; + parsedOpenness = 1; + mode = VarjoOpennessMode.Widen; } - - // Only set the Eye Dilation, if we actually have Pupil data - if (pupilSize > 0 && external.status == GazeStatus.Valid) + else { - data.EyesDilation = (float)calculateEyeDilation(ref pupilSize); + parsedOpenness = (externalOpenness - EYE_SQUEEZE_THRESHOLD) / (EYE_WIDEN_THRESHOLD - EYE_SQUEEZE_THRESHOLD); + mode = VarjoOpennessMode.Openness; } - // Set the Pupil Diameter anyways - data.EyesPupilDiameter = (float)(pupilSize > 10 ? 1 : pupilSize / 10); - } - // This function is used to calculate the Eye Dilation based on the lowest and highest measured Pupil Size - private static double calculateEyeDilation(ref double pupilSize) - { - // Adjust the bounds if Pupil Size exceeds the last thought maximum bounds - if (pupilSize > _maxPupilSize) + // Check if new Openness is within allowable margin + if (eyeStatus >= GazeEyeStatus.Compensated || parsedOpenness < currentOpenness + MAX_OPENNESS_DEVIATION) { - _maxPupilSize = pupilSize; - } - if (pupilSize < _minPupilSize) - { - _minPupilSize = pupilSize; + // ...and calculate squeeze/widen if called for + switch (mode) + { + case VarjoOpennessMode.Squeeze: + squeeze = (externalOpenness / -EYE_SQUEEZE_THRESHOLD) + 1; + break; + case VarjoOpennessMode.Widen: + widen = (externalOpenness - EYE_WIDEN_THRESHOLD) / (1 - EYE_WIDEN_THRESHOLD); + break; + } } - - // In case both max and min are the same, we need to return 0.5; Don't wanna run into a divide by 0 situation ^^" - // We also don't want to run the maths if the pupil size bounds haven't been initialized yet... - if (_maxPupilSize == _minPupilSize || _maxPupilSize == -1) + else { - return 0.5; + // Otherwise set the openness to -1 to indicate no changes + parsedOpenness = -1; } - // Pretty typical number range convertion. - // We assume we want 1 for max dilation and 0 for min dilation; simplifies the maths a bit - return (pupilSize - _minPupilSize) / (_maxPupilSize - _minPupilSize); + return (parsedOpenness, squeeze, widen); } - // This function is used to disect the single Varjo Openness Float into the SRanipal Openness, Widen & Squeeze values - // As the three SRanipal Parameters are exclusive to one another (if one is between 0 or 1, the others have to be either 0 or 1), we only need to do maths for one parameter - private static void parseOpenness(ref Eye data, float openness, bool trackingValid) + // Main Update function + // Mapps Varjo Eye Data to VRCFT Parameters + public static void Update(ref UnifiedEyeData data, ref UnifiedExpressionShape[] expressionData, GazeData external, EyeMeasurements externalMeasurements) { - float srOpenness; - VarjoOpennessMode mode; - - - if (openness <= EYE_SQUEEZE_THRESHOLD) + // Set the Gaze and Pupil Size for each eye when their status is somewhat reliable according to the SDK + if (external.rightStatus >= GazeEyeStatus.Compensated) { - srOpenness = 0; - mode = VarjoOpennessMode.Squeeze; + data.Right.Gaze = GetGazeVector(external.rightEye); + data.Right.PupilDiameter_MM = externalMeasurements.rightPupilDiameterInMM; } - else if (openness >= EYE_WIDEN_THRESHOLD) + if (external.leftStatus >= GazeEyeStatus.Compensated) { - srOpenness = 1; - mode = VarjoOpennessMode.Widen; + data.Left.Gaze = GetGazeVector(external.leftEye); + data.Left.PupilDiameter_MM = externalMeasurements.leftPupilDiameterInMM; } - else + + // Parse Openness as before, but instead of writing them immideatly, we store them in variables temporarely + (float rightOpenness, float rightSqueeze, float rightWiden) = ParseOpenness(data.Right.Openness, externalMeasurements.rightEyeOpenness, external.rightStatus); + (float leftOpenness, float leftSqueeze, float leftWiden) = ParseOpenness(data.Left.Openness, externalMeasurements.leftEyeOpenness, external.leftStatus); + + // Set Openness Values for each eye; if they should change + if (rightOpenness >= 0.0f) { - srOpenness = (openness - EYE_SQUEEZE_THRESHOLD) / (EYE_WIDEN_THRESHOLD - EYE_SQUEEZE_THRESHOLD); - mode = VarjoOpennessMode.Openness; - } + data.Right.Openness = rightOpenness; + expressionData[(int)UnifiedExpressions.EyeWideRight].Weight = rightWiden; + expressionData[(int)UnifiedExpressions.EyeSquintRight].Weight = rightSqueeze; - if (trackingValid || srOpenness < data.Openness + MAX_OPENNESS_DEVIATION) + // Duplicated like on the SRanipal Module + expressionData[(int)UnifiedExpressions.BrowInnerUpRight].Weight = rightWiden; + expressionData[(int)UnifiedExpressions.BrowOuterUpRight].Weight = rightWiden; + expressionData[(int)UnifiedExpressions.BrowPinchRight].Weight = rightSqueeze; + expressionData[(int)UnifiedExpressions.BrowLowererRight].Weight = rightSqueeze; + } + if (leftOpenness >= 0.0f) { - data.Openness = srOpenness; + data.Left.Openness = leftOpenness; + expressionData[(int)UnifiedExpressions.EyeWideLeft].Weight = leftWiden; + expressionData[(int)UnifiedExpressions.EyeSquintLeft].Weight = leftSqueeze; - switch(mode) - { - case VarjoOpennessMode.Squeeze: - data.Squeeze = (openness / -EYE_SQUEEZE_THRESHOLD) + 1; - data.Widen = 0; - break; - case VarjoOpennessMode.Widen: - data.Squeeze = 0; - data.Widen = (openness - EYE_WIDEN_THRESHOLD) / (1 - EYE_WIDEN_THRESHOLD); - break; - default: - data.Squeeze = 0; - data.Widen = 0; - break; - } + // Duplicated like on the SRanipal Module + expressionData[(int)UnifiedExpressions.BrowInnerUpLeft].Weight = leftWiden; + expressionData[(int)UnifiedExpressions.BrowOuterUpLeft].Weight = leftWiden; + expressionData[(int)UnifiedExpressions.BrowPinchLeft].Weight = leftSqueeze; + expressionData[(int)UnifiedExpressions.BrowLowererLeft].Weight = leftSqueeze; } } - } public class VarjoTrackingModule : ExtTrackingModule { - private static VarjoInterface tracker; - private static CancellationTokenSource _cancellationToken; - + private static VarjoNativeInterface tracker; - // eye image stuff - private MemoryMappedFile MemMapFile; - private MemoryMappedViewAccessor ViewAccessor; - private IntPtr EyeImagePointer; - // Values for the Camera buffer size in VRCFT -#if GEN3HMD - private static readonly int CAMERA_WIDTH = 1280, CAMERA_HEIGHT = 400; // 3rd Gen Varjo HMDs (VR-3, XR-3, Aero) -#else - private static readonly int CAMERA_WIDTH = 2560, CAMERA_HEIGHT = 800; // 1st & 2nd Gen Varjo HMDs (VR-1, VR-2, XR-1) -#endif + // Mark this module as only supporting Eye Tracking + public override (bool SupportsEye, bool SupportsExpression) Supported => (true, false); - - public override (bool SupportsEye, bool SupportsLip) Supported => (true, false); - - // Synchronous module initialization. Take as much time as you need to initialize any external modules. This runs in the init-thread - public override (bool eyeSuccess, bool lipSuccess) Initialize(bool eye, bool lip) + // Prepares the Varjo Interface for communication with the SDK and sets module display name and icon + public override (bool eyeSuccess, bool expressionSuccess) Initialize(bool eye, bool lip) { - if (IsStandalone()) - { - tracker = new VarjoNativeInterface(); - } - else - { - tracker = new VarjoCompanionInterface(); - } - Logger.Msg(string.Format("Initializing {0} Varjo module", tracker.GetName())); - bool pipeConnected = tracker.Initialize(); + // Init our tracker first + tracker = new VarjoNativeInterface(); + bool pipeConnected = tracker.Initialize(Logger); + if (pipeConnected) { - unsafe + // if the tracker has init'ed, get the first 4 chars of the HMD name for Icon and Module name (for VR-#, XR-# or AERO) + string hmdName = tracker.GetHMDName().Substring(0, 4); + + // in case we're dealing with the Aero, capitilize the name properly + if (hmdName == "AERO") hmdName = "Aero"; + var hmdIcon = GetType().Assembly.GetManifestResourceStream("VRCFTVarjoModule.Assets." + hmdName + ".png"); + + // if no icon can be found the the reported HMD, use defaults and log the full name + if (hmdIcon == null) { - try - { - MemMapFile = MemoryMappedFile.OpenExisting("Global\\VarjoTrackerInfo"); - ViewAccessor = MemMapFile.CreateViewAccessor(); - byte* ptr = null; - ViewAccessor.SafeMemoryMappedViewHandle.AcquirePointer(ref ptr); - EyeImagePointer = new IntPtr(ptr); - UnifiedTrackingData.LatestEyeData.SupportsImage = true; - UnifiedTrackingData.LatestEyeData.ImageSize = (CAMERA_WIDTH, CAMERA_HEIGHT); - } - catch (FileNotFoundException) - { - Logger.Warning("Varjo camera mapped file doesn't exist; is Varjo Base running?"); - } + Logger.LogInformation("Unknown HMD Name: " + tracker.GetHMDName()); + ModuleInformation.Name = "Varjo Eye Tracking"; + hmdIcon = GetType().Assembly.GetManifestResourceStream("VRCFTVarjoModule.Assets.unknown.png"); } + else + { + ModuleInformation.Name = "Varjo " + hmdName + " Eye Tracking"; + } + + ModuleInformation.StaticImages = hmdIcon != null ? new List { hmdIcon } : ModuleInformation.StaticImages; } - return (pipeConnected, false); - } - // Detects if the module is running in the standalone version of VRCFT - private bool IsStandalone() - { - return true; // uuuh that will do anyway + // Tell the lib manager our result with the init and that we (again) do not support Lip Tracking + return (pipeConnected, false); } - // This will be run in the tracking thread. This is exposed so you can control when and if the tracking data is updated down to the lowest level. - public override Action GetUpdateThreadFunc() + // Update function to be called in a while(true) loop. Keeping a delay is necessary to not fully load an entire CPU thread! (despite what the docs state) + public override void Update() { - _cancellationToken = new CancellationTokenSource(); - return () => + if (Status == ModuleState.Active) { - while (!_cancellationToken.IsCancellationRequested) + // try and update data; log an error on failure and wait 250ms + if (tracker.Update()) { - Update(); - Thread.Sleep(10); + TrackingData.Update(ref UnifiedTracking.Data.Eye, ref UnifiedTracking.Data.Shapes, tracker.GetGazeData(), tracker.GetEyeMeasurements()); + } + else + { + Logger.LogWarning("There seems to be an issue with getting Tracking data. Will try again in 250ms."); + Thread.Sleep(240); } - }; - } - - private unsafe void UpdateEyeImage() - { - if (MemMapFile == null || EyeImagePointer == null || !UnifiedTrackingData.LatestEyeData.SupportsImage) - { - return; - } - if (UnifiedTrackingData.LatestEyeData.ImageData == null) - { - UnifiedTrackingData.LatestEyeData.ImageData = new byte[CAMERA_WIDTH * CAMERA_HEIGHT]; } - Marshal.Copy(EyeImagePointer, UnifiedTrackingData.LatestEyeData.ImageData, 0, CAMERA_WIDTH * CAMERA_HEIGHT); - } - // The update function needs to be defined separately in case the user is running with the --vrcft-nothread launch parameter - public void Update() - { - if (Status.EyeState == ModuleState.Active) - { - tracker.Update(); - TrackingData.Update(ref UnifiedTrackingData.LatestEyeData, tracker.GetGazeData(), tracker.GetEyeMeasurements()); - } - UpdateEyeImage(); + // Sleep the thread for 10ms (aka let the update run at 100Hz) + Thread.Sleep(10); } - // A chance to de-initialize everything. This runs synchronously inside main game thread. Do not touch any Unity objects here. + // Function to be called when the module is torn down; this call should be passed through to the Varjo Interface public override void Teardown() { - _cancellationToken.Cancel(); tracker.Teardown(); - ViewAccessor.SafeMemoryMappedViewHandle.ReleasePointer(); - _cancellationToken.Dispose(); } } } \ No newline at end of file diff --git a/VarjoInterface.cs b/VarjoTypes.cs similarity index 78% rename from VarjoInterface.cs rename to VarjoTypes.cs index 16c2935..25521bc 100644 --- a/VarjoInterface.cs +++ b/VarjoTypes.cs @@ -1,11 +1,4 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Runtime.InteropServices; -using System.Text; -using System.Threading.Tasks; -using VRCFaceTracking; +using System.Runtime.InteropServices; namespace VRCFTVarjoModule { @@ -118,39 +111,21 @@ public struct GazeCalibrationQuality public GazeEyeCalibrationQuality right; } - - public abstract class VarjoInterface + public enum VarjoPropertyKey { - protected GazeData gazeData; - protected EyeMeasurements eyeMeasurements; - - public GazeData GetGazeData() - { - return gazeData; - } - - public EyeMeasurements GetEyeMeasurements() - { - return eyeMeasurements; - } - - public abstract void Teardown(); - public abstract bool Initialize(); - public abstract void Update(); - - public abstract string GetName(); - - protected string GetModuleDir() - { - return Utils.PersistentDataDirectory + "\\CustomLibs\\Varjo"; - } - - protected bool VarjoAvailable() - { - // totally not how the official Varjo library works under the hood - return File.Exists("\\\\.\\pipe\\Varjo\\InfoService"); - } - + Invalid = 0x0, + UserPresence = 0x2000, + GazeCalibrating = 0xA000, + GazeCalibrated = 0xA001, + GazeCalibrationQuality = 0xA002, + GazeAllowed = 0xA003, + GazeEyeCalibrationQuality_Left = 0xA004, + GazeEyeCalibrationQuality_Right = 0xA005, + GazeIPDEstimate = 0xA010, + HMDConnected = 0xE001, + HMDProductName = 0xE002, + HMDSerialNumber = 0xE003, + MRAvailable = 0xD000 }