Desktop Head Tracking mod

README.md

@@ -5,6 +5,7 @@ Merged set of MelonLoader mods for ChilloutVR.
 |-----------|------------|----------------|-----------------------------------------------------------------|----------------|-------|
 | Avatar Change Info | ml_aci | 1.0.2 | Yes | Working |
 | Avatar Motion Tweaker | ml_amt | 1.1.0 | On review | Working |
+| Desktop Head Tracking | ml_dht | 1.0.0. | No | Working |
 | Desktop Reticle Switch | ml_drs | 1.0.0 | Yes | Working |
 | Four Point Tracking | ml_fpt | 1.0.6 | On review | Working |
 | Leap Motion Extension | ml_lme | 1.1.8 | Yes | Working |

ml_dht/FaceTracked.cs

@@ -0,0 +1,101 @@
+using ABI_RC.Core.Player;
+using UnityEngine;
+
+namespace ml_dht
+{
+    class FaceTracked : MonoBehaviour
+    {
+        bool m_enabled = true;
+        float m_smoothing = 0.5f;
+        bool m_mirrored = false;
+
+        RootMotion.FinalIK.LookAtIK m_lookIK = null;
+        Transform m_camera = null;
+        Transform m_headBone = null;
+
+        Vector3 m_headPosition;
+        Quaternion m_headRotation;
+        Vector2 m_gazeDirection;
+        float m_blinkProgress = 0f;
+        Vector2 m_mouthShapes;
+        float m_eyebrowsProgress = 0f;
+
+        Quaternion m_bindRotation;
+        Quaternion m_lastHeadRotation;
+
+        void Start()
+        {
+            m_camera = PlayerSetup.Instance.desktopCamera.transform;
+        }
+
+        public void UpdateTrackingData(ref TrackingData p_data)
+        {
+            m_headPosition.Set(p_data.m_headPositionX * (m_mirrored ? -1f : 1f), p_data.m_headPositionY, p_data.m_headPositionZ);
+            m_headRotation.Set(p_data.m_headRotationX, p_data.m_headRotationY * (m_mirrored ? -1f : 1f), p_data.m_headRotationZ * (m_mirrored ? -1f : 1f), p_data.m_headRotationW);
+            m_gazeDirection.Set(m_mirrored ? (1f - p_data.m_gazeX) : p_data.m_gazeX, p_data.m_gazeY);
+            m_blinkProgress = p_data.m_blink;
+            m_mouthShapes.Set(p_data.m_mouthOpen, p_data.m_mouthShape);
+            m_eyebrowsProgress = p_data.m_brows;
+        }
+
+        public void OnEyeControllerUpdate()
+        {
+            if(m_enabled)
+            {
+                // Gaze
+                PlayerSetup.Instance.eyeMovement.manualViewTarget = true;
+                PlayerSetup.Instance.eyeMovement.targetViewPosition = m_camera.position + m_camera.rotation * new Vector3((m_gazeDirection.x - 0.5f) * -2f, (m_gazeDirection.y - 0.5f) * 2f, 1f);
+
+                // Blink
+                PlayerSetup.Instance.eyeMovement.manualBlinking = true;
+                PlayerSetup.Instance.eyeMovement.blinkProgress = m_blinkProgress;
+            }
+        }
+
+        void OnLookIKPostUpdate()
+        {
+            if(m_enabled && (m_headBone != null))
+            {
+                m_lastHeadRotation = Quaternion.Slerp(m_lastHeadRotation, m_headRotation * m_bindRotation, m_smoothing);
+                m_headBone.localRotation = m_lastHeadRotation;
+            }
+        }
+
+        public void OnSetupAvatarGeneral()
+        {
+            m_headBone = PlayerSetup.Instance._animator.GetBoneTransform(HumanBodyBones.Head);
+            m_lookIK = PlayerSetup.Instance._avatar.GetComponent<RootMotion.FinalIK.LookAtIK>();
+
+            if(m_headBone != null)
+                m_bindRotation = m_headBone.localRotation;
+
+            if(m_lookIK != null)
+                m_lookIK.solver.OnPostUpdate += this.OnLookIKPostUpdate;
+        }
+        public void OnAvatarClear()
+        {
+            m_lookIK = null;
+            m_headBone = null;
+            m_lastHeadRotation = Quaternion.identity;
+            m_bindRotation = Quaternion.identity;
+        }
+
+        public void SetEnabled(bool p_state)
+        {
+            if(m_enabled != p_state)
+            {
+                m_enabled = p_state;
+                if(m_enabled)
+                    m_lastHeadRotation = m_bindRotation;
+            }
+        }
+        public void SetSmoothing(float p_value)
+        {
+            m_smoothing = 1f - Mathf.Clamp(p_value, 0f, 0.99f);
+        }
+        public void SetMirrored(bool p_state)
+        {
+            m_mirrored = p_state;
+        }
+    }
+}

ml_dht/Main.cs

@@ -0,0 +1,133 @@
+using ABI_RC.Core.Player;
+
+namespace ml_dht
+{
+    public class DesktopHeadTracking : MelonLoader.MelonMod
+    {
+        static DesktopHeadTracking ms_instance = null;
+
+        MemoryMapReader m_mapReader = null;
+        byte[] m_buffer = null;
+        TrackingData m_trackingData;
+
+        FaceTracked m_localTracked = null;
+
+        public override void OnApplicationStart()
+        {
+            if(ms_instance == null)
+                ms_instance = this;
+
+            Settings.Init();
+            Settings.EnabledChange += this.OnEnabledChanged;
+            Settings.MirroredChange += this.OnMirroredChanged;
+            Settings.SmoothingChange += this.OnSmoothingChanged;
+
+            m_mapReader = new MemoryMapReader();
+            m_buffer = new byte[1024];
+
+            m_mapReader.Open("head/data");
+
+            // Patches
+            HarmonyInstance.Patch(
+                typeof(PlayerSetup).GetMethod(nameof(PlayerSetup.ClearAvatar)),
+                null,
+                new HarmonyLib.HarmonyMethod(typeof(DesktopHeadTracking).GetMethod(nameof(OnAvatarClear_Postfix), System.Reflection.BindingFlags.Static | System.Reflection.BindingFlags.NonPublic))
+            );
+            HarmonyInstance.Patch(
+                typeof(PlayerSetup).GetMethod("SetupAvatarGeneral", System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic),
+                null,
+                new HarmonyLib.HarmonyMethod(typeof(DesktopHeadTracking).GetMethod(nameof(OnSetupAvatarGeneral_Postfix), System.Reflection.BindingFlags.Static | System.Reflection.BindingFlags.NonPublic))
+            );
+            HarmonyInstance.Patch(
+                typeof(CVREyeController).GetMethod("Update", System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic),
+                null,
+                new HarmonyLib.HarmonyMethod(typeof(DesktopHeadTracking).GetMethod(nameof(OnEyeControllerUpdate_Postfix), System.Reflection.BindingFlags.Static | System.Reflection.BindingFlags.NonPublic))
+            );
+
+            MelonLoader.MelonCoroutines.Start(WaitForPlayer());    
+        }
+
+        System.Collections.IEnumerator WaitForPlayer()
+        {
+            while(PlayerSetup.Instance == null)
+                yield return null;
+
+            m_localTracked = PlayerSetup.Instance.gameObject.AddComponent<FaceTracked>();
+            m_localTracked.SetEnabled(Settings.Enabled);
+            m_localTracked.SetMirrored(Settings.Mirrored);
+            m_localTracked.SetSmoothing(Settings.Smoothing);
+        }
+
+        public override void OnUpdate()
+        {
+            if(Settings.Enabled && m_mapReader.Read(ref m_buffer))
+            {
+                m_trackingData = TrackingData.ToObject(m_buffer);
+                if(m_localTracked != null)
+                    m_localTracked.UpdateTrackingData(ref m_trackingData);
+            }
+        }
+
+        void OnEnabledChanged(bool p_state)
+        {
+            if(m_localTracked != null)
+                m_localTracked.SetEnabled(p_state);
+        }
+        void OnMirroredChanged(bool p_state)
+        {
+            if(m_localTracked != null)
+                m_localTracked.SetMirrored(p_state);
+        }
+        void OnSmoothingChanged(float p_value)
+        {
+            if(m_localTracked != null)
+                m_localTracked.SetSmoothing(p_value);
+        }
+
+        static void OnSetupAvatarGeneral_Postfix() => ms_instance?.OnSetupAvatarGeneral();
+        void OnSetupAvatarGeneral()
+        {
+            try
+            {
+                if(m_localTracked != null)
+                    m_localTracked.OnSetupAvatarGeneral();
+            }
+            catch(System.Exception e)
+            {
+                MelonLoader.MelonLogger.Error(e);
+            }
+        }
+
+        static void OnAvatarClear_Postfix() => ms_instance?.OnAvatarClear();
+        void OnAvatarClear()
+        {
+            try
+            {
+                if(m_localTracked != null)
+                    m_localTracked.OnAvatarClear();
+            }
+            catch(System.Exception e)
+            {
+                MelonLoader.MelonLogger.Error(e);
+            }
+        }
+
+        static void OnEyeControllerUpdate_Postfix(ref CVREyeController __instance)
+        {
+            try
+            {
+                if(__instance == PlayerSetup.Instance.eyeMovement)
+                    ms_instance?.OnEyeControllerUpdate();
+            }
+            catch(System.Exception e)
+            {
+                MelonLoader.MelonLogger.Error(e);
+            }
+        }
+        void OnEyeControllerUpdate()
+        {
+            if(m_localTracked != null)
+                m_localTracked.OnEyeControllerUpdate();
+        }
+    }
+}

ml_dht/MemoryMapReader.cs

@@ -0,0 +1,55 @@
+using System.IO;
+using System.IO.MemoryMappedFiles;
+
+namespace ml_dht
+{
+    class MemoryMapReader
+    {
+        MemoryMappedFile m_file = null;
+        MemoryMappedViewStream m_stream = null;
+        int m_dataSize = 0;
+
+        public bool Open(string p_path, int p_dataSize = 1024)
+        {
+            if(m_file == null)
+            {
+                m_dataSize = p_dataSize;
+
+                m_file = MemoryMappedFile.CreateOrOpen(p_path, m_dataSize, MemoryMappedFileAccess.ReadWrite);
+                m_stream = m_file.CreateViewStream(0, m_dataSize, MemoryMappedFileAccess.Read);
+            }
+            return (m_file != null);
+        }
+
+        public bool Read(ref byte[] p_data)
+        {
+            bool l_result = false;
+            if((m_stream != null) && m_stream.CanRead)
+            {
+                try
+                {
+                    m_stream.Seek(0, SeekOrigin.Begin);
+                    m_stream.Read(p_data, 0, (p_data.Length > m_dataSize) ? m_dataSize : p_data.Length);
+                    l_result = true;
+                }
+                catch(System.Exception) { }
+            }
+            return l_result;
+        }
+
+        public void Close()
+        {
+            if(m_file != null)
+            {
+                m_stream.Close();
+                m_stream.Dispose();
+                m_stream = null;
+
+                m_file.Dispose();
+                m_file = null;
+
+                m_dataSize = 0;
+            }
+        }
+    }
+}

ml_dht/Properties/AssemblyInfo.cs

@@ -0,0 +1,10 @@
+using System.Reflection;
+
+[assembly: AssemblyTitle("DesktopHeadTracking")]
+[assembly: AssemblyVersion("1.0.0")]
+[assembly: AssemblyFileVersion("1.0.0")]
+
+[assembly: MelonLoader.MelonInfo(typeof(ml_dht.DesktopHeadTracking), "DesktopHeadTracking", "1.0.0", "SDraw", "https://github.com/SDraw/ml_mods_cvr")]
+[assembly: MelonLoader.MelonGame(null, "ChilloutVR")]
+[assembly: MelonLoader.MelonPlatform(MelonLoader.MelonPlatformAttribute.CompatiblePlatforms.WINDOWS_X64)]
+[assembly: MelonLoader.MelonPlatformDomain(MelonLoader.MelonPlatformDomainAttribute.CompatibleDomains.MONO)]

ml_dht/README.md

@@ -0,0 +1,27 @@
+# Desktop Head Tracking
+This mod adds desktop head tracking based on data from memory-mapped file `head/data`.  
+Refer to `TrackingData.cs` for reference in case of implementing own software.
+
+[![](.github/img_01.png)](https://youtu.be/jgcFhSNi9DM)
+
+# Features
+* Head rotation
+* Eyes gaze direction
+* Blinking
+
+# Installation
+* Install [latest MelonLoader](https://github.com/LavaGang/MelonLoader)
+* Get [latest release DLL](../../../releases/latest):
+  * Put `ml_dht.dll` in `Mods` folder of game
+
+# Usage
+Available mod's settings in `Settings - Implementation - Desktop Head Tracking`:
+* **Enabled:** enabled head tracking; default value - `false`.
+* **Mirrored movement:** mirrors movement and gaze along 0YZ plane; default value - `false`.
+* **Movement smoothing:** smoothing factor between new and old movement data; default value - `50`;
+
+# Known compatible tracking software
+* VSeeFace with [Tracking Data Parser mod](https://github.com/SDraw/ml_mods_vsf)
+
+# Notes
+* Blinking doesn't work for remote players to due [game's bug](https://feedback.abinteractive.net/p/overrided-blinking-state-isn-t-copied-to-movement-data-from-network).

ml_dht/Scripts.cs

@@ -0,0 +1,26 @@
+using System;
+using System.IO;
+using System.Reflection;
+
+namespace ml_dht
+{
+    static class Scripts
+    {
+        public static string GetEmbeddedScript(string p_name)
+        {
+            string l_result = "";
+            Assembly l_assembly = Assembly.GetExecutingAssembly();
+            string l_assemblyName = l_assembly.GetName().Name;
+
+            try
+            {
+                Stream l_libraryStream = l_assembly.GetManifestResourceStream(l_assemblyName + ".resources." + p_name);
+                StreamReader l_streadReader = new StreamReader(l_libraryStream);
+                l_result = l_streadReader.ReadToEnd();
+            }
+            catch(Exception) { }
+
+            return l_result;
+        }
+    }
+}