Face tracking components override

SDraw · Sep 23, 2022 · b6a4e0c · b6a4e0c
1 parent 65f8992
commit b6a4e0c
Show file tree

Hide file tree

Showing 8 changed files with 99 additions and 10 deletions.
diff --git a/README.md b/README.md
@@ -5,7 +5,7 @@ Merged set of MelonLoader mods for ChilloutVR.
 |-----------|------------|----------------|-----------------------------------------------------------------|----------------|-------|
 | Avatar Change Info | ml_aci | 1.0.2 | Yes | Working |
 | Avatar Motion Tweaker | ml_amt | 1.1.0 | On review | Working |
-| Desktop Head Tracking | ml_dht | 1.0.0 | On review | Working |
+| Desktop Head Tracking | ml_dht | 1.0.1 | On review | Working |
 | Desktop Reticle Switch | ml_drs | 1.0.0 | Yes | Working |
 | Four Point Tracking | ml_fpt | 1.0.6 | On review | Working |
 | Leap Motion Extension | ml_lme | 1.1.8 | Yes | Working |

diff --git a/ml_dht/FaceTracked.cs b/ml_dht/FaceTracked.cs
@@ -1,4 +1,5 @@
-using ABI_RC.Core.Player;
+using ABI.CCK.Components;
+using ABI_RC.Core.Player;
 using UnityEngine;
 
 namespace ml_dht
@@ -8,7 +9,9 @@ class FaceTracked : MonoBehaviour
         bool m_enabled = true;
         float m_smoothing = 0.5f;
         bool m_mirrored = false;
+        bool m_faceOverride = true;
 
+        CVRAvatar m_avatarDescriptior = null;
         RootMotion.FinalIK.LookAtIK m_lookIK = null;
         Transform m_camera = null;
         Transform m_headBone = null;
@@ -61,8 +64,38 @@ void OnLookIKPostUpdate()
             }
         }
 
+        public void OnFaceTrackingUpdate(CVRFaceTracking p_component)
+        {
+            if(m_enabled && m_faceOverride)
+            {
+                if(m_avatarDescriptior != null)
+                    m_avatarDescriptior.useVisemeLipsync = false;
+
+                p_component.BlendShapeValues[(int)ViveSR.anipal.Lip.LipShape_v2.Jaw_Open] = m_mouthShapes.x * 100f;
+
+                if(m_mouthShapes.y < 0f)
+                {
+                    float l_weight = Mathf.Clamp(Mathf.InverseLerp(0.25f, 1f, -m_mouthShapes.y), 0f, 1f) * 100f;
+                    p_component.BlendShapeValues[(int)ViveSR.anipal.Lip.LipShape_v2.Mouth_Pout] = 0f;
+                    p_component.BlendShapeValues[(int)ViveSR.anipal.Lip.LipShape_v2.Mouth_Smile_Left] = l_weight;
+                    p_component.BlendShapeValues[(int)ViveSR.anipal.Lip.LipShape_v2.Mouth_Smile_Right] = l_weight;
+                }
+                if(m_mouthShapes.y > 0f)
+                {
+                    float l_weight = Mathf.Clamp(Mathf.InverseLerp(0.25f, 1f, m_mouthShapes.y), 0f, 1f) * 100f;
+                    p_component.BlendShapeValues[(int)ViveSR.anipal.Lip.LipShape_v2.Mouth_Pout] = l_weight;
+                    p_component.BlendShapeValues[(int)ViveSR.anipal.Lip.LipShape_v2.Mouth_Smile_Left] = 0f;
+                    p_component.BlendShapeValues[(int)ViveSR.anipal.Lip.LipShape_v2.Mouth_Smile_Right] = 0f;
+                }
+
+                p_component.LipSyncWasUpdated = true;
+                p_component.UpdateLipShapes();
+            }
+        }
+
         public void OnSetupAvatarGeneral()
         {
+            m_avatarDescriptior = PlayerSetup.Instance._avatar.GetComponent<CVRAvatar>();
             m_headBone = PlayerSetup.Instance._animator.GetBoneTransform(HumanBodyBones.Head);
             m_lookIK = PlayerSetup.Instance._avatar.GetComponent<RootMotion.FinalIK.LookAtIK>();
 
@@ -71,9 +104,11 @@ public void OnSetupAvatarGeneral()
 
             if(m_lookIK != null)
                 m_lookIK.solver.OnPostUpdate += this.OnLookIKPostUpdate;
+
         }
         public void OnAvatarClear()
         {
+            m_avatarDescriptior = null;
             m_lookIK = null;
             m_headBone = null;
             m_lastHeadRotation = Quaternion.identity;
@@ -97,5 +132,9 @@ public void SetMirrored(bool p_state)
         {
             m_mirrored = p_state;
         }
+        public void SetFaceOverride(bool p_state)
+        {
+            m_faceOverride = p_state;
+        }
     }
 }
diff --git a/ml_dht/Main.cs b/ml_dht/Main.cs
@@ -1,3 +1,4 @@
+using ABI.CCK.Components;
 using ABI_RC.Core.Player;
 
 namespace ml_dht
@@ -21,6 +22,7 @@ public override void OnApplicationStart()
             Settings.EnabledChange += this.OnEnabledChanged;
             Settings.MirroredChange += this.OnMirroredChanged;
             Settings.SmoothingChange += this.OnSmoothingChanged;
+            Settings.FaceOverrideChange += this.OnFaceOverrideChange;
 
             m_mapReader = new MemoryMapReader();
             m_buffer = new byte[1024];
@@ -43,6 +45,11 @@ public override void OnApplicationStart()
                 null,
                 new HarmonyLib.HarmonyMethod(typeof(DesktopHeadTracking).GetMethod(nameof(OnEyeControllerUpdate_Postfix), System.Reflection.BindingFlags.Static | System.Reflection.BindingFlags.NonPublic))
             );
+            HarmonyInstance.Patch(
+                typeof(CVRFaceTracking).GetMethod("Update", System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic),
+                null,
+                new HarmonyLib.HarmonyMethod(typeof(DesktopHeadTracking).GetMethod(nameof(OnFaceTrackingUpdate_Postfix), System.Reflection.BindingFlags.Static | System.Reflection.BindingFlags.NonPublic))
+            );
 
             MelonLoader.MelonCoroutines.Start(WaitForPlayer());    
         }
@@ -83,6 +90,11 @@ void OnSmoothingChanged(float p_value)
             if(m_localTracked != null)
                 m_localTracked.SetSmoothing(p_value);
         }
+        void OnFaceOverrideChange(bool p_state)
+        {
+            if(m_localTracked != null)
+                m_localTracked.SetFaceOverride(p_state);
+        }
 
         static void OnSetupAvatarGeneral_Postfix() => ms_instance?.OnSetupAvatarGeneral();
         void OnSetupAvatarGeneral()
@@ -129,5 +141,19 @@ void OnEyeControllerUpdate()
             if(m_localTracked != null)
                 m_localTracked.OnEyeControllerUpdate();
         }
+
+        static void OnFaceTrackingUpdate_Postfix(ref CVRFaceTracking __instance) => ms_instance?.OnFaceTrackingUpdate(__instance);
+        void OnFaceTrackingUpdate(CVRFaceTracking p_component)
+        {
+            try
+            {
+                if(p_component.isLocal && (m_localTracked != null))
+                    m_localTracked.OnFaceTrackingUpdate(p_component);
+            }
+            catch(System.Exception e)
+            {
+                MelonLoader.MelonLogger.Error(e);
+            }
+        }
     }
 }
diff --git a/ml_dht/Properties/AssemblyInfo.cs b/ml_dht/Properties/AssemblyInfo.cs
@@ -1,10 +1,10 @@
 using System.Reflection;
 
 [assembly: AssemblyTitle("DesktopHeadTracking")]
-[assembly: AssemblyVersion("1.0.0")]
-[assembly: AssemblyFileVersion("1.0.0")]
+[assembly: AssemblyVersion("1.0.1")]
+[assembly: AssemblyFileVersion("1.0.1")]
 
-[assembly: MelonLoader.MelonInfo(typeof(ml_dht.DesktopHeadTracking), "DesktopHeadTracking", "1.0.0", "SDraw", "https://github.com/SDraw/ml_mods_cvr")]
+[assembly: MelonLoader.MelonInfo(typeof(ml_dht.DesktopHeadTracking), "DesktopHeadTracking", "1.0.1", "SDraw", "https://github.com/SDraw/ml_mods_cvr")]
 [assembly: MelonLoader.MelonGame(null, "ChilloutVR")]
 [assembly: MelonLoader.MelonPlatform(MelonLoader.MelonPlatformAttribute.CompatiblePlatforms.WINDOWS_X64)]
 [assembly: MelonLoader.MelonPlatformDomain(MelonLoader.MelonPlatformDomainAttribute.CompatibleDomains.MONO)]
diff --git a/ml_dht/README.md b/ml_dht/README.md
@@ -8,6 +8,7 @@ Refer to `TrackingData.cs` for reference in case of implementing own software.
 * Head rotation
 * Eyes gaze direction
 * Blinking
+* Basic mouth shapes
 
 # Installation
 * Install [latest MelonLoader](https://github.com/LavaGang/MelonLoader)
@@ -18,7 +19,8 @@ Refer to `TrackingData.cs` for reference in case of implementing own software.
 Available mod's settings in `Settings - Implementation - Desktop Head Tracking`:
 * **Enabled:** enabled head tracking; default value - `false`.
 * **Mirrored movement:** mirrors movement and gaze along 0YZ plane; default value - `false`.
-* **Movement smoothing:** smoothing factor between new and old movement data; default value - `50`;
+* **Movement smoothing:** smoothing factor between new and old movement data; default value - `50`.
+* **Override face tracking:** Overrides and activates avatar's `VRC Face Tracking` components. List of used shapes: `Jaw_Open`, `Mouth_Pout`, `Mouth_Smile_Left`, `Mouth_Smile_Right`.
 
 # Known compatible tracking software
 * [VSeeFace](https://www.vseeface.icu) with [Tracking Data Parser mod](https://github.com/SDraw/ml_mods_vsf)

diff --git a/ml_dht/Settings.cs b/ml_dht/Settings.cs
@@ -11,19 +11,22 @@ enum ModSetting
         {
             Enabled = 0,
             Mirrored,
-            Smoothing
+            Smoothing,
+            FaceOverride
         }
 
         static bool ms_enabled = false;
         static bool ms_mirrored = false;
         static float ms_smoothing = 0.5f;
+        static bool ms_faceOverride = true;
 
         static MelonLoader.MelonPreferences_Category ms_category = null;
         static List<MelonLoader.MelonPreferences_Entry> ms_entries = null;
 
         static public event Action<bool> EnabledChange;
         static public event Action<bool> MirroredChange;
         static public event Action<float> SmoothingChange;
+        static public event Action<bool> FaceOverrideChange;
 
         public static void Init()
         {
@@ -33,6 +36,7 @@ public static void Init()
             ms_entries.Add(ms_category.CreateEntry(ModSetting.Enabled.ToString(), false));
             ms_entries.Add(ms_category.CreateEntry(ModSetting.Mirrored.ToString(), false));
             ms_entries.Add(ms_category.CreateEntry(ModSetting.Smoothing.ToString(), 50));
+            ms_entries.Add(ms_category.CreateEntry(ModSetting.FaceOverride.ToString(), true));
 
             Load();
 
@@ -66,6 +70,7 @@ static void Load()
             ms_enabled = (bool)ms_entries[(int)ModSetting.Enabled].BoxedValue;
             ms_mirrored = (bool)ms_entries[(int)ModSetting.Mirrored].BoxedValue;
             ms_smoothing = ((int)ms_entries[(int)ModSetting.Smoothing].BoxedValue) * 0.01f;
+            ms_faceOverride = (bool)ms_entries[(int)ModSetting.FaceOverride].BoxedValue;
         }
 
         static void OnSliderUpdate(string p_name, string p_value)
@@ -105,6 +110,12 @@ static void OnToggleUpdate(string p_name, string p_value)
                         MirroredChange?.Invoke(ms_mirrored);
                     }
                     break;
+
+                    case ModSetting.FaceOverride:
+                    {
+                        ms_faceOverride = bool.Parse(p_value);
+                        FaceOverrideChange?.Invoke(ms_faceOverride);
+                    } break;
                 }
 
                 ms_entries[(int)l_setting].BoxedValue = bool.Parse(p_value);
@@ -123,5 +134,9 @@ public static float Smoothing
         {
             get => ms_smoothing;
         }
+        public static bool FaceOverride
+        {
+            get => ms_faceOverride;
+        }
     }
 }
diff --git a/ml_dht/TrackingData.cs b/ml_dht/TrackingData.cs
@@ -13,9 +13,9 @@ struct TrackingData
     public float m_gazeX; // Range - [0;1], 0.5 - center
     public float m_gazeY; // Range - [0;1], 0.5 - center
     public float m_blink; // Range - [0;1], 1.0 - closed
-    public float m_mouthOpen; // Range - [0;1], not used yet
-    public float m_mouthShape; // Range - [0;1], not used yet
-    public float m_brows; // Range - [0;1], not used yet
+    public float m_mouthOpen; // Range - [0;1]
+    public float m_mouthShape; // Range - [-1;1], -1 - wide, 1 - narrow
+    public float m_brows; // Range - [-1;1], -1 - up, 1 - down; not used yet
 
     static public byte[] ToBytes(TrackingData p_faceData)
     {

diff --git a/ml_dht/resources/menu.js b/ml_dht/resources/menu.js
@@ -200,6 +200,13 @@ function inp_toggle_mod_dht(_obj, _callbackName) {
                 <div id="Smoothing" class ="inp_slider no-scroll" data-min="0" data-max="99" data-current="50"></div>
             </div>
         </div>
+
+        <div class ="row-wrapper">
+            <div class ="option-caption">Override face tracking: </div>
+            <div class ="option-input">
+                <div id="FaceOverride" class ="inp_toggle no-scroll" data-current="true"></div>
+            </div>
+        </div>
     `;
     document.getElementById('settings-implementation').appendChild(l_block);