diff --git a/BabbleConfig.cs b/BabbleConfig.cs new file mode 100644 index 0000000..817d7b8 --- /dev/null +++ b/BabbleConfig.cs @@ -0,0 +1,17 @@ +using Newtonsoft.Json; +using System.Reflection; + +namespace VRCFaceTracking.Babble; + +public static class BabbleConfig +{ + private const string BabbleConfigFile = "BabbleConfig.json"; + + public static Config GetBabbleConfig() + { + string directoryName = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)!; + string path = Path.Combine(directoryName, "BabbleConfig.json"); + string value = File.ReadAllText(path); + return JsonConvert.DeserializeObject(value)!; + } +} diff --git a/BabbleExpressions.cs b/BabbleExpressions.cs index e1454fd..10e4ed6 100644 --- a/BabbleExpressions.cs +++ b/BabbleExpressions.cs @@ -1,86 +1,286 @@ -using Newtonsoft.Json; -using System.Reflection; +using VRCFaceTracking.Babble.Collections; using VRCFaceTracking.Core.Params.Expressions; -using VRCFaceTracking.Babble.Collections; namespace VRCFaceTracking.Babble; + public static class BabbleExpressions { - // private const string BabbleExpressionsPath = "BabbleExpressions.json"; - - // public static readonly TwoKeyDictionary BabbleExpressionMap; - // static BabbleExpressions() - // { - // var expressions = Path.Combine( - // Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)!, - // BabbleExpressionsPath); - // var converter = new TwoKeyDictionaryConverter(); - // if (!File.Exists(expressions)) - // { - // string json = JsonConvert.SerializeObject( - // DefaultBabbleExpressionMap, converter); - // File.WriteAllText(expressions, json); - // } - - // BabbleExpressionMap = JsonConvert.DeserializeObject - // > - // (File.ReadAllText(expressions), converter)!; - // } - - private static readonly TwoKeyDictionary BabbleExpressionMap = new() + public static readonly TwoKeyDictionary BabbleExpressionMap = new TwoKeyDictionary { - { UnifiedExpressions.CheekPuffLeft, "/cheekPuffLeft", 0f }, - { UnifiedExpressions.CheekPuffRight, "/cheekPuffRight", 0f }, - { UnifiedExpressions.CheekSuckLeft, "/cheekSuckLeft", 0f }, - { UnifiedExpressions.CheekSuckRight, "/cheekSuckRight", 0f }, - { UnifiedExpressions.JawOpen, "/jawOpen", 0f }, - { UnifiedExpressions.JawForward, "/jawForward", 0f }, - { UnifiedExpressions.JawLeft, "/jawLeft", 0f }, - { UnifiedExpressions.JawRight, "/jawRight", 0f }, - { UnifiedExpressions.NoseSneerLeft, "/noseSneerLeft", 0f }, - { UnifiedExpressions.NoseSneerRight, "/noseSneerRight", 0f }, - { UnifiedExpressions.LipFunnelLowerLeft, "/mouthFunnel", 0f }, - { UnifiedExpressions.LipFunnelLowerRight, "/mouthFunnel", 0f }, - { UnifiedExpressions.LipFunnelUpperLeft, "/mouthFunnel", 0f }, - { UnifiedExpressions.LipFunnelUpperRight, "/mouthFunnel", 0f }, - { UnifiedExpressions.LipPuckerLowerLeft, "/mouthPucker", 0f }, - { UnifiedExpressions.LipPuckerLowerRight, "/mouthPucker", 0f }, - { UnifiedExpressions.LipPuckerUpperLeft, "/mouthPucker", 0f }, - { UnifiedExpressions.LipPuckerUpperRight, "/mouthPucker", 0f }, - { UnifiedExpressions.MouthPressLeft, "/mouthLeft", 0f }, - { UnifiedExpressions.MouthPressRight, "/mouthRight", 0f }, - { UnifiedExpressions.LipSuckUpperLeft, "/mouthRollUpper", 0f }, - { UnifiedExpressions.LipSuckUpperRight, "/mouthRollUpper", 0f }, - { UnifiedExpressions.LipSuckLowerLeft, "/mouthRollLower", 0f }, - { UnifiedExpressions.LipSuckLowerRight, "/mouthRollLower", 0f }, - { UnifiedExpressions.MouthRaiserUpper, "/mouthShrugUpper", 0f }, - { UnifiedExpressions.MouthRaiserLower, "/mouthShrugLower", 0f }, - { UnifiedExpressions.MouthClosed, "/mouthClose", 0f }, - { UnifiedExpressions.MouthCornerPullLeft, "/mouthSmileLeft", 0f }, - { UnifiedExpressions.MouthCornerPullRight, "/mouthSmileRight", 0f }, - { UnifiedExpressions.MouthFrownLeft, "/mouthFrownLeft", 0f }, - { UnifiedExpressions.MouthFrownRight, "/mouthFrownRight", 0f }, - { UnifiedExpressions.MouthDimpleLeft, "/mouthDimpleLeft", 0f }, - { UnifiedExpressions.MouthDimpleRight, "/mouthDimpleRight", 0f }, - { UnifiedExpressions.MouthUpperUpLeft, "/mouthUpperUpLeft", 0f }, - { UnifiedExpressions.MouthUpperUpRight, "/mouthUpperUpRight", 0f }, - { UnifiedExpressions.MouthLowerDownLeft, "/mouthLowerDownLeft", 0f }, - { UnifiedExpressions.MouthLowerDownRight, "/mouthLowerDownRight", 0f }, - { UnifiedExpressions.MouthPressLeft, "/mouthPressLeft", 0f }, - { UnifiedExpressions.MouthPressRight, "/mouthPressRight", 0f }, - { UnifiedExpressions.MouthStretchLeft, "/mouthStretchLeft", 0f }, - { UnifiedExpressions.MouthStretchRight, "/mouthStretchRight", 0f }, - { UnifiedExpressions.TongueOut, "/tongueOut", 0f }, - { UnifiedExpressions.TongueUp, "/tongueUp", 0f }, - { UnifiedExpressions.TongueDown, "/tongueDown", 0f }, - { UnifiedExpressions.TongueLeft, "/tongueLeft", 0f }, - { UnifiedExpressions.TongueRight, "/tongueRight", 0f }, - { UnifiedExpressions.TongueRoll, "/tongueRoll", 0f }, - { UnifiedExpressions.TongueBendDown, "/tongueBendDown", 0f }, - { UnifiedExpressions.TongueCurlUp, "/tongueCurlUp", 0f }, - { UnifiedExpressions.TongueSquish, "/tongueSquish", 0f }, - { UnifiedExpressions.TongueFlat, "/tongueFlat", 0f }, - { UnifiedExpressions.TongueTwistLeft, "/tongueTwistLeft", 0f }, - { UnifiedExpressions.TongueTwistRight, "/tongueTwistRight", 0 } + { + UnifiedExpressions.CheekPuffLeft, + "/cheekPuffLeft", + 0f + }, + { + UnifiedExpressions.CheekPuffRight, + "/cheekPuffRight", + 0f + }, + { + UnifiedExpressions.CheekSuckLeft, + "/cheekSuckLeft", + 0f + }, + { + UnifiedExpressions.CheekSuckRight, + "/cheekSuckRight", + 0f + }, + { + UnifiedExpressions.JawOpen, + "/jawOpen", + 0f + }, + { + UnifiedExpressions.JawForward, + "/jawForward", + 0f + }, + { + UnifiedExpressions.JawLeft, + "/jawLeft", + 0f + }, + { + UnifiedExpressions.JawRight, + "/jawRight", + 0f + }, + { + UnifiedExpressions.NoseSneerLeft, + "/noseSneerLeft", + 0f + }, + { + UnifiedExpressions.NoseSneerRight, + "/noseSneerRight", + 0f + }, + { + UnifiedExpressions.LipFunnelLowerLeft, + "/mouthFunnel", + 0f + }, + { + UnifiedExpressions.LipFunnelLowerRight, + "/mouthFunnel", + 0f + }, + { + UnifiedExpressions.LipFunnelUpperLeft, + "/mouthFunnel", + 0f + }, + { + UnifiedExpressions.LipFunnelUpperRight, + "/mouthFunnel", + 0f + }, + { + UnifiedExpressions.LipPuckerLowerLeft, + "/mouthPucker", + 0f + }, + { + UnifiedExpressions.LipPuckerLowerRight, + "/mouthPucker", + 0f + }, + { + UnifiedExpressions.LipPuckerUpperLeft, + "/mouthPucker", + 0f + }, + { + UnifiedExpressions.LipPuckerUpperRight, + "/mouthPucker", + 0f + }, + { + UnifiedExpressions.MouthUpperLeft, + "/mouthLeft", + 0f + }, + { + UnifiedExpressions.MouthLowerLeft, + "/mouthLeft", + 0f + }, + { + UnifiedExpressions.MouthUpperRight, + "/mouthRight", + 0f + }, + { + UnifiedExpressions.MouthLowerRight, + "/mouthRight", + 0f + }, + { + UnifiedExpressions.LipSuckUpperLeft, + "/mouthRollUpper", + 0f + }, + { + UnifiedExpressions.LipSuckUpperRight, + "/mouthRollUpper", + 0f + }, + { + UnifiedExpressions.LipSuckLowerLeft, + "/mouthRollLower", + 0f + }, + { + UnifiedExpressions.LipSuckLowerRight, + "/mouthRollLower", + 0f + }, + { + UnifiedExpressions.MouthRaiserUpper, + "/mouthShrugUpper", + 0f + }, + { + UnifiedExpressions.MouthRaiserLower, + "/mouthShrugLower", + 0f + }, + { + UnifiedExpressions.MouthClosed, + "/mouthClose", + 0f + }, + { + UnifiedExpressions.MouthCornerPullLeft, + "/mouthSmileLeft", + 0f + }, + { + UnifiedExpressions.MouthCornerPullRight, + "/mouthSmileRight", + 0f + }, + { + UnifiedExpressions.MouthFrownLeft, + "/mouthFrownLeft", + 0f + }, + { + UnifiedExpressions.MouthFrownRight, + "/mouthFrownRight", + 0f + }, + { + UnifiedExpressions.MouthDimpleLeft, + "/mouthDimpleLeft", + 0f + }, + { + UnifiedExpressions.MouthDimpleRight, + "/mouthDimpleRight", + 0f + }, + { + UnifiedExpressions.MouthUpperUpLeft, + "/mouthUpperUpLeft", + 0f + }, + { + UnifiedExpressions.MouthUpperUpRight, + "/mouthUpperUpRight", + 0f + }, + { + UnifiedExpressions.MouthLowerDownLeft, + "/mouthLowerDownLeft", + 0f + }, + { + UnifiedExpressions.MouthLowerDownRight, + "/mouthLowerDownRight", + 0f + }, + { + UnifiedExpressions.MouthPressLeft, + "/mouthPressLeft", + 0f + }, + { + UnifiedExpressions.MouthPressRight, + "/mouthPressRight", + 0f + }, + { + UnifiedExpressions.MouthStretchLeft, + "/mouthStretchLeft", + 0f + }, + { + UnifiedExpressions.MouthStretchRight, + "/mouthStretchRight", + 0f + }, + { + UnifiedExpressions.TongueOut, + "/tongueOut", + 0f + }, + { + UnifiedExpressions.TongueUp, + "/tongueUp", + 0f + }, + { + UnifiedExpressions.TongueDown, + "/tongueDown", + 0f + }, + { + UnifiedExpressions.TongueLeft, + "/tongueLeft", + 0f + }, + { + UnifiedExpressions.TongueRight, + "/tongueRight", + 0f + }, + { + UnifiedExpressions.TongueRoll, + "/tongueRoll", + 0f + }, + { + UnifiedExpressions.TongueBendDown, + "/tongueBendDown", + 0f + }, + { + UnifiedExpressions.TongueCurlUp, + "/tongueCurlUp", + 0f + }, + { + UnifiedExpressions.TongueSquish, + "/tongueSquish", + 0f + }, + { + UnifiedExpressions.TongueFlat, + "/tongueFlat", + 0f + }, + { + UnifiedExpressions.TongueTwistLeft, + "/tongueTwistLeft", + 0f + }, + { + UnifiedExpressions.TongueTwistRight, + "/tongueTwistRight", + 0f + } }; } diff --git a/BabbleOSC.cs b/BabbleOSC.cs index cbeec68..7837c2b 100644 --- a/BabbleOSC.cs +++ b/BabbleOSC.cs @@ -1,109 +1,107 @@ -using System; +using Microsoft.Extensions.Logging; using System.Net; using System.Net.Sockets; -using System.Threading; -using Microsoft.Extensions.Logging; using VRCFaceTracking.Core.OSC; namespace VRCFaceTracking.Babble; public class BabbleOSC { - private Socket _receiver; + private Socket _receiver; - private bool _loop = true; + private bool _loop = true; - private readonly Thread _thread; + private readonly Thread _thread; - private readonly ILogger _logger; + private readonly ILogger _logger; - private readonly int _resolvedPort; - private readonly string _resolvedHost; + private readonly int _resolvedPort; - private const string DEFAULT_HOST = "127.0.0.1"; - private const int DEFAULT_PORT = 8888; + private readonly string _resolvedHost; - private const int TIMEOUT_MS = 10000; + public const string DEFAULT_HOST = "127.0.0.1"; - public BabbleOSC(ILogger iLogger, int? port = null, string? host = null) - { - _logger = iLogger; - if (_receiver != null) - { - _logger.LogError("BabbleOSC connection already exists."); - return; - } + public const int DEFAULT_PORT = 8888; - _resolvedHost = host ?? DEFAULT_HOST; - _resolvedPort = port ?? DEFAULT_PORT; + private const int TIMEOUT_MS = 10000; - ConfigureReceiver(); + public BabbleOSC(ILogger iLogger, string? host = null, int? port = null) + { + _logger = iLogger; + if (_receiver != null) + { + _logger.LogError("BabbleOSC connection already exists."); + return; + } + _resolvedHost = host ?? DEFAULT_HOST; + _resolvedPort = port ?? TIMEOUT_MS; - _loop = true; - _thread = new Thread(ListenLoop); - _thread.Start(); - } + _logger.LogInformation($"Started BabbleOSC with Host: {_resolvedHost} and Port {_resolvedPort}"); + ConfigureReceiver(); + _loop = true; + _thread = new Thread(new ThreadStart(ListenLoop)); + _thread.Start(); + } - private void ConfigureReceiver() - { - var ipAddress = IPAddress.Parse(_resolvedHost); - var ipEndPoint = new IPEndPoint(ipAddress, _resolvedPort); - - _receiver = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp); - _receiver.Bind(ipEndPoint); - _receiver.ReceiveTimeout = TIMEOUT_MS; - } + private void ConfigureReceiver() + { + IPAddress address = IPAddress.Parse(_resolvedHost); + IPEndPoint localEP = new IPEndPoint(address, _resolvedPort); + _receiver = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp); + _receiver.Bind(localEP); + _receiver.ReceiveTimeout = TIMEOUT_MS; + } - private void ListenLoop() - { - byte[] array = new byte[4096]; - while (_loop) - { - try - { - if (_receiver.IsBound) - { - int num = _receiver.Receive(array); - int num2 = 0; - OscMessage val; - try - { - val = new OscMessage(array, num, ref num2); - } - catch (Exception) - { - continue; - } - if (val.Value is float) - { - if (val.Address == "/mouthFunnel" || val.Address == "/mouthPucker") - { - BabbleExpressions.BabbleExpressionMap.SetByKey2(val.Address, (float)val.Value * 4f); - } - else if (BabbleExpressions.BabbleExpressionMap.ContainsKey2(val.Address)) - { - BabbleExpressions.BabbleExpressionMap.SetByKey2(val.Address, (float)val.Value); - } - } - } - else - { - _receiver.Close(); - _receiver.Dispose(); - ConfigureReceiver(); - } - } - catch (Exception) - { - } - } - } + private void ListenLoop() + { + byte[] array = new byte[4096]; + while (_loop) + { + try + { + if (_receiver.IsBound) + { + int len = _receiver.Receive(array); + int messageIndex = 0; + OscMessage oscMessage; + try + { + oscMessage = new OscMessage(array, len, ref messageIndex); + } + catch (Exception) + { + continue; + } + if (oscMessage.Value is float) + { + if (oscMessage.Address == "/mouthFunnel" || oscMessage.Address == "/mouthPucker") + { + BabbleExpressions.BabbleExpressionMap.SetByKey2(oscMessage.Address, (float)oscMessage.Value * 4f); + } + else if (BabbleExpressions.BabbleExpressionMap.ContainsKey2(oscMessage.Address)) + { + BabbleExpressions.BabbleExpressionMap.SetByKey2(oscMessage.Address, (float)oscMessage.Value); + } + } + } + else + { + _receiver.Close(); + _receiver.Dispose(); + ConfigureReceiver(); + } + } + catch (Exception) + { + } + } + } - public void Teardown() - { - _loop = false; - _receiver.Close(); - _receiver.Dispose(); - _thread.Join(); - } + public void Teardown() + { + _loop = false; + _receiver.Close(); + _receiver.Dispose(); + _thread.Join(); + } } diff --git a/BabbleVRC.cs b/BabbleVRC.cs index 3796486..7eba7d5 100644 --- a/BabbleVRC.cs +++ b/BabbleVRC.cs @@ -1,8 +1,8 @@ using System.Reflection; -using VRCFaceTracking.Core.Params.Data; using VRCFaceTracking.Core.Params.Expressions; namespace VRCFaceTracking.Babble; + public class BabbleVRC : ExtTrackingModule { private BabbleOSC babbleOSC; @@ -11,30 +11,31 @@ public class BabbleVRC : ExtTrackingModule public override (bool eyeSuccess, bool expressionSuccess) Initialize(bool eyeAvailable, bool expressionAvailable) { - babbleOSC = new BabbleOSC(Logger); - - List streams = new List(); - Assembly a = Assembly.GetExecutingAssembly(); - var hmdStream = a.GetManifestResourceStream - ("VRCFaceTracking.Babble.BabbleLogo.png"); - streams.Add(hmdStream!); - ModuleInformation = new ModuleMetadata() + Config babbleConfig = BabbleConfig.GetBabbleConfig(); + babbleOSC = new BabbleOSC(Logger, babbleConfig.Host, babbleConfig.Port); + List list = new List(); + Assembly executingAssembly = Assembly.GetExecutingAssembly(); + Stream manifestResourceStream = executingAssembly.GetManifestResourceStream("VRCFaceTracking.Babble.BabbleLogo.png")!; + list.Add(manifestResourceStream); + ModuleInformation = new ModuleMetadata { - Name = "Project Babble Face Tracking\nInference Model v2.0.7", - StaticImages = streams + Name = "Project Babble Face Tracking\nInference Model v2.1.1", + StaticImages = list }; - return (false, true); } - public override void Teardown() => babbleOSC.Teardown(); - + public override void Teardown() + { + babbleOSC.Teardown(); + } + public override void Update() { - foreach (var unifiedExpression in BabbleExpressions.BabbleExpressionMap) + foreach (UnifiedExpressions expression in BabbleExpressions.BabbleExpressionMap) { - UnifiedTracking.Data.Shapes[(int) unifiedExpression].Weight = BabbleExpressions.BabbleExpressionMap.GetByKey1 (unifiedExpression); + UnifiedTracking.Data.Shapes[(int)expression].Weight = BabbleExpressions.BabbleExpressionMap.GetByKey1(expression); } - Thread.Sleep(10)' + Thread.Sleep(10); } } diff --git a/Config.cs b/Config.cs new file mode 100644 index 0000000..508be8f --- /dev/null +++ b/Config.cs @@ -0,0 +1,8 @@ +namespace VRCFaceTracking.Babble; + +public class Config +{ + public string Host { get; set; } + + public int Port { get; set; } +} diff --git a/TwoKeyDictionaryConverter.cs b/TwoKeyDictionaryConverter.cs deleted file mode 100644 index c3ae88e..0000000 --- a/TwoKeyDictionaryConverter.cs +++ /dev/null @@ -1,43 +0,0 @@ -using System; -using System.Collections.Generic; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; - -namespace VRCFaceTracking.Babble; - -public class TwoKeyDictionaryConverter : JsonConverter> -{ - public override bool CanRead => true; - - public override bool CanWrite => true; - - public override void WriteJson(JsonWriter writer, TwoKeyDictionary tkd, JsonSerializer serializer) - { - List list = new List(); - for (int i = 0; i < tkd.Count; i++) - { - (TKey1, TKey2, TValue) tuple = tkd.ElementAt(i); - JObject item = new JObject(new JProperty("unifiedExpression", tuple.Item1.ToString()), new JProperty("oscAddress", tuple.Item2), new JProperty("weight", tuple.Item3)); - list.Add(item); - } - serializer.Serialize(writer, list); - } - - public override TwoKeyDictionary ReadJson(JsonReader reader, Type objectType, TwoKeyDictionary existingValue, bool hasExistingValue, JsonSerializer serializer) - { - List list = serializer.Deserialize>(reader); - if (list == null) - { - return null; - } - TwoKeyDictionary twoKeyDictionary = new TwoKeyDictionary(); - foreach (JObject item in list) - { - TKey1 key = (TKey1)Enum.Parse(typeof(TKey1), item["unifiedExpression"].Value()); - TKey2 key2 = item["oscAddress"].Value(); - TValue value = item["weight"].Value(); - twoKeyDictionary.Add(key, key2, value); - } - return twoKeyDictionary; - } -} diff --git a/VRCFaceTracking.Babble.csproj b/VRCFaceTracking.Babble.csproj index 5989a1e..de3a6ca 100644 --- a/VRCFaceTracking.Babble.csproj +++ b/VRCFaceTracking.Babble.csproj @@ -15,6 +15,11 @@ + + + + + ..\..\..\..\..\Program Files\WindowsApps\96ba052f-0948-44d8-86c4-a0212e4ae047_5.1.1.0_x64__d7rcq4vxghz0r\Microsoft.Extensions.Logging.dll @@ -31,12 +36,6 @@ ..\..\..\..\..\Program Files\WindowsApps\96ba052f-0948-44d8-86c4-a0212e4ae047_5.1.1.0_x64__d7rcq4vxghz0r\Newtonsoft.Json.dll - - ..\..\..\..\..\Program Files\WindowsApps\96ba052f-0948-44d8-86c4-a0212e4ae047_5.1.1.0_x64__d7rcq4vxghz0r\VRCFaceTracking.dll - - - ..\..\..\..\..\Program Files\WindowsApps\96ba052f-0948-44d8-86c4-a0212e4ae047_5.1.1.0_x64__d7rcq4vxghz0r\VRCFaceTracking.Core.dll -