Upgrade to NatML 1.1.3

natmlx · Mar 3, 2023 · a80fc20 · a80fc20
1 parent c57949a
commit a80fc20
Show file tree

Hide file tree

Showing 19 changed files with 521 additions and 388 deletions.
diff --git a/Assembly-CSharp.csproj b/Assembly-CSharp.csproj
diff --git a/Assets/BlazePalmSample.cs b/Assets/BlazePalmSample.cs
@@ -1,58 +1,44 @@
 /* 
 *   BlazePalm
-*   Copyright (c) 2022 NatML Inc. All Rights Reserved.
+*   Copyright © 2023 NatML Inc. All Rights Reserved.
 */
 
 namespace NatML.Examples {
 
     using UnityEngine;
-    using NatML.Devices;
-    using NatML.Devices.Outputs;
+    using NatML.VideoKit;
     using NatML.Vision;
     using NatML.Visualizers;
 
-    [MLModelDataEmbed("@natml/blazepalm-detector"), MLModelDataEmbed("@natml/blazepalm-landmark")]
     public sealed class BlazePalmSample : MonoBehaviour {
-
+
+        [Header(@"Camera")]
+        public VideoKitCameraManager cameraManager;
+
         [Header(@"UI")]
         public BlazePalmVisualizer visualizer;
 
-        private CameraDevice cameraDevice;
-        private TextureOutput previewTextureOutput;
-
         private BlazePalmPipeline pipeline;
 
-        async void Start () {
-            // Request camera permissions
-            var permissionStatus = await MediaDeviceQuery.RequestPermissions<CameraDevice>();
-            if (permissionStatus != PermissionStatus.Authorized) {
-                Debug.Log(@"User did not grant camera permissions");
-                return;
-            }
-            // Get a camera device
-            var query = new MediaDeviceQuery(MediaDeviceCriteria.CameraDevice);
-            cameraDevice = query.current as CameraDevice;
-            // Start the camera preview
-            previewTextureOutput = new TextureOutput();
-            cameraDevice.StartRunning(previewTextureOutput);
-            // Display the preview
-            var previewTexture = await previewTextureOutput;
-            visualizer.image = previewTexture;
+        private async void Start () {
             // Create the BlazePalm predictor
-            var detectorModelData = await MLModelData.FromHub("@natml/blazepalm-detector");
-            var predictorModelData = await MLModelData.FromHub("@natml/blazepalm-landmark");
-            pipeline = new BlazePalmPipeline(detectorModelData, predictorModelData);
+            pipeline = await BlazePalmPipeline.Create();
+            // Listen for camera frames
+            cameraManager.OnCameraFrame.AddListener(OnCameraFrame);
         }
 
-        void Update () {
-            // Check that the predictor has been created
-            if (pipeline == null)
-                return;
+        private void OnCameraFrame (CameraFrame frame) {
             // Predict
-            var hands = pipeline.Predict(previewTextureOutput.texture);
+            var hands = pipeline.Predict(frame);
+            // Visualize
             visualizer.Render(hands);
         }
 
-        void OnDisable () => pipeline?.Dispose();
+        private void OnDisable () {
+            // Stop listening for camera frames
+            cameraManager.OnCameraFrame.RemoveListener(OnCameraFrame);
+            // Dispose the BlazePalm pipeline
+            pipeline?.Dispose();
+        }
     }
 }
diff --git a/Assets/BlazePalmSample.unity b/Assets/BlazePalmSample.unity
@@ -38,7 +38,7 @@ RenderSettings:
   m_ReflectionIntensity: 1
   m_CustomReflection: {fileID: 0}
   m_Sun: {fileID: 705507994}
-  m_IndirectSpecularColor: {r: 0.44657838, g: 0.49641234, b: 0.57481676, a: 1}
+  m_IndirectSpecularColor: {r: 0.4440765, g: 0.49331635, b: 0.57239074, a: 1}
   m_UseRadianceAmbientProbe: 0
 --- !u!157 &3
 LightmapSettings:
@@ -135,6 +135,8 @@ GameObject:
   - component: {fileID: 231924671}
   - component: {fileID: 231924670}
   - component: {fileID: 231924673}
+  - component: {fileID: 231924675}
+  - component: {fileID: 231924674}
   - component: {fileID: 231924672}
   m_Layer: 5
   m_Name: Preview
@@ -227,6 +229,39 @@ MonoBehaviour:
   m_EditorClassIdentifier: 
   m_AspectMode: 4
   m_AspectRatio: 1
+--- !u!114 &231924674
+MonoBehaviour:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 231924668}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: 145a5a333d6e14379ae91956c0b3373e, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
+  cameraManager: {fileID: 976522267}
+  viewMode: 0
+  focusMode: 0
+  exposureMode: 0
+  zoomMode: 0
+  OnPresent:
+    m_PersistentCalls:
+      m_Calls: []
+--- !u!114 &231924675
+MonoBehaviour:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 231924668}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: d0b148fe25e99eb48b9724523833bab1, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
+  m_Delegates: []
 --- !u!1 &284054438
 GameObject:
   m_ObjectHideFlags: 0
@@ -439,6 +474,7 @@ MonoBehaviour:
   m_Script: {fileID: 11500000, guid: 4f231c4fb786f3946a6b90b886c48677, type: 3}
   m_Name: 
   m_EditorClassIdentifier: 
+  m_SendPointerHoverToParent: 1
   m_HorizontalAxis: Horizontal
   m_VerticalAxis: Vertical
   m_SubmitButton: Submit
@@ -663,6 +699,7 @@ GameObject:
   serializedVersion: 6
   m_Component:
   - component: {fileID: 976522266}
+  - component: {fileID: 976522267}
   - component: {fileID: 976522265}
   m_Layer: 0
   m_Name: BlazePalmSample
@@ -683,6 +720,7 @@ MonoBehaviour:
   m_Script: {fileID: 11500000, guid: 1f9f4ff1037ce40c3b511a71b9ad6f69, type: 3}
   m_Name: 
   m_EditorClassIdentifier: 
+  cameraManager: {fileID: 976522267}
   visualizer: {fileID: 231924672}
 --- !u!4 &976522266
 Transform:
@@ -699,6 +737,28 @@ Transform:
   m_Father: {fileID: 0}
   m_RootOrder: 4
   m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
+--- !u!114 &976522267
+MonoBehaviour:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 976522264}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: 98622613d505143a9a06b4b11b0a82cf, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
+  capabilities: 2
+  playOnAwake: 1
+  _facing: 0
+  resolution: 3
+  frameRate: 30
+  focusMode: 0
+  exposureMode: 0
+  OnCameraFrame:
+    m_PersistentCalls:
+      m_Calls: []
 --- !u!1 &1380056287
 GameObject:
   m_ObjectHideFlags: 0

diff --git a/Assets/BlazePalmVisualizer.cs b/Assets/BlazePalmVisualizer.cs
@@ -1,6 +1,6 @@
 /* 
 *   BlazePalm
-*   Copyright (c) 2022 NatML Inc. All Rights Reserved.
+*   Copyright © 2023 NatML Inc. All Rights Reserved.
 */
 
 namespace NatML.Visualizers {
@@ -10,11 +10,12 @@ namespace NatML.Visualizers {
     using UnityEngine;
     using UnityEngine.UI;
     using UnityEngine.UI.Extensions;
-    using Vision;
+    using NatML.VideoKit.UI;
+    using NatML.Vision;
 
     /// <summary>
     /// </summary>
-    [RequireComponent(typeof(RawImage), typeof(AspectRatioFitter))]
+    [RequireComponent(typeof(VideoKitCameraView))]
     public sealed class BlazePalmVisualizer : MonoBehaviour {
 
         #region --Inspector--
@@ -24,16 +25,6 @@ public sealed class BlazePalmVisualizer : MonoBehaviour {
 
 
         #region --Client API--
-        /// <summary>
-        /// </summary>
-        public Texture2D image {
-            get => rawImage.texture as Texture2D;
-            set {
-                rawImage.texture = value;
-                aspectFitter.aspectRatio = (float)value.width / value.height;
-            }
-        }
-
         /// <summary>
         /// </summary>
         /// <param name="hands"></param>

diff --git a/Packages/ai.natml.vision.blazepalm/Changelog.md b/Packages/ai.natml.vision.blazepalm/Changelog.md
@@ -1,2 +1,10 @@
+## 1.0.1
++ Added `BlazePalmPipeline.Create` static method for creating the pipeline.
++ Added `BlazePalmPredictor.Create` static method for creating the predictor.
++ Improved memory behaviour of `BlazePalmPipeline` by removing new ROI image allocations.
++ Removed `BlazePalmPipeline` public constructor. Use the `BlazePalmPipeline.Create` method instead.
++ Removed `BlazePalmPredictor` public constructor. Use the `BlazePalmPredictor.Create` method instead.
++ Upgraded to NatML 1.1.3.
+
 ## 1.0.0
 + First release.
diff --git a/Packages/ai.natml.vision.blazepalm/README.md b/Packages/ai.natml.vision.blazepalm/README.md
@@ -13,19 +13,16 @@ Add the following items to your Unity project's `Packages/manifest.json`:
     }
   ],
   "dependencies": {
-    "ai.natml.vision.blazepalm": "1.0.0"
+    "ai.natml.vision.blazepalm": "1.0.1"
   }
 }
 ```
 
 ## Predicting Hands in an Image
 First, create the BlazePalm pipeline:
 ```csharp
-// Fetch the model data
-var detectorModelData = await MLModelData.FromHub("@natml/blazepalm-detector");
-var predictorModelData = await MLModelData.FromHub("@natml/blazepalm-landmark");
 // Create the BlazePalm pipeline
-var pipeline = new BlazePalmPipeline(detectorModelData, predictorModelData);
+var pipeline = await BlazePalmPipeline.Create();
 ```
 
 Then detect hands in the image:
@@ -41,10 +38,9 @@ ___
 - Unity 2021.2+
 
 ## Quick Tips
-- Join the [NatML community on Discord](https://hub.natml.ai/community).
+- Join the [NatML community on Discord](https://natml.ai/community).
 - Discover more ML models on [NatML Hub](https://hub.natml.ai).
 - See the [NatML documentation](https://docs.natml.ai/unity).
-- Discuss [NatML on Unity Forums](https://forum.unity.com/threads/open-beta-natml-machine-learning-runtime.1109339/).
 - Contact us at [hi@natml.ai](mailto:hi@natml.ai).
 
 Thank you very much!