added hand-tracking blaze-palm predictor

Assets/ARPlayground/DemoScenes/ARPlayground.unity

@@ -216,6 +216,52 @@ Transform:
   m_Father: {fileID: 0}
   m_RootOrder: 1
   m_LocalEulerAnglesHint: {x: 50, y: -30, z: 0}
+--- !u!1 &171043883
+GameObject:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  serializedVersion: 6
+  m_Component:
+  - component: {fileID: 171043884}
+  - component: {fileID: 171043886}
+  m_Layer: 0
+  m_Name: HandTracking
+  m_TagString: Untagged
+  m_Icon: {fileID: 0}
+  m_NavMeshLayer: 0
+  m_StaticEditorFlags: 0
+  m_IsActive: 1
+--- !u!4 &171043884
+Transform:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 171043883}
+  m_LocalRotation: {x: -0, y: -0, z: -0, w: 1}
+  m_LocalPosition: {x: 0, y: 0, z: 0}
+  m_LocalScale: {x: 1, y: 1, z: 1}
+  m_Children: []
+  m_Father: {fileID: 1187653469}
+  m_RootOrder: 4
+  m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
+--- !u!114 &171043886
+MonoBehaviour:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 171043883}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: 965d1baf78f41e047aaf4b68a67399cf, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
+  model: {fileID: 5022602860645237092, guid: b76656a7f65d61e47b5897b3bf81b2e0, type: 3}
+  workerType: 32768
+  threshold: 0.75
 --- !u!1 &287728605
 GameObject:
   m_ObjectHideFlags: 0
@@ -609,7 +655,7 @@ MonoBehaviour:
   m_PrefabInstance: {fileID: 0}
   m_PrefabAsset: {fileID: 0}
   m_GameObject: {fileID: 474984181}
-  m_Enabled: 1
+  m_Enabled: 0
   m_EditorHideFlags: 0
   m_Script: {fileID: 11500000, guid: 26b650973440a9c4c89851b6aa481350, type: 3}
   m_Name: 
@@ -1164,6 +1210,7 @@ Transform:
   - {fileID: 474984182}
   - {fileID: 1321703897}
   - {fileID: 731385816}
+  - {fileID: 171043884}
   m_Father: {fileID: 0}
   m_RootOrder: 4
   m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
@@ -1420,7 +1467,7 @@ MonoBehaviour:
   m_PrefabInstance: {fileID: 0}
   m_PrefabAsset: {fileID: 0}
   m_GameObject: {fileID: 1443897844}
-  m_Enabled: 1
+  m_Enabled: 0
   m_EditorHideFlags: 0
   m_Script: {fileID: 11500000, guid: 04e78f0aed57ae440a5f46b6a630a207, type: 3}
   m_Name: 
@@ -1430,7 +1477,6 @@ MonoBehaviour:
   minDistance: 0.5
   maxDistance: 5
   depthImage: {fileID: 1050482430}
-  debugText: {fileID: 311577892}
 --- !u!1 &1479813792
 GameObject:
   m_ObjectHideFlags: 0

Assets/ARPlayground/PredictorPacks/HandTrackingBlaze/Resources/BlazePalmCommonShader.hlsl

@@ -0,0 +1,61 @@
+#ifndef _BLAZEPALMBARRACUDA_COMMON_H_
+#define _BLAZEPALMBARRACUDA_COMMON_H_
+
+#include "BlazePalmDetectionStruct.hlsl"
+
+// Maximum number of detections. This value must be matched with MaxDetection
+// in PalmDetector.cs.
+#define MAX_DETECTION 64
+
+// We can encode the geometric features of Detection into a float4x4 matrix.
+// This is handy for calculating weighted means of detections.
+
+float4x4 DetectionToMatrix(in PalmDetection d)
+{
+    return float4x4(d.center, d.extent,
+                    d.keyPoints[0], d.keyPoints[1],
+                    d.keyPoints[2], d.keyPoints[3],
+                    d.keyPoints[4], d.keyPoints[5]);
+}
+
+PalmDetection MatrixToDetection(float4x4 m, float score)
+{
+    PalmDetection d;
+    d.center = m._m00_m01;
+    d.extent = m._m02_m03;
+    d.keyPoints[0] = m._m10_m11;
+    d.keyPoints[1] = m._m12_m13;
+    d.keyPoints[2] = m._m20_m21;
+    d.keyPoints[3] = m._m22_m23;
+    d.keyPoints[4] = m._m30_m31;
+    d.keyPoints[5] = m._m32_m33;
+    d.score = score;
+    d.pad = 0;
+    return d;
+}
+
+// Common math functions
+
+float2 VFlip(float2 p)
+{
+    return float2(p.x, 1 - p.y);
+}
+
+float Sigmoid(float x)
+{
+    return 1 / (1 + exp(-x));
+}
+
+float CalculateIOU(in PalmDetection d1, in PalmDetection d2)
+{
+    float area0 = d1.extent.x * d1.extent.y;
+    float area1 = d2.extent.x * d2.extent.y;
+
+    float2 p0 = max(d1.center - d1.extent / 2, d2.center - d2.extent / 2);
+    float2 p1 = min(d1.center + d1.extent / 2, d2.center + d2.extent / 2);
+    float areaInner = max(0, p1.x - p0.x) * max(0, p1.y - p0.y);
+
+    return areaInner / (area0 + area1 - areaInner);
+}
+
+#endif

Assets/ARPlayground/PredictorPacks/HandTrackingBlaze/Resources/BlazePalmDetectionStruct.hlsl

@@ -0,0 +1,15 @@
+#ifndef _BLAZEPALMBARRACUDA_STRUCT_H_
+#define _BLAZEPALMBARRACUDA_STRUCT_H_
+
+// Detection structure: The layout of this structure must be matched with the
+// one defined in Detection.cs
+struct PalmDetection
+{
+    float2 center;
+    float2 extent;
+    float2 keyPoints[6];
+    float score;
+    float3 pad;
+};
+
+#endif

Assets/ARPlayground/PredictorPacks/HandTrackingBlaze/Resources/BlazePalmPostprocess1Kernel.hlsl

@@ -0,0 +1,47 @@
+//
+// Function template for Postprocess1 (bounding box aggregation)
+//
+[numthreads(CELLS_IN_ROW, CELLS_IN_ROW, 1)]
+void KERNEL_NAME(uint2 id : SV_DispatchThreadID)
+{
+    // Scale factor based on the input image size
+    float scale = 1 / _ImageSize;
+
+    // Corresponding row number in the input texture
+    uint row0 = (id.y * CELLS_IN_ROW + id.x) * ANCHOR_COUNT + INDEX_OFFSET;
+
+    // Anchor point coordinates
+    float2 anchor = (CELLS_IN_ROW - 0.5 - id) / CELLS_IN_ROW;
+
+    for (uint ai = 0; ai < ANCHOR_COUNT; ai++)
+    {
+        PalmDetection d;
+        d.pad = 0;
+
+        // Row number of this anchor
+        uint row = row0 + ai;
+
+        // Confidence score
+        d.score = Sigmoid(_Scores[uint2(0, row)]);
+
+        // Bounding box
+        float x = _Boxes[uint2(0, row)];
+        float y = _Boxes[uint2(1, row)];
+        float w = _Boxes[uint2(2, row)];
+        float h = _Boxes[uint2(3, row)];
+
+        d.center = VFlip(anchor + float2(x, y) * scale);
+        d.extent = float2(w, h) * scale;
+
+        // Key points
+        [unroll] for (uint ki = 0; ki < 6; ki++)
+        {
+            float kx = _Boxes[uint2(4 + 2 * ki + 0, row)];
+            float ky = _Boxes[uint2(4 + 2 * ki + 1, row)];
+            d.keyPoints[ki] = VFlip(anchor + float2(kx, ky) * scale);
+        }
+
+        // Thresholding
+        if (d.score > _Threshold) _Output.Append(d);
+    }
+}

Assets/ARPlayground/PredictorPacks/HandTrackingBlaze/Resources/BlazePalmPostprocess1Shader.compute

@@ -0,0 +1,36 @@
+#pragma kernel Postprocess1Layer1
+#pragma kernel Postprocess1Layer2
+
+// 
+// 1st postprocessor (bounding box aggregation)
+//
+
+#include "BlazePalmCommonShader.hlsl"
+
+// Input uniforms
+Texture2D<float> _Scores;
+Texture2D<float> _Boxes;
+float _ImageSize;
+float _Threshold;
+
+// Output uniforms
+AppendStructuredBuffer<PalmDetection> _Output;
+
+// First layer: 8-pixel stride, two anchors
+#define CELLS_IN_ROW 16
+#define ANCHOR_COUNT 2
+#define INDEX_OFFSET 384
+#define KERNEL_NAME Postprocess1Layer1
+#include "BlazePalmPostprocess1Kernel.hlsl"
+
+#undef CELLS_IN_ROW
+#undef ANCHOR_COUNT
+#undef KERNEL_NAME
+#undef INDEX_OFFSET
+
+// Second layer: 16-pixel stride, six anchors
+#define CELLS_IN_ROW 8
+#define ANCHOR_COUNT 6
+#define INDEX_OFFSET 0
+#define KERNEL_NAME Postprocess1Layer2
+#include "BlazePalmPostprocess1Kernel.hlsl"

Assets/ARPlayground/PredictorPacks/HandTrackingBlaze/Resources/BlazePalmPostprocess2Shader.compute

@@ -0,0 +1,61 @@
+#pragma kernel Postprocess2
+
+//
+// 2nd postprocessor (overlap removal)
+//
+
+#include "BlazePalmCommonShader.hlsl"
+
+#define IOU_THRESHOLD 0.3
+
+// Input uniforms
+ConsumeStructuredBuffer<PalmDetection> _Input;
+ByteAddressBuffer _Count;
+
+// Output uniforms
+AppendStructuredBuffer<PalmDetection> _Output;
+
+// Detection entry cache
+groupshared PalmDetection _entry[MAX_DETECTION];
+
+[numthreads(1, 1, 1)]
+void Postprocess2(uint3 id : SV_DispatchThreadID)
+{
+    // Detection entry cache initialization
+    uint entry_count = _Count.Load(0);
+    if (entry_count == 0) return;
+
+    for (uint i = 0; i < entry_count; i++)
+        _entry[i] = _Input.Consume();
+
+    // Try all the permutations.
+    for (i = 0; i < entry_count - 1; i++)
+    {
+        if (_entry[i].score == 0) continue;
+
+        // Variables for accumulation
+        float max_score = _entry[i].score;
+        float4x4 g_acc = DetectionToMatrix(_entry[i]) * max_score;
+        float s_acc = max_score;
+
+        for (uint j = i + 1; j < entry_count; j++)
+        {
+            if (_entry[j].score == 0) continue;
+
+            // Overlap test
+            if (CalculateIOU(_entry[i], _entry[j]) < IOU_THRESHOLD) continue;
+
+            // Accumulation
+            float score = _entry[j].score;
+            g_acc += DetectionToMatrix(_entry[j]) * score;
+            s_acc += score;
+            max_score = max(max_score, score);
+
+            // Remove the entry.
+            _entry[j].score = 0;
+        }
+
+        // Output the blended detection.
+        _Output.Append(MatrixToDetection(g_acc / s_acc, max_score));
+    }
+}