Skip to content

Commit 969b6c2

Browse files
hai-vrhai-vr
committed
Work towards adding eye tracking input:
- Compute the intersection between the eye gaze and the overlay, which is used in place of the overlay hand laser input. - In our ImGui window, add support for transparent backgrounds for the eye tracking overlay. - Add ImGuiWindowFlags.NoBackground flag to the window when applicable. - Invoke ClearColorTarget with a transparent alpha color. - Add eye tracking tab with a custom eye tracking menu tab. - Buttons are not yet functional. The code between the Shortcuts tab and the Eye Tracking Menu tab need to be mutualized. - Add radial and staggered layouts. - Add frame counter code for debugging update rate.
1 parent 89a3417 commit 969b6c2

14 files changed

+495
-112
lines changed

‎h-view/src/OVR/HVOpenVRThread.cs

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,9 @@ public class HVOpenVRThread
1414
private const int TotalWindowHeight = 510;
1515
public const string VrManifestAppKey = "Hai.HView";
1616

17+
// TODO: Add a runtime switch
18+
private const bool EnableEyeTrackingProvider = false;
19+
1720
private readonly HVRoutine _routine;
1821
private readonly HVOpenVRManagement _ovr;
1922
private readonly bool _registerAppManifest;
@@ -82,14 +85,14 @@ public void Run()
8285
innerWindow.SetupUi(true);
8386

8487
var windowRatio = width / (height * 1f);
85-
var dashboard = new HVOverlayInstance(innerWindow, "main", true, windowRatio);
88+
var dashboard = new HVImGuiOverlay(innerWindow, "main", true, windowRatio);
8689
dashboard.Start();
8790

8891
var overlayables = new List<IOverlayable>();
8992
overlayables.Add(dashboard);
9093

9194
HEyeTrackingOverlay eyeTrackingOptional = null;
92-
if (false)
95+
if (EnableEyeTrackingProvider)
9396
{
9497
eyeTrackingOptional = new HEyeTrackingOverlay(_routine, dashboard);
9598
eyeTrackingOptional.Start();
@@ -103,6 +106,7 @@ public void Run()
103106
handOverlay.Start();
104107
handOverlay.MoveToInitialPosition(ovr.PoseData());
105108
eyeTrackingOptional?.SetHandOverlay(handOverlay);
109+
innerWindow.SetIsHandOverlay(true);
106110

107111
overlayables.Add(handOverlay);
108112
});
@@ -116,6 +120,7 @@ public void Run()
116120
handOverlay.Teardown();
117121
eyeTrackingOptional?.SetHandOverlay(null);
118122
overlayables.Remove(handOverlay);
123+
innerWindow.SetIsHandOverlay(false);
119124

120125
handOverlay = null;
121126
});

‎h-view/src/OVR/OpenVRUtils.cs

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
1-
using System.Runtime.InteropServices;
1+
using System.Numerics;
2+
using System.Runtime.InteropServices;
23
using Valve.VR;
34

45
namespace Hai.HView.OVR;
@@ -13,4 +14,29 @@ public static InputDigitalActionData_t GetDigitalInput(ulong action)
1314
OpenVR.Input.GetDigitalActionData(action, ref data, SizeOfDigitalActionData, 0);
1415
return data;
1516
}
17+
18+
/// Invokes ComputeOverlayIntersection but restricts the accepted U and V components to [0, 1], returning false when not in that range.
19+
public static bool ComputeOverlayIntersectionStrictUVs(ulong ulOverlayHandle, VROverlayIntersectionParams_t pParams, out Vector2 uv)
20+
{
21+
VROverlayIntersectionResults_t results = default;
22+
var success = OpenVR.Overlay.ComputeOverlayIntersection(ulOverlayHandle, ref pParams, ref results);
23+
if (success)
24+
{
25+
var x01 = results.vUVs.v0;
26+
var y01 = results.vUVs.v1;
27+
if (x01 is >= 0f and <= 1f && y01 is >= 0f and <= 1f)
28+
{
29+
uv = new Vector2(x01, y01);
30+
return true;
31+
}
32+
}
33+
34+
uv = Vector2.Zero;
35+
return false;
36+
}
37+
38+
public static bool IsValidDeviceIndex(uint deviceIndex)
39+
{
40+
return deviceIndex != OpenVR.k_unTrackedDeviceIndexInvalid;
41+
}
1642
}

‎h-view/src/Overlay/HEyeTrackingOverlay.cs

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ namespace Hai.HView.Overlay;
88
public class HEyeTrackingOverlay : IOverlayable
99
{
1010
private readonly HVRoutine _routine;
11-
private readonly HVOverlayInstance _dashboard;
11+
private readonly HVImGuiOverlay _dashboard;
1212
private HHandOverlay _handOverlayNullable;
1313
private const string Name = "eyetracking";
1414

@@ -22,15 +22,15 @@ public class HEyeTrackingOverlay : IOverlayable
2222
public Vector3 EyePos => _eyePos;
2323
public Quaternion EyeGaze => _eyeGaze;
2424

25-
public HEyeTrackingOverlay(HVRoutine routine, HVOverlayInstance dashboard)
25+
public HEyeTrackingOverlay(HVRoutine routine, HVImGuiOverlay dashboard)
2626
{
2727
_routine = routine;
2828
_dashboard = dashboard;
2929
}
3030

3131
public void Start()
3232
{
33-
OpenVR.Overlay.CreateOverlay($"{HVOverlayInstance.OverlayKey}-{Name}", HVApp.AppTitle, ref _handle);
33+
OpenVR.Overlay.CreateOverlay($"{HVImGuiOverlay.OverlayKey}-{Name}", HVApp.AppTitle, ref _handle);
3434
OpenVR.Overlay.SetOverlayFromFile(_handle, Path.GetFullPath("EyeTrackingCursor.png"));
3535
OpenVR.Overlay.SetOverlayAlpha(_handle, 1f);
3636
OpenVR.Overlay.SetOverlayColor(_handle, 1f, 1f, 1f);
@@ -48,8 +48,8 @@ public void ProvidePoseData(HVPoseData poseData)
4848
var eyeRot = HVGeofunctions.QuaternionFromAngles(new Vector3(yy, xx, 0), HVRotationMulOrder.YZX);
4949

5050
var absToHead = HVOvrGeofunctions.OvrToOvrnum(_poseData.Poses[0].mDeviceToAbsoluteTracking);
51-
var headToEyeTrackingFocus = HVOvrGeofunctions.OvrToOvrnum(HVOvrGeofunctions.OvrTR(new Vector3(0, 0, 0), eyeRot));
52-
var move = HVOvrGeofunctions.OvrToOvrnum(HVOvrGeofunctions.OvrTR(new Vector3(0, 0, -1), Quaternion.Identity));
51+
var headToEyeTrackingFocus = HVGeofunctions.TR(new Vector3(0, 0, 0), eyeRot);
52+
var move = HVGeofunctions.TR(new Vector3(0, 0, -1), Quaternion.Identity);
5353

5454
_overlayPlace = HVOvrGeofunctions.OvrnumToOvr(absToHead * headToEyeTrackingFocus * move);
5555
HVGeofunctions.ToPosRotV3(absToHead * headToEyeTrackingFocus, out _eyePos, out _eyeGaze);

‎h-view/src/Overlay/HHandOverlay.cs

Lines changed: 19 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -2,22 +2,23 @@
22
using System.Numerics;
33
using Hai.HView.Core;
44
using Hai.HView.Gui;
5+
using Hai.HView.OVR;
56
using Valve.VR;
67

78
namespace Hai.HView.Overlay;
89

910
public class HHandOverlay : IOverlayable
1011
{
1112
private readonly HVRoutine _routine;
12-
private readonly HVOverlayInstance _overlay;
13+
private readonly HVImGuiOverlay _overlay;
1314
private readonly Stopwatch _stopwatch;
1415
private readonly bool _useLeftHand;
1516

1617
public HHandOverlay(HVInnerWindow innerWindow, float windowRatio, HVRoutine routine, bool useLeftHand)
1718
{
1819
_routine = routine;
1920
_useLeftHand = useLeftHand;
20-
_overlay = new HVOverlayInstance(innerWindow, "costumes", false, windowRatio);
21+
_overlay = new HVImGuiOverlay(innerWindow, "costumes", false, windowRatio);
2122
_stopwatch = new Stopwatch();
2223
}
2324

@@ -30,12 +31,12 @@ public void Start()
3031
public void MoveToInitialPosition(HVPoseData poseData)
3132
{
3233
var deviceIndex = GetHandedDeviceIndex(poseData);
33-
if (!HVOverlayMovement.IsValidDeviceIndex(deviceIndex)) return;
34+
if (!OpenVRUtils.IsValidDeviceIndex(deviceIndex)) return;
3435

3536
var pos = new Vector3(0f, -0.01f, -0.2f);
3637
var angles = new Vector3(65, 0, 0);
3738
var rot = HVGeofunctions.QuaternionFromAngles(angles, HVRotationMulOrder.YZX);
38-
var handToOverlayPlace = HVOvrGeofunctions.OvrToOvrnum(HVOvrGeofunctions.OvrTRS(pos, rot, Vector3.One));
39+
var handToOverlayPlace = HVGeofunctions.TR(pos, rot);
3940

4041
var absToHandPlace = HVOvrGeofunctions.OvrToOvrnum(poseData.Poses[deviceIndex].mDeviceToAbsoluteTracking);
4142
var absToOverlayPlace = HVOvrGeofunctions.OvrnumToOvr(absToHandPlace * handToOverlayPlace);
@@ -59,9 +60,10 @@ public void ProvidePoseData(HVPoseData poseData)
5960
private void HandleIntersection(HVPoseData poseData)
6061
{
6162
var whichHand = GetHandedDeviceIndex(poseData);
62-
if (HVOverlayMovement.IsValidDeviceIndex(whichHand))
63+
if (OpenVRUtils.IsValidDeviceIndex(whichHand))
6364
{
64-
// var isIntersecting = IsIntersecting(poseData, whichHand);
65+
// var isIntersecting = IsHandLaserIntersecting(poseData, whichHand);
66+
// OpenVR.Overlay.SetOverlayFlag(_overlay.GetOverlayHandle(), VROverlayFlags.MakeOverlaysInteractiveIfVisible, isIntersecting);
6567
var isIntersecting = OpenVR.Overlay.IsHoverTargetOverlay(_overlay.GetOverlayHandle());
6668
if (isIntersecting)
6769
{
@@ -75,28 +77,22 @@ private uint GetHandedDeviceIndex(HVPoseData poseData)
7577
return _useLeftHand ? poseData.LeftHandDeviceIndex : poseData.RightHandDeviceIndex;
7678
}
7779

78-
private bool IsIntersecting(HVPoseData poseData, uint whichHand)
80+
private bool IsHandLaserIntersecting(HVPoseData poseData, uint whichHand)
7981
{
80-
// FIXME: None of this works, unsure why. Intersection is offset, or has wrong angle or something. It feels squished vertically.
82+
// FIXME: ~~None of this works, unsure why. Intersection is offset, or has wrong angle or something. It feels squished vertically.~~
83+
// FIXME: After working on another part of the program, check out HEyeTrackingOverlay.cs,
84+
// as it might be that the quaternion of the hand pose (or the matrix itself) needs to be inverted before being passed to the intersection params.
85+
// FIXME: It still doesn't seem to work properly, probably the laser angle or origin point needs to be sourced from one of the controller poses.
8186

8287
var handPose = poseData.Poses[whichHand].mDeviceToAbsoluteTracking;
83-
var move = new Vector3(0, -0.2f, 0);
84-
var absToOverHand = HVOvrGeofunctions.OvrToOvrnum(HVOvrGeofunctions.OvrTranslate(move)) * HVOvrGeofunctions.OvrToOvrnum(handPose);
85-
var pos = HVOvrGeofunctions.PosOvr(HVOvrGeofunctions.OvrnumToOvr(absToOverHand));
88+
HVGeofunctions.ToPosRotV3(HVOvrGeofunctions.OvrToOvrnum(handPose), out var pos, out var rot);
8689
var intersection = new VROverlayIntersectionParams_t
8790
{
8891
eOrigin = OpenVR.Compositor.GetTrackingSpace(),
89-
vSource = pos,
90-
vDirection = HVOvrGeofunctions.ForwardOvr(handPose)
92+
vSource = HVOvrGeofunctions.Vec(pos),
93+
vDirection = HVOvrGeofunctions.Vec(Vector3.Transform(new Vector3(0, 0, -1), Quaternion.Inverse(rot)))
9194
};
92-
93-
VROverlayIntersectionResults_t results = default;
94-
var isIntersecting = OpenVR.Overlay.ComputeOverlayIntersection(_overlay.GetOverlayHandle(), ref intersection, ref results);
95-
96-
// HACK: somehow ComputeOverlayIntersection doesn't return false when it's not intersecting
97-
if (results.vPoint is { v0: 0, v1: 0, v2: 0 }) isIntersecting = false;
98-
Console.WriteLine($"IsIntersecting? {isIntersecting} {results.vPoint.v0} {results.vPoint.v1} {results.vPoint.v2}");
99-
return isIntersecting;
95+
return OpenVRUtils.ComputeOverlayIntersectionStrictUVs(_overlay.GetOverlayHandle(), intersection, out _);
10096
}
10197

10298
public void ProcessThatOverlay(Stopwatch stopwatch)
@@ -107,7 +103,8 @@ public void ProcessThatOverlay(Stopwatch stopwatch)
107103
{
108104
_routine.EjectUserFromCostumeMenu();
109105

110-
// We eject and also hide, because we want this to work even when VRC is not running.
106+
// We eject and also hide, because we want this to work even when VRC is not running,
107+
// and also when VRC is going into a loading screen.
111108
_routine.HideCostumes();
112109
}
113110
}

‎h-view/src/Overlay/HVGeofunctions.cs

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,32 @@ public static class HVGeofunctions
66
{
77
// Ovrnum: System.Numerics matrix functions using OpenVR coordinate system
88
// Unity: System.Numerics matrix functions using Unity coordinate system
9+
10+
public static Matrix4x4 TR(Vector3 translation, Quaternion rotation)
11+
{
12+
var numRot = Matrix4x4.CreateFromQuaternion(rotation);
13+
14+
return new Matrix4x4
15+
(
16+
numRot.M11, numRot.M12, numRot.M13, translation.X,
17+
numRot.M21, numRot.M22, numRot.M23, translation.Y,
18+
numRot.M31, numRot.M32, numRot.M33, translation.Z,
19+
0, 0, 0, 1
20+
);
21+
}
22+
23+
public static Matrix4x4 TRS(Vector3 translation, Quaternion rotation, Vector3 scale)
24+
{
25+
var numRot = Matrix4x4.CreateFromQuaternion(rotation);
26+
27+
return new Matrix4x4
28+
(
29+
numRot.M11 * scale.X, numRot.M12 * scale.X, numRot.M13 * scale.X, translation.X,
30+
numRot.M21 * scale.Y, numRot.M22 * scale.Y, numRot.M23 * scale.Y, translation.Y,
31+
numRot.M31 * scale.Z, numRot.M32 * scale.Z, numRot.M33 * scale.Z, translation.Z,
32+
0, 0, 0, 1
33+
);
34+
}
935

1036
public static Matrix4x4 OvrnumToUnity(Matrix4x4 generic)
1137
{

‎h-view/src/Overlay/HVOverlay.cs renamed to ‎h-view/src/Overlay/HVImGuiOverlay.cs

Lines changed: 24 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,13 @@
22
using System.Numerics;
33
using Hai.HView.Core;
44
using Hai.HView.Gui;
5+
using Hai.HView.OVR;
56
using Valve.VR;
67
using Veldrid;
78

89
namespace Hai.HView.Overlay;
910

10-
public class HVOverlayInstance : IOverlayable
11+
public class HVImGuiOverlay : IOverlayable
1112
{
1213
private const string DashboardKey = "hview-dashboard";
1314
public const string OverlayKey = "hview-overlay";
@@ -31,7 +32,7 @@ public class HVOverlayInstance : IOverlayable
3132
private Vector3 _eyePos;
3233
private Quaternion _eyeGaze;
3334

34-
public HVOverlayInstance(HVInnerWindow innerWindow, string name, bool isDashboard, float ratio)
35+
public HVImGuiOverlay(HVInnerWindow innerWindow, string name, bool isDashboard, float ratio)
3536
{
3637
_innerWindow = innerWindow;
3738
_name = name;
@@ -207,32 +208,40 @@ public void ProvideEyeTracking(Vector3 eyePos, Quaternion eyeGaze)
207208
_usingEyeTracking = true;
208209
_eyePos = eyePos;
209210
_eyeGaze = eyeGaze;
211+
_innerWindow.SetEyeTracking(_usingEyeTracking);
210212
}
211213

212214
private void ProcessEyeTracking()
213215
{
214216
if (!_usingEyeTracking) return;
215217

216-
// (??????) Why do I have to inverse the eye gaze quaternion? (??????)
217-
var quaternion = Quaternion.Inverse(_eyeGaze);
218+
/*
219+
Conversation between 2024-09-15 and 2024-09-18:
220+
Haï~:
221+
When I tried to use OpenVR.Overlay.ComputeOverlayIntersection, which takes a vSource and a vDirection vectors,
222+
it seemed like the vector in vSource is in world space, and the vector in vDirection is in overlay space,
223+
rather than my expectation that vSource and vDirection were both in world space.
224+
Going backwards, the only way I could make this function work was to extract the position out of mDeviceToAbsoluteTracking matrix,
225+
but the inverse rotation out of the mDeviceToAbsoluteTracking matrix
226+
that doesn't make sense to me, so I'm wondering if there's something fundamental about matrices or some other concept that I'm not getting
227+
228+
cnlohr (reply to Haï~):
229+
I don't see any rhyme or reason. It's possible it could be because of historical reasons that no longer apply, but now it is how it is.
230+
231+
*/
232+
// We invert the gaze quaternion because apparently the vDirection part of the raycast is in overlay space (see conversation above)
233+
var openVrEyeGazeRaycastQuat = Quaternion.Inverse(_eyeGaze);
218234

219-
var gazeDir = Vector3.Transform(new Vector3(0, 0, -1), quaternion);
220-
VROverlayIntersectionParams_t intersectionParams = new VROverlayIntersectionParams_t
235+
var gazeDir = Vector3.Transform(new Vector3(0, 0, -1), openVrEyeGazeRaycastQuat);
236+
var intersectionParams = new VROverlayIntersectionParams_t
221237
{
222238
eOrigin = OpenVR.Compositor.GetTrackingSpace(),
223239
vSource = HVOvrGeofunctions.Vec(_eyePos),
224240
vDirection = HVOvrGeofunctions.Vec(gazeDir)
225241
};
226-
VROverlayIntersectionResults_t results = default;
227-
var success = OpenVR.Overlay.ComputeOverlayIntersection(_handle, ref intersectionParams, ref results);
228-
if (success)
242+
if (OpenVRUtils.ComputeOverlayIntersectionStrictUVs(_handle, intersectionParams, out var uv))
229243
{
230-
var x01 = results.vUVs.v0;
231-
var y01 = results.vUVs.v1;
232-
if (x01 is > 0f and < 1f && y01 is > 0f and < 1f)
233-
{
234-
_inputSnapshot.MouseMove(new Vector2(x01, 1 - y01));
235-
}
244+
_inputSnapshot.MouseMove(new Vector2(uv.X, 1 - uv.Y));
236245
}
237246
}
238247
}

‎h-view/src/Overlay/HVOverlayMovement.cs

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
using System.Numerics;
2+
using Hai.HView.OVR;
23
using Valve.VR;
34

45
namespace Hai.HView.Overlay;
@@ -10,7 +11,7 @@ public void Evaluate(ulong hOverlay, HVPoseData poseData)
1011
// FIXME: Disable this code for now as this interferes with immovable non-dashboard overlays.
1112
return;
1213
var controllerIndex = poseData.RightHandDeviceIndex;
13-
if (IsValidDeviceIndex(controllerIndex))
14+
if (OpenVRUtils.IsValidDeviceIndex(controllerIndex))
1415
{
1516
// TODO: The following is just test values.
1617
var quaternion = HVGeofunctions.QuaternionFromAngles(new Vector3(35, -25, -9), HVRotationMulOrder.YZX);
@@ -22,11 +23,6 @@ public void Evaluate(ulong hOverlay, HVPoseData poseData)
2223
// FIXME: This breaks OVR Advanced Settings motion / playspace mover!
2324
OpenVR.Overlay.SetOverlayFlag(hOverlay, VROverlayFlags.MakeOverlaysInteractiveIfVisible, true);
2425
}
25-
26-
public static bool IsValidDeviceIndex(uint deviceIndex)
27-
{
28-
return deviceIndex != OpenVR.k_unTrackedDeviceIndexInvalid;
29-
}
3026
}
3127

3228
public class HVPoseData

‎h-view/src/Overlay/HVOvrGeofunctions.cs

Lines changed: 0 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -74,23 +74,6 @@ public static HmdMatrix34_t OvrnumToOvr(Matrix4x4 overnum)
7474
};
7575
}
7676

77-
public static HmdVector3_t PosOvr(HmdMatrix34_t ovr)
78-
{
79-
return new HmdVector3_t
80-
{
81-
v0 = ovr.m3, v1 = ovr.m7, v2 = ovr.m11
82-
};
83-
}
84-
85-
// TODO: Is this correct?
86-
public static HmdVector3_t ForwardOvr(HmdMatrix34_t ovr)
87-
{
88-
return new HmdVector3_t
89-
{
90-
v0 = -ovr.m2, v1 = -ovr.m6, v2 = -ovr.m10
91-
};
92-
}
93-
9477
public static HmdVector3_t Vec(Vector3 num)
9578
{
9679
return new HmdVector3_t

0 commit comments

Comments
 (0)