Skip to content

Commit

Permalink
Improve scrolling using eye tracking (#1570)
Browse files Browse the repository at this point in the history 
* Improve scrolling using eye tracking

This PR fixes a couple of issues with the current gesture to scroll
using eye tracking:
1. it allows users to scroll with both hands, initially it was only
tied to the right hand
2. it prevents scrolling by moving the eyes. The scroll gesture
should be done with the hand or the controller but not with the gaze.
The current code scrolls if user moves their eyes up and down while
pinching/clicking and holding.

To fix that we pass a new parameter to the Update() method in the
input source carrying the transform for the eye gaze. That transform
would be used to set the starting point of the scroll action if
there is a scroll or the pointer position otherwise.

Methods like Update(), EmulateControllerFromHands() or
HandleEyeTrackingScroll() are suffering from bad design
decisions and this PR does not improve the situation. In any case
that's a different topic and should be addressed in a separate PR.

Tested in:
* Pico 4E (5.11.1)
* Meta Quest Pro (v71)
* Magic Leap 2
  • Loading branch information
@svillar
svillar authored Oct 17, 2024
1 parent e2d7ade commit e72b8a8
Show file tree
Hide file tree
Showing 4 changed files with 28 additions and 17 deletions.
5 changes: 2 additions & 3 deletions app/src/openxr/cpp/OpenXRInput.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ XrResult OpenXRInput::Update(const XrFrameState& frameState, XrSpace baseSpace,
bool usingEyeTracking = pointerMode == DeviceDelegate::PointerMode::TRACKED_EYE && updateEyeGaze(frameState, head, delegate);

for (auto& input : mInputSources) {
input->Update(frameState, baseSpace, head, offsets, renderMode, pointerMode, usingEyeTracking, handTrackingEnabled, delegate);
input->Update(frameState, baseSpace, head, offsets, renderMode, pointerMode, usingEyeTracking, handTrackingEnabled, mEyeTrackingTransform, delegate);
}

// Update tracked keyboard
Expand Down Expand Up @@ -374,8 +374,7 @@ bool OpenXRInput::updateEyeGaze(XrFrameState frameState, const vrb::Matrix& head
vrb::Quaternion gazeOrientation(gazeLocation.pose.orientation.x, gazeLocation.pose.orientation.y, gazeLocation.pose.orientation.z, gazeLocation.pose.orientation.w);
float* filteredOrientation = mOneEuroFilterGazeOrientation->filter(frameState.predictedDisplayTime, gazeOrientation.Data());
gazeOrientation = {filteredOrientation[0], filteredOrientation[1], filteredOrientation[2], filteredOrientation[3]};
delegate.SetTransform(0, vrb::Matrix::Rotation(gazeOrientation).Translate(gazePosition));
delegate.SetImmersiveBeamTransform(0, vrb::Matrix::Identity());
mEyeTrackingTransform = vrb::Matrix::Rotation(gazeOrientation).Translate(gazePosition);

return true;
}
Expand Down
1 change: 1 addition & 0 deletions app/src/openxr/cpp/OpenXRInput.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ class OpenXRInput {
XrSpace mEyeGazeActionSpace {XR_NULL_HANDLE };
XrSpace mLocalReferenceSpace { XR_NULL_HANDLE};
std::unique_ptr<OneEuroFilterQuaternion> mOneEuroFilterGazeOrientation;
vrb::Matrix mEyeTrackingTransform;

public:
static OpenXRInputPtr Create(XrInstance, XrSession, XrSystemProperties, XrSpace localSpace,
Expand Down
30 changes: 20 additions & 10 deletions app/src/openxr/cpp/OpenXRInputSource.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -639,7 +639,7 @@ OpenXRInputSource::PopulateHandJointLocations(device::RenderMode renderMode, std
}
}

void OpenXRInputSource::EmulateControllerFromHand(device::RenderMode renderMode, XrTime predictedDisplayTime, const vrb::Matrix& head, const vrb::Matrix& handJointForAim, DeviceDelegate::PointerMode pointerMode, bool usingEyeTracking, ControllerDelegate& delegate)
void OpenXRInputSource::EmulateControllerFromHand(device::RenderMode renderMode, XrTime predictedDisplayTime, const vrb::Matrix& head, const vrb::Matrix& handJointForAim, DeviceDelegate::PointerMode pointerMode, bool usingEyeTracking, const vrb::Matrix& eyeTrackingTransform, ControllerDelegate& delegate)
{
assert(mHasHandJoints);

Expand Down Expand Up @@ -731,13 +731,15 @@ void OpenXRInputSource::EmulateControllerFromHand(device::RenderMode renderMode,
delegate.SetImmersiveBeamTransform(mIndex, pointerTransform);
} else {
assert(pointerMode == DeviceDelegate::PointerMode::TRACKED_EYE);
HandleEyeTrackingScroll(predictedDisplayTime, triggerButtonPressed, pointerTransform, delegate);
HandleEyeTrackingScroll(predictedDisplayTime, triggerButtonPressed, pointerTransform, eyeTrackingTransform, delegate);
if (!triggerButtonPressed)
delegate.SetTransform(mIndex, eyeTrackingTransform);
}

delegate.SetCapabilityFlags(mIndex, flags);
}

void OpenXRInputSource::Update(const XrFrameState& frameState, XrSpace localSpace, const vrb::Matrix &head, const vrb::Vector& offsets, device::RenderMode renderMode, DeviceDelegate::PointerMode pointerMode, bool usingEyeTracking, bool handTrackingEnabled, ControllerDelegate& delegate)
void OpenXRInputSource::Update(const XrFrameState& frameState, XrSpace localSpace, const vrb::Matrix &head, const vrb::Vector& offsets, device::RenderMode renderMode, DeviceDelegate::PointerMode pointerMode, bool usingEyeTracking, bool handTrackingEnabled, const vrb::Matrix& eyeTrackingTransform, ControllerDelegate& delegate)
{
if (mActiveMapping &&
((mHandeness == OpenXRHandFlags::Left && !mActiveMapping->leftControllerModel) ||
Expand Down Expand Up @@ -801,7 +803,7 @@ void OpenXRInputSource::Update(const XrFrameState& frameState, XrSpace localSpac
std::vector<float> jointRadii;
PopulateHandJointLocations(renderMode, jointTransforms, jointRadii);
if (!mIsHandInteractionSupported) {
EmulateControllerFromHand(renderMode, frameState.predictedDisplayTime, head, jointTransforms[HAND_JOINT_FOR_AIM], pointerMode, usingEyeTracking, delegate);
EmulateControllerFromHand(renderMode, frameState.predictedDisplayTime, head, jointTransforms[HAND_JOINT_FOR_AIM], pointerMode, usingEyeTracking, eyeTrackingTransform, delegate);
delegate.SetHandJointLocations(mIndex, std::move(jointTransforms), std::move(jointRadii));
return;
}
Expand Down Expand Up @@ -932,8 +934,11 @@ void OpenXRInputSource::Update(const XrFrameState& frameState, XrSpace localSpac

if (button.type == OpenXRButtonType::Trigger) {
delegate.SetSelectFactor(mIndex, state->value);
if (pointerMode == DeviceDelegate::PointerMode::TRACKED_EYE)
HandleEyeTrackingScroll(frameState.predictedDisplayTime, state->clicked, pointerTransform, delegate);
if (pointerMode == DeviceDelegate::PointerMode::TRACKED_EYE) {
HandleEyeTrackingScroll(frameState.predictedDisplayTime, state->clicked, pointerTransform, eyeTrackingTransform, delegate);
if (!state->clicked)
delegate.SetTransform(mIndex, eyeTrackingTransform);
}
}

// Select action
Expand Down Expand Up @@ -1069,16 +1074,21 @@ OpenXRInputSource::GetNextHandMeshBuffer() {
return mHandMeshMSFT.buffer;
}

void OpenXRInputSource::HandleEyeTrackingScroll(XrTime predictedDisplayTime, bool triggerClicked, const vrb::Matrix &pointerTransform, ControllerDelegate &controllerDelegate) {
void
OpenXRInputSource::HandleEyeTrackingScroll(XrTime predictedDisplayTime, bool triggerClicked, const vrb::Matrix &pointerTransform, const vrb::Matrix& eyeTrackingTransform, ControllerDelegate &controllerDelegate) {
if (!mTriggerWasClicked && triggerClicked) {
mControllerPositionOnGestureStart = pointerTransform.GetTranslation();
mEyeGazeTransformOnPinchStart = eyeTrackingTransform;
mEyeTrackingPinchStartTime = predictedDisplayTime;
} else if (mTriggerWasClicked && triggerClicked) {
// Throttle the start of the scroll gesture to avoid scrolling on pinch.
if (predictedDisplayTime - mEyeTrackingPinchStartTime > kEyeTrackingScrollThreshold) {
vrb::Vector currentControllerPosition = pointerTransform.GetTranslation();
auto delta = currentControllerPosition - mControllerPositionOnGestureStart;
controllerDelegate.TranslateTransform(mIndex, delta * 10);
auto delta = currentControllerPosition - mControllerPositionOnScrollStart;
controllerDelegate.SetTransform(mIndex, mEyeGazeTransformOnPinchStart.Translate(delta * 10));
} else {
// Keep updating the initial position while throttling to avoid a sudden jump when
// scrolling starts.
mControllerPositionOnScrollStart = pointerTransform.GetTranslation();
}
}
mTriggerWasClicked = triggerClicked;
Expand Down
9 changes: 5 additions & 4 deletions app/src/openxr/cpp/OpenXRInputSource.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ class OpenXRInputSource {
void UpdateHaptics(ControllerDelegate&);
bool GetHandTrackingInfo(XrTime predictedDisplayTime, XrSpace, const vrb::Matrix& head);
float GetDistanceBetweenJoints (XrHandJointEXT jointA, XrHandJointEXT jointB);
void EmulateControllerFromHand(device::RenderMode renderMode, XrTime predictedDisplayTime, const vrb::Matrix& head, const vrb::Matrix& handJointForAim, DeviceDelegate::PointerMode pointerMode, bool usingEyeTracking, ControllerDelegate& delegate);
void EmulateControllerFromHand(device::RenderMode renderMode, XrTime predictedDisplayTime, const vrb::Matrix& head, const vrb::Matrix& handJointForAim, DeviceDelegate::PointerMode pointerMode, bool usingEyeTracking, const vrb::Matrix& eyeTrackingTransform, ControllerDelegate& delegate);
void PopulateHandJointLocations(device::RenderMode, std::vector<vrb::Matrix>& jointTransforms, std::vector<float>& jointRadii);

XrInstance mInstance { XR_NULL_HANDLE };
Expand Down Expand Up @@ -85,7 +85,8 @@ class OpenXRInputSource {
bool mUsingHandInteractionProfile { false };
device::DeviceType mDeviceType { device::UnknownType };
bool mTriggerWasClicked { false };
vrb::Vector mControllerPositionOnGestureStart;
vrb::Vector mControllerPositionOnScrollStart;
vrb::Matrix mEyeGazeTransformOnPinchStart;
XrTime mEyeTrackingPinchStartTime { 0 };

struct HandMeshMSFT {
Expand All @@ -107,13 +108,13 @@ class OpenXRInputSource {

bool mIsHandInteractionSupported { false };

void HandleEyeTrackingScroll(XrTime predictedDisplayTime, bool triggerClicked, const vrb::Matrix& pointerTransform, ControllerDelegate &controllerDelegate);
void HandleEyeTrackingScroll(XrTime predictedDisplayTime, bool triggerClicked, const vrb::Matrix& pointerTransform, const vrb::Matrix& eyeTrackingTransform, ControllerDelegate &controllerDelegate);
public:
static OpenXRInputSourcePtr Create(XrInstance, XrSession, OpenXRActionSet&, const XrSystemProperties&, OpenXRHandFlags, int index);
~OpenXRInputSource();

XrResult SuggestBindings(SuggestedBindings&) const;
void Update(const XrFrameState&, XrSpace, const vrb::Matrix& head, const vrb::Vector& offsets, device::RenderMode, DeviceDelegate::PointerMode pointerMode, bool usingEyeTracking, bool handTrackingEnabled, ControllerDelegate& delegate);
void Update(const XrFrameState&, XrSpace, const vrb::Matrix& head, const vrb::Vector& offsets, device::RenderMode, DeviceDelegate::PointerMode pointerMode, bool usingEyeTracking, bool handTrackingEnabled, const vrb::Matrix& eyeTrackingTransform, ControllerDelegate& delegate);
XrResult UpdateInteractionProfile(ControllerDelegate&);
std::string ControllerModelName() const;
OpenXRInputMapping* GetActiveMapping() const { return mActiveMapping; }
Expand Down

0 comments on commit e72b8a8

Please sign in to comment.