diff --git a/app/src/openxr/cpp/OpenXRInput.cpp b/app/src/openxr/cpp/OpenXRInput.cpp index eb6b3e0b9e..4e3c5ee913 100644 --- a/app/src/openxr/cpp/OpenXRInput.cpp +++ b/app/src/openxr/cpp/OpenXRInput.cpp @@ -90,7 +90,7 @@ XrResult OpenXRInput::Update(const XrFrameState& frameState, XrSpace baseSpace, bool usingEyeTracking = pointerMode == DeviceDelegate::PointerMode::TRACKED_EYE && updateEyeGaze(frameState, head, delegate); for (auto& input : mInputSources) { - input->Update(frameState, baseSpace, head, offsets, renderMode, pointerMode, usingEyeTracking, handTrackingEnabled, delegate); + input->Update(frameState, baseSpace, head, offsets, renderMode, pointerMode, usingEyeTracking, handTrackingEnabled, mEyeTrackingTransform, delegate); } // Update tracked keyboard @@ -374,8 +374,7 @@ bool OpenXRInput::updateEyeGaze(XrFrameState frameState, const vrb::Matrix& head vrb::Quaternion gazeOrientation(gazeLocation.pose.orientation.x, gazeLocation.pose.orientation.y, gazeLocation.pose.orientation.z, gazeLocation.pose.orientation.w); float* filteredOrientation = mOneEuroFilterGazeOrientation->filter(frameState.predictedDisplayTime, gazeOrientation.Data()); gazeOrientation = {filteredOrientation[0], filteredOrientation[1], filteredOrientation[2], filteredOrientation[3]}; - delegate.SetTransform(0, vrb::Matrix::Rotation(gazeOrientation).Translate(gazePosition)); - delegate.SetImmersiveBeamTransform(0, vrb::Matrix::Identity()); + mEyeTrackingTransform = vrb::Matrix::Rotation(gazeOrientation).Translate(gazePosition); return true; } diff --git a/app/src/openxr/cpp/OpenXRInput.h b/app/src/openxr/cpp/OpenXRInput.h index 405dc6777a..4338a679b2 100644 --- a/app/src/openxr/cpp/OpenXRInput.h +++ b/app/src/openxr/cpp/OpenXRInput.h @@ -56,6 +56,7 @@ class OpenXRInput { XrSpace mEyeGazeActionSpace {XR_NULL_HANDLE }; XrSpace mLocalReferenceSpace { XR_NULL_HANDLE}; std::unique_ptr mOneEuroFilterGazeOrientation; + vrb::Matrix mEyeTrackingTransform; public: static OpenXRInputPtr Create(XrInstance, XrSession, XrSystemProperties, XrSpace localSpace, diff --git a/app/src/openxr/cpp/OpenXRInputSource.cpp b/app/src/openxr/cpp/OpenXRInputSource.cpp index 9989063f76..a668951c34 100644 --- a/app/src/openxr/cpp/OpenXRInputSource.cpp +++ b/app/src/openxr/cpp/OpenXRInputSource.cpp @@ -639,7 +639,7 @@ OpenXRInputSource::PopulateHandJointLocations(device::RenderMode renderMode, std } } -void OpenXRInputSource::EmulateControllerFromHand(device::RenderMode renderMode, XrTime predictedDisplayTime, const vrb::Matrix& head, const vrb::Matrix& handJointForAim, DeviceDelegate::PointerMode pointerMode, bool usingEyeTracking, ControllerDelegate& delegate) +void OpenXRInputSource::EmulateControllerFromHand(device::RenderMode renderMode, XrTime predictedDisplayTime, const vrb::Matrix& head, const vrb::Matrix& handJointForAim, DeviceDelegate::PointerMode pointerMode, bool usingEyeTracking, const vrb::Matrix& eyeTrackingTransform, ControllerDelegate& delegate) { assert(mHasHandJoints); @@ -731,13 +731,15 @@ void OpenXRInputSource::EmulateControllerFromHand(device::RenderMode renderMode, delegate.SetImmersiveBeamTransform(mIndex, pointerTransform); } else { assert(pointerMode == DeviceDelegate::PointerMode::TRACKED_EYE); - HandleEyeTrackingScroll(predictedDisplayTime, triggerButtonPressed, pointerTransform, delegate); + HandleEyeTrackingScroll(predictedDisplayTime, triggerButtonPressed, pointerTransform, eyeTrackingTransform, delegate); + if (!triggerButtonPressed) + delegate.SetTransform(mIndex, eyeTrackingTransform); } delegate.SetCapabilityFlags(mIndex, flags); } -void OpenXRInputSource::Update(const XrFrameState& frameState, XrSpace localSpace, const vrb::Matrix &head, const vrb::Vector& offsets, device::RenderMode renderMode, DeviceDelegate::PointerMode pointerMode, bool usingEyeTracking, bool handTrackingEnabled, ControllerDelegate& delegate) +void OpenXRInputSource::Update(const XrFrameState& frameState, XrSpace localSpace, const vrb::Matrix &head, const vrb::Vector& offsets, device::RenderMode renderMode, DeviceDelegate::PointerMode pointerMode, bool usingEyeTracking, bool handTrackingEnabled, const vrb::Matrix& eyeTrackingTransform, ControllerDelegate& delegate) { if (mActiveMapping && ((mHandeness == OpenXRHandFlags::Left && !mActiveMapping->leftControllerModel) || @@ -801,7 +803,7 @@ void OpenXRInputSource::Update(const XrFrameState& frameState, XrSpace localSpac std::vector jointRadii; PopulateHandJointLocations(renderMode, jointTransforms, jointRadii); if (!mIsHandInteractionSupported) { - EmulateControllerFromHand(renderMode, frameState.predictedDisplayTime, head, jointTransforms[HAND_JOINT_FOR_AIM], pointerMode, usingEyeTracking, delegate); + EmulateControllerFromHand(renderMode, frameState.predictedDisplayTime, head, jointTransforms[HAND_JOINT_FOR_AIM], pointerMode, usingEyeTracking, eyeTrackingTransform, delegate); delegate.SetHandJointLocations(mIndex, std::move(jointTransforms), std::move(jointRadii)); return; } @@ -932,8 +934,11 @@ void OpenXRInputSource::Update(const XrFrameState& frameState, XrSpace localSpac if (button.type == OpenXRButtonType::Trigger) { delegate.SetSelectFactor(mIndex, state->value); - if (pointerMode == DeviceDelegate::PointerMode::TRACKED_EYE) - HandleEyeTrackingScroll(frameState.predictedDisplayTime, state->clicked, pointerTransform, delegate); + if (pointerMode == DeviceDelegate::PointerMode::TRACKED_EYE) { + HandleEyeTrackingScroll(frameState.predictedDisplayTime, state->clicked, pointerTransform, eyeTrackingTransform, delegate); + if (!state->clicked) + delegate.SetTransform(mIndex, eyeTrackingTransform); + } } // Select action @@ -1069,16 +1074,21 @@ OpenXRInputSource::GetNextHandMeshBuffer() { return mHandMeshMSFT.buffer; } -void OpenXRInputSource::HandleEyeTrackingScroll(XrTime predictedDisplayTime, bool triggerClicked, const vrb::Matrix &pointerTransform, ControllerDelegate &controllerDelegate) { +void +OpenXRInputSource::HandleEyeTrackingScroll(XrTime predictedDisplayTime, bool triggerClicked, const vrb::Matrix &pointerTransform, const vrb::Matrix& eyeTrackingTransform, ControllerDelegate &controllerDelegate) { if (!mTriggerWasClicked && triggerClicked) { - mControllerPositionOnGestureStart = pointerTransform.GetTranslation(); + mEyeGazeTransformOnPinchStart = eyeTrackingTransform; mEyeTrackingPinchStartTime = predictedDisplayTime; } else if (mTriggerWasClicked && triggerClicked) { // Throttle the start of the scroll gesture to avoid scrolling on pinch. if (predictedDisplayTime - mEyeTrackingPinchStartTime > kEyeTrackingScrollThreshold) { vrb::Vector currentControllerPosition = pointerTransform.GetTranslation(); - auto delta = currentControllerPosition - mControllerPositionOnGestureStart; - controllerDelegate.TranslateTransform(mIndex, delta * 10); + auto delta = currentControllerPosition - mControllerPositionOnScrollStart; + controllerDelegate.SetTransform(mIndex, mEyeGazeTransformOnPinchStart.Translate(delta * 10)); + } else { + // Keep updating the initial position while throttling to avoid a sudden jump when + // scrolling starts. + mControllerPositionOnScrollStart = pointerTransform.GetTranslation(); } } mTriggerWasClicked = triggerClicked; diff --git a/app/src/openxr/cpp/OpenXRInputSource.h b/app/src/openxr/cpp/OpenXRInputSource.h index a276390160..f65b4d5c7b 100644 --- a/app/src/openxr/cpp/OpenXRInputSource.h +++ b/app/src/openxr/cpp/OpenXRInputSource.h @@ -47,7 +47,7 @@ class OpenXRInputSource { void UpdateHaptics(ControllerDelegate&); bool GetHandTrackingInfo(XrTime predictedDisplayTime, XrSpace, const vrb::Matrix& head); float GetDistanceBetweenJoints (XrHandJointEXT jointA, XrHandJointEXT jointB); - void EmulateControllerFromHand(device::RenderMode renderMode, XrTime predictedDisplayTime, const vrb::Matrix& head, const vrb::Matrix& handJointForAim, DeviceDelegate::PointerMode pointerMode, bool usingEyeTracking, ControllerDelegate& delegate); + void EmulateControllerFromHand(device::RenderMode renderMode, XrTime predictedDisplayTime, const vrb::Matrix& head, const vrb::Matrix& handJointForAim, DeviceDelegate::PointerMode pointerMode, bool usingEyeTracking, const vrb::Matrix& eyeTrackingTransform, ControllerDelegate& delegate); void PopulateHandJointLocations(device::RenderMode, std::vector& jointTransforms, std::vector& jointRadii); XrInstance mInstance { XR_NULL_HANDLE }; @@ -85,7 +85,8 @@ class OpenXRInputSource { bool mUsingHandInteractionProfile { false }; device::DeviceType mDeviceType { device::UnknownType }; bool mTriggerWasClicked { false }; - vrb::Vector mControllerPositionOnGestureStart; + vrb::Vector mControllerPositionOnScrollStart; + vrb::Matrix mEyeGazeTransformOnPinchStart; XrTime mEyeTrackingPinchStartTime { 0 }; struct HandMeshMSFT { @@ -107,13 +108,13 @@ class OpenXRInputSource { bool mIsHandInteractionSupported { false }; - void HandleEyeTrackingScroll(XrTime predictedDisplayTime, bool triggerClicked, const vrb::Matrix& pointerTransform, ControllerDelegate &controllerDelegate); + void HandleEyeTrackingScroll(XrTime predictedDisplayTime, bool triggerClicked, const vrb::Matrix& pointerTransform, const vrb::Matrix& eyeTrackingTransform, ControllerDelegate &controllerDelegate); public: static OpenXRInputSourcePtr Create(XrInstance, XrSession, OpenXRActionSet&, const XrSystemProperties&, OpenXRHandFlags, int index); ~OpenXRInputSource(); XrResult SuggestBindings(SuggestedBindings&) const; - void Update(const XrFrameState&, XrSpace, const vrb::Matrix& head, const vrb::Vector& offsets, device::RenderMode, DeviceDelegate::PointerMode pointerMode, bool usingEyeTracking, bool handTrackingEnabled, ControllerDelegate& delegate); + void Update(const XrFrameState&, XrSpace, const vrb::Matrix& head, const vrb::Vector& offsets, device::RenderMode, DeviceDelegate::PointerMode pointerMode, bool usingEyeTracking, bool handTrackingEnabled, const vrb::Matrix& eyeTrackingTransform, ControllerDelegate& delegate); XrResult UpdateInteractionProfile(ControllerDelegate&); std::string ControllerModelName() const; OpenXRInputMapping* GetActiveMapping() const { return mActiveMapping; }