Skip to content

Commit

Permalink
Add support for MSFT hand interaction profile
Browse files Browse the repository at this point in the history 
Support for the Khronos XR_EXT_HAND_INTERACTION extension was added
not so long ago. That extension defines a new interaction profile
that can be used to get the input from hands using the same OpenXR
concepts that are valid for physical controllers.

There is another vendor extension that does the same which is
XR_MSFT_HAND_INTERACTION which is supported in devices that do not
support the Khronos one, like for example Meta devices or the Lynx-R1.

Even if they are 2 different extensions, the nice thing is that we
could use the same code path for both reducing the mess and complexity
of the hand tracking code. Not only that, as they both define
interaction profiles, they integrate nicely with the code that
handles physical controllers.

Apart from that we had to fine tune the condition that disables hands
when aim is not available because it was disabling the hand even if
we had valid data which is not correct.

This profile is supported, among others, in
* Meta Quest2, Quest3, Quest Pro
* Lynx-R1
  • Loading branch information
@svillar
svillar committed Oct 7, 2024
1 parent 3aea808 commit 7176e94
Show file tree
Hide file tree
Showing 4 changed files with 48 additions and 14 deletions.
3 changes: 3 additions & 0 deletions app/src/openxr/cpp/DeviceDelegateOpenXR.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,9 @@ struct DeviceDelegateOpenXR::State {
if (OpenXRExtensions::IsExtensionSupported(XR_BD_CONTROLLER_INTERACTION_EXTENSION_NAME))
extensions.push_back(XR_BD_CONTROLLER_INTERACTION_EXTENSION_NAME);

if (OpenXRExtensions::IsExtensionSupported(XR_MSFT_HAND_INTERACTION_EXTENSION_NAME))
extensions.push_back(XR_MSFT_HAND_INTERACTION_EXTENSION_NAME);

java = {XR_TYPE_INSTANCE_CREATE_INFO_ANDROID_KHR};
java.applicationVM = javaContext->vm;
java.applicationActivity = javaContext->activity;
Expand Down
21 changes: 19 additions & 2 deletions app/src/openxr/cpp/OpenXRInputMappings.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ namespace crow {
constexpr const char* kPathActionValue { "value" };
constexpr const char* kPathActionReady { "ready_ext" };
constexpr const char* kInteractionProfileHandInteraction { "/interaction_profiles/ext/hand_interaction_ext" };
constexpr const char* kInteractionProfileMSFTHandInteraction { "/interaction_profiles/microsoft/hand_interaction" };

// OpenXR Button List
enum class OpenXRButtonType {
Expand Down Expand Up @@ -436,8 +437,24 @@ namespace crow {
{}
};

const std::array<OpenXRInputMapping, 12> OpenXRInputMappings {
OculusTouch, OculusTouch2, MetaQuestTouchPro, Pico4x, Pico4xOld, PicoNeo3, Hvr6DOF, Hvr3DOF, LenovoVRX, MagicLeap2, MetaTouchPlus, HandInteraction
const OpenXRInputMapping MSFTHandInteraction {
kInteractionProfileMSFTHandInteraction,
IS_6DOF,
"",
"",
device::UnknownType,
std::vector<OpenXRInputProfile> { "generic-hand-select-grasp", "generic-hand-select", "generic-hand" },
std::vector<OpenXRButton> {
{ OpenXRButtonType::Trigger, "input/select", OpenXRButtonFlags::Value, OpenXRHandFlags::Both },
{ OpenXRButtonType::Squeeze, kPathSqueeze, OpenXRButtonFlags::Value, OpenXRHandFlags::Both },
},
{},
{}
};


const std::array<OpenXRInputMapping, 13> OpenXRInputMappings {
OculusTouch, OculusTouch2, MetaQuestTouchPro, Pico4x, Pico4xOld, PicoNeo3, Hvr6DOF, Hvr3DOF, LenovoVRX, MagicLeap2, MetaTouchPlus, HandInteraction, MSFTHandInteraction
};

} // namespace crow
36 changes: 25 additions & 11 deletions app/src/openxr/cpp/OpenXRInputSource.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,15 +51,16 @@ XrResult OpenXRInputSource::Initialize()
{
mSubactionPathName = mHandeness == OpenXRHandFlags::Left ? kPathLeftHand : kPathRightHand;
mSubactionPath = mActionSet.GetSubactionPath(mHandeness);
mIsHandInteractionEXTSupported = OpenXRExtensions::IsExtensionSupported(XR_EXT_HAND_INTERACTION_EXTENSION_NAME);
mIsHandInteractionSupported = OpenXRExtensions::IsExtensionSupported(XR_EXT_HAND_INTERACTION_EXTENSION_NAME) ||
OpenXRExtensions::IsExtensionSupported(XR_MSFT_HAND_INTERACTION_EXTENSION_NAME);

// Initialize Action Set.
std::string prefix = std::string("input_") + (mHandeness == OpenXRHandFlags::Left ? "left" : "right");

// Initialize pose actions and spaces.
RETURN_IF_XR_FAILED(mActionSet.GetOrCreateAction(XR_ACTION_TYPE_POSE_INPUT, "grip", OpenXRHandFlags::Both, mGripAction));
RETURN_IF_XR_FAILED(mActionSet.GetOrCreateAction(XR_ACTION_TYPE_POSE_INPUT, "pointer", OpenXRHandFlags::Both, mPointerAction));
if (mIsHandInteractionEXTSupported) {
if (OpenXRExtensions::IsExtensionSupported(XR_EXT_HAND_INTERACTION_EXTENSION_NAME)) {
RETURN_IF_XR_FAILED(mActionSet.GetOrCreateAction(XR_ACTION_TYPE_POSE_INPUT, "pinch_ext", OpenXRHandFlags::Both, mPinchPoseAction));
RETURN_IF_XR_FAILED(mActionSet.GetOrCreateAction(XR_ACTION_TYPE_POSE_INPUT, "poke_ext", OpenXRHandFlags::Both, mPokePoseAction));
}
Expand Down Expand Up @@ -142,7 +143,7 @@ XrResult OpenXRInputSource::Initialize()
#else
mSupportsFBHandTrackingAim = OpenXRExtensions::IsExtensionSupported(XR_FB_HAND_TRACKING_AIM_EXTENSION_NAME);
#endif
if (!mIsHandInteractionEXTSupported) {
if (!mIsHandInteractionSupported) {
if (mSupportsFBHandTrackingAim) {
mGestureManager = std::make_unique<OpenXRGestureManagerFBHandTrackingAim>();
} else {
Expand All @@ -152,7 +153,10 @@ XrResult OpenXRInputSource::Initialize()
}
VRB_LOG("OpenXR: using %s to compute hands aim", mSupportsFBHandTrackingAim ? XR_FB_HAND_TRACKING_AIM_EXTENSION_NAME : "hand joints");
} else {
VRB_LOG("OpenXR: using %s to compute hands aim", XR_EXT_HAND_INTERACTION_EXTENSION_NAME);
if (OpenXRExtensions::IsExtensionSupported(XR_EXT_HAND_INTERACTION_EXTENSION_NAME))
VRB_LOG("OpenXR: using %s to compute hands aim", XR_EXT_HAND_INTERACTION_EXTENSION_NAME);
if (OpenXRExtensions::IsExtensionSupported(XR_MSFT_HAND_INTERACTION_EXTENSION_NAME))
VRB_LOG("OpenXR: using %s to compute hands aim", XR_MSFT_HAND_INTERACTION_EXTENSION_NAME);
}
}

Expand Down Expand Up @@ -424,7 +428,7 @@ XrResult OpenXRInputSource::SuggestBindings(SuggestedBindings& bindings) const

// FIXME: reenable this once MagicLeap OS v1.8 is released as it has a fix for this.
// Otherwise the hand interaction profile is not properly used.
if (mIsHandInteractionEXTSupported && (DeviceUtils::GetDeviceTypeFromSystem(true) != device::MagicLeap2)) {
if (OpenXRExtensions::IsExtensionSupported(XR_EXT_HAND_INTERACTION_EXTENSION_NAME) && (DeviceUtils::GetDeviceTypeFromSystem(true) != device::MagicLeap2)) {
RETURN_IF_XR_FAILED(CreateBinding(mapping.path, mPinchPoseAction, mSubactionPathName + "/" + kPinchPose, bindings));
RETURN_IF_XR_FAILED(CreateBinding(mapping.path, mPokePoseAction, mSubactionPathName + "/" + kPokePose, bindings));
}
Expand Down Expand Up @@ -752,7 +756,7 @@ void OpenXRInputSource::Update(const XrFrameState& frameState, XrSpace localSpac
if (mPointerSpace == XR_NULL_HANDLE) {
CHECK_XRCMD(CreateActionSpace(mPointerAction, mPointerSpace));
}
if (mIsHandInteractionEXTSupported) {
if (OpenXRExtensions::IsExtensionSupported(XR_EXT_HAND_INTERACTION_EXTENSION_NAME)) {
if (mPinchSpace == XR_NULL_HANDLE)
CHECK_XRCMD(CreateActionSpace(mPinchPoseAction, mPinchSpace));
if (mPokeSpace == XR_NULL_HANDLE)
Expand Down Expand Up @@ -795,7 +799,7 @@ void OpenXRInputSource::Update(const XrFrameState& frameState, XrSpace localSpac
std::vector<vrb::Matrix> jointTransforms;
std::vector<float> jointRadii;
PopulateHandJointLocations(renderMode, jointTransforms, jointRadii);
if (!mIsHandInteractionEXTSupported) {
if (!mIsHandInteractionSupported) {
EmulateControllerFromHand(renderMode, frameState.predictedDisplayTime, head, jointTransforms[HAND_JOINT_FOR_AIM], pointerMode, usingEyeTracking, delegate);
delegate.SetHandJointLocations(mIndex, std::move(jointTransforms), std::move(jointRadii));
return;
Expand All @@ -813,15 +817,23 @@ void OpenXRInputSource::Update(const XrFrameState& frameState, XrSpace localSpac
bool usingTrackedPointer = pointerMode == DeviceDelegate::PointerMode::TRACKED_POINTER;
bool hasAim = isPoseActive && (poseLocation.locationFlags & XR_SPACE_LOCATION_ORIENTATION_VALID_BIT);


if (hasAim && mUsingHandInteractionProfile && handFacesHead) {
// XR_MSFT_hand_interaction returns true for aim when doing palm pinching. That's unexpected because
// as long as pinching finishes aim becomes false again. Force it to false to avoid showing
// the aim ray when palm pinching.
hasAim = false;
}

// Don't enable aim for eye tracking as we don't want to paint the beam. Note that we'd still
// have an aim, meaning that we can point, focus, click... UI elements.
delegate.SetAimEnabled(mIndex, hasAim && usingTrackedPointer);

// Disable the controller if there is no aim unless:
// a) we're using hand interaction profile and we have hand tracking info. In that case the user
// is facing their palm to do a hand gesture (that's why there is no aim)
// b) we're using eye tracking. In that case the eye gaze will be the aim, not the contorller's
if (!hasAim && !(mUsingHandInteractionProfile && gotHandTrackingInfo && handFacesHead) && !usingEyeTracking) {
// b) we're using eye tracking. In that case the eye gaze will be the aim, not the controller's
if (!hasAim && !(mUsingHandInteractionProfile && gotHandTrackingInfo) && !usingEyeTracking) {
delegate.SetEnabled(mIndex, false);
return;
}
Expand All @@ -835,7 +847,7 @@ void OpenXRInputSource::Update(const XrFrameState& frameState, XrSpace localSpac
};
adjustPoseLocation(offsets);

// XR_EXT_hand_interaction does not really require the hand joints data, but if we
// Hand interaction profiles do not really require the hand joints data, but if we
// set ControllerMode::Hand then Wolvic code assumes that it does.
delegate.SetMode(mIndex, mUsingHandInteractionProfile && gotHandTrackingInfo ? ControllerMode::Hand : ControllerMode::Device);
delegate.SetEnabled(mIndex, true);
Expand Down Expand Up @@ -1018,11 +1030,13 @@ XrResult OpenXRInputSource::UpdateInteractionProfile(ControllerDelegate& delegat
path = buffer;
path_len = writtenCount;
mActiveMapping = nullptr;
bool pathDefinesHandInteractionProfile = !strncmp(path, kInteractionProfileHandInteraction, path_len) ||
!strncmp(path, kInteractionProfileMSFTHandInteraction, path_len);

for (auto& mapping : mMappings) {
if (!strncmp(mapping.path, path, path_len)) {
mActiveMapping = &mapping;
mUsingHandInteractionProfile = !strncmp(path, kInteractionProfileHandInteraction, path_len);
mUsingHandInteractionProfile = pathDefinesHandInteractionProfile;
VRB_LOG("OpenXR: NEW active mapping %s", mActiveMapping->path);
break;
}
Expand Down
2 changes: 1 addition & 1 deletion app/src/openxr/cpp/OpenXRInputSource.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ class OpenXRInputSource {
HandMeshBufferPtr AcquireHandMeshBuffer();
void ReleaseHandMeshBuffer();

bool mIsHandInteractionEXTSupported { false };
bool mIsHandInteractionSupported { false };

void HandleEyeTrackingScroll(XrTime predictedDisplayTime, bool triggerClicked, const vrb::Matrix& pointerTransform, ControllerDelegate &controllerDelegate);
public:
Expand Down

0 comments on commit 7176e94

Please sign in to comment.