1
0
mirror of https://github.com/xiaopeng12138/MaiDXR.git synced 2024-12-19 04:15:53 +01:00
MaiDXR/Assets/Oculus/VR/Scripts/OVRTrackedKeyboard/OVRTrackedKeyboardHands.cs
2022-08-20 21:35:57 +02:00

524 lines
17 KiB
C#

/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
* All rights reserved.
*
* Licensed under the Oculus SDK License Agreement (the "License");
* you may not use the Oculus SDK except in compliance with the License,
* which is provided at the time of installation or download, or which
* otherwise accompanies this software in either electronic or hard copy form.
*
* You may obtain a copy of the License at
*
* https://developer.oculus.com/licenses/oculussdk/
*
* Unless required by applicable law or agreed to in writing, the Oculus SDK
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
using UnityEngine;
using UnityEngine.Assertions;
public class OVRTrackedKeyboardHands : MonoBehaviour
{
public GameObject LeftHandPresence;
public GameObject RightHandPresence;
private bool handPresenceInitialized_ = false;
private Transform leftHandRoot_;
private Transform rightHandRoot_;
public OVRTrackedKeyboard KeyboardTracker;
private OVRCameraRig cameraRig_;
private OVRHand leftHand_;
private OVRSkeleton leftHandSkeleton_;
private OVRSkeletonRenderer leftHandSkeletonRenderer_;
private GameObject leftHandSkeletonRendererGO_;
private SkinnedMeshRenderer leftHandSkinnedMeshRenderer_;
private OVRMeshRenderer leftHandMeshRenderer_;
private OVRHand rightHand_;
private OVRSkeleton rightHandSkeleton_;
private OVRSkeletonRenderer rightHandSkeletonRenderer_;
private GameObject rightHandSkeletonRendererGO_;
private OVRMeshRenderer rightHandMeshRenderer_;
private SkinnedMeshRenderer rightHandSkinnedMeshRenderer_;
public bool RightHandOverKeyboard { get; private set; } = false;
public bool LeftHandOverKeyboard { get; private set; } = false;
private static readonly float handInnerAlphaThreshold_ = 0.08f;
private static readonly float handOuterAlphaThreshold_ = 0.20f;
private static readonly float maximumPassthroughHandsDistance_ = 0.18f;
private static readonly float minimumModelHandsDistance_ = 0.11f;
private TrackedKeyboardHandsVisibilityChangedEvent? lastVisibilityEvent_ = null;
private struct HandBoneMapping
{
public Transform LeftHandTransform;
public Transform LeftPresenceTransform;
public Transform RightHandTransform;
public Transform RightPresenceTransform;
public OVRSkeleton.BoneId BoneName;
public string HandPresenceLeftBoneName;
public string HandPresenceRightBoneName;
};
private readonly HandBoneMapping[] boneMappings_ = new HandBoneMapping[]
{
new HandBoneMapping
{
BoneName = OVRSkeleton.BoneId.Hand_WristRoot,
HandPresenceLeftBoneName = "b_l_wrist",
HandPresenceRightBoneName = "b_r_wrist"
},
new HandBoneMapping
{
BoneName = OVRSkeleton.BoneId.Hand_Thumb0,
HandPresenceLeftBoneName = "b_l_thumb0",
HandPresenceRightBoneName = "b_r_thumb0"
},
new HandBoneMapping
{
BoneName = OVRSkeleton.BoneId.Hand_Thumb1,
HandPresenceLeftBoneName = "b_l_thumb1",
HandPresenceRightBoneName = "b_r_thumb1"
},
new HandBoneMapping
{
BoneName = OVRSkeleton.BoneId.Hand_Thumb2,
HandPresenceLeftBoneName = "b_l_thumb2",
HandPresenceRightBoneName = "b_r_thumb2"
},
new HandBoneMapping
{
BoneName = OVRSkeleton.BoneId.Hand_Thumb3,
HandPresenceLeftBoneName = "b_l_thumb3",
HandPresenceRightBoneName = "b_r_thumb3"
},
new HandBoneMapping
{
BoneName = OVRSkeleton.BoneId.Hand_Index1,
HandPresenceLeftBoneName = "b_l_index1",
HandPresenceRightBoneName = "b_r_index1"
},
new HandBoneMapping
{
BoneName = OVRSkeleton.BoneId.Hand_Index2,
HandPresenceLeftBoneName = "b_l_index2",
HandPresenceRightBoneName = "b_r_index2"
},
new HandBoneMapping
{
BoneName = OVRSkeleton.BoneId.Hand_Index3,
HandPresenceLeftBoneName = "b_l_index3",
HandPresenceRightBoneName = "b_r_index3"
},
new HandBoneMapping
{
BoneName = OVRSkeleton.BoneId.Hand_Middle1,
HandPresenceLeftBoneName = "b_l_middle1",
HandPresenceRightBoneName = "b_r_middle1"
},
new HandBoneMapping
{
BoneName = OVRSkeleton.BoneId.Hand_Middle2,
HandPresenceLeftBoneName = "b_l_middle2",
HandPresenceRightBoneName = "b_r_middle2"
},
new HandBoneMapping
{
BoneName = OVRSkeleton.BoneId.Hand_Middle3,
HandPresenceLeftBoneName = "b_l_middle3",
HandPresenceRightBoneName = "b_r_middle3"
},
new HandBoneMapping
{
BoneName = OVRSkeleton.BoneId.Hand_Ring1,
HandPresenceLeftBoneName = "b_l_ring1",
HandPresenceRightBoneName = "b_r_ring1"
},
new HandBoneMapping
{
BoneName = OVRSkeleton.BoneId.Hand_Ring2,
HandPresenceLeftBoneName = "b_l_ring2",
HandPresenceRightBoneName = "b_r_ring2"
},
new HandBoneMapping
{
BoneName = OVRSkeleton.BoneId.Hand_Ring3,
HandPresenceLeftBoneName = "b_l_ring3",
HandPresenceRightBoneName = "b_r_ring3"
},
new HandBoneMapping
{
BoneName = OVRSkeleton.BoneId.Hand_Pinky0,
HandPresenceLeftBoneName = "b_l_pinky0",
HandPresenceRightBoneName = "b_r_pinky0"
},
new HandBoneMapping
{
BoneName = OVRSkeleton.BoneId.Hand_Pinky1,
HandPresenceLeftBoneName = "b_l_pinky1",
HandPresenceRightBoneName = "b_r_pinky1"
},
new HandBoneMapping
{
BoneName = OVRSkeleton.BoneId.Hand_Pinky2,
HandPresenceLeftBoneName = "b_l_pinky2",
HandPresenceRightBoneName = "b_r_pinky2"
},
new HandBoneMapping
{
BoneName = OVRSkeleton.BoneId.Hand_Pinky3,
HandPresenceLeftBoneName = "b_l_pinky3",
HandPresenceRightBoneName = "b_r_pinky3"
}
};
public Material HandsMaterial;
#region MATERIAL PROPERTIES
private const float XSCALE = 0.73f;
private const float YSCALE = 0.8f;
private const float FORWARD_OFFSET = -0.02f;
private int keyboardPositionID_;
private int keyboardRotationID_;
private int keyboardScaleID_;
#endregion
private void Awake() {
KeyboardTracker.TrackedKeyboardActiveChanged += TrackedKeyboardActiveUpdated;
KeyboardTracker.TrackedKeyboardVisibilityChanged += TrackedKeyboardVisibilityChanged;
keyboardPositionID_ = Shader.PropertyToID("_KeyboardPosition");
keyboardRotationID_ = Shader.PropertyToID("_KeyboardRotation");
keyboardScaleID_ = Shader.PropertyToID("_KeyboardScale");
}
private void Start()
{
cameraRig_ = FindObjectOfType<OVRCameraRig>();
leftHand_ = cameraRig_.leftHandAnchor.GetComponentInChildren<OVRHand>();
rightHand_ = cameraRig_.rightHandAnchor.GetComponentInChildren<OVRHand>();
leftHandSkeleton_ = leftHand_.GetComponent<OVRSkeleton>();
rightHandSkeleton_ = rightHand_.GetComponent<OVRSkeleton>();
leftHandMeshRenderer_ = leftHand_.GetComponent<OVRMeshRenderer>();
rightHandMeshRenderer_ = rightHand_.GetComponent<OVRMeshRenderer>();
leftHandSkeletonRenderer_ = leftHand_.GetComponent<OVRSkeletonRenderer>();
rightHandSkeletonRenderer_ = rightHand_.GetComponent<OVRSkeletonRenderer>();
if (!leftHandSkeletonRenderer_.enabled)
{
// App is not using skeleton renderer
leftHandSkeletonRenderer_ = null;
rightHandSkeletonRenderer_ = null;
}
leftHandSkinnedMeshRenderer_ = leftHand_.GetComponent<SkinnedMeshRenderer>();
rightHandSkinnedMeshRenderer_ = rightHand_.GetComponent<SkinnedMeshRenderer>();
var leftHand = GameObject.Instantiate(LeftHandPresence);
var rightHand = GameObject.Instantiate(RightHandPresence);
leftHandRoot_ = leftHand.transform;
rightHandRoot_ = rightHand.transform;
leftHand.SetActive(false);
rightHand.SetActive(false);
#if !UNITY_EDITOR // Initialized in LateUpdate() in editor
RetargetHandTrackingToHandPresence();
enabled = false;
#endif
}
private bool AreControllersActive =>
!(leftHand_.IsTracked || rightHand_.IsTracked);
private void LateUpdate()
{
#if UNITY_EDITOR
if (!handPresenceInitialized_)
{
if (leftHandSkeleton_.IsInitialized && rightHandSkeleton_.IsInitialized)
{
RetargetHandTrackingToHandPresence();
}
else
{
return;
}
}
#endif
if (AreControllersActive)
{
DisableHandObjects();
return;
}
foreach (var boneEntry in boneMappings_)
{
boneEntry.LeftPresenceTransform.localRotation = boneEntry.LeftHandTransform.localRotation;
boneEntry.RightPresenceTransform.localRotation = boneEntry.RightHandTransform.localRotation;
if (boneEntry.BoneName == OVRSkeleton.BoneId.Hand_WristRoot)
{
boneEntry.LeftPresenceTransform.rotation = boneEntry.LeftHandTransform.rotation;
boneEntry.RightPresenceTransform.rotation = boneEntry.RightHandTransform.rotation;
var leftScale = leftHand_.HandScale;
var rightScale = rightHand_.HandScale;
boneEntry.RightPresenceTransform.localScale = new Vector3(rightScale, rightScale, rightScale);
boneEntry.LeftPresenceTransform.localScale = new Vector3(leftScale, leftScale, leftScale);
}
}
rightHandRoot_.position = rightHand_.transform.position;
rightHandRoot_.rotation = rightHand_.transform.rotation;
leftHandRoot_.position = leftHand_.transform.position;
leftHandRoot_.rotation = leftHand_.transform.rotation;
var leftHandDistance = GetHandDistanceToKeyboard(leftHandSkeleton_);
var rightHandDistance = GetHandDistanceToKeyboard(rightHandSkeleton_);
LeftHandOverKeyboard = ShouldEnablePassthrough(leftHandDistance);
RightHandOverKeyboard = ShouldEnablePassthrough(rightHandDistance);
KeyboardTracker.HandsOverKeyboard = RightHandOverKeyboard || LeftHandOverKeyboard;
var enableLeftModel = ShouldEnableModel(leftHandDistance);
var enableRightModel = ShouldEnableModel(rightHandDistance);
SetHandModelsEnabled(enableLeftModel, enableRightModel);
if (KeyboardTracker.Presentation == OVRTrackedKeyboard.KeyboardPresentation.PreferOpaque)
{
// Used mixed reality service hands
leftHandRoot_.gameObject.SetActive(false);
rightHandRoot_.gameObject.SetActive(false);
}
else
{
leftHandRoot_.gameObject.SetActive(LeftHandOverKeyboard);
rightHandRoot_.gameObject.SetActive(RightHandOverKeyboard);
}
var position = KeyboardTracker.ActiveKeyboardTransform?.position;
var rotation = KeyboardTracker.ActiveKeyboardTransform?.rotation;
var offset = KeyboardTracker.ActiveKeyboardTransform == null
? Vector3.zero
: KeyboardTracker.ActiveKeyboardTransform.forward * FORWARD_OFFSET;
HandsMaterial.SetVector(keyboardPositionID_, position.HasValue ? position.Value + offset : Vector3.zero);
HandsMaterial.SetVector(keyboardRotationID_, rotation.HasValue ? rotation.Value.eulerAngles : Vector3.zero);
HandsMaterial.SetVector(
keyboardScaleID_,
new Vector4(
KeyboardTracker.ActiveKeyboardInfo.Dimensions.x * XSCALE,
0.1f,
KeyboardTracker.ActiveKeyboardInfo.Dimensions.z * YSCALE,
1f
)
);
if (lastVisibilityEvent_ == null
|| LeftHandOverKeyboard != lastVisibilityEvent_.Value.leftVisible
|| RightHandOverKeyboard != lastVisibilityEvent_.Value.rightVisible)
{
lastVisibilityEvent_ = new TrackedKeyboardHandsVisibilityChangedEvent
{
leftVisible = LeftHandOverKeyboard,
rightVisible = RightHandOverKeyboard
};
KeyboardTracker.UpdateKeyboardVisibility();
}
if (LeftHandOverKeyboard || RightHandOverKeyboard)
{
var handsIntensity = new OVRPlugin.InsightPassthroughKeyboardHandsIntensity
{
LeftHandIntensity =
ComputeOpacity(leftHandDistance, handInnerAlphaThreshold_, handOuterAlphaThreshold_),
RightHandIntensity =
ComputeOpacity(rightHandDistance, handInnerAlphaThreshold_, handOuterAlphaThreshold_)
};
OVRPlugin.SetInsightPassthroughKeyboardHandsIntensity(KeyboardTracker.PassthroughOverlay.layerId, handsIntensity);
}
}
private bool ShouldEnablePassthrough(float distance)
{
return distance <= maximumPassthroughHandsDistance_;
}
private bool ShouldEnableModel(float distance)
{
return distance >= minimumModelHandsDistance_;
}
private float GetHandDistanceToKeyboard(OVRSkeleton handSkeleton)
{
// TODO: Switch back to PointerPose once it's working in OpenXR
var pinchPosition = handSkeleton.Bones[(int) OVRSkeleton.BoneId.Hand_Index3].Transform.position;
var handPosition = handSkeleton.Bones[(int) OVRSkeleton.BoneId.Hand_Middle1].Transform.position;
var pinkyPosition = handSkeleton.Bones[(int) OVRSkeleton.BoneId.Hand_Pinky3].Transform.position;
return Mathf.Min(KeyboardTracker.GetDistanceToKeyboard(pinchPosition),
KeyboardTracker.GetDistanceToKeyboard(handPosition),
KeyboardTracker.GetDistanceToKeyboard(pinkyPosition));
}
private float ComputeOpacity(float distance, float innerThreshold, float outerThreshold)
{
return Mathf.Clamp((outerThreshold - distance) / (outerThreshold - innerThreshold), 0.0f, 1.0f);
}
private void SetHandModelsEnabled(bool enableLeftModel, bool enableRightModel)
{
leftHandMeshRenderer_.enabled = enableLeftModel;
rightHandMeshRenderer_.enabled = enableRightModel;
leftHandSkinnedMeshRenderer_.enabled = enableLeftModel;
rightHandSkinnedMeshRenderer_.enabled = enableRightModel;
if (leftHandSkeletonRenderer_ != null)
{
if (leftHandSkeletonRendererGO_ == null)
{
leftHandSkeletonRendererGO_ = leftHandSkeletonRenderer_.gameObject.transform.Find("SkeletonRenderer")?.gameObject;
rightHandSkeletonRendererGO_ = rightHandSkeletonRenderer_.gameObject.transform.Find("SkeletonRenderer")?.gameObject;
}
if (leftHandSkeletonRendererGO_ != null)
{
leftHandSkeletonRendererGO_.SetActive(enableLeftModel);
}
if (rightHandSkeletonRendererGO_ != null)
{
rightHandSkeletonRendererGO_.SetActive(enableRightModel);
}
}
}
private void RetargetHandTrackingToHandPresence()
{
Assert.IsTrue(LeftHandPresence != null && RightHandPresence != null);
for (int index = 0; index < boneMappings_.Length; index++)
{
var entry = boneMappings_[index];
var ovrBoneStringLeft = OVRSkeleton.BoneLabelFromBoneId(OVRSkeleton.SkeletonType.HandLeft, entry.BoneName);
var ovrBoneStringRight = OVRSkeleton.BoneLabelFromBoneId(OVRSkeleton.SkeletonType.HandRight, entry.BoneName);
boneMappings_[index].LeftHandTransform =
leftHand_.transform.FindChildRecursive(ovrBoneStringLeft);
boneMappings_[index].LeftPresenceTransform = leftHandRoot_.FindChildRecursive(entry.HandPresenceLeftBoneName);
boneMappings_[index].RightHandTransform =
rightHand_.transform.FindChildRecursive(ovrBoneStringRight);
boneMappings_[index].RightPresenceTransform = rightHandRoot_.FindChildRecursive(entry.HandPresenceRightBoneName);
Assert.IsTrue(
boneMappings_[index].LeftPresenceTransform != null
&& boneMappings_[index].RightPresenceTransform != null
&& boneMappings_[index].RightHandTransform != null
&& boneMappings_[index].LeftHandTransform != null,
string.Format(
"[tracked_keyboard] - entry.lp {0} && entry.rp {1} && entry.rt {2} && entry.lt {3}, {4}, {5}",
boneMappings_[index].LeftPresenceTransform,
boneMappings_[index].RightPresenceTransform,
boneMappings_[index].RightHandTransform,
boneMappings_[index].LeftHandTransform,
ovrBoneStringRight,
ovrBoneStringLeft
)
);
}
handPresenceInitialized_ = true;
}
private void StopHandPresence()
{
enabled = false;
// Re-enable hand models if they are disabled, let OVRHand handle controller/hands switching
SetHandModelsEnabled(true, true);
DisableHandObjects();
}
private void DisableHandObjects()
{
KeyboardTracker.HandsOverKeyboard = false;
RightHandOverKeyboard = false;
LeftHandOverKeyboard = false;
if (leftHandRoot_ != null)
{
leftHandRoot_.gameObject.SetActive(false);
}
if (rightHandRoot_ != null)
{
rightHandRoot_.gameObject.SetActive(false);
}
}
public void TrackedKeyboardActiveUpdated(OVRTrackedKeyboard.TrackedKeyboardSetActiveEvent e)
{
if (!e.IsEnabled)
{
StopHandPresence();
}
}
public void TrackedKeyboardVisibilityChanged(OVRTrackedKeyboard.TrackedKeyboardVisibilityChangedEvent e)
{
switch (e.State)
{
case OVRTrackedKeyboard.TrackedKeyboardState.Offline:
case OVRTrackedKeyboard.TrackedKeyboardState.NoTrackableKeyboard:
case OVRTrackedKeyboard.TrackedKeyboardState.StartedNotTracked:
StopHandPresence();
break;
case OVRTrackedKeyboard.TrackedKeyboardState.Valid:
enabled = handPresenceInitialized_;
break;
case OVRTrackedKeyboard.TrackedKeyboardState.Stale:
if (e.TrackingTimeout)
{
StopHandPresence();
}
break;
case OVRTrackedKeyboard.TrackedKeyboardState.Uninitialized:
case OVRTrackedKeyboard.TrackedKeyboardState.Error:
case OVRTrackedKeyboard.TrackedKeyboardState.ErrorExtensionFailed:
StopHandPresence();
Debug.LogWarning("Invalid state passed into TrackedKeyboardVisibilityChanged " + e.State.ToString());
break;
default:
throw new System.Exception(
$"[tracked_keyboard] - unhandled state: TrackedKeyboardVisibilityChanged {e.State}"
);
}
}
public struct TrackedKeyboardHandsVisibilityChangedEvent
{
public bool leftVisible;
public bool rightVisible;
}
}