adding hand and fix detection for image not using worldlandmarks

AmyangXYZ · Sep 24, 2024 · 3bf1ee7 · 3bf1ee7
1 parent 62c4072
commit 3bf1ee7
Show file tree

Hide file tree

Showing 3 changed files with 50 additions and 5 deletions.
diff --git a/src/App.tsx b/src/App.tsx
@@ -10,6 +10,8 @@ import { Download } from "@mui/icons-material"
 function App(): JSX.Element {
   const [pose, setPose] = useState<NormalizedLandmark[] | null>(null)
   const [face, setFace] = useState<NormalizedLandmark[] | null>(null)
+  const [leftHand, setLeftHand] = useState<NormalizedLandmark[] | null>(null)
+  const [rightHand, setRightHand] = useState<NormalizedLandmark[] | null>(null)
   const [lerpFactor, setLerpFactor] = useState<number>(0.5)
   const [fps, setFps] = useState<number>(0)
 
@@ -38,8 +40,21 @@ function App(): JSX.Element {
           </a>
         </div>
       </header>
-      <Video setPose={setPose} setFace={setFace} setLerpFactor={setLerpFactor}></Video>
-      <MMDScene pose={pose} face={face} lerpFactor={lerpFactor} setFps={setFps}></MMDScene>
+      <Video
+        setPose={setPose}
+        setFace={setFace}
+        setLeftHand={setLeftHand}
+        setRightHand={setRightHand}
+        setLerpFactor={setLerpFactor}
+      ></Video>
+      <MMDScene
+        pose={pose}
+        face={face}
+        leftHand={leftHand}
+        rightHand={rightHand}
+        lerpFactor={lerpFactor}
+        setFps={setFps}
+      ></MMDScene>
     </>
   )
 }

diff --git a/src/MMDScene.tsx b/src/MMDScene.tsx
@@ -31,11 +31,15 @@ const availableModels = ["深空之眼-托特", "深空之眼-托特2", "鸣潮-
 function MMDScene({
   pose,
   face,
+  leftHand,
+  rightHand,
   lerpFactor,
   setFps,
 }: {
   pose: NormalizedLandmark[] | null
   face: NormalizedLandmark[] | null
+  leftHand: NormalizedLandmark[] | null
+  rightHand: NormalizedLandmark[] | null
   lerpFactor: number
   setFps: (fps: number) => void
 }): JSX.Element {
@@ -656,6 +660,8 @@ function MMDScene({
       updateMMDFace(mmdModelRef.current, face)
     }
   }, [face])
+
+  useEffect(() => {}, [leftHand, rightHand])
   return (
     <>
       <canvas ref={canvasRef} className="scene"></canvas>

diff --git a/src/Video.tsx b/src/Video.tsx
@@ -39,10 +39,14 @@ const VisuallyHiddenInput = styled("input")({
 function Video({
   setPose,
   setFace,
+  setLeftHand,
+  setRightHand,
   setLerpFactor,
 }: {
   setPose: (pose: NormalizedLandmark[]) => void
   setFace: (face: NormalizedLandmark[]) => void
+  setLeftHand: (leftHand: NormalizedLandmark[]) => void
+  setRightHand: (rightHand: NormalizedLandmark[]) => void
   setLerpFactor: (lerpFactor: number) => void
 }): JSX.Element {
   const videoRef = useRef<HTMLVideoElement>(null)
@@ -216,6 +220,16 @@ function Video({
               } else {
                 setFace([])
               }
+              if (result.leftHandWorldLandmarks && result.leftHandWorldLandmarks.length > 0) {
+                setLeftHand(result.leftHandWorldLandmarks[0])
+              } else {
+                setLeftHand([])
+              }
+              if (result.rightHandWorldLandmarks && result.rightHandWorldLandmarks.length > 0) {
+                setRightHand(result.rightHandWorldLandmarks[0])
+              } else {
+                setRightHand([])
+              }
             })
           } else if (
             imgRef.current &&
@@ -225,22 +239,32 @@ function Video({
           ) {
             lastImgSrc = imgRef.current.src
             holisticLandmarkerRef.current!.detect(imgRef.current, (result) => {
-              if (result.poseLandmarks[0]) {
-                setPose(result.poseLandmarks[0])
+              if (result.poseWorldLandmarks[0]) {
+                setPose(result.poseWorldLandmarks[0])
               } else {
                 setPose([])
               }
               if (result.faceLandmarks && result.faceLandmarks.length > 0) {
                 setFace(result.faceLandmarks[0])
               }
+              if (result.leftHandWorldLandmarks && result.leftHandWorldLandmarks.length > 0) {
+                setLeftHand(result.leftHandWorldLandmarks[0])
+              } else {
+                setLeftHand([])
+              }
+              if (result.rightHandWorldLandmarks && result.rightHandWorldLandmarks.length > 0) {
+                setRightHand(result.rightHandWorldLandmarks[0])
+              } else {
+                setRightHand([])
+              }
             })
           }
           requestAnimationFrame(detect)
         }
         detect()
       }
     )
-  }, [setPose, setFace, imgRef, videoRef, isRecordingRef])
+  }, [setPose, setFace, setLeftHand, setRightHand, imgRef, videoRef, isRecordingRef])
 
   const replayCallback = (fps: number) => {
     setIsReplaying(true)