Added slerp from from three to interpolate rotations

src/components/AnnotationView.js

@@ -2,12 +2,12 @@ import React, { useRef } from "react";
 import { Dropdown, Button, ButtonGroup } from "react-bootstrap";
 import ReactPlayer from "react-player";
 import SceneContainer from "./SceneContainer";
-import { getFixedNumber } from "../utils/MathUtils";
+import { getFixedNumber, frameToS } from "../utils/MathUtils";
 import FormattedTime from "../components/videoPlayer/FormattedTime";
 import { useObjectStore, useCameraStore, useAnnotationStore, useVideoStore } from '../stores';
 
 /**
- * Video player slide component.
+ * Video player slider component.
  *
  * @param {void} onMouseUp Mouse up listener
  * @param {void} onMouseDown Mouse down listener
@@ -45,6 +45,7 @@ const AnnotationView = () => {
     playing,
     played,
     duration,
+    fps,
     playbackRate,
     loop,
     seeking,
@@ -65,15 +66,15 @@ const AnnotationView = () => {
   const objectStore = useObjectStore();
   const { camPosition } = useCameraStore();
 
-  const findAnnotation = (t) => {
+  const findAnnotation = (played) => {
     if (!annotationStore.reviewMode) return
-    const secondsLapsed = t * duration;
+    const framesLapsed = played * duration;
     let currentAnnotations = annotationStore.outputAnnotation.annotations;
     if (currentAnnotations.length <= 0) return;
     const annotation = currentAnnotations.reduce((prev, curr) => {
       let closest =
-        Math.abs(curr.time - secondsLapsed) <
-        Math.abs(prev.time - secondsLapsed)
+        Math.abs(curr.time - framesLapsed) <
+        Math.abs(prev.time - framesLapsed)
           ? curr
           : prev;
       return closest;
@@ -88,8 +89,9 @@ const AnnotationView = () => {
   };
 
   const handleSeekMouseUp = (e) => {
+    let fraction = getFixedNumber(e.target.value, 5);
     handleSeekingtoFalse();
-    player.current.seekTo(getFixedNumber(e.target.value, 5));
+    player.current.seekTo(fraction);
   };
 
   const handleRewind = () => {
@@ -106,14 +108,15 @@ const AnnotationView = () => {
 
   const onProgress = (state) => {
     const { played } = state;
-    if (!seeking) handleProgress(getFixedNumber(played, 5));
-    findAnnotation(played);
+    let fraction = getFixedNumber(played, 5);
+    if (!seeking) handleProgress(fraction);
+    findAnnotation(fraction);
   };
 
   const onSliderChange = (e) => {
-    let t = getFixedNumber(e.target.value, 5);
-    handleSeekChange(t);
-    findAnnotation(t);
+    let fraction = getFixedNumber(e.target.value, 5);
+    handleSeekChange(fraction);
+    findAnnotation(fraction);
   };
 
   return (
@@ -135,7 +138,7 @@ const AnnotationView = () => {
             controls={false}
             loop={loop}
             playbackRate={playbackRate}
-            progressInterval={1000 / 30}
+            progressInterval={1000 / fps}
             onReady={(e) => onVideoReady(e)}
             onStart={() => console.log("onStart")}
             onBuffer={() => console.log("onBuffer")}
@@ -200,9 +203,9 @@ const AnnotationView = () => {
           </div>
           <div className="d-flex align-items-center">
             <div className="player-control__time">
-              <FormattedTime seconds={played * duration} />
+              <FormattedTime seconds={played * frameToS(duration, fps)} />
               {" / "}
-              <FormattedTime seconds={duration} />
+              <FormattedTime seconds={frameToS(duration, fps)} />
             </div>
           </div>
         </div>

src/components/SceneContainer.js

@@ -1,14 +1,22 @@
 import React, { useRef, useEffect, useCallback, Suspense, useState } from "react";
 import { Canvas, useFrame, useThree, extend } from "@react-three/fiber";
-import { useGLTF, Environment, Html, softShadows, Shadow } from "@react-three/drei";
+import { useGLTF, Environment, Html, softShadows } from "@react-three/drei";
 import { useObjectStore, useAnnotationStore, useVideoStore } from '../stores';
-import { ObjectControls } from '../ObjectControls';
+import { ObjectControls } from '../extensions/ObjectControls';
 import { DragControls } from 'three-stdlib';
 import * as THREE from "three";
 extend({ DragControls, ObjectControls })
 
 softShadows()
 
+/**
+ * The 3D Object component
+ * 
+ * @param {Object} camPosition The initial position of camera
+ * @param {props} props The props from parent copmponent {SceneContainer}
+ * @returns 
+ */
+
 const Banana = ({camPosition, ...props}) => {
     const group = useRef();
     const object = useRef();
@@ -18,7 +26,6 @@ const Banana = ({camPosition, ...props}) => {
     const { addAnnotation, reviewMode } = useAnnotationStore()
 
     const { nodes, materials } = useGLTF(objectSelected.gltfFile);
-    const [ hovered, setHover ] = useState(false)
 
     const { camera, gl: { domElement }} = useThree();
 
@@ -27,10 +34,23 @@ const Banana = ({camPosition, ...props}) => {
     const rotation = Object.values(objRotation)
 
     const [vec] = useState(() => new THREE.Vector3())
+    const [euler] = useState(() => new THREE.Euler())
+    const [quaternion] = useState(() => new THREE.Quaternion());
 
     useFrame(() => {
-      if (reviewMode) object.current.position.lerp(vec.set(objPosition.x, objPosition.y, objPosition.z), 0.1)
-    })
+      if (reviewMode) {
+        object.current.position.lerp(
+          vec.set(objPosition.x, objPosition.y, objPosition.z),
+          0.1
+        ); // Use lerp to linearly interpolate between two vectors.
+        object.current.quaternion.slerp(
+          quaternion.setFromEuler(
+            euler.set(objRotation._x, objRotation._y, objRotation._z, "XYZ")
+          ),
+          0.1
+        ); // Use slerp to handle the spherical linear interpolation between quaternions.
+      }
+    });
 
     const handleKeys = useCallback((e) => {
       if (!e.shiftKey || !e.ctrlKey) {
@@ -57,13 +77,14 @@ const Banana = ({camPosition, ...props}) => {
 
     const annotateFrame = () => {
       let dataAnnotation = {
-          id:(videoState.played * videoState.duration), 
-          time: (videoState.played * videoState.duration),
+          id: Math.round(videoState.played * videoState.duration), 
+          time: Math.round(videoState.played * videoState.duration), // Frame
           position: {x: object.current.position.x,y: object.current.position.y,z: object.current.position.z},
           rotation: {_x: object.current.rotation._x, _y: object.current.rotation._y, _z: object.current.rotation._z},
           scale: object.current.scale.x
       }
-      addAnnotation(dataAnnotation)
+      if (!reviewMode) addAnnotation(dataAnnotation)
+      return
     }
 
     return (
@@ -75,30 +96,12 @@ const Banana = ({camPosition, ...props}) => {
               material={materials.material_0}
               scale={objScale}
               position={!reviewMode ? position : null} 
-              rotation={rotation}
-              onPointerUp={(e) => handleTranslation(object.current.position) }
+              rotation={!reviewMode ? rotation : [Math.PI / 2, 0, 0]}
+              onPointerUp={(e) => handleTranslation(object.current.position)}
+              onClick={annotateFrame}
             /> 
         </group>
 
-        <mesh 
-          receiveShadow 
-          castShadow 
-          position={[0, 0, -0.5]} 
-          onClick={annotateFrame}
-          onPointerOver={(e) => setHover(true)}
-          onPointerOut={(e) => setHover(false)}
-        >
-            <sphereBufferGeometry args={[0.05, 64, 64]} />
-            <meshPhysicalMaterial color={'purple'} clearcoat={1} clearcoatRoughness={0} />
-            <Shadow position-y={-0.79} rotation-x={-Math.PI / 2} opacity={0.6} scale={[0.8, 0.8, 1]} />
-            <directionalLight position={[4,3,3]} castShadow intensity={1.5} shadow-camera-far={70} />
-            { hovered && 
-              <Html>
-                <div className="scene-tooltip">Click to annotate</div> 
-              </Html> 
-            }
-        </mesh>
-
         { object.current && <dragControls args={[[ object.current ], camera, domElement]} /> }
         { object.current && <objectControls args={[camera, domElement, object.current ]}/>  }
       </>

src/stores/VideoStore.js

@@ -1,11 +1,13 @@
 import create from 'zustand';
+import { secondsToFrame } from '../utils/MathUtils';
 
 const useVideoStore = create(set => ({
     url: "https://storage.turbo360.co/instagram_clone-vjcpvp/rbgt_banana.mp4",
-    duration: 0,
+    duration: 0, // frames
     id :'react-player',
     played: 0,
     seekedTime: 0,
+    fps: 30,
     playing: false,
     controls: false ,
     muted: true,
@@ -22,7 +24,7 @@ const useVideoStore = create(set => ({
     handleToggleLoop : () => set ( state => ({ loop: !state.loop})),
     handleVideoPlayPause : () => set (state => ({ playing : !state.playing })),
     handleVideoRewind : () => set (state => ({ playing: false, played: 0})),
-    handleVideoDuration : (duration) => set (state => ({ duration : duration })), 
+    handleVideoDuration : (duration) => set (state => ({ duration : secondsToFrame(duration, state.fps) })), 
     handleEnded: () => set (state => ({ playing : state.loop})),
     handleProgress: (played) => set(state => ({ played: played })),
     handleSeekingtoFalse: () => set (state => ({ seeking: false })), 

src/utils/MathUtils.js

@@ -1,5 +1,8 @@
-
 const getFixedNumber = (number, digits) =>
   Math.round(number * 10 ** digits) / 10 ** digits;
 
-export { getFixedNumber };
+const secondsToFrame = (t, fps) => Math.round(t * fps);
+
+const frameToS = (frame, fps) => frame / fps;
+
+export { getFixedNumber, secondsToFrame, frameToS };