Added form to enter video url and select object

sahirgomez1 · Jun 25, 2021 · 4c0070c · 4c0070c
1 parent 5da7465
commit 4c0070c
Show file tree

Hide file tree

Showing 12 changed files with 502 additions and 48 deletions.
diff --git a/README.md b/README.md
@@ -1,19 +1,26 @@
 # YCB Object 3d annotation
 
-## Available Scripts
-
-To install
-`git clone https://github.com/sahirgomez1/ycb_objects.git`
-
-In the project directory :
-`cd ycb_objects_app`.\
-you can run
-`npm install` to install all dependencies.\
-then run
-`npm start`.
+## Installation and running
+
+To install. First clone the repo using:
+```
+git clone https://github.com/sahirgomez1/ycb_objects.git
+cd ycb_objects_app
+```
+
+In the project directory, you can run
+```
+npm install
+```
+to install all dependencies. Then run.\
+```
+npm start
+```
 
 ### `npm start`
 
+Add your 3D objects to the public folder, use `.gltf` files.
+
 Runs the app in the development mode.\
 Open [http://localhost:3000](http://localhost:3000) to view it in the browser.
 

diff --git a/public/hammer.gltf b/public/hammer.gltf
diff --git a/src/rbgt_banana.mp4 → public/rbgt_banana.mp4 b/src/rbgt_banana.mp4 → public/rbgt_banana.mp4
diff --git a/public/tomato_can.gltf b/public/tomato_can.gltf
diff --git a/src/assets/scss/main_content.scss b/src/assets/scss/main_content.scss
@@ -52,6 +52,17 @@
     }
 }
 
+.player-wrapper {
+    position: relative;
+    padding-top: 56.25%; /* 720 / 1280 = 0.5625 */
+  }
+
+.react-player {
+    position: absolute;
+    top: 0;
+    left: 0;
+}
+
 .scene-container {
     position: relative;
     width: 100%;

diff --git a/src/assets/scss/variables.scss b/src/assets/scss/variables.scss
@@ -39,7 +39,7 @@ $blue2:   #5CA5B2 !default;
 
 $default:       #172b4d !default;
 $primary:       #2D303D !default;
-$secondary:     #f7fafc !default;
+$secondary:     $gray-200 !default;
 $success:       $green !default;
 $info:          $teal !default;
 $warning:       $yellow !default;

diff --git a/src/components/AnnotationView.js b/src/components/AnnotationView.js
@@ -33,6 +33,7 @@ const AnnotationView = ({camPosition, objScale}) => {
         playbackRate,
         loop, 
         seeking,
+        setVideoDimensions,
         handleProgress,
         handleVideoPlayPause,
         handleVideoRewind,
@@ -67,24 +68,27 @@ const AnnotationView = ({camPosition, objScale}) => {
                   objScale={objScale}/>
               </div>
             </div>
-            <ReactPlayer
-              ref={player}
-              width='100%'
-              height='auto'
-              url={url}
-              playing={playing}
-              controls={false}
-              loop={loop}
-              playbackRate={playbackRate}
-              onReady={() => console.log('onReady')}
-              onStart={() => console.log('onStart')}
-              onBuffer={() => console.log('onBuffer')}
-              onSeek={e => console.log('onSeek', e)}
-              onEnded={handleEnded}
-              onError={e => console.log('onError', e)}
-              onProgress={onProgress}
-              onDuration={e => handleVideoDuration(e)}
-            />                         
+            <div className="player-wrapper">
+              <ReactPlayer
+                ref={player}
+                className='react-player'
+                width='100%'
+                height='100%'
+                url={url}
+                playing={playing}
+                controls={false}
+                loop={loop}
+                playbackRate={playbackRate}
+                onReady={(e) => setVideoDimensions(e)}
+                onStart={() => console.log('onStart')}
+                onBuffer={() => console.log('onBuffer')}
+                onSeek={e => console.log('onSeek', e)}
+                onEnded={handleEnded}
+                onError={e => console.log('onError', e)}
+                onProgress={onProgress}
+                onDuration={e => handleVideoDuration(e)}
+              />   
+            </div>                      
           </div>
           <div className="mt-2">
               <Slider

diff --git a/src/components/EnvironmentContainer.js b/src/components/EnvironmentContainer.js
@@ -13,12 +13,13 @@ softShadows()
 const Banana = ({camPosition, ...props}) => {
     const group = useRef();
     const object = useRef();
-    const { nodes, materials } = useGLTF("/banana.gltf");
-    const [ hovered, setHover ] = useState(false)
 
-    const { objPosition, objRotation, handleTranslation, handleRotation } = useObjectStore();
+    const { objectSelected, objPosition, objRotation, handleTranslation, handleRotation } = useObjectStore();
     const videoState = useVideoStore()
     const { addAnnotation } = useAnnotationStore()
+
+    const { nodes, materials } = useGLTF(objectSelected.gltfFile);
+    const [ hovered, setHover ] = useState(false)
 
     const { camera, gl: { domElement }} = useThree();
     camera.position.set( camPosition.xCamPos, camPosition.yCamPos, camPosition.zCamPos)
@@ -48,8 +49,12 @@ const Banana = ({camPosition, ...props}) => {
     })
 
     const annotateFrame = () => {
-      console.log(object.current)
-      let video_metadata = { url: videoState.url }
+      //console.log(object.current)
+      let video_metadata = { 
+          url: videoState.url, 
+          videoHeight: videoState.videoHeight, 
+          videoWidth: videoState.videoWidth 
+      }
       let dataAnnotation = {
           id: videoState.played, 
           time: (videoState.played * videoState.duration),

diff --git a/src/components/TopSection.js b/src/components/TopSection.js
@@ -1,14 +1,16 @@
 import React, { useState } from 'react';
-import { Container, Row, Col, Tab, Nav, FormControl, InputGroup } from "react-bootstrap";
+import { Container, Row, Col, Tab, Nav, Form, FormControl, InputGroup } from "react-bootstrap";
 import { useObjectStore } from '../stores/ObjectStore';
+import { useVideoStore } from '../stores/VideoStore';
 import AnnotationView from './AnnotationView';
 
 const TopSection = () => {
 
     const [camPosition, setCamPosition] = useState({ xCamPos: 5, yCamPos:5, zCamPos:8, fov:4 })
     const [objScale, setObjScale] = useState(3)
 
-    const { objPosition, objRotation, handleTranslation, handleRotation } = useObjectStore()
+    const { objectSelected, defaultObjects, handleSelectObject, objPosition, objRotation, handleTranslation, handleRotation } = useObjectStore()
+    const { url, setVideoOnScene } = useVideoStore()
 
     const handleChangeCam = e => {
       const { name, value } = e.target;
@@ -26,8 +28,34 @@ const TopSection = () => {
 
     return (
         <Container fluid="md" className="">
-        <Row className="justify-content-center">
+          <Row className="justify-content-center">
             <Col md={3} className="section-wrapper" >
+              <div className="controls">
+                <Form.Group>
+                  <Form.Label><h6>Video</h6></Form.Label>
+                  <Form.Control 
+                    size="sm" 
+                    className="span2" 
+                    type="text" 
+                    placeholder="Enter video URL" 
+                    value={url}
+                    onChange={setVideoOnScene}
+                  />
+                  <Form.Label><h6>Select Object</h6></Form.Label>
+                  <Form.Control 
+                    size="sm" 
+                    as="select" 
+                    value={objectSelected.name} 
+                    onChange={(e) => handleSelectObject(e.target.value)}
+                  >
+                    {defaultObjects.map((obj, index) => (
+                      <option key={index} value={obj.name}>
+                        {obj.name}
+                      </option>
+                    ))}
+                  </Form.Control>
+                </Form.Group>
+              </div>
               <div className="controls">
                 <Tab.Container id="left-tabs-example" defaultActiveKey="camera" >
                     <Row>

diff --git a/src/stores/AnnotationStore.js b/src/stores/AnnotationStore.js
@@ -8,12 +8,14 @@ const addRecord = (list, annotation) => {
 }
 
 const useAnnotationStore = create(set => ({
-    outputAnnotation : { url: "", videoWidth: 500, videoHeight: 400, annotations:[] },
+    outputAnnotation : { url: "", videoWidth: 0, videoHeight: 0, annotations:[] },
     addAnnotation : (video_metadata, annotation) => 
         set(state => ({ 
             outputAnnotation: {
                 ...state.outputAnnotation,
                 url: video_metadata.url, 
+                videoWidth: video_metadata.videoWidth, 
+                videoHeight: video_metadata.videoHeight, 
                 annotations: addRecord(state.outputAnnotation.annotations, annotation)
             }
         })),

diff --git a/src/stores/ObjectStore.js b/src/stores/ObjectStore.js
@@ -1,11 +1,29 @@
-import create from 'zustand';
+import create from "zustand";
 
-const useObjectStore = create(set => ({
-    objPosition: { x: 0, y: 0.2, z: 0 },
-    objRotation: { _x: Math.PI/2, _y: 0, _z: 0 },
-    handleTranslation: (newPosition) => set(state => ({ objPosition: {...newPosition} })),
-    handleRotation: (newRotation) => set(state => ({ objRotation: {_x: newRotation._x, _y: newRotation._y, _z: newRotation._z, } })),
-}))
+const useObjectStore = create((set) => ({
+  defaultObjects: [
+    { name: "Tomato Can", gltfFile: "/tomato_can.gltf" },
+    { name: "Banana", gltfFile: "/banana.gltf" },
+    { name: "Hammer", gltfFile: "/hammer.gltf" },
+  ],
+  objectSelected: { name: "Banana", gltfFile: "/banana.gltf" },
+  objPosition: { x: 0, y: 0.2, z: 0 },
+  objRotation: { _x: Math.PI / 2, _y: 0, _z: 0 },
+
+  handleSelectObject: (e) =>
+    set((state) => ({
+      objectSelected: state.defaultObjects.find((i) => i.name === e),
+    })),
+  handleTranslation: (newPosition) =>
+    set((state) => ({ objPosition: { ...newPosition } })),
+  handleRotation: (newRotation) =>
+    set((state) => ({
+      objRotation: {
+        _x: newRotation._x,
+        _y: newRotation._y,
+        _z: newRotation._z,
+      },
+    })),
+}));
 
-
-export { useObjectStore };
+export { useObjectStore };
diff --git a/src/stores/VideoStore.js b/src/stores/VideoStore.js
@@ -14,6 +14,8 @@ const useVideoStore = create(set => ({
     width: '100%',
     height: 'auto',
 
+    setVideoOnScene : (e) => set ( state => ({ url: e.target.value })),
+    setVideoDimensions : (e) => set (state => ({ videoHeight: e.wrapper.clientHeight, videoWidth: e.wrapper.clientWidth})),
     handleToggleLoop : () => set ( state => ({ loop: !state.loop})),
     handleVideoPlayPause : () => set (state => ({ playing : !state.playing })),
     handleVideoRewind : () => set (state => ({ playing: false, played: 0})),