smooth 3d trajectories, add visualizations

Signed-off-by: Konstantinos Rematas <[email protected]>

demo/estimate_poses.py

@@ -14,7 +14,7 @@
 db.digest_metadata()
 
 db.get_boxes_from_detectron()
-# db.dump_video('detections')
+db.dump_video('detections')
 
 db.estimate_poses(openpose_dir=opt.openpose_dir)
 db.refine_poses(keypoint_thresh=7, score_thresh=0.4, neck_thresh=0.4)

demo/generate_mesh.py

@@ -0,0 +1,114 @@
+import argparse
+import soccer3d
+import json
+from os.path import join
+from tqdm import tqdm
+import utils.camera as cam_utils
+import utils.io as io
+import utils.mesh as mesh_utils
+import utils.misc as misc_utils
+import utils.files as file_utils
+import openmesh as om
+import numpy as np
+import cv2
+
+
+parser = argparse.ArgumentParser(description='Calibrate a soccer video')
+parser.add_argument('--path_to_data', default='/home/krematas/Mountpoints/grail/data/barcelona', help='path')
+parser.add_argument('--decimate_to', type=int, default=500, help='Margin around the pose')
+opt, _ = parser.parse_known_args()
+
+
+with open(join(opt.path_to_data, 'players', 'metadata', 'position.json')) as f:
+    data = json.load(f)
+
+
+db = soccer3d.YoutubeVideo(opt.path_to_data)
+db.digest_metadata()
+
+db.refine_poses(keypoint_thresh=7, score_thresh=0.4, neck_thresh=0.4)
+
+file_utils.mkdir(join(db.path_to_dataset, 'scene3d'))
+
+for sel_frame in tqdm(range(db.n_frames)):
+    img = db.get_frame(sel_frame)
+    basename = db.frame_basenames[sel_frame]
+
+    cam_data = db.calib[basename]
+    cam = cam_utils.Camera(basename, cam_data['A'], cam_data['R'], cam_data['T'], db.shape[0], db.shape[1])
+
+    player_list = data[basename]
+
+    frame_mesh_points = np.zeros((0, 3))
+    frame_mesh_faces = np.zeros((0, 3))
+
+    uvs_atlas = []
+    textures_atlas = []
+
+    cnt = 0
+    for p in range(len(player_list)):
+        mesh = om.PolyMesh()
+
+        ply_name = join(opt.path_to_data, 'players', 'meshes', player_list[p]['mesh']+'.ply')
+        vertex_data, face_data, _, _ = io.read_ply(ply_name)
+
+        pointcloud, pc_colors, _ = io.ply_to_numpy(vertex_data)
+        pc_centered = pointcloud - np.mean(pointcloud, axis=0)
+
+        points2d, depth = cam.project(pointcloud)
+        faces = mesh_utils.triangulate_depthmap_points(points2d, depth, depth_thresh=0.1)
+
+        vertex_handle = []
+        for i in range(pointcloud.shape[0]):
+            vh = mesh.add_vertex(pointcloud[i, :])
+            vertex_handle.append(vh)
+
+        face_handle = []
+        for i in range(len(faces)):
+            fh0 = mesh.add_face(vertex_handle[faces[i][0]], vertex_handle[faces[i][1]], vertex_handle[faces[i][2]])
+            face_handle.append(fh0)
+
+        d = om.PolyMeshDecimater(mesh)
+        mh = om.PolyMeshModQuadricHandle()
+
+        # add modules
+        d.add(mh)
+        d.module(mh).set_max_err(0.001)
+
+        # decimate
+        d.initialize()
+        d.decimate_to(opt.decimate_to)
+
+        mesh.garbage_collection()
+
+        _vertices = mesh.points()
+        _faces = mesh.face_vertex_indices()
+
+        uv, _ = cam.project(_vertices, dtype=np.float32)
+        _vertices -= np.mean(_vertices, axis=0)
+        _vertices[:, 0] += player_list[p]['x']
+        # _vertices[:, 1] = _vertices[:, 1] + pc_center[1] - np.min(_vertices[:, 1])
+        _vertices[:, 2] += player_list[p]['z']
+
+        min_u, max_u = int(np.min(np.round(uv[:, 0]))), int(np.max(np.round(uv[:, 0]))) + 1
+        min_v, max_v = int(np.min(np.round(uv[:, 1]))), int(np.max(np.round(uv[:, 1]))) + 1
+        crop = img[min_v:max_v, min_u:max_u, :]
+
+        tmp = uv.copy()
+        tmp[:, 0] -= np.min(np.round(uv[:, 0]))
+        tmp[:, 1] -= np.min(np.round(uv[:, 1]))
+
+        uvs_atlas.append(tmp)
+        textures_atlas.append(crop)
+
+        frame_mesh_points = np.vstack((frame_mesh_points, _vertices))
+        frame_mesh_faces = np.vstack((frame_mesh_faces, _faces + cnt))
+        cnt += _vertices.shape[0]
+
+    atlas, final_uvs = misc_utils.pack_textures(textures_atlas, uvs_atlas, n_rows=1)
+
+    texture_name = join(db.path_to_dataset, 'scene3d', '{0}.jpg'.format(basename))
+    cv2.imwrite(join(texture_name), atlas[:, :, (2, 1, 0)] * 255)
+
+    ply_out = io.numpy_to_ply(frame_mesh_points)
+    io.write_obj(join(db.path_to_dataset, 'scene3d', '{0}.obj'.format(basename)), ply_out, list(frame_mesh_faces), final_uvs, texture_name)

demo/simple_visualization.py

@@ -0,0 +1,235 @@
+import numpy as np
+from glumpy import app, gl, gloo, data
+from glumpy.transforms import Trackball, Position
+import utils.io as io
+import utils.files as file_utils
+from os.path import join
+import time
+import argparse
+
+W, H = 104.73, 67.74
+
+
+def objload(filename):
+    V = []  # vertex
+    T = []  # texcoords
+    N = []  # normals
+    F = []  # face indices
+    F_V = []  # face indices
+    F_T = []  # face indices
+    F_N = []  # face indices
+
+    for lineno, line in enumerate(open(filename)):
+        if line[0] == '#':
+            continue
+        values = line.strip().split(' ')
+        code = values[0]
+        values = values[1:]
+        # vertex (v)
+        if code == 'v':
+            V.append([float(x) for x in values])
+        # tex-coord (vt)
+        elif code == 'vt':
+            T.append([float(x) for x in values])
+        # normal (n)
+        elif code == 'vn':
+            N.append([float(x) for x in values])
+        # face (f)
+        elif code == 'f':
+            if len(values) != 3:
+                raise ValueError('not a triangle at line' % lineno)
+            for v in values:
+                for j, index in enumerate(v.split('/')):
+                    if len(index):
+                        if j == 0:
+                            F_V.append(int(index) - 1)
+                        elif j == 1:
+                            F_T.append(int(index) - 1)
+                        elif j == 2:
+                            F_N.append(int(index) - 1)
+
+    # Building the vertices
+    V = np.array(V)
+    F_V = np.array(F_V)
+    vtype = [('position', np.float32, 3)]
+
+    if len(T):
+        T = np.array(T)
+        F_T = np.array(F_T)
+        vtype.append(('texcoord', np.float32, 2))
+    if len(N):
+        N = np.array(N)
+        F_N = np.array(F_N)
+        vtype.append(('normal', np.float32, 3))
+
+    vertices = np.empty(len(F_V), vtype)
+    vertices["position"] = V[F_V]
+    if len(T):
+        vertices["texcoord"] = T[F_T]
+    if len(N):
+        vertices["normal"] = N[F_N]
+    vertices = vertices.view(gloo.VertexBuffer)
+
+    itype = np.uint32
+    indices = np.arange(len(vertices), dtype=np.uint32)
+    indices = indices.view(gloo.IndexBuffer)
+    return vertices, indices
+
+
+def plane():
+    vtype = [('position', np.float32, 3),
+             ('texcoord', np.float32, 2),
+             ('normal', np.float32, 3)]
+    itype = np.uint32
+
+    y_disp = -0.8
+    p = np.array([[-W / 2, y_disp, -H / 2], [-W / 2., y_disp, H / 2], [W / 2, y_disp, H / 2], [W / 2, y_disp, -H / 2], ], dtype=float)
+
+    n = np.array([[0, 1, 0]])
+    t = np.array([[0, 0], [0, 1], [1, 1], [1, 0]])
+
+    faces_p = [0, 1, 2, 3]
+    faces_n = [0, 0, 0, 0]
+    faces_t = [0, 1, 2, 3]
+
+    _vertices = np.zeros(4, vtype)
+    _vertices['position'] = p[faces_p]
+    _vertices['normal'] = n[faces_n]
+    _vertices['texcoord'] = t[faces_t]
+
+    filled = np.resize(np.array([0, 1, 2, 0, 2, 3], dtype=itype), 6 * (2 * 3))
+    filled += np.repeat(4 * np.arange(6, dtype=itype), 6)
+    _vertices = _vertices.view(gloo.VertexBuffer)
+    filled = filled.view(gloo.IndexBuffer)
+
+    return _vertices, filled
+
+
+vertex_tex = """
+attribute vec3 position;
+attribute vec2 texcoord;      // Vertex texture coordinates
+varying vec2   v_texcoord;      // Interpolated fragment texture coordinates (out)
+void main()
+{
+    v_texcoord  = texcoord;
+    gl_Position = <transform>;
+}
+"""
+
+fragment_tex = """
+uniform sampler2D u_texture;  // Texture 
+varying vec2      v_texcoord; // Interpolated fragment texture coordinates (in)
+void main()
+{
+    // Get texture color
+    vec4 t_color = texture2D(u_texture, v_texcoord);
+    // Final color
+    gl_FragColor = t_color;
+}
+"""
+
+parser = argparse.ArgumentParser(description='Track camera given an initial estimate')
+parser.add_argument('--path_to_data', default='/home/krematas/Mountpoints/grail/data/barcelona', help='path')
+parser.add_argument('--start', type=int, default=0, help='Starting frame')
+parser.add_argument('--end', type=int, default=-1, help='Ending frame')
+parser.add_argument('--fps', type=int, default=15, help='Ending frame')
+
+opt, _ = parser.parse_known_args()
+
+
+filenames = np.genfromtxt(join(opt.path_to_data, 'youtube.txt'), dtype=str)
+if opt.end == -1:
+    opt.end = len(filenames)
+filenames = filenames[opt.start:opt.end]
+
+fps = opt.fps
+n_frames = len(filenames)
+
+trackball = Trackball(Position("position"), aspect=1, theta=45, phi=45, distance=100, zoom=135)
+
+vertices_field, indices_field = plane()
+
+field = gloo.Program(vertex_tex, fragment_tex)
+field.bind(vertices_field)
+field['position'] = vertices_field
+field['u_texture'] = data.get(join(opt.path_to_data, 'texture.png'))
+field['transform'] = trackball
+
+all_programs = []
+
+for fid, fname in enumerate(filenames):
+
+    (basename, ext) = file_utils.extract_basename(fname)
+    print('Loading model {0}/{1}: {2}'.format(fid, len(filenames), basename))
+
+    path_to_pc = join(opt.path_to_data, 'scene3d')
+    img = io.imread(join(path_to_pc, '{0}.jpg'.format(basename)), dtype=np.float32)
+
+    vertices, indices = objload(join(path_to_pc, '{0}.obj'.format(basename)))
+    vertices['texcoord'][:, 1] = 1.0-vertices['texcoord'][:, 1]
+
+    tex_program = gloo.Program(vertex_tex, fragment_tex)
+    tex_program.bind(vertices)
+    tex_program['u_texture'] = img
+    tex_program['transform'] = trackball
+
+    all_programs.append(tex_program)
+
+trackball.theta, trackball.phi, trackball.zoom = -10, 0, 15
+
+
+window = app.Window(width=512, height=512, color=(0.30, 0.30, 0.35, 1.00))
+
+time_counter = 0
+play_or_pause = 0
+
+
+@window.event
+def on_draw(dt):
+
+    global time_counter, n_frames, play_or_pause
+    window.clear()
+    field.draw(gl.GL_TRIANGLES, indices_field)
+    all_programs[time_counter].draw(gl.GL_TRIANGLES)
+
+    if time_counter == n_frames-1:
+        time_counter = -1
+
+    if play_or_pause:
+        time_counter += 1
+
+    time.sleep(1./fps)
+
+
+@window.event
+def on_init():
+    gl.glEnable(gl.GL_DEPTH_TEST)
+    gl.glEnable(gl.GL_BLEND)
+    gl.glBlendFunc(gl.GL_SRC_ALPHA, gl.GL_ONE_MINUS_SRC_ALPHA)
+
+
+@window.event
+def on_character(text):
+    global play_or_pause, time_counter, fps
+    if text == ' ':
+        if play_or_pause == 1:
+            play_or_pause = 0
+        else:
+            play_or_pause = 1
+    if text == '+':
+        fps += 1
+
+    if text == '-':
+        fps -= 1
+
+    if text == 'a':
+        time_counter -= 1
+
+    if text == 'd':
+        time_counter += 1
+
+    print(time_counter)
+
+
+window.attach(field['transform'])
+app.run()