diff --git a/examples/1_pipeline.py b/examples/1_pipeline.py new file mode 100644 index 0000000..2593b51 --- /dev/null +++ b/examples/1_pipeline.py @@ -0,0 +1,49 @@ +''' Simple example of pipeline +3D obj(process) --> 2d image +''' +import os, sys +import numpy as np +import scipy.io as sio +from skimage import io +from time import time +import matplotlib.pyplot as plt + +sys.path.append('..') +import face3d +from face3d import mesh +from face3d import mesh_cython + +# ------------------------------ 1. load mesh data +C = sio.loadmat('Data/example1.mat') +vertices = C['vertices']; colors = C['colors']; triangles = C['triangles'] +colors = colors/np.max(colors) + +# ------------------------------ 2. modify vertices(transformation. change position of obj) +# scale. target size=200 for example +s = 200/(np.max(vertices[:,1]) - np.min(vertices[:,1])) +# rotate 30 degree for example +R = mesh.transform.angle2matrix([0, 30, 0]) +# no translation. center of obj:[0,0] +t = [0, 0, 0] +transformed_vertices = mesh.transform.similarity_transform(vertices, s, R, t) + +# ------------------------------ 3. modify colors/texture(add light) +# set lights +light_positions = np.array([[-128, -128, 300]]) +light_intensities = np.array([[1, 1, 1]]) +lit_colors = mesh.light.add_light(transformed_vertices, triangles, colors, light_positions, light_intensities) + +# ------------------------------ 4. modify vertices(projection. change position of camera) +projected_vertices = mesh.transform.lookat_camera(transformed_vertices, eye = [0, 0, 200], at = np.array([0, 0, 0]), up = None) + +# ------------------------------ 5. render(to 2d image) +# set h, w of rendering +h = w = 256 +# change to image coords for rendering +image_vertices = mesh.transform.to_image(projected_vertices, h, w) +rendering = face3d.mesh_cython.render.render_colors(image_vertices, triangles, lit_colors, h, w) + +# ---- show +plt.imshow(rendering) +plt.show() + diff --git a/examples/2_3dmm.py b/examples/2_3dmm.py new file mode 100644 index 0000000..9616be2 --- /dev/null +++ b/examples/2_3dmm.py @@ -0,0 +1,69 @@ +''' 3d morphable model example +3dmm parameters --> mesh (and inverse) +''' +import os, sys +import numpy as np +import scipy.io as sio +from skimage import io +from time import time +import matplotlib.pyplot as plt + +sys.path.append('..') +import face3d +from face3d import mesh +from face3d import mesh_cython +from face3d.morphable_model import MorphabelModel + +# --------------------- Forward: parameters(shape, expression, pose) --> 3D obj --> 2D image --------------- +# --- 1. load model +bfm = MorphabelModel('Data/BFM/Out/BFM.mat') +print('init bfm model success') + +# --- 2. generate face mesh: vertices(represent shape) & colors(represent texture) +sp = bfm.get_shape_para('random') +ep = bfm.get_exp_para('random') +vertices = bfm.generate_vertices(sp, ep) + +tp = bfm.get_tex_para('random') +colors = bfm.generate_colors(tp) +colors = np.minimum(np.maximum(colors, 0), 1) + +# --- 3. transform vertices to proper position +s = 1e-03 +angles = [10, 30, 20] +t = [0, 0, 0] +transformed_vertices = bfm.transform(vertices, s, angles, t) +projected_vertices = transformed_vertices.copy() # using stantard camera & orth projection + +# --- 4. render(3d obj --> 2d image) +# set prop of rendering +h = w = 256; c = 3 +image_vertices = mesh.transform.to_image(projected_vertices, h, w) +image = mesh_cython.render.render_colors(image_vertices, bfm.triangles, colors, h, w) + +# -------------------- Back: 2D image points and corresponding 3D vertex indices--> parameters(pose, shape, expression) ------ +## only use 68 key points to fit +x = projected_vertices[bfm.kpt_ind, :2] # 2d keypoint, which can be detected from image +X_ind = bfm.kpt_ind # index of keypoints in 3DMM. fixed. + +# fit +fitted_sp, fitted_ep, fitted_s, fitted_angles, fitted_t = bfm.fit(x, X_ind, max_iter = 4) + +# verify fitted parameters +fitted_vertices = bfm.generate_vertices(fitted_sp, fitted_ep) +transformed_vertices = bfm.transform(fitted_vertices, fitted_s, fitted_angles, fitted_t) + +image_vertices = mesh.transform.to_image(transformed_vertices, h, w) +fitted_image = mesh_cython.render.render_colors(image_vertices, bfm.triangles, colors, h, w) + + +# ------------- print & show +print('pose, groudtruth: \n', s, angles[0], angles[1], angles[2], t[0], t[1]) +print('pose, fitted: \n', fitted_s, fitted_angles[0], fitted_angles[1], fitted_angles[2], fitted_t[0], fitted_t[1]) + +save_folder = 'results/3dmm' +if not os.path.exists(save_folder): + os.mkdir(save_folder) + +io.imsave('{}/generated.jpg'.format(save_folder), image) +io.imsave('{}/fitted.jpg'.format(save_folder), fitted_image) diff --git a/examples/3_transform.py b/examples/3_transform.py new file mode 100644 index 0000000..926e34e --- /dev/null +++ b/examples/3_transform.py @@ -0,0 +1,136 @@ +''' Examples of transformation & camera model. +''' +import os, sys +import numpy as np +import math +import scipy.io as sio +from skimage import io +from time import time +import subprocess + +sys.path.append('..') +import face3d +from face3d import mesh +from face3d import mesh_cython + + +def transform_test(vertices, obj, camera, h = 256, w = 256): + ''' + Args: + obj: dict contains obj transform paras + camera: dict contains camera paras + ''' + R = mesh.transform.angle2matrix(obj['angles']) + transformed_vertices = mesh.transform.similarity_transform(vertices, obj['s'], R, obj['t']) + + if camera['proj_type'] == 'orthographic': + projected_vertices = transformed_vertices + image_vertices = mesh.transform.to_image(projected_vertices, h, w) + else: + + ## world space to camera space. (Look at camera.) + projected_vertices = mesh.transform.lookat_camera(transformed_vertices, camera['eye'], camera['at'], camera['up']) + ## camera space to image space. (Projection) if orth project, ignore + projected_vertices = mesh.transform.perspective_project(projected_vertices, camera['fovy'], near = camera['near'], far = camera['far']) + ## to image coords(position in image) + image_vertices = mesh.transform.to_image(projected_vertices, h, w, True) + + rendering = mesh_cython.render.render_colors(image_vertices, triangles, colors, h, w) + rendering = np.minimum((np.maximum(rendering, 0)), 1) + return rendering + +# --------- load mesh data +C = sio.loadmat('Data/example1.mat') +vertices = C['vertices']; +global colors +global triangles +colors = C['colors']; triangles = C['triangles'] +colors = colors/np.max(colors) +# move center to [0,0,0] +vertices = vertices - np.mean(vertices, 0)[np.newaxis, :] + +# save folder +save_folder = 'results/transform' +if not os.path.exists(save_folder): + os.mkdir(save_folder) +options = '-delay 10 -loop 0 -layers optimize' # gif options. need ImageMagick installed. + +# ---- start +obj = {} +camera = {} +### face in reality: ~18cm height/width. set 180 = 18cm. image size: 256 x 256 +scale_init = 180/(np.max(vertices[:,1]) - np.min(vertices[:,1])) # scale face model to real size + +## 1. fix camera model(stadard camera& orth proj). change obj position. +camera['proj_type'] = 'orthographic' +# scale +for factor in np.arange(0.5, 1.2, 0.1): + obj['s'] = scale_init*factor + obj['angles'] = [0, 0, 0] + obj['t'] = [0, 0, 0] + image = transform_test(vertices, obj, camera) + io.imsave('{}/1_1_{:.2f}.jpg'.format(save_folder, factor), image) + +# angles +for i in range(3): + for angle in np.arange(-50, 51, 10): + obj['s'] = scale_init + obj['angles'] = [0, 0, 0] + obj['angles'][i] = angle + obj['t'] = [0, 0, 0] + image = transform_test(vertices, obj, camera) + io.imsave('{}/1_2_{}_{}.jpg'.format(save_folder, i, angle), image) +subprocess.call('convert {} {}/1_*.jpg {}'.format(options, save_folder, save_folder + '/obj.gif'), shell=True) + +## 2. fix obj position(center=[0,0,0], front pose). change camera position&direction, using perspective proj(fovy fixed) +obj['s'] = scale_init +obj['angles'] = [0, 0, 0] +obj['t'] = [0, 0, 0] +# obj: center at [0,0,0]. size:200 + +camera['proj_type'] = 'perspective' +camera['at'] = [0, 0, 0] +camera['near'] = 1000 +camera['far'] = -100 + +# eye position +camera['fovy'] = 30 +camera['up'] = [0, 1, 0] # +# z-axis: eye from far to near, looking at the center of face +for p in np.arange(500, 250-1, -40): # 0.5m->0.25m + camera['eye'] = [0, 0, p] # stay in front of face + image = transform_test(vertices, obj, camera) + io.imsave('{}/2_eye_1_{}.jpg'.format(save_folder, 1000-p), image) + +# y-axis: eye from down to up, looking at the center of face +for p in np.arange(-300, 301, 60): # up 0.3m -> down 0.3m + camera['eye'] = [0, p, 250] # stay 0.25m far + image = transform_test(vertices, obj, camera) + io.imsave('{}/2_eye_2_{}.jpg'.format(save_folder, p/6), image) + +# x-axis: eye from left to right, looking at the center of face +for p in np.arange(-300, 301, 60): # left 0.3m -> right 0.3m + camera['eye'] = [p, 0, 250] # stay 0.25m far + image = transform_test(vertices, obj, camera) + io.imsave('{}/2_eye_3_{}.jpg'.format(save_folder, -p/6), image) + +# up direction +camera['eye'] = [0, 0, 250] # stay in front +for p in np.arange(-50, 51, 10): + world_up = np.array([0, 1, 0]) # default direction + z = np.deg2rad(p) + Rz=np.array([[math.cos(z), -math.sin(z), 0], + [math.sin(z), math.cos(z), 0], + [ 0, 0, 1]]) + up = Rz.dot(world_up[:, np.newaxis]) # rotate up direction + # note that: rotating up direction is opposite to rotating obj + # just imagine: rotating camera 20 degree clockwise, is equal to keeping camera fixed and rotating obj 20 degree anticlockwise. + camera['up'] = np.squeeze(up) + image = transform_test(vertices, obj, camera) + io.imsave('{}/2_eye_4_{}.jpg'.format(save_folder, -p), image) + +subprocess.call('convert {} {}/2_*.jpg {}'.format(options, save_folder, save_folder + '/camera.gif'), shell=True) + +# -- delete jpg files +print('gifs have been generated, now delete jpgs') +subprocess.call('rm {}/*.jpg'.format(save_folder), shell=True) diff --git a/examples/4_light.py b/examples/4_light.py new file mode 100644 index 0000000..eaf4887 --- /dev/null +++ b/examples/4_light.py @@ -0,0 +1,77 @@ +''' +test light +''' +import os, sys +import numpy as np +import scipy.io as sio +from skimage import io +from time import time +import subprocess + +sys.path.append('..') +import face3d +from face3d import mesh +from face3d import mesh_cython + + +def light_test(vertices, light_positions, light_intensities, h = 256, w = 256): + lit_colors = mesh_cython.light.add_light(vertices, triangles, colors, light_positions, light_intensities) + image_vertices = mesh.transform.to_image(vertices, h, w) + rendering = mesh_cython.render.render_colors(image_vertices, triangles, lit_colors, h, w) + rendering = np.minimum((np.maximum(rendering, 0)), 1) + return rendering + +# --------- load mesh data +C = sio.loadmat('Data/example1.mat') +vertices = C['vertices']; +global colors +global triangles +colors = C['colors']; triangles = C['triangles'] +colors = colors/np.max(colors) +# move center to [0,0,0] +vertices = vertices - np.mean(vertices, 0)[np.newaxis, :] +s = 200/(np.max(vertices[:,1]) - np.min(vertices[:,1])) +R = mesh.transform.angle2matrix([0, 0, 0]) +t = [0, 0, 0] +vertices = mesh.transform.similarity_transform(vertices, s, R, t) + + +save_folder = 'results/light' +if not os.path.exists(save_folder): + os.mkdir(save_folder) +options = '-delay 12 -loop 0 -layers optimize' # gif. need ImageMagick. + +# ---- start +# 1. fix light intensities. change light positions. +# x axis: light from left to right +light_intensities = np.array([[1, 1, 1]]) +for i,p in enumerate(range(-200, 201, 40)): + light_positions = np.array([[p, 0, 300]]) + image = light_test(vertices, light_positions, light_intensities) + io.imsave('{}/1_1_{:0>2d}.jpg'.format(save_folder, i), image) +# y axis: light from up to down +for i,p in enumerate(range(200, -201, -40)): + light_positions = np.array([[0, p, 300]]) + image = light_test(vertices, light_positions, light_intensities) + io.imsave('{}/1_2_{:0>2d}.jpg'.format(save_folder, i), image) +# z axis: light near down to far +for i,p in enumerate(range(100, 461, 40)): + light_positions = np.array([[0, 0, p]]) + image = light_test(vertices, light_positions, light_intensities) + io.imsave('{}/1_3_{:0>2d}.jpg'.format(save_folder, i), image) +subprocess.call('convert {} {}/1_*.jpg {}'.format(options, save_folder, save_folder + '/position.gif'), shell=True) + + +# 2. fix light positions. change light intensities. +light_positions = np.array([[0, 0, 300]]) +for k in range(3): + for i,p in enumerate(np.arange(0.4,1.1,0.2)): + light_intensities = np.array([[0, 0, 0]], dtype = np.float32) + light_intensities[0,k] = p + image = light_test(vertices, light_positions, light_intensities) + io.imsave('{}/2_{}_{:0>2d}.jpg'.format(save_folder, k, i), image) +subprocess.call('convert {} {}/2_*.jpg {}'.format(options, save_folder, save_folder + '/intensity.gif'), shell=True) + +# -- delete jpg files +print('gifs have been generated, now delete jpgs') +subprocess.call('rm {}/*.jpg'.format(save_folder), shell=True) diff --git a/examples/5_render.py b/examples/5_render.py new file mode 100644 index 0000000..88cbf1e --- /dev/null +++ b/examples/5_render.py @@ -0,0 +1,56 @@ +''' Test render speed. +''' +import os, sys +import numpy as np +import scipy.io as sio +from skimage import io +from time import time +import matplotlib.pyplot as plt + +np.set_printoptions(suppress=True) + +sys.path.append('..') +import face3d +from face3d import mesh +from face3d import mesh_cython +from face3d.morphable_model import MorphabelModel + +# load data +C = sio.loadmat('Data/example1.mat') +vertices = C['vertices']; colors = C['colors']; triangles = C['triangles'] +colors = colors/np.max(colors) +# move center to [0,0,0] +vertices = vertices - np.mean(vertices, 0)[np.newaxis, :] +s = 200/(np.max(vertices[:,1]) - np.min(vertices[:,1])) +R = mesh.transform.angle2matrix([0, 0, 0]) +t = [0, 0, 0] +vertices = mesh.transform.similarity_transform(vertices, s, R, t) +# h, w of rendering +h = w = 256 +image_vertices = mesh.transform.to_image(vertices, h, w) + +# -----------------------------------------render +# # render texture python +# st = time() +# rendering_tp = face3d.mesh.render.render_texture(vertices, triangles, texture, texcoord, triangles, h, w) +# print('----------texture python: ', time() - st) + +# # render texture c++ +# st = time() +# rendering_tc = face3d.mesh_cython.render.render_texture(vertices, triangles, texture, texcoord, triangles, h, w) +# print('----------texture c++: ', time() - st) + +# render colors python +st = time() +rendering_cp = face3d.mesh.render.render_colors(image_vertices, triangles, colors, h, w) +print('----------colors python: ', time() - st) + +# render colors python ras +st = time() +rendering_cpr = face3d.mesh.render.render_colors_ras(image_vertices, triangles, colors, h, w) +print('----------colors python ras: ', time() - st) + +# render colors python c++ +st = time() +rendering_cc = face3d.mesh_cython.render.render_colors(image_vertices, triangles, colors, h, w) +print('----------colors c++: ', time() - st) \ No newline at end of file diff --git a/examples/6_generate_image_map.py b/examples/6_generate_image_map.py new file mode 100644 index 0000000..7ff7aa5 --- /dev/null +++ b/examples/6_generate_image_map.py @@ -0,0 +1,68 @@ +''' +Generate 2d maps representing different attributes(colors, depth, pncc, etc) +: render attributes to image space. +''' +import os, sys +import numpy as np +import scipy.io as sio +from skimage import io +from time import time +import matplotlib.pyplot as plt + +sys.path.append('..') +import face3d +from face3d import mesh +from face3d import mesh_cython + +# ------------------------------ load mesh data +C = sio.loadmat('Data/example1.mat') +vertices = C['vertices']; colors = C['colors']; triangles = C['triangles'] +colors = colors/np.max(colors) + +# ------------------------------ modify vertices(transformation. change position of obj) +# scale. target size=200 for example +s = 200/(np.max(vertices[:,1]) - np.min(vertices[:,1])) +# rotate 30 degree for example +R = mesh.transform.angle2matrix([0, 30, 0]) +# no translation. center of obj:[0,0] +t = [0, 0, 0] +transformed_vertices = mesh.transform.similarity_transform(vertices, s, R, t) + +# ------------------------------ render(to 2d image) +# set h, w of rendering +h = w = 256 +# change to image coords for rendering +image_vertices = mesh.transform.to_image(transformed_vertices, h, w) + +## --- start +save_folder = 'results/image_map' +if not os.path.exists(save_folder): + os.mkdir(save_folder) + +## 0. color map +attribute = colors +color_image = mesh_cython.render.render_colors(image_vertices, triangles, attribute, h, w, c=3) +io.imsave('{}/color.jpg'.format(save_folder), np.squeeze(color_image)) + +## 1. depth map +z = image_vertices[:,2:] +z = z - np.min(z) +z = z/np.max(z) +attribute = z +depth_image = mesh_cython.render.render_colors(image_vertices, triangles, attribute, h, w, c=1) +io.imsave('{}/depth.jpg'.format(save_folder), np.squeeze(depth_image)) + +## 2. pncc in 'Face Alignment Across Large Poses: A 3D Solution'. for dense correspondences +pncc = face3d.morphable_model.load.load_pncc_code('Data/BFM/Out/pncc_code.mat') +attribute = pncc +pncc_image = mesh_cython.render.render_colors(image_vertices, triangles, attribute, h, w, c=3) +io.imsave('{}/pncc.jpg'.format(save_folder), np.squeeze(pncc_image)) + +## 3. uv coordinates in 'DenseReg: Fully convolutional dense shape regression in-the-wild'. for dense correspondences +uv_coords = face3d.morphable_model.load.load_uv_coords('Data/BFM/Out/BFM_UV.mat') # +attribute = uv_coords # note that: original paper used quantized coords, here not +uv_coords_image = mesh_cython.render.render_colors(image_vertices, triangles, attribute, h, w, c=2) # two channels: u, v +# add one channel for show +uv_coords_image = np.concatenate((np.zeros((h, w, 1)), uv_coords_image), 2) +io.imsave('{}/uv_coords.jpg'.format(save_folder), np.squeeze(uv_coords_image)) + diff --git a/examples/7_generate_uv_map.py b/examples/7_generate_uv_map.py new file mode 100644 index 0000000..244d892 --- /dev/null +++ b/examples/7_generate_uv_map.py @@ -0,0 +1,78 @@ +''' +Generate 2d uv maps representing different attributes(colors, depth, image position, etc) +: render attributes to uv space. +''' +import os, sys +import numpy as np +import scipy.io as sio +from skimage import io +import skimage.transform +from time import time +import matplotlib.pyplot as plt + +sys.path.append('..') +import face3d +from face3d import mesh +from face3d import mesh_cython +from face3d.morphable_model import MorphabelModel + +def process_uv(uv_coords, uv_h = 256, uv_w = 256): + uv_coords[:,0] = uv_coords[:,0]*(uv_h - 1) + uv_coords[:,1] = uv_coords[:,1]*(uv_h - 1) + uv_coords[:,1] = uv_h - uv_coords[:,1] - 1 + uv_coords = np.hstack((uv_coords, np.zeros((uv_coords.shape[0], 1)))) # add z + return uv_coords + + +# --load mesh data +C = sio.loadmat('Data/example1.mat') +vertices = C['vertices']; colors = C['colors']; triangles = C['full_triangles']; +colors = colors/np.max(colors) +# --modify vertices(transformation. change position of obj) +s = 200/(np.max(vertices[:,1]) - np.min(vertices[:,1])) +R = mesh.transform.angle2matrix([0, 30, 0]) +t = [0, 0, 0] +transformed_vertices = mesh.transform.similarity_transform(vertices, s, R, t) +# --load uv coords +uv_coords = face3d.morphable_model.load.load_uv_coords('Data/BFM/Out/BFM_UV.mat') # + +# -- start +save_folder = 'results/uv_map' +if not os.path.exists(save_folder): + os.mkdir(save_folder) + +uv_h = uv_w = 256 +image_h = image_w = 256 +uv_coords = process_uv(uv_coords, uv_h, uv_w) + +#-- 1. uv texture map +attribute = colors +uv_texture_map = mesh_cython.render.render_colors(uv_coords, triangles, attribute, uv_h, uv_w, c=3) +io.imsave('{}/uv_texture_map.jpg'.format(save_folder), np.squeeze(uv_texture_map)) + +#-- 2. uv position map in 'Joint 3D Face Reconstruction and Dense Alignment with Position Map Regression Network' +#-- for face reconstruction & alginment(dense correspondences) +# To some extent, when uv space is regular, position map is a subclass of geometry image(recording geometry information in regular image) +# Notice: position map doesn't exit alone, it depends on the corresponding rendering(2d facical image). +# Attribute is position(with respect to image space, be careful when using perpestive projection) +image_vertices = mesh.transform.to_image(transformed_vertices, image_h, image_w) # use orth projection here +position = image_vertices.copy() +position[:,2] = position[:,2] - np.min(position[:,2]) # translate z +attribute = position +# corresponding 2d facial image +image = mesh_cython.render.render_colors(image_vertices, triangles, colors, image_h, image_w, c=3) +uv_position_map = mesh_cython.render.render_colors(uv_coords, triangles, attribute, uv_h, uv_w, c=3) +io.imsave('{}/image.jpg'.format(save_folder), np.squeeze(image)) +np.save('{}/uv_position_map.npy'.format(save_folder), uv_position_map) +io.imsave('{}/uv_position_map.jpg'.format(save_folder), (uv_position_map)/max(image_h, image_w)) # only for show + +# - verify +# import cv2 +# uv_texture_map_rec = cv2.remap(image, uv_position_map[:,:,:2].astype(np.float32), None, interpolation=cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT,borderValue=(0)) +# io.imsave('{}/uv_texture_map_rec.jpg'.format(save_folder), np.squeeze(uv_texture_map_rec)) + +#-- 3. general geometry image. attribute = vertices/transformed_vertices +# TODO +#-- 4. attribute = normals +# TODO + diff --git a/examples/8_generate_posmap_300WLP.py b/examples/8_generate_posmap_300WLP.py new file mode 100644 index 0000000..f60109a --- /dev/null +++ b/examples/8_generate_posmap_300WLP.py @@ -0,0 +1,111 @@ +''' +Generate uv position map of 300W_LP. +''' +import os, sys +import numpy as np +import scipy.io as sio +from skimage import io +import skimage.transform +from time import time +import matplotlib.pyplot as plt + +sys.path.append('..') +import face3d +from face3d import mesh +from face3d import mesh_cython +from face3d.morphable_model import MorphabelModel + +def process_uv(uv_coords, uv_h = 256, uv_w = 256): + uv_coords[:,0] = uv_coords[:,0]*(uv_h - 1) + uv_coords[:,1] = uv_coords[:,1]*(uv_h - 1) + uv_coords[:,1] = uv_h - uv_coords[:,1] - 1 + uv_coords = np.hstack((uv_coords, np.zeros((uv_coords.shape[0], 1)))) # add z + return uv_coords + +def run_posmap_300W_LP(image_path, save_folder, uv_h = 256, uv_w = 256, image_h = 256, image_w = 256): + # 1. load image and fitted parameters + image_name = image_path.strip().split('/')[-1] + image = io.imread(image_path)/255. + [h, w, c] = image.shape + + mat_path = image_path.replace('jpg', 'mat') + info = sio.loadmat(mat_path) + pose_para = info['Pose_Para'].T.astype(np.float32) + shape_para = info['Shape_Para'].astype(np.float32) + exp_para = info['Exp_Para'].astype(np.float32) + + # 2. generate mesh + # load bfm + bfm = MorphabelModel('Data/BFM/Out/BFM.mat') + # generate shape + vertices = bfm.generate_vertices(shape_para, exp_para) + # transform mesh + s = pose_para[-1, 0] + angles = pose_para[:3, 0] + t = pose_para[3:6, 0] + transformed_vertices = bfm.transform_3ddfa(vertices, s, angles, t) + projected_vertices = transformed_vertices.copy() # using stantard camera & orth projection as in 3DDFA + image_vertices = projected_vertices.copy() + image_vertices[:,1] = h - image_vertices[:,1] - 1 + + # 3. crop image with key points + kpt = image_vertices[bfm.kpt_ind, :].astype(np.int32) + left = np.min(kpt[:, 0]) + right = np.max(kpt[:, 0]) + top = np.min(kpt[:, 1]) + bottom = np.max(kpt[:, 1]) + center = np.array([right - (right - left) / 2.0, + bottom - (bottom - top) / 2.0]) + old_size = (right - left + bottom - top)/2 + size = int(old_size*1.5) + # random pertube + marg = old_size*0.1 + t_x = np.random.rand()*marg*2 - marg + t_y = np.random.rand()*marg*2 - marg + center[0] = center[0]+t_x; center[1] = center[1]+t_y + size = size*(np.random.rand()*0.2 + 0.9) + + src_pts = np.array([[center[0]-size/2, center[1]-size/2], [center[0] - size/2, center[1]+size/2], [center[0]+size/2, center[1]-size/2]]) + DST_PTS = np.array([[0, 0], [0, image_h - 1], [image_w - 1, 0]]) + tform = skimage.transform.estimate_transform('similarity', src_pts, DST_PTS) + cropped_image = skimage.transform.warp(image, tform.inverse, output_shape=(image_h, image_w)) + + # transform face position(image vertices) along with 2d facial image + position = image_vertices.copy() + position[:, 2] = 1 + position = np.dot(position, tform.params.T) + position[:, 2] = projected_vertices[:, 2]*tform.params[0, 0] # scale z + position[:, 2] = position[:, 2] - np.min(position[:, 2]) # translate z + + # 4. uv position map: render position to uv space + uv_position_map = mesh_cython.render.render_colors(uv_coords, bfm.full_triangles, position, uv_h, uv_w, c = 3) + + # 5. save files + io.imsave('{}/{}'.format(save_folder, image_name), np.squeeze(cropped_image)) + np.save('{}/{}'.format(save_folder, image_name.replace('jpg', 'npy')), uv_position_map) + io.imsave('{}/{}'.format(save_folder, image_name.replace('.jpg', '_posmap.jpg')), (uv_position_map)/max(image_h, image_w)) # only for show + + # --verify + # import cv2 + # uv_texture_map_rec = cv2.remap(cropped_image, uv_position_map[:,:,:2].astype(np.float32), None, interpolation=cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT,borderValue=(0)) + # io.imsave('{}/{}'.format(save_folder, image_name.replace('.jpg', '_tex.jpg')), np.squeeze(uv_texture_map_rec)) + + +if __name__ == '__main__': + save_folder = 'results/posmap_300WLP' + if not os.path.exists(save_folder): + os.mkdir(save_folder) + + # set para + uv_h = uv_w = 256 + image_h = image_w = 256 + + # load uv coords + global uv_coords + uv_coords = face3d.morphable_model.load.load_uv_coords('Data/BFM/Out/BFM_UV.mat') # + uv_coords = process_uv(uv_coords, uv_h, uv_w) + + # run + image_path = 'Data/IBUG_image_008_1_0.jpg' + run_posmap_300W_LP(image_path, save_folder) +