From fd08f0464849666e01dd34f823f6d13ac85f7a5c Mon Sep 17 00:00:00 2001 From: YadiraF Date: Thu, 19 Jul 2018 09:30:08 +0800 Subject: [PATCH] add codes generating gif --- examples/2_3dmm.py | 24 ++++++++- face3d/morphable_model/fit.py | 62 ++++++++++++++++++++++- face3d/morphable_model/morphabel_model.py | 13 +++-- 3 files changed, 93 insertions(+), 6 deletions(-) diff --git a/examples/2_3dmm.py b/examples/2_3dmm.py index e0be9bb..ffe1e95 100644 --- a/examples/2_3dmm.py +++ b/examples/2_3dmm.py @@ -1,7 +1,9 @@ ''' 3d morphable model example -3dmm parameters --> mesh (and inverse) +3dmm parameters --> mesh +fitting: 2d image + 3dmm -> 3d face ''' import os, sys +import subprocess import numpy as np import scipy.io as sio from skimage import io @@ -47,7 +49,7 @@ X_ind = bfm.kpt_ind # index of keypoints in 3DMM. fixed. # fit -fitted_sp, fitted_ep, fitted_s, fitted_angles, fitted_t = bfm.fit(x, X_ind, max_iter = 4) +fitted_sp, fitted_ep, fitted_s, fitted_angles, fitted_t = bfm.fit(x, X_ind, max_iter = 3) # verify fitted parameters fitted_vertices = bfm.generate_vertices(fitted_sp, fitted_ep) @@ -67,3 +69,21 @@ io.imsave('{}/generated.jpg'.format(save_folder), image) io.imsave('{}/fitted.jpg'.format(save_folder), fitted_image) + + +### ----------------- visualize fitting process +# fit +fitted_sp, fitted_ep, fitted_s, fitted_angles, fitted_t = bfm.fit(x, X_ind, max_iter = 3, isShow = True) + +# verify fitted parameters +for i in range(fitted_sp.shape[0]): + fitted_vertices = bfm.generate_vertices(fitted_sp[i], fitted_ep[i]) + transformed_vertices = bfm.transform(fitted_vertices, fitted_s[i], fitted_angles[i], fitted_t[i]) + + image_vertices = mesh.transform.to_image(transformed_vertices, h, w) + fitted_image = mesh_cython.render.render_colors(image_vertices, bfm.triangles, colors, h, w) + io.imsave('{}/show_{:0>2d}.jpg'.format(save_folder, i), fitted_image) + +options = '-delay 20 -loop 0 -layers optimize' # gif. need ImageMagick. +subprocess.call('convert {} {}/show_*.jpg {}'.format(options, save_folder, save_folder + '/3dmm.gif'), shell=True) +subprocess.call('rm {}/show_*.jpg'.format(save_folder), shell=True) diff --git a/face3d/morphable_model/fit.py b/face3d/morphable_model/fit.py index 16aea86..c282fd4 100644 --- a/face3d/morphable_model/fit.py +++ b/face3d/morphable_model/fit.py @@ -209,4 +209,64 @@ def fit_points(x, X_ind, model, n_sp, n_ep, max_iter = 4): expression = np.reshape(expression, [int(len(expression)/3), 3]).T sp = estimate_shape(x, shapeMU, shapePC, model['shapeEV'][:n_sp,:], expression, s, R, t[:2], lamb = 40) - return sp, ep, s, R, t \ No newline at end of file + return sp, ep, s, R, t + + +# ---------------- fit +def fit_points_for_show(x, X_ind, model, n_sp, n_ep, max_iter = 4): + ''' + Args: + x: (n, 2) image points + X_ind: (n,) corresponding Model vertex indices + model: 3DMM + max_iter: iteration + Returns: + sp: (n_sp, 1). shape parameters + ep: (n_ep, 1). exp parameters + s, R, t + ''' + x = x.copy().T + + #-- init + sp = np.zeros((n_sp, 1), dtype = np.float32) + ep = np.zeros((n_ep, 1), dtype = np.float32) + + #-------------------- estimate + X_ind_all = np.tile(X_ind[np.newaxis, :], [3, 1])*3 + X_ind_all[1, :] += 1 + X_ind_all[2, :] += 2 + valid_ind = X_ind_all.flatten('F') + + shapeMU = model['shapeMU'][valid_ind, :] + shapePC = model['shapePC'][valid_ind, :n_sp] + expPC = model['expPC'][valid_ind, :n_ep] + + s = 4e-04 + R = mesh.transform.angle2matrix([0, 0, 0]) + t = [0, 0, 0] + lsp = []; lep = []; ls = []; lR = []; lt = [] + for i in range(max_iter): + X = shapeMU + shapePC.dot(sp) + expPC.dot(ep) + X = np.reshape(X, [int(len(X)/3), 3]).T + lsp.append(sp); lep.append(ep); ls.append(s), lR.append(R), lt.append(t) + + #----- estimate pose + P = mesh.transform.estimate_affine_matrix_3d22d(X.T, x.T) + s, R, t = mesh.transform.P2sRt(P) + lsp.append(sp); lep.append(ep); ls.append(s), lR.append(R), lt.append(t) + + #----- estimate shape + # expression + shape = shapePC.dot(sp) + shape = np.reshape(shape, [int(len(shape)/3), 3]).T + ep = estimate_expression(x, shapeMU, expPC, model['expEV'][:n_ep,:], shape, s, R, t[:2], lamb = 20) + lsp.append(sp); lep.append(ep); ls.append(s), lR.append(R), lt.append(t) + + # shape + expression = expPC.dot(ep) + expression = np.reshape(expression, [int(len(expression)/3), 3]).T + sp = estimate_shape(x, shapeMU, shapePC, model['shapeEV'][:n_sp,:], expression, s, R, t[:2], lamb = 40) + + # print('ls', ls) + # print('lR', lR) + return np.array(lsp), np.array(lep), np.array(ls), np.array(lR), np.array(lt) \ No newline at end of file diff --git a/face3d/morphable_model/morphabel_model.py b/face3d/morphable_model/morphabel_model.py index ed9d753..ecbdc99 100644 --- a/face3d/morphable_model/morphabel_model.py +++ b/face3d/morphable_model/morphabel_model.py @@ -117,19 +117,26 @@ def transform_3ddfa(self, vertices, s, angles, t3d): # only used for processing R = mesh.transform.angle2matrix_3ddfa(angles) return mesh.transform.similarity_transform(vertices, s, R, t3d) # --------------------------------------------------- fitting - def fit(self, x, X_ind, max_iter = 4): + def fit(self, x, X_ind, max_iter = 4, isShow = False): ''' fit 3dmm & pose parameters Args: x: (n, 2) image points X_ind: (n,) corresponding Model vertex indices max_iter: iteration + isShow: whether to reserve middle results for show Returns: fitted_sp: (n_sp, 1). shape parameters fitted_ep: (n_ep, 1). exp parameters s, angles, t ''' - fitted_sp, fitted_ep, s, R, t = fit.fit_points(x, X_ind, self.model, n_sp = self.n_shape_para, n_ep = self.n_exp_para, max_iter = max_iter) - angles = mesh.transform.matrix2angle(R) + if isShow: + fitted_sp, fitted_ep, s, R, t = fit.fit_points_for_show(x, X_ind, self.model, n_sp = self.n_shape_para, n_ep = self.n_exp_para, max_iter = max_iter) + angles = np.zeros((R.shape[0], 3)) + for i in range(R.shape[0]): + angles[i] = mesh.transform.matrix2angle(R[i]) + else: + fitted_sp, fitted_ep, s, R, t = fit.fit_points(x, X_ind, self.model, n_sp = self.n_shape_para, n_ep = self.n_exp_para, max_iter = max_iter) + angles = mesh.transform.matrix2angle(R) return fitted_sp, fitted_ep, s, angles, t