diff --git a/face_decoder.py b/face_decoder.py index ef22388..0f07e43 100644 --- a/face_decoder.py +++ b/face_decoder.py @@ -48,6 +48,7 @@ class Face3D(): # landmark_p: [batchsize,68,2] face_landmark_t = self.Compute_landmark(face_shape_t,self.facemodel) landmark_p = self.Projection_block(face_landmark_t) # 256*256 image + landmark_p = tf.stack([landmark_p[:,:,0],223. - landmark_p[:,:,1]],axis = 2) self.landmark_p = landmark_p # [batchsize,N,3] vertex color (in RGB order) diff --git a/images/camera.png b/images/camera.png new file mode 100644 index 0000000..8812268 Binary files /dev/null and b/images/camera.png differ diff --git a/images/lm3d.png b/images/lm3d.png new file mode 100644 index 0000000..c2b0818 Binary files /dev/null and b/images/lm3d.png differ diff --git a/images/lm5p.png b/images/lm5p.png new file mode 100644 index 0000000..0fdef99 Binary files /dev/null and b/images/lm5p.png differ diff --git a/output/blank.txt b/output/blank.txt deleted file mode 100644 index e69de29..0000000 diff --git a/preprocess_img.py b/preprocess_img.py index 76a2b4d..4a699db 100644 --- a/preprocess_img.py +++ b/preprocess_img.py @@ -2,7 +2,7 @@ import numpy as np from scipy.io import loadmat,savemat from PIL import Image -#calculating least sqaures problem +#calculating least square problem def POS(xp,x): npts = xp.shape[1] @@ -56,7 +56,7 @@ def Preprocess(img,lm,lm3D): # change from image plane coordinates to 3D sapce coordinates(X-Y plane) lm = np.stack([lm[:,0],h0 - 1 - lm[:,1]], axis = 1) - # calculate translation and scale factors using 5 facial landmarks and standard landmarks + # calculate translation and scale factors using 5 facial landmarks and standard landmarks of a 3D face t,s = POS(lm.transpose(),lm3D.transpose()) # processing the image diff --git a/readme.md b/readme.md index 4981fa3..137c4ed 100644 --- a/readme.md +++ b/readme.md @@ -58,18 +58,18 @@ Faces are represented with Basel Face Model 2009, which is easy for further mani ## Getting Started -### Prerequisite ### - -- Python >= 3.5 (numpy, scipy, pillow, opencv) -- Tensorflow >= 1.4 -- [Basel Face Model 2009 (BFM09)](https://faces.dmi.unibas.ch/bfm/main.php?nav=1-0&id=basel_face_model) -- [Expression Basis (transferred from Facewarehouse by Guo et al.)](https://github.com/Juyong/3DFace) - -Optional: - -- [tf mesh renderer](https://github.com/google/tf_mesh_renderer) (We use it as renderer while training. Can be used at test stage too. Only on Linux.) - +### System Requirements ### +- Reconstructions can be done on both Windows and Linux. However, we suggest running on Linux because the rendering process is only supported on Linux currently. If you wish to run on Windows, you have to comment out the rendering part. +- Python >= 3.5 (numpy, scipy, pillow, opencv). +- Tensorflow 1.4 ~ 1.12. +- [Basel Face Model 2009 (BFM09)](https://faces.dmi.unibas.ch/bfm/main.php?nav=1-0&id=basel_face_model). Our method depends on BFM09 to reconstruct 3D faces from regressed coefficients. +- [Expression Basis (transferred from Facewarehouse by Guo et al.)](https://github.com/Juyong/3DFace). The original BFM09 model does not handle expression variations so extra expression basis are needed. +- [tf mesh renderer](https://github.com/google/tf_mesh_renderer). Install the library via +``` +pip install mesh_renderer +``` +. Or you can follow the instruction of tf mesh render to install it using Bazel. Note that current rendering tool does not support tensorflow version higher than 1.13 and can only be used on Linux. ### Usage ### 1. Clone the repository @@ -79,11 +79,11 @@ git clone https://github.com/Microsoft/Deep3DFaceReconstruction cd Deep3DFaceReconstruction ``` -2. Download the BFM09 model and put "01_MorphableModel.mat" into ./BFM subfolder. +2. Due to the license agreement of Basel Face Model, you have to download the BFM09 model after submitting an application on its [home page](https://faces.dmi.unibas.ch/bfm/main.php?nav=1-2&id=downloads). After getting the access to BFM data, download "01_MorphableModel.mat" and put it into ./BFM subfolder. -3. Download the Expression Basis provided by Guo (You can find a link named CoarseData in the first row of Introduction part in their repository. Download and unzip the Coarse\_Dataset.zip), and put "Exp_Pca.bin" into ./BFM subfolder. +3. Download the Expression Basis provided by [Guo](https://github.com/Juyong/3DFace). Here we provide a [quick link](https://drive.google.com/open?id=1bw5Xf8C12pWmcMhNEu6PtsYVZkVucEN6) to it. Download "Exp_Pca.bin" and put it into ./BFM subfolder. The expression basis are constructed using [Facewarehouse](kunzhou.net/zjugaps/facewarehouse/) data and transferred to BFM topology. -4. Download the trained model at [GoogleDrive](https://drive.google.com/file/d/1RSEkXwF5BGelvBaIJFtKIxjUcR5ULSK0/view?usp=sharing), and put it into ./network subfolder. +4. Download the trained [reconstruction network](https://drive.google.com/file/d/1RSEkXwF5BGelvBaIJFtKIxjUcR5ULSK0/view?usp=sharing), and put it into ./network subfolder. 5. Run the demo code. @@ -91,13 +91,25 @@ cd Deep3DFaceReconstruction python demo.py ``` -6. To check the results, see ./output subfolder which contains: - - "xxx.mat" : consists of cropped input image, corresponding 5p and 68p landmarks, and output coefficients of R-Net. +6. ./input subfolder contains several test images and ./output subfolder stores their reconstruction results. For each input test image, two output files can be obtained after running the demo code: + - "xxx.mat" : + - cropped_img: an RGB image after alignment, which is the input to the R-Net + - recon_img: an RGBA reconstruction image aligned with the input image. + - coeff: output coefficients of R-Net. + - face_shape: vertex positions of 3D face in the world coordinate. + - face_texture: vertex texture of 3D face, which excludes lighting information. + - face_color: vertex color of 3D face, which takes lighting into consideration. + - lm\_68p: 68 2D facial landmarks derived from the reconstructed 3D face. The landmarks are aligned with cropped_img. + - lm\_5p: 5 detected landmarks aligned with cropped_img. - "xxx_mesh.obj" : 3D face mesh in canonical view (best viewed in MeshLab). -### Tips ### +### Common Issues ### -1. The model is trained without augmentation so that a pre-alignment with 5 facial landmarks is necessary. We put some examples in the ./input subfolder for reference. +1. The model is trained without position augmentation so that a pre-alignment with 5 facial landmarks is necessary. In our image pre-processing stage, we solve a least square problem between 5 facial landmarks detected on the image and 5 facial landmarks of an average 3D face to cancel out face scales and misalignment. +

+ +

+2. We put some examples in the ./input subfolder for reference. 2. Current model is trained under the assumption of 3-channel scene illumination (instead of white light described in the paper).