Skip to content

Commit

Permalink
Merge pull request liuyulvv#1 from liuyulvv/dev
Browse files Browse the repository at this point in the history 
眨眼眉毛嘴巴
  • Loading branch information
@liuyulvv
liuyulvv authored Aug 29, 2022
2 parents 713f2c7 + eadfe73 commit 7837bab
Show file tree
Hide file tree
Showing 5 changed files with 296 additions and 8 deletions.
64 changes: 61 additions & 3 deletions blend_shape_config.json
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"eye": {
"left": [
33,
130,
133,
160,
159,
Expand All @@ -20,8 +20,66 @@
374,
380
],
"low": 0.6,
"high": 1.0,
"low": 0.25,
"high": 0.75,
"maxRatio": 0.285
},
"mouth": {
"eye": [
133,
362
],
"mouth": [
13,
14
],
"low": 0.17,
"high": 0.5
},
"brow": {
"left": [
35,
244,
63,
105,
66,
229,
230,
231
],
"right": [
265,
464,
293,
334,
296,
449,
450,
451
],
"maxRatio": 1.15,
"low": 0.07,
"high": 0.125
},
"pupil": {
"left": [
468,
469,
470,
471,
472
],
"right": [
473,
474,
475,
476,
477
]
},
"nose": {
"tip": [
1
]
}
}
31 changes: 31 additions & 0 deletions include/livelink/blend_shape_config.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,37 @@ enum class FaceBlendShape {
RightEyeRoll
};

// struct PCF {
// PCF(int near = 1,
// int far = 10000,
// int height = 1920,
// int width = 1080,
// double fy = 1074.520446598223) : near_(near),
// far_(far),
// height_(height),
// width_(width),
// fy_(fy) {
// fov_y_ = 2 * std::atan(height / (2 * fy_));
// auto heightAtNear = 2 * near_ * std::tan(0.5 * fov_y_);
// auto widthAtNear = width * heightAtNear / height;
// left_ = -0.5 * widthAtNear;
// right_ = 0.5 * widthAtNear;
// bottom_ = -0.5 * heightAtNear;
// top_ = 0.5 * heightAtNear;
// }

// int near_;
// int far_;
// int height_;
// int width_;
// int fy_;
// int fov_y_;
// int left_;
// int right_;
// int bottom_;
// int top_;
// };

} // namespace LiveLink

#endif
114 changes: 112 additions & 2 deletions include/livelink/face_live_link.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,13 @@

#include "live_link_base.hpp"
#include "util.hpp"
#include <fstream>
#include <iostream>
#include <random>
#include <vector>

using json = nlohmann::json;

namespace LiveLink {

class FaceLiveLink : public LiveLinkBase {
Expand All @@ -17,13 +21,118 @@ class FaceLiveLink : public LiveLinkBase {
virtual ~FaceLiveLink() = default;

virtual void Process(const SendCallback& callback, const Landmark& landmark) override {
UpdateEyeOpen(landmark);
// // 54, 53, 52, 51
// json data(blendShape_);
// std::ofstream file("res/blendshape/" + std::to_string(index_) + ".json");
// file << data;
// json data1(landmark);
// std::ofstream file1("res/landmark/" + std::to_string(index_++) + ".json");
// file1 << data1;
// for (int i = 0; i < blendShape_.size(); ++i) {
// if (i >= 51 && i <= 54) {
// blendShape_[i] = 0;
// } else {
// std::random_device rd;
// std::mt19937 seed(rd());
// // std::uniform_int_distribution<> gen(0, 255);
// std::uniform_real_distribution<float> gen(0, 1);
// blendShape_[i] = gen(seed);
// }
// }
UpdateEyeBlink(landmark);
UpdateMouth(landmark);
UpdateBrow(landmark);
UpdatePupil(landmark);
Encode();
callback(buffer_);
}

void Renew(const SendCallback& callback) {
for (int i = 0; i < blendShape_.size(); ++i) {
blendShape_[i] = 0;
}
Encode();
callback(buffer_);
}

private:
void UpdateEyeOpen(const Landmark& landmark) {
void UpdatePupil(const Landmark& landmark) {
auto pupil = json_["pupil"];
auto leftPupilIndex = pupil["left"].get<std::vector<int>>();
auto rightPupilIndex = pupil["right"].get<std::vector<int>>();
auto leftPupil = GetLandmark(landmark, leftPupilIndex);
auto rightPupil = GetLandmark(landmark, rightPupilIndex);

auto eye = json_["eye"];
auto leftEyeIndex = eye["left"].get<std::vector<int>>();
auto rightEyeIndex = eye["right"].get<std::vector<int>>();

auto& leftEyeOuterCorner = landmark[leftEyeIndex[0]];
auto& leftEyeInnerCorner = landmark[leftEyeIndex[1]];

auto& rightEyeOuterCorner = landmark[rightEyeIndex[0]];
auto& rightEyeInnerCorner = landmark[rightEyeIndex[1]];

auto leftEyeWidth = Distance(leftEyeOuterCorner, leftEyeInnerCorner);
auto rightEyeWidth = Distance(rightEyeOuterCorner, rightEyeInnerCorner);

auto leftEyeMidPoint = Average(leftEyeOuterCorner, leftEyeInnerCorner);
auto rightEyeMidPoint = Average(rightEyeOuterCorner, rightEyeInnerCorner);

auto leftDx = leftEyeMidPoint[0] - leftPupil[0][0];
auto leftDy = leftEyeMidPoint[1] - leftPupil[0][1];

auto rightDx = rightEyeMidPoint[0] - rightPupil[0][0];
auto rightDy = rightEyeMidPoint[1] - rightPupil[0][1];

auto leftRatioX = leftDx / (leftEyeWidth / 2) * 4;
auto rightRatioX = rightDx / (rightEyeWidth / 2) * 4;

auto leftRatioY = leftDy / (leftEyeWidth / 4) * 4;
auto rightRatioY = rightDy / (rightEyeWidth / 4) * 4;

// if (leftRatioX >= 1) {
// blendShape_[int(FaceBlendShape::)]
// }

std::cout << leftRatioX << " " << rightRatioX << " " << leftRatioY << " " << rightRatioY << std::endl;
}

void UpdateBrow(const Landmark& landmark) {
auto brow = json_["brow"];
auto maxRatio = brow["maxRatio"].get<double>();
auto low = brow["low"].get<double>();
auto high = brow["high"].get<double>();
auto leftBrowIndex = brow["left"].get<std::vector<int>>();
auto rightBrowIndex = brow["right"].get<std::vector<int>>();
auto leftBrowDistance = GetEyeLidRatio(landmark, leftBrowIndex);
auto rightBrowDistance = GetEyeLidRatio(landmark, rightBrowIndex);
auto leftBrowRatio = leftBrowDistance / maxRatio - 1;
auto rightBrowRatio = leftBrowDistance / maxRatio - 1;
auto leftRaiseRatio = Remap(leftBrowRatio, low, high);
auto rightRaiseRatio = Remap(rightBrowRatio, low, high);
blendShape_[int(FaceBlendShape::BrowOuterUpLeft)] = leftRaiseRatio;
blendShape_[int(FaceBlendShape::BrowOuterUpRight)] = rightRaiseRatio;
// std::cout << leftRaiseRatio << " " << rightRaiseRatio << std::endl;
}

void UpdateMouth(const Landmark& landmark) {
auto mouth = json_["mouth"];
auto eyeIndex = mouth["eye"].get<std::vector<int>>();
auto mouthIndex = mouth["mouth"].get<std::vector<int>>();

auto eyeLandmark = GetLandmark(landmark, eyeIndex);
auto eyeInnerDistance = Distance(eyeLandmark[0], eyeLandmark[1]);

auto mouthLandmark = GetLandmark(landmark, mouthIndex);
auto mouthOpen = Distance(mouthLandmark[0], mouthLandmark[1]);

auto mouthY = Remap(mouthOpen / eyeInnerDistance, mouth["low"].get<double>(), mouth["high"].get<double>());
blendShape_[int(FaceBlendShape::MouthLowerDownLeft)] = mouthY;
blendShape_[int(FaceBlendShape::MouthLowerDownRight)] = mouthY;
}

void UpdateEyeBlink(const Landmark& landmark) {
auto eye = json_["eye"];
auto leftIndex = eye["left"].get<std::vector<int>>();
auto rightIndex = eye["right"].get<std::vector<int>>();
Expand Down Expand Up @@ -82,6 +191,7 @@ class FaceLiveLink : public LiveLinkBase {
int fps = 60;
int size_ = 61;
std::vector<char> buffer_;
int index_ = 1700;
};

} // namespace LiveLink
Expand Down
88 changes: 88 additions & 0 deletions include/livelink/live_link_base.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
#include "blend_shape_config.hpp"
#include "nlohmann/json.hpp"
#include <functional>
#include <opencv.hpp>
#include <string>
#include <vector>

Expand All @@ -23,6 +24,22 @@ class LiveLinkBase {
virtual void Process(const SendCallback& callback, const Landmark& landmark) = 0;
virtual void Encode() = 0;

Landmark GetLandmark(const Landmark& landmark, std::vector<int>& index) {
Landmark ret;
for (const auto& i : index) {
ret.push_back(landmark[i]);
}
return std::move(ret);
}

std::vector<double> Average(const std::vector<double>& x, const std::vector<double>& y) {
std::vector<double> ret;
for (int i = 0; i < x.size(); ++i) {
ret.push_back((x[i] + y[i]) / 2);
}
return std::move(ret);
}

double Distance(const std::vector<double>& x, const std::vector<double>& y, bool threeD = false) const {
if (threeD) {
return std::pow(std::pow(x[0] - y[0], 2) + std::pow(x[1] - y[1], 2) + std::pow(x[2] - y[2], 2), 0.5);
Expand All @@ -39,6 +56,77 @@ class LiveLinkBase {
return (Clamp(value, min, max) - min) / (max - min);
}

// void CalculateRotation(const Landmark& landmark, const PCF& pcf) {
// double width = pcf.width_;
// double height = pcf.height_;
// double focalLength = width;
// std::vector<double> center{width / 2, height / 2};
// std::vector<std::vector<double>> cameraMatrix{
// {focalLength, 0, center[0]},
// {0, focalLength, center[1]},
// {0, 0, 1}};
// std::vector<std::vector<int>> distCoeff{{0}, {0}, {0}, {0}};

// Landmark landmarks;
// for (int j = 0; j < 3; ++j) {
// std::vector<double> temp;
// for (int i = 0; i < 468; ++i) {
// temp.push_back({landmark[i][j]});
// }
// landmarks.push_back(temp);
// }
// // auto landmarksMat = ToMat(landmarks);
// }

// void GetMetricLandmarks(Landmark& landmarks, const PCF& pcf) {
// auto landmarksMat = ProjectXY(landmarks, pcf);
// auto res = landmarksMat.row(2);
// }
//
// cv::Mat ProjectXY(Landmark& landmarks, const PCF& pcf) {
// // landmarks: 3 * 468
// assert(landmarks.size() == 3 && landmarks[0].size() == 468);
// auto xScale = pcf.right_ - pcf.left_;
// auto yScale = pcf.top_ - pcf.bottom_;
// auto xTranslation = pcf.left_;
// auto yTranslation = pcf.bottom_;
// for (int i = 0; i < 468; ++i) {
// landmarks[1][i] = 1.0 - landmarks[1][i];
// }
// auto mat = ToMat(landmarks);
// cv::Mat mat1(1, 3, cv::DataType<double>::type);
// mat1.at<double>(0, 0) = xScale;
// mat1.at<double>(0, 1) = yScale;
// mat1.at<double>(0, 2) = xScale;
// mat = mat * mat1.t();
// cv::Mat mat2(1, 3, cv::DataType<double>::type);
// mat2.at<double>(0, 0) = xTranslation;
// mat2.at<double>(0, 1) = yTranslation;
// mat2.at<double>(0, 2) = 0;
// mat = mat + mat2.t();
// return std::move(mat);
// }
//
// cv::Mat ToMat(const std::vector<std::vector<double>>& value) {
// cv::Mat mat(value.size(), value.at(0).size(), cv::DataType<double>::type);
// for (int i = 0; i < mat.rows; ++i) {
// for (int j = 0; j < mat.cols; ++j) {
// mat.at<double>(i, j) = value.at(i).at(j);
// }
// }
// return std::move(mat);
// }
//
// std::vector<std::vector<double>> ToVector(const cv::Mat& mat) {
// std::vector<std::vector<double>> ret(mat.rows, std::vector<double>(mat.cols, 0));
// for (int i = 0; i < mat.rows; ++i) {
// for (int j = 0; j < mat.cols; ++j) {
// ret.at(i).at(j) = mat.at<double>(i, j);
// }
// }
// return ret;
// }

protected:
const json& json_;
};
Expand Down
Loading

0 comments on commit 7837bab

Please sign in to comment.