diff --git a/blend_shape_config.json b/blend_shape_config.json index 4a1abb4..eef7ccf 100644 --- a/blend_shape_config.json +++ b/blend_shape_config.json @@ -1,7 +1,7 @@ { "eye": { "left": [ - 33, + 130, 133, 160, 159, @@ -20,8 +20,66 @@ 374, 380 ], - "low": 0.6, - "high": 1.0, + "low": 0.25, + "high": 0.75, "maxRatio": 0.285 + }, + "mouth": { + "eye": [ + 133, + 362 + ], + "mouth": [ + 13, + 14 + ], + "low": 0.17, + "high": 0.5 + }, + "brow": { + "left": [ + 35, + 244, + 63, + 105, + 66, + 229, + 230, + 231 + ], + "right": [ + 265, + 464, + 293, + 334, + 296, + 449, + 450, + 451 + ], + "maxRatio": 1.15, + "low": 0.07, + "high": 0.125 + }, + "pupil": { + "left": [ + 468, + 469, + 470, + 471, + 472 + ], + "right": [ + 473, + 474, + 475, + 476, + 477 + ] + }, + "nose": { + "tip": [ + 1 + ] } } \ No newline at end of file diff --git a/include/livelink/blend_shape_config.hpp b/include/livelink/blend_shape_config.hpp index c2b2553..7fa0548 100644 --- a/include/livelink/blend_shape_config.hpp +++ b/include/livelink/blend_shape_config.hpp @@ -67,6 +67,37 @@ enum class FaceBlendShape { RightEyeRoll }; +// struct PCF { +// PCF(int near = 1, +// int far = 10000, +// int height = 1920, +// int width = 1080, +// double fy = 1074.520446598223) : near_(near), +// far_(far), +// height_(height), +// width_(width), +// fy_(fy) { +// fov_y_ = 2 * std::atan(height / (2 * fy_)); +// auto heightAtNear = 2 * near_ * std::tan(0.5 * fov_y_); +// auto widthAtNear = width * heightAtNear / height; +// left_ = -0.5 * widthAtNear; +// right_ = 0.5 * widthAtNear; +// bottom_ = -0.5 * heightAtNear; +// top_ = 0.5 * heightAtNear; +// } + +// int near_; +// int far_; +// int height_; +// int width_; +// int fy_; +// int fov_y_; +// int left_; +// int right_; +// int bottom_; +// int top_; +// }; + } // namespace LiveLink #endif \ No newline at end of file diff --git a/include/livelink/face_live_link.hpp b/include/livelink/face_live_link.hpp index 0fb1d83..769fc98 100644 --- a/include/livelink/face_live_link.hpp +++ b/include/livelink/face_live_link.hpp @@ -3,9 +3,13 @@ #include "live_link_base.hpp" #include "util.hpp" +#include #include +#include #include +using json = nlohmann::json; + namespace LiveLink { class FaceLiveLink : public LiveLinkBase { @@ -17,13 +21,118 @@ class FaceLiveLink : public LiveLinkBase { virtual ~FaceLiveLink() = default; virtual void Process(const SendCallback& callback, const Landmark& landmark) override { - UpdateEyeOpen(landmark); + // // 54, 53, 52, 51 + // json data(blendShape_); + // std::ofstream file("res/blendshape/" + std::to_string(index_) + ".json"); + // file << data; + // json data1(landmark); + // std::ofstream file1("res/landmark/" + std::to_string(index_++) + ".json"); + // file1 << data1; + // for (int i = 0; i < blendShape_.size(); ++i) { + // if (i >= 51 && i <= 54) { + // blendShape_[i] = 0; + // } else { + // std::random_device rd; + // std::mt19937 seed(rd()); + // // std::uniform_int_distribution<> gen(0, 255); + // std::uniform_real_distribution gen(0, 1); + // blendShape_[i] = gen(seed); + // } + // } + UpdateEyeBlink(landmark); + UpdateMouth(landmark); + UpdateBrow(landmark); + UpdatePupil(landmark); + Encode(); + callback(buffer_); + } + + void Renew(const SendCallback& callback) { + for (int i = 0; i < blendShape_.size(); ++i) { + blendShape_[i] = 0; + } Encode(); callback(buffer_); } private: - void UpdateEyeOpen(const Landmark& landmark) { + void UpdatePupil(const Landmark& landmark) { + auto pupil = json_["pupil"]; + auto leftPupilIndex = pupil["left"].get>(); + auto rightPupilIndex = pupil["right"].get>(); + auto leftPupil = GetLandmark(landmark, leftPupilIndex); + auto rightPupil = GetLandmark(landmark, rightPupilIndex); + + auto eye = json_["eye"]; + auto leftEyeIndex = eye["left"].get>(); + auto rightEyeIndex = eye["right"].get>(); + + auto& leftEyeOuterCorner = landmark[leftEyeIndex[0]]; + auto& leftEyeInnerCorner = landmark[leftEyeIndex[1]]; + + auto& rightEyeOuterCorner = landmark[rightEyeIndex[0]]; + auto& rightEyeInnerCorner = landmark[rightEyeIndex[1]]; + + auto leftEyeWidth = Distance(leftEyeOuterCorner, leftEyeInnerCorner); + auto rightEyeWidth = Distance(rightEyeOuterCorner, rightEyeInnerCorner); + + auto leftEyeMidPoint = Average(leftEyeOuterCorner, leftEyeInnerCorner); + auto rightEyeMidPoint = Average(rightEyeOuterCorner, rightEyeInnerCorner); + + auto leftDx = leftEyeMidPoint[0] - leftPupil[0][0]; + auto leftDy = leftEyeMidPoint[1] - leftPupil[0][1]; + + auto rightDx = rightEyeMidPoint[0] - rightPupil[0][0]; + auto rightDy = rightEyeMidPoint[1] - rightPupil[0][1]; + + auto leftRatioX = leftDx / (leftEyeWidth / 2) * 4; + auto rightRatioX = rightDx / (rightEyeWidth / 2) * 4; + + auto leftRatioY = leftDy / (leftEyeWidth / 4) * 4; + auto rightRatioY = rightDy / (rightEyeWidth / 4) * 4; + + // if (leftRatioX >= 1) { + // blendShape_[int(FaceBlendShape::)] + // } + + std::cout << leftRatioX << " " << rightRatioX << " " << leftRatioY << " " << rightRatioY << std::endl; + } + + void UpdateBrow(const Landmark& landmark) { + auto brow = json_["brow"]; + auto maxRatio = brow["maxRatio"].get(); + auto low = brow["low"].get(); + auto high = brow["high"].get(); + auto leftBrowIndex = brow["left"].get>(); + auto rightBrowIndex = brow["right"].get>(); + auto leftBrowDistance = GetEyeLidRatio(landmark, leftBrowIndex); + auto rightBrowDistance = GetEyeLidRatio(landmark, rightBrowIndex); + auto leftBrowRatio = leftBrowDistance / maxRatio - 1; + auto rightBrowRatio = leftBrowDistance / maxRatio - 1; + auto leftRaiseRatio = Remap(leftBrowRatio, low, high); + auto rightRaiseRatio = Remap(rightBrowRatio, low, high); + blendShape_[int(FaceBlendShape::BrowOuterUpLeft)] = leftRaiseRatio; + blendShape_[int(FaceBlendShape::BrowOuterUpRight)] = rightRaiseRatio; + // std::cout << leftRaiseRatio << " " << rightRaiseRatio << std::endl; + } + + void UpdateMouth(const Landmark& landmark) { + auto mouth = json_["mouth"]; + auto eyeIndex = mouth["eye"].get>(); + auto mouthIndex = mouth["mouth"].get>(); + + auto eyeLandmark = GetLandmark(landmark, eyeIndex); + auto eyeInnerDistance = Distance(eyeLandmark[0], eyeLandmark[1]); + + auto mouthLandmark = GetLandmark(landmark, mouthIndex); + auto mouthOpen = Distance(mouthLandmark[0], mouthLandmark[1]); + + auto mouthY = Remap(mouthOpen / eyeInnerDistance, mouth["low"].get(), mouth["high"].get()); + blendShape_[int(FaceBlendShape::MouthLowerDownLeft)] = mouthY; + blendShape_[int(FaceBlendShape::MouthLowerDownRight)] = mouthY; + } + + void UpdateEyeBlink(const Landmark& landmark) { auto eye = json_["eye"]; auto leftIndex = eye["left"].get>(); auto rightIndex = eye["right"].get>(); @@ -82,6 +191,7 @@ class FaceLiveLink : public LiveLinkBase { int fps = 60; int size_ = 61; std::vector buffer_; + int index_ = 1700; }; } // namespace LiveLink diff --git a/include/livelink/live_link_base.hpp b/include/livelink/live_link_base.hpp index 0b307d2..f4722f1 100644 --- a/include/livelink/live_link_base.hpp +++ b/include/livelink/live_link_base.hpp @@ -4,6 +4,7 @@ #include "blend_shape_config.hpp" #include "nlohmann/json.hpp" #include +#include #include #include @@ -23,6 +24,22 @@ class LiveLinkBase { virtual void Process(const SendCallback& callback, const Landmark& landmark) = 0; virtual void Encode() = 0; + Landmark GetLandmark(const Landmark& landmark, std::vector& index) { + Landmark ret; + for (const auto& i : index) { + ret.push_back(landmark[i]); + } + return std::move(ret); + } + + std::vector Average(const std::vector& x, const std::vector& y) { + std::vector ret; + for (int i = 0; i < x.size(); ++i) { + ret.push_back((x[i] + y[i]) / 2); + } + return std::move(ret); + } + double Distance(const std::vector& x, const std::vector& y, bool threeD = false) const { if (threeD) { return std::pow(std::pow(x[0] - y[0], 2) + std::pow(x[1] - y[1], 2) + std::pow(x[2] - y[2], 2), 0.5); @@ -39,6 +56,77 @@ class LiveLinkBase { return (Clamp(value, min, max) - min) / (max - min); } + // void CalculateRotation(const Landmark& landmark, const PCF& pcf) { + // double width = pcf.width_; + // double height = pcf.height_; + // double focalLength = width; + // std::vector center{width / 2, height / 2}; + // std::vector> cameraMatrix{ + // {focalLength, 0, center[0]}, + // {0, focalLength, center[1]}, + // {0, 0, 1}}; + // std::vector> distCoeff{{0}, {0}, {0}, {0}}; + + // Landmark landmarks; + // for (int j = 0; j < 3; ++j) { + // std::vector temp; + // for (int i = 0; i < 468; ++i) { + // temp.push_back({landmark[i][j]}); + // } + // landmarks.push_back(temp); + // } + // // auto landmarksMat = ToMat(landmarks); + // } + + // void GetMetricLandmarks(Landmark& landmarks, const PCF& pcf) { + // auto landmarksMat = ProjectXY(landmarks, pcf); + // auto res = landmarksMat.row(2); + // } + // + // cv::Mat ProjectXY(Landmark& landmarks, const PCF& pcf) { + // // landmarks: 3 * 468 + // assert(landmarks.size() == 3 && landmarks[0].size() == 468); + // auto xScale = pcf.right_ - pcf.left_; + // auto yScale = pcf.top_ - pcf.bottom_; + // auto xTranslation = pcf.left_; + // auto yTranslation = pcf.bottom_; + // for (int i = 0; i < 468; ++i) { + // landmarks[1][i] = 1.0 - landmarks[1][i]; + // } + // auto mat = ToMat(landmarks); + // cv::Mat mat1(1, 3, cv::DataType::type); + // mat1.at(0, 0) = xScale; + // mat1.at(0, 1) = yScale; + // mat1.at(0, 2) = xScale; + // mat = mat * mat1.t(); + // cv::Mat mat2(1, 3, cv::DataType::type); + // mat2.at(0, 0) = xTranslation; + // mat2.at(0, 1) = yTranslation; + // mat2.at(0, 2) = 0; + // mat = mat + mat2.t(); + // return std::move(mat); + // } + // + // cv::Mat ToMat(const std::vector>& value) { + // cv::Mat mat(value.size(), value.at(0).size(), cv::DataType::type); + // for (int i = 0; i < mat.rows; ++i) { + // for (int j = 0; j < mat.cols; ++j) { + // mat.at(i, j) = value.at(i).at(j); + // } + // } + // return std::move(mat); + // } + // + // std::vector> ToVector(const cv::Mat& mat) { + // std::vector> ret(mat.rows, std::vector(mat.cols, 0)); + // for (int i = 0; i < mat.rows; ++i) { + // for (int j = 0; j < mat.cols; ++j) { + // ret.at(i).at(j) = mat.at(i, j); + // } + // } + // return ret; + // } + protected: const json& json_; }; diff --git a/src/main.cpp b/src/main.cpp index 462da78..7cb056a 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -43,6 +43,7 @@ int main() { bool isCamera = true; cv::Mat outputBGRFrame; + cv::Mat cameraBGRFrame; bool grabFrames = true; if (!capture.isOpened()) { logger->Log("VideoCapture is not open"); @@ -60,6 +61,8 @@ int main() { socket.send_to(asio::buffer(buffer), serverEndpoint); }; + faceLiveLink.Renew(sendCallback); + auto landmarkCallback = [&](const std::vector>& data) { faceLiveLink.Process(sendCallback, data); }; @@ -69,11 +72,9 @@ int main() { }; interface->CreateObserver("face_landmarks_with_iris", landmarkCallback); - interface->CreateObserver("face_geometry", tempCallback); interface->Start(); while (grabFrames) { - cv::Mat cameraBGRFrame; capture >> cameraBGRFrame; if (isCamera) { cv::flip(cameraBGRFrame, cameraBGRFrame, 1); @@ -85,7 +86,7 @@ int main() { cv::Mat cameraRGBFrame; cv::cvtColor(cameraBGRFrame, cameraRGBFrame, cv::COLOR_BGR2RGB); interface->Detect(cameraRGBFrame); - const int pressed_key = cv::waitKey(40); + const int pressed_key = cv::waitKey(100); if (pressed_key >= 0 && pressed_key != 255) grabFrames = false; }