From 794352857062077d6a379aa18763df446ccb2a4e Mon Sep 17 00:00:00 2001 From: liuyulvv Date: Thu, 25 Aug 2022 11:32:58 +0800 Subject: [PATCH] =?UTF-8?q?=E7=9C=A8=E7=9C=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CMakeLists.txt | 24 +++++-- blend_shape_config.json | 27 ++++++++ include/livelink.hpp | 8 +++ include/livelink/blend_shape_config.hpp | 72 ++++++++++++++++++++ include/livelink/face_live_link.hpp | 89 +++++++++++++++++++++++++ include/livelink/live_link_base.hpp | 48 +++++++++++++ include/mediapipe.hpp | 8 +++ include/mediapipe/mediapipe_api.h | 13 ++++ include/mediapipe/mediapipe_interface.h | 26 ++++++++ include/mediapipe/mediapipe_log.h | 14 ++++ include/util.hpp | 22 ++++++ main.cpp => src/main.cpp | 25 +++++-- 12 files changed, 363 insertions(+), 13 deletions(-) create mode 100644 blend_shape_config.json create mode 100644 include/livelink.hpp create mode 100644 include/livelink/blend_shape_config.hpp create mode 100644 include/livelink/face_live_link.hpp create mode 100644 include/livelink/live_link_base.hpp create mode 100644 include/mediapipe.hpp create mode 100644 include/mediapipe/mediapipe_api.h create mode 100644 include/mediapipe/mediapipe_interface.h create mode 100644 include/mediapipe/mediapipe_log.h create mode 100644 include/util.hpp rename main.cpp => src/main.cpp (75%) diff --git a/CMakeLists.txt b/CMakeLists.txt index b45a9b7..53155b2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,6 +12,7 @@ include_directories( "./include" "./include/livelink" "./include/mediapipe" + "./include/nlohmann" "./include/blendshape" ) @@ -22,10 +23,19 @@ link_directories( "./lib" ) -set (SRC ./main.cpp) - -add_executable(${PROJECT_NAME} ${SRC}) - -target_link_libraries(${PROJECT_NAME} opencv_world3410 onnxruntime onnxruntime_providers_cuda onnxruntime_providers_shared onnxruntime_providers_tensorrt mediapipe) - -SET(EXECUTABLE_OUTPUT_PATH ${PROJECT_SOURCE_DIR}/bin) \ No newline at end of file +SET(EXECUTABLE_OUTPUT_PATH ${PROJECT_SOURCE_DIR}/bin) + +file(GLOB srcs RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}/src/*.cpp") + +foreach(mainfile IN LISTS srcs) + get_filename_component(srcname ${mainfile} NAME_WE) + add_executable(${srcname} ${mainfile}) + target_link_libraries(${srcname} + opencv_world3410 + onnxruntime + onnxruntime_providers_cuda + onnxruntime_providers_shared + onnxruntime_providers_tensorrt + mediapipe + ) +endforeach() \ No newline at end of file diff --git a/blend_shape_config.json b/blend_shape_config.json new file mode 100644 index 0000000..4a1abb4 --- /dev/null +++ b/blend_shape_config.json @@ -0,0 +1,27 @@ +{ + "eye": { + "left": [ + 33, + 133, + 160, + 159, + 158, + 144, + 145, + 153 + ], + "right": [ + 263, + 362, + 387, + 386, + 385, + 373, + 374, + 380 + ], + "low": 0.6, + "high": 1.0, + "maxRatio": 0.285 + } +} \ No newline at end of file diff --git a/include/livelink.hpp b/include/livelink.hpp new file mode 100644 index 0000000..8b11101 --- /dev/null +++ b/include/livelink.hpp @@ -0,0 +1,8 @@ +#ifndef LIVELINK_HPP +#define LIVELINK_HPP + +#include "livelink/blend_shape_config.hpp" +#include "livelink/face_live_link.hpp" +#include "livelink/live_link_base.hpp" + +#endif \ No newline at end of file diff --git a/include/livelink/blend_shape_config.hpp b/include/livelink/blend_shape_config.hpp new file mode 100644 index 0000000..c2b2553 --- /dev/null +++ b/include/livelink/blend_shape_config.hpp @@ -0,0 +1,72 @@ +#ifndef BLEND_SHAPE_CONFIG_HPP +#define BLEND_SHAPE_CONFIG_HPP + +namespace LiveLink { + +enum class FaceBlendShape { + EyeBlinkLeft, + EyeLookDownLeft, + EyeLookInLeft, + EyeLookOutLeft, + EyeLookUpLeft, + EyeSquintLeft, + EyeWideLeft, + EyeBlinkRight, + EyeLookDownRight, + EyeLookInRight, + EyeLookOutRight, + EyeLookUpRight, + EyeSquintRight, + EyeWideRight, + JawForward, + JawLeft, + JawRight, + JawOpen, + MouthClose, + MouthFunnel, + MouthPucker, + MouthLeft, + MouthRight, + MouthSmileLeft, + MouthSmileRight, + MouthFrownLeft, + MouthFrownRight, + MouthDimpleLeft, + MouthDimpleRight, + MouthStretchLeft, + MouthStretchRight, + MouthRollLower, + MouthRollUpper, + MouthShrugLower, + MouthShrugUpper, + MouthPressLeft, + MouthPressRight, + MouthLowerDownLeft, + MouthLowerDownRight, + MouthUpperUpLeft, + MouthUpperUpRight, + BrowDownLeft, + BrowDownRight, + BrowInnerUp, + BrowOuterUpLeft, + BrowOuterUpRight, + CheekPuff, + CheekSquintLeft, + CheekSquintRight, + NoseSneerLeft, + NoseSneerRight, + TongueOut, + HeadYaw, + HeadPitch, + HeadRoll, + LeftEyeYaw, + LeftEyePitch, + LeftEyeRoll, + RightEyeYaw, + RightEyePitch, + RightEyeRoll +}; + +} // namespace LiveLink + +#endif \ No newline at end of file diff --git a/include/livelink/face_live_link.hpp b/include/livelink/face_live_link.hpp new file mode 100644 index 0000000..0fb1d83 --- /dev/null +++ b/include/livelink/face_live_link.hpp @@ -0,0 +1,89 @@ +#ifndef FACE_LIVE_LINK_HPP +#define FACE_LIVE_LINK_HPP + +#include "live_link_base.hpp" +#include "util.hpp" +#include +#include + +namespace LiveLink { + +class FaceLiveLink : public LiveLinkBase { +public: + FaceLiveLink(const json& config, int size = 61) : LiveLinkBase(config) { + blendShape_ = std::vector(size, 0); + } + + virtual ~FaceLiveLink() = default; + + virtual void Process(const SendCallback& callback, const Landmark& landmark) override { + UpdateEyeOpen(landmark); + Encode(); + callback(buffer_); + } + +private: + void UpdateEyeOpen(const Landmark& landmark) { + auto eye = json_["eye"]; + auto leftIndex = eye["left"].get>(); + auto rightIndex = eye["right"].get>(); + auto maxRatio = eye["maxRatio"].get(); + auto low = eye["low"].get(); + auto high = eye["high"].get(); + auto leftRatio = Clamp(GetEyeLidRatio(landmark, leftIndex) / maxRatio, 0, 2); + auto rightRatio = Clamp(GetEyeLidRatio(landmark, rightIndex) / maxRatio, 0, 2); + auto leftRes = Remap(leftRatio, low, high); + auto rightRes = Remap(rightRatio, low, high); + blendShape_[int(FaceBlendShape::EyeBlinkLeft)] = (1 - leftRes) || 0; + blendShape_[int(FaceBlendShape::EyeBlinkRight)] = (1 - rightRes) || 0; + } + + double GetEyeLidRatio(const Landmark& landmark, const std::vector& index) const { + auto& eyeOuterCorner = landmark[index[0]]; + auto& eyeInnerCorner = landmark[index[1]]; + auto& eyeOuterUpperLid = landmark[index[2]]; + auto& eyeMidUpperLid = landmark[index[3]]; + auto& eyeInnerUpperLid = landmark[index[4]]; + auto& eyeOuterLowerLid = landmark[index[5]]; + auto& eyeMidLowerLid = landmark[index[6]]; + auto& eyeInnerLowerLid = landmark[index[7]]; + auto eyeWidth = Distance(eyeOuterCorner, eyeInnerCorner); + auto eyeOuterLidDistance = Distance(eyeOuterUpperLid, eyeOuterLowerLid); + auto eyeMidLidDistance = Distance(eyeMidUpperLid, eyeMidLowerLid); + auto eyeInnerLidDistance = Distance(eyeInnerUpperLid, eyeInnerLowerLid); + auto eyeLidAvg = (eyeOuterLidDistance + eyeMidLidDistance + eyeInnerLidDistance) / 3; + return eyeLidAvg / eyeWidth; + } + + virtual void Encode() override { + buffer_.clear(); + Util::ValueToBuffer(6, buffer_, false); // version + for (const auto& c : uuid_) { // uuid + buffer_.push_back(c); + } + Util::ValueToBuffer(int(name_.size()), buffer_, true); // name length + for (const auto& c : name_) { // name + buffer_.push_back(c); + } + Util::ValueToBuffer(0, buffer_, true); // ??? + Util::ValueToBuffer(0, buffer_, true); // ??? + Util::ValueToBuffer(60, buffer_, true); // fps + Util::ValueToBuffer(1, buffer_, true); // int(fps/60) + buffer_.push_back(0x3d); // blendshape length + std::for_each(blendShape_.begin(), blendShape_.end(), [&](const float value) { // blendshape data + Util::ValueToBuffer(value, buffer_, true); + }); + } + + BlendShape blendShape_; + int version_ = 6; + std::string uuid_ = "$fcaf7061-2964-459b-87a9-ad78290e21e6"; + std::string name_ = "vzan"; + int fps = 60; + int size_ = 61; + std::vector buffer_; +}; + +} // namespace LiveLink + +#endif \ No newline at end of file diff --git a/include/livelink/live_link_base.hpp b/include/livelink/live_link_base.hpp new file mode 100644 index 0000000..0b307d2 --- /dev/null +++ b/include/livelink/live_link_base.hpp @@ -0,0 +1,48 @@ +#ifndef LIVE_LINK_BASE_HPP +#define LIVE_LINK_BASE_HPP + +#include "blend_shape_config.hpp" +#include "nlohmann/json.hpp" +#include +#include +#include + +using json = nlohmann::json; + +namespace LiveLink { + +class LiveLinkBase { +public: + using SendCallback = std::function& buffer)>; + using Landmark = std::vector>; + using BlendShape = std::vector; + + explicit LiveLinkBase(const json& config) : json_(config) {} + virtual ~LiveLinkBase() = default; + + virtual void Process(const SendCallback& callback, const Landmark& landmark) = 0; + virtual void Encode() = 0; + + double Distance(const std::vector& x, const std::vector& y, bool threeD = false) const { + if (threeD) { + return std::pow(std::pow(x[0] - y[0], 2) + std::pow(x[1] - y[1], 2) + std::pow(x[2] - y[2], 2), 0.5); + } else { + return std::pow(std::pow(x[0] - y[0], 2) + std::pow(x[1] - y[1], 2), 0.5); + } + } + + double Clamp(double value, double min, double max) const { + return std::max(std::min(value, max), min); + } + + double Remap(double value, double min, double max) const { + return (Clamp(value, min, max) - min) / (max - min); + } + +protected: + const json& json_; +}; + +} // namespace LiveLink + +#endif \ No newline at end of file diff --git a/include/mediapipe.hpp b/include/mediapipe.hpp new file mode 100644 index 0000000..232ea9d --- /dev/null +++ b/include/mediapipe.hpp @@ -0,0 +1,8 @@ +#ifndef MEDIAPIPE_HPP +#define MEDIAPIPE_HPP + +#include "mediapipe_api.h" +#include "mediapipe_interface.h" +#include "mediapipe_log.h" + +#endif \ No newline at end of file diff --git a/include/mediapipe/mediapipe_api.h b/include/mediapipe/mediapipe_api.h new file mode 100644 index 0000000..e98744f --- /dev/null +++ b/include/mediapipe/mediapipe_api.h @@ -0,0 +1,13 @@ +#ifndef MEDIAPIPE_API_H_ +#define MEDIAPIPE_API_H_ + +#include +#include "mediapipe_interface.h" + +#ifndef DllExport +#define DllExport __declspec(dllexport) +#endif + +DllExport MediapipeInterface* CreateMediapipeInterface(); + +#endif \ No newline at end of file diff --git a/include/mediapipe/mediapipe_interface.h b/include/mediapipe/mediapipe_interface.h new file mode 100644 index 0000000..3c223a9 --- /dev/null +++ b/include/mediapipe/mediapipe_interface.h @@ -0,0 +1,26 @@ +#ifndef MEDIAPIPE_INTERFACE_H_ +#define MEDIAPIPE_INTERFACE_H_ + +#include "mediapipe_log.h" +#include +#include + +class MediapipeInterface { +public: + MediapipeInterface() = default; + virtual ~MediapipeInterface() = default; + + using LandmarkCallback = std::function>& landmarks)>; + using MatCallback = std::function; + + virtual void SetLogger(const std::shared_ptr& logger) = 0; + virtual void SetResourceDir(const std::string& path) = 0; + virtual void SetGraph(const std::string& path) = 0; + virtual void Detect(const cv::Mat& frame) = 0; + virtual void CreateObserver(const std::string& name, const LandmarkCallback& callback) = 0; + virtual void OpenPreview(const MatCallback& callback) = 0; + virtual void Start() = 0; + virtual void Stop() = 0; +}; + +#endif \ No newline at end of file diff --git a/include/mediapipe/mediapipe_log.h b/include/mediapipe/mediapipe_log.h new file mode 100644 index 0000000..f74287f --- /dev/null +++ b/include/mediapipe/mediapipe_log.h @@ -0,0 +1,14 @@ +#ifndef MEDIAPIPE_LOG_H_ +#define MEDIAPIPE_LOG_H_ + +#include + +class MediapipeLogger { +public: + MediapipeLogger() = default; + virtual ~MediapipeLogger() = default; + + virtual void Log(const std::string& content) const = 0; +}; + +#endif \ No newline at end of file diff --git a/include/util.hpp b/include/util.hpp new file mode 100644 index 0000000..1528e22 --- /dev/null +++ b/include/util.hpp @@ -0,0 +1,22 @@ +#ifndef UTIL_HPP +#define UTIL_HPP + +#include +#include + +namespace Util { + +template +void ValueToBuffer(T value, std::vector& buffer, bool net = false) { + std::vector data(sizeof(value)); + std::memcpy(data.data(), &value, sizeof(value)); + if (net) { + std::for_each(data.rbegin(), data.rend(), [&](const char c) { buffer.push_back(c); }); + } else { + std::for_each(data.begin(), data.end(), [&](const char c) { buffer.push_back(c); }); + } +} + +} // namespace Util + +#endif diff --git a/main.cpp b/src/main.cpp similarity index 75% rename from main.cpp rename to src/main.cpp index 31fa4fd..462da78 100644 --- a/main.cpp +++ b/src/main.cpp @@ -1,10 +1,14 @@ #include "asio.hpp" #include "livelink.hpp" #include "mediapipe.hpp" +#include "nlohmann/json.hpp" +#include #include #include #include +using json = nlohmann::json; + class Logger : public MediapipeLogger { public: virtual void Log(const std::string& content) const override { @@ -25,14 +29,18 @@ int main() { auto interface = CreateMediapipeInterface(); interface->SetLogger(logger); - LiveLink::FaceLiveLink faceLiveLink; + // init blend shape config + std::ifstream config("blend_shape_config.json"); + auto jsonData = json::parse(config); + + LiveLink::FaceLiveLink faceLiveLink(jsonData); constexpr char windowName[] = "MediaPipe"; cv::namedWindow(windowName); cv::VideoCapture capture; - capture.open("D:/video/pml.mp4"); - bool isCamera = false; + capture.open(0); + bool isCamera = true; cv::Mat outputBGRFrame; bool grabFrames = true; @@ -40,7 +48,7 @@ int main() { logger->Log("VideoCapture is not open"); } interface->SetResourceDir(""); - interface->SetGraph("holistic_tracking_onnx_cuda.pbtxt"); + interface->SetGraph("iris_tracking_cpu.pbtxt"); auto matCallback = [&](const cv::Mat& frame) { cv::cvtColor(frame, outputBGRFrame, cv::COLOR_RGB2BGR); @@ -53,10 +61,15 @@ int main() { }; auto landmarkCallback = [&](const std::vector>& data) { - faceLiveLink.Send(sendCallback); + faceLiveLink.Process(sendCallback, data); + }; + + auto tempCallback = [&](const std::vector>& data) { + // faceLiveLink.Process(sendCallback, data); }; - interface->CreateObserver("face_landmarks", landmarkCallback); + interface->CreateObserver("face_landmarks_with_iris", landmarkCallback); + interface->CreateObserver("face_geometry", tempCallback); interface->Start(); while (grabFrames) {