眨眼

CMakeLists.txt

@@ -12,6 +12,7 @@ include_directories(
     "./include"
     "./include/livelink"
     "./include/mediapipe"
+    "./include/nlohmann"
     "./include/blendshape"
 )
 
@@ -22,10 +23,19 @@ link_directories(
     "./lib"
 )
 
-set (SRC ./main.cpp)
-
-add_executable(${PROJECT_NAME} ${SRC})
-
-target_link_libraries(${PROJECT_NAME} opencv_world3410 onnxruntime onnxruntime_providers_cuda onnxruntime_providers_shared onnxruntime_providers_tensorrt mediapipe)
-
-SET(EXECUTABLE_OUTPUT_PATH ${PROJECT_SOURCE_DIR}/bin)
+SET(EXECUTABLE_OUTPUT_PATH ${PROJECT_SOURCE_DIR}/bin)
+
+file(GLOB srcs RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}/src/*.cpp")
+
+foreach(mainfile IN LISTS srcs)
+    get_filename_component(srcname ${mainfile} NAME_WE)
+    add_executable(${srcname} ${mainfile})
+    target_link_libraries(${srcname}
+        opencv_world3410
+        onnxruntime
+        onnxruntime_providers_cuda
+        onnxruntime_providers_shared
+        onnxruntime_providers_tensorrt
+        mediapipe
+    )
+endforeach()

blend_shape_config.json

@@ -0,0 +1,27 @@
+{
+    "eye": {
+        "left": [
+            33,
+            133,
+            160,
+            159,
+            158,
+            144,
+            145,
+            153
+        ],
+        "right": [
+            263,
+            362,
+            387,
+            386,
+            385,
+            373,
+            374,
+            380
+        ],
+        "low": 0.6,
+        "high": 1.0,
+        "maxRatio": 0.285
+    }
+}

include/livelink.hpp

@@ -0,0 +1,8 @@
+#ifndef LIVELINK_HPP
+#define LIVELINK_HPP
+
+#include "livelink/blend_shape_config.hpp"
+#include "livelink/face_live_link.hpp"
+#include "livelink/live_link_base.hpp"
+
+#endif

include/livelink/blend_shape_config.hpp

@@ -0,0 +1,72 @@
+#ifndef BLEND_SHAPE_CONFIG_HPP
+#define BLEND_SHAPE_CONFIG_HPP
+
+namespace LiveLink {
+
+enum class FaceBlendShape {
+    EyeBlinkLeft,
+    EyeLookDownLeft,
+    EyeLookInLeft,
+    EyeLookOutLeft,
+    EyeLookUpLeft,
+    EyeSquintLeft,
+    EyeWideLeft,
+    EyeBlinkRight,
+    EyeLookDownRight,
+    EyeLookInRight,
+    EyeLookOutRight,
+    EyeLookUpRight,
+    EyeSquintRight,
+    EyeWideRight,
+    JawForward,
+    JawLeft,
+    JawRight,
+    JawOpen,
+    MouthClose,
+    MouthFunnel,
+    MouthPucker,
+    MouthLeft,
+    MouthRight,
+    MouthSmileLeft,
+    MouthSmileRight,
+    MouthFrownLeft,
+    MouthFrownRight,
+    MouthDimpleLeft,
+    MouthDimpleRight,
+    MouthStretchLeft,
+    MouthStretchRight,
+    MouthRollLower,
+    MouthRollUpper,
+    MouthShrugLower,
+    MouthShrugUpper,
+    MouthPressLeft,
+    MouthPressRight,
+    MouthLowerDownLeft,
+    MouthLowerDownRight,
+    MouthUpperUpLeft,
+    MouthUpperUpRight,
+    BrowDownLeft,
+    BrowDownRight,
+    BrowInnerUp,
+    BrowOuterUpLeft,
+    BrowOuterUpRight,
+    CheekPuff,
+    CheekSquintLeft,
+    CheekSquintRight,
+    NoseSneerLeft,
+    NoseSneerRight,
+    TongueOut,
+    HeadYaw,
+    HeadPitch,
+    HeadRoll,
+    LeftEyeYaw,
+    LeftEyePitch,
+    LeftEyeRoll,
+    RightEyeYaw,
+    RightEyePitch,
+    RightEyeRoll
+};
+
+}  // namespace LiveLink
+
+#endif

include/livelink/face_live_link.hpp

@@ -0,0 +1,89 @@
+#ifndef FACE_LIVE_LINK_HPP
+#define FACE_LIVE_LINK_HPP
+
+#include "live_link_base.hpp"
+#include "util.hpp"
+#include <iostream>
+#include <vector>
+
+namespace LiveLink {
+
+class FaceLiveLink : public LiveLinkBase {
+public:
+    FaceLiveLink(const json& config, int size = 61) : LiveLinkBase(config) {
+        blendShape_ = std::vector<float>(size, 0);
+    }
+
+    virtual ~FaceLiveLink() = default;
+
+    virtual void Process(const SendCallback& callback, const Landmark& landmark) override {
+        UpdateEyeOpen(landmark);
+        Encode();
+        callback(buffer_);
+    }
+
+private:
+    void UpdateEyeOpen(const Landmark& landmark) {
+        auto eye = json_["eye"];
+        auto leftIndex = eye["left"].get<std::vector<int>>();
+        auto rightIndex = eye["right"].get<std::vector<int>>();
+        auto maxRatio = eye["maxRatio"].get<double>();
+        auto low = eye["low"].get<double>();
+        auto high = eye["high"].get<double>();
+        auto leftRatio = Clamp(GetEyeLidRatio(landmark, leftIndex) / maxRatio, 0, 2);
+        auto rightRatio = Clamp(GetEyeLidRatio(landmark, rightIndex) / maxRatio, 0, 2);
+        auto leftRes = Remap(leftRatio, low, high);
+        auto rightRes = Remap(rightRatio, low, high);
+        blendShape_[int(FaceBlendShape::EyeBlinkLeft)] = (1 - leftRes) || 0;
+        blendShape_[int(FaceBlendShape::EyeBlinkRight)] = (1 - rightRes) || 0;
+    }
+
+    double GetEyeLidRatio(const Landmark& landmark, const std::vector<int>& index) const {
+        auto& eyeOuterCorner = landmark[index[0]];
+        auto& eyeInnerCorner = landmark[index[1]];
+        auto& eyeOuterUpperLid = landmark[index[2]];
+        auto& eyeMidUpperLid = landmark[index[3]];
+        auto& eyeInnerUpperLid = landmark[index[4]];
+        auto& eyeOuterLowerLid = landmark[index[5]];
+        auto& eyeMidLowerLid = landmark[index[6]];
+        auto& eyeInnerLowerLid = landmark[index[7]];
+        auto eyeWidth = Distance(eyeOuterCorner, eyeInnerCorner);
+        auto eyeOuterLidDistance = Distance(eyeOuterUpperLid, eyeOuterLowerLid);
+        auto eyeMidLidDistance = Distance(eyeMidUpperLid, eyeMidLowerLid);
+        auto eyeInnerLidDistance = Distance(eyeInnerUpperLid, eyeInnerLowerLid);
+        auto eyeLidAvg = (eyeOuterLidDistance + eyeMidLidDistance + eyeInnerLidDistance) / 3;
+        return eyeLidAvg / eyeWidth;
+    }
+
+    virtual void Encode() override {
+        buffer_.clear();
+        Util::ValueToBuffer(6, buffer_, false);  // version
+        for (const auto& c : uuid_) {            // uuid
+            buffer_.push_back(c);
+        }
+        Util::ValueToBuffer(int(name_.size()), buffer_, true);  // name length
+        for (const auto& c : name_) {                           // name
+            buffer_.push_back(c);
+        }
+        Util::ValueToBuffer(0, buffer_, true);                                          // ???
+        Util::ValueToBuffer(0, buffer_, true);                                          // ???
+        Util::ValueToBuffer(60, buffer_, true);                                         // fps
+        Util::ValueToBuffer(1, buffer_, true);                                          // int(fps/60)
+        buffer_.push_back(0x3d);                                                        // blendshape length
+        std::for_each(blendShape_.begin(), blendShape_.end(), [&](const float value) {  // blendshape data
+            Util::ValueToBuffer(value, buffer_, true);
+        });
+    }
+
+    BlendShape blendShape_;
+    int version_ = 6;
+    std::string uuid_ = "$fcaf7061-2964-459b-87a9-ad78290e21e6";
+    std::string name_ = "vzan";
+    int fps = 60;
+    int size_ = 61;
+    std::vector<char> buffer_;
+};
+
+}  // namespace LiveLink
+
+#endif

include/livelink/live_link_base.hpp

@@ -0,0 +1,48 @@
+#ifndef LIVE_LINK_BASE_HPP
+#define LIVE_LINK_BASE_HPP
+
+#include "blend_shape_config.hpp"
+#include "nlohmann/json.hpp"
+#include <functional>
+#include <string>
+#include <vector>
+
+using json = nlohmann::json;
+
+namespace LiveLink {
+
+class LiveLinkBase {
+public:
+    using SendCallback = std::function<void(const std::vector<char>& buffer)>;
+    using Landmark = std::vector<std::vector<double>>;
+    using BlendShape = std::vector<float>;
+
+    explicit LiveLinkBase(const json& config) : json_(config) {}
+    virtual ~LiveLinkBase() = default;
+
+    virtual void Process(const SendCallback& callback, const Landmark& landmark) = 0;
+    virtual void Encode() = 0;
+
+    double Distance(const std::vector<double>& x, const std::vector<double>& y, bool threeD = false) const {
+        if (threeD) {
+            return std::pow(std::pow(x[0] - y[0], 2) + std::pow(x[1] - y[1], 2) + std::pow(x[2] - y[2], 2), 0.5);
+        } else {
+            return std::pow(std::pow(x[0] - y[0], 2) + std::pow(x[1] - y[1], 2), 0.5);
+        }
+    }
+
+    double Clamp(double value, double min, double max) const {
+        return std::max(std::min(value, max), min);
+    }
+
+    double Remap(double value, double min, double max) const {
+        return (Clamp(value, min, max) - min) / (max - min);
+    }
+
+protected:
+    const json& json_;
+};
+
+}  // namespace LiveLink
+
+#endif

include/mediapipe.hpp

@@ -0,0 +1,8 @@
+#ifndef MEDIAPIPE_HPP
+#define MEDIAPIPE_HPP
+
+#include "mediapipe_api.h"
+#include "mediapipe_interface.h"
+#include "mediapipe_log.h"
+
+#endif

include/mediapipe/mediapipe_api.h

@@ -0,0 +1,13 @@
+#ifndef MEDIAPIPE_API_H_
+#define MEDIAPIPE_API_H_
+
+#include <memory>
+#include "mediapipe_interface.h"
+
+#ifndef DllExport
+#define DllExport __declspec(dllexport)
+#endif
+
+DllExport MediapipeInterface* CreateMediapipeInterface();
+
+#endif

include/mediapipe/mediapipe_interface.h

@@ -0,0 +1,26 @@
+#ifndef MEDIAPIPE_INTERFACE_H_
+#define MEDIAPIPE_INTERFACE_H_
+
+#include "mediapipe_log.h"
+#include <opencv2/opencv.hpp>
+#include <functional>
+
+class MediapipeInterface {
+public:
+    MediapipeInterface() = default;
+    virtual ~MediapipeInterface() = default;
+
+    using LandmarkCallback = std::function<void(std::vector<std::vector<double>>& landmarks)>;
+    using MatCallback = std::function<void(cv::Mat& frame)>;
+
+    virtual void SetLogger(const std::shared_ptr<MediapipeLogger>& logger) = 0;
+    virtual void SetResourceDir(const std::string& path) = 0;
+    virtual void SetGraph(const std::string& path) = 0;
+    virtual void Detect(const cv::Mat& frame) = 0;
+    virtual void CreateObserver(const std::string& name, const LandmarkCallback& callback) = 0;
+    virtual void OpenPreview(const MatCallback& callback) = 0;
+    virtual void Start() = 0;
+    virtual void Stop() = 0;
+};
+
+#endif

include/mediapipe/mediapipe_log.h

@@ -0,0 +1,14 @@
+#ifndef MEDIAPIPE_LOG_H_
+#define MEDIAPIPE_LOG_H_
+
+#include <string>
+
+class MediapipeLogger {
+public:
+    MediapipeLogger() = default;
+    virtual ~MediapipeLogger() = default;
+
+    virtual void Log(const std::string& content) const = 0;
+};
+
+#endif

include/util.hpp

@@ -0,0 +1,22 @@
+#ifndef UTIL_HPP
+#define UTIL_HPP
+
+#include <iostream>
+#include <vector>
+
+namespace Util {
+
+template <typename T>
+void ValueToBuffer(T value, std::vector<char>& buffer, bool net = false) {
+    std::vector<char> data(sizeof(value));
+    std::memcpy(data.data(), &value, sizeof(value));
+    if (net) {
+        std::for_each(data.rbegin(), data.rend(), [&](const char c) { buffer.push_back(c); });
+    } else {
+        std::for_each(data.begin(), data.end(), [&](const char c) { buffer.push_back(c); });
+    }
+}
+
+}  // namespace Util
+
+#endif