Skip to content
This repository has been archived by the owner on Jun 15, 2023. It is now read-only.

眨眼眉毛嘴巴 #1

Merged
merged 2 commits into from
Aug 29, 2022
Merged

眨眼眉毛嘴巴 #1

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
02485cf
dev
liuyulvv Aug 26, 2022
eadfe73
眉毛
liuyulvv Aug 29, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
眉毛
  • Loading branch information
@liuyulvv
liuyulvv committed Aug 29, 2022
commit eadfe734d27d553a13a1e9af17905bd0ea546e18
68 changes: 45 additions & 23 deletions blend_shape_config.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,33 +27,55 @@
"mouth": {
"eye": [
133,
362,
130,
263
362
],
"mouth": [
13,
14,
61,
291
14
],
"low": 0.17,
"high": 0.5
},
"brow": {
"left": [
35,
244,
63,
105,
66,
229,
230,
231
],
"right": [
265,
464,
293,
334,
296,
449,
450,
451
],
"maxRatio": 1.15,
"low": 0.07,
"high": 0.125
},
"pupil": {
"left": [
468,
469,
470,
471,
472
],
"open": {
"low": 0.15,
"high": 0.7
},
"shape": {
"low": 0.45,
"high": 0.9
},
"mouthY": {
"low": 0.17,
"high": 0.5
},
"ratioA": {},
"ratioE": {},
"ratioI": {},
"ratioO": {},
"ratioU": {}
"right": [
473,
474,
475,
476,
477
]
},
"nose": {
"tip": [
Expand Down
122 changes: 84 additions & 38 deletions include/livelink/face_live_link.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,26 +21,28 @@ class FaceLiveLink : public LiveLinkBase {
virtual ~FaceLiveLink() = default;

virtual void Process(const SendCallback& callback, const Landmark& landmark) override {
// 54, 53, 52, 51
json data(blendShape_);
std::ofstream file("res/blendshape/" + std::to_string(index_) + ".json");
file << data;
json data1(landmark);
std::ofstream file1("res/landmark/" + std::to_string(index_++) + ".json");
file1 << data1;
for (int i = 0; i < blendShape_.size(); ++i) {
if (i >= 51 && i <= 54) {
blendShape_[i] = 0;
} else {
std::random_device rd;
std::mt19937 seed(rd());
// std::uniform_int_distribution<> gen(0, 255);
std::uniform_real_distribution<float> gen(0, 1);
blendShape_[i] = gen(seed);
}
}
// UpdateEyeBlink(landmark);
// UpdateMouth(landmark);
// // 54, 53, 52, 51
// json data(blendShape_);
// std::ofstream file("res/blendshape/" + std::to_string(index_) + ".json");
// file << data;
// json data1(landmark);
// std::ofstream file1("res/landmark/" + std::to_string(index_++) + ".json");
// file1 << data1;
// for (int i = 0; i < blendShape_.size(); ++i) {
// if (i >= 51 && i <= 54) {
// blendShape_[i] = 0;
// } else {
// std::random_device rd;
// std::mt19937 seed(rd());
// // std::uniform_int_distribution<> gen(0, 255);
// std::uniform_real_distribution<float> gen(0, 1);
// blendShape_[i] = gen(seed);
// }
// }
UpdateEyeBlink(landmark);
UpdateMouth(landmark);
UpdateBrow(landmark);
UpdatePupil(landmark);
Encode();
callback(buffer_);
}
Expand All @@ -54,36 +56,80 @@ class FaceLiveLink : public LiveLinkBase {
}

private:
void UpdatePupil(const Landmark& landmark) {
auto pupil = json_["pupil"];
auto leftPupilIndex = pupil["left"].get<std::vector<int>>();
auto rightPupilIndex = pupil["right"].get<std::vector<int>>();
auto leftPupil = GetLandmark(landmark, leftPupilIndex);
auto rightPupil = GetLandmark(landmark, rightPupilIndex);

auto eye = json_["eye"];
auto leftEyeIndex = eye["left"].get<std::vector<int>>();
auto rightEyeIndex = eye["right"].get<std::vector<int>>();

auto& leftEyeOuterCorner = landmark[leftEyeIndex[0]];
auto& leftEyeInnerCorner = landmark[leftEyeIndex[1]];

auto& rightEyeOuterCorner = landmark[rightEyeIndex[0]];
auto& rightEyeInnerCorner = landmark[rightEyeIndex[1]];

auto leftEyeWidth = Distance(leftEyeOuterCorner, leftEyeInnerCorner);
auto rightEyeWidth = Distance(rightEyeOuterCorner, rightEyeInnerCorner);

auto leftEyeMidPoint = Average(leftEyeOuterCorner, leftEyeInnerCorner);
auto rightEyeMidPoint = Average(rightEyeOuterCorner, rightEyeInnerCorner);

auto leftDx = leftEyeMidPoint[0] - leftPupil[0][0];
auto leftDy = leftEyeMidPoint[1] - leftPupil[0][1];

auto rightDx = rightEyeMidPoint[0] - rightPupil[0][0];
auto rightDy = rightEyeMidPoint[1] - rightPupil[0][1];

auto leftRatioX = leftDx / (leftEyeWidth / 2) * 4;
auto rightRatioX = rightDx / (rightEyeWidth / 2) * 4;

auto leftRatioY = leftDy / (leftEyeWidth / 4) * 4;
auto rightRatioY = rightDy / (rightEyeWidth / 4) * 4;

// if (leftRatioX >= 1) {
// blendShape_[int(FaceBlendShape::)]
// }

std::cout << leftRatioX << " " << rightRatioX << " " << leftRatioY << " " << rightRatioY << std::endl;
}

void UpdateBrow(const Landmark& landmark) {
auto brow = json_["brow"];
auto maxRatio = brow["maxRatio"].get<double>();
auto low = brow["low"].get<double>();
auto high = brow["high"].get<double>();
auto leftBrowIndex = brow["left"].get<std::vector<int>>();
auto rightBrowIndex = brow["right"].get<std::vector<int>>();
auto leftBrowDistance = GetEyeLidRatio(landmark, leftBrowIndex);
auto rightBrowDistance = GetEyeLidRatio(landmark, rightBrowIndex);
auto leftBrowRatio = leftBrowDistance / maxRatio - 1;
auto rightBrowRatio = leftBrowDistance / maxRatio - 1;
auto leftRaiseRatio = Remap(leftBrowRatio, low, high);
auto rightRaiseRatio = Remap(rightBrowRatio, low, high);
blendShape_[int(FaceBlendShape::BrowOuterUpLeft)] = leftRaiseRatio;
blendShape_[int(FaceBlendShape::BrowOuterUpRight)] = rightRaiseRatio;
// std::cout << leftRaiseRatio << " " << rightRaiseRatio << std::endl;
}

void UpdateMouth(const Landmark& landmark) {
auto mouth = json_["mouth"];
auto eyeIndex = mouth["eye"].get<std::vector<int>>();
auto mouthIndex = mouth["mouth"].get<std::vector<int>>();

auto eyeLandmark = GetLandmark(landmark, eyeIndex);
auto eyeInnerDistance = Distance(eyeLandmark[0], eyeLandmark[1]);
auto eyeOuterDistance = Distance(eyeLandmark[2], eyeLandmark[3]);

auto mouthLandmark = GetLandmark(landmark, mouthIndex);
auto mouthOpen = Distance(mouthLandmark[0], mouthLandmark[1]);
auto mouthWidth = Distance(mouthLandmark[2], mouthLandmark[3]);

auto ratioY = mouthOpen / eyeInnerDistance;
auto ratioX = mouthWidth / eyeOuterDistance;

// ratioY = Remap(ratioY, mouth["open"]["low"].get<double>(), mouth["open"]["high"].get<double>());
// ratioX = Remap(ratioX, mouth["shape"]["low"].get<double>(), mouth["shape"]["high"].get<double>());
// ratioX = (ratioX - 0.3) * 2;
auto mouthY = Remap(mouthOpen / eyeInnerDistance, mouth["mouthY"]["low"].get<double>(), mouth["mouthY"]["high"].get<double>());
// auto mouthX = ratioX;
// auto ratioI = Clamp(Remap(mouthX, 0, 1) * 2 * Remap(mouthY, 0.2, 0.7), 0, 1);
// auto ratioA = mouthY * 0.4 + mouthY * (1 - ratioI) * 0.6;
// auto ratioU = mouthY + Remap(1 - ratioI, 0, 0.3) * 0.1;
// auto ratioE = Remap(ratioU, 0.2, 1) * (1 - ratioI) * 0.3;
// auto ratioO = (1 - ratioI) * Remap(mouthY, 0.3, 1) * 0.4;
blendShape_[int(FaceBlendShape::MouthClose)] = -0.3f; // ??? 传输负数没有效果

auto mouthY = Remap(mouthOpen / eyeInnerDistance, mouth["low"].get<double>(), mouth["high"].get<double>());
blendShape_[int(FaceBlendShape::MouthLowerDownLeft)] = mouthY;
blendShape_[int(FaceBlendShape::MouthLowerDownRight)] = mouthY;
// std::cout << ratioA << " " << ratioE << " " << ratioI << " " << ratioO << " " << ratioU << " " << (mouthY || 0) << std::endl;
}

void UpdateEyeBlink(const Landmark& landmark) {
Expand Down
11 changes: 3 additions & 8 deletions src/main.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ int main() {
cv::namedWindow(windowName);
cv::VideoCapture capture;

capture.open(1);
capture.open(0);
bool isCamera = true;

cv::Mat outputBGRFrame;
Expand All @@ -51,8 +51,6 @@ int main() {
interface->SetResourceDir("");
interface->SetGraph("iris_tracking_cpu.pbtxt");

int i = 1;

auto matCallback = [&](const cv::Mat& frame) {
cv::cvtColor(frame, outputBGRFrame, cv::COLOR_RGB2BGR);
cv::imshow(windowName, outputBGRFrame);
Expand All @@ -63,13 +61,10 @@ int main() {
socket.send_to(asio::buffer(buffer), serverEndpoint);
};

faceLiveLink.Renew(sendCallback);

auto landmarkCallback = [&](const std::vector<std::vector<double>>& data) {
++i;
faceLiveLink.Process(sendCallback, data);
if (i == 51) {
faceLiveLink.Renew(sendCallback);
exit(0);
}
};

auto tempCallback = [&](const std::vector<std::vector<double>>& data) {
Expand Down