added gesture creation. added gesture listen. created more gestures.

qihang0233 · Jul 21, 2015 · 0832f04 · 0832f04
1 parent 9a08809
commit 0832f04
Show file tree

Hide file tree

Showing 10 changed files with 160 additions and 114 deletions.
diff --git a/Makefile b/Makefile
@@ -16,15 +16,18 @@ LDFLAGS = -ljpeg -mavx -lm -lpthread -lX11 `pkg-config --libs opencv` -DDLIB_HAV
 # $(VAR) gives value of the variable.
 # $@ stores the target
 # $^ stores the dependency
-all: bin/oic bin/facegesmatch bin/facegescreate
+all: bin/oic bin/facegesmatch bin/facegescreate bin/facegeslisten
 
 bin/oic: obj/dlib.o obj/faceDetection.o obj/pupilDetection.o obj/kalmanFilters.o obj/util.o obj/kmeansUtils.o obj/oic.o
 	$(CXX) -o $@ $^ $(LDFLAGS)
 
 bin/facegescreate: obj/dlib.o obj/faceDetection.o obj/util.o obj/gestureDetection.o obj/facegescreate.o
 	$(CXX) -o $@ $^ $(LDFLAGS)
 
-bin/facegesmatch: obj/dlib.o obj/faceDetection.o obj/util.o obj/facegesmatch.o
+bin/facegesmatch: obj/dlib.o obj/faceDetection.o obj/util.o obj/gestureDetection.o obj/facegesmatch.o
+	$(CXX) -o $@ $^ $(LDFLAGS)
+
+bin/facegeslisten: obj/dlib.o obj/faceDetection.o obj/gestureDetection.o obj/util.o obj/facegeslisten.o
 	$(CXX) -o $@ $^ $(LDFLAGS)
 
 obj/dlib.o: src/dlib/all/source.cpp
@@ -58,7 +61,10 @@ obj/facegesmatch.o: src/facegesMatch.cpp
 obj/facegescreate.o: src/facegesCreate.cpp
 	$(CXX) -c $(CXXFLAGS) -o $@ $<
 
-obj/gestureDetection.o: src/gestureDetection.cpp src/util.cpp
+obj/facegeslisten.o: src/facegesListen.cpp
+	$(CXX) -c $(CXXFLAGS) -o $@ $<
+
+obj/gestureDetection.o: src/gestureDetection.cpp
 	$(CXX) -c $(CXXFLAGS) -o $@ $<
 
 # .PHONY tells make that 'all' or 'clean' aren't _actually_ files, and always

diff --git a/bin/facegescreate b/bin/facegescreate
diff --git a/bin/facegeslisten b/bin/facegeslisten
diff --git a/bin/facegesmatch b/bin/facegesmatch
diff --git a/ges/left_rock.txt b/ges/left_rock.txt
@@ -0,0 +1,10 @@
+-0.08628 -0.143821 -0.985835
+-0.0879976 -0.175979 -0.980453
+-0.121439 -0.151811 -0.980921
+-0.126898 -0.190397 -0.973471
+-0.162617 -0.162563 -0.973206
+-0.151878 -0.121492 -0.980904
+-0.151681 -0.121334 -0.980954
+-0.127055 -0.0317673 -0.991387
+-7.65805e-14 -0.0643828 -0.997925
+-0.0288098 -0.0576145 -0.997923
diff --git a/ges/nod.txt b/ges/nod.txt
@@ -0,0 +1,10 @@
+-0.0975447 -0.130089 -0.986692
+-0.130573 -0.0979078 -0.986592
+-0.130518 -0.0978662 -0.986604
+-0.127166 -0.031795 -0.991372
+-0.0525942 0.0262994 -0.99827
+-0.111594 0.111556 -0.987473
+-0.0779051 0.0519232 -0.995608
+-0.0917627 -0.0305942 -0.995311
+-0.0880408 -0.176066 -0.980433
+-0.129177 -0.193816 -0.972496
diff --git a/ges/right_rock.txt b/ges/right_rock.txt
@@ -0,0 +1,10 @@
+-0.126983 1.00731e-13 -0.991905
+-0.0903152 0.0301116 -0.995458
+-0.0903152 0.0301116 -0.995458
+-0.0903152 0.0301116 -0.995458
+-0.0567651 0.0283851 -0.997984
+0.0889263 0.0296485 -0.995597
+0.0902386 -0.0300861 -0.995466
+-0.0888819 0.0296337 -0.995601
+-0.124999 9.91575e-14 -0.992157
+-0.126983 1.00731e-13 -0.991905
diff --git a/src/facegesCreate.cpp b/src/facegesCreate.cpp
@@ -15,7 +15,6 @@
 
 #include "faceDetection.h"
 #include "util.h"
-#include "gestureDetection.h"
 
 int main(int argc, char **argv) {
 
@@ -24,9 +23,6 @@ int main(int argc, char **argv) {
 	FaceFeatures *face_features = new FaceFeatures();
 	FaceData *face_data = new FaceData();
 	FacePose *face_pose = new FacePose();
-	FaceGesture *face_gesture = new FaceGesture();
-
-	face_gesture->assign(15);
 
 	cv::Mat frame, frame_clr;
 
@@ -35,15 +31,6 @@ int main(int argc, char **argv) {
 
 	dlib::deserialize("./res/shape_predictor_68_face_landmarks.dat") >> pose_model;
 
-	std::vector<std::vector<std::vector<double> > > gestures_learned(2);
-
-	read_vector_from_file(argv[1], gestures_learned[0]);
-	read_vector_from_file(argv[2], gestures_learned[1]);
-
-	int score_temp_1, score_temp_2, pos;
-	std::vector<double> vec_normal(3);
-	std::vector<std::vector<double> > vec_current_bin(0);
-
 	while(1) {
 		try {
 			cap>>frame;
@@ -78,42 +65,15 @@ int main(int argc, char **argv) {
 				face_pose->assign(face_features, face_data);
 
 				std::cout<<face_pose->normal[0]<<" "<<face_pose->normal[1]<<" "<<face_pose->normal[2]<<"\n";
-
-				vec_normal[0] = face_pose->normal[0];
-				vec_normal[1] = face_pose->normal[1];
-				vec_normal[2] = face_pose->normal[2];
-
-				face_gesture->normal->push(vec_normal);
-				//std::cout<<"Filled : "<<face_gesture->normal->get_filled()<<std::endl;
-
-				vec_current_bin = face_gesture->normal->clone();
-				//std::cout<<"Size : "<<vec_current_bin.size()<<std::endl;
-
-				for(int i=0;i<vec_current_bin.size();i++) {
-					std::cout<<vec_current_bin[i][0]<<std::endl;
-				}
-
-				score_temp_1 = DTWScore(vec_current_bin, gestures_learned[0]);
-				pos = 0;
-
-				for(int i=1;i<gestures_learned.size();i++) {
-					score_temp_2 = DTWScore(vec_current_bin, gestures_learned[i]);
-					if(score_temp_2 < score_temp_1) {
-						score_temp_1 = score_temp_2;
-						pos = i;
-					}
-				}
-
-				std::cout<<"Matched with gesture["<<pos<<"]"<<std::endl;
 			}
 			else {
 				std::cout<<"Zero faces"<<std::endl;
 			}
 		}
 		catch(std::exception& e) {
-			std::cout<<e.what()<<std::endl;
 			break;
 		}
 	}
+
 	return 0;
 }
diff --git a/src/facegesListen.cpp b/src/facegesListen.cpp
@@ -0,0 +1,119 @@
+#include <math.h>
+#include <stdlib.h>
+#include <string>
+#include <fstream>
+#include <iostream>
+
+#include <opencv2/highgui/highgui.hpp>
+#include <opencv2/opencv.hpp>
+#include <opencv2/legacy/compat.hpp>
+
+#include "dlib/opencv.h"
+#include "dlib/image_processing/frontal_face_detector.h"
+#include "dlib/image_processing/render_face_detections.h"
+#include "dlib/gui_widgets.h"
+
+#include "faceDetection.h"
+#include "util.h"
+#include "gestureDetection.h"
+
+int main(int argc, char **argv) {
+
+	cv::VideoCapture cap(0);
+
+	FaceFeatures *face_features = new FaceFeatures();
+	FaceData *face_data = new FaceData();
+	FacePose *face_pose = new FacePose();
+	FaceGesture *face_gesture = new FaceGesture();
+
+	face_gesture->assign(15);
+
+	cv::Mat frame, frame_clr;
+
+	dlib::frontal_face_detector detector = dlib::get_frontal_face_detector();
+	dlib::shape_predictor pose_model;
+
+	dlib::deserialize("./res/shape_predictor_68_face_landmarks.dat") >> pose_model;
+
+	std::vector<std::vector<std::vector<double> > > gestures_learned(5);
+
+	read_vector_from_file("left2right_fast.txt", gestures_learned[0]);
+	read_vector_from_file("left_rock.txt", gestures_learned[1]);
+	read_vector_from_file("nod.txt", gestures_learned[2]);
+	read_vector_from_file("right_rock.txt", gestures_learned[3]);
+	read_vector_from_file("up2down_fast.txt", gestures_learned[4]);
+
+	int score_temp_1, score_temp_2, pos;
+	std::vector<double> vec_normal(3);
+	std::vector<std::vector<double> > vec_current_bin(0);
+
+	while(1) {
+		try {
+			cap>>frame;
+			cv::flip(frame, frame, 1);
+			frame.copyTo(frame_clr);
+			cv::cvtColor(frame, frame, CV_BGR2GRAY);
+
+			dlib::cv_image<unsigned char> cimg_gray(frame);
+			dlib::cv_image<dlib::bgr_pixel> cimg_clr(frame_clr);
+
+			std::vector<dlib::rectangle> faces = detector(cimg_gray);
+
+			std::vector<dlib::full_object_detection> shapes;
+			for (unsigned long i = 0; i < faces.size(); ++i)
+				shapes.push_back(pose_model(cimg_gray, faces[i]));
+
+			if(faces.size()) {
+
+				dlib::full_object_detection shape = shapes[0];
+
+				face_features->assign(cv::Point(0,0),
+					get_mid_point(cv::Point(shape.part(42).x(), shape.part(42).y()),
+						cv::Point(shape.part(45).x(), shape.part(45).y())),
+					get_mid_point(cv::Point(shape.part(36).x(), shape.part(36).y()),
+						cv::Point(shape.part(39).x(), shape.part(39).y())),
+					cv::Point(shape.part(30).x(), shape.part(30).y()), 
+					get_mid_point(cv::Point(shape.part(48).x(), shape.part(48).y()),
+						cv::Point(shape.part(54).x(), shape.part(54).y())));
+
+				face_data->assign(face_features);
+
+				face_pose->assign(face_features, face_data);
+
+				std::cout<<face_pose->normal[0]<<" "<<face_pose->normal[1]<<" "<<face_pose->normal[2]<<"\n";
+
+				vec_normal[0] = face_pose->normal[0];
+				vec_normal[1] = face_pose->normal[1];
+				vec_normal[2] = face_pose->normal[2];
+
+				face_gesture->normal->push(vec_normal);
+				//std::cout<<"Filled : "<<face_gesture->normal->get_filled()<<std::endl;
+
+				vec_current_bin = face_gesture->normal->clone();
+				//std::cout<<"Size : "<<vec_current_bin.size()<<std::endl;
+
+				score_temp_1 = DTWScore(vec_current_bin, gestures_learned[0]);
+				pos = 0;
+
+				for(int i=1;i<gestures_learned.size();i++) {
+					score_temp_2 = DTWScore(vec_current_bin, gestures_learned[i]);
+					std::cout<<score_temp_2<<" ";
+					if(score_temp_2 < score_temp_1) {
+						score_temp_1 = score_temp_2;
+						pos = i;
+					}
+				}
+
+				std::cout<<"Matched with gesture["<<pos<<"]"<<std::endl;
+			}
+			else {
+				std::cout<<"Zero faces"<<std::endl;
+			}
+		}
+		catch(std::exception& e) {
+			std::cout<<e.what()<<std::endl;
+			break;
+		}
+	}
+	return 0;
+}
diff --git a/src/facegesMatch.cpp b/src/facegesMatch.cpp
@@ -15,76 +15,7 @@
 
 #include "faceDetection.h"
 #include "util.h"
-
-#define DTW_INFINITY 1e30
-
-double maximum(double a, double b, double c) {
-	if(a>b && a>c) {
-		return a;
-	}
-	else if(b>a && b>c) {
-		return b;
-	}
-	else return c;
-}
-
-double measure_deviation(std::vector<double> arr1, std::vector<double> arr2) {
-
-	/*
-		Function to find deviation/difference between two measured vectors having size = 3.
-
-		@params:
-		arr1	Input array 1
-		arr2	Input array 2
-	*/
-
-	return maximum(std::fabs(arr1[0] - arr2[0]), std::fabs(arr1[1] - arr2[1]), std::fabs(arr1[2] - arr2[2]));
-}
-
-double minimum(double a, double b, double c) {
-	if(a<b && a<c) {
-		return a;
-	}
-	else if(b<a && b<c) {
-		return b;
-	}
-	else return c;
-}
-
-double DTWScore(std::vector<std::vector<double> > arr1, std::vector<std::vector<double> > arr2) {
-
-	/*
-		Function to estimate how close two measurements are. The closeness is denoted by the score, which is computed
-		using the DTW(Dynamic Time Warping) Algorithm. Lower the score, more close they are.
-
-		@params:
-		arr1	Input array 1
-		arr2	Input array 2
-	*/
-
-	int m = arr1.size() - 1;
-	int n = arr2.size() - 1;
-
-	double DTW[m+1][n+1], dev;
-
-	for(int i=1; i<=m; i++) {
-		DTW[i][0] = DTW_INFINITY;
-	}
-	for(int j=1; j<=n; j++) {
-		DTW[0][j] = DTW_INFINITY;
-	}
-	DTW[0][0] = 0;
-
-	for(int i=1; i<=m; i++) {
-		for(int j=1; j<=n; j++) {
-			dev = measure_deviation(arr1[i], arr2[j]);
-			DTW[i][j] = dev + minimum(DTW[i-1][j], DTW[i][j-1], DTW[i-1][j-1]);
-		}
-	}
-
-	//DTW[m][n] is the score.
-	return DTW[m][n];
-}
+#include "gestureDetection.h"
 
 int main(int argc, char **argv) {