started working on face gesture recognition.

qihang0233 · Jul 18, 2015 · ebca10d · ebca10d
1 parent 2d82aa9
commit ebca10d
Show file tree

Hide file tree

Showing 23 changed files with 307 additions and 64 deletions.
diff --git a/Makefile b/Makefile
@@ -16,11 +16,17 @@ LDFLAGS = -ljpeg -mavx -lm -lpthread -lX11 `pkg-config --libs opencv` -DDLIB_HAV
 # $(VAR) gives value of the variable.
 # $@ stores the target
 # $^ stores the dependency
-all: bin/oic
+all: bin/oic bin/facegesmatch bin/facegescreate
 
 bin/oic: obj/dlib.o obj/faceDetection.o obj/pupilDetection.o obj/kalmanFilters.o obj/util.o obj/kmeansUtils.o obj/oic.o
 	$(CXX) -o $@ $^ $(LDFLAGS)
 
+bin/facegescreate: obj/dlib.o obj/faceDetection.o obj/util.o obj/facegescreate.o
+	$(CXX) -o $@ $^ $(LDFLAGS)
+
+bin/facegesmatch: obj/dlib.o obj/faceDetection.o obj/util.o obj/facegesmatch.o
+	$(CXX) -o $@ $^ $(LDFLAGS)
+
 obj/dlib.o: src/dlib/all/source.cpp
 	mkdir -p obj bin
 	$(CXX) -c $(CXXFLAGS) -o $@ $<
@@ -46,6 +52,12 @@ obj/kmeansUtils.o: src/kmeansUtils.cpp
 obj/oic.o: src/oic.cpp
 	$(CXX) -c $(CXXFLAGS) -o $@ $<
 
+obj/facegesmatch.o: src/facegesMatch.cpp
+	$(CXX) -c $(CXXFLAGS) -o $@ $<
+
+obj/facegescreate.o: src/facegesCreate.cpp
+	$(CXX) -c $(CXXFLAGS) -o $@ $<
+
 # .PHONY tells make that 'all' or 'clean' aren't _actually_ files, and always
 # execute the compilation action when 'make all' or 'make clean' are used
 .PHONY: all oic

diff --git a/Makefile~ b/Makefile~
diff --git a/bin/facegescreate b/bin/facegescreate
diff --git a/bin/facegesmatch b/bin/facegesmatch
diff --git a/bin/oic b/bin/oic
diff --git a/ges/left2right_fast.txt b/ges/left2right_fast.txt
@@ -0,0 +1,9 @@
+-0.0261852 -0.0261765 -0.999314
+-0.0531895 -0.0177337 -0.998427
+-0.106346 -0.0354562 -0.993697
+-0.527942 -0.0391 -0.84838
+-0.435288 -0.0757005 -0.897103
+-0.473714 0.0378908 -0.879863
+0.103073 0.0515412 -0.993338
+0.249236 -0.0191687 -0.968253
+0.170367 0.0189272 -0.985199
diff --git a/ges/left2right_slow.txt b/ges/left2right_slow.txt
@@ -0,0 +1,28 @@
+-0.0331207 0.0165619 -0.999314
+-0.0264294 -0.0264206 -0.999301
+-0.0466407 -0.0310857 -0.998428
+-0.0753685 -0.0565137 -0.995553
+-0.106339 -0.035454 -0.993698
+-0.0890488 -0.0712331 -0.993477
+-0.102103 -0.0850901 -0.991128
+-0.178882 -0.0536744 -0.982405
+-0.227623 0.0189655 -0.973565
+-0.273769 -0.109532 -0.955538
+-0.299362 -0.0561305 -0.952487
+-0.273058 -0.0728258 -0.959237
+-0.275916 0.148571 -0.94963
+-0.268546 -0.0191785 -0.963076
+-0.2212 -0.0368654 -0.974531
+-0.16877 -0.0187498 -0.985477
+-0.150938 1.19734e-13 -0.988543
+-0.109561 0.0182595 -0.993812
+-0.0550456 4.36658e-14 -0.998484
+7.21032e-15 0.0181788 -0.999835
+0.0304981 0.045759 -0.998487
+0.120082 -0.0514742 -0.991429
+0.185184 0 -0.982704
+0.226413 0 -0.974031
+0.260969 0.0186373 -0.965167
+0.288457 0 -0.957493
+0.23633 -0.0545422 -0.970141
+
diff --git a/ges/up2down_fast.txt b/ges/up2down_fast.txt
@@ -0,0 +1,8 @@
+2.87701e-14 0.0725358 -0.997366
+-0.0174386 0.0697465 -0.997412
+-0.0167098 0.0334166 -0.999302
+-0.0183973 -0.05518 -0.998307
+-0.0363338 -0.272529 -0.961461
+-0.0540409 -0.19812 -0.978687
+-0.117233 -0.039086 -0.992335
+3.65693e-14 0.0921994 -0.995741
diff --git a/ges/up2down_slow.txt b/ges/up2down_slow.txt
@@ -0,0 +1,25 @@
+-0.0310636 0.0466075 -0.99843
+-0.0333616 0.0667173 -0.997214
+-0.0167105 -0.033418 -0.999302
+-6.72204e-14 -0.0565136 -0.998402
+-0.0377358 2.99345e-14 -0.999288
+-0.0133428 -0.0133383 -0.999822
+-1.11718e-13 -0.0939239 -0.995579
+-2.00979e-13 -0.168967 -0.985622
+0.0186724 -0.224105 -0.974386
+-0.0185102 -0.222159 -0.974835
+-0.0360191 -0.216122 -0.975702
+-0.091791 -0.201906 -0.975094
+-2.85909e-13 -0.240369 -0.970681
+-0.0180742 -0.198847 -0.979864
+-1.95553e-13 -0.164406 -0.986393
+-0.111485 -0.16727 -0.979588
+0.0189418 -0.246286 -0.969012
+0.0370328 -0.203683 -0.978336
+0.0370462 -0.185226 -0.981997
+0.0186905 -0.130844 -0.991227
+0 0 -1
+-2.26522e-14 -0.0190442 -0.999819
+2.20014e-14 0.0554705 -0.99846
+0.0373831 0 -0.999301
+0.0132181 0.0132137 -0.999825
diff --git a/include/kmeansUtils.h b/include/kmeansUtils.h
@@ -1,7 +1,7 @@
 #ifndef KMEANS_UTILs_H
 #define KMEANS_UTILs_H
 
-void kmeans_array_generate(cv::Mat src, std::vector<std::vector<double> >& vec, int mode);
-void kmeans_clusters_view(cv::Mat& src, cv::Mat labels);
+void kmeans_array_generate(cv::Mat src, std::vector<float >& vec, int mode);
+void kmeans_clusters_view(cv::Mat& src, std::vector<int> labels);
 
 #endif
diff --git a/obj/dlib.o b/obj/dlib.o
diff --git a/obj/faceDetection.o b/obj/faceDetection.o
diff --git a/obj/faceges.o b/obj/faceges.o
diff --git a/obj/facegescreate.o b/obj/facegescreate.o
diff --git a/obj/facegesmatch.o b/obj/facegesmatch.o
diff --git a/obj/kalmanFilters.o b/obj/kalmanFilters.o
diff --git a/obj/kmeansUtils.o b/obj/kmeansUtils.o
diff --git a/obj/oic.o b/obj/oic.o
diff --git a/obj/pupilDetection.o b/obj/pupilDetection.o
diff --git a/obj/util.o b/obj/util.o
diff --git a/src/facegesCreate.cpp b/src/facegesCreate.cpp
@@ -0,0 +1,80 @@
+#include <math.h>
+#include <stdlib.h>
+#include <string>
+#include <fstream>
+#include <iostream>
+
+#include <opencv2/highgui/highgui.hpp>
+#include <opencv2/opencv.hpp>
+#include <opencv2/legacy/compat.hpp>
+
+#include "dlib/opencv.h"
+#include "dlib/image_processing/frontal_face_detector.h"
+#include "dlib/image_processing/render_face_detections.h"
+#include "dlib/gui_widgets.h"
+
+#include "faceDetection.h"
+#include "util.h"
+
+int main(int argc, char **argv) {
+
+	std::ofstream file_out(argv[1], std::ios::out);
+	if(file_out) {
+		std::cout<<"file opened";
+	}
+
+	cv::VideoCapture cap(0);
+
+	FaceFeatures *face_features = new FaceFeatures();
+	FaceData *face_data = new FaceData();
+	FacePose *face_pose = new FacePose();
+
+	cv::Mat frame, frame_clr;
+
+	dlib::frontal_face_detector detector = dlib::get_frontal_face_detector();
+	dlib::shape_predictor pose_model;
+
+	dlib::deserialize("./res/shape_predictor_68_face_landmarks.dat") >> pose_model;
+
+	while(1) {
+		try {
+			cap>>frame;
+			cv::flip(frame, frame, 1);
+			frame.copyTo(frame_clr);
+			cv::cvtColor(frame, frame, CV_BGR2GRAY);
+
+			dlib::cv_image<unsigned char> cimg_gray(frame);
+			dlib::cv_image<dlib::bgr_pixel> cimg_clr(frame_clr);
+
+			std::vector<dlib::rectangle> faces = detector(cimg_gray);
+
+			std::vector<dlib::full_object_detection> shapes;
+			for (unsigned long i = 0; i < faces.size(); ++i)
+				shapes.push_back(pose_model(cimg_gray, faces[i]));
+
+			dlib::full_object_detection shape = shapes[0];
+
+			face_features->assign(cv::Point(0,0),
+				get_mid_point(cv::Point(shape.part(42).x(), shape.part(42).y()),
+					cv::Point(shape.part(45).x(), shape.part(45).y())),
+				get_mid_point(cv::Point(shape.part(36).x(), shape.part(36).y()),
+					cv::Point(shape.part(39).x(), shape.part(39).y())),
+				cv::Point(shape.part(30).x(), shape.part(30).y()), 
+				get_mid_point(cv::Point(shape.part(48).x(), shape.part(48).y()),
+					cv::Point(shape.part(54).x(), shape.part(54).y())));
+
+			face_data->assign(face_features);
+
+			face_pose->assign(face_features, face_data);
+
+			std::cout<<face_pose->normal[0]<<" "<<face_pose->normal[1]<<" "<<face_pose->normal[2]<<"\n";
+			file_out<<face_pose->normal[0]<<" "<<face_pose->normal[1]<<" "<<face_pose->normal[2]<<"\n";
+		}
+		catch(std::exception& e) {
+			file_out.close();
+			break;
+		}
+	}
+
+	return 0;
+}
diff --git a/src/facegesMatch.cpp b/src/facegesMatch.cpp
@@ -0,0 +1,125 @@
+#include <math.h>
+#include <stdlib.h>
+#include <string>
+#include <fstream>
+#include <sstream>
+
+#include <opencv2/highgui/highgui.hpp>
+#include <opencv2/opencv.hpp>
+#include <opencv2/legacy/compat.hpp>
+
+#include "dlib/opencv.h"
+#include "dlib/image_processing/frontal_face_detector.h"
+#include "dlib/image_processing/render_face_detections.h"
+#include "dlib/gui_widgets.h"
+
+#include "faceDetection.h"
+#include "util.h"
+
+#define DTW_INFINITY 1e30
+
+double maximum(double a, double b, double c) {
+	if(a>b && a>c) {
+		return a;
+	}
+	else if(b>a && b>c) {
+		return b;
+	}
+	else return c;
+}
+
+double measure_deviation(std::vector<double> arr1, std::vector<double> arr2) {
+
+	/*
+		Function to find deviation/difference between two measured vectors having size = 3.
+
+		@params:
+		arr1	Input array 1
+		arr2	Input array 2
+	*/
+
+	return maximum(std::fabs(arr1[0] - arr2[0]), std::fabs(arr1[1] - arr2[1]), std::fabs(arr1[2] - arr2[2]));
+}
+
+double minimum(double a, double b, double c) {
+	if(a<b && a<c) {
+		return a;
+	}
+	else if(b<a && b<c) {
+		return b;
+	}
+	else return c;
+}
+
+double DTWScore(std::vector<std::vector<double> > arr1, std::vector<std::vector<double> > arr2) {
+
+	/*
+		Function to estimate how close two measurements are. The closeness is denoted by the score, which is computed
+		using the DTW(Dynamic Time Warping) Algorithm. Lower the score, more close they are.
+
+		@params:
+		arr1	Input array 1
+		arr2	Input array 2
+	*/
+
+	int m = arr1.size() - 1;
+	int n = arr2.size() - 1;
+
+	double DTW[m+1][n+1], dev;
+
+	for(int i=1; i<=m; i++) {
+		DTW[i][0] = DTW_INFINITY;
+	}
+	for(int j=1; j<=n; j++) {
+		DTW[0][j] = DTW_INFINITY;
+	}
+	DTW[0][0] = 0;
+
+	for(int i=1; i<=m; i++) {
+		for(int j=1; j<=n; j++) {
+			dev = measure_deviation(arr1[i], arr2[j]);
+			DTW[i][j] = dev + minimum(DTW[i-1][j], DTW[i][j-1], DTW[i-1][j-1]);
+		}
+	}
+
+	//DTW[m][n] is the score.
+	return DTW[m][n];
+}
+
+void read_vector_from_file(char* file_name, std::vector<std::vector<double> >& arr) {
+
+	/*
+		Function to read a vector of measurements from a text file.
+
+		@params:
+		file_name	Name of the file to be read
+		arr 		Vector to be constructed, passed by reference
+	*/
+
+	std::ifstream file_in(file_name);
+	std::string line;
+
+	while(std::getline(file_in, line)) {
+		std::istringstream iss(line);
+		std::vector<double> vec(3);
+
+		if(!(iss >> vec[0] >> vec[1] >> vec[2])) {
+			break;
+		}
+
+		//std::cout<<vec[0]<<" "<<vec[1]<<" "<<vec[2]<<std::endl;
+		arr.push_back(vec);
+	}
+}
+
+int main(int argc, char **argv) {
+
+	std::vector<std::vector<double> > arr1, arr2;
+
+	read_vector_from_file(argv[1], arr1);
+	read_vector_from_file(argv[2], arr2);
+
+	std::cout<<"DTW Score : "<<DTWScore(arr1, arr2)<<std::endl;
+
+	return 0;
+}