From 58da7275a659c26b6a1fd9ec07e76f4cf521375e Mon Sep 17 00:00:00 2001
From: houxiao <houxiao@454eff88-639b-444f-9e54-f578c98de674>
Date: 星期三, 04 一月 2017 18:16:35 +0800
Subject: [PATCH] add dlib face trace

---
 RtspFace/make.sh                   |   44 +++--
 RtspFace/PL_SensetimeFaceTrack.h   |    6 
 RtspFace/PL_SensetimeFaceTrack.cpp |    4 
 RtspFace/PL_DlibFaceTrack.cpp      |  309 ++++++++++++++++++++++++++++++++++++++++++++
 RtspFace/main.cpp                  |   18 ++
 RtspFace/PL_AVFrameBGRA.cpp        |    4 
 RtspFace/PL_DlibFaceTrack.h        |   33 ++++
 7 files changed, 390 insertions(+), 28 deletions(-)

diff --git a/RtspFace/PL_AVFrameBGRA.cpp b/RtspFace/PL_AVFrameBGRA.cpp
index 7ec2d11..0826c5b 100644
--- a/RtspFace/PL_AVFrameBGRA.cpp
+++ b/RtspFace/PL_AVFrameBGRA.cpp
@@ -7,10 +7,10 @@
 	#include <libavcodec/avcodec.h>
 	#include <libavutil/frame.h>
 	#include <libavformat/avformat.h>
-	
-	#include <libyuv.h>
 }
 
+#include <libyuv.h>
+
 struct PL_AVFrameBGRA_Internal
 {
 	uint8_t buffer[1920*1080*4];//#todo from config
diff --git a/RtspFace/PL_DlibFaceTrack.cpp b/RtspFace/PL_DlibFaceTrack.cpp
new file mode 100644
index 0000000..5c5dcea
--- /dev/null
+++ b/RtspFace/PL_DlibFaceTrack.cpp
@@ -0,0 +1,309 @@
+#include "PL_DlibFaceTrack.h"
+#include "MaterialBuffer.h"
+#include "logger.h"
+
+#include <opencv2/opencv.hpp>
+#include <opencv2/highgui/highgui.hpp>
+
+#include <dlib/opencv.h>
+#include <dlib/image_processing/frontal_face_detector.h>
+#include <dlib/image_processing/render_face_detections.h>
+#include <dlib/image_processing.h>
+#include <dlib/gui_widgets.h>
+
+#include <libyuv.h>
+
+#define DLIB_POSE_MODEL_PATH "/opt/dlib/models/shape_predictor_68_face_landmarks.dat"
+
+struct PL_DlibFaceTrack_Internal
+{
+	//uint8_t buffer[1920*1080*4];
+	//size_t buffSize;
+	//size_t buffSizeMax;
+	MB_Frame lastFrame;
+	PL_DlibFaceTrack_Config config;
+
+	bool payError;
+	
+	dlib::frontal_face_detector detector; // #todo reset
+	dlib::shape_predictor pose_model;
+
+	PL_DlibFaceTrack_Internal() : 
+		//buffSize(0), buffSizeMax(sizeof(buffer)), 
+		lastFrame(), config(), payError(true)
+	{
+	}
+	
+	~PL_DlibFaceTrack_Internal()
+	{
+	}
+	
+	void reset()
+	{
+		//buffSize = 0;
+		payError = true;
+		
+		MB_Frame _lastFrame;
+		lastFrame = _lastFrame;
+		PL_DlibFaceTrack_Config _config;
+		config = _config;
+	}
+};
+
+PipeLineElem* create_PL_DlibFaceTrack()
+{
+	return new PL_DlibFaceTrack;
+}
+
+PL_DlibFaceTrack::PL_DlibFaceTrack() : internal(new PL_DlibFaceTrack_Internal)
+{
+}
+
+PL_DlibFaceTrack::~PL_DlibFaceTrack()
+{
+	delete (PL_DlibFaceTrack_Internal*)internal;
+	internal= nullptr;
+}
+
+bool PL_DlibFaceTrack::init(void* args)
+{
+	PL_DlibFaceTrack_Internal* in = (PL_DlibFaceTrack_Internal*)internal;
+	in->reset();
+	
+	PL_DlibFaceTrack_Config* config = (PL_DlibFaceTrack_Config*)args;
+	if (config != nullptr)
+		in->config = *config;
+	
+#ifdef __AVX__
+	LOG_DEBUG << "AVX on";
+	#ifdef DLIB_HAVE_SSE2
+		LOG_DEBUG << "DLIB_HAVE_SSE2 on";
+	#endif
+	#ifdef DLIB_HAVE_SSE3
+		LOG_DEBUG << "DLIB_HAVE_SSE3 on";
+	#endif
+	#ifdef DLIB_HAVE_SSE41
+		LOG_DEBUG << "DLIB_HAVE_SSE41 on";
+	#endif
+	#ifdef DLIB_HAVE_AVX
+		LOG_DEBUG << "DLIB_HAVE_AVX on";
+	#endif
+#endif
+	
+	in->detector = dlib::get_frontal_face_detector();
+	dlib::deserialize(DLIB_POSE_MODEL_PATH) >> in->pose_model;
+	
+	return true;
+}
+
+void PL_DlibFaceTrack::finit()
+{
+	PL_DlibFaceTrack_Internal* in = (PL_DlibFaceTrack_Internal*)internal;
+
+}
+
+uint64_t time_msec()
+{
+	timeval tv;
+	gettimeofday(&tv, nullptr);
+	
+	return (tv.tv_sec * 1000 * 1000 + tv.tv_usec) / 1000;
+}
+
+int doFaceTrack(PL_DlibFaceTrack_Internal* in, 
+				uint8_t* buffer, size_t width, size_t height, MB_Frame::MBFType pixFmt)
+{
+	uint64_t tbegin = time_msec();
+
+	//uint8_t* pBuffer = new uint8_t[width * height * 3];
+	
+	//#define SUBSAMPLE(v, a) ((((v) + (a) - 1)) / (a))
+
+	//libyuv::I420ToRGB24(buffer, width, 
+	//				   buffer + width * height, SUBSAMPLE(width, 2), 
+	//				   buffer + width * height + width * height / 4, SUBSAMPLE(width, 2), 
+	//				   pBuffer, 3 * width, 
+	//				   width, height);
+
+	//#test
+	//static size_t f=0;
+	//char fname[50];
+	//sprintf(fname, "%u.rgb", ++f);
+	//FILE * pFile = fopen (fname, "wb");
+	//fwrite (pBuffer , sizeof(char), width * height * 4, pFile);
+	//fclose(pFile);
+
+	cv::Mat yMat(cv::Size(width,height), CV_8UC1, buffer);
+	//cv::Mat rgbMat(cv::Size(width,height), CV_8UC3, pBuffer);
+
+	//dlib::cv_image<dlib::bgr_pixel> rgbImg(rgbMat);
+	dlib::cv_image<unsigned char> yImg(yMat);
+	
+	dlib::array2d<unsigned char> downImg;
+	// Call pyr N times should have the same result, but one time may be faster
+	
+	switch(in->config.pyramid_down_layers)
+	{
+	case 2:
+	{
+		// best for 800x600 image on a 2-core Intel(R) Xeon(R) CPU E3-1220 v5 @ 3.00GHz in VMware
+		// face distance from camera is about 50cm
+		dlib::pyramid_down<2> pyr;
+		if (in->config.pyramid_down_n >= 1)
+		{
+			pyr(yImg, downImg);
+			for(uint8_t i = 0; i < in->config.pyramid_down_n - 1; i++)
+				pyr(downImg, downImg);
+		}
+	}
+	break;
+	case 3:
+	{
+		dlib::pyramid_down<3> pyr;
+		pyr(yImg, downImg);
+		if (in->config.pyramid_down_n >= 1)
+		{
+			pyr(yImg, downImg);
+			for(uint8_t i = 0; i < in->config.pyramid_down_n - 1; i++)
+				pyr(downImg, downImg);
+		}
+	}
+	case 4:
+	{
+		dlib::pyramid_down<4> pyr;
+		pyr(yImg, downImg);
+		if (in->config.pyramid_down_n >= 1)
+		{
+			pyr(yImg, downImg);
+			for(uint8_t i = 0; i < in->config.pyramid_down_n - 1; i++)
+				pyr(downImg, downImg);
+		}
+	}
+	break;
+	case 0:
+	default:
+		// do nothing
+	break;
+	}
+
+	LOGP(DEBUG, "downImg L=%u, n=%u, nr=%u, nc=%u", 
+		in->config.pyramid_down_layers, in->config.pyramid_down_n, downImg.nr(), downImg.nc());
+
+	//LOGP(DEBUG, "doFaceTrack time1=%llu (ms)", time_msec() - tbegin);
+
+	uint8_t factPosFactor = 0;
+	if (downImg.nr() > 0 && downImg.nc() > 0)
+		factPosFactor = width / downImg.nc();
+	
+	// Detect faces 
+	std::vector<dlib::rectangle> faces;
+	if (factPosFactor > 0)
+		faces = in->detector(downImg);
+	else
+		faces = in->detector(yImg);
+	
+	//LOGP(DEBUG, "doFaceTrack time2=%llu (ms)", time_msec() - tbegin);
+
+	// Find the pose of each face.
+	std::vector<dlib::full_object_detection> shapes;
+	for (unsigned long i = 0; i < faces.size(); ++i)
+	{
+		dlib::full_object_detection posePoint;
+		if (factPosFactor > 0)
+		{
+			posePoint = in->pose_model(downImg, faces[i]);
+			
+			for (size_t i = 0; i < 68; i++)
+				posePoint.part(i) = dlib::point(posePoint.part(i).x() * factPosFactor, posePoint.part(i).y() * factPosFactor);
+		}
+		else
+		{
+			posePoint = in->pose_model(yImg, faces[i]);
+		}
+		
+		shapes.push_back(posePoint);
+
+		if (factPosFactor > 0)
+		{
+			faces[i].set_left(faces[i].left() * factPosFactor);
+			faces[i].set_top(faces[i].top() * factPosFactor);
+			faces[i].set_right(faces[i].right() * factPosFactor);
+			faces[i].set_bottom(faces[i].bottom() * factPosFactor);
+		}
+
+		LOGP(DEBUG, "face: %d, poseLTRB: [%d, %d, %d, %d]", i, 
+			faces[i].left(), faces[i].top(), faces[i].right(), faces[i].bottom());
+
+		cv::rectangle(yMat, 
+			cv::Point2i(faces[i].left(), faces[i].top()), 
+			cv::Point2i(faces[i].right(), faces[i].bottom()), CV_RGB(128, 128, 128), 2);
+			
+		for (int i = 0; i < shapes.size(); i++)
+		{  
+			for (int j = 0; j < 68; j++)
+				cv::circle(yMat, cvPoint(shapes[i].part(j).x(), shapes[i].part(j).y()), 2, cvScalar(255, 0, 0), -1);
+		}  
+	}
+	
+	LOGP(DEBUG, "doFaceTrack time3=%llu (ms)", time_msec() - tbegin);
+
+	return faces.size();
+}
+
+bool PL_DlibFaceTrack::pay(const PipeMaterial& pm)
+{
+	PL_DlibFaceTrack_Internal* in = (PL_DlibFaceTrack_Internal*)internal;
+
+	if (pm.type != PipeMaterial::PMT_FRAME)
+	{
+		LOG_ERROR << "Only support PMT_FRAME";
+		return false;
+	}
+	
+	if (pm.buffer == nullptr)
+		return false;
+	
+	MB_Frame* frame = (MB_Frame*)pm.buffer;
+	if (frame->type != MB_Frame::MBFT_YUV420)
+	{
+		LOG_ERROR << "Only support MBFT_YUV420";
+		return false;
+	}
+
+	int face_count = doFaceTrack(
+						in, (uint8_t*)frame->buffer, frame->width, frame->height, MB_Frame::MBFT_YUV420);
+	if (face_count < 0)
+	{
+		in->payError = true;
+		return false;
+	}
+	else
+		in->payError = false;
+	
+	//in->buffer readly
+in->payError = false;//#todo
+	in->lastFrame.type = MB_Frame::MBFT_YUV420;
+	in->lastFrame.buffer = frame->buffer;//#todo should copy
+	in->lastFrame.buffSize = frame->buffSize;
+	in->lastFrame.width = frame->width;
+	in->lastFrame.height = frame->height;
+	in->lastFrame.pts = frame->pts;
+	gettimeofday(&(in->lastFrame.pts),NULL);
+
+	return true;
+}
+
+bool PL_DlibFaceTrack::gain(PipeMaterial& pm)
+{
+	PL_DlibFaceTrack_Internal* in = (PL_DlibFaceTrack_Internal*)internal;
+
+	if (!in->payError)
+	{
+		pm.type = PipeMaterial::PMT_FRAME;
+		pm.buffer = &(in->lastFrame);
+		pm.buffSize = 0;
+		pm.former = this;
+	}
+	pm.former = this;
+	return !in->payError;
+}
diff --git a/RtspFace/PL_DlibFaceTrack.h b/RtspFace/PL_DlibFaceTrack.h
new file mode 100644
index 0000000..158b5e8
--- /dev/null
+++ b/RtspFace/PL_DlibFaceTrack.h
@@ -0,0 +1,33 @@
+#ifndef _PL_DLIBFACETRACK_H_
+#define _PL_DLIBFACETRACK_H_
+
+#include "PipeLine.h"
+
+struct PL_DlibFaceTrack_Config
+{
+	uint8_t pyramid_down_layers;
+	uint8_t pyramid_down_n;
+	
+	PL_DlibFaceTrack_Config() : pyramid_down_layers(0), pyramid_down_n(0)
+	{ }
+};
+
+class PL_DlibFaceTrack : public PipeLineElem
+{
+public:
+	PL_DlibFaceTrack();
+	virtual ~PL_DlibFaceTrack();
+
+	virtual bool init(void* args);
+	virtual void finit();
+
+	virtual bool pay(const PipeMaterial& pm);
+	virtual bool gain(PipeMaterial& pm);
+	
+private:
+	void* internal;
+};
+
+PipeLineElem* create_PL_DlibFaceTrack();
+
+#endif
diff --git a/RtspFace/PL_SensetimeFaceTrack.cpp b/RtspFace/PL_SensetimeFaceTrack.cpp
index 71cb4d3..541c79c 100644
--- a/RtspFace/PL_SensetimeFaceTrack.cpp
+++ b/RtspFace/PL_SensetimeFaceTrack.cpp
@@ -189,7 +189,7 @@
 
 	if (pm.type != PipeMaterial::PMT_FRAME)
 	{
-		LOG_ERROR << "PL_H264Encoder::pay only support PMT_FRAME";
+		LOG_ERROR << "Only support PMT_FRAME";
 		return false;
 	}
 	
@@ -199,7 +199,7 @@
 	MB_Frame* frame = (MB_Frame*)pm.buffer;
 	if (frame->type != MB_Frame::MBFT_YUV420)
 	{
-		LOG_ERROR << "PL_H264Encoder::pay only support MBFT_YUV420";
+		LOG_ERROR << "Only support MBFT_YUV420";
 		return false;
 	}
 
diff --git a/RtspFace/PL_SensetimeFaceTrack.h b/RtspFace/PL_SensetimeFaceTrack.h
index 8ce9f5c..48030e8 100644
--- a/RtspFace/PL_SensetimeFaceTrack.h
+++ b/RtspFace/PL_SensetimeFaceTrack.h
@@ -1,5 +1,5 @@
-#ifndef _PL_PL_SENSETIMEFACETRACK_H_
-#define _PL_PL_SENSETIMEFACETRACK_H_
+#ifndef _PL_SENSETIMEFACETRACK_H_
+#define _PL_SENSETIMEFACETRACK_H_
 
 #include "PipeLine.h"
 
@@ -34,6 +34,6 @@
 	void* internal;
 };
 
-PipeLineElem* create_PL_SensetimeFaceDetect();
+PipeLineElem* create_PL_SensetimeFaceTrack();
 
 #endif
diff --git a/RtspFace/main.cpp b/RtspFace/main.cpp
index 412c1ac..738b9b1 100644
--- a/RtspFace/main.cpp
+++ b/RtspFace/main.cpp
@@ -10,6 +10,8 @@
 
 #include "PL_SensetimeFaceTrack.h"
 
+#include "PL_DlibFaceTrack.h"
+
 #include "logger.h"
 
 int main(int argc, char** argv)
@@ -27,6 +29,8 @@
 	pipeLine.register_elem_creator("PL_Scale", create_PL_Scale);
 	
 	pipeLine.register_elem_creator("PL_SensetimeFaceTrack", create_PL_SensetimeFaceTrack);
+	
+	pipeLine.register_elem_creator("PL_DlibFaceTrack", create_PL_DlibFaceTrack);
 	
 	{
 		PL_RTSPClient* rtspClient = (PL_RTSPClient*)pipeLine.push_elem("PL_RTSPClient");
@@ -78,9 +82,17 @@
 		}
 	}
 
+	//{
+	//	SensetimeFaceTrackConfig config;
+	//	PL_SensetimeFaceTrack* pl = (PL_SensetimeFaceTrack*)pipeLine.push_elem("PL_SensetimeFaceTrack");
+	//	pl->init(&config);
+	//}
+	
 	{
-		SensetimeFaceTrackConfig config;
-		PL_SensetimeFaceTrack* pl = (PL_SensetimeFaceTrack*)pipeLine.push_elem("PL_SensetimeFaceTrack");
+		PL_DlibFaceTrack_Config config;
+		config.pyramid_down_layers = 2;
+		config.pyramid_down_n = 1;
+		PL_DlibFaceTrack* pl = (PL_DlibFaceTrack*)pipeLine.push_elem("PL_DlibFaceTrack");
 		pl->init(&config);
 	}
 
@@ -104,7 +116,7 @@
 			exit(EXIT_FAILURE);
 		}
 	}
-	
+
 	{
 		PL_RTSPServer* rtspServer = (PL_RTSPServer*)pipeLine.push_elem("PL_RTSPServer");
 		bool ret = rtspServer->init(nullptr);
diff --git a/RtspFace/make.sh b/RtspFace/make.sh
index acdd829..c15d0d2 100644
--- a/RtspFace/make.sh
+++ b/RtspFace/make.sh
@@ -29,42 +29,50 @@
 OPENCV_INC=
 OPENCV_LIB="-lopencv_core"
 
+DLIB_BASE=/opt/dlib
+DLIB_INC="-I$DLIB_BASE"
+DLIB_LIB="-L$DLIB_BASE/build/dlib -ldlib"
+
 LIBLOG4CPP_BASE=/opt/log4cpp/inst
 LIBLOG4CPP_INC="-I$LIBLOG4CPP_BASE/include"
 LIBLOG4CPP_LIB="-L$LIBLOG4CPP_BASE/lib -llog4cpp"
 
-CPPFLAGS+="-pthread $LIVEMEDIA_INC $FFMPEG_INC $LIBBASE64_INC $LIBYUV_INC $SENSETIMEFACESDK_INC $LIBLOG4CPP_INC"
-LDFLAGS+="-pthread $LIVEMEDIA_LIB $FFMPEG_LIB $LIBBASE64_LIB $LIBYUV_LIB $LIBX264_LIB $SENSETIMEFACESDK_LIB $OPENCV_LIB $LIBLOG4CPP_LIB"
+# -O3
+CPPFLAGS+="-g -mavx -c -std=c++11 -pthread $LIVEMEDIA_INC $FFMPEG_INC $LIBBASE64_INC $LIBYUV_INC $SENSETIMEFACESDK_INC $LIBLOG4CPP_INC $DLIB_INC"
+LDFLAGS+="-pthread $LIVEMEDIA_LIB $FFMPEG_LIB $LIBBASE64_LIB $LIBYUV_LIB $LIBX264_LIB $SENSETIMEFACESDK_LIB $OPENCV_LIB $LIBLOG4CPP_LIB $DLIB_LIB"
 
 CFLAGS+="-D__STDC_CONSTANT_MACROS"
 
 rm rtsp_face
 rm *.o
 
-g++ -g -c -std=c++11 main.cpp $CFLAGS $CPPFLAGS
-g++ -g -c -std=c++11 PipeLine.cpp $CFLAGS $CPPFLAGS
+g++ main.cpp $CFLAGS $CPPFLAGS
+g++ PipeLine.cpp $CFLAGS $CPPFLAGS
 
-g++ -g -c -std=c++11 PL_RTSPClient.cpp $CFLAGS $CPPFLAGS
-g++ -g -c -std=c++11 PL_RTSPServer.cpp $CFLAGS $CPPFLAGS
-g++ -g -c -std=c++11 PL_H264Decoder.cpp $CFLAGS $CPPFLAGS
-g++ -g -c -std=c++11 PL_H264Encoder.cpp $CFLAGS $CPPFLAGS
-g++ -g -c -std=c++11 PL_AVFrameYUV420.cpp $CFLAGS $CPPFLAGS
-g++ -g -c -std=c++11 PL_AVFrameBGRA.cpp $CFLAGS $CPPFLAGS
-g++ -g -c -std=c++11 PL_Queue.cpp $CFLAGS $CPPFLAGS
-g++ -g -c -std=c++11 PL_Scale.cpp $CFLAGS $CPPFLAGS
+g++ PL_RTSPClient.cpp $CFLAGS $CPPFLAGS
+g++ PL_RTSPServer.cpp $CFLAGS $CPPFLAGS
+g++ PL_H264Decoder.cpp $CFLAGS $CPPFLAGS
+g++ PL_H264Encoder.cpp -O3 $CFLAGS $CPPFLAGS
+g++ PL_AVFrameYUV420.cpp $CFLAGS $CPPFLAGS
+g++ PL_AVFrameBGRA.cpp $CFLAGS $CPPFLAGS
+g++ PL_Queue.cpp $CFLAGS $CPPFLAGS
+g++ PL_Scale.cpp $CFLAGS $CPPFLAGS
 
-g++ -g -c -std=c++11 PL_SensetimeFaceTrack.cpp $CFLAGS $CPPFLAGS
+g++ PL_SensetimeFaceTrack.cpp $CFLAGS $CPPFLAGS
 
-g++ -g -c -std=c++11 $FFMPEGRTSPSERVER_BASE/FFmpegH264Source.cpp $CFLAGS $CPPFLAGS
-g++ -g -c -std=c++11 $FFMPEGRTSPSERVER_BASE/LiveRTSPServer.cpp $CFLAGS $CPPFLAGS
-g++ -g -c -std=c++11 $FFMPEGRTSPSERVER_BASE/LiveServerMediaSubsession.cpp $CFLAGS $CPPFLAGS
+g++ PL_DlibFaceTrack.cpp -O3 $CFLAGS $CPPFLAGS
+
+g++ $FFMPEGRTSPSERVER_BASE/FFmpegH264Source.cpp $CFLAGS $CPPFLAGS
+g++ $FFMPEGRTSPSERVER_BASE/LiveRTSPServer.cpp $CFLAGS $CPPFLAGS
+g++ $FFMPEGRTSPSERVER_BASE/LiveServerMediaSubsession.cpp $CFLAGS $CPPFLAGS
 
 g++ -g -std=c++11 \
   main.o PipeLine.o \
   PL_RTSPClient.o PL_H264Decoder.o PL_H264Encoder.o PL_AVFrameYUV420.o PL_AVFrameBGRA.o PL_Queue.o PL_Scale.o \
   PL_SensetimeFaceTrack.o \
+  PL_DlibFaceTrack.o \
   $FFMPEGRTSPSERVER_OBJ PL_RTSPServer.o \
   $LDFLAGS -o rtsp_face
 
-#export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$LIBX264_BASE/lib:$FFMPEG_BASE/lib:$SENSETIMEFACESDK_BASE/libs/linux-x86_64:$LIBLOG4CPP_BASE/lib
-#./rtsp_face rtsp://admin:admin12345@192.168.1.64:554/h264/ch1/main/av_stream
+#export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$LIBX264_BASE/lib:$FFMPEG_BASE/lib:$SENSETIMEFACESDK_BASE/libs/linux-x86_64:$LIBLOG4CPP_BASE/lib:$DLIB_BASE/build/dlib
+#./rtsp_face rtsp://admin:admin12345@192.168.1.70:554/h264/ch1/main/av_stream

--
Gitblit v1.8.0