add dlib face trace

houxiao

2017-01-04 58da7275a659c26b6a1fd9ec07e76f4cf521375e

add dlib face trace

git-svn-id: http://192.168.1.226/svn/proxy@75 454eff88-639b-444f-9e54-f578c98de674

 RtspFace/PL_AVFrameBGRA.cpp

@@ -7,10 +7,10 @@
    #include <libavcodec/avcodec.h>

    #include <libavutil/frame.h>

    #include <libavformat/avformat.h>

	
    #include <libyuv.h>

}



#include <libyuv.h>



struct PL_AVFrameBGRA_Internal

{

    uint8_t buffer[1920*1080*4];//#todo from config


 RtspFace/PL_DlibFaceTrack.cpp

New file
@@ -0,0 +1,309 @@
#include "PL_DlibFaceTrack.h"

#include "MaterialBuffer.h"

#include "logger.h"



#include <opencv2/opencv.hpp>

#include <opencv2/highgui/highgui.hpp>



#include <dlib/opencv.h>

#include <dlib/image_processing/frontal_face_detector.h>

#include <dlib/image_processing/render_face_detections.h>

#include <dlib/image_processing.h>

#include <dlib/gui_widgets.h>



#include <libyuv.h>



#define DLIB_POSE_MODEL_PATH "/opt/dlib/models/shape_predictor_68_face_landmarks.dat"



struct PL_DlibFaceTrack_Internal

{

    //uint8_t buffer[1920*1080*4];

    //size_t buffSize;

    //size_t buffSizeMax;

    MB_Frame lastFrame;

    PL_DlibFaceTrack_Config config;



    bool payError;

	
    dlib::frontal_face_detector detector; // #todo reset

    dlib::shape_predictor pose_model;



    PL_DlibFaceTrack_Internal() : 
        //buffSize(0), buffSizeMax(sizeof(buffer)), 
        lastFrame(), config(), payError(true)

    {

    }

	
    ~PL_DlibFaceTrack_Internal()

    {

    }

	
    void reset()

    {

        //buffSize = 0;

        payError = true;

		
        MB_Frame _lastFrame;

        lastFrame = _lastFrame;

        PL_DlibFaceTrack_Config _config;

        config = _config;

    }

};



PipeLineElem* create_PL_DlibFaceTrack()

{

    return new PL_DlibFaceTrack;

}



PL_DlibFaceTrack::PL_DlibFaceTrack() : internal(new PL_DlibFaceTrack_Internal)

{

}



PL_DlibFaceTrack::~PL_DlibFaceTrack()

{

    delete (PL_DlibFaceTrack_Internal*)internal;

    internal= nullptr;

}



bool PL_DlibFaceTrack::init(void* args)

{

    PL_DlibFaceTrack_Internal* in = (PL_DlibFaceTrack_Internal*)internal;

    in->reset();

	
    PL_DlibFaceTrack_Config* config = (PL_DlibFaceTrack_Config*)args;

    if (config != nullptr)

        in->config = *config;

	
#ifdef __AVX__

    LOG_DEBUG << "AVX on";

    #ifdef DLIB_HAVE_SSE2

        LOG_DEBUG << "DLIB_HAVE_SSE2 on";

    #endif

    #ifdef DLIB_HAVE_SSE3

        LOG_DEBUG << "DLIB_HAVE_SSE3 on";

    #endif

    #ifdef DLIB_HAVE_SSE41

        LOG_DEBUG << "DLIB_HAVE_SSE41 on";

    #endif

    #ifdef DLIB_HAVE_AVX

        LOG_DEBUG << "DLIB_HAVE_AVX on";

    #endif

#endif

	
    in->detector = dlib::get_frontal_face_detector();

    dlib::deserialize(DLIB_POSE_MODEL_PATH) >> in->pose_model;

	
    return true;

}



void PL_DlibFaceTrack::finit()

{

    PL_DlibFaceTrack_Internal* in = (PL_DlibFaceTrack_Internal*)internal;



}



uint64_t time_msec()

{

    timeval tv;

    gettimeofday(&tv, nullptr);

	
    return (tv.tv_sec * 1000 * 1000 + tv.tv_usec) / 1000;

}



int doFaceTrack(PL_DlibFaceTrack_Internal* in, 
                uint8_t* buffer, size_t width, size_t height, MB_Frame::MBFType pixFmt)

{

    uint64_t tbegin = time_msec();



    //uint8_t* pBuffer = new uint8_t[width * height * 3];

	
    //#define SUBSAMPLE(v, a) ((((v) + (a) - 1)) / (a))



    //libyuv::I420ToRGB24(buffer, width, 
    //                   buffer + width * height, SUBSAMPLE(width, 2), 
    //                   buffer + width * height + width * height / 4, SUBSAMPLE(width, 2), 
    //                   pBuffer, 3 * width, 
    //                   width, height);



    //#test

    //static size_t f=0;

    //char fname[50];

    //sprintf(fname, "%u.rgb", ++f);

    //FILE * pFile = fopen (fname, "wb");

    //fwrite (pBuffer , sizeof(char), width * height * 4, pFile);

    //fclose(pFile);



    cv::Mat yMat(cv::Size(width,height), CV_8UC1, buffer);

    //cv::Mat rgbMat(cv::Size(width,height), CV_8UC3, pBuffer);



    //dlib::cv_image<dlib::bgr_pixel> rgbImg(rgbMat);

    dlib::cv_image<unsigned char> yImg(yMat);

	
    dlib::array2d<unsigned char> downImg;

    // Call pyr N times should have the same result, but one time may be faster

	
    switch(in->config.pyramid_down_layers)

    {

    case 2:

    {

        // best for 800x600 image on a 2-core Intel(R) Xeon(R) CPU E3-1220 v5 @ 3.00GHz in VMware

        // face distance from camera is about 50cm

        dlib::pyramid_down<2> pyr;

        if (in->config.pyramid_down_n >= 1)

        {

            pyr(yImg, downImg);

            for(uint8_t i = 0; i < in->config.pyramid_down_n - 1; i++)

                pyr(downImg, downImg);

        }

    }

    break;

    case 3:

    {

        dlib::pyramid_down<3> pyr;

        pyr(yImg, downImg);

        if (in->config.pyramid_down_n >= 1)

        {

            pyr(yImg, downImg);

            for(uint8_t i = 0; i < in->config.pyramid_down_n - 1; i++)

                pyr(downImg, downImg);

        }

    }

    case 4:

    {

        dlib::pyramid_down<4> pyr;

        pyr(yImg, downImg);

        if (in->config.pyramid_down_n >= 1)

        {

            pyr(yImg, downImg);

            for(uint8_t i = 0; i < in->config.pyramid_down_n - 1; i++)

                pyr(downImg, downImg);

        }

    }

    break;

    case 0:

    default:

        // do nothing

    break;

    }



    LOGP(DEBUG, "downImg L=%u, n=%u, nr=%u, nc=%u", 
        in->config.pyramid_down_layers, in->config.pyramid_down_n, downImg.nr(), downImg.nc());



    //LOGP(DEBUG, "doFaceTrack time1=%llu (ms)", time_msec() - tbegin);



    uint8_t factPosFactor = 0;

    if (downImg.nr() > 0 && downImg.nc() > 0)

        factPosFactor = width / downImg.nc();

	
    // Detect faces 
    std::vector<dlib::rectangle> faces;

    if (factPosFactor > 0)

        faces = in->detector(downImg);

    else

        faces = in->detector(yImg);

	
    //LOGP(DEBUG, "doFaceTrack time2=%llu (ms)", time_msec() - tbegin);



    // Find the pose of each face.

    std::vector<dlib::full_object_detection> shapes;

    for (unsigned long i = 0; i < faces.size(); ++i)

    {

        dlib::full_object_detection posePoint;

        if (factPosFactor > 0)

        {

            posePoint = in->pose_model(downImg, faces[i]);

			
            for (size_t i = 0; i < 68; i++)

                posePoint.part(i) = dlib::point(posePoint.part(i).x() * factPosFactor, posePoint.part(i).y() * factPosFactor);

        }

        else

        {

            posePoint = in->pose_model(yImg, faces[i]);

        }

		
        shapes.push_back(posePoint);



        if (factPosFactor > 0)

        {

            faces[i].set_left(faces[i].left() * factPosFactor);

            faces[i].set_top(faces[i].top() * factPosFactor);

            faces[i].set_right(faces[i].right() * factPosFactor);

            faces[i].set_bottom(faces[i].bottom() * factPosFactor);

        }



        LOGP(DEBUG, "face: %d, poseLTRB: [%d, %d, %d, %d]", i, 
            faces[i].left(), faces[i].top(), faces[i].right(), faces[i].bottom());



        cv::rectangle(yMat, 
            cv::Point2i(faces[i].left(), faces[i].top()), 
            cv::Point2i(faces[i].right(), faces[i].bottom()), CV_RGB(128, 128, 128), 2);

			
        for (int i = 0; i < shapes.size(); i++)

        {  
            for (int j = 0; j < 68; j++)

                cv::circle(yMat, cvPoint(shapes[i].part(j).x(), shapes[i].part(j).y()), 2, cvScalar(255, 0, 0), -1);

        }  
    }

	
    LOGP(DEBUG, "doFaceTrack time3=%llu (ms)", time_msec() - tbegin);



    return faces.size();

}



bool PL_DlibFaceTrack::pay(const PipeMaterial& pm)

{

    PL_DlibFaceTrack_Internal* in = (PL_DlibFaceTrack_Internal*)internal;



    if (pm.type != PipeMaterial::PMT_FRAME)

    {

        LOG_ERROR << "Only support PMT_FRAME";

        return false;

    }

	
    if (pm.buffer == nullptr)

        return false;

	
    MB_Frame* frame = (MB_Frame*)pm.buffer;

    if (frame->type != MB_Frame::MBFT_YUV420)

    {

        LOG_ERROR << "Only support MBFT_YUV420";

        return false;

    }



    int face_count = doFaceTrack(

                        in, (uint8_t*)frame->buffer, frame->width, frame->height, MB_Frame::MBFT_YUV420);

    if (face_count < 0)

    {

        in->payError = true;

        return false;

    }

    else

        in->payError = false;

	
    //in->buffer readly

in->payError = false;//#todo

    in->lastFrame.type = MB_Frame::MBFT_YUV420;

    in->lastFrame.buffer = frame->buffer;//#todo should copy

    in->lastFrame.buffSize = frame->buffSize;

    in->lastFrame.width = frame->width;

    in->lastFrame.height = frame->height;

    in->lastFrame.pts = frame->pts;

    gettimeofday(&(in->lastFrame.pts),NULL);



    return true;

}



bool PL_DlibFaceTrack::gain(PipeMaterial& pm)

{

    PL_DlibFaceTrack_Internal* in = (PL_DlibFaceTrack_Internal*)internal;



    if (!in->payError)

    {

        pm.type = PipeMaterial::PMT_FRAME;

        pm.buffer = &(in->lastFrame);

        pm.buffSize = 0;

        pm.former = this;

    }

    pm.former = this;

    return !in->payError;

}


 RtspFace/PL_DlibFaceTrack.h

New file
@@ -0,0 +1,33 @@
#ifndef _PL_DLIBFACETRACK_H_

#define _PL_DLIBFACETRACK_H_



#include "PipeLine.h"



struct PL_DlibFaceTrack_Config

{

    uint8_t pyramid_down_layers;

    uint8_t pyramid_down_n;

	
    PL_DlibFaceTrack_Config() : pyramid_down_layers(0), pyramid_down_n(0)

    { }

};



class PL_DlibFaceTrack : public PipeLineElem

{

public:

    PL_DlibFaceTrack();

    virtual ~PL_DlibFaceTrack();



    virtual bool init(void* args);

    virtual void finit();



    virtual bool pay(const PipeMaterial& pm);

    virtual bool gain(PipeMaterial& pm);

	
private:

    void* internal;

};



PipeLineElem* create_PL_DlibFaceTrack();



#endif


 RtspFace/PL_SensetimeFaceTrack.cpp

@@ -189,7 +189,7 @@


    if (pm.type != PipeMaterial::PMT_FRAME)

    {

        LOG_ERROR << "PL_H264Encoder::pay only support PMT_FRAME";

        LOG_ERROR << "Only support PMT_FRAME";

        return false;

    }

    

@@ -199,7 +199,7 @@
    MB_Frame* frame = (MB_Frame*)pm.buffer;

    if (frame->type != MB_Frame::MBFT_YUV420)

    {

        LOG_ERROR << "PL_H264Encoder::pay only support MBFT_YUV420";

        LOG_ERROR << "Only support MBFT_YUV420";

        return false;

    }




 RtspFace/PL_SensetimeFaceTrack.h

@@ -1,5 +1,5 @@
#ifndef _PL_PL_SENSETIMEFACETRACK_H_

#define _PL_PL_SENSETIMEFACETRACK_H_

#ifndef _PL_SENSETIMEFACETRACK_H_

#define _PL_SENSETIMEFACETRACK_H_



#include "PipeLine.h"



@@ -34,6 +34,6 @@
    void* internal;

};



PipeLineElem* create_PL_SensetimeFaceDetect();

PipeLineElem* create_PL_SensetimeFaceTrack();



#endif


 RtspFace/main.cpp

@@ -10,6 +10,8 @@


#include "PL_SensetimeFaceTrack.h"



#include "PL_DlibFaceTrack.h"



#include "logger.h"



int main(int argc, char** argv)

@@ -27,6 +29,8 @@
    pipeLine.register_elem_creator("PL_Scale", create_PL_Scale);

    

    pipeLine.register_elem_creator("PL_SensetimeFaceTrack", create_PL_SensetimeFaceTrack);

	
    pipeLine.register_elem_creator("PL_DlibFaceTrack", create_PL_DlibFaceTrack);

    

    {

        PL_RTSPClient* rtspClient = (PL_RTSPClient*)pipeLine.push_elem("PL_RTSPClient");

@@ -78,9 +82,17 @@
        }

    }



    //{

    //    SensetimeFaceTrackConfig config;

    //    PL_SensetimeFaceTrack* pl = (PL_SensetimeFaceTrack*)pipeLine.push_elem("PL_SensetimeFaceTrack");

    //    pl->init(&config);

    //}

	
    {

        SensetimeFaceTrackConfig config;

        PL_SensetimeFaceTrack* pl = (PL_SensetimeFaceTrack*)pipeLine.push_elem("PL_SensetimeFaceTrack");

        PL_DlibFaceTrack_Config config;

        config.pyramid_down_layers = 2;

        config.pyramid_down_n = 1;

        PL_DlibFaceTrack* pl = (PL_DlibFaceTrack*)pipeLine.push_elem("PL_DlibFaceTrack");

        pl->init(&config);

    }



@@ -104,7 +116,7 @@
            exit(EXIT_FAILURE);

        }

    }

	


    {

        PL_RTSPServer* rtspServer = (PL_RTSPServer*)pipeLine.push_elem("PL_RTSPServer");

        bool ret = rtspServer->init(nullptr);


 RtspFace/make.sh

@@ -29,42 +29,50 @@
OPENCV_INC=
OPENCV_LIB="-lopencv_core"

DLIB_BASE=/opt/dlib
DLIB_INC="-I$DLIB_BASE"
DLIB_LIB="-L$DLIB_BASE/build/dlib -ldlib"

LIBLOG4CPP_BASE=/opt/log4cpp/inst
LIBLOG4CPP_INC="-I$LIBLOG4CPP_BASE/include"
LIBLOG4CPP_LIB="-L$LIBLOG4CPP_BASE/lib -llog4cpp"

CPPFLAGS+="-pthread $LIVEMEDIA_INC $FFMPEG_INC $LIBBASE64_INC $LIBYUV_INC $SENSETIMEFACESDK_INC $LIBLOG4CPP_INC"
LDFLAGS+="-pthread $LIVEMEDIA_LIB $FFMPEG_LIB $LIBBASE64_LIB $LIBYUV_LIB $LIBX264_LIB $SENSETIMEFACESDK_LIB $OPENCV_LIB $LIBLOG4CPP_LIB"
# -O3
CPPFLAGS+="-g -mavx -c -std=c++11 -pthread $LIVEMEDIA_INC $FFMPEG_INC $LIBBASE64_INC $LIBYUV_INC $SENSETIMEFACESDK_INC $LIBLOG4CPP_INC $DLIB_INC"
LDFLAGS+="-pthread $LIVEMEDIA_LIB $FFMPEG_LIB $LIBBASE64_LIB $LIBYUV_LIB $LIBX264_LIB $SENSETIMEFACESDK_LIB $OPENCV_LIB $LIBLOG4CPP_LIB $DLIB_LIB"

CFLAGS+="-D__STDC_CONSTANT_MACROS"

rm rtsp_face
rm *.o

g++ -g -c -std=c++11 main.cpp $CFLAGS $CPPFLAGS
g++ -g -c -std=c++11 PipeLine.cpp $CFLAGS $CPPFLAGS
g++ main.cpp $CFLAGS $CPPFLAGS
g++ PipeLine.cpp $CFLAGS $CPPFLAGS

g++ -g -c -std=c++11 PL_RTSPClient.cpp $CFLAGS $CPPFLAGS
g++ -g -c -std=c++11 PL_RTSPServer.cpp $CFLAGS $CPPFLAGS
g++ -g -c -std=c++11 PL_H264Decoder.cpp $CFLAGS $CPPFLAGS
g++ -g -c -std=c++11 PL_H264Encoder.cpp $CFLAGS $CPPFLAGS
g++ -g -c -std=c++11 PL_AVFrameYUV420.cpp $CFLAGS $CPPFLAGS
g++ -g -c -std=c++11 PL_AVFrameBGRA.cpp $CFLAGS $CPPFLAGS
g++ -g -c -std=c++11 PL_Queue.cpp $CFLAGS $CPPFLAGS
g++ -g -c -std=c++11 PL_Scale.cpp $CFLAGS $CPPFLAGS
g++ PL_RTSPClient.cpp $CFLAGS $CPPFLAGS
g++ PL_RTSPServer.cpp $CFLAGS $CPPFLAGS
g++ PL_H264Decoder.cpp $CFLAGS $CPPFLAGS
g++ PL_H264Encoder.cpp -O3 $CFLAGS $CPPFLAGS
g++ PL_AVFrameYUV420.cpp $CFLAGS $CPPFLAGS
g++ PL_AVFrameBGRA.cpp $CFLAGS $CPPFLAGS
g++ PL_Queue.cpp $CFLAGS $CPPFLAGS
g++ PL_Scale.cpp $CFLAGS $CPPFLAGS

g++ -g -c -std=c++11 PL_SensetimeFaceTrack.cpp $CFLAGS $CPPFLAGS
g++ PL_SensetimeFaceTrack.cpp $CFLAGS $CPPFLAGS

g++ -g -c -std=c++11 $FFMPEGRTSPSERVER_BASE/FFmpegH264Source.cpp $CFLAGS $CPPFLAGS
g++ -g -c -std=c++11 $FFMPEGRTSPSERVER_BASE/LiveRTSPServer.cpp $CFLAGS $CPPFLAGS
g++ -g -c -std=c++11 $FFMPEGRTSPSERVER_BASE/LiveServerMediaSubsession.cpp $CFLAGS $CPPFLAGS
g++ PL_DlibFaceTrack.cpp -O3 $CFLAGS $CPPFLAGS

g++ $FFMPEGRTSPSERVER_BASE/FFmpegH264Source.cpp $CFLAGS $CPPFLAGS
g++ $FFMPEGRTSPSERVER_BASE/LiveRTSPServer.cpp $CFLAGS $CPPFLAGS
g++ $FFMPEGRTSPSERVER_BASE/LiveServerMediaSubsession.cpp $CFLAGS $CPPFLAGS

g++ -g -std=c++11 \
  main.o PipeLine.o \
  PL_RTSPClient.o PL_H264Decoder.o PL_H264Encoder.o PL_AVFrameYUV420.o PL_AVFrameBGRA.o PL_Queue.o PL_Scale.o \
  PL_SensetimeFaceTrack.o \
  PL_DlibFaceTrack.o \
  $FFMPEGRTSPSERVER_OBJ PL_RTSPServer.o \
  $LDFLAGS -o rtsp_face

#export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$LIBX264_BASE/lib:$FFMPEG_BASE/lib:$SENSETIMEFACESDK_BASE/libs/linux-x86_64:$LIBLOG4CPP_BASE/lib
#./rtsp_face rtsp://admin:admin12345@192.168.1.64:554/h264/ch1/main/av_stream
#export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$LIBX264_BASE/lib:$FFMPEG_BASE/lib:$SENSETIMEFACESDK_BASE/libs/linux-x86_64:$LIBLOG4CPP_BASE/lib:$DLIB_BASE/build/dlib
#./rtsp_face rtsp://admin:admin12345@192.168.1.70:554/h264/ch1/main/av_stream