From 59408e43480b48b6acd6e76641ff1a97811e7ab5 Mon Sep 17 00:00:00 2001 From: pansen <pansen626@sina.com> Date: 星期四, 10 一月 2019 18:19:49 +0800 Subject: [PATCH] 修改yoloServer 优化多线程? --- .gitignore | 4 QiaoJiaSystem/YoloServer/YoloDetectServerI.h | 36 + QiaoJiaSystem/YoloServer/DnDetect.h | 89 ++++- QiaoJiaSystem/YoloServer/ImageDrawElement.cpp | 55 ++- QiaoJiaSystem/YoloServer/YoloRpcElement.cpp | 126 +++++++ QiaoJiaSystem/VideoAnalysFromHC/main.cpp | 5 QiaoJiaSystem/StructureApp/FaceRpcElement.cpp | 1 QiaoJiaSystem/YoloServer/DnDetect.cpp | 164 ++++++---- QiaoJiaSystem/YoloServer/main.cpp | 21 + QiaoJiaSystem/YoloServer/ImageDrawElement.h | 30 + QiaoJiaSystem/YoloServer/YoloRpcElement.h | 52 +++ QiaoJiaSystem/YoloServer/YoloDetectServerI.cpp | 160 +++------- QiaoJiaSystem/YoloServer/CMakeLists.txt | 108 ++++- QiaoJiaSystem/CMakeLists.txt | 2 QiaoJiaSystem/YoloServer/YoloDetectClientTest.cpp | 98 +++-- 15 files changed, 645 insertions(+), 306 deletions(-) diff --git a/.gitignore b/.gitignore index ccb23fd..6435583 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,7 @@ build-QiaoJiaSystem-unknown-Default/ QiaoJiaSystem/build/glog_dump\.log + +\.idea/ + +QiaoJiaSystem/VideoAnalysFromHC/cmake-build-debug/ diff --git a/QiaoJiaSystem/CMakeLists.txt b/QiaoJiaSystem/CMakeLists.txt index 2da842a..400e343 100644 --- a/QiaoJiaSystem/CMakeLists.txt +++ b/QiaoJiaSystem/CMakeLists.txt @@ -16,7 +16,7 @@ add_subdirectory(StructureApp) add_subdirectory(FaceDetectServer) -#add_subdirectory(YoloServer) +add_subdirectory(YoloServer) add_subdirectory(FaceSearchServer) add_subdirectory(VideoAnalysFromHC) add_subdirectory(RapidStructureApp) diff --git a/QiaoJiaSystem/StructureApp/FaceRpcElement.cpp b/QiaoJiaSystem/StructureApp/FaceRpcElement.cpp index 6f2cdd1..d661aa0 100644 --- a/QiaoJiaSystem/StructureApp/FaceRpcElement.cpp +++ b/QiaoJiaSystem/StructureApp/FaceRpcElement.cpp @@ -107,6 +107,7 @@ trackingTrigger->getLastRect().properties["smile"] = to_string(property.smile_level); trackingTrigger->getLastRect().properties["race"] = to_string(property.race); trackingTrigger->getLastRect().properties["time"] = getProperty("time"); + trackingTrigger->getLastRect().properties["detectscore"] = scoredRect.score; triggerFaces.push_back(face); triggerMats.push_back(image( diff --git a/QiaoJiaSystem/VideoAnalysFromHC/main.cpp b/QiaoJiaSystem/VideoAnalysFromHC/main.cpp index cfb0eba..2b01d43 100644 --- a/QiaoJiaSystem/VideoAnalysFromHC/main.cpp +++ b/QiaoJiaSystem/VideoAnalysFromHC/main.cpp @@ -66,7 +66,10 @@ int main(int argc, char **argv) { SAVE_APP_ARGS - ENABLEGLOG(GET_STR_CONFIG("logPath").c_str()); + +#ifdef GLOG + ENABLEGLOG(GET_STR_CONFIG("logPath").c_str()); +#endif appPref.setLongData("gpu.index", 0); appPref.setIntData("show.image", 0); diff --git a/QiaoJiaSystem/YoloServer/CMakeLists.txt b/QiaoJiaSystem/YoloServer/CMakeLists.txt index ff8a39e..646fbf1 100644 --- a/QiaoJiaSystem/YoloServer/CMakeLists.txt +++ b/QiaoJiaSystem/YoloServer/CMakeLists.txt @@ -59,11 +59,16 @@ ../../../BasicPlatForm/libs/Ice-3.7.0/include ../../../BasicPlatForm/libs/DarkNet/lib ../../../BasicPlatForm/libs/ffmpeg/lib + ../../../BasicPlatForm/libs/FastDFS/lib + ../../../BasicPlatForm/libs/crul/lib + ../../../BasicPlatForm/libs/jsoncpp/lib + ../../../BasicPlatForm/libs/libuuid/lib ) add_executable(${PROJECT_NAME} main.cpp YoloDetectServerI.cpp + DnDetect.cpp rpc/YoloServer.cpp ) @@ -71,33 +76,76 @@ ${LIBS} ) -#add_executable(YoloDetectClientTest -# ../../BasicPlatForm/basic/util/opencv/CvUtil.cpp -# ../../BasicPlatForm/basic/util/opencv/CvUtil.h -# ../../BasicPlatForm/basic/pipe/PipeConnector.cpp -# ../../BasicPlatForm/basic/pipe/PipeConnector.h -# ../../BasicPlatForm/basic/pipe/PipeController.cpp -# ../../BasicPlatForm/basic/pipe/PipeController.h -# ../../BasicPlatForm/basic/pipe/PipeElement.cpp -# ../../BasicPlatForm/basic/pipe/PipeElement.h -# -# ../../BasicPlatForm/basic/pipe_element/ImageFactoryElement.cpp -# ../../BasicPlatForm/basic/pipe_element/ImageFactoryElement.h -# ../../BasicPlatForm/basic/pipe_element/ImageShowElement.cpp -# ../../BasicPlatForm/basic/pipe_element/ImageShowElement.h -# ../../BasicPlatForm/basic/pipe_element/ffmpeg/VideoCaptureElement.cpp -# ../../BasicPlatForm/basic/pipe_element/ffmpeg/VideoCaptureElement.h -# rpc/YoloServer.cpp -# YoloDetectClientTest.cpp -# YoloRpcElement.cpp -# YoloRpcElement.h -# ImageDrawElement.cpp -# ) -# -#target_link_libraries(YoloDetectClientTest -# Ice -# Qt5Core -# avformat avcodec avutil avfilter swscale swresample -# opencv_world -# pthread -# ) +add_executable(YoloDetectClientTest + ../../../BasicPlatForm/basic/util/opencv/CvUtil.cpp + ../../../BasicPlatForm/basic/util/opencv/CvUtil.h + ../../../BasicPlatForm/basic/pipe/PipeConnector.cpp + ../../../BasicPlatForm/basic/pipe/PipeConnector.h + ../../../BasicPlatForm/basic/pipe/PipeController.cpp + ../../../BasicPlatForm/basic/pipe/PipeController.h + ../../../BasicPlatForm/basic/pipe/PipeElement.cpp + ../../../BasicPlatForm/basic/pipe/PipeElement.h + ../../../BasicPlatForm/basic/timer_counter/Clocktimer.h + ../../../BasicPlatForm/basic/timer_counter/Clocktimer.cpp + + ../../../BasicPlatForm/basic/pipe_element/ImageFactoryElement.cpp + ../../../BasicPlatForm/basic/pipe_element/ImageFactoryElement.h + ../../../BasicPlatForm/basic/pipe_element/ImageShowElement.cpp + ../../../BasicPlatForm/basic/pipe_element/ImageShowElement.h + ../../../BasicPlatForm/basic/pipe_element/ffmpeg/VideoCaptureElement.cpp + ../../../BasicPlatForm/basic/pipe_element/ffmpeg/VideoCaptureElement.h + ../../../BasicPlatForm/basic/util/fastdfs/FastdfsClient.cpp + ../../../BasicPlatForm/basic/pipe/TimerElement.cpp + ../RapidStructureApp/TriggerElement.cpp + ../../../BasicPlatForm/basic/db/Elasticsearch/EsDBTool.cpp + ../../../BasicPlatForm/basic/util/curl/HttpRequestWithCrul.hpp + rpc/YoloServer.cpp + YoloDetectClientTest.cpp + ./YoloRpcElement.cpp + ./YoloRpcElement.h + ImageDrawElement.cpp + ) + +link_directories( + + ../../../BasicPlatForm/libs/FastDFS/lib + ../../../BasicPlatForm/libs/crul/lib + ../../../BasicPlatForm/libs/libuuid/lib +) +include_directories( + ./rpc + ../StructureApp + ../RapidStructureApp/ + ../../../BasicPlatForm + ../../../BasicPlatForm/libs/opencv/include + ../../../BasicPlatForm/libs/jsoncpp/include + ../../../BasicPlatForm/libs/Ice-3.7.0/include + ../../../BasicPlatForm/libs/ffmpeg/include + ../../../BasicPlatForm/libs/DarkNet/include + /usr/local/cuda/include/ + /usr/include/x86_64-linux-gnu/qt5 + ../../../BasicPlatForm/basic/util/fastdfs/ + ../../../BasicPlatForm/libs/FastDFS/include + ../../../BasicPlatForm/libs/FastDFS/include/fastdfs + ../../../BasicPlatForm/libs/FastDFS/include/fastcommon + ../../../BasicPlatForm/basic/pipe/ + ../../../BasicPlatForm/libs/crul/include + ../../../BasicPlatForm/basic/util/curl/ + ../../../BasicPlatForm/libs/libuuid/include + + #glog + ../../../BasicPlatForm/libs/glog/include +) +target_link_libraries(YoloDetectClientTest + Ice + Qt5Core + avformat avcodec avutil avfilter swscale swresample + opencv_world + curl + uuid + jsoncpp + fastcommon + fdfsclient + glog + pthread + ) diff --git a/QiaoJiaSystem/YoloServer/DnDetect.cpp b/QiaoJiaSystem/YoloServer/DnDetect.cpp index 498ba0f..61ad8b0 100644 --- a/QiaoJiaSystem/YoloServer/DnDetect.cpp +++ b/QiaoJiaSystem/YoloServer/DnDetect.cpp @@ -1,90 +1,92 @@ +#include <basic/util/app/AppPreference.hpp> #include "DnDetect.h" -DnDetect::DnDetect(const int gpuIndex):m_net(nullptr),m_thresh(0.5),m_hier_thresh(0.5),m_nms(0.45),names(nullptr),alphabet(nullptr) -{ - cuda_set_device(gpuIndex); - - char *datacfg = "cfg/coco.data"; - char *cfgfile = "cfg/yolov3.cfg"; - char *weightfile = "./yolov3.weights"; - - double loadtime = what_time_is_it_now(); - list *options = read_data_cfg(datacfg); - char *name_list = option_find_str(options, "names", "data/names.list"); - names = get_labels(name_list); - - alphabet = load_alphabet(); - m_net = load_network(cfgfile, weightfile, 0); - set_batch_network(m_net, 1); - printf("load mod use %f seconds.\n", what_time_is_it_now()-loadtime); - - srand(2222222); - double attime; - float nms=.45; -} - -DnDetect::~DnDetect() -{ +DnDetect::DnDetect::DnDetect(const int gpuIndex) : m_net(nullptr), m_thresh(0.5), m_hier_thresh(0.5), m_nms(0.45), + names(nullptr), alphabet(nullptr), m_thdInit(init, this), + m_bInitThd(false) { } -std::vector<cv::Rect2f> DnDetect::detect(cv::Mat & img) -{ +DnDetect::DnDetect::~DnDetect() { - double bttime=what_time_is_it_now(); +} + +std::vector<DnDetect::YoloObjInfo> DnDetect::DnDetect::detect(cv::Mat &img) { + std::lock_guard<std::mutex> dataGuard(dataMtx); + + ClockTimer cl("DnDetect::DnDetect"); + std::vector<YoloObjInfo> objInfos; + if (!m_bInitThd) return objInfos; + double bttime = what_time_is_it_now(); image im = matToImg(img); +// DBG("matToImg : "<<what_time_is_it_now()-bttime); +// printf("matToImg %f seconds.\n", what_time_is_it_now()-bttime); image sized = letterbox_image(im, m_net->w, m_net->h); - layer l = m_net->layers[m_net->n-1]; + layer l = m_net->layers[m_net->n - 1]; float *X = sized.data; - //attime=what_time_is_it_now(); + //attime=what_time_is_it_now();p-> network_predict(m_net, X); //printf("Predicted in %f seconds.\n", what_time_is_it_now()-attime); int nboxes = 0; - detection *dets = get_network_boxes(m_net, im.w, im.h, 0.5f, 0.1f, 0, 1, &nboxes); - if (m_nms) do_nms_sort(dets, nboxes, l.classes, m_nms); - // draw_detections(im, dets, nboxes, m_thresh, names, alphabet, l.classes); - std::vector<cv::Rect2f> rects; - for(int i=0;i<nboxes;i++) + + detection *dets; + { +// std::lock_guard<std::mutex> dataGuard(dataMtx); +// dataMtx.lock(); + dets = get_network_boxes(m_net, im.w, im.h, m_thresh, m_hier_thresh, 0, 1, &nboxes); + if (nboxes > 100) { +// dataMtx.unlock(); + return objInfos; + } + if (m_nms) do_nms_sort(dets, nboxes, l.classes, m_nms); +// dataMtx.unlock(); + } + + // draw_detections(im, dets, nboxes, m_thresh, names, alphabet, l.classes); + for (int i = 0; i < nboxes; i++) { + YoloObjInfo objInfo; std::vector<float> vec(80); - memcpy(&vec[0],dets[i].prob,sizeof(float)*80); + memcpy(&vec[0], dets[i].prob, sizeof(float) * 80); int type = -1; - for(int j = 0; j < l.classes; ++j){ - if(j != 0){ + for (int j = 0; j < l.classes; ++j) { +//#todo new func in list out bool + if (j != 0) { continue; } - if (dets[i].prob[j] > m_thresh){ +//#todo get score + if (dets[i].prob[j] > 0.0f) { if (type < 0) { type = j; + objInfo.prob = dets[i].prob[j]; } else { } - } else{ + } else { } } - if(type >= 0){ - if(type != 0){ - continue; - } - float left = (dets[i].bbox.x-dets[i].bbox.w/2.); - float top = (dets[i].bbox.y-dets[i].bbox.h/2.); - cv::Rect2f rect(left,top,dets[i].bbox.w,dets[i].bbox.h); - std::cout<<"rect.x : "<<left<<" rect.y : "<<top<<" rect.w : "<<rect.width<<" rect.h : "<<rect.height<<std::endl; - rects.push_back(rect); + if (type >= 0) { +// if(type != 0){ +// continue; +// } + objInfo.type = type; + objInfo.rcObj.left = (dets[i].bbox.x - dets[i].bbox.w / 2.); + objInfo.rcObj.top = (dets[i].bbox.y - dets[i].bbox.h / 2.); + objInfo.rcObj.right = (dets[i].bbox.x + dets[i].bbox.w / 2.); + objInfo.rcObj.bottom = (dets[i].bbox.y + dets[i].bbox.h / 2.); + objInfos.push_back(objInfo); } - - } free_detections(dets, nboxes); // show_image(im, "Video"); // cv::waitKey(10); free_image(im); free_image(sized); - printf("all time use %f seconds.\n", what_time_is_it_now()-bttime); - return rects; + //printf("all time use %f seconds.\n", what_time_is_it_now()-bttime); + return objInfos; } -image DnDetect::matToImg(cv::Mat& RefImg) { +image DnDetect::DnDetect::matToImg(cv::Mat &RefImg) { CV_Assert(RefImg.depth() == CV_8U); int h = RefImg.rows; @@ -92,23 +94,28 @@ int channels = RefImg.channels(); image im = make_image(w, h, 3); int count = 0; - switch(channels){ - case 1:{ + switch (channels) { + case 1: { cv::MatIterator_<unsigned char> it, end; - for (it = RefImg.begin<unsigned char>(), end = RefImg.end<unsigned char>(); it != end; ++it){ - im.data[count] = im.data[w*h + count] = im.data[w*h*2 + count] = (float)(*it)/255.0; - + for (it = RefImg.begin<unsigned char>(), end = RefImg.end<unsigned char>(); it != end; ++it) { + im.data[count] = im.data[w * h + count] = im.data[w * h * 2 + count] = (float) (*it) / 255.0; ++count; } break; } - case 3:{ - cv::MatIterator_<cv::Vec3b> it, end; - for (it = RefImg.begin<cv::Vec3b>(), end = RefImg.end<cv::Vec3b>(); it != end; ++it){ - im.data[count] = (float)(*it)[2]/255.0; - im.data[w*h + count] = (float)(*it)[1]/255.0; - im.data[w*h*2 + count] = (float)(*it)[0]/255.0; - ++count; + case 3: { + float *desData = im.data; + uchar *srcData = RefImg.data; + + int size = w * h; + int size2 = size * 2; + for (int i = 0; i < size; i++) { + *(desData) = *(srcData + 2) / 255.0f; + *(desData + size) = *(srcData + 1) / 255.0f; + *(desData + size2) = *(srcData) / 255.0f; + + desData++; + srcData += 3; } break; } @@ -119,3 +126,28 @@ } return im; } + +int DnDetect::DnDetect::init(void *arg) { + DnDetect *p = (DnDetect *) arg; + + p->m_thresh = appPref.getFloatData("thresh.detect"); + cuda_set_device(appPref.getIntData("gpu.index")); + + char *datacfg = "cfg/coco.data"; + char *cfgfile = "cfg/yolov3.cfg"; + char *weightfile = "./yolov3.weights"; + + double loadtime = what_time_is_it_now(); + list *options = read_data_cfg(datacfg); + char *name_list = option_find_str(options, "names", "data/names.list"); + p->names = get_labels(name_list); + + p->alphabet = load_alphabet(); + p->m_net = load_network(cfgfile, weightfile, 0); + set_batch_network(p->m_net, 1); + printf("load mod use %f seconds.\n", what_time_is_it_now() - loadtime); + + srand(2222222); + p->m_bInitThd = true; + return 0; +} diff --git a/QiaoJiaSystem/YoloServer/DnDetect.h b/QiaoJiaSystem/YoloServer/DnDetect.h index 3718aed..fc1a37c 100644 --- a/QiaoJiaSystem/YoloServer/DnDetect.h +++ b/QiaoJiaSystem/YoloServer/DnDetect.h @@ -1,26 +1,83 @@ #ifndef DNDETECT_H #define DNDETECT_H #define GPU + #include <darknet.h> #include <opencv2/opencv.hpp> -class DnDetect -{ -public: - DnDetect(const int gpuIndex); - virtual ~DnDetect(); +#include <thread> - std::vector<cv::Rect2f> detect(cv::Mat&); -private: - image matToImg(cv::Mat& RefImg); +namespace DnDetect { -private: - network *m_net; - float m_thresh; - float m_hier_thresh; - float m_nms; - char **names; + class ClockTimer { + public: + ClockTimer(const std::string &flag, int nMscTimeout = 0) { + timeval time; + gettimeofday(&time, nullptr); + m_start = time.tv_sec * 1000000 + time.tv_usec; + m_flag = flag; + m_nMscTimeout = nMscTimeout; + } - image **alphabet; -}; + ~ClockTimer() { + timeval time1; + gettimeofday(&time1, nullptr); + double end = time1.tv_sec * 1000000 + time1.tv_usec; + int n = (end - m_start) / 1000; + if (0 >= m_nMscTimeout) { +// std::cout << m_flag << ">> Time elapsed: " << n << "ms" << std::endl; + INFO(m_flag << ">> Time elapsed: " << n << "ms"); + } else if (n >= m_nMscTimeout) { +// std::cout << m_flag << ">> Time out: " << n << "ms" << std::endl; + INFO(m_flag << ">> Time out: " << n << "ms"); + } + } + + private: + double m_start; + std::string m_flag; + int m_nMscTimeout; + }; + + struct Rect { + float left; + float top; + float right; + float bottom; + }; + + struct YoloObjInfo { + int type; + float prob; + Rect rcObj; + }; + + class DnDetect { + public: + DnDetect(const int gpuIndex); + + virtual ~DnDetect(); + + std::vector<YoloObjInfo> detect(cv::Mat &); + + private: + image matToImg(cv::Mat &RefImg); + + static int init(void *arg); + + private: + network *m_net; + float m_thresh; + float m_hier_thresh; + float m_nms; + char **names; + image **alphabet; + + std::mutex dataMtx; + + bool m_bInitThd; + std::thread m_thdInit; + }; +} + #endif diff --git a/QiaoJiaSystem/YoloServer/ImageDrawElement.cpp b/QiaoJiaSystem/YoloServer/ImageDrawElement.cpp index 0084b09..528bc12 100644 --- a/QiaoJiaSystem/YoloServer/ImageDrawElement.cpp +++ b/QiaoJiaSystem/YoloServer/ImageDrawElement.cpp @@ -1,30 +1,43 @@ #include "ImageDrawElement.h" +#include <basic/util/opencv/CvUtil.h> +ImageDrawElement::ImageDrawElement() { -ImageDrawElement::ImageDrawElement() -{ } -void ImageDrawElement::setRects(const std::vector<cv::Rect2f> &value) -{ - rectsMtx.lock(); - rectsBuffer = value; - rectsUpdated = true; - rectsMtx.unlock(); +void ImageDrawElement::darwProperty(cv::Mat &image, string key, string value, int x, int y) { + cv::putText(image, key + ": " + value, cv::Point(x, y), cv::HersheyFonts::FONT_HERSHEY_PLAIN, 1.5, + cv::Scalar(255, 255, 0), 2); } -void ImageDrawElement::processImage(cv::Mat &image) -{ - if(rectsUpdated){ - rectsMtx.lock(); - rects = rectsBuffer; - rectsUpdated = false; - rectsMtx.unlock(); +void ImageDrawElement::processImage(cv::Mat &image) { +// auto faceData = faces.getData(); + auto yoloObjectsData = yoloObjects.getData(); + +// for (auto face: faceData) { +// auto rect = CvUtil::zoomRect(face.rect, 1.2, 1.2); +// cv::rectangle(image, rect, face.id >= 0 ? cv::Scalar(0, 255, 0) : cv::Scalar(0, 255, 255), 2); +// int i = 0; +// for (auto &property:face.properties) { +// darwProperty(image, property.first, property.second, rect.x + rect.width, rect.y + 40 * i++); +// } +// } + +// DBG("yoloObjectsData size is " << yoloObjectsData.size()); + for (auto yoloObj: yoloObjectsData) { + auto rect = CvUtil::zoomRect(yoloObj.rect, 1, 1); + cv::rectangle(image, rect, yoloObj.id >= 0 ? cv::Scalar(255, 0, 0) : cv::Scalar(0, 255, 255), 2); + int i = 0; + for (auto &property:yoloObj.properties) { + darwProperty(image, property.first, property.second, rect.x + rect.width, rect.y + 40 * i++); + } } - for(size_t i = 0;i<rects.size();i++){ - cv::Rect2f& rectf = rects[i]; - cv::Rect rect(rectf.x*image.cols,rectf.y*image.rows,rectf.width*image.cols,rectf.height*image.rows); - cv::rectangle(image, rect,cv::Scalar(0,0,255),4); - } - fireConnectors(); +} + +//void ImageDrawElement::setFaces(std::vector<ScoredRect> value) { +// faces = value; +//} + +void ImageDrawElement::setYoloObjects(std::vector<ScoredRect> value) { + yoloObjects = value; } diff --git a/QiaoJiaSystem/YoloServer/ImageDrawElement.h b/QiaoJiaSystem/YoloServer/ImageDrawElement.h index 03fd9df..21d6012 100644 --- a/QiaoJiaSystem/YoloServer/ImageDrawElement.h +++ b/QiaoJiaSystem/YoloServer/ImageDrawElement.h @@ -1,20 +1,30 @@ #ifndef IMAGEDRAWELEMENT_H #define IMAGEDRAWELEMENT_H -#include <basic/pipe_element/ImageFactoryElement.h> -#include <mutex> -class ImageDrawElement : public ImageFactoryElement -{ +#include <YoloServer.h> +//#include <FaceServer.h> +#include <basic/pipe_element/ImageFactoryElement.h> +#include <basic/core/DoubleBufferedData.h> +#include "TrackingTrigger.h" + +class ImageDrawElement : public ImageFactoryElement { public: ImageDrawElement(); - void setRects(const std::vector<cv::Rect2f> &value); + + // ImageFactoryElement interface +// void setFaces(std::vector<ScoredRect> value); + + void setYoloObjects(std::vector<ScoredRect> value); private: - std::mutex rectsMtx; - std::vector<cv::Rect2f> rects; - std::vector<cv::Rect2f> rectsBuffer; - bool rectsUpdated; - virtual void processImage(cv::Mat& image); + virtual void processImage(cv::Mat &) override; + + void darwProperty(cv::Mat &image, string key, string value, int x, int y); + +private: +// DoubleBufferedData<std::vector<ScoredRect>> faces; + DoubleBufferedData<std::vector<ScoredRect>> yoloObjects; }; + #endif // IMAGEDRAWELEMENT_H diff --git a/QiaoJiaSystem/YoloServer/YoloDetectClientTest.cpp b/QiaoJiaSystem/YoloServer/YoloDetectClientTest.cpp index 7d7804d..974c7cc 100644 --- a/QiaoJiaSystem/YoloServer/YoloDetectClientTest.cpp +++ b/QiaoJiaSystem/YoloServer/YoloDetectClientTest.cpp @@ -9,71 +9,95 @@ #include <basic/pipe/PipeController.h> #include <basic/pipe_element/ffmpeg/VideoCaptureElement.h> #include "ImageDrawElement.h" -#include "YoloRpcElement.h" -class Controllor : public PipeController -{ +#include <YoloRpcElement.h> +#include <basic/util/app/AppPreference.hpp> + +class Controllor : public PipeController { public: - Controllor(const int index,const std::string& rtsp):m_videoCaptureElement(rtsp,25,1000,0), - m_YoloRpcElement(std::to_string(index)+"YoloRpc") - { + Controllor(const int index, const std::string &rtsp) : m_videoCaptureElement(rtsp, 25, 1000, 0), + m_YoloRpcElement(std::to_string(index) + "YoloRpc") { + m_index = index; m_rtsp = rtsp; - m_videoCaptureElement.registerConnector([&]{ - if(!m_YoloRpcElement.isBusy()){ + m_videoCaptureElement.registerConnector([&] { + if (!m_YoloRpcElement.isBusy()) { m_YoloRpcElement.setImage(m_videoCaptureElement.getImage()); - // paElement.setImage(videoCaptureElement.getImage()); - // paNumElement.setImage(videoCaptureElement.getImage()); + // paElement.setImage(videoCaptureElement.getImage()); + // paNumElement.setImage(videoCaptureElement.getImage()); m_YoloRpcElement.submit(); } - if(!m_imageDrawElement.isBusy()){ + if (!m_imageDrawElement.isBusy()) { m_imageDrawElement.setImage(m_videoCaptureElement.getImage()); m_imageDrawElement.submit(); } }); - m_YoloRpcElement.registerConnector([&]{ - m_imageDrawElement.setRects(m_YoloRpcElement.getRects()); + m_YoloRpcElement.registerConnector([&] { + m_imageDrawElement.setYoloObjects(m_YoloRpcElement.getLastScoreRects()); }); - m_imageDrawElement.registerConnector([&]{ - ImageShowElement::showImage(m_rtsp,*m_imageDrawElement.getImage()); + m_imageDrawElement.registerConnector([&] { +// ImageShowElement::showImage(std::to_string(m_index), *m_imageDrawElement.getImage()); }); - m_videoCaptureElement.setOutPutInterval(2); + m_videoCaptureElement.setOutPutInterval(3); registerElement(m_videoCaptureElement); registerElement(m_YoloRpcElement); registerElement(m_imageDrawElement); } - virtual ~Controllor() - {} + + virtual ~Controllor() {} private: std::string m_rtsp; ffmpeg::VideoCaptureElement m_videoCaptureElement; YoloRpcElement m_YoloRpcElement; ImageDrawElement m_imageDrawElement; + int m_index; }; //using namespace std; -int main(int argc, char *argv[]) -{ +int main(int argc, char *argv[]) { SAVE_APP_ARGS - Controllor _Controllor(0,"/home/basic/video/201804/day0416/01010002606000000-slice1.mp4"); - _Controllor.start(); - Controllor _Controllor0(1,"/home/basic/video/201804/day0416/01010002606000000-slice2.mp4"); - _Controllor0.start(); - cout << "Hello World!" << endl; - Controllor _Controllor1(2,"/home/basic/video/201804/day0416/01010002609000000-slice.mp4"); - _Controllor1.start(); - Controllor _Controllor2(3,"/home/basic/video/201804/day0416/01010002611000000-slice.mp4"); - _Controllor2.start(); - Controllor _Controllor3(4,"/home/basic/video/201804/night0418/02000000652000100-slice.mp4"); - _Controllor3.start(); - Controllor _Controllor4(5,"/home/basic/video/201804/night0418/02000000654000100-slice.mp4"); - _Controllor4.start(); - Controllor _Controllor5(6,"/home/basic/video/201804/night0418/02000000659000100-slice.mp4"); - _Controllor5.start(); - while(true) - {sleep(1);} + int num = atoi(argv[1]); + std::string path(argv[2]); + int portNum = atoi(argv[3]); + //yolo server + appPref.setStringData("yolo.proxy", "yoloServer"); + //#todo + appPref.setStringData("yolo.ip", ""); + appPref.setIntData("yolo.port", portNum); +// rpcClient(appPref.getStringData("yolo.proxy"), appPref.getStringData("yolo.ip"), appPref.getLongData("yolo.port"),"tcp"), + + for (int i = 0; i < num; i++) { + Controllor *_Controllor = new Controllor(i, path); + _Controllor->start(); + } + +// Controllor _Controllor(0, "/home/bsk/2.mp4"); +// _Controllor.start(); +// Controllor _Controllor1(1, "/home/bsk/2.mp4"); +// _Controllor1.start(); +// Controllor _Controllor2(2, "/home/bsk/2.mp4"); +// _Controllor2.start(); +// Controllor _Controllor3(3, "/home/bsk/2.mp4"); +// _Controllor3.start(); +// Controllor _Controllor4(4, "/home/bsk/2.mp4"); +// _Controllor4.start(); + +// Controllor _Controllor0(1,"/home/basic/video/201804/day0416/01010002606000000-slice2.mp4"); +// _Controllor0.start(); +// cout << "Hello World!" << endl; +// Controllor _Controllor1(2,"/home/basic/video/201804/day0416/01010002609000000-slice.mp4"); +// _Controllor1.start(); +// Controllor _Controllor2(3,"/home/basic/video/201804/day0416/01010002611000000-slice.mp4"); +// _Controllor2.start(); +// Controllor _Controllor3(4,"/home/basic/video/201804/night0418/02000000652000100-slice.mp4"); +// _Controllor3.start(); +// Controllor _Controllor4(5,"/home/basic/video/201804/night0418/02000000654000100-slice.mp4"); +// _Controllor4.start(); +// Controllor _Controllor5(6,"/home/basic/video/201804/night0418/02000000659000100-slice.mp4"); +// _Controllor5.start(); + while (true) { sleep(10); } return 0; } diff --git a/QiaoJiaSystem/YoloServer/YoloDetectServerI.cpp b/QiaoJiaSystem/YoloServer/YoloDetectServerI.cpp index cb0e612..02ef93f 100644 --- a/QiaoJiaSystem/YoloServer/YoloDetectServerI.cpp +++ b/QiaoJiaSystem/YoloServer/YoloDetectServerI.cpp @@ -14,67 +14,53 @@ ::YoloDetect::ObjInfos YoloDetectServerI::YoloDetect(::Ice::Int w, ::Ice::Int h, const ::std::string &shM, const ::Ice::Current &) { - ClockTimer ct("YoloDetectServerI::YoloDetect"); + DnDetect::ClockTimer ct("YoloDetectServerI::YoloDetect"); + + std::thread::id key = std::this_thread::get_id(); + DBG("key is " << key); + DnDetect::DnDetect *t_dnDetect = nullptr; + for (auto &item : map_dnDetRes) { + //鏄惁鍙戠幇pid + bool find_pid = false; + + auto &pid_map = item.second.map_pid; + for (auto &pid_item : pid_map) { + if (pid_item.second == key) { +// 鍙戠幇pid锛岃幏鍙栬祫婧愬彞鏌� + find_pid = true; + break; + } + } + int map_pid_size = item.second.map_pid.size(); + if (!find_pid && map_pid_size < item.second.i) { + //娌″彂鐜板彞鏌勶紝骞朵笖map瀹归噺瓒冲鏀惧叆鏂扮殑pid + map_pid_size++; + //#todo lock ? + item.second.map_pid[map_pid_size] = key; + find_pid = true; + } + //鍙戠幇浜唒id閫�鍑哄惊鐜� + if (find_pid) { + t_dnDetect = item.second.dnDetect; + break; + } + } + ::YoloDetect::ObjInfos objInfos; - if (!m_bInitThd) return objInfos; + if (!m_bInitThd || t_dnDetect == nullptr) { + ERR("error "); + return objInfos; + } QSharedMemory shareMemory(QString(shM.c_str())); if (shareMemory.attach()) { int channel = 3; cv::Mat _mat = bufferToMat(w, h, channel, shareMemory.constData()); - -// double bttime=what_time_is_it_now(); - image im = matToImg(_mat); -// DBG("matToImg : "<<what_time_is_it_now()-bttime); -// printf("matToImg %f seconds.\n", what_time_is_it_now()-bttime); - image sized = letterbox_image(im, m_net->w, m_net->h); - layer l = m_net->layers[m_net->n - 1]; - - float *X = sized.data; - //attime=what_time_is_it_now();p-> - network_predict(m_net, X); - //printf("Predicted in %f seconds.\n", what_time_is_it_now()-attime); - int nboxes = 0; - detection *dets = get_network_boxes(m_net, im.w, im.h, m_thresh, m_hier_thresh, 0, 1, &nboxes); - if (m_nms) do_nms_sort(dets, nboxes, l.classes, m_nms); - // draw_detections(im, dets, nboxes, m_thresh, names, alphabet, l.classes); - for (int i = 0; i < nboxes; i++) { + auto res = t_dnDetect->detect(_mat); + for (auto &item : res) { ::YoloDetect::ObjInfo objInfo; - std::vector<float> vec(80); - memcpy(&vec[0], dets[i].prob, sizeof(float) * 80); - int type = -1; - for (int j = 0; j < l.classes; ++j) { -// if(j != 0){ -// continue; -// } - if (dets[i].prob[j] > 0.0f) { - if (type < 0) { - type = j; - objInfo.prob = dets[i].prob[j]; - } else { - } - } else { - } - } - if (type >= 0) { -// if(type != 0){ -// continue; -// } - objInfo.type = type; - objInfo.rcObj.left = (dets[i].bbox.x - dets[i].bbox.w / 2.); - objInfo.rcObj.top = (dets[i].bbox.y - dets[i].bbox.h / 2.); - objInfo.rcObj.right = (dets[i].bbox.x + dets[i].bbox.w / 2.); - objInfo.rcObj.bottom = (dets[i].bbox.y + dets[i].bbox.h / 2.); - objInfos.push_back(objInfo); - } - - + memcpy(&objInfo, &item, sizeof(item)); + objInfos.push_back(objInfo); } - free_detections(dets, nboxes); - // show_image(im, "Video"); - // cv::waitKey(10); - free_image(im); - free_image(sized); - //printf("all time use %f seconds.\n", what_time_is_it_now()-bttime); } return objInfos; } @@ -82,24 +68,15 @@ int YoloDetectServerI::init(void *arg) { YoloDetectServerI *p = (YoloDetectServerI *) arg; - p->m_thresh = appPref.getFloatData("thresh.detect"); - cuda_set_device(appPref.getIntData("gpu.index")); + for (int i = 0; i < 1; i++) { + DnDetectRes t_detectRes; +// t_detectRes.dnDetect = new DnDetect::DnDetect(i % 2); + t_detectRes.dnDetect = new DnDetect::DnDetect(1); + int size = appPref.getIntData("poolNum"); + t_detectRes.i = size > 0 ? size : 1; + p->map_dnDetRes[i] = t_detectRes; - char *datacfg = "cfg/coco.data"; - char *cfgfile = "cfg/yolov3.cfg"; - char *weightfile = "./yolov3.weights"; - - double loadtime = what_time_is_it_now(); - list *options = read_data_cfg(datacfg); - char *name_list = option_find_str(options, "names", "data/names.list"); - p->names = get_labels(name_list); - - p->alphabet = load_alphabet(); - p->m_net = load_network(cfgfile, weightfile, 0); - set_batch_network(p->m_net, 1); - printf("load mod use %f seconds.\n", what_time_is_it_now() - loadtime); - - srand(2222222); + } p->m_bInitThd = true; return 0; } @@ -128,47 +105,6 @@ return mat; } -image YoloDetectServerI::matToImg(cv::Mat &RefImg) { - CV_Assert(RefImg.depth() == CV_8U); - - int h = RefImg.rows; - int w = RefImg.cols; - int channels = RefImg.channels(); - image im = make_image(w, h, 3); - int count = 0; - switch (channels) { - case 1: { - cv::MatIterator_<unsigned char> it, end; - for (it = RefImg.begin<unsigned char>(), end = RefImg.end<unsigned char>(); it != end; ++it) { - im.data[count] = im.data[w * h + count] = im.data[w * h * 2 + count] = (float) (*it) / 255.0; - - ++count; - } - break; - } - case 3: { - float *desData = im.data; - uchar *srcData = RefImg.data; - - int size = w * h; - int size2 = size * 2; - for (int i = 0; i < size; i++) { - *(desData) = *(srcData + 2) / 255.0f; - *(desData + size) = *(srcData + 1) / 255.0f; - *(desData + size2) = *(srcData) / 255.0f; - - desData++; - srcData += 3; - } - break; - } - - default: - printf("Channel number not supported.\n"); - break; - } - return im; -} YoloDetect::stringData YoloDetectServerI::getCocoData(const Ice::Current &) { YoloDetect::stringData retval; diff --git a/QiaoJiaSystem/YoloServer/YoloDetectServerI.h b/QiaoJiaSystem/YoloServer/YoloDetectServerI.h index ad18554..ed5203a 100644 --- a/QiaoJiaSystem/YoloServer/YoloDetectServerI.h +++ b/QiaoJiaSystem/YoloServer/YoloDetectServerI.h @@ -1,20 +1,40 @@ #ifndef YOLODETECTSERVERI_H #define YOLODETECTSERVERI_H #define GPU + #include "YoloServer.h" #include <thread> #include <darknet.h> #include <opencv2/opencv.hpp> #include <basic/util/resource/ResourcesManager.h> -class YoloDetectServerI : public YoloDetect::YoloDetectServer -{ + +#include "DnDetect.h" +#include <atomic> + +struct DnDetectRes { +public: + DnDetectRes() : i(3) { + } + + DnDetect::DnDetect *dnDetect; +// 搴忓彿鍙婄嚎绋媔d + std::unordered_map<int, std::thread::id> map_pid; +// 鎺ュ彈鐨勭嚎绋嬩釜鏁� + int i; +}; + +class YoloDetectServerI : public YoloDetect::YoloDetectServer { public: YoloDetectServerI(); + virtual ~YoloDetectServerI(); // FaceDetectServer interface public: - virtual ::YoloDetect::ObjInfos YoloDetect(::Ice::Int, ::Ice::Int, const ::std::string&, const ::Ice::Current& = ::Ice::emptyCurrent)override; + virtual ::YoloDetect::ObjInfos + YoloDetect(::Ice::Int, ::Ice::Int, const ::std::string &, const ::Ice::Current & = ::Ice::emptyCurrent) override; + virtual YoloDetect::stringData getCocoData(const Ice::Current &) override; + private: network *m_net; float m_thresh; @@ -25,13 +45,15 @@ bool m_bInitThd; std::thread m_thdInit; - // ResourcesManager<int> resourcesManager; + std::map<int, DnDetectRes> map_dnDetRes; + + // ResourcesManager<int> resourcesManager; private: - static int init(void* arg); - image matToImg(cv::Mat& RefImg); - cv::Mat bufferToMat(const int w,const int h,const int channels,const void* buffer); + static int init(void *arg); + + cv::Mat bufferToMat(const int w, const int h, const int channels, const void *buffer); }; diff --git a/QiaoJiaSystem/YoloServer/YoloRpcElement.cpp b/QiaoJiaSystem/YoloServer/YoloRpcElement.cpp new file mode 100644 index 0000000..cdecab1 --- /dev/null +++ b/QiaoJiaSystem/YoloServer/YoloRpcElement.cpp @@ -0,0 +1,126 @@ +#include "YoloRpcElement.h" +#include <basic/util/app/AppPreference.hpp> +#include <QtCore/QSharedMemory> +#include <QtCore/QString> +#include <basic/timer_counter/Clocktimer.h> +#include <basic/util/opencv/CvUtil.h> + + +#include <basic/db/Elasticsearch/EsDBTool.h> +#include <uuid/uuid.h> +#include <jsoncpp/json/json.h> + +#include <QtCore/QJsonDocument> +#include <QtCore/QJsonObject> + +YoloRpcElement::YoloRpcElement(string shareMemoryName) : + rpcClient(appPref.getStringData("yolo.proxy"), appPref.getStringData("yolo.ip"), + appPref.getIntData("yolo.port"), "tcp"), fdfsClient(nullptr), sharedMemory(nullptr), + m_triggerElement(0, 50), trackingTrigger(nullptr) { + sharedMemory = new QSharedMemory(QString(shareMemoryName.c_str())); +// DBG(shareMemoryName); + //1520 x 2688 1080 x 1920 //2560 * 1440 * 4 + if (!sharedMemory->create(4608 * 2592 * 4)) { + sharedMemory->attach(); + } + trackingTrigger = new TrackingTrigger(0.5); + try { + auto server = rpcClient.getServer(); + cocoData = server->getCocoData(); + } + catch (std::exception &e) { + ERR(e.what()) + } +} + +YoloRpcElement::~YoloRpcElement() { + if (sharedMemory) { + delete sharedMemory; + } + if (trackingTrigger) { + delete trackingTrigger; + } +} + +void YoloRpcElement::threadFunc() { + ClockTimer ct("YoloRpcElement::threadFunc"); + triggerMats.clear(); + try { + auto server = rpcClient.getServer(); + if (!server) { + ERR("server is null"); + return; + } + objs = server->YoloDetect(image.cols, image.rows, sharedMemory->key().toStdString());//TODO + + DBG("objs size is " << objs.size()); + if (objs.size() <= 0) { + m_triggerElement.setState(false); + m_triggerElement.triggerOnce(false); + trackingTrigger->triggerLine(); + return; + } else { + m_triggerElement.setState(true); + } + string t_camIdex = getProperty("dev_id") + getProperty("ch_id"); + float t_det_sc = 0.75; +// appPref.getFloatData(t_camIdex + "yolo.det") == -1 ? 0.75 : appPref.getFloatData(t_camIdex + "yolo.det"); + + for (auto &obj: objs) { + if (obj.type != 0) + continue; + if (obj.prob < t_det_sc) { + continue; + } + ScoredRect scoredRect; + + int x = obj.rcObj.left * image.cols; + int y = obj.rcObj.top * image.rows; + int w = (obj.rcObj.right - obj.rcObj.left) * image.cols; + int h = (obj.rcObj.bottom - obj.rcObj.top) * image.rows; + scoredRect.rect = cv::Rect(x, y, w, h); + + scoredRect.score = scoredRect.rect.area() > 0 ? obj.prob : 0; + if (trackingTrigger->triggerOnce(scoredRect)) { + trackingTrigger->getLastRect().properties["id"] = to_string(scoredRect.id); + trackingTrigger->getLastRect().properties["type"] = cocoData[obj.type]; + auto t_image = image(scoredRect.rect & cv::Rect(0, 0, image.cols, image.rows)).clone(); + triggerMats.push_back(t_image); + + } + } + trackingTrigger->triggerLine(); + m_triggerElement.triggerOnce(false); + if (triggerMats.size() > 0)fireConnectors("YoloTrigger"); + fireConnectors(); + } catch (std::exception &e) { + ERR(e.what()) + } +} + +::YoloDetect::ObjInfos YoloRpcElement::getObjects() const { + return objs; +} + +bool YoloRpcElement::getTrigger() const { + return (objs.size() > 0 ? true : false); +} + +std::vector<cv::Mat> YoloRpcElement::getTriggerMats() { + return triggerMats; +} + +std::vector<ScoredRect> YoloRpcElement::getLastScoreRects() const { + return trackingTrigger->getLastScoreRects(); +} + +void YoloRpcElement::setImage(const cv::Mat &value) { + if (value.size != image.size) { + image = cv::Mat(value.rows, value.cols, CV_8UC3, sharedMemory->data()); + } + value.copyTo(image); +} + +bool YoloRpcElement::getTriggerState() const { + return m_triggerElement.getTriggerState(); +} diff --git a/QiaoJiaSystem/YoloServer/YoloRpcElement.h b/QiaoJiaSystem/YoloServer/YoloRpcElement.h new file mode 100644 index 0000000..e31e3fa --- /dev/null +++ b/QiaoJiaSystem/YoloServer/YoloRpcElement.h @@ -0,0 +1,52 @@ +#ifndef YOLORPCELEMENT_H +#define YOLORPCELEMENT_H + +#include <YoloServer.h> +#include <basic/pipe/PipeElement.h> +#include <basic/rpc/IceRpc.hpp> +#include <opencv2/opencv.hpp> +#include <basic/util/fastdfs/FastFds.hpp> +#include <TriggerElement.h> +#include "TrackingTrigger.h" + +class QSharedMemory; + +class YoloRpcElement : public basic::PipeElement { +public: + YoloRpcElement(string); + + ~YoloRpcElement(); + + void setImage(const cv::Mat &value); + + ::YoloDetect::ObjInfos getObjects() const; + + std::vector<cv::Mat> getTriggerMats(); + + std::vector<ScoredRect> getLastScoreRects() const; + + void setFdfs(FastFdsWithLock *p_fdfsClient) { + fdfsClient = p_fdfsClient; + } + + bool getTriggerState() const; + + bool getTrigger() const; + +private: + virtual void threadFunc() override; + +private: + IceRpcClient<YoloDetect::YoloDetectServerPrx> rpcClient; + cv::Mat image; + QSharedMemory *sharedMemory; + ::YoloDetect::ObjInfos objs; + TrackingTrigger *trackingTrigger; + std::vector<cv::Mat> triggerMats; + ::YoloDetect::stringData cocoData; + FastFdsWithLock *fdfsClient; + + TriggerElement m_triggerElement; +}; + +#endif // YOLORPCELEMENT_H diff --git a/QiaoJiaSystem/YoloServer/main.cpp b/QiaoJiaSystem/YoloServer/main.cpp index e223edd..438ff3e 100644 --- a/QiaoJiaSystem/YoloServer/main.cpp +++ b/QiaoJiaSystem/YoloServer/main.cpp @@ -23,13 +23,24 @@ // adapter->add(new YoloDetectServerI(), Ice::stringToIdentity(identity)); // adapter->activate(); // ich->waitForShutdown(); - appPref.setIntData("gpu.index", 0); - appPref.setFloatData("thresh.detect", 0.7); - IceRpcServer<YoloDetectServerI> server("yoloServer", 10003, "tcp"); + if (argc < 3) { + ERR("argc error"); + return -1; + } + + int gpuindex = atoi(argv[1]); + int poolNum = atoi(argv[2]); + int portNum = atoi(argv[3]); + + appPref.setIntData("gpu.index", gpuindex); + appPref.setFloatData("thresh.detect", 0.7); + appPref.setIntData("poolNum", poolNum); + + IceRpcServer<YoloDetectServerI> server("yoloServer", portNum, "tcp"); server.setMessageSizeMax(1024 * 1024 * 50); - server.setPoolInitSize(5); - server.setPoolMaxSize(5); + server.setPoolInitSize(poolNum); + server.setPoolMaxSize(poolNum); server.runWaitShutDown(); return 0; } -- Gitblit v1.8.0