From 72481c0ede11920945d886ff9ad4be4b266863a3 Mon Sep 17 00:00:00 2001
From: houxiao <houxiao@454eff88-639b-444f-9e54-f578c98de674>
Date: 星期四, 20 七月 2017 20:00:08 +0800
Subject: [PATCH] lib
---
VisitFace/RtspNativeCodec/app/src/main/cpp/FaceCache.cpp | 459 ++++++++++++++++++++++++++++++++++++++++++++++++++++++--
1 files changed, 437 insertions(+), 22 deletions(-)
diff --git a/VisitFace/RtspNativeCodec/app/src/main/cpp/FaceCache.cpp b/VisitFace/RtspNativeCodec/app/src/main/cpp/FaceCache.cpp
index 0b8e658..0871fdc 100644
--- a/VisitFace/RtspNativeCodec/app/src/main/cpp/FaceCache.cpp
+++ b/VisitFace/RtspNativeCodec/app/src/main/cpp/FaceCache.cpp
@@ -5,33 +5,146 @@
#include <logger.h>
#include <Logger/src/logger.hpp>
+#include <opencv/cv.h>
+#include <opencv/cxcore.h>
+#include <opencv/highgui.h>
+
+#include <vector>
+
+#include <PbFaceList.pb.h>
+
+#include <libyuv.h>
+#include <MediaHelper.h>
+
+//#define YUV420_TO_RGB888 1
+#define YUV420_TO_RGB565 1
+//#define YUV420_TO_ABGR8888 1
+//#define NV12_TO_ABGR8888 1
+
struct FcPmBreackerContext
{
uint8_t frameYUV[1920*1080*4];
size_t frameYUVSize;
+ MB_Frame::MBFType frameYUVType;
- uint8_t frameRGB[1920*1080*4];
+ uint8_t frameRGB[1920*1088*4];
size_t frameRGBSize;
+
+ size_t width;
+ size_t height;
st_ff_vect_t faceFeatures;
bool dataAvailable;
- FcPmBreackerContext() : frameYUV(), frameYUVSize(0), frameRGB(), frameRGBSize(0), faceFeatures(), dataAvailable(false)
+ FcPmBreackerContext() :
+ frameYUV(), frameYUVSize(0), frameYUVType(MB_Frame::MBFT__FIRST), frameRGB(), frameRGBSize(0),
+ faceFeatures(), width(0), height(0), dataAvailable(false)
{
}
void reset()
{
frameYUVSize = 0;
- frameRGBSize = 0;
+ frameRGBSize = 0;
+ width = 0;
+ height = 0;
dataAvailable = false;
faceFeatures.clear();
}
- bool convertYUV420ToRGB()
+ bool convertYUV420ToRGB888()
{
+ int src_height = height;
+ int src_width = width;
+ const uint8* src_y = (const uint8*)(frameYUV);
+ const uint8* src_u = (const uint8*)(src_y + (src_height * src_width));
+ const uint8* src_v = (const uint8*)(src_u + (src_height * src_width / 4));
+
+ libyuv::I420ToRGB24(src_y, src_width,
+ src_u, MH_SUBSAMPLE1(src_width, 2),
+ src_v, MH_SUBSAMPLE1(src_width, 2),
+ frameRGB, 3 * src_width,
+ src_width, src_height);
+ frameRGBSize = src_height * src_width * 3;
+
+ //{
+ // static size_t f = 0;
+ // char fname[50];
+ // sprintf(fname, "/sdcard/face-%u.rgb", ++f);
+ // FILE *pFile = fopen(fname, "wb");
+ // fwrite(frameRGB, 1, frameRGBSize, pFile);
+ // fclose(pFile);
+ // if (f > 20)exit(0);
+ //}
+
+ return true;
}
+
+ bool convertYUV420ToRGB565()
+ {
+ int src_height = height;
+ int src_width = width;
+ const uint8* src_y = (const uint8*)(frameYUV);
+ const uint8* src_u = (const uint8*)(src_y + (src_height * src_width));
+ const uint8* src_v = (const uint8*)(src_u + (src_height * src_width / 4));
+
+ if (frameYUVType == MB_Frame::MBFT_YUV420)
+ {
+ libyuv::I420ToRGB565(src_y, src_width,
+ src_u, MH_SUBSAMPLE1(src_width, 2),
+ src_v, MH_SUBSAMPLE1(src_width, 2),
+ frameRGB, 2 * src_width,
+ src_width, src_height);
+ }
+ else if (frameYUVType == MB_Frame::MBFT_NV12)
+ {
+ libyuv::NV12ToRGB565(src_y, src_width,
+ src_u, src_width,
+ frameRGB, 2 * src_width,
+ src_width, src_height);
+
+ //{
+ // static size_t f = 0;
+ // char fname[50];
+ // sprintf(fname, "/sdcard/face-%u.rgb", ++f);
+ // FILE *pFile = fopen(fname, "wb");
+ // fwrite(frameRGB, 1, frameRGBSize, pFile);
+ // fclose(pFile);
+ // if (f > 20)exit(0);
+ //}
+ }
+ frameRGBSize = src_height * src_width * 2;
+ return true;
+ }
+
+ bool convertYUV420ToABGR8888()
+ {
+ int src_height = height;
+ int src_width = width;
+ const uint8* src_y = (const uint8*)(frameYUV);
+ const uint8* src_u = (const uint8*)(src_y + (src_height * src_width));
+ const uint8* src_v = (const uint8*)(src_u + (src_height * src_width / 4));
+
+ libyuv::I420ToABGR(src_y, src_width, // android ARGB_8888 is ABGR
+ src_u, MH_SUBSAMPLE1(src_width, 2),
+ src_v, MH_SUBSAMPLE1(src_width, 2),
+ frameRGB, 4 * src_width,
+ src_width, src_height);
+ frameRGBSize = src_height * src_width * 4;
+
+ //{
+ // static size_t f = 0;
+ // char fname[50];
+ // sprintf(fname, "/sdcard/face-%u.argb", ++f);
+ // FILE *pFile = fopen(fname, "wb");
+ // fwrite(frameRGB, 1, frameRGBSize, pFile);
+ // fclose(pFile);
+ // if (f > 10)exit(0);
+ //}
+
+ return true;
+ }
};
bool fc_pm_breaker_ptr(const PipeMaterial* pm, void* args)
@@ -43,7 +156,16 @@
ctx->faceFeatures = faceFeatures;
ctx->dataAvailable &= true;
- return false;
+ return true;
+}
+
+bool fc_pm_breaker_ptr_count(const PipeMaterial* pm, void* args)
+{
+ size_t* count = (size_t*)args;
+
+ const st_ff_vect_t& faceFeatures(*(const st_ff_vect_t*)(pm->buffer));
+ *count = faceFeatures.size();
+ return false;
}
bool fc_pm_breaker_frame(const PipeMaterial* pm, void* args)
@@ -51,7 +173,7 @@
FcPmBreackerContext* ctx = (FcPmBreackerContext*)args;
const MB_Frame* lastFrame((const MB_Frame*)(pm->buffer));
- if (lastFrame->type != MB_Frame::MBFT_YUV420)
+ if (lastFrame->type != MB_Frame::MBFT_YUV420 && lastFrame->type != MB_Frame::MBFT_NV12)
{
ctx->dataAvailable &= false;
return false;
@@ -63,13 +185,24 @@
return false;
}
- ctx->frameYUVSize = lastFrame->buffSize;
+ ctx->frameYUVSize = lastFrame->buffSize;
+ ctx->frameYUVType = lastFrame->type;
+ ctx->width = lastFrame->width;
+ ctx->height = lastFrame->height;
memcpy(ctx->frameYUV, lastFrame->buffer, ctx->frameYUVSize);
ctx->dataAvailable &= true;
- return false;
+ return true;
}
+NativeImgIdx::operator std::string() const
+{
+ char buf[256];
+ sprintf(buf, "offset=%d, size=%d, type=%d, width=%d, height=%d", offset, size, type, width, height);
+ return std::string(buf);
+}
+
+#ifdef USE_ST_SDK
FaceCache::FaceCache() : _ctx(new FcPmBreackerContext)
{
}
@@ -78,10 +211,14 @@
{
delete (FcPmBreackerContext*)_ctx;
}
+#endif
// returns count of face
int FaceCache::cachePm(const PipeMaterial& pm)
{
+ if (_ctx == nullptr)
+ _ctx = new FcPmBreackerContext;
+
FcPmBreackerContext& ctx(*(FcPmBreackerContext*)_ctx);
ctx.reset();
ctx.dataAvailable = true;
@@ -94,25 +231,303 @@
return 0;
}
-bool FaceCache::getFaceListPb(uint8_t* buffer, size_t& buffSize)
+// returns count of face
+size_t FaceCache::getFaceCount(const PipeMaterial& pm) const
{
- return false;
+ size_t count = 0;
+ pm.breake(PipeMaterial::PMT_PTR, MB_Frame::MBFT__FIRST, fc_pm_breaker_ptr_count, &count);
+
+ return count;
}
-bool FaceCache::getFaceListImage(int* buffIdx, size_t& count, uint8_t* buffImg, size_t& buffImgSize)
+bool FaceCache::getFaceListPb(uint8_t* buffer, size_t& buffMaxSize)
{
- return false;
+ FcPmBreackerContext& ctx(*(FcPmBreackerContext*)_ctx);
+ if (!ctx.dataAvailable)
+ return false;
+
+ PbFaceList pbFaceList;
+ pbFaceList.set_magic(pbFaceList.magic());
+ pbFaceList.set_image_count(ctx.faceFeatures.size());
+ pbFaceList.set_src_width(ctx.width);
+ pbFaceList.set_src_height(ctx.height);
+
+ cv::Mat yMat(cv::Size(ctx.width, ctx.height), CV_8UC1, ctx.frameYUV);
+
+ for (int i = 0; i < ctx.faceFeatures.size(); i++)
+ {
+ if (! ctx.faceFeatures[i].test_face_in_cone(35.0f, 35.0f, 35.0f))
+ continue;
+
+ const PLGH_Rect& faceRect(ctx.faceFeatures[i].rect);
+ cv::Mat roiMat1(yMat, cv::Rect(faceRect.leftTop.X, faceRect.leftTop.Y, faceRect.width(), faceRect.height()));
+ cv::Mat roiMat(roiMat1.clone()); // #todo copy data should be avoid!!!!
+ size_t s = roiMat.total() * roiMat.elemSize();
+
+ PbFaceList_FaceListImage& pbFaceListImage(*(pbFaceList.add_images()));
+ pbFaceListImage.set_idx(i);
+ pbFaceListImage.set_size(s);
+ pbFaceListImage.set_type(PbFaceList_FaceListImage_ImageType_MBFT_Y8);
+ pbFaceListImage.set_width(roiMat.cols);
+ pbFaceListImage.set_height(roiMat.rows);
+ pbFaceListImage.set_top_left_x(faceRect.leftTop.X);
+ pbFaceListImage.set_top_left_y(faceRect.leftTop.Y);
+
+ pbFaceListImage.add_img(roiMat.ptr(), s);
+
+ //{
+ // static size_t f = 0;
+ // char fname[50];
+ // sprintf(fname, "/sdcard/face-%u.yuv", ++f);
+ // FILE *pFile = fopen(fname, "wb");
+ // fwrite(roiMat.ptr(), 1, s, pFile);
+ // fclose(pFile);
+ // if (f > 20)exit(0);
+ //}
+
+ //{
+ // static size_t f = 0;
+ // char fname[50];
+ // sprintf(fname, "/sdcard/face-%u.yuv", ++f);
+ // FILE *pFile = fopen(fname, "wb");
+ // const std::string* img(*pbFaceListImage.img().data());
+ // fwrite(img->data(), 1, img->size(), pFile);
+ // fclose(pFile);
+ // if (f > 20)exit(0);
+ //}
+ }
+
+ size_t s = pbFaceList.ByteSize();
+ buffMaxSize = std::min(s, buffMaxSize);
+ pbFaceList.SerializeToArray(buffer, buffMaxSize);
+
+ //{
+ // static size_t f = 0;
+ // char fname[50];
+ // sprintf(fname, "/sdcard/facelist-%u.pb", ++f);
+ // FILE *pFile = fopen(fname, "wb");
+ // fwrite(buffer, 1, buffMaxSize, pFile);
+ // fclose(pFile);
+ // if (f > 10)exit(0);
+ //}
+
+ return buffMaxSize > 0;
}
-bool FaceCache::extractFace()
+#ifdef YUV420_TO_RGB888
+
+bool FaceCache::getFaceListImage(std::vector<NativeImgIdx>& imgIdxes, uint8_t* buffImg, size_t& buffImgMaxSize)
{
- FcPmBreackerContext& ctx(*(FcPmBreackerContext*)_ctx);
-
- if (ctx.frameRGBSize == 0)
- {
- if (! ctx.convertYUV420ToRGB())
- return false;
- }
-
-
+ FcPmBreackerContext& ctx(*(FcPmBreackerContext*)_ctx);
+ if (!ctx.dataAvailable)
+ return false;
+
+ if (ctx.frameRGBSize == 0)
+ {
+ //#todo should optimize not convert the whole image
+ if (! ctx.convertYUV420ToRGB888())
+ return false;
+ }
+
+ imgIdxes.clear();
+ size_t offset = 0;
+
+ cv::Mat rgbMat(cv::Size(ctx.width, ctx.height), CV_8UC3, ctx.frameRGB);
+ for (st_ff_vect_t::const_iterator ffiter = ctx.faceFeatures.begin(); ffiter != ctx.faceFeatures.end(); ++ffiter)
+ {
+ const FaceRect& faceRect(ffiter->rect);
+ cv::Mat roiMat1(rgbMat, cv::Rect(faceRect.leftTop.x, faceRect.leftTop.y, faceRect.width(), faceRect.height()));
+ cv::Mat roiMat(roiMat1.clone()); // #todo copy data should be avoid!!!!
+
+ NativeImgIdx imgidx;
+ imgidx.offset = offset;
+ imgidx.size = roiMat.total() * roiMat.elemSize();
+ imgidx.type = MB_Frame::MBFT_RGB888;
+ imgidx.width = roiMat.cols;
+ imgidx.height = roiMat.rows;
+
+ const size_t newSize = imgidx.offset + imgidx.size;
+
+ //isContinuous
+ if (newSize > buffImgMaxSize)
+ {
+ LOG_ERROR << "FaceCache::getFaceListImage buffImgMaxSize truncated" << LOG_ENDL;
+ return false;
+ }
+
+
+ uint8_t* pbuf = buffImg + imgidx.offset;
+ memcpy(pbuf, roiMat.ptr(), imgidx.size);
+
+ //uint8_t* pbuf = buffImg + imgidx.offset;
+ //cv::Mat roiMat1(roiMat.cols, roiMat.rows, CV_8UC3, pbuf);
+ //pbuf[0]=123;
+ //roiMat.copyTo(roiMat1);
+ //memcpy(buffImg + imgidx.offset, roiMat.ptr(), imgidx.size);
+
+ //uint8_t* pbuf = buffImg + imgidx.offset;
+ //for(size_t r = 0; r < roiMat.rows; r++)
+ //{
+ // const uint8_t* pcol = roiMat.ptr<uint8_t>(r);
+ // memcpy(pbuf, pcol, roiMat.cols);
+ // pbuf += roiMat.cols;
+ //}
+
+ //{
+ // static size_t f = 0;
+ // char fname[50];
+ // sprintf(fname, "/sdcard/face-%u.rgb", ++f);
+ // FILE *pFile = fopen(fname, "wb");
+ // fwrite(pbuf, 1, imgidx.size, pFile);
+ // fclose(pFile);
+ // if (f > 20)exit(0);
+ //}
+
+ imgIdxes.push_back(imgidx);
+ offset = newSize;
+ }
+
+ buffImgMaxSize = offset;
+ return true;
}
+
+#elif YUV420_TO_RGB565
+
+bool FaceCache::getFaceListImage(std::vector<NativeImgIdx>& imgIdxes, uint8_t* buffImg, size_t& buffImgMaxSize)
+{
+ FcPmBreackerContext& ctx(*(FcPmBreackerContext*)_ctx);
+ if (!ctx.dataAvailable)
+ return false;
+
+ if (ctx.frameRGBSize == 0)
+ {
+ //#todo should optimize not convert the whole image
+ if (! ctx.convertYUV420ToRGB565())
+ return false;
+ }
+
+ imgIdxes.clear();
+ size_t offset = 0;
+
+ cv::Mat rgbMat(cv::Size(ctx.width, ctx.height), CV_16UC1, ctx.frameRGB);
+ for (st_ff_vect_t::const_iterator ffiter = ctx.faceFeatures.begin(); ffiter != ctx.faceFeatures.end(); ++ffiter)
+ {
+ if (! ffiter->test_face_in_cone(35.0f, 35.0f, 35.0f))
+ continue;
+
+ const PLGH_Rect& faceRect(ffiter->rect);
+ cv::Mat roiMat1(rgbMat, cv::Rect(faceRect.leftTop.X, faceRect.leftTop.Y, faceRect.width(), faceRect.height()));
+ cv::Mat roiMat(roiMat1.clone()); // #todo copy data should be avoid!!!!
+
+ NativeImgIdx imgidx;
+ imgidx.offset = offset;
+ imgidx.size = roiMat.total() * roiMat.elemSize();
+ imgidx.type = MB_Frame::MBFT_RGB565;
+ imgidx.width = roiMat.cols;
+ imgidx.height = roiMat.rows;
+
+ const size_t newSize = imgidx.offset + imgidx.size;
+
+ //isContinuous
+ if (newSize > buffImgMaxSize)
+ {
+ LOG_ERROR << "FaceCache::getFaceListImage buffImgMaxSize truncated" << LOG_ENDL;
+ return false;
+ }
+
+ /*
+ uint8_t* pbuf = buffImg + imgidx.offset;
+ uint8_t* psrc = ctx.frameRGB + (ctx.width * faceRect.leftTop.y + faceRect.leftTop.x) * 2;
+ for (int row = 0; row < faceRect.height(); row++)
+ {
+ memcpy(pbuf, psrc, imgidx.width * 2);
+ pbuf += imgidx.width * 2;
+ psrc += ctx.width * 2;
+ }
+ */
+
+ uint8_t* pbuf = buffImg + imgidx.offset;
+ memcpy(pbuf, roiMat.ptr(), imgidx.size);
+
+ //{
+ // static size_t f = 0;
+ // char fname[50];
+ // sprintf(fname, "/sdcard/face-%u-w%d-h%d.rgb565", ++f, imgidx.width, imgidx.height);
+ // FILE *pFile = fopen(fname, "wb");
+ // fwrite(pbuf, 1, imgidx.size, pFile);
+ // fclose(pFile);
+ // if (f > 20)exit(0);
+ //}
+
+ imgIdxes.push_back(imgidx);
+ offset = newSize;
+ }
+
+ buffImgMaxSize = offset;
+ return true;
+}
+
+#elif YUV420_TO_ABGR8888
+
+bool FaceCache::getFaceListImage(std::vector<NativeImgIdx>& imgIdxes, uint8_t* buffImg, size_t& buffImgMaxSize)
+{
+ FcPmBreackerContext& ctx(*(FcPmBreackerContext*)_ctx);
+ if (!ctx.dataAvailable)
+ return false;
+
+ if (ctx.frameRGBSize == 0)
+ {
+ //#todo should optimize not convert the whole image
+ if (! ctx.convertYUV420ToABGR8888())
+ return false;
+ }
+
+ imgIdxes.clear();
+ size_t offset = 0;
+
+ cv::Mat rgbMat(cv::Size(ctx.width, ctx.height), CV_8UC4, ctx.frameRGB);
+ for (st_ff_vect_t::const_iterator ffiter = ctx.faceFeatures.begin(); ffiter != ctx.faceFeatures.end(); ++ffiter)
+ {
+ const FaceRect& faceRect(ffiter->rect);
+
+ cv::Mat roiMat1(rgbMat, cv::Rect(faceRect.leftTop.x, faceRect.leftTop.y, faceRect.rightBottom.x - faceRect.leftTop.x, faceRect.rightBottom.y - faceRect.leftTop.y));
+ cv::Mat roiMat(roiMat1.clone()); // #todo copy data should be avoid!!!!
+
+ NativeImgIdx imgidx;
+ imgidx.offset = offset;
+ imgidx.size = roiMat.total() * roiMat.elemSize();
+ imgidx.type = MB_Frame::MBFT_ABGR8888;
+ imgidx.width = roiMat.cols;
+ imgidx.height = roiMat.rows;
+
+ const size_t newSize = imgidx.offset + imgidx.size;
+
+ //isContinuous
+ if (newSize > buffImgMaxSize)
+ {
+ LOG_ERROR << "FaceCache::getFaceListImage buffImgMaxSize truncated" << LOG_ENDL;
+ return false;
+ }
+
+ uint8_t* pbuf = buffImg + imgidx.offset;
+ memcpy(pbuf, roiMat.ptr(), imgidx.size);
+
+ //{
+ // static size_t f = 0;
+ // char fname[50];
+ // sprintf(fname, "/sdcard/face-%u-w%d-h%d.argb", ++f, imgidx.width, imgidx.height);
+ // FILE *pFile = fopen(fname, "wb");
+ // fwrite(pbuf, 1, imgidx.size, pFile);
+ // fclose(pFile);
+ // if (f > 20)exit(0);
+ //}
+
+ imgIdxes.push_back(imgidx);
+ offset = newSize;
+ }
+
+ buffImgMaxSize = offset;
+ return true;
+}
+
+#endif
--
Gitblit v1.8.0