#include "FaceCache.h" #include "PipeLine.h" #include "MaterialBuffer.h" #include "PL_SensetimeFaceTrack.h" #include #include #include #include #include #include #include "PbFaceList.pb.h" #include struct FcPmBreackerContext { uint8_t frameYUV[1920*1080*4]; size_t frameYUVSize; uint8_t frameRGB[1920*1080*4]; size_t frameRGBSize; size_t width; size_t height; st_ff_vect_t faceFeatures; bool dataAvailable; FcPmBreackerContext() : frameYUV(), frameYUVSize(0), frameRGB(), frameRGBSize(0), faceFeatures(), width(0), height(0), dataAvailable(false) { } void reset() { frameYUVSize = 0; frameRGBSize = 0; width = 0; height = 0; dataAvailable = false; faceFeatures.clear(); } bool convertYUV420ToRGB() { #define SUBSAMPLE(v, a) ((((v) + (a) - 1)) / (a)) int src_height = height; int src_width = width; const uint8* src_y = (const uint8*)(frameYUV); const uint8* src_u = (const uint8*)(src_y + (src_height * src_width)); const uint8* src_v = (const uint8*)(src_u + (src_height * src_width / 4)); libyuv::I420ToRGB24(src_y, src_width, src_u, SUBSAMPLE(src_width, 2), src_v, SUBSAMPLE(src_width, 2), frameRGB, 3 * src_width, src_width, src_height); frameRGBSize = src_height * src_width * 3; return true; } }; bool fc_pm_breaker_ptr(const PipeMaterial* pm, void* args) { FcPmBreackerContext* ctx = (FcPmBreackerContext*)args; const st_ff_vect_t& faceFeatures(*(const st_ff_vect_t*)(pm->buffer)); ctx->faceFeatures = faceFeatures; ctx->dataAvailable &= true; return true; } bool fc_pm_breaker_frame(const PipeMaterial* pm, void* args) { FcPmBreackerContext* ctx = (FcPmBreackerContext*)args; const MB_Frame* lastFrame((const MB_Frame*)(pm->buffer)); if (lastFrame->type != MB_Frame::MBFT_YUV420) { ctx->dataAvailable &= false; return false; } if (lastFrame->buffSize == 0) { ctx->dataAvailable &= false; return false; } ctx->frameYUVSize = lastFrame->buffSize; ctx->width = lastFrame->width; ctx->height = lastFrame->height; memcpy(ctx->frameYUV, lastFrame->buffer, ctx->frameYUVSize); ctx->dataAvailable &= true; return true; } #ifdef USE_ST_SDK FaceCache::FaceCache() : _ctx(new FcPmBreackerContext) { } FaceCache::~FaceCache() { delete (FcPmBreackerContext*)_ctx; } #endif // returns count of face int FaceCache::cachePm(const PipeMaterial& pm) { if (_ctx == nullptr) _ctx = new FcPmBreackerContext; FcPmBreackerContext& ctx(*(FcPmBreackerContext*)_ctx); ctx.reset(); ctx.dataAvailable = true; pm.breake(PipeMaterial::PMT_PTR, MB_Frame::MBFT__FIRST, fc_pm_breaker_ptr, _ctx); pm.breake(PipeMaterial::PMT_FRAME, MB_Frame::MBFT__FIRST, fc_pm_breaker_frame, _ctx); if (ctx.dataAvailable) return ctx.faceFeatures.size(); else return 0; } bool FaceCache::getFaceListPb(uint8_t* buffer, size_t& buffMaxSize) { FcPmBreackerContext& ctx(*(FcPmBreackerContext*)_ctx); if (!ctx.dataAvailable) return false; PbFaceList pbFaceList; pbFaceList.set_magic(pbFaceList.magic()); pbFaceList.set_image_count(ctx.faceFeatures.size()); pbFaceList.set_src_width(ctx.width); pbFaceList.set_src_height(ctx.height); cv::Mat yMat(cv::Size(ctx.width, ctx.height), CV_8UC1, ctx.frameYUV); for (int i = 0; i < ctx.faceFeatures.size(); i++) { const FaceRect& faceRect(ctx.faceFeatures[i].rect); cv::Mat roiMat(yMat, cv::Rect(faceRect.leftTop.x, faceRect.leftTop.y, faceRect.rightBottom.x - faceRect.leftTop.x, faceRect.rightBottom.y - faceRect.leftTop.y)); PbFaceList_FaceListImage pbFaceListImage; pbFaceListImage.set_idx(i); pbFaceListImage.set_size(roiMat.elemSize()); pbFaceListImage.set_type(PbFaceList_FaceListImage_ImageType_IT_Y); pbFaceListImage.set_width(roiMat.cols); pbFaceListImage.set_height(roiMat.rows); pbFaceListImage.set_top_left_x(faceRect.leftTop.x); pbFaceListImage.set_top_left_y(faceRect.leftTop.y); pbFaceListImage.add_img(roiMat.ptr(), roiMat.total() * roiMat.elemSize()); } size_t s = pbFaceList.ByteSize(); buffMaxSize = std::min(s, buffMaxSize); pbFaceList.SerializeToArray(buffer, buffMaxSize); return buffMaxSize > 0; } bool FaceCache::getFaceListImage(int* buffIdx, size_t& count, uint8_t* buffImg, size_t& buffImgMaxSize) { FcPmBreackerContext& ctx(*(FcPmBreackerContext*)_ctx); if (ctx.frameRGBSize == 0) { //#todo should optimize not convert the whole image if (! ctx.convertYUV420ToRGB()) return false; } count = 0; uint8_t* pBuff = buffImg; size_t totalSize = 0; cv::Mat rgbMat(cv::Size(ctx.width, ctx.height), CV_8UC3, ctx.frameRGB); for (st_ff_vect_t::const_iterator ffiter = ctx.faceFeatures.begin(); ffiter != ctx.faceFeatures.end(); ++ffiter) { const FaceRect& faceRect(ffiter->rect); cv::Mat roiMat(rgbMat, cv::Rect(faceRect.leftTop.x, faceRect.leftTop.y, faceRect.rightBottom.x - faceRect.leftTop.x, faceRect.rightBottom.y - faceRect.leftTop.y)); //isContinuous size_t s = roiMat.total() * roiMat.elemSize(); if (totalSize + s <= buffImgMaxSize) { memcpy(pBuff, roiMat.ptr(), s); *buffIdx = int(totalSize); pBuff += s; totalSize += s; count++; buffIdx++; } } buffImgMaxSize = totalSize; return (count != 0); }