#include "FaceCache.h" #include "PipeLine.h" #include "MaterialBuffer.h" #include "PL_SensetimeFaceTrackMultiTrd.h" #include #include #include #include #include #include #include #include #include //#define YUV420_TO_RGB888 1 #define YUV420_TO_RGB565 1 //#define YUV420_TO_ABGR8888 1 //#define NV12_TO_ABGR8888 1 struct FcPmBreackerContext { uint8_t frameYUV[1920*1080*4]; size_t frameYUVSize; MB_Frame::MBFType frameYUVType; uint8_t frameRGB[1920*1088*4]; size_t frameRGBSize; size_t width; size_t height; st_ff_vect_t faceFeatures; bool dataAvailable; FcPmBreackerContext() : frameYUV(), frameYUVSize(0), frameYUVType(MB_Frame::MBFT__FIRST), frameRGB(), frameRGBSize(0), faceFeatures(), width(0), height(0), dataAvailable(false) { } void reset() { frameYUVSize = 0; frameRGBSize = 0; width = 0; height = 0; dataAvailable = false; faceFeatures.clear(); } bool convertYUV420ToRGB888() { int src_height = height; int src_width = width; const uint8* src_y = (const uint8*)(frameYUV); const uint8* src_u = (const uint8*)(src_y + (src_height * src_width)); const uint8* src_v = (const uint8*)(src_u + (src_height * src_width / 4)); libyuv::I420ToRGB24(src_y, src_width, src_u, MH_SUBSAMPLE1(src_width, 2), src_v, MH_SUBSAMPLE1(src_width, 2), frameRGB, 3 * src_width, src_width, src_height); frameRGBSize = src_height * src_width * 3; //{ // static size_t f = 0; // char fname[50]; // sprintf(fname, "/sdcard/face-%u.rgb", ++f); // FILE *pFile = fopen(fname, "wb"); // fwrite(frameRGB, 1, frameRGBSize, pFile); // fclose(pFile); // if (f > 20)exit(0); //} return true; } bool convertYUV420ToRGB565() { int src_height = height; int src_width = width; const uint8* src_y = (const uint8*)(frameYUV); const uint8* src_u = (const uint8*)(src_y + (src_height * src_width)); const uint8* src_v = (const uint8*)(src_u + (src_height * src_width / 4)); if (frameYUVType == MB_Frame::MBFT_YUV420) { libyuv::I420ToRGB565(src_y, src_width, src_u, MH_SUBSAMPLE1(src_width, 2), src_v, MH_SUBSAMPLE1(src_width, 2), frameRGB, 2 * src_width, src_width, src_height); } else if (frameYUVType == MB_Frame::MBFT_NV12) { libyuv::NV12ToRGB565(src_y, src_width, src_u, src_width, frameRGB, 2 * src_width, src_width, src_height); //{ // static size_t f = 0; // char fname[50]; // sprintf(fname, "/sdcard/face-%u.rgb", ++f); // FILE *pFile = fopen(fname, "wb"); // fwrite(frameRGB, 1, frameRGBSize, pFile); // fclose(pFile); // if (f > 20)exit(0); //} } frameRGBSize = src_height * src_width * 2; return true; } bool convertYUV420ToABGR8888() { int src_height = height; int src_width = width; const uint8* src_y = (const uint8*)(frameYUV); const uint8* src_u = (const uint8*)(src_y + (src_height * src_width)); const uint8* src_v = (const uint8*)(src_u + (src_height * src_width / 4)); libyuv::I420ToABGR(src_y, src_width, // android ARGB_8888 is ABGR src_u, MH_SUBSAMPLE1(src_width, 2), src_v, MH_SUBSAMPLE1(src_width, 2), frameRGB, 4 * src_width, src_width, src_height); frameRGBSize = src_height * src_width * 4; //{ // static size_t f = 0; // char fname[50]; // sprintf(fname, "/sdcard/face-%u.argb", ++f); // FILE *pFile = fopen(fname, "wb"); // fwrite(frameRGB, 1, frameRGBSize, pFile); // fclose(pFile); // if (f > 10)exit(0); //} return true; } }; bool fc_pm_breaker_ptr(const PipeMaterial* pm, void* args) { FcPmBreackerContext* ctx = (FcPmBreackerContext*)args; const st_ff_vect_t& faceFeatures(*(const st_ff_vect_t*)(pm->buffer)); ctx->faceFeatures = faceFeatures; ctx->dataAvailable &= true; return true; } bool fc_pm_breaker_ptr_count(const PipeMaterial* pm, void* args) { size_t* count = (size_t*)args; const st_ff_vect_t& faceFeatures(*(const st_ff_vect_t*)(pm->buffer)); *count = faceFeatures.size(); return false; } bool fc_pm_breaker_frame(const PipeMaterial* pm, void* args) { FcPmBreackerContext* ctx = (FcPmBreackerContext*)args; const MB_Frame* lastFrame((const MB_Frame*)(pm->buffer)); if (lastFrame->type != MB_Frame::MBFT_YUV420 && lastFrame->type != MB_Frame::MBFT_NV12) { ctx->dataAvailable &= false; return false; } if (lastFrame->buffSize == 0) { ctx->dataAvailable &= false; return false; } ctx->frameYUVSize = lastFrame->buffSize; ctx->frameYUVType = lastFrame->type; ctx->width = lastFrame->width; ctx->height = lastFrame->height; memcpy(ctx->frameYUV, lastFrame->buffer, ctx->frameYUVSize); ctx->dataAvailable &= true; return true; } NativeImgIdx::operator std::string() const { char buf[256]; sprintf(buf, "offset=%d, size=%d, type=%d, width=%d, height=%d", offset, size, type, width, height); return std::string(buf); } #ifdef USE_ST_SDK FaceCache::FaceCache() : _ctx(new FcPmBreackerContext) { } FaceCache::~FaceCache() { delete (FcPmBreackerContext*)_ctx; } #endif // returns count of face int FaceCache::cachePm(const PipeMaterial& pm) { if (_ctx == nullptr) _ctx = new FcPmBreackerContext; FcPmBreackerContext& ctx(*(FcPmBreackerContext*)_ctx); ctx.reset(); ctx.dataAvailable = true; pm.breake(PipeMaterial::PMT_PTR, MB_Frame::MBFT__FIRST, fc_pm_breaker_ptr, _ctx); pm.breake(PipeMaterial::PMT_FRAME, MB_Frame::MBFT__FIRST, fc_pm_breaker_frame, _ctx); if (ctx.dataAvailable) return ctx.faceFeatures.size(); else return 0; } // returns count of face size_t FaceCache::getFaceCount(const PipeMaterial& pm) const { size_t count = 0; pm.breake(PipeMaterial::PMT_PTR, MB_Frame::MBFT__FIRST, fc_pm_breaker_ptr_count, &count); return count; } bool FaceCache::getFaceListPb(uint8_t* buffer, size_t& buffMaxSize) { FcPmBreackerContext& ctx(*(FcPmBreackerContext*)_ctx); if (!ctx.dataAvailable) return false; PbFaceList pbFaceList; pbFaceList.set_magic(pbFaceList.magic()); pbFaceList.set_image_count(ctx.faceFeatures.size()); pbFaceList.set_src_width(ctx.width); pbFaceList.set_src_height(ctx.height); cv::Mat yMat(cv::Size(ctx.width, ctx.height), CV_8UC1, ctx.frameYUV); for (int i = 0; i < ctx.faceFeatures.size(); i++) { if (! ctx.faceFeatures[i].test_face_in_cone(35.0f, 35.0f, 35.0f)) continue; const PLGH_Rect& faceRect(ctx.faceFeatures[i].rect); cv::Mat roiMat1(yMat, cv::Rect(faceRect.leftTop.X, faceRect.leftTop.Y, faceRect.width(), faceRect.height())); cv::Mat roiMat(roiMat1.clone()); // #todo copy data should be avoid!!!! size_t s = roiMat.total() * roiMat.elemSize(); PbFaceList_FaceListImage& pbFaceListImage(*(pbFaceList.add_images())); pbFaceListImage.set_idx(i); pbFaceListImage.set_size(s); pbFaceListImage.set_type(PbFaceList_FaceListImage_ImageType_MBFT_Y8); pbFaceListImage.set_width(roiMat.cols); pbFaceListImage.set_height(roiMat.rows); pbFaceListImage.set_top_left_x(faceRect.leftTop.X); pbFaceListImage.set_top_left_y(faceRect.leftTop.Y); pbFaceListImage.add_img(roiMat.ptr(), s); //{ // static size_t f = 0; // char fname[50]; // sprintf(fname, "/sdcard/face-%u.yuv", ++f); // FILE *pFile = fopen(fname, "wb"); // fwrite(roiMat.ptr(), 1, s, pFile); // fclose(pFile); // if (f > 20)exit(0); //} //{ // static size_t f = 0; // char fname[50]; // sprintf(fname, "/sdcard/face-%u.yuv", ++f); // FILE *pFile = fopen(fname, "wb"); // const std::string* img(*pbFaceListImage.img().data()); // fwrite(img->data(), 1, img->size(), pFile); // fclose(pFile); // if (f > 20)exit(0); //} } size_t s = pbFaceList.ByteSize(); buffMaxSize = std::min(s, buffMaxSize); pbFaceList.SerializeToArray(buffer, buffMaxSize); //{ // static size_t f = 0; // char fname[50]; // sprintf(fname, "/sdcard/facelist-%u.pb", ++f); // FILE *pFile = fopen(fname, "wb"); // fwrite(buffer, 1, buffMaxSize, pFile); // fclose(pFile); // if (f > 10)exit(0); //} return buffMaxSize > 0; } #ifdef YUV420_TO_RGB888 bool FaceCache::getFaceListImage(std::vector& imgIdxes, uint8_t* buffImg, size_t& buffImgMaxSize) { FcPmBreackerContext& ctx(*(FcPmBreackerContext*)_ctx); if (!ctx.dataAvailable) return false; if (ctx.frameRGBSize == 0) { //#todo should optimize not convert the whole image if (! ctx.convertYUV420ToRGB888()) return false; } imgIdxes.clear(); size_t offset = 0; cv::Mat rgbMat(cv::Size(ctx.width, ctx.height), CV_8UC3, ctx.frameRGB); for (st_ff_vect_t::const_iterator ffiter = ctx.faceFeatures.begin(); ffiter != ctx.faceFeatures.end(); ++ffiter) { const FaceRect& faceRect(ffiter->rect); cv::Mat roiMat1(rgbMat, cv::Rect(faceRect.leftTop.x, faceRect.leftTop.y, faceRect.width(), faceRect.height())); cv::Mat roiMat(roiMat1.clone()); // #todo copy data should be avoid!!!! NativeImgIdx imgidx; imgidx.offset = offset; imgidx.size = roiMat.total() * roiMat.elemSize(); imgidx.type = MB_Frame::MBFT_RGB888; imgidx.width = roiMat.cols; imgidx.height = roiMat.rows; const size_t newSize = imgidx.offset + imgidx.size; //isContinuous if (newSize > buffImgMaxSize) { LOG_ERROR << "FaceCache::getFaceListImage buffImgMaxSize truncated" << LOG_ENDL; return false; } uint8_t* pbuf = buffImg + imgidx.offset; memcpy(pbuf, roiMat.ptr(), imgidx.size); //uint8_t* pbuf = buffImg + imgidx.offset; //cv::Mat roiMat1(roiMat.cols, roiMat.rows, CV_8UC3, pbuf); //pbuf[0]=123; //roiMat.copyTo(roiMat1); //memcpy(buffImg + imgidx.offset, roiMat.ptr(), imgidx.size); //uint8_t* pbuf = buffImg + imgidx.offset; //for(size_t r = 0; r < roiMat.rows; r++) //{ // const uint8_t* pcol = roiMat.ptr(r); // memcpy(pbuf, pcol, roiMat.cols); // pbuf += roiMat.cols; //} //{ // static size_t f = 0; // char fname[50]; // sprintf(fname, "/sdcard/face-%u.rgb", ++f); // FILE *pFile = fopen(fname, "wb"); // fwrite(pbuf, 1, imgidx.size, pFile); // fclose(pFile); // if (f > 20)exit(0); //} imgIdxes.push_back(imgidx); offset = newSize; } buffImgMaxSize = offset; return true; } #elif YUV420_TO_RGB565 bool FaceCache::getFaceListImage(std::vector& imgIdxes, uint8_t* buffImg, size_t& buffImgMaxSize) { FcPmBreackerContext& ctx(*(FcPmBreackerContext*)_ctx); if (!ctx.dataAvailable) return false; if (ctx.frameRGBSize == 0) { //#todo should optimize not convert the whole image if (! ctx.convertYUV420ToRGB565()) return false; } imgIdxes.clear(); size_t offset = 0; cv::Mat rgbMat(cv::Size(ctx.width, ctx.height), CV_16UC1, ctx.frameRGB); for (st_ff_vect_t::const_iterator ffiter = ctx.faceFeatures.begin(); ffiter != ctx.faceFeatures.end(); ++ffiter) { if (! ffiter->test_face_in_cone(35.0f, 35.0f, 35.0f)) continue; const PLGH_Rect& faceRect(ffiter->rect); cv::Mat roiMat1(rgbMat, cv::Rect(faceRect.leftTop.X, faceRect.leftTop.Y, faceRect.width(), faceRect.height())); cv::Mat roiMat(roiMat1.clone()); // #todo copy data should be avoid!!!! NativeImgIdx imgidx; imgidx.offset = offset; imgidx.size = roiMat.total() * roiMat.elemSize(); imgidx.type = MB_Frame::MBFT_RGB565; imgidx.width = roiMat.cols; imgidx.height = roiMat.rows; const size_t newSize = imgidx.offset + imgidx.size; //isContinuous if (newSize > buffImgMaxSize) { LOG_ERROR << "FaceCache::getFaceListImage buffImgMaxSize truncated" << LOG_ENDL; return false; } /* uint8_t* pbuf = buffImg + imgidx.offset; uint8_t* psrc = ctx.frameRGB + (ctx.width * faceRect.leftTop.y + faceRect.leftTop.x) * 2; for (int row = 0; row < faceRect.height(); row++) { memcpy(pbuf, psrc, imgidx.width * 2); pbuf += imgidx.width * 2; psrc += ctx.width * 2; } */ uint8_t* pbuf = buffImg + imgidx.offset; memcpy(pbuf, roiMat.ptr(), imgidx.size); //{ // static size_t f = 0; // char fname[50]; // sprintf(fname, "/sdcard/face-%u-w%d-h%d.rgb565", ++f, imgidx.width, imgidx.height); // FILE *pFile = fopen(fname, "wb"); // fwrite(pbuf, 1, imgidx.size, pFile); // fclose(pFile); // if (f > 20)exit(0); //} imgIdxes.push_back(imgidx); offset = newSize; } buffImgMaxSize = offset; return true; } #elif YUV420_TO_ABGR8888 bool FaceCache::getFaceListImage(std::vector& imgIdxes, uint8_t* buffImg, size_t& buffImgMaxSize) { FcPmBreackerContext& ctx(*(FcPmBreackerContext*)_ctx); if (!ctx.dataAvailable) return false; if (ctx.frameRGBSize == 0) { //#todo should optimize not convert the whole image if (! ctx.convertYUV420ToABGR8888()) return false; } imgIdxes.clear(); size_t offset = 0; cv::Mat rgbMat(cv::Size(ctx.width, ctx.height), CV_8UC4, ctx.frameRGB); for (st_ff_vect_t::const_iterator ffiter = ctx.faceFeatures.begin(); ffiter != ctx.faceFeatures.end(); ++ffiter) { const FaceRect& faceRect(ffiter->rect); cv::Mat roiMat1(rgbMat, cv::Rect(faceRect.leftTop.x, faceRect.leftTop.y, faceRect.rightBottom.x - faceRect.leftTop.x, faceRect.rightBottom.y - faceRect.leftTop.y)); cv::Mat roiMat(roiMat1.clone()); // #todo copy data should be avoid!!!! NativeImgIdx imgidx; imgidx.offset = offset; imgidx.size = roiMat.total() * roiMat.elemSize(); imgidx.type = MB_Frame::MBFT_ABGR8888; imgidx.width = roiMat.cols; imgidx.height = roiMat.rows; const size_t newSize = imgidx.offset + imgidx.size; //isContinuous if (newSize > buffImgMaxSize) { LOG_ERROR << "FaceCache::getFaceListImage buffImgMaxSize truncated" << LOG_ENDL; return false; } uint8_t* pbuf = buffImg + imgidx.offset; memcpy(pbuf, roiMat.ptr(), imgidx.size); //{ // static size_t f = 0; // char fname[50]; // sprintf(fname, "/sdcard/face-%u-w%d-h%d.argb", ++f, imgidx.width, imgidx.height); // FILE *pFile = fopen(fname, "wb"); // fwrite(pbuf, 1, imgidx.size, pFile); // fclose(pFile); // if (f > 20)exit(0); //} imgIdxes.push_back(imgidx); offset = newSize; } buffImgMaxSize = offset; return true; } #endif