#include "PL_BlockGrouping.h" #include "MaterialBuffer.h" #include "logger.h" #include "MediaHelper.h" #include "GraphicHelper.h" #include // for memcpy #include #include #include #include #include struct MBFrameWrapper { MB_Frame frame; int frameRefCount; MBFrameWrapper(bool _copyData, const MB_Frame _frame) : frame(_frame), frameRefCount(_copyData ? 0 : -1) { if (_copyData) { uint8_t* newBuffer = new uint8_t[_frame.buffSize]; frame.buffer = newBuffer; memcpy(newBuffer, _frame.buffer, _frame.buffSize); } } void reference() { if (frameRefCount >= 0) frameRefCount++; else frameRefCount--; } void release() { if (frameRefCount == 0) { delete[] (uint8_t*)frame.buffer; frame.reset(); } else if (frameRefCount == -1) { frame.reset(); } if (frameRefCount < -1) { frameRefCount++; } else if (frameRefCount > 0) { frameRefCount--; } } ~MBFrameWrapper() { release(); } }; struct PLBG_Block { const uint16_t blockID; RectWrapper rectWrapper; MBFrameWrapper* mbfWrapper; uint8_t* croppedData; size_t croppedDataSize; PLBG_Block() : blockID(++_blockID), rectWrapper(), mbfWrapper(nullptr), croppedData(nullptr), croppedDataSize(0) { } void crop_data() { if (croppedData != nullptr || mbfWrapper == nullptr) return; PLGH_Rect& rect(rectWrapper.rect); if (mbfWrapper->frame.type == MB_Frame::MBFT_YUV420) { int src_width = mbfWrapper->frame.width; int src_height = mbfWrapper->frame.height; uint8_t* src_y = (uint8_t*)(mbfWrapper->frame.buffer); uint8_t* src_u = (uint8_t*)(src_y + (src_height * src_width)); uint8_t* src_v = (uint8_t*)(src_u + (src_height * src_width / 4)); cv::Mat matY(cv::Size(src_width, src_height), CV_8UC1, src_y); cv::Mat roiMatY(matY, cv::Rect(rect.leftTop.X, rect.leftTop.Y, rect.width(), rect.height())); cv::Mat cloneRoiMatY(roiMatY.clone()); cv::Mat matU(cv::Size(MH_SUBSAMPLE1(src_width, 2), MH_SUBSAMPLE1(src_height, 2)), CV_8UC1, src_u); cv::Mat roiMatU(matU, cv::Rect(MH_SUBSAMPLE1(rect.leftTop.X, 2), MH_SUBSAMPLE1(rect.leftTop.Y, 2), MH_SUBSAMPLE1(rect.width(), 2), MH_SUBSAMPLE1(rect.height(), 2))); cv::Mat cloneRoiMatU(roiMatU.clone()); cv::Mat matV(cv::Size(MH_SUBSAMPLE1(src_width, 2), MH_SUBSAMPLE1(src_height, 2)), CV_8UC1, src_v); cv::Mat roiMatV(matV, cv::Rect(MH_SUBSAMPLE1(rect.leftTop.X, 2), MH_SUBSAMPLE1(rect.leftTop.Y, 2), MH_SUBSAMPLE1(rect.width(), 2), MH_SUBSAMPLE1(rect.height(), 2))); cv::Mat cloneRoiMatV(roiMatV.clone()); const size_t yPlanarSize = rect.width() * rect.height(); croppedDataSize = yPlanarSize * 1.5; croppedData = new uint8_t[croppedDataSize]; uint8_t* dst_y = (uint8_t*)croppedData; uint8_t* dst_u = (uint8_t*)(dst_y + yPlanarSize); uint8_t* dst_v = (uint8_t*)(dst_u + yPlanarSize / 4); memcpy(dst_y, cloneRoiMatY.ptr(), yPlanarSize); memcpy(dst_u, cloneRoiMatU.ptr(), yPlanarSize / 4); memcpy(dst_v, cloneRoiMatV.ptr(), yPlanarSize / 4); //{ // static size_t f = 0; // char fname[50]; // sprintf(fname, "/sdcard/face-%u-w%u-h%u.yuv420", ++f, rect.width(), rect.height()); // FILE *pFile = fopen(fname, "wb"); // fwrite(croppedData, 1, croppedDataSize, pFile); // fclose(pFile); // if (f > 10)exit(0); //} } else if (mbfWrapper->frame.type == MB_Frame::MBFT_NV12) { int src_width = mbfWrapper->frame.width; int src_height = mbfWrapper->frame.height; uint8_t* src_y = (uint8_t*)(mbfWrapper->frame.buffer); uint8_t* src_uv = (uint8_t*)(src_y + (src_height * src_width)); cv::Mat matY(cv::Size(src_width, src_height), CV_8UC1, src_y); cv::Mat roiMatY(matY, cv::Rect(rect.leftTop.X, rect.leftTop.Y, rect.width(), rect.height())); cv::Mat cloneRoiMatY(roiMatY.clone()); cv::Mat matUV(cv::Size(MH_SUBSAMPLE1(src_width, 2), MH_SUBSAMPLE1(src_height, 2)), CV_16UC1, src_uv); cv::Mat roiMatUV(matUV, cv::Rect(MH_SUBSAMPLE1(rect.leftTop.X, 2), MH_SUBSAMPLE1(rect.leftTop.Y, 2), MH_SUBSAMPLE1(rect.width(), 2), MH_SUBSAMPLE1(rect.height(), 2))); //cv::Mat roiMatUV(matUV, cv::Rect(rect.leftTop.X, rect.leftTop.Y, rect.width(), rect.height())); cv::Mat cloneRoiMatUV(roiMatUV.clone()); const size_t yPlanarSize = rect.width() * rect.height(); croppedDataSize = yPlanarSize * 1.5; croppedData = new uint8_t[croppedDataSize];//#todo optimize static memory uint8_t* dst_y = (uint8_t*)croppedData; uint8_t* dst_uv = (uint8_t*)(dst_y + yPlanarSize); memcpy(dst_y, cloneRoiMatY.ptr(), yPlanarSize); memcpy(dst_uv, cloneRoiMatUV.ptr(), yPlanarSize * 0.5); } else if (mbfWrapper->frame.type == MB_Frame::MBFT_RGB888) { LOG_WARN << "frame type not supported" << LOG_ENDL; } else { LOG_WARN << "frame type not supported" << LOG_ENDL; } } void release_crop() { delete[] (uint8_t*)croppedData; croppedDataSize = 0; } ~PLBG_Block() { } private: static uint16_t _blockID; }; uint16_t PLBG_Block::_blockID = 0; struct PLBG_Group { const uint16_t groupID; std::vector blocks; // sequence of paid blocks int paid_rect_interval; int paid_frame_count; int _current_frame_accepted_blocks; PLBG_Group() : groupID(++_groupID), blocks(), paid_rect_interval(0), paid_frame_count(0), _current_frame_accepted_blocks(0) { } private: static uint16_t _groupID; }; uint16_t PLBG_Group::_groupID = 0; struct PL_BlockGrouping_Internal { PL_BlockGrouping_Config config; std::set mbfws; std::list groups; std::list groupsMature; bool payError; plbg_output_vec_t outputs; PL_BlockGrouping_Internal() : config(), mbfws(), groups(), groupsMature(), payError(true), outputs() { } ~PL_BlockGrouping_Internal() { reset(); } void reset() { PL_BlockGrouping_Config _config; config = _config; //for (std::set::iterator iter = buffers.begin(); iter != buffers.end(); iter++) // delete[] *iter; mbfws.clear(); //#todo delete groups.clear(); groupsMature.clear(); //#todo delete payError = true; outputs.clear(); //#todo delete croppedData } }; PipeLineElem* create_PL_BlockGrouping() { return new PL_BlockGrouping; } PL_BlockGrouping::PL_BlockGrouping() : internal(new PL_BlockGrouping_Internal) { } PL_BlockGrouping::~PL_BlockGrouping() { delete (PL_BlockGrouping_Internal*)internal; internal= nullptr; } bool PL_BlockGrouping::init(void* args) { PL_BlockGrouping_Internal* in = (PL_BlockGrouping_Internal*)internal; in->reset(); if (args != nullptr) { PL_BlockGrouping_Config* config = (PL_BlockGrouping_Config*)args; in->config = *config; } return true; } void PL_BlockGrouping::finit() { PL_BlockGrouping_Internal* in = (PL_BlockGrouping_Internal*)internal; } bool plbg_pay_breaker(const PipeMaterial* pm, void* args) { *(PipeMaterial**)args = const_cast(pm); return false; } bool test_group_accept_rect(PL_BlockGrouping_Internal* in, const PLBG_Group& group, const PLGH_Rect& rect) { const PLGH_Rect& lastRect(group.blocks.back().rectWrapper.rect); // test block center diff PLGH_Point lastBlockCenter(lastRect.center()); PLGH_Point testBlockCenter(rect.center()); if (std::abs(lastBlockCenter.X - testBlockCenter.X) > in->config.block_center_x_diff || std::abs(lastBlockCenter.Y - testBlockCenter.Y > in->config.block_center_y_diff)) return false; // test block area if (std::abs(lastRect.area() - rect.area()) > in->config.block_area_diff) return false; // test frame continuity if (group.paid_rect_interval > in->config.continuity_max) return false; // test blocks count alreadly accepted this frame if (group._current_frame_accepted_blocks >= in->config.accept_blocks_max_per_frame) return false; return true; } bool PL_BlockGrouping::pay(const PipeMaterial& pm) { PL_BlockGrouping_Internal* in = (PL_BlockGrouping_Internal*)internal; //#todo //if (in->mbfs.size() + 1 >= PLBG_MAX_CACHE_FRAMES) //{ // LOG_WARN << "cached frame exceed " << PLBG_MAX_CACHE_FRAMES << LOG_ENDL; // return false; //} PipeMaterial pmPtr; { PipeMaterial* _pmPtr = nullptr; pm.breake(PipeMaterial::PMT_PTR, MB_Frame::MBFT__FIRST, plbg_pay_breaker, &_pmPtr); if (_pmPtr == nullptr) { LOG_WARN << "PMT_PM_LIST need PMT_PTR" << LOG_ENDL; return false; } pmPtr = *_pmPtr; } MB_Frame mbfFrame; { PipeMaterial* _pmFrame = nullptr; if (_pmFrame == nullptr) pm.breake(PipeMaterial::PMT_FRAME, MB_Frame::MBFT_YUV420, plbg_pay_breaker, &_pmFrame); if (_pmFrame == nullptr) pm.breake(PipeMaterial::PMT_FRAME, MB_Frame::MBFT_NV12, plbg_pay_breaker, &_pmFrame); if (_pmFrame == nullptr) { LOG_WARN << "PMT_PM_LIST need PMT_FRAME of (MBFT_YUV420 or MBFT_NV12)" << LOG_ENDL; return false; } mbfFrame = *(MB_Frame*)_pmFrame->buffer; } // cache frame MBFrameWrapper* mbfWrapper = new MBFrameWrapper(in->config.copyData, mbfFrame); // get blocks std::list rectws; in->config.get_rect_func(pmPtr, mbfFrame, rectws); // classify blocks to exists group for (std::list::iterator iterGrp = in->groups.begin(); iterGrp != in->groups.end(); ++iterGrp) { PLBG_Group& currGroup(*iterGrp); currGroup.paid_rect_interval++; currGroup.paid_frame_count++; for (std::list::iterator iterRectw = rectws.begin(); iterRectw != rectws.end(); ) { const RectWrapper& currRectw(*iterRectw); if (!test_group_accept_rect(in, currGroup, currRectw.rect)) { ++iterRectw; continue; } // accept this block into group PLBG_Block currBlock; currBlock.rectWrapper = currRectw; currBlock.mbfWrapper = mbfWrapper; mbfWrapper->reference(); currBlock.crop_data(); currGroup.blocks.push_back(currBlock); currGroup.paid_rect_interval = 0; currGroup._current_frame_accepted_blocks++; iterRectw = rectws.erase(iterRectw); } } //#todo // we test remain groups asume remain rectws //if (in->groups.size() > PLBG_MAX_GROUPS - rectws.size()) //{ // if (mbfWrapper->frameRefCount == 0) // { // in->mbfs.erase(mbfWrapper); // delete mbfWrapper; // mbfWrapper = nullptr; // } // // LOG_WARN << "Exceed max groups " << PLBG_MAX_GROUPS << LOG_ENDL; // return false; //} // grouping left blocks std::list::iterator iterNewGrp = in->groups.end(); if (!rectws.empty()) { // add the first rect (not classified) into new group PLBG_Block currBlock; currBlock.rectWrapper = *rectws.begin(); currBlock.mbfWrapper = mbfWrapper; mbfWrapper->reference(); currBlock.crop_data(); rectws.erase(rectws.begin()); PLBG_Group currGroup; currGroup.blocks.push_back(currBlock); currGroup.paid_rect_interval = 0; currGroup._current_frame_accepted_blocks++; iterNewGrp = in->groups.insert(in->groups.end(), currGroup); } for (std::list::iterator iterRectw = rectws.begin(); iterRectw != rectws.end(); ++iterRectw) { const RectWrapper& currRectw(*iterRectw); std::list::iterator iterGrp = iterNewGrp; for ( ; iterGrp != in->groups.end(); ++iterGrp) { if (test_group_accept_rect(in, *iterGrp, iterRectw->rect)) break; } // add rect PLBG_Block currBlock; currBlock.rectWrapper = *iterRectw; currBlock.mbfWrapper = mbfWrapper; mbfWrapper->reference(); currBlock.crop_data(); if (iterGrp == in->groups.end()) { PLBG_Group currGroup; iterGrp = in->groups.insert(in->groups.end(), currGroup); } iterGrp->blocks.push_back(currBlock); iterGrp->paid_rect_interval = 0; iterGrp->_current_frame_accepted_blocks++; } // reset for (std::list::iterator iterGrp = in->groups.begin(); iterGrp != in->groups.end(); ++iterGrp) iterGrp->_current_frame_accepted_blocks = 0; if (mbfWrapper->frameRefCount != -1 && mbfWrapper->frameRefCount != 0) in->mbfws.insert(mbfWrapper); else delete mbfWrapper; return true; } float scoring_canny_focus(const PLBG_Block& block) { const int w = block.rectWrapper.rect.width(); const int h = block.rectWrapper.rect.height(); if (block.croppedData == nullptr || block.croppedDataSize < w * h) return 0.0f; cv::Mat yMat(cv::Size(w, h), CV_8UC1, block.croppedData); cv::Mat edges; cv::GaussianBlur(yMat, edges, cv::Size(5, 5), 1.5, 1.5); cv::Canny(edges, edges, 50, 100, 3); //cv::Sobel(yMat, edges, CV_8UC1, 1, 0, 3); //memcpy(block.croppedData, edges.data, w * h); size_t sum = 0; std::for_each(edges.begin(), edges.end(), [&](uint8_t v) { sum += (v != 0); }); float focusRate = (float)sum / (w * h); return focusRate; } float scoring_histogram_focus(const PLBG_Block& block) { return 1.0f; } bool PL_BlockGrouping::gain(PipeMaterial& pm) { PL_BlockGrouping_Internal* in = (PL_BlockGrouping_Internal*)internal; // clean last mature groups and outputs for (std::list::iterator iterGrp = in->groupsMature.begin(); iterGrp != in->groupsMature.end(); ++iterGrp) { for (std::vector::iterator iterBlk = iterGrp->blocks.begin(); iterBlk != iterGrp->blocks.end(); ++iterBlk) { iterBlk->mbfWrapper->release(); iterBlk->release_crop(); if (iterBlk->rectWrapper.user_data_deleter != nullptr) iterBlk->rectWrapper.user_data_deleter(iterBlk->rectWrapper); } } in->groupsMature.clear(); for (std::set::iterator iter = in->mbfws.begin(); iter != in->mbfws.end(); ) { MBFrameWrapper* mbfw = *iter; if (mbfw->frameRefCount == 0 || mbfw->frameRefCount == -1) { delete mbfw; iter = in->mbfws.erase(iter); } else ++iter; } in->outputs.clear(); // find "mature" groups for (std::list::iterator iterGrp = in->groups.begin(); iterGrp != in->groups.end(); ) { bool grpMature = false; PLBG_Group& currGroup(*iterGrp); if (currGroup.blocks.size() >= in->config.group_size_max) grpMature = true; if (currGroup.paid_frame_count >= in->config.frame_count_max) grpMature = true; if (grpMature && (currGroup.blocks.size() < in->config.group_size_min)) { //drop group for (std::vector::iterator iterBlk = currGroup.blocks.begin(); iterBlk != currGroup.blocks.end(); ++iterBlk) { iterBlk->mbfWrapper->release(); iterBlk->release_crop(); } iterGrp = in->groups.erase(iterGrp); continue; } if (grpMature) { // generate output group in->groupsMature.push_back(currGroup); iterGrp = in->groups.erase(iterGrp); continue; } ++iterGrp; } // scoring blocks const bool calc_canny = !MH_F_ZEQ(in->config.canny_focus); const bool calc_histogram = !MH_F_ZEQ(in->config.histogram_uniformy); const bool calc_user_score_1 = !MH_F_ZEQ(in->config.user_score_1_rate); const float user_score_2_rate = 1.0f - in->config.canny_focus - in->config.histogram_uniformy - in->config.user_score_1_rate; const bool calc_user_score_2 = (!MH_F_ZEQ(user_score_2_rate) && in->config.user_score_2_func != nullptr); for (std::list::iterator iterGrp = in->groupsMature.begin(); iterGrp != in->groupsMature.end(); ++iterGrp) { std::vector::iterator iterBestBlk = iterGrp->blocks.end(); float bestScore = 0.0f; for (std::vector::iterator iterBlk = iterGrp->blocks.begin(); iterBlk != iterGrp->blocks.end(); ++iterBlk) { float totalScore = 0.0f; if (calc_canny) { float cannyScore = scoring_canny_focus(*iterBlk) * 10.0f; totalScore += in->config.canny_focus * cannyScore; } if (calc_histogram) { float histScore = scoring_histogram_focus(*iterBlk); totalScore += in->config.histogram_uniformy * histScore; } if (calc_user_score_1) { totalScore += in->config.user_score_1_rate * iterBlk->rectWrapper.user_score_1; } if (calc_user_score_2) { float userScore2 = in->config.user_score_2_func(&(iterBlk->mbfWrapper->frame), iterBlk->rectWrapper.rect, iterBlk->croppedData); totalScore += user_score_2_rate * userScore2; } if (MH_F_GT(totalScore, bestScore)) { iterBestBlk = iterBlk; bestScore = totalScore; } } if (iterBestBlk == iterGrp->blocks.end() || MH_F_ZEQ(bestScore)) continue; PLBG_Output output; output.groupID = iterGrp->groupID; output.blockID = iterBestBlk->blockID; output.rectInOriginFrame = iterBestBlk->rectWrapper; output.originframe = &(iterBestBlk->mbfWrapper->frame); output.score = bestScore; output.croppedData = iterBestBlk->croppedData; output.croppedDataSize = iterBestBlk->croppedDataSize; in->outputs.push_back(output); } pm.type = PipeMaterial::PMT_PTR; pm.buffer = &(in->outputs); pm.buffSize = 0; pm.former = this; pm.deleter = nullptr; pm.args = nullptr; return true; }