#include "PL_AndroidSurfaceViewRender.h" #include "MaterialBuffer.h" #include "logger.h" #include "MediaHelper.h" #include #include #include #include #include struct PL_ASVR_Internal { uint8_t buffer[1920*1080*3];//#todo new from config uint8_t buffer1[1920*1080*3];//#todo new from config size_t buffSize; size_t buff1Size; const size_t buffSizeMax; const size_t buff1SizeMax; bool payError; MB_Frame lastMbfBuffOrigin; MB_Frame lastMbfBuffRender; MB_Frame* lastMbfList[2]; PL_AndroidSurfaceViewRender_Config config; int nativeWindowFormat; int nativeWindowStride; int nativeWindowWidth; int nativeWindowHeight; PL_ASVR_Internal() : buffSize(0), buff1Size(0), buffSizeMax(sizeof(buffer)), buff1SizeMax(sizeof(buffer1)), payError(true), lastMbfBuffOrigin(), lastMbfBuffRender(), config(), nativeWindowFormat(0), nativeWindowStride(0), nativeWindowWidth(0), nativeWindowHeight(0) { lastMbfBuffOrigin.reset(); lastMbfBuffRender.reset(); lastMbfList[0] = &lastMbfBuffOrigin; lastMbfList[1] = &lastMbfBuffRender; } ~PL_ASVR_Internal() { } void reset() { buffSize = 0; buff1Size = 0; payError = true; lastMbfBuffOrigin.reset(); lastMbfBuffRender.reset(); lastMbfList[0] = &lastMbfBuffOrigin; lastMbfList[1] = &lastMbfBuffRender; nativeWindowFormat = 0; nativeWindowStride = 0; nativeWindowWidth = 0; nativeWindowHeight = 0; } }; PipeLineElem* create_PL_AndroidSurfaceViewRender() { return new PL_AndroidSurfaceViewRender; } PL_AndroidSurfaceViewRender::PL_AndroidSurfaceViewRender() : internal(new PL_ASVR_Internal) { } PL_AndroidSurfaceViewRender::~PL_AndroidSurfaceViewRender() { delete (PL_ASVR_Internal*)internal; internal= nullptr; } bool PL_AndroidSurfaceViewRender::init(void* args) { PL_ASVR_Internal* in = (PL_ASVR_Internal*)internal; PL_AndroidSurfaceViewRender_Config* config = (PL_AndroidSurfaceViewRender_Config*)args; in->config = *config; if (in->config.windowSurface == nullptr) { LOG_ERROR << "windowSurfaceRender can not be null" << LOG_ENDL; return false; } ANativeWindow* window = (ANativeWindow*)(in->config.windowSurface); if (in->config.windowSurface == nullptr) { LOG_ERROR << "ANativeWindow can not be null" << LOG_ENDL; return false; } in->nativeWindowFormat = ANativeWindow_getFormat(window); in->nativeWindowWidth = ANativeWindow_getWidth(window); in->nativeWindowHeight = ANativeWindow_getHeight(window); ANativeWindow_Buffer buffer; if (ANativeWindow_lock(window, &buffer, NULL) == 0) { in->nativeWindowStride = buffer.stride; ANativeWindow_unlockAndPost(window); } LOGP(INFO, "nwF=%d, nwS=%d, nwW=%d, nwH=%d", in->nativeWindowFormat, in->nativeWindowStride, in->nativeWindowWidth, in->nativeWindowHeight); return true; } void PL_AndroidSurfaceViewRender::finit() { PL_ASVR_Internal* in = (PL_ASVR_Internal*)internal; } bool convert_yuv420_origin_to_render(PL_ASVR_Internal* in) { if (in->config.directlyDisplay) return true; int src_width = in->lastMbfBuffOrigin.width; int src_height = in->lastMbfBuffOrigin.height; const uint8_t* src_y = (const uint8_t*)(in->lastMbfBuffOrigin.buffer); const uint8_t* src_u = (const uint8_t*)(src_y + (src_height * src_width)); const uint8_t* src_v = (const uint8_t*)(src_u + (src_height * src_width / 4)); const int dst_width = (in->config.scaleToWidth <= 0 ? in->nativeWindowStride : in->config.scaleToWidth); const int dst_height = (in->config.scaleToHeight <= 0 ? in->nativeWindowHeight : in->config.scaleToHeight); if (src_width != dst_width || src_height != dst_height) { uint8_t* dst_y = (uint8_t*)(in->buffer1); uint8_t* dst_u = (uint8_t*)(dst_y + (dst_height * dst_width)); uint8_t* dst_v = (uint8_t*)(dst_u + (dst_height * dst_width / 4)); // libyuv irregular scale not ok libyuv::I420Scale(src_y, src_width, src_u, MH_SUBSAMPLE1(src_width, 2), src_v, MH_SUBSAMPLE1(src_width, 2), src_width, src_height, dst_y, dst_width, dst_u, MH_SUBSAMPLE1(dst_width, 2), dst_v, MH_SUBSAMPLE1(dst_width, 2), dst_width, dst_height, libyuv::kFilterNone // kFilterLinear ); in->buff1Size = dst_width * dst_height * 2; src_width = dst_width; src_height = dst_height; src_y = (const uint8_t*)(in->buffer1); src_u = (const uint8_t*)(src_y + (src_height * src_width)); src_v = (const uint8_t*)(src_u + (src_height * src_width / 4)); //{ // static size_t f = 0; // char fname[50]; // sprintf(fname, "/sdcard/face-%u.yuv", ++f); // FILE *pFile = fopen(fname, "wb"); // fwrite(dst_y, sizeof(char), dst_width * dst_height * 2, pFile); // printf("write face file %s\n", fname); // fclose(pFile); // if (f > 50) exit(0); //} } else { in->buff1Size = 0; } if (in->nativeWindowFormat == WINDOW_FORMAT_RGBA_8888 || in->nativeWindowFormat == WINDOW_FORMAT_RGBX_8888) { libyuv::I420ToRGBA(src_y, src_width, src_u, MH_SUBSAMPLE1(src_width, 2), src_v, MH_SUBSAMPLE1(src_width, 2), in->buffer, 4 * dst_width, src_width, src_height ); in->buffSize = src_width * src_height * 4; in->lastMbfBuffRender.type = MB_Frame::MBFT_RGBA; } else if (in->nativeWindowFormat == WINDOW_FORMAT_RGB_565) { libyuv::I420ToRGB565(src_y, src_width, src_u, MH_SUBSAMPLE1(src_width, 2), src_v, MH_SUBSAMPLE1(src_width, 2), in->buffer, 2 * dst_width, src_width, src_height); in->buffSize = src_width * src_height * 2; //{ // static size_t f = 0; // char fname[50]; // sprintf(fname, "/sdcard/face-%u.yuv", ++f); // FILE *pFile = fopen(fname, "wb"); // fwrite(src_y, sizeof(char), src_width * src_height * 2, pFile); // printf("write face file %s\n", fname); // fclose(pFile); // if (f > 50) exit(0); //} //{ // static size_t f = 0; // char fname[50]; // sprintf(fname, "/sdcard/face-%u.rgb565", ++f); // FILE *pFile = fopen(fname, "wb"); // fwrite(in->buffer, sizeof(char), in->buffSize, pFile); // printf("write face file %s\n", fname); // fclose(pFile); // if (f > 20) exit(0); //} in->lastMbfBuffRender.type = MB_Frame::MBFT_RGB565; } //else if (in->nativeWindowFormat == 0x32315659) //{ // in->buffSize = src_width * src_height * 2; // in->lastMbfBuffRender.type = MB_Frame::MBFT_YUV420; //} else { LOG_ERROR << "nativeWindowFormat not support, nativeWindowFormat=" << in->nativeWindowFormat << LOG_ENDL; in->lastMbfBuffRender.type = MB_Frame::MBFT__FIRST; return false; } in->lastMbfBuffRender.buffer = in->buffer; in->lastMbfBuffRender.buffSize = in->buffSize; in->lastMbfBuffRender.width = dst_width;//in->lastMbfBuffOrigin.width; in->lastMbfBuffRender.height = dst_height;//in->lastMbfBuffOrigin.height; in->lastMbfBuffRender.pts = in->lastMbfBuffOrigin.pts; LOGP(DEBUG, "render bs=%d, f=%d, w=%d, h=%d", int(in->lastMbfBuffRender.buffSize), int(in->nativeWindowFormat), int(in->lastMbfBuffRender.width), int(in->lastMbfBuffRender.height)); return true; } bool convert_nv12_origin_to_render(PL_ASVR_Internal* in) { if (in->config.directlyDisplay) return true; if (false) {//#test test nv12 on yuv420 frame const int src_width = in->lastMbfBuffOrigin.width; const int src_height = in->lastMbfBuffOrigin.height; const uint8_t* src_y = (const uint8_t*)(in->lastMbfBuffOrigin.buffer); const uint8_t* src_u = (const uint8_t*)(src_y + (src_height * src_width)); const uint8_t* src_v = (const uint8_t*)(src_u + (src_height * src_width / 4)); const int dst_width = src_width; const int dst_height = src_height; uint8_t* dst_y = (uint8_t*)(in->buffer); uint8_t* dst_uv = (uint8_t*)(dst_y + (dst_width * dst_height)); libyuv::I420ToNV12(src_y, src_width, src_u, MH_SUBSAMPLE1(src_width, 2), src_v, MH_SUBSAMPLE1(src_width, 2), dst_y, dst_width, dst_uv, dst_width, src_width, src_height); in->lastMbfBuffOrigin.type = MB_Frame::MBFT_NV12; in->lastMbfBuffOrigin.width = dst_width; in->lastMbfBuffOrigin.height = dst_height; in->lastMbfBuffOrigin.buffer = in->buffer; in->buffSize = dst_width * dst_height * 1.5; //{ // static size_t f = 0; // char fname[50]; // sprintf(fname, "/sdcard/face-%u.nv12", ++f); // FILE *pFile = fopen(fname, "wb"); // fwrite(in->buffer, sizeof(char), in->buffSize, pFile); // printf("write face file %s\n", fname); // fclose(pFile); // if (f > 20) exit(0); //} } int src_width = in->lastMbfBuffOrigin.width; int src_height = in->lastMbfBuffOrigin.height; const uint8_t* src_y = (const uint8_t*)(in->lastMbfBuffOrigin.buffer); const uint8_t* src_uv = (const uint8_t*)(src_y + (src_height * src_width)); const int dst_width = (in->config.scaleToWidth <= 0 ? in->nativeWindowStride : in->config.scaleToWidth); const int dst_height = (in->config.scaleToHeight <= 0 ? in->nativeWindowHeight : in->config.scaleToHeight); if (src_width != dst_width || src_height != dst_height) { // RK3288, 1920->640: 2.8~12ms, avg=4ms uint8_t* dst_y = (uint8_t*)(in->buffer1); uint8_t* dst_uv = (uint8_t*)(dst_y + (dst_height * dst_width)); libyuv::ScalePlane(src_y, src_width, src_width, src_height, dst_y, dst_width, dst_width, dst_height, libyuv::kFilterNone); libyuv::ScalePlane_16((uint16*)src_uv, MH_SUBSAMPLE1(src_width, 2), MH_SUBSAMPLE1(src_width, 2), MH_SUBSAMPLE1(src_height, 2), (uint16*)dst_uv, MH_SUBSAMPLE1(dst_width, 2), MH_SUBSAMPLE1(dst_width, 2), MH_SUBSAMPLE1(dst_height, 2), libyuv::kFilterNone); in->buff1Size = dst_width * dst_height * 1.5; src_width = dst_width; src_height = dst_height; src_y = (const uint8_t*)(in->buffer1); src_uv = (const uint8_t*)(src_y + (src_height * src_width)); } else { in->buff1Size = 0; } //{ // static size_t f = 0; // char fname[50]; // sprintf(fname, "/sdcard/face-%u.nv12", ++f); // FILE *pFile = fopen(fname, "wb"); // fwrite(in->buffer1, sizeof(char), in->buff1Size, pFile); // printf("write face file %s\n", fname); // fclose(pFile); // if (f > 20) exit(0); //} if (in->nativeWindowFormat == WINDOW_FORMAT_RGBA_8888 || in->nativeWindowFormat == WINDOW_FORMAT_RGBX_8888) { // we have no NV12ToRGBA //#todo here has bug libyuv::NV12ToARGB(src_y, src_width, src_uv, src_width, in->buffer, 4 * dst_width, src_width, src_height ); in->buffSize = src_width * src_height * 4; in->lastMbfBuffRender.type = MB_Frame::MBFT_RGBA; } else if (in->nativeWindowFormat == WINDOW_FORMAT_RGB_565) { // RK3288, 640, 1ms libyuv::NV12ToRGB565(src_y, src_width, src_uv, src_width, in->buffer, 2 * dst_width, src_width, src_height); in->buffSize = src_width * src_height * 2; in->lastMbfBuffRender.type = MB_Frame::MBFT_RGB565; } else { LOG_ERROR << "nativeWindowFormat not support, nativeWindowFormat=" << in->nativeWindowFormat << LOG_ENDL; in->lastMbfBuffRender.type = MB_Frame::MBFT__FIRST; return false; } in->lastMbfBuffRender.buffer = in->buffer; in->lastMbfBuffRender.buffSize = in->buffSize; in->lastMbfBuffRender.width = dst_width;//in->lastMbfBuffOrigin.width; in->lastMbfBuffRender.height = dst_height;//in->lastMbfBuffOrigin.height; in->lastMbfBuffRender.pts = in->lastMbfBuffOrigin.pts; LOGP(DEBUG, "render bs=%d, f=%d, w=%d, h=%d", int(in->lastMbfBuffRender.buffSize), int(in->nativeWindowFormat), int(in->lastMbfBuffRender.width), int(in->lastMbfBuffRender.height)); return true; } bool convert_rgb565_origin_to_render(PL_ASVR_Internal *in) { if (in->config.directlyDisplay) return true; ANativeWindow* window = (ANativeWindow*)(in->config.windowSurface); int src_width = in->lastMbfBuffOrigin.width; int src_height = in->lastMbfBuffOrigin.height; const int dst_width = (in->config.scaleToWidth <= 0 ? in->nativeWindowStride : in->config.scaleToWidth); const int dst_height = (in->config.scaleToHeight <= 0 ? in->nativeWindowHeight : in->config.scaleToHeight); if (src_width != dst_width || src_height != dst_height) { uint8_t* src = (uint8_t*)in->lastMbfBuffOrigin.buffer; uint8_t* dst = (uint8_t*)in->buffer1; libyuv::ScalePlane_16((uint16*)src, src_width, src_width, src_height, (uint16*)dst, dst_width, dst_width, dst_height, libyuv::kFilterNone); in->buff1Size = dst_width * dst_height * 2; memcpy(in->buffer, in->buffer1, in->buff1Size); in->buffSize = in->buff1Size; } else { if (!in->config.directlyDisplay) { memcpy(in->buffer, in->lastMbfBuffOrigin.buffer, in->lastMbfBuffOrigin.buffSize); in->buffSize = in->lastMbfBuffOrigin.buffSize; } } return true; } bool render_surface(PL_ASVR_Internal* in) { ANativeWindow* window = (ANativeWindow*)(in->config.windowSurface); ANativeWindow_Buffer buffer; if (ANativeWindow_lock(window, &buffer, NULL) == 0) { size_t bitsSize = 0; if (buffer.format == WINDOW_FORMAT_RGBA_8888 || buffer.format == WINDOW_FORMAT_RGBX_8888) bitsSize = buffer.stride * buffer.height * 4; else if (buffer.format == WINDOW_FORMAT_RGB_565) bitsSize = buffer.stride * buffer.height * 2; else bitsSize = buffer.stride * buffer.height; if (in->config.directlyDisplay) memcpy(buffer.bits, in->lastMbfBuffOrigin.buffer, bitsSize); else { if (bitsSize > in->buffSize) { LOG_WARN << "surface buffer truncated" << LOG_ENDL; bitsSize = in->buffSize; } else if (bitsSize < in->buffSize) { LOG_WARN << "in buffer truncated" << LOG_ENDL; } memcpy(buffer.bits, in->buffer, bitsSize); } ANativeWindow_unlockAndPost(window); } else { LOG_WARN << "ANativeWindow_lock error" << LOG_ENDL; } return true; } /*static*/ bool PL_AndroidSurfaceViewRender::pay_breaker_MBFT_YUV(const PipeMaterial* pm, void* args) { PL_ASVR_Internal* in = (PL_ASVR_Internal*)args; if (pm->type != PipeMaterial::PMT_FRAME) { LOG_ERROR << "Only support PMT_FRAME" << LOG_ENDL; return false; } if (pm->buffer == nullptr) return false; MB_Frame* frame = (MB_Frame*)pm->buffer; if (frame->type != MB_Frame::MBFT_YUV420 && frame->type != MB_Frame::MBFT_NV12 && frame->type != MB_Frame::MBFT_RGB565) { LOG_ERROR << "Only support MBFT_YUV420 、MBFT_NV12 and MBFT_RGB565" << LOG_ENDL; in->payError = true; return false; } in->payError = false; in->lastMbfBuffOrigin.type = frame->type; in->lastMbfBuffOrigin.buffer = frame->buffer; in->lastMbfBuffOrigin.buffSize = frame->buffSize; in->lastMbfBuffOrigin.width = frame->width; in->lastMbfBuffOrigin.height = frame->height; in->lastMbfBuffOrigin.pts = frame->pts; // read from lastMbfBuffOrigin // write to in->buffer // write to lastMbfBuffRender bool ret = false; if (in->lastMbfBuffOrigin.type == MB_Frame::MBFT_YUV420) ret = convert_yuv420_origin_to_render(in); else if (in->lastMbfBuffOrigin.type == MB_Frame::MBFT_NV12) ret = convert_nv12_origin_to_render(in); else if (in->lastMbfBuffOrigin.type == MB_Frame::MBFT_RGB565) ret = convert_rgb565_origin_to_render(in); if (!ret) { LOG_ERROR << "convert yuv origin to render error" << LOG_ENDL; in->payError = true; return false; } // read from lastMbfBuffOrigin and in->buffer ret = render_surface(in); if (!ret) { LOG_ERROR << "render_surface error" << LOG_ENDL; in->payError = true; return false; } in->payError = false; return false; } bool PL_AndroidSurfaceViewRender::pay(const PipeMaterial& pm) { PL_ASVR_Internal* in = (PL_ASVR_Internal*)internal; in->payError = true; if (in->payError) pm.breake(PipeMaterial::PMT_FRAME_LIST, MB_Frame::MBFT_YUV420, PL_AndroidSurfaceViewRender::pay_breaker_MBFT_YUV, in); if (in->payError) pm.breake(PipeMaterial::PMT_FRAME_LIST, MB_Frame::MBFT_NV12, PL_AndroidSurfaceViewRender::pay_breaker_MBFT_YUV, in); if (in->payError) pm.breake(PipeMaterial::PMT_FRAME, MB_Frame::MBFT_YUV420, PL_AndroidSurfaceViewRender::pay_breaker_MBFT_YUV, in); if (in->payError) pm.breake(PipeMaterial::PMT_FRAME, MB_Frame::MBFT_NV12, PL_AndroidSurfaceViewRender::pay_breaker_MBFT_YUV, in); if (in->payError) pm.breake(PipeMaterial::PMT_FRAME, MB_Frame::MBFT_RGB565, PL_AndroidSurfaceViewRender::pay_breaker_MBFT_YUV, in); return !(in->payError); } bool PL_AndroidSurfaceViewRender::gain(PipeMaterial& pm) { PL_ASVR_Internal* in = (PL_ASVR_Internal*)internal; if (in->payError) return false; if (in->config.outputOriginFrame && in->config.outputRenderFrame) { pm.type = PipeMaterial::PMT_FRAME_LIST; pm.buffer = *(in->lastMbfList); pm.buffSize = sizeof(in->lastMbfList) / sizeof(MB_Frame*); // 2 } else if (in->config.outputOriginFrame) { pm.type = PipeMaterial::PMT_FRAME; pm.buffer = &(in->lastMbfBuffOrigin); pm.buffSize = 0; } else if (in->config.outputRenderFrame) { pm.type = PipeMaterial::PMT_FRAME; pm.buffer = &(in->lastMbfBuffRender); pm.buffSize = 0; } else { return false; } pm.deleter = nullptr; pm.former = this; return true; }