#include "PL_AndroidSurfaceViewRender.h"
|
#include "MaterialBuffer.h"
|
#include "logger.h"
|
#include "MediaHelper.h"
|
|
#include <media/NdkMediaFormat.h>
|
|
#include <android/bitmap.h>
|
#include <android/native_window.h>
|
|
#include <stdlib.h>
|
|
#include <libyuv.h>
|
|
struct PL_ASVR_Internal
|
{
|
uint8_t buffer[1920*1080*3];//#todo new from config
|
uint8_t buffer1[1920*1080*3];//#todo new from config
|
size_t buffSize;
|
size_t buff1Size;
|
const size_t buffSizeMax;
|
const size_t buff1SizeMax;
|
|
bool payError;
|
|
MB_Frame lastMbfBuffOrigin;
|
MB_Frame lastMbfBuffRender;
|
MB_Frame* lastMbfList[2];
|
|
PL_AndroidSurfaceViewRender_Config config;
|
|
int nativeWindowFormat;
|
int nativeWindowStride;
|
int nativeWindowWidth;
|
int nativeWindowHeight;
|
|
PL_ASVR_Internal() :
|
buffSize(0), buff1Size(0), buffSizeMax(sizeof(buffer)), buff1SizeMax(sizeof(buffer1)),
|
payError(true),
|
lastMbfBuffOrigin(), lastMbfBuffRender(),
|
config(),
|
nativeWindowFormat(0), nativeWindowStride(0), nativeWindowWidth(0), nativeWindowHeight(0)
|
{
|
lastMbfBuffOrigin.reset();
|
lastMbfBuffRender.reset();
|
|
lastMbfList[0] = &lastMbfBuffOrigin;
|
lastMbfList[1] = &lastMbfBuffRender;
|
}
|
|
~PL_ASVR_Internal()
|
{
|
}
|
|
void reset()
|
{
|
buffSize = 0;
|
buff1Size = 0;
|
|
payError = true;
|
|
lastMbfBuffOrigin.reset();
|
lastMbfBuffRender.reset();
|
|
lastMbfList[0] = &lastMbfBuffOrigin;
|
lastMbfList[1] = &lastMbfBuffRender;
|
|
nativeWindowFormat = 0;
|
nativeWindowStride = 0;
|
nativeWindowWidth = 0;
|
nativeWindowHeight = 0;
|
}
|
};
|
|
PipeLineElem* create_PL_AndroidSurfaceViewRender()
|
{
|
return new PL_AndroidSurfaceViewRender;
|
}
|
|
PL_AndroidSurfaceViewRender::PL_AndroidSurfaceViewRender() : internal(new PL_ASVR_Internal)
|
{
|
}
|
|
PL_AndroidSurfaceViewRender::~PL_AndroidSurfaceViewRender()
|
{
|
delete (PL_ASVR_Internal*)internal;
|
internal= nullptr;
|
}
|
|
bool PL_AndroidSurfaceViewRender::init(void* args)
|
{
|
PL_ASVR_Internal* in = (PL_ASVR_Internal*)internal;
|
|
PL_AndroidSurfaceViewRender_Config* config = (PL_AndroidSurfaceViewRender_Config*)args;
|
in->config = *config;
|
|
if (in->config.windowSurface == nullptr)
|
{
|
LOG_ERROR << "windowSurfaceRender can not be null" << LOG_ENDL;
|
return false;
|
}
|
|
ANativeWindow* window = (ANativeWindow*)(in->config.windowSurface);
|
if (in->config.windowSurface == nullptr)
|
{
|
LOG_ERROR << "ANativeWindow can not be null" << LOG_ENDL;
|
return false;
|
}
|
|
in->nativeWindowFormat = ANativeWindow_getFormat(window);
|
in->nativeWindowWidth = ANativeWindow_getWidth(window);
|
in->nativeWindowHeight = ANativeWindow_getHeight(window);
|
|
ANativeWindow_Buffer buffer;
|
if (ANativeWindow_lock(window, &buffer, NULL) == 0)
|
{
|
in->nativeWindowStride = buffer.stride;
|
ANativeWindow_unlockAndPost(window);
|
}
|
|
LOGP(INFO, "nwF=%d, nwS=%d, nwW=%d, nwH=%d", in->nativeWindowFormat, in->nativeWindowStride, in->nativeWindowWidth, in->nativeWindowHeight);
|
|
return true;
|
}
|
|
void PL_AndroidSurfaceViewRender::finit()
|
{
|
PL_ASVR_Internal* in = (PL_ASVR_Internal*)internal;
|
}
|
|
bool convert_yuv420_origin_to_render(PL_ASVR_Internal* in)
|
{
|
if (in->config.directlyDisplay)
|
return true;
|
|
int src_width = in->lastMbfBuffOrigin.width;
|
int src_height = in->lastMbfBuffOrigin.height;
|
const uint8_t* src_y = (const uint8_t*)(in->lastMbfBuffOrigin.buffer);
|
const uint8_t* src_u = (const uint8_t*)(src_y + (src_height * src_width));
|
const uint8_t* src_v = (const uint8_t*)(src_u + (src_height * src_width / 4));
|
|
const int dst_width = (in->config.scaleToWidth <= 0 ? in->nativeWindowStride : in->config.scaleToWidth);
|
const int dst_height = (in->config.scaleToHeight <= 0 ? in->nativeWindowHeight : in->config.scaleToHeight);
|
|
if (src_width != dst_width || src_height != dst_height)
|
{
|
uint8_t* dst_y = (uint8_t*)(in->buffer1);
|
uint8_t* dst_u = (uint8_t*)(dst_y + (dst_height * dst_width));
|
uint8_t* dst_v = (uint8_t*)(dst_u + (dst_height * dst_width / 4));
|
|
// libyuv irregular scale not ok
|
libyuv::I420Scale(src_y, src_width,
|
src_u, MH_SUBSAMPLE1(src_width, 2),
|
src_v, MH_SUBSAMPLE1(src_width, 2),
|
src_width, src_height,
|
dst_y, dst_width,
|
dst_u, MH_SUBSAMPLE1(dst_width, 2),
|
dst_v, MH_SUBSAMPLE1(dst_width, 2),
|
dst_width, dst_height,
|
libyuv::kFilterNone // kFilterLinear
|
);
|
|
in->buff1Size = dst_width * dst_height * 2;
|
|
src_width = dst_width;
|
src_height = dst_height;
|
src_y = (const uint8_t*)(in->buffer1);
|
src_u = (const uint8_t*)(src_y + (src_height * src_width));
|
src_v = (const uint8_t*)(src_u + (src_height * src_width / 4));
|
|
//{
|
// static size_t f = 0;
|
// char fname[50];
|
// sprintf(fname, "/sdcard/face-%u.yuv", ++f);
|
// FILE *pFile = fopen(fname, "wb");
|
// fwrite(dst_y, sizeof(char), dst_width * dst_height * 2, pFile);
|
// printf("write face file %s\n", fname);
|
// fclose(pFile);
|
// if (f > 50) exit(0);
|
//}
|
}
|
else
|
{
|
in->buff1Size = 0;
|
}
|
|
if (in->nativeWindowFormat == WINDOW_FORMAT_RGBA_8888 || in->nativeWindowFormat == WINDOW_FORMAT_RGBX_8888)
|
{
|
libyuv::I420ToRGBA(src_y, src_width,
|
src_u, MH_SUBSAMPLE1(src_width, 2),
|
src_v, MH_SUBSAMPLE1(src_width, 2),
|
in->buffer, 4 * dst_width,
|
src_width, src_height
|
);
|
in->buffSize = src_width * src_height * 4;
|
|
in->lastMbfBuffRender.type = MB_Frame::MBFT_RGBA;
|
}
|
else if (in->nativeWindowFormat == WINDOW_FORMAT_RGB_565)
|
{
|
libyuv::I420ToRGB565(src_y, src_width,
|
src_u, MH_SUBSAMPLE1(src_width, 2),
|
src_v, MH_SUBSAMPLE1(src_width, 2),
|
in->buffer, 2 * dst_width,
|
src_width, src_height);
|
in->buffSize = src_width * src_height * 2;
|
|
//{
|
// static size_t f = 0;
|
// char fname[50];
|
// sprintf(fname, "/sdcard/face-%u.yuv", ++f);
|
// FILE *pFile = fopen(fname, "wb");
|
// fwrite(src_y, sizeof(char), src_width * src_height * 2, pFile);
|
// printf("write face file %s\n", fname);
|
// fclose(pFile);
|
// if (f > 50) exit(0);
|
//}
|
|
//{
|
// static size_t f = 0;
|
// char fname[50];
|
// sprintf(fname, "/sdcard/face-%u.rgb565", ++f);
|
// FILE *pFile = fopen(fname, "wb");
|
// fwrite(in->buffer, sizeof(char), in->buffSize, pFile);
|
// printf("write face file %s\n", fname);
|
// fclose(pFile);
|
// if (f > 20) exit(0);
|
//}
|
|
in->lastMbfBuffRender.type = MB_Frame::MBFT_RGB565;
|
}
|
//else if (in->nativeWindowFormat == 0x32315659)
|
//{
|
// in->buffSize = src_width * src_height * 2;
|
// in->lastMbfBuffRender.type = MB_Frame::MBFT_YUV420;
|
//}
|
else
|
{
|
LOG_ERROR << "nativeWindowFormat not support, nativeWindowFormat=" << in->nativeWindowFormat << LOG_ENDL;
|
in->lastMbfBuffRender.type = MB_Frame::MBFT__FIRST;
|
return false;
|
}
|
|
in->lastMbfBuffRender.buffer = in->buffer;
|
in->lastMbfBuffRender.buffSize = in->buffSize;
|
in->lastMbfBuffRender.width = dst_width;//in->lastMbfBuffOrigin.width;
|
in->lastMbfBuffRender.height = dst_height;//in->lastMbfBuffOrigin.height;
|
in->lastMbfBuffRender.pts = in->lastMbfBuffOrigin.pts;
|
|
LOGP(DEBUG, "render bs=%d, f=%d, w=%d, h=%d", int(in->lastMbfBuffRender.buffSize), int(in->nativeWindowFormat), int(in->lastMbfBuffRender.width), int(in->lastMbfBuffRender.height));
|
|
return true;
|
}
|
|
bool convert_nv12_origin_to_render(PL_ASVR_Internal* in)
|
{
|
if (in->config.directlyDisplay)
|
return true;
|
|
if (false)
|
{//#test test nv12 on yuv420 frame
|
const int src_width = in->lastMbfBuffOrigin.width;
|
const int src_height = in->lastMbfBuffOrigin.height;
|
const uint8_t* src_y = (const uint8_t*)(in->lastMbfBuffOrigin.buffer);
|
const uint8_t* src_u = (const uint8_t*)(src_y + (src_height * src_width));
|
const uint8_t* src_v = (const uint8_t*)(src_u + (src_height * src_width / 4));
|
|
const int dst_width = src_width;
|
const int dst_height = src_height;
|
uint8_t* dst_y = (uint8_t*)(in->buffer);
|
uint8_t* dst_uv = (uint8_t*)(dst_y + (dst_width * dst_height));
|
|
libyuv::I420ToNV12(src_y, src_width,
|
src_u, MH_SUBSAMPLE1(src_width, 2),
|
src_v, MH_SUBSAMPLE1(src_width, 2),
|
dst_y, dst_width,
|
dst_uv, dst_width,
|
src_width, src_height);
|
|
in->lastMbfBuffOrigin.type = MB_Frame::MBFT_NV12;
|
in->lastMbfBuffOrigin.width = dst_width;
|
in->lastMbfBuffOrigin.height = dst_height;
|
in->lastMbfBuffOrigin.buffer = in->buffer;
|
in->buffSize = dst_width * dst_height * 1.5;
|
|
//{
|
// static size_t f = 0;
|
// char fname[50];
|
// sprintf(fname, "/sdcard/face-%u.nv12", ++f);
|
// FILE *pFile = fopen(fname, "wb");
|
// fwrite(in->buffer, sizeof(char), in->buffSize, pFile);
|
// printf("write face file %s\n", fname);
|
// fclose(pFile);
|
// if (f > 20) exit(0);
|
//}
|
}
|
|
int src_width = in->lastMbfBuffOrigin.width;
|
int src_height = in->lastMbfBuffOrigin.height;
|
const uint8_t* src_y = (const uint8_t*)(in->lastMbfBuffOrigin.buffer);
|
const uint8_t* src_uv = (const uint8_t*)(src_y + (src_height * src_width));
|
|
const int dst_width = (in->config.scaleToWidth <= 0 ? in->nativeWindowStride : in->config.scaleToWidth);
|
const int dst_height = (in->config.scaleToHeight <= 0 ? in->nativeWindowHeight : in->config.scaleToHeight);
|
|
if (src_width != dst_width || src_height != dst_height)
|
{
|
// RK3288, 1920->640: 2.8~12ms, avg=4ms
|
|
uint8_t* dst_y = (uint8_t*)(in->buffer1);
|
uint8_t* dst_uv = (uint8_t*)(dst_y + (dst_height * dst_width));
|
|
libyuv::ScalePlane(src_y, src_width,
|
src_width, src_height,
|
dst_y, dst_width,
|
dst_width, dst_height,
|
libyuv::kFilterNone);
|
|
libyuv::ScalePlane_16((uint16*)src_uv, MH_SUBSAMPLE1(src_width, 2),
|
MH_SUBSAMPLE1(src_width, 2), MH_SUBSAMPLE1(src_height, 2),
|
(uint16*)dst_uv, MH_SUBSAMPLE1(dst_width, 2),
|
MH_SUBSAMPLE1(dst_width, 2), MH_SUBSAMPLE1(dst_height, 2),
|
libyuv::kFilterNone);
|
|
in->buff1Size = dst_width * dst_height * 1.5;
|
|
src_width = dst_width;
|
src_height = dst_height;
|
src_y = (const uint8_t*)(in->buffer1);
|
src_uv = (const uint8_t*)(src_y + (src_height * src_width));
|
}
|
else
|
{
|
in->buff1Size = 0;
|
}
|
|
//{
|
// static size_t f = 0;
|
// char fname[50];
|
// sprintf(fname, "/sdcard/face-%u.nv12", ++f);
|
// FILE *pFile = fopen(fname, "wb");
|
// fwrite(in->buffer1, sizeof(char), in->buff1Size, pFile);
|
// printf("write face file %s\n", fname);
|
// fclose(pFile);
|
// if (f > 20) exit(0);
|
//}
|
|
if (in->nativeWindowFormat == WINDOW_FORMAT_RGBA_8888 || in->nativeWindowFormat == WINDOW_FORMAT_RGBX_8888)
|
{
|
// we have no NV12ToRGBA
|
//#todo here has bug
|
libyuv::NV12ToARGB(src_y, src_width,
|
src_uv, src_width,
|
in->buffer, 4 * dst_width,
|
src_width, src_height
|
);
|
in->buffSize = src_width * src_height * 4;
|
|
in->lastMbfBuffRender.type = MB_Frame::MBFT_RGBA;
|
}
|
else if (in->nativeWindowFormat == WINDOW_FORMAT_RGB_565)
|
{
|
// RK3288, 640, 1ms
|
|
libyuv::NV12ToRGB565(src_y, src_width,
|
src_uv, src_width,
|
in->buffer, 2 * dst_width,
|
src_width, src_height);
|
in->buffSize = src_width * src_height * 2;
|
|
in->lastMbfBuffRender.type = MB_Frame::MBFT_RGB565;
|
}
|
else
|
{
|
LOG_ERROR << "nativeWindowFormat not support, nativeWindowFormat=" << in->nativeWindowFormat << LOG_ENDL;
|
in->lastMbfBuffRender.type = MB_Frame::MBFT__FIRST;
|
return false;
|
}
|
|
in->lastMbfBuffRender.buffer = in->buffer;
|
in->lastMbfBuffRender.buffSize = in->buffSize;
|
in->lastMbfBuffRender.width = dst_width;//in->lastMbfBuffOrigin.width;
|
in->lastMbfBuffRender.height = dst_height;//in->lastMbfBuffOrigin.height;
|
in->lastMbfBuffRender.pts = in->lastMbfBuffOrigin.pts;
|
|
LOGP(DEBUG, "render bs=%d, f=%d, w=%d, h=%d", int(in->lastMbfBuffRender.buffSize), int(in->nativeWindowFormat), int(in->lastMbfBuffRender.width), int(in->lastMbfBuffRender.height));
|
|
return true;
|
}
|
|
bool convert_rgb565_origin_to_render(PL_ASVR_Internal *in)
|
{
|
if (in->config.directlyDisplay)
|
return true;
|
|
ANativeWindow* window = (ANativeWindow*)(in->config.windowSurface);
|
int src_width = in->lastMbfBuffOrigin.width;
|
int src_height = in->lastMbfBuffOrigin.height;
|
|
const int dst_width = (in->config.scaleToWidth <= 0 ? in->nativeWindowStride : in->config.scaleToWidth);
|
const int dst_height = (in->config.scaleToHeight <= 0 ? in->nativeWindowHeight : in->config.scaleToHeight);
|
|
if (src_width != dst_width || src_height != dst_height)
|
{
|
uint8_t* src = (uint8_t*)in->lastMbfBuffOrigin.buffer;
|
uint8_t* dst = (uint8_t*)in->buffer1;
|
libyuv::ScalePlane_16((uint16*)src, src_width,
|
src_width, src_height,
|
(uint16*)dst, dst_width,
|
dst_width, dst_height,
|
libyuv::kFilterNone);
|
in->buff1Size = dst_width * dst_height * 2;
|
memcpy(in->buffer, in->buffer1, in->buff1Size);
|
in->buffSize = in->buff1Size;
|
}
|
else
|
{
|
if (!in->config.directlyDisplay)
|
{
|
memcpy(in->buffer, in->lastMbfBuffOrigin.buffer, in->lastMbfBuffOrigin.buffSize);
|
in->buffSize = in->lastMbfBuffOrigin.buffSize;
|
}
|
}
|
return true;
|
}
|
|
bool render_surface(PL_ASVR_Internal* in)
|
{
|
ANativeWindow* window = (ANativeWindow*)(in->config.windowSurface);
|
ANativeWindow_Buffer buffer;
|
if (ANativeWindow_lock(window, &buffer, NULL) == 0)
|
{
|
size_t bitsSize = 0;
|
if (buffer.format == WINDOW_FORMAT_RGBA_8888 || buffer.format == WINDOW_FORMAT_RGBX_8888)
|
bitsSize = buffer.stride * buffer.height * 4;
|
else if (buffer.format == WINDOW_FORMAT_RGB_565)
|
bitsSize = buffer.stride * buffer.height * 2;
|
else
|
bitsSize = buffer.stride * buffer.height;
|
|
if (in->config.directlyDisplay)
|
memcpy(buffer.bits, in->lastMbfBuffOrigin.buffer, bitsSize);
|
else
|
{
|
if (bitsSize > in->buffSize)
|
{
|
LOG_WARN << "surface buffer truncated" << LOG_ENDL;
|
bitsSize = in->buffSize;
|
}
|
else if (bitsSize < in->buffSize)
|
{
|
LOG_WARN << "in buffer truncated" << LOG_ENDL;
|
}
|
memcpy(buffer.bits, in->buffer, bitsSize);
|
}
|
|
ANativeWindow_unlockAndPost(window);
|
}
|
else
|
{
|
LOG_WARN << "ANativeWindow_lock error" << LOG_ENDL;
|
}
|
|
|
return true;
|
}
|
|
/*static*/ bool PL_AndroidSurfaceViewRender::pay_breaker_MBFT_YUV(const PipeMaterial* pm, void* args)
|
{
|
PL_ASVR_Internal* in = (PL_ASVR_Internal*)args;
|
|
if (pm->type != PipeMaterial::PMT_FRAME)
|
{
|
LOG_ERROR << "Only support PMT_FRAME" << LOG_ENDL;
|
return false;
|
}
|
|
if (pm->buffer == nullptr)
|
return false;
|
|
MB_Frame* frame = (MB_Frame*)pm->buffer;
|
if (frame->type != MB_Frame::MBFT_YUV420 && frame->type != MB_Frame::MBFT_NV12 && frame->type != MB_Frame::MBFT_RGB565)
|
{
|
LOG_ERROR << "Only support MBFT_YUV420 、MBFT_NV12 and MBFT_RGB565" << LOG_ENDL;
|
in->payError = true;
|
return false;
|
}
|
|
in->payError = false;
|
|
in->lastMbfBuffOrigin.type = frame->type;
|
in->lastMbfBuffOrigin.buffer = frame->buffer;
|
in->lastMbfBuffOrigin.buffSize = frame->buffSize;
|
in->lastMbfBuffOrigin.width = frame->width;
|
in->lastMbfBuffOrigin.height = frame->height;
|
in->lastMbfBuffOrigin.pts = frame->pts;
|
|
// read from lastMbfBuffOrigin
|
// write to in->buffer
|
// write to lastMbfBuffRender
|
bool ret = false;
|
if (in->lastMbfBuffOrigin.type == MB_Frame::MBFT_YUV420)
|
ret = convert_yuv420_origin_to_render(in);
|
else if (in->lastMbfBuffOrigin.type == MB_Frame::MBFT_NV12)
|
ret = convert_nv12_origin_to_render(in);
|
else if (in->lastMbfBuffOrigin.type == MB_Frame::MBFT_RGB565)
|
ret = convert_rgb565_origin_to_render(in);
|
if (!ret)
|
{
|
LOG_ERROR << "convert yuv origin to render error" << LOG_ENDL;
|
in->payError = true;
|
return false;
|
}
|
|
// read from lastMbfBuffOrigin and in->buffer
|
ret = render_surface(in);
|
if (!ret)
|
{
|
LOG_ERROR << "render_surface error" << LOG_ENDL;
|
in->payError = true;
|
return false;
|
}
|
|
in->payError = false;
|
return false;
|
}
|
|
bool PL_AndroidSurfaceViewRender::pay(const PipeMaterial& pm)
|
{
|
PL_ASVR_Internal* in = (PL_ASVR_Internal*)internal;
|
in->payError = true;
|
if (in->payError)
|
pm.breake(PipeMaterial::PMT_FRAME_LIST, MB_Frame::MBFT_YUV420, PL_AndroidSurfaceViewRender::pay_breaker_MBFT_YUV, in);
|
if (in->payError)
|
pm.breake(PipeMaterial::PMT_FRAME_LIST, MB_Frame::MBFT_NV12, PL_AndroidSurfaceViewRender::pay_breaker_MBFT_YUV, in);
|
if (in->payError)
|
pm.breake(PipeMaterial::PMT_FRAME, MB_Frame::MBFT_YUV420, PL_AndroidSurfaceViewRender::pay_breaker_MBFT_YUV, in);
|
if (in->payError)
|
pm.breake(PipeMaterial::PMT_FRAME, MB_Frame::MBFT_NV12, PL_AndroidSurfaceViewRender::pay_breaker_MBFT_YUV, in);
|
if (in->payError)
|
pm.breake(PipeMaterial::PMT_FRAME, MB_Frame::MBFT_RGB565, PL_AndroidSurfaceViewRender::pay_breaker_MBFT_YUV, in);
|
return !(in->payError);
|
}
|
|
bool PL_AndroidSurfaceViewRender::gain(PipeMaterial& pm)
|
{
|
PL_ASVR_Internal* in = (PL_ASVR_Internal*)internal;
|
|
if (in->payError)
|
return false;
|
|
if (in->config.outputOriginFrame && in->config.outputRenderFrame)
|
{
|
pm.type = PipeMaterial::PMT_FRAME_LIST;
|
pm.buffer = *(in->lastMbfList);
|
pm.buffSize = sizeof(in->lastMbfList) / sizeof(MB_Frame*); // 2
|
}
|
else if (in->config.outputOriginFrame)
|
{
|
pm.type = PipeMaterial::PMT_FRAME;
|
pm.buffer = &(in->lastMbfBuffOrigin);
|
pm.buffSize = 0;
|
}
|
else if (in->config.outputRenderFrame)
|
{
|
pm.type = PipeMaterial::PMT_FRAME;
|
pm.buffer = &(in->lastMbfBuffRender);
|
pm.buffSize = 0;
|
}
|
else
|
{
|
return false;
|
}
|
|
pm.deleter = nullptr;
|
pm.former = this;
|
|
return true;
|
}
|
|
