#include "PL_AndroidMediaCodecDecoder.h"
|
#include "MaterialBuffer.h"
|
#include "logger.h"
|
#include "MediaHelper.h"
|
|
#include <media/NdkMediaCodec.h>
|
#include <media/NdkMediaFormat.h>
|
|
#include <android/bitmap.h>
|
|
#include <stdlib.h>
|
#include <liveMedia/liveMedia.hh>
|
|
//from android source
|
//https://android.googlesource.com/platform/system/core/+/android-5.1.1_r38/
|
#include <system/window.h>
|
#include <ui/GraphicBuffer.h>
|
//#include <gui/Surface.h>
|
|
#define EGL_EGLEXT_PROTOTYPES 1
|
#define GL_GLEXT_PROTOTYPES 1
|
#define GL3_PROTOTYPES 1
|
#include <EGL/egl.h>
|
#include <EGL/eglext.h>
|
#include <GLES/gl.h>
|
#include <GLES/glext.h>
|
#include <GLES2/gl2.h>
|
#include <GLES3/gl3.h>
|
#include <GLES3/gl3ext.h>
|
#include <GLES3/gl31.h>
|
|
struct PL_AMCD_Internal
|
{
|
uint8_t buffer[1920*1080*3];//#todo new from config
|
size_t buffSize;
|
const size_t buffSizeMax;
|
|
size_t inputFrameCount;
|
int lastOutputBuffIdx; // -1 is invalid for android omx
|
|
MB_Frame lastMbfBuffIdx; // pm for bufidx
|
MB_Frame lastMbfBuffer; // frame for buffer (decoded data)
|
MB_Frame* lastMbfList[2];
|
|
PL_AndroidMediaCodecDecoder_Config config;
|
|
AMediaCodec* codec;
|
|
PL_AMCD_Internal() :
|
buffSize(0), buffSizeMax(sizeof(buffer)),
|
inputFrameCount(0), lastOutputBuffIdx(-1),
|
lastMbfBuffIdx(), lastMbfBuffer(),
|
config(),
|
codec(nullptr)
|
{
|
}
|
|
~PL_AMCD_Internal()
|
{
|
}
|
|
void reset()
|
{
|
buffSize = 0;
|
|
inputFrameCount = 0;
|
lastOutputBuffIdx = 0;
|
|
lastMbfBuffIdx.reset();
|
lastMbfBuffer.reset();
|
lastMbfList[0] = &lastMbfBuffIdx;
|
lastMbfList[1] = &lastMbfBuffer;
|
|
PL_AndroidMediaCodecDecoder_Config _config;
|
config = _config;
|
|
codec = nullptr;
|
}
|
};
|
|
PipeLineElem* create_PL_AndroidMediaCodecDecoder()
|
{
|
return new PL_AndroidMediaCodecDecoder;
|
}
|
|
PL_AndroidMediaCodecDecoder::PL_AndroidMediaCodecDecoder() : internal(new PL_AMCD_Internal)
|
{
|
}
|
|
PL_AndroidMediaCodecDecoder::~PL_AndroidMediaCodecDecoder()
|
{
|
delete (PL_AMCD_Internal*)internal;
|
internal= nullptr;
|
}
|
|
bool PL_AndroidMediaCodecDecoder::init(void* args)
|
{
|
PL_AMCD_Internal* in = (PL_AMCD_Internal*)internal;
|
in->reset();
|
|
PL_AndroidMediaCodecDecoder_Config* config = (PL_AndroidMediaCodecDecoder_Config*)args;
|
in->config = *config;
|
|
AMediaFormat* format = AMediaFormat_new();
|
|
AMediaFormat_setString(format, AMEDIAFORMAT_KEY_MIME, config->ak_mime.c_str());
|
AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_HEIGHT, config->ak_height);
|
AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_WIDTH, config->ak_width);
|
|
// see: https://developer.android.com/reference/android/media/MediaCodecInfo.CodecCapabilities.html#COLOR_FormatYUV420Flexible
|
#define AMEDIA_COLOR_FormatYUV420Flexible 0x7f420888
|
//AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_COLOR_FORMAT, AMEDIA_COLOR_FormatYUV420Flexible);
|
//AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_STRIDE, config->ak_width*2);
|
|
// #19 COLOR_FormatYUV420Planar (I420)
|
// #20 COLOR_FormatYUV420PackedPlanar (also I420)
|
// #21 COLOR_FormatYUV420SemiPlanar (NV12)
|
// #39 COLOR_FormatYUV420PackedSemiPlanar (also NV12)
|
// #0x7f000100 COLOR_TI_FormatYUV420PackedSemiPlanar (also also NV12)
|
|
//tegra: I420
|
//qcom: NV12
|
|
/*
|
std::string fmtp(manager->get_param(PLGP_RTSP_FMTP));
|
if (fmtp.empty())
|
return false;
|
|
uint32_t numSPropRecords = 0;
|
SPropRecord *p_record = parseSPropParameterSets(fmtp.c_str(), numSPropRecords);
|
if (numSPropRecords < 2)
|
{
|
LOG_WARN << "numSPropRecords < 2" << std::endl;
|
return false;
|
}
|
|
SPropRecord &sps = p_record[0];
|
SPropRecord &pps = p_record[1];
|
|
AMediaFormat_setBuffer(format, "csd-0", sps.sPropBytes, sps.sPropLength); // sps
|
AMediaFormat_setBuffer(format, "csd-1", pps.sPropBytes, pps.sPropLength); // pps
|
|
//uint8_t sps[] = {0x0,0x0,0x0,0x1, 0x67, 0x42, 0x00, 0x2A, 0x95, 0xA8, 0x1E, 0x00, 0x89, 0xF9, 0x61, 0x00, 0x00, 0x07, 0x08, 0x00, 0x01, 0x5F, 0x90, 0x04};
|
//uint8_t pps[] = {0x0,0x0,0x0,0x1, 0x68, 0xCE, 0x3C, 0x80};
|
//AMediaFormat_setBuffer(format, "csd-0", sps, sizeof(sps)); // sps
|
//AMediaFormat_setBuffer(format, "csd-1", pps, sizeof(pps)); // pps
|
*/
|
|
// should like:
|
// mime: string(video/avc), durationUs: int64(10000000), width: int32(480), height: int32(360), max-input-size: int32(55067), csd-0: data, csd-1: data}
|
LOG_INFO << "AMediaFormat_toString: " << AMediaFormat_toString(format) << LOG_ENDL;
|
|
in->codec = AMediaCodec_createDecoderByType(config->ak_mime.c_str());
|
|
// if no sureface android use software decoder
|
// see: http://stackoverflow.com/questions/15500290/access-violation-in-native-code-with-hardware-accelerated-android-mediacodec-dec
|
ANativeWindow* windowDecode = (ANativeWindow*)config->windowSurfaceDecode;
|
if (AMediaCodec_configure(in->codec, format, windowDecode, NULL, 0) != AMEDIA_OK)
|
{
|
AMediaFormat_delete(format);
|
LOG_ERROR << "AMediaCodec_configure error" << LOG_ENDL;
|
return false;
|
}
|
|
if (AMediaCodec_start(in->codec) != AMEDIA_OK)
|
{
|
AMediaFormat_delete(format);
|
LOG_ERROR << "AMediaCodec_start error" << LOG_ENDL;
|
return false;
|
}
|
|
AMediaFormat_delete(format);
|
return true;
|
}
|
|
void PL_AndroidMediaCodecDecoder::finit()
|
{
|
PL_AMCD_Internal* in = (PL_AMCD_Internal*)internal;
|
//todo release codec
|
// call AMediaCodec_stop
|
}
|
|
bool PL_AndroidMediaCodecDecoder::pay(const PipeMaterial& pm)
|
{
|
PL_AMCD_Internal* in = (PL_AMCD_Internal*)internal;
|
|
if (pm.type != PipeMaterial::PMT_FRAME)
|
{
|
LOG_ERROR << "Only support PMT_FRAME" << std::endl;
|
return false;
|
}
|
|
if (pm.buffer == nullptr)
|
return false;
|
|
MB_Frame* frame = (MB_Frame*)pm.buffer;
|
if (frame->type != MB_Frame::MBFT_H264_NALU)
|
{
|
LOG_ERROR << "Only support MBFT_H264_NALU" << std::endl;
|
return false;
|
}
|
|
ssize_t bufidx = AMediaCodec_dequeueInputBuffer(in->codec, 2000);
|
LOGP(DEBUG, "input buffer bufidx=%zd, inputFrameCount=%d", bufidx, in->inputFrameCount++);
|
|
if (bufidx >= 0)
|
{
|
size_t bufsize;
|
uint8_t* inputBuff = AMediaCodec_getInputBuffer(in->codec, bufidx, &bufsize);
|
size_t sampleSize = std::min(bufsize, frame->buffSize);
|
memcpy(inputBuff, frame->buffer, sampleSize); // fill buffer
|
|
uint64_t presentationTimeUs = timeval_to_microseconds(frame->pts); //microseconds
|
|
media_status_t ms = AMediaCodec_queueInputBuffer(in->codec, bufidx, 0, sampleSize, presentationTimeUs, 0);
|
LOGP(DEBUG, "media_status_t=%d", ms);
|
}
|
else
|
{
|
LOG_WARN << "bufidx=" << bufidx << LOG_ENDL;
|
return false; // return true for gain
|
}
|
|
if (in->config.releaseOutputBuffIdxInPay)
|
{
|
AMediaCodecBufferInfo info;
|
in->lastOutputBuffIdx = AMediaCodec_dequeueOutputBuffer(in->codec, &info, 0);
|
LOG_WARN << "releaseOutputBuffIdxInPay bufidx=" << in->lastOutputBuffIdx << ", flags=" << info.flags << LOG_ENDL;
|
|
if (in->lastOutputBuffIdx >= 0)
|
{
|
if (in->config.releaseOutputBuffIdx)
|
{
|
AMediaCodec_releaseOutputBuffer(in->codec, in->lastOutputBuffIdx, info.size != 0); // in->config.windowSurfaceRender != nullptr
|
in->lastOutputBuffIdx = -1;
|
}
|
}
|
}
|
|
return true;
|
}
|
|
static bool rended = false;
|
|
bool PL_AndroidMediaCodecDecoder::gain(PipeMaterial& pm)
|
{
|
PL_AMCD_Internal* in = (PL_AMCD_Internal*)internal;
|
|
AMediaCodecBufferInfo info;
|
in->lastOutputBuffIdx = AMediaCodec_dequeueOutputBuffer(in->codec, &info, 0);
|
rended = false;
|
|
if (in->lastOutputBuffIdx >= 0)
|
{
|
if (info.flags & AMEDIACODEC_BUFFER_FLAG_END_OF_STREAM)
|
{
|
LOGP(WARNING, "output EOS");
|
}
|
//int64_t presentationNano = info.presentationTimeUs * 1000;
|
//if (d->renderstart < 0) {
|
// d->renderstart = systemnanotime() - presentationNano;
|
//}
|
//int64_t delay = (d->renderstart + presentationNano) - systemnanotime();
|
//if (delay > 0) {
|
// usleep(delay / 1000);
|
//}
|
|
AMediaFormat* format = AMediaCodec_getOutputFormat(in->codec);
|
if (format != NULL)
|
{
|
int32_t width, height, color;
|
AMediaFormat_getInt32(format, AMEDIAFORMAT_KEY_WIDTH, &width);
|
AMediaFormat_getInt32(format, AMEDIAFORMAT_KEY_HEIGHT, &height);
|
AMediaFormat_getInt32(format, AMEDIAFORMAT_KEY_COLOR_FORMAT, &color);
|
AMediaFormat_delete(format);
|
format = nullptr;
|
|
LOGP(DEBUG, "output media format, w=%d, h=%d, c=%d", width, height, color);
|
}
|
|
in->lastMbfBuffIdx.reset();
|
if (! in->config.releaseOutputBuffIdx)
|
{
|
in->lastMbfBuffIdx.type = MB_Frame::MBFT_INDEX;
|
in->lastMbfBuffIdx.buffer = (void*)(in->lastOutputBuffIdx);
|
in->lastMbfBuffIdx.buffSize = sizeof(in->lastOutputBuffIdx);
|
in->lastMbfBuffIdx.width = in->config.ak_width;
|
in->lastMbfBuffIdx.height = in->config.ak_height;
|
microseconds_to_timeval(info.presentationTimeUs, in->lastMbfBuffIdx.pts);
|
}
|
|
in->lastMbfBuffer.reset();
|
if ((in->config.generateDecodedDataPerFrame != 0) && (in->inputFrameCount % in->config.generateDecodedDataPerFrame == 0))
|
{
|
size_t outSize = in->buffSizeMax;
|
uint8_t* outputBuff = AMediaCodec_getOutputBuffer(in->codec, in->lastOutputBuffIdx, &outSize);
|
if (outputBuff != nullptr)
|
{
|
in->buffSize = std::min((size_t) info.size, in->buffSizeMax);
|
memcpy(in->buffer, outputBuff + info.offset, in->buffSize);
|
|
/*
|
if (in->buffSize <= 8 && in->config.windowSurfaceDecode != nullptr)
|
{
|
ANativeWindow* window = (ANativeWindow*)(in->config.windowSurfaceDecode);
|
|
ANativeWindow_Buffer wbuffer;
|
if (ANativeWindow_lock(window, &wbuffer, NULL) == 0)
|
{
|
size_t bitsSize = 0;
|
if (wbuffer.format == WINDOW_FORMAT_RGBA_8888 || wbuffer.format == WINDOW_FORMAT_RGBX_8888)
|
bitsSize = wbuffer.height * wbuffer.width * 4;
|
else if (wbuffer.format == WINDOW_FORMAT_RGB_565)
|
bitsSize = wbuffer.height * wbuffer.width * 2;
|
else
|
bitsSize = wbuffer.height * wbuffer.width;
|
|
memcpy(in->buffer, wbuffer.bits, bitsSize);//#test copy opposite
|
in->buffSize = bitsSize;
|
|
ANativeWindow_unlockAndPost(window);
|
}
|
}
|
*/
|
|
/*
|
if (in->config.windowSurfaceRender != nullptr)
|
{
|
//glReadPixels(0, 0, 10, 10, GL_RGBA, GL_UNSIGNED_BYTE, in->buffer);
|
//LOG_WARN<<"glReadPixels err="<<glGetError()<<LOG_ENDL;
|
|
ANativeWindow* window = (ANativeWindow*)(in->config.windowSurfaceRender);
|
|
ANativeWindow_Buffer wbuffer;
|
if (ANativeWindow_lock(window, &wbuffer, NULL) == 0)
|
{
|
size_t bitsSize = 0;
|
if (wbuffer.format == WINDOW_FORMAT_RGBA_8888 || wbuffer.format == WINDOW_FORMAT_RGBX_8888)
|
bitsSize = wbuffer.height * wbuffer.width * 4;
|
else if (wbuffer.format == WINDOW_FORMAT_RGB_565)
|
bitsSize = wbuffer.height * wbuffer.width * 2;
|
else
|
bitsSize = wbuffer.height * wbuffer.width;
|
|
in->buffSize = bitsSize;
|
memcpy(wbuffer.bits, in->buffer, bitsSize);
|
ANativeWindow_unlockAndPost(window);
|
}
|
}
|
*/
|
|
in->lastMbfBuffer.type = MB_Frame::MBFT_YUV420;
|
in->lastMbfBuffer.buffer = in->buffer;
|
in->lastMbfBuffer.buffSize = in->buffSize;
|
in->lastMbfBuffer.width = in->config.ak_width;
|
in->lastMbfBuffer.height = in->config.ak_height;
|
microseconds_to_timeval(info.presentationTimeUs, in->lastMbfBuffer.pts);
|
|
//if (in->lastMbfBuffer.buffSize > 10)
|
//{
|
// static size_t f = 0;
|
// char fname[50];
|
// sprintf(fname, "/sdcard/face-%u.yuv", ++f);
|
// FILE *pFile = fopen(fname, "wb");
|
// fwrite(in->lastMbfBuffer.buffer, sizeof(char), in->lastMbfBuffer.buffSize, pFile);
|
// printf("write face file %s\n", fname);
|
// fclose(pFile);
|
// if (f > 50) exit(0);
|
//}
|
}
|
}
|
|
pm.type = PipeMaterial::PMT_FRAME_LIST;
|
pm.buffer = *(in->lastMbfList); // in->lastMbfList is typeof MB_Frame**
|
pm.buffSize = sizeof(in->lastMbfList) / sizeof(MB_Frame*); // 2
|
pm.deleter = nullptr;
|
pm.former = this;
|
|
if (in->config.releaseOutputBuffIdx)
|
{
|
AMediaCodec_releaseOutputBuffer(in->codec, in->lastOutputBuffIdx, info.size != 0);
|
in->lastOutputBuffIdx = -1;
|
rended = true;
|
}
|
|
return true;
|
}
|
else if (in->lastOutputBuffIdx == AMEDIACODEC_INFO_OUTPUT_BUFFERS_CHANGED)
|
{
|
LOGP(DEBUG, "output buffers changed");
|
}
|
else if (in->lastOutputBuffIdx == AMEDIACODEC_INFO_OUTPUT_FORMAT_CHANGED)
|
{
|
auto format = AMediaCodec_getOutputFormat(in->codec);
|
LOGP(INFO, "format changed to: %s", AMediaFormat_toString(format));
|
AMediaFormat_delete(format);
|
}
|
else if (in->lastOutputBuffIdx == AMEDIACODEC_INFO_TRY_AGAIN_LATER)
|
{
|
LOGP(DEBUG, "no output buffer right now");
|
}
|
else
|
{
|
LOGP(WARNING, "unexpected info code: %zd", in->lastOutputBuffIdx);
|
}
|
|
return false;
|
}
|
|
void dbgReadTexture()
|
{
|
using namespace android;
|
|
static size_t i = 0;
|
if (i++ <300)
|
return;
|
|
//static GLuint pboTextureOES = 0;
|
//glGenBuffers(1, &pboTextureOES);
|
//glBindBuffer(GL_PIXEL_PACK_BUFFER, pboTextureOES);
|
//glBufferData(GL_PIXEL_PACK_BUFFER, 640*480*4, 0, GL_STREAM_READ);
|
//glBindBuffer(GL_PIXEL_PACK_BUFFER, 0);
|
//
|
//
|
//glReadBuffer(GL_COLOR_ATTACHMENT0);
|
//glBindBuffer(GL_PIXEL_PACK_BUFFER, pbo_id);
|
//glReadPixels(0, 0, width, height, GL_RGBA, GL_UNSIGNED_BYTE, 0);
|
//GLubyte *ptr = glMapBufferRange(GL_PIXEL_PACK_BUFFER, 0, pbo_size, GL_MAP_READ_BIT);
|
//memcpy(pixels, ptr, pbo_size);
|
//glUnmapBuffer(GL_PIXEL_PACK_BUFFER);
|
//glBindBuffer(GL_PIXEL_PACK_BUFFER, 0);
|
|
|
static GraphicBuffer *glWin = nullptr;
|
static EGLImageKHR eglImg = nullptr;
|
|
if (glWin == nullptr)
|
{
|
glWin = new GraphicBuffer(640, 480, HAL_PIXEL_FORMAT_RGB_565, GraphicBuffer::USAGE_SW_READ_OFTEN | GraphicBuffer::USAGE_SW_WRITE_RARELY | GraphicBuffer::USAGE_HW_TEXTURE);
|
LOG_WARN << "glWin->initCheck()=" << glWin->initCheck()<<LOG_ENDL;
|
|
glWin->dumpAllocationsToSystemLog();
|
|
//GraphicBufferAlloc* mGraphicBufferAlloc = new GraphicBufferAlloc();
|
//sp<GraphicBuffer> window = mGraphicBufferAlloc->createGraphicBuffer(width, height, PIXEL_FORMAT_RGBA_8888, GraphicBuffer::USAGE_SW_READ_OFTEN | GraphicBuffer::USAGE_HW_TEXTURE,&error);
|
}
|
|
if (eglImg == nullptr)
|
{
|
EGLDisplay eglDisp = eglGetDisplay(EGL_DEFAULT_DISPLAY);
|
EGLContext eglCtx = eglGetCurrentContext();
|
ANativeWindowBuffer* winBuf = glWin->getNativeBuffer();
|
|
const EGLint renderImageAttrs[] = {
|
//EGL_WIDTH, 640,
|
//EGL_HEIGHT, 480,
|
//EGL_MATCH_FORMAT_KHR, EGL_FORMAT_RGBA_8888_KHR,
|
EGL_IMAGE_PRESERVED_KHR, EGL_TRUE,
|
//EGL_MATCH_FORMAT_KHR, EGL_FORMAT_RGB_565_KHR,
|
EGL_NONE, EGL_NONE
|
};
|
eglImg = eglCreateImageKHR(eglDisp, EGL_NO_CONTEXT, EGL_NATIVE_BUFFER_ANDROID, (EGLClientBuffer)winBuf, renderImageAttrs);
|
LOG_ERROR << "eglCreateImageKHR=" << (eglImg != EGL_NO_IMAGE_KHR) << LOG_ENDL;
|
EGLint eglerr = eglGetError(); // EGL_SUCCESS=12288
|
GLenum glerr = glGetError();
|
LOG_ERROR << "eglerr="<<eglerr<<", glerr="<<glerr<<LOG_ENDL;
|
}
|
|
//if (!rended)
|
// return;;
|
|
static int f=0;
|
f++;
|
static uint8_t* texbuf = new uint8_t[640 * 480 * 4];
|
size_t buffSize = 0;
|
|
if (eglImg != nullptr)
|
{
|
GLenum glerr = 0;
|
//glBindTexture(GL_TEXTURE_2D, 1);
|
//if ((glerr=glGetError()) != GL_NO_ERROR) LOG_WARN<<"glBindTexture err="<<(glerr=glGetError())<<LOG_ENDL;
|
//static PFNGLEGLIMAGETARGETTEXTURE2DOESPROC imageTargetTexture2DProc = (PFNGLEGLIMAGETARGETTEXTURE2DOESPROC)eglGetProcAddress("glEGLImageTargetTexture2DOES");
|
glEGLImageTargetTexture2DOES(GL_TEXTURE_EXTERNAL_OES, eglImg);
|
//glEGLImageTargetRenderbufferStorageOES(GL_TEXTURE_EXTERNAL_OES, eglImg);
|
|
if ((glerr=glGetError()) != GL_NO_ERROR) LOG_WARN<<"glEGLImageTargetTexture2DOES err="<<glerr<<LOG_ENDL;
|
|
//glActiveTexture(GL_TEXTURE0);
|
//LOG_WARN<<"glActiveTexture err="<<(glerr=glGetError())<<LOG_ENDL;
|
//
|
//glBindTexture(GL_TEXTURE_EXTERNAL_OES, 1);
|
//LOG_WARN<<"glBindTexture err="<<(glerr=glGetError())<<LOG_ENDL;
|
//
|
//glCopyTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, 0, 0, 640, 480, 0);
|
//LOG_WARN<<"glCopyTexImage2D err="<<(glerr=glGetError())<<LOG_ENDL;
|
|
|
//
|
uint8_t *ptr = nullptr;
|
//status_t lockYCbCr(uint32_t usage, android_ycbcr *ycbcr);
|
//
|
status_t s = glWin->lock(GraphicBuffer::USAGE_SW_READ_OFTEN, (void**)&ptr);
|
memcpy(texbuf, ptr, 640 * 480);
|
if (texbuf[100] != 0||texbuf[101] != 0)
|
{
|
int aa=123;
|
}
|
glWin->unlock();
|
buffSize = 640 * 480;
|
|
|
}
|
|
|
//glReadPixels(0, 0, 640, 480, GL_RGBA, GL_UNSIGNED_BYTE, texbuf);
|
//LOG_WARN<<"glReadPixels err="<<glGetError()<<LOG_ENDL;
|
|
//char fname[50];
|
//sprintf(fname, "/sdcard/dec-%u.yuv", ++f);
|
//FILE *pFile = fopen(fname, "wb");
|
//fwrite(texbuf, sizeof(char), buffSize, pFile);
|
//printf("write face file %s\n", fname);
|
//fclose(pFile);
|
//if (f > 50) exit(0);
|
|
}
|
