From b022b91c0c6fa807424b6c12cc92ac5946838083 Mon Sep 17 00:00:00 2001
From: houxiao <houxiao@454eff88-639b-444f-9e54-f578c98de674>
Date: 星期四, 13 七月 2017 16:34:39 +0800
Subject: [PATCH] update pipeline
---
RtspFace/PL_AndroidMediaCodecDecoder.cpp | 322 ++++++++++++++++++++++++++++++++++++++++-------------
1 files changed, 242 insertions(+), 80 deletions(-)
diff --git a/RtspFace/PL_AndroidMediaCodecDecoder.cpp b/RtspFace/PL_AndroidMediaCodecDecoder.cpp
index 23ed474..65f6b35 100644
--- a/RtspFace/PL_AndroidMediaCodecDecoder.cpp
+++ b/RtspFace/PL_AndroidMediaCodecDecoder.cpp
@@ -1,23 +1,38 @@
#include "PL_AndroidMediaCodecDecoder.h"
#include "MaterialBuffer.h"
#include "logger.h"
+#include "MediaHelper.h"
+
+#include <media/NdkMediaCodec.h>
+#include <media/NdkMediaFormat.h>
+
+#include <android/bitmap.h>
+
+#include <stdlib.h>
+#include <liveMedia/liveMedia.hh>
struct PL_AMCD_Internal
{
- uint8_t buffer[1920*1080*4];//#todo from config
+ uint8_t buffer[1920*1080*3];//#todo new from config
size_t buffSize;
- size_t buffSizeMax;
- MB_Frame lastFrame;
+ const size_t buffSizeMax;
+
+ size_t inputFrameCount;
+ int lastOutputBuffIdx; // -1 is invalid for android omx
+
+ MB_Frame lastMbfBuffIdx; // pm for bufidx
+ MB_Frame lastMbfBuffer; // frame for buffer (decoded data)
+ MB_Frame* lastMbfList[2];
+
PL_AndroidMediaCodecDecoder_Config config;
- bool payError;
-
AMediaCodec* codec;
-
+
PL_AMCD_Internal() :
- buffSize(0), buffSizeMax(sizeof(buffer)), lastFrame(),
+ buffSize(0), buffSizeMax(sizeof(buffer)),
+ inputFrameCount(0), lastOutputBuffIdx(-1),
+ lastMbfBuffIdx(), lastMbfBuffer(),
config(),
- payError(true),
codec(nullptr)
{
}
@@ -29,10 +44,14 @@
void reset()
{
buffSize = 0;
- payError = true;
- MB_Frame _lastFrame;
- lastFrame = _lastFrame;
+ inputFrameCount = 0;
+ lastOutputBuffIdx = 0;
+
+ lastMbfBuffIdx.reset();
+ lastMbfBuffer.reset();
+ lastMbfList[0] = &lastMbfBuffIdx;
+ lastMbfList[1] = &lastMbfBuffer;
PL_AndroidMediaCodecDecoder_Config _config;
config = _config;
@@ -66,23 +85,73 @@
AMediaFormat* format = AMediaFormat_new();
- AMediaFormat_setString(format, AMEDIAFORMAT_KEY_MIME, config.ak_mime.c_str());
- AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_HEIGHT, config.ak_height);
- AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_WIDTH, config.ak_width);
+ AMediaFormat_setString(format, AMEDIAFORMAT_KEY_MIME, config->ak_mime.c_str());
+ AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_HEIGHT, config->ak_height);
+ AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_WIDTH, config->ak_width);
+
+// see: https://developer.android.com/reference/android/media/MediaCodecInfo.CodecCapabilities.html#COLOR_FormatYUV420Flexible
+#define AMEDIA_COLOR_FormatYUV420Flexible 0x7f420888
+ //AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_COLOR_FORMAT, AMEDIA_COLOR_FormatYUV420Flexible);
+ //AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_STRIDE, config->ak_width*2);
+
+ // #19 COLOR_FormatYUV420Planar (I420)
+ // #20 COLOR_FormatYUV420PackedPlanar (also I420)
+ // #21 COLOR_FormatYUV420SemiPlanar (NV12)
+ // #39 COLOR_FormatYUV420PackedSemiPlanar (also NV12)
+ // #0x7f000100 COLOR_TI_FormatYUV420PackedSemiPlanar (also also NV12)
+
+ //tegra: I420
+ //qcom: NV12
+
+ /*
+ std::string fmtp(manager->get_param(PLGP_RTSP_FMTP));
+ if (fmtp.empty())
+ return false;
+
+ uint32_t numSPropRecords = 0;
+ SPropRecord *p_record = parseSPropParameterSets(fmtp.c_str(), numSPropRecords);
+ if (numSPropRecords < 2)
+ {
+ LOG_WARN << "numSPropRecords < 2" << std::endl;
+ return false;
+ }
+
+ SPropRecord &sps = p_record[0];
+ SPropRecord &pps = p_record[1];
+
+ AMediaFormat_setBuffer(format, "csd-0", sps.sPropBytes, sps.sPropLength); // sps
+ AMediaFormat_setBuffer(format, "csd-1", pps.sPropBytes, pps.sPropLength); // pps
+
+ //uint8_t sps[] = {0x0,0x0,0x0,0x1, 0x67, 0x42, 0x00, 0x2A, 0x95, 0xA8, 0x1E, 0x00, 0x89, 0xF9, 0x61, 0x00, 0x00, 0x07, 0x08, 0x00, 0x01, 0x5F, 0x90, 0x04};
+ //uint8_t pps[] = {0x0,0x0,0x0,0x1, 0x68, 0xCE, 0x3C, 0x80};
+ //AMediaFormat_setBuffer(format, "csd-0", sps, sizeof(sps)); // sps
+ //AMediaFormat_setBuffer(format, "csd-1", pps, sizeof(pps)); // pps
+ */
// should like:
// mime: string(video/avc), durationUs: int64(10000000), width: int32(480), height: int32(360), max-input-size: int32(55067), csd-0: data, csd-1: data}
LOG_INFO << "AMediaFormat_toString: " << AMediaFormat_toString(format) << LOG_ENDL;
- in->codec = AMediaCodec_createDecoderByType(config.ak_mime.c_str());
- if (AMediaCodec_configure(in->codec, format, data.window, NULL, 0) != AMEDIA_OK)
+ in->codec = AMediaCodec_createDecoderByType(config->ak_mime.c_str());
+
+ // if no sureface android use software decoder
+ // see: http://stackoverflow.com/questions/15500290/access-violation-in-native-code-with-hardware-accelerated-android-mediacodec-dec
+ ANativeWindow* window = (config->renderFromOutputBuffIdx ? (ANativeWindow*)config->windowSurface : nullptr);
+ if (AMediaCodec_configure(in->codec, format, window, NULL, 0) != AMEDIA_OK)
+ {
+ AMediaFormat_delete(format);
LOG_ERROR << "AMediaCodec_configure error" << LOG_ENDL;
+ return false;
+ }
if (AMediaCodec_start(in->codec) != AMEDIA_OK)
+ {
+ AMediaFormat_delete(format);
LOG_ERROR << "AMediaCodec_start error" << LOG_ENDL;
+ return false;
+ }
AMediaFormat_delete(format);
-
return true;
}
@@ -90,6 +159,7 @@
{
PL_AMCD_Internal* in = (PL_AMCD_Internal*)internal;
//todo release codec
+ // call AMediaCodec_stop
}
bool PL_AndroidMediaCodecDecoder::pay(const PipeMaterial& pm)
@@ -111,73 +181,44 @@
LOG_ERROR << "Only support MBFT_H264_NALU" << std::endl;
return false;
}
-
-
-
-
+
ssize_t bufidx = AMediaCodec_dequeueInputBuffer(in->codec, 2000);
- static int framecount = 0;
- LOGP(DEBUG, "input buffer bufidx=%zd, framecount=%d", bufidx, framecount++);
+ LOGP(DEBUG, "input buffer bufidx=%zd, inputFrameCount=%d", bufidx, in->inputFrameCount++);
if (bufidx >= 0)
{
size_t bufsize;
- uint8_t* buf = AMediaCodec_getInputBuffer(in->codec, bufidx, &bufsize);
+ uint8_t* inputBuff = AMediaCodec_getInputBuffer(in->codec, bufidx, &bufsize);
size_t sampleSize = std::min(bufsize, frame->buffSize);
- memcpy(buf, buffer, sampleSize);
- //auto sampleSize = AMediaExtractor_readSampleData(d->ex, buf, bufsize);
- //if (sampleSize < 0) {
- // sampleSize = 0;
- // d->sawInputEOS = true;
- // LOGV("EOS");
- //}
- //auto presentationTimeUs = AMediaExtractor_getSampleTime(d->ex);
- uint64_t presentationTimeUs = presentationTime.tv_sec * 1000 * 1000 + presentationTime.tv_usec; //microseconds
+ memcpy(inputBuff, frame->buffer, sampleSize); // fill buffer
- media_status_t ms = AMediaCodec_queueInputBuffer(data.codec, bufidx, 0, sampleSize, presentationTimeUs, 0);
- //LOGV("media_status_t=%d", ms);
- //AMediaExtractor_advance(d->ex);
+ uint64_t presentationTimeUs = timeval_to_microseconds(frame->pts); //microseconds
+
+ media_status_t ms = AMediaCodec_queueInputBuffer(in->codec, bufidx, 0, sampleSize, presentationTimeUs, 0);
+ LOGP(DEBUG, "media_status_t=%d", ms);
}
+ else
+ {
+ LOG_WARN << "bufidx=" << bufidx << LOG_ENDL;
+ return false; // return true for gain
+ }
-
-//
-// AVFrame* pAVFrame = (AVFrame*)frame->buffer;
-// if (pAVFrame == nullptr)
-// return false;
-//
-// const int height = pAVFrame->height;
-// const int width = pAVFrame->width;
-//
-////int I420ToBGRA(const uint8* src_y, int src_stride_y,
-//// const uint8* src_u, int src_stride_u,
-//// const uint8* src_v, int src_stride_v,
-//// uint8* dst_argb, int dst_stride_argb,
-//// int width, int height);
-//
-// libyuv::I420ToBGRA(pAVFrame->data[0], width,
-// pAVFrame->data[1], SUBSAMPLE(width, 2),
-// pAVFrame->data[2], SUBSAMPLE(width, 2),
-// in->buffer, 4 * width,
-// width, height);
-//
-// in->buffSize = in->buffSizeMax;
-// //in->buffer readly
-//
-// in->lastFrame.type = MB_Frame::MBFT_BGRA;
-// in->lastFrame.buffer = in->buffer;
-// in->lastFrame.buffSize = in->buffSize;
-// in->lastFrame.width = width;
-// in->lastFrame.height = height;
-// in->lastFrame.pts = frame->pts;
-//
-// //#test
-// //static size_t f=0;
-// //char fname[50];
-// //sprintf(fname, "%u.bgra", ++f);
-// //FILE * pFile = fopen (fname,"wb");
-// //fwrite (in->buffer , sizeof(char), in->buffSize, pFile);
-// //fclose(pFile);
-//
+ if (in->config.releaseOutputBuffIdxInPay)
+ {
+ AMediaCodecBufferInfo info;
+ in->lastOutputBuffIdx = AMediaCodec_dequeueOutputBuffer(in->codec, &info, 0);
+ LOG_WARN << "releaseOutputBuffIdxInPay bufidx=" << in->lastOutputBuffIdx << ", flags=" << info.flags << LOG_ENDL;
+
+ if (in->lastOutputBuffIdx >= 0)
+ {
+ if (in->config.releaseOutputBuffIdx)
+ {
+ AMediaCodec_releaseOutputBuffer(in->codec, in->lastOutputBuffIdx, info.size != 0);
+ in->lastOutputBuffIdx = -1;
+ }
+ }
+ }
+
return true;
}
@@ -185,9 +226,130 @@
{
PL_AMCD_Internal* in = (PL_AMCD_Internal*)internal;
- pm.type = PipeMaterial::PMT_FRAME;
- pm.buffer = &(in->lastFrame);
- pm.buffSize = 0;
- pm.former = this;
- return true;
+ AMediaCodecBufferInfo info;
+ in->lastOutputBuffIdx = AMediaCodec_dequeueOutputBuffer(in->codec, &info, 0);
+ if (in->lastOutputBuffIdx >= 0)
+ {
+ if (info.flags & AMEDIACODEC_BUFFER_FLAG_END_OF_STREAM)
+ {
+ LOGP(WARNING, "output EOS");
+ }
+ //int64_t presentationNano = info.presentationTimeUs * 1000;
+ //if (d->renderstart < 0) {
+ // d->renderstart = systemnanotime() - presentationNano;
+ //}
+ //int64_t delay = (d->renderstart + presentationNano) - systemnanotime();
+ //if (delay > 0) {
+ // usleep(delay / 1000);
+ //}
+
+ AMediaFormat* format = AMediaCodec_getOutputFormat(in->codec);
+ if (format != NULL)
+ {
+ int32_t width, height, color;
+ AMediaFormat_getInt32(format, AMEDIAFORMAT_KEY_WIDTH, &width);
+ AMediaFormat_getInt32(format, AMEDIAFORMAT_KEY_HEIGHT, &height);
+ AMediaFormat_getInt32(format, AMEDIAFORMAT_KEY_COLOR_FORMAT, &color);
+ AMediaFormat_delete(format);
+ format = nullptr;
+
+ LOGP(DEBUG, "output media format, w=%d, h=%d, c=%d", width, height, color);
+ }
+
+ in->lastMbfBuffIdx.reset();
+ if (! in->config.releaseOutputBuffIdx)
+ {
+ in->lastMbfBuffIdx.type = MB_Frame::MBFT_INDEX;
+ in->lastMbfBuffIdx.buffer = (void*)(in->lastOutputBuffIdx);
+ in->lastMbfBuffIdx.buffSize = sizeof(in->lastOutputBuffIdx);
+ in->lastMbfBuffIdx.width = in->config.ak_width;
+ in->lastMbfBuffIdx.height = in->config.ak_height;
+ microseconds_to_timeval(info.presentationTimeUs, in->lastMbfBuffIdx.pts);
+ }
+
+ in->lastMbfBuffer.reset();
+ if ((in->config.generateDecodedDataPerFrame != 0) && (in->inputFrameCount % in->config.generateDecodedDataPerFrame == 0))
+ {
+ size_t outSize = in->buffSizeMax;
+ uint8_t* outputBuff = AMediaCodec_getOutputBuffer(in->codec, in->lastOutputBuffIdx, &outSize);
+ if (outputBuff != nullptr)
+ {
+ in->buffSize = std::min((size_t) info.size, in->buffSizeMax);
+ memcpy(in->buffer, outputBuff + info.offset, in->buffSize);
+
+ if (in->config.renderFromOutputBuff)
+ {
+ ANativeWindow* window = (ANativeWindow*)(in->config.windowSurface);
+ ANativeWindow_Buffer wbuffer;
+ if (ANativeWindow_lock(window, &wbuffer, NULL) == 0)
+ {
+ size_t bitsSize = 0;
+ if (wbuffer.format == WINDOW_FORMAT_RGBA_8888 || wbuffer.format == WINDOW_FORMAT_RGBX_8888)
+ bitsSize = wbuffer.height*wbuffer.width*4;
+ else if (wbuffer.format == WINDOW_FORMAT_RGB_565)
+ bitsSize = wbuffer.height*wbuffer.width*2;
+ else
+ bitsSize = wbuffer.height*wbuffer.width;
+
+ memcpy(wbuffer.bits, in->buffer, bitsSize);
+ //memcpy(in->buffer, wbuffer.bits, bitsSize);//#test copy opposite
+ ANativeWindow_unlockAndPost(window);
+ }
+ }
+
+ in->lastMbfBuffer.type = MB_Frame::MBFT_YUV420;
+ in->lastMbfBuffer.buffer = in->buffer;
+ in->lastMbfBuffer.buffSize = in->buffSize;
+ in->lastMbfBuffer.width = in->config.ak_width;
+ in->lastMbfBuffer.height = in->config.ak_height;
+ microseconds_to_timeval(info.presentationTimeUs, in->lastMbfBuffer.pts);
+
+ //if (in->lastMbfBuffer.buffSize > 10)
+ //{
+ // static size_t f = 0;
+ // char fname[50];
+ // sprintf(fname, "/sdcard/face-%u.yuv", ++f);
+ // FILE *pFile = fopen(fname, "wb");
+ // fwrite(in->lastMbfBuffer.buffer, sizeof(char), in->lastMbfBuffer.buffSize, pFile);
+ // printf("write face file %s\n", fname);
+ // fclose(pFile);
+ // if (f > 50) exit(0);
+ //}
+ }
+ }
+
+ pm.type = PipeMaterial::PMT_FRAME_LIST;
+ pm.buffer = *(in->lastMbfList); // in->lastMbfList is typeof MB_Frame**
+ pm.buffSize = sizeof(in->lastMbfList) / sizeof(MB_Frame*); // 2
+ pm.deleter = nullptr;
+ pm.former = this;
+
+ if (in->config.releaseOutputBuffIdx)
+ {
+ AMediaCodec_releaseOutputBuffer(in->codec, in->lastOutputBuffIdx, in->config.renderFromOutputBuffIdx);//info.size != 0
+ in->lastOutputBuffIdx = -1;
+ }
+
+ return true;
+ }
+ else if (in->lastOutputBuffIdx == AMEDIACODEC_INFO_OUTPUT_BUFFERS_CHANGED)
+ {
+ LOGP(DEBUG, "output buffers changed");
+ }
+ else if (in->lastOutputBuffIdx == AMEDIACODEC_INFO_OUTPUT_FORMAT_CHANGED)
+ {
+ auto format = AMediaCodec_getOutputFormat(in->codec);
+ LOGP(INFO, "format changed to: %s", AMediaFormat_toString(format));
+ AMediaFormat_delete(format);
+ }
+ else if (in->lastOutputBuffIdx == AMEDIACODEC_INFO_TRY_AGAIN_LATER)
+ {
+ LOGP(DEBUG, "no output buffer right now");
+ }
+ else
+ {
+ LOGP(WARNING, "unexpected info code: %zd", in->lastOutputBuffIdx);
+ }
+
+ return false;
}
--
Gitblit v1.8.0