#include "PL_AndroidMediaCodecDecoder.h" #include "MaterialBuffer.h" #include "logger.h" #include "MediaHelper.h" #include #include #include #include #include // see: https://developer.android.com/reference/android/media/MediaCodecInfo.CodecCapabilities.html#COLOR_FormatYUV420Flexible #define AMEDIA_COLOR_FormatYUV420Flexible 0x7f420888 #define AMEDIA_COLOR_FormatYUV420Planar 0x00000013 // 19, I420 #define AMEDIA_COLOR_FormatYUV420PackedPlanar 0x00000014 // 20, I420 #define AMEDIA_COLOR_FormatYUV420SemiPlanar 0x00000015 // 21, NV12 #define AMEDIA_COLOR_FormatYUV420PackedSemiPlanar 0x00000027 // 39, NV12 struct PL_AMCD_Internal { uint8_t buffer[1920*1090*3];//#todo new from config size_t buffSize; const size_t buffSizeMax; size_t inputFrameCount; int lastOutputBuffIdx; // -1 is invalid for android omx MB_Frame lastMbfBuffIdx; // pm for bufidx MB_Frame lastMbfBuffer; // frame for buffer (decoded data) MB_Frame* lastMbfList[2]; int lastWidth; int lastHeight; PL_AndroidMediaCodecDecoder_Config config; AMediaCodec* codec; bool auxIsSet; PL_AMCD_Internal() : buffSize(0), buffSizeMax(sizeof(buffer)), inputFrameCount(0), lastOutputBuffIdx(-1), lastMbfBuffIdx(), lastMbfBuffer(), config(), codec(nullptr), auxIsSet(false) { } ~PL_AMCD_Internal() { } void reset() { auxIsSet = false; buffSize = 0; inputFrameCount = 0; lastOutputBuffIdx = 0; lastMbfBuffIdx.reset(); lastMbfBuffer.reset(); lastMbfList[0] = &lastMbfBuffIdx; lastMbfList[1] = &lastMbfBuffer; lastWidth = 0; lastHeight = 0; PL_AndroidMediaCodecDecoder_Config _config; config = _config; codec = nullptr; } }; PipeLineElem* create_PL_AndroidMediaCodecDecoder() { return new PL_AndroidMediaCodecDecoder; } PL_AndroidMediaCodecDecoder::PL_AndroidMediaCodecDecoder() : internal(new PL_AMCD_Internal) { } PL_AndroidMediaCodecDecoder::~PL_AndroidMediaCodecDecoder() { delete (PL_AMCD_Internal*)internal; internal= nullptr; } bool PL_AndroidMediaCodecDecoder::init_codec() { PL_AMCD_Internal* in = (PL_AMCD_Internal*)internal; const PL_AndroidMediaCodecDecoder_Config* config(&(in->config)); if (in->codec != nullptr) { LOG_ERROR << "codec not null" << LOG_ENDL; return false; } in->auxIsSet = false; AMediaFormat* format = AMediaFormat_new(); AMediaFormat_setString(format, AMEDIAFORMAT_KEY_MIME, config->ak_mime.c_str()); AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_HEIGHT, config->ak_height); AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_WIDTH, config->ak_width); //AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_COLOR_FORMAT, AMEDIA_COLOR_FormatYUV420); AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_MAX_INPUT_SIZE, config->ak_height * config->ak_width * 1.5); //AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_STRIDE, config->ak_width*2); // #19 COLOR_FormatYUV420Planar (I420) // #20 COLOR_FormatYUV420PackedPlanar (also I420) // #21 COLOR_FormatYUV420SemiPlanar (NV12) // #39 COLOR_FormatYUV420PackedSemiPlanar (also NV12) // #0x7f000100 COLOR_TI_FormatYUV420PackedSemiPlanar (also also NV12) //tegra: I420 //qcom: NV12 /* std::string fmtp(manager->get_param(PLGP_RTSP_FMTP)); if (fmtp.empty()) return false; uint32_t numSPropRecords = 0; SPropRecord *p_record = parseSPropParameterSets(fmtp.c_str(), numSPropRecords); if (numSPropRecords < 2) { LOG_WARN << "numSPropRecords < 2" << std::endl; return false; } SPropRecord &sps = p_record[0]; SPropRecord &pps = p_record[1]; AMediaFormat_setBuffer(format, "csd-0", sps.sPropBytes, sps.sPropLength); // sps AMediaFormat_setBuffer(format, "csd-1", pps.sPropBytes, pps.sPropLength); // pps */ // J0LgKI1oCgPaEAAAAwAQAAADAoDxB6gA,KM4ySA== // uint8_t sps[] = {0x27,0x42,0xe0,0x28,0x8d,0x68,0x0a,0x03,0xda,0x10,0x00,0x00,0x03,0x00,0x10,0x00,0x00,0x03,0x02,0x80,0xf1,0x07,0xa8,0x00}; // uint8_t pps[] = {0x28,0xce,0x32,0x48}; // AMediaFormat_setBuffer(format, "csd-0", sps, sizeof(sps)); // sps // AMediaFormat_setBuffer(format, "csd-1", pps, sizeof(pps)); // pps if (!in->auxIsSet) { std::string base64_sps = this->manager->get_param(PLGP_DEC_SPS_B64); std::string base64_pps = this->manager->get_param(PLGP_DEC_PPS_B64); if ((!base64_sps.empty()) && (!base64_pps.empty())) { size_t result_sps = 0; size_t result_pps = 0; uint8_t *uc_sps = base64_decode(base64_sps.c_str(), base64_sps.length(), result_sps); uint8_t *uc_pps = base64_decode(base64_pps.c_str(), base64_pps.length(), result_pps); char tmp[100] = {0x00,0x00,0x00,0x01,0x00}; memcpy(tmp + 4, uc_sps, result_sps); AMediaFormat_setBuffer(format, "csd-0", tmp, result_sps + 4); // sps memcpy(tmp + 4, uc_pps, result_pps); AMediaFormat_setBuffer(format, "csd-1", tmp, result_pps + 4); // pps in->auxIsSet = true; delete[] uc_sps; delete[] uc_pps; } } // should like: // mime: string(video/avc), durationUs: int64(10000000), width: int32(480), height: int32(360), max-input-size: int32(55067), csd-0: data, csd-1: data} const char* fmt = AMediaFormat_toString(format); LOG_INFO << "AMediaFormat_toString: " << fmt << LOG_ENDL; in->codec = AMediaCodec_createDecoderByType(config->ak_mime.c_str()); // if no sureface android use software decoder // see: http://stackoverflow.com/questions/15500290/access-violation-in-native-code-with-hardware-accelerated-android-mediacodec-dec ANativeWindow* windowDecode = (ANativeWindow*)config->windowSurfaceDecode; if (AMediaCodec_configure(in->codec, format, windowDecode, NULL, 0) != AMEDIA_OK) { AMediaCodec_delete(in->codec); in->codec = nullptr; AMediaFormat_delete(format); LOG_ERROR << "AMediaCodec_configure error" << LOG_ENDL; return false; } if (AMediaCodec_start(in->codec) != AMEDIA_OK) { AMediaCodec_delete(in->codec); in->codec = nullptr; AMediaFormat_delete(format); LOG_ERROR << "AMediaCodec_start error" << LOG_ENDL; return false; } AMediaFormat_delete(format); return true; } bool PL_AndroidMediaCodecDecoder::init(void* args) { PL_AMCD_Internal* in = (PL_AMCD_Internal*)internal; in->reset(); PL_AndroidMediaCodecDecoder_Config* config = (PL_AndroidMediaCodecDecoder_Config*)args; in->config = *config; if (!in->config.initCodecInPay) return init_codec(); return true; } void PL_AndroidMediaCodecDecoder::finit() { PL_AMCD_Internal* in = (PL_AMCD_Internal*)internal; AMediaCodec_stop(in->codec); AMediaCodec_delete(in->codec); in->codec = nullptr; in->reset(); } void PL_AndroidMediaCodecDecoder::aux_is_set() { // support only in android-26 (8.0) #if 0 PL_AMCD_Internal* in = (PL_AMCD_Internal*)internal; if (!in->auxIsSet) { std::string base64_sps = this->manager->get_param(PLGP_DEC_SPS_B64); std::string base64_pps = this->manager->get_param(PLGP_DEC_PPS_B64); if((!base64_sps.empty()) && (!base64_pps.empty())) { size_t result_sps = 0; size_t result_pps = 0; uint8_t* uc_sps = base64_decode(base64_sps.c_str(),base64_sps.length(),result_sps); uint8_t* uc_pps = base64_decode(base64_pps.c_str(),base64_pps.length(),result_pps); AMediaFormat* format = AMediaCodec_getOutputFormat(in->codec); AMediaFormat_setBuffer(format, "csd-0", uc_sps, result_sps); // sps AMediaFormat_setBuffer(format, "csd-1", uc_pps, result_pps); // pps AMediaCodec_setParameters(in->codec,format); AMediaFormat_delete(format); delete[] uc_sps; uc_sps = nullptr; delete[] uc_pps; uc_pps = nullptr; in->auxIsSet = true; } } #endif } bool PL_AndroidMediaCodecDecoder::pay(const PipeMaterial& pm) { PL_AMCD_Internal* in = (PL_AMCD_Internal*)internal; //PipeLineElemTimingDebugger td(this); // RK3288, 300~1300us@1920, MAX=7400us if (pm.type != PipeMaterial::PMT_FRAME) { LOG_ERROR << "Only support PMT_FRAME" << std::endl; return false; } if (pm.buffer == nullptr) return false; if (in->config.initCodecInPay && in->codec == nullptr) { if (!init_codec()) { LOG_ERROR << "init_codec error" << std::endl; return false; } } else { aux_is_set(); } MB_Frame* frame = (MB_Frame*)pm.buffer; if (frame->type != MB_Frame::MBFT_H264_NALU) { LOG_ERROR << "Only support MBFT_H264_NALU" << std::endl; return false; } // static FILE *pFile = fopen("/data/bb.264", "wb"); // fwrite(frame->buffer, sizeof(char), frame->buffSize, pFile); // fflush(pFile); ssize_t bufidx = AMediaCodec_dequeueInputBuffer(in->codec, 2000); LOGP(DEBUG, "input buffer bufidx=%zd, inputFrameCount=%d", bufidx, in->inputFrameCount++); if (bufidx >= 0) { size_t bufsize; uint8_t* inputBuff = AMediaCodec_getInputBuffer(in->codec, bufidx, &bufsize); size_t sampleSize = std::min(bufsize, frame->buffSize); memcpy(inputBuff, frame->buffer, sampleSize); // fill buffer uint64_t presentationTimeUs = timeval_to_microseconds(frame->pts); //microseconds media_status_t ms = AMediaCodec_queueInputBuffer(in->codec, bufidx, 0, sampleSize, presentationTimeUs, 0); LOGP(DEBUG, "media_status_t=%d", ms); } else { LOG_WARN << "bufidx=" << bufidx << LOG_ENDL; return false; // return true for gain } if (in->config.releaseOutputBuffIdxInPay) { AMediaCodecBufferInfo info; in->lastOutputBuffIdx = AMediaCodec_dequeueOutputBuffer(in->codec, &info, 0); LOG_WARN << "releaseOutputBuffIdxInPay bufidx=" << in->lastOutputBuffIdx << ", flags=" << info.flags << LOG_ENDL; if (in->lastOutputBuffIdx >= 0) { if (in->config.releaseOutputBuffIdx) { AMediaCodec_releaseOutputBuffer(in->codec, in->lastOutputBuffIdx, info.size != 0); // in->config.windowSurfaceRender != nullptr in->lastOutputBuffIdx = -1; } } } return true; } static bool rended = false; bool PL_AndroidMediaCodecDecoder::gain(PipeMaterial& pm) { PL_AMCD_Internal* in = (PL_AMCD_Internal*)internal; //PipeLineElemTimingDebugger td(this); // RK3288, 51~16000@1920, MAX=26000us AMediaCodecBufferInfo info; in->lastOutputBuffIdx = AMediaCodec_dequeueOutputBuffer(in->codec, &info, 0); rended = false; if (in->lastOutputBuffIdx >= 0) { if (info.flags & AMEDIACODEC_BUFFER_FLAG_END_OF_STREAM) { LOGP(WARNING, "output EOS"); } //int64_t presentationNano = info.presentationTimeUs * 1000; //if (d->renderstart < 0) { // d->renderstart = systemnanotime() - presentationNano; //} //int64_t delay = (d->renderstart + presentationNano) - systemnanotime(); //if (delay > 0) { // usleep(delay / 1000); //} AMediaFormat* format = AMediaCodec_getOutputFormat(in->codec); int32_t width = in->config.ak_width; int32_t height = in->config.ak_height; int32_t color = 0; if (format != NULL) { AMediaFormat_getInt32(format, AMEDIAFORMAT_KEY_WIDTH, &width); AMediaFormat_getInt32(format, AMEDIAFORMAT_KEY_HEIGHT, &height); AMediaFormat_getInt32(format, AMEDIAFORMAT_KEY_COLOR_FORMAT, &color); AMediaFormat_delete(format); format = nullptr; LOGP(DEBUG, "output media format, w=%d, h=%d, c=%d", width, height, color); } in->lastMbfBuffIdx.reset(); if (! in->config.releaseOutputBuffIdx) { in->lastMbfBuffIdx.type = MB_Frame::MBFT_INDEX; in->lastMbfBuffIdx.buffer = (void*)(in->lastOutputBuffIdx); in->lastMbfBuffIdx.buffSize = sizeof(in->lastOutputBuffIdx); in->lastMbfBuffIdx.width = width; in->lastMbfBuffIdx.height = height; microseconds_to_timeval(info.presentationTimeUs, in->lastMbfBuffIdx.pts); } in->lastMbfBuffer.reset(); if ((in->config.generateDecodedDataPerFrame != 0) && (in->inputFrameCount % in->config.generateDecodedDataPerFrame == 0)) { size_t outSize = in->buffSizeMax; uint8_t* outputBuff = AMediaCodec_getOutputBuffer(in->codec, in->lastOutputBuffIdx, &outSize); if (outputBuff != nullptr) { in->buffSize = std::min((size_t) info.size, in->buffSizeMax); memcpy(in->buffer, outputBuff + info.offset, in->buffSize); /* if (in->buffSize <= 8 && in->config.windowSurfaceDecode != nullptr) { ANativeWindow* window = (ANativeWindow*)(in->config.windowSurfaceDecode); ANativeWindow_Buffer wbuffer; if (ANativeWindow_lock(window, &wbuffer, NULL) == 0) { size_t bitsSize = 0; if (wbuffer.format == WINDOW_FORMAT_RGBA_8888 || wbuffer.format == WINDOW_FORMAT_RGBX_8888) bitsSize = wbuffer.height * wbuffer.width * 4; else if (wbuffer.format == WINDOW_FORMAT_RGB_565) bitsSize = wbuffer.height * wbuffer.width * 2; else bitsSize = wbuffer.height * wbuffer.width; memcpy(in->buffer, wbuffer.bits, bitsSize);//#test copy opposite in->buffSize = bitsSize; ANativeWindow_unlockAndPost(window); } } */ in->lastMbfBuffer.type = MB_Frame::MBFT_YUV420; if (color == AMEDIA_COLOR_FormatYUV420Planar || color == AMEDIA_COLOR_FormatYUV420PackedPlanar) // I420 in->lastMbfBuffer.type = MB_Frame::MBFT_YUV420; else if (color == AMEDIA_COLOR_FormatYUV420SemiPlanar || color == AMEDIA_COLOR_FormatYUV420PackedSemiPlanar) // NV12 in->lastMbfBuffer.type = MB_Frame::MBFT_NV12; else { LOG_WARN << "color format not support" << LOG_ENDL; return false; } in->lastMbfBuffer.buffer = in->buffer; in->lastMbfBuffer.buffSize = in->buffSize; in->lastMbfBuffer.width = in->config.ak_width; in->lastMbfBuffer.height = in->config.ak_height; if (in->config.ptsUseAbsoluteTime) gettimeofday(&(in->lastMbfBuffer.pts), nullptr); else microseconds_to_timeval(info.presentationTimeUs, in->lastMbfBuffer.pts); //if (in->lastMbfBuffer.buffSize > 10) //{ // static size_t f = 0; // char fname[50]; // sprintf(fname, "/sdcard/amcd-%u.yuv", ++f); // FILE *pFile = fopen(fname, "wb"); // fwrite(in->lastMbfBuffer.buffer, sizeof(char), in->lastMbfBuffer.buffSize, pFile); // printf("write face file %s\n", fname); // fclose(pFile); // if (f > 50) exit(0); //} } } pm.type = PipeMaterial::PMT_FRAME_LIST; pm.buffer = *(in->lastMbfList); // in->lastMbfList is typeof MB_Frame** pm.buffSize = sizeof(in->lastMbfList) / sizeof(MB_Frame*); // 2 pm.deleter = nullptr; pm.former = this; if (in->config.releaseOutputBuffIdx) { AMediaCodec_releaseOutputBuffer(in->codec, in->lastOutputBuffIdx, info.size != 0); in->lastOutputBuffIdx = -1; rended = true; } return true; } else if (in->lastOutputBuffIdx == AMEDIACODEC_INFO_OUTPUT_BUFFERS_CHANGED) { LOGP(DEBUG, "output buffers changed"); } else if (in->lastOutputBuffIdx == AMEDIACODEC_INFO_OUTPUT_FORMAT_CHANGED) { auto format = AMediaCodec_getOutputFormat(in->codec); LOGP(INFO, "format changed to: %s", AMediaFormat_toString(format)); AMediaFormat_delete(format); } else if (in->lastOutputBuffIdx == AMEDIACODEC_INFO_TRY_AGAIN_LATER) { LOGP(DEBUG, "no output buffer right now"); } else { LOGP(WARNING, "unexpected info code: %zd", in->lastOutputBuffIdx); } return false; }