From 9e5babf9db52e64bdae60137be7696e56241fca6 Mon Sep 17 00:00:00 2001 From: xingzilong <xingzilong@454eff88-639b-444f-9e54-f578c98de674> Date: 星期五, 18 八月 2017 18:12:17 +0800 Subject: [PATCH] H264 NALU解析 并在RTSPServer判断 --- RtspFace/PL_AndroidMediaCodecDecoder_ndk.cpp | 406 +++++++++++++++++++++++++++++++++++---------------------- 1 files changed, 247 insertions(+), 159 deletions(-) diff --git a/RtspFace/PL_AndroidMediaCodecDecoder_ndk.cpp b/RtspFace/PL_AndroidMediaCodecDecoder_ndk.cpp index 79a8a46..dd7e505 100644 --- a/RtspFace/PL_AndroidMediaCodecDecoder_ndk.cpp +++ b/RtspFace/PL_AndroidMediaCodecDecoder_ndk.cpp @@ -11,6 +11,7 @@ #include <stdlib.h> #include <liveMedia/liveMedia.hh> + // see: https://developer.android.com/reference/android/media/MediaCodecInfo.CodecCapabilities.html#COLOR_FormatYUV420Flexible #define AMEDIA_COLOR_FormatYUV420Flexible 0x7f420888 #define AMEDIA_COLOR_FormatYUV420Planar 0x00000013 // 19, I420 @@ -31,10 +32,10 @@ MB_Frame lastMbfBuffer; // frame for buffer (decoded data) MB_Frame* lastMbfList[2]; - int lastWidth; - int lastHeight; + int lastWidth; + int lastHeight; - PL_AndroidMediaCodecDecoder_Config config; + PL_AndroidMediaCodecDecoder_Config config; AMediaCodec* codec; @@ -56,6 +57,7 @@ void reset() { + auxIsSet = false; buffSize = 0; inputFrameCount = 0; @@ -66,8 +68,8 @@ lastMbfList[0] = &lastMbfBuffIdx; lastMbfList[1] = &lastMbfBuffer; - lastWidth = 0; - lastHeight = 0; + lastWidth = 0; + lastHeight = 0; PL_AndroidMediaCodecDecoder_Config _config; config = _config; @@ -91,6 +93,125 @@ internal= nullptr; } +bool PL_AndroidMediaCodecDecoder::init_codec() +{ + PL_AMCD_Internal* in = (PL_AMCD_Internal*)internal; + const PL_AndroidMediaCodecDecoder_Config* config(&(in->config)); + + if (in->codec != nullptr) + { + LOG_ERROR << "codec not null" << LOG_ENDL; + return false; + } + + in->auxIsSet = false; + + AMediaFormat* format = AMediaFormat_new(); + + AMediaFormat_setString(format, AMEDIAFORMAT_KEY_MIME, config->ak_mime.c_str()); + AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_HEIGHT, config->ak_height); + AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_WIDTH, config->ak_width); + + //AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_COLOR_FORMAT, AMEDIA_COLOR_FormatYUV420); + AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_MAX_INPUT_SIZE, config->ak_height * config->ak_width * 1.5); + + //AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_STRIDE, config->ak_width*2); + + // #19 COLOR_FormatYUV420Planar (I420) + // #20 COLOR_FormatYUV420PackedPlanar (also I420) + // #21 COLOR_FormatYUV420SemiPlanar (NV12) + // #39 COLOR_FormatYUV420PackedSemiPlanar (also NV12) + // #0x7f000100 COLOR_TI_FormatYUV420PackedSemiPlanar (also also NV12) + + //tegra: I420 + //qcom: NV12 + + /* + std::string fmtp(manager->get_param(PLGP_RTSP_FMTP)); + if (fmtp.empty()) + return false; + + uint32_t numSPropRecords = 0; + SPropRecord *p_record = parseSPropParameterSets(fmtp.c_str(), numSPropRecords); + if (numSPropRecords < 2) + { + LOG_WARN << "numSPropRecords < 2" << std::endl; + return false; + } + + SPropRecord &sps = p_record[0]; + SPropRecord &pps = p_record[1]; + + AMediaFormat_setBuffer(format, "csd-0", sps.sPropBytes, sps.sPropLength); // sps + AMediaFormat_setBuffer(format, "csd-1", pps.sPropBytes, pps.sPropLength); // pps + */ + + // J0LgKI1oCgPaEAAAAwAQAAADAoDxB6gA,KM4ySA== + +// uint8_t sps[] = {0x27,0x42,0xe0,0x28,0x8d,0x68,0x0a,0x03,0xda,0x10,0x00,0x00,0x03,0x00,0x10,0x00,0x00,0x03,0x02,0x80,0xf1,0x07,0xa8,0x00}; +// uint8_t pps[] = {0x28,0xce,0x32,0x48}; +// AMediaFormat_setBuffer(format, "csd-0", sps, sizeof(sps)); // sps +// AMediaFormat_setBuffer(format, "csd-1", pps, sizeof(pps)); // pps + + if (!in->auxIsSet) + { + std::string base64_sps = this->manager->get_param(PLGP_DEC_SPS_B64); + std::string base64_pps = this->manager->get_param(PLGP_DEC_PPS_B64); + if ((!base64_sps.empty()) && (!base64_pps.empty())) + { + size_t result_sps = 0; + size_t result_pps = 0; + + uint8_t *uc_sps = base64_decode(base64_sps.c_str(), base64_sps.length(), result_sps); + uint8_t *uc_pps = base64_decode(base64_pps.c_str(), base64_pps.length(), result_pps); + + char tmp[100] = {0x00,0x00,0x00,0x01,0x00}; + memcpy(tmp + 4, uc_sps, result_sps); + AMediaFormat_setBuffer(format, "csd-0", tmp, result_sps + 4); // sps + + memcpy(tmp + 4, uc_pps, result_pps); + AMediaFormat_setBuffer(format, "csd-1", tmp, result_pps + 4); // pps + in->auxIsSet = true; + + delete[] uc_sps; + delete[] uc_pps; + } + } + + // should like: + // mime: string(video/avc), durationUs: int64(10000000), width: int32(480), height: int32(360), max-input-size: int32(55067), csd-0: data, csd-1: data} + const char* fmt = AMediaFormat_toString(format); + LOG_INFO << "AMediaFormat_toString: " << fmt << LOG_ENDL; + + in->codec = AMediaCodec_createDecoderByType(config->ak_mime.c_str()); + + // if no sureface android use software decoder + // see: http://stackoverflow.com/questions/15500290/access-violation-in-native-code-with-hardware-accelerated-android-mediacodec-dec + ANativeWindow* windowDecode = (ANativeWindow*)config->windowSurfaceDecode; + if (AMediaCodec_configure(in->codec, format, windowDecode, NULL, 0) != AMEDIA_OK) + { + AMediaCodec_delete(in->codec); + in->codec = nullptr; + + AMediaFormat_delete(format); + LOG_ERROR << "AMediaCodec_configure error" << LOG_ENDL; + return false; + } + + if (AMediaCodec_start(in->codec) != AMEDIA_OK) + { + AMediaCodec_delete(in->codec); + in->codec = nullptr; + + AMediaFormat_delete(format); + LOG_ERROR << "AMediaCodec_start error" << LOG_ENDL; + return false; + } + + AMediaFormat_delete(format); + return true; +} + bool PL_AndroidMediaCodecDecoder::init(void* args) { PL_AMCD_Internal* in = (PL_AMCD_Internal*)internal; @@ -99,77 +220,9 @@ PL_AndroidMediaCodecDecoder_Config* config = (PL_AndroidMediaCodecDecoder_Config*)args; in->config = *config; - AMediaFormat* format = AMediaFormat_new(); + if (!in->config.initCodecInPay) + return init_codec(); - AMediaFormat_setString(format, AMEDIAFORMAT_KEY_MIME, config->ak_mime.c_str()); - AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_HEIGHT, config->ak_height); - AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_WIDTH, config->ak_width); - - //AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_COLOR_FORMAT, AMEDIA_COLOR_FormatYUV420); - AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_MAX_INPUT_SIZE, config->ak_height * config->ak_width * 1.5); - - //AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_STRIDE, config->ak_width*2); - - // #19 COLOR_FormatYUV420Planar (I420) - // #20 COLOR_FormatYUV420PackedPlanar (also I420) - // #21 COLOR_FormatYUV420SemiPlanar (NV12) - // #39 COLOR_FormatYUV420PackedSemiPlanar (also NV12) - // #0x7f000100 COLOR_TI_FormatYUV420PackedSemiPlanar (also also NV12) - - //tegra: I420 - //qcom: NV12 - - /* - std::string fmtp(manager->get_param(PLGP_RTSP_FMTP)); - if (fmtp.empty()) - return false; - - uint32_t numSPropRecords = 0; - SPropRecord *p_record = parseSPropParameterSets(fmtp.c_str(), numSPropRecords); - if (numSPropRecords < 2) - { - LOG_WARN << "numSPropRecords < 2" << std::endl; - return false; - } - - SPropRecord &sps = p_record[0]; - SPropRecord &pps = p_record[1]; - - AMediaFormat_setBuffer(format, "csd-0", sps.sPropBytes, sps.sPropLength); // sps - AMediaFormat_setBuffer(format, "csd-1", pps.sPropBytes, pps.sPropLength); // pps - */ - - // J0LgKI1oCgPaEAAAAwAQAAADAoDxB6gA,KM4ySA== - //uint8_t sps[] = {0x27,0x42,0xe0,0x28,0x8d,0x68,0x0a,0x03,0xda,0x10,0x00,0x00,0x03,0x00,0x10,0x00,0x00,0x03,0x02,0x80,0xf1,0x07,0xa8,0x00}; - //uint8_t pps[] = {0x28,0xce,0x32,0x48}; - //AMediaFormat_setBuffer(format, "csd-0", sps, sizeof(sps)); // sps - //AMediaFormat_setBuffer(format, "csd-1", pps, sizeof(pps)); // pps - - // should like: - // mime: string(video/avc), durationUs: int64(10000000), width: int32(480), height: int32(360), max-input-size: int32(55067), csd-0: data, csd-1: data} - const char* fmt = AMediaFormat_toString(format); - LOG_INFO << "AMediaFormat_toString: " << fmt << LOG_ENDL; - - in->codec = AMediaCodec_createDecoderByType(config->ak_mime.c_str()); - - // if no sureface android use software decoder - // see: http://stackoverflow.com/questions/15500290/access-violation-in-native-code-with-hardware-accelerated-android-mediacodec-dec - ANativeWindow* windowDecode = (ANativeWindow*)config->windowSurfaceDecode; - if (AMediaCodec_configure(in->codec, format, windowDecode, NULL, 0) != AMEDIA_OK) - { - AMediaFormat_delete(format); - LOG_ERROR << "AMediaCodec_configure error" << LOG_ENDL; - return false; - } - - if (AMediaCodec_start(in->codec) != AMEDIA_OK) - { - AMediaFormat_delete(format); - LOG_ERROR << "AMediaCodec_start error" << LOG_ENDL; - return false; - } - - AMediaFormat_delete(format); return true; } @@ -179,8 +232,44 @@ AMediaCodec_stop(in->codec); AMediaCodec_delete(in->codec); + in->codec = nullptr; in->reset(); +} + +void PL_AndroidMediaCodecDecoder::aux_is_set() +{ + // support only in android-26 (8.0) +#if 0 + PL_AMCD_Internal* in = (PL_AMCD_Internal*)internal; + + if (!in->auxIsSet) + { + std::string base64_sps = this->manager->get_param(PLGP_DEC_SPS_B64); + std::string base64_pps = this->manager->get_param(PLGP_DEC_PPS_B64); + if((!base64_sps.empty()) && (!base64_pps.empty())) + { + size_t result_sps = 0; + size_t result_pps = 0; + + uint8_t* uc_sps = base64_decode(base64_sps.c_str(),base64_sps.length(),result_sps); + uint8_t* uc_pps = base64_decode(base64_pps.c_str(),base64_pps.length(),result_pps); + + AMediaFormat* format = AMediaCodec_getOutputFormat(in->codec); + AMediaFormat_setBuffer(format, "csd-0", uc_sps, result_sps); // sps + AMediaFormat_setBuffer(format, "csd-1", uc_pps, result_pps); // pps + AMediaCodec_setParameters(in->codec,format); + AMediaFormat_delete(format); + + delete[] uc_sps; + uc_sps = nullptr; + delete[] uc_pps; + uc_pps = nullptr; + + in->auxIsSet = true; + } + } +#endif } bool PL_AndroidMediaCodecDecoder::pay(const PipeMaterial& pm) @@ -197,18 +286,17 @@ if (pm.buffer == nullptr) return false; - if (!in->auxIsSet) + if (in->config.initCodecInPay && in->codec == nullptr) { - //#todo - // find PLGP_DEC_SPS_B64 PLGP_DEC_PPS_B64 in this->manager else nothing - // base64 decode - - //AMediaFormat* format = AMediaCodec_getOutputFormat(in->codec); - //AMediaFormat_setBuffer(format, "csd-1", pps, sizeof(pps)); // pps - //AMediaCodec_setParameters - // #todo delete format - - in->auxIsSet = true; + if (!init_codec()) + { + LOG_ERROR << "init_codec error" << std::endl; + return false; + } + } + else + { + aux_is_set(); } MB_Frame* frame = (MB_Frame*)pm.buffer; @@ -218,25 +306,25 @@ return false; } - //static FILE *pFile = fopen("/data/bb.264", "wb"); - //fwrite(frame->buffer, sizeof(char), frame->buffSize, pFile); - //fflush(pFile); +// static FILE *pFile = fopen("/data/bb.264", "wb"); +// fwrite(frame->buffer, sizeof(char), frame->buffSize, pFile); +// fflush(pFile); - ssize_t bufidx = AMediaCodec_dequeueInputBuffer(in->codec, 2000); - LOGP(DEBUG, "input buffer bufidx=%zd, inputFrameCount=%d", bufidx, in->inputFrameCount++); + ssize_t bufidx = AMediaCodec_dequeueInputBuffer(in->codec, 2000); + LOGP(DEBUG, "input buffer bufidx=%zd, inputFrameCount=%d", bufidx, in->inputFrameCount++); - if (bufidx >= 0) + if (bufidx >= 0) { - size_t bufsize; - uint8_t* inputBuff = AMediaCodec_getInputBuffer(in->codec, bufidx, &bufsize); - size_t sampleSize = std::min(bufsize, frame->buffSize); - memcpy(inputBuff, frame->buffer, sampleSize); // fill buffer + size_t bufsize; + uint8_t* inputBuff = AMediaCodec_getInputBuffer(in->codec, bufidx, &bufsize); + size_t sampleSize = std::min(bufsize, frame->buffSize); + memcpy(inputBuff, frame->buffer, sampleSize); // fill buffer - uint64_t presentationTimeUs = timeval_to_microseconds(frame->pts); //microseconds + uint64_t presentationTimeUs = timeval_to_microseconds(frame->pts); //microseconds - media_status_t ms = AMediaCodec_queueInputBuffer(in->codec, bufidx, 0, sampleSize, presentationTimeUs, 0); - LOGP(DEBUG, "media_status_t=%d", ms); - } + media_status_t ms = AMediaCodec_queueInputBuffer(in->codec, bufidx, 0, sampleSize, presentationTimeUs, 0); + LOGP(DEBUG, "media_status_t=%d", ms); + } else { LOG_WARN << "bufidx=" << bufidx << LOG_ENDL; @@ -267,13 +355,13 @@ bool PL_AndroidMediaCodecDecoder::gain(PipeMaterial& pm) { PL_AMCD_Internal* in = (PL_AMCD_Internal*)internal; - //PipeLineElemTimingDebugger td(this); // RK3288, 51~16000@1920, MAX=26000us + //PipeLineElemTimingDebugger td(this); // RK3288, 51~16000@1920, MAX=26000us AMediaCodecBufferInfo info; in->lastOutputBuffIdx = AMediaCodec_dequeueOutputBuffer(in->codec, &info, 0); - rended = false; + rended = false; - if (in->lastOutputBuffIdx >= 0) + if (in->lastOutputBuffIdx >= 0) { if (info.flags & AMEDIACODEC_BUFFER_FLAG_END_OF_STREAM) { @@ -288,21 +376,21 @@ // usleep(delay / 1000); //} - AMediaFormat* format = AMediaCodec_getOutputFormat(in->codec); - int32_t width = in->config.ak_width; - int32_t height = in->config.ak_height; - int32_t color = 0; + AMediaFormat* format = AMediaCodec_getOutputFormat(in->codec); + int32_t width = in->config.ak_width; + int32_t height = in->config.ak_height; + int32_t color = 0; - if (format != NULL) - { - AMediaFormat_getInt32(format, AMEDIAFORMAT_KEY_WIDTH, &width); - AMediaFormat_getInt32(format, AMEDIAFORMAT_KEY_HEIGHT, &height); - AMediaFormat_getInt32(format, AMEDIAFORMAT_KEY_COLOR_FORMAT, &color); - AMediaFormat_delete(format); - format = nullptr; + if (format != NULL) + { + AMediaFormat_getInt32(format, AMEDIAFORMAT_KEY_WIDTH, &width); + AMediaFormat_getInt32(format, AMEDIAFORMAT_KEY_HEIGHT, &height); + AMediaFormat_getInt32(format, AMEDIAFORMAT_KEY_COLOR_FORMAT, &color); + AMediaFormat_delete(format); + format = nullptr; - LOGP(DEBUG, "output media format, w=%d, h=%d, c=%d", width, height, color); - } + LOGP(DEBUG, "output media format, w=%d, h=%d, c=%d", width, height, color); + } in->lastMbfBuffIdx.reset(); if (! in->config.releaseOutputBuffIdx) @@ -318,60 +406,60 @@ in->lastMbfBuffer.reset(); if ((in->config.generateDecodedDataPerFrame != 0) && (in->inputFrameCount % in->config.generateDecodedDataPerFrame == 0)) { - size_t outSize = in->buffSizeMax; - uint8_t* outputBuff = AMediaCodec_getOutputBuffer(in->codec, in->lastOutputBuffIdx, &outSize); - if (outputBuff != nullptr) - { - in->buffSize = std::min((size_t) info.size, in->buffSizeMax); - memcpy(in->buffer, outputBuff + info.offset, in->buffSize); + size_t outSize = in->buffSizeMax; + uint8_t* outputBuff = AMediaCodec_getOutputBuffer(in->codec, in->lastOutputBuffIdx, &outSize); + if (outputBuff != nullptr) + { + in->buffSize = std::min((size_t) info.size, in->buffSizeMax); + memcpy(in->buffer, outputBuff + info.offset, in->buffSize); - /* - if (in->buffSize <= 8 && in->config.windowSurfaceDecode != nullptr) - { - ANativeWindow* window = (ANativeWindow*)(in->config.windowSurfaceDecode); + /* + if (in->buffSize <= 8 && in->config.windowSurfaceDecode != nullptr) + { + ANativeWindow* window = (ANativeWindow*)(in->config.windowSurfaceDecode); - ANativeWindow_Buffer wbuffer; - if (ANativeWindow_lock(window, &wbuffer, NULL) == 0) - { - size_t bitsSize = 0; - if (wbuffer.format == WINDOW_FORMAT_RGBA_8888 || wbuffer.format == WINDOW_FORMAT_RGBX_8888) - bitsSize = wbuffer.height * wbuffer.width * 4; - else if (wbuffer.format == WINDOW_FORMAT_RGB_565) - bitsSize = wbuffer.height * wbuffer.width * 2; - else - bitsSize = wbuffer.height * wbuffer.width; + ANativeWindow_Buffer wbuffer; + if (ANativeWindow_lock(window, &wbuffer, NULL) == 0) + { + size_t bitsSize = 0; + if (wbuffer.format == WINDOW_FORMAT_RGBA_8888 || wbuffer.format == WINDOW_FORMAT_RGBX_8888) + bitsSize = wbuffer.height * wbuffer.width * 4; + else if (wbuffer.format == WINDOW_FORMAT_RGB_565) + bitsSize = wbuffer.height * wbuffer.width * 2; + else + bitsSize = wbuffer.height * wbuffer.width; - memcpy(in->buffer, wbuffer.bits, bitsSize);//#test copy opposite - in->buffSize = bitsSize; + memcpy(in->buffer, wbuffer.bits, bitsSize);//#test copy opposite + in->buffSize = bitsSize; - ANativeWindow_unlockAndPost(window); - } - } - */ + ANativeWindow_unlockAndPost(window); + } + } + */ - in->lastMbfBuffer.type = MB_Frame::MBFT_YUV420; + in->lastMbfBuffer.type = MB_Frame::MBFT_YUV420; - if (color == AMEDIA_COLOR_FormatYUV420Planar || color == AMEDIA_COLOR_FormatYUV420PackedPlanar) // I420 - in->lastMbfBuffer.type = MB_Frame::MBFT_YUV420; - else if (color == AMEDIA_COLOR_FormatYUV420SemiPlanar || color == AMEDIA_COLOR_FormatYUV420PackedSemiPlanar) // NV12 - in->lastMbfBuffer.type = MB_Frame::MBFT_NV12; - else - { - LOG_WARN << "color format not support" << LOG_ENDL; - return false; - } + if (color == AMEDIA_COLOR_FormatYUV420Planar || color == AMEDIA_COLOR_FormatYUV420PackedPlanar) // I420 + in->lastMbfBuffer.type = MB_Frame::MBFT_YUV420; + else if (color == AMEDIA_COLOR_FormatYUV420SemiPlanar || color == AMEDIA_COLOR_FormatYUV420PackedSemiPlanar) // NV12 + in->lastMbfBuffer.type = MB_Frame::MBFT_NV12; + else + { + LOG_WARN << "color format not support" << LOG_ENDL; + return false; + } - in->lastMbfBuffer.buffer = in->buffer; - in->lastMbfBuffer.buffSize = in->buffSize; - in->lastMbfBuffer.width = in->config.ak_width; - in->lastMbfBuffer.height = in->config.ak_height; + in->lastMbfBuffer.buffer = in->buffer; + in->lastMbfBuffer.buffSize = in->buffSize; + in->lastMbfBuffer.width = in->config.ak_width; + in->lastMbfBuffer.height = in->config.ak_height; if (in->config.ptsUseAbsoluteTime) gettimeofday(&(in->lastMbfBuffer.pts), nullptr); else microseconds_to_timeval(info.presentationTimeUs, in->lastMbfBuffer.pts); - //if (in->lastMbfBuffer.buffSize > 10) + //if (in->lastMbfBuffer.buffSize > 10) //{ // static size_t f = 0; // char fname[50]; @@ -382,7 +470,7 @@ // fclose(pFile); // if (f > 50) exit(0); //} - } + } } pm.type = PipeMaterial::PMT_FRAME_LIST; @@ -395,7 +483,7 @@ { AMediaCodec_releaseOutputBuffer(in->codec, in->lastOutputBuffIdx, info.size != 0); in->lastOutputBuffIdx = -1; - rended = true; + rended = true; } return true; -- Gitblit v1.8.0