xingzilong
2017-08-18 9e5babf9db52e64bdae60137be7696e56241fca6
RtspFace/PL_AndroidMediaCodecDecoder_ndk.cpp
@@ -11,6 +11,7 @@
#include <stdlib.h>
#include <liveMedia/liveMedia.hh>
// see: https://developer.android.com/reference/android/media/MediaCodecInfo.CodecCapabilities.html#COLOR_FormatYUV420Flexible
#define AMEDIA_COLOR_FormatYUV420Flexible 0x7f420888
#define AMEDIA_COLOR_FormatYUV420Planar 0x00000013 // 19, I420
@@ -31,19 +32,22 @@
   MB_Frame lastMbfBuffer; // frame for buffer (decoded data)
   MB_Frame* lastMbfList[2];
    int lastWidth;
    int lastHeight;
   int lastWidth;
   int lastHeight;
    PL_AndroidMediaCodecDecoder_Config config;
   PL_AndroidMediaCodecDecoder_Config config;
   AMediaCodec* codec;
   bool auxIsSet;
   PL_AMCD_Internal() : 
      buffSize(0), buffSizeMax(sizeof(buffer)), 
      inputFrameCount(0), lastOutputBuffIdx(-1), 
      lastMbfBuffIdx(), lastMbfBuffer(),
      config(), 
      codec(nullptr)
      codec(nullptr),
      auxIsSet(false)
   {
   }
   
@@ -53,6 +57,7 @@
   
   void reset()
   {
      auxIsSet = false;
      buffSize = 0;
      
      inputFrameCount = 0;
@@ -63,8 +68,8 @@
      lastMbfList[0] = &lastMbfBuffIdx;
      lastMbfList[1] = &lastMbfBuffer;
        lastWidth = 0;
        lastHeight = 0;
      lastWidth = 0;
      lastHeight = 0;
      PL_AndroidMediaCodecDecoder_Config _config;
      config = _config;
@@ -88,6 +93,125 @@
   internal= nullptr;
}
bool PL_AndroidMediaCodecDecoder::init_codec()
{
   PL_AMCD_Internal* in = (PL_AMCD_Internal*)internal;
   const PL_AndroidMediaCodecDecoder_Config* config(&(in->config));
   if (in->codec != nullptr)
   {
      LOG_ERROR << "codec not null" << LOG_ENDL;
      return false;
   }
   in->auxIsSet = false;
   AMediaFormat* format = AMediaFormat_new();
   AMediaFormat_setString(format, AMEDIAFORMAT_KEY_MIME, config->ak_mime.c_str());
   AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_HEIGHT, config->ak_height);
   AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_WIDTH, config->ak_width);
   //AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_COLOR_FORMAT, AMEDIA_COLOR_FormatYUV420);
   AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_MAX_INPUT_SIZE, config->ak_height * config->ak_width * 1.5);
   //AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_STRIDE, config->ak_width*2);
   // #19 COLOR_FormatYUV420Planar (I420)
   // #20 COLOR_FormatYUV420PackedPlanar (also I420)
   // #21 COLOR_FormatYUV420SemiPlanar (NV12)
   // #39 COLOR_FormatYUV420PackedSemiPlanar (also NV12)
   // #0x7f000100 COLOR_TI_FormatYUV420PackedSemiPlanar (also also NV12)
   //tegra: I420
   //qcom: NV12
   /*
      std::string fmtp(manager->get_param(PLGP_RTSP_FMTP));
      if (fmtp.empty())
         return false;
      uint32_t numSPropRecords = 0;
      SPropRecord *p_record = parseSPropParameterSets(fmtp.c_str(), numSPropRecords);
      if (numSPropRecords < 2)
      {
         LOG_WARN << "numSPropRecords < 2" << std::endl;
         return false;
      }
      SPropRecord &sps = p_record[0];
      SPropRecord &pps = p_record[1];
      AMediaFormat_setBuffer(format, "csd-0", sps.sPropBytes, sps.sPropLength); // sps
      AMediaFormat_setBuffer(format, "csd-1", pps.sPropBytes, pps.sPropLength); // pps
   */
   // J0LgKI1oCgPaEAAAAwAQAAADAoDxB6gA,KM4ySA==
//   uint8_t sps[] = {0x27,0x42,0xe0,0x28,0x8d,0x68,0x0a,0x03,0xda,0x10,0x00,0x00,0x03,0x00,0x10,0x00,0x00,0x03,0x02,0x80,0xf1,0x07,0xa8,0x00};
//   uint8_t pps[] = {0x28,0xce,0x32,0x48};
//   AMediaFormat_setBuffer(format, "csd-0", sps, sizeof(sps)); // sps
//   AMediaFormat_setBuffer(format, "csd-1", pps, sizeof(pps)); // pps
   if (!in->auxIsSet)
   {
      std::string base64_sps = this->manager->get_param(PLGP_DEC_SPS_B64);
      std::string base64_pps = this->manager->get_param(PLGP_DEC_PPS_B64);
      if ((!base64_sps.empty()) && (!base64_pps.empty()))
      {
         size_t result_sps = 0;
         size_t result_pps = 0;
         uint8_t *uc_sps = base64_decode(base64_sps.c_str(), base64_sps.length(), result_sps);
         uint8_t *uc_pps = base64_decode(base64_pps.c_str(), base64_pps.length(), result_pps);
         char tmp[100] = {0x00,0x00,0x00,0x01,0x00};
         memcpy(tmp + 4, uc_sps, result_sps);
         AMediaFormat_setBuffer(format, "csd-0", tmp, result_sps + 4); // sps
         memcpy(tmp + 4, uc_pps, result_pps);
         AMediaFormat_setBuffer(format, "csd-1", tmp, result_pps + 4); // pps
         in->auxIsSet = true;
         delete[] uc_sps;
         delete[] uc_pps;
      }
   }
   // should like:
   // mime: string(video/avc), durationUs: int64(10000000), width: int32(480), height: int32(360), max-input-size: int32(55067), csd-0: data, csd-1: data}
   const char* fmt =  AMediaFormat_toString(format);
   LOG_INFO << "AMediaFormat_toString: " << fmt << LOG_ENDL;
   in->codec = AMediaCodec_createDecoderByType(config->ak_mime.c_str());
   // if no sureface android use software decoder
   // see: http://stackoverflow.com/questions/15500290/access-violation-in-native-code-with-hardware-accelerated-android-mediacodec-dec
   ANativeWindow* windowDecode = (ANativeWindow*)config->windowSurfaceDecode;
   if (AMediaCodec_configure(in->codec, format, windowDecode, NULL, 0) != AMEDIA_OK)
   {
      AMediaCodec_delete(in->codec);
      in->codec = nullptr;
      AMediaFormat_delete(format);
      LOG_ERROR << "AMediaCodec_configure error" << LOG_ENDL;
      return false;
   }
   if (AMediaCodec_start(in->codec) != AMEDIA_OK)
   {
      AMediaCodec_delete(in->codec);
      in->codec = nullptr;
      AMediaFormat_delete(format);
      LOG_ERROR << "AMediaCodec_start error" << LOG_ENDL;
      return false;
   }
   AMediaFormat_delete(format);
   return true;
}
bool PL_AndroidMediaCodecDecoder::init(void* args)
{
   PL_AMCD_Internal* in = (PL_AMCD_Internal*)internal;
@@ -96,78 +220,9 @@
   PL_AndroidMediaCodecDecoder_Config* config = (PL_AndroidMediaCodecDecoder_Config*)args;
   in->config = *config;
    AMediaFormat* format = AMediaFormat_new();
   if (!in->config.initCodecInPay)
      return init_codec();
    AMediaFormat_setString(format, AMEDIAFORMAT_KEY_MIME, config->ak_mime.c_str());
    AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_HEIGHT, config->ak_height);
    AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_WIDTH, config->ak_width);
    //AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_COLOR_FORMAT, AMEDIA_COLOR_FormatYUV420);
    AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_MAX_INPUT_SIZE, config->ak_height * config->ak_width * 1.5);
    //AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_STRIDE, config->ak_width*2);
   // #19 COLOR_FormatYUV420Planar (I420)
   // #20 COLOR_FormatYUV420PackedPlanar (also I420)
   // #21 COLOR_FormatYUV420SemiPlanar (NV12)
   // #39 COLOR_FormatYUV420PackedSemiPlanar (also NV12)
   // #0x7f000100 COLOR_TI_FormatYUV420PackedSemiPlanar (also also NV12)
   //tegra: I420
   //qcom: NV12
    /*
        std::string fmtp(manager->get_param(PLGP_RTSP_FMTP));
        if (fmtp.empty())
            return false;
        uint32_t numSPropRecords = 0;
        SPropRecord *p_record = parseSPropParameterSets(fmtp.c_str(), numSPropRecords);
        if (numSPropRecords < 2)
        {
            LOG_WARN << "numSPropRecords < 2" << std::endl;
            return false;
        }
        SPropRecord &sps = p_record[0];
        SPropRecord &pps = p_record[1];
        AMediaFormat_setBuffer(format, "csd-0", sps.sPropBytes, sps.sPropLength); // sps
        AMediaFormat_setBuffer(format, "csd-1", pps.sPropBytes, pps.sPropLength); // pps
    */
      // J0LgKI1oCgPaEAAAAwAQAAADAoDxB6gA,KM4ySA==
        uint8_t sps[] = {0x27,0x42,0xe0,0x28,0x8d,0x68,0x0a,0x03,0xda,0x10,0x00,0x00,0x03,0x00,0x10,0x00,0x00,0x03,0x02,0x80,0xf1,0x07,0xa8,0x00};
        uint8_t pps[] = {0x28,0xce,0x32,0x48};
        AMediaFormat_setBuffer(format, "csd-0", sps, sizeof(sps)); // sps
        AMediaFormat_setBuffer(format, "csd-1", pps, sizeof(pps)); // pps
    // should like:
    // mime: string(video/avc), durationUs: int64(10000000), width: int32(480), height: int32(360), max-input-size: int32(55067), csd-0: data, csd-1: data}
    const char* fmt =  AMediaFormat_toString(format);
    LOG_INFO << "AMediaFormat_toString: " << fmt << LOG_ENDL;
    in->codec = AMediaCodec_createDecoderByType(config->ak_mime.c_str());
    // if no sureface android use software decoder
    // see: http://stackoverflow.com/questions/15500290/access-violation-in-native-code-with-hardware-accelerated-android-mediacodec-dec
    ANativeWindow* windowDecode = (ANativeWindow*)config->windowSurfaceDecode;
    if (AMediaCodec_configure(in->codec, format, windowDecode, NULL, 0) != AMEDIA_OK)
   {
      AMediaFormat_delete(format);
        LOG_ERROR << "AMediaCodec_configure error" << LOG_ENDL;
      return false;
   }
    if (AMediaCodec_start(in->codec) != AMEDIA_OK)
   {
      AMediaFormat_delete(format);
        LOG_ERROR << "AMediaCodec_start error" << LOG_ENDL;
      return false;
   }
    AMediaFormat_delete(format);
   return true;
}
@@ -177,8 +232,44 @@
   AMediaCodec_stop(in->codec);
   AMediaCodec_delete(in->codec);
   in->codec = nullptr;
   in->reset();
}
void PL_AndroidMediaCodecDecoder::aux_is_set()
{
   // support only in android-26 (8.0)
#if 0
   PL_AMCD_Internal* in = (PL_AMCD_Internal*)internal;
   if (!in->auxIsSet)
   {
      std::string base64_sps = this->manager->get_param(PLGP_DEC_SPS_B64);
      std::string base64_pps = this->manager->get_param(PLGP_DEC_PPS_B64);
      if((!base64_sps.empty()) && (!base64_pps.empty()))
      {
         size_t result_sps = 0;
         size_t result_pps = 0;
         uint8_t* uc_sps = base64_decode(base64_sps.c_str(),base64_sps.length(),result_sps);
         uint8_t* uc_pps = base64_decode(base64_pps.c_str(),base64_pps.length(),result_pps);
         AMediaFormat* format = AMediaCodec_getOutputFormat(in->codec);
         AMediaFormat_setBuffer(format, "csd-0", uc_sps, result_sps); // sps
         AMediaFormat_setBuffer(format, "csd-1", uc_pps, result_pps); // pps
         AMediaCodec_setParameters(in->codec,format);
         AMediaFormat_delete(format);
         delete[] uc_sps;
         uc_sps = nullptr;
         delete[] uc_pps;
         uc_pps = nullptr;
         in->auxIsSet = true;
      }
   }
#endif
}
bool PL_AndroidMediaCodecDecoder::pay(const PipeMaterial& pm)
@@ -194,7 +285,20 @@
   
   if (pm.buffer == nullptr)
      return false;
   if (in->config.initCodecInPay && in->codec == nullptr)
   {
      if (!init_codec())
      {
         LOG_ERROR << "init_codec error" << std::endl;
         return false;
      }
   }
   else
   {
      aux_is_set();
   }
   MB_Frame* frame = (MB_Frame*)pm.buffer;
   if (frame->type != MB_Frame::MBFT_H264_NALU)
   {
@@ -202,25 +306,25 @@
      return false;
   }
   static FILE *pFile = fopen("/data/bb.264", "wb");
   fwrite(frame->buffer, sizeof(char), frame->buffSize, pFile);
   fflush(pFile);
//   static FILE *pFile = fopen("/data/bb.264", "wb");
//   fwrite(frame->buffer, sizeof(char), frame->buffSize, pFile);
//   fflush(pFile);
    ssize_t bufidx = AMediaCodec_dequeueInputBuffer(in->codec, 2000);
    LOGP(DEBUG, "input buffer bufidx=%zd, inputFrameCount=%d", bufidx, in->inputFrameCount++);
   ssize_t bufidx = AMediaCodec_dequeueInputBuffer(in->codec, 2000);
   LOGP(DEBUG, "input buffer bufidx=%zd, inputFrameCount=%d", bufidx, in->inputFrameCount++);
    if (bufidx >= 0)
   if (bufidx >= 0)
   {
        size_t bufsize;
        uint8_t* inputBuff = AMediaCodec_getInputBuffer(in->codec, bufidx, &bufsize);
        size_t sampleSize = std::min(bufsize, frame->buffSize);
        memcpy(inputBuff, frame->buffer, sampleSize); // fill buffer
      size_t bufsize;
      uint8_t* inputBuff = AMediaCodec_getInputBuffer(in->codec, bufidx, &bufsize);
      size_t sampleSize = std::min(bufsize, frame->buffSize);
      memcpy(inputBuff, frame->buffer, sampleSize); // fill buffer
        uint64_t presentationTimeUs = timeval_to_microseconds(frame->pts); //microseconds
      uint64_t presentationTimeUs = timeval_to_microseconds(frame->pts); //microseconds
        media_status_t ms = AMediaCodec_queueInputBuffer(in->codec, bufidx, 0, sampleSize, presentationTimeUs, 0);
        LOGP(DEBUG, "media_status_t=%d", ms);
    }
      media_status_t ms = AMediaCodec_queueInputBuffer(in->codec, bufidx, 0, sampleSize, presentationTimeUs, 0);
      LOGP(DEBUG, "media_status_t=%d", ms);
   }
   else
   {
      LOG_WARN << "bufidx=" << bufidx << LOG_ENDL;
@@ -251,13 +355,13 @@
bool PL_AndroidMediaCodecDecoder::gain(PipeMaterial& pm)
{
   PL_AMCD_Internal* in = (PL_AMCD_Internal*)internal;
    //PipeLineElemTimingDebugger td(this); // RK3288, 51~16000@1920, MAX=26000us
   //PipeLineElemTimingDebugger td(this); // RK3288, 51~16000@1920, MAX=26000us
   AMediaCodecBufferInfo info;
   in->lastOutputBuffIdx = AMediaCodec_dequeueOutputBuffer(in->codec, &info, 0);
    rended = false;
   rended = false;
    if (in->lastOutputBuffIdx >= 0)
   if (in->lastOutputBuffIdx >= 0)
   {
      if (info.flags & AMEDIACODEC_BUFFER_FLAG_END_OF_STREAM)
      {
@@ -272,21 +376,21 @@
      //   usleep(delay / 1000);
      //}
        AMediaFormat* format = AMediaCodec_getOutputFormat(in->codec);
        int32_t width = in->config.ak_width;
        int32_t height = in->config.ak_height;
        int32_t color = 0;
      AMediaFormat* format = AMediaCodec_getOutputFormat(in->codec);
      int32_t width = in->config.ak_width;
      int32_t height = in->config.ak_height;
      int32_t color = 0;
        if (format != NULL)
        {
            AMediaFormat_getInt32(format, AMEDIAFORMAT_KEY_WIDTH, &width);
            AMediaFormat_getInt32(format, AMEDIAFORMAT_KEY_HEIGHT, &height);
            AMediaFormat_getInt32(format, AMEDIAFORMAT_KEY_COLOR_FORMAT, &color);
            AMediaFormat_delete(format);
            format = nullptr;
      if (format != NULL)
      {
         AMediaFormat_getInt32(format, AMEDIAFORMAT_KEY_WIDTH, &width);
         AMediaFormat_getInt32(format, AMEDIAFORMAT_KEY_HEIGHT, &height);
         AMediaFormat_getInt32(format, AMEDIAFORMAT_KEY_COLOR_FORMAT, &color);
         AMediaFormat_delete(format);
         format = nullptr;
            LOGP(DEBUG, "output media format, w=%d, h=%d, c=%d", width, height, color);
        }
         LOGP(DEBUG, "output media format, w=%d, h=%d, c=%d", width, height, color);
      }
      in->lastMbfBuffIdx.reset();
      if (! in->config.releaseOutputBuffIdx)
@@ -302,60 +406,60 @@
      in->lastMbfBuffer.reset();
      if ((in->config.generateDecodedDataPerFrame != 0) && (in->inputFrameCount % in->config.generateDecodedDataPerFrame == 0))
      {
            size_t outSize = in->buffSizeMax;
            uint8_t* outputBuff = AMediaCodec_getOutputBuffer(in->codec, in->lastOutputBuffIdx, &outSize);
            if (outputBuff != nullptr)
            {
                in->buffSize = std::min((size_t) info.size, in->buffSizeMax);
                memcpy(in->buffer, outputBuff + info.offset, in->buffSize);
         size_t outSize = in->buffSizeMax;
         uint8_t* outputBuff = AMediaCodec_getOutputBuffer(in->codec, in->lastOutputBuffIdx, &outSize);
         if (outputBuff != nullptr)
         {
            in->buffSize = std::min((size_t) info.size, in->buffSizeMax);
            memcpy(in->buffer, outputBuff + info.offset, in->buffSize);
                /*
                if (in->buffSize <= 8 && in->config.windowSurfaceDecode != nullptr)
                {
                    ANativeWindow* window = (ANativeWindow*)(in->config.windowSurfaceDecode);
            /*
            if (in->buffSize <= 8 && in->config.windowSurfaceDecode != nullptr)
            {
               ANativeWindow* window = (ANativeWindow*)(in->config.windowSurfaceDecode);
                    ANativeWindow_Buffer wbuffer;
                    if (ANativeWindow_lock(window, &wbuffer, NULL) == 0)
                    {
                        size_t bitsSize = 0;
                        if (wbuffer.format == WINDOW_FORMAT_RGBA_8888 || wbuffer.format == WINDOW_FORMAT_RGBX_8888)
                            bitsSize = wbuffer.height * wbuffer.width * 4;
                        else if (wbuffer.format == WINDOW_FORMAT_RGB_565)
                            bitsSize = wbuffer.height * wbuffer.width * 2;
                        else
                            bitsSize = wbuffer.height * wbuffer.width;
               ANativeWindow_Buffer wbuffer;
               if (ANativeWindow_lock(window, &wbuffer, NULL) == 0)
               {
                  size_t bitsSize = 0;
                  if (wbuffer.format == WINDOW_FORMAT_RGBA_8888 || wbuffer.format == WINDOW_FORMAT_RGBX_8888)
                     bitsSize = wbuffer.height * wbuffer.width * 4;
                  else if (wbuffer.format == WINDOW_FORMAT_RGB_565)
                     bitsSize = wbuffer.height * wbuffer.width * 2;
                  else
                     bitsSize = wbuffer.height * wbuffer.width;
                        memcpy(in->buffer, wbuffer.bits,  bitsSize);//#test copy opposite
                        in->buffSize = bitsSize;
                  memcpy(in->buffer, wbuffer.bits,  bitsSize);//#test copy opposite
                  in->buffSize = bitsSize;
                        ANativeWindow_unlockAndPost(window);
                    }
                }
                 */
                  ANativeWindow_unlockAndPost(window);
               }
            }
             */
                in->lastMbfBuffer.type = MB_Frame::MBFT_YUV420;
            in->lastMbfBuffer.type = MB_Frame::MBFT_YUV420;
                if (color == AMEDIA_COLOR_FormatYUV420Planar || color == AMEDIA_COLOR_FormatYUV420PackedPlanar) // I420
                    in->lastMbfBuffer.type = MB_Frame::MBFT_YUV420;
                else if (color == AMEDIA_COLOR_FormatYUV420SemiPlanar || color == AMEDIA_COLOR_FormatYUV420PackedSemiPlanar) // NV12
                    in->lastMbfBuffer.type = MB_Frame::MBFT_NV12;
                else
                {
                    LOG_WARN << "color format not support" << LOG_ENDL;
                    return  false;
                }
            if (color == AMEDIA_COLOR_FormatYUV420Planar || color == AMEDIA_COLOR_FormatYUV420PackedPlanar) // I420
               in->lastMbfBuffer.type = MB_Frame::MBFT_YUV420;
            else if (color == AMEDIA_COLOR_FormatYUV420SemiPlanar || color == AMEDIA_COLOR_FormatYUV420PackedSemiPlanar) // NV12
               in->lastMbfBuffer.type = MB_Frame::MBFT_NV12;
            else
            {
               LOG_WARN << "color format not support" << LOG_ENDL;
               return  false;
            }
                in->lastMbfBuffer.buffer = in->buffer;
                in->lastMbfBuffer.buffSize = in->buffSize;
                in->lastMbfBuffer.width = in->config.ak_width;
                in->lastMbfBuffer.height = in->config.ak_height;
            in->lastMbfBuffer.buffer = in->buffer;
            in->lastMbfBuffer.buffSize = in->buffSize;
            in->lastMbfBuffer.width = in->config.ak_width;
            in->lastMbfBuffer.height = in->config.ak_height;
            if (in->config.ptsUseAbsoluteTime)
               gettimeofday(&(in->lastMbfBuffer.pts), nullptr);
            else
               microseconds_to_timeval(info.presentationTimeUs, in->lastMbfBuffer.pts);
                //if (in->lastMbfBuffer.buffSize > 10)
            //if (in->lastMbfBuffer.buffSize > 10)
            //{
            //   static size_t f = 0;
            //   char fname[50];
@@ -366,7 +470,7 @@
            //   fclose(pFile);
            //   if (f > 50) exit(0);
            //}
            }
         }
      }
      pm.type = PipeMaterial::PMT_FRAME_LIST;
@@ -379,7 +483,7 @@
      {
         AMediaCodec_releaseOutputBuffer(in->codec, in->lastOutputBuffIdx, info.size != 0);
         in->lastOutputBuffIdx = -1;
            rended = true;
         rended = true;
      }
      return true;