git.sesse.net Git - ffmpeg/blob - libavcodec/libstagefright.cpp

   1 /*
   2  * Interface to the Android Stagefright library for
   3  * H/W accelerated H.264 decoding
   4  *
   5  * Copyright (C) 2011 Mohamed Naufal
   6  * Copyright (C) 2011 Martin Storsjö
   7  *
   8  * This file is part of FFmpeg.
   9  *
  10  * FFmpeg is free software; you can redistribute it and/or
  11  * modify it under the terms of the GNU Lesser General Public
  12  * License as published by the Free Software Foundation; either
  13  * version 2.1 of the License, or (at your option) any later version.
  14  *
  15  * FFmpeg is distributed in the hope that it will be useful,
  16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18  * Lesser General Public License for more details.
  19  *
  20  * You should have received a copy of the GNU Lesser General Public
  21  * License along with FFmpeg; if not, write to the Free Software
  22  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  23  */
  24
  25 #include <binder/ProcessState.h>
  26 #include <media/stagefright/MetaData.h>
  27 #include <media/stagefright/MediaBufferGroup.h>
  28 #include <media/stagefright/MediaDebug.h>
  29 #include <media/stagefright/MediaDefs.h>
  30 #include <media/stagefright/OMXClient.h>
  31 #include <media/stagefright/OMXCodec.h>
  32 #include <utils/List.h>
  33 #include <new>
  34
  35 extern "C" {
  36 #include "avcodec.h"
  37 #include "libavutil/imgutils.h"
  38 }
  39
  40 #define OMX_QCOM_COLOR_FormatYVU420SemiPlanar 0x7FA30C00
  41
  42 using namespace android;
  43
  44 struct Frame {
  45     status_t status;
  46     size_t size;
  47     int64_t time;
  48     int key;
  49     uint8_t *buffer;
  50     MediaBuffer* mbuffer;
  51     int32_t w, h;
  52 };
  53
  54 class CustomSource;
  55
  56 struct StagefrightContext {
  57     AVCodecContext *avctx;
  58     AVBitStreamFilterContext *bsfc;
  59     uint8_t* orig_extradata;
  60     int orig_extradata_size;
  61     sp<MediaSource> *source;
  62     List<Frame*> *in_queue, *out_queue;
  63     pthread_mutex_t in_mutex, out_mutex;
  64     pthread_cond_t condition;
  65     pthread_t decode_thread_id;
  66
  67     Frame *end_frame;
  68     bool source_done;
  69     volatile sig_atomic_t thread_started, thread_exited, stop_decode;
  70
  71     AVFrame ret_frame;
  72
  73     uint8_t *dummy_buf;
  74     int dummy_bufsize;
  75
  76     OMXClient *client;
  77     sp<MediaSource> *decoder;
  78     const char *decoder_component;
  79 };
  80
  81 class CustomSource : public MediaSource {
  82 public:
  83     CustomSource(AVCodecContext *avctx, sp<MetaData> meta) {
  84         s = (StagefrightContext*)avctx->priv_data;
  85         source_meta = meta;
  86         frame_size  = (avctx->width * avctx->height * 3) / 2;
  87         buf_group.add_buffer(new MediaBuffer(frame_size));
  88     }
  89
  90     virtual sp<MetaData> getFormat() {
  91         return source_meta;
  92     }
  93
  94     virtual status_t start(MetaData *params) {
  95         return OK;
  96     }
  97
  98     virtual status_t stop() {
  99         return OK;
 100     }
 101
 102     virtual status_t read(MediaBuffer **buffer,
 103                           const MediaSource::ReadOptions *options) {
 104         Frame *frame;
 105         status_t ret;
 106
 107         if (s->thread_exited)
 108             return ERROR_END_OF_STREAM;
 109         pthread_mutex_lock(&s->in_mutex);
 110
 111         while (s->in_queue->empty())
 112             pthread_cond_wait(&s->condition, &s->in_mutex);
 113
 114         frame = *s->in_queue->begin();
 115         ret = frame->status;
 116
 117         if (ret == OK) {
 118             ret = buf_group.acquire_buffer(buffer);
 119             if (ret == OK) {
 120                 memcpy((*buffer)->data(), frame->buffer, frame->size);
 121                 (*buffer)->set_range(0, frame->size);
 122                 (*buffer)->meta_data()->clear();
 123                 (*buffer)->meta_data()->setInt32(kKeyIsSyncFrame,frame->key);
 124                 (*buffer)->meta_data()->setInt64(kKeyTime, frame->time);
 125             } else {
 126                 av_log(s->avctx, AV_LOG_ERROR, "Failed to acquire MediaBuffer\n");
 127             }
 128             av_freep(&frame->buffer);
 129         }
 130
 131         s->in_queue->erase(s->in_queue->begin());
 132         pthread_mutex_unlock(&s->in_mutex);
 133
 134         av_freep(&frame);
 135         return ret;
 136     }
 137
 138 private:
 139     MediaBufferGroup buf_group;
 140     sp<MetaData> source_meta;
 141     StagefrightContext *s;
 142     int frame_size;
 143 };
 144
 145 void* decode_thread(void *arg)
 146 {
 147     AVCodecContext *avctx = (AVCodecContext*)arg;
 148     StagefrightContext *s = (StagefrightContext*)avctx->priv_data;
 149     Frame* frame;
 150     MediaBuffer *buffer;
 151     int decode_done = 0;
 152     do {
 153         buffer = NULL;
 154         frame = (Frame*)av_mallocz(sizeof(Frame));
 155         if (!frame) {
 156             frame         = s->end_frame;
 157             frame->status = AVERROR(ENOMEM);
 158             decode_done   = 1;
 159             s->end_frame  = NULL;
 160         } else {
 161             frame->status = (*s->decoder)->read(&buffer);
 162             if (frame->status == OK) {
 163                 sp<MetaData> outFormat = (*s->decoder)->getFormat();
 164                 outFormat->findInt32(kKeyWidth , &frame->w);
 165                 outFormat->findInt32(kKeyHeight, &frame->h);
 166                 frame->size    = buffer->range_length();
 167                 frame->mbuffer = buffer;
 168             } else if (frame->status == INFO_FORMAT_CHANGED) {
 169                 if (buffer)
 170                     buffer->release();
 171                 av_free(frame);
 172                 continue;
 173             } else {
 174                 decode_done = 1;
 175             }
 176         }
 177         while (true) {
 178             pthread_mutex_lock(&s->out_mutex);
 179             if (s->out_queue->size() >= 10) {
 180                 pthread_mutex_unlock(&s->out_mutex);
 181                 usleep(10000);
 182                 continue;
 183             }
 184             break;
 185         }
 186         s->out_queue->push_back(frame);
 187         pthread_mutex_unlock(&s->out_mutex);
 188     } while (!decode_done && !s->stop_decode);
 189
 190     s->thread_exited = true;
 191
 192     return 0;
 193 }
 194
 195 static av_cold int Stagefright_init(AVCodecContext *avctx)
 196 {
 197     StagefrightContext *s = (StagefrightContext*)avctx->priv_data;
 198     sp<MetaData> meta, outFormat;
 199     int32_t colorFormat = 0;
 200     int ret;
 201
 202     if (!avctx->extradata || !avctx->extradata_size || avctx->extradata[0] != 1)
 203         return -1;
 204
 205     s->avctx = avctx;
 206     s->bsfc  = av_bitstream_filter_init("h264_mp4toannexb");
 207     if (!s->bsfc) {
 208         av_log(avctx, AV_LOG_ERROR, "Cannot open the h264_mp4toannexb BSF!\n");
 209         return -1;
 210     }
 211
 212     s->orig_extradata_size = avctx->extradata_size;
 213     s->orig_extradata = (uint8_t*) av_mallocz(avctx->extradata_size +
 214                                               FF_INPUT_BUFFER_PADDING_SIZE);
 215     if (!s->orig_extradata) {
 216         ret = AVERROR(ENOMEM);
 217         goto fail;
 218     }
 219     memcpy(s->orig_extradata, avctx->extradata, avctx->extradata_size);
 220
 221     meta = new MetaData;
 222     if (meta == NULL) {
 223         ret = AVERROR(ENOMEM);
 224         goto fail;
 225     }
 226     meta->setCString(kKeyMIMEType, MEDIA_MIMETYPE_VIDEO_AVC);
 227     meta->setInt32(kKeyWidth, avctx->width);
 228     meta->setInt32(kKeyHeight, avctx->height);
 229     meta->setData(kKeyAVCC, kTypeAVCC, avctx->extradata, avctx->extradata_size);
 230
 231     android::ProcessState::self()->startThreadPool();
 232
 233     s->source    = new sp<MediaSource>();
 234     *s->source   = new CustomSource(avctx, meta);
 235     s->in_queue  = new List<Frame*>;
 236     s->out_queue = new List<Frame*>;
 237     s->client    = new OMXClient;
 238     s->end_frame = (Frame*)av_mallocz(sizeof(Frame));
 239     if (s->source == NULL || !s->in_queue || !s->out_queue || !s->client ||
 240         !s->end_frame) {
 241         ret = AVERROR(ENOMEM);
 242         goto fail;
 243     }
 244
 245     if (s->client->connect() !=  OK) {
 246         av_log(avctx, AV_LOG_ERROR, "Cannot connect OMX client\n");
 247         ret = -1;
 248         goto fail;
 249     }
 250
 251     s->decoder  = new sp<MediaSource>();
 252     *s->decoder = OMXCodec::Create(s->client->interface(), meta,
 253                                   false, *s->source, NULL,
 254                                   OMXCodec::kClientNeedsFramebuffer);
 255     if ((*s->decoder)->start() !=  OK) {
 256         av_log(avctx, AV_LOG_ERROR, "Cannot start decoder\n");
 257         ret = -1;
 258         s->client->disconnect();
 259         goto fail;
 260     }
 261
 262     outFormat = (*s->decoder)->getFormat();
 263     outFormat->findInt32(kKeyColorFormat, &colorFormat);
 264     if (colorFormat == OMX_QCOM_COLOR_FormatYVU420SemiPlanar ||
 265         colorFormat == OMX_COLOR_FormatYUV420SemiPlanar)
 266         avctx->pix_fmt = PIX_FMT_NV21;
 267     else
 268         avctx->pix_fmt = PIX_FMT_YUV420P;
 269
 270     outFormat->findCString(kKeyDecoderComponent, &s->decoder_component);
 271     if (s->decoder_component)
 272         s->decoder_component = av_strdup(s->decoder_component);
 273
 274     pthread_mutex_init(&s->in_mutex, NULL);
 275     pthread_mutex_init(&s->out_mutex, NULL);
 276     pthread_cond_init(&s->condition, NULL);
 277     return 0;
 278
 279 fail:
 280     av_bitstream_filter_close(s->bsfc);
 281     av_freep(&s->orig_extradata);
 282     av_freep(&s->end_frame);
 283     delete s->in_queue;
 284     delete s->out_queue;
 285     delete s->client;
 286     return ret;
 287 }
 288
 289 static int Stagefright_decode_frame(AVCodecContext *avctx, void *data,
 290                                     int *data_size, AVPacket *avpkt)
 291 {
 292     StagefrightContext *s = (StagefrightContext*)avctx->priv_data;
 293     Frame *frame;
 294     MediaBuffer *mbuffer;
 295     status_t status;
 296     size_t size;
 297     uint8_t *buf;
 298     const uint8_t *src_data[3];
 299     int w, h;
 300     int src_linesize[3];
 301     int orig_size = avpkt->size;
 302     AVPacket pkt = *avpkt;
 303     int ret;
 304
 305     if (!s->thread_started) {
 306         pthread_create(&s->decode_thread_id, NULL, &decode_thread, avctx);
 307         s->thread_started = true;
 308     }
 309
 310     if (avpkt && avpkt->data) {
 311         av_bitstream_filter_filter(s->bsfc, avctx, NULL, &pkt.data, &pkt.size,
 312                                    avpkt->data, avpkt->size, avpkt->flags & AV_PKT_FLAG_KEY);
 313         avpkt = &pkt;
 314     }
 315
 316     if (!s->source_done) {
 317         if(!s->dummy_buf) {
 318             s->dummy_buf = (uint8_t*)av_malloc(avpkt->size);
 319             if (!s->dummy_buf)
 320                 return AVERROR(ENOMEM);
 321             s->dummy_bufsize = avpkt->size;
 322             memcpy(s->dummy_buf, avpkt->data, avpkt->size);
 323         }
 324
 325         frame = (Frame*)av_mallocz(sizeof(Frame));
 326         if (avpkt->data) {
 327             frame->status  = OK;
 328             frame->size    = avpkt->size;
 329             // Stagefright can't handle negative timestamps -
 330             // if needed, work around this by offsetting them manually?
 331             if (avpkt->pts >= 0)
 332                 frame->time    = avpkt->pts;
 333             frame->key     = avpkt->flags & AV_PKT_FLAG_KEY ? 1 : 0;
 334             frame->buffer  = (uint8_t*)av_malloc(avpkt->size);
 335             if (!frame->buffer) {
 336                 av_freep(&frame);
 337                 return AVERROR(ENOMEM);
 338             }
 339             uint8_t *ptr = avpkt->data;
 340             // The OMX.SEC decoder fails without this.
 341             if (avpkt->size == orig_size + avctx->extradata_size) {
 342                 ptr += avctx->extradata_size;
 343                 frame->size = orig_size;
 344             }
 345             memcpy(frame->buffer, ptr, orig_size);
 346         } else {
 347             frame->status  = ERROR_END_OF_STREAM;
 348             s->source_done = true;
 349         }
 350
 351         while (true) {
 352             if (s->thread_exited) {
 353                 s->source_done = true;
 354                 break;
 355             }
 356             pthread_mutex_lock(&s->in_mutex);
 357             if (s->in_queue->size() >= 10) {
 358                 pthread_mutex_unlock(&s->in_mutex);
 359                 usleep(10000);
 360                 continue;
 361             }
 362             s->in_queue->push_back(frame);
 363             pthread_cond_signal(&s->condition);
 364             pthread_mutex_unlock(&s->in_mutex);
 365             break;
 366         }
 367     }
 368     while (true) {
 369         pthread_mutex_lock(&s->out_mutex);
 370         if (!s->out_queue->empty()) break;
 371         pthread_mutex_unlock(&s->out_mutex);
 372         if (s->source_done) {
 373             usleep(10000);
 374             continue;
 375         } else {
 376             return orig_size;
 377         }
 378     }
 379
 380     frame = *s->out_queue->begin();
 381     s->out_queue->erase(s->out_queue->begin());
 382     pthread_mutex_unlock(&s->out_mutex);
 383
 384     mbuffer = frame->mbuffer;
 385     status  = frame->status;
 386     size    = frame->size;
 387     w       = frame->w;
 388     h       = frame->h;
 389     av_freep(&frame);
 390
 391     if (status == ERROR_END_OF_STREAM)
 392         return 0;
 393     if (status != OK) {
 394         if (status == AVERROR(ENOMEM))
 395             return status;
 396         av_log(avctx, AV_LOG_ERROR, "Decode failed: %x\n", status);
 397         return -1;
 398     }
 399
 400     // The OMX.SEC decoder doesn't signal the modified width/height
 401     if (s->decoder_component && !strncmp(s->decoder_component, "OMX.SEC", 7) &&
 402         (w & 15 || h & 15)) {
 403         if (((w + 15)&~15) * ((h + 15)&~15) * 3/2 == size) {
 404             w = (w + 15)&~15;
 405             h = (h + 15)&~15;
 406         }
 407     }
 408
 409     if (!avctx->width || !avctx->height || avctx->width > w || avctx->height > h) {
 410         avctx->width  = w;
 411         avctx->height = h;
 412     }
 413
 414     ret = avctx->reget_buffer(avctx, &s->ret_frame);
 415     if (ret < 0) {
 416         av_log(avctx, AV_LOG_ERROR, "reget buffer() failed\n");
 417         goto end;
 418     }
 419
 420     src_linesize[0] = w;
 421     if (avctx->pix_fmt == PIX_FMT_YUV420P)
 422         src_linesize[1] = src_linesize[2] = w/2;
 423     else if (avctx->pix_fmt == PIX_FMT_NV21)
 424         src_linesize[1] = w;
 425
 426     buf = (uint8_t*)mbuffer->data();
 427     src_data[0] = buf;
 428     src_data[1] = buf + src_linesize[0] * h;
 429     src_data[2] = src_data[1] + src_linesize[1] * h/2;
 430     av_image_copy(s->ret_frame.data, s->ret_frame.linesize,
 431                   src_data, src_linesize,
 432                   avctx->pix_fmt, avctx->width, avctx->height);
 433
 434     *data_size = sizeof(AVFrame);
 435     *(AVFrame*)data = s->ret_frame;
 436     ret = orig_size;
 437 end:
 438     mbuffer->release();
 439     return ret;
 440 }
 441
 442 static av_cold int Stagefright_close(AVCodecContext *avctx)
 443 {
 444     StagefrightContext *s = (StagefrightContext*)avctx->priv_data;
 445     Frame *frame;
 446
 447     if (s->thread_started) {
 448         if (!s->thread_exited) {
 449             s->stop_decode = 1;
 450
 451             // Make sure decode_thread() doesn't get stuck
 452             pthread_mutex_lock(&s->out_mutex);
 453             while (!s->out_queue->empty()) {
 454                 frame = *s->out_queue->begin();
 455                 s->out_queue->erase(s->out_queue->begin());
 456                 if (frame->size)
 457                     frame->mbuffer->release();
 458                 av_freep(&frame);
 459             }
 460             pthread_mutex_unlock(&s->out_mutex);
 461
 462             // Feed a dummy frame prior to signalling EOF.
 463             // This is required to terminate the decoder(OMX.SEC)
 464             // when only one frame is read during stream info detection.
 465             if (s->dummy_buf && (frame = (Frame*)av_mallocz(sizeof(Frame)))) {
 466                 frame->status = OK;
 467                 frame->size   = s->dummy_bufsize;
 468                 frame->key    = 1;
 469                 frame->buffer = s->dummy_buf;
 470                 pthread_mutex_lock(&s->in_mutex);
 471                 s->in_queue->push_back(frame);
 472                 pthread_cond_signal(&s->condition);
 473                 pthread_mutex_unlock(&s->in_mutex);
 474                 s->dummy_buf = NULL;
 475             }
 476
 477             pthread_mutex_lock(&s->in_mutex);
 478             s->end_frame->status = ERROR_END_OF_STREAM;
 479             s->in_queue->push_back(s->end_frame);
 480             pthread_cond_signal(&s->condition);
 481             pthread_mutex_unlock(&s->in_mutex);
 482             s->end_frame = NULL;
 483         }
 484
 485         pthread_join(s->decode_thread_id, NULL);
 486
 487         if (s->ret_frame.data[0])
 488             avctx->release_buffer(avctx, &s->ret_frame);
 489
 490         s->thread_started = false;
 491     }
 492
 493     while (!s->in_queue->empty()) {
 494         frame = *s->in_queue->begin();
 495         s->in_queue->erase(s->in_queue->begin());
 496         if (frame->size)
 497             av_freep(&frame->buffer);
 498         av_freep(&frame);
 499     }
 500
 501     while (!s->out_queue->empty()) {
 502         frame = *s->out_queue->begin();
 503         s->out_queue->erase(s->out_queue->begin());
 504         if (frame->size)
 505             frame->mbuffer->release();
 506         av_freep(&frame);
 507     }
 508
 509     (*s->decoder)->stop();
 510     s->client->disconnect();
 511
 512     if (s->decoder_component)
 513         av_freep(&s->decoder_component);
 514     av_freep(&s->dummy_buf);
 515     av_freep(&s->end_frame);
 516
 517     // Reset the extradata back to the original mp4 format, so that
 518     // the next invocation (both when decoding and when called from
 519     // av_find_stream_info) get the original mp4 format extradata.
 520     av_freep(&avctx->extradata);
 521     avctx->extradata = s->orig_extradata;
 522     avctx->extradata_size = s->orig_extradata_size;
 523
 524     delete s->in_queue;
 525     delete s->out_queue;
 526     delete s->client;
 527     delete s->decoder;
 528     delete s->source;
 529
 530     pthread_mutex_destroy(&s->in_mutex);
 531     pthread_mutex_destroy(&s->out_mutex);
 532     pthread_cond_destroy(&s->condition);
 533     av_bitstream_filter_close(s->bsfc);
 534     return 0;
 535 }
 536
 537 AVCodec ff_libstagefright_h264_decoder = {
 538     "libstagefright_h264",
 539     AVMEDIA_TYPE_VIDEO,
 540     CODEC_ID_H264,
 541     sizeof(StagefrightContext),
 542     Stagefright_init,
 543     NULL, //encode
 544     Stagefright_close,
 545     Stagefright_decode_frame,
 546     CODEC_CAP_DELAY,
 547     NULL, //next
 548     NULL, //flush
 549     NULL, //supported_framerates
 550     NULL, //pixel_formats
 551     NULL_IF_CONFIG_SMALL("libstagefright H.264"),
 552 };