git.sesse.net Git - ffmpeg/blob - libavcodec/libstagefright.cpp

   1 /*
   2  * Interface to the Android Stagefright library for
   3  * H/W accelerated H.264 decoding
   4  *
   5  * Copyright (C) 2011 Mohamed Naufal
   6  * Copyright (C) 2011 Martin Storsjö
   7  *
   8  * This file is part of FFmpeg.
   9  *
  10  * FFmpeg is free software; you can redistribute it and/or
  11  * modify it under the terms of the GNU Lesser General Public
  12  * License as published by the Free Software Foundation; either
  13  * version 2.1 of the License, or (at your option) any later version.
  14  *
  15  * FFmpeg is distributed in the hope that it will be useful,
  16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18  * Lesser General Public License for more details.
  19  *
  20  * You should have received a copy of the GNU Lesser General Public
  21  * License along with FFmpeg; if not, write to the Free Software
  22  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  23  */
  24
  25 #include <binder/ProcessState.h>
  26 #include <media/stagefright/MetaData.h>
  27 #include <media/stagefright/MediaBufferGroup.h>
  28 #include <media/stagefright/MediaDebug.h>
  29 #include <media/stagefright/MediaDefs.h>
  30 #include <media/stagefright/OMXClient.h>
  31 #include <media/stagefright/OMXCodec.h>
  32 #include <utils/List.h>
  33 #include <new>
  34 #include <map>
  35
  36 extern "C" {
  37 #include "avcodec.h"
  38 #include "libavutil/imgutils.h"
  39 }
  40
  41 #define OMX_QCOM_COLOR_FormatYVU420SemiPlanar 0x7FA30C00
  42
  43 using namespace android;
  44
  45 struct Frame {
  46     status_t status;
  47     size_t size;
  48     int64_t time;
  49     int key;
  50     uint8_t *buffer;
  51     AVFrame *vframe;
  52 };
  53
  54 struct TimeStamp {
  55     int64_t pts;
  56     int64_t reordered_opaque;
  57 };
  58
  59 class CustomSource;
  60
  61 struct StagefrightContext {
  62     AVCodecContext *avctx;
  63     AVBitStreamFilterContext *bsfc;
  64     uint8_t* orig_extradata;
  65     int orig_extradata_size;
  66     sp<MediaSource> *source;
  67     List<Frame*> *in_queue, *out_queue;
  68     pthread_mutex_t in_mutex, out_mutex;
  69     pthread_cond_t condition;
  70     pthread_t decode_thread_id;
  71
  72     Frame *end_frame;
  73     bool source_done;
  74     volatile sig_atomic_t thread_started, thread_exited, stop_decode;
  75
  76     AVFrame *prev_frame;
  77     std::map<int64_t, TimeStamp> *ts_map;
  78     int64_t frame_index;
  79
  80     uint8_t *dummy_buf;
  81     int dummy_bufsize;
  82
  83     OMXClient *client;
  84     sp<MediaSource> *decoder;
  85     const char *decoder_component;
  86 };
  87
  88 class CustomSource : public MediaSource {
  89 public:
  90     CustomSource(AVCodecContext *avctx, sp<MetaData> meta) {
  91         s = (StagefrightContext*)avctx->priv_data;
  92         source_meta = meta;
  93         frame_size  = (avctx->width * avctx->height * 3) / 2;
  94         buf_group.add_buffer(new MediaBuffer(frame_size));
  95     }
  96
  97     virtual sp<MetaData> getFormat() {
  98         return source_meta;
  99     }
 100
 101     virtual status_t start(MetaData *params) {
 102         return OK;
 103     }
 104
 105     virtual status_t stop() {
 106         return OK;
 107     }
 108
 109     virtual status_t read(MediaBuffer **buffer,
 110                           const MediaSource::ReadOptions *options) {
 111         Frame *frame;
 112         status_t ret;
 113
 114         if (s->thread_exited)
 115             return ERROR_END_OF_STREAM;
 116         pthread_mutex_lock(&s->in_mutex);
 117
 118         while (s->in_queue->empty())
 119             pthread_cond_wait(&s->condition, &s->in_mutex);
 120
 121         frame = *s->in_queue->begin();
 122         ret = frame->status;
 123
 124         if (ret == OK) {
 125             ret = buf_group.acquire_buffer(buffer);
 126             if (ret == OK) {
 127                 memcpy((*buffer)->data(), frame->buffer, frame->size);
 128                 (*buffer)->set_range(0, frame->size);
 129                 (*buffer)->meta_data()->clear();
 130                 (*buffer)->meta_data()->setInt32(kKeyIsSyncFrame,frame->key);
 131                 (*buffer)->meta_data()->setInt64(kKeyTime, frame->time);
 132             } else {
 133                 av_log(s->avctx, AV_LOG_ERROR, "Failed to acquire MediaBuffer\n");
 134             }
 135             av_freep(&frame->buffer);
 136         }
 137
 138         s->in_queue->erase(s->in_queue->begin());
 139         pthread_mutex_unlock(&s->in_mutex);
 140
 141         av_freep(&frame);
 142         return ret;
 143     }
 144
 145 private:
 146     MediaBufferGroup buf_group;
 147     sp<MetaData> source_meta;
 148     StagefrightContext *s;
 149     int frame_size;
 150 };
 151
 152 void* decode_thread(void *arg)
 153 {
 154     AVCodecContext *avctx = (AVCodecContext*)arg;
 155     StagefrightContext *s = (StagefrightContext*)avctx->priv_data;
 156     const AVPixFmtDescriptor *pix_desc = av_pix_fmt_desc_get(avctx->pix_fmt);
 157     Frame* frame;
 158     MediaBuffer *buffer;
 159     int32_t w, h;
 160     int decode_done = 0;
 161     int ret;
 162     int src_linesize[3];
 163     const uint8_t *src_data[3];
 164     int64_t out_frame_index = 0;
 165
 166     do {
 167         buffer = NULL;
 168         frame = (Frame*)av_mallocz(sizeof(Frame));
 169         if (!frame) {
 170             frame         = s->end_frame;
 171             frame->status = AVERROR(ENOMEM);
 172             decode_done   = 1;
 173             s->end_frame  = NULL;
 174             goto push_frame;
 175         }
 176         frame->status = (*s->decoder)->read(&buffer);
 177         if (frame->status == OK) {
 178             sp<MetaData> outFormat = (*s->decoder)->getFormat();
 179             outFormat->findInt32(kKeyWidth , &w);
 180             outFormat->findInt32(kKeyHeight, &h);
 181             frame->vframe = (AVFrame*)av_mallocz(sizeof(AVFrame));
 182             if (!frame->vframe) {
 183                 frame->status = AVERROR(ENOMEM);
 184                 decode_done   = 1;
 185                 buffer->release();
 186                 goto push_frame;
 187             }
 188             ret = ff_get_buffer(avctx, frame->vframe);
 189             if (ret < 0) {
 190                 frame->status = ret;
 191                 decode_done   = 1;
 192                 buffer->release();
 193                 goto push_frame;
 194             }
 195
 196             // The OMX.SEC decoder doesn't signal the modified width/height
 197             if (s->decoder_component && !strncmp(s->decoder_component, "OMX.SEC", 7) &&
 198                 (w & 15 || h & 15)) {
 199                 if (((w + 15)&~15) * ((h + 15)&~15) * 3/2 == buffer->range_length()) {
 200                     w = (w + 15)&~15;
 201                     h = (h + 15)&~15;
 202                 }
 203             }
 204
 205             if (!avctx->width || !avctx->height || avctx->width > w || avctx->height > h) {
 206                 avctx->width  = w;
 207                 avctx->height = h;
 208             }
 209
 210             src_linesize[0] = av_image_get_linesize(avctx->pix_fmt, w, 0);
 211             src_linesize[1] = av_image_get_linesize(avctx->pix_fmt, w, 1);
 212             src_linesize[2] = av_image_get_linesize(avctx->pix_fmt, w, 2);
 213
 214             src_data[0] = (uint8_t*)buffer->data();
 215             src_data[1] = src_data[0] + src_linesize[0] * h;
 216             src_data[2] = src_data[1] + src_linesize[1] * -(-h>>pix_desc->log2_chroma_h);
 217             av_image_copy(frame->vframe->data, frame->vframe->linesize,
 218                           src_data, src_linesize,
 219                           avctx->pix_fmt, avctx->width, avctx->height);
 220
 221             buffer->meta_data()->findInt64(kKeyTime, &out_frame_index);
 222             if (out_frame_index && s->ts_map->count(out_frame_index) > 0) {
 223                 frame->vframe->pts = (*s->ts_map)[out_frame_index].pts;
 224                 frame->vframe->reordered_opaque = (*s->ts_map)[out_frame_index].reordered_opaque;
 225                 s->ts_map->erase(out_frame_index);
 226             }
 227             buffer->release();
 228             } else if (frame->status == INFO_FORMAT_CHANGED) {
 229                 if (buffer)
 230                     buffer->release();
 231                 av_free(frame);
 232                 continue;
 233             } else {
 234                 decode_done = 1;
 235             }
 236 push_frame:
 237         while (true) {
 238             pthread_mutex_lock(&s->out_mutex);
 239             if (s->out_queue->size() >= 10) {
 240                 pthread_mutex_unlock(&s->out_mutex);
 241                 usleep(10000);
 242                 continue;
 243             }
 244             break;
 245         }
 246         s->out_queue->push_back(frame);
 247         pthread_mutex_unlock(&s->out_mutex);
 248     } while (!decode_done && !s->stop_decode);
 249
 250     s->thread_exited = true;
 251
 252     return 0;
 253 }
 254
 255 static av_cold int Stagefright_init(AVCodecContext *avctx)
 256 {
 257     StagefrightContext *s = (StagefrightContext*)avctx->priv_data;
 258     sp<MetaData> meta, outFormat;
 259     int32_t colorFormat = 0;
 260     int ret;
 261
 262     if (!avctx->extradata || !avctx->extradata_size || avctx->extradata[0] != 1)
 263         return -1;
 264
 265     s->avctx = avctx;
 266     s->bsfc  = av_bitstream_filter_init("h264_mp4toannexb");
 267     if (!s->bsfc) {
 268         av_log(avctx, AV_LOG_ERROR, "Cannot open the h264_mp4toannexb BSF!\n");
 269         return -1;
 270     }
 271
 272     s->orig_extradata_size = avctx->extradata_size;
 273     s->orig_extradata = (uint8_t*) av_mallocz(avctx->extradata_size +
 274                                               FF_INPUT_BUFFER_PADDING_SIZE);
 275     if (!s->orig_extradata) {
 276         ret = AVERROR(ENOMEM);
 277         goto fail;
 278     }
 279     memcpy(s->orig_extradata, avctx->extradata, avctx->extradata_size);
 280
 281     meta = new MetaData;
 282     if (meta == NULL) {
 283         ret = AVERROR(ENOMEM);
 284         goto fail;
 285     }
 286     meta->setCString(kKeyMIMEType, MEDIA_MIMETYPE_VIDEO_AVC);
 287     meta->setInt32(kKeyWidth, avctx->width);
 288     meta->setInt32(kKeyHeight, avctx->height);
 289     meta->setData(kKeyAVCC, kTypeAVCC, avctx->extradata, avctx->extradata_size);
 290
 291     android::ProcessState::self()->startThreadPool();
 292
 293     s->source    = new sp<MediaSource>();
 294     *s->source   = new CustomSource(avctx, meta);
 295     s->in_queue  = new List<Frame*>;
 296     s->out_queue = new List<Frame*>;
 297     s->ts_map    = new std::map<int64_t, TimeStamp>;
 298     s->client    = new OMXClient;
 299     s->end_frame = (Frame*)av_mallocz(sizeof(Frame));
 300     if (s->source == NULL || !s->in_queue || !s->out_queue || !s->client ||
 301         !s->ts_map || !s->end_frame) {
 302         ret = AVERROR(ENOMEM);
 303         goto fail;
 304     }
 305
 306     if (s->client->connect() !=  OK) {
 307         av_log(avctx, AV_LOG_ERROR, "Cannot connect OMX client\n");
 308         ret = -1;
 309         goto fail;
 310     }
 311
 312     s->decoder  = new sp<MediaSource>();
 313     *s->decoder = OMXCodec::Create(s->client->interface(), meta,
 314                                   false, *s->source, NULL,
 315                                   OMXCodec::kClientNeedsFramebuffer);
 316     if ((*s->decoder)->start() !=  OK) {
 317         av_log(avctx, AV_LOG_ERROR, "Cannot start decoder\n");
 318         ret = -1;
 319         s->client->disconnect();
 320         goto fail;
 321     }
 322
 323     outFormat = (*s->decoder)->getFormat();
 324     outFormat->findInt32(kKeyColorFormat, &colorFormat);
 325     if (colorFormat == OMX_QCOM_COLOR_FormatYVU420SemiPlanar ||
 326         colorFormat == OMX_COLOR_FormatYUV420SemiPlanar)
 327         avctx->pix_fmt = AV_PIX_FMT_NV21;
 328     else if (colorFormat == OMX_COLOR_FormatYCbYCr)
 329         avctx->pix_fmt = AV_PIX_FMT_YUYV422;
 330     else if (colorFormat == OMX_COLOR_FormatCbYCrY)
 331         avctx->pix_fmt = AV_PIX_FMT_UYVY422;
 332     else
 333         avctx->pix_fmt = AV_PIX_FMT_YUV420P;
 334
 335     outFormat->findCString(kKeyDecoderComponent, &s->decoder_component);
 336     if (s->decoder_component)
 337         s->decoder_component = av_strdup(s->decoder_component);
 338
 339     pthread_mutex_init(&s->in_mutex, NULL);
 340     pthread_mutex_init(&s->out_mutex, NULL);
 341     pthread_cond_init(&s->condition, NULL);
 342     return 0;
 343
 344 fail:
 345     av_bitstream_filter_close(s->bsfc);
 346     av_freep(&s->orig_extradata);
 347     av_freep(&s->end_frame);
 348     delete s->in_queue;
 349     delete s->out_queue;
 350     delete s->ts_map;
 351     delete s->client;
 352     return ret;
 353 }
 354
 355 static int Stagefright_decode_frame(AVCodecContext *avctx, void *data,
 356                                     int *got_frame, AVPacket *avpkt)
 357 {
 358     StagefrightContext *s = (StagefrightContext*)avctx->priv_data;
 359     Frame *frame;
 360     status_t status;
 361     int orig_size = avpkt->size;
 362     AVPacket pkt = *avpkt;
 363     AVFrame *ret_frame;
 364
 365     if (!s->thread_started) {
 366         pthread_create(&s->decode_thread_id, NULL, &decode_thread, avctx);
 367         s->thread_started = true;
 368     }
 369
 370     if (avpkt && avpkt->data) {
 371         av_bitstream_filter_filter(s->bsfc, avctx, NULL, &pkt.data, &pkt.size,
 372                                    avpkt->data, avpkt->size, avpkt->flags & AV_PKT_FLAG_KEY);
 373         avpkt = &pkt;
 374     }
 375
 376     if (!s->source_done) {
 377         if(!s->dummy_buf) {
 378             s->dummy_buf = (uint8_t*)av_malloc(avpkt->size);
 379             if (!s->dummy_buf)
 380                 return AVERROR(ENOMEM);
 381             s->dummy_bufsize = avpkt->size;
 382             memcpy(s->dummy_buf, avpkt->data, avpkt->size);
 383         }
 384
 385         frame = (Frame*)av_mallocz(sizeof(Frame));
 386         if (avpkt->data) {
 387             frame->status  = OK;
 388             frame->size    = avpkt->size;
 389             frame->key     = avpkt->flags & AV_PKT_FLAG_KEY ? 1 : 0;
 390             frame->buffer  = (uint8_t*)av_malloc(avpkt->size);
 391             if (!frame->buffer) {
 392                 av_freep(&frame);
 393                 return AVERROR(ENOMEM);
 394             }
 395             uint8_t *ptr = avpkt->data;
 396             // The OMX.SEC decoder fails without this.
 397             if (avpkt->size == orig_size + avctx->extradata_size) {
 398                 ptr += avctx->extradata_size;
 399                 frame->size = orig_size;
 400             }
 401             memcpy(frame->buffer, ptr, orig_size);
 402             if (avpkt == &pkt)
 403                 av_free(avpkt->data);
 404
 405             frame->time = ++s->frame_index;
 406             (*s->ts_map)[s->frame_index].pts = avpkt->pts;
 407             (*s->ts_map)[s->frame_index].reordered_opaque = avctx->reordered_opaque;
 408         } else {
 409             frame->status  = ERROR_END_OF_STREAM;
 410             s->source_done = true;
 411         }
 412
 413         while (true) {
 414             if (s->thread_exited) {
 415                 s->source_done = true;
 416                 break;
 417             }
 418             pthread_mutex_lock(&s->in_mutex);
 419             if (s->in_queue->size() >= 10) {
 420                 pthread_mutex_unlock(&s->in_mutex);
 421                 usleep(10000);
 422                 continue;
 423             }
 424             s->in_queue->push_back(frame);
 425             pthread_cond_signal(&s->condition);
 426             pthread_mutex_unlock(&s->in_mutex);
 427             break;
 428         }
 429     }
 430     while (true) {
 431         pthread_mutex_lock(&s->out_mutex);
 432         if (!s->out_queue->empty()) break;
 433         pthread_mutex_unlock(&s->out_mutex);
 434         if (s->source_done) {
 435             usleep(10000);
 436             continue;
 437         } else {
 438             return orig_size;
 439         }
 440     }
 441
 442     frame = *s->out_queue->begin();
 443     s->out_queue->erase(s->out_queue->begin());
 444     pthread_mutex_unlock(&s->out_mutex);
 445
 446     ret_frame = frame->vframe;
 447     status  = frame->status;
 448     av_freep(&frame);
 449
 450     if (status == ERROR_END_OF_STREAM)
 451         return 0;
 452     if (status != OK) {
 453         if (status == AVERROR(ENOMEM))
 454             return status;
 455         av_log(avctx, AV_LOG_ERROR, "Decode failed: %x\n", status);
 456         return -1;
 457     }
 458
 459     if (s->prev_frame) {
 460         avctx->release_buffer(avctx, s->prev_frame);
 461         av_freep(&s->prev_frame);
 462     }
 463     s->prev_frame = ret_frame;
 464
 465     *got_frame = 1;
 466     *(AVFrame*)data = *ret_frame;
 467     return orig_size;
 468 }
 469
 470 static av_cold int Stagefright_close(AVCodecContext *avctx)
 471 {
 472     StagefrightContext *s = (StagefrightContext*)avctx->priv_data;
 473     Frame *frame;
 474
 475     if (s->thread_started) {
 476         if (!s->thread_exited) {
 477             s->stop_decode = 1;
 478
 479             // Make sure decode_thread() doesn't get stuck
 480             pthread_mutex_lock(&s->out_mutex);
 481             while (!s->out_queue->empty()) {
 482                 frame = *s->out_queue->begin();
 483                 s->out_queue->erase(s->out_queue->begin());
 484                 if (frame->vframe) {
 485                     avctx->release_buffer(avctx, frame->vframe);
 486                     av_freep(&frame->vframe);
 487                 }
 488                 av_freep(&frame);
 489             }
 490             pthread_mutex_unlock(&s->out_mutex);
 491
 492             // Feed a dummy frame prior to signalling EOF.
 493             // This is required to terminate the decoder(OMX.SEC)
 494             // when only one frame is read during stream info detection.
 495             if (s->dummy_buf && (frame = (Frame*)av_mallocz(sizeof(Frame)))) {
 496                 frame->status = OK;
 497                 frame->size   = s->dummy_bufsize;
 498                 frame->key    = 1;
 499                 frame->buffer = s->dummy_buf;
 500                 pthread_mutex_lock(&s->in_mutex);
 501                 s->in_queue->push_back(frame);
 502                 pthread_cond_signal(&s->condition);
 503                 pthread_mutex_unlock(&s->in_mutex);
 504                 s->dummy_buf = NULL;
 505             }
 506
 507             pthread_mutex_lock(&s->in_mutex);
 508             s->end_frame->status = ERROR_END_OF_STREAM;
 509             s->in_queue->push_back(s->end_frame);
 510             pthread_cond_signal(&s->condition);
 511             pthread_mutex_unlock(&s->in_mutex);
 512             s->end_frame = NULL;
 513         }
 514
 515         pthread_join(s->decode_thread_id, NULL);
 516
 517         if (s->prev_frame) {
 518             avctx->release_buffer(avctx, s->prev_frame);
 519             av_freep(&s->prev_frame);
 520         }
 521
 522         s->thread_started = false;
 523     }
 524
 525     while (!s->in_queue->empty()) {
 526         frame = *s->in_queue->begin();
 527         s->in_queue->erase(s->in_queue->begin());
 528         if (frame->size)
 529             av_freep(&frame->buffer);
 530         av_freep(&frame);
 531     }
 532
 533     while (!s->out_queue->empty()) {
 534         frame = *s->out_queue->begin();
 535         s->out_queue->erase(s->out_queue->begin());
 536         if (frame->vframe) {
 537             avctx->release_buffer(avctx, frame->vframe);
 538             av_freep(&frame->vframe);
 539         }
 540         av_freep(&frame);
 541     }
 542
 543     (*s->decoder)->stop();
 544     s->client->disconnect();
 545
 546     if (s->decoder_component)
 547         av_freep(&s->decoder_component);
 548     av_freep(&s->dummy_buf);
 549     av_freep(&s->end_frame);
 550
 551     // Reset the extradata back to the original mp4 format, so that
 552     // the next invocation (both when decoding and when called from
 553     // av_find_stream_info) get the original mp4 format extradata.
 554     av_freep(&avctx->extradata);
 555     avctx->extradata = s->orig_extradata;
 556     avctx->extradata_size = s->orig_extradata_size;
 557
 558     delete s->in_queue;
 559     delete s->out_queue;
 560     delete s->ts_map;
 561     delete s->client;
 562     delete s->decoder;
 563     delete s->source;
 564
 565     pthread_mutex_destroy(&s->in_mutex);
 566     pthread_mutex_destroy(&s->out_mutex);
 567     pthread_cond_destroy(&s->condition);
 568     av_bitstream_filter_close(s->bsfc);
 569     return 0;
 570 }
 571
 572 AVCodec ff_libstagefright_h264_decoder = {
 573     "libstagefright_h264",
 574     NULL_IF_CONFIG_SMALL("libstagefright H.264"),
 575     AVMEDIA_TYPE_VIDEO,
 576     AV_CODEC_ID_H264,
 577     CODEC_CAP_DELAY,
 578     NULL, //supported_framerates
 579     NULL, //pix_fmts
 580     NULL, //supported_samplerates
 581     NULL, //sample_fmts
 582     NULL, //channel_layouts
 583     0,    //max_lowres
 584     NULL, //priv_class
 585     NULL, //profiles
 586     sizeof(StagefrightContext),
 587     NULL, //next
 588     NULL, //init_thread_copy
 589     NULL, //update_thread_context
 590     NULL, //defaults
 591     NULL, //init_static_data
 592     Stagefright_init,
 593     NULL, //encode
 594     NULL, //encode2
 595     Stagefright_decode_frame,
 596     Stagefright_close,
 597 };