2 * Interface to the Android Stagefright library for
3 * H/W accelerated H.264 decoding
5 * Copyright (C) 2011 Mohamed Naufal
6 * Copyright (C) 2011 Martin Storsjö
8 * This file is part of FFmpeg.
10 * FFmpeg is free software; you can redistribute it and/or
11 * modify it under the terms of the GNU Lesser General Public
12 * License as published by the Free Software Foundation; either
13 * version 2.1 of the License, or (at your option) any later version.
15 * FFmpeg is distributed in the hope that it will be useful,
16 * but WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
18 * Lesser General Public License for more details.
20 * You should have received a copy of the GNU Lesser General Public
21 * License along with FFmpeg; if not, write to the Free Software
22 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
25 #include <binder/ProcessState.h>
26 #include <media/stagefright/MetaData.h>
27 #include <media/stagefright/MediaBufferGroup.h>
28 #include <media/stagefright/MediaDebug.h>
29 #include <media/stagefright/MediaDefs.h>
30 #include <media/stagefright/OMXClient.h>
31 #include <media/stagefright/OMXCodec.h>
32 #include <utils/List.h>
38 #include "libavutil/imgutils.h"
41 #define OMX_QCOM_COLOR_FormatYVU420SemiPlanar 0x7FA30C00
43 using namespace android;
56 int64_t reordered_opaque;
61 struct StagefrightContext {
62 AVCodecContext *avctx;
63 AVBitStreamFilterContext *bsfc;
64 uint8_t* orig_extradata;
65 int orig_extradata_size;
66 sp<MediaSource> *source;
67 List<Frame*> *in_queue, *out_queue;
68 pthread_mutex_t in_mutex, out_mutex;
69 pthread_cond_t condition;
70 pthread_t decode_thread_id;
74 volatile sig_atomic_t thread_started, thread_exited, stop_decode;
77 std::map<int64_t, TimeStamp> *ts_map;
84 sp<MediaSource> *decoder;
85 const char *decoder_component;
88 class CustomSource : public MediaSource {
90 CustomSource(AVCodecContext *avctx, sp<MetaData> meta) {
91 s = (StagefrightContext*)avctx->priv_data;
93 frame_size = (avctx->width * avctx->height * 3) / 2;
94 buf_group.add_buffer(new MediaBuffer(frame_size));
97 virtual sp<MetaData> getFormat() {
101 virtual status_t start(MetaData *params) {
105 virtual status_t stop() {
109 virtual status_t read(MediaBuffer **buffer,
110 const MediaSource::ReadOptions *options) {
114 if (s->thread_exited)
115 return ERROR_END_OF_STREAM;
116 pthread_mutex_lock(&s->in_mutex);
118 while (s->in_queue->empty())
119 pthread_cond_wait(&s->condition, &s->in_mutex);
121 frame = *s->in_queue->begin();
125 ret = buf_group.acquire_buffer(buffer);
127 memcpy((*buffer)->data(), frame->buffer, frame->size);
128 (*buffer)->set_range(0, frame->size);
129 (*buffer)->meta_data()->clear();
130 (*buffer)->meta_data()->setInt32(kKeyIsSyncFrame,frame->key);
131 (*buffer)->meta_data()->setInt64(kKeyTime, frame->time);
133 av_log(s->avctx, AV_LOG_ERROR, "Failed to acquire MediaBuffer\n");
135 av_freep(&frame->buffer);
138 s->in_queue->erase(s->in_queue->begin());
139 pthread_mutex_unlock(&s->in_mutex);
146 MediaBufferGroup buf_group;
147 sp<MetaData> source_meta;
148 StagefrightContext *s;
152 void* decode_thread(void *arg)
154 AVCodecContext *avctx = (AVCodecContext*)arg;
155 StagefrightContext *s = (StagefrightContext*)avctx->priv_data;
156 const AVPixFmtDescriptor *pix_desc = av_pix_fmt_desc_get(avctx->pix_fmt);
163 const uint8_t *src_data[3];
164 int64_t out_frame_index = 0;
168 frame = (Frame*)av_mallocz(sizeof(Frame));
170 frame = s->end_frame;
171 frame->status = AVERROR(ENOMEM);
176 frame->status = (*s->decoder)->read(&buffer);
177 if (frame->status == OK) {
178 sp<MetaData> outFormat = (*s->decoder)->getFormat();
179 outFormat->findInt32(kKeyWidth , &w);
180 outFormat->findInt32(kKeyHeight, &h);
181 frame->vframe = (AVFrame*)av_mallocz(sizeof(AVFrame));
182 if (!frame->vframe) {
183 frame->status = AVERROR(ENOMEM);
188 ret = ff_get_buffer(avctx, frame->vframe);
190 av_log(avctx, AV_LOG_ERROR, "get_buffer() failed\n");
197 // The OMX.SEC decoder doesn't signal the modified width/height
198 if (s->decoder_component && !strncmp(s->decoder_component, "OMX.SEC", 7) &&
199 (w & 15 || h & 15)) {
200 if (((w + 15)&~15) * ((h + 15)&~15) * 3/2 == buffer->range_length()) {
206 if (!avctx->width || !avctx->height || avctx->width > w || avctx->height > h) {
211 src_linesize[0] = av_image_get_linesize(avctx->pix_fmt, w, 0);
212 src_linesize[1] = av_image_get_linesize(avctx->pix_fmt, w, 1);
213 src_linesize[2] = av_image_get_linesize(avctx->pix_fmt, w, 2);
215 src_data[0] = (uint8_t*)buffer->data();
216 src_data[1] = src_data[0] + src_linesize[0] * h;
217 src_data[2] = src_data[1] + src_linesize[1] * -(-h>>pix_desc->log2_chroma_h);
218 av_image_copy(frame->vframe->data, frame->vframe->linesize,
219 src_data, src_linesize,
220 avctx->pix_fmt, avctx->width, avctx->height);
222 buffer->meta_data()->findInt64(kKeyTime, &out_frame_index);
223 if (out_frame_index && s->ts_map->count(out_frame_index) > 0) {
224 frame->vframe->pts = (*s->ts_map)[out_frame_index].pts;
225 frame->vframe->reordered_opaque = (*s->ts_map)[out_frame_index].reordered_opaque;
226 s->ts_map->erase(out_frame_index);
229 } else if (frame->status == INFO_FORMAT_CHANGED) {
239 pthread_mutex_lock(&s->out_mutex);
240 if (s->out_queue->size() >= 10) {
241 pthread_mutex_unlock(&s->out_mutex);
247 s->out_queue->push_back(frame);
248 pthread_mutex_unlock(&s->out_mutex);
249 } while (!decode_done && !s->stop_decode);
251 s->thread_exited = true;
256 static av_cold int Stagefright_init(AVCodecContext *avctx)
258 StagefrightContext *s = (StagefrightContext*)avctx->priv_data;
259 sp<MetaData> meta, outFormat;
260 int32_t colorFormat = 0;
263 if (!avctx->extradata || !avctx->extradata_size || avctx->extradata[0] != 1)
267 s->bsfc = av_bitstream_filter_init("h264_mp4toannexb");
269 av_log(avctx, AV_LOG_ERROR, "Cannot open the h264_mp4toannexb BSF!\n");
273 s->orig_extradata_size = avctx->extradata_size;
274 s->orig_extradata = (uint8_t*) av_mallocz(avctx->extradata_size +
275 FF_INPUT_BUFFER_PADDING_SIZE);
276 if (!s->orig_extradata) {
277 ret = AVERROR(ENOMEM);
280 memcpy(s->orig_extradata, avctx->extradata, avctx->extradata_size);
284 ret = AVERROR(ENOMEM);
287 meta->setCString(kKeyMIMEType, MEDIA_MIMETYPE_VIDEO_AVC);
288 meta->setInt32(kKeyWidth, avctx->width);
289 meta->setInt32(kKeyHeight, avctx->height);
290 meta->setData(kKeyAVCC, kTypeAVCC, avctx->extradata, avctx->extradata_size);
292 android::ProcessState::self()->startThreadPool();
294 s->source = new sp<MediaSource>();
295 *s->source = new CustomSource(avctx, meta);
296 s->in_queue = new List<Frame*>;
297 s->out_queue = new List<Frame*>;
298 s->ts_map = new std::map<int64_t, TimeStamp>;
299 s->client = new OMXClient;
300 s->end_frame = (Frame*)av_mallocz(sizeof(Frame));
301 if (s->source == NULL || !s->in_queue || !s->out_queue || !s->client ||
302 !s->ts_map || !s->end_frame) {
303 ret = AVERROR(ENOMEM);
307 if (s->client->connect() != OK) {
308 av_log(avctx, AV_LOG_ERROR, "Cannot connect OMX client\n");
313 s->decoder = new sp<MediaSource>();
314 *s->decoder = OMXCodec::Create(s->client->interface(), meta,
315 false, *s->source, NULL,
316 OMXCodec::kClientNeedsFramebuffer);
317 if ((*s->decoder)->start() != OK) {
318 av_log(avctx, AV_LOG_ERROR, "Cannot start decoder\n");
320 s->client->disconnect();
324 outFormat = (*s->decoder)->getFormat();
325 outFormat->findInt32(kKeyColorFormat, &colorFormat);
326 if (colorFormat == OMX_QCOM_COLOR_FormatYVU420SemiPlanar ||
327 colorFormat == OMX_COLOR_FormatYUV420SemiPlanar)
328 avctx->pix_fmt = AV_PIX_FMT_NV21;
329 else if (colorFormat == OMX_COLOR_FormatYCbYCr)
330 avctx->pix_fmt = AV_PIX_FMT_YUYV422;
331 else if (colorFormat == OMX_COLOR_FormatCbYCrY)
332 avctx->pix_fmt = AV_PIX_FMT_UYVY422;
334 avctx->pix_fmt = AV_PIX_FMT_YUV420P;
336 outFormat->findCString(kKeyDecoderComponent, &s->decoder_component);
337 if (s->decoder_component)
338 s->decoder_component = av_strdup(s->decoder_component);
340 pthread_mutex_init(&s->in_mutex, NULL);
341 pthread_mutex_init(&s->out_mutex, NULL);
342 pthread_cond_init(&s->condition, NULL);
346 av_bitstream_filter_close(s->bsfc);
347 av_freep(&s->orig_extradata);
348 av_freep(&s->end_frame);
356 static int Stagefright_decode_frame(AVCodecContext *avctx, void *data,
357 int *got_frame, AVPacket *avpkt)
359 StagefrightContext *s = (StagefrightContext*)avctx->priv_data;
362 int orig_size = avpkt->size;
363 AVPacket pkt = *avpkt;
366 if (!s->thread_started) {
367 pthread_create(&s->decode_thread_id, NULL, &decode_thread, avctx);
368 s->thread_started = true;
371 if (avpkt && avpkt->data) {
372 av_bitstream_filter_filter(s->bsfc, avctx, NULL, &pkt.data, &pkt.size,
373 avpkt->data, avpkt->size, avpkt->flags & AV_PKT_FLAG_KEY);
377 if (!s->source_done) {
379 s->dummy_buf = (uint8_t*)av_malloc(avpkt->size);
381 return AVERROR(ENOMEM);
382 s->dummy_bufsize = avpkt->size;
383 memcpy(s->dummy_buf, avpkt->data, avpkt->size);
386 frame = (Frame*)av_mallocz(sizeof(Frame));
389 frame->size = avpkt->size;
390 frame->key = avpkt->flags & AV_PKT_FLAG_KEY ? 1 : 0;
391 frame->buffer = (uint8_t*)av_malloc(avpkt->size);
392 if (!frame->buffer) {
394 return AVERROR(ENOMEM);
396 uint8_t *ptr = avpkt->data;
397 // The OMX.SEC decoder fails without this.
398 if (avpkt->size == orig_size + avctx->extradata_size) {
399 ptr += avctx->extradata_size;
400 frame->size = orig_size;
402 memcpy(frame->buffer, ptr, orig_size);
404 av_free(avpkt->data);
406 frame->time = ++s->frame_index;
407 (*s->ts_map)[s->frame_index].pts = avpkt->pts;
408 (*s->ts_map)[s->frame_index].reordered_opaque = avctx->reordered_opaque;
410 frame->status = ERROR_END_OF_STREAM;
411 s->source_done = true;
415 if (s->thread_exited) {
416 s->source_done = true;
419 pthread_mutex_lock(&s->in_mutex);
420 if (s->in_queue->size() >= 10) {
421 pthread_mutex_unlock(&s->in_mutex);
425 s->in_queue->push_back(frame);
426 pthread_cond_signal(&s->condition);
427 pthread_mutex_unlock(&s->in_mutex);
432 pthread_mutex_lock(&s->out_mutex);
433 if (!s->out_queue->empty()) break;
434 pthread_mutex_unlock(&s->out_mutex);
435 if (s->source_done) {
443 frame = *s->out_queue->begin();
444 s->out_queue->erase(s->out_queue->begin());
445 pthread_mutex_unlock(&s->out_mutex);
447 ret_frame = frame->vframe;
448 status = frame->status;
451 if (status == ERROR_END_OF_STREAM)
454 if (status == AVERROR(ENOMEM))
456 av_log(avctx, AV_LOG_ERROR, "Decode failed: %x\n", status);
461 avctx->release_buffer(avctx, s->prev_frame);
462 av_freep(&s->prev_frame);
464 s->prev_frame = ret_frame;
467 *(AVFrame*)data = *ret_frame;
471 static av_cold int Stagefright_close(AVCodecContext *avctx)
473 StagefrightContext *s = (StagefrightContext*)avctx->priv_data;
476 if (s->thread_started) {
477 if (!s->thread_exited) {
480 // Make sure decode_thread() doesn't get stuck
481 pthread_mutex_lock(&s->out_mutex);
482 while (!s->out_queue->empty()) {
483 frame = *s->out_queue->begin();
484 s->out_queue->erase(s->out_queue->begin());
486 avctx->release_buffer(avctx, frame->vframe);
487 av_freep(&frame->vframe);
491 pthread_mutex_unlock(&s->out_mutex);
493 // Feed a dummy frame prior to signalling EOF.
494 // This is required to terminate the decoder(OMX.SEC)
495 // when only one frame is read during stream info detection.
496 if (s->dummy_buf && (frame = (Frame*)av_mallocz(sizeof(Frame)))) {
498 frame->size = s->dummy_bufsize;
500 frame->buffer = s->dummy_buf;
501 pthread_mutex_lock(&s->in_mutex);
502 s->in_queue->push_back(frame);
503 pthread_cond_signal(&s->condition);
504 pthread_mutex_unlock(&s->in_mutex);
508 pthread_mutex_lock(&s->in_mutex);
509 s->end_frame->status = ERROR_END_OF_STREAM;
510 s->in_queue->push_back(s->end_frame);
511 pthread_cond_signal(&s->condition);
512 pthread_mutex_unlock(&s->in_mutex);
516 pthread_join(s->decode_thread_id, NULL);
519 avctx->release_buffer(avctx, s->prev_frame);
520 av_freep(&s->prev_frame);
523 s->thread_started = false;
526 while (!s->in_queue->empty()) {
527 frame = *s->in_queue->begin();
528 s->in_queue->erase(s->in_queue->begin());
530 av_freep(&frame->buffer);
534 while (!s->out_queue->empty()) {
535 frame = *s->out_queue->begin();
536 s->out_queue->erase(s->out_queue->begin());
538 avctx->release_buffer(avctx, frame->vframe);
539 av_freep(&frame->vframe);
544 (*s->decoder)->stop();
545 s->client->disconnect();
547 if (s->decoder_component)
548 av_freep(&s->decoder_component);
549 av_freep(&s->dummy_buf);
550 av_freep(&s->end_frame);
552 // Reset the extradata back to the original mp4 format, so that
553 // the next invocation (both when decoding and when called from
554 // av_find_stream_info) get the original mp4 format extradata.
555 av_freep(&avctx->extradata);
556 avctx->extradata = s->orig_extradata;
557 avctx->extradata_size = s->orig_extradata_size;
566 pthread_mutex_destroy(&s->in_mutex);
567 pthread_mutex_destroy(&s->out_mutex);
568 pthread_cond_destroy(&s->condition);
569 av_bitstream_filter_close(s->bsfc);
573 AVCodec ff_libstagefright_h264_decoder = {
574 "libstagefright_h264",
575 NULL_IF_CONFIG_SMALL("libstagefright H.264"),
579 NULL, //supported_framerates
581 NULL, //supported_samplerates
583 NULL, //channel_layouts
587 sizeof(StagefrightContext),
589 NULL, //init_thread_copy
590 NULL, //update_thread_context
592 NULL, //init_static_data
596 Stagefright_decode_frame,