2 * Interface to the Android Stagefright library for
3 * H/W accelerated H.264 decoding
5 * Copyright (C) 2011 Mohamed Naufal
6 * Copyright (C) 2011 Martin Storsjö
8 * This file is part of FFmpeg.
10 * FFmpeg is free software; you can redistribute it and/or
11 * modify it under the terms of the GNU Lesser General Public
12 * License as published by the Free Software Foundation; either
13 * version 2.1 of the License, or (at your option) any later version.
15 * FFmpeg is distributed in the hope that it will be useful,
16 * but WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
18 * Lesser General Public License for more details.
20 * You should have received a copy of the GNU Lesser General Public
21 * License along with FFmpeg; if not, write to the Free Software
22 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
25 #include <binder/ProcessState.h>
26 #include <media/stagefright/MetaData.h>
27 #include <media/stagefright/MediaBufferGroup.h>
28 #include <media/stagefright/MediaDebug.h>
29 #include <media/stagefright/MediaDefs.h>
30 #include <media/stagefright/OMXClient.h>
31 #include <media/stagefright/OMXCodec.h>
32 #include <utils/List.h>
37 #include "libavutil/imgutils.h"
40 #define OMX_QCOM_COLOR_FormatYVU420SemiPlanar 0x7FA30C00
42 using namespace android;
56 struct StagefrightContext {
57 AVCodecContext *avctx;
58 AVBitStreamFilterContext *bsfc;
59 uint8_t* orig_extradata;
60 int orig_extradata_size;
61 sp<MediaSource> *source;
62 List<Frame*> *in_queue, *out_queue;
63 pthread_mutex_t in_mutex, out_mutex;
64 pthread_cond_t condition;
65 pthread_t decode_thread_id;
69 volatile sig_atomic_t thread_started, thread_exited, stop_decode;
77 sp<MediaSource> *decoder;
78 const char *decoder_component;
81 class CustomSource : public MediaSource {
83 CustomSource(AVCodecContext *avctx, sp<MetaData> meta) {
84 s = (StagefrightContext*)avctx->priv_data;
86 frame_size = (avctx->width * avctx->height * 3) / 2;
87 buf_group.add_buffer(new MediaBuffer(frame_size));
90 virtual sp<MetaData> getFormat() {
94 virtual status_t start(MetaData *params) {
98 virtual status_t stop() {
102 virtual status_t read(MediaBuffer **buffer,
103 const MediaSource::ReadOptions *options) {
107 if (s->thread_exited)
108 return ERROR_END_OF_STREAM;
109 pthread_mutex_lock(&s->in_mutex);
111 while (s->in_queue->empty())
112 pthread_cond_wait(&s->condition, &s->in_mutex);
114 frame = *s->in_queue->begin();
118 ret = buf_group.acquire_buffer(buffer);
120 memcpy((*buffer)->data(), frame->buffer, frame->size);
121 (*buffer)->set_range(0, frame->size);
122 (*buffer)->meta_data()->clear();
123 (*buffer)->meta_data()->setInt32(kKeyIsSyncFrame,frame->key);
124 (*buffer)->meta_data()->setInt64(kKeyTime, frame->time);
126 av_log(s->avctx, AV_LOG_ERROR, "Failed to acquire MediaBuffer\n");
128 av_freep(&frame->buffer);
131 s->in_queue->erase(s->in_queue->begin());
132 pthread_mutex_unlock(&s->in_mutex);
139 MediaBufferGroup buf_group;
140 sp<MetaData> source_meta;
141 StagefrightContext *s;
145 void* decode_thread(void *arg)
147 AVCodecContext *avctx = (AVCodecContext*)arg;
148 StagefrightContext *s = (StagefrightContext*)avctx->priv_data;
154 frame = (Frame*)av_mallocz(sizeof(Frame));
156 frame = s->end_frame;
157 frame->status = AVERROR(ENOMEM);
161 frame->status = (*s->decoder)->read(&buffer);
162 if (frame->status == OK) {
163 sp<MetaData> outFormat = (*s->decoder)->getFormat();
164 outFormat->findInt32(kKeyWidth , &frame->w);
165 outFormat->findInt32(kKeyHeight, &frame->h);
166 frame->size = buffer->range_length();
167 frame->mbuffer = buffer;
168 } else if (frame->status == INFO_FORMAT_CHANGED) {
178 pthread_mutex_lock(&s->out_mutex);
179 if (s->out_queue->size() >= 10) {
180 pthread_mutex_unlock(&s->out_mutex);
186 s->out_queue->push_back(frame);
187 pthread_mutex_unlock(&s->out_mutex);
188 } while (!decode_done && !s->stop_decode);
190 s->thread_exited = true;
195 static av_cold int Stagefright_init(AVCodecContext *avctx)
197 StagefrightContext *s = (StagefrightContext*)avctx->priv_data;
198 sp<MetaData> meta, outFormat;
199 int32_t colorFormat = 0;
202 if (!avctx->extradata || !avctx->extradata_size || avctx->extradata[0] != 1)
206 s->bsfc = av_bitstream_filter_init("h264_mp4toannexb");
208 av_log(avctx, AV_LOG_ERROR, "Cannot open the h264_mp4toannexb BSF!\n");
212 s->orig_extradata_size = avctx->extradata_size;
213 s->orig_extradata = (uint8_t*) av_mallocz(avctx->extradata_size +
214 FF_INPUT_BUFFER_PADDING_SIZE);
215 if (!s->orig_extradata) {
216 ret = AVERROR(ENOMEM);
219 memcpy(s->orig_extradata, avctx->extradata, avctx->extradata_size);
223 ret = AVERROR(ENOMEM);
226 meta->setCString(kKeyMIMEType, MEDIA_MIMETYPE_VIDEO_AVC);
227 meta->setInt32(kKeyWidth, avctx->width);
228 meta->setInt32(kKeyHeight, avctx->height);
229 meta->setData(kKeyAVCC, kTypeAVCC, avctx->extradata, avctx->extradata_size);
231 android::ProcessState::self()->startThreadPool();
233 s->source = new sp<MediaSource>();
234 *s->source = new CustomSource(avctx, meta);
235 s->in_queue = new List<Frame*>;
236 s->out_queue = new List<Frame*>;
237 s->client = new OMXClient;
238 s->end_frame = (Frame*)av_mallocz(sizeof(Frame));
239 if (s->source == NULL || !s->in_queue || !s->out_queue || !s->client ||
241 ret = AVERROR(ENOMEM);
245 if (s->client->connect() != OK) {
246 av_log(avctx, AV_LOG_ERROR, "Cannot connect OMX client\n");
251 s->decoder = new sp<MediaSource>();
252 *s->decoder = OMXCodec::Create(s->client->interface(), meta,
253 false, *s->source, NULL,
254 OMXCodec::kClientNeedsFramebuffer);
255 if ((*s->decoder)->start() != OK) {
256 av_log(avctx, AV_LOG_ERROR, "Cannot start decoder\n");
258 s->client->disconnect();
262 outFormat = (*s->decoder)->getFormat();
263 outFormat->findInt32(kKeyColorFormat, &colorFormat);
264 if (colorFormat == OMX_QCOM_COLOR_FormatYVU420SemiPlanar ||
265 colorFormat == OMX_COLOR_FormatYUV420SemiPlanar)
266 avctx->pix_fmt = PIX_FMT_NV21;
268 avctx->pix_fmt = PIX_FMT_YUV420P;
270 outFormat->findCString(kKeyDecoderComponent, &s->decoder_component);
271 if (s->decoder_component)
272 s->decoder_component = av_strdup(s->decoder_component);
274 pthread_mutex_init(&s->in_mutex, NULL);
275 pthread_mutex_init(&s->out_mutex, NULL);
276 pthread_cond_init(&s->condition, NULL);
280 av_bitstream_filter_close(s->bsfc);
281 av_freep(&s->orig_extradata);
282 av_freep(&s->end_frame);
289 static int Stagefright_decode_frame(AVCodecContext *avctx, void *data,
290 int *data_size, AVPacket *avpkt)
292 StagefrightContext *s = (StagefrightContext*)avctx->priv_data;
294 MediaBuffer *mbuffer;
298 const uint8_t *src_data[3];
301 int orig_size = avpkt->size;
302 AVPacket pkt = *avpkt;
305 if (!s->thread_started) {
306 pthread_create(&s->decode_thread_id, NULL, &decode_thread, avctx);
307 s->thread_started = true;
310 if (avpkt && avpkt->data) {
311 av_bitstream_filter_filter(s->bsfc, avctx, NULL, &pkt.data, &pkt.size,
312 avpkt->data, avpkt->size, avpkt->flags & AV_PKT_FLAG_KEY);
316 if (!s->source_done) {
318 s->dummy_buf = (uint8_t*)av_malloc(avpkt->size);
320 return AVERROR(ENOMEM);
321 s->dummy_bufsize = avpkt->size;
322 memcpy(s->dummy_buf, avpkt->data, avpkt->size);
325 frame = (Frame*)av_mallocz(sizeof(Frame));
328 frame->size = avpkt->size;
329 // Stagefright can't handle negative timestamps -
330 // if needed, work around this by offsetting them manually?
332 frame->time = avpkt->pts;
333 frame->key = avpkt->flags & AV_PKT_FLAG_KEY ? 1 : 0;
334 frame->buffer = (uint8_t*)av_malloc(avpkt->size);
335 if (!frame->buffer) {
337 return AVERROR(ENOMEM);
339 uint8_t *ptr = avpkt->data;
340 // The OMX.SEC decoder fails without this.
341 if (avpkt->size == orig_size + avctx->extradata_size) {
342 ptr += avctx->extradata_size;
343 frame->size = orig_size;
345 memcpy(frame->buffer, ptr, orig_size);
347 frame->status = ERROR_END_OF_STREAM;
348 s->source_done = true;
352 if (s->thread_exited) {
353 s->source_done = true;
356 pthread_mutex_lock(&s->in_mutex);
357 if (s->in_queue->size() >= 10) {
358 pthread_mutex_unlock(&s->in_mutex);
362 s->in_queue->push_back(frame);
363 pthread_cond_signal(&s->condition);
364 pthread_mutex_unlock(&s->in_mutex);
369 pthread_mutex_lock(&s->out_mutex);
370 if (!s->out_queue->empty()) break;
371 pthread_mutex_unlock(&s->out_mutex);
372 if (s->source_done) {
380 frame = *s->out_queue->begin();
381 s->out_queue->erase(s->out_queue->begin());
382 pthread_mutex_unlock(&s->out_mutex);
384 mbuffer = frame->mbuffer;
385 status = frame->status;
391 if (status == ERROR_END_OF_STREAM)
394 if (status == AVERROR(ENOMEM))
396 av_log(avctx, AV_LOG_ERROR, "Decode failed: %x\n", status);
400 // The OMX.SEC decoder doesn't signal the modified width/height
401 if (s->decoder_component && !strncmp(s->decoder_component, "OMX.SEC", 7) &&
402 (w & 15 || h & 15)) {
403 if (((w + 15)&~15) * ((h + 15)&~15) * 3/2 == size) {
409 if (!avctx->width || !avctx->height || avctx->width > w || avctx->height > h) {
414 ret = avctx->reget_buffer(avctx, &s->ret_frame);
416 av_log(avctx, AV_LOG_ERROR, "reget buffer() failed\n");
421 if (avctx->pix_fmt == PIX_FMT_YUV420P)
422 src_linesize[1] = src_linesize[2] = w/2;
423 else if (avctx->pix_fmt == PIX_FMT_NV21)
426 buf = (uint8_t*)mbuffer->data();
428 src_data[1] = buf + src_linesize[0] * h;
429 src_data[2] = src_data[1] + src_linesize[1] * h/2;
430 av_image_copy(s->ret_frame.data, s->ret_frame.linesize,
431 src_data, src_linesize,
432 avctx->pix_fmt, avctx->width, avctx->height);
434 *data_size = sizeof(AVFrame);
435 *(AVFrame*)data = s->ret_frame;
442 static av_cold int Stagefright_close(AVCodecContext *avctx)
444 StagefrightContext *s = (StagefrightContext*)avctx->priv_data;
447 if (s->thread_started) {
448 if (!s->thread_exited) {
451 // Make sure decode_thread() doesn't get stuck
452 pthread_mutex_lock(&s->out_mutex);
453 while (!s->out_queue->empty()) {
454 frame = *s->out_queue->begin();
455 s->out_queue->erase(s->out_queue->begin());
457 frame->mbuffer->release();
460 pthread_mutex_unlock(&s->out_mutex);
462 // Feed a dummy frame prior to signalling EOF.
463 // This is required to terminate the decoder(OMX.SEC)
464 // when only one frame is read during stream info detection.
465 if (s->dummy_buf && (frame = (Frame*)av_mallocz(sizeof(Frame)))) {
467 frame->size = s->dummy_bufsize;
469 frame->buffer = s->dummy_buf;
470 pthread_mutex_lock(&s->in_mutex);
471 s->in_queue->push_back(frame);
472 pthread_cond_signal(&s->condition);
473 pthread_mutex_unlock(&s->in_mutex);
477 pthread_mutex_lock(&s->in_mutex);
478 s->end_frame->status = ERROR_END_OF_STREAM;
479 s->in_queue->push_back(s->end_frame);
480 pthread_cond_signal(&s->condition);
481 pthread_mutex_unlock(&s->in_mutex);
485 pthread_join(s->decode_thread_id, NULL);
487 if (s->ret_frame.data[0])
488 avctx->release_buffer(avctx, &s->ret_frame);
490 s->thread_started = false;
493 while (!s->in_queue->empty()) {
494 frame = *s->in_queue->begin();
495 s->in_queue->erase(s->in_queue->begin());
497 av_freep(&frame->buffer);
501 while (!s->out_queue->empty()) {
502 frame = *s->out_queue->begin();
503 s->out_queue->erase(s->out_queue->begin());
505 frame->mbuffer->release();
509 (*s->decoder)->stop();
510 s->client->disconnect();
512 if (s->decoder_component)
513 av_freep(&s->decoder_component);
514 av_freep(&s->dummy_buf);
515 av_freep(&s->end_frame);
517 // Reset the extradata back to the original mp4 format, so that
518 // the next invocation (both when decoding and when called from
519 // av_find_stream_info) get the original mp4 format extradata.
520 av_freep(&avctx->extradata);
521 avctx->extradata = s->orig_extradata;
522 avctx->extradata_size = s->orig_extradata_size;
530 pthread_mutex_destroy(&s->in_mutex);
531 pthread_mutex_destroy(&s->out_mutex);
532 pthread_cond_destroy(&s->condition);
533 av_bitstream_filter_close(s->bsfc);
537 AVCodec ff_libstagefright_h264_decoder = {
538 "libstagefright_h264",
541 sizeof(StagefrightContext),
545 Stagefright_decode_frame,
549 NULL, //supported_framerates
550 NULL, //pixel_formats
551 NULL_IF_CONFIG_SMALL("libstagefright H.264"),