1 /*****************************************************************************
2 * android_mediacodec.c: Video decoder module using the Android MediaCodec API
3 *****************************************************************************
4 * Copyright (C) 2012 Martin Storsjo
6 * Authors: Martin Storsjo <martin@martin.st>
8 * This program is free software; you can redistribute it and/or modify it
9 * under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation; either version 2.1 of the License, or
11 * (at your option) any later version.
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with this program; if not, write to the Free Software Foundation,
20 * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
21 *****************************************************************************/
23 /*****************************************************************************
25 *****************************************************************************/
33 #include <vlc_common.h>
34 #include <vlc_plugin.h>
35 #include <vlc_codec.h>
36 #include <vlc_block_helper.h>
39 #include "../h264_nal.h"
41 #include <OMX_Component.h>
42 #include "omxil_utils.h"
44 #define INFO_OUTPUT_BUFFERS_CHANGED -3
45 #define INFO_OUTPUT_FORMAT_CHANGED -2
46 #define INFO_TRY_AGAIN_LATER -1
52 jclass media_codec_list_class, media_codec_class, media_format_class;
53 jclass buffer_info_class, byte_buffer_class;
55 jmethodID get_codec_count, get_codec_info_at, is_encoder;
56 jmethodID get_supported_types, get_name;
57 jmethodID create_by_codec_name, configure, start, stop, flush, release;
58 jmethodID get_output_format, get_input_buffers, get_output_buffers;
59 jmethodID dequeue_input_buffer, dequeue_output_buffer, queue_input_buffer;
60 jmethodID release_output_buffer;
61 jmethodID create_video_format, set_integer, set_bytebuffer, get_integer;
62 jmethodID buffer_info_ctor;
63 jmethodID allocate_direct, limit;
64 jfieldID size_field, offset_field, pts_field;
70 jobject input_buffers, output_buffers;
72 int stride, slice_height;
73 int crop_top, crop_left;
82 METHOD, STATIC_METHOD, FIELD
85 #define OFF(x) offsetof(struct decoder_sys_t, x)
91 static const struct classname classes[] = {
92 { "android/media/MediaCodecList", OFF(media_codec_list_class) },
93 { "android/media/MediaCodec", OFF(media_codec_class) },
94 { "android/media/MediaFormat", OFF(media_format_class) },
95 { "android/media/MediaFormat", OFF(media_format_class) },
96 { "android/media/MediaCodec$BufferInfo", OFF(buffer_info_class) },
97 { "java/nio/ByteBuffer", OFF(byte_buffer_class) },
109 static const struct member members[] = {
110 { "toString", "()Ljava/lang/String;", "java/lang/Object", OFF(tostring), METHOD },
112 { "getCodecCount", "()I", "android/media/MediaCodecList", OFF(get_codec_count), STATIC_METHOD },
113 { "getCodecInfoAt", "(I)Landroid/media/MediaCodecInfo;", "android/media/MediaCodecList", OFF(get_codec_info_at), STATIC_METHOD },
115 { "isEncoder", "()Z", "android/media/MediaCodecInfo", OFF(is_encoder), METHOD },
116 { "getSupportedTypes", "()[Ljava/lang/String;", "android/media/MediaCodecInfo", OFF(get_supported_types), METHOD },
117 { "getName", "()Ljava/lang/String;", "android/media/MediaCodecInfo", OFF(get_name), METHOD },
119 { "createByCodecName", "(Ljava/lang/String;)Landroid/media/MediaCodec;", "android/media/MediaCodec", OFF(create_by_codec_name), STATIC_METHOD },
120 { "configure", "(Landroid/media/MediaFormat;Landroid/view/Surface;Landroid/media/MediaCrypto;I)V", "android/media/MediaCodec", OFF(configure), METHOD },
121 { "start", "()V", "android/media/MediaCodec", OFF(start), METHOD },
122 { "stop", "()V", "android/media/MediaCodec", OFF(stop), METHOD },
123 { "flush", "()V", "android/media/MediaCodec", OFF(flush), METHOD },
124 { "release", "()V", "android/media/MediaCodec", OFF(release), METHOD },
125 { "getOutputFormat", "()Landroid/media/MediaFormat;", "android/media/MediaCodec", OFF(get_output_format), METHOD },
126 { "getInputBuffers", "()[Ljava/nio/ByteBuffer;", "android/media/MediaCodec", OFF(get_input_buffers), METHOD },
127 { "getOutputBuffers", "()[Ljava/nio/ByteBuffer;", "android/media/MediaCodec", OFF(get_output_buffers), METHOD },
128 { "dequeueInputBuffer", "(J)I", "android/media/MediaCodec", OFF(dequeue_input_buffer), METHOD },
129 { "dequeueOutputBuffer", "(Landroid/media/MediaCodec$BufferInfo;J)I", "android/media/MediaCodec", OFF(dequeue_output_buffer), METHOD },
130 { "queueInputBuffer", "(IIIJI)V", "android/media/MediaCodec", OFF(queue_input_buffer), METHOD },
131 { "releaseOutputBuffer", "(IZ)V", "android/media/MediaCodec", OFF(release_output_buffer), METHOD },
133 { "createVideoFormat", "(Ljava/lang/String;II)Landroid/media/MediaFormat;", "android/media/MediaFormat", OFF(create_video_format), STATIC_METHOD },
134 { "setInteger", "(Ljava/lang/String;I)V", "android/media/MediaFormat", OFF(set_integer), METHOD },
135 { "getInteger", "(Ljava/lang/String;)I", "android/media/MediaFormat", OFF(get_integer), METHOD },
136 { "setByteBuffer", "(Ljava/lang/String;Ljava/nio/ByteBuffer;)V", "android/media/MediaFormat", OFF(set_bytebuffer), METHOD },
138 { "<init>", "()V", "android/media/MediaCodec$BufferInfo", OFF(buffer_info_ctor), METHOD },
139 { "size", "I", "android/media/MediaCodec$BufferInfo", OFF(size_field), FIELD },
140 { "offset", "I", "android/media/MediaCodec$BufferInfo", OFF(offset_field), FIELD },
141 { "presentationTimeUs", "J", "android/media/MediaCodec$BufferInfo", OFF(pts_field), FIELD },
143 { "allocateDirect", "(I)Ljava/nio/ByteBuffer;", "java/nio/ByteBuffer", OFF(allocate_direct), STATIC_METHOD },
144 { "limit", "(I)Ljava/nio/Buffer;", "java/nio/ByteBuffer", OFF(limit), METHOD },
146 { NULL, NULL, NULL, 0, 0 },
149 #define GET_INTEGER(obj, name) (*env)->CallIntMethod(env, obj, p_sys->get_integer, (*env)->NewStringUTF(env, name))
151 /*****************************************************************************
153 *****************************************************************************/
154 static int OpenDecoder(vlc_object_t *);
155 static void CloseDecoder(vlc_object_t *);
157 static picture_t *DecodeVideo(decoder_t *, block_t **);
159 /*****************************************************************************
161 *****************************************************************************/
163 set_description( N_("Video decoder using Android MediaCodec") )
164 set_category( CAT_INPUT )
165 set_subcategory( SUBCAT_INPUT_VCODEC )
166 set_section( N_("Decoding") , NULL )
167 set_capability( "decoder", 0 ) /* Only enabled via commandline arguments */
168 set_callbacks( OpenDecoder, CloseDecoder )
171 static int jstrcmp(JNIEnv* env, jobject str, const char* str2)
173 jsize len = (*env)->GetStringUTFLength(env, str);
174 if (len != (jsize) strlen(str2))
176 const char *ptr = (*env)->GetStringUTFChars(env, str, NULL);
177 int ret = memcmp(ptr, str2, len);
178 (*env)->ReleaseStringUTFChars(env, str, ptr);
182 /*****************************************************************************
183 * OpenDecoder: Create the decoder instance
184 *****************************************************************************/
185 static int OpenDecoder(vlc_object_t *p_this)
187 decoder_t *p_dec = (decoder_t*)p_this;
188 decoder_sys_t *p_sys;
190 if (p_dec->fmt_in.i_cat != VIDEO_ES && !p_dec->b_force)
193 const char *mime = NULL;
194 switch (p_dec->fmt_in.i_codec) {
195 case VLC_CODEC_H264: mime = "video/avc"; break;
196 case VLC_CODEC_H263: mime = "video/3gpp"; break;
197 case VLC_CODEC_MP4V: mime = "video/mp4v-es"; break;
198 case VLC_CODEC_VC1: mime = "video/wvc1"; break;
200 msg_Dbg(p_dec, "codec %d not supported", p_dec->fmt_in.i_codec);
203 /* Allocate the memory needed to store the decoder's structure */
204 if ((p_dec->p_sys = p_sys = calloc(1, sizeof(*p_sys))) == NULL)
207 p_dec->pf_decode_video = DecodeVideo;
210 p_dec->fmt_out.i_cat = p_dec->fmt_in.i_cat;
211 p_dec->fmt_out.video = p_dec->fmt_in.video;
212 p_dec->fmt_out.audio = p_dec->fmt_in.audio;
213 p_dec->b_need_packetized = true;
216 (*myVm)->AttachCurrentThread(myVm, &env, NULL);
218 for (int i = 0; classes[i].name; i++) {
219 *(jclass*)((uint8_t*)p_sys + classes[i].offset) =
220 (*env)->FindClass(env, classes[i].name);
222 if ((*env)->ExceptionOccurred(env)) {
223 msg_Warn(p_dec, "Unable to find class %s", classes[i].name);
224 (*env)->ExceptionClear(env);
230 for (int i = 0; members[i].name; i++) {
231 if (i == 0 || strcmp(members[i].class, members[i - 1].class))
232 last_class = (*env)->FindClass(env, members[i].class);
234 if ((*env)->ExceptionOccurred(env)) {
235 msg_Warn(p_dec, "Unable to find class %s", members[i].class);
236 (*env)->ExceptionClear(env);
240 switch (members[i].type) {
242 *(jmethodID*)((uint8_t*)p_sys + members[i].offset) =
243 (*env)->GetMethodID(env, last_class, members[i].name, members[i].sig);
246 *(jmethodID*)((uint8_t*)p_sys + members[i].offset) =
247 (*env)->GetStaticMethodID(env, last_class, members[i].name, members[i].sig);
250 *(jfieldID*)((uint8_t*)p_sys + members[i].offset) =
251 (*env)->GetFieldID(env, last_class, members[i].name, members[i].sig);
254 if ((*env)->ExceptionOccurred(env)) {
255 msg_Warn(p_dec, "Unable to find the member %s in %s",
256 members[i].name, members[i].class);
257 (*env)->ExceptionClear(env);
262 int num_codecs = (*env)->CallStaticIntMethod(env, p_sys->media_codec_list_class,
263 p_sys->get_codec_count);
264 jobject codec_name = NULL;
266 for (int i = 0; i < num_codecs; i++) {
267 jobject info = (*env)->CallStaticObjectMethod(env, p_sys->media_codec_list_class,
268 p_sys->get_codec_info_at, i);
269 if ((*env)->CallBooleanMethod(env, info, p_sys->is_encoder)) {
270 (*env)->DeleteLocalRef(env, info);
273 jobject types = (*env)->CallObjectMethod(env, info, p_sys->get_supported_types);
274 int num_types = (*env)->GetArrayLength(env, types);
276 for (int j = 0; j < num_types && !found; j++) {
277 jobject type = (*env)->GetObjectArrayElement(env, types, j);
278 if (!jstrcmp(env, type, mime))
280 (*env)->DeleteLocalRef(env, type);
283 jobject name = (*env)->CallObjectMethod(env, info, p_sys->get_name);
284 jsize name_len = (*env)->GetStringUTFLength(env, name);
285 const char *name_ptr = (*env)->GetStringUTFChars(env, name, NULL);
286 msg_Dbg(p_dec, "using %.*s", name_len, name_ptr);
287 p_sys->name = malloc(name_len + 1);
288 memcpy(p_sys->name, name_ptr, name_len);
289 p_sys->name[name_len] = '\0';
290 (*env)->ReleaseStringUTFChars(env, name, name_ptr);
294 (*env)->DeleteLocalRef(env, info);
298 msg_Dbg(p_dec, "No suitable codec matching %s was found", mime);
302 // This method doesn't handle errors nicely, it crashes if the codec isn't found.
303 // (The same goes for createDecoderByType.) This is fixed in latest AOSP and in 4.2,
304 // but not in 4.1 devices.
305 p_sys->codec = (*env)->CallStaticObjectMethod(env, p_sys->media_codec_class,
306 p_sys->create_by_codec_name, codec_name);
307 p_sys->codec = (*env)->NewGlobalRef(env, p_sys->codec);
309 jobject format = (*env)->CallStaticObjectMethod(env, p_sys->media_format_class,
310 p_sys->create_video_format, (*env)->NewStringUTF(env, mime),
311 p_dec->fmt_in.video.i_width, p_dec->fmt_in.video.i_height);
313 if (p_dec->fmt_in.i_extra) {
314 // Allocate a byte buffer via allocateDirect in java instead of NewDirectByteBuffer,
315 // since the latter doesn't allocate storage of its own, and we don't know how long
316 // the codec uses the buffer.
317 int buf_size = p_dec->fmt_in.i_extra + 20;
318 jobject bytebuf = (*env)->CallStaticObjectMethod(env, p_sys->byte_buffer_class,
319 p_sys->allocate_direct, buf_size);
320 uint32_t size = p_dec->fmt_in.i_extra;
321 uint8_t *ptr = (*env)->GetDirectBufferAddress(env, bytebuf);
322 if (p_dec->fmt_in.i_codec == VLC_CODEC_H264 && ((uint8_t*)p_dec->fmt_in.p_extra)[0] == 1) {
323 convert_sps_pps(p_dec, p_dec->fmt_in.p_extra, p_dec->fmt_in.i_extra,
325 &size, &p_sys->nal_size);
327 memcpy(ptr, p_dec->fmt_in.p_extra, size);
329 (*env)->CallObjectMethod(env, bytebuf, p_sys->limit, size);
330 (*env)->CallVoidMethod(env, format, p_sys->set_bytebuffer,
331 (*env)->NewStringUTF(env, "csd-0"), bytebuf);
332 (*env)->DeleteLocalRef(env, bytebuf);
335 (*env)->CallVoidMethod(env, p_sys->codec, p_sys->configure, format, NULL, NULL, 0);
336 if ((*env)->ExceptionOccurred(env)) {
337 msg_Warn(p_dec, "Exception occurred in MediaCodec.configure");
338 (*env)->ExceptionClear(env);
341 (*env)->CallVoidMethod(env, p_sys->codec, p_sys->start);
342 if ((*env)->ExceptionOccurred(env)) {
343 msg_Warn(p_dec, "Exception occurred in MediaCodec.start");
344 (*env)->ExceptionClear(env);
349 p_sys->input_buffers = (*env)->CallObjectMethod(env, p_sys->codec, p_sys->get_input_buffers);
350 p_sys->output_buffers = (*env)->CallObjectMethod(env, p_sys->codec, p_sys->get_output_buffers);
351 p_sys->buffer_info = (*env)->NewObject(env, p_sys->buffer_info_class, p_sys->buffer_info_ctor);
352 p_sys->input_buffers = (*env)->NewGlobalRef(env, p_sys->input_buffers);
353 p_sys->output_buffers = (*env)->NewGlobalRef(env, p_sys->output_buffers);
354 p_sys->buffer_info = (*env)->NewGlobalRef(env, p_sys->buffer_info);
355 (*env)->DeleteLocalRef(env, format);
357 (*myVm)->DetachCurrentThread(myVm);
361 (*myVm)->DetachCurrentThread(myVm);
362 CloseDecoder(p_this);
366 static void CloseDecoder(vlc_object_t *p_this)
368 decoder_t *p_dec = (decoder_t *)p_this;
369 decoder_sys_t *p_sys = p_dec->p_sys;
375 (*myVm)->AttachCurrentThread(myVm, &env, NULL);
376 if (p_sys->input_buffers)
377 (*env)->DeleteGlobalRef(env, p_sys->input_buffers);
378 if (p_sys->output_buffers)
379 (*env)->DeleteGlobalRef(env, p_sys->output_buffers);
382 (*env)->CallVoidMethod(env, p_sys->codec, p_sys->stop);
383 (*env)->CallVoidMethod(env, p_sys->codec, p_sys->release);
384 (*env)->DeleteGlobalRef(env, p_sys->codec);
386 if (p_sys->buffer_info)
387 (*env)->DeleteGlobalRef(env, p_sys->buffer_info);
388 (*myVm)->DetachCurrentThread(myVm);
394 static void GetOutput(decoder_t *p_dec, JNIEnv *env, picture_t **pp_pic, int loop)
396 decoder_sys_t *p_sys = p_dec->p_sys;
398 int index = (*env)->CallIntMethod(env, p_sys->codec, p_sys->dequeue_output_buffer,
399 p_sys->buffer_info, (jlong) 0);
401 jobject buf = (*env)->GetObjectArrayElement(env, p_sys->output_buffers, index);
402 jsize buf_size = (*env)->GetDirectBufferCapacity(env, buf);
403 uint8_t *ptr = (*env)->GetDirectBufferAddress(env, buf);
405 *pp_pic = decoder_NewPicture(p_dec);
407 picture_t *p_pic = *pp_pic;
408 int size = (*env)->GetIntField(env, p_sys->buffer_info, p_sys->size_field);
409 int offset = (*env)->GetIntField(env, p_sys->buffer_info, p_sys->offset_field);
410 ptr += offset; // Check the size parameter as well
411 // TODO: Use crop_top/crop_left as well? Or is that already taken into account?
412 // On OMX_TI_COLOR_FormatYUV420PackedSemiPlanar the offset already incldues
413 // the cropping, so the top/left cropping params should just be ignored.
414 unsigned int chroma_div;
415 p_pic->date = (*env)->GetLongField(env, p_sys->buffer_info, p_sys->pts_field);
416 GetVlcChromaSizes(p_dec->fmt_out.i_codec, p_dec->fmt_out.video.i_width,
417 p_dec->fmt_out.video.i_height, NULL, NULL, &chroma_div);
418 CopyOmxPicture(p_sys->pixel_format, p_pic, p_sys->slice_height, p_sys->stride,
421 (*env)->CallVoidMethod(env, p_sys->codec, p_sys->release_output_buffer, index, false);
422 jthrowable exception = (*env)->ExceptionOccurred(env);
423 if(exception != NULL) {
424 jclass illegalStateException = (*env)->FindClass(env, "java/lang/IllegalStateException");
425 if((*env)->IsInstanceOf(env, exception, illegalStateException)) {
426 msg_Err(p_dec, "Codec error (IllegalStateException) in MediaCodec.releaseOutputBuffer");
427 (*env)->ExceptionClear(env);
428 (*env)->DeleteLocalRef(env, illegalStateException);
431 (*env)->DeleteLocalRef(env, buf);
432 } else if (index == INFO_OUTPUT_BUFFERS_CHANGED) {
433 msg_Dbg(p_dec, "output buffers changed");
434 (*env)->DeleteGlobalRef(env, p_sys->output_buffers);
435 p_sys->output_buffers = (*env)->CallObjectMethod(env, p_sys->codec,
436 p_sys->get_output_buffers);
437 p_sys->output_buffers = (*env)->NewGlobalRef(env, p_sys->output_buffers);
439 } else if (index == INFO_OUTPUT_FORMAT_CHANGED) {
441 jobject format = (*env)->CallObjectMethod(env, p_sys->codec, p_sys->get_output_format);
442 jobject format_string = (*env)->CallObjectMethod(env, format, p_sys->tostring);
444 jsize format_len = (*env)->GetStringUTFLength(env, format_string);
445 const char *format_ptr = (*env)->GetStringUTFChars(env, format_string, NULL);
446 msg_Dbg(p_dec, "output format changed: %.*s", format_len, format_ptr);
447 (*env)->ReleaseStringUTFChars(env, format_string, format_ptr);
449 int width = GET_INTEGER(format, "width");
450 int height = GET_INTEGER(format, "height");
451 p_sys->stride = GET_INTEGER(format, "stride");
452 p_sys->slice_height = GET_INTEGER(format, "slice-height");
453 p_sys->pixel_format = GET_INTEGER(format, "color-format");
454 p_sys->crop_left = GET_INTEGER(format, "crop-left");
455 p_sys->crop_top = GET_INTEGER(format, "crop-top");
456 int crop_right = GET_INTEGER(format, "crop-right");
457 int crop_bottom = GET_INTEGER(format, "crop-bottom");
459 const char *name = "unknown";
460 GetVlcChromaFormat(p_sys->pixel_format, &p_dec->fmt_out.i_codec, &name);
461 msg_Dbg(p_dec, "output: %d %s, %dx%d stride %d %d, crop %d %d %d %d",
462 p_sys->pixel_format, name, width, height, p_sys->stride, p_sys->slice_height,
463 p_sys->crop_left, p_sys->crop_top, crop_right, crop_bottom);
465 p_dec->fmt_out.video.i_width = crop_right + 1 - p_sys->crop_left;
466 p_dec->fmt_out.video.i_height = crop_bottom + 1 - p_sys->crop_top;
467 if (p_sys->stride <= 0)
468 p_sys->stride = width;
469 if (p_sys->slice_height <= 0)
470 p_sys->slice_height = height;
471 if ((*env)->ExceptionOccurred(env))
472 (*env)->ExceptionClear(env);
473 if (p_sys->pixel_format == OMX_TI_COLOR_FormatYUV420PackedSemiPlanar) {
474 p_sys->slice_height -= p_sys->crop_top/2;
475 /* Reset crop top/left here, since the offset parameter already includes this.
476 * If we'd ignore the offset parameter in the BufferInfo, we could just keep
477 * the original slice height and apply the top/left cropping instead. */
479 p_sys->crop_left = 0;
481 /* Workaround for some Samsung decoders, the ones named e.g.
482 * OMX.SEC.avc.dec don't have any padding between planes (while
483 * the slice height signals that they would have). The ones
484 * named OMX.SEC.AVC.Decoder have proper slice height as the
485 * parameter indicates. */
486 if (!strncmp(p_sys->name, "OMX.SEC.", strlen("OMX.SEC.")) &&
487 !strstr(p_sys->name, ".Decoder")) {
488 p_sys->slice_height = 0;
489 p_sys->stride = p_dec->fmt_out.video.i_width;
499 static picture_t *DecodeVideo(decoder_t *p_dec, block_t **pp_block)
501 decoder_sys_t *p_sys = p_dec->p_sys;
502 picture_t *p_pic = NULL;
505 if (!pp_block || !*pp_block)
508 block_t *p_block = *pp_block;
510 (*myVm)->AttachCurrentThread(myVm, &env, NULL);
512 if (p_block->i_flags & (BLOCK_FLAG_DISCONTINUITY|BLOCK_FLAG_CORRUPTED)) {
513 block_Release(p_block);
515 (*env)->CallVoidMethod(env, p_sys->codec, p_sys->flush);
517 (*myVm)->DetachCurrentThread(myVm);
523 int index = (*env)->CallIntMethod(env, p_sys->codec, p_sys->dequeue_input_buffer, timeout);
525 GetOutput(p_dec, env, &p_pic, timeout > 0);
529 jobject buf = (*env)->GetObjectArrayElement(env, p_sys->input_buffers, index);
530 jsize size = (*env)->GetDirectBufferCapacity(env, buf);
531 uint8_t *bufptr = (*env)->GetDirectBufferAddress(env, buf);
532 if (size > p_block->i_buffer)
533 size = p_block->i_buffer;
534 memcpy(bufptr, p_block->p_buffer, size);
536 convert_h264_to_annexb(bufptr, size, p_sys->nal_size);
538 int64_t ts = p_block->i_pts;
539 if (!ts && p_block->i_dts)
541 (*env)->CallVoidMethod(env, p_sys->codec, p_sys->queue_input_buffer, index, 0, size, ts, 0);
542 (*env)->DeleteLocalRef(env, buf);
547 GetOutput(p_dec, env, &p_pic, 0);
548 (*myVm)->DetachCurrentThread(myVm);
550 block_Release(p_block);