1 /*****************************************************************************
2 * android_mediacodec.c: Video decoder module using the Android MediaCodec API
3 *****************************************************************************
4 * Copyright (C) 2012 Martin Storsjo
6 * Authors: Martin Storsjo <martin@martin.st>
8 * This program is free software; you can redistribute it and/or modify it
9 * under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation; either version 2.1 of the License, or
11 * (at your option) any later version.
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with this program; if not, write to the Free Software Foundation,
20 * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
21 *****************************************************************************/
23 /*****************************************************************************
25 *****************************************************************************/
33 #include <vlc_common.h>
34 #include <vlc_plugin.h>
35 #include <vlc_codec.h>
36 #include <vlc_block_helper.h>
39 #include "../h264_nal.h"
41 #include <OMX_Component.h>
42 #include "omxil_utils.h"
44 #define INFO_OUTPUT_BUFFERS_CHANGED -3
45 #define INFO_OUTPUT_FORMAT_CHANGED -2
46 #define INFO_TRY_AGAIN_LATER -1
52 jclass media_codec_list_class, media_codec_class, media_format_class;
53 jclass buffer_info_class, byte_buffer_class;
55 jmethodID get_codec_count, get_codec_info_at, is_encoder;
56 jmethodID get_supported_types, get_name;
57 jmethodID create_by_codec_name, configure, start, stop, flush, release;
58 jmethodID get_output_format, get_input_buffers, get_output_buffers;
59 jmethodID dequeue_input_buffer, dequeue_output_buffer, queue_input_buffer;
60 jmethodID release_output_buffer;
61 jmethodID create_video_format, set_integer, set_bytebuffer, get_integer;
62 jmethodID buffer_info_ctor;
63 jmethodID allocate_direct, limit;
64 jfieldID size_field, offset_field, pts_field;
70 jobject input_buffers, output_buffers;
72 int stride, slice_height;
73 int crop_top, crop_left;
82 METHOD, STATIC_METHOD, FIELD
85 #define OFF(x) offsetof(struct decoder_sys_t, x)
91 static const struct classname classes[] = {
92 { "android/media/MediaCodecList", OFF(media_codec_list_class) },
93 { "android/media/MediaCodec", OFF(media_codec_class) },
94 { "android/media/MediaFormat", OFF(media_format_class) },
95 { "android/media/MediaFormat", OFF(media_format_class) },
96 { "android/media/MediaCodec$BufferInfo", OFF(buffer_info_class) },
97 { "java/nio/ByteBuffer", OFF(byte_buffer_class) },
109 static const struct member members[] = {
110 { "toString", "()Ljava/lang/String;", "java/lang/Object", OFF(tostring), METHOD },
112 { "getCodecCount", "()I", "android/media/MediaCodecList", OFF(get_codec_count), STATIC_METHOD },
113 { "getCodecInfoAt", "(I)Landroid/media/MediaCodecInfo;", "android/media/MediaCodecList", OFF(get_codec_info_at), STATIC_METHOD },
115 { "isEncoder", "()Z", "android/media/MediaCodecInfo", OFF(is_encoder), METHOD },
116 { "getSupportedTypes", "()[Ljava/lang/String;", "android/media/MediaCodecInfo", OFF(get_supported_types), METHOD },
117 { "getName", "()Ljava/lang/String;", "android/media/MediaCodecInfo", OFF(get_name), METHOD },
119 { "createByCodecName", "(Ljava/lang/String;)Landroid/media/MediaCodec;", "android/media/MediaCodec", OFF(create_by_codec_name), STATIC_METHOD },
120 { "configure", "(Landroid/media/MediaFormat;Landroid/view/Surface;Landroid/media/MediaCrypto;I)V", "android/media/MediaCodec", OFF(configure), METHOD },
121 { "start", "()V", "android/media/MediaCodec", OFF(start), METHOD },
122 { "stop", "()V", "android/media/MediaCodec", OFF(stop), METHOD },
123 { "flush", "()V", "android/media/MediaCodec", OFF(flush), METHOD },
124 { "release", "()V", "android/media/MediaCodec", OFF(release), METHOD },
125 { "getOutputFormat", "()Landroid/media/MediaFormat;", "android/media/MediaCodec", OFF(get_output_format), METHOD },
126 { "getInputBuffers", "()[Ljava/nio/ByteBuffer;", "android/media/MediaCodec", OFF(get_input_buffers), METHOD },
127 { "getOutputBuffers", "()[Ljava/nio/ByteBuffer;", "android/media/MediaCodec", OFF(get_output_buffers), METHOD },
128 { "dequeueInputBuffer", "(J)I", "android/media/MediaCodec", OFF(dequeue_input_buffer), METHOD },
129 { "dequeueOutputBuffer", "(Landroid/media/MediaCodec$BufferInfo;J)I", "android/media/MediaCodec", OFF(dequeue_output_buffer), METHOD },
130 { "queueInputBuffer", "(IIIJI)V", "android/media/MediaCodec", OFF(queue_input_buffer), METHOD },
131 { "releaseOutputBuffer", "(IZ)V", "android/media/MediaCodec", OFF(release_output_buffer), METHOD },
133 { "createVideoFormat", "(Ljava/lang/String;II)Landroid/media/MediaFormat;", "android/media/MediaFormat", OFF(create_video_format), STATIC_METHOD },
134 { "setInteger", "(Ljava/lang/String;I)V", "android/media/MediaFormat", OFF(set_integer), METHOD },
135 { "getInteger", "(Ljava/lang/String;)I", "android/media/MediaFormat", OFF(get_integer), METHOD },
136 { "setByteBuffer", "(Ljava/lang/String;Ljava/nio/ByteBuffer;)V", "android/media/MediaFormat", OFF(set_bytebuffer), METHOD },
138 { "<init>", "()V", "android/media/MediaCodec$BufferInfo", OFF(buffer_info_ctor), METHOD },
139 { "size", "I", "android/media/MediaCodec$BufferInfo", OFF(size_field), FIELD },
140 { "offset", "I", "android/media/MediaCodec$BufferInfo", OFF(offset_field), FIELD },
141 { "presentationTimeUs", "J", "android/media/MediaCodec$BufferInfo", OFF(pts_field), FIELD },
143 { "allocateDirect", "(I)Ljava/nio/ByteBuffer;", "java/nio/ByteBuffer", OFF(allocate_direct), STATIC_METHOD },
144 { "limit", "(I)Ljava/nio/Buffer;", "java/nio/ByteBuffer", OFF(limit), METHOD },
146 { NULL, NULL, NULL, 0, 0 },
149 #define GET_INTEGER(obj, name) (*env)->CallIntMethod(env, obj, p_sys->get_integer, (*env)->NewStringUTF(env, name))
151 /*****************************************************************************
153 *****************************************************************************/
154 static int OpenDecoder(vlc_object_t *);
155 static void CloseDecoder(vlc_object_t *);
157 static picture_t *DecodeVideo(decoder_t *, block_t **);
159 /*****************************************************************************
161 *****************************************************************************/
163 set_description( N_("Video decoder using Android MediaCodec") )
164 set_category( CAT_INPUT )
165 set_subcategory( SUBCAT_INPUT_VCODEC )
166 set_section( N_("Decoding") , NULL )
167 set_capability( "decoder", 0 ) /* Only enabled via commandline arguments */
168 set_callbacks( OpenDecoder, CloseDecoder )
171 static int jstrcmp(JNIEnv* env, jobject str, const char* str2)
173 jsize len = (*env)->GetStringUTFLength(env, str);
174 if (len != (jsize) strlen(str2))
176 const char *ptr = (*env)->GetStringUTFChars(env, str, NULL);
177 int ret = memcmp(ptr, str2, len);
178 (*env)->ReleaseStringUTFChars(env, str, ptr);
182 /*****************************************************************************
183 * OpenDecoder: Create the decoder instance
184 *****************************************************************************/
185 static int OpenDecoder(vlc_object_t *p_this)
187 decoder_t *p_dec = (decoder_t*)p_this;
188 decoder_sys_t *p_sys;
190 if (p_dec->fmt_in.i_cat != VIDEO_ES && !p_dec->b_force)
193 const char *mime = NULL;
194 switch (p_dec->fmt_in.i_codec) {
195 case VLC_CODEC_H264: mime = "video/avc"; break;
196 case VLC_CODEC_H263: mime = "video/3gpp"; break;
197 case VLC_CODEC_MP4V: mime = "video/mp4v-es"; break;
199 msg_Dbg(p_dec, "codec %d not supported", p_dec->fmt_in.i_codec);
202 /* Allocate the memory needed to store the decoder's structure */
203 if ((p_dec->p_sys = p_sys = calloc(1, sizeof(*p_sys))) == NULL)
206 p_dec->pf_decode_video = DecodeVideo;
209 p_dec->fmt_out.i_cat = p_dec->fmt_in.i_cat;
210 p_dec->fmt_out.video = p_dec->fmt_in.video;
211 p_dec->fmt_out.audio = p_dec->fmt_in.audio;
212 p_dec->b_need_packetized = true;
215 (*myVm)->AttachCurrentThread(myVm, &env, NULL);
217 for (int i = 0; classes[i].name; i++) {
218 *(jclass*)((uint8_t*)p_sys + classes[i].offset) =
219 (*env)->FindClass(env, classes[i].name);
221 if ((*env)->ExceptionOccurred(env)) {
222 msg_Warn(p_dec, "Unable to find class %s", classes[i].name);
223 (*env)->ExceptionClear(env);
229 for (int i = 0; members[i].name; i++) {
230 if (i == 0 || strcmp(members[i].class, members[i - 1].class))
231 last_class = (*env)->FindClass(env, members[i].class);
233 if ((*env)->ExceptionOccurred(env)) {
234 msg_Warn(p_dec, "Unable to find class %s", members[i].class);
235 (*env)->ExceptionClear(env);
239 switch (members[i].type) {
241 *(jmethodID*)((uint8_t*)p_sys + members[i].offset) =
242 (*env)->GetMethodID(env, last_class, members[i].name, members[i].sig);
245 *(jmethodID*)((uint8_t*)p_sys + members[i].offset) =
246 (*env)->GetStaticMethodID(env, last_class, members[i].name, members[i].sig);
249 *(jfieldID*)((uint8_t*)p_sys + members[i].offset) =
250 (*env)->GetFieldID(env, last_class, members[i].name, members[i].sig);
253 if ((*env)->ExceptionOccurred(env)) {
254 msg_Warn(p_dec, "Unable to find the member %s in %s",
255 members[i].name, members[i].class);
256 (*env)->ExceptionClear(env);
261 int num_codecs = (*env)->CallStaticIntMethod(env, p_sys->media_codec_list_class,
262 p_sys->get_codec_count);
263 jobject codec_info = NULL, codec_name = NULL;
265 for (int i = 0; i < num_codecs; i++) {
266 jobject info = (*env)->CallStaticObjectMethod(env, p_sys->media_codec_list_class,
267 p_sys->get_codec_info_at, i);
268 if ((*env)->CallBooleanMethod(env, info, p_sys->is_encoder)) {
269 (*env)->DeleteLocalRef(env, info);
272 jobject types = (*env)->CallObjectMethod(env, info, p_sys->get_supported_types);
273 int num_types = (*env)->GetArrayLength(env, types);
275 for (int j = 0; j < num_types && !found; j++) {
276 jobject type = (*env)->GetObjectArrayElement(env, types, j);
277 if (!jstrcmp(env, type, mime))
279 (*env)->DeleteLocalRef(env, type);
282 jobject name = (*env)->CallObjectMethod(env, info, p_sys->get_name);
283 jsize name_len = (*env)->GetStringUTFLength(env, name);
284 const char *name_ptr = (*env)->GetStringUTFChars(env, name, NULL);
285 msg_Dbg(p_dec, "using %.*s", name_len, name_ptr);
286 p_sys->name = malloc(name_len + 1);
287 memcpy(p_sys->name, name_ptr, name_len);
288 p_sys->name[name_len] = '\0';
289 (*env)->ReleaseStringUTFChars(env, name, name_ptr);
294 (*env)->DeleteLocalRef(env, info);
298 msg_Dbg(p_dec, "No suitable codec matching %s was found", mime);
302 // This method doesn't handle errors nicely, it crashes if the codec isn't found.
303 // (The same goes for createDecoderByType.) This is fixed in latest AOSP, but not
304 // in current 4.1 devices.
305 p_sys->codec = (*env)->CallStaticObjectMethod(env, p_sys->media_codec_class,
306 p_sys->create_by_codec_name, codec_name);
307 p_sys->codec = (*env)->NewGlobalRef(env, p_sys->codec);
309 jobject format = (*env)->CallStaticObjectMethod(env, p_sys->media_format_class,
310 p_sys->create_video_format, (*env)->NewStringUTF(env, mime),
311 p_dec->fmt_in.video.i_width, p_dec->fmt_in.video.i_height);
313 if (p_dec->fmt_in.i_extra) {
314 // Allocate a byte buffer via allocateDirect in java instead of NewDirectByteBuffer,
315 // since the latter doesn't allocate storage of its own, and we don't know how long
316 // the codec uses the buffer.
317 int buf_size = p_dec->fmt_in.i_extra + 20;
318 jobject bytebuf = (*env)->CallStaticObjectMethod(env, p_sys->byte_buffer_class,
319 p_sys->allocate_direct, buf_size);
320 uint32_t size = p_dec->fmt_in.i_extra;
321 uint8_t *ptr = (*env)->GetDirectBufferAddress(env, bytebuf);
322 if (p_dec->fmt_in.i_codec == VLC_CODEC_H264 && ((uint8_t*)p_dec->fmt_in.p_extra)[0] == 1) {
323 convert_sps_pps(p_dec, p_dec->fmt_in.p_extra, p_dec->fmt_in.i_extra,
325 &size, &p_sys->nal_size);
327 memcpy(ptr, p_dec->fmt_in.p_extra, size);
329 (*env)->CallObjectMethod(env, bytebuf, p_sys->limit, size);
330 (*env)->CallVoidMethod(env, format, p_sys->set_bytebuffer,
331 (*env)->NewStringUTF(env, "csd-0"), bytebuf);
332 (*env)->DeleteLocalRef(env, bytebuf);
335 (*env)->CallVoidMethod(env, p_sys->codec, p_sys->configure, format, NULL, NULL, 0);
336 if ((*env)->ExceptionOccurred(env)) {
337 (*env)->ExceptionClear(env);
340 (*env)->CallVoidMethod(env, p_sys->codec, p_sys->start);
343 p_sys->input_buffers = (*env)->CallObjectMethod(env, p_sys->codec, p_sys->get_input_buffers);
344 p_sys->output_buffers = (*env)->CallObjectMethod(env, p_sys->codec, p_sys->get_output_buffers);
345 p_sys->buffer_info = (*env)->NewObject(env, p_sys->buffer_info_class, p_sys->buffer_info_ctor);
346 p_sys->input_buffers = (*env)->NewGlobalRef(env, p_sys->input_buffers);
347 p_sys->output_buffers = (*env)->NewGlobalRef(env, p_sys->output_buffers);
348 p_sys->buffer_info = (*env)->NewGlobalRef(env, p_sys->buffer_info);
349 (*env)->DeleteLocalRef(env, format);
351 (*myVm)->DetachCurrentThread(myVm);
355 (*myVm)->DetachCurrentThread(myVm);
356 CloseDecoder(p_this);
360 static void CloseDecoder(vlc_object_t *p_this)
362 decoder_t *p_dec = (decoder_t *)p_this;
363 decoder_sys_t *p_sys = p_dec->p_sys;
369 (*myVm)->AttachCurrentThread(myVm, &env, NULL);
370 if (p_sys->input_buffers)
371 (*env)->DeleteGlobalRef(env, p_sys->input_buffers);
372 if (p_sys->output_buffers)
373 (*env)->DeleteGlobalRef(env, p_sys->output_buffers);
376 (*env)->CallVoidMethod(env, p_sys->codec, p_sys->stop);
377 (*env)->CallVoidMethod(env, p_sys->codec, p_sys->release);
378 (*env)->DeleteGlobalRef(env, p_sys->codec);
380 if (p_sys->buffer_info)
381 (*env)->DeleteGlobalRef(env, p_sys->buffer_info);
382 (*myVm)->DetachCurrentThread(myVm);
388 static void GetOutput(decoder_t *p_dec, JNIEnv *env, picture_t **pp_pic, int loop)
390 decoder_sys_t *p_sys = p_dec->p_sys;
392 int index = (*env)->CallIntMethod(env, p_sys->codec, p_sys->dequeue_output_buffer,
393 p_sys->buffer_info, (jlong) 0);
395 jobject buf = (*env)->GetObjectArrayElement(env, p_sys->output_buffers, index);
396 jsize buf_size = (*env)->GetDirectBufferCapacity(env, buf);
397 uint8_t *ptr = (*env)->GetDirectBufferAddress(env, buf);
399 *pp_pic = decoder_NewPicture(p_dec);
401 picture_t *p_pic = *pp_pic;
402 int size = (*env)->GetIntField(env, p_sys->buffer_info, p_sys->size_field);
403 int offset = (*env)->GetIntField(env, p_sys->buffer_info, p_sys->offset_field);
404 ptr += offset; // Check the size parameter as well
405 // TODO: Use crop_top/crop_left as well? Or is that already taken into account?
406 // On OMX_TI_COLOR_FormatYUV420PackedSemiPlanar the offset already incldues
407 // the cropping, so the top/left cropping params should just be ignored.
408 unsigned int chroma_div;
409 p_pic->date = (*env)->GetLongField(env, p_sys->buffer_info, p_sys->pts_field);
410 GetVlcChromaSizes(p_dec->fmt_out.i_codec, p_dec->fmt_out.video.i_width,
411 p_dec->fmt_out.video.i_height, NULL, NULL, &chroma_div);
412 CopyOmxPicture(p_sys->pixel_format, p_pic, p_sys->slice_height, p_sys->stride,
415 (*env)->CallVoidMethod(env, p_sys->codec, p_sys->release_output_buffer, index, false);
416 (*env)->DeleteLocalRef(env, buf);
417 } else if (index == INFO_OUTPUT_BUFFERS_CHANGED) {
418 msg_Dbg(p_dec, "output buffers changed");
419 (*env)->DeleteGlobalRef(env, p_sys->output_buffers);
420 p_sys->output_buffers = (*env)->CallObjectMethod(env, p_sys->codec,
421 p_sys->get_output_buffers);
422 p_sys->output_buffers = (*env)->NewGlobalRef(env, p_sys->output_buffers);
424 } else if (index == INFO_OUTPUT_FORMAT_CHANGED) {
426 jobject format = (*env)->CallObjectMethod(env, p_sys->codec, p_sys->get_output_format);
427 jobject format_string = (*env)->CallObjectMethod(env, format, p_sys->tostring);
429 jsize format_len = (*env)->GetStringUTFLength(env, format_string);
430 const char *format_ptr = (*env)->GetStringUTFChars(env, format_string, NULL);
431 msg_Dbg(p_dec, "output format changed: %.*s", format_len, format_ptr);
432 (*env)->ReleaseStringUTFChars(env, format_string, format_ptr);
434 int width = GET_INTEGER(format, "width");
435 int height = GET_INTEGER(format, "height");
436 p_sys->stride = GET_INTEGER(format, "stride");
437 p_sys->slice_height = GET_INTEGER(format, "slice-height");
438 p_sys->pixel_format = GET_INTEGER(format, "color-format");
439 p_sys->crop_left = GET_INTEGER(format, "crop-left");
440 p_sys->crop_top = GET_INTEGER(format, "crop-top");
441 int crop_right = GET_INTEGER(format, "crop-right");
442 int crop_bottom = GET_INTEGER(format, "crop-bottom");
444 const char *name = "unknown";
445 GetVlcChromaFormat(p_sys->pixel_format, &p_dec->fmt_out.i_codec, &name);
446 msg_Dbg(p_dec, "output: %d %s, %dx%d stride %d %d, crop %d %d %d %d",
447 p_sys->pixel_format, name, width, height, p_sys->stride, p_sys->slice_height,
448 p_sys->crop_left, p_sys->crop_top, crop_right, crop_bottom);
450 p_dec->fmt_out.video.i_width = crop_right + 1 - p_sys->crop_left;
451 p_dec->fmt_out.video.i_height = crop_bottom + 1 - p_sys->crop_top;
452 if (p_sys->stride <= 0)
453 p_sys->stride = width;
454 if (p_sys->slice_height <= 0)
455 p_sys->slice_height = height;
456 if ((*env)->ExceptionOccurred(env))
457 (*env)->ExceptionClear(env);
458 if (p_sys->pixel_format == OMX_TI_COLOR_FormatYUV420PackedSemiPlanar) {
459 p_sys->slice_height -= p_sys->crop_top/2;
460 /* Reset crop top/left here, since the offset parameter already includes this.
461 * If we'd ignore the offset parameter in the BufferInfo, we could just keep
462 * the original slice height and apply the top/left cropping instead. */
464 p_sys->crop_left = 0;
466 /* Workaround for some Samsung decoders, the ones named e.g.
467 * OMX.SEC.avc.dec don't have any padding between planes (while
468 * the slice height signals that they would have). The ones
469 * named OMX.SEC.AVC.Decoder have proper slice height as the
470 * parameter indicates. */
471 if (!strncmp(p_sys->name, "OMX.SEC.", strlen("OMX.SEC.")) &&
472 !strstr(p_sys->name, ".Decoder")) {
473 p_sys->slice_height = 0;
483 static picture_t *DecodeVideo(decoder_t *p_dec, block_t **pp_block)
485 decoder_sys_t *p_sys = p_dec->p_sys;
486 picture_t *p_pic = NULL;
489 if (!pp_block || !*pp_block)
492 block_t *p_block = *pp_block;
494 (*myVm)->AttachCurrentThread(myVm, &env, NULL);
496 if (p_block->i_flags & (BLOCK_FLAG_DISCONTINUITY|BLOCK_FLAG_CORRUPTED)) {
497 block_Release(p_block);
499 (*env)->CallVoidMethod(env, p_sys->codec, p_sys->flush);
501 (*myVm)->DetachCurrentThread(myVm);
507 int index = (*env)->CallIntMethod(env, p_sys->codec, p_sys->dequeue_input_buffer, timeout);
509 GetOutput(p_dec, env, &p_pic, timeout > 0);
513 jobject buf = (*env)->GetObjectArrayElement(env, p_sys->input_buffers, index);
514 jsize size = (*env)->GetDirectBufferCapacity(env, buf);
515 uint8_t *bufptr = (*env)->GetDirectBufferAddress(env, buf);
516 if (size > p_block->i_buffer)
517 size = p_block->i_buffer;
518 memcpy(bufptr, p_block->p_buffer, size);
520 convert_h264_to_annexb(bufptr, size, p_sys->nal_size);
522 int64_t ts = p_block->i_pts;
523 if (!ts && p_block->i_dts)
525 (*env)->CallVoidMethod(env, p_sys->codec, p_sys->queue_input_buffer, index, 0, size, ts, 0);
526 (*env)->DeleteLocalRef(env, buf);
531 GetOutput(p_dec, env, &p_pic, 0);
532 (*myVm)->DetachCurrentThread(myVm);
534 block_Release(p_block);