1 /*****************************************************************************
2 * android_mediacodec.c: Video decoder module using the Android MediaCodec API
3 *****************************************************************************
4 * Copyright (C) 2012 Martin Storsjo
6 * Authors: Martin Storsjo <martin@martin.st>
8 * This program is free software; you can redistribute it and/or modify it
9 * under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation; either version 2.1 of the License, or
11 * (at your option) any later version.
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with this program; if not, write to the Free Software Foundation,
20 * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
21 *****************************************************************************/
23 /*****************************************************************************
25 *****************************************************************************/
33 #include <vlc_common.h>
34 #include <vlc_plugin.h>
35 #include <vlc_codec.h>
36 #include <vlc_block_helper.h>
39 #include "../h264_nal.h"
40 #include "../hevc_nal.h"
42 #include <OMX_Component.h>
43 #include "omxil_utils.h"
44 #include "android_opaque.h"
45 #include "../../video_output/android/android_window.h"
47 #define INFO_OUTPUT_BUFFERS_CHANGED -3
48 #define INFO_OUTPUT_FORMAT_CHANGED -2
49 #define INFO_TRY_AGAIN_LATER -1
51 #define THREAD_NAME "android_mediacodec"
53 extern int jni_attach_thread(JNIEnv **env, const char *thread_name);
54 extern void jni_detach_thread();
55 /* JNI functions to get/set an Android Surface object. */
56 extern jobject jni_LockAndGetAndroidJavaSurface();
57 extern void jni_UnlockAndroidSurface();
58 extern void jni_EventHardwareAccelerationError();
59 extern bool jni_IsVideoPlayerActivityCreated();
61 /* Implementation of a circular buffer of timestamps with overwriting
62 * of older values. MediaCodec has only one type of timestamp, if a
63 * block has no PTS, we send the DTS instead. Some hardware decoders
64 * cannot cope with this situation and output the frames in the wrong
65 * order. As a workaround in this case, we use a FIFO of timestamps in
66 * order to remember which input packets had no PTS. Since an
67 * hardware decoder can silently drop frames, this might cause a
68 * growing desynchronization with the actual timestamp. Thus the
69 * circular buffer has a limited size and will overwrite older values.
79 static timestamp_fifo_t *timestamp_FifoNew(uint32_t capacity)
81 timestamp_fifo_t *fifo = calloc(1, sizeof(*fifo));
84 fifo->buffer = malloc(capacity * sizeof(*fifo->buffer));
89 fifo->capacity = capacity;
93 static void timestamp_FifoRelease(timestamp_fifo_t *fifo)
99 static bool timestamp_FifoIsEmpty(timestamp_fifo_t *fifo)
101 return fifo->size == 0;
104 static bool timestamp_FifoIsFull(timestamp_fifo_t *fifo)
106 return fifo->size == fifo->capacity;
109 static void timestamp_FifoEmpty(timestamp_fifo_t *fifo)
114 static void timestamp_FifoPut(timestamp_fifo_t *fifo, int64_t ts)
116 uint32_t end = (fifo->begin + fifo->size) % fifo->capacity;
117 fifo->buffer[end] = ts;
118 if (!timestamp_FifoIsFull(fifo))
121 fifo->begin = (fifo->begin + 1) % fifo->capacity;
124 static int64_t timestamp_FifoGet(timestamp_fifo_t *fifo)
126 if (timestamp_FifoIsEmpty(fifo))
127 return VLC_TS_INVALID;
129 int64_t result = fifo->buffer[fifo->begin];
130 fifo->begin = (fifo->begin + 1) % fifo->capacity;
137 jclass media_codec_list_class, media_codec_class, media_format_class;
138 jclass buffer_info_class, byte_buffer_class;
140 jmethodID get_codec_count, get_codec_info_at, is_encoder, get_capabilities_for_type;
141 jfieldID profile_levels_field, profile_field, level_field;
142 jmethodID get_supported_types, get_name;
143 jmethodID create_by_codec_name, configure, start, stop, flush, release;
144 jmethodID get_output_format;
145 jmethodID get_input_buffers, get_input_buffer;
146 jmethodID get_output_buffers, get_output_buffer;
147 jmethodID dequeue_input_buffer, dequeue_output_buffer, queue_input_buffer;
148 jmethodID release_output_buffer;
149 jmethodID create_video_format, set_integer, set_bytebuffer, get_integer;
150 jmethodID buffer_info_ctor;
151 jmethodID allocate_direct, limit;
152 jfieldID size_field, offset_field, pts_field;
158 jobject input_buffers, output_buffers;
160 int stride, slice_height;
167 bool error_event_sent;
169 ArchitectureSpecificCopyData architecture_specific_data;
171 /* Direct rendering members. */
172 bool direct_rendering;
173 picture_t** pp_inflight_pictures; /**< stores the inflight picture for each output buffer or NULL */
174 unsigned int i_inflight_pictures;
176 timestamp_fifo_t *timestamp_fifo;
178 int64_t i_preroll_end;
183 METHOD, STATIC_METHOD, FIELD
186 #define OFF(x) offsetof(struct decoder_sys_t, x)
192 static const struct classname classes[] = {
193 { "android/media/MediaCodecList", OFF(media_codec_list_class) },
194 { "android/media/MediaCodec", OFF(media_codec_class) },
195 { "android/media/MediaFormat", OFF(media_format_class) },
196 { "android/media/MediaFormat", OFF(media_format_class) },
197 { "android/media/MediaCodec$BufferInfo", OFF(buffer_info_class) },
198 { "java/nio/ByteBuffer", OFF(byte_buffer_class) },
211 static const struct member members[] = {
212 { "toString", "()Ljava/lang/String;", "java/lang/Object", OFF(tostring), METHOD, true },
214 { "getCodecCount", "()I", "android/media/MediaCodecList", OFF(get_codec_count), STATIC_METHOD, true },
215 { "getCodecInfoAt", "(I)Landroid/media/MediaCodecInfo;", "android/media/MediaCodecList", OFF(get_codec_info_at), STATIC_METHOD, true },
217 { "isEncoder", "()Z", "android/media/MediaCodecInfo", OFF(is_encoder), METHOD, true },
218 { "getSupportedTypes", "()[Ljava/lang/String;", "android/media/MediaCodecInfo", OFF(get_supported_types), METHOD, true },
219 { "getName", "()Ljava/lang/String;", "android/media/MediaCodecInfo", OFF(get_name), METHOD, true },
220 { "getCapabilitiesForType", "(Ljava/lang/String;)Landroid/media/MediaCodecInfo$CodecCapabilities;", "android/media/MediaCodecInfo", OFF(get_capabilities_for_type), METHOD, true },
222 { "profileLevels", "[Landroid/media/MediaCodecInfo$CodecProfileLevel;", "android/media/MediaCodecInfo$CodecCapabilities", OFF(profile_levels_field), FIELD, true },
223 { "profile", "I", "android/media/MediaCodecInfo$CodecProfileLevel", OFF(profile_field), FIELD, true },
224 { "level", "I", "android/media/MediaCodecInfo$CodecProfileLevel", OFF(level_field), FIELD, true },
226 { "createByCodecName", "(Ljava/lang/String;)Landroid/media/MediaCodec;", "android/media/MediaCodec", OFF(create_by_codec_name), STATIC_METHOD, true },
227 { "configure", "(Landroid/media/MediaFormat;Landroid/view/Surface;Landroid/media/MediaCrypto;I)V", "android/media/MediaCodec", OFF(configure), METHOD, true },
228 { "start", "()V", "android/media/MediaCodec", OFF(start), METHOD, true },
229 { "stop", "()V", "android/media/MediaCodec", OFF(stop), METHOD, true },
230 { "flush", "()V", "android/media/MediaCodec", OFF(flush), METHOD, true },
231 { "release", "()V", "android/media/MediaCodec", OFF(release), METHOD, true },
232 { "getOutputFormat", "()Landroid/media/MediaFormat;", "android/media/MediaCodec", OFF(get_output_format), METHOD, true },
233 { "getInputBuffers", "()[Ljava/nio/ByteBuffer;", "android/media/MediaCodec", OFF(get_input_buffers), METHOD, false },
234 { "getInputBuffer", "(I)Ljava/nio/ByteBuffer;", "android/media/MediaCodec", OFF(get_input_buffer), METHOD, false },
235 { "getOutputBuffers", "()[Ljava/nio/ByteBuffer;", "android/media/MediaCodec", OFF(get_output_buffers), METHOD, false },
236 { "getOutputBuffer", "(I)Ljava/nio/ByteBuffer;", "android/media/MediaCodec", OFF(get_output_buffer), METHOD, false },
237 { "dequeueInputBuffer", "(J)I", "android/media/MediaCodec", OFF(dequeue_input_buffer), METHOD, true },
238 { "dequeueOutputBuffer", "(Landroid/media/MediaCodec$BufferInfo;J)I", "android/media/MediaCodec", OFF(dequeue_output_buffer), METHOD, true },
239 { "queueInputBuffer", "(IIIJI)V", "android/media/MediaCodec", OFF(queue_input_buffer), METHOD, true },
240 { "releaseOutputBuffer", "(IZ)V", "android/media/MediaCodec", OFF(release_output_buffer), METHOD, true },
242 { "createVideoFormat", "(Ljava/lang/String;II)Landroid/media/MediaFormat;", "android/media/MediaFormat", OFF(create_video_format), STATIC_METHOD, true },
243 { "setInteger", "(Ljava/lang/String;I)V", "android/media/MediaFormat", OFF(set_integer), METHOD, true },
244 { "getInteger", "(Ljava/lang/String;)I", "android/media/MediaFormat", OFF(get_integer), METHOD, true },
245 { "setByteBuffer", "(Ljava/lang/String;Ljava/nio/ByteBuffer;)V", "android/media/MediaFormat", OFF(set_bytebuffer), METHOD, true },
247 { "<init>", "()V", "android/media/MediaCodec$BufferInfo", OFF(buffer_info_ctor), METHOD, true },
248 { "size", "I", "android/media/MediaCodec$BufferInfo", OFF(size_field), FIELD, true },
249 { "offset", "I", "android/media/MediaCodec$BufferInfo", OFF(offset_field), FIELD, true },
250 { "presentationTimeUs", "J", "android/media/MediaCodec$BufferInfo", OFF(pts_field), FIELD, true },
252 { "allocateDirect", "(I)Ljava/nio/ByteBuffer;", "java/nio/ByteBuffer", OFF(allocate_direct), STATIC_METHOD, true },
253 { "limit", "(I)Ljava/nio/Buffer;", "java/nio/ByteBuffer", OFF(limit), METHOD, true },
255 { NULL, NULL, NULL, 0, 0, false },
258 #define GET_INTEGER(obj, name) (*env)->CallIntMethod(env, obj, p_sys->get_integer, (*env)->NewStringUTF(env, name))
260 /*****************************************************************************
262 *****************************************************************************/
263 static int OpenDecoder(vlc_object_t *);
264 static void CloseDecoder(vlc_object_t *);
266 static picture_t *DecodeVideo(decoder_t *, block_t **);
268 static void InvalidateAllPictures(decoder_t *);
269 static int InsertInflightPicture(decoder_t *, picture_t *, unsigned int );
271 /*****************************************************************************
273 *****************************************************************************/
274 #define DIRECTRENDERING_TEXT N_("Android direct rendering")
275 #define DIRECTRENDERING_LONGTEXT N_(\
276 "Enable Android direct rendering using opaque buffers.")
278 #define CFG_PREFIX "mediacodec-"
281 set_description( N_("Video decoder using Android MediaCodec") )
282 set_category( CAT_INPUT )
283 set_subcategory( SUBCAT_INPUT_VCODEC )
284 set_section( N_("Decoding") , NULL )
285 set_capability( "decoder", 0 ) /* Only enabled via commandline arguments */
286 add_bool(CFG_PREFIX "dr", true,
287 DIRECTRENDERING_TEXT, DIRECTRENDERING_LONGTEXT, true)
288 set_callbacks( OpenDecoder, CloseDecoder )
291 static int jstrcmp(JNIEnv* env, jobject str, const char* str2)
293 jsize len = (*env)->GetStringUTFLength(env, str);
294 if (len != (jsize) strlen(str2))
296 const char *ptr = (*env)->GetStringUTFChars(env, str, NULL);
297 int ret = memcmp(ptr, str2, len);
298 (*env)->ReleaseStringUTFChars(env, str, ptr);
302 static inline bool check_exception( JNIEnv *env )
304 if ((*env)->ExceptionOccurred(env)) {
305 (*env)->ExceptionClear(env);
311 #define CHECK_EXCEPTION() check_exception( env )
313 static bool codec_is_blacklisted( const char *p_name, int i_name_len )
315 static const char *blacklisted_codecs[] = {
316 /* software decoders */
318 /* crashes mediaserver */
319 "OMX.MTK.VIDEO.DECODER.MPEG4",
323 for( const char **pp_bl_codecs = blacklisted_codecs; *pp_bl_codecs != NULL;
326 if( !strncmp( p_name, *pp_bl_codecs,
327 __MIN( strlen(*pp_bl_codecs), i_name_len ) ) )
333 /*****************************************************************************
334 * OpenDecoder: Create the decoder instance
335 *****************************************************************************/
336 static int OpenDecoder(vlc_object_t *p_this)
338 decoder_t *p_dec = (decoder_t*)p_this;
339 decoder_sys_t *p_sys;
341 if (p_dec->fmt_in.i_cat != VIDEO_ES && !p_dec->b_force)
344 const char *mime = NULL;
345 switch (p_dec->fmt_in.i_codec) {
346 case VLC_CODEC_HEVC: mime = "video/hevc"; break;
347 case VLC_CODEC_H264: mime = "video/avc"; break;
348 case VLC_CODEC_H263: mime = "video/3gpp"; break;
349 case VLC_CODEC_MP4V: mime = "video/mp4v-es"; break;
350 case VLC_CODEC_WMV3: mime = "video/x-ms-wmv"; break;
351 case VLC_CODEC_VC1: mime = "video/wvc1"; break;
352 case VLC_CODEC_VP8: mime = "video/x-vnd.on2.vp8"; break;
353 case VLC_CODEC_VP9: mime = "video/x-vnd.on2.vp9"; break;
355 msg_Dbg(p_dec, "codec %4.4s not supported", (char *)&p_dec->fmt_in.i_codec);
359 size_t fmt_profile = 0;
360 if (p_dec->fmt_in.i_codec == VLC_CODEC_H264)
361 h264_get_profile_level(&p_dec->fmt_in, &fmt_profile, NULL, NULL);
363 /* Allocate the memory needed to store the decoder's structure */
364 if ((p_dec->p_sys = p_sys = calloc(1, sizeof(*p_sys))) == NULL)
367 p_dec->pf_decode_video = DecodeVideo;
369 p_dec->fmt_out.i_cat = p_dec->fmt_in.i_cat;
370 p_dec->fmt_out.video = p_dec->fmt_in.video;
371 p_dec->fmt_out.audio = p_dec->fmt_in.audio;
372 p_dec->b_need_packetized = true;
375 jni_attach_thread(&env, THREAD_NAME);
377 for (int i = 0; classes[i].name; i++) {
378 *(jclass*)((uint8_t*)p_sys + classes[i].offset) =
379 (*env)->FindClass(env, classes[i].name);
381 if (CHECK_EXCEPTION()) {
382 msg_Warn(p_dec, "Unable to find class %s", classes[i].name);
388 for (int i = 0; members[i].name; i++) {
389 if (i == 0 || strcmp(members[i].class, members[i - 1].class))
390 last_class = (*env)->FindClass(env, members[i].class);
392 if (CHECK_EXCEPTION()) {
393 msg_Warn(p_dec, "Unable to find class %s", members[i].class);
397 switch (members[i].type) {
399 *(jmethodID*)((uint8_t*)p_sys + members[i].offset) =
400 (*env)->GetMethodID(env, last_class, members[i].name, members[i].sig);
403 *(jmethodID*)((uint8_t*)p_sys + members[i].offset) =
404 (*env)->GetStaticMethodID(env, last_class, members[i].name, members[i].sig);
407 *(jfieldID*)((uint8_t*)p_sys + members[i].offset) =
408 (*env)->GetFieldID(env, last_class, members[i].name, members[i].sig);
411 if (CHECK_EXCEPTION()) {
412 msg_Warn(p_dec, "Unable to find the member %s in %s",
413 members[i].name, members[i].class);
414 if (members[i].critical)
418 /* getInputBuffers and getOutputBuffers are deprecated if API >= 21
419 * use getInputBuffer and getOutputBuffer instead. */
420 if (p_sys->get_input_buffer && p_sys->get_output_buffer) {
421 p_sys->get_output_buffers =
422 p_sys->get_input_buffers = NULL;
423 } else if (!p_sys->get_output_buffers && !p_sys->get_input_buffers) {
424 msg_Warn(p_dec, "Unable to find get Output/Input Buffer/Buffers");
428 int num_codecs = (*env)->CallStaticIntMethod(env, p_sys->media_codec_list_class,
429 p_sys->get_codec_count);
430 jobject codec_name = NULL;
432 for (int i = 0; i < num_codecs; i++) {
433 jobject codec_capabilities = NULL;
434 jobject profile_levels = NULL;
437 jobject types = NULL;
439 int profile_levels_len = 0, num_types = 0;
440 const char *name_ptr = NULL;
443 info = (*env)->CallStaticObjectMethod(env, p_sys->media_codec_list_class,
444 p_sys->get_codec_info_at, i);
445 if ((*env)->CallBooleanMethod(env, info, p_sys->is_encoder))
448 codec_capabilities = (*env)->CallObjectMethod(env, info, p_sys->get_capabilities_for_type,
449 (*env)->NewStringUTF(env, mime));
450 if (CHECK_EXCEPTION()) {
451 msg_Warn(p_dec, "Exception occurred in MediaCodecInfo.getCapabilitiesForType");
453 } else if (codec_capabilities) {
454 profile_levels = (*env)->GetObjectField(env, codec_capabilities, p_sys->profile_levels_field);
456 profile_levels_len = (*env)->GetArrayLength(env, profile_levels);
458 msg_Dbg(p_dec, "Number of profile levels: %d", profile_levels_len);
460 types = (*env)->CallObjectMethod(env, info, p_sys->get_supported_types);
461 num_types = (*env)->GetArrayLength(env, types);
462 name = (*env)->CallObjectMethod(env, info, p_sys->get_name);
463 name_len = (*env)->GetStringUTFLength(env, name);
464 name_ptr = (*env)->GetStringUTFChars(env, name, NULL);
467 if (codec_is_blacklisted( name_ptr, name_len))
469 for (int j = 0; j < num_types && !found; j++) {
470 jobject type = (*env)->GetObjectArrayElement(env, types, j);
471 if (!jstrcmp(env, type, mime)) {
472 /* The mime type is matching for this component. We
473 now check if the capabilities of the codec is
474 matching the video format. */
475 if (p_dec->fmt_in.i_codec == VLC_CODEC_H264 && fmt_profile) {
476 /* This decoder doesn't expose its profiles and is high
478 if (!strncmp(name_ptr, "OMX.LUMEVideoDecoder", __MIN(20, name_len)))
481 for (int i = 0; i < profile_levels_len && !found; ++i) {
482 jobject profile_level = (*env)->GetObjectArrayElement(env, profile_levels, i);
484 int omx_profile = (*env)->GetIntField(env, profile_level, p_sys->profile_field);
485 size_t codec_profile = convert_omx_to_profile_idc(omx_profile);
486 (*env)->DeleteLocalRef(env, profile_level);
487 if (codec_profile != fmt_profile)
489 /* Some encoders set the level too high, thus we ignore it for the moment.
490 We could try to guess the actual profile based on the resolution. */
497 (*env)->DeleteLocalRef(env, type);
500 msg_Dbg(p_dec, "using %.*s", name_len, name_ptr);
501 p_sys->name = malloc(name_len + 1);
502 memcpy(p_sys->name, name_ptr, name_len);
503 p_sys->name[name_len] = '\0';
508 (*env)->ReleaseStringUTFChars(env, name, name_ptr);
510 (*env)->DeleteLocalRef(env, profile_levels);
512 (*env)->DeleteLocalRef(env, types);
513 if (codec_capabilities)
514 (*env)->DeleteLocalRef(env, codec_capabilities);
516 (*env)->DeleteLocalRef(env, info);
522 msg_Dbg(p_dec, "No suitable codec matching %s was found", mime);
526 // This method doesn't handle errors nicely, it crashes if the codec isn't found.
527 // (The same goes for createDecoderByType.) This is fixed in latest AOSP and in 4.2,
528 // but not in 4.1 devices.
529 p_sys->codec = (*env)->CallStaticObjectMethod(env, p_sys->media_codec_class,
530 p_sys->create_by_codec_name, codec_name);
531 if (CHECK_EXCEPTION()) {
532 msg_Warn(p_dec, "Exception occurred in MediaCodec.createByCodecName.");
535 p_sys->allocated = true;
536 p_sys->codec = (*env)->NewGlobalRef(env, p_sys->codec);
538 jobject format = (*env)->CallStaticObjectMethod(env, p_sys->media_format_class,
539 p_sys->create_video_format, (*env)->NewStringUTF(env, mime),
540 p_dec->fmt_in.video.i_width, p_dec->fmt_in.video.i_height);
542 if (p_dec->fmt_in.i_extra) {
543 // Allocate a byte buffer via allocateDirect in java instead of NewDirectByteBuffer,
544 // since the latter doesn't allocate storage of its own, and we don't know how long
545 // the codec uses the buffer.
546 int buf_size = p_dec->fmt_in.i_extra + 20;
547 jobject bytebuf = (*env)->CallStaticObjectMethod(env, p_sys->byte_buffer_class,
548 p_sys->allocate_direct, buf_size);
549 uint32_t size = p_dec->fmt_in.i_extra;
550 uint8_t *ptr = (*env)->GetDirectBufferAddress(env, bytebuf);
551 if (p_dec->fmt_in.i_codec == VLC_CODEC_H264 && ((uint8_t*)p_dec->fmt_in.p_extra)[0] == 1) {
552 convert_sps_pps(p_dec, p_dec->fmt_in.p_extra, p_dec->fmt_in.i_extra,
554 &size, &p_sys->nal_size);
555 } else if (p_dec->fmt_in.i_codec == VLC_CODEC_HEVC) {
556 convert_hevc_nal_units(p_dec, p_dec->fmt_in.p_extra,
557 p_dec->fmt_in.i_extra, ptr, buf_size,
558 &size, &p_sys->nal_size);
560 memcpy(ptr, p_dec->fmt_in.p_extra, size);
562 (*env)->CallObjectMethod(env, bytebuf, p_sys->limit, size);
563 (*env)->CallVoidMethod(env, format, p_sys->set_bytebuffer,
564 (*env)->NewStringUTF(env, "csd-0"), bytebuf);
565 (*env)->DeleteLocalRef(env, bytebuf);
568 /* If the VideoPlayerActivity is not started, MediaCodec opaque
569 direct rendering should be disabled since no surface will be
570 attached to the JNI. */
571 p_sys->direct_rendering = jni_IsVideoPlayerActivityCreated() && var_InheritBool(p_dec, CFG_PREFIX "dr");
573 /* There is no way to rotate the video using direct rendering (and using a
574 * SurfaceView) before API 21 (Lollipop). Therefore, we deactivate direct
575 * rendering if video doesn't have a normal rotation and if
576 * get_input_buffer method is not present (This method exists since API
578 if (p_sys->direct_rendering
579 && p_dec->fmt_in.video.orientation != ORIENT_NORMAL
580 && !p_sys->get_input_buffer)
581 p_sys->direct_rendering = false;
583 if (p_sys->direct_rendering) {
584 if (p_dec->fmt_in.video.orientation != ORIENT_NORMAL) {
587 switch (p_dec->fmt_in.video.orientation) {
588 case ORIENT_ROTATED_90:
591 case ORIENT_ROTATED_180:
594 case ORIENT_ROTATED_270:
600 (*env)->CallVoidMethod(env, format, p_sys->set_integer,
601 (*env)->NewStringUTF(env, "rotation-degrees"),
605 jobject surf = jni_LockAndGetAndroidJavaSurface();
607 // Configure MediaCodec with the Android surface.
608 (*env)->CallVoidMethod(env, p_sys->codec, p_sys->configure, format, surf, NULL, 0);
609 if (CHECK_EXCEPTION()) {
610 msg_Warn(p_dec, "Exception occurred in MediaCodec.configure with an output surface.");
611 jni_UnlockAndroidSurface();
614 p_dec->fmt_out.i_codec = VLC_CODEC_ANDROID_OPAQUE;
616 msg_Warn(p_dec, "Failed to get the Android Surface, disabling direct rendering.");
617 p_sys->direct_rendering = false;
619 jni_UnlockAndroidSurface();
621 if (!p_sys->direct_rendering) {
622 (*env)->CallVoidMethod(env, p_sys->codec, p_sys->configure, format, NULL, NULL, 0);
623 if (CHECK_EXCEPTION()) {
624 msg_Warn(p_dec, "Exception occurred in MediaCodec.configure");
629 (*env)->CallVoidMethod(env, p_sys->codec, p_sys->start);
630 if (CHECK_EXCEPTION()) {
631 msg_Warn(p_dec, "Exception occurred in MediaCodec.start");
634 p_sys->started = true;
636 if (p_sys->get_input_buffers && p_sys->get_output_buffers) {
637 p_sys->input_buffers = (*env)->CallObjectMethod(env, p_sys->codec, p_sys->get_input_buffers);
638 if (CHECK_EXCEPTION()) {
639 msg_Err(p_dec, "Exception in MediaCodec.getInputBuffers (OpenDecoder)");
642 p_sys->output_buffers = (*env)->CallObjectMethod(env, p_sys->codec, p_sys->get_output_buffers);
643 if (CHECK_EXCEPTION()) {
644 msg_Err(p_dec, "Exception in MediaCodec.getOutputBuffers (OpenDecoder)");
647 p_sys->input_buffers = (*env)->NewGlobalRef(env, p_sys->input_buffers);
648 p_sys->output_buffers = (*env)->NewGlobalRef(env, p_sys->output_buffers);
650 p_sys->buffer_info = (*env)->NewObject(env, p_sys->buffer_info_class, p_sys->buffer_info_ctor);
651 p_sys->buffer_info = (*env)->NewGlobalRef(env, p_sys->buffer_info);
652 (*env)->DeleteLocalRef(env, format);
656 const int timestamp_fifo_size = 32;
657 p_sys->timestamp_fifo = timestamp_FifoNew(timestamp_fifo_size);
658 if (!p_sys->timestamp_fifo)
665 CloseDecoder(p_this);
669 static void CloseDecoder(vlc_object_t *p_this)
671 decoder_t *p_dec = (decoder_t *)p_this;
672 decoder_sys_t *p_sys = p_dec->p_sys;
678 /* Invalidate all pictures that are currently in flight in order
679 * to prevent the vout from using destroyed output buffers. */
680 if (p_sys->direct_rendering)
681 InvalidateAllPictures(p_dec);
682 jni_attach_thread(&env, THREAD_NAME);
683 if (p_sys->input_buffers)
684 (*env)->DeleteGlobalRef(env, p_sys->input_buffers);
685 if (p_sys->output_buffers)
686 (*env)->DeleteGlobalRef(env, p_sys->output_buffers);
690 (*env)->CallVoidMethod(env, p_sys->codec, p_sys->stop);
691 if (CHECK_EXCEPTION())
692 msg_Err(p_dec, "Exception in MediaCodec.stop");
694 if (p_sys->allocated)
696 (*env)->CallVoidMethod(env, p_sys->codec, p_sys->release);
697 if (CHECK_EXCEPTION())
698 msg_Err(p_dec, "Exception in MediaCodec.release");
700 (*env)->DeleteGlobalRef(env, p_sys->codec);
702 if (p_sys->buffer_info)
703 (*env)->DeleteGlobalRef(env, p_sys->buffer_info);
707 ArchitectureSpecificCopyHooksDestroy(p_sys->pixel_format, &p_sys->architecture_specific_data);
708 free(p_sys->pp_inflight_pictures);
709 if (p_sys->timestamp_fifo)
710 timestamp_FifoRelease(p_sys->timestamp_fifo);
714 /*****************************************************************************
715 * ReleaseOutputBuffer
716 *****************************************************************************/
717 static int ReleaseOutputBuffer(decoder_t *p_dec, JNIEnv *env, int i_index,
720 decoder_sys_t *p_sys = p_dec->p_sys;
722 (*env)->CallVoidMethod(env, p_sys->codec, p_sys->release_output_buffer,
724 if (CHECK_EXCEPTION()) {
725 msg_Err(p_dec, "Exception in MediaCodec.releaseOutputBuffer");
731 /*****************************************************************************
733 *****************************************************************************/
734 static void UnlockPicture(picture_t* p_pic, bool b_render)
736 picture_sys_t *p_picsys = p_pic->p_sys;
737 decoder_t *p_dec = p_picsys->priv.hw.p_dec;
739 if (!p_picsys->priv.hw.b_valid)
742 vlc_mutex_lock(get_android_opaque_mutex());
744 /* Picture might have been invalidated while waiting on the mutex. */
745 if (!p_picsys->priv.hw.b_valid) {
746 vlc_mutex_unlock(get_android_opaque_mutex());
750 uint32_t i_index = p_picsys->priv.hw.i_index;
751 InsertInflightPicture(p_dec, NULL, i_index);
753 /* Release the MediaCodec buffer. */
755 jni_attach_thread(&env, THREAD_NAME);
756 ReleaseOutputBuffer(p_dec, env, i_index, b_render);
758 p_picsys->priv.hw.b_valid = false;
760 vlc_mutex_unlock(get_android_opaque_mutex());
763 static void InvalidateAllPictures(decoder_t *p_dec)
765 decoder_sys_t *p_sys = p_dec->p_sys;
767 vlc_mutex_lock(get_android_opaque_mutex());
769 for (unsigned int i = 0; i < p_sys->i_inflight_pictures; ++i) {
770 picture_t *p_pic = p_sys->pp_inflight_pictures[i];
772 p_pic->p_sys->priv.hw.b_valid = false;
773 p_sys->pp_inflight_pictures[i] = NULL;
776 vlc_mutex_unlock(get_android_opaque_mutex());
779 static int InsertInflightPicture(decoder_t *p_dec, picture_t *p_pic,
780 unsigned int i_index)
782 decoder_sys_t *p_sys = p_dec->p_sys;
784 if (i_index >= p_sys->i_inflight_pictures) {
785 picture_t **pp_pics = realloc(p_sys->pp_inflight_pictures,
786 (i_index + 1) * sizeof (picture_t *));
789 if (i_index - p_sys->i_inflight_pictures > 0)
790 memset(&pp_pics[p_sys->i_inflight_pictures], 0,
791 (i_index - p_sys->i_inflight_pictures) * sizeof (picture_t *));
792 p_sys->pp_inflight_pictures = pp_pics;
793 p_sys->i_inflight_pictures = i_index + 1;
795 p_sys->pp_inflight_pictures[i_index] = p_pic;
799 static int PutInput(decoder_t *p_dec, JNIEnv *env, block_t **pp_block, jlong timeout)
801 decoder_sys_t *p_sys = p_dec->p_sys;
802 block_t *p_block = *pp_block;
807 struct H264ConvertState convert_state = { 0, 0 };
809 index = (*env)->CallIntMethod(env, p_sys->codec,
810 p_sys->dequeue_input_buffer, timeout);
811 if (CHECK_EXCEPTION()) {
812 msg_Err(p_dec, "Exception occurred in MediaCodec.dequeueInputBuffer");
818 if (p_sys->get_input_buffers)
819 buf = (*env)->GetObjectArrayElement(env, p_sys->input_buffers, index);
821 buf = (*env)->CallObjectMethod(env, p_sys->codec, p_sys->get_input_buffer, index);
822 size = (*env)->GetDirectBufferCapacity(env, buf);
823 bufptr = (*env)->GetDirectBufferAddress(env, buf);
825 msg_Err(p_dec, "Java buffer has invalid size");
828 if ((size_t) size > p_block->i_buffer)
829 size = p_block->i_buffer;
830 memcpy(bufptr, p_block->p_buffer, size);
832 convert_h264_to_annexb(bufptr, size, p_sys->nal_size, &convert_state);
834 int64_t ts = p_block->i_pts;
835 if (!ts && p_block->i_dts)
837 if (p_block->i_flags & BLOCK_FLAG_PREROLL )
838 p_sys->i_preroll_end = ts;
839 timestamp_FifoPut(p_sys->timestamp_fifo, p_block->i_pts ? VLC_TS_INVALID : p_block->i_dts);
840 (*env)->CallVoidMethod(env, p_sys->codec, p_sys->queue_input_buffer, index, 0, size, ts, 0);
841 (*env)->DeleteLocalRef(env, buf);
842 if (CHECK_EXCEPTION()) {
843 msg_Err(p_dec, "Exception in MediaCodec.queueInputBuffer");
846 block_Release(p_block);
848 p_sys->decoded = true;
853 static int GetOutput(decoder_t *p_dec, JNIEnv *env, picture_t **pp_pic, jlong timeout)
855 decoder_sys_t *p_sys = p_dec->p_sys;
856 int index = (*env)->CallIntMethod(env, p_sys->codec, p_sys->dequeue_output_buffer,
857 p_sys->buffer_info, timeout);
858 if (CHECK_EXCEPTION()) {
859 msg_Err(p_dec, "Exception in MediaCodec.dequeueOutputBuffer (GetOutput)");
864 int64_t i_buffer_pts;
866 if (!p_sys->pixel_format) {
867 msg_Warn(p_dec, "Buffers returned before output format is set, dropping frame");
868 return ReleaseOutputBuffer(p_dec, env, index, false);
871 i_buffer_pts = (*env)->GetLongField(env, p_sys->buffer_info, p_sys->pts_field);
872 if (i_buffer_pts <= p_sys->i_preroll_end)
873 return ReleaseOutputBuffer(p_dec, env, index, false);
875 *pp_pic = decoder_NewPicture(p_dec);
877 msg_Warn(p_dec, "NewPicture failed");
878 return ReleaseOutputBuffer(p_dec, env, index, false);
880 picture_t *p_pic = *pp_pic;
881 /* If the oldest input block had no PTS, the timestamp
882 * of the frame returned by MediaCodec might be wrong
883 * so we overwrite it with the corresponding dts. */
884 int64_t forced_ts = timestamp_FifoGet(p_sys->timestamp_fifo);
885 if (forced_ts == VLC_TS_INVALID)
886 p_pic->date = i_buffer_pts;
888 p_pic->date = forced_ts;
890 if (p_sys->direct_rendering) {
891 picture_sys_t *p_picsys = p_pic->p_sys;
892 p_picsys->pf_lock_pic = NULL;
893 p_picsys->pf_unlock_pic = UnlockPicture;
894 p_picsys->priv.hw.p_dec = p_dec;
895 p_picsys->priv.hw.i_index = index;
896 p_picsys->priv.hw.b_valid = true;
898 vlc_mutex_lock(get_android_opaque_mutex());
899 InsertInflightPicture(p_dec, p_pic, index);
900 vlc_mutex_unlock(get_android_opaque_mutex());
903 if (p_sys->get_output_buffers)
904 buf = (*env)->GetObjectArrayElement(env, p_sys->output_buffers, index);
906 buf = (*env)->CallObjectMethod(env, p_sys->codec,
907 p_sys->get_output_buffer, index);
908 //jsize buf_size = (*env)->GetDirectBufferCapacity(env, buf);
909 uint8_t *ptr = (*env)->GetDirectBufferAddress(env, buf);
911 //int size = (*env)->GetIntField(env, p_sys->buffer_info, p_sys->size_field);
912 int offset = (*env)->GetIntField(env, p_sys->buffer_info, p_sys->offset_field);
913 ptr += offset; // Check the size parameter as well
915 unsigned int chroma_div;
916 GetVlcChromaSizes(p_dec->fmt_out.i_codec, p_dec->fmt_out.video.i_width,
917 p_dec->fmt_out.video.i_height, NULL, NULL, &chroma_div);
918 CopyOmxPicture(p_sys->pixel_format, p_pic, p_sys->slice_height, p_sys->stride,
919 ptr, chroma_div, &p_sys->architecture_specific_data);
920 (*env)->CallVoidMethod(env, p_sys->codec, p_sys->release_output_buffer, index, false);
922 jthrowable exception = (*env)->ExceptionOccurred(env);
923 if (exception != NULL) {
924 jclass illegalStateException = (*env)->FindClass(env, "java/lang/IllegalStateException");
925 if((*env)->IsInstanceOf(env, exception, illegalStateException)) {
926 msg_Err(p_dec, "Codec error (IllegalStateException) in MediaCodec.releaseOutputBuffer");
927 (*env)->ExceptionClear(env);
928 (*env)->DeleteLocalRef(env, illegalStateException);
929 (*env)->DeleteLocalRef(env, buf);
933 (*env)->DeleteLocalRef(env, buf);
935 } else if (index == INFO_OUTPUT_BUFFERS_CHANGED) {
936 msg_Dbg(p_dec, "output buffers changed");
937 if (!p_sys->get_output_buffers)
939 (*env)->DeleteGlobalRef(env, p_sys->output_buffers);
941 p_sys->output_buffers = (*env)->CallObjectMethod(env, p_sys->codec,
942 p_sys->get_output_buffers);
943 if (CHECK_EXCEPTION()) {
944 msg_Err(p_dec, "Exception in MediaCodec.getOutputBuffer (GetOutput)");
945 p_sys->output_buffers = NULL;
949 p_sys->output_buffers = (*env)->NewGlobalRef(env, p_sys->output_buffers);
950 } else if (index == INFO_OUTPUT_FORMAT_CHANGED) {
951 jobject format = (*env)->CallObjectMethod(env, p_sys->codec, p_sys->get_output_format);
952 if (CHECK_EXCEPTION()) {
953 msg_Err(p_dec, "Exception in MediaCodec.getOutputFormat (GetOutput)");
957 jobject format_string = (*env)->CallObjectMethod(env, format, p_sys->tostring);
959 jsize format_len = (*env)->GetStringUTFLength(env, format_string);
960 const char *format_ptr = (*env)->GetStringUTFChars(env, format_string, NULL);
961 msg_Dbg(p_dec, "output format changed: %.*s", format_len, format_ptr);
962 (*env)->ReleaseStringUTFChars(env, format_string, format_ptr);
964 ArchitectureSpecificCopyHooksDestroy(p_sys->pixel_format, &p_sys->architecture_specific_data);
966 int width = GET_INTEGER(format, "width");
967 int height = GET_INTEGER(format, "height");
968 p_sys->stride = GET_INTEGER(format, "stride");
969 p_sys->slice_height = GET_INTEGER(format, "slice-height");
970 p_sys->pixel_format = GET_INTEGER(format, "color-format");
971 int crop_left = GET_INTEGER(format, "crop-left");
972 int crop_top = GET_INTEGER(format, "crop-top");
973 int crop_right = GET_INTEGER(format, "crop-right");
974 int crop_bottom = GET_INTEGER(format, "crop-bottom");
976 const char *name = "unknown";
977 if (!p_sys->direct_rendering) {
978 if (!GetVlcChromaFormat(p_sys->pixel_format,
979 &p_dec->fmt_out.i_codec, &name)) {
980 msg_Err(p_dec, "color-format not recognized");
985 msg_Err(p_dec, "output: %d %s, %dx%d stride %d %d, crop %d %d %d %d",
986 p_sys->pixel_format, name, width, height, p_sys->stride, p_sys->slice_height,
987 crop_left, crop_top, crop_right, crop_bottom);
989 p_dec->fmt_out.video.i_width = crop_right + 1 - crop_left;
990 p_dec->fmt_out.video.i_height = crop_bottom + 1 - crop_top;
991 if (p_dec->fmt_out.video.i_width <= 1
992 || p_dec->fmt_out.video.i_height <= 1) {
993 p_dec->fmt_out.video.i_width = width;
994 p_dec->fmt_out.video.i_height = height;
996 p_dec->fmt_out.video.i_visible_width = p_dec->fmt_out.video.i_width;
997 p_dec->fmt_out.video.i_visible_height = p_dec->fmt_out.video.i_height;
999 if (p_sys->stride <= 0)
1000 p_sys->stride = width;
1001 if (p_sys->slice_height <= 0)
1002 p_sys->slice_height = height;
1005 ArchitectureSpecificCopyHooks(p_dec, p_sys->pixel_format, p_sys->slice_height,
1006 p_sys->stride, &p_sys->architecture_specific_data);
1007 if (p_sys->pixel_format == OMX_TI_COLOR_FormatYUV420PackedSemiPlanar)
1008 p_sys->slice_height -= crop_top/2;
1009 if (IgnoreOmxDecoderPadding(p_sys->name)) {
1010 p_sys->slice_height = 0;
1011 p_sys->stride = p_dec->fmt_out.video.i_width;
1017 static picture_t *DecodeVideo(decoder_t *p_dec, block_t **pp_block)
1019 decoder_sys_t *p_sys = p_dec->p_sys;
1020 picture_t *p_pic = NULL;
1023 if (!pp_block || !*pp_block)
1026 if (p_sys->error_state)
1029 jni_attach_thread(&env, THREAD_NAME);
1033 if ((*pp_block)->i_flags & (BLOCK_FLAG_DISCONTINUITY|BLOCK_FLAG_CORRUPTED)) {
1034 block_Release(*pp_block);
1036 p_sys->i_preroll_end = 0;
1037 timestamp_FifoEmpty(p_sys->timestamp_fifo);
1038 if (p_sys->decoded) {
1039 /* Invalidate all pictures that are currently in flight
1040 * since flushing make all previous indices returned by
1041 * MediaCodec invalid. */
1042 if (p_sys->direct_rendering)
1043 InvalidateAllPictures(p_dec);
1045 (*env)->CallVoidMethod(env, p_sys->codec, p_sys->flush);
1046 if (CHECK_EXCEPTION()) {
1047 msg_Warn(p_dec, "Exception occurred in MediaCodec.flush");
1048 p_sys->error_state = true;
1051 p_sys->decoded = false;
1055 /* Use the aspect ratio provided by the input (ie read from packetizer).
1056 * Don't check the current value of the aspect ratio in fmt_out, since we
1057 * want to allow changes in it to propagate. */
1058 if (p_dec->fmt_in.video.i_sar_num != 0 && p_dec->fmt_in.video.i_sar_den != 0) {
1059 p_dec->fmt_out.video.i_sar_num = p_dec->fmt_in.video.i_sar_num;
1060 p_dec->fmt_out.video.i_sar_den = p_dec->fmt_in.video.i_sar_den;
1064 const int max_polling_attempts = 50;
1066 /* return when pp_block is processed */
1067 while (*pp_block != NULL) {
1068 if (*pp_block != NULL && PutInput(p_dec, env, pp_block, (jlong) 0) != 0) {
1069 p_sys->error_state = true;
1073 if (p_pic == NULL && GetOutput(p_dec, env, &p_pic, timeout) != 0) {
1074 p_sys->error_state = true;
1078 if (p_pic == NULL && *pp_block != NULL) {
1079 timeout = 30 * 1000;
1081 /* With opaque DR the output buffers are released by the
1082 vout therefore we implement a timeout for polling in
1083 order to avoid being indefinitely stalled in this loop. */
1084 if (p_sys->direct_rendering && attempts == max_polling_attempts) {
1085 p_pic = decoder_NewPicture(p_dec);
1087 p_pic->date = VLC_TS_INVALID;
1088 picture_sys_t *p_picsys = p_pic->p_sys;
1089 p_picsys->pf_lock_pic = NULL;
1090 p_picsys->pf_unlock_pic = NULL;
1091 p_picsys->priv.hw.p_dec = NULL;
1092 p_picsys->priv.hw.i_index = -1;
1093 p_picsys->priv.hw.b_valid = false;
1096 /* If we cannot return a picture we must free the
1097 block since the decoder will proceed with the
1099 block_Release(*pp_block);
1107 if (p_sys->error_state) {
1108 if( pp_block && *pp_block )
1110 block_Release(*pp_block);
1114 picture_Release(p_pic);
1117 if (!p_sys->error_event_sent) {
1118 /* Signal the error to the Java. */
1119 jni_EventHardwareAccelerationError();
1120 p_sys->error_event_sent = true;
1124 jni_detach_thread();