git.sesse.net Git - vlc/blob - modules/codec/omxil/android_mediacodec.c

   1 /*****************************************************************************
   2  * android_mediacodec.c: Video decoder module using the Android MediaCodec API
   3  *****************************************************************************
   4  * Copyright (C) 2012 Martin Storsjo
   5  *
   6  * Authors: Martin Storsjo <martin@martin.st>
   7  *
   8  * This program is free software; you can redistribute it and/or modify it
   9  * under the terms of the GNU Lesser General Public License as published by
  10  * the Free Software Foundation; either version 2.1 of the License, or
  11  * (at your option) any later version.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  16  * GNU Lesser General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU Lesser General Public License
  19  * along with this program; if not, write to the Free Software Foundation,
  20  * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
  21  *****************************************************************************/
  22
  23 /*****************************************************************************
  24  * Preamble
  25  *****************************************************************************/
  26 #ifdef HAVE_CONFIG_H
  27 # include "config.h"
  28 #endif
  29
  30 #include <jni.h>
  31 #include <stdint.h>
  32
  33 #include <vlc_common.h>
  34 #include <vlc_plugin.h>
  35 #include <vlc_codec.h>
  36 #include <vlc_block_helper.h>
  37 #include <vlc_cpu.h>
  38
  39 #include "../h264_nal.h"
  40 #include <OMX_Core.h>
  41 #include <OMX_Component.h>
  42 #include "omxil_utils.h"
  43 #include "android_opaque.h"
  44
  45 #define INFO_OUTPUT_BUFFERS_CHANGED -3
  46 #define INFO_OUTPUT_FORMAT_CHANGED  -2
  47 #define INFO_TRY_AGAIN_LATER        -1
  48
  49 extern JavaVM *myVm;
  50 /* JNI functions to get/set an Android Surface object. */
  51 extern jobject jni_LockAndGetAndroidJavaSurface();
  52 extern void jni_UnlockAndroidSurface();
  53 extern void jni_SetAndroidSurfaceSizeEnv(JNIEnv *p_env, int width, int height, int visible_width, int visible_height, int sar_num, int sar_den);
  54 extern void jni_EventHardwareAccelerationError();
  55
  56 struct decoder_sys_t
  57 {
  58     jclass media_codec_list_class, media_codec_class, media_format_class;
  59     jclass buffer_info_class, byte_buffer_class;
  60     jmethodID tostring;
  61     jmethodID get_codec_count, get_codec_info_at, is_encoder, get_capabilities_for_type;
  62     jfieldID profile_levels_field, profile_field, level_field;
  63     jmethodID get_supported_types, get_name;
  64     jmethodID create_by_codec_name, configure, start, stop, flush, release;
  65     jmethodID get_output_format, get_input_buffers, get_output_buffers;
  66     jmethodID dequeue_input_buffer, dequeue_output_buffer, queue_input_buffer;
  67     jmethodID release_output_buffer;
  68     jmethodID create_video_format, set_integer, set_bytebuffer, get_integer;
  69     jmethodID buffer_info_ctor;
  70     jmethodID allocate_direct, limit;
  71     jfieldID size_field, offset_field, pts_field;
  72
  73     uint32_t nal_size;
  74
  75     jobject codec;
  76     jobject buffer_info;
  77     jobject input_buffers, output_buffers;
  78     int pixel_format;
  79     int stride, slice_height;
  80     int crop_top, crop_left;
  81     char *name;
  82
  83     bool started;
  84     bool decoded;
  85     bool error_state;
  86     bool error_event_sent;
  87
  88     ArchitectureSpecificCopyData architecture_specific_data;
  89
  90     /* Direct rendering members. */
  91     bool direct_rendering;
  92     int i_output_buffers; /**< number of MediaCodec output buffers */
  93     picture_t** inflight_picture; /**< stores the inflight picture for each output buffer or NULL */
  94 };
  95
  96 enum Types
  97 {
  98     METHOD, STATIC_METHOD, FIELD
  99 };
 100
 101 #define OFF(x) offsetof(struct decoder_sys_t, x)
 102 struct classname
 103 {
 104     const char *name;
 105     int offset;
 106 };
 107 static const struct classname classes[] = {
 108     { "android/media/MediaCodecList", OFF(media_codec_list_class) },
 109     { "android/media/MediaCodec", OFF(media_codec_class) },
 110     { "android/media/MediaFormat", OFF(media_format_class) },
 111     { "android/media/MediaFormat", OFF(media_format_class) },
 112     { "android/media/MediaCodec$BufferInfo", OFF(buffer_info_class) },
 113     { "java/nio/ByteBuffer", OFF(byte_buffer_class) },
 114     { NULL, 0 },
 115 };
 116
 117 struct member
 118 {
 119     const char *name;
 120     const char *sig;
 121     const char *class;
 122     int offset;
 123     int type;
 124 };
 125 static const struct member members[] = {
 126     { "toString", "()Ljava/lang/String;", "java/lang/Object", OFF(tostring), METHOD },
 127
 128     { "getCodecCount", "()I", "android/media/MediaCodecList", OFF(get_codec_count), STATIC_METHOD },
 129     { "getCodecInfoAt", "(I)Landroid/media/MediaCodecInfo;", "android/media/MediaCodecList", OFF(get_codec_info_at), STATIC_METHOD },
 130
 131     { "isEncoder", "()Z", "android/media/MediaCodecInfo", OFF(is_encoder), METHOD },
 132     { "getSupportedTypes", "()[Ljava/lang/String;", "android/media/MediaCodecInfo", OFF(get_supported_types), METHOD },
 133     { "getName", "()Ljava/lang/String;", "android/media/MediaCodecInfo", OFF(get_name), METHOD },
 134     { "getCapabilitiesForType", "(Ljava/lang/String;)Landroid/media/MediaCodecInfo$CodecCapabilities;", "android/media/MediaCodecInfo", OFF(get_capabilities_for_type), METHOD },
 135
 136     { "profileLevels", "[Landroid/media/MediaCodecInfo$CodecProfileLevel;", "android/media/MediaCodecInfo$CodecCapabilities", OFF(profile_levels_field), FIELD },
 137     { "profile", "I", "android/media/MediaCodecInfo$CodecProfileLevel", OFF(profile_field), FIELD },
 138     { "level", "I", "android/media/MediaCodecInfo$CodecProfileLevel", OFF(level_field), FIELD },
 139
 140     { "createByCodecName", "(Ljava/lang/String;)Landroid/media/MediaCodec;", "android/media/MediaCodec", OFF(create_by_codec_name), STATIC_METHOD },
 141     { "configure", "(Landroid/media/MediaFormat;Landroid/view/Surface;Landroid/media/MediaCrypto;I)V", "android/media/MediaCodec", OFF(configure), METHOD },
 142     { "start", "()V", "android/media/MediaCodec", OFF(start), METHOD },
 143     { "stop", "()V", "android/media/MediaCodec", OFF(stop), METHOD },
 144     { "flush", "()V", "android/media/MediaCodec", OFF(flush), METHOD },
 145     { "release", "()V", "android/media/MediaCodec", OFF(release), METHOD },
 146     { "getOutputFormat", "()Landroid/media/MediaFormat;", "android/media/MediaCodec", OFF(get_output_format), METHOD },
 147     { "getInputBuffers", "()[Ljava/nio/ByteBuffer;", "android/media/MediaCodec", OFF(get_input_buffers), METHOD },
 148     { "getOutputBuffers", "()[Ljava/nio/ByteBuffer;", "android/media/MediaCodec", OFF(get_output_buffers), METHOD },
 149     { "dequeueInputBuffer", "(J)I", "android/media/MediaCodec", OFF(dequeue_input_buffer), METHOD },
 150     { "dequeueOutputBuffer", "(Landroid/media/MediaCodec$BufferInfo;J)I", "android/media/MediaCodec", OFF(dequeue_output_buffer), METHOD },
 151     { "queueInputBuffer", "(IIIJI)V", "android/media/MediaCodec", OFF(queue_input_buffer), METHOD },
 152     { "releaseOutputBuffer", "(IZ)V", "android/media/MediaCodec", OFF(release_output_buffer), METHOD },
 153
 154     { "createVideoFormat", "(Ljava/lang/String;II)Landroid/media/MediaFormat;", "android/media/MediaFormat", OFF(create_video_format), STATIC_METHOD },
 155     { "setInteger", "(Ljava/lang/String;I)V", "android/media/MediaFormat", OFF(set_integer), METHOD },
 156     { "getInteger", "(Ljava/lang/String;)I", "android/media/MediaFormat", OFF(get_integer), METHOD },
 157     { "setByteBuffer", "(Ljava/lang/String;Ljava/nio/ByteBuffer;)V", "android/media/MediaFormat", OFF(set_bytebuffer), METHOD },
 158
 159     { "<init>", "()V", "android/media/MediaCodec$BufferInfo", OFF(buffer_info_ctor), METHOD },
 160     { "size", "I", "android/media/MediaCodec$BufferInfo", OFF(size_field), FIELD },
 161     { "offset", "I", "android/media/MediaCodec$BufferInfo", OFF(offset_field), FIELD },
 162     { "presentationTimeUs", "J", "android/media/MediaCodec$BufferInfo", OFF(pts_field), FIELD },
 163
 164     { "allocateDirect", "(I)Ljava/nio/ByteBuffer;", "java/nio/ByteBuffer", OFF(allocate_direct), STATIC_METHOD },
 165     { "limit", "(I)Ljava/nio/Buffer;", "java/nio/ByteBuffer", OFF(limit), METHOD },
 166
 167     { NULL, NULL, NULL, 0, 0 },
 168 };
 169
 170 #define GET_INTEGER(obj, name) (*env)->CallIntMethod(env, obj, p_sys->get_integer, (*env)->NewStringUTF(env, name))
 171
 172 /*****************************************************************************
 173  * Local prototypes
 174  *****************************************************************************/
 175 static int  OpenDecoder(vlc_object_t *);
 176 static void CloseDecoder(vlc_object_t *);
 177
 178 static picture_t *DecodeVideo(decoder_t *, block_t **);
 179
 180 static void InvalidateAllPictures(decoder_t *);
 181
 182 /*****************************************************************************
 183  * Module descriptor
 184  *****************************************************************************/
 185 #define DIRECTRENDERING_TEXT N_("Android direct rendering")
 186 #define DIRECTRENDERING_LONGTEXT N_(\
 187         "Enable Android direct rendering using opaque buffers.")
 188
 189 #define CFG_PREFIX "mediacodec-"
 190
 191 vlc_module_begin ()
 192     set_description( N_("Video decoder using Android MediaCodec") )
 193     set_category( CAT_INPUT )
 194     set_subcategory( SUBCAT_INPUT_VCODEC )
 195     set_section( N_("Decoding") , NULL )
 196     set_capability( "decoder", 0 ) /* Only enabled via commandline arguments */
 197     add_bool(CFG_PREFIX "dr", true,
 198              DIRECTRENDERING_TEXT, DIRECTRENDERING_LONGTEXT, true)
 199     set_callbacks( OpenDecoder, CloseDecoder )
 200 vlc_module_end ()
 201
 202 static int jstrcmp(JNIEnv* env, jobject str, const char* str2)
 203 {
 204     jsize len = (*env)->GetStringUTFLength(env, str);
 205     if (len != (jsize) strlen(str2))
 206         return -1;
 207     const char *ptr = (*env)->GetStringUTFChars(env, str, NULL);
 208     int ret = memcmp(ptr, str2, len);
 209     (*env)->ReleaseStringUTFChars(env, str, ptr);
 210     return ret;
 211 }
 212
 213 /*****************************************************************************
 214  * OpenDecoder: Create the decoder instance
 215  *****************************************************************************/
 216 static int OpenDecoder(vlc_object_t *p_this)
 217 {
 218     decoder_t *p_dec = (decoder_t*)p_this;
 219     decoder_sys_t *p_sys;
 220
 221     if (p_dec->fmt_in.i_cat != VIDEO_ES && !p_dec->b_force)
 222         return VLC_EGENERIC;
 223
 224     const char *mime = NULL;
 225     switch (p_dec->fmt_in.i_codec) {
 226     case VLC_CODEC_H264: mime = "video/avc"; break;
 227     case VLC_CODEC_H263: mime = "video/3gpp"; break;
 228     case VLC_CODEC_MP4V: mime = "video/mp4v-es"; break;
 229     case VLC_CODEC_VC1:  mime = "video/wvc1"; break;
 230     case VLC_CODEC_VP8:  mime = "video/x-vnd.on2.vp8"; break;
 231     default:
 232         msg_Dbg(p_dec, "codec %d not supported", p_dec->fmt_in.i_codec);
 233         return VLC_EGENERIC;
 234     }
 235
 236     size_t fmt_profile = 0;
 237     if (p_dec->fmt_in.i_codec == VLC_CODEC_H264)
 238         h264_get_profile_level(&p_dec->fmt_in, &fmt_profile, NULL, NULL);
 239
 240     /* Allocate the memory needed to store the decoder's structure */
 241     if ((p_dec->p_sys = p_sys = calloc(1, sizeof(*p_sys))) == NULL)
 242         return VLC_ENOMEM;
 243
 244     p_dec->pf_decode_video = DecodeVideo;
 245
 246
 247     p_dec->fmt_out.i_cat = p_dec->fmt_in.i_cat;
 248     p_dec->fmt_out.video = p_dec->fmt_in.video;
 249     p_dec->fmt_out.audio = p_dec->fmt_in.audio;
 250     p_dec->b_need_packetized = true;
 251
 252     JNIEnv* env = NULL;
 253     (*myVm)->AttachCurrentThread(myVm, &env, NULL);
 254
 255     for (int i = 0; classes[i].name; i++) {
 256         *(jclass*)((uint8_t*)p_sys + classes[i].offset) =
 257             (*env)->FindClass(env, classes[i].name);
 258
 259         if ((*env)->ExceptionOccurred(env)) {
 260             msg_Warn(p_dec, "Unable to find class %s", classes[i].name);
 261             (*env)->ExceptionClear(env);
 262             goto error;
 263         }
 264     }
 265
 266     jclass last_class;
 267     for (int i = 0; members[i].name; i++) {
 268         if (i == 0 || strcmp(members[i].class, members[i - 1].class))
 269             last_class = (*env)->FindClass(env, members[i].class);
 270
 271         if ((*env)->ExceptionOccurred(env)) {
 272             msg_Warn(p_dec, "Unable to find class %s", members[i].class);
 273             (*env)->ExceptionClear(env);
 274             goto error;
 275         }
 276
 277         switch (members[i].type) {
 278         case METHOD:
 279             *(jmethodID*)((uint8_t*)p_sys + members[i].offset) =
 280                 (*env)->GetMethodID(env, last_class, members[i].name, members[i].sig);
 281             break;
 282         case STATIC_METHOD:
 283             *(jmethodID*)((uint8_t*)p_sys + members[i].offset) =
 284                 (*env)->GetStaticMethodID(env, last_class, members[i].name, members[i].sig);
 285             break;
 286         case FIELD:
 287             *(jfieldID*)((uint8_t*)p_sys + members[i].offset) =
 288                 (*env)->GetFieldID(env, last_class, members[i].name, members[i].sig);
 289             break;
 290         }
 291         if ((*env)->ExceptionOccurred(env)) {
 292             msg_Warn(p_dec, "Unable to find the member %s in %s",
 293                      members[i].name, members[i].class);
 294             (*env)->ExceptionClear(env);
 295             goto error;
 296         }
 297     }
 298
 299     int num_codecs = (*env)->CallStaticIntMethod(env, p_sys->media_codec_list_class,
 300                                                  p_sys->get_codec_count);
 301     jobject codec_name = NULL;
 302
 303     for (int i = 0; i < num_codecs; i++) {
 304         jobject info = (*env)->CallStaticObjectMethod(env, p_sys->media_codec_list_class,
 305                                                       p_sys->get_codec_info_at, i);
 306         if ((*env)->CallBooleanMethod(env, info, p_sys->is_encoder)) {
 307             (*env)->DeleteLocalRef(env, info);
 308             continue;
 309         }
 310
 311         jobject codec_capabilities = (*env)->CallObjectMethod(env, info, p_sys->get_capabilities_for_type,
 312                                                               (*env)->NewStringUTF(env, mime));
 313         jobject profile_levels = (*env)->GetObjectField(env, codec_capabilities, p_sys->profile_levels_field);
 314         int profile_levels_len = profile_levels ? (*env)->GetArrayLength(env, profile_levels) : 0;
 315         msg_Dbg(p_dec, "Number of profile levels: %d", profile_levels_len);
 316
 317         jobject types = (*env)->CallObjectMethod(env, info, p_sys->get_supported_types);
 318         int num_types = (*env)->GetArrayLength(env, types);
 319         bool found = false;
 320         for (int j = 0; j < num_types && !found; j++) {
 321             jobject type = (*env)->GetObjectArrayElement(env, types, j);
 322             if (!jstrcmp(env, type, mime)) {
 323                 /* The mime type is matching for this component. We
 324                    now check if the capabilities of the codec is
 325                    matching the video format. */
 326                 if (p_dec->fmt_in.i_codec == VLC_CODEC_H264 && fmt_profile && profile_levels_len) {
 327                     for (int i = 0; i < profile_levels_len && !found; ++i) {
 328                         jobject profile_level = (*env)->GetObjectArrayElement(env, profile_levels, i);
 329
 330                         int omx_profile = (*env)->GetLongField(env, profile_level, p_sys->profile_field);
 331                         size_t codec_profile = convert_omx_to_profile_idc(omx_profile);
 332                         if (codec_profile != fmt_profile)
 333                             continue;
 334                         /* Some encoders set the level too high, thus we ignore it for the moment.
 335                            We could try to guess the actual profile based on the resolution. */
 336                         found = true;
 337                     }
 338                 }
 339                 else
 340                     found = true;
 341             }
 342             (*env)->DeleteLocalRef(env, type);
 343         }
 344         if (found) {
 345             jobject name = (*env)->CallObjectMethod(env, info, p_sys->get_name);
 346             jsize name_len = (*env)->GetStringUTFLength(env, name);
 347             const char *name_ptr = (*env)->GetStringUTFChars(env, name, NULL);
 348             msg_Dbg(p_dec, "using %.*s", name_len, name_ptr);
 349             p_sys->name = malloc(name_len + 1);
 350             memcpy(p_sys->name, name_ptr, name_len);
 351             p_sys->name[name_len] = '\0';
 352             (*env)->ReleaseStringUTFChars(env, name, name_ptr);
 353             codec_name = name;
 354             break;
 355         }
 356         (*env)->DeleteLocalRef(env, info);
 357     }
 358
 359     if (!codec_name) {
 360         msg_Dbg(p_dec, "No suitable codec matching %s was found", mime);
 361         goto error;
 362     }
 363
 364     // This method doesn't handle errors nicely, it crashes if the codec isn't found.
 365     // (The same goes for createDecoderByType.) This is fixed in latest AOSP and in 4.2,
 366     // but not in 4.1 devices.
 367     p_sys->codec = (*env)->CallStaticObjectMethod(env, p_sys->media_codec_class,
 368                                                   p_sys->create_by_codec_name, codec_name);
 369     p_sys->codec = (*env)->NewGlobalRef(env, p_sys->codec);
 370
 371     jobject format = (*env)->CallStaticObjectMethod(env, p_sys->media_format_class,
 372                          p_sys->create_video_format, (*env)->NewStringUTF(env, mime),
 373                          p_dec->fmt_in.video.i_width, p_dec->fmt_in.video.i_height);
 374
 375     if (p_dec->fmt_in.i_extra) {
 376         // Allocate a byte buffer via allocateDirect in java instead of NewDirectByteBuffer,
 377         // since the latter doesn't allocate storage of its own, and we don't know how long
 378         // the codec uses the buffer.
 379         int buf_size = p_dec->fmt_in.i_extra + 20;
 380         jobject bytebuf = (*env)->CallStaticObjectMethod(env, p_sys->byte_buffer_class,
 381                                                          p_sys->allocate_direct, buf_size);
 382         uint32_t size = p_dec->fmt_in.i_extra;
 383         uint8_t *ptr = (*env)->GetDirectBufferAddress(env, bytebuf);
 384         if (p_dec->fmt_in.i_codec == VLC_CODEC_H264 && ((uint8_t*)p_dec->fmt_in.p_extra)[0] == 1) {
 385             convert_sps_pps(p_dec, p_dec->fmt_in.p_extra, p_dec->fmt_in.i_extra,
 386                             ptr, buf_size,
 387                             &size, &p_sys->nal_size);
 388         } else {
 389             memcpy(ptr, p_dec->fmt_in.p_extra, size);
 390         }
 391         (*env)->CallObjectMethod(env, bytebuf, p_sys->limit, size);
 392         (*env)->CallVoidMethod(env, format, p_sys->set_bytebuffer,
 393                                (*env)->NewStringUTF(env, "csd-0"), bytebuf);
 394         (*env)->DeleteLocalRef(env, bytebuf);
 395     }
 396
 397     p_sys->direct_rendering = var_InheritBool(p_dec, CFG_PREFIX "dr");
 398     if (p_sys->direct_rendering) {
 399         jobject surf = jni_LockAndGetAndroidJavaSurface();
 400         if (surf) {
 401             // Configure MediaCodec with the Android surface.
 402             (*env)->CallVoidMethod(env, p_sys->codec, p_sys->configure, format, surf, NULL, 0);
 403             if ((*env)->ExceptionOccurred(env)) {
 404                 msg_Warn(p_dec, "Exception occurred in MediaCodec.configure with an output surface.");
 405                 (*env)->ExceptionClear(env);
 406                 jni_UnlockAndroidSurface();
 407                 goto error;
 408             }
 409             p_dec->fmt_out.i_codec = VLC_CODEC_ANDROID_OPAQUE;
 410         } else {
 411             msg_Warn(p_dec, "Failed to get the Android Surface, disabling direct rendering.");
 412             p_sys->direct_rendering = false;
 413         }
 414         jni_UnlockAndroidSurface();
 415     }
 416     if (!p_sys->direct_rendering) {
 417         (*env)->CallVoidMethod(env, p_sys->codec, p_sys->configure, format, NULL, NULL, 0);
 418         if ((*env)->ExceptionOccurred(env)) {
 419             msg_Warn(p_dec, "Exception occurred in MediaCodec.configure");
 420             (*env)->ExceptionClear(env);
 421             goto error;
 422         }
 423     }
 424
 425     (*env)->CallVoidMethod(env, p_sys->codec, p_sys->start);
 426     if ((*env)->ExceptionOccurred(env)) {
 427         msg_Warn(p_dec, "Exception occurred in MediaCodec.start");
 428         (*env)->ExceptionClear(env);
 429         goto error;
 430     }
 431     p_sys->started = true;
 432
 433     p_sys->input_buffers = (*env)->CallObjectMethod(env, p_sys->codec, p_sys->get_input_buffers);
 434     p_sys->output_buffers = (*env)->CallObjectMethod(env, p_sys->codec, p_sys->get_output_buffers);
 435     p_sys->buffer_info = (*env)->NewObject(env, p_sys->buffer_info_class, p_sys->buffer_info_ctor);
 436     p_sys->input_buffers = (*env)->NewGlobalRef(env, p_sys->input_buffers);
 437     p_sys->output_buffers = (*env)->NewGlobalRef(env, p_sys->output_buffers);
 438     p_sys->buffer_info = (*env)->NewGlobalRef(env, p_sys->buffer_info);
 439     p_sys->i_output_buffers = (*env)->GetArrayLength(env, p_sys->output_buffers);
 440     p_sys->inflight_picture = calloc(1, sizeof(picture_t*) * p_sys->i_output_buffers);
 441     if (!p_sys->inflight_picture)
 442         goto error;
 443     (*env)->DeleteLocalRef(env, format);
 444
 445     (*myVm)->DetachCurrentThread(myVm);
 446     return VLC_SUCCESS;
 447
 448  error:
 449     (*myVm)->DetachCurrentThread(myVm);
 450     CloseDecoder(p_this);
 451     return VLC_EGENERIC;
 452 }
 453
 454 static void CloseDecoder(vlc_object_t *p_this)
 455 {
 456     decoder_t *p_dec = (decoder_t *)p_this;
 457     decoder_sys_t *p_sys = p_dec->p_sys;
 458     JNIEnv *env = NULL;
 459
 460     if (!p_sys)
 461         return;
 462
 463     /* Invalidate all pictures that are currently in flight in order
 464      * to prevent the vout from using destroyed output buffers. */
 465     if (p_sys->direct_rendering)
 466         InvalidateAllPictures(p_dec);
 467     (*myVm)->AttachCurrentThread(myVm, &env, NULL);
 468     if (p_sys->input_buffers)
 469         (*env)->DeleteGlobalRef(env, p_sys->input_buffers);
 470     if (p_sys->output_buffers)
 471         (*env)->DeleteGlobalRef(env, p_sys->output_buffers);
 472     if (p_sys->codec) {
 473         if (p_sys->started)
 474             (*env)->CallVoidMethod(env, p_sys->codec, p_sys->stop);
 475         (*env)->CallVoidMethod(env, p_sys->codec, p_sys->release);
 476         (*env)->DeleteGlobalRef(env, p_sys->codec);
 477     }
 478     if (p_sys->buffer_info)
 479         (*env)->DeleteGlobalRef(env, p_sys->buffer_info);
 480     (*myVm)->DetachCurrentThread(myVm);
 481
 482     free(p_sys->name);
 483     ArchitectureSpecificCopyHooksDestroy(p_sys->pixel_format, &p_sys->architecture_specific_data);
 484     free(p_sys->inflight_picture);
 485     free(p_sys);
 486 }
 487
 488 /*****************************************************************************
 489  * vout callbacks
 490  *****************************************************************************/
 491 static void DisplayBuffer(picture_sys_t* p_picsys, bool b_render)
 492 {
 493     decoder_t *p_dec = p_picsys->p_dec;
 494     decoder_sys_t *p_sys = p_dec->p_sys;
 495
 496     if (!p_picsys->b_valid)
 497         return;
 498
 499     vlc_mutex_lock(get_android_opaque_mutex());
 500
 501     /* Picture might have been invalidated while waiting on the mutex. */
 502     if (!p_picsys->b_valid)
 503     {
 504         vlc_mutex_unlock(get_android_opaque_mutex());
 505         return;
 506     }
 507
 508     uint32_t i_index = p_picsys->i_index;
 509     p_sys->inflight_picture[i_index] = NULL;
 510
 511     /* Release the MediaCodec buffer. */
 512     JNIEnv *env = NULL;
 513     (*myVm)->AttachCurrentThread(myVm, &env, NULL);
 514     (*env)->CallVoidMethod(env, p_sys->codec, p_sys->release_output_buffer, i_index, b_render);
 515     if ((*env)->ExceptionOccurred(env)) {
 516         msg_Err(p_dec, "Exception in MediaCodec.releaseOutputBuffer (DisplayBuffer)");
 517         (*env)->ExceptionClear(env);
 518     }
 519
 520     (*myVm)->DetachCurrentThread(myVm);
 521     p_picsys->b_valid = false;
 522
 523     vlc_mutex_unlock(get_android_opaque_mutex());
 524 }
 525
 526 static void UnlockCallback(picture_sys_t* p_picsys)
 527 {
 528     DisplayBuffer(p_picsys, false);
 529 }
 530
 531 static void DisplayCallback(picture_sys_t* p_picsys)
 532 {
 533     DisplayBuffer(p_picsys, true);
 534 }
 535
 536 static void InvalidateAllPictures(decoder_t *p_dec)
 537 {
 538     decoder_sys_t *p_sys = p_dec->p_sys;
 539
 540     vlc_mutex_lock(get_android_opaque_mutex());
 541     for (int i = 0; i < p_sys->i_output_buffers; ++i)
 542     {
 543         picture_t *p_pic = p_sys->inflight_picture[i];
 544         if (p_pic) {
 545             p_pic->p_sys->b_valid = false;
 546             p_sys->inflight_picture[i] = NULL;
 547         }
 548     }
 549     vlc_mutex_unlock(get_android_opaque_mutex());
 550 }
 551
 552 static void GetOutput(decoder_t *p_dec, JNIEnv *env, picture_t **pp_pic)
 553 {
 554     decoder_sys_t *p_sys = p_dec->p_sys;
 555     while (1) {
 556         int index = (*env)->CallIntMethod(env, p_sys->codec, p_sys->dequeue_output_buffer,
 557                                           p_sys->buffer_info, (jlong) 0);
 558         if ((*env)->ExceptionOccurred(env)) {
 559             (*env)->ExceptionClear(env);
 560             p_sys->error_state = true;
 561             return;
 562         }
 563
 564         if (index >= 0) {
 565             if (!p_sys->pixel_format) {
 566                 msg_Warn(p_dec, "Buffers returned before output format is set, dropping frame");
 567                 (*env)->CallVoidMethod(env, p_sys->codec, p_sys->release_output_buffer, index, false);
 568                 continue;
 569             }
 570
 571             if (!*pp_pic) {
 572                 *pp_pic = decoder_NewPicture(p_dec);
 573             } else if (p_sys->direct_rendering) {
 574                 picture_t *p_pic = *pp_pic;
 575                 picture_sys_t *p_picsys = p_pic->p_sys;
 576                 int i_prev_index = p_picsys->i_index;
 577                 (*env)->CallVoidMethod(env, p_sys->codec, p_sys->release_output_buffer, i_prev_index, false);
 578
 579                 // No need to lock here since the previous picture was not sent.
 580                 p_sys->inflight_picture[i_prev_index] = NULL;
 581             }
 582             if (*pp_pic) {
 583
 584                 picture_t *p_pic = *pp_pic;
 585                 // TODO: Use crop_top/crop_left as well? Or is that already taken into account?
 586                 // On OMX_TI_COLOR_FormatYUV420PackedSemiPlanar the offset already incldues
 587                 // the cropping, so the top/left cropping params should just be ignored.
 588                 p_pic->date = (*env)->GetLongField(env, p_sys->buffer_info, p_sys->pts_field);
 589                 if (p_sys->direct_rendering) {
 590                     picture_sys_t *p_picsys = p_pic->p_sys;
 591                     p_picsys->pf_display_callback = DisplayCallback;
 592                     p_picsys->pf_unlock_callback = UnlockCallback;
 593                     p_picsys->p_dec = p_dec;
 594                     p_picsys->i_index = index;
 595                     p_picsys->b_valid = true;
 596
 597                     p_sys->inflight_picture[index] = p_pic;
 598                 } else {
 599                     jobject buf = (*env)->GetObjectArrayElement(env, p_sys->output_buffers, index);
 600                     jsize buf_size = (*env)->GetDirectBufferCapacity(env, buf);
 601                     uint8_t *ptr = (*env)->GetDirectBufferAddress(env, buf);
 602
 603                     int size = (*env)->GetIntField(env, p_sys->buffer_info, p_sys->size_field);
 604                     int offset = (*env)->GetIntField(env, p_sys->buffer_info, p_sys->offset_field);
 605                     ptr += offset; // Check the size parameter as well
 606
 607                     unsigned int chroma_div;
 608                     GetVlcChromaSizes(p_dec->fmt_out.i_codec, p_dec->fmt_out.video.i_width,
 609                                       p_dec->fmt_out.video.i_height, NULL, NULL, &chroma_div);
 610                     CopyOmxPicture(p_sys->pixel_format, p_pic, p_sys->slice_height, p_sys->stride,
 611                                    ptr, chroma_div, &p_sys->architecture_specific_data);
 612                     (*env)->CallVoidMethod(env, p_sys->codec, p_sys->release_output_buffer, index, false);
 613
 614                     jthrowable exception = (*env)->ExceptionOccurred(env);
 615                     if(exception != NULL) {
 616                         jclass illegalStateException = (*env)->FindClass(env, "java/lang/IllegalStateException");
 617                         if((*env)->IsInstanceOf(env, exception, illegalStateException)) {
 618                             msg_Err(p_dec, "Codec error (IllegalStateException) in MediaCodec.releaseOutputBuffer");
 619                             (*env)->ExceptionClear(env);
 620                             (*env)->DeleteLocalRef(env, illegalStateException);
 621                             p_sys->error_state = true;
 622                         }
 623                     }
 624                     (*env)->DeleteLocalRef(env, buf);
 625                 }
 626             } else {
 627                 msg_Warn(p_dec, "NewPicture failed");
 628                 (*env)->CallVoidMethod(env, p_sys->codec, p_sys->release_output_buffer, index, false);
 629             }
 630
 631             return;
 632         } else if (index == INFO_OUTPUT_BUFFERS_CHANGED) {
 633             msg_Dbg(p_dec, "output buffers changed");
 634             (*env)->DeleteGlobalRef(env, p_sys->output_buffers);
 635
 636             p_sys->output_buffers = (*env)->CallObjectMethod(env, p_sys->codec,
 637                                                              p_sys->get_output_buffers);
 638             p_sys->output_buffers = (*env)->NewGlobalRef(env, p_sys->output_buffers);
 639
 640             vlc_mutex_lock(get_android_opaque_mutex());
 641             free(p_sys->inflight_picture);
 642             p_sys->i_output_buffers = (*env)->GetArrayLength(env, p_sys->output_buffers);
 643             p_sys->inflight_picture = calloc(1, sizeof(picture_t*) * p_sys->i_output_buffers);
 644             vlc_mutex_unlock(get_android_opaque_mutex());
 645
 646         } else if (index == INFO_OUTPUT_FORMAT_CHANGED) {
 647
 648             jobject format = (*env)->CallObjectMethod(env, p_sys->codec, p_sys->get_output_format);
 649             jobject format_string = (*env)->CallObjectMethod(env, format, p_sys->tostring);
 650
 651             jsize format_len = (*env)->GetStringUTFLength(env, format_string);
 652             const char *format_ptr = (*env)->GetStringUTFChars(env, format_string, NULL);
 653             msg_Dbg(p_dec, "output format changed: %.*s", format_len, format_ptr);
 654             (*env)->ReleaseStringUTFChars(env, format_string, format_ptr);
 655
 656             ArchitectureSpecificCopyHooksDestroy(p_sys->pixel_format, &p_sys->architecture_specific_data);
 657
 658             int width           = GET_INTEGER(format, "width");
 659             int height          = GET_INTEGER(format, "height");
 660             p_sys->stride       = GET_INTEGER(format, "stride");
 661             p_sys->slice_height = GET_INTEGER(format, "slice-height");
 662             p_sys->pixel_format = GET_INTEGER(format, "color-format");
 663             p_sys->crop_left    = GET_INTEGER(format, "crop-left");
 664             p_sys->crop_top     = GET_INTEGER(format, "crop-top");
 665             int crop_right      = GET_INTEGER(format, "crop-right");
 666             int crop_bottom     = GET_INTEGER(format, "crop-bottom");
 667
 668             const char *name = "unknown";
 669             if (p_sys->direct_rendering) {
 670                 int sar_num = 1, sar_den = 1;
 671                 if (p_dec->fmt_in.video.i_sar_num != 0 && p_dec->fmt_in.video.i_sar_den != 0) {
 672                     sar_num = p_dec->fmt_in.video.i_sar_num;
 673                     sar_den = p_dec->fmt_in.video.i_sar_den;
 674                 }
 675                 jni_SetAndroidSurfaceSizeEnv(env, width, height, width, height, sar_num, sar_den);
 676             } else
 677                 GetVlcChromaFormat(p_sys->pixel_format, &p_dec->fmt_out.i_codec, &name);
 678
 679             msg_Dbg(p_dec, "output: %d %s, %dx%d stride %d %d, crop %d %d %d %d",
 680                     p_sys->pixel_format, name, width, height, p_sys->stride, p_sys->slice_height,
 681                     p_sys->crop_left, p_sys->crop_top, crop_right, crop_bottom);
 682
 683             p_dec->fmt_out.video.i_width = crop_right + 1 - p_sys->crop_left;
 684             p_dec->fmt_out.video.i_height = crop_bottom + 1 - p_sys->crop_top;
 685             if (p_sys->stride <= 0)
 686                 p_sys->stride = width;
 687             if (p_sys->slice_height <= 0)
 688                 p_sys->slice_height = height;
 689             if ((*env)->ExceptionOccurred(env))
 690                 (*env)->ExceptionClear(env);
 691
 692             ArchitectureSpecificCopyHooks(p_dec, p_sys->pixel_format, p_sys->slice_height,
 693                                           p_sys->stride, &p_sys->architecture_specific_data);
 694             if (p_sys->pixel_format == OMX_TI_COLOR_FormatYUV420PackedSemiPlanar) {
 695                 p_sys->slice_height -= p_sys->crop_top/2;
 696                 /* Reset crop top/left here, since the offset parameter already includes this.
 697                  * If we'd ignore the offset parameter in the BufferInfo, we could just keep
 698                  * the original slice height and apply the top/left cropping instead. */
 699                 p_sys->crop_top = 0;
 700                 p_sys->crop_left = 0;
 701             }
 702             if (IgnoreOmxDecoderPadding(p_sys->name)) {
 703                 p_sys->slice_height = 0;
 704                 p_sys->stride = p_dec->fmt_out.video.i_width;
 705             }
 706
 707         } else {
 708             return;
 709         }
 710     }
 711 }
 712
 713 static picture_t *DecodeVideo(decoder_t *p_dec, block_t **pp_block)
 714 {
 715     decoder_sys_t *p_sys = p_dec->p_sys;
 716     picture_t *p_pic = NULL;
 717     JNIEnv *env = NULL;
 718     struct H264ConvertState convert_state = { 0, 0 };
 719
 720     if (!pp_block || !*pp_block)
 721         return NULL;
 722
 723     block_t *p_block = *pp_block;
 724
 725     if (p_sys->error_state) {
 726         block_Release(p_block);
 727         if (!p_sys->error_event_sent) {
 728             /* Signal the error to the Java. */
 729             jni_EventHardwareAccelerationError();
 730             p_sys->error_event_sent = true;
 731         }
 732         return NULL;
 733     }
 734
 735     (*myVm)->AttachCurrentThread(myVm, &env, NULL);
 736
 737     if (p_block->i_flags & (BLOCK_FLAG_DISCONTINUITY|BLOCK_FLAG_CORRUPTED)) {
 738         block_Release(p_block);
 739         if (p_sys->decoded) {
 740             /* Invalidate all pictures that are currently in flight
 741              * since flushing make all previous indices returned by
 742              * MediaCodec invalid. */
 743             if (p_sys->direct_rendering)
 744                 InvalidateAllPictures(p_dec);
 745
 746             (*env)->CallVoidMethod(env, p_sys->codec, p_sys->flush);
 747             if ((*env)->ExceptionOccurred(env)) {
 748                 msg_Warn(p_dec, "Exception occurred in MediaCodec.flush");
 749                 (*env)->ExceptionClear(env);
 750                 p_sys->error_state = true;
 751             }
 752         }
 753         p_sys->decoded = false;
 754         (*myVm)->DetachCurrentThread(myVm);
 755         return NULL;
 756     }
 757
 758     /* Use the aspect ratio provided by the input (ie read from packetizer).
 759      * Don't check the current value of the aspect ratio in fmt_out, since we
 760      * want to allow changes in it to propagate. */
 761     if (p_dec->fmt_in.video.i_sar_num != 0 && p_dec->fmt_in.video.i_sar_den != 0) {
 762         p_dec->fmt_out.video.i_sar_num = p_dec->fmt_in.video.i_sar_num;
 763         p_dec->fmt_out.video.i_sar_den = p_dec->fmt_in.video.i_sar_den;
 764     }
 765
 766     jlong timeout = 0;
 767     const int max_polling_attempts = 50;
 768     int attempts = 0;
 769     while (true) {
 770         int index = (*env)->CallIntMethod(env, p_sys->codec, p_sys->dequeue_input_buffer, timeout);
 771         if ((*env)->ExceptionOccurred(env)) {
 772             (*env)->ExceptionClear(env);
 773             p_sys->error_state = true;
 774             break;
 775         }
 776         if (index < 0) {
 777             GetOutput(p_dec, env, &p_pic);
 778             if (p_pic) {
 779                 /* If we couldn't get an available input buffer but a
 780                  * decoded frame is available, we return the frame
 781                  * without assigning NULL to *pp_block. The next call
 782                  * to DecodeVideo will try to send the input packet again.
 783                  */
 784                 (*myVm)->DetachCurrentThread(myVm);
 785                 return p_pic;
 786             }
 787             timeout = 30*1000;
 788             ++attempts;
 789             /* With opaque DR the output buffers are released by the
 790                vout therefore we implement a timeout for polling in
 791                order to avoid being indefinitely stalled in this loop. */
 792             if (p_sys->direct_rendering && attempts == max_polling_attempts) {
 793                 picture_t *invalid_picture = decoder_NewPicture(p_dec);
 794                 if (invalid_picture) {
 795                     invalid_picture->date = VLC_TS_INVALID;
 796                     picture_sys_t *p_picsys = invalid_picture->p_sys;
 797                     p_picsys->pf_display_callback = NULL;
 798                     p_picsys->pf_unlock_callback = NULL;
 799                     p_picsys->p_dec = NULL;
 800                     p_picsys->i_index = -1;
 801                     p_picsys->b_valid = false;
 802                 }
 803                 else {
 804                     /* If we cannot return a picture we must free the
 805                        block since the decoder will proceed with the
 806                        next block. */
 807                     block_Release(p_block);
 808                     *pp_block = NULL;
 809                 }
 810                 (*myVm)->DetachCurrentThread(myVm);
 811                 return invalid_picture;
 812             }
 813             continue;
 814         }
 815         jobject buf = (*env)->GetObjectArrayElement(env, p_sys->input_buffers, index);
 816         jsize size = (*env)->GetDirectBufferCapacity(env, buf);
 817         uint8_t *bufptr = (*env)->GetDirectBufferAddress(env, buf);
 818         if (size > p_block->i_buffer)
 819             size = p_block->i_buffer;
 820         memcpy(bufptr, p_block->p_buffer, size);
 821
 822         convert_h264_to_annexb(bufptr, size, p_sys->nal_size, &convert_state);
 823
 824         int64_t ts = p_block->i_pts;
 825         if (!ts && p_block->i_dts)
 826             ts = p_block->i_dts;
 827         (*env)->CallVoidMethod(env, p_sys->codec, p_sys->queue_input_buffer, index, 0, size, ts, 0);
 828         (*env)->DeleteLocalRef(env, buf);
 829         p_sys->decoded = true;
 830         break;
 831     }
 832     if (!p_pic)
 833         GetOutput(p_dec, env, &p_pic);
 834     (*myVm)->DetachCurrentThread(myVm);
 835
 836     block_Release(p_block);
 837     *pp_block = NULL;
 838
 839     return p_pic;
 840 }