2 * RockChip MPP Video Decoder
3 * Copyright (c) 2017 Lionel CHAZALLON
5 * This file is part of FFmpeg.
7 * FFmpeg is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation; either
10 * version 2.1 of the License, or (at your option) any later version.
12 * FFmpeg is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with FFmpeg; if not, write to the Free Software
19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
22 #include <drm_fourcc.h>
24 #include <rockchip/mpp_buffer.h>
25 #include <rockchip/rk_mpi.h>
32 #include "libavutil/buffer.h"
33 #include "libavutil/common.h"
34 #include "libavutil/frame.h"
35 #include "libavutil/hwcontext.h"
36 #include "libavutil/hwcontext_drm.h"
37 #include "libavutil/imgutils.h"
38 #include "libavutil/log.h"
40 #define RECEIVE_FRAME_TIMEOUT 100
41 #define FRAMEGROUP_MAX_FRAMES 16
46 MppBufferGroup frame_group;
52 AVBufferRef *frames_ref;
53 AVBufferRef *device_ref;
58 AVBufferRef *decoder_ref;
63 AVBufferRef *decoder_ref;
66 static MppCodingType rkmpp_get_codingtype(AVCodecContext *avctx)
68 switch (avctx->codec_id) {
69 case AV_CODEC_ID_H264: return MPP_VIDEO_CodingAVC;
70 case AV_CODEC_ID_HEVC: return MPP_VIDEO_CodingHEVC;
71 case AV_CODEC_ID_VP8: return MPP_VIDEO_CodingVP8;
72 case AV_CODEC_ID_VP9: return MPP_VIDEO_CodingVP9;
73 default: return MPP_VIDEO_CodingUnused;
77 static uint32_t rkmpp_get_frameformat(MppFrameFormat mppformat)
80 case MPP_FMT_YUV420SP: return DRM_FORMAT_NV12;
81 #ifdef DRM_FORMAT_NV12_10
82 case MPP_FMT_YUV420SP_10BIT: return DRM_FORMAT_NV12_10;
88 static int rkmpp_write_data(AVCodecContext *avctx, uint8_t *buffer, int size, int64_t pts)
90 RKMPPDecodeContext *rk_context = avctx->priv_data;
91 RKMPPDecoder *decoder = (RKMPPDecoder *)rk_context->decoder_ref->data;
95 // create the MPP packet
96 ret = mpp_packet_init(&packet, buffer, size);
98 av_log(avctx, AV_LOG_ERROR, "Failed to init MPP packet (code = %d)\n", ret);
99 return AVERROR_UNKNOWN;
102 mpp_packet_set_pts(packet, pts);
105 mpp_packet_set_eos(packet);
107 ret = decoder->mpi->decode_put_packet(decoder->ctx, packet);
109 if (ret == MPP_ERR_BUFFER_FULL) {
110 av_log(avctx, AV_LOG_DEBUG, "Buffer full writing %d bytes to decoder\n", size);
111 ret = AVERROR(EAGAIN);
113 ret = AVERROR_UNKNOWN;
116 av_log(avctx, AV_LOG_DEBUG, "Wrote %d bytes to decoder\n", size);
118 mpp_packet_deinit(&packet);
123 static int rkmpp_close_decoder(AVCodecContext *avctx)
125 RKMPPDecodeContext *rk_context = avctx->priv_data;
126 av_buffer_unref(&rk_context->decoder_ref);
130 static void rkmpp_release_decoder(void *opaque, uint8_t *data)
132 RKMPPDecoder *decoder = (RKMPPDecoder *)data;
135 decoder->mpi->reset(decoder->ctx);
136 mpp_destroy(decoder->ctx);
140 if (decoder->frame_group) {
141 mpp_buffer_group_put(decoder->frame_group);
142 decoder->frame_group = NULL;
145 av_buffer_unref(&decoder->frames_ref);
146 av_buffer_unref(&decoder->device_ref);
151 static int rkmpp_init_decoder(AVCodecContext *avctx)
153 RKMPPDecodeContext *rk_context = avctx->priv_data;
154 RKMPPDecoder *decoder = NULL;
155 MppCodingType codectype = MPP_VIDEO_CodingUnused;
160 avctx->pix_fmt = AV_PIX_FMT_DRM_PRIME;
162 // create a decoder and a ref to it
163 decoder = av_mallocz(sizeof(RKMPPDecoder));
165 ret = AVERROR(ENOMEM);
169 rk_context->decoder_ref = av_buffer_create((uint8_t *)decoder, sizeof(*decoder), rkmpp_release_decoder,
170 NULL, AV_BUFFER_FLAG_READONLY);
171 if (!rk_context->decoder_ref) {
173 ret = AVERROR(ENOMEM);
177 av_log(avctx, AV_LOG_DEBUG, "Initializing RKMPP decoder.\n");
179 codectype = rkmpp_get_codingtype(avctx);
180 if (codectype == MPP_VIDEO_CodingUnused) {
181 av_log(avctx, AV_LOG_ERROR, "Unknown codec type (%d).\n", avctx->codec_id);
182 ret = AVERROR_UNKNOWN;
186 ret = mpp_check_support_format(MPP_CTX_DEC, codectype);
188 av_log(avctx, AV_LOG_ERROR, "Codec type (%d) unsupported by MPP\n", avctx->codec_id);
189 ret = AVERROR_UNKNOWN;
193 // Create the MPP context
194 ret = mpp_create(&decoder->ctx, &decoder->mpi);
196 av_log(avctx, AV_LOG_ERROR, "Failed to create MPP context (code = %d).\n", ret);
197 ret = AVERROR_UNKNOWN;
202 ret = mpp_init(decoder->ctx, MPP_CTX_DEC, codectype);
204 av_log(avctx, AV_LOG_ERROR, "Failed to initialize MPP context (code = %d).\n", ret);
205 ret = AVERROR_UNKNOWN;
209 // make decode calls blocking with a timeout
210 paramS32 = MPP_POLL_BLOCK;
211 ret = decoder->mpi->control(decoder->ctx, MPP_SET_OUTPUT_BLOCK, ¶mS32);
213 av_log(avctx, AV_LOG_ERROR, "Failed to set blocking mode on MPI (code = %d).\n", ret);
214 ret = AVERROR_UNKNOWN;
218 paramS64 = RECEIVE_FRAME_TIMEOUT;
219 ret = decoder->mpi->control(decoder->ctx, MPP_SET_OUTPUT_BLOCK_TIMEOUT, ¶mS64);
221 av_log(avctx, AV_LOG_ERROR, "Failed to set block timeout on MPI (code = %d).\n", ret);
222 ret = AVERROR_UNKNOWN;
226 ret = mpp_buffer_group_get_internal(&decoder->frame_group, MPP_BUFFER_TYPE_ION);
228 av_log(avctx, AV_LOG_ERROR, "Failed to retrieve buffer group (code = %d)\n", ret);
229 ret = AVERROR_UNKNOWN;
233 ret = decoder->mpi->control(decoder->ctx, MPP_DEC_SET_EXT_BUF_GROUP, decoder->frame_group);
235 av_log(avctx, AV_LOG_ERROR, "Failed to assign buffer group (code = %d)\n", ret);
236 ret = AVERROR_UNKNOWN;
240 ret = mpp_buffer_group_limit_config(decoder->frame_group, 0, FRAMEGROUP_MAX_FRAMES);
242 av_log(avctx, AV_LOG_ERROR, "Failed to set buffer group limit (code = %d)\n", ret);
243 ret = AVERROR_UNKNOWN;
247 decoder->first_packet = 1;
249 av_log(avctx, AV_LOG_DEBUG, "RKMPP decoder initialized successfully.\n");
251 decoder->device_ref = av_hwdevice_ctx_alloc(AV_HWDEVICE_TYPE_DRM);
252 if (!decoder->device_ref) {
253 ret = AVERROR(ENOMEM);
256 ret = av_hwdevice_ctx_init(decoder->device_ref);
263 av_log(avctx, AV_LOG_ERROR, "Failed to initialize RKMPP decoder.\n");
264 rkmpp_close_decoder(avctx);
268 static int rkmpp_send_packet(AVCodecContext *avctx, const AVPacket *avpkt)
270 RKMPPDecodeContext *rk_context = avctx->priv_data;
271 RKMPPDecoder *decoder = (RKMPPDecoder *)rk_context->decoder_ref->data;
276 av_log(avctx, AV_LOG_DEBUG, "End of stream.\n");
277 decoder->eos_reached = 1;
278 ret = rkmpp_write_data(avctx, NULL, 0, 0);
280 av_log(avctx, AV_LOG_ERROR, "Failed to send EOS to decoder (code = %d)\n", ret);
284 // on first packet, send extradata
285 if (decoder->first_packet) {
286 if (avctx->extradata_size) {
287 ret = rkmpp_write_data(avctx, avctx->extradata,
288 avctx->extradata_size,
291 av_log(avctx, AV_LOG_ERROR, "Failed to write extradata to decoder (code = %d)\n", ret);
295 decoder->first_packet = 0;
299 ret = rkmpp_write_data(avctx, avpkt->data, avpkt->size, avpkt->pts);
300 if (ret && ret!=AVERROR(EAGAIN))
301 av_log(avctx, AV_LOG_ERROR, "Failed to write data to decoder (code = %d)\n", ret);
306 static void rkmpp_release_frame(void *opaque, uint8_t *data)
308 AVDRMFrameDescriptor *desc = (AVDRMFrameDescriptor *)data;
309 AVBufferRef *framecontextref = (AVBufferRef *)opaque;
310 RKMPPFrameContext *framecontext = (RKMPPFrameContext *)framecontextref->data;
312 mpp_frame_deinit(&framecontext->frame);
313 av_buffer_unref(&framecontext->decoder_ref);
314 av_buffer_unref(&framecontextref);
319 static int rkmpp_retrieve_frame(AVCodecContext *avctx, AVFrame *frame)
321 RKMPPDecodeContext *rk_context = avctx->priv_data;
322 RKMPPDecoder *decoder = (RKMPPDecoder *)rk_context->decoder_ref->data;
323 RKMPPFrameContext *framecontext = NULL;
324 AVBufferRef *framecontextref = NULL;
326 MppFrame mppframe = NULL;
327 MppBuffer buffer = NULL;
328 AVDRMFrameDescriptor *desc = NULL;
329 AVDRMLayerDescriptor *layer = NULL;
332 MppFrameFormat mppformat;
335 // on start of decoding, MPP can return -1, which is supposed to be expected
336 // this is due to some internal MPP init which is not completed, that will
337 // only happen in the first few frames queries, but should not be interpreted
338 // as an error, Therefore we need to retry a couple times when we get -1
339 // in order to let it time to complete it's init, then we sleep a bit between retries.
341 ret = decoder->mpi->decode_get_frame(decoder->ctx, &mppframe);
342 if (ret != MPP_OK && ret != MPP_ERR_TIMEOUT && !decoder->first_frame) {
343 if (retrycount < 5) {
344 av_log(avctx, AV_LOG_DEBUG, "Failed to get a frame, retrying (code = %d, retrycount = %d)\n", ret, retrycount);
347 goto retry_get_frame;
349 av_log(avctx, AV_LOG_ERROR, "Failed to get a frame from MPP (code = %d)\n", ret);
355 // Check whether we have a special frame or not
356 if (mpp_frame_get_info_change(mppframe)) {
357 AVHWFramesContext *hwframes;
359 av_log(avctx, AV_LOG_INFO, "Decoder noticed an info change (%dx%d), format=%d\n",
360 (int)mpp_frame_get_width(mppframe), (int)mpp_frame_get_height(mppframe),
361 (int)mpp_frame_get_fmt(mppframe));
363 avctx->width = mpp_frame_get_width(mppframe);
364 avctx->height = mpp_frame_get_height(mppframe);
366 decoder->mpi->control(decoder->ctx, MPP_DEC_SET_INFO_CHANGE_READY, NULL);
367 decoder->first_frame = 1;
369 av_buffer_unref(&decoder->frames_ref);
371 decoder->frames_ref = av_hwframe_ctx_alloc(decoder->device_ref);
372 if (!decoder->frames_ref) {
373 ret = AVERROR(ENOMEM);
377 mppformat = mpp_frame_get_fmt(mppframe);
378 drmformat = rkmpp_get_frameformat(mppformat);
380 hwframes = (AVHWFramesContext*)decoder->frames_ref->data;
381 hwframes->format = AV_PIX_FMT_DRM_PRIME;
382 hwframes->sw_format = drmformat == DRM_FORMAT_NV12 ? AV_PIX_FMT_NV12 : AV_PIX_FMT_NONE;
383 hwframes->width = avctx->width;
384 hwframes->height = avctx->height;
385 ret = av_hwframe_ctx_init(decoder->frames_ref);
389 // here decoder is fully initialized, we need to feed it again with data
390 ret = AVERROR(EAGAIN);
392 } else if (mpp_frame_get_eos(mppframe)) {
393 av_log(avctx, AV_LOG_DEBUG, "Received a EOS frame.\n");
394 decoder->eos_reached = 1;
397 } else if (mpp_frame_get_discard(mppframe)) {
398 av_log(avctx, AV_LOG_DEBUG, "Received a discard frame.\n");
399 ret = AVERROR(EAGAIN);
401 } else if (mpp_frame_get_errinfo(mppframe)) {
402 av_log(avctx, AV_LOG_ERROR, "Received a errinfo frame.\n");
403 ret = AVERROR_UNKNOWN;
407 // here we should have a valid frame
408 av_log(avctx, AV_LOG_DEBUG, "Received a frame.\n");
410 // setup general frame fields
411 frame->format = AV_PIX_FMT_DRM_PRIME;
412 frame->width = mpp_frame_get_width(mppframe);
413 frame->height = mpp_frame_get_height(mppframe);
414 frame->pts = mpp_frame_get_pts(mppframe);
415 frame->color_range = mpp_frame_get_color_range(mppframe);
416 frame->color_primaries = mpp_frame_get_color_primaries(mppframe);
417 frame->color_trc = mpp_frame_get_color_trc(mppframe);
418 frame->colorspace = mpp_frame_get_colorspace(mppframe);
420 mode = mpp_frame_get_mode(mppframe);
421 frame->interlaced_frame = ((mode & MPP_FRAME_FLAG_FIELD_ORDER_MASK) == MPP_FRAME_FLAG_DEINTERLACED);
422 frame->top_field_first = ((mode & MPP_FRAME_FLAG_FIELD_ORDER_MASK) == MPP_FRAME_FLAG_TOP_FIRST);
424 mppformat = mpp_frame_get_fmt(mppframe);
425 drmformat = rkmpp_get_frameformat(mppformat);
427 // now setup the frame buffer info
428 buffer = mpp_frame_get_buffer(mppframe);
430 desc = av_mallocz(sizeof(AVDRMFrameDescriptor));
432 ret = AVERROR(ENOMEM);
436 desc->nb_objects = 1;
437 desc->objects[0].fd = mpp_buffer_get_fd(buffer);
438 desc->objects[0].size = mpp_buffer_get_size(buffer);
441 layer = &desc->layers[0];
442 layer->format = drmformat;
443 layer->nb_planes = 2;
445 layer->planes[0].object_index = 0;
446 layer->planes[0].offset = 0;
447 layer->planes[0].pitch = mpp_frame_get_hor_stride(mppframe);
449 layer->planes[1].object_index = 0;
450 layer->planes[1].offset = layer->planes[0].pitch * mpp_frame_get_ver_stride(mppframe);
451 layer->planes[1].pitch = layer->planes[0].pitch;
453 // we also allocate a struct in buf[0] that will allow to hold additionnal information
454 // for releasing properly MPP frames and decoder
455 framecontextref = av_buffer_allocz(sizeof(*framecontext));
456 if (!framecontextref) {
457 ret = AVERROR(ENOMEM);
461 // MPP decoder needs to be closed only when all frames have been released.
462 framecontext = (RKMPPFrameContext *)framecontextref->data;
463 framecontext->decoder_ref = av_buffer_ref(rk_context->decoder_ref);
464 framecontext->frame = mppframe;
466 frame->data[0] = (uint8_t *)desc;
467 frame->buf[0] = av_buffer_create((uint8_t *)desc, sizeof(*desc), rkmpp_release_frame,
468 framecontextref, AV_BUFFER_FLAG_READONLY);
470 if (!frame->buf[0]) {
471 ret = AVERROR(ENOMEM);
475 frame->hw_frames_ctx = av_buffer_ref(decoder->frames_ref);
476 if (!frame->hw_frames_ctx) {
477 ret = AVERROR(ENOMEM);
481 decoder->first_frame = 0;
484 av_log(avctx, AV_LOG_ERROR, "Failed to retrieve the frame buffer, frame is dropped (code = %d)\n", ret);
485 mpp_frame_deinit(&mppframe);
487 } else if (decoder->eos_reached) {
489 } else if (ret == MPP_ERR_TIMEOUT) {
490 av_log(avctx, AV_LOG_DEBUG, "Timeout when trying to get a frame from MPP\n");
493 return AVERROR(EAGAIN);
497 mpp_frame_deinit(&mppframe);
500 av_buffer_unref(&framecontext->decoder_ref);
503 av_buffer_unref(&framecontextref);
511 static int rkmpp_receive_frame(AVCodecContext *avctx, AVFrame *frame)
513 RKMPPDecodeContext *rk_context = avctx->priv_data;
514 RKMPPDecoder *decoder = (RKMPPDecoder *)rk_context->decoder_ref->data;
519 if (!decoder->eos_reached) {
520 // we get the available slots in decoder
521 ret = decoder->mpi->control(decoder->ctx, MPP_DEC_GET_FREE_PACKET_SLOT_COUNT, &freeslots);
523 av_log(avctx, AV_LOG_ERROR, "Failed to get decoder free slots (code = %d).\n", ret);
528 ret = ff_decode_get_packet(avctx, &pkt);
529 if (ret < 0 && ret != AVERROR_EOF) {
533 ret = rkmpp_send_packet(avctx, &pkt);
534 av_packet_unref(&pkt);
537 av_log(avctx, AV_LOG_ERROR, "Failed to send packet to decoder (code = %d)\n", ret);
542 // make sure we keep decoder full
543 if (freeslots > 1 && decoder->first_frame)
544 return AVERROR(EAGAIN);
547 return rkmpp_retrieve_frame(avctx, frame);
550 static void rkmpp_flush(AVCodecContext *avctx)
552 RKMPPDecodeContext *rk_context = avctx->priv_data;
553 RKMPPDecoder *decoder = (RKMPPDecoder *)rk_context->decoder_ref->data;
556 av_log(avctx, AV_LOG_DEBUG, "Flush.\n");
558 ret = decoder->mpi->reset(decoder->ctx);
560 decoder->first_frame = 1;
561 decoder->first_packet = 1;
563 av_log(avctx, AV_LOG_ERROR, "Failed to reset MPI (code = %d)\n", ret);
567 #define RKMPP_DEC_CLASS(NAME) \
568 static const AVClass rkmpp_##NAME##_dec_class = { \
569 .class_name = "rkmpp_" #NAME "_dec", \
570 .version = LIBAVUTIL_VERSION_INT, \
573 #define RKMPP_DEC(NAME, ID, BSFS) \
574 RKMPP_DEC_CLASS(NAME) \
575 AVCodec ff_##NAME##_rkmpp_decoder = { \
576 .name = #NAME "_rkmpp", \
577 .long_name = NULL_IF_CONFIG_SMALL(#NAME " (rkmpp)"), \
578 .type = AVMEDIA_TYPE_VIDEO, \
580 .priv_data_size = sizeof(RKMPPDecodeContext), \
581 .init = rkmpp_init_decoder, \
582 .close = rkmpp_close_decoder, \
583 .receive_frame = rkmpp_receive_frame, \
584 .flush = rkmpp_flush, \
585 .priv_class = &rkmpp_##NAME##_dec_class, \
586 .capabilities = AV_CODEC_CAP_DELAY, \
587 .caps_internal = AV_CODEC_CAP_AVOID_PROBING, \
588 .pix_fmts = (const enum AVPixelFormat[]) { AV_PIX_FMT_DRM_PRIME, \
593 RKMPP_DEC(h264, AV_CODEC_ID_H264, "h264_mp4toannexb")
594 RKMPP_DEC(hevc, AV_CODEC_ID_HEVC, "hevc_mp4toannexb")
595 RKMPP_DEC(vp8, AV_CODEC_ID_VP8, NULL)
596 RKMPP_DEC(vp9, AV_CODEC_ID_VP9, NULL)