hevc: Fix modulo operations

Keep qp fields within the range.

Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
Signed-off-by: Luca Barbato <lu_zero@gentoo.org>

diff --git a/libavcodec/hevc.c b/libavcodec/hevc.c
index 438794ded70073b4bf9c529f2162376af08e7df4..d5175f52d2cd060bea7a4717866a08f4da892f23 100644 (file)
--- a/libavcodec/hevc.c
+++ b/libavcodec/hevc.c
@@ -771,8 +771,8 @@ static int hls_slice_header(HEVCContext *s)
     s->HEVClc.first_qp_group = !s->sh.dependent_slice_segment_flag;
 
     if (!s->pps->cu_qp_delta_enabled_flag)
-        s->HEVClc.qp_y = ((s->sh.slice_qp + 52 + 2 * s->sps->qp_bd_offset) %
-                          (52 + s->sps->qp_bd_offset)) - s->sps->qp_bd_offset;
+        s->HEVClc.qp_y = FFUMOD(s->sh.slice_qp + 52 + 2 * s->sps->qp_bd_offset,
+                                52 + s->sps->qp_bd_offset) - s->sps->qp_bd_offset;
 
     s->slice_initialized = 1;
 

diff --git a/libavcodec/hevc.h b/libavcodec/hevc.h
index 07d70739ded0d7bf72f7434f3f7de8368a03dd1f..f623887c6e2cf3daaf017304cd74540a35de77c1 100644 (file)
--- a/libavcodec/hevc.h
+++ b/libavcodec/hevc.h
@@ -85,6 +85,9 @@
                    s->nal_unit_type == NAL_BLA_N_LP)
 #define IS_IRAP(s) (s->nal_unit_type >= 16 && s->nal_unit_type <= 23)
 
+#define FFUDIV(a,b) (((a) > 0 ? (a) : (a) - (b) + 1) / (b))
+#define FFUMOD(a,b) ((a) - (b) * FFUDIV(a,b))
+
 /**
  * Table 7-3: NAL unit type codes
  */

diff --git a/libavcodec/hevc_filter.c b/libavcodec/hevc_filter.c
index bb1e360dd47d0461cf2c12dcf352e3c111e23523..f3c655416a859154b3978a1e55fc21dea9a2f630 100644 (file)
--- a/libavcodec/hevc_filter.c
+++ b/libavcodec/hevc_filter.c
@@ -158,8 +158,8 @@ void ff_hevc_set_qPy(HEVCContext *s, int xC, int yC,
 
     if (s->HEVClc.tu.cu_qp_delta != 0) {
         int off = s->sps->qp_bd_offset;
-        s->HEVClc.qp_y = ((qp_y + s->HEVClc.tu.cu_qp_delta + 52 + 2 * off) %
-                          (52 + off)) - off;
+        s->HEVClc.qp_y = FFUMOD(qp_y + s->HEVClc.tu.cu_qp_delta + 52 + 2 * off,
+                                52 + off) - off;
     } else
         s->HEVClc.qp_y = qp_y;
 }