More correct ref comparison in the loop filter.

Originally committed as revision 14322 to svn://svn.ffmpeg.org/ffmpeg/trunk

diff --git a/libavcodec/h264.c b/libavcodec/h264.c
index 7fa1b074737b3e381b9f2f9a128a7556989de479..c35779dfcb0b2776e31ae7f7e7200c470a13f4fc 100644 (file)
--- a/libavcodec/h264.c
+++ b/libavcodec/h264.c
@@ -6727,7 +6727,7 @@ static void filter_mb( H264Context *h, int mb_x, int mb_y, uint8_t *img_y, uint8
                     int b_idx= 8 + 4 + edge * (dir ? 8:1);
                     int bn_idx= b_idx - (dir ? 8:1);
                     int v = 0;
-                    int xn= (h->ref_cache[0][ b_idx] ^ h->ref_cache[0][bn_idx]) < 0;
+                    int xn= h->slice_type == FF_B_TYPE && h->ref2frm[0][h->ref_cache[0][b_idx]+2] != h->ref2frm[0][h->ref_cache[0][bn_idx]+2];
 
                     for( l = 0; !v && l < 1 + (h->slice_type == FF_B_TYPE); l++ ) {
                         int ln= l^xn;
@@ -6753,7 +6753,7 @@ static void filter_mb( H264Context *h, int mb_x, int mb_y, uint8_t *img_y, uint8
                     }
                     else if(!mv_done)
                     {
-                        int xn= (h->ref_cache[0][ b_idx] ^ h->ref_cache[0][bn_idx]) < 0;
+                        int xn= h->slice_type == FF_B_TYPE && h->ref2frm[0][h->ref_cache[0][b_idx]+2] != h->ref2frm[0][h->ref_cache[0][bn_idx]+2];
                         bS[i] = 0;
                         for( l = 0; l < 1 + (h->slice_type == FF_B_TYPE); l++ ) {
                             int ln= l^xn;