opengl: on manual unpacking use picture visible width for calculations, fixes some...

diff --git a/modules/video_output/opengl.c b/modules/video_output/opengl.c
index e89cf63ba7f4b7bb90009012c533144e00aff6fc..36a514d7378f869c3cbe1b3115e40bcfc17f44cb 100644 (file)
--- a/modules/video_output/opengl.c
+++ b/modules/video_output/opengl.c
@@ -659,20 +659,23 @@ int vout_display_opengl_Prepare(vout_display_opengl_t *vgl,
         }
         glBindTexture(vgl->tex_target, vgl->texture[0][j]);
 
-        if ( (picture->p[j].i_pitch / picture->p[j].i_pixel_pitch) != (vgl->fmt.i_width  * vgl->chroma->p[j].w.num / vgl->chroma->p[j].w.den) ) {
-            uint8_t *new_plane = malloc(picture->p[j].i_pitch*picture->p[j].i_pixel_pitch *vgl->fmt.i_height * vgl->chroma->p[j].h.num / vgl->chroma->p[j].h.den);
 #ifndef GL_UNPACK_ROW_LENGTH
+        if ( (picture->p[j].i_pitch / picture->p[j].i_pixel_pitch) !=
+             ( picture->format.i_visible_width * vgl->chroma->p[j].w.num / vgl->chroma->p[j].w.den ) )
+        {
+            uint8_t *new_plane = malloc( picture->format.i_visible_width * vgl->fmt.i_visible_height * vgl->chroma->p[j].h.num / (vgl->chroma->p[j].h.den *  vgl->chroma->p[j].w.den ) );
             uint8_t *destination = new_plane;
             const uint8_t *source = picture->p[j].p_pixels;
-            for( unsigned height = 0; height < vgl->fmt.i_height * vgl->chroma->p[j].h.num / vgl->chroma->p[j].h.den; height++ )
+
+            for( unsigned height = 0; height < (vgl->fmt.i_visible_height * vgl->chroma->p[j].h.num / vgl->chroma->p[j].h.den) ; height++ )
             {
-                memcpy( destination, source, vgl->fmt.i_width  * vgl->chroma->p[j].w.num / vgl->chroma->p[j].w.den );
-                source +=  picture->p[j].i_pitch*picture->p[j].i_pixel_pitch;
-                destination += vgl->fmt.i_width  * vgl->chroma->p[j].w.num / vgl->chroma->p[j].w.den;
+                memcpy( destination, source, picture->format.i_visible_width * vgl->chroma->p[j].w.num / vgl->chroma->p[j].w.den );
+                source += picture->p[j].i_pitch / picture->p[j].i_pixel_pitch;
+                destination += picture->format.i_visible_width * vgl->chroma->p[j].w.num / vgl->chroma->p[j].w.den;
             }
             glTexSubImage2D( vgl->tex_target, 0,
                              0, 0,
-                             vgl->fmt.i_width  * vgl->chroma->p[j].w.num / vgl->chroma->p[j].w.den,
+                             picture->format.i_visible_width * vgl->chroma->p[j].w.num / vgl->chroma->p[j].w.den,
                              vgl->fmt.i_height * vgl->chroma->p[j].h.num / vgl->chroma->p[j].h.den,
                              vgl->tex_format, vgl->tex_type, new_plane );
             free( new_plane );