Add missing overflow checks in av_image_fill_pointers() and

[ffmpeg] / libavdevice / x11grab.c

diff --git a/libavdevice/x11grab.c b/libavdevice/x11grab.c
index 60993276f5aac2ed73cde5c2ce6940d2a07a0a2e..ee3b8e901e116b6a558b794b87af13d71ff25ed6 100644 (file)
--- a/libavdevice/x11grab.c
+++ b/libavdevice/x11grab.c
@@ -30,7 +30,7 @@
  */
 
 /**
- * @file libavdevice/x11grab.c
+ * @file
  * X11 frame device demuxer by Clemens Fruhwirth <clemens@endorphin.org>
  * and Edouard Gomez <ed.gomez@free.fr>.
  */
@@ -71,7 +71,7 @@ struct x11_grab
 };
 
 /**
- * Initializes the x11 grab device demuxer (public device demuxer API).
+ * Initialize the x11 grab device demuxer (public device demuxer API).
  *
  * @param s1 Context from avformat core
  * @param ap Parameters from avformat core
@@ -110,7 +110,7 @@ x11grab_read_header(AVFormatContext *s1, AVFormatParameters *ap)
         return AVERROR(EIO);
     }
 
-    if (!ap || ap->width <= 0 || ap->height <= 0 || ap->time_base.den <= 0) {
+    if (ap->width <= 0 || ap->height <= 0 || ap->time_base.den <= 0) {
         av_log(s1, AV_LOG_ERROR, "AVParameters don't have video size and/or rate. Use -s and -r.\n");
         return AVERROR(EIO);
     }
@@ -226,7 +226,7 @@ x11grab_read_header(AVFormatContext *s1, AVFormatParameters *ap)
     x11grab->image = image;
     x11grab->use_shm = use_shm;
 
-    st->codec->codec_type = CODEC_TYPE_VIDEO;
+    st->codec->codec_type = AVMEDIA_TYPE_VIDEO;
     st->codec->codec_id = CODEC_ID_RAWVIDEO;
     st->codec->width = ap->width;
     st->codec->height = ap->height;
@@ -238,13 +238,11 @@ x11grab_read_header(AVFormatContext *s1, AVFormatParameters *ap)
 }
 
 /**
- * Paints a mouse pointer in an X11 image.
+ * Paint a mouse pointer in an X11 image.
  *
  * @param image image to paint the mouse pointer to
  * @param s context used to retrieve original grabbing rectangle
  *          coordinates
- * @param x Mouse pointer coordinate
- * @param y Mouse pointer coordinate
  */
 static void
 paint_mouse_pointer(XImage *image, struct x11_grab *s)
@@ -258,9 +256,18 @@ paint_mouse_pointer(XImage *image, struct x11_grab *s)
     int x, y;
     int line, column;
     int to_line, to_column;
-    int image_addr, xcim_addr;
+    int pixstride = image->bits_per_pixel >> 3;
+    /* Warning: in its insanity, xlib provides unsigned image data through a
+     * char* pointer, so we have to make it uint8_t to make things not break.
+     * Anyone who performs further investigation of the xlib API likely risks
+     * permanent brain damage. */
+    uint8_t *pix = image->data;
 
-    xcim = XFixesGetCursorImage(dpy);;
+    /* Code doesn't currently support 16-bit or PAL8 */
+    if (image->bits_per_pixel != 24 && image->bits_per_pixel != 32)
+        return;
+
+    xcim = XFixesGetCursorImage(dpy);
 
     x = xcim->x - xcim->xhot;
     y = xcim->y - xcim->yhot;
@@ -270,14 +277,22 @@ paint_mouse_pointer(XImage *image, struct x11_grab *s)
 
     for (line = FFMAX(y, y_off); line < to_line; line++) {
         for (column = FFMAX(x, x_off); column < to_column; column++) {
-            xcim_addr = (line - y) * xcim->width + column - x;
-
-            if ((unsigned char)(xcim->pixels[xcim_addr] >> 24) != 0) { // skip fully transparent pixel
-                image_addr = ((line - y_off) * width + column - x_off) * 4;
-
-                image->data[image_addr] = (unsigned char)(xcim->pixels[xcim_addr] >> 0);
-                image->data[image_addr+1] = (unsigned char)(xcim->pixels[xcim_addr] >> 8);
-                image->data[image_addr+2] = (unsigned char)(xcim->pixels[xcim_addr] >> 16);
+            int  xcim_addr = (line - y) * xcim->width + column - x;
+            int image_addr = ((line - y_off) * width + column - x_off) * pixstride;
+            int r = (uint8_t)(xcim->pixels[xcim_addr] >>  0);
+            int g = (uint8_t)(xcim->pixels[xcim_addr] >>  8);
+            int b = (uint8_t)(xcim->pixels[xcim_addr] >> 16);
+            int a = (uint8_t)(xcim->pixels[xcim_addr] >> 24);
+
+            if (a == 255) {
+                pix[image_addr+0] = r;
+                pix[image_addr+1] = g;
+                pix[image_addr+2] = b;
+            } else if (a) {
+                /* pixel values from XFixesGetCursorImage come premultiplied by alpha */
+                pix[image_addr+0] = r + (pix[image_addr+0]*(255-a) + 255/2) / 255;
+                pix[image_addr+1] = g + (pix[image_addr+1]*(255-a) + 255/2) / 255;
+                pix[image_addr+2] = b + (pix[image_addr+2]*(255-a) + 255/2) / 255;
             }
         }
     }
@@ -288,7 +303,7 @@ paint_mouse_pointer(XImage *image, struct x11_grab *s)
 
 
 /**
- * Reads new data in the image structure.
+ * Read new data in the image structure.
  *
  * @param dpy X11 display to grab from
  * @param d
@@ -335,7 +350,7 @@ xget_zpixmap(Display *dpy, Drawable d, XImage *image, int x, int y)
 }
 
 /**
- * Grabs a frame from x11 (public device demuxer API).
+ * Grab a frame from x11 (public device demuxer API).
  *
  * @param s1 Context from avformat core
  * @param pkt Packet holding the brabbed frame
@@ -398,7 +413,7 @@ x11grab_read_packet(AVFormatContext *s1, AVPacket *pkt)
 }
 
 /**
- * Closes x11 frame grabber (public device demuxer API).
+ * Close x11 frame grabber (public device demuxer API).
  *
  * @param s1 Context from avformat core
  * @return 0 success, !0 failure