commit 5418e8007c248bf9668d22a8c1fa9528149b69f2
Author: Josef Gajdusek <atx@atx.name>
Date:   Mon Nov 14 11:39:01 2016 +0100

    Fix heap overflows in the various rectangle fill functions
    
    Altough rfbproto.c does check whether the overall FramebufferUpdate rectangle is
    too large, some of the individual encoding decoders do not, which allows a
    malicious server to overwrite parts of the heap.

diff --git a/libvncclient/rfbproto.c b/libvncclient/rfbproto.c
index 94b9bdb..9edfbad 100644
--- a/libvncclient/rfbproto.c
+++ b/libvncclient/rfbproto.c
@@ -147,6 +147,10 @@ void* rfbClientGetClientData(rfbClient* client, void* tag)
 
 /* messages */
 
+static boolean CheckRect(rfbClient* client, int x, int y, int w, int h) {
+  return x + w <= client->width && y + h <= client->height;
+}
+
 static void FillRectangle(rfbClient* client, int x, int y, int w, int h, uint32_t colour) {
   int i,j;
 
@@ -154,6 +158,11 @@ static void FillRectangle(rfbClient* client, int x, int y, int w, int h, uint32_
       return;
   }
 
+  if (!CheckRect(client, x, y, w, h)) {
+    rfbClientLog("Rect out of bounds: %dx%d at (%d, %d)\n", x, y, w, h);
+    return;
+  }
+
 #define FILL_RECT(BPP) \
     for(j=y*client->width;j<(y+h)*client->width;j+=client->width) \
       for(i=x;i<x+w;i++) \
@@ -175,6 +184,11 @@ static void CopyRectangle(rfbClient* client, uint8_t* buffer, int x, int y, int
       return;
   }
 
+  if (!CheckRect(client, x, y, w, h)) {
+    rfbClientLog("Rect out of bounds: %dx%d at (%d, %d)\n", x, y, w, h);
+    return;
+  }
+
 #define COPY_RECT(BPP) \
   { \
     int rs = w * BPP / 8, rs2 = client->width * BPP / 8; \
@@ -201,6 +215,16 @@ static void CopyRectangleFromRectangle(rfbClient* client, int src_x, int src_y,
       return;
   }
 
+  if (!CheckRect(client, src_x, src_y, w, h)) {
+    rfbClientLog("Source rect out of bounds: %dx%d at (%d, %d)\n", src_x, src_y, w, h);
+    return;
+  }
+
+  if (!CheckRect(client, dest_x, dest_y, w, h)) {
+    rfbClientLog("Dest rect out of bounds: %dx%d at (%d, %d)\n", dest_x, dest_y, w, h);
+    return;
+  }
+
 #define COPY_RECT_FROM_RECT(BPP) \
   { \
     uint##BPP##_t* _buffer=((uint##BPP##_t*)client->frameBuffer)+(src_y-dest_y)*client->width+src_x-dest_x; \