summaryrefslogtreecommitdiffstats
path: root/main/libxvmc/0001-Use-_XEatDataWords-to-avoid-overflow-of-rep.length-s.patch
blob: bba780364553f9c0e2ddb809abecb9e7ea475eba (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
From cf1a1dc1b9ca34a29d0471da9389f8eae70ddbd9 Mon Sep 17 00:00:00 2001
From: Alan Coopersmith <alan.coopersmith@oracle.com>
Date: Sat, 13 Apr 2013 00:47:57 -0700
Subject: [PATCH 1/6] Use _XEatDataWords to avoid overflow of rep.length
 shifting

rep.length is a CARD32, so rep.length << 2 could overflow in 32-bit builds

Signed-off-by: Alan Coopersmith <alan.coopersmith@oracle.com>
---
 configure.ac |  6 ++++++
 src/XvMC.c   | 24 ++++++++++++++++++------
 2 files changed, 24 insertions(+), 6 deletions(-)

diff --git a/configure.ac b/configure.ac
index b44f80d..f9d59a1 100644
--- a/configure.ac
+++ b/configure.ac
@@ -42,6 +42,12 @@ XORG_CHECK_MALLOC_ZERO
 # Obtain compiler/linker options for depedencies
 PKG_CHECK_MODULES(XVMC, x11 xext xv xextproto videoproto)
 
+# Check for _XEatDataWords function that may be patched into older Xlib release
+SAVE_LIBS="$LIBS"
+LIBS="$XVMC_LIBS"
+AC_CHECK_FUNCS([_XEatDataWords])
+LIBS="$SAVE_LIBS"
+
 # Checks for library functions.
 AC_CHECK_FUNCS([shmat])
 
diff --git a/src/XvMC.c b/src/XvMC.c
index 5a4cf0d..b3e97ec 100644
--- a/src/XvMC.c
+++ b/src/XvMC.c
@@ -16,6 +16,18 @@
 #include <sys/time.h>
 #include <X11/extensions/Xext.h>
 #include <X11/extensions/extutil.h>
+#include <limits.h>
+
+#ifndef HAVE__XEATDATAWORDS
+static inline void _XEatDataWords(Display *dpy, unsigned long n)
+{
+# ifndef LONG64
+    if (n >= (ULONG_MAX >> 2))
+        _XIOError(dpy);
+# endif
+    _XEatData (dpy, n << 2);
+}
+#endif
 
 static XExtensionInfo _xvmc_info_data;
 static XExtensionInfo *xvmc_info = &_xvmc_info_data;
@@ -134,7 +146,7 @@ XvMCSurfaceInfo * XvMCListSurfaceTypes(Display *dpy, XvPortID port, int *num)
 	       surface_info[i].flags = sinfo.flags;
 	    }
 	} else
-	   _XEatData(dpy, rep.length << 2);
+	   _XEatDataWords(dpy, rep.length);
     }
 
     UnlockDisplay (dpy);
@@ -207,7 +219,7 @@ XvImageFormatValues * XvMCListSubpictureTypes (
               ret[i].scanline_order = Info.scanline_order;
             }
         } else
-	   _XEatData(dpy, rep.length << 2);
+	   _XEatDataWords(dpy, rep.length);
     }
 
     UnlockDisplay (dpy);
@@ -278,7 +290,7 @@ Status _xvmc_create_context (
             _XRead(dpy, (char*)(*priv_data), rep.length << 2);
 	    *priv_count = rep.length;
 	} else
-	    _XEatData(dpy, rep.length << 2);
+	    _XEatDataWords(dpy, rep.length);
     }
 
     UnlockDisplay (dpy);
@@ -359,7 +371,7 @@ Status _xvmc_create_surface (
             _XRead(dpy, (char*)(*priv_data), rep.length << 2);
             *priv_count = rep.length;
         } else
-            _XEatData(dpy, rep.length << 2);
+            _XEatDataWords(dpy, rep.length);
     }
 
     UnlockDisplay (dpy);
@@ -449,7 +461,7 @@ Status _xvmc_create_subpicture (
             _XRead(dpy, (char*)(*priv_data), rep.length << 2);
             *priv_count = rep.length;
         } else
-            _XEatData(dpy, rep.length << 2);
+            _XEatDataWords(dpy, rep.length);
     }
 
     UnlockDisplay (dpy);
@@ -579,7 +591,7 @@ Status XvMCGetDRInfo(Display *dpy, XvPortID port,
 
 	} else {
 
-	    _XEatData(dpy, realSize);
+	    _XEatDataWords(dpy, rep.length);
 	    UnlockDisplay (dpy);
 	    SyncHandle ();
 	    return -1;
-- 
1.8.2.3