main/libxvmc: fix CVE-2013-1990,CVE-2013-1999

ref #1931
fixes #1976

(cherry picked from commit dfac4cbecc1c27d53504a0d9a80019146c9c9bfb)

1 files changed, 54 insertions, 0 deletions

diff --git a/main/libxvmc/0004-integer-overflow-in-_xvmc_create_.patch b/main/libxvmc/0004-integer-overflow-in-_xvmc_create_.patch
new file mode 100644
index 0000000000..70298e45a4
--- /dev/null
+++ b/main/libxvmc/0004-integer-overflow-in-_xvmc_create_.patch
@@ -0,0 +1,54 @@
+From 5fd871e5f878810f8f8837725d548e07e89577ab Mon Sep 17 00:00:00 2001
+From: Alan Coopersmith <alan.coopersmith@oracle.com>
+Date: Sat, 13 Apr 2013 00:50:02 -0700
+Subject: [PATCH 4/6] integer overflow in _xvmc_create_*()
+
+rep.length is a CARD32 and should be bounds checked before left-shifting
+by 2 bits to come up with the total size to allocate, though in these
+cases, no buffer overflow should occur here, since the XRead call is passed
+the same rep.length << 2 length argument, but the *priv_count returned to
+the caller could be interpreted or used to calculate a larger buffer size
+than was actually allocated, leading them to go out of bounds.
+
+Signed-off-by: Alan Coopersmith <alan.coopersmith@oracle.com>
+---
+ src/XvMC.c | 9 ++++++---
+ 1 file changed, 6 insertions(+), 3 deletions(-)
+
+diff --git a/src/XvMC.c b/src/XvMC.c
+index 8d602ec..d8bc59d 100644
+--- a/src/XvMC.c
++++ b/src/XvMC.c
+@@ -285,7 +285,8 @@ Status _xvmc_create_context (
+     context->flags = rep.flags_return;
+ 
+     if(rep.length) {
+-	*priv_data = Xmalloc(rep.length << 2);
++	if (rep.length < (INT_MAX >> 2))
++	    *priv_data = Xmalloc(rep.length << 2);
+ 	if(*priv_data) {
+             _XRead(dpy, (char*)(*priv_data), rep.length << 2);
+ 	    *priv_count = rep.length;
+@@ -366,7 +367,8 @@ Status _xvmc_create_surface (
+     }
+ 
+     if(rep.length) {
+-        *priv_data = Xmalloc(rep.length << 2);
++        if (rep.length < (INT_MAX >> 2))
++            *priv_data = Xmalloc(rep.length << 2);
+         if(*priv_data) {
+             _XRead(dpy, (char*)(*priv_data), rep.length << 2);
+             *priv_count = rep.length;
+@@ -456,7 +458,8 @@ Status _xvmc_create_subpicture (
+     subpicture->component_order[3] = rep.component_order[3];
+ 
+     if(rep.length) {
+-        *priv_data = Xmalloc(rep.length << 2);
++        if (rep.length < (INT_MAX >> 2))
++            *priv_data = Xmalloc(rep.length << 2);
+         if(*priv_data) {
+             _XRead(dpy, (char*)(*priv_data), rep.length << 2);
+             *priv_count = rep.length;
+-- 
+1.8.2.3
+