Fix errno values. Fix MALLOC_GLIBC_COMPAT handling in malloc/malloc.c,

which was reversed.  Provide more consistancy between implementations.
Handle it when people do stupid things like malloc(-1);

2 files changed, 24 insertions, 12 deletions

diff --git a/libc/stdlib/malloc/malloc.c b/libc/stdlib/malloc/malloc.c
index 5f88cf48e..98ac41cd7 100644
--- a/libc/stdlib/malloc/malloc.c
+++ b/libc/stdlib/malloc/malloc.c
@@ -13,6 +13,7 @@
 
 #include <stdlib.h>
 #include <unistd.h>
+#include <errno.h>
 #include <sys/mman.h>
 
 #include "malloc.h"
@@ -173,6 +174,7 @@ malloc_from_heap (size_t size, struct heap *heap)
 void *
 malloc (size_t size)
 {
+  void *mem;
 #ifdef MALLOC_DEBUGGING
   static int debugging_initialized = 0;
   if (! debugging_initialized)
@@ -185,12 +187,22 @@ malloc (size_t size)
 #endif
 
 #if defined(__MALLOC_GLIBC_COMPAT__)
-  if (size == 0)
-    return 0;
-#else
-  if (size == 0)
+  if (unlikely(size == 0))
       size++;
+#else
+  /* Some programs will call malloc (0).  Lets be strict and return NULL */
+  if (unlikely(size == 0))
+      goto oom;
 #endif
-
-  return malloc_from_heap (size, &__malloc_heap);
+  /* Check if they are doing something dumb like malloc(-1) */
+  if (unlikely(((unsigned long)size > (unsigned long)(MALLOC_HEADER_SIZE*-2))))
+      goto oom;
+
+  mem = malloc_from_heap (size, &__malloc_heap);
+  if (unlikely(!mem)) {
+oom:
+      __set_errno(ENOMEM);
+      return NULL;
+  }
+  return mem;
 }
diff --git a/libc/stdlib/malloc/realloc.c b/libc/stdlib/malloc/realloc.c
index d4e0d9cb4..9e6f880fe 100644
--- a/libc/stdlib/malloc/realloc.c
+++ b/libc/stdlib/malloc/realloc.c
@@ -13,6 +13,7 @@
 
 #include <stdlib.h>
 #include <string.h>
+#include <errno.h>
 
 #include "malloc.h"
 #include "heap.h"
@@ -25,13 +26,12 @@ realloc (void *mem, size_t new_size)
   char *base_mem;
 
   /* Check for special cases.  */
-  if (! new_size)
-    {
+  if (!mem)
+      return malloc(new_size);
+  if (!new_size) {
       free (mem);
-      return 0;
-    }
-  else if (! mem)
-    return malloc (new_size);
+      return (malloc(new_size));
+  }
 
   /* Normal realloc.  */