Add proper max malloc and errno handling to std malloc/realloc

Allow malloc libc test to run since max alloc is rejected properly

elkscmd/test/libc/malloc.c

@@ -15,18 +15,14 @@ TEST_CASE(malloc_malloc_free) {
         free(p);
     }
 
-    /* TODO:BUG: causes hang at 100% CPU */
-#if 0
     errno = 0;
-    p = malloc((size_t) - 1);
+    p = malloc((size_t)-1);
     if (p == NULL) {
         EXPECT_EQ(errno, ENOMEM);
     } else {
         EXPECT_EQ(errno, 0);
-        memset(p, 0xff, (size_t) - 1);
         free(p);
     }
-#endif
 
     /* strange sizes are fine; memory is writable and free-able */
     for (int i = 1; i < 1024; i += 123) {

libc/malloc/malloc.c

@@ -11,6 +11,7 @@
 
 #include <malloc.h>
 #include <unistd.h>
+#include <errno.h>
 
 #include "_malloc.h"
 
@@ -190,8 +191,16 @@ malloc(size_t size)
    register mem __wcnear *ptr = 0;
    register unsigned int sz;
 
+   errno = 0;
    if (size == 0)
-      return 0;			/* ANSI STD */
+      return 0;			/* ANSI STD, no error */
+
+   /* Minor oops here, sbrk has a signed argument */
+   if((int)size < 0 || size > (((unsigned)-1) >> 1) - sizeof(mem) * 3)
+   {
+      errno = ENOMEM;
+      return 0;
+   }
 
    sz = size + sizeof(mem) * 2 - 1;
    sz /= sizeof(mem);
@@ -281,6 +290,7 @@ malloc(size_t size)
 #ifndef MCHUNK
 	    ptr = __mini_malloc(size);
 #endif
+	    if( ptr == 0) errno = ENOMEM;
 #ifdef VERBOSE
 	    if( ptr == 0 )
 	       __noise("MALLOC FAIL", 0);