using explicit tests for allocation overflow whenever possible

lmem.c

@@ -1,5 +1,5 @@
 /*
-** $Id: lmem.c,v 1.91 2015/03/06 19:45:54 roberto Exp roberto $
+** $Id: lmem.c,v 1.92 2017/12/06 18:36:31 roberto Exp roberto $
 ** Interface to Memory Manager
 ** See Copyright Notice in lua.h
 */
@@ -53,24 +53,26 @@
 #define MINSIZEARRAY	4
 
 
-void *luaM_growaux_ (lua_State *L, void *block, int nelems, int *size,
+void *luaM_growaux_ (lua_State *L, void *block, int nelems, int *psize,
                      int size_elems, int limit, const char *what) {
   void *newblock;
-  int newsize;
-  if (nelems + 1 <= *size)  /* does one extra element still fit? */
+  int size = *psize;
+  if (nelems + 1 <= size)  /* does one extra element still fit? */
     return block;  /* nothing to be done */
-  if (*size >= limit/2) {  /* cannot double it? */
-    if (*size >= limit)  /* cannot grow even a little? */
+  if (size >= limit / 2) {  /* cannot double it? */
+    if (size >= limit)  /* cannot grow even a little? */
       luaG_runerror(L, "too many %s (limit is %d)", what, limit);
-    newsize = limit;  /* still have at least one free place */
+    size = limit;  /* still have at least one free place */
   }
   else {
-    newsize = (*size)*2;
-    if (newsize < MINSIZEARRAY)
-      newsize = MINSIZEARRAY;  /* minimum size */
+    size *= 2;
+    if (size < MINSIZEARRAY)
+      size = MINSIZEARRAY;  /* minimum size */
   }
-  newblock = luaM_reallocv(L, block, *size, newsize, size_elems);
-  *size = newsize;  /* update only when everything else is OK */
+  /* 'limit' ensures that multiplication will not overflow */
+  newblock = luaM_realloc(L, block, cast(size_t, *psize) * size_elems,
+                                    cast(size_t, size) * size_elems);
+  *psize = size;  /* update only when everything else is OK */
   return newblock;
 }
 
@@ -113,7 +115,7 @@ void luaM_free_ (lua_State *L, void *block, size_t osize) {
 /*
 ** generic allocation routine.
 */
-void *luaM_realloc_ (lua_State *L, void *block, size_t osize, size_t nsize) {
+void *luaM_realloc (lua_State *L, void *block, size_t osize, size_t nsize) {
   void *newblock;
   global_State *g = G(L);
   lua_assert((osize == 0) == (block == NULL));

lmem.h

@@ -1,5 +1,5 @@
 /*
-** $Id: lmem.h,v 1.43 2014/12/19 17:26:14 roberto Exp roberto $
+** $Id: lmem.h,v 1.44 2017/12/06 18:36:31 roberto Exp roberto $
 ** Interface to Memory Manager
 ** See Copyright Notice in lua.h
 */
@@ -31,36 +31,40 @@
 #define luaM_checksize(L,n,e)  \
 	(luaM_testsize(n,e) ? luaM_toobig(L) : cast_void(0))
 
+
 /*
-** This macro reallocs a vector 'b' from 'on' to 'n' elements, where
-** each element has size 'e'. In case of arithmetic overflow of the
-** product 'n'*'e', it raises an error (calling 'luaM_toobig').
+** Computes the minimum between 'n' and 'MAX_SIZET/sizeof(t)', so that
+** the result is not larger than 'n' and cannot overflow a 'size_t'
+** when multiplied by the size of type 't'. (Assumes that 'n' is an
+** 'int' or 'unsigned int' and that 'int' is not larger than 'size_t'.)
 */
-#define luaM_reallocv(L,b,on,n,e) \
-  (luaM_checksize(L,n,e), \
-   luaM_realloc_(L, (b), cast(size_t, on)*(e), cast(size_t, n)*(e)))
+#define luaM_limitN(n,t)  \
+  ((cast(size_t, n) > MAX_SIZET/sizeof(t)) ? (MAX_SIZET/sizeof(t)) : (n))
 
 /*
 ** Arrays of chars do not need any test
 */
 #define luaM_reallocvchar(L,b,on,n)  \
-    cast(char *, luaM_realloc_(L, (b), (on)*sizeof(char), (n)*sizeof(char)))
+    cast(char *, luaM_realloc(L, (b), (on)*sizeof(char), (n)*sizeof(char)))
 
 #define luaM_freemem(L, b, s)	luaM_free_(L, (b), (s))
 #define luaM_free(L, b)		luaM_free_(L, (b), sizeof(*(b)))
 #define luaM_freearray(L, b, n)   luaM_free_(L, (b), (n)*sizeof(*(b)))
 
-#define luaM_new(L,t)		cast(t *, luaM_malloc(L, sizeof(t), 0))
-#define luaM_newvector(L,n,t) \
-  (luaM_checksize(L,n,sizeof(t)), cast(t *, luaM_malloc(L, (n)*sizeof(t), 0)))
+#define luaM_new(L,t)		cast(t*, luaM_malloc(L, sizeof(t), 0))
+#define luaM_newvector(L,n,t)	cast(t*, luaM_malloc(L, (n)*sizeof(t), 0))
+#define luaM_newvectorchecked(L,n,t) \
+  (luaM_checksize(L,n,sizeof(t)), luaM_newvector(L,n,t))
 
 #define luaM_newobject(L,tag,s)	luaM_malloc(L, (s), tag)
 
 #define luaM_growvector(L,v,nelems,size,t,limit,e) \
-	((v)=cast(t *, luaM_growaux_(L,v,nelems,&(size),sizeof(t),limit,e)))
+	((v)=cast(t *, luaM_growaux_(L,v,nelems,&(size),sizeof(t), \
+                         luaM_limitN(limit,t),e)))
 
 #define luaM_reallocvector(L, v,oldn,n,t) \
-   ((v)=cast(t *, luaM_reallocv(L, v, oldn, n, sizeof(t))))
+   ((v)=cast(t *, luaM_realloc(L, v, cast(size_t, oldn) * sizeof(t), \
+                                     cast(size_t, n) * sizeof(t))))
 
 #define luaM_shrinkvector(L,v,size,fs,t) \
    ((v)=cast(t *, luaM_shrinkvector_(L, v, &(size), fs, sizeof(t))))
@@ -68,7 +72,7 @@
 LUAI_FUNC l_noret luaM_toobig (lua_State *L);
 
 /* not to be called directly */
-LUAI_FUNC void *luaM_realloc_ (lua_State *L, void *block, size_t oldsize,
+LUAI_FUNC void *luaM_realloc (lua_State *L, void *block, size_t oldsize,
                                                           size_t size);
 LUAI_FUNC void luaM_free_ (lua_State *L, void *block, size_t osize);
 LUAI_FUNC void *luaM_growaux_ (lua_State *L, void *block, int nelems,

lstring.c

@@ -1,5 +1,5 @@
 /*
-** $Id: lstring.c,v 2.57 2017/07/27 13:50:16 roberto Exp roberto $
+** $Id: lstring.c,v 2.58 2017/12/01 16:40:29 roberto Exp roberto $
 ** String table (keeps all strings handled by Lua)
 ** See Copyright Notice in lua.h
 */
@@ -31,6 +31,13 @@
 #endif
 
 
+
+/*
+** Maximum size for string table.
+*/
+#define MAXSTRTB	cast_int(luaM_limitN(MAX_INT, TString*))
+
+
 /*
 ** equality for long strings
 */
@@ -173,7 +180,8 @@ static TString *internshrstr (lua_State *L, const char *str, size_t l) {
       return ts;
     }
   }
-  if (g->strt.nuse >= g->strt.size && g->strt.size <= MAX_INT/2) {
+  if (g->strt.nuse >= g->strt.size &&
+      g->strt.size <= MAXSTRTB / 2) {
     luaS_resize(L, g->strt.size * 2);
     list = &g->strt.hash[lmod(h, g->strt.size)];  /* recompute with new size */
   }

ltable.c

@@ -1,5 +1,5 @@
 /*
-** $Id: ltable.c,v 2.126 2017/11/08 14:50:23 roberto Exp roberto $
+** $Id: ltable.c,v 2.127 2017/11/23 19:29:04 roberto Exp roberto $
 ** Lua tables (hash)
 ** See Copyright Notice in lua.h
 */
@@ -40,21 +40,34 @@
 
 
 /*
-** Maximum size of array part (MAXASIZE) is 2^MAXABITS. MAXABITS is
-** the largest integer such that MAXASIZE fits in an unsigned int.
+** MAXABITS is the largest integer such that MAXASIZE fits in an
+** unsigned int.
 */
 #define MAXABITS	cast_int(sizeof(int) * CHAR_BIT - 1)
-#define MAXASIZE	(1u << MAXABITS)
+
 
 /*
-** Maximum size of hash part is 2^MAXHBITS. MAXHBITS is the largest
-** integer such that 2^MAXHBITS fits in a signed int. (Note that the
-** maximum number of elements in a table, 2^MAXABITS + 2^MAXHBITS, still
-** fits comfortably in an unsigned int.)
+** MAXASIZE is the maximum size of the array part. It is the minimum
+** between 2^MAXABITS and the maximum size such that, measured in bytes,
+** it fits in a 'size_t'.
+*/
+#define MAXASIZE	luaM_limitN(1u << MAXABITS, TValue)
+
+/*
+** MAXHBITS is the largest integer such that 2^MAXHBITS fits in a
+** signed int.
 */
 #define MAXHBITS	(MAXABITS - 1)
 
 
+/*
+** MAXHSIZE is the maximum size of the hash part. It is the minimum
+** between 2^MAXHBITS and the maximum size such that, measured in bytes,
+** it fits in a 'size_t'.
+*/
+#define MAXHSIZE	luaM_limitN(1u << MAXHBITS, Node)
+
+
 #define hashpow2(t,n)		(gnode(t, lmod((n), sizenode(t))))
 
 #define hashstr(t,str)		hashpow2(t, (str)->hash)
@@ -353,6 +366,13 @@ static void setarrayvector (lua_State *L, Table *t, unsigned int size) {
 }
 
 
+/*
+** Creates an array for the hash part of a table with the given
+** size, or reuses the dummy node if size is zero.
+** The computation for size overflow is in two steps: the first
+** comparison ensures that the shift in the second one does not
+** overflow.
+*/
 static void setnodevector (lua_State *L, Table *t, unsigned int size) {
   if (size == 0) {  /* no elements to hash part? */
     t->node = cast(Node *, dummynode);  /* use common 'dummynode' */
@@ -362,7 +382,7 @@ static void setnodevector (lua_State *L, Table *t, unsigned int size) {
   else {
     int i;
     int lsize = luaO_ceillog2(size);
-    if (lsize > MAXHBITS)
+    if (lsize > MAXHBITS || (1u << lsize) > MAXHSIZE)
       luaG_runerror(L, "table overflow");
     size = twoto(lsize);
     t->node = luaM_newvector(L, size, Node);

lundump.c

@@ -1,5 +1,5 @@
 /*
-** $Id: lundump.c,v 2.47 2017/06/29 15:06:44 roberto Exp roberto $
+** $Id: lundump.c,v 2.48 2017/11/28 11:19:07 roberto Exp roberto $
 ** load precompiled Lua chunks
 ** See Copyright Notice in lua.h
 */
@@ -114,7 +114,7 @@ static TString *LoadString (LoadState *S) {
 
 static void LoadCode (LoadState *S, Proto *f) {
   int n = LoadInt(S);
-  f->code = luaM_newvector(S->L, n, Instruction);
+  f->code = luaM_newvectorchecked(S->L, n, Instruction);
   f->sizecode = n;
   LoadVector(S, f->code, n);
 }
@@ -126,7 +126,7 @@ static void LoadFunction(LoadState *S, Proto *f, TString *psource);
 static void LoadConstants (LoadState *S, Proto *f) {
   int i;
   int n = LoadInt(S);
-  f->k = luaM_newvector(S->L, n, TValue);
+  f->k = luaM_newvectorchecked(S->L, n, TValue);
   f->sizek = n;
   for (i = 0; i < n; i++)
     setnilvalue(&f->k[i]);
@@ -159,7 +159,7 @@ static void LoadConstants (LoadState *S, Proto *f) {
 static void LoadProtos (LoadState *S, Proto *f) {
   int i;
   int n = LoadInt(S);
-  f->p = luaM_newvector(S->L, n, Proto *);
+  f->p = luaM_newvectorchecked(S->L, n, Proto *);
   f->sizep = n;
   for (i = 0; i < n; i++)
     f->p[i] = NULL;
@@ -173,7 +173,7 @@ static void LoadProtos (LoadState *S, Proto *f) {
 static void LoadUpvalues (LoadState *S, Proto *f) {
   int i, n;
   n = LoadInt(S);
-  f->upvalues = luaM_newvector(S->L, n, Upvaldesc);
+  f->upvalues = luaM_newvectorchecked(S->L, n, Upvaldesc);
   f->sizeupvalues = n;
   for (i = 0; i < n; i++)
     f->upvalues[i].name = NULL;
@@ -187,18 +187,18 @@ static void LoadUpvalues (LoadState *S, Proto *f) {
 static void LoadDebug (LoadState *S, Proto *f) {
   int i, n;
   n = LoadInt(S);
-  f->lineinfo = luaM_newvector(S->L, n, ls_byte);
+  f->lineinfo = luaM_newvectorchecked(S->L, n, ls_byte);
   f->sizelineinfo = n;
   LoadVector(S, f->lineinfo, n);
   n = LoadInt(S);
-  f->abslineinfo = luaM_newvector(S->L, n, AbsLineInfo);
+  f->abslineinfo = luaM_newvectorchecked(S->L, n, AbsLineInfo);
   f->sizeabslineinfo = n;
   for (i = 0; i < n; i++) {
     f->abslineinfo[i].pc = LoadInt(S);
     f->abslineinfo[i].line = LoadInt(S);
   }
   n = LoadInt(S);
-  f->locvars = luaM_newvector(S->L, n, LocVar);
+  f->locvars = luaM_newvectorchecked(S->L, n, LocVar);
   f->sizelocvars = n;
   for (i = 0; i < n; i++)
     f->locvars[i].varname = NULL;