Changes in the control of C-stack overflow

* unification of the 'nny' and 'nCcalls' counters;
  * external C functions ('lua_CFunction') count more "slots" in
    the C stack (to allow for their possible use of buffers)
  * added a new test script specific for C-stack overflows. (Most
    of those tests were already present, but concentrating them
    in a single script easies the task of checking whether
    'LUAI_MAXCCALLS' is adequate in a system.)
This commit is contained in:
Roberto Ierusalimschy 2018-12-27 14:32:29 -02:00
parent da37ac9c78
commit ba7da13ec5
12 changed files with 170 additions and 74 deletions

4
lapi.c
View File

@ -956,7 +956,7 @@ LUA_API void lua_callk (lua_State *L, int nargs, int nresults,
api_check(L, L->status == LUA_OK, "cannot do calls on non-normal thread");
checkresults(L, nargs, nresults);
func = L->top - (nargs+1);
if (k != NULL && L->nny == 0) { /* need to prepare continuation? */
if (k != NULL && yieldable(L)) { /* need to prepare continuation? */
L->ci->u.c.k = k; /* save continuation */
L->ci->u.c.ctx = ctx; /* save context */
luaD_call(L, func, nresults); /* do the call */
@ -1004,7 +1004,7 @@ LUA_API int lua_pcallk (lua_State *L, int nargs, int nresults, int errfunc,
func = savestack(L, o);
}
c.func = L->top - (nargs+1); /* function to be called */
if (k == NULL || L->nny > 0) { /* no continuation or no yieldable? */
if (k == NULL || !yieldable(L)) { /* no continuation or no yieldable? */
c.nresults = nresults; /* do a 'conventional' protected call */
status = luaD_pcall(L, f_call, &c, savestack(L, c.func), func);
}

40
ldo.c
View File

@ -138,7 +138,7 @@ l_noret luaD_throw (lua_State *L, int errcode) {
int luaD_rawrunprotected (lua_State *L, Pfunc f, void *ud) {
unsigned short oldnCcalls = L->nCcalls - L->nci;
l_uint32 oldnCcalls = L->nCcalls - L->nci;
struct lua_longjmp lj;
lua_assert(L->nCcalls >= L->nci);
lj.status = LUA_OK;
@ -513,12 +513,17 @@ void luaD_call (lua_State *L, StkId func, int nresults) {
/*
** Similar to 'luaD_call', but does not allow yields during the call
** Similar to 'luaD_call', but does not allow yields during the call.
** If there is a stack overflow, freeing all CI structures will
** force the subsequent call to invoke 'luaE_extendCI', which then
** will raise any errors.
*/
void luaD_callnoyield (lua_State *L, StkId func, int nResults) {
L->nny++;
incXCcalls(L);
if (getCcalls(L) >= LUAI_MAXCCALLS) /* possible stack overflow? */
luaE_freeCI(L);
luaD_call(L, func, nResults);
L->nny--;
decXCcalls(L);
}
@ -530,7 +535,7 @@ static void finishCcall (lua_State *L, int status) {
CallInfo *ci = L->ci;
int n;
/* must have a continuation and must be able to call it */
lua_assert(ci->u.c.k != NULL && L->nny == 0);
lua_assert(ci->u.c.k != NULL && yieldable(L));
/* error status can only happen in a protected call */
lua_assert((ci->callstatus & CIST_YPCALL) || status == LUA_YIELD);
if (ci->callstatus & CIST_YPCALL) { /* was inside a pcall? */
@ -601,7 +606,6 @@ static int recover (lua_State *L, int status) {
luaD_seterrorobj(L, status, oldtop);
L->ci = ci;
L->allowhook = getoah(ci->callstatus); /* restore original 'allowhook' */
L->nny = 0; /* should be zero to be yieldable */
luaD_shrinkstack(L);
L->errfunc = ci->u.c.old_errfunc;
return 1; /* continue running the coroutine */
@ -622,13 +626,6 @@ static int resume_error (lua_State *L, const char *msg, int narg) {
}
/*
** "Cost" in the C stack for a coroutine invocation.
*/
#if !defined(LUAL_COROCSTK)
#define LUAL_COROCSTK 3
#endif
/*
** Do the work for 'lua_resume' in protected mode. Most of the work
** depends on the status of the coroutine: initial state, suspended
@ -664,7 +661,6 @@ static void resume (lua_State *L, void *ud) {
LUA_API int lua_resume (lua_State *L, lua_State *from, int nargs,
int *nresults) {
int status;
unsigned short oldnny = L->nny; /* save "number of non-yieldable" calls */
lua_lock(L);
if (L->status == LUA_OK) { /* may be starting a coroutine */
if (L->ci != &L->base_ci) /* not in base level? */
@ -675,11 +671,10 @@ LUA_API int lua_resume (lua_State *L, lua_State *from, int nargs,
if (from == NULL)
L->nCcalls = 1;
else /* correct 'nCcalls' for this thread */
L->nCcalls = from->nCcalls - from->nci + L->nci + LUAL_COROCSTK;
L->nCcalls = getCcalls(from) - from->nci + L->nci + CSTACKCF;
if (L->nCcalls >= LUAI_MAXCCALLS)
return resume_error(L, "C stack overflow", nargs);
luai_userstateresume(L, nargs);
L->nny = 0; /* allow yields */
api_checknelems(L, (L->status == LUA_OK) ? nargs + 1 : nargs);
status = luaD_rawrunprotected(L, resume, &nargs);
/* continue running after recoverable errors */
@ -698,14 +693,13 @@ LUA_API int lua_resume (lua_State *L, lua_State *from, int nargs,
}
*nresults = (status == LUA_YIELD) ? L->ci->u2.nyield
: cast_int(L->top - (L->ci->func + 1));
L->nny = oldnny; /* restore 'nny' */
lua_unlock(L);
return status;
}
LUA_API int lua_isyieldable (lua_State *L) {
return (L->nny == 0);
return yieldable(L);
}
@ -715,7 +709,7 @@ LUA_API int lua_yieldk (lua_State *L, int nresults, lua_KContext ctx,
luai_userstateyield(L, nresults);
lua_lock(L);
api_checknelems(L, nresults);
if (unlikely(L->nny > 0)) {
if (unlikely(!yieldable(L))) {
if (L != G(L)->mainthread)
luaG_runerror(L, "attempt to yield across a C-call boundary");
else
@ -741,7 +735,7 @@ LUA_API int lua_yieldk (lua_State *L, int nresults, lua_KContext ctx,
/*
** Call the C function 'func' in protected mode, restoring basic
** thread information ('allowhook', 'nny', etc.) and in particular
** thread information ('allowhook', etc.) and in particular
** its stack level in case of errors.
*/
int luaD_pcall (lua_State *L, Pfunc func, void *u,
@ -749,7 +743,6 @@ int luaD_pcall (lua_State *L, Pfunc func, void *u,
int status;
CallInfo *old_ci = L->ci;
lu_byte old_allowhooks = L->allowhook;
unsigned short old_nny = L->nny;
ptrdiff_t old_errfunc = L->errfunc;
L->errfunc = ef;
status = luaD_rawrunprotected(L, func, u);
@ -757,7 +750,6 @@ int luaD_pcall (lua_State *L, Pfunc func, void *u,
StkId oldtop = restorestack(L, old_top);
L->ci = old_ci;
L->allowhook = old_allowhooks;
L->nny = old_nny;
status = luaF_close(L, oldtop, status);
oldtop = restorestack(L, old_top); /* previous call may change stack */
luaD_seterrorobj(L, status, oldtop);
@ -811,7 +803,7 @@ int luaD_protectedparser (lua_State *L, ZIO *z, const char *name,
const char *mode) {
struct SParser p;
int status;
L->nny++; /* cannot yield during parsing */
incnny(L); /* cannot yield during parsing */
p.z = z; p.name = name; p.mode = mode;
p.dyd.actvar.arr = NULL; p.dyd.actvar.size = 0;
p.dyd.gt.arr = NULL; p.dyd.gt.size = 0;
@ -822,7 +814,7 @@ int luaD_protectedparser (lua_State *L, ZIO *z, const char *name,
luaM_freearray(L, p.dyd.actvar.arr, p.dyd.actvar.size);
luaM_freearray(L, p.dyd.gt.arr, p.dyd.gt.size);
luaM_freearray(L, p.dyd.label.arr, p.dyd.label.size);
L->nny--;
decnny(L);
return status;
}

View File

@ -184,11 +184,13 @@ typedef LUAI_UACINT l_uacInt;
** must be an unsigned with (at least) 4 bytes (see details in lopcodes.h)
*/
#if LUAI_BITSINT >= 32
typedef unsigned int Instruction;
typedef unsigned int l_uint32;
#else
typedef unsigned long Instruction;
typedef unsigned long l_uint32;
#endif
typedef l_uint32 Instruction;
/*

View File

@ -367,10 +367,12 @@ static void adjust_assign (LexState *ls, int nvars, int nexps, expdesc *e) {
}
#define enterlevel(ls) luaE_incCcalls((ls)->L)
/*
** Macros to limit the maximum recursion depth while parsing
*/
#define enterlevel(ls) luaE_enterCcall((ls)->L)
#define leavelevel(ls) ((ls)->L->nCcalls--)
#define leavelevel(ls) luaE_exitCcall((ls)->L)
/*

View File

@ -99,24 +99,42 @@ void luaE_setdebt (global_State *g, l_mem debt) {
/*
** Increment count of "C calls" and check for overflows. In case of
** a stack overflow, check appropriate error ("regular" overflow or
** overflow while handling stack overflow). If 'nCalls' is larger than
** LUAI_MAXCCALLS (which means it is handling a "regular" overflow) but
** smaller than 9/8 of LUAI_MAXCCALLS, does not report an error (to
** allow overflow handling to work)
** overflow while handling stack overflow).
** If 'nCcalls' is larger than LUAI_MAXCCALLS but smaller than
** LUAI_MAXCCALLS + CSTACKCF (plus 2 to avoid by-one errors), it means
** it has just entered the "overflow zone", so the function raises an
** overflow error.
** If 'nCcalls' is larger than LUAI_MAXCCALLS + CSTACKCF + 2
** (which means it is already handling an overflow) but smaller than
** 9/8 of LUAI_MAXCCALLS, does not report an error (to allow message
** handling to work).
** Otherwise, report a stack overflow while handling a stack overflow
** (probably caused by a repeating error in the message handling
** function).
*/
void luaE_incCcalls (lua_State *L) {
if (++L->nCcalls >= LUAI_MAXCCALLS) {
if (L->nCcalls == LUAI_MAXCCALLS)
luaG_runerror(L, "C stack overflow");
else if (L->nCcalls >= (LUAI_MAXCCALLS + (LUAI_MAXCCALLS>>3)))
luaD_throw(L, LUA_ERRERR); /* error while handing stack error */
void luaE_enterCcall (lua_State *L) {
int ncalls = getCcalls(L);
L->nCcalls++;
if (ncalls >= LUAI_MAXCCALLS) { /* possible overflow? */
luaE_freeCI(L); /* release unused CIs */
ncalls = getCcalls(L); /* update call count */
if (ncalls >= LUAI_MAXCCALLS) { /* still overflow? */
if (ncalls <= LUAI_MAXCCALLS + CSTACKCF + 2) {
/* no error before increments; raise the error now */
L->nCcalls += (CSTACKCF + 4); /* avoid raising it again */
luaG_runerror(L, "C stack overflow");
}
else if (ncalls >= (LUAI_MAXCCALLS + (LUAI_MAXCCALLS >> 3)))
luaD_throw(L, LUA_ERRERR); /* error while handling stack error */
}
}
}
CallInfo *luaE_extendCI (lua_State *L) {
CallInfo *ci;
luaE_incCcalls(L);
lua_assert(L->ci->next == NULL);
luaE_enterCcall(L);
ci = luaM_new(L, CallInfo);
lua_assert(L->ci->next == NULL);
L->ci->next = ci;
@ -135,13 +153,13 @@ void luaE_freeCI (lua_State *L) {
CallInfo *ci = L->ci;
CallInfo *next = ci->next;
ci->next = NULL;
L->nCcalls -= L->nci; /* to subtract removed elements from 'nCcalls' */
L->nCcalls -= L->nci; /* subtract removed elements from 'nCcalls' */
while ((ci = next) != NULL) {
next = ci->next;
luaM_free(L, ci);
L->nci--;
}
L->nCcalls += L->nci; /* to subtract removed elements from 'nCcalls' */
L->nCcalls += L->nci; /* adjust result */
}
@ -151,7 +169,7 @@ void luaE_freeCI (lua_State *L) {
void luaE_shrinkCI (lua_State *L) {
CallInfo *ci = L->ci;
CallInfo *next2; /* next's next */
L->nCcalls -= L->nci; /* to subtract removed elements from 'nCcalls' */
L->nCcalls -= L->nci; /* subtract removed elements from 'nCcalls' */
/* while there are two nexts */
while (ci->next != NULL && (next2 = ci->next->next) != NULL) {
luaM_free(L, ci->next); /* free next */
@ -160,7 +178,7 @@ void luaE_shrinkCI (lua_State *L) {
next2->previous = ci;
ci = next2; /* keep next's next */
}
L->nCcalls += L->nci; /* to subtract removed elements from 'nCcalls' */
L->nCcalls += L->nci; /* adjust result */
}
@ -250,7 +268,6 @@ static void preinit_thread (lua_State *L, global_State *g) {
L->allowhook = 1;
resethookcount(L);
L->openupval = NULL;
L->nny = 1;
L->status = LUA_OK;
L->errfunc = 0;
}

View File

@ -15,7 +15,6 @@
/*
** Some notes about garbage-collected objects: All objects in Lua must
** be kept somehow accessible until being freed, so all objects always
** belong to one (and only one) of these lists, using field 'next' of
@ -43,26 +42,58 @@
** 'weak': tables with weak values to be cleared;
** 'ephemeron': ephemeron tables with white->white entries;
** 'allweak': tables with weak keys and/or weak values to be cleared.
*/
/*
/*
** About 'nCcalls': each thread in Lua (a lua_State) keeps a count of
** how many "C calls" it has in the C stack, to avoid C-stack overflow.
** how many "C calls" it can do in the C stack, to avoid C-stack overflow.
** This count is very rough approximation; it considers only recursive
** functions inside the interpreter, as non-recursive calls can be
** considered using a fixed (although unknown) amount of stack space.
**
** The count itself has two parts: the lower part is the count itself;
** the higher part counts the number of non-yieldable calls in the stack.
**
** Because calls to external C functions can use of unkown amount
** of space (e.g., functions using an auxiliary buffer), calls
** to these functions add more than one to the count.
**
** The proper count also includes the number of CallInfo structures
** allocated by Lua, as a kind of "potential" calls. So, when Lua
** calls a function (and "consumes" one CallInfo), it needs neither to
** increment nor to check 'nCcalls', as its use of C stack is already
** accounted for.
*/
/* number of "C stack slots" used by an external C function */
#define CSTACKCF 10
/* true if this thread does not have non-yieldable calls in the stack */
#define yieldable(L) (((L)->nCcalls & 0xffff0000) == 0)
/* real number of C calls */
#define getCcalls(L) ((L)->nCcalls & 0xffff)
/* Increment the number of non-yieldable calls */
#define incnny(L) ((L)->nCcalls += 0x10000)
/* Decrement the number of non-yieldable calls */
#define decnny(L) ((L)->nCcalls -= 0x10000)
/* Increment the number of non-yieldable calls and nCcalls */
#define incXCcalls(L) ((L)->nCcalls += 0x10000 + CSTACKCF)
/* Decrement the number of non-yieldable calls and nCcalls */
#define decXCcalls(L) ((L)->nCcalls -= 0x10000 + CSTACKCF)
struct lua_longjmp; /* defined in ldo.c */
@ -208,8 +239,9 @@ typedef struct global_State {
*/
struct lua_State {
CommonHeader;
unsigned short nci; /* number of items in 'ci' list */
lu_byte status;
lu_byte allowhook;
unsigned short nci; /* number of items in 'ci' list */
StkId top; /* first free slot in the stack */
global_State *l_G;
CallInfo *ci; /* call info for current function */
@ -223,13 +255,11 @@ struct lua_State {
CallInfo base_ci; /* CallInfo for first level (C calling Lua) */
volatile lua_Hook hook;
ptrdiff_t errfunc; /* current error handling function (stack index) */
l_uint32 nCcalls; /* number of allowed nested C calls - 'nci' */
int stacksize;
int basehookcount;
int hookcount;
unsigned short nny; /* number of non-yieldable calls in stack */
unsigned short nCcalls; /* number of nested C calls + 'nny' */
l_signalT hookmask;
lu_byte allowhook;
};
@ -283,8 +313,10 @@ LUAI_FUNC void luaE_freethread (lua_State *L, lua_State *L1);
LUAI_FUNC CallInfo *luaE_extendCI (lua_State *L);
LUAI_FUNC void luaE_freeCI (lua_State *L);
LUAI_FUNC void luaE_shrinkCI (lua_State *L);
LUAI_FUNC void luaE_incCcalls (lua_State *L);
LUAI_FUNC void luaE_enterCcall (lua_State *L);
#define luaE_exitCcall(L) ((L)->nCcalls--)
#endif

View File

@ -31,7 +31,7 @@
/* compiled with -O0, Lua uses a lot of C stack space... */
#undef LUAI_MAXCCALLS
#define LUAI_MAXCCALLS 200
#define LUAI_MAXCCALLS 400
/* to avoid warnings, and to make sure value is really unused */
#define UNUSED(x) (x=0, (void)(x))

View File

@ -695,14 +695,14 @@
/*
@@ LUAL_BUFFERSIZE is the buffer size used by the lauxlib buffer system.
** CHANGE it if it uses too much C-stack space. (For long double,
** 'string.format("%.99f", -1e4932)' needs 5034 bytes, so a
** 'string.format("%.99f", -1e4932)' needs 5052 bytes, so a
** smaller buffer would force a memory allocation for each call to
** 'string.format'.)
*/
#if LUA_FLOAT_TYPE == LUA_FLOAT_LONGDOUBLE
#define LUAL_BUFFERSIZE 8192
#else
#define LUAL_BUFFERSIZE ((int)(0x80 * sizeof(void*) * sizeof(lua_Integer)))
#define LUAL_BUFFERSIZE ((int)(16 * sizeof(void*) * sizeof(lua_Number)))
#endif
/*

View File

@ -172,6 +172,7 @@ if not _G._soft then
assert(f() == 'b')
assert(f() == 'a')
end
dofile('cstack.lua')
dofile('nextvar.lua')
dofile('pm.lua')
dofile('utf8.lua')

View File

@ -107,7 +107,7 @@ function filter (p, g)
end)
end
local x = gen(100)
local x = gen(80)
local a = {}
while 1 do
local n = x()
@ -116,7 +116,7 @@ while 1 do
x = filter(n, x)
end
assert(#a == 25 and a[#a] == 97)
assert(#a == 22 and a[#a] == 79)
x, a = nil

62
testes/cstack.lua Normal file
View File

@ -0,0 +1,62 @@
-- $Id: testes/cstack.lua $
-- See Copyright Notice in file all.lua
print"testing C-stack overflow detection"
-- Segmentation faults in these tests probably result from a C-stack
-- overflow. To avoid these errors, recompile Lua with a smaller
-- value for the constant 'LUAI_MAXCCALLS' or else ensure a larger
-- stack for the program.
local function checkerror (msg, f, ...)
local s, err = pcall(f, ...)
assert(not s and string.find(err, msg))
end
do -- simple recursion
local count = 0
local function foo ()
count = count + 1
foo()
end
checkerror("stack overflow", foo)
print(" maximum recursion: " .. count)
end
-- bug since 2.5 (C-stack overflow in recursion inside pattern matching)
do
local function f (size)
local s = string.rep("a", size)
local p = string.rep(".?", size)
return string.match(s, p)
end
local m = f(80)
assert(#m == 80)
checkerror("too complex", f, 200000)
end
-- testing stack-overflow in recursive 'gsub'
do
local count = 0
local function foo ()
count = count + 1
string.gsub("a", ".", foo)
end
checkerror("stack overflow", foo)
print(" maximum 'gsub' nest (calls): " .. count)
-- can be done with metamethods, too
count = 0
local t = setmetatable({}, {__index = foo})
foo = function ()
count = count + 1
string.gsub("a", ".", t)
end
checkerror("stack overflow", foo)
print(" maximum 'gsub' nest (metamethods): " .. count)
end
print'OK'

View File

@ -237,18 +237,6 @@ checkerror("invalid capture index %%0", string.gsub, "alo", "(%0)", "a")
checkerror("invalid capture index %%1", string.gsub, "alo", "(%1)", "a")
checkerror("invalid use of '%%'", string.gsub, "alo", ".", "%x")
-- bug since 2.5 (C-stack overflow)
do
local function f (size)
local s = string.rep("a", size)
local p = string.rep(".?", size)
return pcall(string.match, s, p)
end
local r, m = f(80)
assert(r and #m == 80)
r, m = f(200000)
assert(not r and string.find(m, "too complex"))
end
if not _soft then
print("big strings")