From 59c88f846d1dcd901a4420651aedf27816618923 Mon Sep 17 00:00:00 2001 From: Roberto Ierusalimschy Date: Wed, 24 Feb 2021 11:14:44 -0300 Subject: [PATCH] Broadening the use of branch hints More uses of macros 'likely'/'unlikely' (renamed to 'l_likely'/'l_unlikely'), both in range (extended to the libraries) and in scope (extended to hooks, stack growth). --- lauxlib.c | 22 +++++++++++----------- lauxlib.h | 8 ++++++-- lbaselib.c | 12 ++++++------ lcorolib.c | 10 +++++----- ldblib.c | 6 +++--- ldo.c | 24 ++++++++++++------------ ldo.h | 2 +- lgc.c | 2 +- liolib.c | 17 +++++++++-------- llimits.h | 16 ---------------- lmathlib.c | 5 +++-- lmem.c | 8 ++++---- loadlib.c | 17 ++++++++++------- loslib.c | 8 ++++---- lparser.c | 8 ++++---- lstate.c | 2 +- lstring.c | 8 ++++---- lstrlib.c | 41 ++++++++++++++++++++++------------------- ltable.c | 8 ++++---- ltablib.c | 9 +++++---- ltm.c | 5 +++-- luaconf.h | 20 ++++++++++++++++++++ lvm.c | 39 ++++++++++++++++++++------------------- lvm.h | 6 ++++-- 24 files changed, 162 insertions(+), 141 deletions(-) diff --git a/lauxlib.c b/lauxlib.c index e8fc486e..2610f90e 100644 --- a/lauxlib.c +++ b/lauxlib.c @@ -190,7 +190,7 @@ LUALIB_API int luaL_argerror (lua_State *L, int arg, const char *extramsg) { } -int luaL_typeerror (lua_State *L, int arg, const char *tname) { +LUALIB_API int luaL_typeerror (lua_State *L, int arg, const char *tname) { const char *msg; const char *typearg; /* name for the type of the actual argument */ if (luaL_getmetafield(L, arg, "__name") == LUA_TSTRING) @@ -378,7 +378,7 @@ LUALIB_API int luaL_checkoption (lua_State *L, int arg, const char *def, ** but without 'msg'.) */ LUALIB_API void luaL_checkstack (lua_State *L, int space, const char *msg) { - if (!lua_checkstack(L, space)) { + if (l_unlikely(!lua_checkstack(L, space))) { if (msg) luaL_error(L, "stack overflow (%s)", msg); else @@ -388,20 +388,20 @@ LUALIB_API void luaL_checkstack (lua_State *L, int space, const char *msg) { LUALIB_API void luaL_checktype (lua_State *L, int arg, int t) { - if (lua_type(L, arg) != t) + if (l_unlikely(lua_type(L, arg) != t)) tag_error(L, arg, t); } LUALIB_API void luaL_checkany (lua_State *L, int arg) { - if (lua_type(L, arg) == LUA_TNONE) + if (l_unlikely(lua_type(L, arg) == LUA_TNONE)) luaL_argerror(L, arg, "value expected"); } LUALIB_API const char *luaL_checklstring (lua_State *L, int arg, size_t *len) { const char *s = lua_tolstring(L, arg, len); - if (!s) tag_error(L, arg, LUA_TSTRING); + if (l_unlikely(!s)) tag_error(L, arg, LUA_TSTRING); return s; } @@ -420,7 +420,7 @@ LUALIB_API const char *luaL_optlstring (lua_State *L, int arg, LUALIB_API lua_Number luaL_checknumber (lua_State *L, int arg) { int isnum; lua_Number d = lua_tonumberx(L, arg, &isnum); - if (!isnum) + if (l_unlikely(!isnum)) tag_error(L, arg, LUA_TNUMBER); return d; } @@ -442,7 +442,7 @@ static void interror (lua_State *L, int arg) { LUALIB_API lua_Integer luaL_checkinteger (lua_State *L, int arg) { int isnum; lua_Integer d = lua_tointegerx(L, arg, &isnum); - if (!isnum) { + if (l_unlikely(!isnum)) { interror(L, arg); } return d; @@ -475,7 +475,7 @@ static void *resizebox (lua_State *L, int idx, size_t newsize) { lua_Alloc allocf = lua_getallocf(L, &ud); UBox *box = (UBox *)lua_touserdata(L, idx); void *temp = allocf(ud, box->box, box->bsize, newsize); - if (temp == NULL && newsize > 0) { /* allocation error? */ + if (l_unlikely(temp == NULL && newsize > 0)) { /* allocation error? */ lua_pushliteral(L, "not enough memory"); lua_error(L); /* raise a memory error */ } @@ -521,7 +521,7 @@ static void newbox (lua_State *L) { */ static size_t newbuffsize (luaL_Buffer *B, size_t sz) { size_t newsize = B->size * 2; /* double buffer size */ - if (MAX_SIZET - sz < B->n) /* overflow in (B->n + sz)? */ + if (l_unlikely(MAX_SIZET - sz < B->n)) /* overflow in (B->n + sz)? */ return luaL_error(B->L, "buffer too large"); if (newsize < B->n + sz) /* double is not big enough? */ newsize = B->n + sz; @@ -861,7 +861,7 @@ LUALIB_API lua_Integer luaL_len (lua_State *L, int idx) { int isnum; lua_len(L, idx); l = lua_tointegerx(L, -1, &isnum); - if (!isnum) + if (l_unlikely(!isnum)) luaL_error(L, "object length is not an integer"); lua_pop(L, 1); /* remove object */ return l; @@ -1074,7 +1074,7 @@ static void warnfon (void *ud, const char *message, int tocont) { LUALIB_API lua_State *luaL_newstate (void) { lua_State *L = lua_newstate(l_alloc, NULL); - if (L) { + if (l_likely(L)) { lua_atpanic(L, &panic); lua_setwarnf(L, warnfoff, L); /* default is warnings off */ } diff --git a/lauxlib.h b/lauxlib.h index 65714911..9058e262 100644 --- a/lauxlib.h +++ b/lauxlib.h @@ -122,6 +122,10 @@ LUALIB_API void (luaL_requiref) (lua_State *L, const char *modname, ** =============================================================== */ +#if !defined(l_likely) +#define l_likely(x) x +#endif + #define luaL_newlibtable(L,l) \ lua_createtable(L, 0, sizeof(l)/sizeof((l)[0]) - 1) @@ -130,10 +134,10 @@ LUALIB_API void (luaL_requiref) (lua_State *L, const char *modname, (luaL_checkversion(L), luaL_newlibtable(L,l), luaL_setfuncs(L,l,0)) #define luaL_argcheck(L, cond,arg,extramsg) \ - ((void)((cond) || luaL_argerror(L, (arg), (extramsg)))) + ((void)(l_likely(cond) || luaL_argerror(L, (arg), (extramsg)))) #define luaL_argexpected(L,cond,arg,tname) \ - ((void)((cond) || luaL_typeerror(L, (arg), (tname)))) + ((void)(l_likely(cond) || luaL_typeerror(L, (arg), (tname)))) #define luaL_checkstring(L,n) (luaL_checklstring(L, (n), NULL)) #define luaL_optstring(L,n,d) (luaL_optlstring(L, (n), (d), NULL)) diff --git a/lbaselib.c b/lbaselib.c index 60786b3d..83ad306d 100644 --- a/lbaselib.c +++ b/lbaselib.c @@ -138,7 +138,7 @@ static int luaB_setmetatable (lua_State *L) { int t = lua_type(L, 2); luaL_checktype(L, 1, LUA_TTABLE); luaL_argexpected(L, t == LUA_TNIL || t == LUA_TTABLE, 2, "nil or table"); - if (luaL_getmetafield(L, 1, "__metatable") != LUA_TNIL) + if (l_unlikely(luaL_getmetafield(L, 1, "__metatable") != LUA_TNIL)) return luaL_error(L, "cannot change a protected metatable"); lua_settop(L, 2); lua_setmetatable(L, 1); @@ -300,7 +300,7 @@ static int luaB_ipairs (lua_State *L) { static int load_aux (lua_State *L, int status, int envidx) { - if (status == LUA_OK) { + if (l_likely(status == LUA_OK)) { if (envidx != 0) { /* 'env' parameter? */ lua_pushvalue(L, envidx); /* environment for loaded function */ if (!lua_setupvalue(L, -2, 1)) /* set it as 1st upvalue */ @@ -356,7 +356,7 @@ static const char *generic_reader (lua_State *L, void *ud, size_t *size) { *size = 0; return NULL; } - else if (!lua_isstring(L, -1)) + else if (l_unlikely(!lua_isstring(L, -1))) luaL_error(L, "reader function must return a string"); lua_replace(L, RESERVEDSLOT); /* save string in reserved slot */ return lua_tolstring(L, RESERVEDSLOT, size); @@ -394,7 +394,7 @@ static int dofilecont (lua_State *L, int d1, lua_KContext d2) { static int luaB_dofile (lua_State *L) { const char *fname = luaL_optstring(L, 1, NULL); lua_settop(L, 1); - if (luaL_loadfile(L, fname) != LUA_OK) + if (l_unlikely(luaL_loadfile(L, fname) != LUA_OK)) return lua_error(L); lua_callk(L, 0, LUA_MULTRET, 0, dofilecont); return dofilecont(L, 0, 0); @@ -402,7 +402,7 @@ static int luaB_dofile (lua_State *L) { static int luaB_assert (lua_State *L) { - if (lua_toboolean(L, 1)) /* condition is true? */ + if (l_likely(lua_toboolean(L, 1))) /* condition is true? */ return lua_gettop(L); /* return all arguments */ else { /* error */ luaL_checkany(L, 1); /* there must be a condition */ @@ -438,7 +438,7 @@ static int luaB_select (lua_State *L) { ** ignored). */ static int finishpcall (lua_State *L, int status, lua_KContext extra) { - if (status != LUA_OK && status != LUA_YIELD) { /* error? */ + if (l_unlikely(status != LUA_OK && status != LUA_YIELD)) { /* error? */ lua_pushboolean(L, 0); /* first result (false) */ lua_pushvalue(L, -2); /* error message */ return 2; /* return false, msg */ diff --git a/lcorolib.c b/lcorolib.c index ed7c58b2..fedbebec 100644 --- a/lcorolib.c +++ b/lcorolib.c @@ -31,14 +31,14 @@ static lua_State *getco (lua_State *L) { */ static int auxresume (lua_State *L, lua_State *co, int narg) { int status, nres; - if (!lua_checkstack(co, narg)) { + if (l_unlikely(!lua_checkstack(co, narg))) { lua_pushliteral(L, "too many arguments to resume"); return -1; /* error flag */ } lua_xmove(L, co, narg); status = lua_resume(co, L, narg, &nres); - if (status == LUA_OK || status == LUA_YIELD) { - if (!lua_checkstack(L, nres + 1)) { + if (l_likely(status == LUA_OK || status == LUA_YIELD)) { + if (l_unlikely(!lua_checkstack(L, nres + 1))) { lua_pop(co, nres); /* remove results anyway */ lua_pushliteral(L, "too many results to resume"); return -1; /* error flag */ @@ -57,7 +57,7 @@ static int luaB_coresume (lua_State *L) { lua_State *co = getco(L); int r; r = auxresume(L, co, lua_gettop(L) - 1); - if (r < 0) { + if (l_unlikely(r < 0)) { lua_pushboolean(L, 0); lua_insert(L, -2); return 2; /* return false + error message */ @@ -73,7 +73,7 @@ static int luaB_coresume (lua_State *L) { static int luaB_auxwrap (lua_State *L) { lua_State *co = lua_tothread(L, lua_upvalueindex(1)); int r = auxresume(L, co, lua_gettop(L)); - if (r < 0) { /* error? */ + if (l_unlikely(r < 0)) { /* error? */ int stat = lua_status(co); if (stat != LUA_OK && stat != LUA_YIELD) { /* error in the coroutine? */ stat = lua_resetthread(co); /* close its tbc variables */ diff --git a/ldblib.c b/ldblib.c index de6e38b3..6dcbaa98 100644 --- a/ldblib.c +++ b/ldblib.c @@ -33,7 +33,7 @@ static const char *const HOOKKEY = "_HOOKKEY"; ** checked. */ static void checkstack (lua_State *L, lua_State *L1, int n) { - if (L != L1 && !lua_checkstack(L1, n)) + if (l_unlikely(L != L1 && !lua_checkstack(L1, n))) luaL_error(L, "stack overflow"); } @@ -213,7 +213,7 @@ static int db_getlocal (lua_State *L) { lua_Debug ar; const char *name; int level = (int)luaL_checkinteger(L, arg + 1); - if (!lua_getstack(L1, level, &ar)) /* out of range? */ + if (l_unlikely(!lua_getstack(L1, level, &ar))) /* out of range? */ return luaL_argerror(L, arg+1, "level out of range"); checkstack(L, L1, 1); name = lua_getlocal(L1, &ar, nvar); @@ -238,7 +238,7 @@ static int db_setlocal (lua_State *L) { lua_Debug ar; int level = (int)luaL_checkinteger(L, arg + 1); int nvar = (int)luaL_checkinteger(L, arg + 2); - if (!lua_getstack(L1, level, &ar)) /* out of range? */ + if (l_unlikely(!lua_getstack(L1, level, &ar))) /* out of range? */ return luaL_argerror(L, arg+1, "level out of range"); luaL_checkany(L, arg+3); lua_settop(L, arg+3); diff --git a/ldo.c b/ldo.c index 5587b602..7c9ce06e 100644 --- a/ldo.c +++ b/ldo.c @@ -184,7 +184,7 @@ int luaD_reallocstack (lua_State *L, int newsize, int raiseerror) { StkId newstack = luaM_reallocvector(L, L->stack, lim + EXTRA_STACK, newsize + EXTRA_STACK, StackValue); lua_assert(newsize <= LUAI_MAXSTACK || newsize == ERRORSTACKSIZE); - if (unlikely(newstack == NULL)) { /* reallocation failed? */ + if (l_unlikely(newstack == NULL)) { /* reallocation failed? */ if (raiseerror) luaM_error(L); else return 0; /* do not raise an error */ @@ -204,7 +204,7 @@ int luaD_reallocstack (lua_State *L, int newsize, int raiseerror) { */ int luaD_growstack (lua_State *L, int n, int raiseerror) { int size = stacksize(L); - if (unlikely(size > LUAI_MAXSTACK)) { + if (l_unlikely(size > LUAI_MAXSTACK)) { /* if stack is larger than maximum, thread is already using the extra space reserved for errors, that is, thread is handling a stack error; cannot grow further than that. */ @@ -220,7 +220,7 @@ int luaD_growstack (lua_State *L, int n, int raiseerror) { newsize = LUAI_MAXSTACK; if (newsize < needed) /* but must respect what was asked for */ newsize = needed; - if (likely(newsize <= LUAI_MAXSTACK)) + if (l_likely(newsize <= LUAI_MAXSTACK)) return luaD_reallocstack(L, newsize, raiseerror); else { /* stack overflow */ /* add extra size to be able to handle the error message */ @@ -376,7 +376,7 @@ static void rethook (lua_State *L, CallInfo *ci, int nres) { void luaD_tryfuncTM (lua_State *L, StkId func) { const TValue *tm = luaT_gettmbyobj(L, s2v(func), TM_CALL); StkId p; - if (unlikely(ttisnil(tm))) + if (l_unlikely(ttisnil(tm))) luaG_callerror(L, s2v(func)); /* nothing to call */ for (p = L->top; p > func; p--) /* open space for metamethod */ setobjs2s(L, p, p-1); @@ -444,7 +444,7 @@ static void moveresults (lua_State *L, StkId res, int nres, int wanted) { */ void luaD_poscall (lua_State *L, CallInfo *ci, int nres) { int wanted = ci->nresults; - if (L->hookmask && !hastocloseCfunc(wanted)) + if (l_unlikely(L->hookmask && !hastocloseCfunc(wanted))) rethook(L, ci, nres); /* move results to proper place */ moveresults(L, ci->func, nres, wanted); @@ -510,7 +510,7 @@ CallInfo *luaD_precall (lua_State *L, StkId func, int nresults) { ci->top = L->top + LUA_MINSTACK; ci->func = func; lua_assert(ci->top <= L->stack_last); - if (L->hookmask & LUA_MASKCALL) { + if (l_unlikely(L->hookmask & LUA_MASKCALL)) { int narg = cast_int(L->top - func) - 1; luaD_hook(L, LUA_HOOKCALL, -1, 1, narg); } @@ -556,7 +556,7 @@ CallInfo *luaD_precall (lua_State *L, StkId func, int nresults) { static void ccall (lua_State *L, StkId func, int nResults, int inc) { CallInfo *ci; L->nCcalls += inc; - if (unlikely(getCcalls(L) >= LUAI_MAXCCALLS)) + if (l_unlikely(getCcalls(L) >= LUAI_MAXCCALLS)) luaE_checkcstack(L); if ((ci = luaD_precall(L, func, nResults)) != NULL) { /* Lua function? */ ci->callstatus = CIST_FRESH; /* mark that it is a "fresh" execute */ @@ -600,7 +600,7 @@ void luaD_callnoyield (lua_State *L, StkId func, int nResults) { */ static int finishpcallk (lua_State *L, CallInfo *ci) { int status = getcistrecst(ci); /* get original status */ - if (status == LUA_OK) /* no error? */ + if (l_likely(status == LUA_OK)) /* no error? */ status = LUA_YIELD; /* was interrupted by an yield */ else { /* error */ StkId func = restorestack(L, ci->u2.funcidx); @@ -774,7 +774,7 @@ LUA_API int lua_resume (lua_State *L, lua_State *from, int nargs, status = luaD_rawrunprotected(L, resume, &nargs); /* continue running after recoverable errors */ status = precover(L, status); - if (likely(!errorstatus(status))) + if (l_likely(!errorstatus(status))) lua_assert(status == L->status); /* normal end or yield */ else { /* unrecoverable error */ L->status = cast_byte(status); /* mark thread as 'dead' */ @@ -800,7 +800,7 @@ LUA_API int lua_yieldk (lua_State *L, int nresults, lua_KContext ctx, lua_lock(L); ci = L->ci; api_checknelems(L, nresults); - if (unlikely(!yieldable(L))) { + if (l_unlikely(!yieldable(L))) { if (L != G(L)->mainthread) luaG_runerror(L, "attempt to yield across a C-call boundary"); else @@ -853,7 +853,7 @@ int luaD_closeprotected (lua_State *L, ptrdiff_t level, int status) { struct CloseP pcl; pcl.level = restorestack(L, level); pcl.status = status; status = luaD_rawrunprotected(L, &closepaux, &pcl); - if (likely(status == LUA_OK)) /* no more errors? */ + if (l_likely(status == LUA_OK)) /* no more errors? */ return pcl.status; else { /* an error occurred; restore saved state and repeat */ L->ci = old_ci; @@ -876,7 +876,7 @@ int luaD_pcall (lua_State *L, Pfunc func, void *u, ptrdiff_t old_errfunc = L->errfunc; L->errfunc = ef; status = luaD_rawrunprotected(L, func, u); - if (unlikely(status != LUA_OK)) { /* an error occurred? */ + if (l_unlikely(status != LUA_OK)) { /* an error occurred? */ L->ci = old_ci; L->allowhook = old_allowhooks; status = luaD_closeprotected(L, old_top, status); diff --git a/ldo.h b/ldo.h index c7721d62..6bf0ed86 100644 --- a/ldo.h +++ b/ldo.h @@ -23,7 +23,7 @@ ** at every check. */ #define luaD_checkstackaux(L,n,pre,pos) \ - if (L->stack_last - L->top <= (n)) \ + if (l_unlikely(L->stack_last - L->top <= (n))) \ { pre; luaD_growstack(L, n, 1); pos; } \ else { condmovestack(L,pre,pos); } diff --git a/lgc.c b/lgc.c index bab9beb1..94e0486e 100644 --- a/lgc.c +++ b/lgc.c @@ -916,7 +916,7 @@ static void GCTM (lua_State *L) { L->ci->callstatus &= ~CIST_FIN; /* not running a finalizer anymore */ L->allowhook = oldah; /* restore hooks */ g->gcrunning = running; /* restore state */ - if (unlikely(status != LUA_OK)) { /* error while running __gc? */ + if (l_unlikely(status != LUA_OK)) { /* error while running __gc? */ luaE_warnerror(L, "__gc metamethod"); L->top--; /* pops error object */ } diff --git a/liolib.c b/liolib.c index 79516724..b08397da 100644 --- a/liolib.c +++ b/liolib.c @@ -186,7 +186,7 @@ static int f_tostring (lua_State *L) { static FILE *tofile (lua_State *L) { LStream *p = tolstream(L); - if (isclosed(p)) + if (l_unlikely(isclosed(p))) luaL_error(L, "attempt to use a closed file"); lua_assert(p->f); return p->f; @@ -261,7 +261,7 @@ static LStream *newfile (lua_State *L) { static void opencheck (lua_State *L, const char *fname, const char *mode) { LStream *p = newfile(L); p->f = fopen(fname, mode); - if (p->f == NULL) + if (l_unlikely(p->f == NULL)) luaL_error(L, "cannot open file '%s' (%s)", fname, strerror(errno)); } @@ -309,7 +309,7 @@ static FILE *getiofile (lua_State *L, const char *findex) { LStream *p; lua_getfield(L, LUA_REGISTRYINDEX, findex); p = (LStream *)lua_touserdata(L, -1); - if (isclosed(p)) + if (l_unlikely(isclosed(p))) luaL_error(L, "default %s file is closed", findex + IOPREF_LEN); return p->f; } @@ -436,7 +436,7 @@ typedef struct { ** Add current char to buffer (if not out of space) and read next one */ static int nextc (RN *rn) { - if (rn->n >= L_MAXLENNUM) { /* buffer overflow? */ + if (l_unlikely(rn->n >= L_MAXLENNUM)) { /* buffer overflow? */ rn->buff[0] = '\0'; /* invalidate result */ return 0; /* fail */ } @@ -499,8 +499,8 @@ static int read_number (lua_State *L, FILE *f) { ungetc(rn.c, rn.f); /* unread look-ahead char */ l_unlockfile(rn.f); rn.buff[rn.n] = '\0'; /* finish string */ - if (lua_stringtonumber(L, rn.buff)) /* is this a valid number? */ - return 1; /* ok */ + if (l_likely(lua_stringtonumber(L, rn.buff))) + return 1; /* ok, it is a valid number */ else { /* invalid format */ lua_pushnil(L); /* "result" to be removed */ return 0; /* read fails */ @@ -676,7 +676,8 @@ static int g_write (lua_State *L, FILE *f, int arg) { status = status && (fwrite(s, sizeof(char), l, f) == l); } } - if (status) return 1; /* file handle already on stack top */ + if (l_likely(status)) + return 1; /* file handle already on stack top */ else return luaL_fileresult(L, status, NULL); } @@ -703,7 +704,7 @@ static int f_seek (lua_State *L) { luaL_argcheck(L, (lua_Integer)offset == p3, 3, "not an integer in proper range"); op = l_fseek(f, offset, mode[op]); - if (op) + if (l_unlikely(op)) return luaL_fileresult(L, 0, NULL); /* error */ else { lua_pushinteger(L, (lua_Integer)l_ftell(f)); diff --git a/llimits.h b/llimits.h index d0394831..025f1c82 100644 --- a/llimits.h +++ b/llimits.h @@ -149,22 +149,6 @@ typedef LUAI_UACINT l_uacInt; #endif -/* -** macros to improve jump prediction (used mainly for error handling) -*/ -#if !defined(likely) - -#if defined(__GNUC__) -#define likely(x) (__builtin_expect(((x) != 0), 1)) -#define unlikely(x) (__builtin_expect(((x) != 0), 0)) -#else -#define likely(x) (x) -#define unlikely(x) (x) -#endif - -#endif - - /* ** non-return type */ diff --git a/lmathlib.c b/lmathlib.c index 86def470..5f5983a4 100644 --- a/lmathlib.c +++ b/lmathlib.c @@ -73,7 +73,7 @@ static int math_atan (lua_State *L) { static int math_toint (lua_State *L) { int valid; lua_Integer n = lua_tointegerx(L, 1, &valid); - if (valid) + if (l_likely(valid)) lua_pushinteger(L, n); else { luaL_checkany(L, 1); @@ -175,7 +175,8 @@ static int math_log (lua_State *L) { lua_Number base = luaL_checknumber(L, 2); #if !defined(LUA_USE_C89) if (base == l_mathop(2.0)) - res = l_mathop(log2)(x); else + res = l_mathop(log2)(x); + else #endif if (base == l_mathop(10.0)) res = l_mathop(log10)(x); diff --git a/lmem.c b/lmem.c index 4822a0ea..e90f991a 100644 --- a/lmem.c +++ b/lmem.c @@ -83,7 +83,7 @@ void *luaM_growaux_ (lua_State *L, void *block, int nelems, int *psize, if (nelems + 1 <= size) /* does one extra element still fit? */ return block; /* nothing to be done */ if (size >= limit / 2) { /* cannot double it? */ - if (unlikely(size >= limit)) /* cannot grow even a little? */ + if (l_unlikely(size >= limit)) /* cannot grow even a little? */ luaG_runerror(L, "too many %s (limit is %d)", what, limit); size = limit; /* still have at least one free place */ } @@ -164,7 +164,7 @@ void *luaM_realloc_ (lua_State *L, void *block, size_t osize, size_t nsize) { global_State *g = G(L); lua_assert((osize == 0) == (block == NULL)); newblock = firsttry(g, block, osize, nsize); - if (unlikely(newblock == NULL && nsize > 0)) { + if (l_unlikely(newblock == NULL && nsize > 0)) { if (nsize > osize) /* not shrinking a block? */ newblock = tryagain(L, block, osize, nsize); if (newblock == NULL) /* still no memory? */ @@ -179,7 +179,7 @@ void *luaM_realloc_ (lua_State *L, void *block, size_t osize, size_t nsize) { void *luaM_saferealloc_ (lua_State *L, void *block, size_t osize, size_t nsize) { void *newblock = luaM_realloc_(L, block, osize, nsize); - if (unlikely(newblock == NULL && nsize > 0)) /* allocation failed? */ + if (l_unlikely(newblock == NULL && nsize > 0)) /* allocation failed? */ luaM_error(L); return newblock; } @@ -191,7 +191,7 @@ void *luaM_malloc_ (lua_State *L, size_t size, int tag) { else { global_State *g = G(L); void *newblock = firsttry(g, NULL, tag, size); - if (unlikely(newblock == NULL)) { + if (l_unlikely(newblock == NULL)) { newblock = tryagain(L, NULL, tag, size); if (newblock == NULL) luaM_error(L); diff --git a/loadlib.c b/loadlib.c index c0ec9a13..6f9fa373 100644 --- a/loadlib.c +++ b/loadlib.c @@ -132,14 +132,16 @@ static void lsys_unloadlib (void *lib) { static void *lsys_load (lua_State *L, const char *path, int seeglb) { void *lib = dlopen(path, RTLD_NOW | (seeglb ? RTLD_GLOBAL : RTLD_LOCAL)); - if (lib == NULL) lua_pushstring(L, dlerror()); + if (l_unlikely(lib == NULL)) + lua_pushstring(L, dlerror()); return lib; } static lua_CFunction lsys_sym (lua_State *L, void *lib, const char *sym) { lua_CFunction f = cast_func(dlsym(lib, sym)); - if (f == NULL) lua_pushstring(L, dlerror()); + if (l_unlikely(f == NULL)) + lua_pushstring(L, dlerror()); return f; } @@ -410,7 +412,7 @@ static int ll_loadlib (lua_State *L) { const char *path = luaL_checkstring(L, 1); const char *init = luaL_checkstring(L, 2); int stat = lookforfunc(L, path, init); - if (stat == 0) /* no errors? */ + if (l_likely(stat == 0)) /* no errors? */ return 1; /* return the loaded function */ else { /* error; error message is on stack top */ luaL_pushfail(L); @@ -523,14 +525,14 @@ static const char *findfile (lua_State *L, const char *name, const char *path; lua_getfield(L, lua_upvalueindex(1), pname); path = lua_tostring(L, -1); - if (path == NULL) + if (l_unlikely(path == NULL)) luaL_error(L, "'package.%s' must be a string", pname); return searchpath(L, name, path, ".", dirsep); } static int checkload (lua_State *L, int stat, const char *filename) { - if (stat) { /* module loaded successfully? */ + if (l_likely(stat)) { /* module loaded successfully? */ lua_pushstring(L, filename); /* will be 2nd argument to module */ return 2; /* return open function and file name */ } @@ -623,13 +625,14 @@ static void findloader (lua_State *L, const char *name) { int i; luaL_Buffer msg; /* to build error message */ /* push 'package.searchers' to index 3 in the stack */ - if (lua_getfield(L, lua_upvalueindex(1), "searchers") != LUA_TTABLE) + if (l_unlikely(lua_getfield(L, lua_upvalueindex(1), "searchers") + != LUA_TTABLE)) luaL_error(L, "'package.searchers' must be a table"); luaL_buffinit(L, &msg); /* iterate over available searchers to find a loader */ for (i = 1; ; i++) { luaL_addstring(&msg, "\n\t"); /* error-message prefix */ - if (lua_rawgeti(L, 3, i) == LUA_TNIL) { /* no more searchers? */ + if (l_unlikely(lua_rawgeti(L, 3, i) == LUA_TNIL)) { /* no more searchers? */ lua_pop(L, 1); /* remove nil */ luaL_buffsub(&msg, 2); /* remove prefix */ luaL_pushresult(&msg); /* create error message */ diff --git a/loslib.c b/loslib.c index e65e188b..3e20d622 100644 --- a/loslib.c +++ b/loslib.c @@ -170,7 +170,7 @@ static int os_tmpname (lua_State *L) { char buff[LUA_TMPNAMBUFSIZE]; int err; lua_tmpnam(buff, err); - if (err) + if (l_unlikely(err)) return luaL_error(L, "unable to generate a unique filename"); lua_pushstring(L, buff); return 1; @@ -208,7 +208,7 @@ static int os_clock (lua_State *L) { */ static void setfield (lua_State *L, const char *key, int value, int delta) { #if (defined(LUA_NUMTIME) && LUA_MAXINTEGER <= INT_MAX) - if (value > LUA_MAXINTEGER - delta) + if (l_unlikely(value > LUA_MAXINTEGER - delta)) luaL_error(L, "field '%s' is out-of-bound", key); #endif lua_pushinteger(L, (lua_Integer)value + delta); @@ -253,9 +253,9 @@ static int getfield (lua_State *L, const char *key, int d, int delta) { int t = lua_getfield(L, -1, key); /* get field and its type */ lua_Integer res = lua_tointegerx(L, -1, &isnum); if (!isnum) { /* field is not an integer? */ - if (t != LUA_TNIL) /* some other value? */ + if (l_unlikely(t != LUA_TNIL)) /* some other value? */ return luaL_error(L, "field '%s' is not an integer", key); - else if (d < 0) /* absent field; no default? */ + else if (l_unlikely(d < 0)) /* absent field; no default? */ return luaL_error(L, "field '%s' missing in date table", key); res = d; } diff --git a/lparser.c b/lparser.c index 249ba9a4..284ef1f0 100644 --- a/lparser.c +++ b/lparser.c @@ -128,7 +128,7 @@ static void checknext (LexState *ls, int c) { ** in line 'where' (if that is not the current line). */ static void check_match (LexState *ls, int what, int who, int where) { - if (unlikely(!testnext(ls, what))) { + if (l_unlikely(!testnext(ls, what))) { if (where == ls->linenumber) /* all in the same line? */ error_expected(ls, what); /* do not need a complex message */ else { @@ -517,7 +517,7 @@ static void solvegoto (LexState *ls, int g, Labeldesc *label) { Labellist *gl = &ls->dyd->gt; /* list of goto's */ Labeldesc *gt = &gl->arr[g]; /* goto to be resolved */ lua_assert(eqstr(gt->name, label->name)); - if (unlikely(gt->nactvar < label->nactvar)) /* enter some scope? */ + if (l_unlikely(gt->nactvar < label->nactvar)) /* enter some scope? */ jumpscopeerror(ls, gt); luaK_patchlist(ls->fs, gt->pc, label->pc); for (i = g; i < gl->n - 1; i++) /* remove goto from pending list */ @@ -1435,7 +1435,7 @@ static void breakstat (LexState *ls) { */ static void checkrepeated (LexState *ls, TString *name) { Labeldesc *lb = findlabel(ls, name); - if (unlikely(lb != NULL)) { /* already defined? */ + if (l_unlikely(lb != NULL)) { /* already defined? */ const char *msg = "label '%s' already defined on line %d"; msg = luaO_pushfstring(ls->L, msg, getstr(name), lb->line); luaK_semerror(ls, msg); /* error */ @@ -1520,7 +1520,7 @@ static void fixforjump (FuncState *fs, int pc, int dest, int back) { int offset = dest - (pc + 1); if (back) offset = -offset; - if (unlikely(offset > MAXARG_Bx)) + if (l_unlikely(offset > MAXARG_Bx)) luaX_syntaxerror(fs->ls, "control structure too long"); SETARG_Bx(*jmp, offset); } diff --git a/lstate.c b/lstate.c index 38078521..04909db3 100644 --- a/lstate.c +++ b/lstate.c @@ -172,7 +172,7 @@ void luaE_checkcstack (lua_State *L) { LUAI_FUNC void luaE_incCstack (lua_State *L) { L->nCcalls++; - if (unlikely(getCcalls(L) >= LUAI_MAXCCALLS)) + if (l_unlikely(getCcalls(L) >= LUAI_MAXCCALLS)) luaE_checkcstack(L); } diff --git a/lstring.c b/lstring.c index 138871c7..13dcaf42 100644 --- a/lstring.c +++ b/lstring.c @@ -89,7 +89,7 @@ void luaS_resize (lua_State *L, int nsize) { if (nsize < osize) /* shrinking table? */ tablerehash(tb->hash, osize, nsize); /* depopulate shrinking part */ newvect = luaM_reallocvector(L, tb->hash, osize, nsize, TString*); - if (unlikely(newvect == NULL)) { /* reallocation failed? */ + if (l_unlikely(newvect == NULL)) { /* reallocation failed? */ if (nsize < osize) /* was it shrinking table? */ tablerehash(tb->hash, nsize, osize); /* restore to original size */ /* leave table as it was */ @@ -172,7 +172,7 @@ void luaS_remove (lua_State *L, TString *ts) { static void growstrtab (lua_State *L, stringtable *tb) { - if (unlikely(tb->nuse == MAX_INT)) { /* too many strings? */ + if (l_unlikely(tb->nuse == MAX_INT)) { /* too many strings? */ luaC_fullgc(L, 1); /* try to free some... */ if (tb->nuse == MAX_INT) /* still too many? */ luaM_error(L); /* cannot even create a message... */ @@ -223,7 +223,7 @@ TString *luaS_newlstr (lua_State *L, const char *str, size_t l) { return internshrstr(L, str, l); else { TString *ts; - if (unlikely(l >= (MAX_SIZE - sizeof(TString))/sizeof(char))) + if (l_unlikely(l >= (MAX_SIZE - sizeof(TString))/sizeof(char))) luaM_toobig(L); ts = luaS_createlngstrobj(L, l); memcpy(getstr(ts), str, l * sizeof(char)); @@ -259,7 +259,7 @@ Udata *luaS_newudata (lua_State *L, size_t s, int nuvalue) { Udata *u; int i; GCObject *o; - if (unlikely(s > MAX_SIZE - udatamemoffset(nuvalue))) + if (l_unlikely(s > MAX_SIZE - udatamemoffset(nuvalue))) luaM_toobig(L); o = luaC_newobj(L, LUA_VUSERDATA, sizeudata(nuvalue, s)); u = gco2u(o); diff --git a/lstrlib.c b/lstrlib.c index c7242ea4..47e5b27a 100644 --- a/lstrlib.c +++ b/lstrlib.c @@ -152,8 +152,9 @@ static int str_rep (lua_State *L) { const char *s = luaL_checklstring(L, 1, &l); lua_Integer n = luaL_checkinteger(L, 2); const char *sep = luaL_optlstring(L, 3, "", &lsep); - if (n <= 0) lua_pushliteral(L, ""); - else if (l + lsep < l || l + lsep > MAXSIZE / n) /* may overflow? */ + if (n <= 0) + lua_pushliteral(L, ""); + else if (l_unlikely(l + lsep < l || l + lsep > MAXSIZE / n)) return luaL_error(L, "resulting string too large"); else { size_t totallen = (size_t)n * l + (size_t)(n - 1) * lsep; @@ -181,7 +182,7 @@ static int str_byte (lua_State *L) { size_t pose = getendpos(L, 3, pi, l); int n, i; if (posi > pose) return 0; /* empty interval; return no values */ - if (pose - posi >= (size_t)INT_MAX) /* arithmetic overflow? */ + if (l_unlikely(pose - posi >= (size_t)INT_MAX)) /* arithmetic overflow? */ return luaL_error(L, "string slice too long"); n = (int)(pose - posi) + 1; luaL_checkstack(L, n, "string slice too long"); @@ -235,7 +236,7 @@ static int str_dump (lua_State *L) { luaL_checktype(L, 1, LUA_TFUNCTION); lua_settop(L, 1); /* ensure function is on the top of the stack */ state.init = 0; - if (lua_dump(L, writer, &state, strip) != 0) + if (l_unlikely(lua_dump(L, writer, &state, strip) != 0)) return luaL_error(L, "unable to dump given function"); luaL_pushresult(&state.B); return 1; @@ -275,7 +276,8 @@ static int tonum (lua_State *L, int arg) { static void trymt (lua_State *L, const char *mtname) { lua_settop(L, 2); /* back to the original arguments */ - if (lua_type(L, 2) == LUA_TSTRING || !luaL_getmetafield(L, 2, mtname)) + if (l_unlikely(lua_type(L, 2) == LUA_TSTRING || + !luaL_getmetafield(L, 2, mtname))) luaL_error(L, "attempt to %s a '%s' with a '%s'", mtname + 2, luaL_typename(L, -2), luaL_typename(L, -1)); lua_insert(L, -3); /* put metamethod before arguments */ @@ -383,7 +385,8 @@ static const char *match (MatchState *ms, const char *s, const char *p); static int check_capture (MatchState *ms, int l) { l -= '1'; - if (l < 0 || l >= ms->level || ms->capture[l].len == CAP_UNFINISHED) + if (l_unlikely(l < 0 || l >= ms->level || + ms->capture[l].len == CAP_UNFINISHED)) return luaL_error(ms->L, "invalid capture index %%%d", l + 1); return l; } @@ -400,14 +403,14 @@ static int capture_to_close (MatchState *ms) { static const char *classend (MatchState *ms, const char *p) { switch (*p++) { case L_ESC: { - if (p == ms->p_end) + if (l_unlikely(p == ms->p_end)) luaL_error(ms->L, "malformed pattern (ends with '%%')"); return p+1; } case '[': { if (*p == '^') p++; do { /* look for a ']' */ - if (p == ms->p_end) + if (l_unlikely(p == ms->p_end)) luaL_error(ms->L, "malformed pattern (missing ']')"); if (*(p++) == L_ESC && p < ms->p_end) p++; /* skip escapes (e.g. '%]') */ @@ -482,7 +485,7 @@ static int singlematch (MatchState *ms, const char *s, const char *p, static const char *matchbalance (MatchState *ms, const char *s, const char *p) { - if (p >= ms->p_end - 1) + if (l_unlikely(p >= ms->p_end - 1)) luaL_error(ms->L, "malformed pattern (missing arguments to '%%b')"); if (*s != *p) return NULL; else { @@ -565,7 +568,7 @@ static const char *match_capture (MatchState *ms, const char *s, int l) { static const char *match (MatchState *ms, const char *s, const char *p) { - if (ms->matchdepth-- == 0) + if (l_unlikely(ms->matchdepth-- == 0)) luaL_error(ms->L, "pattern too complex"); init: /* using goto's to optimize tail recursion */ if (p != ms->p_end) { /* end of pattern? */ @@ -599,7 +602,7 @@ static const char *match (MatchState *ms, const char *s, const char *p) { case 'f': { /* frontier? */ const char *ep; char previous; p += 2; - if (*p != '[') + if (l_unlikely(*p != '[')) luaL_error(ms->L, "missing '[' after '%%f' in pattern"); ep = classend(ms, p); /* points to what is next */ previous = (s == ms->src_init) ? '\0' : *(s - 1); @@ -699,7 +702,7 @@ static const char *lmemfind (const char *s1, size_t l1, static size_t get_onecapture (MatchState *ms, int i, const char *s, const char *e, const char **cap) { if (i >= ms->level) { - if (i != 0) + if (l_unlikely(i != 0)) luaL_error(ms->L, "invalid capture index %%%d", i + 1); *cap = s; return e - s; @@ -707,7 +710,7 @@ static size_t get_onecapture (MatchState *ms, int i, const char *s, else { ptrdiff_t capl = ms->capture[i].len; *cap = ms->capture[i].init; - if (capl == CAP_UNFINISHED) + if (l_unlikely(capl == CAP_UNFINISHED)) luaL_error(ms->L, "unfinished capture"); else if (capl == CAP_POSITION) lua_pushinteger(ms->L, (ms->capture[i].init - ms->src_init) + 1); @@ -926,7 +929,7 @@ static int add_value (MatchState *ms, luaL_Buffer *b, const char *s, luaL_addlstring(b, s, e - s); /* keep original text */ return 0; /* no changes */ } - else if (!lua_isstring(L, -1)) + else if (l_unlikely(!lua_isstring(L, -1))) return luaL_error(L, "invalid replacement value (a %s)", luaL_typename(L, -1)); else { @@ -1058,7 +1061,7 @@ static int lua_number2strx (lua_State *L, char *buff, int sz, for (i = 0; i < n; i++) buff[i] = toupper(uchar(buff[i])); } - else if (fmt[SIZELENMOD] != 'a') + else if (l_unlikely(fmt[SIZELENMOD] != 'a')) return luaL_error(L, "modifiers for format '%%a'/'%%A' not implemented"); return n; } @@ -1411,7 +1414,7 @@ static int getnum (const char **fmt, int df) { */ static int getnumlimit (Header *h, const char **fmt, int df) { int sz = getnum(fmt, df); - if (sz > MAXINTSIZE || sz <= 0) + if (l_unlikely(sz > MAXINTSIZE || sz <= 0)) return luaL_error(h->L, "integral size (%d) out of limits [1,%d]", sz, MAXINTSIZE); return sz; @@ -1452,7 +1455,7 @@ static KOption getoption (Header *h, const char **fmt, int *size) { case 's': *size = getnumlimit(h, fmt, sizeof(size_t)); return Kstring; case 'c': *size = getnum(fmt, -1); - if (*size == -1) + if (l_unlikely(*size == -1)) luaL_error(h->L, "missing size for format option 'c'"); return Kchar; case 'z': return Kzstr; @@ -1491,7 +1494,7 @@ static KOption getdetails (Header *h, size_t totalsize, else { if (align > h->maxalign) /* enforce maximum alignment */ align = h->maxalign; - if ((align & (align - 1)) != 0) /* is 'align' not a power of 2? */ + if (l_unlikely((align & (align - 1)) != 0)) /* not a power of 2? */ luaL_argerror(h->L, 1, "format asks for alignment not power of 2"); *ntoalign = (align - (int)(totalsize & (align - 1))) & (align - 1); } @@ -1683,7 +1686,7 @@ static lua_Integer unpackint (lua_State *L, const char *str, else if (size > SZINT) { /* must check unread bytes */ int mask = (!issigned || (lua_Integer)res >= 0) ? 0 : MC; for (i = limit; i < size; i++) { - if ((unsigned char)str[islittle ? i : size - 1 - i] != mask) + if (l_unlikely((unsigned char)str[islittle ? i : size - 1 - i] != mask)) luaL_error(L, "%d-byte integer does not fit into Lua Integer", size); } } diff --git a/ltable.c b/ltable.c index e98bab71..b520cdf4 100644 --- a/ltable.c +++ b/ltable.c @@ -307,7 +307,7 @@ static unsigned int findindex (lua_State *L, Table *t, TValue *key, return i; /* yes; that's the index */ else { const TValue *n = getgeneric(t, key, 1); - if (unlikely(isabstkey(n))) + if (l_unlikely(isabstkey(n))) luaG_runerror(L, "invalid key to 'next'"); /* key not found */ i = cast_int(nodefromval(n) - gnode(t, 0)); /* key index in hash table */ /* hash elements are numbered after array ones */ @@ -541,7 +541,7 @@ void luaH_resize (lua_State *L, Table *t, unsigned int newasize, } /* allocate new array */ newarray = luaM_reallocvector(L, t->array, oldasize, newasize, TValue); - if (unlikely(newarray == NULL && newasize > 0)) { /* allocation failed? */ + if (l_unlikely(newarray == NULL && newasize > 0)) { /* allocation failed? */ freehash(L, &newt); /* release new hash part */ luaM_error(L); /* raise error (with array unchanged) */ } @@ -635,7 +635,7 @@ static Node *getfreepos (Table *t) { void luaH_newkey (lua_State *L, Table *t, const TValue *key, TValue *value) { Node *mp; TValue aux; - if (unlikely(ttisnil(key))) + if (l_unlikely(ttisnil(key))) luaG_runerror(L, "table index is nil"); else if (ttisfloat(key)) { lua_Number f = fltvalue(key); @@ -644,7 +644,7 @@ void luaH_newkey (lua_State *L, Table *t, const TValue *key, TValue *value) { setivalue(&aux, k); key = &aux; /* insert it as an integer */ } - else if (unlikely(luai_numisnan(f))) + else if (l_unlikely(luai_numisnan(f))) luaG_runerror(L, "table index is NaN"); } if (ttisnil(value)) diff --git a/ltablib.c b/ltablib.c index c7f0e4dc..d80eb801 100644 --- a/ltablib.c +++ b/ltablib.c @@ -145,7 +145,7 @@ static int tmove (lua_State *L) { static void addfield (lua_State *L, luaL_Buffer *b, lua_Integer i) { lua_geti(L, 1, i); - if (!lua_isstring(L, -1)) + if (l_unlikely(!lua_isstring(L, -1))) luaL_error(L, "invalid value (%s) at index %I in table for 'concat'", luaL_typename(L, -1), i); luaL_addvalue(b); @@ -196,7 +196,8 @@ static int tunpack (lua_State *L) { lua_Integer e = luaL_opt(L, luaL_checkinteger, 3, luaL_len(L, 1)); if (i > e) return 0; /* empty range */ n = (lua_Unsigned)e - i; /* number of elements minus 1 (avoid overflows) */ - if (n >= (unsigned int)INT_MAX || !lua_checkstack(L, (int)(++n))) + if (l_unlikely(n >= (unsigned int)INT_MAX || + !lua_checkstack(L, (int)(++n)))) return luaL_error(L, "too many results to unpack"); for (; i < e; i++) { /* push arg[i..e - 1] (to avoid overflows) */ lua_geti(L, 1, i); @@ -300,14 +301,14 @@ static IdxT partition (lua_State *L, IdxT lo, IdxT up) { for (;;) { /* next loop: repeat ++i while a[i] < P */ while ((void)lua_geti(L, 1, ++i), sort_comp(L, -1, -2)) { - if (i == up - 1) /* a[i] < P but a[up - 1] == P ?? */ + if (l_unlikely(i == up - 1)) /* a[i] < P but a[up - 1] == P ?? */ luaL_error(L, "invalid order function for sorting"); lua_pop(L, 1); /* remove a[i] */ } /* after the loop, a[i] >= P and a[lo .. i - 1] < P */ /* next loop: repeat --j while P < a[j] */ while ((void)lua_geti(L, 1, --j), sort_comp(L, -3, -1)) { - if (j < i) /* j < i but a[j] > P ?? */ + if (l_unlikely(j < i)) /* j < i but a[j] > P ?? */ luaL_error(L, "invalid order function for sorting"); lua_pop(L, 1); /* remove a[j] */ } diff --git a/ltm.c b/ltm.c index 4770f96b..b657b783 100644 --- a/ltm.c +++ b/ltm.c @@ -147,7 +147,7 @@ static int callbinTM (lua_State *L, const TValue *p1, const TValue *p2, void luaT_trybinTM (lua_State *L, const TValue *p1, const TValue *p2, StkId res, TMS event) { - if (!callbinTM(L, p1, p2, res, event)) { + if (l_unlikely(!callbinTM(L, p1, p2, res, event))) { switch (event) { case TM_BAND: case TM_BOR: case TM_BXOR: case TM_SHL: case TM_SHR: case TM_BNOT: { @@ -166,7 +166,8 @@ void luaT_trybinTM (lua_State *L, const TValue *p1, const TValue *p2, void luaT_tryconcatTM (lua_State *L) { StkId top = L->top; - if (!callbinTM(L, s2v(top - 2), s2v(top - 1), top - 2, TM_CONCAT)) + if (l_unlikely(!callbinTM(L, s2v(top - 2), s2v(top - 1), top - 2, + TM_CONCAT))) luaG_concaterror(L, s2v(top - 2), s2v(top - 1)); } diff --git a/luaconf.h b/luaconf.h index a44858c4..ae73e2fd 100644 --- a/luaconf.h +++ b/luaconf.h @@ -660,6 +660,26 @@ #define lua_getlocaledecpoint() (localeconv()->decimal_point[0]) #endif + +/* +** macros to improve jump prediction, used mostly for error handling +** and debug facilities. +*/ +#if (defined(LUA_CORE) || defined(LUA_LIB)) && !defined(l_likely) + +#include +#if defined(__GNUC__) +#define l_likely(x) (__builtin_expect(((x) != 0), 1)) +#define l_unlikely(x) (__builtin_expect(((x) != 0), 0)) +#else +#define l_likely(x) (x) +#define l_unlikely(x) (x) +#endif + +#endif + + + /* }================================================================== */ diff --git a/lvm.c b/lvm.c index 1252ecbf..728acd46 100644 --- a/lvm.c +++ b/lvm.c @@ -235,11 +235,11 @@ static int forprep (lua_State *L, StkId ra) { } else { /* try making all values floats */ lua_Number init; lua_Number limit; lua_Number step; - if (unlikely(!tonumber(plimit, &limit))) + if (l_unlikely(!tonumber(plimit, &limit))) luaG_forerror(L, plimit, "limit"); - if (unlikely(!tonumber(pstep, &step))) + if (l_unlikely(!tonumber(pstep, &step))) luaG_forerror(L, pstep, "step"); - if (unlikely(!tonumber(pinit, &init))) + if (l_unlikely(!tonumber(pinit, &init))) luaG_forerror(L, pinit, "initial value"); if (step == 0) luaG_runerror(L, "'for' step is zero"); @@ -292,7 +292,7 @@ void luaV_finishget (lua_State *L, const TValue *t, TValue *key, StkId val, if (slot == NULL) { /* 't' is not a table? */ lua_assert(!ttistable(t)); tm = luaT_gettmbyobj(L, t, TM_INDEX); - if (unlikely(notm(tm))) + if (l_unlikely(notm(tm))) luaG_typeerror(L, t, "index"); /* no metamethod */ /* else will try the metamethod */ } @@ -346,7 +346,7 @@ void luaV_finishset (lua_State *L, const TValue *t, TValue *key, } else { /* not a table; check metamethod */ tm = luaT_gettmbyobj(L, t, TM_NEWINDEX); - if (unlikely(notm(tm))) + if (l_unlikely(notm(tm))) luaG_typeerror(L, t, "index"); } /* try the metamethod */ @@ -651,7 +651,7 @@ void luaV_concat (lua_State *L, int total) { /* collect total length and number of strings */ for (n = 1; n < total && tostring(L, s2v(top - n - 1)); n++) { size_t l = vslen(s2v(top - n - 1)); - if (unlikely(l >= (MAX_SIZE/sizeof(char)) - tl)) + if (l_unlikely(l >= (MAX_SIZE/sizeof(char)) - tl)) luaG_runerror(L, "string length overflow"); tl += l; } @@ -695,7 +695,7 @@ void luaV_objlen (lua_State *L, StkId ra, const TValue *rb) { } default: { /* try metamethod */ tm = luaT_gettmbyobj(L, rb, TM_LEN); - if (unlikely(notm(tm))) /* no metamethod? */ + if (l_unlikely(notm(tm))) /* no metamethod? */ luaG_typeerror(L, rb, "get length of"); break; } @@ -711,7 +711,7 @@ void luaV_objlen (lua_State *L, StkId ra, const TValue *rb) { ** otherwise 'floor(q) == trunc(q) - 1'. */ lua_Integer luaV_idiv (lua_State *L, lua_Integer m, lua_Integer n) { - if (unlikely(l_castS2U(n) + 1u <= 1u)) { /* special cases: -1 or 0 */ + if (l_unlikely(l_castS2U(n) + 1u <= 1u)) { /* special cases: -1 or 0 */ if (n == 0) luaG_runerror(L, "attempt to divide by zero"); return intop(-, 0, m); /* n==-1; avoid overflow with 0x80000...//-1 */ @@ -731,7 +731,7 @@ lua_Integer luaV_idiv (lua_State *L, lua_Integer m, lua_Integer n) { ** about luaV_idiv.) */ lua_Integer luaV_mod (lua_State *L, lua_Integer m, lua_Integer n) { - if (unlikely(l_castS2U(n) + 1u <= 1u)) { /* special cases: -1 or 0 */ + if (l_unlikely(l_castS2U(n) + 1u <= 1u)) { /* special cases: -1 or 0 */ if (n == 0) luaG_runerror(L, "attempt to perform 'n%%0'"); return 0; /* m % -1 == 0; avoid overflow with 0x80000...%-1 */ @@ -1049,7 +1049,8 @@ void luaV_finishOp (lua_State *L) { #define updatebase(ci) (base = ci->func + 1) -#define updatestack(ci) { if (trap) { updatebase(ci); ra = RA(i); } } +#define updatestack(ci) \ + { if (l_unlikely(trap)) { updatebase(ci); ra = RA(i); } } /* @@ -1107,7 +1108,7 @@ void luaV_finishOp (lua_State *L) { /* fetch an instruction and prepare its execution */ #define vmfetch() { \ - if (trap) { /* stack reallocation or hooks? */ \ + if (l_unlikely(trap)) { /* stack reallocation or hooks? */ \ trap = luaG_traceexec(L, pc); /* handle hooks */ \ updatebase(ci); /* correct stack */ \ } \ @@ -1135,7 +1136,7 @@ void luaV_execute (lua_State *L, CallInfo *ci) { cl = clLvalue(s2v(ci->func)); k = cl->p->k; pc = ci->u.l.savedpc; - if (trap) { + if (l_unlikely(trap)) { if (pc == cl->p->code) { /* first instruction (not resuming)? */ if (cl->p->is_vararg) trap = 0; /* hooks will start after VARARGPREP instruction */ @@ -1678,23 +1679,23 @@ void luaV_execute (lua_State *L, CallInfo *ci) { goto ret; } vmcase(OP_RETURN0) { - if (L->hookmask) { + if (l_unlikely(L->hookmask)) { L->top = ra; savepc(ci); luaD_poscall(L, ci, 0); /* no hurry... */ trap = 1; } else { /* do the 'poscall' here */ - int nres = ci->nresults; + int nres; L->ci = ci->previous; /* back to caller */ L->top = base - 1; - while (nres-- > 0) + for (nres = ci->nresults; l_unlikely(nres > 0); nres--) setnilvalue(s2v(L->top++)); /* all results are nil */ } goto ret; } vmcase(OP_RETURN1) { - if (L->hookmask) { + if (l_unlikely(L->hookmask)) { L->top = ra + 1; savepc(ci); luaD_poscall(L, ci, 1); /* no hurry... */ @@ -1708,8 +1709,8 @@ void luaV_execute (lua_State *L, CallInfo *ci) { else { setobjs2s(L, base - 1, ra); /* at least this result */ L->top = base; - while (--nres > 0) /* complete missing results */ - setnilvalue(s2v(L->top++)); + for (; l_unlikely(nres > 1); nres--) + setnilvalue(s2v(L->top++)); /* complete missing results */ } } ret: /* return from a Lua function */ @@ -1812,7 +1813,7 @@ void luaV_execute (lua_State *L, CallInfo *ci) { } vmcase(OP_VARARGPREP) { ProtectNT(luaT_adjustvarargs(L, GETARG_A(i), ci, cl->p)); - if (trap) { + if (l_unlikely(trap)) { /* previous "Protect" updated trap */ luaD_hookcall(L, ci); L->oldpc = 1; /* next opcode will be seen as a "new" line */ } diff --git a/lvm.h b/lvm.h index 2d4ac160..1bc16f3a 100644 --- a/lvm.h +++ b/lvm.h @@ -60,12 +60,14 @@ typedef enum { /* convert an object to an integer (including string coercion) */ #define tointeger(o,i) \ - (ttisinteger(o) ? (*(i) = ivalue(o), 1) : luaV_tointeger(o,i,LUA_FLOORN2I)) + (l_likely(ttisinteger(o)) ? (*(i) = ivalue(o), 1) \ + : luaV_tointeger(o,i,LUA_FLOORN2I)) /* convert an object to an integer (without string coercion) */ #define tointegerns(o,i) \ - (ttisinteger(o) ? (*(i) = ivalue(o), 1) : luaV_tointegerns(o,i,LUA_FLOORN2I)) + (l_likely(ttisinteger(o)) ? (*(i) = ivalue(o), 1) \ + : luaV_tointegerns(o,i,LUA_FLOORN2I)) #define intop(op,v1,v2) l_castU2S(l_castS2U(v1) op l_castS2U(v2))