Added suport for Fixed Buffers

A fixed buffer keeps a binary chunk "forever", so that the program
does not need to copy some of its parts when loading it.

lbaselib.c

@@ -337,9 +337,20 @@ static int load_aux (lua_State *L, int status, int envidx) {
 }
 
 
+static const char *getmode (lua_State *L, int idx) {
+  const char *mode = luaL_optstring(L, idx, "bt");
+  int i = 0;
+  if (mode[i] == 'b') i++;
+  if (mode[i] == 't') i++;
+  if (mode[i] != '\0')
+    luaL_argerror(L, idx, "invalid mode");
+  return mode;
+}
+
+
 static int luaB_loadfile (lua_State *L) {
   const char *fname = luaL_optstring(L, 1, NULL);
-  const char *mode = luaL_optstring(L, 2, NULL);
+  const char *mode = getmode(L, 2);
   int env = (!lua_isnone(L, 3) ? 3 : 0);  /* 'env' index or 0 if no 'env' */
   int status = luaL_loadfilex(L, fname, mode);
   return load_aux(L, status, env);
@@ -388,7 +399,7 @@ static int luaB_load (lua_State *L) {
   int status;
   size_t l;
   const char *s = lua_tolstring(L, 1, &l);
-  const char *mode = luaL_optstring(L, 3, "bt");
+  const char *mode = getmode(L, 3);
   int env = (!lua_isnone(L, 4) ? 4 : 0);  /* 'env' index or 0 if no 'env' */
   if (s != NULL) {  /* loading a string? */
     const char *chunkname = luaL_optstring(L, 2, s);

ldo.c

@@ -977,7 +977,7 @@ struct SParser {  /* data to 'f_parser' */
 
 
 static void checkmode (lua_State *L, const char *mode, const char *x) {
-  if (mode && strchr(mode, x[0]) == NULL) {
+  if (strchr(mode, x[0]) == NULL) {
     luaO_pushfstring(L,
        "attempt to load a %s chunk (mode is '%s')", x, mode);
     luaD_throw(L, LUA_ERRSYNTAX);
@@ -988,13 +988,18 @@ static void checkmode (lua_State *L, const char *mode, const char *x) {
 static void f_parser (lua_State *L, void *ud) {
   LClosure *cl;
   struct SParser *p = cast(struct SParser *, ud);
+  const char *mode = p->mode ? p->mode : "bt";
   int c = zgetc(p->z);  /* read first character */
   if (c == LUA_SIGNATURE[0]) {
-    checkmode(L, p->mode, "binary");
+    int fixed = 0;
-    cl = luaU_undump(L, p->z, p->name);
+    if (strchr(mode, 'B') != NULL)
+      fixed = 1;
+    else
+      checkmode(L, mode, "binary");
+    cl = luaU_undump(L, p->z, p->name, fixed);
   }
   else {
-    checkmode(L, p->mode, "text");
+    checkmode(L, mode, "text");
     cl = luaY_parser(L, p->z, &p->buff, &p->dyd, p->name, c);
   }
   lua_assert(cl->nupvalues == cl->p->sizeupvalues);

lfunc.c

@@ -265,10 +265,12 @@ Proto *luaF_newproto (lua_State *L) {
 
 
 void luaF_freeproto (lua_State *L, Proto *f) {
-  luaM_freearray(L, f->code, f->sizecode);
+  if (!(f->flag & PF_FIXED)) {
+    luaM_freearray(L, f->code, f->sizecode);
+    luaM_freearray(L, f->lineinfo, f->sizelineinfo);
+  }
   luaM_freearray(L, f->p, f->sizep);
   luaM_freearray(L, f->k, f->sizek);
-  luaM_freearray(L, f->lineinfo, f->sizelineinfo);
   luaM_freearray(L, f->abslineinfo, f->sizeabslineinfo);
   luaM_freearray(L, f->locvars, f->sizelocvars);
   luaM_freearray(L, f->upvalues, f->sizeupvalues);

lobject.h

@@ -556,6 +556,7 @@ typedef struct AbsLineInfo {
 ** Flags in Prototypes
 */
 #define PF_ISVARARG	1
+#define PF_FIXED	2  /* prototype has parts in fixed memory */
 
 
 /*

lstring.c

@@ -207,8 +207,8 @@ static TString *internshrstr (lua_State *L, const char *str, size_t l) {
     list = &tb->hash[lmod(h, tb->size)];  /* rehash with new size */
   }
   ts = createstrobj(L, l, LUA_VSHRSTR, h);
-  memcpy(getshrstr(ts), str, l * sizeof(char));
   ts->shrlen = cast_byte(l);
+  memcpy(getshrstr(ts), str, l * sizeof(char));
   ts->u.hnext = *list;
   *list = ts;
   tb->nuse++;

ltests.c

@@ -1513,8 +1513,11 @@ static int runC (lua_State *L, lua_State *L1, const char *pc) {
       luaL_loadfile(L1, luaL_checkstring(L1, getnum));
     }
     else if EQ("loadstring") {
-      const char *s = luaL_checkstring(L1, getnum);
+      size_t slen;
-      luaL_loadstring(L1, s);
+      const char *s = luaL_checklstring(L1, getnum, &slen);
+      const char *name = getstring;
+      const char *mode = getstring;
+      luaL_loadbufferx(L1, s, slen, name, mode);
     }
     else if EQ("newmetatable") {
       lua_pushboolean(L1, luaL_newmetatable(L1, getstring));

lundump.c

@@ -38,6 +38,7 @@ typedef struct {
   Table *h;  /* list for string reuse */
   lu_mem offset;  /* current position relative to beginning of dump */
   lua_Integer nstr;  /* number of strings in the list */
+  lu_byte fixed;  /* dump is fixed in memory */
 } LoadState;
 
 
@@ -70,6 +71,16 @@ static void loadAlign (LoadState *S, int align) {
 }
 
 
+#define getaddr(S,n,t)	cast(t *, getaddr_(S,n,sizeof(t)))
+
+static const void *getaddr_ (LoadState *S, int n, int sz) {
+  const void *block = luaZ_getaddr(S->Z, n * sz);
+  if (block == NULL)
+    error(S, "truncated fixed buffer");
+  return block;
+}
+
+
 #define loadVar(S,x)		loadVector(S,&x,1)
 
 
@@ -169,10 +180,16 @@ static TString *loadString (LoadState *S, Proto *p) {
 
 static void loadCode (LoadState *S, Proto *f) {
   int n = loadInt(S);
-  f->code = luaM_newvectorchecked(S->L, n, Instruction);
-  f->sizecode = n;
   loadAlign(S, sizeof(f->code[0]));
-  loadVector(S, f->code, n);
+  if (S->fixed) {
+    f->code = getaddr(S, n, Instruction);
+    f->sizecode = n;
+  }
+  else {
+    f->code = luaM_newvectorchecked(S->L, n, Instruction);
+    f->sizecode = n;
+    loadVector(S, f->code, n);
+  }
 }
 
 
@@ -254,9 +271,15 @@ static void loadUpvalues (LoadState *S, Proto *f) {
 static void loadDebug (LoadState *S, Proto *f) {
   int i, n;
   n = loadInt(S);
-  f->lineinfo = luaM_newvectorchecked(S->L, n, ls_byte);
+  if (S->fixed) {
-  f->sizelineinfo = n;
+    f->lineinfo = getaddr(S, n, ls_byte);
-  loadVector(S, f->lineinfo, n);
+    f->sizelineinfo = n;
+  }
+  else {
+    f->lineinfo = luaM_newvectorchecked(S->L, n, ls_byte);
+    f->sizelineinfo = n;
+    loadVector(S, f->lineinfo, n);
+  }
   n = loadInt(S);
   f->abslineinfo = luaM_newvectorchecked(S->L, n, AbsLineInfo);
   f->sizeabslineinfo = n;
@@ -287,7 +310,9 @@ static void loadFunction (LoadState *S, Proto *f) {
   f->linedefined = loadInt(S);
   f->lastlinedefined = loadInt(S);
   f->numparams = loadByte(S);
-  f->flag = loadByte(S) & PF_ISVARARG;  /* keep only the meaningful flags */
+  f->flag = loadByte(S) & PF_ISVARARG;  /* get only the meaningful flags */
+  if (S->fixed)
+    f->flag |= PF_FIXED;  /* signal that code is fixed */
   f->maxstacksize = loadByte(S);
   loadCode(S, f);
   loadConstants(S, f);
@@ -335,7 +360,7 @@ static void checkHeader (LoadState *S) {
 /*
 ** Load precompiled chunk.
 */
-LClosure *luaU_undump (lua_State *L, ZIO *Z, const char *name) {
+LClosure *luaU_undump (lua_State *L, ZIO *Z, const char *name, int fixed) {
   LoadState S;
   LClosure *cl;
   if (*name == '@' || *name == '=')
@@ -346,6 +371,7 @@ LClosure *luaU_undump (lua_State *L, ZIO *Z, const char *name) {
     S.name = name;
   S.L = L;
   S.Z = Z;
+  S.fixed = fixed;
   S.offset = 1;  /* fist byte was already read */
   checkHeader(&S);
   cl = luaF_newLclosure(L, loadByte(&S));

lundump.h

@@ -26,7 +26,8 @@
 #define LUAC_FORMAT	0	/* this is the official format */
 
 /* load one chunk; from lundump.c */
-LUAI_FUNC LClosure* luaU_undump (lua_State* L, ZIO* Z, const char* name);
+LUAI_FUNC LClosure* luaU_undump (lua_State* L, ZIO* Z, const char* name,
+                                               int fixed);
 
 /* dump one chunk; from ldump.c */
 LUAI_FUNC int luaU_dump (lua_State* L, const Proto* f, lua_Writer w,

lzio.c

@@ -45,17 +45,25 @@ void luaZ_init (lua_State *L, ZIO *z, lua_Reader reader, void *data) {
 
 
 /* --------------------------------------------------------------- read --- */
+
+static int checkbuffer (ZIO *z) {
+  if (z->n == 0) {  /* no bytes in buffer? */
+    if (luaZ_fill(z) == EOZ)  /* try to read more */
+      return 0;  /* no more input */
+    else {
+      z->n++;  /* luaZ_fill consumed first byte; put it back */
+      z->p--;
+    }
+  }
+  return 1;  /* now buffer has something */
+}
+
+
 size_t luaZ_read (ZIO *z, void *b, size_t n) {
   while (n) {
     size_t m;
-    if (z->n == 0) {  /* no bytes in buffer? */
+    if (!checkbuffer(z))
-      if (luaZ_fill(z) == EOZ)  /* try to read more */
+      return n;  /* no more input; return number of missing bytes */
-        return n;  /* no more input; return number of missing bytes */
-      else {
-        z->n++;  /* luaZ_fill consumed first byte; put it back */
-        z->p--;
-      }
-    }
     m = (n <= z->n) ? n : z->n;  /* min. between n and z->n */
     memcpy(b, z->p, m);
     z->n -= m;
@@ -66,3 +74,15 @@ size_t luaZ_read (ZIO *z, void *b, size_t n) {
   return 0;
 }
 
+
+const void *luaZ_getaddr (ZIO* z, size_t n) {
+  const void *res;
+  if (!checkbuffer(z))
+    return NULL;  /* no more input */
+  if (z->n < n)  /* not enough bytes? */
+    return NULL;  /* block not whole; cannot give an address */
+  res = z->p;  /* get block address */
+  z->n -= n;  /* consume these bytes */
+  z->p += n;
+  return res;
+}

lzio.h

@@ -48,6 +48,7 @@ LUAI_FUNC void luaZ_init (lua_State *L, ZIO *z, lua_Reader reader,
                                         void *data);
 LUAI_FUNC size_t luaZ_read (ZIO* z, void *b, size_t n);	/* read next n bytes */
 
+LUAI_FUNC const void *luaZ_getaddr (ZIO* z, size_t n);
 
 
 /* --------- Private Part ------------------ */

manual/manual.of

@@ -2730,7 +2730,8 @@ For such errors, Lua does not call the @x{message handler}.
 
 @item{@defid{LUA_ERRERR}| error while running the @x{message handler}.}
 
-@item{@defid{LUA_ERRSYNTAX}| syntax error during precompilation.}
+@item{@defid{LUA_ERRSYNTAX}| syntax error during precompilation
+or format error in a binary chunk.}
 
 @item{@defid{LUA_YIELD}| the thread (coroutine) yields.}
 
@@ -3646,6 +3647,18 @@ and loads it accordingly (see program @idx{luac}).
 The string @id{mode} works as in function @Lid{load},
 with the addition that
 a @id{NULL} value is equivalent to the string @St{bt}.
+Moreover, it may have a @Char{B} instead of a @Char{b},
+meaning a @emphx{fixed buffer} with the binary dump.
+
+A fixed buffer means that the address returned by the reader function
+should contain the chunk until everything created by the chunk has
+been collected.
+(In general, a fixed buffer would keep the chunk
+as its contents until the end of the program,
+for instance with the chunk in ROM.)
+Moreover, for a fixed buffer,
+the reader function should return the entire chunk in the first read.
+(As an example, @Lid{luaL_loadbufferx} does that.)
 
 @id{lua_load} uses the stack internally,
 so the reader function must always leave the stack
@@ -5688,6 +5701,8 @@ This function returns the same results as @Lid{lua_load}.
 @id{name} is the chunk name,
 used for debug information and error messages.
 The string @id{mode} works as in the function @Lid{lua_load}.
+In particular, this function supports mode @Char{B} for
+fixed buffers.
 
 }
 

testes/api.lua

@@ -407,7 +407,7 @@ do
       concat 3]]) == "hi alo mundo")
 
   -- "argerror" without frames
-  assert(T.checkpanic("loadstring 4") ==
+  assert(T.checkpanic("loadstring 4 name bt") ==
       "bad argument #4 (string expected, got no value)")
 
 
@@ -420,7 +420,7 @@ do
   if not _soft then
     local msg = T.checkpanic[[
       pushstring "function f() f() end"
-      loadstring -1; call 0 0
+      loadstring -1 name t; call 0 0
       getglobal f; call 0 0
     ]]
     assert(string.find(msg, "stack overflow"))
@@ -430,7 +430,7 @@ do
   assert(T.checkpanic([[
     pushstring "return {__close = function () Y = 'ho'; end}"
     newtable
-    loadstring -2
+    loadstring -2 name t
     call 0 1
     setmetatable -2
     toclose -1
@@ -458,6 +458,8 @@ if not _soft then
   print'+'
 end
 
+
+
 local lim = _soft and 500 or 12000
 local prog = {"checkstack " .. (lim * 2 + 100) .. "msg", "newtable"}
 for i = 1,lim do
@@ -481,10 +483,20 @@ for i = 1,lim do assert(t[i] == i*10); t[i] = undef end
 assert(next(t) == nil)
 prog, g, t = nil
 
+do   -- shrink stack
+  local m1, m2 = 0, collectgarbage"count" * 1024
+  while m1 ~= m2 do    -- repeat until stable
+    collectgarbage()
+    m1 = m2
+    m2 = collectgarbage"count" * 1024
+  end
+end
+
+
 -- testing errors
 
 a = T.testC([[
-  loadstring 2; pcall 0 1 0;
+  loadstring 2 name t; pcall 0 1 0;
   pushvalue 3; insert -2; pcall 1 1 0;
   pcall 0 0 0;
   return 1
@@ -498,7 +510,7 @@ local function check3(p, ...)
   assert(#arg == 3)
   assert(string.find(arg[3], p))
 end
-check3(":1:", T.testC("loadstring 2; return *", "x="))
+check3(":1:", T.testC("loadstring 2 name t; return *", "x="))
 check3("%.", T.testC("loadfile 2; return *", "."))
 check3("xxxx", T.testC("loadfile 2; return *", "xxxx"))
 
@@ -509,6 +521,35 @@ local function checkerrnopro (code, msg)
   assert(not stt and string.find(err, msg))
 end
 
+
+do
+  print("testing load of binaries in fixed buffers")
+  local source = {}
+  local N = 1000
+  -- create a somewhat "large" source
+  for i = 1, N do source[i] = "X = X + 1; " end
+  source = table.concat(source)
+  -- give chunk an explicit name to avoid using source as name
+  source = load(source, "name1")
+  -- dump without debug information
+  source = string.dump(source, true)
+  -- each "X=X+1" generates 4 opcodes with 4 bytes each
+  assert(#source > N * 4 * 4)
+  collectgarbage(); collectgarbage()
+  local m1 = collectgarbage"count" * 1024
+  -- load dump using fixed buffer
+  local code = T.testC([[
+    loadstring 2 name B;
+    return 1
+  ]], source)
+  collectgarbage()
+  local m2 = collectgarbage"count" * 1024
+  -- load used fewer than 300 bytes
+  assert(m2 > m1 and m2 - m1 < 300)
+  X = 0; code(); assert(X == N); X = nil
+end
+
+
 if not _soft then
   collectgarbage("stop")   -- avoid __gc with full stack
   checkerrnopro("pushnum 3; call 0 0", "attempt to call")