diff --git a/manifest b/manifest index 448d34a0ee..6425c9f1a0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sfixes\sfrom\strunk. -D 2019-12-13T12:14:23.880 +C Cleanup\sand\sperformance\senhancements\sfor\smini-lookaside. +D 2019-12-13T15:48:21.644 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -491,8 +491,8 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 276463aa7a26ee73958b9f4bcbef4eaec516f22bf6b8d7d2428f39c8dea90840 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c d74f5e7bd51f3c9d283442473eb65aef359664efd6513591c03f01881c4ae2da -F src/main.c 94bf8b6a283914cc8fa8743b3db5350206a4be5cccfd9c84d53409ca5d8bd1c1 -F src/malloc.c 02235d039c72102d4bddf203a493c2613164487c70ec84bab679fc30daa883f0 +F src/main.c 34d73303ac76015d3c8961f16d49ab44a56cbc998f178b27ca07847db4cf9273 +F src/malloc.c d74d986f91ebd6a4a9b2995467f4c13a1fdb57ac87df5bb0a7fa3afe1db72d1d F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de F src/mem2.c f1940d9e91948dd6a908fbb9ce3835c36b5d83c3 @@ -532,9 +532,9 @@ F src/shell.c.in 4a3a9e1c11847b1904f2b01d087af1c052f660902755abab457cab1756817de F src/sqlite.h.in 2a23e8161775253d9cf383c2c6aa559005dc787d350dcb0be67a6c4cc3bd1d19 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 72af51aa4e912e14cd495fb6e7fac65f0940db80ed950d90911aff292cc47ce2 -F src/sqliteInt.h a036ca4ee94ac0dea82c90b2c902dc84bce187adf75e68284e890bc0c4cd3d95 +F src/sqliteInt.h 4948c2046c921461d2021e52ac8cec5b65cd56ab0c58aa7150313abace90153c F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b -F src/status.c 0fe55b92810509eac821ded2e177e1da944a78cdcdfec7813ad1d6b36d6d667e +F src/status.c baf38f521ac079be616dab1ff13f60c6ce0703c7f0429a65238c695197a56935 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 F src/tclsqlite.c 8cd2600e8de23dff6cdf84d39f46ca57139b061b28f6f80b166bace17d52ab1c F src/test1.c c654981c1d86ebc90dd23fcc0969e6c85e28112f0acc2e2224a97a2a33e7c42f @@ -1852,7 +1852,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P bda92d92bf69a73bc6dd498f92578ad6e81111a3f97bc8724d1720e7283a83b6 c1014e80b26131200a115beb86929a8f0ded2dd65b075e47373346c0f170576a -R b2be5daabe2de4420861c8338271e7de +P 9c471195f6d3e4b00e2d0f909b306a4036352082dca5f016a8eece226e82163d +R 40d26fbfedcb39151f8726a7fa468e8a U drh -Z 5e37344191b5a16624379099587382f5 +Z 7742eaad4b11604e1540b9f51bda9209 diff --git a/manifest.uuid b/manifest.uuid index 69d04cf3ef..4e1f2b833d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9c471195f6d3e4b00e2d0f909b306a4036352082dca5f016a8eece226e82163d \ No newline at end of file +74805668430051032ae9b256c84e252755ee03075fc08293c948675ed40ec280 \ No newline at end of file diff --git a/src/main.c b/src/main.c index 5b70273af2..9f541ff0d0 100644 --- a/src/main.c +++ b/src/main.c @@ -684,6 +684,8 @@ static int setupLookaside(sqlite3 *db, void *pBuf, int sz, int cnt){ #ifndef SQLITE_OMIT_LOOKASIDE void *pStart; sqlite3_int64 szAlloc = sz*(sqlite3_int64)cnt; + int nBig; /* Number of full-size slots */ + int nSm; /* Number smaller mini-slots */ if( sqlite3LookasideUsed(db,0)>0 ){ return SQLITE_BUSY; @@ -708,56 +710,69 @@ static int setupLookaside(sqlite3 *db, void *pBuf, int sz, int cnt){ sqlite3BeginBenignMalloc(); pStart = sqlite3Malloc( szAlloc ); /* IMP: R-61949-35727 */ sqlite3EndBenignMalloc(); - if( pStart ) cnt = sqlite3MallocSize(pStart)/sz; + if( pStart ) szAlloc = sqlite3MallocSize(pStart); }else{ pStart = pBuf; } +#ifndef SQLITE_OMIT_MINI_LOOKASIDE + if( sz>=MINI_SZ*3 ){ + nBig = szAlloc/(3*MINI_SZ+sz); + nSm = (szAlloc - sz*nBig)/MINI_SZ; + }else if( sz>=MINI_SZ*2 ){ + nBig = szAlloc/(MINI_SZ+sz); + nSm = (szAlloc - sz*nBig)/MINI_SZ; + }else +#endif /* SQLITE_OMIT_MINI_LOOKASIDE */ + if( sz>0 ){ + nBig = szAlloc/sz; + nSm = 0; + }else{ + nBig = nSm = 0; + } db->lookaside.pStart = pStart; db->lookaside.pInit = 0; db->lookaside.pFree = 0; -#ifndef SQLITE_OMIT_MINI_LOOKASIDE - db->lookaside.pMiniInit = 0; - db->lookaside.pMiniFree = 0; - /* The arithmetic below causes the number of large lookaside slots to be 1/3 - ** the number of mini slots, based on the observation that 75% of allocations - ** are <= MINI_SZ bytes. - */ - cnt = szAlloc/(3*MINI_SZ+sz); -#endif db->lookaside.sz = (u16)sz; db->lookaside.szTrue = (u16)sz; if( pStart ){ int i; LookasideSlot *p; assert( sz > (int)sizeof(LookasideSlot*) ); - db->lookaside.nSlot = cnt; p = (LookasideSlot*)pStart; - for(i=cnt-1; i>=0; i--){ + for(i=0; ipNext = db->lookaside.pInit; db->lookaside.pInit = p; p = (LookasideSlot*)&((u8*)p)[sz]; } #ifndef SQLITE_OMIT_MINI_LOOKASIDE + db->lookaside.pMiniInit = 0; + db->lookaside.pMiniFree = 0; db->lookaside.pMiddle = p; - /* Fill the remainder of the buffer with mini slots */ - while(p<=(LookasideSlot*)&((u8*)pStart)[szAlloc-MINI_SZ]){ + for(i=0; ipNext = db->lookaside.pMiniInit; db->lookaside.pMiniInit = p; - db->lookaside.nSlot++; p = (LookasideSlot*)&((u8*)p)[MINI_SZ]; } -#endif +#endif /* SQLITE_OMIT_MINI_LOOKASIDE */ + assert( ((uptr)p)<=szAlloc + (uptr)pStart ); db->lookaside.pEnd = p; db->lookaside.bDisable = 0; db->lookaside.bMalloced = pBuf==0 ?1:0; + db->lookaside.nSlot = nBig+nSm; }else{ db->lookaside.pStart = db; +#ifndef SQLITE_OMIT_MINI_LOOKASIDE + db->lookaside.pMiniInit = 0; + db->lookaside.pMiniFree = 0; + db->lookaside.pMiddle = db; +#endif /* SQLITE_OMIT_MINI_LOOKASIDE */ db->lookaside.pEnd = db; db->lookaside.bDisable = 1; db->lookaside.sz = 0; db->lookaside.bMalloced = 0; db->lookaside.nSlot = 0; } + assert( sqlite3LookasideUsed(db,0)==0 ); #endif /* SQLITE_OMIT_LOOKASIDE */ return SQLITE_OK; } diff --git a/src/malloc.c b/src/malloc.c index 26a258b347..1074156eec 100644 --- a/src/malloc.c +++ b/src/malloc.c @@ -341,8 +341,8 @@ static int lookasideMallocSize(sqlite3 *db, void *p){ } int sqlite3DbMallocSize(sqlite3 *db, void *p){ assert( p!=0 ); - if( db==0 || !isLookaside(db,p) ){ #ifdef SQLITE_DEBUG + if( db==0 || !isLookaside(db,p) ){ if( db==0 ){ assert( sqlite3MemdebugNoType(p, (u8)~MEMTYPE_HEAP) ); assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); @@ -350,12 +350,23 @@ int sqlite3DbMallocSize(sqlite3 *db, void *p){ assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); assert( sqlite3MemdebugNoType(p, (u8)~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); } -#endif - return sqlite3GlobalConfig.m.xSize(p); - }else{ - assert( sqlite3_mutex_held(db->mutex) ); - return lookasideMallocSize(db, p); } +#endif + if( db ){ + if( ((uptr)p)<(uptr)(db->lookaside.pEnd) ){ +#ifndef SQLITE_OMIT_MINI_LOOKASIDE + if( ((uptr)p)>=(uptr)(db->lookaside.pMiddle) ){ + assert( sqlite3_mutex_held(db->mutex) ); + return MINI_SZ; + } +#endif + if( ((uptr)p)>=(uptr)(db->lookaside.pStart) ){ + assert( sqlite3_mutex_held(db->mutex) ); + return db->lookaside.szTrue; + } + } + } + return sqlite3GlobalConfig.m.xSize(p); } sqlite3_uint64 sqlite3_msize(void *p){ assert( sqlite3MemdebugNoType(p, (u8)~MEMTYPE_HEAP) ); @@ -402,26 +413,27 @@ void sqlite3DbFreeNN(sqlite3 *db, void *p){ measureAllocationSize(db, p); return; } - if( isLookaside(db, p) ){ - LookasideSlot *pBuf = (LookasideSlot*)p; + if( ((uptr)p)<(uptr)(db->lookaside.pEnd) ){ #ifndef SQLITE_OMIT_MINI_LOOKASIDE - if( p>=db->lookaside.pMiddle ){ -# ifdef SQLITE_DEBUG - /* Trash all content in the buffer being freed */ - memset(p, 0xaa, MINI_SZ); -# endif + if( ((uptr)p)>=(uptr)(db->lookaside.pMiddle) ){ + LookasideSlot *pBuf = (LookasideSlot*)p; +#ifdef SQLITE_DEBUG + memset(p, 0xaa, MINI_SZ); /* Trash freed content */ +#endif pBuf->pNext = db->lookaside.pMiniFree; db->lookaside.pMiniFree = pBuf; return; } -#endif +#endif /* SQLITE_OMIT_MINI_LOOKASIDE */ + if( ((uptr)p)>=(uptr)(db->lookaside.pStart) ){ + LookasideSlot *pBuf = (LookasideSlot*)p; #ifdef SQLITE_DEBUG - /* Trash all content in the buffer being freed */ - memset(p, 0xaa, db->lookaside.szTrue); + memset(p, 0xaa, db->lookaside.szTrue); /* Trash freed content */ #endif - pBuf->pNext = db->lookaside.pFree; - db->lookaside.pFree = pBuf; - return; + pBuf->pNext = db->lookaside.pFree; + db->lookaside.pFree = pBuf; + return; + } } } assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); @@ -585,7 +597,7 @@ void *sqlite3DbMallocRawNN(sqlite3 *db, u64 n){ } return dbMallocRawFinish(db, n); } -# ifndef SQLITE_OMIT_MINI_LOOKASIDE +#ifndef SQLITE_OMIT_MINI_LOOKASIDE if( n<=MINI_SZ ){ if( (pBuf = db->lookaside.pMiniFree)!=0 ){ db->lookaside.pMiniFree = pBuf->pNext; @@ -597,7 +609,7 @@ void *sqlite3DbMallocRawNN(sqlite3 *db, u64 n){ return (void*)pBuf; } } -# endif +#endif if( (pBuf = db->lookaside.pFree)!=0 ){ db->lookaside.pFree = pBuf->pNext; db->lookaside.anStat[0]++; @@ -631,7 +643,16 @@ void *sqlite3DbRealloc(sqlite3 *db, void *p, u64 n){ assert( db!=0 ); if( p==0 ) return sqlite3DbMallocRawNN(db, n); assert( sqlite3_mutex_held(db->mutex) ); - if( isLookaside(db,p) && nlookaside.pEnd ){ +#ifndef SQLITE_OMIT_MINI_LOOKASIDE + if( ((uptr)p)>=(uptr)db->lookaside.pMiddle ){ + if( n<=MINI_SZ ) return p; + }else +#endif + if( ((uptr)p)>=(uptr)db->lookaside.pStart ){ + if( n<=db->lookaside.szTrue ) return p; + } + } return dbReallocFinish(db, p, n); } static SQLITE_NOINLINE void *dbReallocFinish(sqlite3 *db, void *p, u64 n){ diff --git a/src/sqliteInt.h b/src/sqliteInt.h index da9a7ff811..b18f05c85e 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1277,6 +1277,25 @@ struct Schema { ** disables lookaside without adding a new test for the bDisable flag ** in a performance-critical path. sz should be set by to szTrue whenever ** bDisable changes back to zero. +** +** Lookaside buffers are initially held on the pInit list. As they are +** used and freed, they are added back to the pFree list. New allocations +** come off of pFree first, then pInit as a fallback. This dual-list +** allows use to compute a high-water mark - the maximum number of allocations +** outstanding at any point in the past - by subtracting the number of +** allocations on the pInit list from the total number of allocations. +** +** Enhancement on 2019-12-12: Mini-lookaside +** The default lookaside configuration is 100 slots of 1200 bytes each. +** The larger slot sizes are important for performance, but they waste +** a lot of space, as most lookaside allocations are less than 128 bytes. +** The mini-lookaside enhancement breaks up the lookaside allocation into +** two pools: One of 128-byte slots and the other of the default size +** (1200-byte) slots. Allocations are filled from the mini-pool first, +** failing over to the full-size pool if that does not work. Thus more +** lookaside slots are available while also using less memory. +** This enhancement can be omitted by compiling with +** SQLITE_OMIT_MINI_LOOKASIDE. */ struct Lookaside { u32 bDisable; /* Only operate the lookaside when zero */ @@ -1290,8 +1309,9 @@ struct Lookaside { #ifndef SQLITE_OMIT_MINI_LOOKASIDE LookasideSlot *pMiniInit; /* List of mini buffers not prediously used */ LookasideSlot *pMiniFree; /* List of available mini buffers */ - void *pMiddle; /* An address between the fullsize and mini buffers */ -#endif + void *pMiddle; /* First byte past end of full-size buffers and + ** the first byte of mini-buffers */ +#endif /* SQLITE_OMIT_MINI_LOOKASIDE */ void *pStart; /* First byte of available memory space */ void *pEnd; /* First byte past end of available space */ }; @@ -1304,7 +1324,11 @@ struct LookasideSlot { db->lookaside.sz=db->lookaside.bDisable?0:db->lookaside.szTrue /* Size of the MINI lookside allocation */ -#define MINI_SZ 128 +#ifdef SQLITE_OMIT_MINI_LOOKASIDE +# define MINI_SZ 0 +#else +# define MINI_SZ 128 +#endif /* ** A hash table for built-in function definitions. (Application-defined diff --git a/src/status.c b/src/status.c index d7ffe99ede..977457f8e3 100644 --- a/src/status.c +++ b/src/status.c @@ -191,7 +191,7 @@ int sqlite3LookasideUsed(sqlite3 *db, int *pHighwater){ #ifndef SQLITE_OMIT_MINI_LOOKASIDE nInit += countLookasideSlots(db->lookaside.pMiniInit); nFree += countLookasideSlots(db->lookaside.pMiniFree); -#endif +#endif /* SQLITE_OMIT_MINI_LOOKASIDE */ if( pHighwater ) *pHighwater = db->lookaside.nSlot - nInit; return db->lookaside.nSlot - (nInit+nFree); }