Add support for F2FS atomic writes. Untested at this point.

FossilOrigin-Name: 416973ede3bde8567d1f2699728f72352979e054ef988d1c1e1cfe4290f6f8b8
This commit is contained in:
dan 2017-07-20 19:49:14 +00:00
parent 4562d0db0d
commit efe1697191
7 changed files with 171 additions and 79 deletions

View File

@ -1,5 +1,5 @@
C Enhance\sthe\sbuilt-in\sdate/time\sfunctions\sso\sthat\sthey\scan\sbe\sused\sin\nCHECK\sconstraints,\sin\sthe\sWHERE\sclause\sor\spartial\sindexes,\sand\sindex\nexpressions,\sprovided\sthat\snone\sof\sthe\snon-deterministic\skeywords\n("now",\s"localtime",\s"utc")\sare\sused\sas\sarguments.
D 2017-07-20T15:08:43.378
C Add\ssupport\sfor\sF2FS\satomic\swrites.\sUntested\sat\sthis\spoint.
D 2017-07-20T19:49:14.731
F Makefile.in d9873c9925917cca9990ee24be17eb9613a668012c85a343aef7e5536ae266e8
F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
F Makefile.msc 02b469e9dcd5b7ee63fc1fb05babc174260ee4cfa4e0ef2e48c3c6801567a016
@ -424,7 +424,7 @@ F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de
F src/mem2.c f1940d9e91948dd6a908fbb9ce3835c36b5d83c3
F src/mem3.c 8768ac94694f31ffaf8b4d0ea5dc08af7010a35a
F src/mem5.c 9bf955937b07f8c32541c8a9991f33ce3173d944
F src/memjournal.c 95752936c11dc6995672d1dd783cd633eea0cc95
F src/memjournal.c 3f1d95947cd39b35a6375b2ca384bac904f16353bd54995e9bff78161463ac1c
F src/msvc.h 4942752b6a253116baaa8de75256c51a459a5e81
F src/mutex.c 8e45800ee78e0cd1f1f3fe8e398853307f4a085c
F src/mutex.h 779d588e3b7756ec3ecf7d78cde1d84aba414f85
@ -436,10 +436,10 @@ F src/os.c add02933b1dce7a39a005b00a2f5364b763e9a24
F src/os.h 8e976e59eb4ca1c0fca6d35ee803e38951cb0343
F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85
F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
F src/os_unix.c 30e2c43e4955db990e5b5a81e901f8aa74cc8820
F src/os_unix.c eca7004f2cde959ed233951256e22aa4169c72f69b831e14276d0bdded4372f5
F src/os_win.c 2a6c73eef01c51a048cc4ddccd57f981afbec18a
F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
F src/pager.c 14f6982c470c05b8e85575c69e9c1712010602e20400f8670d8699e21283e0e4
F src/pager.c 7ab906445447d5374971986de3525350d18c0e256723cd5a2456a429cd372f6c
F src/pager.h f2a99646c5533ffe11afa43e9e0bea74054e4efa
F src/parse.y e384cb73f99e1b074085c974b37f4d830e885359e4b60837e30f7d67c16ba65b
F src/pcache.c 62835bed959e2914edd26afadfecce29ece0e870
@ -455,7 +455,7 @@ F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
F src/select.c c6bf96a7f9d7d68f929de84738c599a30d0a725ab0b54420e70545743cd5ee7b
F src/shell.c bd6a37cbe8bf64ef6a6a74fdc50f067d3148149b4ce2b4d03154663e66ded55f
F src/shell.c.in b5725acacba95ccefa57b6d068f710e29ba8239c3aa704628a1902a1f729c175
F src/sqlite.h.in dad804d4e1979a2ddec33cc8da6aa50c04e6ba0dcb4058e7b3609588d010e041
F src/sqlite.h.in 51541eacdbbd26afcb5933ea4fadf9952fa9b3e4d7277c2b7983bf2b4a09a0bf
F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
F src/sqlite3ext.h 967154985ed2ae62f90d9029bb5b5071793d847f1696a2ebe9e8cc0b042ae60b
F src/sqliteInt.h 96197a18f041b9ab99e6cee0db39dbf771ac7762d9f0f63d9e719285f0478664
@ -1232,7 +1232,7 @@ F test/superlock.test ec94f0556b6488d97f71c79f9061ae08d9ab8f12
F test/symlink.test c9ebe7330d228249e447038276bfc8a7b22f4849
F test/sync.test 2f84bdbc2b2df1fcb0220575b4b9f8cea94b7529
F test/sync2.test 6be8ed007fa063b147773c1982b5bdba97a32badc536bdc6077eff5cf8710ece
F test/syscall.test 7a60601770172a8014a4d222d5f3d95a5d2b5c47fbb0374e2698e89c99e37256
F test/syscall.test a39d9a36f852ae6e4800f861bc2f2e83f68bbc2112d9399931ecfadeabd2d69d
F test/sysfault.test c9f2b0d8d677558f74de750c75e12a5454719d04
F test/tabfunc01.test c47171c36b3d411df2bd49719dcaa5d034f8d277477fd41d253940723b969a51
F test/table.test b708f3e5fa2542fa51dfab21fc07b36ea445cb2f
@ -1637,8 +1637,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
P d14fc621e918915bbf8e04597eb238ea78dff3d9c5eb4402cb88692d00dbdfee b7f70c7fcabc10b8b3e62fe5ac68476cec23acaee037c7250ff70bca3f3ab541
R 636bb2618e46215e27cfa1dc613ec36a
T +closed b7f70c7fcabc10b8b3e62fe5ac68476cec23acaee037c7250ff70bca3f3ab541
U drh
Z 8d21c041010891259262753e78bdd45f
P a90c062d46c63a1e6f83064b1c5afb26a16e93b6ee8620ca46d169fdb325c488
R 83821cb44cbf5af73c5132cb66d20c9f
T *branch * f2fs-support
T *sym-f2fs-support *
T -sym-trunk *
U dan
Z 66fb297e5afae668d4c6aa465d37e6d1

View File

@ -1 +1 @@
a90c062d46c63a1e6f83064b1c5afb26a16e93b6ee8620ca46d169fdb325c488
416973ede3bde8567d1f2699728f72352979e054ef988d1c1e1cfe4290f6f8b8

View File

@ -388,13 +388,16 @@ void sqlite3MemJournalOpen(sqlite3_file *pJfd){
/*
** If the argument p points to a MemJournal structure that is not an
** in-memory-only journal file (i.e. is one that was opened with a +ve
** nSpill parameter), and the underlying file has not yet been created,
** create it now.
** nSpill parameter or as SQLITE_OPEN_MAIN_JOURNAL), and the underlying
** file has not yet been created, create it now.
*/
int sqlite3JournalCreate(sqlite3_file *p){
int sqlite3JournalCreate(sqlite3_file *pJfd){
int rc = SQLITE_OK;
if( p->pMethods==&MemJournalMethods && ((MemJournal*)p)->nSpill>0 ){
rc = memjrnlCreateFile((MemJournal*)p);
MemJournal *p = (MemJournal*)pJfd;
if( p->pMethod==&MemJournalMethods
&& (p->nSpill>0 || (p->flags & SQLITE_OPEN_MAIN_JOURNAL))
){
rc = memjrnlCreateFile(p);
}
return rc;
}

View File

@ -90,6 +90,7 @@
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <sys/ioctl.h>
#include <unistd.h>
#include <time.h>
#include <sys/time.h>
@ -220,10 +221,8 @@ struct unixFile {
sqlite3_int64 mmapSizeMax; /* Configured FCNTL_MMAP_SIZE value */
void *pMapRegion; /* Memory mapped region */
#endif
#ifdef __QNXNTO__
int sectorSize; /* Device sector size */
int deviceCharacteristics; /* Precomputed device characteristics */
#endif
#if SQLITE_ENABLE_LOCKING_STYLE
int openFlags; /* The flags specified at open() */
#endif
@ -328,6 +327,13 @@ static pid_t randomnessPid = 0;
# define lseek lseek64
#endif
#define F2FS_IOCTL_MAGIC 0xf5
#define F2FS_IOC_START_ATOMIC_WRITE _IO(F2FS_IOCTL_MAGIC, 1)
#define F2FS_IOC_COMMIT_ATOMIC_WRITE _IO(F2FS_IOCTL_MAGIC, 2)
#define F2FS_IOC_START_VOLATILE_WRITE _IO(F2FS_IOCTL_MAGIC, 3)
#define F2FS_IOC_ABORT_VOLATILE_WRITE _IO(F2FS_IOCTL_MAGIC, 5)
/*
** Different Unix systems declare open() in different ways. Same use
** open(const char*,int,mode_t). Others use open(const char*,int,...).
@ -500,6 +506,9 @@ static struct unix_syscall {
#endif
#define osLstat ((int(*)(const char*,struct stat*))aSyscall[27].pCurrent)
{ "ioctl", (sqlite3_syscall_ptr)ioctl, 0 },
#define osIoctl ((int(*)(int,int))aSyscall[28].pCurrent)
}; /* End of the overrideable system calls */
@ -3777,6 +3786,19 @@ static int unixGetTempname(int nBuf, char *zBuf);
static int unixFileControl(sqlite3_file *id, int op, void *pArg){
unixFile *pFile = (unixFile*)id;
switch( op ){
case SQLITE_FCNTL_BEGIN_ATOMIC_WRITE: {
int rc = osIoctl(pFile->h, F2FS_IOC_START_ATOMIC_WRITE);
return rc ? SQLITE_ERROR : SQLITE_OK;
}
case SQLITE_FCNTL_COMMIT_ATOMIC_WRITE: {
int rc = osIoctl(pFile->h, F2FS_IOC_COMMIT_ATOMIC_WRITE);
return rc ? SQLITE_ERROR : SQLITE_OK;
}
case SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE: {
int rc = osIoctl(pFile->h, F2FS_IOC_ABORT_VOLATILE_WRITE);
return rc ? SQLITE_ERROR : SQLITE_OK;
}
case SQLITE_FCNTL_LOCKSTATE: {
*(int*)pArg = pFile->eFileLock;
return SQLITE_OK;
@ -3860,30 +3882,43 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){
}
/*
** Return the sector size in bytes of the underlying block device for
** the specified file. This is almost always 512 bytes, but may be
** larger for some devices.
** If pFd->sectorSize is non-zero when this function is called, it is a
** no-op. Otherwise, the values of pFd->sectorSize and
** pFd->deviceCharacteristics are set according to the file-system
** characteristics.
**
** SQLite code assumes this function cannot fail. It also assumes that
** if two files are created in the same file-system directory (i.e.
** a database and its journal file) that the sector size will be the
** same for both.
** There are two versions of this function. One for QNX and one for all
** other systems.
*/
#ifndef __QNXNTO__
static int unixSectorSize(sqlite3_file *NotUsed){
UNUSED_PARAMETER(NotUsed);
return SQLITE_DEFAULT_SECTOR_SIZE;
}
#endif
#ifndef __QNXNTO__
static void setDeviceCharacteristics(unixFile *pFd){
if( pFd->sectorSize==0 ){
int res;
assert( pFd->deviceCharacteristics==0 );
/*
** The following version of unixSectorSize() is optimized for QNX.
*/
#ifdef __QNXNTO__
/* Check for support for F2FS atomic batch writes. */
res = osIoctl(pFd->h, F2FS_IOC_START_VOLATILE_WRITE);
if( res==SQLITE_OK ){
osIoctl(pFd->h, F2FS_IOC_ABORT_VOLATILE_WRITE);
pFd->deviceCharacteristics =
SQLITE_IOCAP_BATCH_ATOMIC |
SQLITE_IOCAP_ATOMIC |
SQLITE_IOCAP_SEQUENTIAL |
SQLITE_IOCAP_SAFE_APPEND;
}
/* Set the POWERSAFE_OVERWRITE flag if requested. */
if( pFd->ctrlFlags & UNIXFILE_PSOW ){
pFd->deviceCharacteristics |= SQLITE_IOCAP_POWERSAFE_OVERWRITE;
}
pFd->sectorSize = SQLITE_DEFAULT_SECTOR_SIZE;
}
}
#else
#include <sys/dcmd_blk.h>
#include <sys/statvfs.h>
static int unixSectorSize(sqlite3_file *id){
unixFile *pFile = (unixFile*)id;
static void setDeviceCharacteristics(unixFile *pFile){
if( pFile->sectorSize == 0 ){
struct statvfs fsInfo;
@ -3952,9 +3987,24 @@ static int unixSectorSize(sqlite3_file *id){
pFile->deviceCharacteristics = 0;
pFile->sectorSize = SQLITE_DEFAULT_SECTOR_SIZE;
}
return pFile->sectorSize;
}
#endif /* __QNXNTO__ */
#endif
/*
** Return the sector size in bytes of the underlying block device for
** the specified file. This is almost always 512 bytes, but may be
** larger for some devices.
**
** SQLite code assumes this function cannot fail. It also assumes that
** if two files are created in the same file-system directory (i.e.
** a database and its journal file) that the sector size will be the
** same for both.
*/
static int unixSectorSize(sqlite3_file *id){
unixFile *pFd = (unixFile*)id;
setDeviceCharacteristics(pFd);
return pFd->sectorSize;
}
/*
** Return the device characteristics for the file.
@ -3970,16 +4020,9 @@ static int unixSectorSize(sqlite3_file *id){
** available to turn it off and URI query parameter available to turn it off.
*/
static int unixDeviceCharacteristics(sqlite3_file *id){
unixFile *p = (unixFile*)id;
int rc = 0;
#ifdef __QNXNTO__
if( p->sectorSize==0 ) unixSectorSize(id);
rc = p->deviceCharacteristics;
#endif
if( p->ctrlFlags & UNIXFILE_PSOW ){
rc |= SQLITE_IOCAP_POWERSAFE_OVERWRITE;
}
return rc;
unixFile *pFd = (unixFile*)id;
setDeviceCharacteristics(pFd);
return pFd->deviceCharacteristics;
}
#if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0
@ -7598,7 +7641,7 @@ int sqlite3_os_init(void){
/* Double-check that the aSyscall[] array has been constructed
** correctly. See ticket [bb3a86e890c8e96ab] */
assert( ArraySize(aSyscall)==28 );
assert( ArraySize(aSyscall)==29 );
/* Register all VFSes defined in the aVfs[] array */
for(i=0; i<(sizeof(aVfs)/sizeof(sqlite3_vfs)); i++){

View File

@ -958,6 +958,7 @@ static int assert_pager_state(Pager *p){
assert( isOpen(p->jfd)
|| p->journalMode==PAGER_JOURNALMODE_OFF
|| p->journalMode==PAGER_JOURNALMODE_WAL
|| (sqlite3OsDeviceCharacteristics(p->fd)&SQLITE_IOCAP_BATCH_ATOMIC)
);
break;
@ -1194,6 +1195,11 @@ static int jrnlBufferSize(Pager *pPager){
assert( isOpen(pPager->fd) );
dc = sqlite3OsDeviceCharacteristics(pPager->fd);
/* use in-memory journal */
if( dc&SQLITE_IOCAP_BATCH_ATOMIC ){
return -1;
}
nSector = pPager->sectorSize;
szPage = pPager->pageSize;
@ -2012,7 +2018,9 @@ static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){
}
releaseAllSavepoints(pPager);
assert( isOpen(pPager->jfd) || pPager->pInJournal==0 );
assert( isOpen(pPager->jfd) || pPager->pInJournal==0
|| (sqlite3OsDeviceCharacteristics(pPager->fd)&SQLITE_IOCAP_BATCH_ATOMIC)
);
if( isOpen(pPager->jfd) ){
assert( !pagerUseWal(pPager) );
@ -4567,6 +4575,13 @@ static int pagerStress(void *p, PgHdr *pPg){
rc = pagerWalFrames(pPager, pPg, 0, 0);
}
}else{
#ifdef SQLITE_ENABLE_ATOMIC_WRITE
if( pPager->tempFile==0 ){
rc = sqlite3JournalCreate(pPager->jfd);
if( rc!=SQLITE_OK ) return pager_error(pPager, rc);
}
#endif
/* Sync the journal file if required. */
if( pPg->flags&PGHDR_NEED_SYNC
@ -6371,32 +6386,39 @@ int sqlite3PagerCommitPhaseOne(
** created for this transaction.
*/
#ifdef SQLITE_ENABLE_ATOMIC_WRITE
PgHdr *pPg;
assert( isOpen(pPager->jfd)
|| pPager->journalMode==PAGER_JOURNALMODE_OFF
|| pPager->journalMode==PAGER_JOURNALMODE_WAL
);
if( !zMaster && isOpen(pPager->jfd)
&& pPager->journalOff==jrnlBufferSize(pPager)
&& pPager->dbSize>=pPager->dbOrigSize
&& (0==(pPg = sqlite3PcacheDirtyList(pPager->pPCache)) || 0==pPg->pDirty)
){
/* Update the db file change counter via the direct-write method. The
** following call will modify the in-memory representation of page 1
** to include the updated change counter and then write page 1
** directly to the database file. Because of the atomic-write
** property of the host file-system, this is safe.
*/
rc = pager_incr_changecounter(pPager, 1);
}else{
rc = sqlite3JournalCreate(pPager->jfd);
if( rc==SQLITE_OK ){
rc = pager_incr_changecounter(pPager, 0);
sqlite3_file *fd = pPager->fd;
int bBatch = zMaster==0 /* An SQLITE_IOCAP_BATCH_ATOMIC commit */
&& (sqlite3OsDeviceCharacteristics(fd) & SQLITE_IOCAP_BATCH_ATOMIC)
&& pPager->journalMode!=PAGER_JOURNALMODE_MEMORY
&& sqlite3JournalIsInMemory(pPager->jfd);
if( bBatch==0 ){
PgHdr *pPg;
assert( isOpen(pPager->jfd)
|| pPager->journalMode==PAGER_JOURNALMODE_OFF
|| pPager->journalMode==PAGER_JOURNALMODE_WAL
);
if( !zMaster && isOpen(pPager->jfd)
&& pPager->journalOff==jrnlBufferSize(pPager)
&& pPager->dbSize>=pPager->dbOrigSize
&& (!(pPg = sqlite3PcacheDirtyList(pPager->pPCache)) || 0==pPg->pDirty)
){
/* Update the db file change counter via the direct-write method. The
** following call will modify the in-memory representation of page 1
** to include the updated change counter and then write page 1
** directly to the database file. Because of the atomic-write
** property of the host file-system, this is safe.
*/
rc = pager_incr_changecounter(pPager, 1);
}else{
rc = sqlite3JournalCreate(pPager->jfd);
if( rc==SQLITE_OK ){
rc = pager_incr_changecounter(pPager, 0);
}
}
}
#else
rc = pager_incr_changecounter(pPager, 0);
}else
#endif
rc = pager_incr_changecounter(pPager, 0);
if( rc!=SQLITE_OK ) goto commit_phase_one_exit;
/* Write the master journal name into the journal file. If a master
@ -6419,8 +6441,24 @@ int sqlite3PagerCommitPhaseOne(
*/
rc = syncJournal(pPager, 0);
if( rc!=SQLITE_OK ) goto commit_phase_one_exit;
if( bBatch ){
/* The pager is now in DBMOD state. But regardless of what happens
** next, attempting to play the journal back into the database would
** be unsafe. Close it now to make sure that does not happen. */
sqlite3OsClose(pPager->jfd);
rc = sqlite3OsFileControl(fd, SQLITE_FCNTL_BEGIN_ATOMIC_WRITE, 0);
if( rc!=SQLITE_OK ) goto commit_phase_one_exit;
}
rc = pager_write_pagelist(pPager,sqlite3PcacheDirtyList(pPager->pPCache));
if( bBatch ){
if( rc==SQLITE_OK ){
rc = sqlite3OsFileControl(fd, SQLITE_FCNTL_COMMIT_ATOMIC_WRITE, 0);
}else{
sqlite3OsFileControl(fd, SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE, 0);
}
}
if( rc!=SQLITE_OK ){
assert( rc!=SQLITE_IOERR_BLOCKED );
goto commit_phase_one_exit;

View File

@ -595,6 +595,7 @@ int sqlite3_exec(
#define SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN 0x00000800
#define SQLITE_IOCAP_POWERSAFE_OVERWRITE 0x00001000
#define SQLITE_IOCAP_IMMUTABLE 0x00002000
#define SQLITE_IOCAP_BATCH_ATOMIC 0x00004000
/*
** CAPI3REF: File Locking Levels
@ -729,6 +730,7 @@ struct sqlite3_file {
** <li> [SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN]
** <li> [SQLITE_IOCAP_POWERSAFE_OVERWRITE]
** <li> [SQLITE_IOCAP_IMMUTABLE]
** <li> [SQLITE_IOCAP_BATCH_ATOMIC]
** </ul>
**
** The SQLITE_IOCAP_ATOMIC property means that all writes of
@ -1044,6 +1046,10 @@ struct sqlite3_io_methods {
#define SQLITE_FCNTL_WIN32_GET_HANDLE 29
#define SQLITE_FCNTL_PDB 30
#define SQLITE_FCNTL_BEGIN_ATOMIC_WRITE 31
#define SQLITE_FCNTL_COMMIT_ATOMIC_WRITE 32
#define SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE 33
/* deprecated names */
#define SQLITE_GET_LOCKPROXYFILE SQLITE_FCNTL_GET_LOCKPROXYFILE
#define SQLITE_SET_LOCKPROXYFILE SQLITE_FCNTL_SET_LOCKPROXYFILE

View File

@ -61,7 +61,7 @@ foreach s {
fcntl read pread write pwrite fchmod fallocate
pread64 pwrite64 unlink openDirectory mkdir rmdir
statvfs fchown geteuid umask mmap munmap mremap
getpagesize readlink lstat
getpagesize readlink lstat ioctl
} {
if {[test_syscall exists $s]} {lappend syscall_list $s}
}