Add support for F2FS atomic writes. Untested at this point.

FossilOrigin-Name: 416973ede3bde8567d1f2699728f72352979e054ef988d1c1e1cfe4290f6f8b8

manifest

@@ -1,5 +1,5 @@
-C Enhance\sthe\sbuilt-in\sdate/time\sfunctions\sso\sthat\sthey\scan\sbe\sused\sin\nCHECK\sconstraints,\sin\sthe\sWHERE\sclause\sor\spartial\sindexes,\sand\sindex\nexpressions,\sprovided\sthat\snone\sof\sthe\snon-deterministic\skeywords\n("now",\s"localtime",\s"utc")\sare\sused\sas\sarguments.
+C Add\ssupport\sfor\sF2FS\satomic\swrites.\sUntested\sat\sthis\spoint.
-D 2017-07-20T15:08:43.378
+D 2017-07-20T19:49:14.731
 F Makefile.in d9873c9925917cca9990ee24be17eb9613a668012c85a343aef7e5536ae266e8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 02b469e9dcd5b7ee63fc1fb05babc174260ee4cfa4e0ef2e48c3c6801567a016
@@ -424,7 +424,7 @@ F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de
 F src/mem2.c f1940d9e91948dd6a908fbb9ce3835c36b5d83c3
 F src/mem3.c 8768ac94694f31ffaf8b4d0ea5dc08af7010a35a
 F src/mem5.c 9bf955937b07f8c32541c8a9991f33ce3173d944
-F src/memjournal.c 95752936c11dc6995672d1dd783cd633eea0cc95
+F src/memjournal.c 3f1d95947cd39b35a6375b2ca384bac904f16353bd54995e9bff78161463ac1c
 F src/msvc.h 4942752b6a253116baaa8de75256c51a459a5e81
 F src/mutex.c 8e45800ee78e0cd1f1f3fe8e398853307f4a085c
 F src/mutex.h 779d588e3b7756ec3ecf7d78cde1d84aba414f85
@@ -436,10 +436,10 @@ F src/os.c add02933b1dce7a39a005b00a2f5364b763e9a24
 F src/os.h 8e976e59eb4ca1c0fca6d35ee803e38951cb0343
 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85
 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
-F src/os_unix.c 30e2c43e4955db990e5b5a81e901f8aa74cc8820
+F src/os_unix.c eca7004f2cde959ed233951256e22aa4169c72f69b831e14276d0bdded4372f5
 F src/os_win.c 2a6c73eef01c51a048cc4ddccd57f981afbec18a
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
-F src/pager.c 14f6982c470c05b8e85575c69e9c1712010602e20400f8670d8699e21283e0e4
+F src/pager.c 7ab906445447d5374971986de3525350d18c0e256723cd5a2456a429cd372f6c
 F src/pager.h f2a99646c5533ffe11afa43e9e0bea74054e4efa
 F src/parse.y e384cb73f99e1b074085c974b37f4d830e885359e4b60837e30f7d67c16ba65b
 F src/pcache.c 62835bed959e2914edd26afadfecce29ece0e870
@@ -455,7 +455,7 @@ F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c c6bf96a7f9d7d68f929de84738c599a30d0a725ab0b54420e70545743cd5ee7b
 F src/shell.c bd6a37cbe8bf64ef6a6a74fdc50f067d3148149b4ce2b4d03154663e66ded55f
 F src/shell.c.in b5725acacba95ccefa57b6d068f710e29ba8239c3aa704628a1902a1f729c175
-F src/sqlite.h.in dad804d4e1979a2ddec33cc8da6aa50c04e6ba0dcb4058e7b3609588d010e041
+F src/sqlite.h.in 51541eacdbbd26afcb5933ea4fadf9952fa9b3e4d7277c2b7983bf2b4a09a0bf
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 967154985ed2ae62f90d9029bb5b5071793d847f1696a2ebe9e8cc0b042ae60b
 F src/sqliteInt.h 96197a18f041b9ab99e6cee0db39dbf771ac7762d9f0f63d9e719285f0478664
@@ -1232,7 +1232,7 @@ F test/superlock.test ec94f0556b6488d97f71c79f9061ae08d9ab8f12
 F test/symlink.test c9ebe7330d228249e447038276bfc8a7b22f4849
 F test/sync.test 2f84bdbc2b2df1fcb0220575b4b9f8cea94b7529
 F test/sync2.test 6be8ed007fa063b147773c1982b5bdba97a32badc536bdc6077eff5cf8710ece
-F test/syscall.test 7a60601770172a8014a4d222d5f3d95a5d2b5c47fbb0374e2698e89c99e37256
+F test/syscall.test a39d9a36f852ae6e4800f861bc2f2e83f68bbc2112d9399931ecfadeabd2d69d
 F test/sysfault.test c9f2b0d8d677558f74de750c75e12a5454719d04
 F test/tabfunc01.test c47171c36b3d411df2bd49719dcaa5d034f8d277477fd41d253940723b969a51
 F test/table.test b708f3e5fa2542fa51dfab21fc07b36ea445cb2f
@@ -1637,8 +1637,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P d14fc621e918915bbf8e04597eb238ea78dff3d9c5eb4402cb88692d00dbdfee b7f70c7fcabc10b8b3e62fe5ac68476cec23acaee037c7250ff70bca3f3ab541
+P a90c062d46c63a1e6f83064b1c5afb26a16e93b6ee8620ca46d169fdb325c488
-R 636bb2618e46215e27cfa1dc613ec36a
+R 83821cb44cbf5af73c5132cb66d20c9f
-T +closed b7f70c7fcabc10b8b3e62fe5ac68476cec23acaee037c7250ff70bca3f3ab541
+T *branch * f2fs-support
-U drh
+T *sym-f2fs-support *
-Z 8d21c041010891259262753e78bdd45f
+T -sym-trunk *
+U dan
+Z 66fb297e5afae668d4c6aa465d37e6d1

manifest.uuid

@@ -1 +1 @@
-a90c062d46c63a1e6f83064b1c5afb26a16e93b6ee8620ca46d169fdb325c488
+416973ede3bde8567d1f2699728f72352979e054ef988d1c1e1cfe4290f6f8b8

src/memjournal.c

@@ -388,13 +388,16 @@ void sqlite3MemJournalOpen(sqlite3_file *pJfd){
 /*
 ** If the argument p points to a MemJournal structure that is not an 
 ** in-memory-only journal file (i.e. is one that was opened with a +ve
-** nSpill parameter), and the underlying file has not yet been created, 
+** nSpill parameter or as SQLITE_OPEN_MAIN_JOURNAL), and the underlying 
-** create it now.
+** file has not yet been created, create it now.
 */
-int sqlite3JournalCreate(sqlite3_file *p){
+int sqlite3JournalCreate(sqlite3_file *pJfd){
   int rc = SQLITE_OK;
-  if( p->pMethods==&MemJournalMethods && ((MemJournal*)p)->nSpill>0 ){
+  MemJournal *p = (MemJournal*)pJfd;
-    rc = memjrnlCreateFile((MemJournal*)p);
+  if( p->pMethod==&MemJournalMethods 
+   && (p->nSpill>0 || (p->flags & SQLITE_OPEN_MAIN_JOURNAL))
+  ){
+    rc = memjrnlCreateFile(p);
   }
   return rc;
 }

src/os_unix.c

@@ -90,6 +90,7 @@
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <fcntl.h>
+#include <sys/ioctl.h>
 #include <unistd.h>
 #include <time.h>
 #include <sys/time.h>
@@ -220,10 +221,8 @@ struct unixFile {
   sqlite3_int64 mmapSizeMax;          /* Configured FCNTL_MMAP_SIZE value */
   void *pMapRegion;                   /* Memory mapped region */
 #endif
-#ifdef __QNXNTO__
   int sectorSize;                     /* Device sector size */
   int deviceCharacteristics;          /* Precomputed device characteristics */
-#endif
 #if SQLITE_ENABLE_LOCKING_STYLE
   int openFlags;                      /* The flags specified at open() */
 #endif
@@ -328,6 +327,13 @@ static pid_t randomnessPid = 0;
 # define lseek lseek64
 #endif
 
+#define F2FS_IOCTL_MAGIC        0xf5
+#define F2FS_IOC_START_ATOMIC_WRITE     _IO(F2FS_IOCTL_MAGIC, 1)
+#define F2FS_IOC_COMMIT_ATOMIC_WRITE    _IO(F2FS_IOCTL_MAGIC, 2)
+#define F2FS_IOC_START_VOLATILE_WRITE   _IO(F2FS_IOCTL_MAGIC, 3)
+#define F2FS_IOC_ABORT_VOLATILE_WRITE   _IO(F2FS_IOCTL_MAGIC, 5)
+
+
 /*
 ** Different Unix systems declare open() in different ways.  Same use
 ** open(const char*,int,mode_t).  Others use open(const char*,int,...).
@@ -500,6 +506,9 @@ static struct unix_syscall {
 #endif
 #define osLstat      ((int(*)(const char*,struct stat*))aSyscall[27].pCurrent)
 
+  { "ioctl",         (sqlite3_syscall_ptr)ioctl,          0 },
+#define osIoctl ((int(*)(int,int))aSyscall[28].pCurrent)
+
 }; /* End of the overrideable system calls */
 
 
@@ -3777,6 +3786,19 @@ static int unixGetTempname(int nBuf, char *zBuf);
 static int unixFileControl(sqlite3_file *id, int op, void *pArg){
   unixFile *pFile = (unixFile*)id;
   switch( op ){
+    case SQLITE_FCNTL_BEGIN_ATOMIC_WRITE: {
+      int rc = osIoctl(pFile->h, F2FS_IOC_START_ATOMIC_WRITE);
+      return rc ? SQLITE_ERROR : SQLITE_OK;
+    }
+    case SQLITE_FCNTL_COMMIT_ATOMIC_WRITE: {
+      int rc = osIoctl(pFile->h, F2FS_IOC_COMMIT_ATOMIC_WRITE);
+      return rc ? SQLITE_ERROR : SQLITE_OK;
+    }
+    case SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE: {
+      int rc = osIoctl(pFile->h, F2FS_IOC_ABORT_VOLATILE_WRITE);
+      return rc ? SQLITE_ERROR : SQLITE_OK;
+    }
+
     case SQLITE_FCNTL_LOCKSTATE: {
       *(int*)pArg = pFile->eFileLock;
       return SQLITE_OK;
@@ -3860,30 +3882,43 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){
 }
 
 /*
-** Return the sector size in bytes of the underlying block device for
+** If pFd->sectorSize is non-zero when this function is called, it is a
-** the specified file. This is almost always 512 bytes, but may be
+** no-op. Otherwise, the values of pFd->sectorSize and 
-** larger for some devices.
+** pFd->deviceCharacteristics are set according to the file-system 
+** characteristics. 
 **
-** SQLite code assumes this function cannot fail. It also assumes that
+** There are two versions of this function. One for QNX and one for all
-** if two files are created in the same file-system directory (i.e.
+** other systems.
-** a database and its journal file) that the sector size will be the
-** same for both.
 */
 #ifndef __QNXNTO__
-static int unixSectorSize(sqlite3_file *NotUsed){
+static void setDeviceCharacteristics(unixFile *pFd){
-  UNUSED_PARAMETER(NotUsed);
+  if( pFd->sectorSize==0 ){
-  return SQLITE_DEFAULT_SECTOR_SIZE;
+    int res;
-}
+    assert( pFd->deviceCharacteristics==0 );
-#endif
 
-/*
+    /* Check for support for F2FS atomic batch writes. */
-** The following version of unixSectorSize() is optimized for QNX.
+    res = osIoctl(pFd->h, F2FS_IOC_START_VOLATILE_WRITE);
-*/
+    if( res==SQLITE_OK ){
-#ifdef __QNXNTO__
+      osIoctl(pFd->h, F2FS_IOC_ABORT_VOLATILE_WRITE);
+      pFd->deviceCharacteristics = 
+        SQLITE_IOCAP_BATCH_ATOMIC |
+        SQLITE_IOCAP_ATOMIC |
+        SQLITE_IOCAP_SEQUENTIAL |
+        SQLITE_IOCAP_SAFE_APPEND;
+    }
+
+    /* Set the POWERSAFE_OVERWRITE flag if requested. */
+    if( pFd->ctrlFlags & UNIXFILE_PSOW ){
+      pFd->deviceCharacteristics |= SQLITE_IOCAP_POWERSAFE_OVERWRITE;
+    }
+
+    pFd->sectorSize = SQLITE_DEFAULT_SECTOR_SIZE;
+  }
+}
+#else
 #include <sys/dcmd_blk.h>
 #include <sys/statvfs.h>
-static int unixSectorSize(sqlite3_file *id){
+static void setDeviceCharacteristics(unixFile *pFile){
-  unixFile *pFile = (unixFile*)id;
   if( pFile->sectorSize == 0 ){
     struct statvfs fsInfo;
        
@@ -3952,9 +3987,24 @@ static int unixSectorSize(sqlite3_file *id){
     pFile->deviceCharacteristics = 0;
     pFile->sectorSize = SQLITE_DEFAULT_SECTOR_SIZE;
   }
-  return pFile->sectorSize;
 }
-#endif /* __QNXNTO__ */
+#endif
+
+/*
+** Return the sector size in bytes of the underlying block device for
+** the specified file. This is almost always 512 bytes, but may be
+** larger for some devices.
+**
+** SQLite code assumes this function cannot fail. It also assumes that
+** if two files are created in the same file-system directory (i.e.
+** a database and its journal file) that the sector size will be the
+** same for both.
+*/
+static int unixSectorSize(sqlite3_file *id){
+  unixFile *pFd = (unixFile*)id;
+  setDeviceCharacteristics(pFd);
+  return pFd->sectorSize;
+}
 
 /*
 ** Return the device characteristics for the file.
@@ -3970,16 +4020,9 @@ static int unixSectorSize(sqlite3_file *id){
 ** available to turn it off and URI query parameter available to turn it off.
 */
 static int unixDeviceCharacteristics(sqlite3_file *id){
-  unixFile *p = (unixFile*)id;
+  unixFile *pFd = (unixFile*)id;
-  int rc = 0;
+  setDeviceCharacteristics(pFd);
-#ifdef __QNXNTO__
+  return pFd->deviceCharacteristics;
-  if( p->sectorSize==0 ) unixSectorSize(id);
-  rc = p->deviceCharacteristics;
-#endif
-  if( p->ctrlFlags & UNIXFILE_PSOW ){
-    rc |= SQLITE_IOCAP_POWERSAFE_OVERWRITE;
-  }
-  return rc;
 }
 
 #if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0
@@ -7598,7 +7641,7 @@ int sqlite3_os_init(void){
 
   /* Double-check that the aSyscall[] array has been constructed
   ** correctly.  See ticket [bb3a86e890c8e96ab] */
-  assert( ArraySize(aSyscall)==28 );
+  assert( ArraySize(aSyscall)==29 );
 
   /* Register all VFSes defined in the aVfs[] array */
   for(i=0; i<(sizeof(aVfs)/sizeof(sqlite3_vfs)); i++){

src/pager.c

@@ -958,6 +958,7 @@ static int assert_pager_state(Pager *p){
       assert( isOpen(p->jfd) 
            || p->journalMode==PAGER_JOURNALMODE_OFF 
            || p->journalMode==PAGER_JOURNALMODE_WAL 
+           || (sqlite3OsDeviceCharacteristics(p->fd)&SQLITE_IOCAP_BATCH_ATOMIC)
       );
       break;
 
@@ -1194,6 +1195,11 @@ static int jrnlBufferSize(Pager *pPager){
 
     assert( isOpen(pPager->fd) );
     dc = sqlite3OsDeviceCharacteristics(pPager->fd);
+    /* use in-memory journal */
+    if( dc&SQLITE_IOCAP_BATCH_ATOMIC ){
+      return -1;
+    }
+
     nSector = pPager->sectorSize;
     szPage = pPager->pageSize;
 
@@ -2012,7 +2018,9 @@ static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){
   }
 
   releaseAllSavepoints(pPager);
-  assert( isOpen(pPager->jfd) || pPager->pInJournal==0 );
+  assert( isOpen(pPager->jfd) || pPager->pInJournal==0 
+      || (sqlite3OsDeviceCharacteristics(pPager->fd)&SQLITE_IOCAP_BATCH_ATOMIC)
+  );
   if( isOpen(pPager->jfd) ){
     assert( !pagerUseWal(pPager) );
 
@@ -4568,6 +4576,13 @@ static int pagerStress(void *p, PgHdr *pPg){
     }
   }else{
     
+#ifdef SQLITE_ENABLE_ATOMIC_WRITE
+    if( pPager->tempFile==0 ){
+      rc = sqlite3JournalCreate(pPager->jfd);
+      if( rc!=SQLITE_OK ) return pager_error(pPager, rc);
+    }
+#endif
+  
     /* Sync the journal file if required. */
     if( pPg->flags&PGHDR_NEED_SYNC 
      || pPager->eState==PAGER_WRITER_CACHEMOD
@@ -6371,6 +6386,13 @@ int sqlite3PagerCommitPhaseOne(
       ** created for this transaction.
       */
   #ifdef SQLITE_ENABLE_ATOMIC_WRITE
+      sqlite3_file *fd = pPager->fd;
+      int bBatch = zMaster==0      /* An SQLITE_IOCAP_BATCH_ATOMIC commit */
+        && (sqlite3OsDeviceCharacteristics(fd) & SQLITE_IOCAP_BATCH_ATOMIC)
+        && pPager->journalMode!=PAGER_JOURNALMODE_MEMORY
+        && sqlite3JournalIsInMemory(pPager->jfd);
+
+      if( bBatch==0 ){
         PgHdr *pPg;
         assert( isOpen(pPager->jfd) 
             || pPager->journalMode==PAGER_JOURNALMODE_OFF 
@@ -6379,7 +6401,7 @@ int sqlite3PagerCommitPhaseOne(
         if( !zMaster && isOpen(pPager->jfd) 
          && pPager->journalOff==jrnlBufferSize(pPager) 
          && pPager->dbSize>=pPager->dbOrigSize
-       && (0==(pPg = sqlite3PcacheDirtyList(pPager->pPCache)) || 0==pPg->pDirty)
+         && (!(pPg = sqlite3PcacheDirtyList(pPager->pPCache)) || 0==pPg->pDirty)
         ){
           /* Update the db file change counter via the direct-write method. The 
           ** following call will modify the in-memory representation of page 1 
@@ -6394,9 +6416,9 @@ int sqlite3PagerCommitPhaseOne(
             rc = pager_incr_changecounter(pPager, 0);
           }
         }
-  #else
+      }else
-      rc = pager_incr_changecounter(pPager, 0);
   #endif
+      rc = pager_incr_changecounter(pPager, 0);
       if( rc!=SQLITE_OK ) goto commit_phase_one_exit;
   
       /* Write the master journal name into the journal file. If a master 
@@ -6420,7 +6442,23 @@ int sqlite3PagerCommitPhaseOne(
       rc = syncJournal(pPager, 0);
       if( rc!=SQLITE_OK ) goto commit_phase_one_exit;
 
+      if( bBatch ){
+        /* The pager is now in DBMOD state. But regardless of what happens
+        ** next, attempting to play the journal back into the database would
+        ** be unsafe. Close it now to make sure that does not happen.  */
+        sqlite3OsClose(pPager->jfd);
+        rc = sqlite3OsFileControl(fd, SQLITE_FCNTL_BEGIN_ATOMIC_WRITE, 0);
+        if( rc!=SQLITE_OK ) goto commit_phase_one_exit;
+      }
       rc = pager_write_pagelist(pPager,sqlite3PcacheDirtyList(pPager->pPCache));
+      if( bBatch ){
+        if( rc==SQLITE_OK ){
+          rc = sqlite3OsFileControl(fd, SQLITE_FCNTL_COMMIT_ATOMIC_WRITE, 0);
+        }else{
+          sqlite3OsFileControl(fd, SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE, 0);
+        }
+      }
+
       if( rc!=SQLITE_OK ){
         assert( rc!=SQLITE_IOERR_BLOCKED );
         goto commit_phase_one_exit;

src/sqlite.h.in

@@ -595,6 +595,7 @@ int sqlite3_exec(
 #define SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN  0x00000800
 #define SQLITE_IOCAP_POWERSAFE_OVERWRITE    0x00001000
 #define SQLITE_IOCAP_IMMUTABLE              0x00002000
+#define SQLITE_IOCAP_BATCH_ATOMIC           0x00004000
 
 /*
 ** CAPI3REF: File Locking Levels
@@ -729,6 +730,7 @@ struct sqlite3_file {
 ** <li> [SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN]
 ** <li> [SQLITE_IOCAP_POWERSAFE_OVERWRITE]
 ** <li> [SQLITE_IOCAP_IMMUTABLE]
+** <li> [SQLITE_IOCAP_BATCH_ATOMIC]
 ** </ul>
 **
 ** The SQLITE_IOCAP_ATOMIC property means that all writes of
@@ -1044,6 +1046,10 @@ struct sqlite3_io_methods {
 #define SQLITE_FCNTL_WIN32_GET_HANDLE       29
 #define SQLITE_FCNTL_PDB                    30
 
+#define SQLITE_FCNTL_BEGIN_ATOMIC_WRITE     31
+#define SQLITE_FCNTL_COMMIT_ATOMIC_WRITE    32
+#define SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE  33
+
 /* deprecated names */
 #define SQLITE_GET_LOCKPROXYFILE      SQLITE_FCNTL_GET_LOCKPROXYFILE
 #define SQLITE_SET_LOCKPROXYFILE      SQLITE_FCNTL_SET_LOCKPROXYFILE

test/syscall.test

@@ -61,7 +61,7 @@ foreach s {
     fcntl read pread write pwrite fchmod fallocate
     pread64 pwrite64 unlink openDirectory mkdir rmdir 
     statvfs fchown geteuid umask mmap munmap mremap
-    getpagesize readlink lstat
+    getpagesize readlink lstat ioctl
 } {
   if {[test_syscall exists $s]} {lappend syscall_list $s}
 }