Update ext/misc/zipfile.c to support creating and adding entries to existing

zip archives.

FossilOrigin-Name: 2dec2dec592c0726ebe87b841b9c8d493dea7074a99f278eb1bf0b744d658a9d

ext/misc/zipfile.c

@@ -26,6 +26,8 @@ SQLITE_EXTENSION_INIT1
 #include <utime.h>
 #include <errno.h>
 
+#include <zlib.h>
+
 #ifndef SQLITE_OMIT_VIRTUALTABLE
 
 #ifndef SQLITE_AMALGAMATION
@@ -47,11 +49,40 @@ typedef unsigned long u32;
 ");"
 
 #define ZIPFILE_F_COLUMN_IDX 6    /* Index of column "f" in the above */
-
 #define ZIPFILE_BUFFER_SIZE (64*1024)
 
 
-#define ZIPFILE_EXTRA_TIMESTAMP 0x5455
+/*
+** Magic numbers used to read and write zip files.
+**
+** ZIPFILE_NEWENTRY_MADEBY:
+**   Use this value for the "version-made-by" field in new zip file
+**   entries. The upper byte indicates "unix", and the lower byte 
+**   indicates that the zip file matches pkzip specification 3.0. 
+**   This is what info-zip seems to do.
+**
+** ZIPFILE_NEWENTRY_REQUIRED:
+**   Value for "version-required-to-extract" field of new entries.
+**   Version 2.0 is required to support folders and deflate compression.
+**
+** ZIPFILE_NEWENTRY_FLAGS:
+**   Value for "general-purpose-bit-flags" field of new entries. Bit
+**   11 means "utf-8 filename and comment".
+**
+** ZIPFILE_SIGNATURE_CDS:
+**   First 4 bytes of a valid CDS record.
+**
+** ZIPFILE_SIGNATURE_LFH:
+**   First 4 bytes of a valid LFH record.
+*/
+#define ZIPFILE_EXTRA_TIMESTAMP   0x5455
+#define ZIPFILE_NEWENTRY_MADEBY   ((3<<8) + 30)
+#define ZIPFILE_NEWENTRY_REQUIRED 20
+#define ZIPFILE_NEWENTRY_FLAGS    0x800
+#define ZIPFILE_SIGNATURE_CDS     0x02014b50
+#define ZIPFILE_SIGNATURE_LFH     0x04034b50
+#define ZIPFILE_SIGNATURE_EOCD    0x06054b50
+#define ZIPFILE_LFH_FIXED_SZ      30
 
 /*
 ** Set the error message contained in context ctx to the results of
@@ -188,18 +219,58 @@ struct ZipfileCsr {
   i64 iDataOff;              /* Offset in zipfile to data */
   u32 mTime;                 /* Extended mtime value */
   int flags;
-  u8 *aBuffer;               /* Buffer used for various tasks */
 };
 
+/*
+** Values for ZipfileCsr.flags.
+*/
 #define ZIPFILE_MTIME_VALID 0x0001
 
+typedef struct ZipfileEntry ZipfileEntry;
+struct ZipfileEntry {
+  char *zPath;               /* Path of zipfile entry */
+  u8 *aCdsEntry;             /* Buffer containing entire CDS entry */
+  int nCdsEntry;             /* Size of buffer aCdsEntry[] in bytes */
+  ZipfileEntry *pNext;
+};
+
 typedef struct ZipfileTab ZipfileTab;
 struct ZipfileTab {
   sqlite3_vtab base;         /* Base class - must be first */
+  char *zFile;               /* Zip file this table accesses (may be NULL) */
+  u8 *aBuffer;               /* Temporary buffer used for various tasks */
+
+  /* The following are used by write transactions only */
+  ZipfileEntry *pEntry;      /* Linked list of all files (if pWriteFd!=0) */
+  FILE *pWriteFd;            /* File handle open on zip archive */
+  i64 szCurrent;             /* Current size of zip archive */
+  i64 szOrig;                /* Size of archive at start of transaction */
 };
 
+static void zipfileDequote(char *zIn){
+  char q = zIn[0];
+  if( q=='"' || q=='\'' || q=='`' || q=='[' ){
+    char c;
+    int iIn = 1;
+    int iOut = 0;
+    if( q=='[' ) q = ']';
+    while( (c = zIn[iIn++]) ){
+      if( c==q ){
+        if( zIn[iIn++]!=q ) break;
+      }
+      zIn[iOut++] = c;
+    }
+    zIn[iOut] = '\0';
+  }
+}
+
 /*
 ** Construct a new ZipfileTab virtual table object.
+** 
+**   argv[0]   -> module name  ("zipfile")
+**   argv[1]   -> database name
+**   argv[2]   -> table name
+**   argv[...] -> "column name" and other module argument fields.
 */
 static int zipfileConnect(
   sqlite3 *db,
@@ -208,14 +279,28 @@ static int zipfileConnect(
   sqlite3_vtab **ppVtab,
   char **pzErr
 ){
+  int nByte = sizeof(ZipfileTab) + ZIPFILE_BUFFER_SIZE;
+  int nFile = 0;
+  const char *zFile = 0;
   ZipfileTab *pNew = 0;
   int rc;
 
+  if( argc>3 ){
+    zFile = argv[3];
+    nFile = strlen(zFile)+1;
+  }
+
   rc = sqlite3_declare_vtab(db, ZIPFILE_SCHEMA);
   if( rc==SQLITE_OK ){
-    pNew = (ZipfileTab*)sqlite3_malloc( sizeof(*pNew) );
+    pNew = (ZipfileTab*)sqlite3_malloc(nByte+nFile);
     if( pNew==0 ) return SQLITE_NOMEM;
-    memset(pNew, 0, sizeof(*pNew));
+    memset(pNew, 0, nByte+nFile);
+    pNew->aBuffer = (u8*)&pNew[1];
+    if( zFile ){
+      pNew->zFile = (char*)&pNew->aBuffer[ZIPFILE_BUFFER_SIZE];
+      memcpy(pNew->zFile, zFile, nFile);
+      zipfileDequote(pNew->zFile);
+    }
   }
   *ppVtab = (sqlite3_vtab*)pNew;
   return rc;
@@ -234,11 +319,12 @@ static int zipfileDisconnect(sqlite3_vtab *pVtab){
 */
 static int zipfileOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCsr){
   ZipfileCsr *pCsr;
-  pCsr = sqlite3_malloc( sizeof(*pCsr) + ZIPFILE_BUFFER_SIZE);
+  pCsr = sqlite3_malloc(sizeof(*pCsr));
-  if( pCsr==0 ) return SQLITE_NOMEM;
+  *ppCsr = (sqlite3_vtab_cursor*)pCsr;
+  if( pCsr==0 ){
+    return SQLITE_NOMEM;
+  }
   memset(pCsr, 0, sizeof(*pCsr));
-  pCsr->aBuffer = (u8*)&pCsr[1];
-  *ppCsr = &pCsr->base;
   return SQLITE_OK;
 }
 
@@ -247,6 +333,8 @@ static int zipfileOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCsr){
 ** by zipfileOpen().
 */
 static void zipfileResetCursor(ZipfileCsr *pCsr){
+  sqlite3_free(pCsr->cds.zFile);
+  pCsr->cds.zFile = 0;
   pCsr->iRowid = 0;
   pCsr->bEof = 0;
   if( pCsr->pFile ){
@@ -276,17 +364,39 @@ static void zipfileSetErrmsg(ZipfileCsr *pCsr, const char *zFmt, ...){
   va_end(ap);
 }
 
-static int zipfileReadData(ZipfileCsr *pCsr, u8 *aRead, int nRead, i64 iOff){
+static int zipfileReadData(
+  FILE *pFile,                    /* Read from this file */
+  u8 *aRead,                      /* Read into this buffer */
+  int nRead,                      /* Number of bytes to read */
+  i64 iOff,                       /* Offset to read from */
+  char **pzErrmsg                 /* OUT: Error message (from sqlite3_malloc) */
+){
   size_t n;
-  fseek(pCsr->pFile, iOff, SEEK_SET);
+  fseek(pFile, iOff, SEEK_SET);
-  n = fread(aRead, 1, nRead, pCsr->pFile);
+  n = fread(aRead, 1, nRead, pFile);
   if( n!=nRead ){
-    zipfileSetErrmsg(pCsr, "error in fread()");
+    *pzErrmsg = sqlite3_mprintf("error in fread()");
     return SQLITE_ERROR;
   }
   return SQLITE_OK;
 }
 
+static int zipfileAppendData(
+  ZipfileTab *pTab,
+  const u8 *aWrite,
+  int nWrite
+){
+  size_t n;
+  fseek(pTab->pWriteFd, pTab->szCurrent, SEEK_SET);
+  n = fwrite(aWrite, 1, nWrite, pTab->pWriteFd);
+  if( n!=nWrite ){
+    pTab->base.zErrMsg = sqlite3_mprintf("error in fwrite()");
+    return SQLITE_ERROR;
+  }
+  pTab->szCurrent += nWrite;
+  return SQLITE_OK;
+}
+
 static u16 zipfileGetU16(const u8 *aBuf){
   return (aBuf[1] << 8) + aBuf[0];
 }
@@ -297,18 +407,47 @@ static u32 zipfileGetU32(const u8 *aBuf){
        + ((u32)(aBuf[0]) <<  0);
 }
 
+static void zipfilePutU16(u8 *aBuf, u16 val){
+  aBuf[0] = val & 0xFF;
+  aBuf[1] = (val>>8) & 0xFF;
+}
+static void zipfilePutU32(u8 *aBuf, u32 val){
+  aBuf[0] = val & 0xFF;
+  aBuf[1] = (val>>8) & 0xFF;
+  aBuf[2] = (val>>16) & 0xFF;
+  aBuf[3] = (val>>24) & 0xFF;
+}
+
 #define zipfileRead32(aBuf) ( aBuf+=4, zipfileGetU32(aBuf-4) )
 #define zipfileRead16(aBuf) ( aBuf+=2, zipfileGetU16(aBuf-2) )
 
+#define zipfileWrite32(aBuf,val) { zipfilePutU32(aBuf,val); aBuf+=4; }
+#define zipfileWrite16(aBuf,val) { zipfilePutU16(aBuf,val); aBuf+=2; }
+
+static u8* zipfileCsrBuffer(ZipfileCsr *pCsr){
+  return ((ZipfileTab*)(pCsr->base.pVtab))->aBuffer;
+}
+
+/*
+** Magic numbers used to read CDS records.
+*/
+#define ZIPFILE_CDS_FIXED_SZ  46
+#define ZIPFILE_CDS_NFILE_OFF 28
+
 static int zipfileReadCDS(ZipfileCsr *pCsr){
-  static const int szFix = 46;    /* Size of fixed-size part of CDS */
+  char **pzErr = &pCsr->base.pVtab->zErrMsg;
-  u8 *aRead = pCsr->aBuffer;
+  u8 *aRead = zipfileCsrBuffer(pCsr);
   int rc;
 
-  rc = zipfileReadData(pCsr, aRead, szFix, pCsr->iNextOff);
+  sqlite3_free(pCsr->cds.zFile);
+  pCsr->cds.zFile = 0;
+
+  rc = zipfileReadData(
+      pCsr->pFile, aRead, ZIPFILE_CDS_FIXED_SZ, pCsr->iNextOff, pzErr
+  );
   if( rc==SQLITE_OK ){
     u32 sig = zipfileRead32(aRead);
-    if( sig!=0x02014b50 ){
+    if( sig!=ZIPFILE_SIGNATURE_CDS ){
       zipfileSetErrmsg(pCsr,"failed to read CDS at offset %lld",pCsr->iNextOff);
       rc = SQLITE_ERROR;
     }else{
@@ -322,6 +461,7 @@ static int zipfileReadCDS(ZipfileCsr *pCsr){
       pCsr->cds.crc32 = zipfileRead32(aRead);
       pCsr->cds.szCompressed = zipfileRead32(aRead);
       pCsr->cds.szUncompressed = zipfileRead32(aRead);
+      assert( aRead==zipfileCsrBuffer(pCsr)+ZIPFILE_CDS_NFILE_OFF );
       pCsr->cds.nFile = zipfileRead16(aRead);
       pCsr->cds.nExtra = zipfileRead16(aRead);
       pCsr->cds.nComment = zipfileRead16(aRead);
@@ -330,15 +470,15 @@ static int zipfileReadCDS(ZipfileCsr *pCsr){
       pCsr->cds.iExternalAttr = zipfileRead32(aRead);
       pCsr->cds.iOffset = zipfileRead32(aRead);
 
-      assert( aRead==&pCsr->aBuffer[szFix] );
+      assert( aRead==zipfileCsrBuffer(pCsr)+ZIPFILE_CDS_FIXED_SZ );
 
       nRead = pCsr->cds.nFile + pCsr->cds.nExtra;
-      aRead = pCsr->aBuffer;
+      aRead = zipfileCsrBuffer(pCsr);
-      rc = zipfileReadData(pCsr, aRead, nRead, pCsr->iNextOff+szFix);
+      pCsr->iNextOff += ZIPFILE_CDS_FIXED_SZ;
+      rc = zipfileReadData(pCsr->pFile, aRead, nRead, pCsr->iNextOff, pzErr);
 
       if( rc==SQLITE_OK ){
         pCsr->cds.zFile = sqlite3_mprintf("%.*s", (int)pCsr->cds.nFile, aRead);
-        pCsr->iNextOff += szFix;
         pCsr->iNextOff += pCsr->cds.nFile;
         pCsr->iNextOff += pCsr->cds.nExtra;
         pCsr->iNextOff += pCsr->cds.nComment;
@@ -381,14 +521,15 @@ static int zipfileReadCDS(ZipfileCsr *pCsr){
 }
 
 static int zipfileReadLFH(ZipfileCsr *pCsr){
-  static const int szFix = 30;    /* Size of fixed-size part of LFH */
+  char **pzErr = &pCsr->base.pVtab->zErrMsg;
-  u8 *aRead = pCsr->aBuffer;
+  static const int szFix = ZIPFILE_LFH_FIXED_SZ;
+  u8 *aRead = zipfileCsrBuffer(pCsr);
   int rc;
 
-  rc = zipfileReadData(pCsr, aRead, szFix, pCsr->cds.iOffset);
+  rc = zipfileReadData(pCsr->pFile, aRead, szFix, pCsr->cds.iOffset, pzErr);
   if( rc==SQLITE_OK ){
     u32 sig = zipfileRead32(aRead);
-    if( sig!=0x04034b50 ){
+    if( sig!=ZIPFILE_SIGNATURE_LFH ){
       zipfileSetErrmsg(pCsr, "failed to read LFH at offset %d", 
           (int)pCsr->cds.iOffset
       );
@@ -404,7 +545,7 @@ static int zipfileReadLFH(ZipfileCsr *pCsr){
       pCsr->lfh.szUncompressed = zipfileRead32(aRead);
       pCsr->lfh.nFile = zipfileRead16(aRead);
       pCsr->lfh.nExtra = zipfileRead16(aRead);
-      assert( aRead==&pCsr->aBuffer[szFix] );
+      assert( aRead==zipfileCsrBuffer(pCsr)+szFix );
       pCsr->iDataOff = pCsr->cds.iOffset+szFix+pCsr->lfh.nFile+pCsr->lfh.nExtra;
     }
   }
@@ -460,6 +601,22 @@ static time_t zipfileMtime(ZipfileCsr *pCsr){
   return mktime(&t);
 }
 
+static void zipfileMtimeToDos(ZipfileCDS *pCds, u32 mTime){
+  time_t t = (time_t)mTime;
+  struct tm res;
+  localtime_r(&t, &res);
+
+  pCds->mTime = 
+    (res.tm_sec / 2) + 
+    (res.tm_min << 5) +
+    (res.tm_hour << 11);
+
+  pCds->mDate = 
+    (res.tm_mday-1) +
+    ((res.tm_mon+1) << 5) +
+    ((res.tm_year-80) << 9);
+}
+
 /*
 ** Return values of columns for the row at which the series_cursor
 ** is currently pointing.
@@ -499,7 +656,9 @@ static int zipfileColumn(
         if( aBuf==0 ){
           rc = SQLITE_NOMEM;
         }else{
-          rc = zipfileReadData(pCsr, aBuf, sz, pCsr->iDataOff);
+          rc = zipfileReadData(pCsr->pFile, aBuf, sz, pCsr->iDataOff,
+              &pCsr->base.pVtab->zErrMsg
+          );
         }
         if( rc==SQLITE_OK ){
           sqlite3_result_blob(ctx, aBuf, sz, SQLITE_TRANSIENT);
@@ -537,17 +696,26 @@ static int zipfileEof(sqlite3_vtab_cursor *cur){
 }
 
 /*
-** The zip file has been successfully opened (so pCsr->pFile is valid). 
-** This function attempts to locate and read the End of central
-** directory record from the file.
-**
 */
-static int zipfileReadEOCD(ZipfileCsr *pCsr, ZipfileEOCD *pEOCD){
+static int zipfileReadEOCD(
-  u8 *aRead = pCsr->aBuffer;
+  ZipfileTab *pTab,               /* Return errors here */
-  int nRead = (int)(MIN(pCsr->nByte, ZIPFILE_BUFFER_SIZE));
+  FILE *pFile,                    /* Read from this file */
-  i64 iOff = pCsr->nByte - nRead;
+  ZipfileEOCD *pEOCD              /* Object to populate */
+){
+  u8 *aRead = pTab->aBuffer;      /* Temporary buffer */
+  i64 szFile;                     /* Total size of file in bytes */
+  int nRead;                      /* Bytes to read from file */
+  i64 iOff;                       /* Offset to read from */
 
-  int rc = zipfileReadData(pCsr, aRead, nRead, iOff);
+  fseek(pFile, 0, SEEK_END);
+  szFile = (i64)ftell(pFile);
+  if( szFile==0 ){
+    return SQLITE_EMPTY;
+  }
+  nRead = (int)(MIN(szFile, ZIPFILE_BUFFER_SIZE));
+  iOff = szFile - nRead;
+
+  int rc = zipfileReadData(pFile, aRead, nRead, iOff, &pTab->base.zErrMsg);
   if( rc==SQLITE_OK ){
     int i;
 
@@ -560,7 +728,9 @@ static int zipfileReadEOCD(ZipfileCsr *pCsr, ZipfileEOCD *pEOCD){
       }
     }
     if( i<0 ){
-      zipfileSetErrmsg(pCsr, "cannot find end of central directory record");
+      pTab->base.zErrMsg = sqlite3_mprintf(
+          "cannot find end of central directory record"
+      );
       return SQLITE_ERROR;
     }
 
@@ -592,6 +762,7 @@ static int zipfileFilter(
   int idxNum, const char *idxStr,
   int argc, sqlite3_value **argv
 ){
+  ZipfileTab *pTab = (ZipfileTab*)cur->pVtab;
   ZipfileCsr *pCsr = (ZipfileCsr*)cur;
   const char *zFile;              /* Zip file to scan */
   int rc = SQLITE_OK;             /* Return Code */
@@ -600,23 +771,29 @@ static int zipfileFilter(
 
   assert( idxNum==argc && (idxNum==0 || idxNum==1) );
   if( idxNum==0 ){
-    /* Error. User did not supply a file name. */
+    ZipfileTab *pTab = (ZipfileTab*)cur->pVtab;
-    zipfileSetErrmsg(pCsr, "table function zipfile() requires an argument");
+    zFile = pTab->zFile;
-    return SQLITE_ERROR;
+    if( zFile==0 ){
+      /* Error. This is an eponymous virtual table and the user has not 
+      ** supplied a file name. */
+      zipfileSetErrmsg(pCsr, "table function zipfile() requires an argument");
+      return SQLITE_ERROR;
+    }
+  }else{
+    zFile = (const char*)sqlite3_value_text(argv[0]);
   }
-
-  zFile = sqlite3_value_text(argv[0]);
   pCsr->pFile = fopen(zFile, "rb");
   if( pCsr->pFile==0 ){
     zipfileSetErrmsg(pCsr, "cannot open file: %s", zFile);
     rc = SQLITE_ERROR;
   }else{
-    fseek(pCsr->pFile, 0, SEEK_END);
+    rc = zipfileReadEOCD(pTab, pCsr->pFile, &pCsr->eocd);
-    pCsr->nByte = (i64)ftell(pCsr->pFile);
-    rc = zipfileReadEOCD(pCsr, &pCsr->eocd);
     if( rc==SQLITE_OK ){
       pCsr->iNextOff = pCsr->eocd.iOffset;
       rc = zipfileNext(cur);
+    }else if( rc==SQLITE_EMPTY ){
+      rc = SQLITE_OK;
+      pCsr->bEof = 1;
     }
   }
 
@@ -653,17 +830,404 @@ static int zipfileBestIndex(
   return SQLITE_OK;
 }
 
+static int zipfileLoadDirectory(ZipfileTab *pTab){
+  ZipfileEOCD eocd;
+  int rc;
+
+  rc = zipfileReadEOCD(pTab, pTab->pWriteFd, &eocd);
+  if( rc==SQLITE_OK ){
+    int i;
+    int iOff = 0;
+    u8 *aBuf = sqlite3_malloc(eocd.nSize);
+    if( aBuf==0 ){
+      rc = SQLITE_NOMEM;
+    }else{
+      rc = zipfileReadData(
+          pTab->pWriteFd, aBuf, eocd.nSize, eocd.iOffset, &pTab->base.zErrMsg
+      );
+    }
+
+    for(i=0; rc==SQLITE_OK && i<eocd.nEntry; i++){
+      u16 nFile;
+      u16 nExtra;
+      u16 nComment;
+      ZipfileEntry *pNew;
+      u8 *aRec = &aBuf[iOff];
+
+      nFile = zipfileGetU16(&aRec[ZIPFILE_CDS_NFILE_OFF]);
+      nExtra = zipfileGetU16(&aRec[ZIPFILE_CDS_NFILE_OFF+2]);
+      nComment = zipfileGetU16(&aRec[ZIPFILE_CDS_NFILE_OFF+4]);
+
+      pNew = sqlite3_malloc(
+          sizeof(ZipfileEntry) 
+        + nFile+1 
+        + ZIPFILE_CDS_FIXED_SZ+nFile+nExtra+nComment
+      );
+      if( pNew==0 ){
+        rc = SQLITE_NOMEM;
+      }else{
+        pNew->zPath = (char*)&pNew[1];
+        memcpy(pNew->zPath, &aBuf[ZIPFILE_CDS_FIXED_SZ], nFile);
+        pNew->zPath[nFile] = '\0';
+        pNew->aCdsEntry = (u8*)&pNew->zPath[nFile+1];
+        pNew->nCdsEntry = ZIPFILE_CDS_FIXED_SZ+nFile+nExtra+nComment;
+        memcpy(pNew->aCdsEntry, aRec, pNew->nCdsEntry);
+
+        pNew->pNext = pTab->pEntry;
+        pTab->pEntry = pNew;
+      }
+
+      iOff += ZIPFILE_CDS_FIXED_SZ+nFile+nExtra+nComment;
+    }
+
+    sqlite3_free(aBuf);
+  }else if( rc==SQLITE_EMPTY ){
+    rc = SQLITE_OK;
+  }
+
+  return rc;
+}
+
+static ZipfileEntry *zipfileNewEntry(
+  ZipfileCDS *pCds,               /* Values for fixed size part of CDS */
+  const char *zPath,              /* Path for new entry */
+  int nPath,                      /* strlen(zPath) */
+  u32 mTime                       /* Modification time (or 0) */
+){
+  u8 *aWrite;
+  ZipfileEntry *pNew;
+  pCds->nFile = nPath;
+  pCds->nExtra = mTime ? 9 : 0;
+  pNew = (ZipfileEntry*)sqlite3_malloc(
+    sizeof(ZipfileEntry) + 
+    nPath+1 + 
+    ZIPFILE_CDS_FIXED_SZ + nPath + pCds->nExtra
+  );
+
+  if( pNew ){
+    pNew->zPath = (char*)&pNew[1];
+    pNew->aCdsEntry = (u8*)&pNew->zPath[nPath+1];
+    pNew->nCdsEntry = ZIPFILE_CDS_FIXED_SZ + nPath + pCds->nExtra;
+    memcpy(pNew->zPath, zPath, nPath+1);
+
+    aWrite = pNew->aCdsEntry;
+    zipfileWrite32(aWrite, ZIPFILE_SIGNATURE_CDS);
+    zipfileWrite16(aWrite, pCds->iVersionMadeBy);
+    zipfileWrite16(aWrite, pCds->iVersionExtract);
+    zipfileWrite16(aWrite, pCds->flags);
+    zipfileWrite16(aWrite, pCds->iCompression);
+    zipfileWrite16(aWrite, pCds->mTime);
+    zipfileWrite16(aWrite, pCds->mDate);
+    zipfileWrite32(aWrite, pCds->crc32);
+    zipfileWrite32(aWrite, pCds->szCompressed);
+    zipfileWrite32(aWrite, pCds->szUncompressed);
+    zipfileWrite16(aWrite, pCds->nFile);
+    zipfileWrite16(aWrite, pCds->nExtra);
+    zipfileWrite16(aWrite, pCds->nComment);      assert( pCds->nComment==0 );
+    zipfileWrite16(aWrite, pCds->iDiskStart);
+    zipfileWrite16(aWrite, pCds->iInternalAttr);
+    zipfileWrite32(aWrite, pCds->iExternalAttr);
+    zipfileWrite32(aWrite, pCds->iOffset);
+    assert( aWrite==&pNew->aCdsEntry[ZIPFILE_CDS_FIXED_SZ] );
+    memcpy(aWrite, zPath, nPath);
+    if( pCds->nExtra ){
+      aWrite += nPath;
+      zipfileWrite16(aWrite, ZIPFILE_EXTRA_TIMESTAMP);
+      zipfileWrite16(aWrite, 5);
+      *aWrite++ = 0x01;
+      zipfileWrite32(aWrite, mTime);
+    }
+  }
+
+  return pNew;
+}
+
+static int zipfileAppendEntry(
+  ZipfileTab *pTab,
+  ZipfileCDS *pCds,
+  const char *zPath,              /* Path for new entry */
+  int nPath,                      /* strlen(zPath) */
+  const u8 *pData,
+  int nData,
+  u32 mTime
+){
+  u8 *aBuf = pTab->aBuffer;
+  int rc;
+
+  zipfileWrite32(aBuf, ZIPFILE_SIGNATURE_LFH);
+  zipfileWrite16(aBuf, pCds->iVersionExtract);
+  zipfileWrite16(aBuf, pCds->flags);
+  zipfileWrite16(aBuf, pCds->iCompression);
+  zipfileWrite16(aBuf, pCds->mTime);
+  zipfileWrite16(aBuf, pCds->mDate);
+  zipfileWrite32(aBuf, pCds->crc32);
+  zipfileWrite32(aBuf, pCds->szCompressed);
+  zipfileWrite32(aBuf, pCds->szUncompressed);
+  zipfileWrite16(aBuf, nPath);
+  zipfileWrite16(aBuf, pCds->nExtra);
+  assert( aBuf==&pTab->aBuffer[ZIPFILE_LFH_FIXED_SZ] );
+  rc = zipfileAppendData(pTab, pTab->aBuffer, aBuf - pTab->aBuffer);
+  if( rc==SQLITE_OK ){
+    rc = zipfileAppendData(pTab, (const u8*)zPath, nPath);
+  }
+
+  if( rc==SQLITE_OK && pCds->nExtra ){
+    aBuf = pTab->aBuffer;
+    zipfileWrite16(aBuf, ZIPFILE_EXTRA_TIMESTAMP);
+    zipfileWrite16(aBuf, 5);
+    *aBuf++ = 0x01;
+    zipfileWrite32(aBuf, mTime);
+    rc = zipfileAppendData(pTab, pTab->aBuffer, 9);
+  }
+
+  if( rc==SQLITE_OK ){
+    rc = zipfileAppendData(pTab, pData, nData);
+  }
+
+  return rc;
+}
+
+static int zipfileGetMode(ZipfileTab *pTab, sqlite3_value *pVal, int *pMode){
+  const char *z = (const char*)sqlite3_value_text(pVal);
+  int mode = 0;
+  if( z==0 || (z[0]>=0 && z[0]<=9) ){
+    mode = sqlite3_value_int(pVal);
+  }else{
+    const char zTemplate[10] = "-rwxrwxrwx";
+    int i;
+    if( strlen(z)!=10 ) goto parse_error;
+    switch( z[0] ){
+      case '-': mode |= S_IFREG; break;
+      case 'd': mode |= S_IFDIR; break;
+      case 'l': mode |= S_IFLNK; break;
+      default: goto parse_error;
+    }
+    for(i=1; i<10; i++){
+      if( z[i]==zTemplate[i] ) mode |= 1 << (9-i);
+      else if( z[i]!='-' ) goto parse_error;
+    }
+  }
+  *pMode = mode;
+  return SQLITE_OK;
+
+ parse_error:
+  pTab->base.zErrMsg = sqlite3_mprintf("zipfile: parse error in mode: %s", z);
+  return SQLITE_ERROR;
+}
+
+/*
+** xUpdate method.
+*/
+static int zipfileUpdate(
+  sqlite3_vtab *pVtab, 
+  int nVal, 
+  sqlite3_value **apVal, 
+  sqlite_int64 *pRowid
+){
+  ZipfileTab *pTab = (ZipfileTab*)pVtab;
+  int rc = SQLITE_OK;             /* Return Code */
+  ZipfileEntry *pNew = 0;         /* New in-memory CDS entry */
+
+  int mode;                       /* Mode for new entry */
+  i64 mTime;                      /* Modification time for new entry */
+  i64 sz;                         /* Uncompressed size */
+  const char *zPath;              /* Path for new entry */
+  int nPath;                      /* strlen(zPath) */
+  const u8 *pData;                /* Pointer to buffer containing content */
+  int nData;                      /* Size of pData buffer in bytes */
+  int iMethod = 0;                /* Compression method for new entry */
+  u8 *pFree = 0;                  /* Free this */
+  ZipfileCDS cds;                 /* New Central Directory Structure entry */
+
+  if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
+    pTab->base.zErrMsg = sqlite3_mprintf(
+        "zipfile: UPDATE/DELETE not yet supported"
+    );
+    return SQLITE_ERROR;
+  }
+
+  /* This table is only writable if a default archive path was specified 
+  ** as part of the CREATE VIRTUAL TABLE statement. */
+  if( pTab->zFile==0 ){
+    pTab->base.zErrMsg = sqlite3_mprintf(
+        "zipfile: writing requires a default archive"
+    );
+    return SQLITE_ERROR;
+  }
+
+  /* Open a write fd on the file. Also load the entire central directory
+  ** structure into memory. During the transaction any new file data is 
+  ** appended to the archive file, but the central directory is accumulated
+  ** in main-memory until the transaction is committed.  */
+  if( pTab->pWriteFd==0 ){
+    pTab->pWriteFd = fopen(pTab->zFile, "ab+");
+    if( pTab->pWriteFd==0 ){
+      pTab->base.zErrMsg = sqlite3_mprintf(
+          "zipfile: failed to open file %s for writing", pTab->zFile
+      );
+      return SQLITE_ERROR;
+    }
+    fseek(pTab->pWriteFd, 0, SEEK_END);
+    pTab->szCurrent = pTab->szOrig = (i64)ftell(pTab->pWriteFd);
+    rc = zipfileLoadDirectory(pTab);
+    if( rc!=SQLITE_OK ) return rc;
+  }
+
+  zPath = (const char*)sqlite3_value_text(apVal[2]);
+  nPath = strlen(zPath);
+  rc = zipfileGetMode(pTab, apVal[3], &mode);
+  if( rc!=SQLITE_OK ) return rc;
+  mTime = sqlite3_value_int64(apVal[4]);
+  sz = sqlite3_value_int(apVal[5]);
+  pData = sqlite3_value_blob(apVal[6]);
+  nData = sqlite3_value_bytes(apVal[6]);
+
+  /* If a NULL value is inserted into the 'method' column, do automatic
+  ** compression. */
+  if( nData>0 && sqlite3_value_type(apVal[7])==SQLITE_NULL ){
+    pFree = (u8*)sqlite3_malloc(nData);
+    if( pFree==0 ){
+      rc = SQLITE_NOMEM;
+    }else{
+      int res;
+      z_stream str;
+      memset(&str, 0, sizeof(str));
+      str.next_in = (z_const Bytef*)pData;
+      str.avail_in = nData;
+      str.next_out = pFree;
+      str.avail_out = nData;
+      deflateInit2(&str, 9, Z_DEFLATED, -15, 8, Z_DEFAULT_STRATEGY);
+      res = deflate(&str, Z_FINISH);
+      if( res==Z_STREAM_END ){
+        pData = pFree;
+        nData = str.total_out;
+        iMethod = 8;
+      }else if( res!=Z_OK ){
+        pTab->base.zErrMsg = sqlite3_mprintf("zipfile: deflate() error");
+        rc = SQLITE_ERROR;
+      }
+      deflateEnd(&str);
+    }
+  }else{
+    iMethod = sqlite3_value_int(apVal[7]);
+    if( iMethod<0 || iMethod>65535 ){
+      pTab->base.zErrMsg = sqlite3_mprintf(
+          "zipfile: invalid compression method: %d", iMethod
+      );
+      rc = SQLITE_ERROR;
+    }
+  }
+
+  if( rc==SQLITE_OK ){
+    /* Create the new CDS record. */
+    memset(&cds, 0, sizeof(cds));
+    cds.iVersionMadeBy = ZIPFILE_NEWENTRY_MADEBY;
+    cds.iVersionExtract = ZIPFILE_NEWENTRY_REQUIRED;
+    cds.flags = ZIPFILE_NEWENTRY_FLAGS;
+    cds.iCompression = iMethod;
+    zipfileMtimeToDos(&cds, (u32)mTime);
+    cds.crc32 = crc32(0, pData, nData);
+    cds.szCompressed = nData;
+    cds.szUncompressed = sz;
+    cds.iExternalAttr = (mode<<16);
+    cds.iOffset = pTab->szCurrent;
+    pNew = zipfileNewEntry(&cds, zPath, nPath, (u32)mTime);
+    if( pNew==0 ){
+      rc = SQLITE_NOMEM;
+    }else{
+      pNew->pNext = pTab->pEntry;
+      pTab->pEntry = pNew;
+    }
+  }
+
+  /* Append the new header+file to the archive */
+  if( rc==SQLITE_OK ){
+    rc = zipfileAppendEntry(pTab, &cds, zPath, nPath, pData, nData, (u32)mTime);
+  }
+
+  sqlite3_free(pFree);
+  return rc;
+}
+
+static int zipfileAppendEOCD(ZipfileTab *pTab, ZipfileEOCD *p){
+  u8 *aBuf = pTab->aBuffer;
+
+  zipfileWrite32(aBuf, ZIPFILE_SIGNATURE_EOCD);
+  zipfileWrite16(aBuf, p->iDisk);
+  zipfileWrite16(aBuf, p->iFirstDisk);
+  zipfileWrite16(aBuf, p->nEntry);
+  zipfileWrite16(aBuf, p->nEntryTotal);
+  zipfileWrite32(aBuf, p->nSize);
+  zipfileWrite32(aBuf, p->iOffset);
+  zipfileWrite16(aBuf, 0);        /* Size of trailing comment in bytes*/
+
+  assert( (aBuf-pTab->aBuffer)==22 );
+  return zipfileAppendData(pTab, pTab->aBuffer, aBuf - pTab->aBuffer);
+}
+
+static void zipfileCleanupTransaction(ZipfileTab *pTab){
+  ZipfileEntry *pEntry;
+  ZipfileEntry *pNext;
+
+  for(pEntry=pTab->pEntry; pEntry; pEntry=pNext){
+    pNext = pEntry->pNext;
+    sqlite3_free(pEntry);
+  }
+  pTab->pEntry = 0;
+  fclose(pTab->pWriteFd);
+  pTab->pWriteFd = 0;
+  pTab->szCurrent = 0;
+  pTab->szOrig = 0;
+}
+
+static int zipfileBegin(sqlite3_vtab *pVtab){
+  return SQLITE_OK;
+}
+
+static int zipfileCommit(sqlite3_vtab *pVtab){
+  ZipfileTab *pTab = (ZipfileTab*)pVtab;
+  int rc = SQLITE_OK;
+  if( pTab->pWriteFd ){
+    i64 iOffset = pTab->szCurrent;
+    ZipfileEntry *pEntry;
+    ZipfileEOCD eocd;
+    int nEntry = 0;
+
+    /* Write out all entries */
+    for(pEntry=pTab->pEntry; rc==SQLITE_OK && pEntry; pEntry=pEntry->pNext){
+      rc = zipfileAppendData(pTab, pEntry->aCdsEntry, pEntry->nCdsEntry);
+      nEntry++;
+    }
+
+    /* Write out the EOCD record */
+    eocd.iDisk = 0;
+    eocd.iFirstDisk = 0;
+    eocd.nEntry = nEntry;
+    eocd.nEntryTotal = nEntry;
+    eocd.nSize = pTab->szCurrent - iOffset;;
+    eocd.iOffset = iOffset;
+    rc = zipfileAppendEOCD(pTab, &eocd);
+
+    zipfileCleanupTransaction(pTab);
+  }
+  return rc;
+}
+
+static int zipfileRollback(sqlite3_vtab *pVtab){
+  return zipfileCommit(pVtab);
+}
+
 /*
 ** Register the "zipfile" virtual table.
 */
 static int zipfileRegister(sqlite3 *db){
   static sqlite3_module zipfileModule = {
-    0,                         /* iVersion */
+    1,                         /* iVersion */
-    0,                         /* xCreate */
+    zipfileConnect,            /* xCreate */
     zipfileConnect,            /* xConnect */
     zipfileBestIndex,          /* xBestIndex */
     zipfileDisconnect,         /* xDisconnect */
-    0,                         /* xDestroy */
+    zipfileDisconnect,         /* xDestroy */
     zipfileOpen,               /* xOpen - open a cursor */
     zipfileClose,              /* xClose - close a cursor */
     zipfileFilter,             /* xFilter - configure scan constraints */
@@ -671,11 +1235,11 @@ static int zipfileRegister(sqlite3 *db){
     zipfileEof,                /* xEof - check for end of scan */
     zipfileColumn,             /* xColumn - read data */
     zipfileRowid,              /* xRowid - read data */
-    0,                         /* xUpdate */
+    zipfileUpdate,             /* xUpdate */
-    0,                         /* xBegin */
+    zipfileBegin,              /* xBegin */
     0,                         /* xSync */
-    0,                         /* xCommit */
+    zipfileCommit,             /* xCommit */
-    0,                         /* xRollback */
+    zipfileRollback,           /* xRollback */
     0,                         /* xFindMethod */
     0,                         /* xRename */
   };
@@ -687,8 +1251,6 @@ static int zipfileRegister(sqlite3 *db){
 # define zipfileRegister(x) SQLITE_OK
 #endif
 
-#include <zlib.h>
-
 /*
 ** zipfile_uncompress(DATA, SZ, METHOD)
 */

main.mk

@@ -370,6 +370,7 @@ TESTSRC += \
   $(TOP)/ext/misc/unionvtab.c \
   $(TOP)/ext/misc/wholenumber.c \
   $(TOP)/ext/misc/vfslog.c \
+  $(TOP)/ext/misc/zipfile.c \
   $(TOP)/ext/fts5/fts5_tcl.c \
   $(TOP)/ext/fts5/fts5_test_mi.c \
   $(TOP)/ext/fts5/fts5_test_tok.c 

manifest

@@ -1,5 +1,5 @@
-C Improve\sthe\sshell\stool\s".ar\s--list\s--verbose"\scommand.
+C Update\sext/misc/zipfile.c\sto\ssupport\screating\sand\sadding\sentries\sto\sexisting\nzip\sarchives.
-D 2017-12-27T21:13:21.423
+D 2017-12-29T20:19:03.196
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5
@@ -302,7 +302,7 @@ F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178
 F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9
 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd
 F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212
-F ext/misc/zipfile.c ced1aa768904cf8182c3da93d42df4e3cf30fe7a2e66c9216f6bc64a8df4fd5b
+F ext/misc/zipfile.c d9e0b0a4fdb4cb82ad22fd19d75563efdb1d6f812d62438bd30819781c282357
 F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e
 F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842
 F ext/rbu/rbu10.test 1846519a438697f45e9dcb246908af81b551c29e1078d0304fae83f1fed7e9ee
@@ -405,7 +405,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk fc13303745f7a06e2eac69406ee0e7ba481f6df37a528ceb6ebacc03e310a020
+F main.mk 9109d5786263a20fcf321a77c463d39c9ac84c871ea3774b0d213d8311dda105
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -492,7 +492,7 @@ F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6
 F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
 F src/tclsqlite.c 1833388c01e3b77f4c712185ee7250b9423ee0981ce6ae7e401e47db0319a696
-F src/test1.c 8ef15f7a357f85dfc41c6c748ce9c947b4f676e01bb5ae6a45bee4923dff8b51
+F src/test1.c 1879ff5095def6091f83f54c6233bc19e2b7223068fb02da41f2396d55729764
 F src/test2.c 3efb99ab7f1fc8d154933e02ae1378bac9637da5
 F src/test3.c b8434949dfb8aff8dfa082c8b592109e77844c2135ed3c492113839b6956255b
 F src/test4.c 18ec393bb4d0ad1de729f0b94da7267270f3d8e6
@@ -1596,6 +1596,7 @@ F test/wordcount.c cb589cec469a1d90add05b1f8cee75c7210338d87a5afd65260ed5c0f4bbf
 F test/writecrash.test f1da7f7adfe8d7f09ea79b42e5ca6dcc41102f27f8e334ad71539501ddd910cc
 F test/zeroblob.test 3857870fe681b8185654414a9bccfde80b62a0fa
 F test/zerodamage.test e59a56443d6298ecf7435f618f0b27654f0c849e
+F test/zipfile.test c5167d0d13ed7165b1982ac46b8aa94d65bdf41d94dd82f1d17ad2250650356c
 F tool/GetFile.cs a15e08acb5dd7539b75ba23501581d7c2b462cb5
 F tool/GetTclKit.bat 8995df40c4209808b31f24de0b58f90930239a234f7591e3675d45bfbb990c5d
 F tool/Replace.cs 02c67258801c2fb5f63231e0ac0f220b4b36ba91
@@ -1692,7 +1693,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P a532a0f6fd59e81086d46f09151ba7fb26725198231d902c71d0f95cb01dbe91
+P b64681a644c419bb98d00980a6cb56ef5a0aff5ef5321955631f0b4c88aac283
-R 7075030d907b0ffe212621eb709bb967
+R 407639481a191f472cd3497cc84b9ac5
 U dan
-Z 1c430e112380037cc2bfad44bf8be92e
+Z a9a0bf4010ca982b681b075b8e4b53ca

manifest.uuid

@@ -1 +1 @@
-b64681a644c419bb98d00980a6cb56ef5a0aff5ef5321955631f0b4c88aac283
+2dec2dec592c0726ebe87b841b9c8d493dea7074a99f278eb1bf0b744d658a9d

src/test1.c

@@ -6960,6 +6960,7 @@ static int SQLITE_TCLAPI tclLoadStaticExtensionCmd(
   extern int sqlite3_totype_init(sqlite3*,char**,const sqlite3_api_routines*);
   extern int sqlite3_wholenumber_init(sqlite3*,char**,const sqlite3_api_routines*);
   extern int sqlite3_unionvtab_init(sqlite3*,char**,const sqlite3_api_routines*);
+  extern int sqlite3_zipfile_init(sqlite3*,char**,const sqlite3_api_routines*);
   static const struct {
     const char *zExtName;
     int (*pInit)(sqlite3*,char**,const sqlite3_api_routines*);
@@ -6981,6 +6982,7 @@ static int SQLITE_TCLAPI tclLoadStaticExtensionCmd(
     { "totype",                sqlite3_totype_init               },
     { "unionvtab",             sqlite3_unionvtab_init            },
     { "wholenumber",           sqlite3_wholenumber_init          },
+    { "zipfile",               sqlite3_zipfile_init              },
   };
   sqlite3 *db;
   const char *zName;

test/zipfile.test

@@ -0,0 +1,60 @@
+# 2017 December 9
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix zipfile
+
+load_static_extension db zipfile
+
+forcedelete test.zip
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE temp.zz USING zipfile('test.zip');
+  PRAGMA table_info(zz);
+} {
+  0 name {} 0 {} 0 
+  1 mode {} 0 {} 0 
+  2 mtime {} 0 {} 0 
+  3 sz {} 0 {} 0 
+  4 data {} 0 {} 0
+  5 method {} 0 {} 0
+}
+
+do_execsql_test 1.1 {
+  INSERT INTO zz VALUES('f.txt', '-rw-r--r--', 1000000000, 5, 'abcde', 0);
+  INSERT INTO zz VALUES('g.txt', '-rw-r--r--', 1000000002, 5, '12345', 0);
+}
+
+do_execsql_test 1.2 {
+  SELECT name, mtime, data FROM zipfile('test.zip');
+} {
+  g.txt 1000000002 12345
+  f.txt 1000000000 abcde 
+}
+
+do_execsql_test 1.3 {
+  INSERT INTO zz VALUES('h.txt', 
+    '-rw-r--r--', 1000000004, 20, 'aaaaaaaaaabbbbbbbbbb', NULL
+  );
+}
+
+do_execsql_test 1.4 {
+  SELECT name, mtime, zipfile_uncompress(data, sz, method), method
+  FROM zipfile('test.zip');
+} {
+  h.txt 1000000004 aaaaaaaaaabbbbbbbbbb 8
+  f.txt 1000000000 abcde 0
+  g.txt 1000000002 12345 0
+}
+
+finish_test
+