Out-of-memory cleanup in tokenizers. Handle NULL return from

malloc/calloc/realloc appropriately, and use sizeof(var) instead of
sizeof(type) to make certain that we don't get a mismatch between
them as the code rots. (CVS 3693)

FossilOrigin-Name: fbc53da8c645935c74e49af2ab2cf447dc72ba4e

ext/fts1/fts1_porter.c

@@ -70,7 +70,9 @@ static int porterCreate(
   sqlite3_tokenizer **ppTokenizer
 ){
   porter_tokenizer *t;
-  t = (porter_tokenizer *) calloc(sizeof(porter_tokenizer), 1);
+  t = (porter_tokenizer *) calloc(sizeof(*t), 1);
+  if( t==NULL ) return SQLITE_NOMEM;
+
   *ppTokenizer = &t->base;
   return SQLITE_OK;
 }
@@ -96,7 +98,9 @@ static int porterOpen(
 ){
   porter_tokenizer_cursor *c;
 
-  c = (porter_tokenizer_cursor *) malloc(sizeof(porter_tokenizer_cursor));
+  c = (porter_tokenizer_cursor *) malloc(sizeof(*c));
+  if( c==NULL ) return SQLITE_NOMEM;
+
   c->zInput = zInput;
   if( zInput==0 ){
     c->nInput = 0;
@@ -605,6 +609,7 @@ static int porterNext(
       if( n>c->nAllocated ){
         c->nAllocated = n+20;
         c->zToken = realloc(c->zToken, c->nAllocated);
+        if( c->zToken==NULL ) return SQLITE_NOMEM;
       }
       porter_stemmer(&z[iStartOffset], n, c->zToken, pnBytes);
       *pzToken = c->zToken;

ext/fts1/fts1_tokenizer1.c

@@ -61,7 +61,9 @@ static int simpleCreate(
 ){
   simple_tokenizer *t;
 
-  t = (simple_tokenizer *) calloc(sizeof(simple_tokenizer), 1);
+  t = (simple_tokenizer *) calloc(sizeof(*t), 1);
+  if( t==NULL ) return SQLITE_NOMEM;
+
   /* TODO(shess) Delimiters need to remain the same from run to run,
   ** else we need to reindex.  One solution would be a meta-table to
   ** track such information in the database, then we'd only want this
@@ -111,7 +113,9 @@ static int simpleOpen(
 ){
   simple_tokenizer_cursor *c;
 
-  c = (simple_tokenizer_cursor *) malloc(sizeof(simple_tokenizer_cursor));
+  c = (simple_tokenizer_cursor *) malloc(sizeof(*c));
+  if( c==NULL ) return SQLITE_NOMEM;
+
   c->pInput = pInput;
   if( pInput==0 ){
     c->nBytes = 0;
@@ -175,6 +179,7 @@ static int simpleNext(
       if( n>c->nTokenAllocated ){
         c->nTokenAllocated = n+20;
         c->pToken = realloc(c->pToken, c->nTokenAllocated);
+        if( c->pToken==NULL ) return SQLITE_NOMEM;
       }
       for(i=0; i<n; i++){
         /* TODO(shess) This needs expansion to handle UTF-8

ext/fts2/fts2_porter.c

@@ -70,7 +70,9 @@ static int porterCreate(
   sqlite3_tokenizer **ppTokenizer
 ){
   porter_tokenizer *t;
-  t = (porter_tokenizer *) calloc(sizeof(porter_tokenizer), 1);
+  t = (porter_tokenizer *) calloc(sizeof(*t), 1);
+  if( t==NULL ) return SQLITE_NOMEM;
+
   *ppTokenizer = &t->base;
   return SQLITE_OK;
 }
@@ -96,7 +98,9 @@ static int porterOpen(
 ){
   porter_tokenizer_cursor *c;
 
-  c = (porter_tokenizer_cursor *) malloc(sizeof(porter_tokenizer_cursor));
+  c = (porter_tokenizer_cursor *) malloc(sizeof(*c));
+  if( c==NULL ) return SQLITE_NOMEM;
+
   c->zInput = zInput;
   if( zInput==0 ){
     c->nInput = 0;
@@ -605,6 +609,7 @@ static int porterNext(
       if( n>c->nAllocated ){
         c->nAllocated = n+20;
         c->zToken = realloc(c->zToken, c->nAllocated);
+        if( c->zToken==NULL ) return SQLITE_NOMEM;
       }
       porter_stemmer(&z[iStartOffset], n, c->zToken, pnBytes);
       *pzToken = c->zToken;

ext/fts2/fts2_tokenizer1.c

@@ -61,7 +61,9 @@ static int simpleCreate(
 ){
   simple_tokenizer *t;
 
-  t = (simple_tokenizer *) calloc(sizeof(simple_tokenizer), 1);
+  t = (simple_tokenizer *) calloc(sizeof(*t), 1);
+  if( t==NULL ) return SQLITE_NOMEM;
+
   /* TODO(shess) Delimiters need to remain the same from run to run,
   ** else we need to reindex.  One solution would be a meta-table to
   ** track such information in the database, then we'd only want this
@@ -111,7 +113,9 @@ static int simpleOpen(
 ){
   simple_tokenizer_cursor *c;
 
-  c = (simple_tokenizer_cursor *) malloc(sizeof(simple_tokenizer_cursor));
+  c = (simple_tokenizer_cursor *) malloc(sizeof(*c));
+  if( c==NULL ) return SQLITE_NOMEM;
+
   c->pInput = pInput;
   if( pInput==0 ){
     c->nBytes = 0;
@@ -175,6 +179,7 @@ static int simpleNext(
       if( n>c->nTokenAllocated ){
         c->nTokenAllocated = n+20;
         c->pToken = realloc(c->pToken, c->nTokenAllocated);
+        if( c->pToken==NULL ) return SQLITE_NOMEM;
       }
       for(i=0; i<n; i++){
         /* TODO(shess) This needs expansion to handle UTF-8

manifest

@@ -1,5 +1,5 @@
-C Save\san\sif(...)\sclause\smade\sredundant\sby\s(3691).\s(CVS\s3692)
-D 2007-03-15T15:35:29
+C Out-of-memory\scleanup\sin\stokenizers.\s\sHandle\sNULL\sreturn\sfrom\nmalloc/calloc/realloc\sappropriately,\sand\suse\ssizeof(var)\sinstead\sof\nsizeof(type)\sto\smake\scertain\sthat\swe\sdon't\sget\sa\smismatch\sbetween\nthem\sas\sthe\scode\srots.\s(CVS\s3693)
+D 2007-03-16T18:30:54
 F Makefile.in 1fe3d0b46e40fd684e1e61f8e8056cefed16de9f
 F Makefile.linux-gcc 2d8574d1ba75f129aba2019f0b959db380a90935
 F README 9c4e2d6706bdcc3efdd773ce752a8cdab4f90028
@@ -26,9 +26,9 @@ F ext/fts1/fts1.c 0aab3cf20eefd38935c8f525494d689cb2785f1d
 F ext/fts1/fts1.h 6060b8f62c1d925ea8356cb1a6598073eb9159a6
 F ext/fts1/fts1_hash.c 3196cee866edbebb1c0521e21672e6d599965114
 F ext/fts1/fts1_hash.h 957d378355ed29f672cd5add012ce8b088a5e089
-F ext/fts1/fts1_porter.c dd7db7f640aa648f272ee7d77f46815896c923f3
+F ext/fts1/fts1_porter.c 263e6a89c8769a456152e1abcd8821fcb4699e5b
 F ext/fts1/fts1_tokenizer.h fdea722c38a9f82ed921642981234f666e47919c
-F ext/fts1/fts1_tokenizer1.c 98c2bb9f1feb97294256850bd84baac6799168b8
+F ext/fts1/fts1_tokenizer1.c df09e638156abfe320e705c839baa98d595f45a7
 F ext/fts1/fulltext.c d935e600d87bc86b7d64f55c7520ea41d6034c5c
 F ext/fts1/fulltext.h 08525a47852d1d62a0be81d3fc3fe2d23b094efd
 F ext/fts1/simple_tokenizer.c 1844d72f7194c3fd3d7e4173053911bf0661b70d
@@ -38,9 +38,9 @@ F ext/fts2/fts2.c a49ed7292cbacbfcde6fdee1be4f6529277be3fa
 F ext/fts2/fts2.h bbdab26d34f91974d5b9ade8b7836c140a7c4ce1
 F ext/fts2/fts2_hash.c b3f22116d4ef0bc8f2da6e3fdc435c86d0951a9b
 F ext/fts2/fts2_hash.h e283308156018329f042816eb09334df714e105e
-F ext/fts2/fts2_porter.c 3d074a83d93f34765e26aeb24fc9ba300ecb26dc
+F ext/fts2/fts2_porter.c df529ee9b70f21ebb9c5affd3c7669ee615accc7
 F ext/fts2/fts2_tokenizer.h 4c5ffe31d63622869eb6eec1503df7f6996fd1bd
-F ext/fts2/fts2_tokenizer1.c b26f0d2fae967fbe5722f82d08506d35f10ac4e7
+F ext/fts2/fts2_tokenizer1.c 6067f2f710bc7e91c0688b7b11be1027777553e0
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895
 F ltmain.sh 56abb507100ed2d4261f6dd1653dec3cf4066387
 F main.mk 0347b7c277fab2aa579a8d84f3ba975407e8fba4
@@ -436,7 +436,7 @@ F www/tclsqlite.tcl bb0d1357328a42b1993d78573e587c6dcbc964b9
 F www/vdbe.tcl 87a31ace769f20d3627a64fa1fade7fed47b90d0
 F www/version3.tcl 890248cf7b70e60c383b0e84d77d5132b3ead42b
 F www/whentouse.tcl 97e2b5cd296f7d8057e11f44427dea8a4c2db513
-P 7180874592ffcaf8e2fe3c3b6b37449654da709b
-R 719cc995cbd8893611545d757a98e68f
-U danielk1977
-Z 8386e34fed8da2b1d9cd079c24880e75
+P 8153edf8bea51b9344de1b700482879190f86fb4
+R 193c48ddae813be84d567590d530cec0
+U shess
+Z 0134f9e6ffacefea1f838756fe42f156

manifest.uuid

@@ -1 +1 @@
-8153edf8bea51b9344de1b700482879190f86fb4
+fbc53da8c645935c74e49af2ab2cf447dc72ba4e