From 153d3e91dbace8dc0c56ef7a80ea1e282208e3c5 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Thu, 21 Jan 2016 18:56:10 +0000 Subject: [PATCH 01/39] Fix typo in 'Makefile.in'. FossilOrigin-Name: 29a51c6dcc0ef7cab51aba0024e9c5bec89c6973 --- Makefile.in | 4 ++-- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Makefile.in b/Makefile.in index 6859dcf854..a07279c3c9 100644 --- a/Makefile.in +++ b/Makefile.in @@ -1180,9 +1180,9 @@ lib_install: libsqlite3.la $(INSTALL) -d $(DESTDIR)$(libdir) $(LTINSTALL) libsqlite3.la $(DESTDIR)$(libdir) -install: sqlite3$(BEXE) lib_install sqlite3.h sqlite3.pc ${HAVE_TCL:1=tcl_install} +install: sqlite3$(TEXE) lib_install sqlite3.h sqlite3.pc ${HAVE_TCL:1=tcl_install} $(INSTALL) -d $(DESTDIR)$(bindir) - $(LTINSTALL) sqlite3$(BEXE) $(DESTDIR)$(bindir) + $(LTINSTALL) sqlite3$(TEXE) $(DESTDIR)$(bindir) $(INSTALL) -d $(DESTDIR)$(includedir) $(INSTALL) -m 0644 sqlite3.h $(DESTDIR)$(includedir) $(INSTALL) -m 0644 $(TOP)/src/sqlite3ext.h $(DESTDIR)$(includedir) diff --git a/manifest b/manifest index d459ef1f26..94fd201a3d 100644 --- a/manifest +++ b/manifest @@ -1,6 +1,6 @@ -C Improved\scomments\son\sthe\sFORDELETE\shint.\s\sNo\slogic\schanges. -D 2016-01-21T15:55:37.704 -F Makefile.in 7be88f5b473891e3a8c07245ed60535fcda4f9ee +C Fix\stypo\sin\s'Makefile.in'. +D 2016-01-21T18:56:10.840 +F Makefile.in 027c1603f255390c43a426671055a31c0a65fdb4 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc c5ead4aa22ff6f528c755b07ed1e31184ac5b3d2 F README.md 8ecc12493ff9f820cdea6520a9016001cb2e59b7 @@ -1419,7 +1419,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f60f025027ab3acdbc97701f81ff9c11c8463022 -R 58c662ebb54c7087fe70ba0b9b653627 -U drh -Z 656e64b9ebb4ed29ac85ca9fad2e65b0 +P a3cec529f0238e4ca1196fec420f2de80d28db78 +R 0153ee59df704041be1869f598c173fa +U mistachkin +Z cab32b4afad5b6a6884b5ccdb88994ac diff --git a/manifest.uuid b/manifest.uuid index 80d0db3e83..47bf74ce33 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a3cec529f0238e4ca1196fec420f2de80d28db78 \ No newline at end of file +29a51c6dcc0ef7cab51aba0024e9c5bec89c6973 \ No newline at end of file From edcb4ebba1553304c3a14f127b1991a11f2b1bb0 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Fri, 22 Jan 2016 01:25:15 +0000 Subject: [PATCH 02/39] MSVC makefile enhancements for use with the Windows 10 SDK. FossilOrigin-Name: ac2e9d3de54d299f92fc35927b6df6d3aaa2dc59 --- Makefile.msc | 119 +++++++++++++++++++++++++++++++++++++++----------- manifest | 12 ++--- manifest.uuid | 2 +- 3 files changed, 100 insertions(+), 33 deletions(-) diff --git a/Makefile.msc b/Makefile.msc index 160323a103..3ac19c2f73 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -130,6 +130,12 @@ FOR_WINRT = 0 FOR_UAP = 0 !ENDIF +# Set this non-0 to compile binaries suitable for Windows 10. +# +!IFNDEF FOR_WIN10 +FOR_WIN10 = 0 +!ENDIF + # Set this non-0 to skip attempting to look for and/or link with the Tcl # runtime library. # @@ -186,6 +192,31 @@ DEBUG = 0 OPTIMIZATIONS = 2 !ENDIF +# This is the name to use for the SQLite dynamic link library (DLL). +# +!IFNDEF SQLITE3DLL +SQLITE3DLL = sqlite3.dll +!ENDIF + +# This is the name to use for the SQLite import library (LIB). +# +!IFNDEF SQLITE3LIB +SQLITE3LIB = sqlite3.lib +!ENDIF + +# This is the name to use for the SQLite shell executable (EXE). +# +!IFNDEF SQLITE3EXE +SQLITE3EXE = sqlite3.exe +!ENDIF + +# This is the argument used to set the program database (PDB) file for the +# SQLite shell executable (EXE). +# +!IFNDEF SQLITE3EXEPDB +SQLITE3EXEPDB = /pdb:sqlite3sh.pdb +!ENDIF + # These are the "standard" SQLite compilation options used when compiling for # the Windows platform. # @@ -195,6 +226,19 @@ OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RTREE=1 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1 !ENDIF +# These are the "extended" SQLite compilation options used when compiling for +# the Windows 10 platform. +# +!IFNDEF EXT_FEATURE_FLAGS +!IF $(FOR_WIN10)!=0 +EXT_FEATURE_FLAGS = $(EXT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS4=1 +EXT_FEATURE_FLAGS = $(EXT_FEATURE_FLAGS) -DSQLITE_SYSTEM_MALLOC=1 +EXT_FEATURE_FLAGS = $(EXT_FEATURE_FLAGS) -DSQLITE_OMIT_LOCALTIME=1 +!ELSE +EXT_FEATURE_FLAGS = +!ENDIF +!ENDIF + ############################################################################### ############################### END OF OPTIONS ################################ ############################################################################### @@ -327,9 +371,18 @@ TCC = $(CC) -nologo -W4 -DINCLUDE_MSVC_H=1 $(CCOPTS) $(TCCOPTS) TCC = $(CC) -nologo -W3 $(CCOPTS) $(TCCOPTS) !ENDIF -TCC = $(TCC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) -I$(TOP)\src -fp:precise +TCC = $(TCC) -DSQLITE_OS_WIN=1 -I$(TOP) -I$(TOP)\src -fp:precise RCC = $(RC) -DSQLITE_OS_WIN=1 -I$(TOP) -I$(TOP)\src $(RCOPTS) $(RCCOPTS) +# Adjust the names of the primary targets for use with Windows 10. +# +!IF $(FOR_WIN10)!=0 +SQLITE3DLL = winsqlite3.dll +SQLITE3LIB = winsqlite3.lib +SQLITE3EXE = winsqlite3shell.exe +SQLITE3EXEPDB = +!ENDIF + # Check if we want to use the "stdcall" calling convention when compiling. # This is not supported by the compilers for non-x86 platforms. It should # also be noted here that building any target with these "stdcall" options @@ -337,7 +390,7 @@ RCC = $(RC) -DSQLITE_OS_WIN=1 -I$(TOP) -I$(TOP)\src $(RCOPTS) $(RCCOPTS) # to how the Tcl library functions are declared and exported (i.e. without # an explicit calling convention, which results in "cdecl"). # -!IF $(USE_STDCALL)!=0 +!IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0 !IF "$(PLATFORM)"=="x86" CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall @@ -358,7 +411,7 @@ SHELL_CCONV_OPTS = # These are additional compiler options used for the core library. # !IFNDEF CORE_COMPILE_OPTS -!IF $(DYNAMIC_SHELL)!=0 +!IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0 CORE_COMPILE_OPTS = $(CORE_CCONV_OPTS) -DSQLITE_API=__declspec(dllexport) !ELSE CORE_COMPILE_OPTS = $(CORE_CCONV_OPTS) @@ -369,7 +422,7 @@ CORE_COMPILE_OPTS = $(CORE_CCONV_OPTS) # when linking. # !IFNDEF CORE_LINK_DEP -!IF $(DYNAMIC_SHELL)!=0 +!IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0 CORE_LINK_DEP = !ELSE CORE_LINK_DEP = sqlite3.def @@ -379,7 +432,7 @@ CORE_LINK_DEP = sqlite3.def # These are additional linker options used for the core library. # !IFNDEF CORE_LINK_OPTS -!IF $(DYNAMIC_SHELL)!=0 +!IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0 CORE_LINK_OPTS = !ELSE CORE_LINK_OPTS = /DEF:sqlite3.def @@ -389,7 +442,7 @@ CORE_LINK_OPTS = /DEF:sqlite3.def # These are additional compiler options used for the shell executable. # !IFNDEF SHELL_COMPILE_OPTS -!IF $(DYNAMIC_SHELL)!=0 +!IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0 SHELL_COMPILE_OPTS = -DSQLITE_SHELL_JSON1 $(SHELL_CCONV_OPTS) -DSQLITE_API=__declspec(dllimport) !ELSE SHELL_COMPILE_OPTS = -DSQLITE_SHELL_JSON1 $(SHELL_CCONV_OPTS) @@ -399,8 +452,8 @@ SHELL_COMPILE_OPTS = -DSQLITE_SHELL_JSON1 $(SHELL_CCONV_OPTS) # This is the core library that the shell executable should depend on. # !IFNDEF SHELL_CORE_DEP -!IF $(DYNAMIC_SHELL)!=0 -SHELL_CORE_DEP = sqlite3.dll +!IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0 +SHELL_CORE_DEP = $(SQLITE3DLL) !ELSE SHELL_CORE_DEP = libsqlite3.lib !ENDIF @@ -409,8 +462,8 @@ SHELL_CORE_DEP = libsqlite3.lib # This is the core library that the shell executable should link with. # !IFNDEF SHELL_CORE_LIB -!IF $(DYNAMIC_SHELL)!=0 -SHELL_CORE_LIB = sqlite3.lib +!IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0 +SHELL_CORE_LIB = $(SQLITE3LIB) !ELSE SHELL_CORE_LIB = libsqlite3.lib !ENDIF @@ -441,6 +494,13 @@ TCC = $(TCC) -DWINAPI_FAMILY=WINAPI_FAMILY_APP RCC = $(RCC) -DWINAPI_FAMILY=WINAPI_FAMILY_APP !ENDIF +# C compiler options for Windows 10 (needs Visual Studio 2015). +# +!IF $(FOR_WIN10)!=0 +TCC = $(TCC) /guard:cf -D_ARM_WINAPI_PARTITION_DESKTOP_SDK_AVAILABLE +BCC = $(BCC) /guard:cf -D_ARM_WINAPI_PARTITION_DESKTOP_SDK_AVAILABLE +!ENDIF + # Also, we need to dynamically link to the correct MSVC runtime # when compiling for WinRT (e.g. debug or release) OR if the # USE_CRT_DLL option is set to force dynamically linking to the @@ -498,7 +558,7 @@ BCC = $(BCC) -DNDEBUG RCC = $(RCC) -DNDEBUG !ENDIF -!IF $(DEBUG)>0 || $(API_ARMOR)!=0 +!IF $(DEBUG)>0 || $(API_ARMOR)!=0 || $(FOR_WIN10)!=0 TCC = $(TCC) -DSQLITE_ENABLE_API_ARMOR=1 RCC = $(RCC) -DSQLITE_ENABLE_API_ARMOR=1 !ENDIF @@ -659,8 +719,8 @@ REQ_FEATURE_FLAGS = $(REQ_FEATURE_FLAGS) -DSQLITE_WIN32_USE_UUID=1 # Add the required and optional SQLite compilation options into the command # lines used to invoke the MSVC code and resource compilers. # -TCC = $(TCC) $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) -RCC = $(RCC) $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) +TCC = $(TCC) $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(EXT_FEATURE_FLAGS) +RCC = $(RCC) $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(EXT_FEATURE_FLAGS) # Add in any optional parameters specified on the commane line, e.g. # nmake /f Makefile.msc all "OPTS=-DSQLITE_ENABLE_FOO=1 -DSQLITE_OMIT_FOO=1" @@ -1203,7 +1263,7 @@ EXTHDR = $(EXTHDR) \ # TESTPROGS = \ testfixture.exe \ - sqlite3.exe \ + $(SQLITE3EXE) \ sqlite3_analyzer.exe \ sqldiff.exe @@ -1217,7 +1277,10 @@ FUZZDATA = \ # Extra compiler options for various shell tools # -SHELL_COMPILE_OPTS = -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS5 +!IF $(FOR_WIN10)==0 +SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_FTS5 +!ENDIF + FUZZERSHELL_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 FUZZCHECK_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 @@ -1228,7 +1291,7 @@ TESTOPTS = --verbose=file --output=test-out.txt # This is the default Makefile target. The objects listed here # are what get build when you type just "make" with no arguments. # -all: dll libsqlite3.lib sqlite3.exe libtclsqlite3.lib +all: dll libsqlite3.lib shell libtclsqlite3.lib libsqlite3.lib: $(LIBOBJ) $(LTLIB) $(LTLIBOPTS) /OUT:$@ $(LIBOBJ) $(TLIBS) @@ -1236,9 +1299,9 @@ libsqlite3.lib: $(LIBOBJ) libtclsqlite3.lib: tclsqlite.lo libsqlite3.lib $(LTLIB) $(LTLIBOPTS) $(LTLIBPATHS) /OUT:$@ tclsqlite.lo libsqlite3.lib $(LIBTCLSTUB) $(TLIBS) -sqlite3.exe: $(TOP)\src\shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) sqlite3.h - $(LTLINK) $(SHELL_COMPILE_OPTS) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) $(TOP)\src\shell.c \ - /link /pdb:sqlite3sh.pdb $(LDFLAGS) $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS) +$(SQLITE3EXE): $(TOP)\src\shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) sqlite3.h + $(LTLINK) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) $(TOP)\src\shell.c \ + /link $(SQLITE3EXEPDB) $(LDFLAGS) $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS) sqldiff.exe: $(TOP)\tool\sqldiff.c sqlite3.c sqlite3.h $(LTLINK) $(NO_WARN) $(TOP)\tool\sqldiff.c sqlite3.c /link $(LDFLAGS) $(LTLINKOPTS) @@ -1592,7 +1655,7 @@ parse.h: parse.c parse.c: $(TOP)\src\parse.y lemon.exe $(TOP)\tool\addopcodes.tcl del /Q parse.y parse.h parse.h.temp 2>NUL copy $(TOP)\src\parse.y . - .\lemon.exe $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(OPTS) parse.y + .\lemon.exe $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(EXT_FEATURE_FLAGS) $(OPTS) parse.y move parse.h parse.h.temp $(TCLSH_CMD) $(TOP)\tool\addopcodes.tcl parse.h.temp > parse.h @@ -1603,7 +1666,7 @@ sqlite3ext.h: .target_source copy tsrc\sqlite3ext.h . mkkeywordhash.exe: $(TOP)\tool\mkkeywordhash.c - $(BCC) $(NO_WARN) -Fe$@ $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(OPTS) \ + $(BCC) $(NO_WARN) -Fe$@ $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(EXT_FEATURE_FLAGS) $(OPTS) \ $(TOP)\tool\mkkeywordhash.c /link $(LDFLAGS) $(NLTLINKOPTS) $(NLTLIBPATHS) keywordhash.h: $(TOP)\tool\mkkeywordhash.c mkkeywordhash.exe @@ -1698,7 +1761,7 @@ FTS5_SRC = \ fts5parse.c: $(TOP)\ext\fts5\fts5parse.y lemon.exe copy $(TOP)\ext\fts5\fts5parse.y . del /Q fts5parse.h 2>NUL - .\lemon.exe $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(OPTS) fts5parse.y + .\lemon.exe $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(EXT_FEATURE_FLAGS) $(OPTS) fts5parse.y fts5parse.h: fts5parse.c @@ -1759,7 +1822,7 @@ fulltestonly: $(TESTPROGS) fuzztest @set PATH=$(LIBTCLPATH);$(PATH) .\testfixture.exe $(TOP)\test\full.test -queryplantest: testfixture.exe sqlite3.exe +queryplantest: testfixture.exe shell @set PATH=$(LIBTCLPATH);$(PATH) .\testfixture.exe $(TOP)\test\permutations.test queryplanner $(TESTOPTS) @@ -1864,7 +1927,7 @@ clean: del /Q LogEst.exe fts3view.exe rollback-test.exe showdb.exe 2>NUL del /Q showjournal.exe showstat4.exe showwal.exe speedtest1.exe 2>NUL del /Q mptester.exe wordcount.exe rbu.exe 2>NUL - del /Q sqlite3.exe sqlite3.dll sqlite3.def 2>NUL + del /Q $(SQLITE3EXE) $(SQLITE3DLL) sqlite3.def 2>NUL del /Q sqlite3.c sqlite3-*.c 2>NUL del /Q sqlite3rc.h 2>NUL del /Q shell.c sqlite3ext.h 2>NUL @@ -1873,9 +1936,13 @@ clean: del /Q fuzzershell.exe fuzzcheck.exe sqldiff.exe 2>NUL del /Q fts5.* fts5parse.* 2>NUL +# Shell executable. +# +shell: $(SQLITE3EXE) + # Dynamic link library section. # -dll: sqlite3.dll +dll: $(SQLITE3DLL) sqlite3.def: libsqlite3.lib echo EXPORTS > sqlite3.def @@ -1883,5 +1950,5 @@ sqlite3.def: libsqlite3.lib | $(TCLSH_CMD) $(TOP)\tool\replace.tcl include "^\s+1 _?(sqlite3_.*)$$" \1 \ | sort >> sqlite3.def -sqlite3.dll: $(LIBOBJ) $(LIBRESOBJS) $(CORE_LINK_DEP) +$(SQLITE3DLL): $(LIBOBJ) $(LIBRESOBJS) $(CORE_LINK_DEP) $(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL $(CORE_LINK_OPTS) /OUT:$@ $(LIBOBJ) $(LIBRESOBJS) $(LTLIBS) $(TLIBS) diff --git a/manifest b/manifest index 94fd201a3d..67cd5b4802 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,8 @@ -C Fix\stypo\sin\s'Makefile.in'. -D 2016-01-21T18:56:10.840 +C MSVC\smakefile\senhancements\sfor\suse\swith\sthe\sWindows\s10\sSDK. +D 2016-01-22T01:25:15.110 F Makefile.in 027c1603f255390c43a426671055a31c0a65fdb4 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 -F Makefile.msc c5ead4aa22ff6f528c755b07ed1e31184ac5b3d2 +F Makefile.msc 89b2e9780d97a6a3d7c89c4ad04d093787be3dc2 F README.md 8ecc12493ff9f820cdea6520a9016001cb2e59b7 F VERSION 866588d1edf0ccb5b0d33896974338f97564f719 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -1419,7 +1419,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P a3cec529f0238e4ca1196fec420f2de80d28db78 -R 0153ee59df704041be1869f598c173fa +P 29a51c6dcc0ef7cab51aba0024e9c5bec89c6973 +R 9e26ae539c690ec3cae729a6d81b3775 U mistachkin -Z cab32b4afad5b6a6884b5ccdb88994ac +Z d4ab6d93e282614aa0160682fbcc0beb diff --git a/manifest.uuid b/manifest.uuid index 47bf74ce33..5f1ffc501a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -29a51c6dcc0ef7cab51aba0024e9c5bec89c6973 \ No newline at end of file +ac2e9d3de54d299f92fc35927b6df6d3aaa2dc59 \ No newline at end of file From 5d4d941e94546ee93fe1b1823e903afc6b1f598c Mon Sep 17 00:00:00 2001 From: mistachkin Date: Fri, 22 Jan 2016 03:54:36 +0000 Subject: [PATCH 03/39] Further refinements to the MSVC makefile. FossilOrigin-Name: 67625b963a6f5a0c6dff1a04a323e693569c2d94 --- Makefile.msc | 126 +++++++++++++++++++++++++++++--------------------- manifest | 12 ++--- manifest.uuid | 2 +- 3 files changed, 81 insertions(+), 59 deletions(-) diff --git a/Makefile.msc b/Makefile.msc index 3ac19c2f73..a75045fc98 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -130,7 +130,7 @@ FOR_WINRT = 0 FOR_UAP = 0 !ENDIF -# Set this non-0 to compile binaries suitable for Windows 10. +# Set this non-0 to compile binaries suitable for the Windows 10 platform. # !IFNDEF FOR_WIN10 FOR_WIN10 = 0 @@ -192,6 +192,24 @@ DEBUG = 0 OPTIMIZATIONS = 2 !ENDIF +# Set the source code file to be used by executables and libraries when +# they need the amalgamation. +# +!IFNDEF SQLITE3C +!IF $(SPLIT_AMALGAMATION)!=0 +SQLITE3C = sqlite3-all.c +!ELSE +SQLITE3C = sqlite3.c +!ENDIF +!ENDIF + +# Set the include code file to be used by executables and libraries when +# they need SQLite. +# +!IFNDEF SQLITE3H +SQLITE3H = sqlite3.h +!ENDIF + # This is the name to use for the SQLite dynamic link library (DLL). # !IFNDEF SQLITE3DLL @@ -443,9 +461,20 @@ CORE_LINK_OPTS = /DEF:sqlite3.def # !IFNDEF SHELL_COMPILE_OPTS !IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0 -SHELL_COMPILE_OPTS = -DSQLITE_SHELL_JSON1 $(SHELL_CCONV_OPTS) -DSQLITE_API=__declspec(dllimport) +SHELL_COMPILE_OPTS = $(SHELL_CCONV_OPTS) -DSQLITE_API=__declspec(dllimport) !ELSE -SHELL_COMPILE_OPTS = -DSQLITE_SHELL_JSON1 $(SHELL_CCONV_OPTS) +SHELL_COMPILE_OPTS = $(SHELL_CCONV_OPTS) +!ENDIF +!ENDIF + +# This is the source code that the shell executable should be compiled +# with. +# +!IFNDEF SHELL_CORE_SRC +!IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0 +SHELL_CORE_SRC = +!ELSE +SHELL_CORE_SRC = $(SQLITE3C) !ENDIF !ENDIF @@ -455,7 +484,7 @@ SHELL_COMPILE_OPTS = -DSQLITE_SHELL_JSON1 $(SHELL_CCONV_OPTS) !IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0 SHELL_CORE_DEP = $(SQLITE3DLL) !ELSE -SHELL_CORE_DEP = libsqlite3.lib +SHELL_CORE_DEP = !ENDIF !ENDIF @@ -465,7 +494,7 @@ SHELL_CORE_DEP = libsqlite3.lib !IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0 SHELL_CORE_LIB = $(SQLITE3LIB) !ELSE -SHELL_CORE_LIB = libsqlite3.lib +SHELL_CORE_LIB = !ENDIF !ENDIF @@ -494,7 +523,7 @@ TCC = $(TCC) -DWINAPI_FAMILY=WINAPI_FAMILY_APP RCC = $(RCC) -DWINAPI_FAMILY=WINAPI_FAMILY_APP !ENDIF -# C compiler options for Windows 10 (needs Visual Studio 2015). +# C compiler options for the Windows 10 platform (needs MSVC 2015). # !IF $(FOR_WIN10)!=0 TCC = $(TCC) /guard:cf -D_ARM_WINAPI_PARTITION_DESKTOP_SDK_AVAILABLE @@ -506,7 +535,7 @@ BCC = $(BCC) /guard:cf -D_ARM_WINAPI_PARTITION_DESKTOP_SDK_AVAILABLE # USE_CRT_DLL option is set to force dynamically linking to the # MSVC runtime library. # -!IF $(FOR_WINRT)!=0 || $(USE_CRT_DLL)!=0 +!IF $(FOR_WINRT)!=0 || $(FOR_WIN10)!=0 || $(USE_CRT_DLL)!=0 !IF $(DEBUG)>1 TCC = $(TCC) -MDd BCC = $(BCC) -MDd @@ -1084,7 +1113,7 @@ SRC5 = \ opcodes.h \ parse.c \ parse.h \ - sqlite3.h + $(SQLITE3H) # All source code files. # @@ -1228,7 +1257,7 @@ HDR = \ $(TOP)\src\pcache.h \ parse.h \ $(TOP)\src\pragma.h \ - sqlite3.h \ + $(SQLITE3H) \ $(TOP)\src\sqlite3ext.h \ $(TOP)\src\sqliteInt.h \ $(TOP)\src\sqliteLimit.h \ @@ -1275,16 +1304,20 @@ FUZZDATA = \ $(TOP)\test\fuzzdata3.db \ $(TOP)\test\fuzzdata4.db -# Extra compiler options for various shell tools +# Additional compiler options for the shell. These are only effective +# when the shell is not being dynamically linked. # -!IF $(FOR_WIN10)==0 -SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_FTS5 +!IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0 +SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS5 !ENDIF +# Extra compiler options for various test tools. +# +MPTESTER_COMPILE_OPTS = -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS5 FUZZERSHELL_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 FUZZCHECK_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -# Standard options to testfixture +# Standard options to testfixture. # TESTOPTS = --verbose=file --output=test-out.txt @@ -1299,23 +1332,21 @@ libsqlite3.lib: $(LIBOBJ) libtclsqlite3.lib: tclsqlite.lo libsqlite3.lib $(LTLIB) $(LTLIBOPTS) $(LTLIBPATHS) /OUT:$@ tclsqlite.lo libsqlite3.lib $(LIBTCLSTUB) $(TLIBS) -$(SQLITE3EXE): $(TOP)\src\shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) sqlite3.h - $(LTLINK) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) $(TOP)\src\shell.c \ +$(SQLITE3EXE): $(TOP)\src\shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H) + $(LTLINK) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) $(TOP)\src\shell.c $(SHELL_CORE_SRC) \ /link $(SQLITE3EXEPDB) $(LDFLAGS) $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS) -sqldiff.exe: $(TOP)\tool\sqldiff.c sqlite3.c sqlite3.h - $(LTLINK) $(NO_WARN) $(TOP)\tool\sqldiff.c sqlite3.c /link $(LDFLAGS) $(LTLINKOPTS) +sqldiff.exe: $(TOP)\tool\sqldiff.c $(SQLITE3C) $(SQLITE3H) + $(LTLINK) $(NO_WARN) $(TOP)\tool\sqldiff.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) -fuzzershell.exe: $(TOP)\tool\fuzzershell.c sqlite3.c sqlite3.h - $(LTLINK) $(NO_WARN) $(FUZZERSHELL_COMPILE_OPTS) \ - $(TOP)\tool\fuzzershell.c sqlite3.c /link $(LDFLAGS) $(LTLINKOPTS) +fuzzershell.exe: $(TOP)\tool\fuzzershell.c $(SQLITE3C) $(SQLITE3H) + $(LTLINK) $(NO_WARN) $(FUZZERSHELL_COMPILE_OPTS) $(TOP)\tool\fuzzershell.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) -fuzzcheck.exe: $(TOP)\test\fuzzcheck.c sqlite3.c sqlite3.h - $(LTLINK) $(NO_WARN) $(FUZZCHECK_COMPILE_OPTS) $(TOP)\test\fuzzcheck.c sqlite3.c /link $(LDFLAGS) $(LTLINKOPTS) +fuzzcheck.exe: $(TOP)\test\fuzzcheck.c $(SQLITE3C) $(SQLITE3H) + $(LTLINK) $(NO_WARN) $(FUZZCHECK_COMPILE_OPTS) $(TOP)\test\fuzzcheck.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) -mptester.exe: $(TOP)\mptest\mptest.c $(SHELL_CORE_DEP) $(LIBRESOBJS) sqlite3.h - $(LTLINK) $(NO_WARN) $(SHELL_COMPILE_OPTS) $(TOP)\mptest\mptest.c \ - /link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(SHELL_LINK_OPTS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS) +mptester.exe: $(TOP)\mptest\mptest.c $(SQLITE3C) $(SQLITE3H) + $(LTLINK) $(NO_WARN) $(MPTESTER_COMPILE_OPTS) $(TOP)\mptest\mptest.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) MPTEST1 = mptester mptest.db $(TOP)\mptest\crash01.test --repeat 20 MPTEST2 = mptester mptest.db $(TOP)\mptest\multiwrite01.test --repeat 20 @@ -1359,15 +1390,6 @@ sqlite3.c: .target_source sqlite3ext.h $(TOP)\tool\mksqlite3c.tcl sqlite3-all.c: sqlite3.c $(TOP)\tool\split-sqlite3c.tcl $(TCLSH_CMD) $(TOP)\tool\split-sqlite3c.tcl -# Set the source code file to be used by executables and libraries when -# they need the amalgamation. -# -!IF $(SPLIT_AMALGAMATION)!=0 -SQLITE3C = sqlite3-all.c -!ELSE -SQLITE3C = sqlite3.c -!ENDIF - # Rule to build the amalgamation # sqlite3.lo: $(SQLITE3C) @@ -1637,7 +1659,7 @@ tclsqlite.lo: $(TOP)\src\tclsqlite.c $(HDR) tclsqlite-shell.lo: $(TOP)\src\tclsqlite.c $(HDR) $(LTCOMPILE) $(NO_WARN) -DTCLSH=1 -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c -tclsqlite3.exe: tclsqlite-shell.lo $(SQLITE3C) $(LIBRESOBJS) +tclsqlite3.exe: tclsqlite-shell.lo $(SQLITE3C) $(SQLITE3H) $(LIBRESOBJS) $(LTLINK) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /OUT:$@ tclsqlite-shell.lo $(LIBRESOBJS) $(LTLIBS) $(TLIBS) # Rules to build opcodes.c and opcodes.h @@ -1659,8 +1681,8 @@ parse.c: $(TOP)\src\parse.y lemon.exe $(TOP)\tool\addopcodes.tcl move parse.h parse.h.temp $(TCLSH_CMD) $(TOP)\tool\addopcodes.tcl parse.h.temp > parse.h -sqlite3.h: $(TOP)\src\sqlite.h.in $(TOP)\manifest.uuid $(TOP)\VERSION - $(TCLSH_CMD) $(TOP)\tool\mksqlite3h.tcl $(TOP:\=/) > sqlite3.h +$(SQLITE3H): $(TOP)\src\sqlite.h.in $(TOP)\manifest.uuid $(TOP)\VERSION + $(TCLSH_CMD) $(TOP)\tool\mksqlite3h.tcl $(TOP:\=/) > $(SQLITE3H) sqlite3ext.h: .target_source copy tsrc\sqlite3ext.h . @@ -1800,7 +1822,7 @@ TESTFIXTURE_SRC = $(TESTSRC) $(TOP)\src\tclsqlite.c $(TESTFIXTURE_SRC0) TESTFIXTURE_SRC = $(TESTSRC) $(TOP)\src\tclsqlite.c $(TESTFIXTURE_SRC1) !ENDIF -testfixture.exe: $(TESTFIXTURE_SRC) $(LIBRESOBJS) $(HDR) +testfixture.exe: $(TESTFIXTURE_SRC) $(SQLITE3H) $(LIBRESOBJS) $(HDR) $(LTLINK) -DSQLITE_NO_SYNC=1 $(TESTFIXTURE_FLAGS) \ -DBUILD_sqlite -I$(TCLINCDIR) \ $(TESTFIXTURE_SRC) \ @@ -1849,7 +1871,7 @@ smoketest: $(TESTPROGS) @set PATH=$(LIBTCLPATH);$(PATH) .\testfixture.exe $(TOP)\test\main.test $(TESTOPTS) -sqlite3_analyzer.c: $(SQLITE3C) $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl +sqlite3_analyzer.c: $(SQLITE3C) $(SQLITE3H) $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl echo #define TCLSH 2 > $@ echo #define SQLITE_ENABLE_DBSTAT_VTAB 1 >> $@ copy $@ + $(SQLITE3C) + $(TOP)\src\tclsqlite.c $@ @@ -1868,49 +1890,49 @@ testloadext.lo: $(TOP)\src\test_loadext.c testloadext.dll: testloadext.lo $(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL /OUT:$@ testloadext.lo -showdb.exe: $(TOP)\tool\showdb.c $(SQLITE3C) +showdb.exe: $(TOP)\tool\showdb.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ $(TOP)\tool\showdb.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) -showstat4.exe: $(TOP)\tool\showstat4.c $(SQLITE3C) +showstat4.exe: $(TOP)\tool\showstat4.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ $(TOP)\tool\showstat4.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) -showjournal.exe: $(TOP)\tool\showjournal.c $(SQLITE3C) +showjournal.exe: $(TOP)\tool\showjournal.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ $(TOP)\tool\showjournal.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) -showwal.exe: $(TOP)\tool\showwal.c $(SQLITE3C) +showwal.exe: $(TOP)\tool\showwal.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ $(TOP)\tool\showwal.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) -fts3view.exe: $(TOP)\ext\fts3\tool\fts3view.c $(SQLITE3C) +fts3view.exe: $(TOP)\ext\fts3\tool\fts3view.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ $(TOP)\ext\fts3\tool\fts3view.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) -rollback-test.exe: $(TOP)\tool\rollback-test.c $(SQLITE3C) +rollback-test.exe: $(TOP)\tool\rollback-test.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ $(TOP)\tool\rollback-test.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) -LogEst.exe: $(TOP)\tool\logest.c sqlite3.h +LogEst.exe: $(TOP)\tool\logest.c $(SQLITE3H) $(LTLINK) $(NO_WARN) -Fe$@ $(TOP)\tool\LogEst.c /link $(LDFLAGS) $(LTLINKOPTS) -wordcount.exe: $(TOP)\test\wordcount.c $(SQLITE3C) +wordcount.exe: $(TOP)\test\wordcount.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ $(TOP)\test\wordcount.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) -speedtest1.exe: $(TOP)\test\speedtest1.c $(SQLITE3C) +speedtest1.exe: $(TOP)\test\speedtest1.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ $(TOP)\test\speedtest1.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) -rbu.exe: $(TOP)\ext\rbu\rbu.c $(TOP)\ext\rbu\sqlite3rbu.c $(SQLITE3C) - $(LTLINK) $(NO_WARN) -DSQLITE_ENABLE_RBU -Fe$@ $(TOP)\ext\rbu\rbu.c $(SQLITE3C) \ - /link $(LDFLAGS) $(LTLINKOPTS) +rbu.exe: $(TOP)\ext\rbu\rbu.c $(TOP)\ext\rbu\sqlite3rbu.c $(SQLITE3C) $(SQLITE3H) + $(LTLINK) $(NO_WARN) -DSQLITE_ENABLE_RBU -Fe$@ \ + $(TOP)\ext\rbu\rbu.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) clean: del /Q *.exp *.lo *.ilk *.lib *.obj *.ncb *.pdb *.sdf *.suo 2>NUL del /Q *.bsc *.cod *.da *.bb *.bbg gmon.out 2>NUL - del /Q sqlite3.h opcodes.c opcodes.h 2>NUL + del /Q $(SQLITE3C) $(SQLITE3H) opcodes.c opcodes.h 2>NUL del /Q lemon.* lempar.c parse.* 2>NUL del /Q mkkeywordhash.* keywordhash.h 2>NUL del /Q notasharedlib.* 2>NUL diff --git a/manifest b/manifest index 67cd5b4802..c364abf8fa 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,8 @@ -C MSVC\smakefile\senhancements\sfor\suse\swith\sthe\sWindows\s10\sSDK. -D 2016-01-22T01:25:15.110 +C Further\srefinements\sto\sthe\sMSVC\smakefile. +D 2016-01-22T03:54:36.710 F Makefile.in 027c1603f255390c43a426671055a31c0a65fdb4 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 -F Makefile.msc 89b2e9780d97a6a3d7c89c4ad04d093787be3dc2 +F Makefile.msc 7e45640ca31116b24372d30a4e087078db22fcd8 F README.md 8ecc12493ff9f820cdea6520a9016001cb2e59b7 F VERSION 866588d1edf0ccb5b0d33896974338f97564f719 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -1419,7 +1419,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 29a51c6dcc0ef7cab51aba0024e9c5bec89c6973 -R 9e26ae539c690ec3cae729a6d81b3775 +P ac2e9d3de54d299f92fc35927b6df6d3aaa2dc59 +R 2fd39a9f583616f965419daf8ad2c68a U mistachkin -Z d4ab6d93e282614aa0160682fbcc0beb +Z 2b921b34884313de5b54030da9770cbb diff --git a/manifest.uuid b/manifest.uuid index 5f1ffc501a..51b2a1dd73 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ac2e9d3de54d299f92fc35927b6df6d3aaa2dc59 \ No newline at end of file +67625b963a6f5a0c6dff1a04a323e693569c2d94 \ No newline at end of file From d7d305a25aaa60ce55f9e6ec1840b39309282015 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Fri, 22 Jan 2016 04:22:36 +0000 Subject: [PATCH 04/39] Enable FTS4 when building the shell with the MSVC makefile. FossilOrigin-Name: e4c07df557cd50786b05eecf011bf94708e6e31b --- Makefile.msc | 2 +- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Makefile.msc b/Makefile.msc index a75045fc98..3621d4f92b 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1308,7 +1308,7 @@ FUZZDATA = \ # when the shell is not being dynamically linked. # !IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0 -SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS5 +SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS5 !ENDIF # Extra compiler options for various test tools. diff --git a/manifest b/manifest index c364abf8fa..5d0ee23cca 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,8 @@ -C Further\srefinements\sto\sthe\sMSVC\smakefile. -D 2016-01-22T03:54:36.710 +C Enable\sFTS4\swhen\sbuilding\sthe\sshell\swith\sthe\sMSVC\smakefile. +D 2016-01-22T04:22:36.961 F Makefile.in 027c1603f255390c43a426671055a31c0a65fdb4 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 -F Makefile.msc 7e45640ca31116b24372d30a4e087078db22fcd8 +F Makefile.msc d2b93511a969c0c8fcf52aeb5e426571e8c610d2 F README.md 8ecc12493ff9f820cdea6520a9016001cb2e59b7 F VERSION 866588d1edf0ccb5b0d33896974338f97564f719 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -1419,7 +1419,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ac2e9d3de54d299f92fc35927b6df6d3aaa2dc59 -R 2fd39a9f583616f965419daf8ad2c68a +P 67625b963a6f5a0c6dff1a04a323e693569c2d94 +R a9194694e295a05f59bbd77f1c470bc0 U mistachkin -Z 2b921b34884313de5b54030da9770cbb +Z bf1c280948a8257fdbbacb5c97da93b0 diff --git a/manifest.uuid b/manifest.uuid index 51b2a1dd73..533751ec8a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -67625b963a6f5a0c6dff1a04a323e693569c2d94 \ No newline at end of file +e4c07df557cd50786b05eecf011bf94708e6e31b \ No newline at end of file From b5c557b87e0e54f39ade09a73dc0b7cb13d2c93d Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 22 Jan 2016 15:44:07 +0000 Subject: [PATCH 05/39] Have the dbstat module dequote any argument passed to the CREATE VIRTUAL TABLE statement before attempting to match it against the names of attached databases. FossilOrigin-Name: e60461e984b8df09256bb0d733dbfae52568a145 --- manifest | 16 ++++++------ manifest.uuid | 2 +- src/dbstat.c | 6 ++++- test/stat.test | 66 ++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 80 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 5d0ee23cca..c4697a9a83 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enable\sFTS4\swhen\sbuilding\sthe\sshell\swith\sthe\sMSVC\smakefile. -D 2016-01-22T04:22:36.961 +C Have\sthe\sdbstat\smodule\sdequote\sany\sargument\spassed\sto\sthe\sCREATE\sVIRTUAL\sTABLE\sstatement\sbefore\sattempting\sto\smatch\sit\sagainst\sthe\snames\sof\sattached\sdatabases. +D 2016-01-22T15:44:07.317 F Makefile.in 027c1603f255390c43a426671055a31c0a65fdb4 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc d2b93511a969c0c8fcf52aeb5e426571e8c610d2 @@ -296,7 +296,7 @@ F src/callback.c 29ae4faba226c7ebb9aee93016b5ce8a8f071261 F src/complete.c addcd8160b081131005d5bc2d34adf20c1c5c92f F src/ctime.c 60e135af364d777a9ab41c97e5e89cd224da6198 F src/date.c 997651e3ee6c2818fbf7fcdb7156cef9eb3ece20 -F src/dbstat.c ffd63fc8ba7541476ced189b95e95d7f2bc63f78 +F src/dbstat.c d33af6b426449d7bfd5b3d9f77911c20d1c8abdc F src/delete.c 00af9f08a15ddc5cba5962d3d3e5bf2d67b2e7da F src/expr.c df0d7c3230d59abd679da22ff5ce4cfd0e3a0e63 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb @@ -1052,7 +1052,7 @@ F test/spellfix3.test f7bf7b3482971473d32b6b00f6944c5c066cff97 F test/sqldiff1.test 8f6bc7c6a5b3585d350d779c6078869ba402f8f5 F test/sqllimits1.test a74ee2a3740b9f9c2437c246d8fb77354862a142 F test/sqllog.test a8faa2df39610a037dd372ed872d124260d32953 -F test/stat.test 8de91498c99f5298b303f70f1d1f3b9557af91bf +F test/stat.test fafe6e82dfdb97d8c8be31cd83e36e973079ce0f F test/statfault.test f525a7bf633e50afd027700e9a486090684b1ac1 F test/stmt.test 25d64e3dbf9a3ce89558667d7f39d966fe2a71b9 F test/subquery.test d7268d193dd33d5505df965399d3a594e76ae13f @@ -1419,7 +1419,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 67625b963a6f5a0c6dff1a04a323e693569c2d94 -R a9194694e295a05f59bbd77f1c470bc0 -U mistachkin -Z bf1c280948a8257fdbbacb5c97da93b0 +P e4c07df557cd50786b05eecf011bf94708e6e31b +R ccbd95b5668f9b5a41b204404932d800 +U dan +Z ad5fc4f27f7e976928da8b4460d48d82 diff --git a/manifest.uuid b/manifest.uuid index 533751ec8a..800a36102e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e4c07df557cd50786b05eecf011bf94708e6e31b \ No newline at end of file +e60461e984b8df09256bb0d733dbfae52568a145 \ No newline at end of file diff --git a/src/dbstat.c b/src/dbstat.c index ae55d6b803..e404c71dd3 100644 --- a/src/dbstat.c +++ b/src/dbstat.c @@ -149,7 +149,11 @@ static int statConnect( int iDb; if( argc>=4 ){ - iDb = sqlite3FindDbName(db, argv[3]); + char *zDb = sqlite3DbStrDup(db, argv[3]); + if( zDb==0 ) return SQLITE_NOMEM; + sqlite3Dequote(zDb); + iDb = sqlite3FindDbName(db, zDb); + sqlite3DbFree(db, zDb); if( iDb<0 ){ *pzErr = sqlite3_mprintf("no such database: %s", argv[3]); return SQLITE_ERROR; diff --git a/test/stat.test b/test/stat.test index 57c1b9eae1..14751bd6ff 100644 --- a/test/stat.test +++ b/test/stat.test @@ -14,6 +14,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +set testprefix stat ifcapable !vtab||!compound { finish_test @@ -184,4 +185,69 @@ do_catchsql_test stat-6.1 { CREATE VIRTUAL TABLE temp.s2 USING dbstat(mainx); } {1 {no such database: mainx}} +#------------------------------------------------------------------------- +# Test that the argument passed to the dbstat constructor is dequoted +# before it is matched against the names of attached databases. +# +forcedelete test.db2 +do_execsql_test 7.1 { + ATTACH 'test.db2' AS '123'; + CREATE TABLE "123".x1(a, b); + INSERT INTO x1 VALUES(1, 2); +} + +do_execsql_test 7.1.1 { + SELECT * FROM dbstat('123'); +} { + sqlite_master / 1 leaf 1 37 875 37 0 1024 + x1 / 2 leaf 1 4 1008 4 1024 1024 +} +do_execsql_test 7.1.2 { + SELECT * FROM dbstat(123); +} { + sqlite_master / 1 leaf 1 37 875 37 0 1024 + x1 / 2 leaf 1 4 1008 4 1024 1024 +} +do_execsql_test 7.1.3 { + CREATE VIRTUAL TABLE x2 USING dbstat('123'); + SELECT * FROM x2; +} { + sqlite_master / 1 leaf 1 37 875 37 0 1024 + x1 / 2 leaf 1 4 1008 4 1024 1024 +} +do_execsql_test 7.1.4 { + CREATE VIRTUAL TABLE x3 USING dbstat(123); + SELECT * FROM x3; +} { + sqlite_master / 1 leaf 1 37 875 37 0 1024 + x1 / 2 leaf 1 4 1008 4 1024 1024 +} + +do_execsql_test 7.2 { + DETACH 123; + DROP TABLE x2; + DROP TABLE x3; + ATTACH 'test.db2' AS '123corp'; +} +do_execsql_test 7.2.1 { + SELECT * FROM dbstat('123corp'); +} { + sqlite_master / 1 leaf 1 37 875 37 0 1024 + x1 / 2 leaf 1 4 1008 4 1024 1024 +} +do_catchsql_test 7.2.2 { + SELECT * FROM dbstat(123corp); +} {1 {unrecognized token: "123corp"}} +do_execsql_test 7.2.3 { + CREATE VIRTUAL TABLE x2 USING dbstat('123corp'); + SELECT * FROM x2; +} { + sqlite_master / 1 leaf 1 37 875 37 0 1024 + x1 / 2 leaf 1 4 1008 4 1024 1024 +} +do_catchsql_test 7.2.4 { + CREATE VIRTUAL TABLE x3 USING dbstat(123corp); + SELECT * FROM x3; +} {1 {unrecognized token: "123corp"}} + finish_test From 40aced5c654a3f3322f49fa9e4b2116f43167790 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 22 Jan 2016 17:48:09 +0000 Subject: [PATCH 06/39] Add the sqlite3TokenInit() utility function. FossilOrigin-Name: 7323175337b7ba85ac932ca892b28860f6a5b688 --- manifest | 28 ++++++++++++++-------------- manifest.uuid | 2 +- src/build.c | 6 ++---- src/dbstat.c | 8 +++----- src/expr.c | 3 +-- src/fkey.c | 8 +++----- src/select.c | 3 +-- src/sqliteInt.h | 1 + src/trigger.c | 3 +-- src/util.c | 8 ++++++++ 10 files changed, 35 insertions(+), 35 deletions(-) diff --git a/manifest b/manifest index c4697a9a83..6c3814c3d9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Have\sthe\sdbstat\smodule\sdequote\sany\sargument\spassed\sto\sthe\sCREATE\sVIRTUAL\sTABLE\sstatement\sbefore\sattempting\sto\smatch\sit\sagainst\sthe\snames\sof\sattached\sdatabases. -D 2016-01-22T15:44:07.317 +C Add\sthe\ssqlite3TokenInit()\sutility\sfunction. +D 2016-01-22T17:48:09.470 F Makefile.in 027c1603f255390c43a426671055a31c0a65fdb4 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc d2b93511a969c0c8fcf52aeb5e426571e8c610d2 @@ -291,16 +291,16 @@ F src/btmutex.c bc87dd3b062cc26edfe79918de2200ccb8d41e73 F src/btree.c f224ae877fde69d1a9d430f502edaf8502752dbe F src/btree.h 526137361963e746949ab966a910c7f455ac6b04 F src/btreeInt.h c18b7d2a3494695133e4e60ee36061d37f45d9a5 -F src/build.c 31af80bba31ac159967951ef58f3144cc7db9d70 +F src/build.c b4eba1e84752ec9cae7ff3dacd5a8b6d1ab8deb9 F src/callback.c 29ae4faba226c7ebb9aee93016b5ce8a8f071261 F src/complete.c addcd8160b081131005d5bc2d34adf20c1c5c92f F src/ctime.c 60e135af364d777a9ab41c97e5e89cd224da6198 F src/date.c 997651e3ee6c2818fbf7fcdb7156cef9eb3ece20 -F src/dbstat.c d33af6b426449d7bfd5b3d9f77911c20d1c8abdc +F src/dbstat.c b2ec6793eef97aebb4d171d490a4ffdfa9f2475c F src/delete.c 00af9f08a15ddc5cba5962d3d3e5bf2d67b2e7da -F src/expr.c df0d7c3230d59abd679da22ff5ce4cfd0e3a0e63 +F src/expr.c d10c1cdef5810cdbf73adc9f9b383684230b360a F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb -F src/fkey.c e18b3dff7d47c7bcac5ac4fc178a89b9fd322b44 +F src/fkey.c c66d3e5b35d4d95b5c1e2ee6c12f5df13a7f9ad6 F src/func.c ba6c03f9e440f5693086c08ee88e6e60212b3504 F src/global.c bd5a0af3f30b0c01be6db756c626cd3c33a3d260 F src/hash.c 4263fbc955f26c2e8cdc0cf214bc42435aa4e4f5 @@ -345,12 +345,12 @@ F src/printf.c af589a27b7d40f6f4f704e9eea99f02f18ad6d32 F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 F src/resolve.c 9f7ce3a3c087afb7597b7c916c99126ff3f12f0c F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e -F src/select.c 718954db86277d696c520fe671148db1e9c4ed3c +F src/select.c 1db9a902e89201a0ae3ff5243a7a3f37842a3937 F src/shell.c dcd7a83645ef2a58ee9c6d0ea4714d877d7835c4 F src/sqlite.h.in 214476a62012e578f42133a9a3b4f97a9aa421a3 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h dfbe62ffd95b99afe2140d8c35b180d11924072d -F src/sqliteInt.h 0403328581127bc8ad2f9cc7af2c3bb23d5316da +F src/sqliteInt.h 74e10a74116df0aec9d4a3e134f1a86cc34c2f14 F src/sqliteLimit.h 216557999cb45f2e3578ed53ebefe228d779cb46 F src/status.c 70912d7be68e9e2dbc4010c93d344af61d4c59ba F src/table.c 51b46b2a62d1b3a959633d593b89bab5e2c9155e @@ -405,10 +405,10 @@ F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c bbfb74450643cb5372a43ad4f6cffd7e9dfcecb0 F src/tokenize.c 5606871a377f390af7040ec3c12e0d183512d785 F src/treeview.c 78842e90c1f71269e7a73a1d4221b6fe360bab66 -F src/trigger.c 056e51182a3677434423e3be0c74e61b90b4a663 +F src/trigger.c 72d876b2d0c66604a112362bdae07dae9b104816 F src/update.c 17332f9fe818cbc0444c36a811800af8498af4c3 F src/utf.c 32d7f82aa921322f3e1c956f4b58f019ebd2c6b3 -F src/util.c e802e8e311a0d6c48cd1b3e89db164f6f0248d70 +F src/util.c 72d40df0a52d3f30b462a15f0e094fcbade6dc82 F src/vacuum.c 2ddd5cad2a7b9cef7f9e431b8c7771634c6b1701 F src/vdbe.c b90d9d38e5e0260c2eafa3cb4c2274d8ea94da27 F src/vdbe.h 7a733ea8aac1b77305a67698e784fa3484ee3337 @@ -1419,7 +1419,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P e4c07df557cd50786b05eecf011bf94708e6e31b -R ccbd95b5668f9b5a41b204404932d800 -U dan -Z ad5fc4f27f7e976928da8b4460d48d82 +P e60461e984b8df09256bb0d733dbfae52568a145 +R 03ab6310cefadc63b5c7c7da966c2348 +U drh +Z 0c42ba520d71deaa3e0a21b23ff67f9d diff --git a/manifest.uuid b/manifest.uuid index 800a36102e..e9fd60917b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e60461e984b8df09256bb0d733dbfae52568a145 \ No newline at end of file +7323175337b7ba85ac932ca892b28860f6a5b688 \ No newline at end of file diff --git a/src/build.c b/src/build.c index f7074ab417..06f5433e37 100644 --- a/src/build.c +++ b/src/build.c @@ -1708,8 +1708,7 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ if( pTab->iPKey>=0 ){ ExprList *pList; Token ipkToken; - ipkToken.z = pTab->aCol[pTab->iPKey].zName; - ipkToken.n = sqlite3Strlen30(ipkToken.z); + sqlite3TokenInit(&ipkToken, pTab->aCol[pTab->iPKey].zName); pList = sqlite3ExprListAppend(pParse, 0, sqlite3ExprAlloc(db, TK_ID, &ipkToken, 0)); if( pList==0 ) return; @@ -3057,8 +3056,7 @@ Index *sqlite3CreateIndex( */ if( pList==0 ){ Token prevCol; - prevCol.z = pTab->aCol[pTab->nCol-1].zName; - prevCol.n = sqlite3Strlen30(prevCol.z); + sqlite3TokenInit(&prevCol, pTab->aCol[pTab->nCol-1].zName); pList = sqlite3ExprListAppend(pParse, 0, sqlite3ExprAlloc(db, TK_ID, &prevCol, 0)); if( pList==0 ) goto exit_create_index; diff --git a/src/dbstat.c b/src/dbstat.c index e404c71dd3..5e42cdfe38 100644 --- a/src/dbstat.c +++ b/src/dbstat.c @@ -149,11 +149,9 @@ static int statConnect( int iDb; if( argc>=4 ){ - char *zDb = sqlite3DbStrDup(db, argv[3]); - if( zDb==0 ) return SQLITE_NOMEM; - sqlite3Dequote(zDb); - iDb = sqlite3FindDbName(db, zDb); - sqlite3DbFree(db, zDb); + Token nm; + sqlite3TokenInit(&nm, (char*)argv[3]); + iDb = sqlite3FindDb(db, &nm); if( iDb<0 ){ *pzErr = sqlite3_mprintf("no such database: %s", argv[3]); return SQLITE_ERROR; diff --git a/src/expr.c b/src/expr.c index 403e81cf34..c91db2834f 100644 --- a/src/expr.c +++ b/src/expr.c @@ -85,8 +85,7 @@ Expr *sqlite3ExprAddCollateToken( Expr *sqlite3ExprAddCollateString(Parse *pParse, Expr *pExpr, const char *zC){ Token s; assert( zC!=0 ); - s.z = zC; - s.n = sqlite3Strlen30(s.z); + sqlite3TokenInit(&s, (char*)zC); return sqlite3ExprAddCollateToken(pParse, pExpr, &s, 0); } diff --git a/src/fkey.c b/src/fkey.c index 2abd06c693..7eb188924d 100644 --- a/src/fkey.c +++ b/src/fkey.c @@ -1192,11 +1192,9 @@ static Trigger *fkActionTrigger( assert( iFromCol>=0 ); assert( pIdx!=0 || (pTab->iPKey>=0 && pTab->iPKeynCol) ); assert( pIdx==0 || pIdx->aiColumn[i]>=0 ); - tToCol.z = pTab->aCol[pIdx ? pIdx->aiColumn[i] : pTab->iPKey].zName; - tFromCol.z = pFKey->pFrom->aCol[iFromCol].zName; - - tToCol.n = sqlite3Strlen30(tToCol.z); - tFromCol.n = sqlite3Strlen30(tFromCol.z); + sqlite3TokenInit(&tToCol, + pTab->aCol[pIdx ? pIdx->aiColumn[i] : pTab->iPKey].zName); + sqlite3TokenInit(&tFromCol, pFKey->pFrom->aCol[iFromCol].zName); /* Create the expression "OLD.zToCol = zFromCol". It is important ** that the "OLD.zToCol" term is on the LHS of the = operator, so diff --git a/src/select.c b/src/select.c index 9eb6279c10..ffc0ec2415 100644 --- a/src/select.c +++ b/src/select.c @@ -4438,8 +4438,7 @@ static int selectExpander(Walker *pWalker, Select *p){ pExpr = pRight; } pNew = sqlite3ExprListAppend(pParse, pNew, pExpr); - sColname.z = zColname; - sColname.n = sqlite3Strlen30(zColname); + sqlite3TokenInit(&sColname, zColname); sqlite3ExprListSetName(pParse, pNew, &sColname, 0); if( pNew && (p->selFlags & SF_NestedFrom)!=0 ){ struct ExprList_item *pX = &pNew->a[pNew->nExpr-1]; diff --git a/src/sqliteInt.h b/src/sqliteInt.h index b536b8ab28..1a8a5d3a5b 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3322,6 +3322,7 @@ char *sqlite3VMPrintf(sqlite3*,const char*, va_list); void sqlite3SetString(char **, sqlite3*, const char*); void sqlite3ErrorMsg(Parse*, const char*, ...); int sqlite3Dequote(char*); +void sqlite3TokenInit(Token*,char*); int sqlite3KeywordCode(const unsigned char*, int); int sqlite3RunParser(Parse*, const char*, char **); void sqlite3FinishCoding(Parse*); diff --git a/src/trigger.c b/src/trigger.c index 48d6772992..4bfb55af69 100644 --- a/src/trigger.c +++ b/src/trigger.c @@ -287,8 +287,7 @@ void sqlite3FinishTrigger( pStepList->pTrig = pTrig; pStepList = pStepList->pNext; } - nameToken.z = pTrig->zName; - nameToken.n = sqlite3Strlen30(nameToken.z); + sqlite3TokenInit(&nameToken, pTrig->zName); sqlite3FixInit(&sFix, pParse, iDb, "trigger", &nameToken); if( sqlite3FixTriggerStep(&sFix, pTrig->step_list) || sqlite3FixExpr(&sFix, pTrig->pWhen) diff --git a/src/util.c b/src/util.c index b4c5e62bb3..96f7c7f3dc 100644 --- a/src/util.c +++ b/src/util.c @@ -234,6 +234,14 @@ int sqlite3Dequote(char *z){ return j; } +/* +** Generate a Token object from a string +*/ +void sqlite3TokenInit(Token *p, char *z){ + p->z = z; + p->n = sqlite3Strlen30(z); +} + /* Convenient short-hand */ #define UpperToLower sqlite3UpperToLower From c3dfa5eba74db8eae0dddff1424b24e7f5873dbe Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 22 Jan 2016 19:44:03 +0000 Subject: [PATCH 07/39] Reenable the xCurrentTime and xGetLastError methods on the unix VFSes as long as SQLITE_OMIT_DEPRECATED is not defined. FossilOrigin-Name: c11c85fdb6514cae54bb44945cc197dcaba72307 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_unix.c | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 6c3814c3d9..566db8de82 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\ssqlite3TokenInit()\sutility\sfunction. -D 2016-01-22T17:48:09.470 +C Reenable\sthe\sxCurrentTime\sand\sxGetLastError\smethods\son\sthe\sunix\sVFSes\sas\slong\nas\sSQLITE_OMIT_DEPRECATED\sis\snot\sdefined. +D 2016-01-22T19:44:03.065 F Makefile.in 027c1603f255390c43a426671055a31c0a65fdb4 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc d2b93511a969c0c8fcf52aeb5e426571e8c610d2 @@ -329,7 +329,7 @@ F src/os.c 8fd25588eeba74068d41102d26810e216999b6c8 F src/os.h 3e57a24e2794a94d3cf2342c6d9a884888cd96bf F src/os_common.h abdb9a191a367793268fe553d25bab894e986a0e F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa -F src/os_unix.c b509b49b40a269e7b75ab511b6e92b2dc9444359 +F src/os_unix.c 0eb7f469fcd4e1fbedf30060438e26b839ec5486 F src/os_win.c 386fba30419e8458b13209781c2af5590eab2811 F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca F src/pager.c f4e9ac39fbb1e0fde97af85c0f4e00eb90764b67 @@ -1419,7 +1419,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P e60461e984b8df09256bb0d733dbfae52568a145 -R 03ab6310cefadc63b5c7c7da966c2348 +P 7323175337b7ba85ac932ca892b28860f6a5b688 +R 4fcfc780a1da751b6f90e6d6f3ced841 U drh -Z 0c42ba520d71deaa3e0a21b23ff67f9d +Z 703592c4495d8a71fd67fbd54e7f3927 diff --git a/manifest.uuid b/manifest.uuid index e9fd60917b..a853332581 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7323175337b7ba85ac932ca892b28860f6a5b688 \ No newline at end of file +c11c85fdb6514cae54bb44945cc197dcaba72307 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 5bd6116945..ee9b55674b 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -6187,7 +6187,7 @@ static int unixCurrentTimeInt64(sqlite3_vfs *NotUsed, sqlite3_int64 *piNow){ return rc; } -#if 0 /* Not used */ +#ifndef SQLITE_OMIT_DEPRECATED /* ** Find the current time (in Universal Coordinated Time). Write the ** current time and date as a Julian Day number into *prNow and @@ -6205,7 +6205,7 @@ static int unixCurrentTime(sqlite3_vfs *NotUsed, double *prNow){ # define unixCurrentTime 0 #endif -#if 0 /* Not used */ +#ifndef SQLITE_OMIT_DEPRECATED /* ** We added the xGetLastError() method with the intention of providing ** better low-level error messages when operating-system problems come up From 795502d26c770c06b1d8d586fd316a6e374ed1d6 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 22 Jan 2016 19:48:34 +0000 Subject: [PATCH 08/39] Experimental performance enhancements for fts5. FossilOrigin-Name: b5a57b812fd6a734cf2a342bf0b730ae18912d73 --- ext/fts5/fts5Int.h | 10 +- ext/fts5/fts5_expr.c | 42 ++-- ext/fts5/fts5_index.c | 389 +++++++++++++++++++++++++++----------- ext/fts5/fts5_vocab.c | 56 +++--- ext/fts5/test/fts5ac.test | 1 - manifest | 25 +-- manifest.uuid | 2 +- 7 files changed, 355 insertions(+), 170 deletions(-) diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index b2f0d6c34e..7c50da9656 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -315,6 +315,12 @@ void sqlite3Fts5TermsetFree(Fts5Termset*); typedef struct Fts5Index Fts5Index; typedef struct Fts5IndexIter Fts5IndexIter; +struct Fts5IndexIter { + i64 iRowid; + const u8 *pData; + int nData; +}; + /* ** Values used as part of the flags argument passed to IndexQuery(). */ @@ -396,6 +402,8 @@ void sqlite3Fts5IterClose(Fts5IndexIter*); const char *sqlite3Fts5IterTerm(Fts5IndexIter*, int*); int sqlite3Fts5IterNextScan(Fts5IndexIter*); +int sqlite3Fts5IterCollist(Fts5IndexIter*, const u8 **, int*); + /* ** Insert or remove data to or from the index. Each time a document is @@ -469,8 +477,6 @@ int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge); int sqlite3Fts5IndexLoadConfig(Fts5Index *p); -int sqlite3Fts5IterCollist(Fts5IndexIter*, const u8 **, int*); - /* ** End of interface to code in fts5_index.c. **************************************************************************/ diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index c51ed7939c..290baf4e2c 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -323,11 +323,15 @@ static int fts5ExprSynonymList( const u8 *a; int n; - if( bCollist ){ + if( 0 && bCollist ){ rc = sqlite3Fts5IterCollist(pIter, &a, &n); }else{ i64 dummy; +#if 0 rc = sqlite3Fts5IterPoslist(pIter, pColset, &a, &n, &dummy); +#endif + a = pIter->pData; + n = pIter->nData; } if( rc!=SQLITE_OK ) goto synonym_poslist_out; @@ -436,7 +440,12 @@ static int fts5ExprPhraseIsMatch( pTerm, 0, pColset, pNode->iRowid, &bFlag, (u8**)&a, &n ); }else{ + Fts5IndexIter *pIter = pTerm->pIter; +#if 0 rc = sqlite3Fts5IterPoslist(pTerm->pIter, pColset, &a, &n, &dummy); +#endif + a = pIter->pData; + n = pIter->nData; } if( rc!=SQLITE_OK ) goto ismatch_out; sqlite3Fts5PoslistReaderInit(a, n, &aIter[i]); @@ -775,6 +784,7 @@ static int fts5ExprNearTest( for(pTerm=&pPhrase->aTerm[0]; pTerm; pTerm=pTerm->pSynonym){ Fts5IndexIter *pIter = pTerm->pIter; if( sqlite3Fts5IterEof(pIter)==0 ){ +#if 0 int n; i64 iRowid; rc = sqlite3Fts5IterPoslist(pIter, pNear->pColset, 0, &n, &iRowid); @@ -784,6 +794,10 @@ static int fts5ExprNearTest( }else if( iRowid==pNode->iRowid && n>0 ){ pPhrase->poslist.n = 1; } +#endif + if( pIter->iRowid==pNode->iRowid && pIter->nData>0 ){ + pPhrase->poslist.n = 1; + } } } return pPhrase->poslist.n; @@ -800,9 +814,13 @@ static int fts5ExprNearTest( rc = fts5ExprPhraseIsMatch(pNode, pNear->pColset, pPhrase, &bMatch); if( bMatch==0 ) break; }else{ + Fts5IndexIter *pIter = pPhrase->aTerm[0].pIter; + fts5BufferSet(&rc, &pPhrase->poslist, pIter->nData, pIter->pData); +#if 0 rc = sqlite3Fts5IterPoslistBuffer( pPhrase->aTerm[0].pIter, &pPhrase->poslist ); +#endif } } @@ -823,21 +841,20 @@ static int fts5ExprTokenTest( ** fts5_index.c iterator object. This is much faster than synthesizing ** a new poslist the way we have to for more complicated phrase or NEAR ** expressions. */ - Fts5ExprNearset *pNear = pNode->pNear; - Fts5ExprPhrase *pPhrase = pNear->apPhrase[0]; + Fts5ExprPhrase *pPhrase = pNode->pNear->apPhrase[0]; Fts5IndexIter *pIter = pPhrase->aTerm[0].pIter; - Fts5Colset *pColset = pNear->pColset; - int rc; assert( pNode->eType==FTS5_TERM ); - assert( pNear->nPhrase==1 && pPhrase->nTerm==1 ); + assert( pNode->pNear->nPhrase==1 && pPhrase->nTerm==1 ); assert( pPhrase->aTerm[0].pSynonym==0 ); - rc = sqlite3Fts5IterPoslist(pIter, pColset, - (const u8**)&pPhrase->poslist.p, (int*)&pPhrase->poslist.n, &pNode->iRowid - ); + pPhrase->poslist.n = pIter->nData; + if( pExpr->pConfig->eDetail==FTS5_DETAIL_FULL ){ + pPhrase->poslist.p = (u8*)pIter->pData; + } + pNode->iRowid = pIter->iRowid; pNode->bNomatch = (pPhrase->poslist.n==0); - return rc; + return SQLITE_OK; } /* @@ -2484,6 +2501,8 @@ int sqlite3Fts5ExprPhraseCollist( int rc = SQLITE_OK; assert( iPhrase>=0 && iPhrasenPhrase ); + assert( pExpr->pConfig->eDetail==FTS5_DETAIL_COLUMNS ); + if( pNode->bEof==0 && pNode->iRowid==pExpr->pRoot->iRowid && pPhrase->poslist.n>0 @@ -2503,7 +2522,8 @@ int sqlite3Fts5ExprPhraseCollist( *ppCollist = a; } }else{ - sqlite3Fts5IterCollist(pPhrase->aTerm[0].pIter, ppCollist, pnCollist); + *ppCollist = pPhrase->aTerm[0].pIter->pData; + *pnCollist = pPhrase->aTerm[0].pIter->nData; } }else{ *ppCollist = 0; diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index fc11a23413..2ea57b04f9 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -261,6 +261,7 @@ typedef struct Fts5Data Fts5Data; typedef struct Fts5DlidxIter Fts5DlidxIter; typedef struct Fts5DlidxLvl Fts5DlidxLvl; typedef struct Fts5DlidxWriter Fts5DlidxWriter; +typedef struct Fts5Iter Fts5Iter; typedef struct Fts5PageWriter Fts5PageWriter; typedef struct Fts5SegIter Fts5SegIter; typedef struct Fts5DoclistIter Fts5DoclistIter; @@ -503,10 +504,16 @@ struct Fts5SegIter { ** Used by sqlite3Fts5IterPoslist() when the poslist needs to be buffered. ** There is no way to tell if this is populated or not. */ -struct Fts5IndexIter { +struct Fts5Iter { + Fts5IndexIter base; /* Base class containing output vars */ + Fts5Index *pIndex; /* Index that owns this iterator */ Fts5Structure *pStruct; /* Database structure for this iterator */ Fts5Buffer poslist; /* Buffer containing current poslist */ + Fts5Colset *pColset; /* Restrict matches to these columns */ + + /* Invoked to set output variables. */ + void (*xSetOutputs)(Fts5Iter*, Fts5SegIter*); int nSeg; /* Size of aSeg[] array */ int bRev; /* True to iterate in reverse order */ @@ -1752,7 +1759,7 @@ static void fts5SegIterReverseNewPage(Fts5Index *p, Fts5SegIter *pIter){ ** points to a delete marker. A delete marker is an entry with a 0 byte ** position-list. */ -static int fts5MultiIterIsEmpty(Fts5Index *p, Fts5IndexIter *pIter){ +static int fts5MultiIterIsEmpty(Fts5Index *p, Fts5Iter *pIter){ Fts5SegIter *pSeg = &pIter->aSeg[pIter->aFirst[1].iFirst]; return (p->rc==SQLITE_OK && pSeg->pLeaf && pSeg->nPos==0); } @@ -2406,7 +2413,7 @@ static void fts5SegIterClear(Fts5SegIter *pIter){ ** two iterators. */ static void fts5AssertComparisonResult( - Fts5IndexIter *pIter, + Fts5Iter *pIter, Fts5SegIter *p1, Fts5SegIter *p2, Fts5CResult *pRes @@ -2447,7 +2454,7 @@ static void fts5AssertComparisonResult( ** statement used to verify that the contents of the pIter->aFirst[] array ** are correct. */ -static void fts5AssertMultiIterSetup(Fts5Index *p, Fts5IndexIter *pIter){ +static void fts5AssertMultiIterSetup(Fts5Index *p, Fts5Iter *pIter){ if( p->rc==SQLITE_OK ){ Fts5SegIter *pFirst = &pIter->aSeg[ pIter->aFirst[1].iFirst ]; int i; @@ -2492,7 +2499,7 @@ static void fts5AssertMultiIterSetup(Fts5Index *p, Fts5IndexIter *pIter){ ** to a key that is a duplicate of another, higher priority, ** segment-iterator in the pSeg->aSeg[] array. */ -static int fts5MultiIterDoCompare(Fts5IndexIter *pIter, int iOut){ +static int fts5MultiIterDoCompare(Fts5Iter *pIter, int iOut){ int i1; /* Index of left-hand Fts5SegIter */ int i2; /* Index of right-hand Fts5SegIter */ int iRes; @@ -2638,7 +2645,7 @@ static void fts5SegIterNextFrom( /* ** Free the iterator object passed as the second argument. */ -static void fts5MultiIterFree(Fts5Index *p, Fts5IndexIter *pIter){ +static void fts5MultiIterFree(Fts5Index *p, Fts5Iter *pIter){ if( pIter ){ int i; for(i=0; inSeg; i++){ @@ -2652,7 +2659,7 @@ static void fts5MultiIterFree(Fts5Index *p, Fts5IndexIter *pIter){ static void fts5MultiIterAdvanced( Fts5Index *p, /* FTS5 backend to iterate within */ - Fts5IndexIter *pIter, /* Iterator to update aFirst[] array for */ + Fts5Iter *pIter, /* Iterator to update aFirst[] array for */ int iChanged, /* Index of sub-iterator just advanced */ int iMinset /* Minimum entry in aFirst[] to set */ ){ @@ -2680,8 +2687,9 @@ static void fts5MultiIterAdvanced( */ static int fts5MultiIterAdvanceRowid( Fts5Index *p, /* FTS5 backend to iterate within */ - Fts5IndexIter *pIter, /* Iterator to update aFirst[] array for */ - int iChanged /* Index of sub-iterator just advanced */ + Fts5Iter *pIter, /* Iterator to update aFirst[] array for */ + int iChanged, /* Index of sub-iterator just advanced */ + Fts5SegIter **ppFirst ){ Fts5SegIter *pNew = &pIter->aSeg[iChanged]; @@ -2714,13 +2722,14 @@ static int fts5MultiIterAdvanceRowid( } } + *ppFirst = pNew; return 0; } /* ** Set the pIter->bEof variable based on the state of the sub-iterators. */ -static void fts5MultiIterSetEof(Fts5IndexIter *pIter){ +static void fts5MultiIterSetEof(Fts5Iter *pIter){ Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ]; pIter->bEof = pSeg->pLeaf==0; pIter->iSwitchRowid = pSeg->iRowid; @@ -2735,39 +2744,44 @@ static void fts5MultiIterSetEof(Fts5IndexIter *pIter){ */ static void fts5MultiIterNext( Fts5Index *p, - Fts5IndexIter *pIter, + Fts5Iter *pIter, int bFrom, /* True if argument iFrom is valid */ i64 iFrom /* Advance at least as far as this */ ){ - if( p->rc==SQLITE_OK ){ - int bUseFrom = bFrom; - do { - int iFirst = pIter->aFirst[1].iFirst; - int bNewTerm = 0; - Fts5SegIter *pSeg = &pIter->aSeg[iFirst]; - assert( p->rc==SQLITE_OK ); - if( bUseFrom && pSeg->pDlidx ){ - fts5SegIterNextFrom(p, pSeg, iFrom); - }else{ - pSeg->xNext(p, pSeg, &bNewTerm); - } + int bUseFrom = bFrom; + while( p->rc==SQLITE_OK ){ + int iFirst = pIter->aFirst[1].iFirst; + int bNewTerm = 0; + Fts5SegIter *pSeg = &pIter->aSeg[iFirst]; + assert( p->rc==SQLITE_OK ); + if( bUseFrom && pSeg->pDlidx ){ + fts5SegIterNextFrom(p, pSeg, iFrom); + }else{ + pSeg->xNext(p, pSeg, &bNewTerm); + } - if( pSeg->pLeaf==0 || bNewTerm - || fts5MultiIterAdvanceRowid(p, pIter, iFirst) - ){ - fts5MultiIterAdvanced(p, pIter, iFirst, 1); - fts5MultiIterSetEof(pIter); - } - fts5AssertMultiIterSetup(p, pIter); + if( pSeg->pLeaf==0 || bNewTerm + || fts5MultiIterAdvanceRowid(p, pIter, iFirst, &pSeg) + ){ + fts5MultiIterAdvanced(p, pIter, iFirst, 1); + fts5MultiIterSetEof(pIter); + pSeg = &pIter->aSeg[pIter->aFirst[1].iFirst]; + if( pSeg->pLeaf==0 ) return; + } - bUseFrom = 0; - }while( pIter->bSkipEmpty && fts5MultiIterIsEmpty(p, pIter) ); + fts5AssertMultiIterSetup(p, pIter); + assert( pSeg==&pIter->aSeg[pIter->aFirst[1].iFirst] && pSeg->pLeaf ); + if( pIter->bSkipEmpty==0 || pSeg->nPos ){ + pIter->xSetOutputs(pIter, pSeg); + return; + } + bUseFrom = 0; } } static void fts5MultiIterNext2( Fts5Index *p, - Fts5IndexIter *pIter, + Fts5Iter *pIter, int *pbNewTerm /* OUT: True if *might* be new term */ ){ assert( pIter->bSkipEmpty ); @@ -2780,7 +2794,7 @@ static void fts5MultiIterNext2( assert( p->rc==SQLITE_OK ); pSeg->xNext(p, pSeg, &bNewTerm); if( pSeg->pLeaf==0 || bNewTerm - || fts5MultiIterAdvanceRowid(p, pIter, iFirst) + || fts5MultiIterAdvanceRowid(p, pIter, iFirst, &pSeg) ){ fts5MultiIterAdvanced(p, pIter, iFirst, 1); fts5MultiIterSetEof(pIter); @@ -2794,17 +2808,19 @@ static void fts5MultiIterNext2( } } +static void fts5IterSetOutputs_Noop(Fts5Iter *pIter, Fts5SegIter *pSeg){ +} -static Fts5IndexIter *fts5MultiIterAlloc( +static Fts5Iter *fts5MultiIterAlloc( Fts5Index *p, /* FTS5 backend to iterate within */ int nSeg ){ - Fts5IndexIter *pNew; + Fts5Iter *pNew; int nSlot; /* Power of two >= nSeg */ for(nSlot=2; nSlotaSeg[] */ sizeof(Fts5CResult) * nSlot /* pNew->aFirst[] */ ); @@ -2812,12 +2828,13 @@ static Fts5IndexIter *fts5MultiIterAlloc( pNew->nSeg = nSlot; pNew->aFirst = (Fts5CResult*)&pNew->aSeg[nSlot]; pNew->pIndex = p; + pNew->xSetOutputs = fts5IterSetOutputs_Noop; } return pNew; } /* -** Allocate a new Fts5IndexIter object. +** Allocate a new Fts5Iter object. ** ** The new object will be used to iterate through data in structure pStruct. ** If iLevel is -ve, then all data in all segments is merged. Or, if iLevel @@ -2835,14 +2852,14 @@ static void fts5MultiIterNew( const u8 *pTerm, int nTerm, /* Term to seek to (or NULL/0) */ int iLevel, /* Level to iterate (-1 for all) */ int nSegment, /* Number of segments to merge (iLevel>=0) */ - Fts5IndexIter **ppOut /* New object */ + Fts5Iter **ppOut /* New object */ ){ int nSeg = 0; /* Number of segment-iters in use */ int iIter = 0; /* */ int iSeg; /* Used to iterate through segments */ Fts5Buffer buf = {0,0,0}; /* Buffer used by fts5SegIterSeekInit() */ Fts5StructureLevel *pLvl; - Fts5IndexIter *pNew; + Fts5Iter *pNew; assert( (pTerm==0 && nTerm==0) || iLevel<0 ); @@ -2917,16 +2934,16 @@ static void fts5MultiIterNew( } /* -** Create an Fts5IndexIter that iterates through the doclist provided +** Create an Fts5Iter that iterates through the doclist provided ** as the second argument. */ static void fts5MultiIterNew2( Fts5Index *p, /* FTS5 backend to iterate within */ Fts5Data *pData, /* Doclist to iterate through */ int bDesc, /* True for descending rowid order */ - Fts5IndexIter **ppOut /* New object */ + Fts5Iter **ppOut /* New object */ ){ - Fts5IndexIter *pNew; + Fts5Iter *pNew; pNew = fts5MultiIterAlloc(p, 2); if( pNew ){ Fts5SegIter *pIter = &pNew->aSeg[1]; @@ -2961,7 +2978,7 @@ static void fts5MultiIterNew2( ** Return true if the iterator is at EOF or if an error has occurred. ** False otherwise. */ -static int fts5MultiIterEof(Fts5Index *p, Fts5IndexIter *pIter){ +static int fts5MultiIterEof(Fts5Index *p, Fts5Iter *pIter){ assert( p->rc || (pIter->aSeg[ pIter->aFirst[1].iFirst ].pLeaf==0)==pIter->bEof ); @@ -2973,7 +2990,7 @@ static int fts5MultiIterEof(Fts5Index *p, Fts5IndexIter *pIter){ ** to. If the iterator points to EOF when this function is called the ** results are undefined. */ -static i64 fts5MultiIterRowid(Fts5IndexIter *pIter){ +static i64 fts5MultiIterRowid(Fts5Iter *pIter){ assert( pIter->aSeg[ pIter->aFirst[1].iFirst ].pLeaf ); return pIter->aSeg[ pIter->aFirst[1].iFirst ].iRowid; } @@ -2983,7 +3000,7 @@ static i64 fts5MultiIterRowid(Fts5IndexIter *pIter){ */ static void fts5MultiIterNextFrom( Fts5Index *p, - Fts5IndexIter *pIter, + Fts5Iter *pIter, i64 iMatch ){ while( 1 ){ @@ -3000,7 +3017,7 @@ static void fts5MultiIterNextFrom( ** Return a pointer to a buffer containing the term associated with the ** entry that the iterator currently points to. */ -static const u8 *fts5MultiIterTerm(Fts5IndexIter *pIter, int *pn){ +static const u8 *fts5MultiIterTerm(Fts5Iter *pIter, int *pn){ Fts5SegIter *p = &pIter->aSeg[ pIter->aFirst[1].iFirst ]; *pn = p->term.n; return p->term.p; @@ -3582,7 +3599,7 @@ static void fts5WriteInit( ** incremental merge operation. This function is called if the incremental ** merge step has finished but the input has not been completely exhausted. */ -static void fts5TrimSegments(Fts5Index *p, Fts5IndexIter *pIter){ +static void fts5TrimSegments(Fts5Index *p, Fts5Iter *pIter){ int i; Fts5Buffer buf; memset(&buf, 0, sizeof(Fts5Buffer)); @@ -3660,7 +3677,7 @@ static void fts5IndexMergeLevel( Fts5Structure *pStruct = *ppStruct; Fts5StructureLevel *pLvl = &pStruct->aLevel[iLvl]; Fts5StructureLevel *pLvlOut; - Fts5IndexIter *pIter = 0; /* Iterator to read input data */ + Fts5Iter *pIter = 0; /* Iterator to read input data */ int nRem = pnRem ? *pnRem : 0; /* Output leaf pages left to write */ int nInput; /* Number of input segments */ Fts5SegWriter writer; /* Writer object */ @@ -4342,7 +4359,7 @@ static int fts5IndexExtractCol( static int fts5AppendRowid( Fts5Index *p, i64 iDelta, - Fts5IndexIter *pMulti, + Fts5Iter *pMulti, Fts5Colset *pColset, Fts5Buffer *pBuf ){ @@ -4367,7 +4384,7 @@ static int fts5AppendRowid( static int fts5AppendPoslist( Fts5Index *p, i64 iDelta, - Fts5IndexIter *pMulti, + Fts5Iter *pMulti, Fts5Colset *pColset, Fts5Buffer *pBuf ){ @@ -4645,14 +4662,14 @@ static void fts5SetupPrefixIter( const u8 *pToken, /* Buffer containing prefix to match */ int nToken, /* Size of buffer pToken in bytes */ Fts5Colset *pColset, /* Restrict matches to these columns */ - Fts5IndexIter **ppIter /* OUT: New iterator */ + Fts5Iter **ppIter /* OUT: New iterator */ ){ Fts5Structure *pStruct; Fts5Buffer *aBuf; const int nBuf = 32; void (*xMerge)(Fts5Index*, Fts5Buffer*, Fts5Buffer*); - int (*xAppend)(Fts5Index*, i64, Fts5IndexIter*, Fts5Colset*, Fts5Buffer*); + int (*xAppend)(Fts5Index*, i64, Fts5Iter*, Fts5Colset*, Fts5Buffer*); if( p->pConfig->eDetail==FTS5_DETAIL_NONE ){ xMerge = fts5MergeRowidLists; xAppend = fts5AppendRowid; @@ -4668,7 +4685,7 @@ static void fts5SetupPrefixIter( const int flags = FTS5INDEX_QUERY_SCAN; int i; i64 iLastRowid = 0; - Fts5IndexIter *p1 = 0; /* Iterator used to gather data from index */ + Fts5Iter *p1 = 0; /* Iterator used to gather data from index */ Fts5Data *pData; Fts5Buffer doclist; int bNewTerm = 1; @@ -4932,6 +4949,154 @@ int sqlite3Fts5IndexWrite( return rc; } + +static int fts5IndexExtractColset ( + Fts5Colset *pColset, /* Colset to filter on */ + const u8 *pPos, int nPos, /* Position list */ + Fts5Buffer *pBuf /* Output buffer */ +){ + int rc = SQLITE_OK; + int i; + + fts5BufferZero(pBuf); + for(i=0; inCol; i++){ + const u8 *pSub = pPos; + int nSub = fts5IndexExtractCol(&pSub, nPos, pColset->aiCol[i]); + if( nSub ){ + fts5BufferAppendBlob(&rc, pBuf, nSub, pSub); + } + } + return rc; +} + +/* +** xSetOutputs callback used by detail=none tables. +*/ +static void fts5IterSetOutputs_None(Fts5Iter *pIter, Fts5SegIter *pSeg){ + assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_NONE ); + pIter->base.iRowid = pSeg->iRowid; + pIter->base.nData = pSeg->nPos; +} + +/* +** xSetOutputs callback used by detail=full and detail=col tables when no +** column filters are specified. +*/ +static void fts5IterSetOutputs_Nocolset(Fts5Iter *pIter, Fts5SegIter *pSeg){ + pIter->base.iRowid = pSeg->iRowid; + pIter->base.nData = pSeg->nPos; + + assert( pIter->pIndex->pConfig->eDetail!=FTS5_DETAIL_NONE ); + assert( pIter->pColset==0 || pIter->bFiltered ); + + if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf ){ + /* All data is stored on the current page. Populate the output + ** variables to point into the body of the page object. */ + pIter->base.pData = &pSeg->pLeaf->p[pSeg->iLeafOffset]; + }else{ + /* The data is distributed over two or more pages. Copy it into the + ** Fts5Iter.poslist buffer and then set the output pointer to point + ** to this buffer. */ + fts5BufferZero(&pIter->poslist); + fts5SegiterPoslist(pIter->pIndex, pSeg, 0, &pIter->poslist); + pIter->base.pData = pIter->poslist.p; + } +} + +/* +** xSetOutputs callback used by detail=col when there is a column filter. +*/ +static void fts5IterSetOutputs_Col(Fts5Iter *pIter, Fts5SegIter *pSeg){ + Fts5Colset *pColset = pIter->pColset; + pIter->base.iRowid = pSeg->iRowid; + + assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_COLUMNS ); + assert( pColset ); + + if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf && 0 ){ + /* All data is stored on the current page. Populate the output + ** variables to point into the body of the page object. */ + Fts5PoslistWriter writer = {0}; + const u8 *a = &pSeg->pLeaf->p[pSeg->iLeafOffset]; + int n = pSeg->nPos; + int iCol = 0; + i64 iPos = 0; + int iOff = 0; + + fts5BufferZero(&pIter->poslist); + while( 0==sqlite3Fts5PoslistNext64(a, n, &iOff, &iPos) ){ + if( iPos==pColset->aiCol[iCol] ){ + sqlite3Fts5PoslistWriterAppend(&pIter->poslist, &writer, iPos); + if( ++iCol>=pColset->nCol ) break; + } + } + + }else{ + /* The data is distributed over two or more pages. Copy it into the + ** Fts5Iter.poslist buffer and then set the output pointer to point + ** to this buffer. */ + fts5BufferZero(&pIter->poslist); + fts5SegiterPoslist(pIter->pIndex, pSeg, pColset, &pIter->poslist); + } + + pIter->base.pData = pIter->poslist.p; + pIter->base.nData = pIter->poslist.n; +} + +/* +** xSetOutputs callback used by detail=full when there is a column filter. +*/ +static void fts5IterSetOutputs_Full(Fts5Iter *pIter, Fts5SegIter *pSeg){ + Fts5Colset *pColset = pIter->pColset; + pIter->base.iRowid = pSeg->iRowid; + + assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_FULL ); + assert( pColset ); + + if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf ){ + /* All data is stored on the current page. Populate the output + ** variables to point into the body of the page object. */ + const u8 *a = &pSeg->pLeaf->p[pSeg->iLeafOffset]; + if( pColset->nCol==1 ){ + pIter->base.nData = fts5IndexExtractCol(&a, pSeg->nPos,pColset->aiCol[0]); + pIter->base.pData = a; + }else{ + fts5BufferZero(&pIter->poslist); + fts5IndexExtractColset(pColset, a, pSeg->nPos, &pIter->poslist); + pIter->base.pData = pIter->poslist.p; + pIter->base.nData = pIter->poslist.n; + } + }else{ + /* The data is distributed over two or more pages. Copy it into the + ** Fts5Iter.poslist buffer and then set the output pointer to point + ** to this buffer. */ + fts5BufferZero(&pIter->poslist); + fts5SegiterPoslist(pIter->pIndex, pSeg, pColset, &pIter->poslist); + pIter->base.pData = pIter->poslist.p; + pIter->base.nData = pIter->poslist.n; + } +} + +static void fts5IterSetOutputCb(Fts5Iter *pIter){ + int eDetail = pIter->pIndex->pConfig->eDetail; + if( eDetail==FTS5_DETAIL_NONE ){ + pIter->xSetOutputs = fts5IterSetOutputs_None; + } + + else if( pIter->pColset==0 || pIter->bFiltered ){ + pIter->xSetOutputs = fts5IterSetOutputs_Nocolset; + } + + else if( eDetail==FTS5_DETAIL_FULL ){ + pIter->xSetOutputs = fts5IterSetOutputs_Full; + } + + else{ + assert( eDetail==FTS5_DETAIL_COLUMNS ); + pIter->xSetOutputs = fts5IterSetOutputs_Col; + } +} + /* ** Open a new iterator to iterate though all rowid that match the ** specified token or token prefix. @@ -4944,22 +5109,27 @@ int sqlite3Fts5IndexQuery( Fts5IndexIter **ppIter /* OUT: New iterator object */ ){ Fts5Config *pConfig = p->pConfig; - Fts5IndexIter *pRet = 0; - int iIdx = 0; + Fts5Iter *pRet = 0; Fts5Buffer buf = {0, 0, 0}; /* If the QUERY_SCAN flag is set, all other flags must be clear. */ assert( (flags & FTS5INDEX_QUERY_SCAN)==0 || flags==FTS5INDEX_QUERY_SCAN ); if( sqlite3Fts5BufferSize(&p->rc, &buf, nToken+1)==0 ){ + int iIdx = 0; /* Index to search */ memcpy(&buf.p[1], pToken, nToken); -#ifdef SQLITE_DEBUG - /* If the QUERY_TEST_NOIDX flag was specified, then this must be a + /* Figure out which index to search and set iIdx accordingly. If this + ** is a prefix query for which there is no prefix index, set iIdx to + ** greater than pConfig->nPrefix to indicate that the query will be + ** satisfied by scanning multiple terms in the main index. + ** + ** If the QUERY_TEST_NOIDX flag was specified, then this must be a ** prefix-query. Instead of using a prefix-index (if one exists), ** evaluate the prefix query using the main FTS index. This is used ** for internal sanity checking by the integrity-check in debug ** mode only. */ +#ifdef SQLITE_DEBUG if( pConfig->bPrefixIndex==0 || (flags & FTS5INDEX_QUERY_TEST_NOIDX) ){ assert( flags & FTS5INDEX_QUERY_PREFIX ); iIdx = 1+pConfig->nPrefix; @@ -4973,6 +5143,7 @@ int sqlite3Fts5IndexQuery( } if( iIdx<=pConfig->nPrefix ){ + /* Straight index lookup */ Fts5Structure *pStruct = fts5StructureRead(p); buf.p[0] = (u8)(FTS5_MAIN_PREFIX + iIdx); if( pStruct ){ @@ -4980,17 +5151,25 @@ int sqlite3Fts5IndexQuery( fts5StructureRelease(pStruct); } }else{ + /* Scan multiple terms in the main index */ int bDesc = (flags & FTS5INDEX_QUERY_DESC)!=0; buf.p[0] = FTS5_MAIN_PREFIX; fts5SetupPrefixIter(p, bDesc, buf.p, nToken+1, pColset, &pRet); } + if( p->rc==SQLITE_OK ){ + Fts5SegIter *pSeg = &pRet->aSeg[pRet->aFirst[1].iFirst]; + pRet->pColset = pColset; + fts5IterSetOutputCb(pRet); + if( pSeg->pLeaf ) pRet->xSetOutputs(pRet, pSeg); + } if( p->rc ){ - sqlite3Fts5IterClose(pRet); + sqlite3Fts5IterClose(&pRet->base); pRet = 0; fts5CloseReader(p); } - *ppIter = pRet; + + *ppIter = &pRet->base; sqlite3Fts5BufferFree(&buf); } return fts5IndexReturn(p); @@ -5000,14 +5179,15 @@ int sqlite3Fts5IndexQuery( ** Return true if the iterator passed as the only argument is at EOF. */ int sqlite3Fts5IterEof(Fts5IndexIter *pIter){ - assert( pIter->pIndex->rc==SQLITE_OK ); - return pIter->bEof; + assert( ((Fts5Iter*)pIter)->pIndex->rc==SQLITE_OK ); + return ((Fts5Iter*)pIter)->bEof; } /* ** Move to the next matching rowid. */ -int sqlite3Fts5IterNext(Fts5IndexIter *pIter){ +int sqlite3Fts5IterNext(Fts5IndexIter *pIndexIter){ + Fts5Iter *pIter = (Fts5Iter*)pIndexIter; assert( pIter->pIndex->rc==SQLITE_OK ); fts5MultiIterNext(pIter->pIndex, pIter, 0, 0); return fts5IndexReturn(pIter->pIndex); @@ -5016,7 +5196,8 @@ int sqlite3Fts5IterNext(Fts5IndexIter *pIter){ /* ** Move to the next matching term/rowid. Used by the fts5vocab module. */ -int sqlite3Fts5IterNextScan(Fts5IndexIter *pIter){ +int sqlite3Fts5IterNextScan(Fts5IndexIter *pIndexIter){ + Fts5Iter *pIter = (Fts5Iter*)pIndexIter; Fts5Index *p = pIter->pIndex; assert( pIter->pIndex->rc==SQLITE_OK ); @@ -5039,7 +5220,8 @@ int sqlite3Fts5IterNextScan(Fts5IndexIter *pIter){ ** definition of "at or after" depends on whether this iterator iterates ** in ascending or descending rowid order. */ -int sqlite3Fts5IterNextFrom(Fts5IndexIter *pIter, i64 iMatch){ +int sqlite3Fts5IterNextFrom(Fts5IndexIter *pIndexIter, i64 iMatch){ + Fts5Iter *pIter = (Fts5Iter*)pIndexIter; fts5MultiIterNextFrom(pIter->pIndex, pIter, iMatch); return fts5IndexReturn(pIter->pIndex); } @@ -5047,41 +5229,21 @@ int sqlite3Fts5IterNextFrom(Fts5IndexIter *pIter, i64 iMatch){ /* ** Return the current rowid. */ -i64 sqlite3Fts5IterRowid(Fts5IndexIter *pIter){ - return fts5MultiIterRowid(pIter); +i64 sqlite3Fts5IterRowid(Fts5IndexIter *pIndexIter){ + return fts5MultiIterRowid((Fts5Iter*)pIndexIter); } /* ** Return the current term. */ -const char *sqlite3Fts5IterTerm(Fts5IndexIter *pIter, int *pn){ +const char *sqlite3Fts5IterTerm(Fts5IndexIter *pIndexIter, int *pn){ int n; - const char *z = (const char*)fts5MultiIterTerm(pIter, &n); + const char *z = (const char*)fts5MultiIterTerm((Fts5Iter*)pIndexIter, &n); *pn = n-1; return &z[1]; } -static int fts5IndexExtractColset ( - Fts5Colset *pColset, /* Colset to filter on */ - const u8 *pPos, int nPos, /* Position list */ - Fts5Buffer *pBuf /* Output buffer */ -){ - int rc = SQLITE_OK; - int i; - - fts5BufferZero(pBuf); - for(i=0; inCol; i++){ - const u8 *pSub = pPos; - int nSub = fts5IndexExtractCol(&pSub, nPos, pColset->aiCol[i]); - if( nSub ){ - fts5BufferAppendBlob(&rc, pBuf, nSub, pSub); - } - } - return rc; -} - - /* ** Return a pointer to a buffer containing a copy of the position list for ** the current entry. Output variable *pn is set to the size of the buffer @@ -5091,12 +5253,13 @@ static int fts5IndexExtractColset ( ** field that starts the position list on disk. */ int sqlite3Fts5IterPoslist( - Fts5IndexIter *pIter, + Fts5IndexIter *pIndexIter, Fts5Colset *pColset, /* Column filter (or NULL) */ const u8 **pp, /* OUT: Pointer to position-list data */ int *pn, /* OUT: Size of position-list in bytes */ i64 *piRowid /* OUT: Current rowid */ ){ + Fts5Iter *pIter = (Fts5Iter*)pIndexIter; Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ]; int eDetail = pIter->pIndex->pConfig->eDetail; @@ -5133,10 +5296,11 @@ int sqlite3Fts5IterPoslist( } int sqlite3Fts5IterCollist( - Fts5IndexIter *pIter, + Fts5IndexIter *pIndexIter, const u8 **pp, /* OUT: Pointer to position-list data */ int *pn /* OUT: Size of position-list in bytes */ ){ + Fts5Iter *pIter = (Fts5Iter*)pIndexIter; assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_COLUMNS ); *pp = pIter->poslist.p; *pn = pIter->poslist.n; @@ -5148,7 +5312,8 @@ int sqlite3Fts5IterCollist( ** copies the position list into the buffer supplied as the second ** argument. */ -int sqlite3Fts5IterPoslistBuffer(Fts5IndexIter *pIter, Fts5Buffer *pBuf){ +int sqlite3Fts5IterPoslistBuffer(Fts5IndexIter *pIndexIter, Fts5Buffer *pBuf){ + Fts5Iter *pIter = (Fts5Iter*)pIndexIter; Fts5Index *p = pIter->pIndex; Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ]; assert( p->rc==SQLITE_OK ); @@ -5160,8 +5325,9 @@ int sqlite3Fts5IterPoslistBuffer(Fts5IndexIter *pIter, Fts5Buffer *pBuf){ /* ** Close an iterator opened by an earlier call to sqlite3Fts5IndexQuery(). */ -void sqlite3Fts5IterClose(Fts5IndexIter *pIter){ - if( pIter ){ +void sqlite3Fts5IterClose(Fts5IndexIter *pIndexIter){ + if( pIndexIter ){ + Fts5Iter *pIter = (Fts5Iter*)pIndexIter; Fts5Index *pIndex = pIter->pIndex; fts5MultiIterFree(pIter->pIndex, pIter); fts5CloseReader(pIndex); @@ -5328,35 +5494,30 @@ static int fts5QueryCksum( ){ int eDetail = p->pConfig->eDetail; u64 cksum = *pCksum; - Fts5IndexIter *pIdxIter = 0; - Fts5Buffer buf = {0, 0, 0}; - int rc = sqlite3Fts5IndexQuery(p, z, n, flags, 0, &pIdxIter); + Fts5IndexIter *pIter = 0; + int rc = sqlite3Fts5IndexQuery(p, z, n, flags, 0, &pIter); - while( rc==SQLITE_OK && 0==sqlite3Fts5IterEof(pIdxIter) ){ - i64 rowid = sqlite3Fts5IterRowid(pIdxIter); + while( rc==SQLITE_OK && 0==sqlite3Fts5IterEof(pIter) ){ + i64 rowid = sqlite3Fts5IterRowid(pIter); if( eDetail==FTS5_DETAIL_NONE ){ cksum ^= sqlite3Fts5IndexEntryCksum(rowid, 0, 0, iIdx, z, n); }else{ - rc = sqlite3Fts5IterPoslistBuffer(pIdxIter, &buf); - if( rc==SQLITE_OK ){ - Fts5PoslistReader sReader; - for(sqlite3Fts5PoslistReaderInit(buf.p, buf.n, &sReader); - sReader.bEof==0; - sqlite3Fts5PoslistReaderNext(&sReader) - ){ - int iCol = FTS5_POS2COLUMN(sReader.iPos); - int iOff = FTS5_POS2OFFSET(sReader.iPos); - cksum ^= sqlite3Fts5IndexEntryCksum(rowid, iCol, iOff, iIdx, z, n); - } + Fts5PoslistReader sReader; + for(sqlite3Fts5PoslistReaderInit(pIter->pData, pIter->nData, &sReader); + sReader.bEof==0; + sqlite3Fts5PoslistReaderNext(&sReader) + ){ + int iCol = FTS5_POS2COLUMN(sReader.iPos); + int iOff = FTS5_POS2OFFSET(sReader.iPos); + cksum ^= sqlite3Fts5IndexEntryCksum(rowid, iCol, iOff, iIdx, z, n); } } if( rc==SQLITE_OK ){ - rc = sqlite3Fts5IterNext(pIdxIter); + rc = sqlite3Fts5IterNext(pIter); } } - sqlite3Fts5IterClose(pIdxIter); - fts5BufferFree(&buf); + sqlite3Fts5IterClose(pIter); *pCksum = cksum; return rc; @@ -5661,7 +5822,7 @@ int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){ int eDetail = p->pConfig->eDetail; u64 cksum2 = 0; /* Checksum based on contents of indexes */ Fts5Buffer poslist = {0,0,0}; /* Buffer used to hold a poslist */ - Fts5IndexIter *pIter; /* Used to iterate through entire index */ + Fts5Iter *pIter; /* Used to iterate through entire index */ Fts5Structure *pStruct; /* Index structure */ #ifdef SQLITE_DEBUG diff --git a/ext/fts5/fts5_vocab.c b/ext/fts5/fts5_vocab.c index ddc5576c5d..17aa5816c4 100644 --- a/ext/fts5/fts5_vocab.c +++ b/ext/fts5/fts5_vocab.c @@ -412,28 +412,29 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){ i64 iPos = 0; /* 64-bit position read from poslist */ int iOff = 0; /* Current offset within position list */ + pPos = pCsr->pIter->pData; + nPos = pCsr->pIter->nData; switch( pCsr->pConfig->eDetail ){ case FTS5_DETAIL_FULL: - rc = sqlite3Fts5IterPoslist(pCsr->pIter, 0, &pPos, &nPos, &dummy); - if( rc==SQLITE_OK ){ - if( pTab->eType==FTS5_VOCAB_ROW ){ - while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){ - pCsr->aCnt[0]++; - } - pCsr->aDoc[0]++; - }else{ - int iCol = -1; - while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){ - int ii = FTS5_POS2COLUMN(iPos); - pCsr->aCnt[ii]++; - if( iCol!=ii ){ - if( ii>=nCol ){ - rc = FTS5_CORRUPT; - break; - } - pCsr->aDoc[ii]++; - iCol = ii; + pPos = pCsr->pIter->pData; + nPos = pCsr->pIter->nData; + if( pTab->eType==FTS5_VOCAB_ROW ){ + while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){ + pCsr->aCnt[0]++; + } + pCsr->aDoc[0]++; + }else{ + int iCol = -1; + while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){ + int ii = FTS5_POS2COLUMN(iPos); + pCsr->aCnt[ii]++; + if( iCol!=ii ){ + if( ii>=nCol ){ + rc = FTS5_CORRUPT; + break; } + pCsr->aDoc[ii]++; + iCol = ii; } } } @@ -443,19 +444,14 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){ if( pTab->eType==FTS5_VOCAB_ROW ){ pCsr->aDoc[0]++; }else{ - Fts5Buffer buf = {0, 0, 0}; - rc = sqlite3Fts5IterPoslistBuffer(pCsr->pIter, &buf); - if( rc==SQLITE_OK ){ - while( 0==sqlite3Fts5PoslistNext64(buf.p, buf.n, &iOff,&iPos) ){ - assert_nc( iPos>=0 && iPos=nCol ){ - rc = FTS5_CORRUPT; - break; - } - pCsr->aDoc[iPos]++; + while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff,&iPos) ){ + assert_nc( iPos>=0 && iPos=nCol ){ + rc = FTS5_CORRUPT; + break; } + pCsr->aDoc[iPos]++; } - sqlite3Fts5BufferFree(&buf); } break; diff --git a/ext/fts5/test/fts5ac.test b/ext/fts5/test/fts5ac.test index 00b1328867..a5a522a466 100644 --- a/ext/fts5/test/fts5ac.test +++ b/ext/fts5/test/fts5ac.test @@ -250,7 +250,6 @@ foreach {tn2 sql} { FROM xx WHERE xx match $expr } $res - set res [fts5_query_data $expr xx DESC] do_execsql_test 1.$tn2.$tn.[llength $res].desc { SELECT rowid, fts5_test_poslist(xx), fts5_test_collist(xx) diff --git a/manifest b/manifest index 6c3814c3d9..99a9a8b139 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\ssqlite3TokenInit()\sutility\sfunction. -D 2016-01-22T17:48:09.470 +C Experimental\sperformance\senhancements\sfor\sfts5. +D 2016-01-22T19:48:34.825 F Makefile.in 027c1603f255390c43a426671055a31c0a65fdb4 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc d2b93511a969c0c8fcf52aeb5e426571e8c610d2 @@ -97,13 +97,13 @@ F ext/fts3/unicode/mkunicode.tcl 95cf7ec186e48d4985e433ff8a1c89090a774252 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0 F ext/fts5/fts5.h ff9c2782e8ed890b0de2f697a8d63971939e70c7 -F ext/fts5/fts5Int.h 5599703af9c13512900a9f22fec39d48078d619d +F ext/fts5/fts5Int.h f93aed3d05d1b3686c4dccf70544bcfc7aa117f1 F ext/fts5/fts5_aux.c 2dafc3aee0c70d643140c77d8d70daffa51a9e9e F ext/fts5/fts5_buffer.c 7d3f6f01f8fdc45204e6a33925ef8478a67d28dd F ext/fts5/fts5_config.c 0c384ebdd23fd055e2e50a93277b8d59da538238 -F ext/fts5/fts5_expr.c 4ab4504c54bbe24689c83411d8588f4ec99136e9 +F ext/fts5/fts5_expr.c 48b9131b74c8d3b8c12ba0f7995e2d60eecce9f2 F ext/fts5/fts5_hash.c 1b113977296cf4212c6ec667d5e3f2bd18036955 -F ext/fts5/fts5_index.c 716c301835a122ba36910b4f821c87d26ae9a5d9 +F ext/fts5/fts5_index.c 19062d1f40ba6d88d786a986d1c20f9d741799d0 F ext/fts5/fts5_main.c 833db0a3df10ab26e0221a9baa40cf871c450df3 F ext/fts5/fts5_storage.c fb2eaec3aa954b680d43096dc539f8270bd6390e F ext/fts5/fts5_tcl.c f8731e0508299bd43f1a2eff7dbeaac870768966 @@ -112,13 +112,13 @@ F ext/fts5/fts5_test_tok.c db08af63673c3a7d39f053b36fd6e065017706be F ext/fts5/fts5_tokenize.c 504984ac6993323247221eebe3cd55bead01b5f8 F ext/fts5/fts5_unicode2.c 78273fbd588d1d9bd0a7e4e0ccc9207348bae33c F ext/fts5/fts5_varint.c 3f86ce09cab152e3d45490d7586b7ed2e40c13f1 -F ext/fts5/fts5_vocab.c ee6df1a3be103414d7b7af833ae1885c7b83a9d0 +F ext/fts5/fts5_vocab.c f1b4308b9b7ec8e659d0c9b39ddc8f1aeee47a1a F ext/fts5/fts5parse.y 1647eba089b9b3fc058b4dc989d9da87d15b9580 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba F ext/fts5/test/fts5_common.tcl 6d0d74b695c4be055a8ba1dd807f22a2abc95b5e F ext/fts5/test/fts5aa.test 7e814df4a0e6c22a6fe2d84f210fdc0b5068a084 F ext/fts5/test/fts5ab.test 30325a89453280160106be411bba3acf138e6d1b -F ext/fts5/test/fts5ac.test d5073ca7bd2d9fe8aab0c82c6c75a7e4b0d70ced +F ext/fts5/test/fts5ac.test dec95549e007dd9be52aa435cdcd0f08e14e64d0 F ext/fts5/test/fts5ad.test 0ddaa5b692ff220100ee396228838f4331399eaa F ext/fts5/test/fts5ae.test 612dcb51f4069226791ff14c17dbfb3138c56f20 F ext/fts5/test/fts5af.test be858a96b1f5de66ba6d64f0021bd8b2408e126c @@ -1419,7 +1419,10 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P e60461e984b8df09256bb0d733dbfae52568a145 -R 03ab6310cefadc63b5c7c7da966c2348 -U drh -Z 0c42ba520d71deaa3e0a21b23ff67f9d +P 7323175337b7ba85ac932ca892b28860f6a5b688 +R bc65520844365f59e43048a22901b688 +T *branch * fts5-perf +T *sym-fts5-perf * +T -sym-trunk * +U dan +Z 20f25a74d681d80924173ee060583d9e diff --git a/manifest.uuid b/manifest.uuid index e9fd60917b..fc3a1227d6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7323175337b7ba85ac932ca892b28860f6a5b688 \ No newline at end of file +b5a57b812fd6a734cf2a342bf0b730ae18912d73 \ No newline at end of file From a8a18734857f3ad96403acecc201effaab473fb7 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Fri, 22 Jan 2016 22:16:50 +0000 Subject: [PATCH 09/39] More MSVC makefile cleanup. FossilOrigin-Name: df22556fd75997111e52f96572da8379dfe948be --- Makefile.msc | 36 ++++++++++++++++++------------------ manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 26 insertions(+), 26 deletions(-) diff --git a/Makefile.msc b/Makefile.msc index 3621d4f92b..504b2893ba 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1326,12 +1326,29 @@ TESTOPTS = --verbose=file --output=test-out.txt # all: dll libsqlite3.lib shell libtclsqlite3.lib +# Dynamic link library section. +# +dll: $(SQLITE3DLL) + +# Shell executable. +# +shell: $(SQLITE3EXE) + libsqlite3.lib: $(LIBOBJ) $(LTLIB) $(LTLIBOPTS) /OUT:$@ $(LIBOBJ) $(TLIBS) libtclsqlite3.lib: tclsqlite.lo libsqlite3.lib $(LTLIB) $(LTLIBOPTS) $(LTLIBPATHS) /OUT:$@ tclsqlite.lo libsqlite3.lib $(LIBTCLSTUB) $(TLIBS) +$(SQLITE3DLL): $(LIBOBJ) $(LIBRESOBJS) $(CORE_LINK_DEP) + $(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL $(CORE_LINK_OPTS) /OUT:$@ $(LIBOBJ) $(LIBRESOBJS) $(LTLIBS) $(TLIBS) + +sqlite3.def: libsqlite3.lib + echo EXPORTS > sqlite3.def + dumpbin /all libsqlite3.lib \ + | $(TCLSH_CMD) $(TOP)\tool\replace.tcl include "^\s+1 _?(sqlite3_.*)$$" \1 \ + | sort >> sqlite3.def + $(SQLITE3EXE): $(TOP)\src\shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H) $(LTLINK) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) $(TOP)\src\shell.c $(SHELL_CORE_SRC) \ /link $(SQLITE3EXEPDB) $(LDFLAGS) $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS) @@ -1419,7 +1436,7 @@ opcodes.lo: opcodes.c # Rule to build the Win32 resources object file. # !IF $(USE_RC)!=0 -$(LIBRESOBJS): $(TOP)\src\sqlite3.rc $(HDR) +$(LIBRESOBJS): $(TOP)\src\sqlite3.rc $(SQLITE3H) echo #ifndef SQLITE_RESOURCE_VERSION > sqlite3rc.h for /F %%V in ('type "$(TOP)\VERSION"') do ( \ echo #define SQLITE_RESOURCE_VERSION %%V \ @@ -1957,20 +1974,3 @@ clean: del /Q sqlite-*-output.vsix 2>NUL del /Q fuzzershell.exe fuzzcheck.exe sqldiff.exe 2>NUL del /Q fts5.* fts5parse.* 2>NUL - -# Shell executable. -# -shell: $(SQLITE3EXE) - -# Dynamic link library section. -# -dll: $(SQLITE3DLL) - -sqlite3.def: libsqlite3.lib - echo EXPORTS > sqlite3.def - dumpbin /all libsqlite3.lib \ - | $(TCLSH_CMD) $(TOP)\tool\replace.tcl include "^\s+1 _?(sqlite3_.*)$$" \1 \ - | sort >> sqlite3.def - -$(SQLITE3DLL): $(LIBOBJ) $(LIBRESOBJS) $(CORE_LINK_DEP) - $(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL $(CORE_LINK_OPTS) /OUT:$@ $(LIBOBJ) $(LIBRESOBJS) $(LTLIBS) $(TLIBS) diff --git a/manifest b/manifest index 566db8de82..2e197ef017 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,8 @@ -C Reenable\sthe\sxCurrentTime\sand\sxGetLastError\smethods\son\sthe\sunix\sVFSes\sas\slong\nas\sSQLITE_OMIT_DEPRECATED\sis\snot\sdefined. -D 2016-01-22T19:44:03.065 +C More\sMSVC\smakefile\scleanup. +D 2016-01-22T22:16:50.346 F Makefile.in 027c1603f255390c43a426671055a31c0a65fdb4 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 -F Makefile.msc d2b93511a969c0c8fcf52aeb5e426571e8c610d2 +F Makefile.msc bf507c4168b28d0ba47ac29d53fd6244289f10b8 F README.md 8ecc12493ff9f820cdea6520a9016001cb2e59b7 F VERSION 866588d1edf0ccb5b0d33896974338f97564f719 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -1419,7 +1419,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 7323175337b7ba85ac932ca892b28860f6a5b688 -R 4fcfc780a1da751b6f90e6d6f3ced841 -U drh -Z 703592c4495d8a71fd67fbd54e7f3927 +P c11c85fdb6514cae54bb44945cc197dcaba72307 +R 7e2bc2233b486c9642759e292a9803ed +U mistachkin +Z f537116bba79cbb015eb60c46b78af4d diff --git a/manifest.uuid b/manifest.uuid index a853332581..289f7591c2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c11c85fdb6514cae54bb44945cc197dcaba72307 \ No newline at end of file +df22556fd75997111e52f96572da8379dfe948be \ No newline at end of file From 147ef3948622234303ef6d38c38e6ee9fe325928 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 22 Jan 2016 23:17:51 +0000 Subject: [PATCH 10/39] In the TCL interface, if a database connection object was opened with the -uri 1 option, then also honor URI filenames for the "backup" and "restore" commands. FossilOrigin-Name: a1c8116ced62d81f3f5ca26bbe0877e829d4cc56 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/tclsqlite.c | 8 ++++++-- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 2e197ef017..ffb98b2734 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C More\sMSVC\smakefile\scleanup. -D 2016-01-22T22:16:50.346 +C In\sthe\sTCL\sinterface,\sif\sa\sdatabase\sconnection\sobject\swas\sopened\swith\nthe\s-uri\s1\soption,\sthen\salso\shonor\sURI\sfilenames\sfor\sthe\s"backup"\sand\n"restore"\scommands. +D 2016-01-22T23:17:51.219 F Makefile.in 027c1603f255390c43a426671055a31c0a65fdb4 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc bf507c4168b28d0ba47ac29d53fd6244289f10b8 @@ -354,7 +354,7 @@ F src/sqliteInt.h 74e10a74116df0aec9d4a3e134f1a86cc34c2f14 F src/sqliteLimit.h 216557999cb45f2e3578ed53ebefe228d779cb46 F src/status.c 70912d7be68e9e2dbc4010c93d344af61d4c59ba F src/table.c 51b46b2a62d1b3a959633d593b89bab5e2c9155e -F src/tclsqlite.c 82979239a896992f9b78efec81cfda05d316a7d0 +F src/tclsqlite.c 94ef6e2794220c5b6064d4c78ec7169a8c5cc45d F src/test1.c 4f1b42699068b7806af3111786f5ad760c2c1ff7 F src/test2.c 5586f43fcd9a1be0830793cf9d354082c261b25b F src/test3.c a8887dabbbee3059af338f20d290084a63ed1b0f @@ -1419,7 +1419,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P c11c85fdb6514cae54bb44945cc197dcaba72307 -R 7e2bc2233b486c9642759e292a9803ed -U mistachkin -Z f537116bba79cbb015eb60c46b78af4d +P df22556fd75997111e52f96572da8379dfe948be +R 14f009751ec343de1132abf0a7bfa01f +U drh +Z 3618240e30f243f2df7ae3a3ace24b2d diff --git a/manifest.uuid b/manifest.uuid index 289f7591c2..5a178d2571 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -df22556fd75997111e52f96572da8379dfe948be \ No newline at end of file +a1c8116ced62d81f3f5ca26bbe0877e829d4cc56 \ No newline at end of file diff --git a/src/tclsqlite.c b/src/tclsqlite.c index 604e898265..aa913ca7c7 100644 --- a/src/tclsqlite.c +++ b/src/tclsqlite.c @@ -153,6 +153,7 @@ struct SqliteDb { IncrblobChannel *pIncrblob;/* Linked list of open incrblob channels */ int nStep, nSort, nIndex; /* Statistics for most recent operation */ int nTransaction; /* Number of nested [transaction] methods */ + int openFlags; /* Flags used to open. (SQLITE_OPEN_URI) */ #ifdef SQLITE_TEST int bLegacyPrepare; /* True to use sqlite3_prepare() */ #endif @@ -1750,7 +1751,8 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){ Tcl_WrongNumArgs(interp, 2, objv, "?DATABASE? FILENAME"); return TCL_ERROR; } - rc = sqlite3_open(zDestFile, &pDest); + rc = sqlite3_open_v2(zDestFile, &pDest, + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE| pDb->openFlags, 0); if( rc!=SQLITE_OK ){ Tcl_AppendResult(interp, "cannot open target database: ", sqlite3_errmsg(pDest), (char*)0); @@ -2613,7 +2615,8 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){ Tcl_WrongNumArgs(interp, 2, objv, "?DATABASE? FILENAME"); return TCL_ERROR; } - rc = sqlite3_open_v2(zSrcFile, &pSrc, SQLITE_OPEN_READONLY, 0); + rc = sqlite3_open_v2(zSrcFile, &pSrc, + SQLITE_OPEN_READONLY | pDb->openFlags, 0); if( rc!=SQLITE_OK ){ Tcl_AppendResult(interp, "cannot open source database: ", sqlite3_errmsg(pSrc), (char*)0); @@ -3088,6 +3091,7 @@ static int DbMain(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){ return TCL_ERROR; } p->maxStmt = NUM_PREPARED_STMTS; + p->openFlags = flags & SQLITE_OPEN_URI; p->interp = interp; zArg = Tcl_GetStringFromObj(objv[1], 0); if( DbUseNre() ){ From 4cfea5c794983b03bc7fcf7b43b91b83d27b945c Mon Sep 17 00:00:00 2001 From: mistachkin Date: Sat, 23 Jan 2016 00:07:51 +0000 Subject: [PATCH 11/39] More work on the MSVC build. FossilOrigin-Name: a79c46bc61a35edbaf6112c26e8052314b0eb16a --- Makefile.msc | 2 +- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/sqlite3.rc | 2 ++ 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/Makefile.msc b/Makefile.msc index 504b2893ba..5921e9375a 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1948,7 +1948,7 @@ rbu.exe: $(TOP)\ext\rbu\rbu.c $(TOP)\ext\rbu\sqlite3rbu.c $(SQLITE3C) $(SQLITE3H clean: del /Q *.exp *.lo *.ilk *.lib *.obj *.ncb *.pdb *.sdf *.suo 2>NUL - del /Q *.bsc *.cod *.da *.bb *.bbg gmon.out 2>NUL + del /Q *.bsc *.cod *.da *.bb *.bbg *.vc gmon.out 2>NUL del /Q $(SQLITE3C) $(SQLITE3H) opcodes.c opcodes.h 2>NUL del /Q lemon.* lempar.c parse.* 2>NUL del /Q mkkeywordhash.* keywordhash.h 2>NUL diff --git a/manifest b/manifest index ffb98b2734..61cf5d66ed 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,8 @@ -C In\sthe\sTCL\sinterface,\sif\sa\sdatabase\sconnection\sobject\swas\sopened\swith\nthe\s-uri\s1\soption,\sthen\salso\shonor\sURI\sfilenames\sfor\sthe\s"backup"\sand\n"restore"\scommands. -D 2016-01-22T23:17:51.219 +C More\swork\son\sthe\sMSVC\sbuild. +D 2016-01-23T00:07:51.151 F Makefile.in 027c1603f255390c43a426671055a31c0a65fdb4 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 -F Makefile.msc bf507c4168b28d0ba47ac29d53fd6244289f10b8 +F Makefile.msc 1708a78eda223b6daa302b140037fcc214a779f9 F README.md 8ecc12493ff9f820cdea6520a9016001cb2e59b7 F VERSION 866588d1edf0ccb5b0d33896974338f97564f719 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -348,7 +348,7 @@ F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 1db9a902e89201a0ae3ff5243a7a3f37842a3937 F src/shell.c dcd7a83645ef2a58ee9c6d0ea4714d877d7835c4 F src/sqlite.h.in 214476a62012e578f42133a9a3b4f97a9aa421a3 -F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad +F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h dfbe62ffd95b99afe2140d8c35b180d11924072d F src/sqliteInt.h 74e10a74116df0aec9d4a3e134f1a86cc34c2f14 F src/sqliteLimit.h 216557999cb45f2e3578ed53ebefe228d779cb46 @@ -1419,7 +1419,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P df22556fd75997111e52f96572da8379dfe948be -R 14f009751ec343de1132abf0a7bfa01f -U drh -Z 3618240e30f243f2df7ae3a3ace24b2d +P a1c8116ced62d81f3f5ca26bbe0877e829d4cc56 +R b34246fa017fc0ae2e7bf7cb484e3eeb +U mistachkin +Z ba075ff448ff99fa520b3503324fe6e9 diff --git a/manifest.uuid b/manifest.uuid index 5a178d2571..d1ea2b33a4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a1c8116ced62d81f3f5ca26bbe0877e829d4cc56 \ No newline at end of file +a79c46bc61a35edbaf6112c26e8052314b0eb16a \ No newline at end of file diff --git a/src/sqlite3.rc b/src/sqlite3.rc index 04dd086488..5a856490d6 100644 --- a/src/sqlite3.rc +++ b/src/sqlite3.rc @@ -39,9 +39,11 @@ LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US * Icon */ +#if !defined(RC_VERONLY) #define IDI_SQLITE 101 IDI_SQLITE ICON "..\\art\\sqlite370.ico" +#endif /* !defined(RC_VERONLY) */ /* * Version From 3e7860938b58cae4388007a2bc6506679f13ddff Mon Sep 17 00:00:00 2001 From: mistachkin Date: Sat, 23 Jan 2016 07:53:04 +0000 Subject: [PATCH 12/39] Add tooling to create an MSVC Makefile capable of building the core library and shell only, using a pre-built amalgamation. FossilOrigin-Name: c46f1a13cb00264d1164399059250884c7be4ea0 --- Makefile.min.msc | 885 +++++++++++++++++++++++++++++++++++++++++++++ Makefile.msc | 38 ++ manifest | 17 +- manifest.uuid | 2 +- tool/mkmsvcmin.tcl | 88 +++++ 5 files changed, 1023 insertions(+), 7 deletions(-) create mode 100644 Makefile.min.msc create mode 100644 tool/mkmsvcmin.tcl diff --git a/Makefile.min.msc b/Makefile.min.msc new file mode 100644 index 0000000000..2ec986fa4b --- /dev/null +++ b/Makefile.min.msc @@ -0,0 +1,885 @@ +# +# nmake Makefile for SQLite +# +############################################################################### +############################## START OF OPTIONS ############################### +############################################################################### + +# The toplevel directory of the source tree. This is the directory +# that contains this "Makefile.msc". +# +TOP = . + + +# Set this non-0 to enable full warnings (-W4, etc) when compiling. +# +!IFNDEF USE_FULLWARN +USE_FULLWARN = 0 +!ENDIF + +# Set this non-0 to use "stdcall" calling convention for the core library +# and shell executable. +# +!IFNDEF USE_STDCALL +USE_STDCALL = 0 +!ENDIF + +# Set this non-0 to have the shell executable link against the core dynamic +# link library. +# +!IFNDEF DYNAMIC_SHELL +DYNAMIC_SHELL = 0 +!ENDIF + +# Set this non-0 to enable extra code that attempts to detect misuse of the +# SQLite API. +# +!IFNDEF API_ARMOR +API_ARMOR = 0 +!ENDIF + +# If necessary, create a list of harmless compiler warnings to disable when +# compiling the various tools. For the SQLite source code itself, warnings, +# if any, will be disabled from within it. +# +!IFNDEF NO_WARN +!IF $(USE_FULLWARN)!=0 +NO_WARN = -wd4054 -wd4055 -wd4100 -wd4127 -wd4130 -wd4152 -wd4189 -wd4206 +NO_WARN = $(NO_WARN) -wd4210 -wd4232 -wd4305 -wd4306 -wd4702 -wd4706 +!ENDIF +!ENDIF + +# Set this non-0 to use the library paths and other options necessary for +# Windows Phone 8.1. +# +!IFNDEF USE_WP81_OPTS +USE_WP81_OPTS = 0 +!ENDIF + +# Set this non-0 to split the SQLite amalgamation file into chunks to +# be used for debugging with Visual Studio. +# +!IFNDEF SPLIT_AMALGAMATION +SPLIT_AMALGAMATION = 0 +!ENDIF + + +# Set this non-0 to dynamically link to the MSVC runtime library. +# +!IFNDEF USE_CRT_DLL +USE_CRT_DLL = 0 +!ENDIF + +# Set this non-0 to link to the RPCRT4 library. +# +!IFNDEF USE_RPCRT4_LIB +USE_RPCRT4_LIB = 0 +!ENDIF + +# Set this non-0 to generate assembly code listings for the source code +# files. +# +!IFNDEF USE_LISTINGS +USE_LISTINGS = 0 +!ENDIF + +# Set this non-0 to attempt setting the native compiler automatically +# for cross-compiling the command line tools needed during the compilation +# process. +# +!IFNDEF XCOMPILE +XCOMPILE = 0 +!ENDIF + +# Set this non-0 to use the native libraries paths for cross-compiling +# the command line tools needed during the compilation process. +# +!IFNDEF USE_NATIVE_LIBPATHS +USE_NATIVE_LIBPATHS = 0 +!ENDIF + +# Set this 0 to skip the compiling and embedding of version resources. +# +!IFNDEF USE_RC +USE_RC = 1 +!ENDIF + +# Set this non-0 to compile binaries suitable for the WinRT environment. +# This setting does not apply to any binaries that require Tcl to operate +# properly (i.e. the text fixture, etc). +# +!IFNDEF FOR_WINRT +FOR_WINRT = 0 +!ENDIF + +# Set this non-0 to compile binaries suitable for the UAP environment. +# This setting does not apply to any binaries that require Tcl to operate +# properly (i.e. the text fixture, etc). +# +!IFNDEF FOR_UAP +FOR_UAP = 0 +!ENDIF + +# Set this non-0 to compile binaries suitable for the Windows 10 platform. +# +!IFNDEF FOR_WIN10 +FOR_WIN10 = 0 +!ENDIF + + +# Set this to non-0 to create and use PDBs. +# +!IFNDEF SYMBOLS +SYMBOLS = 1 +!ENDIF + +# Set this to non-0 to use the SQLite debugging heap subsystem. +# +!IFNDEF MEMDEBUG +MEMDEBUG = 0 +!ENDIF + +# Set this to non-0 to use the Win32 native heap subsystem. +# +!IFNDEF WIN32HEAP +WIN32HEAP = 0 +!ENDIF + +# Set this to non-0 to enable OSTRACE() macros, which can be useful when +# debugging. +# +!IFNDEF OSTRACE +OSTRACE = 0 +!ENDIF + +# Set this to one of the following values to enable various debugging +# features. Each level includes the debugging options from the previous +# levels. Currently, the recognized values for DEBUG are: +# +# 0 == NDEBUG: Disables assert() and other runtime diagnostics. +# 1 == SQLITE_ENABLE_API_ARMOR: extra attempts to detect misuse of the API. +# 2 == Disables NDEBUG and all optimizations and then enables PDBs. +# 3 == SQLITE_DEBUG: Enables various diagnostics messages and code. +# 4 == SQLITE_WIN32_MALLOC_VALIDATE: Validate the Win32 native heap per call. +# 5 == SQLITE_DEBUG_OS_TRACE: Enables output from the OSTRACE() macros. +# 6 == SQLITE_ENABLE_IOTRACE: Enables output from the IOTRACE() macros. +# +!IFNDEF DEBUG +DEBUG = 0 +!ENDIF + +# Enable use of available compiler optimizations? Normally, this should be +# non-zero. Setting this to zero, thus disabling all compiler optimizations, +# can be useful for testing. +# +!IFNDEF OPTIMIZATIONS +OPTIMIZATIONS = 2 +!ENDIF + +# Set the source code file to be used by executables and libraries when +# they need the amalgamation. +# +!IFNDEF SQLITE3C +!IF $(SPLIT_AMALGAMATION)!=0 +SQLITE3C = sqlite3-all.c +!ELSE +SQLITE3C = sqlite3.c +!ENDIF +!ENDIF + +# Set the include code file to be used by executables and libraries when +# they need SQLite. +# +!IFNDEF SQLITE3H +SQLITE3H = sqlite3.h +!ENDIF + +# This is the name to use for the SQLite dynamic link library (DLL). +# +!IFNDEF SQLITE3DLL +SQLITE3DLL = sqlite3.dll +!ENDIF + +# This is the name to use for the SQLite import library (LIB). +# +!IFNDEF SQLITE3LIB +SQLITE3LIB = sqlite3.lib +!ENDIF + +# This is the name to use for the SQLite shell executable (EXE). +# +!IFNDEF SQLITE3EXE +SQLITE3EXE = sqlite3.exe +!ENDIF + +# This is the argument used to set the program database (PDB) file for the +# SQLite shell executable (EXE). +# +!IFNDEF SQLITE3EXEPDB +SQLITE3EXEPDB = /pdb:sqlite3sh.pdb +!ENDIF + +# These are the "standard" SQLite compilation options used when compiling for +# the Windows platform. +# +!IFNDEF OPT_FEATURE_FLAGS +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS3=1 +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RTREE=1 +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1 +!ENDIF + +# These are the "extended" SQLite compilation options used when compiling for +# the Windows 10 platform. +# +!IFNDEF EXT_FEATURE_FLAGS +!IF $(FOR_WIN10)!=0 +EXT_FEATURE_FLAGS = $(EXT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS4=1 +EXT_FEATURE_FLAGS = $(EXT_FEATURE_FLAGS) -DSQLITE_SYSTEM_MALLOC=1 +EXT_FEATURE_FLAGS = $(EXT_FEATURE_FLAGS) -DSQLITE_OMIT_LOCALTIME=1 +!ELSE +EXT_FEATURE_FLAGS = +!ENDIF +!ENDIF + +############################################################################### +############################### END OF OPTIONS ################################ +############################################################################### + +# This assumes that MSVC is always installed in 32-bit Program Files directory +# and sets the variable for use in locating other 32-bit installs accordingly. +# +PROGRAMFILES_X86 = $(VCINSTALLDIR)\..\.. +PROGRAMFILES_X86 = $(PROGRAMFILES_X86:\\=\) + +# Check for the predefined command macro CC. This should point to the compiler +# binary for the target platform. If it is not defined, simply define it to +# the legacy default value 'cl.exe'. +# +!IFNDEF CC +CC = cl.exe +!ENDIF + +# Check for the command macro LD. This should point to the linker binary for +# the target platform. If it is not defined, simply define it to the legacy +# default value 'link.exe'. +# +!IFNDEF LD +LD = link.exe +!ENDIF + +# Check for the predefined command macro RC. This should point to the resource +# compiler binary for the target platform. If it is not defined, simply define +# it to the legacy default value 'rc.exe'. +# +!IFNDEF RC +RC = rc.exe +!ENDIF + +# Check for the MSVC runtime library path macro. Othertise, this value will +# default to the 'lib' directory underneath the MSVC installation directory. +# +!IFNDEF CRTLIBPATH +CRTLIBPATH = $(VCINSTALLDIR)\lib +!ENDIF + +CRTLIBPATH = $(CRTLIBPATH:\\=\) + +# Check for the command macro NCC. This should point to the compiler binary +# for the platform the compilation process is taking place on. If it is not +# defined, simply define it to have the same value as the CC macro. When +# cross-compiling, it is suggested that this macro be modified via the command +# line (since nmake itself does not provide a built-in method to guess it). +# For example, to use the x86 compiler when cross-compiling for x64, a command +# line similar to the following could be used (all on one line): +# +# nmake /f Makefile.msc sqlite3.dll +# XCOMPILE=1 USE_NATIVE_LIBPATHS=1 +# +# Alternatively, the full path and file name to the compiler binary for the +# platform the compilation process is taking place may be specified (all on +# one line): +# +# nmake /f Makefile.msc sqlite3.dll +# "NCC=""%VCINSTALLDIR%\bin\cl.exe""" +# USE_NATIVE_LIBPATHS=1 +# +!IFDEF NCC +NCC = $(NCC:\\=\) +!ELSEIF $(XCOMPILE)!=0 +NCC = "$(VCINSTALLDIR)\bin\$(CC)" +NCC = $(NCC:\\=\) +!ELSE +NCC = $(CC) +!ENDIF + +# Check for the MSVC native runtime library path macro. Othertise, +# this value will default to the 'lib' directory underneath the MSVC +# installation directory. +# +!IFNDEF NCRTLIBPATH +NCRTLIBPATH = $(VCINSTALLDIR)\lib +!ENDIF + +NCRTLIBPATH = $(NCRTLIBPATH:\\=\) + +# Check for the Platform SDK library path macro. Othertise, this +# value will default to the 'lib' directory underneath the Windows +# SDK installation directory (the environment variable used appears +# to be available when using Visual C++ 2008 or later via the +# command line). +# +!IFNDEF NSDKLIBPATH +NSDKLIBPATH = $(WINDOWSSDKDIR)\lib +!ENDIF + +NSDKLIBPATH = $(NSDKLIBPATH:\\=\) + +# C compiler and options for use in building executables that +# will run on the platform that is doing the build. +# +!IF $(USE_FULLWARN)!=0 +BCC = $(NCC) -nologo -W4 $(CCOPTS) $(BCCOPTS) +!ELSE +BCC = $(NCC) -nologo -W3 $(CCOPTS) $(BCCOPTS) +!ENDIF + +# Check if assembly code listings should be generated for the source +# code files to be compiled. +# +!IF $(USE_LISTINGS)!=0 +BCC = $(BCC) -FAcs +!ENDIF + +# Check if the native library paths should be used when compiling +# the command line tools used during the compilation process. If +# so, set the necessary macro now. +# +!IF $(USE_NATIVE_LIBPATHS)!=0 +NLTLIBPATHS = "/LIBPATH:$(NCRTLIBPATH)" "/LIBPATH:$(NSDKLIBPATH)" + +!IFDEF NUCRTLIBPATH +NUCRTLIBPATH = $(NUCRTLIBPATH:\\=\) +NLTLIBPATHS = $(NLTLIBPATHS) "/LIBPATH:$(NUCRTLIBPATH)" +!ENDIF +!ENDIF + +# C compiler and options for use in building executables that +# will run on the target platform. (BCC and TCC are usually the +# same unless your are cross-compiling.) +# +!IF $(USE_FULLWARN)!=0 +TCC = $(CC) -nologo -W4 -DINCLUDE_MSVC_H=1 $(CCOPTS) $(TCCOPTS) +!ELSE +TCC = $(CC) -nologo -W3 $(CCOPTS) $(TCCOPTS) +!ENDIF + +TCC = $(TCC) -DSQLITE_OS_WIN=1 -I$(TOP) -fp:precise +RCC = $(RC) -DSQLITE_OS_WIN=1 -I$(TOP) $(RCOPTS) $(RCCOPTS) + +# Adjust the names of the primary targets for use with Windows 10. +# +!IF $(FOR_WIN10)!=0 +SQLITE3DLL = winsqlite3.dll +SQLITE3LIB = winsqlite3.lib +SQLITE3EXE = winsqlite3shell.exe +SQLITE3EXEPDB = +!ENDIF + +# Check if we want to use the "stdcall" calling convention when compiling. +# This is not supported by the compilers for non-x86 platforms. It should +# also be noted here that building any target with these "stdcall" options +# will most likely fail if the Tcl library is also required. This is due +# to how the Tcl library functions are declared and exported (i.e. without +# an explicit calling convention, which results in "cdecl"). +# +!IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0 +!IF "$(PLATFORM)"=="x86" +CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall +SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall +!ELSE +!IFNDEF PLATFORM +CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall +SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall +!ELSE +CORE_CCONV_OPTS = +SHELL_CCONV_OPTS = +!ENDIF +!ENDIF +!ELSE +CORE_CCONV_OPTS = +SHELL_CCONV_OPTS = +!ENDIF + +# These are additional compiler options used for the core library. +# +!IFNDEF CORE_COMPILE_OPTS +!IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0 +CORE_COMPILE_OPTS = $(CORE_CCONV_OPTS) -DSQLITE_API=__declspec(dllexport) +!ELSE +CORE_COMPILE_OPTS = $(CORE_CCONV_OPTS) +!ENDIF +!ENDIF + +# These are the additional targets that the core library should depend on +# when linking. +# +!IFNDEF CORE_LINK_DEP +!IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0 +CORE_LINK_DEP = +!ELSE +CORE_LINK_DEP = +!ENDIF +!ENDIF + +# These are additional linker options used for the core library. +# +!IFNDEF CORE_LINK_OPTS +!IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0 +CORE_LINK_OPTS = +!ELSE +CORE_LINK_OPTS = +!ENDIF +!ENDIF + +# These are additional compiler options used for the shell executable. +# +!IFNDEF SHELL_COMPILE_OPTS +!IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0 +SHELL_COMPILE_OPTS = $(SHELL_CCONV_OPTS) -DSQLITE_API=__declspec(dllimport) +!ELSE +SHELL_COMPILE_OPTS = $(SHELL_CCONV_OPTS) +!ENDIF +!ENDIF + +# This is the source code that the shell executable should be compiled +# with. +# +!IFNDEF SHELL_CORE_SRC +!IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0 +SHELL_CORE_SRC = +!ELSE +SHELL_CORE_SRC = $(SQLITE3C) +!ENDIF +!ENDIF + +# This is the core library that the shell executable should depend on. +# +!IFNDEF SHELL_CORE_DEP +!IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0 +SHELL_CORE_DEP = $(SQLITE3DLL) +!ELSE +SHELL_CORE_DEP = +!ENDIF +!ENDIF + +# This is the core library that the shell executable should link with. +# +!IFNDEF SHELL_CORE_LIB +!IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0 +SHELL_CORE_LIB = $(SQLITE3LIB) +!ELSE +SHELL_CORE_LIB = +!ENDIF +!ENDIF + +# These are additional linker options used for the shell executable. +# +!IFNDEF SHELL_LINK_OPTS +SHELL_LINK_OPTS = $(SHELL_CORE_LIB) +!ENDIF + +# Check if assembly code listings should be generated for the source +# code files to be compiled. +# +!IF $(USE_LISTINGS)!=0 +TCC = $(TCC) -FAcs +!ENDIF + +# When compiling the library for use in the WinRT environment, +# the following compile-time options must be used as well to +# disable use of Win32 APIs that are not available and to enable +# use of Win32 APIs that are specific to Windows 8 and/or WinRT. +# +!IF $(FOR_WINRT)!=0 +TCC = $(TCC) -DSQLITE_OS_WINRT=1 +RCC = $(RCC) -DSQLITE_OS_WINRT=1 +TCC = $(TCC) -DWINAPI_FAMILY=WINAPI_FAMILY_APP +RCC = $(RCC) -DWINAPI_FAMILY=WINAPI_FAMILY_APP +!ENDIF + +# C compiler options for the Windows 10 platform (needs MSVC 2015). +# +!IF $(FOR_WIN10)!=0 +TCC = $(TCC) /guard:cf -D_ARM_WINAPI_PARTITION_DESKTOP_SDK_AVAILABLE +BCC = $(BCC) /guard:cf -D_ARM_WINAPI_PARTITION_DESKTOP_SDK_AVAILABLE +!ENDIF + +# Also, we need to dynamically link to the correct MSVC runtime +# when compiling for WinRT (e.g. debug or release) OR if the +# USE_CRT_DLL option is set to force dynamically linking to the +# MSVC runtime library. +# +!IF $(FOR_WINRT)!=0 || $(FOR_WIN10)!=0 || $(USE_CRT_DLL)!=0 +!IF $(DEBUG)>1 +TCC = $(TCC) -MDd +BCC = $(BCC) -MDd +!ELSE +TCC = $(TCC) -MD +BCC = $(BCC) -MD +!ENDIF +!ELSE +!IF $(DEBUG)>1 +TCC = $(TCC) -MTd +BCC = $(BCC) -MTd +!ELSE +TCC = $(TCC) -MT +BCC = $(BCC) -MT +!ENDIF +!ENDIF + + +# Define -DNDEBUG to compile without debugging (i.e., for production usage) +# Omitting the define will cause extra debugging code to be inserted and +# includes extra comments when "EXPLAIN stmt" is used. +# +!IF $(DEBUG)==0 +TCC = $(TCC) -DNDEBUG +BCC = $(BCC) -DNDEBUG +RCC = $(RCC) -DNDEBUG +!ENDIF + +!IF $(DEBUG)>0 || $(API_ARMOR)!=0 || $(FOR_WIN10)!=0 +TCC = $(TCC) -DSQLITE_ENABLE_API_ARMOR=1 +RCC = $(RCC) -DSQLITE_ENABLE_API_ARMOR=1 +!ENDIF + +!IF $(DEBUG)>2 +TCC = $(TCC) -DSQLITE_DEBUG=1 +RCC = $(RCC) -DSQLITE_DEBUG=1 +!ENDIF + +!IF $(DEBUG)>4 || $(OSTRACE)!=0 +TCC = $(TCC) -DSQLITE_FORCE_OS_TRACE=1 -DSQLITE_DEBUG_OS_TRACE=1 +RCC = $(RCC) -DSQLITE_FORCE_OS_TRACE=1 -DSQLITE_DEBUG_OS_TRACE=1 +!ENDIF + +!IF $(DEBUG)>5 +TCC = $(TCC) -DSQLITE_ENABLE_IOTRACE=1 +RCC = $(RCC) -DSQLITE_ENABLE_IOTRACE=1 +!ENDIF + +# Prevent warnings about "insecure" MSVC runtime library functions +# being used. +# +TCC = $(TCC) -D_CRT_SECURE_NO_DEPRECATE -D_CRT_SECURE_NO_WARNINGS +BCC = $(BCC) -D_CRT_SECURE_NO_DEPRECATE -D_CRT_SECURE_NO_WARNINGS +RCC = $(RCC) -D_CRT_SECURE_NO_DEPRECATE -D_CRT_SECURE_NO_WARNINGS + +# Prevent warnings about "deprecated" POSIX functions being used. +# +TCC = $(TCC) -D_CRT_NONSTDC_NO_DEPRECATE -D_CRT_NONSTDC_NO_WARNINGS +BCC = $(BCC) -D_CRT_NONSTDC_NO_DEPRECATE -D_CRT_NONSTDC_NO_WARNINGS +RCC = $(RCC) -D_CRT_NONSTDC_NO_DEPRECATE -D_CRT_NONSTDC_NO_WARNINGS + +# Use the SQLite debugging heap subsystem? +# +!IF $(MEMDEBUG)!=0 +TCC = $(TCC) -DSQLITE_MEMDEBUG=1 +RCC = $(RCC) -DSQLITE_MEMDEBUG=1 + +# Use native Win32 heap subsystem instead of malloc/free? +# +!ELSEIF $(WIN32HEAP)!=0 +TCC = $(TCC) -DSQLITE_WIN32_MALLOC=1 +RCC = $(RCC) -DSQLITE_WIN32_MALLOC=1 + +# Validate the heap on every call into the native Win32 heap subsystem? +# +!IF $(DEBUG)>3 +TCC = $(TCC) -DSQLITE_WIN32_MALLOC_VALIDATE=1 +RCC = $(RCC) -DSQLITE_WIN32_MALLOC_VALIDATE=1 +!ENDIF +!ENDIF + + +# Compiler options needed for programs that use the readline() library. +# +!IFNDEF READLINE_FLAGS +READLINE_FLAGS = -DHAVE_READLINE=0 +!ENDIF + +# The library that programs using readline() must link against. +# +!IFNDEF LIBREADLINE +LIBREADLINE = +!ENDIF + +# Should the database engine be compiled threadsafe +# +TCC = $(TCC) -DSQLITE_THREADSAFE=1 +RCC = $(RCC) -DSQLITE_THREADSAFE=1 + +# Do threads override each others locks by default (1), or do we test (-1) +# +TCC = $(TCC) -DSQLITE_THREAD_OVERRIDE_LOCK=-1 +RCC = $(RCC) -DSQLITE_THREAD_OVERRIDE_LOCK=-1 + +# Any target libraries which libsqlite must be linked against +# +!IFNDEF TLIBS +TLIBS = +!ENDIF + +# Flags controlling use of the in memory btree implementation +# +# SQLITE_TEMP_STORE is 0 to force temporary tables to be in a file, 1 to +# default to file, 2 to default to memory, and 3 to force temporary +# tables to always be in memory. +# +TCC = $(TCC) -DSQLITE_TEMP_STORE=1 +RCC = $(RCC) -DSQLITE_TEMP_STORE=1 + +# Enable/disable loadable extensions, and other optional features +# based on configuration. (-DSQLITE_OMIT*, -DSQLITE_ENABLE*). +# The same set of OMIT and ENABLE flags should be passed to the +# LEMON parser generator and the mkkeywordhash tool as well. + +# These are the required SQLite compilation options used when compiling for +# the Windows platform. +# +REQ_FEATURE_FLAGS = $(REQ_FEATURE_FLAGS) -DSQLITE_MAX_TRIGGER_DEPTH=100 + +# If we are linking to the RPCRT4 library, enable features that need it. +# +!IF $(USE_RPCRT4_LIB)!=0 +REQ_FEATURE_FLAGS = $(REQ_FEATURE_FLAGS) -DSQLITE_WIN32_USE_UUID=1 +!ENDIF + +# Add the required and optional SQLite compilation options into the command +# lines used to invoke the MSVC code and resource compilers. +# +TCC = $(TCC) $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(EXT_FEATURE_FLAGS) +RCC = $(RCC) $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(EXT_FEATURE_FLAGS) + +# Add in any optional parameters specified on the commane line, e.g. +# nmake /f Makefile.msc all "OPTS=-DSQLITE_ENABLE_FOO=1 -DSQLITE_OMIT_FOO=1" +# +TCC = $(TCC) $(OPTS) +RCC = $(RCC) $(OPTS) + +# If compiling for debugging, add some defines. +# +!IF $(DEBUG)>1 +TCC = $(TCC) -D_DEBUG +BCC = $(BCC) -D_DEBUG +RCC = $(RCC) -D_DEBUG +!ENDIF + +# If optimizations are enabled or disabled (either implicitly or +# explicitly), add the necessary flags. +# +!IF $(DEBUG)>1 || $(OPTIMIZATIONS)==0 +TCC = $(TCC) -Od +BCC = $(BCC) -Od +!ELSEIF $(OPTIMIZATIONS)>=3 +TCC = $(TCC) -Ox +BCC = $(BCC) -Ox +!ELSEIF $(OPTIMIZATIONS)==2 +TCC = $(TCC) -O2 +BCC = $(BCC) -O2 +!ELSEIF $(OPTIMIZATIONS)==1 +TCC = $(TCC) -O1 +BCC = $(BCC) -O1 +!ENDIF + +# If symbols are enabled (or compiling for debugging), enable PDBs. +# +!IF $(DEBUG)>1 || $(SYMBOLS)!=0 +TCC = $(TCC) -Zi +BCC = $(BCC) -Zi +!ENDIF + + +# Command line prefixes for compiling code, compiling resources, +# linking, etc. +# +LTCOMPILE = $(TCC) -Fo$@ +LTRCOMPILE = $(RCC) -r +LTLIB = lib.exe +LTLINK = $(TCC) -Fe$@ + +# If requested, link to the RPCRT4 library. +# +!IF $(USE_RPCRT4_LIB)!=0 +LTLINK = $(LTLINK) rpcrt4.lib +!ENDIF + +# If a platform was set, force the linker to target that. +# Note that the vcvars*.bat family of batch files typically +# set this for you. Otherwise, the linker will attempt +# to deduce the binary type based on the object files. +!IFDEF PLATFORM +LTLINKOPTS = /NOLOGO /MACHINE:$(PLATFORM) +LTLIBOPTS = /NOLOGO /MACHINE:$(PLATFORM) +!ELSE +LTLINKOPTS = /NOLOGO +LTLIBOPTS = /NOLOGO +!ENDIF + +# When compiling for use in the WinRT environment, the following +# linker option must be used to mark the executable as runnable +# only in the context of an application container. +# +!IF $(FOR_WINRT)!=0 +LTLINKOPTS = $(LTLINKOPTS) /APPCONTAINER +!IF "$(VISUALSTUDIOVERSION)"=="12.0" || "$(VISUALSTUDIOVERSION)"=="14.0" +!IFNDEF STORELIBPATH +!IF "$(PLATFORM)"=="x86" +STORELIBPATH = $(CRTLIBPATH)\store +!ELSEIF "$(PLATFORM)"=="x64" +STORELIBPATH = $(CRTLIBPATH)\store\amd64 +!ELSEIF "$(PLATFORM)"=="ARM" +STORELIBPATH = $(CRTLIBPATH)\store\arm +!ELSE +STORELIBPATH = $(CRTLIBPATH)\store +!ENDIF +!ENDIF +STORELIBPATH = $(STORELIBPATH:\\=\) +LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(STORELIBPATH)" +!ENDIF +!ENDIF + +# When compiling for Windows Phone 8.1, an extra library path is +# required. +# +!IF $(USE_WP81_OPTS)!=0 +!IFNDEF WP81LIBPATH +!IF "$(PLATFORM)"=="x86" +WP81LIBPATH = $(PROGRAMFILES_X86)\Windows Phone Kits\8.1\lib\x86 +!ELSEIF "$(PLATFORM)"=="ARM" +WP81LIBPATH = $(PROGRAMFILES_X86)\Windows Phone Kits\8.1\lib\ARM +!ELSE +WP81LIBPATH = $(PROGRAMFILES_X86)\Windows Phone Kits\8.1\lib\x86 +!ENDIF +!ENDIF +!ENDIF + +# When compiling for Windows Phone 8.1, some extra linker options +# are also required. +# +!IF $(USE_WP81_OPTS)!=0 +!IFDEF WP81LIBPATH +LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(WP81LIBPATH)" +!ENDIF +LTLINKOPTS = $(LTLINKOPTS) /DYNAMICBASE +LTLINKOPTS = $(LTLINKOPTS) WindowsPhoneCore.lib RuntimeObject.lib PhoneAppModelHost.lib +LTLINKOPTS = $(LTLINKOPTS) /NODEFAULTLIB:kernel32.lib /NODEFAULTLIB:ole32.lib +!ENDIF + +# When compiling for UAP, some extra linker options are also required. +# +!IF $(FOR_UAP)!=0 +LTLINKOPTS = $(LTLINKOPTS) /DYNAMICBASE /NODEFAULTLIB:kernel32.lib +LTLINKOPTS = $(LTLINKOPTS) mincore.lib +!IFDEF PSDKLIBPATH +LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(PSDKLIBPATH)" +!ENDIF +!ENDIF + +# If either debugging or symbols are enabled, enable PDBs. +# +!IF $(DEBUG)>1 || $(SYMBOLS)!=0 +LDFLAGS = /DEBUG $(LDOPTS) +!ELSE +LDFLAGS = $(LDOPTS) +!ENDIF + + +# You should not have to change anything below this line +############################################################################### + + +# Object files for the amalgamation. +# +LIBOBJS1 = sqlite3.lo + +# Determine the real value of LIBOBJ based on the 'configure' script +# +LIBOBJ = $(LIBOBJS1) + +# Determine if embedded resource compilation and usage are enabled. +# +!IF $(USE_RC)!=0 +LIBRESOBJS = sqlite3res.lo +!ELSE +LIBRESOBJS = +!ENDIF + + +# Additional compiler options for the shell. These are only effective +# when the shell is not being dynamically linked. +# +!IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0 +SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS5 +!ENDIF + + +# This is the default Makefile target. The objects listed here +# are what get build when you type just "make" with no arguments. +# +all: dll libsqlite3.lib shell + +# Dynamic link library section. +# +dll: $(SQLITE3DLL) + +# Shell executable. +# +shell: $(SQLITE3EXE) + +libsqlite3.lib: $(LIBOBJ) + $(LTLIB) $(LTLIBOPTS) /OUT:$@ $(LIBOBJ) $(TLIBS) + + +$(SQLITE3DLL): $(LIBOBJ) $(LIBRESOBJS) $(CORE_LINK_DEP) + $(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL $(CORE_LINK_OPTS) /OUT:$@ $(LIBOBJ) $(LIBRESOBJS) $(LTLIBS) $(TLIBS) + + +$(SQLITE3EXE): $(TOP)\shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H) + $(LTLINK) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) $(TOP)\shell.c $(SHELL_CORE_SRC) \ + /link $(SQLITE3EXEPDB) $(LDFLAGS) $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS) + + +# Rule to build the amalgamation +# +sqlite3.lo: $(SQLITE3C) + $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(SQLITE3C) + + +# Rule to build the Win32 resources object file. +# +!IF $(USE_RC)!=0 +_HASHCHAR=^# +!IF ![echo !IFNDEF VERSION > rcver.vc] && \ + ![for /F "delims=" %V in ('type "$(SQLITE3H)" ^| find "$(_HASHCHAR)define SQLITE_VERSION "') do (echo VERSION = ^^%V >> rcver.vc)] && \ + ![echo !ENDIF >> rcver.vc] +!INCLUDE rcver.vc +!ENDIF + +RESOURCE_VERSION = $(VERSION:^#=) +RESOURCE_VERSION = $(RESOURCE_VERSION:define=) +RESOURCE_VERSION = $(RESOURCE_VERSION:SQLITE_VERSION=) +RESOURCE_VERSION = $(RESOURCE_VERSION:"=) +RESOURCE_VERSION = $(RESOURCE_VERSION:.=,) + +$(LIBRESOBJS): $(TOP)\sqlite3.rc rcver.vc $(SQLITE3H) + echo #ifndef SQLITE_RESOURCE_VERSION > sqlite3rc.h + echo #define SQLITE_RESOURCE_VERSION $(RESOURCE_VERSION) >> sqlite3rc.h + echo #endif >> sqlite3rc.h + $(LTRCOMPILE) -fo $(LIBRESOBJS) -DRC_VERONLY $(TOP)\sqlite3.rc +!ENDIF + + +clean: + del /Q *.exp *.lo *.ilk *.lib *.obj *.ncb *.pdb *.sdf *.suo 2>NUL + del /Q *.bsc *.cod *.da *.bb *.bbg *.vc gmon.out 2>NUL diff --git a/Makefile.msc b/Makefile.msc index 5921e9375a..ab0ba95d53 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -10,11 +10,13 @@ # TOP = . +# <> # Set this non-0 to create and use the SQLite amalgamation file. # !IFNDEF USE_AMALGAMATION USE_AMALGAMATION = 1 !ENDIF +# <> # Set this non-0 to enable full warnings (-W4, etc) when compiling. # @@ -68,11 +70,13 @@ USE_WP81_OPTS = 0 SPLIT_AMALGAMATION = 0 !ENDIF +# <> # Set this non-0 to use the International Components for Unicode (ICU). # !IFNDEF USE_ICU USE_ICU = 0 !ENDIF +# <> # Set this non-0 to dynamically link to the MSVC runtime library. # @@ -136,12 +140,14 @@ FOR_UAP = 0 FOR_WIN10 = 0 !ENDIF +# <> # Set this non-0 to skip attempting to look for and/or link with the Tcl # runtime library. # !IFNDEF NO_TCL NO_TCL = 0 !ENDIF +# <> # Set this to non-0 to create and use PDBs. # @@ -553,6 +559,7 @@ BCC = $(BCC) -MT !ENDIF !ENDIF +# <> # The mksqlite3c.tcl and mksqlite3h.tcl scripts will pull in # any extension header files by default. For non-amalgamation # builds, we need to make sure the compiler can find these. @@ -576,6 +583,7 @@ MKSQLITE3C_ARGS = --linemacros MKSQLITE3C_ARGS = !ENDIF !ENDIF +# <> # Define -DNDEBUG to compile without debugging (i.e., for production usage) # Omitting the define will cause extra debugging code to be inserted and @@ -640,6 +648,7 @@ RCC = $(RCC) -DSQLITE_WIN32_MALLOC_VALIDATE=1 !ENDIF !ENDIF +# <> # The locations of the Tcl header and library files. Also, the library that # non-stubs enabled programs using Tcl must link against. These variables # (TCLINCDIR, TCLLIBDIR, and LIBTCL) may be overridden via the environment @@ -691,6 +700,7 @@ LIBICU = icuuc.lib icuin.lib !IFNDEF TCLSH_CMD TCLSH_CMD = tclsh85 !ENDIF +# <> # Compiler options needed for programs that use the readline() library. # @@ -789,6 +799,7 @@ TCC = $(TCC) -Zi BCC = $(BCC) -Zi !ENDIF +# <> # If ICU support is enabled, add the compiler options for it. # !IF $(USE_ICU)!=0 @@ -799,6 +810,7 @@ RCC = $(RCC) -I$(TOP)\ext\icu TCC = $(TCC) -I$(ICUINCDIR) RCC = $(RCC) -I$(ICUINCDIR) !ENDIF +# <> # Command line prefixes for compiling code, compiling resources, # linking, etc. @@ -894,6 +906,7 @@ LDFLAGS = /DEBUG $(LDOPTS) LDFLAGS = $(LDOPTS) !ENDIF +# <> # Start with the Tcl related linker options. # !IF $(NO_TCL)==0 @@ -907,10 +920,12 @@ LTLIBS = $(LIBTCL) LTLIBPATHS = $(LTLIBPATHS) /LIBPATH:$(ICULIBDIR) LTLIBS = $(LTLIBS) $(LIBICU) !ENDIF +# <> # You should not have to change anything below this line ############################################################################### +# <> # Object files for the SQLite library (non-amalgamation). # LIBOBJS0 = vdbe.lo parse.lo alter.lo analyze.lo attach.lo auth.lo \ @@ -934,6 +949,7 @@ LIBOBJS0 = vdbe.lo parse.lo alter.lo analyze.lo attach.lo auth.lo \ vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \ vdbetrace.lo wal.lo walker.lo where.lo wherecode.lo whereexpr.lo \ utf.lo vtab.lo +# <> # Object files for the amalgamation. # @@ -941,11 +957,15 @@ LIBOBJS1 = sqlite3.lo # Determine the real value of LIBOBJ based on the 'configure' script # +# <> !IF $(USE_AMALGAMATION)==0 LIBOBJ = $(LIBOBJS0) !ELSE +# <> LIBOBJ = $(LIBOBJS1) +# <> !ENDIF +# <> # Determine if embedded resource compilation and usage are enabled. # @@ -955,6 +975,7 @@ LIBRESOBJS = sqlite3res.lo LIBRESOBJS = !ENDIF +# <> # All of the source code files. # SRC1 = \ @@ -1303,6 +1324,7 @@ FUZZDATA = \ $(TOP)\test\fuzzdata2.db \ $(TOP)\test\fuzzdata3.db \ $(TOP)\test\fuzzdata4.db +# <> # Additional compiler options for the shell. These are only effective # when the shell is not being dynamically linked. @@ -1311,6 +1333,7 @@ FUZZDATA = \ SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS5 !ENDIF +# <> # Extra compiler options for various test tools. # MPTESTER_COMPILE_OPTS = -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS5 @@ -1320,6 +1343,7 @@ FUZZCHECK_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 # Standard options to testfixture. # TESTOPTS = --verbose=file --output=test-out.txt +# <> # This is the default Makefile target. The objects listed here # are what get build when you type just "make" with no arguments. @@ -1337,22 +1361,27 @@ shell: $(SQLITE3EXE) libsqlite3.lib: $(LIBOBJ) $(LTLIB) $(LTLIBOPTS) /OUT:$@ $(LIBOBJ) $(TLIBS) +# <> libtclsqlite3.lib: tclsqlite.lo libsqlite3.lib $(LTLIB) $(LTLIBOPTS) $(LTLIBPATHS) /OUT:$@ tclsqlite.lo libsqlite3.lib $(LIBTCLSTUB) $(TLIBS) +# <> $(SQLITE3DLL): $(LIBOBJ) $(LIBRESOBJS) $(CORE_LINK_DEP) $(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL $(CORE_LINK_OPTS) /OUT:$@ $(LIBOBJ) $(LIBRESOBJS) $(LTLIBS) $(TLIBS) +# <> sqlite3.def: libsqlite3.lib echo EXPORTS > sqlite3.def dumpbin /all libsqlite3.lib \ | $(TCLSH_CMD) $(TOP)\tool\replace.tcl include "^\s+1 _?(sqlite3_.*)$$" \1 \ | sort >> sqlite3.def +# <> $(SQLITE3EXE): $(TOP)\src\shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H) $(LTLINK) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) $(TOP)\src\shell.c $(SHELL_CORE_SRC) \ /link $(SQLITE3EXEPDB) $(LDFLAGS) $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS) +# <> sqldiff.exe: $(TOP)\tool\sqldiff.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) $(TOP)\tool\sqldiff.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) @@ -1406,12 +1435,14 @@ sqlite3.c: .target_source sqlite3ext.h $(TOP)\tool\mksqlite3c.tcl sqlite3-all.c: sqlite3.c $(TOP)\tool\split-sqlite3c.tcl $(TCLSH_CMD) $(TOP)\tool\split-sqlite3c.tcl +# <> # Rule to build the amalgamation # sqlite3.lo: $(SQLITE3C) $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(SQLITE3C) +# <> # Rules to build the LEMON compiler generator # lempar.c: $(TOP)\tool\lempar.c @@ -1432,10 +1463,12 @@ parse.lo: parse.c $(HDR) opcodes.lo: opcodes.c $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c opcodes.c +# <> # Rule to build the Win32 resources object file. # !IF $(USE_RC)!=0 +# <> $(LIBRESOBJS): $(TOP)\src\sqlite3.rc $(SQLITE3H) echo #ifndef SQLITE_RESOURCE_VERSION > sqlite3rc.h for /F %%V in ('type "$(TOP)\VERSION"') do ( \ @@ -1444,8 +1477,10 @@ $(LIBRESOBJS): $(TOP)\src\sqlite3.rc $(SQLITE3H) ) echo #endif >> sqlite3rc.h $(LTRCOMPILE) -fo $(LIBRESOBJS) $(TOP)\src\sqlite3.rc +# <> !ENDIF +# <> # Rules to build individual *.lo files from files in the src directory. # alter.lo: $(TOP)\src\alter.c $(HDR) @@ -1945,10 +1980,12 @@ speedtest1.exe: $(TOP)\test\speedtest1.c $(SQLITE3C) $(SQLITE3H) rbu.exe: $(TOP)\ext\rbu\rbu.c $(TOP)\ext\rbu\sqlite3rbu.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) -DSQLITE_ENABLE_RBU -Fe$@ \ $(TOP)\ext\rbu\rbu.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) +# <> clean: del /Q *.exp *.lo *.ilk *.lib *.obj *.ncb *.pdb *.sdf *.suo 2>NUL del /Q *.bsc *.cod *.da *.bb *.bbg *.vc gmon.out 2>NUL +# <> del /Q $(SQLITE3C) $(SQLITE3H) opcodes.c opcodes.h 2>NUL del /Q lemon.* lempar.c parse.* 2>NUL del /Q mkkeywordhash.* keywordhash.h 2>NUL @@ -1974,3 +2011,4 @@ clean: del /Q sqlite-*-output.vsix 2>NUL del /Q fuzzershell.exe fuzzcheck.exe sqldiff.exe 2>NUL del /Q fts5.* fts5parse.* 2>NUL +# <> diff --git a/manifest b/manifest index 61cf5d66ed..12db615b93 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,9 @@ -C More\swork\son\sthe\sMSVC\sbuild. -D 2016-01-23T00:07:51.151 +C Add\stooling\sto\screate\san\sMSVC\sMakefile\scapable\sof\sbuilding\sthe\score\slibrary\sand\sshell\sonly,\susing\sa\spre-built\samalgamation. +D 2016-01-23T07:53:04.796 F Makefile.in 027c1603f255390c43a426671055a31c0a65fdb4 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 -F Makefile.msc 1708a78eda223b6daa302b140037fcc214a779f9 +F Makefile.min.msc c805e4a474fc7610d4f0cd0b6a9f56334a14c6ad +F Makefile.msc 6fca5455aaecbd14479f33f091aa19df2d3d2969 F README.md 8ecc12493ff9f820cdea6520a9016001cb2e59b7 F VERSION 866588d1edf0ccb5b0d33896974338f97564f719 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -1378,6 +1379,7 @@ F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862 F tool/logest.c eef612f8adf4d0993dafed0416064cf50d5d33c6 F tool/mkautoconfamal.sh 5a5441280b509d2bb3bdc71bfb63781b0d570373 F tool/mkkeywordhash.c 06ec0b78bd4fa68c12d90ef2bdfe76b039133ff8 +F tool/mkmsvcmin.tcl 859737ff14668827738a3f7ca17ba643a88bcecc F tool/mkopcodec.tcl edde8adc42621b5e598127f8cdc6d52cfe21f52b F tool/mkopcodeh.tcl e04177031532b7aa9379ded50e820231ac4abd6e F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e @@ -1419,7 +1421,10 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P a1c8116ced62d81f3f5ca26bbe0877e829d4cc56 -R b34246fa017fc0ae2e7bf7cb484e3eeb +P a79c46bc61a35edbaf6112c26e8052314b0eb16a +R bd00cfb25453f633191b5f1755909575 +T *branch * msvcMakeMin +T *sym-msvcMakeMin * +T -sym-trunk * U mistachkin -Z ba075ff448ff99fa520b3503324fe6e9 +Z 18f99743dd26f3a3f31830457ad1b4ef diff --git a/manifest.uuid b/manifest.uuid index d1ea2b33a4..06bbe8845c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a79c46bc61a35edbaf6112c26e8052314b0eb16a \ No newline at end of file +c46f1a13cb00264d1164399059250884c7be4ea0 \ No newline at end of file diff --git a/tool/mkmsvcmin.tcl b/tool/mkmsvcmin.tcl new file mode 100644 index 0000000000..56c9d52811 --- /dev/null +++ b/tool/mkmsvcmin.tcl @@ -0,0 +1,88 @@ +#!/usr/bin/tcl +# +# Removes and/or replaces specially marked sections from the Makefile +# for MSVC, writing the resulting output to another (different) file. +# In addition, some other strings are also removed and/or replaced if +# they are present in the Makefile. +# +set fromFileName [lindex $argv 0] + +if {![file exists $fromFileName]} then { + error "input file \"$fromFileName\" does not exist" +} + +set toFileName [lindex $argv 1] + +if {[file exists $toFileName]} then { + error "output file \"$toFileName\" already exists" +} + +proc readFile { fileName } { + set file_id [open $fileName RDONLY] + fconfigure $file_id -encoding binary -translation binary + set result [read $file_id] + close $file_id + return $result +} + +proc writeFile { fileName data } { + set file_id [open $fileName {WRONLY CREAT TRUNC}] + fconfigure $file_id -encoding binary -translation binary + puts -nonewline $file_id $data + close $file_id + return "" +} + +proc escapeSubSpec { data } { + regsub -all -- {&} $data {\\\&} data + regsub -all -- {\\(\d+)} $data {\\\\\1} data + return $data +} + +proc substVars { data } { + return [uplevel 1 [list subst -nocommands -nobackslashes $data]] +} + +# +# NOTE: This block is used to replace the section marked <> in +# the Makefile, if it exists. +# +set blocks(1) [string trimleft [string map [list \\\\ \\] { +_HASHCHAR=^# +!IF ![echo !IFNDEF VERSION > rcver.vc] && \\ + ![for /F "delims=" %V in ('type "$(SQLITE3H)" ^| find "$(_HASHCHAR)define SQLITE_VERSION "') do (echo VERSION = ^^%V >> rcver.vc)] && \\ + ![echo !ENDIF >> rcver.vc] +!INCLUDE rcver.vc +!ENDIF + +RESOURCE_VERSION = $(VERSION:^#=) +RESOURCE_VERSION = $(RESOURCE_VERSION:define=) +RESOURCE_VERSION = $(RESOURCE_VERSION:SQLITE_VERSION=) +RESOURCE_VERSION = $(RESOURCE_VERSION:"=) +RESOURCE_VERSION = $(RESOURCE_VERSION:.=,) + +$(LIBRESOBJS): $(TOP)\sqlite3.rc rcver.vc $(SQLITE3H) + echo #ifndef SQLITE_RESOURCE_VERSION > sqlite3rc.h + echo #define SQLITE_RESOURCE_VERSION $(RESOURCE_VERSION) >> sqlite3rc.h + echo #endif >> sqlite3rc.h + $(LTRCOMPILE) -fo $(LIBRESOBJS) -DRC_VERONLY $(TOP)\sqlite3.rc +}]] + +set data [readFile $fromFileName] + +regsub -all -- {# <>\n.*?# <>\n} \ + $data "" data + +foreach i [lsort [array names blocks]] { + regsub -all -- [substVars \ + {# <>\n.*?# <>\n}] \ + $data [escapeSubSpec $blocks($i)] data +} + +set data [string map [list " -I\$(TOP)\\src" ""] $data] +set data [string map [list " /DEF:sqlite3.def" ""] $data] +set data [string map [list " sqlite3.def" ""] $data] +set data [string map [list " libtclsqlite3.lib" ""] $data] +set data [string map [list "\$(TOP)\\src\\" "\$(TOP)\\"] $data] + +writeFile $toFileName $data From c615097d3d9140f9ea2d23faf2dd4ed58475a6ad Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 23 Jan 2016 14:05:27 +0000 Subject: [PATCH 13/39] Remove an assert() that can be false if compiled with SQLITE_USE_ALLOCA. FossilOrigin-Name: f0a551edf87ef061deae34d88b02c3b484ae9adc --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/prepare.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 61cf5d66ed..7b54ea0ab0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C More\swork\son\sthe\sMSVC\sbuild. -D 2016-01-23T00:07:51.151 +C Remove\san\sassert()\sthat\scan\sbe\sfalse\sif\scompiled\swith\sSQLITE_USE_ALLOCA. +D 2016-01-23T14:05:27.109 F Makefile.in 027c1603f255390c43a426671055a31c0a65fdb4 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1708a78eda223b6daa302b140037fcc214a779f9 @@ -340,7 +340,7 @@ F src/pcache.h 4d0ccaad264d360981ec5e6a2b596d6e85242545 F src/pcache1.c 72f644dc9e1468c72922eff5904048427b817051 F src/pragma.c ea290193369faa0a26ae2f924e7b86289b4a7987 F src/pragma.h 64c78a648751b9f4f297276c4eb7507b14b4628c -F src/prepare.c 74855ddbdfad6a1c4a4d5c4b0913ebb01174ba19 +F src/prepare.c 8ca7237428f372a04717d558555ea67ee1c5df93 F src/printf.c af589a27b7d40f6f4f704e9eea99f02f18ad6d32 F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 F src/resolve.c 9f7ce3a3c087afb7597b7c916c99126ff3f12f0c @@ -1419,7 +1419,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P a1c8116ced62d81f3f5ca26bbe0877e829d4cc56 -R b34246fa017fc0ae2e7bf7cb484e3eeb -U mistachkin -Z ba075ff448ff99fa520b3503324fe6e9 +P a79c46bc61a35edbaf6112c26e8052314b0eb16a +R 78c84a447f1adb3672b2aa032cc33ea6 +U drh +Z 2d3baf83036e6553ecd28fa4c13c4bc2 diff --git a/manifest.uuid b/manifest.uuid index d1ea2b33a4..d11f7b73ef 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a79c46bc61a35edbaf6112c26e8052314b0eb16a \ No newline at end of file +f0a551edf87ef061deae34d88b02c3b484ae9adc \ No newline at end of file diff --git a/src/prepare.c b/src/prepare.c index acc70dd2d5..acd50fcaf7 100644 --- a/src/prepare.c +++ b/src/prepare.c @@ -525,7 +525,7 @@ static int sqlite3Prepare( } pParse->pReprepare = pReprepare; assert( ppStmt && *ppStmt==0 ); - assert( !db->mallocFailed ); + /* assert( !db->mallocFailed ); // not true with SQLITE_USE_ALLOCA */ assert( sqlite3_mutex_held(db->mutex) ); /* Check to verify that it is possible to get a read lock on all From 8498f387be1e02e4053c4bf5dcf4ded8a7103484 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 23 Jan 2016 14:45:36 +0000 Subject: [PATCH 14/39] Fix a problem involving detail=col and column filters. FossilOrigin-Name: 7558a0ad2276e91f2faced8ea405d9fdb4fa0c6e --- ext/fts5/fts5_index.c | 14 ++++++++++---- manifest | 15 ++++++--------- manifest.uuid | 2 +- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 2ea57b04f9..8323842dc6 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -5013,21 +5013,26 @@ static void fts5IterSetOutputs_Col(Fts5Iter *pIter, Fts5SegIter *pSeg){ assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_COLUMNS ); assert( pColset ); - if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf && 0 ){ + if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf ){ /* All data is stored on the current page. Populate the output ** variables to point into the body of the page object. */ Fts5PoslistWriter writer = {0}; const u8 *a = &pSeg->pLeaf->p[pSeg->iLeafOffset]; int n = pSeg->nPos; int iCol = 0; + int iCVal = pColset->aiCol[0]; i64 iPos = 0; int iOff = 0; fts5BufferZero(&pIter->poslist); while( 0==sqlite3Fts5PoslistNext64(a, n, &iOff, &iPos) ){ - if( iPos==pColset->aiCol[iCol] ){ - sqlite3Fts5PoslistWriterAppend(&pIter->poslist, &writer, iPos); - if( ++iCol>=pColset->nCol ) break; + while( iPos>=iCVal ){ + if( iPos==iCVal ){ + sqlite3Fts5PoslistWriterAppend(&pIter->poslist, &writer, iPos); + } + if( ++iCol>=pColset->nCol ) goto setoutputs_col_out; + assert( pColset->aiCol[iCol]>iCVal ); + iCVal = pColset->aiCol[iCol]; } } @@ -5039,6 +5044,7 @@ static void fts5IterSetOutputs_Col(Fts5Iter *pIter, Fts5SegIter *pSeg){ fts5SegiterPoslist(pIter->pIndex, pSeg, pColset, &pIter->poslist); } +setoutputs_col_out: pIter->base.pData = pIter->poslist.p; pIter->base.nData = pIter->poslist.n; } diff --git a/manifest b/manifest index 99a9a8b139..0168850d28 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Experimental\sperformance\senhancements\sfor\sfts5. -D 2016-01-22T19:48:34.825 +C Fix\sa\sproblem\sinvolving\sdetail=col\sand\scolumn\sfilters. +D 2016-01-23T14:45:36.706 F Makefile.in 027c1603f255390c43a426671055a31c0a65fdb4 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc d2b93511a969c0c8fcf52aeb5e426571e8c610d2 @@ -103,7 +103,7 @@ F ext/fts5/fts5_buffer.c 7d3f6f01f8fdc45204e6a33925ef8478a67d28dd F ext/fts5/fts5_config.c 0c384ebdd23fd055e2e50a93277b8d59da538238 F ext/fts5/fts5_expr.c 48b9131b74c8d3b8c12ba0f7995e2d60eecce9f2 F ext/fts5/fts5_hash.c 1b113977296cf4212c6ec667d5e3f2bd18036955 -F ext/fts5/fts5_index.c 19062d1f40ba6d88d786a986d1c20f9d741799d0 +F ext/fts5/fts5_index.c bd5476edd4d6ef37fc389794a78f47f29f521634 F ext/fts5/fts5_main.c 833db0a3df10ab26e0221a9baa40cf871c450df3 F ext/fts5/fts5_storage.c fb2eaec3aa954b680d43096dc539f8270bd6390e F ext/fts5/fts5_tcl.c f8731e0508299bd43f1a2eff7dbeaac870768966 @@ -1419,10 +1419,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 7323175337b7ba85ac932ca892b28860f6a5b688 -R bc65520844365f59e43048a22901b688 -T *branch * fts5-perf -T *sym-fts5-perf * -T -sym-trunk * +P b5a57b812fd6a734cf2a342bf0b730ae18912d73 +R 20bdd3799e5ba562672d44f9df7f8c2a U dan -Z 20f25a74d681d80924173ee060583d9e +Z fc938fd5a5718db0c9c10bacac344b80 diff --git a/manifest.uuid b/manifest.uuid index fc3a1227d6..7a9fdf27eb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b5a57b812fd6a734cf2a342bf0b730ae18912d73 \ No newline at end of file +7558a0ad2276e91f2faced8ea405d9fdb4fa0c6e \ No newline at end of file From c66d03f1e13e8f1746e063d642480cd951d4ca44 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 23 Jan 2016 15:57:06 +0000 Subject: [PATCH 15/39] Fix some signed/unsigned comparison compiler warnings in fts5. FossilOrigin-Name: 3be336aa893f9eb0837d7d66c83bab1489792b9a --- ext/fts5/fts5Int.h | 6 +++--- ext/fts5/fts5_buffer.c | 2 +- manifest | 16 ++++++++-------- manifest.uuid | 2 +- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index b2f0d6c34e..5d7facc948 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -225,8 +225,8 @@ int sqlite3Fts5ConfigParseRank(const char*, char**, char**); typedef struct Fts5Buffer Fts5Buffer; struct Fts5Buffer { u8 *p; - u32 n; - u32 nSpace; + int n; + int nSpace; }; int sqlite3Fts5BufferSize(int*, Fts5Buffer*, u32); @@ -247,7 +247,7 @@ char *sqlite3Fts5Mprintf(int *pRc, const char *zFmt, ...); #define fts5BufferSet(a,b,c,d) sqlite3Fts5BufferSet(a,b,c,d) #define fts5BufferGrow(pRc,pBuf,nn) ( \ - (pBuf)->n + (nn) <= (pBuf)->nSpace ? 0 : \ + (u32)((pBuf)->n) + (u32)(nn) <= (u32)((pBuf)->nSpace) ? 0 : \ sqlite3Fts5BufferSize((pRc),(pBuf),(nn)+(pBuf)->n) \ ) diff --git a/ext/fts5/fts5_buffer.c b/ext/fts5/fts5_buffer.c index 94591188d3..9bd4fddb82 100644 --- a/ext/fts5/fts5_buffer.c +++ b/ext/fts5/fts5_buffer.c @@ -363,7 +363,7 @@ int sqlite3Fts5TermsetAdd( void sqlite3Fts5TermsetFree(Fts5Termset *p){ if( p ){ - int i; + u32 i; for(i=0; iapHash); i++){ Fts5TermsetEntry *pEntry = p->apHash[i]; while( pEntry ){ diff --git a/manifest b/manifest index 7b54ea0ab0..9f8ab447a0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\sassert()\sthat\scan\sbe\sfalse\sif\scompiled\swith\sSQLITE_USE_ALLOCA. -D 2016-01-23T14:05:27.109 +C Fix\ssome\ssigned/unsigned\scomparison\scompiler\swarnings\sin\sfts5. +D 2016-01-23T15:57:06.186 F Makefile.in 027c1603f255390c43a426671055a31c0a65fdb4 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1708a78eda223b6daa302b140037fcc214a779f9 @@ -97,9 +97,9 @@ F ext/fts3/unicode/mkunicode.tcl 95cf7ec186e48d4985e433ff8a1c89090a774252 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0 F ext/fts5/fts5.h ff9c2782e8ed890b0de2f697a8d63971939e70c7 -F ext/fts5/fts5Int.h 5599703af9c13512900a9f22fec39d48078d619d +F ext/fts5/fts5Int.h e4c7ca41a7841f180d78a24850757e0ffcb20f27 F ext/fts5/fts5_aux.c 2dafc3aee0c70d643140c77d8d70daffa51a9e9e -F ext/fts5/fts5_buffer.c 7d3f6f01f8fdc45204e6a33925ef8478a67d28dd +F ext/fts5/fts5_buffer.c 0b8e1f84fec3ec01f7e17f8d4f17c46856b31e10 F ext/fts5/fts5_config.c 0c384ebdd23fd055e2e50a93277b8d59da538238 F ext/fts5/fts5_expr.c 4ab4504c54bbe24689c83411d8588f4ec99136e9 F ext/fts5/fts5_hash.c 1b113977296cf4212c6ec667d5e3f2bd18036955 @@ -1419,7 +1419,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P a79c46bc61a35edbaf6112c26e8052314b0eb16a -R 78c84a447f1adb3672b2aa032cc33ea6 -U drh -Z 2d3baf83036e6553ecd28fa4c13c4bc2 +P f0a551edf87ef061deae34d88b02c3b484ae9adc +R 962289f1674397e1ac71fafb409f527f +U dan +Z 5f630552e065d9e5a5f7f6919a0222d3 diff --git a/manifest.uuid b/manifest.uuid index d11f7b73ef..e3c087f5fd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f0a551edf87ef061deae34d88b02c3b484ae9adc \ No newline at end of file +3be336aa893f9eb0837d7d66c83bab1489792b9a \ No newline at end of file From c6aab321441e0ce3eefa2a2ace0864cbca116231 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 23 Jan 2016 18:24:52 +0000 Subject: [PATCH 16/39] Changes to spellfix to try to get it to use stack space instead of heap space in cases where that makes sense. FossilOrigin-Name: dfcebc7393a3a780ee9c4f257723c303bb633064 --- ext/misc/spellfix.c | 17 +++++++++++++---- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/ext/misc/spellfix.c b/ext/misc/spellfix.c index c0da6b339b..d17a05a3cd 100644 --- a/ext/misc/spellfix.c +++ b/ext/misc/spellfix.c @@ -365,8 +365,8 @@ static int editdist1(const char *zA, const char *zB, int *pnMatch){ int *m; /* The cost matrix */ char *cx; /* Corresponding character values */ int *toFree = 0; /* Malloced space */ - int mStack[60+15]; /* Stack space to use if not too much is needed */ int nMatch = 0; + int mStack[60+15]; /* Stack space to use if not too much is needed */ /* Early out if either input is NULL */ if( zA==0 || zB==0 ) return -1; @@ -899,15 +899,24 @@ static int editDist3Core( EditDist3FromString f = *pFrom; EditDist3To *a2; unsigned int *m; + unsigned int *pToFree; int szRow; EditDist3Cost *p; int res; + sqlite3_uint64 nByte; + unsigned int stackSpace[16*1024]; /* allocate the Wagner matrix and the aTo[] array for the TO string */ n = (f.n+1)*(n2+1); n = (n+1)&~1; - m = sqlite3_malloc( n*sizeof(m[0]) + sizeof(a2[0])*n2 ); - if( m==0 ) return -1; /* Out of memory */ + nByte = n*sizeof(m[0]) + sizeof(a2[0])*n2; + if( nByte<=sizeof(stackSpace) ){ + m = stackSpace; + pToFree = 0; + }else{ + m = pToFree = sqlite3_malloc( nByte ); + if( m==0 ) return -1; /* Out of memory */ + } a2 = (EditDist3To*)&m[n]; memset(a2, 0, sizeof(a2[0])*n2); @@ -1029,7 +1038,7 @@ static int editDist3Core( editDist3Abort: for(i2=0; i2 Date: Sat, 23 Jan 2016 18:51:59 +0000 Subject: [PATCH 17/39] Fix an fts5 problem with using both xPhraseFirst() and xPhraseFirstColumn() within a single statement in detail=col mode. FossilOrigin-Name: 72d53699bf0dcdb9d2a22e229989d7435f061399 --- ext/fts5/fts5Int.h | 6 +-- ext/fts5/fts5_aux.c | 2 +- ext/fts5/fts5_buffer.c | 4 +- ext/fts5/fts5_expr.c | 79 +++++++++++---------------------- ext/fts5/fts5_index.c | 79 --------------------------------- ext/fts5/fts5_main.c | 4 +- ext/fts5/fts5_storage.c | 2 +- ext/fts5/fts5_tokenize.c | 2 +- ext/fts5/fts5_vocab.c | 3 +- ext/fts5/test/fts5_common.tcl | 3 +- ext/fts5/test/fts5synonym2.test | 37 ++++++++++++++- manifest | 32 ++++++------- manifest.uuid | 2 +- 13 files changed, 90 insertions(+), 165 deletions(-) diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index b8486dfdfb..48ab980921 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -29,7 +29,7 @@ typedef unsigned short u16; typedef sqlite3_int64 i64; typedef sqlite3_uint64 u64; -#define ArraySize(x) (sizeof(x) / sizeof(x[0])) +#define ArraySize(x) ((int)(sizeof(x) / sizeof(x[0]))) #define testcase(x) #define ALWAYS(x) 1 @@ -388,8 +388,6 @@ int sqlite3Fts5IterEof(Fts5IndexIter*); int sqlite3Fts5IterNext(Fts5IndexIter*); int sqlite3Fts5IterNextFrom(Fts5IndexIter*, i64 iMatch); i64 sqlite3Fts5IterRowid(Fts5IndexIter*); -int sqlite3Fts5IterPoslist(Fts5IndexIter*,Fts5Colset*, const u8**, int*, i64*); -int sqlite3Fts5IterPoslistBuffer(Fts5IndexIter *pIter, Fts5Buffer *pBuf); /* ** Close an iterator opened by sqlite3Fts5IndexQuery(). @@ -402,8 +400,6 @@ void sqlite3Fts5IterClose(Fts5IndexIter*); const char *sqlite3Fts5IterTerm(Fts5IndexIter*, int*); int sqlite3Fts5IterNextScan(Fts5IndexIter*); -int sqlite3Fts5IterCollist(Fts5IndexIter*, const u8 **, int*); - /* ** Insert or remove data to or from the index. Each time a document is diff --git a/ext/fts5/fts5_aux.c b/ext/fts5/fts5_aux.c index 011064d405..79bed1c8f7 100644 --- a/ext/fts5/fts5_aux.c +++ b/ext/fts5/fts5_aux.c @@ -544,7 +544,7 @@ int sqlite3Fts5AuxInit(fts5_api *pApi){ int rc = SQLITE_OK; /* Return code */ int i; /* To iterate through builtin functions */ - for(i=0; rc==SQLITE_OK && i<(int)ArraySize(aBuiltin); i++){ + for(i=0; rc==SQLITE_OK && ixCreateFunction(pApi, aBuiltin[i].zFunc, aBuiltin[i].pUserData, diff --git a/ext/fts5/fts5_buffer.c b/ext/fts5/fts5_buffer.c index 9bd4fddb82..d69adf7d24 100644 --- a/ext/fts5/fts5_buffer.c +++ b/ext/fts5/fts5_buffer.c @@ -322,7 +322,7 @@ int sqlite3Fts5TermsetAdd( *pbPresent = 0; if( p ){ int i; - int hash = 13; + u32 hash = 13; Fts5TermsetEntry *pEntry; /* Calculate a hash value for this term. This is the same hash checksum @@ -339,7 +339,7 @@ int sqlite3Fts5TermsetAdd( if( pEntry->iIdx==iIdx && pEntry->nTerm==nTerm && memcmp(pEntry->pTerm, pTerm, nTerm)==0 - ){ + ){ *pbPresent = 1; break; } diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index 290baf4e2c..3a1497f05d 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -306,7 +306,7 @@ static int fts5ExprSynonymList( int bCollist, Fts5Colset *pColset, i64 iRowid, - int *pbDel, /* OUT: Caller should sqlite3_free(*pa) */ + Fts5Buffer *pBuf, /* Use this buffer for space if required */ u8 **pa, int *pn ){ Fts5PoslistReader aStatic[4]; @@ -320,22 +320,7 @@ static int fts5ExprSynonymList( for(p=pTerm; p; p=p->pSynonym){ Fts5IndexIter *pIter = p->pIter; if( sqlite3Fts5IterEof(pIter)==0 && sqlite3Fts5IterRowid(pIter)==iRowid ){ - const u8 *a; - int n; - - if( 0 && bCollist ){ - rc = sqlite3Fts5IterCollist(pIter, &a, &n); - }else{ - i64 dummy; -#if 0 - rc = sqlite3Fts5IterPoslist(pIter, pColset, &a, &n, &dummy); -#endif - a = pIter->pData; - n = pIter->nData; - } - - if( rc!=SQLITE_OK ) goto synonym_poslist_out; - if( n==0 ) continue; + if( pIter->nData==0 ) continue; if( nIter==nAlloc ){ int nByte = sizeof(Fts5PoslistReader) * nAlloc * 2; Fts5PoslistReader *aNew = (Fts5PoslistReader*)sqlite3_malloc(nByte); @@ -348,20 +333,19 @@ static int fts5ExprSynonymList( if( aIter!=aStatic ) sqlite3_free(aIter); aIter = aNew; } - sqlite3Fts5PoslistReaderInit(a, n, &aIter[nIter]); + sqlite3Fts5PoslistReaderInit(pIter->pData, pIter->nData, &aIter[nIter]); assert( aIter[nIter].bEof==0 ); nIter++; } } - assert( *pbDel==0 ); if( nIter==1 ){ *pa = (u8*)aIter[0].a; *pn = aIter[0].n; }else{ Fts5PoslistWriter writer = {0}; - Fts5Buffer buf = {0,0,0}; i64 iPrev = -1; + fts5BufferZero(pBuf); while( 1 ){ int i; i64 iMin = FTS5_LARGEST_INT64; @@ -376,15 +360,12 @@ static int fts5ExprSynonymList( } } if( iMin==FTS5_LARGEST_INT64 || rc!=SQLITE_OK ) break; - rc = sqlite3Fts5PoslistWriterAppend(&buf, &writer, iMin); + rc = sqlite3Fts5PoslistWriterAppend(pBuf, &writer, iMin); iPrev = iMin; } - if( rc ){ - sqlite3_free(buf.p); - }else{ - *pa = buf.p; - *pn = buf.n; - *pbDel = 1; + if( rc==SQLITE_OK ){ + *pa = pBuf->p; + *pn = pBuf->n; } } @@ -421,7 +402,7 @@ static int fts5ExprPhraseIsMatch( /* If the aStatic[] array is not large enough, allocate a large array ** using sqlite3_malloc(). This approach could be improved upon. */ - if( pPhrase->nTerm>(int)ArraySize(aStatic) ){ + if( pPhrase->nTerm>ArraySize(aStatic) ){ int nByte = sizeof(Fts5PoslistReader) * pPhrase->nTerm; aIter = (Fts5PoslistReader*)sqlite3_malloc(nByte); if( !aIter ) return SQLITE_NOMEM; @@ -431,23 +412,23 @@ static int fts5ExprPhraseIsMatch( /* Initialize a term iterator for each term in the phrase */ for(i=0; inTerm; i++){ Fts5ExprTerm *pTerm = &pPhrase->aTerm[i]; - i64 dummy; int n = 0; int bFlag = 0; - const u8 *a = 0; + u8 *a = 0; if( pTerm->pSynonym ){ + Fts5Buffer buf = {0, 0, 0}; rc = fts5ExprSynonymList( - pTerm, 0, pColset, pNode->iRowid, &bFlag, (u8**)&a, &n + pTerm, 0, pColset, pNode->iRowid, &buf, &a, &n ); + if( rc ){ + sqlite3_free(a); + goto ismatch_out; + } + if( a==buf.p ) bFlag = 1; }else{ - Fts5IndexIter *pIter = pTerm->pIter; -#if 0 - rc = sqlite3Fts5IterPoslist(pTerm->pIter, pColset, &a, &n, &dummy); -#endif - a = pIter->pData; - n = pIter->nData; + a = (u8*)pTerm->pIter->pData; + n = pTerm->pIter->nData; } - if( rc!=SQLITE_OK ) goto ismatch_out; sqlite3Fts5PoslistReaderInit(a, n, &aIter[i]); aIter[i].bFlag = (u8)bFlag; if( aIter[i].bEof ) goto ismatch_out; @@ -562,7 +543,7 @@ static int fts5ExprNearIsMatch(int *pRc, Fts5ExprNearset *pNear){ /* If the aStatic[] array is not large enough, allocate a large array ** using sqlite3_malloc(). This approach could be improved upon. */ - if( pNear->nPhrase>(int)ArraySize(aStatic) ){ + if( pNear->nPhrase>ArraySize(aStatic) ){ int nByte = sizeof(Fts5NearTrimmer) * pNear->nPhrase; a = (Fts5NearTrimmer*)sqlite3Fts5MallocZero(&rc, nByte); }else{ @@ -1387,10 +1368,10 @@ static void fts5ExprPhraseFree(Fts5ExprPhrase *pPhrase){ Fts5ExprTerm *pTerm = &pPhrase->aTerm[i]; sqlite3_free(pTerm->zTerm); sqlite3Fts5IterClose(pTerm->pIter); - for(pSyn=pTerm->pSynonym; pSyn; pSyn=pNext){ pNext = pSyn->pSynonym; sqlite3Fts5IterClose(pSyn->pIter); + fts5BufferFree((Fts5Buffer*)&pSyn[1]); sqlite3_free(pSyn); } } @@ -1478,13 +1459,13 @@ static int fts5ParseTokenize( assert( pPhrase==0 || pPhrase->nTerm>0 ); if( pPhrase && (tflags & FTS5_TOKEN_COLOCATED) ){ Fts5ExprTerm *pSyn; - int nByte = sizeof(Fts5ExprTerm) + nToken+1; + int nByte = sizeof(Fts5ExprTerm) + sizeof(Fts5Buffer) + nToken+1; pSyn = (Fts5ExprTerm*)sqlite3_malloc(nByte); if( pSyn==0 ){ rc = SQLITE_NOMEM; }else{ memset(pSyn, 0, nByte); - pSyn->zTerm = (char*)&pSyn[1]; + pSyn->zTerm = ((char*)pSyn) + sizeof(Fts5ExprTerm) + sizeof(Fts5Buffer); memcpy(pSyn->zTerm, pToken, nToken); pSyn->pSynonym = pPhrase->aTerm[pPhrase->nTerm-1].pSynonym; pPhrase->aTerm[pPhrase->nTerm-1].pSynonym = pSyn; @@ -2263,7 +2244,7 @@ int sqlite3Fts5ExprInit(Fts5Global *pGlobal, sqlite3 *db){ int rc = SQLITE_OK; void *pCtx = (void*)pGlobal; - for(i=0; rc==SQLITE_OK && i<(int)ArraySize(aFunc); i++){ + for(i=0; rc==SQLITE_OK && iz, -1, SQLITE_UTF8, pCtx, p->x, 0, 0); } @@ -2509,18 +2490,10 @@ int sqlite3Fts5ExprPhraseCollist( ){ Fts5ExprTerm *pTerm = &pPhrase->aTerm[0]; if( pTerm->pSynonym ){ - int bDel = 0; - u8 *a; + Fts5Buffer *pBuf = (Fts5Buffer*)&pTerm->pSynonym[1]; rc = fts5ExprSynonymList( - pTerm, 1, 0, pNode->iRowid, &bDel, &a, pnCollist + pTerm, 1, 0, pNode->iRowid, pBuf, (u8**)ppCollist, pnCollist ); - if( bDel ){ - sqlite3Fts5BufferSet(&rc, &pPhrase->poslist, *pnCollist, a); - *ppCollist = pPhrase->poslist.p; - sqlite3_free(a); - }else{ - *ppCollist = a; - } }else{ *ppCollist = pPhrase->aTerm[0].pIter->pData; *pnCollist = pPhrase->aTerm[0].pIter->nData; diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 8323842dc6..e147199ef8 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -5249,85 +5249,6 @@ const char *sqlite3Fts5IterTerm(Fts5IndexIter *pIndexIter, int *pn){ return &z[1]; } - -/* -** Return a pointer to a buffer containing a copy of the position list for -** the current entry. Output variable *pn is set to the size of the buffer -** in bytes before returning. -** -** The returned position list does not include the "number of bytes" varint -** field that starts the position list on disk. -*/ -int sqlite3Fts5IterPoslist( - Fts5IndexIter *pIndexIter, - Fts5Colset *pColset, /* Column filter (or NULL) */ - const u8 **pp, /* OUT: Pointer to position-list data */ - int *pn, /* OUT: Size of position-list in bytes */ - i64 *piRowid /* OUT: Current rowid */ -){ - Fts5Iter *pIter = (Fts5Iter*)pIndexIter; - Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ]; - int eDetail = pIter->pIndex->pConfig->eDetail; - - assert( pIter->pIndex->rc==SQLITE_OK ); - *piRowid = pSeg->iRowid; - if( eDetail==FTS5_DETAIL_NONE ){ - *pn = pSeg->nPos; - }else - if( eDetail==FTS5_DETAIL_FULL - && pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf - ){ - u8 *pPos = &pSeg->pLeaf->p[pSeg->iLeafOffset]; - if( pColset==0 || pIter->bFiltered ){ - *pn = pSeg->nPos; - *pp = pPos; - }else if( pColset->nCol==1 ){ - *pp = pPos; - *pn = fts5IndexExtractCol(pp, pSeg->nPos, pColset->aiCol[0]); - }else{ - fts5BufferZero(&pIter->poslist); - fts5IndexExtractColset(pColset, pPos, pSeg->nPos, &pIter->poslist); - *pp = pIter->poslist.p; - *pn = pIter->poslist.n; - } - }else{ - fts5BufferZero(&pIter->poslist); - fts5SegiterPoslist(pIter->pIndex, pSeg, pColset, &pIter->poslist); - if( eDetail==FTS5_DETAIL_FULL ){ - *pp = pIter->poslist.p; - } - *pn = pIter->poslist.n; - } - return fts5IndexReturn(pIter->pIndex); -} - -int sqlite3Fts5IterCollist( - Fts5IndexIter *pIndexIter, - const u8 **pp, /* OUT: Pointer to position-list data */ - int *pn /* OUT: Size of position-list in bytes */ -){ - Fts5Iter *pIter = (Fts5Iter*)pIndexIter; - assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_COLUMNS ); - *pp = pIter->poslist.p; - *pn = pIter->poslist.n; - return SQLITE_OK; -} - -/* -** This function is similar to sqlite3Fts5IterPoslist(), except that it -** copies the position list into the buffer supplied as the second -** argument. -*/ -int sqlite3Fts5IterPoslistBuffer(Fts5IndexIter *pIndexIter, Fts5Buffer *pBuf){ - Fts5Iter *pIter = (Fts5Iter*)pIndexIter; - Fts5Index *p = pIter->pIndex; - Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ]; - assert( p->rc==SQLITE_OK ); - fts5BufferZero(pBuf); - fts5SegiterPoslist(p, pSeg, 0, pBuf); - return fts5IndexReturn(p); -} - /* ** Close an iterator opened by an earlier call to sqlite3Fts5IndexQuery(). */ diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index 7447f9b5e1..ec045da6bf 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -538,7 +538,7 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ for(i=0; inConstraint; i++){ struct sqlite3_index_constraint *p = &pInfo->aConstraint[i]; int j; - for(j=0; j<(int)ArraySize(aConstraint); j++){ + for(j=0; jiColumn==aColMap[pC->iCol] && p->op & pC->op ){ if( p->usable ){ @@ -585,7 +585,7 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ /* Assign argvIndex values to each constraint in use. */ iNext = 1; - for(i=0; i<(int)ArraySize(aConstraint); i++){ + for(i=0; iiConsIndex>=0 ){ pInfo->aConstraintUsage[pC->iConsIndex].argvIndex = iNext++; diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c index 56383619d1..da9524dea3 100644 --- a/ext/fts5/fts5_storage.c +++ b/ext/fts5/fts5_storage.c @@ -338,7 +338,7 @@ int sqlite3Fts5StorageClose(Fts5Storage *p){ int i; /* Finalize all SQL statements */ - for(i=0; i<(int)ArraySize(p->aStmt); i++){ + for(i=0; iaStmt); i++){ sqlite3_finalize(p->aStmt[i]); } diff --git a/ext/fts5/fts5_tokenize.c b/ext/fts5/fts5_tokenize.c index e60183c095..790adaae47 100644 --- a/ext/fts5/fts5_tokenize.c +++ b/ext/fts5/fts5_tokenize.c @@ -1220,7 +1220,7 @@ int sqlite3Fts5TokenizerInit(fts5_api *pApi){ int rc = SQLITE_OK; /* Return code */ int i; /* To iterate through builtin functions */ - for(i=0; rc==SQLITE_OK && i<(int)ArraySize(aBuiltin); i++){ + for(i=0; rc==SQLITE_OK && ixCreateTokenizer(pApi, aBuiltin[i].zName, (void*)pApi, diff --git a/ext/fts5/fts5_vocab.c b/ext/fts5/fts5_vocab.c index 17aa5816c4..b17e8974fb 100644 --- a/ext/fts5/fts5_vocab.c +++ b/ext/fts5/fts5_vocab.c @@ -184,7 +184,7 @@ static int fts5VocabInitVtab( rc = fts5VocabTableType(zType, pzErr, &eType); if( rc==SQLITE_OK ){ - assert( eType>=0 && eType=0 && eTypeeType==FTS5_VOCAB_COL || pTab->eType==FTS5_VOCAB_ROW ); while( rc==SQLITE_OK ){ - i64 dummy; const u8 *pPos; int nPos; /* Position list */ i64 iPos = 0; /* 64-bit position read from poslist */ int iOff = 0; /* Current offset within position list */ diff --git a/ext/fts5/test/fts5_common.tcl b/ext/fts5/test/fts5_common.tcl index e7c61af00c..cf688dcc0d 100644 --- a/ext/fts5/test/fts5_common.tcl +++ b/ext/fts5/test/fts5_common.tcl @@ -48,7 +48,8 @@ proc fts5_test_poslist2 {cmd} { } } - set res + #set res + sort_poslist $res } proc fts5_test_collist {cmd} { diff --git a/ext/fts5/test/fts5synonym2.test b/ext/fts5/test/fts5synonym2.test index dddaa17af8..7e92822c33 100644 --- a/ext/fts5/test/fts5synonym2.test +++ b/ext/fts5/test/fts5synonym2.test @@ -27,6 +27,21 @@ foreach_detail_mode $testprefix { fts5_tclnum_register db fts5_aux_test_functions db +proc fts5_test_bothlist {cmd} { + + for {set i 0} {$i < [$cmd xPhraseCount]} {incr i} { + set bFirst 1 + $cmd xPhraseColumnForeach $i c { + lappend CL $i.$c + if {$bFirst} { $cmd xPhraseForeach $i c o { lappend PL $i.$c.$o } } + set bFirst 0 + } + } + + list [sort_poslist $PL] $CL +} +sqlite3_fts5_create_function db fts5_test_bothlist fts5_test_bothlist + proc fts5_rowid {cmd} { expr [$cmd xColumnText -1] } sqlite3_fts5_create_function db fts5_rowid fts5_rowid @@ -89,6 +104,8 @@ do_execsql_test 1.$tok.0.2 { } foreach {tn expr} { + 2.1 "one OR two OR three OR four" + 1.1 "one" 1.2 "two" 1.3 "three" 1.4 "four" 1.5 "v" 1.6 "vi" 1.7 "vii" 1.8 "viii" 1.9 "9" 1.10 "0" 1.11 "1" 1.12 "2" @@ -113,13 +130,31 @@ foreach {tn expr} { set res [fts5_query_data $expr ss ASC ::tclnum_syn] do_execsql_test 1.$tok.$tn.[llength $res].asc.1 { - SELECT rowid, fts5_test_poslist(ss), fts5_test_collist(ss) FROM ss($expr) + SELECT rowid, fts5_test_poslist2(ss), fts5_test_collist(ss) FROM ss($expr) } $res do_execsql_test 1.$tok.$tn.[llength $res].asc.2 { SELECT rowid, fts5_test_poslist(ss), fts5_test_collist(ss) FROM ss($expr) + } $res + + do_execsql_test 1.$tok.$tn.[llength $res].asc.2 { + SELECT rowid, fts5_test_poslist2(ss), fts5_test_collist(ss) FROM ss($expr) ORDER BY rank ASC } $res + + set res2 [list] + foreach {a b c} $res { lappend res2 $a $c $b } + do_execsql_test 1.$tok.$tn.[llength $res].asc.3 { + SELECT rowid, fts5_test_collist(ss), fts5_test_poslist2(ss) FROM ss($expr) + } $res2 + + set res3 [list] + foreach {a b c} $res { lappend res3 $a [list $b $c] } + do_execsql_test 1.$tok.$tn.[llength $res].asc.3 { + SELECT rowid, fts5_test_bothlist(ss) FROM ss($expr) + } $res3 + + } } diff --git a/manifest b/manifest index e3a6b78f3f..fbcb4d2d1b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\strunk\schanges\s(including\sfixes\sfor\swarnings\sin\sfts5)\swith\sthis\sbranch. -D 2016-01-23T16:20:16.389 +C Fix\san\sfts5\sproblem\swith\susing\sboth\sxPhraseFirst()\sand\sxPhraseFirstColumn()\swithin\sa\ssingle\sstatement\sin\sdetail=col\smode. +D 2016-01-23T18:51:59.865 F Makefile.in 027c1603f255390c43a426671055a31c0a65fdb4 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1708a78eda223b6daa302b140037fcc214a779f9 @@ -97,25 +97,25 @@ F ext/fts3/unicode/mkunicode.tcl 95cf7ec186e48d4985e433ff8a1c89090a774252 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0 F ext/fts5/fts5.h ff9c2782e8ed890b0de2f697a8d63971939e70c7 -F ext/fts5/fts5Int.h a810589cf2fedd7dcfc9ba7871d2a5a09e850ace -F ext/fts5/fts5_aux.c 2dafc3aee0c70d643140c77d8d70daffa51a9e9e -F ext/fts5/fts5_buffer.c 0b8e1f84fec3ec01f7e17f8d4f17c46856b31e10 +F ext/fts5/fts5Int.h 6e0f90eb4872654a5b98130dec16965716525c9a +F ext/fts5/fts5_aux.c b9bcce753ef5b451267b2232f0ca153ddeb3951d +F ext/fts5/fts5_buffer.c f6e0c6018ffc8e39fc0b333b5daa8b8d528ae6e4 F ext/fts5/fts5_config.c 0c384ebdd23fd055e2e50a93277b8d59da538238 -F ext/fts5/fts5_expr.c 48b9131b74c8d3b8c12ba0f7995e2d60eecce9f2 +F ext/fts5/fts5_expr.c a66b9694519d9c336d9bdbd46ea22e7e14aef412 F ext/fts5/fts5_hash.c 1b113977296cf4212c6ec667d5e3f2bd18036955 -F ext/fts5/fts5_index.c bd5476edd4d6ef37fc389794a78f47f29f521634 -F ext/fts5/fts5_main.c 833db0a3df10ab26e0221a9baa40cf871c450df3 -F ext/fts5/fts5_storage.c fb2eaec3aa954b680d43096dc539f8270bd6390e +F ext/fts5/fts5_index.c 722d8717e3167dd05fa48af970352932052da318 +F ext/fts5/fts5_main.c 3886bbfc5ac1d9df29979823ddf2b68241e1127e +F ext/fts5/fts5_storage.c 2a1f44deae090cd711f02cec0c2af8e660360d24 F ext/fts5/fts5_tcl.c f8731e0508299bd43f1a2eff7dbeaac870768966 F ext/fts5/fts5_test_mi.c 1ec66ffdf7632077fbd773b7a6df5153272ec070 F ext/fts5/fts5_test_tok.c db08af63673c3a7d39f053b36fd6e065017706be -F ext/fts5/fts5_tokenize.c 504984ac6993323247221eebe3cd55bead01b5f8 +F ext/fts5/fts5_tokenize.c 4d5c4f183c7d07d144bc219b92da1ea0e962fae3 F ext/fts5/fts5_unicode2.c 78273fbd588d1d9bd0a7e4e0ccc9207348bae33c F ext/fts5/fts5_varint.c 3f86ce09cab152e3d45490d7586b7ed2e40c13f1 -F ext/fts5/fts5_vocab.c f1b4308b9b7ec8e659d0c9b39ddc8f1aeee47a1a +F ext/fts5/fts5_vocab.c 3ef401a8d6932db56368de32f446eb9fe73aa623 F ext/fts5/fts5parse.y 1647eba089b9b3fc058b4dc989d9da87d15b9580 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba -F ext/fts5/test/fts5_common.tcl 6d0d74b695c4be055a8ba1dd807f22a2abc95b5e +F ext/fts5/test/fts5_common.tcl 61ff0d1a29d98a91c4553b20b3f410d858834ee9 F ext/fts5/test/fts5aa.test 7e814df4a0e6c22a6fe2d84f210fdc0b5068a084 F ext/fts5/test/fts5ab.test 30325a89453280160106be411bba3acf138e6d1b F ext/fts5/test/fts5ac.test dec95549e007dd9be52aa435cdcd0f08e14e64d0 @@ -177,7 +177,7 @@ F ext/fts5/test/fts5rowid.test 16908a99d6efc9ba21081b4f2b86b3fc699839a6 F ext/fts5/test/fts5simple.test 2bc6451cbe887a9215f5b14ae307c70d850344c9 F ext/fts5/test/fts5simple2.test 98377ae1ff7749a42c21fe1a139c1ed312522c46 F ext/fts5/test/fts5synonym.test 6475d189c2e20d60795808f83e36bf9318708d48 -F ext/fts5/test/fts5synonym2.test eadb00c73ef0653258873e756b7e9102e0687539 +F ext/fts5/test/fts5synonym2.test aa4c43bd3b691ff80f658cb064f5ab40690e834e F ext/fts5/test/fts5tok1.test beb894c6f3468f10a574302f69ebe4436b0287c7 F ext/fts5/test/fts5tok2.test dcacb32d4a2a3f0dd3215d4a3987f78ae4be21a2 F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89 @@ -1419,7 +1419,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 7558a0ad2276e91f2faced8ea405d9fdb4fa0c6e 3be336aa893f9eb0837d7d66c83bab1489792b9a -R 27b07c29574081bf2a6f1649bb5e2393 +P ceccc9ad788fb4da9120915741995b9f088f85ff +R e7d7753020b6f70b109a92a8ac037ba7 U dan -Z 165b70c16fc218fdaf7b93acd8863bd2 +Z 648645363fb131dd10e5d7321a173b9d diff --git a/manifest.uuid b/manifest.uuid index 3e212691ce..1891619c18 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ceccc9ad788fb4da9120915741995b9f088f85ff \ No newline at end of file +72d53699bf0dcdb9d2a22e229989d7435f061399 \ No newline at end of file From 408273e722fbff04e6f02960060529cd85ba772e Mon Sep 17 00:00:00 2001 From: mistachkin Date: Sat, 23 Jan 2016 19:24:19 +0000 Subject: [PATCH 18/39] Use the -integer option when sorting list elements. FossilOrigin-Name: 949aa614e1401f6ab61cbb2e0469c60d764e6880 --- manifest | 15 ++++++--------- manifest.uuid | 2 +- tool/mkmsvcmin.tcl | 2 +- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 12db615b93..d24047ded1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stooling\sto\screate\san\sMSVC\sMakefile\scapable\sof\sbuilding\sthe\score\slibrary\sand\sshell\sonly,\susing\sa\spre-built\samalgamation. -D 2016-01-23T07:53:04.796 +C Use\sthe\s-integer\soption\swhen\ssorting\slist\selements. +D 2016-01-23T19:24:19.182 F Makefile.in 027c1603f255390c43a426671055a31c0a65fdb4 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.min.msc c805e4a474fc7610d4f0cd0b6a9f56334a14c6ad @@ -1379,7 +1379,7 @@ F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862 F tool/logest.c eef612f8adf4d0993dafed0416064cf50d5d33c6 F tool/mkautoconfamal.sh 5a5441280b509d2bb3bdc71bfb63781b0d570373 F tool/mkkeywordhash.c 06ec0b78bd4fa68c12d90ef2bdfe76b039133ff8 -F tool/mkmsvcmin.tcl 859737ff14668827738a3f7ca17ba643a88bcecc +F tool/mkmsvcmin.tcl a0217f49042ec8160ff8665db638e9b5edc17964 F tool/mkopcodec.tcl edde8adc42621b5e598127f8cdc6d52cfe21f52b F tool/mkopcodeh.tcl e04177031532b7aa9379ded50e820231ac4abd6e F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e @@ -1421,10 +1421,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P a79c46bc61a35edbaf6112c26e8052314b0eb16a -R bd00cfb25453f633191b5f1755909575 -T *branch * msvcMakeMin -T *sym-msvcMakeMin * -T -sym-trunk * +P c46f1a13cb00264d1164399059250884c7be4ea0 +R ce47a7131ea57ef586dba1d328c21253 U mistachkin -Z 18f99743dd26f3a3f31830457ad1b4ef +Z c9f6cfce5ff80db91258ef2b40e34be6 diff --git a/manifest.uuid b/manifest.uuid index 06bbe8845c..0c60eb81fc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c46f1a13cb00264d1164399059250884c7be4ea0 \ No newline at end of file +949aa614e1401f6ab61cbb2e0469c60d764e6880 \ No newline at end of file diff --git a/tool/mkmsvcmin.tcl b/tool/mkmsvcmin.tcl index 56c9d52811..188b73a18e 100644 --- a/tool/mkmsvcmin.tcl +++ b/tool/mkmsvcmin.tcl @@ -73,7 +73,7 @@ set data [readFile $fromFileName] regsub -all -- {# <>\n.*?# <>\n} \ $data "" data -foreach i [lsort [array names blocks]] { +foreach i [lsort -integer [array names blocks]] { regsub -all -- [substVars \ {# <>\n.*?# <>\n}] \ $data [escapeSubSpec $blocks($i)] data From 06cee48728424701260e2f3215ad75c257bace44 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 23 Jan 2016 19:47:00 +0000 Subject: [PATCH 19/39] Improved comments on the transformation script and on the generated Makefile. FossilOrigin-Name: c6e633ab73812fe38f3e969f1652f1b5759096e4 --- Makefile.min.msc | 6 ++++++ manifest | 16 +++++++-------- manifest.uuid | 2 +- tool/mkmsvcmin.tcl | 51 +++++++++++++++++++++++++++------------------- 4 files changed, 45 insertions(+), 30 deletions(-) diff --git a/Makefile.min.msc b/Makefile.min.msc index 2ec986fa4b..e2187efad3 100644 --- a/Makefile.min.msc +++ b/Makefile.min.msc @@ -1,3 +1,9 @@ +#### DO NOT EDIT #### +# This makefile is automatically generated from the Makefile.msc at +# the root of the canonical SQLite source tree using the +# tool/mkmsvcmin.tcl script. +# + # # nmake Makefile for SQLite # diff --git a/manifest b/manifest index d24047ded1..504f2ebbaf 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,8 @@ -C Use\sthe\s-integer\soption\swhen\ssorting\slist\selements. -D 2016-01-23T19:24:19.182 +C Improved\scomments\son\sthe\stransformation\sscript\sand\son\sthe\sgenerated\sMakefile. +D 2016-01-23T19:47:00.459 F Makefile.in 027c1603f255390c43a426671055a31c0a65fdb4 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 -F Makefile.min.msc c805e4a474fc7610d4f0cd0b6a9f56334a14c6ad +F Makefile.min.msc cbbc5e37bfe74583cce7c2585319620b7516e6ed F Makefile.msc 6fca5455aaecbd14479f33f091aa19df2d3d2969 F README.md 8ecc12493ff9f820cdea6520a9016001cb2e59b7 F VERSION 866588d1edf0ccb5b0d33896974338f97564f719 @@ -1379,7 +1379,7 @@ F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862 F tool/logest.c eef612f8adf4d0993dafed0416064cf50d5d33c6 F tool/mkautoconfamal.sh 5a5441280b509d2bb3bdc71bfb63781b0d570373 F tool/mkkeywordhash.c 06ec0b78bd4fa68c12d90ef2bdfe76b039133ff8 -F tool/mkmsvcmin.tcl a0217f49042ec8160ff8665db638e9b5edc17964 +F tool/mkmsvcmin.tcl a62d398d1304b36dd41b69e655d875728fd9d509 F tool/mkopcodec.tcl edde8adc42621b5e598127f8cdc6d52cfe21f52b F tool/mkopcodeh.tcl e04177031532b7aa9379ded50e820231ac4abd6e F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e @@ -1421,7 +1421,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P c46f1a13cb00264d1164399059250884c7be4ea0 -R ce47a7131ea57ef586dba1d328c21253 -U mistachkin -Z c9f6cfce5ff80db91258ef2b40e34be6 +P 949aa614e1401f6ab61cbb2e0469c60d764e6880 +R 52e20c37d886e0ca572199fede1eb984 +U drh +Z 5f79d302a18d7c201b23ed73e75ff6ef diff --git a/manifest.uuid b/manifest.uuid index 0c60eb81fc..876d304c70 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -949aa614e1401f6ab61cbb2e0469c60d764e6880 \ No newline at end of file +c6e633ab73812fe38f3e969f1652f1b5759096e4 \ No newline at end of file diff --git a/tool/mkmsvcmin.tcl b/tool/mkmsvcmin.tcl index 188b73a18e..7414f53000 100644 --- a/tool/mkmsvcmin.tcl +++ b/tool/mkmsvcmin.tcl @@ -1,22 +1,26 @@ #!/usr/bin/tcl # -# Removes and/or replaces specially marked sections from the Makefile -# for MSVC, writing the resulting output to another (different) file. -# In addition, some other strings are also removed and/or replaced if -# they are present in the Makefile. -# -set fromFileName [lindex $argv 0] - -if {![file exists $fromFileName]} then { - error "input file \"$fromFileName\" does not exist" +# This script reads the regular MSVC makefile (../Makefile.msc) and outputs +# a revised version of that Makefile that is "minimal" in the sense that +# it uses the sqlite3.c amalgamation as input and does not require tclsh. +# The resulting "../Makefile.min.msc" is suitable for use in the amalgamation +# tarballs. +# +if {$argc==0} { + set basedir [file dir [file dir [file normalize $argv0]]] + set fromFileName [file join $basedir Makefile.msc] + set toFileName [file join $basedir Makefile.min.msc] +} else { + set fromFileName [lindex $argv 0] + if {![file exists $fromFileName]} then { + error "input file \"$fromFileName\" does not exist" + } + set toFileName [lindex $argv 1] + if {[file exists $toFileName]} then { + error "output file \"$toFileName\" already exists" + } } - -set toFileName [lindex $argv 1] -if {[file exists $toFileName]} then { - error "output file \"$toFileName\" already exists" -} - proc readFile { fileName } { set file_id [open $fileName RDONLY] fconfigure $file_id -encoding binary -translation binary @@ -24,7 +28,7 @@ proc readFile { fileName } { close $file_id return $result } - + proc writeFile { fileName data } { set file_id [open $fileName {WRONLY CREAT TRUNC}] fconfigure $file_id -encoding binary -translation binary @@ -32,17 +36,17 @@ proc writeFile { fileName data } { close $file_id return "" } - + proc escapeSubSpec { data } { regsub -all -- {&} $data {\\\&} data regsub -all -- {\\(\d+)} $data {\\\\\1} data return $data } - + proc substVars { data } { return [uplevel 1 [list subst -nocommands -nobackslashes $data]] } - + # # NOTE: This block is used to replace the section marked <> in # the Makefile, if it exists. @@ -67,8 +71,13 @@ $(LIBRESOBJS): $(TOP)\sqlite3.rc rcver.vc $(SQLITE3H) echo #endif >> sqlite3rc.h $(LTRCOMPILE) -fo $(LIBRESOBJS) -DRC_VERONLY $(TOP)\sqlite3.rc }]] - -set data [readFile $fromFileName] + +set data "#### DO NOT EDIT ####\n" +append data "# This makefile is automatically " +append data "generated from the [file tail $fromFileName] at\n" +append data "# the root of the canonical SQLite source tree using the\n" +append data "# tool/[file tail $argv0] script.\n#\n\n" +append data [readFile $fromFileName] regsub -all -- {# <>\n.*?# <>\n} \ $data "" data From ac779bcbe964611322cc9ad4e056e8220784b84a Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 23 Jan 2016 20:09:30 +0000 Subject: [PATCH 20/39] Adjust the amalgamation-tarball build process so that it includes the MSVC makefile and the sqlite3.rc resource file. FossilOrigin-Name: e420f3dc93a0e3364f0773950b7a125f5221a109 --- autoconf/Makefile.am | 2 +- autoconf/{README => README.txt} | 5 ++++- manifest | 16 ++++++++-------- manifest.uuid | 2 +- tool/mkautoconfamal.sh | 17 +++++++++-------- 5 files changed, 23 insertions(+), 19 deletions(-) rename autoconf/{README => README.txt} (88%) diff --git a/autoconf/Makefile.am b/autoconf/Makefile.am index eed3ba41cb..03424a606b 100644 --- a/autoconf/Makefile.am +++ b/autoconf/Makefile.am @@ -14,7 +14,7 @@ sqlite3_CFLAGS = $(AM_CFLAGS) include_HEADERS = sqlite3.h sqlite3ext.h -EXTRA_DIST = sqlite3.1 tea +EXTRA_DIST = sqlite3.1 tea Makefile.msc sqlite3.rc README.txt pkgconfigdir = ${libdir}/pkgconfig pkgconfig_DATA = sqlite3.pc diff --git a/autoconf/README b/autoconf/README.txt similarity index 88% rename from autoconf/README rename to autoconf/README.txt index dd8cca24bd..731158a883 100644 --- a/autoconf/README +++ b/autoconf/README.txt @@ -1,4 +1,3 @@ - This package contains: * the SQLite library amalgamation (single file) source code distribution, @@ -6,6 +5,7 @@ This package contains: * the sqlite3.h and sqlite3ext.h header files required to link programs and sqlite extensions against the installed libary. * autoconf/automake installation infrastucture. + * a Makefile.msc and sqlite3.rc for Microsoft Visual C++ The generic installation instructions for autoconf/automake are found in the INSTALL file. @@ -30,3 +30,6 @@ example: $ CFLAGS="-Os -DSQLITE_OMIT_TRIGGERS" ./configure +To compile for Windows using Microsoft Visual C++: + + $ nmake /f Makefile.msc FOR_WIN10=1 diff --git a/manifest b/manifest index 504f2ebbaf..a6797b77e5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\scomments\son\sthe\stransformation\sscript\sand\son\sthe\sgenerated\sMakefile. -D 2016-01-23T19:47:00.459 +C Adjust\sthe\samalgamation-tarball\sbuild\sprocess\sso\sthat\sit\sincludes\sthe\nMSVC\smakefile\sand\sthe\ssqlite3.rc\sresource\sfile. +D 2016-01-23T20:09:30.205 F Makefile.in 027c1603f255390c43a426671055a31c0a65fdb4 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.min.msc cbbc5e37bfe74583cce7c2585319620b7516e6ed @@ -11,9 +11,9 @@ F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2 F art/sqlite370.ico af56c1d00fee7cd4753e8631ed60703ed0fc6e90 F art/sqlite370.jpg d512473dae7e378a67e28ff96a34da7cb331def2 F autoconf/INSTALL 83e4a25da9fd053c7b3665eaaaf7919707915903 -F autoconf/Makefile.am 089e5ecdb5761e64ea1013ded02feb4d8b29927d -F autoconf/README 14458f1046c118efa721aadec5f227e876d3cd38 +F autoconf/Makefile.am 1c1657650775960804945dc392e14d9e43c5ed84 F autoconf/README.first 6c4f34fe115ff55d4e8dbfa3cecf04a0188292f7 +F autoconf/README.txt c9265657b6ca8c4de1e148e8c0c4c29d60ec860c w autoconf/README F autoconf/configure.ac 7b1ea0dcaf49fafba262ce4b0ee8cb3281b555d1 F autoconf/tea/Makefile.in b438a7020446c8a8156e8d97c8914a04833da6fd F autoconf/tea/README 3e9a3c060f29a44344ab50aec506f4db903fb873 @@ -1377,7 +1377,7 @@ F tool/lemon.c 799e73e19a33b8dd7767a7fa34618ed2a9c2397d F tool/lempar.c 3ec1463a034b37d87d782be5f6b8b10a3b1ecbe7 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862 F tool/logest.c eef612f8adf4d0993dafed0416064cf50d5d33c6 -F tool/mkautoconfamal.sh 5a5441280b509d2bb3bdc71bfb63781b0d570373 +F tool/mkautoconfamal.sh add1d703074e7edb2d8423441c7e245271c5c8cb F tool/mkkeywordhash.c 06ec0b78bd4fa68c12d90ef2bdfe76b039133ff8 F tool/mkmsvcmin.tcl a62d398d1304b36dd41b69e655d875728fd9d509 F tool/mkopcodec.tcl edde8adc42621b5e598127f8cdc6d52cfe21f52b @@ -1421,7 +1421,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 949aa614e1401f6ab61cbb2e0469c60d764e6880 -R 52e20c37d886e0ca572199fede1eb984 +P c6e633ab73812fe38f3e969f1652f1b5759096e4 +R d5543de9eede26dd90664aa004eef406 U drh -Z 5f79d302a18d7c201b23ed73e75ff6ef +Z 98c1051b769380596fbf89ccef90cc39 diff --git a/manifest.uuid b/manifest.uuid index 876d304c70..c32f479829 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c6e633ab73812fe38f3e969f1652f1b5759096e4 \ No newline at end of file +e420f3dc93a0e3364f0773950b7a125f5221a109 \ No newline at end of file diff --git a/tool/mkautoconfamal.sh b/tool/mkautoconfamal.sh index 9f44152e36..d368cd226b 100644 --- a/tool/mkautoconfamal.sh +++ b/tool/mkautoconfamal.sh @@ -34,14 +34,15 @@ set -e ARTIFACT=`printf "3%.2d%.2d%.2d" $xx $yy $zz` rm -rf $TMPSPACE -cp -R $TOP/autoconf $TMPSPACE - -cp sqlite3.c $TMPSPACE -cp sqlite3.h $TMPSPACE -cp sqlite3ext.h $TMPSPACE -cp $TOP/sqlite3.1 $TMPSPACE -cp $TOP/sqlite3.pc.in $TMPSPACE -cp $TOP/src/shell.c $TMPSPACE +cp -R $TOP/autoconf $TMPSPACE +cp sqlite3.c $TMPSPACE +cp sqlite3.h $TMPSPACE +cp sqlite3ext.h $TMPSPACE +cp $TOP/sqlite3.1 $TMPSPACE +cp $TOP/sqlite3.pc.in $TMPSPACE +cp $TOP/src/shell.c $TMPSPACE +cp $TOP/Makefile.min.msc $TMPSPACE/Makefile.msc +cp $TOP/src/sqlite3.rc $TMPSPACE cat $TMPSPACE/configure.ac | sed "s/--SQLITE-VERSION--/$VERSION/" > $TMPSPACE/tmp From 4a25549b087bd576ac413f88f5e2d654cd59e0b4 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Sat, 23 Jan 2016 20:16:40 +0000 Subject: [PATCH 21/39] Minor coding style tweaks for the mkmsvcmin tool. FossilOrigin-Name: c15e0fc4be08b92547bea1f1cac65581ef497409 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- tool/mkmsvcmin.tcl | 6 +++--- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index a6797b77e5..2076ff35e1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Adjust\sthe\samalgamation-tarball\sbuild\sprocess\sso\sthat\sit\sincludes\sthe\nMSVC\smakefile\sand\sthe\ssqlite3.rc\sresource\sfile. -D 2016-01-23T20:09:30.205 +C Minor\scoding\sstyle\stweaks\sfor\sthe\smkmsvcmin\stool. +D 2016-01-23T20:16:40.624 F Makefile.in 027c1603f255390c43a426671055a31c0a65fdb4 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.min.msc cbbc5e37bfe74583cce7c2585319620b7516e6ed @@ -13,7 +13,7 @@ F art/sqlite370.jpg d512473dae7e378a67e28ff96a34da7cb331def2 F autoconf/INSTALL 83e4a25da9fd053c7b3665eaaaf7919707915903 F autoconf/Makefile.am 1c1657650775960804945dc392e14d9e43c5ed84 F autoconf/README.first 6c4f34fe115ff55d4e8dbfa3cecf04a0188292f7 -F autoconf/README.txt c9265657b6ca8c4de1e148e8c0c4c29d60ec860c w autoconf/README +F autoconf/README.txt c9265657b6ca8c4de1e148e8c0c4c29d60ec860c F autoconf/configure.ac 7b1ea0dcaf49fafba262ce4b0ee8cb3281b555d1 F autoconf/tea/Makefile.in b438a7020446c8a8156e8d97c8914a04833da6fd F autoconf/tea/README 3e9a3c060f29a44344ab50aec506f4db903fb873 @@ -1379,7 +1379,7 @@ F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862 F tool/logest.c eef612f8adf4d0993dafed0416064cf50d5d33c6 F tool/mkautoconfamal.sh add1d703074e7edb2d8423441c7e245271c5c8cb F tool/mkkeywordhash.c 06ec0b78bd4fa68c12d90ef2bdfe76b039133ff8 -F tool/mkmsvcmin.tcl a62d398d1304b36dd41b69e655d875728fd9d509 +F tool/mkmsvcmin.tcl c1f92b52505fe4a026e7b57fffe49b439b11b71b F tool/mkopcodec.tcl edde8adc42621b5e598127f8cdc6d52cfe21f52b F tool/mkopcodeh.tcl e04177031532b7aa9379ded50e820231ac4abd6e F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e @@ -1421,7 +1421,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P c6e633ab73812fe38f3e969f1652f1b5759096e4 -R d5543de9eede26dd90664aa004eef406 -U drh -Z 98c1051b769380596fbf89ccef90cc39 +P e420f3dc93a0e3364f0773950b7a125f5221a109 +R 0e64448b1af0ccd7100284f6f222657e +U mistachkin +Z 38e3fbc9582e760d49d364916a96f186 diff --git a/manifest.uuid b/manifest.uuid index c32f479829..6bcba645b0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e420f3dc93a0e3364f0773950b7a125f5221a109 \ No newline at end of file +c15e0fc4be08b92547bea1f1cac65581ef497409 \ No newline at end of file diff --git a/tool/mkmsvcmin.tcl b/tool/mkmsvcmin.tcl index 7414f53000..d0143ddd86 100644 --- a/tool/mkmsvcmin.tcl +++ b/tool/mkmsvcmin.tcl @@ -5,18 +5,18 @@ # it uses the sqlite3.c amalgamation as input and does not require tclsh. # The resulting "../Makefile.min.msc" is suitable for use in the amalgamation # tarballs. -# +# if {$argc==0} { set basedir [file dir [file dir [file normalize $argv0]]] set fromFileName [file join $basedir Makefile.msc] set toFileName [file join $basedir Makefile.min.msc] } else { set fromFileName [lindex $argv 0] - if {![file exists $fromFileName]} then { + if {![file exists $fromFileName]} { error "input file \"$fromFileName\" does not exist" } set toFileName [lindex $argv 1] - if {[file exists $toFileName]} then { + if {[file exists $toFileName]} { error "output file \"$toFileName\" already exists" } } From 021f9a6605c2c1af96db9fd144c64749c5224751 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 23 Jan 2016 20:34:27 +0000 Subject: [PATCH 22/39] Move Makefile.min.msc into the autoconf subdir where it belongs and change its name to Makefile.msc. Adjust build scripts accordingly. FossilOrigin-Name: 12eb8db79697ef55228c78011d275f36f58271e1 --- Makefile.min.msc => autoconf/Makefile.msc | 5 +++-- manifest | 18 +++++++++--------- manifest.uuid | 2 +- tool/mkautoconfamal.sh | 1 - tool/mkmsvcmin.tcl | 7 ++++--- 5 files changed, 17 insertions(+), 16 deletions(-) rename Makefile.min.msc => autoconf/Makefile.msc (99%) diff --git a/Makefile.min.msc b/autoconf/Makefile.msc similarity index 99% rename from Makefile.min.msc rename to autoconf/Makefile.msc index e2187efad3..fce62b2b8a 100644 --- a/Makefile.min.msc +++ b/autoconf/Makefile.msc @@ -1,7 +1,8 @@ #### DO NOT EDIT #### # This makefile is automatically generated from the Makefile.msc at -# the root of the canonical SQLite source tree using the -# tool/mkmsvcmin.tcl script. +# the root of the canonical SQLite source tree (not the +# amalgamation tarball) using the tool/mkmsvcmin.tcl +# script. # # diff --git a/manifest b/manifest index 2076ff35e1..9a914fa3d3 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,7 @@ -C Minor\scoding\sstyle\stweaks\sfor\sthe\smkmsvcmin\stool. -D 2016-01-23T20:16:40.624 +C Move\sMakefile.min.msc\sinto\sthe\sautoconf\ssubdir\swhere\sit\sbelongs\sand\schange\nits\sname\sto\sMakefile.msc.\s\sAdjust\sbuild\sscripts\saccordingly. +D 2016-01-23T20:34:27.088 F Makefile.in 027c1603f255390c43a426671055a31c0a65fdb4 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 -F Makefile.min.msc cbbc5e37bfe74583cce7c2585319620b7516e6ed F Makefile.msc 6fca5455aaecbd14479f33f091aa19df2d3d2969 F README.md 8ecc12493ff9f820cdea6520a9016001cb2e59b7 F VERSION 866588d1edf0ccb5b0d33896974338f97564f719 @@ -12,6 +11,7 @@ F art/sqlite370.ico af56c1d00fee7cd4753e8631ed60703ed0fc6e90 F art/sqlite370.jpg d512473dae7e378a67e28ff96a34da7cb331def2 F autoconf/INSTALL 83e4a25da9fd053c7b3665eaaaf7919707915903 F autoconf/Makefile.am 1c1657650775960804945dc392e14d9e43c5ed84 +F autoconf/Makefile.msc 68ed752a809b611d97b95d8572a34fe6fd1196f1 w Makefile.min.msc F autoconf/README.first 6c4f34fe115ff55d4e8dbfa3cecf04a0188292f7 F autoconf/README.txt c9265657b6ca8c4de1e148e8c0c4c29d60ec860c F autoconf/configure.ac 7b1ea0dcaf49fafba262ce4b0ee8cb3281b555d1 @@ -1377,9 +1377,9 @@ F tool/lemon.c 799e73e19a33b8dd7767a7fa34618ed2a9c2397d F tool/lempar.c 3ec1463a034b37d87d782be5f6b8b10a3b1ecbe7 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862 F tool/logest.c eef612f8adf4d0993dafed0416064cf50d5d33c6 -F tool/mkautoconfamal.sh add1d703074e7edb2d8423441c7e245271c5c8cb +F tool/mkautoconfamal.sh a29b14d54302b33fd892958f6895582ea90e4a45 F tool/mkkeywordhash.c 06ec0b78bd4fa68c12d90ef2bdfe76b039133ff8 -F tool/mkmsvcmin.tcl c1f92b52505fe4a026e7b57fffe49b439b11b71b +F tool/mkmsvcmin.tcl 93167a9e73383465b5716aa8dfa407409fccef1d F tool/mkopcodec.tcl edde8adc42621b5e598127f8cdc6d52cfe21f52b F tool/mkopcodeh.tcl e04177031532b7aa9379ded50e820231ac4abd6e F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e @@ -1421,7 +1421,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P e420f3dc93a0e3364f0773950b7a125f5221a109 -R 0e64448b1af0ccd7100284f6f222657e -U mistachkin -Z 38e3fbc9582e760d49d364916a96f186 +P c15e0fc4be08b92547bea1f1cac65581ef497409 +R ffa1a98b2b7806021300568489ac1126 +U drh +Z ba6195ff75ad05f3ae39571f4a1714e5 diff --git a/manifest.uuid b/manifest.uuid index 6bcba645b0..e3e08e4e7c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c15e0fc4be08b92547bea1f1cac65581ef497409 \ No newline at end of file +12eb8db79697ef55228c78011d275f36f58271e1 \ No newline at end of file diff --git a/tool/mkautoconfamal.sh b/tool/mkautoconfamal.sh index d368cd226b..6081a7b0ec 100644 --- a/tool/mkautoconfamal.sh +++ b/tool/mkautoconfamal.sh @@ -41,7 +41,6 @@ cp sqlite3ext.h $TMPSPACE cp $TOP/sqlite3.1 $TMPSPACE cp $TOP/sqlite3.pc.in $TMPSPACE cp $TOP/src/shell.c $TMPSPACE -cp $TOP/Makefile.min.msc $TMPSPACE/Makefile.msc cp $TOP/src/sqlite3.rc $TMPSPACE cat $TMPSPACE/configure.ac | diff --git a/tool/mkmsvcmin.tcl b/tool/mkmsvcmin.tcl index d0143ddd86..de01490c97 100644 --- a/tool/mkmsvcmin.tcl +++ b/tool/mkmsvcmin.tcl @@ -9,7 +9,7 @@ if {$argc==0} { set basedir [file dir [file dir [file normalize $argv0]]] set fromFileName [file join $basedir Makefile.msc] - set toFileName [file join $basedir Makefile.min.msc] + set toFileName [file join $basedir autoconf Makefile.msc] } else { set fromFileName [lindex $argv 0] if {![file exists $fromFileName]} { @@ -75,8 +75,9 @@ $(LIBRESOBJS): $(TOP)\sqlite3.rc rcver.vc $(SQLITE3H) set data "#### DO NOT EDIT ####\n" append data "# This makefile is automatically " append data "generated from the [file tail $fromFileName] at\n" -append data "# the root of the canonical SQLite source tree using the\n" -append data "# tool/[file tail $argv0] script.\n#\n\n" +append data "# the root of the canonical SQLite source tree (not the\n" +append data "# amalgamation tarball) using the tool/[file tail $argv0]\n" +append data "# script.\n#\n\n" append data [readFile $fromFileName] regsub -all -- {# <>\n.*?# <>\n} \ From 74f6e6ccf877fd3154eff2fabaa0c55e188bb019 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 23 Jan 2016 20:43:48 +0000 Subject: [PATCH 23/39] Updates to the autoconf README file. FossilOrigin-Name: 4ce60fa42d8486d219d40d25d785d7f2cabe5e06 --- autoconf/README.txt | 11 +++++++++-- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/autoconf/README.txt b/autoconf/README.txt index 731158a883..575e0a8dd8 100644 --- a/autoconf/README.txt +++ b/autoconf/README.txt @@ -4,8 +4,11 @@ This package contains: * the shell.c file used to build the sqlite3 shell too, and * the sqlite3.h and sqlite3ext.h header files required to link programs and sqlite extensions against the installed libary. - * autoconf/automake installation infrastucture. - * a Makefile.msc and sqlite3.rc for Microsoft Visual C++ + * autoconf/automake installation infrastucture for building on posix systems. + * a Makefile.msc and sqlite3.rc for building with Microsoft Visual C++ + +BUILDING ON POSIX +================= The generic installation instructions for autoconf/automake are found in the INSTALL file. @@ -30,6 +33,10 @@ example: $ CFLAGS="-Os -DSQLITE_OMIT_TRIGGERS" ./configure + +BUILDING WITH MICROSOFT VISUAL C++: +=================================== + To compile for Windows using Microsoft Visual C++: $ nmake /f Makefile.msc FOR_WIN10=1 diff --git a/manifest b/manifest index 9a914fa3d3..e41fae053e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Move\sMakefile.min.msc\sinto\sthe\sautoconf\ssubdir\swhere\sit\sbelongs\sand\schange\nits\sname\sto\sMakefile.msc.\s\sAdjust\sbuild\sscripts\saccordingly. -D 2016-01-23T20:34:27.088 +C Updates\sto\sthe\sautoconf\sREADME\sfile. +D 2016-01-23T20:43:48.995 F Makefile.in 027c1603f255390c43a426671055a31c0a65fdb4 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6fca5455aaecbd14479f33f091aa19df2d3d2969 @@ -11,9 +11,9 @@ F art/sqlite370.ico af56c1d00fee7cd4753e8631ed60703ed0fc6e90 F art/sqlite370.jpg d512473dae7e378a67e28ff96a34da7cb331def2 F autoconf/INSTALL 83e4a25da9fd053c7b3665eaaaf7919707915903 F autoconf/Makefile.am 1c1657650775960804945dc392e14d9e43c5ed84 -F autoconf/Makefile.msc 68ed752a809b611d97b95d8572a34fe6fd1196f1 w Makefile.min.msc +F autoconf/Makefile.msc 68ed752a809b611d97b95d8572a34fe6fd1196f1 F autoconf/README.first 6c4f34fe115ff55d4e8dbfa3cecf04a0188292f7 -F autoconf/README.txt c9265657b6ca8c4de1e148e8c0c4c29d60ec860c +F autoconf/README.txt ebf930270b2d6258f4d2c62c8c5a5dcc7cff5293 F autoconf/configure.ac 7b1ea0dcaf49fafba262ce4b0ee8cb3281b555d1 F autoconf/tea/Makefile.in b438a7020446c8a8156e8d97c8914a04833da6fd F autoconf/tea/README 3e9a3c060f29a44344ab50aec506f4db903fb873 @@ -1421,7 +1421,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P c15e0fc4be08b92547bea1f1cac65581ef497409 -R ffa1a98b2b7806021300568489ac1126 +P 12eb8db79697ef55228c78011d275f36f58271e1 +R f077ea8ac49bb767871086d72cc4aac2 U drh -Z ba6195ff75ad05f3ae39571f4a1714e5 +Z 47e598e0236f733ea03c00ba7b81ee9b diff --git a/manifest.uuid b/manifest.uuid index e3e08e4e7c..ea84af834d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -12eb8db79697ef55228c78011d275f36f58271e1 \ No newline at end of file +4ce60fa42d8486d219d40d25d785d7f2cabe5e06 \ No newline at end of file From 76d4b1b835be68f6164042e34efb4c31e3dc11c3 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Sat, 23 Jan 2016 21:57:46 +0000 Subject: [PATCH 24/39] More updates to the README. FossilOrigin-Name: 9f02868df71d902a2b9118d0b06df18ee92b666b --- autoconf/README.txt | 77 +++++++++++++++++++++++++++++++++++++++++---- manifest | 14 ++++----- manifest.uuid | 2 +- 3 files changed, 79 insertions(+), 14 deletions(-) diff --git a/autoconf/README.txt b/autoconf/README.txt index 575e0a8dd8..a581c39dea 100644 --- a/autoconf/README.txt +++ b/autoconf/README.txt @@ -4,8 +4,10 @@ This package contains: * the shell.c file used to build the sqlite3 shell too, and * the sqlite3.h and sqlite3ext.h header files required to link programs and sqlite extensions against the installed libary. - * autoconf/automake installation infrastucture for building on posix systems. - * a Makefile.msc and sqlite3.rc for building with Microsoft Visual C++ + * autoconf/automake installation infrastucture for building on POSIX + compliant systems. + * a Makefile.msc and sqlite3.rc for building with Microsoft Visual C++ on + Windows. BUILDING ON POSIX ================= @@ -19,7 +21,7 @@ The following SQLite specific boolean options are supported: --enable-threadsafe build a thread-safe library [default=yes] --enable-dynamic-extensions support loadable extensions [default=yes] -The default value for the CFLAGS variable (options passed to the C +The default value for the CFLAGS variable (options passed to the C compiler) includes debugging symbols in the build, resulting in larger binaries than are necessary. Override it on the configure command line like this: @@ -34,9 +36,72 @@ example: $ CFLAGS="-Os -DSQLITE_OMIT_TRIGGERS" ./configure -BUILDING WITH MICROSOFT VISUAL C++: -=================================== +BUILDING WITH MICROSOFT VISUAL C++ +================================== To compile for Windows using Microsoft Visual C++: - $ nmake /f Makefile.msc FOR_WIN10=1 + $ nmake /f Makefile.msc + +Using Microsoft Visual C++ 2005 (or later) is recommended. Several Windows +platform variants may be built by adding additional macros to the NMAKE +command line. + +BUILDING FOR WINRT 8.0 +---------------------- + + FOR_WINRT=1 + +Using Microsoft Visual C++ 2012 (or later) is required. When using the +above, something like the following macro will need to be added to the +NMAKE command line as well: + + "NSDKLIBPATH=%WindowsSdkDir%\..\8.0\lib\win8\um\x86" + +BUILDING FOR WINRT 8.1 +---------------------- + + FOR_WINRT=1 + +Using Microsoft Visual C++ 2013 (or later) is required. When using the +above, something like the following macro will need to be added to the +NMAKE command line as well: + + "NSDKLIBPATH=%WindowsSdkDir%\..\8.1\lib\winv6.3\um\x86" + +BUILDING FOR UAP 10.0 +--------------------- + + FOR_WINRT=1 FOR_UAP=1 + +Using Microsoft Visual C++ 2015 (or later) is required. When using the +above, something like the following macros will need to be added to the +NMAKE command line as well: + + "NSDKLIBPATH=%WindowsSdkDir%\..\10\lib\10.0.10586.0\um\x86" + "PSDKLIBPATH=%WindowsSdkDir%\..\10\lib\10.0.10586.0\um\x86" + "NUCRTLIBPATH=%UniversalCRTSdkDir%\..\10\lib\10.0.10586.0\ucrt\x86" + +BUILDING FOR THE WINDOWS 10 SDK +------------------------------- + + FOR_WIN10=1 + +Using Microsoft Visual C++ 2015 (or later) is required. When using the +above, no other macros should be needed on the NMAKE command line. + +USING PREPROCESSOR DEFINES +-------------------------- + +Additionally, preprocessor defines may be specified by using the OPTS macro +on the NMAKE command line. However, not all possible preprocessor defines +may be specified in this manner as some require the amalgamation to be built +with them enabled (see http://www.sqlite.org/compile.html). For example, the +following will work: + + "OPTS=-DSQLITE_ENABLE_STAT4=1 -DSQLITE_ENABLE_JSON1=1" + +However, the following will not compile unless the amalgamation was built +with it enabled: + + "OPTS=-DSQLITE_ENABLE_UPDATE_DELETE_LIMIT=1" diff --git a/manifest b/manifest index e41fae053e..0f3e8b7b8d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Updates\sto\sthe\sautoconf\sREADME\sfile. -D 2016-01-23T20:43:48.995 +C More\supdates\sto\sthe\sREADME. +D 2016-01-23T21:57:46.170 F Makefile.in 027c1603f255390c43a426671055a31c0a65fdb4 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6fca5455aaecbd14479f33f091aa19df2d3d2969 @@ -13,7 +13,7 @@ F autoconf/INSTALL 83e4a25da9fd053c7b3665eaaaf7919707915903 F autoconf/Makefile.am 1c1657650775960804945dc392e14d9e43c5ed84 F autoconf/Makefile.msc 68ed752a809b611d97b95d8572a34fe6fd1196f1 F autoconf/README.first 6c4f34fe115ff55d4e8dbfa3cecf04a0188292f7 -F autoconf/README.txt ebf930270b2d6258f4d2c62c8c5a5dcc7cff5293 +F autoconf/README.txt fc86867528c500dfd691620e623bfe6799c98d14 F autoconf/configure.ac 7b1ea0dcaf49fafba262ce4b0ee8cb3281b555d1 F autoconf/tea/Makefile.in b438a7020446c8a8156e8d97c8914a04833da6fd F autoconf/tea/README 3e9a3c060f29a44344ab50aec506f4db903fb873 @@ -1421,7 +1421,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 12eb8db79697ef55228c78011d275f36f58271e1 -R f077ea8ac49bb767871086d72cc4aac2 -U drh -Z 47e598e0236f733ea03c00ba7b81ee9b +P 4ce60fa42d8486d219d40d25d785d7f2cabe5e06 +R 9ddb76e24748b77caceaf34a8b255b23 +U mistachkin +Z 8a8dd4db0d2ee1de0599573a5cb1355c diff --git a/manifest.uuid b/manifest.uuid index ea84af834d..c53057578b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4ce60fa42d8486d219d40d25d785d7f2cabe5e06 \ No newline at end of file +9f02868df71d902a2b9118d0b06df18ee92b666b \ No newline at end of file From 9084ec1d962388a219d32fbed684f15e44de11d8 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 23 Jan 2016 22:55:53 +0000 Subject: [PATCH 25/39] Add the SQLITE_SPELLFIX_STACKALLOC_SZ compile-time option to control how much stack space is available for use as the Wagner matrix in editDest3 of the spellfix extension. FossilOrigin-Name: dd0100dd87e2c7c70fabb2d3188f0906e605583d --- ext/misc/spellfix.c | 13 ++++++++++++- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/ext/misc/spellfix.c b/ext/misc/spellfix.c index d17a05a3cd..68bd2afbb2 100644 --- a/ext/misc/spellfix.c +++ b/ext/misc/spellfix.c @@ -875,6 +875,17 @@ static void updateCost( } } +/* +** How much stack space (int bytes) to use for Wagner matrix in +** editDist3Core(). If more space than this is required, the entire +** matrix is taken from the heap. To reduce the load on the memory +** allocator, make this value as large as practical for the +** architecture in use. +*/ +#ifndef SQLITE_SPELLFIX_STACKALLOC_SZ +# define SQLITE_SPELLFIX_STACKALLOC_SZ (1024) +#endif + /* Compute the edit distance between two strings. ** ** If an error occurs, return a negative number which is the error code. @@ -904,7 +915,7 @@ static int editDist3Core( EditDist3Cost *p; int res; sqlite3_uint64 nByte; - unsigned int stackSpace[16*1024]; + unsigned int stackSpace[SQLITE_SPELLFIX_STACKALLOC_SZ/sizeof(unsigned int)]; /* allocate the Wagner matrix and the aTo[] array for the TO string */ n = (f.n+1)*(n2+1); diff --git a/manifest b/manifest index 8a5302c683..d6adb98190 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Changes\sto\sspellfix\sto\stry\sto\sget\sit\sto\suse\sstack\sspace\sinstead\sof\sheap\sspace\nin\scases\swhere\sthat\smakes\ssense. -D 2016-01-23T18:24:52.238 +C Add\sthe\sSQLITE_SPELLFIX_STACKALLOC_SZ\scompile-time\soption\sto\scontrol\show\smuch\nstack\sspace\sis\savailable\sfor\suse\sas\sthe\sWagner\smatrix\sin\seditDest3\sof\sthe\nspellfix\sextension. +D 2016-01-23T22:55:53.451 F Makefile.in 027c1603f255390c43a426671055a31c0a65fdb4 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1708a78eda223b6daa302b140037fcc214a779f9 @@ -210,7 +210,7 @@ F ext/misc/regexp.c af92cdaa5058fcec1451e49becc7ba44dba023dc F ext/misc/rot13.c 1ac6f95f99b575907b9b09c81a349114cf9be45a F ext/misc/series.c b8fb7befd85b3a9b4a10e701b30b2b79ca92b6d4 F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52 -F ext/misc/spellfix.c 5850a0e05ae1eacd48a3d80991c63b58d41e2d27 +F ext/misc/spellfix.c db4cc4b7aa12384e6c19a289a39cd232d355413d F ext/misc/totype.c 4a167594e791abeed95e0a8db028822b5e8fe512 F ext/misc/vfslog.c fe40fab5c077a40477f7e5eba994309ecac6cc95 F ext/misc/vtshim.c babb0dc2bf116029e3e7c9a618b8a1377045303e @@ -1419,7 +1419,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 3be336aa893f9eb0837d7d66c83bab1489792b9a -R e0cc5654cf9d372268674d3acd0d1178 +P dfcebc7393a3a780ee9c4f257723c303bb633064 +R f2b384711e45db4261b5ef34d3e09d67 U drh -Z 5f4045218ee81aac454594c9cbaf4e94 +Z 57efa8447e9c944d8afed12c4b4dd5da diff --git a/manifest.uuid b/manifest.uuid index ff35bf94f1..dfbb1735b4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -dfcebc7393a3a780ee9c4f257723c303bb633064 \ No newline at end of file +dd0100dd87e2c7c70fabb2d3188f0906e605583d \ No newline at end of file From d0d0f8dc67725287d31be219314e839594a4278f Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 25 Jan 2016 01:07:59 +0000 Subject: [PATCH 26/39] Small simplification and performance improvement in memsys5Free(). FossilOrigin-Name: 0a9cff5c4822874b74e90bfca3963bc7e5c753a5 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/mem5.c | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index d6adb98190..1cbb37b406 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sSQLITE_SPELLFIX_STACKALLOC_SZ\scompile-time\soption\sto\scontrol\show\smuch\nstack\sspace\sis\savailable\sfor\suse\sas\sthe\sWagner\smatrix\sin\seditDest3\sof\sthe\nspellfix\sextension. -D 2016-01-23T22:55:53.451 +C Small\ssimplification\sand\sperformance\simprovement\sin\smemsys5Free(). +D 2016-01-25T01:07:59.493 F Makefile.in 027c1603f255390c43a426671055a31c0a65fdb4 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1708a78eda223b6daa302b140037fcc214a779f9 @@ -316,7 +316,7 @@ F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c 6919bcf12f221868ea066eec27e579fed95ce98b F src/mem2.c f1940d9e91948dd6a908fbb9ce3835c36b5d83c3 F src/mem3.c 8768ac94694f31ffaf8b4d0ea5dc08af7010a35a -F src/mem5.c 71f81a11fc5e29a57428761ab38a7bf2ef4ee19d +F src/mem5.c 9bf955937b07f8c32541c8a9991f33ce3173d944 F src/memjournal.c 3eb2c0b51adbd869cb6a44780323f05fa904dc85 F src/msvc.h d9ba56c6851227ab44b3f228a35f3f5772296495 F src/mutex.c 8e45800ee78e0cd1f1f3fe8e398853307f4a085c @@ -1419,7 +1419,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P dfcebc7393a3a780ee9c4f257723c303bb633064 -R f2b384711e45db4261b5ef34d3e09d67 +P dd0100dd87e2c7c70fabb2d3188f0906e605583d +R a179bcf25d5e17fae98e7434f7b5c754 U drh -Z 57efa8447e9c944d8afed12c4b4dd5da +Z 6a3ae2a8bc97ab052cdf4e5d8140159f diff --git a/manifest.uuid b/manifest.uuid index dfbb1735b4..fed0f2a557 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -dd0100dd87e2c7c70fabb2d3188f0906e605583d \ No newline at end of file +0a9cff5c4822874b74e90bfca3963bc7e5c753a5 \ No newline at end of file diff --git a/src/mem5.c b/src/mem5.c index 7316a630a6..c194a6b778 100644 --- a/src/mem5.c +++ b/src/mem5.c @@ -322,11 +322,11 @@ static void memsys5FreeUnsafe(void *pOld){ int iBuddy; if( (iBlock>>iLogsize) & 1 ){ iBuddy = iBlock - size; + assert( iBuddy>=0 ); }else{ iBuddy = iBlock + size; + if( iBuddy>=mem5.nBlock ) break; } - assert( iBuddy>=0 ); - if( (iBuddy+(1<mem5.nBlock ) break; if( mem5.aCtrl[iBuddy]!=(CTRL_FREE | iLogsize) ) break; memsys5Unlink(iBuddy, iLogsize); iLogsize++; From a7dc4a321d8f537d9257b20634f93dccfabd03cd Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 25 Jan 2016 02:15:02 +0000 Subject: [PATCH 27/39] Small simplification to the prepare statement opcode memory reuse logic. Easier to read, and slightly smaller and faster. FossilOrigin-Name: 8a1deae497edf3fa43fa96152d140405398c5ed6 --- manifest | 12 ++--- manifest.uuid | 2 +- src/vdbeaux.c | 119 ++++++++++++++++++++++++-------------------------- 3 files changed, 64 insertions(+), 69 deletions(-) diff --git a/manifest b/manifest index 1cbb37b406..e1fe265cc4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Small\ssimplification\sand\sperformance\simprovement\sin\smemsys5Free(). -D 2016-01-25T01:07:59.493 +C Small\ssimplification\sto\sthe\sprepare\sstatement\sopcode\smemory\sreuse\slogic.\nEasier\sto\sread,\sand\sslightly\ssmaller\sand\sfaster. +D 2016-01-25T02:15:02.255 F Makefile.in 027c1603f255390c43a426671055a31c0a65fdb4 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1708a78eda223b6daa302b140037fcc214a779f9 @@ -414,7 +414,7 @@ F src/vdbe.c b90d9d38e5e0260c2eafa3cb4c2274d8ea94da27 F src/vdbe.h 7a733ea8aac1b77305a67698e784fa3484ee3337 F src/vdbeInt.h 42eefa4f9e7432b9968d321b44e48821ec13b189 F src/vdbeapi.c ffae8f5af4570fbd548504e815e9fb7227f0822e -F src/vdbeaux.c 07f8f485a6cbc0a62da660f14e303061d45d5cb6 +F src/vdbeaux.c 757f86e6fef8efb3dd4226cb31e2e82b9c44c883 F src/vdbeblob.c 37c3d11a753e403698c69e17383d282e1ae73e75 F src/vdbemem.c b9181e77eca2a095929d46250daf85c8d2621fc0 F src/vdbesort.c 0971557e5d3c289e46f56a52aed2197c13251de7 @@ -1419,7 +1419,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P dd0100dd87e2c7c70fabb2d3188f0906e605583d -R a179bcf25d5e17fae98e7434f7b5c754 +P 0a9cff5c4822874b74e90bfca3963bc7e5c753a5 +R 5ad29c29cc49b0c7f9cd0947deba5a80 U drh -Z 6a3ae2a8bc97ab052cdf4e5d8140159f +Z 5c673e5ee459b55b04084dc37cebc110 diff --git a/manifest.uuid b/manifest.uuid index fed0f2a557..f04ce16d0b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0a9cff5c4822874b74e90bfca3963bc7e5c753a5 \ No newline at end of file +8a1deae497edf3fa43fa96152d140405398c5ed6 \ No newline at end of file diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 17f1cb7dc4..e9be081a20 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -1721,41 +1721,43 @@ void sqlite3VdbeIOTraceSql(Vdbe *p){ } #endif /* !SQLITE_OMIT_TRACE && SQLITE_ENABLE_IOTRACE */ -/* -** Allocate space from a fixed size buffer and return a pointer to -** that space. If insufficient space is available, return NULL. +/* An instance of this object describes bulk memory available for use +** by subcomponents of a prepared statement. Space is allocated out +** of a ReusableSpace object by the allocSpace() routine below. +*/ +struct ReusableSpace { + u8 *pSpace; /* Available memory */ + int nFree; /* Bytes of available memory */ + int nNeeded; /* Total bytes that could not be allocated */ +}; + +/* Try to allocate nByte bytes of 8-byte aligned bulk memory for pBuf +** from the ReusableSpace object. Return a pointer to the allocated +** memory on success. If insufficient memory is available in the +** ReusableSpace object, increase the ReusableSpace.nNeeded +** value by the amount needed and return NULL. ** -** The pBuf parameter is the initial value of a pointer which will -** receive the new memory. pBuf is normally NULL. If pBuf is not -** NULL, it means that memory space has already been allocated and that -** this routine should not allocate any new memory. When pBuf is not -** NULL simply return pBuf. Only allocate new memory space when pBuf -** is NULL. +** If pBuf is not initially NULL, that means that the memory has already +** been allocated by a prior call to this routine, so just return a copy +** of pBuf and leave ReusableSpace unchanged. ** -** nByte is the number of bytes of space needed. -** -** pFrom points to *pnFrom bytes of available space. New space is allocated -** from the end of the pFrom buffer and *pnFrom is decremented. -** -** *pnNeeded is a counter of the number of bytes of space that have failed -** to allocate. If there is insufficient space in pFrom to satisfy the -** request, then increment *pnNeeded by the amount of the request. +** This allocator is employed to repurpose unused slots at the end of the +** opcode array of prepared state for other memory needs of the prepared +** statement. */ static void *allocSpace( - void *pBuf, /* Where return pointer will be stored */ - int nByte, /* Number of bytes to allocate */ - u8 *pFrom, /* Memory available for allocation */ - int *pnFrom, /* IN/OUT: Space available at pFrom */ - int *pnNeeded /* If allocation cannot be made, increment *pnByte */ + struct ReusableSpace *p, /* Bulk memory available for allocation */ + void *pBuf, /* Pointer to a prior allocation */ + int nByte /* Bytes of memory needed */ ){ - assert( EIGHT_BYTE_ALIGNMENT(pFrom) ); + assert( EIGHT_BYTE_ALIGNMENT(p->pSpace) ); if( pBuf==0 ){ nByte = ROUND8(nByte); - if( nByte <= *pnFrom ){ - *pnFrom -= nByte; - pBuf = &pFrom[*pnFrom]; + if( nByte <= p->nFree ){ + p->nFree -= nByte; + pBuf = &p->pSpace[p->nFree]; }else{ - *pnNeeded += nByte; + p->nNeeded += nByte; } } assert( EIGHT_BYTE_ALIGNMENT(pBuf) ); @@ -1831,9 +1833,7 @@ void sqlite3VdbeMakeReady( int nArg; /* Number of arguments in subprograms */ int nOnce; /* Number of OP_Once instructions */ int n; /* Loop counter */ - int nFree; /* Available free space */ - u8 *zCsr; /* Memory available for allocation */ - int nByte; /* How much extra memory is needed */ + struct ReusableSpace x; /* Reusable bulk memory */ assert( p!=0 ); assert( p->nOp>0 ); @@ -1851,7 +1851,7 @@ void sqlite3VdbeMakeReady( /* For each cursor required, also allocate a memory cell. Memory ** cells (nMem+1-nCursor)..nMem, inclusive, will never be used by - ** the vdbe program. Instead they are used to allocate space for + ** the vdbe program. Instead they are used to allocate memory for ** VdbeCursor/BtCursor structures. The blob of memory associated with ** cursor 0 is stored in memory cell nMem. Memory cell (nMem-1) ** stores the blob of memory associated with cursor 1, etc. @@ -1860,20 +1860,18 @@ void sqlite3VdbeMakeReady( */ nMem += nCursor; - /* zCsr will initially point to nFree bytes of unused space at the - ** end of the opcode array, p->aOp. The computation of nFree is - ** conservative - it might be smaller than the true number of free - ** bytes, but never larger. nFree must be a multiple of 8 - it is - ** rounded down if is not. + /* Figure out how much reusable memory is available at the end of the + ** opcode array. This extra memory will be reallocated for other elements + ** of the prepared statement. */ - n = ROUND8(sizeof(Op)*p->nOp); /* Bytes of opcode space used */ - zCsr = &((u8*)p->aOp)[n]; /* Unused opcode space */ - assert( EIGHT_BYTE_ALIGNMENT(zCsr) ); - nFree = ROUNDDOWN8(pParse->szOpAlloc - n); /* Bytes of unused space */ - assert( nFree>=0 ); - if( nFree>0 ){ - memset(zCsr, 0, nFree); - assert( EIGHT_BYTE_ALIGNMENT(&zCsr[nFree]) ); + n = ROUND8(sizeof(Op)*p->nOp); /* Bytes of opcode memory used */ + x.pSpace = &((u8*)p->aOp)[n]; /* Unused opcode memory */ + assert( EIGHT_BYTE_ALIGNMENT(x.pSpace) ); + x.nFree = ROUNDDOWN8(pParse->szOpAlloc - n); /* Bytes of unused memory */ + assert( x.nFree>=0 ); + if( x.nFree>0 ){ + memset(x.pSpace, 0, x.nFree); + assert( EIGHT_BYTE_ALIGNMENT(&x.pSpace[x.nFree]) ); } resolveP2Values(p, &nArg); @@ -1883,33 +1881,30 @@ void sqlite3VdbeMakeReady( } p->expired = 0; - /* Memory for registers, parameters, cursor, etc, is allocated in two - ** passes. On the first pass, we try to reuse unused space at the + /* Memory for registers, parameters, cursor, etc, is allocated in one or two + ** passes. On the first pass, we try to reuse unused memory at the ** end of the opcode array. If we are unable to satisfy all memory ** requirements by reusing the opcode array tail, then the second - ** pass will fill in the rest using a fresh allocation. + ** pass will fill in the remainder using a fresh memory allocation. ** ** This two-pass approach that reuses as much memory as possible from - ** the leftover space at the end of the opcode array can significantly + ** the leftover memory at the end of the opcode array. This can significantly ** reduce the amount of memory held by a prepared statement. */ do { - nByte = 0; - p->aMem = allocSpace(p->aMem, nMem*sizeof(Mem), zCsr, &nFree, &nByte); - p->aVar = allocSpace(p->aVar, nVar*sizeof(Mem), zCsr, &nFree, &nByte); - p->apArg = allocSpace(p->apArg, nArg*sizeof(Mem*), zCsr, &nFree, &nByte); - p->apCsr = allocSpace(p->apCsr, nCursor*sizeof(VdbeCursor*), - zCsr, &nFree, &nByte); - p->aOnceFlag = allocSpace(p->aOnceFlag, nOnce, zCsr, &nFree, &nByte); + x.nNeeded = 0; + p->aMem = allocSpace(&x, p->aMem, nMem*sizeof(Mem)); + p->aVar = allocSpace(&x, p->aVar, nVar*sizeof(Mem)); + p->apArg = allocSpace(&x, p->apArg, nArg*sizeof(Mem*)); + p->apCsr = allocSpace(&x, p->apCsr, nCursor*sizeof(VdbeCursor*)); + p->aOnceFlag = allocSpace(&x, p->aOnceFlag, nOnce); #ifdef SQLITE_ENABLE_STMT_SCANSTATUS - p->anExec = allocSpace(p->anExec, p->nOp*sizeof(i64), zCsr, &nFree, &nByte); + p->anExec = allocSpace(&x, p->anExec, p->nOp*sizeof(i64)); #endif - if( nByte ){ - p->pFree = sqlite3DbMallocZero(db, nByte); - } - zCsr = p->pFree; - nFree = nByte; - }while( nByte && !db->mallocFailed ); + if( x.nNeeded==0 ) break; + x.pSpace = p->pFree = sqlite3DbMallocZero(db, x.nNeeded); + x.nFree = x.nNeeded; + }while( !db->mallocFailed ); p->nCursor = nCursor; p->nOnceFlag = nOnce; From 164c957b091ed8fb0b875481556e9e7d882d11a2 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 25 Jan 2016 13:55:47 +0000 Subject: [PATCH 28/39] Add the SQLITE_EXTRA_DURABLE compile-time option. FossilOrigin-Name: 30671345b1c1ee55a2d1aa17273213f1849efd81 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pager.c | 17 ++++++++++++++++- 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index e1fe265cc4..69e58459fa 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Small\ssimplification\sto\sthe\sprepare\sstatement\sopcode\smemory\sreuse\slogic.\nEasier\sto\sread,\sand\sslightly\ssmaller\sand\sfaster. -D 2016-01-25T02:15:02.255 +C Add\sthe\sSQLITE_EXTRA_DURABLE\scompile-time\soption. +D 2016-01-25T13:55:47.049 F Makefile.in 027c1603f255390c43a426671055a31c0a65fdb4 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1708a78eda223b6daa302b140037fcc214a779f9 @@ -332,7 +332,7 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c 0eb7f469fcd4e1fbedf30060438e26b839ec5486 F src/os_win.c 386fba30419e8458b13209781c2af5590eab2811 F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca -F src/pager.c f4e9ac39fbb1e0fde97af85c0f4e00eb90764b67 +F src/pager.c 2916c66aee50f69d9ec56a7619b62d9c6a3bee61 F src/pager.h 1c2a49143dfba9e69cc8159ef019f472ed8d260b F src/parse.y caad1e98edeca6960493d0c60d31b76820dd7776 F src/pcache.c 73895411fa6b7bd6f0091212feabbe833b358d23 @@ -1419,7 +1419,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 0a9cff5c4822874b74e90bfca3963bc7e5c753a5 -R 5ad29c29cc49b0c7f9cd0947deba5a80 +P 8a1deae497edf3fa43fa96152d140405398c5ed6 +R ae4ccddaf3c593cdf1ed2beb484cfd15 U drh -Z 5c673e5ee459b55b04084dc37cebc110 +Z 297086babbdf85e61ae58e42a437c1ec diff --git a/manifest.uuid b/manifest.uuid index f04ce16d0b..439be2e5fa 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8a1deae497edf3fa43fa96152d140405398c5ed6 \ No newline at end of file +30671345b1c1ee55a2d1aa17273213f1849efd81 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 2c904d2df1..0afbe215c6 100644 --- a/src/pager.c +++ b/src/pager.c @@ -428,6 +428,20 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ */ #define MAX_SECTOR_SIZE 0x10000 +/* +** If the option SQLITE_EXTRA_DURABLE option is set at compile-time, then +** SQLite will do extra fsync() operations when synchronous==FULL to help +** ensure that transactions are durable across a power failure. Most +** applications are happy as long as transactions are consistent across +** a power failure, and are perfectly willing to lose the last transaction +** in exchange for the extra performance of avoiding directory syncs. +** And so the default SQLITE_EXTRA_DURABLE setting is off. +*/ +#ifndef SQLITE_EXTRA_DURABLE +# define SQLITE_EXTRA_DURABLE 0 +#endif + + /* ** An instance of the following structure is allocated for each active ** savepoint and statement transaction in the system. All such structures @@ -1983,7 +1997,8 @@ static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){ ); sqlite3OsClose(pPager->jfd); if( bDelete ){ - rc = sqlite3OsDelete(pPager->pVfs, pPager->zJournal, 0); + rc = sqlite3OsDelete(pPager->pVfs, pPager->zJournal, + pPager->fullSync && SQLITE_EXTRA_DURABLE); } } } From cc2fa4cf5599637bdb8f5a2cc64f0805243e04e4 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 25 Jan 2016 15:57:29 +0000 Subject: [PATCH 29/39] Replace the OP_SetIfNotPos operator with OP_OffsetLimit in the VDBE, for simpler and smaller code. FossilOrigin-Name: 7ac017a498b6fb28343eef2d24e400c7800660d6 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/select.c | 9 +++------ src/vdbe.c | 33 ++++++++++++++++++++++----------- 4 files changed, 33 insertions(+), 25 deletions(-) diff --git a/manifest b/manifest index 69e58459fa..827a3d5126 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sSQLITE_EXTRA_DURABLE\scompile-time\soption. -D 2016-01-25T13:55:47.049 +C Replace\sthe\sOP_SetIfNotPos\soperator\swith\sOP_OffsetLimit\sin\sthe\sVDBE,\sfor\nsimpler\sand\ssmaller\scode. +D 2016-01-25T15:57:29.060 F Makefile.in 027c1603f255390c43a426671055a31c0a65fdb4 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1708a78eda223b6daa302b140037fcc214a779f9 @@ -345,7 +345,7 @@ F src/printf.c af589a27b7d40f6f4f704e9eea99f02f18ad6d32 F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 F src/resolve.c 9f7ce3a3c087afb7597b7c916c99126ff3f12f0c F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e -F src/select.c 1db9a902e89201a0ae3ff5243a7a3f37842a3937 +F src/select.c c34292c8ce7fe69c7cf890d933834a22572bd301 F src/shell.c dcd7a83645ef2a58ee9c6d0ea4714d877d7835c4 F src/sqlite.h.in 214476a62012e578f42133a9a3b4f97a9aa421a3 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -410,7 +410,7 @@ F src/update.c 17332f9fe818cbc0444c36a811800af8498af4c3 F src/utf.c 32d7f82aa921322f3e1c956f4b58f019ebd2c6b3 F src/util.c 72d40df0a52d3f30b462a15f0e094fcbade6dc82 F src/vacuum.c 2ddd5cad2a7b9cef7f9e431b8c7771634c6b1701 -F src/vdbe.c b90d9d38e5e0260c2eafa3cb4c2274d8ea94da27 +F src/vdbe.c 3ffbcc413bf793e3f0b95b79ef2f4bd449a5b5a3 F src/vdbe.h 7a733ea8aac1b77305a67698e784fa3484ee3337 F src/vdbeInt.h 42eefa4f9e7432b9968d321b44e48821ec13b189 F src/vdbeapi.c ffae8f5af4570fbd548504e815e9fb7227f0822e @@ -1419,7 +1419,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 8a1deae497edf3fa43fa96152d140405398c5ed6 -R ae4ccddaf3c593cdf1ed2beb484cfd15 +P 30671345b1c1ee55a2d1aa17273213f1849efd81 +R f342889ba3409bd63b1cf54d88bfe0e2 U drh -Z 297086babbdf85e61ae58e42a437c1ec +Z e7c5fc86067d4269675714897cec0b51 diff --git a/manifest.uuid b/manifest.uuid index 439be2e5fa..276ef7a03f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -30671345b1c1ee55a2d1aa17273213f1849efd81 \ No newline at end of file +7ac017a498b6fb28343eef2d24e400c7800660d6 \ No newline at end of file diff --git a/src/select.c b/src/select.c index ffc0ec2415..ea4298e677 100644 --- a/src/select.c +++ b/src/select.c @@ -1860,10 +1860,8 @@ static void computeLimitRegisters(Parse *pParse, Select *p, int iBreak){ sqlite3ExprCode(pParse, p->pOffset, iOffset); sqlite3VdbeAddOp1(v, OP_MustBeInt, iOffset); VdbeCoverage(v); VdbeComment((v, "OFFSET counter")); - sqlite3VdbeAddOp3(v, OP_SetIfNotPos, iOffset, iOffset, 0); - sqlite3VdbeAddOp3(v, OP_Add, iLimit, iOffset, iOffset+1); + sqlite3VdbeAddOp3(v, OP_OffsetLimit, iLimit, iOffset+1, iOffset); VdbeComment((v, "LIMIT+OFFSET")); - sqlite3VdbeAddOp3(v, OP_SetIfNotPos, iLimit, iOffset+1, -1); } } } @@ -2280,9 +2278,8 @@ static int multiSelect( addr = sqlite3VdbeAddOp1(v, OP_IfNot, p->iLimit); VdbeCoverage(v); VdbeComment((v, "Jump ahead if LIMIT reached")); if( p->iOffset ){ - sqlite3VdbeAddOp3(v, OP_SetIfNotPos, p->iOffset, p->iOffset, 0); - sqlite3VdbeAddOp3(v, OP_Add, p->iLimit, p->iOffset, p->iOffset+1); - sqlite3VdbeAddOp3(v, OP_SetIfNotPos, p->iLimit, p->iOffset+1, -1); + sqlite3VdbeAddOp3(v, OP_OffsetLimit, + p->iLimit, p->iOffset+1, p->iOffset); } } explainSetInteger(iSub2, pParse->iNextSelectId); diff --git a/src/vdbe.c b/src/vdbe.c index c6d5f7b0cc..cfff895c30 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -5757,20 +5757,31 @@ case OP_IfPos: { /* jump, in1 */ break; } -/* Opcode: SetIfNotPos P1 P2 P3 * * -** Synopsis: if r[P1]<=0 then r[P2]=P3 +/* Opcode: OffsetLimit P1 P2 P3 * * +** Synopsis: if r[P1]>0 then r[P2]=r[P1]+max(0,r[P3]) else r[P2]=(-1) ** -** Register P1 must contain an integer. -** If the value of register P1 is not positive (if it is less than 1) then -** set the value of register P2 to be the integer P3. +** This opcode performs a commonly used computation associated with +** LIMIT and OFFSET process. r[P1] holds the limit counter. r[P3] +** holds the offset counter. The opcode computes the combined value +** of the LIMIT and OFFSET and stores that value in r[P2]. The r[P2] +** value computed is the total number of rows that will need to be +** visited in order to complete the query. +** +** If r[P3] is zero or negative, that means there is no OFFSET +** and r[P2] is set to be the value of the LIMIT, r[P1]. +** +** if r[P1] is zero or negative, that means there is no LIMIT +** and r[P2] is set to -1. +** +** Otherwise, r[P2] is set to the sum of r[P1] and r[P3]. */ -case OP_SetIfNotPos: { /* in1, in2 */ +case OP_OffsetLimit: { /* in1, out2, in3 */ pIn1 = &aMem[pOp->p1]; - assert( pIn1->flags&MEM_Int ); - if( pIn1->u.i<=0 ){ - pOut = out2Prerelease(p, pOp); - pOut->u.i = pOp->p3; - } + pIn3 = &aMem[pOp->p3]; + pOut = out2Prerelease(p, pOp); + assert( pIn1->flags & MEM_Int ); + assert( pIn3->flags & MEM_Int ); + pOut->u.i = pIn1->u.i<=0 ? -1 : pIn1->u.i+(pIn3->u.i>0?pIn3->u.i:0); break; } From e88ec187cdd590f7b9155cef3ee276c268a4a522 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 25 Jan 2016 17:04:48 +0000 Subject: [PATCH 30/39] Fix issues on unix with opening database files via symlinks that are not in the current working directory. And with nested symlinks. FossilOrigin-Name: 80398fd44fb232193450103808e1854e0eba5652 --- manifest | 19 +++--- manifest.uuid | 2 +- src/os_unix.c | 154 +++++++++++++++++++++++++++++++++------------- test/symlink.test | 119 ++++++++++++++++++++++++++--------- 4 files changed, 215 insertions(+), 79 deletions(-) diff --git a/manifest b/manifest index 827a3d5126..0b395a1661 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Replace\sthe\sOP_SetIfNotPos\soperator\swith\sOP_OffsetLimit\sin\sthe\sVDBE,\sfor\nsimpler\sand\ssmaller\scode. -D 2016-01-25T15:57:29.060 +C Fix\sissues\son\sunix\swith\sopening\sdatabase\sfiles\svia\ssymlinks\sthat\sare\snot\sin\sthe\scurrent\sworking\sdirectory.\sAnd\swith\snested\ssymlinks. +D 2016-01-25T17:04:48.546 F Makefile.in 027c1603f255390c43a426671055a31c0a65fdb4 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1708a78eda223b6daa302b140037fcc214a779f9 @@ -329,7 +329,7 @@ F src/os.c 8fd25588eeba74068d41102d26810e216999b6c8 F src/os.h 3e57a24e2794a94d3cf2342c6d9a884888cd96bf F src/os_common.h abdb9a191a367793268fe553d25bab894e986a0e F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa -F src/os_unix.c 0eb7f469fcd4e1fbedf30060438e26b839ec5486 +F src/os_unix.c cf524029242b4f878d6b97bad25cc2c0b66c2b31 F src/os_win.c 386fba30419e8458b13209781c2af5590eab2811 F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca F src/pager.c 2916c66aee50f69d9ec56a7619b62d9c6a3bee61 @@ -1061,7 +1061,7 @@ F test/subselect.test d24fd8757daf97dafd2e889c73ea4c4272dcf4e4 F test/substr.test 18f57c4ca8a598805c4d64e304c418734d843c1a F test/subtype1.test 7fe09496352f97053af1437150751be2d0a0cae8 F test/superlock.test 1cde669f68d2dd37d6c9bd35eee1d95491ae3fc2 -F test/symlink.test cbf6cb8c6c4b63a39e9f0f6b0d5c99e249dbc102 +F test/symlink.test 511db82662446bb0d3619002422760ef8e4b1122 F test/sync.test a34cd43e98b7fb84eabbf38f7ed8f7349b3f3d85 F test/syscall.test 2aa9e111b79fb385681ff8940124def6f8faab87 F test/sysfault.test c9f2b0d8d677558f74de750c75e12a5454719d04 @@ -1419,7 +1419,10 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 30671345b1c1ee55a2d1aa17273213f1849efd81 -R f342889ba3409bd63b1cf54d88bfe0e2 -U drh -Z e7c5fc86067d4269675714897cec0b51 +P 7ac017a498b6fb28343eef2d24e400c7800660d6 +R 0289127606d03bb3b649ecbd39bb14a5 +T *branch * follow-symlinks +T *sym-follow-symlinks * +T -sym-trunk * +U dan +Z 3e18b3d01ba737d3635110839e378935 diff --git a/manifest.uuid b/manifest.uuid index 276ef7a03f..6dda1af216 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7ac017a498b6fb28343eef2d24e400c7800660d6 \ No newline at end of file +80398fd44fb232193450103808e1854e0eba5652 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index ee9b55674b..9cd1fa19ad 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -149,6 +149,11 @@ */ #define MAX_PATHNAME 512 +/* +** Maximum supported symbolic links +*/ +#define SQLITE_MAX_SYMLINKS 100 + /* Always cast the getpid() return type for compatibility with ** kernel modules in VxWorks. */ #define osGetpid(X) (pid_t)getpid() @@ -5927,52 +5932,22 @@ static int unixAccess( return SQLITE_OK; } - /* -** Turn a relative pathname into a full pathname. The relative path -** is stored as a nul-terminated string in the buffer pointed to by -** zPath. +** Buffer zOut contains a (possibly) relative pathname. Overwrite it with +** the corresponding full pathname. ** -** zOut points to a buffer of at least sqlite3_vfs.mxPathname bytes -** (in this case, MAX_PATHNAME bytes). The full-path is written to -** this buffer before returning. +** Parameter nOut is the allocated size of buffer zOut. nByte is the number +** of bytes in the nul-terminated string that it contains (not including +** the nul-terminator itself). +** +** Return SQLITE_OK if successful, or an SQLite error code otherwise. */ -static int unixFullPathname( - sqlite3_vfs *pVfs, /* Pointer to vfs object */ - const char *zPath, /* Possibly relative input path */ - int nOut, /* Size of output buffer in bytes */ - char *zOut /* Output buffer */ +static int mkFullPathname( + const char *zPath, /* Use this path to log any errors */ + char *zOut, /* IN/OUT: Buffer to modify */ + int nByte, /* size of nul-terminated zOut in bytes */ + int nOut /* Allocated size of buffer zOut */ ){ - int nByte; - - /* It's odd to simulate an io-error here, but really this is just - ** using the io-error infrastructure to test that SQLite handles this - ** function failing. This function could fail if, for example, the - ** current working directory has been unlinked. - */ - SimulateIOError( return SQLITE_ERROR ); - - assert( pVfs->mxPathname==MAX_PATHNAME ); - UNUSED_PARAMETER(pVfs); - -#if defined(HAVE_READLINK) - /* Attempt to resolve the path as if it were a symbolic link. If it is - ** a symbolic link, the resolved path is stored in buffer zOut[]. Or, if - ** the identified file is not a symbolic link or does not exist, then - ** zPath is copied directly into zOut. Either way, nByte is left set to - ** the size of the string copied into zOut[] in bytes. */ - nByte = osReadlink(zPath, zOut, nOut-1); - if( nByte<0 ){ - if( errno!=EINVAL && errno!=ENOENT ){ - return unixLogError(SQLITE_CANTOPEN_BKPT, "readlink", zPath); - } - sqlite3_snprintf(nOut, zOut, "%s", zPath); - nByte = sqlite3Strlen30(zOut); - }else{ - zOut[nByte] = '\0'; - } -#endif - /* If buffer zOut[] now contains an absolute path there is nothing more ** to do. If it contains a relative path, do the following: ** @@ -5989,6 +5964,7 @@ static int unixFullPathname( ** truncated to make it fit. This is Ok, as SQLite refuses to open any ** file for which this function returns a full path larger than (nOut-8) ** bytes in size. */ + assert( nBytemxPathname==MAX_PATHNAME ); + UNUSED_PARAMETER(pVfs); + + /* It's odd to simulate an io-error here, but really this is just + ** using the io-error infrastructure to test that SQLite handles this + ** function failing. This function could fail if, for example, the + ** current working directory has been unlinked. + */ + SimulateIOError( return SQLITE_ERROR ); + + do { + + /* Attempt to resolve the path as if it were a symbolic link. If it is + ** a symbolic link, the resolved path is stored in buffer zOut[]. Or, if + ** the identified file is not a symbolic link or does not exist, then + ** zPath is copied directly into zOut. Either way, nByte is left set to + ** the size of the string copied into zOut[] in bytes. */ + assert( (zDel==0 && zIn==zPath) || (zDel!=0 && zIn==zDel) ); + if( zDel ){ + assert( zIn==zDel ); + sqlite3_snprintf(nOut, zDel, "%s", zOut); + } + nByte = osReadlink(zIn, zOut, nOut-1); + if( nByte<0 ){ + if( errno!=EINVAL && errno!=ENOENT ){ + rc = unixLogError(SQLITE_CANTOPEN_BKPT, "readlink", zPath); + }else{ + sqlite3_snprintf(nOut, zOut, "%s", zIn); + nByte = sqlite3Strlen30(zOut); + } + bLink = 0; + }else if( ++nLink>SQLITE_MAX_SYMLINKS ){ + sqlite3_log(SQLITE_CANTOPEN, + "too many symbolic links (max=%d)", SQLITE_MAX_SYMLINKS + ); + rc = SQLITE_CANTOPEN_BKPT; + }else{ + zOut[nByte] = '\0'; + if( zOut[0]!='/' ){ + int n; + for(n = sqlite3Strlen30(zIn); n>0 && zIn[n-1]!='/'; n--); + if( nByte+n+1>nOut ){ + rc = SQLITE_CANTOPEN_BKPT; + }else{ + memmove(&zOut[n], zOut, nByte+1); + memcpy(zOut, zIn, n); + nByte += n; + } + } + if( zDel==0 ){ + zDel = sqlite3_malloc(nOut); + if( zDel==0 ) rc = SQLITE_NOMEM; + zIn = (const char*)zDel; + } + bLink = 1; + } + + if( rc==SQLITE_OK ){ + rc = mkFullPathname(zPath, zOut, nByte, nOut); + } + }while( bLink && rc==SQLITE_OK ); + + sqlite3_free(zDel); + return rc; +#endif /* HAVE_READLINK */ +} + #ifndef SQLITE_OMIT_LOAD_EXTENSION /* diff --git a/test/symlink.test b/test/symlink.test index 790624161f..e9e5f3c305 100644 --- a/test/symlink.test +++ b/test/symlink.test @@ -83,38 +83,49 @@ do_test 1.5 { do_test 2.0 { catch { db close } catch { db2 close } - forcedelete test.db test.db2 + forcedelete test.db test.db2 test.db3 sqlite3 db test.db execsql { CREATE TABLE t1(x) } file link test.db2 test.db - sqlite3 db2 test.db2 - file exists test.db-journal -} 0 + file link test.db3 test.db2 + set {} {} +} {} -do_test 2.1 { - execsql { - BEGIN; - INSERT INTO t1 VALUES(1); - } db2 - file exists test.db-journal -} 1 -do_test 2.2 { - file exists test.db2-journal -} 0 -do_test 2.3 { - execsql { - COMMIT; - PRAGMA journal_mode = wal; - INSERT INTO t1 VALUES(2); - } db2 - file exists test.db-wal -} 1 -do_test 2.4 { - file exists test.db2-wal -} 0 -do_execsql_test 2.5 { - SELECT * FROM t1; -} {1 2} +foreach {tn f} {1 test.db2 2 test.db3} { + do_test 2.$tn.1 { + sqlite3 db2 $f + file exists test.db-journal + } 0 + do_test 2.$tn.2 { + execsql { + BEGIN; + INSERT INTO t1 VALUES(1); + } db2 + file exists test.db-journal + } 1 + do_test 2.$tn.3 { + list [file exists test2.db-journal] [file exists test3.db-journal] + } {0 0} + do_test 2.$tn.4 { + execsql { + COMMIT; + PRAGMA journal_mode = wal; + INSERT INTO t1 VALUES(2); + } db2 + file exists test.db-wal + } 1 + do_test 2.$tn.5 { + list [file exists test2.db-wal] [file exists test3.db-wal] + } {0 0} + do_execsql_test 2.$tn.6 { + SELECT * FROM t1; + } {1 2} + db2 close + do_execsql_test 2.$tn.7 { + DELETE FROM t1; + PRAGMA journal_mode = delete; + } delete +} # Try to open a ridiculously long pathname. Bug found by # Kostya Serebryany using libFuzzer on 2015-11-30. @@ -125,5 +136,57 @@ do_test 3.1 { set res } {unable to open database file} +#------------------------------------------------------------------------- +# Test that relative symlinks that are not located in the cwd work. +# +do_test 4.1 { + forcedelete x y z + file mkdir x + file mkdir y + file mkdir z + sqlite3 db x/test.db + file link y/test.db ../x/test.db + file link z/test.db ../y/test.db + execsql { + PRAGMA journal_mode = wal; + CREATE TABLE t1(x, y); + INSERT INTO t1 VALUES('hello', 'world'); + } +} {wal} + +do_test 4.2.1 { + db close + sqlite3 db y/test.db + db eval { SELECT * FROM t1 } +} {hello world} +do_test 4.2.2 { + list [file exists x/test.db-wal] [file exists y/test.db-wal] +} {1 0} + +do_test 4.3.1 { + db close + sqlite3 db z/test.db + db eval { SELECT * FROM t1 } +} {hello world} +do_test 4.3.2 { + list [file exists x/test.db-wal] [file exists y/test.db-wal] \ + [file exists z/test.db-wal] +} {1 0 0} + +do_test 4.4.0 { + forcedelete w + file mkdir w + file link w/test.db [file join [pwd] x/test.db] + set {} {} +} {} +do_test 4.4.1 { + db close + breakpoint + sqlite3 db w/test.db + db eval { SELECT * FROM t1 } +} {hello world} +do_test 4.4.2 { + list [file exists x/test.db-wal] [file exists w/test.db-wal] +} {1 0} finish_test From caf6b150a1077f2d4293b4091247c7e932da3927 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 25 Jan 2016 18:05:49 +0000 Subject: [PATCH 31/39] Simplify the unixFullpathname() function. This adds a dependency on lstat(). FossilOrigin-Name: f71249d3db9242b8f38955db51a7a5789d002803 --- manifest | 15 +++--- manifest.uuid | 2 +- src/os_unix.c | 137 +++++++++++++++++++------------------------------- 3 files changed, 60 insertions(+), 94 deletions(-) diff --git a/manifest b/manifest index 0b395a1661..27188d419f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sissues\son\sunix\swith\sopening\sdatabase\sfiles\svia\ssymlinks\sthat\sare\snot\sin\sthe\scurrent\sworking\sdirectory.\sAnd\swith\snested\ssymlinks. -D 2016-01-25T17:04:48.546 +C Simplify\sthe\sunixFullpathname()\sfunction.\sThis\sadds\sa\sdependency\son\slstat(). +D 2016-01-25T18:05:49.964 F Makefile.in 027c1603f255390c43a426671055a31c0a65fdb4 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1708a78eda223b6daa302b140037fcc214a779f9 @@ -329,7 +329,7 @@ F src/os.c 8fd25588eeba74068d41102d26810e216999b6c8 F src/os.h 3e57a24e2794a94d3cf2342c6d9a884888cd96bf F src/os_common.h abdb9a191a367793268fe553d25bab894e986a0e F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa -F src/os_unix.c cf524029242b4f878d6b97bad25cc2c0b66c2b31 +F src/os_unix.c 6604e7f9e5298b615f729b6bb3c22bd3545cdca6 F src/os_win.c 386fba30419e8458b13209781c2af5590eab2811 F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca F src/pager.c 2916c66aee50f69d9ec56a7619b62d9c6a3bee61 @@ -1419,10 +1419,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 7ac017a498b6fb28343eef2d24e400c7800660d6 -R 0289127606d03bb3b649ecbd39bb14a5 -T *branch * follow-symlinks -T *sym-follow-symlinks * -T -sym-trunk * +P 80398fd44fb232193450103808e1854e0eba5652 +R 084dee05e1353887f6af0d88670b22d0 U dan -Z 3e18b3d01ba737d3635110839e378935 +Z 72572e009a3ce3b7747f3257ebf7dfc7 diff --git a/manifest.uuid b/manifest.uuid index 6dda1af216..b66c39a6ae 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -80398fd44fb232193450103808e1854e0eba5652 \ No newline at end of file +f71249d3db9242b8f38955db51a7a5789d002803 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 9cd1fa19ad..3d113301b8 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -480,6 +480,8 @@ static struct unix_syscall { #endif #define osReadlink ((ssize_t(*)(const char*,char*,size_t))aSyscall[26].pCurrent) + { "lstat", (sqlite3_syscall_ptr)lstat, 0 }, +#define osLstat ((int(*)(const char*,struct stat*))aSyscall[27].pCurrent) }; /* End of the overrideable system calls */ @@ -5933,54 +5935,24 @@ static int unixAccess( } /* -** Buffer zOut contains a (possibly) relative pathname. Overwrite it with -** the corresponding full pathname. ** -** Parameter nOut is the allocated size of buffer zOut. nByte is the number -** of bytes in the nul-terminated string that it contains (not including -** the nul-terminator itself). -** -** Return SQLITE_OK if successful, or an SQLite error code otherwise. */ static int mkFullPathname( - const char *zPath, /* Use this path to log any errors */ - char *zOut, /* IN/OUT: Buffer to modify */ - int nByte, /* size of nul-terminated zOut in bytes */ + const char *zPath, /* Input path */ + char *zOut, /* Output buffer */ int nOut /* Allocated size of buffer zOut */ ){ - /* If buffer zOut[] now contains an absolute path there is nothing more - ** to do. If it contains a relative path, do the following: - ** - ** * move the relative path string so that it is at the end of th - ** zOut[] buffer. - ** * Call getcwd() to read the path of the current working directory - ** into the start of the zOut[] buffer. - ** * Append a '/' character to the cwd string and move the - ** relative path back within the buffer so that it immediately - ** follows the '/'. - ** - ** This code is written so that if the combination of the CWD and relative - ** path are larger than the allocated size of zOut[] the CWD is silently - ** truncated to make it fit. This is Ok, as SQLite refuses to open any - ** file for which this function returns a full path larger than (nOut-8) - ** bytes in size. */ - assert( nBytenOut ) return SQLITE_CANTOPEN_BKPT; + sqlite3_snprintf(nOut-iOff, &zOut[iOff], "%s", zPath); return SQLITE_OK; } @@ -6000,13 +5972,11 @@ static int unixFullPathname( char *zOut /* Output buffer */ ){ #if !defined(HAVE_READLINK) - sqlite3_snprintf(nOut, zOut, "%s", zIn); - nByte = sqlite3Strlen30(zOut); - return mkFullPathname(zPath, zOut, sqlite3Strlen30(zOut), nOut); + return mkFullPathname(zPath, zOut, nOut); #else int rc = SQLITE_OK; int nByte; - int nLink = 0; /* Number of symbolic links followed so far */ + int nLink = 1; /* Number of symbolic links followed so far */ int bLink; /* True for a symbolic link */ const char *zIn = zPath; /* Input path for each iteration of loop */ char *zDel = 0; @@ -6023,55 +5993,54 @@ static int unixFullPathname( do { - /* Attempt to resolve the path as if it were a symbolic link. If it is - ** a symbolic link, the resolved path is stored in buffer zOut[]. Or, if - ** the identified file is not a symbolic link or does not exist, then - ** zPath is copied directly into zOut. Either way, nByte is left set to - ** the size of the string copied into zOut[] in bytes. */ - assert( (zDel==0 && zIn==zPath) || (zDel!=0 && zIn==zDel) ); - if( zDel ){ - assert( zIn==zDel ); - sqlite3_snprintf(nOut, zDel, "%s", zOut); - } - nByte = osReadlink(zIn, zOut, nOut-1); - if( nByte<0 ){ - if( errno!=EINVAL && errno!=ENOENT ){ - rc = unixLogError(SQLITE_CANTOPEN_BKPT, "readlink", zPath); - }else{ - sqlite3_snprintf(nOut, zOut, "%s", zIn); - nByte = sqlite3Strlen30(zOut); + /* Call stat() on path zIn. Set bLink to true if the path is a symbolic + ** link, or false otherwise. */ + int bLink = 0; + struct stat buf; + if( osLstat(zIn, &buf)!=0 ){ + if( errno!=ENOENT ){ + rc = unixLogError(SQLITE_CANTOPEN_BKPT, "stat", zIn); } - bLink = 0; - }else if( ++nLink>SQLITE_MAX_SYMLINKS ){ - sqlite3_log(SQLITE_CANTOPEN, - "too many symbolic links (max=%d)", SQLITE_MAX_SYMLINKS - ); - rc = SQLITE_CANTOPEN_BKPT; }else{ - zOut[nByte] = '\0'; - if( zOut[0]!='/' ){ - int n; - for(n = sqlite3Strlen30(zIn); n>0 && zIn[n-1]!='/'; n--); - if( nByte+n+1>nOut ){ - rc = SQLITE_CANTOPEN_BKPT; - }else{ - memmove(&zOut[n], zOut, nByte+1); - memcpy(zOut, zIn, n); - nByte += n; - } - } + bLink = S_ISLNK(buf.st_mode); + } + + if( bLink ){ if( zDel==0 ){ zDel = sqlite3_malloc(nOut); if( zDel==0 ) rc = SQLITE_NOMEM; - zIn = (const char*)zDel; + }else if( ++nLink>SQLITE_MAX_SYMLINKS ){ + rc = SQLITE_CANTOPEN_BKPT; } - bLink = 1; + + if( rc==SQLITE_OK ){ + nByte = osReadlink(zIn, zDel, nOut-1); + if( nByte<0 ){ + rc = unixLogError(SQLITE_CANTOPEN_BKPT, "readlink", zIn); + }else if( zDel[0]!='/' ){ + int n; + for(n = sqlite3Strlen30(zIn); n>0 && zIn[n-1]!='/'; n--); + if( nByte+n+1>nOut ){ + rc = SQLITE_CANTOPEN_BKPT; + }else{ + memmove(&zDel[n], zDel, nByte+1); + memcpy(zDel, zIn, n); + nByte += n; + } + zDel[nByte] = '\0'; + } + } + + zIn = zDel; } if( rc==SQLITE_OK ){ - rc = mkFullPathname(zPath, zOut, nByte, nOut); + assert( zIn!=zOut || zIn[0]=='/' ); + rc = mkFullPathname(zIn, zOut, nOut); } - }while( bLink && rc==SQLITE_OK ); + if( bLink==0 ) break; + zIn = zOut; + }while( rc==SQLITE_OK ); sqlite3_free(zDel); return rc; @@ -7574,7 +7543,7 @@ int sqlite3_os_init(void){ /* Double-check that the aSyscall[] array has been constructed ** correctly. See ticket [bb3a86e890c8e96ab] */ - assert( ArraySize(aSyscall)==27 ); + assert( ArraySize(aSyscall)==28 ); /* Register all VFSes defined in the aVfs[] array */ for(i=0; i<(sizeof(aVfs)/sizeof(sqlite3_vfs)); i++){ From af1b36b1df592aa8ecf1fd8799db7e415688ed6e Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 25 Jan 2016 18:43:05 +0000 Subject: [PATCH 32/39] Only use lstat() if the HAVE_LSTAT macro is defined. Fix some test file issues. FossilOrigin-Name: 8a6e4147a680ad6c5fdd588468b6daac82349d2c --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/os_unix.c | 12 ++++++++---- src/vxworks.h | 1 + test/oserror.test | 2 +- test/syscall.test | 2 +- 6 files changed, 21 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 27188d419f..33b8dc8b5c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Simplify\sthe\sunixFullpathname()\sfunction.\sThis\sadds\sa\sdependency\son\slstat(). -D 2016-01-25T18:05:49.964 +C Only\suse\slstat()\sif\sthe\sHAVE_LSTAT\smacro\sis\sdefined.\sFix\ssome\stest\sfile\sissues. +D 2016-01-25T18:43:05.007 F Makefile.in 027c1603f255390c43a426671055a31c0a65fdb4 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1708a78eda223b6daa302b140037fcc214a779f9 @@ -329,7 +329,7 @@ F src/os.c 8fd25588eeba74068d41102d26810e216999b6c8 F src/os.h 3e57a24e2794a94d3cf2342c6d9a884888cd96bf F src/os_common.h abdb9a191a367793268fe553d25bab894e986a0e F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa -F src/os_unix.c 6604e7f9e5298b615f729b6bb3c22bd3545cdca6 +F src/os_unix.c 4c217111d7bdebbd7a9b47cf11f434420da432e7 F src/os_win.c 386fba30419e8458b13209781c2af5590eab2811 F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca F src/pager.c 2916c66aee50f69d9ec56a7619b62d9c6a3bee61 @@ -420,7 +420,7 @@ F src/vdbemem.c b9181e77eca2a095929d46250daf85c8d2621fc0 F src/vdbesort.c 0971557e5d3c289e46f56a52aed2197c13251de7 F src/vdbetrace.c 8befe829faff6d9e6f6e4dee5a7d3f85cc85f1a0 F src/vtab.c 320682cca733115b4cbe71320b5c5eeb1074ebde -F src/vxworks.h 974e7d9a98f602d6310d563e1dc4e08f9fc48e47 +F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c d21b99fd1458159d0b1ecdccc8ee6ada4fdc4c54 F src/wal.h 2f7c831cf3b071fa548bf2d5cac640846a7ff19c F src/walker.c 0f142b5bd3ed2041fc52d773880748b212e63354 @@ -924,7 +924,7 @@ F test/orderby6.test 8b38138ab0972588240b3fca0985d2e400432859 F test/orderby7.test 3d1383d52ade5b9eb3a173b3147fdd296f0202da F test/orderby8.test 23ef1a5d72bd3adcc2f65561c654295d1b8047bd F test/orderby9.test 87fb9548debcc2cd141c5299002dd94672fa76a3 -F test/oserror.test 361346396ae18462c7393c1ac5c3f17237bd89b2 +F test/oserror.test b32dc34f2363ef18532e3a0a7358e3e7e321974f F test/ovfl.test 4f7ca651cba5c059a12d8c67dddd49bec5747799 F test/pager1.test 1acbdb14c5952a72dd43129cabdbf69aaa3ed1fa F test/pager2.test 67b8f40ae98112bcdba1f2b2d03ea83266418c71 @@ -1063,7 +1063,7 @@ F test/subtype1.test 7fe09496352f97053af1437150751be2d0a0cae8 F test/superlock.test 1cde669f68d2dd37d6c9bd35eee1d95491ae3fc2 F test/symlink.test 511db82662446bb0d3619002422760ef8e4b1122 F test/sync.test a34cd43e98b7fb84eabbf38f7ed8f7349b3f3d85 -F test/syscall.test 2aa9e111b79fb385681ff8940124def6f8faab87 +F test/syscall.test f59ba4e25f7ba4a4c031026cc2ef8b6e4b4c639c F test/sysfault.test c9f2b0d8d677558f74de750c75e12a5454719d04 F test/tabfunc01.test cc33684f9480fcf1fd5ce287ac28d22971cad1cc F test/table.test b708f3e5fa2542fa51dfab21fc07b36ea445cb2f @@ -1419,7 +1419,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 80398fd44fb232193450103808e1854e0eba5652 -R 084dee05e1353887f6af0d88670b22d0 +P f71249d3db9242b8f38955db51a7a5789d002803 +R 31817b68212f6684c4e9fbfebdae658d U dan -Z 72572e009a3ce3b7747f3257ebf7dfc7 +Z 17d90a7cade21886dca3e4bdfc08ed4c diff --git a/manifest.uuid b/manifest.uuid index b66c39a6ae..35202dd686 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f71249d3db9242b8f38955db51a7a5789d002803 \ No newline at end of file +8a6e4147a680ad6c5fdd588468b6daac82349d2c \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 3d113301b8..a3bb45bdb9 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -480,7 +480,11 @@ static struct unix_syscall { #endif #define osReadlink ((ssize_t(*)(const char*,char*,size_t))aSyscall[26].pCurrent) - { "lstat", (sqlite3_syscall_ptr)lstat, 0 }, +#if defined(HAVE_LSTAT) + { "lstat", (sqlite3_syscall_ptr)lstat, 0 }, +#else + { "lstat", (sqlite3_syscall_ptr)0, 0 }, +#endif #define osLstat ((int(*)(const char*,struct stat*))aSyscall[27].pCurrent) }; /* End of the overrideable system calls */ @@ -5971,7 +5975,7 @@ static int unixFullPathname( int nOut, /* Size of output buffer in bytes */ char *zOut /* Output buffer */ ){ -#if !defined(HAVE_READLINK) +#if !defined(HAVE_READLINK) || !defined(HAVE_LSTAT) return mkFullPathname(zPath, zOut, nOut); #else int rc = SQLITE_OK; @@ -5999,7 +6003,7 @@ static int unixFullPathname( struct stat buf; if( osLstat(zIn, &buf)!=0 ){ if( errno!=ENOENT ){ - rc = unixLogError(SQLITE_CANTOPEN_BKPT, "stat", zIn); + rc = unixLogError(SQLITE_CANTOPEN_BKPT, "lstat", zIn); } }else{ bLink = S_ISLNK(buf.st_mode); @@ -6044,7 +6048,7 @@ static int unixFullPathname( sqlite3_free(zDel); return rc; -#endif /* HAVE_READLINK */ +#endif /* HAVE_READLINK && HAVE_LSTAT */ } diff --git a/src/vxworks.h b/src/vxworks.h index 60c41a19b8..e7013c3f66 100644 --- a/src/vxworks.h +++ b/src/vxworks.h @@ -28,4 +28,5 @@ #define OS_VXWORKS 0 #define HAVE_FCHOWN 1 #define HAVE_READLINK 1 +#define HAVE_LSTAT 1 #endif /* defined(_WRS_KERNEL) */ diff --git a/test/oserror.test b/test/oserror.test index 5fa7f98eb5..246a9d4023 100644 --- a/test/oserror.test +++ b/test/oserror.test @@ -95,7 +95,7 @@ do_test 1.4.1 { do_re_test 1.4.2 { lindex $::log 0 -} {^os_unix.c:\d*: \(\d+\) (open|readlink)\(.*test.db\) - } +} {^os_unix.c:\d*: \(\d+\) (open|readlink|lstat)\(.*test.db\) - } #-------------------------------------------------------------------------- # Tests oserror-1.* test failures in the unlink() system call. diff --git a/test/syscall.test b/test/syscall.test index a935957d39..5716b35a6e 100644 --- a/test/syscall.test +++ b/test/syscall.test @@ -61,7 +61,7 @@ foreach s { fcntl read pread write pwrite fchmod fallocate pread64 pwrite64 unlink openDirectory mkdir rmdir statvfs fchown geteuid umask mmap munmap mremap - getpagesize readlink + getpagesize readlink lstat } { if {[test_syscall exists $s]} {lappend syscall_list $s} } From 2aac8c7ba1e84a90b36906926336a4558360a02e Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 25 Jan 2016 22:08:11 +0000 Subject: [PATCH 33/39] Fix a compiler warning about doing pointer arithmetic involving a NULL pointer even though the result of computation is never used. FossilOrigin-Name: 7c49a9478bd36564e81d33458ca1f4063ddaca83 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbesort.c | 4 +++- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 827a3d5126..5a7fe0a637 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Replace\sthe\sOP_SetIfNotPos\soperator\swith\sOP_OffsetLimit\sin\sthe\sVDBE,\sfor\nsimpler\sand\ssmaller\scode. -D 2016-01-25T15:57:29.060 +C Fix\sa\scompiler\swarning\sabout\sdoing\spointer\sarithmetic\sinvolving\sa\sNULL\spointer\neven\sthough\sthe\sresult\sof\scomputation\sis\snever\sused. +D 2016-01-25T22:08:11.961 F Makefile.in 027c1603f255390c43a426671055a31c0a65fdb4 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1708a78eda223b6daa302b140037fcc214a779f9 @@ -417,7 +417,7 @@ F src/vdbeapi.c ffae8f5af4570fbd548504e815e9fb7227f0822e F src/vdbeaux.c 757f86e6fef8efb3dd4226cb31e2e82b9c44c883 F src/vdbeblob.c 37c3d11a753e403698c69e17383d282e1ae73e75 F src/vdbemem.c b9181e77eca2a095929d46250daf85c8d2621fc0 -F src/vdbesort.c 0971557e5d3c289e46f56a52aed2197c13251de7 +F src/vdbesort.c 3bb1f1f03162e6d223da623714d8e93fcaeac658 F src/vdbetrace.c 8befe829faff6d9e6f6e4dee5a7d3f85cc85f1a0 F src/vtab.c 320682cca733115b4cbe71320b5c5eeb1074ebde F src/vxworks.h 974e7d9a98f602d6310d563e1dc4e08f9fc48e47 @@ -1419,7 +1419,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 30671345b1c1ee55a2d1aa17273213f1849efd81 -R f342889ba3409bd63b1cf54d88bfe0e2 +P 7ac017a498b6fb28343eef2d24e400c7800660d6 +R aa5d51a793ca6d0a7613986a1affa54a U drh -Z e7c5fc86067d4269675714897cec0b51 +Z 142832d114ade2d8f5ff5fab3faed40b diff --git a/manifest.uuid b/manifest.uuid index 276ef7a03f..d34f9b48c8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7ac017a498b6fb28343eef2d24e400c7800660d6 \ No newline at end of file +7c49a9478bd36564e81d33458ca1f4063ddaca83 \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index e095f80912..22029aa6bf 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -1837,7 +1837,9 @@ int sqlite3VdbeSorterWrite( pNew = (SorterRecord*)&pSorter->list.aMemory[pSorter->iMemory]; pSorter->iMemory += ROUND8(nReq); - pNew->u.iNext = (int)((u8*)(pSorter->list.pList) - pSorter->list.aMemory); + if( pSorter->list.pList ){ + pNew->u.iNext = (int)((u8*)(pSorter->list.pList) - pSorter->list.aMemory); + } }else{ pNew = (SorterRecord *)sqlite3Malloc(nReq); if( pNew==0 ){ From a72d15b5f0bbc0d67eb117624df7f005ad89c443 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 25 Jan 2016 23:19:58 +0000 Subject: [PATCH 34/39] Minor changes to the autoconf/README.txt file. FossilOrigin-Name: 6df8a9c00a9d067f67d492da7a4617908070c6c0 --- autoconf/README.txt | 16 +++++++++++----- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/autoconf/README.txt b/autoconf/README.txt index a581c39dea..114831ec5e 100644 --- a/autoconf/README.txt +++ b/autoconf/README.txt @@ -9,6 +9,12 @@ This package contains: * a Makefile.msc and sqlite3.rc for building with Microsoft Visual C++ on Windows. +SUMMARY OF HOW TO BUILD +======================= + + unix: ./configure; make + windows: nmake /f Makefile.msc + BUILDING ON POSIX ================= @@ -47,7 +53,7 @@ Using Microsoft Visual C++ 2005 (or later) is recommended. Several Windows platform variants may be built by adding additional macros to the NMAKE command line. -BUILDING FOR WINRT 8.0 +Building for WinRT 8.0 ---------------------- FOR_WINRT=1 @@ -58,7 +64,7 @@ NMAKE command line as well: "NSDKLIBPATH=%WindowsSdkDir%\..\8.0\lib\win8\um\x86" -BUILDING FOR WINRT 8.1 +Building for WinRT 8.1 ---------------------- FOR_WINRT=1 @@ -69,7 +75,7 @@ NMAKE command line as well: "NSDKLIBPATH=%WindowsSdkDir%\..\8.1\lib\winv6.3\um\x86" -BUILDING FOR UAP 10.0 +Building for UAP 10.0 --------------------- FOR_WINRT=1 FOR_UAP=1 @@ -82,7 +88,7 @@ NMAKE command line as well: "PSDKLIBPATH=%WindowsSdkDir%\..\10\lib\10.0.10586.0\um\x86" "NUCRTLIBPATH=%UniversalCRTSdkDir%\..\10\lib\10.0.10586.0\ucrt\x86" -BUILDING FOR THE WINDOWS 10 SDK +Building for the Windows 10 SDK ------------------------------- FOR_WIN10=1 @@ -90,7 +96,7 @@ BUILDING FOR THE WINDOWS 10 SDK Using Microsoft Visual C++ 2015 (or later) is required. When using the above, no other macros should be needed on the NMAKE command line. -USING PREPROCESSOR DEFINES +Other preprocessor defines -------------------------- Additionally, preprocessor defines may be specified by using the OPTS macro diff --git a/manifest b/manifest index 0f3e8b7b8d..48622ea281 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C More\supdates\sto\sthe\sREADME. -D 2016-01-23T21:57:46.170 +C Minor\schanges\sto\sthe\sautoconf/README.txt\sfile. +D 2016-01-25T23:19:58.651 F Makefile.in 027c1603f255390c43a426671055a31c0a65fdb4 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6fca5455aaecbd14479f33f091aa19df2d3d2969 @@ -13,7 +13,7 @@ F autoconf/INSTALL 83e4a25da9fd053c7b3665eaaaf7919707915903 F autoconf/Makefile.am 1c1657650775960804945dc392e14d9e43c5ed84 F autoconf/Makefile.msc 68ed752a809b611d97b95d8572a34fe6fd1196f1 F autoconf/README.first 6c4f34fe115ff55d4e8dbfa3cecf04a0188292f7 -F autoconf/README.txt fc86867528c500dfd691620e623bfe6799c98d14 +F autoconf/README.txt e3a5cf0ba2d8944c71ca1eb7c06c15f52bfca131 F autoconf/configure.ac 7b1ea0dcaf49fafba262ce4b0ee8cb3281b555d1 F autoconf/tea/Makefile.in b438a7020446c8a8156e8d97c8914a04833da6fd F autoconf/tea/README 3e9a3c060f29a44344ab50aec506f4db903fb873 @@ -1421,7 +1421,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 4ce60fa42d8486d219d40d25d785d7f2cabe5e06 -R 9ddb76e24748b77caceaf34a8b255b23 -U mistachkin -Z 8a8dd4db0d2ee1de0599573a5cb1355c +P 9f02868df71d902a2b9118d0b06df18ee92b666b +R 9aeab9bcdca09d2b8e9198ca7dec65cb +U drh +Z 1be43b84d3a6e7c2425c5f0806c1fce0 diff --git a/manifest.uuid b/manifest.uuid index c53057578b..4cab521d6d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9f02868df71d902a2b9118d0b06df18ee92b666b \ No newline at end of file +6df8a9c00a9d067f67d492da7a4617908070c6c0 \ No newline at end of file From f0fc9929369cda0c7416174d43a299188b8beabf Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 26 Jan 2016 00:12:42 +0000 Subject: [PATCH 35/39] Remove an unused variable. FossilOrigin-Name: 1c2656c1d37906230edc142d3a4253b16b6e925f --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_unix.c | 1 - 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 33b8dc8b5c..ea40dd333f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Only\suse\slstat()\sif\sthe\sHAVE_LSTAT\smacro\sis\sdefined.\sFix\ssome\stest\sfile\sissues. -D 2016-01-25T18:43:05.007 +C Remove\san\sunused\svariable. +D 2016-01-26T00:12:42.634 F Makefile.in 027c1603f255390c43a426671055a31c0a65fdb4 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1708a78eda223b6daa302b140037fcc214a779f9 @@ -329,7 +329,7 @@ F src/os.c 8fd25588eeba74068d41102d26810e216999b6c8 F src/os.h 3e57a24e2794a94d3cf2342c6d9a884888cd96bf F src/os_common.h abdb9a191a367793268fe553d25bab894e986a0e F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa -F src/os_unix.c 4c217111d7bdebbd7a9b47cf11f434420da432e7 +F src/os_unix.c e9e1e6a49892a76fe1fd167b1d475eeddaf1ccb3 F src/os_win.c 386fba30419e8458b13209781c2af5590eab2811 F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca F src/pager.c 2916c66aee50f69d9ec56a7619b62d9c6a3bee61 @@ -1419,7 +1419,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f71249d3db9242b8f38955db51a7a5789d002803 -R 31817b68212f6684c4e9fbfebdae658d -U dan -Z 17d90a7cade21886dca3e4bdfc08ed4c +P 8a6e4147a680ad6c5fdd588468b6daac82349d2c +R d2cf53f7b50c768b631d5f18bba143b9 +U drh +Z 592370e14c792642f017f87ac074cb77 diff --git a/manifest.uuid b/manifest.uuid index 35202dd686..78b51cd18f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8a6e4147a680ad6c5fdd588468b6daac82349d2c \ No newline at end of file +1c2656c1d37906230edc142d3a4253b16b6e925f \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index a3bb45bdb9..cdbcd2c8e0 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -5981,7 +5981,6 @@ static int unixFullPathname( int rc = SQLITE_OK; int nByte; int nLink = 1; /* Number of symbolic links followed so far */ - int bLink; /* True for a symbolic link */ const char *zIn = zPath; /* Input path for each iteration of loop */ char *zDel = 0; From 23496702622549d23f22639b7e2f5c47cfeba9a6 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 26 Jan 2016 13:56:42 +0000 Subject: [PATCH 36/39] Ensure that unixFullpathname() always nul-terminates its output buffer, even when returning an error. FossilOrigin-Name: 4a4385564dd3887a7953820b60c99d6ce289f96a --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/os_unix.c | 31 +++++++++++++++++++------------ test/symlink.test | 1 - 4 files changed, 28 insertions(+), 22 deletions(-) diff --git a/manifest b/manifest index ea40dd333f..2765e104fb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\sunused\svariable. -D 2016-01-26T00:12:42.634 +C Ensure\sthat\sunixFullpathname()\salways\snul-terminates\sits\soutput\sbuffer,\seven\swhen\sreturning\san\serror. +D 2016-01-26T13:56:42.494 F Makefile.in 027c1603f255390c43a426671055a31c0a65fdb4 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1708a78eda223b6daa302b140037fcc214a779f9 @@ -329,7 +329,7 @@ F src/os.c 8fd25588eeba74068d41102d26810e216999b6c8 F src/os.h 3e57a24e2794a94d3cf2342c6d9a884888cd96bf F src/os_common.h abdb9a191a367793268fe553d25bab894e986a0e F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa -F src/os_unix.c e9e1e6a49892a76fe1fd167b1d475eeddaf1ccb3 +F src/os_unix.c 5bb20172d0c9a6afcfa829a88c406970593c848d F src/os_win.c 386fba30419e8458b13209781c2af5590eab2811 F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca F src/pager.c 2916c66aee50f69d9ec56a7619b62d9c6a3bee61 @@ -1061,7 +1061,7 @@ F test/subselect.test d24fd8757daf97dafd2e889c73ea4c4272dcf4e4 F test/substr.test 18f57c4ca8a598805c4d64e304c418734d843c1a F test/subtype1.test 7fe09496352f97053af1437150751be2d0a0cae8 F test/superlock.test 1cde669f68d2dd37d6c9bd35eee1d95491ae3fc2 -F test/symlink.test 511db82662446bb0d3619002422760ef8e4b1122 +F test/symlink.test c9ebe7330d228249e447038276bfc8a7b22f4849 F test/sync.test a34cd43e98b7fb84eabbf38f7ed8f7349b3f3d85 F test/syscall.test f59ba4e25f7ba4a4c031026cc2ef8b6e4b4c639c F test/sysfault.test c9f2b0d8d677558f74de750c75e12a5454719d04 @@ -1419,7 +1419,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 8a6e4147a680ad6c5fdd588468b6daac82349d2c -R d2cf53f7b50c768b631d5f18bba143b9 -U drh -Z 592370e14c792642f017f87ac074cb77 +P 1c2656c1d37906230edc142d3a4253b16b6e925f +R 6c0824142ba46339a29c32338146479b +U dan +Z e95289d1bfc22539e02b85ec76fd69c7 diff --git a/manifest.uuid b/manifest.uuid index 78b51cd18f..72144560be 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1c2656c1d37906230edc142d3a4253b16b6e925f \ No newline at end of file +4a4385564dd3887a7953820b60c99d6ce289f96a \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index cdbcd2c8e0..81422f215d 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -5955,7 +5955,12 @@ static int mkFullPathname( iOff = sqlite3Strlen30(zOut); zOut[iOff++] = '/'; } - if( (iOff+nPath+1)>nOut ) return SQLITE_CANTOPEN_BKPT; + if( (iOff+nPath+1)>nOut ){ + /* SQLite assumes that xFullPathname() nul-terminates the output buffer + ** even if it returns an error. */ + zOut[iOff] = '\0'; + return SQLITE_CANTOPEN_BKPT; + } sqlite3_snprintf(nOut-iOff, &zOut[iOff], "%s", zPath); return SQLITE_OK; } @@ -6020,15 +6025,17 @@ static int unixFullPathname( nByte = osReadlink(zIn, zDel, nOut-1); if( nByte<0 ){ rc = unixLogError(SQLITE_CANTOPEN_BKPT, "readlink", zIn); - }else if( zDel[0]!='/' ){ - int n; - for(n = sqlite3Strlen30(zIn); n>0 && zIn[n-1]!='/'; n--); - if( nByte+n+1>nOut ){ - rc = SQLITE_CANTOPEN_BKPT; - }else{ - memmove(&zDel[n], zDel, nByte+1); - memcpy(zDel, zIn, n); - nByte += n; + }else{ + if( zDel[0]!='/' ){ + int n; + for(n = sqlite3Strlen30(zIn); n>0 && zIn[n-1]!='/'; n--); + if( nByte+n+1>nOut ){ + rc = SQLITE_CANTOPEN_BKPT; + }else{ + memmove(&zDel[n], zDel, nByte+1); + memcpy(zDel, zIn, n); + nByte += n; + } } zDel[nByte] = '\0'; } @@ -6037,8 +6044,8 @@ static int unixFullPathname( zIn = zDel; } - if( rc==SQLITE_OK ){ - assert( zIn!=zOut || zIn[0]=='/' ); + assert( rc!=SQLITE_OK || zIn!=zOut || zIn[0]=='/' ); + if( rc==SQLITE_OK && zIn!=zOut ){ rc = mkFullPathname(zIn, zOut, nOut); } if( bLink==0 ) break; diff --git a/test/symlink.test b/test/symlink.test index e9e5f3c305..949102cf8a 100644 --- a/test/symlink.test +++ b/test/symlink.test @@ -181,7 +181,6 @@ do_test 4.4.0 { } {} do_test 4.4.1 { db close - breakpoint sqlite3 db w/test.db db eval { SELECT * FROM t1 } } {hello world} From fee981e5b97a79b26308c2dc3f946264e4ee8028 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 26 Jan 2016 17:08:22 +0000 Subject: [PATCH 37/39] Enhance fts5txt2db.tcl, a script used to generate fts5/fts4 databases for performance testing. FossilOrigin-Name: c646e40350e5aa91abcf52de61fb31275bad38f9 --- ext/fts5/tool/fts5txt2db.tcl | 200 +++++++++++++++++++++++++---------- manifest | 12 +-- manifest.uuid | 2 +- 3 files changed, 152 insertions(+), 62 deletions(-) diff --git a/ext/fts5/tool/fts5txt2db.tcl b/ext/fts5/tool/fts5txt2db.tcl index c22c5dae72..c6a58e6edd 100644 --- a/ext/fts5/tool/fts5txt2db.tcl +++ b/ext/fts5/tool/fts5txt2db.tcl @@ -1,13 +1,127 @@ -proc usage {} { - puts stderr "$::argv0 ?OPTIONS? DATABASE FILE1..." +#------------------------------------------------------------------------- +# Command line options processor. +# +proc command_line_error {O E {msg ""}} { + if {$msg != ""} { + puts stderr "Error: $msg" + puts stderr "" + } + + set L [list] + foreach o $O { + if {[llength $o]==1} { + lappend L [string toupper $o] + } + } + + puts stderr "Usage: $::argv0 ?SWITCHES? $L" puts stderr "" - puts stderr "Options are" - puts stderr " -fts5" - puts stderr " -fts4" - puts stderr " -colsize " - puts stderr { + puts stderr "Switches are:" + foreach o $O { + if {[llength $o]==3} { + foreach {a b c} $o {} + puts stderr [format " -%-15s %s (default \"%s\")" "$a VAL" $c $b] + } elseif {[llength $o]==2} { + foreach {a b} $o {} + puts stderr [format " -%-15s %s" $a $b] + } + } + puts stderr "" + puts stderr $E + exit -1 +} + +proc process_command_line {avar lArgs O E} { + + upvar $avar A + set zTrailing "" ;# True if ... is present in $O + set lPosargs [list] + + # Populate A() with default values. Also, for each switch in the command + # line spec, set an entry in the idx() array as follows: + # + # {tblname t1 "table name to use"} + # -> [set idx(-tblname) {tblname t1 "table name to use"} + # + # For each position parameter, append its name to $lPosargs. If the ... + # specifier is present, set $zTrailing to the name of the prefix. + # + foreach o $O { + set nm [lindex $o 0] + set nArg [llength $o] + switch -- $nArg { + 1 { + if {[string range $nm end-2 end]=="..."} { + set zTrailing [string range $nm 0 end-3] + } else { + lappend lPosargs $nm + } + } + 2 { + set A($nm) 0 + set idx(-$nm) $o + } + 3 { + set A($nm) [lindex $o 1] + set idx(-$nm) $o + } + default { + error "Error in command line specification" + } + } + } + + # Set explicitly specified option values + # + set nArg [llength $lArgs] + for {set i 0} {$i < $nArg} {incr i} { + set opt [lindex $lArgs $i] + if {[string range $opt 0 0]!="-" || $opt=="--"} break + set c [array names idx "${opt}*"] + if {[llength $c]==0} { command_line_error $O $E "Unrecognized option: $opt"} + if {[llength $c]>1} { command_line_error $O $E "Ambiguous option: $opt"} + + if {[llength $idx($c)]==3} { + if {$i==[llength $lArgs]-1} { + command_line_error $O $E "Option requires argument: $c" + } + incr i + set A([lindex $idx($c) 0]) [lindex $lArgs $i] + } else { + set A([lindex $idx($c) 0]) 1 + } + } + + # Deal with position arguments. + # + set nPosarg [llength $lPosargs] + set nRem [expr $nArg - $i] + if {$nRem < $nPosarg || ($zTrailing=="" && $nRem > $nPosarg)} { + command_line_error $O $E + } + for {set j 0} {$j < $nPosarg} {incr j} { + set A([lindex $lPosargs $j]) [lindex $lArgs [expr $j+$i]] + } + if {$zTrailing!=""} { + set A($zTrailing) [lrange $lArgs [expr $j+$i] end] + } +} +# End of command line options processor. +#------------------------------------------------------------------------- + + +process_command_line A $argv { + {fts5 "use fts5"} + {fts4 "use fts4"} + {colsize "10 10 10" "list of column sizes"} + {tblname "t1" "table name to create"} + {detail "full" "Fts5 detail mode to use"} + {repeat 1 "Load each file this many times"} + database + file... +} { This script is designed to create fts4/5 tables with more than one column. The -colsize option should be set to a Tcl list of integer values, one for each column in the table. Each value is the number of tokens that will be @@ -22,59 +136,27 @@ of the -colsize list. The next N2 are used for the second column of the first row, and so on. Rows are added to the table until the entire list of tokens is exhausted. } - exit -1 + +if {$A(fts4)} { + set A(fts) fts4 +} else { + set A(fts) fts5 } -set O(aColSize) [list 10 10 10] -set O(tblname) t1 -set O(fts) fts5 - - -set options_with_values {-colsize} - -for {set i 0} {$i < [llength $argv]} {incr i} { - set opt [lindex $argv $i] - if {[string range $opt 0 0]!="-"} break - - if {[lsearch $options_with_values $opt]>=0} { - incr i - if {$i==[llength $argv]} usage - set val [lindex $argv $i] - } - - switch -- $opt { - -colsize { - set O(aColSize) $val - } - - -fts4 { - set O(fts) fts4 - } - - -fts5 { - set O(fts) fts5 - } - } -} - -if {$i > [llength $argv]-2} usage -set O(db) [lindex $argv $i] -set O(files) [lrange $argv [expr $i+1] end] - -sqlite3 db $O(db) +sqlite3 db $A(database) # Create the FTS table in the db. Return a list of the table columns. # proc create_table {} { - global O + global A set cols [list a b c d e f g h i j k l m n o p q r s t u v w x y z] - set nCol [llength $O(aColSize)] + set nCol [llength $A(colsize)] set cols [lrange $cols 0 [expr $nCol-1]] - set sql "CREATE VIRTUAL TABLE IF NOT EXISTS $O(tblname) USING $O(fts) (" + set sql "CREATE VIRTUAL TABLE IF NOT EXISTS $A(tblname) USING $A(fts) (" append sql [join $cols ,] - append sql ");" + if {$A(fts)=="fts5"} { append sql ",detail=$A(detail));" } db eval $sql return $cols @@ -89,27 +171,35 @@ proc readfile {file} { split $data } +proc repeat {L n} { + set res [list] + for {set i 0} {$i < $n} {incr i} { + set res [concat $res $L] + } + set res +} + # Load all the data into a big list of tokens. # set tokens [list] -foreach f $O(files) { - set tokens [concat $tokens [readfile $f]] +foreach f $A(file) { + set tokens [concat $tokens [repeat [readfile $f] $A(repeat)]] } set N [llength $tokens] set i 0 set cols [create_table] -set sql "INSERT INTO $O(tblname) VALUES(\$[lindex $cols 0]" +set sql "INSERT INTO $A(tblname) VALUES(\$R([lindex $cols 0])" foreach c [lrange $cols 1 end] { - append sql ", \$A($c)" + append sql ", \$R($c)" } append sql ")" db eval BEGIN while {$i < $N} { - foreach c $cols s $O(aColSize) { - set A($c) [lrange $tokens $i [expr $i+$s-1]] + foreach c $cols s $A(colsize) { + set R($c) [lrange $tokens $i [expr $i+$s-1]] incr i $s } db eval $sql diff --git a/manifest b/manifest index fbcb4d2d1b..32c610bbaf 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sfts5\sproblem\swith\susing\sboth\sxPhraseFirst()\sand\sxPhraseFirstColumn()\swithin\sa\ssingle\sstatement\sin\sdetail=col\smode. -D 2016-01-23T18:51:59.865 +C Enhance\sfts5txt2db.tcl,\sa\sscript\sused\sto\sgenerate\sfts5/fts4\sdatabases\sfor\sperformance\stesting. +D 2016-01-26T17:08:22.193 F Makefile.in 027c1603f255390c43a426671055a31c0a65fdb4 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1708a78eda223b6daa302b140037fcc214a779f9 @@ -189,7 +189,7 @@ F ext/fts5/test/fts5update.test 57c7012a7919889048947addae10e0613df45529 F ext/fts5/test/fts5version.test 978f59541d8cef7e8591f8be2115ec5ccb863e2e F ext/fts5/test/fts5vocab.test 480d780aa6b699816c5066225fbd86f3a0239477 F ext/fts5/tool/fts5speed.tcl aaee41894b552df8fbf8616aad003b2ea9ba3221 -F ext/fts5/tool/fts5txt2db.tcl c374c4c4797e8cdfadabdfaeeb5412dcd6686e84 +F ext/fts5/tool/fts5txt2db.tcl ae308338b2da1646dea456ab66706acdde8c714e F ext/fts5/tool/loadfts5.tcl 95b03429ee6b138645703c6ca192c3ac96eaf093 F ext/fts5/tool/mkfts5c.tcl d1c2a9ab8e0ec690a52316f33dd9b1d379942f45 F ext/fts5/tool/showfts5.tcl d54da0e067306663e2d5d523965ca487698e722c @@ -1419,7 +1419,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ceccc9ad788fb4da9120915741995b9f088f85ff -R e7d7753020b6f70b109a92a8ac037ba7 +P 72d53699bf0dcdb9d2a22e229989d7435f061399 +R e572f7330a1bdca73a606e620f2735b9 U dan -Z 648645363fb131dd10e5d7321a173b9d +Z 9f71d918f9bd09bf4fa550347b1ed369 diff --git a/manifest.uuid b/manifest.uuid index 1891619c18..0db14bcc12 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -72d53699bf0dcdb9d2a22e229989d7435f061399 \ No newline at end of file +c646e40350e5aa91abcf52de61fb31275bad38f9 \ No newline at end of file From 1c16bccb79888f775b4e1fbdca00d8f272afed17 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 26 Jan 2016 19:30:49 +0000 Subject: [PATCH 38/39] Improve the performance of fts5 column filters on detail=col tables. FossilOrigin-Name: 249a2d070c34bf884a04cb248b9691e239f2871c --- ext/fts5/fts5_index.c | 68 ++++++++++++++++++++++++++++++---- ext/fts5/test/fts5ac.test | 2 +- ext/fts5/test/fts5simple3.test | 44 ++++++++++++++++++++++ ext/fts5/tool/fts5speed.tcl | 2 + manifest | 17 +++++---- manifest.uuid | 2 +- 6 files changed, 117 insertions(+), 18 deletions(-) create mode 100644 ext/fts5/test/fts5simple3.test diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index e147199ef8..6d442de799 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -5003,6 +5003,53 @@ static void fts5IterSetOutputs_Nocolset(Fts5Iter *pIter, Fts5SegIter *pSeg){ } } +/* +** xSetOutputs callback used when: detail=col when there is a column filter. +** +** * detail=col, +** * there is a column filter, and +** * the table contains 32 or fewer columns. +*/ +static void fts5IterSetOutputs_Col32(Fts5Iter *pIter, Fts5SegIter *pSeg){ + Fts5Colset *pColset = pIter->pColset; + pIter->base.iRowid = pSeg->iRowid; + + assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_COLUMNS ); + assert( pColset ); + + fts5BufferZero(&pIter->poslist); + if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf ){ + int i; + int iPrev = 0; + u32 m = 0; + u8 *a = (u8*)&pSeg->pLeaf->p[pSeg->iLeafOffset]; + u8 *pEnd = (u8*)&a[pSeg->nPos]; + + while( aposlist.p; + for(i=0; inCol; i++){ + int iCol = pColset->aiCol[i]; + if( m & (1 << iCol) ){ + *a++ = (iCol - iPrev) + 2; + iPrev = iCol; + } + } + pIter->poslist.n = a - pIter->poslist.p; + + }else{ + fts5SegiterPoslist(pIter->pIndex, pSeg, pColset, &pIter->poslist); + } + + pIter->base.pData = pIter->poslist.p; + pIter->base.nData = pIter->poslist.n; +} + /* ** xSetOutputs callback used by detail=col when there is a column filter. */ @@ -5083,9 +5130,9 @@ static void fts5IterSetOutputs_Full(Fts5Iter *pIter, Fts5SegIter *pSeg){ } } -static void fts5IterSetOutputCb(Fts5Iter *pIter){ - int eDetail = pIter->pIndex->pConfig->eDetail; - if( eDetail==FTS5_DETAIL_NONE ){ +static void fts5IterSetOutputCb(int *pRc, Fts5Iter *pIter){ + Fts5Config *pConfig = pIter->pIndex->pConfig; + if( pConfig->eDetail==FTS5_DETAIL_NONE ){ pIter->xSetOutputs = fts5IterSetOutputs_None; } @@ -5093,13 +5140,18 @@ static void fts5IterSetOutputCb(Fts5Iter *pIter){ pIter->xSetOutputs = fts5IterSetOutputs_Nocolset; } - else if( eDetail==FTS5_DETAIL_FULL ){ + else if( pConfig->eDetail==FTS5_DETAIL_FULL ){ pIter->xSetOutputs = fts5IterSetOutputs_Full; } else{ - assert( eDetail==FTS5_DETAIL_COLUMNS ); - pIter->xSetOutputs = fts5IterSetOutputs_Col; + assert( pConfig->eDetail==FTS5_DETAIL_COLUMNS ); + if( pConfig->nCol<=32 ){ + pIter->xSetOutputs = fts5IterSetOutputs_Col32; + sqlite3Fts5BufferSize(pRc, &pIter->poslist, pConfig->nCol); + }else{ + pIter->xSetOutputs = fts5IterSetOutputs_Col; + } } } @@ -5166,8 +5218,8 @@ int sqlite3Fts5IndexQuery( if( p->rc==SQLITE_OK ){ Fts5SegIter *pSeg = &pRet->aSeg[pRet->aFirst[1].iFirst]; pRet->pColset = pColset; - fts5IterSetOutputCb(pRet); - if( pSeg->pLeaf ) pRet->xSetOutputs(pRet, pSeg); + fts5IterSetOutputCb(&p->rc, pRet); + if( p->rc==SQLITE_OK && pSeg->pLeaf ) pRet->xSetOutputs(pRet, pSeg); } if( p->rc ){ sqlite3Fts5IterClose(&pRet->base); diff --git a/ext/fts5/test/fts5ac.test b/ext/fts5/test/fts5ac.test index a5a522a466..61b3230772 100644 --- a/ext/fts5/test/fts5ac.test +++ b/ext/fts5/test/fts5ac.test @@ -158,8 +158,8 @@ foreach {tn2 sql} { #------------------------------------------------------------------------- # foreach {tn expr} { - 1.2 "a OR b" 1.1 "a AND b" + 1.2 "a OR b" 1.3 "o" 1.4 "b q" 1.5 "e a e" diff --git a/ext/fts5/test/fts5simple3.test b/ext/fts5/test/fts5simple3.test new file mode 100644 index 0000000000..b84a3e56dd --- /dev/null +++ b/ext/fts5/test/fts5simple3.test @@ -0,0 +1,44 @@ +# 2015 September 05 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5simple3 + +# If SQLITE_ENABLE_FTS5 is defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +fts5_aux_test_functions db + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, c, detail=col); + INSERT INTO t1 VALUES('a', 'b', 'c'); + INSERT INTO t1 VALUES('x', 'x', 'x'); +} + +do_execsql_test 1.1 { + SELECT rowid, fts5_test_collist(t1) FROM t1('a:a'); +} {1 0.0} + +do_execsql_test 1.2 { + SELECT rowid, fts5_test_collist(t1) FROM t1('b:x'); +} {2 0.1} + +do_execsql_test 1.3 { + SELECT rowid, fts5_test_collist(t1) FROM t1('b:a'); +} {} + + +finish_test + diff --git a/ext/fts5/tool/fts5speed.tcl b/ext/fts5/tool/fts5speed.tcl index 1b060ea759..8e1c7f3899 100644 --- a/ext/fts5/tool/fts5speed.tcl +++ b/ext/fts5/tool/fts5speed.tcl @@ -11,6 +11,8 @@ set Q { {1 "SELECT count(*) FROM t1 WHERE t1 MATCH 'c:t*'"} {1 "SELECT count(*) FROM t1 WHERE t1 MATCH 'a:t* OR b:t* OR c:t* OR d:t* OR e:t* OR f:t* OR g:t*'"} {1 "SELECT count(*) FROM t1 WHERE t1 MATCH 'a:t*'"} + + {2 "SELECT count(*) FROM t1 WHERE t1 MATCH 'c:the'"} } proc usage {} { diff --git a/manifest b/manifest index 32c610bbaf..2e90cb8536 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sfts5txt2db.tcl,\sa\sscript\sused\sto\sgenerate\sfts5/fts4\sdatabases\sfor\sperformance\stesting. -D 2016-01-26T17:08:22.193 +C Improve\sthe\sperformance\sof\sfts5\scolumn\sfilters\son\sdetail=col\stables. +D 2016-01-26T19:30:49.768 F Makefile.in 027c1603f255390c43a426671055a31c0a65fdb4 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1708a78eda223b6daa302b140037fcc214a779f9 @@ -103,7 +103,7 @@ F ext/fts5/fts5_buffer.c f6e0c6018ffc8e39fc0b333b5daa8b8d528ae6e4 F ext/fts5/fts5_config.c 0c384ebdd23fd055e2e50a93277b8d59da538238 F ext/fts5/fts5_expr.c a66b9694519d9c336d9bdbd46ea22e7e14aef412 F ext/fts5/fts5_hash.c 1b113977296cf4212c6ec667d5e3f2bd18036955 -F ext/fts5/fts5_index.c 722d8717e3167dd05fa48af970352932052da318 +F ext/fts5/fts5_index.c b34b7257f73929dc1b25c420aad2453dcbe36128 F ext/fts5/fts5_main.c 3886bbfc5ac1d9df29979823ddf2b68241e1127e F ext/fts5/fts5_storage.c 2a1f44deae090cd711f02cec0c2af8e660360d24 F ext/fts5/fts5_tcl.c f8731e0508299bd43f1a2eff7dbeaac870768966 @@ -118,7 +118,7 @@ F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba F ext/fts5/test/fts5_common.tcl 61ff0d1a29d98a91c4553b20b3f410d858834ee9 F ext/fts5/test/fts5aa.test 7e814df4a0e6c22a6fe2d84f210fdc0b5068a084 F ext/fts5/test/fts5ab.test 30325a89453280160106be411bba3acf138e6d1b -F ext/fts5/test/fts5ac.test dec95549e007dd9be52aa435cdcd0f08e14e64d0 +F ext/fts5/test/fts5ac.test 55cad4275a1f5acabfe14d8442a8046b47e49e5f F ext/fts5/test/fts5ad.test 0ddaa5b692ff220100ee396228838f4331399eaa F ext/fts5/test/fts5ae.test 612dcb51f4069226791ff14c17dbfb3138c56f20 F ext/fts5/test/fts5af.test be858a96b1f5de66ba6d64f0021bd8b2408e126c @@ -176,6 +176,7 @@ F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17 F ext/fts5/test/fts5rowid.test 16908a99d6efc9ba21081b4f2b86b3fc699839a6 F ext/fts5/test/fts5simple.test 2bc6451cbe887a9215f5b14ae307c70d850344c9 F ext/fts5/test/fts5simple2.test 98377ae1ff7749a42c21fe1a139c1ed312522c46 +F ext/fts5/test/fts5simple3.test e671b36bc4dbd4f5095e66cb04473cba9f680f53 F ext/fts5/test/fts5synonym.test 6475d189c2e20d60795808f83e36bf9318708d48 F ext/fts5/test/fts5synonym2.test aa4c43bd3b691ff80f658cb064f5ab40690e834e F ext/fts5/test/fts5tok1.test beb894c6f3468f10a574302f69ebe4436b0287c7 @@ -188,7 +189,7 @@ F ext/fts5/test/fts5unindexed.test e9539d5b78c677315e7ed8ea911d4fd25437c680 F ext/fts5/test/fts5update.test 57c7012a7919889048947addae10e0613df45529 F ext/fts5/test/fts5version.test 978f59541d8cef7e8591f8be2115ec5ccb863e2e F ext/fts5/test/fts5vocab.test 480d780aa6b699816c5066225fbd86f3a0239477 -F ext/fts5/tool/fts5speed.tcl aaee41894b552df8fbf8616aad003b2ea9ba3221 +F ext/fts5/tool/fts5speed.tcl 47f0031e6ac564964f4f4805e439ea665e848df2 F ext/fts5/tool/fts5txt2db.tcl ae308338b2da1646dea456ab66706acdde8c714e F ext/fts5/tool/loadfts5.tcl 95b03429ee6b138645703c6ca192c3ac96eaf093 F ext/fts5/tool/mkfts5c.tcl d1c2a9ab8e0ec690a52316f33dd9b1d379942f45 @@ -1419,7 +1420,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 72d53699bf0dcdb9d2a22e229989d7435f061399 -R e572f7330a1bdca73a606e620f2735b9 +P c646e40350e5aa91abcf52de61fb31275bad38f9 +R a44979856d96ea38951595ad6e15598a U dan -Z 9f71d918f9bd09bf4fa550347b1ed369 +Z a6d46df0ea079f8f36131bee8e2cf5b7 diff --git a/manifest.uuid b/manifest.uuid index 0db14bcc12..54ccb7acb1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c646e40350e5aa91abcf52de61fb31275bad38f9 \ No newline at end of file +249a2d070c34bf884a04cb248b9691e239f2871c \ No newline at end of file From 18205c781d4bb15d5a26801844212f231a4aed9d Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 26 Jan 2016 20:08:50 +0000 Subject: [PATCH 39/39] Further minor performance improvements and code-size reductions related to fts5 column filters on detail=col tables. FossilOrigin-Name: b4ac61aeee976296e7719949cd4fb496147a29e8 --- ext/fts5/fts5_index.c | 119 +++++++++++++++--------------------------- manifest | 12 ++--- manifest.uuid | 2 +- 3 files changed, 50 insertions(+), 83 deletions(-) diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 6d442de799..ca3feb86f7 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -5004,96 +5004,63 @@ static void fts5IterSetOutputs_Nocolset(Fts5Iter *pIter, Fts5SegIter *pSeg){ } /* -** xSetOutputs callback used when: detail=col when there is a column filter. -** -** * detail=col, -** * there is a column filter, and -** * the table contains 32 or fewer columns. +** xSetOutputs callback used by detail=col when there is a column filter +** and there are 100 or more columns. Also called as a fallback from +** fts5IterSetOutputs_Col100 if the column-list spans more than one page. */ -static void fts5IterSetOutputs_Col32(Fts5Iter *pIter, Fts5SegIter *pSeg){ - Fts5Colset *pColset = pIter->pColset; - pIter->base.iRowid = pSeg->iRowid; - - assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_COLUMNS ); - assert( pColset ); - +static void fts5IterSetOutputs_Col(Fts5Iter *pIter, Fts5SegIter *pSeg){ fts5BufferZero(&pIter->poslist); - if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf ){ - int i; - int iPrev = 0; - u32 m = 0; - u8 *a = (u8*)&pSeg->pLeaf->p[pSeg->iLeafOffset]; - u8 *pEnd = (u8*)&a[pSeg->nPos]; - - while( aposlist.p; - for(i=0; inCol; i++){ - int iCol = pColset->aiCol[i]; - if( m & (1 << iCol) ){ - *a++ = (iCol - iPrev) + 2; - iPrev = iCol; - } - } - pIter->poslist.n = a - pIter->poslist.p; - - }else{ - fts5SegiterPoslist(pIter->pIndex, pSeg, pColset, &pIter->poslist); - } - + fts5SegiterPoslist(pIter->pIndex, pSeg, pIter->pColset, &pIter->poslist); + pIter->base.iRowid = pSeg->iRowid; pIter->base.pData = pIter->poslist.p; pIter->base.nData = pIter->poslist.n; } /* -** xSetOutputs callback used by detail=col when there is a column filter. +** xSetOutputs callback used when: +** +** * detail=col, +** * there is a column filter, and +** * the table contains 100 or fewer columns. +** +** The last point is to ensure all column numbers are stored as +** single-byte varints. */ -static void fts5IterSetOutputs_Col(Fts5Iter *pIter, Fts5SegIter *pSeg){ - Fts5Colset *pColset = pIter->pColset; - pIter->base.iRowid = pSeg->iRowid; +static void fts5IterSetOutputs_Col100(Fts5Iter *pIter, Fts5SegIter *pSeg){ assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_COLUMNS ); - assert( pColset ); + assert( pIter->pColset ); - if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf ){ - /* All data is stored on the current page. Populate the output - ** variables to point into the body of the page object. */ - Fts5PoslistWriter writer = {0}; - const u8 *a = &pSeg->pLeaf->p[pSeg->iLeafOffset]; - int n = pSeg->nPos; - int iCol = 0; - int iCVal = pColset->aiCol[0]; - i64 iPos = 0; - int iOff = 0; + if( pSeg->iLeafOffset+pSeg->nPos>pSeg->pLeaf->szLeaf ){ + fts5IterSetOutputs_Col(pIter, pSeg); + }else{ + u8 *a = (u8*)&pSeg->pLeaf->p[pSeg->iLeafOffset]; + u8 *pEnd = (u8*)&a[pSeg->nPos]; + int iPrev = 0; + int *aiCol = pIter->pColset->aiCol; + int *aiColEnd = &aiCol[pIter->pColset->nCol]; - fts5BufferZero(&pIter->poslist); - while( 0==sqlite3Fts5PoslistNext64(a, n, &iOff, &iPos) ){ - while( iPos>=iCVal ){ - if( iPos==iCVal ){ - sqlite3Fts5PoslistWriterAppend(&pIter->poslist, &writer, iPos); - } - if( ++iCol>=pColset->nCol ) goto setoutputs_col_out; - assert( pColset->aiCol[iCol]>iCVal ); - iCVal = pColset->aiCol[iCol]; + u8 *aOut = pIter->poslist.p; + int iPrevOut = 0; + + pIter->base.iRowid = pSeg->iRowid; + + while( aposlist); - fts5SegiterPoslist(pIter->pIndex, pSeg, pColset, &pIter->poslist); - } - setoutputs_col_out: - pIter->base.pData = pIter->poslist.p; - pIter->base.nData = pIter->poslist.n; + pIter->base.pData = pIter->poslist.p; + pIter->base.nData = aOut - pIter->poslist.p; + } } /* @@ -5146,8 +5113,8 @@ static void fts5IterSetOutputCb(int *pRc, Fts5Iter *pIter){ else{ assert( pConfig->eDetail==FTS5_DETAIL_COLUMNS ); - if( pConfig->nCol<=32 ){ - pIter->xSetOutputs = fts5IterSetOutputs_Col32; + if( pConfig->nCol<=100 ){ + pIter->xSetOutputs = fts5IterSetOutputs_Col100; sqlite3Fts5BufferSize(pRc, &pIter->poslist, pConfig->nCol); }else{ pIter->xSetOutputs = fts5IterSetOutputs_Col; diff --git a/manifest b/manifest index 2e90cb8536..173bd3a03a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improve\sthe\sperformance\sof\sfts5\scolumn\sfilters\son\sdetail=col\stables. -D 2016-01-26T19:30:49.768 +C Further\sminor\sperformance\simprovements\sand\scode-size\sreductions\srelated\sto\sfts5\scolumn\sfilters\son\sdetail=col\stables. +D 2016-01-26T20:08:50.348 F Makefile.in 027c1603f255390c43a426671055a31c0a65fdb4 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1708a78eda223b6daa302b140037fcc214a779f9 @@ -103,7 +103,7 @@ F ext/fts5/fts5_buffer.c f6e0c6018ffc8e39fc0b333b5daa8b8d528ae6e4 F ext/fts5/fts5_config.c 0c384ebdd23fd055e2e50a93277b8d59da538238 F ext/fts5/fts5_expr.c a66b9694519d9c336d9bdbd46ea22e7e14aef412 F ext/fts5/fts5_hash.c 1b113977296cf4212c6ec667d5e3f2bd18036955 -F ext/fts5/fts5_index.c b34b7257f73929dc1b25c420aad2453dcbe36128 +F ext/fts5/fts5_index.c 5558bfbeaf364cc67f937e25753ceed8757cb6d1 F ext/fts5/fts5_main.c 3886bbfc5ac1d9df29979823ddf2b68241e1127e F ext/fts5/fts5_storage.c 2a1f44deae090cd711f02cec0c2af8e660360d24 F ext/fts5/fts5_tcl.c f8731e0508299bd43f1a2eff7dbeaac870768966 @@ -1420,7 +1420,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P c646e40350e5aa91abcf52de61fb31275bad38f9 -R a44979856d96ea38951595ad6e15598a +P 249a2d070c34bf884a04cb248b9691e239f2871c +R a28f5f4bedd812396ab85ff0aefd35c4 U dan -Z a6d46df0ea079f8f36131bee8e2cf5b7 +Z 56fb7a3fddaab518f0c4eb551a124662 diff --git a/manifest.uuid b/manifest.uuid index 54ccb7acb1..708359e9ec 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -249a2d070c34bf884a04cb248b9691e239f2871c \ No newline at end of file +b4ac61aeee976296e7719949cd4fb496147a29e8 \ No newline at end of file