From 1d9db01065d3d855d33c257711ef40486b122885 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 23 Sep 2024 20:23:43 +0000 Subject: [PATCH] CLI uses only lib-c for I/O on Windows. No calls to Win32. Works on Win11, at least. Reads and writes unicode to/from the console and UTF-8 to/from files. Prototype code only - must testing and additional work required. FossilOrigin-Name: 5c54530d5a0a4125a1ba44f22537c4f63d5e5708f347c43cbac3e1832c4335da --- Makefile.in | 2 - Makefile.msc | 2 - main.mk | 2 - manifest | 21 +++--- manifest.uuid | 2 +- src/shell.c.in | 190 ++++++++++++++++++++++++++++--------------------- 6 files changed, 122 insertions(+), 97 deletions(-) diff --git a/Makefile.in b/Makefile.in index dba80d38fb..7b693b5225 100644 --- a/Makefile.in +++ b/Makefile.in @@ -1187,8 +1187,6 @@ keywordhash.h: $(TOP)/tool/mkkeywordhash.c # Source and header files that shell.c depends on SHELL_DEP = \ $(TOP)/src/shell.c.in \ - $(TOP)/ext/consio/console_io.c \ - $(TOP)/ext/consio/console_io.h \ $(TOP)/ext/expert/sqlite3expert.c \ $(TOP)/ext/expert/sqlite3expert.h \ $(TOP)/ext/intck/sqlite3intck.c \ diff --git a/Makefile.msc b/Makefile.msc index 434c996608..bb63601518 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -2316,8 +2316,6 @@ keywordhash.h: $(TOP)\tool\mkkeywordhash.c mkkeywordhash.exe # Source and header files that shell.c depends on SHELL_DEP = \ $(TOP)\src\shell.c.in \ - $(TOP)\ext\consio\console_io.c \ - $(TOP)\ext\consio\console_io.h \ $(TOP)\ext\expert\sqlite3expert.c \ $(TOP)\ext\expert\sqlite3expert.h \ $(TOP)\ext\intck\sqlite3intck.c \ diff --git a/main.mk b/main.mk index 79e345d1b8..8aaa7e7d06 100644 --- a/main.mk +++ b/main.mk @@ -763,8 +763,6 @@ keywordhash.h: $(TOP)/tool/mkkeywordhash.c # Source and header files that shell.c depends on SHELL_DEP = \ $(TOP)/src/shell.c.in \ - $(TOP)/ext/consio/console_io.c \ - $(TOP)/ext/consio/console_io.h \ $(TOP)/ext/expert/sqlite3expert.c \ $(TOP)/ext/expert/sqlite3expert.h \ $(TOP)/ext/intck/sqlite3intck.c \ diff --git a/manifest b/manifest index ca881335f6..8b26ee7843 100644 --- a/manifest +++ b/manifest @@ -1,11 +1,11 @@ -C Add\sthe\srun-fuzzcheck\starget\sto\sthe\sMSVC\smakefile. -D 2024-09-21T17:27:47.017 +C CLI\suses\sonly\slib-c\sfor\sI/O\son\sWindows.\s\sNo\scalls\sto\sWin32.\s\sWorks\son\sWin11,\nat\sleast.\s\sReads\sand\swrites\sunicode\sto/from\sthe\sconsole\sand\sUTF-8\sto/from\sfiles.\nPrototype\scode\sonly\s-\smust\stesting\sand\sadditional\swork\srequired. +D 2024-09-23T20:23:43.341 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 -F Makefile.in fa448c4c0567623fd140efebecb570ab58d955d766a5ea0fd8a94e9b5697007c +F Makefile.in aa594119c3c7f699e87a767fca6598452f77d4c32c41a6486c40d4f156d4efc1 F Makefile.linux-gcc f3842a0b1efbfbb74ac0ef60e56b301836d05b4d867d014f714fa750048f1ab6 -F Makefile.msc e3c4723c27464acc31da4420b808c8d2690180ba2b915897bece0a9d5d2cecf6 +F Makefile.msc add7e29bae33ad5b8c464daf6de84a1a01e31e6337d79fb0c1062e53fa7657da F README.md c3c0f19532ce28f6297a71870f3c7b424729f0e6d9ab889616d3587dd2332159 F VERSION 0db40f92c04378404eb45bff93e9e42c148c7e54fd3da99469ed21e22411f5a6 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -687,7 +687,7 @@ F ext/wasm/wasmfs.make 8a4955882aaa0783b3f60a9484a1f0f3d8b6f775c0fcd17c082f31966 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 5ade0bc977aa135e79e3faaea894d5671b26107cc91e70783aa7dc83f22f3ba0 -F main.mk 67e622f31d10fee8f0f62655b4f9b47cd97fe70a125674ca6754b3549d69cc0e +F main.mk b897586c0c7b77b7e39f0a0e9ed79fed7346b09af1ed35a08da745e02b795772 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 F mptest/crash01.test 61e61469e257df0850df4293d7d4d6c2af301421 @@ -768,7 +768,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c b2cd748488012312824508639b6af908461e45403037d5c4e19d9b0e8195507f F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 F src/select.c 4b14337a2742f0c0beeba490e9a05507e9b4b12184b9cd12773501d08d48e3fe -F src/shell.c.in 375f8a183126be96ec73db4e42c57917ff10a0900846b1b722dd4f8cef537812 +F src/shell.c.in 265c877932142ee8ef05a6aa5a0a5bff92905ffef97dc6f566062a27814274a1 F src/sqlite.h.in 77f55bd1978a04a14db211732f0a609077cf60ba4ccf9baf39988f508945419c F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54 @@ -2213,8 +2213,11 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 62e11a3a78edf9853b74d6495ccd8ae9ac1966c7d78eb3682cf2d5885e3740ec -R 44ce456c4e740a869984e636809287df +P 2e5194407a1b34dd0659c350ea8098bfef7b3f11aa5b2a07ecd2bce5582655a2 +R 7c20c2fd1880ef1b062b32c88da2e8e4 +T *branch * cli-stdlib +T *sym-cli-stdlib * +T -sym-trunk * U drh -Z fb345da7ed81e5875e018c3667e473a3 +Z cdf0bc25a30881c9fe4f62b1ff96af22 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index c7adf060ec..7417271d4a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2e5194407a1b34dd0659c350ea8098bfef7b3f11aa5b2a07ecd2bce5582655a2 +5c54530d5a0a4125a1ba44f22537c4f63d5e5708f347c43cbac3e1832c4335da diff --git a/src/shell.c.in b/src/shell.c.in index 2b0e506ed8..a2b22845d6 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -237,6 +237,57 @@ extern char *sqlite3_win32_unicode_to_utf8(LPCWSTR); extern LPWSTR sqlite3_win32_utf8_to_unicode(const char *zText); #endif +#ifdef _WIN32 +/* On Windows, we normally run with output mode of TEXT so that \n characters +** are automatically translated into \r\n. However, this behavior needs +** to be disabled in some cases (ex: when generating CSV output and when +** rendering quoted strings that contain \n characters). The following +** routines take care of that. +*/ +static void setBinaryMode(FILE *file, int isOutput){ + if( isOutput ) fflush(file); + _setmode(_fileno(file), _O_BINARY); +} +static void setTextMode(FILE *file, int isOutput){ + if( isOutput ) fflush(file); + _setmode(_fileno(file), _O_TEXT); +} +#else + /* Unix equivalents to set*Mode() */ +# define setBinaryMode(X,Y) +# define setTextMode(X,Y) +#endif + +#ifdef _WIN32 +/* fgets() for windows */ +static char *cli_fgets(char *buf, int sz, FILE *in){ + if( isatty(_fileno(in)) ){ + /* When reading from the command-prompt in Windows, it is necessary + ** to use _O_WTEXT input mode to read UTF-16 characters, then translate + ** that into UTF-8. Otherwise, non-ASCII characters all get translated + ** into '?'. + */ + wchar_t *b1 = malloc( sz*sizeof(wchar_t) ); + if( b1==0 ) return 0; + _setmode(_fileno(in), _O_WTEXT); + if( fgetws(b1, sz/4, in)==0 ){ + sqlite3_free(b1); + return 0; + } + WideCharToMultiByte(CP_UTF8, 0, b1, -1, buf, sz, 0, 0); + sqlite3_free(b1); + return buf; + }else{ + /* Reading from a file or other input source, just read bytes without + ** any translation. */ + return fgets(buf, sz, in); + } +} +#else +/* library version works for everybody else */ +# define cli_fgets fgets +#endif + /* Use console I/O package as a direct INCLUDE. */ #define SQLITE_INTERNAL_LINKAGE static @@ -248,57 +299,15 @@ extern LPWSTR sqlite3_win32_utf8_to_unicode(const char *zText); # define SQLITE_CIO_NO_SETMODE # define SQLITE_CIO_NO_FLUSH #endif -INCLUDE ../ext/consio/console_io.h -INCLUDE ../ext/consio/console_io.c -#ifndef SQLITE_SHELL_FIDDLE - -/* From here onward, fgets() is redirected to the console_io library. */ -# define fgets(b,n,f) fGetsUtf8(b,n,f) -/* - * Define macros for emitting output text in various ways: - * sputz(s, z) => emit 0-terminated string z to given stream s - * sputf(s, f, ...) => emit varargs per format f to given stream s - * oputz(z) => emit 0-terminated string z to default stream - * oputf(f, ...) => emit varargs per format f to default stream - * eputz(z) => emit 0-terminated string z to error stream - * eputf(f, ...) => emit varargs per format f to error stream - * oputb(b, n) => emit char buffer b[0..n-1] to default stream - * - * Note that the default stream is whatever has been last set via: - * setOutputStream(FILE *pf) - * This is normally the stream that CLI normal output goes to. - * For the stand-alone CLI, it is stdout with no .output redirect. - * - * The ?putz(z) forms are required for the Fiddle builds for string literal - * output, in aid of enforcing format string to argument correspondence. - */ -# define sputz(s,z) fPutsUtf8(z,s) -# define sputf fPrintfUtf8 -# define oputz(z) oPutsUtf8(z) -# define oputf oPrintfUtf8 -# define eputz(z) ePutsUtf8(z) -# define eputf ePrintfUtf8 -# define oputb(buf,na) oPutbUtf8(buf,na) -# define fflush(s) fFlushBuffer(s); - -#else -/* For Fiddle, all console handling and emit redirection is omitted. */ -/* These next 3 macros are for emitting formatted output. When complaints - * from the WASM build are issued for non-formatted output, when a mere - * string literal is to be emitted, the ?putz(z) forms should be used. - * (This permits compile-time checking of format string / argument mismatch.) - */ -# define oputf(fmt, ...) printf(fmt,__VA_ARGS__) -# define eputf(fmt, ...) fprintf(stderr,fmt,__VA_ARGS__) -# define sputf(fp,fmt, ...) fprintf(fp,fmt,__VA_ARGS__) +#define oputf(fmt, ...) printf(fmt,__VA_ARGS__) +#define eputf(fmt, ...) fprintf(stderr,fmt,__VA_ARGS__) +#define sputf(fp,fmt, ...) fprintf(fp,fmt,__VA_ARGS__) /* These next 3 macros are for emitting simple string literals. */ -# define oputz(z) fputs(z,stdout) -# define eputz(z) fputs(z,stderr) -# define sputz(fp,z) fputs(z,fp) -# define oputb(buf,na) fwrite(buf,1,na,stdout) -# undef fflush -#endif +#define oputz(z) fputs(z,stdout) +#define eputz(z) fputs(z,stderr) +#define sputz(fp,z) fputs(z,fp) +#define oputb(buf,na) fwrite(buf,1,na,stdout) /* True if the timer is enabled */ static int enableTimer = 0; @@ -344,6 +353,7 @@ struct rusage { #define getrusage(A,B) memset(B,0,sizeof(*B)) #endif + /* Saved resource information for the beginning of an operation */ static struct rusage sBegin; /* CPU time at start */ static sqlite3_int64 iBegin; /* Wall-clock time at start */ @@ -802,7 +812,7 @@ static char *local_getline(char *zLine, FILE *in){ zLine = realloc(zLine, nLine); shell_check_oom(zLine); } - if( fgets(&zLine[n], nLine - n, in)==0 ){ + if( cli_fgets(&zLine[n], nLine - n, in)==0 ){ if( n==0 ){ free(zLine); return 0; @@ -1766,10 +1776,7 @@ static const char *unused_string( static void output_quoted_string(const char *z){ int i; char c; -#ifndef SQLITE_SHELL_FIDDLE - FILE *pfO = setOutputStream(invalidFileStream); - setBinaryMode(pfO, 1); -#endif + setBinaryMode(stdout, 1); if( z==0 ) return; for(i=0; (c = z[i])!=0 && c!='\''; i++){} if( c==0 ){ @@ -1794,11 +1801,7 @@ static void output_quoted_string(const char *z){ } oputz("'"); } -#ifndef SQLITE_SHELL_FIDDLE - setTextMode(pfO, 1); -#else setTextMode(stdout, 1); -#endif } /* @@ -1813,10 +1816,7 @@ static void output_quoted_string(const char *z){ static void output_quoted_escaped_string(const char *z){ int i; char c; -#ifndef SQLITE_SHELL_FIDDLE - FILE *pfO = setOutputStream(invalidFileStream); - setBinaryMode(pfO, 1); -#endif + setBinaryMode(stdout, 1); for(i=0; (c = z[i])!=0 && c!='\'' && c!='\n' && c!='\r'; i++){} if( c==0 ){ oputf("'%s'",z); @@ -1868,11 +1868,7 @@ static void output_quoted_escaped_string(const char *z){ oputf(",'%s',char(10))", zNL); } } -#ifndef SQLITE_SHELL_FIDDLE - setTextMode(pfO, 1); -#else setTextMode(stdout, 1); -#endif } /* @@ -1892,6 +1888,42 @@ static const char *anyOfInStr(const char *s, const char *zAny, size_t ns){ } return pcFirst; } + +/* Skip over as much z[] input char sequence as is valid UTF-8, +** limited per nAccept char's or whole characters and containing +** no char cn such that ((1<=0 => char count, nAccept<0 => character + */ +const char *zSkipValidUtf8(const char *z, int nAccept, long ccm){ + int ng = (nAccept<0)? -nAccept : 0; + const char *pcLimit = (nAccept>=0)? z+nAccept : 0; + assert(z!=0); + while( (pcLimit)? (z= pcLimit ) return z; + else{ + char ct = *zt++; + if( ct==0 || (zt-z)>4 || (ct & 0xC0)!=0x80 ){ + /* Trailing bytes are too few, too many, or invalid. */ + return z; + } + } + } while( ((c <<= 1) & 0x40) == 0x40 ); /* Eat lead byte's count. */ + z = zt; + } + } + return z; +} + + /* ** Output the given string as a quoted according to C or TCL quoting rules. */ @@ -3027,7 +3059,7 @@ static void displayLinuxIoStats(void){ sqlite3_snprintf(sizeof(z), z, "/proc/%d/io", getpid()); in = fopen(z, "rb"); if( in==0 ) return; - while( fgets(z, sizeof(z), in)!=0 ){ + while( cli_fgets(z, sizeof(z), in)!=0 ){ static const struct { const char *zPattern; const char *zDesc; @@ -5238,7 +5270,7 @@ static unsigned char *readHexDb(ShellState *p, int *pnData){ } *pnData = 0; nLine++; - if( fgets(zLine, sizeof(zLine), in)==0 ) goto readHexDb_error; + if( cli_fgets(zLine, sizeof(zLine), in)==0 ) goto readHexDb_error; rc = sscanf(zLine, "| size %d pagesize %d", &n, &pgsz); if( rc!=2 ) goto readHexDb_error; if( n<0 ) goto readHexDb_error; @@ -5251,7 +5283,7 @@ static unsigned char *readHexDb(ShellState *p, int *pnData){ eputz("invalid pagesize\n"); goto readHexDb_error; } - for(nLine++; fgets(zLine, sizeof(zLine), in)!=0; nLine++){ + for(nLine++; cli_fgets(zLine, sizeof(zLine), in)!=0; nLine++){ rc = sscanf(zLine, "| page %d offset %d", &j, &k); if( rc==2 ){ iOffset = k; @@ -5283,7 +5315,7 @@ readHexDb_error: if( in!=p->in ){ fclose(in); }else{ - while( fgets(zLine, sizeof(zLine), p->in)!=0 ){ + while( cli_fgets(zLine, sizeof(zLine), p->in)!=0 ){ nLine++; if(cli_strncmp(zLine, "| end ", 6)==0 ) break; } @@ -6177,10 +6209,10 @@ static void tryToClone(ShellState *p, const char *zNewDb){ ** Change the output stream (file or pipe or console) to something else. */ static void output_redir(ShellState *p, FILE *pfNew){ - if( p->out != stdout ) eputz("Output already redirected.\n"); - else{ + if( p->out != stdout ){ + eputz("Output already redirected.\n"); + }else{ p->out = pfNew; - setOutputStream(pfNew); } } @@ -6226,7 +6258,6 @@ static void output_reset(ShellState *p){ } p->outfile[0] = 0; p->out = stdout; - setOutputStream(stdout); } #else # define output_redir(SS,pfO) @@ -10719,9 +10750,9 @@ static int do_meta_command(char *zLine, ShellState *p){ zCmd = sqlite3_mprintf(strchr(azArg[i],' ')==0?"%z %s":"%z \"%s\"", zCmd, azArg[i]); } - consoleRestore(); + /*consoleRestore();*/ x = zCmd!=0 ? system(zCmd) : 1; - consoleRenewSetup(); + /*consoleRenewSetup();*/ sqlite3_free(zCmd); if( x ) eputf("System command returns %d\n", x); }else @@ -12374,7 +12405,6 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ # define data shellState #else ShellState data; - StreamsAreConsole consStreams = SAC_NoConsole; #endif const char *zInitFile = 0; int i; @@ -12397,10 +12427,8 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ stdout_is_console = 1; data.wasm.zDefaultDbName = "/fiddle.sqlite3"; #else - consStreams = consoleClassifySetup(stdin, stdout, stderr); - stdin_is_interactive = (consStreams & SAC_InConsole)!=0; - stdout_is_console = (consStreams & SAC_OutConsole)!=0; - atexit(consoleRestore); + stdin_is_interactive = isatty(0); + stdout_is_console = isatty(1); #endif atexit(sayAbnormalExit); #ifdef SQLITE_DEBUG