From 9113c87ef3ee81a19088dad11ab556201c88560d Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 16 Aug 2022 00:04:40 +0000 Subject: [PATCH 1/4] Replace the RC4-based PRNG with one based on ChaCha20. 3x faster. FossilOrigin-Name: 084d8776fa95c75440530028171c56547a341c9a952ba2f29bb533b538603c78 --- manifest | 17 ++++++----- manifest.uuid | 2 +- src/random.c | 83 ++++++++++++++++++++++++++++++++------------------- 3 files changed, 63 insertions(+), 39 deletions(-) diff --git a/manifest b/manifest index bb54c41f76..28553e1e9b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Extra\stest\sfor\sSQLITE_MAX_COMPOUND_SELECT. -D 2022-08-15T19:23:15.090 +C Replace\sthe\sRC4-based\sPRNG\swith\sone\sbased\son\sChaCha20.\s\s3x\sfaster. +D 2022-08-16T00:04:40.434 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -585,7 +585,7 @@ F src/pragma.c 6637d624c37a8909d3edfa9d7cf694d79b49d2a0827d8c52ef15dceb641783fa F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 F src/prepare.c c62820c15dcb63013519c8e41d9f928d7478672cc902cfd0581c733c271dbf45 F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 -F src/random.c 097dc8b31b8fba5a9aca1697aeb9fd82078ec91be734c16bffda620ced7ab83c +F src/random.c ff4b47317d22c9b6af581e25cb9243190082c57daaf175ae93c4adaa223a1e5e F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 4750fbe9d8ecb7236baf7a9bea4299bb87126e08c209645666a0ae8f0efbe0fc @@ -1999,8 +1999,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6170e638ebeb12cc40c3247324237978401f701afc270de88ad03e183e82cefc -R 5fdd415975dfc9f1efaf302c20079f22 -U dan -Z 538d1647acf283a5b618d8240e7b7cf4 +P c271096736889530f392ff37e631a77f3bc9c46b290dbda245fa05249f4410fc +R 169c772b3aaee1590ba8a07f447292d5 +T *branch * chacha20-prng +T *sym-chacha20-prng * +T -sym-trunk * +U drh +Z 1e027b4a09350d19e88468e502211941 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 800ff0f5cb..3e2548ba94 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c271096736889530f392ff37e631a77f3bc9c46b290dbda245fa05249f4410fc \ No newline at end of file +084d8776fa95c75440530028171c56547a341c9a952ba2f29bb533b538603c78 \ No newline at end of file diff --git a/src/random.c b/src/random.c index 87f9e2cecb..11c5a42b6d 100644 --- a/src/random.c +++ b/src/random.c @@ -22,16 +22,38 @@ ** This structure is the current state of the generator. */ static SQLITE_WSD struct sqlite3PrngType { - unsigned char isInit; /* True if initialized */ - unsigned char i, j; /* State variables */ - unsigned char s[256]; /* State variables */ + u32 s[16]; /* 64 bytes of chacha20 state */ + u8 out[64]; /* Output bytes */ + u8 n; /* Output bytes remaining */ } sqlite3Prng; +#define ROTL(a,b) (((a) << (b)) | ((a) >> (32 - (b)))) +#define QR(a, b, c, d) ( \ + a += b, d ^= a, d = ROTL(d,16), \ + c += d, b ^= c, b = ROTL(b,12), \ + a += b, d ^= a, d = ROTL(d, 8), \ + c += d, b ^= c, b = ROTL(b, 7)) +static void chacha_block(u32 *out, const u32 *in){ + int i; + u32 x[16]; + memcpy(x, in, 64); + for(i=0; i<10; i++){ + QR(x[0], x[4], x[ 8], x[12]); + QR(x[1], x[5], x[ 9], x[13]); + QR(x[2], x[6], x[10], x[14]); + QR(x[3], x[7], x[11], x[15]); + QR(x[0], x[5], x[10], x[15]); + QR(x[1], x[6], x[11], x[12]); + QR(x[2], x[7], x[ 8], x[13]); + QR(x[3], x[4], x[ 9], x[14]); + } + for(i=0; i<16; i++) out[i] = x[i]+in[i]; +} + /* ** Return N random bytes. */ void sqlite3_randomness(int N, void *pBuf){ - unsigned char t; unsigned char *zBuf = pBuf; /* The "wsdPrng" macro will resolve to the pseudo-random number generator @@ -61,7 +83,7 @@ void sqlite3_randomness(int N, void *pBuf){ sqlite3_mutex_enter(mutex); if( N<=0 || pBuf==0 ){ - wsdPrng.isInit = 0; + wsdPrng.s[0] = 0; sqlite3_mutex_leave(mutex); return; } @@ -75,39 +97,38 @@ void sqlite3_randomness(int N, void *pBuf){ ** encryption. The RC4 algorithm is being used as a PRNG (pseudo-random ** number generator) not as an encryption device. */ - if( !wsdPrng.isInit ){ + if( wsdPrng.s[0]==0 ){ sqlite3_vfs *pVfs = sqlite3_vfs_find(0); - int i; - char k[256]; - wsdPrng.j = 0; - wsdPrng.i = 0; + static const u32 chacha20_init[] = { + 0x61707865, 0x3320646e, 0x79622d32, 0x6b206574 + }; + memcpy(&wsdPrng.s[0], chacha20_init, 16); if( NEVER(pVfs==0) ){ - memset(k, 0, sizeof(k)); + memset(&wsdPrng.s[4], 0, 44); }else{ - sqlite3OsRandomness(pVfs, 256, k); + sqlite3OsRandomness(pVfs, 44, (char*)&wsdPrng.s[4]); } - for(i=0; i<256; i++){ - wsdPrng.s[i] = (u8)i; - } - for(i=0; i<256; i++){ - wsdPrng.j += wsdPrng.s[i] + k[i]; - t = wsdPrng.s[wsdPrng.j]; - wsdPrng.s[wsdPrng.j] = wsdPrng.s[i]; - wsdPrng.s[i] = t; - } - wsdPrng.isInit = 1; + wsdPrng.s[16] = wsdPrng.s[12]; + wsdPrng.s[12] = 0; + wsdPrng.n = 0; } assert( N>0 ); - do{ - wsdPrng.i++; - t = wsdPrng.s[wsdPrng.i]; - wsdPrng.j += t; - wsdPrng.s[wsdPrng.i] = wsdPrng.s[wsdPrng.j]; - wsdPrng.s[wsdPrng.j] = t; - t += wsdPrng.s[wsdPrng.i]; - *(zBuf++) = wsdPrng.s[t]; - }while( --N ); + while( 1 /* exit by break */ ){ + if( N<=wsdPrng.n ){ + memcpy(zBuf, &wsdPrng.out[wsdPrng.n-N], N); + wsdPrng.n -= N; + break; + } + if( wsdPrng.n>0 ){ + memcpy(zBuf, wsdPrng.out, wsdPrng.n); + N -= wsdPrng.n; + zBuf += wsdPrng.n; + } + wsdPrng.s[12]++; + chacha_block((u32*)wsdPrng.out, wsdPrng.s); + wsdPrng.n = 64; + } sqlite3_mutex_leave(mutex); } From 0f201fccc0b113e98641623a1ba7b25c42fbf346 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 16 Aug 2022 14:09:51 +0000 Subject: [PATCH 2/4] Omit an unnecessary test case from corruptN.test that does not work with the new RFC-7539 PRNG. FossilOrigin-Name: d9e8c65ed25c4f6222c737f4244d5362dbe433d357f7d133765157446cf4e925 --- manifest | 15 ++++++--------- manifest.uuid | 2 +- test/corruptN.test | 15 +++++++++------ 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 28553e1e9b..376b7f52af 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Replace\sthe\sRC4-based\sPRNG\swith\sone\sbased\son\sChaCha20.\s\s3x\sfaster. -D 2022-08-16T00:04:40.434 +C Omit\san\sunnecessary\stest\scase\sfrom\scorruptN.test\sthat\sdoes\snot\swork\swith\sthe\nnew\sRFC-7539\sPRNG. +D 2022-08-16T14:09:51.629 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -854,7 +854,7 @@ F test/corruptJ.test 4d5ccc4bf959464229a836d60142831ef76a5aa4 F test/corruptK.test 5b4212fe346699831c5ad559a62c54e11c0611bdde1ea8423a091f9c01aa32af F test/corruptL.test ecce40d7b9b909a670a42a45d86e30d927735d7e7f09041af438b19529d35532 F test/corruptM.test 7d574320e08c1b36caa3e47262061f186367d593a7e305d35f15289cc2c3e067 -F test/corruptN.test 60b5a62944b4f0029ba07edaa5fd8e670539d6b0a8d99db26c068d435675cbfe +F test/corruptN.test 7c099d153a554001b4fb829c799b01f2ea6276cbc32479131e0db0da4efd9cc4 F test/cost.test b11cdbf9f11ffe8ef99c9881bf390e61fe92baf2182bad1dbe6de59a7295c576 F test/count.test cd4bd531066e8d77ef8fe1e3fc8253d042072e117ccab214b290cf83f1602249 F test/countofview.test e17d6e6688cf74f22783c9ec6e788c0790ee4fbbaee713affd00b1ac0bb39b86 @@ -1999,11 +1999,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c271096736889530f392ff37e631a77f3bc9c46b290dbda245fa05249f4410fc -R 169c772b3aaee1590ba8a07f447292d5 -T *branch * chacha20-prng -T *sym-chacha20-prng * -T -sym-trunk * +P 084d8776fa95c75440530028171c56547a341c9a952ba2f29bb533b538603c78 +R 4a1be98000c410a528d0befc32c8b6da U drh -Z 1e027b4a09350d19e88468e502211941 +Z af7ba7ff2c1330aa966ddca9675bb3a2 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 3e2548ba94..63acf48a4f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -084d8776fa95c75440530028171c56547a341c9a952ba2f29bb533b538603c78 \ No newline at end of file +d9e8c65ed25c4f6222c737f4244d5362dbe433d357f7d133765157446cf4e925 \ No newline at end of file diff --git a/test/corruptN.test b/test/corruptN.test index 4685788223..376325f5c5 100644 --- a/test/corruptN.test +++ b/test/corruptN.test @@ -141,12 +141,15 @@ do_test 2.0 { | end c-b92b.txt.db }]} {} -prng_seed 0 db -do_catchsql_test 2.1 { -SELECT count(*) FROM sqlite_schema; -WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<1000) -INSERT INTO t1(a) SELECT randomblob(null) FROM c; -} {1 {database disk image is malformed}} +# This test only works with the legacy RC4 PRNG +if 0 { + prng_seed 0 db + do_catchsql_test 2.1 { + SELECT count(*) FROM sqlite_schema; + WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<1000) + INSERT INTO t1(a) SELECT randomblob(null) FROM c; + } {1 {database disk image is malformed}} +} reset_db if {![info exists ::G(perm:presql)]} { From 534945ad07002648e46a5f1db0cdeea216d79d79 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 16 Aug 2022 16:40:54 +0000 Subject: [PATCH 3/4] Fix an off-by-one error in the ChaCha20 initialization code. FossilOrigin-Name: 72e220eed446ea9a02a6ef03e09a01bcb8bbca1f3b32d2e0bf52a17d9722e2f0 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/random.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 376b7f52af..33c8ab32a3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Omit\san\sunnecessary\stest\scase\sfrom\scorruptN.test\sthat\sdoes\snot\swork\swith\sthe\nnew\sRFC-7539\sPRNG. -D 2022-08-16T14:09:51.629 +C Fix\san\soff-by-one\serror\sin\sthe\sChaCha20\sinitialization\scode. +D 2022-08-16T16:40:54.599 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -585,7 +585,7 @@ F src/pragma.c 6637d624c37a8909d3edfa9d7cf694d79b49d2a0827d8c52ef15dceb641783fa F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 F src/prepare.c c62820c15dcb63013519c8e41d9f928d7478672cc902cfd0581c733c271dbf45 F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 -F src/random.c ff4b47317d22c9b6af581e25cb9243190082c57daaf175ae93c4adaa223a1e5e +F src/random.c af089dd588a3cbf85df14f9711c347f70050fcb73bc003376c8491065f6c317b F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 4750fbe9d8ecb7236baf7a9bea4299bb87126e08c209645666a0ae8f0efbe0fc @@ -1999,8 +1999,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 084d8776fa95c75440530028171c56547a341c9a952ba2f29bb533b538603c78 -R 4a1be98000c410a528d0befc32c8b6da +P d9e8c65ed25c4f6222c737f4244d5362dbe433d357f7d133765157446cf4e925 +R 6547827ce11f2be716204766ff5dff1b U drh -Z af7ba7ff2c1330aa966ddca9675bb3a2 +Z 947a70a0b2c6f8310490779060824f95 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 63acf48a4f..a9411d2af9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d9e8c65ed25c4f6222c737f4244d5362dbe433d357f7d133765157446cf4e925 \ No newline at end of file +72e220eed446ea9a02a6ef03e09a01bcb8bbca1f3b32d2e0bf52a17d9722e2f0 \ No newline at end of file diff --git a/src/random.c b/src/random.c index 11c5a42b6d..8bf75156aa 100644 --- a/src/random.c +++ b/src/random.c @@ -108,7 +108,7 @@ void sqlite3_randomness(int N, void *pBuf){ }else{ sqlite3OsRandomness(pVfs, 44, (char*)&wsdPrng.s[4]); } - wsdPrng.s[16] = wsdPrng.s[12]; + wsdPrng.s[15] = wsdPrng.s[12]; wsdPrng.s[12] = 0; wsdPrng.n = 0; } From cd05aafa56ec8437c26528bde61be157806a4b3a Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 16 Aug 2022 16:57:33 +0000 Subject: [PATCH 4/4] Fix obsolete comments. Add new comments. Fix non-standard spacing. FossilOrigin-Name: a0d224c6a69941dad1f2b35edcc7ddee343b99eae2aeed74043461f3e97ef5b4 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/random.c | 21 +++++++++------------ 3 files changed, 16 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index 33c8ab32a3..db5ad92d5f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\soff-by-one\serror\sin\sthe\sChaCha20\sinitialization\scode. -D 2022-08-16T16:40:54.599 +C Fix\sobsolete\scomments.\s\sAdd\snew\scomments.\s\sFix\snon-standard\sspacing. +D 2022-08-16T16:57:33.367 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -585,7 +585,7 @@ F src/pragma.c 6637d624c37a8909d3edfa9d7cf694d79b49d2a0827d8c52ef15dceb641783fa F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 F src/prepare.c c62820c15dcb63013519c8e41d9f928d7478672cc902cfd0581c733c271dbf45 F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 -F src/random.c af089dd588a3cbf85df14f9711c347f70050fcb73bc003376c8491065f6c317b +F src/random.c 546d6feb15ec69c1aafe9bb351a277cbb498fd5410e646add673acb805714960 F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 4750fbe9d8ecb7236baf7a9bea4299bb87126e08c209645666a0ae8f0efbe0fc @@ -1999,8 +1999,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d9e8c65ed25c4f6222c737f4244d5362dbe433d357f7d133765157446cf4e925 -R 6547827ce11f2be716204766ff5dff1b +P 72e220eed446ea9a02a6ef03e09a01bcb8bbca1f3b32d2e0bf52a17d9722e2f0 +R cdb53b59d0af11c8f66b0b5bb02ede98 U drh -Z 947a70a0b2c6f8310490779060824f95 +Z 1158c2a25804bd20ee35e9fb8e722661 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a9411d2af9..54413fbc1b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -72e220eed446ea9a02a6ef03e09a01bcb8bbca1f3b32d2e0bf52a17d9722e2f0 \ No newline at end of file +a0d224c6a69941dad1f2b35edcc7ddee343b99eae2aeed74043461f3e97ef5b4 \ No newline at end of file diff --git a/src/random.c b/src/random.c index 8bf75156aa..9335fc49da 100644 --- a/src/random.c +++ b/src/random.c @@ -27,12 +27,15 @@ static SQLITE_WSD struct sqlite3PrngType { u8 n; /* Output bytes remaining */ } sqlite3Prng; + +/* The RFC-7539 ChaCha20 block function +*/ #define ROTL(a,b) (((a) << (b)) | ((a) >> (32 - (b)))) -#define QR(a, b, c, d) ( \ - a += b, d ^= a, d = ROTL(d,16), \ - c += d, b ^= c, b = ROTL(b,12), \ - a += b, d ^= a, d = ROTL(d, 8), \ - c += d, b ^= c, b = ROTL(b, 7)) +#define QR(a, b, c, d) ( \ + a += b, d ^= a, d = ROTL(d,16), \ + c += d, b ^= c, b = ROTL(b,12), \ + a += b, d ^= a, d = ROTL(d, 8), \ + c += d, b ^= c, b = ROTL(b, 7)) static void chacha_block(u32 *out, const u32 *in){ int i; u32 x[16]; @@ -89,13 +92,7 @@ void sqlite3_randomness(int N, void *pBuf){ } /* Initialize the state of the random number generator once, - ** the first time this routine is called. The seed value does - ** not need to contain a lot of randomness since we are not - ** trying to do secure encryption or anything like that... - ** - ** Nothing in this file or anywhere else in SQLite does any kind of - ** encryption. The RC4 algorithm is being used as a PRNG (pseudo-random - ** number generator) not as an encryption device. + ** the first time this routine is called. */ if( wsdPrng.s[0]==0 ){ sqlite3_vfs *pVfs = sqlite3_vfs_find(0);