From 770d0b309151773d833932a3c6960e280ca1cdc4 Mon Sep 17 00:00:00 2001 From: Dmitry Vedenko Date: Wed, 5 May 2021 14:43:27 +0300 Subject: [PATCH] Update SQLite to 3.35.5 This update contains to important features: 1. Improved performance of WAL mode locking primitives in cases where there are hundreds of connections all accessing the same database file at once 2. In WAL mode, if a writer crashes and leaves the shm file in an inconsistent state, subsequent transactions are now able to recover the shm file even if there are active read transactions. Before this enhancement, shm file recovery that scenario would result in an SQLITE_PROTOCOL error. --- lib-src/sqlite/shell.c | 2797 ++- lib-src/sqlite/sqlite3.c | 32144 +++++++++++++++++++--------------- lib-src/sqlite/sqlite3.h | 1494 +- lib-src/sqlite/sqlite3ext.h | 4 + 4 files changed, 21495 insertions(+), 14944 deletions(-) diff --git a/lib-src/sqlite/shell.c b/lib-src/sqlite/shell.c index 974f57af7..27de22a38 100644 --- a/lib-src/sqlite/shell.c +++ b/lib-src/sqlite/shell.c @@ -571,8 +571,6 @@ static void utf8_width_print(FILE *pOut, int w, const char *zUtf){ int i; int n; int aw = w<0 ? -w : w; - char zBuf[1000]; - if( aw>(int)sizeof(zBuf)/3 ) aw = (int)sizeof(zBuf)/3; for(i=n=0; zUtf[i]; i++){ if( (zUtf[i]&0xc0)!=0x80 ){ n++; @@ -641,6 +639,21 @@ static int strlenChar(const char *z){ return n; } +/* +** Return true if zFile does not exist or if it is not an ordinary file. +*/ +#ifdef _WIN32 +# define notNormalFile(X) 0 +#else +static int notNormalFile(const char *zFile){ + struct stat x; + int rc; + memset(&x, 0, sizeof(x)); + rc = stat(zFile, &x); + return rc || !S_ISREG(x.st_mode); +} +#endif + /* ** This routine reads a line of text from FILE in, stores ** the text in memory obtained from malloc() and returns a pointer @@ -953,7 +966,7 @@ static void shellModuleSchema( ** CREATE VIRTUAL TABLE ** ** This UDF is used by the .schema command to insert the schema name of -** attached databases into the middle of the sqlite_master.sql field. +** attached databases into the middle of the sqlite_schema.sql field. */ static void shellAddSchemaName( sqlite3_context *pCtx, @@ -1411,7 +1424,10 @@ SQLITE_EXTENSION_INIT1 #include #include #include + +#ifndef SQLITE_AMALGAMATION /* typedef sqlite3_uint64 u64; */ +#endif /* SQLITE_AMALGAMATION */ /****************************************************************************** ** The Hash Engine @@ -2001,9 +2017,11 @@ static void sha3QueryFunc( } nCol = sqlite3_column_count(pStmt); z = sqlite3_sql(pStmt); - n = (int)strlen(z); - hash_step_vformat(&cx,"S%d:",n); - SHA3Update(&cx,(unsigned char*)z,n); + if( z ){ + n = (int)strlen(z); + hash_step_vformat(&cx,"S%d:",n); + SHA3Update(&cx,(unsigned char*)z,n); + } /* Compute a hash over the result of the query */ while( SQLITE_ROW==sqlite3_step(pStmt) ){ @@ -3330,7 +3348,7 @@ static int completionNext(sqlite3_vtab_cursor *cur){ const char *zDb = (const char*)sqlite3_column_text(pS2, 1); zSql = sqlite3_mprintf( "%z%s" - "SELECT name FROM \"%w\".sqlite_master", + "SELECT name FROM \"%w\".sqlite_schema", zSql, zSep, zDb ); if( zSql==0 ) return SQLITE_NOMEM; @@ -3354,7 +3372,7 @@ static int completionNext(sqlite3_vtab_cursor *cur){ const char *zDb = (const char*)sqlite3_column_text(pS2, 1); zSql = sqlite3_mprintf( "%z%s" - "SELECT pti.name FROM \"%w\".sqlite_master AS sm" + "SELECT pti.name FROM \"%w\".sqlite_schema AS sm" " JOIN pragma_table_info(sm.name,%Q) AS pti" " WHERE sm.type='table'", zSql, zSep, zDb, zDb @@ -3622,24 +3640,23 @@ int sqlite3_completion_init( ** appended onto the end of some other file, such as an executable. ** ** A special record must appear at the end of the file that identifies the -** file as an appended database and provides an offset to page 1. For -** best performance page 1 should be located at a disk page boundary, though -** that is not required. +** file as an appended database and provides the offset to the first page +** of the exposed content. (Or, it is the length of the content prefix.) +** For best performance page 1 should be located at a disk page boundary, +** though that is not required. ** ** When opening a database using this VFS, the connection might treat -** the file as an ordinary SQLite database, or it might treat is as a -** database appended onto some other file. Here are the rules: +** the file as an ordinary SQLite database, or it might treat it as a +** database appended onto some other file. The decision is made by +** applying the following rules in order: ** -** (1) When opening a new empty file, that file is treated as an ordinary -** database. +** (1) An empty file is an ordinary database. ** -** (2) When opening a file that begins with the standard SQLite prefix -** string "SQLite format 3", that file is treated as an ordinary -** database. +** (2) If the file ends with the appendvfs trailer string +** "Start-Of-SQLite3-NNNNNNNN" that file is an appended database. ** -** (3) When opening a file that ends with the appendvfs trailer string -** "Start-Of-SQLite3-NNNNNNNN" that file is treated as an appended -** database. +** (3) If the file begins with the standard SQLite prefix string +** "SQLite format 3", that file is an ordinary database. ** ** (4) If none of the above apply and the SQLITE_OPEN_CREATE flag is ** set, then a new database is appended to the already existing file. @@ -3647,13 +3664,13 @@ int sqlite3_completion_init( ** (5) Otherwise, SQLITE_CANTOPEN is returned. ** ** To avoid unnecessary complications with the PENDING_BYTE, the size of -** the file containing the database is limited to 1GB. This VFS will refuse -** to read or write past the 1GB mark. This restriction might be lifted in -** future versions. For now, if you need a large database, then keep the -** database in a separate file. +** the file containing the database is limited to 1GiB. (1073741824 bytes) +** This VFS will not read or write past the 1GiB mark. This restriction +** might be lifted in future versions. For now, if you need a larger +** database, then keep it in a separate file. ** -** If the file being opened is not an appended database, then this shim is -** a pass-through into the default underlying VFS. +** If the file being opened is a plain database (not an appended one), then +** this shim is a pass-through into the default underlying VFS. (rule 3) **/ /* #include "sqlite3ext.h" */ SQLITE_EXTENSION_INIT1 @@ -3666,17 +3683,27 @@ SQLITE_EXTENSION_INIT1 ** 123456789 123456789 12345 ** ** The NNNNNNNN represents a 64-bit big-endian unsigned integer which is -** the offset to page 1. +** the offset to page 1, and also the length of the prefix content. */ #define APND_MARK_PREFIX "Start-Of-SQLite3-" #define APND_MARK_PREFIX_SZ 17 -#define APND_MARK_SIZE 25 +#define APND_MARK_FOS_SZ 8 +#define APND_MARK_SIZE (APND_MARK_PREFIX_SZ+APND_MARK_FOS_SZ) /* ** Maximum size of the combined prefix + database + append-mark. This ** must be less than 0x40000000 to avoid locking issues on Windows. */ -#define APND_MAX_SIZE (65536*15259) +#define APND_MAX_SIZE (0x40000000) + +/* +** Try to align the database to an even multiple of APND_ROUNDUP bytes. +*/ +#ifndef APND_ROUNDUP +#define APND_ROUNDUP 4096 +#endif +#define APND_ALIGN_MASK ((sqlite3_int64)(APND_ROUNDUP-1)) +#define APND_START_ROUNDUP(fsz) (((fsz)+APND_ALIGN_MASK) & ~APND_ALIGN_MASK) /* ** Forward declaration of objects used by this utility @@ -3690,11 +3717,45 @@ typedef struct ApndFile ApndFile; #define ORIGVFS(p) ((sqlite3_vfs*)((p)->pAppData)) #define ORIGFILE(p) ((sqlite3_file*)(((ApndFile*)(p))+1)) -/* An open file */ +/* An open appendvfs file +** +** An instance of this structure describes the appended database file. +** A separate sqlite3_file object is always appended. The appended +** sqlite3_file object (which can be accessed using ORIGFILE()) describes +** the entire file, including the prefix, the database, and the +** append-mark. +** +** The structure of an AppendVFS database is like this: +** +** +-------------+---------+----------+-------------+ +** | prefix-file | padding | database | append-mark | +** +-------------+---------+----------+-------------+ +** ^ ^ +** | | +** iPgOne iMark +** +** +** "prefix file" - file onto which the database has been appended. +** "padding" - zero or more bytes inserted so that "database" +** starts on an APND_ROUNDUP boundary +** "database" - The SQLite database file +** "append-mark" - The 25-byte "Start-Of-SQLite3-NNNNNNNN" that indicates +** the offset from the start of prefix-file to the start +** of "database". +** +** The size of the database is iMark - iPgOne. +** +** The NNNNNNNN in the "Start-Of-SQLite3-NNNNNNNN" suffix is the value +** of iPgOne stored as a big-ending 64-bit integer. +** +** iMark will be the size of the underlying file minus 25 (APND_MARKSIZE). +** Or, iMark is -1 to indicate that it has not yet been written. +*/ struct ApndFile { - sqlite3_file base; /* IO methods */ - sqlite3_int64 iPgOne; /* File offset to page 1 */ - sqlite3_int64 iMark; /* Start of the append-mark */ + sqlite3_file base; /* Subclass. MUST BE FIRST! */ + sqlite3_int64 iPgOne; /* Offset to the start of the database */ + sqlite3_int64 iMark; /* Offset of the append mark. -1 if unwritten */ + /* Always followed by another sqlite3_file that describes the whole file */ }; /* @@ -3786,8 +3847,6 @@ static const sqlite3_io_methods apnd_io_methods = { apndUnfetch /* xUnfetch */ }; - - /* ** Close an apnd-file. */ @@ -3805,22 +3864,37 @@ static int apndRead( int iAmt, sqlite_int64 iOfst ){ - ApndFile *p = (ApndFile *)pFile; + ApndFile *paf = (ApndFile *)pFile; pFile = ORIGFILE(pFile); - return pFile->pMethods->xRead(pFile, zBuf, iAmt, iOfst+p->iPgOne); + return pFile->pMethods->xRead(pFile, zBuf, iAmt, paf->iPgOne+iOfst); } /* -** Add the append-mark onto the end of the file. +** Add the append-mark onto what should become the end of the file. +* If and only if this succeeds, internal ApndFile.iMark is updated. +* Parameter iWriteEnd is the appendvfs-relative offset of the new mark. */ -static int apndWriteMark(ApndFile *p, sqlite3_file *pFile){ - int i; +static int apndWriteMark( + ApndFile *paf, + sqlite3_file *pFile, + sqlite_int64 iWriteEnd +){ + sqlite_int64 iPgOne = paf->iPgOne; unsigned char a[APND_MARK_SIZE]; + int i = APND_MARK_FOS_SZ; + int rc; + assert(pFile == ORIGFILE(paf)); memcpy(a, APND_MARK_PREFIX, APND_MARK_PREFIX_SZ); - for(i=0; i<8; i++){ - a[APND_MARK_PREFIX_SZ+i] = (p->iPgOne >> (56 - i*8)) & 0xff; + while( --i >= 0 ){ + a[APND_MARK_PREFIX_SZ+i] = (unsigned char)(iPgOne & 0xff); + iPgOne >>= 8; } - return pFile->pMethods->xWrite(pFile, a, APND_MARK_SIZE, p->iMark); + iWriteEnd += paf->iPgOne; + if( SQLITE_OK==(rc = pFile->pMethods->xWrite + (pFile, a, APND_MARK_SIZE, iWriteEnd)) ){ + paf->iMark = iWriteEnd; + } + return rc; } /* @@ -3832,38 +3906,28 @@ static int apndWrite( int iAmt, sqlite_int64 iOfst ){ - int rc; - ApndFile *p = (ApndFile *)pFile; + ApndFile *paf = (ApndFile *)pFile; + sqlite_int64 iWriteEnd = iOfst + iAmt; + if( iWriteEnd>=APND_MAX_SIZE ) return SQLITE_FULL; pFile = ORIGFILE(pFile); - if( iOfst+iAmt>=APND_MAX_SIZE ) return SQLITE_FULL; - rc = pFile->pMethods->xWrite(pFile, zBuf, iAmt, iOfst+p->iPgOne); - if( rc==SQLITE_OK && iOfst + iAmt + p->iPgOne > p->iMark ){ - sqlite3_int64 sz = 0; - rc = pFile->pMethods->xFileSize(pFile, &sz); - if( rc==SQLITE_OK ){ - p->iMark = sz - APND_MARK_SIZE; - if( iOfst + iAmt + p->iPgOne > p->iMark ){ - p->iMark = p->iPgOne + iOfst + iAmt; - rc = apndWriteMark(p, pFile); - } - } + /* If append-mark is absent or will be overwritten, write it. */ + if( paf->iMark < 0 || paf->iPgOne + iWriteEnd > paf->iMark ){ + int rc = apndWriteMark(paf, pFile, iWriteEnd); + if( SQLITE_OK!=rc ) return rc; } - return rc; + return pFile->pMethods->xWrite(pFile, zBuf, iAmt, paf->iPgOne+iOfst); } /* ** Truncate an apnd-file. */ static int apndTruncate(sqlite3_file *pFile, sqlite_int64 size){ - int rc; - ApndFile *p = (ApndFile *)pFile; + ApndFile *paf = (ApndFile *)pFile; pFile = ORIGFILE(pFile); - rc = pFile->pMethods->xTruncate(pFile, size+p->iPgOne+APND_MARK_SIZE); - if( rc==SQLITE_OK ){ - p->iMark = p->iPgOne+size; - rc = apndWriteMark(p, pFile); - } - return rc; + /* The append mark goes out first so truncate failure does not lose it. */ + if( SQLITE_OK!=apndWriteMark(paf, pFile, size) ) return SQLITE_IOERR; + /* Truncate underlying file just past append mark */ + return pFile->pMethods->xTruncate(pFile, paf->iMark+APND_MARK_SIZE); } /* @@ -3876,16 +3940,12 @@ static int apndSync(sqlite3_file *pFile, int flags){ /* ** Return the current file-size of an apnd-file. +** If the append mark is not yet there, the file-size is 0. */ static int apndFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){ - ApndFile *p = (ApndFile *)pFile; - int rc; - pFile = ORIGFILE(p); - rc = pFile->pMethods->xFileSize(pFile, pSize); - if( rc==SQLITE_OK && p->iPgOne ){ - *pSize -= p->iPgOne + APND_MARK_SIZE; - } - return rc; + ApndFile *paf = (ApndFile *)pFile; + *pSize = ( paf->iMark >= 0 )? (paf->iMark - paf->iPgOne) : 0; + return SQLITE_OK; } /* @@ -3916,12 +3976,13 @@ static int apndCheckReservedLock(sqlite3_file *pFile, int *pResOut){ ** File control method. For custom operations on an apnd-file. */ static int apndFileControl(sqlite3_file *pFile, int op, void *pArg){ - ApndFile *p = (ApndFile *)pFile; + ApndFile *paf = (ApndFile *)pFile; int rc; pFile = ORIGFILE(pFile); + if( op==SQLITE_FCNTL_SIZE_HINT ) *(sqlite3_int64*)pArg += paf->iPgOne; rc = pFile->pMethods->xFileControl(pFile, op, pArg); if( rc==SQLITE_OK && op==SQLITE_FCNTL_VFSNAME ){ - *(char**)pArg = sqlite3_mprintf("apnd(%lld)/%z", p->iPgOne, *(char**)pArg); + *(char**)pArg = sqlite3_mprintf("apnd(%lld)/%z", paf->iPgOne,*(char**)pArg); } return rc; } @@ -3980,6 +4041,9 @@ static int apndFetch( void **pp ){ ApndFile *p = (ApndFile *)pFile; + if( p->iMark < 0 || iOfst+iAmt > p->iMark ){ + return SQLITE_IOERR; /* Cannot read what is not yet there. */ + } pFile = ORIGFILE(pFile); return pFile->pMethods->xFetch(pFile, iOfst+p->iPgOne, iAmt, pp); } @@ -3991,95 +4055,153 @@ static int apndUnfetch(sqlite3_file *pFile, sqlite3_int64 iOfst, void *pPage){ return pFile->pMethods->xUnfetch(pFile, iOfst+p->iPgOne, pPage); } -/* -** Check to see if the file is an ordinary SQLite database file. -*/ -static int apndIsOrdinaryDatabaseFile(sqlite3_int64 sz, sqlite3_file *pFile){ - int rc; - char zHdr[16]; - static const char aSqliteHdr[] = "SQLite format 3"; - if( sz<512 ) return 0; - rc = pFile->pMethods->xRead(pFile, zHdr, sizeof(zHdr), 0); - if( rc ) return 0; - return memcmp(zHdr, aSqliteHdr, sizeof(zHdr))==0; -} - /* ** Try to read the append-mark off the end of a file. Return the -** start of the appended database if the append-mark is present. If -** there is no append-mark, return -1; +** start of the appended database if the append-mark is present. +** If there is no valid append-mark, return -1; +** +** An append-mark is only valid if the NNNNNNNN start-of-database offset +** indicates that the appended database contains at least one page. The +** start-of-database value must be a multiple of 512. */ static sqlite3_int64 apndReadMark(sqlite3_int64 sz, sqlite3_file *pFile){ int rc, i; sqlite3_int64 iMark; + int msbs = 8 * (APND_MARK_FOS_SZ-1); unsigned char a[APND_MARK_SIZE]; - if( sz<=APND_MARK_SIZE ) return -1; + if( APND_MARK_SIZE!=(sz & 0x1ff) ) return -1; rc = pFile->pMethods->xRead(pFile, a, APND_MARK_SIZE, sz-APND_MARK_SIZE); if( rc ) return -1; if( memcmp(a, APND_MARK_PREFIX, APND_MARK_PREFIX_SZ)!=0 ) return -1; - iMark = ((sqlite3_int64)(a[APND_MARK_PREFIX_SZ]&0x7f))<<56; - for(i=1; i<8; i++){ - iMark += (sqlite3_int64)a[APND_MARK_PREFIX_SZ+i]<<(56-8*i); + iMark = ((sqlite3_int64)(a[APND_MARK_PREFIX_SZ] & 0x7f)) << msbs; + for(i=1; i<8; i++){ + msbs -= 8; + iMark |= (sqlite3_int64)a[APND_MARK_PREFIX_SZ+i]< (sz - APND_MARK_SIZE - 512) ) return -1; + if( iMark & 0x1ff ) return -1; return iMark; } +static const char apvfsSqliteHdr[] = "SQLite format 3"; +/* +** Check to see if the file is an appendvfs SQLite database file. +** Return true iff it is such. Parameter sz is the file's size. +*/ +static int apndIsAppendvfsDatabase(sqlite3_int64 sz, sqlite3_file *pFile){ + int rc; + char zHdr[16]; + sqlite3_int64 iMark = apndReadMark(sz, pFile); + if( iMark>=0 ){ + /* If file has the correct end-marker, the expected odd size, and the + ** SQLite DB type marker where the end-marker puts it, then it + ** is an appendvfs database. + */ + rc = pFile->pMethods->xRead(pFile, zHdr, sizeof(zHdr), iMark); + if( SQLITE_OK==rc + && memcmp(zHdr, apvfsSqliteHdr, sizeof(zHdr))==0 + && (sz & 0x1ff) == APND_MARK_SIZE + && sz>=512+APND_MARK_SIZE + ){ + return 1; /* It's an appendvfs database */ + } + } + return 0; +} + +/* +** Check to see if the file is an ordinary SQLite database file. +** Return true iff so. Parameter sz is the file's size. +*/ +static int apndIsOrdinaryDatabaseFile(sqlite3_int64 sz, sqlite3_file *pFile){ + char zHdr[16]; + if( apndIsAppendvfsDatabase(sz, pFile) /* rule 2 */ + || (sz & 0x1ff) != 0 + || SQLITE_OK!=pFile->pMethods->xRead(pFile, zHdr, sizeof(zHdr), 0) + || memcmp(zHdr, apvfsSqliteHdr, sizeof(zHdr))!=0 + ){ + return 0; + }else{ + return 1; + } +} + /* ** Open an apnd file handle. */ static int apndOpen( - sqlite3_vfs *pVfs, + sqlite3_vfs *pApndVfs, const char *zName, sqlite3_file *pFile, int flags, int *pOutFlags ){ - ApndFile *p; - sqlite3_file *pSubFile; - sqlite3_vfs *pSubVfs; + ApndFile *pApndFile = (ApndFile*)pFile; + sqlite3_file *pBaseFile = ORIGFILE(pFile); + sqlite3_vfs *pBaseVfs = ORIGVFS(pApndVfs); int rc; - sqlite3_int64 sz; - pSubVfs = ORIGVFS(pVfs); + sqlite3_int64 sz = 0; if( (flags & SQLITE_OPEN_MAIN_DB)==0 ){ - return pSubVfs->xOpen(pSubVfs, zName, pFile, flags, pOutFlags); + /* The appendvfs is not to be used for transient or temporary databases. + ** Just use the base VFS open to initialize the given file object and + ** open the underlying file. (Appendvfs is then unused for this file.) + */ + return pBaseVfs->xOpen(pBaseVfs, zName, pFile, flags, pOutFlags); + } + memset(pApndFile, 0, sizeof(ApndFile)); + pFile->pMethods = &apnd_io_methods; + pApndFile->iMark = -1; /* Append mark not yet written */ + + rc = pBaseVfs->xOpen(pBaseVfs, zName, pBaseFile, flags, pOutFlags); + if( rc==SQLITE_OK ){ + rc = pBaseFile->pMethods->xFileSize(pBaseFile, &sz); } - p = (ApndFile*)pFile; - memset(p, 0, sizeof(*p)); - pSubFile = ORIGFILE(pFile); - p->base.pMethods = &apnd_io_methods; - rc = pSubVfs->xOpen(pSubVfs, zName, pSubFile, flags, pOutFlags); - if( rc ) goto apnd_open_done; - rc = pSubFile->pMethods->xFileSize(pSubFile, &sz); if( rc ){ - pSubFile->pMethods->xClose(pSubFile); - goto apnd_open_done; + pBaseFile->pMethods->xClose(pBaseFile); + pFile->pMethods = 0; + return rc; } - if( apndIsOrdinaryDatabaseFile(sz, pSubFile) ){ - memmove(pFile, pSubFile, pSubVfs->szOsFile); + if( apndIsOrdinaryDatabaseFile(sz, pBaseFile) ){ + /* The file being opened appears to be just an ordinary DB. Copy + ** the base dispatch-table so this instance mimics the base VFS. + */ + memmove(pApndFile, pBaseFile, pBaseVfs->szOsFile); return SQLITE_OK; } - p->iMark = 0; - p->iPgOne = apndReadMark(sz, pFile); - if( p->iPgOne>0 ){ + pApndFile->iPgOne = apndReadMark(sz, pFile); + if( pApndFile->iPgOne>=0 ){ + pApndFile->iMark = sz - APND_MARK_SIZE; /* Append mark found */ return SQLITE_OK; } if( (flags & SQLITE_OPEN_CREATE)==0 ){ - pSubFile->pMethods->xClose(pSubFile); + pBaseFile->pMethods->xClose(pBaseFile); rc = SQLITE_CANTOPEN; + pFile->pMethods = 0; + }else{ + /* Round newly added appendvfs location to #define'd page boundary. + ** Note that nothing has yet been written to the underlying file. + ** The append mark will be written along with first content write. + ** Until then, paf->iMark value indicates it is not yet written. + */ + pApndFile->iPgOne = APND_START_ROUNDUP(sz); } - p->iPgOne = (sz+0xfff) & ~(sqlite3_int64)0xfff; -apnd_open_done: - if( rc ) pFile->pMethods = 0; return rc; } +/* +** Delete an apnd file. +** For an appendvfs, this could mean delete the appendvfs portion, +** leaving the appendee as it was before it gained an appendvfs. +** For now, this code deletes the underlying file too. +*/ +static int apndDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ + return ORIGVFS(pVfs)->xDelete(ORIGVFS(pVfs), zPath, dirSync); +} + /* ** All other VFS methods are pass-thrus. */ -static int apndDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ - return ORIGVFS(pVfs)->xDelete(ORIGVFS(pVfs), zPath, dirSync); -} static int apndAccess( sqlite3_vfs *pVfs, const char *zPath, @@ -4379,6 +4501,1383 @@ int sqlite3_uint_init( } /************************* End ../ext/misc/uint.c ********************/ +/************************* Begin ../ext/misc/decimal.c ******************/ +/* +** 2020-06-22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** Routines to implement arbitrary-precision decimal math. +** +** The focus here is on simplicity and correctness, not performance. +*/ +/* #include "sqlite3ext.h" */ +SQLITE_EXTENSION_INIT1 +#include +#include +#include +#include + +/* Mark a function parameter as unused, to suppress nuisance compiler +** warnings. */ +#ifndef UNUSED_PARAMETER +# define UNUSED_PARAMETER(X) (void)(X) +#endif + + +/* A decimal object */ +typedef struct Decimal Decimal; +struct Decimal { + char sign; /* 0 for positive, 1 for negative */ + char oom; /* True if an OOM is encountered */ + char isNull; /* True if holds a NULL rather than a number */ + char isInit; /* True upon initialization */ + int nDigit; /* Total number of digits */ + int nFrac; /* Number of digits to the right of the decimal point */ + signed char *a; /* Array of digits. Most significant first. */ +}; + +/* +** Release memory held by a Decimal, but do not free the object itself. +*/ +static void decimal_clear(Decimal *p){ + sqlite3_free(p->a); +} + +/* +** Destroy a Decimal object +*/ +static void decimal_free(Decimal *p){ + if( p ){ + decimal_clear(p); + sqlite3_free(p); + } +} + +/* +** Allocate a new Decimal object. Initialize it to the number given +** by the input string. +*/ +static Decimal *decimal_new( + sqlite3_context *pCtx, + sqlite3_value *pIn, + int nAlt, + const unsigned char *zAlt +){ + Decimal *p; + int n, i; + const unsigned char *zIn; + int iExp = 0; + p = sqlite3_malloc( sizeof(*p) ); + if( p==0 ) goto new_no_mem; + p->sign = 0; + p->oom = 0; + p->isInit = 1; + p->isNull = 0; + p->nDigit = 0; + p->nFrac = 0; + if( zAlt ){ + n = nAlt, + zIn = zAlt; + }else{ + if( sqlite3_value_type(pIn)==SQLITE_NULL ){ + p->a = 0; + p->isNull = 1; + return p; + } + n = sqlite3_value_bytes(pIn); + zIn = sqlite3_value_text(pIn); + } + p->a = sqlite3_malloc64( n+1 ); + if( p->a==0 ) goto new_no_mem; + for(i=0; isspace(zIn[i]); i++){} + if( zIn[i]=='-' ){ + p->sign = 1; + i++; + }else if( zIn[i]=='+' ){ + i++; + } + while( i='0' && c<='9' ){ + p->a[p->nDigit++] = c - '0'; + }else if( c=='.' ){ + p->nFrac = p->nDigit + 1; + }else if( c=='e' || c=='E' ){ + int j = i+1; + int neg = 0; + if( j>=n ) break; + if( zIn[j]=='-' ){ + neg = 1; + j++; + }else if( zIn[j]=='+' ){ + j++; + } + while( j='0' && zIn[j]<='9' ){ + iExp = iExp*10 + zIn[j] - '0'; + } + j++; + } + if( neg ) iExp = -iExp; + break; + } + i++; + } + if( p->nFrac ){ + p->nFrac = p->nDigit - (p->nFrac - 1); + } + if( iExp>0 ){ + if( p->nFrac>0 ){ + if( iExp<=p->nFrac ){ + p->nFrac -= iExp; + iExp = 0; + }else{ + iExp -= p->nFrac; + p->nFrac = 0; + } + } + if( iExp>0 ){ + p->a = sqlite3_realloc64(p->a, p->nDigit + iExp + 1 ); + if( p->a==0 ) goto new_no_mem; + memset(p->a+p->nDigit, 0, iExp); + p->nDigit += iExp; + } + }else if( iExp<0 ){ + int nExtra; + iExp = -iExp; + nExtra = p->nDigit - p->nFrac - 1; + if( nExtra ){ + if( nExtra>=iExp ){ + p->nFrac += iExp; + iExp = 0; + }else{ + iExp -= nExtra; + p->nFrac = p->nDigit - 1; + } + } + if( iExp>0 ){ + p->a = sqlite3_realloc64(p->a, p->nDigit + iExp + 1 ); + if( p->a==0 ) goto new_no_mem; + memmove(p->a+iExp, p->a, p->nDigit); + memset(p->a, 0, iExp); + p->nDigit += iExp; + p->nFrac += iExp; + } + } + return p; + +new_no_mem: + if( pCtx ) sqlite3_result_error_nomem(pCtx); + sqlite3_free(p); + return 0; +} + +/* +** Make the given Decimal the result. +*/ +static void decimal_result(sqlite3_context *pCtx, Decimal *p){ + char *z; + int i, j; + int n; + if( p==0 || p->oom ){ + sqlite3_result_error_nomem(pCtx); + return; + } + if( p->isNull ){ + sqlite3_result_null(pCtx); + return; + } + z = sqlite3_malloc( p->nDigit+4 ); + if( z==0 ){ + sqlite3_result_error_nomem(pCtx); + return; + } + i = 0; + if( p->nDigit==0 || (p->nDigit==1 && p->a[0]==0) ){ + p->sign = 0; + } + if( p->sign ){ + z[0] = '-'; + i = 1; + } + n = p->nDigit - p->nFrac; + if( n<=0 ){ + z[i++] = '0'; + } + j = 0; + while( n>1 && p->a[j]==0 ){ + j++; + n--; + } + while( n>0 ){ + z[i++] = p->a[j] + '0'; + j++; + n--; + } + if( p->nFrac ){ + z[i++] = '.'; + do{ + z[i++] = p->a[j] + '0'; + j++; + }while( jnDigit ); + } + z[i] = 0; + sqlite3_result_text(pCtx, z, i, sqlite3_free); +} + +/* +** SQL Function: decimal(X) +** +** Convert input X into decimal and then back into text +*/ +static void decimalFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Decimal *p = decimal_new(context, argv[0], 0, 0); + UNUSED_PARAMETER(argc); + decimal_result(context, p); + decimal_free(p); +} + +/* +** Compare to Decimal objects. Return negative, 0, or positive if the +** first object is less than, equal to, or greater than the second. +** +** Preconditions for this routine: +** +** pA!=0 +** pA->isNull==0 +** pB!=0 +** pB->isNull==0 +*/ +static int decimal_cmp(const Decimal *pA, const Decimal *pB){ + int nASig, nBSig, rc, n; + if( pA->sign!=pB->sign ){ + return pA->sign ? -1 : +1; + } + if( pA->sign ){ + const Decimal *pTemp = pA; + pA = pB; + pB = pTemp; + } + nASig = pA->nDigit - pA->nFrac; + nBSig = pB->nDigit - pB->nFrac; + if( nASig!=nBSig ){ + return nASig - nBSig; + } + n = pA->nDigit; + if( n>pB->nDigit ) n = pB->nDigit; + rc = memcmp(pA->a, pB->a, n); + if( rc==0 ){ + rc = pA->nDigit - pB->nDigit; + } + return rc; +} + +/* +** SQL Function: decimal_cmp(X, Y) +** +** Return negative, zero, or positive if X is less then, equal to, or +** greater than Y. +*/ +static void decimalCmpFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Decimal *pA = 0, *pB = 0; + int rc; + + UNUSED_PARAMETER(argc); + pA = decimal_new(context, argv[0], 0, 0); + if( pA==0 || pA->isNull ) goto cmp_done; + pB = decimal_new(context, argv[1], 0, 0); + if( pB==0 || pB->isNull ) goto cmp_done; + rc = decimal_cmp(pA, pB); + if( rc<0 ) rc = -1; + else if( rc>0 ) rc = +1; + sqlite3_result_int(context, rc); +cmp_done: + decimal_free(pA); + decimal_free(pB); +} + +/* +** Expand the Decimal so that it has a least nDigit digits and nFrac +** digits to the right of the decimal point. +*/ +static void decimal_expand(Decimal *p, int nDigit, int nFrac){ + int nAddSig; + int nAddFrac; + if( p==0 ) return; + nAddFrac = nFrac - p->nFrac; + nAddSig = (nDigit - p->nDigit) - nAddFrac; + if( nAddFrac==0 && nAddSig==0 ) return; + p->a = sqlite3_realloc64(p->a, nDigit+1); + if( p->a==0 ){ + p->oom = 1; + return; + } + if( nAddSig ){ + memmove(p->a+nAddSig, p->a, p->nDigit); + memset(p->a, 0, nAddSig); + p->nDigit += nAddSig; + } + if( nAddFrac ){ + memset(p->a+p->nDigit, 0, nAddFrac); + p->nDigit += nAddFrac; + p->nFrac += nAddFrac; + } +} + +/* +** Add the value pB into pA. +** +** Both pA and pB might become denormalized by this routine. +*/ +static void decimal_add(Decimal *pA, Decimal *pB){ + int nSig, nFrac, nDigit; + int i, rc; + if( pA==0 ){ + return; + } + if( pA->oom || pB==0 || pB->oom ){ + pA->oom = 1; + return; + } + if( pA->isNull || pB->isNull ){ + pA->isNull = 1; + return; + } + nSig = pA->nDigit - pA->nFrac; + if( nSig && pA->a[0]==0 ) nSig--; + if( nSignDigit-pB->nFrac ){ + nSig = pB->nDigit - pB->nFrac; + } + nFrac = pA->nFrac; + if( nFracnFrac ) nFrac = pB->nFrac; + nDigit = nSig + nFrac + 1; + decimal_expand(pA, nDigit, nFrac); + decimal_expand(pB, nDigit, nFrac); + if( pA->oom || pB->oom ){ + pA->oom = 1; + }else{ + if( pA->sign==pB->sign ){ + int carry = 0; + for(i=nDigit-1; i>=0; i--){ + int x = pA->a[i] + pB->a[i] + carry; + if( x>=10 ){ + carry = 1; + pA->a[i] = x - 10; + }else{ + carry = 0; + pA->a[i] = x; + } + } + }else{ + signed char *aA, *aB; + int borrow = 0; + rc = memcmp(pA->a, pB->a, nDigit); + if( rc<0 ){ + aA = pB->a; + aB = pA->a; + pA->sign = !pA->sign; + }else{ + aA = pA->a; + aB = pB->a; + } + for(i=nDigit-1; i>=0; i--){ + int x = aA[i] - aB[i] - borrow; + if( x<0 ){ + pA->a[i] = x+10; + borrow = 1; + }else{ + pA->a[i] = x; + borrow = 0; + } + } + } + } +} + +/* +** Compare text in decimal order. +*/ +static int decimalCollFunc( + void *notUsed, + int nKey1, const void *pKey1, + int nKey2, const void *pKey2 +){ + const unsigned char *zA = (const unsigned char*)pKey1; + const unsigned char *zB = (const unsigned char*)pKey2; + Decimal *pA = decimal_new(0, 0, nKey1, zA); + Decimal *pB = decimal_new(0, 0, nKey2, zB); + int rc; + UNUSED_PARAMETER(notUsed); + if( pA==0 || pB==0 ){ + rc = 0; + }else{ + rc = decimal_cmp(pA, pB); + } + decimal_free(pA); + decimal_free(pB); + return rc; +} + + +/* +** SQL Function: decimal_add(X, Y) +** decimal_sub(X, Y) +** +** Return the sum or difference of X and Y. +*/ +static void decimalAddFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Decimal *pA = decimal_new(context, argv[0], 0, 0); + Decimal *pB = decimal_new(context, argv[1], 0, 0); + UNUSED_PARAMETER(argc); + decimal_add(pA, pB); + decimal_result(context, pA); + decimal_free(pA); + decimal_free(pB); +} +static void decimalSubFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Decimal *pA = decimal_new(context, argv[0], 0, 0); + Decimal *pB = decimal_new(context, argv[1], 0, 0); + UNUSED_PARAMETER(argc); + if( pB==0 ) return; + pB->sign = !pB->sign; + decimal_add(pA, pB); + decimal_result(context, pA); + decimal_free(pA); + decimal_free(pB); +} + +/* Aggregate funcion: decimal_sum(X) +** +** Works like sum() except that it uses decimal arithmetic for unlimited +** precision. +*/ +static void decimalSumStep( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Decimal *p; + Decimal *pArg; + UNUSED_PARAMETER(argc); + p = sqlite3_aggregate_context(context, sizeof(*p)); + if( p==0 ) return; + if( !p->isInit ){ + p->isInit = 1; + p->a = sqlite3_malloc(2); + if( p->a==0 ){ + p->oom = 1; + }else{ + p->a[0] = 0; + } + p->nDigit = 1; + p->nFrac = 0; + } + if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; + pArg = decimal_new(context, argv[0], 0, 0); + decimal_add(p, pArg); + decimal_free(pArg); +} +static void decimalSumInverse( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Decimal *p; + Decimal *pArg; + UNUSED_PARAMETER(argc); + p = sqlite3_aggregate_context(context, sizeof(*p)); + if( p==0 ) return; + if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; + pArg = decimal_new(context, argv[0], 0, 0); + if( pArg ) pArg->sign = !pArg->sign; + decimal_add(p, pArg); + decimal_free(pArg); +} +static void decimalSumValue(sqlite3_context *context){ + Decimal *p = sqlite3_aggregate_context(context, 0); + if( p==0 ) return; + decimal_result(context, p); +} +static void decimalSumFinalize(sqlite3_context *context){ + Decimal *p = sqlite3_aggregate_context(context, 0); + if( p==0 ) return; + decimal_result(context, p); + decimal_clear(p); +} + +/* +** SQL Function: decimal_mul(X, Y) +** +** Return the product of X and Y. +** +** All significant digits after the decimal point are retained. +** Trailing zeros after the decimal point are omitted as long as +** the number of digits after the decimal point is no less than +** either the number of digits in either input. +*/ +static void decimalMulFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Decimal *pA = decimal_new(context, argv[0], 0, 0); + Decimal *pB = decimal_new(context, argv[1], 0, 0); + signed char *acc = 0; + int i, j, k; + int minFrac; + UNUSED_PARAMETER(argc); + if( pA==0 || pA->oom || pA->isNull + || pB==0 || pB->oom || pB->isNull + ){ + goto mul_end; + } + acc = sqlite3_malloc64( pA->nDigit + pB->nDigit + 2 ); + if( acc==0 ){ + sqlite3_result_error_nomem(context); + goto mul_end; + } + memset(acc, 0, pA->nDigit + pB->nDigit + 2); + minFrac = pA->nFrac; + if( pB->nFracnFrac; + for(i=pA->nDigit-1; i>=0; i--){ + signed char f = pA->a[i]; + int carry = 0, x; + for(j=pB->nDigit-1, k=i+j+3; j>=0; j--, k--){ + x = acc[k] + f*pB->a[j] + carry; + acc[k] = x%10; + carry = x/10; + } + x = acc[k] + carry; + acc[k] = x%10; + acc[k-1] += x/10; + } + sqlite3_free(pA->a); + pA->a = acc; + acc = 0; + pA->nDigit += pB->nDigit + 2; + pA->nFrac += pB->nFrac; + pA->sign ^= pB->sign; + while( pA->nFrac>minFrac && pA->a[pA->nDigit-1]==0 ){ + pA->nFrac--; + pA->nDigit--; + } + decimal_result(context, pA); + +mul_end: + sqlite3_free(acc); + decimal_free(pA); + decimal_free(pB); +} + +#ifdef _WIN32 + +#endif +int sqlite3_decimal_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + static const struct { + const char *zFuncName; + int nArg; + void (*xFunc)(sqlite3_context*,int,sqlite3_value**); + } aFunc[] = { + { "decimal", 1, decimalFunc }, + { "decimal_cmp", 2, decimalCmpFunc }, + { "decimal_add", 2, decimalAddFunc }, + { "decimal_sub", 2, decimalSubFunc }, + { "decimal_mul", 2, decimalMulFunc }, + }; + unsigned int i; + (void)pzErrMsg; /* Unused parameter */ + + SQLITE_EXTENSION_INIT2(pApi); + + for(i=0; i 'ieee754(2,0)' +** ieee754(45.25) -> 'ieee754(181,-2)' +** ieee754(2, 0) -> 2.0 +** ieee754(181, -2) -> 45.25 +** +** Two additional functions break apart the one-argument ieee754() +** result into separate integer values: +** +** ieee754_mantissa(45.25) -> 181 +** ieee754_exponent(45.25) -> -2 +** +** These functions convert binary64 numbers into blobs and back again. +** +** ieee754_from_blob(x'3ff0000000000000') -> 1.0 +** ieee754_to_blob(1.0) -> x'3ff0000000000000' +** +** In all single-argument functions, if the argument is an 8-byte blob +** then that blob is interpreted as a big-endian binary64 value. +** +** +** EXACT DECIMAL REPRESENTATION OF BINARY64 VALUES +** ----------------------------------------------- +** +** This extension in combination with the separate 'decimal' extension +** can be used to compute the exact decimal representation of binary64 +** values. To begin, first compute a table of exponent values: +** +** CREATE TABLE pow2(x INTEGER PRIMARY KEY, v TEXT); +** WITH RECURSIVE c(x,v) AS ( +** VALUES(0,'1') +** UNION ALL +** SELECT x+1, decimal_mul(v,'2') FROM c WHERE x+1<=971 +** ) INSERT INTO pow2(x,v) SELECT x, v FROM c; +** WITH RECURSIVE c(x,v) AS ( +** VALUES(-1,'0.5') +** UNION ALL +** SELECT x-1, decimal_mul(v,'0.5') FROM c WHERE x-1>=-1075 +** ) INSERT INTO pow2(x,v) SELECT x, v FROM c; +** +** Then, to compute the exact decimal representation of a floating +** point value (the value 47.49 is used in the example) do: +** +** WITH c(n) AS (VALUES(47.49)) +** ---------------^^^^^---- Replace with whatever you want +** SELECT decimal_mul(ieee754_mantissa(c.n),pow2.v) +** FROM pow2, c WHERE pow2.x=ieee754_exponent(c.n); +** +** Here is a query to show various boundry values for the binary64 +** number format: +** +** WITH c(name,bin) AS (VALUES +** ('minimum positive value', x'0000000000000001'), +** ('maximum subnormal value', x'000fffffffffffff'), +** ('mininum positive nornal value', x'0010000000000000'), +** ('maximum value', x'7fefffffffffffff')) +** SELECT c.name, decimal_mul(ieee754_mantissa(c.bin),pow2.v) +** FROM pow2, c WHERE pow2.x=ieee754_exponent(c.bin); +** +*/ +/* #include "sqlite3ext.h" */ +SQLITE_EXTENSION_INIT1 +#include +#include + +/* Mark a function parameter as unused, to suppress nuisance compiler +** warnings. */ +#ifndef UNUSED_PARAMETER +# define UNUSED_PARAMETER(X) (void)(X) +#endif + +/* +** Implementation of the ieee754() function +*/ +static void ieee754func( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + if( argc==1 ){ + sqlite3_int64 m, a; + double r; + int e; + int isNeg; + char zResult[100]; + assert( sizeof(m)==sizeof(r) ); + if( sqlite3_value_type(argv[0])==SQLITE_BLOB + && sqlite3_value_bytes(argv[0])==sizeof(r) + ){ + const unsigned char *x = sqlite3_value_blob(argv[0]); + unsigned int i; + sqlite3_uint64 v = 0; + for(i=0; i>52; + m = a & ((((sqlite3_int64)1)<<52)-1); + if( e==0 ){ + m <<= 1; + }else{ + m |= ((sqlite3_int64)1)<<52; + } + while( e<1075 && m>0 && (m&1)==0 ){ + m >>= 1; + e++; + } + if( isNeg ) m = -m; + } + switch( *(int*)sqlite3_user_data(context) ){ + case 0: + sqlite3_snprintf(sizeof(zResult), zResult, "ieee754(%lld,%d)", + m, e-1075); + sqlite3_result_text(context, zResult, -1, SQLITE_TRANSIENT); + break; + case 1: + sqlite3_result_int64(context, m); + break; + case 2: + sqlite3_result_int(context, e-1075); + break; + } + }else{ + sqlite3_int64 m, e, a; + double r; + int isNeg = 0; + m = sqlite3_value_int64(argv[0]); + e = sqlite3_value_int64(argv[1]); + + /* Limit the range of e. Ticket 22dea1cfdb9151e4 2021-03-02 */ + if( e>10000 ){ + e = 10000; + }else if( e<-10000 ){ + e = -10000; + } + + if( m<0 ){ + isNeg = 1; + m = -m; + if( m<0 ) return; + }else if( m==0 && e>-1000 && e<1000 ){ + sqlite3_result_double(context, 0.0); + return; + } + while( (m>>32)&0xffe00000 ){ + m >>= 1; + e++; + } + while( m!=0 && ((m>>32)&0xfff00000)==0 ){ + m <<= 1; + e--; + } + e += 1075; + if( e<=0 ){ + /* Subnormal */ + m >>= 1-e; + e = 0; + }else if( e>0x7ff ){ + e = 0x7ff; + } + a = m & ((((sqlite3_int64)1)<<52)-1); + a |= e<<52; + if( isNeg ) a |= ((sqlite3_uint64)1)<<63; + memcpy(&r, &a, sizeof(r)); + sqlite3_result_double(context, r); + } +} + +/* +** Functions to convert between blobs and floats. +*/ +static void ieee754func_from_blob( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + UNUSED_PARAMETER(argc); + if( sqlite3_value_type(argv[0])==SQLITE_BLOB + && sqlite3_value_bytes(argv[0])==sizeof(double) + ){ + double r; + const unsigned char *x = sqlite3_value_blob(argv[0]); + unsigned int i; + sqlite3_uint64 v = 0; + for(i=0; i>= 8; + } + sqlite3_result_blob(context, a, sizeof(r), SQLITE_TRANSIENT); + } +} + + +#ifdef _WIN32 + +#endif +int sqlite3_ieee_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + static const struct { + char *zFName; + int nArg; + int iAux; + void (*xFunc)(sqlite3_context*,int,sqlite3_value**); + } aFunc[] = { + { "ieee754", 1, 0, ieee754func }, + { "ieee754", 2, 0, ieee754func }, + { "ieee754_mantissa", 1, 1, ieee754func }, + { "ieee754_exponent", 1, 2, ieee754func }, + { "ieee754_to_blob", 1, 0, ieee754func_to_blob }, + { "ieee754_from_blob", 1, 0, ieee754func_from_blob }, + + }; + unsigned int i; + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Unused parameter */ + for(i=0; i