From 08119c361d1181b3e8f1abb429236e488a664753 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 13 Aug 2013 15:42:54 -0400 Subject: Imported Upstream version 2.2.1 --- src/alter.c | 6 +- src/analyze.c | 4 +- src/attach.c | 12 +- src/backup.c | 57 +- src/bitvec.c | 2 +- src/btree.c | 501 +++++--- src/btree.h | 7 + src/btreeInt.h | 1 + src/build.c | 129 ++- src/callback.c | 12 +- src/crypto.c | 222 +++- src/crypto.h | 32 +- src/crypto_cc.c | 140 +++ src/crypto_impl.c | 441 +++----- src/crypto_libtomcrypt.c | 215 ++++ src/crypto_openssl.c | 251 ++++ src/ctime.c | 31 +- src/delete.c | 43 +- src/expr.c | 241 ++-- src/fkey.c | 40 +- src/func.c | 193 +++- src/global.c | 11 + src/hash.c | 2 +- src/hash.h | 2 +- src/insert.c | 160 ++- src/journal.c | 18 + src/legacy.c | 12 +- src/lempar.c | 1 + src/loadext.c | 87 +- src/main.c | 190 +++- src/memjournal.c | 4 +- src/os.c | 20 + src/os.h | 10 +- src/os_unix.c | 684 +++++++++-- src/os_win.c | 968 ++++++++++++---- src/pager.c | 401 +++++-- src/pager.h | 21 +- src/parse.y | 53 +- src/pcache.h | 2 + src/pragma.c | 221 +++- src/prepare.c | 13 +- src/resolve.c | 212 ++-- src/select.c | 508 ++++++--- src/shell.c | 308 +++-- src/sqlcipher.h | 75 ++ src/sqlite.h.in | 186 ++- src/sqlite3ext.h | 41 +- src/sqliteInt.h | 400 ++++--- src/status.c | 3 +- src/tclsqlite.c | 29 +- src/test1.c | 249 ++-- src/test2.c | 54 +- src/test3.c | 40 +- src/test4.c | 31 +- src/test6.c | 25 +- src/test7.c | 31 +- src/test8.c | 28 +- src/test_async.c | 8 +- src/test_autoext.c | 2 +- src/test_backup.c | 10 +- src/test_config.c | 14 +- src/test_fs.c | 333 ++++++ src/test_fuzzer.c | 1215 -------------------- src/test_intarray.c | 11 +- src/test_intarray.h | 11 + src/test_malloc.c | 68 +- src/test_multiplex.c | 10 +- src/test_multiplex.h | 8 + src/test_mutex.c | 12 +- src/test_quota.c | 29 +- src/test_rtree.c | 8 +- src/test_spellfix.c | 2830 ---------------------------------------------- src/test_sqllog.c | 507 +++++++++ src/test_syscall.c | 43 +- src/test_thread.c | 16 +- src/test_vfs.c | 44 +- src/test_vfstrace.c | 4 +- src/test_wholenumber.c | 311 ----- src/trigger.c | 9 + src/update.c | 6 +- src/utf.c | 16 +- src/util.c | 38 +- src/vacuum.c | 2 + src/vdbe.c | 143 ++- src/vdbe.h | 2 +- src/vdbeInt.h | 49 +- src/vdbeapi.c | 14 +- src/vdbeaux.c | 126 ++- src/vdbeblob.c | 2 +- src/vdbemem.c | 2 + src/vdbesort.c | 10 +- src/vdbetrace.c | 38 +- src/vtab.c | 12 +- src/wal.c | 74 +- src/wal.h | 5 +- src/walker.c | 17 +- src/where.c | 1798 ++++++++++++++++++----------- 97 files changed, 8121 insertions(+), 7416 deletions(-) create mode 100644 src/crypto_cc.c create mode 100644 src/crypto_libtomcrypt.c create mode 100644 src/crypto_openssl.c create mode 100644 src/sqlcipher.h create mode 100644 src/test_fs.c delete mode 100644 src/test_fuzzer.c delete mode 100644 src/test_spellfix.c create mode 100644 src/test_sqllog.c delete mode 100644 src/test_wholenumber.c (limited to 'src') diff --git a/src/alter.c b/src/alter.c index 7f56ce7..a49d334 100644 --- a/src/alter.c +++ b/src/alter.c @@ -414,7 +414,7 @@ void sqlite3AlterRenameTable( assert( pSrc->nSrc==1 ); assert( sqlite3BtreeHoldsAllMutexes(pParse->db) ); - pTab = sqlite3LocateTable(pParse, 0, pSrc->a[0].zName, pSrc->a[0].zDatabase); + pTab = sqlite3LocateTableItem(pParse, 0, &pSrc->a[0]); if( !pTab ) goto exit_rename_table; iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); zDb = db->aDb[iDb].zName; @@ -664,7 +664,7 @@ void sqlite3AlterFinishAddColumn(Parse *pParse, Token *pColDef){ ** If there is a NOT NULL constraint, then the default value for the ** column must not be NULL. */ - if( pCol->isPrimKey ){ + if( pCol->colFlags & COLFLAG_PRIMKEY ){ sqlite3ErrorMsg(pParse, "Cannot add a PRIMARY KEY column"); return; } @@ -757,7 +757,7 @@ void sqlite3AlterBeginAddColumn(Parse *pParse, SrcList *pSrc){ assert( pParse->pNewTable==0 ); assert( sqlite3BtreeHoldsAllMutexes(db) ); if( db->mallocFailed ) goto exit_begin_add_column; - pTab = sqlite3LocateTable(pParse, 0, pSrc->a[0].zName, pSrc->a[0].zDatabase); + pTab = sqlite3LocateTableItem(pParse, 0, &pSrc->a[0]); if( !pTab ) goto exit_begin_add_column; #ifndef SQLITE_OMIT_VIRTUALTABLE diff --git a/src/analyze.c b/src/analyze.c index 632fdc1..9a3e959 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -473,7 +473,7 @@ static void analyzeOneTable( /* Do not gather statistics on views or virtual tables */ return; } - if( memcmp(pTab->zName, "sqlite_", 7)==0 ){ + if( sqlite3_strnicmp(pTab->zName, "sqlite_", 7)==0 ){ /* Do not gather statistics on system tables */ return; } @@ -883,7 +883,7 @@ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ if( pIndex==0 ) break; pIndex->aiRowEst[i] = v; if( *z==' ' ) z++; - if( memcmp(z, "unordered", 10)==0 ){ + if( strcmp(z, "unordered")==0 ){ pIndex->bUnordered = 1; break; } diff --git a/src/attach.c b/src/attach.c index e295c30..b8e1219 100644 --- a/src/attach.c +++ b/src/attach.c @@ -109,7 +109,7 @@ static void attachFunc( } } - /* Allocate the new entry in the db->aDb[] array and initialise the schema + /* Allocate the new entry in the db->aDb[] array and initialize the schema ** hash tables. */ if( db->aDb==db->aDbStatic ){ @@ -126,7 +126,7 @@ static void attachFunc( /* Open the database file. If the btree is successfully opened, use ** it to obtain the database schema. At this point the schema may - ** or may not be initialised. + ** or may not be initialized. */ flags = db->openFlags; rc = sqlite3ParseUri(db->pVfs->zName, zFile, &flags, &pVfs, &zPath, &zErr); @@ -434,6 +434,7 @@ int sqlite3FixInit( assert( db->nDb>iDb ); pFix->pParse = pParse; pFix->zDb = db->aDb[iDb].zName; + pFix->pSchema = db->aDb[iDb].pSchema; pFix->zType = zType; pFix->pName = pName; return 1; @@ -464,14 +465,15 @@ int sqlite3FixSrcList( if( NEVER(pList==0) ) return 0; zDb = pFix->zDb; for(i=0, pItem=pList->a; inSrc; i++, pItem++){ - if( pItem->zDatabase==0 ){ - pItem->zDatabase = sqlite3DbStrDup(pFix->pParse->db, zDb); - }else if( sqlite3StrICmp(pItem->zDatabase,zDb)!=0 ){ + if( pItem->zDatabase && sqlite3StrICmp(pItem->zDatabase, zDb) ){ sqlite3ErrorMsg(pFix->pParse, "%s %T cannot reference objects in database %s", pFix->zType, pFix->pName, pItem->zDatabase); return 1; } + sqlite3DbFree(pFix->pParse->db, pItem->zDatabase); + pItem->zDatabase = 0; + pItem->pSchema = pFix->pSchema; #if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_TRIGGER) if( sqlite3FixSelect(pFix, pItem->pSelect) ) return 1; if( sqlite3FixExpr(pFix, pItem->pOn) ) return 1; diff --git a/src/backup.c b/src/backup.c index 4881215..252f61c 100644 --- a/src/backup.c +++ b/src/backup.c @@ -212,20 +212,28 @@ static int isFatalError(int rc){ ** page iSrcPg from the source database. Copy this data into the ** destination database. */ -static int backupOnePage(sqlite3_backup *p, Pgno iSrcPg, const u8 *zSrcData){ +static int backupOnePage( + sqlite3_backup *p, /* Backup handle */ + Pgno iSrcPg, /* Source database page to backup */ + const u8 *zSrcData, /* Source database page data */ + int bUpdate /* True for an update, false otherwise */ +){ Pager * const pDestPager = sqlite3BtreePager(p->pDest); const int nSrcPgsz = sqlite3BtreeGetPageSize(p->pSrc); int nDestPgsz = sqlite3BtreeGetPageSize(p->pDest); const int nCopy = MIN(nSrcPgsz, nDestPgsz); const i64 iEnd = (i64)iSrcPg*(i64)nSrcPgsz; #ifdef SQLITE_HAS_CODEC - int nSrcReserve = sqlite3BtreeGetReserve(p->pSrc); + /* Use BtreeGetReserveNoMutex() for the source b-tree, as although it is + ** guaranteed that the shared-mutex is held by this thread, handle + ** p->pSrc may not actually be the owner. */ + int nSrcReserve = sqlite3BtreeGetReserveNoMutex(p->pSrc); int nDestReserve = sqlite3BtreeGetReserve(p->pDest); #endif - int rc = SQLITE_OK; i64 iOff; + assert( sqlite3BtreeGetReserveNoMutex(p->pSrc)>=0 ); assert( p->bDestLocked ); assert( !isFatalError(p->rc) ); assert( iSrcPg!=PENDING_BYTE_PAGE(p->pSrc->pBt) ); @@ -282,6 +290,9 @@ static int backupOnePage(sqlite3_backup *p, Pgno iSrcPg, const u8 *zSrcData){ */ memcpy(zOut, zIn, nCopy); ((u8 *)sqlite3PagerGetExtra(pDestPg))[0] = 0; + if( iOff==0 && bUpdate==0 ){ + sqlite3Put4byte(&zOut[28], sqlite3BtreeLastPage(p->pSrc)); + } } sqlite3PagerUnref(pDestPg); } @@ -386,9 +397,10 @@ int sqlite3_backup_step(sqlite3_backup *p, int nPage){ const Pgno iSrcPg = p->iNext; /* Source page number */ if( iSrcPg!=PENDING_BYTE_PAGE(p->pSrc->pBt) ){ DbPage *pSrcPg; /* Source page object */ - rc = sqlite3PagerGet(pSrcPager, iSrcPg, &pSrcPg); + rc = sqlite3PagerAcquire(pSrcPager, iSrcPg, &pSrcPg, + PAGER_ACQUIRE_READONLY); if( rc==SQLITE_OK ){ - rc = backupOnePage(p, iSrcPg, sqlite3PagerGetData(pSrcPg)); + rc = backupOnePage(p, iSrcPg, sqlite3PagerGetData(pSrcPg), 0); sqlite3PagerUnref(pSrcPg); } } @@ -410,7 +422,13 @@ int sqlite3_backup_step(sqlite3_backup *p, int nPage){ ** same schema version. */ if( rc==SQLITE_DONE ){ - rc = sqlite3BtreeUpdateMeta(p->pDest,1,p->iDestSchema+1); + if( nSrcPage==0 ){ + rc = sqlite3BtreeNewDb(p->pDest); + nSrcPage = 1; + } + if( rc==SQLITE_OK || rc==SQLITE_DONE ){ + rc = sqlite3BtreeUpdateMeta(p->pDest,1,p->iDestSchema+1); + } if( rc==SQLITE_OK ){ if( p->pDestDb ){ sqlite3ResetAllSchemasOfConnection(p->pDestDb); @@ -444,7 +462,7 @@ int sqlite3_backup_step(sqlite3_backup *p, int nPage){ }else{ nDestTruncate = nSrcPage * (pgszSrc/pgszDest); } - sqlite3PagerTruncateImage(pDestPager, nDestTruncate); + assert( nDestTruncate>0 ); if( pgszSrc= iSize || ( + assert( nDestTruncate==0 + || (i64)nDestTruncate*(i64)pgszDest >= iSize || ( nDestTruncate==(int)(PENDING_BYTE_PAGE(p->pDest->pBt)-1) && iSize>=PENDING_BYTE && iSize<=PENDING_BYTE+pgszDest )); - /* This call ensures that all data required to recreate the original + /* This block ensures that all data required to recreate the original ** database has been stored in the journal for pDestPager and the ** journal synced to disk. So at this point we may safely modify ** the database file in any way, knowing that if a power failure ** occurs, the original database will be reconstructed from the ** journal file. */ - rc = sqlite3PagerCommitPhaseOne(pDestPager, 0, 1); + sqlite3PagerPagecount(pDestPager, &nDstPage); + for(iPg=nDestTruncate; rc==SQLITE_OK && iPg<=(Pgno)nDstPage; iPg++){ + if( iPg!=PENDING_BYTE_PAGE(p->pDest->pBt) ){ + DbPage *pPg; + rc = sqlite3PagerGet(pDestPager, iPg, &pPg); + if( rc==SQLITE_OK ){ + rc = sqlite3PagerWrite(pPg); + sqlite3PagerUnref(pPg); + } + } + } + if( rc==SQLITE_OK ){ + rc = sqlite3PagerCommitPhaseOne(pDestPager, 0, 1); + } /* Write the extra pages and truncate the database file as required */ iEnd = MIN(PENDING_BYTE + pgszDest, iSize); @@ -500,6 +534,7 @@ int sqlite3_backup_step(sqlite3_backup *p, int nPage){ rc = sqlite3PagerSync(pDestPager); } }else{ + sqlite3PagerTruncateImage(pDestPager, nDestTruncate); rc = sqlite3PagerCommitPhaseOne(pDestPager, 0, 0); } @@ -628,7 +663,7 @@ void sqlite3BackupUpdate(sqlite3_backup *pBackup, Pgno iPage, const u8 *aData){ int rc; assert( p->pDestDb ); sqlite3_mutex_enter(p->pDestDb->mutex); - rc = backupOnePage(p, iPage, aData); + rc = backupOnePage(p, iPage, aData, 1); sqlite3_mutex_leave(p->pDestDb->mutex); assert( rc!=SQLITE_BUSY && rc!=SQLITE_LOCKED ); if( rc!=SQLITE_OK ){ diff --git a/src/bitvec.c b/src/bitvec.c index 8d805a6..52184aa 100644 --- a/src/bitvec.c +++ b/src/bitvec.c @@ -72,7 +72,7 @@ /* ** A bitmap is an instance of the following structure. ** -** This bitmap records the existance of zero or more bits +** This bitmap records the existence of zero or more bits ** with values between 1 and iSize, inclusive. ** ** There are three possible representations of the bitmap. diff --git a/src/btree.c b/src/btree.c index d34d6ee..3ca6058 100644 --- a/src/btree.c +++ b/src/btree.c @@ -43,6 +43,25 @@ int sqlite3BtreeTrace=1; /* True to enable tracing */ */ #define get2byteNotZero(X) (((((int)get2byte(X))-1)&0xffff)+1) +/* +** Values passed as the 5th argument to allocateBtreePage() +*/ +#define BTALLOC_ANY 0 /* Allocate any page */ +#define BTALLOC_EXACT 1 /* Allocate exact page if possible */ +#define BTALLOC_LE 2 /* Allocate any page <= the parameter */ + +/* +** Macro IfNotOmitAV(x) returns (x) if SQLITE_OMIT_AUTOVACUUM is not +** defined, or 0 if it is. For example: +** +** bIncrVacuum = IfNotOmitAV(pBtShared->incrVacuum); +*/ +#ifndef SQLITE_OMIT_AUTOVACUUM +#define IfNotOmitAV(expr) (expr) +#else +#define IfNotOmitAV(expr) 0 +#endif + #ifndef SQLITE_OMIT_SHARED_CACHE /* ** A list of BtShared objects that are eligible for participation @@ -556,6 +575,19 @@ static void btreeClearHasContent(BtShared *pBt){ pBt->pHasContent = 0; } +/* +** Release all of the apPage[] pages for a cursor. +*/ +static void btreeReleaseAllCursorPages(BtCursor *pCur){ + int i; + for(i=0; i<=pCur->iPage; i++){ + releasePage(pCur->apPage[i]); + pCur->apPage[i] = 0; + } + pCur->iPage = -1; +} + + /* ** Save the current cursor position in the variables BtCursor.nKey ** and BtCursor.pKey. The cursor's state is set to CURSOR_REQUIRESEEK. @@ -595,12 +627,7 @@ static int saveCursorPosition(BtCursor *pCur){ assert( !pCur->apPage[0]->intKey || !pCur->pKey ); if( rc==SQLITE_OK ){ - int i; - for(i=0; i<=pCur->iPage; i++){ - releasePage(pCur->apPage[i]); - pCur->apPage[i] = 0; - } - pCur->iPage = -1; + btreeReleaseAllCursorPages(pCur); pCur->eState = CURSOR_REQUIRESEEK; } @@ -618,11 +645,15 @@ static int saveAllCursors(BtShared *pBt, Pgno iRoot, BtCursor *pExcept){ assert( sqlite3_mutex_held(pBt->mutex) ); assert( pExcept==0 || pExcept->pBt==pBt ); for(p=pBt->pCursor; p; p=p->pNext){ - if( p!=pExcept && (0==iRoot || p->pgnoRoot==iRoot) && - p->eState==CURSOR_VALID ){ - int rc = saveCursorPosition(p); - if( SQLITE_OK!=rc ){ - return rc; + if( p!=pExcept && (0==iRoot || p->pgnoRoot==iRoot) ){ + if( p->eState==CURSOR_VALID ){ + int rc = saveCursorPosition(p); + if( SQLITE_OK!=rc ){ + return rc; + } + }else{ + testcase( p->iPage>0 ); + btreeReleaseAllCursorPages(p); } } } @@ -1550,13 +1581,17 @@ static int btreeGetPage( BtShared *pBt, /* The btree */ Pgno pgno, /* Number of the page to fetch */ MemPage **ppPage, /* Return the page in this parameter */ - int noContent /* Do not load page content if true */ + int noContent, /* Do not load page content if true */ + int bReadonly /* True if a read-only (mmap) page is ok */ ){ int rc; DbPage *pDbPage; + int flags = (noContent ? PAGER_ACQUIRE_NOCONTENT : 0) + | (bReadonly ? PAGER_ACQUIRE_READONLY : 0); + assert( noContent==0 || bReadonly==0 ); assert( sqlite3_mutex_held(pBt->mutex) ); - rc = sqlite3PagerAcquire(pBt->pPager, pgno, (DbPage**)&pDbPage, noContent); + rc = sqlite3PagerAcquire(pBt->pPager, pgno, (DbPage**)&pDbPage, flags); if( rc ) return rc; *ppPage = btreePageFromDbPage(pDbPage, pgno, pBt); return SQLITE_OK; @@ -1599,9 +1634,10 @@ u32 sqlite3BtreeLastPage(Btree *p){ ** may remain unchanged, or it may be set to an invalid value. */ static int getAndInitPage( - BtShared *pBt, /* The database file */ - Pgno pgno, /* Number of the page to get */ - MemPage **ppPage /* Write the page pointer here */ + BtShared *pBt, /* The database file */ + Pgno pgno, /* Number of the page to get */ + MemPage **ppPage, /* Write the page pointer here */ + int bReadonly /* True if a read-only (mmap) page is ok */ ){ int rc; assert( sqlite3_mutex_held(pBt->mutex) ); @@ -1609,7 +1645,7 @@ static int getAndInitPage( if( pgno>btreePagecount(pBt) ){ rc = SQLITE_CORRUPT_BKPT; }else{ - rc = btreeGetPage(pBt, pgno, ppPage, 0); + rc = btreeGetPage(pBt, pgno, ppPage, 0, bReadonly); if( rc==SQLITE_OK ){ rc = btreeInitPage(*ppPage); if( rc!=SQLITE_OK ){ @@ -1840,6 +1876,7 @@ int sqlite3BtreeOpen( rc = sqlite3PagerOpen(pVfs, &pBt->pPager, zFilename, EXTRA_SIZE, flags, vfsFlags, pageReinit); if( rc==SQLITE_OK ){ + sqlite3PagerSetMmapLimit(pBt->pPager, db->szMmap); rc = sqlite3PagerReadFileheader(pBt->pPager,sizeof(zDbHeader),zDbHeader); } if( rc!=SQLITE_OK ){ @@ -2106,6 +2143,19 @@ int sqlite3BtreeSetCacheSize(Btree *p, int mxPage){ return SQLITE_OK; } +/* +** Change the limit on the amount of the database file that may be +** memory mapped. +*/ +int sqlite3BtreeSetMmapLimit(Btree *p, sqlite3_int64 szMmap){ + BtShared *pBt = p->pBt; + assert( sqlite3_mutex_held(p->db->mutex) ); + sqlite3BtreeEnter(p); + sqlite3PagerSetMmapLimit(pBt->pPager, szMmap); + sqlite3BtreeLeave(p); + return SQLITE_OK; +} + /* ** Change the way data is synced to disk in order to increase or decrease ** how well the database resists damage due to OS crashes and power @@ -2200,6 +2250,24 @@ int sqlite3BtreeGetPageSize(Btree *p){ return p->pBt->pageSize; } +#if defined(SQLITE_HAS_CODEC) || defined(SQLITE_DEBUG) +/* +** This function is similar to sqlite3BtreeGetReserve(), except that it +** may only be called if it is guaranteed that the b-tree mutex is already +** held. +** +** This is useful in one special case in the backup API code where it is +** known that the shared b-tree mutex is held, but the mutex on the +** database handle that owns *p is not. In this case if sqlite3BtreeEnter() +** were to be called, it might collide with some other operation on the +** database handle that owns *p, causing undefined behavior. +*/ +int sqlite3BtreeGetReserveNoMutex(Btree *p){ + assert( sqlite3_mutex_held(p->pBt->mutex) ); + return p->pBt->pageSize - p->pBt->usableSize; +} +#endif /* SQLITE_HAS_CODEC || SQLITE_DEBUG */ + #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) || !defined(SQLITE_OMIT_VACUUM) /* ** Return the number of bytes of space at the end of every page that @@ -2313,7 +2381,7 @@ static int lockBtree(BtShared *pBt){ assert( pBt->pPage1==0 ); rc = sqlite3PagerSharedLock(pBt->pPager); if( rc!=SQLITE_OK ) return rc; - rc = btreeGetPage(pBt, 1, &pPage1, 0); + rc = btreeGetPage(pBt, 1, &pPage1, 0, 0); if( rc!=SQLITE_OK ) return rc; /* Do some checking to help insure the file we opened really is @@ -2449,6 +2517,29 @@ page1_init_failed: return rc; } +#ifndef NDEBUG +/* +** Return the number of cursors open on pBt. This is for use +** in assert() expressions, so it is only compiled if NDEBUG is not +** defined. +** +** Only write cursors are counted if wrOnly is true. If wrOnly is +** false then all cursors are counted. +** +** For the purposes of this routine, a cursor is any cursor that +** is capable of reading or writing to the databse. Cursors that +** have been tripped into the CURSOR_FAULT state are not counted. +*/ +static int countValidCursors(BtShared *pBt, int wrOnly){ + BtCursor *pCur; + int r = 0; + for(pCur=pBt->pCursor; pCur; pCur=pCur->pNext){ + if( (wrOnly==0 || pCur->wrFlag) && pCur->eState!=CURSOR_FAULT ) r++; + } + return r; +} +#endif + /* ** If there are no outstanding cursors and we are not in the middle ** of a transaction but there is a read lock on the database, then @@ -2459,7 +2550,7 @@ page1_init_failed: */ static void unlockBtreeIfUnused(BtShared *pBt){ assert( sqlite3_mutex_held(pBt->mutex) ); - assert( pBt->pCursor==0 || pBt->inTransaction>TRANS_NONE ); + assert( countValidCursors(pBt,0)==0 || pBt->inTransaction>TRANS_NONE ); if( pBt->inTransaction==TRANS_NONE && pBt->pPage1!=0 ){ assert( pBt->pPage1->aData ); assert( sqlite3PagerRefcount(pBt->pPager)==1 ); @@ -2513,6 +2604,20 @@ static int newDatabase(BtShared *pBt){ return SQLITE_OK; } +/* +** Initialize the first page of the database file (creating a database +** consisting of a single page and no schema objects). Return SQLITE_OK +** if successful, or an SQLite error code otherwise. +*/ +int sqlite3BtreeNewDb(Btree *p){ + int rc; + sqlite3BtreeEnter(p); + p->pBt->nPage = 0; + rc = newDatabase(p->pBt); + sqlite3BtreeLeave(p); + return rc; +} + /* ** Attempt to start a new transaction. A write-transaction ** is started if the second argument is nonzero, otherwise a read- @@ -2563,6 +2668,7 @@ int sqlite3BtreeBeginTrans(Btree *p, int wrflag){ if( p->inTrans==TRANS_WRITE || (p->inTrans==TRANS_READ && !wrflag) ){ goto trans_begun; } + assert( IfNotOmitAV(pBt->bDoTruncate)==0 ); /* Write transactions are not possible on a read-only database */ if( (pBt->btsFlags & BTS_READ_ONLY)!=0 && wrflag ){ @@ -2857,7 +2963,7 @@ static int relocatePage( ** iPtrPage. */ if( eType!=PTRMAP_ROOTPAGE ){ - rc = btreeGetPage(pBt, iPtrPage, &pPtrPage, 0); + rc = btreeGetPage(pBt, iPtrPage, &pPtrPage, 0, 0); if( rc!=SQLITE_OK ){ return rc; } @@ -2879,24 +2985,23 @@ static int relocatePage( static int allocateBtreePage(BtShared *, MemPage **, Pgno *, Pgno, u8); /* -** Perform a single step of an incremental-vacuum. If successful, -** return SQLITE_OK. If there is no work to do (and therefore no -** point in calling this function again), return SQLITE_DONE. +** Perform a single step of an incremental-vacuum. If successful, return +** SQLITE_OK. If there is no work to do (and therefore no point in +** calling this function again), return SQLITE_DONE. Or, if an error +** occurs, return some other error code. ** -** More specificly, this function attempts to re-organize the -** database so that the last page of the file currently in use -** is no longer in use. +** More specificly, this function attempts to re-organize the database so +** that the last page of the file currently in use is no longer in use. ** -** If the nFin parameter is non-zero, this function assumes -** that the caller will keep calling incrVacuumStep() until -** it returns SQLITE_DONE or an error, and that nFin is the -** number of pages the database file will contain after this -** process is complete. If nFin is zero, it is assumed that -** incrVacuumStep() will be called a finite amount of times -** which may or may not empty the freelist. A full autovacuum -** has nFin>0. A "PRAGMA incremental_vacuum" has nFin==0. +** Parameter nFin is the number of pages that this database would contain +** were this function called until it returns SQLITE_DONE. +** +** If the bCommit parameter is non-zero, this function assumes that the +** caller will keep calling incrVacuumStep() until it returns SQLITE_DONE +** or an error. bCommit is passed true for an auto-vacuum-on-commmit +** operation, or false for an incremental vacuum. */ -static int incrVacuumStep(BtShared *pBt, Pgno nFin, Pgno iLastPg){ +static int incrVacuumStep(BtShared *pBt, Pgno nFin, Pgno iLastPg, int bCommit){ Pgno nFreeList; /* Number of pages still on the free-list */ int rc; @@ -2921,15 +3026,15 @@ static int incrVacuumStep(BtShared *pBt, Pgno nFin, Pgno iLastPg){ } if( eType==PTRMAP_FREEPAGE ){ - if( nFin==0 ){ + if( bCommit==0 ){ /* Remove the page from the files free-list. This is not required - ** if nFin is non-zero. In that case, the free-list will be + ** if bCommit is non-zero. In that case, the free-list will be ** truncated to zero after this function returns, so it doesn't ** matter if it still contains some garbage entries. */ Pgno iFreePg; MemPage *pFreePg; - rc = allocateBtreePage(pBt, &pFreePg, &iFreePg, iLastPg, 1); + rc = allocateBtreePage(pBt, &pFreePg, &iFreePg, iLastPg, BTALLOC_EXACT); if( rc!=SQLITE_OK ){ return rc; } @@ -2939,34 +3044,37 @@ static int incrVacuumStep(BtShared *pBt, Pgno nFin, Pgno iLastPg){ } else { Pgno iFreePg; /* Index of free page to move pLastPg to */ MemPage *pLastPg; + u8 eMode = BTALLOC_ANY; /* Mode parameter for allocateBtreePage() */ + Pgno iNear = 0; /* nearby parameter for allocateBtreePage() */ - rc = btreeGetPage(pBt, iLastPg, &pLastPg, 0); + rc = btreeGetPage(pBt, iLastPg, &pLastPg, 0, 0); if( rc!=SQLITE_OK ){ return rc; } - /* If nFin is zero, this loop runs exactly once and page pLastPg + /* If bCommit is zero, this loop runs exactly once and page pLastPg ** is swapped with the first free page pulled off the free list. ** - ** On the other hand, if nFin is greater than zero, then keep + ** On the other hand, if bCommit is greater than zero, then keep ** looping until a free-page located within the first nFin pages ** of the file is found. */ + if( bCommit==0 ){ + eMode = BTALLOC_LE; + iNear = nFin; + } do { MemPage *pFreePg; - rc = allocateBtreePage(pBt, &pFreePg, &iFreePg, 0, 0); + rc = allocateBtreePage(pBt, &pFreePg, &iFreePg, iNear, eMode); if( rc!=SQLITE_OK ){ releasePage(pLastPg); return rc; } releasePage(pFreePg); - }while( nFin!=0 && iFreePg>nFin ); + }while( bCommit && iFreePg>nFin ); assert( iFreePgpDbPage); - if( rc==SQLITE_OK ){ - rc = relocatePage(pBt, pLastPg, eType, iPtrPage, iFreePg, nFin!=0); - } + rc = relocatePage(pBt, pLastPg, eType, iPtrPage, iFreePg, bCommit); releasePage(pLastPg); if( rc!=SQLITE_OK ){ return rc; @@ -2974,29 +3082,39 @@ static int incrVacuumStep(BtShared *pBt, Pgno nFin, Pgno iLastPg){ } } - if( nFin==0 ){ - iLastPg--; - while( iLastPg==PENDING_BYTE_PAGE(pBt)||PTRMAP_ISPAGE(pBt, iLastPg) ){ - if( PTRMAP_ISPAGE(pBt, iLastPg) ){ - MemPage *pPg; - rc = btreeGetPage(pBt, iLastPg, &pPg, 0); - if( rc!=SQLITE_OK ){ - return rc; - } - rc = sqlite3PagerWrite(pPg->pDbPage); - releasePage(pPg); - if( rc!=SQLITE_OK ){ - return rc; - } - } + if( bCommit==0 ){ + do { iLastPg--; - } - sqlite3PagerTruncateImage(pBt->pPager, iLastPg); + }while( iLastPg==PENDING_BYTE_PAGE(pBt) || PTRMAP_ISPAGE(pBt, iLastPg) ); + pBt->bDoTruncate = 1; pBt->nPage = iLastPg; } return SQLITE_OK; } +/* +** The database opened by the first argument is an auto-vacuum database +** nOrig pages in size containing nFree free pages. Return the expected +** size of the database in pages following an auto-vacuum operation. +*/ +static Pgno finalDbSize(BtShared *pBt, Pgno nOrig, Pgno nFree){ + int nEntry; /* Number of entries on one ptrmap page */ + Pgno nPtrmap; /* Number of PtrMap pages to be freed */ + Pgno nFin; /* Return value */ + + nEntry = pBt->usableSize/5; + nPtrmap = (nFree-nOrig+PTRMAP_PAGENO(pBt, nOrig)+nEntry)/nEntry; + nFin = nOrig - nFree - nPtrmap; + if( nOrig>PENDING_BYTE_PAGE(pBt) && nFinautoVacuum ){ rc = SQLITE_DONE; }else{ - invalidateAllOverflowCache(pBt); - rc = incrVacuumStep(pBt, 0, btreePagecount(pBt)); - if( rc==SQLITE_OK ){ - rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); - put4byte(&pBt->pPage1->aData[28], pBt->nPage); + Pgno nOrig = btreePagecount(pBt); + Pgno nFree = get4byte(&pBt->pPage1->aData[36]); + Pgno nFin = finalDbSize(pBt, nOrig, nFree); + + if( nOrig0 ){ + rc = saveAllCursors(pBt, 0, 0); + if( rc==SQLITE_OK ){ + invalidateAllOverflowCache(pBt); + rc = incrVacuumStep(pBt, nFin, nOrig, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); + put4byte(&pBt->pPage1->aData[28], pBt->nPage); + } + }else{ + rc = SQLITE_DONE; } } sqlite3BtreeLeave(p); @@ -3045,9 +3176,7 @@ static int autoVacuumCommit(BtShared *pBt){ if( !pBt->incrVacuum ){ Pgno nFin; /* Number of pages in database after autovacuuming */ Pgno nFree; /* Number of pages on the freelist initially */ - Pgno nPtrmap; /* Number of PtrMap pages to be freed */ Pgno iFree; /* The next page to be freed */ - int nEntry; /* Number of entries on one ptrmap page */ Pgno nOrig; /* Database size before freeing */ nOrig = btreePagecount(pBt); @@ -3060,26 +3189,20 @@ static int autoVacuumCommit(BtShared *pBt){ } nFree = get4byte(&pBt->pPage1->aData[36]); - nEntry = pBt->usableSize/5; - nPtrmap = (nFree-nOrig+PTRMAP_PAGENO(pBt, nOrig)+nEntry)/nEntry; - nFin = nOrig - nFree - nPtrmap; - if( nOrig>PENDING_BYTE_PAGE(pBt) && nFinnOrig ) return SQLITE_CORRUPT_BKPT; - + if( nFinnFin && rc==SQLITE_OK; iFree--){ - rc = incrVacuumStep(pBt, nFin, iFree); + rc = incrVacuumStep(pBt, nFin, iFree, 1); } if( (rc==SQLITE_DONE || rc==SQLITE_OK) && nFree>0 ){ rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); put4byte(&pBt->pPage1->aData[32], 0); put4byte(&pBt->pPage1->aData[36], 0); put4byte(&pBt->pPage1->aData[28], nFin); - sqlite3PagerTruncateImage(pBt->pPager, nFin); + pBt->bDoTruncate = 1; pBt->nPage = nFin; } if( rc!=SQLITE_OK ){ @@ -3087,7 +3210,7 @@ static int autoVacuumCommit(BtShared *pBt){ } } - assert( nRef==sqlite3PagerRefcount(pPager) ); + assert( nRef>=sqlite3PagerRefcount(pPager) ); return rc; } @@ -3134,6 +3257,9 @@ int sqlite3BtreeCommitPhaseOne(Btree *p, const char *zMaster){ return rc; } } + if( pBt->bDoTruncate ){ + sqlite3PagerTruncateImage(pBt->pPager, pBt->nPage); + } #endif rc = sqlite3PagerCommitPhaseOne(pBt->pPager, zMaster, 0); sqlite3BtreeLeave(p); @@ -3149,7 +3275,9 @@ static void btreeEndTransaction(Btree *p){ BtShared *pBt = p->pBt; assert( sqlite3BtreeHoldsMutex(p) ); - btreeClearHasContent(pBt); +#ifndef SQLITE_OMIT_AUTOVACUUM + pBt->bDoTruncate = 0; +#endif if( p->inTrans>TRANS_NONE && p->db->activeVdbeCnt>1 ){ /* If there are other active statements that belong to this database ** handle, downgrade to a read-only transaction. The other statements @@ -3224,6 +3352,7 @@ int sqlite3BtreeCommitPhaseTwo(Btree *p, int bCleanup){ return rc; } pBt->inTransaction = TRANS_READ; + btreeClearHasContent(pBt); } btreeEndTransaction(p); @@ -3245,27 +3374,6 @@ int sqlite3BtreeCommit(Btree *p){ return rc; } -#ifndef NDEBUG -/* -** Return the number of write-cursors open on this handle. This is for use -** in assert() expressions, so it is only compiled if NDEBUG is not -** defined. -** -** For the purposes of this routine, a write-cursor is any cursor that -** is capable of writing to the databse. That means the cursor was -** originally opened for writing and the cursor has not be disabled -** by having its state changed to CURSOR_FAULT. -*/ -static int countWriteCursors(BtShared *pBt){ - BtCursor *pCur; - int r = 0; - for(pCur=pBt->pCursor; pCur; pCur=pCur->pNext){ - if( pCur->wrFlag && pCur->eState!=CURSOR_FAULT ) r++; - } - return r; -} -#endif - /* ** This routine sets the state to CURSOR_FAULT and the error ** code to errCode for every cursor on BtShared that pBtree @@ -3337,7 +3445,7 @@ int sqlite3BtreeRollback(Btree *p, int tripCode){ /* The rollback may have destroyed the pPage1->aData value. So ** call btreeGetPage() on page 1 again to make ** sure pPage1->aData is set correctly. */ - if( btreeGetPage(pBt, 1, &pPage1, 0)==SQLITE_OK ){ + if( btreeGetPage(pBt, 1, &pPage1, 0, 0)==SQLITE_OK ){ int nPage = get4byte(28+(u8*)pPage1->aData); testcase( nPage==0 ); if( nPage==0 ) sqlite3PagerPagecount(pBt->pPager, &nPage); @@ -3345,8 +3453,9 @@ int sqlite3BtreeRollback(Btree *p, int tripCode){ pBt->nPage = nPage; releasePage(pPage1); } - assert( countWriteCursors(pBt)==0 ); + assert( countValidCursors(pBt, 1)==0 ); pBt->inTransaction = TRANS_READ; + btreeClearHasContent(pBt); } btreeEndTransaction(p); @@ -3771,7 +3880,7 @@ static int getOverflowPage( assert( next==0 || rc==SQLITE_DONE ); if( rc==SQLITE_OK ){ - rc = btreeGetPage(pBt, ovfl, &pPage, 0); + rc = btreeGetPage(pBt, ovfl, &pPage, 0, (ppPage==0)); assert( rc==SQLITE_OK || pPage==0 ); if( rc==SQLITE_OK ){ next = get4byte(pPage->aData); @@ -3992,7 +4101,9 @@ static int accessPayload( { DbPage *pDbPage; - rc = sqlite3PagerGet(pBt->pPager, nextPage, &pDbPage); + rc = sqlite3PagerAcquire(pBt->pPager, nextPage, &pDbPage, + (eOp==0 ? PAGER_ACQUIRE_READONLY : 0) + ); if( rc==SQLITE_OK ){ aPayload = sqlite3PagerGetData(pDbPage); nextPage = get4byte(aPayload); @@ -4171,10 +4282,11 @@ static int moveToChild(BtCursor *pCur, u32 newPgno){ assert( cursorHoldsMutex(pCur) ); assert( pCur->eState==CURSOR_VALID ); assert( pCur->iPageiPage>=0 ); if( pCur->iPage>=(BTCURSOR_MAX_DEPTH-1) ){ return SQLITE_CORRUPT_BKPT; } - rc = getAndInitPage(pBt, newPgno, &pNewPage); + rc = getAndInitPage(pBt, newPgno, &pNewPage, (pCur->wrFlag==0)); if( rc ) return rc; pCur->apPage[i+1] = pNewPage; pCur->aiIdx[i+1] = 0; @@ -4291,7 +4403,7 @@ static int moveToRoot(BtCursor *pCur){ pCur->eState = CURSOR_INVALID; return SQLITE_OK; }else{ - rc = getAndInitPage(pBt, pCur->pgnoRoot, &pCur->apPage[0]); + rc = getAndInitPage(pBt, pCur->pgnoRoot, &pCur->apPage[0], pCur->wrFlag==0); if( rc!=SQLITE_OK ){ pCur->eState = CURSOR_INVALID; return rc; @@ -4821,21 +4933,23 @@ int sqlite3BtreePrevious(BtCursor *pCur, int *pRes){ ** an error. *ppPage and *pPgno are undefined in the event of an error. ** Do not invoke sqlite3PagerUnref() on *ppPage if an error is returned. ** -** If the "nearby" parameter is not 0, then a (feeble) effort is made to +** If the "nearby" parameter is not 0, then an effort is made to ** locate a page close to the page number "nearby". This can be used in an ** attempt to keep related pages close to each other in the database file, ** which in turn can make database access faster. ** -** If the "exact" parameter is not 0, and the page-number nearby exists -** anywhere on the free-list, then it is guarenteed to be returned. This -** is only used by auto-vacuum databases when allocating a new table. +** If the eMode parameter is BTALLOC_EXACT and the nearby page exists +** anywhere on the free-list, then it is guaranteed to be returned. If +** eMode is BTALLOC_LT then the page returned will be less than or equal +** to nearby if any such page exists. If eMode is BTALLOC_ANY then there +** are no restrictions on which page is returned. */ static int allocateBtreePage( - BtShared *pBt, - MemPage **ppPage, - Pgno *pPgno, - Pgno nearby, - u8 exact + BtShared *pBt, /* The btree */ + MemPage **ppPage, /* Store pointer to the allocated page here */ + Pgno *pPgno, /* Store the page number here */ + Pgno nearby, /* Search for a page near this one */ + u8 eMode /* BTALLOC_EXACT, BTALLOC_LT, or BTALLOC_ANY */ ){ MemPage *pPage1; int rc; @@ -4846,6 +4960,7 @@ static int allocateBtreePage( Pgno mxPage; /* Total size of the database file */ assert( sqlite3_mutex_held(pBt->mutex) ); + assert( eMode==BTALLOC_ANY || (nearby>0 && IfNotOmitAV(pBt->autoVacuum)) ); pPage1 = pBt->pPage1; mxPage = btreePagecount(pBt); n = get4byte(&pPage1->aData[36]); @@ -4858,21 +4973,24 @@ static int allocateBtreePage( Pgno iTrunk; u8 searchList = 0; /* If the free-list must be searched for 'nearby' */ - /* If the 'exact' parameter was true and a query of the pointer-map + /* If eMode==BTALLOC_EXACT and a query of the pointer-map ** shows that the page 'nearby' is somewhere on the free-list, then ** the entire-list will be searched for that page. */ #ifndef SQLITE_OMIT_AUTOVACUUM - if( exact && nearby<=mxPage ){ - u8 eType; - assert( nearby>0 ); - assert( pBt->autoVacuum ); - rc = ptrmapGet(pBt, nearby, &eType, 0); - if( rc ) return rc; - if( eType==PTRMAP_FREEPAGE ){ - searchList = 1; + if( eMode==BTALLOC_EXACT ){ + if( nearby<=mxPage ){ + u8 eType; + assert( nearby>0 ); + assert( pBt->autoVacuum ); + rc = ptrmapGet(pBt, nearby, &eType, 0); + if( rc ) return rc; + if( eType==PTRMAP_FREEPAGE ){ + searchList = 1; + } } - *pPgno = nearby; + }else if( eMode==BTALLOC_LE ){ + searchList = 1; } #endif @@ -4885,7 +5003,8 @@ static int allocateBtreePage( /* The code within this loop is run only once if the 'searchList' variable ** is not true. Otherwise, it runs once for each trunk-page on the - ** free-list until the page 'nearby' is located. + ** free-list until the page 'nearby' is located (eMode==BTALLOC_EXACT) + ** or until a page less than 'nearby' is located (eMode==BTALLOC_LT) */ do { pPrevTrunk = pTrunk; @@ -4898,7 +5017,7 @@ static int allocateBtreePage( if( iTrunk>mxPage ){ rc = SQLITE_CORRUPT_BKPT; }else{ - rc = btreeGetPage(pBt, iTrunk, &pTrunk, 0); + rc = btreeGetPage(pBt, iTrunk, &pTrunk, 0, 0); } if( rc ){ pTrunk = 0; @@ -4927,11 +5046,13 @@ static int allocateBtreePage( rc = SQLITE_CORRUPT_BKPT; goto end_allocate_page; #ifndef SQLITE_OMIT_AUTOVACUUM - }else if( searchList && nearby==iTrunk ){ + }else if( searchList + && (nearby==iTrunk || (iTrunkpDbPage); @@ -4960,7 +5081,7 @@ static int allocateBtreePage( goto end_allocate_page; } testcase( iNewTrunk==mxPage ); - rc = btreeGetPage(pBt, iNewTrunk, &pNewTrunk, 0); + rc = btreeGetPage(pBt, iNewTrunk, &pNewTrunk, 0, 0); if( rc!=SQLITE_OK ){ goto end_allocate_page; } @@ -4994,14 +5115,24 @@ static int allocateBtreePage( unsigned char *aData = pTrunk->aData; if( nearby>0 ){ u32 i; - int dist; closest = 0; - dist = sqlite3AbsInt32(get4byte(&aData[8]) - nearby); - for(i=1; ipDbPage); if( rc!=SQLITE_OK ){ @@ -5042,8 +5175,26 @@ static int allocateBtreePage( pPrevTrunk = 0; }while( searchList ); }else{ - /* There are no pages on the freelist, so create a new page at the - ** end of the file */ + /* There are no pages on the freelist, so append a new page to the + ** database image. + ** + ** Normally, new pages allocated by this block can be requested from the + ** pager layer with the 'no-content' flag set. This prevents the pager + ** from trying to read the pages content from disk. However, if the + ** current transaction has already run one or more incremental-vacuum + ** steps, then the page we are about to allocate may contain content + ** that is required in the event of a rollback. In this case, do + ** not set the no-content flag. This causes the pager to load and journal + ** the current page content before overwriting it. + ** + ** Note that the pager will not actually attempt to load or journal + ** content for any page that really does lie past the end of the database + ** file on disk. So the effects of disabling the no-content optimization + ** here are confined to those pages that lie between the end of the + ** database image and the end of the database file. + */ + int bNoContent = (0==IfNotOmitAV(pBt->bDoTruncate)); + rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); if( rc ) return rc; pBt->nPage++; @@ -5058,7 +5209,7 @@ static int allocateBtreePage( MemPage *pPg = 0; TRACE(("ALLOCATE: %d from end of file (pointer-map page)\n", pBt->nPage)); assert( pBt->nPage!=PENDING_BYTE_PAGE(pBt) ); - rc = btreeGetPage(pBt, pBt->nPage, &pPg, 1); + rc = btreeGetPage(pBt, pBt->nPage, &pPg, bNoContent, 0); if( rc==SQLITE_OK ){ rc = sqlite3PagerWrite(pPg->pDbPage); releasePage(pPg); @@ -5072,7 +5223,7 @@ static int allocateBtreePage( *pPgno = pBt->nPage; assert( *pPgno!=PENDING_BYTE_PAGE(pBt) ); - rc = btreeGetPage(pBt, *pPgno, ppPage, 1); + rc = btreeGetPage(pBt, *pPgno, ppPage, bNoContent, 0); if( rc ) return rc; rc = sqlite3PagerWrite((*ppPage)->pDbPage); if( rc!=SQLITE_OK ){ @@ -5140,7 +5291,7 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ /* If the secure_delete option is enabled, then ** always fully overwrite deleted information with zeros. */ - if( (!pPage && ((rc = btreeGetPage(pBt, iPage, &pPage, 0))!=0) ) + if( (!pPage && ((rc = btreeGetPage(pBt, iPage, &pPage, 0, 0))!=0) ) || ((rc = sqlite3PagerWrite(pPage->pDbPage))!=0) ){ goto freepage_out; @@ -5167,7 +5318,7 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ u32 nLeaf; /* Initial number of leaf cells on trunk page */ iTrunk = get4byte(&pPage1->aData[32]); - rc = btreeGetPage(pBt, iTrunk, &pTrunk, 0); + rc = btreeGetPage(pBt, iTrunk, &pTrunk, 0, 0); if( rc!=SQLITE_OK ){ goto freepage_out; } @@ -5213,7 +5364,7 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ ** first trunk in the free-list is full. Either way, the page being freed ** will become the new first trunk page in the free-list. */ - if( pPage==0 && SQLITE_OK!=(rc = btreeGetPage(pBt, iPage, &pPage, 0)) ){ + if( pPage==0 && SQLITE_OK!=(rc = btreeGetPage(pBt, iPage, &pPage, 0, 0)) ){ goto freepage_out; } rc = sqlite3PagerWrite(pPage->pDbPage); @@ -5256,7 +5407,7 @@ static int clearCell(MemPage *pPage, unsigned char *pCell){ return SQLITE_OK; /* No overflow pages. Return without doing anything */ } if( pCell+info.iOverflow+3 > pPage->aData+pPage->maskPage ){ - return SQLITE_CORRUPT; /* Cell extends past end of page */ + return SQLITE_CORRUPT_BKPT; /* Cell extends past end of page */ } ovflPgno = get4byte(&pCell[info.iOverflow]); assert( pBt->usableSize > 4 ); @@ -5400,7 +5551,7 @@ static int fillInCell( ** If this is the first overflow page, then write a partial entry ** to the pointer-map. If we write nothing to this pointer-map slot, ** then the optimistic overflow chain processing in clearCell() - ** may misinterpret the uninitialised values and delete the + ** may misinterpret the uninitialized values and delete the ** wrong pages from the database. */ if( pBt->autoVacuum && rc==SQLITE_OK ){ @@ -5712,7 +5863,7 @@ static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){ assert( pPage->nOverflow==1 ); /* This error condition is now caught prior to reaching this function */ - if( pPage->nCell<=0 ) return SQLITE_CORRUPT_BKPT; + if( pPage->nCell==0 ) return SQLITE_CORRUPT_BKPT; /* Allocate a new page. This page will become the right-sibling of ** pPage. Make the parent page writable, so that the new divider cell @@ -6014,7 +6165,7 @@ static int balance_nonroot( } pgno = get4byte(pRight); while( 1 ){ - rc = getAndInitPage(pBt, pgno, &apOld[i]); + rc = getAndInitPage(pBt, pgno, &apOld[i], 0); if( rc ){ memset(apOld, 0, (i+1)*sizeof(MemPage*)); goto balance_cleanup; @@ -6873,7 +7024,7 @@ int sqlite3BtreeInsert( insertCell(pPage, idx, newCell, szNew, 0, 0, &rc); assert( rc!=SQLITE_OK || pPage->nCell>0 || pPage->nOverflow>0 ); - /* If no error has occured and pPage has an overflow cell, call balance() + /* If no error has occurred and pPage has an overflow cell, call balance() ** to redistribute the cells within the tree. Since balance() may move ** the cursor, zero the BtCursor.info.nSize and BtCursor.validNKey ** variables. @@ -7087,7 +7238,7 @@ static int btreeCreateTable(Btree *p, int *piTable, int createTabFlags){ ** be moved to the allocated page (unless the allocated page happens ** to reside at pgnoRoot). */ - rc = allocateBtreePage(pBt, &pPageMove, &pgnoMove, pgnoRoot, 1); + rc = allocateBtreePage(pBt, &pPageMove, &pgnoMove, pgnoRoot, BTALLOC_EXACT); if( rc!=SQLITE_OK ){ return rc; } @@ -7102,10 +7253,17 @@ static int btreeCreateTable(Btree *p, int *piTable, int createTabFlags){ u8 eType = 0; Pgno iPtrPage = 0; + /* Save the positions of any open cursors. This is required in + ** case they are holding a reference to an xFetch reference + ** corresponding to page pgnoRoot. */ + rc = saveAllCursors(pBt, 0, 0); releasePage(pPageMove); + if( rc!=SQLITE_OK ){ + return rc; + } /* Move the page currently at pgnoRoot to pgnoMove. */ - rc = btreeGetPage(pBt, pgnoRoot, &pRoot, 0); + rc = btreeGetPage(pBt, pgnoRoot, &pRoot, 0, 0); if( rc!=SQLITE_OK ){ return rc; } @@ -7126,7 +7284,7 @@ static int btreeCreateTable(Btree *p, int *piTable, int createTabFlags){ if( rc!=SQLITE_OK ){ return rc; } - rc = btreeGetPage(pBt, pgnoRoot, &pRoot, 0); + rc = btreeGetPage(pBt, pgnoRoot, &pRoot, 0, 0); if( rc!=SQLITE_OK ){ return rc; } @@ -7202,7 +7360,7 @@ static int clearDatabasePage( return SQLITE_CORRUPT_BKPT; } - rc = getAndInitPage(pBt, pgno, &pPage); + rc = getAndInitPage(pBt, pgno, &pPage, 0); if( rc ) return rc; for(i=0; inCell; i++){ pCell = findCell(pPage, i); @@ -7304,7 +7462,7 @@ static int btreeDropTable(Btree *p, Pgno iTable, int *piMoved){ return SQLITE_LOCKED_SHAREDCACHE; } - rc = btreeGetPage(pBt, (Pgno)iTable, &pPage, 0); + rc = btreeGetPage(pBt, (Pgno)iTable, &pPage, 0, 0); if( rc ) return rc; rc = sqlite3BtreeClearTable(p, iTable, 0); if( rc ){ @@ -7339,7 +7497,7 @@ static int btreeDropTable(Btree *p, Pgno iTable, int *piMoved){ */ MemPage *pMove; releasePage(pPage); - rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0); + rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0, 0); if( rc!=SQLITE_OK ){ return rc; } @@ -7349,7 +7507,7 @@ static int btreeDropTable(Btree *p, Pgno iTable, int *piMoved){ return rc; } pMove = 0; - rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0); + rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0, 0); freePage(pMove, &rc); releasePage(pMove); if( rc!=SQLITE_OK ){ @@ -7761,7 +7919,7 @@ static int checkTreePage( usableSize = pBt->usableSize; if( iPage==0 ) return 0; if( checkRef(pCheck, iPage, zParentContext) ) return 0; - if( (rc = btreeGetPage(pBt, (Pgno)iPage, &pPage, 0))!=0 ){ + if( (rc = btreeGetPage(pBt, (Pgno)iPage, &pPage, 0, 0))!=0 ){ checkAppendMsg(pCheck, zContext, "unable to get the page. error code=%d", rc); return 0; @@ -7994,7 +8152,7 @@ char *sqlite3BtreeIntegrityCheck( } i = PENDING_BYTE_PAGE(pBt); if( i<=sCheck.nPage ) setPageReferenced(&sCheck, i); - sqlite3StrAccumInit(&sCheck.errMsg, zErr, sizeof(zErr), 20000); + sqlite3StrAccumInit(&sCheck.errMsg, zErr, sizeof(zErr), SQLITE_MAX_LENGTH); sCheck.errMsg.useMalloc = 2; /* Check the integrity of the freelist @@ -8233,6 +8391,17 @@ int sqlite3BtreePutData(BtCursor *pCsr, u32 offset, u32 amt, void *z){ return SQLITE_ABORT; } + /* Save the positions of all other cursors open on this table. This is + ** required in case any of them are holding references to an xFetch + ** version of the b-tree page modified by the accessPayload call below. + ** + ** Note that pCsr must be open on a BTREE_INTKEY table and saveCursorPosition() + ** and hence saveAllCursors() cannot fail on a BTREE_INTKEY table, hence + ** saveAllCursors can only return SQLITE_OK. + */ + VVA_ONLY(rc =) saveAllCursors(pCsr->pBt, pCsr->pgnoRoot, pCsr); + assert( rc==SQLITE_OK ); + /* Check some assumptions: ** (a) the cursor is open for writing, ** (b) there is a read/write transaction open, diff --git a/src/btree.h b/src/btree.h index 95897d5..ace0f8c 100644 --- a/src/btree.h +++ b/src/btree.h @@ -63,6 +63,7 @@ int sqlite3BtreeOpen( int sqlite3BtreeClose(Btree*); int sqlite3BtreeSetCacheSize(Btree*,int); +int sqlite3BtreeSetMmapLimit(Btree*,sqlite3_int64); int sqlite3BtreeSetSafetyLevel(Btree*,int,int,int); int sqlite3BtreeSyncDisabled(Btree*); int sqlite3BtreeSetPageSize(Btree *p, int nPagesize, int nReserve, int eFix); @@ -71,6 +72,9 @@ int sqlite3BtreeMaxPageCount(Btree*,int); u32 sqlite3BtreeLastPage(Btree*); int sqlite3BtreeSecureDelete(Btree*,int); int sqlite3BtreeGetReserve(Btree*); +#if defined(SQLITE_HAS_CODEC) || defined(SQLITE_DEBUG) +int sqlite3BtreeGetReserveNoMutex(Btree *p); +#endif int sqlite3BtreeSetAutoVacuum(Btree *, int); int sqlite3BtreeGetAutoVacuum(Btree *); int sqlite3BtreeBeginTrans(Btree*,int); @@ -114,6 +118,8 @@ void sqlite3BtreeTripAllCursors(Btree*, int); void sqlite3BtreeGetMeta(Btree *pBtree, int idx, u32 *pValue); int sqlite3BtreeUpdateMeta(Btree*, int idx, u32 value); +int sqlite3BtreeNewDb(Btree *p); + /* ** The second parameter to sqlite3BtreeGetMeta or sqlite3BtreeUpdateMeta ** should be one of the following values. The integer values are assigned @@ -134,6 +140,7 @@ int sqlite3BtreeUpdateMeta(Btree*, int idx, u32 value); #define BTREE_TEXT_ENCODING 5 #define BTREE_USER_VERSION 6 #define BTREE_INCR_VACUUM 7 +#define BTREE_APPLICATION_ID 8 /* ** Values that may be OR'd together to form the second argument of an diff --git a/src/btreeInt.h b/src/btreeInt.h index b157dec..ce3c549 100644 --- a/src/btreeInt.h +++ b/src/btreeInt.h @@ -411,6 +411,7 @@ struct BtShared { #ifndef SQLITE_OMIT_AUTOVACUUM u8 autoVacuum; /* True if auto-vacuum is enabled */ u8 incrVacuum; /* True if incr-vacuum is enabled */ + u8 bDoTruncate; /* True to truncate db on commit */ #endif u8 inTransaction; /* Transaction state */ u8 max1bytePayload; /* Maximum first byte of cell for a 1-byte payload */ diff --git a/src/build.c b/src/build.c index 25e4740..3c91cdc 100644 --- a/src/build.c +++ b/src/build.c @@ -127,6 +127,7 @@ void sqlite3FinishCoding(Parse *pParse){ sqlite3 *db; Vdbe *v; + assert( pParse->pToplevel==0 ); db = pParse->db; if( db->mallocFailed ) return; if( pParse->nested ) return; @@ -319,6 +320,31 @@ Table *sqlite3LocateTable( return p; } +/* +** Locate the table identified by *p. +** +** This is a wrapper around sqlite3LocateTable(). The difference between +** sqlite3LocateTable() and this function is that this function restricts +** the search to schema (p->pSchema) if it is not NULL. p->pSchema may be +** non-NULL if it is part of a view or trigger program definition. See +** sqlite3FixSrcList() for details. +*/ +Table *sqlite3LocateTableItem( + Parse *pParse, + int isView, + struct SrcList_item *p +){ + const char *zDb; + assert( p->pSchema==0 || p->zDatabase==0 ); + if( p->pSchema ){ + int iDb = sqlite3SchemaToIndex(pParse->db, p->pSchema); + zDb = pParse->db->aDb[iDb].zName; + }else{ + zDb = p->zDatabase; + } + return sqlite3LocateTable(pParse, isView, p->zName, zDb); +} + /* ** Locate the in-memory structure that describes ** a particular index given the name of that index @@ -1169,7 +1195,7 @@ void sqlite3AddPrimaryKey( pTab->tabFlags |= TF_HasPrimaryKey; if( pList==0 ){ iCol = pTab->nCol - 1; - pTab->aCol[iCol].isPrimKey = 1; + pTab->aCol[iCol].colFlags |= COLFLAG_PRIMKEY; }else{ for(i=0; inExpr; i++){ for(iCol=0; iColnCol; iCol++){ @@ -1178,7 +1204,7 @@ void sqlite3AddPrimaryKey( } } if( iColnCol ){ - pTab->aCol[iCol].isPrimKey = 1; + pTab->aCol[iCol].colFlags |= COLFLAG_PRIMKEY; } } if( pList->nExpr>1 ) iCol = -1; @@ -1295,10 +1321,7 @@ CollSeq *sqlite3LocateCollSeq(Parse *pParse, const char *zName){ pColl = sqlite3FindCollSeq(db, enc, zName, initbusy); if( !initbusy && (!pColl || !pColl->xCmp) ){ - pColl = sqlite3GetCollSeq(db, enc, pColl, zName); - if( !pColl ){ - sqlite3ErrorMsg(pParse, "no such collation sequence: %s", zName); - } + pColl = sqlite3GetCollSeq(pParse, enc, pColl, zName); } return pColl; @@ -1997,6 +2020,7 @@ static void destroyTable(Parse *pParse, Table *pTab){ return; }else{ int iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); + assert( iDb>=0 && iDbdb->nDb ); destroyRootPage(pParse, iLargest, iDb); iDestroyed = iLargest; } @@ -2076,7 +2100,7 @@ void sqlite3CodeDropTable(Parse *pParse, Table *pTab, int iDb, int isView){ /* Drop all SQLITE_MASTER table and index entries that refer to the ** table. The program name loops through the master table and deletes ** every row that refers to a table of the same name as the one being - ** dropped. Triggers are handled seperately because a trigger can be + ** dropped. Triggers are handled separately because a trigger can be ** created in the temp database that refers to a table in another ** database. */ @@ -2114,8 +2138,7 @@ void sqlite3DropTable(Parse *pParse, SrcList *pName, int isView, int noErr){ assert( pParse->nErr==0 ); assert( pName->nSrc==1 ); if( noErr ) db->suppressErr++; - pTab = sqlite3LocateTable(pParse, isView, - pName->a[0].zName, pName->a[0].zDatabase); + pTab = sqlite3LocateTableItem(pParse, isView, &pName->a[0]); if( noErr ) db->suppressErr--; if( pTab==0 ){ @@ -2369,9 +2392,6 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ int tnum; /* Root page of index */ Vdbe *v; /* Generate code into this virtual machine */ KeyInfo *pKey; /* KeyInfo for index */ -#ifdef SQLITE_OMIT_MERGE_SORT - int regIdxKey; /* Registers containing the index key */ -#endif int regRecord; /* Register holding assemblied index record */ sqlite3 *db = pParse->db; /* The database connection */ int iDb = sqlite3SchemaToIndex(db, pIndex->pSchema); @@ -2399,13 +2419,9 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ (char *)pKey, P4_KEYINFO_HANDOFF); sqlite3VdbeChangeP5(v, OPFLAG_BULKCSR|((memRootPage>=0)?OPFLAG_P2ISREG:0)); -#ifndef SQLITE_OMIT_MERGE_SORT /* Open the sorter cursor if we are to use one. */ iSorter = pParse->nTab++; sqlite3VdbeAddOp4(v, OP_SorterOpen, iSorter, 0, 0, (char*)pKey, P4_KEYINFO); -#else - iSorter = iTab; -#endif /* Open the table. Loop through all rows of the table, inserting index ** records into the sorter. */ @@ -2413,7 +2429,6 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ addr1 = sqlite3VdbeAddOp2(v, OP_Rewind, iTab, 0); regRecord = sqlite3GetTempReg(pParse); -#ifndef SQLITE_OMIT_MERGE_SORT sqlite3GenerateIndexKey(pParse, pIndex, iTab, regRecord, 1); sqlite3VdbeAddOp2(v, OP_SorterInsert, iSorter, regRecord); sqlite3VdbeAddOp2(v, OP_Next, iTab, addr1+1); @@ -2424,8 +2439,8 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ sqlite3VdbeAddOp2(v, OP_Goto, 0, j2); addr2 = sqlite3VdbeCurrentAddr(v); sqlite3VdbeAddOp3(v, OP_SorterCompare, iSorter, j2, regRecord); - sqlite3HaltConstraint( - pParse, OE_Abort, "indexed columns are not unique", P4_STATIC + sqlite3HaltConstraint(pParse, SQLITE_CONSTRAINT_UNIQUE, + OE_Abort, "indexed columns are not unique", P4_STATIC ); }else{ addr2 = sqlite3VdbeCurrentAddr(v); @@ -2433,30 +2448,6 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ sqlite3VdbeAddOp2(v, OP_SorterData, iSorter, regRecord); sqlite3VdbeAddOp3(v, OP_IdxInsert, iIdx, regRecord, 1); sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); -#else - regIdxKey = sqlite3GenerateIndexKey(pParse, pIndex, iTab, regRecord, 1); - addr2 = addr1 + 1; - if( pIndex->onError!=OE_None ){ - const int regRowid = regIdxKey + pIndex->nColumn; - const int j2 = sqlite3VdbeCurrentAddr(v) + 2; - void * const pRegKey = SQLITE_INT_TO_PTR(regIdxKey); - - /* The registers accessed by the OP_IsUnique opcode were allocated - ** using sqlite3GetTempRange() inside of the sqlite3GenerateIndexKey() - ** call above. Just before that function was freed they were released - ** (made available to the compiler for reuse) using - ** sqlite3ReleaseTempRange(). So in some ways having the OP_IsUnique - ** opcode use the values stored within seems dangerous. However, since - ** we can be sure that no other temp registers have been allocated - ** since sqlite3ReleaseTempRange() was called, it is safe to do so. - */ - sqlite3VdbeAddOp4(v, OP_IsUnique, iIdx, j2, regRowid, pRegKey, P4_INT32); - sqlite3HaltConstraint( - pParse, OE_Abort, "indexed columns are not unique", P4_STATIC); - } - sqlite3VdbeAddOp3(v, OP_IdxInsert, iIdx, regRecord, 0); - sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); -#endif sqlite3ReleaseTempReg(pParse, regRecord); sqlite3VdbeAddOp2(v, OP_SorterNext, iSorter, addr2); sqlite3VdbeJumpHere(v, addr1); @@ -2555,9 +2546,9 @@ Index *sqlite3CreateIndex( ** sqlite3FixSrcList can never fail. */ assert(0); } - pTab = sqlite3LocateTable(pParse, 0, pTblName->a[0].zName, - pTblName->a[0].zDatabase); - if( !pTab || db->mallocFailed ) goto exit_create_index; + pTab = sqlite3LocateTableItem(pParse, 0, &pTblName->a[0]); + assert( db->mallocFailed==0 || pTab==0 ); + if( pTab==0 ) goto exit_create_index; assert( db->aDb[iDb].pSchema==pTab->pSchema ); }else{ assert( pName==0 ); @@ -2571,7 +2562,7 @@ Index *sqlite3CreateIndex( assert( pTab!=0 ); assert( pParse->nErr==0 ); if( sqlite3StrNICmp(pTab->zName, "sqlite_", 7)==0 - && memcmp(&pTab->zName[7],"altertab_",9)!=0 ){ + && sqlite3StrNICmp(&pTab->zName[7],"altertab_",9)!=0 ){ sqlite3ErrorMsg(pParse, "table %s may not be indexed", pTab->zName); goto exit_create_index; } @@ -2668,12 +2659,8 @@ Index *sqlite3CreateIndex( for(i=0; inExpr; i++){ Expr *pExpr = pList->a[i].pExpr; if( pExpr ){ - CollSeq *pColl = pExpr->pColl; - /* Either pColl!=0 or there was an OOM failure. But if an OOM - ** failure we have quit before reaching this point. */ - if( ALWAYS(pColl) ){ - nExtra += (1 + sqlite3Strlen30(pColl->zName)); - } + assert( pExpr->op==TK_COLLATE ); + nExtra += (1 + sqlite3Strlen30(pExpr->u.zToken)); } } @@ -2746,14 +2733,10 @@ Index *sqlite3CreateIndex( goto exit_create_index; } pIndex->aiColumn[i] = j; - /* Justification of the ALWAYS(pListItem->pExpr->pColl): Because of - ** the way the "idxlist" non-terminal is constructed by the parser, - ** if pListItem->pExpr is not null then either pListItem->pExpr->pColl - ** must exist or else there must have been an OOM error. But if there - ** was an OOM error, we would never reach this point. */ - if( pListItem->pExpr && ALWAYS(pListItem->pExpr->pColl) ){ + if( pListItem->pExpr ){ int nColl; - zColl = pListItem->pExpr->pColl->zName; + assert( pListItem->pExpr->op==TK_COLLATE ); + zColl = pListItem->pExpr->u.zToken; nColl = sqlite3Strlen30(zColl) + 1; assert( nExtra>=nColl ); memcpy(zExtra, zColl, nColl); @@ -2762,9 +2745,7 @@ Index *sqlite3CreateIndex( nExtra -= nColl; }else{ zColl = pTab->aCol[j].zColl; - if( !zColl ){ - zColl = "BINARY"; - } + if( !zColl ) zColl = "BINARY"; } if( !db->init.busy && !sqlite3LocateCollSeq(pParse, zColl) ){ goto exit_create_index; @@ -2820,7 +2801,7 @@ Index *sqlite3CreateIndex( ** However the ON CONFLICT clauses are different. If both this ** constraint and the previous equivalent constraint have explicit ** ON CONFLICT clauses this is an error. Otherwise, use the - ** explicitly specified behaviour for the index. + ** explicitly specified behavior for the index. */ if( !(pIdx->onError==OE_Default || pIndex->onError==OE_Default) ){ sqlite3ErrorMsg(pParse, @@ -3567,6 +3548,15 @@ int sqlite3OpenTempDatabase(Parse *pParse){ void sqlite3CodeVerifySchema(Parse *pParse, int iDb){ Parse *pToplevel = sqlite3ParseToplevel(pParse); +#ifndef SQLITE_OMIT_TRIGGER + if( pToplevel!=pParse ){ + /* This branch is taken if a trigger is currently being coded. In this + ** case, set cookieGoto to a non-zero value to show that this function + ** has been called. This is used by the sqlite3ExprCodeConstants() + ** function. */ + pParse->cookieGoto = -1; + } +#endif if( pToplevel->cookieGoto==0 ){ Vdbe *v = sqlite3GetVdbe(pToplevel); if( v==0 ) return; /* This only happens if there was a prior error */ @@ -3664,12 +3654,19 @@ void sqlite3MayAbort(Parse *pParse){ ** error. The onError parameter determines which (if any) of the statement ** and/or current transaction is rolled back. */ -void sqlite3HaltConstraint(Parse *pParse, int onError, char *p4, int p4type){ +void sqlite3HaltConstraint( + Parse *pParse, /* Parsing context */ + int errCode, /* extended error code */ + int onError, /* Constraint type */ + char *p4, /* Error message */ + int p4type /* P4_STATIC or P4_TRANSIENT */ +){ Vdbe *v = sqlite3GetVdbe(pParse); + assert( (errCode&0xff)==SQLITE_CONSTRAINT ); if( onError==OE_Abort ){ sqlite3MayAbort(pParse); } - sqlite3VdbeAddOp4(v, OP_Halt, SQLITE_CONSTRAINT, onError, 0, p4, p4type); + sqlite3VdbeAddOp4(v, OP_Halt, errCode, onError, 0, p4, p4type); } /* diff --git a/src/callback.c b/src/callback.c index a515e05..d40c65c 100644 --- a/src/callback.c +++ b/src/callback.c @@ -75,17 +75,18 @@ static int synthCollSeq(sqlite3 *db, CollSeq *pColl){ ** ** The return value is either the collation sequence to be used in database ** db for collation type name zName, length nName, or NULL, if no collation -** sequence can be found. +** sequence can be found. If no collation is found, leave an error message. ** ** See also: sqlite3LocateCollSeq(), sqlite3FindCollSeq() */ CollSeq *sqlite3GetCollSeq( - sqlite3* db, /* The database connection */ + Parse *pParse, /* Parsing context */ u8 enc, /* The desired encoding for the collating sequence */ CollSeq *pColl, /* Collating sequence with native encoding, or NULL */ const char *zName /* Collating sequence name */ ){ CollSeq *p; + sqlite3 *db = pParse->db; p = pColl; if( !p ){ @@ -102,6 +103,9 @@ CollSeq *sqlite3GetCollSeq( p = 0; } assert( !p || p->xCmp ); + if( p==0 ){ + sqlite3ErrorMsg(pParse, "no such collation sequence: %s", zName); + } return p; } @@ -120,10 +124,8 @@ int sqlite3CheckCollSeq(Parse *pParse, CollSeq *pColl){ if( pColl ){ const char *zName = pColl->zName; sqlite3 *db = pParse->db; - CollSeq *p = sqlite3GetCollSeq(db, ENC(db), pColl, zName); + CollSeq *p = sqlite3GetCollSeq(pParse, ENC(db), pColl, zName); if( !p ){ - sqlite3ErrorMsg(pParse, "no such collation sequence: %s", zName); - pParse->nErr++; return SQLITE_ERROR; } assert( p==pColl ); diff --git a/src/crypto.c b/src/crypto.c index 50213b5..2551e6b 100644 --- a/src/crypto.c +++ b/src/crypto.c @@ -1,10 +1,8 @@ /* ** SQLCipher -** crypto.c developed by Stephen Lombardo (Zetetic LLC) -** sjlombardo at zetetic dot net -** http://zetetic.net +** http://sqlcipher.net ** -** Copyright (c) 2009, ZETETIC LLC +** Copyright (c) 2008 - 2013, ZETETIC LLC ** All rights reserved. ** ** Redistribution and use in source and binary forms, with or without @@ -30,7 +28,7 @@ ** SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ** */ -/* BEGIN CRYPTO */ +/* BEGIN SQLCIPHER */ #ifdef SQLITE_HAS_CODEC #include @@ -38,12 +36,12 @@ #include "btreeInt.h" #include "crypto.h" -const char* codec_get_cipher_version() { +static const char* codec_get_cipher_version() { return CIPHER_VERSION; } /* Generate code to return a string value */ -void codec_vdbe_return_static_string(Parse *pParse, const char *zLabel, const char *value){ +static void codec_vdbe_return_static_string(Parse *pParse, const char *zLabel, const char *value){ Vdbe *v = sqlite3GetVdbe(pParse); sqlite3VdbeSetNumCols(v, 1); sqlite3VdbeSetColName(v, 0, COLNAME_NAME, zLabel, SQLITE_STATIC); @@ -69,7 +67,7 @@ static int codec_set_btree_to_codec_pagesize(sqlite3 *db, Db *pDb, codec_ctx *ct return rc; } -int codec_set_pass_key(sqlite3* db, int nDb, const void *zKey, int nKey, int for_ctx) { +static int codec_set_pass_key(sqlite3* db, int nDb, const void *zKey, int nKey, int for_ctx) { struct Db *pDb = &db->aDb[nDb]; CODEC_TRACE(("codec_set_pass_key: entered db=%p nDb=%d zKey=%s nKey=%d for_ctx=%d\n", db, nDb, (char *)zKey, nKey, for_ctx)); if(pDb->pBt) { @@ -91,6 +89,11 @@ int codec_pragma(sqlite3* db, int iDb, Parse *pParse, const char *zLeft, const c CODEC_TRACE(("codec_pragma: entered db=%p iDb=%d pParse=%p zLeft=%s zRight=%s ctx=%p\n", db, iDb, pParse, zLeft, zRight, ctx)); + if( sqlite3StrICmp(zLeft, "cipher_provider")==0 && !zRight ){ + if(ctx) { codec_vdbe_return_static_string(pParse, "cipher_provider", + sqlcipher_codec_get_cipher_provider(ctx)); + } + } else if( sqlite3StrICmp(zLeft, "cipher_version")==0 && !zRight ){ codec_vdbe_return_static_string(pParse, "cipher_version", codec_get_cipher_version()); }else @@ -291,13 +294,13 @@ int sqlite3CodecAttach(sqlite3* db, int nDb, const void *zKey, int nKey) { sqlcipher_activate(); /* perform internal initialization for sqlcipher */ + sqlite3_mutex_enter(db->mutex); + /* point the internal codec argument against the contet to be prepared */ rc = sqlcipher_codec_ctx_init(&ctx, pDb, pDb->pBt->pBt->pPager, fd, zKey, nKey); if(rc != SQLITE_OK) return rc; /* initialization failed, do not attach potentially corrupted context */ - sqlite3_mutex_enter(db->mutex); - sqlite3pager_sqlite3PagerSetCodec(sqlite3BtreePager(pDb->pBt), sqlite3Codec, NULL, sqlite3FreeCodecArg, (void *) ctx); codec_set_btree_to_codec_pagesize(db, pDb, ctx); @@ -423,6 +426,203 @@ void sqlite3CodecGetKey(sqlite3* db, int nDb, void **zKey, int *nKey) { } } +#ifndef OMIT_EXPORT + +/* + * Implementation of an "export" function that allows a caller + * to duplicate the main database to an attached database. This is intended + * as a conveneince for users who need to: + * + * 1. migrate from an non-encrypted database to an encrypted database + * 2. move from an encrypted database to a non-encrypted database + * 3. convert beween the various flavors of encrypted databases. + * + * This implementation is based heavily on the procedure and code used + * in vacuum.c, but is exposed as a function that allows export to any + * named attached database. + */ + +/* +** Finalize a prepared statement. If there was an error, store the +** text of the error message in *pzErrMsg. Return the result code. +** +** Based on vacuumFinalize from vacuum.c +*/ +static int sqlcipher_finalize(sqlite3 *db, sqlite3_stmt *pStmt, char **pzErrMsg){ + int rc; + rc = sqlite3VdbeFinalize((Vdbe*)pStmt); + if( rc ){ + sqlite3SetString(pzErrMsg, db, sqlite3_errmsg(db)); + } + return rc; +} + +/* +** Execute zSql on database db. Return an error code. +** +** Based on execSql from vacuum.c +*/ +static int sqlcipher_execSql(sqlite3 *db, char **pzErrMsg, const char *zSql){ + sqlite3_stmt *pStmt; + VVA_ONLY( int rc; ) + if( !zSql ){ + return SQLITE_NOMEM; + } + if( SQLITE_OK!=sqlite3_prepare(db, zSql, -1, &pStmt, 0) ){ + sqlite3SetString(pzErrMsg, db, sqlite3_errmsg(db)); + return sqlite3_errcode(db); + } + VVA_ONLY( rc = ) sqlite3_step(pStmt); + assert( rc!=SQLITE_ROW ); + return sqlcipher_finalize(db, pStmt, pzErrMsg); +} + +/* +** Execute zSql on database db. The statement returns exactly +** one column. Execute this as SQL on the same database. +** +** Based on execExecSql from vacuum.c +*/ +static int sqlcipher_execExecSql(sqlite3 *db, char **pzErrMsg, const char *zSql){ + sqlite3_stmt *pStmt; + int rc; + + rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); + if( rc!=SQLITE_OK ) return rc; + + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + rc = sqlcipher_execSql(db, pzErrMsg, (char*)sqlite3_column_text(pStmt, 0)); + if( rc!=SQLITE_OK ){ + sqlcipher_finalize(db, pStmt, pzErrMsg); + return rc; + } + } + + return sqlcipher_finalize(db, pStmt, pzErrMsg); +} + +/* + * copy database and schema from the main database to an attached database + * + * Based on sqlite3RunVacuum from vacuum.c +*/ +void sqlcipher_exportFunc(sqlite3_context *context, int argc, sqlite3_value **argv) { + sqlite3 *db = sqlite3_context_db_handle(context); + const char* attachedDb = (const char*) sqlite3_value_text(argv[0]); + int saved_flags; /* Saved value of the db->flags */ + int saved_nChange; /* Saved value of db->nChange */ + int saved_nTotalChange; /* Saved value of db->nTotalChange */ + void (*saved_xTrace)(void*,const char*); /* Saved db->xTrace */ + int rc = SQLITE_OK; /* Return code from service routines */ + char *zSql = NULL; /* SQL statements */ + char *pzErrMsg = NULL; + + saved_flags = db->flags; + saved_nChange = db->nChange; + saved_nTotalChange = db->nTotalChange; + saved_xTrace = db->xTrace; + db->flags |= SQLITE_WriteSchema | SQLITE_IgnoreChecks | SQLITE_PreferBuiltin; + db->flags &= ~(SQLITE_ForeignKeys | SQLITE_ReverseOrder); + db->xTrace = 0; + + /* Query the schema of the main database. Create a mirror schema + ** in the temporary database. + */ + zSql = sqlite3_mprintf( + "SELECT 'CREATE TABLE %s.' || substr(sql,14) " + " FROM sqlite_master WHERE type='table' AND name!='sqlite_sequence'" + " AND rootpage>0" + , attachedDb); + rc = (zSql == NULL) ? SQLITE_NOMEM : sqlcipher_execExecSql(db, &pzErrMsg, zSql); + if( rc!=SQLITE_OK ) goto end_of_export; + sqlite3_free(zSql); + + zSql = sqlite3_mprintf( + "SELECT 'CREATE INDEX %s.' || substr(sql,14)" + " FROM sqlite_master WHERE sql LIKE 'CREATE INDEX %%' " + , attachedDb); + rc = (zSql == NULL) ? SQLITE_NOMEM : sqlcipher_execExecSql(db, &pzErrMsg, zSql); + if( rc!=SQLITE_OK ) goto end_of_export; + sqlite3_free(zSql); + + zSql = sqlite3_mprintf( + "SELECT 'CREATE UNIQUE INDEX %s.' || substr(sql,21) " + " FROM sqlite_master WHERE sql LIKE 'CREATE UNIQUE INDEX %%'" + , attachedDb); + rc = (zSql == NULL) ? SQLITE_NOMEM : sqlcipher_execExecSql(db, &pzErrMsg, zSql); + if( rc!=SQLITE_OK ) goto end_of_export; + sqlite3_free(zSql); + + /* Loop through the tables in the main database. For each, do + ** an "INSERT INTO rekey_db.xxx SELECT * FROM main.xxx;" to copy + ** the contents to the temporary database. + */ + zSql = sqlite3_mprintf( + "SELECT 'INSERT INTO %s.' || quote(name) " + "|| ' SELECT * FROM main.' || quote(name) || ';'" + "FROM main.sqlite_master " + "WHERE type = 'table' AND name!='sqlite_sequence' " + " AND rootpage>0" + , attachedDb); + rc = (zSql == NULL) ? SQLITE_NOMEM : sqlcipher_execExecSql(db, &pzErrMsg, zSql); + if( rc!=SQLITE_OK ) goto end_of_export; + sqlite3_free(zSql); + + /* Copy over the sequence table + */ + zSql = sqlite3_mprintf( + "SELECT 'DELETE FROM %s.' || quote(name) || ';' " + "FROM %s.sqlite_master WHERE name='sqlite_sequence' " + , attachedDb, attachedDb); + rc = (zSql == NULL) ? SQLITE_NOMEM : sqlcipher_execExecSql(db, &pzErrMsg, zSql); + if( rc!=SQLITE_OK ) goto end_of_export; + sqlite3_free(zSql); + + zSql = sqlite3_mprintf( + "SELECT 'INSERT INTO %s.' || quote(name) " + "|| ' SELECT * FROM main.' || quote(name) || ';' " + "FROM %s.sqlite_master WHERE name=='sqlite_sequence';" + , attachedDb, attachedDb); + rc = (zSql == NULL) ? SQLITE_NOMEM : sqlcipher_execExecSql(db, &pzErrMsg, zSql); + if( rc!=SQLITE_OK ) goto end_of_export; + sqlite3_free(zSql); + + /* Copy the triggers, views, and virtual tables from the main database + ** over to the temporary database. None of these objects has any + ** associated storage, so all we have to do is copy their entries + ** from the SQLITE_MASTER table. + */ + zSql = sqlite3_mprintf( + "INSERT INTO %s.sqlite_master " + " SELECT type, name, tbl_name, rootpage, sql" + " FROM main.sqlite_master" + " WHERE type='view' OR type='trigger'" + " OR (type='table' AND rootpage=0)" + , attachedDb); + rc = (zSql == NULL) ? SQLITE_NOMEM : sqlcipher_execSql(db, &pzErrMsg, zSql); + if( rc!=SQLITE_OK ) goto end_of_export; + sqlite3_free(zSql); + + zSql = NULL; +end_of_export: + db->flags = saved_flags; + db->nChange = saved_nChange; + db->nTotalChange = saved_nTotalChange; + db->xTrace = saved_xTrace; + + sqlite3_free(zSql); + + if(rc) { + if(pzErrMsg != NULL) { + sqlite3_result_error(context, pzErrMsg, -1); + sqlite3DbFree(db, pzErrMsg); + } else { + sqlite3_result_error(context, sqlite3ErrStr(rc), -1); + } + } +} + +#endif -/* END CRYPTO */ +/* END SQLCIPHER */ #endif diff --git a/src/crypto.h b/src/crypto.h index 8520152..a45b57c 100644 --- a/src/crypto.h +++ b/src/crypto.h @@ -30,15 +30,21 @@ ** SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ** */ -/* BEGIN CRYPTO */ +/* BEGIN SQLCIPHER */ #ifdef SQLITE_HAS_CODEC #ifndef CRYPTO_H #define CRYPTO_H +#if !defined (SQLCIPHER_CRYPTO_CC) \ + && !defined (SQLCIPHER_CRYPTO_LIBTOMCRYPT) \ + && !defined (SQLCIPHER_CRYPTO_OPENSSL) +#define SQLCIPHER_CRYPTO_OPENSSL +#endif + #define FILE_HEADER_SZ 16 #ifndef CIPHER_VERSION -#define CIPHER_VERSION "2.1.1" +#define CIPHER_VERSION "2.2.1" #endif #ifndef CIPHER @@ -82,6 +88,15 @@ #define HMAC_SALT_MASK 0x3a #endif +#ifndef CIPHER_MAX_IV_SZ +#define CIPHER_MAX_IV_SZ 16 +#endif + +#ifndef CIPHER_MAX_KEY_SZ +#define CIPHER_MAX_KEY_SZ 64 +#endif + + #ifdef CODEC_DEBUG #define CODEC_TRACE(X) {printf X;fflush(stdout);} #else @@ -135,16 +150,8 @@ static void cipher_hex2bin(const char *hex, int sz, unsigned char *out){ } /* extensions defined in crypto_impl.c */ - typedef struct codec_ctx codec_ctx; -/* utility functions */ -void* sqlcipher_memset(void *v, unsigned char value, int len); -int sqlcipher_ismemset(const void *v, unsigned char value, int len); -int sqlcipher_memcmp(const void *v0, const void *v1, int len); -int sqlcipher_pseudorandom(void *, int); -void sqlcipher_free(void *, int); - /* activation and initialization */ void sqlcipher_activate(); void sqlcipher_deactivate(); @@ -194,8 +201,7 @@ int sqlcipher_codec_ctx_set_flag(codec_ctx *ctx, unsigned int flag); int sqlcipher_codec_ctx_unset_flag(codec_ctx *ctx, unsigned int flag); int sqlcipher_codec_ctx_get_flag(codec_ctx *ctx, unsigned int flag, int for_ctx); -/* end extensions defined in crypto_impl.c */ - +const char* sqlcipher_codec_get_cipher_provider(codec_ctx *ctx); #endif #endif -/* END CRYPTO */ +/* END SQLCIPHER */ diff --git a/src/crypto_cc.c b/src/crypto_cc.c new file mode 100644 index 0000000..cf932f6 --- /dev/null +++ b/src/crypto_cc.c @@ -0,0 +1,140 @@ +/* +** SQLCipher +** http://sqlcipher.net +** +** Copyright (c) 2008 - 2013, ZETETIC LLC +** All rights reserved. +** +** Redistribution and use in source and binary forms, with or without +** modification, are permitted provided that the following conditions are met: +** * Redistributions of source code must retain the above copyright +** notice, this list of conditions and the following disclaimer. +** * Redistributions in binary form must reproduce the above copyright +** notice, this list of conditions and the following disclaimer in the +** documentation and/or other materials provided with the distribution. +** * Neither the name of the ZETETIC LLC nor the +** names of its contributors may be used to endorse or promote products +** derived from this software without specific prior written permission. +** +** THIS SOFTWARE IS PROVIDED BY ZETETIC LLC ''AS IS'' AND ANY +** EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +** WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +** DISCLAIMED. IN NO EVENT SHALL ZETETIC LLC BE LIABLE FOR ANY +** DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +** (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +** LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +** ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +** (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +** SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +** +*/ +/* BEGIN SQLCIPHER */ +#ifdef SQLITE_HAS_CODEC +#ifdef SQLCIPHER_CRYPTO_CC +#include "crypto.h" +#include "sqlcipher.h" +#include +#include + +/* generate a defined number of random bytes */ +static int sqlcipher_cc_random (void *ctx, void *buffer, int length) { + return (SecRandomCopyBytes(kSecRandomDefault, length, (uint8_t *)buffer) == 0) ? SQLITE_OK : SQLITE_ERROR; +} + +static const char* sqlcipher_cc_get_provider_name(void *ctx) { + return "commoncrypto"; +} + +static int sqlcipher_cc_hmac(void *ctx, unsigned char *hmac_key, int key_sz, unsigned char *in, int in_sz, unsigned char *in2, int in2_sz, unsigned char *out) { + CCHmacContext hmac_context; + CCHmacInit(&hmac_context, kCCHmacAlgSHA1, hmac_key, key_sz); + CCHmacUpdate(&hmac_context, in, in_sz); + CCHmacUpdate(&hmac_context, in2, in2_sz); + CCHmacFinal(&hmac_context, out); + return SQLITE_OK; +} + +static int sqlcipher_cc_kdf(void *ctx, const char *pass, int pass_sz, unsigned char* salt, int salt_sz, int workfactor, int key_sz, unsigned char *key) { + CCKeyDerivationPBKDF(kCCPBKDF2, pass, pass_sz, salt, salt_sz, kCCPRFHmacAlgSHA1, workfactor, key, key_sz); + return SQLITE_OK; +} + +static int sqlcipher_cc_cipher(void *ctx, int mode, unsigned char *key, int key_sz, unsigned char *iv, unsigned char *in, int in_sz, unsigned char *out) { + CCCryptorRef cryptor; + size_t tmp_csz, csz; + CCOperation op = mode == CIPHER_ENCRYPT ? kCCEncrypt : kCCDecrypt; + + CCCryptorCreate(op, kCCAlgorithmAES128, 0, key, kCCKeySizeAES256, iv, &cryptor); + CCCryptorUpdate(cryptor, in, in_sz, out, in_sz, &tmp_csz); + csz = tmp_csz; + out += tmp_csz; + CCCryptorFinal(cryptor, out, in_sz - csz, &tmp_csz); + csz += tmp_csz; + CCCryptorRelease(cryptor); + assert(size == csz); + + return SQLITE_OK; +} + +static int sqlcipher_cc_set_cipher(void *ctx, const char *cipher_name) { + return SQLITE_OK; +} + +static const char* sqlcipher_cc_get_cipher(void *ctx) { + return "aes-256-cbc"; +} + +static int sqlcipher_cc_get_key_sz(void *ctx) { + return kCCKeySizeAES256; +} + +static int sqlcipher_cc_get_iv_sz(void *ctx) { + return kCCBlockSizeAES128; +} + +static int sqlcipher_cc_get_block_sz(void *ctx) { + return kCCBlockSizeAES128; +} + +static int sqlcipher_cc_get_hmac_sz(void *ctx) { + return CC_SHA1_DIGEST_LENGTH; +} + +static int sqlcipher_cc_ctx_copy(void *target_ctx, void *source_ctx) { + return SQLITE_OK; +} + +static int sqlcipher_cc_ctx_cmp(void *c1, void *c2) { + return SQLITE_OK; +} + +static int sqlcipher_cc_ctx_init(void **ctx) { + return SQLITE_OK; +} + +static int sqlcipher_cc_ctx_free(void **ctx) { + return SQLITE_OK; +} + +int sqlcipher_cc_setup(sqlcipher_provider *p) { + p->random = sqlcipher_cc_random; + p->get_provider_name = sqlcipher_cc_get_provider_name; + p->hmac = sqlcipher_cc_hmac; + p->kdf = sqlcipher_cc_kdf; + p->cipher = sqlcipher_cc_cipher; + p->set_cipher = sqlcipher_cc_set_cipher; + p->get_cipher = sqlcipher_cc_get_cipher; + p->get_key_sz = sqlcipher_cc_get_key_sz; + p->get_iv_sz = sqlcipher_cc_get_iv_sz; + p->get_block_sz = sqlcipher_cc_get_block_sz; + p->get_hmac_sz = sqlcipher_cc_get_hmac_sz; + p->ctx_copy = sqlcipher_cc_ctx_copy; + p->ctx_cmp = sqlcipher_cc_ctx_cmp; + p->ctx_init = sqlcipher_cc_ctx_init; + p->ctx_free = sqlcipher_cc_ctx_free; + return SQLITE_OK; +} + +#endif +#endif +/* END SQLCIPHER */ diff --git a/src/crypto_impl.c b/src/crypto_impl.c index f35144d..1e7fa99 100644 --- a/src/crypto_impl.c +++ b/src/crypto_impl.c @@ -1,10 +1,8 @@ /* ** SQLCipher -** crypto_impl.c developed by Stephen Lombardo (Zetetic LLC) -** sjlombardo at zetetic dot net -** http://zetetic.net +** http://sqlcipher.net ** -** Copyright (c) 2011, ZETETIC LLC +** Copyright (c) 2008 - 2013, ZETETIC LLC ** All rights reserved. ** ** Redistribution and use in source and binary forms, with or without @@ -30,14 +28,12 @@ ** SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ** */ -/* BEGIN CRYPTO */ +/* BEGIN SQLCIPHER */ #ifdef SQLITE_HAS_CODEC -#include -#include -#include #include "sqliteInt.h" #include "btreeInt.h" +#include "sqlcipher.h" #include "crypto.h" #ifndef OMIT_MEMLOCK #if defined(__unix__) || defined(__APPLE__) @@ -52,9 +48,6 @@ struct and associated functions are defined here */ typedef struct { int derive_key; - EVP_CIPHER *evp_cipher; - EVP_CIPHER_CTX ectx; - HMAC_CTX hctx; int kdf_iter; int fast_kdf_iter; int key_sz; @@ -67,23 +60,15 @@ typedef struct { unsigned char *key; unsigned char *hmac_key; char *pass; + sqlcipher_provider *provider; + void *provider_ctx; } cipher_ctx; -void sqlcipher_cipher_ctx_free(cipher_ctx **); -int sqlcipher_cipher_ctx_cmp(cipher_ctx *, cipher_ctx *); -int sqlcipher_cipher_ctx_copy(cipher_ctx *, cipher_ctx *); -int sqlcipher_cipher_ctx_init(cipher_ctx **); -int sqlcipher_cipher_ctx_set_pass(cipher_ctx *, const void *, int); -int sqlcipher_cipher_ctx_key_derive(codec_ctx *, cipher_ctx *); - -/* prototype for pager HMAC function */ -int sqlcipher_page_hmac(cipher_ctx *, Pgno, unsigned char *, int, unsigned char *); - static unsigned int default_flags = DEFAULT_CIPHER_FLAGS; static unsigned char hmac_salt_mask = HMAC_SALT_MASK; - -static unsigned int openssl_external_init = 0; -static unsigned int openssl_init_count = 0; +static unsigned int sqlcipher_activate_count = 0; +static sqlite3_mutex* sqlcipher_provider_mutex = NULL; +static sqlcipher_provider *default_provider = NULL; struct codec_ctx { int kdf_salt_sz; @@ -96,49 +81,78 @@ struct codec_ctx { cipher_ctx *write_ctx; }; -/* activate and initialize sqlcipher. Most importantly, this will automatically - intialize OpenSSL's EVP system if it hasn't already be externally. Note that - this function may be called multiple times as new codecs are intiialized. - Thus it performs some basic counting to ensure that only the last and final - sqlcipher_deactivate() will free the EVP structures. -*/ +int sqlcipher_register_provider(sqlcipher_provider *p) { + sqlite3_mutex_enter(sqlcipher_provider_mutex); + if(default_provider != NULL && default_provider != p) { + /* only free the current registerd provider if it has been initialized + and it isn't a pointer to the same provider passed to the function + (i.e. protect against a caller calling register twice for the same provider) */ + sqlcipher_free(default_provider, sizeof(sqlcipher_provider)); + } + default_provider = p; + sqlite3_mutex_leave(sqlcipher_provider_mutex); + return SQLITE_OK; +} + +/* return a pointer to the currently registered provider. This will + allow an application to fetch the current registered provider and + make minor changes to it */ +sqlcipher_provider* sqlcipher_get_provider() { + return default_provider; +} + void sqlcipher_activate() { - sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); + sqlite3_mutex_enter(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); - /* we'll initialize openssl and increment the internal init counter - but only if it hasn't been initalized outside of SQLCipher by this program - e.g. on startup */ - if(openssl_init_count == 0 && EVP_get_cipherbyname(CIPHER) != NULL) { - openssl_external_init = 1; + if(sqlcipher_provider_mutex == NULL) { + /* allocate a new mutex to guard access to the provider */ + sqlcipher_provider_mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); } - if(openssl_external_init == 0) { - if(openssl_init_count == 0) { - OpenSSL_add_all_algorithms(); - } - openssl_init_count++; - } - sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); + /* check to see if there is a provider registered at this point + if there no provider registered at this point, register the + default provider */ + if(sqlcipher_get_provider() == NULL) { + sqlcipher_provider *p = sqlcipher_malloc(sizeof(sqlcipher_provider)); +#if defined (SQLCIPHER_CRYPTO_CC) + extern int sqlcipher_cc_setup(sqlcipher_provider *p); + sqlcipher_cc_setup(p); +#elif defined (SQLCIPHER_CRYPTO_LIBTOMCRYPT) + extern int sqlcipher_ltc_setup(sqlcipher_provider *p); + sqlcipher_ltc_setup(p); +#elif defined (SQLCIPHER_CRYPTO_OPENSSL) + extern int sqlcipher_openssl_setup(sqlcipher_provider *p); + sqlcipher_openssl_setup(p); +#else +#error "NO DEFAULT SQLCIPHER CRYPTO PROVIDER DEFINED" +#endif + sqlcipher_register_provider(p); + } + + sqlcipher_activate_count++; /* increment activation count */ + + sqlite3_mutex_leave(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); } -/* deactivate SQLCipher, most imporantly decremeting the activation count and - freeing the EVP structures on the final deactivation to ensure that - OpenSSL memory is cleaned up */ void sqlcipher_deactivate() { - sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); - /* If it is initialized externally, then the init counter should never be greater than zero. - This should prevent SQLCipher from "cleaning up" openssl - when something else in the program might be using it. */ - if(openssl_external_init == 0) { - openssl_init_count--; - /* if the counter reaches zero after it's decremented release EVP memory - Note: this code will only be reached if OpensSSL_add_all_algorithms() - is called by SQLCipher internally. */ - if(openssl_init_count == 0) { - EVP_cleanup(); + sqlite3_mutex_enter(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); + sqlcipher_activate_count--; + /* if no connections are using sqlcipher, cleanup globals */ + if(sqlcipher_activate_count < 1) { + sqlite3_mutex_enter(sqlcipher_provider_mutex); + if(default_provider != NULL) { + sqlcipher_free(default_provider, sizeof(sqlcipher_provider)); + default_provider = NULL; } + sqlite3_mutex_leave(sqlcipher_provider_mutex); + + /* last connection closed, free provider mutex*/ + sqlite3_mutex_free(sqlcipher_provider_mutex); + sqlcipher_provider_mutex = NULL; + + sqlcipher_activate_count = 0; /* reset activation count */ } - sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); + sqlite3_mutex_leave(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); } /* constant time memset using volitile to avoid having the memset @@ -185,11 +199,6 @@ int sqlcipher_memcmp(const void *v0, const void *v1, int len) { return (result != 0); } -/* generate a defined number of pseudorandom bytes */ -int sqlcipher_random (void *buffer, int length) { - return RAND_bytes((unsigned char *)buffer, length); -} - /** * Free and wipe memory. Uses SQLites internal sqlite3_free so that memory * can be countend and memory leak detection works in the test suite. @@ -242,14 +251,24 @@ void* sqlcipher_malloc(int sz) { * returns SQLITE_OK if initialization was successful * returns SQLITE_NOMEM if an error occured allocating memory */ -int sqlcipher_cipher_ctx_init(cipher_ctx **iCtx) { +static int sqlcipher_cipher_ctx_init(cipher_ctx **iCtx) { + int rc; cipher_ctx *ctx; *iCtx = (cipher_ctx *) sqlcipher_malloc(sizeof(cipher_ctx)); ctx = *iCtx; if(ctx == NULL) return SQLITE_NOMEM; - ctx->key = (unsigned char *) sqlcipher_malloc(EVP_MAX_KEY_LENGTH); - ctx->hmac_key = (unsigned char *) sqlcipher_malloc(EVP_MAX_KEY_LENGTH); + ctx->provider = (sqlcipher_provider *) sqlcipher_malloc(sizeof(sqlcipher_provider)); + if(ctx->provider == NULL) return SQLITE_NOMEM; + + /* make a copy of the provider to be used for the duration of the context */ + sqlite3_mutex_enter(sqlcipher_provider_mutex); + memcpy(ctx->provider, default_provider, sizeof(sqlcipher_provider)); + sqlite3_mutex_leave(sqlcipher_provider_mutex); + + if((rc = ctx->provider->ctx_init(&ctx->provider_ctx)) != SQLITE_OK) return rc; + ctx->key = (unsigned char *) sqlcipher_malloc(CIPHER_MAX_KEY_SZ); + ctx->hmac_key = (unsigned char *) sqlcipher_malloc(CIPHER_MAX_KEY_SZ); if(ctx->key == NULL) return SQLITE_NOMEM; if(ctx->hmac_key == NULL) return SQLITE_NOMEM; @@ -262,9 +281,11 @@ int sqlcipher_cipher_ctx_init(cipher_ctx **iCtx) { /** * Free and wipe memory associated with a cipher_ctx */ -void sqlcipher_cipher_ctx_free(cipher_ctx **iCtx) { +static void sqlcipher_cipher_ctx_free(cipher_ctx **iCtx) { cipher_ctx *ctx = *iCtx; CODEC_TRACE(("cipher_ctx_free: entered iCtx=%p\n", iCtx)); + ctx->provider->ctx_free(&ctx->provider_ctx); + sqlcipher_free(ctx->provider, sizeof(sqlcipher_provider)); sqlcipher_free(ctx->key, ctx->key_sz); sqlcipher_free(ctx->hmac_key, ctx->key_sz); sqlcipher_free(ctx->pass, ctx->pass_sz); @@ -277,18 +298,18 @@ void sqlcipher_cipher_ctx_free(cipher_ctx **iCtx) { * returns 0 if all the parameters (except the derived key data) are the same * returns 1 otherwise */ -int sqlcipher_cipher_ctx_cmp(cipher_ctx *c1, cipher_ctx *c2) { +static int sqlcipher_cipher_ctx_cmp(cipher_ctx *c1, cipher_ctx *c2) { CODEC_TRACE(("sqlcipher_cipher_ctx_cmp: entered c1=%p c2=%p\n", c1, c2)); if( - c1->evp_cipher == c2->evp_cipher - && c1->iv_sz == c2->iv_sz + c1->iv_sz == c2->iv_sz && c1->kdf_iter == c2->kdf_iter && c1->fast_kdf_iter == c2->fast_kdf_iter && c1->key_sz == c2->key_sz && c1->pass_sz == c2->pass_sz && c1->flags == c2->flags && c1->hmac_sz == c2->hmac_sz + && c1->provider->ctx_cmp(c1->provider_ctx, c2->provider_ctx) && ( c1->pass == c2->pass || !sqlcipher_memcmp((const unsigned char*)c1->pass, @@ -307,19 +328,27 @@ int sqlcipher_cipher_ctx_cmp(cipher_ctx *c1, cipher_ctx *c2) { * returns SQLITE_OK if initialization was successful * returns SQLITE_NOMEM if an error occured allocating memory */ -int sqlcipher_cipher_ctx_copy(cipher_ctx *target, cipher_ctx *source) { +static int sqlcipher_cipher_ctx_copy(cipher_ctx *target, cipher_ctx *source) { void *key = target->key; void *hmac_key = target->hmac_key; + void *provider = target->provider; + void *provider_ctx = target->provider_ctx; CODEC_TRACE(("sqlcipher_cipher_ctx_copy: entered target=%p, source=%p\n", target, source)); sqlcipher_free(target->pass, target->pass_sz); memcpy(target, source, sizeof(cipher_ctx)); target->key = key; //restore pointer to previously allocated key data - memcpy(target->key, source->key, EVP_MAX_KEY_LENGTH); + memcpy(target->key, source->key, CIPHER_MAX_KEY_SZ); target->hmac_key = hmac_key; //restore pointer to previously allocated hmac key data - memcpy(target->hmac_key, source->hmac_key, EVP_MAX_KEY_LENGTH); + memcpy(target->hmac_key, source->hmac_key, CIPHER_MAX_KEY_SZ); + + target->provider = provider; // restore pointer to previouly allocated provider; + memcpy(target->provider, source->provider, sizeof(sqlcipher_provider)); + + target->provider_ctx = provider_ctx; // restore pointer to previouly allocated provider context; + target->provider->ctx_copy(target->provider_ctx, source->provider_ctx); target->pass = sqlcipher_malloc(source->pass_sz); if(target->pass == NULL) return SQLITE_NOMEM; @@ -336,7 +365,7 @@ int sqlcipher_cipher_ctx_copy(cipher_ctx *target, cipher_ctx *source) { * returns SQLITE_NOMEM if an error occured allocating memory * returns SQLITE_ERROR if the key couldn't be set because the pass was null or size was zero */ -int sqlcipher_cipher_ctx_set_pass(cipher_ctx *ctx, const void *zKey, int nKey) { +static int sqlcipher_cipher_ctx_set_pass(cipher_ctx *ctx, const void *zKey, int nKey) { sqlcipher_free(ctx->pass, ctx->pass_sz); ctx->pass_sz = nKey; if(zKey && nKey) { @@ -366,11 +395,12 @@ int sqlcipher_codec_ctx_set_cipher(codec_ctx *ctx, const char *cipher_name, int cipher_ctx *c_ctx = for_ctx ? ctx->write_ctx : ctx->read_ctx; int rc; - c_ctx->evp_cipher = (EVP_CIPHER *) EVP_get_cipherbyname(cipher_name); - c_ctx->key_sz = EVP_CIPHER_key_length(c_ctx->evp_cipher); - c_ctx->iv_sz = EVP_CIPHER_iv_length(c_ctx->evp_cipher); - c_ctx->block_sz = EVP_CIPHER_block_size(c_ctx->evp_cipher); - c_ctx->hmac_sz = EVP_MD_size(EVP_sha1()); + c_ctx->provider->set_cipher(c_ctx->provider_ctx, cipher_name); + + c_ctx->key_sz = c_ctx->provider->get_key_sz(c_ctx->provider_ctx); + c_ctx->iv_sz = c_ctx->provider->get_iv_sz(c_ctx->provider_ctx); + c_ctx->block_sz = c_ctx->provider->get_block_sz(c_ctx->provider_ctx); + c_ctx->hmac_sz = c_ctx->provider->get_hmac_sz(c_ctx->provider_ctx); c_ctx->derive_key = 1; if(for_ctx == 2) @@ -382,8 +412,7 @@ int sqlcipher_codec_ctx_set_cipher(codec_ctx *ctx, const char *cipher_name, int const char* sqlcipher_codec_ctx_get_cipher(codec_ctx *ctx, int for_ctx) { cipher_ctx *c_ctx = for_ctx ? ctx->write_ctx : ctx->read_ctx; - EVP_CIPHER *evp_cipher = c_ctx->evp_cipher; - return EVP_CIPHER_name(evp_cipher); + return c_ctx->provider->get_cipher(c_ctx->provider_ctx); } int sqlcipher_codec_ctx_set_kdf_iter(codec_ctx *ctx, int kdf_iter, int for_ctx) { @@ -444,7 +473,7 @@ unsigned char sqlcipher_get_hmac_salt_mask() { /* set the codec flag for whether this individual database should be using hmac */ int sqlcipher_codec_ctx_set_use_hmac(codec_ctx *ctx, int use) { - int reserve = EVP_MAX_IV_LENGTH; /* base reserve size will be IV only */ + int reserve = CIPHER_MAX_IV_SZ; /* base reserve size will be IV only */ if(use) reserve += ctx->read_ctx->hmac_sz; /* if reserve will include hmac, update that size */ @@ -568,7 +597,7 @@ int sqlcipher_codec_ctx_init(codec_ctx **iCtx, Db *pDb, Pager *pPager, sqlite3_f if(fd == NULL || sqlite3OsRead(fd, ctx->kdf_salt, FILE_HEADER_SZ, 0) != SQLITE_OK) { /* if unable to read the bytes, generate random salt */ - if(sqlcipher_random(ctx->kdf_salt, FILE_HEADER_SZ) != 1) return SQLITE_ERROR; + if(ctx->read_ctx->provider->random(ctx->read_ctx->provider_ctx, ctx->kdf_salt, FILE_HEADER_SZ) != SQLITE_OK) return SQLITE_ERROR; } if((rc = sqlcipher_codec_ctx_set_cipher(ctx, CIPHER, 0)) != SQLITE_OK) return rc; @@ -608,7 +637,7 @@ static void sqlcipher_put4byte_le(unsigned char *p, u32 v) { p[3] = (u8)(v>>24); } -int sqlcipher_page_hmac(cipher_ctx *ctx, Pgno pgno, unsigned char *in, int in_sz, unsigned char *out) { +static int sqlcipher_page_hmac(cipher_ctx *ctx, Pgno pgno, unsigned char *in, int in_sz, unsigned char *out) { unsigned char pgno_raw[sizeof(pgno)]; /* we may convert page number to consistent representation before calculating MAC for compatibility across big-endian and little-endian platforms. @@ -626,16 +655,14 @@ int sqlcipher_page_hmac(cipher_ctx *ctx, Pgno pgno, unsigned char *in, int in_sz memcpy(pgno_raw, &pgno, sizeof(pgno)); } - HMAC_CTX_init(&ctx->hctx); - HMAC_Init_ex(&ctx->hctx, ctx->hmac_key, ctx->key_sz, EVP_sha1(), NULL); - /* include the encrypted page data, initialization vector, and page number in HMAC. This will prevent both tampering with the ciphertext, manipulation of the IV, or resequencing otherwise valid pages out of order in a database */ - HMAC_Update(&ctx->hctx, in, in_sz); - HMAC_Update(&ctx->hctx, (const unsigned char*) pgno_raw, sizeof(pgno)); - HMAC_Final(&ctx->hctx, out, NULL); - HMAC_CTX_cleanup(&ctx->hctx); + ctx->provider->hmac( + ctx->provider_ctx, ctx->hmac_key, + ctx->key_sz, in, + in_sz, (unsigned char*) &pgno_raw, + sizeof(pgno), out); return SQLITE_OK; } @@ -650,7 +677,7 @@ int sqlcipher_page_hmac(cipher_ctx *ctx, Pgno pgno, unsigned char *in, int in_sz int sqlcipher_page_cipher(codec_ctx *ctx, int for_ctx, Pgno pgno, int mode, int page_sz, unsigned char *in, unsigned char *out) { cipher_ctx *c_ctx = for_ctx ? ctx->write_ctx : ctx->read_ctx; unsigned char *iv_in, *iv_out, *hmac_in, *hmac_out, *out_start; - int tmp_csz, csz, size; + int size; /* calculate some required positions into various buffers */ size = page_sz - c_ctx->reserve_sz; /* adjust size to useable size and memset reserve at end of page */ @@ -675,7 +702,7 @@ int sqlcipher_page_cipher(codec_ctx *ctx, int for_ctx, Pgno pgno, int mode, int if(mode == CIPHER_ENCRYPT) { /* start at front of the reserve block, write random data to the end */ - if(sqlcipher_random(iv_out, c_ctx->reserve_sz) != 1) return SQLITE_ERROR; + if(c_ctx->provider->random(c_ctx->provider_ctx, iv_out, c_ctx->reserve_sz) != SQLITE_OK) return SQLITE_ERROR; } else { /* CIPHER_DECRYPT */ memcpy(iv_out, iv_in, c_ctx->iv_sz); /* copy the iv from the input to output buffer */ } @@ -707,17 +734,8 @@ int sqlcipher_page_cipher(codec_ctx *ctx, int for_ctx, Pgno pgno, int mode, int } } } - - EVP_CipherInit(&c_ctx->ectx, c_ctx->evp_cipher, NULL, NULL, mode); - EVP_CIPHER_CTX_set_padding(&c_ctx->ectx, 0); - EVP_CipherInit(&c_ctx->ectx, NULL, c_ctx->key, iv_out, mode); - EVP_CipherUpdate(&c_ctx->ectx, out, &tmp_csz, in, size); - csz = tmp_csz; - out += tmp_csz; - EVP_CipherFinal(&c_ctx->ectx, out, &tmp_csz); - csz += tmp_csz; - EVP_CIPHER_CTX_cleanup(&c_ctx->ectx); - assert(size == csz); + + c_ctx->provider->cipher(c_ctx->provider_ctx, mode, c_ctx->key, c_ctx->key_sz, iv_out, in, size, out); if((c_ctx->flags & CIPHER_FLAG_HMAC) && (mode == CIPHER_ENCRYPT)) { sqlcipher_page_hmac(c_ctx, pgno, out_start, size + c_ctx->iv_sz, hmac_out); @@ -739,7 +757,7 @@ int sqlcipher_page_cipher(codec_ctx *ctx, int for_ctx, Pgno pgno, int mode, int * returns SQLITE_OK if initialization was successful * returns SQLITE_ERROR if the key could't be derived (for instance if pass is NULL or pass_sz is 0) */ -int sqlcipher_cipher_ctx_key_derive(codec_ctx *ctx, cipher_ctx *c_ctx) { +static int sqlcipher_cipher_ctx_key_derive(codec_ctx *ctx, cipher_ctx *c_ctx) { CODEC_TRACE(("codec_key_derive: entered c_ctx->pass=%s, c_ctx->pass_sz=%d \ ctx->kdf_salt=%p ctx->kdf_salt_sz=%d c_ctx->kdf_iter=%d \ ctx->hmac_kdf_salt=%p, c_ctx->fast_kdf_iter=%d c_ctx->key_sz=%d\n", @@ -755,9 +773,9 @@ int sqlcipher_cipher_ctx_key_derive(codec_ctx *ctx, cipher_ctx *c_ctx) { cipher_hex2bin(z, n, c_ctx->key); } else { CODEC_TRACE(("codec_key_derive: deriving key using full PBKDF2 with %d iterations\n", c_ctx->kdf_iter)); - PKCS5_PBKDF2_HMAC_SHA1( c_ctx->pass, c_ctx->pass_sz, - ctx->kdf_salt, ctx->kdf_salt_sz, - c_ctx->kdf_iter, c_ctx->key_sz, c_ctx->key); + c_ctx->provider->kdf(c_ctx->provider_ctx, (const char*) c_ctx->pass, c_ctx->pass_sz, + ctx->kdf_salt, ctx->kdf_salt_sz, c_ctx->kdf_iter, + c_ctx->key_sz, c_ctx->key); } @@ -779,9 +797,11 @@ int sqlcipher_cipher_ctx_key_derive(codec_ctx *ctx, cipher_ctx *c_ctx) { CODEC_TRACE(("codec_key_derive: deriving hmac key from encryption key using PBKDF2 with %d iterations\n", c_ctx->fast_kdf_iter)); - PKCS5_PBKDF2_HMAC_SHA1( (const char*)c_ctx->key, c_ctx->key_sz, - ctx->hmac_kdf_salt, ctx->kdf_salt_sz, - c_ctx->fast_kdf_iter, c_ctx->key_sz, c_ctx->hmac_key); + + + c_ctx->provider->kdf(c_ctx->provider_ctx, (const char*)c_ctx->key, c_ctx->key_sz, + ctx->hmac_kdf_salt, ctx->kdf_salt_sz, c_ctx->fast_kdf_iter, + c_ctx->key_sz, c_ctx->hmac_key); } c_ctx->derive_key = 0; @@ -815,202 +835,9 @@ int sqlcipher_codec_key_copy(codec_ctx *ctx, int source) { } } - -#ifndef OMIT_EXPORT - -/* - * Implementation of an "export" function that allows a caller - * to duplicate the main database to an attached database. This is intended - * as a conveneince for users who need to: - * - * 1. migrate from an non-encrypted database to an encrypted database - * 2. move from an encrypted database to a non-encrypted database - * 3. convert beween the various flavors of encrypted databases. - * - * This implementation is based heavily on the procedure and code used - * in vacuum.c, but is exposed as a function that allows export to any - * named attached database. - */ - -/* -** Finalize a prepared statement. If there was an error, store the -** text of the error message in *pzErrMsg. Return the result code. -** -** Based on vacuumFinalize from vacuum.c -*/ -static int sqlcipher_finalize(sqlite3 *db, sqlite3_stmt *pStmt, char **pzErrMsg){ - int rc; - rc = sqlite3VdbeFinalize((Vdbe*)pStmt); - if( rc ){ - sqlite3SetString(pzErrMsg, db, sqlite3_errmsg(db)); - } - return rc; -} - -/* -** Execute zSql on database db. Return an error code. -** -** Based on execSql from vacuum.c -*/ -static int sqlcipher_execSql(sqlite3 *db, char **pzErrMsg, const char *zSql){ - sqlite3_stmt *pStmt; - VVA_ONLY( int rc; ) - if( !zSql ){ - return SQLITE_NOMEM; - } - if( SQLITE_OK!=sqlite3_prepare(db, zSql, -1, &pStmt, 0) ){ - sqlite3SetString(pzErrMsg, db, sqlite3_errmsg(db)); - return sqlite3_errcode(db); - } - VVA_ONLY( rc = ) sqlite3_step(pStmt); - assert( rc!=SQLITE_ROW ); - return sqlcipher_finalize(db, pStmt, pzErrMsg); +const char* sqlcipher_codec_get_cipher_provider(codec_ctx *ctx) { + return ctx->read_ctx->provider->get_provider_name(ctx->read_ctx); } -/* -** Execute zSql on database db. The statement returns exactly -** one column. Execute this as SQL on the same database. -** -** Based on execExecSql from vacuum.c -*/ -static int sqlcipher_execExecSql(sqlite3 *db, char **pzErrMsg, const char *zSql){ - sqlite3_stmt *pStmt; - int rc; - - rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); - if( rc!=SQLITE_OK ) return rc; - - while( SQLITE_ROW==sqlite3_step(pStmt) ){ - rc = sqlcipher_execSql(db, pzErrMsg, (char*)sqlite3_column_text(pStmt, 0)); - if( rc!=SQLITE_OK ){ - sqlcipher_finalize(db, pStmt, pzErrMsg); - return rc; - } - } - - return sqlcipher_finalize(db, pStmt, pzErrMsg); -} - -/* - * copy database and schema from the main database to an attached database - * - * Based on sqlite3RunVacuum from vacuum.c -*/ -void sqlcipher_exportFunc(sqlite3_context *context, int argc, sqlite3_value **argv) { - sqlite3 *db = sqlite3_context_db_handle(context); - const char* attachedDb = (const char*) sqlite3_value_text(argv[0]); - int saved_flags; /* Saved value of the db->flags */ - int saved_nChange; /* Saved value of db->nChange */ - int saved_nTotalChange; /* Saved value of db->nTotalChange */ - void (*saved_xTrace)(void*,const char*); /* Saved db->xTrace */ - int rc = SQLITE_OK; /* Return code from service routines */ - char *zSql = NULL; /* SQL statements */ - char *pzErrMsg = NULL; - - saved_flags = db->flags; - saved_nChange = db->nChange; - saved_nTotalChange = db->nTotalChange; - saved_xTrace = db->xTrace; - db->flags |= SQLITE_WriteSchema | SQLITE_IgnoreChecks | SQLITE_PreferBuiltin; - db->flags &= ~(SQLITE_ForeignKeys | SQLITE_ReverseOrder); - db->xTrace = 0; - - /* Query the schema of the main database. Create a mirror schema - ** in the temporary database. - */ - zSql = sqlite3_mprintf( - "SELECT 'CREATE TABLE %s.' || substr(sql,14) " - " FROM sqlite_master WHERE type='table' AND name!='sqlite_sequence'" - " AND rootpage>0" - , attachedDb); - rc = (zSql == NULL) ? SQLITE_NOMEM : sqlcipher_execExecSql(db, &pzErrMsg, zSql); - if( rc!=SQLITE_OK ) goto end_of_export; - sqlite3_free(zSql); - - zSql = sqlite3_mprintf( - "SELECT 'CREATE INDEX %s.' || substr(sql,14)" - " FROM sqlite_master WHERE sql LIKE 'CREATE INDEX %%' " - , attachedDb); - rc = (zSql == NULL) ? SQLITE_NOMEM : sqlcipher_execExecSql(db, &pzErrMsg, zSql); - if( rc!=SQLITE_OK ) goto end_of_export; - sqlite3_free(zSql); - - zSql = sqlite3_mprintf( - "SELECT 'CREATE UNIQUE INDEX %s.' || substr(sql,21) " - " FROM sqlite_master WHERE sql LIKE 'CREATE UNIQUE INDEX %%'" - , attachedDb); - rc = (zSql == NULL) ? SQLITE_NOMEM : sqlcipher_execExecSql(db, &pzErrMsg, zSql); - if( rc!=SQLITE_OK ) goto end_of_export; - sqlite3_free(zSql); - - /* Loop through the tables in the main database. For each, do - ** an "INSERT INTO rekey_db.xxx SELECT * FROM main.xxx;" to copy - ** the contents to the temporary database. - */ - zSql = sqlite3_mprintf( - "SELECT 'INSERT INTO %s.' || quote(name) " - "|| ' SELECT * FROM main.' || quote(name) || ';'" - "FROM main.sqlite_master " - "WHERE type = 'table' AND name!='sqlite_sequence' " - " AND rootpage>0" - , attachedDb); - rc = (zSql == NULL) ? SQLITE_NOMEM : sqlcipher_execExecSql(db, &pzErrMsg, zSql); - if( rc!=SQLITE_OK ) goto end_of_export; - sqlite3_free(zSql); - - /* Copy over the sequence table - */ - zSql = sqlite3_mprintf( - "SELECT 'DELETE FROM %s.' || quote(name) || ';' " - "FROM %s.sqlite_master WHERE name='sqlite_sequence' " - , attachedDb, attachedDb); - rc = (zSql == NULL) ? SQLITE_NOMEM : sqlcipher_execExecSql(db, &pzErrMsg, zSql); - if( rc!=SQLITE_OK ) goto end_of_export; - sqlite3_free(zSql); - - zSql = sqlite3_mprintf( - "SELECT 'INSERT INTO %s.' || quote(name) " - "|| ' SELECT * FROM main.' || quote(name) || ';' " - "FROM %s.sqlite_master WHERE name=='sqlite_sequence';" - , attachedDb, attachedDb); - rc = (zSql == NULL) ? SQLITE_NOMEM : sqlcipher_execExecSql(db, &pzErrMsg, zSql); - if( rc!=SQLITE_OK ) goto end_of_export; - sqlite3_free(zSql); - - /* Copy the triggers, views, and virtual tables from the main database - ** over to the temporary database. None of these objects has any - ** associated storage, so all we have to do is copy their entries - ** from the SQLITE_MASTER table. - */ - zSql = sqlite3_mprintf( - "INSERT INTO %s.sqlite_master " - " SELECT type, name, tbl_name, rootpage, sql" - " FROM main.sqlite_master" - " WHERE type='view' OR type='trigger'" - " OR (type='table' AND rootpage=0)" - , attachedDb); - rc = (zSql == NULL) ? SQLITE_NOMEM : sqlcipher_execSql(db, &pzErrMsg, zSql); - if( rc!=SQLITE_OK ) goto end_of_export; - sqlite3_free(zSql); - - zSql = NULL; -end_of_export: - db->flags = saved_flags; - db->nChange = saved_nChange; - db->nTotalChange = saved_nTotalChange; - db->xTrace = saved_xTrace; - - sqlite3_free(zSql); - - if(rc) { - if(pzErrMsg != NULL) { - sqlite3_result_error(context, pzErrMsg, -1); - sqlite3DbFree(db, pzErrMsg); - } else { - sqlite3_result_error(context, sqlite3ErrStr(rc), -1); - } - } -} - -#endif #endif +/* END SQLCIPHER */ diff --git a/src/crypto_libtomcrypt.c b/src/crypto_libtomcrypt.c new file mode 100644 index 0000000..0299771 --- /dev/null +++ b/src/crypto_libtomcrypt.c @@ -0,0 +1,215 @@ +/* +** SQLCipher +** http://sqlcipher.net +** +** Copyright (c) 2008 - 2013, ZETETIC LLC +** All rights reserved. +** +** Redistribution and use in source and binary forms, with or without +** modification, are permitted provided that the following conditions are met: +** * Redistributions of source code must retain the above copyright +** notice, this list of conditions and the following disclaimer. +** * Redistributions in binary form must reproduce the above copyright +** notice, this list of conditions and the following disclaimer in the +** documentation and/or other materials provided with the distribution. +** * Neither the name of the ZETETIC LLC nor the +** names of its contributors may be used to endorse or promote products +** derived from this software without specific prior written permission. +** +** THIS SOFTWARE IS PROVIDED BY ZETETIC LLC ''AS IS'' AND ANY +** EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +** WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +** DISCLAIMED. IN NO EVENT SHALL ZETETIC LLC BE LIABLE FOR ANY +** DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +** (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +** LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +** ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +** (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +** SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +** +*/ +/* BEGIN SQLCIPHER */ +#ifdef SQLITE_HAS_CODEC +#ifdef SQLCIPHER_CRYPTO_LIBTOMCRYPT +#include "sqliteInt.h" +#include "sqlcipher.h" +#include + +typedef struct { + prng_state prng; +} ltc_ctx; + +static unsigned int ltc_init = 0; + +static int sqlcipher_ltc_add_random(void *ctx, void *buffer, int length) { + ltc_ctx *ltc = (ltc_ctx*)ctx; + int rc = fortuna_add_entropy(buffer, length, &(ltc->prng)); + return rc != CRYPT_OK ? SQLITE_ERROR : SQLITE_OK; +} + +static int sqlcipher_ltc_activate(void *ctx) { + ltc_ctx *ltc = (ltc_ctx*)ctx; + int random_buffer_sz = 32; + unsigned char random_buffer[random_buffer_sz]; + + if(ltc_init == 0) { + if(register_prng(&fortuna_desc) != CRYPT_OK) return SQLITE_ERROR; + if(register_cipher(&rijndael_desc) != CRYPT_OK) return SQLITE_ERROR; + if(register_hash(&sha1_desc) != CRYPT_OK) return SQLITE_ERROR; + ltc_init = 1; + } + if(fortuna_start(&(ltc->prng)) != CRYPT_OK) { + return SQLITE_ERROR; + } + sqlite3_randomness(random_buffer_sz, &random_buffer); + if(sqlcipher_ltc_add_random(ctx, random_buffer, random_buffer_sz) != SQLITE_OK) { + return SQLITE_ERROR; + } + if(sqlcipher_ltc_add_random(ctx, <c, sizeof(ltc_ctx*)) != SQLITE_OK) { + return SQLITE_ERROR; + } + if(fortuna_ready(&(ltc->prng)) != CRYPT_OK) { + return SQLITE_ERROR; + } + return SQLITE_OK; +} + +static int sqlcipher_ltc_deactivate(void *ctx) { + ltc_ctx *ltc = (ltc_ctx*)ctx; + fortuna_done(&(ltc->prng)); +} + +static const char* sqlcipher_ltc_get_provider_name(void *ctx) { + return "libtomcrypt"; +} + +static int sqlcipher_ltc_random(void *ctx, void *buffer, int length) { + ltc_ctx *ltc = (ltc_ctx*)ctx; + int rc; + + if((rc = fortuna_ready(&(ltc->prng))) != CRYPT_OK) { + return SQLITE_ERROR; + } + fortuna_read(buffer, length, &(ltc->prng)); + return SQLITE_OK; +} + +static int sqlcipher_ltc_hmac(void *ctx, unsigned char *hmac_key, int key_sz, unsigned char *in, int in_sz, unsigned char *in2, int in2_sz, unsigned char *out) { + int rc, hash_idx; + hmac_state hmac; + unsigned long outlen = key_sz; + + hash_idx = find_hash("sha1"); + if((rc = hmac_init(&hmac, hash_idx, hmac_key, key_sz)) != CRYPT_OK) return SQLITE_ERROR; + if((rc = hmac_process(&hmac, in, in_sz)) != CRYPT_OK) return SQLITE_ERROR; + if((rc = hmac_process(&hmac, in2, in2_sz)) != CRYPT_OK) return SQLITE_ERROR; + if((rc = hmac_done(&hmac, out, &outlen)) != CRYPT_OK) return SQLITE_ERROR; + return SQLITE_OK; +} + +static int sqlcipher_ltc_kdf(void *ctx, const char *pass, int pass_sz, unsigned char* salt, int salt_sz, int workfactor, int key_sz, unsigned char *key) { + int rc, hash_idx; + unsigned long outlen = key_sz; + unsigned long random_buffer_sz = 256; + char random_buffer[random_buffer_sz]; + ltc_ctx *ltc = (ltc_ctx*)ctx; + + hash_idx = find_hash("sha1"); + if((rc = pkcs_5_alg2(pass, pass_sz, salt, salt_sz, + workfactor, hash_idx, key, &outlen)) != CRYPT_OK) { + return SQLITE_ERROR; + } + if((rc = pkcs_5_alg2(key, key_sz, salt, salt_sz, + 1, hash_idx, random_buffer, &random_buffer_sz)) != CRYPT_OK) { + return SQLITE_ERROR; + } + sqlcipher_ltc_add_random(ctx, random_buffer, random_buffer_sz); + return SQLITE_OK; +} + +static const char* sqlcipher_ltc_get_cipher(void *ctx) { + return "rijndael"; +} + +static int sqlcipher_ltc_cipher(void *ctx, int mode, unsigned char *key, int key_sz, unsigned char *iv, unsigned char *in, int in_sz, unsigned char *out) { + int rc, cipher_idx, hash_idx; + symmetric_CBC cbc; + + if((cipher_idx = find_cipher(sqlcipher_ltc_get_cipher(ctx))) == -1) return SQLITE_ERROR; + if((rc = cbc_start(cipher_idx, iv, key, key_sz, 0, &cbc)) != CRYPT_OK) return SQLITE_ERROR; + rc = mode == 1 ? cbc_encrypt(in, out, in_sz, &cbc) : cbc_decrypt(in, out, in_sz, &cbc); + if(rc != CRYPT_OK) return SQLITE_ERROR; + cbc_done(&cbc); + return SQLITE_OK; +} + +static int sqlcipher_ltc_set_cipher(void *ctx, const char *cipher_name) { + return SQLITE_OK; +} + +static int sqlcipher_ltc_get_key_sz(void *ctx) { + int cipher_idx = find_cipher(sqlcipher_ltc_get_cipher(ctx)); + return cipher_descriptor[cipher_idx].max_key_length; +} + +static int sqlcipher_ltc_get_iv_sz(void *ctx) { + int cipher_idx = find_cipher(sqlcipher_ltc_get_cipher(ctx)); + return cipher_descriptor[cipher_idx].block_length; +} + +static int sqlcipher_ltc_get_block_sz(void *ctx) { + int cipher_idx = find_cipher(sqlcipher_ltc_get_cipher(ctx)); + return cipher_descriptor[cipher_idx].block_length; +} + +static int sqlcipher_ltc_get_hmac_sz(void *ctx) { + int hash_idx = find_hash("sha1"); + return hash_descriptor[hash_idx].hashsize; +} + +static int sqlcipher_ltc_ctx_copy(void *target_ctx, void *source_ctx) { + memcpy(target_ctx, source_ctx, sizeof(ltc_ctx)); + return SQLITE_OK; +} + +static int sqlcipher_ltc_ctx_cmp(void *c1, void *c2) { + return 1; +} + +static int sqlcipher_ltc_ctx_init(void **ctx) { + *ctx = sqlcipher_malloc(sizeof(ltc_ctx)); + if(*ctx == NULL) return SQLITE_NOMEM; + sqlcipher_ltc_activate(*ctx); + return SQLITE_OK; +} + +static int sqlcipher_ltc_ctx_free(void **ctx) { + sqlcipher_ltc_deactivate(&ctx); + sqlcipher_free(*ctx, sizeof(ltc_ctx)); + return SQLITE_OK; +} + +int sqlcipher_ltc_setup(sqlcipher_provider *p) { + p->activate = sqlcipher_ltc_activate; + p->deactivate = sqlcipher_ltc_deactivate; + p->get_provider_name = sqlcipher_ltc_get_provider_name; + p->random = sqlcipher_ltc_random; + p->hmac = sqlcipher_ltc_hmac; + p->kdf = sqlcipher_ltc_kdf; + p->cipher = sqlcipher_ltc_cipher; + p->set_cipher = sqlcipher_ltc_set_cipher; + p->get_cipher = sqlcipher_ltc_get_cipher; + p->get_key_sz = sqlcipher_ltc_get_key_sz; + p->get_iv_sz = sqlcipher_ltc_get_iv_sz; + p->get_block_sz = sqlcipher_ltc_get_block_sz; + p->get_hmac_sz = sqlcipher_ltc_get_hmac_sz; + p->ctx_copy = sqlcipher_ltc_ctx_copy; + p->ctx_cmp = sqlcipher_ltc_ctx_cmp; + p->ctx_init = sqlcipher_ltc_ctx_init; + p->ctx_free = sqlcipher_ltc_ctx_free; + p->add_random = sqlcipher_ltc_add_random; +} + +#endif +#endif +/* END SQLCIPHER */ diff --git a/src/crypto_openssl.c b/src/crypto_openssl.c new file mode 100644 index 0000000..6035b7c --- /dev/null +++ b/src/crypto_openssl.c @@ -0,0 +1,251 @@ +/* +** SQLCipher +** http://sqlcipher.net +** +** Copyright (c) 2008 - 2013, ZETETIC LLC +** All rights reserved. +** +** Redistribution and use in source and binary forms, with or without +** modification, are permitted provided that the following conditions are met: +** * Redistributions of source code must retain the above copyright +** notice, this list of conditions and the following disclaimer. +** * Redistributions in binary form must reproduce the above copyright +** notice, this list of conditions and the following disclaimer in the +** documentation and/or other materials provided with the distribution. +** * Neither the name of the ZETETIC LLC nor the +** names of its contributors may be used to endorse or promote products +** derived from this software without specific prior written permission. +** +** THIS SOFTWARE IS PROVIDED BY ZETETIC LLC ''AS IS'' AND ANY +** EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +** WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +** DISCLAIMED. IN NO EVENT SHALL ZETETIC LLC BE LIABLE FOR ANY +** DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +** (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +** LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +** ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +** (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +** SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +** +*/ +/* BEGIN SQLCIPHER */ +#ifdef SQLITE_HAS_CODEC +#ifdef SQLCIPHER_CRYPTO_OPENSSL +#include "sqliteInt.h" +#include "crypto.h" +#include "sqlcipher.h" +#include +#include +#include + +typedef struct { + EVP_CIPHER *evp_cipher; +} openssl_ctx; + + +static unsigned int openssl_external_init = 0; +static unsigned int openssl_init_count = 0; +static sqlite3_mutex* openssl_rand_mutex = NULL; + +static int sqlcipher_openssl_add_random(void *ctx, void *buffer, int length) { +#ifndef SQLCIPHER_OPENSSL_NO_MUTEX_RAND + sqlite3_mutex_enter(openssl_rand_mutex); +#endif + RAND_add(buffer, length, 0); +#ifndef SQLCIPHER_OPENSSL_NO_MUTEX_RAND + sqlite3_mutex_leave(openssl_rand_mutex); +#endif + return SQLITE_OK; +} + +/* activate and initialize sqlcipher. Most importantly, this will automatically + intialize OpenSSL's EVP system if it hasn't already be externally. Note that + this function may be called multiple times as new codecs are intiialized. + Thus it performs some basic counting to ensure that only the last and final + sqlcipher_openssl_deactivate() will free the EVP structures. +*/ +static int sqlcipher_openssl_activate(void *ctx) { + /* initialize openssl and increment the internal init counter + but only if it hasn't been initalized outside of SQLCipher by this program + e.g. on startup */ + sqlite3_mutex_enter(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); + + if(openssl_init_count == 0 && EVP_get_cipherbyname(CIPHER) != NULL) { + /* if openssl has not yet been initialized by this library, but + a call to get_cipherbyname works, then the openssl library + has been initialized externally already. */ + openssl_external_init = 1; + } + + if(openssl_init_count == 0 && openssl_external_init == 0) { + /* if the library was not externally initialized, then should be now */ + OpenSSL_add_all_algorithms(); + } + +#ifndef SQLCIPHER_OPENSSL_NO_MUTEX_RAND + if(openssl_rand_mutex == NULL) { + /* allocate a mutex to guard against concurrent calls to RAND_bytes() */ + openssl_rand_mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); + } +#endif + + openssl_init_count++; + sqlite3_mutex_leave(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); + return SQLITE_OK; +} + +/* deactivate SQLCipher, most imporantly decremeting the activation count and + freeing the EVP structures on the final deactivation to ensure that + OpenSSL memory is cleaned up */ +static int sqlcipher_openssl_deactivate(void *ctx) { + sqlite3_mutex_enter(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); + openssl_init_count--; + + if(openssl_init_count == 0) { + if(openssl_external_init == 0) { + /* if OpenSSL hasn't be initialized externally, and the counter reaches zero + after it's decremented, release EVP memory + Note: this code will only be reached if OpensSSL_add_all_algorithms() + is called by SQLCipher internally. This should prevent SQLCipher from + "cleaning up" openssl when it was initialized externally by the program */ + EVP_cleanup(); + } +#ifndef SQLCIPHER_OPENSSL_NO_MUTEX_RAND + sqlite3_mutex_free(openssl_rand_mutex); + openssl_rand_mutex = NULL; +#endif + } + sqlite3_mutex_leave(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); + return SQLITE_OK; +} + +static const char* sqlcipher_openssl_get_provider_name(void *ctx) { + return "openssl"; +} + +/* generate a defined number of random bytes */ +static int sqlcipher_openssl_random (void *ctx, void *buffer, int length) { + int rc = 0; + /* concurrent calls to RAND_bytes can cause a crash under some openssl versions when a + naive application doesn't use CRYPTO_set_locking_callback and + CRYPTO_THREADID_set_callback to ensure openssl thread safety. + This is simple workaround to prevent this common crash + but a more proper solution is that applications setup platform-appropriate + thread saftey in openssl externally */ +#ifndef SQLCIPHER_OPENSSL_NO_MUTEX_RAND + sqlite3_mutex_enter(openssl_rand_mutex); +#endif + rc = RAND_bytes((unsigned char *)buffer, length); +#ifndef SQLCIPHER_OPENSSL_NO_MUTEX_RAND + sqlite3_mutex_leave(openssl_rand_mutex); +#endif + return (rc == 1) ? SQLITE_OK : SQLITE_ERROR; +} + +static int sqlcipher_openssl_hmac(void *ctx, unsigned char *hmac_key, int key_sz, unsigned char *in, int in_sz, unsigned char *in2, int in2_sz, unsigned char *out) { + HMAC_CTX hctx; + unsigned int outlen; + HMAC_CTX_init(&hctx); + HMAC_Init_ex(&hctx, hmac_key, key_sz, EVP_sha1(), NULL); + HMAC_Update(&hctx, in, in_sz); + HMAC_Update(&hctx, in2, in2_sz); + HMAC_Final(&hctx, out, &outlen); + HMAC_CTX_cleanup(&hctx); + return SQLITE_OK; +} + +static int sqlcipher_openssl_kdf(void *ctx, const char *pass, int pass_sz, unsigned char* salt, int salt_sz, int workfactor, int key_sz, unsigned char *key) { + PKCS5_PBKDF2_HMAC_SHA1(pass, pass_sz, salt, salt_sz, workfactor, key_sz, key); + return SQLITE_OK; +} + +static int sqlcipher_openssl_cipher(void *ctx, int mode, unsigned char *key, int key_sz, unsigned char *iv, unsigned char *in, int in_sz, unsigned char *out) { + EVP_CIPHER_CTX ectx; + int tmp_csz, csz; + + EVP_CipherInit(&ectx, ((openssl_ctx *)ctx)->evp_cipher, NULL, NULL, mode); + EVP_CIPHER_CTX_set_padding(&ectx, 0); // no padding + EVP_CipherInit(&ectx, NULL, key, iv, mode); + EVP_CipherUpdate(&ectx, out, &tmp_csz, in, in_sz); + csz = tmp_csz; + out += tmp_csz; + EVP_CipherFinal(&ectx, out, &tmp_csz); + csz += tmp_csz; + EVP_CIPHER_CTX_cleanup(&ectx); + assert(in_sz == csz); + return SQLITE_OK; +} + +static int sqlcipher_openssl_set_cipher(void *ctx, const char *cipher_name) { + openssl_ctx *o_ctx = (openssl_ctx *)ctx; + o_ctx->evp_cipher = (EVP_CIPHER *) EVP_get_cipherbyname(cipher_name); + return SQLITE_OK; +} + +static const char* sqlcipher_openssl_get_cipher(void *ctx) { + return EVP_CIPHER_name(((openssl_ctx *)ctx)->evp_cipher); +} + +static int sqlcipher_openssl_get_key_sz(void *ctx) { + return EVP_CIPHER_key_length(((openssl_ctx *)ctx)->evp_cipher); +} + +static int sqlcipher_openssl_get_iv_sz(void *ctx) { + return EVP_CIPHER_iv_length(((openssl_ctx *)ctx)->evp_cipher); +} + +static int sqlcipher_openssl_get_block_sz(void *ctx) { + return EVP_CIPHER_block_size(((openssl_ctx *)ctx)->evp_cipher); +} + +static int sqlcipher_openssl_get_hmac_sz(void *ctx) { + return EVP_MD_size(EVP_sha1()); +} + +static int sqlcipher_openssl_ctx_copy(void *target_ctx, void *source_ctx) { + memcpy(target_ctx, source_ctx, sizeof(openssl_ctx)); + return SQLITE_OK; +} + +static int sqlcipher_openssl_ctx_cmp(void *c1, void *c2) { + return ((openssl_ctx *)c1)->evp_cipher == ((openssl_ctx *)c2)->evp_cipher; +} + +static int sqlcipher_openssl_ctx_init(void **ctx) { + *ctx = sqlcipher_malloc(sizeof(openssl_ctx)); + if(*ctx == NULL) return SQLITE_NOMEM; + sqlcipher_openssl_activate(*ctx); + return SQLITE_OK; +} + +static int sqlcipher_openssl_ctx_free(void **ctx) { + sqlcipher_openssl_deactivate(*ctx); + sqlcipher_free(*ctx, sizeof(openssl_ctx)); + return SQLITE_OK; +} + +int sqlcipher_openssl_setup(sqlcipher_provider *p) { + p->activate = sqlcipher_openssl_activate; + p->deactivate = sqlcipher_openssl_deactivate; + p->get_provider_name = sqlcipher_openssl_get_provider_name; + p->random = sqlcipher_openssl_random; + p->hmac = sqlcipher_openssl_hmac; + p->kdf = sqlcipher_openssl_kdf; + p->cipher = sqlcipher_openssl_cipher; + p->set_cipher = sqlcipher_openssl_set_cipher; + p->get_cipher = sqlcipher_openssl_get_cipher; + p->get_key_sz = sqlcipher_openssl_get_key_sz; + p->get_iv_sz = sqlcipher_openssl_get_iv_sz; + p->get_block_sz = sqlcipher_openssl_get_block_sz; + p->get_hmac_sz = sqlcipher_openssl_get_hmac_sz; + p->ctx_copy = sqlcipher_openssl_ctx_copy; + p->ctx_cmp = sqlcipher_openssl_ctx_cmp; + p->ctx_init = sqlcipher_openssl_ctx_init; + p->ctx_free = sqlcipher_openssl_ctx_free; + p->add_random = sqlcipher_openssl_add_random; + return SQLITE_OK; +} + +#endif +#endif +/* END SQLCIPHER */ diff --git a/src/ctime.c b/src/ctime.c index 61cf4e3..60595ff 100644 --- a/src/ctime.c +++ b/src/ctime.c @@ -48,15 +48,15 @@ static const char * const azCompileOpt[] = { #ifdef SQLITE_COVERAGE_TEST "COVERAGE_TEST", #endif -#ifdef SQLITE_CURDIR - "CURDIR", -#endif #ifdef SQLITE_DEBUG "DEBUG", #endif #ifdef SQLITE_DEFAULT_LOCKING_MODE "DEFAULT_LOCKING_MODE=" CTIMEOPT_VAL(SQLITE_DEFAULT_LOCKING_MODE), #endif +#if defined(SQLITE_DEFAULT_MMAP_SIZE) && !defined(SQLITE_DEFAULT_MMAP_SIZE_xc) + "DEFAULT_MMAP_SIZE=" CTIMEOPT_VAL(SQLITE_DEFAULT_MMAP_SIZE), +#endif #ifdef SQLITE_DISABLE_DIRSYNC "DISABLE_DIRSYNC", #endif @@ -147,6 +147,9 @@ static const char * const azCompileOpt[] = { #ifdef SQLITE_LOCK_TRACE "LOCK_TRACE", #endif +#if defined(SQLITE_MAX_MMAP_SIZE) && !defined(SQLITE_MAX_MMAP_SIZE_xc) + "MAX_MMAP_SIZE=" CTIMEOPT_VAL(SQLITE_MAX_MMAP_SIZE), +#endif #ifdef SQLITE_MAX_SCHEMA_RETRY "MAX_SCHEMA_RETRY=" CTIMEOPT_VAL(SQLITE_MAX_SCHEMA_RETRY), #endif @@ -204,11 +207,6 @@ static const char * const azCompileOpt[] = { #ifdef SQLITE_OMIT_CHECK "OMIT_CHECK", #endif -/* // redundant -** #ifdef SQLITE_OMIT_COMPILEOPTION_DIAGS -** "OMIT_COMPILEOPTION_DIAGS", -** #endif -*/ #ifdef SQLITE_OMIT_COMPLETE "OMIT_COMPLETE", #endif @@ -263,9 +261,6 @@ static const char * const azCompileOpt[] = { #ifdef SQLITE_OMIT_MEMORYDB "OMIT_MEMORYDB", #endif -#ifdef SQLITE_OMIT_MERGE_SORT - "OMIT_MERGE_SORT", -#endif #ifdef SQLITE_OMIT_OR_OPTIMIZATION "OMIT_OR_OPTIMIZATION", #endif @@ -338,6 +333,9 @@ static const char * const azCompileOpt[] = { #ifdef SQLITE_PROXY_DEBUG "PROXY_DEBUG", #endif +#ifdef SQLITE_RTREE_INT_ONLY + "RTREE_INT_ONLY", +#endif #ifdef SQLITE_SECURE_DELETE "SECURE_DELETE", #endif @@ -350,13 +348,13 @@ static const char * const azCompileOpt[] = { #ifdef SQLITE_TCL "TCL", #endif -#ifdef SQLITE_TEMP_STORE +#if defined(SQLITE_TEMP_STORE) && !defined(SQLITE_TEMP_STORE_xc) "TEMP_STORE=" CTIMEOPT_VAL(SQLITE_TEMP_STORE), #endif #ifdef SQLITE_TEST "TEST", #endif -#ifdef SQLITE_THREADSAFE +#if defined(SQLITE_THREADSAFE) "THREADSAFE=" CTIMEOPT_VAL(SQLITE_THREADSAFE), #endif #ifdef SQLITE_USE_ALLOCA @@ -382,8 +380,11 @@ int sqlite3_compileoption_used(const char *zOptName){ /* Since ArraySize(azCompileOpt) is normally in single digits, a ** linear search is adequate. No need for a binary search. */ for(i=0; ia; Table *pTab; assert( pItem && pSrc->nSrc==1 ); - pTab = sqlite3LocateTable(pParse, 0, pItem->zName, pItem->zDatabase); + pTab = sqlite3LocateTableItem(pParse, 0, pItem); sqlite3DeleteTable(pParse->db, pItem->pTab); pItem->pTab = pTab; if( pTab ){ @@ -93,29 +93,28 @@ void sqlite3MaterializeView( int iCur /* Cursor number for ephemerial table */ ){ SelectDest dest; - Select *pDup; + Select *pSel; + SrcList *pFrom; sqlite3 *db = pParse->db; + int iDb = sqlite3SchemaToIndex(db, pView->pSchema); - pDup = sqlite3SelectDup(db, pView->pSelect, 0); - if( pWhere ){ - SrcList *pFrom; - - pWhere = sqlite3ExprDup(db, pWhere, 0); - pFrom = sqlite3SrcListAppend(db, 0, 0, 0); - if( pFrom ){ - assert( pFrom->nSrc==1 ); - pFrom->a[0].zAlias = sqlite3DbStrDup(db, pView->zName); - pFrom->a[0].pSelect = pDup; - assert( pFrom->a[0].pOn==0 ); - assert( pFrom->a[0].pUsing==0 ); - }else{ - sqlite3SelectDelete(db, pDup); - } - pDup = sqlite3SelectNew(pParse, 0, pFrom, pWhere, 0, 0, 0, 0, 0, 0); + pWhere = sqlite3ExprDup(db, pWhere, 0); + pFrom = sqlite3SrcListAppend(db, 0, 0, 0); + + if( pFrom ){ + assert( pFrom->nSrc==1 ); + pFrom->a[0].zName = sqlite3DbStrDup(db, pView->zName); + pFrom->a[0].zDatabase = sqlite3DbStrDup(db, db->aDb[iDb].zName); + assert( pFrom->a[0].pOn==0 ); + assert( pFrom->a[0].pUsing==0 ); } + + pSel = sqlite3SelectNew(pParse, 0, pFrom, pWhere, 0, 0, 0, 0, 0, 0); + if( pSel ) pSel->selFlags |= SF_Materialize; + sqlite3SelectDestInit(&dest, SRT_EphemTab, iCur); - sqlite3Select(pParse, pDup, &dest); - sqlite3SelectDelete(db, pDup); + sqlite3Select(pParse, pSel, &dest); + sqlite3SelectDelete(db, pSel); } #endif /* !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER) */ @@ -638,7 +637,9 @@ int sqlite3GenerateIndexKey( } if( doMakeRec ){ const char *zAff; - if( pTab->pSelect || (pParse->db->flags & SQLITE_IdxRealAsInt)!=0 ){ + if( pTab->pSelect + || OptimizationDisabled(pParse->db, SQLITE_IdxRealAsInt) + ){ zAff = 0; }else{ zAff = sqlite3IndexAffinityStr(v, pIdx); diff --git a/src/expr.c b/src/expr.c index 89172f9..660397e 100644 --- a/src/expr.c +++ b/src/expr.c @@ -31,7 +31,9 @@ ** SELECT * FROM t1 WHERE (select a from t1); */ char sqlite3ExprAffinity(Expr *pExpr){ - int op = pExpr->op; + int op; + pExpr = sqlite3ExprSkipCollate(pExpr); + op = pExpr->op; if( op==TK_SELECT ){ assert( pExpr->flags&EP_xIsSelect ); return sqlite3ExprAffinity(pExpr->x.pSelect->pEList->a[0].pExpr); @@ -56,66 +58,89 @@ char sqlite3ExprAffinity(Expr *pExpr){ } /* -** Set the explicit collating sequence for an expression to the -** collating sequence supplied in the second argument. +** Set the collating sequence for expression pExpr to be the collating +** sequence named by pToken. Return a pointer to a new Expr node that +** implements the COLLATE operator. +** +** If a memory allocation error occurs, that fact is recorded in pParse->db +** and the pExpr parameter is returned unchanged. */ -Expr *sqlite3ExprSetColl(Expr *pExpr, CollSeq *pColl){ - if( pExpr && pColl ){ - pExpr->pColl = pColl; - pExpr->flags |= EP_ExpCollate; +Expr *sqlite3ExprAddCollateToken(Parse *pParse, Expr *pExpr, Token *pCollName){ + if( pCollName->n>0 ){ + Expr *pNew = sqlite3ExprAlloc(pParse->db, TK_COLLATE, pCollName, 1); + if( pNew ){ + pNew->pLeft = pExpr; + pNew->flags |= EP_Collate; + pExpr = pNew; + } } return pExpr; } +Expr *sqlite3ExprAddCollateString(Parse *pParse, Expr *pExpr, const char *zC){ + Token s; + assert( zC!=0 ); + s.z = zC; + s.n = sqlite3Strlen30(s.z); + return sqlite3ExprAddCollateToken(pParse, pExpr, &s); +} /* -** Set the collating sequence for expression pExpr to be the collating -** sequence named by pToken. Return a pointer to the revised expression. -** The collating sequence is marked as "explicit" using the EP_ExpCollate -** flag. An explicit collating sequence will override implicit -** collating sequences. +** Skip over any TK_COLLATE and/or TK_AS operators at the root of +** an expression. */ -Expr *sqlite3ExprSetCollByToken(Parse *pParse, Expr *pExpr, Token *pCollName){ - char *zColl = 0; /* Dequoted name of collation sequence */ - CollSeq *pColl; - sqlite3 *db = pParse->db; - zColl = sqlite3NameFromToken(db, pCollName); - pColl = sqlite3LocateCollSeq(pParse, zColl); - sqlite3ExprSetColl(pExpr, pColl); - sqlite3DbFree(db, zColl); +Expr *sqlite3ExprSkipCollate(Expr *pExpr){ + while( pExpr && (pExpr->op==TK_COLLATE || pExpr->op==TK_AS) ){ + pExpr = pExpr->pLeft; + } return pExpr; } /* -** Return the default collation sequence for the expression pExpr. If -** there is no default collation type, return 0. +** Return the collation sequence for the expression pExpr. If +** there is no defined collating sequence, return NULL. +** +** The collating sequence might be determined by a COLLATE operator +** or by the presence of a column with a defined collating sequence. +** COLLATE operators take first precedence. Left operands take +** precedence over right operands. */ CollSeq *sqlite3ExprCollSeq(Parse *pParse, Expr *pExpr){ + sqlite3 *db = pParse->db; CollSeq *pColl = 0; Expr *p = pExpr; while( p ){ - int op; - pColl = p->pColl; - if( pColl ) break; - op = p->op; - if( p->pTab!=0 && ( - op==TK_AGG_COLUMN || op==TK_COLUMN || op==TK_REGISTER || op==TK_TRIGGER - )){ + int op = p->op; + if( op==TK_CAST || op==TK_UPLUS ){ + p = p->pLeft; + continue; + } + assert( op!=TK_REGISTER || p->op2!=TK_COLLATE ); + if( op==TK_COLLATE ){ + pColl = sqlite3GetCollSeq(pParse, ENC(db), 0, p->u.zToken); + break; + } + if( p->pTab!=0 + && (op==TK_AGG_COLUMN || op==TK_COLUMN + || op==TK_REGISTER || op==TK_TRIGGER) + ){ /* op==TK_REGISTER && p->pTab!=0 happens when pExpr was originally ** a TK_COLUMN but was previously evaluated and cached in a register */ - const char *zColl; int j = p->iColumn; if( j>=0 ){ - sqlite3 *db = pParse->db; - zColl = p->pTab->aCol[j].zColl; + const char *zColl = p->pTab->aCol[j].zColl; pColl = sqlite3FindCollSeq(db, ENC(db), zColl, 0); - pExpr->pColl = pColl; } break; } - if( op!=TK_CAST && op!=TK_UPLUS ){ + if( p->flags & EP_Collate ){ + if( ALWAYS(p->pLeft) && (p->pLeft->flags & EP_Collate)!=0 ){ + p = p->pLeft; + }else{ + p = p->pRight; + } + }else{ break; } - p = p->pLeft; } if( sqlite3CheckCollSeq(pParse, pColl) ){ pColl = 0; @@ -219,12 +244,10 @@ CollSeq *sqlite3BinaryCompareCollSeq( ){ CollSeq *pColl; assert( pLeft ); - if( pLeft->flags & EP_ExpCollate ){ - assert( pLeft->pColl ); - pColl = pLeft->pColl; - }else if( pRight && pRight->flags & EP_ExpCollate ){ - assert( pRight->pColl ); - pColl = pRight->pColl; + if( pLeft->flags & EP_Collate ){ + pColl = sqlite3ExprCollSeq(pParse, pLeft); + }else if( pRight && (pRight->flags & EP_Collate)!=0 ){ + pColl = sqlite3ExprCollSeq(pParse, pRight); }else{ pColl = sqlite3ExprCollSeq(pParse, pLeft); if( !pColl ){ @@ -454,17 +477,11 @@ void sqlite3ExprAttachSubtrees( }else{ if( pRight ){ pRoot->pRight = pRight; - if( pRight->flags & EP_ExpCollate ){ - pRoot->flags |= EP_ExpCollate; - pRoot->pColl = pRight->pColl; - } + pRoot->flags |= EP_Collate & pRight->flags; } if( pLeft ){ pRoot->pLeft = pLeft; - if( pLeft->flags & EP_ExpCollate ){ - pRoot->flags |= EP_ExpCollate; - pRoot->pColl = pLeft->pColl; - } + pRoot->flags |= EP_Collate & pLeft->flags; } exprSetHeight(pRoot); } @@ -616,7 +633,7 @@ void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr){ */ ynVar i; for(i=0; inzVar; i++){ - if( pParse->azVar[i] && memcmp(pParse->azVar[i],z,n+1)==0 ){ + if( pParse->azVar[i] && strcmp(pParse->azVar[i],z)==0 ){ pExpr->iColumn = x = (ynVar)i+1; break; } @@ -722,7 +739,7 @@ static int dupedExprStructSize(Expr *p, int flags){ assert( !ExprHasProperty(p, EP_FromJoin) ); assert( (p->flags2 & EP2_MallocedToken)==0 ); assert( (p->flags2 & EP2_Irreducible)==0 ); - if( p->pLeft || p->pRight || p->pColl || p->x.pList ){ + if( p->pLeft || p->pRight || p->x.pList ){ nSize = EXPR_REDUCEDSIZE | EP_Reduced; }else{ nSize = EXPR_TOKENONLYSIZE | EP_TokenOnly; @@ -930,6 +947,7 @@ SrcList *sqlite3SrcListDup(sqlite3 *db, SrcList *p, int flags){ struct SrcList_item *pNewItem = &pNew->a[i]; struct SrcList_item *pOldItem = &p->a[i]; Table *pTab; + pNewItem->pSchema = pOldItem->pSchema; pNewItem->zDatabase = sqlite3DbStrDup(db, pOldItem->zDatabase); pNewItem->zName = sqlite3DbStrDup(db, pOldItem->zName); pNewItem->zAlias = sqlite3DbStrDup(db, pOldItem->zAlias); @@ -938,6 +956,7 @@ SrcList *sqlite3SrcListDup(sqlite3 *db, SrcList *p, int flags){ pNewItem->addrFillSub = pOldItem->addrFillSub; pNewItem->regReturn = pOldItem->regReturn; pNewItem->isCorrelated = pOldItem->isCorrelated; + pNewItem->viaCoroutine = pOldItem->viaCoroutine; pNewItem->zIndex = sqlite3DbStrDup(db, pOldItem->zIndex); pNewItem->notIndexed = pOldItem->notIndexed; pNewItem->pIndex = pOldItem->pIndex; @@ -1190,6 +1209,7 @@ static int selectNodeIsConstant(Walker *pWalker, Select *NotUsed){ } static int exprIsConst(Expr *p, int initFlag){ Walker w; + memset(&w, 0, sizeof(w)); w.u.i = initFlag; w.xExprCallback = exprNodeIsConstant; w.xSelectCallback = selectNodeIsConstant; @@ -1420,24 +1440,34 @@ int sqlite3CodeOnce(Parse *pParse){ /* ** This function is used by the implementation of the IN (...) operator. -** It's job is to find or create a b-tree structure that may be used -** either to test for membership of the (...) set or to iterate through -** its members, skipping duplicates. +** The pX parameter is the expression on the RHS of the IN operator, which +** might be either a list of expressions or a subquery. +** +** The job of this routine is to find or create a b-tree object that can +** be used either to test for membership in the RHS set or to iterate through +** all members of the RHS set, skipping duplicates. +** +** A cursor is opened on the b-tree object that the RHS of the IN operator +** and pX->iTable is set to the index of that cursor. ** -** The index of the cursor opened on the b-tree (database table, database index -** or ephermal table) is stored in pX->iTable before this function returns. ** The returned value of this function indicates the b-tree type, as follows: ** -** IN_INDEX_ROWID - The cursor was opened on a database table. -** IN_INDEX_INDEX - The cursor was opened on a database index. -** IN_INDEX_EPH - The cursor was opened on a specially created and -** populated epheremal table. +** IN_INDEX_ROWID - The cursor was opened on a database table. +** IN_INDEX_INDEX_ASC - The cursor was opened on an ascending index. +** IN_INDEX_INDEX_DESC - The cursor was opened on a descending index. +** IN_INDEX_EPH - The cursor was opened on a specially created and +** populated epheremal table. ** -** An existing b-tree may only be used if the SELECT is of the simple -** form: +** An existing b-tree might be used if the RHS expression pX is a simple +** subquery such as: ** ** SELECT FROM ** +** If the RHS of the IN operator is a list or a more complex subquery, then +** an ephemeral table might need to be generated from the RHS and then +** pX->iTable made to point to the ephermeral table instead of an +** existing table. +** ** If the prNotFound parameter is 0, then the b-tree will be used to iterate ** through the set members, skipping any duplicates. In this case an ** epheremal table must be used unless the selected is guaranteed @@ -1533,8 +1563,7 @@ int sqlite3FindInIndex(Parse *pParse, Expr *pX, int *prNotFound){ ** comparison is the same as the affinity of the column. If ** it is not, it is not possible to use any index. */ - char aff = comparisonAffinity(pX); - int affinity_ok = (pTab->aCol[iCol].affinity==aff||aff==SQLITE_AFF_NONE); + int affinity_ok = sqlite3IndexAffinityOk(pX, pTab->aCol[iCol].affinity); for(pIdx=pTab->pIndex; pIdx && eType==0 && affinity_ok; pIdx=pIdx->pNext){ if( (pIdx->aiColumn[0]==iCol) @@ -1550,7 +1579,8 @@ int sqlite3FindInIndex(Parse *pParse, Expr *pX, int *prNotFound){ sqlite3VdbeAddOp4(v, OP_OpenRead, iTab, pIdx->tnum, iDb, pKey,P4_KEYINFO_HANDOFF); VdbeComment((v, "%s", pIdx->zName)); - eType = IN_INDEX_INDEX; + assert( IN_INDEX_INDEX_DESC == IN_INDEX_INDEX_ASC+1 ); + eType = IN_INDEX_INDEX_ASC + pIdx->aSortOrder[0]; sqlite3VdbeJumpHere(v, iAddr); if( prNotFound && !pTab->aCol[iCol].notNull ){ @@ -1662,6 +1692,7 @@ int sqlite3CodeSubselect( case TK_IN: { char affinity; /* Affinity of the LHS of the IN */ KeyInfo keyInfo; /* Keyinfo for the generated table */ + static u8 sortOrder = 0; /* Fake aSortOrder for keyInfo */ int addr; /* Address of OP_OpenEphemeral instruction */ Expr *pLeft = pExpr->pLeft; /* the LHS of the IN operator */ @@ -1689,6 +1720,7 @@ int sqlite3CodeSubselect( if( rMayHaveNull==0 ) sqlite3VdbeChangeP5(v, BTREE_UNORDERED); memset(&keyInfo, 0, sizeof(keyInfo)); keyInfo.nField = 1; + keyInfo.aSortOrder = &sortOrder; if( ExprHasProperty(pExpr, EP_xIsSelect) ){ /* Case 1: expr IN (SELECT ...) @@ -1729,6 +1761,7 @@ int sqlite3CodeSubselect( affinity = SQLITE_AFF_NONE; } keyInfo.aColl[0] = sqlite3ExprCollSeq(pParse, pExpr->pLeft); + keyInfo.aSortOrder = &sortOrder; /* Loop through each expression in . */ r1 = sqlite3GetTempReg(pParse); @@ -2055,7 +2088,7 @@ void sqlite3ExprCacheStore(Parse *pParse, int iTab, int iCol, int iReg){ ** for testing only - to verify that SQLite always gets the same answer ** with and without the column cache. */ - if( pParse->db->flags & SQLITE_ColumnCache ) return; + if( OptimizationDisabled(pParse->db, SQLITE_ColumnCache) ) return; /* First replace any existing entry. ** @@ -2252,8 +2285,8 @@ void sqlite3ExprCacheAffinityChange(Parse *pParse, int iStart, int iCount){ void sqlite3ExprCodeMove(Parse *pParse, int iFrom, int iTo, int nReg){ int i; struct yColCache *p; - if( NEVER(iFrom==iTo) ) return; - sqlite3VdbeAddOp3(pParse->pVdbe, OP_Move, iFrom, iTo, nReg); + assert( iFrom>=iTo+nReg || iFrom+nReg<=iTo ); + sqlite3VdbeAddOp3(pParse->pVdbe, OP_Move, iFrom, iTo, nReg-1); for(i=0, p=pParse->aColCache; iiReg; if( x>=iFrom && xpVdbe, OP_Copy, iFrom+i, iTo+i); - } -} - #if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) /* ** Return true if any register in the range iFrom..iTo (inclusive) @@ -2745,6 +2766,7 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ sqlite3ReleaseTempReg(pParse, r4); break; } + case TK_COLLATE: case TK_UPLUS: { inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); break; @@ -2911,7 +2933,8 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ sqlite3VdbeAddOp4( v, OP_Halt, SQLITE_OK, OE_Ignore, 0, pExpr->u.zToken,0); }else{ - sqlite3HaltConstraint(pParse, pExpr->affinity, pExpr->u.zToken, 0); + sqlite3HaltConstraint(pParse, SQLITE_CONSTRAINT_TRIGGER, + pExpr->affinity, pExpr->u.zToken, 0); } break; @@ -3114,6 +3137,12 @@ void sqlite3ExplainExpr(Vdbe *pOut, Expr *pExpr){ case TK_ISNULL: zUniOp = "ISNULL"; break; case TK_NOTNULL: zUniOp = "NOTNULL"; break; + case TK_COLLATE: { + sqlite3ExplainExpr(pOut, pExpr->pLeft); + sqlite3ExplainPrintf(pOut,".COLLATE(%s)",pExpr->u.zToken); + break; + } + case TK_AGG_FUNCTION: case TK_CONST_FUNC: case TK_FUNCTION: { @@ -3251,6 +3280,12 @@ void sqlite3ExplainExprList(Vdbe *pOut, ExprList *pList){ sqlite3ExplainPush(pOut); sqlite3ExplainExpr(pOut, pList->a[i].pExpr); sqlite3ExplainPop(pOut); + if( pList->a[i].zName ){ + sqlite3ExplainPrintf(pOut, " AS %s", pList->a[i].zName); + } + if( pList->a[i].bSpanIsTab ){ + sqlite3ExplainPrintf(pOut, " (%s)", pList->a[i].zSpan); + } if( inExpr-1 ){ sqlite3ExplainNL(pOut); } @@ -3332,6 +3367,9 @@ static int evalConstExpr(Walker *pWalker, Expr *pExpr){ case TK_REGISTER: { return WRC_Prune; } + case TK_COLLATE: { + return WRC_Continue; + } case TK_FUNCTION: case TK_AGG_FUNCTION: case TK_CONST_FUNC: { @@ -3353,9 +3391,11 @@ static int evalConstExpr(Walker *pWalker, Expr *pExpr){ } if( isAppropriateForFactoring(pExpr) ){ int r1 = ++pParse->nMem; - int r2; - r2 = sqlite3ExprCodeTarget(pParse, pExpr, r1); - if( NEVER(r1!=r2) ) sqlite3ReleaseTempReg(pParse, r1); + int r2 = sqlite3ExprCodeTarget(pParse, pExpr, r1); + /* If r2!=r1, it means that register r1 is never used. That is harmless + ** but suboptimal, so we want to know about the situation to fix it. + ** Hence the following assert: */ + assert( r2==r1 ); pExpr->op2 = pExpr->op; pExpr->op = TK_REGISTER; pExpr->iTable = r2; @@ -3383,9 +3423,9 @@ static int evalConstExpr(Walker *pWalker, Expr *pExpr){ void sqlite3ExprCodeConstants(Parse *pParse, Expr *pExpr){ Walker w; if( pParse->cookieGoto ) return; - if( (pParse->db->flags & SQLITE_FactorOutConst)!=0 ) return; + if( OptimizationDisabled(pParse->db, SQLITE_FactorOutConst) ) return; + memset(&w, 0, sizeof(w)); w.xExprCallback = evalConstExpr; - w.xSelectCallback = 0; w.pParse = pParse; sqlite3WalkExpr(&w, pExpr); } @@ -3498,7 +3538,7 @@ void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ int r1, r2; assert( jumpIfNull==SQLITE_JUMPIFNULL || jumpIfNull==0 ); - if( NEVER(v==0) ) return; /* Existance of VDBE checked by caller */ + if( NEVER(v==0) ) return; /* Existence of VDBE checked by caller */ if( NEVER(pExpr==0) ) return; /* No way this can happen */ op = pExpr->op; switch( op ){ @@ -3618,7 +3658,7 @@ void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ int r1, r2; assert( jumpIfNull==SQLITE_JUMPIFNULL || jumpIfNull==0 ); - if( NEVER(v==0) ) return; /* Existance of VDBE checked by caller */ + if( NEVER(v==0) ) return; /* Existence of VDBE checked by caller */ if( pExpr==0 ) return; /* The value of pExpr->op and op are related as follows: @@ -3772,7 +3812,15 @@ int sqlite3ExprCompare(Expr *pA, Expr *pB){ return 2; } if( (pA->flags & EP_Distinct)!=(pB->flags & EP_Distinct) ) return 2; - if( pA->op!=pB->op ) return 2; + if( pA->op!=pB->op ){ + if( pA->op==TK_COLLATE && sqlite3ExprCompare(pA->pLeft, pB)<2 ){ + return 1; + } + if( pB->op==TK_COLLATE && sqlite3ExprCompare(pA, pB->pLeft)<2 ){ + return 1; + } + return 2; + } if( sqlite3ExprCompare(pA->pLeft, pB->pLeft) ) return 2; if( sqlite3ExprCompare(pA->pRight, pB->pRight) ) return 2; if( sqlite3ExprListCompare(pA->x.pList, pB->x.pList) ) return 2; @@ -3784,11 +3832,9 @@ int sqlite3ExprCompare(Expr *pA, Expr *pB){ }else if( pA->op!=TK_COLUMN && ALWAYS(pA->op!=TK_AGG_COLUMN) && pA->u.zToken){ if( ExprHasProperty(pB, EP_IntValue) || NEVER(pB->u.zToken==0) ) return 2; if( strcmp(pA->u.zToken,pB->u.zToken)!=0 ){ - return 2; + return pA->op==TK_COLLATE ? 1 : 2; } } - if( (pA->flags & EP_ExpCollate)!=(pB->flags & EP_ExpCollate) ) return 1; - if( (pA->flags & EP_ExpCollate)!=0 && pA->pColl!=pB->pColl ) return 2; return 0; } @@ -4029,8 +4075,10 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ ExprSetIrreducible(pExpr); pExpr->iAgg = (i16)i; pExpr->pAggInfo = pAggInfo; + return WRC_Prune; + }else{ + return WRC_Continue; } - return WRC_Prune; } } return WRC_Continue; @@ -4042,9 +4090,10 @@ static int analyzeAggregatesInSelect(Walker *pWalker, Select *pSelect){ } /* -** Analyze the given expression looking for aggregate functions and -** for variables that need to be added to the pParse->aAgg[] array. -** Make additional entries to the pParse->aAgg[] array as necessary. +** Analyze the pExpr expression looking for aggregate functions and +** for variables that need to be added to AggInfo object that pNC->pAggInfo +** points to. Additional entries are made on the AggInfo object as +** necessary. ** ** This routine should only be called after the expression has been ** analyzed by sqlite3ResolveExprNames(). diff --git a/src/fkey.c b/src/fkey.c index 9db3a71..ac35bc1 100644 --- a/src/fkey.c +++ b/src/fkey.c @@ -21,8 +21,9 @@ ** -------------------------- ** ** Foreign keys in SQLite come in two flavours: deferred and immediate. -** If an immediate foreign key constraint is violated, SQLITE_CONSTRAINT -** is returned and the current statement transaction rolled back. If a +** If an immediate foreign key constraint is violated, +** SQLITE_CONSTRAINT_FOREIGNKEY is returned and the current +** statement transaction rolled back. If a ** deferred foreign key constraint is violated, no action is taken ** immediately. However if the application attempts to commit the ** transaction before fixing the constraint violation, the attempt fails. @@ -86,7 +87,8 @@ ** Immediate constraints are usually handled similarly. The only difference ** is that the counter used is stored as part of each individual statement ** object (struct Vdbe). If, after the statement has run, its immediate -** constraint counter is greater than zero, it returns SQLITE_CONSTRAINT +** constraint counter is greater than zero, +** it returns SQLITE_CONSTRAINT_FOREIGNKEY ** and the statement transaction is rolled back. An exception is an INSERT ** statement that inserts a single row only (no triggers). In this case, ** instead of using a counter, an exception is thrown immediately if the @@ -142,7 +144,7 @@ ** A foreign key constraint requires that the key columns in the parent ** table are collectively subject to a UNIQUE or PRIMARY KEY constraint. ** Given that pParent is the parent table for foreign key constraint pFKey, -** search the schema a unique index on the parent key columns. +** search the schema for a unique index on the parent key columns. ** ** If successful, zero is returned. If the parent key is an INTEGER PRIMARY ** KEY column, then output variable *ppIdx is set to NULL. Otherwise, *ppIdx @@ -178,7 +180,7 @@ ** into pParse. If an OOM error occurs, non-zero is returned and the ** pParse->db->mallocFailed flag is set. */ -static int locateFkeyIndex( +int sqlite3FkLocateIndex( Parse *pParse, /* Parse context to store any error in */ Table *pParent, /* Parent table of FK constraint pFKey */ FKey *pFKey, /* Foreign key to find index for */ @@ -275,7 +277,9 @@ static int locateFkeyIndex( if( !pIdx ){ if( !pParse->disableTriggers ){ - sqlite3ErrorMsg(pParse, "foreign key mismatch"); + sqlite3ErrorMsg(pParse, + "foreign key mismatch - \"%w\" referencing \"%w\"", + pFKey->pFrom->zName, pFKey->zTo); } sqlite3DbFree(pParse->db, aiCol); return 1; @@ -424,8 +428,8 @@ static void fkLookupParent( ** incrementing a counter. This is necessary as the VM code is being ** generated for will not open a statement transaction. */ assert( nIncr==1 ); - sqlite3HaltConstraint( - pParse, OE_Abort, "foreign key constraint failed", P4_STATIC + sqlite3HaltConstraint(pParse, SQLITE_CONSTRAINT_FOREIGNKEY, + OE_Abort, "foreign key constraint failed", P4_STATIC ); }else{ if( nIncr>0 && pFKey->isDeferred==0 ){ @@ -511,12 +515,15 @@ static void fkScanChildren( ** expression to the parent key column defaults. */ if( pIdx ){ Column *pCol; + const char *zColl; iCol = pIdx->aiColumn[i]; pCol = &pTab->aCol[iCol]; if( pTab->iPKey==iCol ) iCol = -1; pLeft->iTable = regData+iCol+1; pLeft->affinity = pCol->affinity; - pLeft->pColl = sqlite3LocateCollSeq(pParse, pCol->zColl); + zColl = pCol->zColl; + if( zColl==0 ) zColl = db->pDfltColl->zName; + pLeft = sqlite3ExprAddCollateString(pParse, pLeft, zColl); }else{ pLeft->iTable = regData; pLeft->affinity = SQLITE_AFF_INTEGER; @@ -662,8 +669,8 @@ void sqlite3FkDropTable(Parse *pParse, SrcList *pName, Table *pTab){ ** any modifications to the schema are made. This is because statement ** transactions are not able to rollback schema changes. */ sqlite3VdbeAddOp2(v, OP_FkIfZero, 0, sqlite3VdbeCurrentAddr(v)+2); - sqlite3HaltConstraint( - pParse, OE_Abort, "foreign key constraint failed", P4_STATIC + sqlite3HaltConstraint(pParse, SQLITE_CONSTRAINT_FOREIGNKEY, + OE_Abort, "foreign key constraint failed", P4_STATIC ); if( iSkip ){ @@ -733,7 +740,7 @@ void sqlite3FkCheck( }else{ pTo = sqlite3LocateTable(pParse, 0, pFKey->zTo, zDb); } - if( !pTo || locateFkeyIndex(pParse, pTo, pFKey, &pIdx, &aiFree) ){ + if( !pTo || sqlite3FkLocateIndex(pParse, pTo, pFKey, &pIdx, &aiFree) ){ assert( isIgnoreErrors==0 || (regOld!=0 && regNew==0) ); if( !isIgnoreErrors || db->mallocFailed ) return; if( pTo==0 ){ @@ -813,7 +820,7 @@ void sqlite3FkCheck( continue; } - if( locateFkeyIndex(pParse, pTab, pFKey, &pIdx, &aiCol) ){ + if( sqlite3FkLocateIndex(pParse, pTab, pFKey, &pIdx, &aiCol) ){ if( !isIgnoreErrors || db->mallocFailed ) return; continue; } @@ -868,7 +875,7 @@ u32 sqlite3FkOldmask( } for(p=sqlite3FkReferences(pTab); p; p=p->pNextTo){ Index *pIdx = 0; - locateFkeyIndex(pParse, pTab, p, &pIdx, 0); + sqlite3FkLocateIndex(pParse, pTab, p, &pIdx, 0); if( pIdx ){ for(i=0; inColumn; i++) mask |= COLUMN_MASK(pIdx->aiColumn[i]); } @@ -925,7 +932,8 @@ int sqlite3FkRequired( int iKey; for(iKey=0; iKeynCol; iKey++){ Column *pCol = &pTab->aCol[iKey]; - if( (zKey ? !sqlite3StrICmp(pCol->zName, zKey) : pCol->isPrimKey) ){ + if( (zKey ? !sqlite3StrICmp(pCol->zName, zKey) + : (pCol->colFlags & COLFLAG_PRIMKEY)!=0) ){ if( aChange[iKey]>=0 ) return 1; if( iKey==pTab->iPKey && chngRowid ) return 1; } @@ -993,7 +1001,7 @@ static Trigger *fkActionTrigger( int i; /* Iterator variable */ Expr *pWhen = 0; /* WHEN clause for the trigger */ - if( locateFkeyIndex(pParse, pTab, pFKey, &pIdx, &aiCol) ) return 0; + if( sqlite3FkLocateIndex(pParse, pTab, pFKey, &pIdx, &aiCol) ) return 0; assert( aiCol || pFKey->nCol==1 ); for(i=0; inCol; i++){ diff --git a/src/func.c b/src/func.c index 2cbaddc..c02f096 100644 --- a/src/func.c +++ b/src/func.c @@ -168,6 +168,56 @@ static void absFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ } } +/* +** Implementation of the instr() function. +** +** instr(haystack,needle) finds the first occurrence of needle +** in haystack and returns the number of previous characters plus 1, +** or 0 if needle does not occur within haystack. +** +** If both haystack and needle are BLOBs, then the result is one more than +** the number of bytes in haystack prior to the first occurrence of needle, +** or 0 if needle never occurs in haystack. +*/ +static void instrFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const unsigned char *zHaystack; + const unsigned char *zNeedle; + int nHaystack; + int nNeedle; + int typeHaystack, typeNeedle; + int N = 1; + int isText; + + UNUSED_PARAMETER(argc); + typeHaystack = sqlite3_value_type(argv[0]); + typeNeedle = sqlite3_value_type(argv[1]); + if( typeHaystack==SQLITE_NULL || typeNeedle==SQLITE_NULL ) return; + nHaystack = sqlite3_value_bytes(argv[0]); + nNeedle = sqlite3_value_bytes(argv[1]); + if( typeHaystack==SQLITE_BLOB && typeNeedle==SQLITE_BLOB ){ + zHaystack = sqlite3_value_blob(argv[0]); + zNeedle = sqlite3_value_blob(argv[1]); + isText = 0; + }else{ + zHaystack = sqlite3_value_text(argv[0]); + zNeedle = sqlite3_value_text(argv[1]); + isText = 1; + } + while( nNeedle<=nHaystack && memcmp(zHaystack, zNeedle, nNeedle)!=0 ){ + N++; + do{ + nHaystack--; + zHaystack++; + }while( isText && (zHaystack[0]&0xc0)==0x80 ); + } + if( nNeedle>nHaystack ) N = 0; + sqlite3_result_int(context, N); +} + /* ** Implementation of the substr() function. ** @@ -367,33 +417,14 @@ static void lowerFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ } } - -#if 0 /* This function is never used. */ -/* -** The COALESCE() and IFNULL() functions used to be implemented as shown -** here. But now they are implemented as VDBE code so that unused arguments -** do not have to be computed. This legacy implementation is retained as -** comment. -*/ /* -** Implementation of the IFNULL(), NVL(), and COALESCE() functions. -** All three do the same thing. They return the first non-NULL -** argument. +** The COALESCE() and IFNULL() functions are implemented as VDBE code so +** that unused argument values do not have to be computed. However, we +** still need some kind of function implementation for this routines in +** the function table. That function implementation will never be called +** so it doesn't matter what the implementation is. We might as well use +** the "version()" function as a substitute. */ -static void ifnullFunc( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - int i; - for(i=0; inoCase; int prevEscape = 0; /* True if the previous character was 'escape' */ - while( (c = sqlite3Utf8Read(zPattern,&zPattern))!=0 ){ - if( !prevEscape && c==matchAll ){ - while( (c=sqlite3Utf8Read(zPattern,&zPattern)) == matchAll + while( (c = sqlite3Utf8Read(&zPattern))!=0 ){ + if( c==matchAll && !prevEscape ){ + while( (c=sqlite3Utf8Read(&zPattern)) == matchAll || c == matchOne ){ - if( c==matchOne && sqlite3Utf8Read(zString, &zString)==0 ){ + if( c==matchOne && sqlite3Utf8Read(&zString)==0 ){ return 0; } } if( c==0 ){ return 1; }else if( c==esc ){ - c = sqlite3Utf8Read(zPattern, &zPattern); + c = sqlite3Utf8Read(&zPattern); if( c==0 ){ return 0; } @@ -592,25 +623,25 @@ static int patternCompare( } return *zString!=0; } - while( (c2 = sqlite3Utf8Read(zString,&zString))!=0 ){ + while( (c2 = sqlite3Utf8Read(&zString))!=0 ){ if( noCase ){ GlogUpperToLower(c2); GlogUpperToLower(c); while( c2 != 0 && c2 != c ){ - c2 = sqlite3Utf8Read(zString, &zString); + c2 = sqlite3Utf8Read(&zString); GlogUpperToLower(c2); } }else{ while( c2 != 0 && c2 != c ){ - c2 = sqlite3Utf8Read(zString, &zString); + c2 = sqlite3Utf8Read(&zString); } } if( c2==0 ) return 0; if( patternCompare(zPattern,zString,pInfo,esc) ) return 1; } return 0; - }else if( !prevEscape && c==matchOne ){ - if( sqlite3Utf8Read(zString, &zString)==0 ){ + }else if( c==matchOne && !prevEscape ){ + if( sqlite3Utf8Read(&zString)==0 ){ return 0; } }else if( c==matchSet ){ @@ -618,20 +649,20 @@ static int patternCompare( assert( esc==0 ); /* This only occurs for GLOB, not LIKE */ seen = 0; invert = 0; - c = sqlite3Utf8Read(zString, &zString); + c = sqlite3Utf8Read(&zString); if( c==0 ) return 0; - c2 = sqlite3Utf8Read(zPattern, &zPattern); + c2 = sqlite3Utf8Read(&zPattern); if( c2=='^' ){ invert = 1; - c2 = sqlite3Utf8Read(zPattern, &zPattern); + c2 = sqlite3Utf8Read(&zPattern); } if( c2==']' ){ if( c==']' ) seen = 1; - c2 = sqlite3Utf8Read(zPattern, &zPattern); + c2 = sqlite3Utf8Read(&zPattern); } while( c2 && c2!=']' ){ if( c2=='-' && zPattern[0]!=']' && zPattern[0]!=0 && prior_c>0 ){ - c2 = sqlite3Utf8Read(zPattern, &zPattern); + c2 = sqlite3Utf8Read(&zPattern); if( c>=prior_c && c<=c2 ) seen = 1; prior_c = 0; }else{ @@ -640,7 +671,7 @@ static int patternCompare( } prior_c = c2; } - c2 = sqlite3Utf8Read(zPattern, &zPattern); + c2 = sqlite3Utf8Read(&zPattern); } if( c2==0 || (seen ^ invert)==0 ){ return 0; @@ -648,7 +679,7 @@ static int patternCompare( }else if( esc==c && !prevEscape ){ prevEscape = 1; }else{ - c2 = sqlite3Utf8Read(zString, &zString); + c2 = sqlite3Utf8Read(&zString); if( noCase ){ GlogUpperToLower(c); GlogUpperToLower(c2); @@ -662,6 +693,13 @@ static int patternCompare( return *zString==0; } +/* +** The sqlite3_strglob() interface. +*/ +int sqlite3_strglob(const char *zGlobPattern, const char *zString){ + return patternCompare((u8*)zGlobPattern, (u8*)zString, &globInfo, 0)==0; +} + /* ** Count the number of times that the LIKE operator (or GLOB which is ** just a variation of LIKE) gets called. This is used for testing @@ -720,7 +758,7 @@ static void likeFunc( "ESCAPE expression must be a single character", -1); return; } - escape = sqlite3Utf8Read(zEsc, &zEsc); + escape = sqlite3Utf8Read(&zEsc); } if( zA && zB ){ struct compareInfo *pInfo = sqlite3_user_data(context); @@ -931,6 +969,62 @@ static void quoteFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ } } +/* +** The unicode() function. Return the integer unicode code-point value +** for the first character of the input string. +*/ +static void unicodeFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const unsigned char *z = sqlite3_value_text(argv[0]); + (void)argc; + if( z && z[0] ) sqlite3_result_int(context, sqlite3Utf8Read(&z)); +} + +/* +** The char() function takes zero or more arguments, each of which is +** an integer. It constructs a string where each character of the string +** is the unicode character for the corresponding integer argument. +*/ +static void charFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + unsigned char *z, *zOut; + int i; + zOut = z = sqlite3_malloc( argc*4 ); + if( z==0 ){ + sqlite3_result_error_nomem(context); + return; + } + for(i=0; i0x10ffff ) x = 0xfffd; + c = (unsigned)(x & 0x1fffff); + if( c<0x00080 ){ + *zOut++ = (u8)(c&0xFF); + }else if( c<0x00800 ){ + *zOut++ = 0xC0 + (u8)((c>>6)&0x1F); + *zOut++ = 0x80 + (u8)(c & 0x3F); + }else if( c<0x10000 ){ + *zOut++ = 0xE0 + (u8)((c>>12)&0x0F); + *zOut++ = 0x80 + (u8)((c>>6) & 0x3F); + *zOut++ = 0x80 + (u8)(c & 0x3F); + }else{ + *zOut++ = 0xF0 + (u8)((c>>18) & 0x07); + *zOut++ = 0x80 + (u8)((c>>12) & 0x3F); + *zOut++ = 0x80 + (u8)((c>>6) & 0x3F); + *zOut++ = 0x80 + (u8)(c & 0x3F); + } \ + } + sqlite3_result_text(context, (char*)z, (int)(zOut-z), sqlite3_free); +} + /* ** The hex() function. Interpret the argument as a blob. Return ** a hexadecimal rendering as text. @@ -1450,16 +1544,24 @@ static void groupConcatFinalize(sqlite3_context *context){ */ void sqlite3RegisterBuiltinFunctions(sqlite3 *db){ int rc = sqlite3_overload_function(db, "MATCH", 2); +/* BEGIN SQLCIPHER */ +#ifdef SQLITE_HAS_CODEC #ifndef OMIT_EXPORT extern void sqlcipher_exportFunc(sqlite3_context *, int, sqlite3_value **); #endif +#endif +/* END SQLCIPHER */ assert( rc==SQLITE_NOMEM || rc==SQLITE_OK ); if( rc==SQLITE_NOMEM ){ db->mallocFailed = 1; } +/* BEGIN SQLCIPHER */ +#ifdef SQLITE_HAS_CODEC #ifndef OMIT_EXPORT sqlite3CreateFunc(db, "sqlcipher_export", 1, SQLITE_TEXT, 0, sqlcipher_exportFunc, 0, 0, 0); #endif +#endif +/* END SQLCIPHER */ } /* @@ -1561,8 +1663,11 @@ void sqlite3RegisterGlobalFunctions(void){ AGGREGATE(max, 1, 1, 1, minmaxStep, minMaxFinalize ), FUNCTION2(typeof, 1, 0, 0, typeofFunc, SQLITE_FUNC_TYPEOF), FUNCTION2(length, 1, 0, 0, lengthFunc, SQLITE_FUNC_LENGTH), + FUNCTION(instr, 2, 0, 0, instrFunc ), FUNCTION(substr, 2, 0, 0, substrFunc ), FUNCTION(substr, 3, 0, 0, substrFunc ), + FUNCTION(unicode, 1, 0, 0, unicodeFunc ), + FUNCTION(char, -1, 0, 0, charFunc ), FUNCTION(abs, 1, 0, 0, absFunc ), #ifndef SQLITE_OMIT_FLOATING_POINT FUNCTION(round, 1, 0, 0, roundFunc ), diff --git a/src/global.c b/src/global.c index 7de0668..7b02cf2 100644 --- a/src/global.c +++ b/src/global.c @@ -133,6 +133,10 @@ const unsigned char sqlite3CtypeMap[256] = { # define SQLITE_USE_URI 0 #endif +#ifndef SQLITE_ALLOW_COVERING_INDEX_SCAN +# define SQLITE_ALLOW_COVERING_INDEX_SCAN 1 +#endif + /* ** The following singleton contains the global configuration for ** the SQLite library. @@ -142,6 +146,7 @@ SQLITE_WSD struct Sqlite3Config sqlite3Config = { 1, /* bCoreMutex */ SQLITE_THREADSAFE==1, /* bFullMutex */ SQLITE_USE_URI, /* bOpenUri */ + SQLITE_ALLOW_COVERING_INDEX_SCAN, /* bUseCis */ 0x7ffffffe, /* mxStrlen */ 128, /* szLookaside */ 500, /* nLookaside */ @@ -151,6 +156,8 @@ SQLITE_WSD struct Sqlite3Config sqlite3Config = { (void*)0, /* pHeap */ 0, /* nHeap */ 0, 0, /* mnHeap, mxHeap */ + SQLITE_DEFAULT_MMAP_SIZE, /* szMmap */ + SQLITE_MAX_MMAP_SIZE, /* mxMmap */ (void*)0, /* pScratch */ 0, /* szScratch */ 0, /* nScratch */ @@ -170,6 +177,10 @@ SQLITE_WSD struct Sqlite3Config sqlite3Config = { 0, /* xLog */ 0, /* pLogArg */ 0, /* bLocaltimeFault */ +#ifdef SQLITE_ENABLE_SQLLOG + 0, /* xSqllog */ + 0 /* pSqllogArg */ +#endif }; diff --git a/src/hash.c b/src/hash.c index d7625d3..e81dcf9 100644 --- a/src/hash.c +++ b/src/hash.c @@ -193,7 +193,7 @@ static void removeElementGivenHash( } sqlite3_free( elem ); pH->count--; - if( pH->count<=0 ){ + if( pH->count==0 ){ assert( pH->first==0 ); assert( pH->count==0 ); sqlite3HashClear(pH); diff --git a/src/hash.h b/src/hash.h index 990a2d6..82b7c58 100644 --- a/src/hash.h +++ b/src/hash.h @@ -9,7 +9,7 @@ ** May you share freely, never taking more than you give. ** ************************************************************************* -** This is the header file for the generic hash-table implemenation +** This is the header file for the generic hash-table implementation ** used in SQLite. */ #ifndef _SQLITE_HASH_H_ diff --git a/src/insert.c b/src/insert.c index a24e8f9..9a5661f 100644 --- a/src/insert.c +++ b/src/insert.c @@ -25,7 +25,7 @@ void sqlite3OpenTable( int opcode /* OP_OpenRead or OP_OpenWrite */ ){ Vdbe *v; - if( IsVirtual(pTab) ) return; + assert( !IsVirtual(pTab) ); v = sqlite3GetVdbe(p); assert( opcode==OP_OpenWrite || opcode==OP_OpenRead ); sqlite3TableLock(p, iDb, pTab->tnum, (opcode==OP_OpenWrite)?1:0, pTab->zName); @@ -320,6 +320,97 @@ void sqlite3AutoincrementEnd(Parse *pParse){ #endif /* SQLITE_OMIT_AUTOINCREMENT */ +/* +** Generate code for a co-routine that will evaluate a subquery one +** row at a time. +** +** The pSelect parameter is the subquery that the co-routine will evaluation. +** Information about the location of co-routine and the registers it will use +** is returned by filling in the pDest object. +** +** Registers are allocated as follows: +** +** pDest->iSDParm The register holding the next entry-point of the +** co-routine. Run the co-routine to its next breakpoint +** by calling "OP_Yield $X" where $X is pDest->iSDParm. +** +** pDest->iSDParm+1 The register holding the "completed" flag for the +** co-routine. This register is 0 if the previous Yield +** generated a new result row, or 1 if the subquery +** has completed. If the Yield is called again +** after this register becomes 1, then the VDBE will +** halt with an SQLITE_INTERNAL error. +** +** pDest->iSdst First result register. +** +** pDest->nSdst Number of result registers. +** +** This routine handles all of the register allocation and fills in the +** pDest structure appropriately. +** +** Here is a schematic of the generated code assuming that X is the +** co-routine entry-point register reg[pDest->iSDParm], that EOF is the +** completed flag reg[pDest->iSDParm+1], and R and S are the range of +** registers that hold the result set, reg[pDest->iSdst] through +** reg[pDest->iSdst+pDest->nSdst-1]: +** +** X <- A +** EOF <- 0 +** goto B +** A: setup for the SELECT +** loop rows in the SELECT +** load results into registers R..S +** yield X +** end loop +** cleanup after the SELECT +** EOF <- 1 +** yield X +** halt-error +** B: +** +** To use this subroutine, the caller generates code as follows: +** +** [ Co-routine generated by this subroutine, shown above ] +** S: yield X +** if EOF goto E +** if skip this row, goto C +** if terminate loop, goto E +** deal with this row +** C: goto S +** E: +*/ +int sqlite3CodeCoroutine(Parse *pParse, Select *pSelect, SelectDest *pDest){ + int regYield; /* Register holding co-routine entry-point */ + int regEof; /* Register holding co-routine completion flag */ + int addrTop; /* Top of the co-routine */ + int j1; /* Jump instruction */ + int rc; /* Result code */ + Vdbe *v; /* VDBE under construction */ + + regYield = ++pParse->nMem; + regEof = ++pParse->nMem; + v = sqlite3GetVdbe(pParse); + addrTop = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp2(v, OP_Integer, addrTop+2, regYield); /* X <- A */ + VdbeComment((v, "Co-routine entry point")); + sqlite3VdbeAddOp2(v, OP_Integer, 0, regEof); /* EOF <- 0 */ + VdbeComment((v, "Co-routine completion flag")); + sqlite3SelectDestInit(pDest, SRT_Coroutine, regYield); + j1 = sqlite3VdbeAddOp2(v, OP_Goto, 0, 0); + rc = sqlite3Select(pParse, pSelect, pDest); + assert( pParse->nErr==0 || rc ); + if( pParse->db->mallocFailed && rc==SQLITE_OK ) rc = SQLITE_NOMEM; + if( rc ) return rc; + sqlite3VdbeAddOp2(v, OP_Integer, 1, regEof); /* EOF <- 1 */ + sqlite3VdbeAddOp1(v, OP_Yield, regYield); /* yield X */ + sqlite3VdbeAddOp2(v, OP_Halt, SQLITE_INTERNAL, OE_Abort); + VdbeComment((v, "End of coroutine")); + sqlite3VdbeJumpHere(v, j1); /* label B: */ + return rc; +} + + + /* Forward declaration */ static int xferOptimization( Parse *pParse, /* Parser context */ @@ -568,51 +659,12 @@ void sqlite3Insert( ** co-routine is the common header to the 3rd and 4th templates. */ if( pSelect ){ - /* Data is coming from a SELECT. Generate code to implement that SELECT - ** as a co-routine. The code is common to both the 3rd and 4th - ** templates: - ** - ** EOF <- 0 - ** X <- A - ** goto B - ** A: setup for the SELECT - ** loop over the tables in the SELECT - ** load value into register R..R+n - ** yield X - ** end loop - ** cleanup after the SELECT - ** EOF <- 1 - ** yield X - ** halt-error - ** - ** On each invocation of the co-routine, it puts a single row of the - ** SELECT result into registers dest.iMem...dest.iMem+dest.nMem-1. - ** (These output registers are allocated by sqlite3Select().) When - ** the SELECT completes, it sets the EOF flag stored in regEof. - */ - int rc, j1; - - regEof = ++pParse->nMem; - sqlite3VdbeAddOp2(v, OP_Integer, 0, regEof); /* EOF <- 0 */ - VdbeComment((v, "SELECT eof flag")); - sqlite3SelectDestInit(&dest, SRT_Coroutine, ++pParse->nMem); - addrSelect = sqlite3VdbeCurrentAddr(v)+2; - sqlite3VdbeAddOp2(v, OP_Integer, addrSelect-1, dest.iSDParm); - j1 = sqlite3VdbeAddOp2(v, OP_Goto, 0, 0); - VdbeComment((v, "Jump over SELECT coroutine")); - - /* Resolve the expressions in the SELECT statement and execute it. */ - rc = sqlite3Select(pParse, pSelect, &dest); - assert( pParse->nErr==0 || rc ); - if( rc || NEVER(pParse->nErr) || db->mallocFailed ){ - goto insert_cleanup; - } - sqlite3VdbeAddOp2(v, OP_Integer, 1, regEof); /* EOF <- 1 */ - sqlite3VdbeAddOp1(v, OP_Yield, dest.iSDParm); /* yield X */ - sqlite3VdbeAddOp2(v, OP_Halt, SQLITE_INTERNAL, OE_Abort); - VdbeComment((v, "End of SELECT coroutine")); - sqlite3VdbeJumpHere(v, j1); /* label B: */ + /* Data is coming from a SELECT. Generate a co-routine to run that + ** SELECT. */ + int rc = sqlite3CodeCoroutine(pParse, pSelect, &dest); + if( rc ) goto insert_cleanup; + regEof = dest.iSDParm + 1; regFromSelect = dest.iSdst; assert( pSelect->pEList ); nColumn = pSelect->pEList->nExpr; @@ -1193,7 +1245,7 @@ void sqlite3GenerateConstraintChecks( case OE_Fail: { char *zMsg; sqlite3VdbeAddOp3(v, OP_HaltIfNull, - SQLITE_CONSTRAINT, onError, regData+i); + SQLITE_CONSTRAINT_NOTNULL, onError, regData+i); zMsg = sqlite3MPrintf(db, "%s.%s may not be NULL", pTab->zName, pTab->aCol[i].zName); sqlite3VdbeChangeP4(v, -1, zMsg, P4_DYNAMIC); @@ -1233,7 +1285,8 @@ void sqlite3GenerateConstraintChecks( }else{ zConsName = 0; } - sqlite3HaltConstraint(pParse, onError, zConsName, P4_DYNAMIC); + sqlite3HaltConstraint(pParse, SQLITE_CONSTRAINT_CHECK, + onError, zConsName, P4_DYNAMIC); } sqlite3VdbeResolveLabel(v, allOk); } @@ -1264,8 +1317,8 @@ void sqlite3GenerateConstraintChecks( case OE_Rollback: case OE_Abort: case OE_Fail: { - sqlite3HaltConstraint( - pParse, onError, "PRIMARY KEY must be unique", P4_STATIC); + sqlite3HaltConstraint(pParse, SQLITE_CONSTRAINT_PRIMARYKEY, + onError, "PRIMARY KEY must be unique", P4_STATIC); break; } case OE_Replace: { @@ -1392,7 +1445,8 @@ void sqlite3GenerateConstraintChecks( sqlite3StrAccumAppend(&errMsg, pIdx->nColumn>1 ? " are not unique" : " is not unique", -1); zErr = sqlite3StrAccumFinish(&errMsg); - sqlite3HaltConstraint(pParse, onError, zErr, 0); + sqlite3HaltConstraint(pParse, SQLITE_CONSTRAINT_UNIQUE, + onError, zErr, 0); sqlite3DbFree(errMsg.db, zErr); break; } @@ -1691,7 +1745,7 @@ static int xferOptimization( ** we have to check the semantics. */ pItem = pSelect->pSrc->a; - pSrc = sqlite3LocateTable(pParse, 0, pItem->zName, pItem->zDatabase); + pSrc = sqlite3LocateTableItem(pParse, 0, pItem); if( pSrc==0 ){ return 0; /* FROM clause does not contain a real table */ } @@ -1800,8 +1854,8 @@ static int xferOptimization( if( pDest->iPKey>=0 ){ addr1 = sqlite3VdbeAddOp2(v, OP_Rowid, iSrc, regRowid); addr2 = sqlite3VdbeAddOp3(v, OP_NotExists, iDest, 0, regRowid); - sqlite3HaltConstraint( - pParse, onError, "PRIMARY KEY must be unique", P4_STATIC); + sqlite3HaltConstraint(pParse, SQLITE_CONSTRAINT_PRIMARYKEY, + onError, "PRIMARY KEY must be unique", P4_STATIC); sqlite3VdbeJumpHere(v, addr2); autoIncStep(pParse, regAutoinc, regRowid); }else if( pDest->pIndex==0 ){ diff --git a/src/journal.c b/src/journal.c index 2f9e222..fed27be 100644 --- a/src/journal.c +++ b/src/journal.c @@ -59,6 +59,14 @@ static int createFile(JournalFile *p){ assert(p->iSize<=p->nBuf); rc = sqlite3OsWrite(p->pReal, p->zBuf, p->iSize, 0); } + if( rc!=SQLITE_OK ){ + /* If an error occurred while writing to the file, close it before + ** returning. This way, SQLite uses the in-memory journal data to + ** roll back changes made to the internal page-cache before this + ** function was called. */ + sqlite3OsClose(pReal); + p->pReal = 0; + } } } return rc; @@ -228,6 +236,16 @@ int sqlite3JournalCreate(sqlite3_file *p){ return createFile((JournalFile *)p); } +/* +** The file-handle passed as the only argument is guaranteed to be an open +** file. It may or may not be of class JournalFile. If the file is a +** JournalFile, and the underlying file on disk has not yet been opened, +** return 0. Otherwise, return 1. +*/ +int sqlite3JournalExists(sqlite3_file *p){ + return (p->pMethods!=&JournalFileMethods || ((JournalFile *)p)->pReal!=0); +} + /* ** Return the number of bytes required to store a JournalFile that uses vfs ** pVfs to create the underlying on-disk files. diff --git a/src/legacy.c b/src/legacy.c index ebab2de..94649ae 100644 --- a/src/legacy.c +++ b/src/legacy.c @@ -38,7 +38,6 @@ int sqlite3_exec( const char *zLeftover; /* Tail of unprocessed SQL */ sqlite3_stmt *pStmt = 0; /* The current SQL statement */ char **azCols = 0; /* Names of result columns */ - int nRetry = 0; /* Number of retry attempts */ int callbackIsInit; /* True if callback data is initialized */ if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; @@ -46,12 +45,12 @@ int sqlite3_exec( sqlite3_mutex_enter(db->mutex); sqlite3Error(db, SQLITE_OK, 0); - while( (rc==SQLITE_OK || (rc==SQLITE_SCHEMA && (++nRetry)<2)) && zSql[0] ){ + while( rc==SQLITE_OK && zSql[0] ){ int nCol; char **azVals = 0; pStmt = 0; - rc = sqlite3_prepare(db, zSql, -1, &pStmt, &zLeftover); + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &zLeftover); assert( rc==SQLITE_OK || pStmt==0 ); if( rc!=SQLITE_OK ){ continue; @@ -108,11 +107,8 @@ int sqlite3_exec( if( rc!=SQLITE_ROW ){ rc = sqlite3VdbeFinalize((Vdbe *)pStmt); pStmt = 0; - if( rc!=SQLITE_SCHEMA ){ - nRetry = 0; - zSql = zLeftover; - while( sqlite3Isspace(zSql[0]) ) zSql++; - } + zSql = zLeftover; + while( sqlite3Isspace(zSql[0]) ) zSql++; break; } } diff --git a/src/lempar.c b/src/lempar.c index cb6025e..2afaa6c 100644 --- a/src/lempar.c +++ b/src/lempar.c @@ -608,6 +608,7 @@ static void yy_reduce( */ %% }; + assert( yyruleno>=0 && yyrulenoyyidx -= yysize; diff --git a/src/loadext.c b/src/loadext.c index 3fcf500..cdcf6a9 100644 --- a/src/loadext.c +++ b/src/loadext.c @@ -378,6 +378,19 @@ static const sqlite3_api_routines sqlite3Apis = { sqlite3_blob_reopen, sqlite3_vtab_config, sqlite3_vtab_on_conflict, + sqlite3_close_v2, + sqlite3_db_filename, + sqlite3_db_readonly, + sqlite3_db_release_memory, + sqlite3_errstr, + sqlite3_stmt_busy, + sqlite3_stmt_readonly, + sqlite3_stricmp, + sqlite3_uri_boolean, + sqlite3_uri_int64, + sqlite3_uri_parameter, + sqlite3_vsnprintf, + sqlite3_wal_checkpoint_v2 }; /* @@ -402,8 +415,23 @@ static int sqlite3LoadExtension( void *handle; int (*xInit)(sqlite3*,char**,const sqlite3_api_routines*); char *zErrmsg = 0; + const char *zEntry; + char *zAltEntry = 0; void **aHandle; int nMsg = 300 + sqlite3Strlen30(zFile); + int ii; + + /* Shared library endings to try if zFile cannot be loaded as written */ + static const char *azEndings[] = { +#if SQLITE_OS_WIN + "dll" +#elif defined(__APPLE__) + "dylib" +#else + "so" +#endif + }; + if( pzErrMsg ) *pzErrMsg = 0; @@ -420,11 +448,17 @@ static int sqlite3LoadExtension( return SQLITE_ERROR; } - if( zProc==0 ){ - zProc = "sqlite3_extension_init"; - } + zEntry = zProc ? zProc : "sqlite3_extension_init"; handle = sqlite3OsDlOpen(pVfs, zFile); +#if SQLITE_OS_UNIX || SQLITE_OS_WIN + for(ii=0; ii sqlite3_example_init + ** C:/lib/mathfuncs.dll ==> sqlite3_mathfuncs_init + */ + if( xInit==0 && zProc==0 ){ + int iFile, iEntry, c; + int ncFile = sqlite3Strlen30(zFile); + zAltEntry = sqlite3_malloc(ncFile+30); + if( zAltEntry==0 ){ + sqlite3OsDlClose(pVfs, handle); + return SQLITE_NOMEM; + } + memcpy(zAltEntry, "sqlite3_", 8); + for(iFile=ncFile-1; iFile>=0 && zFile[iFile]!='/'; iFile--){} + iFile++; + if( sqlite3_strnicmp(zFile+iFile, "lib", 3)==0 ) iFile += 3; + for(iEntry=8; (c = zFile[iFile])!=0 && c!='.'; iFile++){ + if( sqlite3Isalpha(c) ){ + zAltEntry[iEntry++] = (char)sqlite3UpperToLower[(unsigned)c]; + } + } + memcpy(zAltEntry+iEntry, "_init", 6); + zEntry = zAltEntry; + xInit = (int(*)(sqlite3*,char**,const sqlite3_api_routines*)) + sqlite3OsDlSym(pVfs, handle, zEntry); + } if( xInit==0 ){ if( pzErrMsg ){ - nMsg += sqlite3Strlen30(zProc); + nMsg += sqlite3Strlen30(zEntry); *pzErrMsg = zErrmsg = sqlite3_malloc(nMsg); if( zErrmsg ){ sqlite3_snprintf(nMsg, zErrmsg, - "no entry point [%s] in shared library [%s]", zProc,zFile); + "no entry point [%s] in shared library [%s]", zEntry, zFile); sqlite3OsDlError(pVfs, nMsg-1, zErrmsg); } - sqlite3OsDlClose(pVfs, handle); } + sqlite3OsDlClose(pVfs, handle); + sqlite3_free(zAltEntry); return SQLITE_ERROR; - }else if( xInit(db, &zErrmsg, &sqlite3Apis) ){ + } + sqlite3_free(zAltEntry); + if( xInit(db, &zErrmsg, &sqlite3Apis) ){ if( pzErrMsg ){ *pzErrMsg = sqlite3_mprintf("error during initialization: %s", zErrmsg); } diff --git a/src/main.c b/src/main.c index 16294a6..39f6042 100644 --- a/src/main.c +++ b/src/main.c @@ -132,6 +132,13 @@ int sqlite3_initialize(void){ */ if( sqlite3GlobalConfig.isInit ) return SQLITE_OK; +#ifdef SQLITE_ENABLE_SQLLOG + { + extern void sqlite3_init_sqllog(void); + sqlite3_init_sqllog(); + } +#endif + /* Make sure the mutex subsystem is initialized. If unable to ** initialize the mutex subsystem, return early with the error. ** If the system is so sick that we are unable to allocate a mutex, @@ -475,6 +482,33 @@ int sqlite3_config(int op, ...){ break; } + case SQLITE_CONFIG_COVERING_INDEX_SCAN: { + sqlite3GlobalConfig.bUseCis = va_arg(ap, int); + break; + } + +#ifdef SQLITE_ENABLE_SQLLOG + case SQLITE_CONFIG_SQLLOG: { + typedef void(*SQLLOGFUNC_t)(void*, sqlite3*, const char*, int); + sqlite3GlobalConfig.xSqllog = va_arg(ap, SQLLOGFUNC_t); + sqlite3GlobalConfig.pSqllogArg = va_arg(ap, void *); + break; + } +#endif + + case SQLITE_CONFIG_MMAP_SIZE: { + sqlite3_int64 szMmap = va_arg(ap, sqlite3_int64); + sqlite3_int64 mxMmap = va_arg(ap, sqlite3_int64); + if( mxMmap<0 || mxMmap>SQLITE_MAX_MMAP_SIZE ){ + mxMmap = SQLITE_MAX_MMAP_SIZE; + } + sqlite3GlobalConfig.mxMmap = mxMmap; + if( szMmap<0 ) szMmap = SQLITE_DEFAULT_MMAP_SIZE; + if( szMmap>mxMmap) szMmap = mxMmap; + sqlite3GlobalConfig.szMmap = szMmap; + break; + } + default: { rc = SQLITE_ERROR; break; @@ -814,6 +848,13 @@ static int sqlite3Close(sqlite3 *db, int forceZombie){ return SQLITE_BUSY; } +#ifdef SQLITE_ENABLE_SQLLOG + if( sqlite3GlobalConfig.xSqllog ){ + /* Closing the handle. Fourth parameter is passed the value 2. */ + sqlite3GlobalConfig.xSqllog(sqlite3GlobalConfig.pSqllogArg, db, 0, 2); + } +#endif + /* Convert the connection into a zombie and then close it. */ db->magic = SQLITE_MAGIC_ZOMBIE; @@ -857,10 +898,16 @@ void sqlite3LeaveMutexAndCloseZombie(sqlite3 *db){ /* If we reach this point, it means that the database connection has ** closed all sqlite3_stmt and sqlite3_backup objects and has been - ** pased to sqlite3_close (meaning that it is a zombie). Therefore, + ** passed to sqlite3_close (meaning that it is a zombie). Therefore, ** go ahead and free all resources. */ + /* If a transaction is open, roll it back. This also ensures that if + ** any database schemas have been modified by an uncommitted transaction + ** they are reset. And that the required b-tree mutex is held to make + ** the pager rollback and schema reset an atomic operation. */ + sqlite3RollbackAll(db, SQLITE_OK); + /* Free any outstanding Savepoint structures. */ sqlite3CloseSavepoints(db); @@ -961,6 +1008,15 @@ void sqlite3RollbackAll(sqlite3 *db, int tripCode){ int inTrans = 0; assert( sqlite3_mutex_held(db->mutex) ); sqlite3BeginBenignMalloc(); + + /* Obtain all b-tree mutexes before making any calls to BtreeRollback(). + ** This is important in case the transaction being rolled back has + ** modified the database schema. If the b-tree mutexes are not taken + ** here, then another shared-cache connection might sneak in between + ** the database rollback and schema reset, which can cause false + ** corruption reports in some cases. */ + sqlite3BtreeEnterAll(db); + for(i=0; inDb; i++){ Btree *p = db->aDb[i].pBt; if( p ){ @@ -974,10 +1030,11 @@ void sqlite3RollbackAll(sqlite3 *db, int tripCode){ sqlite3VtabRollback(db); sqlite3EndBenignMalloc(); - if( db->flags&SQLITE_InternChanges ){ + if( (db->flags&SQLITE_InternChanges)!=0 && db->init.busy==0 ){ sqlite3ExpirePreparedStatements(db); sqlite3ResetAllSchemasOfConnection(db); } + sqlite3BtreeLeaveAll(db); /* Any deferred constraint violations have now been resolved. */ db->nDeferredCons = 0; @@ -988,6 +1045,110 @@ void sqlite3RollbackAll(sqlite3 *db, int tripCode){ } } +/* +** Return a static string containing the name corresponding to the error code +** specified in the argument. +*/ +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) || \ + defined(SQLITE_DEBUG_OS_TRACE) +const char *sqlite3ErrName(int rc){ + const char *zName = 0; + int i, origRc = rc; + for(i=0; i<2 && zName==0; i++, rc &= 0xff){ + switch( rc ){ + case SQLITE_OK: zName = "SQLITE_OK"; break; + case SQLITE_ERROR: zName = "SQLITE_ERROR"; break; + case SQLITE_INTERNAL: zName = "SQLITE_INTERNAL"; break; + case SQLITE_PERM: zName = "SQLITE_PERM"; break; + case SQLITE_ABORT: zName = "SQLITE_ABORT"; break; + case SQLITE_ABORT_ROLLBACK: zName = "SQLITE_ABORT_ROLLBACK"; break; + case SQLITE_BUSY: zName = "SQLITE_BUSY"; break; + case SQLITE_BUSY_RECOVERY: zName = "SQLITE_BUSY_RECOVERY"; break; + case SQLITE_LOCKED: zName = "SQLITE_LOCKED"; break; + case SQLITE_LOCKED_SHAREDCACHE: zName = "SQLITE_LOCKED_SHAREDCACHE";break; + case SQLITE_NOMEM: zName = "SQLITE_NOMEM"; break; + case SQLITE_READONLY: zName = "SQLITE_READONLY"; break; + case SQLITE_READONLY_RECOVERY: zName = "SQLITE_READONLY_RECOVERY"; break; + case SQLITE_READONLY_CANTLOCK: zName = "SQLITE_READONLY_CANTLOCK"; break; + case SQLITE_READONLY_ROLLBACK: zName = "SQLITE_READONLY_ROLLBACK"; break; + case SQLITE_INTERRUPT: zName = "SQLITE_INTERRUPT"; break; + case SQLITE_IOERR: zName = "SQLITE_IOERR"; break; + case SQLITE_IOERR_READ: zName = "SQLITE_IOERR_READ"; break; + case SQLITE_IOERR_SHORT_READ: zName = "SQLITE_IOERR_SHORT_READ"; break; + case SQLITE_IOERR_WRITE: zName = "SQLITE_IOERR_WRITE"; break; + case SQLITE_IOERR_FSYNC: zName = "SQLITE_IOERR_FSYNC"; break; + case SQLITE_IOERR_DIR_FSYNC: zName = "SQLITE_IOERR_DIR_FSYNC"; break; + case SQLITE_IOERR_TRUNCATE: zName = "SQLITE_IOERR_TRUNCATE"; break; + case SQLITE_IOERR_FSTAT: zName = "SQLITE_IOERR_FSTAT"; break; + case SQLITE_IOERR_UNLOCK: zName = "SQLITE_IOERR_UNLOCK"; break; + case SQLITE_IOERR_RDLOCK: zName = "SQLITE_IOERR_RDLOCK"; break; + case SQLITE_IOERR_DELETE: zName = "SQLITE_IOERR_DELETE"; break; + case SQLITE_IOERR_BLOCKED: zName = "SQLITE_IOERR_BLOCKED"; break; + case SQLITE_IOERR_NOMEM: zName = "SQLITE_IOERR_NOMEM"; break; + case SQLITE_IOERR_ACCESS: zName = "SQLITE_IOERR_ACCESS"; break; + case SQLITE_IOERR_CHECKRESERVEDLOCK: + zName = "SQLITE_IOERR_CHECKRESERVEDLOCK"; break; + case SQLITE_IOERR_LOCK: zName = "SQLITE_IOERR_LOCK"; break; + case SQLITE_IOERR_CLOSE: zName = "SQLITE_IOERR_CLOSE"; break; + case SQLITE_IOERR_DIR_CLOSE: zName = "SQLITE_IOERR_DIR_CLOSE"; break; + case SQLITE_IOERR_SHMOPEN: zName = "SQLITE_IOERR_SHMOPEN"; break; + case SQLITE_IOERR_SHMSIZE: zName = "SQLITE_IOERR_SHMSIZE"; break; + case SQLITE_IOERR_SHMLOCK: zName = "SQLITE_IOERR_SHMLOCK"; break; + case SQLITE_IOERR_SHMMAP: zName = "SQLITE_IOERR_SHMMAP"; break; + case SQLITE_IOERR_SEEK: zName = "SQLITE_IOERR_SEEK"; break; + case SQLITE_IOERR_DELETE_NOENT: zName = "SQLITE_IOERR_DELETE_NOENT";break; + case SQLITE_IOERR_MMAP: zName = "SQLITE_IOERR_MMAP"; break; + case SQLITE_CORRUPT: zName = "SQLITE_CORRUPT"; break; + case SQLITE_CORRUPT_VTAB: zName = "SQLITE_CORRUPT_VTAB"; break; + case SQLITE_NOTFOUND: zName = "SQLITE_NOTFOUND"; break; + case SQLITE_FULL: zName = "SQLITE_FULL"; break; + case SQLITE_CANTOPEN: zName = "SQLITE_CANTOPEN"; break; + case SQLITE_CANTOPEN_NOTEMPDIR: zName = "SQLITE_CANTOPEN_NOTEMPDIR";break; + case SQLITE_CANTOPEN_ISDIR: zName = "SQLITE_CANTOPEN_ISDIR"; break; + case SQLITE_CANTOPEN_FULLPATH: zName = "SQLITE_CANTOPEN_FULLPATH"; break; + case SQLITE_PROTOCOL: zName = "SQLITE_PROTOCOL"; break; + case SQLITE_EMPTY: zName = "SQLITE_EMPTY"; break; + case SQLITE_SCHEMA: zName = "SQLITE_SCHEMA"; break; + case SQLITE_TOOBIG: zName = "SQLITE_TOOBIG"; break; + case SQLITE_CONSTRAINT: zName = "SQLITE_CONSTRAINT"; break; + case SQLITE_CONSTRAINT_UNIQUE: zName = "SQLITE_CONSTRAINT_UNIQUE"; break; + case SQLITE_CONSTRAINT_TRIGGER: zName = "SQLITE_CONSTRAINT_TRIGGER";break; + case SQLITE_CONSTRAINT_FOREIGNKEY: + zName = "SQLITE_CONSTRAINT_FOREIGNKEY"; break; + case SQLITE_CONSTRAINT_CHECK: zName = "SQLITE_CONSTRAINT_CHECK"; break; + case SQLITE_CONSTRAINT_PRIMARYKEY: + zName = "SQLITE_CONSTRAINT_PRIMARYKEY"; break; + case SQLITE_CONSTRAINT_NOTNULL: zName = "SQLITE_CONSTRAINT_NOTNULL";break; + case SQLITE_CONSTRAINT_COMMITHOOK: + zName = "SQLITE_CONSTRAINT_COMMITHOOK"; break; + case SQLITE_CONSTRAINT_VTAB: zName = "SQLITE_CONSTRAINT_VTAB"; break; + case SQLITE_CONSTRAINT_FUNCTION: + zName = "SQLITE_CONSTRAINT_FUNCTION"; break; + case SQLITE_MISMATCH: zName = "SQLITE_MISMATCH"; break; + case SQLITE_MISUSE: zName = "SQLITE_MISUSE"; break; + case SQLITE_NOLFS: zName = "SQLITE_NOLFS"; break; + case SQLITE_AUTH: zName = "SQLITE_AUTH"; break; + case SQLITE_FORMAT: zName = "SQLITE_FORMAT"; break; + case SQLITE_RANGE: zName = "SQLITE_RANGE"; break; + case SQLITE_NOTADB: zName = "SQLITE_NOTADB"; break; + case SQLITE_ROW: zName = "SQLITE_ROW"; break; + case SQLITE_NOTICE: zName = "SQLITE_NOTICE"; break; + case SQLITE_NOTICE_RECOVER_WAL: zName = "SQLITE_NOTICE_RECOVER_WAL";break; + case SQLITE_NOTICE_RECOVER_ROLLBACK: + zName = "SQLITE_NOTICE_RECOVER_ROLLBACK"; break; + case SQLITE_WARNING: zName = "SQLITE_WARNING"; break; + case SQLITE_DONE: zName = "SQLITE_DONE"; break; + } + } + if( zName==0 ){ + static char zBuf[50]; + sqlite3_snprintf(sizeof(zBuf), zBuf, "SQLITE_UNKNOWN(%d)", origRc); + zName = zBuf; + } + return zName; +} +#endif + /* ** Return a static string that describes the kind of error specified in the ** argument. @@ -1116,6 +1277,7 @@ int sqlite3_busy_handler( db->busyHandler.xFunc = xBusy; db->busyHandler.pArg = pArg; db->busyHandler.nBusy = 0; + db->busyTimeout = 0; sqlite3_mutex_leave(db->mutex); return SQLITE_OK; } @@ -1153,8 +1315,8 @@ void sqlite3_progress_handler( */ int sqlite3_busy_timeout(sqlite3 *db, int ms){ if( ms>0 ){ - db->busyTimeout = ms; sqlite3_busy_handler(db, sqliteDefaultBusyCallback, (void*)db); + db->busyTimeout = ms; }else{ sqlite3_busy_handler(db, 0, 0); } @@ -1767,6 +1929,15 @@ int sqlite3_extended_errcode(sqlite3 *db){ return db->errCode; } +/* +** Return a string that describes the kind of error specified in the +** argument. For now, this simply calls the internal sqlite3ErrStr() +** function. +*/ +const char *sqlite3_errstr(int rc){ + return sqlite3ErrStr(rc); +} + /* ** Create a new collating function for database "db". The name is zName ** and the encoding is enc. @@ -2278,6 +2449,7 @@ static int openDatabase( memcpy(db->aLimit, aHardLimit, sizeof(db->aLimit)); db->autoCommit = 1; db->nextAutovac = -1; + db->szMmap = sqlite3GlobalConfig.szMmap; db->nextPagesize = 0; db->flags |= SQLITE_ShortColNames | SQLITE_AutoIndex | SQLITE_EnableTrigger #if SQLITE_DEFAULT_FILE_FORMAT<4 @@ -2436,6 +2608,13 @@ opendb_out: db->magic = SQLITE_MAGIC_SICK; } *ppDb = db; +#ifdef SQLITE_ENABLE_SQLLOG + if( sqlite3GlobalConfig.xSqllog ){ + /* Opening a db handle. Fourth parameter is passed 0. */ + void *pArg = sqlite3GlobalConfig.pSqllogArg; + sqlite3GlobalConfig.xSqllog(pArg, db, zFilename, 0); + } +#endif return sqlite3ApiExit(0, rc); } @@ -2741,7 +2920,7 @@ int sqlite3_table_column_metadata( zDataType = pCol->zType; zCollSeq = pCol->zColl; notnull = pCol->notNull!=0; - primarykey = pCol->isPrimKey!=0; + primarykey = (pCol->colFlags & COLFLAG_PRIMKEY)!=0; autoinc = pTab->iPKey==iCol && (pTab->tabFlags & TF_Autoincrement)!=0; }else{ zDataType = "INTEGER"; @@ -3004,8 +3183,7 @@ int sqlite3_test_control(int op, ...){ */ case SQLITE_TESTCTRL_OPTIMIZATIONS: { sqlite3 *db = va_arg(ap, sqlite3*); - int x = va_arg(ap,int); - db->flags = (x & SQLITE_OptMask) | (db->flags & ~SQLITE_OptMask); + db->dbOptFlags = (u16)(va_arg(ap, int) & 0xffff); break; } diff --git a/src/memjournal.c b/src/memjournal.c index 3e66e21..0572594 100644 --- a/src/memjournal.c +++ b/src/memjournal.c @@ -230,7 +230,9 @@ static const struct sqlite3_io_methods MemJournalMethods = { 0, /* xShmMap */ 0, /* xShmLock */ 0, /* xShmBarrier */ - 0 /* xShmUnlock */ + 0, /* xShmUnmap */ + 0, /* xFetch */ + 0 /* xUnfetch */ }; /* diff --git a/src/os.c b/src/os.c index b5e918a..be2ea4c 100644 --- a/src/os.c +++ b/src/os.c @@ -141,6 +141,26 @@ int sqlite3OsShmMap( return id->pMethods->xShmMap(id, iPage, pgsz, bExtend, pp); } +#if SQLITE_MAX_MMAP_SIZE>0 +/* The real implementation of xFetch and xUnfetch */ +int sqlite3OsFetch(sqlite3_file *id, i64 iOff, int iAmt, void **pp){ + DO_OS_MALLOC_TEST(id); + return id->pMethods->xFetch(id, iOff, iAmt, pp); +} +int sqlite3OsUnfetch(sqlite3_file *id, i64 iOff, void *p){ + return id->pMethods->xUnfetch(id, iOff, p); +} +#else +/* No-op stubs to use when memory-mapped I/O is disabled */ +int sqlite3OsFetch(sqlite3_file *id, i64 iOff, int iAmt, void **pp){ + *pp = 0; + return SQLITE_OK; +} +int sqlite3OsUnfetch(sqlite3_file *id, i64 iOff, void *p){ + return SQLITE_OK; +} +#endif + /* ** The next group of routines are convenience wrappers around the ** VFS methods. diff --git a/src/os.h b/src/os.h index 1ec7d4b..070a2dd 100644 --- a/src/os.h +++ b/src/os.h @@ -99,14 +99,6 @@ # define SQLITE_OS_WINRT 0 #endif -/* -** When compiled for WinCE or WinRT, there is no concept of the current -** directory. - */ -#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT -# define SQLITE_CURDIR 1 -#endif - /* If the SET_FULLSYNC macro is not defined above, then make it ** a no-op */ @@ -259,6 +251,8 @@ int sqlite3OsShmMap(sqlite3_file *,int,int,int,void volatile **); int sqlite3OsShmLock(sqlite3_file *id, int, int, int); void sqlite3OsShmBarrier(sqlite3_file *id); int sqlite3OsShmUnmap(sqlite3_file *id, int); +int sqlite3OsFetch(sqlite3_file *id, i64, int, void **); +int sqlite3OsUnfetch(sqlite3_file *, i64, void *); /* diff --git a/src/os_unix.c b/src/os_unix.c index c0df66e..abc23a4 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -46,6 +46,13 @@ #include "sqliteInt.h" #if SQLITE_OS_UNIX /* This file is used on unix only */ +/* Use posix_fallocate() if it is available +*/ +#if !defined(HAVE_POSIX_FALLOCATE) \ + && (_XOPEN_SOURCE >= 600 || _POSIX_C_SOURCE >= 200112L) +# define HAVE_POSIX_FALLOCATE 1 +#endif + /* ** There are various methods for file locking used for concurrency ** control: @@ -119,7 +126,7 @@ #include #include #include -#ifndef SQLITE_OMIT_WAL +#if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 #include #endif @@ -218,6 +225,15 @@ struct unixFile { const char *zPath; /* Name of the file */ unixShm *pShm; /* Shared memory segment information */ int szChunk; /* Configured by FCNTL_CHUNK_SIZE */ + int nFetchOut; /* Number of outstanding xFetch refs */ + sqlite3_int64 mmapSize; /* Usable size of mapping at pMapRegion */ + sqlite3_int64 mmapSizeActual; /* Actual size of mapping at pMapRegion */ + sqlite3_int64 mmapSizeMax; /* Configured FCNTL_MMAP_SIZE value */ + void *pMapRegion; /* Memory mapped region */ +#ifdef __QNXNTO__ + int sectorSize; /* Device sector size */ + int deviceCharacteristics; /* Precomputed device characteristics */ +#endif #if SQLITE_ENABLE_LOCKING_STYLE int openFlags; /* The flags specified at open() */ #endif @@ -238,7 +254,9 @@ struct unixFile { unsigned char transCntrChng; /* True if the transaction counter changed */ unsigned char dbUpdate; /* True if any part of database file changed */ unsigned char inNormalWrite; /* True if in a normal write operation */ + #endif + #ifdef SQLITE_TEST /* In test mode, increase the size of this structure a bit so that ** it is larger than the struct CrashFile defined in test6.c. @@ -262,6 +280,7 @@ struct unixFile { #define UNIXFILE_DELETE 0x20 /* Delete on close */ #define UNIXFILE_URI 0x40 /* Filename might have query parameters */ #define UNIXFILE_NOLOCK 0x80 /* Do no file locking */ +#define UNIXFILE_WARNED 0x0100 /* verifyDbFile() warnings have been issued */ /* ** Include code that is common to all os_*.c files @@ -295,6 +314,17 @@ struct unixFile { #define threadid 0 #endif +/* +** HAVE_MREMAP defaults to true on Linux and false everywhere else. +*/ +#if !defined(HAVE_MREMAP) +# if defined(__linux__) && defined(_GNU_SOURCE) +# define HAVE_MREMAP 1 +# else +# define HAVE_MREMAP 0 +# endif +#endif + /* ** Different Unix systems declare open() in different ways. Same use ** open(const char*,int,mode_t). Others use open(const char*,int,...). @@ -326,7 +356,7 @@ static int openDirectory(const char*, int*); ** to all overrideable system calls. */ static struct unix_syscall { - const char *zName; /* Name of the sytem call */ + const char *zName; /* Name of the system call */ sqlite3_syscall_ptr pCurrent; /* Current value of the system call */ sqlite3_syscall_ptr pDefault; /* Default value */ } aSyscall[] = { @@ -401,11 +431,7 @@ static struct unix_syscall { #define osPwrite64 ((ssize_t(*)(int,const void*,size_t,off_t))\ aSyscall[13].pCurrent) -#if SQLITE_ENABLE_LOCKING_STYLE { "fchmod", (sqlite3_syscall_ptr)fchmod, 0 }, -#else - { "fchmod", (sqlite3_syscall_ptr)0, 0 }, -#endif #define osFchmod ((int(*)(int,mode_t))aSyscall[14].pCurrent) #if defined(HAVE_POSIX_FALLOCATE) && HAVE_POSIX_FALLOCATE @@ -430,8 +456,18 @@ static struct unix_syscall { { "fchown", (sqlite3_syscall_ptr)posixFchown, 0 }, #define osFchown ((int(*)(int,uid_t,gid_t))aSyscall[20].pCurrent) - { "umask", (sqlite3_syscall_ptr)umask, 0 }, -#define osUmask ((mode_t(*)(mode_t))aSyscall[21].pCurrent) + { "mmap", (sqlite3_syscall_ptr)mmap, 0 }, +#define osMmap ((void*(*)(void*,size_t,int,int,int,off_t))aSyscall[21].pCurrent) + + { "munmap", (sqlite3_syscall_ptr)munmap, 0 }, +#define osMunmap ((void*(*)(void*,size_t))aSyscall[22].pCurrent) + +#if HAVE_MREMAP + { "mremap", (sqlite3_syscall_ptr)mremap, 0 }, +#else + { "mremap", (sqlite3_syscall_ptr)0, 0 }, +#endif +#define osMremap ((void*(*)(void*,size_t,size_t,int,...))aSyscall[23].pCurrent) }; /* End of the overrideable system calls */ @@ -537,14 +573,7 @@ static const char *unixNextSystemCall(sqlite3_vfs *p, const char *zName){ */ static int robust_open(const char *z, int f, mode_t m){ int fd; - mode_t m2; - mode_t origM = 0; - if( m==0 ){ - m2 = SQLITE_DEFAULT_FILE_PERMISSIONS; - }else{ - m2 = m; - origM = osUmask(0); - } + mode_t m2 = m ? m : SQLITE_DEFAULT_FILE_PERMISSIONS; do{ #if defined(O_CLOEXEC) fd = osOpen(z,f|O_CLOEXEC,m2); @@ -552,12 +581,20 @@ static int robust_open(const char *z, int f, mode_t m){ fd = osOpen(z,f,m2); #endif }while( fd<0 && errno==EINTR ); - if( m ){ - osUmask(origM); - } + if( fd>=0 ){ + if( m!=0 ){ + struct stat statbuf; + if( osFstat(fd, &statbuf)==0 + && statbuf.st_size==0 + && (statbuf.st_mode&0777)!=m + ){ + osFchmod(fd, m); + } + } #if defined(FD_CLOEXEC) && (!defined(O_CLOEXEC) || O_CLOEXEC==0) - if( fd>=0 ) osFcntl(fd, F_SETFD, osFcntl(fd, F_GETFD, 0) | FD_CLOEXEC); + osFcntl(fd, F_SETFD, osFcntl(fd, F_GETFD, 0) | FD_CLOEXEC); #endif + } return fd; } @@ -763,7 +800,6 @@ static int sqliteErrorFromPosixError(int posixError, int sqliteIOErr) { } - /****************************************************************************** ****************** Begin Unique File ID Utility Used By VxWorks *************** ** @@ -1099,7 +1135,6 @@ static int unixLogErrorAtLine( zErr = strerror(iErrno); #endif - assert( errcode!=SQLITE_OK ); if( zPath==0 ) zPath = ""; sqlite3_log(errcode, "os_unix.c:%d: (%d) %s(%s) - %s", @@ -1265,6 +1300,50 @@ static int findInodeInfo( } +/* +** Check a unixFile that is a database. Verify the following: +** +** (1) There is exactly one hard link on the file +** (2) The file is not a symbolic link +** (3) The file has not been renamed or unlinked +** +** Issue sqlite3_log(SQLITE_WARNING,...) messages if anything is not right. +*/ +static void verifyDbFile(unixFile *pFile){ + struct stat buf; + int rc; + if( pFile->ctrlFlags & UNIXFILE_WARNED ){ + /* One or more of the following warnings have already been issued. Do not + ** repeat them so as not to clutter the error log */ + return; + } + rc = osFstat(pFile->h, &buf); + if( rc!=0 ){ + sqlite3_log(SQLITE_WARNING, "cannot fstat db file %s", pFile->zPath); + pFile->ctrlFlags |= UNIXFILE_WARNED; + return; + } + if( buf.st_nlink==0 && (pFile->ctrlFlags & UNIXFILE_DELETE)==0 ){ + sqlite3_log(SQLITE_WARNING, "file unlinked while open: %s", pFile->zPath); + pFile->ctrlFlags |= UNIXFILE_WARNED; + return; + } + if( buf.st_nlink>1 ){ + sqlite3_log(SQLITE_WARNING, "multiple links to file: %s", pFile->zPath); + pFile->ctrlFlags |= UNIXFILE_WARNED; + return; + } + if( pFile->pInode!=0 + && ((rc = osStat(pFile->zPath, &buf))!=0 + || buf.st_ino!=pFile->pInode->fileId.ino) + ){ + sqlite3_log(SQLITE_WARNING, "file renamed while open: %s", pFile->zPath); + pFile->ctrlFlags |= UNIXFILE_WARNED; + return; + } +} + + /* ** This routine checks if there is a RESERVED lock held on the specified ** file by this or any other process. If such a lock is held, set *pResOut @@ -1795,9 +1874,13 @@ end_unlock: ** the requested locking level, this routine is a no-op. */ static int unixUnlock(sqlite3_file *id, int eFileLock){ + assert( eFileLock==SHARED_LOCK || ((unixFile *)id)->nFetchOut==0 ); return posixUnlock(id, eFileLock, 0); } +static int unixMapfile(unixFile *pFd, i64 nByte); +static void unixUnmapfile(unixFile *pFd); + /* ** This function performs the parts of the "close file" operation ** common to all locking schemes. It closes the directory and file @@ -1810,6 +1893,7 @@ static int unixUnlock(sqlite3_file *id, int eFileLock){ */ static int closeUnixFile(sqlite3_file *id){ unixFile *pFile = (unixFile*)id; + unixUnmapfile(pFile); if( pFile->h>=0 ){ robust_close(pFile, pFile->h, __LINE__); pFile->h = -1; @@ -1836,6 +1920,7 @@ static int closeUnixFile(sqlite3_file *id){ static int unixClose(sqlite3_file *id){ int rc = SQLITE_OK; unixFile *pFile = (unixFile *)id; + verifyDbFile(pFile); unixUnlock(id, NO_LOCK); unixEnterMutex(); @@ -1904,7 +1989,7 @@ static int nolockClose(sqlite3_file *id) { /****************************************************************************** ************************* Begin dot-file Locking ****************************** ** -** The dotfile locking implementation uses the existance of separate lock +** The dotfile locking implementation uses the existence of separate lock ** files (really a directory) to control access to the database. This works ** on just about every filesystem imaginable. But there are serious downsides: ** @@ -1919,7 +2004,7 @@ static int nolockClose(sqlite3_file *id) { ** ** Dotfile locking works by creating a subdirectory in the same directory as ** the database and with the same name but with a ".lock" extension added. -** The existance of a lock directory implies an EXCLUSIVE lock. All other +** The existence of a lock directory implies an EXCLUSIVE lock. All other ** lock types (SHARED, RESERVED, PENDING) are mapped into EXCLUSIVE. */ @@ -2086,13 +2171,13 @@ static int dotlockUnlock(sqlite3_file *id, int eFileLock) { ** Close a file. Make sure the lock has been released before closing. */ static int dotlockClose(sqlite3_file *id) { - int rc; + int rc = SQLITE_OK; if( id ){ unixFile *pFile = (unixFile*)id; dotlockUnlock(id, NO_LOCK); sqlite3_free(pFile->lockingContext); + rc = closeUnixFile(id); } - rc = closeUnixFile(id); return rc; } /****************** End of the dot-file lock implementation ******************* @@ -2296,10 +2381,12 @@ static int flockUnlock(sqlite3_file *id, int eFileLock) { ** Close a file. */ static int flockClose(sqlite3_file *id) { + int rc = SQLITE_OK; if( id ){ flockUnlock(id, NO_LOCK); + rc = closeUnixFile(id); } - return closeUnixFile(id); + return rc; } #endif /* SQLITE_ENABLE_LOCKING_STYLE && !OS_VXWORK */ @@ -3010,6 +3097,8 @@ static int seekAndRead(unixFile *id, sqlite3_int64 offset, void *pBuf, int cnt){ i64 newOffset; #endif TIMER_START; + assert( cnt==(cnt&0x1ffff) ); + cnt &= 0x1ffff; do{ #if defined(USE_PREAD) got = osPread(id->h, pBuf, cnt, offset); @@ -3063,6 +3152,8 @@ static int unixRead( unixFile *pFile = (unixFile *)id; int got; assert( id ); + assert( offset>=0 ); + assert( amt>0 ); /* If this is a database file (not a journal, master-journal or temp ** file), the bytes in the locking range should never be read or written. */ @@ -3073,6 +3164,23 @@ static int unixRead( ); #endif +#if SQLITE_MAX_MMAP_SIZE>0 + /* Deal with as much of this read request as possible by transfering + ** data from the memory mapping using memcpy(). */ + if( offsetmmapSize ){ + if( offset+amt <= pFile->mmapSize ){ + memcpy(pBuf, &((u8 *)(pFile->pMapRegion))[offset], amt); + return SQLITE_OK; + }else{ + int nCopy = pFile->mmapSize - offset; + memcpy(pBuf, &((u8 *)(pFile->pMapRegion))[offset], nCopy); + pBuf = &((u8 *)pBuf)[nCopy]; + amt -= nCopy; + offset += nCopy; + } + } +#endif + got = seekAndRead(pFile, offset, pBuf, amt); if( got==amt ){ return SQLITE_OK; @@ -3088,44 +3196,59 @@ static int unixRead( } /* -** Seek to the offset in id->offset then read cnt bytes into pBuf. -** Return the number of bytes actually read. Update the offset. -** -** To avoid stomping the errno value on a failed write the lastErrno value -** is set before returning. +** Attempt to seek the file-descriptor passed as the first argument to +** absolute offset iOff, then attempt to write nBuf bytes of data from +** pBuf to it. If an error occurs, return -1 and set *piErrno. Otherwise, +** return the actual number of bytes written (which may be less than +** nBuf). */ -static int seekAndWrite(unixFile *id, i64 offset, const void *pBuf, int cnt){ - int got; -#if (!defined(USE_PREAD) && !defined(USE_PREAD64)) - i64 newOffset; -#endif +static int seekAndWriteFd( + int fd, /* File descriptor to write to */ + i64 iOff, /* File offset to begin writing at */ + const void *pBuf, /* Copy data from this buffer to the file */ + int nBuf, /* Size of buffer pBuf in bytes */ + int *piErrno /* OUT: Error number if error occurs */ +){ + int rc = 0; /* Value returned by system call */ + + assert( nBuf==(nBuf&0x1ffff) ); + nBuf &= 0x1ffff; TIMER_START; + #if defined(USE_PREAD) - do{ got = osPwrite(id->h, pBuf, cnt, offset); }while( got<0 && errno==EINTR ); + do{ rc = osPwrite(fd, pBuf, nBuf, iOff); }while( rc<0 && errno==EINTR ); #elif defined(USE_PREAD64) - do{ got = osPwrite64(id->h, pBuf, cnt, offset);}while( got<0 && errno==EINTR); + do{ rc = osPwrite64(fd, pBuf, nBuf, iOff);}while( rc<0 && errno==EINTR); #else do{ - newOffset = lseek(id->h, offset, SEEK_SET); - SimulateIOError( newOffset-- ); - if( newOffset!=offset ){ - if( newOffset == -1 ){ - ((unixFile*)id)->lastErrno = errno; - }else{ - ((unixFile*)id)->lastErrno = 0; - } + i64 iSeek = lseek(fd, iOff, SEEK_SET); + SimulateIOError( iSeek-- ); + + if( iSeek!=iOff ){ + if( piErrno ) *piErrno = (iSeek==-1 ? errno : 0); return -1; } - got = osWrite(id->h, pBuf, cnt); - }while( got<0 && errno==EINTR ); + rc = osWrite(fd, pBuf, nBuf); + }while( rc<0 && errno==EINTR ); #endif + TIMER_END; - if( got<0 ){ - ((unixFile*)id)->lastErrno = errno; - } + OSTRACE(("WRITE %-3d %5d %7lld %llu\n", fd, rc, iOff, TIMER_ELAPSED)); - OSTRACE(("WRITE %-3d %5d %7lld %llu\n", id->h, got, offset, TIMER_ELAPSED)); - return got; + if( rc<0 && piErrno ) *piErrno = errno; + return rc; +} + + +/* +** Seek to the offset in id->offset then read cnt bytes into pBuf. +** Return the number of bytes actually read. Update the offset. +** +** To avoid stomping the errno value on a failed write the lastErrno value +** is set before returning. +*/ +static int seekAndWrite(unixFile *id, i64 offset, const void *pBuf, int cnt){ + return seekAndWriteFd(id->h, offset, pBuf, cnt, &id->lastErrno); } @@ -3175,6 +3298,23 @@ static int unixWrite( } #endif +#if SQLITE_MAX_MMAP_SIZE>0 + /* Deal with as much of this write request as possible by transfering + ** data from the memory mapping using memcpy(). */ + if( offsetmmapSize ){ + if( offset+amt <= pFile->mmapSize ){ + memcpy(&((u8 *)(pFile->pMapRegion))[offset], pBuf, amt); + return SQLITE_OK; + }else{ + int nCopy = pFile->mmapSize - offset; + memcpy(&((u8 *)(pFile->pMapRegion))[offset], pBuf, nCopy); + pBuf = &((u8 *)pBuf)[nCopy]; + amt -= nCopy; + offset += nCopy; + } + } +#endif + while( amt>0 && (wrote = seekAndWrite(pFile, offset, pBuf, amt))>0 ){ amt -= wrote; offset += wrote; @@ -3402,7 +3542,7 @@ static int unixSync(sqlite3_file *id, int flags){ } /* Also fsync the directory containing the file if the DIRSYNC flag - ** is set. This is a one-time occurrance. Many systems (examples: AIX) + ** is set. This is a one-time occurrence. Many systems (examples: AIX) ** are unable to fsync a directory, so ignore errors on the fsync. */ if( pFile->ctrlFlags & UNIXFILE_DIRSYNC ){ @@ -3457,6 +3597,14 @@ static int unixTruncate(sqlite3_file *id, i64 nByte){ } #endif + /* If the file was just truncated to a size smaller than the currently + ** mapped region, reduce the effective mapping size as well. SQLite will + ** use read() and write() to access data beyond this point from now on. + */ + if( nBytemmapSize ){ + pFile->mmapSize = nByte; + } + return SQLITE_OK; } } @@ -3545,6 +3693,19 @@ static int fcntlSizeHint(unixFile *pFile, i64 nByte){ } } + if( pFile->mmapSizeMax>0 && nByte>pFile->mmapSize ){ + int rc; + if( pFile->szChunk<=0 ){ + if( robust_ftruncate(pFile->h, nByte) ){ + pFile->lastErrno = errno; + return unixLogError(SQLITE_IOERR_TRUNCATE, "ftruncate", pFile->zPath); + } + } + + rc = unixMapfile(pFile, nByte); + return rc; + } + return SQLITE_OK; } @@ -3564,6 +3725,9 @@ static void unixModeBit(unixFile *pFile, unsigned char mask, int *pArg){ } } +/* Forward declaration */ +static int unixGetTempname(int nBuf, char *zBuf); + /* ** Information and control of an open file handle. */ @@ -3601,6 +3765,26 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ *(char**)pArg = sqlite3_mprintf("%s", pFile->pVfs->zName); return SQLITE_OK; } + case SQLITE_FCNTL_TEMPFILENAME: { + char *zTFile = sqlite3_malloc( pFile->pVfs->mxPathname ); + if( zTFile ){ + unixGetTempname(pFile->pVfs->mxPathname, zTFile); + *(char**)pArg = zTFile; + } + return SQLITE_OK; + } + case SQLITE_FCNTL_MMAP_SIZE: { + i64 newLimit = *(i64*)pArg; + if( newLimit>sqlite3GlobalConfig.mxMmap ){ + newLimit = sqlite3GlobalConfig.mxMmap; + } + *(i64*)pArg = pFile->mmapSizeMax; + if( newLimit>=0 ){ + pFile->mmapSizeMax = newLimit; + if( newLimitmmapSize ) pFile->mmapSize = newLimit; + } + return SQLITE_OK; + } #ifdef SQLITE_DEBUG /* The pager calls this method to signal that it has done ** a rollback and that the database is therefore unchanged and @@ -3632,10 +3816,92 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ ** a database and its journal file) that the sector size will be the ** same for both. */ -static int unixSectorSize(sqlite3_file *pFile){ - (void)pFile; +#ifndef __QNXNTO__ +static int unixSectorSize(sqlite3_file *NotUsed){ + UNUSED_PARAMETER(NotUsed); return SQLITE_DEFAULT_SECTOR_SIZE; } +#endif + +/* +** The following version of unixSectorSize() is optimized for QNX. +*/ +#ifdef __QNXNTO__ +#include +#include +static int unixSectorSize(sqlite3_file *id){ + unixFile *pFile = (unixFile*)id; + if( pFile->sectorSize == 0 ){ + struct statvfs fsInfo; + + /* Set defaults for non-supported filesystems */ + pFile->sectorSize = SQLITE_DEFAULT_SECTOR_SIZE; + pFile->deviceCharacteristics = 0; + if( fstatvfs(pFile->h, &fsInfo) == -1 ) { + return pFile->sectorSize; + } + + if( !strcmp(fsInfo.f_basetype, "tmp") ) { + pFile->sectorSize = fsInfo.f_bsize; + pFile->deviceCharacteristics = + SQLITE_IOCAP_ATOMIC4K | /* All ram filesystem writes are atomic */ + SQLITE_IOCAP_SAFE_APPEND | /* growing the file does not occur until + ** the write succeeds */ + SQLITE_IOCAP_SEQUENTIAL | /* The ram filesystem has no write behind + ** so it is ordered */ + 0; + }else if( strstr(fsInfo.f_basetype, "etfs") ){ + pFile->sectorSize = fsInfo.f_bsize; + pFile->deviceCharacteristics = + /* etfs cluster size writes are atomic */ + (pFile->sectorSize / 512 * SQLITE_IOCAP_ATOMIC512) | + SQLITE_IOCAP_SAFE_APPEND | /* growing the file does not occur until + ** the write succeeds */ + SQLITE_IOCAP_SEQUENTIAL | /* The ram filesystem has no write behind + ** so it is ordered */ + 0; + }else if( !strcmp(fsInfo.f_basetype, "qnx6") ){ + pFile->sectorSize = fsInfo.f_bsize; + pFile->deviceCharacteristics = + SQLITE_IOCAP_ATOMIC | /* All filesystem writes are atomic */ + SQLITE_IOCAP_SAFE_APPEND | /* growing the file does not occur until + ** the write succeeds */ + SQLITE_IOCAP_SEQUENTIAL | /* The ram filesystem has no write behind + ** so it is ordered */ + 0; + }else if( !strcmp(fsInfo.f_basetype, "qnx4") ){ + pFile->sectorSize = fsInfo.f_bsize; + pFile->deviceCharacteristics = + /* full bitset of atomics from max sector size and smaller */ + ((pFile->sectorSize / 512 * SQLITE_IOCAP_ATOMIC512) << 1) - 2 | + SQLITE_IOCAP_SEQUENTIAL | /* The ram filesystem has no write behind + ** so it is ordered */ + 0; + }else if( strstr(fsInfo.f_basetype, "dos") ){ + pFile->sectorSize = fsInfo.f_bsize; + pFile->deviceCharacteristics = + /* full bitset of atomics from max sector size and smaller */ + ((pFile->sectorSize / 512 * SQLITE_IOCAP_ATOMIC512) << 1) - 2 | + SQLITE_IOCAP_SEQUENTIAL | /* The ram filesystem has no write behind + ** so it is ordered */ + 0; + }else{ + pFile->deviceCharacteristics = + SQLITE_IOCAP_ATOMIC512 | /* blocks are atomic */ + SQLITE_IOCAP_SAFE_APPEND | /* growing the file does not occur until + ** the write succeeds */ + 0; + } + } + /* Last chance verification. If the sector size isn't a multiple of 512 + ** then it isn't valid.*/ + if( pFile->sectorSize % 512 != 0 ){ + pFile->deviceCharacteristics = 0; + pFile->sectorSize = SQLITE_DEFAULT_SECTOR_SIZE; + } + return pFile->sectorSize; +} +#endif /* __QNXNTO__ */ /* ** Return the device characteristics for the file. @@ -3652,11 +3918,15 @@ static int unixSectorSize(sqlite3_file *pFile){ */ static int unixDeviceCharacteristics(sqlite3_file *id){ unixFile *p = (unixFile*)id; + int rc = 0; +#ifdef __QNXNTO__ + if( p->sectorSize==0 ) unixSectorSize(id); + rc = p->deviceCharacteristics; +#endif if( p->ctrlFlags & UNIXFILE_PSOW ){ - return SQLITE_IOCAP_POWERSAFE_OVERWRITE; - }else{ - return 0; + rc |= SQLITE_IOCAP_POWERSAFE_OVERWRITE; } + return rc; } #ifndef SQLITE_OMIT_WAL @@ -3827,7 +4097,7 @@ static void unixShmPurge(unixFile *pFd){ sqlite3_mutex_free(p->mutex); for(i=0; inRegion; i++){ if( p->h>=0 ){ - munmap(p->apRegion[i], p->szRegion); + osMunmap(p->apRegion[i], p->szRegion); }else{ sqlite3_free(p->apRegion[i]); } @@ -4067,16 +4337,32 @@ static int unixShmMap( if( sStat.st_sizeh, nByte) ){ - rc = unixLogError(SQLITE_IOERR_SHMSIZE, "ftruncate", - pShmNode->zFilename); + if( !bExtend ){ goto shmpage_out; } + + /* Alternatively, if bExtend is true, extend the file. Do this by + ** writing a single byte to the end of each (OS) page being + ** allocated or extended. Technically, we need only write to the + ** last page in order to extend the file. But writing to all new + ** pages forces the OS to allocate them immediately, which reduces + ** the chances of SIGBUS while accessing the mapped region later on. + */ + else{ + static const int pgsz = 4096; + int iPg; + + /* Write to the last byte of each newly allocated or extended page */ + assert( (nByte % pgsz)==0 ); + for(iPg=(sStat.st_size/pgsz); iPg<(nByte/pgsz); iPg++){ + if( seekAndWriteFd(pShmNode->h, iPg*pgsz + pgsz-1, "", 1, 0)!=1 ){ + const char *zFile = pShmNode->zFilename; + rc = unixLogError(SQLITE_IOERR_SHMSIZE, "write", zFile); + goto shmpage_out; + } + } + } } } @@ -4092,9 +4378,9 @@ static int unixShmMap( while(pShmNode->nRegion<=iRegion){ void *pMem; if( pShmNode->h>=0 ){ - pMem = mmap(0, szRegion, + pMem = osMmap(0, szRegion, pShmNode->isReadonly ? PROT_READ : PROT_READ|PROT_WRITE, - MAP_SHARED, pShmNode->h, pShmNode->nRegion*szRegion + MAP_SHARED, pShmNode->h, szRegion*(i64)pShmNode->nRegion ); if( pMem==MAP_FAILED ){ rc = unixLogError(SQLITE_IOERR_SHMMAP, "mmap", pShmNode->zFilename); @@ -4309,6 +4595,236 @@ static int unixShmUnmap( # define unixShmUnmap 0 #endif /* #ifndef SQLITE_OMIT_WAL */ +/* +** If it is currently memory mapped, unmap file pFd. +*/ +static void unixUnmapfile(unixFile *pFd){ + assert( pFd->nFetchOut==0 ); +#if SQLITE_MAX_MMAP_SIZE>0 + if( pFd->pMapRegion ){ + osMunmap(pFd->pMapRegion, pFd->mmapSizeActual); + pFd->pMapRegion = 0; + pFd->mmapSize = 0; + pFd->mmapSizeActual = 0; + } +#endif +} + +#if SQLITE_MAX_MMAP_SIZE>0 +/* +** Return the system page size. +*/ +static int unixGetPagesize(void){ +#if HAVE_MREMAP + return 512; +#elif defined(_BSD_SOURCE) + return getpagesize(); +#else + return (int)sysconf(_SC_PAGESIZE); +#endif +} +#endif /* SQLITE_MAX_MMAP_SIZE>0 */ + +#if SQLITE_MAX_MMAP_SIZE>0 +/* +** Attempt to set the size of the memory mapping maintained by file +** descriptor pFd to nNew bytes. Any existing mapping is discarded. +** +** If successful, this function sets the following variables: +** +** unixFile.pMapRegion +** unixFile.mmapSize +** unixFile.mmapSizeActual +** +** If unsuccessful, an error message is logged via sqlite3_log() and +** the three variables above are zeroed. In this case SQLite should +** continue accessing the database using the xRead() and xWrite() +** methods. +*/ +static void unixRemapfile( + unixFile *pFd, /* File descriptor object */ + i64 nNew /* Required mapping size */ +){ + const char *zErr = "mmap"; + int h = pFd->h; /* File descriptor open on db file */ + u8 *pOrig = (u8 *)pFd->pMapRegion; /* Pointer to current file mapping */ + i64 nOrig = pFd->mmapSizeActual; /* Size of pOrig region in bytes */ + u8 *pNew = 0; /* Location of new mapping */ + int flags = PROT_READ; /* Flags to pass to mmap() */ + + assert( pFd->nFetchOut==0 ); + assert( nNew>pFd->mmapSize ); + assert( nNew<=pFd->mmapSizeMax ); + assert( nNew>0 ); + assert( pFd->mmapSizeActual>=pFd->mmapSize ); + assert( MAP_FAILED!=0 ); + + if( (pFd->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE; + + if( pOrig ){ + const int szSyspage = unixGetPagesize(); + i64 nReuse = (pFd->mmapSize & ~(szSyspage-1)); + u8 *pReq = &pOrig[nReuse]; + + /* Unmap any pages of the existing mapping that cannot be reused. */ + if( nReuse!=nOrig ){ + osMunmap(pReq, nOrig-nReuse); + } + +#if HAVE_MREMAP + pNew = osMremap(pOrig, nReuse, nNew, MREMAP_MAYMOVE); + zErr = "mremap"; +#else + pNew = osMmap(pReq, nNew-nReuse, flags, MAP_SHARED, h, nReuse); + if( pNew!=MAP_FAILED ){ + if( pNew!=pReq ){ + osMunmap(pNew, nNew - nReuse); + pNew = 0; + }else{ + pNew = pOrig; + } + } +#endif + + /* The attempt to extend the existing mapping failed. Free it. */ + if( pNew==MAP_FAILED || pNew==0 ){ + osMunmap(pOrig, nReuse); + } + } + + /* If pNew is still NULL, try to create an entirely new mapping. */ + if( pNew==0 ){ + pNew = osMmap(0, nNew, flags, MAP_SHARED, h, 0); + } + + if( pNew==MAP_FAILED ){ + pNew = 0; + nNew = 0; + unixLogError(SQLITE_OK, zErr, pFd->zPath); + + /* If the mmap() above failed, assume that all subsequent mmap() calls + ** will probably fail too. Fall back to using xRead/xWrite exclusively + ** in this case. */ + pFd->mmapSizeMax = 0; + } + pFd->pMapRegion = (void *)pNew; + pFd->mmapSize = pFd->mmapSizeActual = nNew; +} +#endif + +/* +** Memory map or remap the file opened by file-descriptor pFd (if the file +** is already mapped, the existing mapping is replaced by the new). Or, if +** there already exists a mapping for this file, and there are still +** outstanding xFetch() references to it, this function is a no-op. +** +** If parameter nByte is non-negative, then it is the requested size of +** the mapping to create. Otherwise, if nByte is less than zero, then the +** requested size is the size of the file on disk. The actual size of the +** created mapping is either the requested size or the value configured +** using SQLITE_FCNTL_MMAP_LIMIT, whichever is smaller. +** +** SQLITE_OK is returned if no error occurs (even if the mapping is not +** recreated as a result of outstanding references) or an SQLite error +** code otherwise. +*/ +static int unixMapfile(unixFile *pFd, i64 nByte){ +#if SQLITE_MAX_MMAP_SIZE>0 + i64 nMap = nByte; + int rc; + + assert( nMap>=0 || pFd->nFetchOut==0 ); + if( pFd->nFetchOut>0 ) return SQLITE_OK; + + if( nMap<0 ){ + struct stat statbuf; /* Low-level file information */ + rc = osFstat(pFd->h, &statbuf); + if( rc!=SQLITE_OK ){ + return SQLITE_IOERR_FSTAT; + } + nMap = statbuf.st_size; + } + if( nMap>pFd->mmapSizeMax ){ + nMap = pFd->mmapSizeMax; + } + + if( nMap!=pFd->mmapSize ){ + if( nMap>0 ){ + unixRemapfile(pFd, nMap); + }else{ + unixUnmapfile(pFd); + } + } +#endif + + return SQLITE_OK; +} + +/* +** If possible, return a pointer to a mapping of file fd starting at offset +** iOff. The mapping must be valid for at least nAmt bytes. +** +** If such a pointer can be obtained, store it in *pp and return SQLITE_OK. +** Or, if one cannot but no error occurs, set *pp to 0 and return SQLITE_OK. +** Finally, if an error does occur, return an SQLite error code. The final +** value of *pp is undefined in this case. +** +** If this function does return a pointer, the caller must eventually +** release the reference by calling unixUnfetch(). +*/ +static int unixFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ +#if SQLITE_MAX_MMAP_SIZE>0 + unixFile *pFd = (unixFile *)fd; /* The underlying database file */ +#endif + *pp = 0; + +#if SQLITE_MAX_MMAP_SIZE>0 + if( pFd->mmapSizeMax>0 ){ + if( pFd->pMapRegion==0 ){ + int rc = unixMapfile(pFd, -1); + if( rc!=SQLITE_OK ) return rc; + } + if( pFd->mmapSize >= iOff+nAmt ){ + *pp = &((u8 *)pFd->pMapRegion)[iOff]; + pFd->nFetchOut++; + } + } +#endif + return SQLITE_OK; +} + +/* +** If the third argument is non-NULL, then this function releases a +** reference obtained by an earlier call to unixFetch(). The second +** argument passed to this function must be the same as the corresponding +** argument that was passed to the unixFetch() invocation. +** +** Or, if the third argument is NULL, then this function is being called +** to inform the VFS layer that, according to POSIX, any existing mapping +** may now be invalid and should be unmapped. +*/ +static int unixUnfetch(sqlite3_file *fd, i64 iOff, void *p){ + unixFile *pFd = (unixFile *)fd; /* The underlying database file */ + UNUSED_PARAMETER(iOff); + + /* If p==0 (unmap the entire file) then there must be no outstanding + ** xFetch references. Or, if p!=0 (meaning it is an xFetch reference), + ** then there must be at least one outstanding. */ + assert( (p==0)==(pFd->nFetchOut==0) ); + + /* If p!=0, it must match the iOff value. */ + assert( p==0 || p==&((u8 *)pFd->pMapRegion)[iOff] ); + + if( p ){ + pFd->nFetchOut--; + }else{ + unixUnmapfile(pFd); + } + + assert( pFd->nFetchOut>=0 ); + return SQLITE_OK; +} + /* ** Here ends the implementation of all sqlite3_file methods. ** @@ -4367,7 +4883,9 @@ static const sqlite3_io_methods METHOD = { \ unixShmMap, /* xShmMap */ \ unixShmLock, /* xShmLock */ \ unixShmBarrier, /* xShmBarrier */ \ - unixShmUnmap /* xShmUnmap */ \ + unixShmUnmap, /* xShmUnmap */ \ + unixFetch, /* xFetch */ \ + unixUnfetch, /* xUnfetch */ \ }; \ static const sqlite3_io_methods *FINDER##Impl(const char *z, unixFile *p){ \ UNUSED_PARAMETER(z); UNUSED_PARAMETER(p); \ @@ -4384,7 +4902,7 @@ static const sqlite3_io_methods *(*const FINDER)(const char*,unixFile *p) \ IOMETHODS( posixIoFinder, /* Finder function name */ posixIoMethods, /* sqlite3_io_methods object name */ - 2, /* shared memory is enabled */ + 3, /* shared memory and mmap are enabled */ unixClose, /* xClose method */ unixLock, /* xLock method */ unixUnlock, /* xUnlock method */ @@ -4635,11 +5153,12 @@ static int fillInUnixFile( pNew->pVfs = pVfs; pNew->zPath = zFilename; pNew->ctrlFlags = (u8)ctrlFlags; + pNew->mmapSizeMax = sqlite3GlobalConfig.szMmap; if( sqlite3_uri_boolean(((ctrlFlags & UNIXFILE_URI) ? zFilename : 0), "psow", SQLITE_POWERSAFE_OVERWRITE) ){ pNew->ctrlFlags |= UNIXFILE_PSOW; } - if( memcmp(pVfs->zName,"unix-excl",10)==0 ){ + if( strcmp(pVfs->zName,"unix-excl")==0 ){ pNew->ctrlFlags |= UNIXFILE_EXCL; } @@ -4671,7 +5190,7 @@ static int fillInUnixFile( unixEnterMutex(); rc = findInodeInfo(pNew, &pNew->pInode); if( rc!=SQLITE_OK ){ - /* If an error occured in findInodeInfo(), close the file descriptor + /* If an error occurred in findInodeInfo(), close the file descriptor ** immediately, before releasing the mutex. findInodeInfo() may fail ** in two scenarios: ** @@ -4770,15 +5289,15 @@ static int fillInUnixFile( if( h>=0 ) robust_close(pNew, h, __LINE__); h = -1; osUnlink(zFilename); - isDelete = 0; + pNew->ctrlFlags |= UNIXFILE_DELETE; } - if( isDelete ) pNew->ctrlFlags |= UNIXFILE_DELETE; #endif if( rc!=SQLITE_OK ){ if( h>=0 ) robust_close(pNew, h, __LINE__); }else{ pNew->pMethod = pLockingStyle; OpenCounter(+1); + verifyDbFile(pNew); } return rc; } @@ -5278,8 +5797,13 @@ static int unixDelete( int rc = SQLITE_OK; UNUSED_PARAMETER(NotUsed); SimulateIOError(return SQLITE_IOERR_DELETE); - if( osUnlink(zPath)==(-1) && errno!=ENOENT ){ - return unixLogError(SQLITE_IOERR_DELETE, "unlink", zPath); + if( osUnlink(zPath)==(-1) ){ + if( errno==ENOENT ){ + rc = SQLITE_IOERR_DELETE_NOENT; + }else{ + rc = unixLogError(SQLITE_IOERR_DELETE, "unlink", zPath); + } + return rc; } #ifndef SQLITE_DISABLE_DIRSYNC if( (dirSync & 1)!=0 ){ @@ -5304,7 +5828,7 @@ static int unixDelete( } /* -** Test the existance of or access permissions of file zPath. The +** Test the existence of or access permissions of file zPath. The ** test performed depends on the value of flags: ** ** SQLITE_ACCESS_EXISTS: Return 1 if the file exists @@ -6867,7 +7391,7 @@ int sqlite3_os_init(void){ /* Double-check that the aSyscall[] array has been constructed ** correctly. See ticket [bb3a86e890c8e96ab] */ - assert( ArraySize(aSyscall)==22 ); + assert( ArraySize(aSyscall)==24 ); /* Register all VFSes defined in the aVfs[] array */ for(i=0; i<(sizeof(aVfs)/sizeof(sqlite3_vfs)); i++){ diff --git a/src/os_win.c b/src/os_win.c index cbf17b1..aeb0881 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -24,6 +24,66 @@ */ #include "os_common.h" +/* +** Compiling and using WAL mode requires several APIs that are only +** available in Windows platforms based on the NT kernel. +*/ +#if !SQLITE_OS_WINNT && !defined(SQLITE_OMIT_WAL) +# error "WAL mode requires support from the Windows NT kernel, compile\ + with SQLITE_OMIT_WAL." +#endif + +/* +** Are most of the Win32 ANSI APIs available (i.e. with certain exceptions +** based on the sub-platform)? +*/ +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT +# define SQLITE_WIN32_HAS_ANSI +#endif + +/* +** Are most of the Win32 Unicode APIs available (i.e. with certain exceptions +** based on the sub-platform)? +*/ +#if SQLITE_OS_WINCE || SQLITE_OS_WINNT || SQLITE_OS_WINRT +# define SQLITE_WIN32_HAS_WIDE +#endif + +/* +** Do we need to manually define the Win32 file mapping APIs for use with WAL +** mode (e.g. these APIs are available in the Windows CE SDK; however, they +** are not present in the header file)? +*/ +#if SQLITE_WIN32_FILEMAPPING_API && !defined(SQLITE_OMIT_WAL) +/* +** Two of the file mapping APIs are different under WinRT. Figure out which +** set we need. +*/ +#if SQLITE_OS_WINRT +WINBASEAPI HANDLE WINAPI CreateFileMappingFromApp(HANDLE, \ + LPSECURITY_ATTRIBUTES, ULONG, ULONG64, LPCWSTR); + +WINBASEAPI LPVOID WINAPI MapViewOfFileFromApp(HANDLE, ULONG, ULONG64, SIZE_T); +#else +#if defined(SQLITE_WIN32_HAS_ANSI) +WINBASEAPI HANDLE WINAPI CreateFileMappingA(HANDLE, LPSECURITY_ATTRIBUTES, \ + DWORD, DWORD, DWORD, LPCSTR); +#endif /* defined(SQLITE_WIN32_HAS_ANSI) */ + +#if defined(SQLITE_WIN32_HAS_WIDE) +WINBASEAPI HANDLE WINAPI CreateFileMappingW(HANDLE, LPSECURITY_ATTRIBUTES, \ + DWORD, DWORD, DWORD, LPCWSTR); +#endif /* defined(SQLITE_WIN32_HAS_WIDE) */ + +WINBASEAPI LPVOID WINAPI MapViewOfFile(HANDLE, DWORD, DWORD, DWORD, SIZE_T); +#endif /* SQLITE_OS_WINRT */ + +/* +** This file mapping API is common to both Win32 and WinRT. +*/ +WINBASEAPI BOOL WINAPI UnmapViewOfFile(LPCVOID); +#endif /* SQLITE_WIN32_FILEMAPPING_API && !defined(SQLITE_OMIT_WAL) */ + /* ** Macro to find the minimum of two numeric values. */ @@ -90,11 +150,20 @@ struct winFile { winceLock local; /* Locks obtained by this instance of winFile */ winceLock *shared; /* Global shared lock memory for the file */ #endif +#if SQLITE_MAX_MMAP_SIZE>0 + int nFetchOut; /* Number of outstanding xFetch references */ + HANDLE hMap; /* Handle for accessing memory mapping */ + void *pMapRegion; /* Area memory mapped */ + sqlite3_int64 mmapSize; /* Usable size of mapped region */ + sqlite3_int64 mmapSizeActual; /* Actual size of mapped region */ + sqlite3_int64 mmapSizeMax; /* Configured FCNTL_MMAP_SIZE value */ +#endif }; /* ** Allowed values for winFile.ctrlFlags */ +#define WINFILE_RDONLY 0x02 /* Connection is read only */ #define WINFILE_PERSIST_WAL 0x04 /* Persistent WAL mode */ #define WINFILE_PSOW 0x10 /* SQLITE_IOCAP_POWERSAFE_OVERWRITE */ @@ -229,14 +298,6 @@ int sqlite3_os_type = 0; static int sqlite3_os_type = 0; #endif -#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT -# define SQLITE_WIN32_HAS_ANSI -#endif - -#if SQLITE_OS_WINCE || SQLITE_OS_WINNT || SQLITE_OS_WINRT -# define SQLITE_WIN32_HAS_WIDE -#endif - #ifndef SYSCALL # define SYSCALL sqlite3_syscall_ptr #endif @@ -256,7 +317,7 @@ static int sqlite3_os_type = 0; ** to all overrideable system calls. */ static struct win_syscall { - const char *zName; /* Name of the sytem call */ + const char *zName; /* Name of the system call */ sqlite3_syscall_ptr pCurrent; /* Current value of the system call */ sqlite3_syscall_ptr pDefault; /* Default value */ } aSyscall[] = { @@ -308,6 +369,16 @@ static struct win_syscall { #define osCreateFileW ((HANDLE(WINAPI*)(LPCWSTR,DWORD,DWORD, \ LPSECURITY_ATTRIBUTES,DWORD,DWORD,HANDLE))aSyscall[5].pCurrent) +#if (!SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_ANSI) && \ + !defined(SQLITE_OMIT_WAL)) + { "CreateFileMappingA", (SYSCALL)CreateFileMappingA, 0 }, +#else + { "CreateFileMappingA", (SYSCALL)0, 0 }, +#endif + +#define osCreateFileMappingA ((HANDLE(WINAPI*)(HANDLE,LPSECURITY_ATTRIBUTES, \ + DWORD,DWORD,DWORD,LPCSTR))aSyscall[6].pCurrent) + #if SQLITE_OS_WINCE || (!SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) && \ !defined(SQLITE_OMIT_WAL)) { "CreateFileMappingW", (SYSCALL)CreateFileMappingW, 0 }, @@ -316,7 +387,7 @@ static struct win_syscall { #endif #define osCreateFileMappingW ((HANDLE(WINAPI*)(HANDLE,LPSECURITY_ATTRIBUTES, \ - DWORD,DWORD,DWORD,LPCWSTR))aSyscall[6].pCurrent) + DWORD,DWORD,DWORD,LPCWSTR))aSyscall[7].pCurrent) #if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) { "CreateMutexW", (SYSCALL)CreateMutexW, 0 }, @@ -325,7 +396,7 @@ static struct win_syscall { #endif #define osCreateMutexW ((HANDLE(WINAPI*)(LPSECURITY_ATTRIBUTES,BOOL, \ - LPCWSTR))aSyscall[7].pCurrent) + LPCWSTR))aSyscall[8].pCurrent) #if defined(SQLITE_WIN32_HAS_ANSI) { "DeleteFileA", (SYSCALL)DeleteFileA, 0 }, @@ -333,7 +404,7 @@ static struct win_syscall { { "DeleteFileA", (SYSCALL)0, 0 }, #endif -#define osDeleteFileA ((BOOL(WINAPI*)(LPCSTR))aSyscall[8].pCurrent) +#define osDeleteFileA ((BOOL(WINAPI*)(LPCSTR))aSyscall[9].pCurrent) #if defined(SQLITE_WIN32_HAS_WIDE) { "DeleteFileW", (SYSCALL)DeleteFileW, 0 }, @@ -341,7 +412,7 @@ static struct win_syscall { { "DeleteFileW", (SYSCALL)0, 0 }, #endif -#define osDeleteFileW ((BOOL(WINAPI*)(LPCWSTR))aSyscall[9].pCurrent) +#define osDeleteFileW ((BOOL(WINAPI*)(LPCWSTR))aSyscall[10].pCurrent) #if SQLITE_OS_WINCE { "FileTimeToLocalFileTime", (SYSCALL)FileTimeToLocalFileTime, 0 }, @@ -350,7 +421,7 @@ static struct win_syscall { #endif #define osFileTimeToLocalFileTime ((BOOL(WINAPI*)(CONST FILETIME*, \ - LPFILETIME))aSyscall[10].pCurrent) + LPFILETIME))aSyscall[11].pCurrent) #if SQLITE_OS_WINCE { "FileTimeToSystemTime", (SYSCALL)FileTimeToSystemTime, 0 }, @@ -359,11 +430,11 @@ static struct win_syscall { #endif #define osFileTimeToSystemTime ((BOOL(WINAPI*)(CONST FILETIME*, \ - LPSYSTEMTIME))aSyscall[11].pCurrent) + LPSYSTEMTIME))aSyscall[12].pCurrent) { "FlushFileBuffers", (SYSCALL)FlushFileBuffers, 0 }, -#define osFlushFileBuffers ((BOOL(WINAPI*)(HANDLE))aSyscall[12].pCurrent) +#define osFlushFileBuffers ((BOOL(WINAPI*)(HANDLE))aSyscall[13].pCurrent) #if defined(SQLITE_WIN32_HAS_ANSI) { "FormatMessageA", (SYSCALL)FormatMessageA, 0 }, @@ -372,7 +443,7 @@ static struct win_syscall { #endif #define osFormatMessageA ((DWORD(WINAPI*)(DWORD,LPCVOID,DWORD,DWORD,LPSTR, \ - DWORD,va_list*))aSyscall[13].pCurrent) + DWORD,va_list*))aSyscall[14].pCurrent) #if defined(SQLITE_WIN32_HAS_WIDE) { "FormatMessageW", (SYSCALL)FormatMessageW, 0 }, @@ -381,15 +452,19 @@ static struct win_syscall { #endif #define osFormatMessageW ((DWORD(WINAPI*)(DWORD,LPCVOID,DWORD,DWORD,LPWSTR, \ - DWORD,va_list*))aSyscall[14].pCurrent) + DWORD,va_list*))aSyscall[15].pCurrent) +#if !defined(SQLITE_OMIT_LOAD_EXTENSION) { "FreeLibrary", (SYSCALL)FreeLibrary, 0 }, +#else + { "FreeLibrary", (SYSCALL)0, 0 }, +#endif -#define osFreeLibrary ((BOOL(WINAPI*)(HMODULE))aSyscall[15].pCurrent) +#define osFreeLibrary ((BOOL(WINAPI*)(HMODULE))aSyscall[16].pCurrent) { "GetCurrentProcessId", (SYSCALL)GetCurrentProcessId, 0 }, -#define osGetCurrentProcessId ((DWORD(WINAPI*)(VOID))aSyscall[16].pCurrent) +#define osGetCurrentProcessId ((DWORD(WINAPI*)(VOID))aSyscall[17].pCurrent) #if !SQLITE_OS_WINCE && defined(SQLITE_WIN32_HAS_ANSI) { "GetDiskFreeSpaceA", (SYSCALL)GetDiskFreeSpaceA, 0 }, @@ -398,7 +473,7 @@ static struct win_syscall { #endif #define osGetDiskFreeSpaceA ((BOOL(WINAPI*)(LPCSTR,LPDWORD,LPDWORD,LPDWORD, \ - LPDWORD))aSyscall[17].pCurrent) + LPDWORD))aSyscall[18].pCurrent) #if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) { "GetDiskFreeSpaceW", (SYSCALL)GetDiskFreeSpaceW, 0 }, @@ -407,7 +482,7 @@ static struct win_syscall { #endif #define osGetDiskFreeSpaceW ((BOOL(WINAPI*)(LPCWSTR,LPDWORD,LPDWORD,LPDWORD, \ - LPDWORD))aSyscall[18].pCurrent) + LPDWORD))aSyscall[19].pCurrent) #if defined(SQLITE_WIN32_HAS_ANSI) { "GetFileAttributesA", (SYSCALL)GetFileAttributesA, 0 }, @@ -415,7 +490,7 @@ static struct win_syscall { { "GetFileAttributesA", (SYSCALL)0, 0 }, #endif -#define osGetFileAttributesA ((DWORD(WINAPI*)(LPCSTR))aSyscall[19].pCurrent) +#define osGetFileAttributesA ((DWORD(WINAPI*)(LPCSTR))aSyscall[20].pCurrent) #if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) { "GetFileAttributesW", (SYSCALL)GetFileAttributesW, 0 }, @@ -423,7 +498,7 @@ static struct win_syscall { { "GetFileAttributesW", (SYSCALL)0, 0 }, #endif -#define osGetFileAttributesW ((DWORD(WINAPI*)(LPCWSTR))aSyscall[20].pCurrent) +#define osGetFileAttributesW ((DWORD(WINAPI*)(LPCWSTR))aSyscall[21].pCurrent) #if defined(SQLITE_WIN32_HAS_WIDE) { "GetFileAttributesExW", (SYSCALL)GetFileAttributesExW, 0 }, @@ -432,7 +507,7 @@ static struct win_syscall { #endif #define osGetFileAttributesExW ((BOOL(WINAPI*)(LPCWSTR,GET_FILEEX_INFO_LEVELS, \ - LPVOID))aSyscall[21].pCurrent) + LPVOID))aSyscall[22].pCurrent) #if !SQLITE_OS_WINRT { "GetFileSize", (SYSCALL)GetFileSize, 0 }, @@ -440,7 +515,7 @@ static struct win_syscall { { "GetFileSize", (SYSCALL)0, 0 }, #endif -#define osGetFileSize ((DWORD(WINAPI*)(HANDLE,LPDWORD))aSyscall[22].pCurrent) +#define osGetFileSize ((DWORD(WINAPI*)(HANDLE,LPDWORD))aSyscall[23].pCurrent) #if !SQLITE_OS_WINCE && defined(SQLITE_WIN32_HAS_ANSI) { "GetFullPathNameA", (SYSCALL)GetFullPathNameA, 0 }, @@ -449,7 +524,7 @@ static struct win_syscall { #endif #define osGetFullPathNameA ((DWORD(WINAPI*)(LPCSTR,DWORD,LPSTR, \ - LPSTR*))aSyscall[23].pCurrent) + LPSTR*))aSyscall[24].pCurrent) #if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) { "GetFullPathNameW", (SYSCALL)GetFullPathNameW, 0 }, @@ -458,12 +533,13 @@ static struct win_syscall { #endif #define osGetFullPathNameW ((DWORD(WINAPI*)(LPCWSTR,DWORD,LPWSTR, \ - LPWSTR*))aSyscall[24].pCurrent) + LPWSTR*))aSyscall[25].pCurrent) { "GetLastError", (SYSCALL)GetLastError, 0 }, -#define osGetLastError ((DWORD(WINAPI*)(VOID))aSyscall[25].pCurrent) +#define osGetLastError ((DWORD(WINAPI*)(VOID))aSyscall[26].pCurrent) +#if !defined(SQLITE_OMIT_LOAD_EXTENSION) #if SQLITE_OS_WINCE /* The GetProcAddressA() routine is only available on Windows CE. */ { "GetProcAddressA", (SYSCALL)GetProcAddressA, 0 }, @@ -472,9 +548,12 @@ static struct win_syscall { ** an ANSI string regardless of the _UNICODE setting */ { "GetProcAddressA", (SYSCALL)GetProcAddress, 0 }, #endif +#else + { "GetProcAddressA", (SYSCALL)0, 0 }, +#endif #define osGetProcAddressA ((FARPROC(WINAPI*)(HMODULE, \ - LPCSTR))aSyscall[26].pCurrent) + LPCSTR))aSyscall[27].pCurrent) #if !SQLITE_OS_WINRT { "GetSystemInfo", (SYSCALL)GetSystemInfo, 0 }, @@ -482,11 +561,11 @@ static struct win_syscall { { "GetSystemInfo", (SYSCALL)0, 0 }, #endif -#define osGetSystemInfo ((VOID(WINAPI*)(LPSYSTEM_INFO))aSyscall[27].pCurrent) +#define osGetSystemInfo ((VOID(WINAPI*)(LPSYSTEM_INFO))aSyscall[28].pCurrent) { "GetSystemTime", (SYSCALL)GetSystemTime, 0 }, -#define osGetSystemTime ((VOID(WINAPI*)(LPSYSTEMTIME))aSyscall[28].pCurrent) +#define osGetSystemTime ((VOID(WINAPI*)(LPSYSTEMTIME))aSyscall[29].pCurrent) #if !SQLITE_OS_WINCE { "GetSystemTimeAsFileTime", (SYSCALL)GetSystemTimeAsFileTime, 0 }, @@ -495,7 +574,7 @@ static struct win_syscall { #endif #define osGetSystemTimeAsFileTime ((VOID(WINAPI*)( \ - LPFILETIME))aSyscall[29].pCurrent) + LPFILETIME))aSyscall[30].pCurrent) #if defined(SQLITE_WIN32_HAS_ANSI) { "GetTempPathA", (SYSCALL)GetTempPathA, 0 }, @@ -503,7 +582,7 @@ static struct win_syscall { { "GetTempPathA", (SYSCALL)0, 0 }, #endif -#define osGetTempPathA ((DWORD(WINAPI*)(DWORD,LPSTR))aSyscall[30].pCurrent) +#define osGetTempPathA ((DWORD(WINAPI*)(DWORD,LPSTR))aSyscall[31].pCurrent) #if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) { "GetTempPathW", (SYSCALL)GetTempPathW, 0 }, @@ -511,7 +590,7 @@ static struct win_syscall { { "GetTempPathW", (SYSCALL)0, 0 }, #endif -#define osGetTempPathW ((DWORD(WINAPI*)(DWORD,LPWSTR))aSyscall[31].pCurrent) +#define osGetTempPathW ((DWORD(WINAPI*)(DWORD,LPWSTR))aSyscall[32].pCurrent) #if !SQLITE_OS_WINRT { "GetTickCount", (SYSCALL)GetTickCount, 0 }, @@ -519,7 +598,7 @@ static struct win_syscall { { "GetTickCount", (SYSCALL)0, 0 }, #endif -#define osGetTickCount ((DWORD(WINAPI*)(VOID))aSyscall[32].pCurrent) +#define osGetTickCount ((DWORD(WINAPI*)(VOID))aSyscall[33].pCurrent) #if defined(SQLITE_WIN32_HAS_ANSI) { "GetVersionExA", (SYSCALL)GetVersionExA, 0 }, @@ -528,12 +607,12 @@ static struct win_syscall { #endif #define osGetVersionExA ((BOOL(WINAPI*)( \ - LPOSVERSIONINFOA))aSyscall[33].pCurrent) + LPOSVERSIONINFOA))aSyscall[34].pCurrent) { "HeapAlloc", (SYSCALL)HeapAlloc, 0 }, #define osHeapAlloc ((LPVOID(WINAPI*)(HANDLE,DWORD, \ - SIZE_T))aSyscall[34].pCurrent) + SIZE_T))aSyscall[35].pCurrent) #if !SQLITE_OS_WINRT { "HeapCreate", (SYSCALL)HeapCreate, 0 }, @@ -542,7 +621,7 @@ static struct win_syscall { #endif #define osHeapCreate ((HANDLE(WINAPI*)(DWORD,SIZE_T, \ - SIZE_T))aSyscall[35].pCurrent) + SIZE_T))aSyscall[36].pCurrent) #if !SQLITE_OS_WINRT { "HeapDestroy", (SYSCALL)HeapDestroy, 0 }, @@ -550,21 +629,21 @@ static struct win_syscall { { "HeapDestroy", (SYSCALL)0, 0 }, #endif -#define osHeapDestroy ((BOOL(WINAPI*)(HANDLE))aSyscall[36].pCurrent) +#define osHeapDestroy ((BOOL(WINAPI*)(HANDLE))aSyscall[37].pCurrent) { "HeapFree", (SYSCALL)HeapFree, 0 }, -#define osHeapFree ((BOOL(WINAPI*)(HANDLE,DWORD,LPVOID))aSyscall[37].pCurrent) +#define osHeapFree ((BOOL(WINAPI*)(HANDLE,DWORD,LPVOID))aSyscall[38].pCurrent) { "HeapReAlloc", (SYSCALL)HeapReAlloc, 0 }, #define osHeapReAlloc ((LPVOID(WINAPI*)(HANDLE,DWORD,LPVOID, \ - SIZE_T))aSyscall[38].pCurrent) + SIZE_T))aSyscall[39].pCurrent) { "HeapSize", (SYSCALL)HeapSize, 0 }, #define osHeapSize ((SIZE_T(WINAPI*)(HANDLE,DWORD, \ - LPCVOID))aSyscall[39].pCurrent) + LPCVOID))aSyscall[40].pCurrent) #if !SQLITE_OS_WINRT { "HeapValidate", (SYSCALL)HeapValidate, 0 }, @@ -573,23 +652,24 @@ static struct win_syscall { #endif #define osHeapValidate ((BOOL(WINAPI*)(HANDLE,DWORD, \ - LPCVOID))aSyscall[40].pCurrent) + LPCVOID))aSyscall[41].pCurrent) -#if defined(SQLITE_WIN32_HAS_ANSI) +#if defined(SQLITE_WIN32_HAS_ANSI) && !defined(SQLITE_OMIT_LOAD_EXTENSION) { "LoadLibraryA", (SYSCALL)LoadLibraryA, 0 }, #else { "LoadLibraryA", (SYSCALL)0, 0 }, #endif -#define osLoadLibraryA ((HMODULE(WINAPI*)(LPCSTR))aSyscall[41].pCurrent) +#define osLoadLibraryA ((HMODULE(WINAPI*)(LPCSTR))aSyscall[42].pCurrent) -#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) +#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) && \ + !defined(SQLITE_OMIT_LOAD_EXTENSION) { "LoadLibraryW", (SYSCALL)LoadLibraryW, 0 }, #else { "LoadLibraryW", (SYSCALL)0, 0 }, #endif -#define osLoadLibraryW ((HMODULE(WINAPI*)(LPCWSTR))aSyscall[42].pCurrent) +#define osLoadLibraryW ((HMODULE(WINAPI*)(LPCWSTR))aSyscall[43].pCurrent) #if !SQLITE_OS_WINRT { "LocalFree", (SYSCALL)LocalFree, 0 }, @@ -597,7 +677,7 @@ static struct win_syscall { { "LocalFree", (SYSCALL)0, 0 }, #endif -#define osLocalFree ((HLOCAL(WINAPI*)(HLOCAL))aSyscall[43].pCurrent) +#define osLocalFree ((HLOCAL(WINAPI*)(HLOCAL))aSyscall[44].pCurrent) #if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT { "LockFile", (SYSCALL)LockFile, 0 }, @@ -607,7 +687,7 @@ static struct win_syscall { #ifndef osLockFile #define osLockFile ((BOOL(WINAPI*)(HANDLE,DWORD,DWORD,DWORD, \ - DWORD))aSyscall[44].pCurrent) + DWORD))aSyscall[45].pCurrent) #endif #if !SQLITE_OS_WINCE @@ -618,7 +698,7 @@ static struct win_syscall { #ifndef osLockFileEx #define osLockFileEx ((BOOL(WINAPI*)(HANDLE,DWORD,DWORD,DWORD,DWORD, \ - LPOVERLAPPED))aSyscall[45].pCurrent) + LPOVERLAPPED))aSyscall[46].pCurrent) #endif #if SQLITE_OS_WINCE || (!SQLITE_OS_WINRT && !defined(SQLITE_OMIT_WAL)) @@ -628,26 +708,26 @@ static struct win_syscall { #endif #define osMapViewOfFile ((LPVOID(WINAPI*)(HANDLE,DWORD,DWORD,DWORD, \ - SIZE_T))aSyscall[46].pCurrent) + SIZE_T))aSyscall[47].pCurrent) { "MultiByteToWideChar", (SYSCALL)MultiByteToWideChar, 0 }, #define osMultiByteToWideChar ((int(WINAPI*)(UINT,DWORD,LPCSTR,int,LPWSTR, \ - int))aSyscall[47].pCurrent) + int))aSyscall[48].pCurrent) { "QueryPerformanceCounter", (SYSCALL)QueryPerformanceCounter, 0 }, #define osQueryPerformanceCounter ((BOOL(WINAPI*)( \ - LARGE_INTEGER*))aSyscall[48].pCurrent) + LARGE_INTEGER*))aSyscall[49].pCurrent) { "ReadFile", (SYSCALL)ReadFile, 0 }, #define osReadFile ((BOOL(WINAPI*)(HANDLE,LPVOID,DWORD,LPDWORD, \ - LPOVERLAPPED))aSyscall[49].pCurrent) + LPOVERLAPPED))aSyscall[50].pCurrent) { "SetEndOfFile", (SYSCALL)SetEndOfFile, 0 }, -#define osSetEndOfFile ((BOOL(WINAPI*)(HANDLE))aSyscall[50].pCurrent) +#define osSetEndOfFile ((BOOL(WINAPI*)(HANDLE))aSyscall[51].pCurrent) #if !SQLITE_OS_WINRT { "SetFilePointer", (SYSCALL)SetFilePointer, 0 }, @@ -656,7 +736,7 @@ static struct win_syscall { #endif #define osSetFilePointer ((DWORD(WINAPI*)(HANDLE,LONG,PLONG, \ - DWORD))aSyscall[51].pCurrent) + DWORD))aSyscall[52].pCurrent) #if !SQLITE_OS_WINRT { "Sleep", (SYSCALL)Sleep, 0 }, @@ -664,12 +744,12 @@ static struct win_syscall { { "Sleep", (SYSCALL)0, 0 }, #endif -#define osSleep ((VOID(WINAPI*)(DWORD))aSyscall[52].pCurrent) +#define osSleep ((VOID(WINAPI*)(DWORD))aSyscall[53].pCurrent) { "SystemTimeToFileTime", (SYSCALL)SystemTimeToFileTime, 0 }, #define osSystemTimeToFileTime ((BOOL(WINAPI*)(CONST SYSTEMTIME*, \ - LPFILETIME))aSyscall[53].pCurrent) + LPFILETIME))aSyscall[54].pCurrent) #if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT { "UnlockFile", (SYSCALL)UnlockFile, 0 }, @@ -679,7 +759,7 @@ static struct win_syscall { #ifndef osUnlockFile #define osUnlockFile ((BOOL(WINAPI*)(HANDLE,DWORD,DWORD,DWORD, \ - DWORD))aSyscall[54].pCurrent) + DWORD))aSyscall[55].pCurrent) #endif #if !SQLITE_OS_WINCE @@ -689,7 +769,7 @@ static struct win_syscall { #endif #define osUnlockFileEx ((BOOL(WINAPI*)(HANDLE,DWORD,DWORD,DWORD, \ - LPOVERLAPPED))aSyscall[55].pCurrent) + LPOVERLAPPED))aSyscall[56].pCurrent) #if SQLITE_OS_WINCE || !defined(SQLITE_OMIT_WAL) { "UnmapViewOfFile", (SYSCALL)UnmapViewOfFile, 0 }, @@ -697,17 +777,17 @@ static struct win_syscall { { "UnmapViewOfFile", (SYSCALL)0, 0 }, #endif -#define osUnmapViewOfFile ((BOOL(WINAPI*)(LPCVOID))aSyscall[56].pCurrent) +#define osUnmapViewOfFile ((BOOL(WINAPI*)(LPCVOID))aSyscall[57].pCurrent) { "WideCharToMultiByte", (SYSCALL)WideCharToMultiByte, 0 }, #define osWideCharToMultiByte ((int(WINAPI*)(UINT,DWORD,LPCWSTR,int,LPSTR,int, \ - LPCSTR,LPBOOL))aSyscall[57].pCurrent) + LPCSTR,LPBOOL))aSyscall[58].pCurrent) { "WriteFile", (SYSCALL)WriteFile, 0 }, #define osWriteFile ((BOOL(WINAPI*)(HANDLE,LPCVOID,DWORD,LPDWORD, \ - LPOVERLAPPED))aSyscall[58].pCurrent) + LPOVERLAPPED))aSyscall[59].pCurrent) #if SQLITE_OS_WINRT { "CreateEventExW", (SYSCALL)CreateEventExW, 0 }, @@ -716,7 +796,7 @@ static struct win_syscall { #endif #define osCreateEventExW ((HANDLE(WINAPI*)(LPSECURITY_ATTRIBUTES,LPCWSTR, \ - DWORD,DWORD))aSyscall[59].pCurrent) + DWORD,DWORD))aSyscall[60].pCurrent) #if !SQLITE_OS_WINRT { "WaitForSingleObject", (SYSCALL)WaitForSingleObject, 0 }, @@ -725,7 +805,7 @@ static struct win_syscall { #endif #define osWaitForSingleObject ((DWORD(WINAPI*)(HANDLE, \ - DWORD))aSyscall[60].pCurrent) + DWORD))aSyscall[61].pCurrent) #if SQLITE_OS_WINRT { "WaitForSingleObjectEx", (SYSCALL)WaitForSingleObjectEx, 0 }, @@ -734,7 +814,7 @@ static struct win_syscall { #endif #define osWaitForSingleObjectEx ((DWORD(WINAPI*)(HANDLE,DWORD, \ - BOOL))aSyscall[61].pCurrent) + BOOL))aSyscall[62].pCurrent) #if SQLITE_OS_WINRT { "SetFilePointerEx", (SYSCALL)SetFilePointerEx, 0 }, @@ -743,7 +823,7 @@ static struct win_syscall { #endif #define osSetFilePointerEx ((BOOL(WINAPI*)(HANDLE,LARGE_INTEGER, \ - PLARGE_INTEGER,DWORD))aSyscall[62].pCurrent) + PLARGE_INTEGER,DWORD))aSyscall[63].pCurrent) #if SQLITE_OS_WINRT { "GetFileInformationByHandleEx", (SYSCALL)GetFileInformationByHandleEx, 0 }, @@ -752,7 +832,7 @@ static struct win_syscall { #endif #define osGetFileInformationByHandleEx ((BOOL(WINAPI*)(HANDLE, \ - FILE_INFO_BY_HANDLE_CLASS,LPVOID,DWORD))aSyscall[63].pCurrent) + FILE_INFO_BY_HANDLE_CLASS,LPVOID,DWORD))aSyscall[64].pCurrent) #if SQLITE_OS_WINRT && !defined(SQLITE_OMIT_WAL) { "MapViewOfFileFromApp", (SYSCALL)MapViewOfFileFromApp, 0 }, @@ -761,7 +841,7 @@ static struct win_syscall { #endif #define osMapViewOfFileFromApp ((LPVOID(WINAPI*)(HANDLE,ULONG,ULONG64, \ - SIZE_T))aSyscall[64].pCurrent) + SIZE_T))aSyscall[65].pCurrent) #if SQLITE_OS_WINRT { "CreateFile2", (SYSCALL)CreateFile2, 0 }, @@ -770,16 +850,16 @@ static struct win_syscall { #endif #define osCreateFile2 ((HANDLE(WINAPI*)(LPCWSTR,DWORD,DWORD,DWORD, \ - LPCREATEFILE2_EXTENDED_PARAMETERS))aSyscall[65].pCurrent) + LPCREATEFILE2_EXTENDED_PARAMETERS))aSyscall[66].pCurrent) -#if SQLITE_OS_WINRT +#if SQLITE_OS_WINRT && !defined(SQLITE_OMIT_LOAD_EXTENSION) { "LoadPackagedLibrary", (SYSCALL)LoadPackagedLibrary, 0 }, #else { "LoadPackagedLibrary", (SYSCALL)0, 0 }, #endif #define osLoadPackagedLibrary ((HMODULE(WINAPI*)(LPCWSTR, \ - DWORD))aSyscall[66].pCurrent) + DWORD))aSyscall[67].pCurrent) #if SQLITE_OS_WINRT { "GetTickCount64", (SYSCALL)GetTickCount64, 0 }, @@ -787,7 +867,7 @@ static struct win_syscall { { "GetTickCount64", (SYSCALL)0, 0 }, #endif -#define osGetTickCount64 ((ULONGLONG(WINAPI*)(VOID))aSyscall[67].pCurrent) +#define osGetTickCount64 ((ULONGLONG(WINAPI*)(VOID))aSyscall[68].pCurrent) #if SQLITE_OS_WINRT { "GetNativeSystemInfo", (SYSCALL)GetNativeSystemInfo, 0 }, @@ -796,7 +876,7 @@ static struct win_syscall { #endif #define osGetNativeSystemInfo ((VOID(WINAPI*)( \ - LPSYSTEM_INFO))aSyscall[68].pCurrent) + LPSYSTEM_INFO))aSyscall[69].pCurrent) #if defined(SQLITE_WIN32_HAS_ANSI) { "OutputDebugStringA", (SYSCALL)OutputDebugStringA, 0 }, @@ -804,7 +884,7 @@ static struct win_syscall { { "OutputDebugStringA", (SYSCALL)0, 0 }, #endif -#define osOutputDebugStringA ((VOID(WINAPI*)(LPCSTR))aSyscall[69].pCurrent) +#define osOutputDebugStringA ((VOID(WINAPI*)(LPCSTR))aSyscall[70].pCurrent) #if defined(SQLITE_WIN32_HAS_WIDE) { "OutputDebugStringW", (SYSCALL)OutputDebugStringW, 0 }, @@ -812,11 +892,11 @@ static struct win_syscall { { "OutputDebugStringW", (SYSCALL)0, 0 }, #endif -#define osOutputDebugStringW ((VOID(WINAPI*)(LPCWSTR))aSyscall[70].pCurrent) +#define osOutputDebugStringW ((VOID(WINAPI*)(LPCWSTR))aSyscall[71].pCurrent) { "GetProcessHeap", (SYSCALL)GetProcessHeap, 0 }, -#define osGetProcessHeap ((HANDLE(WINAPI*)(VOID))aSyscall[71].pCurrent) +#define osGetProcessHeap ((HANDLE(WINAPI*)(VOID))aSyscall[72].pCurrent) #if SQLITE_OS_WINRT && !defined(SQLITE_OMIT_WAL) { "CreateFileMappingFromApp", (SYSCALL)CreateFileMappingFromApp, 0 }, @@ -825,7 +905,7 @@ static struct win_syscall { #endif #define osCreateFileMappingFromApp ((HANDLE(WINAPI*)(HANDLE, \ - LPSECURITY_ATTRIBUTES,ULONG,ULONG64,LPCWSTR))aSyscall[72].pCurrent) + LPSECURITY_ATTRIBUTES,ULONG,ULONG64,LPCWSTR))aSyscall[73].pCurrent) }; /* End of the overrideable system calls */ @@ -917,7 +997,7 @@ static const char *winNextSystemCall(sqlite3_vfs *p, const char *zName){ ** (if available). */ -void sqlite3_win32_write_debug(char *zBuf, int nBuf){ +void sqlite3_win32_write_debug(const char *zBuf, int nBuf){ char zDbgBuf[SQLITE_WIN32_DBG_BUF_SIZE]; int nMin = MIN(nBuf, (SQLITE_WIN32_DBG_BUF_SIZE - 1)); /* may be negative. */ if( nMin<-1 ) nMin = -1; /* all negative values become -1. */ @@ -983,6 +1063,8 @@ void sqlite3_win32_sleep(DWORD milliseconds){ */ #if SQLITE_OS_WINCE || SQLITE_OS_WINRT # define isNT() (1) +#elif !defined(SQLITE_WIN32_HAS_WIDE) +# define isNT() (0) #else static int isNT(void){ if( sqlite3_os_type==0 ){ @@ -993,7 +1075,7 @@ void sqlite3_win32_sleep(DWORD milliseconds){ } return sqlite3_os_type==2; } -#endif /* SQLITE_OS_WINCE */ +#endif #ifdef SQLITE_WIN32_MALLOC /* @@ -1203,7 +1285,7 @@ static LPWSTR utf8ToUnicode(const char *zFilename){ if( nChar==0 ){ return 0; } - zWideFilename = sqlite3_malloc( nChar*sizeof(zWideFilename[0]) ); + zWideFilename = sqlite3MallocZero( nChar*sizeof(zWideFilename[0]) ); if( zWideFilename==0 ){ return 0; } @@ -1228,7 +1310,7 @@ static char *unicodeToUtf8(LPCWSTR zWideFilename){ if( nByte == 0 ){ return 0; } - zFilename = sqlite3_malloc( nByte ); + zFilename = sqlite3MallocZero( nByte ); if( zFilename==0 ){ return 0; } @@ -1258,7 +1340,7 @@ static LPWSTR mbcsToUnicode(const char *zFilename){ if( nByte==0 ){ return 0; } - zMbcsFilename = sqlite3_malloc( nByte*sizeof(zMbcsFilename[0]) ); + zMbcsFilename = sqlite3MallocZero( nByte*sizeof(zMbcsFilename[0]) ); if( zMbcsFilename==0 ){ return 0; } @@ -1287,7 +1369,7 @@ static char *unicodeToMbcs(LPCWSTR zWideFilename){ if( nByte == 0 ){ return 0; } - zFilename = sqlite3_malloc( nByte ); + zFilename = sqlite3MallocZero( nByte ); if( zFilename==0 ){ return 0; } @@ -1441,7 +1523,7 @@ static int getLastErrorMsg(DWORD lastErrno, int nBuf, char *zBuf){ } #endif if( 0 == dwLen ){ - sqlite3_snprintf(nBuf, zBuf, "OsError 0x%x (%u)", lastErrno, lastErrno); + sqlite3_snprintf(nBuf, zBuf, "OsError 0x%lx (%lu)", lastErrno, lastErrno); }else{ /* copy a maximum of nBuf chars to output buffer */ sqlite3_snprintf(nBuf, zBuf, "%s", zOut); @@ -1484,7 +1566,7 @@ static int winLogErrorAtLine( for(i=0; zMsg[i] && zMsg[i]!='\r' && zMsg[i]!='\n'; i++){} zMsg[i] = 0; sqlite3_log(errcode, - "os_win.c:%d: (%d) %s(%s) - %s", + "os_win.c:%d: (%lu) %s(%s) - %s", iLine, lastErrno, zFunc, zPath, zMsg ); @@ -1548,9 +1630,10 @@ static void logIoerr(int nRetry){ /************************************************************************* ** This section contains code for WinCE only. */ +#if !defined(SQLITE_MSVC_LOCALTIME_API) || !SQLITE_MSVC_LOCALTIME_API /* -** Windows CE does not have a localtime() function. So create a -** substitute. +** The MSVC CRT on Windows CE may not have a localtime() function. So +** create a substitute. */ #include struct tm *__cdecl localtime(const time_t *t) @@ -1574,6 +1657,7 @@ struct tm *__cdecl localtime(const time_t *t) y.tm_sec = pTm.wSecond; return &y; } +#endif #define HANDLE_TO_WINFILE(a) (winFile*)&((char*)a)[-(int)offsetof(winFile,h)] @@ -1595,15 +1679,17 @@ static void winceMutexAcquire(HANDLE h){ ** Create the mutex and shared memory used for locking in the file ** descriptor pFile */ -static BOOL winceCreateLock(const char *zFilename, winFile *pFile){ +static int winceCreateLock(const char *zFilename, winFile *pFile){ LPWSTR zTok; LPWSTR zName; + DWORD lastErrno; + BOOL bLogged = FALSE; BOOL bInit = TRUE; zName = utf8ToUnicode(zFilename); if( zName==0 ){ /* out of memory */ - return FALSE; + return SQLITE_IOERR_NOMEM; } /* Initialize the local lockdata */ @@ -1620,9 +1706,10 @@ static BOOL winceCreateLock(const char *zFilename, winFile *pFile){ pFile->hMutex = osCreateMutexW(NULL, FALSE, zName); if (!pFile->hMutex){ pFile->lastErrno = osGetLastError(); - winLogError(SQLITE_ERROR, pFile->lastErrno, "winceCreateLock1", zFilename); + winLogError(SQLITE_IOERR, pFile->lastErrno, + "winceCreateLock1", zFilename); sqlite3_free(zName); - return FALSE; + return SQLITE_IOERR; } /* Acquire the mutex before continuing */ @@ -1639,41 +1726,49 @@ static BOOL winceCreateLock(const char *zFilename, winFile *pFile){ /* Set a flag that indicates we're the first to create the memory so it ** must be zero-initialized */ - if (osGetLastError() == ERROR_ALREADY_EXISTS){ + lastErrno = osGetLastError(); + if (lastErrno == ERROR_ALREADY_EXISTS){ bInit = FALSE; } sqlite3_free(zName); /* If we succeeded in making the shared memory handle, map it. */ - if (pFile->hShared){ + if( pFile->hShared ){ pFile->shared = (winceLock*)osMapViewOfFile(pFile->hShared, FILE_MAP_READ|FILE_MAP_WRITE, 0, 0, sizeof(winceLock)); /* If mapping failed, close the shared memory handle and erase it */ - if (!pFile->shared){ + if( !pFile->shared ){ pFile->lastErrno = osGetLastError(); - winLogError(SQLITE_ERROR, pFile->lastErrno, - "winceCreateLock2", zFilename); + winLogError(SQLITE_IOERR, pFile->lastErrno, + "winceCreateLock2", zFilename); + bLogged = TRUE; osCloseHandle(pFile->hShared); pFile->hShared = NULL; } } /* If shared memory could not be created, then close the mutex and fail */ - if (pFile->hShared == NULL){ + if( pFile->hShared==NULL ){ + if( !bLogged ){ + pFile->lastErrno = lastErrno; + winLogError(SQLITE_IOERR, pFile->lastErrno, + "winceCreateLock3", zFilename); + bLogged = TRUE; + } winceMutexRelease(pFile->hMutex); osCloseHandle(pFile->hMutex); pFile->hMutex = NULL; - return FALSE; + return SQLITE_IOERR; } /* Initialize the shared memory if we're supposed to */ - if (bInit) { + if( bInit ){ memset(pFile->shared, 0, sizeof(winceLock)); } winceMutexRelease(pFile->hMutex); - return TRUE; + return SQLITE_OK; } /* @@ -1752,7 +1847,8 @@ static BOOL winceLockFile( } /* Want a pending lock? */ - else if (dwFileOffsetLow == (DWORD)PENDING_BYTE && nNumberOfBytesToLockLow == 1){ + else if (dwFileOffsetLow == (DWORD)PENDING_BYTE + && nNumberOfBytesToLockLow == 1){ /* If no pending lock has been acquired, then acquire it */ if (pFile->shared->bPending == 0) { pFile->shared->bPending = TRUE; @@ -1762,7 +1858,8 @@ static BOOL winceLockFile( } /* Want a reserved lock? */ - else if (dwFileOffsetLow == (DWORD)RESERVED_BYTE && nNumberOfBytesToLockLow == 1){ + else if (dwFileOffsetLow == (DWORD)RESERVED_BYTE + && nNumberOfBytesToLockLow == 1){ if (pFile->shared->bReserved == 0) { pFile->shared->bReserved = TRUE; pFile->local.bReserved = TRUE; @@ -1805,7 +1902,8 @@ static BOOL winceUnlockFile( /* Did we just have a reader lock? */ else if (pFile->local.nReaders){ - assert(nNumberOfBytesToUnlockLow == (DWORD)SHARED_SIZE || nNumberOfBytesToUnlockLow == 1); + assert(nNumberOfBytesToUnlockLow == (DWORD)SHARED_SIZE + || nNumberOfBytesToUnlockLow == 1); pFile->local.nReaders --; if (pFile->local.nReaders == 0) { @@ -1816,7 +1914,8 @@ static BOOL winceUnlockFile( } /* Releasing a pending lock */ - else if (dwFileOffsetLow == (DWORD)PENDING_BYTE && nNumberOfBytesToUnlockLow == 1){ + else if (dwFileOffsetLow == (DWORD)PENDING_BYTE + && nNumberOfBytesToUnlockLow == 1){ if (pFile->local.bPending){ pFile->local.bPending = FALSE; pFile->shared->bPending = FALSE; @@ -1824,7 +1923,8 @@ static BOOL winceUnlockFile( } } /* Releasing a reserved lock */ - else if (dwFileOffsetLow == (DWORD)RESERVED_BYTE && nNumberOfBytesToUnlockLow == 1){ + else if (dwFileOffsetLow == (DWORD)RESERVED_BYTE + && nNumberOfBytesToUnlockLow == 1){ if (pFile->local.bReserved) { pFile->local.bReserved = FALSE; pFile->shared->bReserved = FALSE; @@ -1927,6 +2027,8 @@ static int seekWinFile(winFile *pFile, sqlite3_int64 iOffset){ DWORD dwRet; /* Value returned by SetFilePointer() */ DWORD lastErrno; /* Value returned by GetLastError() */ + OSTRACE(("SEEK file=%p, offset=%lld\n", pFile->h, iOffset)); + upperBits = (LONG)((iOffset>>32) & 0x7fffffff); lowerBits = (LONG)(iOffset & 0xffffffff); @@ -1934,7 +2036,7 @@ static int seekWinFile(winFile *pFile, sqlite3_int64 iOffset){ ** containing the lower 32-bits of the new file-offset. Or, if it fails, ** it returns INVALID_SET_FILE_POINTER. However according to MSDN, ** INVALID_SET_FILE_POINTER may also be a valid new offset. So to determine - ** whether an error has actually occured, it is also necessary to call + ** whether an error has actually occurred, it is also necessary to call ** GetLastError(). */ dwRet = osSetFilePointer(pFile->h, lowerBits, &upperBits, FILE_BEGIN); @@ -1944,9 +2046,11 @@ static int seekWinFile(winFile *pFile, sqlite3_int64 iOffset){ pFile->lastErrno = lastErrno; winLogError(SQLITE_IOERR_SEEK, pFile->lastErrno, "seekWinFile", pFile->zPath); + OSTRACE(("SEEK file=%p, rc=SQLITE_IOERR_SEEK\n", pFile->h)); return 1; } + OSTRACE(("SEEK file=%p, rc=SQLITE_OK\n", pFile->h)); return 0; #else /* @@ -1963,13 +2067,20 @@ static int seekWinFile(winFile *pFile, sqlite3_int64 iOffset){ pFile->lastErrno = osGetLastError(); winLogError(SQLITE_IOERR_SEEK, pFile->lastErrno, "seekWinFile", pFile->zPath); + OSTRACE(("SEEK file=%p, rc=SQLITE_IOERR_SEEK\n", pFile->h)); return 1; } + OSTRACE(("SEEK file=%p, rc=SQLITE_OK\n", pFile->h)); return 0; #endif } +#if SQLITE_MAX_MMAP_SIZE>0 +/* Forward references to VFS methods */ +static int winUnmapfile(winFile*); +#endif + /* ** Close a file. ** @@ -1989,7 +2100,14 @@ static int winClose(sqlite3_file *id){ #ifndef SQLITE_OMIT_WAL assert( pFile->pShm==0 ); #endif - OSTRACE(("CLOSE %d\n", pFile->h)); + assert( pFile->h!=NULL && pFile->h!=INVALID_HANDLE_VALUE ); + OSTRACE(("CLOSE file=%p\n", pFile->h)); + +#if SQLITE_MAX_MMAP_SIZE>0 + rc = winUnmapfile(pFile); + if( rc!=SQLITE_OK ) return rc; +#endif + do{ rc = osCloseHandle(pFile->h); /* SimulateIOError( rc=0; cnt=MX_CLOSE_ATTEMPT; ); */ @@ -2009,11 +2127,11 @@ static int winClose(sqlite3_file *id){ sqlite3_free(pFile->zDeleteOnClose); } #endif - OSTRACE(("CLOSE %d %s\n", pFile->h, rc ? "ok" : "failed")); if( rc ){ pFile->h = NULL; } OpenCounter(-1); + OSTRACE(("CLOSE file=%p, rc=%s\n", pFile->h, rc ? "ok" : "failed")); return rc ? SQLITE_OK : winLogError(SQLITE_IOERR_CLOSE, osGetLastError(), "winClose", pFile->zPath); @@ -2038,11 +2156,33 @@ static int winRead( int nRetry = 0; /* Number of retrys */ assert( id!=0 ); + assert( amt>0 ); + assert( offset>=0 ); SimulateIOError(return SQLITE_IOERR_READ); - OSTRACE(("READ %d lock=%d\n", pFile->h, pFile->locktype)); + OSTRACE(("READ file=%p, buffer=%p, amount=%d, offset=%lld, lock=%d\n", + pFile->h, pBuf, amt, offset, pFile->locktype)); + +#if SQLITE_MAX_MMAP_SIZE>0 + /* Deal with as much of this read request as possible by transfering + ** data from the memory mapping using memcpy(). */ + if( offsetmmapSize ){ + if( offset+amt <= pFile->mmapSize ){ + memcpy(pBuf, &((u8 *)(pFile->pMapRegion))[offset], amt); + OSTRACE(("READ-MMAP file=%p, rc=SQLITE_OK\n", pFile->h)); + return SQLITE_OK; + }else{ + int nCopy = (int)(pFile->mmapSize - offset); + memcpy(pBuf, &((u8 *)(pFile->pMapRegion))[offset], nCopy); + pBuf = &((u8 *)pBuf)[nCopy]; + amt -= nCopy; + offset += nCopy; + } + } +#endif #if SQLITE_OS_WINCE if( seekWinFile(pFile, offset) ){ + OSTRACE(("READ file=%p, rc=SQLITE_FULL\n", pFile->h)); return SQLITE_FULL; } while( !osReadFile(pFile->h, pBuf, amt, &nRead, 0) ){ @@ -2056,6 +2196,7 @@ static int winRead( DWORD lastErrno; if( retryIoerr(&nRetry, &lastErrno) ) continue; pFile->lastErrno = lastErrno; + OSTRACE(("READ file=%p, rc=SQLITE_IOERR_READ\n", pFile->h)); return winLogError(SQLITE_IOERR_READ, pFile->lastErrno, "winRead", pFile->zPath); } @@ -2063,9 +2204,11 @@ static int winRead( if( nRead<(DWORD)amt ){ /* Unread parts of the buffer must be zero-filled */ memset(&((char*)pBuf)[nRead], 0, amt-nRead); + OSTRACE(("READ file=%p, rc=SQLITE_IOERR_SHORT_READ\n", pFile->h)); return SQLITE_IOERR_SHORT_READ; } + OSTRACE(("READ file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } @@ -2079,7 +2222,7 @@ static int winWrite( int amt, /* Number of bytes to write */ sqlite3_int64 offset /* Offset into the file to begin writing at */ ){ - int rc = 0; /* True if error has occured, else false */ + int rc = 0; /* True if error has occurred, else false */ winFile *pFile = (winFile*)id; /* File handle */ int nRetry = 0; /* Number of retries */ @@ -2088,7 +2231,26 @@ static int winWrite( SimulateIOError(return SQLITE_IOERR_WRITE); SimulateDiskfullError(return SQLITE_FULL); - OSTRACE(("WRITE %d lock=%d\n", pFile->h, pFile->locktype)); + OSTRACE(("WRITE file=%p, buffer=%p, amount=%d, offset=%lld, lock=%d\n", + pFile->h, pBuf, amt, offset, pFile->locktype)); + +#if SQLITE_MAX_MMAP_SIZE>0 + /* Deal with as much of this write request as possible by transfering + ** data from the memory mapping using memcpy(). */ + if( offsetmmapSize ){ + if( offset+amt <= pFile->mmapSize ){ + memcpy(&((u8 *)(pFile->pMapRegion))[offset], pBuf, amt); + OSTRACE(("WRITE-MMAP file=%p, rc=SQLITE_OK\n", pFile->h)); + return SQLITE_OK; + }else{ + int nCopy = (int)(pFile->mmapSize - offset); + memcpy(&((u8 *)(pFile->pMapRegion))[offset], pBuf, nCopy); + pBuf = &((u8 *)pBuf)[nCopy]; + amt -= nCopy; + offset += nCopy; + } + } +#endif #if SQLITE_OS_WINCE rc = seekWinFile(pFile, offset); @@ -2119,7 +2281,8 @@ static int winWrite( if( retryIoerr(&nRetry, &lastErrno) ) continue; break; } - if( nWrite<=0 ){ + assert( nWrite==0 || nWrite<=(DWORD)nRem ); + if( nWrite==0 || nWrite>(DWORD)nRem ){ lastErrno = osGetLastError(); break; } @@ -2140,13 +2303,16 @@ static int winWrite( if( rc ){ if( ( pFile->lastErrno==ERROR_HANDLE_DISK_FULL ) || ( pFile->lastErrno==ERROR_DISK_FULL )){ + OSTRACE(("WRITE file=%p, rc=SQLITE_FULL\n", pFile->h)); return SQLITE_FULL; } + OSTRACE(("WRITE file=%p, rc=SQLITE_IOERR_WRITE\n", pFile->h)); return winLogError(SQLITE_IOERR_WRITE, pFile->lastErrno, "winWrite", pFile->zPath); }else{ logIoerr(nRetry); } + OSTRACE(("WRITE file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } @@ -2156,11 +2322,12 @@ static int winWrite( static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ winFile *pFile = (winFile*)id; /* File handle object */ int rc = SQLITE_OK; /* Return code for this function */ + DWORD lastErrno; assert( pFile ); - - OSTRACE(("TRUNCATE %d %lld\n", pFile->h, nByte)); SimulateIOError(return SQLITE_IOERR_TRUNCATE); + OSTRACE(("TRUNCATE file=%p, size=%lld, lock=%d\n", + pFile->h, nByte, pFile->locktype)); /* If the user has configured a chunk-size for this file, truncate the ** file so that it consists of an integer number of chunks (i.e. the @@ -2174,14 +2341,25 @@ static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ /* SetEndOfFile() returns non-zero when successful, or zero when it fails. */ if( seekWinFile(pFile, nByte) ){ rc = winLogError(SQLITE_IOERR_TRUNCATE, pFile->lastErrno, - "winTruncate1", pFile->zPath); - }else if( 0==osSetEndOfFile(pFile->h) ){ - pFile->lastErrno = osGetLastError(); + "winTruncate1", pFile->zPath); + }else if( 0==osSetEndOfFile(pFile->h) && + ((lastErrno = osGetLastError())!=ERROR_USER_MAPPED_FILE) ){ + pFile->lastErrno = lastErrno; rc = winLogError(SQLITE_IOERR_TRUNCATE, pFile->lastErrno, - "winTruncate2", pFile->zPath); + "winTruncate2", pFile->zPath); } - OSTRACE(("TRUNCATE %d %lld %s\n", pFile->h, nByte, rc ? "failed" : "ok")); +#if SQLITE_MAX_MMAP_SIZE>0 + /* If the file was truncated to a size smaller than the currently + ** mapped region, reduce the effective mapping size as well. SQLite will + ** use read() and write() to access data beyond this point from now on. + */ + if( pFile->pMapRegion && nBytemmapSize ){ + pFile->mmapSize = nByte; + } +#endif + + OSTRACE(("TRUNCATE file=%p, rc=%s\n", pFile->h, sqlite3ErrName(rc))); return rc; } @@ -2221,13 +2399,14 @@ static int winSync(sqlite3_file *id, int flags){ || (flags&0x0F)==SQLITE_SYNC_FULL ); - OSTRACE(("SYNC %d lock=%d\n", pFile->h, pFile->locktype)); - /* Unix cannot, but some systems may return SQLITE_FULL from here. This ** line is to test that doing so does not cause any problems. */ SimulateDiskfullError( return SQLITE_FULL ); + OSTRACE(("SYNC file=%p, flags=%x, lock=%d\n", + pFile->h, flags, pFile->locktype)); + #ifndef SQLITE_TEST UNUSED_PARAMETER(flags); #else @@ -2246,9 +2425,11 @@ static int winSync(sqlite3_file *id, int flags){ rc = osFlushFileBuffers(pFile->h); SimulateIOError( rc=FALSE ); if( rc ){ + OSTRACE(("SYNC file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; }else{ pFile->lastErrno = osGetLastError(); + OSTRACE(("SYNC file=%p, rc=SQLITE_IOERR_FSYNC\n", pFile->h)); return winLogError(SQLITE_IOERR_FSYNC, pFile->lastErrno, "winSync", pFile->zPath); } @@ -2263,7 +2444,10 @@ static int winFileSize(sqlite3_file *id, sqlite3_int64 *pSize){ int rc = SQLITE_OK; assert( id!=0 ); + assert( pSize!=0 ); SimulateIOError(return SQLITE_IOERR_FSTAT); + OSTRACE(("SIZE file=%p, pSize=%p\n", pFile->h, pSize)); + #if SQLITE_OS_WINRT { FILE_STANDARD_INFO info; @@ -2292,6 +2476,8 @@ static int winFileSize(sqlite3_file *id, sqlite3_int64 *pSize){ } } #endif + OSTRACE(("SIZE file=%p, pSize=%p, *pSize=%lld, rc=%s\n", + pFile->h, pSize, *pSize, sqlite3ErrName(rc))); return rc; } @@ -2333,6 +2519,7 @@ static int winFileSize(sqlite3_file *id, sqlite3_int64 *pSize){ */ static int getReadLock(winFile *pFile){ int res; + OSTRACE(("READ-LOCK file=%p, lock=%d\n", pFile->h, pFile->locktype)); if( isNT() ){ #if SQLITE_OS_WINCE /* @@ -2358,6 +2545,7 @@ static int getReadLock(winFile *pFile){ pFile->lastErrno = osGetLastError(); /* No need to log a failure to lock */ } + OSTRACE(("READ-LOCK file=%p, rc=%s\n", pFile->h, sqlite3ErrName(res))); return res; } @@ -2367,6 +2555,7 @@ static int getReadLock(winFile *pFile){ static int unlockReadLock(winFile *pFile){ int res; DWORD lastErrno; + OSTRACE(("READ-UNLOCK file=%p, lock=%d\n", pFile->h, pFile->locktype)); if( isNT() ){ res = winUnlockFile(&pFile->h, SHARED_FIRST, 0, SHARED_SIZE, 0); } @@ -2380,6 +2569,7 @@ static int unlockReadLock(winFile *pFile){ winLogError(SQLITE_IOERR_UNLOCK, pFile->lastErrno, "unlockReadLock", pFile->zPath); } + OSTRACE(("READ-UNLOCK file=%p, rc=%s\n", pFile->h, sqlite3ErrName(res))); return res; } @@ -2418,14 +2608,15 @@ static int winLock(sqlite3_file *id, int locktype){ DWORD lastErrno = NO_ERROR; assert( id!=0 ); - OSTRACE(("LOCK %d %d was %d(%d)\n", - pFile->h, locktype, pFile->locktype, pFile->sharedLockByte)); + OSTRACE(("LOCK file=%p, oldLock=%d(%d), newLock=%d\n", + pFile->h, pFile->locktype, pFile->sharedLockByte, locktype)); /* If there is already a lock of this type or more restrictive on the ** OsFile, do nothing. Don't use the end_lock: exit path, as ** sqlite3OsEnterMutex() hasn't been called yet. */ if( pFile->locktype>=locktype ){ + OSTRACE(("LOCK-HELD file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } @@ -2453,7 +2644,8 @@ static int winLock(sqlite3_file *id, int locktype){ ** If you are using this code as a model for alternative VFSes, do not ** copy this retry logic. It is a hack intended for Windows only. */ - OSTRACE(("could not get a PENDING lock. cnt=%d\n", cnt)); + OSTRACE(("LOCK-PENDING-FAIL file=%p, count=%d, rc=%s\n", + pFile->h, cnt, sqlite3ErrName(res))); if( cnt ) sqlite3_win32_sleep(1); } gotPendingLock = res; @@ -2498,14 +2690,12 @@ static int winLock(sqlite3_file *id, int locktype){ if( locktype==EXCLUSIVE_LOCK && res ){ assert( pFile->locktype>=SHARED_LOCK ); res = unlockReadLock(pFile); - OSTRACE(("unreadlock = %d\n", res)); res = winLockFile(&pFile->h, SQLITE_LOCKFILE_FLAGS, SHARED_FIRST, 0, SHARED_SIZE, 0); if( res ){ newLocktype = EXCLUSIVE_LOCK; }else{ lastErrno = osGetLastError(); - OSTRACE(("error-code = %d\n", lastErrno)); getReadLock(pFile); } } @@ -2523,12 +2713,14 @@ static int winLock(sqlite3_file *id, int locktype){ if( res ){ rc = SQLITE_OK; }else{ - OSTRACE(("LOCK FAILED %d trying for %d but got %d\n", pFile->h, - locktype, newLocktype)); + OSTRACE(("LOCK-FAIL file=%p, wanted=%d, got=%d\n", + pFile->h, locktype, newLocktype)); pFile->lastErrno = lastErrno; rc = SQLITE_BUSY; } pFile->locktype = (u8)newLocktype; + OSTRACE(("LOCK file=%p, lock=%d, rc=%s\n", + pFile->h, pFile->locktype, sqlite3ErrName(rc))); return rc; } @@ -2542,20 +2734,23 @@ static int winCheckReservedLock(sqlite3_file *id, int *pResOut){ winFile *pFile = (winFile*)id; SimulateIOError( return SQLITE_IOERR_CHECKRESERVEDLOCK; ); + OSTRACE(("TEST-WR-LOCK file=%p, pResOut=%p\n", pFile->h, pResOut)); assert( id!=0 ); if( pFile->locktype>=RESERVED_LOCK ){ rc = 1; - OSTRACE(("TEST WR-LOCK %d %d (local)\n", pFile->h, rc)); + OSTRACE(("TEST-WR-LOCK file=%p, rc=%d (local)\n", pFile->h, rc)); }else{ - rc = winLockFile(&pFile->h, SQLITE_LOCKFILE_FLAGS, RESERVED_BYTE, 0, 1, 0); + rc = winLockFile(&pFile->h, SQLITE_LOCKFILEEX_FLAGS,RESERVED_BYTE, 0, 1, 0); if( rc ){ winUnlockFile(&pFile->h, RESERVED_BYTE, 0, 1, 0); } rc = !rc; - OSTRACE(("TEST WR-LOCK %d %d (remote)\n", pFile->h, rc)); + OSTRACE(("TEST-WR-LOCK file=%p, rc=%d (remote)\n", pFile->h, rc)); } *pResOut = rc; + OSTRACE(("TEST-WR-LOCK file=%p, pResOut=%p, *pResOut=%d, rc=SQLITE_OK\n", + pFile->h, pResOut, *pResOut)); return SQLITE_OK; } @@ -2576,8 +2771,8 @@ static int winUnlock(sqlite3_file *id, int locktype){ int rc = SQLITE_OK; assert( pFile!=0 ); assert( locktype<=SHARED_LOCK ); - OSTRACE(("UNLOCK %d to %d was %d(%d)\n", pFile->h, locktype, - pFile->locktype, pFile->sharedLockByte)); + OSTRACE(("UNLOCK file=%p, oldLock=%d(%d), newLock=%d\n", + pFile->h, pFile->locktype, pFile->sharedLockByte, locktype)); type = pFile->locktype; if( type>=EXCLUSIVE_LOCK ){ winUnlockFile(&pFile->h, SHARED_FIRST, 0, SHARED_SIZE, 0); @@ -2598,6 +2793,8 @@ static int winUnlock(sqlite3_file *id, int locktype){ winUnlockFile(&pFile->h, PENDING_BYTE, 0, 1, 0); } pFile->locktype = (u8)locktype; + OSTRACE(("UNLOCK file=%p, lock=%d, rc=%s\n", + pFile->h, pFile->locktype, sqlite3ErrName(rc))); return rc; } @@ -2617,22 +2814,29 @@ static void winModeBit(winFile *pFile, unsigned char mask, int *pArg){ } } +/* Forward declaration */ +static int getTempname(int nBuf, char *zBuf); + /* ** Control and query of the open file handle. */ static int winFileControl(sqlite3_file *id, int op, void *pArg){ winFile *pFile = (winFile*)id; + OSTRACE(("FCNTL file=%p, op=%d, pArg=%p\n", pFile->h, op, pArg)); switch( op ){ case SQLITE_FCNTL_LOCKSTATE: { *(int*)pArg = pFile->locktype; + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } case SQLITE_LAST_ERRNO: { *(int*)pArg = (int)pFile->lastErrno; + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } case SQLITE_FCNTL_CHUNK_SIZE: { pFile->szChunk = *(int *)pArg; + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } case SQLITE_FCNTL_SIZE_HINT: { @@ -2647,20 +2851,25 @@ static int winFileControl(sqlite3_file *id, int op, void *pArg){ SimulateIOErrorBenign(0); } } + OSTRACE(("FCNTL file=%p, rc=%s\n", pFile->h, sqlite3ErrName(rc))); return rc; } + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } case SQLITE_FCNTL_PERSIST_WAL: { winModeBit(pFile, WINFILE_PERSIST_WAL, (int*)pArg); + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } case SQLITE_FCNTL_POWERSAFE_OVERWRITE: { winModeBit(pFile, WINFILE_PSOW, (int*)pArg); + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } case SQLITE_FCNTL_VFSNAME: { *(char**)pArg = sqlite3_mprintf("win32"); + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } case SQLITE_FCNTL_WIN32_AV_RETRY: { @@ -2675,9 +2884,32 @@ static int winFileControl(sqlite3_file *id, int op, void *pArg){ }else{ a[1] = win32IoerrRetryDelay; } + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } + case SQLITE_FCNTL_TEMPFILENAME: { + char *zTFile = sqlite3MallocZero( pFile->pVfs->mxPathname ); + if( zTFile ){ + getTempname(pFile->pVfs->mxPathname, zTFile); + *(char**)pArg = zTFile; + } + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); + return SQLITE_OK; + } +#if SQLITE_MAX_MMAP_SIZE>0 + case SQLITE_FCNTL_MMAP_SIZE: { + i64 newLimit = *(i64*)pArg; + if( newLimit>sqlite3GlobalConfig.mxMmap ){ + newLimit = sqlite3GlobalConfig.mxMmap; + } + *(i64*)pArg = pFile->mmapSizeMax; + if( newLimit>=0 ) pFile->mmapSizeMax = newLimit; + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); + return SQLITE_OK; + } +#endif } + OSTRACE(("FCNTL file=%p, rc=SQLITE_NOTFOUND\n", pFile->h)); return SQLITE_NOTFOUND; } @@ -2705,8 +2937,6 @@ static int winDeviceCharacteristics(sqlite3_file *id){ ((p->ctrlFlags & WINFILE_PSOW)?SQLITE_IOCAP_POWERSAFE_OVERWRITE:0); } -#ifndef SQLITE_OMIT_WAL - /* ** Windows will only let you create file view mappings ** on allocation size granularity boundaries. @@ -2715,6 +2945,8 @@ static int winDeviceCharacteristics(sqlite3_file *id){ */ SYSTEM_INFO winSysInfo; +#ifndef SQLITE_OMIT_WAL + /* ** Helper functions to obtain and relinquish the global mutex. The ** global mutex is used to protect the winLockInfo objects used by @@ -2838,6 +3070,9 @@ static int winShmSystemLock( /* Access to the winShmNode object is serialized by the caller */ assert( sqlite3_mutex_held(pFile->mutex) || pFile->nRef==0 ); + OSTRACE(("SHM-LOCK file=%p, lock=%d, offset=%d, size=%d\n", + pFile->hFile.h, lockType, ofst, nByte)); + /* Release/Acquire the system-level lock */ if( lockType==_SHM_UNLCK ){ rc = winUnlockFile(&pFile->hFile.h, ofst, 0, nByte, 0); @@ -2855,11 +3090,9 @@ static int winShmSystemLock( rc = SQLITE_BUSY; } - OSTRACE(("SHM-LOCK %d %s %s 0x%08lx\n", - pFile->hFile.h, - rc==SQLITE_OK ? "ok" : "failed", - lockType==_SHM_UNLCK ? "UnlockFileEx" : "LockFileEx", - pFile->lastErrno)); + OSTRACE(("SHM-LOCK file=%p, func=%s, errno=%lu, rc=%s\n", + pFile->hFile.h, (lockType == _SHM_UNLCK) ? "winUnlockFile" : + "winLockFile", pFile->lastErrno, sqlite3ErrName(rc))); return rc; } @@ -2879,6 +3112,8 @@ static void winShmPurge(sqlite3_vfs *pVfs, int deleteFlag){ winShmNode *p; BOOL bRc; assert( winShmMutexHeld() ); + OSTRACE(("SHM-PURGE pid=%lu, deleteFlag=%d\n", + osGetCurrentProcessId(), deleteFlag)); pp = &winShmNodeList; while( (p = *pp)!=0 ){ if( p->nRef==0 ){ @@ -2886,15 +3121,13 @@ static void winShmPurge(sqlite3_vfs *pVfs, int deleteFlag){ if( p->mutex ) sqlite3_mutex_free(p->mutex); for(i=0; inRegion; i++){ bRc = osUnmapViewOfFile(p->aRegion[i].pMap); - OSTRACE(("SHM-PURGE pid-%d unmap region=%d %s\n", - (int)osGetCurrentProcessId(), i, - bRc ? "ok" : "failed")); + OSTRACE(("SHM-PURGE-UNMAP pid=%lu, region=%d, rc=%s\n", + osGetCurrentProcessId(), i, bRc ? "ok" : "failed")); bRc = osCloseHandle(p->aRegion[i].hMap); - OSTRACE(("SHM-PURGE pid-%d close region=%d %s\n", - (int)osGetCurrentProcessId(), i, - bRc ? "ok" : "failed")); + OSTRACE(("SHM-PURGE-CLOSE pid=%lu, region=%d, rc=%s\n", + osGetCurrentProcessId(), i, bRc ? "ok" : "failed")); } - if( p->hFile.h != INVALID_HANDLE_VALUE ){ + if( p->hFile.h!=NULL && p->hFile.h!=INVALID_HANDLE_VALUE ){ SimulateIOErrorBenign(1); winClose((sqlite3_file *)&p->hFile); SimulateIOErrorBenign(0); @@ -2934,16 +3167,14 @@ static int winOpenSharedMemory(winFile *pDbFd){ /* Allocate space for the new sqlite3_shm object. Also speculatively ** allocate space for a new winShmNode and filename. */ - p = sqlite3_malloc( sizeof(*p) ); + p = sqlite3MallocZero( sizeof(*p) ); if( p==0 ) return SQLITE_IOERR_NOMEM; - memset(p, 0, sizeof(*p)); nName = sqlite3Strlen30(pDbFd->zPath); - pNew = sqlite3_malloc( sizeof(*pShmNode) + nName + 17 ); + pNew = sqlite3MallocZero( sizeof(*pShmNode) + nName + 17 ); if( pNew==0 ){ sqlite3_free(p); return SQLITE_IOERR_NOMEM; } - memset(pNew, 0, sizeof(*pNew) + nName + 17); pNew->zFilename = (char*)&pNew[1]; sqlite3_snprintf(nName+15, pNew->zFilename, "%s-shm", pDbFd->zPath); sqlite3FileSuffix3(pDbFd->zPath, pNew->zFilename); @@ -2976,7 +3207,7 @@ static int winOpenSharedMemory(winFile *pDbFd){ rc = winOpen(pDbFd->pVfs, pShmNode->zFilename, /* Name of the file (UTF-8) */ (sqlite3_file*)&pShmNode->hFile, /* File handle here */ - SQLITE_OPEN_WAL | SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, /* Mode flags */ + SQLITE_OPEN_WAL | SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0); if( SQLITE_OK!=rc ){ goto shm_open_err; @@ -3173,9 +3404,9 @@ static int winShmLock( } } sqlite3_mutex_leave(pShmNode->mutex); - OSTRACE(("SHM-LOCK shmid-%d, pid-%d got %03x,%03x %s\n", - p->id, (int)osGetCurrentProcessId(), p->sharedMask, p->exclMask, - rc ? "failed" : "ok")); + OSTRACE(("SHM-LOCK pid=%lu, id=%d, sharedMask=%03x, exclMask=%03x, rc=%s\n", + osGetCurrentProcessId(), p->id, p->sharedMask, p->exclMask, + sqlite3ErrName(rc))); return rc; } @@ -3280,20 +3511,24 @@ static int winShmMap( pShmNode->aRegion = apNew; while( pShmNode->nRegion<=iRegion ){ - HANDLE hMap; /* file-mapping handle */ + HANDLE hMap = NULL; /* file-mapping handle */ void *pMap = 0; /* Mapped memory region */ #if SQLITE_OS_WINRT hMap = osCreateFileMappingFromApp(pShmNode->hFile.h, NULL, PAGE_READWRITE, nByte, NULL ); -#else +#elif defined(SQLITE_WIN32_HAS_WIDE) hMap = osCreateFileMappingW(pShmNode->hFile.h, NULL, PAGE_READWRITE, 0, nByte, NULL ); +#elif defined(SQLITE_WIN32_HAS_ANSI) + hMap = osCreateFileMappingA(pShmNode->hFile.h, + NULL, PAGE_READWRITE, 0, nByte, NULL + ); #endif - OSTRACE(("SHM-MAP pid-%d create region=%d nbyte=%d %s\n", - (int)osGetCurrentProcessId(), pShmNode->nRegion, nByte, + OSTRACE(("SHM-MAP-CREATE pid=%lu, region=%d, size=%d, rc=%s\n", + osGetCurrentProcessId(), pShmNode->nRegion, nByte, hMap ? "ok" : "failed")); if( hMap ){ int iOffset = pShmNode->nRegion*szRegion; @@ -3307,8 +3542,8 @@ static int winShmMap( 0, iOffset - iOffsetShift, szRegion + iOffsetShift ); #endif - OSTRACE(("SHM-MAP pid-%d map region=%d offset=%d size=%d %s\n", - (int)osGetCurrentProcessId(), pShmNode->nRegion, iOffset, + OSTRACE(("SHM-MAP-MAP pid=%lu, region=%d, offset=%d, size=%d, rc=%s\n", + osGetCurrentProcessId(), pShmNode->nRegion, iOffset, szRegion, pMap ? "ok" : "failed")); } if( !pMap ){ @@ -3345,6 +3580,230 @@ shmpage_out: # define winShmUnmap 0 #endif /* #ifndef SQLITE_OMIT_WAL */ +/* +** Cleans up the mapped region of the specified file, if any. +*/ +#if SQLITE_MAX_MMAP_SIZE>0 +static int winUnmapfile(winFile *pFile){ + assert( pFile!=0 ); + OSTRACE(("UNMAP-FILE pid=%lu, pFile=%p, hMap=%p, pMapRegion=%p, " + "mmapSize=%lld, mmapSizeActual=%lld, mmapSizeMax=%lld\n", + osGetCurrentProcessId(), pFile, pFile->hMap, pFile->pMapRegion, + pFile->mmapSize, pFile->mmapSizeActual, pFile->mmapSizeMax)); + if( pFile->pMapRegion ){ + if( !osUnmapViewOfFile(pFile->pMapRegion) ){ + pFile->lastErrno = osGetLastError(); + OSTRACE(("UNMAP-FILE pid=%lu, pFile=%p, pMapRegion=%p, " + "rc=SQLITE_IOERR_MMAP\n", osGetCurrentProcessId(), pFile, + pFile->pMapRegion)); + return winLogError(SQLITE_IOERR_MMAP, pFile->lastErrno, + "winUnmap1", pFile->zPath); + } + pFile->pMapRegion = 0; + pFile->mmapSize = 0; + pFile->mmapSizeActual = 0; + } + if( pFile->hMap!=NULL ){ + if( !osCloseHandle(pFile->hMap) ){ + pFile->lastErrno = osGetLastError(); + OSTRACE(("UNMAP-FILE pid=%lu, pFile=%p, hMap=%p, rc=SQLITE_IOERR_MMAP\n", + osGetCurrentProcessId(), pFile, pFile->hMap)); + return winLogError(SQLITE_IOERR_MMAP, pFile->lastErrno, + "winUnmap2", pFile->zPath); + } + pFile->hMap = NULL; + } + OSTRACE(("UNMAP-FILE pid=%lu, pFile=%p, rc=SQLITE_OK\n", + osGetCurrentProcessId(), pFile)); + return SQLITE_OK; +} + +/* +** Memory map or remap the file opened by file-descriptor pFd (if the file +** is already mapped, the existing mapping is replaced by the new). Or, if +** there already exists a mapping for this file, and there are still +** outstanding xFetch() references to it, this function is a no-op. +** +** If parameter nByte is non-negative, then it is the requested size of +** the mapping to create. Otherwise, if nByte is less than zero, then the +** requested size is the size of the file on disk. The actual size of the +** created mapping is either the requested size or the value configured +** using SQLITE_FCNTL_MMAP_SIZE, whichever is smaller. +** +** SQLITE_OK is returned if no error occurs (even if the mapping is not +** recreated as a result of outstanding references) or an SQLite error +** code otherwise. +*/ +static int winMapfile(winFile *pFd, sqlite3_int64 nByte){ + sqlite3_int64 nMap = nByte; + int rc; + + assert( nMap>=0 || pFd->nFetchOut==0 ); + OSTRACE(("MAP-FILE pid=%lu, pFile=%p, size=%lld\n", + osGetCurrentProcessId(), pFd, nByte)); + + if( pFd->nFetchOut>0 ) return SQLITE_OK; + + if( nMap<0 ){ + rc = winFileSize((sqlite3_file*)pFd, &nMap); + if( rc ){ + OSTRACE(("MAP-FILE pid=%lu, pFile=%p, rc=SQLITE_IOERR_FSTAT\n", + osGetCurrentProcessId(), pFd)); + return SQLITE_IOERR_FSTAT; + } + } + if( nMap>pFd->mmapSizeMax ){ + nMap = pFd->mmapSizeMax; + } + nMap &= ~(sqlite3_int64)(winSysInfo.dwPageSize - 1); + + if( nMap==0 && pFd->mmapSize>0 ){ + winUnmapfile(pFd); + } + if( nMap!=pFd->mmapSize ){ + void *pNew = 0; + DWORD protect = PAGE_READONLY; + DWORD flags = FILE_MAP_READ; + + winUnmapfile(pFd); + if( (pFd->ctrlFlags & WINFILE_RDONLY)==0 ){ + protect = PAGE_READWRITE; + flags |= FILE_MAP_WRITE; + } +#if SQLITE_OS_WINRT + pFd->hMap = osCreateFileMappingFromApp(pFd->h, NULL, protect, nMap, NULL); +#elif defined(SQLITE_WIN32_HAS_WIDE) + pFd->hMap = osCreateFileMappingW(pFd->h, NULL, protect, + (DWORD)((nMap>>32) & 0xffffffff), + (DWORD)(nMap & 0xffffffff), NULL); +#elif defined(SQLITE_WIN32_HAS_ANSI) + pFd->hMap = osCreateFileMappingA(pFd->h, NULL, protect, + (DWORD)((nMap>>32) & 0xffffffff), + (DWORD)(nMap & 0xffffffff), NULL); +#endif + if( pFd->hMap==NULL ){ + pFd->lastErrno = osGetLastError(); + rc = winLogError(SQLITE_IOERR_MMAP, pFd->lastErrno, + "winMapfile", pFd->zPath); + /* Log the error, but continue normal operation using xRead/xWrite */ + OSTRACE(("MAP-FILE-CREATE pid=%lu, pFile=%p, rc=SQLITE_IOERR_MMAP\n", + osGetCurrentProcessId(), pFd)); + return SQLITE_OK; + } + assert( (nMap % winSysInfo.dwPageSize)==0 ); +#if SQLITE_OS_WINRT + pNew = osMapViewOfFileFromApp(pFd->hMap, flags, 0, nMap); +#else + assert( sizeof(SIZE_T)==sizeof(sqlite3_int64) || nMap<=0xffffffff ); + pNew = osMapViewOfFile(pFd->hMap, flags, 0, 0, (SIZE_T)nMap); +#endif + if( pNew==NULL ){ + osCloseHandle(pFd->hMap); + pFd->hMap = NULL; + pFd->lastErrno = osGetLastError(); + winLogError(SQLITE_IOERR_MMAP, pFd->lastErrno, + "winMapfile", pFd->zPath); + OSTRACE(("MAP-FILE-MAP pid=%lu, pFile=%p, rc=SQLITE_IOERR_MMAP\n", + osGetCurrentProcessId(), pFd)); + return SQLITE_OK; + } + pFd->pMapRegion = pNew; + pFd->mmapSize = nMap; + pFd->mmapSizeActual = nMap; + } + + OSTRACE(("MAP-FILE pid=%lu, pFile=%p, rc=SQLITE_OK\n", + osGetCurrentProcessId(), pFd)); + return SQLITE_OK; +} +#endif /* SQLITE_MAX_MMAP_SIZE>0 */ + +/* +** If possible, return a pointer to a mapping of file fd starting at offset +** iOff. The mapping must be valid for at least nAmt bytes. +** +** If such a pointer can be obtained, store it in *pp and return SQLITE_OK. +** Or, if one cannot but no error occurs, set *pp to 0 and return SQLITE_OK. +** Finally, if an error does occur, return an SQLite error code. The final +** value of *pp is undefined in this case. +** +** If this function does return a pointer, the caller must eventually +** release the reference by calling winUnfetch(). +*/ +static int winFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ +#if SQLITE_MAX_MMAP_SIZE>0 + winFile *pFd = (winFile*)fd; /* The underlying database file */ +#endif + *pp = 0; + + OSTRACE(("FETCH pid=%lu, pFile=%p, offset=%lld, amount=%d, pp=%p\n", + osGetCurrentProcessId(), fd, iOff, nAmt, pp)); + +#if SQLITE_MAX_MMAP_SIZE>0 + if( pFd->mmapSizeMax>0 ){ + if( pFd->pMapRegion==0 ){ + int rc = winMapfile(pFd, -1); + if( rc!=SQLITE_OK ){ + OSTRACE(("FETCH pid=%lu, pFile=%p, rc=%s\n", + osGetCurrentProcessId(), pFd, sqlite3ErrName(rc))); + return rc; + } + } + if( pFd->mmapSize >= iOff+nAmt ){ + *pp = &((u8 *)pFd->pMapRegion)[iOff]; + pFd->nFetchOut++; + } + } +#endif + + OSTRACE(("FETCH pid=%lu, pFile=%p, pp=%p, *pp=%p, rc=SQLITE_OK\n", + osGetCurrentProcessId(), fd, pp, *pp)); + return SQLITE_OK; +} + +/* +** If the third argument is non-NULL, then this function releases a +** reference obtained by an earlier call to winFetch(). The second +** argument passed to this function must be the same as the corresponding +** argument that was passed to the winFetch() invocation. +** +** Or, if the third argument is NULL, then this function is being called +** to inform the VFS layer that, according to POSIX, any existing mapping +** may now be invalid and should be unmapped. +*/ +static int winUnfetch(sqlite3_file *fd, i64 iOff, void *p){ +#if SQLITE_MAX_MMAP_SIZE>0 + winFile *pFd = (winFile*)fd; /* The underlying database file */ + + /* If p==0 (unmap the entire file) then there must be no outstanding + ** xFetch references. Or, if p!=0 (meaning it is an xFetch reference), + ** then there must be at least one outstanding. */ + assert( (p==0)==(pFd->nFetchOut==0) ); + + /* If p!=0, it must match the iOff value. */ + assert( p==0 || p==&((u8 *)pFd->pMapRegion)[iOff] ); + + OSTRACE(("UNFETCH pid=%lu, pFile=%p, offset=%lld, p=%p\n", + osGetCurrentProcessId(), pFd, iOff, p)); + + if( p ){ + pFd->nFetchOut--; + }else{ + /* FIXME: If Windows truly always prevents truncating or deleting a + ** file while a mapping is held, then the following winUnmapfile() call + ** is unnecessary can can be omitted - potentially improving + ** performance. */ + winUnmapfile(pFd); + } + + assert( pFd->nFetchOut>=0 ); +#endif + + OSTRACE(("UNFETCH pid=%lu, pFile=%p, rc=SQLITE_OK\n", + osGetCurrentProcessId(), fd)); + return SQLITE_OK; +} + /* ** Here ends the implementation of all sqlite3_file methods. ** @@ -3356,7 +3815,7 @@ shmpage_out: ** sqlite3_file for win32. */ static const sqlite3_io_methods winIoMethod = { - 2, /* iVersion */ + 3, /* iVersion */ winClose, /* xClose */ winRead, /* xRead */ winWrite, /* xWrite */ @@ -3372,7 +3831,9 @@ static const sqlite3_io_methods winIoMethod = { winShmMap, /* xShmMap */ winShmLock, /* xShmLock */ winShmBarrier, /* xShmBarrier */ - winShmUnmap /* xShmUnmap */ + winShmUnmap, /* xShmUnmap */ + winFetch, /* xFetch */ + winUnfetch /* xUnfetch */ }; /**************************************************************************** @@ -3436,6 +3897,7 @@ static int getTempname(int nBuf, char *zBuf){ sqlite3_snprintf(MAX_PATH-30, zTempPath, "%s", zMulti); sqlite3_free(zMulti); }else{ + OSTRACE(("TEMP-FILENAME rc=SQLITE_IOERR_NOMEM\n")); return SQLITE_IOERR_NOMEM; } } @@ -3449,6 +3911,7 @@ static int getTempname(int nBuf, char *zBuf){ sqlite3_snprintf(MAX_PATH-30, zTempPath, "%s", zUtf8); sqlite3_free(zUtf8); }else{ + OSTRACE(("TEMP-FILENAME rc=SQLITE_IOERR_NOMEM\n")); return SQLITE_IOERR_NOMEM; } } @@ -3461,6 +3924,7 @@ static int getTempname(int nBuf, char *zBuf){ nTempPath = sqlite3Strlen30(zTempPath); if( (nTempPath + sqlite3Strlen30(SQLITE_TEMP_FILE_PREFIX) + 18) >= nBuf ){ + OSTRACE(("TEMP-FILENAME rc=SQLITE_ERROR\n")); return SQLITE_ERROR; } @@ -3478,8 +3942,8 @@ static int getTempname(int nBuf, char *zBuf){ zBuf[j] = 0; zBuf[j+1] = 0; - OSTRACE(("TEMP FILENAME: %s\n", zBuf)); - return SQLITE_OK; + OSTRACE(("TEMP-FILENAME name=%s, rc=SQLITE_OK\n", zBuf)); + return SQLITE_OK; } /* @@ -3548,9 +4012,7 @@ static int winOpen( int isExclusive = (flags & SQLITE_OPEN_EXCLUSIVE); int isDelete = (flags & SQLITE_OPEN_DELETEONCLOSE); int isCreate = (flags & SQLITE_OPEN_CREATE); -#ifndef NDEBUG int isReadonly = (flags & SQLITE_OPEN_READONLY); -#endif int isReadWrite = (flags & SQLITE_OPEN_READWRITE); #ifndef NDEBUG @@ -3561,6 +4023,9 @@ static int winOpen( )); #endif + OSTRACE(("OPEN name=%s, pFile=%p, flags=%x, pOutFlags=%p\n", + zUtf8Name, id, flags, pOutFlags)); + /* Check the following statements are true: ** ** (a) Exactly one of the READWRITE and READONLY flags must be set, and @@ -3587,8 +4052,9 @@ static int winOpen( || eType==SQLITE_OPEN_TRANSIENT_DB || eType==SQLITE_OPEN_WAL ); - assert( id!=0 ); - UNUSED_PARAMETER(pVfs); + assert( pFile!=0 ); + memset(pFile, 0, sizeof(winFile)); + pFile->h = INVALID_HANDLE_VALUE; #if SQLITE_OS_WINRT if( !sqlite3_temp_directory ){ @@ -3597,15 +4063,15 @@ static int winOpen( } #endif - pFile->h = INVALID_HANDLE_VALUE; - /* If the second argument to this function is NULL, generate a ** temporary file name to use */ if( !zUtf8Name ){ assert(isDelete && !isOpenJournal); + memset(zTmpname, 0, MAX_PATH+2); rc = getTempname(MAX_PATH+2, zTmpname); if( rc!=SQLITE_OK ){ + OSTRACE(("OPEN name=%s, rc=%s", zUtf8Name, sqlite3ErrName(rc))); return rc; } zUtf8Name = zTmpname; @@ -3621,11 +4087,13 @@ static int winOpen( /* Convert the filename to the system encoding. */ zConverted = convertUtf8Filename(zUtf8Name); if( zConverted==0 ){ + OSTRACE(("OPEN name=%s, rc=SQLITE_IOERR_NOMEM", zUtf8Name)); return SQLITE_IOERR_NOMEM; } if( winIsDir(zConverted) ){ sqlite3_free(zConverted); + OSTRACE(("OPEN name=%s, rc=SQLITE_CANTOPEN_ISDIR", zUtf8Name)); return SQLITE_CANTOPEN_ISDIR; } @@ -3716,9 +4184,8 @@ static int winOpen( #endif logIoerr(cnt); - OSTRACE(("OPEN %d %s 0x%lx %s\n", - h, zName, dwDesiredAccess, - h==INVALID_HANDLE_VALUE ? "failed" : "ok")); + OSTRACE(("OPEN file=%p, name=%s, access=%lx, rc=%s\n", h, zUtf8Name, + dwDesiredAccess, (h==INVALID_HANDLE_VALUE) ? "failed" : "ok")); if( h==INVALID_HANDLE_VALUE ){ pFile->lastErrno = lastErrno; @@ -3726,7 +4193,9 @@ static int winOpen( sqlite3_free(zConverted); if( isReadWrite && !isExclusive ){ return winOpen(pVfs, zName, id, - ((flags|SQLITE_OPEN_READONLY)&~(SQLITE_OPEN_CREATE|SQLITE_OPEN_READWRITE)), pOutFlags); + ((flags|SQLITE_OPEN_READONLY) & + ~(SQLITE_OPEN_CREATE|SQLITE_OPEN_READWRITE)), + pOutFlags); }else{ return SQLITE_CANTOPEN_BKPT; } @@ -3740,26 +4209,18 @@ static int winOpen( } } - memset(pFile, 0, sizeof(*pFile)); - pFile->pMethod = &winIoMethod; - pFile->h = h; - pFile->lastErrno = NO_ERROR; - pFile->pVfs = pVfs; -#ifndef SQLITE_OMIT_WAL - pFile->pShm = 0; -#endif - pFile->zPath = zName; - if( sqlite3_uri_boolean(zName, "psow", SQLITE_POWERSAFE_OVERWRITE) ){ - pFile->ctrlFlags |= WINFILE_PSOW; - } + OSTRACE(("OPEN file=%p, name=%s, access=%lx, pOutFlags=%p, *pOutFlags=%d, " + "rc=%s\n", h, zUtf8Name, dwDesiredAccess, pOutFlags, pOutFlags ? + *pOutFlags : 0, (h==INVALID_HANDLE_VALUE) ? "failed" : "ok")); #if SQLITE_OS_WINCE if( isReadWrite && eType==SQLITE_OPEN_MAIN_DB - && !winceCreateLock(zName, pFile) + && (rc = winceCreateLock(zName, pFile))!=SQLITE_OK ){ osCloseHandle(h); sqlite3_free(zConverted); - return SQLITE_CANTOPEN_BKPT; + OSTRACE(("OPEN-CE-LOCK name=%s, rc=%s\n", zName, sqlite3ErrName(rc))); + return rc; } if( isTemp ){ pFile->zDeleteOnClose = zConverted; @@ -3769,6 +4230,25 @@ static int winOpen( sqlite3_free(zConverted); } + pFile->pMethod = &winIoMethod; + pFile->pVfs = pVfs; + pFile->h = h; + if( isReadonly ){ + pFile->ctrlFlags |= WINFILE_RDONLY; + } + if( sqlite3_uri_boolean(zName, "psow", SQLITE_POWERSAFE_OVERWRITE) ){ + pFile->ctrlFlags |= WINFILE_PSOW; + } + pFile->lastErrno = NO_ERROR; + pFile->zPath = zName; +#if SQLITE_MAX_MMAP_SIZE>0 + pFile->hMap = NULL; + pFile->pMapRegion = 0; + pFile->mmapSize = 0; + pFile->mmapSizeActual = 0; + pFile->mmapSizeMax = sqlite3GlobalConfig.szMmap; +#endif + OpenCounter(+1); return rc; } @@ -3799,6 +4279,8 @@ static int winDelete( UNUSED_PARAMETER(syncDir); SimulateIOError(return SQLITE_IOERR_DELETE); + OSTRACE(("DELETE name=%s, syncDir=%d\n", zFilename, syncDir)); + zConverted = convertUtf8Filename(zFilename); if( zConverted==0 ){ return SQLITE_IOERR_NOMEM; @@ -3812,14 +4294,26 @@ static int winDelete( &sAttrData) ){ attr = sAttrData.dwFileAttributes; }else{ - rc = SQLITE_OK; /* Already gone? */ + lastErrno = osGetLastError(); + if( lastErrno==ERROR_FILE_NOT_FOUND + || lastErrno==ERROR_PATH_NOT_FOUND ){ + rc = SQLITE_IOERR_DELETE_NOENT; /* Already gone? */ + }else{ + rc = SQLITE_ERROR; + } break; } #else attr = osGetFileAttributesW(zConverted); #endif if ( attr==INVALID_FILE_ATTRIBUTES ){ - rc = SQLITE_OK; /* Already gone? */ + lastErrno = osGetLastError(); + if( lastErrno==ERROR_FILE_NOT_FOUND + || lastErrno==ERROR_PATH_NOT_FOUND ){ + rc = SQLITE_IOERR_DELETE_NOENT; /* Already gone? */ + }else{ + rc = SQLITE_ERROR; + } break; } if ( attr&FILE_ATTRIBUTE_DIRECTORY ){ @@ -3841,7 +4335,13 @@ static int winDelete( do { attr = osGetFileAttributesA(zConverted); if ( attr==INVALID_FILE_ATTRIBUTES ){ - rc = SQLITE_OK; /* Already gone? */ + lastErrno = osGetLastError(); + if( lastErrno==ERROR_FILE_NOT_FOUND + || lastErrno==ERROR_PATH_NOT_FOUND ){ + rc = SQLITE_IOERR_DELETE_NOENT; /* Already gone? */ + }else{ + rc = SQLITE_ERROR; + } break; } if ( attr&FILE_ATTRIBUTE_DIRECTORY ){ @@ -3859,19 +4359,19 @@ static int winDelete( } while(1); } #endif - if( rc ){ + if( rc && rc!=SQLITE_IOERR_DELETE_NOENT ){ rc = winLogError(SQLITE_IOERR_DELETE, lastErrno, "winDelete", zFilename); }else{ logIoerr(cnt); } sqlite3_free(zConverted); - OSTRACE(("DELETE \"%s\" %s\n", zFilename, (rc ? "failed" : "ok" ))); + OSTRACE(("DELETE name=%s, rc=%s\n", zFilename, sqlite3ErrName(rc))); return rc; } /* -** Check the existance and status of a file. +** Check the existence and status of a file. */ static int winAccess( sqlite3_vfs *pVfs, /* Not used on win32 */ @@ -3886,8 +4386,12 @@ static int winAccess( UNUSED_PARAMETER(pVfs); SimulateIOError( return SQLITE_IOERR_ACCESS; ); + OSTRACE(("ACCESS name=%s, flags=%x, pResOut=%p\n", + zFilename, flags, pResOut)); + zConverted = convertUtf8Filename(zFilename); if( zConverted==0 ){ + OSTRACE(("ACCESS name=%s, rc=SQLITE_IOERR_NOMEM\n", zFilename)); return SQLITE_IOERR_NOMEM; } if( isNT() ){ @@ -3938,6 +4442,8 @@ static int winAccess( assert(!"Invalid flags argument"); } *pResOut = rc; + OSTRACE(("ACCESS name=%s, pResOut=%p, *pResOut=%d, rc=SQLITE_OK\n", + zFilename, pResOut, *pResOut)); return SQLITE_OK; } @@ -4005,16 +4511,12 @@ static int winFullPathname( */ char zOut[MAX_PATH+1]; memset(zOut, 0, MAX_PATH+1); - cygwin_conv_to_win32_path(zRelative, zOut); /* POSIX to Win32 */ + cygwin_conv_path(CCP_POSIX_TO_WIN_A|CCP_RELATIVE, zRelative, zOut, + MAX_PATH+1); sqlite3_snprintf(MIN(nFull, pVfs->mxPathname), zFull, "%s\\%s", sqlite3_data_directory, zOut); }else{ - /* - ** NOTE: The Cygwin docs state that the maximum length needed - ** for the buffer passed to cygwin_conv_to_full_win32_path - ** is MAX_PATH. - */ - cygwin_conv_to_full_win32_path(zRelative, zFull); + cygwin_conv_path(CCP_POSIX_TO_WIN_A, zRelative, zFull, nFull); } return SQLITE_OK; #endif @@ -4039,7 +4541,7 @@ static int winFullPathname( #endif #if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && !defined(__CYGWIN__) - int nByte; + DWORD nByte; void *zConverted; char *zOut; @@ -4073,13 +4575,27 @@ static int winFullPathname( } if( isNT() ){ LPWSTR zTemp; - nByte = osGetFullPathNameW((LPCWSTR)zConverted, 0, 0, 0) + 3; - zTemp = sqlite3_malloc( nByte*sizeof(zTemp[0]) ); + nByte = osGetFullPathNameW((LPCWSTR)zConverted, 0, 0, 0); + if( nByte==0 ){ + winLogError(SQLITE_ERROR, osGetLastError(), + "GetFullPathNameW1", zConverted); + sqlite3_free(zConverted); + return SQLITE_CANTOPEN_FULLPATH; + } + nByte += 3; + zTemp = sqlite3MallocZero( nByte*sizeof(zTemp[0]) ); if( zTemp==0 ){ sqlite3_free(zConverted); return SQLITE_IOERR_NOMEM; } - osGetFullPathNameW((LPCWSTR)zConverted, nByte, zTemp, 0); + nByte = osGetFullPathNameW((LPCWSTR)zConverted, nByte, zTemp, 0); + if( nByte==0 ){ + winLogError(SQLITE_ERROR, osGetLastError(), + "GetFullPathNameW2", zConverted); + sqlite3_free(zConverted); + sqlite3_free(zTemp); + return SQLITE_CANTOPEN_FULLPATH; + } sqlite3_free(zConverted); zOut = unicodeToUtf8(zTemp); sqlite3_free(zTemp); @@ -4087,13 +4603,27 @@ static int winFullPathname( #ifdef SQLITE_WIN32_HAS_ANSI else{ char *zTemp; - nByte = osGetFullPathNameA((char*)zConverted, 0, 0, 0) + 3; - zTemp = sqlite3_malloc( nByte*sizeof(zTemp[0]) ); + nByte = osGetFullPathNameA((char*)zConverted, 0, 0, 0); + if( nByte==0 ){ + winLogError(SQLITE_ERROR, osGetLastError(), + "GetFullPathNameA1", zConverted); + sqlite3_free(zConverted); + return SQLITE_CANTOPEN_FULLPATH; + } + nByte += 3; + zTemp = sqlite3MallocZero( nByte*sizeof(zTemp[0]) ); if( zTemp==0 ){ sqlite3_free(zConverted); return SQLITE_IOERR_NOMEM; } - osGetFullPathNameA((char*)zConverted, nByte, zTemp, 0); + nByte = osGetFullPathNameA((char*)zConverted, nByte, zTemp, 0); + if( nByte==0 ){ + winLogError(SQLITE_ERROR, osGetLastError(), + "GetFullPathNameA2", zConverted); + sqlite3_free(zConverted); + sqlite3_free(zTemp); + return SQLITE_CANTOPEN_FULLPATH; + } sqlite3_free(zConverted); zOut = sqlite3_win32_mbcs_to_utf8(zTemp); sqlite3_free(zTemp); @@ -4144,9 +4674,9 @@ static void winDlError(sqlite3_vfs *pVfs, int nBuf, char *zBufOut){ UNUSED_PARAMETER(pVfs); getLastErrorMsg(osGetLastError(), nBuf, zBufOut); } -static void (*winDlSym(sqlite3_vfs *pVfs, void *pHandle, const char *zSymbol))(void){ +static void (*winDlSym(sqlite3_vfs *pVfs,void *pH,const char *zSym))(void){ UNUSED_PARAMETER(pVfs); - return (void(*)(void))osGetProcAddressA((HANDLE)pHandle, zSymbol); + return (void(*)(void))osGetProcAddressA((HANDLE)pH, zSym); } static void winDlClose(sqlite3_vfs *pVfs, void *pHandle){ UNUSED_PARAMETER(pVfs); @@ -4244,7 +4774,8 @@ static int winCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *piNow){ #endif /* 2^32 - to avoid use of LL and warnings in gcc */ static const sqlite3_int64 max32BitValue = - (sqlite3_int64)2000000000 + (sqlite3_int64)2000000000 + (sqlite3_int64)294967296; + (sqlite3_int64)2000000000 + (sqlite3_int64)2000000000 + + (sqlite3_int64)294967296; #if SQLITE_OS_WINCE SYSTEMTIME time; @@ -4351,9 +4882,8 @@ int sqlite3_os_init(void){ /* Double-check that the aSyscall[] array has been constructed ** correctly. See ticket [bb3a86e890c8e96ab] */ - assert( ArraySize(aSyscall)==73 ); + assert( ArraySize(aSyscall)==74 ); -#ifndef SQLITE_OMIT_WAL /* get memory map allocation granularity */ memset(&winSysInfo, 0, sizeof(SYSTEM_INFO)); #if SQLITE_OS_WINRT @@ -4361,8 +4891,8 @@ int sqlite3_os_init(void){ #else osGetSystemInfo(&winSysInfo); #endif - assert(winSysInfo.dwAllocationGranularity > 0); -#endif + assert( winSysInfo.dwAllocationGranularity>0 ); + assert( winSysInfo.dwPageSize>0 ); sqlite3_vfs_register(&winVfs, 1); return SQLITE_OK; @@ -4370,7 +4900,7 @@ int sqlite3_os_init(void){ int sqlite3_os_end(void){ #if SQLITE_OS_WINRT - if( sleepObj != NULL ){ + if( sleepObj!=NULL ){ osCloseHandle(sleepObj); sleepObj = NULL; } diff --git a/src/pager.c b/src/pager.c index faf0223..61727fa 100644 --- a/src/pager.c +++ b/src/pager.c @@ -273,7 +273,7 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** * A write transaction is active. ** * An EXCLUSIVE or greater lock is held on the database file. ** * All writing and syncing of journal and database data has finished. -** If no error occured, all that remains is to finalize the journal to +** If no error occurred, all that remains is to finalize the journal to ** commit the transaction. If an error did occur, the caller will need ** to rollback the transaction. ** @@ -521,7 +521,7 @@ struct PagerSavepoint { ** ** doNotSpill, doNotSyncSpill ** -** These two boolean variables control the behaviour of cache-spills +** These two boolean variables control the behavior of cache-spills ** (calls made by the pcache module to the pagerStress() routine to ** write cached data to the file-system in order to free up memory). ** @@ -655,6 +655,11 @@ struct Pager { PagerSavepoint *aSavepoint; /* Array of active savepoints */ int nSavepoint; /* Number of elements in aSavepoint[] */ char dbFileVers[16]; /* Changes whenever database file changes */ + + u8 bUseFetch; /* True to use xFetch() */ + int nMmapOut; /* Number of mmap pages currently outstanding */ + sqlite3_int64 szMmap; /* Desired maximum mmap size */ + PgHdr *pMmapFreelist; /* List of free mmap page headers (pDirty) */ /* ** End of the routinely-changing class members ***************************************************************************/ @@ -765,6 +770,16 @@ static const unsigned char aJournalMagic[] = { # define MEMDB pPager->memDb #endif +/* +** The macro USEFETCH is true if we are allowed to use the xFetch and xUnfetch +** interfaces to access the database using memory-mapped I/O. +*/ +#if SQLITE_MAX_MMAP_SIZE>0 +# define USEFETCH(x) ((x)->bUseFetch) +#else +# define USEFETCH(x) 0 +#endif + /* ** The maximum legal page number is (2^31 - 1). */ @@ -1399,7 +1414,7 @@ static int writeJournalHdr(Pager *pPager){ memset(zHeader, 0, sizeof(aJournalMagic)+4); } - /* The random check-hash initialiser */ + /* The random check-hash initializer */ sqlite3_randomness(sizeof(pPager->cksumInit), &pPager->cksumInit); put32bits(&zHeader[sizeof(aJournalMagic)+4], pPager->cksumInit); /* The initial database size */ @@ -1838,6 +1853,8 @@ static int pager_error(Pager *pPager, int rc){ return rc; } +static int pager_truncate(Pager *pPager, Pgno nPage); + /* ** This routine ends a transaction. A transaction is usually ended by ** either a COMMIT or a ROLLBACK operation. This routine may be called @@ -1891,7 +1908,7 @@ static int pager_error(Pager *pPager, int rc){ ** to the first error encountered (the journal finalization one) is ** returned. */ -static int pager_end_transaction(Pager *pPager, int hasMaster){ +static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){ int rc = SQLITE_OK; /* Error code from journal finalization operation */ int rc2 = SQLITE_OK; /* Error code from db file unlock operation */ @@ -1941,12 +1958,13 @@ static int pager_end_transaction(Pager *pPager, int hasMaster){ ** file should be closed and deleted. If this connection writes to ** the database file, it will do so using an in-memory journal. */ + int bDelete = (!pPager->tempFile && sqlite3JournalExists(pPager->jfd)); assert( pPager->journalMode==PAGER_JOURNALMODE_DELETE || pPager->journalMode==PAGER_JOURNALMODE_MEMORY || pPager->journalMode==PAGER_JOURNALMODE_WAL ); sqlite3OsClose(pPager->jfd); - if( !pPager->tempFile ){ + if( bDelete ){ rc = sqlite3OsDelete(pPager->pVfs, pPager->zJournal, 0); } } @@ -1976,7 +1994,17 @@ static int pager_end_transaction(Pager *pPager, int hasMaster){ */ rc2 = sqlite3WalEndWriteTransaction(pPager->pWal); assert( rc2==SQLITE_OK ); + }else if( rc==SQLITE_OK && bCommit && pPager->dbFileSize>pPager->dbSize ){ + /* This branch is taken when committing a transaction in rollback-journal + ** mode if the database file on disk is larger than the database image. + ** At this point the journal has been finalized and the transaction + ** successfully committed, but the EXCLUSIVE lock is still held on the + ** file. So it is safe to truncate the database file to its minimum + ** required size. */ + assert( pPager->eLock==EXCLUSIVE_LOCK ); + rc = pager_truncate(pPager, pPager->dbSize); } + if( !pPager->exclusiveMode && (!pagerUseWal(pPager) || sqlite3WalExclusiveMode(pPager->pWal, 0)) ){ @@ -2015,7 +2043,7 @@ static void pagerUnlockAndRollback(Pager *pPager){ sqlite3EndBenignMalloc(); }else if( !pPager->exclusiveMode ){ assert( pPager->eState==PAGER_READER ); - pager_end_transaction(pPager, 0); + pager_end_transaction(pPager, 0, 0); } } pager_unlock(pPager); @@ -2239,7 +2267,7 @@ static int pager_playback_one_page( i64 ofst = (pgno-1)*(i64)pPager->pageSize; testcase( !isSavepnt && pPg!=0 && (pPg->flags&PGHDR_NEED_SYNC)!=0 ); assert( !pagerUseWal(pPager) ); - rc = sqlite3OsWrite(pPager->fd, (u8*)aData, pPager->pageSize, ofst); + rc = sqlite3OsWrite(pPager->fd, (u8 *)aData, pPager->pageSize, ofst); if( pgno>pPager->dbFileSize ){ pPager->dbFileSize = pgno; } @@ -2509,6 +2537,21 @@ static int pager_truncate(Pager *pPager, Pgno nPage){ return rc; } +/* +** Return a sanitized version of the sector-size of OS file pFile. The +** return value is guaranteed to lie between 32 and MAX_SECTOR_SIZE. +*/ +int sqlite3SectorSize(sqlite3_file *pFile){ + int iRet = sqlite3OsSectorSize(pFile); + if( iRet<32 ){ + iRet = 512; + }else if( iRet>MAX_SECTOR_SIZE ){ + assert( MAX_SECTOR_SIZE>=512 ); + iRet = MAX_SECTOR_SIZE; + } + return iRet; +} + /* ** Set the value of the Pager.sectorSize variable for the given ** pager based on the value returned by the xSectorSize method @@ -2544,14 +2587,7 @@ static void setSectorSize(Pager *pPager){ ** call will segfault. */ pPager->sectorSize = 512; }else{ - pPager->sectorSize = sqlite3OsSectorSize(pPager->fd); - if( pPager->sectorSize<32 ){ - pPager->sectorSize = 512; - } - if( pPager->sectorSize>MAX_SECTOR_SIZE ){ - assert( MAX_SECTOR_SIZE>=512 ); - pPager->sectorSize = MAX_SECTOR_SIZE; - } + pPager->sectorSize = sqlite3SectorSize(pPager->fd); } } @@ -2622,6 +2658,7 @@ static int pager_playback(Pager *pPager, int isHot){ int res = 1; /* Value returned by sqlite3OsAccess() */ char *zMaster = 0; /* Name of master journal file if any */ int needPagerReset; /* True to reset page prior to first page rollback */ + int nPlayback = 0; /* Total number of pages restored from journal */ /* Figure out how many records are in the journal. Abort early if ** the journal is empty. @@ -2722,7 +2759,9 @@ static int pager_playback(Pager *pPager, int isHot){ needPagerReset = 0; } rc = pager_playback_one_page(pPager,&pPager->journalOff,0,1,0); - if( rc!=SQLITE_OK ){ + if( rc==SQLITE_OK ){ + nPlayback++; + }else{ if( rc==SQLITE_DONE ){ pPager->journalOff = szJ; break; @@ -2782,7 +2821,7 @@ end_playback: rc = sqlite3PagerSync(pPager); } if( rc==SQLITE_OK ){ - rc = pager_end_transaction(pPager, zMaster[0]!='\0'); + rc = pager_end_transaction(pPager, zMaster[0]!='\0', 0); testcase( rc!=SQLITE_OK ); } if( rc==SQLITE_OK && zMaster[0] && res ){ @@ -2792,6 +2831,10 @@ end_playback: rc = pager_delmaster(pPager, zMaster); testcase( rc!=SQLITE_OK ); } + if( isHot && nPlayback ){ + sqlite3_log(SQLITE_NOTICE_RECOVER_ROLLBACK, "recovered %d pages from %s", + nPlayback, pPager->zJournal); + } /* The Pager.sectorSize variable may have been updated while rolling ** back a journal created by a process with a different sector size @@ -2813,11 +2856,10 @@ end_playback: ** If an IO error occurs, then the IO error is returned to the caller. ** Otherwise, SQLITE_OK is returned. */ -static int readDbPage(PgHdr *pPg){ +static int readDbPage(PgHdr *pPg, u32 iFrame){ Pager *pPager = pPg->pPager; /* Pager object associated with page pPg */ Pgno pgno = pPg->pgno; /* Page number to read */ int rc = SQLITE_OK; /* Return code */ - int isInWal = 0; /* True if page is in log file */ int pgsz = pPager->pageSize; /* Number of bytes to read */ assert( pPager->eState>=PAGER_READER && !MEMDB ); @@ -2829,11 +2871,13 @@ static int readDbPage(PgHdr *pPg){ return SQLITE_OK; } - if( pagerUseWal(pPager) ){ +#ifndef SQLITE_OMIT_WAL + if( iFrame ){ /* Try to pull the page from the write-ahead log. */ - rc = sqlite3WalRead(pPager->pWal, pgno, &isInWal, pgsz, pPg->pData); - } - if( rc==SQLITE_OK && !isInWal ){ + rc = sqlite3WalReadFrame(pPager->pWal, iFrame, pgsz, pPg->pData); + }else +#endif + { i64 iOffset = (pgno-1)*(i64)pPager->pageSize; rc = sqlite3OsRead(pPager->fd, pPg->pData, pgsz, iOffset); if( rc==SQLITE_IOERR_SHORT_READ ){ @@ -2912,12 +2956,17 @@ static int pagerUndoCallback(void *pCtx, Pgno iPg){ Pager *pPager = (Pager *)pCtx; PgHdr *pPg; + assert( pagerUseWal(pPager) ); pPg = sqlite3PagerLookup(pPager, iPg); if( pPg ){ if( sqlite3PcachePageRefcount(pPg)==1 ){ sqlite3PcacheDrop(pPg); }else{ - rc = readDbPage(pPg); + u32 iFrame = 0; + rc = sqlite3WalFindFrame(pPager->pWal, pPg->pgno, &iFrame); + if( rc==SQLITE_OK ){ + rc = readDbPage(pPg, iFrame); + } if( rc==SQLITE_OK ){ pPager->xReiniter(pPg); } @@ -3061,6 +3110,7 @@ static int pagerBeginReadTransaction(Pager *pPager){ rc = sqlite3WalBeginReadTransaction(pPager->pWal, &changed); if( rc!=SQLITE_OK || changed ){ pager_reset(pPager); + if( USEFETCH(pPager) ) sqlite3OsUnfetch(pPager->fd, 0, 0); } return rc; @@ -3151,6 +3201,7 @@ static int pagerOpenWalIfPresent(Pager *pPager){ if( rc ) return rc; if( nPage==0 ){ rc = sqlite3OsDelete(pPager->pVfs, pPager->zWal, 0); + if( rc==SQLITE_IOERR_DELETE_NOENT ) rc = SQLITE_OK; isWal = 0; }else{ rc = sqlite3OsAccess( @@ -3321,6 +3372,29 @@ void sqlite3PagerSetCachesize(Pager *pPager, int mxPage){ sqlite3PcacheSetCachesize(pPager->pPCache, mxPage); } +/* +** Invoke SQLITE_FCNTL_MMAP_SIZE based on the current value of szMmap. +*/ +static void pagerFixMaplimit(Pager *pPager){ +#if SQLITE_MAX_MMAP_SIZE>0 + sqlite3_file *fd = pPager->fd; + if( isOpen(fd) ){ + sqlite3_int64 sz; + pPager->bUseFetch = (fd->pMethods->iVersion>=3) && pPager->szMmap>0; + sz = pPager->szMmap; + sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_MMAP_SIZE, &sz); + } +#endif +} + +/* +** Change the maximum size of any memory mapping made of the database file. +*/ +void sqlite3PagerSetMmapLimit(Pager *pPager, sqlite3_int64 szMmap){ + pPager->szMmap = szMmap; + pagerFixMaplimit(pPager); +} + /* ** Free as much memory as possible from the pager. */ @@ -3468,9 +3542,16 @@ void sqlite3PagerSetBusyhandler( Pager *pPager, /* Pager object */ int (*xBusyHandler)(void *), /* Pointer to busy-handler function */ void *pBusyHandlerArg /* Argument to pass to xBusyHandler */ -){ +){ pPager->xBusyHandler = xBusyHandler; pPager->pBusyHandlerArg = pBusyHandlerArg; + + if( isOpen(pPager->fd) ){ + void **ap = (void **)&pPager->xBusyHandler; + assert( ((int(*)(void *))(ap[0]))==xBusyHandler ); + assert( ap[1]==pBusyHandlerArg ); + sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_BUSYHANDLER, (void *)ap); + } } /* @@ -3549,6 +3630,7 @@ int sqlite3PagerSetPagesize(Pager *pPager, u32 *pPageSize, int nReserve){ assert( nReserve>=0 && nReserve<1000 ); pPager->nReserve = (i16)nReserve; pagerReportSize(pPager); + pagerFixMaplimit(pPager); } return rc; } @@ -3702,7 +3784,7 @@ static int pager_wait_on_lock(Pager *pPager, int locktype){ ** dirty page were to be discarded from the cache via the pagerStress() ** routine, pagerStress() would not write the current page content to ** the database file. If a savepoint transaction were rolled back after -** this happened, the correct behaviour would be to restore the current +** this happened, the correct behavior would be to restore the current ** content of the page. However, since this content is not present in either ** the database file or the portion of the rollback journal and ** sub-journal rolled back the content could not be restored and the @@ -3726,12 +3808,26 @@ static void assertTruncateConstraint(Pager *pPager){ ** function does not actually modify the database file on disk. It ** just sets the internal state of the pager object so that the ** truncation will be done when the current transaction is committed. +** +** This function is only called right before committing a transaction. +** Once this function has been called, the transaction must either be +** rolled back or committed. It is not safe to call this function and +** then continue writing to the database. */ void sqlite3PagerTruncateImage(Pager *pPager, Pgno nPage){ assert( pPager->dbSize>=nPage ); assert( pPager->eState>=PAGER_WRITER_CACHEMOD ); pPager->dbSize = nPage; - assertTruncateConstraint(pPager); + + /* At one point the code here called assertTruncateConstraint() to + ** ensure that all pages being truncated away by this operation are, + ** if one or more savepoints are open, present in the savepoint + ** journal so that they can be restored if the savepoint is rolled + ** back. This is no longer necessary as this function is now only + ** called right before committing a transaction. So although the + ** Pager object may still have open savepoints (Pager.nSavepoint!=0), + ** they cannot be rolled back. So the assertTruncateConstraint() call + ** is no longer correct. */ } @@ -3760,6 +3856,81 @@ static int pagerSyncHotJournal(Pager *pPager){ return rc; } +/* +** Obtain a reference to a memory mapped page object for page number pgno. +** The new object will use the pointer pData, obtained from xFetch(). +** If successful, set *ppPage to point to the new page reference +** and return SQLITE_OK. Otherwise, return an SQLite error code and set +** *ppPage to zero. +** +** Page references obtained by calling this function should be released +** by calling pagerReleaseMapPage(). +*/ +static int pagerAcquireMapPage( + Pager *pPager, /* Pager object */ + Pgno pgno, /* Page number */ + void *pData, /* xFetch()'d data for this page */ + PgHdr **ppPage /* OUT: Acquired page object */ +){ + PgHdr *p; /* Memory mapped page to return */ + + if( pPager->pMmapFreelist ){ + *ppPage = p = pPager->pMmapFreelist; + pPager->pMmapFreelist = p->pDirty; + p->pDirty = 0; + memset(p->pExtra, 0, pPager->nExtra); + }else{ + *ppPage = p = (PgHdr *)sqlite3MallocZero(sizeof(PgHdr) + pPager->nExtra); + if( p==0 ){ + sqlite3OsUnfetch(pPager->fd, (i64)(pgno-1) * pPager->pageSize, pData); + return SQLITE_NOMEM; + } + p->pExtra = (void *)&p[1]; + p->flags = PGHDR_MMAP; + p->nRef = 1; + p->pPager = pPager; + } + + assert( p->pExtra==(void *)&p[1] ); + assert( p->pPage==0 ); + assert( p->flags==PGHDR_MMAP ); + assert( p->pPager==pPager ); + assert( p->nRef==1 ); + + p->pgno = pgno; + p->pData = pData; + pPager->nMmapOut++; + + return SQLITE_OK; +} + +/* +** Release a reference to page pPg. pPg must have been returned by an +** earlier call to pagerAcquireMapPage(). +*/ +static void pagerReleaseMapPage(PgHdr *pPg){ + Pager *pPager = pPg->pPager; + pPager->nMmapOut--; + pPg->pDirty = pPager->pMmapFreelist; + pPager->pMmapFreelist = pPg; + + assert( pPager->fd->pMethods->iVersion>=3 ); + sqlite3OsUnfetch(pPager->fd, (i64)(pPg->pgno-1)*pPager->pageSize, pPg->pData); +} + +/* +** Free all PgHdr objects stored in the Pager.pMmapFreelist list. +*/ +static void pagerFreeMapHdrs(Pager *pPager){ + PgHdr *p; + PgHdr *pNext; + for(p=pPager->pMmapFreelist; p; p=pNext){ + pNext = p->pDirty; + sqlite3_free(p); + } +} + + /* ** Shutdown the page cache. Free all memory and close all files. ** @@ -3780,6 +3951,7 @@ int sqlite3PagerClose(Pager *pPager){ assert( assert_pager_state(pPager) ); disable_simulated_io_errors(); sqlite3BeginBenignMalloc(); + pagerFreeMapHdrs(pPager); /* pPager->errCode = 0; */ pPager->exclusiveMode = 0; #ifndef SQLITE_OMIT_WAL @@ -4041,7 +4213,9 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ ** file size will be. */ assert( rc!=SQLITE_OK || isOpen(pPager->fd) ); - if( rc==SQLITE_OK && pPager->dbSize>pPager->dbHintSize ){ + if( rc==SQLITE_OK + && (pList->pDirty ? pPager->dbSize : pList->pgno+1)>pPager->dbHintSize + ){ sqlite3_int64 szFile = pPager->pageSize * (sqlite3_int64)pPager->dbSize; sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_SIZE_HINT, &szFile); pPager->dbHintSize = pPager->dbSize; @@ -4448,7 +4622,7 @@ int sqlite3PagerOpen( memcpy(pPager->zFilename, zPathname, nPathname); if( nUri ) memcpy(&pPager->zFilename[nPathname+1], zUri, nUri); memcpy(pPager->zJournal, zPathname, nPathname); - memcpy(&pPager->zJournal[nPathname], "-journal\000", 8+1); + memcpy(&pPager->zJournal[nPathname], "-journal\000", 8+2); sqlite3FileSuffix3(pPager->zFilename, pPager->zJournal); #ifndef SQLITE_OMIT_WAL pPager->zWal = &pPager->zJournal[nPathname+8+1]; @@ -4595,6 +4769,7 @@ int sqlite3PagerOpen( /* pPager->pBusyHandlerArg = 0; */ pPager->xReiniter = xReinit; /* memset(pPager->aHash, 0, sizeof(pPager->aHash)); */ + /* pPager->szMmap = SQLITE_DEFAULT_MMAP_SIZE // will be set by btree.c */ *ppPager = pPager; return SQLITE_OK; @@ -4784,6 +4959,11 @@ int sqlite3PagerSharedLock(Pager *pPager){ goto failed; } if( bHotJournal ){ + if( pPager->readOnly ){ + rc = SQLITE_READONLY_ROLLBACK; + goto failed; + } + /* Get an EXCLUSIVE lock on the database file. At this point it is ** important that a RESERVED lock is not obtained on the way to the ** EXCLUSIVE lock. If it were, another process might open the @@ -4881,9 +5061,11 @@ int sqlite3PagerSharedLock(Pager *pPager){ ); } - if( !pPager->tempFile - && (pPager->pBackup || sqlite3PcachePagecount(pPager->pPCache)>0) - ){ + if( !pPager->tempFile && ( + pPager->pBackup + || sqlite3PcachePagecount(pPager->pPCache)>0 + || USEFETCH(pPager) + )){ /* The shared-lock has just been acquired on the database file ** and there are already pages in the cache (from a previous ** read or write transaction). Check to see if the database @@ -4909,7 +5091,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ if( nPage>0 ){ IOTRACE(("CKVERS %p %d\n", pPager, sizeof(dbFileVers))); rc = sqlite3OsRead(pPager->fd, &dbFileVers, sizeof(dbFileVers), 24); - if( rc!=SQLITE_OK ){ + if( rc!=SQLITE_OK && rc!=SQLITE_IOERR_SHORT_READ ){ goto failed; } }else{ @@ -4918,6 +5100,16 @@ int sqlite3PagerSharedLock(Pager *pPager){ if( memcmp(pPager->dbFileVers, dbFileVers, sizeof(dbFileVers))!=0 ){ pager_reset(pPager); + + /* Unmap the database file. It is possible that external processes + ** may have truncated the database file and then extended it back + ** to its original size while this process was not holding a lock. + ** In this case there may exist a Pager.pMap mapping that appears + ** to be the right size but is not actually valid. Avoid this + ** possibility by unmapping the db here. */ + if( USEFETCH(pPager) ){ + sqlite3OsUnfetch(pPager->fd, 0, 0); + } } } @@ -4959,7 +5151,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ ** nothing to rollback, so this routine is a no-op. */ static void pagerUnlockIfUnused(Pager *pPager){ - if( (sqlite3PcacheRefCount(pPager->pPCache)==0) ){ + if( pPager->nMmapOut==0 && (sqlite3PcacheRefCount(pPager->pPCache)==0) ){ pagerUnlockAndRollback(pPager); } } @@ -5018,13 +5210,27 @@ int sqlite3PagerAcquire( Pager *pPager, /* The pager open on the database file */ Pgno pgno, /* Page number to fetch */ DbPage **ppPage, /* Write a pointer to the page here */ - int noContent /* Do not bother reading content from disk if true */ + int flags /* PAGER_ACQUIRE_XXX flags */ ){ - int rc; - PgHdr *pPg; + int rc = SQLITE_OK; + PgHdr *pPg = 0; + u32 iFrame = 0; /* Frame to read from WAL file */ + const int noContent = (flags & PAGER_ACQUIRE_NOCONTENT); + + /* It is acceptable to use a read-only (mmap) page for any page except + ** page 1 if there is no write-transaction open or the ACQUIRE_READONLY + ** flag was specified by the caller. And so long as the db is not a + ** temporary or in-memory database. */ + const int bMmapOk = (pgno!=1 && USEFETCH(pPager) + && (pPager->eState==PAGER_READER || (flags & PAGER_ACQUIRE_READONLY)) +#ifdef SQLITE_HAS_CODEC + && pPager->xCodec==0 +#endif + ); assert( pPager->eState>=PAGER_READER ); assert( assert_pager_state(pPager) ); + assert( noContent==0 || bMmapOk==0 ); if( pgno==0 ){ return SQLITE_CORRUPT_BKPT; @@ -5035,6 +5241,39 @@ int sqlite3PagerAcquire( if( pPager->errCode!=SQLITE_OK ){ rc = pPager->errCode; }else{ + + if( bMmapOk && pagerUseWal(pPager) ){ + rc = sqlite3WalFindFrame(pPager->pWal, pgno, &iFrame); + if( rc!=SQLITE_OK ) goto pager_acquire_err; + } + + if( iFrame==0 && bMmapOk ){ + void *pData = 0; + + rc = sqlite3OsFetch(pPager->fd, + (i64)(pgno-1) * pPager->pageSize, pPager->pageSize, &pData + ); + + if( rc==SQLITE_OK && pData ){ + if( pPager->eState>PAGER_READER ){ + (void)sqlite3PcacheFetch(pPager->pPCache, pgno, 0, &pPg); + } + if( pPg==0 ){ + rc = pagerAcquireMapPage(pPager, pgno, pData, &pPg); + }else{ + sqlite3OsUnfetch(pPager->fd, (i64)(pgno-1)*pPager->pageSize, pData); + } + if( pPg ){ + assert( rc==SQLITE_OK ); + *ppPage = pPg; + return SQLITE_OK; + } + } + if( rc!=SQLITE_OK ){ + goto pager_acquire_err; + } + } + rc = sqlite3PcacheFetch(pPager->pPCache, pgno, 1, ppPage); } @@ -5093,9 +5332,13 @@ int sqlite3PagerAcquire( memset(pPg->pData, 0, pPager->pageSize); IOTRACE(("ZERO %p %d\n", pPager, pgno)); }else{ + if( pagerUseWal(pPager) && bMmapOk==0 ){ + rc = sqlite3WalFindFrame(pPager->pWal, pgno, &iFrame); + if( rc!=SQLITE_OK ) goto pager_acquire_err; + } assert( pPg->pPager==pPager ); pPager->aStat[PAGER_STAT_MISS]++; - rc = readDbPage(pPg); + rc = readDbPage(pPg, iFrame); if( rc!=SQLITE_OK ){ goto pager_acquire_err; } @@ -5148,7 +5391,11 @@ DbPage *sqlite3PagerLookup(Pager *pPager, Pgno pgno){ void sqlite3PagerUnref(DbPage *pPg){ if( pPg ){ Pager *pPager = pPg->pPager; - sqlite3PcacheRelease(pPg); + if( pPg->flags & PGHDR_MMAP ){ + pagerReleaseMapPage(pPg); + }else{ + sqlite3PcacheRelease(pPg); + } pagerUnlockIfUnused(pPager); } } @@ -5483,6 +5730,7 @@ int sqlite3PagerWrite(DbPage *pDbPage){ Pager *pPager = pPg->pPager; Pgno nPagePerSector = (pPager->sectorSize/pPager->pageSize); + assert( (pPg->flags & PGHDR_MMAP)==0 ); assert( pPager->eState>=PAGER_WRITER_LOCKED ); assert( pPager->eState!=PAGER_ERROR ); assert( assert_pager_state(pPager) ); @@ -5650,7 +5898,7 @@ static int pager_incr_changecounter(Pager *pPager, int isDirectMode){ # define DIRECT_MODE isDirectMode #endif - if( !pPager->changeCountDone && pPager->dbSize>0 ){ + if( !pPager->changeCountDone && ALWAYS(pPager->dbSize>0) ){ PgHdr *pPgHdr; /* Reference to page 1 */ assert( !pPager->tempFile && isOpen(pPager->fd) ); @@ -5682,6 +5930,11 @@ static int pager_incr_changecounter(Pager *pPager, int isDirectMode){ pPager->aStat[PAGER_STAT_WRITE]++; } if( rc==SQLITE_OK ){ + /* Update the pager's copy of the change-counter. Otherwise, the + ** next time a read transaction is opened the cache will be + ** flushed (as the change-counter values will not match). */ + const void *pCopy = (const void *)&((const char *)zBuf)[24]; + memcpy(&pPager->dbFileVers, pCopy, sizeof(pPager->dbFileVers)); pPager->changeCountDone = 1; } }else{ @@ -5868,38 +6121,6 @@ int sqlite3PagerCommitPhaseOne( #endif if( rc!=SQLITE_OK ) goto commit_phase_one_exit; - /* If this transaction has made the database smaller, then all pages - ** being discarded by the truncation must be written to the journal - ** file. This can only happen in auto-vacuum mode. - ** - ** Before reading the pages with page numbers larger than the - ** current value of Pager.dbSize, set dbSize back to the value - ** that it took at the start of the transaction. Otherwise, the - ** calls to sqlite3PagerGet() return zeroed pages instead of - ** reading data from the database file. - */ - #ifndef SQLITE_OMIT_AUTOVACUUM - if( pPager->dbSizedbOrigSize - && pPager->journalMode!=PAGER_JOURNALMODE_OFF - ){ - Pgno i; /* Iterator variable */ - const Pgno iSkip = PAGER_MJ_PGNO(pPager); /* Pending lock page */ - const Pgno dbSize = pPager->dbSize; /* Database image size */ - pPager->dbSize = pPager->dbOrigSize; - for( i=dbSize+1; i<=pPager->dbOrigSize; i++ ){ - if( !sqlite3BitvecTest(pPager->pInJournal, i) && i!=iSkip ){ - PgHdr *pPage; /* Page to journal */ - rc = sqlite3PagerGet(pPager, i, &pPage); - if( rc!=SQLITE_OK ) goto commit_phase_one_exit; - rc = sqlite3PagerWrite(pPage); - sqlite3PagerUnref(pPage); - if( rc!=SQLITE_OK ) goto commit_phase_one_exit; - } - } - pPager->dbSize = dbSize; - } - #endif - /* Write the master journal name into the journal file. If a master ** journal file name has already been written to the journal file, ** or if zMaster is NULL (no master journal), then this call is a no-op. @@ -5927,11 +6148,14 @@ int sqlite3PagerCommitPhaseOne( goto commit_phase_one_exit; } sqlite3PcacheCleanAll(pPager->pPCache); - - /* If the file on disk is not the same size as the database image, - ** then use pager_truncate to grow or shrink the file here. - */ - if( pPager->dbSize!=pPager->dbFileSize ){ + + /* If the file on disk is smaller than the database image, use + ** pager_truncate to grow the file here. This can happen if the database + ** image was extended as part of the current transaction and then the + ** last page in the db image moved to the free-list. In this case the + ** last page is never written out to disk, leaving the database file + ** undersized. Fix this now if it is the case. */ + if( pPager->dbSize>pPager->dbFileSize ){ Pgno nNew = pPager->dbSize - (pPager->dbSize==PAGER_MJ_PGNO(pPager)); assert( pPager->eState==PAGER_WRITER_DBMOD ); rc = pager_truncate(pPager, nNew); @@ -6004,7 +6228,7 @@ int sqlite3PagerCommitPhaseTwo(Pager *pPager){ } PAGERTRACE(("COMMIT %d\n", PAGERID(pPager))); - rc = pager_end_transaction(pPager, pPager->setMaster); + rc = pager_end_transaction(pPager, pPager->setMaster, 1); return pager_error(pPager, rc); } @@ -6049,11 +6273,11 @@ int sqlite3PagerRollback(Pager *pPager){ if( pagerUseWal(pPager) ){ int rc2; rc = sqlite3PagerSavepoint(pPager, SAVEPOINT_ROLLBACK, -1); - rc2 = pager_end_transaction(pPager, pPager->setMaster); + rc2 = pager_end_transaction(pPager, pPager->setMaster, 0); if( rc==SQLITE_OK ) rc = rc2; }else if( !isOpen(pPager->jfd) || pPager->eState==PAGER_WRITER_LOCKED ){ int eState = pPager->eState; - rc = pager_end_transaction(pPager, 0); + rc = pager_end_transaction(pPager, 0, 0); if( !MEMDB && eState>PAGER_WRITER_LOCKED ){ /* This can happen using journal_mode=off. Move the pager to the error ** state to indicate that the contents of the cache may not be trusted. @@ -6068,7 +6292,7 @@ int sqlite3PagerRollback(Pager *pPager){ } assert( pPager->eState==PAGER_READER || rc!=SQLITE_OK ); - assert( rc==SQLITE_OK || rc==SQLITE_FULL + assert( rc==SQLITE_OK || rc==SQLITE_FULL || rc==SQLITE_CORRUPT || rc==SQLITE_NOMEM || (rc&0xFF)==SQLITE_IOERR ); /* If an error occurs during a ROLLBACK, we can no longer trust the pager @@ -6451,7 +6675,8 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ */ if( (pPg->flags&PGHDR_NEED_SYNC) && !isCommit ){ needSyncPgno = pPg->pgno; - assert( pageInJournal(pPg) || pPg->pgno>pPager->dbOrigSize ); + assert( pPager->journalMode==PAGER_JOURNALMODE_OFF || + pageInJournal(pPg) || pPg->pgno>pPager->dbOrigSize ); assert( pPg->flags&PGHDR_DIRTY ); } @@ -6801,11 +7026,12 @@ static int pagerOpenWal(Pager *pPager){ ** (e.g. due to malloc() failure), return an error code. */ if( rc==SQLITE_OK ){ - rc = sqlite3WalOpen(pPager->pVfs, + rc = sqlite3WalOpen(pPager->pVfs, pPager->fd, pPager->zWal, pPager->exclusiveMode, pPager->journalSizeLimit, &pPager->pWal ); } + pagerFixMaplimit(pPager); return rc; } @@ -6896,11 +7122,14 @@ int sqlite3PagerCloseWal(Pager *pPager){ rc = sqlite3WalClose(pPager->pWal, pPager->ckptSyncFlags, pPager->pageSize, (u8*)pPager->pTmpSpace); pPager->pWal = 0; + pagerFixMaplimit(pPager); } } return rc; } +#endif /* !SQLITE_OMIT_WAL */ + #ifdef SQLITE_ENABLE_ZIPVFS /* ** A read-lock must be held on the pager when this function is called. If @@ -6930,11 +7159,9 @@ void *sqlite3PagerCodec(PgHdr *pPg){ } #endif /* SQLITE_HAS_CODEC */ -#endif /* !SQLITE_OMIT_WAL */ - #endif /* SQLITE_OMIT_DISKIO */ -/* BEGIN CRYPTO */ +/* BEGIN SQLCIPHER */ #ifdef SQLITE_HAS_CODEC void sqlite3pager_get_codec(Pager *pPager, void **ctx) { *ctx = pPager->pCodec; @@ -6963,5 +7190,5 @@ void sqlite3pager_sqlite3PagerSetError( Pager *pPager, int error) { } #endif -/* END CRYPTO */ +/* END SQLCIPHER */ diff --git a/src/pager.h b/src/pager.h index 2b60e05..6f65913 100644 --- a/src/pager.h +++ b/src/pager.h @@ -78,6 +78,12 @@ typedef struct PgHdr DbPage; #define PAGER_JOURNALMODE_MEMORY 4 /* In-memory journal file */ #define PAGER_JOURNALMODE_WAL 5 /* Use write-ahead logging */ +/* +** Flags that make up the mask passed to sqlite3PagerAcquire(). +*/ +#define PAGER_ACQUIRE_NOCONTENT 0x01 /* Do not load data from disk */ +#define PAGER_ACQUIRE_READONLY 0x02 /* Read-only page is acceptable */ + /* ** The remainder of this file contains the declarations of the functions ** that make up the Pager sub-system API. See source code comments for @@ -102,6 +108,7 @@ void sqlite3PagerSetBusyhandler(Pager*, int(*)(void *), void *); int sqlite3PagerSetPagesize(Pager*, u32*, int); int sqlite3PagerMaxPageCount(Pager*, int); void sqlite3PagerSetCachesize(Pager*, int); +void sqlite3PagerSetMmapLimit(Pager *, sqlite3_int64); void sqlite3PagerShrink(Pager*); void sqlite3PagerSetSafetyLevel(Pager*,int,int,int); int sqlite3PagerLockingMode(Pager *, int); @@ -138,11 +145,14 @@ int sqlite3PagerOpenSavepoint(Pager *pPager, int n); int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint); int sqlite3PagerSharedLock(Pager *pPager); -int sqlite3PagerCheckpoint(Pager *pPager, int, int*, int*); -int sqlite3PagerWalSupported(Pager *pPager); -int sqlite3PagerWalCallback(Pager *pPager); -int sqlite3PagerOpenWal(Pager *pPager, int *pisOpen); -int sqlite3PagerCloseWal(Pager *pPager); +#ifndef SQLITE_OMIT_WAL + int sqlite3PagerCheckpoint(Pager *pPager, int, int*, int*); + int sqlite3PagerWalSupported(Pager *pPager); + int sqlite3PagerWalCallback(Pager *pPager); + int sqlite3PagerOpenWal(Pager *pPager, int *pisOpen); + int sqlite3PagerCloseWal(Pager *pPager); +#endif + #ifdef SQLITE_ENABLE_ZIPVFS int sqlite3PagerWalFramesize(Pager *pPager); #endif @@ -160,6 +170,7 @@ void *sqlite3PagerTempSpace(Pager*); int sqlite3PagerIsMemdb(Pager*); void sqlite3PagerCacheStat(Pager *, int, int, int *); void sqlite3PagerClearCache(Pager *); +int sqlite3SectorSize(sqlite3_file *); /* Functions used to truncate the database file. */ void sqlite3PagerTruncateImage(Pager*,Pgno); diff --git a/src/parse.y b/src/parse.y index 94433d5..8310b26 100644 --- a/src/parse.y +++ b/src/parse.y @@ -435,8 +435,8 @@ oneselect(A) ::= SELECT distinct(D) selcollist(W) from(X) where_opt(Y) // The "distinct" nonterminal is true (1) if the DISTINCT keyword is // present and false (0) if it is not. // -%type distinct {int} -distinct(A) ::= DISTINCT. {A = 1;} +%type distinct {u16} +distinct(A) ::= DISTINCT. {A = SF_Distinct;} distinct(A) ::= ALL. {A = 0;} distinct(A) ::= . {A = 0;} @@ -499,7 +499,8 @@ stl_prefix(A) ::= seltablist(X) joinop(Y). { if( ALWAYS(A && A->nSrc>0) ) A->a[A->nSrc-1].jointype = (u8)Y; } stl_prefix(A) ::= . {A = 0;} -seltablist(A) ::= stl_prefix(X) nm(Y) dbnm(D) as(Z) indexed_opt(I) on_opt(N) using_opt(U). { +seltablist(A) ::= stl_prefix(X) nm(Y) dbnm(D) as(Z) indexed_opt(I) + on_opt(N) using_opt(U). { A = sqlite3SrcListAppendFromTerm(pParse,X,&Y,&D,&Z,0,N,U); sqlite3SrcListIndexedBy(pParse, A, &I); } @@ -512,25 +513,25 @@ seltablist(A) ::= stl_prefix(X) nm(Y) dbnm(D) as(Z) indexed_opt(I) on_opt(N) usi as(Z) on_opt(N) using_opt(U). { if( X==0 && Z.n==0 && N==0 && U==0 ){ A = F; + }else if( F->nSrc==1 ){ + A = sqlite3SrcListAppendFromTerm(pParse,X,0,0,&Z,0,N,U); + if( A ){ + struct SrcList_item *pNew = &A->a[A->nSrc-1]; + struct SrcList_item *pOld = F->a; + pNew->zName = pOld->zName; + pNew->zDatabase = pOld->zDatabase; + pNew->pSelect = pOld->pSelect; + pOld->zName = pOld->zDatabase = 0; + pOld->pSelect = 0; + } + sqlite3SrcListDelete(pParse->db, F); }else{ Select *pSubquery; sqlite3SrcListShiftJoinType(F); - pSubquery = sqlite3SelectNew(pParse,0,F,0,0,0,0,0,0,0); + pSubquery = sqlite3SelectNew(pParse,0,F,0,0,0,0,SF_NestedFrom,0,0); A = sqlite3SrcListAppendFromTerm(pParse,X,0,0,&Z,pSubquery,N,U); } } - - // A seltablist_paren nonterminal represents anything in a FROM that - // is contained inside parentheses. This can be either a subquery or - // a grouping of table and subqueries. - // -// %type seltablist_paren {Select*} -// %destructor seltablist_paren {sqlite3SelectDelete(pParse->db, $$);} -// seltablist_paren(A) ::= select(S). {A = S;} -// seltablist_paren(A) ::= seltablist(F). { -// sqlite3SrcListShiftJoinType(F); -// A = sqlite3SelectNew(pParse,0,F,0,0,0,0,0,0,0); -// } %endif SQLITE_OMIT_SUBQUERY %type dbnm {Token} @@ -653,7 +654,8 @@ where_opt(A) ::= WHERE expr(X). {A = X.pExpr;} ////////////////////////// The UPDATE command //////////////////////////////// // %ifdef SQLITE_ENABLE_UPDATE_DELETE_LIMIT -cmd ::= UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y) where_opt(W) orderby_opt(O) limit_opt(L). { +cmd ::= UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y) where_opt(W) + orderby_opt(O) limit_opt(L). { sqlite3SrcListIndexedBy(pParse, X, &I); sqlite3ExprListCheckLength(pParse,Y,"set list"); W = sqlite3LimitWhere(pParse, X, W, O, L.pLimit, L.pOffset, "UPDATE"); @@ -661,7 +663,8 @@ cmd ::= UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y) where_opt(W) } %endif %ifndef SQLITE_ENABLE_UPDATE_DELETE_LIMIT -cmd ::= UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y) where_opt(W). { +cmd ::= UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y) + where_opt(W). { sqlite3SrcListIndexedBy(pParse, X, &I); sqlite3ExprListCheckLength(pParse,Y,"set list"); sqlite3Update(pParse,X,Y,W,R); @@ -815,7 +818,7 @@ expr(A) ::= VARIABLE(X). { spanSet(&A, &X, &X); } expr(A) ::= expr(E) COLLATE ids(C). { - A.pExpr = sqlite3ExprSetCollByToken(pParse, E.pExpr, &C); + A.pExpr = sqlite3ExprAddCollateToken(pParse, E.pExpr, &C); A.zStart = E.zStart; A.zEnd = &C.z[C.n]; } @@ -1140,22 +1143,14 @@ uniqueflag(A) ::= . {A = OE_None;} idxlist_opt(A) ::= . {A = 0;} idxlist_opt(A) ::= LP idxlist(X) RP. {A = X;} idxlist(A) ::= idxlist(X) COMMA nm(Y) collate(C) sortorder(Z). { - Expr *p = 0; - if( C.n>0 ){ - p = sqlite3Expr(pParse->db, TK_COLUMN, 0); - sqlite3ExprSetCollByToken(pParse, p, &C); - } + Expr *p = sqlite3ExprAddCollateToken(pParse, 0, &C); A = sqlite3ExprListAppend(pParse,X, p); sqlite3ExprListSetName(pParse,A,&Y,1); sqlite3ExprListCheckLength(pParse, A, "index"); if( A ) A->a[A->nExpr-1].sortOrder = (u8)Z; } idxlist(A) ::= nm(Y) collate(C) sortorder(Z). { - Expr *p = 0; - if( C.n>0 ){ - p = sqlite3PExpr(pParse, TK_COLUMN, 0, 0, 0); - sqlite3ExprSetCollByToken(pParse, p, &C); - } + Expr *p = sqlite3ExprAddCollateToken(pParse, 0, &C); A = sqlite3ExprListAppend(pParse,0, p); sqlite3ExprListSetName(pParse, A, &Y, 1); sqlite3ExprListCheckLength(pParse, A, "index"); diff --git a/src/pcache.h b/src/pcache.h index b9135fd..f4d4ad7 100644 --- a/src/pcache.h +++ b/src/pcache.h @@ -53,6 +53,8 @@ struct PgHdr { #define PGHDR_REUSE_UNLIKELY 0x010 /* A hint that reuse is unlikely */ #define PGHDR_DONT_WRITE 0x020 /* Do not write content to disk */ +#define PGHDR_MMAP 0x040 /* This is an mmap page object */ + /* Initialize and shutdown the page cache subsystem */ int sqlite3PcacheInitialize(void); void sqlite3PcacheShutdown(void); diff --git a/src/pragma.c b/src/pragma.c index e5a9ac2..2297716 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -184,6 +184,9 @@ static int flagPragma(Parse *pParse, const char *zLeft, const char *zRight){ { "sql_trace", SQLITE_SqlTrace }, { "vdbe_listing", SQLITE_VdbeListing }, { "vdbe_trace", SQLITE_VdbeTrace }, + { "vdbe_addoptrace", SQLITE_VdbeAddopTrace}, + { "vdbe_debug", SQLITE_SqlTrace | SQLITE_VdbeListing + | SQLITE_VdbeTrace }, #endif #ifndef SQLITE_OMIT_CHECK { "ignore_check_constraints", SQLITE_IgnoreChecks }, @@ -316,12 +319,12 @@ void sqlite3Pragma( int rc; /* return value form SQLITE_FCNTL_PRAGMA */ sqlite3 *db = pParse->db; /* The database connection */ Db *pDb; /* The specific database being pragmaed */ - Vdbe *v = pParse->pVdbe = sqlite3VdbeCreate(db); /* Prepared statement */ -/** BEGIN CRYPTO **/ + Vdbe *v = sqlite3GetVdbe(pParse); /* Prepared statement */ +/* BEGIN SQLCIPHER */ #ifdef SQLITE_HAS_CODEC extern int codec_pragma(sqlite3*, int, Parse *, const char *, const char *); #endif -/** END CRYPTO **/ +/* END SQLCIPHER */ if( v==0 ) return; @@ -363,6 +366,7 @@ void sqlite3Pragma( aFcntl[1] = zLeft; aFcntl[2] = zRight; aFcntl[3] = 0; + db->busyHandler.nBusy = 0; rc = sqlite3_file_control(db, zDb, SQLITE_FCNTL_PRAGMA, (void*)aFcntl); if( rc==SQLITE_OK ){ if( aFcntl[0] ){ @@ -382,13 +386,13 @@ void sqlite3Pragma( pParse->rc = rc; }else -/** BEGIN CRYPTO **/ +/* BEGIN SQLCIPHER */ #ifdef SQLITE_HAS_CODEC if(codec_pragma(db, iDb, pParse, zLeft, zRight)) { /* codec_pragma executes internal */ }else #endif -/** END CRYPTO **/ +/* END SQLCIPHER */ #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) && !defined(SQLITE_OMIT_DEPRECATED) /* @@ -411,11 +415,12 @@ void sqlite3Pragma( static const VdbeOpList getCacheSize[] = { { OP_Transaction, 0, 0, 0}, /* 0 */ { OP_ReadCookie, 0, 1, BTREE_DEFAULT_CACHE_SIZE}, /* 1 */ - { OP_IfPos, 1, 7, 0}, + { OP_IfPos, 1, 8, 0}, { OP_Integer, 0, 2, 0}, { OP_Subtract, 1, 2, 1}, - { OP_IfPos, 1, 7, 0}, + { OP_IfPos, 1, 8, 0}, { OP_Integer, 0, 1, 0}, /* 6 */ + { OP_Noop, 0, 0, 0}, { OP_ResultRow, 1, 1, 0}, }; int addr; @@ -753,6 +758,43 @@ void sqlite3Pragma( } }else + /* + ** PRAGMA [database.]mmap_size(N) + ** + ** Used to set mapping size limit. The mapping size limit is + ** used to limit the aggregate size of all memory mapped regions of the + ** database file. If this parameter is set to zero, then memory mapping + ** is not used at all. If N is negative, then the default memory map + ** limit determined by sqlite3_config(SQLITE_CONFIG_MMAP_SIZE) is set. + ** The parameter N is measured in bytes. + ** + ** This value is advisory. The underlying VFS is free to memory map + ** as little or as much as it wants. Except, if N is set to 0 then the + ** upper layers will never invoke the xFetch interfaces to the VFS. + */ + if( sqlite3StrICmp(zLeft,"mmap_size")==0 ){ + sqlite3_int64 sz; + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + if( zRight ){ + int ii; + sqlite3Atoi64(zRight, &sz, 1000, SQLITE_UTF8); + if( sz<0 ) sz = sqlite3GlobalConfig.szMmap; + if( pId2->n==0 ) db->szMmap = sz; + for(ii=db->nDb-1; ii>=0; ii--){ + if( db->aDb[ii].pBt && (ii==iDb || pId2->n==0) ){ + sqlite3BtreeSetMmapLimit(db->aDb[ii].pBt, sz); + } + } + } + sz = -1; + if( sqlite3_file_control(db,zDb,SQLITE_FCNTL_MMAP_SIZE,&sz)==SQLITE_OK ){ +#if SQLITE_MAX_MMAP_SIZE==0 + sz = 0; +#endif + returnSingleInt(pParse, "mmap_size", sz); + } + }else + /* ** PRAGMA temp_store ** PRAGMA temp_store = "default"|"memory"|"file" @@ -960,11 +1002,14 @@ void sqlite3Pragma( if( sqlite3ReadSchema(pParse) ) goto pragma_out; pTab = sqlite3FindTable(db, zRight, zDb); if( pTab ){ - int i; + int i, k; int nHidden = 0; Column *pCol; + Index *pPk; + for(pPk=pTab->pIndex; pPk && pPk->autoIndex!=2; pPk=pPk->pNext){} sqlite3VdbeSetNumCols(v, 6); pParse->nMem = 6; + sqlite3CodeVerifySchema(pParse, iDb); sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "cid", SQLITE_STATIC); sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "name", SQLITE_STATIC); sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "type", SQLITE_STATIC); @@ -987,7 +1032,14 @@ void sqlite3Pragma( }else{ sqlite3VdbeAddOp2(v, OP_Null, 0, 5); } - sqlite3VdbeAddOp2(v, OP_Integer, pCol->isPrimKey, 6); + if( (pCol->colFlags & COLFLAG_PRIMKEY)==0 ){ + k = 0; + }else if( pPk==0 ){ + k = 1; + }else{ + for(k=1; ALWAYS(k<=pTab->nCol) && pPk->aiColumn[k-1]!=i; k++){} + } + sqlite3VdbeAddOp2(v, OP_Integer, k, 6); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 6); } } @@ -1003,6 +1055,7 @@ void sqlite3Pragma( pTab = pIdx->pTable; sqlite3VdbeSetNumCols(v, 3); pParse->nMem = 3; + sqlite3CodeVerifySchema(pParse, iDb); sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "seqno", SQLITE_STATIC); sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "cid", SQLITE_STATIC); sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "name", SQLITE_STATIC); @@ -1029,6 +1082,7 @@ void sqlite3Pragma( int i = 0; sqlite3VdbeSetNumCols(v, 3); pParse->nMem = 3; + sqlite3CodeVerifySchema(pParse, iDb); sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "seq", SQLITE_STATIC); sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "name", SQLITE_STATIC); sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "unique", SQLITE_STATIC); @@ -1092,6 +1146,7 @@ void sqlite3Pragma( int i = 0; sqlite3VdbeSetNumCols(v, 8); pParse->nMem = 8; + sqlite3CodeVerifySchema(pParse, iDb); sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "id", SQLITE_STATIC); sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "seq", SQLITE_STATIC); sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "table", SQLITE_STATIC); @@ -1125,6 +1180,122 @@ void sqlite3Pragma( }else #endif /* !defined(SQLITE_OMIT_FOREIGN_KEY) */ +#ifndef SQLITE_OMIT_FOREIGN_KEY +#ifndef SQLITE_OMIT_TRIGGER + if( sqlite3StrICmp(zLeft, "foreign_key_check")==0 ){ + FKey *pFK; /* A foreign key constraint */ + Table *pTab; /* Child table contain "REFERENCES" keyword */ + Table *pParent; /* Parent table that child points to */ + Index *pIdx; /* Index in the parent table */ + int i; /* Loop counter: Foreign key number for pTab */ + int j; /* Loop counter: Field of the foreign key */ + HashElem *k; /* Loop counter: Next table in schema */ + int x; /* result variable */ + int regResult; /* 3 registers to hold a result row */ + int regKey; /* Register to hold key for checking the FK */ + int regRow; /* Registers to hold a row from pTab */ + int addrTop; /* Top of a loop checking foreign keys */ + int addrOk; /* Jump here if the key is OK */ + int *aiCols; /* child to parent column mapping */ + + if( sqlite3ReadSchema(pParse) ) goto pragma_out; + regResult = pParse->nMem+1; + pParse->nMem += 4; + regKey = ++pParse->nMem; + regRow = ++pParse->nMem; + v = sqlite3GetVdbe(pParse); + sqlite3VdbeSetNumCols(v, 4); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "table", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "rowid", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "parent", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 3, COLNAME_NAME, "fkid", SQLITE_STATIC); + sqlite3CodeVerifySchema(pParse, iDb); + k = sqliteHashFirst(&db->aDb[iDb].pSchema->tblHash); + while( k ){ + if( zRight ){ + pTab = sqlite3LocateTable(pParse, 0, zRight, zDb); + k = 0; + }else{ + pTab = (Table*)sqliteHashData(k); + k = sqliteHashNext(k); + } + if( pTab==0 || pTab->pFKey==0 ) continue; + sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); + if( pTab->nCol+regRow>pParse->nMem ) pParse->nMem = pTab->nCol + regRow; + sqlite3OpenTable(pParse, 0, iDb, pTab, OP_OpenRead); + sqlite3VdbeAddOp4(v, OP_String8, 0, regResult, 0, pTab->zName, + P4_TRANSIENT); + for(i=1, pFK=pTab->pFKey; pFK; i++, pFK=pFK->pNextFrom){ + pParent = sqlite3LocateTable(pParse, 0, pFK->zTo, zDb); + if( pParent==0 ) break; + pIdx = 0; + sqlite3TableLock(pParse, iDb, pParent->tnum, 0, pParent->zName); + x = sqlite3FkLocateIndex(pParse, pParent, pFK, &pIdx, 0); + if( x==0 ){ + if( pIdx==0 ){ + sqlite3OpenTable(pParse, i, iDb, pParent, OP_OpenRead); + }else{ + KeyInfo *pKey = sqlite3IndexKeyinfo(pParse, pIdx); + sqlite3VdbeAddOp3(v, OP_OpenRead, i, pIdx->tnum, iDb); + sqlite3VdbeChangeP4(v, -1, (char*)pKey, P4_KEYINFO_HANDOFF); + } + }else{ + k = 0; + break; + } + } + if( pFK ) break; + if( pParse->nTabnTab = i; + addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, 0); + for(i=1, pFK=pTab->pFKey; pFK; i++, pFK=pFK->pNextFrom){ + pParent = sqlite3LocateTable(pParse, 0, pFK->zTo, zDb); + assert( pParent!=0 ); + pIdx = 0; + aiCols = 0; + x = sqlite3FkLocateIndex(pParse, pParent, pFK, &pIdx, &aiCols); + assert( x==0 ); + addrOk = sqlite3VdbeMakeLabel(v); + if( pIdx==0 ){ + int iKey = pFK->aCol[0].iFrom; + assert( iKey>=0 && iKeynCol ); + if( iKey!=pTab->iPKey ){ + sqlite3VdbeAddOp3(v, OP_Column, 0, iKey, regRow); + sqlite3ColumnDefault(v, pTab, iKey, regRow); + sqlite3VdbeAddOp2(v, OP_IsNull, regRow, addrOk); + sqlite3VdbeAddOp2(v, OP_MustBeInt, regRow, + sqlite3VdbeCurrentAddr(v)+3); + }else{ + sqlite3VdbeAddOp2(v, OP_Rowid, 0, regRow); + } + sqlite3VdbeAddOp3(v, OP_NotExists, i, 0, regRow); + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrOk); + sqlite3VdbeJumpHere(v, sqlite3VdbeCurrentAddr(v)-2); + }else{ + for(j=0; jnCol; j++){ + sqlite3ExprCodeGetColumnOfTable(v, pTab, 0, + aiCols ? aiCols[j] : pFK->aCol[0].iFrom, regRow+j); + sqlite3VdbeAddOp2(v, OP_IsNull, regRow+j, addrOk); + } + sqlite3VdbeAddOp3(v, OP_MakeRecord, regRow, pFK->nCol, regKey); + sqlite3VdbeChangeP4(v, -1, + sqlite3IndexAffinityStr(v,pIdx), P4_TRANSIENT); + sqlite3VdbeAddOp4Int(v, OP_Found, i, addrOk, regKey, 0); + } + sqlite3VdbeAddOp2(v, OP_Rowid, 0, regResult+1); + sqlite3VdbeAddOp4(v, OP_String8, 0, regResult+2, 0, + pFK->zTo, P4_TRANSIENT); + sqlite3VdbeAddOp2(v, OP_Integer, i-1, regResult+3); + sqlite3VdbeAddOp2(v, OP_ResultRow, regResult, 4); + sqlite3VdbeResolveLabel(v, addrOk); + sqlite3DbFree(db, aiCols); + } + sqlite3VdbeAddOp2(v, OP_Next, 0, addrTop+1); + sqlite3VdbeJumpHere(v, addrTop); + } + }else +#endif /* !defined(SQLITE_OMIT_TRIGGER) */ +#endif /* !defined(SQLITE_OMIT_FOREIGN_KEY) */ + #ifndef NDEBUG if( sqlite3StrICmp(zLeft, "parser_trace")==0 ){ if( zRight ){ @@ -1246,7 +1417,7 @@ void sqlite3Pragma( sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, sqlite3MPrintf(db, "*** in database %s ***\n", db->aDb[i].zName), P4_DYNAMIC); - sqlite3VdbeAddOp3(v, OP_Move, 2, 4, 1); + sqlite3VdbeAddOp2(v, OP_Move, 2, 4); sqlite3VdbeAddOp3(v, OP_Concat, 4, 3, 2); sqlite3VdbeAddOp2(v, OP_ResultRow, 2, 1); sqlite3VdbeJumpHere(v, addr); @@ -1409,6 +1580,11 @@ void sqlite3Pragma( ** PRAGMA [database.]user_version ** PRAGMA [database.]user_version = ** + ** PRAGMA [database.]freelist_count = + ** + ** PRAGMA [database.]application_id + ** PRAGMA [database.]application_id = + ** ** The pragma's schema_version and user_version are used to set or get ** the value of the schema-version and user-version, respectively. Both ** the schema-version and the user-version are 32-bit signed integers @@ -1430,10 +1606,14 @@ void sqlite3Pragma( if( sqlite3StrICmp(zLeft, "schema_version")==0 || sqlite3StrICmp(zLeft, "user_version")==0 || sqlite3StrICmp(zLeft, "freelist_count")==0 + || sqlite3StrICmp(zLeft, "application_id")==0 ){ int iCookie; /* Cookie index. 1 for schema-cookie, 6 for user-cookie. */ sqlite3VdbeUsesBtree(v, iDb); switch( zLeft[0] ){ + case 'a': case 'A': + iCookie = BTREE_APPLICATION_ID; + break; case 'f': case 'F': iCookie = BTREE_FREE_PAGE_COUNT; break; @@ -1549,6 +1729,22 @@ void sqlite3Pragma( sqlite3_db_release_memory(db); }else + /* + ** PRAGMA busy_timeout + ** PRAGMA busy_timeout = N + ** + ** Call sqlite3_busy_timeout(db, N). Return the current timeout value + ** if one is set. If no busy handler or a different busy handler is set + ** then 0 is returned. Setting the busy_timeout to 0 or negative + ** disables the timeout. + */ + if( sqlite3StrICmp(zLeft, "busy_timeout")==0 ){ + if( zRight ){ + sqlite3_busy_timeout(db, sqlite3Atoi(zRight)); + } + returnSingleInt(pParse, "timeout", db->busyTimeout); + }else + #if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) /* ** Report the current state of file logs for all databases @@ -1564,13 +1760,12 @@ void sqlite3Pragma( sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "status", SQLITE_STATIC); for(i=0; inDb; i++){ Btree *pBt; - Pager *pPager; const char *zState = "unknown"; int j; if( db->aDb[i].zName==0 ) continue; sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, db->aDb[i].zName, P4_STATIC); pBt = db->aDb[i].pBt; - if( pBt==0 || (pPager = sqlite3BtreePager(pBt))==0 ){ + if( pBt==0 || sqlite3BtreePager(pBt)==0 ){ zState = "closed"; }else if( sqlite3_file_control(db, i ? db->aDb[i].zName : 0, SQLITE_FCNTL_LOCKSTATE, &j)==SQLITE_OK ){ @@ -1607,7 +1802,7 @@ void sqlite3Pragma( }else #endif #if defined(SQLITE_HAS_CODEC) || defined(SQLITE_ENABLE_CEROD) - if( sqlite3StrICmp(zLeft, "activate_extensions")==0 ){ + if( sqlite3StrICmp(zLeft, "activate_extensions")==0 && zRight ){ #ifdef SQLITE_HAS_CODEC if( sqlite3StrNICmp(zRight, "see-", 4)==0 ){ sqlite3_activate_see(&zRight[4]); diff --git a/src/prepare.c b/src/prepare.c index bd2cf7a..d78d83c 100644 --- a/src/prepare.c +++ b/src/prepare.c @@ -134,7 +134,9 @@ int sqlite3InitCallback(void *pInit, int argc, char **argv, char **NotUsed){ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ int rc; int i; +#ifndef SQLITE_OMIT_DEPRECATED int size; +#endif Table *pTab; Db *pDb; char const *azArg[4]; @@ -177,7 +179,7 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ /* zMasterSchema and zInitScript are set to point at the master schema ** and initialisation script appropriate for the database being - ** initialised. zMasterName is the name of the master table. + ** initialized. zMasterName is the name of the master table. */ if( !OMIT_TEMPDB && iDb==1 ){ zMasterSchema = temp_master_schema; @@ -257,11 +259,15 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ */ if( meta[BTREE_TEXT_ENCODING-1] ){ /* text encoding */ if( iDb==0 ){ +#ifndef SQLITE_OMIT_UTF16 u8 encoding; /* If opening the main database, set ENC(db). */ encoding = (u8)meta[BTREE_TEXT_ENCODING-1] & 3; if( encoding==0 ) encoding = SQLITE_UTF8; ENC(db) = encoding; +#else + ENC(db) = SQLITE_UTF8; +#endif }else{ /* If opening an attached database, the encoding much match ENC(db) */ if( meta[BTREE_TEXT_ENCODING-1]!=ENC(db) ){ @@ -398,7 +404,7 @@ int sqlite3Init(sqlite3 *db, char **pzErrMsg){ } } - /* Once all the other databases have been initialised, load the schema + /* Once all the other databases have been initialized, load the schema ** for the TEMP database. This is loaded last, as the TEMP database ** schema may contain references to objects in other databases. */ @@ -421,7 +427,7 @@ int sqlite3Init(sqlite3 *db, char **pzErrMsg){ } /* -** This routine is a no-op if the database schema is already initialised. +** This routine is a no-op if the database schema is already initialized. ** Otherwise, the schema is loaded. An error code is returned. */ int sqlite3ReadSchema(Parse *pParse){ @@ -648,7 +654,6 @@ static int sqlite3Prepare( } #endif - assert( db->init.busy==0 || saveSqlFlag==0 ); if( db->init.busy==0 ){ Vdbe *pVdbe = pParse->pVdbe; sqlite3VdbeSetSql(pVdbe, zSql, (int)(pParse->zTail-zSql), saveSqlFlag); diff --git a/src/resolve.c b/src/resolve.c index b87d231..91efcaa 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -68,6 +68,15 @@ static void incrAggFunctionDepth(Expr *pExpr, int N){ ** from the result in the result-set. We might fix this someday. Or ** then again, we might not... ** +** If the reference is followed by a COLLATE operator, then make sure +** the COLLATE operator is preserved. For example: +** +** SELECT a+b, c+d FROM t1 ORDER BY 1 COLLATE nocase; +** +** Should be transformed into: +** +** SELECT a+b, c+d FROM t1 ORDER BY (a+b) COLLATE nocase; +** ** The nSubquery parameter specifies how many levels of subquery the ** alias is removed from the original expression. The usually value is ** zero but it might be more if the alias is contained within a subquery @@ -91,8 +100,9 @@ static void resolveAlias( assert( pOrig!=0 ); assert( pOrig->flags & EP_Resolved ); db = pParse->db; + pDup = sqlite3ExprDup(db, pOrig, 0); + if( pDup==0 ) return; if( pOrig->op!=TK_COLUMN && zType[0]!='G' ){ - pDup = sqlite3ExprDup(db, pOrig, 0); incrAggFunctionDepth(pDup, nSubquery); pDup = sqlite3PExpr(pParse, TK_AS, pDup, 0, 0); if( pDup==0 ) return; @@ -100,32 +110,26 @@ static void resolveAlias( pEList->a[iCol].iAlias = (u16)(++pParse->nAlias); } pDup->iTable = pEList->a[iCol].iAlias; - }else if( ExprHasProperty(pOrig, EP_IntValue) || pOrig->u.zToken==0 ){ - pDup = sqlite3ExprDup(db, pOrig, 0); - if( pDup==0 ) return; - }else{ - char *zToken = pOrig->u.zToken; - assert( zToken!=0 ); - pOrig->u.zToken = 0; - pDup = sqlite3ExprDup(db, pOrig, 0); - pOrig->u.zToken = zToken; - if( pDup==0 ) return; - assert( (pDup->flags & (EP_Reduced|EP_TokenOnly))==0 ); - pDup->flags2 |= EP2_MallocedToken; - pDup->u.zToken = sqlite3DbStrDup(db, zToken); } - if( pExpr->flags & EP_ExpCollate ){ - pDup->pColl = pExpr->pColl; - pDup->flags |= EP_ExpCollate; + if( pExpr->op==TK_COLLATE ){ + pDup = sqlite3ExprAddCollateString(pParse, pDup, pExpr->u.zToken); } /* Before calling sqlite3ExprDelete(), set the EP_Static flag. This ** prevents ExprDelete() from deleting the Expr structure itself, ** allowing it to be repopulated by the memcpy() on the following line. + ** The pExpr->u.zToken might point into memory that will be freed by the + ** sqlite3DbFree(db, pDup) on the last line of this block, so be sure to + ** make a copy of the token before doing the sqlite3DbFree(). */ ExprSetProperty(pExpr, EP_Static); sqlite3ExprDelete(db, pExpr); memcpy(pExpr, pDup, sizeof(*pExpr)); + if( !ExprHasProperty(pExpr, EP_IntValue) && pExpr->u.zToken!=0 ){ + assert( (pExpr->flags & (EP_Reduced|EP_TokenOnly))==0 ); + pExpr->u.zToken = sqlite3DbStrDup(db, pExpr->u.zToken); + pExpr->flags2 |= EP2_MallocedToken; + } sqlite3DbFree(db, pDup); } @@ -146,6 +150,35 @@ static int nameInUsingClause(IdList *pUsing, const char *zCol){ return 0; } +/* +** Subqueries stores the original database, table and column names for their +** result sets in ExprList.a[].zSpan, in the form "DATABASE.TABLE.COLUMN". +** Check to see if the zSpan given to this routine matches the zDb, zTab, +** and zCol. If any of zDb, zTab, and zCol are NULL then those fields will +** match anything. +*/ +int sqlite3MatchSpanName( + const char *zSpan, + const char *zCol, + const char *zTab, + const char *zDb +){ + int n; + for(n=0; ALWAYS(zSpan[n]) && zSpan[n]!='.'; n++){} + if( zDb && (sqlite3StrNICmp(zSpan, zDb, n)!=0 || zDb[n]!=0) ){ + return 0; + } + zSpan += n+1; + for(n=0; ALWAYS(zSpan[n]) && zSpan[n]!='.'; n++){} + if( zTab && (sqlite3StrNICmp(zSpan, zTab, n)!=0 || zTab[n]!=0) ){ + return 0; + } + zSpan += n+1; + if( zCol && sqlite3StrICmp(zSpan, zCol)!=0 ){ + return 0; + } + return 1; +} /* ** Given the name of a column of the form X.Y.Z or Y.Z or just Z, look up @@ -195,13 +228,27 @@ static int lookupName( assert( pNC ); /* the name context cannot be NULL. */ assert( zCol ); /* The Z in X.Y.Z cannot be NULL */ - assert( ~ExprHasAnyProperty(pExpr, EP_TokenOnly|EP_Reduced) ); + assert( !ExprHasAnyProperty(pExpr, EP_TokenOnly|EP_Reduced) ); /* Initialize the node to no-match */ pExpr->iTable = -1; pExpr->pTab = 0; ExprSetIrreducible(pExpr); + /* Translate the schema name in zDb into a pointer to the corresponding + ** schema. If not found, pSchema will remain NULL and nothing will match + ** resulting in an appropriate error message toward the end of this routine + */ + if( zDb ){ + for(i=0; inDb; i++){ + assert( db->aDb[i].zName ); + if( sqlite3StrICmp(db->aDb[i].zName,zDb)==0 ){ + pSchema = db->aDb[i].pSchema; + break; + } + } + } + /* Start at the inner-most context and move outward until a match is found */ while( pNC && cnt==0 ){ ExprList *pEList; @@ -210,31 +257,36 @@ static int lookupName( if( pSrcList ){ for(i=0, pItem=pSrcList->a; inSrc; i++, pItem++){ Table *pTab; - int iDb; Column *pCol; pTab = pItem->pTab; assert( pTab!=0 && pTab->zName!=0 ); - iDb = sqlite3SchemaToIndex(db, pTab->pSchema); assert( pTab->nCol>0 ); - if( zTab ){ - if( pItem->zAlias ){ - char *zTabName = pItem->zAlias; - if( sqlite3StrICmp(zTabName, zTab)!=0 ) continue; - }else{ - char *zTabName = pTab->zName; - if( NEVER(zTabName==0) || sqlite3StrICmp(zTabName, zTab)!=0 ){ - continue; - } - if( zDb!=0 && sqlite3StrICmp(db->aDb[iDb].zName, zDb)!=0 ){ - continue; + if( pItem->pSelect && (pItem->pSelect->selFlags & SF_NestedFrom)!=0 ){ + int hit = 0; + pEList = pItem->pSelect->pEList; + for(j=0; jnExpr; j++){ + if( sqlite3MatchSpanName(pEList->a[j].zSpan, zCol, zTab, zDb) ){ + cnt++; + cntTab = 2; + pMatch = pItem; + pExpr->iColumn = j; + hit = 1; } } + if( hit || zTab==0 ) continue; + } + if( zDb && pTab->pSchema!=pSchema ){ + continue; + } + if( zTab ){ + const char *zTabName = pItem->zAlias ? pItem->zAlias : pTab->zName; + assert( zTabName!=0 ); + if( sqlite3StrICmp(zTabName, zTab)!=0 ){ + continue; + } } if( 0==(cntTab++) ){ - pExpr->iTable = pItem->iCursor; - pExpr->pTab = pTab; - pSchema = pTab->pSchema; pMatch = pItem; } for(j=0, pCol=pTab->aCol; jnCol; j++, pCol++){ @@ -248,17 +300,19 @@ static int lookupName( if( nameInUsingClause(pItem->pUsing, zCol) ) continue; } cnt++; - pExpr->iTable = pItem->iCursor; - pExpr->pTab = pTab; pMatch = pItem; - pSchema = pTab->pSchema; /* Substitute the rowid (column -1) for the INTEGER PRIMARY KEY */ pExpr->iColumn = j==pTab->iPKey ? -1 : (i16)j; break; } } } - } + if( pMatch ){ + pExpr->iTable = pMatch->iCursor; + pExpr->pTab = pMatch->pTab; + pSchema = pExpr->pTab->pSchema; + } + } /* if( pSrcList ) */ #ifndef SQLITE_OMIT_TRIGGER /* If we have not already resolved the name, then maybe @@ -334,7 +388,10 @@ static int lookupName( ** Note that the expression in the result set should have already been ** resolved by the time the WHERE clause is resolved. */ - if( cnt==0 && (pEList = pNC->pEList)!=0 && zTab==0 ){ + if( (pEList = pNC->pEList)!=0 + && zTab==0 + && ((pNC->ncFlags & NC_AsMaybe)==0 || cnt==0) + ){ for(j=0; jnExpr; j++){ char *zAs = pEList->a[j].zName; if( zAs!=0 && sqlite3StrICmp(zAs, zCol)==0 ){ @@ -425,7 +482,9 @@ static int lookupName( lookupname_end: if( cnt==1 ){ assert( pNC!=0 ); - sqlite3AuthRead(pParse, pExpr, pSchema, pNC->pSrcList); + if( pExpr->op!=TK_AS ){ + sqlite3AuthRead(pParse, pExpr, pSchema, pNC->pSrcList); + } /* Increment the nRef value on all name contexts from TopNC up to ** the point where the name matched. */ for(;;){ @@ -593,7 +652,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ sqlite3ErrorMsg(pParse, "misuse of aggregate function %.*s()", nId,zId); pNC->nErr++; is_agg = 0; - }else if( no_such_func ){ + }else if( no_such_func && pParse->db->init.busy==0 ){ sqlite3ErrorMsg(pParse, "no such function: %.*s", nId, zId); pNC->nErr++; }else if( wrong_num_args ){ @@ -812,7 +871,7 @@ static int resolveCompoundOrderBy( int iCol = -1; Expr *pE, *pDup; if( pItem->done ) continue; - pE = pItem->pExpr; + pE = sqlite3ExprSkipCollate(pItem->pExpr); if( sqlite3ExprIsInteger(pE, &iCol) ){ if( iCol<=0 || iCol>pEList->nExpr ){ resolveOutOfRangeError(pParse, "ORDER", i+1, pEList->nExpr); @@ -830,14 +889,20 @@ static int resolveCompoundOrderBy( } } if( iCol>0 ){ - CollSeq *pColl = pE->pColl; - int flags = pE->flags & EP_ExpCollate; + /* Convert the ORDER BY term into an integer column number iCol, + ** taking care to preserve the COLLATE clause if it exists */ + Expr *pNew = sqlite3Expr(db, TK_INTEGER, 0); + if( pNew==0 ) return 1; + pNew->flags |= EP_IntValue; + pNew->u.iValue = iCol; + if( pItem->pExpr==pE ){ + pItem->pExpr = pNew; + }else{ + assert( pItem->pExpr->op==TK_COLLATE ); + assert( pItem->pExpr->pLeft==pE ); + pItem->pExpr->pLeft = pNew; + } sqlite3ExprDelete(db, pE); - pItem->pExpr = pE = sqlite3Expr(db, TK_INTEGER, 0); - if( pE==0 ) return 1; - pE->pColl = pColl; - pE->flags |= EP_IntValue | flags; - pE->u.iValue = iCol; pItem->iOrderByCol = (u16)iCol; pItem->done = 1; }else{ @@ -942,11 +1007,11 @@ static int resolveOrderGroupBy( pItem->iOrderByCol = (u16)iCol; continue; } - if( sqlite3ExprIsInteger(pE, &iCol) ){ + if( sqlite3ExprIsInteger(sqlite3ExprSkipCollate(pE), &iCol) ){ /* The ORDER BY term is an integer constant. Again, set the column ** number so that sqlite3ResolveOrderGroupBy() will convert the ** order-by term to a copy of the result-set expression */ - if( iCol<1 ){ + if( iCol<1 || iCol>0xffff ){ resolveOutOfRangeError(pParse, zType, i+1, nResult); return 1; } @@ -1023,23 +1088,6 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ return WRC_Abort; } - /* Set up the local name-context to pass to sqlite3ResolveExprNames() to - ** resolve the result-set expression list. - */ - sNC.ncFlags = NC_AllowAgg; - sNC.pSrcList = p->pSrc; - sNC.pNext = pOuterNC; - - /* Resolve names in the result set. */ - pEList = p->pEList; - assert( pEList!=0 ); - for(i=0; inExpr; i++){ - Expr *pX = pEList->a[i].pExpr; - if( sqlite3ResolveExprNames(&sNC, pX) ){ - return WRC_Abort; - } - } - /* Recursively resolve names in all subqueries */ for(i=0; ipSrc->nSrc; i++){ @@ -1067,6 +1115,23 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ } } + /* Set up the local name-context to pass to sqlite3ResolveExprNames() to + ** resolve the result-set expression list. + */ + sNC.ncFlags = NC_AllowAgg; + sNC.pSrcList = p->pSrc; + sNC.pNext = pOuterNC; + + /* Resolve names in the result set. */ + pEList = p->pEList; + assert( pEList!=0 ); + for(i=0; inExpr; i++){ + Expr *pX = pEList->a[i].pExpr; + if( sqlite3ResolveExprNames(&sNC, pX) ){ + return WRC_Abort; + } + } + /* If there are no aggregate functions in the result-set, and no GROUP BY ** expression, do not allow aggregates in any of the other expressions. */ @@ -1094,11 +1159,10 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ ** re-evaluated for each reference to it. */ sNC.pEList = p->pEList; - if( sqlite3ResolveExprNames(&sNC, p->pWhere) || - sqlite3ResolveExprNames(&sNC, p->pHaving) - ){ - return WRC_Abort; - } + sNC.ncFlags |= NC_AsMaybe; + if( sqlite3ResolveExprNames(&sNC, p->pHaving) ) return WRC_Abort; + if( sqlite3ResolveExprNames(&sNC, p->pWhere) ) return WRC_Abort; + sNC.ncFlags &= ~NC_AsMaybe; /* The ORDER BY and GROUP BY clauses may not refer to terms in ** outer queries @@ -1219,6 +1283,7 @@ int sqlite3ResolveExprNames( #endif savedHasAgg = pNC->ncFlags & NC_HasAgg; pNC->ncFlags &= ~NC_HasAgg; + memset(&w, 0, sizeof(w)); w.xExprCallback = resolveExprStep; w.xSelectCallback = resolveSelectStep; w.pParse = pNC->pParse; @@ -1259,6 +1324,7 @@ void sqlite3ResolveSelectNames( Walker w; assert( p!=0 ); + memset(&w, 0, sizeof(w)); w.xExprCallback = resolveExprStep; w.xSelectCallback = resolveSelectStep; w.pParse = pParse; diff --git a/src/select.c b/src/select.c index 6ec9da3..f3f1490 100644 --- a/src/select.c +++ b/src/select.c @@ -55,7 +55,7 @@ Select *sqlite3SelectNew( ExprList *pGroupBy, /* the GROUP BY clause */ Expr *pHaving, /* the HAVING clause */ ExprList *pOrderBy, /* the ORDER BY clause */ - int isDistinct, /* true if the DISTINCT keyword is present */ + u16 selFlags, /* Flag parameters, such as SF_Distinct */ Expr *pLimit, /* LIMIT value. NULL means not used */ Expr *pOffset /* OFFSET value. NULL means no offset */ ){ @@ -79,7 +79,7 @@ Select *sqlite3SelectNew( pNew->pGroupBy = pGroupBy; pNew->pHaving = pHaving; pNew->pOrderBy = pOrderBy; - pNew->selFlags = isDistinct ? SF_Distinct : 0; + pNew->selFlags = selFlags; pNew->op = TK_SELECT; pNew->pLimit = pLimit; pNew->pOffset = pOffset; @@ -525,6 +525,19 @@ static int checkForMultiColumnSelectError( } #endif +/* +** An instance of the following object is used to record information about +** how to process the DISTINCT keyword, to simplify passing that information +** into the selectInnerLoop() routine. +*/ +typedef struct DistinctCtx DistinctCtx; +struct DistinctCtx { + u8 isTnct; /* True if the DISTINCT keyword is present */ + u8 eTnctType; /* One of the WHERE_DISTINCT_* operators */ + int tabTnct; /* Ephemeral table used for DISTINCT processing */ + int addrTnct; /* Address of OP_OpenEphemeral opcode for tabTnct */ +}; + /* ** This routine generates the code for the inside of the inner loop ** of a SELECT. @@ -541,7 +554,7 @@ static void selectInnerLoop( int srcTab, /* Pull data from this table */ int nColumn, /* Number of columns in the source table */ ExprList *pOrderBy, /* If not NULL, sort results using this key */ - int distinct, /* If >=0, make sure results are distinct */ + DistinctCtx *pDistinct, /* If not NULL, info on how to process DISTINCT */ SelectDest *pDest, /* How to dispose of the results */ int iContinue, /* Jump here to continue with next row */ int iBreak /* Jump here to break out of the inner loop */ @@ -557,7 +570,7 @@ static void selectInnerLoop( assert( v ); if( NEVER(v==0) ) return; assert( pEList!=0 ); - hasDistinct = distinct>=0; + hasDistinct = pDistinct ? pDistinct->eTnctType : WHERE_DISTINCT_NOOP; if( pOrderBy==0 && !hasDistinct ){ codeOffset(v, p, iContinue); } @@ -597,7 +610,55 @@ static void selectInnerLoop( if( hasDistinct ){ assert( pEList!=0 ); assert( pEList->nExpr==nColumn ); - codeDistinct(pParse, distinct, iContinue, nColumn, regResult); + switch( pDistinct->eTnctType ){ + case WHERE_DISTINCT_ORDERED: { + VdbeOp *pOp; /* No longer required OpenEphemeral instr. */ + int iJump; /* Jump destination */ + int regPrev; /* Previous row content */ + + /* Allocate space for the previous row */ + regPrev = pParse->nMem+1; + pParse->nMem += nColumn; + + /* Change the OP_OpenEphemeral coded earlier to an OP_Null + ** sets the MEM_Cleared bit on the first register of the + ** previous value. This will cause the OP_Ne below to always + ** fail on the first iteration of the loop even if the first + ** row is all NULLs. + */ + sqlite3VdbeChangeToNoop(v, pDistinct->addrTnct); + pOp = sqlite3VdbeGetOp(v, pDistinct->addrTnct); + pOp->opcode = OP_Null; + pOp->p1 = 1; + pOp->p2 = regPrev; + + iJump = sqlite3VdbeCurrentAddr(v) + nColumn; + for(i=0; ia[i].pExpr); + if( iaddrTnct); + break; + } + + default: { + assert( pDistinct->eTnctType==WHERE_DISTINCT_UNORDERED ); + codeDistinct(pParse, pDistinct->tabTnct, iContinue, nColumn, regResult); + break; + } + } if( pOrderBy==0 ){ codeOffset(v, p, iContinue); } @@ -655,7 +716,8 @@ static void selectInnerLoop( */ case SRT_Set: { assert( nColumn==1 ); - p->affinity = sqlite3CompareAffinity(pEList->a[0].pExpr, pDest->affSdst); + pDest->affSdst = + sqlite3CompareAffinity(pEList->a[0].pExpr, pDest->affSdst); if( pOrderBy ){ /* At first glance you would think we could optimize out the ** ORDER BY in this case since the order of entries in the set @@ -664,7 +726,7 @@ static void selectInnerLoop( pushOntoSorter(pParse, pOrderBy, p, regResult); }else{ int r1 = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp4(v, OP_MakeRecord, regResult, 1, r1, &p->affinity, 1); + sqlite3VdbeAddOp4(v, OP_MakeRecord, regResult,1,r1, &pDest->affSdst, 1); sqlite3ExprCacheAffinityChange(pParse, regResult, 1); sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm, r1); sqlite3ReleaseTempReg(pParse, r1); @@ -931,7 +993,8 @@ static void generateSortTail( #ifndef SQLITE_OMIT_SUBQUERY case SRT_Set: { assert( nColumn==1 ); - sqlite3VdbeAddOp4(v, OP_MakeRecord, regRow, 1, regRowid, &p->affinity, 1); + sqlite3VdbeAddOp4(v, OP_MakeRecord, regRow, 1, regRowid, + &pDest->affSdst, 1); sqlite3ExprCacheAffinityChange(pParse, regRow, 1); sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm, regRowid); break; @@ -1246,7 +1309,7 @@ static void generateColumnNames( static int selectColumnsFromExprList( Parse *pParse, /* Parsing context */ ExprList *pEList, /* Expr list from which to derive column names */ - int *pnCol, /* Write the number of columns here */ + i16 *pnCol, /* Write the number of columns here */ Column **paCol /* Write the new column list here */ ){ sqlite3 *db = pParse->db; /* Database connection */ @@ -1272,9 +1335,7 @@ static int selectColumnsFromExprList( for(i=0, pCol=aCol; ia[i].pExpr; - assert( p->pRight==0 || ExprHasProperty(p->pRight, EP_IntValue) - || p->pRight->u.zToken==0 || p->pRight->u.zToken[0]!=0 ); + p = sqlite3ExprSkipCollate(pEList->a[i].pExpr); if( (zName = pEList->a[i].zName)!=0 ){ /* If the column contains an "AS " phrase, use as the name */ zName = sqlite3DbStrDup(db, zName); @@ -1312,6 +1373,9 @@ static int selectColumnsFromExprList( for(j=cnt=0; j1 && sqlite3Isdigit(zName[k]); k--){} + if( zName[k]==':' ) nName = k; zName[nName] = 0; zNewName = sqlite3MPrintf(db, "%s:%d", zName, ++cnt); sqlite3DbFree(db, zName); @@ -1643,6 +1707,8 @@ static int multiSelect( int addr = 0; int nLimit; assert( !pPrior->pLimit ); + pPrior->iLimit = p->iLimit; + pPrior->iOffset = p->iOffset; pPrior->pLimit = p->pLimit; pPrior->pOffset = p->pOffset; explainSetInteger(iSub1, pParse->iNextSelectId); @@ -1768,7 +1834,7 @@ static int multiSelect( sqlite3VdbeAddOp2(v, OP_Rewind, unionTab, iBreak); iStart = sqlite3VdbeCurrentAddr(v); selectInnerLoop(pParse, p, p->pEList, unionTab, p->pEList->nExpr, - 0, -1, &dest, iCont, iBreak); + 0, 0, &dest, iCont, iBreak); sqlite3VdbeResolveLabel(v, iCont); sqlite3VdbeAddOp2(v, OP_Next, unionTab, iStart); sqlite3VdbeResolveLabel(v, iBreak); @@ -1846,7 +1912,7 @@ static int multiSelect( sqlite3VdbeAddOp4Int(v, OP_NotFound, tab2, iCont, r1, 0); sqlite3ReleaseTempReg(pParse, r1); selectInnerLoop(pParse, p, p->pEList, tab1, p->pEList->nExpr, - 0, -1, &dest, iCont, iBreak); + 0, 0, &dest, iCont, iBreak); sqlite3VdbeResolveLabel(v, iCont); sqlite3VdbeAddOp2(v, OP_Next, tab1, iStart); sqlite3VdbeResolveLabel(v, iBreak); @@ -1892,6 +1958,7 @@ static int multiSelect( *apColl = db->pDfltColl; } } + pKeyInfo->aSortOrder = (u8*)apColl; for(pLoop=p; pLoop; pLoop=pLoop->pPrior){ for(i=0; i<2; i++){ @@ -1965,7 +2032,7 @@ static int generateOutputSubroutine( (char*)pKeyInfo, p4type); sqlite3VdbeAddOp3(v, OP_Jump, j2+2, iContinue, j2+2); sqlite3VdbeJumpHere(v, j1); - sqlite3ExprCodeCopy(pParse, pIn->iSdst, regPrev+1, pIn->nSdst); + sqlite3VdbeAddOp3(v, OP_Copy, pIn->iSdst, regPrev+1, pIn->nSdst-1); sqlite3VdbeAddOp2(v, OP_Integer, 1, regPrev); } if( pParse->db->mallocFailed ) return 0; @@ -2000,10 +2067,10 @@ static int generateOutputSubroutine( case SRT_Set: { int r1; assert( pIn->nSdst==1 ); - p->affinity = + pDest->affSdst = sqlite3CompareAffinity(p->pEList->a[0].pExpr, pDest->affSdst); r1 = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp4(v, OP_MakeRecord, pIn->iSdst, 1, r1, &p->affinity, 1); + sqlite3VdbeAddOp4(v, OP_MakeRecord, pIn->iSdst, 1, r1, &pDest->affSdst,1); sqlite3ExprCacheAffinityChange(pParse, pIn->iSdst, 1); sqlite3VdbeAddOp2(v, OP_IdxInsert, pDest->iSDParm, r1); sqlite3ReleaseTempReg(pParse, r1); @@ -2269,12 +2336,13 @@ static int multiSelectOrderBy( for(i=0; ia[i].pExpr; - if( pTerm->flags & EP_ExpCollate ){ - pColl = pTerm->pColl; + if( pTerm->flags & EP_Collate ){ + pColl = sqlite3ExprCollSeq(pParse, pTerm); }else{ pColl = multiSelectCollSeq(pParse, p, aPermute[i]); - pTerm->flags |= EP_ExpCollate; - pTerm->pColl = pColl; + if( pColl==0 ) pColl = db->pDfltColl; + pOrderBy->a[i].pExpr = + sqlite3ExprAddCollateString(pParse, pTerm, pColl->zName); } pKeyMerge->aColl[i] = pColl; pKeyMerge->aSortOrder[i] = pOrderBy->a[i].sortOrder; @@ -2298,7 +2366,8 @@ static int multiSelectOrderBy( }else{ int nExpr = p->pEList->nExpr; assert( nOrderBy>=nExpr || db->mallocFailed ); - regPrev = sqlite3GetTempRange(pParse, nExpr+1); + regPrev = pParse->nMem+1; + pParse->nMem += nExpr+1; sqlite3VdbeAddOp2(v, OP_Integer, 0, regPrev); pKeyDup = sqlite3DbMallocZero(db, sizeof(*pKeyDup) + nExpr*(sizeof(CollSeq*)+1) ); @@ -2477,14 +2546,9 @@ static int multiSelectOrderBy( sqlite3VdbeAddOp4(v, OP_Permutation, 0, 0, 0, (char*)aPermute, P4_INTARRAY); sqlite3VdbeAddOp4(v, OP_Compare, destA.iSdst, destB.iSdst, nOrderBy, (char*)pKeyMerge, P4_KEYINFO_HANDOFF); + sqlite3VdbeChangeP5(v, OPFLAG_PERMUTE); sqlite3VdbeAddOp3(v, OP_Jump, addrAltB, addrAeqB, addrAgtB); - /* Release temporary registers - */ - if( regPrev ){ - sqlite3ReleaseTempRange(pParse, regPrev, nOrderBy+1); - } - /* Jump to the this point in order to terminate the query. */ sqlite3VdbeResolveLabel(v, labelEnd); @@ -2544,9 +2608,6 @@ static Expr *substExpr( assert( pEList!=0 && pExpr->iColumnnExpr ); assert( pExpr->pLeft==0 && pExpr->pRight==0 ); pNew = sqlite3ExprDup(db, pEList->a[pExpr->iColumn].pExpr, 0); - if( pNew && pExpr->pColl ){ - pNew->pColl = pExpr->pColl; - } sqlite3ExprDelete(db, pExpr); pExpr = pNew; } @@ -2745,7 +2806,7 @@ static int flattenSubquery( */ assert( p!=0 ); assert( p->pPrior==0 ); /* Unable to flatten compound queries */ - if( db->flags & SQLITE_QueryFlattener ) return 0; + if( OptimizationDisabled(db, SQLITE_QueryFlattener) ) return 0; pSrc = p->pSrc; assert( pSrc && iFrom>=0 && iFromnSrc ); pSubitem = &pSrc->a[iFrom]; @@ -2899,12 +2960,15 @@ static int flattenSubquery( Select *pNew; ExprList *pOrderBy = p->pOrderBy; Expr *pLimit = p->pLimit; + Expr *pOffset = p->pOffset; Select *pPrior = p->pPrior; p->pOrderBy = 0; p->pSrc = 0; p->pPrior = 0; p->pLimit = 0; + p->pOffset = 0; pNew = sqlite3SelectDup(db, p, 0); + p->pOffset = pOffset; p->pLimit = pLimit; p->pOrderBy = pOrderBy; p->pSrc = pSrc; @@ -3034,10 +3098,9 @@ static int flattenSubquery( pList = pParent->pEList; for(i=0; inExpr; i++){ if( pList->a[i].zName==0 ){ - const char *zSpan = pList->a[i].zSpan; - if( ALWAYS(zSpan) ){ - pList->a[i].zName = sqlite3DbStrDup(db, zSpan); - } + char *zName = sqlite3DbStrDup(db, pList->a[i].zSpan); + sqlite3Dequote(zName); + pList->a[i].zName = zName; } } substExprList(db, pParent->pEList, iParent, pSub->pEList); @@ -3098,34 +3161,43 @@ static int flattenSubquery( #endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ /* -** Analyze the SELECT statement passed as an argument to see if it -** is a min() or max() query. Return WHERE_ORDERBY_MIN or WHERE_ORDERBY_MAX if -** it is, or 0 otherwise. At present, a query is considered to be -** a min()/max() query if: +** Based on the contents of the AggInfo structure indicated by the first +** argument, this function checks if the following are true: +** +** * the query contains just a single aggregate function, +** * the aggregate function is either min() or max(), and +** * the argument to the aggregate function is a column value. ** -** 1. There is a single object in the FROM clause. +** If all of the above are true, then WHERE_ORDERBY_MIN or WHERE_ORDERBY_MAX +** is returned as appropriate. Also, *ppMinMax is set to point to the +** list of arguments passed to the aggregate before returning. ** -** 2. There is a single expression in the result set, and it is -** either min(x) or max(x), where x is a column reference. +** Or, if the conditions above are not met, *ppMinMax is set to 0 and +** WHERE_ORDERBY_NORMAL is returned. */ -static u8 minMaxQuery(Select *p){ - Expr *pExpr; - ExprList *pEList = p->pEList; - - if( pEList->nExpr!=1 ) return WHERE_ORDERBY_NORMAL; - pExpr = pEList->a[0].pExpr; - if( pExpr->op!=TK_AGG_FUNCTION ) return 0; - if( NEVER(ExprHasProperty(pExpr, EP_xIsSelect)) ) return 0; - pEList = pExpr->x.pList; - if( pEList==0 || pEList->nExpr!=1 ) return 0; - if( pEList->a[0].pExpr->op!=TK_AGG_COLUMN ) return WHERE_ORDERBY_NORMAL; - assert( !ExprHasProperty(pExpr, EP_IntValue) ); - if( sqlite3StrICmp(pExpr->u.zToken,"min")==0 ){ - return WHERE_ORDERBY_MIN; - }else if( sqlite3StrICmp(pExpr->u.zToken,"max")==0 ){ - return WHERE_ORDERBY_MAX; +static u8 minMaxQuery(AggInfo *pAggInfo, ExprList **ppMinMax){ + int eRet = WHERE_ORDERBY_NORMAL; /* Return value */ + + *ppMinMax = 0; + if( pAggInfo->nFunc==1 ){ + Expr *pExpr = pAggInfo->aFunc[0].pExpr; /* Aggregate function */ + ExprList *pEList = pExpr->x.pList; /* Arguments to agg function */ + + assert( pExpr->op==TK_AGG_FUNCTION ); + if( pEList && pEList->nExpr==1 && pEList->a[0].pExpr->op==TK_AGG_COLUMN ){ + const char *zFunc = pExpr->u.zToken; + if( sqlite3StrICmp(zFunc, "min")==0 ){ + eRet = WHERE_ORDERBY_MIN; + *ppMinMax = pEList; + }else if( sqlite3StrICmp(zFunc, "max")==0 ){ + eRet = WHERE_ORDERBY_MAX; + *ppMinMax = pEList; + } + } } - return WHERE_ORDERBY_NORMAL; + + assert( *ppMinMax==0 || (*ppMinMax)->nExpr==1 ); + return eRet; } /* @@ -3188,6 +3260,69 @@ int sqlite3IndexedByLookup(Parse *pParse, struct SrcList_item *pFrom){ } return SQLITE_OK; } +/* +** Detect compound SELECT statements that use an ORDER BY clause with +** an alternative collating sequence. +** +** SELECT ... FROM t1 EXCEPT SELECT ... FROM t2 ORDER BY .. COLLATE ... +** +** These are rewritten as a subquery: +** +** SELECT * FROM (SELECT ... FROM t1 EXCEPT SELECT ... FROM t2) +** ORDER BY ... COLLATE ... +** +** This transformation is necessary because the multiSelectOrderBy() routine +** above that generates the code for a compound SELECT with an ORDER BY clause +** uses a merge algorithm that requires the same collating sequence on the +** result columns as on the ORDER BY clause. See ticket +** http://www.sqlite.org/src/info/6709574d2a +** +** This transformation is only needed for EXCEPT, INTERSECT, and UNION. +** The UNION ALL operator works fine with multiSelectOrderBy() even when +** there are COLLATE terms in the ORDER BY. +*/ +static int convertCompoundSelectToSubquery(Walker *pWalker, Select *p){ + int i; + Select *pNew; + Select *pX; + sqlite3 *db; + struct ExprList_item *a; + SrcList *pNewSrc; + Parse *pParse; + Token dummy; + + if( p->pPrior==0 ) return WRC_Continue; + if( p->pOrderBy==0 ) return WRC_Continue; + for(pX=p; pX && (pX->op==TK_ALL || pX->op==TK_SELECT); pX=pX->pPrior){} + if( pX==0 ) return WRC_Continue; + a = p->pOrderBy->a; + for(i=p->pOrderBy->nExpr-1; i>=0; i--){ + if( a[i].pExpr->flags & EP_Collate ) break; + } + if( i<0 ) return WRC_Continue; + + /* If we reach this point, that means the transformation is required. */ + + pParse = pWalker->pParse; + db = pParse->db; + pNew = sqlite3DbMallocZero(db, sizeof(*pNew) ); + if( pNew==0 ) return WRC_Abort; + memset(&dummy, 0, sizeof(dummy)); + pNewSrc = sqlite3SrcListAppendFromTerm(pParse,0,0,0,&dummy,pNew,0,0); + if( pNewSrc==0 ) return WRC_Abort; + *pNew = *p; + p->pSrc = pNewSrc; + p->pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db, TK_ALL, 0)); + p->op = TK_SELECT; + p->pWhere = 0; + pNew->pGroupBy = 0; + pNew->pHaving = 0; + pNew->pOrderBy = 0; + p->pPrior = 0; + pNew->pLimit = 0; + pNew->pOffset = 0; + return WRC_Continue; +} /* ** This routine is a Walker callback for "expanding" a SELECT statement. @@ -3220,14 +3355,16 @@ static int selectExpander(Walker *pWalker, Select *p){ ExprList *pEList; struct SrcList_item *pFrom; sqlite3 *db = pParse->db; + Expr *pE, *pRight, *pExpr; + u16 selFlags = p->selFlags; + p->selFlags |= SF_Expanded; if( db->mallocFailed ){ return WRC_Abort; } - if( NEVER(p->pSrc==0) || (p->selFlags & SF_Expanded)!=0 ){ + if( NEVER(p->pSrc==0) || (selFlags & SF_Expanded)!=0 ){ return WRC_Prune; } - p->selFlags |= SF_Expanded; pTabList = p->pSrc; pEList = p->pEList; @@ -3268,9 +3405,14 @@ static int selectExpander(Walker *pWalker, Select *p){ }else{ /* An ordinary table or view name in the FROM clause */ assert( pFrom->pTab==0 ); - pFrom->pTab = pTab = - sqlite3LocateTable(pParse,0,pFrom->zName,pFrom->zDatabase); + pFrom->pTab = pTab = sqlite3LocateTableItem(pParse, 0, pFrom); if( pTab==0 ) return WRC_Abort; + if( pTab->nRef==0xffff ){ + sqlite3ErrorMsg(pParse, "too many references to \"%s\": max 65535", + pTab->zName); + pFrom->pTab = 0; + return WRC_Abort; + } pTab->nRef++; #if !defined(SQLITE_OMIT_VIEW) || !defined (SQLITE_OMIT_VIRTUALTABLE) if( pTab->pSelect || IsVirtual(pTab) ){ @@ -3306,7 +3448,7 @@ static int selectExpander(Walker *pWalker, Select *p){ ** that need expanding. */ for(k=0; knExpr; k++){ - Expr *pE = pEList->a[k].pExpr; + pE = pEList->a[k].pExpr; if( pE->op==TK_ALL ) break; assert( pE->op!=TK_DOT || pE->pRight!=0 ); assert( pE->op!=TK_DOT || (pE->pLeft!=0 && pE->pLeft->op==TK_ID) ); @@ -3324,10 +3466,18 @@ static int selectExpander(Walker *pWalker, Select *p){ int longNames = (flags & SQLITE_FullColNames)!=0 && (flags & SQLITE_ShortColNames)==0; + /* When processing FROM-clause subqueries, it is always the case + ** that full_column_names=OFF and short_column_names=ON. The + ** sqlite3ResultSetOfSelect() routine makes it so. */ + assert( (p->selFlags & SF_NestedFrom)==0 + || ((flags & SQLITE_FullColNames)==0 && + (flags & SQLITE_ShortColNames)!=0) ); + for(k=0; knExpr; k++){ - Expr *pE = a[k].pExpr; - assert( pE->op!=TK_DOT || pE->pRight!=0 ); - if( pE->op!=TK_ALL && (pE->op!=TK_DOT || pE->pRight->op!=TK_ALL) ){ + pE = a[k].pExpr; + pRight = pE->pRight; + assert( pE->op!=TK_DOT || pRight!=0 ); + if( pE->op!=TK_ALL && (pE->op!=TK_DOT || pRight->op!=TK_ALL) ){ /* This particular expression does not need to be expanded. */ pNew = sqlite3ExprListAppend(pParse, pNew, a[k].pExpr); @@ -3342,32 +3492,43 @@ static int selectExpander(Walker *pWalker, Select *p){ /* This expression is a "*" or a "TABLE.*" and needs to be ** expanded. */ int tableSeen = 0; /* Set to 1 when TABLE matches */ - char *zTName; /* text of name of TABLE */ + char *zTName = 0; /* text of name of TABLE */ if( pE->op==TK_DOT ){ assert( pE->pLeft!=0 ); assert( !ExprHasProperty(pE->pLeft, EP_IntValue) ); zTName = pE->pLeft->u.zToken; - }else{ - zTName = 0; } for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ Table *pTab = pFrom->pTab; + Select *pSub = pFrom->pSelect; char *zTabName = pFrom->zAlias; + const char *zSchemaName = 0; + int iDb; if( zTabName==0 ){ zTabName = pTab->zName; } if( db->mallocFailed ) break; - if( zTName && sqlite3StrICmp(zTName, zTabName)!=0 ){ - continue; + if( pSub==0 || (pSub->selFlags & SF_NestedFrom)==0 ){ + pSub = 0; + if( zTName && sqlite3StrICmp(zTName, zTabName)!=0 ){ + continue; + } + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + zSchemaName = iDb>=0 ? db->aDb[iDb].zName : "*"; } - tableSeen = 1; for(j=0; jnCol; j++){ - Expr *pExpr, *pRight; char *zName = pTab->aCol[j].zName; char *zColname; /* The computed column name */ char *zToFree; /* Malloced string that needs to be freed */ Token sColname; /* Computed column name as a token */ + assert( zName ); + if( zTName && pSub + && sqlite3MatchSpanName(pSub->pEList->a[j].zSpan, 0, zTName, 0)==0 + ){ + continue; + } + /* If a column is marked as 'hidden' (currently only possible ** for virtual tables), do not include it in the expanded ** result-set list. @@ -3376,6 +3537,7 @@ static int selectExpander(Walker *pWalker, Select *p){ assert(IsVirtual(pTab)); continue; } + tableSeen = 1; if( i>0 && zTName==0 ){ if( (pFrom->jointype & JT_NATURAL)!=0 @@ -3398,6 +3560,10 @@ static int selectExpander(Walker *pWalker, Select *p){ Expr *pLeft; pLeft = sqlite3Expr(db, TK_ID, zTabName); pExpr = sqlite3PExpr(pParse, TK_DOT, pLeft, pRight, 0); + if( zSchemaName ){ + pLeft = sqlite3Expr(db, TK_ID, zSchemaName); + pExpr = sqlite3PExpr(pParse, TK_DOT, pLeft, pExpr, 0); + } if( longNames ){ zColname = sqlite3MPrintf(db, "%s.%s", zTabName, zName); zToFree = zColname; @@ -3409,6 +3575,18 @@ static int selectExpander(Walker *pWalker, Select *p){ sColname.z = zColname; sColname.n = sqlite3Strlen30(zColname); sqlite3ExprListSetName(pParse, pNew, &sColname, 0); + if( pNew && (p->selFlags & SF_NestedFrom)!=0 ){ + struct ExprList_item *pX = &pNew->a[pNew->nExpr-1]; + if( pSub ){ + pX->zSpan = sqlite3DbStrDup(db, pSub->pEList->a[j].zSpan); + testcase( pX->zSpan==0 ); + }else{ + pX->zSpan = sqlite3MPrintf(db, "%s.%s.%s", + zSchemaName, zTabName, zColname); + testcase( pX->zSpan==0 ); + } + pX->bSpanIsTab = 1; + } sqlite3DbFree(db, zToFree); } } @@ -3461,10 +3639,13 @@ static int exprWalkNoop(Walker *NotUsed, Expr *NotUsed2){ */ static void sqlite3SelectExpand(Parse *pParse, Select *pSelect){ Walker w; - w.xSelectCallback = selectExpander; + memset(&w, 0, sizeof(w)); + w.xSelectCallback = convertCompoundSelectToSubquery; w.xExprCallback = exprWalkNoop; w.pParse = pParse; sqlite3WalkSelect(&w, pSelect); + w.xSelectCallback = selectExpander; + sqlite3WalkSelect(&w, pSelect); } @@ -3519,9 +3700,11 @@ static int selectAddSubqueryTypeInfo(Walker *pWalker, Select *p){ static void sqlite3SelectAddTypeInfo(Parse *pParse, Select *pSelect){ #ifndef SQLITE_OMIT_SUBQUERY Walker w; + memset(&w, 0, sizeof(w)); w.xSelectCallback = selectAddSubqueryTypeInfo; w.xExprCallback = exprWalkNoop; w.pParse = pParse; + w.bSelectDepthFirst = 1; sqlite3WalkSelect(&w, pSelect); #endif } @@ -3547,6 +3730,7 @@ void sqlite3SelectPrep( sqlite3 *db; if( NEVER(p==0) ) return; db = pParse->db; + if( db->mallocFailed ) return; if( p->selFlags & SF_HasTypeInfo ) return; sqlite3SelectExpand(pParse, p); if( pParse->nErr || db->mallocFailed ) return; @@ -3785,11 +3969,9 @@ int sqlite3Select( ExprList *pOrderBy; /* The ORDER BY clause. May be NULL */ ExprList *pGroupBy; /* The GROUP BY clause. May be NULL */ Expr *pHaving; /* The HAVING clause. May be NULL */ - int isDistinct; /* True if the DISTINCT keyword is present */ - int distinct; /* Table to use for the distinct set */ int rc = 1; /* Value to return from this function */ int addrSortIndex; /* Address of an OP_OpenEphemeral instruction */ - int addrDistinctIndex; /* Address of an OP_OpenEphemeral instruction */ + DistinctCtx sDistinct; /* Info on how to code the DISTINCT keyword */ AggInfo sAggInfo; /* Information used by aggregate queries */ int iEnd; /* Address of the end of the query */ sqlite3 *db; /* The database connection */ @@ -3849,8 +4031,17 @@ int sqlite3Select( int isAggSub; if( pSub==0 ) continue; + + /* Sometimes the code for a subquery will be generated more than + ** once, if the subquery is part of the WHERE clause in a LEFT JOIN, + ** for example. In that case, do not regenerate the code to manifest + ** a view or the co-routine to implement a view. The first instance + ** is sufficient, though the subroutine to manifest the view does need + ** to be invoked again. */ if( pItem->addrFillSub ){ - sqlite3VdbeAddOp2(v, OP_Gosub, pItem->regReturn, pItem->addrFillSub); + if( pItem->viaCoroutine==0 ){ + sqlite3VdbeAddOp2(v, OP_Gosub, pItem->regReturn, pItem->addrFillSub); + } continue; } @@ -3871,6 +4062,44 @@ int sqlite3Select( p->selFlags |= SF_Aggregate; } i = -1; + }else if( pTabList->nSrc==1 && (p->selFlags & SF_Materialize)==0 + && OptimizationEnabled(db, SQLITE_SubqCoroutine) + ){ + /* Implement a co-routine that will return a single row of the result + ** set on each invocation. + */ + int addrTop; + int addrEof; + pItem->regReturn = ++pParse->nMem; + addrEof = ++pParse->nMem; + /* Before coding the OP_Goto to jump to the start of the main routine, + ** ensure that the jump to the verify-schema routine has already + ** been coded. Otherwise, the verify-schema would likely be coded as + ** part of the co-routine. If the main routine then accessed the + ** database before invoking the co-routine for the first time (for + ** example to initialize a LIMIT register from a sub-select), it would + ** be doing so without having verified the schema version and obtained + ** the required db locks. See ticket d6b36be38. */ + sqlite3CodeVerifySchema(pParse, -1); + sqlite3VdbeAddOp0(v, OP_Goto); + addrTop = sqlite3VdbeAddOp1(v, OP_OpenPseudo, pItem->iCursor); + sqlite3VdbeChangeP5(v, 1); + VdbeComment((v, "coroutine for %s", pItem->pTab->zName)); + pItem->addrFillSub = addrTop; + sqlite3VdbeAddOp2(v, OP_Integer, 0, addrEof); + sqlite3VdbeChangeP5(v, 1); + sqlite3SelectDestInit(&dest, SRT_Coroutine, pItem->regReturn); + explainSetInteger(pItem->iSelectId, (u8)pParse->iNextSelectId); + sqlite3Select(pParse, pSub, &dest); + pItem->pTab->nRowEst = (unsigned)pSub->nSelectRow; + pItem->viaCoroutine = 1; + sqlite3VdbeChangeP2(v, addrTop, dest.iSdst); + sqlite3VdbeChangeP3(v, addrTop, dest.nSdst); + sqlite3VdbeAddOp2(v, OP_Integer, 1, addrEof); + sqlite3VdbeAddOp1(v, OP_Yield, pItem->regReturn); + VdbeComment((v, "end %s", pItem->pTab->zName)); + sqlite3VdbeJumpHere(v, addrTop-1); + sqlite3ClearTempRegCache(pParse); }else{ /* Generate a subroutine that will fill an ephemeral table with ** the content of this subquery. pItem->addrFillSub will point @@ -3886,7 +4115,7 @@ int sqlite3Select( pItem->addrFillSub = topAddr+1; VdbeNoopComment((v, "materialize %s", pItem->pTab->zName)); if( pItem->isCorrelated==0 ){ - /* If the subquery is no correlated and if we are not inside of + /* If the subquery is not correlated and if we are not inside of ** a trigger, then we only need to compute the value of the subquery ** once. */ onceAddr = sqlite3CodeOnce(pParse); @@ -3915,7 +4144,7 @@ int sqlite3Select( pWhere = p->pWhere; pGroupBy = p->pGroupBy; pHaving = p->pHaving; - isDistinct = (p->selFlags & SF_Distinct)!=0; + sDistinct.isTnct = (p->selFlags & SF_Distinct)!=0; #ifndef SQLITE_OMIT_COMPOUND_SELECT /* If there is are a sequence of queries, do the earlier ones first. @@ -3950,7 +4179,7 @@ int sqlite3Select( ** to disable this optimization for testing purposes. */ if( sqlite3ExprListCompare(p->pGroupBy, pOrderBy)==0 - && (db->flags & SQLITE_GroupByOrder)==0 ){ + && OptimizationEnabled(db, SQLITE_GroupByOrder) ){ pOrderBy = 0; } @@ -3976,6 +4205,10 @@ int sqlite3Select( p->pGroupBy = sqlite3ExprListDup(db, p->pEList, 0); pGroupBy = p->pGroupBy; pOrderBy = 0; + /* Notice that even thought SF_Distinct has been cleared from p->selFlags, + ** the sDistinct.isTnct is still set. Hence, isTnct represents the + ** original setting of the SF_Distinct flag, not the current setting */ + assert( sDistinct.isTnct ); } /* If there is an ORDER BY clause, then this sorting @@ -4016,24 +4249,27 @@ int sqlite3Select( /* Open a virtual index to use for the distinct set. */ if( p->selFlags & SF_Distinct ){ - KeyInfo *pKeyInfo; - distinct = pParse->nTab++; - pKeyInfo = keyInfoFromExprList(pParse, p->pEList); - addrDistinctIndex = sqlite3VdbeAddOp4(v, OP_OpenEphemeral, distinct, 0, 0, - (char*)pKeyInfo, P4_KEYINFO_HANDOFF); + sDistinct.tabTnct = pParse->nTab++; + sDistinct.addrTnct = sqlite3VdbeAddOp4(v, OP_OpenEphemeral, + sDistinct.tabTnct, 0, 0, + (char*)keyInfoFromExprList(pParse, p->pEList), + P4_KEYINFO_HANDOFF); sqlite3VdbeChangeP5(v, BTREE_UNORDERED); + sDistinct.eTnctType = WHERE_DISTINCT_UNORDERED; }else{ - distinct = addrDistinctIndex = -1; + sDistinct.eTnctType = WHERE_DISTINCT_NOOP; } - /* Aggregate and non-aggregate queries are handled differently */ if( !isAgg && pGroupBy==0 ){ - ExprList *pDist = (isDistinct ? p->pEList : 0); + /* No aggregate functions and no GROUP BY clause */ + ExprList *pDist = (sDistinct.isTnct ? p->pEList : 0); /* Begin the database scan. */ - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pOrderBy, pDist, 0,0); + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pOrderBy, pDist, 0,0); if( pWInfo==0 ) goto select_end; if( pWInfo->nRowOut < p->nSelectRow ) p->nSelectRow = pWInfo->nRowOut; + if( pWInfo->eDistinct ) sDistinct.eTnctType = pWInfo->eDistinct; + if( pOrderBy && pWInfo->nOBSat==pOrderBy->nExpr ) pOrderBy = 0; /* If sorting index that was created by a prior OP_OpenEphemeral ** instruction ended up not being needed, then change the OP_OpenEphemeral @@ -4044,59 +4280,16 @@ int sqlite3Select( p->addrOpenEphm[2] = -1; } - if( pWInfo->eDistinct ){ - VdbeOp *pOp; /* No longer required OpenEphemeral instr. */ - - assert( addrDistinctIndex>=0 ); - pOp = sqlite3VdbeGetOp(v, addrDistinctIndex); - - assert( isDistinct ); - assert( pWInfo->eDistinct==WHERE_DISTINCT_ORDERED - || pWInfo->eDistinct==WHERE_DISTINCT_UNIQUE - ); - distinct = -1; - if( pWInfo->eDistinct==WHERE_DISTINCT_ORDERED ){ - int iJump; - int iExpr; - int iFlag = ++pParse->nMem; - int iBase = pParse->nMem+1; - int iBase2 = iBase + pEList->nExpr; - pParse->nMem += (pEList->nExpr*2); - - /* Change the OP_OpenEphemeral coded earlier to an OP_Integer. The - ** OP_Integer initializes the "first row" flag. */ - pOp->opcode = OP_Integer; - pOp->p1 = 1; - pOp->p2 = iFlag; - - sqlite3ExprCodeExprList(pParse, pEList, iBase, 1); - iJump = sqlite3VdbeCurrentAddr(v) + 1 + pEList->nExpr + 1 + 1; - sqlite3VdbeAddOp2(v, OP_If, iFlag, iJump-1); - for(iExpr=0; iExprnExpr; iExpr++){ - CollSeq *pColl = sqlite3ExprCollSeq(pParse, pEList->a[iExpr].pExpr); - sqlite3VdbeAddOp3(v, OP_Ne, iBase+iExpr, iJump, iBase2+iExpr); - sqlite3VdbeChangeP4(v, -1, (const char *)pColl, P4_COLLSEQ); - sqlite3VdbeChangeP5(v, SQLITE_NULLEQ); - } - sqlite3VdbeAddOp2(v, OP_Goto, 0, pWInfo->iContinue); - - sqlite3VdbeAddOp2(v, OP_Integer, 0, iFlag); - assert( sqlite3VdbeCurrentAddr(v)==iJump ); - sqlite3VdbeAddOp3(v, OP_Move, iBase, iBase2, pEList->nExpr); - }else{ - pOp->opcode = OP_Noop; - } - } - /* Use the standard inner loop. */ - selectInnerLoop(pParse, p, pEList, 0, 0, pOrderBy, distinct, pDest, + selectInnerLoop(pParse, p, pEList, 0, 0, pOrderBy, &sDistinct, pDest, pWInfo->iContinue, pWInfo->iBreak); /* End the database scan loop. */ sqlite3WhereEnd(pWInfo); }else{ - /* This is the processing for aggregate queries */ + /* This case when there exist aggregate functions or a GROUP BY clause + ** or both */ NameContext sNC; /* Name context for processing aggregate information */ int iAMem; /* First Mem address for storing current GROUP BY */ int iBMem; /* First Mem address for previous GROUP BY */ @@ -4204,14 +4397,13 @@ int sqlite3Select( ** in the right order to begin with. */ sqlite3VdbeAddOp2(v, OP_Gosub, regReset, addrReset); - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pGroupBy, 0, 0, 0); + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pGroupBy, 0, 0, 0); if( pWInfo==0 ) goto select_end; - if( pGroupBy==0 ){ + if( pWInfo->nOBSat==pGroupBy->nExpr ){ /* The optimizer is able to deliver rows in group by order so ** we do not have to sort. The OP_OpenEphemeral table will be ** cancelled later because we still need to use the pKeyInfo */ - pGroupBy = p->pGroupBy; groupBySort = 0; }else{ /* Rows are coming out in undetermined order. We have to push @@ -4225,7 +4417,8 @@ int sqlite3Select( int nGroupBy; explainTempTable(pParse, - isDistinct && !(p->selFlags&SF_Distinct)?"DISTINCT":"GROUP BY"); + (sDistinct.isTnct && (p->selFlags&SF_Distinct)==0) ? + "DISTINCT" : "GROUP BY"); groupBySort = 1; nGroupBy = pGroupBy->nExpr; @@ -4357,7 +4550,7 @@ int sqlite3Select( finalizeAggFunctions(pParse, &sAggInfo); sqlite3ExprIfFalse(pParse, pHaving, addrOutputRow+1, SQLITE_JUMPIFNULL); selectInnerLoop(pParse, p, p->pEList, 0, 0, pOrderBy, - distinct, pDest, + &sDistinct, pDest, addrOutputRow+1, addrSetAbort); sqlite3VdbeAddOp1(v, OP_Return, regOutputRow); VdbeComment((v, "end groupby result generator")); @@ -4445,7 +4638,7 @@ int sqlite3Select( ** value of x, the only row required). ** ** A special flag must be passed to sqlite3WhereBegin() to slightly - ** modify behaviour as follows: + ** modify behavior as follows: ** ** + If the query is a "SELECT min(x)", then the loop coded by ** where.c should not iterate over any values with a NULL value @@ -4457,10 +4650,17 @@ int sqlite3Select( ** Refer to code and comments in where.c for details. */ ExprList *pMinMax = 0; - u8 flag = minMaxQuery(p); + u8 flag = WHERE_ORDERBY_NORMAL; + + assert( p->pGroupBy==0 ); + assert( flag==0 ); + if( p->pHaving==0 ){ + flag = minMaxQuery(&sAggInfo, &pMinMax); + } + assert( flag==0 || (pMinMax!=0 && pMinMax->nExpr==1) ); + if( flag ){ - assert( !ExprHasProperty(p->pEList->a[0].pExpr, EP_xIsSelect) ); - pMinMax = sqlite3ExprListDup(db, p->pEList->a[0].pExpr->x.pList,0); + pMinMax = sqlite3ExprListDup(db, pMinMax, 0); pDel = pMinMax; if( pMinMax && !db->mallocFailed ){ pMinMax->a[0].sortOrder = flag!=WHERE_ORDERBY_MIN ?1:0; @@ -4473,13 +4673,14 @@ int sqlite3Select( ** of output. */ resetAccumulator(pParse, &sAggInfo); - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pMinMax,0,flag,0); + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pMinMax,0,flag,0); if( pWInfo==0 ){ sqlite3ExprListDelete(db, pDel); goto select_end; } updateAccumulator(pParse, &sAggInfo); - if( !pMinMax && flag ){ + assert( pMinMax==0 || pMinMax->nExpr==1 ); + if( pWInfo->nOBSat>0 ){ sqlite3VdbeAddOp2(v, OP_Goto, 0, pWInfo->iBreak); VdbeComment((v, "%s() by index", (flag==WHERE_ORDERBY_MIN?"min":"max"))); @@ -4490,7 +4691,7 @@ int sqlite3Select( pOrderBy = 0; sqlite3ExprIfFalse(pParse, pHaving, addrEnd, SQLITE_JUMPIFNULL); - selectInnerLoop(pParse, p, p->pEList, 0, 0, 0, -1, + selectInnerLoop(pParse, p, p->pEList, 0, 0, 0, 0, pDest, addrEnd, addrEnd); sqlite3ExprListDelete(db, pDel); } @@ -4498,7 +4699,7 @@ int sqlite3Select( } /* endif aggregate query */ - if( distinct>=0 ){ + if( sDistinct.eTnctType==WHERE_DISTINCT_UNORDERED ){ explainTempTable(pParse, "DISTINCT"); } @@ -4615,7 +4816,10 @@ void sqlite3ExplainSelect(Vdbe *pVdbe, Select *p){ sqlite3ExplainPrintf(pVdbe, "(null-select)"); return; } - while( p->pPrior ) p = p->pPrior; + while( p->pPrior ){ + p->pPrior->pNext = p; + p = p->pPrior; + } sqlite3ExplainPush(pVdbe); while( p ){ explainOneSelect(pVdbe, p); diff --git a/src/shell.c b/src/shell.c index a17d966..faaec80 100644 --- a/src/shell.c +++ b/src/shell.c @@ -90,7 +90,8 @@ static int enableTimer = 0; #define IsDigit(X) isdigit((unsigned char)X) #define ToLower(X) (char)tolower((unsigned char)X) -#if !defined(_WIN32) && !defined(WIN32) && !defined(_WRS_KERNEL) +#if !defined(_WIN32) && !defined(WIN32) && !defined(_WRS_KERNEL) \ + && !defined(__minux) #include #include @@ -541,6 +542,9 @@ static void output_c_string(FILE *out, const char *z){ if( c=='\\' ){ fputc(c, out); fputc(c, out); + }else if( c=='"' ){ + fputc('\\', out); + fputc('"', out); }else if( c=='\t' ){ fputc('\\', out); fputc('t', out); @@ -696,7 +700,7 @@ static int shell_callback(void *pArg, int nArg, char **azArg, char **azCol, int }else{ w = 0; } - if( w<=0 ){ + if( w==0 ){ w = strlen30(azCol[i] ? azCol[i] : ""); if( w<10 ) w = 10; n = strlen30(azArg && azArg[i] ? azArg[i] : p->nullvalue); @@ -706,7 +710,11 @@ static int shell_callback(void *pArg, int nArg, char **azArg, char **azCol, int p->actualWidth[i] = w; } if( p->showHeader ){ - fprintf(p->out,"%-*.*s%s",w,w,azCol[i], i==nArg-1 ? "\n": " "); + if( w<0 ){ + fprintf(p->out,"%*.*s%s",-w,-w,azCol[i], i==nArg-1 ? "\n": " "); + }else{ + fprintf(p->out,"%-*.*s%s",w,w,azCol[i], i==nArg-1 ? "\n": " "); + } } } if( p->showHeader ){ @@ -714,6 +722,7 @@ static int shell_callback(void *pArg, int nArg, char **azArg, char **azCol, int int w; if( iactualWidth) ){ w = p->actualWidth[i]; + if( w<0 ) w = -w; }else{ w = 10; } @@ -735,8 +744,13 @@ static int shell_callback(void *pArg, int nArg, char **azArg, char **azCol, int strlen30(azArg[i])>w ){ w = strlen30(azArg[i]); } - fprintf(p->out,"%-*.*s%s",w,w, - azArg[i] ? azArg[i] : p->nullvalue, i==nArg-1 ? "\n": " "); + if( w<0 ){ + fprintf(p->out,"%*.*s%s",-w,-w, + azArg[i] ? azArg[i] : p->nullvalue, i==nArg-1 ? "\n": " "); + }else{ + fprintf(p->out,"%-*.*s%s",w,w, + azArg[i] ? azArg[i] : p->nullvalue, i==nArg-1 ? "\n": " "); + } } break; } @@ -786,14 +800,14 @@ static int shell_callback(void *pArg, int nArg, char **azArg, char **azCol, int if( p->cnt++==0 && p->showHeader ){ for(i=0; iout,azCol[i] ? azCol[i] : ""); - fprintf(p->out, "%s", p->separator); + if(iout, "%s", p->separator); } fprintf(p->out,"\n"); } if( azArg==0 ) break; for(i=0; iout, azArg[i] ? azArg[i] : p->nullvalue); - fprintf(p->out, "%s", p->separator); + if(iout, "%s", p->separator); } fprintf(p->out,"\n"); break; @@ -1416,9 +1430,10 @@ static char zHelp[] = " list Values delimited by .separator string\n" " tabs Tab-separated values\n" " tcl TCL list elements\n" - ".nullvalue STRING Print STRING in place of NULL values\n" + ".nullvalue STRING Use STRING in place of NULL values\n" ".output FILENAME Send output to FILENAME\n" ".output stdout Send output to the screen\n" + ".print STRING... Print literal STRING\n" ".prompt MAIN CONTINUE Replace the standard prompts\n" ".quit Exit this program\n" ".read FILENAME Execute SQL in FILENAME\n" @@ -1511,17 +1526,55 @@ static void resolve_backslashes(char *z){ ** Interpret zArg as a boolean value. Return either 0 or 1. */ static int booleanValue(char *zArg){ - int val = atoi(zArg); - int j; - for(j=0; zArg[j]; j++){ - zArg[j] = ToLower(zArg[j]); + int i; + for(i=0; zArg[i]>='0' && zArg[i]<='9'; i++){} + if( i>0 && zArg[i]==0 ) return atoi(zArg); + if( sqlite3_stricmp(zArg, "on")==0 || sqlite3_stricmp(zArg,"yes")==0 ){ + return 1; } - if( strcmp(zArg,"on")==0 ){ - val = 1; - }else if( strcmp(zArg,"yes")==0 ){ - val = 1; + if( sqlite3_stricmp(zArg, "off")==0 || sqlite3_stricmp(zArg,"no")==0 ){ + return 0; + } + fprintf(stderr, "ERROR: Not a boolean value: \"%s\". Assuming \"no\".\n", + zArg); + return 0; +} + +/* +** Interpret zArg as an integer value, possibly with suffixes. +*/ +static sqlite3_int64 integerValue(const char *zArg){ + sqlite3_int64 v = 0; + static const struct { char *zSuffix; int iMult; } aMult[] = { + { "KiB", 1024 }, + { "MiB", 1024*1024 }, + { "GiB", 1024*1024*1024 }, + { "KB", 1000 }, + { "MB", 1000000 }, + { "GB", 1000000000 }, + { "K", 1000 }, + { "M", 1000000 }, + { "G", 1000000000 }, + }; + int i; + int isNeg = 0; + if( zArg[0]=='-' ){ + isNeg = 1; + zArg++; + }else if( zArg[0]=='+' ){ + zArg++; + } + while( isdigit(zArg[0]) ){ + v = v*10 + zArg[0] - '0'; + zArg++; + } + for(i=0; i=3 && strncmp(azArg[0], "backup", n)==0 && nArg>1 && nArg<4){ - const char *zDestFile; - const char *zDb; + if( c=='b' && n>=3 && strncmp(azArg[0], "backup", n)==0 ){ + const char *zDestFile = 0; + const char *zDb = 0; + const char *zKey = 0; sqlite3 *pDest; sqlite3_backup *pBackup; - if( nArg==2 ){ - zDestFile = azArg[1]; - zDb = "main"; - }else{ - zDestFile = azArg[2]; - zDb = azArg[1]; + int j; + for(j=1; jdb, zDb); if( pBackup==0 ){ @@ -1728,7 +1807,8 @@ static int do_meta_command(char *zLine, struct callback_data *p){ p->echoOn = booleanValue(azArg[1]); }else - if( c=='e' && strncmp(azArg[0], "exit", n)==0 && nArg==1 ){ + if( c=='e' && strncmp(azArg[0], "exit", n)==0 ){ + if( nArg>1 && (rc = atoi(azArg[1]))!=0 ) exit(rc); rc = 2; }else @@ -2007,6 +2087,7 @@ static int do_meta_command(char *zLine, struct callback_data *p){ p->mode = MODE_Html; }else if( n2==3 && strncmp(azArg[1],"tcl",n2)==0 ){ p->mode = MODE_Tcl; + sqlite3_snprintf(sizeof(p->separator), p->separator, " "); }else if( n2==3 && strncmp(azArg[1],"csv",n2)==0 ){ p->mode = MODE_Csv; sqlite3_snprintf(sizeof(p->separator), p->separator, ","); @@ -2070,6 +2151,15 @@ static int do_meta_command(char *zLine, struct callback_data *p){ } }else + if( c=='p' && n>=3 && strncmp(azArg[0], "print", n)==0 ){ + int i; + for(i=1; i1 ) fprintf(p->out, " "); + fprintf(p->out, "%s", azArg[i]); + } + fprintf(p->out, "\n"); + }else + if( c=='p' && strncmp(azArg[0], "prompt", n)==0 && (nArg==2 || nArg==3)){ if( nArg >= 2) { strncpy(mainPrompt,azArg[1],(int)ArraySize(mainPrompt)-1); @@ -2188,8 +2278,7 @@ static int do_meta_command(char *zLine, struct callback_data *p){ " SELECT sql, type, tbl_name, name, rowid FROM sqlite_temp_master) " "WHERE lower(tbl_name) LIKE shellstatic()" " AND type!='meta' AND sql NOTNULL " - "ORDER BY substr(type,2,1), " - " CASE type WHEN 'view' THEN rowid ELSE name END", + "ORDER BY rowid", callback, &data, &zErrMsg); zShellStatic = 0; } @@ -2200,8 +2289,7 @@ static int do_meta_command(char *zLine, struct callback_data *p){ " FROM sqlite_master UNION ALL" " SELECT sql, type, tbl_name, name, rowid FROM sqlite_temp_master) " "WHERE type!='meta' AND sql NOTNULL AND name NOT LIKE 'sqlite_%'" - "ORDER BY substr(type,2,1)," - " CASE type WHEN 'view' THEN rowid ELSE name END", + "ORDER BY rowid", callback, &data, &zErrMsg ); } @@ -2323,9 +2411,9 @@ static int do_meta_command(char *zLine, struct callback_data *p){ for(i=0; iout, "%s%-*s", zSp, maxlen, azResult[j] ? azResult[j] : ""); } - printf("\n"); + fprintf(p->out, "\n"); } } for(ii=0; iidb, opt); - printf("%d (0x%08x)\n", rc, rc); + fprintf(p->out, "%d (0x%08x)\n", rc, rc); } else { fprintf(stderr,"Error: testctrl %s takes a single int option\n", azArg[1]); @@ -2395,7 +2483,7 @@ static int do_meta_command(char *zLine, struct callback_data *p){ case SQLITE_TESTCTRL_PRNG_RESET: if( nArg==2 ){ rc = sqlite3_test_control(testctrl); - printf("%d (0x%08x)\n", rc, rc); + fprintf(p->out, "%d (0x%08x)\n", rc, rc); } else { fprintf(stderr,"Error: testctrl %s takes no options\n", azArg[1]); } @@ -2404,9 +2492,9 @@ static int do_meta_command(char *zLine, struct callback_data *p){ /* sqlite3_test_control(int, uint) */ case SQLITE_TESTCTRL_PENDING_BYTE: if( nArg==3 ){ - unsigned int opt = (unsigned int)atoi(azArg[2]); + unsigned int opt = (unsigned int)integerValue(azArg[2]); rc = sqlite3_test_control(testctrl, opt); - printf("%d (0x%08x)\n", rc, rc); + fprintf(p->out, "%d (0x%08x)\n", rc, rc); } else { fprintf(stderr,"Error: testctrl %s takes a single unsigned" " int option\n", azArg[1]); @@ -2419,7 +2507,7 @@ static int do_meta_command(char *zLine, struct callback_data *p){ if( nArg==3 ){ int opt = atoi(azArg[2]); rc = sqlite3_test_control(testctrl, opt); - printf("%d (0x%08x)\n", rc, rc); + fprintf(p->out, "%d (0x%08x)\n", rc, rc); } else { fprintf(stderr,"Error: testctrl %s takes a single int option\n", azArg[1]); @@ -2432,7 +2520,7 @@ static int do_meta_command(char *zLine, struct callback_data *p){ if( nArg==3 ){ const char *opt = azArg[2]; rc = sqlite3_test_control(testctrl, opt); - printf("%d (0x%08x)\n", rc, rc); + fprintf(p->out, "%d (0x%08x)\n", rc, rc); } else { fprintf(stderr,"Error: testctrl %s takes a single char * option\n", azArg[1]); @@ -2477,7 +2565,7 @@ static int do_meta_command(char *zLine, struct callback_data *p){ }else if( c=='v' && strncmp(azArg[0], "version", n)==0 ){ - printf("SQLite %s %s\n" /*extra-version-info*/, + fprintf(p->out, "SQLite %s %s\n" /*extra-version-info*/, sqlite3_libversion(), sqlite3_sourceid()); }else @@ -2487,12 +2575,19 @@ static int do_meta_command(char *zLine, struct callback_data *p){ if( p->db ){ sqlite3_file_control(p->db, zDbName, SQLITE_FCNTL_VFSNAME, &zVfsName); if( zVfsName ){ - printf("%s\n", zVfsName); + fprintf(p->out, "%s\n", zVfsName); sqlite3_free(zVfsName); } } }else +#if defined(SQLITE_DEBUG) && defined(SQLITE_ENABLE_WHERETRACE) + if( c=='w' && strncmp(azArg[0], "wheretrace", n)==0 ){ + extern int sqlite3WhereTrace; + sqlite3WhereTrace = booleanValue(azArg[1]); + }else +#endif + if( c=='w' && strncmp(azArg[0], "width", n)==0 && nArg>1 ){ int j; assert( nArg<=ArraySize(azArg) ); @@ -2675,6 +2770,10 @@ static int process_input(struct callback_data *p, FILE *in){ free(zSql); zSql = 0; nSql = 0; + }else if( zSql && _all_whitespace(zSql) ){ + free(zSql); + zSql = 0; + nSql = 0; } } if( zSql ){ @@ -2684,7 +2783,7 @@ static int process_input(struct callback_data *p, FILE *in){ free(zSql); } free(zLine); - return errCnt; + return errCnt>0; } /* @@ -2797,21 +2896,25 @@ static const char zOptions[] = " -bail stop after hitting an error\n" " -batch force batch I/O\n" " -column set output mode to 'column'\n" - " -cmd command run \"command\" before reading stdin\n" + " -cmd COMMAND run \"COMMAND\" before reading stdin\n" " -csv set output mode to 'csv'\n" " -echo print commands before execution\n" - " -init filename read/process named file\n" + " -init FILENAME read/process named file\n" " -[no]header turn headers on or off\n" +#if defined(SQLITE_ENABLE_MEMSYS3) || defined(SQLITE_ENABLE_MEMSYS5) + " -heap SIZE Size of heap for memsys3 or memsys5\n" +#endif " -help show this message\n" " -html set output mode to HTML\n" " -interactive force interactive I/O\n" " -line set output mode to 'line'\n" " -list set output mode to 'list'\n" + " -mmap N default mmap size set to N\n" #ifdef SQLITE_ENABLE_MULTIPLEX " -multiplex enable the multiplexor VFS\n" #endif - " -nullvalue 'text' set text string for NULL values\n" - " -separator 'x' set output field separator (|)\n" + " -nullvalue TEXT set text string for NULL values. Default ''\n" + " -separator SEP set output field separator. Default: '|'\n" " -stats print memory stats before each finalize\n" " -version show SQLite version\n" " -vfs NAME use NAME as the default VFS\n" @@ -2847,6 +2950,19 @@ static void main_init(struct callback_data *data) { sqlite3_config(SQLITE_CONFIG_SINGLETHREAD); } +/* +** Get the argument to an --option. Throw an error and die if no argument +** is available. +*/ +static char *cmdline_option_value(int argc, char **argv, int i){ + if( i==argc ){ + fprintf(stderr, "%s: Error: missing argument to %s\n", + argv[0], argv[argc-1]); + exit(1); + } + return argv[i]; +} + int main(int argc, char **argv){ char *zErrMsg = 0; struct callback_data data; @@ -2876,24 +2992,35 @@ int main(int argc, char **argv){ ** the size of the alternative malloc heap, ** and the first command to execute. */ - for(i=1; i0x7fff0000 ) szHeap = 0x7fff0000; sqlite3_config(SQLITE_CONFIG_HEAP, malloc((int)szHeap), (int)szHeap, 64); #endif @@ -2927,8 +3049,11 @@ int main(int argc, char **argv){ extern int sqlite3_multiple_initialize(const char*,int); sqlite3_multiplex_initialize(0, 1); #endif + }else if( strcmp(z,"-mmap")==0 ){ + sqlite3_int64 sz = integerValue(cmdline_option_value(argc,argv,++i)); + sqlite3_config(SQLITE_CONFIG_MMAP_SIZE, sz, sz); }else if( strcmp(z,"-vfs")==0 ){ - sqlite3_vfs *pVfs = sqlite3_vfs_find(argv[++i]); + sqlite3_vfs *pVfs = sqlite3_vfs_find(cmdline_option_value(argc,argv,++i)); if( pVfs ){ sqlite3_vfs_register(pVfs, 1); }else{ @@ -2937,31 +3062,15 @@ int main(int argc, char **argv){ } } } - if( i=argc){ - fprintf(stderr,"%s: Error: missing argument for option: %s\n", - Argv0, z); - fprintf(stderr,"Use -help for a list of options.\n"); - return 1; - } sqlite3_snprintf(sizeof(data.separator), data.separator, - "%.*s",(int)sizeof(data.separator)-1,argv[i]); + "%s",cmdline_option_value(argc,argv,++i)); }else if( strcmp(z,"-nullvalue")==0 ){ - i++; - if(i>=argc){ - fprintf(stderr,"%s: Error: missing argument for option: %s\n", - Argv0, z); - fprintf(stderr,"Use -help for a list of options.\n"); - return 1; - } sqlite3_snprintf(sizeof(data.nullvalue), data.nullvalue, - "%.*s",(int)sizeof(data.nullvalue)-1,argv[i]); + "%s",cmdline_option_value(argc,argv,++i)); }else if( strcmp(z,"-header")==0 ){ data.showHeader = 1; }else if( strcmp(z,"-noheader")==0 ){ @@ -3041,6 +3137,8 @@ int main(int argc, char **argv){ stdin_is_interactive = 0; }else if( strcmp(z,"-heap")==0 ){ i++; + }else if( strcmp(z,"-mmap")==0 ){ + i++; }else if( strcmp(z,"-vfs")==0 ){ i++; #ifdef SQLITE_ENABLE_VFSTRACE @@ -3055,11 +3153,10 @@ int main(int argc, char **argv){ usage(1); }else if( strcmp(z,"-cmd")==0 ){ if( i==argc-1 ) break; - i++; - z = argv[i]; + z = cmdline_option_value(argc,argv,++i); if( z[0]=='.' ){ rc = do_meta_command(z, &data); - if( rc && bail_on_error ) return rc; + if( rc && bail_on_error ) return rc==2 ? 0 : rc; }else{ open_db(&data); rc = shell_exec(data.db, z, shell_callback, &data, &zErrMsg); @@ -3083,6 +3180,7 @@ int main(int argc, char **argv){ */ if( zFirstCmd[0]=='.' ){ rc = do_meta_command(zFirstCmd, &data); + if( rc==2 ) rc = 0; }else{ open_db(&data); rc = shell_exec(data.db, zFirstCmd, shell_callback, &data, &zErrMsg); @@ -3102,7 +3200,13 @@ int main(int argc, char **argv){ char *zHistory = 0; int nHistory; printf( +/* BEGIN SQLCIPHER */ +#ifdef SQLITE_HAS_CODEC + "SQLCipher version %s %.19s\n" /*extra-version-info*/ +#else "SQLite version %s %.19s\n" /*extra-version-info*/ +#endif +/* END SQLCIPHER */ "Enter \".help\" for instructions\n" "Enter SQL statements terminated with a \";\"\n", sqlite3_libversion(), sqlite3_sourceid() diff --git a/src/sqlcipher.h b/src/sqlcipher.h new file mode 100644 index 0000000..41f8f83 --- /dev/null +++ b/src/sqlcipher.h @@ -0,0 +1,75 @@ +/* +** SQLCipher +** sqlcipher.h developed by Stephen Lombardo (Zetetic LLC) +** sjlombardo at zetetic dot net +** http://zetetic.net +** +** Copyright (c) 2008, ZETETIC LLC +** All rights reserved. +** +** Redistribution and use in source and binary forms, with or without +** modification, are permitted provided that the following conditions are met: +** * Redistributions of source code must retain the above copyright +** notice, this list of conditions and the following disclaimer. +** * Redistributions in binary form must reproduce the above copyright +** notice, this list of conditions and the following disclaimer in the +** documentation and/or other materials provided with the distribution. +** * Neither the name of the ZETETIC LLC nor the +** names of its contributors may be used to endorse or promote products +** derived from this software without specific prior written permission. +** +** THIS SOFTWARE IS PROVIDED BY ZETETIC LLC ''AS IS'' AND ANY +** EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +** WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +** DISCLAIMED. IN NO EVENT SHALL ZETETIC LLC BE LIABLE FOR ANY +** DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +** (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +** LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +** ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +** (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +** SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +** +*/ +/* BEGIN SQLCIPHER */ +#ifdef SQLITE_HAS_CODEC +#ifndef SQLCIPHER_H +#define SQLCIPHER_H + + +typedef struct { + int (*activate)(void *ctx); + int (*deactivate)(void *ctx); + const char* (*get_provider_name)(void *ctx); + int (*add_random)(void *ctx, void *buffer, int length); + int (*random)(void *ctx, void *buffer, int length); + int (*hmac)(void *ctx, unsigned char *hmac_key, int key_sz, unsigned char *in, int in_sz, unsigned char *in2, int in2_sz, unsigned char *out); + int (*kdf)(void *ctx, const char *pass, int pass_sz, unsigned char* salt, int salt_sz, int workfactor, int key_sz, unsigned char *key); + int (*cipher)(void *ctx, int mode, unsigned char *key, int key_sz, unsigned char *iv, unsigned char *in, int in_sz, unsigned char *out); + int (*set_cipher)(void *ctx, const char *cipher_name); + const char* (*get_cipher)(void *ctx); + int (*get_key_sz)(void *ctx); + int (*get_iv_sz)(void *ctx); + int (*get_block_sz)(void *ctx); + int (*get_hmac_sz)(void *ctx); + int (*ctx_copy)(void *target_ctx, void *source_ctx); + int (*ctx_cmp)(void *c1, void *c2); + int (*ctx_init)(void **ctx); + int (*ctx_free)(void **ctx); +} sqlcipher_provider; + +/* utility functions */ +void sqlcipher_free(void *ptr, int sz); +void* sqlcipher_malloc(int sz); +void* sqlcipher_memset(void *v, unsigned char value, int len); +int sqlcipher_ismemset(const void *v, unsigned char value, int len); +int sqlcipher_memcmp(const void *v0, const void *v1, int len); +void sqlcipher_free(void *, int); + +/* provider interfaces */ +int sqlcipher_register_provider(sqlcipher_provider *p); +sqlcipher_provider* sqlcipher_get_provider(); + +#endif +#endif +/* END SQLCIPHER */ + diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 9a31e22..6608823 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -283,7 +283,7 @@ typedef sqlite_uint64 sqlite3_uint64; ** [sqlite3_blob_close | close] all [BLOB handles], and ** [sqlite3_backup_finish | finish] all [sqlite3_backup] objects associated ** with the [sqlite3] object prior to attempting to close the object. ^If -** sqlite3_close() is called on a [database connection] that still has +** sqlite3_close_v2() is called on a [database connection] that still has ** outstanding [prepared statements], [BLOB handles], and/or ** [sqlite3_backup] objects then it returns SQLITE_OK but the deallocation ** of resources is deferred until all [prepared statements], [BLOB handles], @@ -420,6 +420,8 @@ int sqlite3_exec( #define SQLITE_FORMAT 24 /* Auxiliary database format error */ #define SQLITE_RANGE 25 /* 2nd parameter to sqlite3_bind out of range */ #define SQLITE_NOTADB 26 /* File opened that is not a database file */ +#define SQLITE_NOTICE 27 /* Notifications from sqlite3_log() */ +#define SQLITE_WARNING 28 /* Warnings from sqlite3_log() */ #define SQLITE_ROW 100 /* sqlite3_step() has another row ready */ #define SQLITE_DONE 101 /* sqlite3_step() has finished executing */ /* end-of-error-codes */ @@ -469,14 +471,29 @@ int sqlite3_exec( #define SQLITE_IOERR_SHMLOCK (SQLITE_IOERR | (20<<8)) #define SQLITE_IOERR_SHMMAP (SQLITE_IOERR | (21<<8)) #define SQLITE_IOERR_SEEK (SQLITE_IOERR | (22<<8)) +#define SQLITE_IOERR_DELETE_NOENT (SQLITE_IOERR | (23<<8)) +#define SQLITE_IOERR_MMAP (SQLITE_IOERR | (24<<8)) #define SQLITE_LOCKED_SHAREDCACHE (SQLITE_LOCKED | (1<<8)) #define SQLITE_BUSY_RECOVERY (SQLITE_BUSY | (1<<8)) #define SQLITE_CANTOPEN_NOTEMPDIR (SQLITE_CANTOPEN | (1<<8)) #define SQLITE_CANTOPEN_ISDIR (SQLITE_CANTOPEN | (2<<8)) +#define SQLITE_CANTOPEN_FULLPATH (SQLITE_CANTOPEN | (3<<8)) #define SQLITE_CORRUPT_VTAB (SQLITE_CORRUPT | (1<<8)) #define SQLITE_READONLY_RECOVERY (SQLITE_READONLY | (1<<8)) #define SQLITE_READONLY_CANTLOCK (SQLITE_READONLY | (2<<8)) +#define SQLITE_READONLY_ROLLBACK (SQLITE_READONLY | (3<<8)) #define SQLITE_ABORT_ROLLBACK (SQLITE_ABORT | (2<<8)) +#define SQLITE_CONSTRAINT_CHECK (SQLITE_CONSTRAINT | (1<<8)) +#define SQLITE_CONSTRAINT_COMMITHOOK (SQLITE_CONSTRAINT | (2<<8)) +#define SQLITE_CONSTRAINT_FOREIGNKEY (SQLITE_CONSTRAINT | (3<<8)) +#define SQLITE_CONSTRAINT_FUNCTION (SQLITE_CONSTRAINT | (4<<8)) +#define SQLITE_CONSTRAINT_NOTNULL (SQLITE_CONSTRAINT | (5<<8)) +#define SQLITE_CONSTRAINT_PRIMARYKEY (SQLITE_CONSTRAINT | (6<<8)) +#define SQLITE_CONSTRAINT_TRIGGER (SQLITE_CONSTRAINT | (7<<8)) +#define SQLITE_CONSTRAINT_UNIQUE (SQLITE_CONSTRAINT | (8<<8)) +#define SQLITE_CONSTRAINT_VTAB (SQLITE_CONSTRAINT | (9<<8)) +#define SQLITE_NOTICE_RECOVER_WAL (SQLITE_NOTICE | (1<<8)) +#define SQLITE_NOTICE_RECOVER_ROLLBACK (SQLITE_NOTICE | (2<<8)) /* ** CAPI3REF: Flags For File Open Operations @@ -716,6 +733,9 @@ struct sqlite3_io_methods { void (*xShmBarrier)(sqlite3_file*); int (*xShmUnmap)(sqlite3_file*, int deleteFlag); /* Methods above are valid for version 2 */ + int (*xFetch)(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp); + int (*xUnfetch)(sqlite3_file*, sqlite3_int64 iOfst, void *p); + /* Methods above are valid for version 3 */ /* Additional methods may be added in future releases */ }; @@ -850,6 +870,38 @@ struct sqlite3_io_methods { ** compilation of the PRAGMA fails with an error. ^The [SQLITE_FCNTL_PRAGMA] ** file control occurs at the beginning of pragma statement analysis and so ** it is able to override built-in [PRAGMA] statements. +** +**
  • [[SQLITE_FCNTL_BUSYHANDLER]] +** ^The [SQLITE_FCNTL_BUSYHANDLER] +** file-control may be invoked by SQLite on the database file handle +** shortly after it is opened in order to provide a custom VFS with access +** to the connections busy-handler callback. The argument is of type (void **) +** - an array of two (void *) values. The first (void *) actually points +** to a function of type (int (*)(void *)). In order to invoke the connections +** busy-handler, this function should be invoked with the second (void *) in +** the array as the only argument. If it returns non-zero, then the operation +** should be retried. If it returns zero, the custom VFS should abandon the +** current operation. +** +**
  • [[SQLITE_FCNTL_TEMPFILENAME]] +** ^Application can invoke the [SQLITE_FCNTL_TEMPFILENAME] file-control +** to have SQLite generate a +** temporary filename using the same algorithm that is followed to generate +** temporary filenames for TEMP tables and other internal uses. The +** argument should be a char** which will be filled with the filename +** written into memory obtained from [sqlite3_malloc()]. The caller should +** invoke [sqlite3_free()] on the result to avoid a memory leak. +** +**
  • [[SQLITE_FCNTL_MMAP_SIZE]] +** The [SQLITE_FCNTL_MMAP_SIZE] file control is used to query or set the +** maximum number of bytes that will be used for memory-mapped I/O. +** The argument is a pointer to a value of type sqlite3_int64 that +** is an advisory maximum number of bytes in the file to memory map. The +** pointer is overwritten with the old value. The limit is not changed if +** the value originally pointed to is negative, and so the current limit +** can be queried by passing in a pointer to a negative number. This +** file-control is used internally to implement [PRAGMA mmap_size]. +** ** */ #define SQLITE_FCNTL_LOCKSTATE 1 @@ -866,6 +918,9 @@ struct sqlite3_io_methods { #define SQLITE_FCNTL_VFSNAME 12 #define SQLITE_FCNTL_POWERSAFE_OVERWRITE 13 #define SQLITE_FCNTL_PRAGMA 14 +#define SQLITE_FCNTL_BUSYHANDLER 15 +#define SQLITE_FCNTL_TEMPFILENAME 16 +#define SQLITE_FCNTL_MMAP_SIZE 18 /* ** CAPI3REF: Mutex Handle @@ -1532,7 +1587,9 @@ struct sqlite3_mem_methods { ** page cache implementation into that object.)^ ** ** [[SQLITE_CONFIG_LOG]]
    SQLITE_CONFIG_LOG
    -**
    ^The SQLITE_CONFIG_LOG option takes two arguments: a pointer to a +**
    The SQLITE_CONFIG_LOG option is used to configure the SQLite +** global [error log]. +** (^The SQLITE_CONFIG_LOG option takes two arguments: a pointer to a ** function with a call signature of void(*)(void*,int,const char*), ** and a pointer to void. ^If the function pointer is not NULL, it is ** invoked by [sqlite3_log()] to process each logging event. ^If the @@ -1562,10 +1619,54 @@ struct sqlite3_mem_methods { ** disabled. The default value may be changed by compiling with the ** [SQLITE_USE_URI] symbol defined. ** +** [[SQLITE_CONFIG_COVERING_INDEX_SCAN]]
    SQLITE_CONFIG_COVERING_INDEX_SCAN +**
    This option takes a single integer argument which is interpreted as +** a boolean in order to enable or disable the use of covering indices for +** full table scans in the query optimizer. The default setting is determined +** by the [SQLITE_ALLOW_COVERING_INDEX_SCAN] compile-time option, or is "on" +** if that compile-time option is omitted. +** The ability to disable the use of covering indices for full table scans +** is because some incorrectly coded legacy applications might malfunction +** malfunction when the optimization is enabled. Providing the ability to +** disable the optimization allows the older, buggy application code to work +** without change even with newer versions of SQLite. +** ** [[SQLITE_CONFIG_PCACHE]] [[SQLITE_CONFIG_GETPCACHE]] **
    SQLITE_CONFIG_PCACHE and SQLITE_CONFIG_GETPCACHE **
    These options are obsolete and should not be used by new code. ** They are retained for backwards compatibility but are now no-ops. +**
    +** +** [[SQLITE_CONFIG_SQLLOG]] +**
    SQLITE_CONFIG_SQLLOG +**
    This option is only available if sqlite is compiled with the +** [SQLITE_ENABLE_SQLLOG] pre-processor macro defined. The first argument should +** be a pointer to a function of type void(*)(void*,sqlite3*,const char*, int). +** The second should be of type (void*). The callback is invoked by the library +** in three separate circumstances, identified by the value passed as the +** fourth parameter. If the fourth parameter is 0, then the database connection +** passed as the second argument has just been opened. The third argument +** points to a buffer containing the name of the main database file. If the +** fourth parameter is 1, then the SQL statement that the third parameter +** points to has just been executed. Or, if the fourth parameter is 2, then +** the connection being passed as the second parameter is being closed. The +** third parameter is passed NULL In this case. An example of using this +** configuration option can be seen in the "test_sqllog.c" source file in +** the canonical SQLite source tree.
    +** +** [[SQLITE_CONFIG_MMAP_SIZE]] +**
    SQLITE_CONFIG_MMAP_SIZE +**
    SQLITE_CONFIG_MMAP_SIZE takes two 64-bit integer (sqlite3_int64) values +** that are the default mmap size limit (the default setting for +** [PRAGMA mmap_size]) and the maximum allowed mmap size limit. +** The default setting can be overridden by each database connection using +** either the [PRAGMA mmap_size] command, or by using the +** [SQLITE_FCNTL_MMAP_SIZE] file control. The maximum allowed mmap size +** cannot be changed at run-time. Nor may the maximum allowed mmap size +** exceed the compile-time maximum mmap size set by the +** [SQLITE_MAX_MMAP_SIZE] compile-time option. +** If either argument to this option is negative, then that argument is +** changed to its compile-time default. ** */ #define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ @@ -1587,6 +1688,9 @@ struct sqlite3_mem_methods { #define SQLITE_CONFIG_URI 17 /* int */ #define SQLITE_CONFIG_PCACHE2 18 /* sqlite3_pcache_methods2* */ #define SQLITE_CONFIG_GETPCACHE2 19 /* sqlite3_pcache_methods2* */ +#define SQLITE_CONFIG_COVERING_INDEX_SCAN 20 /* int */ +#define SQLITE_CONFIG_SQLLOG 21 /* xSqllog, void* */ +#define SQLITE_CONFIG_MMAP_SIZE 22 /* sqlite3_int64, sqlite3_int64 */ /* ** CAPI3REF: Database Connection Configuration Options @@ -2420,6 +2524,9 @@ int sqlite3_set_authorizer( ** as each triggered subprogram is entered. The callbacks for triggers ** contain a UTF-8 SQL comment that identifies the trigger.)^ ** +** The [SQLITE_TRACE_SIZE_LIMIT] compile-time option can be used to limit +** the length of [bound parameter] expansion in the output of sqlite3_trace(). +** ** ^The callback function registered by sqlite3_profile() is invoked ** as each SQL statement finishes. ^The profile callback contains ** the original statement text and an estimate of wall-clock time @@ -2595,7 +2702,7 @@ void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** an error)^. ** ^If "ro" is specified, then the database is opened for read-only ** access, just as if the [SQLITE_OPEN_READONLY] flag had been set in the -** third argument to sqlite3_prepare_v2(). ^If the mode option is set to +** third argument to sqlite3_open_v2(). ^If the mode option is set to ** "rw", then the database is opened for read-write (but not create) ** access, as if SQLITE_OPEN_READWRITE (but not SQLITE_OPEN_CREATE) had ** been set. ^Value "rwc" is equivalent to setting both @@ -2611,7 +2718,7 @@ void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** sqlite3_open_v2(). ^Setting the cache parameter to "private" is ** equivalent to setting the SQLITE_OPEN_PRIVATECACHE bit. ** ^If sqlite3_open_v2() is used and the "cache" parameter is present in -** a URI filename, its value overrides any behaviour requested by setting +** a URI filename, its value overrides any behavior requested by setting ** SQLITE_OPEN_PRIVATECACHE or SQLITE_OPEN_SHAREDCACHE flag. ** ** @@ -2747,6 +2854,11 @@ sqlite3_int64 sqlite3_uri_int64(const char*, const char*, sqlite3_int64); ** However, the error string might be overwritten or deallocated by ** subsequent calls to other SQLite interface functions.)^ ** +** ^The sqlite3_errstr() interface returns the English-language text +** that describes the [result code], as UTF-8. +** ^(Memory to hold the error message string is managed internally +** and must not be freed by the application)^. +** ** When the serialized [threading mode] is in use, it might be the ** case that a second error occurs on a separate thread in between ** the time of the first error and the call to these interfaces. @@ -2765,6 +2877,7 @@ int sqlite3_errcode(sqlite3 *db); int sqlite3_extended_errcode(sqlite3 *db); const char *sqlite3_errmsg(sqlite3*); const void *sqlite3_errmsg16(sqlite3*); +const char *sqlite3_errstr(int); /* ** CAPI3REF: SQL Statement Object @@ -2952,7 +3065,8 @@ int sqlite3_limit(sqlite3*, int id, int newVal); **
  • ** ^If the database schema changes, instead of returning [SQLITE_SCHEMA] as it ** always used to do, [sqlite3_step()] will automatically recompile the SQL -** statement and try to run it again. +** statement and try to run it again. As many as [SQLITE_MAX_SCHEMA_RETRY] +** retries will occur before sqlite3_step() gives up and returns an error. **
    • ** **
  • @@ -3156,6 +3270,9 @@ typedef struct sqlite3_context sqlite3_context; ** parameter [SQLITE_LIMIT_VARIABLE_NUMBER] (default value: 999). ** ** ^The third argument is the value to bind to the parameter. +** ^If the third parameter to sqlite3_bind_text() or sqlite3_bind_text16() +** or sqlite3_bind_blob() is a NULL pointer then the fourth parameter +** is ignored and the end result is the same as sqlite3_bind_null(). ** ** ^(In those routines that have a fourth argument, its value is the ** number of bytes in the parameter. To be clear: the value is the @@ -3923,7 +4040,8 @@ SQLITE_DEPRECATED int sqlite3_expired(sqlite3_stmt*); SQLITE_DEPRECATED int sqlite3_transfer_bindings(sqlite3_stmt*, sqlite3_stmt*); SQLITE_DEPRECATED int sqlite3_global_recover(void); SQLITE_DEPRECATED void sqlite3_thread_cleanup(void); -SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int64,int),void*,sqlite3_int64); +SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int64,int), + void*,sqlite3_int64); #endif /* @@ -4003,14 +4121,17 @@ int sqlite3_value_numeric_type(sqlite3_value*); ** In those cases, sqlite3_aggregate_context() might be called for the ** first time from within xFinal().)^ ** -** ^The sqlite3_aggregate_context(C,N) routine returns a NULL pointer if N is -** less than or equal to zero or if a memory allocate error occurs. +** ^The sqlite3_aggregate_context(C,N) routine returns a NULL pointer +** when first called if N is less than or equal to zero or if a memory +** allocate error occurs. ** ** ^(The amount of space allocated by sqlite3_aggregate_context(C,N) is ** determined by the N parameter on first successful call. Changing the ** value of N in subsequent call to sqlite3_aggregate_context() within ** the same aggregate function instance will not resize the memory -** allocation.)^ +** allocation.)^ Within the xFinal callback, it is customary to set +** N=0 in calls to sqlite3_aggregate_context(C,N) so that no +** pointless memory allocations occur. ** ** ^SQLite automatically frees the memory allocated by ** sqlite3_aggregate_context() when the aggregate query concludes. @@ -4108,7 +4229,7 @@ void sqlite3_set_auxdata(sqlite3_context*, int N, void*, void (*)(void*)); ** the content before returning. ** ** The typedef is necessary to work around problems in certain -** C++ compilers. See ticket #2191. +** C++ compilers. */ typedef void (*sqlite3_destructor_type)(void*); #define SQLITE_STATIC ((sqlite3_destructor_type)0) @@ -4727,6 +4848,9 @@ void *sqlite3_update_hook( ** future releases of SQLite. Applications that care about shared ** cache setting should set it explicitly. ** +** This interface is threadsafe on processors where writing a +** 32-bit integer is atomic. +** ** See Also: [SQLite Shared-Cache Mode] */ int sqlite3_enable_shared_cache(int); @@ -4904,11 +5028,20 @@ int sqlite3_table_column_metadata( ** ^This interface loads an SQLite extension library from the named file. ** ** ^The sqlite3_load_extension() interface attempts to load an -** SQLite extension library contained in the file zFile. +** [SQLite extension] library contained in the file zFile. If +** the file cannot be loaded directly, attempts are made to load +** with various operating-system specific extensions added. +** So for example, if "samplelib" cannot be loaded, then names like +** "samplelib.so" or "samplelib.dylib" or "samplelib.dll" might +** be tried also. ** ** ^The entry point is zProc. -** ^zProc may be 0, in which case the name of the entry point -** defaults to "sqlite3_extension_init". +** ^(zProc may be 0, in which case SQLite will try to come up with an +** entry point name on its own. It first tries "sqlite3_extension_init". +** If that does not work, it constructs a name "sqlite3_X_init" where the +** X is consists of the lower-case equivalent of all ASCII alphabetic +** characters in the filename from the last "/" to the first following +** "." and omitting any initial "lib".)^ ** ^The sqlite3_load_extension() interface returns ** [SQLITE_OK] on success and [SQLITE_ERROR] if something goes wrong. ** ^If an error occurs and pzErrMsg is not 0, then the @@ -4934,11 +5067,11 @@ int sqlite3_load_extension( ** CAPI3REF: Enable Or Disable Extension Loading ** ** ^So as not to open security holes in older applications that are -** unprepared to deal with extension loading, and as a means of disabling -** extension loading while evaluating user-entered SQL, the following API +** unprepared to deal with [extension loading], and as a means of disabling +** [extension loading] while evaluating user-entered SQL, the following API ** is provided to turn the [sqlite3_load_extension()] mechanism on and off. ** -** ^Extension loading is off by default. See ticket #1863. +** ^Extension loading is off by default. ** ^Call the sqlite3_enable_load_extension() routine with onoff==1 ** to turn extension loading on and call it with onoff==0 to turn ** it back off again. @@ -4950,7 +5083,7 @@ int sqlite3_enable_load_extension(sqlite3 *db, int onoff); ** ** ^This interface causes the xEntryPoint() function to be invoked for ** each new [database connection] that is created. The idea here is that -** xEntryPoint() is the entry point for a statically linked SQLite extension +** xEntryPoint() is the entry point for a statically linked [SQLite extension] ** that is to be automatically loaded into all new database connections. ** ** ^(Even though the function prototype shows that xEntryPoint() takes @@ -6301,7 +6434,7 @@ struct sqlite3_pcache_page { ** parameter to help it determined what action to take: ** **
    • -**
    createFlag Behaviour when page is not already in cache +**
    createFlag Behavior when page is not already in cache **
    0 Do not allocate a new page. Return NULL. **
    1 Allocate a new page if it easy and convenient to do so. ** Otherwise return NULL. @@ -6730,10 +6863,25 @@ int sqlite3_unlock_notify( int sqlite3_stricmp(const char *, const char *); int sqlite3_strnicmp(const char *, const char *, int); +/* +** CAPI3REF: String Globbing +* +** ^The [sqlite3_strglob(P,X)] interface returns zero if string X matches +** the glob pattern P, and it returns non-zero if string X does not match +** the glob pattern P. ^The definition of glob pattern matching used in +** [sqlite3_strglob(P,X)] is the same as for the "X GLOB P" operator in the +** SQL dialect used by SQLite. ^The sqlite3_strglob(P,X) function is case +** sensitive. +** +** Note that this routine returns zero on a match and non-zero if the strings +** do not match, the same as [sqlite3_stricmp()] and [sqlite3_strnicmp()]. +*/ +int sqlite3_strglob(const char *zGlob, const char *zStr); + /* ** CAPI3REF: Error Logging Interface ** -** ^The [sqlite3_log()] interface writes a message into the error log +** ^The [sqlite3_log()] interface writes a message into the [error log] ** established by the [SQLITE_CONFIG_LOG] option to [sqlite3_config()]. ** ^If logging is enabled, the zFormat string and subsequent arguments are ** used with [sqlite3_snprintf()] to generate the final output string. diff --git a/src/sqlite3ext.h b/src/sqlite3ext.h index 5abcde2..928bb3b 100644 --- a/src/sqlite3ext.h +++ b/src/sqlite3ext.h @@ -236,6 +236,20 @@ struct sqlite3_api_routines { int (*blob_reopen)(sqlite3_blob*,sqlite3_int64); int (*vtab_config)(sqlite3*,int op,...); int (*vtab_on_conflict)(sqlite3*); + /* Version 3.7.16 and later */ + int (*close_v2)(sqlite3*); + const char *(*db_filename)(sqlite3*,const char*); + int (*db_readonly)(sqlite3*,const char*); + int (*db_release_memory)(sqlite3*); + const char *(*errstr)(int); + int (*stmt_busy)(sqlite3_stmt*); + int (*stmt_readonly)(sqlite3_stmt*); + int (*stricmp)(const char*,const char*); + int (*uri_boolean)(const char*,const char*,int); + sqlite3_int64 (*uri_int64)(const char*,const char*,sqlite3_int64); + const char *(*uri_parameter)(const char*,const char*); + char *(*vsnprintf)(int,char*,const char*,va_list); + int (*wal_checkpoint_v2)(sqlite3*,const char*,int,int*,int*); }; /* @@ -439,9 +453,32 @@ struct sqlite3_api_routines { #define sqlite3_blob_reopen sqlite3_api->blob_reopen #define sqlite3_vtab_config sqlite3_api->vtab_config #define sqlite3_vtab_on_conflict sqlite3_api->vtab_on_conflict +/* Version 3.7.16 and later */ +#define sqlite3_close_v2 sqlite3_api->close_v2 +#define sqlite3_db_filename sqlite3_api->db_filename +#define sqlite3_db_readonly sqlite3_api->db_readonly +#define sqlite3_db_release_memory sqlite3_api->db_release_memory +#define sqlite3_errstr sqlite3_api->errstr +#define sqlite3_stmt_busy sqlite3_api->stmt_busy +#define sqlite3_stmt_readonly sqlite3_api->stmt_readonly +#define sqlite3_stricmp sqlite3_api->stricmp +#define sqlite3_uri_boolean sqlite3_api->uri_boolean +#define sqlite3_uri_int64 sqlite3_api->uri_int64 +#define sqlite3_uri_parameter sqlite3_api->uri_parameter +#define sqlite3_uri_vsnprintf sqlite3_api->vsnprintf +#define sqlite3_wal_checkpoint_v2 sqlite3_api->wal_checkpoint_v2 #endif /* SQLITE_CORE */ -#define SQLITE_EXTENSION_INIT1 const sqlite3_api_routines *sqlite3_api = 0; -#define SQLITE_EXTENSION_INIT2(v) sqlite3_api = v; +#ifndef SQLITE_CORE + /* This case when the file really is being compiled as a loadable + ** extension */ +# define SQLITE_EXTENSION_INIT1 const sqlite3_api_routines *sqlite3_api=0; +# define SQLITE_EXTENSION_INIT2(v) sqlite3_api=v; +#else + /* This case when the file is being statically linked into the + ** application */ +# define SQLITE_EXTENSION_INIT1 /*no-op*/ +# define SQLITE_EXTENSION_INIT2(v) (void)v; /* unused parameter */ +#endif #endif /* _SQLITE3EXT_H_ */ diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 09163bf..5950f23 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -66,6 +66,10 @@ # define _GNU_SOURCE #endif +#if defined(__OpenBSD__) && !defined(_BSD_SOURCE) +# define _BSD_SOURCE +#endif + /* ** Include standard header files as necessary */ @@ -118,11 +122,11 @@ ** We support that for legacy. */ #if !defined(SQLITE_THREADSAFE) -#if defined(THREADSAFE) -# define SQLITE_THREADSAFE THREADSAFE -#else -# define SQLITE_THREADSAFE 1 /* IMP: R-07272-22309 */ -#endif +# if defined(THREADSAFE) +# define SQLITE_THREADSAFE THREADSAFE +# else +# define SQLITE_THREADSAFE 1 /* IMP: R-07272-22309 */ +# endif #endif /* @@ -200,7 +204,8 @@ ** ** See also ticket #2741. */ -#if !defined(_XOPEN_SOURCE) && !defined(__DARWIN__) && !defined(__APPLE__) && SQLITE_THREADSAFE +#if !defined(_XOPEN_SOURCE) && !defined(__DARWIN__) \ + && !defined(__APPLE__) && SQLITE_THREADSAFE # define _XOPEN_SOURCE 500 /* Needed to enable pthread recursive mutexes */ #endif @@ -387,6 +392,7 @@ */ #ifndef SQLITE_TEMP_STORE # define SQLITE_TEMP_STORE 1 +# define SQLITE_TEMP_STORE_xc 1 /* Exclude from ctime.c */ #endif /* @@ -534,6 +540,49 @@ extern const int sqlite3one; # define EIGHT_BYTE_ALIGNMENT(X) ((((char*)(X) - (char*)0)&7)==0) #endif +/* +** Disable MMAP on platforms where it is known to not work +*/ +#if defined(__OpenBSD__) || defined(__QNXNTO__) +# undef SQLITE_MAX_MMAP_SIZE +# define SQLITE_MAX_MMAP_SIZE 0 +#endif + +/* +** Default maximum size of memory used by memory-mapped I/O in the VFS +*/ +#ifdef __APPLE__ +# include +# if TARGET_OS_IPHONE +# undef SQLITE_MAX_MMAP_SIZE +# define SQLITE_MAX_MMAP_SIZE 0 +# endif +#endif +#ifndef SQLITE_MAX_MMAP_SIZE +# if defined(__linux__) \ + || defined(_WIN32) \ + || (defined(__APPLE__) && defined(__MACH__)) \ + || defined(__sun) +# define SQLITE_MAX_MMAP_SIZE 0x7fff0000 /* 2147418112 */ +# else +# define SQLITE_MAX_MMAP_SIZE 0 +# endif +# define SQLITE_MAX_MMAP_SIZE_xc 1 /* exclude from ctime.c */ +#endif + +/* +** The default MMAP_SIZE is zero on all platforms. Or, even if a larger +** default MMAP_SIZE is specified at compile-time, make sure that it does +** not exceed the maximum mmap size. +*/ +#ifndef SQLITE_DEFAULT_MMAP_SIZE +# define SQLITE_DEFAULT_MMAP_SIZE 0 +# define SQLITE_DEFAULT_MMAP_SIZE_xc 1 /* Exclude from ctime.c */ +#endif +#if SQLITE_DEFAULT_MMAP_SIZE>SQLITE_MAX_MMAP_SIZE +# undef SQLITE_DEFAULT_MMAP_SIZE +# define SQLITE_DEFAULT_MMAP_SIZE SQLITE_MAX_MMAP_SIZE +#endif /* ** An instance of the following structure is used to store the busy-handler @@ -575,6 +624,11 @@ struct BusyHandler { */ #define ArraySize(X) ((int)(sizeof(X)/sizeof(X[0]))) +/* +** Determine if the argument is a power of two +*/ +#define IsPowerOfTwo(X) (((X)&((X)-1))==0) + /* ** The following value as a destructor means to use sqlite3DbFree(). ** The sqlite3DbFree() routine requires two parameters instead of the @@ -661,6 +715,7 @@ typedef struct Parse Parse; typedef struct RowSet RowSet; typedef struct Savepoint Savepoint; typedef struct Select Select; +typedef struct SelectDest SelectDest; typedef struct SrcList SrcList; typedef struct StrAccum StrAccum; typedef struct Table Table; @@ -823,9 +878,11 @@ struct sqlite3 { int nDb; /* Number of backends currently in use */ int flags; /* Miscellaneous flags. See below */ i64 lastRowid; /* ROWID of most recent insert (see above) */ + i64 szMmap; /* Default mmap_size setting */ unsigned int openFlags; /* Flags passed to sqlite3_vfs.xOpen() */ int errCode; /* Most recent error code (SQLITE_*) */ int errMask; /* & result codes with this before returning */ + u16 dbOptFlags; /* Flags to enable/disable optimizations */ u8 autoCommit; /* The auto-commit flag. */ u8 temp_store; /* 1: file 2: memory 0: default */ u8 mallocFailed; /* True if we have seen a malloc failure */ @@ -930,48 +987,60 @@ struct sqlite3 { /* ** Possible values for the sqlite3.flags. */ -#define SQLITE_VdbeTrace 0x00000100 /* True to trace VDBE execution */ -#define SQLITE_InternChanges 0x00000200 /* Uncommitted Hash table changes */ -#define SQLITE_FullColNames 0x00000400 /* Show full column names on SELECT */ -#define SQLITE_ShortColNames 0x00000800 /* Show short columns names */ -#define SQLITE_CountRows 0x00001000 /* Count rows changed by INSERT, */ +#define SQLITE_VdbeTrace 0x00000001 /* True to trace VDBE execution */ +#define SQLITE_InternChanges 0x00000002 /* Uncommitted Hash table changes */ +#define SQLITE_FullColNames 0x00000004 /* Show full column names on SELECT */ +#define SQLITE_ShortColNames 0x00000008 /* Show short columns names */ +#define SQLITE_CountRows 0x00000010 /* Count rows changed by INSERT, */ /* DELETE, or UPDATE and return */ /* the count using a callback. */ -#define SQLITE_NullCallback 0x00002000 /* Invoke the callback once if the */ +#define SQLITE_NullCallback 0x00000020 /* Invoke the callback once if the */ /* result set is empty */ -#define SQLITE_SqlTrace 0x00004000 /* Debug print SQL as it executes */ -#define SQLITE_VdbeListing 0x00008000 /* Debug listings of VDBE programs */ -#define SQLITE_WriteSchema 0x00010000 /* OK to update SQLITE_MASTER */ - /* 0x00020000 Unused */ -#define SQLITE_IgnoreChecks 0x00040000 /* Do not enforce check constraints */ -#define SQLITE_ReadUncommitted 0x0080000 /* For shared-cache mode */ -#define SQLITE_LegacyFileFmt 0x00100000 /* Create new databases in format 1 */ -#define SQLITE_FullFSync 0x00200000 /* Use full fsync on the backend */ -#define SQLITE_CkptFullFSync 0x00400000 /* Use full fsync for checkpoint */ -#define SQLITE_RecoveryMode 0x00800000 /* Ignore schema errors */ -#define SQLITE_ReverseOrder 0x01000000 /* Reverse unordered SELECTs */ -#define SQLITE_RecTriggers 0x02000000 /* Enable recursive triggers */ -#define SQLITE_ForeignKeys 0x04000000 /* Enforce foreign key constraints */ -#define SQLITE_AutoIndex 0x08000000 /* Enable automatic indexes */ -#define SQLITE_PreferBuiltin 0x10000000 /* Preference to built-in funcs */ -#define SQLITE_LoadExtension 0x20000000 /* Enable load_extension */ -#define SQLITE_EnableTrigger 0x40000000 /* True to enable triggers */ - -/* -** Bits of the sqlite3.flags field that are used by the -** sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS,...) interface. -** These must be the low-order bits of the flags field. -*/ -#define SQLITE_QueryFlattener 0x01 /* Disable query flattening */ -#define SQLITE_ColumnCache 0x02 /* Disable the column cache */ -#define SQLITE_IndexSort 0x04 /* Disable indexes for sorting */ -#define SQLITE_IndexSearch 0x08 /* Disable indexes for searching */ -#define SQLITE_IndexCover 0x10 /* Disable index covering table */ -#define SQLITE_GroupByOrder 0x20 /* Disable GROUPBY cover of ORDERBY */ -#define SQLITE_FactorOutConst 0x40 /* Disable factoring out constants */ -#define SQLITE_IdxRealAsInt 0x80 /* Store REAL as INT in indices */ -#define SQLITE_DistinctOpt 0x80 /* DISTINCT using indexes */ -#define SQLITE_OptMask 0xff /* Mask of all disablable opts */ +#define SQLITE_SqlTrace 0x00000040 /* Debug print SQL as it executes */ +#define SQLITE_VdbeListing 0x00000080 /* Debug listings of VDBE programs */ +#define SQLITE_WriteSchema 0x00000100 /* OK to update SQLITE_MASTER */ +#define SQLITE_VdbeAddopTrace 0x00000200 /* Trace sqlite3VdbeAddOp() calls */ +#define SQLITE_IgnoreChecks 0x00000400 /* Do not enforce check constraints */ +#define SQLITE_ReadUncommitted 0x0000800 /* For shared-cache mode */ +#define SQLITE_LegacyFileFmt 0x00001000 /* Create new databases in format 1 */ +#define SQLITE_FullFSync 0x00002000 /* Use full fsync on the backend */ +#define SQLITE_CkptFullFSync 0x00004000 /* Use full fsync for checkpoint */ +#define SQLITE_RecoveryMode 0x00008000 /* Ignore schema errors */ +#define SQLITE_ReverseOrder 0x00010000 /* Reverse unordered SELECTs */ +#define SQLITE_RecTriggers 0x00020000 /* Enable recursive triggers */ +#define SQLITE_ForeignKeys 0x00040000 /* Enforce foreign key constraints */ +#define SQLITE_AutoIndex 0x00080000 /* Enable automatic indexes */ +#define SQLITE_PreferBuiltin 0x00100000 /* Preference to built-in funcs */ +#define SQLITE_LoadExtension 0x00200000 /* Enable load_extension */ +#define SQLITE_EnableTrigger 0x00400000 /* True to enable triggers */ + +/* +** Bits of the sqlite3.dbOptFlags field that are used by the +** sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS,...) interface to +** selectively disable various optimizations. +*/ +#define SQLITE_QueryFlattener 0x0001 /* Query flattening */ +#define SQLITE_ColumnCache 0x0002 /* Column cache */ +#define SQLITE_GroupByOrder 0x0004 /* GROUPBY cover of ORDERBY */ +#define SQLITE_FactorOutConst 0x0008 /* Constant factoring */ +#define SQLITE_IdxRealAsInt 0x0010 /* Store REAL as INT in indices */ +#define SQLITE_DistinctOpt 0x0020 /* DISTINCT using indexes */ +#define SQLITE_CoverIdxScan 0x0040 /* Covering index scans */ +#define SQLITE_OrderByIdxJoin 0x0080 /* ORDER BY of joins via index */ +#define SQLITE_SubqCoroutine 0x0100 /* Evaluate subqueries as coroutines */ +#define SQLITE_Transitive 0x0200 /* Transitive constraints */ +#define SQLITE_AllOpts 0xffff /* All optimizations */ + +/* +** Macros for testing whether or not optimizations are enabled or disabled. +*/ +#ifndef SQLITE_OMIT_BUILTIN_TEST +#define OptimizationDisabled(db, mask) (((db)->dbOptFlags&(mask))!=0) +#define OptimizationEnabled(db, mask) (((db)->dbOptFlags&(mask))==0) +#else +#define OptimizationDisabled(db, mask) 0 +#define OptimizationEnabled(db, mask) 1 +#endif /* ** Possible values for the sqlite.magic field. @@ -1122,32 +1191,22 @@ struct Column { char *zDflt; /* Original text of the default value */ char *zType; /* Data type for this column */ char *zColl; /* Collating sequence. If NULL, use the default */ - u8 notNull; /* True if there is a NOT NULL constraint */ - u8 isPrimKey; /* True if this column is part of the PRIMARY KEY */ + u8 notNull; /* An OE_ code for handling a NOT NULL constraint */ char affinity; /* One of the SQLITE_AFF_... values */ -#ifndef SQLITE_OMIT_VIRTUALTABLE - u8 isHidden; /* True if this column is 'hidden' */ -#endif + u16 colFlags; /* Boolean properties. See COLFLAG_ defines below */ }; +/* Allowed values for Column.colFlags: +*/ +#define COLFLAG_PRIMKEY 0x0001 /* Column is part of the primary key */ +#define COLFLAG_HIDDEN 0x0002 /* A hidden column in a virtual table */ + /* ** A "Collating Sequence" is defined by an instance of the following ** structure. Conceptually, a collating sequence consists of a name and ** a comparison routine that defines the order of that sequence. ** -** There may two separate implementations of the collation function, one -** that processes text in UTF-8 encoding (CollSeq.xCmp) and another that -** processes text encoded in UTF-16 (CollSeq.xCmp16), using the machine -** native byte order. When a collation sequence is invoked, SQLite selects -** the version that will require the least expensive encoding -** translations, if any. -** -** The CollSeq.pUser member variable is an extra parameter that passed in -** as the first argument to the UTF-8 comparison function, xCmp. -** CollSeq.pUser16 is the equivalent for the UTF-16 comparison function, -** xCmp16. -** -** If both CollSeq.xCmp and CollSeq.xCmp16 are NULL, it means that the +** If CollSeq.xCmp is NULL, it means that the ** collating sequence is undefined. Indices built on an undefined ** collating sequence may not be read or written. */ @@ -1285,28 +1344,28 @@ struct VTable { */ struct Table { char *zName; /* Name of the table or view */ - int iPKey; /* If not negative, use aCol[iPKey] as the primary key */ - int nCol; /* Number of columns in this table */ Column *aCol; /* Information about each column */ Index *pIndex; /* List of SQL indexes on this table. */ - int tnum; /* Root BTree node for this table (see note above) */ - tRowcnt nRowEst; /* Estimated rows in table - from sqlite_stat1 table */ Select *pSelect; /* NULL for tables. Points to definition if a view. */ - u16 nRef; /* Number of pointers to this Table */ - u8 tabFlags; /* Mask of TF_* values */ - u8 keyConf; /* What to do in case of uniqueness conflict on iPKey */ FKey *pFKey; /* Linked list of all foreign keys in this table */ char *zColAff; /* String defining the affinity of each column */ #ifndef SQLITE_OMIT_CHECK ExprList *pCheck; /* All CHECK constraints */ #endif + tRowcnt nRowEst; /* Estimated rows in table - from sqlite_stat1 table */ + int tnum; /* Root BTree node for this table (see note above) */ + i16 iPKey; /* If not negative, use aCol[iPKey] as the primary key */ + i16 nCol; /* Number of columns in this table */ + u16 nRef; /* Number of pointers to this Table */ + u8 tabFlags; /* Mask of TF_* values */ + u8 keyConf; /* What to do in case of uniqueness conflict on iPKey */ #ifndef SQLITE_OMIT_ALTERTABLE int addColOffset; /* Offset in CREATE TABLE stmt to add a new column */ #endif #ifndef SQLITE_OMIT_VIRTUALTABLE - VTable *pVTable; /* List of VTable objects. */ int nModuleArg; /* Number of arguments to the module */ char **azModuleArg; /* Text of all module args. [0] is module name */ + VTable *pVTable; /* List of VTable objects. */ #endif Trigger *pTrigger; /* List of triggers stored in pSchema */ Schema *pSchema; /* Schema that contains this table */ @@ -1330,7 +1389,7 @@ struct Table { */ #ifndef SQLITE_OMIT_VIRTUALTABLE # define IsVirtual(X) (((X)->tabFlags & TF_Virtual)!=0) -# define IsHiddenColumn(X) ((X)->isHidden) +# define IsHiddenColumn(X) (((X)->colFlags & COLFLAG_HIDDEN)!=0) #else # define IsVirtual(X) 0 # define IsHiddenColumn(X) 0 @@ -1481,20 +1540,20 @@ struct UnpackedRecord { ** element. */ struct Index { - char *zName; /* Name of this index */ - int *aiColumn; /* Which columns are used by this index. 1st is 0 */ - tRowcnt *aiRowEst; /* Result of ANALYZE: Est. rows selected by each column */ - Table *pTable; /* The SQL table being indexed */ - char *zColAff; /* String defining the affinity of each column */ - Index *pNext; /* The next index associated with the same table */ - Schema *pSchema; /* Schema containing this index */ - u8 *aSortOrder; /* Array of size Index.nColumn. True==DESC, False==ASC */ - char **azColl; /* Array of collation sequence names for index */ - int nColumn; /* Number of columns in the table used by this index */ - int tnum; /* Page containing root of this index in database file */ - u8 onError; /* OE_Abort, OE_Ignore, OE_Replace, or OE_None */ - u8 autoIndex; /* True if is automatically created (ex: by UNIQUE) */ - u8 bUnordered; /* Use this index for == or IN queries only */ + char *zName; /* Name of this index */ + int *aiColumn; /* Which columns are used by this index. 1st is 0 */ + tRowcnt *aiRowEst; /* From ANALYZE: Est. rows selected by each column */ + Table *pTable; /* The SQL table being indexed */ + char *zColAff; /* String defining the affinity of each column */ + Index *pNext; /* The next index associated with the same table */ + Schema *pSchema; /* Schema containing this index */ + u8 *aSortOrder; /* for each column: True==DESC, False==ASC */ + char **azColl; /* Array of collation sequence names for index */ + int tnum; /* DB Page containing root of this index */ + u16 nColumn; /* Number of columns in table used by this index */ + u8 onError; /* OE_Abort, OE_Ignore, OE_Replace, or OE_None */ + unsigned autoIndex:2; /* 1==UNIQUE, 2==PRIMARY KEY, 0==CREATE INDEX */ + unsigned bUnordered:1; /* Use this index for == or IN queries only */ #ifdef SQLITE_ENABLE_STAT3 int nSample; /* Number of elements in aSample[] */ tRowcnt avgEq; /* Average nEq value for key values not in aSample */ @@ -1675,13 +1734,15 @@ struct Expr { ExprList *pList; /* Function arguments or in " IN ( IN (