diff options
Diffstat (limited to 'src')
97 files changed, 8121 insertions, 7416 deletions
diff --git a/src/alter.c b/src/alter.c index 7f56ce7..a49d334 100644 --- a/src/alter.c +++ b/src/alter.c @@ -414,7 +414,7 @@ void sqlite3AlterRenameTable( assert( pSrc->nSrc==1 ); assert( sqlite3BtreeHoldsAllMutexes(pParse->db) ); - pTab = sqlite3LocateTable(pParse, 0, pSrc->a[0].zName, pSrc->a[0].zDatabase); + pTab = sqlite3LocateTableItem(pParse, 0, &pSrc->a[0]); if( !pTab ) goto exit_rename_table; iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); zDb = db->aDb[iDb].zName; @@ -664,7 +664,7 @@ void sqlite3AlterFinishAddColumn(Parse *pParse, Token *pColDef){ ** If there is a NOT NULL constraint, then the default value for the ** column must not be NULL. */ - if( pCol->isPrimKey ){ + if( pCol->colFlags & COLFLAG_PRIMKEY ){ sqlite3ErrorMsg(pParse, "Cannot add a PRIMARY KEY column"); return; } @@ -757,7 +757,7 @@ void sqlite3AlterBeginAddColumn(Parse *pParse, SrcList *pSrc){ assert( pParse->pNewTable==0 ); assert( sqlite3BtreeHoldsAllMutexes(db) ); if( db->mallocFailed ) goto exit_begin_add_column; - pTab = sqlite3LocateTable(pParse, 0, pSrc->a[0].zName, pSrc->a[0].zDatabase); + pTab = sqlite3LocateTableItem(pParse, 0, &pSrc->a[0]); if( !pTab ) goto exit_begin_add_column; #ifndef SQLITE_OMIT_VIRTUALTABLE diff --git a/src/analyze.c b/src/analyze.c index 632fdc1..9a3e959 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -473,7 +473,7 @@ static void analyzeOneTable( /* Do not gather statistics on views or virtual tables */ return; } - if( memcmp(pTab->zName, "sqlite_", 7)==0 ){ + if( sqlite3_strnicmp(pTab->zName, "sqlite_", 7)==0 ){ /* Do not gather statistics on system tables */ return; } @@ -883,7 +883,7 @@ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ if( pIndex==0 ) break; pIndex->aiRowEst[i] = v; if( *z==' ' ) z++; - if( memcmp(z, "unordered", 10)==0 ){ + if( strcmp(z, "unordered")==0 ){ pIndex->bUnordered = 1; break; } diff --git a/src/attach.c b/src/attach.c index e295c30..b8e1219 100644 --- a/src/attach.c +++ b/src/attach.c @@ -109,7 +109,7 @@ static void attachFunc( } } - /* Allocate the new entry in the db->aDb[] array and initialise the schema + /* Allocate the new entry in the db->aDb[] array and initialize the schema ** hash tables. */ if( db->aDb==db->aDbStatic ){ @@ -126,7 +126,7 @@ static void attachFunc( /* Open the database file. If the btree is successfully opened, use ** it to obtain the database schema. At this point the schema may - ** or may not be initialised. + ** or may not be initialized. */ flags = db->openFlags; rc = sqlite3ParseUri(db->pVfs->zName, zFile, &flags, &pVfs, &zPath, &zErr); @@ -434,6 +434,7 @@ int sqlite3FixInit( assert( db->nDb>iDb ); pFix->pParse = pParse; pFix->zDb = db->aDb[iDb].zName; + pFix->pSchema = db->aDb[iDb].pSchema; pFix->zType = zType; pFix->pName = pName; return 1; @@ -464,14 +465,15 @@ int sqlite3FixSrcList( if( NEVER(pList==0) ) return 0; zDb = pFix->zDb; for(i=0, pItem=pList->a; i<pList->nSrc; i++, pItem++){ - if( pItem->zDatabase==0 ){ - pItem->zDatabase = sqlite3DbStrDup(pFix->pParse->db, zDb); - }else if( sqlite3StrICmp(pItem->zDatabase,zDb)!=0 ){ + if( pItem->zDatabase && sqlite3StrICmp(pItem->zDatabase, zDb) ){ sqlite3ErrorMsg(pFix->pParse, "%s %T cannot reference objects in database %s", pFix->zType, pFix->pName, pItem->zDatabase); return 1; } + sqlite3DbFree(pFix->pParse->db, pItem->zDatabase); + pItem->zDatabase = 0; + pItem->pSchema = pFix->pSchema; #if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_TRIGGER) if( sqlite3FixSelect(pFix, pItem->pSelect) ) return 1; if( sqlite3FixExpr(pFix, pItem->pOn) ) return 1; diff --git a/src/backup.c b/src/backup.c index 4881215..252f61c 100644 --- a/src/backup.c +++ b/src/backup.c @@ -212,20 +212,28 @@ static int isFatalError(int rc){ ** page iSrcPg from the source database. Copy this data into the ** destination database. */ -static int backupOnePage(sqlite3_backup *p, Pgno iSrcPg, const u8 *zSrcData){ +static int backupOnePage( + sqlite3_backup *p, /* Backup handle */ + Pgno iSrcPg, /* Source database page to backup */ + const u8 *zSrcData, /* Source database page data */ + int bUpdate /* True for an update, false otherwise */ +){ Pager * const pDestPager = sqlite3BtreePager(p->pDest); const int nSrcPgsz = sqlite3BtreeGetPageSize(p->pSrc); int nDestPgsz = sqlite3BtreeGetPageSize(p->pDest); const int nCopy = MIN(nSrcPgsz, nDestPgsz); const i64 iEnd = (i64)iSrcPg*(i64)nSrcPgsz; #ifdef SQLITE_HAS_CODEC - int nSrcReserve = sqlite3BtreeGetReserve(p->pSrc); + /* Use BtreeGetReserveNoMutex() for the source b-tree, as although it is + ** guaranteed that the shared-mutex is held by this thread, handle + ** p->pSrc may not actually be the owner. */ + int nSrcReserve = sqlite3BtreeGetReserveNoMutex(p->pSrc); int nDestReserve = sqlite3BtreeGetReserve(p->pDest); #endif - int rc = SQLITE_OK; i64 iOff; + assert( sqlite3BtreeGetReserveNoMutex(p->pSrc)>=0 ); assert( p->bDestLocked ); assert( !isFatalError(p->rc) ); assert( iSrcPg!=PENDING_BYTE_PAGE(p->pSrc->pBt) ); @@ -282,6 +290,9 @@ static int backupOnePage(sqlite3_backup *p, Pgno iSrcPg, const u8 *zSrcData){ */ memcpy(zOut, zIn, nCopy); ((u8 *)sqlite3PagerGetExtra(pDestPg))[0] = 0; + if( iOff==0 && bUpdate==0 ){ + sqlite3Put4byte(&zOut[28], sqlite3BtreeLastPage(p->pSrc)); + } } sqlite3PagerUnref(pDestPg); } @@ -386,9 +397,10 @@ int sqlite3_backup_step(sqlite3_backup *p, int nPage){ const Pgno iSrcPg = p->iNext; /* Source page number */ if( iSrcPg!=PENDING_BYTE_PAGE(p->pSrc->pBt) ){ DbPage *pSrcPg; /* Source page object */ - rc = sqlite3PagerGet(pSrcPager, iSrcPg, &pSrcPg); + rc = sqlite3PagerAcquire(pSrcPager, iSrcPg, &pSrcPg, + PAGER_ACQUIRE_READONLY); if( rc==SQLITE_OK ){ - rc = backupOnePage(p, iSrcPg, sqlite3PagerGetData(pSrcPg)); + rc = backupOnePage(p, iSrcPg, sqlite3PagerGetData(pSrcPg), 0); sqlite3PagerUnref(pSrcPg); } } @@ -410,7 +422,13 @@ int sqlite3_backup_step(sqlite3_backup *p, int nPage){ ** same schema version. */ if( rc==SQLITE_DONE ){ - rc = sqlite3BtreeUpdateMeta(p->pDest,1,p->iDestSchema+1); + if( nSrcPage==0 ){ + rc = sqlite3BtreeNewDb(p->pDest); + nSrcPage = 1; + } + if( rc==SQLITE_OK || rc==SQLITE_DONE ){ + rc = sqlite3BtreeUpdateMeta(p->pDest,1,p->iDestSchema+1); + } if( rc==SQLITE_OK ){ if( p->pDestDb ){ sqlite3ResetAllSchemasOfConnection(p->pDestDb); @@ -444,7 +462,7 @@ int sqlite3_backup_step(sqlite3_backup *p, int nPage){ }else{ nDestTruncate = nSrcPage * (pgszSrc/pgszDest); } - sqlite3PagerTruncateImage(pDestPager, nDestTruncate); + assert( nDestTruncate>0 ); if( pgszSrc<pgszDest ){ /* If the source page-size is smaller than the destination page-size, @@ -458,22 +476,38 @@ int sqlite3_backup_step(sqlite3_backup *p, int nPage){ */ const i64 iSize = (i64)pgszSrc * (i64)nSrcPage; sqlite3_file * const pFile = sqlite3PagerFile(pDestPager); + Pgno iPg; + int nDstPage; i64 iOff; i64 iEnd; assert( pFile ); - assert( (i64)nDestTruncate*(i64)pgszDest >= iSize || ( + assert( nDestTruncate==0 + || (i64)nDestTruncate*(i64)pgszDest >= iSize || ( nDestTruncate==(int)(PENDING_BYTE_PAGE(p->pDest->pBt)-1) && iSize>=PENDING_BYTE && iSize<=PENDING_BYTE+pgszDest )); - /* This call ensures that all data required to recreate the original + /* This block ensures that all data required to recreate the original ** database has been stored in the journal for pDestPager and the ** journal synced to disk. So at this point we may safely modify ** the database file in any way, knowing that if a power failure ** occurs, the original database will be reconstructed from the ** journal file. */ - rc = sqlite3PagerCommitPhaseOne(pDestPager, 0, 1); + sqlite3PagerPagecount(pDestPager, &nDstPage); + for(iPg=nDestTruncate; rc==SQLITE_OK && iPg<=(Pgno)nDstPage; iPg++){ + if( iPg!=PENDING_BYTE_PAGE(p->pDest->pBt) ){ + DbPage *pPg; + rc = sqlite3PagerGet(pDestPager, iPg, &pPg); + if( rc==SQLITE_OK ){ + rc = sqlite3PagerWrite(pPg); + sqlite3PagerUnref(pPg); + } + } + } + if( rc==SQLITE_OK ){ + rc = sqlite3PagerCommitPhaseOne(pDestPager, 0, 1); + } /* Write the extra pages and truncate the database file as required */ iEnd = MIN(PENDING_BYTE + pgszDest, iSize); @@ -500,6 +534,7 @@ int sqlite3_backup_step(sqlite3_backup *p, int nPage){ rc = sqlite3PagerSync(pDestPager); } }else{ + sqlite3PagerTruncateImage(pDestPager, nDestTruncate); rc = sqlite3PagerCommitPhaseOne(pDestPager, 0, 0); } @@ -628,7 +663,7 @@ void sqlite3BackupUpdate(sqlite3_backup *pBackup, Pgno iPage, const u8 *aData){ int rc; assert( p->pDestDb ); sqlite3_mutex_enter(p->pDestDb->mutex); - rc = backupOnePage(p, iPage, aData); + rc = backupOnePage(p, iPage, aData, 1); sqlite3_mutex_leave(p->pDestDb->mutex); assert( rc!=SQLITE_BUSY && rc!=SQLITE_LOCKED ); if( rc!=SQLITE_OK ){ diff --git a/src/bitvec.c b/src/bitvec.c index 8d805a6..52184aa 100644 --- a/src/bitvec.c +++ b/src/bitvec.c @@ -72,7 +72,7 @@ /* ** A bitmap is an instance of the following structure. ** -** This bitmap records the existance of zero or more bits +** This bitmap records the existence of zero or more bits ** with values between 1 and iSize, inclusive. ** ** There are three possible representations of the bitmap. diff --git a/src/btree.c b/src/btree.c index d34d6ee..3ca6058 100644 --- a/src/btree.c +++ b/src/btree.c @@ -43,6 +43,25 @@ int sqlite3BtreeTrace=1; /* True to enable tracing */ */ #define get2byteNotZero(X) (((((int)get2byte(X))-1)&0xffff)+1) +/* +** Values passed as the 5th argument to allocateBtreePage() +*/ +#define BTALLOC_ANY 0 /* Allocate any page */ +#define BTALLOC_EXACT 1 /* Allocate exact page if possible */ +#define BTALLOC_LE 2 /* Allocate any page <= the parameter */ + +/* +** Macro IfNotOmitAV(x) returns (x) if SQLITE_OMIT_AUTOVACUUM is not +** defined, or 0 if it is. For example: +** +** bIncrVacuum = IfNotOmitAV(pBtShared->incrVacuum); +*/ +#ifndef SQLITE_OMIT_AUTOVACUUM +#define IfNotOmitAV(expr) (expr) +#else +#define IfNotOmitAV(expr) 0 +#endif + #ifndef SQLITE_OMIT_SHARED_CACHE /* ** A list of BtShared objects that are eligible for participation @@ -557,6 +576,19 @@ static void btreeClearHasContent(BtShared *pBt){ } /* +** Release all of the apPage[] pages for a cursor. +*/ +static void btreeReleaseAllCursorPages(BtCursor *pCur){ + int i; + for(i=0; i<=pCur->iPage; i++){ + releasePage(pCur->apPage[i]); + pCur->apPage[i] = 0; + } + pCur->iPage = -1; +} + + +/* ** Save the current cursor position in the variables BtCursor.nKey ** and BtCursor.pKey. The cursor's state is set to CURSOR_REQUIRESEEK. ** @@ -595,12 +627,7 @@ static int saveCursorPosition(BtCursor *pCur){ assert( !pCur->apPage[0]->intKey || !pCur->pKey ); if( rc==SQLITE_OK ){ - int i; - for(i=0; i<=pCur->iPage; i++){ - releasePage(pCur->apPage[i]); - pCur->apPage[i] = 0; - } - pCur->iPage = -1; + btreeReleaseAllCursorPages(pCur); pCur->eState = CURSOR_REQUIRESEEK; } @@ -618,11 +645,15 @@ static int saveAllCursors(BtShared *pBt, Pgno iRoot, BtCursor *pExcept){ assert( sqlite3_mutex_held(pBt->mutex) ); assert( pExcept==0 || pExcept->pBt==pBt ); for(p=pBt->pCursor; p; p=p->pNext){ - if( p!=pExcept && (0==iRoot || p->pgnoRoot==iRoot) && - p->eState==CURSOR_VALID ){ - int rc = saveCursorPosition(p); - if( SQLITE_OK!=rc ){ - return rc; + if( p!=pExcept && (0==iRoot || p->pgnoRoot==iRoot) ){ + if( p->eState==CURSOR_VALID ){ + int rc = saveCursorPosition(p); + if( SQLITE_OK!=rc ){ + return rc; + } + }else{ + testcase( p->iPage>0 ); + btreeReleaseAllCursorPages(p); } } } @@ -1550,13 +1581,17 @@ static int btreeGetPage( BtShared *pBt, /* The btree */ Pgno pgno, /* Number of the page to fetch */ MemPage **ppPage, /* Return the page in this parameter */ - int noContent /* Do not load page content if true */ + int noContent, /* Do not load page content if true */ + int bReadonly /* True if a read-only (mmap) page is ok */ ){ int rc; DbPage *pDbPage; + int flags = (noContent ? PAGER_ACQUIRE_NOCONTENT : 0) + | (bReadonly ? PAGER_ACQUIRE_READONLY : 0); + assert( noContent==0 || bReadonly==0 ); assert( sqlite3_mutex_held(pBt->mutex) ); - rc = sqlite3PagerAcquire(pBt->pPager, pgno, (DbPage**)&pDbPage, noContent); + rc = sqlite3PagerAcquire(pBt->pPager, pgno, (DbPage**)&pDbPage, flags); if( rc ) return rc; *ppPage = btreePageFromDbPage(pDbPage, pgno, pBt); return SQLITE_OK; @@ -1599,9 +1634,10 @@ u32 sqlite3BtreeLastPage(Btree *p){ ** may remain unchanged, or it may be set to an invalid value. */ static int getAndInitPage( - BtShared *pBt, /* The database file */ - Pgno pgno, /* Number of the page to get */ - MemPage **ppPage /* Write the page pointer here */ + BtShared *pBt, /* The database file */ + Pgno pgno, /* Number of the page to get */ + MemPage **ppPage, /* Write the page pointer here */ + int bReadonly /* True if a read-only (mmap) page is ok */ ){ int rc; assert( sqlite3_mutex_held(pBt->mutex) ); @@ -1609,7 +1645,7 @@ static int getAndInitPage( if( pgno>btreePagecount(pBt) ){ rc = SQLITE_CORRUPT_BKPT; }else{ - rc = btreeGetPage(pBt, pgno, ppPage, 0); + rc = btreeGetPage(pBt, pgno, ppPage, 0, bReadonly); if( rc==SQLITE_OK ){ rc = btreeInitPage(*ppPage); if( rc!=SQLITE_OK ){ @@ -1840,6 +1876,7 @@ int sqlite3BtreeOpen( rc = sqlite3PagerOpen(pVfs, &pBt->pPager, zFilename, EXTRA_SIZE, flags, vfsFlags, pageReinit); if( rc==SQLITE_OK ){ + sqlite3PagerSetMmapLimit(pBt->pPager, db->szMmap); rc = sqlite3PagerReadFileheader(pBt->pPager,sizeof(zDbHeader),zDbHeader); } if( rc!=SQLITE_OK ){ @@ -2107,6 +2144,19 @@ int sqlite3BtreeSetCacheSize(Btree *p, int mxPage){ } /* +** Change the limit on the amount of the database file that may be +** memory mapped. +*/ +int sqlite3BtreeSetMmapLimit(Btree *p, sqlite3_int64 szMmap){ + BtShared *pBt = p->pBt; + assert( sqlite3_mutex_held(p->db->mutex) ); + sqlite3BtreeEnter(p); + sqlite3PagerSetMmapLimit(pBt->pPager, szMmap); + sqlite3BtreeLeave(p); + return SQLITE_OK; +} + +/* ** Change the way data is synced to disk in order to increase or decrease ** how well the database resists damage due to OS crashes and power ** failures. Level 1 is the same as asynchronous (no syncs() occur and @@ -2200,6 +2250,24 @@ int sqlite3BtreeGetPageSize(Btree *p){ return p->pBt->pageSize; } +#if defined(SQLITE_HAS_CODEC) || defined(SQLITE_DEBUG) +/* +** This function is similar to sqlite3BtreeGetReserve(), except that it +** may only be called if it is guaranteed that the b-tree mutex is already +** held. +** +** This is useful in one special case in the backup API code where it is +** known that the shared b-tree mutex is held, but the mutex on the +** database handle that owns *p is not. In this case if sqlite3BtreeEnter() +** were to be called, it might collide with some other operation on the +** database handle that owns *p, causing undefined behavior. +*/ +int sqlite3BtreeGetReserveNoMutex(Btree *p){ + assert( sqlite3_mutex_held(p->pBt->mutex) ); + return p->pBt->pageSize - p->pBt->usableSize; +} +#endif /* SQLITE_HAS_CODEC || SQLITE_DEBUG */ + #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) || !defined(SQLITE_OMIT_VACUUM) /* ** Return the number of bytes of space at the end of every page that @@ -2313,7 +2381,7 @@ static int lockBtree(BtShared *pBt){ assert( pBt->pPage1==0 ); rc = sqlite3PagerSharedLock(pBt->pPager); if( rc!=SQLITE_OK ) return rc; - rc = btreeGetPage(pBt, 1, &pPage1, 0); + rc = btreeGetPage(pBt, 1, &pPage1, 0, 0); if( rc!=SQLITE_OK ) return rc; /* Do some checking to help insure the file we opened really is @@ -2449,6 +2517,29 @@ page1_init_failed: return rc; } +#ifndef NDEBUG +/* +** Return the number of cursors open on pBt. This is for use +** in assert() expressions, so it is only compiled if NDEBUG is not +** defined. +** +** Only write cursors are counted if wrOnly is true. If wrOnly is +** false then all cursors are counted. +** +** For the purposes of this routine, a cursor is any cursor that +** is capable of reading or writing to the databse. Cursors that +** have been tripped into the CURSOR_FAULT state are not counted. +*/ +static int countValidCursors(BtShared *pBt, int wrOnly){ + BtCursor *pCur; + int r = 0; + for(pCur=pBt->pCursor; pCur; pCur=pCur->pNext){ + if( (wrOnly==0 || pCur->wrFlag) && pCur->eState!=CURSOR_FAULT ) r++; + } + return r; +} +#endif + /* ** If there are no outstanding cursors and we are not in the middle ** of a transaction but there is a read lock on the database, then @@ -2459,7 +2550,7 @@ page1_init_failed: */ static void unlockBtreeIfUnused(BtShared *pBt){ assert( sqlite3_mutex_held(pBt->mutex) ); - assert( pBt->pCursor==0 || pBt->inTransaction>TRANS_NONE ); + assert( countValidCursors(pBt,0)==0 || pBt->inTransaction>TRANS_NONE ); if( pBt->inTransaction==TRANS_NONE && pBt->pPage1!=0 ){ assert( pBt->pPage1->aData ); assert( sqlite3PagerRefcount(pBt->pPager)==1 ); @@ -2514,6 +2605,20 @@ static int newDatabase(BtShared *pBt){ } /* +** Initialize the first page of the database file (creating a database +** consisting of a single page and no schema objects). Return SQLITE_OK +** if successful, or an SQLite error code otherwise. +*/ +int sqlite3BtreeNewDb(Btree *p){ + int rc; + sqlite3BtreeEnter(p); + p->pBt->nPage = 0; + rc = newDatabase(p->pBt); + sqlite3BtreeLeave(p); + return rc; +} + +/* ** Attempt to start a new transaction. A write-transaction ** is started if the second argument is nonzero, otherwise a read- ** transaction. If the second argument is 2 or more and exclusive @@ -2563,6 +2668,7 @@ int sqlite3BtreeBeginTrans(Btree *p, int wrflag){ if( p->inTrans==TRANS_WRITE || (p->inTrans==TRANS_READ && !wrflag) ){ goto trans_begun; } + assert( IfNotOmitAV(pBt->bDoTruncate)==0 ); /* Write transactions are not possible on a read-only database */ if( (pBt->btsFlags & BTS_READ_ONLY)!=0 && wrflag ){ @@ -2857,7 +2963,7 @@ static int relocatePage( ** iPtrPage. */ if( eType!=PTRMAP_ROOTPAGE ){ - rc = btreeGetPage(pBt, iPtrPage, &pPtrPage, 0); + rc = btreeGetPage(pBt, iPtrPage, &pPtrPage, 0, 0); if( rc!=SQLITE_OK ){ return rc; } @@ -2879,24 +2985,23 @@ static int relocatePage( static int allocateBtreePage(BtShared *, MemPage **, Pgno *, Pgno, u8); /* -** Perform a single step of an incremental-vacuum. If successful, -** return SQLITE_OK. If there is no work to do (and therefore no -** point in calling this function again), return SQLITE_DONE. +** Perform a single step of an incremental-vacuum. If successful, return +** SQLITE_OK. If there is no work to do (and therefore no point in +** calling this function again), return SQLITE_DONE. Or, if an error +** occurs, return some other error code. ** -** More specificly, this function attempts to re-organize the -** database so that the last page of the file currently in use -** is no longer in use. +** More specificly, this function attempts to re-organize the database so +** that the last page of the file currently in use is no longer in use. ** -** If the nFin parameter is non-zero, this function assumes -** that the caller will keep calling incrVacuumStep() until -** it returns SQLITE_DONE or an error, and that nFin is the -** number of pages the database file will contain after this -** process is complete. If nFin is zero, it is assumed that -** incrVacuumStep() will be called a finite amount of times -** which may or may not empty the freelist. A full autovacuum -** has nFin>0. A "PRAGMA incremental_vacuum" has nFin==0. +** Parameter nFin is the number of pages that this database would contain +** were this function called until it returns SQLITE_DONE. +** +** If the bCommit parameter is non-zero, this function assumes that the +** caller will keep calling incrVacuumStep() until it returns SQLITE_DONE +** or an error. bCommit is passed true for an auto-vacuum-on-commmit +** operation, or false for an incremental vacuum. */ -static int incrVacuumStep(BtShared *pBt, Pgno nFin, Pgno iLastPg){ +static int incrVacuumStep(BtShared *pBt, Pgno nFin, Pgno iLastPg, int bCommit){ Pgno nFreeList; /* Number of pages still on the free-list */ int rc; @@ -2921,15 +3026,15 @@ static int incrVacuumStep(BtShared *pBt, Pgno nFin, Pgno iLastPg){ } if( eType==PTRMAP_FREEPAGE ){ - if( nFin==0 ){ + if( bCommit==0 ){ /* Remove the page from the files free-list. This is not required - ** if nFin is non-zero. In that case, the free-list will be + ** if bCommit is non-zero. In that case, the free-list will be ** truncated to zero after this function returns, so it doesn't ** matter if it still contains some garbage entries. */ Pgno iFreePg; MemPage *pFreePg; - rc = allocateBtreePage(pBt, &pFreePg, &iFreePg, iLastPg, 1); + rc = allocateBtreePage(pBt, &pFreePg, &iFreePg, iLastPg, BTALLOC_EXACT); if( rc!=SQLITE_OK ){ return rc; } @@ -2939,34 +3044,37 @@ static int incrVacuumStep(BtShared *pBt, Pgno nFin, Pgno iLastPg){ } else { Pgno iFreePg; /* Index of free page to move pLastPg to */ MemPage *pLastPg; + u8 eMode = BTALLOC_ANY; /* Mode parameter for allocateBtreePage() */ + Pgno iNear = 0; /* nearby parameter for allocateBtreePage() */ - rc = btreeGetPage(pBt, iLastPg, &pLastPg, 0); + rc = btreeGetPage(pBt, iLastPg, &pLastPg, 0, 0); if( rc!=SQLITE_OK ){ return rc; } - /* If nFin is zero, this loop runs exactly once and page pLastPg + /* If bCommit is zero, this loop runs exactly once and page pLastPg ** is swapped with the first free page pulled off the free list. ** - ** On the other hand, if nFin is greater than zero, then keep + ** On the other hand, if bCommit is greater than zero, then keep ** looping until a free-page located within the first nFin pages ** of the file is found. */ + if( bCommit==0 ){ + eMode = BTALLOC_LE; + iNear = nFin; + } do { MemPage *pFreePg; - rc = allocateBtreePage(pBt, &pFreePg, &iFreePg, 0, 0); + rc = allocateBtreePage(pBt, &pFreePg, &iFreePg, iNear, eMode); if( rc!=SQLITE_OK ){ releasePage(pLastPg); return rc; } releasePage(pFreePg); - }while( nFin!=0 && iFreePg>nFin ); + }while( bCommit && iFreePg>nFin ); assert( iFreePg<iLastPg ); - rc = sqlite3PagerWrite(pLastPg->pDbPage); - if( rc==SQLITE_OK ){ - rc = relocatePage(pBt, pLastPg, eType, iPtrPage, iFreePg, nFin!=0); - } + rc = relocatePage(pBt, pLastPg, eType, iPtrPage, iFreePg, bCommit); releasePage(pLastPg); if( rc!=SQLITE_OK ){ return rc; @@ -2974,30 +3082,40 @@ static int incrVacuumStep(BtShared *pBt, Pgno nFin, Pgno iLastPg){ } } - if( nFin==0 ){ - iLastPg--; - while( iLastPg==PENDING_BYTE_PAGE(pBt)||PTRMAP_ISPAGE(pBt, iLastPg) ){ - if( PTRMAP_ISPAGE(pBt, iLastPg) ){ - MemPage *pPg; - rc = btreeGetPage(pBt, iLastPg, &pPg, 0); - if( rc!=SQLITE_OK ){ - return rc; - } - rc = sqlite3PagerWrite(pPg->pDbPage); - releasePage(pPg); - if( rc!=SQLITE_OK ){ - return rc; - } - } + if( bCommit==0 ){ + do { iLastPg--; - } - sqlite3PagerTruncateImage(pBt->pPager, iLastPg); + }while( iLastPg==PENDING_BYTE_PAGE(pBt) || PTRMAP_ISPAGE(pBt, iLastPg) ); + pBt->bDoTruncate = 1; pBt->nPage = iLastPg; } return SQLITE_OK; } /* +** The database opened by the first argument is an auto-vacuum database +** nOrig pages in size containing nFree free pages. Return the expected +** size of the database in pages following an auto-vacuum operation. +*/ +static Pgno finalDbSize(BtShared *pBt, Pgno nOrig, Pgno nFree){ + int nEntry; /* Number of entries on one ptrmap page */ + Pgno nPtrmap; /* Number of PtrMap pages to be freed */ + Pgno nFin; /* Return value */ + + nEntry = pBt->usableSize/5; + nPtrmap = (nFree-nOrig+PTRMAP_PAGENO(pBt, nOrig)+nEntry)/nEntry; + nFin = nOrig - nFree - nPtrmap; + if( nOrig>PENDING_BYTE_PAGE(pBt) && nFin<PENDING_BYTE_PAGE(pBt) ){ + nFin--; + } + while( PTRMAP_ISPAGE(pBt, nFin) || nFin==PENDING_BYTE_PAGE(pBt) ){ + nFin--; + } + + return nFin; +} + +/* ** A write-transaction must be opened before calling this function. ** It performs a single unit of work towards an incremental vacuum. ** @@ -3014,11 +3132,24 @@ int sqlite3BtreeIncrVacuum(Btree *p){ if( !pBt->autoVacuum ){ rc = SQLITE_DONE; }else{ - invalidateAllOverflowCache(pBt); - rc = incrVacuumStep(pBt, 0, btreePagecount(pBt)); - if( rc==SQLITE_OK ){ - rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); - put4byte(&pBt->pPage1->aData[28], pBt->nPage); + Pgno nOrig = btreePagecount(pBt); + Pgno nFree = get4byte(&pBt->pPage1->aData[36]); + Pgno nFin = finalDbSize(pBt, nOrig, nFree); + + if( nOrig<nFin ){ + rc = SQLITE_CORRUPT_BKPT; + }else if( nFree>0 ){ + rc = saveAllCursors(pBt, 0, 0); + if( rc==SQLITE_OK ){ + invalidateAllOverflowCache(pBt); + rc = incrVacuumStep(pBt, nFin, nOrig, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); + put4byte(&pBt->pPage1->aData[28], pBt->nPage); + } + }else{ + rc = SQLITE_DONE; } } sqlite3BtreeLeave(p); @@ -3045,9 +3176,7 @@ static int autoVacuumCommit(BtShared *pBt){ if( !pBt->incrVacuum ){ Pgno nFin; /* Number of pages in database after autovacuuming */ Pgno nFree; /* Number of pages on the freelist initially */ - Pgno nPtrmap; /* Number of PtrMap pages to be freed */ Pgno iFree; /* The next page to be freed */ - int nEntry; /* Number of entries on one ptrmap page */ Pgno nOrig; /* Database size before freeing */ nOrig = btreePagecount(pBt); @@ -3060,26 +3189,20 @@ static int autoVacuumCommit(BtShared *pBt){ } nFree = get4byte(&pBt->pPage1->aData[36]); - nEntry = pBt->usableSize/5; - nPtrmap = (nFree-nOrig+PTRMAP_PAGENO(pBt, nOrig)+nEntry)/nEntry; - nFin = nOrig - nFree - nPtrmap; - if( nOrig>PENDING_BYTE_PAGE(pBt) && nFin<PENDING_BYTE_PAGE(pBt) ){ - nFin--; - } - while( PTRMAP_ISPAGE(pBt, nFin) || nFin==PENDING_BYTE_PAGE(pBt) ){ - nFin--; - } + nFin = finalDbSize(pBt, nOrig, nFree); if( nFin>nOrig ) return SQLITE_CORRUPT_BKPT; - + if( nFin<nOrig ){ + rc = saveAllCursors(pBt, 0, 0); + } for(iFree=nOrig; iFree>nFin && rc==SQLITE_OK; iFree--){ - rc = incrVacuumStep(pBt, nFin, iFree); + rc = incrVacuumStep(pBt, nFin, iFree, 1); } if( (rc==SQLITE_DONE || rc==SQLITE_OK) && nFree>0 ){ rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); put4byte(&pBt->pPage1->aData[32], 0); put4byte(&pBt->pPage1->aData[36], 0); put4byte(&pBt->pPage1->aData[28], nFin); - sqlite3PagerTruncateImage(pBt->pPager, nFin); + pBt->bDoTruncate = 1; pBt->nPage = nFin; } if( rc!=SQLITE_OK ){ @@ -3087,7 +3210,7 @@ static int autoVacuumCommit(BtShared *pBt){ } } - assert( nRef==sqlite3PagerRefcount(pPager) ); + assert( nRef>=sqlite3PagerRefcount(pPager) ); return rc; } @@ -3134,6 +3257,9 @@ int sqlite3BtreeCommitPhaseOne(Btree *p, const char *zMaster){ return rc; } } + if( pBt->bDoTruncate ){ + sqlite3PagerTruncateImage(pBt->pPager, pBt->nPage); + } #endif rc = sqlite3PagerCommitPhaseOne(pBt->pPager, zMaster, 0); sqlite3BtreeLeave(p); @@ -3149,7 +3275,9 @@ static void btreeEndTransaction(Btree *p){ BtShared *pBt = p->pBt; assert( sqlite3BtreeHoldsMutex(p) ); - btreeClearHasContent(pBt); +#ifndef SQLITE_OMIT_AUTOVACUUM + pBt->bDoTruncate = 0; +#endif if( p->inTrans>TRANS_NONE && p->db->activeVdbeCnt>1 ){ /* If there are other active statements that belong to this database ** handle, downgrade to a read-only transaction. The other statements @@ -3224,6 +3352,7 @@ int sqlite3BtreeCommitPhaseTwo(Btree *p, int bCleanup){ return rc; } pBt->inTransaction = TRANS_READ; + btreeClearHasContent(pBt); } btreeEndTransaction(p); @@ -3245,27 +3374,6 @@ int sqlite3BtreeCommit(Btree *p){ return rc; } -#ifndef NDEBUG -/* -** Return the number of write-cursors open on this handle. This is for use -** in assert() expressions, so it is only compiled if NDEBUG is not -** defined. -** -** For the purposes of this routine, a write-cursor is any cursor that -** is capable of writing to the databse. That means the cursor was -** originally opened for writing and the cursor has not be disabled -** by having its state changed to CURSOR_FAULT. -*/ -static int countWriteCursors(BtShared *pBt){ - BtCursor *pCur; - int r = 0; - for(pCur=pBt->pCursor; pCur; pCur=pCur->pNext){ - if( pCur->wrFlag && pCur->eState!=CURSOR_FAULT ) r++; - } - return r; -} -#endif - /* ** This routine sets the state to CURSOR_FAULT and the error ** code to errCode for every cursor on BtShared that pBtree @@ -3337,7 +3445,7 @@ int sqlite3BtreeRollback(Btree *p, int tripCode){ /* The rollback may have destroyed the pPage1->aData value. So ** call btreeGetPage() on page 1 again to make ** sure pPage1->aData is set correctly. */ - if( btreeGetPage(pBt, 1, &pPage1, 0)==SQLITE_OK ){ + if( btreeGetPage(pBt, 1, &pPage1, 0, 0)==SQLITE_OK ){ int nPage = get4byte(28+(u8*)pPage1->aData); testcase( nPage==0 ); if( nPage==0 ) sqlite3PagerPagecount(pBt->pPager, &nPage); @@ -3345,8 +3453,9 @@ int sqlite3BtreeRollback(Btree *p, int tripCode){ pBt->nPage = nPage; releasePage(pPage1); } - assert( countWriteCursors(pBt)==0 ); + assert( countValidCursors(pBt, 1)==0 ); pBt->inTransaction = TRANS_READ; + btreeClearHasContent(pBt); } btreeEndTransaction(p); @@ -3771,7 +3880,7 @@ static int getOverflowPage( assert( next==0 || rc==SQLITE_DONE ); if( rc==SQLITE_OK ){ - rc = btreeGetPage(pBt, ovfl, &pPage, 0); + rc = btreeGetPage(pBt, ovfl, &pPage, 0, (ppPage==0)); assert( rc==SQLITE_OK || pPage==0 ); if( rc==SQLITE_OK ){ next = get4byte(pPage->aData); @@ -3992,7 +4101,9 @@ static int accessPayload( { DbPage *pDbPage; - rc = sqlite3PagerGet(pBt->pPager, nextPage, &pDbPage); + rc = sqlite3PagerAcquire(pBt->pPager, nextPage, &pDbPage, + (eOp==0 ? PAGER_ACQUIRE_READONLY : 0) + ); if( rc==SQLITE_OK ){ aPayload = sqlite3PagerGetData(pDbPage); nextPage = get4byte(aPayload); @@ -4171,10 +4282,11 @@ static int moveToChild(BtCursor *pCur, u32 newPgno){ assert( cursorHoldsMutex(pCur) ); assert( pCur->eState==CURSOR_VALID ); assert( pCur->iPage<BTCURSOR_MAX_DEPTH ); + assert( pCur->iPage>=0 ); if( pCur->iPage>=(BTCURSOR_MAX_DEPTH-1) ){ return SQLITE_CORRUPT_BKPT; } - rc = getAndInitPage(pBt, newPgno, &pNewPage); + rc = getAndInitPage(pBt, newPgno, &pNewPage, (pCur->wrFlag==0)); if( rc ) return rc; pCur->apPage[i+1] = pNewPage; pCur->aiIdx[i+1] = 0; @@ -4291,7 +4403,7 @@ static int moveToRoot(BtCursor *pCur){ pCur->eState = CURSOR_INVALID; return SQLITE_OK; }else{ - rc = getAndInitPage(pBt, pCur->pgnoRoot, &pCur->apPage[0]); + rc = getAndInitPage(pBt, pCur->pgnoRoot, &pCur->apPage[0], pCur->wrFlag==0); if( rc!=SQLITE_OK ){ pCur->eState = CURSOR_INVALID; return rc; @@ -4821,21 +4933,23 @@ int sqlite3BtreePrevious(BtCursor *pCur, int *pRes){ ** an error. *ppPage and *pPgno are undefined in the event of an error. ** Do not invoke sqlite3PagerUnref() on *ppPage if an error is returned. ** -** If the "nearby" parameter is not 0, then a (feeble) effort is made to +** If the "nearby" parameter is not 0, then an effort is made to ** locate a page close to the page number "nearby". This can be used in an ** attempt to keep related pages close to each other in the database file, ** which in turn can make database access faster. ** -** If the "exact" parameter is not 0, and the page-number nearby exists -** anywhere on the free-list, then it is guarenteed to be returned. This -** is only used by auto-vacuum databases when allocating a new table. +** If the eMode parameter is BTALLOC_EXACT and the nearby page exists +** anywhere on the free-list, then it is guaranteed to be returned. If +** eMode is BTALLOC_LT then the page returned will be less than or equal +** to nearby if any such page exists. If eMode is BTALLOC_ANY then there +** are no restrictions on which page is returned. */ static int allocateBtreePage( - BtShared *pBt, - MemPage **ppPage, - Pgno *pPgno, - Pgno nearby, - u8 exact + BtShared *pBt, /* The btree */ + MemPage **ppPage, /* Store pointer to the allocated page here */ + Pgno *pPgno, /* Store the page number here */ + Pgno nearby, /* Search for a page near this one */ + u8 eMode /* BTALLOC_EXACT, BTALLOC_LT, or BTALLOC_ANY */ ){ MemPage *pPage1; int rc; @@ -4846,6 +4960,7 @@ static int allocateBtreePage( Pgno mxPage; /* Total size of the database file */ assert( sqlite3_mutex_held(pBt->mutex) ); + assert( eMode==BTALLOC_ANY || (nearby>0 && IfNotOmitAV(pBt->autoVacuum)) ); pPage1 = pBt->pPage1; mxPage = btreePagecount(pBt); n = get4byte(&pPage1->aData[36]); @@ -4858,21 +4973,24 @@ static int allocateBtreePage( Pgno iTrunk; u8 searchList = 0; /* If the free-list must be searched for 'nearby' */ - /* If the 'exact' parameter was true and a query of the pointer-map + /* If eMode==BTALLOC_EXACT and a query of the pointer-map ** shows that the page 'nearby' is somewhere on the free-list, then ** the entire-list will be searched for that page. */ #ifndef SQLITE_OMIT_AUTOVACUUM - if( exact && nearby<=mxPage ){ - u8 eType; - assert( nearby>0 ); - assert( pBt->autoVacuum ); - rc = ptrmapGet(pBt, nearby, &eType, 0); - if( rc ) return rc; - if( eType==PTRMAP_FREEPAGE ){ - searchList = 1; + if( eMode==BTALLOC_EXACT ){ + if( nearby<=mxPage ){ + u8 eType; + assert( nearby>0 ); + assert( pBt->autoVacuum ); + rc = ptrmapGet(pBt, nearby, &eType, 0); + if( rc ) return rc; + if( eType==PTRMAP_FREEPAGE ){ + searchList = 1; + } } - *pPgno = nearby; + }else if( eMode==BTALLOC_LE ){ + searchList = 1; } #endif @@ -4885,7 +5003,8 @@ static int allocateBtreePage( /* The code within this loop is run only once if the 'searchList' variable ** is not true. Otherwise, it runs once for each trunk-page on the - ** free-list until the page 'nearby' is located. + ** free-list until the page 'nearby' is located (eMode==BTALLOC_EXACT) + ** or until a page less than 'nearby' is located (eMode==BTALLOC_LT) */ do { pPrevTrunk = pTrunk; @@ -4898,7 +5017,7 @@ static int allocateBtreePage( if( iTrunk>mxPage ){ rc = SQLITE_CORRUPT_BKPT; }else{ - rc = btreeGetPage(pBt, iTrunk, &pTrunk, 0); + rc = btreeGetPage(pBt, iTrunk, &pTrunk, 0, 0); } if( rc ){ pTrunk = 0; @@ -4927,11 +5046,13 @@ static int allocateBtreePage( rc = SQLITE_CORRUPT_BKPT; goto end_allocate_page; #ifndef SQLITE_OMIT_AUTOVACUUM - }else if( searchList && nearby==iTrunk ){ + }else if( searchList + && (nearby==iTrunk || (iTrunk<nearby && eMode==BTALLOC_LE)) + ){ /* The list is being searched and this trunk page is the page ** to allocate, regardless of whether it has leaves. */ - assert( *pPgno==iTrunk ); + *pPgno = iTrunk; *ppPage = pTrunk; searchList = 0; rc = sqlite3PagerWrite(pTrunk->pDbPage); @@ -4960,7 +5081,7 @@ static int allocateBtreePage( goto end_allocate_page; } testcase( iNewTrunk==mxPage ); - rc = btreeGetPage(pBt, iNewTrunk, &pNewTrunk, 0); + rc = btreeGetPage(pBt, iNewTrunk, &pNewTrunk, 0, 0); if( rc!=SQLITE_OK ){ goto end_allocate_page; } @@ -4994,14 +5115,24 @@ static int allocateBtreePage( unsigned char *aData = pTrunk->aData; if( nearby>0 ){ u32 i; - int dist; closest = 0; - dist = sqlite3AbsInt32(get4byte(&aData[8]) - nearby); - for(i=1; i<k; i++){ - int d2 = sqlite3AbsInt32(get4byte(&aData[8+i*4]) - nearby); - if( d2<dist ){ - closest = i; - dist = d2; + if( eMode==BTALLOC_LE ){ + for(i=0; i<k; i++){ + iPage = get4byte(&aData[8+i*4]); + if( iPage<=nearby ){ + closest = i; + break; + } + } + }else{ + int dist; + dist = sqlite3AbsInt32(get4byte(&aData[8]) - nearby); + for(i=1; i<k; i++){ + int d2 = sqlite3AbsInt32(get4byte(&aData[8+i*4]) - nearby); + if( d2<dist ){ + closest = i; + dist = d2; + } } } }else{ @@ -5015,7 +5146,9 @@ static int allocateBtreePage( goto end_allocate_page; } testcase( iPage==mxPage ); - if( !searchList || iPage==nearby ){ + if( !searchList + || (iPage==nearby || (iPage<nearby && eMode==BTALLOC_LE)) + ){ int noContent; *pPgno = iPage; TRACE(("ALLOCATE: %d was leaf %d of %d on trunk %d" @@ -5028,7 +5161,7 @@ static int allocateBtreePage( } put4byte(&aData[4], k-1); noContent = !btreeGetHasContent(pBt, *pPgno); - rc = btreeGetPage(pBt, *pPgno, ppPage, noContent); + rc = btreeGetPage(pBt, *pPgno, ppPage, noContent, 0); if( rc==SQLITE_OK ){ rc = sqlite3PagerWrite((*ppPage)->pDbPage); if( rc!=SQLITE_OK ){ @@ -5042,8 +5175,26 @@ static int allocateBtreePage( pPrevTrunk = 0; }while( searchList ); }else{ - /* There are no pages on the freelist, so create a new page at the - ** end of the file */ + /* There are no pages on the freelist, so append a new page to the + ** database image. + ** + ** Normally, new pages allocated by this block can be requested from the + ** pager layer with the 'no-content' flag set. This prevents the pager + ** from trying to read the pages content from disk. However, if the + ** current transaction has already run one or more incremental-vacuum + ** steps, then the page we are about to allocate may contain content + ** that is required in the event of a rollback. In this case, do + ** not set the no-content flag. This causes the pager to load and journal + ** the current page content before overwriting it. + ** + ** Note that the pager will not actually attempt to load or journal + ** content for any page that really does lie past the end of the database + ** file on disk. So the effects of disabling the no-content optimization + ** here are confined to those pages that lie between the end of the + ** database image and the end of the database file. + */ + int bNoContent = (0==IfNotOmitAV(pBt->bDoTruncate)); + rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); if( rc ) return rc; pBt->nPage++; @@ -5058,7 +5209,7 @@ static int allocateBtreePage( MemPage *pPg = 0; TRACE(("ALLOCATE: %d from end of file (pointer-map page)\n", pBt->nPage)); assert( pBt->nPage!=PENDING_BYTE_PAGE(pBt) ); - rc = btreeGetPage(pBt, pBt->nPage, &pPg, 1); + rc = btreeGetPage(pBt, pBt->nPage, &pPg, bNoContent, 0); if( rc==SQLITE_OK ){ rc = sqlite3PagerWrite(pPg->pDbPage); releasePage(pPg); @@ -5072,7 +5223,7 @@ static int allocateBtreePage( *pPgno = pBt->nPage; assert( *pPgno!=PENDING_BYTE_PAGE(pBt) ); - rc = btreeGetPage(pBt, *pPgno, ppPage, 1); + rc = btreeGetPage(pBt, *pPgno, ppPage, bNoContent, 0); if( rc ) return rc; rc = sqlite3PagerWrite((*ppPage)->pDbPage); if( rc!=SQLITE_OK ){ @@ -5140,7 +5291,7 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ /* If the secure_delete option is enabled, then ** always fully overwrite deleted information with zeros. */ - if( (!pPage && ((rc = btreeGetPage(pBt, iPage, &pPage, 0))!=0) ) + if( (!pPage && ((rc = btreeGetPage(pBt, iPage, &pPage, 0, 0))!=0) ) || ((rc = sqlite3PagerWrite(pPage->pDbPage))!=0) ){ goto freepage_out; @@ -5167,7 +5318,7 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ u32 nLeaf; /* Initial number of leaf cells on trunk page */ iTrunk = get4byte(&pPage1->aData[32]); - rc = btreeGetPage(pBt, iTrunk, &pTrunk, 0); + rc = btreeGetPage(pBt, iTrunk, &pTrunk, 0, 0); if( rc!=SQLITE_OK ){ goto freepage_out; } @@ -5213,7 +5364,7 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ ** first trunk in the free-list is full. Either way, the page being freed ** will become the new first trunk page in the free-list. */ - if( pPage==0 && SQLITE_OK!=(rc = btreeGetPage(pBt, iPage, &pPage, 0)) ){ + if( pPage==0 && SQLITE_OK!=(rc = btreeGetPage(pBt, iPage, &pPage, 0, 0)) ){ goto freepage_out; } rc = sqlite3PagerWrite(pPage->pDbPage); @@ -5256,7 +5407,7 @@ static int clearCell(MemPage *pPage, unsigned char *pCell){ return SQLITE_OK; /* No overflow pages. Return without doing anything */ } if( pCell+info.iOverflow+3 > pPage->aData+pPage->maskPage ){ - return SQLITE_CORRUPT; /* Cell extends past end of page */ + return SQLITE_CORRUPT_BKPT; /* Cell extends past end of page */ } ovflPgno = get4byte(&pCell[info.iOverflow]); assert( pBt->usableSize > 4 ); @@ -5400,7 +5551,7 @@ static int fillInCell( ** If this is the first overflow page, then write a partial entry ** to the pointer-map. If we write nothing to this pointer-map slot, ** then the optimistic overflow chain processing in clearCell() - ** may misinterpret the uninitialised values and delete the + ** may misinterpret the uninitialized values and delete the ** wrong pages from the database. */ if( pBt->autoVacuum && rc==SQLITE_OK ){ @@ -5712,7 +5863,7 @@ static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){ assert( pPage->nOverflow==1 ); /* This error condition is now caught prior to reaching this function */ - if( pPage->nCell<=0 ) return SQLITE_CORRUPT_BKPT; + if( pPage->nCell==0 ) return SQLITE_CORRUPT_BKPT; /* Allocate a new page. This page will become the right-sibling of ** pPage. Make the parent page writable, so that the new divider cell @@ -6014,7 +6165,7 @@ static int balance_nonroot( } pgno = get4byte(pRight); while( 1 ){ - rc = getAndInitPage(pBt, pgno, &apOld[i]); + rc = getAndInitPage(pBt, pgno, &apOld[i], 0); if( rc ){ memset(apOld, 0, (i+1)*sizeof(MemPage*)); goto balance_cleanup; @@ -6873,7 +7024,7 @@ int sqlite3BtreeInsert( insertCell(pPage, idx, newCell, szNew, 0, 0, &rc); assert( rc!=SQLITE_OK || pPage->nCell>0 || pPage->nOverflow>0 ); - /* If no error has occured and pPage has an overflow cell, call balance() + /* If no error has occurred and pPage has an overflow cell, call balance() ** to redistribute the cells within the tree. Since balance() may move ** the cursor, zero the BtCursor.info.nSize and BtCursor.validNKey ** variables. @@ -7087,7 +7238,7 @@ static int btreeCreateTable(Btree *p, int *piTable, int createTabFlags){ ** be moved to the allocated page (unless the allocated page happens ** to reside at pgnoRoot). */ - rc = allocateBtreePage(pBt, &pPageMove, &pgnoMove, pgnoRoot, 1); + rc = allocateBtreePage(pBt, &pPageMove, &pgnoMove, pgnoRoot, BTALLOC_EXACT); if( rc!=SQLITE_OK ){ return rc; } @@ -7102,10 +7253,17 @@ static int btreeCreateTable(Btree *p, int *piTable, int createTabFlags){ u8 eType = 0; Pgno iPtrPage = 0; + /* Save the positions of any open cursors. This is required in + ** case they are holding a reference to an xFetch reference + ** corresponding to page pgnoRoot. */ + rc = saveAllCursors(pBt, 0, 0); releasePage(pPageMove); + if( rc!=SQLITE_OK ){ + return rc; + } /* Move the page currently at pgnoRoot to pgnoMove. */ - rc = btreeGetPage(pBt, pgnoRoot, &pRoot, 0); + rc = btreeGetPage(pBt, pgnoRoot, &pRoot, 0, 0); if( rc!=SQLITE_OK ){ return rc; } @@ -7126,7 +7284,7 @@ static int btreeCreateTable(Btree *p, int *piTable, int createTabFlags){ if( rc!=SQLITE_OK ){ return rc; } - rc = btreeGetPage(pBt, pgnoRoot, &pRoot, 0); + rc = btreeGetPage(pBt, pgnoRoot, &pRoot, 0, 0); if( rc!=SQLITE_OK ){ return rc; } @@ -7202,7 +7360,7 @@ static int clearDatabasePage( return SQLITE_CORRUPT_BKPT; } - rc = getAndInitPage(pBt, pgno, &pPage); + rc = getAndInitPage(pBt, pgno, &pPage, 0); if( rc ) return rc; for(i=0; i<pPage->nCell; i++){ pCell = findCell(pPage, i); @@ -7304,7 +7462,7 @@ static int btreeDropTable(Btree *p, Pgno iTable, int *piMoved){ return SQLITE_LOCKED_SHAREDCACHE; } - rc = btreeGetPage(pBt, (Pgno)iTable, &pPage, 0); + rc = btreeGetPage(pBt, (Pgno)iTable, &pPage, 0, 0); if( rc ) return rc; rc = sqlite3BtreeClearTable(p, iTable, 0); if( rc ){ @@ -7339,7 +7497,7 @@ static int btreeDropTable(Btree *p, Pgno iTable, int *piMoved){ */ MemPage *pMove; releasePage(pPage); - rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0); + rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0, 0); if( rc!=SQLITE_OK ){ return rc; } @@ -7349,7 +7507,7 @@ static int btreeDropTable(Btree *p, Pgno iTable, int *piMoved){ return rc; } pMove = 0; - rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0); + rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0, 0); freePage(pMove, &rc); releasePage(pMove); if( rc!=SQLITE_OK ){ @@ -7761,7 +7919,7 @@ static int checkTreePage( usableSize = pBt->usableSize; if( iPage==0 ) return 0; if( checkRef(pCheck, iPage, zParentContext) ) return 0; - if( (rc = btreeGetPage(pBt, (Pgno)iPage, &pPage, 0))!=0 ){ + if( (rc = btreeGetPage(pBt, (Pgno)iPage, &pPage, 0, 0))!=0 ){ checkAppendMsg(pCheck, zContext, "unable to get the page. error code=%d", rc); return 0; @@ -7994,7 +8152,7 @@ char *sqlite3BtreeIntegrityCheck( } i = PENDING_BYTE_PAGE(pBt); if( i<=sCheck.nPage ) setPageReferenced(&sCheck, i); - sqlite3StrAccumInit(&sCheck.errMsg, zErr, sizeof(zErr), 20000); + sqlite3StrAccumInit(&sCheck.errMsg, zErr, sizeof(zErr), SQLITE_MAX_LENGTH); sCheck.errMsg.useMalloc = 2; /* Check the integrity of the freelist @@ -8233,6 +8391,17 @@ int sqlite3BtreePutData(BtCursor *pCsr, u32 offset, u32 amt, void *z){ return SQLITE_ABORT; } + /* Save the positions of all other cursors open on this table. This is + ** required in case any of them are holding references to an xFetch + ** version of the b-tree page modified by the accessPayload call below. + ** + ** Note that pCsr must be open on a BTREE_INTKEY table and saveCursorPosition() + ** and hence saveAllCursors() cannot fail on a BTREE_INTKEY table, hence + ** saveAllCursors can only return SQLITE_OK. + */ + VVA_ONLY(rc =) saveAllCursors(pCsr->pBt, pCsr->pgnoRoot, pCsr); + assert( rc==SQLITE_OK ); + /* Check some assumptions: ** (a) the cursor is open for writing, ** (b) there is a read/write transaction open, diff --git a/src/btree.h b/src/btree.h index 95897d5..ace0f8c 100644 --- a/src/btree.h +++ b/src/btree.h @@ -63,6 +63,7 @@ int sqlite3BtreeOpen( int sqlite3BtreeClose(Btree*); int sqlite3BtreeSetCacheSize(Btree*,int); +int sqlite3BtreeSetMmapLimit(Btree*,sqlite3_int64); int sqlite3BtreeSetSafetyLevel(Btree*,int,int,int); int sqlite3BtreeSyncDisabled(Btree*); int sqlite3BtreeSetPageSize(Btree *p, int nPagesize, int nReserve, int eFix); @@ -71,6 +72,9 @@ int sqlite3BtreeMaxPageCount(Btree*,int); u32 sqlite3BtreeLastPage(Btree*); int sqlite3BtreeSecureDelete(Btree*,int); int sqlite3BtreeGetReserve(Btree*); +#if defined(SQLITE_HAS_CODEC) || defined(SQLITE_DEBUG) +int sqlite3BtreeGetReserveNoMutex(Btree *p); +#endif int sqlite3BtreeSetAutoVacuum(Btree *, int); int sqlite3BtreeGetAutoVacuum(Btree *); int sqlite3BtreeBeginTrans(Btree*,int); @@ -114,6 +118,8 @@ void sqlite3BtreeTripAllCursors(Btree*, int); void sqlite3BtreeGetMeta(Btree *pBtree, int idx, u32 *pValue); int sqlite3BtreeUpdateMeta(Btree*, int idx, u32 value); +int sqlite3BtreeNewDb(Btree *p); + /* ** The second parameter to sqlite3BtreeGetMeta or sqlite3BtreeUpdateMeta ** should be one of the following values. The integer values are assigned @@ -134,6 +140,7 @@ int sqlite3BtreeUpdateMeta(Btree*, int idx, u32 value); #define BTREE_TEXT_ENCODING 5 #define BTREE_USER_VERSION 6 #define BTREE_INCR_VACUUM 7 +#define BTREE_APPLICATION_ID 8 /* ** Values that may be OR'd together to form the second argument of an diff --git a/src/btreeInt.h b/src/btreeInt.h index b157dec..ce3c549 100644 --- a/src/btreeInt.h +++ b/src/btreeInt.h @@ -411,6 +411,7 @@ struct BtShared { #ifndef SQLITE_OMIT_AUTOVACUUM u8 autoVacuum; /* True if auto-vacuum is enabled */ u8 incrVacuum; /* True if incr-vacuum is enabled */ + u8 bDoTruncate; /* True to truncate db on commit */ #endif u8 inTransaction; /* Transaction state */ u8 max1bytePayload; /* Maximum first byte of cell for a 1-byte payload */ diff --git a/src/build.c b/src/build.c index 25e4740..3c91cdc 100644 --- a/src/build.c +++ b/src/build.c @@ -127,6 +127,7 @@ void sqlite3FinishCoding(Parse *pParse){ sqlite3 *db; Vdbe *v; + assert( pParse->pToplevel==0 ); db = pParse->db; if( db->mallocFailed ) return; if( pParse->nested ) return; @@ -320,6 +321,31 @@ Table *sqlite3LocateTable( } /* +** Locate the table identified by *p. +** +** This is a wrapper around sqlite3LocateTable(). The difference between +** sqlite3LocateTable() and this function is that this function restricts +** the search to schema (p->pSchema) if it is not NULL. p->pSchema may be +** non-NULL if it is part of a view or trigger program definition. See +** sqlite3FixSrcList() for details. +*/ +Table *sqlite3LocateTableItem( + Parse *pParse, + int isView, + struct SrcList_item *p +){ + const char *zDb; + assert( p->pSchema==0 || p->zDatabase==0 ); + if( p->pSchema ){ + int iDb = sqlite3SchemaToIndex(pParse->db, p->pSchema); + zDb = pParse->db->aDb[iDb].zName; + }else{ + zDb = p->zDatabase; + } + return sqlite3LocateTable(pParse, isView, p->zName, zDb); +} + +/* ** Locate the in-memory structure that describes ** a particular index given the name of that index ** and the name of the database that contains the index. @@ -1169,7 +1195,7 @@ void sqlite3AddPrimaryKey( pTab->tabFlags |= TF_HasPrimaryKey; if( pList==0 ){ iCol = pTab->nCol - 1; - pTab->aCol[iCol].isPrimKey = 1; + pTab->aCol[iCol].colFlags |= COLFLAG_PRIMKEY; }else{ for(i=0; i<pList->nExpr; i++){ for(iCol=0; iCol<pTab->nCol; iCol++){ @@ -1178,7 +1204,7 @@ void sqlite3AddPrimaryKey( } } if( iCol<pTab->nCol ){ - pTab->aCol[iCol].isPrimKey = 1; + pTab->aCol[iCol].colFlags |= COLFLAG_PRIMKEY; } } if( pList->nExpr>1 ) iCol = -1; @@ -1295,10 +1321,7 @@ CollSeq *sqlite3LocateCollSeq(Parse *pParse, const char *zName){ pColl = sqlite3FindCollSeq(db, enc, zName, initbusy); if( !initbusy && (!pColl || !pColl->xCmp) ){ - pColl = sqlite3GetCollSeq(db, enc, pColl, zName); - if( !pColl ){ - sqlite3ErrorMsg(pParse, "no such collation sequence: %s", zName); - } + pColl = sqlite3GetCollSeq(pParse, enc, pColl, zName); } return pColl; @@ -1997,6 +2020,7 @@ static void destroyTable(Parse *pParse, Table *pTab){ return; }else{ int iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); + assert( iDb>=0 && iDb<pParse->db->nDb ); destroyRootPage(pParse, iLargest, iDb); iDestroyed = iLargest; } @@ -2076,7 +2100,7 @@ void sqlite3CodeDropTable(Parse *pParse, Table *pTab, int iDb, int isView){ /* Drop all SQLITE_MASTER table and index entries that refer to the ** table. The program name loops through the master table and deletes ** every row that refers to a table of the same name as the one being - ** dropped. Triggers are handled seperately because a trigger can be + ** dropped. Triggers are handled separately because a trigger can be ** created in the temp database that refers to a table in another ** database. */ @@ -2114,8 +2138,7 @@ void sqlite3DropTable(Parse *pParse, SrcList *pName, int isView, int noErr){ assert( pParse->nErr==0 ); assert( pName->nSrc==1 ); if( noErr ) db->suppressErr++; - pTab = sqlite3LocateTable(pParse, isView, - pName->a[0].zName, pName->a[0].zDatabase); + pTab = sqlite3LocateTableItem(pParse, isView, &pName->a[0]); if( noErr ) db->suppressErr--; if( pTab==0 ){ @@ -2369,9 +2392,6 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ int tnum; /* Root page of index */ Vdbe *v; /* Generate code into this virtual machine */ KeyInfo *pKey; /* KeyInfo for index */ -#ifdef SQLITE_OMIT_MERGE_SORT - int regIdxKey; /* Registers containing the index key */ -#endif int regRecord; /* Register holding assemblied index record */ sqlite3 *db = pParse->db; /* The database connection */ int iDb = sqlite3SchemaToIndex(db, pIndex->pSchema); @@ -2399,13 +2419,9 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ (char *)pKey, P4_KEYINFO_HANDOFF); sqlite3VdbeChangeP5(v, OPFLAG_BULKCSR|((memRootPage>=0)?OPFLAG_P2ISREG:0)); -#ifndef SQLITE_OMIT_MERGE_SORT /* Open the sorter cursor if we are to use one. */ iSorter = pParse->nTab++; sqlite3VdbeAddOp4(v, OP_SorterOpen, iSorter, 0, 0, (char*)pKey, P4_KEYINFO); -#else - iSorter = iTab; -#endif /* Open the table. Loop through all rows of the table, inserting index ** records into the sorter. */ @@ -2413,7 +2429,6 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ addr1 = sqlite3VdbeAddOp2(v, OP_Rewind, iTab, 0); regRecord = sqlite3GetTempReg(pParse); -#ifndef SQLITE_OMIT_MERGE_SORT sqlite3GenerateIndexKey(pParse, pIndex, iTab, regRecord, 1); sqlite3VdbeAddOp2(v, OP_SorterInsert, iSorter, regRecord); sqlite3VdbeAddOp2(v, OP_Next, iTab, addr1+1); @@ -2424,8 +2439,8 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ sqlite3VdbeAddOp2(v, OP_Goto, 0, j2); addr2 = sqlite3VdbeCurrentAddr(v); sqlite3VdbeAddOp3(v, OP_SorterCompare, iSorter, j2, regRecord); - sqlite3HaltConstraint( - pParse, OE_Abort, "indexed columns are not unique", P4_STATIC + sqlite3HaltConstraint(pParse, SQLITE_CONSTRAINT_UNIQUE, + OE_Abort, "indexed columns are not unique", P4_STATIC ); }else{ addr2 = sqlite3VdbeCurrentAddr(v); @@ -2433,30 +2448,6 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ sqlite3VdbeAddOp2(v, OP_SorterData, iSorter, regRecord); sqlite3VdbeAddOp3(v, OP_IdxInsert, iIdx, regRecord, 1); sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); -#else - regIdxKey = sqlite3GenerateIndexKey(pParse, pIndex, iTab, regRecord, 1); - addr2 = addr1 + 1; - if( pIndex->onError!=OE_None ){ - const int regRowid = regIdxKey + pIndex->nColumn; - const int j2 = sqlite3VdbeCurrentAddr(v) + 2; - void * const pRegKey = SQLITE_INT_TO_PTR(regIdxKey); - - /* The registers accessed by the OP_IsUnique opcode were allocated - ** using sqlite3GetTempRange() inside of the sqlite3GenerateIndexKey() - ** call above. Just before that function was freed they were released - ** (made available to the compiler for reuse) using - ** sqlite3ReleaseTempRange(). So in some ways having the OP_IsUnique - ** opcode use the values stored within seems dangerous. However, since - ** we can be sure that no other temp registers have been allocated - ** since sqlite3ReleaseTempRange() was called, it is safe to do so. - */ - sqlite3VdbeAddOp4(v, OP_IsUnique, iIdx, j2, regRowid, pRegKey, P4_INT32); - sqlite3HaltConstraint( - pParse, OE_Abort, "indexed columns are not unique", P4_STATIC); - } - sqlite3VdbeAddOp3(v, OP_IdxInsert, iIdx, regRecord, 0); - sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); -#endif sqlite3ReleaseTempReg(pParse, regRecord); sqlite3VdbeAddOp2(v, OP_SorterNext, iSorter, addr2); sqlite3VdbeJumpHere(v, addr1); @@ -2555,9 +2546,9 @@ Index *sqlite3CreateIndex( ** sqlite3FixSrcList can never fail. */ assert(0); } - pTab = sqlite3LocateTable(pParse, 0, pTblName->a[0].zName, - pTblName->a[0].zDatabase); - if( !pTab || db->mallocFailed ) goto exit_create_index; + pTab = sqlite3LocateTableItem(pParse, 0, &pTblName->a[0]); + assert( db->mallocFailed==0 || pTab==0 ); + if( pTab==0 ) goto exit_create_index; assert( db->aDb[iDb].pSchema==pTab->pSchema ); }else{ assert( pName==0 ); @@ -2571,7 +2562,7 @@ Index *sqlite3CreateIndex( assert( pTab!=0 ); assert( pParse->nErr==0 ); if( sqlite3StrNICmp(pTab->zName, "sqlite_", 7)==0 - && memcmp(&pTab->zName[7],"altertab_",9)!=0 ){ + && sqlite3StrNICmp(&pTab->zName[7],"altertab_",9)!=0 ){ sqlite3ErrorMsg(pParse, "table %s may not be indexed", pTab->zName); goto exit_create_index; } @@ -2668,12 +2659,8 @@ Index *sqlite3CreateIndex( for(i=0; i<pList->nExpr; i++){ Expr *pExpr = pList->a[i].pExpr; if( pExpr ){ - CollSeq *pColl = pExpr->pColl; - /* Either pColl!=0 or there was an OOM failure. But if an OOM - ** failure we have quit before reaching this point. */ - if( ALWAYS(pColl) ){ - nExtra += (1 + sqlite3Strlen30(pColl->zName)); - } + assert( pExpr->op==TK_COLLATE ); + nExtra += (1 + sqlite3Strlen30(pExpr->u.zToken)); } } @@ -2746,14 +2733,10 @@ Index *sqlite3CreateIndex( goto exit_create_index; } pIndex->aiColumn[i] = j; - /* Justification of the ALWAYS(pListItem->pExpr->pColl): Because of - ** the way the "idxlist" non-terminal is constructed by the parser, - ** if pListItem->pExpr is not null then either pListItem->pExpr->pColl - ** must exist or else there must have been an OOM error. But if there - ** was an OOM error, we would never reach this point. */ - if( pListItem->pExpr && ALWAYS(pListItem->pExpr->pColl) ){ + if( pListItem->pExpr ){ int nColl; - zColl = pListItem->pExpr->pColl->zName; + assert( pListItem->pExpr->op==TK_COLLATE ); + zColl = pListItem->pExpr->u.zToken; nColl = sqlite3Strlen30(zColl) + 1; assert( nExtra>=nColl ); memcpy(zExtra, zColl, nColl); @@ -2762,9 +2745,7 @@ Index *sqlite3CreateIndex( nExtra -= nColl; }else{ zColl = pTab->aCol[j].zColl; - if( !zColl ){ - zColl = "BINARY"; - } + if( !zColl ) zColl = "BINARY"; } if( !db->init.busy && !sqlite3LocateCollSeq(pParse, zColl) ){ goto exit_create_index; @@ -2820,7 +2801,7 @@ Index *sqlite3CreateIndex( ** However the ON CONFLICT clauses are different. If both this ** constraint and the previous equivalent constraint have explicit ** ON CONFLICT clauses this is an error. Otherwise, use the - ** explicitly specified behaviour for the index. + ** explicitly specified behavior for the index. */ if( !(pIdx->onError==OE_Default || pIndex->onError==OE_Default) ){ sqlite3ErrorMsg(pParse, @@ -3567,6 +3548,15 @@ int sqlite3OpenTempDatabase(Parse *pParse){ void sqlite3CodeVerifySchema(Parse *pParse, int iDb){ Parse *pToplevel = sqlite3ParseToplevel(pParse); +#ifndef SQLITE_OMIT_TRIGGER + if( pToplevel!=pParse ){ + /* This branch is taken if a trigger is currently being coded. In this + ** case, set cookieGoto to a non-zero value to show that this function + ** has been called. This is used by the sqlite3ExprCodeConstants() + ** function. */ + pParse->cookieGoto = -1; + } +#endif if( pToplevel->cookieGoto==0 ){ Vdbe *v = sqlite3GetVdbe(pToplevel); if( v==0 ) return; /* This only happens if there was a prior error */ @@ -3664,12 +3654,19 @@ void sqlite3MayAbort(Parse *pParse){ ** error. The onError parameter determines which (if any) of the statement ** and/or current transaction is rolled back. */ -void sqlite3HaltConstraint(Parse *pParse, int onError, char *p4, int p4type){ +void sqlite3HaltConstraint( + Parse *pParse, /* Parsing context */ + int errCode, /* extended error code */ + int onError, /* Constraint type */ + char *p4, /* Error message */ + int p4type /* P4_STATIC or P4_TRANSIENT */ +){ Vdbe *v = sqlite3GetVdbe(pParse); + assert( (errCode&0xff)==SQLITE_CONSTRAINT ); if( onError==OE_Abort ){ sqlite3MayAbort(pParse); } - sqlite3VdbeAddOp4(v, OP_Halt, SQLITE_CONSTRAINT, onError, 0, p4, p4type); + sqlite3VdbeAddOp4(v, OP_Halt, errCode, onError, 0, p4, p4type); } /* diff --git a/src/callback.c b/src/callback.c index a515e05..d40c65c 100644 --- a/src/callback.c +++ b/src/callback.c @@ -75,17 +75,18 @@ static int synthCollSeq(sqlite3 *db, CollSeq *pColl){ ** ** The return value is either the collation sequence to be used in database ** db for collation type name zName, length nName, or NULL, if no collation -** sequence can be found. +** sequence can be found. If no collation is found, leave an error message. ** ** See also: sqlite3LocateCollSeq(), sqlite3FindCollSeq() */ CollSeq *sqlite3GetCollSeq( - sqlite3* db, /* The database connection */ + Parse *pParse, /* Parsing context */ u8 enc, /* The desired encoding for the collating sequence */ CollSeq *pColl, /* Collating sequence with native encoding, or NULL */ const char *zName /* Collating sequence name */ ){ CollSeq *p; + sqlite3 *db = pParse->db; p = pColl; if( !p ){ @@ -102,6 +103,9 @@ CollSeq *sqlite3GetCollSeq( p = 0; } assert( !p || p->xCmp ); + if( p==0 ){ + sqlite3ErrorMsg(pParse, "no such collation sequence: %s", zName); + } return p; } @@ -120,10 +124,8 @@ int sqlite3CheckCollSeq(Parse *pParse, CollSeq *pColl){ if( pColl ){ const char *zName = pColl->zName; sqlite3 *db = pParse->db; - CollSeq *p = sqlite3GetCollSeq(db, ENC(db), pColl, zName); + CollSeq *p = sqlite3GetCollSeq(pParse, ENC(db), pColl, zName); if( !p ){ - sqlite3ErrorMsg(pParse, "no such collation sequence: %s", zName); - pParse->nErr++; return SQLITE_ERROR; } assert( p==pColl ); diff --git a/src/crypto.c b/src/crypto.c index 50213b5..2551e6b 100644 --- a/src/crypto.c +++ b/src/crypto.c @@ -1,10 +1,8 @@ /* ** SQLCipher -** crypto.c developed by Stephen Lombardo (Zetetic LLC) -** sjlombardo at zetetic dot net -** http://zetetic.net +** http://sqlcipher.net ** -** Copyright (c) 2009, ZETETIC LLC +** Copyright (c) 2008 - 2013, ZETETIC LLC ** All rights reserved. ** ** Redistribution and use in source and binary forms, with or without @@ -30,7 +28,7 @@ ** SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ** */ -/* BEGIN CRYPTO */ +/* BEGIN SQLCIPHER */ #ifdef SQLITE_HAS_CODEC #include <assert.h> @@ -38,12 +36,12 @@ #include "btreeInt.h" #include "crypto.h" -const char* codec_get_cipher_version() { +static const char* codec_get_cipher_version() { return CIPHER_VERSION; } /* Generate code to return a string value */ -void codec_vdbe_return_static_string(Parse *pParse, const char *zLabel, const char *value){ +static void codec_vdbe_return_static_string(Parse *pParse, const char *zLabel, const char *value){ Vdbe *v = sqlite3GetVdbe(pParse); sqlite3VdbeSetNumCols(v, 1); sqlite3VdbeSetColName(v, 0, COLNAME_NAME, zLabel, SQLITE_STATIC); @@ -69,7 +67,7 @@ static int codec_set_btree_to_codec_pagesize(sqlite3 *db, Db *pDb, codec_ctx *ct return rc; } -int codec_set_pass_key(sqlite3* db, int nDb, const void *zKey, int nKey, int for_ctx) { +static int codec_set_pass_key(sqlite3* db, int nDb, const void *zKey, int nKey, int for_ctx) { struct Db *pDb = &db->aDb[nDb]; CODEC_TRACE(("codec_set_pass_key: entered db=%p nDb=%d zKey=%s nKey=%d for_ctx=%d\n", db, nDb, (char *)zKey, nKey, for_ctx)); if(pDb->pBt) { @@ -91,6 +89,11 @@ int codec_pragma(sqlite3* db, int iDb, Parse *pParse, const char *zLeft, const c CODEC_TRACE(("codec_pragma: entered db=%p iDb=%d pParse=%p zLeft=%s zRight=%s ctx=%p\n", db, iDb, pParse, zLeft, zRight, ctx)); + if( sqlite3StrICmp(zLeft, "cipher_provider")==0 && !zRight ){ + if(ctx) { codec_vdbe_return_static_string(pParse, "cipher_provider", + sqlcipher_codec_get_cipher_provider(ctx)); + } + } else if( sqlite3StrICmp(zLeft, "cipher_version")==0 && !zRight ){ codec_vdbe_return_static_string(pParse, "cipher_version", codec_get_cipher_version()); }else @@ -291,13 +294,13 @@ int sqlite3CodecAttach(sqlite3* db, int nDb, const void *zKey, int nKey) { sqlcipher_activate(); /* perform internal initialization for sqlcipher */ + sqlite3_mutex_enter(db->mutex); + /* point the internal codec argument against the contet to be prepared */ rc = sqlcipher_codec_ctx_init(&ctx, pDb, pDb->pBt->pBt->pPager, fd, zKey, nKey); if(rc != SQLITE_OK) return rc; /* initialization failed, do not attach potentially corrupted context */ - sqlite3_mutex_enter(db->mutex); - sqlite3pager_sqlite3PagerSetCodec(sqlite3BtreePager(pDb->pBt), sqlite3Codec, NULL, sqlite3FreeCodecArg, (void *) ctx); codec_set_btree_to_codec_pagesize(db, pDb, ctx); @@ -423,6 +426,203 @@ void sqlite3CodecGetKey(sqlite3* db, int nDb, void **zKey, int *nKey) { } } +#ifndef OMIT_EXPORT + +/* + * Implementation of an "export" function that allows a caller + * to duplicate the main database to an attached database. This is intended + * as a conveneince for users who need to: + * + * 1. migrate from an non-encrypted database to an encrypted database + * 2. move from an encrypted database to a non-encrypted database + * 3. convert beween the various flavors of encrypted databases. + * + * This implementation is based heavily on the procedure and code used + * in vacuum.c, but is exposed as a function that allows export to any + * named attached database. + */ + +/* +** Finalize a prepared statement. If there was an error, store the +** text of the error message in *pzErrMsg. Return the result code. +** +** Based on vacuumFinalize from vacuum.c +*/ +static int sqlcipher_finalize(sqlite3 *db, sqlite3_stmt *pStmt, char **pzErrMsg){ + int rc; + rc = sqlite3VdbeFinalize((Vdbe*)pStmt); + if( rc ){ + sqlite3SetString(pzErrMsg, db, sqlite3_errmsg(db)); + } + return rc; +} + +/* +** Execute zSql on database db. Return an error code. +** +** Based on execSql from vacuum.c +*/ +static int sqlcipher_execSql(sqlite3 *db, char **pzErrMsg, const char *zSql){ + sqlite3_stmt *pStmt; + VVA_ONLY( int rc; ) + if( !zSql ){ + return SQLITE_NOMEM; + } + if( SQLITE_OK!=sqlite3_prepare(db, zSql, -1, &pStmt, 0) ){ + sqlite3SetString(pzErrMsg, db, sqlite3_errmsg(db)); + return sqlite3_errcode(db); + } + VVA_ONLY( rc = ) sqlite3_step(pStmt); + assert( rc!=SQLITE_ROW ); + return sqlcipher_finalize(db, pStmt, pzErrMsg); +} + +/* +** Execute zSql on database db. The statement returns exactly +** one column. Execute this as SQL on the same database. +** +** Based on execExecSql from vacuum.c +*/ +static int sqlcipher_execExecSql(sqlite3 *db, char **pzErrMsg, const char *zSql){ + sqlite3_stmt *pStmt; + int rc; + + rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); + if( rc!=SQLITE_OK ) return rc; + + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + rc = sqlcipher_execSql(db, pzErrMsg, (char*)sqlite3_column_text(pStmt, 0)); + if( rc!=SQLITE_OK ){ + sqlcipher_finalize(db, pStmt, pzErrMsg); + return rc; + } + } + + return sqlcipher_finalize(db, pStmt, pzErrMsg); +} + +/* + * copy database and schema from the main database to an attached database + * + * Based on sqlite3RunVacuum from vacuum.c +*/ +void sqlcipher_exportFunc(sqlite3_context *context, int argc, sqlite3_value **argv) { + sqlite3 *db = sqlite3_context_db_handle(context); + const char* attachedDb = (const char*) sqlite3_value_text(argv[0]); + int saved_flags; /* Saved value of the db->flags */ + int saved_nChange; /* Saved value of db->nChange */ + int saved_nTotalChange; /* Saved value of db->nTotalChange */ + void (*saved_xTrace)(void*,const char*); /* Saved db->xTrace */ + int rc = SQLITE_OK; /* Return code from service routines */ + char *zSql = NULL; /* SQL statements */ + char *pzErrMsg = NULL; + + saved_flags = db->flags; + saved_nChange = db->nChange; + saved_nTotalChange = db->nTotalChange; + saved_xTrace = db->xTrace; + db->flags |= SQLITE_WriteSchema | SQLITE_IgnoreChecks | SQLITE_PreferBuiltin; + db->flags &= ~(SQLITE_ForeignKeys | SQLITE_ReverseOrder); + db->xTrace = 0; + + /* Query the schema of the main database. Create a mirror schema + ** in the temporary database. + */ + zSql = sqlite3_mprintf( + "SELECT 'CREATE TABLE %s.' || substr(sql,14) " + " FROM sqlite_master WHERE type='table' AND name!='sqlite_sequence'" + " AND rootpage>0" + , attachedDb); + rc = (zSql == NULL) ? SQLITE_NOMEM : sqlcipher_execExecSql(db, &pzErrMsg, zSql); + if( rc!=SQLITE_OK ) goto end_of_export; + sqlite3_free(zSql); + + zSql = sqlite3_mprintf( + "SELECT 'CREATE INDEX %s.' || substr(sql,14)" + " FROM sqlite_master WHERE sql LIKE 'CREATE INDEX %%' " + , attachedDb); + rc = (zSql == NULL) ? SQLITE_NOMEM : sqlcipher_execExecSql(db, &pzErrMsg, zSql); + if( rc!=SQLITE_OK ) goto end_of_export; + sqlite3_free(zSql); + + zSql = sqlite3_mprintf( + "SELECT 'CREATE UNIQUE INDEX %s.' || substr(sql,21) " + " FROM sqlite_master WHERE sql LIKE 'CREATE UNIQUE INDEX %%'" + , attachedDb); + rc = (zSql == NULL) ? SQLITE_NOMEM : sqlcipher_execExecSql(db, &pzErrMsg, zSql); + if( rc!=SQLITE_OK ) goto end_of_export; + sqlite3_free(zSql); + + /* Loop through the tables in the main database. For each, do + ** an "INSERT INTO rekey_db.xxx SELECT * FROM main.xxx;" to copy + ** the contents to the temporary database. + */ + zSql = sqlite3_mprintf( + "SELECT 'INSERT INTO %s.' || quote(name) " + "|| ' SELECT * FROM main.' || quote(name) || ';'" + "FROM main.sqlite_master " + "WHERE type = 'table' AND name!='sqlite_sequence' " + " AND rootpage>0" + , attachedDb); + rc = (zSql == NULL) ? SQLITE_NOMEM : sqlcipher_execExecSql(db, &pzErrMsg, zSql); + if( rc!=SQLITE_OK ) goto end_of_export; + sqlite3_free(zSql); + + /* Copy over the sequence table + */ + zSql = sqlite3_mprintf( + "SELECT 'DELETE FROM %s.' || quote(name) || ';' " + "FROM %s.sqlite_master WHERE name='sqlite_sequence' " + , attachedDb, attachedDb); + rc = (zSql == NULL) ? SQLITE_NOMEM : sqlcipher_execExecSql(db, &pzErrMsg, zSql); + if( rc!=SQLITE_OK ) goto end_of_export; + sqlite3_free(zSql); + + zSql = sqlite3_mprintf( + "SELECT 'INSERT INTO %s.' || quote(name) " + "|| ' SELECT * FROM main.' || quote(name) || ';' " + "FROM %s.sqlite_master WHERE name=='sqlite_sequence';" + , attachedDb, attachedDb); + rc = (zSql == NULL) ? SQLITE_NOMEM : sqlcipher_execExecSql(db, &pzErrMsg, zSql); + if( rc!=SQLITE_OK ) goto end_of_export; + sqlite3_free(zSql); + + /* Copy the triggers, views, and virtual tables from the main database + ** over to the temporary database. None of these objects has any + ** associated storage, so all we have to do is copy their entries + ** from the SQLITE_MASTER table. + */ + zSql = sqlite3_mprintf( + "INSERT INTO %s.sqlite_master " + " SELECT type, name, tbl_name, rootpage, sql" + " FROM main.sqlite_master" + " WHERE type='view' OR type='trigger'" + " OR (type='table' AND rootpage=0)" + , attachedDb); + rc = (zSql == NULL) ? SQLITE_NOMEM : sqlcipher_execSql(db, &pzErrMsg, zSql); + if( rc!=SQLITE_OK ) goto end_of_export; + sqlite3_free(zSql); + + zSql = NULL; +end_of_export: + db->flags = saved_flags; + db->nChange = saved_nChange; + db->nTotalChange = saved_nTotalChange; + db->xTrace = saved_xTrace; + + sqlite3_free(zSql); + + if(rc) { + if(pzErrMsg != NULL) { + sqlite3_result_error(context, pzErrMsg, -1); + sqlite3DbFree(db, pzErrMsg); + } else { + sqlite3_result_error(context, sqlite3ErrStr(rc), -1); + } + } +} + +#endif -/* END CRYPTO */ +/* END SQLCIPHER */ #endif diff --git a/src/crypto.h b/src/crypto.h index 8520152..a45b57c 100644 --- a/src/crypto.h +++ b/src/crypto.h @@ -30,15 +30,21 @@ ** SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ** */ -/* BEGIN CRYPTO */ +/* BEGIN SQLCIPHER */ #ifdef SQLITE_HAS_CODEC #ifndef CRYPTO_H #define CRYPTO_H +#if !defined (SQLCIPHER_CRYPTO_CC) \ + && !defined (SQLCIPHER_CRYPTO_LIBTOMCRYPT) \ + && !defined (SQLCIPHER_CRYPTO_OPENSSL) +#define SQLCIPHER_CRYPTO_OPENSSL +#endif + #define FILE_HEADER_SZ 16 #ifndef CIPHER_VERSION -#define CIPHER_VERSION "2.1.1" +#define CIPHER_VERSION "2.2.1" #endif #ifndef CIPHER @@ -82,6 +88,15 @@ #define HMAC_SALT_MASK 0x3a #endif +#ifndef CIPHER_MAX_IV_SZ +#define CIPHER_MAX_IV_SZ 16 +#endif + +#ifndef CIPHER_MAX_KEY_SZ +#define CIPHER_MAX_KEY_SZ 64 +#endif + + #ifdef CODEC_DEBUG #define CODEC_TRACE(X) {printf X;fflush(stdout);} #else @@ -135,16 +150,8 @@ static void cipher_hex2bin(const char *hex, int sz, unsigned char *out){ } /* extensions defined in crypto_impl.c */ - typedef struct codec_ctx codec_ctx; -/* utility functions */ -void* sqlcipher_memset(void *v, unsigned char value, int len); -int sqlcipher_ismemset(const void *v, unsigned char value, int len); -int sqlcipher_memcmp(const void *v0, const void *v1, int len); -int sqlcipher_pseudorandom(void *, int); -void sqlcipher_free(void *, int); - /* activation and initialization */ void sqlcipher_activate(); void sqlcipher_deactivate(); @@ -194,8 +201,7 @@ int sqlcipher_codec_ctx_set_flag(codec_ctx *ctx, unsigned int flag); int sqlcipher_codec_ctx_unset_flag(codec_ctx *ctx, unsigned int flag); int sqlcipher_codec_ctx_get_flag(codec_ctx *ctx, unsigned int flag, int for_ctx); -/* end extensions defined in crypto_impl.c */ - +const char* sqlcipher_codec_get_cipher_provider(codec_ctx *ctx); #endif #endif -/* END CRYPTO */ +/* END SQLCIPHER */ diff --git a/src/crypto_cc.c b/src/crypto_cc.c new file mode 100644 index 0000000..cf932f6 --- /dev/null +++ b/src/crypto_cc.c @@ -0,0 +1,140 @@ +/* +** SQLCipher +** http://sqlcipher.net +** +** Copyright (c) 2008 - 2013, ZETETIC LLC +** All rights reserved. +** +** Redistribution and use in source and binary forms, with or without +** modification, are permitted provided that the following conditions are met: +** * Redistributions of source code must retain the above copyright +** notice, this list of conditions and the following disclaimer. +** * Redistributions in binary form must reproduce the above copyright +** notice, this list of conditions and the following disclaimer in the +** documentation and/or other materials provided with the distribution. +** * Neither the name of the ZETETIC LLC nor the +** names of its contributors may be used to endorse or promote products +** derived from this software without specific prior written permission. +** +** THIS SOFTWARE IS PROVIDED BY ZETETIC LLC ''AS IS'' AND ANY +** EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +** WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +** DISCLAIMED. IN NO EVENT SHALL ZETETIC LLC BE LIABLE FOR ANY +** DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +** (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +** LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +** ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +** (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +** SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +** +*/ +/* BEGIN SQLCIPHER */ +#ifdef SQLITE_HAS_CODEC +#ifdef SQLCIPHER_CRYPTO_CC +#include "crypto.h" +#include "sqlcipher.h" +#include <CommonCrypto/CommonCrypto.h> +#include <Security/SecRandom.h> + +/* generate a defined number of random bytes */ +static int sqlcipher_cc_random (void *ctx, void *buffer, int length) { + return (SecRandomCopyBytes(kSecRandomDefault, length, (uint8_t *)buffer) == 0) ? SQLITE_OK : SQLITE_ERROR; +} + +static const char* sqlcipher_cc_get_provider_name(void *ctx) { + return "commoncrypto"; +} + +static int sqlcipher_cc_hmac(void *ctx, unsigned char *hmac_key, int key_sz, unsigned char *in, int in_sz, unsigned char *in2, int in2_sz, unsigned char *out) { + CCHmacContext hmac_context; + CCHmacInit(&hmac_context, kCCHmacAlgSHA1, hmac_key, key_sz); + CCHmacUpdate(&hmac_context, in, in_sz); + CCHmacUpdate(&hmac_context, in2, in2_sz); + CCHmacFinal(&hmac_context, out); + return SQLITE_OK; +} + +static int sqlcipher_cc_kdf(void *ctx, const char *pass, int pass_sz, unsigned char* salt, int salt_sz, int workfactor, int key_sz, unsigned char *key) { + CCKeyDerivationPBKDF(kCCPBKDF2, pass, pass_sz, salt, salt_sz, kCCPRFHmacAlgSHA1, workfactor, key, key_sz); + return SQLITE_OK; +} + +static int sqlcipher_cc_cipher(void *ctx, int mode, unsigned char *key, int key_sz, unsigned char *iv, unsigned char *in, int in_sz, unsigned char *out) { + CCCryptorRef cryptor; + size_t tmp_csz, csz; + CCOperation op = mode == CIPHER_ENCRYPT ? kCCEncrypt : kCCDecrypt; + + CCCryptorCreate(op, kCCAlgorithmAES128, 0, key, kCCKeySizeAES256, iv, &cryptor); + CCCryptorUpdate(cryptor, in, in_sz, out, in_sz, &tmp_csz); + csz = tmp_csz; + out += tmp_csz; + CCCryptorFinal(cryptor, out, in_sz - csz, &tmp_csz); + csz += tmp_csz; + CCCryptorRelease(cryptor); + assert(size == csz); + + return SQLITE_OK; +} + +static int sqlcipher_cc_set_cipher(void *ctx, const char *cipher_name) { + return SQLITE_OK; +} + +static const char* sqlcipher_cc_get_cipher(void *ctx) { + return "aes-256-cbc"; +} + +static int sqlcipher_cc_get_key_sz(void *ctx) { + return kCCKeySizeAES256; +} + +static int sqlcipher_cc_get_iv_sz(void *ctx) { + return kCCBlockSizeAES128; +} + +static int sqlcipher_cc_get_block_sz(void *ctx) { + return kCCBlockSizeAES128; +} + +static int sqlcipher_cc_get_hmac_sz(void *ctx) { + return CC_SHA1_DIGEST_LENGTH; +} + +static int sqlcipher_cc_ctx_copy(void *target_ctx, void *source_ctx) { + return SQLITE_OK; +} + +static int sqlcipher_cc_ctx_cmp(void *c1, void *c2) { + return SQLITE_OK; +} + +static int sqlcipher_cc_ctx_init(void **ctx) { + return SQLITE_OK; +} + +static int sqlcipher_cc_ctx_free(void **ctx) { + return SQLITE_OK; +} + +int sqlcipher_cc_setup(sqlcipher_provider *p) { + p->random = sqlcipher_cc_random; + p->get_provider_name = sqlcipher_cc_get_provider_name; + p->hmac = sqlcipher_cc_hmac; + p->kdf = sqlcipher_cc_kdf; + p->cipher = sqlcipher_cc_cipher; + p->set_cipher = sqlcipher_cc_set_cipher; + p->get_cipher = sqlcipher_cc_get_cipher; + p->get_key_sz = sqlcipher_cc_get_key_sz; + p->get_iv_sz = sqlcipher_cc_get_iv_sz; + p->get_block_sz = sqlcipher_cc_get_block_sz; + p->get_hmac_sz = sqlcipher_cc_get_hmac_sz; + p->ctx_copy = sqlcipher_cc_ctx_copy; + p->ctx_cmp = sqlcipher_cc_ctx_cmp; + p->ctx_init = sqlcipher_cc_ctx_init; + p->ctx_free = sqlcipher_cc_ctx_free; + return SQLITE_OK; +} + +#endif +#endif +/* END SQLCIPHER */ diff --git a/src/crypto_impl.c b/src/crypto_impl.c index f35144d..1e7fa99 100644 --- a/src/crypto_impl.c +++ b/src/crypto_impl.c @@ -1,10 +1,8 @@ /* ** SQLCipher -** crypto_impl.c developed by Stephen Lombardo (Zetetic LLC) -** sjlombardo at zetetic dot net -** http://zetetic.net +** http://sqlcipher.net ** -** Copyright (c) 2011, ZETETIC LLC +** Copyright (c) 2008 - 2013, ZETETIC LLC ** All rights reserved. ** ** Redistribution and use in source and binary forms, with or without @@ -30,14 +28,12 @@ ** SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ** */ -/* BEGIN CRYPTO */ +/* BEGIN SQLCIPHER */ #ifdef SQLITE_HAS_CODEC -#include <openssl/rand.h> -#include <openssl/evp.h> -#include <openssl/hmac.h> #include "sqliteInt.h" #include "btreeInt.h" +#include "sqlcipher.h" #include "crypto.h" #ifndef OMIT_MEMLOCK #if defined(__unix__) || defined(__APPLE__) @@ -52,9 +48,6 @@ struct and associated functions are defined here */ typedef struct { int derive_key; - EVP_CIPHER *evp_cipher; - EVP_CIPHER_CTX ectx; - HMAC_CTX hctx; int kdf_iter; int fast_kdf_iter; int key_sz; @@ -67,23 +60,15 @@ typedef struct { unsigned char *key; unsigned char *hmac_key; char *pass; + sqlcipher_provider *provider; + void *provider_ctx; } cipher_ctx; -void sqlcipher_cipher_ctx_free(cipher_ctx **); -int sqlcipher_cipher_ctx_cmp(cipher_ctx *, cipher_ctx *); -int sqlcipher_cipher_ctx_copy(cipher_ctx *, cipher_ctx *); -int sqlcipher_cipher_ctx_init(cipher_ctx **); -int sqlcipher_cipher_ctx_set_pass(cipher_ctx *, const void *, int); -int sqlcipher_cipher_ctx_key_derive(codec_ctx *, cipher_ctx *); - -/* prototype for pager HMAC function */ -int sqlcipher_page_hmac(cipher_ctx *, Pgno, unsigned char *, int, unsigned char *); - static unsigned int default_flags = DEFAULT_CIPHER_FLAGS; static unsigned char hmac_salt_mask = HMAC_SALT_MASK; - -static unsigned int openssl_external_init = 0; -static unsigned int openssl_init_count = 0; +static unsigned int sqlcipher_activate_count = 0; +static sqlite3_mutex* sqlcipher_provider_mutex = NULL; +static sqlcipher_provider *default_provider = NULL; struct codec_ctx { int kdf_salt_sz; @@ -96,49 +81,78 @@ struct codec_ctx { cipher_ctx *write_ctx; }; -/* activate and initialize sqlcipher. Most importantly, this will automatically - intialize OpenSSL's EVP system if it hasn't already be externally. Note that - this function may be called multiple times as new codecs are intiialized. - Thus it performs some basic counting to ensure that only the last and final - sqlcipher_deactivate() will free the EVP structures. -*/ +int sqlcipher_register_provider(sqlcipher_provider *p) { + sqlite3_mutex_enter(sqlcipher_provider_mutex); + if(default_provider != NULL && default_provider != p) { + /* only free the current registerd provider if it has been initialized + and it isn't a pointer to the same provider passed to the function + (i.e. protect against a caller calling register twice for the same provider) */ + sqlcipher_free(default_provider, sizeof(sqlcipher_provider)); + } + default_provider = p; + sqlite3_mutex_leave(sqlcipher_provider_mutex); + return SQLITE_OK; +} + +/* return a pointer to the currently registered provider. This will + allow an application to fetch the current registered provider and + make minor changes to it */ +sqlcipher_provider* sqlcipher_get_provider() { + return default_provider; +} + void sqlcipher_activate() { - sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); + sqlite3_mutex_enter(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); - /* we'll initialize openssl and increment the internal init counter - but only if it hasn't been initalized outside of SQLCipher by this program - e.g. on startup */ - if(openssl_init_count == 0 && EVP_get_cipherbyname(CIPHER) != NULL) { - openssl_external_init = 1; + if(sqlcipher_provider_mutex == NULL) { + /* allocate a new mutex to guard access to the provider */ + sqlcipher_provider_mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); } - if(openssl_external_init == 0) { - if(openssl_init_count == 0) { - OpenSSL_add_all_algorithms(); - } - openssl_init_count++; - } - sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); + /* check to see if there is a provider registered at this point + if there no provider registered at this point, register the + default provider */ + if(sqlcipher_get_provider() == NULL) { + sqlcipher_provider *p = sqlcipher_malloc(sizeof(sqlcipher_provider)); +#if defined (SQLCIPHER_CRYPTO_CC) + extern int sqlcipher_cc_setup(sqlcipher_provider *p); + sqlcipher_cc_setup(p); +#elif defined (SQLCIPHER_CRYPTO_LIBTOMCRYPT) + extern int sqlcipher_ltc_setup(sqlcipher_provider *p); + sqlcipher_ltc_setup(p); +#elif defined (SQLCIPHER_CRYPTO_OPENSSL) + extern int sqlcipher_openssl_setup(sqlcipher_provider *p); + sqlcipher_openssl_setup(p); +#else +#error "NO DEFAULT SQLCIPHER CRYPTO PROVIDER DEFINED" +#endif + sqlcipher_register_provider(p); + } + + sqlcipher_activate_count++; /* increment activation count */ + + sqlite3_mutex_leave(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); } -/* deactivate SQLCipher, most imporantly decremeting the activation count and - freeing the EVP structures on the final deactivation to ensure that - OpenSSL memory is cleaned up */ void sqlcipher_deactivate() { - sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); - /* If it is initialized externally, then the init counter should never be greater than zero. - This should prevent SQLCipher from "cleaning up" openssl - when something else in the program might be using it. */ - if(openssl_external_init == 0) { - openssl_init_count--; - /* if the counter reaches zero after it's decremented release EVP memory - Note: this code will only be reached if OpensSSL_add_all_algorithms() - is called by SQLCipher internally. */ - if(openssl_init_count == 0) { - EVP_cleanup(); + sqlite3_mutex_enter(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); + sqlcipher_activate_count--; + /* if no connections are using sqlcipher, cleanup globals */ + if(sqlcipher_activate_count < 1) { + sqlite3_mutex_enter(sqlcipher_provider_mutex); + if(default_provider != NULL) { + sqlcipher_free(default_provider, sizeof(sqlcipher_provider)); + default_provider = NULL; } + sqlite3_mutex_leave(sqlcipher_provider_mutex); + + /* last connection closed, free provider mutex*/ + sqlite3_mutex_free(sqlcipher_provider_mutex); + sqlcipher_provider_mutex = NULL; + + sqlcipher_activate_count = 0; /* reset activation count */ } - sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); + sqlite3_mutex_leave(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); } /* constant time memset using volitile to avoid having the memset @@ -185,11 +199,6 @@ int sqlcipher_memcmp(const void *v0, const void *v1, int len) { return (result != 0); } -/* generate a defined number of pseudorandom bytes */ -int sqlcipher_random (void *buffer, int length) { - return RAND_bytes((unsigned char *)buffer, length); -} - /** * Free and wipe memory. Uses SQLites internal sqlite3_free so that memory * can be countend and memory leak detection works in the test suite. @@ -242,14 +251,24 @@ void* sqlcipher_malloc(int sz) { * returns SQLITE_OK if initialization was successful * returns SQLITE_NOMEM if an error occured allocating memory */ -int sqlcipher_cipher_ctx_init(cipher_ctx **iCtx) { +static int sqlcipher_cipher_ctx_init(cipher_ctx **iCtx) { + int rc; cipher_ctx *ctx; *iCtx = (cipher_ctx *) sqlcipher_malloc(sizeof(cipher_ctx)); ctx = *iCtx; if(ctx == NULL) return SQLITE_NOMEM; - ctx->key = (unsigned char *) sqlcipher_malloc(EVP_MAX_KEY_LENGTH); - ctx->hmac_key = (unsigned char *) sqlcipher_malloc(EVP_MAX_KEY_LENGTH); + ctx->provider = (sqlcipher_provider *) sqlcipher_malloc(sizeof(sqlcipher_provider)); + if(ctx->provider == NULL) return SQLITE_NOMEM; + + /* make a copy of the provider to be used for the duration of the context */ + sqlite3_mutex_enter(sqlcipher_provider_mutex); + memcpy(ctx->provider, default_provider, sizeof(sqlcipher_provider)); + sqlite3_mutex_leave(sqlcipher_provider_mutex); + + if((rc = ctx->provider->ctx_init(&ctx->provider_ctx)) != SQLITE_OK) return rc; + ctx->key = (unsigned char *) sqlcipher_malloc(CIPHER_MAX_KEY_SZ); + ctx->hmac_key = (unsigned char *) sqlcipher_malloc(CIPHER_MAX_KEY_SZ); if(ctx->key == NULL) return SQLITE_NOMEM; if(ctx->hmac_key == NULL) return SQLITE_NOMEM; @@ -262,9 +281,11 @@ int sqlcipher_cipher_ctx_init(cipher_ctx **iCtx) { /** * Free and wipe memory associated with a cipher_ctx */ -void sqlcipher_cipher_ctx_free(cipher_ctx **iCtx) { +static void sqlcipher_cipher_ctx_free(cipher_ctx **iCtx) { cipher_ctx *ctx = *iCtx; CODEC_TRACE(("cipher_ctx_free: entered iCtx=%p\n", iCtx)); + ctx->provider->ctx_free(&ctx->provider_ctx); + sqlcipher_free(ctx->provider, sizeof(sqlcipher_provider)); sqlcipher_free(ctx->key, ctx->key_sz); sqlcipher_free(ctx->hmac_key, ctx->key_sz); sqlcipher_free(ctx->pass, ctx->pass_sz); @@ -277,18 +298,18 @@ void sqlcipher_cipher_ctx_free(cipher_ctx **iCtx) { * returns 0 if all the parameters (except the derived key data) are the same * returns 1 otherwise */ -int sqlcipher_cipher_ctx_cmp(cipher_ctx *c1, cipher_ctx *c2) { +static int sqlcipher_cipher_ctx_cmp(cipher_ctx *c1, cipher_ctx *c2) { CODEC_TRACE(("sqlcipher_cipher_ctx_cmp: entered c1=%p c2=%p\n", c1, c2)); if( - c1->evp_cipher == c2->evp_cipher - && c1->iv_sz == c2->iv_sz + c1->iv_sz == c2->iv_sz && c1->kdf_iter == c2->kdf_iter && c1->fast_kdf_iter == c2->fast_kdf_iter && c1->key_sz == c2->key_sz && c1->pass_sz == c2->pass_sz && c1->flags == c2->flags && c1->hmac_sz == c2->hmac_sz + && c1->provider->ctx_cmp(c1->provider_ctx, c2->provider_ctx) && ( c1->pass == c2->pass || !sqlcipher_memcmp((const unsigned char*)c1->pass, @@ -307,19 +328,27 @@ int sqlcipher_cipher_ctx_cmp(cipher_ctx *c1, cipher_ctx *c2) { * returns SQLITE_OK if initialization was successful * returns SQLITE_NOMEM if an error occured allocating memory */ -int sqlcipher_cipher_ctx_copy(cipher_ctx *target, cipher_ctx *source) { +static int sqlcipher_cipher_ctx_copy(cipher_ctx *target, cipher_ctx *source) { void *key = target->key; void *hmac_key = target->hmac_key; + void *provider = target->provider; + void *provider_ctx = target->provider_ctx; CODEC_TRACE(("sqlcipher_cipher_ctx_copy: entered target=%p, source=%p\n", target, source)); sqlcipher_free(target->pass, target->pass_sz); memcpy(target, source, sizeof(cipher_ctx)); target->key = key; //restore pointer to previously allocated key data - memcpy(target->key, source->key, EVP_MAX_KEY_LENGTH); + memcpy(target->key, source->key, CIPHER_MAX_KEY_SZ); target->hmac_key = hmac_key; //restore pointer to previously allocated hmac key data - memcpy(target->hmac_key, source->hmac_key, EVP_MAX_KEY_LENGTH); + memcpy(target->hmac_key, source->hmac_key, CIPHER_MAX_KEY_SZ); + + target->provider = provider; // restore pointer to previouly allocated provider; + memcpy(target->provider, source->provider, sizeof(sqlcipher_provider)); + + target->provider_ctx = provider_ctx; // restore pointer to previouly allocated provider context; + target->provider->ctx_copy(target->provider_ctx, source->provider_ctx); target->pass = sqlcipher_malloc(source->pass_sz); if(target->pass == NULL) return SQLITE_NOMEM; @@ -336,7 +365,7 @@ int sqlcipher_cipher_ctx_copy(cipher_ctx *target, cipher_ctx *source) { * returns SQLITE_NOMEM if an error occured allocating memory * returns SQLITE_ERROR if the key couldn't be set because the pass was null or size was zero */ -int sqlcipher_cipher_ctx_set_pass(cipher_ctx *ctx, const void *zKey, int nKey) { +static int sqlcipher_cipher_ctx_set_pass(cipher_ctx *ctx, const void *zKey, int nKey) { sqlcipher_free(ctx->pass, ctx->pass_sz); ctx->pass_sz = nKey; if(zKey && nKey) { @@ -366,11 +395,12 @@ int sqlcipher_codec_ctx_set_cipher(codec_ctx *ctx, const char *cipher_name, int cipher_ctx *c_ctx = for_ctx ? ctx->write_ctx : ctx->read_ctx; int rc; - c_ctx->evp_cipher = (EVP_CIPHER *) EVP_get_cipherbyname(cipher_name); - c_ctx->key_sz = EVP_CIPHER_key_length(c_ctx->evp_cipher); - c_ctx->iv_sz = EVP_CIPHER_iv_length(c_ctx->evp_cipher); - c_ctx->block_sz = EVP_CIPHER_block_size(c_ctx->evp_cipher); - c_ctx->hmac_sz = EVP_MD_size(EVP_sha1()); + c_ctx->provider->set_cipher(c_ctx->provider_ctx, cipher_name); + + c_ctx->key_sz = c_ctx->provider->get_key_sz(c_ctx->provider_ctx); + c_ctx->iv_sz = c_ctx->provider->get_iv_sz(c_ctx->provider_ctx); + c_ctx->block_sz = c_ctx->provider->get_block_sz(c_ctx->provider_ctx); + c_ctx->hmac_sz = c_ctx->provider->get_hmac_sz(c_ctx->provider_ctx); c_ctx->derive_key = 1; if(for_ctx == 2) @@ -382,8 +412,7 @@ int sqlcipher_codec_ctx_set_cipher(codec_ctx *ctx, const char *cipher_name, int const char* sqlcipher_codec_ctx_get_cipher(codec_ctx *ctx, int for_ctx) { cipher_ctx *c_ctx = for_ctx ? ctx->write_ctx : ctx->read_ctx; - EVP_CIPHER *evp_cipher = c_ctx->evp_cipher; - return EVP_CIPHER_name(evp_cipher); + return c_ctx->provider->get_cipher(c_ctx->provider_ctx); } int sqlcipher_codec_ctx_set_kdf_iter(codec_ctx *ctx, int kdf_iter, int for_ctx) { @@ -444,7 +473,7 @@ unsigned char sqlcipher_get_hmac_salt_mask() { /* set the codec flag for whether this individual database should be using hmac */ int sqlcipher_codec_ctx_set_use_hmac(codec_ctx *ctx, int use) { - int reserve = EVP_MAX_IV_LENGTH; /* base reserve size will be IV only */ + int reserve = CIPHER_MAX_IV_SZ; /* base reserve size will be IV only */ if(use) reserve += ctx->read_ctx->hmac_sz; /* if reserve will include hmac, update that size */ @@ -568,7 +597,7 @@ int sqlcipher_codec_ctx_init(codec_ctx **iCtx, Db *pDb, Pager *pPager, sqlite3_f if(fd == NULL || sqlite3OsRead(fd, ctx->kdf_salt, FILE_HEADER_SZ, 0) != SQLITE_OK) { /* if unable to read the bytes, generate random salt */ - if(sqlcipher_random(ctx->kdf_salt, FILE_HEADER_SZ) != 1) return SQLITE_ERROR; + if(ctx->read_ctx->provider->random(ctx->read_ctx->provider_ctx, ctx->kdf_salt, FILE_HEADER_SZ) != SQLITE_OK) return SQLITE_ERROR; } if((rc = sqlcipher_codec_ctx_set_cipher(ctx, CIPHER, 0)) != SQLITE_OK) return rc; @@ -608,7 +637,7 @@ static void sqlcipher_put4byte_le(unsigned char *p, u32 v) { p[3] = (u8)(v>>24); } -int sqlcipher_page_hmac(cipher_ctx *ctx, Pgno pgno, unsigned char *in, int in_sz, unsigned char *out) { +static int sqlcipher_page_hmac(cipher_ctx *ctx, Pgno pgno, unsigned char *in, int in_sz, unsigned char *out) { unsigned char pgno_raw[sizeof(pgno)]; /* we may convert page number to consistent representation before calculating MAC for compatibility across big-endian and little-endian platforms. @@ -626,16 +655,14 @@ int sqlcipher_page_hmac(cipher_ctx *ctx, Pgno pgno, unsigned char *in, int in_sz memcpy(pgno_raw, &pgno, sizeof(pgno)); } - HMAC_CTX_init(&ctx->hctx); - HMAC_Init_ex(&ctx->hctx, ctx->hmac_key, ctx->key_sz, EVP_sha1(), NULL); - /* include the encrypted page data, initialization vector, and page number in HMAC. This will prevent both tampering with the ciphertext, manipulation of the IV, or resequencing otherwise valid pages out of order in a database */ - HMAC_Update(&ctx->hctx, in, in_sz); - HMAC_Update(&ctx->hctx, (const unsigned char*) pgno_raw, sizeof(pgno)); - HMAC_Final(&ctx->hctx, out, NULL); - HMAC_CTX_cleanup(&ctx->hctx); + ctx->provider->hmac( + ctx->provider_ctx, ctx->hmac_key, + ctx->key_sz, in, + in_sz, (unsigned char*) &pgno_raw, + sizeof(pgno), out); return SQLITE_OK; } @@ -650,7 +677,7 @@ int sqlcipher_page_hmac(cipher_ctx *ctx, Pgno pgno, unsigned char *in, int in_sz int sqlcipher_page_cipher(codec_ctx *ctx, int for_ctx, Pgno pgno, int mode, int page_sz, unsigned char *in, unsigned char *out) { cipher_ctx *c_ctx = for_ctx ? ctx->write_ctx : ctx->read_ctx; unsigned char *iv_in, *iv_out, *hmac_in, *hmac_out, *out_start; - int tmp_csz, csz, size; + int size; /* calculate some required positions into various buffers */ size = page_sz - c_ctx->reserve_sz; /* adjust size to useable size and memset reserve at end of page */ @@ -675,7 +702,7 @@ int sqlcipher_page_cipher(codec_ctx *ctx, int for_ctx, Pgno pgno, int mode, int if(mode == CIPHER_ENCRYPT) { /* start at front of the reserve block, write random data to the end */ - if(sqlcipher_random(iv_out, c_ctx->reserve_sz) != 1) return SQLITE_ERROR; + if(c_ctx->provider->random(c_ctx->provider_ctx, iv_out, c_ctx->reserve_sz) != SQLITE_OK) return SQLITE_ERROR; } else { /* CIPHER_DECRYPT */ memcpy(iv_out, iv_in, c_ctx->iv_sz); /* copy the iv from the input to output buffer */ } @@ -707,17 +734,8 @@ int sqlcipher_page_cipher(codec_ctx *ctx, int for_ctx, Pgno pgno, int mode, int } } } - - EVP_CipherInit(&c_ctx->ectx, c_ctx->evp_cipher, NULL, NULL, mode); - EVP_CIPHER_CTX_set_padding(&c_ctx->ectx, 0); - EVP_CipherInit(&c_ctx->ectx, NULL, c_ctx->key, iv_out, mode); - EVP_CipherUpdate(&c_ctx->ectx, out, &tmp_csz, in, size); - csz = tmp_csz; - out += tmp_csz; - EVP_CipherFinal(&c_ctx->ectx, out, &tmp_csz); - csz += tmp_csz; - EVP_CIPHER_CTX_cleanup(&c_ctx->ectx); - assert(size == csz); + + c_ctx->provider->cipher(c_ctx->provider_ctx, mode, c_ctx->key, c_ctx->key_sz, iv_out, in, size, out); if((c_ctx->flags & CIPHER_FLAG_HMAC) && (mode == CIPHER_ENCRYPT)) { sqlcipher_page_hmac(c_ctx, pgno, out_start, size + c_ctx->iv_sz, hmac_out); @@ -739,7 +757,7 @@ int sqlcipher_page_cipher(codec_ctx *ctx, int for_ctx, Pgno pgno, int mode, int * returns SQLITE_OK if initialization was successful * returns SQLITE_ERROR if the key could't be derived (for instance if pass is NULL or pass_sz is 0) */ -int sqlcipher_cipher_ctx_key_derive(codec_ctx *ctx, cipher_ctx *c_ctx) { +static int sqlcipher_cipher_ctx_key_derive(codec_ctx *ctx, cipher_ctx *c_ctx) { CODEC_TRACE(("codec_key_derive: entered c_ctx->pass=%s, c_ctx->pass_sz=%d \ ctx->kdf_salt=%p ctx->kdf_salt_sz=%d c_ctx->kdf_iter=%d \ ctx->hmac_kdf_salt=%p, c_ctx->fast_kdf_iter=%d c_ctx->key_sz=%d\n", @@ -755,9 +773,9 @@ int sqlcipher_cipher_ctx_key_derive(codec_ctx *ctx, cipher_ctx *c_ctx) { cipher_hex2bin(z, n, c_ctx->key); } else { CODEC_TRACE(("codec_key_derive: deriving key using full PBKDF2 with %d iterations\n", c_ctx->kdf_iter)); - PKCS5_PBKDF2_HMAC_SHA1( c_ctx->pass, c_ctx->pass_sz, - ctx->kdf_salt, ctx->kdf_salt_sz, - c_ctx->kdf_iter, c_ctx->key_sz, c_ctx->key); + c_ctx->provider->kdf(c_ctx->provider_ctx, (const char*) c_ctx->pass, c_ctx->pass_sz, + ctx->kdf_salt, ctx->kdf_salt_sz, c_ctx->kdf_iter, + c_ctx->key_sz, c_ctx->key); } @@ -779,9 +797,11 @@ int sqlcipher_cipher_ctx_key_derive(codec_ctx *ctx, cipher_ctx *c_ctx) { CODEC_TRACE(("codec_key_derive: deriving hmac key from encryption key using PBKDF2 with %d iterations\n", c_ctx->fast_kdf_iter)); - PKCS5_PBKDF2_HMAC_SHA1( (const char*)c_ctx->key, c_ctx->key_sz, - ctx->hmac_kdf_salt, ctx->kdf_salt_sz, - c_ctx->fast_kdf_iter, c_ctx->key_sz, c_ctx->hmac_key); + + + c_ctx->provider->kdf(c_ctx->provider_ctx, (const char*)c_ctx->key, c_ctx->key_sz, + ctx->hmac_kdf_salt, ctx->kdf_salt_sz, c_ctx->fast_kdf_iter, + c_ctx->key_sz, c_ctx->hmac_key); } c_ctx->derive_key = 0; @@ -815,202 +835,9 @@ int sqlcipher_codec_key_copy(codec_ctx *ctx, int source) { } } - -#ifndef OMIT_EXPORT - -/* - * Implementation of an "export" function that allows a caller - * to duplicate the main database to an attached database. This is intended - * as a conveneince for users who need to: - * - * 1. migrate from an non-encrypted database to an encrypted database - * 2. move from an encrypted database to a non-encrypted database - * 3. convert beween the various flavors of encrypted databases. - * - * This implementation is based heavily on the procedure and code used - * in vacuum.c, but is exposed as a function that allows export to any - * named attached database. - */ - -/* -** Finalize a prepared statement. If there was an error, store the -** text of the error message in *pzErrMsg. Return the result code. -** -** Based on vacuumFinalize from vacuum.c -*/ -static int sqlcipher_finalize(sqlite3 *db, sqlite3_stmt *pStmt, char **pzErrMsg){ - int rc; - rc = sqlite3VdbeFinalize((Vdbe*)pStmt); - if( rc ){ - sqlite3SetString(pzErrMsg, db, sqlite3_errmsg(db)); - } - return rc; -} - -/* -** Execute zSql on database db. Return an error code. -** -** Based on execSql from vacuum.c -*/ -static int sqlcipher_execSql(sqlite3 *db, char **pzErrMsg, const char *zSql){ - sqlite3_stmt *pStmt; - VVA_ONLY( int rc; ) - if( !zSql ){ - return SQLITE_NOMEM; - } - if( SQLITE_OK!=sqlite3_prepare(db, zSql, -1, &pStmt, 0) ){ - sqlite3SetString(pzErrMsg, db, sqlite3_errmsg(db)); - return sqlite3_errcode(db); - } - VVA_ONLY( rc = ) sqlite3_step(pStmt); - assert( rc!=SQLITE_ROW ); - return sqlcipher_finalize(db, pStmt, pzErrMsg); +const char* sqlcipher_codec_get_cipher_provider(codec_ctx *ctx) { + return ctx->read_ctx->provider->get_provider_name(ctx->read_ctx); } -/* -** Execute zSql on database db. The statement returns exactly -** one column. Execute this as SQL on the same database. -** -** Based on execExecSql from vacuum.c -*/ -static int sqlcipher_execExecSql(sqlite3 *db, char **pzErrMsg, const char *zSql){ - sqlite3_stmt *pStmt; - int rc; - - rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); - if( rc!=SQLITE_OK ) return rc; - - while( SQLITE_ROW==sqlite3_step(pStmt) ){ - rc = sqlcipher_execSql(db, pzErrMsg, (char*)sqlite3_column_text(pStmt, 0)); - if( rc!=SQLITE_OK ){ - sqlcipher_finalize(db, pStmt, pzErrMsg); - return rc; - } - } - - return sqlcipher_finalize(db, pStmt, pzErrMsg); -} - -/* - * copy database and schema from the main database to an attached database - * - * Based on sqlite3RunVacuum from vacuum.c -*/ -void sqlcipher_exportFunc(sqlite3_context *context, int argc, sqlite3_value **argv) { - sqlite3 *db = sqlite3_context_db_handle(context); - const char* attachedDb = (const char*) sqlite3_value_text(argv[0]); - int saved_flags; /* Saved value of the db->flags */ - int saved_nChange; /* Saved value of db->nChange */ - int saved_nTotalChange; /* Saved value of db->nTotalChange */ - void (*saved_xTrace)(void*,const char*); /* Saved db->xTrace */ - int rc = SQLITE_OK; /* Return code from service routines */ - char *zSql = NULL; /* SQL statements */ - char *pzErrMsg = NULL; - - saved_flags = db->flags; - saved_nChange = db->nChange; - saved_nTotalChange = db->nTotalChange; - saved_xTrace = db->xTrace; - db->flags |= SQLITE_WriteSchema | SQLITE_IgnoreChecks | SQLITE_PreferBuiltin; - db->flags &= ~(SQLITE_ForeignKeys | SQLITE_ReverseOrder); - db->xTrace = 0; - - /* Query the schema of the main database. Create a mirror schema - ** in the temporary database. - */ - zSql = sqlite3_mprintf( - "SELECT 'CREATE TABLE %s.' || substr(sql,14) " - " FROM sqlite_master WHERE type='table' AND name!='sqlite_sequence'" - " AND rootpage>0" - , attachedDb); - rc = (zSql == NULL) ? SQLITE_NOMEM : sqlcipher_execExecSql(db, &pzErrMsg, zSql); - if( rc!=SQLITE_OK ) goto end_of_export; - sqlite3_free(zSql); - - zSql = sqlite3_mprintf( - "SELECT 'CREATE INDEX %s.' || substr(sql,14)" - " FROM sqlite_master WHERE sql LIKE 'CREATE INDEX %%' " - , attachedDb); - rc = (zSql == NULL) ? SQLITE_NOMEM : sqlcipher_execExecSql(db, &pzErrMsg, zSql); - if( rc!=SQLITE_OK ) goto end_of_export; - sqlite3_free(zSql); - - zSql = sqlite3_mprintf( - "SELECT 'CREATE UNIQUE INDEX %s.' || substr(sql,21) " - " FROM sqlite_master WHERE sql LIKE 'CREATE UNIQUE INDEX %%'" - , attachedDb); - rc = (zSql == NULL) ? SQLITE_NOMEM : sqlcipher_execExecSql(db, &pzErrMsg, zSql); - if( rc!=SQLITE_OK ) goto end_of_export; - sqlite3_free(zSql); - - /* Loop through the tables in the main database. For each, do - ** an "INSERT INTO rekey_db.xxx SELECT * FROM main.xxx;" to copy - ** the contents to the temporary database. - */ - zSql = sqlite3_mprintf( - "SELECT 'INSERT INTO %s.' || quote(name) " - "|| ' SELECT * FROM main.' || quote(name) || ';'" - "FROM main.sqlite_master " - "WHERE type = 'table' AND name!='sqlite_sequence' " - " AND rootpage>0" - , attachedDb); - rc = (zSql == NULL) ? SQLITE_NOMEM : sqlcipher_execExecSql(db, &pzErrMsg, zSql); - if( rc!=SQLITE_OK ) goto end_of_export; - sqlite3_free(zSql); - - /* Copy over the sequence table - */ - zSql = sqlite3_mprintf( - "SELECT 'DELETE FROM %s.' || quote(name) || ';' " - "FROM %s.sqlite_master WHERE name='sqlite_sequence' " - , attachedDb, attachedDb); - rc = (zSql == NULL) ? SQLITE_NOMEM : sqlcipher_execExecSql(db, &pzErrMsg, zSql); - if( rc!=SQLITE_OK ) goto end_of_export; - sqlite3_free(zSql); - - zSql = sqlite3_mprintf( - "SELECT 'INSERT INTO %s.' || quote(name) " - "|| ' SELECT * FROM main.' || quote(name) || ';' " - "FROM %s.sqlite_master WHERE name=='sqlite_sequence';" - , attachedDb, attachedDb); - rc = (zSql == NULL) ? SQLITE_NOMEM : sqlcipher_execExecSql(db, &pzErrMsg, zSql); - if( rc!=SQLITE_OK ) goto end_of_export; - sqlite3_free(zSql); - - /* Copy the triggers, views, and virtual tables from the main database - ** over to the temporary database. None of these objects has any - ** associated storage, so all we have to do is copy their entries - ** from the SQLITE_MASTER table. - */ - zSql = sqlite3_mprintf( - "INSERT INTO %s.sqlite_master " - " SELECT type, name, tbl_name, rootpage, sql" - " FROM main.sqlite_master" - " WHERE type='view' OR type='trigger'" - " OR (type='table' AND rootpage=0)" - , attachedDb); - rc = (zSql == NULL) ? SQLITE_NOMEM : sqlcipher_execSql(db, &pzErrMsg, zSql); - if( rc!=SQLITE_OK ) goto end_of_export; - sqlite3_free(zSql); - - zSql = NULL; -end_of_export: - db->flags = saved_flags; - db->nChange = saved_nChange; - db->nTotalChange = saved_nTotalChange; - db->xTrace = saved_xTrace; - - sqlite3_free(zSql); - - if(rc) { - if(pzErrMsg != NULL) { - sqlite3_result_error(context, pzErrMsg, -1); - sqlite3DbFree(db, pzErrMsg); - } else { - sqlite3_result_error(context, sqlite3ErrStr(rc), -1); - } - } -} - -#endif #endif +/* END SQLCIPHER */ diff --git a/src/crypto_libtomcrypt.c b/src/crypto_libtomcrypt.c new file mode 100644 index 0000000..0299771 --- /dev/null +++ b/src/crypto_libtomcrypt.c @@ -0,0 +1,215 @@ +/* +** SQLCipher +** http://sqlcipher.net +** +** Copyright (c) 2008 - 2013, ZETETIC LLC +** All rights reserved. +** +** Redistribution and use in source and binary forms, with or without +** modification, are permitted provided that the following conditions are met: +** * Redistributions of source code must retain the above copyright +** notice, this list of conditions and the following disclaimer. +** * Redistributions in binary form must reproduce the above copyright +** notice, this list of conditions and the following disclaimer in the +** documentation and/or other materials provided with the distribution. +** * Neither the name of the ZETETIC LLC nor the +** names of its contributors may be used to endorse or promote products +** derived from this software without specific prior written permission. +** +** THIS SOFTWARE IS PROVIDED BY ZETETIC LLC ''AS IS'' AND ANY +** EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +** WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +** DISCLAIMED. IN NO EVENT SHALL ZETETIC LLC BE LIABLE FOR ANY +** DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +** (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +** LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +** ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +** (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +** SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +** +*/ +/* BEGIN SQLCIPHER */ +#ifdef SQLITE_HAS_CODEC +#ifdef SQLCIPHER_CRYPTO_LIBTOMCRYPT +#include "sqliteInt.h" +#include "sqlcipher.h" +#include <tomcrypt.h> + +typedef struct { + prng_state prng; +} ltc_ctx; + +static unsigned int ltc_init = 0; + +static int sqlcipher_ltc_add_random(void *ctx, void *buffer, int length) { + ltc_ctx *ltc = (ltc_ctx*)ctx; + int rc = fortuna_add_entropy(buffer, length, &(ltc->prng)); + return rc != CRYPT_OK ? SQLITE_ERROR : SQLITE_OK; +} + +static int sqlcipher_ltc_activate(void *ctx) { + ltc_ctx *ltc = (ltc_ctx*)ctx; + int random_buffer_sz = 32; + unsigned char random_buffer[random_buffer_sz]; + + if(ltc_init == 0) { + if(register_prng(&fortuna_desc) != CRYPT_OK) return SQLITE_ERROR; + if(register_cipher(&rijndael_desc) != CRYPT_OK) return SQLITE_ERROR; + if(register_hash(&sha1_desc) != CRYPT_OK) return SQLITE_ERROR; + ltc_init = 1; + } + if(fortuna_start(&(ltc->prng)) != CRYPT_OK) { + return SQLITE_ERROR; + } + sqlite3_randomness(random_buffer_sz, &random_buffer); + if(sqlcipher_ltc_add_random(ctx, random_buffer, random_buffer_sz) != SQLITE_OK) { + return SQLITE_ERROR; + } + if(sqlcipher_ltc_add_random(ctx, <c, sizeof(ltc_ctx*)) != SQLITE_OK) { + return SQLITE_ERROR; + } + if(fortuna_ready(&(ltc->prng)) != CRYPT_OK) { + return SQLITE_ERROR; + } + return SQLITE_OK; +} + +static int sqlcipher_ltc_deactivate(void *ctx) { + ltc_ctx *ltc = (ltc_ctx*)ctx; + fortuna_done(&(ltc->prng)); +} + +static const char* sqlcipher_ltc_get_provider_name(void *ctx) { + return "libtomcrypt"; +} + +static int sqlcipher_ltc_random(void *ctx, void *buffer, int length) { + ltc_ctx *ltc = (ltc_ctx*)ctx; + int rc; + + if((rc = fortuna_ready(&(ltc->prng))) != CRYPT_OK) { + return SQLITE_ERROR; + } + fortuna_read(buffer, length, &(ltc->prng)); + return SQLITE_OK; +} + +static int sqlcipher_ltc_hmac(void *ctx, unsigned char *hmac_key, int key_sz, unsigned char *in, int in_sz, unsigned char *in2, int in2_sz, unsigned char *out) { + int rc, hash_idx; + hmac_state hmac; + unsigned long outlen = key_sz; + + hash_idx = find_hash("sha1"); + if((rc = hmac_init(&hmac, hash_idx, hmac_key, key_sz)) != CRYPT_OK) return SQLITE_ERROR; + if((rc = hmac_process(&hmac, in, in_sz)) != CRYPT_OK) return SQLITE_ERROR; + if((rc = hmac_process(&hmac, in2, in2_sz)) != CRYPT_OK) return SQLITE_ERROR; + if((rc = hmac_done(&hmac, out, &outlen)) != CRYPT_OK) return SQLITE_ERROR; + return SQLITE_OK; +} + +static int sqlcipher_ltc_kdf(void *ctx, const char *pass, int pass_sz, unsigned char* salt, int salt_sz, int workfactor, int key_sz, unsigned char *key) { + int rc, hash_idx; + unsigned long outlen = key_sz; + unsigned long random_buffer_sz = 256; + char random_buffer[random_buffer_sz]; + ltc_ctx *ltc = (ltc_ctx*)ctx; + + hash_idx = find_hash("sha1"); + if((rc = pkcs_5_alg2(pass, pass_sz, salt, salt_sz, + workfactor, hash_idx, key, &outlen)) != CRYPT_OK) { + return SQLITE_ERROR; + } + if((rc = pkcs_5_alg2(key, key_sz, salt, salt_sz, + 1, hash_idx, random_buffer, &random_buffer_sz)) != CRYPT_OK) { + return SQLITE_ERROR; + } + sqlcipher_ltc_add_random(ctx, random_buffer, random_buffer_sz); + return SQLITE_OK; +} + +static const char* sqlcipher_ltc_get_cipher(void *ctx) { + return "rijndael"; +} + +static int sqlcipher_ltc_cipher(void *ctx, int mode, unsigned char *key, int key_sz, unsigned char *iv, unsigned char *in, int in_sz, unsigned char *out) { + int rc, cipher_idx, hash_idx; + symmetric_CBC cbc; + + if((cipher_idx = find_cipher(sqlcipher_ltc_get_cipher(ctx))) == -1) return SQLITE_ERROR; + if((rc = cbc_start(cipher_idx, iv, key, key_sz, 0, &cbc)) != CRYPT_OK) return SQLITE_ERROR; + rc = mode == 1 ? cbc_encrypt(in, out, in_sz, &cbc) : cbc_decrypt(in, out, in_sz, &cbc); + if(rc != CRYPT_OK) return SQLITE_ERROR; + cbc_done(&cbc); + return SQLITE_OK; +} + +static int sqlcipher_ltc_set_cipher(void *ctx, const char *cipher_name) { + return SQLITE_OK; +} + +static int sqlcipher_ltc_get_key_sz(void *ctx) { + int cipher_idx = find_cipher(sqlcipher_ltc_get_cipher(ctx)); + return cipher_descriptor[cipher_idx].max_key_length; +} + +static int sqlcipher_ltc_get_iv_sz(void *ctx) { + int cipher_idx = find_cipher(sqlcipher_ltc_get_cipher(ctx)); + return cipher_descriptor[cipher_idx].block_length; +} + +static int sqlcipher_ltc_get_block_sz(void *ctx) { + int cipher_idx = find_cipher(sqlcipher_ltc_get_cipher(ctx)); + return cipher_descriptor[cipher_idx].block_length; +} + +static int sqlcipher_ltc_get_hmac_sz(void *ctx) { + int hash_idx = find_hash("sha1"); + return hash_descriptor[hash_idx].hashsize; +} + +static int sqlcipher_ltc_ctx_copy(void *target_ctx, void *source_ctx) { + memcpy(target_ctx, source_ctx, sizeof(ltc_ctx)); + return SQLITE_OK; +} + +static int sqlcipher_ltc_ctx_cmp(void *c1, void *c2) { + return 1; +} + +static int sqlcipher_ltc_ctx_init(void **ctx) { + *ctx = sqlcipher_malloc(sizeof(ltc_ctx)); + if(*ctx == NULL) return SQLITE_NOMEM; + sqlcipher_ltc_activate(*ctx); + return SQLITE_OK; +} + +static int sqlcipher_ltc_ctx_free(void **ctx) { + sqlcipher_ltc_deactivate(&ctx); + sqlcipher_free(*ctx, sizeof(ltc_ctx)); + return SQLITE_OK; +} + +int sqlcipher_ltc_setup(sqlcipher_provider *p) { + p->activate = sqlcipher_ltc_activate; + p->deactivate = sqlcipher_ltc_deactivate; + p->get_provider_name = sqlcipher_ltc_get_provider_name; + p->random = sqlcipher_ltc_random; + p->hmac = sqlcipher_ltc_hmac; + p->kdf = sqlcipher_ltc_kdf; + p->cipher = sqlcipher_ltc_cipher; + p->set_cipher = sqlcipher_ltc_set_cipher; + p->get_cipher = sqlcipher_ltc_get_cipher; + p->get_key_sz = sqlcipher_ltc_get_key_sz; + p->get_iv_sz = sqlcipher_ltc_get_iv_sz; + p->get_block_sz = sqlcipher_ltc_get_block_sz; + p->get_hmac_sz = sqlcipher_ltc_get_hmac_sz; + p->ctx_copy = sqlcipher_ltc_ctx_copy; + p->ctx_cmp = sqlcipher_ltc_ctx_cmp; + p->ctx_init = sqlcipher_ltc_ctx_init; + p->ctx_free = sqlcipher_ltc_ctx_free; + p->add_random = sqlcipher_ltc_add_random; +} + +#endif +#endif +/* END SQLCIPHER */ diff --git a/src/crypto_openssl.c b/src/crypto_openssl.c new file mode 100644 index 0000000..6035b7c --- /dev/null +++ b/src/crypto_openssl.c @@ -0,0 +1,251 @@ +/* +** SQLCipher +** http://sqlcipher.net +** +** Copyright (c) 2008 - 2013, ZETETIC LLC +** All rights reserved. +** +** Redistribution and use in source and binary forms, with or without +** modification, are permitted provided that the following conditions are met: +** * Redistributions of source code must retain the above copyright +** notice, this list of conditions and the following disclaimer. +** * Redistributions in binary form must reproduce the above copyright +** notice, this list of conditions and the following disclaimer in the +** documentation and/or other materials provided with the distribution. +** * Neither the name of the ZETETIC LLC nor the +** names of its contributors may be used to endorse or promote products +** derived from this software without specific prior written permission. +** +** THIS SOFTWARE IS PROVIDED BY ZETETIC LLC ''AS IS'' AND ANY +** EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +** WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +** DISCLAIMED. IN NO EVENT SHALL ZETETIC LLC BE LIABLE FOR ANY +** DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +** (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +** LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +** ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +** (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +** SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +** +*/ +/* BEGIN SQLCIPHER */ +#ifdef SQLITE_HAS_CODEC +#ifdef SQLCIPHER_CRYPTO_OPENSSL +#include "sqliteInt.h" +#include "crypto.h" +#include "sqlcipher.h" +#include <openssl/rand.h> +#include <openssl/evp.h> +#include <openssl/hmac.h> + +typedef struct { + EVP_CIPHER *evp_cipher; +} openssl_ctx; + + +static unsigned int openssl_external_init = 0; +static unsigned int openssl_init_count = 0; +static sqlite3_mutex* openssl_rand_mutex = NULL; + +static int sqlcipher_openssl_add_random(void *ctx, void *buffer, int length) { +#ifndef SQLCIPHER_OPENSSL_NO_MUTEX_RAND + sqlite3_mutex_enter(openssl_rand_mutex); +#endif + RAND_add(buffer, length, 0); +#ifndef SQLCIPHER_OPENSSL_NO_MUTEX_RAND + sqlite3_mutex_leave(openssl_rand_mutex); +#endif + return SQLITE_OK; +} + +/* activate and initialize sqlcipher. Most importantly, this will automatically + intialize OpenSSL's EVP system if it hasn't already be externally. Note that + this function may be called multiple times as new codecs are intiialized. + Thus it performs some basic counting to ensure that only the last and final + sqlcipher_openssl_deactivate() will free the EVP structures. +*/ +static int sqlcipher_openssl_activate(void *ctx) { + /* initialize openssl and increment the internal init counter + but only if it hasn't been initalized outside of SQLCipher by this program + e.g. on startup */ + sqlite3_mutex_enter(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); + + if(openssl_init_count == 0 && EVP_get_cipherbyname(CIPHER) != NULL) { + /* if openssl has not yet been initialized by this library, but + a call to get_cipherbyname works, then the openssl library + has been initialized externally already. */ + openssl_external_init = 1; + } + + if(openssl_init_count == 0 && openssl_external_init == 0) { + /* if the library was not externally initialized, then should be now */ + OpenSSL_add_all_algorithms(); + } + +#ifndef SQLCIPHER_OPENSSL_NO_MUTEX_RAND + if(openssl_rand_mutex == NULL) { + /* allocate a mutex to guard against concurrent calls to RAND_bytes() */ + openssl_rand_mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); + } +#endif + + openssl_init_count++; + sqlite3_mutex_leave(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); + return SQLITE_OK; +} + +/* deactivate SQLCipher, most imporantly decremeting the activation count and + freeing the EVP structures on the final deactivation to ensure that + OpenSSL memory is cleaned up */ +static int sqlcipher_openssl_deactivate(void *ctx) { + sqlite3_mutex_enter(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); + openssl_init_count--; + + if(openssl_init_count == 0) { + if(openssl_external_init == 0) { + /* if OpenSSL hasn't be initialized externally, and the counter reaches zero + after it's decremented, release EVP memory + Note: this code will only be reached if OpensSSL_add_all_algorithms() + is called by SQLCipher internally. This should prevent SQLCipher from + "cleaning up" openssl when it was initialized externally by the program */ + EVP_cleanup(); + } +#ifndef SQLCIPHER_OPENSSL_NO_MUTEX_RAND + sqlite3_mutex_free(openssl_rand_mutex); + openssl_rand_mutex = NULL; +#endif + } + sqlite3_mutex_leave(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); + return SQLITE_OK; +} + +static const char* sqlcipher_openssl_get_provider_name(void *ctx) { + return "openssl"; +} + +/* generate a defined number of random bytes */ +static int sqlcipher_openssl_random (void *ctx, void *buffer, int length) { + int rc = 0; + /* concurrent calls to RAND_bytes can cause a crash under some openssl versions when a + naive application doesn't use CRYPTO_set_locking_callback and + CRYPTO_THREADID_set_callback to ensure openssl thread safety. + This is simple workaround to prevent this common crash + but a more proper solution is that applications setup platform-appropriate + thread saftey in openssl externally */ +#ifndef SQLCIPHER_OPENSSL_NO_MUTEX_RAND + sqlite3_mutex_enter(openssl_rand_mutex); +#endif + rc = RAND_bytes((unsigned char *)buffer, length); +#ifndef SQLCIPHER_OPENSSL_NO_MUTEX_RAND + sqlite3_mutex_leave(openssl_rand_mutex); +#endif + return (rc == 1) ? SQLITE_OK : SQLITE_ERROR; +} + +static int sqlcipher_openssl_hmac(void *ctx, unsigned char *hmac_key, int key_sz, unsigned char *in, int in_sz, unsigned char *in2, int in2_sz, unsigned char *out) { + HMAC_CTX hctx; + unsigned int outlen; + HMAC_CTX_init(&hctx); + HMAC_Init_ex(&hctx, hmac_key, key_sz, EVP_sha1(), NULL); + HMAC_Update(&hctx, in, in_sz); + HMAC_Update(&hctx, in2, in2_sz); + HMAC_Final(&hctx, out, &outlen); + HMAC_CTX_cleanup(&hctx); + return SQLITE_OK; +} + +static int sqlcipher_openssl_kdf(void *ctx, const char *pass, int pass_sz, unsigned char* salt, int salt_sz, int workfactor, int key_sz, unsigned char *key) { + PKCS5_PBKDF2_HMAC_SHA1(pass, pass_sz, salt, salt_sz, workfactor, key_sz, key); + return SQLITE_OK; +} + +static int sqlcipher_openssl_cipher(void *ctx, int mode, unsigned char *key, int key_sz, unsigned char *iv, unsigned char *in, int in_sz, unsigned char *out) { + EVP_CIPHER_CTX ectx; + int tmp_csz, csz; + + EVP_CipherInit(&ectx, ((openssl_ctx *)ctx)->evp_cipher, NULL, NULL, mode); + EVP_CIPHER_CTX_set_padding(&ectx, 0); // no padding + EVP_CipherInit(&ectx, NULL, key, iv, mode); + EVP_CipherUpdate(&ectx, out, &tmp_csz, in, in_sz); + csz = tmp_csz; + out += tmp_csz; + EVP_CipherFinal(&ectx, out, &tmp_csz); + csz += tmp_csz; + EVP_CIPHER_CTX_cleanup(&ectx); + assert(in_sz == csz); + return SQLITE_OK; +} + +static int sqlcipher_openssl_set_cipher(void *ctx, const char *cipher_name) { + openssl_ctx *o_ctx = (openssl_ctx *)ctx; + o_ctx->evp_cipher = (EVP_CIPHER *) EVP_get_cipherbyname(cipher_name); + return SQLITE_OK; +} + +static const char* sqlcipher_openssl_get_cipher(void *ctx) { + return EVP_CIPHER_name(((openssl_ctx *)ctx)->evp_cipher); +} + +static int sqlcipher_openssl_get_key_sz(void *ctx) { + return EVP_CIPHER_key_length(((openssl_ctx *)ctx)->evp_cipher); +} + +static int sqlcipher_openssl_get_iv_sz(void *ctx) { + return EVP_CIPHER_iv_length(((openssl_ctx *)ctx)->evp_cipher); +} + +static int sqlcipher_openssl_get_block_sz(void *ctx) { + return EVP_CIPHER_block_size(((openssl_ctx *)ctx)->evp_cipher); +} + +static int sqlcipher_openssl_get_hmac_sz(void *ctx) { + return EVP_MD_size(EVP_sha1()); +} + +static int sqlcipher_openssl_ctx_copy(void *target_ctx, void *source_ctx) { + memcpy(target_ctx, source_ctx, sizeof(openssl_ctx)); + return SQLITE_OK; +} + +static int sqlcipher_openssl_ctx_cmp(void *c1, void *c2) { + return ((openssl_ctx *)c1)->evp_cipher == ((openssl_ctx *)c2)->evp_cipher; +} + +static int sqlcipher_openssl_ctx_init(void **ctx) { + *ctx = sqlcipher_malloc(sizeof(openssl_ctx)); + if(*ctx == NULL) return SQLITE_NOMEM; + sqlcipher_openssl_activate(*ctx); + return SQLITE_OK; +} + +static int sqlcipher_openssl_ctx_free(void **ctx) { + sqlcipher_openssl_deactivate(*ctx); + sqlcipher_free(*ctx, sizeof(openssl_ctx)); + return SQLITE_OK; +} + +int sqlcipher_openssl_setup(sqlcipher_provider *p) { + p->activate = sqlcipher_openssl_activate; + p->deactivate = sqlcipher_openssl_deactivate; + p->get_provider_name = sqlcipher_openssl_get_provider_name; + p->random = sqlcipher_openssl_random; + p->hmac = sqlcipher_openssl_hmac; + p->kdf = sqlcipher_openssl_kdf; + p->cipher = sqlcipher_openssl_cipher; + p->set_cipher = sqlcipher_openssl_set_cipher; + p->get_cipher = sqlcipher_openssl_get_cipher; + p->get_key_sz = sqlcipher_openssl_get_key_sz; + p->get_iv_sz = sqlcipher_openssl_get_iv_sz; + p->get_block_sz = sqlcipher_openssl_get_block_sz; + p->get_hmac_sz = sqlcipher_openssl_get_hmac_sz; + p->ctx_copy = sqlcipher_openssl_ctx_copy; + p->ctx_cmp = sqlcipher_openssl_ctx_cmp; + p->ctx_init = sqlcipher_openssl_ctx_init; + p->ctx_free = sqlcipher_openssl_ctx_free; + p->add_random = sqlcipher_openssl_add_random; + return SQLITE_OK; +} + +#endif +#endif +/* END SQLCIPHER */ diff --git a/src/ctime.c b/src/ctime.c index 61cf4e3..60595ff 100644 --- a/src/ctime.c +++ b/src/ctime.c @@ -48,15 +48,15 @@ static const char * const azCompileOpt[] = { #ifdef SQLITE_COVERAGE_TEST "COVERAGE_TEST", #endif -#ifdef SQLITE_CURDIR - "CURDIR", -#endif #ifdef SQLITE_DEBUG "DEBUG", #endif #ifdef SQLITE_DEFAULT_LOCKING_MODE "DEFAULT_LOCKING_MODE=" CTIMEOPT_VAL(SQLITE_DEFAULT_LOCKING_MODE), #endif +#if defined(SQLITE_DEFAULT_MMAP_SIZE) && !defined(SQLITE_DEFAULT_MMAP_SIZE_xc) + "DEFAULT_MMAP_SIZE=" CTIMEOPT_VAL(SQLITE_DEFAULT_MMAP_SIZE), +#endif #ifdef SQLITE_DISABLE_DIRSYNC "DISABLE_DIRSYNC", #endif @@ -147,6 +147,9 @@ static const char * const azCompileOpt[] = { #ifdef SQLITE_LOCK_TRACE "LOCK_TRACE", #endif +#if defined(SQLITE_MAX_MMAP_SIZE) && !defined(SQLITE_MAX_MMAP_SIZE_xc) + "MAX_MMAP_SIZE=" CTIMEOPT_VAL(SQLITE_MAX_MMAP_SIZE), +#endif #ifdef SQLITE_MAX_SCHEMA_RETRY "MAX_SCHEMA_RETRY=" CTIMEOPT_VAL(SQLITE_MAX_SCHEMA_RETRY), #endif @@ -204,11 +207,6 @@ static const char * const azCompileOpt[] = { #ifdef SQLITE_OMIT_CHECK "OMIT_CHECK", #endif -/* // redundant -** #ifdef SQLITE_OMIT_COMPILEOPTION_DIAGS -** "OMIT_COMPILEOPTION_DIAGS", -** #endif -*/ #ifdef SQLITE_OMIT_COMPLETE "OMIT_COMPLETE", #endif @@ -263,9 +261,6 @@ static const char * const azCompileOpt[] = { #ifdef SQLITE_OMIT_MEMORYDB "OMIT_MEMORYDB", #endif -#ifdef SQLITE_OMIT_MERGE_SORT - "OMIT_MERGE_SORT", -#endif #ifdef SQLITE_OMIT_OR_OPTIMIZATION "OMIT_OR_OPTIMIZATION", #endif @@ -338,6 +333,9 @@ static const char * const azCompileOpt[] = { #ifdef SQLITE_PROXY_DEBUG "PROXY_DEBUG", #endif +#ifdef SQLITE_RTREE_INT_ONLY + "RTREE_INT_ONLY", +#endif #ifdef SQLITE_SECURE_DELETE "SECURE_DELETE", #endif @@ -350,13 +348,13 @@ static const char * const azCompileOpt[] = { #ifdef SQLITE_TCL "TCL", #endif -#ifdef SQLITE_TEMP_STORE +#if defined(SQLITE_TEMP_STORE) && !defined(SQLITE_TEMP_STORE_xc) "TEMP_STORE=" CTIMEOPT_VAL(SQLITE_TEMP_STORE), #endif #ifdef SQLITE_TEST "TEST", #endif -#ifdef SQLITE_THREADSAFE +#if defined(SQLITE_THREADSAFE) "THREADSAFE=" CTIMEOPT_VAL(SQLITE_THREADSAFE), #endif #ifdef SQLITE_USE_ALLOCA @@ -382,8 +380,11 @@ int sqlite3_compileoption_used(const char *zOptName){ /* Since ArraySize(azCompileOpt) is normally in single digits, a ** linear search is adequate. No need for a binary search. */ for(i=0; i<ArraySize(azCompileOpt); i++){ - if( (sqlite3StrNICmp(zOptName, azCompileOpt[i], n)==0) - && ( (azCompileOpt[i][n]==0) || (azCompileOpt[i][n]=='=') ) ) return 1; + if( sqlite3StrNICmp(zOptName, azCompileOpt[i], n)==0 + && sqlite3CtypeMap[(unsigned char)azCompileOpt[i][n]]==0 + ){ + return 1; + } } return 0; } diff --git a/src/delete.c b/src/delete.c index 44e5995..634e115 100644 --- a/src/delete.c +++ b/src/delete.c @@ -32,7 +32,7 @@ Table *sqlite3SrcListLookup(Parse *pParse, SrcList *pSrc){ struct SrcList_item *pItem = pSrc->a; Table *pTab; assert( pItem && pSrc->nSrc==1 ); - pTab = sqlite3LocateTable(pParse, 0, pItem->zName, pItem->zDatabase); + pTab = sqlite3LocateTableItem(pParse, 0, pItem); sqlite3DeleteTable(pParse->db, pItem->pTab); pItem->pTab = pTab; if( pTab ){ @@ -93,29 +93,28 @@ void sqlite3MaterializeView( int iCur /* Cursor number for ephemerial table */ ){ SelectDest dest; - Select *pDup; + Select *pSel; + SrcList *pFrom; sqlite3 *db = pParse->db; + int iDb = sqlite3SchemaToIndex(db, pView->pSchema); - pDup = sqlite3SelectDup(db, pView->pSelect, 0); - if( pWhere ){ - SrcList *pFrom; - - pWhere = sqlite3ExprDup(db, pWhere, 0); - pFrom = sqlite3SrcListAppend(db, 0, 0, 0); - if( pFrom ){ - assert( pFrom->nSrc==1 ); - pFrom->a[0].zAlias = sqlite3DbStrDup(db, pView->zName); - pFrom->a[0].pSelect = pDup; - assert( pFrom->a[0].pOn==0 ); - assert( pFrom->a[0].pUsing==0 ); - }else{ - sqlite3SelectDelete(db, pDup); - } - pDup = sqlite3SelectNew(pParse, 0, pFrom, pWhere, 0, 0, 0, 0, 0, 0); + pWhere = sqlite3ExprDup(db, pWhere, 0); + pFrom = sqlite3SrcListAppend(db, 0, 0, 0); + + if( pFrom ){ + assert( pFrom->nSrc==1 ); + pFrom->a[0].zName = sqlite3DbStrDup(db, pView->zName); + pFrom->a[0].zDatabase = sqlite3DbStrDup(db, db->aDb[iDb].zName); + assert( pFrom->a[0].pOn==0 ); + assert( pFrom->a[0].pUsing==0 ); } + + pSel = sqlite3SelectNew(pParse, 0, pFrom, pWhere, 0, 0, 0, 0, 0, 0); + if( pSel ) pSel->selFlags |= SF_Materialize; + sqlite3SelectDestInit(&dest, SRT_EphemTab, iCur); - sqlite3Select(pParse, pDup, &dest); - sqlite3SelectDelete(db, pDup); + sqlite3Select(pParse, pSel, &dest); + sqlite3SelectDelete(db, pSel); } #endif /* !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER) */ @@ -638,7 +637,9 @@ int sqlite3GenerateIndexKey( } if( doMakeRec ){ const char *zAff; - if( pTab->pSelect || (pParse->db->flags & SQLITE_IdxRealAsInt)!=0 ){ + if( pTab->pSelect + || OptimizationDisabled(pParse->db, SQLITE_IdxRealAsInt) + ){ zAff = 0; }else{ zAff = sqlite3IndexAffinityStr(v, pIdx); @@ -31,7 +31,9 @@ ** SELECT * FROM t1 WHERE (select a from t1); */ char sqlite3ExprAffinity(Expr *pExpr){ - int op = pExpr->op; + int op; + pExpr = sqlite3ExprSkipCollate(pExpr); + op = pExpr->op; if( op==TK_SELECT ){ assert( pExpr->flags&EP_xIsSelect ); return sqlite3ExprAffinity(pExpr->x.pSelect->pEList->a[0].pExpr); @@ -56,66 +58,89 @@ char sqlite3ExprAffinity(Expr *pExpr){ } /* -** Set the explicit collating sequence for an expression to the -** collating sequence supplied in the second argument. +** Set the collating sequence for expression pExpr to be the collating +** sequence named by pToken. Return a pointer to a new Expr node that +** implements the COLLATE operator. +** +** If a memory allocation error occurs, that fact is recorded in pParse->db +** and the pExpr parameter is returned unchanged. */ -Expr *sqlite3ExprSetColl(Expr *pExpr, CollSeq *pColl){ - if( pExpr && pColl ){ - pExpr->pColl = pColl; - pExpr->flags |= EP_ExpCollate; +Expr *sqlite3ExprAddCollateToken(Parse *pParse, Expr *pExpr, Token *pCollName){ + if( pCollName->n>0 ){ + Expr *pNew = sqlite3ExprAlloc(pParse->db, TK_COLLATE, pCollName, 1); + if( pNew ){ + pNew->pLeft = pExpr; + pNew->flags |= EP_Collate; + pExpr = pNew; + } } return pExpr; } +Expr *sqlite3ExprAddCollateString(Parse *pParse, Expr *pExpr, const char *zC){ + Token s; + assert( zC!=0 ); + s.z = zC; + s.n = sqlite3Strlen30(s.z); + return sqlite3ExprAddCollateToken(pParse, pExpr, &s); +} /* -** Set the collating sequence for expression pExpr to be the collating -** sequence named by pToken. Return a pointer to the revised expression. -** The collating sequence is marked as "explicit" using the EP_ExpCollate -** flag. An explicit collating sequence will override implicit -** collating sequences. +** Skip over any TK_COLLATE and/or TK_AS operators at the root of +** an expression. */ -Expr *sqlite3ExprSetCollByToken(Parse *pParse, Expr *pExpr, Token *pCollName){ - char *zColl = 0; /* Dequoted name of collation sequence */ - CollSeq *pColl; - sqlite3 *db = pParse->db; - zColl = sqlite3NameFromToken(db, pCollName); - pColl = sqlite3LocateCollSeq(pParse, zColl); - sqlite3ExprSetColl(pExpr, pColl); - sqlite3DbFree(db, zColl); +Expr *sqlite3ExprSkipCollate(Expr *pExpr){ + while( pExpr && (pExpr->op==TK_COLLATE || pExpr->op==TK_AS) ){ + pExpr = pExpr->pLeft; + } return pExpr; } /* -** Return the default collation sequence for the expression pExpr. If -** there is no default collation type, return 0. +** Return the collation sequence for the expression pExpr. If +** there is no defined collating sequence, return NULL. +** +** The collating sequence might be determined by a COLLATE operator +** or by the presence of a column with a defined collating sequence. +** COLLATE operators take first precedence. Left operands take +** precedence over right operands. */ CollSeq *sqlite3ExprCollSeq(Parse *pParse, Expr *pExpr){ + sqlite3 *db = pParse->db; CollSeq *pColl = 0; Expr *p = pExpr; while( p ){ - int op; - pColl = p->pColl; - if( pColl ) break; - op = p->op; - if( p->pTab!=0 && ( - op==TK_AGG_COLUMN || op==TK_COLUMN || op==TK_REGISTER || op==TK_TRIGGER - )){ + int op = p->op; + if( op==TK_CAST || op==TK_UPLUS ){ + p = p->pLeft; + continue; + } + assert( op!=TK_REGISTER || p->op2!=TK_COLLATE ); + if( op==TK_COLLATE ){ + pColl = sqlite3GetCollSeq(pParse, ENC(db), 0, p->u.zToken); + break; + } + if( p->pTab!=0 + && (op==TK_AGG_COLUMN || op==TK_COLUMN + || op==TK_REGISTER || op==TK_TRIGGER) + ){ /* op==TK_REGISTER && p->pTab!=0 happens when pExpr was originally ** a TK_COLUMN but was previously evaluated and cached in a register */ - const char *zColl; int j = p->iColumn; if( j>=0 ){ - sqlite3 *db = pParse->db; - zColl = p->pTab->aCol[j].zColl; + const char *zColl = p->pTab->aCol[j].zColl; pColl = sqlite3FindCollSeq(db, ENC(db), zColl, 0); - pExpr->pColl = pColl; } break; } - if( op!=TK_CAST && op!=TK_UPLUS ){ + if( p->flags & EP_Collate ){ + if( ALWAYS(p->pLeft) && (p->pLeft->flags & EP_Collate)!=0 ){ + p = p->pLeft; + }else{ + p = p->pRight; + } + }else{ break; } - p = p->pLeft; } if( sqlite3CheckCollSeq(pParse, pColl) ){ pColl = 0; @@ -219,12 +244,10 @@ CollSeq *sqlite3BinaryCompareCollSeq( ){ CollSeq *pColl; assert( pLeft ); - if( pLeft->flags & EP_ExpCollate ){ - assert( pLeft->pColl ); - pColl = pLeft->pColl; - }else if( pRight && pRight->flags & EP_ExpCollate ){ - assert( pRight->pColl ); - pColl = pRight->pColl; + if( pLeft->flags & EP_Collate ){ + pColl = sqlite3ExprCollSeq(pParse, pLeft); + }else if( pRight && (pRight->flags & EP_Collate)!=0 ){ + pColl = sqlite3ExprCollSeq(pParse, pRight); }else{ pColl = sqlite3ExprCollSeq(pParse, pLeft); if( !pColl ){ @@ -454,17 +477,11 @@ void sqlite3ExprAttachSubtrees( }else{ if( pRight ){ pRoot->pRight = pRight; - if( pRight->flags & EP_ExpCollate ){ - pRoot->flags |= EP_ExpCollate; - pRoot->pColl = pRight->pColl; - } + pRoot->flags |= EP_Collate & pRight->flags; } if( pLeft ){ pRoot->pLeft = pLeft; - if( pLeft->flags & EP_ExpCollate ){ - pRoot->flags |= EP_ExpCollate; - pRoot->pColl = pLeft->pColl; - } + pRoot->flags |= EP_Collate & pLeft->flags; } exprSetHeight(pRoot); } @@ -616,7 +633,7 @@ void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr){ */ ynVar i; for(i=0; i<pParse->nzVar; i++){ - if( pParse->azVar[i] && memcmp(pParse->azVar[i],z,n+1)==0 ){ + if( pParse->azVar[i] && strcmp(pParse->azVar[i],z)==0 ){ pExpr->iColumn = x = (ynVar)i+1; break; } @@ -722,7 +739,7 @@ static int dupedExprStructSize(Expr *p, int flags){ assert( !ExprHasProperty(p, EP_FromJoin) ); assert( (p->flags2 & EP2_MallocedToken)==0 ); assert( (p->flags2 & EP2_Irreducible)==0 ); - if( p->pLeft || p->pRight || p->pColl || p->x.pList ){ + if( p->pLeft || p->pRight || p->x.pList ){ nSize = EXPR_REDUCEDSIZE | EP_Reduced; }else{ nSize = EXPR_TOKENONLYSIZE | EP_TokenOnly; @@ -930,6 +947,7 @@ SrcList *sqlite3SrcListDup(sqlite3 *db, SrcList *p, int flags){ struct SrcList_item *pNewItem = &pNew->a[i]; struct SrcList_item *pOldItem = &p->a[i]; Table *pTab; + pNewItem->pSchema = pOldItem->pSchema; pNewItem->zDatabase = sqlite3DbStrDup(db, pOldItem->zDatabase); pNewItem->zName = sqlite3DbStrDup(db, pOldItem->zName); pNewItem->zAlias = sqlite3DbStrDup(db, pOldItem->zAlias); @@ -938,6 +956,7 @@ SrcList *sqlite3SrcListDup(sqlite3 *db, SrcList *p, int flags){ pNewItem->addrFillSub = pOldItem->addrFillSub; pNewItem->regReturn = pOldItem->regReturn; pNewItem->isCorrelated = pOldItem->isCorrelated; + pNewItem->viaCoroutine = pOldItem->viaCoroutine; pNewItem->zIndex = sqlite3DbStrDup(db, pOldItem->zIndex); pNewItem->notIndexed = pOldItem->notIndexed; pNewItem->pIndex = pOldItem->pIndex; @@ -1190,6 +1209,7 @@ static int selectNodeIsConstant(Walker *pWalker, Select *NotUsed){ } static int exprIsConst(Expr *p, int initFlag){ Walker w; + memset(&w, 0, sizeof(w)); w.u.i = initFlag; w.xExprCallback = exprNodeIsConstant; w.xSelectCallback = selectNodeIsConstant; @@ -1420,24 +1440,34 @@ int sqlite3CodeOnce(Parse *pParse){ /* ** This function is used by the implementation of the IN (...) operator. -** It's job is to find or create a b-tree structure that may be used -** either to test for membership of the (...) set or to iterate through -** its members, skipping duplicates. +** The pX parameter is the expression on the RHS of the IN operator, which +** might be either a list of expressions or a subquery. +** +** The job of this routine is to find or create a b-tree object that can +** be used either to test for membership in the RHS set or to iterate through +** all members of the RHS set, skipping duplicates. +** +** A cursor is opened on the b-tree object that the RHS of the IN operator +** and pX->iTable is set to the index of that cursor. ** -** The index of the cursor opened on the b-tree (database table, database index -** or ephermal table) is stored in pX->iTable before this function returns. ** The returned value of this function indicates the b-tree type, as follows: ** -** IN_INDEX_ROWID - The cursor was opened on a database table. -** IN_INDEX_INDEX - The cursor was opened on a database index. -** IN_INDEX_EPH - The cursor was opened on a specially created and -** populated epheremal table. +** IN_INDEX_ROWID - The cursor was opened on a database table. +** IN_INDEX_INDEX_ASC - The cursor was opened on an ascending index. +** IN_INDEX_INDEX_DESC - The cursor was opened on a descending index. +** IN_INDEX_EPH - The cursor was opened on a specially created and +** populated epheremal table. ** -** An existing b-tree may only be used if the SELECT is of the simple -** form: +** An existing b-tree might be used if the RHS expression pX is a simple +** subquery such as: ** ** SELECT <column> FROM <table> ** +** If the RHS of the IN operator is a list or a more complex subquery, then +** an ephemeral table might need to be generated from the RHS and then +** pX->iTable made to point to the ephermeral table instead of an +** existing table. +** ** If the prNotFound parameter is 0, then the b-tree will be used to iterate ** through the set members, skipping any duplicates. In this case an ** epheremal table must be used unless the selected <column> is guaranteed @@ -1533,8 +1563,7 @@ int sqlite3FindInIndex(Parse *pParse, Expr *pX, int *prNotFound){ ** comparison is the same as the affinity of the column. If ** it is not, it is not possible to use any index. */ - char aff = comparisonAffinity(pX); - int affinity_ok = (pTab->aCol[iCol].affinity==aff||aff==SQLITE_AFF_NONE); + int affinity_ok = sqlite3IndexAffinityOk(pX, pTab->aCol[iCol].affinity); for(pIdx=pTab->pIndex; pIdx && eType==0 && affinity_ok; pIdx=pIdx->pNext){ if( (pIdx->aiColumn[0]==iCol) @@ -1550,7 +1579,8 @@ int sqlite3FindInIndex(Parse *pParse, Expr *pX, int *prNotFound){ sqlite3VdbeAddOp4(v, OP_OpenRead, iTab, pIdx->tnum, iDb, pKey,P4_KEYINFO_HANDOFF); VdbeComment((v, "%s", pIdx->zName)); - eType = IN_INDEX_INDEX; + assert( IN_INDEX_INDEX_DESC == IN_INDEX_INDEX_ASC+1 ); + eType = IN_INDEX_INDEX_ASC + pIdx->aSortOrder[0]; sqlite3VdbeJumpHere(v, iAddr); if( prNotFound && !pTab->aCol[iCol].notNull ){ @@ -1662,6 +1692,7 @@ int sqlite3CodeSubselect( case TK_IN: { char affinity; /* Affinity of the LHS of the IN */ KeyInfo keyInfo; /* Keyinfo for the generated table */ + static u8 sortOrder = 0; /* Fake aSortOrder for keyInfo */ int addr; /* Address of OP_OpenEphemeral instruction */ Expr *pLeft = pExpr->pLeft; /* the LHS of the IN operator */ @@ -1689,6 +1720,7 @@ int sqlite3CodeSubselect( if( rMayHaveNull==0 ) sqlite3VdbeChangeP5(v, BTREE_UNORDERED); memset(&keyInfo, 0, sizeof(keyInfo)); keyInfo.nField = 1; + keyInfo.aSortOrder = &sortOrder; if( ExprHasProperty(pExpr, EP_xIsSelect) ){ /* Case 1: expr IN (SELECT ...) @@ -1729,6 +1761,7 @@ int sqlite3CodeSubselect( affinity = SQLITE_AFF_NONE; } keyInfo.aColl[0] = sqlite3ExprCollSeq(pParse, pExpr->pLeft); + keyInfo.aSortOrder = &sortOrder; /* Loop through each expression in <exprlist>. */ r1 = sqlite3GetTempReg(pParse); @@ -2055,7 +2088,7 @@ void sqlite3ExprCacheStore(Parse *pParse, int iTab, int iCol, int iReg){ ** for testing only - to verify that SQLite always gets the same answer ** with and without the column cache. */ - if( pParse->db->flags & SQLITE_ColumnCache ) return; + if( OptimizationDisabled(pParse->db, SQLITE_ColumnCache) ) return; /* First replace any existing entry. ** @@ -2252,8 +2285,8 @@ void sqlite3ExprCacheAffinityChange(Parse *pParse, int iStart, int iCount){ void sqlite3ExprCodeMove(Parse *pParse, int iFrom, int iTo, int nReg){ int i; struct yColCache *p; - if( NEVER(iFrom==iTo) ) return; - sqlite3VdbeAddOp3(pParse->pVdbe, OP_Move, iFrom, iTo, nReg); + assert( iFrom>=iTo+nReg || iFrom+nReg<=iTo ); + sqlite3VdbeAddOp3(pParse->pVdbe, OP_Move, iFrom, iTo, nReg-1); for(i=0, p=pParse->aColCache; i<SQLITE_N_COLCACHE; i++, p++){ int x = p->iReg; if( x>=iFrom && x<iFrom+nReg ){ @@ -2262,18 +2295,6 @@ void sqlite3ExprCodeMove(Parse *pParse, int iFrom, int iTo, int nReg){ } } -/* -** Generate code to copy content from registers iFrom...iFrom+nReg-1 -** over to iTo..iTo+nReg-1. -*/ -void sqlite3ExprCodeCopy(Parse *pParse, int iFrom, int iTo, int nReg){ - int i; - if( NEVER(iFrom==iTo) ) return; - for(i=0; i<nReg; i++){ - sqlite3VdbeAddOp2(pParse->pVdbe, OP_Copy, iFrom+i, iTo+i); - } -} - #if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) /* ** Return true if any register in the range iFrom..iTo (inclusive) @@ -2745,6 +2766,7 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ sqlite3ReleaseTempReg(pParse, r4); break; } + case TK_COLLATE: case TK_UPLUS: { inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); break; @@ -2911,7 +2933,8 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ sqlite3VdbeAddOp4( v, OP_Halt, SQLITE_OK, OE_Ignore, 0, pExpr->u.zToken,0); }else{ - sqlite3HaltConstraint(pParse, pExpr->affinity, pExpr->u.zToken, 0); + sqlite3HaltConstraint(pParse, SQLITE_CONSTRAINT_TRIGGER, + pExpr->affinity, pExpr->u.zToken, 0); } break; @@ -3114,6 +3137,12 @@ void sqlite3ExplainExpr(Vdbe *pOut, Expr *pExpr){ case TK_ISNULL: zUniOp = "ISNULL"; break; case TK_NOTNULL: zUniOp = "NOTNULL"; break; + case TK_COLLATE: { + sqlite3ExplainExpr(pOut, pExpr->pLeft); + sqlite3ExplainPrintf(pOut,".COLLATE(%s)",pExpr->u.zToken); + break; + } + case TK_AGG_FUNCTION: case TK_CONST_FUNC: case TK_FUNCTION: { @@ -3251,6 +3280,12 @@ void sqlite3ExplainExprList(Vdbe *pOut, ExprList *pList){ sqlite3ExplainPush(pOut); sqlite3ExplainExpr(pOut, pList->a[i].pExpr); sqlite3ExplainPop(pOut); + if( pList->a[i].zName ){ + sqlite3ExplainPrintf(pOut, " AS %s", pList->a[i].zName); + } + if( pList->a[i].bSpanIsTab ){ + sqlite3ExplainPrintf(pOut, " (%s)", pList->a[i].zSpan); + } if( i<pList->nExpr-1 ){ sqlite3ExplainNL(pOut); } @@ -3332,6 +3367,9 @@ static int evalConstExpr(Walker *pWalker, Expr *pExpr){ case TK_REGISTER: { return WRC_Prune; } + case TK_COLLATE: { + return WRC_Continue; + } case TK_FUNCTION: case TK_AGG_FUNCTION: case TK_CONST_FUNC: { @@ -3353,9 +3391,11 @@ static int evalConstExpr(Walker *pWalker, Expr *pExpr){ } if( isAppropriateForFactoring(pExpr) ){ int r1 = ++pParse->nMem; - int r2; - r2 = sqlite3ExprCodeTarget(pParse, pExpr, r1); - if( NEVER(r1!=r2) ) sqlite3ReleaseTempReg(pParse, r1); + int r2 = sqlite3ExprCodeTarget(pParse, pExpr, r1); + /* If r2!=r1, it means that register r1 is never used. That is harmless + ** but suboptimal, so we want to know about the situation to fix it. + ** Hence the following assert: */ + assert( r2==r1 ); pExpr->op2 = pExpr->op; pExpr->op = TK_REGISTER; pExpr->iTable = r2; @@ -3383,9 +3423,9 @@ static int evalConstExpr(Walker *pWalker, Expr *pExpr){ void sqlite3ExprCodeConstants(Parse *pParse, Expr *pExpr){ Walker w; if( pParse->cookieGoto ) return; - if( (pParse->db->flags & SQLITE_FactorOutConst)!=0 ) return; + if( OptimizationDisabled(pParse->db, SQLITE_FactorOutConst) ) return; + memset(&w, 0, sizeof(w)); w.xExprCallback = evalConstExpr; - w.xSelectCallback = 0; w.pParse = pParse; sqlite3WalkExpr(&w, pExpr); } @@ -3498,7 +3538,7 @@ void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ int r1, r2; assert( jumpIfNull==SQLITE_JUMPIFNULL || jumpIfNull==0 ); - if( NEVER(v==0) ) return; /* Existance of VDBE checked by caller */ + if( NEVER(v==0) ) return; /* Existence of VDBE checked by caller */ if( NEVER(pExpr==0) ) return; /* No way this can happen */ op = pExpr->op; switch( op ){ @@ -3618,7 +3658,7 @@ void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ int r1, r2; assert( jumpIfNull==SQLITE_JUMPIFNULL || jumpIfNull==0 ); - if( NEVER(v==0) ) return; /* Existance of VDBE checked by caller */ + if( NEVER(v==0) ) return; /* Existence of VDBE checked by caller */ if( pExpr==0 ) return; /* The value of pExpr->op and op are related as follows: @@ -3772,7 +3812,15 @@ int sqlite3ExprCompare(Expr *pA, Expr *pB){ return 2; } if( (pA->flags & EP_Distinct)!=(pB->flags & EP_Distinct) ) return 2; - if( pA->op!=pB->op ) return 2; + if( pA->op!=pB->op ){ + if( pA->op==TK_COLLATE && sqlite3ExprCompare(pA->pLeft, pB)<2 ){ + return 1; + } + if( pB->op==TK_COLLATE && sqlite3ExprCompare(pA, pB->pLeft)<2 ){ + return 1; + } + return 2; + } if( sqlite3ExprCompare(pA->pLeft, pB->pLeft) ) return 2; if( sqlite3ExprCompare(pA->pRight, pB->pRight) ) return 2; if( sqlite3ExprListCompare(pA->x.pList, pB->x.pList) ) return 2; @@ -3784,11 +3832,9 @@ int sqlite3ExprCompare(Expr *pA, Expr *pB){ }else if( pA->op!=TK_COLUMN && ALWAYS(pA->op!=TK_AGG_COLUMN) && pA->u.zToken){ if( ExprHasProperty(pB, EP_IntValue) || NEVER(pB->u.zToken==0) ) return 2; if( strcmp(pA->u.zToken,pB->u.zToken)!=0 ){ - return 2; + return pA->op==TK_COLLATE ? 1 : 2; } } - if( (pA->flags & EP_ExpCollate)!=(pB->flags & EP_ExpCollate) ) return 1; - if( (pA->flags & EP_ExpCollate)!=0 && pA->pColl!=pB->pColl ) return 2; return 0; } @@ -4029,8 +4075,10 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ ExprSetIrreducible(pExpr); pExpr->iAgg = (i16)i; pExpr->pAggInfo = pAggInfo; + return WRC_Prune; + }else{ + return WRC_Continue; } - return WRC_Prune; } } return WRC_Continue; @@ -4042,9 +4090,10 @@ static int analyzeAggregatesInSelect(Walker *pWalker, Select *pSelect){ } /* -** Analyze the given expression looking for aggregate functions and -** for variables that need to be added to the pParse->aAgg[] array. -** Make additional entries to the pParse->aAgg[] array as necessary. +** Analyze the pExpr expression looking for aggregate functions and +** for variables that need to be added to AggInfo object that pNC->pAggInfo +** points to. Additional entries are made on the AggInfo object as +** necessary. ** ** This routine should only be called after the expression has been ** analyzed by sqlite3ResolveExprNames(). @@ -21,8 +21,9 @@ ** -------------------------- ** ** Foreign keys in SQLite come in two flavours: deferred and immediate. -** If an immediate foreign key constraint is violated, SQLITE_CONSTRAINT -** is returned and the current statement transaction rolled back. If a +** If an immediate foreign key constraint is violated, +** SQLITE_CONSTRAINT_FOREIGNKEY is returned and the current +** statement transaction rolled back. If a ** deferred foreign key constraint is violated, no action is taken ** immediately. However if the application attempts to commit the ** transaction before fixing the constraint violation, the attempt fails. @@ -86,7 +87,8 @@ ** Immediate constraints are usually handled similarly. The only difference ** is that the counter used is stored as part of each individual statement ** object (struct Vdbe). If, after the statement has run, its immediate -** constraint counter is greater than zero, it returns SQLITE_CONSTRAINT +** constraint counter is greater than zero, +** it returns SQLITE_CONSTRAINT_FOREIGNKEY ** and the statement transaction is rolled back. An exception is an INSERT ** statement that inserts a single row only (no triggers). In this case, ** instead of using a counter, an exception is thrown immediately if the @@ -142,7 +144,7 @@ ** A foreign key constraint requires that the key columns in the parent ** table are collectively subject to a UNIQUE or PRIMARY KEY constraint. ** Given that pParent is the parent table for foreign key constraint pFKey, -** search the schema a unique index on the parent key columns. +** search the schema for a unique index on the parent key columns. ** ** If successful, zero is returned. If the parent key is an INTEGER PRIMARY ** KEY column, then output variable *ppIdx is set to NULL. Otherwise, *ppIdx @@ -178,7 +180,7 @@ ** into pParse. If an OOM error occurs, non-zero is returned and the ** pParse->db->mallocFailed flag is set. */ -static int locateFkeyIndex( +int sqlite3FkLocateIndex( Parse *pParse, /* Parse context to store any error in */ Table *pParent, /* Parent table of FK constraint pFKey */ FKey *pFKey, /* Foreign key to find index for */ @@ -275,7 +277,9 @@ static int locateFkeyIndex( if( !pIdx ){ if( !pParse->disableTriggers ){ - sqlite3ErrorMsg(pParse, "foreign key mismatch"); + sqlite3ErrorMsg(pParse, + "foreign key mismatch - \"%w\" referencing \"%w\"", + pFKey->pFrom->zName, pFKey->zTo); } sqlite3DbFree(pParse->db, aiCol); return 1; @@ -424,8 +428,8 @@ static void fkLookupParent( ** incrementing a counter. This is necessary as the VM code is being ** generated for will not open a statement transaction. */ assert( nIncr==1 ); - sqlite3HaltConstraint( - pParse, OE_Abort, "foreign key constraint failed", P4_STATIC + sqlite3HaltConstraint(pParse, SQLITE_CONSTRAINT_FOREIGNKEY, + OE_Abort, "foreign key constraint failed", P4_STATIC ); }else{ if( nIncr>0 && pFKey->isDeferred==0 ){ @@ -511,12 +515,15 @@ static void fkScanChildren( ** expression to the parent key column defaults. */ if( pIdx ){ Column *pCol; + const char *zColl; iCol = pIdx->aiColumn[i]; pCol = &pTab->aCol[iCol]; if( pTab->iPKey==iCol ) iCol = -1; pLeft->iTable = regData+iCol+1; pLeft->affinity = pCol->affinity; - pLeft->pColl = sqlite3LocateCollSeq(pParse, pCol->zColl); + zColl = pCol->zColl; + if( zColl==0 ) zColl = db->pDfltColl->zName; + pLeft = sqlite3ExprAddCollateString(pParse, pLeft, zColl); }else{ pLeft->iTable = regData; pLeft->affinity = SQLITE_AFF_INTEGER; @@ -662,8 +669,8 @@ void sqlite3FkDropTable(Parse *pParse, SrcList *pName, Table *pTab){ ** any modifications to the schema are made. This is because statement ** transactions are not able to rollback schema changes. */ sqlite3VdbeAddOp2(v, OP_FkIfZero, 0, sqlite3VdbeCurrentAddr(v)+2); - sqlite3HaltConstraint( - pParse, OE_Abort, "foreign key constraint failed", P4_STATIC + sqlite3HaltConstraint(pParse, SQLITE_CONSTRAINT_FOREIGNKEY, + OE_Abort, "foreign key constraint failed", P4_STATIC ); if( iSkip ){ @@ -733,7 +740,7 @@ void sqlite3FkCheck( }else{ pTo = sqlite3LocateTable(pParse, 0, pFKey->zTo, zDb); } - if( !pTo || locateFkeyIndex(pParse, pTo, pFKey, &pIdx, &aiFree) ){ + if( !pTo || sqlite3FkLocateIndex(pParse, pTo, pFKey, &pIdx, &aiFree) ){ assert( isIgnoreErrors==0 || (regOld!=0 && regNew==0) ); if( !isIgnoreErrors || db->mallocFailed ) return; if( pTo==0 ){ @@ -813,7 +820,7 @@ void sqlite3FkCheck( continue; } - if( locateFkeyIndex(pParse, pTab, pFKey, &pIdx, &aiCol) ){ + if( sqlite3FkLocateIndex(pParse, pTab, pFKey, &pIdx, &aiCol) ){ if( !isIgnoreErrors || db->mallocFailed ) return; continue; } @@ -868,7 +875,7 @@ u32 sqlite3FkOldmask( } for(p=sqlite3FkReferences(pTab); p; p=p->pNextTo){ Index *pIdx = 0; - locateFkeyIndex(pParse, pTab, p, &pIdx, 0); + sqlite3FkLocateIndex(pParse, pTab, p, &pIdx, 0); if( pIdx ){ for(i=0; i<pIdx->nColumn; i++) mask |= COLUMN_MASK(pIdx->aiColumn[i]); } @@ -925,7 +932,8 @@ int sqlite3FkRequired( int iKey; for(iKey=0; iKey<pTab->nCol; iKey++){ Column *pCol = &pTab->aCol[iKey]; - if( (zKey ? !sqlite3StrICmp(pCol->zName, zKey) : pCol->isPrimKey) ){ + if( (zKey ? !sqlite3StrICmp(pCol->zName, zKey) + : (pCol->colFlags & COLFLAG_PRIMKEY)!=0) ){ if( aChange[iKey]>=0 ) return 1; if( iKey==pTab->iPKey && chngRowid ) return 1; } @@ -993,7 +1001,7 @@ static Trigger *fkActionTrigger( int i; /* Iterator variable */ Expr *pWhen = 0; /* WHEN clause for the trigger */ - if( locateFkeyIndex(pParse, pTab, pFKey, &pIdx, &aiCol) ) return 0; + if( sqlite3FkLocateIndex(pParse, pTab, pFKey, &pIdx, &aiCol) ) return 0; assert( aiCol || pFKey->nCol==1 ); for(i=0; i<pFKey->nCol; i++){ @@ -169,6 +169,56 @@ static void absFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ } /* +** Implementation of the instr() function. +** +** instr(haystack,needle) finds the first occurrence of needle +** in haystack and returns the number of previous characters plus 1, +** or 0 if needle does not occur within haystack. +** +** If both haystack and needle are BLOBs, then the result is one more than +** the number of bytes in haystack prior to the first occurrence of needle, +** or 0 if needle never occurs in haystack. +*/ +static void instrFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const unsigned char *zHaystack; + const unsigned char *zNeedle; + int nHaystack; + int nNeedle; + int typeHaystack, typeNeedle; + int N = 1; + int isText; + + UNUSED_PARAMETER(argc); + typeHaystack = sqlite3_value_type(argv[0]); + typeNeedle = sqlite3_value_type(argv[1]); + if( typeHaystack==SQLITE_NULL || typeNeedle==SQLITE_NULL ) return; + nHaystack = sqlite3_value_bytes(argv[0]); + nNeedle = sqlite3_value_bytes(argv[1]); + if( typeHaystack==SQLITE_BLOB && typeNeedle==SQLITE_BLOB ){ + zHaystack = sqlite3_value_blob(argv[0]); + zNeedle = sqlite3_value_blob(argv[1]); + isText = 0; + }else{ + zHaystack = sqlite3_value_text(argv[0]); + zNeedle = sqlite3_value_text(argv[1]); + isText = 1; + } + while( nNeedle<=nHaystack && memcmp(zHaystack, zNeedle, nNeedle)!=0 ){ + N++; + do{ + nHaystack--; + zHaystack++; + }while( isText && (zHaystack[0]&0xc0)==0x80 ); + } + if( nNeedle>nHaystack ) N = 0; + sqlite3_result_int(context, N); +} + +/* ** Implementation of the substr() function. ** ** substr(x,p1,p2) returns p2 characters of x[] beginning with p1. @@ -367,33 +417,14 @@ static void lowerFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ } } - -#if 0 /* This function is never used. */ -/* -** The COALESCE() and IFNULL() functions used to be implemented as shown -** here. But now they are implemented as VDBE code so that unused arguments -** do not have to be computed. This legacy implementation is retained as -** comment. -*/ /* -** Implementation of the IFNULL(), NVL(), and COALESCE() functions. -** All three do the same thing. They return the first non-NULL -** argument. +** The COALESCE() and IFNULL() functions are implemented as VDBE code so +** that unused argument values do not have to be computed. However, we +** still need some kind of function implementation for this routines in +** the function table. That function implementation will never be called +** so it doesn't matter what the implementation is. We might as well use +** the "version()" function as a substitute. */ -static void ifnullFunc( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - int i; - for(i=0; i<argc; i++){ - if( SQLITE_NULL!=sqlite3_value_type(argv[i]) ){ - sqlite3_result_value(context, argv[i]); - break; - } - } -} -#endif /* NOT USED */ #define ifnullFunc versionFunc /* Substitute function - never called */ /* @@ -512,7 +543,7 @@ struct compareInfo { ** whereas only characters less than 0x80 do in ASCII. */ #if defined(SQLITE_EBCDIC) -# define sqlite3Utf8Read(A,C) (*(A++)) +# define sqlite3Utf8Read(A) (*((*A)++)) # define GlogUpperToLower(A) A = sqlite3UpperToLower[A] #else # define GlogUpperToLower(A) if( !((A)&~0x7f) ){ A = sqlite3UpperToLower[A]; } @@ -569,18 +600,18 @@ static int patternCompare( u8 noCase = pInfo->noCase; int prevEscape = 0; /* True if the previous character was 'escape' */ - while( (c = sqlite3Utf8Read(zPattern,&zPattern))!=0 ){ - if( !prevEscape && c==matchAll ){ - while( (c=sqlite3Utf8Read(zPattern,&zPattern)) == matchAll + while( (c = sqlite3Utf8Read(&zPattern))!=0 ){ + if( c==matchAll && !prevEscape ){ + while( (c=sqlite3Utf8Read(&zPattern)) == matchAll || c == matchOne ){ - if( c==matchOne && sqlite3Utf8Read(zString, &zString)==0 ){ + if( c==matchOne && sqlite3Utf8Read(&zString)==0 ){ return 0; } } if( c==0 ){ return 1; }else if( c==esc ){ - c = sqlite3Utf8Read(zPattern, &zPattern); + c = sqlite3Utf8Read(&zPattern); if( c==0 ){ return 0; } @@ -592,25 +623,25 @@ static int patternCompare( } return *zString!=0; } - while( (c2 = sqlite3Utf8Read(zString,&zString))!=0 ){ + while( (c2 = sqlite3Utf8Read(&zString))!=0 ){ if( noCase ){ GlogUpperToLower(c2); GlogUpperToLower(c); while( c2 != 0 && c2 != c ){ - c2 = sqlite3Utf8Read(zString, &zString); + c2 = sqlite3Utf8Read(&zString); GlogUpperToLower(c2); } }else{ while( c2 != 0 && c2 != c ){ - c2 = sqlite3Utf8Read(zString, &zString); + c2 = sqlite3Utf8Read(&zString); } } if( c2==0 ) return 0; if( patternCompare(zPattern,zString,pInfo,esc) ) return 1; } return 0; - }else if( !prevEscape && c==matchOne ){ - if( sqlite3Utf8Read(zString, &zString)==0 ){ + }else if( c==matchOne && !prevEscape ){ + if( sqlite3Utf8Read(&zString)==0 ){ return 0; } }else if( c==matchSet ){ @@ -618,20 +649,20 @@ static int patternCompare( assert( esc==0 ); /* This only occurs for GLOB, not LIKE */ seen = 0; invert = 0; - c = sqlite3Utf8Read(zString, &zString); + c = sqlite3Utf8Read(&zString); if( c==0 ) return 0; - c2 = sqlite3Utf8Read(zPattern, &zPattern); + c2 = sqlite3Utf8Read(&zPattern); if( c2=='^' ){ invert = 1; - c2 = sqlite3Utf8Read(zPattern, &zPattern); + c2 = sqlite3Utf8Read(&zPattern); } if( c2==']' ){ if( c==']' ) seen = 1; - c2 = sqlite3Utf8Read(zPattern, &zPattern); + c2 = sqlite3Utf8Read(&zPattern); } while( c2 && c2!=']' ){ if( c2=='-' && zPattern[0]!=']' && zPattern[0]!=0 && prior_c>0 ){ - c2 = sqlite3Utf8Read(zPattern, &zPattern); + c2 = sqlite3Utf8Read(&zPattern); if( c>=prior_c && c<=c2 ) seen = 1; prior_c = 0; }else{ @@ -640,7 +671,7 @@ static int patternCompare( } prior_c = c2; } - c2 = sqlite3Utf8Read(zPattern, &zPattern); + c2 = sqlite3Utf8Read(&zPattern); } if( c2==0 || (seen ^ invert)==0 ){ return 0; @@ -648,7 +679,7 @@ static int patternCompare( }else if( esc==c && !prevEscape ){ prevEscape = 1; }else{ - c2 = sqlite3Utf8Read(zString, &zString); + c2 = sqlite3Utf8Read(&zString); if( noCase ){ GlogUpperToLower(c); GlogUpperToLower(c2); @@ -663,6 +694,13 @@ static int patternCompare( } /* +** The sqlite3_strglob() interface. +*/ +int sqlite3_strglob(const char *zGlobPattern, const char *zString){ + return patternCompare((u8*)zGlobPattern, (u8*)zString, &globInfo, 0)==0; +} + +/* ** Count the number of times that the LIKE operator (or GLOB which is ** just a variation of LIKE) gets called. This is used for testing ** only. @@ -720,7 +758,7 @@ static void likeFunc( "ESCAPE expression must be a single character", -1); return; } - escape = sqlite3Utf8Read(zEsc, &zEsc); + escape = sqlite3Utf8Read(&zEsc); } if( zA && zB ){ struct compareInfo *pInfo = sqlite3_user_data(context); @@ -932,6 +970,62 @@ static void quoteFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ } /* +** The unicode() function. Return the integer unicode code-point value +** for the first character of the input string. +*/ +static void unicodeFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const unsigned char *z = sqlite3_value_text(argv[0]); + (void)argc; + if( z && z[0] ) sqlite3_result_int(context, sqlite3Utf8Read(&z)); +} + +/* +** The char() function takes zero or more arguments, each of which is +** an integer. It constructs a string where each character of the string +** is the unicode character for the corresponding integer argument. +*/ +static void charFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + unsigned char *z, *zOut; + int i; + zOut = z = sqlite3_malloc( argc*4 ); + if( z==0 ){ + sqlite3_result_error_nomem(context); + return; + } + for(i=0; i<argc; i++){ + sqlite3_int64 x; + unsigned c; + x = sqlite3_value_int64(argv[i]); + if( x<0 || x>0x10ffff ) x = 0xfffd; + c = (unsigned)(x & 0x1fffff); + if( c<0x00080 ){ + *zOut++ = (u8)(c&0xFF); + }else if( c<0x00800 ){ + *zOut++ = 0xC0 + (u8)((c>>6)&0x1F); + *zOut++ = 0x80 + (u8)(c & 0x3F); + }else if( c<0x10000 ){ + *zOut++ = 0xE0 + (u8)((c>>12)&0x0F); + *zOut++ = 0x80 + (u8)((c>>6) & 0x3F); + *zOut++ = 0x80 + (u8)(c & 0x3F); + }else{ + *zOut++ = 0xF0 + (u8)((c>>18) & 0x07); + *zOut++ = 0x80 + (u8)((c>>12) & 0x3F); + *zOut++ = 0x80 + (u8)((c>>6) & 0x3F); + *zOut++ = 0x80 + (u8)(c & 0x3F); + } \ + } + sqlite3_result_text(context, (char*)z, (int)(zOut-z), sqlite3_free); +} + +/* ** The hex() function. Interpret the argument as a blob. Return ** a hexadecimal rendering as text. */ @@ -1450,16 +1544,24 @@ static void groupConcatFinalize(sqlite3_context *context){ */ void sqlite3RegisterBuiltinFunctions(sqlite3 *db){ int rc = sqlite3_overload_function(db, "MATCH", 2); +/* BEGIN SQLCIPHER */ +#ifdef SQLITE_HAS_CODEC #ifndef OMIT_EXPORT extern void sqlcipher_exportFunc(sqlite3_context *, int, sqlite3_value **); #endif +#endif +/* END SQLCIPHER */ assert( rc==SQLITE_NOMEM || rc==SQLITE_OK ); if( rc==SQLITE_NOMEM ){ db->mallocFailed = 1; } +/* BEGIN SQLCIPHER */ +#ifdef SQLITE_HAS_CODEC #ifndef OMIT_EXPORT sqlite3CreateFunc(db, "sqlcipher_export", 1, SQLITE_TEXT, 0, sqlcipher_exportFunc, 0, 0, 0); #endif +#endif +/* END SQLCIPHER */ } /* @@ -1561,8 +1663,11 @@ void sqlite3RegisterGlobalFunctions(void){ AGGREGATE(max, 1, 1, 1, minmaxStep, minMaxFinalize ), FUNCTION2(typeof, 1, 0, 0, typeofFunc, SQLITE_FUNC_TYPEOF), FUNCTION2(length, 1, 0, 0, lengthFunc, SQLITE_FUNC_LENGTH), + FUNCTION(instr, 2, 0, 0, instrFunc ), FUNCTION(substr, 2, 0, 0, substrFunc ), FUNCTION(substr, 3, 0, 0, substrFunc ), + FUNCTION(unicode, 1, 0, 0, unicodeFunc ), + FUNCTION(char, -1, 0, 0, charFunc ), FUNCTION(abs, 1, 0, 0, absFunc ), #ifndef SQLITE_OMIT_FLOATING_POINT FUNCTION(round, 1, 0, 0, roundFunc ), diff --git a/src/global.c b/src/global.c index 7de0668..7b02cf2 100644 --- a/src/global.c +++ b/src/global.c @@ -133,6 +133,10 @@ const unsigned char sqlite3CtypeMap[256] = { # define SQLITE_USE_URI 0 #endif +#ifndef SQLITE_ALLOW_COVERING_INDEX_SCAN +# define SQLITE_ALLOW_COVERING_INDEX_SCAN 1 +#endif + /* ** The following singleton contains the global configuration for ** the SQLite library. @@ -142,6 +146,7 @@ SQLITE_WSD struct Sqlite3Config sqlite3Config = { 1, /* bCoreMutex */ SQLITE_THREADSAFE==1, /* bFullMutex */ SQLITE_USE_URI, /* bOpenUri */ + SQLITE_ALLOW_COVERING_INDEX_SCAN, /* bUseCis */ 0x7ffffffe, /* mxStrlen */ 128, /* szLookaside */ 500, /* nLookaside */ @@ -151,6 +156,8 @@ SQLITE_WSD struct Sqlite3Config sqlite3Config = { (void*)0, /* pHeap */ 0, /* nHeap */ 0, 0, /* mnHeap, mxHeap */ + SQLITE_DEFAULT_MMAP_SIZE, /* szMmap */ + SQLITE_MAX_MMAP_SIZE, /* mxMmap */ (void*)0, /* pScratch */ 0, /* szScratch */ 0, /* nScratch */ @@ -170,6 +177,10 @@ SQLITE_WSD struct Sqlite3Config sqlite3Config = { 0, /* xLog */ 0, /* pLogArg */ 0, /* bLocaltimeFault */ +#ifdef SQLITE_ENABLE_SQLLOG + 0, /* xSqllog */ + 0 /* pSqllogArg */ +#endif }; @@ -193,7 +193,7 @@ static void removeElementGivenHash( } sqlite3_free( elem ); pH->count--; - if( pH->count<=0 ){ + if( pH->count==0 ){ assert( pH->first==0 ); assert( pH->count==0 ); sqlite3HashClear(pH); @@ -9,7 +9,7 @@ ** May you share freely, never taking more than you give. ** ************************************************************************* -** This is the header file for the generic hash-table implemenation +** This is the header file for the generic hash-table implementation ** used in SQLite. */ #ifndef _SQLITE_HASH_H_ diff --git a/src/insert.c b/src/insert.c index a24e8f9..9a5661f 100644 --- a/src/insert.c +++ b/src/insert.c @@ -25,7 +25,7 @@ void sqlite3OpenTable( int opcode /* OP_OpenRead or OP_OpenWrite */ ){ Vdbe *v; - if( IsVirtual(pTab) ) return; + assert( !IsVirtual(pTab) ); v = sqlite3GetVdbe(p); assert( opcode==OP_OpenWrite || opcode==OP_OpenRead ); sqlite3TableLock(p, iDb, pTab->tnum, (opcode==OP_OpenWrite)?1:0, pTab->zName); @@ -320,6 +320,97 @@ void sqlite3AutoincrementEnd(Parse *pParse){ #endif /* SQLITE_OMIT_AUTOINCREMENT */ +/* +** Generate code for a co-routine that will evaluate a subquery one +** row at a time. +** +** The pSelect parameter is the subquery that the co-routine will evaluation. +** Information about the location of co-routine and the registers it will use +** is returned by filling in the pDest object. +** +** Registers are allocated as follows: +** +** pDest->iSDParm The register holding the next entry-point of the +** co-routine. Run the co-routine to its next breakpoint +** by calling "OP_Yield $X" where $X is pDest->iSDParm. +** +** pDest->iSDParm+1 The register holding the "completed" flag for the +** co-routine. This register is 0 if the previous Yield +** generated a new result row, or 1 if the subquery +** has completed. If the Yield is called again +** after this register becomes 1, then the VDBE will +** halt with an SQLITE_INTERNAL error. +** +** pDest->iSdst First result register. +** +** pDest->nSdst Number of result registers. +** +** This routine handles all of the register allocation and fills in the +** pDest structure appropriately. +** +** Here is a schematic of the generated code assuming that X is the +** co-routine entry-point register reg[pDest->iSDParm], that EOF is the +** completed flag reg[pDest->iSDParm+1], and R and S are the range of +** registers that hold the result set, reg[pDest->iSdst] through +** reg[pDest->iSdst+pDest->nSdst-1]: +** +** X <- A +** EOF <- 0 +** goto B +** A: setup for the SELECT +** loop rows in the SELECT +** load results into registers R..S +** yield X +** end loop +** cleanup after the SELECT +** EOF <- 1 +** yield X +** halt-error +** B: +** +** To use this subroutine, the caller generates code as follows: +** +** [ Co-routine generated by this subroutine, shown above ] +** S: yield X +** if EOF goto E +** if skip this row, goto C +** if terminate loop, goto E +** deal with this row +** C: goto S +** E: +*/ +int sqlite3CodeCoroutine(Parse *pParse, Select *pSelect, SelectDest *pDest){ + int regYield; /* Register holding co-routine entry-point */ + int regEof; /* Register holding co-routine completion flag */ + int addrTop; /* Top of the co-routine */ + int j1; /* Jump instruction */ + int rc; /* Result code */ + Vdbe *v; /* VDBE under construction */ + + regYield = ++pParse->nMem; + regEof = ++pParse->nMem; + v = sqlite3GetVdbe(pParse); + addrTop = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp2(v, OP_Integer, addrTop+2, regYield); /* X <- A */ + VdbeComment((v, "Co-routine entry point")); + sqlite3VdbeAddOp2(v, OP_Integer, 0, regEof); /* EOF <- 0 */ + VdbeComment((v, "Co-routine completion flag")); + sqlite3SelectDestInit(pDest, SRT_Coroutine, regYield); + j1 = sqlite3VdbeAddOp2(v, OP_Goto, 0, 0); + rc = sqlite3Select(pParse, pSelect, pDest); + assert( pParse->nErr==0 || rc ); + if( pParse->db->mallocFailed && rc==SQLITE_OK ) rc = SQLITE_NOMEM; + if( rc ) return rc; + sqlite3VdbeAddOp2(v, OP_Integer, 1, regEof); /* EOF <- 1 */ + sqlite3VdbeAddOp1(v, OP_Yield, regYield); /* yield X */ + sqlite3VdbeAddOp2(v, OP_Halt, SQLITE_INTERNAL, OE_Abort); + VdbeComment((v, "End of coroutine")); + sqlite3VdbeJumpHere(v, j1); /* label B: */ + return rc; +} + + + /* Forward declaration */ static int xferOptimization( Parse *pParse, /* Parser context */ @@ -568,51 +659,12 @@ void sqlite3Insert( ** co-routine is the common header to the 3rd and 4th templates. */ if( pSelect ){ - /* Data is coming from a SELECT. Generate code to implement that SELECT - ** as a co-routine. The code is common to both the 3rd and 4th - ** templates: - ** - ** EOF <- 0 - ** X <- A - ** goto B - ** A: setup for the SELECT - ** loop over the tables in the SELECT - ** load value into register R..R+n - ** yield X - ** end loop - ** cleanup after the SELECT - ** EOF <- 1 - ** yield X - ** halt-error - ** - ** On each invocation of the co-routine, it puts a single row of the - ** SELECT result into registers dest.iMem...dest.iMem+dest.nMem-1. - ** (These output registers are allocated by sqlite3Select().) When - ** the SELECT completes, it sets the EOF flag stored in regEof. - */ - int rc, j1; - - regEof = ++pParse->nMem; - sqlite3VdbeAddOp2(v, OP_Integer, 0, regEof); /* EOF <- 0 */ - VdbeComment((v, "SELECT eof flag")); - sqlite3SelectDestInit(&dest, SRT_Coroutine, ++pParse->nMem); - addrSelect = sqlite3VdbeCurrentAddr(v)+2; - sqlite3VdbeAddOp2(v, OP_Integer, addrSelect-1, dest.iSDParm); - j1 = sqlite3VdbeAddOp2(v, OP_Goto, 0, 0); - VdbeComment((v, "Jump over SELECT coroutine")); - - /* Resolve the expressions in the SELECT statement and execute it. */ - rc = sqlite3Select(pParse, pSelect, &dest); - assert( pParse->nErr==0 || rc ); - if( rc || NEVER(pParse->nErr) || db->mallocFailed ){ - goto insert_cleanup; - } - sqlite3VdbeAddOp2(v, OP_Integer, 1, regEof); /* EOF <- 1 */ - sqlite3VdbeAddOp1(v, OP_Yield, dest.iSDParm); /* yield X */ - sqlite3VdbeAddOp2(v, OP_Halt, SQLITE_INTERNAL, OE_Abort); - VdbeComment((v, "End of SELECT coroutine")); - sqlite3VdbeJumpHere(v, j1); /* label B: */ + /* Data is coming from a SELECT. Generate a co-routine to run that + ** SELECT. */ + int rc = sqlite3CodeCoroutine(pParse, pSelect, &dest); + if( rc ) goto insert_cleanup; + regEof = dest.iSDParm + 1; regFromSelect = dest.iSdst; assert( pSelect->pEList ); nColumn = pSelect->pEList->nExpr; @@ -1193,7 +1245,7 @@ void sqlite3GenerateConstraintChecks( case OE_Fail: { char *zMsg; sqlite3VdbeAddOp3(v, OP_HaltIfNull, - SQLITE_CONSTRAINT, onError, regData+i); + SQLITE_CONSTRAINT_NOTNULL, onError, regData+i); zMsg = sqlite3MPrintf(db, "%s.%s may not be NULL", pTab->zName, pTab->aCol[i].zName); sqlite3VdbeChangeP4(v, -1, zMsg, P4_DYNAMIC); @@ -1233,7 +1285,8 @@ void sqlite3GenerateConstraintChecks( }else{ zConsName = 0; } - sqlite3HaltConstraint(pParse, onError, zConsName, P4_DYNAMIC); + sqlite3HaltConstraint(pParse, SQLITE_CONSTRAINT_CHECK, + onError, zConsName, P4_DYNAMIC); } sqlite3VdbeResolveLabel(v, allOk); } @@ -1264,8 +1317,8 @@ void sqlite3GenerateConstraintChecks( case OE_Rollback: case OE_Abort: case OE_Fail: { - sqlite3HaltConstraint( - pParse, onError, "PRIMARY KEY must be unique", P4_STATIC); + sqlite3HaltConstraint(pParse, SQLITE_CONSTRAINT_PRIMARYKEY, + onError, "PRIMARY KEY must be unique", P4_STATIC); break; } case OE_Replace: { @@ -1392,7 +1445,8 @@ void sqlite3GenerateConstraintChecks( sqlite3StrAccumAppend(&errMsg, pIdx->nColumn>1 ? " are not unique" : " is not unique", -1); zErr = sqlite3StrAccumFinish(&errMsg); - sqlite3HaltConstraint(pParse, onError, zErr, 0); + sqlite3HaltConstraint(pParse, SQLITE_CONSTRAINT_UNIQUE, + onError, zErr, 0); sqlite3DbFree(errMsg.db, zErr); break; } @@ -1691,7 +1745,7 @@ static int xferOptimization( ** we have to check the semantics. */ pItem = pSelect->pSrc->a; - pSrc = sqlite3LocateTable(pParse, 0, pItem->zName, pItem->zDatabase); + pSrc = sqlite3LocateTableItem(pParse, 0, pItem); if( pSrc==0 ){ return 0; /* FROM clause does not contain a real table */ } @@ -1800,8 +1854,8 @@ static int xferOptimization( if( pDest->iPKey>=0 ){ addr1 = sqlite3VdbeAddOp2(v, OP_Rowid, iSrc, regRowid); addr2 = sqlite3VdbeAddOp3(v, OP_NotExists, iDest, 0, regRowid); - sqlite3HaltConstraint( - pParse, onError, "PRIMARY KEY must be unique", P4_STATIC); + sqlite3HaltConstraint(pParse, SQLITE_CONSTRAINT_PRIMARYKEY, + onError, "PRIMARY KEY must be unique", P4_STATIC); sqlite3VdbeJumpHere(v, addr2); autoIncStep(pParse, regAutoinc, regRowid); }else if( pDest->pIndex==0 ){ diff --git a/src/journal.c b/src/journal.c index 2f9e222..fed27be 100644 --- a/src/journal.c +++ b/src/journal.c @@ -59,6 +59,14 @@ static int createFile(JournalFile *p){ assert(p->iSize<=p->nBuf); rc = sqlite3OsWrite(p->pReal, p->zBuf, p->iSize, 0); } + if( rc!=SQLITE_OK ){ + /* If an error occurred while writing to the file, close it before + ** returning. This way, SQLite uses the in-memory journal data to + ** roll back changes made to the internal page-cache before this + ** function was called. */ + sqlite3OsClose(pReal); + p->pReal = 0; + } } } return rc; @@ -228,6 +236,16 @@ int sqlite3JournalCreate(sqlite3_file *p){ return createFile((JournalFile *)p); } +/* +** The file-handle passed as the only argument is guaranteed to be an open +** file. It may or may not be of class JournalFile. If the file is a +** JournalFile, and the underlying file on disk has not yet been opened, +** return 0. Otherwise, return 1. +*/ +int sqlite3JournalExists(sqlite3_file *p){ + return (p->pMethods!=&JournalFileMethods || ((JournalFile *)p)->pReal!=0); +} + /* ** Return the number of bytes required to store a JournalFile that uses vfs ** pVfs to create the underlying on-disk files. diff --git a/src/legacy.c b/src/legacy.c index ebab2de..94649ae 100644 --- a/src/legacy.c +++ b/src/legacy.c @@ -38,7 +38,6 @@ int sqlite3_exec( const char *zLeftover; /* Tail of unprocessed SQL */ sqlite3_stmt *pStmt = 0; /* The current SQL statement */ char **azCols = 0; /* Names of result columns */ - int nRetry = 0; /* Number of retry attempts */ int callbackIsInit; /* True if callback data is initialized */ if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; @@ -46,12 +45,12 @@ int sqlite3_exec( sqlite3_mutex_enter(db->mutex); sqlite3Error(db, SQLITE_OK, 0); - while( (rc==SQLITE_OK || (rc==SQLITE_SCHEMA && (++nRetry)<2)) && zSql[0] ){ + while( rc==SQLITE_OK && zSql[0] ){ int nCol; char **azVals = 0; pStmt = 0; - rc = sqlite3_prepare(db, zSql, -1, &pStmt, &zLeftover); + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &zLeftover); assert( rc==SQLITE_OK || pStmt==0 ); if( rc!=SQLITE_OK ){ continue; @@ -108,11 +107,8 @@ int sqlite3_exec( if( rc!=SQLITE_ROW ){ rc = sqlite3VdbeFinalize((Vdbe *)pStmt); pStmt = 0; - if( rc!=SQLITE_SCHEMA ){ - nRetry = 0; - zSql = zLeftover; - while( sqlite3Isspace(zSql[0]) ) zSql++; - } + zSql = zLeftover; + while( sqlite3Isspace(zSql[0]) ) zSql++; break; } } diff --git a/src/lempar.c b/src/lempar.c index cb6025e..2afaa6c 100644 --- a/src/lempar.c +++ b/src/lempar.c @@ -608,6 +608,7 @@ static void yy_reduce( */ %% }; + assert( yyruleno>=0 && yyruleno<sizeof(yyRuleInfo)/sizeof(yyRuleInfo[0]) ); yygoto = yyRuleInfo[yyruleno].lhs; yysize = yyRuleInfo[yyruleno].nrhs; yypParser->yyidx -= yysize; diff --git a/src/loadext.c b/src/loadext.c index 3fcf500..cdcf6a9 100644 --- a/src/loadext.c +++ b/src/loadext.c @@ -378,6 +378,19 @@ static const sqlite3_api_routines sqlite3Apis = { sqlite3_blob_reopen, sqlite3_vtab_config, sqlite3_vtab_on_conflict, + sqlite3_close_v2, + sqlite3_db_filename, + sqlite3_db_readonly, + sqlite3_db_release_memory, + sqlite3_errstr, + sqlite3_stmt_busy, + sqlite3_stmt_readonly, + sqlite3_stricmp, + sqlite3_uri_boolean, + sqlite3_uri_int64, + sqlite3_uri_parameter, + sqlite3_vsnprintf, + sqlite3_wal_checkpoint_v2 }; /* @@ -402,8 +415,23 @@ static int sqlite3LoadExtension( void *handle; int (*xInit)(sqlite3*,char**,const sqlite3_api_routines*); char *zErrmsg = 0; + const char *zEntry; + char *zAltEntry = 0; void **aHandle; int nMsg = 300 + sqlite3Strlen30(zFile); + int ii; + + /* Shared library endings to try if zFile cannot be loaded as written */ + static const char *azEndings[] = { +#if SQLITE_OS_WIN + "dll" +#elif defined(__APPLE__) + "dylib" +#else + "so" +#endif + }; + if( pzErrMsg ) *pzErrMsg = 0; @@ -420,11 +448,17 @@ static int sqlite3LoadExtension( return SQLITE_ERROR; } - if( zProc==0 ){ - zProc = "sqlite3_extension_init"; - } + zEntry = zProc ? zProc : "sqlite3_extension_init"; handle = sqlite3OsDlOpen(pVfs, zFile); +#if SQLITE_OS_UNIX || SQLITE_OS_WIN + for(ii=0; ii<ArraySize(azEndings) && handle==0; ii++){ + char *zAltFile = sqlite3_mprintf("%s.%s", zFile, azEndings[ii]); + if( zAltFile==0 ) return SQLITE_NOMEM; + handle = sqlite3OsDlOpen(pVfs, zAltFile); + sqlite3_free(zAltFile); + } +#endif if( handle==0 ){ if( pzErrMsg ){ *pzErrMsg = zErrmsg = sqlite3_malloc(nMsg); @@ -437,20 +471,57 @@ static int sqlite3LoadExtension( return SQLITE_ERROR; } xInit = (int(*)(sqlite3*,char**,const sqlite3_api_routines*)) - sqlite3OsDlSym(pVfs, handle, zProc); + sqlite3OsDlSym(pVfs, handle, zEntry); + + /* If no entry point was specified and the default legacy + ** entry point name "sqlite3_extension_init" was not found, then + ** construct an entry point name "sqlite3_X_init" where the X is + ** replaced by the lowercase value of every ASCII alphabetic + ** character in the filename after the last "/" upto the first ".", + ** and eliding the first three characters if they are "lib". + ** Examples: + ** + ** /usr/local/lib/libExample5.4.3.so ==> sqlite3_example_init + ** C:/lib/mathfuncs.dll ==> sqlite3_mathfuncs_init + */ + if( xInit==0 && zProc==0 ){ + int iFile, iEntry, c; + int ncFile = sqlite3Strlen30(zFile); + zAltEntry = sqlite3_malloc(ncFile+30); + if( zAltEntry==0 ){ + sqlite3OsDlClose(pVfs, handle); + return SQLITE_NOMEM; + } + memcpy(zAltEntry, "sqlite3_", 8); + for(iFile=ncFile-1; iFile>=0 && zFile[iFile]!='/'; iFile--){} + iFile++; + if( sqlite3_strnicmp(zFile+iFile, "lib", 3)==0 ) iFile += 3; + for(iEntry=8; (c = zFile[iFile])!=0 && c!='.'; iFile++){ + if( sqlite3Isalpha(c) ){ + zAltEntry[iEntry++] = (char)sqlite3UpperToLower[(unsigned)c]; + } + } + memcpy(zAltEntry+iEntry, "_init", 6); + zEntry = zAltEntry; + xInit = (int(*)(sqlite3*,char**,const sqlite3_api_routines*)) + sqlite3OsDlSym(pVfs, handle, zEntry); + } if( xInit==0 ){ if( pzErrMsg ){ - nMsg += sqlite3Strlen30(zProc); + nMsg += sqlite3Strlen30(zEntry); *pzErrMsg = zErrmsg = sqlite3_malloc(nMsg); if( zErrmsg ){ sqlite3_snprintf(nMsg, zErrmsg, - "no entry point [%s] in shared library [%s]", zProc,zFile); + "no entry point [%s] in shared library [%s]", zEntry, zFile); sqlite3OsDlError(pVfs, nMsg-1, zErrmsg); } - sqlite3OsDlClose(pVfs, handle); } + sqlite3OsDlClose(pVfs, handle); + sqlite3_free(zAltEntry); return SQLITE_ERROR; - }else if( xInit(db, &zErrmsg, &sqlite3Apis) ){ + } + sqlite3_free(zAltEntry); + if( xInit(db, &zErrmsg, &sqlite3Apis) ){ if( pzErrMsg ){ *pzErrMsg = sqlite3_mprintf("error during initialization: %s", zErrmsg); } @@ -132,6 +132,13 @@ int sqlite3_initialize(void){ */ if( sqlite3GlobalConfig.isInit ) return SQLITE_OK; +#ifdef SQLITE_ENABLE_SQLLOG + { + extern void sqlite3_init_sqllog(void); + sqlite3_init_sqllog(); + } +#endif + /* Make sure the mutex subsystem is initialized. If unable to ** initialize the mutex subsystem, return early with the error. ** If the system is so sick that we are unable to allocate a mutex, @@ -475,6 +482,33 @@ int sqlite3_config(int op, ...){ break; } + case SQLITE_CONFIG_COVERING_INDEX_SCAN: { + sqlite3GlobalConfig.bUseCis = va_arg(ap, int); + break; + } + +#ifdef SQLITE_ENABLE_SQLLOG + case SQLITE_CONFIG_SQLLOG: { + typedef void(*SQLLOGFUNC_t)(void*, sqlite3*, const char*, int); + sqlite3GlobalConfig.xSqllog = va_arg(ap, SQLLOGFUNC_t); + sqlite3GlobalConfig.pSqllogArg = va_arg(ap, void *); + break; + } +#endif + + case SQLITE_CONFIG_MMAP_SIZE: { + sqlite3_int64 szMmap = va_arg(ap, sqlite3_int64); + sqlite3_int64 mxMmap = va_arg(ap, sqlite3_int64); + if( mxMmap<0 || mxMmap>SQLITE_MAX_MMAP_SIZE ){ + mxMmap = SQLITE_MAX_MMAP_SIZE; + } + sqlite3GlobalConfig.mxMmap = mxMmap; + if( szMmap<0 ) szMmap = SQLITE_DEFAULT_MMAP_SIZE; + if( szMmap>mxMmap) szMmap = mxMmap; + sqlite3GlobalConfig.szMmap = szMmap; + break; + } + default: { rc = SQLITE_ERROR; break; @@ -814,6 +848,13 @@ static int sqlite3Close(sqlite3 *db, int forceZombie){ return SQLITE_BUSY; } +#ifdef SQLITE_ENABLE_SQLLOG + if( sqlite3GlobalConfig.xSqllog ){ + /* Closing the handle. Fourth parameter is passed the value 2. */ + sqlite3GlobalConfig.xSqllog(sqlite3GlobalConfig.pSqllogArg, db, 0, 2); + } +#endif + /* Convert the connection into a zombie and then close it. */ db->magic = SQLITE_MAGIC_ZOMBIE; @@ -857,10 +898,16 @@ void sqlite3LeaveMutexAndCloseZombie(sqlite3 *db){ /* If we reach this point, it means that the database connection has ** closed all sqlite3_stmt and sqlite3_backup objects and has been - ** pased to sqlite3_close (meaning that it is a zombie). Therefore, + ** passed to sqlite3_close (meaning that it is a zombie). Therefore, ** go ahead and free all resources. */ + /* If a transaction is open, roll it back. This also ensures that if + ** any database schemas have been modified by an uncommitted transaction + ** they are reset. And that the required b-tree mutex is held to make + ** the pager rollback and schema reset an atomic operation. */ + sqlite3RollbackAll(db, SQLITE_OK); + /* Free any outstanding Savepoint structures. */ sqlite3CloseSavepoints(db); @@ -961,6 +1008,15 @@ void sqlite3RollbackAll(sqlite3 *db, int tripCode){ int inTrans = 0; assert( sqlite3_mutex_held(db->mutex) ); sqlite3BeginBenignMalloc(); + + /* Obtain all b-tree mutexes before making any calls to BtreeRollback(). + ** This is important in case the transaction being rolled back has + ** modified the database schema. If the b-tree mutexes are not taken + ** here, then another shared-cache connection might sneak in between + ** the database rollback and schema reset, which can cause false + ** corruption reports in some cases. */ + sqlite3BtreeEnterAll(db); + for(i=0; i<db->nDb; i++){ Btree *p = db->aDb[i].pBt; if( p ){ @@ -974,10 +1030,11 @@ void sqlite3RollbackAll(sqlite3 *db, int tripCode){ sqlite3VtabRollback(db); sqlite3EndBenignMalloc(); - if( db->flags&SQLITE_InternChanges ){ + if( (db->flags&SQLITE_InternChanges)!=0 && db->init.busy==0 ){ sqlite3ExpirePreparedStatements(db); sqlite3ResetAllSchemasOfConnection(db); } + sqlite3BtreeLeaveAll(db); /* Any deferred constraint violations have now been resolved. */ db->nDeferredCons = 0; @@ -989,6 +1046,110 @@ void sqlite3RollbackAll(sqlite3 *db, int tripCode){ } /* +** Return a static string containing the name corresponding to the error code +** specified in the argument. +*/ +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) || \ + defined(SQLITE_DEBUG_OS_TRACE) +const char *sqlite3ErrName(int rc){ + const char *zName = 0; + int i, origRc = rc; + for(i=0; i<2 && zName==0; i++, rc &= 0xff){ + switch( rc ){ + case SQLITE_OK: zName = "SQLITE_OK"; break; + case SQLITE_ERROR: zName = "SQLITE_ERROR"; break; + case SQLITE_INTERNAL: zName = "SQLITE_INTERNAL"; break; + case SQLITE_PERM: zName = "SQLITE_PERM"; break; + case SQLITE_ABORT: zName = "SQLITE_ABORT"; break; + case SQLITE_ABORT_ROLLBACK: zName = "SQLITE_ABORT_ROLLBACK"; break; + case SQLITE_BUSY: zName = "SQLITE_BUSY"; break; + case SQLITE_BUSY_RECOVERY: zName = "SQLITE_BUSY_RECOVERY"; break; + case SQLITE_LOCKED: zName = "SQLITE_LOCKED"; break; + case SQLITE_LOCKED_SHAREDCACHE: zName = "SQLITE_LOCKED_SHAREDCACHE";break; + case SQLITE_NOMEM: zName = "SQLITE_NOMEM"; break; + case SQLITE_READONLY: zName = "SQLITE_READONLY"; break; + case SQLITE_READONLY_RECOVERY: zName = "SQLITE_READONLY_RECOVERY"; break; + case SQLITE_READONLY_CANTLOCK: zName = "SQLITE_READONLY_CANTLOCK"; break; + case SQLITE_READONLY_ROLLBACK: zName = "SQLITE_READONLY_ROLLBACK"; break; + case SQLITE_INTERRUPT: zName = "SQLITE_INTERRUPT"; break; + case SQLITE_IOERR: zName = "SQLITE_IOERR"; break; + case SQLITE_IOERR_READ: zName = "SQLITE_IOERR_READ"; break; + case SQLITE_IOERR_SHORT_READ: zName = "SQLITE_IOERR_SHORT_READ"; break; + case SQLITE_IOERR_WRITE: zName = "SQLITE_IOERR_WRITE"; break; + case SQLITE_IOERR_FSYNC: zName = "SQLITE_IOERR_FSYNC"; break; + case SQLITE_IOERR_DIR_FSYNC: zName = "SQLITE_IOERR_DIR_FSYNC"; break; + case SQLITE_IOERR_TRUNCATE: zName = "SQLITE_IOERR_TRUNCATE"; break; + case SQLITE_IOERR_FSTAT: zName = "SQLITE_IOERR_FSTAT"; break; + case SQLITE_IOERR_UNLOCK: zName = "SQLITE_IOERR_UNLOCK"; break; + case SQLITE_IOERR_RDLOCK: zName = "SQLITE_IOERR_RDLOCK"; break; + case SQLITE_IOERR_DELETE: zName = "SQLITE_IOERR_DELETE"; break; + case SQLITE_IOERR_BLOCKED: zName = "SQLITE_IOERR_BLOCKED"; break; + case SQLITE_IOERR_NOMEM: zName = "SQLITE_IOERR_NOMEM"; break; + case SQLITE_IOERR_ACCESS: zName = "SQLITE_IOERR_ACCESS"; break; + case SQLITE_IOERR_CHECKRESERVEDLOCK: + zName = "SQLITE_IOERR_CHECKRESERVEDLOCK"; break; + case SQLITE_IOERR_LOCK: zName = "SQLITE_IOERR_LOCK"; break; + case SQLITE_IOERR_CLOSE: zName = "SQLITE_IOERR_CLOSE"; break; + case SQLITE_IOERR_DIR_CLOSE: zName = "SQLITE_IOERR_DIR_CLOSE"; break; + case SQLITE_IOERR_SHMOPEN: zName = "SQLITE_IOERR_SHMOPEN"; break; + case SQLITE_IOERR_SHMSIZE: zName = "SQLITE_IOERR_SHMSIZE"; break; + case SQLITE_IOERR_SHMLOCK: zName = "SQLITE_IOERR_SHMLOCK"; break; + case SQLITE_IOERR_SHMMAP: zName = "SQLITE_IOERR_SHMMAP"; break; + case SQLITE_IOERR_SEEK: zName = "SQLITE_IOERR_SEEK"; break; + case SQLITE_IOERR_DELETE_NOENT: zName = "SQLITE_IOERR_DELETE_NOENT";break; + case SQLITE_IOERR_MMAP: zName = "SQLITE_IOERR_MMAP"; break; + case SQLITE_CORRUPT: zName = "SQLITE_CORRUPT"; break; + case SQLITE_CORRUPT_VTAB: zName = "SQLITE_CORRUPT_VTAB"; break; + case SQLITE_NOTFOUND: zName = "SQLITE_NOTFOUND"; break; + case SQLITE_FULL: zName = "SQLITE_FULL"; break; + case SQLITE_CANTOPEN: zName = "SQLITE_CANTOPEN"; break; + case SQLITE_CANTOPEN_NOTEMPDIR: zName = "SQLITE_CANTOPEN_NOTEMPDIR";break; + case SQLITE_CANTOPEN_ISDIR: zName = "SQLITE_CANTOPEN_ISDIR"; break; + case SQLITE_CANTOPEN_FULLPATH: zName = "SQLITE_CANTOPEN_FULLPATH"; break; + case SQLITE_PROTOCOL: zName = "SQLITE_PROTOCOL"; break; + case SQLITE_EMPTY: zName = "SQLITE_EMPTY"; break; + case SQLITE_SCHEMA: zName = "SQLITE_SCHEMA"; break; + case SQLITE_TOOBIG: zName = "SQLITE_TOOBIG"; break; + case SQLITE_CONSTRAINT: zName = "SQLITE_CONSTRAINT"; break; + case SQLITE_CONSTRAINT_UNIQUE: zName = "SQLITE_CONSTRAINT_UNIQUE"; break; + case SQLITE_CONSTRAINT_TRIGGER: zName = "SQLITE_CONSTRAINT_TRIGGER";break; + case SQLITE_CONSTRAINT_FOREIGNKEY: + zName = "SQLITE_CONSTRAINT_FOREIGNKEY"; break; + case SQLITE_CONSTRAINT_CHECK: zName = "SQLITE_CONSTRAINT_CHECK"; break; + case SQLITE_CONSTRAINT_PRIMARYKEY: + zName = "SQLITE_CONSTRAINT_PRIMARYKEY"; break; + case SQLITE_CONSTRAINT_NOTNULL: zName = "SQLITE_CONSTRAINT_NOTNULL";break; + case SQLITE_CONSTRAINT_COMMITHOOK: + zName = "SQLITE_CONSTRAINT_COMMITHOOK"; break; + case SQLITE_CONSTRAINT_VTAB: zName = "SQLITE_CONSTRAINT_VTAB"; break; + case SQLITE_CONSTRAINT_FUNCTION: + zName = "SQLITE_CONSTRAINT_FUNCTION"; break; + case SQLITE_MISMATCH: zName = "SQLITE_MISMATCH"; break; + case SQLITE_MISUSE: zName = "SQLITE_MISUSE"; break; + case SQLITE_NOLFS: zName = "SQLITE_NOLFS"; break; + case SQLITE_AUTH: zName = "SQLITE_AUTH"; break; + case SQLITE_FORMAT: zName = "SQLITE_FORMAT"; break; + case SQLITE_RANGE: zName = "SQLITE_RANGE"; break; + case SQLITE_NOTADB: zName = "SQLITE_NOTADB"; break; + case SQLITE_ROW: zName = "SQLITE_ROW"; break; + case SQLITE_NOTICE: zName = "SQLITE_NOTICE"; break; + case SQLITE_NOTICE_RECOVER_WAL: zName = "SQLITE_NOTICE_RECOVER_WAL";break; + case SQLITE_NOTICE_RECOVER_ROLLBACK: + zName = "SQLITE_NOTICE_RECOVER_ROLLBACK"; break; + case SQLITE_WARNING: zName = "SQLITE_WARNING"; break; + case SQLITE_DONE: zName = "SQLITE_DONE"; break; + } + } + if( zName==0 ){ + static char zBuf[50]; + sqlite3_snprintf(sizeof(zBuf), zBuf, "SQLITE_UNKNOWN(%d)", origRc); + zName = zBuf; + } + return zName; +} +#endif + +/* ** Return a static string that describes the kind of error specified in the ** argument. */ @@ -1116,6 +1277,7 @@ int sqlite3_busy_handler( db->busyHandler.xFunc = xBusy; db->busyHandler.pArg = pArg; db->busyHandler.nBusy = 0; + db->busyTimeout = 0; sqlite3_mutex_leave(db->mutex); return SQLITE_OK; } @@ -1153,8 +1315,8 @@ void sqlite3_progress_handler( */ int sqlite3_busy_timeout(sqlite3 *db, int ms){ if( ms>0 ){ - db->busyTimeout = ms; sqlite3_busy_handler(db, sqliteDefaultBusyCallback, (void*)db); + db->busyTimeout = ms; }else{ sqlite3_busy_handler(db, 0, 0); } @@ -1768,6 +1930,15 @@ int sqlite3_extended_errcode(sqlite3 *db){ } /* +** Return a string that describes the kind of error specified in the +** argument. For now, this simply calls the internal sqlite3ErrStr() +** function. +*/ +const char *sqlite3_errstr(int rc){ + return sqlite3ErrStr(rc); +} + +/* ** Create a new collating function for database "db". The name is zName ** and the encoding is enc. */ @@ -2278,6 +2449,7 @@ static int openDatabase( memcpy(db->aLimit, aHardLimit, sizeof(db->aLimit)); db->autoCommit = 1; db->nextAutovac = -1; + db->szMmap = sqlite3GlobalConfig.szMmap; db->nextPagesize = 0; db->flags |= SQLITE_ShortColNames | SQLITE_AutoIndex | SQLITE_EnableTrigger #if SQLITE_DEFAULT_FILE_FORMAT<4 @@ -2436,6 +2608,13 @@ opendb_out: db->magic = SQLITE_MAGIC_SICK; } *ppDb = db; +#ifdef SQLITE_ENABLE_SQLLOG + if( sqlite3GlobalConfig.xSqllog ){ + /* Opening a db handle. Fourth parameter is passed 0. */ + void *pArg = sqlite3GlobalConfig.pSqllogArg; + sqlite3GlobalConfig.xSqllog(pArg, db, zFilename, 0); + } +#endif return sqlite3ApiExit(0, rc); } @@ -2741,7 +2920,7 @@ int sqlite3_table_column_metadata( zDataType = pCol->zType; zCollSeq = pCol->zColl; notnull = pCol->notNull!=0; - primarykey = pCol->isPrimKey!=0; + primarykey = (pCol->colFlags & COLFLAG_PRIMKEY)!=0; autoinc = pTab->iPKey==iCol && (pTab->tabFlags & TF_Autoincrement)!=0; }else{ zDataType = "INTEGER"; @@ -3004,8 +3183,7 @@ int sqlite3_test_control(int op, ...){ */ case SQLITE_TESTCTRL_OPTIMIZATIONS: { sqlite3 *db = va_arg(ap, sqlite3*); - int x = va_arg(ap,int); - db->flags = (x & SQLITE_OptMask) | (db->flags & ~SQLITE_OptMask); + db->dbOptFlags = (u16)(va_arg(ap, int) & 0xffff); break; } diff --git a/src/memjournal.c b/src/memjournal.c index 3e66e21..0572594 100644 --- a/src/memjournal.c +++ b/src/memjournal.c @@ -230,7 +230,9 @@ static const struct sqlite3_io_methods MemJournalMethods = { 0, /* xShmMap */ 0, /* xShmLock */ 0, /* xShmBarrier */ - 0 /* xShmUnlock */ + 0, /* xShmUnmap */ + 0, /* xFetch */ + 0 /* xUnfetch */ }; /* @@ -141,6 +141,26 @@ int sqlite3OsShmMap( return id->pMethods->xShmMap(id, iPage, pgsz, bExtend, pp); } +#if SQLITE_MAX_MMAP_SIZE>0 +/* The real implementation of xFetch and xUnfetch */ +int sqlite3OsFetch(sqlite3_file *id, i64 iOff, int iAmt, void **pp){ + DO_OS_MALLOC_TEST(id); + return id->pMethods->xFetch(id, iOff, iAmt, pp); +} +int sqlite3OsUnfetch(sqlite3_file *id, i64 iOff, void *p){ + return id->pMethods->xUnfetch(id, iOff, p); +} +#else +/* No-op stubs to use when memory-mapped I/O is disabled */ +int sqlite3OsFetch(sqlite3_file *id, i64 iOff, int iAmt, void **pp){ + *pp = 0; + return SQLITE_OK; +} +int sqlite3OsUnfetch(sqlite3_file *id, i64 iOff, void *p){ + return SQLITE_OK; +} +#endif + /* ** The next group of routines are convenience wrappers around the ** VFS methods. @@ -99,14 +99,6 @@ # define SQLITE_OS_WINRT 0 #endif -/* -** When compiled for WinCE or WinRT, there is no concept of the current -** directory. - */ -#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT -# define SQLITE_CURDIR 1 -#endif - /* If the SET_FULLSYNC macro is not defined above, then make it ** a no-op */ @@ -259,6 +251,8 @@ int sqlite3OsShmMap(sqlite3_file *,int,int,int,void volatile **); int sqlite3OsShmLock(sqlite3_file *id, int, int, int); void sqlite3OsShmBarrier(sqlite3_file *id); int sqlite3OsShmUnmap(sqlite3_file *id, int); +int sqlite3OsFetch(sqlite3_file *id, i64, int, void **); +int sqlite3OsUnfetch(sqlite3_file *, i64, void *); /* diff --git a/src/os_unix.c b/src/os_unix.c index c0df66e..abc23a4 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -46,6 +46,13 @@ #include "sqliteInt.h" #if SQLITE_OS_UNIX /* This file is used on unix only */ +/* Use posix_fallocate() if it is available +*/ +#if !defined(HAVE_POSIX_FALLOCATE) \ + && (_XOPEN_SOURCE >= 600 || _POSIX_C_SOURCE >= 200112L) +# define HAVE_POSIX_FALLOCATE 1 +#endif + /* ** There are various methods for file locking used for concurrency ** control: @@ -119,7 +126,7 @@ #include <time.h> #include <sys/time.h> #include <errno.h> -#ifndef SQLITE_OMIT_WAL +#if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 #include <sys/mman.h> #endif @@ -218,6 +225,15 @@ struct unixFile { const char *zPath; /* Name of the file */ unixShm *pShm; /* Shared memory segment information */ int szChunk; /* Configured by FCNTL_CHUNK_SIZE */ + int nFetchOut; /* Number of outstanding xFetch refs */ + sqlite3_int64 mmapSize; /* Usable size of mapping at pMapRegion */ + sqlite3_int64 mmapSizeActual; /* Actual size of mapping at pMapRegion */ + sqlite3_int64 mmapSizeMax; /* Configured FCNTL_MMAP_SIZE value */ + void *pMapRegion; /* Memory mapped region */ +#ifdef __QNXNTO__ + int sectorSize; /* Device sector size */ + int deviceCharacteristics; /* Precomputed device characteristics */ +#endif #if SQLITE_ENABLE_LOCKING_STYLE int openFlags; /* The flags specified at open() */ #endif @@ -238,7 +254,9 @@ struct unixFile { unsigned char transCntrChng; /* True if the transaction counter changed */ unsigned char dbUpdate; /* True if any part of database file changed */ unsigned char inNormalWrite; /* True if in a normal write operation */ + #endif + #ifdef SQLITE_TEST /* In test mode, increase the size of this structure a bit so that ** it is larger than the struct CrashFile defined in test6.c. @@ -262,6 +280,7 @@ struct unixFile { #define UNIXFILE_DELETE 0x20 /* Delete on close */ #define UNIXFILE_URI 0x40 /* Filename might have query parameters */ #define UNIXFILE_NOLOCK 0x80 /* Do no file locking */ +#define UNIXFILE_WARNED 0x0100 /* verifyDbFile() warnings have been issued */ /* ** Include code that is common to all os_*.c files @@ -296,6 +315,17 @@ struct unixFile { #endif /* +** HAVE_MREMAP defaults to true on Linux and false everywhere else. +*/ +#if !defined(HAVE_MREMAP) +# if defined(__linux__) && defined(_GNU_SOURCE) +# define HAVE_MREMAP 1 +# else +# define HAVE_MREMAP 0 +# endif +#endif + +/* ** Different Unix systems declare open() in different ways. Same use ** open(const char*,int,mode_t). Others use open(const char*,int,...). ** The difference is important when using a pointer to the function. @@ -326,7 +356,7 @@ static int openDirectory(const char*, int*); ** to all overrideable system calls. */ static struct unix_syscall { - const char *zName; /* Name of the sytem call */ + const char *zName; /* Name of the system call */ sqlite3_syscall_ptr pCurrent; /* Current value of the system call */ sqlite3_syscall_ptr pDefault; /* Default value */ } aSyscall[] = { @@ -401,11 +431,7 @@ static struct unix_syscall { #define osPwrite64 ((ssize_t(*)(int,const void*,size_t,off_t))\ aSyscall[13].pCurrent) -#if SQLITE_ENABLE_LOCKING_STYLE { "fchmod", (sqlite3_syscall_ptr)fchmod, 0 }, -#else - { "fchmod", (sqlite3_syscall_ptr)0, 0 }, -#endif #define osFchmod ((int(*)(int,mode_t))aSyscall[14].pCurrent) #if defined(HAVE_POSIX_FALLOCATE) && HAVE_POSIX_FALLOCATE @@ -430,8 +456,18 @@ static struct unix_syscall { { "fchown", (sqlite3_syscall_ptr)posixFchown, 0 }, #define osFchown ((int(*)(int,uid_t,gid_t))aSyscall[20].pCurrent) - { "umask", (sqlite3_syscall_ptr)umask, 0 }, -#define osUmask ((mode_t(*)(mode_t))aSyscall[21].pCurrent) + { "mmap", (sqlite3_syscall_ptr)mmap, 0 }, +#define osMmap ((void*(*)(void*,size_t,int,int,int,off_t))aSyscall[21].pCurrent) + + { "munmap", (sqlite3_syscall_ptr)munmap, 0 }, +#define osMunmap ((void*(*)(void*,size_t))aSyscall[22].pCurrent) + +#if HAVE_MREMAP + { "mremap", (sqlite3_syscall_ptr)mremap, 0 }, +#else + { "mremap", (sqlite3_syscall_ptr)0, 0 }, +#endif +#define osMremap ((void*(*)(void*,size_t,size_t,int,...))aSyscall[23].pCurrent) }; /* End of the overrideable system calls */ @@ -537,14 +573,7 @@ static const char *unixNextSystemCall(sqlite3_vfs *p, const char *zName){ */ static int robust_open(const char *z, int f, mode_t m){ int fd; - mode_t m2; - mode_t origM = 0; - if( m==0 ){ - m2 = SQLITE_DEFAULT_FILE_PERMISSIONS; - }else{ - m2 = m; - origM = osUmask(0); - } + mode_t m2 = m ? m : SQLITE_DEFAULT_FILE_PERMISSIONS; do{ #if defined(O_CLOEXEC) fd = osOpen(z,f|O_CLOEXEC,m2); @@ -552,12 +581,20 @@ static int robust_open(const char *z, int f, mode_t m){ fd = osOpen(z,f,m2); #endif }while( fd<0 && errno==EINTR ); - if( m ){ - osUmask(origM); - } + if( fd>=0 ){ + if( m!=0 ){ + struct stat statbuf; + if( osFstat(fd, &statbuf)==0 + && statbuf.st_size==0 + && (statbuf.st_mode&0777)!=m + ){ + osFchmod(fd, m); + } + } #if defined(FD_CLOEXEC) && (!defined(O_CLOEXEC) || O_CLOEXEC==0) - if( fd>=0 ) osFcntl(fd, F_SETFD, osFcntl(fd, F_GETFD, 0) | FD_CLOEXEC); + osFcntl(fd, F_SETFD, osFcntl(fd, F_GETFD, 0) | FD_CLOEXEC); #endif + } return fd; } @@ -763,7 +800,6 @@ static int sqliteErrorFromPosixError(int posixError, int sqliteIOErr) { } - /****************************************************************************** ****************** Begin Unique File ID Utility Used By VxWorks *************** ** @@ -1099,7 +1135,6 @@ static int unixLogErrorAtLine( zErr = strerror(iErrno); #endif - assert( errcode!=SQLITE_OK ); if( zPath==0 ) zPath = ""; sqlite3_log(errcode, "os_unix.c:%d: (%d) %s(%s) - %s", @@ -1266,6 +1301,50 @@ static int findInodeInfo( /* +** Check a unixFile that is a database. Verify the following: +** +** (1) There is exactly one hard link on the file +** (2) The file is not a symbolic link +** (3) The file has not been renamed or unlinked +** +** Issue sqlite3_log(SQLITE_WARNING,...) messages if anything is not right. +*/ +static void verifyDbFile(unixFile *pFile){ + struct stat buf; + int rc; + if( pFile->ctrlFlags & UNIXFILE_WARNED ){ + /* One or more of the following warnings have already been issued. Do not + ** repeat them so as not to clutter the error log */ + return; + } + rc = osFstat(pFile->h, &buf); + if( rc!=0 ){ + sqlite3_log(SQLITE_WARNING, "cannot fstat db file %s", pFile->zPath); + pFile->ctrlFlags |= UNIXFILE_WARNED; + return; + } + if( buf.st_nlink==0 && (pFile->ctrlFlags & UNIXFILE_DELETE)==0 ){ + sqlite3_log(SQLITE_WARNING, "file unlinked while open: %s", pFile->zPath); + pFile->ctrlFlags |= UNIXFILE_WARNED; + return; + } + if( buf.st_nlink>1 ){ + sqlite3_log(SQLITE_WARNING, "multiple links to file: %s", pFile->zPath); + pFile->ctrlFlags |= UNIXFILE_WARNED; + return; + } + if( pFile->pInode!=0 + && ((rc = osStat(pFile->zPath, &buf))!=0 + || buf.st_ino!=pFile->pInode->fileId.ino) + ){ + sqlite3_log(SQLITE_WARNING, "file renamed while open: %s", pFile->zPath); + pFile->ctrlFlags |= UNIXFILE_WARNED; + return; + } +} + + +/* ** This routine checks if there is a RESERVED lock held on the specified ** file by this or any other process. If such a lock is held, set *pResOut ** to a non-zero value otherwise *pResOut is set to zero. The return value @@ -1795,9 +1874,13 @@ end_unlock: ** the requested locking level, this routine is a no-op. */ static int unixUnlock(sqlite3_file *id, int eFileLock){ + assert( eFileLock==SHARED_LOCK || ((unixFile *)id)->nFetchOut==0 ); return posixUnlock(id, eFileLock, 0); } +static int unixMapfile(unixFile *pFd, i64 nByte); +static void unixUnmapfile(unixFile *pFd); + /* ** This function performs the parts of the "close file" operation ** common to all locking schemes. It closes the directory and file @@ -1810,6 +1893,7 @@ static int unixUnlock(sqlite3_file *id, int eFileLock){ */ static int closeUnixFile(sqlite3_file *id){ unixFile *pFile = (unixFile*)id; + unixUnmapfile(pFile); if( pFile->h>=0 ){ robust_close(pFile, pFile->h, __LINE__); pFile->h = -1; @@ -1836,6 +1920,7 @@ static int closeUnixFile(sqlite3_file *id){ static int unixClose(sqlite3_file *id){ int rc = SQLITE_OK; unixFile *pFile = (unixFile *)id; + verifyDbFile(pFile); unixUnlock(id, NO_LOCK); unixEnterMutex(); @@ -1904,7 +1989,7 @@ static int nolockClose(sqlite3_file *id) { /****************************************************************************** ************************* Begin dot-file Locking ****************************** ** -** The dotfile locking implementation uses the existance of separate lock +** The dotfile locking implementation uses the existence of separate lock ** files (really a directory) to control access to the database. This works ** on just about every filesystem imaginable. But there are serious downsides: ** @@ -1919,7 +2004,7 @@ static int nolockClose(sqlite3_file *id) { ** ** Dotfile locking works by creating a subdirectory in the same directory as ** the database and with the same name but with a ".lock" extension added. -** The existance of a lock directory implies an EXCLUSIVE lock. All other +** The existence of a lock directory implies an EXCLUSIVE lock. All other ** lock types (SHARED, RESERVED, PENDING) are mapped into EXCLUSIVE. */ @@ -2086,13 +2171,13 @@ static int dotlockUnlock(sqlite3_file *id, int eFileLock) { ** Close a file. Make sure the lock has been released before closing. */ static int dotlockClose(sqlite3_file *id) { - int rc; + int rc = SQLITE_OK; if( id ){ unixFile *pFile = (unixFile*)id; dotlockUnlock(id, NO_LOCK); sqlite3_free(pFile->lockingContext); + rc = closeUnixFile(id); } - rc = closeUnixFile(id); return rc; } /****************** End of the dot-file lock implementation ******************* @@ -2296,10 +2381,12 @@ static int flockUnlock(sqlite3_file *id, int eFileLock) { ** Close a file. */ static int flockClose(sqlite3_file *id) { + int rc = SQLITE_OK; if( id ){ flockUnlock(id, NO_LOCK); + rc = closeUnixFile(id); } - return closeUnixFile(id); + return rc; } #endif /* SQLITE_ENABLE_LOCKING_STYLE && !OS_VXWORK */ @@ -3010,6 +3097,8 @@ static int seekAndRead(unixFile *id, sqlite3_int64 offset, void *pBuf, int cnt){ i64 newOffset; #endif TIMER_START; + assert( cnt==(cnt&0x1ffff) ); + cnt &= 0x1ffff; do{ #if defined(USE_PREAD) got = osPread(id->h, pBuf, cnt, offset); @@ -3063,6 +3152,8 @@ static int unixRead( unixFile *pFile = (unixFile *)id; int got; assert( id ); + assert( offset>=0 ); + assert( amt>0 ); /* If this is a database file (not a journal, master-journal or temp ** file), the bytes in the locking range should never be read or written. */ @@ -3073,6 +3164,23 @@ static int unixRead( ); #endif +#if SQLITE_MAX_MMAP_SIZE>0 + /* Deal with as much of this read request as possible by transfering + ** data from the memory mapping using memcpy(). */ + if( offset<pFile->mmapSize ){ + if( offset+amt <= pFile->mmapSize ){ + memcpy(pBuf, &((u8 *)(pFile->pMapRegion))[offset], amt); + return SQLITE_OK; + }else{ + int nCopy = pFile->mmapSize - offset; + memcpy(pBuf, &((u8 *)(pFile->pMapRegion))[offset], nCopy); + pBuf = &((u8 *)pBuf)[nCopy]; + amt -= nCopy; + offset += nCopy; + } + } +#endif + got = seekAndRead(pFile, offset, pBuf, amt); if( got==amt ){ return SQLITE_OK; @@ -3088,44 +3196,59 @@ static int unixRead( } /* -** Seek to the offset in id->offset then read cnt bytes into pBuf. -** Return the number of bytes actually read. Update the offset. -** -** To avoid stomping the errno value on a failed write the lastErrno value -** is set before returning. +** Attempt to seek the file-descriptor passed as the first argument to +** absolute offset iOff, then attempt to write nBuf bytes of data from +** pBuf to it. If an error occurs, return -1 and set *piErrno. Otherwise, +** return the actual number of bytes written (which may be less than +** nBuf). */ -static int seekAndWrite(unixFile *id, i64 offset, const void *pBuf, int cnt){ - int got; -#if (!defined(USE_PREAD) && !defined(USE_PREAD64)) - i64 newOffset; -#endif +static int seekAndWriteFd( + int fd, /* File descriptor to write to */ + i64 iOff, /* File offset to begin writing at */ + const void *pBuf, /* Copy data from this buffer to the file */ + int nBuf, /* Size of buffer pBuf in bytes */ + int *piErrno /* OUT: Error number if error occurs */ +){ + int rc = 0; /* Value returned by system call */ + + assert( nBuf==(nBuf&0x1ffff) ); + nBuf &= 0x1ffff; TIMER_START; + #if defined(USE_PREAD) - do{ got = osPwrite(id->h, pBuf, cnt, offset); }while( got<0 && errno==EINTR ); + do{ rc = osPwrite(fd, pBuf, nBuf, iOff); }while( rc<0 && errno==EINTR ); #elif defined(USE_PREAD64) - do{ got = osPwrite64(id->h, pBuf, cnt, offset);}while( got<0 && errno==EINTR); + do{ rc = osPwrite64(fd, pBuf, nBuf, iOff);}while( rc<0 && errno==EINTR); #else do{ - newOffset = lseek(id->h, offset, SEEK_SET); - SimulateIOError( newOffset-- ); - if( newOffset!=offset ){ - if( newOffset == -1 ){ - ((unixFile*)id)->lastErrno = errno; - }else{ - ((unixFile*)id)->lastErrno = 0; - } + i64 iSeek = lseek(fd, iOff, SEEK_SET); + SimulateIOError( iSeek-- ); + + if( iSeek!=iOff ){ + if( piErrno ) *piErrno = (iSeek==-1 ? errno : 0); return -1; } - got = osWrite(id->h, pBuf, cnt); - }while( got<0 && errno==EINTR ); + rc = osWrite(fd, pBuf, nBuf); + }while( rc<0 && errno==EINTR ); #endif + TIMER_END; - if( got<0 ){ - ((unixFile*)id)->lastErrno = errno; - } + OSTRACE(("WRITE %-3d %5d %7lld %llu\n", fd, rc, iOff, TIMER_ELAPSED)); - OSTRACE(("WRITE %-3d %5d %7lld %llu\n", id->h, got, offset, TIMER_ELAPSED)); - return got; + if( rc<0 && piErrno ) *piErrno = errno; + return rc; +} + + +/* +** Seek to the offset in id->offset then read cnt bytes into pBuf. +** Return the number of bytes actually read. Update the offset. +** +** To avoid stomping the errno value on a failed write the lastErrno value +** is set before returning. +*/ +static int seekAndWrite(unixFile *id, i64 offset, const void *pBuf, int cnt){ + return seekAndWriteFd(id->h, offset, pBuf, cnt, &id->lastErrno); } @@ -3175,6 +3298,23 @@ static int unixWrite( } #endif +#if SQLITE_MAX_MMAP_SIZE>0 + /* Deal with as much of this write request as possible by transfering + ** data from the memory mapping using memcpy(). */ + if( offset<pFile->mmapSize ){ + if( offset+amt <= pFile->mmapSize ){ + memcpy(&((u8 *)(pFile->pMapRegion))[offset], pBuf, amt); + return SQLITE_OK; + }else{ + int nCopy = pFile->mmapSize - offset; + memcpy(&((u8 *)(pFile->pMapRegion))[offset], pBuf, nCopy); + pBuf = &((u8 *)pBuf)[nCopy]; + amt -= nCopy; + offset += nCopy; + } + } +#endif + while( amt>0 && (wrote = seekAndWrite(pFile, offset, pBuf, amt))>0 ){ amt -= wrote; offset += wrote; @@ -3402,7 +3542,7 @@ static int unixSync(sqlite3_file *id, int flags){ } /* Also fsync the directory containing the file if the DIRSYNC flag - ** is set. This is a one-time occurrance. Many systems (examples: AIX) + ** is set. This is a one-time occurrence. Many systems (examples: AIX) ** are unable to fsync a directory, so ignore errors on the fsync. */ if( pFile->ctrlFlags & UNIXFILE_DIRSYNC ){ @@ -3457,6 +3597,14 @@ static int unixTruncate(sqlite3_file *id, i64 nByte){ } #endif + /* If the file was just truncated to a size smaller than the currently + ** mapped region, reduce the effective mapping size as well. SQLite will + ** use read() and write() to access data beyond this point from now on. + */ + if( nByte<pFile->mmapSize ){ + pFile->mmapSize = nByte; + } + return SQLITE_OK; } } @@ -3545,6 +3693,19 @@ static int fcntlSizeHint(unixFile *pFile, i64 nByte){ } } + if( pFile->mmapSizeMax>0 && nByte>pFile->mmapSize ){ + int rc; + if( pFile->szChunk<=0 ){ + if( robust_ftruncate(pFile->h, nByte) ){ + pFile->lastErrno = errno; + return unixLogError(SQLITE_IOERR_TRUNCATE, "ftruncate", pFile->zPath); + } + } + + rc = unixMapfile(pFile, nByte); + return rc; + } + return SQLITE_OK; } @@ -3564,6 +3725,9 @@ static void unixModeBit(unixFile *pFile, unsigned char mask, int *pArg){ } } +/* Forward declaration */ +static int unixGetTempname(int nBuf, char *zBuf); + /* ** Information and control of an open file handle. */ @@ -3601,6 +3765,26 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ *(char**)pArg = sqlite3_mprintf("%s", pFile->pVfs->zName); return SQLITE_OK; } + case SQLITE_FCNTL_TEMPFILENAME: { + char *zTFile = sqlite3_malloc( pFile->pVfs->mxPathname ); + if( zTFile ){ + unixGetTempname(pFile->pVfs->mxPathname, zTFile); + *(char**)pArg = zTFile; + } + return SQLITE_OK; + } + case SQLITE_FCNTL_MMAP_SIZE: { + i64 newLimit = *(i64*)pArg; + if( newLimit>sqlite3GlobalConfig.mxMmap ){ + newLimit = sqlite3GlobalConfig.mxMmap; + } + *(i64*)pArg = pFile->mmapSizeMax; + if( newLimit>=0 ){ + pFile->mmapSizeMax = newLimit; + if( newLimit<pFile->mmapSize ) pFile->mmapSize = newLimit; + } + return SQLITE_OK; + } #ifdef SQLITE_DEBUG /* The pager calls this method to signal that it has done ** a rollback and that the database is therefore unchanged and @@ -3632,10 +3816,92 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ ** a database and its journal file) that the sector size will be the ** same for both. */ -static int unixSectorSize(sqlite3_file *pFile){ - (void)pFile; +#ifndef __QNXNTO__ +static int unixSectorSize(sqlite3_file *NotUsed){ + UNUSED_PARAMETER(NotUsed); return SQLITE_DEFAULT_SECTOR_SIZE; } +#endif + +/* +** The following version of unixSectorSize() is optimized for QNX. +*/ +#ifdef __QNXNTO__ +#include <sys/dcmd_blk.h> +#include <sys/statvfs.h> +static int unixSectorSize(sqlite3_file *id){ + unixFile *pFile = (unixFile*)id; + if( pFile->sectorSize == 0 ){ + struct statvfs fsInfo; + + /* Set defaults for non-supported filesystems */ + pFile->sectorSize = SQLITE_DEFAULT_SECTOR_SIZE; + pFile->deviceCharacteristics = 0; + if( fstatvfs(pFile->h, &fsInfo) == -1 ) { + return pFile->sectorSize; + } + + if( !strcmp(fsInfo.f_basetype, "tmp") ) { + pFile->sectorSize = fsInfo.f_bsize; + pFile->deviceCharacteristics = + SQLITE_IOCAP_ATOMIC4K | /* All ram filesystem writes are atomic */ + SQLITE_IOCAP_SAFE_APPEND | /* growing the file does not occur until + ** the write succeeds */ + SQLITE_IOCAP_SEQUENTIAL | /* The ram filesystem has no write behind + ** so it is ordered */ + 0; + }else if( strstr(fsInfo.f_basetype, "etfs") ){ + pFile->sectorSize = fsInfo.f_bsize; + pFile->deviceCharacteristics = + /* etfs cluster size writes are atomic */ + (pFile->sectorSize / 512 * SQLITE_IOCAP_ATOMIC512) | + SQLITE_IOCAP_SAFE_APPEND | /* growing the file does not occur until + ** the write succeeds */ + SQLITE_IOCAP_SEQUENTIAL | /* The ram filesystem has no write behind + ** so it is ordered */ + 0; + }else if( !strcmp(fsInfo.f_basetype, "qnx6") ){ + pFile->sectorSize = fsInfo.f_bsize; + pFile->deviceCharacteristics = + SQLITE_IOCAP_ATOMIC | /* All filesystem writes are atomic */ + SQLITE_IOCAP_SAFE_APPEND | /* growing the file does not occur until + ** the write succeeds */ + SQLITE_IOCAP_SEQUENTIAL | /* The ram filesystem has no write behind + ** so it is ordered */ + 0; + }else if( !strcmp(fsInfo.f_basetype, "qnx4") ){ + pFile->sectorSize = fsInfo.f_bsize; + pFile->deviceCharacteristics = + /* full bitset of atomics from max sector size and smaller */ + ((pFile->sectorSize / 512 * SQLITE_IOCAP_ATOMIC512) << 1) - 2 | + SQLITE_IOCAP_SEQUENTIAL | /* The ram filesystem has no write behind + ** so it is ordered */ + 0; + }else if( strstr(fsInfo.f_basetype, "dos") ){ + pFile->sectorSize = fsInfo.f_bsize; + pFile->deviceCharacteristics = + /* full bitset of atomics from max sector size and smaller */ + ((pFile->sectorSize / 512 * SQLITE_IOCAP_ATOMIC512) << 1) - 2 | + SQLITE_IOCAP_SEQUENTIAL | /* The ram filesystem has no write behind + ** so it is ordered */ + 0; + }else{ + pFile->deviceCharacteristics = + SQLITE_IOCAP_ATOMIC512 | /* blocks are atomic */ + SQLITE_IOCAP_SAFE_APPEND | /* growing the file does not occur until + ** the write succeeds */ + 0; + } + } + /* Last chance verification. If the sector size isn't a multiple of 512 + ** then it isn't valid.*/ + if( pFile->sectorSize % 512 != 0 ){ + pFile->deviceCharacteristics = 0; + pFile->sectorSize = SQLITE_DEFAULT_SECTOR_SIZE; + } + return pFile->sectorSize; +} +#endif /* __QNXNTO__ */ /* ** Return the device characteristics for the file. @@ -3652,11 +3918,15 @@ static int unixSectorSize(sqlite3_file *pFile){ */ static int unixDeviceCharacteristics(sqlite3_file *id){ unixFile *p = (unixFile*)id; + int rc = 0; +#ifdef __QNXNTO__ + if( p->sectorSize==0 ) unixSectorSize(id); + rc = p->deviceCharacteristics; +#endif if( p->ctrlFlags & UNIXFILE_PSOW ){ - return SQLITE_IOCAP_POWERSAFE_OVERWRITE; - }else{ - return 0; + rc |= SQLITE_IOCAP_POWERSAFE_OVERWRITE; } + return rc; } #ifndef SQLITE_OMIT_WAL @@ -3827,7 +4097,7 @@ static void unixShmPurge(unixFile *pFd){ sqlite3_mutex_free(p->mutex); for(i=0; i<p->nRegion; i++){ if( p->h>=0 ){ - munmap(p->apRegion[i], p->szRegion); + osMunmap(p->apRegion[i], p->szRegion); }else{ sqlite3_free(p->apRegion[i]); } @@ -4067,16 +4337,32 @@ static int unixShmMap( if( sStat.st_size<nByte ){ /* The requested memory region does not exist. If bExtend is set to ** false, exit early. *pp will be set to NULL and SQLITE_OK returned. - ** - ** Alternatively, if bExtend is true, use ftruncate() to allocate - ** the requested memory region. */ - if( !bExtend ) goto shmpage_out; - if( robust_ftruncate(pShmNode->h, nByte) ){ - rc = unixLogError(SQLITE_IOERR_SHMSIZE, "ftruncate", - pShmNode->zFilename); + if( !bExtend ){ goto shmpage_out; } + + /* Alternatively, if bExtend is true, extend the file. Do this by + ** writing a single byte to the end of each (OS) page being + ** allocated or extended. Technically, we need only write to the + ** last page in order to extend the file. But writing to all new + ** pages forces the OS to allocate them immediately, which reduces + ** the chances of SIGBUS while accessing the mapped region later on. + */ + else{ + static const int pgsz = 4096; + int iPg; + + /* Write to the last byte of each newly allocated or extended page */ + assert( (nByte % pgsz)==0 ); + for(iPg=(sStat.st_size/pgsz); iPg<(nByte/pgsz); iPg++){ + if( seekAndWriteFd(pShmNode->h, iPg*pgsz + pgsz-1, "", 1, 0)!=1 ){ + const char *zFile = pShmNode->zFilename; + rc = unixLogError(SQLITE_IOERR_SHMSIZE, "write", zFile); + goto shmpage_out; + } + } + } } } @@ -4092,9 +4378,9 @@ static int unixShmMap( while(pShmNode->nRegion<=iRegion){ void *pMem; if( pShmNode->h>=0 ){ - pMem = mmap(0, szRegion, + pMem = osMmap(0, szRegion, pShmNode->isReadonly ? PROT_READ : PROT_READ|PROT_WRITE, - MAP_SHARED, pShmNode->h, pShmNode->nRegion*szRegion + MAP_SHARED, pShmNode->h, szRegion*(i64)pShmNode->nRegion ); if( pMem==MAP_FAILED ){ rc = unixLogError(SQLITE_IOERR_SHMMAP, "mmap", pShmNode->zFilename); @@ -4310,6 +4596,236 @@ static int unixShmUnmap( #endif /* #ifndef SQLITE_OMIT_WAL */ /* +** If it is currently memory mapped, unmap file pFd. +*/ +static void unixUnmapfile(unixFile *pFd){ + assert( pFd->nFetchOut==0 ); +#if SQLITE_MAX_MMAP_SIZE>0 + if( pFd->pMapRegion ){ + osMunmap(pFd->pMapRegion, pFd->mmapSizeActual); + pFd->pMapRegion = 0; + pFd->mmapSize = 0; + pFd->mmapSizeActual = 0; + } +#endif +} + +#if SQLITE_MAX_MMAP_SIZE>0 +/* +** Return the system page size. +*/ +static int unixGetPagesize(void){ +#if HAVE_MREMAP + return 512; +#elif defined(_BSD_SOURCE) + return getpagesize(); +#else + return (int)sysconf(_SC_PAGESIZE); +#endif +} +#endif /* SQLITE_MAX_MMAP_SIZE>0 */ + +#if SQLITE_MAX_MMAP_SIZE>0 +/* +** Attempt to set the size of the memory mapping maintained by file +** descriptor pFd to nNew bytes. Any existing mapping is discarded. +** +** If successful, this function sets the following variables: +** +** unixFile.pMapRegion +** unixFile.mmapSize +** unixFile.mmapSizeActual +** +** If unsuccessful, an error message is logged via sqlite3_log() and +** the three variables above are zeroed. In this case SQLite should +** continue accessing the database using the xRead() and xWrite() +** methods. +*/ +static void unixRemapfile( + unixFile *pFd, /* File descriptor object */ + i64 nNew /* Required mapping size */ +){ + const char *zErr = "mmap"; + int h = pFd->h; /* File descriptor open on db file */ + u8 *pOrig = (u8 *)pFd->pMapRegion; /* Pointer to current file mapping */ + i64 nOrig = pFd->mmapSizeActual; /* Size of pOrig region in bytes */ + u8 *pNew = 0; /* Location of new mapping */ + int flags = PROT_READ; /* Flags to pass to mmap() */ + + assert( pFd->nFetchOut==0 ); + assert( nNew>pFd->mmapSize ); + assert( nNew<=pFd->mmapSizeMax ); + assert( nNew>0 ); + assert( pFd->mmapSizeActual>=pFd->mmapSize ); + assert( MAP_FAILED!=0 ); + + if( (pFd->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE; + + if( pOrig ){ + const int szSyspage = unixGetPagesize(); + i64 nReuse = (pFd->mmapSize & ~(szSyspage-1)); + u8 *pReq = &pOrig[nReuse]; + + /* Unmap any pages of the existing mapping that cannot be reused. */ + if( nReuse!=nOrig ){ + osMunmap(pReq, nOrig-nReuse); + } + +#if HAVE_MREMAP + pNew = osMremap(pOrig, nReuse, nNew, MREMAP_MAYMOVE); + zErr = "mremap"; +#else + pNew = osMmap(pReq, nNew-nReuse, flags, MAP_SHARED, h, nReuse); + if( pNew!=MAP_FAILED ){ + if( pNew!=pReq ){ + osMunmap(pNew, nNew - nReuse); + pNew = 0; + }else{ + pNew = pOrig; + } + } +#endif + + /* The attempt to extend the existing mapping failed. Free it. */ + if( pNew==MAP_FAILED || pNew==0 ){ + osMunmap(pOrig, nReuse); + } + } + + /* If pNew is still NULL, try to create an entirely new mapping. */ + if( pNew==0 ){ + pNew = osMmap(0, nNew, flags, MAP_SHARED, h, 0); + } + + if( pNew==MAP_FAILED ){ + pNew = 0; + nNew = 0; + unixLogError(SQLITE_OK, zErr, pFd->zPath); + + /* If the mmap() above failed, assume that all subsequent mmap() calls + ** will probably fail too. Fall back to using xRead/xWrite exclusively + ** in this case. */ + pFd->mmapSizeMax = 0; + } + pFd->pMapRegion = (void *)pNew; + pFd->mmapSize = pFd->mmapSizeActual = nNew; +} +#endif + +/* +** Memory map or remap the file opened by file-descriptor pFd (if the file +** is already mapped, the existing mapping is replaced by the new). Or, if +** there already exists a mapping for this file, and there are still +** outstanding xFetch() references to it, this function is a no-op. +** +** If parameter nByte is non-negative, then it is the requested size of +** the mapping to create. Otherwise, if nByte is less than zero, then the +** requested size is the size of the file on disk. The actual size of the +** created mapping is either the requested size or the value configured +** using SQLITE_FCNTL_MMAP_LIMIT, whichever is smaller. +** +** SQLITE_OK is returned if no error occurs (even if the mapping is not +** recreated as a result of outstanding references) or an SQLite error +** code otherwise. +*/ +static int unixMapfile(unixFile *pFd, i64 nByte){ +#if SQLITE_MAX_MMAP_SIZE>0 + i64 nMap = nByte; + int rc; + + assert( nMap>=0 || pFd->nFetchOut==0 ); + if( pFd->nFetchOut>0 ) return SQLITE_OK; + + if( nMap<0 ){ + struct stat statbuf; /* Low-level file information */ + rc = osFstat(pFd->h, &statbuf); + if( rc!=SQLITE_OK ){ + return SQLITE_IOERR_FSTAT; + } + nMap = statbuf.st_size; + } + if( nMap>pFd->mmapSizeMax ){ + nMap = pFd->mmapSizeMax; + } + + if( nMap!=pFd->mmapSize ){ + if( nMap>0 ){ + unixRemapfile(pFd, nMap); + }else{ + unixUnmapfile(pFd); + } + } +#endif + + return SQLITE_OK; +} + +/* +** If possible, return a pointer to a mapping of file fd starting at offset +** iOff. The mapping must be valid for at least nAmt bytes. +** +** If such a pointer can be obtained, store it in *pp and return SQLITE_OK. +** Or, if one cannot but no error occurs, set *pp to 0 and return SQLITE_OK. +** Finally, if an error does occur, return an SQLite error code. The final +** value of *pp is undefined in this case. +** +** If this function does return a pointer, the caller must eventually +** release the reference by calling unixUnfetch(). +*/ +static int unixFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ +#if SQLITE_MAX_MMAP_SIZE>0 + unixFile *pFd = (unixFile *)fd; /* The underlying database file */ +#endif + *pp = 0; + +#if SQLITE_MAX_MMAP_SIZE>0 + if( pFd->mmapSizeMax>0 ){ + if( pFd->pMapRegion==0 ){ + int rc = unixMapfile(pFd, -1); + if( rc!=SQLITE_OK ) return rc; + } + if( pFd->mmapSize >= iOff+nAmt ){ + *pp = &((u8 *)pFd->pMapRegion)[iOff]; + pFd->nFetchOut++; + } + } +#endif + return SQLITE_OK; +} + +/* +** If the third argument is non-NULL, then this function releases a +** reference obtained by an earlier call to unixFetch(). The second +** argument passed to this function must be the same as the corresponding +** argument that was passed to the unixFetch() invocation. +** +** Or, if the third argument is NULL, then this function is being called +** to inform the VFS layer that, according to POSIX, any existing mapping +** may now be invalid and should be unmapped. +*/ +static int unixUnfetch(sqlite3_file *fd, i64 iOff, void *p){ + unixFile *pFd = (unixFile *)fd; /* The underlying database file */ + UNUSED_PARAMETER(iOff); + + /* If p==0 (unmap the entire file) then there must be no outstanding + ** xFetch references. Or, if p!=0 (meaning it is an xFetch reference), + ** then there must be at least one outstanding. */ + assert( (p==0)==(pFd->nFetchOut==0) ); + + /* If p!=0, it must match the iOff value. */ + assert( p==0 || p==&((u8 *)pFd->pMapRegion)[iOff] ); + + if( p ){ + pFd->nFetchOut--; + }else{ + unixUnmapfile(pFd); + } + + assert( pFd->nFetchOut>=0 ); + return SQLITE_OK; +} + +/* ** Here ends the implementation of all sqlite3_file methods. ** ********************** End sqlite3_file Methods ******************************* @@ -4367,7 +4883,9 @@ static const sqlite3_io_methods METHOD = { \ unixShmMap, /* xShmMap */ \ unixShmLock, /* xShmLock */ \ unixShmBarrier, /* xShmBarrier */ \ - unixShmUnmap /* xShmUnmap */ \ + unixShmUnmap, /* xShmUnmap */ \ + unixFetch, /* xFetch */ \ + unixUnfetch, /* xUnfetch */ \ }; \ static const sqlite3_io_methods *FINDER##Impl(const char *z, unixFile *p){ \ UNUSED_PARAMETER(z); UNUSED_PARAMETER(p); \ @@ -4384,7 +4902,7 @@ static const sqlite3_io_methods *(*const FINDER)(const char*,unixFile *p) \ IOMETHODS( posixIoFinder, /* Finder function name */ posixIoMethods, /* sqlite3_io_methods object name */ - 2, /* shared memory is enabled */ + 3, /* shared memory and mmap are enabled */ unixClose, /* xClose method */ unixLock, /* xLock method */ unixUnlock, /* xUnlock method */ @@ -4635,11 +5153,12 @@ static int fillInUnixFile( pNew->pVfs = pVfs; pNew->zPath = zFilename; pNew->ctrlFlags = (u8)ctrlFlags; + pNew->mmapSizeMax = sqlite3GlobalConfig.szMmap; if( sqlite3_uri_boolean(((ctrlFlags & UNIXFILE_URI) ? zFilename : 0), "psow", SQLITE_POWERSAFE_OVERWRITE) ){ pNew->ctrlFlags |= UNIXFILE_PSOW; } - if( memcmp(pVfs->zName,"unix-excl",10)==0 ){ + if( strcmp(pVfs->zName,"unix-excl")==0 ){ pNew->ctrlFlags |= UNIXFILE_EXCL; } @@ -4671,7 +5190,7 @@ static int fillInUnixFile( unixEnterMutex(); rc = findInodeInfo(pNew, &pNew->pInode); if( rc!=SQLITE_OK ){ - /* If an error occured in findInodeInfo(), close the file descriptor + /* If an error occurred in findInodeInfo(), close the file descriptor ** immediately, before releasing the mutex. findInodeInfo() may fail ** in two scenarios: ** @@ -4770,15 +5289,15 @@ static int fillInUnixFile( if( h>=0 ) robust_close(pNew, h, __LINE__); h = -1; osUnlink(zFilename); - isDelete = 0; + pNew->ctrlFlags |= UNIXFILE_DELETE; } - if( isDelete ) pNew->ctrlFlags |= UNIXFILE_DELETE; #endif if( rc!=SQLITE_OK ){ if( h>=0 ) robust_close(pNew, h, __LINE__); }else{ pNew->pMethod = pLockingStyle; OpenCounter(+1); + verifyDbFile(pNew); } return rc; } @@ -5278,8 +5797,13 @@ static int unixDelete( int rc = SQLITE_OK; UNUSED_PARAMETER(NotUsed); SimulateIOError(return SQLITE_IOERR_DELETE); - if( osUnlink(zPath)==(-1) && errno!=ENOENT ){ - return unixLogError(SQLITE_IOERR_DELETE, "unlink", zPath); + if( osUnlink(zPath)==(-1) ){ + if( errno==ENOENT ){ + rc = SQLITE_IOERR_DELETE_NOENT; + }else{ + rc = unixLogError(SQLITE_IOERR_DELETE, "unlink", zPath); + } + return rc; } #ifndef SQLITE_DISABLE_DIRSYNC if( (dirSync & 1)!=0 ){ @@ -5304,7 +5828,7 @@ static int unixDelete( } /* -** Test the existance of or access permissions of file zPath. The +** Test the existence of or access permissions of file zPath. The ** test performed depends on the value of flags: ** ** SQLITE_ACCESS_EXISTS: Return 1 if the file exists @@ -6867,7 +7391,7 @@ int sqlite3_os_init(void){ /* Double-check that the aSyscall[] array has been constructed ** correctly. See ticket [bb3a86e890c8e96ab] */ - assert( ArraySize(aSyscall)==22 ); + assert( ArraySize(aSyscall)==24 ); /* Register all VFSes defined in the aVfs[] array */ for(i=0; i<(sizeof(aVfs)/sizeof(sqlite3_vfs)); i++){ diff --git a/src/os_win.c b/src/os_win.c index cbf17b1..aeb0881 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -25,6 +25,66 @@ #include "os_common.h" /* +** Compiling and using WAL mode requires several APIs that are only +** available in Windows platforms based on the NT kernel. +*/ +#if !SQLITE_OS_WINNT && !defined(SQLITE_OMIT_WAL) +# error "WAL mode requires support from the Windows NT kernel, compile\ + with SQLITE_OMIT_WAL." +#endif + +/* +** Are most of the Win32 ANSI APIs available (i.e. with certain exceptions +** based on the sub-platform)? +*/ +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT +# define SQLITE_WIN32_HAS_ANSI +#endif + +/* +** Are most of the Win32 Unicode APIs available (i.e. with certain exceptions +** based on the sub-platform)? +*/ +#if SQLITE_OS_WINCE || SQLITE_OS_WINNT || SQLITE_OS_WINRT +# define SQLITE_WIN32_HAS_WIDE +#endif + +/* +** Do we need to manually define the Win32 file mapping APIs for use with WAL +** mode (e.g. these APIs are available in the Windows CE SDK; however, they +** are not present in the header file)? +*/ +#if SQLITE_WIN32_FILEMAPPING_API && !defined(SQLITE_OMIT_WAL) +/* +** Two of the file mapping APIs are different under WinRT. Figure out which +** set we need. +*/ +#if SQLITE_OS_WINRT +WINBASEAPI HANDLE WINAPI CreateFileMappingFromApp(HANDLE, \ + LPSECURITY_ATTRIBUTES, ULONG, ULONG64, LPCWSTR); + +WINBASEAPI LPVOID WINAPI MapViewOfFileFromApp(HANDLE, ULONG, ULONG64, SIZE_T); +#else +#if defined(SQLITE_WIN32_HAS_ANSI) +WINBASEAPI HANDLE WINAPI CreateFileMappingA(HANDLE, LPSECURITY_ATTRIBUTES, \ + DWORD, DWORD, DWORD, LPCSTR); +#endif /* defined(SQLITE_WIN32_HAS_ANSI) */ + +#if defined(SQLITE_WIN32_HAS_WIDE) +WINBASEAPI HANDLE WINAPI CreateFileMappingW(HANDLE, LPSECURITY_ATTRIBUTES, \ + DWORD, DWORD, DWORD, LPCWSTR); +#endif /* defined(SQLITE_WIN32_HAS_WIDE) */ + +WINBASEAPI LPVOID WINAPI MapViewOfFile(HANDLE, DWORD, DWORD, DWORD, SIZE_T); +#endif /* SQLITE_OS_WINRT */ + +/* +** This file mapping API is common to both Win32 and WinRT. +*/ +WINBASEAPI BOOL WINAPI UnmapViewOfFile(LPCVOID); +#endif /* SQLITE_WIN32_FILEMAPPING_API && !defined(SQLITE_OMIT_WAL) */ + +/* ** Macro to find the minimum of two numeric values. */ #ifndef MIN @@ -90,11 +150,20 @@ struct winFile { winceLock local; /* Locks obtained by this instance of winFile */ winceLock *shared; /* Global shared lock memory for the file */ #endif +#if SQLITE_MAX_MMAP_SIZE>0 + int nFetchOut; /* Number of outstanding xFetch references */ + HANDLE hMap; /* Handle for accessing memory mapping */ + void *pMapRegion; /* Area memory mapped */ + sqlite3_int64 mmapSize; /* Usable size of mapped region */ + sqlite3_int64 mmapSizeActual; /* Actual size of mapped region */ + sqlite3_int64 mmapSizeMax; /* Configured FCNTL_MMAP_SIZE value */ +#endif }; /* ** Allowed values for winFile.ctrlFlags */ +#define WINFILE_RDONLY 0x02 /* Connection is read only */ #define WINFILE_PERSIST_WAL 0x04 /* Persistent WAL mode */ #define WINFILE_PSOW 0x10 /* SQLITE_IOCAP_POWERSAFE_OVERWRITE */ @@ -229,14 +298,6 @@ int sqlite3_os_type = 0; static int sqlite3_os_type = 0; #endif -#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT -# define SQLITE_WIN32_HAS_ANSI -#endif - -#if SQLITE_OS_WINCE || SQLITE_OS_WINNT || SQLITE_OS_WINRT -# define SQLITE_WIN32_HAS_WIDE -#endif - #ifndef SYSCALL # define SYSCALL sqlite3_syscall_ptr #endif @@ -256,7 +317,7 @@ static int sqlite3_os_type = 0; ** to all overrideable system calls. */ static struct win_syscall { - const char *zName; /* Name of the sytem call */ + const char *zName; /* Name of the system call */ sqlite3_syscall_ptr pCurrent; /* Current value of the system call */ sqlite3_syscall_ptr pDefault; /* Default value */ } aSyscall[] = { @@ -308,6 +369,16 @@ static struct win_syscall { #define osCreateFileW ((HANDLE(WINAPI*)(LPCWSTR,DWORD,DWORD, \ LPSECURITY_ATTRIBUTES,DWORD,DWORD,HANDLE))aSyscall[5].pCurrent) +#if (!SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_ANSI) && \ + !defined(SQLITE_OMIT_WAL)) + { "CreateFileMappingA", (SYSCALL)CreateFileMappingA, 0 }, +#else + { "CreateFileMappingA", (SYSCALL)0, 0 }, +#endif + +#define osCreateFileMappingA ((HANDLE(WINAPI*)(HANDLE,LPSECURITY_ATTRIBUTES, \ + DWORD,DWORD,DWORD,LPCSTR))aSyscall[6].pCurrent) + #if SQLITE_OS_WINCE || (!SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) && \ !defined(SQLITE_OMIT_WAL)) { "CreateFileMappingW", (SYSCALL)CreateFileMappingW, 0 }, @@ -316,7 +387,7 @@ static struct win_syscall { #endif #define osCreateFileMappingW ((HANDLE(WINAPI*)(HANDLE,LPSECURITY_ATTRIBUTES, \ - DWORD,DWORD,DWORD,LPCWSTR))aSyscall[6].pCurrent) + DWORD,DWORD,DWORD,LPCWSTR))aSyscall[7].pCurrent) #if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) { "CreateMutexW", (SYSCALL)CreateMutexW, 0 }, @@ -325,7 +396,7 @@ static struct win_syscall { #endif #define osCreateMutexW ((HANDLE(WINAPI*)(LPSECURITY_ATTRIBUTES,BOOL, \ - LPCWSTR))aSyscall[7].pCurrent) + LPCWSTR))aSyscall[8].pCurrent) #if defined(SQLITE_WIN32_HAS_ANSI) { "DeleteFileA", (SYSCALL)DeleteFileA, 0 }, @@ -333,7 +404,7 @@ static struct win_syscall { { "DeleteFileA", (SYSCALL)0, 0 }, #endif -#define osDeleteFileA ((BOOL(WINAPI*)(LPCSTR))aSyscall[8].pCurrent) +#define osDeleteFileA ((BOOL(WINAPI*)(LPCSTR))aSyscall[9].pCurrent) #if defined(SQLITE_WIN32_HAS_WIDE) { "DeleteFileW", (SYSCALL)DeleteFileW, 0 }, @@ -341,7 +412,7 @@ static struct win_syscall { { "DeleteFileW", (SYSCALL)0, 0 }, #endif -#define osDeleteFileW ((BOOL(WINAPI*)(LPCWSTR))aSyscall[9].pCurrent) +#define osDeleteFileW ((BOOL(WINAPI*)(LPCWSTR))aSyscall[10].pCurrent) #if SQLITE_OS_WINCE { "FileTimeToLocalFileTime", (SYSCALL)FileTimeToLocalFileTime, 0 }, @@ -350,7 +421,7 @@ static struct win_syscall { #endif #define osFileTimeToLocalFileTime ((BOOL(WINAPI*)(CONST FILETIME*, \ - LPFILETIME))aSyscall[10].pCurrent) + LPFILETIME))aSyscall[11].pCurrent) #if SQLITE_OS_WINCE { "FileTimeToSystemTime", (SYSCALL)FileTimeToSystemTime, 0 }, @@ -359,11 +430,11 @@ static struct win_syscall { #endif #define osFileTimeToSystemTime ((BOOL(WINAPI*)(CONST FILETIME*, \ - LPSYSTEMTIME))aSyscall[11].pCurrent) + LPSYSTEMTIME))aSyscall[12].pCurrent) { "FlushFileBuffers", (SYSCALL)FlushFileBuffers, 0 }, -#define osFlushFileBuffers ((BOOL(WINAPI*)(HANDLE))aSyscall[12].pCurrent) +#define osFlushFileBuffers ((BOOL(WINAPI*)(HANDLE))aSyscall[13].pCurrent) #if defined(SQLITE_WIN32_HAS_ANSI) { "FormatMessageA", (SYSCALL)FormatMessageA, 0 }, @@ -372,7 +443,7 @@ static struct win_syscall { #endif #define osFormatMessageA ((DWORD(WINAPI*)(DWORD,LPCVOID,DWORD,DWORD,LPSTR, \ - DWORD,va_list*))aSyscall[13].pCurrent) + DWORD,va_list*))aSyscall[14].pCurrent) #if defined(SQLITE_WIN32_HAS_WIDE) { "FormatMessageW", (SYSCALL)FormatMessageW, 0 }, @@ -381,15 +452,19 @@ static struct win_syscall { #endif #define osFormatMessageW ((DWORD(WINAPI*)(DWORD,LPCVOID,DWORD,DWORD,LPWSTR, \ - DWORD,va_list*))aSyscall[14].pCurrent) + DWORD,va_list*))aSyscall[15].pCurrent) +#if !defined(SQLITE_OMIT_LOAD_EXTENSION) { "FreeLibrary", (SYSCALL)FreeLibrary, 0 }, +#else + { "FreeLibrary", (SYSCALL)0, 0 }, +#endif -#define osFreeLibrary ((BOOL(WINAPI*)(HMODULE))aSyscall[15].pCurrent) +#define osFreeLibrary ((BOOL(WINAPI*)(HMODULE))aSyscall[16].pCurrent) { "GetCurrentProcessId", (SYSCALL)GetCurrentProcessId, 0 }, -#define osGetCurrentProcessId ((DWORD(WINAPI*)(VOID))aSyscall[16].pCurrent) +#define osGetCurrentProcessId ((DWORD(WINAPI*)(VOID))aSyscall[17].pCurrent) #if !SQLITE_OS_WINCE && defined(SQLITE_WIN32_HAS_ANSI) { "GetDiskFreeSpaceA", (SYSCALL)GetDiskFreeSpaceA, 0 }, @@ -398,7 +473,7 @@ static struct win_syscall { #endif #define osGetDiskFreeSpaceA ((BOOL(WINAPI*)(LPCSTR,LPDWORD,LPDWORD,LPDWORD, \ - LPDWORD))aSyscall[17].pCurrent) + LPDWORD))aSyscall[18].pCurrent) #if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) { "GetDiskFreeSpaceW", (SYSCALL)GetDiskFreeSpaceW, 0 }, @@ -407,7 +482,7 @@ static struct win_syscall { #endif #define osGetDiskFreeSpaceW ((BOOL(WINAPI*)(LPCWSTR,LPDWORD,LPDWORD,LPDWORD, \ - LPDWORD))aSyscall[18].pCurrent) + LPDWORD))aSyscall[19].pCurrent) #if defined(SQLITE_WIN32_HAS_ANSI) { "GetFileAttributesA", (SYSCALL)GetFileAttributesA, 0 }, @@ -415,7 +490,7 @@ static struct win_syscall { { "GetFileAttributesA", (SYSCALL)0, 0 }, #endif -#define osGetFileAttributesA ((DWORD(WINAPI*)(LPCSTR))aSyscall[19].pCurrent) +#define osGetFileAttributesA ((DWORD(WINAPI*)(LPCSTR))aSyscall[20].pCurrent) #if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) { "GetFileAttributesW", (SYSCALL)GetFileAttributesW, 0 }, @@ -423,7 +498,7 @@ static struct win_syscall { { "GetFileAttributesW", (SYSCALL)0, 0 }, #endif -#define osGetFileAttributesW ((DWORD(WINAPI*)(LPCWSTR))aSyscall[20].pCurrent) +#define osGetFileAttributesW ((DWORD(WINAPI*)(LPCWSTR))aSyscall[21].pCurrent) #if defined(SQLITE_WIN32_HAS_WIDE) { "GetFileAttributesExW", (SYSCALL)GetFileAttributesExW, 0 }, @@ -432,7 +507,7 @@ static struct win_syscall { #endif #define osGetFileAttributesExW ((BOOL(WINAPI*)(LPCWSTR,GET_FILEEX_INFO_LEVELS, \ - LPVOID))aSyscall[21].pCurrent) + LPVOID))aSyscall[22].pCurrent) #if !SQLITE_OS_WINRT { "GetFileSize", (SYSCALL)GetFileSize, 0 }, @@ -440,7 +515,7 @@ static struct win_syscall { { "GetFileSize", (SYSCALL)0, 0 }, #endif -#define osGetFileSize ((DWORD(WINAPI*)(HANDLE,LPDWORD))aSyscall[22].pCurrent) +#define osGetFileSize ((DWORD(WINAPI*)(HANDLE,LPDWORD))aSyscall[23].pCurrent) #if !SQLITE_OS_WINCE && defined(SQLITE_WIN32_HAS_ANSI) { "GetFullPathNameA", (SYSCALL)GetFullPathNameA, 0 }, @@ -449,7 +524,7 @@ static struct win_syscall { #endif #define osGetFullPathNameA ((DWORD(WINAPI*)(LPCSTR,DWORD,LPSTR, \ - LPSTR*))aSyscall[23].pCurrent) + LPSTR*))aSyscall[24].pCurrent) #if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) { "GetFullPathNameW", (SYSCALL)GetFullPathNameW, 0 }, @@ -458,12 +533,13 @@ static struct win_syscall { #endif #define osGetFullPathNameW ((DWORD(WINAPI*)(LPCWSTR,DWORD,LPWSTR, \ - LPWSTR*))aSyscall[24].pCurrent) + LPWSTR*))aSyscall[25].pCurrent) { "GetLastError", (SYSCALL)GetLastError, 0 }, -#define osGetLastError ((DWORD(WINAPI*)(VOID))aSyscall[25].pCurrent) +#define osGetLastError ((DWORD(WINAPI*)(VOID))aSyscall[26].pCurrent) +#if !defined(SQLITE_OMIT_LOAD_EXTENSION) #if SQLITE_OS_WINCE /* The GetProcAddressA() routine is only available on Windows CE. */ { "GetProcAddressA", (SYSCALL)GetProcAddressA, 0 }, @@ -472,9 +548,12 @@ static struct win_syscall { ** an ANSI string regardless of the _UNICODE setting */ { "GetProcAddressA", (SYSCALL)GetProcAddress, 0 }, #endif +#else + { "GetProcAddressA", (SYSCALL)0, 0 }, +#endif #define osGetProcAddressA ((FARPROC(WINAPI*)(HMODULE, \ - LPCSTR))aSyscall[26].pCurrent) + LPCSTR))aSyscall[27].pCurrent) #if !SQLITE_OS_WINRT { "GetSystemInfo", (SYSCALL)GetSystemInfo, 0 }, @@ -482,11 +561,11 @@ static struct win_syscall { { "GetSystemInfo", (SYSCALL)0, 0 }, #endif -#define osGetSystemInfo ((VOID(WINAPI*)(LPSYSTEM_INFO))aSyscall[27].pCurrent) +#define osGetSystemInfo ((VOID(WINAPI*)(LPSYSTEM_INFO))aSyscall[28].pCurrent) { "GetSystemTime", (SYSCALL)GetSystemTime, 0 }, -#define osGetSystemTime ((VOID(WINAPI*)(LPSYSTEMTIME))aSyscall[28].pCurrent) +#define osGetSystemTime ((VOID(WINAPI*)(LPSYSTEMTIME))aSyscall[29].pCurrent) #if !SQLITE_OS_WINCE { "GetSystemTimeAsFileTime", (SYSCALL)GetSystemTimeAsFileTime, 0 }, @@ -495,7 +574,7 @@ static struct win_syscall { #endif #define osGetSystemTimeAsFileTime ((VOID(WINAPI*)( \ - LPFILETIME))aSyscall[29].pCurrent) + LPFILETIME))aSyscall[30].pCurrent) #if defined(SQLITE_WIN32_HAS_ANSI) { "GetTempPathA", (SYSCALL)GetTempPathA, 0 }, @@ -503,7 +582,7 @@ static struct win_syscall { { "GetTempPathA", (SYSCALL)0, 0 }, #endif -#define osGetTempPathA ((DWORD(WINAPI*)(DWORD,LPSTR))aSyscall[30].pCurrent) +#define osGetTempPathA ((DWORD(WINAPI*)(DWORD,LPSTR))aSyscall[31].pCurrent) #if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) { "GetTempPathW", (SYSCALL)GetTempPathW, 0 }, @@ -511,7 +590,7 @@ static struct win_syscall { { "GetTempPathW", (SYSCALL)0, 0 }, #endif -#define osGetTempPathW ((DWORD(WINAPI*)(DWORD,LPWSTR))aSyscall[31].pCurrent) +#define osGetTempPathW ((DWORD(WINAPI*)(DWORD,LPWSTR))aSyscall[32].pCurrent) #if !SQLITE_OS_WINRT { "GetTickCount", (SYSCALL)GetTickCount, 0 }, @@ -519,7 +598,7 @@ static struct win_syscall { { "GetTickCount", (SYSCALL)0, 0 }, #endif -#define osGetTickCount ((DWORD(WINAPI*)(VOID))aSyscall[32].pCurrent) +#define osGetTickCount ((DWORD(WINAPI*)(VOID))aSyscall[33].pCurrent) #if defined(SQLITE_WIN32_HAS_ANSI) { "GetVersionExA", (SYSCALL)GetVersionExA, 0 }, @@ -528,12 +607,12 @@ static struct win_syscall { #endif #define osGetVersionExA ((BOOL(WINAPI*)( \ - LPOSVERSIONINFOA))aSyscall[33].pCurrent) + LPOSVERSIONINFOA))aSyscall[34].pCurrent) { "HeapAlloc", (SYSCALL)HeapAlloc, 0 }, #define osHeapAlloc ((LPVOID(WINAPI*)(HANDLE,DWORD, \ - SIZE_T))aSyscall[34].pCurrent) + SIZE_T))aSyscall[35].pCurrent) #if !SQLITE_OS_WINRT { "HeapCreate", (SYSCALL)HeapCreate, 0 }, @@ -542,7 +621,7 @@ static struct win_syscall { #endif #define osHeapCreate ((HANDLE(WINAPI*)(DWORD,SIZE_T, \ - SIZE_T))aSyscall[35].pCurrent) + SIZE_T))aSyscall[36].pCurrent) #if !SQLITE_OS_WINRT { "HeapDestroy", (SYSCALL)HeapDestroy, 0 }, @@ -550,21 +629,21 @@ static struct win_syscall { { "HeapDestroy", (SYSCALL)0, 0 }, #endif -#define osHeapDestroy ((BOOL(WINAPI*)(HANDLE))aSyscall[36].pCurrent) +#define osHeapDestroy ((BOOL(WINAPI*)(HANDLE))aSyscall[37].pCurrent) { "HeapFree", (SYSCALL)HeapFree, 0 }, -#define osHeapFree ((BOOL(WINAPI*)(HANDLE,DWORD,LPVOID))aSyscall[37].pCurrent) +#define osHeapFree ((BOOL(WINAPI*)(HANDLE,DWORD,LPVOID))aSyscall[38].pCurrent) { "HeapReAlloc", (SYSCALL)HeapReAlloc, 0 }, #define osHeapReAlloc ((LPVOID(WINAPI*)(HANDLE,DWORD,LPVOID, \ - SIZE_T))aSyscall[38].pCurrent) + SIZE_T))aSyscall[39].pCurrent) { "HeapSize", (SYSCALL)HeapSize, 0 }, #define osHeapSize ((SIZE_T(WINAPI*)(HANDLE,DWORD, \ - LPCVOID))aSyscall[39].pCurrent) + LPCVOID))aSyscall[40].pCurrent) #if !SQLITE_OS_WINRT { "HeapValidate", (SYSCALL)HeapValidate, 0 }, @@ -573,23 +652,24 @@ static struct win_syscall { #endif #define osHeapValidate ((BOOL(WINAPI*)(HANDLE,DWORD, \ - LPCVOID))aSyscall[40].pCurrent) + LPCVOID))aSyscall[41].pCurrent) -#if defined(SQLITE_WIN32_HAS_ANSI) +#if defined(SQLITE_WIN32_HAS_ANSI) && !defined(SQLITE_OMIT_LOAD_EXTENSION) { "LoadLibraryA", (SYSCALL)LoadLibraryA, 0 }, #else { "LoadLibraryA", (SYSCALL)0, 0 }, #endif -#define osLoadLibraryA ((HMODULE(WINAPI*)(LPCSTR))aSyscall[41].pCurrent) +#define osLoadLibraryA ((HMODULE(WINAPI*)(LPCSTR))aSyscall[42].pCurrent) -#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) +#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) && \ + !defined(SQLITE_OMIT_LOAD_EXTENSION) { "LoadLibraryW", (SYSCALL)LoadLibraryW, 0 }, #else { "LoadLibraryW", (SYSCALL)0, 0 }, #endif -#define osLoadLibraryW ((HMODULE(WINAPI*)(LPCWSTR))aSyscall[42].pCurrent) +#define osLoadLibraryW ((HMODULE(WINAPI*)(LPCWSTR))aSyscall[43].pCurrent) #if !SQLITE_OS_WINRT { "LocalFree", (SYSCALL)LocalFree, 0 }, @@ -597,7 +677,7 @@ static struct win_syscall { { "LocalFree", (SYSCALL)0, 0 }, #endif -#define osLocalFree ((HLOCAL(WINAPI*)(HLOCAL))aSyscall[43].pCurrent) +#define osLocalFree ((HLOCAL(WINAPI*)(HLOCAL))aSyscall[44].pCurrent) #if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT { "LockFile", (SYSCALL)LockFile, 0 }, @@ -607,7 +687,7 @@ static struct win_syscall { #ifndef osLockFile #define osLockFile ((BOOL(WINAPI*)(HANDLE,DWORD,DWORD,DWORD, \ - DWORD))aSyscall[44].pCurrent) + DWORD))aSyscall[45].pCurrent) #endif #if !SQLITE_OS_WINCE @@ -618,7 +698,7 @@ static struct win_syscall { #ifndef osLockFileEx #define osLockFileEx ((BOOL(WINAPI*)(HANDLE,DWORD,DWORD,DWORD,DWORD, \ - LPOVERLAPPED))aSyscall[45].pCurrent) + LPOVERLAPPED))aSyscall[46].pCurrent) #endif #if SQLITE_OS_WINCE || (!SQLITE_OS_WINRT && !defined(SQLITE_OMIT_WAL)) @@ -628,26 +708,26 @@ static struct win_syscall { #endif #define osMapViewOfFile ((LPVOID(WINAPI*)(HANDLE,DWORD,DWORD,DWORD, \ - SIZE_T))aSyscall[46].pCurrent) + SIZE_T))aSyscall[47].pCurrent) { "MultiByteToWideChar", (SYSCALL)MultiByteToWideChar, 0 }, #define osMultiByteToWideChar ((int(WINAPI*)(UINT,DWORD,LPCSTR,int,LPWSTR, \ - int))aSyscall[47].pCurrent) + int))aSyscall[48].pCurrent) { "QueryPerformanceCounter", (SYSCALL)QueryPerformanceCounter, 0 }, #define osQueryPerformanceCounter ((BOOL(WINAPI*)( \ - LARGE_INTEGER*))aSyscall[48].pCurrent) + LARGE_INTEGER*))aSyscall[49].pCurrent) { "ReadFile", (SYSCALL)ReadFile, 0 }, #define osReadFile ((BOOL(WINAPI*)(HANDLE,LPVOID,DWORD,LPDWORD, \ - LPOVERLAPPED))aSyscall[49].pCurrent) + LPOVERLAPPED))aSyscall[50].pCurrent) { "SetEndOfFile", (SYSCALL)SetEndOfFile, 0 }, -#define osSetEndOfFile ((BOOL(WINAPI*)(HANDLE))aSyscall[50].pCurrent) +#define osSetEndOfFile ((BOOL(WINAPI*)(HANDLE))aSyscall[51].pCurrent) #if !SQLITE_OS_WINRT { "SetFilePointer", (SYSCALL)SetFilePointer, 0 }, @@ -656,7 +736,7 @@ static struct win_syscall { #endif #define osSetFilePointer ((DWORD(WINAPI*)(HANDLE,LONG,PLONG, \ - DWORD))aSyscall[51].pCurrent) + DWORD))aSyscall[52].pCurrent) #if !SQLITE_OS_WINRT { "Sleep", (SYSCALL)Sleep, 0 }, @@ -664,12 +744,12 @@ static struct win_syscall { { "Sleep", (SYSCALL)0, 0 }, #endif -#define osSleep ((VOID(WINAPI*)(DWORD))aSyscall[52].pCurrent) +#define osSleep ((VOID(WINAPI*)(DWORD))aSyscall[53].pCurrent) { "SystemTimeToFileTime", (SYSCALL)SystemTimeToFileTime, 0 }, #define osSystemTimeToFileTime ((BOOL(WINAPI*)(CONST SYSTEMTIME*, \ - LPFILETIME))aSyscall[53].pCurrent) + LPFILETIME))aSyscall[54].pCurrent) #if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT { "UnlockFile", (SYSCALL)UnlockFile, 0 }, @@ -679,7 +759,7 @@ static struct win_syscall { #ifndef osUnlockFile #define osUnlockFile ((BOOL(WINAPI*)(HANDLE,DWORD,DWORD,DWORD, \ - DWORD))aSyscall[54].pCurrent) + DWORD))aSyscall[55].pCurrent) #endif #if !SQLITE_OS_WINCE @@ -689,7 +769,7 @@ static struct win_syscall { #endif #define osUnlockFileEx ((BOOL(WINAPI*)(HANDLE,DWORD,DWORD,DWORD, \ - LPOVERLAPPED))aSyscall[55].pCurrent) + LPOVERLAPPED))aSyscall[56].pCurrent) #if SQLITE_OS_WINCE || !defined(SQLITE_OMIT_WAL) { "UnmapViewOfFile", (SYSCALL)UnmapViewOfFile, 0 }, @@ -697,17 +777,17 @@ static struct win_syscall { { "UnmapViewOfFile", (SYSCALL)0, 0 }, #endif -#define osUnmapViewOfFile ((BOOL(WINAPI*)(LPCVOID))aSyscall[56].pCurrent) +#define osUnmapViewOfFile ((BOOL(WINAPI*)(LPCVOID))aSyscall[57].pCurrent) { "WideCharToMultiByte", (SYSCALL)WideCharToMultiByte, 0 }, #define osWideCharToMultiByte ((int(WINAPI*)(UINT,DWORD,LPCWSTR,int,LPSTR,int, \ - LPCSTR,LPBOOL))aSyscall[57].pCurrent) + LPCSTR,LPBOOL))aSyscall[58].pCurrent) { "WriteFile", (SYSCALL)WriteFile, 0 }, #define osWriteFile ((BOOL(WINAPI*)(HANDLE,LPCVOID,DWORD,LPDWORD, \ - LPOVERLAPPED))aSyscall[58].pCurrent) + LPOVERLAPPED))aSyscall[59].pCurrent) #if SQLITE_OS_WINRT { "CreateEventExW", (SYSCALL)CreateEventExW, 0 }, @@ -716,7 +796,7 @@ static struct win_syscall { #endif #define osCreateEventExW ((HANDLE(WINAPI*)(LPSECURITY_ATTRIBUTES,LPCWSTR, \ - DWORD,DWORD))aSyscall[59].pCurrent) + DWORD,DWORD))aSyscall[60].pCurrent) #if !SQLITE_OS_WINRT { "WaitForSingleObject", (SYSCALL)WaitForSingleObject, 0 }, @@ -725,7 +805,7 @@ static struct win_syscall { #endif #define osWaitForSingleObject ((DWORD(WINAPI*)(HANDLE, \ - DWORD))aSyscall[60].pCurrent) + DWORD))aSyscall[61].pCurrent) #if SQLITE_OS_WINRT { "WaitForSingleObjectEx", (SYSCALL)WaitForSingleObjectEx, 0 }, @@ -734,7 +814,7 @@ static struct win_syscall { #endif #define osWaitForSingleObjectEx ((DWORD(WINAPI*)(HANDLE,DWORD, \ - BOOL))aSyscall[61].pCurrent) + BOOL))aSyscall[62].pCurrent) #if SQLITE_OS_WINRT { "SetFilePointerEx", (SYSCALL)SetFilePointerEx, 0 }, @@ -743,7 +823,7 @@ static struct win_syscall { #endif #define osSetFilePointerEx ((BOOL(WINAPI*)(HANDLE,LARGE_INTEGER, \ - PLARGE_INTEGER,DWORD))aSyscall[62].pCurrent) + PLARGE_INTEGER,DWORD))aSyscall[63].pCurrent) #if SQLITE_OS_WINRT { "GetFileInformationByHandleEx", (SYSCALL)GetFileInformationByHandleEx, 0 }, @@ -752,7 +832,7 @@ static struct win_syscall { #endif #define osGetFileInformationByHandleEx ((BOOL(WINAPI*)(HANDLE, \ - FILE_INFO_BY_HANDLE_CLASS,LPVOID,DWORD))aSyscall[63].pCurrent) + FILE_INFO_BY_HANDLE_CLASS,LPVOID,DWORD))aSyscall[64].pCurrent) #if SQLITE_OS_WINRT && !defined(SQLITE_OMIT_WAL) { "MapViewOfFileFromApp", (SYSCALL)MapViewOfFileFromApp, 0 }, @@ -761,7 +841,7 @@ static struct win_syscall { #endif #define osMapViewOfFileFromApp ((LPVOID(WINAPI*)(HANDLE,ULONG,ULONG64, \ - SIZE_T))aSyscall[64].pCurrent) + SIZE_T))aSyscall[65].pCurrent) #if SQLITE_OS_WINRT { "CreateFile2", (SYSCALL)CreateFile2, 0 }, @@ -770,16 +850,16 @@ static struct win_syscall { #endif #define osCreateFile2 ((HANDLE(WINAPI*)(LPCWSTR,DWORD,DWORD,DWORD, \ - LPCREATEFILE2_EXTENDED_PARAMETERS))aSyscall[65].pCurrent) + LPCREATEFILE2_EXTENDED_PARAMETERS))aSyscall[66].pCurrent) -#if SQLITE_OS_WINRT +#if SQLITE_OS_WINRT && !defined(SQLITE_OMIT_LOAD_EXTENSION) { "LoadPackagedLibrary", (SYSCALL)LoadPackagedLibrary, 0 }, #else { "LoadPackagedLibrary", (SYSCALL)0, 0 }, #endif #define osLoadPackagedLibrary ((HMODULE(WINAPI*)(LPCWSTR, \ - DWORD))aSyscall[66].pCurrent) + DWORD))aSyscall[67].pCurrent) #if SQLITE_OS_WINRT { "GetTickCount64", (SYSCALL)GetTickCount64, 0 }, @@ -787,7 +867,7 @@ static struct win_syscall { { "GetTickCount64", (SYSCALL)0, 0 }, #endif -#define osGetTickCount64 ((ULONGLONG(WINAPI*)(VOID))aSyscall[67].pCurrent) +#define osGetTickCount64 ((ULONGLONG(WINAPI*)(VOID))aSyscall[68].pCurrent) #if SQLITE_OS_WINRT { "GetNativeSystemInfo", (SYSCALL)GetNativeSystemInfo, 0 }, @@ -796,7 +876,7 @@ static struct win_syscall { #endif #define osGetNativeSystemInfo ((VOID(WINAPI*)( \ - LPSYSTEM_INFO))aSyscall[68].pCurrent) + LPSYSTEM_INFO))aSyscall[69].pCurrent) #if defined(SQLITE_WIN32_HAS_ANSI) { "OutputDebugStringA", (SYSCALL)OutputDebugStringA, 0 }, @@ -804,7 +884,7 @@ static struct win_syscall { { "OutputDebugStringA", (SYSCALL)0, 0 }, #endif -#define osOutputDebugStringA ((VOID(WINAPI*)(LPCSTR))aSyscall[69].pCurrent) +#define osOutputDebugStringA ((VOID(WINAPI*)(LPCSTR))aSyscall[70].pCurrent) #if defined(SQLITE_WIN32_HAS_WIDE) { "OutputDebugStringW", (SYSCALL)OutputDebugStringW, 0 }, @@ -812,11 +892,11 @@ static struct win_syscall { { "OutputDebugStringW", (SYSCALL)0, 0 }, #endif -#define osOutputDebugStringW ((VOID(WINAPI*)(LPCWSTR))aSyscall[70].pCurrent) +#define osOutputDebugStringW ((VOID(WINAPI*)(LPCWSTR))aSyscall[71].pCurrent) { "GetProcessHeap", (SYSCALL)GetProcessHeap, 0 }, -#define osGetProcessHeap ((HANDLE(WINAPI*)(VOID))aSyscall[71].pCurrent) +#define osGetProcessHeap ((HANDLE(WINAPI*)(VOID))aSyscall[72].pCurrent) #if SQLITE_OS_WINRT && !defined(SQLITE_OMIT_WAL) { "CreateFileMappingFromApp", (SYSCALL)CreateFileMappingFromApp, 0 }, @@ -825,7 +905,7 @@ static struct win_syscall { #endif #define osCreateFileMappingFromApp ((HANDLE(WINAPI*)(HANDLE, \ - LPSECURITY_ATTRIBUTES,ULONG,ULONG64,LPCWSTR))aSyscall[72].pCurrent) + LPSECURITY_ATTRIBUTES,ULONG,ULONG64,LPCWSTR))aSyscall[73].pCurrent) }; /* End of the overrideable system calls */ @@ -917,7 +997,7 @@ static const char *winNextSystemCall(sqlite3_vfs *p, const char *zName){ ** (if available). */ -void sqlite3_win32_write_debug(char *zBuf, int nBuf){ +void sqlite3_win32_write_debug(const char *zBuf, int nBuf){ char zDbgBuf[SQLITE_WIN32_DBG_BUF_SIZE]; int nMin = MIN(nBuf, (SQLITE_WIN32_DBG_BUF_SIZE - 1)); /* may be negative. */ if( nMin<-1 ) nMin = -1; /* all negative values become -1. */ @@ -983,6 +1063,8 @@ void sqlite3_win32_sleep(DWORD milliseconds){ */ #if SQLITE_OS_WINCE || SQLITE_OS_WINRT # define isNT() (1) +#elif !defined(SQLITE_WIN32_HAS_WIDE) +# define isNT() (0) #else static int isNT(void){ if( sqlite3_os_type==0 ){ @@ -993,7 +1075,7 @@ void sqlite3_win32_sleep(DWORD milliseconds){ } return sqlite3_os_type==2; } -#endif /* SQLITE_OS_WINCE */ +#endif #ifdef SQLITE_WIN32_MALLOC /* @@ -1203,7 +1285,7 @@ static LPWSTR utf8ToUnicode(const char *zFilename){ if( nChar==0 ){ return 0; } - zWideFilename = sqlite3_malloc( nChar*sizeof(zWideFilename[0]) ); + zWideFilename = sqlite3MallocZero( nChar*sizeof(zWideFilename[0]) ); if( zWideFilename==0 ){ return 0; } @@ -1228,7 +1310,7 @@ static char *unicodeToUtf8(LPCWSTR zWideFilename){ if( nByte == 0 ){ return 0; } - zFilename = sqlite3_malloc( nByte ); + zFilename = sqlite3MallocZero( nByte ); if( zFilename==0 ){ return 0; } @@ -1258,7 +1340,7 @@ static LPWSTR mbcsToUnicode(const char *zFilename){ if( nByte==0 ){ return 0; } - zMbcsFilename = sqlite3_malloc( nByte*sizeof(zMbcsFilename[0]) ); + zMbcsFilename = sqlite3MallocZero( nByte*sizeof(zMbcsFilename[0]) ); if( zMbcsFilename==0 ){ return 0; } @@ -1287,7 +1369,7 @@ static char *unicodeToMbcs(LPCWSTR zWideFilename){ if( nByte == 0 ){ return 0; } - zFilename = sqlite3_malloc( nByte ); + zFilename = sqlite3MallocZero( nByte ); if( zFilename==0 ){ return 0; } @@ -1441,7 +1523,7 @@ static int getLastErrorMsg(DWORD lastErrno, int nBuf, char *zBuf){ } #endif if( 0 == dwLen ){ - sqlite3_snprintf(nBuf, zBuf, "OsError 0x%x (%u)", lastErrno, lastErrno); + sqlite3_snprintf(nBuf, zBuf, "OsError 0x%lx (%lu)", lastErrno, lastErrno); }else{ /* copy a maximum of nBuf chars to output buffer */ sqlite3_snprintf(nBuf, zBuf, "%s", zOut); @@ -1484,7 +1566,7 @@ static int winLogErrorAtLine( for(i=0; zMsg[i] && zMsg[i]!='\r' && zMsg[i]!='\n'; i++){} zMsg[i] = 0; sqlite3_log(errcode, - "os_win.c:%d: (%d) %s(%s) - %s", + "os_win.c:%d: (%lu) %s(%s) - %s", iLine, lastErrno, zFunc, zPath, zMsg ); @@ -1548,9 +1630,10 @@ static void logIoerr(int nRetry){ /************************************************************************* ** This section contains code for WinCE only. */ +#if !defined(SQLITE_MSVC_LOCALTIME_API) || !SQLITE_MSVC_LOCALTIME_API /* -** Windows CE does not have a localtime() function. So create a -** substitute. +** The MSVC CRT on Windows CE may not have a localtime() function. So +** create a substitute. */ #include <time.h> struct tm *__cdecl localtime(const time_t *t) @@ -1574,6 +1657,7 @@ struct tm *__cdecl localtime(const time_t *t) y.tm_sec = pTm.wSecond; return &y; } +#endif #define HANDLE_TO_WINFILE(a) (winFile*)&((char*)a)[-(int)offsetof(winFile,h)] @@ -1595,15 +1679,17 @@ static void winceMutexAcquire(HANDLE h){ ** Create the mutex and shared memory used for locking in the file ** descriptor pFile */ -static BOOL winceCreateLock(const char *zFilename, winFile *pFile){ +static int winceCreateLock(const char *zFilename, winFile *pFile){ LPWSTR zTok; LPWSTR zName; + DWORD lastErrno; + BOOL bLogged = FALSE; BOOL bInit = TRUE; zName = utf8ToUnicode(zFilename); if( zName==0 ){ /* out of memory */ - return FALSE; + return SQLITE_IOERR_NOMEM; } /* Initialize the local lockdata */ @@ -1620,9 +1706,10 @@ static BOOL winceCreateLock(const char *zFilename, winFile *pFile){ pFile->hMutex = osCreateMutexW(NULL, FALSE, zName); if (!pFile->hMutex){ pFile->lastErrno = osGetLastError(); - winLogError(SQLITE_ERROR, pFile->lastErrno, "winceCreateLock1", zFilename); + winLogError(SQLITE_IOERR, pFile->lastErrno, + "winceCreateLock1", zFilename); sqlite3_free(zName); - return FALSE; + return SQLITE_IOERR; } /* Acquire the mutex before continuing */ @@ -1639,41 +1726,49 @@ static BOOL winceCreateLock(const char *zFilename, winFile *pFile){ /* Set a flag that indicates we're the first to create the memory so it ** must be zero-initialized */ - if (osGetLastError() == ERROR_ALREADY_EXISTS){ + lastErrno = osGetLastError(); + if (lastErrno == ERROR_ALREADY_EXISTS){ bInit = FALSE; } sqlite3_free(zName); /* If we succeeded in making the shared memory handle, map it. */ - if (pFile->hShared){ + if( pFile->hShared ){ pFile->shared = (winceLock*)osMapViewOfFile(pFile->hShared, FILE_MAP_READ|FILE_MAP_WRITE, 0, 0, sizeof(winceLock)); /* If mapping failed, close the shared memory handle and erase it */ - if (!pFile->shared){ + if( !pFile->shared ){ pFile->lastErrno = osGetLastError(); - winLogError(SQLITE_ERROR, pFile->lastErrno, - "winceCreateLock2", zFilename); + winLogError(SQLITE_IOERR, pFile->lastErrno, + "winceCreateLock2", zFilename); + bLogged = TRUE; osCloseHandle(pFile->hShared); pFile->hShared = NULL; } } /* If shared memory could not be created, then close the mutex and fail */ - if (pFile->hShared == NULL){ + if( pFile->hShared==NULL ){ + if( !bLogged ){ + pFile->lastErrno = lastErrno; + winLogError(SQLITE_IOERR, pFile->lastErrno, + "winceCreateLock3", zFilename); + bLogged = TRUE; + } winceMutexRelease(pFile->hMutex); osCloseHandle(pFile->hMutex); pFile->hMutex = NULL; - return FALSE; + return SQLITE_IOERR; } /* Initialize the shared memory if we're supposed to */ - if (bInit) { + if( bInit ){ memset(pFile->shared, 0, sizeof(winceLock)); } winceMutexRelease(pFile->hMutex); - return TRUE; + return SQLITE_OK; } /* @@ -1752,7 +1847,8 @@ static BOOL winceLockFile( } /* Want a pending lock? */ - else if (dwFileOffsetLow == (DWORD)PENDING_BYTE && nNumberOfBytesToLockLow == 1){ + else if (dwFileOffsetLow == (DWORD)PENDING_BYTE + && nNumberOfBytesToLockLow == 1){ /* If no pending lock has been acquired, then acquire it */ if (pFile->shared->bPending == 0) { pFile->shared->bPending = TRUE; @@ -1762,7 +1858,8 @@ static BOOL winceLockFile( } /* Want a reserved lock? */ - else if (dwFileOffsetLow == (DWORD)RESERVED_BYTE && nNumberOfBytesToLockLow == 1){ + else if (dwFileOffsetLow == (DWORD)RESERVED_BYTE + && nNumberOfBytesToLockLow == 1){ if (pFile->shared->bReserved == 0) { pFile->shared->bReserved = TRUE; pFile->local.bReserved = TRUE; @@ -1805,7 +1902,8 @@ static BOOL winceUnlockFile( /* Did we just have a reader lock? */ else if (pFile->local.nReaders){ - assert(nNumberOfBytesToUnlockLow == (DWORD)SHARED_SIZE || nNumberOfBytesToUnlockLow == 1); + assert(nNumberOfBytesToUnlockLow == (DWORD)SHARED_SIZE + || nNumberOfBytesToUnlockLow == 1); pFile->local.nReaders --; if (pFile->local.nReaders == 0) { @@ -1816,7 +1914,8 @@ static BOOL winceUnlockFile( } /* Releasing a pending lock */ - else if (dwFileOffsetLow == (DWORD)PENDING_BYTE && nNumberOfBytesToUnlockLow == 1){ + else if (dwFileOffsetLow == (DWORD)PENDING_BYTE + && nNumberOfBytesToUnlockLow == 1){ if (pFile->local.bPending){ pFile->local.bPending = FALSE; pFile->shared->bPending = FALSE; @@ -1824,7 +1923,8 @@ static BOOL winceUnlockFile( } } /* Releasing a reserved lock */ - else if (dwFileOffsetLow == (DWORD)RESERVED_BYTE && nNumberOfBytesToUnlockLow == 1){ + else if (dwFileOffsetLow == (DWORD)RESERVED_BYTE + && nNumberOfBytesToUnlockLow == 1){ if (pFile->local.bReserved) { pFile->local.bReserved = FALSE; pFile->shared->bReserved = FALSE; @@ -1927,6 +2027,8 @@ static int seekWinFile(winFile *pFile, sqlite3_int64 iOffset){ DWORD dwRet; /* Value returned by SetFilePointer() */ DWORD lastErrno; /* Value returned by GetLastError() */ + OSTRACE(("SEEK file=%p, offset=%lld\n", pFile->h, iOffset)); + upperBits = (LONG)((iOffset>>32) & 0x7fffffff); lowerBits = (LONG)(iOffset & 0xffffffff); @@ -1934,7 +2036,7 @@ static int seekWinFile(winFile *pFile, sqlite3_int64 iOffset){ ** containing the lower 32-bits of the new file-offset. Or, if it fails, ** it returns INVALID_SET_FILE_POINTER. However according to MSDN, ** INVALID_SET_FILE_POINTER may also be a valid new offset. So to determine - ** whether an error has actually occured, it is also necessary to call + ** whether an error has actually occurred, it is also necessary to call ** GetLastError(). */ dwRet = osSetFilePointer(pFile->h, lowerBits, &upperBits, FILE_BEGIN); @@ -1944,9 +2046,11 @@ static int seekWinFile(winFile *pFile, sqlite3_int64 iOffset){ pFile->lastErrno = lastErrno; winLogError(SQLITE_IOERR_SEEK, pFile->lastErrno, "seekWinFile", pFile->zPath); + OSTRACE(("SEEK file=%p, rc=SQLITE_IOERR_SEEK\n", pFile->h)); return 1; } + OSTRACE(("SEEK file=%p, rc=SQLITE_OK\n", pFile->h)); return 0; #else /* @@ -1963,13 +2067,20 @@ static int seekWinFile(winFile *pFile, sqlite3_int64 iOffset){ pFile->lastErrno = osGetLastError(); winLogError(SQLITE_IOERR_SEEK, pFile->lastErrno, "seekWinFile", pFile->zPath); + OSTRACE(("SEEK file=%p, rc=SQLITE_IOERR_SEEK\n", pFile->h)); return 1; } + OSTRACE(("SEEK file=%p, rc=SQLITE_OK\n", pFile->h)); return 0; #endif } +#if SQLITE_MAX_MMAP_SIZE>0 +/* Forward references to VFS methods */ +static int winUnmapfile(winFile*); +#endif + /* ** Close a file. ** @@ -1989,7 +2100,14 @@ static int winClose(sqlite3_file *id){ #ifndef SQLITE_OMIT_WAL assert( pFile->pShm==0 ); #endif - OSTRACE(("CLOSE %d\n", pFile->h)); + assert( pFile->h!=NULL && pFile->h!=INVALID_HANDLE_VALUE ); + OSTRACE(("CLOSE file=%p\n", pFile->h)); + +#if SQLITE_MAX_MMAP_SIZE>0 + rc = winUnmapfile(pFile); + if( rc!=SQLITE_OK ) return rc; +#endif + do{ rc = osCloseHandle(pFile->h); /* SimulateIOError( rc=0; cnt=MX_CLOSE_ATTEMPT; ); */ @@ -2009,11 +2127,11 @@ static int winClose(sqlite3_file *id){ sqlite3_free(pFile->zDeleteOnClose); } #endif - OSTRACE(("CLOSE %d %s\n", pFile->h, rc ? "ok" : "failed")); if( rc ){ pFile->h = NULL; } OpenCounter(-1); + OSTRACE(("CLOSE file=%p, rc=%s\n", pFile->h, rc ? "ok" : "failed")); return rc ? SQLITE_OK : winLogError(SQLITE_IOERR_CLOSE, osGetLastError(), "winClose", pFile->zPath); @@ -2038,11 +2156,33 @@ static int winRead( int nRetry = 0; /* Number of retrys */ assert( id!=0 ); + assert( amt>0 ); + assert( offset>=0 ); SimulateIOError(return SQLITE_IOERR_READ); - OSTRACE(("READ %d lock=%d\n", pFile->h, pFile->locktype)); + OSTRACE(("READ file=%p, buffer=%p, amount=%d, offset=%lld, lock=%d\n", + pFile->h, pBuf, amt, offset, pFile->locktype)); + +#if SQLITE_MAX_MMAP_SIZE>0 + /* Deal with as much of this read request as possible by transfering + ** data from the memory mapping using memcpy(). */ + if( offset<pFile->mmapSize ){ + if( offset+amt <= pFile->mmapSize ){ + memcpy(pBuf, &((u8 *)(pFile->pMapRegion))[offset], amt); + OSTRACE(("READ-MMAP file=%p, rc=SQLITE_OK\n", pFile->h)); + return SQLITE_OK; + }else{ + int nCopy = (int)(pFile->mmapSize - offset); + memcpy(pBuf, &((u8 *)(pFile->pMapRegion))[offset], nCopy); + pBuf = &((u8 *)pBuf)[nCopy]; + amt -= nCopy; + offset += nCopy; + } + } +#endif #if SQLITE_OS_WINCE if( seekWinFile(pFile, offset) ){ + OSTRACE(("READ file=%p, rc=SQLITE_FULL\n", pFile->h)); return SQLITE_FULL; } while( !osReadFile(pFile->h, pBuf, amt, &nRead, 0) ){ @@ -2056,6 +2196,7 @@ static int winRead( DWORD lastErrno; if( retryIoerr(&nRetry, &lastErrno) ) continue; pFile->lastErrno = lastErrno; + OSTRACE(("READ file=%p, rc=SQLITE_IOERR_READ\n", pFile->h)); return winLogError(SQLITE_IOERR_READ, pFile->lastErrno, "winRead", pFile->zPath); } @@ -2063,9 +2204,11 @@ static int winRead( if( nRead<(DWORD)amt ){ /* Unread parts of the buffer must be zero-filled */ memset(&((char*)pBuf)[nRead], 0, amt-nRead); + OSTRACE(("READ file=%p, rc=SQLITE_IOERR_SHORT_READ\n", pFile->h)); return SQLITE_IOERR_SHORT_READ; } + OSTRACE(("READ file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } @@ -2079,7 +2222,7 @@ static int winWrite( int amt, /* Number of bytes to write */ sqlite3_int64 offset /* Offset into the file to begin writing at */ ){ - int rc = 0; /* True if error has occured, else false */ + int rc = 0; /* True if error has occurred, else false */ winFile *pFile = (winFile*)id; /* File handle */ int nRetry = 0; /* Number of retries */ @@ -2088,7 +2231,26 @@ static int winWrite( SimulateIOError(return SQLITE_IOERR_WRITE); SimulateDiskfullError(return SQLITE_FULL); - OSTRACE(("WRITE %d lock=%d\n", pFile->h, pFile->locktype)); + OSTRACE(("WRITE file=%p, buffer=%p, amount=%d, offset=%lld, lock=%d\n", + pFile->h, pBuf, amt, offset, pFile->locktype)); + +#if SQLITE_MAX_MMAP_SIZE>0 + /* Deal with as much of this write request as possible by transfering + ** data from the memory mapping using memcpy(). */ + if( offset<pFile->mmapSize ){ + if( offset+amt <= pFile->mmapSize ){ + memcpy(&((u8 *)(pFile->pMapRegion))[offset], pBuf, amt); + OSTRACE(("WRITE-MMAP file=%p, rc=SQLITE_OK\n", pFile->h)); + return SQLITE_OK; + }else{ + int nCopy = (int)(pFile->mmapSize - offset); + memcpy(&((u8 *)(pFile->pMapRegion))[offset], pBuf, nCopy); + pBuf = &((u8 *)pBuf)[nCopy]; + amt -= nCopy; + offset += nCopy; + } + } +#endif #if SQLITE_OS_WINCE rc = seekWinFile(pFile, offset); @@ -2119,7 +2281,8 @@ static int winWrite( if( retryIoerr(&nRetry, &lastErrno) ) continue; break; } - if( nWrite<=0 ){ + assert( nWrite==0 || nWrite<=(DWORD)nRem ); + if( nWrite==0 || nWrite>(DWORD)nRem ){ lastErrno = osGetLastError(); break; } @@ -2140,13 +2303,16 @@ static int winWrite( if( rc ){ if( ( pFile->lastErrno==ERROR_HANDLE_DISK_FULL ) || ( pFile->lastErrno==ERROR_DISK_FULL )){ + OSTRACE(("WRITE file=%p, rc=SQLITE_FULL\n", pFile->h)); return SQLITE_FULL; } + OSTRACE(("WRITE file=%p, rc=SQLITE_IOERR_WRITE\n", pFile->h)); return winLogError(SQLITE_IOERR_WRITE, pFile->lastErrno, "winWrite", pFile->zPath); }else{ logIoerr(nRetry); } + OSTRACE(("WRITE file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } @@ -2156,11 +2322,12 @@ static int winWrite( static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ winFile *pFile = (winFile*)id; /* File handle object */ int rc = SQLITE_OK; /* Return code for this function */ + DWORD lastErrno; assert( pFile ); - - OSTRACE(("TRUNCATE %d %lld\n", pFile->h, nByte)); SimulateIOError(return SQLITE_IOERR_TRUNCATE); + OSTRACE(("TRUNCATE file=%p, size=%lld, lock=%d\n", + pFile->h, nByte, pFile->locktype)); /* If the user has configured a chunk-size for this file, truncate the ** file so that it consists of an integer number of chunks (i.e. the @@ -2174,14 +2341,25 @@ static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ /* SetEndOfFile() returns non-zero when successful, or zero when it fails. */ if( seekWinFile(pFile, nByte) ){ rc = winLogError(SQLITE_IOERR_TRUNCATE, pFile->lastErrno, - "winTruncate1", pFile->zPath); - }else if( 0==osSetEndOfFile(pFile->h) ){ - pFile->lastErrno = osGetLastError(); + "winTruncate1", pFile->zPath); + }else if( 0==osSetEndOfFile(pFile->h) && + ((lastErrno = osGetLastError())!=ERROR_USER_MAPPED_FILE) ){ + pFile->lastErrno = lastErrno; rc = winLogError(SQLITE_IOERR_TRUNCATE, pFile->lastErrno, - "winTruncate2", pFile->zPath); + "winTruncate2", pFile->zPath); } - OSTRACE(("TRUNCATE %d %lld %s\n", pFile->h, nByte, rc ? "failed" : "ok")); +#if SQLITE_MAX_MMAP_SIZE>0 + /* If the file was truncated to a size smaller than the currently + ** mapped region, reduce the effective mapping size as well. SQLite will + ** use read() and write() to access data beyond this point from now on. + */ + if( pFile->pMapRegion && nByte<pFile->mmapSize ){ + pFile->mmapSize = nByte; + } +#endif + + OSTRACE(("TRUNCATE file=%p, rc=%s\n", pFile->h, sqlite3ErrName(rc))); return rc; } @@ -2221,13 +2399,14 @@ static int winSync(sqlite3_file *id, int flags){ || (flags&0x0F)==SQLITE_SYNC_FULL ); - OSTRACE(("SYNC %d lock=%d\n", pFile->h, pFile->locktype)); - /* Unix cannot, but some systems may return SQLITE_FULL from here. This ** line is to test that doing so does not cause any problems. */ SimulateDiskfullError( return SQLITE_FULL ); + OSTRACE(("SYNC file=%p, flags=%x, lock=%d\n", + pFile->h, flags, pFile->locktype)); + #ifndef SQLITE_TEST UNUSED_PARAMETER(flags); #else @@ -2246,9 +2425,11 @@ static int winSync(sqlite3_file *id, int flags){ rc = osFlushFileBuffers(pFile->h); SimulateIOError( rc=FALSE ); if( rc ){ + OSTRACE(("SYNC file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; }else{ pFile->lastErrno = osGetLastError(); + OSTRACE(("SYNC file=%p, rc=SQLITE_IOERR_FSYNC\n", pFile->h)); return winLogError(SQLITE_IOERR_FSYNC, pFile->lastErrno, "winSync", pFile->zPath); } @@ -2263,7 +2444,10 @@ static int winFileSize(sqlite3_file *id, sqlite3_int64 *pSize){ int rc = SQLITE_OK; assert( id!=0 ); + assert( pSize!=0 ); SimulateIOError(return SQLITE_IOERR_FSTAT); + OSTRACE(("SIZE file=%p, pSize=%p\n", pFile->h, pSize)); + #if SQLITE_OS_WINRT { FILE_STANDARD_INFO info; @@ -2292,6 +2476,8 @@ static int winFileSize(sqlite3_file *id, sqlite3_int64 *pSize){ } } #endif + OSTRACE(("SIZE file=%p, pSize=%p, *pSize=%lld, rc=%s\n", + pFile->h, pSize, *pSize, sqlite3ErrName(rc))); return rc; } @@ -2333,6 +2519,7 @@ static int winFileSize(sqlite3_file *id, sqlite3_int64 *pSize){ */ static int getReadLock(winFile *pFile){ int res; + OSTRACE(("READ-LOCK file=%p, lock=%d\n", pFile->h, pFile->locktype)); if( isNT() ){ #if SQLITE_OS_WINCE /* @@ -2358,6 +2545,7 @@ static int getReadLock(winFile *pFile){ pFile->lastErrno = osGetLastError(); /* No need to log a failure to lock */ } + OSTRACE(("READ-LOCK file=%p, rc=%s\n", pFile->h, sqlite3ErrName(res))); return res; } @@ -2367,6 +2555,7 @@ static int getReadLock(winFile *pFile){ static int unlockReadLock(winFile *pFile){ int res; DWORD lastErrno; + OSTRACE(("READ-UNLOCK file=%p, lock=%d\n", pFile->h, pFile->locktype)); if( isNT() ){ res = winUnlockFile(&pFile->h, SHARED_FIRST, 0, SHARED_SIZE, 0); } @@ -2380,6 +2569,7 @@ static int unlockReadLock(winFile *pFile){ winLogError(SQLITE_IOERR_UNLOCK, pFile->lastErrno, "unlockReadLock", pFile->zPath); } + OSTRACE(("READ-UNLOCK file=%p, rc=%s\n", pFile->h, sqlite3ErrName(res))); return res; } @@ -2418,14 +2608,15 @@ static int winLock(sqlite3_file *id, int locktype){ DWORD lastErrno = NO_ERROR; assert( id!=0 ); - OSTRACE(("LOCK %d %d was %d(%d)\n", - pFile->h, locktype, pFile->locktype, pFile->sharedLockByte)); + OSTRACE(("LOCK file=%p, oldLock=%d(%d), newLock=%d\n", + pFile->h, pFile->locktype, pFile->sharedLockByte, locktype)); /* If there is already a lock of this type or more restrictive on the ** OsFile, do nothing. Don't use the end_lock: exit path, as ** sqlite3OsEnterMutex() hasn't been called yet. */ if( pFile->locktype>=locktype ){ + OSTRACE(("LOCK-HELD file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } @@ -2453,7 +2644,8 @@ static int winLock(sqlite3_file *id, int locktype){ ** If you are using this code as a model for alternative VFSes, do not ** copy this retry logic. It is a hack intended for Windows only. */ - OSTRACE(("could not get a PENDING lock. cnt=%d\n", cnt)); + OSTRACE(("LOCK-PENDING-FAIL file=%p, count=%d, rc=%s\n", + pFile->h, cnt, sqlite3ErrName(res))); if( cnt ) sqlite3_win32_sleep(1); } gotPendingLock = res; @@ -2498,14 +2690,12 @@ static int winLock(sqlite3_file *id, int locktype){ if( locktype==EXCLUSIVE_LOCK && res ){ assert( pFile->locktype>=SHARED_LOCK ); res = unlockReadLock(pFile); - OSTRACE(("unreadlock = %d\n", res)); res = winLockFile(&pFile->h, SQLITE_LOCKFILE_FLAGS, SHARED_FIRST, 0, SHARED_SIZE, 0); if( res ){ newLocktype = EXCLUSIVE_LOCK; }else{ lastErrno = osGetLastError(); - OSTRACE(("error-code = %d\n", lastErrno)); getReadLock(pFile); } } @@ -2523,12 +2713,14 @@ static int winLock(sqlite3_file *id, int locktype){ if( res ){ rc = SQLITE_OK; }else{ - OSTRACE(("LOCK FAILED %d trying for %d but got %d\n", pFile->h, - locktype, newLocktype)); + OSTRACE(("LOCK-FAIL file=%p, wanted=%d, got=%d\n", + pFile->h, locktype, newLocktype)); pFile->lastErrno = lastErrno; rc = SQLITE_BUSY; } pFile->locktype = (u8)newLocktype; + OSTRACE(("LOCK file=%p, lock=%d, rc=%s\n", + pFile->h, pFile->locktype, sqlite3ErrName(rc))); return rc; } @@ -2542,20 +2734,23 @@ static int winCheckReservedLock(sqlite3_file *id, int *pResOut){ winFile *pFile = (winFile*)id; SimulateIOError( return SQLITE_IOERR_CHECKRESERVEDLOCK; ); + OSTRACE(("TEST-WR-LOCK file=%p, pResOut=%p\n", pFile->h, pResOut)); assert( id!=0 ); if( pFile->locktype>=RESERVED_LOCK ){ rc = 1; - OSTRACE(("TEST WR-LOCK %d %d (local)\n", pFile->h, rc)); + OSTRACE(("TEST-WR-LOCK file=%p, rc=%d (local)\n", pFile->h, rc)); }else{ - rc = winLockFile(&pFile->h, SQLITE_LOCKFILE_FLAGS, RESERVED_BYTE, 0, 1, 0); + rc = winLockFile(&pFile->h, SQLITE_LOCKFILEEX_FLAGS,RESERVED_BYTE, 0, 1, 0); if( rc ){ winUnlockFile(&pFile->h, RESERVED_BYTE, 0, 1, 0); } rc = !rc; - OSTRACE(("TEST WR-LOCK %d %d (remote)\n", pFile->h, rc)); + OSTRACE(("TEST-WR-LOCK file=%p, rc=%d (remote)\n", pFile->h, rc)); } *pResOut = rc; + OSTRACE(("TEST-WR-LOCK file=%p, pResOut=%p, *pResOut=%d, rc=SQLITE_OK\n", + pFile->h, pResOut, *pResOut)); return SQLITE_OK; } @@ -2576,8 +2771,8 @@ static int winUnlock(sqlite3_file *id, int locktype){ int rc = SQLITE_OK; assert( pFile!=0 ); assert( locktype<=SHARED_LOCK ); - OSTRACE(("UNLOCK %d to %d was %d(%d)\n", pFile->h, locktype, - pFile->locktype, pFile->sharedLockByte)); + OSTRACE(("UNLOCK file=%p, oldLock=%d(%d), newLock=%d\n", + pFile->h, pFile->locktype, pFile->sharedLockByte, locktype)); type = pFile->locktype; if( type>=EXCLUSIVE_LOCK ){ winUnlockFile(&pFile->h, SHARED_FIRST, 0, SHARED_SIZE, 0); @@ -2598,6 +2793,8 @@ static int winUnlock(sqlite3_file *id, int locktype){ winUnlockFile(&pFile->h, PENDING_BYTE, 0, 1, 0); } pFile->locktype = (u8)locktype; + OSTRACE(("UNLOCK file=%p, lock=%d, rc=%s\n", + pFile->h, pFile->locktype, sqlite3ErrName(rc))); return rc; } @@ -2617,22 +2814,29 @@ static void winModeBit(winFile *pFile, unsigned char mask, int *pArg){ } } +/* Forward declaration */ +static int getTempname(int nBuf, char *zBuf); + /* ** Control and query of the open file handle. */ static int winFileControl(sqlite3_file *id, int op, void *pArg){ winFile *pFile = (winFile*)id; + OSTRACE(("FCNTL file=%p, op=%d, pArg=%p\n", pFile->h, op, pArg)); switch( op ){ case SQLITE_FCNTL_LOCKSTATE: { *(int*)pArg = pFile->locktype; + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } case SQLITE_LAST_ERRNO: { *(int*)pArg = (int)pFile->lastErrno; + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } case SQLITE_FCNTL_CHUNK_SIZE: { pFile->szChunk = *(int *)pArg; + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } case SQLITE_FCNTL_SIZE_HINT: { @@ -2647,20 +2851,25 @@ static int winFileControl(sqlite3_file *id, int op, void *pArg){ SimulateIOErrorBenign(0); } } + OSTRACE(("FCNTL file=%p, rc=%s\n", pFile->h, sqlite3ErrName(rc))); return rc; } + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } case SQLITE_FCNTL_PERSIST_WAL: { winModeBit(pFile, WINFILE_PERSIST_WAL, (int*)pArg); + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } case SQLITE_FCNTL_POWERSAFE_OVERWRITE: { winModeBit(pFile, WINFILE_PSOW, (int*)pArg); + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } case SQLITE_FCNTL_VFSNAME: { *(char**)pArg = sqlite3_mprintf("win32"); + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } case SQLITE_FCNTL_WIN32_AV_RETRY: { @@ -2675,9 +2884,32 @@ static int winFileControl(sqlite3_file *id, int op, void *pArg){ }else{ a[1] = win32IoerrRetryDelay; } + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } + case SQLITE_FCNTL_TEMPFILENAME: { + char *zTFile = sqlite3MallocZero( pFile->pVfs->mxPathname ); + if( zTFile ){ + getTempname(pFile->pVfs->mxPathname, zTFile); + *(char**)pArg = zTFile; + } + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); + return SQLITE_OK; + } +#if SQLITE_MAX_MMAP_SIZE>0 + case SQLITE_FCNTL_MMAP_SIZE: { + i64 newLimit = *(i64*)pArg; + if( newLimit>sqlite3GlobalConfig.mxMmap ){ + newLimit = sqlite3GlobalConfig.mxMmap; + } + *(i64*)pArg = pFile->mmapSizeMax; + if( newLimit>=0 ) pFile->mmapSizeMax = newLimit; + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); + return SQLITE_OK; + } +#endif } + OSTRACE(("FCNTL file=%p, rc=SQLITE_NOTFOUND\n", pFile->h)); return SQLITE_NOTFOUND; } @@ -2705,8 +2937,6 @@ static int winDeviceCharacteristics(sqlite3_file *id){ ((p->ctrlFlags & WINFILE_PSOW)?SQLITE_IOCAP_POWERSAFE_OVERWRITE:0); } -#ifndef SQLITE_OMIT_WAL - /* ** Windows will only let you create file view mappings ** on allocation size granularity boundaries. @@ -2715,6 +2945,8 @@ static int winDeviceCharacteristics(sqlite3_file *id){ */ SYSTEM_INFO winSysInfo; +#ifndef SQLITE_OMIT_WAL + /* ** Helper functions to obtain and relinquish the global mutex. The ** global mutex is used to protect the winLockInfo objects used by @@ -2838,6 +3070,9 @@ static int winShmSystemLock( /* Access to the winShmNode object is serialized by the caller */ assert( sqlite3_mutex_held(pFile->mutex) || pFile->nRef==0 ); + OSTRACE(("SHM-LOCK file=%p, lock=%d, offset=%d, size=%d\n", + pFile->hFile.h, lockType, ofst, nByte)); + /* Release/Acquire the system-level lock */ if( lockType==_SHM_UNLCK ){ rc = winUnlockFile(&pFile->hFile.h, ofst, 0, nByte, 0); @@ -2855,11 +3090,9 @@ static int winShmSystemLock( rc = SQLITE_BUSY; } - OSTRACE(("SHM-LOCK %d %s %s 0x%08lx\n", - pFile->hFile.h, - rc==SQLITE_OK ? "ok" : "failed", - lockType==_SHM_UNLCK ? "UnlockFileEx" : "LockFileEx", - pFile->lastErrno)); + OSTRACE(("SHM-LOCK file=%p, func=%s, errno=%lu, rc=%s\n", + pFile->hFile.h, (lockType == _SHM_UNLCK) ? "winUnlockFile" : + "winLockFile", pFile->lastErrno, sqlite3ErrName(rc))); return rc; } @@ -2879,6 +3112,8 @@ static void winShmPurge(sqlite3_vfs *pVfs, int deleteFlag){ winShmNode *p; BOOL bRc; assert( winShmMutexHeld() ); + OSTRACE(("SHM-PURGE pid=%lu, deleteFlag=%d\n", + osGetCurrentProcessId(), deleteFlag)); pp = &winShmNodeList; while( (p = *pp)!=0 ){ if( p->nRef==0 ){ @@ -2886,15 +3121,13 @@ static void winShmPurge(sqlite3_vfs *pVfs, int deleteFlag){ if( p->mutex ) sqlite3_mutex_free(p->mutex); for(i=0; i<p->nRegion; i++){ bRc = osUnmapViewOfFile(p->aRegion[i].pMap); - OSTRACE(("SHM-PURGE pid-%d unmap region=%d %s\n", - (int)osGetCurrentProcessId(), i, - bRc ? "ok" : "failed")); + OSTRACE(("SHM-PURGE-UNMAP pid=%lu, region=%d, rc=%s\n", + osGetCurrentProcessId(), i, bRc ? "ok" : "failed")); bRc = osCloseHandle(p->aRegion[i].hMap); - OSTRACE(("SHM-PURGE pid-%d close region=%d %s\n", - (int)osGetCurrentProcessId(), i, - bRc ? "ok" : "failed")); + OSTRACE(("SHM-PURGE-CLOSE pid=%lu, region=%d, rc=%s\n", + osGetCurrentProcessId(), i, bRc ? "ok" : "failed")); } - if( p->hFile.h != INVALID_HANDLE_VALUE ){ + if( p->hFile.h!=NULL && p->hFile.h!=INVALID_HANDLE_VALUE ){ SimulateIOErrorBenign(1); winClose((sqlite3_file *)&p->hFile); SimulateIOErrorBenign(0); @@ -2934,16 +3167,14 @@ static int winOpenSharedMemory(winFile *pDbFd){ /* Allocate space for the new sqlite3_shm object. Also speculatively ** allocate space for a new winShmNode and filename. */ - p = sqlite3_malloc( sizeof(*p) ); + p = sqlite3MallocZero( sizeof(*p) ); if( p==0 ) return SQLITE_IOERR_NOMEM; - memset(p, 0, sizeof(*p)); nName = sqlite3Strlen30(pDbFd->zPath); - pNew = sqlite3_malloc( sizeof(*pShmNode) + nName + 17 ); + pNew = sqlite3MallocZero( sizeof(*pShmNode) + nName + 17 ); if( pNew==0 ){ sqlite3_free(p); return SQLITE_IOERR_NOMEM; } - memset(pNew, 0, sizeof(*pNew) + nName + 17); pNew->zFilename = (char*)&pNew[1]; sqlite3_snprintf(nName+15, pNew->zFilename, "%s-shm", pDbFd->zPath); sqlite3FileSuffix3(pDbFd->zPath, pNew->zFilename); @@ -2976,7 +3207,7 @@ static int winOpenSharedMemory(winFile *pDbFd){ rc = winOpen(pDbFd->pVfs, pShmNode->zFilename, /* Name of the file (UTF-8) */ (sqlite3_file*)&pShmNode->hFile, /* File handle here */ - SQLITE_OPEN_WAL | SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, /* Mode flags */ + SQLITE_OPEN_WAL | SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0); if( SQLITE_OK!=rc ){ goto shm_open_err; @@ -3173,9 +3404,9 @@ static int winShmLock( } } sqlite3_mutex_leave(pShmNode->mutex); - OSTRACE(("SHM-LOCK shmid-%d, pid-%d got %03x,%03x %s\n", - p->id, (int)osGetCurrentProcessId(), p->sharedMask, p->exclMask, - rc ? "failed" : "ok")); + OSTRACE(("SHM-LOCK pid=%lu, id=%d, sharedMask=%03x, exclMask=%03x, rc=%s\n", + osGetCurrentProcessId(), p->id, p->sharedMask, p->exclMask, + sqlite3ErrName(rc))); return rc; } @@ -3280,20 +3511,24 @@ static int winShmMap( pShmNode->aRegion = apNew; while( pShmNode->nRegion<=iRegion ){ - HANDLE hMap; /* file-mapping handle */ + HANDLE hMap = NULL; /* file-mapping handle */ void *pMap = 0; /* Mapped memory region */ #if SQLITE_OS_WINRT hMap = osCreateFileMappingFromApp(pShmNode->hFile.h, NULL, PAGE_READWRITE, nByte, NULL ); -#else +#elif defined(SQLITE_WIN32_HAS_WIDE) hMap = osCreateFileMappingW(pShmNode->hFile.h, NULL, PAGE_READWRITE, 0, nByte, NULL ); +#elif defined(SQLITE_WIN32_HAS_ANSI) + hMap = osCreateFileMappingA(pShmNode->hFile.h, + NULL, PAGE_READWRITE, 0, nByte, NULL + ); #endif - OSTRACE(("SHM-MAP pid-%d create region=%d nbyte=%d %s\n", - (int)osGetCurrentProcessId(), pShmNode->nRegion, nByte, + OSTRACE(("SHM-MAP-CREATE pid=%lu, region=%d, size=%d, rc=%s\n", + osGetCurrentProcessId(), pShmNode->nRegion, nByte, hMap ? "ok" : "failed")); if( hMap ){ int iOffset = pShmNode->nRegion*szRegion; @@ -3307,8 +3542,8 @@ static int winShmMap( 0, iOffset - iOffsetShift, szRegion + iOffsetShift ); #endif - OSTRACE(("SHM-MAP pid-%d map region=%d offset=%d size=%d %s\n", - (int)osGetCurrentProcessId(), pShmNode->nRegion, iOffset, + OSTRACE(("SHM-MAP-MAP pid=%lu, region=%d, offset=%d, size=%d, rc=%s\n", + osGetCurrentProcessId(), pShmNode->nRegion, iOffset, szRegion, pMap ? "ok" : "failed")); } if( !pMap ){ @@ -3346,6 +3581,230 @@ shmpage_out: #endif /* #ifndef SQLITE_OMIT_WAL */ /* +** Cleans up the mapped region of the specified file, if any. +*/ +#if SQLITE_MAX_MMAP_SIZE>0 +static int winUnmapfile(winFile *pFile){ + assert( pFile!=0 ); + OSTRACE(("UNMAP-FILE pid=%lu, pFile=%p, hMap=%p, pMapRegion=%p, " + "mmapSize=%lld, mmapSizeActual=%lld, mmapSizeMax=%lld\n", + osGetCurrentProcessId(), pFile, pFile->hMap, pFile->pMapRegion, + pFile->mmapSize, pFile->mmapSizeActual, pFile->mmapSizeMax)); + if( pFile->pMapRegion ){ + if( !osUnmapViewOfFile(pFile->pMapRegion) ){ + pFile->lastErrno = osGetLastError(); + OSTRACE(("UNMAP-FILE pid=%lu, pFile=%p, pMapRegion=%p, " + "rc=SQLITE_IOERR_MMAP\n", osGetCurrentProcessId(), pFile, + pFile->pMapRegion)); + return winLogError(SQLITE_IOERR_MMAP, pFile->lastErrno, + "winUnmap1", pFile->zPath); + } + pFile->pMapRegion = 0; + pFile->mmapSize = 0; + pFile->mmapSizeActual = 0; + } + if( pFile->hMap!=NULL ){ + if( !osCloseHandle(pFile->hMap) ){ + pFile->lastErrno = osGetLastError(); + OSTRACE(("UNMAP-FILE pid=%lu, pFile=%p, hMap=%p, rc=SQLITE_IOERR_MMAP\n", + osGetCurrentProcessId(), pFile, pFile->hMap)); + return winLogError(SQLITE_IOERR_MMAP, pFile->lastErrno, + "winUnmap2", pFile->zPath); + } + pFile->hMap = NULL; + } + OSTRACE(("UNMAP-FILE pid=%lu, pFile=%p, rc=SQLITE_OK\n", + osGetCurrentProcessId(), pFile)); + return SQLITE_OK; +} + +/* +** Memory map or remap the file opened by file-descriptor pFd (if the file +** is already mapped, the existing mapping is replaced by the new). Or, if +** there already exists a mapping for this file, and there are still +** outstanding xFetch() references to it, this function is a no-op. +** +** If parameter nByte is non-negative, then it is the requested size of +** the mapping to create. Otherwise, if nByte is less than zero, then the +** requested size is the size of the file on disk. The actual size of the +** created mapping is either the requested size or the value configured +** using SQLITE_FCNTL_MMAP_SIZE, whichever is smaller. +** +** SQLITE_OK is returned if no error occurs (even if the mapping is not +** recreated as a result of outstanding references) or an SQLite error +** code otherwise. +*/ +static int winMapfile(winFile *pFd, sqlite3_int64 nByte){ + sqlite3_int64 nMap = nByte; + int rc; + + assert( nMap>=0 || pFd->nFetchOut==0 ); + OSTRACE(("MAP-FILE pid=%lu, pFile=%p, size=%lld\n", + osGetCurrentProcessId(), pFd, nByte)); + + if( pFd->nFetchOut>0 ) return SQLITE_OK; + + if( nMap<0 ){ + rc = winFileSize((sqlite3_file*)pFd, &nMap); + if( rc ){ + OSTRACE(("MAP-FILE pid=%lu, pFile=%p, rc=SQLITE_IOERR_FSTAT\n", + osGetCurrentProcessId(), pFd)); + return SQLITE_IOERR_FSTAT; + } + } + if( nMap>pFd->mmapSizeMax ){ + nMap = pFd->mmapSizeMax; + } + nMap &= ~(sqlite3_int64)(winSysInfo.dwPageSize - 1); + + if( nMap==0 && pFd->mmapSize>0 ){ + winUnmapfile(pFd); + } + if( nMap!=pFd->mmapSize ){ + void *pNew = 0; + DWORD protect = PAGE_READONLY; + DWORD flags = FILE_MAP_READ; + + winUnmapfile(pFd); + if( (pFd->ctrlFlags & WINFILE_RDONLY)==0 ){ + protect = PAGE_READWRITE; + flags |= FILE_MAP_WRITE; + } +#if SQLITE_OS_WINRT + pFd->hMap = osCreateFileMappingFromApp(pFd->h, NULL, protect, nMap, NULL); +#elif defined(SQLITE_WIN32_HAS_WIDE) + pFd->hMap = osCreateFileMappingW(pFd->h, NULL, protect, + (DWORD)((nMap>>32) & 0xffffffff), + (DWORD)(nMap & 0xffffffff), NULL); +#elif defined(SQLITE_WIN32_HAS_ANSI) + pFd->hMap = osCreateFileMappingA(pFd->h, NULL, protect, + (DWORD)((nMap>>32) & 0xffffffff), + (DWORD)(nMap & 0xffffffff), NULL); +#endif + if( pFd->hMap==NULL ){ + pFd->lastErrno = osGetLastError(); + rc = winLogError(SQLITE_IOERR_MMAP, pFd->lastErrno, + "winMapfile", pFd->zPath); + /* Log the error, but continue normal operation using xRead/xWrite */ + OSTRACE(("MAP-FILE-CREATE pid=%lu, pFile=%p, rc=SQLITE_IOERR_MMAP\n", + osGetCurrentProcessId(), pFd)); + return SQLITE_OK; + } + assert( (nMap % winSysInfo.dwPageSize)==0 ); +#if SQLITE_OS_WINRT + pNew = osMapViewOfFileFromApp(pFd->hMap, flags, 0, nMap); +#else + assert( sizeof(SIZE_T)==sizeof(sqlite3_int64) || nMap<=0xffffffff ); + pNew = osMapViewOfFile(pFd->hMap, flags, 0, 0, (SIZE_T)nMap); +#endif + if( pNew==NULL ){ + osCloseHandle(pFd->hMap); + pFd->hMap = NULL; + pFd->lastErrno = osGetLastError(); + winLogError(SQLITE_IOERR_MMAP, pFd->lastErrno, + "winMapfile", pFd->zPath); + OSTRACE(("MAP-FILE-MAP pid=%lu, pFile=%p, rc=SQLITE_IOERR_MMAP\n", + osGetCurrentProcessId(), pFd)); + return SQLITE_OK; + } + pFd->pMapRegion = pNew; + pFd->mmapSize = nMap; + pFd->mmapSizeActual = nMap; + } + + OSTRACE(("MAP-FILE pid=%lu, pFile=%p, rc=SQLITE_OK\n", + osGetCurrentProcessId(), pFd)); + return SQLITE_OK; +} +#endif /* SQLITE_MAX_MMAP_SIZE>0 */ + +/* +** If possible, return a pointer to a mapping of file fd starting at offset +** iOff. The mapping must be valid for at least nAmt bytes. +** +** If such a pointer can be obtained, store it in *pp and return SQLITE_OK. +** Or, if one cannot but no error occurs, set *pp to 0 and return SQLITE_OK. +** Finally, if an error does occur, return an SQLite error code. The final +** value of *pp is undefined in this case. +** +** If this function does return a pointer, the caller must eventually +** release the reference by calling winUnfetch(). +*/ +static int winFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ +#if SQLITE_MAX_MMAP_SIZE>0 + winFile *pFd = (winFile*)fd; /* The underlying database file */ +#endif + *pp = 0; + + OSTRACE(("FETCH pid=%lu, pFile=%p, offset=%lld, amount=%d, pp=%p\n", + osGetCurrentProcessId(), fd, iOff, nAmt, pp)); + +#if SQLITE_MAX_MMAP_SIZE>0 + if( pFd->mmapSizeMax>0 ){ + if( pFd->pMapRegion==0 ){ + int rc = winMapfile(pFd, -1); + if( rc!=SQLITE_OK ){ + OSTRACE(("FETCH pid=%lu, pFile=%p, rc=%s\n", + osGetCurrentProcessId(), pFd, sqlite3ErrName(rc))); + return rc; + } + } + if( pFd->mmapSize >= iOff+nAmt ){ + *pp = &((u8 *)pFd->pMapRegion)[iOff]; + pFd->nFetchOut++; + } + } +#endif + + OSTRACE(("FETCH pid=%lu, pFile=%p, pp=%p, *pp=%p, rc=SQLITE_OK\n", + osGetCurrentProcessId(), fd, pp, *pp)); + return SQLITE_OK; +} + +/* +** If the third argument is non-NULL, then this function releases a +** reference obtained by an earlier call to winFetch(). The second +** argument passed to this function must be the same as the corresponding +** argument that was passed to the winFetch() invocation. +** +** Or, if the third argument is NULL, then this function is being called +** to inform the VFS layer that, according to POSIX, any existing mapping +** may now be invalid and should be unmapped. +*/ +static int winUnfetch(sqlite3_file *fd, i64 iOff, void *p){ +#if SQLITE_MAX_MMAP_SIZE>0 + winFile *pFd = (winFile*)fd; /* The underlying database file */ + + /* If p==0 (unmap the entire file) then there must be no outstanding + ** xFetch references. Or, if p!=0 (meaning it is an xFetch reference), + ** then there must be at least one outstanding. */ + assert( (p==0)==(pFd->nFetchOut==0) ); + + /* If p!=0, it must match the iOff value. */ + assert( p==0 || p==&((u8 *)pFd->pMapRegion)[iOff] ); + + OSTRACE(("UNFETCH pid=%lu, pFile=%p, offset=%lld, p=%p\n", + osGetCurrentProcessId(), pFd, iOff, p)); + + if( p ){ + pFd->nFetchOut--; + }else{ + /* FIXME: If Windows truly always prevents truncating or deleting a + ** file while a mapping is held, then the following winUnmapfile() call + ** is unnecessary can can be omitted - potentially improving + ** performance. */ + winUnmapfile(pFd); + } + + assert( pFd->nFetchOut>=0 ); +#endif + + OSTRACE(("UNFETCH pid=%lu, pFile=%p, rc=SQLITE_OK\n", + osGetCurrentProcessId(), fd)); + return SQLITE_OK; +} + +/* ** Here ends the implementation of all sqlite3_file methods. ** ********************** End sqlite3_file Methods ******************************* @@ -3356,7 +3815,7 @@ shmpage_out: ** sqlite3_file for win32. */ static const sqlite3_io_methods winIoMethod = { - 2, /* iVersion */ + 3, /* iVersion */ winClose, /* xClose */ winRead, /* xRead */ winWrite, /* xWrite */ @@ -3372,7 +3831,9 @@ static const sqlite3_io_methods winIoMethod = { winShmMap, /* xShmMap */ winShmLock, /* xShmLock */ winShmBarrier, /* xShmBarrier */ - winShmUnmap /* xShmUnmap */ + winShmUnmap, /* xShmUnmap */ + winFetch, /* xFetch */ + winUnfetch /* xUnfetch */ }; /**************************************************************************** @@ -3436,6 +3897,7 @@ static int getTempname(int nBuf, char *zBuf){ sqlite3_snprintf(MAX_PATH-30, zTempPath, "%s", zMulti); sqlite3_free(zMulti); }else{ + OSTRACE(("TEMP-FILENAME rc=SQLITE_IOERR_NOMEM\n")); return SQLITE_IOERR_NOMEM; } } @@ -3449,6 +3911,7 @@ static int getTempname(int nBuf, char *zBuf){ sqlite3_snprintf(MAX_PATH-30, zTempPath, "%s", zUtf8); sqlite3_free(zUtf8); }else{ + OSTRACE(("TEMP-FILENAME rc=SQLITE_IOERR_NOMEM\n")); return SQLITE_IOERR_NOMEM; } } @@ -3461,6 +3924,7 @@ static int getTempname(int nBuf, char *zBuf){ nTempPath = sqlite3Strlen30(zTempPath); if( (nTempPath + sqlite3Strlen30(SQLITE_TEMP_FILE_PREFIX) + 18) >= nBuf ){ + OSTRACE(("TEMP-FILENAME rc=SQLITE_ERROR\n")); return SQLITE_ERROR; } @@ -3478,8 +3942,8 @@ static int getTempname(int nBuf, char *zBuf){ zBuf[j] = 0; zBuf[j+1] = 0; - OSTRACE(("TEMP FILENAME: %s\n", zBuf)); - return SQLITE_OK; + OSTRACE(("TEMP-FILENAME name=%s, rc=SQLITE_OK\n", zBuf)); + return SQLITE_OK; } /* @@ -3548,9 +4012,7 @@ static int winOpen( int isExclusive = (flags & SQLITE_OPEN_EXCLUSIVE); int isDelete = (flags & SQLITE_OPEN_DELETEONCLOSE); int isCreate = (flags & SQLITE_OPEN_CREATE); -#ifndef NDEBUG int isReadonly = (flags & SQLITE_OPEN_READONLY); -#endif int isReadWrite = (flags & SQLITE_OPEN_READWRITE); #ifndef NDEBUG @@ -3561,6 +4023,9 @@ static int winOpen( )); #endif + OSTRACE(("OPEN name=%s, pFile=%p, flags=%x, pOutFlags=%p\n", + zUtf8Name, id, flags, pOutFlags)); + /* Check the following statements are true: ** ** (a) Exactly one of the READWRITE and READONLY flags must be set, and @@ -3587,8 +4052,9 @@ static int winOpen( || eType==SQLITE_OPEN_TRANSIENT_DB || eType==SQLITE_OPEN_WAL ); - assert( id!=0 ); - UNUSED_PARAMETER(pVfs); + assert( pFile!=0 ); + memset(pFile, 0, sizeof(winFile)); + pFile->h = INVALID_HANDLE_VALUE; #if SQLITE_OS_WINRT if( !sqlite3_temp_directory ){ @@ -3597,15 +4063,15 @@ static int winOpen( } #endif - pFile->h = INVALID_HANDLE_VALUE; - /* If the second argument to this function is NULL, generate a ** temporary file name to use */ if( !zUtf8Name ){ assert(isDelete && !isOpenJournal); + memset(zTmpname, 0, MAX_PATH+2); rc = getTempname(MAX_PATH+2, zTmpname); if( rc!=SQLITE_OK ){ + OSTRACE(("OPEN name=%s, rc=%s", zUtf8Name, sqlite3ErrName(rc))); return rc; } zUtf8Name = zTmpname; @@ -3621,11 +4087,13 @@ static int winOpen( /* Convert the filename to the system encoding. */ zConverted = convertUtf8Filename(zUtf8Name); if( zConverted==0 ){ + OSTRACE(("OPEN name=%s, rc=SQLITE_IOERR_NOMEM", zUtf8Name)); return SQLITE_IOERR_NOMEM; } if( winIsDir(zConverted) ){ sqlite3_free(zConverted); + OSTRACE(("OPEN name=%s, rc=SQLITE_CANTOPEN_ISDIR", zUtf8Name)); return SQLITE_CANTOPEN_ISDIR; } @@ -3716,9 +4184,8 @@ static int winOpen( #endif logIoerr(cnt); - OSTRACE(("OPEN %d %s 0x%lx %s\n", - h, zName, dwDesiredAccess, - h==INVALID_HANDLE_VALUE ? "failed" : "ok")); + OSTRACE(("OPEN file=%p, name=%s, access=%lx, rc=%s\n", h, zUtf8Name, + dwDesiredAccess, (h==INVALID_HANDLE_VALUE) ? "failed" : "ok")); if( h==INVALID_HANDLE_VALUE ){ pFile->lastErrno = lastErrno; @@ -3726,7 +4193,9 @@ static int winOpen( sqlite3_free(zConverted); if( isReadWrite && !isExclusive ){ return winOpen(pVfs, zName, id, - ((flags|SQLITE_OPEN_READONLY)&~(SQLITE_OPEN_CREATE|SQLITE_OPEN_READWRITE)), pOutFlags); + ((flags|SQLITE_OPEN_READONLY) & + ~(SQLITE_OPEN_CREATE|SQLITE_OPEN_READWRITE)), + pOutFlags); }else{ return SQLITE_CANTOPEN_BKPT; } @@ -3740,26 +4209,18 @@ static int winOpen( } } - memset(pFile, 0, sizeof(*pFile)); - pFile->pMethod = &winIoMethod; - pFile->h = h; - pFile->lastErrno = NO_ERROR; - pFile->pVfs = pVfs; -#ifndef SQLITE_OMIT_WAL - pFile->pShm = 0; -#endif - pFile->zPath = zName; - if( sqlite3_uri_boolean(zName, "psow", SQLITE_POWERSAFE_OVERWRITE) ){ - pFile->ctrlFlags |= WINFILE_PSOW; - } + OSTRACE(("OPEN file=%p, name=%s, access=%lx, pOutFlags=%p, *pOutFlags=%d, " + "rc=%s\n", h, zUtf8Name, dwDesiredAccess, pOutFlags, pOutFlags ? + *pOutFlags : 0, (h==INVALID_HANDLE_VALUE) ? "failed" : "ok")); #if SQLITE_OS_WINCE if( isReadWrite && eType==SQLITE_OPEN_MAIN_DB - && !winceCreateLock(zName, pFile) + && (rc = winceCreateLock(zName, pFile))!=SQLITE_OK ){ osCloseHandle(h); sqlite3_free(zConverted); - return SQLITE_CANTOPEN_BKPT; + OSTRACE(("OPEN-CE-LOCK name=%s, rc=%s\n", zName, sqlite3ErrName(rc))); + return rc; } if( isTemp ){ pFile->zDeleteOnClose = zConverted; @@ -3769,6 +4230,25 @@ static int winOpen( sqlite3_free(zConverted); } + pFile->pMethod = &winIoMethod; + pFile->pVfs = pVfs; + pFile->h = h; + if( isReadonly ){ + pFile->ctrlFlags |= WINFILE_RDONLY; + } + if( sqlite3_uri_boolean(zName, "psow", SQLITE_POWERSAFE_OVERWRITE) ){ + pFile->ctrlFlags |= WINFILE_PSOW; + } + pFile->lastErrno = NO_ERROR; + pFile->zPath = zName; +#if SQLITE_MAX_MMAP_SIZE>0 + pFile->hMap = NULL; + pFile->pMapRegion = 0; + pFile->mmapSize = 0; + pFile->mmapSizeActual = 0; + pFile->mmapSizeMax = sqlite3GlobalConfig.szMmap; +#endif + OpenCounter(+1); return rc; } @@ -3799,6 +4279,8 @@ static int winDelete( UNUSED_PARAMETER(syncDir); SimulateIOError(return SQLITE_IOERR_DELETE); + OSTRACE(("DELETE name=%s, syncDir=%d\n", zFilename, syncDir)); + zConverted = convertUtf8Filename(zFilename); if( zConverted==0 ){ return SQLITE_IOERR_NOMEM; @@ -3812,14 +4294,26 @@ static int winDelete( &sAttrData) ){ attr = sAttrData.dwFileAttributes; }else{ - rc = SQLITE_OK; /* Already gone? */ + lastErrno = osGetLastError(); + if( lastErrno==ERROR_FILE_NOT_FOUND + || lastErrno==ERROR_PATH_NOT_FOUND ){ + rc = SQLITE_IOERR_DELETE_NOENT; /* Already gone? */ + }else{ + rc = SQLITE_ERROR; + } break; } #else attr = osGetFileAttributesW(zConverted); #endif if ( attr==INVALID_FILE_ATTRIBUTES ){ - rc = SQLITE_OK; /* Already gone? */ + lastErrno = osGetLastError(); + if( lastErrno==ERROR_FILE_NOT_FOUND + || lastErrno==ERROR_PATH_NOT_FOUND ){ + rc = SQLITE_IOERR_DELETE_NOENT; /* Already gone? */ + }else{ + rc = SQLITE_ERROR; + } break; } if ( attr&FILE_ATTRIBUTE_DIRECTORY ){ @@ -3841,7 +4335,13 @@ static int winDelete( do { attr = osGetFileAttributesA(zConverted); if ( attr==INVALID_FILE_ATTRIBUTES ){ - rc = SQLITE_OK; /* Already gone? */ + lastErrno = osGetLastError(); + if( lastErrno==ERROR_FILE_NOT_FOUND + || lastErrno==ERROR_PATH_NOT_FOUND ){ + rc = SQLITE_IOERR_DELETE_NOENT; /* Already gone? */ + }else{ + rc = SQLITE_ERROR; + } break; } if ( attr&FILE_ATTRIBUTE_DIRECTORY ){ @@ -3859,19 +4359,19 @@ static int winDelete( } while(1); } #endif - if( rc ){ + if( rc && rc!=SQLITE_IOERR_DELETE_NOENT ){ rc = winLogError(SQLITE_IOERR_DELETE, lastErrno, "winDelete", zFilename); }else{ logIoerr(cnt); } sqlite3_free(zConverted); - OSTRACE(("DELETE \"%s\" %s\n", zFilename, (rc ? "failed" : "ok" ))); + OSTRACE(("DELETE name=%s, rc=%s\n", zFilename, sqlite3ErrName(rc))); return rc; } /* -** Check the existance and status of a file. +** Check the existence and status of a file. */ static int winAccess( sqlite3_vfs *pVfs, /* Not used on win32 */ @@ -3886,8 +4386,12 @@ static int winAccess( UNUSED_PARAMETER(pVfs); SimulateIOError( return SQLITE_IOERR_ACCESS; ); + OSTRACE(("ACCESS name=%s, flags=%x, pResOut=%p\n", + zFilename, flags, pResOut)); + zConverted = convertUtf8Filename(zFilename); if( zConverted==0 ){ + OSTRACE(("ACCESS name=%s, rc=SQLITE_IOERR_NOMEM\n", zFilename)); return SQLITE_IOERR_NOMEM; } if( isNT() ){ @@ -3938,6 +4442,8 @@ static int winAccess( assert(!"Invalid flags argument"); } *pResOut = rc; + OSTRACE(("ACCESS name=%s, pResOut=%p, *pResOut=%d, rc=SQLITE_OK\n", + zFilename, pResOut, *pResOut)); return SQLITE_OK; } @@ -4005,16 +4511,12 @@ static int winFullPathname( */ char zOut[MAX_PATH+1]; memset(zOut, 0, MAX_PATH+1); - cygwin_conv_to_win32_path(zRelative, zOut); /* POSIX to Win32 */ + cygwin_conv_path(CCP_POSIX_TO_WIN_A|CCP_RELATIVE, zRelative, zOut, + MAX_PATH+1); sqlite3_snprintf(MIN(nFull, pVfs->mxPathname), zFull, "%s\\%s", sqlite3_data_directory, zOut); }else{ - /* - ** NOTE: The Cygwin docs state that the maximum length needed - ** for the buffer passed to cygwin_conv_to_full_win32_path - ** is MAX_PATH. - */ - cygwin_conv_to_full_win32_path(zRelative, zFull); + cygwin_conv_path(CCP_POSIX_TO_WIN_A, zRelative, zFull, nFull); } return SQLITE_OK; #endif @@ -4039,7 +4541,7 @@ static int winFullPathname( #endif #if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && !defined(__CYGWIN__) - int nByte; + DWORD nByte; void *zConverted; char *zOut; @@ -4073,13 +4575,27 @@ static int winFullPathname( } if( isNT() ){ LPWSTR zTemp; - nByte = osGetFullPathNameW((LPCWSTR)zConverted, 0, 0, 0) + 3; - zTemp = sqlite3_malloc( nByte*sizeof(zTemp[0]) ); + nByte = osGetFullPathNameW((LPCWSTR)zConverted, 0, 0, 0); + if( nByte==0 ){ + winLogError(SQLITE_ERROR, osGetLastError(), + "GetFullPathNameW1", zConverted); + sqlite3_free(zConverted); + return SQLITE_CANTOPEN_FULLPATH; + } + nByte += 3; + zTemp = sqlite3MallocZero( nByte*sizeof(zTemp[0]) ); if( zTemp==0 ){ sqlite3_free(zConverted); return SQLITE_IOERR_NOMEM; } - osGetFullPathNameW((LPCWSTR)zConverted, nByte, zTemp, 0); + nByte = osGetFullPathNameW((LPCWSTR)zConverted, nByte, zTemp, 0); + if( nByte==0 ){ + winLogError(SQLITE_ERROR, osGetLastError(), + "GetFullPathNameW2", zConverted); + sqlite3_free(zConverted); + sqlite3_free(zTemp); + return SQLITE_CANTOPEN_FULLPATH; + } sqlite3_free(zConverted); zOut = unicodeToUtf8(zTemp); sqlite3_free(zTemp); @@ -4087,13 +4603,27 @@ static int winFullPathname( #ifdef SQLITE_WIN32_HAS_ANSI else{ char *zTemp; - nByte = osGetFullPathNameA((char*)zConverted, 0, 0, 0) + 3; - zTemp = sqlite3_malloc( nByte*sizeof(zTemp[0]) ); + nByte = osGetFullPathNameA((char*)zConverted, 0, 0, 0); + if( nByte==0 ){ + winLogError(SQLITE_ERROR, osGetLastError(), + "GetFullPathNameA1", zConverted); + sqlite3_free(zConverted); + return SQLITE_CANTOPEN_FULLPATH; + } + nByte += 3; + zTemp = sqlite3MallocZero( nByte*sizeof(zTemp[0]) ); if( zTemp==0 ){ sqlite3_free(zConverted); return SQLITE_IOERR_NOMEM; } - osGetFullPathNameA((char*)zConverted, nByte, zTemp, 0); + nByte = osGetFullPathNameA((char*)zConverted, nByte, zTemp, 0); + if( nByte==0 ){ + winLogError(SQLITE_ERROR, osGetLastError(), + "GetFullPathNameA2", zConverted); + sqlite3_free(zConverted); + sqlite3_free(zTemp); + return SQLITE_CANTOPEN_FULLPATH; + } sqlite3_free(zConverted); zOut = sqlite3_win32_mbcs_to_utf8(zTemp); sqlite3_free(zTemp); @@ -4144,9 +4674,9 @@ static void winDlError(sqlite3_vfs *pVfs, int nBuf, char *zBufOut){ UNUSED_PARAMETER(pVfs); getLastErrorMsg(osGetLastError(), nBuf, zBufOut); } -static void (*winDlSym(sqlite3_vfs *pVfs, void *pHandle, const char *zSymbol))(void){ +static void (*winDlSym(sqlite3_vfs *pVfs,void *pH,const char *zSym))(void){ UNUSED_PARAMETER(pVfs); - return (void(*)(void))osGetProcAddressA((HANDLE)pHandle, zSymbol); + return (void(*)(void))osGetProcAddressA((HANDLE)pH, zSym); } static void winDlClose(sqlite3_vfs *pVfs, void *pHandle){ UNUSED_PARAMETER(pVfs); @@ -4244,7 +4774,8 @@ static int winCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *piNow){ #endif /* 2^32 - to avoid use of LL and warnings in gcc */ static const sqlite3_int64 max32BitValue = - (sqlite3_int64)2000000000 + (sqlite3_int64)2000000000 + (sqlite3_int64)294967296; + (sqlite3_int64)2000000000 + (sqlite3_int64)2000000000 + + (sqlite3_int64)294967296; #if SQLITE_OS_WINCE SYSTEMTIME time; @@ -4351,9 +4882,8 @@ int sqlite3_os_init(void){ /* Double-check that the aSyscall[] array has been constructed ** correctly. See ticket [bb3a86e890c8e96ab] */ - assert( ArraySize(aSyscall)==73 ); + assert( ArraySize(aSyscall)==74 ); -#ifndef SQLITE_OMIT_WAL /* get memory map allocation granularity */ memset(&winSysInfo, 0, sizeof(SYSTEM_INFO)); #if SQLITE_OS_WINRT @@ -4361,8 +4891,8 @@ int sqlite3_os_init(void){ #else osGetSystemInfo(&winSysInfo); #endif - assert(winSysInfo.dwAllocationGranularity > 0); -#endif + assert( winSysInfo.dwAllocationGranularity>0 ); + assert( winSysInfo.dwPageSize>0 ); sqlite3_vfs_register(&winVfs, 1); return SQLITE_OK; @@ -4370,7 +4900,7 @@ int sqlite3_os_init(void){ int sqlite3_os_end(void){ #if SQLITE_OS_WINRT - if( sleepObj != NULL ){ + if( sleepObj!=NULL ){ osCloseHandle(sleepObj); sleepObj = NULL; } diff --git a/src/pager.c b/src/pager.c index faf0223..61727fa 100644 --- a/src/pager.c +++ b/src/pager.c @@ -273,7 +273,7 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** * A write transaction is active. ** * An EXCLUSIVE or greater lock is held on the database file. ** * All writing and syncing of journal and database data has finished. -** If no error occured, all that remains is to finalize the journal to +** If no error occurred, all that remains is to finalize the journal to ** commit the transaction. If an error did occur, the caller will need ** to rollback the transaction. ** @@ -521,7 +521,7 @@ struct PagerSavepoint { ** ** doNotSpill, doNotSyncSpill ** -** These two boolean variables control the behaviour of cache-spills +** These two boolean variables control the behavior of cache-spills ** (calls made by the pcache module to the pagerStress() routine to ** write cached data to the file-system in order to free up memory). ** @@ -655,6 +655,11 @@ struct Pager { PagerSavepoint *aSavepoint; /* Array of active savepoints */ int nSavepoint; /* Number of elements in aSavepoint[] */ char dbFileVers[16]; /* Changes whenever database file changes */ + + u8 bUseFetch; /* True to use xFetch() */ + int nMmapOut; /* Number of mmap pages currently outstanding */ + sqlite3_int64 szMmap; /* Desired maximum mmap size */ + PgHdr *pMmapFreelist; /* List of free mmap page headers (pDirty) */ /* ** End of the routinely-changing class members ***************************************************************************/ @@ -766,6 +771,16 @@ static const unsigned char aJournalMagic[] = { #endif /* +** The macro USEFETCH is true if we are allowed to use the xFetch and xUnfetch +** interfaces to access the database using memory-mapped I/O. +*/ +#if SQLITE_MAX_MMAP_SIZE>0 +# define USEFETCH(x) ((x)->bUseFetch) +#else +# define USEFETCH(x) 0 +#endif + +/* ** The maximum legal page number is (2^31 - 1). */ #define PAGER_MAX_PGNO 2147483647 @@ -1399,7 +1414,7 @@ static int writeJournalHdr(Pager *pPager){ memset(zHeader, 0, sizeof(aJournalMagic)+4); } - /* The random check-hash initialiser */ + /* The random check-hash initializer */ sqlite3_randomness(sizeof(pPager->cksumInit), &pPager->cksumInit); put32bits(&zHeader[sizeof(aJournalMagic)+4], pPager->cksumInit); /* The initial database size */ @@ -1838,6 +1853,8 @@ static int pager_error(Pager *pPager, int rc){ return rc; } +static int pager_truncate(Pager *pPager, Pgno nPage); + /* ** This routine ends a transaction. A transaction is usually ended by ** either a COMMIT or a ROLLBACK operation. This routine may be called @@ -1891,7 +1908,7 @@ static int pager_error(Pager *pPager, int rc){ ** to the first error encountered (the journal finalization one) is ** returned. */ -static int pager_end_transaction(Pager *pPager, int hasMaster){ +static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){ int rc = SQLITE_OK; /* Error code from journal finalization operation */ int rc2 = SQLITE_OK; /* Error code from db file unlock operation */ @@ -1941,12 +1958,13 @@ static int pager_end_transaction(Pager *pPager, int hasMaster){ ** file should be closed and deleted. If this connection writes to ** the database file, it will do so using an in-memory journal. */ + int bDelete = (!pPager->tempFile && sqlite3JournalExists(pPager->jfd)); assert( pPager->journalMode==PAGER_JOURNALMODE_DELETE || pPager->journalMode==PAGER_JOURNALMODE_MEMORY || pPager->journalMode==PAGER_JOURNALMODE_WAL ); sqlite3OsClose(pPager->jfd); - if( !pPager->tempFile ){ + if( bDelete ){ rc = sqlite3OsDelete(pPager->pVfs, pPager->zJournal, 0); } } @@ -1976,7 +1994,17 @@ static int pager_end_transaction(Pager *pPager, int hasMaster){ */ rc2 = sqlite3WalEndWriteTransaction(pPager->pWal); assert( rc2==SQLITE_OK ); + }else if( rc==SQLITE_OK && bCommit && pPager->dbFileSize>pPager->dbSize ){ + /* This branch is taken when committing a transaction in rollback-journal + ** mode if the database file on disk is larger than the database image. + ** At this point the journal has been finalized and the transaction + ** successfully committed, but the EXCLUSIVE lock is still held on the + ** file. So it is safe to truncate the database file to its minimum + ** required size. */ + assert( pPager->eLock==EXCLUSIVE_LOCK ); + rc = pager_truncate(pPager, pPager->dbSize); } + if( !pPager->exclusiveMode && (!pagerUseWal(pPager) || sqlite3WalExclusiveMode(pPager->pWal, 0)) ){ @@ -2015,7 +2043,7 @@ static void pagerUnlockAndRollback(Pager *pPager){ sqlite3EndBenignMalloc(); }else if( !pPager->exclusiveMode ){ assert( pPager->eState==PAGER_READER ); - pager_end_transaction(pPager, 0); + pager_end_transaction(pPager, 0, 0); } } pager_unlock(pPager); @@ -2239,7 +2267,7 @@ static int pager_playback_one_page( i64 ofst = (pgno-1)*(i64)pPager->pageSize; testcase( !isSavepnt && pPg!=0 && (pPg->flags&PGHDR_NEED_SYNC)!=0 ); assert( !pagerUseWal(pPager) ); - rc = sqlite3OsWrite(pPager->fd, (u8*)aData, pPager->pageSize, ofst); + rc = sqlite3OsWrite(pPager->fd, (u8 *)aData, pPager->pageSize, ofst); if( pgno>pPager->dbFileSize ){ pPager->dbFileSize = pgno; } @@ -2510,6 +2538,21 @@ static int pager_truncate(Pager *pPager, Pgno nPage){ } /* +** Return a sanitized version of the sector-size of OS file pFile. The +** return value is guaranteed to lie between 32 and MAX_SECTOR_SIZE. +*/ +int sqlite3SectorSize(sqlite3_file *pFile){ + int iRet = sqlite3OsSectorSize(pFile); + if( iRet<32 ){ + iRet = 512; + }else if( iRet>MAX_SECTOR_SIZE ){ + assert( MAX_SECTOR_SIZE>=512 ); + iRet = MAX_SECTOR_SIZE; + } + return iRet; +} + +/* ** Set the value of the Pager.sectorSize variable for the given ** pager based on the value returned by the xSectorSize method ** of the open database file. The sector size will be used used @@ -2544,14 +2587,7 @@ static void setSectorSize(Pager *pPager){ ** call will segfault. */ pPager->sectorSize = 512; }else{ - pPager->sectorSize = sqlite3OsSectorSize(pPager->fd); - if( pPager->sectorSize<32 ){ - pPager->sectorSize = 512; - } - if( pPager->sectorSize>MAX_SECTOR_SIZE ){ - assert( MAX_SECTOR_SIZE>=512 ); - pPager->sectorSize = MAX_SECTOR_SIZE; - } + pPager->sectorSize = sqlite3SectorSize(pPager->fd); } } @@ -2622,6 +2658,7 @@ static int pager_playback(Pager *pPager, int isHot){ int res = 1; /* Value returned by sqlite3OsAccess() */ char *zMaster = 0; /* Name of master journal file if any */ int needPagerReset; /* True to reset page prior to first page rollback */ + int nPlayback = 0; /* Total number of pages restored from journal */ /* Figure out how many records are in the journal. Abort early if ** the journal is empty. @@ -2722,7 +2759,9 @@ static int pager_playback(Pager *pPager, int isHot){ needPagerReset = 0; } rc = pager_playback_one_page(pPager,&pPager->journalOff,0,1,0); - if( rc!=SQLITE_OK ){ + if( rc==SQLITE_OK ){ + nPlayback++; + }else{ if( rc==SQLITE_DONE ){ pPager->journalOff = szJ; break; @@ -2782,7 +2821,7 @@ end_playback: rc = sqlite3PagerSync(pPager); } if( rc==SQLITE_OK ){ - rc = pager_end_transaction(pPager, zMaster[0]!='\0'); + rc = pager_end_transaction(pPager, zMaster[0]!='\0', 0); testcase( rc!=SQLITE_OK ); } if( rc==SQLITE_OK && zMaster[0] && res ){ @@ -2792,6 +2831,10 @@ end_playback: rc = pager_delmaster(pPager, zMaster); testcase( rc!=SQLITE_OK ); } + if( isHot && nPlayback ){ + sqlite3_log(SQLITE_NOTICE_RECOVER_ROLLBACK, "recovered %d pages from %s", + nPlayback, pPager->zJournal); + } /* The Pager.sectorSize variable may have been updated while rolling ** back a journal created by a process with a different sector size @@ -2813,11 +2856,10 @@ end_playback: ** If an IO error occurs, then the IO error is returned to the caller. ** Otherwise, SQLITE_OK is returned. */ -static int readDbPage(PgHdr *pPg){ +static int readDbPage(PgHdr *pPg, u32 iFrame){ Pager *pPager = pPg->pPager; /* Pager object associated with page pPg */ Pgno pgno = pPg->pgno; /* Page number to read */ int rc = SQLITE_OK; /* Return code */ - int isInWal = 0; /* True if page is in log file */ int pgsz = pPager->pageSize; /* Number of bytes to read */ assert( pPager->eState>=PAGER_READER && !MEMDB ); @@ -2829,11 +2871,13 @@ static int readDbPage(PgHdr *pPg){ return SQLITE_OK; } - if( pagerUseWal(pPager) ){ +#ifndef SQLITE_OMIT_WAL + if( iFrame ){ /* Try to pull the page from the write-ahead log. */ - rc = sqlite3WalRead(pPager->pWal, pgno, &isInWal, pgsz, pPg->pData); - } - if( rc==SQLITE_OK && !isInWal ){ + rc = sqlite3WalReadFrame(pPager->pWal, iFrame, pgsz, pPg->pData); + }else +#endif + { i64 iOffset = (pgno-1)*(i64)pPager->pageSize; rc = sqlite3OsRead(pPager->fd, pPg->pData, pgsz, iOffset); if( rc==SQLITE_IOERR_SHORT_READ ){ @@ -2912,12 +2956,17 @@ static int pagerUndoCallback(void *pCtx, Pgno iPg){ Pager *pPager = (Pager *)pCtx; PgHdr *pPg; + assert( pagerUseWal(pPager) ); pPg = sqlite3PagerLookup(pPager, iPg); if( pPg ){ if( sqlite3PcachePageRefcount(pPg)==1 ){ sqlite3PcacheDrop(pPg); }else{ - rc = readDbPage(pPg); + u32 iFrame = 0; + rc = sqlite3WalFindFrame(pPager->pWal, pPg->pgno, &iFrame); + if( rc==SQLITE_OK ){ + rc = readDbPage(pPg, iFrame); + } if( rc==SQLITE_OK ){ pPager->xReiniter(pPg); } @@ -3061,6 +3110,7 @@ static int pagerBeginReadTransaction(Pager *pPager){ rc = sqlite3WalBeginReadTransaction(pPager->pWal, &changed); if( rc!=SQLITE_OK || changed ){ pager_reset(pPager); + if( USEFETCH(pPager) ) sqlite3OsUnfetch(pPager->fd, 0, 0); } return rc; @@ -3151,6 +3201,7 @@ static int pagerOpenWalIfPresent(Pager *pPager){ if( rc ) return rc; if( nPage==0 ){ rc = sqlite3OsDelete(pPager->pVfs, pPager->zWal, 0); + if( rc==SQLITE_IOERR_DELETE_NOENT ) rc = SQLITE_OK; isWal = 0; }else{ rc = sqlite3OsAccess( @@ -3322,6 +3373,29 @@ void sqlite3PagerSetCachesize(Pager *pPager, int mxPage){ } /* +** Invoke SQLITE_FCNTL_MMAP_SIZE based on the current value of szMmap. +*/ +static void pagerFixMaplimit(Pager *pPager){ +#if SQLITE_MAX_MMAP_SIZE>0 + sqlite3_file *fd = pPager->fd; + if( isOpen(fd) ){ + sqlite3_int64 sz; + pPager->bUseFetch = (fd->pMethods->iVersion>=3) && pPager->szMmap>0; + sz = pPager->szMmap; + sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_MMAP_SIZE, &sz); + } +#endif +} + +/* +** Change the maximum size of any memory mapping made of the database file. +*/ +void sqlite3PagerSetMmapLimit(Pager *pPager, sqlite3_int64 szMmap){ + pPager->szMmap = szMmap; + pagerFixMaplimit(pPager); +} + +/* ** Free as much memory as possible from the pager. */ void sqlite3PagerShrink(Pager *pPager){ @@ -3468,9 +3542,16 @@ void sqlite3PagerSetBusyhandler( Pager *pPager, /* Pager object */ int (*xBusyHandler)(void *), /* Pointer to busy-handler function */ void *pBusyHandlerArg /* Argument to pass to xBusyHandler */ -){ +){ pPager->xBusyHandler = xBusyHandler; pPager->pBusyHandlerArg = pBusyHandlerArg; + + if( isOpen(pPager->fd) ){ + void **ap = (void **)&pPager->xBusyHandler; + assert( ((int(*)(void *))(ap[0]))==xBusyHandler ); + assert( ap[1]==pBusyHandlerArg ); + sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_BUSYHANDLER, (void *)ap); + } } /* @@ -3549,6 +3630,7 @@ int sqlite3PagerSetPagesize(Pager *pPager, u32 *pPageSize, int nReserve){ assert( nReserve>=0 && nReserve<1000 ); pPager->nReserve = (i16)nReserve; pagerReportSize(pPager); + pagerFixMaplimit(pPager); } return rc; } @@ -3702,7 +3784,7 @@ static int pager_wait_on_lock(Pager *pPager, int locktype){ ** dirty page were to be discarded from the cache via the pagerStress() ** routine, pagerStress() would not write the current page content to ** the database file. If a savepoint transaction were rolled back after -** this happened, the correct behaviour would be to restore the current +** this happened, the correct behavior would be to restore the current ** content of the page. However, since this content is not present in either ** the database file or the portion of the rollback journal and ** sub-journal rolled back the content could not be restored and the @@ -3726,12 +3808,26 @@ static void assertTruncateConstraint(Pager *pPager){ ** function does not actually modify the database file on disk. It ** just sets the internal state of the pager object so that the ** truncation will be done when the current transaction is committed. +** +** This function is only called right before committing a transaction. +** Once this function has been called, the transaction must either be +** rolled back or committed. It is not safe to call this function and +** then continue writing to the database. */ void sqlite3PagerTruncateImage(Pager *pPager, Pgno nPage){ assert( pPager->dbSize>=nPage ); assert( pPager->eState>=PAGER_WRITER_CACHEMOD ); pPager->dbSize = nPage; - assertTruncateConstraint(pPager); + + /* At one point the code here called assertTruncateConstraint() to + ** ensure that all pages being truncated away by this operation are, + ** if one or more savepoints are open, present in the savepoint + ** journal so that they can be restored if the savepoint is rolled + ** back. This is no longer necessary as this function is now only + ** called right before committing a transaction. So although the + ** Pager object may still have open savepoints (Pager.nSavepoint!=0), + ** they cannot be rolled back. So the assertTruncateConstraint() call + ** is no longer correct. */ } @@ -3761,6 +3857,81 @@ static int pagerSyncHotJournal(Pager *pPager){ } /* +** Obtain a reference to a memory mapped page object for page number pgno. +** The new object will use the pointer pData, obtained from xFetch(). +** If successful, set *ppPage to point to the new page reference +** and return SQLITE_OK. Otherwise, return an SQLite error code and set +** *ppPage to zero. +** +** Page references obtained by calling this function should be released +** by calling pagerReleaseMapPage(). +*/ +static int pagerAcquireMapPage( + Pager *pPager, /* Pager object */ + Pgno pgno, /* Page number */ + void *pData, /* xFetch()'d data for this page */ + PgHdr **ppPage /* OUT: Acquired page object */ +){ + PgHdr *p; /* Memory mapped page to return */ + + if( pPager->pMmapFreelist ){ + *ppPage = p = pPager->pMmapFreelist; + pPager->pMmapFreelist = p->pDirty; + p->pDirty = 0; + memset(p->pExtra, 0, pPager->nExtra); + }else{ + *ppPage = p = (PgHdr *)sqlite3MallocZero(sizeof(PgHdr) + pPager->nExtra); + if( p==0 ){ + sqlite3OsUnfetch(pPager->fd, (i64)(pgno-1) * pPager->pageSize, pData); + return SQLITE_NOMEM; + } + p->pExtra = (void *)&p[1]; + p->flags = PGHDR_MMAP; + p->nRef = 1; + p->pPager = pPager; + } + + assert( p->pExtra==(void *)&p[1] ); + assert( p->pPage==0 ); + assert( p->flags==PGHDR_MMAP ); + assert( p->pPager==pPager ); + assert( p->nRef==1 ); + + p->pgno = pgno; + p->pData = pData; + pPager->nMmapOut++; + + return SQLITE_OK; +} + +/* +** Release a reference to page pPg. pPg must have been returned by an +** earlier call to pagerAcquireMapPage(). +*/ +static void pagerReleaseMapPage(PgHdr *pPg){ + Pager *pPager = pPg->pPager; + pPager->nMmapOut--; + pPg->pDirty = pPager->pMmapFreelist; + pPager->pMmapFreelist = pPg; + + assert( pPager->fd->pMethods->iVersion>=3 ); + sqlite3OsUnfetch(pPager->fd, (i64)(pPg->pgno-1)*pPager->pageSize, pPg->pData); +} + +/* +** Free all PgHdr objects stored in the Pager.pMmapFreelist list. +*/ +static void pagerFreeMapHdrs(Pager *pPager){ + PgHdr *p; + PgHdr *pNext; + for(p=pPager->pMmapFreelist; p; p=pNext){ + pNext = p->pDirty; + sqlite3_free(p); + } +} + + +/* ** Shutdown the page cache. Free all memory and close all files. ** ** If a transaction was in progress when this routine is called, that @@ -3780,6 +3951,7 @@ int sqlite3PagerClose(Pager *pPager){ assert( assert_pager_state(pPager) ); disable_simulated_io_errors(); sqlite3BeginBenignMalloc(); + pagerFreeMapHdrs(pPager); /* pPager->errCode = 0; */ pPager->exclusiveMode = 0; #ifndef SQLITE_OMIT_WAL @@ -4041,7 +4213,9 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ ** file size will be. */ assert( rc!=SQLITE_OK || isOpen(pPager->fd) ); - if( rc==SQLITE_OK && pPager->dbSize>pPager->dbHintSize ){ + if( rc==SQLITE_OK + && (pList->pDirty ? pPager->dbSize : pList->pgno+1)>pPager->dbHintSize + ){ sqlite3_int64 szFile = pPager->pageSize * (sqlite3_int64)pPager->dbSize; sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_SIZE_HINT, &szFile); pPager->dbHintSize = pPager->dbSize; @@ -4448,7 +4622,7 @@ int sqlite3PagerOpen( memcpy(pPager->zFilename, zPathname, nPathname); if( nUri ) memcpy(&pPager->zFilename[nPathname+1], zUri, nUri); memcpy(pPager->zJournal, zPathname, nPathname); - memcpy(&pPager->zJournal[nPathname], "-journal\000", 8+1); + memcpy(&pPager->zJournal[nPathname], "-journal\000", 8+2); sqlite3FileSuffix3(pPager->zFilename, pPager->zJournal); #ifndef SQLITE_OMIT_WAL pPager->zWal = &pPager->zJournal[nPathname+8+1]; @@ -4595,6 +4769,7 @@ int sqlite3PagerOpen( /* pPager->pBusyHandlerArg = 0; */ pPager->xReiniter = xReinit; /* memset(pPager->aHash, 0, sizeof(pPager->aHash)); */ + /* pPager->szMmap = SQLITE_DEFAULT_MMAP_SIZE // will be set by btree.c */ *ppPager = pPager; return SQLITE_OK; @@ -4784,6 +4959,11 @@ int sqlite3PagerSharedLock(Pager *pPager){ goto failed; } if( bHotJournal ){ + if( pPager->readOnly ){ + rc = SQLITE_READONLY_ROLLBACK; + goto failed; + } + /* Get an EXCLUSIVE lock on the database file. At this point it is ** important that a RESERVED lock is not obtained on the way to the ** EXCLUSIVE lock. If it were, another process might open the @@ -4881,9 +5061,11 @@ int sqlite3PagerSharedLock(Pager *pPager){ ); } - if( !pPager->tempFile - && (pPager->pBackup || sqlite3PcachePagecount(pPager->pPCache)>0) - ){ + if( !pPager->tempFile && ( + pPager->pBackup + || sqlite3PcachePagecount(pPager->pPCache)>0 + || USEFETCH(pPager) + )){ /* The shared-lock has just been acquired on the database file ** and there are already pages in the cache (from a previous ** read or write transaction). Check to see if the database @@ -4909,7 +5091,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ if( nPage>0 ){ IOTRACE(("CKVERS %p %d\n", pPager, sizeof(dbFileVers))); rc = sqlite3OsRead(pPager->fd, &dbFileVers, sizeof(dbFileVers), 24); - if( rc!=SQLITE_OK ){ + if( rc!=SQLITE_OK && rc!=SQLITE_IOERR_SHORT_READ ){ goto failed; } }else{ @@ -4918,6 +5100,16 @@ int sqlite3PagerSharedLock(Pager *pPager){ if( memcmp(pPager->dbFileVers, dbFileVers, sizeof(dbFileVers))!=0 ){ pager_reset(pPager); + + /* Unmap the database file. It is possible that external processes + ** may have truncated the database file and then extended it back + ** to its original size while this process was not holding a lock. + ** In this case there may exist a Pager.pMap mapping that appears + ** to be the right size but is not actually valid. Avoid this + ** possibility by unmapping the db here. */ + if( USEFETCH(pPager) ){ + sqlite3OsUnfetch(pPager->fd, 0, 0); + } } } @@ -4959,7 +5151,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ ** nothing to rollback, so this routine is a no-op. */ static void pagerUnlockIfUnused(Pager *pPager){ - if( (sqlite3PcacheRefCount(pPager->pPCache)==0) ){ + if( pPager->nMmapOut==0 && (sqlite3PcacheRefCount(pPager->pPCache)==0) ){ pagerUnlockAndRollback(pPager); } } @@ -5018,13 +5210,27 @@ int sqlite3PagerAcquire( Pager *pPager, /* The pager open on the database file */ Pgno pgno, /* Page number to fetch */ DbPage **ppPage, /* Write a pointer to the page here */ - int noContent /* Do not bother reading content from disk if true */ + int flags /* PAGER_ACQUIRE_XXX flags */ ){ - int rc; - PgHdr *pPg; + int rc = SQLITE_OK; + PgHdr *pPg = 0; + u32 iFrame = 0; /* Frame to read from WAL file */ + const int noContent = (flags & PAGER_ACQUIRE_NOCONTENT); + + /* It is acceptable to use a read-only (mmap) page for any page except + ** page 1 if there is no write-transaction open or the ACQUIRE_READONLY + ** flag was specified by the caller. And so long as the db is not a + ** temporary or in-memory database. */ + const int bMmapOk = (pgno!=1 && USEFETCH(pPager) + && (pPager->eState==PAGER_READER || (flags & PAGER_ACQUIRE_READONLY)) +#ifdef SQLITE_HAS_CODEC + && pPager->xCodec==0 +#endif + ); assert( pPager->eState>=PAGER_READER ); assert( assert_pager_state(pPager) ); + assert( noContent==0 || bMmapOk==0 ); if( pgno==0 ){ return SQLITE_CORRUPT_BKPT; @@ -5035,6 +5241,39 @@ int sqlite3PagerAcquire( if( pPager->errCode!=SQLITE_OK ){ rc = pPager->errCode; }else{ + + if( bMmapOk && pagerUseWal(pPager) ){ + rc = sqlite3WalFindFrame(pPager->pWal, pgno, &iFrame); + if( rc!=SQLITE_OK ) goto pager_acquire_err; + } + + if( iFrame==0 && bMmapOk ){ + void *pData = 0; + + rc = sqlite3OsFetch(pPager->fd, + (i64)(pgno-1) * pPager->pageSize, pPager->pageSize, &pData + ); + + if( rc==SQLITE_OK && pData ){ + if( pPager->eState>PAGER_READER ){ + (void)sqlite3PcacheFetch(pPager->pPCache, pgno, 0, &pPg); + } + if( pPg==0 ){ + rc = pagerAcquireMapPage(pPager, pgno, pData, &pPg); + }else{ + sqlite3OsUnfetch(pPager->fd, (i64)(pgno-1)*pPager->pageSize, pData); + } + if( pPg ){ + assert( rc==SQLITE_OK ); + *ppPage = pPg; + return SQLITE_OK; + } + } + if( rc!=SQLITE_OK ){ + goto pager_acquire_err; + } + } + rc = sqlite3PcacheFetch(pPager->pPCache, pgno, 1, ppPage); } @@ -5093,9 +5332,13 @@ int sqlite3PagerAcquire( memset(pPg->pData, 0, pPager->pageSize); IOTRACE(("ZERO %p %d\n", pPager, pgno)); }else{ + if( pagerUseWal(pPager) && bMmapOk==0 ){ + rc = sqlite3WalFindFrame(pPager->pWal, pgno, &iFrame); + if( rc!=SQLITE_OK ) goto pager_acquire_err; + } assert( pPg->pPager==pPager ); pPager->aStat[PAGER_STAT_MISS]++; - rc = readDbPage(pPg); + rc = readDbPage(pPg, iFrame); if( rc!=SQLITE_OK ){ goto pager_acquire_err; } @@ -5148,7 +5391,11 @@ DbPage *sqlite3PagerLookup(Pager *pPager, Pgno pgno){ void sqlite3PagerUnref(DbPage *pPg){ if( pPg ){ Pager *pPager = pPg->pPager; - sqlite3PcacheRelease(pPg); + if( pPg->flags & PGHDR_MMAP ){ + pagerReleaseMapPage(pPg); + }else{ + sqlite3PcacheRelease(pPg); + } pagerUnlockIfUnused(pPager); } } @@ -5483,6 +5730,7 @@ int sqlite3PagerWrite(DbPage *pDbPage){ Pager *pPager = pPg->pPager; Pgno nPagePerSector = (pPager->sectorSize/pPager->pageSize); + assert( (pPg->flags & PGHDR_MMAP)==0 ); assert( pPager->eState>=PAGER_WRITER_LOCKED ); assert( pPager->eState!=PAGER_ERROR ); assert( assert_pager_state(pPager) ); @@ -5650,7 +5898,7 @@ static int pager_incr_changecounter(Pager *pPager, int isDirectMode){ # define DIRECT_MODE isDirectMode #endif - if( !pPager->changeCountDone && pPager->dbSize>0 ){ + if( !pPager->changeCountDone && ALWAYS(pPager->dbSize>0) ){ PgHdr *pPgHdr; /* Reference to page 1 */ assert( !pPager->tempFile && isOpen(pPager->fd) ); @@ -5682,6 +5930,11 @@ static int pager_incr_changecounter(Pager *pPager, int isDirectMode){ pPager->aStat[PAGER_STAT_WRITE]++; } if( rc==SQLITE_OK ){ + /* Update the pager's copy of the change-counter. Otherwise, the + ** next time a read transaction is opened the cache will be + ** flushed (as the change-counter values will not match). */ + const void *pCopy = (const void *)&((const char *)zBuf)[24]; + memcpy(&pPager->dbFileVers, pCopy, sizeof(pPager->dbFileVers)); pPager->changeCountDone = 1; } }else{ @@ -5868,38 +6121,6 @@ int sqlite3PagerCommitPhaseOne( #endif if( rc!=SQLITE_OK ) goto commit_phase_one_exit; - /* If this transaction has made the database smaller, then all pages - ** being discarded by the truncation must be written to the journal - ** file. This can only happen in auto-vacuum mode. - ** - ** Before reading the pages with page numbers larger than the - ** current value of Pager.dbSize, set dbSize back to the value - ** that it took at the start of the transaction. Otherwise, the - ** calls to sqlite3PagerGet() return zeroed pages instead of - ** reading data from the database file. - */ - #ifndef SQLITE_OMIT_AUTOVACUUM - if( pPager->dbSize<pPager->dbOrigSize - && pPager->journalMode!=PAGER_JOURNALMODE_OFF - ){ - Pgno i; /* Iterator variable */ - const Pgno iSkip = PAGER_MJ_PGNO(pPager); /* Pending lock page */ - const Pgno dbSize = pPager->dbSize; /* Database image size */ - pPager->dbSize = pPager->dbOrigSize; - for( i=dbSize+1; i<=pPager->dbOrigSize; i++ ){ - if( !sqlite3BitvecTest(pPager->pInJournal, i) && i!=iSkip ){ - PgHdr *pPage; /* Page to journal */ - rc = sqlite3PagerGet(pPager, i, &pPage); - if( rc!=SQLITE_OK ) goto commit_phase_one_exit; - rc = sqlite3PagerWrite(pPage); - sqlite3PagerUnref(pPage); - if( rc!=SQLITE_OK ) goto commit_phase_one_exit; - } - } - pPager->dbSize = dbSize; - } - #endif - /* Write the master journal name into the journal file. If a master ** journal file name has already been written to the journal file, ** or if zMaster is NULL (no master journal), then this call is a no-op. @@ -5927,11 +6148,14 @@ int sqlite3PagerCommitPhaseOne( goto commit_phase_one_exit; } sqlite3PcacheCleanAll(pPager->pPCache); - - /* If the file on disk is not the same size as the database image, - ** then use pager_truncate to grow or shrink the file here. - */ - if( pPager->dbSize!=pPager->dbFileSize ){ + + /* If the file on disk is smaller than the database image, use + ** pager_truncate to grow the file here. This can happen if the database + ** image was extended as part of the current transaction and then the + ** last page in the db image moved to the free-list. In this case the + ** last page is never written out to disk, leaving the database file + ** undersized. Fix this now if it is the case. */ + if( pPager->dbSize>pPager->dbFileSize ){ Pgno nNew = pPager->dbSize - (pPager->dbSize==PAGER_MJ_PGNO(pPager)); assert( pPager->eState==PAGER_WRITER_DBMOD ); rc = pager_truncate(pPager, nNew); @@ -6004,7 +6228,7 @@ int sqlite3PagerCommitPhaseTwo(Pager *pPager){ } PAGERTRACE(("COMMIT %d\n", PAGERID(pPager))); - rc = pager_end_transaction(pPager, pPager->setMaster); + rc = pager_end_transaction(pPager, pPager->setMaster, 1); return pager_error(pPager, rc); } @@ -6049,11 +6273,11 @@ int sqlite3PagerRollback(Pager *pPager){ if( pagerUseWal(pPager) ){ int rc2; rc = sqlite3PagerSavepoint(pPager, SAVEPOINT_ROLLBACK, -1); - rc2 = pager_end_transaction(pPager, pPager->setMaster); + rc2 = pager_end_transaction(pPager, pPager->setMaster, 0); if( rc==SQLITE_OK ) rc = rc2; }else if( !isOpen(pPager->jfd) || pPager->eState==PAGER_WRITER_LOCKED ){ int eState = pPager->eState; - rc = pager_end_transaction(pPager, 0); + rc = pager_end_transaction(pPager, 0, 0); if( !MEMDB && eState>PAGER_WRITER_LOCKED ){ /* This can happen using journal_mode=off. Move the pager to the error ** state to indicate that the contents of the cache may not be trusted. @@ -6068,7 +6292,7 @@ int sqlite3PagerRollback(Pager *pPager){ } assert( pPager->eState==PAGER_READER || rc!=SQLITE_OK ); - assert( rc==SQLITE_OK || rc==SQLITE_FULL + assert( rc==SQLITE_OK || rc==SQLITE_FULL || rc==SQLITE_CORRUPT || rc==SQLITE_NOMEM || (rc&0xFF)==SQLITE_IOERR ); /* If an error occurs during a ROLLBACK, we can no longer trust the pager @@ -6451,7 +6675,8 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ */ if( (pPg->flags&PGHDR_NEED_SYNC) && !isCommit ){ needSyncPgno = pPg->pgno; - assert( pageInJournal(pPg) || pPg->pgno>pPager->dbOrigSize ); + assert( pPager->journalMode==PAGER_JOURNALMODE_OFF || + pageInJournal(pPg) || pPg->pgno>pPager->dbOrigSize ); assert( pPg->flags&PGHDR_DIRTY ); } @@ -6801,11 +7026,12 @@ static int pagerOpenWal(Pager *pPager){ ** (e.g. due to malloc() failure), return an error code. */ if( rc==SQLITE_OK ){ - rc = sqlite3WalOpen(pPager->pVfs, + rc = sqlite3WalOpen(pPager->pVfs, pPager->fd, pPager->zWal, pPager->exclusiveMode, pPager->journalSizeLimit, &pPager->pWal ); } + pagerFixMaplimit(pPager); return rc; } @@ -6896,11 +7122,14 @@ int sqlite3PagerCloseWal(Pager *pPager){ rc = sqlite3WalClose(pPager->pWal, pPager->ckptSyncFlags, pPager->pageSize, (u8*)pPager->pTmpSpace); pPager->pWal = 0; + pagerFixMaplimit(pPager); } } return rc; } +#endif /* !SQLITE_OMIT_WAL */ + #ifdef SQLITE_ENABLE_ZIPVFS /* ** A read-lock must be held on the pager when this function is called. If @@ -6930,11 +7159,9 @@ void *sqlite3PagerCodec(PgHdr *pPg){ } #endif /* SQLITE_HAS_CODEC */ -#endif /* !SQLITE_OMIT_WAL */ - #endif /* SQLITE_OMIT_DISKIO */ -/* BEGIN CRYPTO */ +/* BEGIN SQLCIPHER */ #ifdef SQLITE_HAS_CODEC void sqlite3pager_get_codec(Pager *pPager, void **ctx) { *ctx = pPager->pCodec; @@ -6963,5 +7190,5 @@ void sqlite3pager_sqlite3PagerSetError( Pager *pPager, int error) { } #endif -/* END CRYPTO */ +/* END SQLCIPHER */ diff --git a/src/pager.h b/src/pager.h index 2b60e05..6f65913 100644 --- a/src/pager.h +++ b/src/pager.h @@ -79,6 +79,12 @@ typedef struct PgHdr DbPage; #define PAGER_JOURNALMODE_WAL 5 /* Use write-ahead logging */ /* +** Flags that make up the mask passed to sqlite3PagerAcquire(). +*/ +#define PAGER_ACQUIRE_NOCONTENT 0x01 /* Do not load data from disk */ +#define PAGER_ACQUIRE_READONLY 0x02 /* Read-only page is acceptable */ + +/* ** The remainder of this file contains the declarations of the functions ** that make up the Pager sub-system API. See source code comments for ** a detailed description of each routine. @@ -102,6 +108,7 @@ void sqlite3PagerSetBusyhandler(Pager*, int(*)(void *), void *); int sqlite3PagerSetPagesize(Pager*, u32*, int); int sqlite3PagerMaxPageCount(Pager*, int); void sqlite3PagerSetCachesize(Pager*, int); +void sqlite3PagerSetMmapLimit(Pager *, sqlite3_int64); void sqlite3PagerShrink(Pager*); void sqlite3PagerSetSafetyLevel(Pager*,int,int,int); int sqlite3PagerLockingMode(Pager *, int); @@ -138,11 +145,14 @@ int sqlite3PagerOpenSavepoint(Pager *pPager, int n); int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint); int sqlite3PagerSharedLock(Pager *pPager); -int sqlite3PagerCheckpoint(Pager *pPager, int, int*, int*); -int sqlite3PagerWalSupported(Pager *pPager); -int sqlite3PagerWalCallback(Pager *pPager); -int sqlite3PagerOpenWal(Pager *pPager, int *pisOpen); -int sqlite3PagerCloseWal(Pager *pPager); +#ifndef SQLITE_OMIT_WAL + int sqlite3PagerCheckpoint(Pager *pPager, int, int*, int*); + int sqlite3PagerWalSupported(Pager *pPager); + int sqlite3PagerWalCallback(Pager *pPager); + int sqlite3PagerOpenWal(Pager *pPager, int *pisOpen); + int sqlite3PagerCloseWal(Pager *pPager); +#endif + #ifdef SQLITE_ENABLE_ZIPVFS int sqlite3PagerWalFramesize(Pager *pPager); #endif @@ -160,6 +170,7 @@ void *sqlite3PagerTempSpace(Pager*); int sqlite3PagerIsMemdb(Pager*); void sqlite3PagerCacheStat(Pager *, int, int, int *); void sqlite3PagerClearCache(Pager *); +int sqlite3SectorSize(sqlite3_file *); /* Functions used to truncate the database file. */ void sqlite3PagerTruncateImage(Pager*,Pgno); diff --git a/src/parse.y b/src/parse.y index 94433d5..8310b26 100644 --- a/src/parse.y +++ b/src/parse.y @@ -435,8 +435,8 @@ oneselect(A) ::= SELECT distinct(D) selcollist(W) from(X) where_opt(Y) // The "distinct" nonterminal is true (1) if the DISTINCT keyword is // present and false (0) if it is not. // -%type distinct {int} -distinct(A) ::= DISTINCT. {A = 1;} +%type distinct {u16} +distinct(A) ::= DISTINCT. {A = SF_Distinct;} distinct(A) ::= ALL. {A = 0;} distinct(A) ::= . {A = 0;} @@ -499,7 +499,8 @@ stl_prefix(A) ::= seltablist(X) joinop(Y). { if( ALWAYS(A && A->nSrc>0) ) A->a[A->nSrc-1].jointype = (u8)Y; } stl_prefix(A) ::= . {A = 0;} -seltablist(A) ::= stl_prefix(X) nm(Y) dbnm(D) as(Z) indexed_opt(I) on_opt(N) using_opt(U). { +seltablist(A) ::= stl_prefix(X) nm(Y) dbnm(D) as(Z) indexed_opt(I) + on_opt(N) using_opt(U). { A = sqlite3SrcListAppendFromTerm(pParse,X,&Y,&D,&Z,0,N,U); sqlite3SrcListIndexedBy(pParse, A, &I); } @@ -512,25 +513,25 @@ seltablist(A) ::= stl_prefix(X) nm(Y) dbnm(D) as(Z) indexed_opt(I) on_opt(N) usi as(Z) on_opt(N) using_opt(U). { if( X==0 && Z.n==0 && N==0 && U==0 ){ A = F; + }else if( F->nSrc==1 ){ + A = sqlite3SrcListAppendFromTerm(pParse,X,0,0,&Z,0,N,U); + if( A ){ + struct SrcList_item *pNew = &A->a[A->nSrc-1]; + struct SrcList_item *pOld = F->a; + pNew->zName = pOld->zName; + pNew->zDatabase = pOld->zDatabase; + pNew->pSelect = pOld->pSelect; + pOld->zName = pOld->zDatabase = 0; + pOld->pSelect = 0; + } + sqlite3SrcListDelete(pParse->db, F); }else{ Select *pSubquery; sqlite3SrcListShiftJoinType(F); - pSubquery = sqlite3SelectNew(pParse,0,F,0,0,0,0,0,0,0); + pSubquery = sqlite3SelectNew(pParse,0,F,0,0,0,0,SF_NestedFrom,0,0); A = sqlite3SrcListAppendFromTerm(pParse,X,0,0,&Z,pSubquery,N,U); } } - - // A seltablist_paren nonterminal represents anything in a FROM that - // is contained inside parentheses. This can be either a subquery or - // a grouping of table and subqueries. - // -// %type seltablist_paren {Select*} -// %destructor seltablist_paren {sqlite3SelectDelete(pParse->db, $$);} -// seltablist_paren(A) ::= select(S). {A = S;} -// seltablist_paren(A) ::= seltablist(F). { -// sqlite3SrcListShiftJoinType(F); -// A = sqlite3SelectNew(pParse,0,F,0,0,0,0,0,0,0); -// } %endif SQLITE_OMIT_SUBQUERY %type dbnm {Token} @@ -653,7 +654,8 @@ where_opt(A) ::= WHERE expr(X). {A = X.pExpr;} ////////////////////////// The UPDATE command //////////////////////////////// // %ifdef SQLITE_ENABLE_UPDATE_DELETE_LIMIT -cmd ::= UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y) where_opt(W) orderby_opt(O) limit_opt(L). { +cmd ::= UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y) where_opt(W) + orderby_opt(O) limit_opt(L). { sqlite3SrcListIndexedBy(pParse, X, &I); sqlite3ExprListCheckLength(pParse,Y,"set list"); W = sqlite3LimitWhere(pParse, X, W, O, L.pLimit, L.pOffset, "UPDATE"); @@ -661,7 +663,8 @@ cmd ::= UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y) where_opt(W) } %endif %ifndef SQLITE_ENABLE_UPDATE_DELETE_LIMIT -cmd ::= UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y) where_opt(W). { +cmd ::= UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y) + where_opt(W). { sqlite3SrcListIndexedBy(pParse, X, &I); sqlite3ExprListCheckLength(pParse,Y,"set list"); sqlite3Update(pParse,X,Y,W,R); @@ -815,7 +818,7 @@ expr(A) ::= VARIABLE(X). { spanSet(&A, &X, &X); } expr(A) ::= expr(E) COLLATE ids(C). { - A.pExpr = sqlite3ExprSetCollByToken(pParse, E.pExpr, &C); + A.pExpr = sqlite3ExprAddCollateToken(pParse, E.pExpr, &C); A.zStart = E.zStart; A.zEnd = &C.z[C.n]; } @@ -1140,22 +1143,14 @@ uniqueflag(A) ::= . {A = OE_None;} idxlist_opt(A) ::= . {A = 0;} idxlist_opt(A) ::= LP idxlist(X) RP. {A = X;} idxlist(A) ::= idxlist(X) COMMA nm(Y) collate(C) sortorder(Z). { - Expr *p = 0; - if( C.n>0 ){ - p = sqlite3Expr(pParse->db, TK_COLUMN, 0); - sqlite3ExprSetCollByToken(pParse, p, &C); - } + Expr *p = sqlite3ExprAddCollateToken(pParse, 0, &C); A = sqlite3ExprListAppend(pParse,X, p); sqlite3ExprListSetName(pParse,A,&Y,1); sqlite3ExprListCheckLength(pParse, A, "index"); if( A ) A->a[A->nExpr-1].sortOrder = (u8)Z; } idxlist(A) ::= nm(Y) collate(C) sortorder(Z). { - Expr *p = 0; - if( C.n>0 ){ - p = sqlite3PExpr(pParse, TK_COLUMN, 0, 0, 0); - sqlite3ExprSetCollByToken(pParse, p, &C); - } + Expr *p = sqlite3ExprAddCollateToken(pParse, 0, &C); A = sqlite3ExprListAppend(pParse,0, p); sqlite3ExprListSetName(pParse, A, &Y, 1); sqlite3ExprListCheckLength(pParse, A, "index"); diff --git a/src/pcache.h b/src/pcache.h index b9135fd..f4d4ad7 100644 --- a/src/pcache.h +++ b/src/pcache.h @@ -53,6 +53,8 @@ struct PgHdr { #define PGHDR_REUSE_UNLIKELY 0x010 /* A hint that reuse is unlikely */ #define PGHDR_DONT_WRITE 0x020 /* Do not write content to disk */ +#define PGHDR_MMAP 0x040 /* This is an mmap page object */ + /* Initialize and shutdown the page cache subsystem */ int sqlite3PcacheInitialize(void); void sqlite3PcacheShutdown(void); diff --git a/src/pragma.c b/src/pragma.c index e5a9ac2..2297716 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -184,6 +184,9 @@ static int flagPragma(Parse *pParse, const char *zLeft, const char *zRight){ { "sql_trace", SQLITE_SqlTrace }, { "vdbe_listing", SQLITE_VdbeListing }, { "vdbe_trace", SQLITE_VdbeTrace }, + { "vdbe_addoptrace", SQLITE_VdbeAddopTrace}, + { "vdbe_debug", SQLITE_SqlTrace | SQLITE_VdbeListing + | SQLITE_VdbeTrace }, #endif #ifndef SQLITE_OMIT_CHECK { "ignore_check_constraints", SQLITE_IgnoreChecks }, @@ -316,12 +319,12 @@ void sqlite3Pragma( int rc; /* return value form SQLITE_FCNTL_PRAGMA */ sqlite3 *db = pParse->db; /* The database connection */ Db *pDb; /* The specific database being pragmaed */ - Vdbe *v = pParse->pVdbe = sqlite3VdbeCreate(db); /* Prepared statement */ -/** BEGIN CRYPTO **/ + Vdbe *v = sqlite3GetVdbe(pParse); /* Prepared statement */ +/* BEGIN SQLCIPHER */ #ifdef SQLITE_HAS_CODEC extern int codec_pragma(sqlite3*, int, Parse *, const char *, const char *); #endif -/** END CRYPTO **/ +/* END SQLCIPHER */ if( v==0 ) return; @@ -363,6 +366,7 @@ void sqlite3Pragma( aFcntl[1] = zLeft; aFcntl[2] = zRight; aFcntl[3] = 0; + db->busyHandler.nBusy = 0; rc = sqlite3_file_control(db, zDb, SQLITE_FCNTL_PRAGMA, (void*)aFcntl); if( rc==SQLITE_OK ){ if( aFcntl[0] ){ @@ -382,13 +386,13 @@ void sqlite3Pragma( pParse->rc = rc; }else -/** BEGIN CRYPTO **/ +/* BEGIN SQLCIPHER */ #ifdef SQLITE_HAS_CODEC if(codec_pragma(db, iDb, pParse, zLeft, zRight)) { /* codec_pragma executes internal */ }else #endif -/** END CRYPTO **/ +/* END SQLCIPHER */ #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) && !defined(SQLITE_OMIT_DEPRECATED) /* @@ -411,11 +415,12 @@ void sqlite3Pragma( static const VdbeOpList getCacheSize[] = { { OP_Transaction, 0, 0, 0}, /* 0 */ { OP_ReadCookie, 0, 1, BTREE_DEFAULT_CACHE_SIZE}, /* 1 */ - { OP_IfPos, 1, 7, 0}, + { OP_IfPos, 1, 8, 0}, { OP_Integer, 0, 2, 0}, { OP_Subtract, 1, 2, 1}, - { OP_IfPos, 1, 7, 0}, + { OP_IfPos, 1, 8, 0}, { OP_Integer, 0, 1, 0}, /* 6 */ + { OP_Noop, 0, 0, 0}, { OP_ResultRow, 1, 1, 0}, }; int addr; @@ -754,6 +759,43 @@ void sqlite3Pragma( }else /* + ** PRAGMA [database.]mmap_size(N) + ** + ** Used to set mapping size limit. The mapping size limit is + ** used to limit the aggregate size of all memory mapped regions of the + ** database file. If this parameter is set to zero, then memory mapping + ** is not used at all. If N is negative, then the default memory map + ** limit determined by sqlite3_config(SQLITE_CONFIG_MMAP_SIZE) is set. + ** The parameter N is measured in bytes. + ** + ** This value is advisory. The underlying VFS is free to memory map + ** as little or as much as it wants. Except, if N is set to 0 then the + ** upper layers will never invoke the xFetch interfaces to the VFS. + */ + if( sqlite3StrICmp(zLeft,"mmap_size")==0 ){ + sqlite3_int64 sz; + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + if( zRight ){ + int ii; + sqlite3Atoi64(zRight, &sz, 1000, SQLITE_UTF8); + if( sz<0 ) sz = sqlite3GlobalConfig.szMmap; + if( pId2->n==0 ) db->szMmap = sz; + for(ii=db->nDb-1; ii>=0; ii--){ + if( db->aDb[ii].pBt && (ii==iDb || pId2->n==0) ){ + sqlite3BtreeSetMmapLimit(db->aDb[ii].pBt, sz); + } + } + } + sz = -1; + if( sqlite3_file_control(db,zDb,SQLITE_FCNTL_MMAP_SIZE,&sz)==SQLITE_OK ){ +#if SQLITE_MAX_MMAP_SIZE==0 + sz = 0; +#endif + returnSingleInt(pParse, "mmap_size", sz); + } + }else + + /* ** PRAGMA temp_store ** PRAGMA temp_store = "default"|"memory"|"file" ** @@ -960,11 +1002,14 @@ void sqlite3Pragma( if( sqlite3ReadSchema(pParse) ) goto pragma_out; pTab = sqlite3FindTable(db, zRight, zDb); if( pTab ){ - int i; + int i, k; int nHidden = 0; Column *pCol; + Index *pPk; + for(pPk=pTab->pIndex; pPk && pPk->autoIndex!=2; pPk=pPk->pNext){} sqlite3VdbeSetNumCols(v, 6); pParse->nMem = 6; + sqlite3CodeVerifySchema(pParse, iDb); sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "cid", SQLITE_STATIC); sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "name", SQLITE_STATIC); sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "type", SQLITE_STATIC); @@ -987,7 +1032,14 @@ void sqlite3Pragma( }else{ sqlite3VdbeAddOp2(v, OP_Null, 0, 5); } - sqlite3VdbeAddOp2(v, OP_Integer, pCol->isPrimKey, 6); + if( (pCol->colFlags & COLFLAG_PRIMKEY)==0 ){ + k = 0; + }else if( pPk==0 ){ + k = 1; + }else{ + for(k=1; ALWAYS(k<=pTab->nCol) && pPk->aiColumn[k-1]!=i; k++){} + } + sqlite3VdbeAddOp2(v, OP_Integer, k, 6); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 6); } } @@ -1003,6 +1055,7 @@ void sqlite3Pragma( pTab = pIdx->pTable; sqlite3VdbeSetNumCols(v, 3); pParse->nMem = 3; + sqlite3CodeVerifySchema(pParse, iDb); sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "seqno", SQLITE_STATIC); sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "cid", SQLITE_STATIC); sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "name", SQLITE_STATIC); @@ -1029,6 +1082,7 @@ void sqlite3Pragma( int i = 0; sqlite3VdbeSetNumCols(v, 3); pParse->nMem = 3; + sqlite3CodeVerifySchema(pParse, iDb); sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "seq", SQLITE_STATIC); sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "name", SQLITE_STATIC); sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "unique", SQLITE_STATIC); @@ -1092,6 +1146,7 @@ void sqlite3Pragma( int i = 0; sqlite3VdbeSetNumCols(v, 8); pParse->nMem = 8; + sqlite3CodeVerifySchema(pParse, iDb); sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "id", SQLITE_STATIC); sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "seq", SQLITE_STATIC); sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "table", SQLITE_STATIC); @@ -1125,6 +1180,122 @@ void sqlite3Pragma( }else #endif /* !defined(SQLITE_OMIT_FOREIGN_KEY) */ +#ifndef SQLITE_OMIT_FOREIGN_KEY +#ifndef SQLITE_OMIT_TRIGGER + if( sqlite3StrICmp(zLeft, "foreign_key_check")==0 ){ + FKey *pFK; /* A foreign key constraint */ + Table *pTab; /* Child table contain "REFERENCES" keyword */ + Table *pParent; /* Parent table that child points to */ + Index *pIdx; /* Index in the parent table */ + int i; /* Loop counter: Foreign key number for pTab */ + int j; /* Loop counter: Field of the foreign key */ + HashElem *k; /* Loop counter: Next table in schema */ + int x; /* result variable */ + int regResult; /* 3 registers to hold a result row */ + int regKey; /* Register to hold key for checking the FK */ + int regRow; /* Registers to hold a row from pTab */ + int addrTop; /* Top of a loop checking foreign keys */ + int addrOk; /* Jump here if the key is OK */ + int *aiCols; /* child to parent column mapping */ + + if( sqlite3ReadSchema(pParse) ) goto pragma_out; + regResult = pParse->nMem+1; + pParse->nMem += 4; + regKey = ++pParse->nMem; + regRow = ++pParse->nMem; + v = sqlite3GetVdbe(pParse); + sqlite3VdbeSetNumCols(v, 4); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "table", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "rowid", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "parent", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 3, COLNAME_NAME, "fkid", SQLITE_STATIC); + sqlite3CodeVerifySchema(pParse, iDb); + k = sqliteHashFirst(&db->aDb[iDb].pSchema->tblHash); + while( k ){ + if( zRight ){ + pTab = sqlite3LocateTable(pParse, 0, zRight, zDb); + k = 0; + }else{ + pTab = (Table*)sqliteHashData(k); + k = sqliteHashNext(k); + } + if( pTab==0 || pTab->pFKey==0 ) continue; + sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); + if( pTab->nCol+regRow>pParse->nMem ) pParse->nMem = pTab->nCol + regRow; + sqlite3OpenTable(pParse, 0, iDb, pTab, OP_OpenRead); + sqlite3VdbeAddOp4(v, OP_String8, 0, regResult, 0, pTab->zName, + P4_TRANSIENT); + for(i=1, pFK=pTab->pFKey; pFK; i++, pFK=pFK->pNextFrom){ + pParent = sqlite3LocateTable(pParse, 0, pFK->zTo, zDb); + if( pParent==0 ) break; + pIdx = 0; + sqlite3TableLock(pParse, iDb, pParent->tnum, 0, pParent->zName); + x = sqlite3FkLocateIndex(pParse, pParent, pFK, &pIdx, 0); + if( x==0 ){ + if( pIdx==0 ){ + sqlite3OpenTable(pParse, i, iDb, pParent, OP_OpenRead); + }else{ + KeyInfo *pKey = sqlite3IndexKeyinfo(pParse, pIdx); + sqlite3VdbeAddOp3(v, OP_OpenRead, i, pIdx->tnum, iDb); + sqlite3VdbeChangeP4(v, -1, (char*)pKey, P4_KEYINFO_HANDOFF); + } + }else{ + k = 0; + break; + } + } + if( pFK ) break; + if( pParse->nTab<i ) pParse->nTab = i; + addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, 0); + for(i=1, pFK=pTab->pFKey; pFK; i++, pFK=pFK->pNextFrom){ + pParent = sqlite3LocateTable(pParse, 0, pFK->zTo, zDb); + assert( pParent!=0 ); + pIdx = 0; + aiCols = 0; + x = sqlite3FkLocateIndex(pParse, pParent, pFK, &pIdx, &aiCols); + assert( x==0 ); + addrOk = sqlite3VdbeMakeLabel(v); + if( pIdx==0 ){ + int iKey = pFK->aCol[0].iFrom; + assert( iKey>=0 && iKey<pTab->nCol ); + if( iKey!=pTab->iPKey ){ + sqlite3VdbeAddOp3(v, OP_Column, 0, iKey, regRow); + sqlite3ColumnDefault(v, pTab, iKey, regRow); + sqlite3VdbeAddOp2(v, OP_IsNull, regRow, addrOk); + sqlite3VdbeAddOp2(v, OP_MustBeInt, regRow, + sqlite3VdbeCurrentAddr(v)+3); + }else{ + sqlite3VdbeAddOp2(v, OP_Rowid, 0, regRow); + } + sqlite3VdbeAddOp3(v, OP_NotExists, i, 0, regRow); + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrOk); + sqlite3VdbeJumpHere(v, sqlite3VdbeCurrentAddr(v)-2); + }else{ + for(j=0; j<pFK->nCol; j++){ + sqlite3ExprCodeGetColumnOfTable(v, pTab, 0, + aiCols ? aiCols[j] : pFK->aCol[0].iFrom, regRow+j); + sqlite3VdbeAddOp2(v, OP_IsNull, regRow+j, addrOk); + } + sqlite3VdbeAddOp3(v, OP_MakeRecord, regRow, pFK->nCol, regKey); + sqlite3VdbeChangeP4(v, -1, + sqlite3IndexAffinityStr(v,pIdx), P4_TRANSIENT); + sqlite3VdbeAddOp4Int(v, OP_Found, i, addrOk, regKey, 0); + } + sqlite3VdbeAddOp2(v, OP_Rowid, 0, regResult+1); + sqlite3VdbeAddOp4(v, OP_String8, 0, regResult+2, 0, + pFK->zTo, P4_TRANSIENT); + sqlite3VdbeAddOp2(v, OP_Integer, i-1, regResult+3); + sqlite3VdbeAddOp2(v, OP_ResultRow, regResult, 4); + sqlite3VdbeResolveLabel(v, addrOk); + sqlite3DbFree(db, aiCols); + } + sqlite3VdbeAddOp2(v, OP_Next, 0, addrTop+1); + sqlite3VdbeJumpHere(v, addrTop); + } + }else +#endif /* !defined(SQLITE_OMIT_TRIGGER) */ +#endif /* !defined(SQLITE_OMIT_FOREIGN_KEY) */ + #ifndef NDEBUG if( sqlite3StrICmp(zLeft, "parser_trace")==0 ){ if( zRight ){ @@ -1246,7 +1417,7 @@ void sqlite3Pragma( sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, sqlite3MPrintf(db, "*** in database %s ***\n", db->aDb[i].zName), P4_DYNAMIC); - sqlite3VdbeAddOp3(v, OP_Move, 2, 4, 1); + sqlite3VdbeAddOp2(v, OP_Move, 2, 4); sqlite3VdbeAddOp3(v, OP_Concat, 4, 3, 2); sqlite3VdbeAddOp2(v, OP_ResultRow, 2, 1); sqlite3VdbeJumpHere(v, addr); @@ -1409,6 +1580,11 @@ void sqlite3Pragma( ** PRAGMA [database.]user_version ** PRAGMA [database.]user_version = <integer> ** + ** PRAGMA [database.]freelist_count = <integer> + ** + ** PRAGMA [database.]application_id + ** PRAGMA [database.]application_id = <integer> + ** ** The pragma's schema_version and user_version are used to set or get ** the value of the schema-version and user-version, respectively. Both ** the schema-version and the user-version are 32-bit signed integers @@ -1430,10 +1606,14 @@ void sqlite3Pragma( if( sqlite3StrICmp(zLeft, "schema_version")==0 || sqlite3StrICmp(zLeft, "user_version")==0 || sqlite3StrICmp(zLeft, "freelist_count")==0 + || sqlite3StrICmp(zLeft, "application_id")==0 ){ int iCookie; /* Cookie index. 1 for schema-cookie, 6 for user-cookie. */ sqlite3VdbeUsesBtree(v, iDb); switch( zLeft[0] ){ + case 'a': case 'A': + iCookie = BTREE_APPLICATION_ID; + break; case 'f': case 'F': iCookie = BTREE_FREE_PAGE_COUNT; break; @@ -1549,6 +1729,22 @@ void sqlite3Pragma( sqlite3_db_release_memory(db); }else + /* + ** PRAGMA busy_timeout + ** PRAGMA busy_timeout = N + ** + ** Call sqlite3_busy_timeout(db, N). Return the current timeout value + ** if one is set. If no busy handler or a different busy handler is set + ** then 0 is returned. Setting the busy_timeout to 0 or negative + ** disables the timeout. + */ + if( sqlite3StrICmp(zLeft, "busy_timeout")==0 ){ + if( zRight ){ + sqlite3_busy_timeout(db, sqlite3Atoi(zRight)); + } + returnSingleInt(pParse, "timeout", db->busyTimeout); + }else + #if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) /* ** Report the current state of file logs for all databases @@ -1564,13 +1760,12 @@ void sqlite3Pragma( sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "status", SQLITE_STATIC); for(i=0; i<db->nDb; i++){ Btree *pBt; - Pager *pPager; const char *zState = "unknown"; int j; if( db->aDb[i].zName==0 ) continue; sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, db->aDb[i].zName, P4_STATIC); pBt = db->aDb[i].pBt; - if( pBt==0 || (pPager = sqlite3BtreePager(pBt))==0 ){ + if( pBt==0 || sqlite3BtreePager(pBt)==0 ){ zState = "closed"; }else if( sqlite3_file_control(db, i ? db->aDb[i].zName : 0, SQLITE_FCNTL_LOCKSTATE, &j)==SQLITE_OK ){ @@ -1607,7 +1802,7 @@ void sqlite3Pragma( }else #endif #if defined(SQLITE_HAS_CODEC) || defined(SQLITE_ENABLE_CEROD) - if( sqlite3StrICmp(zLeft, "activate_extensions")==0 ){ + if( sqlite3StrICmp(zLeft, "activate_extensions")==0 && zRight ){ #ifdef SQLITE_HAS_CODEC if( sqlite3StrNICmp(zRight, "see-", 4)==0 ){ sqlite3_activate_see(&zRight[4]); diff --git a/src/prepare.c b/src/prepare.c index bd2cf7a..d78d83c 100644 --- a/src/prepare.c +++ b/src/prepare.c @@ -134,7 +134,9 @@ int sqlite3InitCallback(void *pInit, int argc, char **argv, char **NotUsed){ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ int rc; int i; +#ifndef SQLITE_OMIT_DEPRECATED int size; +#endif Table *pTab; Db *pDb; char const *azArg[4]; @@ -177,7 +179,7 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ /* zMasterSchema and zInitScript are set to point at the master schema ** and initialisation script appropriate for the database being - ** initialised. zMasterName is the name of the master table. + ** initialized. zMasterName is the name of the master table. */ if( !OMIT_TEMPDB && iDb==1 ){ zMasterSchema = temp_master_schema; @@ -257,11 +259,15 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ */ if( meta[BTREE_TEXT_ENCODING-1] ){ /* text encoding */ if( iDb==0 ){ +#ifndef SQLITE_OMIT_UTF16 u8 encoding; /* If opening the main database, set ENC(db). */ encoding = (u8)meta[BTREE_TEXT_ENCODING-1] & 3; if( encoding==0 ) encoding = SQLITE_UTF8; ENC(db) = encoding; +#else + ENC(db) = SQLITE_UTF8; +#endif }else{ /* If opening an attached database, the encoding much match ENC(db) */ if( meta[BTREE_TEXT_ENCODING-1]!=ENC(db) ){ @@ -398,7 +404,7 @@ int sqlite3Init(sqlite3 *db, char **pzErrMsg){ } } - /* Once all the other databases have been initialised, load the schema + /* Once all the other databases have been initialized, load the schema ** for the TEMP database. This is loaded last, as the TEMP database ** schema may contain references to objects in other databases. */ @@ -421,7 +427,7 @@ int sqlite3Init(sqlite3 *db, char **pzErrMsg){ } /* -** This routine is a no-op if the database schema is already initialised. +** This routine is a no-op if the database schema is already initialized. ** Otherwise, the schema is loaded. An error code is returned. */ int sqlite3ReadSchema(Parse *pParse){ @@ -648,7 +654,6 @@ static int sqlite3Prepare( } #endif - assert( db->init.busy==0 || saveSqlFlag==0 ); if( db->init.busy==0 ){ Vdbe *pVdbe = pParse->pVdbe; sqlite3VdbeSetSql(pVdbe, zSql, (int)(pParse->zTail-zSql), saveSqlFlag); diff --git a/src/resolve.c b/src/resolve.c index b87d231..91efcaa 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -68,6 +68,15 @@ static void incrAggFunctionDepth(Expr *pExpr, int N){ ** from the result in the result-set. We might fix this someday. Or ** then again, we might not... ** +** If the reference is followed by a COLLATE operator, then make sure +** the COLLATE operator is preserved. For example: +** +** SELECT a+b, c+d FROM t1 ORDER BY 1 COLLATE nocase; +** +** Should be transformed into: +** +** SELECT a+b, c+d FROM t1 ORDER BY (a+b) COLLATE nocase; +** ** The nSubquery parameter specifies how many levels of subquery the ** alias is removed from the original expression. The usually value is ** zero but it might be more if the alias is contained within a subquery @@ -91,8 +100,9 @@ static void resolveAlias( assert( pOrig!=0 ); assert( pOrig->flags & EP_Resolved ); db = pParse->db; + pDup = sqlite3ExprDup(db, pOrig, 0); + if( pDup==0 ) return; if( pOrig->op!=TK_COLUMN && zType[0]!='G' ){ - pDup = sqlite3ExprDup(db, pOrig, 0); incrAggFunctionDepth(pDup, nSubquery); pDup = sqlite3PExpr(pParse, TK_AS, pDup, 0, 0); if( pDup==0 ) return; @@ -100,32 +110,26 @@ static void resolveAlias( pEList->a[iCol].iAlias = (u16)(++pParse->nAlias); } pDup->iTable = pEList->a[iCol].iAlias; - }else if( ExprHasProperty(pOrig, EP_IntValue) || pOrig->u.zToken==0 ){ - pDup = sqlite3ExprDup(db, pOrig, 0); - if( pDup==0 ) return; - }else{ - char *zToken = pOrig->u.zToken; - assert( zToken!=0 ); - pOrig->u.zToken = 0; - pDup = sqlite3ExprDup(db, pOrig, 0); - pOrig->u.zToken = zToken; - if( pDup==0 ) return; - assert( (pDup->flags & (EP_Reduced|EP_TokenOnly))==0 ); - pDup->flags2 |= EP2_MallocedToken; - pDup->u.zToken = sqlite3DbStrDup(db, zToken); } - if( pExpr->flags & EP_ExpCollate ){ - pDup->pColl = pExpr->pColl; - pDup->flags |= EP_ExpCollate; + if( pExpr->op==TK_COLLATE ){ + pDup = sqlite3ExprAddCollateString(pParse, pDup, pExpr->u.zToken); } /* Before calling sqlite3ExprDelete(), set the EP_Static flag. This ** prevents ExprDelete() from deleting the Expr structure itself, ** allowing it to be repopulated by the memcpy() on the following line. + ** The pExpr->u.zToken might point into memory that will be freed by the + ** sqlite3DbFree(db, pDup) on the last line of this block, so be sure to + ** make a copy of the token before doing the sqlite3DbFree(). */ ExprSetProperty(pExpr, EP_Static); sqlite3ExprDelete(db, pExpr); memcpy(pExpr, pDup, sizeof(*pExpr)); + if( !ExprHasProperty(pExpr, EP_IntValue) && pExpr->u.zToken!=0 ){ + assert( (pExpr->flags & (EP_Reduced|EP_TokenOnly))==0 ); + pExpr->u.zToken = sqlite3DbStrDup(db, pExpr->u.zToken); + pExpr->flags2 |= EP2_MallocedToken; + } sqlite3DbFree(db, pDup); } @@ -146,6 +150,35 @@ static int nameInUsingClause(IdList *pUsing, const char *zCol){ return 0; } +/* +** Subqueries stores the original database, table and column names for their +** result sets in ExprList.a[].zSpan, in the form "DATABASE.TABLE.COLUMN". +** Check to see if the zSpan given to this routine matches the zDb, zTab, +** and zCol. If any of zDb, zTab, and zCol are NULL then those fields will +** match anything. +*/ +int sqlite3MatchSpanName( + const char *zSpan, + const char *zCol, + const char *zTab, + const char *zDb +){ + int n; + for(n=0; ALWAYS(zSpan[n]) && zSpan[n]!='.'; n++){} + if( zDb && (sqlite3StrNICmp(zSpan, zDb, n)!=0 || zDb[n]!=0) ){ + return 0; + } + zSpan += n+1; + for(n=0; ALWAYS(zSpan[n]) && zSpan[n]!='.'; n++){} + if( zTab && (sqlite3StrNICmp(zSpan, zTab, n)!=0 || zTab[n]!=0) ){ + return 0; + } + zSpan += n+1; + if( zCol && sqlite3StrICmp(zSpan, zCol)!=0 ){ + return 0; + } + return 1; +} /* ** Given the name of a column of the form X.Y.Z or Y.Z or just Z, look up @@ -195,13 +228,27 @@ static int lookupName( assert( pNC ); /* the name context cannot be NULL. */ assert( zCol ); /* The Z in X.Y.Z cannot be NULL */ - assert( ~ExprHasAnyProperty(pExpr, EP_TokenOnly|EP_Reduced) ); + assert( !ExprHasAnyProperty(pExpr, EP_TokenOnly|EP_Reduced) ); /* Initialize the node to no-match */ pExpr->iTable = -1; pExpr->pTab = 0; ExprSetIrreducible(pExpr); + /* Translate the schema name in zDb into a pointer to the corresponding + ** schema. If not found, pSchema will remain NULL and nothing will match + ** resulting in an appropriate error message toward the end of this routine + */ + if( zDb ){ + for(i=0; i<db->nDb; i++){ + assert( db->aDb[i].zName ); + if( sqlite3StrICmp(db->aDb[i].zName,zDb)==0 ){ + pSchema = db->aDb[i].pSchema; + break; + } + } + } + /* Start at the inner-most context and move outward until a match is found */ while( pNC && cnt==0 ){ ExprList *pEList; @@ -210,31 +257,36 @@ static int lookupName( if( pSrcList ){ for(i=0, pItem=pSrcList->a; i<pSrcList->nSrc; i++, pItem++){ Table *pTab; - int iDb; Column *pCol; pTab = pItem->pTab; assert( pTab!=0 && pTab->zName!=0 ); - iDb = sqlite3SchemaToIndex(db, pTab->pSchema); assert( pTab->nCol>0 ); - if( zTab ){ - if( pItem->zAlias ){ - char *zTabName = pItem->zAlias; - if( sqlite3StrICmp(zTabName, zTab)!=0 ) continue; - }else{ - char *zTabName = pTab->zName; - if( NEVER(zTabName==0) || sqlite3StrICmp(zTabName, zTab)!=0 ){ - continue; - } - if( zDb!=0 && sqlite3StrICmp(db->aDb[iDb].zName, zDb)!=0 ){ - continue; + if( pItem->pSelect && (pItem->pSelect->selFlags & SF_NestedFrom)!=0 ){ + int hit = 0; + pEList = pItem->pSelect->pEList; + for(j=0; j<pEList->nExpr; j++){ + if( sqlite3MatchSpanName(pEList->a[j].zSpan, zCol, zTab, zDb) ){ + cnt++; + cntTab = 2; + pMatch = pItem; + pExpr->iColumn = j; + hit = 1; } } + if( hit || zTab==0 ) continue; + } + if( zDb && pTab->pSchema!=pSchema ){ + continue; + } + if( zTab ){ + const char *zTabName = pItem->zAlias ? pItem->zAlias : pTab->zName; + assert( zTabName!=0 ); + if( sqlite3StrICmp(zTabName, zTab)!=0 ){ + continue; + } } if( 0==(cntTab++) ){ - pExpr->iTable = pItem->iCursor; - pExpr->pTab = pTab; - pSchema = pTab->pSchema; pMatch = pItem; } for(j=0, pCol=pTab->aCol; j<pTab->nCol; j++, pCol++){ @@ -248,17 +300,19 @@ static int lookupName( if( nameInUsingClause(pItem->pUsing, zCol) ) continue; } cnt++; - pExpr->iTable = pItem->iCursor; - pExpr->pTab = pTab; pMatch = pItem; - pSchema = pTab->pSchema; /* Substitute the rowid (column -1) for the INTEGER PRIMARY KEY */ pExpr->iColumn = j==pTab->iPKey ? -1 : (i16)j; break; } } } - } + if( pMatch ){ + pExpr->iTable = pMatch->iCursor; + pExpr->pTab = pMatch->pTab; + pSchema = pExpr->pTab->pSchema; + } + } /* if( pSrcList ) */ #ifndef SQLITE_OMIT_TRIGGER /* If we have not already resolved the name, then maybe @@ -334,7 +388,10 @@ static int lookupName( ** Note that the expression in the result set should have already been ** resolved by the time the WHERE clause is resolved. */ - if( cnt==0 && (pEList = pNC->pEList)!=0 && zTab==0 ){ + if( (pEList = pNC->pEList)!=0 + && zTab==0 + && ((pNC->ncFlags & NC_AsMaybe)==0 || cnt==0) + ){ for(j=0; j<pEList->nExpr; j++){ char *zAs = pEList->a[j].zName; if( zAs!=0 && sqlite3StrICmp(zAs, zCol)==0 ){ @@ -425,7 +482,9 @@ static int lookupName( lookupname_end: if( cnt==1 ){ assert( pNC!=0 ); - sqlite3AuthRead(pParse, pExpr, pSchema, pNC->pSrcList); + if( pExpr->op!=TK_AS ){ + sqlite3AuthRead(pParse, pExpr, pSchema, pNC->pSrcList); + } /* Increment the nRef value on all name contexts from TopNC up to ** the point where the name matched. */ for(;;){ @@ -593,7 +652,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ sqlite3ErrorMsg(pParse, "misuse of aggregate function %.*s()", nId,zId); pNC->nErr++; is_agg = 0; - }else if( no_such_func ){ + }else if( no_such_func && pParse->db->init.busy==0 ){ sqlite3ErrorMsg(pParse, "no such function: %.*s", nId, zId); pNC->nErr++; }else if( wrong_num_args ){ @@ -812,7 +871,7 @@ static int resolveCompoundOrderBy( int iCol = -1; Expr *pE, *pDup; if( pItem->done ) continue; - pE = pItem->pExpr; + pE = sqlite3ExprSkipCollate(pItem->pExpr); if( sqlite3ExprIsInteger(pE, &iCol) ){ if( iCol<=0 || iCol>pEList->nExpr ){ resolveOutOfRangeError(pParse, "ORDER", i+1, pEList->nExpr); @@ -830,14 +889,20 @@ static int resolveCompoundOrderBy( } } if( iCol>0 ){ - CollSeq *pColl = pE->pColl; - int flags = pE->flags & EP_ExpCollate; + /* Convert the ORDER BY term into an integer column number iCol, + ** taking care to preserve the COLLATE clause if it exists */ + Expr *pNew = sqlite3Expr(db, TK_INTEGER, 0); + if( pNew==0 ) return 1; + pNew->flags |= EP_IntValue; + pNew->u.iValue = iCol; + if( pItem->pExpr==pE ){ + pItem->pExpr = pNew; + }else{ + assert( pItem->pExpr->op==TK_COLLATE ); + assert( pItem->pExpr->pLeft==pE ); + pItem->pExpr->pLeft = pNew; + } sqlite3ExprDelete(db, pE); - pItem->pExpr = pE = sqlite3Expr(db, TK_INTEGER, 0); - if( pE==0 ) return 1; - pE->pColl = pColl; - pE->flags |= EP_IntValue | flags; - pE->u.iValue = iCol; pItem->iOrderByCol = (u16)iCol; pItem->done = 1; }else{ @@ -942,11 +1007,11 @@ static int resolveOrderGroupBy( pItem->iOrderByCol = (u16)iCol; continue; } - if( sqlite3ExprIsInteger(pE, &iCol) ){ + if( sqlite3ExprIsInteger(sqlite3ExprSkipCollate(pE), &iCol) ){ /* The ORDER BY term is an integer constant. Again, set the column ** number so that sqlite3ResolveOrderGroupBy() will convert the ** order-by term to a copy of the result-set expression */ - if( iCol<1 ){ + if( iCol<1 || iCol>0xffff ){ resolveOutOfRangeError(pParse, zType, i+1, nResult); return 1; } @@ -1023,23 +1088,6 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ return WRC_Abort; } - /* Set up the local name-context to pass to sqlite3ResolveExprNames() to - ** resolve the result-set expression list. - */ - sNC.ncFlags = NC_AllowAgg; - sNC.pSrcList = p->pSrc; - sNC.pNext = pOuterNC; - - /* Resolve names in the result set. */ - pEList = p->pEList; - assert( pEList!=0 ); - for(i=0; i<pEList->nExpr; i++){ - Expr *pX = pEList->a[i].pExpr; - if( sqlite3ResolveExprNames(&sNC, pX) ){ - return WRC_Abort; - } - } - /* Recursively resolve names in all subqueries */ for(i=0; i<p->pSrc->nSrc; i++){ @@ -1067,6 +1115,23 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ } } + /* Set up the local name-context to pass to sqlite3ResolveExprNames() to + ** resolve the result-set expression list. + */ + sNC.ncFlags = NC_AllowAgg; + sNC.pSrcList = p->pSrc; + sNC.pNext = pOuterNC; + + /* Resolve names in the result set. */ + pEList = p->pEList; + assert( pEList!=0 ); + for(i=0; i<pEList->nExpr; i++){ + Expr *pX = pEList->a[i].pExpr; + if( sqlite3ResolveExprNames(&sNC, pX) ){ + return WRC_Abort; + } + } + /* If there are no aggregate functions in the result-set, and no GROUP BY ** expression, do not allow aggregates in any of the other expressions. */ @@ -1094,11 +1159,10 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ ** re-evaluated for each reference to it. */ sNC.pEList = p->pEList; - if( sqlite3ResolveExprNames(&sNC, p->pWhere) || - sqlite3ResolveExprNames(&sNC, p->pHaving) - ){ - return WRC_Abort; - } + sNC.ncFlags |= NC_AsMaybe; + if( sqlite3ResolveExprNames(&sNC, p->pHaving) ) return WRC_Abort; + if( sqlite3ResolveExprNames(&sNC, p->pWhere) ) return WRC_Abort; + sNC.ncFlags &= ~NC_AsMaybe; /* The ORDER BY and GROUP BY clauses may not refer to terms in ** outer queries @@ -1219,6 +1283,7 @@ int sqlite3ResolveExprNames( #endif savedHasAgg = pNC->ncFlags & NC_HasAgg; pNC->ncFlags &= ~NC_HasAgg; + memset(&w, 0, sizeof(w)); w.xExprCallback = resolveExprStep; w.xSelectCallback = resolveSelectStep; w.pParse = pNC->pParse; @@ -1259,6 +1324,7 @@ void sqlite3ResolveSelectNames( Walker w; assert( p!=0 ); + memset(&w, 0, sizeof(w)); w.xExprCallback = resolveExprStep; w.xSelectCallback = resolveSelectStep; w.pParse = pParse; diff --git a/src/select.c b/src/select.c index 6ec9da3..f3f1490 100644 --- a/src/select.c +++ b/src/select.c @@ -55,7 +55,7 @@ Select *sqlite3SelectNew( ExprList *pGroupBy, /* the GROUP BY clause */ Expr *pHaving, /* the HAVING clause */ ExprList *pOrderBy, /* the ORDER BY clause */ - int isDistinct, /* true if the DISTINCT keyword is present */ + u16 selFlags, /* Flag parameters, such as SF_Distinct */ Expr *pLimit, /* LIMIT value. NULL means not used */ Expr *pOffset /* OFFSET value. NULL means no offset */ ){ @@ -79,7 +79,7 @@ Select *sqlite3SelectNew( pNew->pGroupBy = pGroupBy; pNew->pHaving = pHaving; pNew->pOrderBy = pOrderBy; - pNew->selFlags = isDistinct ? SF_Distinct : 0; + pNew->selFlags = selFlags; pNew->op = TK_SELECT; pNew->pLimit = pLimit; pNew->pOffset = pOffset; @@ -526,6 +526,19 @@ static int checkForMultiColumnSelectError( #endif /* +** An instance of the following object is used to record information about +** how to process the DISTINCT keyword, to simplify passing that information +** into the selectInnerLoop() routine. +*/ +typedef struct DistinctCtx DistinctCtx; +struct DistinctCtx { + u8 isTnct; /* True if the DISTINCT keyword is present */ + u8 eTnctType; /* One of the WHERE_DISTINCT_* operators */ + int tabTnct; /* Ephemeral table used for DISTINCT processing */ + int addrTnct; /* Address of OP_OpenEphemeral opcode for tabTnct */ +}; + +/* ** This routine generates the code for the inside of the inner loop ** of a SELECT. ** @@ -541,7 +554,7 @@ static void selectInnerLoop( int srcTab, /* Pull data from this table */ int nColumn, /* Number of columns in the source table */ ExprList *pOrderBy, /* If not NULL, sort results using this key */ - int distinct, /* If >=0, make sure results are distinct */ + DistinctCtx *pDistinct, /* If not NULL, info on how to process DISTINCT */ SelectDest *pDest, /* How to dispose of the results */ int iContinue, /* Jump here to continue with next row */ int iBreak /* Jump here to break out of the inner loop */ @@ -557,7 +570,7 @@ static void selectInnerLoop( assert( v ); if( NEVER(v==0) ) return; assert( pEList!=0 ); - hasDistinct = distinct>=0; + hasDistinct = pDistinct ? pDistinct->eTnctType : WHERE_DISTINCT_NOOP; if( pOrderBy==0 && !hasDistinct ){ codeOffset(v, p, iContinue); } @@ -597,7 +610,55 @@ static void selectInnerLoop( if( hasDistinct ){ assert( pEList!=0 ); assert( pEList->nExpr==nColumn ); - codeDistinct(pParse, distinct, iContinue, nColumn, regResult); + switch( pDistinct->eTnctType ){ + case WHERE_DISTINCT_ORDERED: { + VdbeOp *pOp; /* No longer required OpenEphemeral instr. */ + int iJump; /* Jump destination */ + int regPrev; /* Previous row content */ + + /* Allocate space for the previous row */ + regPrev = pParse->nMem+1; + pParse->nMem += nColumn; + + /* Change the OP_OpenEphemeral coded earlier to an OP_Null + ** sets the MEM_Cleared bit on the first register of the + ** previous value. This will cause the OP_Ne below to always + ** fail on the first iteration of the loop even if the first + ** row is all NULLs. + */ + sqlite3VdbeChangeToNoop(v, pDistinct->addrTnct); + pOp = sqlite3VdbeGetOp(v, pDistinct->addrTnct); + pOp->opcode = OP_Null; + pOp->p1 = 1; + pOp->p2 = regPrev; + + iJump = sqlite3VdbeCurrentAddr(v) + nColumn; + for(i=0; i<nColumn; i++){ + CollSeq *pColl = sqlite3ExprCollSeq(pParse, pEList->a[i].pExpr); + if( i<nColumn-1 ){ + sqlite3VdbeAddOp3(v, OP_Ne, regResult+i, iJump, regPrev+i); + }else{ + sqlite3VdbeAddOp3(v, OP_Eq, regResult+i, iContinue, regPrev+i); + } + sqlite3VdbeChangeP4(v, -1, (const char *)pColl, P4_COLLSEQ); + sqlite3VdbeChangeP5(v, SQLITE_NULLEQ); + } + assert( sqlite3VdbeCurrentAddr(v)==iJump ); + sqlite3VdbeAddOp3(v, OP_Copy, regResult, regPrev, nColumn-1); + break; + } + + case WHERE_DISTINCT_UNIQUE: { + sqlite3VdbeChangeToNoop(v, pDistinct->addrTnct); + break; + } + + default: { + assert( pDistinct->eTnctType==WHERE_DISTINCT_UNORDERED ); + codeDistinct(pParse, pDistinct->tabTnct, iContinue, nColumn, regResult); + break; + } + } if( pOrderBy==0 ){ codeOffset(v, p, iContinue); } @@ -655,7 +716,8 @@ static void selectInnerLoop( */ case SRT_Set: { assert( nColumn==1 ); - p->affinity = sqlite3CompareAffinity(pEList->a[0].pExpr, pDest->affSdst); + pDest->affSdst = + sqlite3CompareAffinity(pEList->a[0].pExpr, pDest->affSdst); if( pOrderBy ){ /* At first glance you would think we could optimize out the ** ORDER BY in this case since the order of entries in the set @@ -664,7 +726,7 @@ static void selectInnerLoop( pushOntoSorter(pParse, pOrderBy, p, regResult); }else{ int r1 = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp4(v, OP_MakeRecord, regResult, 1, r1, &p->affinity, 1); + sqlite3VdbeAddOp4(v, OP_MakeRecord, regResult,1,r1, &pDest->affSdst, 1); sqlite3ExprCacheAffinityChange(pParse, regResult, 1); sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm, r1); sqlite3ReleaseTempReg(pParse, r1); @@ -931,7 +993,8 @@ static void generateSortTail( #ifndef SQLITE_OMIT_SUBQUERY case SRT_Set: { assert( nColumn==1 ); - sqlite3VdbeAddOp4(v, OP_MakeRecord, regRow, 1, regRowid, &p->affinity, 1); + sqlite3VdbeAddOp4(v, OP_MakeRecord, regRow, 1, regRowid, + &pDest->affSdst, 1); sqlite3ExprCacheAffinityChange(pParse, regRow, 1); sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm, regRowid); break; @@ -1246,7 +1309,7 @@ static void generateColumnNames( static int selectColumnsFromExprList( Parse *pParse, /* Parsing context */ ExprList *pEList, /* Expr list from which to derive column names */ - int *pnCol, /* Write the number of columns here */ + i16 *pnCol, /* Write the number of columns here */ Column **paCol /* Write the new column list here */ ){ sqlite3 *db = pParse->db; /* Database connection */ @@ -1272,9 +1335,7 @@ static int selectColumnsFromExprList( for(i=0, pCol=aCol; i<nCol; i++, pCol++){ /* Get an appropriate name for the column */ - p = pEList->a[i].pExpr; - assert( p->pRight==0 || ExprHasProperty(p->pRight, EP_IntValue) - || p->pRight->u.zToken==0 || p->pRight->u.zToken[0]!=0 ); + p = sqlite3ExprSkipCollate(pEList->a[i].pExpr); if( (zName = pEList->a[i].zName)!=0 ){ /* If the column contains an "AS <name>" phrase, use <name> as the name */ zName = sqlite3DbStrDup(db, zName); @@ -1312,6 +1373,9 @@ static int selectColumnsFromExprList( for(j=cnt=0; j<i; j++){ if( sqlite3StrICmp(aCol[j].zName, zName)==0 ){ char *zNewName; + int k; + for(k=nName-1; k>1 && sqlite3Isdigit(zName[k]); k--){} + if( zName[k]==':' ) nName = k; zName[nName] = 0; zNewName = sqlite3MPrintf(db, "%s:%d", zName, ++cnt); sqlite3DbFree(db, zName); @@ -1643,6 +1707,8 @@ static int multiSelect( int addr = 0; int nLimit; assert( !pPrior->pLimit ); + pPrior->iLimit = p->iLimit; + pPrior->iOffset = p->iOffset; pPrior->pLimit = p->pLimit; pPrior->pOffset = p->pOffset; explainSetInteger(iSub1, pParse->iNextSelectId); @@ -1768,7 +1834,7 @@ static int multiSelect( sqlite3VdbeAddOp2(v, OP_Rewind, unionTab, iBreak); iStart = sqlite3VdbeCurrentAddr(v); selectInnerLoop(pParse, p, p->pEList, unionTab, p->pEList->nExpr, - 0, -1, &dest, iCont, iBreak); + 0, 0, &dest, iCont, iBreak); sqlite3VdbeResolveLabel(v, iCont); sqlite3VdbeAddOp2(v, OP_Next, unionTab, iStart); sqlite3VdbeResolveLabel(v, iBreak); @@ -1846,7 +1912,7 @@ static int multiSelect( sqlite3VdbeAddOp4Int(v, OP_NotFound, tab2, iCont, r1, 0); sqlite3ReleaseTempReg(pParse, r1); selectInnerLoop(pParse, p, p->pEList, tab1, p->pEList->nExpr, - 0, -1, &dest, iCont, iBreak); + 0, 0, &dest, iCont, iBreak); sqlite3VdbeResolveLabel(v, iCont); sqlite3VdbeAddOp2(v, OP_Next, tab1, iStart); sqlite3VdbeResolveLabel(v, iBreak); @@ -1892,6 +1958,7 @@ static int multiSelect( *apColl = db->pDfltColl; } } + pKeyInfo->aSortOrder = (u8*)apColl; for(pLoop=p; pLoop; pLoop=pLoop->pPrior){ for(i=0; i<2; i++){ @@ -1965,7 +2032,7 @@ static int generateOutputSubroutine( (char*)pKeyInfo, p4type); sqlite3VdbeAddOp3(v, OP_Jump, j2+2, iContinue, j2+2); sqlite3VdbeJumpHere(v, j1); - sqlite3ExprCodeCopy(pParse, pIn->iSdst, regPrev+1, pIn->nSdst); + sqlite3VdbeAddOp3(v, OP_Copy, pIn->iSdst, regPrev+1, pIn->nSdst-1); sqlite3VdbeAddOp2(v, OP_Integer, 1, regPrev); } if( pParse->db->mallocFailed ) return 0; @@ -2000,10 +2067,10 @@ static int generateOutputSubroutine( case SRT_Set: { int r1; assert( pIn->nSdst==1 ); - p->affinity = + pDest->affSdst = sqlite3CompareAffinity(p->pEList->a[0].pExpr, pDest->affSdst); r1 = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp4(v, OP_MakeRecord, pIn->iSdst, 1, r1, &p->affinity, 1); + sqlite3VdbeAddOp4(v, OP_MakeRecord, pIn->iSdst, 1, r1, &pDest->affSdst,1); sqlite3ExprCacheAffinityChange(pParse, pIn->iSdst, 1); sqlite3VdbeAddOp2(v, OP_IdxInsert, pDest->iSDParm, r1); sqlite3ReleaseTempReg(pParse, r1); @@ -2269,12 +2336,13 @@ static int multiSelectOrderBy( for(i=0; i<nOrderBy; i++){ CollSeq *pColl; Expr *pTerm = pOrderBy->a[i].pExpr; - if( pTerm->flags & EP_ExpCollate ){ - pColl = pTerm->pColl; + if( pTerm->flags & EP_Collate ){ + pColl = sqlite3ExprCollSeq(pParse, pTerm); }else{ pColl = multiSelectCollSeq(pParse, p, aPermute[i]); - pTerm->flags |= EP_ExpCollate; - pTerm->pColl = pColl; + if( pColl==0 ) pColl = db->pDfltColl; + pOrderBy->a[i].pExpr = + sqlite3ExprAddCollateString(pParse, pTerm, pColl->zName); } pKeyMerge->aColl[i] = pColl; pKeyMerge->aSortOrder[i] = pOrderBy->a[i].sortOrder; @@ -2298,7 +2366,8 @@ static int multiSelectOrderBy( }else{ int nExpr = p->pEList->nExpr; assert( nOrderBy>=nExpr || db->mallocFailed ); - regPrev = sqlite3GetTempRange(pParse, nExpr+1); + regPrev = pParse->nMem+1; + pParse->nMem += nExpr+1; sqlite3VdbeAddOp2(v, OP_Integer, 0, regPrev); pKeyDup = sqlite3DbMallocZero(db, sizeof(*pKeyDup) + nExpr*(sizeof(CollSeq*)+1) ); @@ -2477,14 +2546,9 @@ static int multiSelectOrderBy( sqlite3VdbeAddOp4(v, OP_Permutation, 0, 0, 0, (char*)aPermute, P4_INTARRAY); sqlite3VdbeAddOp4(v, OP_Compare, destA.iSdst, destB.iSdst, nOrderBy, (char*)pKeyMerge, P4_KEYINFO_HANDOFF); + sqlite3VdbeChangeP5(v, OPFLAG_PERMUTE); sqlite3VdbeAddOp3(v, OP_Jump, addrAltB, addrAeqB, addrAgtB); - /* Release temporary registers - */ - if( regPrev ){ - sqlite3ReleaseTempRange(pParse, regPrev, nOrderBy+1); - } - /* Jump to the this point in order to terminate the query. */ sqlite3VdbeResolveLabel(v, labelEnd); @@ -2544,9 +2608,6 @@ static Expr *substExpr( assert( pEList!=0 && pExpr->iColumn<pEList->nExpr ); assert( pExpr->pLeft==0 && pExpr->pRight==0 ); pNew = sqlite3ExprDup(db, pEList->a[pExpr->iColumn].pExpr, 0); - if( pNew && pExpr->pColl ){ - pNew->pColl = pExpr->pColl; - } sqlite3ExprDelete(db, pExpr); pExpr = pNew; } @@ -2745,7 +2806,7 @@ static int flattenSubquery( */ assert( p!=0 ); assert( p->pPrior==0 ); /* Unable to flatten compound queries */ - if( db->flags & SQLITE_QueryFlattener ) return 0; + if( OptimizationDisabled(db, SQLITE_QueryFlattener) ) return 0; pSrc = p->pSrc; assert( pSrc && iFrom>=0 && iFrom<pSrc->nSrc ); pSubitem = &pSrc->a[iFrom]; @@ -2899,12 +2960,15 @@ static int flattenSubquery( Select *pNew; ExprList *pOrderBy = p->pOrderBy; Expr *pLimit = p->pLimit; + Expr *pOffset = p->pOffset; Select *pPrior = p->pPrior; p->pOrderBy = 0; p->pSrc = 0; p->pPrior = 0; p->pLimit = 0; + p->pOffset = 0; pNew = sqlite3SelectDup(db, p, 0); + p->pOffset = pOffset; p->pLimit = pLimit; p->pOrderBy = pOrderBy; p->pSrc = pSrc; @@ -3034,10 +3098,9 @@ static int flattenSubquery( pList = pParent->pEList; for(i=0; i<pList->nExpr; i++){ if( pList->a[i].zName==0 ){ - const char *zSpan = pList->a[i].zSpan; - if( ALWAYS(zSpan) ){ - pList->a[i].zName = sqlite3DbStrDup(db, zSpan); - } + char *zName = sqlite3DbStrDup(db, pList->a[i].zSpan); + sqlite3Dequote(zName); + pList->a[i].zName = zName; } } substExprList(db, pParent->pEList, iParent, pSub->pEList); @@ -3098,34 +3161,43 @@ static int flattenSubquery( #endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ /* -** Analyze the SELECT statement passed as an argument to see if it -** is a min() or max() query. Return WHERE_ORDERBY_MIN or WHERE_ORDERBY_MAX if -** it is, or 0 otherwise. At present, a query is considered to be -** a min()/max() query if: +** Based on the contents of the AggInfo structure indicated by the first +** argument, this function checks if the following are true: +** +** * the query contains just a single aggregate function, +** * the aggregate function is either min() or max(), and +** * the argument to the aggregate function is a column value. ** -** 1. There is a single object in the FROM clause. +** If all of the above are true, then WHERE_ORDERBY_MIN or WHERE_ORDERBY_MAX +** is returned as appropriate. Also, *ppMinMax is set to point to the +** list of arguments passed to the aggregate before returning. ** -** 2. There is a single expression in the result set, and it is -** either min(x) or max(x), where x is a column reference. +** Or, if the conditions above are not met, *ppMinMax is set to 0 and +** WHERE_ORDERBY_NORMAL is returned. */ -static u8 minMaxQuery(Select *p){ - Expr *pExpr; - ExprList *pEList = p->pEList; - - if( pEList->nExpr!=1 ) return WHERE_ORDERBY_NORMAL; - pExpr = pEList->a[0].pExpr; - if( pExpr->op!=TK_AGG_FUNCTION ) return 0; - if( NEVER(ExprHasProperty(pExpr, EP_xIsSelect)) ) return 0; - pEList = pExpr->x.pList; - if( pEList==0 || pEList->nExpr!=1 ) return 0; - if( pEList->a[0].pExpr->op!=TK_AGG_COLUMN ) return WHERE_ORDERBY_NORMAL; - assert( !ExprHasProperty(pExpr, EP_IntValue) ); - if( sqlite3StrICmp(pExpr->u.zToken,"min")==0 ){ - return WHERE_ORDERBY_MIN; - }else if( sqlite3StrICmp(pExpr->u.zToken,"max")==0 ){ - return WHERE_ORDERBY_MAX; +static u8 minMaxQuery(AggInfo *pAggInfo, ExprList **ppMinMax){ + int eRet = WHERE_ORDERBY_NORMAL; /* Return value */ + + *ppMinMax = 0; + if( pAggInfo->nFunc==1 ){ + Expr *pExpr = pAggInfo->aFunc[0].pExpr; /* Aggregate function */ + ExprList *pEList = pExpr->x.pList; /* Arguments to agg function */ + + assert( pExpr->op==TK_AGG_FUNCTION ); + if( pEList && pEList->nExpr==1 && pEList->a[0].pExpr->op==TK_AGG_COLUMN ){ + const char *zFunc = pExpr->u.zToken; + if( sqlite3StrICmp(zFunc, "min")==0 ){ + eRet = WHERE_ORDERBY_MIN; + *ppMinMax = pEList; + }else if( sqlite3StrICmp(zFunc, "max")==0 ){ + eRet = WHERE_ORDERBY_MAX; + *ppMinMax = pEList; + } + } } - return WHERE_ORDERBY_NORMAL; + + assert( *ppMinMax==0 || (*ppMinMax)->nExpr==1 ); + return eRet; } /* @@ -3188,6 +3260,69 @@ int sqlite3IndexedByLookup(Parse *pParse, struct SrcList_item *pFrom){ } return SQLITE_OK; } +/* +** Detect compound SELECT statements that use an ORDER BY clause with +** an alternative collating sequence. +** +** SELECT ... FROM t1 EXCEPT SELECT ... FROM t2 ORDER BY .. COLLATE ... +** +** These are rewritten as a subquery: +** +** SELECT * FROM (SELECT ... FROM t1 EXCEPT SELECT ... FROM t2) +** ORDER BY ... COLLATE ... +** +** This transformation is necessary because the multiSelectOrderBy() routine +** above that generates the code for a compound SELECT with an ORDER BY clause +** uses a merge algorithm that requires the same collating sequence on the +** result columns as on the ORDER BY clause. See ticket +** http://www.sqlite.org/src/info/6709574d2a +** +** This transformation is only needed for EXCEPT, INTERSECT, and UNION. +** The UNION ALL operator works fine with multiSelectOrderBy() even when +** there are COLLATE terms in the ORDER BY. +*/ +static int convertCompoundSelectToSubquery(Walker *pWalker, Select *p){ + int i; + Select *pNew; + Select *pX; + sqlite3 *db; + struct ExprList_item *a; + SrcList *pNewSrc; + Parse *pParse; + Token dummy; + + if( p->pPrior==0 ) return WRC_Continue; + if( p->pOrderBy==0 ) return WRC_Continue; + for(pX=p; pX && (pX->op==TK_ALL || pX->op==TK_SELECT); pX=pX->pPrior){} + if( pX==0 ) return WRC_Continue; + a = p->pOrderBy->a; + for(i=p->pOrderBy->nExpr-1; i>=0; i--){ + if( a[i].pExpr->flags & EP_Collate ) break; + } + if( i<0 ) return WRC_Continue; + + /* If we reach this point, that means the transformation is required. */ + + pParse = pWalker->pParse; + db = pParse->db; + pNew = sqlite3DbMallocZero(db, sizeof(*pNew) ); + if( pNew==0 ) return WRC_Abort; + memset(&dummy, 0, sizeof(dummy)); + pNewSrc = sqlite3SrcListAppendFromTerm(pParse,0,0,0,&dummy,pNew,0,0); + if( pNewSrc==0 ) return WRC_Abort; + *pNew = *p; + p->pSrc = pNewSrc; + p->pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db, TK_ALL, 0)); + p->op = TK_SELECT; + p->pWhere = 0; + pNew->pGroupBy = 0; + pNew->pHaving = 0; + pNew->pOrderBy = 0; + p->pPrior = 0; + pNew->pLimit = 0; + pNew->pOffset = 0; + return WRC_Continue; +} /* ** This routine is a Walker callback for "expanding" a SELECT statement. @@ -3220,14 +3355,16 @@ static int selectExpander(Walker *pWalker, Select *p){ ExprList *pEList; struct SrcList_item *pFrom; sqlite3 *db = pParse->db; + Expr *pE, *pRight, *pExpr; + u16 selFlags = p->selFlags; + p->selFlags |= SF_Expanded; if( db->mallocFailed ){ return WRC_Abort; } - if( NEVER(p->pSrc==0) || (p->selFlags & SF_Expanded)!=0 ){ + if( NEVER(p->pSrc==0) || (selFlags & SF_Expanded)!=0 ){ return WRC_Prune; } - p->selFlags |= SF_Expanded; pTabList = p->pSrc; pEList = p->pEList; @@ -3268,9 +3405,14 @@ static int selectExpander(Walker *pWalker, Select *p){ }else{ /* An ordinary table or view name in the FROM clause */ assert( pFrom->pTab==0 ); - pFrom->pTab = pTab = - sqlite3LocateTable(pParse,0,pFrom->zName,pFrom->zDatabase); + pFrom->pTab = pTab = sqlite3LocateTableItem(pParse, 0, pFrom); if( pTab==0 ) return WRC_Abort; + if( pTab->nRef==0xffff ){ + sqlite3ErrorMsg(pParse, "too many references to \"%s\": max 65535", + pTab->zName); + pFrom->pTab = 0; + return WRC_Abort; + } pTab->nRef++; #if !defined(SQLITE_OMIT_VIEW) || !defined (SQLITE_OMIT_VIRTUALTABLE) if( pTab->pSelect || IsVirtual(pTab) ){ @@ -3306,7 +3448,7 @@ static int selectExpander(Walker *pWalker, Select *p){ ** that need expanding. */ for(k=0; k<pEList->nExpr; k++){ - Expr *pE = pEList->a[k].pExpr; + pE = pEList->a[k].pExpr; if( pE->op==TK_ALL ) break; assert( pE->op!=TK_DOT || pE->pRight!=0 ); assert( pE->op!=TK_DOT || (pE->pLeft!=0 && pE->pLeft->op==TK_ID) ); @@ -3324,10 +3466,18 @@ static int selectExpander(Walker *pWalker, Select *p){ int longNames = (flags & SQLITE_FullColNames)!=0 && (flags & SQLITE_ShortColNames)==0; + /* When processing FROM-clause subqueries, it is always the case + ** that full_column_names=OFF and short_column_names=ON. The + ** sqlite3ResultSetOfSelect() routine makes it so. */ + assert( (p->selFlags & SF_NestedFrom)==0 + || ((flags & SQLITE_FullColNames)==0 && + (flags & SQLITE_ShortColNames)!=0) ); + for(k=0; k<pEList->nExpr; k++){ - Expr *pE = a[k].pExpr; - assert( pE->op!=TK_DOT || pE->pRight!=0 ); - if( pE->op!=TK_ALL && (pE->op!=TK_DOT || pE->pRight->op!=TK_ALL) ){ + pE = a[k].pExpr; + pRight = pE->pRight; + assert( pE->op!=TK_DOT || pRight!=0 ); + if( pE->op!=TK_ALL && (pE->op!=TK_DOT || pRight->op!=TK_ALL) ){ /* This particular expression does not need to be expanded. */ pNew = sqlite3ExprListAppend(pParse, pNew, a[k].pExpr); @@ -3342,32 +3492,43 @@ static int selectExpander(Walker *pWalker, Select *p){ /* This expression is a "*" or a "TABLE.*" and needs to be ** expanded. */ int tableSeen = 0; /* Set to 1 when TABLE matches */ - char *zTName; /* text of name of TABLE */ + char *zTName = 0; /* text of name of TABLE */ if( pE->op==TK_DOT ){ assert( pE->pLeft!=0 ); assert( !ExprHasProperty(pE->pLeft, EP_IntValue) ); zTName = pE->pLeft->u.zToken; - }else{ - zTName = 0; } for(i=0, pFrom=pTabList->a; i<pTabList->nSrc; i++, pFrom++){ Table *pTab = pFrom->pTab; + Select *pSub = pFrom->pSelect; char *zTabName = pFrom->zAlias; + const char *zSchemaName = 0; + int iDb; if( zTabName==0 ){ zTabName = pTab->zName; } if( db->mallocFailed ) break; - if( zTName && sqlite3StrICmp(zTName, zTabName)!=0 ){ - continue; + if( pSub==0 || (pSub->selFlags & SF_NestedFrom)==0 ){ + pSub = 0; + if( zTName && sqlite3StrICmp(zTName, zTabName)!=0 ){ + continue; + } + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + zSchemaName = iDb>=0 ? db->aDb[iDb].zName : "*"; } - tableSeen = 1; for(j=0; j<pTab->nCol; j++){ - Expr *pExpr, *pRight; char *zName = pTab->aCol[j].zName; char *zColname; /* The computed column name */ char *zToFree; /* Malloced string that needs to be freed */ Token sColname; /* Computed column name as a token */ + assert( zName ); + if( zTName && pSub + && sqlite3MatchSpanName(pSub->pEList->a[j].zSpan, 0, zTName, 0)==0 + ){ + continue; + } + /* If a column is marked as 'hidden' (currently only possible ** for virtual tables), do not include it in the expanded ** result-set list. @@ -3376,6 +3537,7 @@ static int selectExpander(Walker *pWalker, Select *p){ assert(IsVirtual(pTab)); continue; } + tableSeen = 1; if( i>0 && zTName==0 ){ if( (pFrom->jointype & JT_NATURAL)!=0 @@ -3398,6 +3560,10 @@ static int selectExpander(Walker *pWalker, Select *p){ Expr *pLeft; pLeft = sqlite3Expr(db, TK_ID, zTabName); pExpr = sqlite3PExpr(pParse, TK_DOT, pLeft, pRight, 0); + if( zSchemaName ){ + pLeft = sqlite3Expr(db, TK_ID, zSchemaName); + pExpr = sqlite3PExpr(pParse, TK_DOT, pLeft, pExpr, 0); + } if( longNames ){ zColname = sqlite3MPrintf(db, "%s.%s", zTabName, zName); zToFree = zColname; @@ -3409,6 +3575,18 @@ static int selectExpander(Walker *pWalker, Select *p){ sColname.z = zColname; sColname.n = sqlite3Strlen30(zColname); sqlite3ExprListSetName(pParse, pNew, &sColname, 0); + if( pNew && (p->selFlags & SF_NestedFrom)!=0 ){ + struct ExprList_item *pX = &pNew->a[pNew->nExpr-1]; + if( pSub ){ + pX->zSpan = sqlite3DbStrDup(db, pSub->pEList->a[j].zSpan); + testcase( pX->zSpan==0 ); + }else{ + pX->zSpan = sqlite3MPrintf(db, "%s.%s.%s", + zSchemaName, zTabName, zColname); + testcase( pX->zSpan==0 ); + } + pX->bSpanIsTab = 1; + } sqlite3DbFree(db, zToFree); } } @@ -3461,10 +3639,13 @@ static int exprWalkNoop(Walker *NotUsed, Expr *NotUsed2){ */ static void sqlite3SelectExpand(Parse *pParse, Select *pSelect){ Walker w; - w.xSelectCallback = selectExpander; + memset(&w, 0, sizeof(w)); + w.xSelectCallback = convertCompoundSelectToSubquery; w.xExprCallback = exprWalkNoop; w.pParse = pParse; sqlite3WalkSelect(&w, pSelect); + w.xSelectCallback = selectExpander; + sqlite3WalkSelect(&w, pSelect); } @@ -3519,9 +3700,11 @@ static int selectAddSubqueryTypeInfo(Walker *pWalker, Select *p){ static void sqlite3SelectAddTypeInfo(Parse *pParse, Select *pSelect){ #ifndef SQLITE_OMIT_SUBQUERY Walker w; + memset(&w, 0, sizeof(w)); w.xSelectCallback = selectAddSubqueryTypeInfo; w.xExprCallback = exprWalkNoop; w.pParse = pParse; + w.bSelectDepthFirst = 1; sqlite3WalkSelect(&w, pSelect); #endif } @@ -3547,6 +3730,7 @@ void sqlite3SelectPrep( sqlite3 *db; if( NEVER(p==0) ) return; db = pParse->db; + if( db->mallocFailed ) return; if( p->selFlags & SF_HasTypeInfo ) return; sqlite3SelectExpand(pParse, p); if( pParse->nErr || db->mallocFailed ) return; @@ -3785,11 +3969,9 @@ int sqlite3Select( ExprList *pOrderBy; /* The ORDER BY clause. May be NULL */ ExprList *pGroupBy; /* The GROUP BY clause. May be NULL */ Expr *pHaving; /* The HAVING clause. May be NULL */ - int isDistinct; /* True if the DISTINCT keyword is present */ - int distinct; /* Table to use for the distinct set */ int rc = 1; /* Value to return from this function */ int addrSortIndex; /* Address of an OP_OpenEphemeral instruction */ - int addrDistinctIndex; /* Address of an OP_OpenEphemeral instruction */ + DistinctCtx sDistinct; /* Info on how to code the DISTINCT keyword */ AggInfo sAggInfo; /* Information used by aggregate queries */ int iEnd; /* Address of the end of the query */ sqlite3 *db; /* The database connection */ @@ -3849,8 +4031,17 @@ int sqlite3Select( int isAggSub; if( pSub==0 ) continue; + + /* Sometimes the code for a subquery will be generated more than + ** once, if the subquery is part of the WHERE clause in a LEFT JOIN, + ** for example. In that case, do not regenerate the code to manifest + ** a view or the co-routine to implement a view. The first instance + ** is sufficient, though the subroutine to manifest the view does need + ** to be invoked again. */ if( pItem->addrFillSub ){ - sqlite3VdbeAddOp2(v, OP_Gosub, pItem->regReturn, pItem->addrFillSub); + if( pItem->viaCoroutine==0 ){ + sqlite3VdbeAddOp2(v, OP_Gosub, pItem->regReturn, pItem->addrFillSub); + } continue; } @@ -3871,6 +4062,44 @@ int sqlite3Select( p->selFlags |= SF_Aggregate; } i = -1; + }else if( pTabList->nSrc==1 && (p->selFlags & SF_Materialize)==0 + && OptimizationEnabled(db, SQLITE_SubqCoroutine) + ){ + /* Implement a co-routine that will return a single row of the result + ** set on each invocation. + */ + int addrTop; + int addrEof; + pItem->regReturn = ++pParse->nMem; + addrEof = ++pParse->nMem; + /* Before coding the OP_Goto to jump to the start of the main routine, + ** ensure that the jump to the verify-schema routine has already + ** been coded. Otherwise, the verify-schema would likely be coded as + ** part of the co-routine. If the main routine then accessed the + ** database before invoking the co-routine for the first time (for + ** example to initialize a LIMIT register from a sub-select), it would + ** be doing so without having verified the schema version and obtained + ** the required db locks. See ticket d6b36be38. */ + sqlite3CodeVerifySchema(pParse, -1); + sqlite3VdbeAddOp0(v, OP_Goto); + addrTop = sqlite3VdbeAddOp1(v, OP_OpenPseudo, pItem->iCursor); + sqlite3VdbeChangeP5(v, 1); + VdbeComment((v, "coroutine for %s", pItem->pTab->zName)); + pItem->addrFillSub = addrTop; + sqlite3VdbeAddOp2(v, OP_Integer, 0, addrEof); + sqlite3VdbeChangeP5(v, 1); + sqlite3SelectDestInit(&dest, SRT_Coroutine, pItem->regReturn); + explainSetInteger(pItem->iSelectId, (u8)pParse->iNextSelectId); + sqlite3Select(pParse, pSub, &dest); + pItem->pTab->nRowEst = (unsigned)pSub->nSelectRow; + pItem->viaCoroutine = 1; + sqlite3VdbeChangeP2(v, addrTop, dest.iSdst); + sqlite3VdbeChangeP3(v, addrTop, dest.nSdst); + sqlite3VdbeAddOp2(v, OP_Integer, 1, addrEof); + sqlite3VdbeAddOp1(v, OP_Yield, pItem->regReturn); + VdbeComment((v, "end %s", pItem->pTab->zName)); + sqlite3VdbeJumpHere(v, addrTop-1); + sqlite3ClearTempRegCache(pParse); }else{ /* Generate a subroutine that will fill an ephemeral table with ** the content of this subquery. pItem->addrFillSub will point @@ -3886,7 +4115,7 @@ int sqlite3Select( pItem->addrFillSub = topAddr+1; VdbeNoopComment((v, "materialize %s", pItem->pTab->zName)); if( pItem->isCorrelated==0 ){ - /* If the subquery is no correlated and if we are not inside of + /* If the subquery is not correlated and if we are not inside of ** a trigger, then we only need to compute the value of the subquery ** once. */ onceAddr = sqlite3CodeOnce(pParse); @@ -3915,7 +4144,7 @@ int sqlite3Select( pWhere = p->pWhere; pGroupBy = p->pGroupBy; pHaving = p->pHaving; - isDistinct = (p->selFlags & SF_Distinct)!=0; + sDistinct.isTnct = (p->selFlags & SF_Distinct)!=0; #ifndef SQLITE_OMIT_COMPOUND_SELECT /* If there is are a sequence of queries, do the earlier ones first. @@ -3950,7 +4179,7 @@ int sqlite3Select( ** to disable this optimization for testing purposes. */ if( sqlite3ExprListCompare(p->pGroupBy, pOrderBy)==0 - && (db->flags & SQLITE_GroupByOrder)==0 ){ + && OptimizationEnabled(db, SQLITE_GroupByOrder) ){ pOrderBy = 0; } @@ -3976,6 +4205,10 @@ int sqlite3Select( p->pGroupBy = sqlite3ExprListDup(db, p->pEList, 0); pGroupBy = p->pGroupBy; pOrderBy = 0; + /* Notice that even thought SF_Distinct has been cleared from p->selFlags, + ** the sDistinct.isTnct is still set. Hence, isTnct represents the + ** original setting of the SF_Distinct flag, not the current setting */ + assert( sDistinct.isTnct ); } /* If there is an ORDER BY clause, then this sorting @@ -4016,24 +4249,27 @@ int sqlite3Select( /* Open a virtual index to use for the distinct set. */ if( p->selFlags & SF_Distinct ){ - KeyInfo *pKeyInfo; - distinct = pParse->nTab++; - pKeyInfo = keyInfoFromExprList(pParse, p->pEList); - addrDistinctIndex = sqlite3VdbeAddOp4(v, OP_OpenEphemeral, distinct, 0, 0, - (char*)pKeyInfo, P4_KEYINFO_HANDOFF); + sDistinct.tabTnct = pParse->nTab++; + sDistinct.addrTnct = sqlite3VdbeAddOp4(v, OP_OpenEphemeral, + sDistinct.tabTnct, 0, 0, + (char*)keyInfoFromExprList(pParse, p->pEList), + P4_KEYINFO_HANDOFF); sqlite3VdbeChangeP5(v, BTREE_UNORDERED); + sDistinct.eTnctType = WHERE_DISTINCT_UNORDERED; }else{ - distinct = addrDistinctIndex = -1; + sDistinct.eTnctType = WHERE_DISTINCT_NOOP; } - /* Aggregate and non-aggregate queries are handled differently */ if( !isAgg && pGroupBy==0 ){ - ExprList *pDist = (isDistinct ? p->pEList : 0); + /* No aggregate functions and no GROUP BY clause */ + ExprList *pDist = (sDistinct.isTnct ? p->pEList : 0); /* Begin the database scan. */ - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pOrderBy, pDist, 0,0); + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pOrderBy, pDist, 0,0); if( pWInfo==0 ) goto select_end; if( pWInfo->nRowOut < p->nSelectRow ) p->nSelectRow = pWInfo->nRowOut; + if( pWInfo->eDistinct ) sDistinct.eTnctType = pWInfo->eDistinct; + if( pOrderBy && pWInfo->nOBSat==pOrderBy->nExpr ) pOrderBy = 0; /* If sorting index that was created by a prior OP_OpenEphemeral ** instruction ended up not being needed, then change the OP_OpenEphemeral @@ -4044,59 +4280,16 @@ int sqlite3Select( p->addrOpenEphm[2] = -1; } - if( pWInfo->eDistinct ){ - VdbeOp *pOp; /* No longer required OpenEphemeral instr. */ - - assert( addrDistinctIndex>=0 ); - pOp = sqlite3VdbeGetOp(v, addrDistinctIndex); - - assert( isDistinct ); - assert( pWInfo->eDistinct==WHERE_DISTINCT_ORDERED - || pWInfo->eDistinct==WHERE_DISTINCT_UNIQUE - ); - distinct = -1; - if( pWInfo->eDistinct==WHERE_DISTINCT_ORDERED ){ - int iJump; - int iExpr; - int iFlag = ++pParse->nMem; - int iBase = pParse->nMem+1; - int iBase2 = iBase + pEList->nExpr; - pParse->nMem += (pEList->nExpr*2); - - /* Change the OP_OpenEphemeral coded earlier to an OP_Integer. The - ** OP_Integer initializes the "first row" flag. */ - pOp->opcode = OP_Integer; - pOp->p1 = 1; - pOp->p2 = iFlag; - - sqlite3ExprCodeExprList(pParse, pEList, iBase, 1); - iJump = sqlite3VdbeCurrentAddr(v) + 1 + pEList->nExpr + 1 + 1; - sqlite3VdbeAddOp2(v, OP_If, iFlag, iJump-1); - for(iExpr=0; iExpr<pEList->nExpr; iExpr++){ - CollSeq *pColl = sqlite3ExprCollSeq(pParse, pEList->a[iExpr].pExpr); - sqlite3VdbeAddOp3(v, OP_Ne, iBase+iExpr, iJump, iBase2+iExpr); - sqlite3VdbeChangeP4(v, -1, (const char *)pColl, P4_COLLSEQ); - sqlite3VdbeChangeP5(v, SQLITE_NULLEQ); - } - sqlite3VdbeAddOp2(v, OP_Goto, 0, pWInfo->iContinue); - - sqlite3VdbeAddOp2(v, OP_Integer, 0, iFlag); - assert( sqlite3VdbeCurrentAddr(v)==iJump ); - sqlite3VdbeAddOp3(v, OP_Move, iBase, iBase2, pEList->nExpr); - }else{ - pOp->opcode = OP_Noop; - } - } - /* Use the standard inner loop. */ - selectInnerLoop(pParse, p, pEList, 0, 0, pOrderBy, distinct, pDest, + selectInnerLoop(pParse, p, pEList, 0, 0, pOrderBy, &sDistinct, pDest, pWInfo->iContinue, pWInfo->iBreak); /* End the database scan loop. */ sqlite3WhereEnd(pWInfo); }else{ - /* This is the processing for aggregate queries */ + /* This case when there exist aggregate functions or a GROUP BY clause + ** or both */ NameContext sNC; /* Name context for processing aggregate information */ int iAMem; /* First Mem address for storing current GROUP BY */ int iBMem; /* First Mem address for previous GROUP BY */ @@ -4204,14 +4397,13 @@ int sqlite3Select( ** in the right order to begin with. */ sqlite3VdbeAddOp2(v, OP_Gosub, regReset, addrReset); - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pGroupBy, 0, 0, 0); + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pGroupBy, 0, 0, 0); if( pWInfo==0 ) goto select_end; - if( pGroupBy==0 ){ + if( pWInfo->nOBSat==pGroupBy->nExpr ){ /* The optimizer is able to deliver rows in group by order so ** we do not have to sort. The OP_OpenEphemeral table will be ** cancelled later because we still need to use the pKeyInfo */ - pGroupBy = p->pGroupBy; groupBySort = 0; }else{ /* Rows are coming out in undetermined order. We have to push @@ -4225,7 +4417,8 @@ int sqlite3Select( int nGroupBy; explainTempTable(pParse, - isDistinct && !(p->selFlags&SF_Distinct)?"DISTINCT":"GROUP BY"); + (sDistinct.isTnct && (p->selFlags&SF_Distinct)==0) ? + "DISTINCT" : "GROUP BY"); groupBySort = 1; nGroupBy = pGroupBy->nExpr; @@ -4357,7 +4550,7 @@ int sqlite3Select( finalizeAggFunctions(pParse, &sAggInfo); sqlite3ExprIfFalse(pParse, pHaving, addrOutputRow+1, SQLITE_JUMPIFNULL); selectInnerLoop(pParse, p, p->pEList, 0, 0, pOrderBy, - distinct, pDest, + &sDistinct, pDest, addrOutputRow+1, addrSetAbort); sqlite3VdbeAddOp1(v, OP_Return, regOutputRow); VdbeComment((v, "end groupby result generator")); @@ -4445,7 +4638,7 @@ int sqlite3Select( ** value of x, the only row required). ** ** A special flag must be passed to sqlite3WhereBegin() to slightly - ** modify behaviour as follows: + ** modify behavior as follows: ** ** + If the query is a "SELECT min(x)", then the loop coded by ** where.c should not iterate over any values with a NULL value @@ -4457,10 +4650,17 @@ int sqlite3Select( ** Refer to code and comments in where.c for details. */ ExprList *pMinMax = 0; - u8 flag = minMaxQuery(p); + u8 flag = WHERE_ORDERBY_NORMAL; + + assert( p->pGroupBy==0 ); + assert( flag==0 ); + if( p->pHaving==0 ){ + flag = minMaxQuery(&sAggInfo, &pMinMax); + } + assert( flag==0 || (pMinMax!=0 && pMinMax->nExpr==1) ); + if( flag ){ - assert( !ExprHasProperty(p->pEList->a[0].pExpr, EP_xIsSelect) ); - pMinMax = sqlite3ExprListDup(db, p->pEList->a[0].pExpr->x.pList,0); + pMinMax = sqlite3ExprListDup(db, pMinMax, 0); pDel = pMinMax; if( pMinMax && !db->mallocFailed ){ pMinMax->a[0].sortOrder = flag!=WHERE_ORDERBY_MIN ?1:0; @@ -4473,13 +4673,14 @@ int sqlite3Select( ** of output. */ resetAccumulator(pParse, &sAggInfo); - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pMinMax,0,flag,0); + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pMinMax,0,flag,0); if( pWInfo==0 ){ sqlite3ExprListDelete(db, pDel); goto select_end; } updateAccumulator(pParse, &sAggInfo); - if( !pMinMax && flag ){ + assert( pMinMax==0 || pMinMax->nExpr==1 ); + if( pWInfo->nOBSat>0 ){ sqlite3VdbeAddOp2(v, OP_Goto, 0, pWInfo->iBreak); VdbeComment((v, "%s() by index", (flag==WHERE_ORDERBY_MIN?"min":"max"))); @@ -4490,7 +4691,7 @@ int sqlite3Select( pOrderBy = 0; sqlite3ExprIfFalse(pParse, pHaving, addrEnd, SQLITE_JUMPIFNULL); - selectInnerLoop(pParse, p, p->pEList, 0, 0, 0, -1, + selectInnerLoop(pParse, p, p->pEList, 0, 0, 0, 0, pDest, addrEnd, addrEnd); sqlite3ExprListDelete(db, pDel); } @@ -4498,7 +4699,7 @@ int sqlite3Select( } /* endif aggregate query */ - if( distinct>=0 ){ + if( sDistinct.eTnctType==WHERE_DISTINCT_UNORDERED ){ explainTempTable(pParse, "DISTINCT"); } @@ -4615,7 +4816,10 @@ void sqlite3ExplainSelect(Vdbe *pVdbe, Select *p){ sqlite3ExplainPrintf(pVdbe, "(null-select)"); return; } - while( p->pPrior ) p = p->pPrior; + while( p->pPrior ){ + p->pPrior->pNext = p; + p = p->pPrior; + } sqlite3ExplainPush(pVdbe); while( p ){ explainOneSelect(pVdbe, p); diff --git a/src/shell.c b/src/shell.c index a17d966..faaec80 100644 --- a/src/shell.c +++ b/src/shell.c @@ -90,7 +90,8 @@ static int enableTimer = 0; #define IsDigit(X) isdigit((unsigned char)X) #define ToLower(X) (char)tolower((unsigned char)X) -#if !defined(_WIN32) && !defined(WIN32) && !defined(_WRS_KERNEL) +#if !defined(_WIN32) && !defined(WIN32) && !defined(_WRS_KERNEL) \ + && !defined(__minux) #include <sys/time.h> #include <sys/resource.h> @@ -541,6 +542,9 @@ static void output_c_string(FILE *out, const char *z){ if( c=='\\' ){ fputc(c, out); fputc(c, out); + }else if( c=='"' ){ + fputc('\\', out); + fputc('"', out); }else if( c=='\t' ){ fputc('\\', out); fputc('t', out); @@ -696,7 +700,7 @@ static int shell_callback(void *pArg, int nArg, char **azArg, char **azCol, int }else{ w = 0; } - if( w<=0 ){ + if( w==0 ){ w = strlen30(azCol[i] ? azCol[i] : ""); if( w<10 ) w = 10; n = strlen30(azArg && azArg[i] ? azArg[i] : p->nullvalue); @@ -706,7 +710,11 @@ static int shell_callback(void *pArg, int nArg, char **azArg, char **azCol, int p->actualWidth[i] = w; } if( p->showHeader ){ - fprintf(p->out,"%-*.*s%s",w,w,azCol[i], i==nArg-1 ? "\n": " "); + if( w<0 ){ + fprintf(p->out,"%*.*s%s",-w,-w,azCol[i], i==nArg-1 ? "\n": " "); + }else{ + fprintf(p->out,"%-*.*s%s",w,w,azCol[i], i==nArg-1 ? "\n": " "); + } } } if( p->showHeader ){ @@ -714,6 +722,7 @@ static int shell_callback(void *pArg, int nArg, char **azArg, char **azCol, int int w; if( i<ArraySize(p->actualWidth) ){ w = p->actualWidth[i]; + if( w<0 ) w = -w; }else{ w = 10; } @@ -735,8 +744,13 @@ static int shell_callback(void *pArg, int nArg, char **azArg, char **azCol, int strlen30(azArg[i])>w ){ w = strlen30(azArg[i]); } - fprintf(p->out,"%-*.*s%s",w,w, - azArg[i] ? azArg[i] : p->nullvalue, i==nArg-1 ? "\n": " "); + if( w<0 ){ + fprintf(p->out,"%*.*s%s",-w,-w, + azArg[i] ? azArg[i] : p->nullvalue, i==nArg-1 ? "\n": " "); + }else{ + fprintf(p->out,"%-*.*s%s",w,w, + azArg[i] ? azArg[i] : p->nullvalue, i==nArg-1 ? "\n": " "); + } } break; } @@ -786,14 +800,14 @@ static int shell_callback(void *pArg, int nArg, char **azArg, char **azCol, int if( p->cnt++==0 && p->showHeader ){ for(i=0; i<nArg; i++){ output_c_string(p->out,azCol[i] ? azCol[i] : ""); - fprintf(p->out, "%s", p->separator); + if(i<nArg-1) fprintf(p->out, "%s", p->separator); } fprintf(p->out,"\n"); } if( azArg==0 ) break; for(i=0; i<nArg; i++){ output_c_string(p->out, azArg[i] ? azArg[i] : p->nullvalue); - fprintf(p->out, "%s", p->separator); + if(i<nArg-1) fprintf(p->out, "%s", p->separator); } fprintf(p->out,"\n"); break; @@ -1416,9 +1430,10 @@ static char zHelp[] = " list Values delimited by .separator string\n" " tabs Tab-separated values\n" " tcl TCL list elements\n" - ".nullvalue STRING Print STRING in place of NULL values\n" + ".nullvalue STRING Use STRING in place of NULL values\n" ".output FILENAME Send output to FILENAME\n" ".output stdout Send output to the screen\n" + ".print STRING... Print literal STRING\n" ".prompt MAIN CONTINUE Replace the standard prompts\n" ".quit Exit this program\n" ".read FILENAME Execute SQL in FILENAME\n" @@ -1511,17 +1526,55 @@ static void resolve_backslashes(char *z){ ** Interpret zArg as a boolean value. Return either 0 or 1. */ static int booleanValue(char *zArg){ - int val = atoi(zArg); - int j; - for(j=0; zArg[j]; j++){ - zArg[j] = ToLower(zArg[j]); + int i; + for(i=0; zArg[i]>='0' && zArg[i]<='9'; i++){} + if( i>0 && zArg[i]==0 ) return atoi(zArg); + if( sqlite3_stricmp(zArg, "on")==0 || sqlite3_stricmp(zArg,"yes")==0 ){ + return 1; } - if( strcmp(zArg,"on")==0 ){ - val = 1; - }else if( strcmp(zArg,"yes")==0 ){ - val = 1; + if( sqlite3_stricmp(zArg, "off")==0 || sqlite3_stricmp(zArg,"no")==0 ){ + return 0; + } + fprintf(stderr, "ERROR: Not a boolean value: \"%s\". Assuming \"no\".\n", + zArg); + return 0; +} + +/* +** Interpret zArg as an integer value, possibly with suffixes. +*/ +static sqlite3_int64 integerValue(const char *zArg){ + sqlite3_int64 v = 0; + static const struct { char *zSuffix; int iMult; } aMult[] = { + { "KiB", 1024 }, + { "MiB", 1024*1024 }, + { "GiB", 1024*1024*1024 }, + { "KB", 1000 }, + { "MB", 1000000 }, + { "GB", 1000000000 }, + { "K", 1000 }, + { "M", 1000000 }, + { "G", 1000000000 }, + }; + int i; + int isNeg = 0; + if( zArg[0]=='-' ){ + isNeg = 1; + zArg++; + }else if( zArg[0]=='+' ){ + zArg++; + } + while( isdigit(zArg[0]) ){ + v = v*10 + zArg[0] - '0'; + zArg++; + } + for(i=0; i<sizeof(aMult)/sizeof(aMult[0]); i++){ + if( sqlite3_stricmp(aMult[i].zSuffix, zArg)==0 ){ + v *= aMult[i].iMult; + break; + } } - return val; + return isNeg? -v : v; } /* @@ -1609,24 +1662,50 @@ static int do_meta_command(char *zLine, struct callback_data *p){ if( nArg==0 ) return 0; /* no tokens, no error */ n = strlen30(azArg[0]); c = azArg[0][0]; - if( c=='b' && n>=3 && strncmp(azArg[0], "backup", n)==0 && nArg>1 && nArg<4){ - const char *zDestFile; - const char *zDb; + if( c=='b' && n>=3 && strncmp(azArg[0], "backup", n)==0 ){ + const char *zDestFile = 0; + const char *zDb = 0; + const char *zKey = 0; sqlite3 *pDest; sqlite3_backup *pBackup; - if( nArg==2 ){ - zDestFile = azArg[1]; - zDb = "main"; - }else{ - zDestFile = azArg[2]; - zDb = azArg[1]; + int j; + for(j=1; j<nArg; j++){ + const char *z = azArg[j]; + if( z[0]=='-' ){ + while( z[0]=='-' ) z++; + if( strcmp(z,"key")==0 && j<nArg-1 ){ + zKey = azArg[++j]; + }else + { + fprintf(stderr, "unknown option: %s\n", azArg[j]); + return 1; + } + }else if( zDestFile==0 ){ + zDestFile = azArg[j]; + }else if( zDb==0 ){ + zDb = zDestFile; + zDestFile = azArg[j]; + }else{ + fprintf(stderr, "too many arguments to .backup\n"); + return 1; + } + } + if( zDestFile==0 ){ + fprintf(stderr, "missing FILENAME argument on .backup\n"); + return 1; } + if( zDb==0 ) zDb = "main"; rc = sqlite3_open(zDestFile, &pDest); if( rc!=SQLITE_OK ){ fprintf(stderr, "Error: cannot open \"%s\"\n", zDestFile); sqlite3_close(pDest); return 1; } +#ifdef SQLITE_HAS_CODEC + sqlite3_key(pDest, zKey, (int)strlen(zKey)); +#else + (void)zKey; +#endif open_db(p); pBackup = sqlite3_backup_init(pDest, "main", p->db, zDb); if( pBackup==0 ){ @@ -1728,7 +1807,8 @@ static int do_meta_command(char *zLine, struct callback_data *p){ p->echoOn = booleanValue(azArg[1]); }else - if( c=='e' && strncmp(azArg[0], "exit", n)==0 && nArg==1 ){ + if( c=='e' && strncmp(azArg[0], "exit", n)==0 ){ + if( nArg>1 && (rc = atoi(azArg[1]))!=0 ) exit(rc); rc = 2; }else @@ -2007,6 +2087,7 @@ static int do_meta_command(char *zLine, struct callback_data *p){ p->mode = MODE_Html; }else if( n2==3 && strncmp(azArg[1],"tcl",n2)==0 ){ p->mode = MODE_Tcl; + sqlite3_snprintf(sizeof(p->separator), p->separator, " "); }else if( n2==3 && strncmp(azArg[1],"csv",n2)==0 ){ p->mode = MODE_Csv; sqlite3_snprintf(sizeof(p->separator), p->separator, ","); @@ -2070,6 +2151,15 @@ static int do_meta_command(char *zLine, struct callback_data *p){ } }else + if( c=='p' && n>=3 && strncmp(azArg[0], "print", n)==0 ){ + int i; + for(i=1; i<nArg; i++){ + if( i>1 ) fprintf(p->out, " "); + fprintf(p->out, "%s", azArg[i]); + } + fprintf(p->out, "\n"); + }else + if( c=='p' && strncmp(azArg[0], "prompt", n)==0 && (nArg==2 || nArg==3)){ if( nArg >= 2) { strncpy(mainPrompt,azArg[1],(int)ArraySize(mainPrompt)-1); @@ -2188,8 +2278,7 @@ static int do_meta_command(char *zLine, struct callback_data *p){ " SELECT sql, type, tbl_name, name, rowid FROM sqlite_temp_master) " "WHERE lower(tbl_name) LIKE shellstatic()" " AND type!='meta' AND sql NOTNULL " - "ORDER BY substr(type,2,1), " - " CASE type WHEN 'view' THEN rowid ELSE name END", + "ORDER BY rowid", callback, &data, &zErrMsg); zShellStatic = 0; } @@ -2200,8 +2289,7 @@ static int do_meta_command(char *zLine, struct callback_data *p){ " FROM sqlite_master UNION ALL" " SELECT sql, type, tbl_name, name, rowid FROM sqlite_temp_master) " "WHERE type!='meta' AND sql NOTNULL AND name NOT LIKE 'sqlite_%'" - "ORDER BY substr(type,2,1)," - " CASE type WHEN 'view' THEN rowid ELSE name END", + "ORDER BY rowid", callback, &data, &zErrMsg ); } @@ -2323,9 +2411,9 @@ static int do_meta_command(char *zLine, struct callback_data *p){ for(i=0; i<nPrintRow; i++){ for(j=i; j<nRow; j+=nPrintRow){ char *zSp = j<nPrintRow ? "" : " "; - printf("%s%-*s", zSp, maxlen, azResult[j] ? azResult[j] : ""); + fprintf(p->out, "%s%-*s", zSp, maxlen, azResult[j] ? azResult[j] : ""); } - printf("\n"); + fprintf(p->out, "\n"); } } for(ii=0; ii<nRow; ii++) sqlite3_free(azResult[ii]); @@ -2382,7 +2470,7 @@ static int do_meta_command(char *zLine, struct callback_data *p){ if( nArg==3 ){ int opt = (int)strtol(azArg[2], 0, 0); rc = sqlite3_test_control(testctrl, p->db, opt); - printf("%d (0x%08x)\n", rc, rc); + fprintf(p->out, "%d (0x%08x)\n", rc, rc); } else { fprintf(stderr,"Error: testctrl %s takes a single int option\n", azArg[1]); @@ -2395,7 +2483,7 @@ static int do_meta_command(char *zLine, struct callback_data *p){ case SQLITE_TESTCTRL_PRNG_RESET: if( nArg==2 ){ rc = sqlite3_test_control(testctrl); - printf("%d (0x%08x)\n", rc, rc); + fprintf(p->out, "%d (0x%08x)\n", rc, rc); } else { fprintf(stderr,"Error: testctrl %s takes no options\n", azArg[1]); } @@ -2404,9 +2492,9 @@ static int do_meta_command(char *zLine, struct callback_data *p){ /* sqlite3_test_control(int, uint) */ case SQLITE_TESTCTRL_PENDING_BYTE: if( nArg==3 ){ - unsigned int opt = (unsigned int)atoi(azArg[2]); + unsigned int opt = (unsigned int)integerValue(azArg[2]); rc = sqlite3_test_control(testctrl, opt); - printf("%d (0x%08x)\n", rc, rc); + fprintf(p->out, "%d (0x%08x)\n", rc, rc); } else { fprintf(stderr,"Error: testctrl %s takes a single unsigned" " int option\n", azArg[1]); @@ -2419,7 +2507,7 @@ static int do_meta_command(char *zLine, struct callback_data *p){ if( nArg==3 ){ int opt = atoi(azArg[2]); rc = sqlite3_test_control(testctrl, opt); - printf("%d (0x%08x)\n", rc, rc); + fprintf(p->out, "%d (0x%08x)\n", rc, rc); } else { fprintf(stderr,"Error: testctrl %s takes a single int option\n", azArg[1]); @@ -2432,7 +2520,7 @@ static int do_meta_command(char *zLine, struct callback_data *p){ if( nArg==3 ){ const char *opt = azArg[2]; rc = sqlite3_test_control(testctrl, opt); - printf("%d (0x%08x)\n", rc, rc); + fprintf(p->out, "%d (0x%08x)\n", rc, rc); } else { fprintf(stderr,"Error: testctrl %s takes a single char * option\n", azArg[1]); @@ -2477,7 +2565,7 @@ static int do_meta_command(char *zLine, struct callback_data *p){ }else if( c=='v' && strncmp(azArg[0], "version", n)==0 ){ - printf("SQLite %s %s\n" /*extra-version-info*/, + fprintf(p->out, "SQLite %s %s\n" /*extra-version-info*/, sqlite3_libversion(), sqlite3_sourceid()); }else @@ -2487,12 +2575,19 @@ static int do_meta_command(char *zLine, struct callback_data *p){ if( p->db ){ sqlite3_file_control(p->db, zDbName, SQLITE_FCNTL_VFSNAME, &zVfsName); if( zVfsName ){ - printf("%s\n", zVfsName); + fprintf(p->out, "%s\n", zVfsName); sqlite3_free(zVfsName); } } }else +#if defined(SQLITE_DEBUG) && defined(SQLITE_ENABLE_WHERETRACE) + if( c=='w' && strncmp(azArg[0], "wheretrace", n)==0 ){ + extern int sqlite3WhereTrace; + sqlite3WhereTrace = booleanValue(azArg[1]); + }else +#endif + if( c=='w' && strncmp(azArg[0], "width", n)==0 && nArg>1 ){ int j; assert( nArg<=ArraySize(azArg) ); @@ -2675,6 +2770,10 @@ static int process_input(struct callback_data *p, FILE *in){ free(zSql); zSql = 0; nSql = 0; + }else if( zSql && _all_whitespace(zSql) ){ + free(zSql); + zSql = 0; + nSql = 0; } } if( zSql ){ @@ -2684,7 +2783,7 @@ static int process_input(struct callback_data *p, FILE *in){ free(zSql); } free(zLine); - return errCnt; + return errCnt>0; } /* @@ -2797,21 +2896,25 @@ static const char zOptions[] = " -bail stop after hitting an error\n" " -batch force batch I/O\n" " -column set output mode to 'column'\n" - " -cmd command run \"command\" before reading stdin\n" + " -cmd COMMAND run \"COMMAND\" before reading stdin\n" " -csv set output mode to 'csv'\n" " -echo print commands before execution\n" - " -init filename read/process named file\n" + " -init FILENAME read/process named file\n" " -[no]header turn headers on or off\n" +#if defined(SQLITE_ENABLE_MEMSYS3) || defined(SQLITE_ENABLE_MEMSYS5) + " -heap SIZE Size of heap for memsys3 or memsys5\n" +#endif " -help show this message\n" " -html set output mode to HTML\n" " -interactive force interactive I/O\n" " -line set output mode to 'line'\n" " -list set output mode to 'list'\n" + " -mmap N default mmap size set to N\n" #ifdef SQLITE_ENABLE_MULTIPLEX " -multiplex enable the multiplexor VFS\n" #endif - " -nullvalue 'text' set text string for NULL values\n" - " -separator 'x' set output field separator (|)\n" + " -nullvalue TEXT set text string for NULL values. Default ''\n" + " -separator SEP set output field separator. Default: '|'\n" " -stats print memory stats before each finalize\n" " -version show SQLite version\n" " -vfs NAME use NAME as the default VFS\n" @@ -2847,6 +2950,19 @@ static void main_init(struct callback_data *data) { sqlite3_config(SQLITE_CONFIG_SINGLETHREAD); } +/* +** Get the argument to an --option. Throw an error and die if no argument +** is available. +*/ +static char *cmdline_option_value(int argc, char **argv, int i){ + if( i==argc ){ + fprintf(stderr, "%s: Error: missing argument to %s\n", + argv[0], argv[argc-1]); + exit(1); + } + return argv[i]; +} + int main(int argc, char **argv){ char *zErrMsg = 0; struct callback_data data; @@ -2876,24 +2992,35 @@ int main(int argc, char **argv){ ** the size of the alternative malloc heap, ** and the first command to execute. */ - for(i=1; i<argc-1; i++){ + for(i=1; i<argc; i++){ char *z; - if( argv[i][0]!='-' ) break; z = argv[i]; + if( z[0]!='-' ){ + if( data.zDbFilename==0 ){ + data.zDbFilename = z; + continue; + } + if( zFirstCmd==0 ){ + zFirstCmd = z; + continue; + } + fprintf(stderr,"%s: Error: too many options: \"%s\"\n", Argv0, argv[i]); + fprintf(stderr,"Use -help for a list of options.\n"); + return 1; + } if( z[1]=='-' ) z++; if( strcmp(z,"-separator")==0 || strcmp(z,"-nullvalue")==0 || strcmp(z,"-cmd")==0 ){ - i++; + (void)cmdline_option_value(argc, argv, ++i); }else if( strcmp(z,"-init")==0 ){ - i++; - zInitFile = argv[i]; - /* Need to check for batch mode here to so we can avoid printing - ** informational messages (like from process_sqliterc) before - ** we do the actual processing of arguments later in a second pass. - */ + zInitFile = cmdline_option_value(argc, argv, ++i); }else if( strcmp(z,"-batch")==0 ){ + /* Need to check for batch mode here to so we can avoid printing + ** informational messages (like from process_sqliterc) before + ** we do the actual processing of arguments later in a second pass. + */ stdin_is_interactive = 0; }else if( strcmp(z,"-heap")==0 ){ #if defined(SQLITE_ENABLE_MEMSYS3) || defined(SQLITE_ENABLE_MEMSYS5) @@ -2901,13 +3028,8 @@ int main(int argc, char **argv){ const char *zSize; sqlite3_int64 szHeap; - zSize = argv[++i]; - szHeap = atoi(zSize); - for(j=0; (c = zSize[j])!=0; j++){ - if( c=='M' ){ szHeap *= 1000000; break; } - if( c=='K' ){ szHeap *= 1000; break; } - if( c=='G' ){ szHeap *= 1000000000; break; } - } + zSize = cmdline_option_value(argc, argv, ++i); + szHeap = integerValue(zSize); if( szHeap>0x7fff0000 ) szHeap = 0x7fff0000; sqlite3_config(SQLITE_CONFIG_HEAP, malloc((int)szHeap), (int)szHeap, 64); #endif @@ -2927,8 +3049,11 @@ int main(int argc, char **argv){ extern int sqlite3_multiple_initialize(const char*,int); sqlite3_multiplex_initialize(0, 1); #endif + }else if( strcmp(z,"-mmap")==0 ){ + sqlite3_int64 sz = integerValue(cmdline_option_value(argc,argv,++i)); + sqlite3_config(SQLITE_CONFIG_MMAP_SIZE, sz, sz); }else if( strcmp(z,"-vfs")==0 ){ - sqlite3_vfs *pVfs = sqlite3_vfs_find(argv[++i]); + sqlite3_vfs *pVfs = sqlite3_vfs_find(cmdline_option_value(argc,argv,++i)); if( pVfs ){ sqlite3_vfs_register(pVfs, 1); }else{ @@ -2937,31 +3062,15 @@ int main(int argc, char **argv){ } } } - if( i<argc ){ - data.zDbFilename = argv[i++]; - }else{ + if( data.zDbFilename==0 ){ #ifndef SQLITE_OMIT_MEMORYDB data.zDbFilename = ":memory:"; #else - data.zDbFilename = 0; -#endif - } - if( i<argc ){ - zFirstCmd = argv[i++]; - } - if( i<argc ){ - fprintf(stderr,"%s: Error: too many options: \"%s\"\n", Argv0, argv[i]); - fprintf(stderr,"Use -help for a list of options.\n"); - return 1; - } - data.out = stdout; - -#ifdef SQLITE_OMIT_MEMORYDB - if( data.zDbFilename==0 ){ fprintf(stderr,"%s: Error: no database filename specified\n", Argv0); return 1; - } #endif + } + data.out = stdout; /* Go ahead and open the database file if it already exists. If the ** file does not exist, delay opening it. This prevents empty database @@ -2986,8 +3095,9 @@ int main(int argc, char **argv){ ** file is processed so that the command-line arguments will override ** settings in the initialization file. */ - for(i=1; i<argc && argv[i][0]=='-'; i++){ + for(i=1; i<argc; i++){ char *z = argv[i]; + if( z[0]!='-' ) continue; if( z[1]=='-' ){ z++; } if( strcmp(z,"-init")==0 ){ i++; @@ -3003,25 +3113,11 @@ int main(int argc, char **argv){ data.mode = MODE_Csv; memcpy(data.separator,",",2); }else if( strcmp(z,"-separator")==0 ){ - i++; - if(i>=argc){ - fprintf(stderr,"%s: Error: missing argument for option: %s\n", - Argv0, z); - fprintf(stderr,"Use -help for a list of options.\n"); - return 1; - } sqlite3_snprintf(sizeof(data.separator), data.separator, - "%.*s",(int)sizeof(data.separator)-1,argv[i]); + "%s",cmdline_option_value(argc,argv,++i)); }else if( strcmp(z,"-nullvalue")==0 ){ - i++; - if(i>=argc){ - fprintf(stderr,"%s: Error: missing argument for option: %s\n", - Argv0, z); - fprintf(stderr,"Use -help for a list of options.\n"); - return 1; - } sqlite3_snprintf(sizeof(data.nullvalue), data.nullvalue, - "%.*s",(int)sizeof(data.nullvalue)-1,argv[i]); + "%s",cmdline_option_value(argc,argv,++i)); }else if( strcmp(z,"-header")==0 ){ data.showHeader = 1; }else if( strcmp(z,"-noheader")==0 ){ @@ -3041,6 +3137,8 @@ int main(int argc, char **argv){ stdin_is_interactive = 0; }else if( strcmp(z,"-heap")==0 ){ i++; + }else if( strcmp(z,"-mmap")==0 ){ + i++; }else if( strcmp(z,"-vfs")==0 ){ i++; #ifdef SQLITE_ENABLE_VFSTRACE @@ -3055,11 +3153,10 @@ int main(int argc, char **argv){ usage(1); }else if( strcmp(z,"-cmd")==0 ){ if( i==argc-1 ) break; - i++; - z = argv[i]; + z = cmdline_option_value(argc,argv,++i); if( z[0]=='.' ){ rc = do_meta_command(z, &data); - if( rc && bail_on_error ) return rc; + if( rc && bail_on_error ) return rc==2 ? 0 : rc; }else{ open_db(&data); rc = shell_exec(data.db, z, shell_callback, &data, &zErrMsg); @@ -3083,6 +3180,7 @@ int main(int argc, char **argv){ */ if( zFirstCmd[0]=='.' ){ rc = do_meta_command(zFirstCmd, &data); + if( rc==2 ) rc = 0; }else{ open_db(&data); rc = shell_exec(data.db, zFirstCmd, shell_callback, &data, &zErrMsg); @@ -3102,7 +3200,13 @@ int main(int argc, char **argv){ char *zHistory = 0; int nHistory; printf( +/* BEGIN SQLCIPHER */ +#ifdef SQLITE_HAS_CODEC + "SQLCipher version %s %.19s\n" /*extra-version-info*/ +#else "SQLite version %s %.19s\n" /*extra-version-info*/ +#endif +/* END SQLCIPHER */ "Enter \".help\" for instructions\n" "Enter SQL statements terminated with a \";\"\n", sqlite3_libversion(), sqlite3_sourceid() diff --git a/src/sqlcipher.h b/src/sqlcipher.h new file mode 100644 index 0000000..41f8f83 --- /dev/null +++ b/src/sqlcipher.h @@ -0,0 +1,75 @@ +/* +** SQLCipher +** sqlcipher.h developed by Stephen Lombardo (Zetetic LLC) +** sjlombardo at zetetic dot net +** http://zetetic.net +** +** Copyright (c) 2008, ZETETIC LLC +** All rights reserved. +** +** Redistribution and use in source and binary forms, with or without +** modification, are permitted provided that the following conditions are met: +** * Redistributions of source code must retain the above copyright +** notice, this list of conditions and the following disclaimer. +** * Redistributions in binary form must reproduce the above copyright +** notice, this list of conditions and the following disclaimer in the +** documentation and/or other materials provided with the distribution. +** * Neither the name of the ZETETIC LLC nor the +** names of its contributors may be used to endorse or promote products +** derived from this software without specific prior written permission. +** +** THIS SOFTWARE IS PROVIDED BY ZETETIC LLC ''AS IS'' AND ANY +** EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +** WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +** DISCLAIMED. IN NO EVENT SHALL ZETETIC LLC BE LIABLE FOR ANY +** DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +** (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +** LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +** ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +** (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +** SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +** +*/ +/* BEGIN SQLCIPHER */ +#ifdef SQLITE_HAS_CODEC +#ifndef SQLCIPHER_H +#define SQLCIPHER_H + + +typedef struct { + int (*activate)(void *ctx); + int (*deactivate)(void *ctx); + const char* (*get_provider_name)(void *ctx); + int (*add_random)(void *ctx, void *buffer, int length); + int (*random)(void *ctx, void *buffer, int length); + int (*hmac)(void *ctx, unsigned char *hmac_key, int key_sz, unsigned char *in, int in_sz, unsigned char *in2, int in2_sz, unsigned char *out); + int (*kdf)(void *ctx, const char *pass, int pass_sz, unsigned char* salt, int salt_sz, int workfactor, int key_sz, unsigned char *key); + int (*cipher)(void *ctx, int mode, unsigned char *key, int key_sz, unsigned char *iv, unsigned char *in, int in_sz, unsigned char *out); + int (*set_cipher)(void *ctx, const char *cipher_name); + const char* (*get_cipher)(void *ctx); + int (*get_key_sz)(void *ctx); + int (*get_iv_sz)(void *ctx); + int (*get_block_sz)(void *ctx); + int (*get_hmac_sz)(void *ctx); + int (*ctx_copy)(void *target_ctx, void *source_ctx); + int (*ctx_cmp)(void *c1, void *c2); + int (*ctx_init)(void **ctx); + int (*ctx_free)(void **ctx); +} sqlcipher_provider; + +/* utility functions */ +void sqlcipher_free(void *ptr, int sz); +void* sqlcipher_malloc(int sz); +void* sqlcipher_memset(void *v, unsigned char value, int len); +int sqlcipher_ismemset(const void *v, unsigned char value, int len); +int sqlcipher_memcmp(const void *v0, const void *v1, int len); +void sqlcipher_free(void *, int); + +/* provider interfaces */ +int sqlcipher_register_provider(sqlcipher_provider *p); +sqlcipher_provider* sqlcipher_get_provider(); + +#endif +#endif +/* END SQLCIPHER */ + diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 9a31e22..6608823 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -283,7 +283,7 @@ typedef sqlite_uint64 sqlite3_uint64; ** [sqlite3_blob_close | close] all [BLOB handles], and ** [sqlite3_backup_finish | finish] all [sqlite3_backup] objects associated ** with the [sqlite3] object prior to attempting to close the object. ^If -** sqlite3_close() is called on a [database connection] that still has +** sqlite3_close_v2() is called on a [database connection] that still has ** outstanding [prepared statements], [BLOB handles], and/or ** [sqlite3_backup] objects then it returns SQLITE_OK but the deallocation ** of resources is deferred until all [prepared statements], [BLOB handles], @@ -420,6 +420,8 @@ int sqlite3_exec( #define SQLITE_FORMAT 24 /* Auxiliary database format error */ #define SQLITE_RANGE 25 /* 2nd parameter to sqlite3_bind out of range */ #define SQLITE_NOTADB 26 /* File opened that is not a database file */ +#define SQLITE_NOTICE 27 /* Notifications from sqlite3_log() */ +#define SQLITE_WARNING 28 /* Warnings from sqlite3_log() */ #define SQLITE_ROW 100 /* sqlite3_step() has another row ready */ #define SQLITE_DONE 101 /* sqlite3_step() has finished executing */ /* end-of-error-codes */ @@ -469,14 +471,29 @@ int sqlite3_exec( #define SQLITE_IOERR_SHMLOCK (SQLITE_IOERR | (20<<8)) #define SQLITE_IOERR_SHMMAP (SQLITE_IOERR | (21<<8)) #define SQLITE_IOERR_SEEK (SQLITE_IOERR | (22<<8)) +#define SQLITE_IOERR_DELETE_NOENT (SQLITE_IOERR | (23<<8)) +#define SQLITE_IOERR_MMAP (SQLITE_IOERR | (24<<8)) #define SQLITE_LOCKED_SHAREDCACHE (SQLITE_LOCKED | (1<<8)) #define SQLITE_BUSY_RECOVERY (SQLITE_BUSY | (1<<8)) #define SQLITE_CANTOPEN_NOTEMPDIR (SQLITE_CANTOPEN | (1<<8)) #define SQLITE_CANTOPEN_ISDIR (SQLITE_CANTOPEN | (2<<8)) +#define SQLITE_CANTOPEN_FULLPATH (SQLITE_CANTOPEN | (3<<8)) #define SQLITE_CORRUPT_VTAB (SQLITE_CORRUPT | (1<<8)) #define SQLITE_READONLY_RECOVERY (SQLITE_READONLY | (1<<8)) #define SQLITE_READONLY_CANTLOCK (SQLITE_READONLY | (2<<8)) +#define SQLITE_READONLY_ROLLBACK (SQLITE_READONLY | (3<<8)) #define SQLITE_ABORT_ROLLBACK (SQLITE_ABORT | (2<<8)) +#define SQLITE_CONSTRAINT_CHECK (SQLITE_CONSTRAINT | (1<<8)) +#define SQLITE_CONSTRAINT_COMMITHOOK (SQLITE_CONSTRAINT | (2<<8)) +#define SQLITE_CONSTRAINT_FOREIGNKEY (SQLITE_CONSTRAINT | (3<<8)) +#define SQLITE_CONSTRAINT_FUNCTION (SQLITE_CONSTRAINT | (4<<8)) +#define SQLITE_CONSTRAINT_NOTNULL (SQLITE_CONSTRAINT | (5<<8)) +#define SQLITE_CONSTRAINT_PRIMARYKEY (SQLITE_CONSTRAINT | (6<<8)) +#define SQLITE_CONSTRAINT_TRIGGER (SQLITE_CONSTRAINT | (7<<8)) +#define SQLITE_CONSTRAINT_UNIQUE (SQLITE_CONSTRAINT | (8<<8)) +#define SQLITE_CONSTRAINT_VTAB (SQLITE_CONSTRAINT | (9<<8)) +#define SQLITE_NOTICE_RECOVER_WAL (SQLITE_NOTICE | (1<<8)) +#define SQLITE_NOTICE_RECOVER_ROLLBACK (SQLITE_NOTICE | (2<<8)) /* ** CAPI3REF: Flags For File Open Operations @@ -716,6 +733,9 @@ struct sqlite3_io_methods { void (*xShmBarrier)(sqlite3_file*); int (*xShmUnmap)(sqlite3_file*, int deleteFlag); /* Methods above are valid for version 2 */ + int (*xFetch)(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp); + int (*xUnfetch)(sqlite3_file*, sqlite3_int64 iOfst, void *p); + /* Methods above are valid for version 3 */ /* Additional methods may be added in future releases */ }; @@ -850,6 +870,38 @@ struct sqlite3_io_methods { ** compilation of the PRAGMA fails with an error. ^The [SQLITE_FCNTL_PRAGMA] ** file control occurs at the beginning of pragma statement analysis and so ** it is able to override built-in [PRAGMA] statements. +** +** <li>[[SQLITE_FCNTL_BUSYHANDLER]] +** ^The [SQLITE_FCNTL_BUSYHANDLER] +** file-control may be invoked by SQLite on the database file handle +** shortly after it is opened in order to provide a custom VFS with access +** to the connections busy-handler callback. The argument is of type (void **) +** - an array of two (void *) values. The first (void *) actually points +** to a function of type (int (*)(void *)). In order to invoke the connections +** busy-handler, this function should be invoked with the second (void *) in +** the array as the only argument. If it returns non-zero, then the operation +** should be retried. If it returns zero, the custom VFS should abandon the +** current operation. +** +** <li>[[SQLITE_FCNTL_TEMPFILENAME]] +** ^Application can invoke the [SQLITE_FCNTL_TEMPFILENAME] file-control +** to have SQLite generate a +** temporary filename using the same algorithm that is followed to generate +** temporary filenames for TEMP tables and other internal uses. The +** argument should be a char** which will be filled with the filename +** written into memory obtained from [sqlite3_malloc()]. The caller should +** invoke [sqlite3_free()] on the result to avoid a memory leak. +** +** <li>[[SQLITE_FCNTL_MMAP_SIZE]] +** The [SQLITE_FCNTL_MMAP_SIZE] file control is used to query or set the +** maximum number of bytes that will be used for memory-mapped I/O. +** The argument is a pointer to a value of type sqlite3_int64 that +** is an advisory maximum number of bytes in the file to memory map. The +** pointer is overwritten with the old value. The limit is not changed if +** the value originally pointed to is negative, and so the current limit +** can be queried by passing in a pointer to a negative number. This +** file-control is used internally to implement [PRAGMA mmap_size]. +** ** </ul> */ #define SQLITE_FCNTL_LOCKSTATE 1 @@ -866,6 +918,9 @@ struct sqlite3_io_methods { #define SQLITE_FCNTL_VFSNAME 12 #define SQLITE_FCNTL_POWERSAFE_OVERWRITE 13 #define SQLITE_FCNTL_PRAGMA 14 +#define SQLITE_FCNTL_BUSYHANDLER 15 +#define SQLITE_FCNTL_TEMPFILENAME 16 +#define SQLITE_FCNTL_MMAP_SIZE 18 /* ** CAPI3REF: Mutex Handle @@ -1532,7 +1587,9 @@ struct sqlite3_mem_methods { ** page cache implementation into that object.)^ </dd> ** ** [[SQLITE_CONFIG_LOG]] <dt>SQLITE_CONFIG_LOG</dt> -** <dd> ^The SQLITE_CONFIG_LOG option takes two arguments: a pointer to a +** <dd> The SQLITE_CONFIG_LOG option is used to configure the SQLite +** global [error log]. +** (^The SQLITE_CONFIG_LOG option takes two arguments: a pointer to a ** function with a call signature of void(*)(void*,int,const char*), ** and a pointer to void. ^If the function pointer is not NULL, it is ** invoked by [sqlite3_log()] to process each logging event. ^If the @@ -1562,10 +1619,54 @@ struct sqlite3_mem_methods { ** disabled. The default value may be changed by compiling with the ** [SQLITE_USE_URI] symbol defined. ** +** [[SQLITE_CONFIG_COVERING_INDEX_SCAN]] <dt>SQLITE_CONFIG_COVERING_INDEX_SCAN +** <dd> This option takes a single integer argument which is interpreted as +** a boolean in order to enable or disable the use of covering indices for +** full table scans in the query optimizer. The default setting is determined +** by the [SQLITE_ALLOW_COVERING_INDEX_SCAN] compile-time option, or is "on" +** if that compile-time option is omitted. +** The ability to disable the use of covering indices for full table scans +** is because some incorrectly coded legacy applications might malfunction +** malfunction when the optimization is enabled. Providing the ability to +** disable the optimization allows the older, buggy application code to work +** without change even with newer versions of SQLite. +** ** [[SQLITE_CONFIG_PCACHE]] [[SQLITE_CONFIG_GETPCACHE]] ** <dt>SQLITE_CONFIG_PCACHE and SQLITE_CONFIG_GETPCACHE ** <dd> These options are obsolete and should not be used by new code. ** They are retained for backwards compatibility but are now no-ops. +** </dd> +** +** [[SQLITE_CONFIG_SQLLOG]] +** <dt>SQLITE_CONFIG_SQLLOG +** <dd>This option is only available if sqlite is compiled with the +** [SQLITE_ENABLE_SQLLOG] pre-processor macro defined. The first argument should +** be a pointer to a function of type void(*)(void*,sqlite3*,const char*, int). +** The second should be of type (void*). The callback is invoked by the library +** in three separate circumstances, identified by the value passed as the +** fourth parameter. If the fourth parameter is 0, then the database connection +** passed as the second argument has just been opened. The third argument +** points to a buffer containing the name of the main database file. If the +** fourth parameter is 1, then the SQL statement that the third parameter +** points to has just been executed. Or, if the fourth parameter is 2, then +** the connection being passed as the second parameter is being closed. The +** third parameter is passed NULL In this case. An example of using this +** configuration option can be seen in the "test_sqllog.c" source file in +** the canonical SQLite source tree.</dd> +** +** [[SQLITE_CONFIG_MMAP_SIZE]] +** <dt>SQLITE_CONFIG_MMAP_SIZE +** <dd>SQLITE_CONFIG_MMAP_SIZE takes two 64-bit integer (sqlite3_int64) values +** that are the default mmap size limit (the default setting for +** [PRAGMA mmap_size]) and the maximum allowed mmap size limit. +** The default setting can be overridden by each database connection using +** either the [PRAGMA mmap_size] command, or by using the +** [SQLITE_FCNTL_MMAP_SIZE] file control. The maximum allowed mmap size +** cannot be changed at run-time. Nor may the maximum allowed mmap size +** exceed the compile-time maximum mmap size set by the +** [SQLITE_MAX_MMAP_SIZE] compile-time option. +** If either argument to this option is negative, then that argument is +** changed to its compile-time default. ** </dl> */ #define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ @@ -1587,6 +1688,9 @@ struct sqlite3_mem_methods { #define SQLITE_CONFIG_URI 17 /* int */ #define SQLITE_CONFIG_PCACHE2 18 /* sqlite3_pcache_methods2* */ #define SQLITE_CONFIG_GETPCACHE2 19 /* sqlite3_pcache_methods2* */ +#define SQLITE_CONFIG_COVERING_INDEX_SCAN 20 /* int */ +#define SQLITE_CONFIG_SQLLOG 21 /* xSqllog, void* */ +#define SQLITE_CONFIG_MMAP_SIZE 22 /* sqlite3_int64, sqlite3_int64 */ /* ** CAPI3REF: Database Connection Configuration Options @@ -2420,6 +2524,9 @@ int sqlite3_set_authorizer( ** as each triggered subprogram is entered. The callbacks for triggers ** contain a UTF-8 SQL comment that identifies the trigger.)^ ** +** The [SQLITE_TRACE_SIZE_LIMIT] compile-time option can be used to limit +** the length of [bound parameter] expansion in the output of sqlite3_trace(). +** ** ^The callback function registered by sqlite3_profile() is invoked ** as each SQL statement finishes. ^The profile callback contains ** the original statement text and an estimate of wall-clock time @@ -2595,7 +2702,7 @@ void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** an error)^. ** ^If "ro" is specified, then the database is opened for read-only ** access, just as if the [SQLITE_OPEN_READONLY] flag had been set in the -** third argument to sqlite3_prepare_v2(). ^If the mode option is set to +** third argument to sqlite3_open_v2(). ^If the mode option is set to ** "rw", then the database is opened for read-write (but not create) ** access, as if SQLITE_OPEN_READWRITE (but not SQLITE_OPEN_CREATE) had ** been set. ^Value "rwc" is equivalent to setting both @@ -2611,7 +2718,7 @@ void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** sqlite3_open_v2(). ^Setting the cache parameter to "private" is ** equivalent to setting the SQLITE_OPEN_PRIVATECACHE bit. ** ^If sqlite3_open_v2() is used and the "cache" parameter is present in -** a URI filename, its value overrides any behaviour requested by setting +** a URI filename, its value overrides any behavior requested by setting ** SQLITE_OPEN_PRIVATECACHE or SQLITE_OPEN_SHAREDCACHE flag. ** </ul> ** @@ -2747,6 +2854,11 @@ sqlite3_int64 sqlite3_uri_int64(const char*, const char*, sqlite3_int64); ** However, the error string might be overwritten or deallocated by ** subsequent calls to other SQLite interface functions.)^ ** +** ^The sqlite3_errstr() interface returns the English-language text +** that describes the [result code], as UTF-8. +** ^(Memory to hold the error message string is managed internally +** and must not be freed by the application)^. +** ** When the serialized [threading mode] is in use, it might be the ** case that a second error occurs on a separate thread in between ** the time of the first error and the call to these interfaces. @@ -2765,6 +2877,7 @@ int sqlite3_errcode(sqlite3 *db); int sqlite3_extended_errcode(sqlite3 *db); const char *sqlite3_errmsg(sqlite3*); const void *sqlite3_errmsg16(sqlite3*); +const char *sqlite3_errstr(int); /* ** CAPI3REF: SQL Statement Object @@ -2952,7 +3065,8 @@ int sqlite3_limit(sqlite3*, int id, int newVal); ** <li> ** ^If the database schema changes, instead of returning [SQLITE_SCHEMA] as it ** always used to do, [sqlite3_step()] will automatically recompile the SQL -** statement and try to run it again. +** statement and try to run it again. As many as [SQLITE_MAX_SCHEMA_RETRY] +** retries will occur before sqlite3_step() gives up and returns an error. ** </li> ** ** <li> @@ -3156,6 +3270,9 @@ typedef struct sqlite3_context sqlite3_context; ** parameter [SQLITE_LIMIT_VARIABLE_NUMBER] (default value: 999). ** ** ^The third argument is the value to bind to the parameter. +** ^If the third parameter to sqlite3_bind_text() or sqlite3_bind_text16() +** or sqlite3_bind_blob() is a NULL pointer then the fourth parameter +** is ignored and the end result is the same as sqlite3_bind_null(). ** ** ^(In those routines that have a fourth argument, its value is the ** number of bytes in the parameter. To be clear: the value is the @@ -3923,7 +4040,8 @@ SQLITE_DEPRECATED int sqlite3_expired(sqlite3_stmt*); SQLITE_DEPRECATED int sqlite3_transfer_bindings(sqlite3_stmt*, sqlite3_stmt*); SQLITE_DEPRECATED int sqlite3_global_recover(void); SQLITE_DEPRECATED void sqlite3_thread_cleanup(void); -SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int64,int),void*,sqlite3_int64); +SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int64,int), + void*,sqlite3_int64); #endif /* @@ -4003,14 +4121,17 @@ int sqlite3_value_numeric_type(sqlite3_value*); ** In those cases, sqlite3_aggregate_context() might be called for the ** first time from within xFinal().)^ ** -** ^The sqlite3_aggregate_context(C,N) routine returns a NULL pointer if N is -** less than or equal to zero or if a memory allocate error occurs. +** ^The sqlite3_aggregate_context(C,N) routine returns a NULL pointer +** when first called if N is less than or equal to zero or if a memory +** allocate error occurs. ** ** ^(The amount of space allocated by sqlite3_aggregate_context(C,N) is ** determined by the N parameter on first successful call. Changing the ** value of N in subsequent call to sqlite3_aggregate_context() within ** the same aggregate function instance will not resize the memory -** allocation.)^ +** allocation.)^ Within the xFinal callback, it is customary to set +** N=0 in calls to sqlite3_aggregate_context(C,N) so that no +** pointless memory allocations occur. ** ** ^SQLite automatically frees the memory allocated by ** sqlite3_aggregate_context() when the aggregate query concludes. @@ -4108,7 +4229,7 @@ void sqlite3_set_auxdata(sqlite3_context*, int N, void*, void (*)(void*)); ** the content before returning. ** ** The typedef is necessary to work around problems in certain -** C++ compilers. See ticket #2191. +** C++ compilers. */ typedef void (*sqlite3_destructor_type)(void*); #define SQLITE_STATIC ((sqlite3_destructor_type)0) @@ -4727,6 +4848,9 @@ void *sqlite3_update_hook( ** future releases of SQLite. Applications that care about shared ** cache setting should set it explicitly. ** +** This interface is threadsafe on processors where writing a +** 32-bit integer is atomic. +** ** See Also: [SQLite Shared-Cache Mode] */ int sqlite3_enable_shared_cache(int); @@ -4904,11 +5028,20 @@ int sqlite3_table_column_metadata( ** ^This interface loads an SQLite extension library from the named file. ** ** ^The sqlite3_load_extension() interface attempts to load an -** SQLite extension library contained in the file zFile. +** [SQLite extension] library contained in the file zFile. If +** the file cannot be loaded directly, attempts are made to load +** with various operating-system specific extensions added. +** So for example, if "samplelib" cannot be loaded, then names like +** "samplelib.so" or "samplelib.dylib" or "samplelib.dll" might +** be tried also. ** ** ^The entry point is zProc. -** ^zProc may be 0, in which case the name of the entry point -** defaults to "sqlite3_extension_init". +** ^(zProc may be 0, in which case SQLite will try to come up with an +** entry point name on its own. It first tries "sqlite3_extension_init". +** If that does not work, it constructs a name "sqlite3_X_init" where the +** X is consists of the lower-case equivalent of all ASCII alphabetic +** characters in the filename from the last "/" to the first following +** "." and omitting any initial "lib".)^ ** ^The sqlite3_load_extension() interface returns ** [SQLITE_OK] on success and [SQLITE_ERROR] if something goes wrong. ** ^If an error occurs and pzErrMsg is not 0, then the @@ -4934,11 +5067,11 @@ int sqlite3_load_extension( ** CAPI3REF: Enable Or Disable Extension Loading ** ** ^So as not to open security holes in older applications that are -** unprepared to deal with extension loading, and as a means of disabling -** extension loading while evaluating user-entered SQL, the following API +** unprepared to deal with [extension loading], and as a means of disabling +** [extension loading] while evaluating user-entered SQL, the following API ** is provided to turn the [sqlite3_load_extension()] mechanism on and off. ** -** ^Extension loading is off by default. See ticket #1863. +** ^Extension loading is off by default. ** ^Call the sqlite3_enable_load_extension() routine with onoff==1 ** to turn extension loading on and call it with onoff==0 to turn ** it back off again. @@ -4950,7 +5083,7 @@ int sqlite3_enable_load_extension(sqlite3 *db, int onoff); ** ** ^This interface causes the xEntryPoint() function to be invoked for ** each new [database connection] that is created. The idea here is that -** xEntryPoint() is the entry point for a statically linked SQLite extension +** xEntryPoint() is the entry point for a statically linked [SQLite extension] ** that is to be automatically loaded into all new database connections. ** ** ^(Even though the function prototype shows that xEntryPoint() takes @@ -6301,7 +6434,7 @@ struct sqlite3_pcache_page { ** parameter to help it determined what action to take: ** ** <table border=1 width=85% align=center> -** <tr><th> createFlag <th> Behaviour when page is not already in cache +** <tr><th> createFlag <th> Behavior when page is not already in cache ** <tr><td> 0 <td> Do not allocate a new page. Return NULL. ** <tr><td> 1 <td> Allocate a new page if it easy and convenient to do so. ** Otherwise return NULL. @@ -6731,9 +6864,24 @@ int sqlite3_stricmp(const char *, const char *); int sqlite3_strnicmp(const char *, const char *, int); /* +** CAPI3REF: String Globbing +* +** ^The [sqlite3_strglob(P,X)] interface returns zero if string X matches +** the glob pattern P, and it returns non-zero if string X does not match +** the glob pattern P. ^The definition of glob pattern matching used in +** [sqlite3_strglob(P,X)] is the same as for the "X GLOB P" operator in the +** SQL dialect used by SQLite. ^The sqlite3_strglob(P,X) function is case +** sensitive. +** +** Note that this routine returns zero on a match and non-zero if the strings +** do not match, the same as [sqlite3_stricmp()] and [sqlite3_strnicmp()]. +*/ +int sqlite3_strglob(const char *zGlob, const char *zStr); + +/* ** CAPI3REF: Error Logging Interface ** -** ^The [sqlite3_log()] interface writes a message into the error log +** ^The [sqlite3_log()] interface writes a message into the [error log] ** established by the [SQLITE_CONFIG_LOG] option to [sqlite3_config()]. ** ^If logging is enabled, the zFormat string and subsequent arguments are ** used with [sqlite3_snprintf()] to generate the final output string. diff --git a/src/sqlite3ext.h b/src/sqlite3ext.h index 5abcde2..928bb3b 100644 --- a/src/sqlite3ext.h +++ b/src/sqlite3ext.h @@ -236,6 +236,20 @@ struct sqlite3_api_routines { int (*blob_reopen)(sqlite3_blob*,sqlite3_int64); int (*vtab_config)(sqlite3*,int op,...); int (*vtab_on_conflict)(sqlite3*); + /* Version 3.7.16 and later */ + int (*close_v2)(sqlite3*); + const char *(*db_filename)(sqlite3*,const char*); + int (*db_readonly)(sqlite3*,const char*); + int (*db_release_memory)(sqlite3*); + const char *(*errstr)(int); + int (*stmt_busy)(sqlite3_stmt*); + int (*stmt_readonly)(sqlite3_stmt*); + int (*stricmp)(const char*,const char*); + int (*uri_boolean)(const char*,const char*,int); + sqlite3_int64 (*uri_int64)(const char*,const char*,sqlite3_int64); + const char *(*uri_parameter)(const char*,const char*); + char *(*vsnprintf)(int,char*,const char*,va_list); + int (*wal_checkpoint_v2)(sqlite3*,const char*,int,int*,int*); }; /* @@ -439,9 +453,32 @@ struct sqlite3_api_routines { #define sqlite3_blob_reopen sqlite3_api->blob_reopen #define sqlite3_vtab_config sqlite3_api->vtab_config #define sqlite3_vtab_on_conflict sqlite3_api->vtab_on_conflict +/* Version 3.7.16 and later */ +#define sqlite3_close_v2 sqlite3_api->close_v2 +#define sqlite3_db_filename sqlite3_api->db_filename +#define sqlite3_db_readonly sqlite3_api->db_readonly +#define sqlite3_db_release_memory sqlite3_api->db_release_memory +#define sqlite3_errstr sqlite3_api->errstr +#define sqlite3_stmt_busy sqlite3_api->stmt_busy +#define sqlite3_stmt_readonly sqlite3_api->stmt_readonly +#define sqlite3_stricmp sqlite3_api->stricmp +#define sqlite3_uri_boolean sqlite3_api->uri_boolean +#define sqlite3_uri_int64 sqlite3_api->uri_int64 +#define sqlite3_uri_parameter sqlite3_api->uri_parameter +#define sqlite3_uri_vsnprintf sqlite3_api->vsnprintf +#define sqlite3_wal_checkpoint_v2 sqlite3_api->wal_checkpoint_v2 #endif /* SQLITE_CORE */ -#define SQLITE_EXTENSION_INIT1 const sqlite3_api_routines *sqlite3_api = 0; -#define SQLITE_EXTENSION_INIT2(v) sqlite3_api = v; +#ifndef SQLITE_CORE + /* This case when the file really is being compiled as a loadable + ** extension */ +# define SQLITE_EXTENSION_INIT1 const sqlite3_api_routines *sqlite3_api=0; +# define SQLITE_EXTENSION_INIT2(v) sqlite3_api=v; +#else + /* This case when the file is being statically linked into the + ** application */ +# define SQLITE_EXTENSION_INIT1 /*no-op*/ +# define SQLITE_EXTENSION_INIT2(v) (void)v; /* unused parameter */ +#endif #endif /* _SQLITE3EXT_H_ */ diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 09163bf..5950f23 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -66,6 +66,10 @@ # define _GNU_SOURCE #endif +#if defined(__OpenBSD__) && !defined(_BSD_SOURCE) +# define _BSD_SOURCE +#endif + /* ** Include standard header files as necessary */ @@ -118,11 +122,11 @@ ** We support that for legacy. */ #if !defined(SQLITE_THREADSAFE) -#if defined(THREADSAFE) -# define SQLITE_THREADSAFE THREADSAFE -#else -# define SQLITE_THREADSAFE 1 /* IMP: R-07272-22309 */ -#endif +# if defined(THREADSAFE) +# define SQLITE_THREADSAFE THREADSAFE +# else +# define SQLITE_THREADSAFE 1 /* IMP: R-07272-22309 */ +# endif #endif /* @@ -200,7 +204,8 @@ ** ** See also ticket #2741. */ -#if !defined(_XOPEN_SOURCE) && !defined(__DARWIN__) && !defined(__APPLE__) && SQLITE_THREADSAFE +#if !defined(_XOPEN_SOURCE) && !defined(__DARWIN__) \ + && !defined(__APPLE__) && SQLITE_THREADSAFE # define _XOPEN_SOURCE 500 /* Needed to enable pthread recursive mutexes */ #endif @@ -387,6 +392,7 @@ */ #ifndef SQLITE_TEMP_STORE # define SQLITE_TEMP_STORE 1 +# define SQLITE_TEMP_STORE_xc 1 /* Exclude from ctime.c */ #endif /* @@ -534,6 +540,49 @@ extern const int sqlite3one; # define EIGHT_BYTE_ALIGNMENT(X) ((((char*)(X) - (char*)0)&7)==0) #endif +/* +** Disable MMAP on platforms where it is known to not work +*/ +#if defined(__OpenBSD__) || defined(__QNXNTO__) +# undef SQLITE_MAX_MMAP_SIZE +# define SQLITE_MAX_MMAP_SIZE 0 +#endif + +/* +** Default maximum size of memory used by memory-mapped I/O in the VFS +*/ +#ifdef __APPLE__ +# include <TargetConditionals.h> +# if TARGET_OS_IPHONE +# undef SQLITE_MAX_MMAP_SIZE +# define SQLITE_MAX_MMAP_SIZE 0 +# endif +#endif +#ifndef SQLITE_MAX_MMAP_SIZE +# if defined(__linux__) \ + || defined(_WIN32) \ + || (defined(__APPLE__) && defined(__MACH__)) \ + || defined(__sun) +# define SQLITE_MAX_MMAP_SIZE 0x7fff0000 /* 2147418112 */ +# else +# define SQLITE_MAX_MMAP_SIZE 0 +# endif +# define SQLITE_MAX_MMAP_SIZE_xc 1 /* exclude from ctime.c */ +#endif + +/* +** The default MMAP_SIZE is zero on all platforms. Or, even if a larger +** default MMAP_SIZE is specified at compile-time, make sure that it does +** not exceed the maximum mmap size. +*/ +#ifndef SQLITE_DEFAULT_MMAP_SIZE +# define SQLITE_DEFAULT_MMAP_SIZE 0 +# define SQLITE_DEFAULT_MMAP_SIZE_xc 1 /* Exclude from ctime.c */ +#endif +#if SQLITE_DEFAULT_MMAP_SIZE>SQLITE_MAX_MMAP_SIZE +# undef SQLITE_DEFAULT_MMAP_SIZE +# define SQLITE_DEFAULT_MMAP_SIZE SQLITE_MAX_MMAP_SIZE +#endif /* ** An instance of the following structure is used to store the busy-handler @@ -576,6 +625,11 @@ struct BusyHandler { #define ArraySize(X) ((int)(sizeof(X)/sizeof(X[0]))) /* +** Determine if the argument is a power of two +*/ +#define IsPowerOfTwo(X) (((X)&((X)-1))==0) + +/* ** The following value as a destructor means to use sqlite3DbFree(). ** The sqlite3DbFree() routine requires two parameters instead of the ** one parameter that destructors normally want. So we have to introduce @@ -661,6 +715,7 @@ typedef struct Parse Parse; typedef struct RowSet RowSet; typedef struct Savepoint Savepoint; typedef struct Select Select; +typedef struct SelectDest SelectDest; typedef struct SrcList SrcList; typedef struct StrAccum StrAccum; typedef struct Table Table; @@ -823,9 +878,11 @@ struct sqlite3 { int nDb; /* Number of backends currently in use */ int flags; /* Miscellaneous flags. See below */ i64 lastRowid; /* ROWID of most recent insert (see above) */ + i64 szMmap; /* Default mmap_size setting */ unsigned int openFlags; /* Flags passed to sqlite3_vfs.xOpen() */ int errCode; /* Most recent error code (SQLITE_*) */ int errMask; /* & result codes with this before returning */ + u16 dbOptFlags; /* Flags to enable/disable optimizations */ u8 autoCommit; /* The auto-commit flag. */ u8 temp_store; /* 1: file 2: memory 0: default */ u8 mallocFailed; /* True if we have seen a malloc failure */ @@ -930,48 +987,60 @@ struct sqlite3 { /* ** Possible values for the sqlite3.flags. */ -#define SQLITE_VdbeTrace 0x00000100 /* True to trace VDBE execution */ -#define SQLITE_InternChanges 0x00000200 /* Uncommitted Hash table changes */ -#define SQLITE_FullColNames 0x00000400 /* Show full column names on SELECT */ -#define SQLITE_ShortColNames 0x00000800 /* Show short columns names */ -#define SQLITE_CountRows 0x00001000 /* Count rows changed by INSERT, */ +#define SQLITE_VdbeTrace 0x00000001 /* True to trace VDBE execution */ +#define SQLITE_InternChanges 0x00000002 /* Uncommitted Hash table changes */ +#define SQLITE_FullColNames 0x00000004 /* Show full column names on SELECT */ +#define SQLITE_ShortColNames 0x00000008 /* Show short columns names */ +#define SQLITE_CountRows 0x00000010 /* Count rows changed by INSERT, */ /* DELETE, or UPDATE and return */ /* the count using a callback. */ -#define SQLITE_NullCallback 0x00002000 /* Invoke the callback once if the */ +#define SQLITE_NullCallback 0x00000020 /* Invoke the callback once if the */ /* result set is empty */ -#define SQLITE_SqlTrace 0x00004000 /* Debug print SQL as it executes */ -#define SQLITE_VdbeListing 0x00008000 /* Debug listings of VDBE programs */ -#define SQLITE_WriteSchema 0x00010000 /* OK to update SQLITE_MASTER */ - /* 0x00020000 Unused */ -#define SQLITE_IgnoreChecks 0x00040000 /* Do not enforce check constraints */ -#define SQLITE_ReadUncommitted 0x0080000 /* For shared-cache mode */ -#define SQLITE_LegacyFileFmt 0x00100000 /* Create new databases in format 1 */ -#define SQLITE_FullFSync 0x00200000 /* Use full fsync on the backend */ -#define SQLITE_CkptFullFSync 0x00400000 /* Use full fsync for checkpoint */ -#define SQLITE_RecoveryMode 0x00800000 /* Ignore schema errors */ -#define SQLITE_ReverseOrder 0x01000000 /* Reverse unordered SELECTs */ -#define SQLITE_RecTriggers 0x02000000 /* Enable recursive triggers */ -#define SQLITE_ForeignKeys 0x04000000 /* Enforce foreign key constraints */ -#define SQLITE_AutoIndex 0x08000000 /* Enable automatic indexes */ -#define SQLITE_PreferBuiltin 0x10000000 /* Preference to built-in funcs */ -#define SQLITE_LoadExtension 0x20000000 /* Enable load_extension */ -#define SQLITE_EnableTrigger 0x40000000 /* True to enable triggers */ - -/* -** Bits of the sqlite3.flags field that are used by the -** sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS,...) interface. -** These must be the low-order bits of the flags field. -*/ -#define SQLITE_QueryFlattener 0x01 /* Disable query flattening */ -#define SQLITE_ColumnCache 0x02 /* Disable the column cache */ -#define SQLITE_IndexSort 0x04 /* Disable indexes for sorting */ -#define SQLITE_IndexSearch 0x08 /* Disable indexes for searching */ -#define SQLITE_IndexCover 0x10 /* Disable index covering table */ -#define SQLITE_GroupByOrder 0x20 /* Disable GROUPBY cover of ORDERBY */ -#define SQLITE_FactorOutConst 0x40 /* Disable factoring out constants */ -#define SQLITE_IdxRealAsInt 0x80 /* Store REAL as INT in indices */ -#define SQLITE_DistinctOpt 0x80 /* DISTINCT using indexes */ -#define SQLITE_OptMask 0xff /* Mask of all disablable opts */ +#define SQLITE_SqlTrace 0x00000040 /* Debug print SQL as it executes */ +#define SQLITE_VdbeListing 0x00000080 /* Debug listings of VDBE programs */ +#define SQLITE_WriteSchema 0x00000100 /* OK to update SQLITE_MASTER */ +#define SQLITE_VdbeAddopTrace 0x00000200 /* Trace sqlite3VdbeAddOp() calls */ +#define SQLITE_IgnoreChecks 0x00000400 /* Do not enforce check constraints */ +#define SQLITE_ReadUncommitted 0x0000800 /* For shared-cache mode */ +#define SQLITE_LegacyFileFmt 0x00001000 /* Create new databases in format 1 */ +#define SQLITE_FullFSync 0x00002000 /* Use full fsync on the backend */ +#define SQLITE_CkptFullFSync 0x00004000 /* Use full fsync for checkpoint */ +#define SQLITE_RecoveryMode 0x00008000 /* Ignore schema errors */ +#define SQLITE_ReverseOrder 0x00010000 /* Reverse unordered SELECTs */ +#define SQLITE_RecTriggers 0x00020000 /* Enable recursive triggers */ +#define SQLITE_ForeignKeys 0x00040000 /* Enforce foreign key constraints */ +#define SQLITE_AutoIndex 0x00080000 /* Enable automatic indexes */ +#define SQLITE_PreferBuiltin 0x00100000 /* Preference to built-in funcs */ +#define SQLITE_LoadExtension 0x00200000 /* Enable load_extension */ +#define SQLITE_EnableTrigger 0x00400000 /* True to enable triggers */ + +/* +** Bits of the sqlite3.dbOptFlags field that are used by the +** sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS,...) interface to +** selectively disable various optimizations. +*/ +#define SQLITE_QueryFlattener 0x0001 /* Query flattening */ +#define SQLITE_ColumnCache 0x0002 /* Column cache */ +#define SQLITE_GroupByOrder 0x0004 /* GROUPBY cover of ORDERBY */ +#define SQLITE_FactorOutConst 0x0008 /* Constant factoring */ +#define SQLITE_IdxRealAsInt 0x0010 /* Store REAL as INT in indices */ +#define SQLITE_DistinctOpt 0x0020 /* DISTINCT using indexes */ +#define SQLITE_CoverIdxScan 0x0040 /* Covering index scans */ +#define SQLITE_OrderByIdxJoin 0x0080 /* ORDER BY of joins via index */ +#define SQLITE_SubqCoroutine 0x0100 /* Evaluate subqueries as coroutines */ +#define SQLITE_Transitive 0x0200 /* Transitive constraints */ +#define SQLITE_AllOpts 0xffff /* All optimizations */ + +/* +** Macros for testing whether or not optimizations are enabled or disabled. +*/ +#ifndef SQLITE_OMIT_BUILTIN_TEST +#define OptimizationDisabled(db, mask) (((db)->dbOptFlags&(mask))!=0) +#define OptimizationEnabled(db, mask) (((db)->dbOptFlags&(mask))==0) +#else +#define OptimizationDisabled(db, mask) 0 +#define OptimizationEnabled(db, mask) 1 +#endif /* ** Possible values for the sqlite.magic field. @@ -1122,32 +1191,22 @@ struct Column { char *zDflt; /* Original text of the default value */ char *zType; /* Data type for this column */ char *zColl; /* Collating sequence. If NULL, use the default */ - u8 notNull; /* True if there is a NOT NULL constraint */ - u8 isPrimKey; /* True if this column is part of the PRIMARY KEY */ + u8 notNull; /* An OE_ code for handling a NOT NULL constraint */ char affinity; /* One of the SQLITE_AFF_... values */ -#ifndef SQLITE_OMIT_VIRTUALTABLE - u8 isHidden; /* True if this column is 'hidden' */ -#endif + u16 colFlags; /* Boolean properties. See COLFLAG_ defines below */ }; +/* Allowed values for Column.colFlags: +*/ +#define COLFLAG_PRIMKEY 0x0001 /* Column is part of the primary key */ +#define COLFLAG_HIDDEN 0x0002 /* A hidden column in a virtual table */ + /* ** A "Collating Sequence" is defined by an instance of the following ** structure. Conceptually, a collating sequence consists of a name and ** a comparison routine that defines the order of that sequence. ** -** There may two separate implementations of the collation function, one -** that processes text in UTF-8 encoding (CollSeq.xCmp) and another that -** processes text encoded in UTF-16 (CollSeq.xCmp16), using the machine -** native byte order. When a collation sequence is invoked, SQLite selects -** the version that will require the least expensive encoding -** translations, if any. -** -** The CollSeq.pUser member variable is an extra parameter that passed in -** as the first argument to the UTF-8 comparison function, xCmp. -** CollSeq.pUser16 is the equivalent for the UTF-16 comparison function, -** xCmp16. -** -** If both CollSeq.xCmp and CollSeq.xCmp16 are NULL, it means that the +** If CollSeq.xCmp is NULL, it means that the ** collating sequence is undefined. Indices built on an undefined ** collating sequence may not be read or written. */ @@ -1285,28 +1344,28 @@ struct VTable { */ struct Table { char *zName; /* Name of the table or view */ - int iPKey; /* If not negative, use aCol[iPKey] as the primary key */ - int nCol; /* Number of columns in this table */ Column *aCol; /* Information about each column */ Index *pIndex; /* List of SQL indexes on this table. */ - int tnum; /* Root BTree node for this table (see note above) */ - tRowcnt nRowEst; /* Estimated rows in table - from sqlite_stat1 table */ Select *pSelect; /* NULL for tables. Points to definition if a view. */ - u16 nRef; /* Number of pointers to this Table */ - u8 tabFlags; /* Mask of TF_* values */ - u8 keyConf; /* What to do in case of uniqueness conflict on iPKey */ FKey *pFKey; /* Linked list of all foreign keys in this table */ char *zColAff; /* String defining the affinity of each column */ #ifndef SQLITE_OMIT_CHECK ExprList *pCheck; /* All CHECK constraints */ #endif + tRowcnt nRowEst; /* Estimated rows in table - from sqlite_stat1 table */ + int tnum; /* Root BTree node for this table (see note above) */ + i16 iPKey; /* If not negative, use aCol[iPKey] as the primary key */ + i16 nCol; /* Number of columns in this table */ + u16 nRef; /* Number of pointers to this Table */ + u8 tabFlags; /* Mask of TF_* values */ + u8 keyConf; /* What to do in case of uniqueness conflict on iPKey */ #ifndef SQLITE_OMIT_ALTERTABLE int addColOffset; /* Offset in CREATE TABLE stmt to add a new column */ #endif #ifndef SQLITE_OMIT_VIRTUALTABLE - VTable *pVTable; /* List of VTable objects. */ int nModuleArg; /* Number of arguments to the module */ char **azModuleArg; /* Text of all module args. [0] is module name */ + VTable *pVTable; /* List of VTable objects. */ #endif Trigger *pTrigger; /* List of triggers stored in pSchema */ Schema *pSchema; /* Schema that contains this table */ @@ -1330,7 +1389,7 @@ struct Table { */ #ifndef SQLITE_OMIT_VIRTUALTABLE # define IsVirtual(X) (((X)->tabFlags & TF_Virtual)!=0) -# define IsHiddenColumn(X) ((X)->isHidden) +# define IsHiddenColumn(X) (((X)->colFlags & COLFLAG_HIDDEN)!=0) #else # define IsVirtual(X) 0 # define IsHiddenColumn(X) 0 @@ -1481,20 +1540,20 @@ struct UnpackedRecord { ** element. */ struct Index { - char *zName; /* Name of this index */ - int *aiColumn; /* Which columns are used by this index. 1st is 0 */ - tRowcnt *aiRowEst; /* Result of ANALYZE: Est. rows selected by each column */ - Table *pTable; /* The SQL table being indexed */ - char *zColAff; /* String defining the affinity of each column */ - Index *pNext; /* The next index associated with the same table */ - Schema *pSchema; /* Schema containing this index */ - u8 *aSortOrder; /* Array of size Index.nColumn. True==DESC, False==ASC */ - char **azColl; /* Array of collation sequence names for index */ - int nColumn; /* Number of columns in the table used by this index */ - int tnum; /* Page containing root of this index in database file */ - u8 onError; /* OE_Abort, OE_Ignore, OE_Replace, or OE_None */ - u8 autoIndex; /* True if is automatically created (ex: by UNIQUE) */ - u8 bUnordered; /* Use this index for == or IN queries only */ + char *zName; /* Name of this index */ + int *aiColumn; /* Which columns are used by this index. 1st is 0 */ + tRowcnt *aiRowEst; /* From ANALYZE: Est. rows selected by each column */ + Table *pTable; /* The SQL table being indexed */ + char *zColAff; /* String defining the affinity of each column */ + Index *pNext; /* The next index associated with the same table */ + Schema *pSchema; /* Schema containing this index */ + u8 *aSortOrder; /* for each column: True==DESC, False==ASC */ + char **azColl; /* Array of collation sequence names for index */ + int tnum; /* DB Page containing root of this index */ + u16 nColumn; /* Number of columns in table used by this index */ + u8 onError; /* OE_Abort, OE_Ignore, OE_Replace, or OE_None */ + unsigned autoIndex:2; /* 1==UNIQUE, 2==PRIMARY KEY, 0==CREATE INDEX */ + unsigned bUnordered:1; /* Use this index for == or IN queries only */ #ifdef SQLITE_ENABLE_STAT3 int nSample; /* Number of elements in aSample[] */ tRowcnt avgEq; /* Average nEq value for key values not in aSample */ @@ -1675,13 +1734,15 @@ struct Expr { ExprList *pList; /* Function arguments or in "<expr> IN (<expr-list)" */ Select *pSelect; /* Used for sub-selects and "<expr> IN (<select>)" */ } x; - CollSeq *pColl; /* The collation type of the column or 0 */ /* If the EP_Reduced flag is set in the Expr.flags mask, then no ** space is allocated for the fields below this point. An attempt to ** access them will result in a segfault or malfunction. *********************************************************************/ +#if SQLITE_MAX_EXPR_DEPTH>0 + int nHeight; /* Height of the tree headed by this node */ +#endif int iTable; /* TK_COLUMN: cursor number of table holding column ** TK_REGISTER: register number ** TK_TRIGGER: 1 -> new, 0 -> old */ @@ -1695,9 +1756,6 @@ struct Expr { ** TK_AGG_FUNCTION: nesting depth */ AggInfo *pAggInfo; /* Used by TK_AGG_COLUMN and TK_AGG_FUNCTION */ Table *pTab; /* Table for TK_COLUMN expressions. */ -#if SQLITE_MAX_EXPR_DEPTH>0 - int nHeight; /* Height of the tree headed by this node */ -#endif }; /* @@ -1711,7 +1769,7 @@ struct Expr { #define EP_VarSelect 0x0020 /* pSelect is correlated, not constant */ #define EP_DblQuoted 0x0040 /* token.z was originally in "..." */ #define EP_InfixFunc 0x0080 /* True for an infix function: LIKE, GLOB, etc */ -#define EP_ExpCollate 0x0100 /* Collating sequence specified explicitly */ +#define EP_Collate 0x0100 /* Tree contains a TK_COLLATE opeartor */ #define EP_FixedDest 0x0200 /* Result needed in a specific register */ #define EP_IntValue 0x0400 /* Integer value contained in u.iValue */ #define EP_xIsSelect 0x0800 /* x.pSelect is valid (otherwise x.pList is) */ @@ -1769,18 +1827,27 @@ struct Expr { ** list of "ID = expr" items in an UPDATE. A list of expressions can ** also be used as the argument to a function, in which case the a.zName ** field is not used. +** +** By default the Expr.zSpan field holds a human-readable description of +** the expression that is used in the generation of error messages and +** column labels. In this case, Expr.zSpan is typically the text of a +** column expression as it exists in a SELECT statement. However, if +** the bSpanIsTab flag is set, then zSpan is overloaded to mean the name +** of the result column in the form: DATABASE.TABLE.COLUMN. This later +** form is used for name resolution with nested FROM clauses. */ struct ExprList { int nExpr; /* Number of expressions on the list */ int iECursor; /* VDBE Cursor associated with this ExprList */ struct ExprList_item { /* For each expression in the list */ - Expr *pExpr; /* The list of expressions */ - char *zName; /* Token associated with this expression */ - char *zSpan; /* Original text of the expression */ - u8 sortOrder; /* 1 for DESC or 0 for ASC */ - u8 done; /* A flag to indicate when processing is finished */ - u16 iOrderByCol; /* For ORDER BY, column number in result set */ - u16 iAlias; /* Index into Parse.aAlias[] for zName */ + Expr *pExpr; /* The list of expressions */ + char *zName; /* Token associated with this expression */ + char *zSpan; /* Original text of the expression */ + u8 sortOrder; /* 1 for DESC or 0 for ASC */ + unsigned done :1; /* A flag to indicate when processing is finished */ + unsigned bSpanIsTab :1; /* zSpan holds DB.TABLE.COLUMN */ + u16 iOrderByCol; /* For ORDER BY, column number in result set */ + u16 iAlias; /* Index into Parse.aAlias[] for zName */ } *a; /* Alloc a power of two greater or equal to nExpr */ }; @@ -1855,6 +1922,7 @@ struct SrcList { i16 nSrc; /* Number of tables or subqueries in the FROM clause */ i16 nAlloc; /* Number of entries allocated in a[] below */ struct SrcList_item { + Schema *pSchema; /* Schema to which this item is fixed */ char *zDatabase; /* Name of database holding this table */ char *zName; /* Name of the table */ char *zAlias; /* The "B" part of a "A AS B" phrase. zName is the "A" */ @@ -1863,8 +1931,9 @@ struct SrcList { int addrFillSub; /* Address of subroutine to manifest a subquery */ int regReturn; /* Register holding return address of addrFillSub */ u8 jointype; /* Type of join between this able and the previous */ - u8 notIndexed; /* True if there is a NOT INDEXED clause */ - u8 isCorrelated; /* True if sub-query is correlated */ + unsigned notIndexed :1; /* True if there is a NOT INDEXED clause */ + unsigned isCorrelated :1; /* True if sub-query is correlated */ + unsigned viaCoroutine :1; /* Implemented as a co-routine */ #ifndef SQLITE_OMIT_EXPLAIN u8 iSelectId; /* If pSelect!=0, the id of the sub-select in EQP */ #endif @@ -1905,7 +1974,8 @@ struct SrcList { */ struct WherePlan { u32 wsFlags; /* WHERE_* flags that describe the strategy */ - u32 nEq; /* Number of == constraints */ + u16 nEq; /* Number of == constraints */ + u16 nOBSat; /* Number of ORDER BY terms satisfied */ double nRow; /* Estimated number of rows (for EQP) */ union { Index *pIdx; /* Index when WHERE_INDEXED is true */ @@ -1945,10 +2015,12 @@ struct WhereLevel { struct InLoop { int iCur; /* The VDBE cursor used by this IN operator */ int addrInTop; /* Top of the IN loop */ + u8 eEndLoopOp; /* IN Loop terminator. OP_Next or OP_Prev */ } *aInLoop; /* Information about each nested IN operator */ } in; /* Used when plan.wsFlags&WHERE_IN_ABLE */ Index *pCovidx; /* Possible covering index for WHERE_MULTI_OR */ } u; + double rOptCost; /* "Optimal" cost for this level */ /* The following field is really not part of the current level. But ** we need a place to cache virtual table index information for each @@ -1981,24 +2053,28 @@ struct WhereLevel { ** into the second half to give some continuity. */ struct WhereInfo { - Parse *pParse; /* Parsing and code generating context */ - u16 wctrlFlags; /* Flags originally passed to sqlite3WhereBegin() */ - u8 okOnePass; /* Ok to use one-pass algorithm for UPDATE or DELETE */ - u8 untestedTerms; /* Not all WHERE terms resolved by outer loop */ - u8 eDistinct; - SrcList *pTabList; /* List of tables in the join */ - int iTop; /* The very beginning of the WHERE loop */ - int iContinue; /* Jump here to continue with next record */ - int iBreak; /* Jump here to break out of the loop */ - int nLevel; /* Number of nested loop */ - struct WhereClause *pWC; /* Decomposition of the WHERE clause */ - double savedNQueryLoop; /* pParse->nQueryLoop outside the WHERE loop */ - double nRowOut; /* Estimated number of output rows */ - WhereLevel a[1]; /* Information about each nest loop in WHERE */ + Parse *pParse; /* Parsing and code generating context */ + SrcList *pTabList; /* List of tables in the join */ + u16 nOBSat; /* Number of ORDER BY terms satisfied by indices */ + u16 wctrlFlags; /* Flags originally passed to sqlite3WhereBegin() */ + u8 okOnePass; /* Ok to use one-pass algorithm for UPDATE/DELETE */ + u8 untestedTerms; /* Not all WHERE terms resolved by outer loop */ + u8 eDistinct; /* One of the WHERE_DISTINCT_* values below */ + int iTop; /* The very beginning of the WHERE loop */ + int iContinue; /* Jump here to continue with next record */ + int iBreak; /* Jump here to break out of the loop */ + int nLevel; /* Number of nested loop */ + struct WhereClause *pWC; /* Decomposition of the WHERE clause */ + double savedNQueryLoop; /* pParse->nQueryLoop outside the WHERE loop */ + double nRowOut; /* Estimated number of output rows */ + WhereLevel a[1]; /* Information about each nest loop in WHERE */ }; -#define WHERE_DISTINCT_UNIQUE 1 -#define WHERE_DISTINCT_ORDERED 2 +/* Allowed values for WhereInfo.eDistinct and DistinctCtx.eTnctType */ +#define WHERE_DISTINCT_NOOP 0 /* DISTINCT keyword not used */ +#define WHERE_DISTINCT_UNIQUE 1 /* No duplicates */ +#define WHERE_DISTINCT_ORDERED 2 /* All duplicates are adjacent */ +#define WHERE_DISTINCT_UNORDERED 3 /* Duplicates are scattered */ /* ** A NameContext defines a context in which to resolve table and column @@ -2039,6 +2115,8 @@ struct NameContext { #define NC_HasAgg 0x02 /* One or more aggregate functions seen */ #define NC_IsCheck 0x04 /* True if resolving names in a CHECK constraint */ #define NC_InAggFunc 0x08 /* True if analyzing arguments to an agg func */ +#define NC_AsMaybe 0x10 /* Resolve to AS terms of the result set only + ** if no other resolution is available */ /* ** An instance of the following structure contains all information @@ -2057,13 +2135,12 @@ struct NameContext { ** as the OP_OpenEphm instruction is coded because not ** enough information about the compound query is known at that point. ** The KeyInfo for addrOpenTran[0] and [1] contains collating sequences -** for the result set. The KeyInfo for addrOpenTran[2] contains collating +** for the result set. The KeyInfo for addrOpenEphm[2] contains collating ** sequences for the ORDER BY clause. */ struct Select { ExprList *pEList; /* The fields of the result */ u8 op; /* One of: TK_UNION TK_ALL TK_INTERSECT TK_EXCEPT */ - char affinity; /* MakeRecord with this affinity for SRT_Set */ u16 selFlags; /* Various SF_* values */ int iLimit, iOffset; /* Memory registers holding LIMIT & OFFSET counters */ int addrOpenEphm[3]; /* OP_OpenEphem opcodes related to this select */ @@ -2084,14 +2161,16 @@ struct Select { ** Allowed values for Select.selFlags. The "SF" prefix stands for ** "Select Flag". */ -#define SF_Distinct 0x01 /* Output should be DISTINCT */ -#define SF_Resolved 0x02 /* Identifiers have been resolved */ -#define SF_Aggregate 0x04 /* Contains aggregate functions */ -#define SF_UsesEphemeral 0x08 /* Uses the OpenEphemeral opcode */ -#define SF_Expanded 0x10 /* sqlite3SelectExpand() called on this */ -#define SF_HasTypeInfo 0x20 /* FROM subqueries have Table metadata */ -#define SF_UseSorter 0x40 /* Sort using a sorter */ -#define SF_Values 0x80 /* Synthesized from VALUES clause */ +#define SF_Distinct 0x0001 /* Output should be DISTINCT */ +#define SF_Resolved 0x0002 /* Identifiers have been resolved */ +#define SF_Aggregate 0x0004 /* Contains aggregate functions */ +#define SF_UsesEphemeral 0x0008 /* Uses the OpenEphemeral opcode */ +#define SF_Expanded 0x0010 /* sqlite3SelectExpand() called on this */ +#define SF_HasTypeInfo 0x0020 /* FROM subqueries have Table metadata */ +#define SF_UseSorter 0x0040 /* Sort using a sorter */ +#define SF_Values 0x0080 /* Synthesized from VALUES clause */ +#define SF_Materialize 0x0100 /* Force materialization of views */ +#define SF_NestedFrom 0x0200 /* Part of a parenthesized FROM clause */ /* @@ -2114,13 +2193,12 @@ struct Select { #define SRT_Coroutine 10 /* Generate a single row of result */ /* -** A structure used to customize the behavior of sqlite3Select(). See -** comments above sqlite3Select() for details. +** An instance of this object describes where to put of the results of +** a SELECT statement. */ -typedef struct SelectDest SelectDest; struct SelectDest { - u8 eDest; /* How to dispose of the results */ - u8 affSdst; /* Affinity used when eDest==SRT_Set */ + u8 eDest; /* How to dispose of the results. On of SRT_* above. */ + char affSdst; /* Affinity used when eDest==SRT_Set */ int iSDParm; /* A parameter used by the eDest disposal method */ int iSdst; /* Base register where results are written */ int nSdst; /* Number of registers allocated */ @@ -2321,6 +2399,7 @@ struct AuthContext { #define OPFLAG_TYPEOFARG 0x80 /* OP_Column only used for typeof() */ #define OPFLAG_BULKCSR 0x01 /* OP_Open** used to open bulk cursor */ #define OPFLAG_P2ISREG 0x02 /* P2 to OP_Open** is a register number */ +#define OPFLAG_PERMUTE 0x01 /* OP_Compare: use the permutation */ /* * Each trigger present in the database schema is stored as an instance of @@ -2420,6 +2499,7 @@ struct TriggerStep { typedef struct DbFixer DbFixer; struct DbFixer { Parse *pParse; /* The parsing context. Error messages written here */ + Schema *pSchema; /* Fix items to this schema */ const char *zDb; /* Make sure all objects are contained in this database */ const char *zType; /* Type of the container - used for error messages */ const Token *pName; /* Name of the container - used for error messages */ @@ -2462,6 +2542,7 @@ struct Sqlite3Config { int bCoreMutex; /* True to enable core mutexing */ int bFullMutex; /* True to enable full mutexing */ int bOpenUri; /* True to interpret filenames as URIs */ + int bUseCis; /* Use covering indices for full-scans */ int mxStrlen; /* Maximum string length */ int szLookaside; /* Default lookaside buffer size */ int nLookaside; /* Default lookaside buffer count */ @@ -2471,6 +2552,8 @@ struct Sqlite3Config { void *pHeap; /* Heap storage space */ int nHeap; /* Size of pHeap[] */ int mnReq, mxReq; /* Min and max heap requests sizes */ + sqlite3_int64 szMmap; /* mmap() space per open file */ + sqlite3_int64 mxMmap; /* Maximum value for szMmap */ void *pScratch; /* Scratch memory */ int szScratch; /* Size of each scratch buffer */ int nScratch; /* Number of scratch buffers */ @@ -2491,6 +2574,10 @@ struct Sqlite3Config { void (*xLog)(void*,int,const char*); /* Function for logging */ void *pLogArg; /* First argument to xLog() */ int bLocaltimeFault; /* True to fail localtime() calls */ +#ifdef SQLITE_ENABLE_SQLLOG + void(*xSqllog)(void*,sqlite3*,const char*, int); + void *pSqllogArg; +#endif }; /* @@ -2501,6 +2588,7 @@ struct Walker { int (*xSelectCallback)(Walker*,Select*); /* Callback for SELECTs */ Parse *pParse; /* Parser context. */ int walkerDepth; /* Number of subqueries */ + u8 bSelectDepthFirst; /* Do subqueries first */ union { /* Extra data for callback */ NameContext *pNC; /* Naming context */ int i; /* Integer value */ @@ -2778,6 +2866,7 @@ void sqlite3DeleteTable(sqlite3*, Table*); # define sqlite3AutoincrementBegin(X) # define sqlite3AutoincrementEnd(X) #endif +int sqlite3CodeCoroutine(Parse*, Select*, SelectDest*); void sqlite3Insert(Parse*, SrcList*, ExprList*, Select*, IdList*, int); void *sqlite3ArrayAllocate(sqlite3*,void*,int,int*,int*); IdList *sqlite3IdListAppend(sqlite3*, IdList*, Token*); @@ -2797,23 +2886,21 @@ Index *sqlite3CreateIndex(Parse*,Token*,Token*,SrcList*,ExprList*,int,Token*, void sqlite3DropIndex(Parse*, SrcList*, int); int sqlite3Select(Parse*, Select*, SelectDest*); Select *sqlite3SelectNew(Parse*,ExprList*,SrcList*,Expr*,ExprList*, - Expr*,ExprList*,int,Expr*,Expr*); + Expr*,ExprList*,u16,Expr*,Expr*); void sqlite3SelectDelete(sqlite3*, Select*); Table *sqlite3SrcListLookup(Parse*, SrcList*); int sqlite3IsReadOnly(Parse*, Table*, int); void sqlite3OpenTable(Parse*, int iCur, int iDb, Table*, int); #if defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) && !defined(SQLITE_OMIT_SUBQUERY) -Expr *sqlite3LimitWhere(Parse *, SrcList *, Expr *, ExprList *, Expr *, Expr *, char *); +Expr *sqlite3LimitWhere(Parse*,SrcList*,Expr*,ExprList*,Expr*,Expr*,char*); #endif void sqlite3DeleteFrom(Parse*, SrcList*, Expr*); void sqlite3Update(Parse*, SrcList*, ExprList*, Expr*, int); -WhereInfo *sqlite3WhereBegin( - Parse*,SrcList*,Expr*,ExprList**,ExprList*,u16,int); +WhereInfo *sqlite3WhereBegin(Parse*,SrcList*,Expr*,ExprList*,ExprList*,u16,int); void sqlite3WhereEnd(WhereInfo*); int sqlite3ExprCodeGetColumn(Parse*, Table*, int, int, int, u8); void sqlite3ExprCodeGetColumnOfTable(Vdbe*, Table*, int, int, int); void sqlite3ExprCodeMove(Parse*, int, int, int); -void sqlite3ExprCodeCopy(Parse*, int, int, int); void sqlite3ExprCacheStore(Parse*, int, int, int); void sqlite3ExprCachePush(Parse*); void sqlite3ExprCachePop(Parse*, int); @@ -2830,6 +2917,7 @@ void sqlite3ExprIfTrue(Parse*, Expr*, int, int); void sqlite3ExprIfFalse(Parse*, Expr*, int, int); Table *sqlite3FindTable(sqlite3*,const char*, const char*); Table *sqlite3LocateTable(Parse*,int isView,const char*, const char*); +Table *sqlite3LocateTableItem(Parse*,int isView,struct SrcList_item *); Index *sqlite3FindIndex(sqlite3*,const char*, const char*); void sqlite3UnlinkAndDeleteTable(sqlite3*,int,const char*); void sqlite3UnlinkAndDeleteIndex(sqlite3*,int,const char*); @@ -2872,7 +2960,7 @@ int sqlite3OpenTableAndIndices(Parse*, Table*, int, int); void sqlite3BeginWriteOperation(Parse*, int, int); void sqlite3MultiWrite(Parse*); void sqlite3MayAbort(Parse*); -void sqlite3HaltConstraint(Parse*, int, char*, int); +void sqlite3HaltConstraint(Parse*, int, int, char*, int); Expr *sqlite3ExprDup(sqlite3*,Expr*,int); ExprList *sqlite3ExprListDup(sqlite3*,ExprList*,int); SrcList *sqlite3SrcListDup(sqlite3*,SrcList*,int); @@ -2953,7 +3041,7 @@ int sqlite3GetInt32(const char *, int*); int sqlite3Atoi(const char*); int sqlite3Utf16ByteLen(const void *pData, int nChar); int sqlite3Utf8CharLen(const char *pData, int nByte); -u32 sqlite3Utf8Read(const u8*, const u8**); +u32 sqlite3Utf8Read(const u8**); /* ** Routines to read and write variable-length integers. These used to @@ -2985,8 +3073,11 @@ int sqlite3VarintLen(u64 v); ** x = putVarint32( A, B ); ** */ -#define getVarint32(A,B) (u8)((*(A)<(u8)0x80) ? ((B) = (u32)*(A)),1 : sqlite3GetVarint32((A), (u32 *)&(B))) -#define putVarint32(A,B) (u8)(((u32)(B)<(u32)0x80) ? (*(A) = (unsigned char)(B)),1 : sqlite3PutVarint32((A), (B))) +#define getVarint32(A,B) \ + (u8)((*(A)<(u8)0x80)?((B)=(u32)*(A)),1:sqlite3GetVarint32((A),(u32 *)&(B))) +#define putVarint32(A,B) \ + (u8)(((u32)(B)<(u32)0x80)?(*(A)=(unsigned char)(B)),1:\ + sqlite3PutVarint32((A),(B))) #define getVarint sqlite3GetVarint #define putVarint sqlite3PutVarint @@ -3001,13 +3092,20 @@ void sqlite3Error(sqlite3*, int, const char*,...); void *sqlite3HexToBlob(sqlite3*, const char *z, int n); u8 sqlite3HexToInt(int h); int sqlite3TwoPartName(Parse *, Token *, Token *, Token **); + +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) || \ + defined(SQLITE_DEBUG_OS_TRACE) +const char *sqlite3ErrName(int); +#endif + const char *sqlite3ErrStr(int); int sqlite3ReadSchema(Parse *pParse); CollSeq *sqlite3FindCollSeq(sqlite3*,u8 enc, const char*,int); CollSeq *sqlite3LocateCollSeq(Parse *pParse, const char*zName); CollSeq *sqlite3ExprCollSeq(Parse *pParse, Expr *pExpr); -Expr *sqlite3ExprSetColl(Expr*, CollSeq*); -Expr *sqlite3ExprSetCollByToken(Parse *pParse, Expr*, Token*); +Expr *sqlite3ExprAddCollateToken(Parse *pParse, Expr*, Token*); +Expr *sqlite3ExprAddCollateString(Parse*,Expr*,const char*); +Expr *sqlite3ExprSkipCollate(Expr*); int sqlite3CheckCollSeq(Parse *, CollSeq *); int sqlite3CheckObjectName(Parse *, const char *); void sqlite3VdbeSetChanges(sqlite3 *, int); @@ -3054,13 +3152,14 @@ void sqlite3NestedParse(Parse*, const char*, ...); void sqlite3ExpirePreparedStatements(sqlite3*); int sqlite3CodeSubselect(Parse *, Expr *, int, int); void sqlite3SelectPrep(Parse*, Select*, NameContext*); +int sqlite3MatchSpanName(const char*, const char*, const char*, const char*); int sqlite3ResolveExprNames(NameContext*, Expr*); void sqlite3ResolveSelectNames(Parse*, Select*, NameContext*); int sqlite3ResolveOrderGroupBy(Parse*, Select*, ExprList*, const char*); void sqlite3ColumnDefault(Vdbe *, Table *, int, int); void sqlite3AlterFinishAddColumn(Parse *, Token *); void sqlite3AlterBeginAddColumn(Parse *, SrcList *); -CollSeq *sqlite3GetCollSeq(sqlite3*, u8, CollSeq *, const char*); +CollSeq *sqlite3GetCollSeq(Parse*, u8, CollSeq *, const char*); char sqlite3AffinityType(const char*); void sqlite3Analyze(Parse*, Token*, Token*); int sqlite3InvokeBusyHandler(BusyHandler*); @@ -3164,8 +3263,10 @@ void sqlite3ExprListCheckLength(Parse*, ExprList*, const char*); CollSeq *sqlite3BinaryCompareCollSeq(Parse *, Expr *, Expr *); int sqlite3TempInMemory(const sqlite3*); const char *sqlite3JournalModename(int); -int sqlite3Checkpoint(sqlite3*, int, int, int*, int*); -int sqlite3WalDefaultHook(void*,sqlite3*,const char*,int); +#ifndef SQLITE_OMIT_WAL + int sqlite3Checkpoint(sqlite3*, int, int, int*, int*); + int sqlite3WalDefaultHook(void*,sqlite3*,const char*,int); +#endif /* Declarations for functions in fkey.c. All of these are replaced by ** no-op macros if OMIT_FOREIGN_KEY is defined. In this case no foreign @@ -3190,8 +3291,10 @@ int sqlite3WalDefaultHook(void*,sqlite3*,const char*,int); #endif #ifndef SQLITE_OMIT_FOREIGN_KEY void sqlite3FkDelete(sqlite3 *, Table*); + int sqlite3FkLocateIndex(Parse*,Table*,FKey*,Index**,int**); #else #define sqlite3FkDelete(a,b) + #define sqlite3FkLocateIndex(a,b,c,d,e) #endif @@ -3216,15 +3319,18 @@ int sqlite3WalDefaultHook(void*,sqlite3*,const char*,int); #define IN_INDEX_ROWID 1 #define IN_INDEX_EPH 2 -#define IN_INDEX_INDEX 3 +#define IN_INDEX_INDEX_ASC 3 +#define IN_INDEX_INDEX_DESC 4 int sqlite3FindInIndex(Parse *, Expr *, int*); #ifdef SQLITE_ENABLE_ATOMIC_WRITE int sqlite3JournalOpen(sqlite3_vfs *, const char *, sqlite3_file *, int, int); int sqlite3JournalSize(sqlite3_vfs *); int sqlite3JournalCreate(sqlite3_file *); + int sqlite3JournalExists(sqlite3_file *p); #else #define sqlite3JournalSize(pVfs) ((pVfs)->szOsFile) + #define sqlite3JournalExists(p) 1 #endif void sqlite3MemJournalOpen(sqlite3_file *); diff --git a/src/status.c b/src/status.c index 04b7656..28349e6 100644 --- a/src/status.c +++ b/src/status.c @@ -208,7 +208,8 @@ int sqlite3_db_status( db->pnBytesFreed = &nByte; for(pVdbe=db->pVdbe; pVdbe; pVdbe=pVdbe->pNext){ - sqlite3VdbeDeleteObject(db, pVdbe); + sqlite3VdbeClearObject(db, pVdbe); + sqlite3DbFree(db, pVdbe); } db->pnBytesFreed = 0; diff --git a/src/tclsqlite.c b/src/tclsqlite.c index a2072f8..f1bb292 100644 --- a/src/tclsqlite.c +++ b/src/tclsqlite.c @@ -1005,7 +1005,7 @@ static int DbTransPostCmd( /* This is a tricky scenario to handle. The most likely cause of an ** error is that the exec() above was an attempt to commit the ** top-level transaction that returned SQLITE_BUSY. Or, less likely, - ** that an IO-error has occured. In either case, throw a Tcl exception + ** that an IO-error has occurred. In either case, throw a Tcl exception ** and try to rollback the transaction. ** ** But it could also be that the user executed one or more BEGIN, @@ -1489,7 +1489,7 @@ static Tcl_Obj *dbEvalColumnValue(DbEvalContext *p, int iCol){ } } - return Tcl_NewStringObj(sqlite3_column_text(pStmt, iCol), -1); + return Tcl_NewStringObj((char*)sqlite3_column_text(pStmt, iCol), -1); } /* @@ -2343,7 +2343,7 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){ const char *zDb = "main"; const char *zTable; const char *zColumn; - sqlite_int64 iRow; + Tcl_WideInt iRow; /* Check for the -readonly option */ if( objc>3 && strcmp(Tcl_GetString(objv[2]), "-readonly")==0 ){ @@ -2534,7 +2534,7 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){ pKey = Tcl_GetByteArrayFromObj(objv[2], &nKey); rc = sqlite3_rekey(pDb->db, pKey, nKey); if( rc ){ - Tcl_AppendResult(interp, sqlite3ErrStr(rc), 0); + Tcl_AppendResult(interp, sqlite3_errstr(rc), 0); rc = TCL_ERROR; } #endif @@ -2905,6 +2905,7 @@ static int DbMain(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){ void *pKey = 0; int nKey = 0; #endif + int rc; /* In normal use, each TCL interpreter runs in a single thread. So ** by default, we can turn of mutexing on SQLite database connections. @@ -3009,12 +3010,16 @@ static int DbMain(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){ memset(p, 0, sizeof(*p)); zFile = Tcl_GetStringFromObj(objv[2], 0); zFile = Tcl_TranslateFileName(interp, zFile, &translatedFilename); - sqlite3_open_v2(zFile, &p->db, flags, zVfs); + rc = sqlite3_open_v2(zFile, &p->db, flags, zVfs); Tcl_DStringFree(&translatedFilename); - if( SQLITE_OK!=sqlite3_errcode(p->db) ){ - zErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(p->db)); - sqlite3_close(p->db); - p->db = 0; + if( p->db ){ + if( SQLITE_OK!=sqlite3_errcode(p->db) ){ + zErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(p->db)); + sqlite3_close(p->db); + p->db = 0; + } + }else{ + zErrMsg = sqlite3_mprintf("%s", sqlite3_errstr(rc)); } #ifdef SQLITE_HAS_CODEC if( p->db ){ @@ -3666,6 +3671,7 @@ static void init_all(Tcl_Interp *interp){ extern int Sqlitetestschema_Init(Tcl_Interp*); extern int Sqlitetestsse_Init(Tcl_Interp*); extern int Sqlitetesttclvar_Init(Tcl_Interp*); + extern int Sqlitetestfs_Init(Tcl_Interp*); extern int SqlitetestThread_Init(Tcl_Interp*); extern int SqlitetestOnefile_Init(); extern int SqlitetestOsinst_Init(Tcl_Interp*); @@ -3677,8 +3683,6 @@ static void init_all(Tcl_Interp *interp){ extern int Sqlitemultiplex_Init(Tcl_Interp*); extern int SqliteSuperlock_Init(Tcl_Interp*); extern int SqlitetestSyscall_Init(Tcl_Interp*); - extern int Sqlitetestfuzzer_Init(Tcl_Interp*); - extern int Sqlitetestwholenumber_Init(Tcl_Interp*); #if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) extern int Sqlitetestfts3_Init(Tcl_Interp *interp); @@ -3709,6 +3713,7 @@ static void init_all(Tcl_Interp *interp){ Sqlitetest_mutex_Init(interp); Sqlitetestschema_Init(interp); Sqlitetesttclvar_Init(interp); + Sqlitetestfs_Init(interp); SqlitetestThread_Init(interp); SqlitetestOnefile_Init(interp); SqlitetestOsinst_Init(interp); @@ -3720,8 +3725,6 @@ static void init_all(Tcl_Interp *interp){ Sqlitemultiplex_Init(interp); SqliteSuperlock_Init(interp); SqlitetestSyscall_Init(interp); - Sqlitetestfuzzer_Init(interp); - Sqlitetestwholenumber_Init(interp); #if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) Sqlitetestfts3_Init(interp); diff --git a/src/test1.c b/src/test1.c index c3dac06..a638e48 100644 --- a/src/test1.c +++ b/src/test1.c @@ -113,64 +113,8 @@ int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite3 **ppDb){ return TCL_OK; } - -const char *sqlite3TestErrorName(int rc){ - const char *zName = 0; - switch( rc ){ - case SQLITE_OK: zName = "SQLITE_OK"; break; - case SQLITE_ERROR: zName = "SQLITE_ERROR"; break; - case SQLITE_INTERNAL: zName = "SQLITE_INTERNAL"; break; - case SQLITE_PERM: zName = "SQLITE_PERM"; break; - case SQLITE_ABORT: zName = "SQLITE_ABORT"; break; - case SQLITE_BUSY: zName = "SQLITE_BUSY"; break; - case SQLITE_LOCKED: zName = "SQLITE_LOCKED"; break; - case SQLITE_LOCKED_SHAREDCACHE: zName = "SQLITE_LOCKED_SHAREDCACHE";break; - case SQLITE_NOMEM: zName = "SQLITE_NOMEM"; break; - case SQLITE_READONLY: zName = "SQLITE_READONLY"; break; - case SQLITE_INTERRUPT: zName = "SQLITE_INTERRUPT"; break; - case SQLITE_IOERR: zName = "SQLITE_IOERR"; break; - case SQLITE_CORRUPT: zName = "SQLITE_CORRUPT"; break; - case SQLITE_NOTFOUND: zName = "SQLITE_NOTFOUND"; break; - case SQLITE_FULL: zName = "SQLITE_FULL"; break; - case SQLITE_CANTOPEN: zName = "SQLITE_CANTOPEN"; break; - case SQLITE_PROTOCOL: zName = "SQLITE_PROTOCOL"; break; - case SQLITE_EMPTY: zName = "SQLITE_EMPTY"; break; - case SQLITE_SCHEMA: zName = "SQLITE_SCHEMA"; break; - case SQLITE_TOOBIG: zName = "SQLITE_TOOBIG"; break; - case SQLITE_CONSTRAINT: zName = "SQLITE_CONSTRAINT"; break; - case SQLITE_MISMATCH: zName = "SQLITE_MISMATCH"; break; - case SQLITE_MISUSE: zName = "SQLITE_MISUSE"; break; - case SQLITE_NOLFS: zName = "SQLITE_NOLFS"; break; - case SQLITE_AUTH: zName = "SQLITE_AUTH"; break; - case SQLITE_FORMAT: zName = "SQLITE_FORMAT"; break; - case SQLITE_RANGE: zName = "SQLITE_RANGE"; break; - case SQLITE_NOTADB: zName = "SQLITE_NOTADB"; break; - case SQLITE_ROW: zName = "SQLITE_ROW"; break; - case SQLITE_DONE: zName = "SQLITE_DONE"; break; - case SQLITE_IOERR_READ: zName = "SQLITE_IOERR_READ"; break; - case SQLITE_IOERR_SHORT_READ: zName = "SQLITE_IOERR_SHORT_READ"; break; - case SQLITE_IOERR_WRITE: zName = "SQLITE_IOERR_WRITE"; break; - case SQLITE_IOERR_FSYNC: zName = "SQLITE_IOERR_FSYNC"; break; - case SQLITE_IOERR_DIR_FSYNC: zName = "SQLITE_IOERR_DIR_FSYNC"; break; - case SQLITE_IOERR_TRUNCATE: zName = "SQLITE_IOERR_TRUNCATE"; break; - case SQLITE_IOERR_FSTAT: zName = "SQLITE_IOERR_FSTAT"; break; - case SQLITE_IOERR_UNLOCK: zName = "SQLITE_IOERR_UNLOCK"; break; - case SQLITE_IOERR_RDLOCK: zName = "SQLITE_IOERR_RDLOCK"; break; - case SQLITE_IOERR_DELETE: zName = "SQLITE_IOERR_DELETE"; break; - case SQLITE_IOERR_BLOCKED: zName = "SQLITE_IOERR_BLOCKED"; break; - case SQLITE_IOERR_NOMEM: zName = "SQLITE_IOERR_NOMEM"; break; - case SQLITE_IOERR_ACCESS: zName = "SQLITE_IOERR_ACCESS"; break; - case SQLITE_IOERR_CHECKRESERVEDLOCK: - zName = "SQLITE_IOERR_CHECKRESERVEDLOCK"; break; - case SQLITE_IOERR_LOCK: zName = "SQLITE_IOERR_LOCK"; break; - case SQLITE_CORRUPT_VTAB: zName = "SQLITE_CORRUPT_VTAB"; break; - case SQLITE_READONLY_RECOVERY: zName = "SQLITE_READONLY_RECOVERY"; break; - case SQLITE_READONLY_CANTLOCK: zName = "SQLITE_READONLY_CANTLOCK"; break; - default: zName = "SQLITE_Unknown"; break; - } - return zName; -} -#define t1ErrorName sqlite3TestErrorName +extern const char *sqlite3ErrName(int); +#define t1ErrorName sqlite3ErrName /* ** Convert an sqlite3_stmt* into an sqlite3*. This depends on the @@ -738,6 +682,30 @@ static int sqlite_test_close( } /* +** Usage: sqlite3_close_v2 DB +** +** Closes the database opened by sqlite3_open. +*/ +static int sqlite_test_close_v2( + void *NotUsed, + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int argc, /* Number of arguments */ + char **argv /* Text of each argument */ +){ + sqlite3 *db; + int rc; + if( argc!=2 ){ + Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], + " FILENAME\"", 0); + return TCL_ERROR; + } + if( getDbPointer(interp, argv[1], &db) ) return TCL_ERROR; + rc = sqlite3_close_v2(db); + Tcl_SetResult(interp, (char *)t1ErrorName(rc), TCL_STATIC); + return TCL_OK; +} + +/* ** Implementation of the x_coalesce() function. ** Return the first argument non-NULL argument. */ @@ -1715,7 +1683,7 @@ static int test_blob_read( if( rc==SQLITE_OK ){ Tcl_SetObjResult(interp, Tcl_NewByteArrayObj(zBuf, nByte)); }else{ - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_VOLATILE); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_VOLATILE); } Tcl_Free((char *)zBuf); @@ -1765,7 +1733,7 @@ static int test_blob_write( } rc = sqlite3_blob_write(pBlob, zBuf, nBuf, iOffset); if( rc!=SQLITE_OK ){ - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_VOLATILE); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_VOLATILE); } return (rc==SQLITE_OK ? TCL_OK : TCL_ERROR); @@ -1791,7 +1759,7 @@ static int test_blob_reopen( rc = sqlite3_blob_reopen(pBlob, iRowid); if( rc!=SQLITE_OK ){ - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_VOLATILE); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_VOLATILE); } return (rc==SQLITE_OK ? TCL_OK : TCL_ERROR); @@ -2001,7 +1969,7 @@ static int test_create_function_v2( ); if( rc!=SQLITE_OK ){ Tcl_ResetResult(interp); - Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } return TCL_OK; @@ -2677,7 +2645,7 @@ static int test_collate( if( sqlite3TestErrCode(interp, db, rc) ) return TCL_ERROR; if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } return TCL_OK; @@ -3067,7 +3035,7 @@ static int test_bind_int64( ){ sqlite3_stmt *pStmt; int idx; - i64 value; + Tcl_WideInt value; int rc; if( objc!=4 ){ @@ -3235,7 +3203,7 @@ static int test_bind_text( rc = sqlite3_bind_text(pStmt, idx, value, bytes, SQLITE_TRANSIENT); if( sqlite3TestErrCode(interp, StmtToDb(pStmt), rc) ) return TCL_ERROR; if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } @@ -3283,7 +3251,7 @@ static int test_bind_text16( rc = sqlite3_bind_text16(pStmt, idx, (void *)value, bytes, xDel); if( sqlite3TestErrCode(interp, StmtToDb(pStmt), rc) ) return TCL_ERROR; if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } @@ -4557,7 +4525,7 @@ static int test_busy_timeout( if( getDbPointer(interp, argv[1], &db) ) return TCL_ERROR; if( Tcl_GetInt(interp, argv[2], &ms) ) return TCL_ERROR; rc = sqlite3_busy_timeout(db, ms); - Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_OK; } @@ -4703,7 +4671,7 @@ static int test_soft_heap_limit( Tcl_Obj *CONST objv[] ){ sqlite3_int64 amt; - sqlite3_int64 N = -1; + Tcl_WideInt N = -1; if( objc!=1 && objc!=2 ){ Tcl_WrongNumArgs(interp, 1, objv, "?N?"); return TCL_ERROR; @@ -5078,7 +5046,7 @@ static int file_control_chunksize_test( rc = sqlite3_file_control(db, zDb, SQLITE_FCNTL_CHUNK_SIZE, (void *)&nSize); if( rc ){ - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_STATIC); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_STATIC); return TCL_ERROR; } return TCL_OK; @@ -5096,7 +5064,7 @@ static int file_control_sizehint_test( int objc, /* Number of arguments */ Tcl_Obj *CONST objv[] /* Command arguments */ ){ - sqlite3_int64 nSize; /* Hinted size */ + Tcl_WideInt nSize; /* Hinted size */ char *zDb; /* Db name ("main", "temp" etc.) */ sqlite3 *db; /* Database handle */ int rc; /* file_control() return code */ @@ -5115,7 +5083,7 @@ static int file_control_sizehint_test( rc = sqlite3_file_control(db, zDb, SQLITE_FCNTL_SIZE_HINT, (void *)&nSize); if( rc ){ - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_STATIC); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_STATIC); return TCL_ERROR; } return TCL_OK; @@ -5321,6 +5289,38 @@ static int file_control_vfsname( return TCL_OK; } +/* +** tclcmd: file_control_tempfilename DB ?AUXDB? +** +** Return a string that is a temporary filename +*/ +static int file_control_tempfilename( + ClientData clientData, /* Pointer to sqlite3_enable_XXX function */ + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int objc, /* Number of arguments */ + Tcl_Obj *CONST objv[] /* Command arguments */ +){ + sqlite3 *db; + const char *zDbName = "main"; + char *zTName = 0; + + if( objc!=2 && objc!=3 ){ + Tcl_AppendResult(interp, "wrong # args: should be \"", + Tcl_GetStringFromObj(objv[0], 0), " DB ?AUXDB?", 0); + return TCL_ERROR; + } + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ){ + return TCL_ERROR; + } + if( objc==3 ){ + zDbName = Tcl_GetString(objv[2]); + } + sqlite3_file_control(db, zDbName, SQLITE_FCNTL_TEMPFILENAME, (void*)&zTName); + Tcl_AppendResult(interp, zTName, (char*)0); + sqlite3_free(zTName); + return TCL_OK; +} + /* ** tclcmd: sqlite3_vfs_list @@ -5627,7 +5627,7 @@ static void xLogcallback(void *unused, int err, char *zMsg){ Tcl_Obj *pNew = Tcl_DuplicateObj(logcallback.pObj); Tcl_IncrRefCount(pNew); Tcl_ListObjAppendElement( - 0, pNew, Tcl_NewStringObj(sqlite3TestErrorName(err), -1) + 0, pNew, Tcl_NewStringObj(sqlite3ErrName(err), -1) ); Tcl_ListObjAppendElement(0, pNew, Tcl_NewStringObj(zMsg, -1)); Tcl_EvalObjEx(logcallback.pInterp, pNew, TCL_EVAL_GLOBAL|TCL_EVAL_DIRECT); @@ -5799,6 +5799,31 @@ static int test_test_control( return TCL_OK; } +#if SQLITE_OS_UNIX +#include <sys/time.h> +#include <sys/resource.h> + +static int test_getrusage( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + char buf[1024]; + struct rusage r; + memset(&r, 0, sizeof(r)); + getrusage(RUSAGE_SELF, &r); + + sprintf(buf, "ru_utime=%d.%06d ru_stime=%d.%06d ru_minflt=%d ru_majflt=%d", + (int)r.ru_utime.tv_sec, (int)r.ru_utime.tv_usec, + (int)r.ru_stime.tv_sec, (int)r.ru_stime.tv_usec, + (int)r.ru_minflt, (int)r.ru_majflt + ); + Tcl_SetObjResult(interp, Tcl_NewStringObj(buf, -1)); + return TCL_OK; +} +#endif + #if SQLITE_OS_WIN /* ** Information passed from the main thread into the windows file locker @@ -5933,15 +5958,15 @@ static int optimization_control( const char *zOptName; int mask; } aOpt[] = { - { "all", SQLITE_OptMask }, + { "all", SQLITE_AllOpts }, { "query-flattener", SQLITE_QueryFlattener }, { "column-cache", SQLITE_ColumnCache }, - { "index-sort", SQLITE_IndexSort }, - { "index-search", SQLITE_IndexSearch }, - { "index-cover", SQLITE_IndexCover }, { "groupby-order", SQLITE_GroupByOrder }, { "factor-constants", SQLITE_FactorOutConst }, { "real-as-int", SQLITE_IdxRealAsInt }, + { "distinct-opt", SQLITE_DistinctOpt }, + { "cover-idx-scan", SQLITE_CoverIdxScan }, + { "order-by-idx-join",SQLITE_OrderByIdxJoin }, }; if( objc!=4 ){ @@ -5970,6 +5995,69 @@ static int optimization_control( return TCL_OK; } +typedef struct sqlite3_api_routines sqlite3_api_routines; +/* +** load_static_extension DB NAME ... +** +** Load one or more statically linked extensions. +*/ +static int tclLoadStaticExtensionCmd( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + extern int sqlite3_amatch_init(sqlite3*,char**,const sqlite3_api_routines*); + extern int sqlite3_closure_init(sqlite3*,char**,const sqlite3_api_routines*); + extern int sqlite3_fuzzer_init(sqlite3*,char**,const sqlite3_api_routines*); + extern int sqlite3_ieee_init(sqlite3*,char**,const sqlite3_api_routines*); + extern int sqlite3_nextchar_init(sqlite3*,char**,const sqlite3_api_routines*); + extern int sqlite3_regexp_init(sqlite3*,char**,const sqlite3_api_routines*); + extern int sqlite3_spellfix_init(sqlite3*,char**,const sqlite3_api_routines*); + extern int sqlite3_wholenumber_init(sqlite3*,char**,const sqlite3_api_routines*); + static const struct { + const char *zExtName; + int (*pInit)(sqlite3*,char**,const sqlite3_api_routines*); + } aExtension[] = { + { "amatch", sqlite3_amatch_init }, + { "closure", sqlite3_closure_init }, + { "fuzzer", sqlite3_fuzzer_init }, + { "ieee754", sqlite3_ieee_init }, + { "nextchar", sqlite3_nextchar_init }, + { "regexp", sqlite3_regexp_init }, + { "spellfix", sqlite3_spellfix_init }, + { "wholenumber", sqlite3_wholenumber_init }, + }; + sqlite3 *db; + const char *zName; + int i, j, rc; + char *zErrMsg = 0; + if( objc<3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB NAME ..."); + return TCL_ERROR; + } + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; + for(j=2; j<objc; j++){ + zName = Tcl_GetString(objv[j]); + for(i=0; i<ArraySize(aExtension); i++){ + if( strcmp(zName, aExtension[i].zExtName)==0 ) break; + } + if( i>=ArraySize(aExtension) ){ + Tcl_AppendResult(interp, "no such extension: ", zName, (char*)0); + return TCL_ERROR; + } + rc = aExtension[i].pInit(db, &zErrMsg, 0); + if( rc!=SQLITE_OK || zErrMsg ){ + Tcl_AppendResult(interp, "initialization of ", zName, " failed: ", zErrMsg, + (char*)0); + sqlite3_free(zErrMsg); + return TCL_ERROR; + } + } + return TCL_OK; +} + + /* ** Register commands with the TCL interpreter. */ @@ -6013,6 +6101,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ { "sqlite3_get_table_printf", (Tcl_CmdProc*)test_get_table_printf }, #endif { "sqlite3_close", (Tcl_CmdProc*)sqlite_test_close }, + { "sqlite3_close_v2", (Tcl_CmdProc*)sqlite_test_close_v2 }, { "sqlite3_create_function", (Tcl_CmdProc*)test_create_function }, { "sqlite3_create_aggregate", (Tcl_CmdProc*)test_create_aggregate }, { "sqlite_register_test_function", (Tcl_CmdProc*)test_register_func }, @@ -6150,6 +6239,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ { "file_control_persist_wal", file_control_persist_wal, 0 }, { "file_control_powersafe_overwrite",file_control_powersafe_overwrite,0}, { "file_control_vfsname", file_control_vfsname, 0 }, + { "file_control_tempfilename", file_control_tempfilename, 0 }, { "sqlite3_vfs_list", vfs_list, 0 }, { "sqlite3_create_function_v2", test_create_function_v2, 0 }, @@ -6187,6 +6277,10 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ { "print_explain_query_plan", test_print_eqp, 0 }, #endif { "sqlite3_test_control", test_test_control }, +#if SQLITE_OS_UNIX + { "getrusage", test_getrusage }, +#endif + { "load_static_extension", tclLoadStaticExtensionCmd }, }; static int bitmask_size = sizeof(Bitmask)*8; int i; @@ -6203,7 +6297,6 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ #ifdef SQLITE_DEBUG extern int sqlite3WhereTrace; extern int sqlite3OSTrace; - extern int sqlite3VdbeAddopTrace; extern int sqlite3WalTrace; #endif #ifdef SQLITE_TEST @@ -6266,8 +6359,6 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ (char*)&query_plan, TCL_LINK_STRING|TCL_LINK_READ_ONLY); #endif #ifdef SQLITE_DEBUG - Tcl_LinkVar(interp, "sqlite_addop_trace", - (char*)&sqlite3VdbeAddopTrace, TCL_LINK_INT); Tcl_LinkVar(interp, "sqlite_where_trace", (char*)&sqlite3WhereTrace, TCL_LINK_INT); Tcl_LinkVar(interp, "sqlite_os_trace", diff --git a/src/test2.c b/src/test2.c index 8acdf6f..d130e9d 100644 --- a/src/test2.c +++ b/src/test2.c @@ -19,35 +19,7 @@ #include <string.h> #include <ctype.h> -/* -** Interpret an SQLite error number -*/ -static char *errorName(int rc){ - char *zName; - switch( rc ){ - case SQLITE_OK: zName = "SQLITE_OK"; break; - case SQLITE_ERROR: zName = "SQLITE_ERROR"; break; - case SQLITE_PERM: zName = "SQLITE_PERM"; break; - case SQLITE_ABORT: zName = "SQLITE_ABORT"; break; - case SQLITE_BUSY: zName = "SQLITE_BUSY"; break; - case SQLITE_NOMEM: zName = "SQLITE_NOMEM"; break; - case SQLITE_READONLY: zName = "SQLITE_READONLY"; break; - case SQLITE_INTERRUPT: zName = "SQLITE_INTERRUPT"; break; - case SQLITE_IOERR: zName = "SQLITE_IOERR"; break; - case SQLITE_CORRUPT: zName = "SQLITE_CORRUPT"; break; - case SQLITE_FULL: zName = "SQLITE_FULL"; break; - case SQLITE_CANTOPEN: zName = "SQLITE_CANTOPEN"; break; - case SQLITE_PROTOCOL: zName = "SQLITE_PROTOCOL"; break; - case SQLITE_EMPTY: zName = "SQLITE_EMPTY"; break; - case SQLITE_SCHEMA: zName = "SQLITE_SCHEMA"; break; - case SQLITE_CONSTRAINT: zName = "SQLITE_CONSTRAINT"; break; - case SQLITE_MISMATCH: zName = "SQLITE_MISMATCH"; break; - case SQLITE_MISUSE: zName = "SQLITE_MISUSE"; break; - case SQLITE_NOLFS: zName = "SQLITE_NOLFS"; break; - default: zName = "SQLITE_Unknown"; break; - } - return zName; -} +extern const char *sqlite3ErrName(int); /* ** Page size and reserved size used for testing. @@ -87,7 +59,7 @@ static int pager_open( SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_MAIN_DB, pager_test_reiniter); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } sqlite3PagerSetCachesize(pPager, nPage); @@ -119,7 +91,7 @@ static int pager_close( pPager = sqlite3TestTextToPtr(argv[1]); rc = sqlite3PagerClose(pPager); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } return TCL_OK; @@ -146,7 +118,7 @@ static int pager_rollback( pPager = sqlite3TestTextToPtr(argv[1]); rc = sqlite3PagerRollback(pPager); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } return TCL_OK; @@ -173,12 +145,12 @@ static int pager_commit( pPager = sqlite3TestTextToPtr(argv[1]); rc = sqlite3PagerCommitPhaseOne(pPager, 0, 0); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } rc = sqlite3PagerCommitPhaseTwo(pPager); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } return TCL_OK; @@ -205,7 +177,7 @@ static int pager_stmt_begin( pPager = sqlite3TestTextToPtr(argv[1]); rc = sqlite3PagerOpenSavepoint(pPager, 1); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } return TCL_OK; @@ -233,7 +205,7 @@ static int pager_stmt_rollback( rc = sqlite3PagerSavepoint(pPager, SAVEPOINT_ROLLBACK, 0); sqlite3PagerSavepoint(pPager, SAVEPOINT_RELEASE, 0); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } return TCL_OK; @@ -260,7 +232,7 @@ static int pager_stmt_commit( pPager = sqlite3TestTextToPtr(argv[1]); rc = sqlite3PagerSavepoint(pPager, SAVEPOINT_RELEASE, 0); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } return TCL_OK; @@ -353,7 +325,7 @@ static int page_get( rc = sqlite3PagerGet(pPager, pgno, &pPage); } if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } sqlite3_snprintf(sizeof(zBuf),zBuf,"%p",pPage); @@ -507,7 +479,7 @@ static int page_write( pPage = (DbPage *)sqlite3TestTextToPtr(argv[1]); rc = sqlite3PagerWrite(pPage); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } pData = sqlite3PagerGetData(pPage); @@ -556,7 +528,7 @@ static int fake_big_file( (SQLITE_OPEN_CREATE|SQLITE_OPEN_READWRITE|SQLITE_OPEN_MAIN_DB), 0 ); if( rc ){ - Tcl_AppendResult(interp, "open failed: ", errorName(rc), 0); + Tcl_AppendResult(interp, "open failed: ", sqlite3ErrName(rc), 0); sqlite3_free(zFile); return TCL_ERROR; } @@ -566,7 +538,7 @@ static int fake_big_file( sqlite3OsCloseFree(fd); sqlite3_free(zFile); if( rc ){ - Tcl_AppendResult(interp, "write failed: ", errorName(rc), 0); + Tcl_AppendResult(interp, "write failed: ", sqlite3ErrName(rc), 0); return TCL_ERROR; } return TCL_OK; diff --git a/src/test3.c b/src/test3.c index e460c42..e3ed310 100644 --- a/src/test3.c +++ b/src/test3.c @@ -19,31 +19,7 @@ #include <stdlib.h> #include <string.h> -/* -** Interpret an SQLite error number -*/ -static char *errorName(int rc){ - char *zName; - switch( rc ){ - case SQLITE_OK: zName = "SQLITE_OK"; break; - case SQLITE_ERROR: zName = "SQLITE_ERROR"; break; - case SQLITE_PERM: zName = "SQLITE_PERM"; break; - case SQLITE_ABORT: zName = "SQLITE_ABORT"; break; - case SQLITE_BUSY: zName = "SQLITE_BUSY"; break; - case SQLITE_NOMEM: zName = "SQLITE_NOMEM"; break; - case SQLITE_READONLY: zName = "SQLITE_READONLY"; break; - case SQLITE_INTERRUPT: zName = "SQLITE_INTERRUPT"; break; - case SQLITE_IOERR: zName = "SQLITE_IOERR"; break; - case SQLITE_CORRUPT: zName = "SQLITE_CORRUPT"; break; - case SQLITE_FULL: zName = "SQLITE_FULL"; break; - case SQLITE_CANTOPEN: zName = "SQLITE_CANTOPEN"; break; - case SQLITE_PROTOCOL: zName = "SQLITE_PROTOCOL"; break; - case SQLITE_EMPTY: zName = "SQLITE_EMPTY"; break; - case SQLITE_LOCKED: zName = "SQLITE_LOCKED"; break; - default: zName = "SQLITE_Unknown"; break; - } - return zName; -} +extern const char *sqlite3ErrName(int); /* ** A bogus sqlite3 connection structure for use in the btree @@ -89,7 +65,7 @@ static int btree_open( SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_MAIN_DB); sqlite3_free(zFilename); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } sqlite3BtreeSetCacheSize(pBt, nCache); @@ -119,7 +95,7 @@ static int btree_close( pBt = sqlite3TestTextToPtr(argv[1]); rc = sqlite3BtreeClose(pBt); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } nRefSqlite3--; @@ -156,7 +132,7 @@ static int btree_begin_transaction( rc = sqlite3BtreeBeginTrans(pBt, 1); sqlite3BtreeLeave(pBt); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } return TCL_OK; @@ -250,7 +226,7 @@ static int btree_cursor( sqlite3BtreeLeave(pBt); if( rc ){ ckfree((char *)pCur); - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } sqlite3_snprintf(sizeof(zBuf), zBuf,"%p", pCur); @@ -285,7 +261,7 @@ static int btree_close_cursor( sqlite3BtreeLeave(pBt); ckfree((char *)pCur); if( rc ){ - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } return SQLITE_OK; @@ -319,7 +295,7 @@ static int btree_next( rc = sqlite3BtreeNext(pCur, &res); sqlite3BtreeLeave(pCur->pBtree); if( rc ){ - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } sqlite3_snprintf(sizeof(zBuf),zBuf,"%d",res); @@ -354,7 +330,7 @@ static int btree_first( rc = sqlite3BtreeFirst(pCur, &res); sqlite3BtreeLeave(pCur->pBtree); if( rc ){ - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } sqlite3_snprintf(sizeof(zBuf),zBuf,"%d",res); diff --git a/src/test4.c b/src/test4.c index 5b4c5a1..a6375c7 100644 --- a/src/test4.c +++ b/src/test4.c @@ -20,6 +20,8 @@ #include <sched.h> #include <ctype.h> +extern const char *sqlite3ErrName(int); + /* ** Each thread is controlled by an instance of the following ** structure. @@ -372,34 +374,7 @@ static int tcl_thread_result( return TCL_ERROR; } thread_wait(&threadset[i]); - switch( threadset[i].rc ){ - case SQLITE_OK: zName = "SQLITE_OK"; break; - case SQLITE_ERROR: zName = "SQLITE_ERROR"; break; - case SQLITE_PERM: zName = "SQLITE_PERM"; break; - case SQLITE_ABORT: zName = "SQLITE_ABORT"; break; - case SQLITE_BUSY: zName = "SQLITE_BUSY"; break; - case SQLITE_LOCKED: zName = "SQLITE_LOCKED"; break; - case SQLITE_NOMEM: zName = "SQLITE_NOMEM"; break; - case SQLITE_READONLY: zName = "SQLITE_READONLY"; break; - case SQLITE_INTERRUPT: zName = "SQLITE_INTERRUPT"; break; - case SQLITE_IOERR: zName = "SQLITE_IOERR"; break; - case SQLITE_CORRUPT: zName = "SQLITE_CORRUPT"; break; - case SQLITE_FULL: zName = "SQLITE_FULL"; break; - case SQLITE_CANTOPEN: zName = "SQLITE_CANTOPEN"; break; - case SQLITE_PROTOCOL: zName = "SQLITE_PROTOCOL"; break; - case SQLITE_EMPTY: zName = "SQLITE_EMPTY"; break; - case SQLITE_SCHEMA: zName = "SQLITE_SCHEMA"; break; - case SQLITE_CONSTRAINT: zName = "SQLITE_CONSTRAINT"; break; - case SQLITE_MISMATCH: zName = "SQLITE_MISMATCH"; break; - case SQLITE_MISUSE: zName = "SQLITE_MISUSE"; break; - case SQLITE_NOLFS: zName = "SQLITE_NOLFS"; break; - case SQLITE_AUTH: zName = "SQLITE_AUTH"; break; - case SQLITE_FORMAT: zName = "SQLITE_FORMAT"; break; - case SQLITE_RANGE: zName = "SQLITE_RANGE"; break; - case SQLITE_ROW: zName = "SQLITE_ROW"; break; - case SQLITE_DONE: zName = "SQLITE_DONE"; break; - default: zName = "SQLITE_Unknown"; break; - } + zName = sqlite3ErrName(threadset[i].rc); Tcl_AppendResult(interp, zName, 0); return TCL_OK; } diff --git a/src/test6.c b/src/test6.c index bae6b65..c151ea4 100644 --- a/src/test6.c +++ b/src/test6.c @@ -87,7 +87,7 @@ typedef struct WriteBuffer WriteBuffer; ** an aligned write() of an integer number of 512 byte regions, then ** option (3) above is never selected. Instead, each 512 byte region ** is either correctly written or left completely untouched. Similar -** logic governs the behaviour if any of the other ATOMICXXX flags +** logic governs the behavior if any of the other ATOMICXXX flags ** is set. ** ** If either the IOCAP_SAFEAPPEND or IOCAP_SEQUENTIAL flags are set @@ -312,8 +312,8 @@ static int writeListSync(CrashFile *pFile, int isCrash){ assert(pWrite->zBuf); #ifdef TRACE_CRASHTEST - printf("Trashing %d sectors @ sector %d (%s)\n", - 1+iLast-iFirst, iFirst, pWrite->pFile->zName + printf("Trashing %d sectors @ %lld (sector %d) (%s)\n", + 1+iLast-iFirst, pWrite->iOffset, iFirst, pWrite->pFile->zName ); #endif @@ -628,18 +628,19 @@ static int cfOpen( ** to read data from the 512-byte locking region of a file opened ** with the SQLITE_OPEN_MAIN_DB flag. This region of a database file ** never contains valid data anyhow. So avoid doing such a read here. + ** + ** UPDATE: It also contains an assert() verifying that each call + ** to the xRead() method reads less than 128KB of data. */ const int isDb = (flags&SQLITE_OPEN_MAIN_DB); - i64 iChunk = pWrapper->iSize; - if( iChunk>PENDING_BYTE && isDb ){ - iChunk = PENDING_BYTE; - } + i64 iOff; + memset(pWrapper->zData, 0, pWrapper->nData); - rc = sqlite3OsRead(pReal, pWrapper->zData, (int)iChunk, 0); - if( SQLITE_OK==rc && pWrapper->iSize>(PENDING_BYTE+512) && isDb ){ - i64 iOff = PENDING_BYTE+512; - iChunk = pWrapper->iSize - iOff; - rc = sqlite3OsRead(pReal, &pWrapper->zData[iOff], (int)iChunk, iOff); + for(iOff=0; iOff<pWrapper->iSize; iOff += 512){ + int nRead = pWrapper->iSize - (int)iOff; + if( nRead>512 ) nRead = 512; + if( isDb && iOff==PENDING_BYTE ) continue; + rc = sqlite3OsRead(pReal, &pWrapper->zData[iOff], nRead, iOff); } }else{ rc = SQLITE_NOMEM; diff --git a/src/test7.c b/src/test7.c index 852cd1d..3cd4a22 100644 --- a/src/test7.c +++ b/src/test7.c @@ -376,6 +376,8 @@ static int tcl_client_colname( return TCL_OK; } +extern const char *sqlite3ErrName(int); + /* ** Usage: client_result ID ** @@ -403,34 +405,7 @@ static int tcl_client_result( return TCL_ERROR; } client_wait(&threadset[i]); - switch( threadset[i].rc ){ - case SQLITE_OK: zName = "SQLITE_OK"; break; - case SQLITE_ERROR: zName = "SQLITE_ERROR"; break; - case SQLITE_PERM: zName = "SQLITE_PERM"; break; - case SQLITE_ABORT: zName = "SQLITE_ABORT"; break; - case SQLITE_BUSY: zName = "SQLITE_BUSY"; break; - case SQLITE_LOCKED: zName = "SQLITE_LOCKED"; break; - case SQLITE_NOMEM: zName = "SQLITE_NOMEM"; break; - case SQLITE_READONLY: zName = "SQLITE_READONLY"; break; - case SQLITE_INTERRUPT: zName = "SQLITE_INTERRUPT"; break; - case SQLITE_IOERR: zName = "SQLITE_IOERR"; break; - case SQLITE_CORRUPT: zName = "SQLITE_CORRUPT"; break; - case SQLITE_FULL: zName = "SQLITE_FULL"; break; - case SQLITE_CANTOPEN: zName = "SQLITE_CANTOPEN"; break; - case SQLITE_PROTOCOL: zName = "SQLITE_PROTOCOL"; break; - case SQLITE_EMPTY: zName = "SQLITE_EMPTY"; break; - case SQLITE_SCHEMA: zName = "SQLITE_SCHEMA"; break; - case SQLITE_CONSTRAINT: zName = "SQLITE_CONSTRAINT"; break; - case SQLITE_MISMATCH: zName = "SQLITE_MISMATCH"; break; - case SQLITE_MISUSE: zName = "SQLITE_MISUSE"; break; - case SQLITE_NOLFS: zName = "SQLITE_NOLFS"; break; - case SQLITE_AUTH: zName = "SQLITE_AUTH"; break; - case SQLITE_FORMAT: zName = "SQLITE_FORMAT"; break; - case SQLITE_RANGE: zName = "SQLITE_RANGE"; break; - case SQLITE_ROW: zName = "SQLITE_ROW"; break; - case SQLITE_DONE: zName = "SQLITE_DONE"; break; - default: zName = "SQLITE_Unknown"; break; - } + zName = sqlite3ErrName(threadset[i].rc); Tcl_AppendResult(interp, zName, 0); return TCL_OK; } diff --git a/src/test8.c b/src/test8.c index 53cb149..c573933 100644 --- a/src/test8.c +++ b/src/test8.c @@ -1300,7 +1300,7 @@ static sqlite3_module echoModuleV2 = { ** Decode a pointer to an sqlite3 object. */ extern int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite3 **ppDb); -extern const char *sqlite3TestErrorName(int rc); +extern const char *sqlite3ErrName(int); static void moduleDestroy(void *p){ sqlite3_free(p); @@ -1340,7 +1340,7 @@ static int register_echo_module( ); } - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_STATIC); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_STATIC); return TCL_OK; } @@ -1370,29 +1370,6 @@ static int declare_vtab( return TCL_OK; } -#include "test_spellfix.c" - -/* -** Register the spellfix virtual table module. -*/ -static int register_spellfix_module( - ClientData clientData, - Tcl_Interp *interp, - int objc, - Tcl_Obj *CONST objv[] -){ - sqlite3 *db; - - if( objc!=2 ){ - Tcl_WrongNumArgs(interp, 1, objv, "DB"); - return TCL_ERROR; - } - if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; - - sqlite3Spellfix1Register(db); - return TCL_OK; -} - #endif /* ifndef SQLITE_OMIT_VIRTUALTABLE */ /* @@ -1406,7 +1383,6 @@ int Sqlitetest8_Init(Tcl_Interp *interp){ void *clientData; } aObjCmd[] = { { "register_echo_module", register_echo_module, 0 }, - { "register_spellfix_module", register_spellfix_module, 0 }, { "sqlite3_declare_vtab", declare_vtab, 0 }, }; int i; diff --git a/src/test_async.c b/src/test_async.c index c760eea..b0b9431 100644 --- a/src/test_async.c +++ b/src/test_async.c @@ -23,8 +23,8 @@ #include "sqlite3.h" #include <assert.h> -/* From test1.c */ -const char *sqlite3TestErrorName(int); +/* From main.c */ +extern const char *sqlite3ErrName(int); struct TestAsyncGlobal { @@ -60,7 +60,7 @@ static int testAsyncInit( rc = sqlite3async_initialize(zParent, isDefault); if( rc!=SQLITE_OK ){ - Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3TestErrorName(rc), -1)); + Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1)); return TCL_ERROR; } return TCL_OK; @@ -208,7 +208,7 @@ static int testAsyncControl( } if( rc!=SQLITE_OK ){ - Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3TestErrorName(rc), -1)); + Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1)); return TCL_ERROR; } diff --git a/src/test_autoext.c b/src/test_autoext.c index 6b1e297..b5013f3 100644 --- a/src/test_autoext.c +++ b/src/test_autoext.c @@ -15,7 +15,7 @@ #include "sqlite3ext.h" #ifndef SQLITE_OMIT_LOAD_EXTENSION -static SQLITE_EXTENSION_INIT1 +SQLITE_EXTENSION_INIT1 /* ** The sqr() SQL function returns the square of its input value. diff --git a/src/test_backup.c b/src/test_backup.c index 2727137..e967424 100644 --- a/src/test_backup.c +++ b/src/test_backup.c @@ -17,9 +17,11 @@ #include <sqlite3.h> #include <assert.h> +/* These functions are implemented in main.c. */ +extern const char *sqlite3ErrName(int); + /* These functions are implemented in test1.c. */ -int getDbPointer(Tcl_Interp *, const char *, sqlite3 **); -const char *sqlite3TestErrorName(int); +extern int getDbPointer(Tcl_Interp *, const char *, sqlite3 **); static int backupTestCmd( ClientData clientData, @@ -70,7 +72,7 @@ static int backupTestCmd( Tcl_DeleteCommand(interp, zCmdName); rc = sqlite3_backup_finish(p); - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_STATIC); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_STATIC); break; } @@ -80,7 +82,7 @@ static int backupTestCmd( return TCL_ERROR; } rc = sqlite3_backup_step(p, nPage); - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_STATIC); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_STATIC); break; } diff --git a/src/test_config.c b/src/test_config.c index f79b455..534727a 100644 --- a/src/test_config.c +++ b/src/test_config.c @@ -57,7 +57,7 @@ static void set_options(Tcl_Interp *interp){ Tcl_SetVar2(interp, "sqlite_options","casesensitivelike","0",TCL_GLOBAL_ONLY); #endif -#ifdef SQLITE_CURDIR +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT Tcl_SetVar2(interp, "sqlite_options", "curdir", "1", TCL_GLOBAL_ONLY); #else Tcl_SetVar2(interp, "sqlite_options", "curdir", "0", TCL_GLOBAL_ONLY); @@ -87,6 +87,12 @@ static void set_options(Tcl_Interp *interp){ Tcl_SetVar2(interp, "sqlite_options", "lfs", "1", TCL_GLOBAL_ONLY); #endif +#if SQLITE_MAX_MMAP_SIZE>0 + Tcl_SetVar2(interp, "sqlite_options", "mmap", "1", TCL_GLOBAL_ONLY); +#else + Tcl_SetVar2(interp, "sqlite_options", "mmap", "0", TCL_GLOBAL_ONLY); +#endif + #if 1 /* def SQLITE_MEMDEBUG */ Tcl_SetVar2(interp, "sqlite_options", "memdebug", "1", TCL_GLOBAL_ONLY); #else @@ -395,11 +401,7 @@ Tcl_SetVar2(interp, "sqlite_options", "long_double", Tcl_SetVar2(interp, "sqlite_options", "memorymanage", "0", TCL_GLOBAL_ONLY); #endif -#ifdef SQLITE_OMIT_MERGE_SORT - Tcl_SetVar2(interp, "sqlite_options", "mergesort", "0", TCL_GLOBAL_ONLY); -#else - Tcl_SetVar2(interp, "sqlite_options", "mergesort", "1", TCL_GLOBAL_ONLY); -#endif +Tcl_SetVar2(interp, "sqlite_options", "mergesort", "1", TCL_GLOBAL_ONLY); #ifdef SQLITE_OMIT_OR_OPTIMIZATION Tcl_SetVar2(interp, "sqlite_options", "or_opt", "0", TCL_GLOBAL_ONLY); diff --git a/src/test_fs.c b/src/test_fs.c new file mode 100644 index 0000000..478cad8 --- /dev/null +++ b/src/test_fs.c @@ -0,0 +1,333 @@ +/* +** 2013 Jan 11 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** Code for testing the virtual table interfaces. This code +** is not included in the SQLite library. It is used for automated +** testing of the SQLite library. +** +** The FS virtual table is created as follows: +** +** CREATE VIRTUAL TABLE tbl USING fs(idx); +** +** where idx is the name of a table in the db with 2 columns. The virtual +** table also has two columns - file path and file contents. +** +** The first column of table idx must be an IPK, and the second contains file +** paths. For example: +** +** CREATE TABLE idx(id INTEGER PRIMARY KEY, path TEXT); +** INSERT INTO idx VALUES(4, '/etc/passwd'); +** +** Adding the row to the idx table automatically creates a row in the +** virtual table with rowid=4, path=/etc/passwd and a text field that +** contains data read from file /etc/passwd on disk. +*/ +#include "sqliteInt.h" +#include "tcl.h" + +#include <stdlib.h> +#include <string.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> + +#if SQLITE_OS_UNIX +# include <unistd.h> +#endif +#if SQLITE_OS_WIN +# include <io.h> +#endif + +#ifndef SQLITE_OMIT_VIRTUALTABLE + +typedef struct fs_vtab fs_vtab; +typedef struct fs_cursor fs_cursor; + +/* +** A fs virtual-table object +*/ +struct fs_vtab { + sqlite3_vtab base; + sqlite3 *db; + char *zDb; /* Name of db containing zTbl */ + char *zTbl; /* Name of docid->file map table */ +}; + +/* A fs cursor object */ +struct fs_cursor { + sqlite3_vtab_cursor base; + sqlite3_stmt *pStmt; + char *zBuf; + int nBuf; + int nAlloc; +}; + +/* +** This function is the implementation of both the xConnect and xCreate +** methods of the fs virtual table. +** +** The argv[] array contains the following: +** +** argv[0] -> module name ("fs") +** argv[1] -> database name +** argv[2] -> table name +** argv[...] -> other module argument fields. +*/ +static int fsConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + fs_vtab *pVtab; + int nByte; + const char *zTbl; + const char *zDb = argv[1]; + + if( argc!=4 ){ + *pzErr = sqlite3_mprintf("wrong number of arguments"); + return SQLITE_ERROR; + } + zTbl = argv[3]; + + nByte = sizeof(fs_vtab) + (int)strlen(zTbl) + 1 + (int)strlen(zDb) + 1; + pVtab = (fs_vtab *)sqlite3MallocZero( nByte ); + if( !pVtab ) return SQLITE_NOMEM; + + pVtab->zTbl = (char *)&pVtab[1]; + pVtab->zDb = &pVtab->zTbl[strlen(zTbl)+1]; + pVtab->db = db; + memcpy(pVtab->zTbl, zTbl, strlen(zTbl)); + memcpy(pVtab->zDb, zDb, strlen(zDb)); + *ppVtab = &pVtab->base; + sqlite3_declare_vtab(db, "CREATE TABLE xyz(path TEXT, data TEXT)"); + + return SQLITE_OK; +} +/* Note that for this virtual table, the xCreate and xConnect +** methods are identical. */ + +static int fsDisconnect(sqlite3_vtab *pVtab){ + sqlite3_free(pVtab); + return SQLITE_OK; +} +/* The xDisconnect and xDestroy methods are also the same */ + +/* +** Open a new fs cursor. +*/ +static int fsOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ + fs_cursor *pCur; + pCur = sqlite3MallocZero(sizeof(fs_cursor)); + *ppCursor = &pCur->base; + return SQLITE_OK; +} + +/* +** Close a fs cursor. +*/ +static int fsClose(sqlite3_vtab_cursor *cur){ + fs_cursor *pCur = (fs_cursor *)cur; + sqlite3_finalize(pCur->pStmt); + sqlite3_free(pCur->zBuf); + sqlite3_free(pCur); + return SQLITE_OK; +} + +static int fsNext(sqlite3_vtab_cursor *cur){ + fs_cursor *pCur = (fs_cursor *)cur; + int rc; + + rc = sqlite3_step(pCur->pStmt); + if( rc==SQLITE_ROW || rc==SQLITE_DONE ) rc = SQLITE_OK; + + return rc; +} + +static int fsFilter( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + int rc; + fs_cursor *pCur = (fs_cursor *)pVtabCursor; + fs_vtab *p = (fs_vtab *)(pVtabCursor->pVtab); + + assert( (idxNum==0 && argc==0) || (idxNum==1 && argc==1) ); + if( idxNum==1 ){ + char *zStmt = sqlite3_mprintf( + "SELECT * FROM %Q.%Q WHERE rowid=?", p->zDb, p->zTbl); + if( !zStmt ) return SQLITE_NOMEM; + rc = sqlite3_prepare_v2(p->db, zStmt, -1, &pCur->pStmt, 0); + sqlite3_free(zStmt); + if( rc==SQLITE_OK ){ + sqlite3_bind_value(pCur->pStmt, 1, argv[0]); + } + }else{ + char *zStmt = sqlite3_mprintf("SELECT * FROM %Q.%Q", p->zDb, p->zTbl); + if( !zStmt ) return SQLITE_NOMEM; + rc = sqlite3_prepare_v2(p->db, zStmt, -1, &pCur->pStmt, 0); + sqlite3_free(zStmt); + } + + if( rc==SQLITE_OK ){ + rc = fsNext(pVtabCursor); + } + return rc; +} + +static int fsColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){ + fs_cursor *pCur = (fs_cursor*)cur; + + assert( i==0 || i==1 ); + if( i==0 ){ + sqlite3_result_value(ctx, sqlite3_column_value(pCur->pStmt, 0)); + }else{ + const char *zFile = (const char *)sqlite3_column_text(pCur->pStmt, 1); + struct stat sbuf; + int fd; + + fd = open(zFile, O_RDONLY); + if( fd<0 ) return SQLITE_IOERR; + fstat(fd, &sbuf); + + if( sbuf.st_size>=pCur->nAlloc ){ + int nNew = sbuf.st_size*2; + char *zNew; + if( nNew<1024 ) nNew = 1024; + + zNew = sqlite3Realloc(pCur->zBuf, nNew); + if( zNew==0 ){ + close(fd); + return SQLITE_NOMEM; + } + pCur->zBuf = zNew; + pCur->nAlloc = nNew; + } + + read(fd, pCur->zBuf, sbuf.st_size); + close(fd); + pCur->nBuf = sbuf.st_size; + pCur->zBuf[pCur->nBuf] = '\0'; + + sqlite3_result_text(ctx, pCur->zBuf, -1, SQLITE_TRANSIENT); + } + return SQLITE_OK; +} + +static int fsRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + fs_cursor *pCur = (fs_cursor*)cur; + *pRowid = sqlite3_column_int64(pCur->pStmt, 0); + return SQLITE_OK; +} + +static int fsEof(sqlite3_vtab_cursor *cur){ + fs_cursor *pCur = (fs_cursor*)cur; + return (sqlite3_data_count(pCur->pStmt)==0); +} + +static int fsBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ + int ii; + + for(ii=0; ii<pIdxInfo->nConstraint; ii++){ + struct sqlite3_index_constraint const *pCons = &pIdxInfo->aConstraint[ii]; + if( pCons->iColumn<0 && pCons->usable + && pCons->op==SQLITE_INDEX_CONSTRAINT_EQ ){ + struct sqlite3_index_constraint_usage *pUsage; + pUsage = &pIdxInfo->aConstraintUsage[ii]; + pUsage->omit = 0; + pUsage->argvIndex = 1; + pIdxInfo->idxNum = 1; + pIdxInfo->estimatedCost = 1.0; + break; + } + } + + return SQLITE_OK; +} + +/* +** A virtual table module that provides read-only access to a +** Tcl global variable namespace. +*/ +static sqlite3_module fsModule = { + 0, /* iVersion */ + fsConnect, + fsConnect, + fsBestIndex, + fsDisconnect, + fsDisconnect, + fsOpen, /* xOpen - open a cursor */ + fsClose, /* xClose - close a cursor */ + fsFilter, /* xFilter - configure scan constraints */ + fsNext, /* xNext - advance a cursor */ + fsEof, /* xEof - check for end of scan */ + fsColumn, /* xColumn - read data */ + fsRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ +}; + +/* +** Decode a pointer to an sqlite3 object. +*/ +extern int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite3 **ppDb); + +/* +** Register the echo virtual table module. +*/ +static int register_fs_module( + ClientData clientData, /* Pointer to sqlite3_enable_XXX function */ + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int objc, /* Number of arguments */ + Tcl_Obj *CONST objv[] /* Command arguments */ +){ + sqlite3 *db; + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB"); + return TCL_ERROR; + } + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; +#ifndef SQLITE_OMIT_VIRTUALTABLE + sqlite3_create_module(db, "fs", &fsModule, (void *)interp); +#endif + return TCL_OK; +} + +#endif + + +/* +** Register commands with the TCL interpreter. +*/ +int Sqlitetestfs_Init(Tcl_Interp *interp){ +#ifndef SQLITE_OMIT_VIRTUALTABLE + static struct { + char *zName; + Tcl_ObjCmdProc *xProc; + void *clientData; + } aObjCmd[] = { + { "register_fs_module", register_fs_module, 0 }, + }; + int i; + for(i=0; i<sizeof(aObjCmd)/sizeof(aObjCmd[0]); i++){ + Tcl_CreateObjCommand(interp, aObjCmd[i].zName, + aObjCmd[i].xProc, aObjCmd[i].clientData, 0); + } +#endif + return TCL_OK; +} diff --git a/src/test_fuzzer.c b/src/test_fuzzer.c deleted file mode 100644 index 10496f2..0000000 --- a/src/test_fuzzer.c +++ /dev/null @@ -1,1215 +0,0 @@ -/* -** 2011 March 24 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** Code for a demonstration virtual table that generates variations -** on an input word at increasing edit distances from the original. -** -** A fuzzer virtual table is created like this: -** -** CREATE VIRTUAL TABLE f USING fuzzer(<fuzzer-data-table>); -** -** When it is created, the new fuzzer table must be supplied with the -** name of a "fuzzer data table", which must reside in the same database -** file as the new fuzzer table. The fuzzer data table contains the various -** transformations and their costs that the fuzzer logic uses to generate -** variations. -** -** The fuzzer data table must contain exactly four columns (more precisely, -** the statement "SELECT * FROM <fuzzer_data_table>" must return records -** that consist of four columns). It does not matter what the columns are -** named. -** -** Each row in the fuzzer data table represents a single character -** transformation. The left most column of the row (column 0) contains an -** integer value - the identifier of the ruleset to which the transformation -** rule belongs (see "MULTIPLE RULE SETS" below). The second column of the -** row (column 0) contains the input character or characters. The third -** column contains the output character or characters. And the fourth column -** contains the integer cost of making the transformation. For example: -** -** CREATE TABLE f_data(ruleset, cFrom, cTo, Cost); -** INSERT INTO f_data(ruleset, cFrom, cTo, Cost) VALUES(0, '', 'a', 100); -** INSERT INTO f_data(ruleset, cFrom, cTo, Cost) VALUES(0, 'b', '', 87); -** INSERT INTO f_data(ruleset, cFrom, cTo, Cost) VALUES(0, 'o', 'oe', 38); -** INSERT INTO f_data(ruleset, cFrom, cTo, Cost) VALUES(0, 'oe', 'o', 40); -** -** The first row inserted into the fuzzer data table by the SQL script -** above indicates that the cost of inserting a letter 'a' is 100. (All -** costs are integers. We recommend that costs be scaled so that the -** average cost is around 100.) The second INSERT statement creates a rule -** saying that the cost of deleting a single letter 'b' is 87. The third -** and fourth INSERT statements mean that the cost of transforming a -** single letter "o" into the two-letter sequence "oe" is 38 and that the -** cost of transforming "oe" back into "o" is 40. -** -** The contents of the fuzzer data table are loaded into main memory when -** a fuzzer table is first created, and may be internally reloaded by the -** system at any subsequent time. Therefore, the fuzzer data table should be -** populated before the fuzzer table is created and not modified thereafter. -** If you do need to modify the contents of the fuzzer data table, it is -** recommended that the associated fuzzer table be dropped, the fuzzer data -** table edited, and the fuzzer table recreated within a single transaction. -** Alternatively, the fuzzer data table can be edited then the database -** connection can be closed and reopened. -** -** Once it has been created, the fuzzer table can be queried as follows: -** -** SELECT word, distance FROM f -** WHERE word MATCH 'abcdefg' -** AND distance<200; -** -** This first query outputs the string "abcdefg" and all strings that -** can be derived from that string by appling the specified transformations. -** The strings are output together with their total transformation cost -** (called "distance") and appear in order of increasing cost. No string -** is output more than once. If there are multiple ways to transform the -** target string into the output string then the lowest cost transform is -** the one that is returned. In the example, the search is limited to -** strings with a total distance of less than 200. -** -** The fuzzer is a read-only table. Any attempt to DELETE, INSERT, or -** UPDATE on a fuzzer table will throw an error. -** -** It is important to put some kind of a limit on the fuzzer output. This -** can be either in the form of a LIMIT clause at the end of the query, -** or better, a "distance<NNN" constraint where NNN is some number. The -** running time and memory requirement is exponential in the value of NNN -** so you want to make sure that NNN is not too big. A value of NNN that -** is about twice the average transformation cost seems to give good results. -** -** The fuzzer table can be useful for tasks such as spelling correction. -** Suppose there is a second table vocabulary(w) where the w column contains -** all correctly spelled words. Let $word be a word you want to look up. -** -** SELECT vocabulary.w FROM f, vocabulary -** WHERE f.word MATCH $word -** AND f.distance<=200 -** AND f.word=vocabulary.w -** LIMIT 20 -** -** The query above gives the 20 closest words to the $word being tested. -** (Note that for good performance, the vocubulary.w column should be -** indexed.) -** -** A similar query can be used to find all words in the dictionary that -** begin with some prefix $prefix: -** -** SELECT vocabulary.w FROM f, vocabulary -** WHERE f.word MATCH $prefix -** AND f.distance<=200 -** AND vocabulary.w BETWEEN f.word AND (f.word || x'F7BFBFBF') -** LIMIT 50 -** -** This last query will show up to 50 words out of the vocabulary that -** match or nearly match the $prefix. -** -** MULTIPLE RULE SETS -** -** Normally, the "ruleset" value associated with all character transformations -** in the fuzzer data table is zero. However, if required, the fuzzer table -** allows multiple rulesets to be defined. Each query uses only a single -** ruleset. This allows, for example, a single fuzzer table to support -** multiple languages. -** -** By default, only the rules from ruleset 0 are used. To specify an -** alternative ruleset, a "ruleset = ?" expression must be added to the -** WHERE clause of a SELECT, where ? is the identifier of the desired -** ruleset. For example: -** -** SELECT vocabulary.w FROM f, vocabulary -** WHERE f.word MATCH $word -** AND f.distance<=200 -** AND f.word=vocabulary.w -** AND f.ruleset=1 -- Specify the ruleset to use here -** LIMIT 20 -** -** If no "ruleset = ?" constraint is specified in the WHERE clause, ruleset -** 0 is used. -** -** LIMITS -** -** The maximum ruleset number is 2147483647. The maximum length of either -** of the strings in the second or third column of the fuzzer data table -** is 50 bytes. The maximum cost on a rule is 1000. -*/ - -/* If SQLITE_DEBUG is not defined, disable assert statements. */ -#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) -# define NDEBUG -#endif - -#include "sqlite3.h" -#include <stdlib.h> -#include <string.h> -#include <assert.h> -#include <stdio.h> - -#ifndef SQLITE_OMIT_VIRTUALTABLE - -/* -** Forward declaration of objects used by this implementation -*/ -typedef struct fuzzer_vtab fuzzer_vtab; -typedef struct fuzzer_cursor fuzzer_cursor; -typedef struct fuzzer_rule fuzzer_rule; -typedef struct fuzzer_seen fuzzer_seen; -typedef struct fuzzer_stem fuzzer_stem; - -/* -** Various types. -** -** fuzzer_cost is the "cost" of an edit operation. -** -** fuzzer_len is the length of a matching string. -** -** fuzzer_ruleid is an ruleset identifier. -*/ -typedef int fuzzer_cost; -typedef signed char fuzzer_len; -typedef int fuzzer_ruleid; - -/* -** Limits -*/ -#define FUZZER_MX_LENGTH 50 /* Maximum length of a rule string */ -#define FUZZER_MX_RULEID 2147483647 /* Maximum rule ID */ -#define FUZZER_MX_COST 1000 /* Maximum single-rule cost */ -#define FUZZER_MX_OUTPUT_LENGTH 100 /* Maximum length of an output string */ - - -/* -** Each transformation rule is stored as an instance of this object. -** All rules are kept on a linked list sorted by rCost. -*/ -struct fuzzer_rule { - fuzzer_rule *pNext; /* Next rule in order of increasing rCost */ - char *zFrom; /* Transform from */ - fuzzer_cost rCost; /* Cost of this transformation */ - fuzzer_len nFrom, nTo; /* Length of the zFrom and zTo strings */ - fuzzer_ruleid iRuleset; /* The rule set to which this rule belongs */ - char zTo[4]; /* Transform to (extra space appended) */ -}; - -/* -** A stem object is used to generate variants. It is also used to record -** previously generated outputs. -** -** Every stem is added to a hash table as it is output. Generation of -** duplicate stems is suppressed. -** -** Active stems (those that might generate new outputs) are kepts on a linked -** list sorted by increasing cost. The cost is the sum of rBaseCost and -** pRule->rCost. -*/ -struct fuzzer_stem { - char *zBasis; /* Word being fuzzed */ - const fuzzer_rule *pRule; /* Current rule to apply */ - fuzzer_stem *pNext; /* Next stem in rCost order */ - fuzzer_stem *pHash; /* Next stem with same hash on zBasis */ - fuzzer_cost rBaseCost; /* Base cost of getting to zBasis */ - fuzzer_cost rCostX; /* Precomputed rBaseCost + pRule->rCost */ - fuzzer_len nBasis; /* Length of the zBasis string */ - fuzzer_len n; /* Apply pRule at this character offset */ -}; - -/* -** A fuzzer virtual-table object -*/ -struct fuzzer_vtab { - sqlite3_vtab base; /* Base class - must be first */ - char *zClassName; /* Name of this class. Default: "fuzzer" */ - fuzzer_rule *pRule; /* All active rules in this fuzzer */ - int nCursor; /* Number of active cursors */ -}; - -#define FUZZER_HASH 4001 /* Hash table size */ -#define FUZZER_NQUEUE 20 /* Number of slots on the stem queue */ - -/* A fuzzer cursor object */ -struct fuzzer_cursor { - sqlite3_vtab_cursor base; /* Base class - must be first */ - sqlite3_int64 iRowid; /* The rowid of the current word */ - fuzzer_vtab *pVtab; /* The virtual table this cursor belongs to */ - fuzzer_cost rLimit; /* Maximum cost of any term */ - fuzzer_stem *pStem; /* Stem with smallest rCostX */ - fuzzer_stem *pDone; /* Stems already processed to completion */ - fuzzer_stem *aQueue[FUZZER_NQUEUE]; /* Queue of stems with higher rCostX */ - int mxQueue; /* Largest used index in aQueue[] */ - char *zBuf; /* Temporary use buffer */ - int nBuf; /* Bytes allocated for zBuf */ - int nStem; /* Number of stems allocated */ - int iRuleset; /* Only process rules from this ruleset */ - fuzzer_rule nullRule; /* Null rule used first */ - fuzzer_stem *apHash[FUZZER_HASH]; /* Hash of previously generated terms */ -}; - -/* -** The two input rule lists are both sorted in order of increasing -** cost. Merge them together into a single list, sorted by cost, and -** return a pointer to the head of that list. -*/ -static fuzzer_rule *fuzzerMergeRules(fuzzer_rule *pA, fuzzer_rule *pB){ - fuzzer_rule head; - fuzzer_rule *pTail; - - pTail = &head; - while( pA && pB ){ - if( pA->rCost<=pB->rCost ){ - pTail->pNext = pA; - pTail = pA; - pA = pA->pNext; - }else{ - pTail->pNext = pB; - pTail = pB; - pB = pB->pNext; - } - } - if( pA==0 ){ - pTail->pNext = pB; - }else{ - pTail->pNext = pA; - } - return head.pNext; -} - -/* -** Statement pStmt currently points to a row in the fuzzer data table. This -** function allocates and populates a fuzzer_rule structure according to -** the content of the row. -** -** If successful, *ppRule is set to point to the new object and SQLITE_OK -** is returned. Otherwise, *ppRule is zeroed, *pzErr may be set to point -** to an error message and an SQLite error code returned. -*/ -static int fuzzerLoadOneRule( - fuzzer_vtab *p, /* Fuzzer virtual table handle */ - sqlite3_stmt *pStmt, /* Base rule on statements current row */ - fuzzer_rule **ppRule, /* OUT: New rule object */ - char **pzErr /* OUT: Error message */ -){ - sqlite3_int64 iRuleset = sqlite3_column_int64(pStmt, 0); - const char *zFrom = (const char *)sqlite3_column_text(pStmt, 1); - const char *zTo = (const char *)sqlite3_column_text(pStmt, 2); - int nCost = sqlite3_column_int(pStmt, 3); - - int rc = SQLITE_OK; /* Return code */ - int nFrom; /* Size of string zFrom, in bytes */ - int nTo; /* Size of string zTo, in bytes */ - fuzzer_rule *pRule = 0; /* New rule object to return */ - - if( zFrom==0 ) zFrom = ""; - if( zTo==0 ) zTo = ""; - nFrom = (int)strlen(zFrom); - nTo = (int)strlen(zTo); - - /* Silently ignore null transformations */ - if( strcmp(zFrom, zTo)==0 ){ - *ppRule = 0; - return SQLITE_OK; - } - - if( nCost<=0 || nCost>FUZZER_MX_COST ){ - *pzErr = sqlite3_mprintf("%s: cost must be between 1 and %d", - p->zClassName, FUZZER_MX_COST - ); - rc = SQLITE_ERROR; - }else - if( nFrom>FUZZER_MX_LENGTH || nTo>FUZZER_MX_LENGTH ){ - *pzErr = sqlite3_mprintf("%s: maximum string length is %d", - p->zClassName, FUZZER_MX_LENGTH - ); - rc = SQLITE_ERROR; - }else - if( iRuleset<0 || iRuleset>FUZZER_MX_RULEID ){ - *pzErr = sqlite3_mprintf("%s: ruleset must be between 0 and %d", - p->zClassName, FUZZER_MX_RULEID - ); - rc = SQLITE_ERROR; - }else{ - - pRule = sqlite3_malloc( sizeof(*pRule) + nFrom + nTo ); - if( pRule==0 ){ - rc = SQLITE_NOMEM; - }else{ - memset(pRule, 0, sizeof(*pRule)); - pRule->zFrom = &pRule->zTo[nTo+1]; - pRule->nFrom = nFrom; - memcpy(pRule->zFrom, zFrom, nFrom+1); - memcpy(pRule->zTo, zTo, nTo+1); - pRule->nTo = nTo; - pRule->rCost = nCost; - pRule->iRuleset = (int)iRuleset; - } - } - - *ppRule = pRule; - return rc; -} - -/* -** Load the content of the fuzzer data table into memory. -*/ -static int fuzzerLoadRules( - sqlite3 *db, /* Database handle */ - fuzzer_vtab *p, /* Virtual fuzzer table to configure */ - const char *zDb, /* Database containing rules data */ - const char *zData, /* Table containing rules data */ - char **pzErr /* OUT: Error message */ -){ - int rc = SQLITE_OK; /* Return code */ - char *zSql; /* SELECT used to read from rules table */ - fuzzer_rule *pHead = 0; - - zSql = sqlite3_mprintf("SELECT * FROM %Q.%Q", zDb, zData); - if( zSql==0 ){ - rc = SQLITE_NOMEM; - }else{ - int rc2; /* finalize() return code */ - sqlite3_stmt *pStmt = 0; - rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); - if( rc!=SQLITE_OK ){ - *pzErr = sqlite3_mprintf("%s: %s", p->zClassName, sqlite3_errmsg(db)); - }else if( sqlite3_column_count(pStmt)!=4 ){ - *pzErr = sqlite3_mprintf("%s: %s has %d columns, expected 4", - p->zClassName, zData, sqlite3_column_count(pStmt) - ); - rc = SQLITE_ERROR; - }else{ - while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ - fuzzer_rule *pRule = 0; - rc = fuzzerLoadOneRule(p, pStmt, &pRule, pzErr); - if( pRule ){ - pRule->pNext = pHead; - pHead = pRule; - } - } - } - rc2 = sqlite3_finalize(pStmt); - if( rc==SQLITE_OK ) rc = rc2; - } - sqlite3_free(zSql); - - /* All rules are now in a singly linked list starting at pHead. This - ** block sorts them by cost and then sets fuzzer_vtab.pRule to point to - ** point to the head of the sorted list. - */ - if( rc==SQLITE_OK ){ - unsigned int i; - fuzzer_rule *pX; - fuzzer_rule *a[15]; - for(i=0; i<sizeof(a)/sizeof(a[0]); i++) a[i] = 0; - while( (pX = pHead)!=0 ){ - pHead = pX->pNext; - pX->pNext = 0; - for(i=0; a[i] && i<sizeof(a)/sizeof(a[0])-1; i++){ - pX = fuzzerMergeRules(a[i], pX); - a[i] = 0; - } - a[i] = fuzzerMergeRules(a[i], pX); - } - for(pX=a[0], i=1; i<sizeof(a)/sizeof(a[0]); i++){ - pX = fuzzerMergeRules(a[i], pX); - } - p->pRule = fuzzerMergeRules(p->pRule, pX); - }else{ - /* An error has occurred. Setting p->pRule to point to the head of the - ** allocated list ensures that the list will be cleaned up in this case. - */ - assert( p->pRule==0 ); - p->pRule = pHead; - } - - return rc; -} - -/* -** This function converts an SQL quoted string into an unquoted string -** and returns a pointer to a buffer allocated using sqlite3_malloc() -** containing the result. The caller should eventually free this buffer -** using sqlite3_free. -** -** Examples: -** -** "abc" becomes abc -** 'xyz' becomes xyz -** [pqr] becomes pqr -** `mno` becomes mno -*/ -static char *fuzzerDequote(const char *zIn){ - int nIn; /* Size of input string, in bytes */ - char *zOut; /* Output (dequoted) string */ - - nIn = (int)strlen(zIn); - zOut = sqlite3_malloc(nIn+1); - if( zOut ){ - char q = zIn[0]; /* Quote character (if any ) */ - - if( q!='[' && q!= '\'' && q!='"' && q!='`' ){ - memcpy(zOut, zIn, nIn+1); - }else{ - int iOut = 0; /* Index of next byte to write to output */ - int iIn; /* Index of next byte to read from input */ - - if( q=='[' ) q = ']'; - for(iIn=1; iIn<nIn; iIn++){ - if( zIn[iIn]==q ) iIn++; - zOut[iOut++] = zIn[iIn]; - } - } - assert( (int)strlen(zOut)<=nIn ); - } - return zOut; -} - -/* -** xDisconnect/xDestroy method for the fuzzer module. -*/ -static int fuzzerDisconnect(sqlite3_vtab *pVtab){ - fuzzer_vtab *p = (fuzzer_vtab*)pVtab; - assert( p->nCursor==0 ); - while( p->pRule ){ - fuzzer_rule *pRule = p->pRule; - p->pRule = pRule->pNext; - sqlite3_free(pRule); - } - sqlite3_free(p); - return SQLITE_OK; -} - -/* -** xConnect/xCreate method for the fuzzer module. Arguments are: -** -** argv[0] -> module name ("fuzzer") -** argv[1] -> database name -** argv[2] -> table name -** argv[3] -> fuzzer rule table name -*/ -static int fuzzerConnect( - sqlite3 *db, - void *pAux, - int argc, const char *const*argv, - sqlite3_vtab **ppVtab, - char **pzErr -){ - int rc = SQLITE_OK; /* Return code */ - fuzzer_vtab *pNew = 0; /* New virtual table */ - const char *zModule = argv[0]; - const char *zDb = argv[1]; - - if( argc!=4 ){ - *pzErr = sqlite3_mprintf( - "%s: wrong number of CREATE VIRTUAL TABLE arguments", zModule - ); - rc = SQLITE_ERROR; - }else{ - int nModule; /* Length of zModule, in bytes */ - - nModule = (int)strlen(zModule); - pNew = sqlite3_malloc( sizeof(*pNew) + nModule + 1); - if( pNew==0 ){ - rc = SQLITE_NOMEM; - }else{ - char *zTab; /* Dequoted name of fuzzer data table */ - - memset(pNew, 0, sizeof(*pNew)); - pNew->zClassName = (char*)&pNew[1]; - memcpy(pNew->zClassName, zModule, nModule+1); - - zTab = fuzzerDequote(argv[3]); - if( zTab==0 ){ - rc = SQLITE_NOMEM; - }else{ - rc = fuzzerLoadRules(db, pNew, zDb, zTab, pzErr); - sqlite3_free(zTab); - } - - if( rc==SQLITE_OK ){ - rc = sqlite3_declare_vtab(db, "CREATE TABLE x(word,distance,ruleset)"); - } - if( rc!=SQLITE_OK ){ - fuzzerDisconnect((sqlite3_vtab *)pNew); - pNew = 0; - } - } - } - - *ppVtab = (sqlite3_vtab *)pNew; - return rc; -} - -/* -** Open a new fuzzer cursor. -*/ -static int fuzzerOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ - fuzzer_vtab *p = (fuzzer_vtab*)pVTab; - fuzzer_cursor *pCur; - pCur = sqlite3_malloc( sizeof(*pCur) ); - if( pCur==0 ) return SQLITE_NOMEM; - memset(pCur, 0, sizeof(*pCur)); - pCur->pVtab = p; - *ppCursor = &pCur->base; - p->nCursor++; - return SQLITE_OK; -} - -/* -** Free all stems in a list. -*/ -static void fuzzerClearStemList(fuzzer_stem *pStem){ - while( pStem ){ - fuzzer_stem *pNext = pStem->pNext; - sqlite3_free(pStem); - pStem = pNext; - } -} - -/* -** Free up all the memory allocated by a cursor. Set it rLimit to 0 -** to indicate that it is at EOF. -*/ -static void fuzzerClearCursor(fuzzer_cursor *pCur, int clearHash){ - int i; - fuzzerClearStemList(pCur->pStem); - fuzzerClearStemList(pCur->pDone); - for(i=0; i<FUZZER_NQUEUE; i++) fuzzerClearStemList(pCur->aQueue[i]); - pCur->rLimit = (fuzzer_cost)0; - if( clearHash && pCur->nStem ){ - pCur->mxQueue = 0; - pCur->pStem = 0; - pCur->pDone = 0; - memset(pCur->aQueue, 0, sizeof(pCur->aQueue)); - memset(pCur->apHash, 0, sizeof(pCur->apHash)); - } - pCur->nStem = 0; -} - -/* -** Close a fuzzer cursor. -*/ -static int fuzzerClose(sqlite3_vtab_cursor *cur){ - fuzzer_cursor *pCur = (fuzzer_cursor *)cur; - fuzzerClearCursor(pCur, 0); - sqlite3_free(pCur->zBuf); - pCur->pVtab->nCursor--; - sqlite3_free(pCur); - return SQLITE_OK; -} - -/* -** Compute the current output term for a fuzzer_stem. -*/ -static int fuzzerRender( - fuzzer_stem *pStem, /* The stem to be rendered */ - char **pzBuf, /* Write results into this buffer. realloc if needed */ - int *pnBuf /* Size of the buffer */ -){ - const fuzzer_rule *pRule = pStem->pRule; - int n; /* Size of output term without nul-term */ - char *z; /* Buffer to assemble output term in */ - - n = pStem->nBasis + pRule->nTo - pRule->nFrom; - if( (*pnBuf)<n+1 ){ - (*pzBuf) = sqlite3_realloc((*pzBuf), n+100); - if( (*pzBuf)==0 ) return SQLITE_NOMEM; - (*pnBuf) = n+100; - } - n = pStem->n; - z = *pzBuf; - if( n<0 ){ - memcpy(z, pStem->zBasis, pStem->nBasis+1); - }else{ - memcpy(z, pStem->zBasis, n); - memcpy(&z[n], pRule->zTo, pRule->nTo); - memcpy(&z[n+pRule->nTo], &pStem->zBasis[n+pRule->nFrom], - pStem->nBasis-n-pRule->nFrom+1); - } - - assert( z[pStem->nBasis + pRule->nTo - pRule->nFrom]==0 ); - return SQLITE_OK; -} - -/* -** Compute a hash on zBasis. -*/ -static unsigned int fuzzerHash(const char *z){ - unsigned int h = 0; - while( *z ){ h = (h<<3) ^ (h>>29) ^ *(z++); } - return h % FUZZER_HASH; -} - -/* -** Current cost of a stem -*/ -static fuzzer_cost fuzzerCost(fuzzer_stem *pStem){ - return pStem->rCostX = pStem->rBaseCost + pStem->pRule->rCost; -} - -#if 0 -/* -** Print a description of a fuzzer_stem on stderr. -*/ -static void fuzzerStemPrint( - const char *zPrefix, - fuzzer_stem *pStem, - const char *zSuffix -){ - if( pStem->n<0 ){ - fprintf(stderr, "%s[%s](%d)-->self%s", - zPrefix, - pStem->zBasis, pStem->rBaseCost, - zSuffix - ); - }else{ - char *zBuf = 0; - int nBuf = 0; - if( fuzzerRender(pStem, &zBuf, &nBuf)!=SQLITE_OK ) return; - fprintf(stderr, "%s[%s](%d)-->{%s}(%d)%s", - zPrefix, - pStem->zBasis, pStem->rBaseCost, zBuf, pStem->, - zSuffix - ); - sqlite3_free(zBuf); - } -} -#endif - -/* -** Return 1 if the string to which the cursor is point has already -** been emitted. Return 0 if not. Return -1 on a memory allocation -** failures. -*/ -static int fuzzerSeen(fuzzer_cursor *pCur, fuzzer_stem *pStem){ - unsigned int h; - fuzzer_stem *pLookup; - - if( fuzzerRender(pStem, &pCur->zBuf, &pCur->nBuf)==SQLITE_NOMEM ){ - return -1; - } - h = fuzzerHash(pCur->zBuf); - pLookup = pCur->apHash[h]; - while( pLookup && strcmp(pLookup->zBasis, pCur->zBuf)!=0 ){ - pLookup = pLookup->pHash; - } - return pLookup!=0; -} - -/* -** If argument pRule is NULL, this function returns false. -** -** Otherwise, it returns true if rule pRule should be skipped. A rule -** should be skipped if it does not belong to rule-set iRuleset, or if -** applying it to stem pStem would create a string longer than -** FUZZER_MX_OUTPUT_LENGTH bytes. -*/ -static int fuzzerSkipRule( - const fuzzer_rule *pRule, /* Determine whether or not to skip this */ - fuzzer_stem *pStem, /* Stem rule may be applied to */ - int iRuleset /* Rule-set used by the current query */ -){ - return pRule && ( - (pRule->iRuleset!=iRuleset) - || (pStem->nBasis + pRule->nTo - pRule->nFrom)>FUZZER_MX_OUTPUT_LENGTH - ); -} - -/* -** Advance a fuzzer_stem to its next value. Return 0 if there are -** no more values that can be generated by this fuzzer_stem. Return -** -1 on a memory allocation failure. -*/ -static int fuzzerAdvance(fuzzer_cursor *pCur, fuzzer_stem *pStem){ - const fuzzer_rule *pRule; - while( (pRule = pStem->pRule)!=0 ){ - assert( pRule==&pCur->nullRule || pRule->iRuleset==pCur->iRuleset ); - while( pStem->n < pStem->nBasis - pRule->nFrom ){ - pStem->n++; - if( pRule->nFrom==0 - || memcmp(&pStem->zBasis[pStem->n], pRule->zFrom, pRule->nFrom)==0 - ){ - /* Found a rewrite case. Make sure it is not a duplicate */ - int rc = fuzzerSeen(pCur, pStem); - if( rc<0 ) return -1; - if( rc==0 ){ - fuzzerCost(pStem); - return 1; - } - } - } - pStem->n = -1; - do{ - pRule = pRule->pNext; - }while( fuzzerSkipRule(pRule, pStem, pCur->iRuleset) ); - pStem->pRule = pRule; - if( pRule && fuzzerCost(pStem)>pCur->rLimit ) pStem->pRule = 0; - } - return 0; -} - -/* -** The two input stem lists are both sorted in order of increasing -** rCostX. Merge them together into a single list, sorted by rCostX, and -** return a pointer to the head of that new list. -*/ -static fuzzer_stem *fuzzerMergeStems(fuzzer_stem *pA, fuzzer_stem *pB){ - fuzzer_stem head; - fuzzer_stem *pTail; - - pTail = &head; - while( pA && pB ){ - if( pA->rCostX<=pB->rCostX ){ - pTail->pNext = pA; - pTail = pA; - pA = pA->pNext; - }else{ - pTail->pNext = pB; - pTail = pB; - pB = pB->pNext; - } - } - if( pA==0 ){ - pTail->pNext = pB; - }else{ - pTail->pNext = pA; - } - return head.pNext; -} - -/* -** Load pCur->pStem with the lowest-cost stem. Return a pointer -** to the lowest-cost stem. -*/ -static fuzzer_stem *fuzzerLowestCostStem(fuzzer_cursor *pCur){ - fuzzer_stem *pBest, *pX; - int iBest; - int i; - - if( pCur->pStem==0 ){ - iBest = -1; - pBest = 0; - for(i=0; i<=pCur->mxQueue; i++){ - pX = pCur->aQueue[i]; - if( pX==0 ) continue; - if( pBest==0 || pBest->rCostX>pX->rCostX ){ - pBest = pX; - iBest = i; - } - } - if( pBest ){ - pCur->aQueue[iBest] = pBest->pNext; - pBest->pNext = 0; - pCur->pStem = pBest; - } - } - return pCur->pStem; -} - -/* -** Insert pNew into queue of pending stems. Then find the stem -** with the lowest rCostX and move it into pCur->pStem. -** list. The insert is done such the pNew is in the correct order -** according to fuzzer_stem.zBaseCost+fuzzer_stem.pRule->rCost. -*/ -static fuzzer_stem *fuzzerInsert(fuzzer_cursor *pCur, fuzzer_stem *pNew){ - fuzzer_stem *pX; - int i; - - /* If pCur->pStem exists and is greater than pNew, then make pNew - ** the new pCur->pStem and insert the old pCur->pStem instead. - */ - if( (pX = pCur->pStem)!=0 && pX->rCostX>pNew->rCostX ){ - pNew->pNext = 0; - pCur->pStem = pNew; - pNew = pX; - } - - /* Insert the new value */ - pNew->pNext = 0; - pX = pNew; - for(i=0; i<=pCur->mxQueue; i++){ - if( pCur->aQueue[i] ){ - pX = fuzzerMergeStems(pX, pCur->aQueue[i]); - pCur->aQueue[i] = 0; - }else{ - pCur->aQueue[i] = pX; - break; - } - } - if( i>pCur->mxQueue ){ - if( i<FUZZER_NQUEUE ){ - pCur->mxQueue = i; - pCur->aQueue[i] = pX; - }else{ - assert( pCur->mxQueue==FUZZER_NQUEUE-1 ); - pX = fuzzerMergeStems(pX, pCur->aQueue[FUZZER_NQUEUE-1]); - pCur->aQueue[FUZZER_NQUEUE-1] = pX; - } - } - - return fuzzerLowestCostStem(pCur); -} - -/* -** Allocate a new fuzzer_stem. Add it to the hash table but do not -** link it into either the pCur->pStem or pCur->pDone lists. -*/ -static fuzzer_stem *fuzzerNewStem( - fuzzer_cursor *pCur, - const char *zWord, - fuzzer_cost rBaseCost -){ - fuzzer_stem *pNew; - fuzzer_rule *pRule; - unsigned int h; - - pNew = sqlite3_malloc( sizeof(*pNew) + (int)strlen(zWord) + 1 ); - if( pNew==0 ) return 0; - memset(pNew, 0, sizeof(*pNew)); - pNew->zBasis = (char*)&pNew[1]; - pNew->nBasis = (int)strlen(zWord); - memcpy(pNew->zBasis, zWord, pNew->nBasis+1); - pRule = pCur->pVtab->pRule; - while( fuzzerSkipRule(pRule, pNew, pCur->iRuleset) ){ - pRule = pRule->pNext; - } - pNew->pRule = pRule; - pNew->n = -1; - pNew->rBaseCost = pNew->rCostX = rBaseCost; - h = fuzzerHash(pNew->zBasis); - pNew->pHash = pCur->apHash[h]; - pCur->apHash[h] = pNew; - pCur->nStem++; - return pNew; -} - - -/* -** Advance a cursor to its next row of output -*/ -static int fuzzerNext(sqlite3_vtab_cursor *cur){ - fuzzer_cursor *pCur = (fuzzer_cursor*)cur; - int rc; - fuzzer_stem *pStem, *pNew; - - pCur->iRowid++; - - /* Use the element the cursor is currently point to to create - ** a new stem and insert the new stem into the priority queue. - */ - pStem = pCur->pStem; - if( pStem->rCostX>0 ){ - rc = fuzzerRender(pStem, &pCur->zBuf, &pCur->nBuf); - if( rc==SQLITE_NOMEM ) return SQLITE_NOMEM; - pNew = fuzzerNewStem(pCur, pCur->zBuf, pStem->rCostX); - if( pNew ){ - if( fuzzerAdvance(pCur, pNew)==0 ){ - pNew->pNext = pCur->pDone; - pCur->pDone = pNew; - }else{ - if( fuzzerInsert(pCur, pNew)==pNew ){ - return SQLITE_OK; - } - } - }else{ - return SQLITE_NOMEM; - } - } - - /* Adjust the priority queue so that the first element of the - ** stem list is the next lowest cost word. - */ - while( (pStem = pCur->pStem)!=0 ){ - int res = fuzzerAdvance(pCur, pStem); - if( res<0 ){ - return SQLITE_NOMEM; - }else if( res>0 ){ - pCur->pStem = 0; - pStem = fuzzerInsert(pCur, pStem); - if( (rc = fuzzerSeen(pCur, pStem))!=0 ){ - if( rc<0 ) return SQLITE_NOMEM; - continue; - } - return SQLITE_OK; /* New word found */ - } - pCur->pStem = 0; - pStem->pNext = pCur->pDone; - pCur->pDone = pStem; - if( fuzzerLowestCostStem(pCur) ){ - rc = fuzzerSeen(pCur, pCur->pStem); - if( rc<0 ) return SQLITE_NOMEM; - if( rc==0 ){ - return SQLITE_OK; - } - } - } - - /* Reach this point only if queue has been exhausted and there is - ** nothing left to be output. */ - pCur->rLimit = (fuzzer_cost)0; - return SQLITE_OK; -} - -/* -** Called to "rewind" a cursor back to the beginning so that -** it starts its output over again. Always called at least once -** prior to any fuzzerColumn, fuzzerRowid, or fuzzerEof call. -*/ -static int fuzzerFilter( - sqlite3_vtab_cursor *pVtabCursor, - int idxNum, const char *idxStr, - int argc, sqlite3_value **argv -){ - fuzzer_cursor *pCur = (fuzzer_cursor *)pVtabCursor; - const char *zWord = ""; - fuzzer_stem *pStem; - int idx; - - fuzzerClearCursor(pCur, 1); - pCur->rLimit = 2147483647; - idx = 0; - if( idxNum & 1 ){ - zWord = (const char*)sqlite3_value_text(argv[0]); - idx++; - } - if( idxNum & 2 ){ - pCur->rLimit = (fuzzer_cost)sqlite3_value_int(argv[idx]); - idx++; - } - if( idxNum & 4 ){ - pCur->iRuleset = (fuzzer_cost)sqlite3_value_int(argv[idx]); - idx++; - } - pCur->nullRule.pNext = pCur->pVtab->pRule; - pCur->nullRule.rCost = 0; - pCur->nullRule.nFrom = 0; - pCur->nullRule.nTo = 0; - pCur->nullRule.zFrom = ""; - pCur->iRowid = 1; - assert( pCur->pStem==0 ); - - /* If the query term is longer than FUZZER_MX_OUTPUT_LENGTH bytes, this - ** query will return zero rows. */ - if( (int)strlen(zWord)<FUZZER_MX_OUTPUT_LENGTH ){ - pCur->pStem = pStem = fuzzerNewStem(pCur, zWord, (fuzzer_cost)0); - if( pStem==0 ) return SQLITE_NOMEM; - pStem->pRule = &pCur->nullRule; - pStem->n = pStem->nBasis; - }else{ - pCur->rLimit = 0; - } - - return SQLITE_OK; -} - -/* -** Only the word and distance columns have values. All other columns -** return NULL -*/ -static int fuzzerColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){ - fuzzer_cursor *pCur = (fuzzer_cursor*)cur; - if( i==0 ){ - /* the "word" column */ - if( fuzzerRender(pCur->pStem, &pCur->zBuf, &pCur->nBuf)==SQLITE_NOMEM ){ - return SQLITE_NOMEM; - } - sqlite3_result_text(ctx, pCur->zBuf, -1, SQLITE_TRANSIENT); - }else if( i==1 ){ - /* the "distance" column */ - sqlite3_result_int(ctx, pCur->pStem->rCostX); - }else{ - /* All other columns are NULL */ - sqlite3_result_null(ctx); - } - return SQLITE_OK; -} - -/* -** The rowid. -*/ -static int fuzzerRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ - fuzzer_cursor *pCur = (fuzzer_cursor*)cur; - *pRowid = pCur->iRowid; - return SQLITE_OK; -} - -/* -** When the fuzzer_cursor.rLimit value is 0 or less, that is a signal -** that the cursor has nothing more to output. -*/ -static int fuzzerEof(sqlite3_vtab_cursor *cur){ - fuzzer_cursor *pCur = (fuzzer_cursor*)cur; - return pCur->rLimit<=(fuzzer_cost)0; -} - -/* -** Search for terms of these forms: -** -** (A) word MATCH $str -** (B1) distance < $value -** (B2) distance <= $value -** (C) ruleid == $ruleid -** -** The distance< and distance<= are both treated as distance<=. -** The query plan number is a bit vector: -** -** bit 1: Term of the form (A) found -** bit 2: Term like (B1) or (B2) found -** bit 3: Term like (C) found -** -** If bit-1 is set, $str is always in filter.argv[0]. If bit-2 is set -** then $value is in filter.argv[0] if bit-1 is clear and is in -** filter.argv[1] if bit-1 is set. If bit-3 is set, then $ruleid is -** in filter.argv[0] if bit-1 and bit-2 are both zero, is in -** filter.argv[1] if exactly one of bit-1 and bit-2 are set, and is in -** filter.argv[2] if both bit-1 and bit-2 are set. -*/ -static int fuzzerBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ - int iPlan = 0; - int iDistTerm = -1; - int iRulesetTerm = -1; - int i; - const struct sqlite3_index_constraint *pConstraint; - pConstraint = pIdxInfo->aConstraint; - for(i=0; i<pIdxInfo->nConstraint; i++, pConstraint++){ - if( pConstraint->usable==0 ) continue; - if( (iPlan & 1)==0 - && pConstraint->iColumn==0 - && pConstraint->op==SQLITE_INDEX_CONSTRAINT_MATCH - ){ - iPlan |= 1; - pIdxInfo->aConstraintUsage[i].argvIndex = 1; - pIdxInfo->aConstraintUsage[i].omit = 1; - } - if( (iPlan & 2)==0 - && pConstraint->iColumn==1 - && (pConstraint->op==SQLITE_INDEX_CONSTRAINT_LT - || pConstraint->op==SQLITE_INDEX_CONSTRAINT_LE) - ){ - iPlan |= 2; - iDistTerm = i; - } - if( (iPlan & 4)==0 - && pConstraint->iColumn==2 - && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ - ){ - iPlan |= 4; - pIdxInfo->aConstraintUsage[i].omit = 1; - iRulesetTerm = i; - } - } - if( iPlan & 2 ){ - pIdxInfo->aConstraintUsage[iDistTerm].argvIndex = 1+((iPlan&1)!=0); - } - if( iPlan & 4 ){ - int idx = 1; - if( iPlan & 1 ) idx++; - if( iPlan & 2 ) idx++; - pIdxInfo->aConstraintUsage[iRulesetTerm].argvIndex = idx; - } - pIdxInfo->idxNum = iPlan; - if( pIdxInfo->nOrderBy==1 - && pIdxInfo->aOrderBy[0].iColumn==1 - && pIdxInfo->aOrderBy[0].desc==0 - ){ - pIdxInfo->orderByConsumed = 1; - } - pIdxInfo->estimatedCost = (double)10000; - - return SQLITE_OK; -} - -/* -** A virtual table module that implements the "fuzzer". -*/ -static sqlite3_module fuzzerModule = { - 0, /* iVersion */ - fuzzerConnect, - fuzzerConnect, - fuzzerBestIndex, - fuzzerDisconnect, - fuzzerDisconnect, - fuzzerOpen, /* xOpen - open a cursor */ - fuzzerClose, /* xClose - close a cursor */ - fuzzerFilter, /* xFilter - configure scan constraints */ - fuzzerNext, /* xNext - advance a cursor */ - fuzzerEof, /* xEof - check for end of scan */ - fuzzerColumn, /* xColumn - read data */ - fuzzerRowid, /* xRowid - read data */ - 0, /* xUpdate */ - 0, /* xBegin */ - 0, /* xSync */ - 0, /* xCommit */ - 0, /* xRollback */ - 0, /* xFindMethod */ - 0, /* xRename */ -}; - -#endif /* SQLITE_OMIT_VIRTUALTABLE */ - - -/* -** Register the fuzzer virtual table -*/ -int fuzzer_register(sqlite3 *db){ - int rc = SQLITE_OK; -#ifndef SQLITE_OMIT_VIRTUALTABLE - rc = sqlite3_create_module(db, "fuzzer", &fuzzerModule, 0); -#endif - return rc; -} - -#ifdef SQLITE_TEST -#include <tcl.h> -/* -** Decode a pointer to an sqlite3 object. -*/ -extern int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite3 **ppDb); - -/* -** Register the echo virtual table module. -*/ -static int register_fuzzer_module( - ClientData clientData, /* Pointer to sqlite3_enable_XXX function */ - Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ - int objc, /* Number of arguments */ - Tcl_Obj *CONST objv[] /* Command arguments */ -){ - sqlite3 *db; - if( objc!=2 ){ - Tcl_WrongNumArgs(interp, 1, objv, "DB"); - return TCL_ERROR; - } - getDbPointer(interp, Tcl_GetString(objv[1]), &db); - fuzzer_register(db); - return TCL_OK; -} - - -/* -** Register commands with the TCL interpreter. -*/ -int Sqlitetestfuzzer_Init(Tcl_Interp *interp){ - static struct { - char *zName; - Tcl_ObjCmdProc *xProc; - void *clientData; - } aObjCmd[] = { - { "register_fuzzer_module", register_fuzzer_module, 0 }, - }; - int i; - for(i=0; i<sizeof(aObjCmd)/sizeof(aObjCmd[0]); i++){ - Tcl_CreateObjCommand(interp, aObjCmd[i].zName, - aObjCmd[i].xProc, aObjCmd[i].clientData, 0); - } - return TCL_OK; -} - -#endif /* SQLITE_TEST */ diff --git a/src/test_intarray.c b/src/test_intarray.c index 8651d01..f5c3d9e 100644 --- a/src/test_intarray.c +++ b/src/test_intarray.c @@ -278,7 +278,7 @@ int sqlite3_intarray_bind( extern int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite3 **ppDb); extern void *sqlite3TestTextToPtr(const char*); extern int sqlite3TestMakePointerStr(Tcl_Interp*, char *zPtr, void*); -extern const char *sqlite3TestErrorName(int); +extern const char *sqlite3ErrName(int); /* ** sqlite3_intarray_create DB NAME @@ -309,7 +309,7 @@ static int test_intarray_create( #endif if( rc!=SQLITE_OK ){ assert( pArray==0 ); - Tcl_AppendResult(interp, sqlite3TestErrorName(rc), (char*)0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), (char*)0); return TCL_ERROR; } sqlite3TestMakePointerStr(interp, zPtr, pArray); @@ -346,12 +346,13 @@ static int test_intarray_bind( return TCL_ERROR; } for(i=0; i<n; i++){ - a[i] = 0; - Tcl_GetWideIntFromObj(0, objv[i+2], &a[i]); + Tcl_WideInt x = 0; + Tcl_GetWideIntFromObj(0, objv[i+2], &x); + a[i] = x; } rc = sqlite3_intarray_bind(pArray, n, a, sqlite3_free); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, sqlite3TestErrorName(rc), (char*)0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), (char*)0); return TCL_ERROR; } #endif diff --git a/src/test_intarray.h b/src/test_intarray.h index e994367..691337d 100644 --- a/src/test_intarray.h +++ b/src/test_intarray.h @@ -77,6 +77,13 @@ #include "sqlite3.h" /* +** Make sure we can call this stuff from C++. +*/ +#ifdef __cplusplus +extern "C" { +#endif + +/* ** An sqlite3_intarray is an abstract type to stores an instance of ** an integer array. */ @@ -112,3 +119,7 @@ int sqlite3_intarray_bind( sqlite3_int64 *aElements, /* Content of the intarray */ void (*xFree)(void*) /* How to dispose of the intarray when done */ ); + +#ifdef __cplusplus +} /* End of the 'extern "C"' block */ +#endif diff --git a/src/test_malloc.c b/src/test_malloc.c index 09b8f73..cf98a8f 100644 --- a/src/test_malloc.c +++ b/src/test_malloc.c @@ -234,14 +234,14 @@ static int faultsimInstall(int install){ #ifdef SQLITE_TEST /* -** This function is implemented in test1.c. Returns a pointer to a static +** This function is implemented in main.c. Returns a pointer to a static ** buffer containing the symbolic SQLite error code that corresponds to ** the least-significant 8-bits of the integer passed as an argument. ** For example: ** -** sqlite3TestErrorName(1) -> "SQLITE_ERROR" +** sqlite3ErrName(1) -> "SQLITE_ERROR" */ -const char *sqlite3TestErrorName(int); +extern const char *sqlite3ErrName(int); /* ** Transform pointers to text and back again @@ -720,8 +720,8 @@ static int test_memdebug_settitle( #ifdef SQLITE_MEMDEBUG { const char *zTitle; - zTitle = Tcl_GetString(objv[1]); extern int sqlite3MemdebugSettitle(const char*); + zTitle = Tcl_GetString(objv[1]); sqlite3MemdebugSettitle(zTitle); } #endif @@ -1072,7 +1072,7 @@ static int test_db_config_lookaside( sqlite3 *db; int bufid; static char azBuf[2][10000]; - int getDbPointer(Tcl_Interp*, const char*, sqlite3**); + extern int getDbPointer(Tcl_Interp*, const char*, sqlite3**); if( objc!=5 ){ Tcl_WrongNumArgs(interp, 1, objv, "BUFID SIZE COUNT"); return TCL_ERROR; @@ -1094,9 +1094,7 @@ static int test_db_config_lookaside( } /* -** Usage: -** -** sqlite3_config_heap NBYTE NMINALLOC +** Usage: sqlite3_config_heap NBYTE NMINALLOC */ static int test_config_heap( void * clientData, @@ -1128,12 +1126,12 @@ static int test_config_heap( rc = sqlite3_config(SQLITE_CONFIG_HEAP, zBuf, nByte, nMinAlloc); } - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_VOLATILE); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_VOLATILE); return TCL_OK; } /* -** tclcmd: sqlite3_config_error [DB] +** Usage: sqlite3_config_error [DB] ** ** Invoke sqlite3_config() or sqlite3_db_config() with invalid ** opcodes and verify that they return errors. @@ -1145,7 +1143,7 @@ static int test_config_error( Tcl_Obj *CONST objv[] ){ sqlite3 *db; - int getDbPointer(Tcl_Interp*, const char*, sqlite3**); + extern int getDbPointer(Tcl_Interp*, const char*, sqlite3**); if( objc!=2 && objc!=1 ){ Tcl_WrongNumArgs(interp, 1, objv, "[DB]"); @@ -1171,10 +1169,10 @@ static int test_config_error( } /* -** tclcmd: sqlite3_config_uri BOOLEAN +** Usage: sqlite3_config_uri BOOLEAN ** -** Invoke sqlite3_config() or sqlite3_db_config() with invalid -** opcodes and verify that they return errors. +** Enables or disables interpretation of URI parameters by default using +** SQLITE_CONFIG_URI. */ static int test_config_uri( void * clientData, @@ -1194,16 +1192,43 @@ static int test_config_uri( } rc = sqlite3_config(SQLITE_CONFIG_URI, bOpenUri); - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_VOLATILE); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_VOLATILE); return TCL_OK; } /* -** Usage: +** Usage: sqlite3_config_cis BOOLEAN ** -** sqlite3_dump_memsys3 FILENAME -** sqlite3_dump_memsys5 FILENAME +** Enables or disables the use of the covering-index scan optimization. +** SQLITE_CONFIG_COVERING_INDEX_SCAN. +*/ +static int test_config_cis( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + int rc; + int bUseCis; + + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "BOOL"); + return TCL_ERROR; + } + if( Tcl_GetBooleanFromObj(interp, objv[1], &bUseCis) ){ + return TCL_ERROR; + } + + rc = sqlite3_config(SQLITE_CONFIG_COVERING_INDEX_SCAN, bUseCis); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_VOLATILE); + + return TCL_OK; +} + +/* +** Usage: sqlite3_dump_memsys3 FILENAME +** sqlite3_dump_memsys5 FILENAME ** ** Write a summary of unfreed memsys3 allocations to FILENAME. */ @@ -1310,7 +1335,7 @@ static int test_db_status( int i, op, resetFlag; const char *zOpName; sqlite3 *db; - int getDbPointer(Tcl_Interp*, const char*, sqlite3**); + extern int getDbPointer(Tcl_Interp*, const char*, sqlite3**); static const struct { const char *zName; int op; @@ -1376,7 +1401,7 @@ static int test_install_malloc_faultsim( return TCL_ERROR; } rc = faultsimInstall(isInstall); - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_VOLATILE); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_VOLATILE); return TCL_OK; } @@ -1394,7 +1419,7 @@ static int test_install_memsys3( const sqlite3_mem_methods *sqlite3MemGetMemsys3(void); rc = sqlite3_config(SQLITE_CONFIG_MALLOC, sqlite3MemGetMemsys3()); #endif - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_VOLATILE); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_VOLATILE); return TCL_OK; } @@ -1451,6 +1476,7 @@ int Sqlitetest_malloc_Init(Tcl_Interp *interp){ { "sqlite3_config_lookaside", test_config_lookaside ,0 }, { "sqlite3_config_error", test_config_error ,0 }, { "sqlite3_config_uri", test_config_uri ,0 }, + { "sqlite3_config_cis", test_config_cis ,0 }, { "sqlite3_db_config_lookaside",test_db_config_lookaside ,0 }, { "sqlite3_dump_memsys3", test_dump_memsys3 ,3 }, { "sqlite3_dump_memsys5", test_dump_memsys3 ,5 }, diff --git a/src/test_multiplex.c b/src/test_multiplex.c index 23df347..624541b 100644 --- a/src/test_multiplex.c +++ b/src/test_multiplex.c @@ -60,7 +60,7 @@ /* ** These should be defined to be the same as the values in -** sqliteInt.h. They are defined seperately here so that +** sqliteInt.h. They are defined separately here so that ** the multiplex VFS shim can be built as a loadable ** module. */ @@ -1183,7 +1183,7 @@ int sqlite3_multiplex_shutdown(void){ /***************************** Test Code ***********************************/ #ifdef SQLITE_TEST #include <tcl.h> -extern const char *sqlite3TestErrorName(int); +extern const char *sqlite3ErrName(int); /* @@ -1212,7 +1212,7 @@ static int test_multiplex_initialize( /* Call sqlite3_multiplex_initialize() */ rc = sqlite3_multiplex_initialize(zName, makeDefault); - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_STATIC); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_STATIC); return TCL_OK; } @@ -1237,7 +1237,7 @@ static int test_multiplex_shutdown( /* Call sqlite3_multiplex_shutdown() */ rc = sqlite3_multiplex_shutdown(); - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_STATIC); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_STATIC); return TCL_OK; } @@ -1355,7 +1355,7 @@ static int test_multiplex_control( } rc = sqlite3_file_control(db, Tcl_GetString(objv[2]), aSub[idx].op, pArg); - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_STATIC); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_STATIC); return (rc==SQLITE_OK) ? TCL_OK : TCL_ERROR; } diff --git a/src/test_multiplex.h b/src/test_multiplex.h index ec1ba9b..b7e1afe 100644 --- a/src/test_multiplex.h +++ b/src/test_multiplex.h @@ -46,6 +46,10 @@ #define MULTIPLEX_CTRL_SET_CHUNK_SIZE 214015 #define MULTIPLEX_CTRL_SET_MAX_CHUNKS 214016 +#ifdef __cplusplus +extern "C" { +#endif + /* ** CAPI: Initialize the multiplex VFS shim - sqlite3_multiplex_initialize() ** @@ -88,4 +92,8 @@ extern int sqlite3_multiplex_initialize(const char *zOrigVfsName, int makeDefaul */ extern int sqlite3_multiplex_shutdown(void); +#ifdef __cplusplus +} /* End of the 'extern "C"' block */ +#endif + #endif diff --git a/src/test_mutex.c b/src/test_mutex.c index 0bb7437..c9b4a29 100644 --- a/src/test_mutex.c +++ b/src/test_mutex.c @@ -19,8 +19,8 @@ #include <assert.h> #include <string.h> -/* defined in test1.c */ -const char *sqlite3TestErrorName(int); +/* defined in main.c */ +extern const char *sqlite3ErrName(int); /* A countable mutex */ struct sqlite3_mutex { @@ -148,7 +148,7 @@ static int test_shutdown( } rc = sqlite3_shutdown(); - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_VOLATILE); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_VOLATILE); return TCL_OK; } @@ -169,7 +169,7 @@ static int test_initialize( } rc = sqlite3_initialize(); - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_VOLATILE); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_VOLATILE); return TCL_OK; } @@ -230,7 +230,7 @@ static int test_install_mutex_counters( g.isInstalled = isInstall; } - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_VOLATILE); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_VOLATILE); return TCL_OK; } @@ -354,7 +354,7 @@ static int test_config( } rc = sqlite3_config(i); - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_VOLATILE); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_VOLATILE); return TCL_OK; } diff --git a/src/test_quota.c b/src/test_quota.c index 2ce46ac..e590996 100644 --- a/src/test_quota.c +++ b/src/test_quota.c @@ -1073,7 +1073,7 @@ size_t sqlite3_quota_fwrite( /* If the write was incomplete, adjust the file size and group size ** downward */ if( rc<nmemb && pFile ){ - size_t nWritten = rc>=0 ? rc : 0; + size_t nWritten = rc; sqlite3_int64 iNewEnd = iOfst + size*nWritten; if( iNewEnd<iEnd ) iNewEnd = iEnd; quotaEnter(); @@ -1179,7 +1179,13 @@ int sqlite3_quota_ftruncate(quota_FILE *p, sqlite3_int64 szNew){ rc = ftruncate(fileno(p->f), szNew); #endif #if SQLITE_OS_WIN - rc = _chsize_s(_fileno(p->f), szNew); +# if defined(__MINGW32__) && defined(SQLITE_TEST) + /* _chsize_s() is missing from MingW (as of 2012-11-06). Use + ** _chsize() as a work-around for testing purposes. */ + rc = _chsize(_fileno(p->f), (long)szNew); +# else + rc = _chsize_s(_fileno(p->f), szNew); +# endif #endif if( pFile && rc==0 ){ quotaGroup *pGroup = pFile->pGroup; @@ -1289,7 +1295,7 @@ int sqlite3_quota_remove(const char *zFilename){ if( pGroup ){ for(pFile=pGroup->pFiles; pFile && rc==SQLITE_OK; pFile=pNextFile){ pNextFile = pFile->pNext; - diff = memcmp(zFull, pFile->zFilename, nFull); + diff = strncmp(zFull, pFile->zFilename, nFull); if( diff==0 && ((c = pFile->zFilename[nFull])==0 || c=='/' || c=='\\') ){ if( pFile->nRef ){ pFile->deleteOnClose = 1; @@ -1319,7 +1325,7 @@ struct TclQuotaCallback { Tcl_Obj *pScript; /* Script to be run */ }; -extern const char *sqlite3TestErrorName(int); +extern const char *sqlite3ErrName(int); /* @@ -1354,8 +1360,10 @@ static void tclQuotaCallback( rc = Tcl_EvalObjEx(p->interp, pEval, TCL_EVAL_GLOBAL); if( rc==TCL_OK ){ + Tcl_WideInt x; Tcl_Obj *pLimit = Tcl_ObjGetVar2(p->interp, pVarname, 0, 0); - rc = Tcl_GetWideIntFromObj(p->interp, pLimit, piLimit); + rc = Tcl_GetWideIntFromObj(p->interp, pLimit, &x); + *piLimit = x; Tcl_UnsetVar(p->interp, Tcl_GetString(pVarname), 0); } @@ -1399,7 +1407,7 @@ static int test_quota_initialize( /* Call sqlite3_quota_initialize() */ rc = sqlite3_quota_initialize(zName, makeDefault); - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_STATIC); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_STATIC); return TCL_OK; } @@ -1422,7 +1430,7 @@ static int test_quota_shutdown( /* Call sqlite3_quota_shutdown() */ rc = sqlite3_quota_shutdown(); - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_STATIC); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_STATIC); return TCL_OK; } @@ -1437,7 +1445,7 @@ static int test_quota_set( Tcl_Obj *CONST objv[] ){ const char *zPattern; /* File pattern to configure */ - sqlite3_int64 iLimit; /* Initial quota in bytes */ + Tcl_WideInt iLimit; /* Initial quota in bytes */ Tcl_Obj *pScript; /* Tcl script to invoke to increase quota */ int rc; /* Value returned by quota_set() */ TclQuotaCallback *p; /* Callback object */ @@ -1477,7 +1485,7 @@ static int test_quota_set( /* Invoke sqlite3_quota_set() */ rc = sqlite3_quota_set(zPattern, iLimit, xCallback, (void*)p, xDestroy); - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_STATIC); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_STATIC); return TCL_OK; } @@ -1503,7 +1511,7 @@ static int test_quota_file( /* Invoke sqlite3_quota_file() */ rc = sqlite3_quota_file(zFilename); - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_STATIC); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_STATIC); return TCL_OK; } @@ -1613,7 +1621,6 @@ static int test_quota_fread( return TCL_ERROR; } got = sqlite3_quota_fread(zBuf, sz, nElem, p); - if( got<0 ) got = 0; zBuf[got*sz] = 0; Tcl_SetResult(interp, zBuf, TCL_VOLATILE); sqlite3_free(zBuf); diff --git a/src/test_rtree.c b/src/test_rtree.c index d3c9e0c..f54ae9b 100644 --- a/src/test_rtree.c +++ b/src/test_rtree.c @@ -254,7 +254,7 @@ static int register_cube_geom( UNUSED_PARAMETER(objv); #else extern int getDbPointer(Tcl_Interp*, const char*, sqlite3**); - extern const char *sqlite3TestErrorName(int); + extern const char *sqlite3ErrName(int); sqlite3 *db; int rc; @@ -264,7 +264,7 @@ static int register_cube_geom( } if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; rc = sqlite3_rtree_geometry_callback(db, "cube", cube_geom, (void *)&gHere); - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_STATIC); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_STATIC); #endif return TCL_OK; } @@ -282,7 +282,7 @@ static int register_circle_geom( UNUSED_PARAMETER(objv); #else extern int getDbPointer(Tcl_Interp*, const char*, sqlite3**); - extern const char *sqlite3TestErrorName(int); + extern const char *sqlite3ErrName(int); sqlite3 *db; int rc; @@ -292,7 +292,7 @@ static int register_circle_geom( } if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; rc = sqlite3_rtree_geometry_callback(db, "circle", circle_geom, 0); - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_STATIC); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_STATIC); #endif return TCL_OK; } diff --git a/src/test_spellfix.c b/src/test_spellfix.c deleted file mode 100644 index 3f21d73..0000000 --- a/src/test_spellfix.c +++ /dev/null @@ -1,2830 +0,0 @@ -/* -** 2012 April 10 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** This module implements the spellfix1 VIRTUAL TABLE that can be used -** to search a large vocabulary for close matches. See separate -** documentation files (spellfix1.wiki and editdist3.wiki) for details. -*/ -#if SQLITE_CORE -# include "sqliteInt.h" -#else -# include <string.h> -# include <stdio.h> -# include <stdlib.h> -# include "sqlite3ext.h" - SQLITE_EXTENSION_INIT1 -#endif /* !SQLITE_CORE */ -#include <ctype.h> - -/* -** Character classes for ASCII characters: -** -** 0 '' Silent letters: H W -** 1 'A' Any vowel: A E I O U (Y) -** 2 'B' A bilabeal stop or fricative: B F P V W -** 3 'C' Other fricatives or back stops: C G J K Q S X Z -** 4 'D' Alveolar stops: D T -** 5 'H' Letter H at the beginning of a word -** 6 'L' Glide: L -** 7 'R' Semivowel: R -** 8 'M' Nasals: M N -** 9 'Y' Letter Y at the beginning of a word. -** 10 '9' Digits: 0 1 2 3 4 5 6 7 8 9 -** 11 ' ' White space -** 12 '?' Other. -*/ -#define CCLASS_SILENT 0 -#define CCLASS_VOWEL 1 -#define CCLASS_B 2 -#define CCLASS_C 3 -#define CCLASS_D 4 -#define CCLASS_H 5 -#define CCLASS_L 6 -#define CCLASS_R 7 -#define CCLASS_M 8 -#define CCLASS_Y 9 -#define CCLASS_DIGIT 10 -#define CCLASS_SPACE 11 -#define CCLASS_OTHER 12 - -/* -** The following table gives the character class for non-initial ASCII -** characters. -*/ -static const unsigned char midClass[] = { - /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, - /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, - /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, - /* */ CCLASS_SPACE, /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, - /* */ CCLASS_SPACE, /* */ CCLASS_SPACE, /* */ CCLASS_OTHER, - /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, - /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, - /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, - /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, - /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, - /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, /* */ CCLASS_SPACE, - /* ! */ CCLASS_OTHER, /* " */ CCLASS_OTHER, /* # */ CCLASS_OTHER, - /* $ */ CCLASS_OTHER, /* % */ CCLASS_OTHER, /* & */ CCLASS_OTHER, - /* ' */ CCLASS_SILENT, /* ( */ CCLASS_OTHER, /* ) */ CCLASS_OTHER, - /* * */ CCLASS_OTHER, /* + */ CCLASS_OTHER, /* , */ CCLASS_OTHER, - /* - */ CCLASS_OTHER, /* . */ CCLASS_OTHER, /* / */ CCLASS_OTHER, - /* 0 */ CCLASS_DIGIT, /* 1 */ CCLASS_DIGIT, /* 2 */ CCLASS_DIGIT, - /* 3 */ CCLASS_DIGIT, /* 4 */ CCLASS_DIGIT, /* 5 */ CCLASS_DIGIT, - /* 6 */ CCLASS_DIGIT, /* 7 */ CCLASS_DIGIT, /* 8 */ CCLASS_DIGIT, - /* 9 */ CCLASS_DIGIT, /* : */ CCLASS_OTHER, /* ; */ CCLASS_OTHER, - /* < */ CCLASS_OTHER, /* = */ CCLASS_OTHER, /* > */ CCLASS_OTHER, - /* ? */ CCLASS_OTHER, /* @ */ CCLASS_OTHER, /* A */ CCLASS_VOWEL, - /* B */ CCLASS_B, /* C */ CCLASS_C, /* D */ CCLASS_D, - /* E */ CCLASS_VOWEL, /* F */ CCLASS_B, /* G */ CCLASS_C, - /* H */ CCLASS_SILENT, /* I */ CCLASS_VOWEL, /* J */ CCLASS_C, - /* K */ CCLASS_C, /* L */ CCLASS_L, /* M */ CCLASS_M, - /* N */ CCLASS_M, /* O */ CCLASS_VOWEL, /* P */ CCLASS_B, - /* Q */ CCLASS_C, /* R */ CCLASS_R, /* S */ CCLASS_C, - /* T */ CCLASS_D, /* U */ CCLASS_VOWEL, /* V */ CCLASS_B, - /* W */ CCLASS_B, /* X */ CCLASS_C, /* Y */ CCLASS_VOWEL, - /* Z */ CCLASS_C, /* [ */ CCLASS_OTHER, /* \ */ CCLASS_OTHER, - /* ] */ CCLASS_OTHER, /* ^ */ CCLASS_OTHER, /* _ */ CCLASS_OTHER, - /* ` */ CCLASS_OTHER, /* a */ CCLASS_VOWEL, /* b */ CCLASS_B, - /* c */ CCLASS_C, /* d */ CCLASS_D, /* e */ CCLASS_VOWEL, - /* f */ CCLASS_B, /* g */ CCLASS_C, /* h */ CCLASS_SILENT, - /* i */ CCLASS_VOWEL, /* j */ CCLASS_C, /* k */ CCLASS_C, - /* l */ CCLASS_L, /* m */ CCLASS_M, /* n */ CCLASS_M, - /* o */ CCLASS_VOWEL, /* p */ CCLASS_B, /* q */ CCLASS_C, - /* r */ CCLASS_R, /* s */ CCLASS_C, /* t */ CCLASS_D, - /* u */ CCLASS_VOWEL, /* v */ CCLASS_B, /* w */ CCLASS_B, - /* x */ CCLASS_C, /* y */ CCLASS_VOWEL, /* z */ CCLASS_C, - /* { */ CCLASS_OTHER, /* | */ CCLASS_OTHER, /* } */ CCLASS_OTHER, - /* ~ */ CCLASS_OTHER, /* */ CCLASS_OTHER, -}; -/* -** This tables gives the character class for ASCII characters that form the -** initial character of a word. The only difference from midClass is with -** the letters H, W, and Y. -*/ -static const unsigned char initClass[] = { - /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, - /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, - /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, - /* */ CCLASS_SPACE, /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, - /* */ CCLASS_SPACE, /* */ CCLASS_SPACE, /* */ CCLASS_OTHER, - /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, - /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, - /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, - /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, - /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, - /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, /* */ CCLASS_SPACE, - /* ! */ CCLASS_OTHER, /* " */ CCLASS_OTHER, /* # */ CCLASS_OTHER, - /* $ */ CCLASS_OTHER, /* % */ CCLASS_OTHER, /* & */ CCLASS_OTHER, - /* ' */ CCLASS_OTHER, /* ( */ CCLASS_OTHER, /* ) */ CCLASS_OTHER, - /* * */ CCLASS_OTHER, /* + */ CCLASS_OTHER, /* , */ CCLASS_OTHER, - /* - */ CCLASS_OTHER, /* . */ CCLASS_OTHER, /* / */ CCLASS_OTHER, - /* 0 */ CCLASS_DIGIT, /* 1 */ CCLASS_DIGIT, /* 2 */ CCLASS_DIGIT, - /* 3 */ CCLASS_DIGIT, /* 4 */ CCLASS_DIGIT, /* 5 */ CCLASS_DIGIT, - /* 6 */ CCLASS_DIGIT, /* 7 */ CCLASS_DIGIT, /* 8 */ CCLASS_DIGIT, - /* 9 */ CCLASS_DIGIT, /* : */ CCLASS_OTHER, /* ; */ CCLASS_OTHER, - /* < */ CCLASS_OTHER, /* = */ CCLASS_OTHER, /* > */ CCLASS_OTHER, - /* ? */ CCLASS_OTHER, /* @ */ CCLASS_OTHER, /* A */ CCLASS_VOWEL, - /* B */ CCLASS_B, /* C */ CCLASS_C, /* D */ CCLASS_D, - /* E */ CCLASS_VOWEL, /* F */ CCLASS_B, /* G */ CCLASS_C, - /* H */ CCLASS_SILENT, /* I */ CCLASS_VOWEL, /* J */ CCLASS_C, - /* K */ CCLASS_C, /* L */ CCLASS_L, /* M */ CCLASS_M, - /* N */ CCLASS_M, /* O */ CCLASS_VOWEL, /* P */ CCLASS_B, - /* Q */ CCLASS_C, /* R */ CCLASS_R, /* S */ CCLASS_C, - /* T */ CCLASS_D, /* U */ CCLASS_VOWEL, /* V */ CCLASS_B, - /* W */ CCLASS_B, /* X */ CCLASS_C, /* Y */ CCLASS_Y, - /* Z */ CCLASS_C, /* [ */ CCLASS_OTHER, /* \ */ CCLASS_OTHER, - /* ] */ CCLASS_OTHER, /* ^ */ CCLASS_OTHER, /* _ */ CCLASS_OTHER, - /* ` */ CCLASS_OTHER, /* a */ CCLASS_VOWEL, /* b */ CCLASS_B, - /* c */ CCLASS_C, /* d */ CCLASS_D, /* e */ CCLASS_VOWEL, - /* f */ CCLASS_B, /* g */ CCLASS_C, /* h */ CCLASS_SILENT, - /* i */ CCLASS_VOWEL, /* j */ CCLASS_C, /* k */ CCLASS_C, - /* l */ CCLASS_L, /* m */ CCLASS_M, /* n */ CCLASS_M, - /* o */ CCLASS_VOWEL, /* p */ CCLASS_B, /* q */ CCLASS_C, - /* r */ CCLASS_R, /* s */ CCLASS_C, /* t */ CCLASS_D, - /* u */ CCLASS_VOWEL, /* v */ CCLASS_B, /* w */ CCLASS_B, - /* x */ CCLASS_C, /* y */ CCLASS_Y, /* z */ CCLASS_C, - /* { */ CCLASS_OTHER, /* | */ CCLASS_OTHER, /* } */ CCLASS_OTHER, - /* ~ */ CCLASS_OTHER, /* */ CCLASS_OTHER, -}; - -/* -** Mapping from the character class number (0-13) to a symbol for each -** character class. Note that initClass[] can be used to map the class -** symbol back into the class number. -*/ -static const unsigned char className[] = ".ABCDHLRMY9 ?"; - -/* -** Generate a "phonetic hash" from a string of ASCII characters -** in zIn[0..nIn-1]. -** -** * Map characters by character class as defined above. -** * Omit double-letters -** * Omit vowels beside R and L -** * Omit T when followed by CH -** * Omit W when followed by R -** * Omit D when followed by J or G -** * Omit K in KN or G in GN at the beginning of a word -** -** Space to hold the result is obtained from sqlite3_malloc() -** -** Return NULL if memory allocation fails. -*/ -static unsigned char *phoneticHash(const unsigned char *zIn, int nIn){ - unsigned char *zOut = sqlite3_malloc( nIn + 1 ); - int i; - int nOut = 0; - char cPrev = 0x77; - char cPrevX = 0x77; - const unsigned char *aClass = initClass; - - if( zOut==0 ) return 0; - if( nIn>2 ){ - switch( zIn[0] ){ - case 'g': - case 'k': { - if( zIn[1]=='n' ){ zIn++; nIn--; } - break; - } - } - } - for(i=0; i<nIn; i++){ - unsigned char c = zIn[i]; - if( i+1<nIn ){ - if( c=='w' && zIn[i+1]=='r' ) continue; - if( c=='d' && (zIn[i+1]=='j' || zIn[i+1]=='g') ) continue; - if( i+2<nIn ){ - if( c=='t' && zIn[i+1]=='c' && zIn[i+2]=='h' ) continue; - } - } - c = aClass[c&0x7f]; - if( c==CCLASS_SPACE ) continue; - if( c==CCLASS_OTHER && cPrev!=CCLASS_DIGIT ) continue; - aClass = midClass; - if( c==CCLASS_VOWEL && (cPrevX==CCLASS_R || cPrevX==CCLASS_L) ){ - continue; /* No vowels beside L or R */ - } - if( (c==CCLASS_R || c==CCLASS_L) && cPrevX==CCLASS_VOWEL ){ - nOut--; /* No vowels beside L or R */ - } - cPrev = c; - if( c==CCLASS_SILENT ) continue; - cPrevX = c; - c = className[c]; - assert( nOut>=0 ); - if( nOut==0 || c!=zOut[nOut-1] ) zOut[nOut++] = c; - } - zOut[nOut] = 0; - return zOut; -} - -/* -** This is an SQL function wrapper around phoneticHash(). See -** the description of phoneticHash() for additional information. -*/ -static void phoneticHashSqlFunc( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - const unsigned char *zIn; - unsigned char *zOut; - - zIn = sqlite3_value_text(argv[0]); - if( zIn==0 ) return; - zOut = phoneticHash(zIn, sqlite3_value_bytes(argv[0])); - if( zOut==0 ){ - sqlite3_result_error_nomem(context); - }else{ - sqlite3_result_text(context, (char*)zOut, -1, sqlite3_free); - } -} - -/* -** Return the character class number for a character given its -** context. -*/ -static char characterClass(char cPrev, char c){ - return cPrev==0 ? initClass[c&0x7f] : midClass[c&0x7f]; -} - -/* -** Return the cost of inserting or deleting character c immediately -** following character cPrev. If cPrev==0, that means c is the first -** character of the word. -*/ -static int insertOrDeleteCost(char cPrev, char c, char cNext){ - char classC = characterClass(cPrev, c); - char classCprev; - - if( classC==CCLASS_SILENT ){ - /* Insert or delete "silent" characters such as H or W */ - return 1; - } - if( cPrev==c ){ - /* Repeated characters, or miss a repeat */ - return 10; - } - if( classC==CCLASS_VOWEL && (cPrev=='r' || cNext=='r') ){ - return 20; /* Insert a vowel before or after 'r' */ - } - classCprev = characterClass(cPrev, cPrev); - if( classC==classCprev ){ - if( classC==CCLASS_VOWEL ){ - /* Remove or add a new vowel to a vowel cluster */ - return 15; - }else{ - /* Remove or add a consonant not in the same class */ - return 50; - } - } - - /* any other character insertion or deletion */ - return 100; -} - -/* -** Divide the insertion cost by this factor when appending to the -** end of the word. -*/ -#define FINAL_INS_COST_DIV 4 - -/* -** Return the cost of substituting cTo in place of cFrom assuming -** the previous character is cPrev. If cPrev==0 then cTo is the first -** character of the word. -*/ -static int substituteCost(char cPrev, char cFrom, char cTo){ - char classFrom, classTo; - if( cFrom==cTo ){ - /* Exact match */ - return 0; - } - if( cFrom==(cTo^0x20) && ((cTo>='A' && cTo<='Z') || (cTo>='a' && cTo<='z')) ){ - /* differ only in case */ - return 0; - } - classFrom = characterClass(cPrev, cFrom); - classTo = characterClass(cPrev, cTo); - if( classFrom==classTo ){ - /* Same character class */ - return 40; - } - if( classFrom>=CCLASS_B && classFrom<=CCLASS_Y - && classTo>=CCLASS_B && classTo<=CCLASS_Y ){ - /* Convert from one consonant to another, but in a different class */ - return 75; - } - /* Any other subsitution */ - return 100; -} - -/* -** Given two strings zA and zB which are pure ASCII, return the cost -** of transforming zA into zB. If zA ends with '*' assume that it is -** a prefix of zB and give only minimal penalty for extra characters -** on the end of zB. -** -** Smaller numbers mean a closer match. -** -** Negative values indicate an error: -** -1 One of the inputs is NULL -** -2 Non-ASCII characters on input -** -3 Unable to allocate memory -** -** If pnMatch is not NULL, then *pnMatch is set to the number of bytes -** of zB that matched the pattern in zA. If zA does not end with a '*', -** then this value is always the number of bytes in zB (i.e. strlen(zB)). -** If zA does end in a '*', then it is the number of bytes in the prefix -** of zB that was deemed to match zA. -*/ -static int editdist1(const char *zA, const char *zB, int *pnMatch){ - int nA, nB; /* Number of characters in zA[] and zB[] */ - int xA, xB; /* Loop counters for zA[] and zB[] */ - char cA, cB; /* Current character of zA and zB */ - char cAprev, cBprev; /* Previous character of zA and zB */ - char cAnext, cBnext; /* Next character in zA and zB */ - int d; /* North-west cost value */ - int dc = 0; /* North-west character value */ - int res; /* Final result */ - int *m; /* The cost matrix */ - char *cx; /* Corresponding character values */ - int *toFree = 0; /* Malloced space */ - int mStack[60+15]; /* Stack space to use if not too much is needed */ - int nMatch = 0; - - /* Early out if either input is NULL */ - if( zA==0 || zB==0 ) return -1; - - /* Skip any common prefix */ - while( zA[0] && zA[0]==zB[0] ){ dc = zA[0]; zA++; zB++; nMatch++; } - if( pnMatch ) *pnMatch = nMatch; - if( zA[0]==0 && zB[0]==0 ) return 0; - -#if 0 - printf("A=\"%s\" B=\"%s\" dc=%c\n", zA, zB, dc?dc:' '); -#endif - - /* Verify input strings and measure their lengths */ - for(nA=0; zA[nA]; nA++){ - if( zA[nA]&0x80 ) return -2; - } - for(nB=0; zB[nB]; nB++){ - if( zB[nB]&0x80 ) return -2; - } - - /* Special processing if either string is empty */ - if( nA==0 ){ - cBprev = dc; - for(xB=res=0; (cB = zB[xB])!=0; xB++){ - res += insertOrDeleteCost(cBprev, cB, zB[xB+1])/FINAL_INS_COST_DIV; - cBprev = cB; - } - return res; - } - if( nB==0 ){ - cAprev = dc; - for(xA=res=0; (cA = zA[xA])!=0; xA++){ - res += insertOrDeleteCost(cAprev, cA, zA[xA+1]); - cAprev = cA; - } - return res; - } - - /* A is a prefix of B */ - if( zA[0]=='*' && zA[1]==0 ) return 0; - - /* Allocate and initialize the Wagner matrix */ - if( nB<(sizeof(mStack)*4)/(sizeof(mStack[0])*5) ){ - m = mStack; - }else{ - m = toFree = sqlite3_malloc( (nB+1)*5*sizeof(m[0])/4 ); - if( m==0 ) return -3; - } - cx = (char*)&m[nB+1]; - - /* Compute the Wagner edit distance */ - m[0] = 0; - cx[0] = dc; - cBprev = dc; - for(xB=1; xB<=nB; xB++){ - cBnext = zB[xB]; - cB = zB[xB-1]; - cx[xB] = cB; - m[xB] = m[xB-1] + insertOrDeleteCost(cBprev, cB, cBnext); - cBprev = cB; - } - cAprev = dc; - for(xA=1; xA<=nA; xA++){ - int lastA = (xA==nA); - cA = zA[xA-1]; - cAnext = zA[xA]; - if( cA=='*' && lastA ) break; - d = m[0]; - dc = cx[0]; - m[0] = d + insertOrDeleteCost(cAprev, cA, cAnext); - cBprev = 0; - for(xB=1; xB<=nB; xB++){ - int totalCost, insCost, delCost, subCost, ncx; - cB = zB[xB-1]; - cBnext = zB[xB]; - - /* Cost to insert cB */ - insCost = insertOrDeleteCost(cx[xB-1], cB, cBnext); - if( lastA ) insCost /= FINAL_INS_COST_DIV; - - /* Cost to delete cA */ - delCost = insertOrDeleteCost(cx[xB], cA, cBnext); - - /* Cost to substitute cA->cB */ - subCost = substituteCost(cx[xB-1], cA, cB); - - /* Best cost */ - totalCost = insCost + m[xB-1]; - ncx = cB; - if( (delCost + m[xB])<totalCost ){ - totalCost = delCost + m[xB]; - ncx = cA; - } - if( (subCost + d)<totalCost ){ - totalCost = subCost + d; - } - -#if 0 - printf("%d,%d d=%4d u=%4d r=%4d dc=%c cA=%c cB=%c" - " ins=%4d del=%4d sub=%4d t=%4d ncx=%c\n", - xA, xB, d, m[xB], m[xB-1], dc?dc:' ', cA, cB, - insCost, delCost, subCost, totalCost, ncx?ncx:' '); -#endif - - /* Update the matrix */ - d = m[xB]; - dc = cx[xB]; - m[xB] = totalCost; - cx[xB] = ncx; - cBprev = cB; - } - cAprev = cA; - } - - /* Free the wagner matrix and return the result */ - if( cA=='*' ){ - res = m[1]; - for(xB=1; xB<=nB; xB++){ - if( m[xB]<res ){ - res = m[xB]; - if( pnMatch ) *pnMatch = xB+nMatch; - } - } - }else{ - res = m[nB]; - /* In the current implementation, pnMatch is always NULL if zA does - ** not end in "*" */ - assert( pnMatch==0 ); - } - sqlite3_free(toFree); - return res; -} - -/* -** Function: editdist(A,B) -** -** Return the cost of transforming string A into string B. Both strings -** must be pure ASCII text. If A ends with '*' then it is assumed to be -** a prefix of B and extra characters on the end of B have minimal additional -** cost. -*/ -static void editdistSqlFunc( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - int res = editdist1( - (const char*)sqlite3_value_text(argv[0]), - (const char*)sqlite3_value_text(argv[1]), - 0); - if( res<0 ){ - if( res==(-3) ){ - sqlite3_result_error_nomem(context); - }else if( res==(-2) ){ - sqlite3_result_error(context, "non-ASCII input to editdist()", -1); - }else{ - sqlite3_result_error(context, "NULL input to editdist()", -1); - } - }else{ - sqlite3_result_int(context, res); - } -} - -/* End of the fixed-cost edit distance implementation -****************************************************************************** -***************************************************************************** -** Begin: Configurable cost unicode edit distance routines -*/ -/* Forward declaration of structures */ -typedef struct EditDist3Cost EditDist3Cost; -typedef struct EditDist3Config EditDist3Config; -typedef struct EditDist3Point EditDist3Point; -typedef struct EditDist3From EditDist3From; -typedef struct EditDist3FromString EditDist3FromString; -typedef struct EditDist3To EditDist3To; -typedef struct EditDist3ToString EditDist3ToString; -typedef struct EditDist3Lang EditDist3Lang; - - -/* -** An entry in the edit cost table -*/ -struct EditDist3Cost { - EditDist3Cost *pNext; /* Next cost element */ - u8 nFrom; /* Number of bytes in aFrom */ - u8 nTo; /* Number of bytes in aTo */ - u16 iCost; /* Cost of this transformation */ - char a[4] ; /* FROM string followed by TO string */ - /* Additional TO and FROM string bytes appended as necessary */ -}; - -/* -** Edit costs for a particular language ID -*/ -struct EditDist3Lang { - int iLang; /* Language ID */ - int iInsCost; /* Default insertion cost */ - int iDelCost; /* Default deletion cost */ - int iSubCost; /* Default substitution cost */ - EditDist3Cost *pCost; /* Costs */ -}; - - -/* -** The default EditDist3Lang object, with default costs. -*/ -static const EditDist3Lang editDist3Lang = { 0, 100, 100, 150, 0 }; - -/* -** Complete configuration -*/ -struct EditDist3Config { - int nLang; /* Number of language IDs. Size of a[] */ - EditDist3Lang *a; /* One for each distinct language ID */ -}; - -/* -** Extra information about each character in the FROM string. -*/ -struct EditDist3From { - int nSubst; /* Number of substitution cost entries */ - int nDel; /* Number of deletion cost entries */ - int nByte; /* Number of bytes in this character */ - EditDist3Cost **apSubst; /* Array of substitution costs for this element */ - EditDist3Cost **apDel; /* Array of deletion cost entries */ -}; - -/* -** A precompiled FROM string. -* -** In the common case we expect the FROM string to be reused multiple times. -** In other words, the common case will be to measure the edit distance -** from a single origin string to multiple target strings. -*/ -struct EditDist3FromString { - char *z; /* The complete text of the FROM string */ - int n; /* Number of characters in the FROM string */ - int isPrefix; /* True if ends with '*' character */ - EditDist3From *a; /* Extra info about each char of the FROM string */ -}; - -/* -** Extra information about each character in the TO string. -*/ -struct EditDist3To { - int nIns; /* Number of insertion cost entries */ - int nByte; /* Number of bytes in this character */ - EditDist3Cost **apIns; /* Array of deletion cost entries */ -}; - -/* -** A precompiled FROM string -*/ -struct EditDist3ToString { - char *z; /* The complete text of the TO string */ - int n; /* Number of characters in the TO string */ - EditDist3To *a; /* Extra info about each char of the TO string */ -}; - -/* -** Clear or delete an instance of the object that records all edit-distance -** weights. -*/ -static void editDist3ConfigClear(EditDist3Config *p){ - int i; - if( p==0 ) return; - for(i=0; i<p->nLang; i++){ - EditDist3Cost *pCost, *pNext; - pCost = p->a[i].pCost; - while( pCost ){ - pNext = pCost->pNext; - sqlite3_free(pCost); - pCost = pNext; - } - } - sqlite3_free(p->a); - memset(p, 0, sizeof(*p)); -} -static void editDist3ConfigDelete(void *pIn){ - EditDist3Config *p = (EditDist3Config*)pIn; - editDist3ConfigClear(p); - sqlite3_free(p); -} - -/* -** Load all edit-distance weights from a table. -*/ -static int editDist3ConfigLoad( - EditDist3Config *p, /* The edit distance configuration to load */ - sqlite3 *db, /* Load from this database */ - const char *zTable /* Name of the table from which to load */ -){ - sqlite3_stmt *pStmt; - int rc, rc2; - char *zSql; - int iLangPrev = -9999; - EditDist3Lang *pLang = 0; - - zSql = sqlite3_mprintf("SELECT iLang, cFrom, cTo, iCost" - " FROM \"%w\" WHERE iLang>=0 ORDER BY iLang", zTable); - if( zSql==0 ) return SQLITE_NOMEM; - rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); - sqlite3_free(zSql); - if( rc ) return rc; - editDist3ConfigClear(p); - while( sqlite3_step(pStmt)==SQLITE_ROW ){ - int iLang = sqlite3_column_int(pStmt, 0); - const char *zFrom = (const char*)sqlite3_column_text(pStmt, 1); - int nFrom = zFrom ? sqlite3_column_bytes(pStmt, 1) : 0; - const char *zTo = (const char*)sqlite3_column_text(pStmt, 2); - int nTo = zTo ? sqlite3_column_bytes(pStmt, 2) : 0; - int iCost = sqlite3_column_int(pStmt, 3); - - assert( zFrom!=0 || nFrom==0 ); - assert( zTo!=0 || nTo==0 ); - if( nFrom>100 || nTo>100 ) continue; - if( iCost<0 ) continue; - if( pLang==0 || iLang!=iLangPrev ){ - EditDist3Lang *pNew; - pNew = sqlite3_realloc(p->a, (p->nLang+1)*sizeof(p->a[0])); - if( pNew==0 ){ rc = SQLITE_NOMEM; break; } - p->a = pNew; - pLang = &p->a[p->nLang]; - p->nLang++; - pLang->iLang = iLang; - pLang->iInsCost = 100; - pLang->iDelCost = 100; - pLang->iSubCost = 150; - pLang->pCost = 0; - iLangPrev = iLang; - } - if( nFrom==1 && zFrom[0]=='?' && nTo==0 ){ - pLang->iDelCost = iCost; - }else if( nFrom==0 && nTo==1 && zTo[0]=='?' ){ - pLang->iInsCost = iCost; - }else if( nFrom==1 && nTo==1 && zFrom[0]=='?' && zTo[0]=='?' ){ - pLang->iSubCost = iCost; - }else{ - EditDist3Cost *pCost; - int nExtra = nFrom + nTo - 4; - if( nExtra<0 ) nExtra = 0; - pCost = sqlite3_malloc( sizeof(*pCost) + nExtra ); - if( pCost==0 ){ rc = SQLITE_NOMEM; break; } - pCost->nFrom = nFrom; - pCost->nTo = nTo; - pCost->iCost = iCost; - memcpy(pCost->a, zFrom, nFrom); - memcpy(pCost->a + nFrom, zTo, nTo); - pCost->pNext = pLang->pCost; - pLang->pCost = pCost; - } - } - rc2 = sqlite3_finalize(pStmt); - if( rc==SQLITE_OK ) rc = rc2; - return rc; -} - -/* -** Return the length (in bytes) of a utf-8 character. Or return a maximum -** of N. -*/ -static int utf8Len(unsigned char c, int N){ - int len = 1; - if( c>0x7f ){ - if( (c&0xe0)==0xc0 ){ - len = 2; - }else if( (c&0xf0)==0xe0 ){ - len = 3; - }else{ - len = 4; - } - } - if( len>N ) len = N; - return len; -} - -/* -** Return TRUE (non-zero) of the To side of the given cost matches -** the given string. -*/ -static int matchTo(EditDist3Cost *p, const char *z, int n){ - if( p->nTo>n ) return 0; - if( memcmp(p->a+p->nFrom, z, p->nTo)!=0 ) return 0; - return 1; -} - -/* -** Return TRUE (non-zero) of the To side of the given cost matches -** the given string. -*/ -static int matchFrom(EditDist3Cost *p, const char *z, int n){ - assert( p->nFrom<=n ); - if( memcmp(p->a, z, p->nFrom)!=0 ) return 0; - return 1; -} - -/* -** Return TRUE (non-zero) of the next FROM character and the next TO -** character are the same. -*/ -static int matchFromTo( - EditDist3FromString *pStr, /* Left hand string */ - int n1, /* Index of comparison character on the left */ - const char *z2, /* Right-handl comparison character */ - int n2 /* Bytes remaining in z2[] */ -){ - int b1 = pStr->a[n1].nByte; - if( b1>n2 ) return 0; - if( memcmp(pStr->z+n1, z2, b1)!=0 ) return 0; - return 1; -} - -/* -** Delete an EditDist3FromString objecct -*/ -static void editDist3FromStringDelete(EditDist3FromString *p){ - int i; - if( p ){ - for(i=0; i<p->n; i++){ - sqlite3_free(p->a[i].apDel); - sqlite3_free(p->a[i].apSubst); - } - sqlite3_free(p); - } -} - -/* -** Create a EditDist3FromString object. -*/ -static EditDist3FromString *editDist3FromStringNew( - const EditDist3Lang *pLang, - const char *z, - int n -){ - EditDist3FromString *pStr; - EditDist3Cost *p; - int i; - - if( z==0 ) return 0; - if( n<0 ) n = (int)strlen(z); - pStr = sqlite3_malloc( sizeof(*pStr) + sizeof(pStr->a[0])*n + n + 1 ); - if( pStr==0 ) return 0; - pStr->a = (EditDist3From*)&pStr[1]; - memset(pStr->a, 0, sizeof(pStr->a[0])*n); - pStr->n = n; - pStr->z = (char*)&pStr->a[n]; - memcpy(pStr->z, z, n+1); - if( n && z[n-1]=='*' ){ - pStr->isPrefix = 1; - n--; - pStr->n--; - pStr->z[n] = 0; - }else{ - pStr->isPrefix = 0; - } - - for(i=0; i<n; i++){ - EditDist3From *pFrom = &pStr->a[i]; - memset(pFrom, 0, sizeof(*pFrom)); - pFrom->nByte = utf8Len((unsigned char)z[i], n-i); - for(p=pLang->pCost; p; p=p->pNext){ - EditDist3Cost **apNew; - if( i+p->nFrom>n ) continue; - if( matchFrom(p, z+i, n-i)==0 ) continue; - if( p->nTo==0 ){ - apNew = sqlite3_realloc(pFrom->apDel, - sizeof(*apNew)*(pFrom->nDel+1)); - if( apNew==0 ) break; - pFrom->apDel = apNew; - apNew[pFrom->nDel++] = p; - }else{ - apNew = sqlite3_realloc(pFrom->apSubst, - sizeof(*apNew)*(pFrom->nSubst+1)); - if( apNew==0 ) break; - pFrom->apSubst = apNew; - apNew[pFrom->nSubst++] = p; - } - } - if( p ){ - editDist3FromStringDelete(pStr); - pStr = 0; - break; - } - } - return pStr; -} - -/* -** Update entry m[i] such that it is the minimum of its current value -** and m[j]+iCost. -** -** If the iCost is 1,000,000 or greater, then consider the cost to be -** infinite and skip the update. -*/ -static void updateCost( - unsigned int *m, - int i, - int j, - int iCost -){ - assert( iCost>=0 ); - if( iCost<10000 ){ - unsigned int b = m[j] + iCost; - if( b<m[i] ) m[i] = b; - } -} - -/* Compute the edit distance between two strings. -** -** If an error occurs, return a negative number which is the error code. -** -** If pnMatch is not NULL, then *pnMatch is set to the number of characters -** (not bytes) in z2 that matched the search pattern in *pFrom. If pFrom does -** not contain the pattern for a prefix-search, then this is always the number -** of characters in z2. If pFrom does contain a prefix search pattern, then -** it is the number of characters in the prefix of z2 that was deemed to -** match pFrom. -*/ -static int editDist3Core( - EditDist3FromString *pFrom, /* The FROM string */ - const char *z2, /* The TO string */ - int n2, /* Length of the TO string */ - const EditDist3Lang *pLang, /* Edit weights for a particular language ID */ - int *pnMatch /* OUT: Characters in matched prefix */ -){ - int k, n; - int i1, b1; - int i2, b2; - EditDist3FromString f = *pFrom; - EditDist3To *a2; - unsigned int *m; - int szRow; - EditDist3Cost *p; - int res; - - /* allocate the Wagner matrix and the aTo[] array for the TO string */ - n = (f.n+1)*(n2+1); - n = (n+1)&~1; - m = sqlite3_malloc( n*sizeof(m[0]) + sizeof(a2[0])*n2 ); - if( m==0 ) return -1; /* Out of memory */ - a2 = (EditDist3To*)&m[n]; - memset(a2, 0, sizeof(a2[0])*n2); - - /* Fill in the a1[] matrix for all characters of the TO string */ - for(i2=0; i2<n2; i2++){ - a2[i2].nByte = utf8Len((unsigned char)z2[i2], n2-i2); - for(p=pLang->pCost; p; p=p->pNext){ - EditDist3Cost **apNew; - if( p->nFrom>0 ) continue; - if( i2+p->nTo>n2 ) continue; - if( matchTo(p, z2+i2, n2-i2)==0 ) continue; - a2[i2].nIns++; - apNew = sqlite3_realloc(a2[i2].apIns, sizeof(*apNew)*a2[i2].nIns); - if( apNew==0 ){ - res = -1; /* Out of memory */ - goto editDist3Abort; - } - a2[i2].apIns = apNew; - a2[i2].apIns[a2[i2].nIns-1] = p; - } - } - - /* Prepare to compute the minimum edit distance */ - szRow = f.n+1; - memset(m, 0x01, (n2+1)*szRow*sizeof(m[0])); - m[0] = 0; - - /* First fill in the top-row of the matrix with FROM deletion costs */ - for(i1=0; i1<f.n; i1 += b1){ - b1 = f.a[i1].nByte; - updateCost(m, i1+b1, i1, pLang->iDelCost); - for(k=0; k<f.a[i1].nDel; k++){ - p = f.a[i1].apDel[k]; - updateCost(m, i1+p->nFrom, i1, p->iCost); - } - } - - /* Fill in all subsequent rows, top-to-bottom, left-to-right */ - for(i2=0; i2<n2; i2 += b2){ - int rx; /* Starting index for current row */ - int rxp; /* Starting index for previous row */ - b2 = a2[i2].nByte; - rx = szRow*(i2+b2); - rxp = szRow*i2; - updateCost(m, rx, rxp, pLang->iInsCost); - for(k=0; k<a2[i2].nIns; k++){ - p = a2[i2].apIns[k]; - updateCost(m, szRow*(i2+p->nTo), rxp, p->iCost); - } - for(i1=0; i1<f.n; i1+=b1){ - int cx; /* Index of current cell */ - int cxp; /* Index of cell immediately to the left */ - int cxd; /* Index of cell to the left and one row above */ - int cxu; /* Index of cell immediately above */ - b1 = f.a[i1].nByte; - cxp = rx + i1; - cx = cxp + b1; - cxd = rxp + i1; - cxu = cxd + b1; - updateCost(m, cx, cxp, pLang->iDelCost); - for(k=0; k<f.a[i1].nDel; k++){ - p = f.a[i1].apDel[k]; - updateCost(m, cxp+p->nFrom, cxp, p->iCost); - } - updateCost(m, cx, cxu, pLang->iInsCost); - if( matchFromTo(&f, i1, z2+i2, n2-i2) ){ - updateCost(m, cx, cxd, 0); - } - updateCost(m, cx, cxd, pLang->iSubCost); - for(k=0; k<f.a[i1].nSubst; k++){ - p = f.a[i1].apSubst[k]; - if( matchTo(p, z2+i2, n2-i2) ){ - updateCost(m, cxd+p->nFrom+szRow*p->nTo, cxd, p->iCost); - } - } - } - } - -#if 0 /* Enable for debugging */ - printf(" ^"); - for(i1=0; i1<f.n; i1++) printf(" %c-%2x", f.z[i1], f.z[i1]&0xff); - printf("\n ^:"); - for(i1=0; i1<szRow; i1++){ - int v = m[i1]; - if( v>9999 ) printf(" ****"); - else printf(" %4d", v); - } - printf("\n"); - for(i2=0; i2<n2; i2++){ - printf("%c-%02x:", z2[i2], z2[i2]&0xff); - for(i1=0; i1<szRow; i1++){ - int v = m[(i2+1)*szRow+i1]; - if( v>9999 ) printf(" ****"); - else printf(" %4d", v); - } - printf("\n"); - } -#endif - - /* Free memory allocations and return the result */ - res = (int)m[szRow*(n2+1)-1]; - n = n2; - if( f.isPrefix ){ - for(i2=1; i2<=n2; i2++){ - int b = m[szRow*i2-1]; - if( b<=res ){ - res = b; - n = i2 - 1; - } - } - } - if( pnMatch ){ - int nExtra = 0; - for(k=0; k<n; k++){ - if( (z2[k] & 0xc0)==0x80 ) nExtra++; - } - *pnMatch = n - nExtra; - } - -editDist3Abort: - for(i2=0; i2<n2; i2++) sqlite3_free(a2[i2].apIns); - sqlite3_free(m); - return res; -} - -/* -** Get an appropriate EditDist3Lang object. -*/ -static const EditDist3Lang *editDist3FindLang( - EditDist3Config *pConfig, - int iLang -){ - int i; - for(i=0; i<pConfig->nLang; i++){ - if( pConfig->a[i].iLang==iLang ) return &pConfig->a[i]; - } - return &editDist3Lang; -} - -/* -** Function: editdist3(A,B,iLang) -** editdist3(tablename) -** -** Return the cost of transforming string A into string B using edit -** weights for iLang. -** -** The second form loads edit weights into memory from a table. -*/ -static void editDist3SqlFunc( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - EditDist3Config *pConfig = (EditDist3Config*)sqlite3_user_data(context); - sqlite3 *db = sqlite3_context_db_handle(context); - int rc; - if( argc==1 ){ - const char *zTable = (const char*)sqlite3_value_text(argv[0]); - rc = editDist3ConfigLoad(pConfig, db, zTable); - if( rc ) sqlite3_result_error_code(context, rc); - }else{ - const char *zA = (const char*)sqlite3_value_text(argv[0]); - const char *zB = (const char*)sqlite3_value_text(argv[1]); - int nA = sqlite3_value_bytes(argv[0]); - int nB = sqlite3_value_bytes(argv[1]); - int iLang = argc==3 ? sqlite3_value_int(argv[2]) : 0; - const EditDist3Lang *pLang = editDist3FindLang(pConfig, iLang); - EditDist3FromString *pFrom; - int dist; - - pFrom = editDist3FromStringNew(pLang, zA, nA); - if( pFrom==0 ){ - sqlite3_result_error_nomem(context); - return; - } - dist = editDist3Core(pFrom, zB, nB, pLang, 0); - editDist3FromStringDelete(pFrom); - if( dist==(-1) ){ - sqlite3_result_error_nomem(context); - }else{ - sqlite3_result_int(context, dist); - } - } -} - -/* -** Register the editDist3 function with SQLite -*/ -static int editDist3Install(sqlite3 *db){ - int rc; - EditDist3Config *pConfig = sqlite3_malloc( sizeof(*pConfig) ); - if( pConfig==0 ) return SQLITE_NOMEM; - memset(pConfig, 0, sizeof(*pConfig)); - rc = sqlite3_create_function_v2(db, "editdist3", - 2, SQLITE_UTF8, pConfig, editDist3SqlFunc, 0, 0, 0); - if( rc==SQLITE_OK ){ - rc = sqlite3_create_function_v2(db, "editdist3", - 3, SQLITE_UTF8, pConfig, editDist3SqlFunc, 0, 0, 0); - } - if( rc==SQLITE_OK ){ - rc = sqlite3_create_function_v2(db, "editdist3", - 1, SQLITE_UTF8, pConfig, editDist3SqlFunc, 0, 0, - editDist3ConfigDelete); - }else{ - sqlite3_free(pConfig); - } - return rc; -} -/* End configurable cost unicode edit distance routines -****************************************************************************** -****************************************************************************** -** Begin transliterate unicode-to-ascii implementation -*/ - -#if !SQLITE_AMALGAMATION -/* -** This lookup table is used to help decode the first byte of -** a multi-byte UTF8 character. -*/ -static const unsigned char sqlite3Utf8Trans1[] = { - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, - 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, - 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x00, 0x00, -}; -#endif - -/* -** Return the value of the first UTF-8 character in the string. -*/ -static int utf8Read(const unsigned char *z, int n, int *pSize){ - int c, i; - - /* All callers to this routine (in the current implementation) - ** always have n>0. */ - if( NEVER(n==0) ){ - c = i = 0; - }else{ - c = z[0]; - i = 1; - if( c>=0xc0 ){ - c = sqlite3Utf8Trans1[c-0xc0]; - while( i<n && (z[i] & 0xc0)==0x80 ){ - c = (c<<6) + (0x3f & z[i++]); - } - } - } - *pSize = i; - return c; -} - -/* -** Return the number of characters in the utf-8 string in the nIn byte -** buffer pointed to by zIn. -*/ -static int utf8Charlen(const char *zIn, int nIn){ - int i; - int nChar = 0; - for(i=0; i<nIn; nChar++){ - int sz; - utf8Read((const unsigned char *)&zIn[i], nIn-i, &sz); - i += sz; - } - return nChar; -} - -/* -** Table of translations from unicode characters into ASCII. -*/ -static const struct { - unsigned short int cFrom; - unsigned char cTo0, cTo1; -} translit[] = { - { 0x00A0, 0x20, 0x00 }, /* to */ - { 0x00B5, 0x75, 0x00 }, /* µ to u */ - { 0x00C0, 0x41, 0x00 }, /* À to A */ - { 0x00C1, 0x41, 0x00 }, /* Á to A */ - { 0x00C2, 0x41, 0x00 }, /*  to A */ - { 0x00C3, 0x41, 0x00 }, /* à to A */ - { 0x00C4, 0x41, 0x65 }, /* Ä to Ae */ - { 0x00C5, 0x41, 0x61 }, /* Å to Aa */ - { 0x00C6, 0x41, 0x45 }, /* Æ to AE */ - { 0x00C7, 0x43, 0x00 }, /* Ç to C */ - { 0x00C8, 0x45, 0x00 }, /* È to E */ - { 0x00C9, 0x45, 0x00 }, /* É to E */ - { 0x00CA, 0x45, 0x00 }, /* Ê to E */ - { 0x00CB, 0x45, 0x00 }, /* Ë to E */ - { 0x00CC, 0x49, 0x00 }, /* Ì to I */ - { 0x00CD, 0x49, 0x00 }, /* Í to I */ - { 0x00CE, 0x49, 0x00 }, /* Î to I */ - { 0x00CF, 0x49, 0x00 }, /* Ï to I */ - { 0x00D0, 0x44, 0x00 }, /* Ð to D */ - { 0x00D1, 0x4E, 0x00 }, /* Ñ to N */ - { 0x00D2, 0x4F, 0x00 }, /* Ò to O */ - { 0x00D3, 0x4F, 0x00 }, /* Ó to O */ - { 0x00D4, 0x4F, 0x00 }, /* Ô to O */ - { 0x00D5, 0x4F, 0x00 }, /* Õ to O */ - { 0x00D6, 0x4F, 0x65 }, /* Ö to Oe */ - { 0x00D7, 0x78, 0x00 }, /* × to x */ - { 0x00D8, 0x4F, 0x00 }, /* Ø to O */ - { 0x00D9, 0x55, 0x00 }, /* Ù to U */ - { 0x00DA, 0x55, 0x00 }, /* Ú to U */ - { 0x00DB, 0x55, 0x00 }, /* Û to U */ - { 0x00DC, 0x55, 0x65 }, /* Ü to Ue */ - { 0x00DD, 0x59, 0x00 }, /* Ý to Y */ - { 0x00DE, 0x54, 0x68 }, /* Þ to Th */ - { 0x00DF, 0x73, 0x73 }, /* ß to ss */ - { 0x00E0, 0x61, 0x00 }, /* à to a */ - { 0x00E1, 0x61, 0x00 }, /* á to a */ - { 0x00E2, 0x61, 0x00 }, /* â to a */ - { 0x00E3, 0x61, 0x00 }, /* ã to a */ - { 0x00E4, 0x61, 0x65 }, /* ä to ae */ - { 0x00E5, 0x61, 0x61 }, /* å to aa */ - { 0x00E6, 0x61, 0x65 }, /* æ to ae */ - { 0x00E7, 0x63, 0x00 }, /* ç to c */ - { 0x00E8, 0x65, 0x00 }, /* è to e */ - { 0x00E9, 0x65, 0x00 }, /* é to e */ - { 0x00EA, 0x65, 0x00 }, /* ê to e */ - { 0x00EB, 0x65, 0x00 }, /* ë to e */ - { 0x00EC, 0x69, 0x00 }, /* ì to i */ - { 0x00ED, 0x69, 0x00 }, /* í to i */ - { 0x00EE, 0x69, 0x00 }, /* î to i */ - { 0x00EF, 0x69, 0x00 }, /* ï to i */ - { 0x00F0, 0x64, 0x00 }, /* ð to d */ - { 0x00F1, 0x6E, 0x00 }, /* ñ to n */ - { 0x00F2, 0x6F, 0x00 }, /* ò to o */ - { 0x00F3, 0x6F, 0x00 }, /* ó to o */ - { 0x00F4, 0x6F, 0x00 }, /* ô to o */ - { 0x00F5, 0x6F, 0x00 }, /* õ to o */ - { 0x00F6, 0x6F, 0x65 }, /* ö to oe */ - { 0x00F7, 0x3A, 0x00 }, /* ÷ to : */ - { 0x00F8, 0x6F, 0x00 }, /* ø to o */ - { 0x00F9, 0x75, 0x00 }, /* ù to u */ - { 0x00FA, 0x75, 0x00 }, /* ú to u */ - { 0x00FB, 0x75, 0x00 }, /* û to u */ - { 0x00FC, 0x75, 0x65 }, /* ü to ue */ - { 0x00FD, 0x79, 0x00 }, /* ý to y */ - { 0x00FE, 0x74, 0x68 }, /* þ to th */ - { 0x00FF, 0x79, 0x00 }, /* ÿ to y */ - { 0x0100, 0x41, 0x00 }, /* Ā to A */ - { 0x0101, 0x61, 0x00 }, /* ā to a */ - { 0x0102, 0x41, 0x00 }, /* Ă to A */ - { 0x0103, 0x61, 0x00 }, /* ă to a */ - { 0x0104, 0x41, 0x00 }, /* Ą to A */ - { 0x0105, 0x61, 0x00 }, /* ą to a */ - { 0x0106, 0x43, 0x00 }, /* Ć to C */ - { 0x0107, 0x63, 0x00 }, /* ć to c */ - { 0x0108, 0x43, 0x68 }, /* Ĉ to Ch */ - { 0x0109, 0x63, 0x68 }, /* ĉ to ch */ - { 0x010A, 0x43, 0x00 }, /* Ċ to C */ - { 0x010B, 0x63, 0x00 }, /* ċ to c */ - { 0x010C, 0x43, 0x00 }, /* Č to C */ - { 0x010D, 0x63, 0x00 }, /* č to c */ - { 0x010E, 0x44, 0x00 }, /* Ď to D */ - { 0x010F, 0x64, 0x00 }, /* ď to d */ - { 0x0110, 0x44, 0x00 }, /* Đ to D */ - { 0x0111, 0x64, 0x00 }, /* đ to d */ - { 0x0112, 0x45, 0x00 }, /* Ē to E */ - { 0x0113, 0x65, 0x00 }, /* ē to e */ - { 0x0114, 0x45, 0x00 }, /* Ĕ to E */ - { 0x0115, 0x65, 0x00 }, /* ĕ to e */ - { 0x0116, 0x45, 0x00 }, /* Ė to E */ - { 0x0117, 0x65, 0x00 }, /* ė to e */ - { 0x0118, 0x45, 0x00 }, /* Ę to E */ - { 0x0119, 0x65, 0x00 }, /* ę to e */ - { 0x011A, 0x45, 0x00 }, /* Ě to E */ - { 0x011B, 0x65, 0x00 }, /* ě to e */ - { 0x011C, 0x47, 0x68 }, /* Ĝ to Gh */ - { 0x011D, 0x67, 0x68 }, /* ĝ to gh */ - { 0x011E, 0x47, 0x00 }, /* Ğ to G */ - { 0x011F, 0x67, 0x00 }, /* ğ to g */ - { 0x0120, 0x47, 0x00 }, /* Ġ to G */ - { 0x0121, 0x67, 0x00 }, /* ġ to g */ - { 0x0122, 0x47, 0x00 }, /* Ģ to G */ - { 0x0123, 0x67, 0x00 }, /* ģ to g */ - { 0x0124, 0x48, 0x68 }, /* Ĥ to Hh */ - { 0x0125, 0x68, 0x68 }, /* ĥ to hh */ - { 0x0126, 0x48, 0x00 }, /* Ħ to H */ - { 0x0127, 0x68, 0x00 }, /* ħ to h */ - { 0x0128, 0x49, 0x00 }, /* Ĩ to I */ - { 0x0129, 0x69, 0x00 }, /* ĩ to i */ - { 0x012A, 0x49, 0x00 }, /* Ī to I */ - { 0x012B, 0x69, 0x00 }, /* ī to i */ - { 0x012C, 0x49, 0x00 }, /* Ĭ to I */ - { 0x012D, 0x69, 0x00 }, /* ĭ to i */ - { 0x012E, 0x49, 0x00 }, /* Į to I */ - { 0x012F, 0x69, 0x00 }, /* į to i */ - { 0x0130, 0x49, 0x00 }, /* İ to I */ - { 0x0131, 0x69, 0x00 }, /* ı to i */ - { 0x0132, 0x49, 0x4A }, /* IJ to IJ */ - { 0x0133, 0x69, 0x6A }, /* ij to ij */ - { 0x0134, 0x4A, 0x68 }, /* Ĵ to Jh */ - { 0x0135, 0x6A, 0x68 }, /* ĵ to jh */ - { 0x0136, 0x4B, 0x00 }, /* Ķ to K */ - { 0x0137, 0x6B, 0x00 }, /* ķ to k */ - { 0x0138, 0x6B, 0x00 }, /* ĸ to k */ - { 0x0139, 0x4C, 0x00 }, /* Ĺ to L */ - { 0x013A, 0x6C, 0x00 }, /* ĺ to l */ - { 0x013B, 0x4C, 0x00 }, /* Ļ to L */ - { 0x013C, 0x6C, 0x00 }, /* ļ to l */ - { 0x013D, 0x4C, 0x00 }, /* Ľ to L */ - { 0x013E, 0x6C, 0x00 }, /* ľ to l */ - { 0x013F, 0x4C, 0x2E }, /* Ŀ to L. */ - { 0x0140, 0x6C, 0x2E }, /* ŀ to l. */ - { 0x0141, 0x4C, 0x00 }, /* Ł to L */ - { 0x0142, 0x6C, 0x00 }, /* ł to l */ - { 0x0143, 0x4E, 0x00 }, /* Ń to N */ - { 0x0144, 0x6E, 0x00 }, /* ń to n */ - { 0x0145, 0x4E, 0x00 }, /* Ņ to N */ - { 0x0146, 0x6E, 0x00 }, /* ņ to n */ - { 0x0147, 0x4E, 0x00 }, /* Ň to N */ - { 0x0148, 0x6E, 0x00 }, /* ň to n */ - { 0x0149, 0x27, 0x6E }, /* ʼn to 'n */ - { 0x014A, 0x4E, 0x47 }, /* Ŋ to NG */ - { 0x014B, 0x6E, 0x67 }, /* ŋ to ng */ - { 0x014C, 0x4F, 0x00 }, /* Ō to O */ - { 0x014D, 0x6F, 0x00 }, /* ō to o */ - { 0x014E, 0x4F, 0x00 }, /* Ŏ to O */ - { 0x014F, 0x6F, 0x00 }, /* ŏ to o */ - { 0x0150, 0x4F, 0x00 }, /* Ő to O */ - { 0x0151, 0x6F, 0x00 }, /* ő to o */ - { 0x0152, 0x4F, 0x45 }, /* Œ to OE */ - { 0x0153, 0x6F, 0x65 }, /* œ to oe */ - { 0x0154, 0x52, 0x00 }, /* Ŕ to R */ - { 0x0155, 0x72, 0x00 }, /* ŕ to r */ - { 0x0156, 0x52, 0x00 }, /* Ŗ to R */ - { 0x0157, 0x72, 0x00 }, /* ŗ to r */ - { 0x0158, 0x52, 0x00 }, /* Ř to R */ - { 0x0159, 0x72, 0x00 }, /* ř to r */ - { 0x015A, 0x53, 0x00 }, /* Ś to S */ - { 0x015B, 0x73, 0x00 }, /* ś to s */ - { 0x015C, 0x53, 0x68 }, /* Ŝ to Sh */ - { 0x015D, 0x73, 0x68 }, /* ŝ to sh */ - { 0x015E, 0x53, 0x00 }, /* Ş to S */ - { 0x015F, 0x73, 0x00 }, /* ş to s */ - { 0x0160, 0x53, 0x00 }, /* Š to S */ - { 0x0161, 0x73, 0x00 }, /* š to s */ - { 0x0162, 0x54, 0x00 }, /* Ţ to T */ - { 0x0163, 0x74, 0x00 }, /* ţ to t */ - { 0x0164, 0x54, 0x00 }, /* Ť to T */ - { 0x0165, 0x74, 0x00 }, /* ť to t */ - { 0x0166, 0x54, 0x00 }, /* Ŧ to T */ - { 0x0167, 0x74, 0x00 }, /* ŧ to t */ - { 0x0168, 0x55, 0x00 }, /* Ũ to U */ - { 0x0169, 0x75, 0x00 }, /* ũ to u */ - { 0x016A, 0x55, 0x00 }, /* Ū to U */ - { 0x016B, 0x75, 0x00 }, /* ū to u */ - { 0x016C, 0x55, 0x00 }, /* Ŭ to U */ - { 0x016D, 0x75, 0x00 }, /* ŭ to u */ - { 0x016E, 0x55, 0x00 }, /* Ů to U */ - { 0x016F, 0x75, 0x00 }, /* ů to u */ - { 0x0170, 0x55, 0x00 }, /* Ű to U */ - { 0x0171, 0x75, 0x00 }, /* ű to u */ - { 0x0172, 0x55, 0x00 }, /* Ų to U */ - { 0x0173, 0x75, 0x00 }, /* ų to u */ - { 0x0174, 0x57, 0x00 }, /* Ŵ to W */ - { 0x0175, 0x77, 0x00 }, /* ŵ to w */ - { 0x0176, 0x59, 0x00 }, /* Ŷ to Y */ - { 0x0177, 0x79, 0x00 }, /* ŷ to y */ - { 0x0178, 0x59, 0x00 }, /* Ÿ to Y */ - { 0x0179, 0x5A, 0x00 }, /* Ź to Z */ - { 0x017A, 0x7A, 0x00 }, /* ź to z */ - { 0x017B, 0x5A, 0x00 }, /* Ż to Z */ - { 0x017C, 0x7A, 0x00 }, /* ż to z */ - { 0x017D, 0x5A, 0x00 }, /* Ž to Z */ - { 0x017E, 0x7A, 0x00 }, /* ž to z */ - { 0x017F, 0x73, 0x00 }, /* ſ to s */ - { 0x0192, 0x66, 0x00 }, /* ƒ to f */ - { 0x0218, 0x53, 0x00 }, /* Ș to S */ - { 0x0219, 0x73, 0x00 }, /* ș to s */ - { 0x021A, 0x54, 0x00 }, /* Ț to T */ - { 0x021B, 0x74, 0x00 }, /* ț to t */ - { 0x0386, 0x41, 0x00 }, /* Ά to A */ - { 0x0388, 0x45, 0x00 }, /* Έ to E */ - { 0x0389, 0x49, 0x00 }, /* Ή to I */ - { 0x038A, 0x49, 0x00 }, /* Ί to I */ - { 0x038C, 0x4f, 0x00 }, /* Ό to O */ - { 0x038E, 0x59, 0x00 }, /* Ύ to Y */ - { 0x038F, 0x4f, 0x00 }, /* Ώ to O */ - { 0x0390, 0x69, 0x00 }, /* ΐ to i */ - { 0x0391, 0x41, 0x00 }, /* Α to A */ - { 0x0392, 0x42, 0x00 }, /* Β to B */ - { 0x0393, 0x47, 0x00 }, /* Γ to G */ - { 0x0394, 0x44, 0x00 }, /* Δ to D */ - { 0x0395, 0x45, 0x00 }, /* Ε to E */ - { 0x0396, 0x5a, 0x00 }, /* Ζ to Z */ - { 0x0397, 0x49, 0x00 }, /* Η to I */ - { 0x0398, 0x54, 0x68 }, /* Θ to Th */ - { 0x0399, 0x49, 0x00 }, /* Ι to I */ - { 0x039A, 0x4b, 0x00 }, /* Κ to K */ - { 0x039B, 0x4c, 0x00 }, /* Λ to L */ - { 0x039C, 0x4d, 0x00 }, /* Μ to M */ - { 0x039D, 0x4e, 0x00 }, /* Ν to N */ - { 0x039E, 0x58, 0x00 }, /* Ξ to X */ - { 0x039F, 0x4f, 0x00 }, /* Ο to O */ - { 0x03A0, 0x50, 0x00 }, /* Π to P */ - { 0x03A1, 0x52, 0x00 }, /* Ρ to R */ - { 0x03A3, 0x53, 0x00 }, /* Σ to S */ - { 0x03A4, 0x54, 0x00 }, /* Τ to T */ - { 0x03A5, 0x59, 0x00 }, /* Υ to Y */ - { 0x03A6, 0x46, 0x00 }, /* Φ to F */ - { 0x03A7, 0x43, 0x68 }, /* Χ to Ch */ - { 0x03A8, 0x50, 0x73 }, /* Ψ to Ps */ - { 0x03A9, 0x4f, 0x00 }, /* Ω to O */ - { 0x03AA, 0x49, 0x00 }, /* Ϊ to I */ - { 0x03AB, 0x59, 0x00 }, /* Ϋ to Y */ - { 0x03AC, 0x61, 0x00 }, /* ά to a */ - { 0x03AD, 0x65, 0x00 }, /* έ to e */ - { 0x03AE, 0x69, 0x00 }, /* ή to i */ - { 0x03AF, 0x69, 0x00 }, /* ί to i */ - { 0x03B1, 0x61, 0x00 }, /* α to a */ - { 0x03B2, 0x62, 0x00 }, /* β to b */ - { 0x03B3, 0x67, 0x00 }, /* γ to g */ - { 0x03B4, 0x64, 0x00 }, /* δ to d */ - { 0x03B5, 0x65, 0x00 }, /* ε to e */ - { 0x03B6, 0x7a, 0x00 }, /* ζ to z */ - { 0x03B7, 0x69, 0x00 }, /* η to i */ - { 0x03B8, 0x74, 0x68 }, /* θ to th */ - { 0x03B9, 0x69, 0x00 }, /* ι to i */ - { 0x03BA, 0x6b, 0x00 }, /* κ to k */ - { 0x03BB, 0x6c, 0x00 }, /* λ to l */ - { 0x03BC, 0x6d, 0x00 }, /* μ to m */ - { 0x03BD, 0x6e, 0x00 }, /* ν to n */ - { 0x03BE, 0x78, 0x00 }, /* ξ to x */ - { 0x03BF, 0x6f, 0x00 }, /* ο to o */ - { 0x03C0, 0x70, 0x00 }, /* π to p */ - { 0x03C1, 0x72, 0x00 }, /* ρ to r */ - { 0x03C3, 0x73, 0x00 }, /* σ to s */ - { 0x03C4, 0x74, 0x00 }, /* τ to t */ - { 0x03C5, 0x79, 0x00 }, /* υ to y */ - { 0x03C6, 0x66, 0x00 }, /* φ to f */ - { 0x03C7, 0x63, 0x68 }, /* χ to ch */ - { 0x03C8, 0x70, 0x73 }, /* ψ to ps */ - { 0x03C9, 0x6f, 0x00 }, /* ω to o */ - { 0x03CA, 0x69, 0x00 }, /* ϊ to i */ - { 0x03CB, 0x79, 0x00 }, /* ϋ to y */ - { 0x03CC, 0x6f, 0x00 }, /* ό to o */ - { 0x03CD, 0x79, 0x00 }, /* ύ to y */ - { 0x03CE, 0x69, 0x00 }, /* ώ to i */ - { 0x0400, 0x45, 0x00 }, /* Ѐ to E */ - { 0x0401, 0x45, 0x00 }, /* Ё to E */ - { 0x0402, 0x44, 0x00 }, /* Ђ to D */ - { 0x0403, 0x47, 0x00 }, /* Ѓ to G */ - { 0x0404, 0x45, 0x00 }, /* Є to E */ - { 0x0405, 0x5a, 0x00 }, /* Ѕ to Z */ - { 0x0406, 0x49, 0x00 }, /* І to I */ - { 0x0407, 0x49, 0x00 }, /* Ї to I */ - { 0x0408, 0x4a, 0x00 }, /* Ј to J */ - { 0x0409, 0x49, 0x00 }, /* Љ to I */ - { 0x040A, 0x4e, 0x00 }, /* Њ to N */ - { 0x040B, 0x44, 0x00 }, /* Ћ to D */ - { 0x040C, 0x4b, 0x00 }, /* Ќ to K */ - { 0x040D, 0x49, 0x00 }, /* Ѝ to I */ - { 0x040E, 0x55, 0x00 }, /* Ў to U */ - { 0x040F, 0x44, 0x00 }, /* Џ to D */ - { 0x0410, 0x41, 0x00 }, /* А to A */ - { 0x0411, 0x42, 0x00 }, /* Б to B */ - { 0x0412, 0x56, 0x00 }, /* В to V */ - { 0x0413, 0x47, 0x00 }, /* Г to G */ - { 0x0414, 0x44, 0x00 }, /* Д to D */ - { 0x0415, 0x45, 0x00 }, /* Е to E */ - { 0x0416, 0x5a, 0x68 }, /* Ж to Zh */ - { 0x0417, 0x5a, 0x00 }, /* З to Z */ - { 0x0418, 0x49, 0x00 }, /* И to I */ - { 0x0419, 0x49, 0x00 }, /* Й to I */ - { 0x041A, 0x4b, 0x00 }, /* К to K */ - { 0x041B, 0x4c, 0x00 }, /* Л to L */ - { 0x041C, 0x4d, 0x00 }, /* М to M */ - { 0x041D, 0x4e, 0x00 }, /* Н to N */ - { 0x041E, 0x4f, 0x00 }, /* О to O */ - { 0x041F, 0x50, 0x00 }, /* П to P */ - { 0x0420, 0x52, 0x00 }, /* Р to R */ - { 0x0421, 0x53, 0x00 }, /* С to S */ - { 0x0422, 0x54, 0x00 }, /* Т to T */ - { 0x0423, 0x55, 0x00 }, /* У to U */ - { 0x0424, 0x46, 0x00 }, /* Ф to F */ - { 0x0425, 0x4b, 0x68 }, /* Х to Kh */ - { 0x0426, 0x54, 0x63 }, /* Ц to Tc */ - { 0x0427, 0x43, 0x68 }, /* Ч to Ch */ - { 0x0428, 0x53, 0x68 }, /* Ш to Sh */ - { 0x0429, 0x53, 0x68 }, /* Щ to Shch */ - { 0x042A, 0x61, 0x00 }, /* to A */ - { 0x042B, 0x59, 0x00 }, /* Ы to Y */ - { 0x042C, 0x59, 0x00 }, /* to Y */ - { 0x042D, 0x45, 0x00 }, /* Э to E */ - { 0x042E, 0x49, 0x75 }, /* Ю to Iu */ - { 0x042F, 0x49, 0x61 }, /* Я to Ia */ - { 0x0430, 0x61, 0x00 }, /* а to a */ - { 0x0431, 0x62, 0x00 }, /* б to b */ - { 0x0432, 0x76, 0x00 }, /* в to v */ - { 0x0433, 0x67, 0x00 }, /* г to g */ - { 0x0434, 0x64, 0x00 }, /* д to d */ - { 0x0435, 0x65, 0x00 }, /* е to e */ - { 0x0436, 0x7a, 0x68 }, /* ж to zh */ - { 0x0437, 0x7a, 0x00 }, /* з to z */ - { 0x0438, 0x69, 0x00 }, /* и to i */ - { 0x0439, 0x69, 0x00 }, /* й to i */ - { 0x043A, 0x6b, 0x00 }, /* к to k */ - { 0x043B, 0x6c, 0x00 }, /* л to l */ - { 0x043C, 0x6d, 0x00 }, /* м to m */ - { 0x043D, 0x6e, 0x00 }, /* н to n */ - { 0x043E, 0x6f, 0x00 }, /* о to o */ - { 0x043F, 0x70, 0x00 }, /* п to p */ - { 0x0440, 0x72, 0x00 }, /* р to r */ - { 0x0441, 0x73, 0x00 }, /* с to s */ - { 0x0442, 0x74, 0x00 }, /* т to t */ - { 0x0443, 0x75, 0x00 }, /* у to u */ - { 0x0444, 0x66, 0x00 }, /* ф to f */ - { 0x0445, 0x6b, 0x68 }, /* х to kh */ - { 0x0446, 0x74, 0x63 }, /* ц to tc */ - { 0x0447, 0x63, 0x68 }, /* ч to ch */ - { 0x0448, 0x73, 0x68 }, /* ш to sh */ - { 0x0449, 0x73, 0x68 }, /* щ to shch */ - { 0x044A, 0x61, 0x00 }, /* to a */ - { 0x044B, 0x79, 0x00 }, /* ы to y */ - { 0x044C, 0x79, 0x00 }, /* to y */ - { 0x044D, 0x65, 0x00 }, /* э to e */ - { 0x044E, 0x69, 0x75 }, /* ю to iu */ - { 0x044F, 0x69, 0x61 }, /* я to ia */ - { 0x0450, 0x65, 0x00 }, /* ѐ to e */ - { 0x0451, 0x65, 0x00 }, /* ё to e */ - { 0x0452, 0x64, 0x00 }, /* ђ to d */ - { 0x0453, 0x67, 0x00 }, /* ѓ to g */ - { 0x0454, 0x65, 0x00 }, /* є to e */ - { 0x0455, 0x7a, 0x00 }, /* ѕ to z */ - { 0x0456, 0x69, 0x00 }, /* і to i */ - { 0x0457, 0x69, 0x00 }, /* ї to i */ - { 0x0458, 0x6a, 0x00 }, /* ј to j */ - { 0x0459, 0x69, 0x00 }, /* љ to i */ - { 0x045A, 0x6e, 0x00 }, /* њ to n */ - { 0x045B, 0x64, 0x00 }, /* ћ to d */ - { 0x045C, 0x6b, 0x00 }, /* ќ to k */ - { 0x045D, 0x69, 0x00 }, /* ѝ to i */ - { 0x045E, 0x75, 0x00 }, /* ў to u */ - { 0x045F, 0x64, 0x00 }, /* џ to d */ - { 0x1E02, 0x42, 0x00 }, /* Ḃ to B */ - { 0x1E03, 0x62, 0x00 }, /* ḃ to b */ - { 0x1E0A, 0x44, 0x00 }, /* Ḋ to D */ - { 0x1E0B, 0x64, 0x00 }, /* ḋ to d */ - { 0x1E1E, 0x46, 0x00 }, /* Ḟ to F */ - { 0x1E1F, 0x66, 0x00 }, /* ḟ to f */ - { 0x1E40, 0x4D, 0x00 }, /* Ṁ to M */ - { 0x1E41, 0x6D, 0x00 }, /* ṁ to m */ - { 0x1E56, 0x50, 0x00 }, /* Ṗ to P */ - { 0x1E57, 0x70, 0x00 }, /* ṗ to p */ - { 0x1E60, 0x53, 0x00 }, /* Ṡ to S */ - { 0x1E61, 0x73, 0x00 }, /* ṡ to s */ - { 0x1E6A, 0x54, 0x00 }, /* Ṫ to T */ - { 0x1E6B, 0x74, 0x00 }, /* ṫ to t */ - { 0x1E80, 0x57, 0x00 }, /* Ẁ to W */ - { 0x1E81, 0x77, 0x00 }, /* ẁ to w */ - { 0x1E82, 0x57, 0x00 }, /* Ẃ to W */ - { 0x1E83, 0x77, 0x00 }, /* ẃ to w */ - { 0x1E84, 0x57, 0x00 }, /* Ẅ to W */ - { 0x1E85, 0x77, 0x00 }, /* ẅ to w */ - { 0x1EF2, 0x59, 0x00 }, /* Ỳ to Y */ - { 0x1EF3, 0x79, 0x00 }, /* ỳ to y */ - { 0xFB00, 0x66, 0x66 }, /* ff to ff */ - { 0xFB01, 0x66, 0x69 }, /* fi to fi */ - { 0xFB02, 0x66, 0x6C }, /* fl to fl */ - { 0xFB05, 0x73, 0x74 }, /* ſt to st */ - { 0xFB06, 0x73, 0x74 }, /* st to st */ -}; - -/* -** Convert the input string from UTF-8 into pure ASCII by converting -** all non-ASCII characters to some combination of characters in the -** ASCII subset. -** -** The returned string might contain more characters than the input. -** -** Space to hold the returned string comes from sqlite3_malloc() and -** should be freed by the caller. -*/ -static unsigned char *transliterate(const unsigned char *zIn, int nIn){ - unsigned char *zOut = sqlite3_malloc( nIn*4 + 1 ); - int c, sz, nOut; - if( zOut==0 ) return 0; - nOut = 0; - while( nIn>0 ){ - c = utf8Read(zIn, nIn, &sz); - zIn += sz; - nIn -= sz; - if( c<=127 ){ - zOut[nOut++] = c; - }else{ - int xTop, xBtm, x; - xTop = sizeof(translit)/sizeof(translit[0]) - 1; - xBtm = 0; - while( xTop>=xBtm ){ - x = (xTop + xBtm)/2; - if( translit[x].cFrom==c ){ - zOut[nOut++] = translit[x].cTo0; - if( translit[x].cTo1 ){ - zOut[nOut++] = translit[x].cTo1; - /* Add an extra "ch" after the "sh" for Щ and щ */ - if( c==0x0429 || c== 0x0449 ){ - zOut[nOut++] = 'c'; - zOut[nOut++] = 'h'; - } - } - c = 0; - break; - }else if( translit[x].cFrom>c ){ - xTop = x-1; - }else{ - xBtm = x+1; - } - } - if( c ) zOut[nOut++] = '?'; - } - } - zOut[nOut] = 0; - return zOut; -} - -/* -** Return the number of characters in the shortest prefix of the input -** string that transliterates to an ASCII string nTrans bytes or longer. -** Or, if the transliteration of the input string is less than nTrans -** bytes in size, return the number of characters in the input string. -*/ -static int translen_to_charlen(const char *zIn, int nIn, int nTrans){ - int i, c, sz, nOut; - int nChar; - - i = nOut = 0; - for(nChar=0; i<nIn && nOut<nTrans; nChar++){ - c = utf8Read((const unsigned char *)&zIn[i], nIn-i, &sz); - i += sz; - - nOut++; - if( c>=128 ){ - int xTop, xBtm, x; - xTop = sizeof(translit)/sizeof(translit[0]) - 1; - xBtm = 0; - while( xTop>=xBtm ){ - x = (xTop + xBtm)/2; - if( translit[x].cFrom==c ){ - if( translit[x].cTo1 ) nOut++; - if( c==0x0429 || c== 0x0449 ) nOut += 2; - break; - }else if( translit[x].cFrom>c ){ - xTop = x-1; - }else{ - xBtm = x+1; - } - } - } - } - - return nChar; -} - - -/* -** spellfix1_translit(X) -** -** Convert a string that contains non-ASCII Roman characters into -** pure ASCII. -*/ -static void transliterateSqlFunc( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - const unsigned char *zIn = sqlite3_value_text(argv[0]); - int nIn = sqlite3_value_bytes(argv[0]); - unsigned char *zOut = transliterate(zIn, nIn); - if( zOut==0 ){ - sqlite3_result_error_nomem(context); - }else{ - sqlite3_result_text(context, (char*)zOut, -1, sqlite3_free); - } -} - -/* -** spellfix1_scriptcode(X) -** -** Try to determine the dominant script used by the word X and return -** its ISO 15924 numeric code. -** -** The current implementation only understands the following scripts: -** -** 215 (Latin) -** 220 (Cyrillic) -** 200 (Greek) -** -** This routine will return 998 if the input X contains characters from -** two or more of the above scripts or 999 if X contains no characters -** from any of the above scripts. -*/ -static void scriptCodeSqlFunc( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - const unsigned char *zIn = sqlite3_value_text(argv[0]); - int nIn = sqlite3_value_bytes(argv[0]); - int c, sz; - int scriptMask = 0; - int res; -# define SCRIPT_LATIN 0x0001 -# define SCRIPT_CYRILLIC 0x0002 -# define SCRIPT_GREEK 0x0004 - - while( nIn>0 ){ - c = utf8Read(zIn, nIn, &sz); - zIn += sz; - nIn -= sz; - if( c<0x02af ){ - scriptMask |= SCRIPT_LATIN; - }else if( c>=0x0400 && c<=0x04ff ){ - scriptMask |= SCRIPT_CYRILLIC; - }else if( c>=0x0386 && c<=0x03ce ){ - scriptMask |= SCRIPT_GREEK; - } - } - switch( scriptMask ){ - case 0: res = 999; break; - case SCRIPT_LATIN: res = 215; break; - case SCRIPT_CYRILLIC: res = 220; break; - case SCRIPT_GREEK: res = 200; break; - default: res = 998; break; - } - sqlite3_result_int(context, res); -} - -/* End transliterate -****************************************************************************** -****************************************************************************** -** Begin spellfix1 virtual table. -*/ - -/* Maximum length of a phonehash used for querying the shadow table */ -#define SPELLFIX_MX_HASH 8 - -/* Maximum number of hash strings to examine per query */ -#define SPELLFIX_MX_RUN 1 - -typedef struct spellfix1_vtab spellfix1_vtab; -typedef struct spellfix1_cursor spellfix1_cursor; - -/* Fuzzy-search virtual table object */ -struct spellfix1_vtab { - sqlite3_vtab base; /* Base class - must be first */ - sqlite3 *db; /* Database connection */ - char *zDbName; /* Name of database holding this table */ - char *zTableName; /* Name of the virtual table */ - char *zCostTable; /* Table holding edit-distance cost numbers */ - EditDist3Config *pConfig3; /* Parsed edit distance costs */ -}; - -/* Fuzzy-search cursor object */ -struct spellfix1_cursor { - sqlite3_vtab_cursor base; /* Base class - must be first */ - spellfix1_vtab *pVTab; /* The table to which this cursor belongs */ - char *zPattern; /* rhs of MATCH clause */ - int nRow; /* Number of rows of content */ - int nAlloc; /* Number of allocated rows */ - int iRow; /* Current row of content */ - int iLang; /* Value of the langid= constraint */ - int iTop; /* Value of the top= constraint */ - int iScope; /* Value of the scope= constraint */ - int nSearch; /* Number of vocabulary items checked */ - sqlite3_stmt *pFullScan; /* Shadow query for a full table scan */ - struct spellfix1_row { /* For each row of content */ - sqlite3_int64 iRowid; /* Rowid for this row */ - char *zWord; /* Text for this row */ - int iRank; /* Rank for this row */ - int iDistance; /* Distance from pattern for this row */ - int iScore; /* Score for sorting */ - int iMatchlen; /* Value of matchlen column (or -1) */ - char zHash[SPELLFIX_MX_HASH]; /* the phonehash used for this match */ - } *a; -}; - -/* -** Construct one or more SQL statements from the format string given -** and then evaluate those statements. The success code is written -** into *pRc. -** -** If *pRc is initially non-zero then this routine is a no-op. -*/ -static void spellfix1DbExec( - int *pRc, /* Success code */ - sqlite3 *db, /* Database in which to run SQL */ - const char *zFormat, /* Format string for SQL */ - ... /* Arguments to the format string */ -){ - va_list ap; - char *zSql; - if( *pRc ) return; - va_start(ap, zFormat); - zSql = sqlite3_vmprintf(zFormat, ap); - va_end(ap); - if( zSql==0 ){ - *pRc = SQLITE_NOMEM; - }else{ - *pRc = sqlite3_exec(db, zSql, 0, 0, 0); - sqlite3_free(zSql); - } -} - -/* -** xDisconnect/xDestroy method for the fuzzy-search module. -*/ -static int spellfix1Uninit(int isDestroy, sqlite3_vtab *pVTab){ - spellfix1_vtab *p = (spellfix1_vtab*)pVTab; - int rc = SQLITE_OK; - if( isDestroy ){ - sqlite3 *db = p->db; - spellfix1DbExec(&rc, db, "DROP TABLE IF EXISTS \"%w\".\"%w_vocab\"", - p->zDbName, p->zTableName); - } - if( rc==SQLITE_OK ){ - sqlite3_free(p->zTableName); - editDist3ConfigDelete(p->pConfig3); - sqlite3_free(p->zCostTable); - sqlite3_free(p); - } - return rc; -} -static int spellfix1Disconnect(sqlite3_vtab *pVTab){ - return spellfix1Uninit(0, pVTab); -} -static int spellfix1Destroy(sqlite3_vtab *pVTab){ - return spellfix1Uninit(1, pVTab); -} - -/* -** Make a copy of a string. Remove leading and trailing whitespace -** and dequote it. -*/ -static char *spellfix1Dequote(const char *zIn){ - char *zOut; - int i, j; - char c; - while( isspace(zIn[0]) ) zIn++; - zOut = sqlite3_mprintf("%s", zIn); - if( zOut==0 ) return 0; - i = (int)strlen(zOut); -#if 0 /* The parser will never leave spaces at the end */ - while( i>0 && isspace(zOut[i-1]) ){ i--; } -#endif - zOut[i] = 0; - c = zOut[0]; - if( c=='\'' || c=='"' ){ - for(i=1, j=0; ALWAYS(zOut[i]); i++){ - zOut[j++] = zOut[i]; - if( zOut[i]==c ){ - if( zOut[i+1]==c ){ - i++; - }else{ - zOut[j-1] = 0; - break; - } - } - } - } - return zOut; -} - - -/* -** xConnect/xCreate method for the spellfix1 module. Arguments are: -** -** argv[0] -> module name ("spellfix1") -** argv[1] -> database name -** argv[2] -> table name -** argv[3].. -> optional arguments (i.e. "edit_cost_table" parameter) -*/ -static int spellfix1Init( - int isCreate, - sqlite3 *db, - void *pAux, - int argc, const char *const*argv, - sqlite3_vtab **ppVTab, - char **pzErr -){ - spellfix1_vtab *pNew = 0; - const char *zModule = argv[0]; - const char *zDbName = argv[1]; - const char *zTableName = argv[2]; - int nDbName; - int rc = SQLITE_OK; - int i; - - nDbName = (int)strlen(zDbName); - pNew = sqlite3_malloc( sizeof(*pNew) + nDbName + 1); - if( pNew==0 ){ - rc = SQLITE_NOMEM; - }else{ - memset(pNew, 0, sizeof(*pNew)); - pNew->zDbName = (char*)&pNew[1]; - memcpy(pNew->zDbName, zDbName, nDbName+1); - pNew->zTableName = sqlite3_mprintf("%s", zTableName); - pNew->db = db; - if( pNew->zTableName==0 ){ - rc = SQLITE_NOMEM; - }else{ - rc = sqlite3_declare_vtab(db, - "CREATE TABLE x(word,rank,distance,langid, " - "score, matchlen, phonehash HIDDEN, " - "top HIDDEN, scope HIDDEN, srchcnt HIDDEN, " - "soundslike HIDDEN, command HIDDEN)" - ); -#define SPELLFIX_COL_WORD 0 -#define SPELLFIX_COL_RANK 1 -#define SPELLFIX_COL_DISTANCE 2 -#define SPELLFIX_COL_LANGID 3 -#define SPELLFIX_COL_SCORE 4 -#define SPELLFIX_COL_MATCHLEN 5 -#define SPELLFIX_COL_PHONEHASH 6 -#define SPELLFIX_COL_TOP 7 -#define SPELLFIX_COL_SCOPE 8 -#define SPELLFIX_COL_SRCHCNT 9 -#define SPELLFIX_COL_SOUNDSLIKE 10 -#define SPELLFIX_COL_COMMAND 11 - } - if( rc==SQLITE_OK && isCreate ){ - sqlite3_uint64 r; - spellfix1DbExec(&rc, db, - "CREATE TABLE IF NOT EXISTS \"%w\".\"%w_vocab\"(\n" - " id INTEGER PRIMARY KEY,\n" - " rank INT,\n" - " langid INT,\n" - " word TEXT,\n" - " k1 TEXT,\n" - " k2 TEXT\n" - ");\n", - zDbName, zTableName - ); - sqlite3_randomness(sizeof(r), &r); - spellfix1DbExec(&rc, db, - "CREATE INDEX IF NOT EXISTS \"%w\".\"%w_index_%llx\" " - "ON \"%w_vocab\"(langid,k2);", - zDbName, zModule, r, zTableName - ); - } - for(i=3; rc==SQLITE_OK && i<argc; i++){ - if( memcmp(argv[i],"edit_cost_table=",16)==0 && pNew->zCostTable==0 ){ - pNew->zCostTable = spellfix1Dequote(&argv[i][16]); - if( pNew->zCostTable==0 ) rc = SQLITE_NOMEM; - continue; - } - *pzErr = sqlite3_mprintf("bad argument to spellfix1(): \"%s\"", argv[i]); - rc = SQLITE_ERROR; - } - } - - if( rc && pNew ){ - *ppVTab = 0; - spellfix1Uninit(0, &pNew->base); - }else{ - *ppVTab = (sqlite3_vtab *)pNew; - } - return rc; -} - -/* -** The xConnect and xCreate methods -*/ -static int spellfix1Connect( - sqlite3 *db, - void *pAux, - int argc, const char *const*argv, - sqlite3_vtab **ppVTab, - char **pzErr -){ - return spellfix1Init(0, db, pAux, argc, argv, ppVTab, pzErr); -} -static int spellfix1Create( - sqlite3 *db, - void *pAux, - int argc, const char *const*argv, - sqlite3_vtab **ppVTab, - char **pzErr -){ - return spellfix1Init(1, db, pAux, argc, argv, ppVTab, pzErr); -} - -/* -** Clear all of the content from a cursor. -*/ -static void spellfix1ResetCursor(spellfix1_cursor *pCur){ - int i; - for(i=0; i<pCur->nRow; i++){ - sqlite3_free(pCur->a[i].zWord); - } - pCur->nRow = 0; - pCur->iRow = 0; - pCur->nSearch = 0; - if( pCur->pFullScan ){ - sqlite3_finalize(pCur->pFullScan); - pCur->pFullScan = 0; - } -} - -/* -** Resize the cursor to hold up to N rows of content -*/ -static void spellfix1ResizeCursor(spellfix1_cursor *pCur, int N){ - struct spellfix1_row *aNew; - assert( N>=pCur->nRow ); - aNew = sqlite3_realloc(pCur->a, sizeof(pCur->a[0])*N); - if( aNew==0 && N>0 ){ - spellfix1ResetCursor(pCur); - sqlite3_free(pCur->a); - pCur->nAlloc = 0; - pCur->a = 0; - }else{ - pCur->nAlloc = N; - pCur->a = aNew; - } -} - - -/* -** Close a fuzzy-search cursor. -*/ -static int spellfix1Close(sqlite3_vtab_cursor *cur){ - spellfix1_cursor *pCur = (spellfix1_cursor *)cur; - spellfix1ResetCursor(pCur); - spellfix1ResizeCursor(pCur, 0); - sqlite3_free(pCur->zPattern); - sqlite3_free(pCur); - return SQLITE_OK; -} - -/* -** Search for terms of these forms: -** -** (A) word MATCH $str -** (B) langid == $langid -** (C) top = $top -** (D) scope = $scope -** (E) distance < $distance -** (F) distance <= $distance -** -** The plan number is a bit mask formed with these bits: -** -** 0x01 (A) is found -** 0x02 (B) is found -** 0x04 (C) is found -** 0x08 (D) is found -** 0x10 (E) is found -** 0x20 (F) is found -** -** filter.argv[*] values contains $str, $langid, $top, and $scope, -** if specified and in that order. -*/ -static int spellfix1BestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ - int iPlan = 0; - int iLangTerm = -1; - int iTopTerm = -1; - int iScopeTerm = -1; - int iDistTerm = -1; - int i; - const struct sqlite3_index_constraint *pConstraint; - pConstraint = pIdxInfo->aConstraint; - for(i=0; i<pIdxInfo->nConstraint; i++, pConstraint++){ - if( pConstraint->usable==0 ) continue; - - /* Terms of the form: word MATCH $str */ - if( (iPlan & 1)==0 - && pConstraint->iColumn==SPELLFIX_COL_WORD - && pConstraint->op==SQLITE_INDEX_CONSTRAINT_MATCH - ){ - iPlan |= 1; - pIdxInfo->aConstraintUsage[i].argvIndex = 1; - pIdxInfo->aConstraintUsage[i].omit = 1; - } - - /* Terms of the form: langid = $langid */ - if( (iPlan & 2)==0 - && pConstraint->iColumn==SPELLFIX_COL_LANGID - && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ - ){ - iPlan |= 2; - iLangTerm = i; - } - - /* Terms of the form: top = $top */ - if( (iPlan & 4)==0 - && pConstraint->iColumn==SPELLFIX_COL_TOP - && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ - ){ - iPlan |= 4; - iTopTerm = i; - } - - /* Terms of the form: scope = $scope */ - if( (iPlan & 8)==0 - && pConstraint->iColumn==SPELLFIX_COL_SCOPE - && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ - ){ - iPlan |= 8; - iScopeTerm = i; - } - - /* Terms of the form: distance < $dist or distance <= $dist */ - if( (iPlan & (16|32))==0 - && pConstraint->iColumn==SPELLFIX_COL_DISTANCE - && (pConstraint->op==SQLITE_INDEX_CONSTRAINT_LT - || pConstraint->op==SQLITE_INDEX_CONSTRAINT_LE) - ){ - iPlan |= pConstraint->op==SQLITE_INDEX_CONSTRAINT_LT ? 16 : 32; - iDistTerm = i; - } - } - if( iPlan&1 ){ - int idx = 2; - pIdxInfo->idxNum = iPlan; - if( pIdxInfo->nOrderBy==1 - && pIdxInfo->aOrderBy[0].iColumn==SPELLFIX_COL_SCORE - && pIdxInfo->aOrderBy[0].desc==0 - ){ - pIdxInfo->orderByConsumed = 1; /* Default order by iScore */ - } - if( iPlan&2 ){ - pIdxInfo->aConstraintUsage[iLangTerm].argvIndex = idx++; - pIdxInfo->aConstraintUsage[iLangTerm].omit = 1; - } - if( iPlan&4 ){ - pIdxInfo->aConstraintUsage[iTopTerm].argvIndex = idx++; - pIdxInfo->aConstraintUsage[iTopTerm].omit = 1; - } - if( iPlan&8 ){ - pIdxInfo->aConstraintUsage[iScopeTerm].argvIndex = idx++; - pIdxInfo->aConstraintUsage[iScopeTerm].omit = 1; - } - if( iPlan&(16|32) ){ - pIdxInfo->aConstraintUsage[iDistTerm].argvIndex = idx++; - pIdxInfo->aConstraintUsage[iDistTerm].omit = 1; - } - pIdxInfo->estimatedCost = (double)10000; - }else{ - pIdxInfo->idxNum = 0; - pIdxInfo->estimatedCost = (double)10000000; - } - return SQLITE_OK; -} - -/* -** Open a new fuzzy-search cursor. -*/ -static int spellfix1Open(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ - spellfix1_vtab *p = (spellfix1_vtab*)pVTab; - spellfix1_cursor *pCur; - pCur = sqlite3_malloc( sizeof(*pCur) ); - if( pCur==0 ) return SQLITE_NOMEM; - memset(pCur, 0, sizeof(*pCur)); - pCur->pVTab = p; - *ppCursor = &pCur->base; - return SQLITE_OK; -} - -/* -** Adjust a distance measurement by the words rank in order to show -** preference to common words. -*/ -static int spellfix1Score(int iDistance, int iRank){ - int iLog2; - for(iLog2=0; iRank>0; iLog2++, iRank>>=1){} - return iDistance + 32 - iLog2; -} - -/* -** Compare two spellfix1_row objects for sorting purposes in qsort() such -** that they sort in order of increasing distance. -*/ -static int spellfix1RowCompare(const void *A, const void *B){ - const struct spellfix1_row *a = (const struct spellfix1_row*)A; - const struct spellfix1_row *b = (const struct spellfix1_row*)B; - return a->iScore - b->iScore; -} - -/* -** A structure used to pass information from spellfix1FilterForMatch() -** into spellfix1RunQuery(). -*/ -typedef struct MatchQuery { - spellfix1_cursor *pCur; /* The cursor being queried */ - sqlite3_stmt *pStmt; /* shadow table query statment */ - char zHash[SPELLFIX_MX_HASH]; /* The current phonehash for zPattern */ - const char *zPattern; /* Transliterated input string */ - int nPattern; /* Length of zPattern */ - EditDist3FromString *pMatchStr3; /* Original unicode string */ - EditDist3Config *pConfig3; /* Edit-distance cost coefficients */ - const EditDist3Lang *pLang; /* The selected language coefficients */ - int iLang; /* The language id */ - int iScope; /* Default scope */ - int iMaxDist; /* Maximum allowed edit distance, or -1 */ - int rc; /* Error code */ - int nRun; /* Number of prior runs for the same zPattern */ - char azPrior[SPELLFIX_MX_RUN][SPELLFIX_MX_HASH]; /* Prior hashes */ -} MatchQuery; - -/* -** Run a query looking for the best matches against zPattern using -** zHash as the character class seed hash. -*/ -static void spellfix1RunQuery(MatchQuery *p, const char *zQuery, int nQuery){ - const char *zK1; - const char *zWord; - int iDist; - int iRank; - int iScore; - int iWorst = 0; - int idx; - int idxWorst = -1; - int i; - int iScope = p->iScope; - spellfix1_cursor *pCur = p->pCur; - sqlite3_stmt *pStmt = p->pStmt; - char zHash1[SPELLFIX_MX_HASH]; - char zHash2[SPELLFIX_MX_HASH]; - char *zClass; - int nClass; - int rc; - - if( pCur->a==0 || p->rc ) return; /* Prior memory allocation failure */ - zClass = (char*)phoneticHash((unsigned char*)zQuery, nQuery); - if( zClass==0 ){ - p->rc = SQLITE_NOMEM; - return; - } - nClass = (int)strlen(zClass); - if( nClass>SPELLFIX_MX_HASH-2 ){ - nClass = SPELLFIX_MX_HASH-2; - zClass[nClass] = 0; - } - if( nClass<=iScope ){ - if( nClass>2 ){ - iScope = nClass-1; - }else{ - iScope = nClass; - } - } - memcpy(zHash1, zClass, iScope); - sqlite3_free(zClass); - zHash1[iScope] = 0; - memcpy(zHash2, zHash1, iScope); - zHash2[iScope] = 'Z'; - zHash2[iScope+1] = 0; -#if SPELLFIX_MX_RUN>1 - for(i=0; i<p->nRun; i++){ - if( strcmp(p->azPrior[i], zHash1)==0 ) return; - } -#endif - assert( p->nRun<SPELLFIX_MX_RUN ); - memcpy(p->azPrior[p->nRun++], zHash1, iScope+1); - if( sqlite3_bind_text(pStmt, 1, zHash1, -1, SQLITE_STATIC)==SQLITE_NOMEM - || sqlite3_bind_text(pStmt, 2, zHash2, -1, SQLITE_STATIC)==SQLITE_NOMEM - ){ - p->rc = SQLITE_NOMEM; - return; - } -#if SPELLFIX_MX_RUN>1 - for(i=0; i<pCur->nRow; i++){ - if( pCur->a[i].iScore>iWorst ){ - iWorst = pCur->a[i].iScore; - idxWorst = i; - } - } -#endif - while( sqlite3_step(pStmt)==SQLITE_ROW ){ - int iMatchlen = -1; - iRank = sqlite3_column_int(pStmt, 2); - if( p->pMatchStr3 ){ - int nWord = sqlite3_column_bytes(pStmt, 1); - zWord = (const char*)sqlite3_column_text(pStmt, 1); - iDist = editDist3Core(p->pMatchStr3, zWord, nWord, p->pLang, &iMatchlen); - }else{ - zK1 = (const char*)sqlite3_column_text(pStmt, 3); - if( zK1==0 ) continue; - iDist = editdist1(p->zPattern, zK1, 0); - } - if( iDist<0 ){ - p->rc = SQLITE_NOMEM; - break; - } - pCur->nSearch++; - iScore = spellfix1Score(iDist,iRank); - if( p->iMaxDist>=0 ){ - if( iDist>p->iMaxDist ) continue; - if( pCur->nRow>=pCur->nAlloc-1 ){ - spellfix1ResizeCursor(pCur, pCur->nAlloc*2 + 10); - if( pCur->a==0 ) break; - } - idx = pCur->nRow; - }else if( pCur->nRow<pCur->nAlloc ){ - idx = pCur->nRow; - }else if( iScore<iWorst ){ - idx = idxWorst; - sqlite3_free(pCur->a[idx].zWord); - }else{ - continue; - } - pCur->a[idx].zWord = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 1)); - if( pCur->a[idx].zWord==0 ){ - p->rc = SQLITE_NOMEM; - break; - } - pCur->a[idx].iRowid = sqlite3_column_int64(pStmt, 0); - pCur->a[idx].iRank = iRank; - pCur->a[idx].iDistance = iDist; - pCur->a[idx].iScore = iScore; - pCur->a[idx].iMatchlen = iMatchlen; - memcpy(pCur->a[idx].zHash, zHash1, iScope+1); - if( pCur->nRow<pCur->nAlloc ) pCur->nRow++; - if( pCur->nRow==pCur->nAlloc ){ - iWorst = pCur->a[0].iScore; - idxWorst = 0; - for(i=1; i<pCur->nRow; i++){ - iScore = pCur->a[i].iScore; - if( iWorst<iScore ){ - iWorst = iScore; - idxWorst = i; - } - } - } - } - rc = sqlite3_reset(pStmt); - if( rc ) p->rc = rc; -} - -/* -** This version of the xFilter method work if the MATCH term is present -** and we are doing a scan. -*/ -static int spellfix1FilterForMatch( - spellfix1_cursor *pCur, - int idxNum, - int argc, - sqlite3_value **argv -){ - const unsigned char *zMatchThis; /* RHS of the MATCH operator */ - EditDist3FromString *pMatchStr3 = 0; /* zMatchThis as an editdist string */ - char *zPattern; /* Transliteration of zMatchThis */ - int nPattern; /* Length of zPattern */ - int iLimit = 20; /* Max number of rows of output */ - int iScope = 3; /* Use this many characters of zClass */ - int iLang = 0; /* Language code */ - char *zSql; /* SQL of shadow table query */ - sqlite3_stmt *pStmt = 0; /* Shadow table query */ - int rc; /* Result code */ - int idx = 1; /* Next available filter parameter */ - spellfix1_vtab *p = pCur->pVTab; /* The virtual table that owns pCur */ - MatchQuery x; /* For passing info to RunQuery() */ - - /* Load the cost table if we have not already done so */ - if( p->zCostTable!=0 && p->pConfig3==0 ){ - p->pConfig3 = sqlite3_malloc( sizeof(p->pConfig3[0]) ); - if( p->pConfig3==0 ) return SQLITE_NOMEM; - memset(p->pConfig3, 0, sizeof(p->pConfig3[0])); - rc = editDist3ConfigLoad(p->pConfig3, p->db, p->zCostTable); - if( rc ) return rc; - } - memset(&x, 0, sizeof(x)); - x.iScope = 3; /* Default scope if none specified by "WHERE scope=N" */ - x.iMaxDist = -1; /* Maximum allowed edit distance */ - - if( idxNum&2 ){ - iLang = sqlite3_value_int(argv[idx++]); - } - if( idxNum&4 ){ - iLimit = sqlite3_value_int(argv[idx++]); - if( iLimit<1 ) iLimit = 1; - } - if( idxNum&8 ){ - x.iScope = sqlite3_value_int(argv[idx++]); - if( x.iScope<1 ) x.iScope = 1; - if( x.iScope>SPELLFIX_MX_HASH-2 ) x.iScope = SPELLFIX_MX_HASH-2; - } - if( idxNum&(16|32) ){ - x.iMaxDist = sqlite3_value_int(argv[idx++]); - if( idxNum&16 ) x.iMaxDist--; - if( x.iMaxDist<0 ) x.iMaxDist = 0; - } - spellfix1ResetCursor(pCur); - spellfix1ResizeCursor(pCur, iLimit); - zMatchThis = sqlite3_value_text(argv[0]); - if( zMatchThis==0 ) return SQLITE_OK; - if( p->pConfig3 ){ - x.pLang = editDist3FindLang(p->pConfig3, iLang); - pMatchStr3 = editDist3FromStringNew(x.pLang, (const char*)zMatchThis, -1); - if( pMatchStr3==0 ){ - x.rc = SQLITE_NOMEM; - goto filter_exit; - } - }else{ - x.pLang = 0; - } - zPattern = (char*)transliterate(zMatchThis, sqlite3_value_bytes(argv[0])); - sqlite3_free(pCur->zPattern); - pCur->zPattern = zPattern; - if( zPattern==0 ){ - x.rc = SQLITE_NOMEM; - goto filter_exit; - } - nPattern = (int)strlen(zPattern); - if( zPattern[nPattern-1]=='*' ) nPattern--; - zSql = sqlite3_mprintf( - "SELECT id, word, rank, k1" - " FROM \"%w\".\"%w_vocab\"" - " WHERE langid=%d AND k2>=?1 AND k2<?2", - p->zDbName, p->zTableName, iLang - ); - if( zSql==0 ){ - x.rc = SQLITE_NOMEM; - pStmt = 0; - goto filter_exit; - } - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); - sqlite3_free(zSql); - pCur->iLang = iLang; - x.pCur = pCur; - x.pStmt = pStmt; - x.zPattern = zPattern; - x.nPattern = nPattern; - x.pMatchStr3 = pMatchStr3; - x.iLang = iLang; - x.rc = rc; - x.pConfig3 = p->pConfig3; - if( x.rc==SQLITE_OK ){ - spellfix1RunQuery(&x, zPattern, nPattern); - } - - if( pCur->a ){ - qsort(pCur->a, pCur->nRow, sizeof(pCur->a[0]), spellfix1RowCompare); - pCur->iTop = iLimit; - pCur->iScope = iScope; - }else{ - x.rc = SQLITE_NOMEM; - } - -filter_exit: - sqlite3_finalize(pStmt); - editDist3FromStringDelete(pMatchStr3); - return x.rc; -} - -/* -** This version of xFilter handles a full-table scan case -*/ -static int spellfix1FilterForFullScan( - spellfix1_cursor *pCur, - int idxNum, - int argc, - sqlite3_value **argv -){ - int rc; - char *zSql; - spellfix1_vtab *pVTab = pCur->pVTab; - spellfix1ResetCursor(pCur); - zSql = sqlite3_mprintf( - "SELECT word, rank, NULL, langid, id FROM \"%w\".\"%w_vocab\"", - pVTab->zDbName, pVTab->zTableName); - if( zSql==0 ) return SQLITE_NOMEM; - rc = sqlite3_prepare_v2(pVTab->db, zSql, -1, &pCur->pFullScan, 0); - sqlite3_free(zSql); - pCur->nRow = pCur->iRow = 0; - if( rc==SQLITE_OK ){ - rc = sqlite3_step(pCur->pFullScan); - if( rc==SQLITE_ROW ){ pCur->iRow = -1; rc = SQLITE_OK; } - if( rc==SQLITE_DONE ){ rc = SQLITE_OK; } - }else{ - pCur->iRow = 0; - } - return rc; -} - - -/* -** Called to "rewind" a cursor back to the beginning so that -** it starts its output over again. Always called at least once -** prior to any spellfix1Column, spellfix1Rowid, or spellfix1Eof call. -*/ -static int spellfix1Filter( - sqlite3_vtab_cursor *cur, - int idxNum, const char *idxStr, - int argc, sqlite3_value **argv -){ - spellfix1_cursor *pCur = (spellfix1_cursor *)cur; - int rc; - if( idxNum & 1 ){ - rc = spellfix1FilterForMatch(pCur, idxNum, argc, argv); - }else{ - rc = spellfix1FilterForFullScan(pCur, idxNum, argc, argv); - } - return rc; -} - - -/* -** Advance a cursor to its next row of output -*/ -static int spellfix1Next(sqlite3_vtab_cursor *cur){ - spellfix1_cursor *pCur = (spellfix1_cursor *)cur; - int rc = SQLITE_OK; - if( pCur->iRow < pCur->nRow ){ - if( pCur->pFullScan ){ - rc = sqlite3_step(pCur->pFullScan); - if( rc!=SQLITE_ROW ) pCur->iRow = pCur->nRow; - if( rc==SQLITE_ROW || rc==SQLITE_DONE ) rc = SQLITE_OK; - }else{ - pCur->iRow++; - } - } - return rc; -} - -/* -** Return TRUE if we are at the end-of-file -*/ -static int spellfix1Eof(sqlite3_vtab_cursor *cur){ - spellfix1_cursor *pCur = (spellfix1_cursor *)cur; - return pCur->iRow>=pCur->nRow; -} - -/* -** Return columns from the current row. -*/ -static int spellfix1Column( - sqlite3_vtab_cursor *cur, - sqlite3_context *ctx, - int i -){ - spellfix1_cursor *pCur = (spellfix1_cursor*)cur; - if( pCur->pFullScan ){ - if( i<=SPELLFIX_COL_LANGID ){ - sqlite3_result_value(ctx, sqlite3_column_value(pCur->pFullScan, i)); - }else{ - sqlite3_result_null(ctx); - } - return SQLITE_OK; - } - switch( i ){ - case SPELLFIX_COL_WORD: { - sqlite3_result_text(ctx, pCur->a[pCur->iRow].zWord, -1, SQLITE_STATIC); - break; - } - case SPELLFIX_COL_RANK: { - sqlite3_result_int(ctx, pCur->a[pCur->iRow].iRank); - break; - } - case SPELLFIX_COL_DISTANCE: { - sqlite3_result_int(ctx, pCur->a[pCur->iRow].iDistance); - break; - } - case SPELLFIX_COL_LANGID: { - sqlite3_result_int(ctx, pCur->iLang); - break; - } - case SPELLFIX_COL_SCORE: { - sqlite3_result_int(ctx, pCur->a[pCur->iRow].iScore); - break; - } - case SPELLFIX_COL_MATCHLEN: { - int iMatchlen = pCur->a[pCur->iRow].iMatchlen; - if( iMatchlen<0 ){ - int nPattern = (int)strlen(pCur->zPattern); - char *zWord = pCur->a[pCur->iRow].zWord; - int nWord = (int)strlen(zWord); - - if( nPattern>0 && pCur->zPattern[nPattern-1]=='*' ){ - char *zTranslit; - int res; - zTranslit = (char *)transliterate((unsigned char *)zWord, nWord); - if( !zTranslit ) return SQLITE_NOMEM; - res = editdist1(pCur->zPattern, zTranslit, &iMatchlen); - sqlite3_free(zTranslit); - if( res<0 ) return SQLITE_NOMEM; - iMatchlen = translen_to_charlen(zWord, nWord, iMatchlen); - }else{ - iMatchlen = utf8Charlen(zWord, nWord); - } - } - - sqlite3_result_int(ctx, iMatchlen); - break; - } - case SPELLFIX_COL_PHONEHASH: { - sqlite3_result_text(ctx, pCur->a[pCur->iRow].zHash, -1, SQLITE_STATIC); - break; - } - case SPELLFIX_COL_TOP: { - sqlite3_result_int(ctx, pCur->iTop); - break; - } - case SPELLFIX_COL_SCOPE: { - sqlite3_result_int(ctx, pCur->iScope); - break; - } - case SPELLFIX_COL_SRCHCNT: { - sqlite3_result_int(ctx, pCur->nSearch); - break; - } - default: { - sqlite3_result_null(ctx); - break; - } - } - return SQLITE_OK; -} - -/* -** The rowid. -*/ -static int spellfix1Rowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ - spellfix1_cursor *pCur = (spellfix1_cursor*)cur; - if( pCur->pFullScan ){ - *pRowid = sqlite3_column_int64(pCur->pFullScan, 4); - }else{ - *pRowid = pCur->a[pCur->iRow].iRowid; - } - return SQLITE_OK; -} - -/* -** The xUpdate() method. -*/ -static int spellfix1Update( - sqlite3_vtab *pVTab, - int argc, - sqlite3_value **argv, - sqlite_int64 *pRowid -){ - int rc = SQLITE_OK; - sqlite3_int64 rowid, newRowid; - spellfix1_vtab *p = (spellfix1_vtab*)pVTab; - sqlite3 *db = p->db; - - if( argc==1 ){ - /* A delete operation on the rowid given by argv[0] */ - rowid = *pRowid = sqlite3_value_int64(argv[0]); - spellfix1DbExec(&rc, db, "DELETE FROM \"%w\".\"%w_vocab\" " - " WHERE id=%lld", - p->zDbName, p->zTableName, rowid); - }else{ - const unsigned char *zWord = sqlite3_value_text(argv[SPELLFIX_COL_WORD+2]); - int nWord = sqlite3_value_bytes(argv[SPELLFIX_COL_WORD+2]); - int iLang = sqlite3_value_int(argv[SPELLFIX_COL_LANGID+2]); - int iRank = sqlite3_value_int(argv[SPELLFIX_COL_RANK+2]); - const unsigned char *zSoundslike = - sqlite3_value_text(argv[SPELLFIX_COL_SOUNDSLIKE+2]); - int nSoundslike = sqlite3_value_bytes(argv[SPELLFIX_COL_SOUNDSLIKE+2]); - char *zK1, *zK2; - int i; - char c; - - if( zWord==0 ){ - /* Inserts of the form: INSERT INTO table(command) VALUES('xyzzy'); - ** cause zWord to be NULL, so we look at the "command" column to see - ** what special actions to take */ - const char *zCmd = - (const char*)sqlite3_value_text(argv[SPELLFIX_COL_COMMAND+2]); - if( zCmd==0 ){ - pVTab->zErrMsg = sqlite3_mprintf("%s.word may not be NULL", - p->zTableName); - return SQLITE_CONSTRAINT; - } - if( strcmp(zCmd,"reset")==0 ){ - /* Reset the edit cost table (if there is one). */ - editDist3ConfigDelete(p->pConfig3); - p->pConfig3 = 0; - return SQLITE_OK; - } - pVTab->zErrMsg = sqlite3_mprintf("unknown value for %s.command: \"%w\"", - p->zTableName, zCmd); - return SQLITE_ERROR; - } - if( iRank<1 ) iRank = 1; - if( zSoundslike ){ - zK1 = (char*)transliterate(zSoundslike, nSoundslike); - }else{ - zK1 = (char*)transliterate(zWord, nWord); - } - if( zK1==0 ) return SQLITE_NOMEM; - for(i=0; (c = zK1[i])!=0; i++){ - if( c>='A' && c<='Z' ) zK1[i] += 'a' - 'A'; - } - zK2 = (char*)phoneticHash((const unsigned char*)zK1, i); - if( zK2==0 ){ - sqlite3_free(zK1); - return SQLITE_NOMEM; - } - if( sqlite3_value_type(argv[0])==SQLITE_NULL ){ - spellfix1DbExec(&rc, db, - "INSERT INTO \"%w\".\"%w_vocab\"(rank,langid,word,k1,k2) " - "VALUES(%d,%d,%Q,%Q,%Q)", - p->zDbName, p->zTableName, - iRank, iLang, zWord, zK1, zK2 - ); - *pRowid = sqlite3_last_insert_rowid(db); - }else{ - rowid = sqlite3_value_int64(argv[0]); - newRowid = *pRowid = sqlite3_value_int64(argv[1]); - spellfix1DbExec(&rc, db, - "UPDATE \"%w\".\"%w_vocab\" SET id=%lld, rank=%d, langid=%d," - " word=%Q, k1=%Q, k2=%Q WHERE id=%lld", - p->zDbName, p->zTableName, newRowid, iRank, iLang, - zWord, zK1, zK2, rowid - ); - } - sqlite3_free(zK1); - sqlite3_free(zK2); - } - return rc; -} - -/* -** Rename the spellfix1 table. -*/ -static int spellfix1Rename(sqlite3_vtab *pVTab, const char *zNew){ - spellfix1_vtab *p = (spellfix1_vtab*)pVTab; - sqlite3 *db = p->db; - int rc = SQLITE_OK; - char *zNewName = sqlite3_mprintf("%s", zNew); - if( zNewName==0 ){ - return SQLITE_NOMEM; - } - spellfix1DbExec(&rc, db, - "ALTER TABLE \"%w\".\"%w_vocab\" RENAME TO \"%w_vocab\"", - p->zDbName, p->zTableName, zNewName - ); - if( rc==SQLITE_OK ){ - sqlite3_free(p->zTableName); - p->zTableName = zNewName; - }else{ - sqlite3_free(zNewName); - } - return rc; -} - - -/* -** A virtual table module that provides fuzzy search. -*/ -static sqlite3_module spellfix1Module = { - 0, /* iVersion */ - spellfix1Create, /* xCreate - handle CREATE VIRTUAL TABLE */ - spellfix1Connect, /* xConnect - reconnected to an existing table */ - spellfix1BestIndex, /* xBestIndex - figure out how to do a query */ - spellfix1Disconnect, /* xDisconnect - close a connection */ - spellfix1Destroy, /* xDestroy - handle DROP TABLE */ - spellfix1Open, /* xOpen - open a cursor */ - spellfix1Close, /* xClose - close a cursor */ - spellfix1Filter, /* xFilter - configure scan constraints */ - spellfix1Next, /* xNext - advance a cursor */ - spellfix1Eof, /* xEof - check for end of scan */ - spellfix1Column, /* xColumn - read data */ - spellfix1Rowid, /* xRowid - read data */ - spellfix1Update, /* xUpdate */ - 0, /* xBegin */ - 0, /* xSync */ - 0, /* xCommit */ - 0, /* xRollback */ - 0, /* xFindMethod */ - spellfix1Rename, /* xRename */ -}; - -/* -** Register the various functions and the virtual table. -*/ -static int spellfix1Register(sqlite3 *db){ - int rc = SQLITE_OK; - int i; - rc = sqlite3_create_function(db, "spellfix1_translit", 1, SQLITE_UTF8, 0, - transliterateSqlFunc, 0, 0); - if( rc==SQLITE_OK ){ - rc = sqlite3_create_function(db, "spellfix1_editdist", 2, SQLITE_UTF8, 0, - editdistSqlFunc, 0, 0); - } - if( rc==SQLITE_OK ){ - rc = sqlite3_create_function(db, "spellfix1_phonehash", 1, SQLITE_UTF8, 0, - phoneticHashSqlFunc, 0, 0); - } - if( rc==SQLITE_OK ){ - rc = sqlite3_create_function(db, "spellfix1_scriptcode", 1, SQLITE_UTF8, 0, - scriptCodeSqlFunc, 0, 0); - } - if( rc==SQLITE_OK ){ - rc = sqlite3_create_module(db, "spellfix1", &spellfix1Module, 0); - } - if( rc==SQLITE_OK ){ - rc = editDist3Install(db); - } - - /* Verify sanity of the translit[] table */ - for(i=0; i<sizeof(translit)/sizeof(translit[0])-1; i++){ - assert( translit[i].cFrom<translit[i+1].cFrom ); - } - - return rc; -} - -#if SQLITE_CORE || defined(SQLITE_TEST) -/* -** Register the spellfix1 virtual table and its associated functions. -*/ -int sqlite3Spellfix1Register(sqlite3 *db){ - return spellfix1Register(db); -} -#endif - - -#if !SQLITE_CORE -/* -** Extension load function. -*/ -int sqlite3_extension_init( - sqlite3 *db, - char **pzErrMsg, - const sqlite3_api_routines *pApi -){ - SQLITE_EXTENSION_INIT2(pApi); - return spellfix1Register(db); -} -#endif /* !SQLITE_CORE */ diff --git a/src/test_sqllog.c b/src/test_sqllog.c new file mode 100644 index 0000000..4aa68b7 --- /dev/null +++ b/src/test_sqllog.c @@ -0,0 +1,507 @@ +/* +** 2012 November 26 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** OVERVIEW +** +** This file contains experimental code used to record data from live +** SQLite applications that may be useful for offline analysis. +** Specifically, this module can be used to capture the following +** information: +** +** 1) The initial contents of all database files opened by the +** application, and +** +** 2) All SQL statements executed by the application. +** +** The captured information can then be used to run (for example) +** performance analysis looking for slow queries or to look for +** optimization opportunities in either the application or in SQLite +** itself. +** +** USAGE +** +** To use this module, SQLite must be compiled with the SQLITE_ENABLE_SQLLOG +** pre-processor symbol defined and this file linked into the application. +** One way to link this file into the application is to append the content +** of this file onto the end of the "sqlite3.c" amalgamation and then +** recompile the application as normal except with the addition of the +** -DSQLITE_ENABLE_SQLLOG option. +** +** At runtime, logging is enabled by setting environment variable +** SQLITE_SQLLOG_DIR to the name of a directory in which to store logged +** data. The logging directory must already exist. +** +** Usually, if the application opens the same database file more than once +** (either by attaching it or by using more than one database handle), only +** a single copy is made. This behavior may be overridden (so that a +** separate copy is taken each time the database file is opened or attached) +** by setting the environment variable SQLITE_SQLLOG_REUSE_FILES to 0. +** +** OUTPUT: +** +** The SQLITE_SQLLOG_DIR is populated with three types of files: +** +** sqllog_N.db - Copies of database files. N may be any integer. +** +** sqllog_N.sql - A list of SQL statements executed by a single +** connection. N may be any integer. +** +** sqllog.idx - An index mapping from integer N to a database +** file name - indicating the full path of the +** database from which sqllog_N.db was copied. +** +** ERROR HANDLING: +** +** This module attempts to make a best effort to continue logging if an +** IO or other error is encountered. For example, if a log file cannot +** be opened logs are not collected for that connection, but other +** logging proceeds as expected. Errors are logged by calling sqlite3_log(). +*/ + +#ifndef _SQLITE3_H_ +#include "sqlite3.h" +#endif +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <assert.h> + +#include <sys/types.h> +#include <unistd.h> +static int getProcessId(void){ +#if SQLITE_OS_WIN + return (int)_getpid(); +#else + return (int)getpid(); +#endif +} + +/* Names of environment variables to be used */ +#define ENVIRONMENT_VARIABLE1_NAME "SQLITE_SQLLOG_DIR" +#define ENVIRONMENT_VARIABLE2_NAME "SQLITE_SQLLOG_REUSE_FILES" + +/* Assume that all database and database file names are shorted than this. */ +#define SQLLOG_NAMESZ 512 + +/* Maximum number of simultaneous database connections the process may +** open (if any more are opened an error is logged using sqlite3_log() +** and processing is halted). +*/ +#define MAX_CONNECTIONS 256 + +/* There is one instance of this object for each SQLite database connection +** that is being logged. +*/ +struct SLConn { + int isErr; /* True if an error has occurred */ + sqlite3 *db; /* Connection handle */ + int iLog; /* First integer value used in file names */ + FILE *fd; /* File descriptor for log file */ +}; + +/* This object is a singleton that keeps track of all data loggers. +*/ +static struct SLGlobal { + /* Protected by MUTEX_STATIC_MASTER */ + sqlite3_mutex *mutex; /* Recursive mutex */ + int nConn; /* Size of aConn[] array */ + + /* Protected by SLGlobal.mutex */ + int bReuse; /* True to avoid extra copies of db files */ + char zPrefix[SQLLOG_NAMESZ]; /* Prefix for all created files */ + char zIdx[SQLLOG_NAMESZ]; /* Full path to *.idx file */ + int iNextLog; /* Used to allocate file names */ + int iNextDb; /* Used to allocate database file names */ + int bRec; /* True if testSqllog() is called rec. */ + int iClock; /* Clock value */ + struct SLConn aConn[MAX_CONNECTIONS]; +} sqllogglobal; + +/* +** Return true if c is an ASCII whitespace character. +*/ +static int sqllog_isspace(char c){ + return (c==' ' || c=='\t' || c=='\n' || c=='\v' || c=='\f' || c=='\r'); +} + +/* +** The first argument points to a nul-terminated string containing an SQL +** command. Before returning, this function sets *pz to point to the start +** of the first token in this command, and *pn to the number of bytes in +** the token. This is used to check if the SQL command is an "ATTACH" or +** not. +*/ +static void sqllogTokenize(const char *z, const char **pz, int *pn){ + const char *p = z; + int n; + + /* Skip past any whitespace */ + while( sqllog_isspace(*p) ){ + p++; + } + + /* Figure out how long the first token is */ + *pz = p; + n = 0; + while( (p[n]>='a' && p[n]<='z') || (p[n]>='A' && p[n]<='Z') ) n++; + *pn = n; +} + +/* +** Check if the logs directory already contains a copy of database file +** zFile. If so, return a pointer to the full path of the copy. Otherwise, +** return NULL. +** +** If a non-NULL value is returned, then the caller must arrange to +** eventually free it using sqlite3_free(). +*/ +static char *sqllogFindFile(const char *zFile){ + char *zRet = 0; + FILE *fd = 0; + + /* Open the index file for reading */ + fd = fopen(sqllogglobal.zIdx, "r"); + if( fd==0 ){ + sqlite3_log(SQLITE_IOERR, "sqllogFindFile(): error in fopen()"); + return 0; + } + + /* Loop through each entry in the index file. If zFile is not NULL and the + ** entry is a match, then set zRet to point to the filename of the existing + ** copy and break out of the loop. */ + while( feof(fd)==0 ){ + char zLine[SQLLOG_NAMESZ*2+5]; + if( fgets(zLine, sizeof(zLine), fd) ){ + int n; + char *z; + + zLine[sizeof(zLine)-1] = '\0'; + z = zLine; + while( *z>='0' && *z<='9' ) z++; + while( *z==' ' ) z++; + + n = strlen(z); + while( n>0 && sqllog_isspace(z[n-1]) ) n--; + + if( n==strlen(zFile) && 0==memcmp(zFile, z, n) ){ + char zBuf[16]; + memset(zBuf, 0, sizeof(zBuf)); + z = zLine; + while( *z>='0' && *z<='9' ){ + zBuf[z-zLine] = *z; + z++; + } + zRet = sqlite3_mprintf("%s_%s.db", sqllogglobal.zPrefix, zBuf); + break; + } + } + } + + if( ferror(fd) ){ + sqlite3_log(SQLITE_IOERR, "sqllogFindFile(): error reading index file"); + } + + fclose(fd); + return zRet; +} + +static int sqllogFindAttached( + struct SLConn *p, /* Database connection */ + const char *zSearch, /* Name to search for (or NULL) */ + char *zName, /* OUT: Name of attached database */ + char *zFile /* OUT: Name of attached file */ +){ + sqlite3_stmt *pStmt; + int rc; + + /* The "PRAGMA database_list" command returns a list of databases in the + ** order that they were attached. So a newly attached database is + ** described by the last row returned. */ + assert( sqllogglobal.bRec==0 ); + sqllogglobal.bRec = 1; + rc = sqlite3_prepare_v2(p->db, "PRAGMA database_list", -1, &pStmt, 0); + if( rc==SQLITE_OK ){ + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + const char *zVal1; int nVal1; + const char *zVal2; int nVal2; + + zVal1 = (const char*)sqlite3_column_text(pStmt, 1); + nVal1 = sqlite3_column_bytes(pStmt, 1); + memcpy(zName, zVal1, nVal1+1); + + zVal2 = (const char*)sqlite3_column_text(pStmt, 2); + nVal2 = sqlite3_column_bytes(pStmt, 2); + memcpy(zFile, zVal2, nVal2+1); + + if( zSearch && strlen(zSearch)==nVal1 + && 0==sqlite3_strnicmp(zSearch, zVal1, nVal1) + ){ + break; + } + } + rc = sqlite3_finalize(pStmt); + } + sqllogglobal.bRec = 0; + + if( rc!=SQLITE_OK ){ + sqlite3_log(rc, "sqllogFindAttached(): error in \"PRAGMA database_list\""); + } + return rc; +} + + +/* +** Parameter zSearch is the name of a database attached to the database +** connection associated with the first argument. This function creates +** a backup of this database in the logs directory. +** +** The name used for the backup file is automatically generated. Call +** it zFile. +** +** If the bLog parameter is true, then a statement of the following form +** is written to the log file associated with *p: +** +** ATTACH 'zFile' AS 'zName'; +** +** Otherwise, if bLog is false, a comment is added to the log file: +** +** -- Main database file is 'zFile' +** +** The SLGlobal.mutex mutex is always held when this function is called. +*/ +static void sqllogCopydb(struct SLConn *p, const char *zSearch, int bLog){ + char zName[SQLLOG_NAMESZ]; /* Attached database name */ + char zFile[SQLLOG_NAMESZ]; /* Database file name */ + char *zFree; + char *zInit = 0; + int rc; + + rc = sqllogFindAttached(p, zSearch, zName, zFile); + if( rc!=SQLITE_OK ) return; + + if( zFile[0]=='\0' ){ + zInit = sqlite3_mprintf(""); + }else{ + if( sqllogglobal.bReuse ){ + zInit = sqllogFindFile(zFile); + }else{ + zInit = 0; + } + if( zInit==0 ){ + int rc; + sqlite3 *copy = 0; + int iDb; + + /* Generate a file-name to use for the copy of this database */ + iDb = sqllogglobal.iNextDb++; + zInit = sqlite3_mprintf("%s_%d.db", sqllogglobal.zPrefix, iDb); + + /* Create the backup */ + assert( sqllogglobal.bRec==0 ); + sqllogglobal.bRec = 1; + rc = sqlite3_open(zInit, ©); + if( rc==SQLITE_OK ){ + sqlite3_backup *pBak; + sqlite3_exec(copy, "PRAGMA synchronous = 0", 0, 0, 0); + pBak = sqlite3_backup_init(copy, "main", p->db, zName); + if( pBak ){ + sqlite3_backup_step(pBak, -1); + rc = sqlite3_backup_finish(pBak); + }else{ + rc = sqlite3_errcode(copy); + } + sqlite3_close(copy); + } + sqllogglobal.bRec = 0; + + if( rc==SQLITE_OK ){ + /* Write an entry into the database index file */ + FILE *fd = fopen(sqllogglobal.zIdx, "a"); + if( fd ){ + fprintf(fd, "%d %s\n", iDb, zFile); + fclose(fd); + } + }else{ + sqlite3_log(rc, "sqllogCopydb(): error backing up database"); + } + } + } + + if( bLog ){ + zFree = sqlite3_mprintf("ATTACH '%q' AS '%q'; -- clock=%d\n", + zInit, zName, sqllogglobal.iClock++ + ); + }else{ + zFree = sqlite3_mprintf("-- Main database is '%q'\n", zInit); + } + fprintf(p->fd, "%s", zFree); + sqlite3_free(zFree); + + sqlite3_free(zInit); +} + +/* +** If it is not already open, open the log file for connection *p. +** +** The SLGlobal.mutex mutex is always held when this function is called. +*/ +static void sqllogOpenlog(struct SLConn *p){ + /* If the log file has not yet been opened, open it now. */ + if( p->fd==0 ){ + char *zLog; + + /* If it is still NULL, have global.zPrefix point to a copy of + ** environment variable $ENVIRONMENT_VARIABLE1_NAME. */ + if( sqllogglobal.zPrefix[0]==0 ){ + FILE *fd; + char *zVar = getenv(ENVIRONMENT_VARIABLE1_NAME); + if( zVar==0 || strlen(zVar)+10>=(sizeof(sqllogglobal.zPrefix)) ) return; + sprintf(sqllogglobal.zPrefix, "%s/sqllog_%d", zVar, getProcessId()); + sprintf(sqllogglobal.zIdx, "%s.idx", sqllogglobal.zPrefix); + if( getenv(ENVIRONMENT_VARIABLE2_NAME) ){ + sqllogglobal.bReuse = atoi(getenv(ENVIRONMENT_VARIABLE2_NAME)); + } + fd = fopen(sqllogglobal.zIdx, "w"); + if( fd ) fclose(fd); + } + + /* Open the log file */ + zLog = sqlite3_mprintf("%s_%d.sql", sqllogglobal.zPrefix, p->iLog); + p->fd = fopen(zLog, "w"); + sqlite3_free(zLog); + if( p->fd==0 ){ + sqlite3_log(SQLITE_IOERR, "sqllogOpenlog(): Failed to open log file"); + } + } +} + +/* +** This function is called if the SQLLOG callback is invoked to report +** execution of an SQL statement. Parameter p is the connection the statement +** was executed by and parameter zSql is the text of the statement itself. +*/ +static void testSqllogStmt(struct SLConn *p, const char *zSql){ + const char *zFirst; /* Pointer to first token in zSql */ + int nFirst; /* Size of token zFirst in bytes */ + + sqllogTokenize(zSql, &zFirst, &nFirst); + if( nFirst!=6 || 0!=sqlite3_strnicmp("ATTACH", zFirst, 6) ){ + /* Not an ATTACH statement. Write this directly to the log. */ + fprintf(p->fd, "%s; -- clock=%d\n", zSql, sqllogglobal.iClock++); + }else{ + /* This is an ATTACH statement. Copy the database. */ + sqllogCopydb(p, 0, 1); + } +} + +/* +** The SQLITE_CONFIG_SQLLOG callback registered by sqlite3_init_sqllog(). +** +** The eType parameter has the following values: +** +** 0: Opening a new database connection. zSql is the name of the +** file being opened. db is a pointer to the newly created database +** connection. +** +** 1: An SQL statement has run to completion. zSql is the text of the +** SQL statement with all parameters expanded to their actual values. +** +** 2: Closing a database connection. zSql is NULL. The db pointer to +** the database connection being closed has already been shut down +** and cannot be used for any further SQL. +** +** The pCtx parameter is a copy of the pointer that was originally passed +** into the sqlite3_config(SQLITE_CONFIG_SQLLOG) statement. In this +** particular implementation, pCtx is always a pointer to the +** sqllogglobal global variable define above. +*/ +static void testSqllog(void *pCtx, sqlite3 *db, const char *zSql, int eType){ + struct SLConn *p = 0; + sqlite3_mutex *master = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER); + + assert( eType==0 || eType==1 || eType==2 ); + assert( (eType==2)==(zSql==0) ); + + /* This is a database open command. */ + if( eType==0 ){ + sqlite3_mutex_enter(master); + if( sqllogglobal.mutex==0 ){ + sqllogglobal.mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_RECURSIVE); + } + p = &sqllogglobal.aConn[sqllogglobal.nConn++]; + p->fd = 0; + p->db = db; + p->iLog = sqllogglobal.iNextLog++; + sqlite3_mutex_leave(master); + + /* Open the log and take a copy of the main database file */ + sqlite3_mutex_enter(sqllogglobal.mutex); + if( sqllogglobal.bRec==0 ){ + sqllogOpenlog(p); + if( p->fd ) sqllogCopydb(p, "main", 0); + } + sqlite3_mutex_leave(sqllogglobal.mutex); + } + + else{ + + int i; + for(i=0; i<sqllogglobal.nConn; i++){ + p = &sqllogglobal.aConn[i]; + if( p->db==db ) break; + } + if( i==sqllogglobal.nConn ) return; + + /* A database handle close command */ + if( eType==2 ){ + sqlite3_mutex_enter(master); + if( p->fd ) fclose(p->fd); + p->db = 0; + p->fd = 0; + + sqllogglobal.nConn--; + if( sqllogglobal.nConn==0 ){ + sqlite3_mutex_free(sqllogglobal.mutex); + sqllogglobal.mutex = 0; + }else{ + int nShift = &sqllogglobal.aConn[sqllogglobal.nConn] - p; + if( nShift>0 ){ + memmove(p, &p[1], nShift*sizeof(struct SLConn)); + } + } + sqlite3_mutex_leave(master); + + /* An ordinary SQL command. */ + }else if( p->fd ){ + sqlite3_mutex_enter(sqllogglobal.mutex); + if( sqllogglobal.bRec==0 ){ + testSqllogStmt(p, zSql); + } + sqlite3_mutex_leave(sqllogglobal.mutex); + } + } +} + +/* +** This function is called either before sqlite3_initialized() or by it. +** It checks if the SQLITE_SQLLOG_DIR variable is defined, and if so +** registers an SQLITE_CONFIG_SQLLOG callback to record the applications +** database activity. +*/ +void sqlite3_init_sqllog(void){ + if( getenv(ENVIRONMENT_VARIABLE1_NAME) ){ + if( SQLITE_OK==sqlite3_config(SQLITE_CONFIG_SQLLOG, testSqllog, 0) ){ + memset(&sqllogglobal, 0, sizeof(sqllogglobal)); + sqllogglobal.bReuse = 1; + } + } +} diff --git a/src/test_syscall.c b/src/test_syscall.c index d484f22..7c0873c 100644 --- a/src/test_syscall.c +++ b/src/test_syscall.c @@ -23,7 +23,7 @@ ** ** open close access getcwd stat fstat ** ftruncate fcntl read pread pread64 write -** pwrite pwrite64 fchmod fallocate +** pwrite pwrite64 fchmod fallocate mmap ** ** test_syscall uninstall ** Uninstall all wrapper functions. @@ -69,18 +69,19 @@ ** the xNextSystemCall() VFS method. */ +#include "sqliteInt.h" #include "sqlite3.h" #include "tcl.h" #include <stdlib.h> #include <string.h> #include <assert.h> -#include "sqliteInt.h" #if SQLITE_OS_UNIX -/* From test1.c */ -extern const char *sqlite3TestErrorName(int); +/* From main.c */ +extern const char *sqlite3ErrName(int); +#include <sys/mman.h> #include <sys/types.h> #include <errno.h> @@ -106,7 +107,8 @@ static int ts_pwrite(int fd, const void *aBuf, size_t nBuf, off_t off); static int ts_pwrite64(int fd, const void *aBuf, size_t nBuf, off_t off); static int ts_fchmod(int fd, mode_t mode); static int ts_fallocate(int fd, off_t off, off_t len); - +static void *ts_mmap(void *, size_t, int, int, int, off_t); +static void *ts_mremap(void*, size_t, size_t, int, ...); struct TestSyscallArray { const char *zName; @@ -131,6 +133,8 @@ struct TestSyscallArray { /* 13 */ { "pwrite64", (sqlite3_syscall_ptr)ts_pwrite64, 0, 0, 0 }, /* 14 */ { "fchmod", (sqlite3_syscall_ptr)ts_fchmod, 0, 0, 0 }, /* 15 */ { "fallocate", (sqlite3_syscall_ptr)ts_fallocate, 0, 0, 0 }, + /* 16 */ { "mmap", (sqlite3_syscall_ptr)ts_mmap, 0, 0, 0 }, + /* 17 */ { "mremap", (sqlite3_syscall_ptr)ts_mremap, 0, 0, 0 }, { 0, 0, 0, 0, 0 } }; @@ -152,6 +156,8 @@ struct TestSyscallArray { aSyscall[13].xOrig) #define orig_fchmod ((int(*)(int,mode_t))aSyscall[14].xOrig) #define orig_fallocate ((int(*)(int,off_t,off_t))aSyscall[15].xOrig) +#define orig_mmap ((void*(*)(void*,size_t,int,int,int,off_t))aSyscall[16].xOrig) +#define orig_mremap ((void*(*)(void*,size_t,size_t,int,...))aSyscall[17].xOrig) /* ** This function is called exactly once from within each invocation of a @@ -377,6 +383,31 @@ static int ts_fallocate(int fd, off_t off, off_t len){ return orig_fallocate(fd, off, len); } +static void *ts_mmap( + void *pAddr, + size_t nByte, + int prot, + int flags, + int fd, + off_t iOff +){ + if( tsIsFailErrno("mmap") ){ + return MAP_FAILED; + } + return orig_mmap(pAddr, nByte, prot, flags, fd, iOff); +} + +static void *ts_mremap(void *a, size_t b, size_t c, int d, ...){ + va_list ap; + void *pArg; + if( tsIsFailErrno("mremap") ){ + return MAP_FAILED; + } + va_start(ap, d); + pArg = va_arg(ap, void *); + return orig_mremap(a, b, c, d, pArg); +} + static int test_syscall_install( void * clientData, Tcl_Interp *interp, @@ -467,7 +498,7 @@ static int test_syscall_reset( } } if( rc!=SQLITE_OK ){ - Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3TestErrorName(rc), -1)); + Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1)); return TCL_ERROR; } diff --git a/src/test_thread.c b/src/test_thread.c index ae62de8..2f9363b 100644 --- a/src/test_thread.c +++ b/src/test_thread.c @@ -60,12 +60,14 @@ static Tcl_ObjCmdProc blocking_prepare_v2_proc; int Sqlitetest1_Init(Tcl_Interp *); int Sqlite3_Init(Tcl_Interp *); +/* Functions from main.c */ +extern const char *sqlite3ErrName(int); + /* Functions from test1.c */ -void *sqlite3TestTextToPtr(const char *); -const char *sqlite3TestErrorName(int); -int getDbPointer(Tcl_Interp *, const char *, sqlite3 **); -int sqlite3TestMakePointerStr(Tcl_Interp *, char *, void *); -int sqlite3TestErrCode(Tcl_Interp *, sqlite3 *, int); +extern void *sqlite3TestTextToPtr(const char *); +extern int getDbPointer(Tcl_Interp *, const char *, sqlite3 **); +extern int sqlite3TestMakePointerStr(Tcl_Interp *, char *, void *); +extern int sqlite3TestErrCode(Tcl_Interp *, sqlite3 *, int); /* ** Handler for events of type EvalEvent. @@ -559,7 +561,7 @@ static int blocking_step_proc( pStmt = (sqlite3_stmt*)sqlite3TestTextToPtr(Tcl_GetString(objv[1])); rc = sqlite3_blocking_step(pStmt); - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), 0); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), 0); return TCL_OK; } @@ -606,7 +608,7 @@ static int blocking_prepare_v2_proc( } if( rc!=SQLITE_OK ){ assert( pStmt==0 ); - sprintf(zBuf, "%s ", (char *)sqlite3TestErrorName(rc)); + sprintf(zBuf, "%s ", (char *)sqlite3ErrName(rc)); Tcl_AppendResult(interp, zBuf, sqlite3_errmsg(db), 0); return TCL_ERROR; } diff --git a/src/test_vfs.c b/src/test_vfs.c index 93c556b..fcd5774 100644 --- a/src/test_vfs.c +++ b/src/test_vfs.c @@ -125,8 +125,9 @@ struct Testvfs { #define TESTVFS_ACCESS_MASK 0x00004000 #define TESTVFS_FULLPATHNAME_MASK 0x00008000 #define TESTVFS_READ_MASK 0x00010000 +#define TESTVFS_UNLOCK_MASK 0x00020000 -#define TESTVFS_ALL_MASK 0x0001FFFF +#define TESTVFS_ALL_MASK 0x0003FFFF #define TESTVFS_MAX_PAGES 1024 @@ -265,7 +266,8 @@ static void tvfsExecTcl( const char *zMethod, Tcl_Obj *arg1, Tcl_Obj *arg2, - Tcl_Obj *arg3 + Tcl_Obj *arg3, + Tcl_Obj *arg4 ){ int rc; /* Return code from Tcl_EvalObj() */ Tcl_Obj *pEval; @@ -282,6 +284,7 @@ static void tvfsExecTcl( if( arg1 ) Tcl_ListObjAppendElement(p->interp, pEval, arg1); if( arg2 ) Tcl_ListObjAppendElement(p->interp, pEval, arg2); if( arg3 ) Tcl_ListObjAppendElement(p->interp, pEval, arg3); + if( arg4 ) Tcl_ListObjAppendElement(p->interp, pEval, arg4); rc = Tcl_EvalObjEx(p->interp, pEval, TCL_EVAL_GLOBAL); if( rc!=TCL_OK ){ @@ -302,7 +305,7 @@ static int tvfsClose(sqlite3_file *pFile){ if( p->pScript && p->mask&TESTVFS_CLOSE_MASK ){ tvfsExecTcl(p, "xClose", - Tcl_NewStringObj(pFd->zFilename, -1), pFd->pShmId, 0 + Tcl_NewStringObj(pFd->zFilename, -1), pFd->pShmId, 0, 0 ); } @@ -333,7 +336,7 @@ static int tvfsRead( Testvfs *p = (Testvfs *)pFd->pVfs->pAppData; if( p->pScript && p->mask&TESTVFS_READ_MASK ){ tvfsExecTcl(p, "xRead", - Tcl_NewStringObj(pFd->zFilename, -1), pFd->pShmId, 0 + Tcl_NewStringObj(pFd->zFilename, -1), pFd->pShmId, 0, 0 ); tvfsResultCode(p, &rc); } @@ -362,7 +365,7 @@ static int tvfsWrite( if( p->pScript && p->mask&TESTVFS_WRITE_MASK ){ tvfsExecTcl(p, "xWrite", Tcl_NewStringObj(pFd->zFilename, -1), pFd->pShmId, - Tcl_NewWideIntObj(iOfst) + Tcl_NewWideIntObj(iOfst), Tcl_NewIntObj(iAmt) ); tvfsResultCode(p, &rc); } @@ -390,7 +393,7 @@ static int tvfsTruncate(sqlite3_file *pFile, sqlite_int64 size){ if( p->pScript && p->mask&TESTVFS_TRUNCATE_MASK ){ tvfsExecTcl(p, "xTruncate", - Tcl_NewStringObj(pFd->zFilename, -1), pFd->pShmId, 0 + Tcl_NewStringObj(pFd->zFilename, -1), pFd->pShmId, 0, 0 ); tvfsResultCode(p, &rc); } @@ -431,7 +434,7 @@ static int tvfsSync(sqlite3_file *pFile, int flags){ tvfsExecTcl(p, "xSync", Tcl_NewStringObj(pFd->zFilename, -1), pFd->pShmId, - Tcl_NewStringObj(zFlags, -1) + Tcl_NewStringObj(zFlags, -1), 0 ); tvfsResultCode(p, &rc); } @@ -465,8 +468,12 @@ static int tvfsLock(sqlite3_file *pFile, int eLock){ ** Unlock an tvfs-file. */ static int tvfsUnlock(sqlite3_file *pFile, int eLock){ - TestvfsFd *p = tvfsGetFd(pFile); - return sqlite3OsUnlock(p->pReal, eLock); + TestvfsFd *pFd = tvfsGetFd(pFile); + Testvfs *p = (Testvfs *)pFd->pVfs->pAppData; + if( p->mask&TESTVFS_WRITE_MASK && tvfsInjectIoerr(p) ){ + return SQLITE_IOERR_UNLOCK; + } + return sqlite3OsUnlock(pFd->pReal, eLock); } /* @@ -578,7 +585,7 @@ static int tvfsOpen( z += strlen(z) + 1; } } - tvfsExecTcl(p, "xOpen", Tcl_NewStringObj(pFd->zFilename, -1), pArg, 0); + tvfsExecTcl(p, "xOpen", Tcl_NewStringObj(pFd->zFilename, -1), pArg, 0, 0); Tcl_DecrRefCount(pArg); if( tvfsResultCode(p, &rc) ){ if( rc!=SQLITE_OK ) return rc; @@ -635,7 +642,7 @@ static int tvfsDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ if( p->pScript && p->mask&TESTVFS_DELETE_MASK ){ tvfsExecTcl(p, "xDelete", - Tcl_NewStringObj(zPath, -1), Tcl_NewIntObj(dirSync), 0 + Tcl_NewStringObj(zPath, -1), Tcl_NewIntObj(dirSync), 0, 0 ); tvfsResultCode(p, &rc); } @@ -663,7 +670,7 @@ static int tvfsAccess( if( flags==SQLITE_ACCESS_READWRITE ) zArg = "SQLITE_ACCESS_READWRITE"; if( flags==SQLITE_ACCESS_READ ) zArg = "SQLITE_ACCESS_READ"; tvfsExecTcl(p, "xAccess", - Tcl_NewStringObj(zPath, -1), Tcl_NewStringObj(zArg, -1), 0 + Tcl_NewStringObj(zPath, -1), Tcl_NewStringObj(zArg, -1), 0, 0 ); if( tvfsResultCode(p, &rc) ){ if( rc!=SQLITE_OK ) return rc; @@ -691,7 +698,7 @@ static int tvfsFullPathname( Testvfs *p = (Testvfs *)pVfs->pAppData; if( p->pScript && p->mask&TESTVFS_FULLPATHNAME_MASK ){ int rc; - tvfsExecTcl(p, "xFullPathname", Tcl_NewStringObj(zPath, -1), 0, 0); + tvfsExecTcl(p, "xFullPathname", Tcl_NewStringObj(zPath, -1), 0, 0, 0); if( tvfsResultCode(p, &rc) ){ if( rc!=SQLITE_OK ) return rc; } @@ -771,7 +778,7 @@ static int tvfsShmOpen(sqlite3_file *pFile){ */ Tcl_ResetResult(p->interp); if( p->pScript && p->mask&TESTVFS_SHMOPEN_MASK ){ - tvfsExecTcl(p, "xShmOpen", Tcl_NewStringObj(pFd->zFilename, -1), 0, 0); + tvfsExecTcl(p, "xShmOpen", Tcl_NewStringObj(pFd->zFilename, -1), 0, 0, 0); if( tvfsResultCode(p, &rc) ){ if( rc!=SQLITE_OK ) return rc; } @@ -841,7 +848,7 @@ static int tvfsShmMap( Tcl_ListObjAppendElement(p->interp, pArg, Tcl_NewIntObj(pgsz)); Tcl_ListObjAppendElement(p->interp, pArg, Tcl_NewIntObj(isWrite)); tvfsExecTcl(p, "xShmMap", - Tcl_NewStringObj(pFd->pShm->zFile, -1), pFd->pShmId, pArg + Tcl_NewStringObj(pFd->pShm->zFile, -1), pFd->pShmId, pArg, 0 ); tvfsResultCode(p, &rc); Tcl_DecrRefCount(pArg); @@ -891,7 +898,7 @@ static int tvfsShmLock( } tvfsExecTcl(p, "xShmLock", Tcl_NewStringObj(pFd->pShm->zFile, -1), pFd->pShmId, - Tcl_NewStringObj(zLock, -1) + Tcl_NewStringObj(zLock, -1), 0 ); tvfsResultCode(p, &rc); } @@ -937,7 +944,7 @@ static void tvfsShmBarrier(sqlite3_file *pFile){ if( p->pScript && p->mask&TESTVFS_SHMBARRIER_MASK ){ tvfsExecTcl(p, "xShmBarrier", - Tcl_NewStringObj(pFd->pShm->zFile, -1), pFd->pShmId, 0 + Tcl_NewStringObj(pFd->pShm->zFile, -1), pFd->pShmId, 0, 0 ); } } @@ -961,7 +968,7 @@ static int tvfsShmUnmap( if( p->pScript && p->mask&TESTVFS_SHMCLOSE_MASK ){ tvfsExecTcl(p, "xShmUnmap", - Tcl_NewStringObj(pFd->pShm->zFile, -1), pFd->pShmId, 0 + Tcl_NewStringObj(pFd->pShm->zFile, -1), pFd->pShmId, 0, 0 ); tvfsResultCode(p, &rc); } @@ -1099,6 +1106,7 @@ static int testvfs_obj_cmd( { "xClose", TESTVFS_CLOSE_MASK }, { "xAccess", TESTVFS_ACCESS_MASK }, { "xFullPathname", TESTVFS_FULLPATHNAME_MASK }, + { "xUnlock", TESTVFS_UNLOCK_MASK }, }; Tcl_Obj **apElem = 0; int nElem = 0; diff --git a/src/test_vfstrace.c b/src/test_vfstrace.c index d2f7455..0aacc01 100644 --- a/src/test_vfstrace.c +++ b/src/test_vfstrace.c @@ -475,6 +475,7 @@ static int vfstraceFileControl(sqlite3_file *pFile, int op, void *pArg){ case SQLITE_FCNTL_PERSIST_WAL: zOp = "PERSIST_WAL"; break; case SQLITE_FCNTL_OVERWRITE: zOp = "OVERWRITE"; break; case SQLITE_FCNTL_VFSNAME: zOp = "VFSNAME"; break; + case SQLITE_FCNTL_TEMPFILENAME: zOp = "TEMPFILENAME"; break; case 0xca093fa0: zOp = "DB_UNCHANGED"; break; case SQLITE_FCNTL_PRAGMA: { const char *const* a = (const char*const*)pArg; @@ -496,7 +497,8 @@ static int vfstraceFileControl(sqlite3_file *pFile, int op, void *pArg){ *(char**)pArg = sqlite3_mprintf("vfstrace.%s/%z", pInfo->zVfsName, *(char**)pArg); } - if( op==SQLITE_FCNTL_PRAGMA && rc==SQLITE_OK && *(char**)pArg ){ + if( (op==SQLITE_FCNTL_PRAGMA || op==SQLITE_FCNTL_TEMPFILENAME) + && rc==SQLITE_OK && *(char**)pArg ){ vfstrace_printf(pInfo, "%s.xFileControl(%s,%s) returns %s", pInfo->zVfsName, p->zFName, zOp, *(char**)pArg); } diff --git a/src/test_wholenumber.c b/src/test_wholenumber.c deleted file mode 100644 index 7c42d01..0000000 --- a/src/test_wholenumber.c +++ /dev/null @@ -1,311 +0,0 @@ -/* -** 2011 April 02 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** This file implements a virtual table that returns the whole numbers -** between 1 and 4294967295, inclusive. -** -** Example: -** -** CREATE VIRTUAL TABLE nums USING wholenumber; -** SELECT value FROM nums WHERE value<10; -** -** Results in: -** -** 1 2 3 4 5 6 7 8 9 -*/ -#include "sqlite3.h" -#include <assert.h> -#include <string.h> - -#ifndef SQLITE_OMIT_VIRTUALTABLE - - -/* A wholenumber cursor object */ -typedef struct wholenumber_cursor wholenumber_cursor; -struct wholenumber_cursor { - sqlite3_vtab_cursor base; /* Base class - must be first */ - sqlite3_int64 iValue; /* Current value */ - sqlite3_int64 mxValue; /* Maximum value */ -}; - -/* Methods for the wholenumber module */ -static int wholenumberConnect( - sqlite3 *db, - void *pAux, - int argc, const char *const*argv, - sqlite3_vtab **ppVtab, - char **pzErr -){ - sqlite3_vtab *pNew; - pNew = *ppVtab = sqlite3_malloc( sizeof(*pNew) ); - if( pNew==0 ) return SQLITE_NOMEM; - sqlite3_declare_vtab(db, "CREATE TABLE x(value)"); - memset(pNew, 0, sizeof(*pNew)); - return SQLITE_OK; -} -/* Note that for this virtual table, the xCreate and xConnect -** methods are identical. */ - -static int wholenumberDisconnect(sqlite3_vtab *pVtab){ - sqlite3_free(pVtab); - return SQLITE_OK; -} -/* The xDisconnect and xDestroy methods are also the same */ - - -/* -** Open a new wholenumber cursor. -*/ -static int wholenumberOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ - wholenumber_cursor *pCur; - pCur = sqlite3_malloc( sizeof(*pCur) ); - if( pCur==0 ) return SQLITE_NOMEM; - memset(pCur, 0, sizeof(*pCur)); - *ppCursor = &pCur->base; - return SQLITE_OK; -} - -/* -** Close a wholenumber cursor. -*/ -static int wholenumberClose(sqlite3_vtab_cursor *cur){ - sqlite3_free(cur); - return SQLITE_OK; -} - - -/* -** Advance a cursor to its next row of output -*/ -static int wholenumberNext(sqlite3_vtab_cursor *cur){ - wholenumber_cursor *pCur = (wholenumber_cursor*)cur; - pCur->iValue++; - return SQLITE_OK; -} - -/* -** Return the value associated with a wholenumber. -*/ -static int wholenumberColumn( - sqlite3_vtab_cursor *cur, - sqlite3_context *ctx, - int i -){ - wholenumber_cursor *pCur = (wholenumber_cursor*)cur; - sqlite3_result_int64(ctx, pCur->iValue); - return SQLITE_OK; -} - -/* -** The rowid. -*/ -static int wholenumberRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ - wholenumber_cursor *pCur = (wholenumber_cursor*)cur; - *pRowid = pCur->iValue; - return SQLITE_OK; -} - -/* -** When the wholenumber_cursor.rLimit value is 0 or less, that is a signal -** that the cursor has nothing more to output. -*/ -static int wholenumberEof(sqlite3_vtab_cursor *cur){ - wholenumber_cursor *pCur = (wholenumber_cursor*)cur; - return pCur->iValue>pCur->mxValue || pCur->iValue==0; -} - -/* -** Called to "rewind" a cursor back to the beginning so that -** it starts its output over again. Always called at least once -** prior to any wholenumberColumn, wholenumberRowid, or wholenumberEof call. -** -** idxNum Constraints -** ------ --------------------- -** 0 (none) -** 1 value > $argv0 -** 2 value >= $argv0 -** 4 value < $argv0 -** 8 value <= $argv0 -** -** 5 value > $argv0 AND value < $argv1 -** 6 value >= $argv0 AND value < $argv1 -** 9 value > $argv0 AND value <= $argv1 -** 10 value >= $argv0 AND value <= $argv1 -*/ -static int wholenumberFilter( - sqlite3_vtab_cursor *pVtabCursor, - int idxNum, const char *idxStr, - int argc, sqlite3_value **argv -){ - wholenumber_cursor *pCur = (wholenumber_cursor *)pVtabCursor; - sqlite3_int64 v; - int i = 0; - pCur->iValue = 1; - pCur->mxValue = 0xffffffff; /* 4294967295 */ - if( idxNum & 3 ){ - v = sqlite3_value_int64(argv[0]) + (idxNum&1); - if( v>pCur->iValue && v<=pCur->mxValue ) pCur->iValue = v; - i++; - } - if( idxNum & 12 ){ - v = sqlite3_value_int64(argv[i]) - ((idxNum>>2)&1); - if( v>=pCur->iValue && v<pCur->mxValue ) pCur->mxValue = v; - } - return SQLITE_OK; -} - -/* -** Search for terms of these forms: -** -** (1) value > $value -** (2) value >= $value -** (4) value < $value -** (8) value <= $value -** -** idxNum is an ORed combination of 1 or 2 with 4 or 8. -*/ -static int wholenumberBestIndex( - sqlite3_vtab *tab, - sqlite3_index_info *pIdxInfo -){ - int i; - int idxNum = 0; - int argvIdx = 1; - int ltIdx = -1; - int gtIdx = -1; - const struct sqlite3_index_constraint *pConstraint; - pConstraint = pIdxInfo->aConstraint; - for(i=0; i<pIdxInfo->nConstraint; i++, pConstraint++){ - if( pConstraint->usable==0 ) continue; - if( (idxNum & 3)==0 && pConstraint->op==SQLITE_INDEX_CONSTRAINT_GT ){ - idxNum |= 1; - ltIdx = i; - } - if( (idxNum & 3)==0 && pConstraint->op==SQLITE_INDEX_CONSTRAINT_GE ){ - idxNum |= 2; - ltIdx = i; - } - if( (idxNum & 12)==0 && pConstraint->op==SQLITE_INDEX_CONSTRAINT_LT ){ - idxNum |= 4; - gtIdx = i; - } - if( (idxNum & 12)==0 && pConstraint->op==SQLITE_INDEX_CONSTRAINT_LE ){ - idxNum |= 8; - gtIdx = i; - } - } - pIdxInfo->idxNum = idxNum; - if( ltIdx>=0 ){ - pIdxInfo->aConstraintUsage[ltIdx].argvIndex = argvIdx++; - pIdxInfo->aConstraintUsage[ltIdx].omit = 1; - } - if( gtIdx>=0 ){ - pIdxInfo->aConstraintUsage[gtIdx].argvIndex = argvIdx; - pIdxInfo->aConstraintUsage[gtIdx].omit = 1; - } - if( pIdxInfo->nOrderBy==1 - && pIdxInfo->aOrderBy[0].desc==0 - ){ - pIdxInfo->orderByConsumed = 1; - } - pIdxInfo->estimatedCost = (double)1; - return SQLITE_OK; -} - -/* -** A virtual table module that provides read-only access to a -** Tcl global variable namespace. -*/ -static sqlite3_module wholenumberModule = { - 0, /* iVersion */ - wholenumberConnect, - wholenumberConnect, - wholenumberBestIndex, - wholenumberDisconnect, - wholenumberDisconnect, - wholenumberOpen, /* xOpen - open a cursor */ - wholenumberClose, /* xClose - close a cursor */ - wholenumberFilter, /* xFilter - configure scan constraints */ - wholenumberNext, /* xNext - advance a cursor */ - wholenumberEof, /* xEof - check for end of scan */ - wholenumberColumn, /* xColumn - read data */ - wholenumberRowid, /* xRowid - read data */ - 0, /* xUpdate */ - 0, /* xBegin */ - 0, /* xSync */ - 0, /* xCommit */ - 0, /* xRollback */ - 0, /* xFindMethod */ - 0, /* xRename */ -}; - -#endif /* SQLITE_OMIT_VIRTUALTABLE */ - - -/* -** Register the wholenumber virtual table -*/ -int wholenumber_register(sqlite3 *db){ - int rc = SQLITE_OK; -#ifndef SQLITE_OMIT_VIRTUALTABLE - rc = sqlite3_create_module(db, "wholenumber", &wholenumberModule, 0); -#endif - return rc; -} - -#ifdef SQLITE_TEST -#include <tcl.h> -/* -** Decode a pointer to an sqlite3 object. -*/ -extern int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite3 **ppDb); - -/* -** Register the echo virtual table module. -*/ -static int register_wholenumber_module( - ClientData clientData, /* Pointer to sqlite3_enable_XXX function */ - Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ - int objc, /* Number of arguments */ - Tcl_Obj *CONST objv[] /* Command arguments */ -){ - sqlite3 *db; - if( objc!=2 ){ - Tcl_WrongNumArgs(interp, 1, objv, "DB"); - return TCL_ERROR; - } - if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; - wholenumber_register(db); - return TCL_OK; -} - - -/* -** Register commands with the TCL interpreter. -*/ -int Sqlitetestwholenumber_Init(Tcl_Interp *interp){ - static struct { - char *zName; - Tcl_ObjCmdProc *xProc; - void *clientData; - } aObjCmd[] = { - { "register_wholenumber_module", register_wholenumber_module, 0 }, - }; - int i; - for(i=0; i<sizeof(aObjCmd)/sizeof(aObjCmd[0]); i++){ - Tcl_CreateObjCommand(interp, aObjCmd[i].zName, - aObjCmd[i].xProc, aObjCmd[i].clientData, 0); - } - return TCL_OK; -} - -#endif /* SQLITE_TEST */ diff --git a/src/trigger.c b/src/trigger.c index 8985ec6..f1ff766 100644 --- a/src/trigger.c +++ b/src/trigger.c @@ -729,6 +729,15 @@ static int codeTriggerProgram( */ pParse->eOrconf = (orconf==OE_Default)?pStep->orconf:(u8)orconf; + /* Clear the cookieGoto flag. When coding triggers, the cookieGoto + ** variable is used as a flag to indicate to sqlite3ExprCodeConstants() + ** that it is not safe to refactor constants (this happens after the + ** start of the first loop in the SQL statement is coded - at that + ** point code may be conditionally executed, so it is no longer safe to + ** initialize constant register values). */ + assert( pParse->cookieGoto==0 || pParse->cookieGoto==-1 ); + pParse->cookieGoto = 0; + switch( pStep->op ){ case TK_UPDATE: { sqlite3Update(pParse, diff --git a/src/update.c b/src/update.c index 96ba4df..3ab1ab2 100644 --- a/src/update.c +++ b/src/update.c @@ -208,6 +208,7 @@ void sqlite3Update( } if( j>=pTab->nCol ){ if( sqlite3IsRowid(pChanges->a[i].zName) ){ + j = -1; chngRowid = 1; pRowidExpr = pChanges->a[i].pExpr; }else{ @@ -220,7 +221,8 @@ void sqlite3Update( { int rc; rc = sqlite3AuthCheck(pParse, SQLITE_UPDATE, pTab->zName, - pTab->aCol[j].zName, db->aDb[iDb].zName); + j<0 ? "ROWID" : pTab->aCol[j].zName, + db->aDb[iDb].zName); if( rc==SQLITE_DENY ){ goto update_cleanup; }else if( rc==SQLITE_IGNORE ){ @@ -458,7 +460,7 @@ void sqlite3Update( /* The row-trigger may have deleted the row being updated. In this ** case, jump to the next row. No updates or AFTER triggers are - ** required. This behaviour - what happens when the row being updated + ** required. This behavior - what happens when the row being updated ** is deleted or renamed by a BEFORE trigger - is left undefined in the ** documentation. */ @@ -164,25 +164,23 @@ static const unsigned char sqlite3Utf8Trans1[] = { || (c&0xFFFFFFFE)==0xFFFE ){ c = 0xFFFD; } \ } u32 sqlite3Utf8Read( - const unsigned char *zIn, /* First byte of UTF-8 character */ - const unsigned char **pzNext /* Write first byte past UTF-8 char here */ + const unsigned char **pz /* Pointer to string from which to read char */ ){ unsigned int c; /* Same as READ_UTF8() above but without the zTerm parameter. ** For this routine, we assume the UTF8 string is always zero-terminated. */ - c = *(zIn++); + c = *((*pz)++); if( c>=0xc0 ){ c = sqlite3Utf8Trans1[c-0xc0]; - while( (*zIn & 0xc0)==0x80 ){ - c = (c<<6) + (0x3f & *(zIn++)); + while( (*(*pz) & 0xc0)==0x80 ){ + c = (c<<6) + (0x3f & *((*pz)++)); } if( c<0x80 || (c&0xFFFFF800)==0xD800 || (c&0xFFFFFFFE)==0xFFFE ){ c = 0xFFFD; } } - *pzNext = zIn; return c; } @@ -283,7 +281,6 @@ int sqlite3VdbeMemTranslate(Mem *pMem, u8 desiredEnc){ if( desiredEnc==SQLITE_UTF16LE ){ /* UTF-8 -> UTF-16 Little-endian */ while( zIn<zTerm ){ - /* c = sqlite3Utf8Read(zIn, zTerm, (const u8**)&zIn); */ READ_UTF8(zIn, zTerm, c); WRITE_UTF16LE(z, c); } @@ -291,7 +288,6 @@ int sqlite3VdbeMemTranslate(Mem *pMem, u8 desiredEnc){ assert( desiredEnc==SQLITE_UTF16BE ); /* UTF-8 -> UTF-16 Big-endian */ while( zIn<zTerm ){ - /* c = sqlite3Utf8Read(zIn, zTerm, (const u8**)&zIn); */ READ_UTF8(zIn, zTerm, c); WRITE_UTF16BE(z, c); } @@ -419,7 +415,7 @@ int sqlite3Utf8To8(unsigned char *zIn){ u32 c; while( zIn[0] && zOut<=zIn ){ - c = sqlite3Utf8Read(zIn, (const u8**)&zIn); + c = sqlite3Utf8Read((const u8**)&zIn); if( c!=0xfffd ){ WRITE_UTF8(zOut, c); } @@ -524,7 +520,7 @@ void sqlite3UtfSelfTest(void){ assert( n>0 && n<=4 ); z[0] = 0; z = zBuf; - c = sqlite3Utf8Read(z, (const u8**)&z); + c = sqlite3Utf8Read((const u8**)&z); t = i; if( i>=0xD800 && i<=0xDFFF ) t = 0xFFFD; if( (i&0xFFFFFFFE)==0xFFFE ) t = 0xFFFD; @@ -261,7 +261,7 @@ int sqlite3_strnicmp(const char *zLeft, const char *zRight, int N){ */ int sqlite3AtoF(const char *z, double *pResult, int length, u8 enc){ #ifndef SQLITE_OMIT_FLOATING_POINT - int incr = (enc==SQLITE_UTF8?1:2); + int incr; const char *zEnd = z + length; /* sign * significand * (10 ^ (esign * exponent)) */ int sign = 1; /* sign of significand */ @@ -272,10 +272,22 @@ int sqlite3AtoF(const char *z, double *pResult, int length, u8 enc){ int eValid = 1; /* True exponent is either not used or is well-formed */ double result; int nDigits = 0; + int nonNum = 0; + assert( enc==SQLITE_UTF8 || enc==SQLITE_UTF16LE || enc==SQLITE_UTF16BE ); *pResult = 0.0; /* Default return value, in case of an error */ - if( enc==SQLITE_UTF16BE ) z++; + if( enc==SQLITE_UTF8 ){ + incr = 1; + }else{ + int i; + incr = 2; + assert( SQLITE_UTF16LE==2 && SQLITE_UTF16BE==3 ); + for(i=3-enc; i<length && z[i]==0; i+=2){} + nonNum = i<length; + zEnd = z+i+enc-3; + z += (enc&1); + } /* skip leading spaces */ while( z<zEnd && sqlite3Isspace(*z) ) z+=incr; @@ -408,7 +420,7 @@ do_atof_calc: *pResult = result; /* return true if number and no extra non-whitespace chracters after */ - return z>=zEnd && nDigits>0 && eValid; + return z>=zEnd && nDigits>0 && eValid && nonNum==0; #else return !sqlite3Atoi64(z, pResult, length, enc); #endif /* SQLITE_OMIT_FLOATING_POINT */ @@ -457,21 +469,33 @@ static int compare2pow63(const char *zNum, int incr){ ** signed 64-bit integer, its negative -9223372036854665808 can be. ** ** If zNum is too big for a 64-bit integer and is not -** 9223372036854665808 then return 1. +** 9223372036854665808 or if zNum contains any non-numeric text, +** then return 1. ** ** length is the number of bytes in the string (bytes, not characters). ** The string is not necessarily zero-terminated. The encoding is ** given by enc. */ int sqlite3Atoi64(const char *zNum, i64 *pNum, int length, u8 enc){ - int incr = (enc==SQLITE_UTF8?1:2); + int incr; u64 u = 0; int neg = 0; /* assume positive */ int i; int c = 0; + int nonNum = 0; const char *zStart; const char *zEnd = zNum + length; - if( enc==SQLITE_UTF16BE ) zNum++; + assert( enc==SQLITE_UTF8 || enc==SQLITE_UTF16LE || enc==SQLITE_UTF16BE ); + if( enc==SQLITE_UTF8 ){ + incr = 1; + }else{ + incr = 2; + assert( SQLITE_UTF16LE==2 && SQLITE_UTF16BE==3 ); + for(i=3-enc; i<length && zNum[i]==0; i+=2){} + nonNum = i<length; + zEnd = zNum+i+enc-3; + zNum += (enc&1); + } while( zNum<zEnd && sqlite3Isspace(*zNum) ) zNum+=incr; if( zNum<zEnd ){ if( *zNum=='-' ){ @@ -496,7 +520,7 @@ int sqlite3Atoi64(const char *zNum, i64 *pNum, int length, u8 enc){ testcase( i==18 ); testcase( i==19 ); testcase( i==20 ); - if( (c!=0 && &zNum[i]<zEnd) || (i==0 && zStart==zNum) || i>19*incr ){ + if( (c!=0 && &zNum[i]<zEnd) || (i==0 && zStart==zNum) || i>19*incr || nonNum ){ /* zNum is empty or contains non-numeric text or is longer ** than 19 digits (thus guaranteeing that it is too large) */ return 1; diff --git a/src/vacuum.c b/src/vacuum.c index 401d41d..4afb2cc 100644 --- a/src/vacuum.c +++ b/src/vacuum.c @@ -85,6 +85,7 @@ void sqlite3Vacuum(Parse *pParse){ Vdbe *v = sqlite3GetVdbe(pParse); if( v ){ sqlite3VdbeAddOp2(v, OP_Vacuum, 0, 0); + sqlite3VdbeUsesBtree(v, 0); } return; } @@ -288,6 +289,7 @@ int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){ BTREE_DEFAULT_CACHE_SIZE, 0, /* Preserve the default page cache size */ BTREE_TEXT_ENCODING, 0, /* Preserve the text encoding */ BTREE_USER_VERSION, 0, /* Preserve the user version */ + BTREE_APPLICATION_ID, 0, /* Preserve the application id */ }; assert( 1==sqlite3BtreeIsInTrans(pTemp) ); @@ -152,11 +152,7 @@ int sqlite3_found_count = 0; && sqlite3VdbeMemMakeWriteable(P) ){ goto no_mem;} /* Return true if the cursor was opened using the OP_OpenSorter opcode. */ -#ifdef SQLITE_OMIT_MERGE_SORT -# define isSorter(x) 0 -#else # define isSorter(x) ((x)->pSorter!=0) -#endif /* ** Argument pMem points at a register that will be passed to a @@ -422,7 +418,9 @@ void sqlite3VdbeMemPrettyPrint(Mem *pMem, char *zBuf){ ** Print the value of a register for tracing purposes: */ static void memTracePrint(FILE *out, Mem *p){ - if( p->flags & MEM_Null ){ + if( p->flags & MEM_Invalid ){ + fprintf(out, " undefined"); + }else if( p->flags & MEM_Null ){ fprintf(out, " NULL"); }else if( (p->flags & (MEM_Int|MEM_Str))==(MEM_Int|MEM_Str) ){ fprintf(out, " si:%lld", p->u.i); @@ -867,7 +865,7 @@ case OP_Halt: { if( rc==SQLITE_BUSY ){ p->rc = rc = SQLITE_BUSY; }else{ - assert( rc==SQLITE_OK || p->rc==SQLITE_CONSTRAINT ); + assert( rc==SQLITE_OK || (p->rc&0xff)==SQLITE_CONSTRAINT ); assert( rc==SQLITE_OK || db->nDeferredCons>0 ); rc = p->rc ? SQLITE_ERROR : SQLITE_DONE; } @@ -956,23 +954,28 @@ case OP_String: { /* out2-prerelease */ break; } -/* Opcode: Null * P2 P3 * * +/* Opcode: Null P1 P2 P3 * * ** ** Write a NULL into registers P2. If P3 greater than P2, then also write -** NULL into register P3 and ever register in between P2 and P3. If P3 +** NULL into register P3 and every register in between P2 and P3. If P3 ** is less than P2 (typically P3 is zero) then only register P2 is -** set to NULL +** set to NULL. +** +** If the P1 value is non-zero, then also set the MEM_Cleared flag so that +** NULL values will not compare equal even if SQLITE_NULLEQ is set on +** OP_Ne or OP_Eq. */ case OP_Null: { /* out2-prerelease */ int cnt; + u16 nullFlag; cnt = pOp->p3-pOp->p2; assert( pOp->p3<=p->nMem ); - pOut->flags = MEM_Null; + pOut->flags = nullFlag = pOp->p1 ? (MEM_Null|MEM_Cleared) : MEM_Null; while( cnt>0 ){ pOut++; memAboutToChange(p, pOut); VdbeMemRelease(pOut); - pOut->flags = MEM_Null; + pOut->flags = nullFlag; cnt--; } break; @@ -1015,10 +1018,10 @@ case OP_Variable: { /* out2-prerelease */ /* Opcode: Move P1 P2 P3 * * ** -** Move the values in register P1..P1+P3-1 over into -** registers P2..P2+P3-1. Registers P1..P1+P1-1 are +** Move the values in register P1..P1+P3 over into +** registers P2..P2+P3. Registers P1..P1+P3 are ** left holding a NULL. It is an error for register ranges -** P1..P1+P3-1 and P2..P2+P3-1 to overlap. +** P1..P1+P3 and P2..P2+P3 to overlap. */ case OP_Move: { char *zMalloc; /* Holding variable for allocated memory */ @@ -1026,7 +1029,7 @@ case OP_Move: { int p1; /* Register to copy from */ int p2; /* Register to copy to */ - n = pOp->p3; + n = pOp->p3 + 1; p1 = pOp->p1; p2 = pOp->p2; assert( n>0 && p1>0 && p2>0 ); @@ -1055,20 +1058,31 @@ case OP_Move: { break; } -/* Opcode: Copy P1 P2 * * * +/* Opcode: Copy P1 P2 P3 * * ** -** Make a copy of register P1 into register P2. +** Make a copy of registers P1..P1+P3 into registers P2..P2+P3. ** ** This instruction makes a deep copy of the value. A duplicate ** is made of any string or blob constant. See also OP_SCopy. */ -case OP_Copy: { /* in1, out2 */ +case OP_Copy: { + int n; + + n = pOp->p3; pIn1 = &aMem[pOp->p1]; pOut = &aMem[pOp->p2]; assert( pOut!=pIn1 ); - sqlite3VdbeMemShallowCopy(pOut, pIn1, MEM_Ephem); - Deephemeralize(pOut); - REGISTER_TRACE(pOp->p2, pOut); + while( 1 ){ + sqlite3VdbeMemShallowCopy(pOut, pIn1, MEM_Ephem); + Deephemeralize(pOut); +#ifdef SQLITE_DEBUG + pOut->pScopyFrom = 0; +#endif + REGISTER_TRACE(pOp->p2+pOp->p3-n, pOut); + if( (n--)==0 ) break; + pOut++; + pIn1++; + } break; } @@ -1252,6 +1266,7 @@ case OP_Subtract: /* same as TK_MINUS, in1, in2, out3 */ case OP_Multiply: /* same as TK_STAR, in1, in2, out3 */ case OP_Divide: /* same as TK_SLASH, in1, in2, out3 */ case OP_Remainder: { /* same as TK_REM, in1, in2, out3 */ + char bIntint; /* Started out as two integer operands */ int flags; /* Combined MEM_* flags from both inputs */ i64 iA; /* Integer value of left operand */ i64 iB; /* Integer value of right operand */ @@ -1268,6 +1283,7 @@ case OP_Remainder: { /* same as TK_REM, in1, in2, out3 */ if( (pIn1->flags & pIn2->flags & MEM_Int)==MEM_Int ){ iA = pIn1->u.i; iB = pIn2->u.i; + bIntint = 1; switch( pOp->opcode ){ case OP_Add: if( sqlite3AddInt64(&iB,iA) ) goto fp_math; break; case OP_Subtract: if( sqlite3SubInt64(&iB,iA) ) goto fp_math; break; @@ -1288,6 +1304,7 @@ case OP_Remainder: { /* same as TK_REM, in1, in2, out3 */ pOut->u.i = iB; MemSetTypeFlag(pOut, MEM_Int); }else{ + bIntint = 0; fp_math: rA = sqlite3VdbeRealValue(pIn1); rB = sqlite3VdbeRealValue(pIn2); @@ -1319,7 +1336,7 @@ fp_math: } pOut->r = rB; MemSetTypeFlag(pOut, MEM_Real); - if( (flags & MEM_Real)==0 ){ + if( (flags & MEM_Real)==0 && !bIntint ){ sqlite3VdbeIntegerAffinity(pOut); } #endif @@ -1737,6 +1754,10 @@ case OP_ToReal: { /* same as TK_TO_REAL, in1 */ ** ** If the SQLITE_STOREP2 bit of P5 is set, then do not jump. Instead, ** store a boolean result (either 0, or 1, or NULL) in register P2. +** +** If the SQLITE_NULLEQ bit is set in P5, then NULL values are considered +** equal to one another, provided that they do not have their MEM_Cleared +** bit set. */ /* Opcode: Ne P1 P2 P3 P4 P5 ** @@ -1803,7 +1824,15 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ ** or not both operands are null. */ assert( pOp->opcode==OP_Eq || pOp->opcode==OP_Ne ); - res = (flags1 & flags3 & MEM_Null)==0; + assert( (flags1 & MEM_Cleared)==0 ); + if( (flags1&MEM_Null)!=0 + && (flags3&MEM_Null)!=0 + && (flags3&MEM_Cleared)==0 + ){ + res = 0; /* Results are equal */ + }else{ + res = 1; /* Results are not equal */ + } }else{ /* SQLITE_NULLEQ is clear and at least one operand is NULL, ** then the result is always NULL. @@ -1862,9 +1891,9 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ ** Set the permutation used by the OP_Compare operator to be the array ** of integers in P4. ** -** The permutation is only valid until the next OP_Permutation, OP_Compare, -** OP_Halt, or OP_ResultRow. Typically the OP_Permutation should occur -** immediately prior to the OP_Compare. +** The permutation is only valid until the next OP_Compare that has +** the OPFLAG_PERMUTE bit set in P5. Typically the OP_Permutation should +** occur immediately prior to the OP_Compare. */ case OP_Permutation: { assert( pOp->p4type==P4_INTARRAY ); @@ -1873,12 +1902,17 @@ case OP_Permutation: { break; } -/* Opcode: Compare P1 P2 P3 P4 * +/* Opcode: Compare P1 P2 P3 P4 P5 ** ** Compare two vectors of registers in reg(P1)..reg(P1+P3-1) (call this ** vector "A") and in reg(P2)..reg(P2+P3-1) ("B"). Save the result of ** the comparison for use by the next OP_Jump instruct. ** +** If P5 has the OPFLAG_PERMUTE bit set, then the order of comparison is +** determined by the most recent OP_Permutation operator. If the +** OPFLAG_PERMUTE bit is clear, then register are compared in sequential +** order. +** ** P4 is a KeyInfo structure that defines collating sequences and sort ** orders for the comparison. The permutation applies to registers ** only. The KeyInfo elements are used sequentially. @@ -1897,6 +1931,7 @@ case OP_Compare: { CollSeq *pColl; /* Collating sequence to use on this term */ int bRev; /* True for DESCENDING sort order */ + if( (pOp->p5 & OPFLAG_PERMUTE)==0 ) aPermute = 0; n = pOp->p3; pKeyInfo = pOp->p4.pKeyInfo; assert( n>0 ); @@ -2040,8 +2075,6 @@ case OP_BitNot: { /* same as TK_BITNOT, in1, out2 */ ** ** Check if OP_Once flag P1 is set. If so, jump to instruction P2. Otherwise, ** set the flag and fall through to the next instruction. -** -** See also: JumpOnce */ case OP_Once: { /* jump */ assert( pOp->p1<p->nOnceFlag ); @@ -2212,6 +2245,11 @@ case OP_Column: { } }else if( ALWAYS(pC->pseudoTableReg>0) ){ pReg = &aMem[pC->pseudoTableReg]; + if( pC->multiPseudo ){ + sqlite3VdbeMemShallowCopy(pDest, pReg+p2, MEM_Ephem); + Deephemeralize(pDest); + goto op_column_out; + } assert( pReg->flags & MEM_Blob ); assert( memIsValid(pReg) ); payloadSize = pReg->n; @@ -3270,7 +3308,7 @@ case OP_OpenEphemeral: { break; } -/* Opcode: OpenSorter P1 P2 * P4 * +/* Opcode: SorterOpen P1 P2 * P4 * ** ** This opcode works like OP_OpenEphemeral except that it opens ** a transient index that is specifically designed to sort large @@ -3278,26 +3316,23 @@ case OP_OpenEphemeral: { */ case OP_SorterOpen: { VdbeCursor *pCx; -#ifndef SQLITE_OMIT_MERGE_SORT + pCx = allocateCursor(p, pOp->p1, pOp->p2, -1, 1); if( pCx==0 ) goto no_mem; pCx->pKeyInfo = pOp->p4.pKeyInfo; pCx->pKeyInfo->enc = ENC(p->db); pCx->isSorter = 1; rc = sqlite3VdbeSorterInit(db, pCx); -#else - pOp->opcode = OP_OpenEphemeral; - pc--; -#endif break; } -/* Opcode: OpenPseudo P1 P2 P3 * * +/* Opcode: OpenPseudo P1 P2 P3 * P5 ** ** Open a new cursor that points to a fake table that contains a single ** row of data. The content of that one row in the content of memory -** register P2. In other words, cursor P1 becomes an alias for the -** MEM_Blob content contained in register P2. +** register P2 when P5==0. In other words, cursor P1 becomes an alias for the +** MEM_Blob content contained in register P2. When P5==1, then the +** row is represented by P3 consecutive registers beginning with P2. ** ** A pseudo-table created by this opcode is used to hold a single ** row output from the sorter so that the row can be decomposed into @@ -3317,6 +3352,7 @@ case OP_OpenPseudo: { pCx->pseudoTableReg = pOp->p2; pCx->isTable = 1; pCx->isIndex = 0; + pCx->multiPseudo = pOp->p5; break; } @@ -3479,7 +3515,7 @@ case OP_SeekGt: { /* jump, in3 */ ** r.flags = 0; ** } */ - r.flags = (u16)(UNPACKED_INCRKEY * (1 & (oc - OP_SeekLt))); + r.flags = (u8)(UNPACKED_INCRKEY * (1 & (oc - OP_SeekLt))); assert( oc!=OP_SeekGt || r.flags==UNPACKED_INCRKEY ); assert( oc!=OP_SeekLe || r.flags==UNPACKED_INCRKEY ); assert( oc!=OP_SeekGe || r.flags==0 ); @@ -4168,15 +4204,11 @@ case OP_SorterCompare: { */ case OP_SorterData: { VdbeCursor *pC; -#ifndef SQLITE_OMIT_MERGE_SORT + pOut = &aMem[pOp->p2]; pC = p->apCsr[pOp->p1]; assert( pC->isSorter ); rc = sqlite3VdbeSorterRowkey(pC, pOut); -#else - pOp->opcode = OP_RowKey; - pc--; -#endif break; } @@ -4280,7 +4312,7 @@ case OP_Rowid: { /* out2-prerelease */ assert( pOp->p1>=0 && pOp->p1<p->nCursor ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); - assert( pC->pseudoTableReg==0 ); + assert( pC->pseudoTableReg==0 || pC->nullRow ); if( pC->nullRow ){ pOut->flags = MEM_Null; break; @@ -4375,9 +4407,6 @@ case OP_Last: { /* jump */ ** correctly optimizing out sorts. */ case OP_SorterSort: /* jump */ -#ifdef SQLITE_OMIT_MERGE_SORT - pOp->opcode = OP_Sort; -#endif case OP_Sort: { /* jump */ #ifdef SQLITE_TEST sqlite3_sort_count++; @@ -4456,9 +4485,6 @@ case OP_Rewind: { /* jump */ ** number P5-1 in the prepared statement is incremented. */ case OP_SorterNext: /* jump */ -#ifdef SQLITE_OMIT_MERGE_SORT - pOp->opcode = OP_Next; -#endif case OP_Prev: /* jump */ case OP_Next: { /* jump */ VdbeCursor *pC; @@ -4509,9 +4535,6 @@ case OP_Next: { /* jump */ ** for tables is OP_Insert. */ case OP_SorterInsert: /* in2 */ -#ifdef SQLITE_OMIT_MERGE_SORT - pOp->opcode = OP_IdxInsert; -#endif case OP_IdxInsert: { /* in2 */ VdbeCursor *pC; BtCursor *pCrsr; @@ -4706,6 +4729,7 @@ case OP_Destroy: { /* out2-prerelease */ int iCnt; Vdbe *pVdbe; int iDb; + #ifndef SQLITE_OMIT_VIRTUALTABLE iCnt = 0; for(pVdbe=db->pVdbe; pVdbe; pVdbe = pVdbe->pNext){ @@ -5495,7 +5519,9 @@ case OP_JournalMode: { /* out2-prerelease */ Pager *pPager; /* Pager associated with pBt */ int eNew; /* New journal mode */ int eOld; /* The old journal mode */ +#ifndef SQLITE_OMIT_WAL const char *zFilename; /* Name of database file for pPager */ +#endif eNew = pOp->p3; assert( eNew==PAGER_JOURNALMODE_DELETE @@ -5735,7 +5761,7 @@ case OP_VOpen: { /* Initialize sqlite3_vtab_cursor base class */ pVtabCursor->pVtab = pVtab; - /* Initialise vdbe cursor object */ + /* Initialize vdbe cursor object */ pCur = allocateCursor(p, pOp->p1, 0, -1, 0); if( pCur ){ pCur->pVtabCursor = pVtabCursor; @@ -6014,7 +6040,7 @@ case OP_VUpdate: { assert( nArg>1 && apArg[0] && (apArg[0]->flags&MEM_Null) ); db->lastRowid = lastRowid = rowid; } - if( rc==SQLITE_CONSTRAINT && pOp->p4.pVtab->bConstraint ){ + if( (rc&0xff)==SQLITE_CONSTRAINT && pOp->p4.pVtab->bConstraint ){ if( pOp->p5==OE_Ignore ){ rc = SQLITE_OK; }else{ @@ -6075,7 +6101,10 @@ case OP_Trace: { char *zTrace; char *z; - if( db->xTrace && (zTrace = (pOp->p4.z ? pOp->p4.z : p->zSql))!=0 ){ + if( db->xTrace + && !p->doingRerun + && (zTrace = (pOp->p4.z ? pOp->p4.z : p->zSql))!=0 + ){ z = sqlite3VdbeExpandSql(p, zTrace); db->xTrace(db->pTraceArg, z); sqlite3DbFree(db, z); @@ -188,7 +188,7 @@ VdbeOp *sqlite3VdbeGetOp(Vdbe*, int); int sqlite3VdbeMakeLabel(Vdbe*); void sqlite3VdbeRunOnlyOnce(Vdbe*); void sqlite3VdbeDelete(Vdbe*); -void sqlite3VdbeDeleteObject(sqlite3*,Vdbe*); +void sqlite3VdbeClearObject(sqlite3*,Vdbe*); void sqlite3VdbeMakeReady(Vdbe*,Parse*); int sqlite3VdbeFinalize(Vdbe*); void sqlite3VdbeResolveLabel(Vdbe*, int); diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 1f5694a..3a5b402 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -19,6 +19,14 @@ #define _VDBEINT_H_ /* +** The maximum number of times that a statement will try to reparse +** itself before giving up and returning SQLITE_SCHEMA. +*/ +#ifndef SQLITE_MAX_SCHEMA_RETRY +# define SQLITE_MAX_SCHEMA_RETRY 50 +#endif + +/* ** SQL is translated into a sequence of instructions to be ** executed by a virtual machine. Each instruction is an instance ** of the following structure. @@ -63,6 +71,7 @@ struct VdbeCursor { Bool isIndex; /* True if an index containing keys only - no data */ Bool isOrdered; /* True if the underlying table is BTREE_UNORDERED */ Bool isSorter; /* True if a new-style sorter */ + Bool multiPseudo; /* Multi-register pseudo-cursor */ sqlite3_vtab_cursor *pVtabCursor; /* The cursor for a virtual table */ const sqlite3_module *pModule; /* Module for cursor pVtabCursor */ i64 seqCount; /* Sequence counter */ @@ -122,7 +131,7 @@ struct VdbeFrame { VdbeCursor **apCsr; /* Array of Vdbe cursors for parent frame */ void *token; /* Copy of SubProgram.token */ i64 lastRowid; /* Last insert rowid (sqlite3.lastRowid) */ - u16 nCursor; /* Number of entries in apCsr */ + int nCursor; /* Number of entries in apCsr */ int pc; /* Program Counter in parent (calling) frame */ int nOp; /* Size of aOp array */ int nMem; /* Number of entries in aMem */ @@ -187,7 +196,9 @@ struct Mem { #define MEM_RowSet 0x0020 /* Value is a RowSet object */ #define MEM_Frame 0x0040 /* Value is a VdbeFrame object */ #define MEM_Invalid 0x0080 /* Value is undefined */ -#define MEM_TypeMask 0x00ff /* Mask of type bits */ +#define MEM_Cleared 0x0100 /* NULL set by OP_Null, not from data */ +#define MEM_TypeMask 0x01ff /* Mask of type bits */ + /* Whenever Mem contains a valid string or blob representation, one of ** the following flags must be set to determine the memory management @@ -273,6 +284,11 @@ struct Explain { char zBase[100]; /* Initial space */ }; +/* A bitfield type for use inside of structures. Always follow with :N where +** N is the number of bits. +*/ +typedef unsigned bft; /* Bit Field Type */ + /* ** An instance of the virtual machine. This structure contains the complete ** state of the virtual machine. @@ -301,7 +317,7 @@ struct Vdbe { int nLabel; /* Number of labels used */ int *aLabel; /* Space to hold the labels */ u16 nResColumn; /* Number of columns in one row of the result set */ - u16 nCursor; /* Number of slots in apCsr[] */ + int nCursor; /* Number of slots in apCsr[] */ u32 magic; /* Magic number for sanity checking */ char *zErrMsg; /* Error message written here */ Vdbe *pPrev,*pNext; /* Linked list of VDBEs with the same Vdbe.db */ @@ -314,15 +330,16 @@ struct Vdbe { int pc; /* The program counter */ int rc; /* Value to return */ u8 errorAction; /* Recovery action to do in case of an error */ - u8 explain; /* True if EXPLAIN present on SQL command */ - u8 changeCntOn; /* True to update the change-counter */ - u8 expired; /* True if the VM needs to be recompiled */ - u8 runOnlyOnce; /* Automatically expire on reset */ u8 minWriteFileFormat; /* Minimum file format for writable database files */ - u8 inVtabMethod; /* See comments above */ - u8 usesStmtJournal; /* True if uses a statement journal */ - u8 readOnly; /* True for read-only statements */ - u8 isPrepareV2; /* True if prepared with prepare_v2() */ + bft explain:2; /* True if EXPLAIN present on SQL command */ + bft inVtabMethod:2; /* See comments above */ + bft changeCntOn:1; /* True to update the change-counter */ + bft expired:1; /* True if the VM needs to be recompiled */ + bft runOnlyOnce:1; /* Automatically expire on reset */ + bft usesStmtJournal:1; /* True if uses a statement journal */ + bft readOnly:1; /* True for read-only statements */ + bft isPrepareV2:1; /* True if prepared with prepare_v2() */ + bft doingRerun:1; /* True if rerunning after an auto-reprepare */ int nChange; /* Number of db changes made since last reset */ yDbMask btreeMask; /* Bitmask of db->aDb[] entries referenced */ yDbMask lockMask; /* Subset of btreeMask that requires a lock */ @@ -420,15 +437,6 @@ int sqlite3VdbeFrameRestore(VdbeFrame *); void sqlite3VdbeMemStoreType(Mem *pMem); int sqlite3VdbeTransferError(Vdbe *p); -#ifdef SQLITE_OMIT_MERGE_SORT -# define sqlite3VdbeSorterInit(Y,Z) SQLITE_OK -# define sqlite3VdbeSorterWrite(X,Y,Z) SQLITE_OK -# define sqlite3VdbeSorterClose(Y,Z) -# define sqlite3VdbeSorterRowkey(Y,Z) SQLITE_OK -# define sqlite3VdbeSorterRewind(X,Y,Z) SQLITE_OK -# define sqlite3VdbeSorterNext(X,Y,Z) SQLITE_OK -# define sqlite3VdbeSorterCompare(X,Y,Z) SQLITE_OK -#else int sqlite3VdbeSorterInit(sqlite3 *, VdbeCursor *); void sqlite3VdbeSorterClose(sqlite3 *, VdbeCursor *); int sqlite3VdbeSorterRowkey(const VdbeCursor *, Mem *); @@ -436,7 +444,6 @@ int sqlite3VdbeSorterNext(sqlite3 *, const VdbeCursor *, int *); int sqlite3VdbeSorterRewind(sqlite3 *, const VdbeCursor *, int *); int sqlite3VdbeSorterWrite(sqlite3 *, const VdbeCursor *, Mem *); int sqlite3VdbeSorterCompare(const VdbeCursor *, Mem *, int *); -#endif #if !defined(SQLITE_OMIT_SHARED_CACHE) && SQLITE_THREADSAFE>0 void sqlite3VdbeEnter(Vdbe*); diff --git a/src/vdbeapi.c b/src/vdbeapi.c index b9a88a6..7c861e2 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -445,7 +445,7 @@ end_of_step: assert( p->rc!=SQLITE_ROW && p->rc!=SQLITE_DONE ); if( p->isPrepareV2 && rc!=SQLITE_ROW && rc!=SQLITE_DONE ){ /* If this statement was prepared using sqlite3_prepare_v2(), and an - ** error has occured, then return the error code in p->rc to the + ** error has occurred, then return the error code in p->rc to the ** caller. Set the error code in the database handle to the same value. */ rc = sqlite3VdbeTransferError(p); @@ -454,14 +454,6 @@ end_of_step: } /* -** The maximum number of times that a statement will try to reparse -** itself before giving up and returning SQLITE_SCHEMA. -*/ -#ifndef SQLITE_MAX_SCHEMA_RETRY -# define SQLITE_MAX_SCHEMA_RETRY 5 -#endif - -/* ** This is the top-level implementation of sqlite3_step(). Call ** sqlite3Step() to do most of the work. If a schema error occurs, ** call sqlite3Reprepare() and try again. @@ -478,10 +470,12 @@ int sqlite3_step(sqlite3_stmt *pStmt){ } db = v->db; sqlite3_mutex_enter(db->mutex); + v->doingRerun = 0; while( (rc = sqlite3Step(v))==SQLITE_SCHEMA && cnt++ < SQLITE_MAX_SCHEMA_RETRY && (rc2 = rc = sqlite3Reprepare(v))==SQLITE_OK ){ sqlite3_reset(pStmt); + v->doingRerun = 1; assert( v->expired==0 ); } if( rc2!=SQLITE_OK && ALWAYS(v->isPrepareV2) && ALWAYS(db->pErr) ){ @@ -1196,7 +1190,7 @@ int sqlite3VdbeParameterIndex(Vdbe *p, const char *zName, int nName){ if( zName ){ for(i=0; i<p->nzVar; i++){ const char *z = p->azVar[i]; - if( z && memcmp(z,zName,nName)==0 && z[nName]==0 ){ + if( z && strncmp(z,zName,nName)==0 && z[nName]==0 ){ return i+1; } } diff --git a/src/vdbeaux.c b/src/vdbeaux.c index d4f9864..2c4269a 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -17,18 +17,6 @@ #include "sqliteInt.h" #include "vdbeInt.h" - - -/* -** When debugging the code generator in a symbolic debugger, one can -** set the sqlite3VdbeAddopTrace to 1 and all opcodes will be printed -** as they are added to the instruction stream. -*/ -#ifdef SQLITE_DEBUG -int sqlite3VdbeAddopTrace = 0; -#endif - - /* ** Create a new virtual database engine. */ @@ -53,7 +41,7 @@ Vdbe *sqlite3VdbeCreate(sqlite3 *db){ void sqlite3VdbeSetSql(Vdbe *p, const char *z, int n, int isPrepareV2){ assert( isPrepareV2==1 || isPrepareV2==0 ); if( p==0 ) return; -#ifdef SQLITE_OMIT_TRACE +#if defined(SQLITE_OMIT_TRACE) && !defined(SQLITE_ENABLE_SQLLOG) if( !isPrepareV2 ) return; #endif assert( p->zSql==0 ); @@ -158,7 +146,9 @@ int sqlite3VdbeAddOp3(Vdbe *p, int op, int p1, int p2, int p3){ pOp->p4type = P4_NOTUSED; #ifdef SQLITE_DEBUG pOp->zComment = 0; - if( sqlite3VdbeAddopTrace ) sqlite3VdbePrintOp(0, i, &p->aOp[i]); + if( p->db->flags & SQLITE_VdbeAddopTrace ){ + sqlite3VdbePrintOp(0, i, &p->aOp[i]); + } #endif #ifdef VDBE_PROFILE pOp->cycles = 0; @@ -377,7 +367,7 @@ int sqlite3VdbeAssertMayAbort(Vdbe *v, int mayAbort){ || (opcode==OP_FkCounter && pOp->p1==0 && pOp->p2==1) #endif || ((opcode==OP_Halt || opcode==OP_HaltIfNull) - && (pOp->p1==SQLITE_CONSTRAINT && pOp->p2==OE_Abort)) + && ((pOp->p1&0xff)==SQLITE_CONSTRAINT && pOp->p2==OE_Abort)) ){ hasAbort = 1; break; @@ -385,7 +375,7 @@ int sqlite3VdbeAssertMayAbort(Vdbe *v, int mayAbort){ } sqlite3DbFree(v->db, sIter.apSub); - /* Return true if hasAbort==mayAbort. Or if a malloc failure occured. + /* Return true if hasAbort==mayAbort. Or if a malloc failure occurred. ** If malloc failed, then the while() loop above may not have iterated ** through all opcodes and hasAbort may be set incorrectly. Return ** true for this case to prevent the assert() in the callers frame @@ -512,7 +502,7 @@ int sqlite3VdbeAddOpList(Vdbe *p, int nOp, VdbeOpList const *aOp){ pOut->p5 = 0; #ifdef SQLITE_DEBUG pOut->zComment = 0; - if( sqlite3VdbeAddopTrace ){ + if( p->db->flags & SQLITE_VdbeAddopTrace ){ sqlite3VdbePrintOp(0, i+addr, &p->aOp[i+addr]); } #endif @@ -723,6 +713,7 @@ void sqlite3VdbeChangeP4(Vdbe *p, int addr, const char *zP4, int n){ addr = p->nOp - 1; } pOp = &p->aOp[addr]; + assert( pOp->p4type==P4_NOTUSED || pOp->p4type==P4_INT32 ); freeP4(db, pOp->p4type, pOp->p4.p); pOp->p4.p = 0; if( n==P4_INT32 ){ @@ -745,10 +736,9 @@ void sqlite3VdbeChangeP4(Vdbe *p, int addr, const char *zP4, int n){ u8 *aSortOrder; memcpy((char*)pKeyInfo, zP4, nByte - nField); aSortOrder = pKeyInfo->aSortOrder; - if( aSortOrder ){ - pKeyInfo->aSortOrder = (unsigned char*)&pKeyInfo->aColl[nField]; - memcpy(pKeyInfo->aSortOrder, aSortOrder, nField); - } + assert( aSortOrder!=0 ); + pKeyInfo->aSortOrder = (unsigned char*)&pKeyInfo->aColl[nField]; + memcpy(pKeyInfo->aSortOrder, aSortOrder, nField); pOp->p4type = P4_KEYINFO; }else{ p->db->mallocFailed = 1; @@ -861,26 +851,23 @@ static char *displayP4(Op *pOp, char *zTemp, int nTemp){ case P4_KEYINFO: { int i, j; KeyInfo *pKeyInfo = pOp->p4.pKeyInfo; + assert( pKeyInfo->aSortOrder!=0 ); sqlite3_snprintf(nTemp, zTemp, "keyinfo(%d", pKeyInfo->nField); i = sqlite3Strlen30(zTemp); for(j=0; j<pKeyInfo->nField; j++){ CollSeq *pColl = pKeyInfo->aColl[j]; - if( pColl ){ - int n = sqlite3Strlen30(pColl->zName); - if( i+n>nTemp-6 ){ - memcpy(&zTemp[i],",...",4); - break; - } - zTemp[i++] = ','; - if( pKeyInfo->aSortOrder && pKeyInfo->aSortOrder[j] ){ - zTemp[i++] = '-'; - } - memcpy(&zTemp[i], pColl->zName,n+1); - i += n; - }else if( i+4<nTemp-6 ){ - memcpy(&zTemp[i],",nil",4); - i += 4; + const char *zColl = pColl ? pColl->zName : "nil"; + int n = sqlite3Strlen30(zColl); + if( i+n>nTemp-6 ){ + memcpy(&zTemp[i],",...",4); + break; + } + zTemp[i++] = ','; + if( pKeyInfo->aSortOrder[j] ){ + zTemp[i++] = '-'; } + memcpy(&zTemp[i], zColl, n+1); + i += n; } zTemp[i++] = ')'; zTemp[i] = 0; @@ -1541,7 +1528,7 @@ void sqlite3VdbeMakeReady( zEnd = &zCsr[nByte]; }while( nByte && !db->mallocFailed ); - p->nCursor = (u16)nCursor; + p->nCursor = nCursor; p->nOnceFlag = nOnce; if( p->aVar ){ p->nVar = (ynVar)nVar; @@ -1770,7 +1757,9 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ if( sqlite3BtreeIsInTrans(pBt) ){ needXcommit = 1; if( i!=1 ) nTrans++; + sqlite3BtreeEnter(pBt); rc = sqlite3PagerExclusiveLock(sqlite3BtreePager(pBt)); + sqlite3BtreeLeave(pBt); } } if( rc!=SQLITE_OK ){ @@ -1781,7 +1770,7 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ if( needXcommit && db->xCommitCallback ){ rc = db->xCommitCallback(db->pCommitArg); if( rc ){ - return SQLITE_CONSTRAINT; + return SQLITE_CONSTRAINT_COMMITHOOK; } } @@ -2018,7 +2007,7 @@ int sqlite3VdbeCloseStatement(Vdbe *p, int eOp){ /* If p->iStatement is greater than zero, then this Vdbe opened a ** statement transaction that should be closed here. The only exception - ** is that an IO error may have occured, causing an emergency rollback. + ** is that an IO error may have occurred, causing an emergency rollback. ** In this case (db->nStatement==0), and there is nothing to do. */ if( db->nStatement && p->iStatement ){ @@ -2073,14 +2062,14 @@ int sqlite3VdbeCloseStatement(Vdbe *p, int eOp){ ** violations, return SQLITE_ERROR. Otherwise, SQLITE_OK. ** ** If there are outstanding FK violations and this function returns -** SQLITE_ERROR, set the result of the VM to SQLITE_CONSTRAINT and write -** an error message to it. Then return SQLITE_ERROR. +** SQLITE_ERROR, set the result of the VM to SQLITE_CONSTRAINT_FOREIGNKEY +** and write an error message to it. Then return SQLITE_ERROR. */ #ifndef SQLITE_OMIT_FOREIGN_KEY int sqlite3VdbeCheckFk(Vdbe *p, int deferred){ sqlite3 *db = p->db; if( (deferred && db->nDeferredCons>0) || (!deferred && p->nFkConstraint>0) ){ - p->rc = SQLITE_CONSTRAINT; + p->rc = SQLITE_CONSTRAINT_FOREIGNKEY; p->errorAction = OE_Abort; sqlite3SetString(&p->zErrMsg, db, "foreign key constraint failed"); return SQLITE_ERROR; @@ -2154,7 +2143,7 @@ int sqlite3VdbeHalt(Vdbe *p){ ** ** Even if the statement is read-only, it is important to perform ** a statement or transaction rollback operation. If the error - ** occured while writing to the journal, sub-journal or database + ** occurred while writing to the journal, sub-journal or database ** file as part of an effort to free up cache space (see function ** pagerStress() in pager.c), the rollback is required to restore ** the pager to a consistent state. @@ -2195,7 +2184,7 @@ int sqlite3VdbeHalt(Vdbe *p){ sqlite3VdbeLeave(p); return SQLITE_ERROR; } - rc = SQLITE_CONSTRAINT; + rc = SQLITE_CONSTRAINT_FOREIGNKEY; }else{ /* The auto-commit flag is true, the vdbe program was successful ** or hit an 'OR FAIL' constraint and there are no deferred foreign @@ -2238,7 +2227,7 @@ int sqlite3VdbeHalt(Vdbe *p){ if( eStatementOp ){ rc = sqlite3VdbeCloseStatement(p, eStatementOp); if( rc ){ - if( p->rc==SQLITE_OK || p->rc==SQLITE_CONSTRAINT ){ + if( p->rc==SQLITE_OK || (p->rc&0xff)==SQLITE_CONSTRAINT ){ p->rc = rc; sqlite3DbFree(db, p->zErrMsg); p->zErrMsg = 0; @@ -2324,6 +2313,27 @@ int sqlite3VdbeTransferError(Vdbe *p){ return rc; } +#ifdef SQLITE_ENABLE_SQLLOG +/* +** If an SQLITE_CONFIG_SQLLOG hook is registered and the VM has been run, +** invoke it. +*/ +static void vdbeInvokeSqllog(Vdbe *v){ + if( sqlite3GlobalConfig.xSqllog && v->rc==SQLITE_OK && v->zSql && v->pc>=0 ){ + char *zExpanded = sqlite3VdbeExpandSql(v, v->zSql); + assert( v->db->init.busy==0 ); + if( zExpanded ){ + sqlite3GlobalConfig.xSqllog( + sqlite3GlobalConfig.pSqllogArg, v->db, zExpanded, 1 + ); + sqlite3DbFree(v->db, zExpanded); + } + } +} +#else +# define vdbeInvokeSqllog(x) +#endif + /* ** Clean up a VDBE after execution but do not delete the VDBE just yet. ** Write any error messages into *pzErrMsg. Return the result code. @@ -2351,6 +2361,7 @@ int sqlite3VdbeReset(Vdbe *p){ ** instructions yet, leave the main database error information unchanged. */ if( p->pc>=0 ){ + vdbeInvokeSqllog(p); sqlite3VdbeTransferError(p); sqlite3DbFree(db, p->zErrMsg); p->zErrMsg = 0; @@ -2432,12 +2443,14 @@ void sqlite3VdbeDeleteAuxData(VdbeFunc *pVdbeFunc, int mask){ } /* -** Free all memory associated with the Vdbe passed as the second argument. +** Free all memory associated with the Vdbe passed as the second argument, +** except for object itself, which is preserved. +** ** The difference between this function and sqlite3VdbeDelete() is that ** VdbeDelete() also unlinks the Vdbe from the list of VMs associated with -** the database connection. +** the database connection and frees the object itself. */ -void sqlite3VdbeDeleteObject(sqlite3 *db, Vdbe *p){ +void sqlite3VdbeClearObject(sqlite3 *db, Vdbe *p){ SubProgram *pSub, *pNext; int i; assert( p->db==0 || p->db==db ); @@ -2458,7 +2471,6 @@ void sqlite3VdbeDeleteObject(sqlite3 *db, Vdbe *p){ sqlite3DbFree(db, p->zExplain); sqlite3DbFree(db, p->pExplain); #endif - sqlite3DbFree(db, p); } /* @@ -2470,6 +2482,7 @@ void sqlite3VdbeDelete(Vdbe *p){ if( NEVER(p==0) ) return; db = p->db; assert( sqlite3_mutex_held(db->mutex) ); + sqlite3VdbeClearObject(db, p); if( p->pPrev ){ p->pPrev->pNext = p->pNext; }else{ @@ -2481,7 +2494,7 @@ void sqlite3VdbeDelete(Vdbe *p){ } p->magic = VDBE_MAGIC_DEAD; p->db = 0; - sqlite3VdbeDeleteObject(db, p); + sqlite3DbFree(db, p); } /* @@ -2544,7 +2557,7 @@ int sqlite3VdbeCursorMoveto(VdbeCursor *p){ ** the blob of data that it corresponds to. In a table record, all serial ** types are stored at the start of the record, and the blobs of data at ** the end. Hence these functions allow the caller to handle the -** serial-type and data blob seperately. +** serial-type and data blob separately. ** ** The following table describes the various storage classes for data: ** @@ -2583,9 +2596,6 @@ u32 sqlite3VdbeSerialType(Mem *pMem, int file_format){ # define MAX_6BYTE ((((i64)0x00008000)<<32)-1) i64 i = pMem->u.i; u64 u; - if( file_format>=4 && (i&1)==i ){ - return 8+(u32)i; - } if( i<0 ){ if( i<(-MAX_6BYTE) ) return 6; /* Previous test prevents: u = -(-9223372036854775808) */ @@ -2593,7 +2603,9 @@ u32 sqlite3VdbeSerialType(Mem *pMem, int file_format){ }else{ u = i; } - if( u<=127 ) return 1; + if( u<=127 ){ + return ((i&1)==i && file_format>=4) ? 8+(u32)u : 1; + } if( u<=32767 ) return 2; if( u<=8388607 ) return 3; if( u<=2147483647 ) return 4; @@ -2878,6 +2890,7 @@ UnpackedRecord *sqlite3VdbeAllocUnpackedRecord( } p->aMem = (Mem*)&((char*)p)[ROUND8(sizeof(UnpackedRecord))]; + assert( pKeyInfo->aSortOrder!=0 ); p->pKeyInfo = pKeyInfo; p->nField = pKeyInfo->nField + 1; return p; @@ -2971,6 +2984,7 @@ int sqlite3VdbeRecordCompare( idx1 = getVarint32(aKey1, szHdr1); d1 = szHdr1; nField = pKeyInfo->nField; + assert( pKeyInfo->aSortOrder!=0 ); while( idx1<szHdr1 && i<pPKey2->nField ){ u32 serial_type1; @@ -2990,7 +3004,7 @@ int sqlite3VdbeRecordCompare( assert( mem1.zMalloc==0 ); /* See comment below */ /* Invert the result if we are using DESC sort order. */ - if( pKeyInfo->aSortOrder && i<nField && pKeyInfo->aSortOrder[i] ){ + if( i<nField && pKeyInfo->aSortOrder[i] ){ rc = -rc; } diff --git a/src/vdbeblob.c b/src/vdbeblob.c index ae77a47..2e8fd8e 100644 --- a/src/vdbeblob.c +++ b/src/vdbeblob.c @@ -313,7 +313,7 @@ int sqlite3_blob_open( } sqlite3_bind_int64(pBlob->pStmt, 1, iRow); rc = blobSeekToRow(pBlob, iRow, &zErr); - } while( (++nAttempt)<5 && rc==SQLITE_SCHEMA ); + } while( (++nAttempt)<SQLITE_MAX_SCHEMA_RETRY && rc==SQLITE_SCHEMA ); blob_open_out: if( rc==SQLITE_OK && db->mallocFailed==0 ){ diff --git a/src/vdbemem.c b/src/vdbemem.c index fd964de..8fc222e 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -32,7 +32,9 @@ ** between formats. */ int sqlite3VdbeChangeEncoding(Mem *pMem, int desiredEnc){ +#ifndef SQLITE_OMIT_UTF16 int rc; +#endif assert( (pMem->flags&MEM_RowSet)==0 ); assert( desiredEnc==SQLITE_UTF8 || desiredEnc==SQLITE_UTF16LE || desiredEnc==SQLITE_UTF16BE ); diff --git a/src/vdbesort.c b/src/vdbesort.c index ba1e9f0..fdfc4a7 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -18,7 +18,6 @@ #include "sqliteInt.h" #include "vdbeInt.h" -#ifndef SQLITE_OMIT_MERGE_SORT typedef struct VdbeSorterIter VdbeSorterIter; typedef struct SorterRecord SorterRecord; @@ -195,8 +194,11 @@ static int vdbeSorterIterRead( int rc; /* sqlite3OsRead() return code */ /* Determine how many bytes of data to read. */ - nRead = (int)(p->iEof - p->iReadOff); - if( nRead>p->nBuffer ) nRead = p->nBuffer; + if( (p->iEof - p->iReadOff) > (i64)p->nBuffer ){ + nRead = p->nBuffer; + }else{ + nRead = (int)(p->iEof - p->iReadOff); + } assert( nRead>0 ); /* Read data from the file. Return early if an error occurs. */ @@ -1034,5 +1036,3 @@ int sqlite3VdbeSorterCompare( vdbeSorterCompare(pCsr, 1, pVal->z, pVal->n, pKey, nKey, pRes); return SQLITE_OK; } - -#endif /* #ifndef SQLITE_OMIT_MERGE_SORT */ diff --git a/src/vdbetrace.c b/src/vdbetrace.c index 35825c8..356277e 100644 --- a/src/vdbetrace.c +++ b/src/vdbetrace.c @@ -53,6 +53,11 @@ static int findNextHostParameter(const char *zSql, int *pnToken){ ** then the returned string holds a copy of zRawSql with "-- " prepended ** to each line of text. ** +** If the SQLITE_TRACE_SIZE_LIMIT macro is defined to an integer, then +** then long strings and blobs are truncated to that many bytes. This +** can be used to prevent unreasonably large trace strings when dealing +** with large (multi-megabyte) strings and blobs. +** ** The calling function is responsible for making sure the memory returned ** is eventually freed. ** @@ -123,30 +128,49 @@ char *sqlite3VdbeExpandSql( }else if( pVar->flags & MEM_Real ){ sqlite3XPrintf(&out, "%!.15g", pVar->r); }else if( pVar->flags & MEM_Str ){ + int nOut; /* Number of bytes of the string text to include in output */ #ifndef SQLITE_OMIT_UTF16 u8 enc = ENC(db); + Mem utf8; if( enc!=SQLITE_UTF8 ){ - Mem utf8; memset(&utf8, 0, sizeof(utf8)); utf8.db = db; sqlite3VdbeMemSetStr(&utf8, pVar->z, pVar->n, enc, SQLITE_STATIC); sqlite3VdbeChangeEncoding(&utf8, SQLITE_UTF8); - sqlite3XPrintf(&out, "'%.*q'", utf8.n, utf8.z); - sqlite3VdbeMemRelease(&utf8); - }else + pVar = &utf8; + } #endif - { - sqlite3XPrintf(&out, "'%.*q'", pVar->n, pVar->z); + nOut = pVar->n; +#ifdef SQLITE_TRACE_SIZE_LIMIT + if( nOut>SQLITE_TRACE_SIZE_LIMIT ){ + nOut = SQLITE_TRACE_SIZE_LIMIT; + while( nOut<pVar->n && (pVar->z[nOut]&0xc0)==0x80 ){ nOut++; } } +#endif + sqlite3XPrintf(&out, "'%.*q'", nOut, pVar->z); +#ifdef SQLITE_TRACE_SIZE_LIMIT + if( nOut<pVar->n ) sqlite3XPrintf(&out, "/*+%d bytes*/", pVar->n-nOut); +#endif +#ifndef SQLITE_OMIT_UTF16 + if( enc!=SQLITE_UTF8 ) sqlite3VdbeMemRelease(&utf8); +#endif }else if( pVar->flags & MEM_Zero ){ sqlite3XPrintf(&out, "zeroblob(%d)", pVar->u.nZero); }else{ + int nOut; /* Number of bytes of the blob to include in output */ assert( pVar->flags & MEM_Blob ); sqlite3StrAccumAppend(&out, "x'", 2); - for(i=0; i<pVar->n; i++){ + nOut = pVar->n; +#ifdef SQLITE_TRACE_SIZE_LIMIT + if( nOut>SQLITE_TRACE_SIZE_LIMIT ) nOut = SQLITE_TRACE_SIZE_LIMIT; +#endif + for(i=0; i<nOut; i++){ sqlite3XPrintf(&out, "%02x", pVar->z[i]&0xff); } sqlite3StrAccumAppend(&out, "'", 1); +#ifdef SQLITE_TRACE_SIZE_LIMIT + if( nOut<pVar->n ) sqlite3XPrintf(&out, "/*+%d bytes*/", pVar->n-nOut); +#endif } } } @@ -264,7 +264,7 @@ void sqlite3VtabClear(sqlite3 *db, Table *p){ if( p->azModuleArg ){ int i; for(i=0; i<p->nModuleArg; i++){ - sqlite3DbFree(db, p->azModuleArg[i]); + if( i!=1 ) sqlite3DbFree(db, p->azModuleArg[i]); } sqlite3DbFree(db, p->azModuleArg); } @@ -324,7 +324,7 @@ void sqlite3VtabBeginParse( pTable->tabFlags |= TF_Virtual; pTable->nModuleArg = 0; addModuleArgument(db, pTable, sqlite3NameFromToken(db, pModuleName)); - addModuleArgument(db, pTable, sqlite3DbStrDup(db, db->aDb[iDb].zName)); + addModuleArgument(db, pTable, 0); addModuleArgument(db, pTable, sqlite3DbStrDup(db, pTable->zName)); pParse->sNameToken.n = (int)(&pModuleName->z[pModuleName->n] - pName1->z); @@ -481,6 +481,7 @@ static int vtabCallConstructor( int nArg = pTab->nModuleArg; char *zErr = 0; char *zModuleName = sqlite3MPrintf(db, "%s", pTab->zName); + int iDb; if( !zModuleName ){ return SQLITE_NOMEM; @@ -494,6 +495,9 @@ static int vtabCallConstructor( pVTable->db = db; pVTable->pMod = pMod; + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + pTab->azModuleArg[1] = db->aDb[iDb].zName; + /* Invoke the virtual table constructor */ assert( &db->pVtabCtx ); assert( xConstruct ); @@ -528,7 +532,7 @@ static int vtabCallConstructor( /* If everything went according to plan, link the new VTable structure ** into the linked list headed by pTab->pVTable. Then loop through the ** columns of the table to see if any of them contain the token "hidden". - ** If so, set the Column.isHidden flag and remove the token from + ** If so, set the Column COLFLAG_HIDDEN flag and remove the token from ** the type string. */ pVTable->pNext = pTab->pVTable; pTab->pVTable = pVTable; @@ -559,7 +563,7 @@ static int vtabCallConstructor( assert(zType[i-1]==' '); zType[i-1] = '\0'; } - pTab->aCol[iCol].isHidden = 1; + pTab->aCol[iCol].colFlags |= COLFLAG_HIDDEN; } } } @@ -1207,8 +1207,9 @@ finished: ** checkpointing the log file. */ if( pWal->hdr.nPage ){ - sqlite3_log(SQLITE_OK, "Recovered %d frames from WAL file %s", - pWal->hdr.nPage, pWal->zWalName + sqlite3_log(SQLITE_NOTICE_RECOVER_WAL, + "recovered %d frames from WAL file %s", + pWal->hdr.mxFrame, pWal->zWalName ); } } @@ -1722,8 +1723,8 @@ static int walCheckpoint( rc = sqlite3OsSync(pWal->pWalFd, sync_flags); } - /* If the database file may grow as a result of this checkpoint, hint - ** about the eventual size of the db file to the VFS layer. + /* If the database may grow as a result of this checkpoint, hint + ** about the eventual size of the db file to the VFS layer. */ if( rc==SQLITE_OK ){ i64 nReq = ((i64)mxPage * szPage); @@ -1733,6 +1734,7 @@ static int walCheckpoint( } } + /* Iterate through the contents of the WAL, copying data to the db file. */ while( rc==SQLITE_OK && 0==walIteratorNext(pIter, &iDbpage, &iFrame) ){ i64 iOffset; @@ -2287,19 +2289,17 @@ void sqlite3WalEndReadTransaction(Wal *pWal){ } /* -** Read a page from the WAL, if it is present in the WAL and if the -** current read transaction is configured to use the WAL. +** Search the wal file for page pgno. If found, set *piRead to the frame that +** contains the page. Otherwise, if pgno is not in the wal file, set *piRead +** to zero. ** -** The *pInWal is set to 1 if the requested page is in the WAL and -** has been loaded. Or *pInWal is set to 0 if the page was not in -** the WAL and needs to be read out of the database. +** Return SQLITE_OK if successful, or an error code if an error occurs. If an +** error does occur, the final value of *piRead is undefined. */ -int sqlite3WalRead( +int sqlite3WalFindFrame( Wal *pWal, /* WAL handle */ Pgno pgno, /* Database page number to read data for */ - int *pInWal, /* OUT: True if data is read from WAL */ - int nOut, /* Size of buffer pOut in bytes */ - u8 *pOut /* Buffer to write page data to */ + u32 *piRead /* OUT: Frame number (or zero) */ ){ u32 iRead = 0; /* If !=0, WAL frame to return data from */ u32 iLast = pWal->hdr.mxFrame; /* Last page in WAL for this reader */ @@ -2315,7 +2315,7 @@ int sqlite3WalRead( ** WAL were empty. */ if( iLast==0 || pWal->readLock==0 ){ - *pInWal = 0; + *piRead = 0; return SQLITE_OK; } @@ -2386,26 +2386,31 @@ int sqlite3WalRead( } #endif - /* If iRead is non-zero, then it is the log frame number that contains the - ** required page. Read and return data from the log file. - */ - if( iRead ){ - int sz; - i64 iOffset; - sz = pWal->hdr.szPage; - sz = (sz&0xfe00) + ((sz&0x0001)<<16); - testcase( sz<=32768 ); - testcase( sz>=65536 ); - iOffset = walFrameOffset(iRead, sz) + WAL_FRAME_HDRSIZE; - *pInWal = 1; - /* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL */ - return sqlite3OsRead(pWal->pWalFd, pOut, (nOut>sz ? sz : nOut), iOffset); - } - - *pInWal = 0; + *piRead = iRead; return SQLITE_OK; } +/* +** Read the contents of frame iRead from the wal file into buffer pOut +** (which is nOut bytes in size). Return SQLITE_OK if successful, or an +** error code otherwise. +*/ +int sqlite3WalReadFrame( + Wal *pWal, /* WAL handle */ + u32 iRead, /* Frame to read */ + int nOut, /* Size of buffer pOut in bytes */ + u8 *pOut /* Buffer to write page data to */ +){ + int sz; + i64 iOffset; + sz = pWal->hdr.szPage; + sz = (sz&0xfe00) + ((sz&0x0001)<<16); + testcase( sz<=32768 ); + testcase( sz>=65536 ); + iOffset = walFrameOffset(iRead, sz) + WAL_FRAME_HDRSIZE; + /* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL */ + return sqlite3OsRead(pWal->pWalFd, pOut, (nOut>sz ? sz : nOut), iOffset); +} /* ** Return the size of the database in pages (or zero, if unknown). @@ -2518,7 +2523,7 @@ int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx){ assert( walFramePgno(pWal, iFrame)!=1 ); rc = xUndo(pUndoCtx, walFramePgno(pWal, iFrame)); } - walCleanupHash(pWal); + if( iMax!=pWal->hdr.mxFrame ) walCleanupHash(pWal); } assert( rc==SQLITE_OK ); return rc; @@ -2828,7 +2833,7 @@ int sqlite3WalFrames( */ if( isCommit && (sync_flags & WAL_SYNC_TRANSACTIONS)!=0 ){ if( pWal->padToSectorBoundary ){ - int sectorSize = sqlite3OsSectorSize(pWal->pWalFd); + int sectorSize = sqlite3SectorSize(pWal->pWalFd); w.iSyncPoint = ((iOffset+sectorSize-1)/sectorSize)*sectorSize; while( iOffset<w.iSyncPoint ){ rc = walWriteOneFrame(&w, pLast, nTruncate, iOffset); @@ -2952,6 +2957,9 @@ int sqlite3WalCheckpoint( /* Read the wal-index header. */ if( rc==SQLITE_OK ){ rc = walIndexReadHdr(pWal, &isChanged); + if( isChanged && pWal->pDbFd->pMethods->iVersion>=3 ){ + sqlite3OsUnfetch(pWal->pDbFd, 0, 0); + } } /* Copy data from the log to the database file. */ @@ -31,7 +31,6 @@ # define sqlite3WalClose(w,x,y,z) 0 # define sqlite3WalBeginReadTransaction(y,z) 0 # define sqlite3WalEndReadTransaction(z) -# define sqlite3WalRead(v,w,x,y,z) 0 # define sqlite3WalDbsize(y) 0 # define sqlite3WalBeginWriteTransaction(y) 0 # define sqlite3WalEndWriteTransaction(x) 0 @@ -44,6 +43,7 @@ # define sqlite3WalExclusiveMode(y,z) 0 # define sqlite3WalHeapMemory(z) 0 # define sqlite3WalFramesize(z) 0 +# define sqlite3WalFindFrame(x,y,z) 0 #else #define WAL_SAVEPOINT_NDATA 4 @@ -71,7 +71,8 @@ int sqlite3WalBeginReadTransaction(Wal *pWal, int *); void sqlite3WalEndReadTransaction(Wal *pWal); /* Read a page from the write-ahead log, if it is present. */ -int sqlite3WalRead(Wal *pWal, Pgno pgno, int *pInWal, int nOut, u8 *pOut); +int sqlite3WalFindFrame(Wal *, Pgno, u32 *); +int sqlite3WalReadFrame(Wal *, u32, int, u8 *); /* If the WAL is not empty, return the size of the database. */ Pgno sqlite3WalDbsize(Wal *pWal); diff --git a/src/walker.c b/src/walker.c index eab96ea..e71ed2a 100644 --- a/src/walker.c +++ b/src/walker.c @@ -113,7 +113,9 @@ int sqlite3WalkSelectFrom(Walker *pWalker, Select *p){ /* ** Call sqlite3WalkExpr() for every expression in Select statement p. ** Invoke sqlite3WalkSelect() for subqueries in the FROM clause and -** on the compound select chain, p->pPrior. +** on the compound select chain, p->pPrior. Invoke the xSelectCallback() +** either before or after the walk of expressions and FROM clause, depending +** on whether pWalker->bSelectDepthFirst is false or true, respectively. ** ** Return WRC_Continue under normal conditions. Return WRC_Abort if ** there is an abort request. @@ -127,14 +129,23 @@ int sqlite3WalkSelect(Walker *pWalker, Select *p){ rc = WRC_Continue; pWalker->walkerDepth++; while( p ){ - rc = pWalker->xSelectCallback(pWalker, p); - if( rc ) break; + if( !pWalker->bSelectDepthFirst ){ + rc = pWalker->xSelectCallback(pWalker, p); + if( rc ) break; + } if( sqlite3WalkSelectExpr(pWalker, p) || sqlite3WalkSelectFrom(pWalker, p) ){ pWalker->walkerDepth--; return WRC_Abort; } + if( pWalker->bSelectDepthFirst ){ + rc = pWalker->xSelectCallback(pWalker, p); + /* Depth-first search is currently only used for + ** selectAddSubqueryTypeInfo() and that routine always returns + ** WRC_Continue (0). So the following branch is never taken. */ + if( NEVER(rc) ) break; + } p = p->pPrior; } pWalker->walkerDepth--; diff --git a/src/where.c b/src/where.c index 216a47f..e614f4a 100644 --- a/src/where.c +++ b/src/where.c @@ -23,9 +23,10 @@ ** Trace output macros */ #if defined(SQLITE_TEST) || defined(SQLITE_DEBUG) -int sqlite3WhereTrace = 0; +/***/ int sqlite3WhereTrace = 0; #endif -#if defined(SQLITE_TEST) && defined(SQLITE_DEBUG) +#if defined(SQLITE_DEBUG) \ + && (defined(SQLITE_TEST) || defined(SQLITE_ENABLE_WHERETRACE)) # define WHERETRACE(X) if(sqlite3WhereTrace) sqlite3DebugPrintf X #else # define WHERETRACE(X) @@ -97,8 +98,8 @@ struct WhereTerm { int leftCursor; /* Cursor number of X in "X <op> <expr>" */ union { int leftColumn; /* Column number of X in "X <op> <expr>" */ - WhereOrInfo *pOrInfo; /* Extra information if eOperator==WO_OR */ - WhereAndInfo *pAndInfo; /* Extra information if eOperator==WO_AND */ + WhereOrInfo *pOrInfo; /* Extra information if (eOperator & WO_OR)!=0 */ + WhereAndInfo *pAndInfo; /* Extra information if (eOperator& WO_AND)!=0 */ } u; u16 eOperator; /* A WO_xx value describing <op> */ u8 wtFlags; /* TERM_xxx bit flags. See below */ @@ -139,7 +140,6 @@ struct WhereTerm { struct WhereClause { Parse *pParse; /* The parser context */ WhereMaskSet *pMaskSet; /* Mapping of table cursor numbers to bitmasks */ - Bitmask vmask; /* Bitmask identifying virtual table cursors */ WhereClause *pOuter; /* Outer conjunction */ u8 op; /* Split operator. TK_AND or TK_OR */ u16 wctrlFlags; /* Might include WHERE_AND_ONLY */ @@ -226,6 +226,7 @@ struct WhereCost { #define WO_ISNULL 0x080 #define WO_OR 0x100 /* Two or more OR-connected terms */ #define WO_AND 0x200 /* Two or more AND-connected terms */ +#define WO_EQUIV 0x400 /* Of the form A==B, both columns */ #define WO_NOOP 0x800 /* This term does not restrict search space */ #define WO_ALL 0xfff /* Mask of all possible WO_* values */ @@ -252,18 +253,55 @@ struct WhereCost { #define WHERE_COLUMN_NULL 0x00080000 /* x IS NULL */ #define WHERE_INDEXED 0x000f0000 /* Anything that uses an index */ #define WHERE_NOT_FULLSCAN 0x100f3000 /* Does not do a full table scan */ -#define WHERE_IN_ABLE 0x000f1000 /* Able to support an IN operator */ +#define WHERE_IN_ABLE 0x080f1000 /* Able to support an IN operator */ #define WHERE_TOP_LIMIT 0x00100000 /* x<EXPR or x<=EXPR constraint */ #define WHERE_BTM_LIMIT 0x00200000 /* x>EXPR or x>=EXPR constraint */ #define WHERE_BOTH_LIMIT 0x00300000 /* Both x>EXPR and x<EXPR */ -#define WHERE_IDX_ONLY 0x00800000 /* Use index only - omit table */ -#define WHERE_ORDERBY 0x01000000 /* Output will appear in correct order */ -#define WHERE_REVERSE 0x02000000 /* Scan in reverse order */ -#define WHERE_UNIQUE 0x04000000 /* Selects no more than one row */ +#define WHERE_IDX_ONLY 0x00400000 /* Use index only - omit table */ +#define WHERE_ORDERED 0x00800000 /* Output will appear in correct order */ +#define WHERE_REVERSE 0x01000000 /* Scan in reverse order */ +#define WHERE_UNIQUE 0x02000000 /* Selects no more than one row */ +#define WHERE_ALL_UNIQUE 0x04000000 /* This and all prior have one row */ +#define WHERE_OB_UNIQUE 0x00004000 /* Values in ORDER BY columns are + ** different for every output row */ #define WHERE_VIRTUALTABLE 0x08000000 /* Use virtual-table processing */ #define WHERE_MULTI_OR 0x10000000 /* OR using multiple indices */ #define WHERE_TEMP_INDEX 0x20000000 /* Uses an ephemeral index */ #define WHERE_DISTINCT 0x40000000 /* Correct order for DISTINCT */ +#define WHERE_COVER_SCAN 0x80000000 /* Full scan of a covering index */ + +/* +** This module contains many separate subroutines that work together to +** find the best indices to use for accessing a particular table in a query. +** An instance of the following structure holds context information about the +** index search so that it can be more easily passed between the various +** routines. +*/ +typedef struct WhereBestIdx WhereBestIdx; +struct WhereBestIdx { + Parse *pParse; /* Parser context */ + WhereClause *pWC; /* The WHERE clause */ + struct SrcList_item *pSrc; /* The FROM clause term to search */ + Bitmask notReady; /* Mask of cursors not available */ + Bitmask notValid; /* Cursors not available for any purpose */ + ExprList *pOrderBy; /* The ORDER BY clause */ + ExprList *pDistinct; /* The select-list if query is DISTINCT */ + sqlite3_index_info **ppIdxInfo; /* Index information passed to xBestIndex */ + int i, n; /* Which loop is being coded; # of loops */ + WhereLevel *aLevel; /* Info about outer loops */ + WhereCost cost; /* Lowest cost query plan */ +}; + +/* +** Return TRUE if the probe cost is less than the baseline cost +*/ +static int compareCost(const WhereCost *pProbe, const WhereCost *pBaseline){ + if( pProbe->rCost<pBaseline->rCost ) return 1; + if( pProbe->rCost>pBaseline->rCost ) return 0; + if( pProbe->plan.nOBSat>pBaseline->plan.nOBSat ) return 1; + if( pProbe->plan.nRow<pBaseline->plan.nRow ) return 1; + return 0; +} /* ** Initialize a preallocated WhereClause structure. @@ -280,7 +318,6 @@ static void whereClauseInit( pWC->nTerm = 0; pWC->nSlot = ArraySize(pWC->aStatic); pWC->a = pWC->aStatic; - pWC->vmask = 0; pWC->wctrlFlags = wctrlFlags; } @@ -367,7 +404,7 @@ static int whereClauseInsert(WhereClause *pWC, Expr *p, u8 wtFlags){ pWC->nSlot = sqlite3DbMallocSize(db, pWC->a)/sizeof(pWC->a[0]); } pTerm = &pWC->a[idx = pWC->nTerm++]; - pTerm->pExpr = p; + pTerm->pExpr = sqlite3ExprSkipCollate(p); pTerm->wtFlags = wtFlags; pTerm->pWC = pWC; pTerm->iParent = -1; @@ -527,23 +564,32 @@ static int allowedOp(int op){ ** Commute a comparison operator. Expressions of the form "X op Y" ** are converted into "Y op X". ** -** If a collation sequence is associated with either the left or right +** If left/right precedence rules come into play when determining the +** collating ** side of the comparison, it remains associated with the same side after ** the commutation. So "Y collate NOCASE op X" becomes -** "X collate NOCASE op Y". This is because any collation sequence on +** "X op Y". This is because any collation sequence on ** the left hand side of a comparison overrides any collation sequence -** attached to the right. For the same reason the EP_ExpCollate flag +** attached to the right. For the same reason the EP_Collate flag ** is not commuted. */ static void exprCommute(Parse *pParse, Expr *pExpr){ - u16 expRight = (pExpr->pRight->flags & EP_ExpCollate); - u16 expLeft = (pExpr->pLeft->flags & EP_ExpCollate); + u16 expRight = (pExpr->pRight->flags & EP_Collate); + u16 expLeft = (pExpr->pLeft->flags & EP_Collate); assert( allowedOp(pExpr->op) && pExpr->op!=TK_IN ); - pExpr->pRight->pColl = sqlite3ExprCollSeq(pParse, pExpr->pRight); - pExpr->pLeft->pColl = sqlite3ExprCollSeq(pParse, pExpr->pLeft); - SWAP(CollSeq*,pExpr->pRight->pColl,pExpr->pLeft->pColl); - pExpr->pRight->flags = (pExpr->pRight->flags & ~EP_ExpCollate) | expLeft; - pExpr->pLeft->flags = (pExpr->pLeft->flags & ~EP_ExpCollate) | expRight; + if( expRight==expLeft ){ + /* Either X and Y both have COLLATE operator or neither do */ + if( expRight ){ + /* Both X and Y have COLLATE operators. Make sure X is always + ** used by clearing the EP_Collate flag from Y. */ + pExpr->pRight->flags &= ~EP_Collate; + }else if( sqlite3ExprCollSeq(pParse, pExpr->pLeft)!=0 ){ + /* Neither X nor Y have COLLATE operators, but X has a non-default + ** collating sequence. So add the EP_Collate marker on X to cause + ** it to be searched first. */ + pExpr->pLeft->flags |= EP_Collate; + } + } SWAP(Expr*,pExpr->pRight,pExpr->pLeft); if( pExpr->op>=TK_GT ){ assert( TK_LT==TK_GT+2 ); @@ -584,6 +630,23 @@ static u16 operatorMask(int op){ ** where X is a reference to the iColumn of table iCur and <op> is one of ** the WO_xx operator codes specified by the op parameter. ** Return a pointer to the term. Return 0 if not found. +** +** The term returned might by Y=<expr> if there is another constraint in +** the WHERE clause that specifies that X=Y. Any such constraints will be +** identified by the WO_EQUIV bit in the pTerm->eOperator field. The +** aEquiv[] array holds X and all its equivalents, with each SQL variable +** taking up two slots in aEquiv[]. The first slot is for the cursor number +** and the second is for the column number. There are 22 slots in aEquiv[] +** so that means we can look for X plus up to 10 other equivalent values. +** Hence a search for X will return <expr> if X=A1 and A1=A2 and A2=A3 +** and ... and A9=A10 and A10=<expr>. +** +** If there are multiple terms in the WHERE clause of the form "X <op> <expr>" +** then try for the one with no dependencies on <expr> - in other words where +** <expr> is a constant expression of some kind. Only return entries of +** the form "X <op> Y" where Y is a column in another table if no terms of +** the form "X <op> <const-expr>" exist. If no terms with a constant RHS +** exist, try to return a term that does not use WO_EQUIV. */ static WhereTerm *findTerm( WhereClause *pWC, /* The WHERE clause to be searched */ @@ -593,45 +656,85 @@ static WhereTerm *findTerm( u32 op, /* Mask of WO_xx values describing operator */ Index *pIdx /* Must be compatible with this index, if not NULL */ ){ - WhereTerm *pTerm; - int k; + WhereTerm *pTerm; /* Term being examined as possible result */ + WhereTerm *pResult = 0; /* The answer to return */ + WhereClause *pWCOrig = pWC; /* Original pWC value */ + int j, k; /* Loop counters */ + Expr *pX; /* Pointer to an expression */ + Parse *pParse; /* Parsing context */ + int iOrigCol = iColumn; /* Original value of iColumn */ + int nEquiv = 2; /* Number of entires in aEquiv[] */ + int iEquiv = 2; /* Number of entries of aEquiv[] processed so far */ + int aEquiv[22]; /* iCur,iColumn and up to 10 other equivalents */ + assert( iCur>=0 ); - op &= WO_ALL; - for(; pWC; pWC=pWC->pOuter){ - for(pTerm=pWC->a, k=pWC->nTerm; k; k--, pTerm++){ - if( pTerm->leftCursor==iCur - && (pTerm->prereqRight & notReady)==0 - && pTerm->u.leftColumn==iColumn - && (pTerm->eOperator & op)!=0 - ){ - if( iColumn>=0 && pIdx && pTerm->eOperator!=WO_ISNULL ){ - Expr *pX = pTerm->pExpr; - CollSeq *pColl; - char idxaff; - int j; - Parse *pParse = pWC->pParse; - - idxaff = pIdx->pTable->aCol[iColumn].affinity; - if( !sqlite3IndexAffinityOk(pX, idxaff) ) continue; - - /* Figure out the collation sequence required from an index for - ** it to be useful for optimising expression pX. Store this - ** value in variable pColl. - */ - assert(pX->pLeft); - pColl = sqlite3BinaryCompareCollSeq(pParse, pX->pLeft, pX->pRight); - assert(pColl || pParse->nErr); - - for(j=0; pIdx->aiColumn[j]!=iColumn; j++){ - if( NEVER(j>=pIdx->nColumn) ) return 0; + aEquiv[0] = iCur; + aEquiv[1] = iColumn; + for(;;){ + for(pWC=pWCOrig; pWC; pWC=pWC->pOuter){ + for(pTerm=pWC->a, k=pWC->nTerm; k; k--, pTerm++){ + if( pTerm->leftCursor==iCur + && pTerm->u.leftColumn==iColumn + ){ + if( (pTerm->prereqRight & notReady)==0 + && (pTerm->eOperator & op & WO_ALL)!=0 + ){ + if( iOrigCol>=0 && pIdx && (pTerm->eOperator & WO_ISNULL)==0 ){ + CollSeq *pColl; + char idxaff; + + pX = pTerm->pExpr; + pParse = pWC->pParse; + idxaff = pIdx->pTable->aCol[iOrigCol].affinity; + if( !sqlite3IndexAffinityOk(pX, idxaff) ){ + continue; + } + + /* Figure out the collation sequence required from an index for + ** it to be useful for optimising expression pX. Store this + ** value in variable pColl. + */ + assert(pX->pLeft); + pColl = sqlite3BinaryCompareCollSeq(pParse,pX->pLeft,pX->pRight); + if( pColl==0 ) pColl = pParse->db->pDfltColl; + + for(j=0; pIdx->aiColumn[j]!=iOrigCol; j++){ + if( NEVER(j>=pIdx->nColumn) ) return 0; + } + if( sqlite3StrICmp(pColl->zName, pIdx->azColl[j]) ){ + continue; + } + } + if( pTerm->prereqRight==0 && (pTerm->eOperator&WO_EQ)!=0 ){ + pResult = pTerm; + goto findTerm_success; + }else if( pResult==0 ){ + pResult = pTerm; + } + } + if( (pTerm->eOperator & WO_EQUIV)!=0 + && nEquiv<ArraySize(aEquiv) + ){ + pX = sqlite3ExprSkipCollate(pTerm->pExpr->pRight); + assert( pX->op==TK_COLUMN ); + for(j=0; j<nEquiv; j+=2){ + if( aEquiv[j]==pX->iTable && aEquiv[j+1]==pX->iColumn ) break; + } + if( j==nEquiv ){ + aEquiv[j] = pX->iTable; + aEquiv[j+1] = pX->iColumn; + nEquiv += 2; + } } - if( pColl && sqlite3StrICmp(pColl->zName, pIdx->azColl[j]) ) continue; } - return pTerm; } } + if( iEquiv>=nEquiv ) break; + iCur = aEquiv[iEquiv++]; + iColumn = aEquiv[iEquiv++]; } - return 0; +findTerm_success: + return pResult; } /* Forward reference */ @@ -817,7 +920,7 @@ static void transferJoinMarkings(Expr *pDerived, Expr *pBase){ ** ** CASE 1: ** -** If all subterms are of the form T.C=expr for some single column of C +** If all subterms are of the form T.C=expr for some single column of C and ** a single table T (as shown in example B above) then create a new virtual ** term that is an equivalent IN expression. In other words, if the term ** being analyzed is: @@ -905,11 +1008,10 @@ static void exprAnalyzeOrTerm( ** Compute the set of tables that might satisfy cases 1 or 2. */ indexable = ~(Bitmask)0; - chngToIN = ~(pWC->vmask); + chngToIN = ~(Bitmask)0; for(i=pOrWc->nTerm-1, pOrTerm=pOrWc->a; i>=0 && indexable; i--, pOrTerm++){ if( (pOrTerm->eOperator & WO_SINGLE)==0 ){ WhereAndInfo *pAndInfo; - assert( pOrTerm->eOperator==0 ); assert( (pOrTerm->wtFlags & (TERM_ANDINFO|TERM_ORINFO))==0 ); chngToIN = 0; pAndInfo = sqlite3DbMallocRaw(db, sizeof(*pAndInfo)); @@ -948,7 +1050,7 @@ static void exprAnalyzeOrTerm( b |= getMask(pMaskSet, pOther->leftCursor); } indexable &= b; - if( pOrTerm->eOperator!=WO_EQ ){ + if( (pOrTerm->eOperator & WO_EQ)==0 ){ chngToIN = 0; }else{ chngToIN &= b; @@ -999,7 +1101,7 @@ static void exprAnalyzeOrTerm( for(j=0; j<2 && !okToChngToIN; j++){ pOrTerm = pOrWc->a; for(i=pOrWc->nTerm-1; i>=0; i--, pOrTerm++){ - assert( pOrTerm->eOperator==WO_EQ ); + assert( pOrTerm->eOperator & WO_EQ ); pOrTerm->wtFlags &= ~TERM_OR_OK; if( pOrTerm->leftCursor==iCursor ){ /* This is the 2-bit case and we are on the second iteration and @@ -1025,7 +1127,7 @@ static void exprAnalyzeOrTerm( /* No candidate table+column was found. This can only occur ** on the second iteration */ assert( j==1 ); - assert( (chngToIN&(chngToIN-1))==0 ); + assert( IsPowerOfTwo(chngToIN) ); assert( chngToIN==getMask(pMaskSet, iCursor) ); break; } @@ -1035,7 +1137,7 @@ static void exprAnalyzeOrTerm( ** table and column is common to every term in the OR clause */ okToChngToIN = 1; for(; i>=0 && okToChngToIN; i--, pOrTerm++){ - assert( pOrTerm->eOperator==WO_EQ ); + assert( pOrTerm->eOperator & WO_EQ ); if( pOrTerm->leftCursor!=iCursor ){ pOrTerm->wtFlags &= ~TERM_OR_OK; }else if( pOrTerm->u.leftColumn!=iColumn ){ @@ -1071,7 +1173,7 @@ static void exprAnalyzeOrTerm( for(i=pOrWc->nTerm-1, pOrTerm=pOrWc->a; i>=0; i--, pOrTerm++){ if( (pOrTerm->wtFlags & TERM_OR_OK)==0 ) continue; - assert( pOrTerm->eOperator==WO_EQ ); + assert( pOrTerm->eOperator & WO_EQ ); assert( pOrTerm->leftCursor==iCursor ); assert( pOrTerm->u.leftColumn==iColumn ); pDup = sqlite3ExprDup(db, pOrTerm->pExpr->pRight, 0); @@ -1101,7 +1203,6 @@ static void exprAnalyzeOrTerm( } #endif /* !SQLITE_OMIT_OR_OPTIMIZATION && !SQLITE_OMIT_SUBQUERY */ - /* ** The input to this routine is an WhereTerm structure with only the ** "pExpr" field filled in. The job of this routine is to analyze the @@ -1144,6 +1245,7 @@ static void exprAnalyze( pTerm = &pWC->a[idxTerm]; pMaskSet = pWC->pMaskSet; pExpr = pTerm->pExpr; + assert( pExpr->op!=TK_AS && pExpr->op!=TK_COLLATE ); prereqLeft = exprTableUsage(pMaskSet, pExpr->pLeft); op = pExpr->op; if( op==TK_IN ){ @@ -1169,17 +1271,19 @@ static void exprAnalyze( pTerm->leftCursor = -1; pTerm->iParent = -1; pTerm->eOperator = 0; - if( allowedOp(op) && (pTerm->prereqRight & prereqLeft)==0 ){ - Expr *pLeft = pExpr->pLeft; - Expr *pRight = pExpr->pRight; + if( allowedOp(op) ){ + Expr *pLeft = sqlite3ExprSkipCollate(pExpr->pLeft); + Expr *pRight = sqlite3ExprSkipCollate(pExpr->pRight); + u16 opMask = (pTerm->prereqRight & prereqLeft)==0 ? WO_ALL : WO_EQUIV; if( pLeft->op==TK_COLUMN ){ pTerm->leftCursor = pLeft->iTable; pTerm->u.leftColumn = pLeft->iColumn; - pTerm->eOperator = operatorMask(op); + pTerm->eOperator = operatorMask(op) & opMask; } if( pRight && pRight->op==TK_COLUMN ){ WhereTerm *pNew; Expr *pDup; + u16 eExtraOp = 0; /* Extra bits for pNew->eOperator */ if( pTerm->leftCursor>=0 ){ int idxNew; pDup = sqlite3ExprDup(db, pExpr, 0); @@ -1194,18 +1298,25 @@ static void exprAnalyze( pTerm = &pWC->a[idxTerm]; pTerm->nChild = 1; pTerm->wtFlags |= TERM_COPIED; + if( pExpr->op==TK_EQ + && !ExprHasProperty(pExpr, EP_FromJoin) + && OptimizationEnabled(db, SQLITE_Transitive) + ){ + pTerm->eOperator |= WO_EQUIV; + eExtraOp = WO_EQUIV; + } }else{ pDup = pExpr; pNew = pTerm; } exprCommute(pParse, pDup); - pLeft = pDup->pLeft; + pLeft = sqlite3ExprSkipCollate(pDup->pLeft); pNew->leftCursor = pLeft->iTable; pNew->u.leftColumn = pLeft->iColumn; testcase( (prereqLeft | extraRight) != prereqLeft ); pNew->prereqRight = prereqLeft | extraRight; pNew->prereqAll = prereqAll; - pNew->eOperator = operatorMask(pDup->op); + pNew->eOperator = (operatorMask(pDup->op) + eExtraOp) & opMask; } } @@ -1278,7 +1389,7 @@ static void exprAnalyze( Expr *pNewExpr2; int idxNew1; int idxNew2; - CollSeq *pColl; /* Collating sequence to use */ + Token sCollSeqName; /* Name of collating sequence */ pLeft = pExpr->x.pList->a[1].pExpr; pStr2 = sqlite3ExprDup(db, pStr1, 0); @@ -1300,16 +1411,19 @@ static void exprAnalyze( } *pC = c + 1; } - pColl = sqlite3FindCollSeq(db, SQLITE_UTF8, noCase ? "NOCASE" : "BINARY",0); + sCollSeqName.z = noCase ? "NOCASE" : "BINARY"; + sCollSeqName.n = 6; + pNewExpr1 = sqlite3ExprDup(db, pLeft, 0); pNewExpr1 = sqlite3PExpr(pParse, TK_GE, - sqlite3ExprSetColl(sqlite3ExprDup(db,pLeft,0), pColl), - pStr1, 0); + sqlite3ExprAddCollateToken(pParse,pNewExpr1,&sCollSeqName), + pStr1, 0); idxNew1 = whereClauseInsert(pWC, pNewExpr1, TERM_VIRTUAL|TERM_DYNAMIC); testcase( idxNew1==0 ); exprAnalyze(pSrc, pWC, idxNew1); + pNewExpr2 = sqlite3ExprDup(db, pLeft, 0); pNewExpr2 = sqlite3PExpr(pParse, TK_LT, - sqlite3ExprSetColl(sqlite3ExprDup(db,pLeft,0), pColl), - pStr2, 0); + sqlite3ExprAddCollateToken(pParse,pNewExpr2,&sCollSeqName), + pStr2, 0); idxNew2 = whereClauseInsert(pWC, pNewExpr2, TERM_VIRTUAL|TERM_DYNAMIC); testcase( idxNew2==0 ); exprAnalyze(pSrc, pWC, idxNew2); @@ -1407,25 +1521,6 @@ static void exprAnalyze( } /* -** Return TRUE if any of the expressions in pList->a[iFirst...] contain -** a reference to any table other than the iBase table. -*/ -static int referencesOtherTables( - ExprList *pList, /* Search expressions in ths list */ - WhereMaskSet *pMaskSet, /* Mapping from tables to bitmaps */ - int iFirst, /* Be searching with the iFirst-th expression */ - int iBase /* Ignore references to this table */ -){ - Bitmask allowed = ~getMask(pMaskSet, iBase); - while( iFirst<pList->nExpr ){ - if( (exprTableUsage(pMaskSet, pList->a[iFirst++].pExpr)&allowed)!=0 ){ - return 1; - } - } - return 0; -} - -/* ** This function searches the expression list passed as the second argument ** for an expression of type TK_COLUMN that refers to the same column and ** uses the same collation sequence as the iCol'th column of index pIdx. @@ -1446,12 +1541,12 @@ static int findIndexCol( const char *zColl = pIdx->azColl[iCol]; for(i=0; i<pList->nExpr; i++){ - Expr *p = pList->a[i].pExpr; + Expr *p = sqlite3ExprSkipCollate(pList->a[i].pExpr); if( p->op==TK_COLUMN && p->iColumn==pIdx->aiColumn[iCol] && p->iTable==iBase ){ - CollSeq *pColl = sqlite3ExprCollSeq(pParse, p); + CollSeq *pColl = sqlite3ExprCollSeq(pParse, pList->a[i].pExpr); if( ALWAYS(pColl) && 0==sqlite3StrICmp(pColl->zName, zColl) ){ return i; } @@ -1484,7 +1579,8 @@ static int isDistinctIndex( Bitmask mask = 0; /* Mask of unaccounted for pDistinct exprs */ int i; /* Iterator variable */ - if( pIdx->zName==0 || pDistinct==0 || pDistinct->nExpr>=BMS ) return 0; + assert( pDistinct!=0 ); + if( pIdx->zName==0 || pDistinct->nExpr>=BMS ) return 0; testcase( pDistinct->nExpr==BMS-1 ); /* Loop through all the expressions in the distinct list. If any of them @@ -1497,7 +1593,7 @@ static int isDistinctIndex( */ for(i=0; i<pDistinct->nExpr; i++){ WhereTerm *pTerm; - Expr *p = pDistinct->a[i].pExpr; + Expr *p = sqlite3ExprSkipCollate(pDistinct->a[i].pExpr); if( p->op!=TK_COLUMN ) return 0; pTerm = findTerm(pWC, p->iTable, p->iColumn, ~(Bitmask)0, WO_EQ, 0); if( pTerm ){ @@ -1549,7 +1645,7 @@ static int isDistinctRedundant( ** current SELECT is a correlated sub-query. */ for(i=0; i<pDistinct->nExpr; i++){ - Expr *p = pDistinct->a[i].pExpr; + Expr *p = sqlite3ExprSkipCollate(pDistinct->a[i].pExpr); if( p->op==TK_COLUMN && p->iTable==iBase && p->iColumn<0 ) return 1; } @@ -1587,165 +1683,6 @@ static int isDistinctRedundant( } /* -** This routine decides if pIdx can be used to satisfy the ORDER BY -** clause. If it can, it returns 1. If pIdx cannot satisfy the -** ORDER BY clause, this routine returns 0. -** -** pOrderBy is an ORDER BY clause from a SELECT statement. pTab is the -** left-most table in the FROM clause of that same SELECT statement and -** the table has a cursor number of "base". pIdx is an index on pTab. -** -** nEqCol is the number of columns of pIdx that are used as equality -** constraints. Any of these columns may be missing from the ORDER BY -** clause and the match can still be a success. -** -** All terms of the ORDER BY that match against the index must be either -** ASC or DESC. (Terms of the ORDER BY clause past the end of a UNIQUE -** index do not need to satisfy this constraint.) The *pbRev value is -** set to 1 if the ORDER BY clause is all DESC and it is set to 0 if -** the ORDER BY clause is all ASC. -*/ -static int isSortingIndex( - Parse *pParse, /* Parsing context */ - WhereMaskSet *pMaskSet, /* Mapping from table cursor numbers to bitmaps */ - Index *pIdx, /* The index we are testing */ - int base, /* Cursor number for the table to be sorted */ - ExprList *pOrderBy, /* The ORDER BY clause */ - int nEqCol, /* Number of index columns with == constraints */ - int wsFlags, /* Index usages flags */ - int *pbRev /* Set to 1 if ORDER BY is DESC */ -){ - int i, j; /* Loop counters */ - int sortOrder = 0; /* XOR of index and ORDER BY sort direction */ - int nTerm; /* Number of ORDER BY terms */ - struct ExprList_item *pTerm; /* A term of the ORDER BY clause */ - sqlite3 *db = pParse->db; - - if( !pOrderBy ) return 0; - if( wsFlags & WHERE_COLUMN_IN ) return 0; - if( pIdx->bUnordered ) return 0; - - nTerm = pOrderBy->nExpr; - assert( nTerm>0 ); - - /* Argument pIdx must either point to a 'real' named index structure, - ** or an index structure allocated on the stack by bestBtreeIndex() to - ** represent the rowid index that is part of every table. */ - assert( pIdx->zName || (pIdx->nColumn==1 && pIdx->aiColumn[0]==-1) ); - - /* Match terms of the ORDER BY clause against columns of - ** the index. - ** - ** Note that indices have pIdx->nColumn regular columns plus - ** one additional column containing the rowid. The rowid column - ** of the index is also allowed to match against the ORDER BY - ** clause. - */ - for(i=j=0, pTerm=pOrderBy->a; j<nTerm && i<=pIdx->nColumn; i++){ - Expr *pExpr; /* The expression of the ORDER BY pTerm */ - CollSeq *pColl; /* The collating sequence of pExpr */ - int termSortOrder; /* Sort order for this term */ - int iColumn; /* The i-th column of the index. -1 for rowid */ - int iSortOrder; /* 1 for DESC, 0 for ASC on the i-th index term */ - const char *zColl; /* Name of the collating sequence for i-th index term */ - - pExpr = pTerm->pExpr; - if( pExpr->op!=TK_COLUMN || pExpr->iTable!=base ){ - /* Can not use an index sort on anything that is not a column in the - ** left-most table of the FROM clause */ - break; - } - pColl = sqlite3ExprCollSeq(pParse, pExpr); - if( !pColl ){ - pColl = db->pDfltColl; - } - if( pIdx->zName && i<pIdx->nColumn ){ - iColumn = pIdx->aiColumn[i]; - if( iColumn==pIdx->pTable->iPKey ){ - iColumn = -1; - } - iSortOrder = pIdx->aSortOrder[i]; - zColl = pIdx->azColl[i]; - }else{ - iColumn = -1; - iSortOrder = 0; - zColl = pColl->zName; - } - if( pExpr->iColumn!=iColumn || sqlite3StrICmp(pColl->zName, zColl) ){ - /* Term j of the ORDER BY clause does not match column i of the index */ - if( i<nEqCol ){ - /* If an index column that is constrained by == fails to match an - ** ORDER BY term, that is OK. Just ignore that column of the index - */ - continue; - }else if( i==pIdx->nColumn ){ - /* Index column i is the rowid. All other terms match. */ - break; - }else{ - /* If an index column fails to match and is not constrained by == - ** then the index cannot satisfy the ORDER BY constraint. - */ - return 0; - } - } - assert( pIdx->aSortOrder!=0 || iColumn==-1 ); - assert( pTerm->sortOrder==0 || pTerm->sortOrder==1 ); - assert( iSortOrder==0 || iSortOrder==1 ); - termSortOrder = iSortOrder ^ pTerm->sortOrder; - if( i>nEqCol ){ - if( termSortOrder!=sortOrder ){ - /* Indices can only be used if all ORDER BY terms past the - ** equality constraints are all either DESC or ASC. */ - return 0; - } - }else{ - sortOrder = termSortOrder; - } - j++; - pTerm++; - if( iColumn<0 && !referencesOtherTables(pOrderBy, pMaskSet, j, base) ){ - /* If the indexed column is the primary key and everything matches - ** so far and none of the ORDER BY terms to the right reference other - ** tables in the join, then we are assured that the index can be used - ** to sort because the primary key is unique and so none of the other - ** columns will make any difference - */ - j = nTerm; - } - } - - *pbRev = sortOrder!=0; - if( j>=nTerm ){ - /* All terms of the ORDER BY clause are covered by this index so - ** this index can be used for sorting. */ - return 1; - } - if( pIdx->onError!=OE_None && i==pIdx->nColumn - && (wsFlags & WHERE_COLUMN_NULL)==0 - && !referencesOtherTables(pOrderBy, pMaskSet, j, base) - ){ - Column *aCol = pIdx->pTable->aCol; - - /* All terms of this index match some prefix of the ORDER BY clause, - ** the index is UNIQUE, and no terms on the tail of the ORDER BY - ** refer to other tables in a join. So, assuming that the index entries - ** visited contain no NULL values, then this index delivers rows in - ** the required order. - ** - ** It is not possible for any of the first nEqCol index fields to be - ** NULL (since the corresponding "=" operator in the WHERE clause would - ** not be true). So if all remaining index columns have NOT NULL - ** constaints attached to them, we can be confident that the visited - ** index entries are free of NULLs. */ - for(i=nEqCol; i<pIdx->nColumn; i++){ - if( aCol[pIdx->aiColumn[i]].notNull==0 ) break; - } - return (i==pIdx->nColumn); - } - return 0; -} - -/* ** Prepare a crude estimate of the logarithm of the input value. ** The results need not be exact. This is only used for estimating ** the total cost of performing operations with O(logN) or O(NlogN) @@ -1809,9 +1746,7 @@ static void TRACE_IDX_OUTPUTS(sqlite3_index_info *p){ /* ** Required because bestIndex() is called by bestOrClauseIndex() */ -static void bestIndex( - Parse*, WhereClause*, struct SrcList_item*, - Bitmask, Bitmask, ExprList*, WhereCost*); +static void bestIndex(WhereBestIdx*); /* ** This routine attempts to find an scanning strategy that can be used @@ -1820,20 +1755,14 @@ static void bestIndex( ** The table associated with FROM clause term pSrc may be either a ** regular B-Tree table or a virtual table. */ -static void bestOrClauseIndex( - Parse *pParse, /* The parsing context */ - WhereClause *pWC, /* The WHERE clause */ - struct SrcList_item *pSrc, /* The FROM clause term to search */ - Bitmask notReady, /* Mask of cursors not available for indexing */ - Bitmask notValid, /* Cursors not available for any purpose */ - ExprList *pOrderBy, /* The ORDER BY clause */ - WhereCost *pCost /* Lowest cost query plan */ -){ +static void bestOrClauseIndex(WhereBestIdx *p){ #ifndef SQLITE_OMIT_OR_OPTIMIZATION - const int iCur = pSrc->iCursor; /* The cursor of the table to be accessed */ + WhereClause *pWC = p->pWC; /* The WHERE clause */ + struct SrcList_item *pSrc = p->pSrc; /* The FROM clause term to search */ + const int iCur = pSrc->iCursor; /* The cursor of the table */ const Bitmask maskSrc = getMask(pWC->pMaskSet, iCur); /* Bitmask for pSrc */ WhereTerm * const pWCEnd = &pWC->a[pWC->nTerm]; /* End of pWC->a[] */ - WhereTerm *pTerm; /* A single term of the WHERE clause */ + WhereTerm *pTerm; /* A single term of the WHERE clause */ /* The OR-clause optimization is disallowed if the INDEXED BY or ** NOT INDEXED clauses are used or if the WHERE_AND_ONLY bit is set. */ @@ -1846,8 +1775,8 @@ static void bestOrClauseIndex( /* Search the WHERE clause terms for a usable WO_OR term. */ for(pTerm=pWC->a; pTerm<pWCEnd; pTerm++){ - if( pTerm->eOperator==WO_OR - && ((pTerm->prereqAll & ~maskSrc) & notReady)==0 + if( (pTerm->eOperator & WO_OR)!=0 + && ((pTerm->prereqAll & ~maskSrc) & p->notReady)==0 && (pTerm->u.pOrInfo->indexable & maskSrc)!=0 ){ WhereClause * const pOrWC = &pTerm->u.pOrInfo->wc; @@ -1857,15 +1786,19 @@ static void bestOrClauseIndex( double rTotal = 0; double nRow = 0; Bitmask used = 0; + WhereBestIdx sBOI; + sBOI = *p; + sBOI.pOrderBy = 0; + sBOI.pDistinct = 0; + sBOI.ppIdxInfo = 0; for(pOrTerm=pOrWC->a; pOrTerm<pOrWCEnd; pOrTerm++){ - WhereCost sTermCost; WHERETRACE(("... Multi-index OR testing for term %d of %d....\n", (pOrTerm - pOrWC->a), (pTerm - pWC->a) )); - if( pOrTerm->eOperator==WO_AND ){ - WhereClause *pAndWC = &pOrTerm->u.pAndInfo->wc; - bestIndex(pParse, pAndWC, pSrc, notReady, notValid, 0, &sTermCost); + if( (pOrTerm->eOperator& WO_AND)!=0 ){ + sBOI.pWC = &pOrTerm->u.pAndInfo->wc; + bestIndex(&sBOI); }else if( pOrTerm->leftCursor==iCur ){ WhereClause tempWC; tempWC.pParse = pWC->pParse; @@ -1875,19 +1808,20 @@ static void bestOrClauseIndex( tempWC.a = pOrTerm; tempWC.wctrlFlags = 0; tempWC.nTerm = 1; - bestIndex(pParse, &tempWC, pSrc, notReady, notValid, 0, &sTermCost); + sBOI.pWC = &tempWC; + bestIndex(&sBOI); }else{ continue; } - rTotal += sTermCost.rCost; - nRow += sTermCost.plan.nRow; - used |= sTermCost.used; - if( rTotal>=pCost->rCost ) break; + rTotal += sBOI.cost.rCost; + nRow += sBOI.cost.plan.nRow; + used |= sBOI.cost.used; + if( rTotal>=p->cost.rCost ) break; } /* If there is an ORDER BY clause, increase the scan cost to account ** for the cost of the sort. */ - if( pOrderBy!=0 ){ + if( p->pOrderBy!=0 ){ WHERETRACE(("... sorting increases OR cost %.9g to %.9g\n", rTotal, rTotal+nRow*estLog(nRow))); rTotal += nRow*estLog(nRow); @@ -1897,12 +1831,13 @@ static void bestOrClauseIndex( ** less than the current cost stored in pCost, replace the contents ** of pCost. */ WHERETRACE(("... multi-index OR cost=%.9g nrow=%.9g\n", rTotal, nRow)); - if( rTotal<pCost->rCost ){ - pCost->rCost = rTotal; - pCost->used = used; - pCost->plan.nRow = nRow; - pCost->plan.wsFlags = flags; - pCost->plan.u.pTerm = pTerm; + if( rTotal<p->cost.rCost ){ + p->cost.rCost = rTotal; + p->cost.used = used; + p->cost.plan.nRow = nRow; + p->cost.plan.nOBSat = p->i ? p->aLevel[p->i-1].plan.nOBSat : 0; + p->cost.plan.wsFlags = flags; + p->cost.plan.u.pTerm = pTerm; } } } @@ -1922,7 +1857,7 @@ static int termCanDriveIndex( ){ char aff; if( pTerm->leftCursor!=pSrc->iCursor ) return 0; - if( pTerm->eOperator!=WO_EQ ) return 0; + if( (pTerm->eOperator & WO_EQ)==0 ) return 0; if( (pTerm->prereqRight & notReady)!=0 ) return 0; aff = pSrc->pTab->aCol[pTerm->u.leftColumn].affinity; if( !sqlite3IndexAffinityOk(pTerm->pExpr, aff) ) return 0; @@ -1939,15 +1874,12 @@ static int termCanDriveIndex( ** is taken into account, then alter the query plan to use the ** transient index. */ -static void bestAutomaticIndex( - Parse *pParse, /* The parsing context */ - WhereClause *pWC, /* The WHERE clause */ - struct SrcList_item *pSrc, /* The FROM clause term to search */ - Bitmask notReady, /* Mask of cursors that are not available */ - WhereCost *pCost /* Lowest cost query plan */ -){ - double nTableRow; /* Rows in the input table */ - double logN; /* log(nTableRow) */ +static void bestAutomaticIndex(WhereBestIdx *p){ + Parse *pParse = p->pParse; /* The parsing context */ + WhereClause *pWC = p->pWC; /* The WHERE clause */ + struct SrcList_item *pSrc = p->pSrc; /* The FROM clause term to search */ + double nTableRow; /* Rows in the input table */ + double logN; /* log(nTableRow) */ double costTempIdx; /* per-query cost of the transient index */ WhereTerm *pTerm; /* A single term of the WHERE clause */ WhereTerm *pWCEnd; /* End of pWC->a[] */ @@ -1961,10 +1893,16 @@ static void bestAutomaticIndex( /* Automatic indices are disabled at run-time */ return; } - if( (pCost->plan.wsFlags & WHERE_NOT_FULLSCAN)!=0 ){ + if( (p->cost.plan.wsFlags & WHERE_NOT_FULLSCAN)!=0 + && (p->cost.plan.wsFlags & WHERE_COVER_SCAN)==0 + ){ /* We already have some kind of index in use for this query. */ return; } + if( pSrc->viaCoroutine ){ + /* Cannot index a co-routine */ + return; + } if( pSrc->notIndexed ){ /* The NOT INDEXED clause appears in the SQL. */ return; @@ -1979,7 +1917,7 @@ static void bestAutomaticIndex( nTableRow = pTable->nRowEst; logN = estLog(nTableRow); costTempIdx = 2*logN*(nTableRow/pParse->nQueryLoop + 1); - if( costTempIdx>=pCost->rCost ){ + if( costTempIdx>=p->cost.rCost ){ /* The cost of creating the transient table would be greater than ** doing the full table scan */ return; @@ -1988,19 +1926,19 @@ static void bestAutomaticIndex( /* Search for any equality comparison term */ pWCEnd = &pWC->a[pWC->nTerm]; for(pTerm=pWC->a; pTerm<pWCEnd; pTerm++){ - if( termCanDriveIndex(pTerm, pSrc, notReady) ){ + if( termCanDriveIndex(pTerm, pSrc, p->notReady) ){ WHERETRACE(("auto-index reduces cost from %.1f to %.1f\n", - pCost->rCost, costTempIdx)); - pCost->rCost = costTempIdx; - pCost->plan.nRow = logN + 1; - pCost->plan.wsFlags = WHERE_TEMP_INDEX; - pCost->used = pTerm->prereqRight; + p->cost.rCost, costTempIdx)); + p->cost.rCost = costTempIdx; + p->cost.plan.nRow = logN + 1; + p->cost.plan.wsFlags = WHERE_TEMP_INDEX; + p->cost.used = pTerm->prereqRight; break; } } } #else -# define bestAutomaticIndex(A,B,C,D,E) /* no-op */ +# define bestAutomaticIndex(A) /* no-op */ #endif /* SQLITE_OMIT_AUTOMATIC_INDEX */ @@ -2161,12 +2099,11 @@ static void constructAutomaticIndex( ** responsibility of the caller to eventually release the structure ** by passing the pointer returned by this function to sqlite3_free(). */ -static sqlite3_index_info *allocateIndexInfo( - Parse *pParse, - WhereClause *pWC, - struct SrcList_item *pSrc, - ExprList *pOrderBy -){ +static sqlite3_index_info *allocateIndexInfo(WhereBestIdx *p){ + Parse *pParse = p->pParse; + WhereClause *pWC = p->pWC; + struct SrcList_item *pSrc = p->pSrc; + ExprList *pOrderBy = p->pOrderBy; int i, j; int nTerm; struct sqlite3_index_constraint *pIdxCons; @@ -2182,10 +2119,10 @@ static sqlite3_index_info *allocateIndexInfo( ** to this virtual table */ for(i=nTerm=0, pTerm=pWC->a; i<pWC->nTerm; i++, pTerm++){ if( pTerm->leftCursor != pSrc->iCursor ) continue; - assert( (pTerm->eOperator&(pTerm->eOperator-1))==0 ); - testcase( pTerm->eOperator==WO_IN ); - testcase( pTerm->eOperator==WO_ISNULL ); - if( pTerm->eOperator & (WO_IN|WO_ISNULL) ) continue; + assert( IsPowerOfTwo(pTerm->eOperator & ~WO_EQUIV) ); + testcase( pTerm->eOperator & WO_IN ); + testcase( pTerm->eOperator & WO_ISNULL ); + if( pTerm->eOperator & (WO_ISNULL) ) continue; if( pTerm->wtFlags & TERM_VNULL ) continue; nTerm++; } @@ -2196,12 +2133,13 @@ static sqlite3_index_info *allocateIndexInfo( */ nOrderBy = 0; if( pOrderBy ){ - for(i=0; i<pOrderBy->nExpr; i++){ + int n = pOrderBy->nExpr; + for(i=0; i<n; i++){ Expr *pExpr = pOrderBy->a[i].pExpr; if( pExpr->op!=TK_COLUMN || pExpr->iTable!=pSrc->iCursor ) break; } - if( i==pOrderBy->nExpr ){ - nOrderBy = pOrderBy->nExpr; + if( i==n){ + nOrderBy = n; } } @@ -2232,15 +2170,18 @@ static sqlite3_index_info *allocateIndexInfo( pUsage; for(i=j=0, pTerm=pWC->a; i<pWC->nTerm; i++, pTerm++){ + u8 op; if( pTerm->leftCursor != pSrc->iCursor ) continue; - assert( (pTerm->eOperator&(pTerm->eOperator-1))==0 ); - testcase( pTerm->eOperator==WO_IN ); - testcase( pTerm->eOperator==WO_ISNULL ); - if( pTerm->eOperator & (WO_IN|WO_ISNULL) ) continue; + assert( IsPowerOfTwo(pTerm->eOperator & ~WO_EQUIV) ); + testcase( pTerm->eOperator & WO_IN ); + testcase( pTerm->eOperator & WO_ISNULL ); + if( pTerm->eOperator & (WO_ISNULL) ) continue; if( pTerm->wtFlags & TERM_VNULL ) continue; pIdxCons[j].iColumn = pTerm->u.leftColumn; pIdxCons[j].iTermOffset = i; - pIdxCons[j].op = (u8)pTerm->eOperator; + op = (u8)pTerm->eOperator & WO_ALL; + if( op==WO_IN ) op = WO_EQ; + pIdxCons[j].op = op; /* The direct assignment in the previous line is possible only because ** the WO_ and SQLITE_INDEX_CONSTRAINT_ codes are identical. The ** following asserts verify this fact. */ @@ -2250,7 +2191,7 @@ static sqlite3_index_info *allocateIndexInfo( assert( WO_GT==SQLITE_INDEX_CONSTRAINT_GT ); assert( WO_GE==SQLITE_INDEX_CONSTRAINT_GE ); assert( WO_MATCH==SQLITE_INDEX_CONSTRAINT_MATCH ); - assert( pTerm->eOperator & (WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE|WO_MATCH) ); + assert( pTerm->eOperator & (WO_IN|WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE|WO_MATCH) ); j++; } for(i=0; i<nOrderBy; i++){ @@ -2325,16 +2266,10 @@ static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){ ** routine takes care of freeing the sqlite3_index_info structure after ** everybody has finished with it. */ -static void bestVirtualIndex( - Parse *pParse, /* The parsing context */ - WhereClause *pWC, /* The WHERE clause */ - struct SrcList_item *pSrc, /* The FROM clause term to search */ - Bitmask notReady, /* Mask of cursors not available for index */ - Bitmask notValid, /* Cursors not valid for any purpose */ - ExprList *pOrderBy, /* The order by clause */ - WhereCost *pCost, /* Lowest cost query plan */ - sqlite3_index_info **ppIdxInfo /* Index information passed to xBestIndex */ -){ +static void bestVirtualIndex(WhereBestIdx *p){ + Parse *pParse = p->pParse; /* The parsing context */ + WhereClause *pWC = p->pWC; /* The WHERE clause */ + struct SrcList_item *pSrc = p->pSrc; /* The FROM clause term to search */ Table *pTab = pSrc->pTab; sqlite3_index_info *pIdxInfo; struct sqlite3_index_constraint *pIdxCons; @@ -2342,21 +2277,22 @@ static void bestVirtualIndex( WhereTerm *pTerm; int i, j; int nOrderBy; + int bAllowIN; /* Allow IN optimizations */ double rCost; /* Make sure wsFlags is initialized to some sane value. Otherwise, if the ** malloc in allocateIndexInfo() fails and this function returns leaving ** wsFlags in an uninitialized state, the caller may behave unpredictably. */ - memset(pCost, 0, sizeof(*pCost)); - pCost->plan.wsFlags = WHERE_VIRTUALTABLE; + memset(&p->cost, 0, sizeof(p->cost)); + p->cost.plan.wsFlags = WHERE_VIRTUALTABLE; /* If the sqlite3_index_info structure has not been previously ** allocated and initialized, then allocate and initialize it now. */ - pIdxInfo = *ppIdxInfo; + pIdxInfo = *p->ppIdxInfo; if( pIdxInfo==0 ){ - *ppIdxInfo = pIdxInfo = allocateIndexInfo(pParse, pWC, pSrc, pOrderBy); + *p->ppIdxInfo = pIdxInfo = allocateIndexInfo(p); } if( pIdxInfo==0 ){ return; @@ -2376,65 +2312,112 @@ static void bestVirtualIndex( assert( pTab->azModuleArg && pTab->azModuleArg[0] ); assert( sqlite3GetVTable(pParse->db, pTab) ); - /* Set the aConstraint[].usable fields and initialize all - ** output variables to zero. - ** - ** aConstraint[].usable is true for constraints where the right-hand - ** side contains only references to tables to the left of the current - ** table. In other words, if the constraint is of the form: - ** - ** column = expr - ** - ** and we are evaluating a join, then the constraint on column is - ** only valid if all tables referenced in expr occur to the left - ** of the table containing column. - ** - ** The aConstraints[] array contains entries for all constraints - ** on the current table. That way we only have to compute it once - ** even though we might try to pick the best index multiple times. - ** For each attempt at picking an index, the order of tables in the - ** join might be different so we have to recompute the usable flag - ** each time. + /* Try once or twice. On the first attempt, allow IN optimizations. + ** If an IN optimization is accepted by the virtual table xBestIndex + ** method, but the pInfo->aConstrainUsage.omit flag is not set, then + ** the query will not work because it might allow duplicate rows in + ** output. In that case, run the xBestIndex method a second time + ** without the IN constraints. Usually this loop only runs once. + ** The loop will exit using a "break" statement. */ - pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint; - pUsage = pIdxInfo->aConstraintUsage; - for(i=0; i<pIdxInfo->nConstraint; i++, pIdxCons++){ - j = pIdxCons->iTermOffset; - pTerm = &pWC->a[j]; - pIdxCons->usable = (pTerm->prereqRight¬Ready) ? 0 : 1; - } - memset(pUsage, 0, sizeof(pUsage[0])*pIdxInfo->nConstraint); - if( pIdxInfo->needToFreeIdxStr ){ - sqlite3_free(pIdxInfo->idxStr); - } - pIdxInfo->idxStr = 0; - pIdxInfo->idxNum = 0; - pIdxInfo->needToFreeIdxStr = 0; - pIdxInfo->orderByConsumed = 0; - /* ((double)2) In case of SQLITE_OMIT_FLOATING_POINT... */ - pIdxInfo->estimatedCost = SQLITE_BIG_DBL / ((double)2); - nOrderBy = pIdxInfo->nOrderBy; - if( !pOrderBy ){ - pIdxInfo->nOrderBy = 0; - } - - if( vtabBestIndex(pParse, pTab, pIdxInfo) ){ - return; - } + for(bAllowIN=1; 1; bAllowIN--){ + assert( bAllowIN==0 || bAllowIN==1 ); - pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint; - for(i=0; i<pIdxInfo->nConstraint; i++){ - if( pUsage[i].argvIndex>0 ){ - pCost->used |= pWC->a[pIdxCons[i].iTermOffset].prereqRight; + /* Set the aConstraint[].usable fields and initialize all + ** output variables to zero. + ** + ** aConstraint[].usable is true for constraints where the right-hand + ** side contains only references to tables to the left of the current + ** table. In other words, if the constraint is of the form: + ** + ** column = expr + ** + ** and we are evaluating a join, then the constraint on column is + ** only valid if all tables referenced in expr occur to the left + ** of the table containing column. + ** + ** The aConstraints[] array contains entries for all constraints + ** on the current table. That way we only have to compute it once + ** even though we might try to pick the best index multiple times. + ** For each attempt at picking an index, the order of tables in the + ** join might be different so we have to recompute the usable flag + ** each time. + */ + pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint; + pUsage = pIdxInfo->aConstraintUsage; + for(i=0; i<pIdxInfo->nConstraint; i++, pIdxCons++){ + j = pIdxCons->iTermOffset; + pTerm = &pWC->a[j]; + if( (pTerm->prereqRight&p->notReady)==0 + && (bAllowIN || (pTerm->eOperator & WO_IN)==0) + ){ + pIdxCons->usable = 1; + }else{ + pIdxCons->usable = 0; + } + } + memset(pUsage, 0, sizeof(pUsage[0])*pIdxInfo->nConstraint); + if( pIdxInfo->needToFreeIdxStr ){ + sqlite3_free(pIdxInfo->idxStr); + } + pIdxInfo->idxStr = 0; + pIdxInfo->idxNum = 0; + pIdxInfo->needToFreeIdxStr = 0; + pIdxInfo->orderByConsumed = 0; + /* ((double)2) In case of SQLITE_OMIT_FLOATING_POINT... */ + pIdxInfo->estimatedCost = SQLITE_BIG_DBL / ((double)2); + nOrderBy = pIdxInfo->nOrderBy; + if( !p->pOrderBy ){ + pIdxInfo->nOrderBy = 0; } + + if( vtabBestIndex(pParse, pTab, pIdxInfo) ){ + return; + } + + pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint; + for(i=0; i<pIdxInfo->nConstraint; i++, pIdxCons++){ + if( pUsage[i].argvIndex>0 ){ + j = pIdxCons->iTermOffset; + pTerm = &pWC->a[j]; + p->cost.used |= pTerm->prereqRight; + if( (pTerm->eOperator & WO_IN)!=0 ){ + if( pUsage[i].omit==0 ){ + /* Do not attempt to use an IN constraint if the virtual table + ** says that the equivalent EQ constraint cannot be safely omitted. + ** If we do attempt to use such a constraint, some rows might be + ** repeated in the output. */ + break; + } + /* A virtual table that is constrained by an IN clause may not + ** consume the ORDER BY clause because (1) the order of IN terms + ** is not necessarily related to the order of output terms and + ** (2) Multiple outputs from a single IN value will not merge + ** together. */ + pIdxInfo->orderByConsumed = 0; + } + } + } + if( i>=pIdxInfo->nConstraint ) break; } + /* The orderByConsumed signal is only valid if all outer loops collectively + ** generate just a single row of output. + */ + if( pIdxInfo->orderByConsumed ){ + for(i=0; i<p->i; i++){ + if( (p->aLevel[i].plan.wsFlags & WHERE_UNIQUE)==0 ){ + pIdxInfo->orderByConsumed = 0; + } + } + } + /* If there is an ORDER BY clause, and the selected virtual table index ** does not satisfy it, increase the cost of the scan accordingly. This ** matches the processing for non-virtual tables in bestBtreeIndex(). */ rCost = pIdxInfo->estimatedCost; - if( pOrderBy && pIdxInfo->orderByConsumed==0 ){ + if( p->pOrderBy && pIdxInfo->orderByConsumed==0 ){ rCost += estLog(rCost)*rCost; } @@ -2446,21 +2429,24 @@ static void bestVirtualIndex( ** is defined. */ if( (SQLITE_BIG_DBL/((double)2))<rCost ){ - pCost->rCost = (SQLITE_BIG_DBL/((double)2)); + p->cost.rCost = (SQLITE_BIG_DBL/((double)2)); }else{ - pCost->rCost = rCost; + p->cost.rCost = rCost; } - pCost->plan.u.pVtabIdx = pIdxInfo; + p->cost.plan.u.pVtabIdx = pIdxInfo; if( pIdxInfo->orderByConsumed ){ - pCost->plan.wsFlags |= WHERE_ORDERBY; + p->cost.plan.wsFlags |= WHERE_ORDERED; + p->cost.plan.nOBSat = nOrderBy; + }else{ + p->cost.plan.nOBSat = p->i ? p->aLevel[p->i-1].plan.nOBSat : 0; } - pCost->plan.nEq = 0; + p->cost.plan.nEq = 0; pIdxInfo->nOrderBy = nOrderBy; /* Try to find a more efficient access pattern by using multiple indexes ** to optimize an OR expression within the WHERE clause. */ - bestOrClauseIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, pCost); + bestOrClauseIndex(p); } #endif /* SQLITE_OMIT_VIRTUALTABLE */ @@ -2548,10 +2534,8 @@ static int whereKeyStats( pColl = db->pDfltColl; assert( pColl->enc==SQLITE_UTF8 ); }else{ - pColl = sqlite3GetCollSeq(db, SQLITE_UTF8, 0, *pIdx->azColl); + pColl = sqlite3GetCollSeq(pParse, SQLITE_UTF8, 0, *pIdx->azColl); if( pColl==0 ){ - sqlite3ErrorMsg(pParse, "no such collation sequence: %s", - *pIdx->azColl); return SQLITE_ERROR; } z = (const u8 *)sqlite3ValueText(pVal, pColl->enc); @@ -2722,24 +2706,24 @@ static int whereRangeScanEst( if( pLower ){ Expr *pExpr = pLower->pExpr->pRight; rc = valueFromExpr(pParse, pExpr, aff, &pRangeVal); - assert( pLower->eOperator==WO_GT || pLower->eOperator==WO_GE ); + assert( (pLower->eOperator & (WO_GT|WO_GE))!=0 ); if( rc==SQLITE_OK && whereKeyStats(pParse, p, pRangeVal, 0, a)==SQLITE_OK ){ iLower = a[0]; - if( pLower->eOperator==WO_GT ) iLower += a[1]; + if( (pLower->eOperator & WO_GT)!=0 ) iLower += a[1]; } sqlite3ValueFree(pRangeVal); } if( rc==SQLITE_OK && pUpper ){ Expr *pExpr = pUpper->pExpr->pRight; rc = valueFromExpr(pParse, pExpr, aff, &pRangeVal); - assert( pUpper->eOperator==WO_LT || pUpper->eOperator==WO_LE ); + assert( (pUpper->eOperator & (WO_LT|WO_LE))!=0 ); if( rc==SQLITE_OK && whereKeyStats(pParse, p, pRangeVal, 1, a)==SQLITE_OK ){ iUpper = a[0]; - if( pUpper->eOperator==WO_LE ) iUpper += a[1]; + if( (pUpper->eOperator & WO_LE)!=0 ) iUpper += a[1]; } sqlite3ValueFree(pRangeVal); } @@ -2859,11 +2843,295 @@ static int whereInScanEst( } #endif /* defined(SQLITE_ENABLE_STAT3) */ +/* +** Check to see if column iCol of the table with cursor iTab will appear +** in sorted order according to the current query plan. +** +** Return values: +** +** 0 iCol is not ordered +** 1 iCol has only a single value +** 2 iCol is in ASC order +** 3 iCol is in DESC order +*/ +static int isOrderedColumn( + WhereBestIdx *p, + int iTab, + int iCol +){ + int i, j; + WhereLevel *pLevel = &p->aLevel[p->i-1]; + Index *pIdx; + u8 sortOrder; + for(i=p->i-1; i>=0; i--, pLevel--){ + if( pLevel->iTabCur!=iTab ) continue; + if( (pLevel->plan.wsFlags & WHERE_ALL_UNIQUE)!=0 ){ + return 1; + } + assert( (pLevel->plan.wsFlags & WHERE_ORDERED)!=0 ); + if( (pIdx = pLevel->plan.u.pIdx)!=0 ){ + if( iCol<0 ){ + sortOrder = 0; + testcase( (pLevel->plan.wsFlags & WHERE_REVERSE)!=0 ); + }else{ + int n = pIdx->nColumn; + for(j=0; j<n; j++){ + if( iCol==pIdx->aiColumn[j] ) break; + } + if( j>=n ) return 0; + sortOrder = pIdx->aSortOrder[j]; + testcase( (pLevel->plan.wsFlags & WHERE_REVERSE)!=0 ); + } + }else{ + if( iCol!=(-1) ) return 0; + sortOrder = 0; + testcase( (pLevel->plan.wsFlags & WHERE_REVERSE)!=0 ); + } + if( (pLevel->plan.wsFlags & WHERE_REVERSE)!=0 ){ + assert( sortOrder==0 || sortOrder==1 ); + testcase( sortOrder==1 ); + sortOrder = 1 - sortOrder; + } + return sortOrder+2; + } + return 0; +} + +/* +** This routine decides if pIdx can be used to satisfy the ORDER BY +** clause, either in whole or in part. The return value is the +** cumulative number of terms in the ORDER BY clause that are satisfied +** by the index pIdx and other indices in outer loops. +** +** The table being queried has a cursor number of "base". pIdx is the +** index that is postulated for use to access the table. +** +** The *pbRev value is set to 0 order 1 depending on whether or not +** pIdx should be run in the forward order or in reverse order. +*/ +static int isSortingIndex( + WhereBestIdx *p, /* Best index search context */ + Index *pIdx, /* The index we are testing */ + int base, /* Cursor number for the table to be sorted */ + int *pbRev, /* Set to 1 for reverse-order scan of pIdx */ + int *pbObUnique /* ORDER BY column values will different in every row */ +){ + int i; /* Number of pIdx terms used */ + int j; /* Number of ORDER BY terms satisfied */ + int sortOrder = 2; /* 0: forward. 1: backward. 2: unknown */ + int nTerm; /* Number of ORDER BY terms */ + struct ExprList_item *pOBItem;/* A term of the ORDER BY clause */ + Table *pTab = pIdx->pTable; /* Table that owns index pIdx */ + ExprList *pOrderBy; /* The ORDER BY clause */ + Parse *pParse = p->pParse; /* Parser context */ + sqlite3 *db = pParse->db; /* Database connection */ + int nPriorSat; /* ORDER BY terms satisfied by outer loops */ + int seenRowid = 0; /* True if an ORDER BY rowid term is seen */ + int uniqueNotNull; /* pIdx is UNIQUE with all terms are NOT NULL */ + int outerObUnique; /* Outer loops generate different values in + ** every row for the ORDER BY columns */ + + if( p->i==0 ){ + nPriorSat = 0; + outerObUnique = 1; + }else{ + u32 wsFlags = p->aLevel[p->i-1].plan.wsFlags; + nPriorSat = p->aLevel[p->i-1].plan.nOBSat; + if( (wsFlags & WHERE_ORDERED)==0 ){ + /* This loop cannot be ordered unless the next outer loop is + ** also ordered */ + return nPriorSat; + } + if( OptimizationDisabled(db, SQLITE_OrderByIdxJoin) ){ + /* Only look at the outer-most loop if the OrderByIdxJoin + ** optimization is disabled */ + return nPriorSat; + } + testcase( wsFlags & WHERE_OB_UNIQUE ); + testcase( wsFlags & WHERE_ALL_UNIQUE ); + outerObUnique = (wsFlags & (WHERE_OB_UNIQUE|WHERE_ALL_UNIQUE))!=0; + } + pOrderBy = p->pOrderBy; + assert( pOrderBy!=0 ); + if( pIdx->bUnordered ){ + /* Hash indices (indicated by the "unordered" tag on sqlite_stat1) cannot + ** be used for sorting */ + return nPriorSat; + } + nTerm = pOrderBy->nExpr; + uniqueNotNull = pIdx->onError!=OE_None; + assert( nTerm>0 ); + + /* Argument pIdx must either point to a 'real' named index structure, + ** or an index structure allocated on the stack by bestBtreeIndex() to + ** represent the rowid index that is part of every table. */ + assert( pIdx->zName || (pIdx->nColumn==1 && pIdx->aiColumn[0]==-1) ); + + /* Match terms of the ORDER BY clause against columns of + ** the index. + ** + ** Note that indices have pIdx->nColumn regular columns plus + ** one additional column containing the rowid. The rowid column + ** of the index is also allowed to match against the ORDER BY + ** clause. + */ + j = nPriorSat; + for(i=0,pOBItem=&pOrderBy->a[j]; j<nTerm && i<=pIdx->nColumn; i++){ + Expr *pOBExpr; /* The expression of the ORDER BY pOBItem */ + CollSeq *pColl; /* The collating sequence of pOBExpr */ + int termSortOrder; /* Sort order for this term */ + int iColumn; /* The i-th column of the index. -1 for rowid */ + int iSortOrder; /* 1 for DESC, 0 for ASC on the i-th index term */ + int isEq; /* Subject to an == or IS NULL constraint */ + int isMatch; /* ORDER BY term matches the index term */ + const char *zColl; /* Name of collating sequence for i-th index term */ + WhereTerm *pConstraint; /* A constraint in the WHERE clause */ + + /* If the next term of the ORDER BY clause refers to anything other than + ** a column in the "base" table, then this index will not be of any + ** further use in handling the ORDER BY. */ + pOBExpr = sqlite3ExprSkipCollate(pOBItem->pExpr); + if( pOBExpr->op!=TK_COLUMN || pOBExpr->iTable!=base ){ + break; + } + + /* Find column number and collating sequence for the next entry + ** in the index */ + if( pIdx->zName && i<pIdx->nColumn ){ + iColumn = pIdx->aiColumn[i]; + if( iColumn==pIdx->pTable->iPKey ){ + iColumn = -1; + } + iSortOrder = pIdx->aSortOrder[i]; + zColl = pIdx->azColl[i]; + assert( zColl!=0 ); + }else{ + iColumn = -1; + iSortOrder = 0; + zColl = 0; + } + + /* Check to see if the column number and collating sequence of the + ** index match the column number and collating sequence of the ORDER BY + ** clause entry. Set isMatch to 1 if they both match. */ + if( pOBExpr->iColumn==iColumn ){ + if( zColl ){ + pColl = sqlite3ExprCollSeq(pParse, pOBItem->pExpr); + if( !pColl ) pColl = db->pDfltColl; + isMatch = sqlite3StrICmp(pColl->zName, zColl)==0; + }else{ + isMatch = 1; + } + }else{ + isMatch = 0; + } + + /* termSortOrder is 0 or 1 for whether or not the access loop should + ** run forward or backwards (respectively) in order to satisfy this + ** term of the ORDER BY clause. */ + assert( pOBItem->sortOrder==0 || pOBItem->sortOrder==1 ); + assert( iSortOrder==0 || iSortOrder==1 ); + termSortOrder = iSortOrder ^ pOBItem->sortOrder; + + /* If X is the column in the index and ORDER BY clause, check to see + ** if there are any X= or X IS NULL constraints in the WHERE clause. */ + pConstraint = findTerm(p->pWC, base, iColumn, p->notReady, + WO_EQ|WO_ISNULL|WO_IN, pIdx); + if( pConstraint==0 ){ + isEq = 0; + }else if( (pConstraint->eOperator & WO_IN)!=0 ){ + isEq = 0; + }else if( (pConstraint->eOperator & WO_ISNULL)!=0 ){ + uniqueNotNull = 0; + isEq = 1; /* "X IS NULL" means X has only a single value */ + }else if( pConstraint->prereqRight==0 ){ + isEq = 1; /* Constraint "X=constant" means X has only a single value */ + }else{ + Expr *pRight = pConstraint->pExpr->pRight; + if( pRight->op==TK_COLUMN ){ + WHERETRACE((" .. isOrderedColumn(tab=%d,col=%d)", + pRight->iTable, pRight->iColumn)); + isEq = isOrderedColumn(p, pRight->iTable, pRight->iColumn); + WHERETRACE((" -> isEq=%d\n", isEq)); + + /* If the constraint is of the form X=Y where Y is an ordered value + ** in an outer loop, then make sure the sort order of Y matches the + ** sort order required for X. */ + if( isMatch && isEq>=2 && isEq!=pOBItem->sortOrder+2 ){ + testcase( isEq==2 ); + testcase( isEq==3 ); + break; + } + }else{ + isEq = 0; /* "X=expr" places no ordering constraints on X */ + } + } + if( !isMatch ){ + if( isEq==0 ){ + break; + }else{ + continue; + } + }else if( isEq!=1 ){ + if( sortOrder==2 ){ + sortOrder = termSortOrder; + }else if( termSortOrder!=sortOrder ){ + break; + } + } + j++; + pOBItem++; + if( iColumn<0 ){ + seenRowid = 1; + break; + }else if( pTab->aCol[iColumn].notNull==0 && isEq!=1 ){ + testcase( isEq==0 ); + testcase( isEq==2 ); + testcase( isEq==3 ); + uniqueNotNull = 0; + } + } + if( seenRowid ){ + uniqueNotNull = 1; + }else if( uniqueNotNull==0 || i<pIdx->nColumn ){ + uniqueNotNull = 0; + } + + /* If we have not found at least one ORDER BY term that matches the + ** index, then show no progress. */ + if( pOBItem==&pOrderBy->a[nPriorSat] ) return nPriorSat; + + /* Either the outer queries must generate rows where there are no two + ** rows with the same values in all ORDER BY columns, or else this + ** loop must generate just a single row of output. Example: Suppose + ** the outer loops generate A=1 and A=1, and this loop generates B=3 + ** and B=4. Then without the following test, ORDER BY A,B would + ** generate the wrong order output: 1,3 1,4 1,3 1,4 + */ + if( outerObUnique==0 && uniqueNotNull==0 ) return nPriorSat; + *pbObUnique = uniqueNotNull; + + /* Return the necessary scan order back to the caller */ + *pbRev = sortOrder & 1; + + /* If there was an "ORDER BY rowid" term that matched, or it is only + ** possible for a single row from this table to match, then skip over + ** any additional ORDER BY terms dealing with this table. + */ + if( uniqueNotNull ){ + /* Advance j over additional ORDER BY terms associated with base */ + WhereMaskSet *pMS = p->pWC->pMaskSet; + Bitmask m = ~getMask(pMS, base); + while( j<nTerm && (exprTableUsage(pMS, pOrderBy->a[j].pExpr)&m)==0 ){ + j++; + } + } + return j; +} /* ** Find the best query plan for accessing a particular table. Write the -** best query plan and its cost into the WhereCost object supplied as the -** last parameter. +** best query plan and its cost into the p->cost. ** ** The lowest cost plan wins. The cost is an estimate of the amount of ** CPU and disk I/O needed to process the requested result. @@ -2883,21 +3151,15 @@ static int whereInScanEst( ** SQLITE_BIG_DBL. If a plan is found that uses the named index, ** then the cost is calculated in the usual way. ** -** If a NOT INDEXED clause (pSrc->notIndexed!=0) was attached to the table +** If a NOT INDEXED clause was attached to the table ** in the SELECT statement, then no indexes are considered. However, the ** selected plan may still take advantage of the built-in rowid primary key ** index. */ -static void bestBtreeIndex( - Parse *pParse, /* The parsing context */ - WhereClause *pWC, /* The WHERE clause */ - struct SrcList_item *pSrc, /* The FROM clause term to search */ - Bitmask notReady, /* Mask of cursors not available for indexing */ - Bitmask notValid, /* Cursors not available for any purpose */ - ExprList *pOrderBy, /* The ORDER BY clause */ - ExprList *pDistinct, /* The select-list if query is DISTINCT */ - WhereCost *pCost /* Lowest cost query plan */ -){ +static void bestBtreeIndex(WhereBestIdx *p){ + Parse *pParse = p->pParse; /* The parsing context */ + WhereClause *pWC = p->pWC; /* The WHERE clause */ + struct SrcList_item *pSrc = p->pSrc; /* The FROM clause term to search */ int iCur = pSrc->iCursor; /* The cursor of the table to be accessed */ Index *pProbe; /* An index we are evaluating */ Index *pIdx; /* Copy of pProbe, or zero for IPK index */ @@ -2906,11 +3168,16 @@ static void bestBtreeIndex( Index sPk; /* A fake index object for the primary key */ tRowcnt aiRowEstPk[2]; /* The aiRowEst[] value for the sPk index */ int aiColumnPk = -1; /* The aColumn[] value for the sPk index */ - int wsFlagMask; /* Allowed flags in pCost->plan.wsFlag */ + int wsFlagMask; /* Allowed flags in p->cost.plan.wsFlag */ + int nPriorSat; /* ORDER BY terms satisfied by outer loops */ + int nOrderBy; /* Number of ORDER BY terms */ + char bSortInit; /* Initializer for bSort in inner loop */ + char bDistInit; /* Initializer for bDist in inner loop */ + /* Initialize the cost to a worst-case value */ - memset(pCost, 0, sizeof(*pCost)); - pCost->rCost = SQLITE_BIG_DBL; + memset(&p->cost, 0, sizeof(p->cost)); + p->cost.rCost = SQLITE_BIG_DBL; /* If the pSrc table is the right table of a LEFT JOIN then we may not ** use an index to satisfy IS NULL constraints on that table. This is @@ -2956,22 +3223,29 @@ static void bestBtreeIndex( pIdx = 0; } + nOrderBy = p->pOrderBy ? p->pOrderBy->nExpr : 0; + if( p->i ){ + nPriorSat = p->aLevel[p->i-1].plan.nOBSat; + bSortInit = nPriorSat<nOrderBy; + bDistInit = 0; + }else{ + nPriorSat = 0; + bSortInit = nOrderBy>0; + bDistInit = p->pDistinct!=0; + } + /* Loop over all indices looking for the best one to use */ for(; pProbe; pIdx=pProbe=pProbe->pNext){ const tRowcnt * const aiRowEst = pProbe->aiRowEst; - double cost; /* Cost of using pProbe */ - double nRow; /* Estimated number of rows in result set */ + WhereCost pc; /* Cost of using pProbe */ double log10N = (double)1; /* base-10 logarithm of nRow (inexact) */ - int rev; /* True to scan in reverse order */ - int wsFlags = 0; - Bitmask used = 0; /* The following variables are populated based on the properties of ** index being evaluated. They are then used to determine the expected ** cost and number of rows returned. ** - ** nEq: + ** pc.plan.nEq: ** Number of equality terms that can be implemented using the index. ** In other words, the number of initial fields in the index that ** are used in == or IN or NOT NULL constraints of the WHERE clause. @@ -3014,6 +3288,10 @@ static void bestBtreeIndex( ** external sort (i.e. scanning the index being evaluated will not ** correctly order records). ** + ** bDist: + ** Boolean. True if there is a DISTINCT clause that will require an + ** external btree. + ** ** bLookup: ** Boolean. True if a table lookup is required for each index entry ** visited. In other words, true if this is not a covering index. @@ -3029,29 +3307,35 @@ static void bestBtreeIndex( ** SELECT a, b FROM tbl WHERE a = 1; ** SELECT a, b, c FROM tbl WHERE a = 1; */ - int nEq; /* Number of == or IN terms matching index */ int bInEst = 0; /* True if "x IN (SELECT...)" seen */ int nInMul = 1; /* Number of distinct equalities to lookup */ double rangeDiv = (double)1; /* Estimated reduction in search space */ int nBound = 0; /* Number of range constraints seen */ - int bSort = !!pOrderBy; /* True if external sort required */ - int bDist = !!pDistinct; /* True if index cannot help with DISTINCT */ - int bLookup = 0; /* True if not a covering index */ + char bSort = bSortInit; /* True if external sort required */ + char bDist = bDistInit; /* True if index cannot help with DISTINCT */ + char bLookup = 0; /* True if not a covering index */ WhereTerm *pTerm; /* A single term of the WHERE clause */ #ifdef SQLITE_ENABLE_STAT3 WhereTerm *pFirstTerm = 0; /* First term matching the index */ #endif - /* Determine the values of nEq and nInMul */ - for(nEq=0; nEq<pProbe->nColumn; nEq++){ - int j = pProbe->aiColumn[nEq]; - pTerm = findTerm(pWC, iCur, j, notReady, eqTermMask, pIdx); + WHERETRACE(( + " %s(%s):\n", + pSrc->pTab->zName, (pIdx ? pIdx->zName : "ipk") + )); + memset(&pc, 0, sizeof(pc)); + pc.plan.nOBSat = nPriorSat; + + /* Determine the values of pc.plan.nEq and nInMul */ + for(pc.plan.nEq=0; pc.plan.nEq<pProbe->nColumn; pc.plan.nEq++){ + int j = pProbe->aiColumn[pc.plan.nEq]; + pTerm = findTerm(pWC, iCur, j, p->notReady, eqTermMask, pIdx); if( pTerm==0 ) break; - wsFlags |= (WHERE_COLUMN_EQ|WHERE_ROWID_EQ); + pc.plan.wsFlags |= (WHERE_COLUMN_EQ|WHERE_ROWID_EQ); testcase( pTerm->pWC!=pWC ); if( pTerm->eOperator & WO_IN ){ Expr *pExpr = pTerm->pExpr; - wsFlags |= WHERE_COLUMN_IN; + pc.plan.wsFlags |= WHERE_COLUMN_IN; if( ExprHasProperty(pExpr, EP_xIsSelect) ){ /* "x IN (SELECT ...)": Assume the SELECT returns 25 rows */ nInMul *= 25; @@ -3061,12 +3345,12 @@ static void bestBtreeIndex( nInMul *= pExpr->x.pList->nExpr; } }else if( pTerm->eOperator & WO_ISNULL ){ - wsFlags |= WHERE_COLUMN_NULL; + pc.plan.wsFlags |= WHERE_COLUMN_NULL; } #ifdef SQLITE_ENABLE_STAT3 - if( nEq==0 && pProbe->aSample ) pFirstTerm = pTerm; + if( pc.plan.nEq==0 && pProbe->aSample ) pFirstTerm = pTerm; #endif - used |= pTerm->prereqRight; + pc.used |= pTerm->prereqRight; } /* If the index being considered is UNIQUE, and there is an equality @@ -3075,65 +3359,82 @@ static void bestBtreeIndex( ** indicate this to the caller. ** ** Otherwise, if the search may find more than one row, test to see if - ** there is a range constraint on indexed column (nEq+1) that can be - ** optimized using the index. + ** there is a range constraint on indexed column (pc.plan.nEq+1) that + ** can be optimized using the index. */ - if( nEq==pProbe->nColumn && pProbe->onError!=OE_None ){ - testcase( wsFlags & WHERE_COLUMN_IN ); - testcase( wsFlags & WHERE_COLUMN_NULL ); - if( (wsFlags & (WHERE_COLUMN_IN|WHERE_COLUMN_NULL))==0 ){ - wsFlags |= WHERE_UNIQUE; + if( pc.plan.nEq==pProbe->nColumn && pProbe->onError!=OE_None ){ + testcase( pc.plan.wsFlags & WHERE_COLUMN_IN ); + testcase( pc.plan.wsFlags & WHERE_COLUMN_NULL ); + if( (pc.plan.wsFlags & (WHERE_COLUMN_IN|WHERE_COLUMN_NULL))==0 ){ + pc.plan.wsFlags |= WHERE_UNIQUE; + if( p->i==0 || (p->aLevel[p->i-1].plan.wsFlags & WHERE_ALL_UNIQUE)!=0 ){ + pc.plan.wsFlags |= WHERE_ALL_UNIQUE; + } } }else if( pProbe->bUnordered==0 ){ - int j = (nEq==pProbe->nColumn ? -1 : pProbe->aiColumn[nEq]); - if( findTerm(pWC, iCur, j, notReady, WO_LT|WO_LE|WO_GT|WO_GE, pIdx) ){ - WhereTerm *pTop = findTerm(pWC, iCur, j, notReady, WO_LT|WO_LE, pIdx); - WhereTerm *pBtm = findTerm(pWC, iCur, j, notReady, WO_GT|WO_GE, pIdx); - whereRangeScanEst(pParse, pProbe, nEq, pBtm, pTop, &rangeDiv); + int j; + j = (pc.plan.nEq==pProbe->nColumn ? -1 : pProbe->aiColumn[pc.plan.nEq]); + if( findTerm(pWC, iCur, j, p->notReady, WO_LT|WO_LE|WO_GT|WO_GE, pIdx) ){ + WhereTerm *pTop, *pBtm; + pTop = findTerm(pWC, iCur, j, p->notReady, WO_LT|WO_LE, pIdx); + pBtm = findTerm(pWC, iCur, j, p->notReady, WO_GT|WO_GE, pIdx); + whereRangeScanEst(pParse, pProbe, pc.plan.nEq, pBtm, pTop, &rangeDiv); if( pTop ){ nBound = 1; - wsFlags |= WHERE_TOP_LIMIT; - used |= pTop->prereqRight; + pc.plan.wsFlags |= WHERE_TOP_LIMIT; + pc.used |= pTop->prereqRight; testcase( pTop->pWC!=pWC ); } if( pBtm ){ nBound++; - wsFlags |= WHERE_BTM_LIMIT; - used |= pBtm->prereqRight; + pc.plan.wsFlags |= WHERE_BTM_LIMIT; + pc.used |= pBtm->prereqRight; testcase( pBtm->pWC!=pWC ); } - wsFlags |= (WHERE_COLUMN_RANGE|WHERE_ROWID_RANGE); + pc.plan.wsFlags |= (WHERE_COLUMN_RANGE|WHERE_ROWID_RANGE); } } /* If there is an ORDER BY clause and the index being considered will ** naturally scan rows in the required order, set the appropriate flags - ** in wsFlags. Otherwise, if there is an ORDER BY clause but the index - ** will scan rows in a different order, set the bSort variable. */ - if( isSortingIndex( - pParse, pWC->pMaskSet, pProbe, iCur, pOrderBy, nEq, wsFlags, &rev) - ){ - bSort = 0; - wsFlags |= WHERE_ROWID_RANGE|WHERE_COLUMN_RANGE|WHERE_ORDERBY; - wsFlags |= (rev ? WHERE_REVERSE : 0); + ** in pc.plan.wsFlags. Otherwise, if there is an ORDER BY clause but + ** the index will scan rows in a different order, set the bSort + ** variable. */ + if( bSort && (pSrc->jointype & JT_LEFT)==0 ){ + int bRev = 2; + int bObUnique = 0; + WHERETRACE((" --> before isSortIndex: nPriorSat=%d\n",nPriorSat)); + pc.plan.nOBSat = isSortingIndex(p, pProbe, iCur, &bRev, &bObUnique); + WHERETRACE((" --> after isSortIndex: bRev=%d bObU=%d nOBSat=%d\n", + bRev, bObUnique, pc.plan.nOBSat)); + if( nPriorSat<pc.plan.nOBSat || (pc.plan.wsFlags & WHERE_ALL_UNIQUE)!=0 ){ + pc.plan.wsFlags |= WHERE_ORDERED; + if( bObUnique ) pc.plan.wsFlags |= WHERE_OB_UNIQUE; + } + if( nOrderBy==pc.plan.nOBSat ){ + bSort = 0; + pc.plan.wsFlags |= WHERE_ROWID_RANGE|WHERE_COLUMN_RANGE; + } + if( bRev & 1 ) pc.plan.wsFlags |= WHERE_REVERSE; } /* If there is a DISTINCT qualifier and this index will scan rows in ** order of the DISTINCT expressions, clear bDist and set the appropriate - ** flags in wsFlags. */ - if( isDistinctIndex(pParse, pWC, pProbe, iCur, pDistinct, nEq) - && (wsFlags & WHERE_COLUMN_IN)==0 + ** flags in pc.plan.wsFlags. */ + if( bDist + && isDistinctIndex(pParse, pWC, pProbe, iCur, p->pDistinct, pc.plan.nEq) + && (pc.plan.wsFlags & WHERE_COLUMN_IN)==0 ){ bDist = 0; - wsFlags |= WHERE_ROWID_RANGE|WHERE_COLUMN_RANGE|WHERE_DISTINCT; + pc.plan.wsFlags |= WHERE_ROWID_RANGE|WHERE_COLUMN_RANGE|WHERE_DISTINCT; } /* If currently calculating the cost of using an index (not the IPK ** index), determine if all required column data may be obtained without ** using the main table (i.e. if the index is a covering ** index for this query). If it is, set the WHERE_IDX_ONLY flag in - ** wsFlags. Otherwise, set the bLookup variable to true. */ - if( pIdx && wsFlags ){ + ** pc.plan.wsFlags. Otherwise, set the bLookup variable to true. */ + if( pIdx ){ Bitmask m = pSrc->colUsed; int j; for(j=0; j<pIdx->nColumn; j++){ @@ -3143,7 +3444,7 @@ static void bestBtreeIndex( } } if( m==0 ){ - wsFlags |= WHERE_IDX_ONLY; + pc.plan.wsFlags |= WHERE_IDX_ONLY; }else{ bLookup = 1; } @@ -3153,10 +3454,10 @@ static void bestBtreeIndex( ** Estimate the number of rows of output. For an "x IN (SELECT...)" ** constraint, do not let the estimate exceed half the rows in the table. */ - nRow = (double)(aiRowEst[nEq] * nInMul); - if( bInEst && nRow*2>aiRowEst[0] ){ - nRow = aiRowEst[0]/2; - nInMul = (int)(nRow / aiRowEst[nEq]); + pc.plan.nRow = (double)(aiRowEst[pc.plan.nEq] * nInMul); + if( bInEst && pc.plan.nRow*2>aiRowEst[0] ){ + pc.plan.nRow = aiRowEst[0]/2; + nInMul = (int)(pc.plan.nRow / aiRowEst[pc.plan.nEq]); } #ifdef SQLITE_ENABLE_STAT3 @@ -3166,15 +3467,19 @@ static void bestBtreeIndex( ** to get a better estimate on the number of rows based on ** VALUE and how common that value is according to the histogram. */ - if( nRow>(double)1 && nEq==1 && pFirstTerm!=0 && aiRowEst[1]>1 ){ + if( pc.plan.nRow>(double)1 && pc.plan.nEq==1 + && pFirstTerm!=0 && aiRowEst[1]>1 ){ assert( (pFirstTerm->eOperator & (WO_EQ|WO_ISNULL|WO_IN))!=0 ); if( pFirstTerm->eOperator & (WO_EQ|WO_ISNULL) ){ - testcase( pFirstTerm->eOperator==WO_EQ ); - testcase( pFirstTerm->eOperator==WO_ISNULL ); - whereEqualScanEst(pParse, pProbe, pFirstTerm->pExpr->pRight, &nRow); + testcase( pFirstTerm->eOperator & WO_EQ ); + testcase( pFirstTerm->eOperator & WO_EQUIV ); + testcase( pFirstTerm->eOperator & WO_ISNULL ); + whereEqualScanEst(pParse, pProbe, pFirstTerm->pExpr->pRight, + &pc.plan.nRow); }else if( bInEst==0 ){ - assert( pFirstTerm->eOperator==WO_IN ); - whereInScanEst(pParse, pProbe, pFirstTerm->pExpr->x.pList, &nRow); + assert( pFirstTerm->eOperator & WO_IN ); + whereInScanEst(pParse, pProbe, pFirstTerm->pExpr->x.pList, + &pc.plan.nRow); } } #endif /* SQLITE_ENABLE_STAT3 */ @@ -3182,8 +3487,8 @@ static void bestBtreeIndex( /* Adjust the number of output rows and downward to reflect rows ** that are excluded by range constraints. */ - nRow = nRow/rangeDiv; - if( nRow<1 ) nRow = 1; + pc.plan.nRow = pc.plan.nRow/rangeDiv; + if( pc.plan.nRow<1 ) pc.plan.nRow = 1; /* Experiments run on real SQLite databases show that the time needed ** to do a binary search to locate a row in a table or index is roughly @@ -3198,7 +3503,19 @@ static void bestBtreeIndex( ** So this computation assumes table records are about twice as big ** as index records */ - if( (wsFlags & WHERE_NOT_FULLSCAN)==0 ){ + if( (pc.plan.wsFlags&~(WHERE_REVERSE|WHERE_ORDERED|WHERE_OB_UNIQUE)) + ==WHERE_IDX_ONLY + && (pWC->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 + && sqlite3GlobalConfig.bUseCis + && OptimizationEnabled(pParse->db, SQLITE_CoverIdxScan) + ){ + /* This index is not useful for indexing, but it is a covering index. + ** A full-scan of the index might be a little faster than a full-scan + ** of the table, so give this case a cost slightly less than a table + ** scan. */ + pc.rCost = aiRowEst[0]*3 + pProbe->nColumn; + pc.plan.wsFlags |= WHERE_COVER_SCAN|WHERE_COLUMN_RANGE; + }else if( (pc.plan.wsFlags & WHERE_NOT_FULLSCAN)==0 ){ /* The cost of a full table scan is a number of move operations equal ** to the number of rows in the table. ** @@ -3208,10 +3525,15 @@ static void bestBtreeIndex( ** decision and one which we expect to revisit in the future. But ** it seems to be working well enough at the moment. */ - cost = aiRowEst[0]*4; + pc.rCost = aiRowEst[0]*4; + pc.plan.wsFlags &= ~WHERE_IDX_ONLY; + if( pIdx ){ + pc.plan.wsFlags &= ~WHERE_ORDERED; + pc.plan.nOBSat = nPriorSat; + } }else{ log10N = estLog(aiRowEst[0]); - cost = nRow; + pc.rCost = pc.plan.nRow; if( pIdx ){ if( bLookup ){ /* For an index lookup followed by a table lookup: @@ -3219,20 +3541,20 @@ static void bestBtreeIndex( ** + nRow steps through the index ** + nRow table searches to lookup the table entry using the rowid */ - cost += (nInMul + nRow)*log10N; + pc.rCost += (nInMul + pc.plan.nRow)*log10N; }else{ /* For a covering index: ** nInMul index searches to find the initial entry ** + nRow steps through the index */ - cost += nInMul*log10N; + pc.rCost += nInMul*log10N; } }else{ /* For a rowid primary key lookup: ** nInMult table searches to find the initial entry for each range ** + nRow steps through the table */ - cost += nInMul*log10N; + pc.rCost += nInMul*log10N; } } @@ -3243,10 +3565,12 @@ static void bestBtreeIndex( ** difference and select C of 3.0. */ if( bSort ){ - cost += nRow*estLog(nRow)*3; + double m = estLog(pc.plan.nRow*(nOrderBy - pc.plan.nOBSat)/nOrderBy); + m *= (double)(pc.plan.nOBSat ? 2 : 3); + pc.rCost += pc.plan.nRow*m; } if( bDist ){ - cost += nRow*estLog(nRow)*3; + pc.rCost += pc.plan.nRow*estLog(pc.plan.nRow)*3; } /**** Cost of using this index has now been computed ****/ @@ -3267,25 +3591,25 @@ static void bestBtreeIndex( ** might be selected even when there exists an optimal index that has ** no such dependency. */ - if( nRow>2 && cost<=pCost->rCost ){ + if( pc.plan.nRow>2 && pc.rCost<=p->cost.rCost ){ int k; /* Loop counter */ - int nSkipEq = nEq; /* Number of == constraints to skip */ + int nSkipEq = pc.plan.nEq; /* Number of == constraints to skip */ int nSkipRange = nBound; /* Number of < constraints to skip */ Bitmask thisTab; /* Bitmap for pSrc */ thisTab = getMask(pWC->pMaskSet, iCur); - for(pTerm=pWC->a, k=pWC->nTerm; nRow>2 && k; k--, pTerm++){ + for(pTerm=pWC->a, k=pWC->nTerm; pc.plan.nRow>2 && k; k--, pTerm++){ if( pTerm->wtFlags & TERM_VIRTUAL ) continue; - if( (pTerm->prereqAll & notValid)!=thisTab ) continue; + if( (pTerm->prereqAll & p->notValid)!=thisTab ) continue; if( pTerm->eOperator & (WO_EQ|WO_IN|WO_ISNULL) ){ if( nSkipEq ){ - /* Ignore the first nEq equality matches since the index + /* Ignore the first pc.plan.nEq equality matches since the index ** has already accounted for these */ nSkipEq--; }else{ /* Assume each additional equality match reduces the result ** set size by a factor of 10 */ - nRow /= 10; + pc.plan.nRow /= 10; } }else if( pTerm->eOperator & (WO_LT|WO_LE|WO_GT|WO_GE) ){ if( nSkipRange ){ @@ -3299,37 +3623,33 @@ static void bestBtreeIndex( ** more selective intentionally because of the subjective ** observation that indexed range constraints really are more ** selective in practice, on average. */ - nRow /= 3; + pc.plan.nRow /= 3; } - }else if( pTerm->eOperator!=WO_NOOP ){ + }else if( (pTerm->eOperator & WO_NOOP)==0 ){ /* Any other expression lowers the output row count by half */ - nRow /= 2; + pc.plan.nRow /= 2; } } - if( nRow<2 ) nRow = 2; + if( pc.plan.nRow<2 ) pc.plan.nRow = 2; } WHERETRACE(( - "%s(%s): nEq=%d nInMul=%d rangeDiv=%d bSort=%d bLookup=%d wsFlags=0x%x\n" - " notReady=0x%llx log10N=%.1f nRow=%.1f cost=%.1f used=0x%llx\n", - pSrc->pTab->zName, (pIdx ? pIdx->zName : "ipk"), - nEq, nInMul, (int)rangeDiv, bSort, bLookup, wsFlags, - notReady, log10N, nRow, cost, used + " nEq=%d nInMul=%d rangeDiv=%d bSort=%d bLookup=%d wsFlags=0x%08x\n" + " notReady=0x%llx log10N=%.1f nRow=%.1f cost=%.1f\n" + " used=0x%llx nOBSat=%d\n", + pc.plan.nEq, nInMul, (int)rangeDiv, bSort, bLookup, pc.plan.wsFlags, + p->notReady, log10N, pc.plan.nRow, pc.rCost, pc.used, + pc.plan.nOBSat )); /* If this index is the best we have seen so far, then record this - ** index and its cost in the pCost structure. + ** index and its cost in the p->cost structure. */ - if( (!pIdx || wsFlags) - && (cost<pCost->rCost || (cost<=pCost->rCost && nRow<pCost->plan.nRow)) - ){ - pCost->rCost = cost; - pCost->used = used; - pCost->plan.nRow = nRow; - pCost->plan.wsFlags = (wsFlags&wsFlagMask); - pCost->plan.nEq = nEq; - pCost->plan.u.pIdx = pIdx; + if( (!pIdx || pc.plan.wsFlags) && compareCost(&pc, &p->cost) ){ + p->cost = pc; + p->cost.plan.wsFlags &= wsFlagMask; + p->cost.plan.u.pIdx = pIdx; } /* If there was an INDEXED BY clause, then only that one index is @@ -3344,27 +3664,26 @@ static void bestBtreeIndex( /* If there is no ORDER BY clause and the SQLITE_ReverseOrder flag ** is set, then reverse the order that the index will be scanned ** in. This is used for application testing, to help find cases - ** where application behaviour depends on the (undefined) order that + ** where application behavior depends on the (undefined) order that ** SQLite outputs rows in in the absence of an ORDER BY clause. */ - if( !pOrderBy && pParse->db->flags & SQLITE_ReverseOrder ){ - pCost->plan.wsFlags |= WHERE_REVERSE; + if( !p->pOrderBy && pParse->db->flags & SQLITE_ReverseOrder ){ + p->cost.plan.wsFlags |= WHERE_REVERSE; } - assert( pOrderBy || (pCost->plan.wsFlags&WHERE_ORDERBY)==0 ); - assert( pCost->plan.u.pIdx==0 || (pCost->plan.wsFlags&WHERE_ROWID_EQ)==0 ); + assert( p->pOrderBy || (p->cost.plan.wsFlags&WHERE_ORDERED)==0 ); + assert( p->cost.plan.u.pIdx==0 || (p->cost.plan.wsFlags&WHERE_ROWID_EQ)==0 ); assert( pSrc->pIndex==0 - || pCost->plan.u.pIdx==0 - || pCost->plan.u.pIdx==pSrc->pIndex + || p->cost.plan.u.pIdx==0 + || p->cost.plan.u.pIdx==pSrc->pIndex ); - WHERETRACE(("best index is: %s\n", - ((pCost->plan.wsFlags & WHERE_NOT_FULLSCAN)==0 ? "none" : - pCost->plan.u.pIdx ? pCost->plan.u.pIdx->zName : "ipk") - )); + WHERETRACE((" best index is %s cost=%.1f\n", + p->cost.plan.u.pIdx ? p->cost.plan.u.pIdx->zName : "ipk", + p->cost.rCost)); - bestOrClauseIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, pCost); - bestAutomaticIndex(pParse, pWC, pSrc, notReady, pCost); - pCost->plan.wsFlags |= eqTermMask; + bestOrClauseIndex(p); + bestAutomaticIndex(p); + p->cost.plan.wsFlags |= eqTermMask; } /* @@ -3372,28 +3691,28 @@ static void bestBtreeIndex( ** best query plan and its cost into the WhereCost object supplied ** as the last parameter. This function may calculate the cost of ** both real and virtual table scans. +** +** This function does not take ORDER BY or DISTINCT into account. Nor +** does it remember the virtual table query plan. All it does is compute +** the cost while determining if an OR optimization is applicable. The +** details will be reconsidered later if the optimization is found to be +** applicable. */ -static void bestIndex( - Parse *pParse, /* The parsing context */ - WhereClause *pWC, /* The WHERE clause */ - struct SrcList_item *pSrc, /* The FROM clause term to search */ - Bitmask notReady, /* Mask of cursors not available for indexing */ - Bitmask notValid, /* Cursors not available for any purpose */ - ExprList *pOrderBy, /* The ORDER BY clause */ - WhereCost *pCost /* Lowest cost query plan */ -){ +static void bestIndex(WhereBestIdx *p){ #ifndef SQLITE_OMIT_VIRTUALTABLE - if( IsVirtual(pSrc->pTab) ){ - sqlite3_index_info *p = 0; - bestVirtualIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, pCost,&p); - if( p->needToFreeIdxStr ){ - sqlite3_free(p->idxStr); + if( IsVirtual(p->pSrc->pTab) ){ + sqlite3_index_info *pIdxInfo = 0; + p->ppIdxInfo = &pIdxInfo; + bestVirtualIndex(p); + assert( pIdxInfo!=0 || p->pParse->db->mallocFailed ); + if( pIdxInfo && pIdxInfo->needToFreeIdxStr ){ + sqlite3_free(pIdxInfo->idxStr); } - sqlite3DbFree(pParse->db, p); + sqlite3DbFree(p->pParse->db, pIdxInfo); }else #endif { - bestBtreeIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, 0, pCost); + bestBtreeIndex(p); } } @@ -3492,7 +3811,8 @@ static void codeApplyAffinity(Parse *pParse, int base, int n, char *zAff){ static int codeEqualityTerm( Parse *pParse, /* The parsing context */ WhereTerm *pTerm, /* The term of the WHERE clause to be coded */ - WhereLevel *pLevel, /* When level of the FROM clause we are working on */ + WhereLevel *pLevel, /* The level of the FROM clause we are working on */ + int iEq, /* Index of the equality term within this level */ int iTarget /* Attempt to leave results in this register */ ){ Expr *pX = pTerm->pExpr; @@ -3510,12 +3830,26 @@ static int codeEqualityTerm( int eType; int iTab; struct InLoop *pIn; + u8 bRev = (pLevel->plan.wsFlags & WHERE_REVERSE)!=0; + if( (pLevel->plan.wsFlags & WHERE_INDEXED)!=0 + && pLevel->plan.u.pIdx->aSortOrder[iEq] + ){ + testcase( iEq==0 ); + testcase( iEq==pLevel->plan.u.pIdx->nColumn-1 ); + testcase( iEq>0 && iEq+1<pLevel->plan.u.pIdx->nColumn ); + testcase( bRev ); + bRev = !bRev; + } assert( pX->op==TK_IN ); iReg = iTarget; eType = sqlite3FindInIndex(pParse, pX, 0); + if( eType==IN_INDEX_INDEX_DESC ){ + testcase( bRev ); + bRev = !bRev; + } iTab = pX->iTable; - sqlite3VdbeAddOp2(v, OP_Rewind, iTab, 0); + sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iTab, 0); assert( pLevel->plan.wsFlags & WHERE_IN_ABLE ); if( pLevel->u.in.nIn==0 ){ pLevel->addrNxt = sqlite3VdbeMakeLabel(v); @@ -3533,6 +3867,7 @@ static int codeEqualityTerm( }else{ pIn->addrInTop = sqlite3VdbeAddOp3(v, OP_Column, iTab, 0, iReg); } + pIn->eEndLoopOp = bRev ? OP_Prev : OP_Next; sqlite3VdbeAddOp1(v, OP_IsNull, iReg); }else{ pLevel->u.in.nIn = 0; @@ -3627,7 +3962,7 @@ static int codeAllEqualityTerms( ** Ex: CREATE INDEX i1 ON t1(a,b,a); SELECT * FROM t1 WHERE a=0 AND b=0; */ testcase( (pTerm->wtFlags & TERM_CODED)!=0 ); testcase( pTerm->wtFlags & TERM_VIRTUAL ); /* EV: R-30575-11662 */ - r1 = codeEqualityTerm(pParse, pTerm, pLevel, regBase+j); + r1 = codeEqualityTerm(pParse, pTerm, pLevel, j, regBase+j); if( r1!=regBase+j ){ if( nReg==1 ){ sqlite3ReleaseTempReg(pParse, regBase); @@ -3837,6 +4172,7 @@ static Bitmask codeOneLoopStart( int addrCont; /* Jump here to continue with next cycle */ int iRowidReg = 0; /* Rowid is stored in this register, if not zero */ int iReleaseReg = 0; /* Temp register to free before returning */ + Bitmask newNotReady; /* Return value */ pParse = pWInfo->pParse; v = pParse->pVdbe; @@ -3847,6 +4183,7 @@ static Bitmask codeOneLoopStart( bRev = (pLevel->plan.wsFlags & WHERE_REVERSE)!=0; omitTable = (pLevel->plan.wsFlags & WHERE_IDX_ONLY)!=0 && (wctrlFlags & WHERE_FORCE_TABLE)==0; + VdbeNoopComment((v, "Begin Join Loop %d", iLevel)); /* Create labels for the "break" and "continue" instructions ** for the current loop. Jump to addrBrk to break out of a loop. @@ -3871,12 +4208,23 @@ static Bitmask codeOneLoopStart( VdbeComment((v, "init LEFT JOIN no-match flag")); } + /* Special case of a FROM clause subquery implemented as a co-routine */ + if( pTabItem->viaCoroutine ){ + int regYield = pTabItem->regReturn; + sqlite3VdbeAddOp2(v, OP_Integer, pTabItem->addrFillSub-1, regYield); + pLevel->p2 = sqlite3VdbeAddOp1(v, OP_Yield, regYield); + VdbeComment((v, "next row of co-routine %s", pTabItem->pTab->zName)); + sqlite3VdbeAddOp2(v, OP_If, regYield+1, addrBrk); + pLevel->op = OP_Goto; + }else + #ifndef SQLITE_OMIT_VIRTUALTABLE if( (pLevel->plan.wsFlags & WHERE_VIRTUALTABLE)!=0 ){ /* Case 0: The table is a virtual-table. Use the VFilter and VNext ** to access the data. */ int iReg; /* P3 Value for OP_VFilter */ + int addrNotFound; sqlite3_index_info *pVtabIdx = pLevel->plan.u.pVtabIdx; int nConstraint = pVtabIdx->nConstraint; struct sqlite3_index_constraint_usage *aUsage = @@ -3886,11 +4234,18 @@ static Bitmask codeOneLoopStart( sqlite3ExprCachePush(pParse); iReg = sqlite3GetTempRange(pParse, nConstraint+2); + addrNotFound = pLevel->addrBrk; for(j=1; j<=nConstraint; j++){ for(k=0; k<nConstraint; k++){ if( aUsage[k].argvIndex==j ){ - int iTerm = aConstraint[k].iTermOffset; - sqlite3ExprCode(pParse, pWC->a[iTerm].pExpr->pRight, iReg+j+1); + int iTarget = iReg+j+1; + pTerm = &pWC->a[aConstraint[k].iTermOffset]; + if( pTerm->eOperator & WO_IN ){ + codeEqualityTerm(pParse, pTerm, pLevel, k, iTarget); + addrNotFound = pLevel->addrNxt; + }else{ + sqlite3ExprCode(pParse, pTerm->pExpr->pRight, iTarget); + } break; } } @@ -3898,7 +4253,7 @@ static Bitmask codeOneLoopStart( } sqlite3VdbeAddOp2(v, OP_Integer, pVtabIdx->idxNum, iReg); sqlite3VdbeAddOp2(v, OP_Integer, j-1, iReg+1); - sqlite3VdbeAddOp4(v, OP_VFilter, iCur, addrBrk, iReg, pVtabIdx->idxStr, + sqlite3VdbeAddOp4(v, OP_VFilter, iCur, addrNotFound, iReg, pVtabIdx->idxStr, pVtabIdx->needToFreeIdxStr ? P4_MPRINTF : P4_STATIC); pVtabIdx->needToFreeIdxStr = 0; for(j=0; j<nConstraint; j++){ @@ -3925,13 +4280,13 @@ static Bitmask codeOneLoopStart( pTerm = findTerm(pWC, iCur, -1, notReady, WO_EQ|WO_IN, 0); assert( pTerm!=0 ); assert( pTerm->pExpr!=0 ); - assert( pTerm->leftCursor==iCur ); assert( omitTable==0 ); testcase( pTerm->wtFlags & TERM_VIRTUAL ); /* EV: R-30575-11662 */ - iRowidReg = codeEqualityTerm(pParse, pTerm, pLevel, iReleaseReg); + iRowidReg = codeEqualityTerm(pParse, pTerm, pLevel, 0, iReleaseReg); addrNxt = pLevel->addrNxt; sqlite3VdbeAddOp2(v, OP_MustBeInt, iRowidReg, addrNxt); sqlite3VdbeAddOp3(v, OP_NotExists, iCur, addrNxt, iRowidReg); + sqlite3ExprCacheAffinityChange(pParse, iRowidReg, 1); sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg); VdbeComment((v, "pk")); pLevel->op = OP_Noop; @@ -4089,7 +4444,7 @@ static Bitmask codeOneLoopStart( ** this requires some special handling. */ if( (wctrlFlags&WHERE_ORDERBY_MIN)!=0 - && (pLevel->plan.wsFlags&WHERE_ORDERBY) + && (pLevel->plan.wsFlags&WHERE_ORDERED) && (pIdx->nColumn>nEq) ){ /* assert( pOrderBy->nExpr==1 ); */ @@ -4252,6 +4607,11 @@ static Bitmask codeOneLoopStart( pLevel->op = OP_Next; } pLevel->p1 = iIdxCur; + if( pLevel->plan.wsFlags & WHERE_COVER_SCAN ){ + pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP; + }else{ + assert( pLevel->p5==0 ); + } }else #ifndef SQLITE_OMIT_OR_OPTIMIZATION @@ -4311,7 +4671,7 @@ static Bitmask codeOneLoopStart( pTerm = pLevel->plan.u.pTerm; assert( pTerm!=0 ); - assert( pTerm->eOperator==WO_OR ); + assert( pTerm->eOperator & WO_OR ); assert( (pTerm->wtFlags & TERM_ORINFO)!=0 ); pOrWc = &pTerm->u.pOrInfo->wc; pLevel->op = OP_Return; @@ -4366,6 +4726,10 @@ static Bitmask codeOneLoopStart( ** the "interesting" terms of z - terms that did not originate in the ** ON or USING clause of a LEFT JOIN, and terms that are usable as ** indices. + ** + ** This optimization also only applies if the (x1 OR x2 OR ...) term + ** is not contained in the ON clause of a LEFT JOIN. + ** See ticket http://www.sqlite.org/src/info/f2369304e4 */ if( pWC->nTerm>1 ){ int iTerm; @@ -4384,10 +4748,10 @@ static Bitmask codeOneLoopStart( for(ii=0; ii<pOrWc->nTerm; ii++){ WhereTerm *pOrTerm = &pOrWc->a[ii]; - if( pOrTerm->leftCursor==iCur || pOrTerm->eOperator==WO_AND ){ + if( pOrTerm->leftCursor==iCur || (pOrTerm->eOperator & WO_AND)!=0 ){ WhereInfo *pSubWInfo; /* Info for single OR-term scan */ Expr *pOrExpr = pOrTerm->pExpr; - if( pAndExpr ){ + if( pAndExpr && !ExprHasProperty(pOrExpr, EP_FromJoin) ){ pAndExpr->pLeft = pOrExpr; pOrExpr = pAndExpr; } @@ -4474,7 +4838,7 @@ static Bitmask codeOneLoopStart( pLevel->p2 = 1 + sqlite3VdbeAddOp2(v, aStart[bRev], iCur, addrBrk); pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP; } - notReady &= ~getMask(pWC->pMaskSet, iCur); + newNotReady = notReady & ~getMask(pWC->pMaskSet, iCur); /* Insert code to test every subexpression that can be completely ** computed using the current set of tables. @@ -4488,7 +4852,7 @@ static Bitmask codeOneLoopStart( testcase( pTerm->wtFlags & TERM_VIRTUAL ); /* IMP: R-30575-11662 */ testcase( pTerm->wtFlags & TERM_CODED ); if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; - if( (pTerm->prereqAll & notReady)!=0 ){ + if( (pTerm->prereqAll & newNotReady)!=0 ){ testcase( pWInfo->untestedTerms==0 && (pWInfo->wctrlFlags & WHERE_ONETABLE_ONLY)!=0 ); pWInfo->untestedTerms = 1; @@ -4503,6 +4867,33 @@ static Bitmask codeOneLoopStart( pTerm->wtFlags |= TERM_CODED; } + /* Insert code to test for implied constraints based on transitivity + ** of the "==" operator. + ** + ** Example: If the WHERE clause contains "t1.a=t2.b" and "t2.b=123" + ** and we are coding the t1 loop and the t2 loop has not yet coded, + ** then we cannot use the "t1.a=t2.b" constraint, but we can code + ** the implied "t1.a=123" constraint. + */ + for(pTerm=pWC->a, j=pWC->nTerm; j>0; j--, pTerm++){ + Expr *pE; + WhereTerm *pAlt; + Expr sEq; + if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; + if( pTerm->eOperator!=(WO_EQUIV|WO_EQ) ) continue; + if( pTerm->leftCursor!=iCur ) continue; + pE = pTerm->pExpr; + assert( !ExprHasProperty(pE, EP_FromJoin) ); + assert( (pTerm->prereqRight & newNotReady)!=0 ); + pAlt = findTerm(pWC, iCur, pTerm->u.leftColumn, notReady, WO_EQ|WO_IN, 0); + if( pAlt==0 ) continue; + if( pAlt->wtFlags & (TERM_CODED) ) continue; + VdbeNoopComment((v, "begin transitive constraint")); + sEq = *pAlt->pExpr; + sEq.pLeft = pE->pLeft; + sqlite3ExprIfFalse(pParse, &sEq, addrCont, SQLITE_JUMPIFNULL); + } + /* For a LEFT OUTER JOIN, generate code that will record the fact that ** at least one row of the right table has matched the left table. */ @@ -4515,7 +4906,7 @@ static Bitmask codeOneLoopStart( testcase( pTerm->wtFlags & TERM_VIRTUAL ); /* IMP: R-30575-11662 */ testcase( pTerm->wtFlags & TERM_CODED ); if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; - if( (pTerm->prereqAll & notReady)!=0 ){ + if( (pTerm->prereqAll & newNotReady)!=0 ){ assert( pWInfo->untestedTerms ); continue; } @@ -4526,7 +4917,7 @@ static Bitmask codeOneLoopStart( } sqlite3ReleaseTempReg(pParse, iReleaseReg); - return notReady; + return newNotReady; } #if defined(SQLITE_TEST) @@ -4646,42 +5037,46 @@ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ ** ** ORDER BY CLAUSE PROCESSING ** -** *ppOrderBy is a pointer to the ORDER BY clause of a SELECT statement, +** pOrderBy is a pointer to the ORDER BY clause of a SELECT statement, ** if there is one. If there is no ORDER BY clause or if this routine -** is called from an UPDATE or DELETE statement, then ppOrderBy is NULL. +** is called from an UPDATE or DELETE statement, then pOrderBy is NULL. ** ** If an index can be used so that the natural output order of the table ** scan is correct for the ORDER BY clause, then that index is used and -** *ppOrderBy is set to NULL. This is an optimization that prevents an -** unnecessary sort of the result set if an index appropriate for the -** ORDER BY clause already exists. +** the returned WhereInfo.nOBSat field is set to pOrderBy->nExpr. This +** is an optimization that prevents an unnecessary sort of the result set +** if an index appropriate for the ORDER BY clause already exists. ** ** If the where clause loops cannot be arranged to provide the correct -** output order, then the *ppOrderBy is unchanged. +** output order, then WhereInfo.nOBSat is 0. */ WhereInfo *sqlite3WhereBegin( Parse *pParse, /* The parser context */ SrcList *pTabList, /* A list of all tables to be scanned */ Expr *pWhere, /* The WHERE clause */ - ExprList **ppOrderBy, /* An ORDER BY clause, or NULL */ + ExprList *pOrderBy, /* An ORDER BY clause, or NULL */ ExprList *pDistinct, /* The select-list for DISTINCT queries - or NULL */ u16 wctrlFlags, /* One of the WHERE_* flags defined in sqliteInt.h */ int iIdxCur /* If WHERE_ONETABLE_ONLY is set, index cursor number */ ){ - int i; /* Loop counter */ int nByteWInfo; /* Num. bytes allocated for WhereInfo struct */ int nTabList; /* Number of elements in pTabList */ WhereInfo *pWInfo; /* Will become the return value of this function */ Vdbe *v = pParse->pVdbe; /* The virtual database engine */ Bitmask notReady; /* Cursors that are not yet positioned */ + WhereBestIdx sWBI; /* Best index search context */ WhereMaskSet *pMaskSet; /* The expression mask set */ - WhereClause *pWC; /* Decomposition of the WHERE clause */ - struct SrcList_item *pTabItem; /* A single entry from pTabList */ - WhereLevel *pLevel; /* A single level in the pWInfo list */ - int iFrom; /* First unused FROM clause element */ + WhereLevel *pLevel; /* A single level in pWInfo->a[] */ + int iFrom; /* First unused FROM clause element */ int andFlags; /* AND-ed combination of all pWC->a[].wtFlags */ + int ii; /* Loop counter */ sqlite3 *db; /* Database connection */ + + /* Variable initialization */ + memset(&sWBI, 0, sizeof(sWBI)); + sWBI.pParse = pParse; + /* The number of tables in the FROM clause is limited by the number of ** bits in a Bitmask */ @@ -4721,22 +5116,23 @@ WhereInfo *sqlite3WhereBegin( pWInfo->pParse = pParse; pWInfo->pTabList = pTabList; pWInfo->iBreak = sqlite3VdbeMakeLabel(v); - pWInfo->pWC = pWC = (WhereClause *)&((u8 *)pWInfo)[nByteWInfo]; + pWInfo->pWC = sWBI.pWC = (WhereClause *)&((u8 *)pWInfo)[nByteWInfo]; pWInfo->wctrlFlags = wctrlFlags; pWInfo->savedNQueryLoop = pParse->nQueryLoop; - pMaskSet = (WhereMaskSet*)&pWC[1]; + pMaskSet = (WhereMaskSet*)&sWBI.pWC[1]; + sWBI.aLevel = pWInfo->a; /* Disable the DISTINCT optimization if SQLITE_DistinctOpt is set via ** sqlite3_test_ctrl(SQLITE_TESTCTRL_OPTIMIZATIONS,...) */ - if( db->flags & SQLITE_DistinctOpt ) pDistinct = 0; + if( OptimizationDisabled(db, SQLITE_DistinctOpt) ) pDistinct = 0; /* Split the WHERE clause into separate subexpressions where each ** subexpression is separated by an AND operator. */ initMaskSet(pMaskSet); - whereClauseInit(pWC, pParse, pMaskSet, wctrlFlags); + whereClauseInit(sWBI.pWC, pParse, pMaskSet, wctrlFlags); sqlite3ExprCodeConstants(pParse, pWhere); - whereSplit(pWC, pWhere, TK_AND); /* IMP: R-15842-53296 */ + whereSplit(sWBI.pWC, pWhere, TK_AND); /* IMP: R-15842-53296 */ /* Special case: a WHERE clause that is constant. Evaluate the ** expression and either jump over all of the code or fall thru. @@ -4757,30 +5153,19 @@ WhereInfo *sqlite3WhereBegin( ** bitmask for all tables to the left of the join. Knowing the bitmask ** for all tables to the left of a left join is important. Ticket #3015. ** - ** Configure the WhereClause.vmask variable so that bits that correspond - ** to virtual table cursors are set. This is used to selectively disable - ** the OR-to-IN transformation in exprAnalyzeOrTerm(). It is not helpful - ** with virtual tables. - ** ** Note that bitmasks are created for all pTabList->nSrc tables in ** pTabList, not just the first nTabList tables. nTabList is normally ** equal to pTabList->nSrc but might be shortened to 1 if the ** WHERE_ONETABLE_ONLY flag is set. */ - assert( pWC->vmask==0 && pMaskSet->n==0 ); - for(i=0; i<pTabList->nSrc; i++){ - createMask(pMaskSet, pTabList->a[i].iCursor); -#ifndef SQLITE_OMIT_VIRTUALTABLE - if( ALWAYS(pTabList->a[i].pTab) && IsVirtual(pTabList->a[i].pTab) ){ - pWC->vmask |= ((Bitmask)1 << i); - } -#endif + for(ii=0; ii<pTabList->nSrc; ii++){ + createMask(pMaskSet, pTabList->a[ii].iCursor); } #ifndef NDEBUG { Bitmask toTheLeft = 0; - for(i=0; i<pTabList->nSrc; i++){ - Bitmask m = getMask(pMaskSet, pTabList->a[i].iCursor); + for(ii=0; ii<pTabList->nSrc; ii++){ + Bitmask m = getMask(pMaskSet, pTabList->a[ii].iCursor); assert( (m-1)==toTheLeft ); toTheLeft |= m; } @@ -4792,7 +5177,7 @@ WhereInfo *sqlite3WhereBegin( ** want to analyze these virtual terms, so start analyzing at the end ** and work forward so that the added virtual terms are never processed. */ - exprAnalyzeAll(pTabList, pWC); + exprAnalyzeAll(pTabList, sWBI.pWC); if( db->mallocFailed ){ goto whereBeginError; } @@ -4801,7 +5186,7 @@ WhereInfo *sqlite3WhereBegin( ** If it is, then set pDistinct to NULL and WhereInfo.eDistinct to ** WHERE_DISTINCT_UNIQUE to tell the caller to ignore the DISTINCT. */ - if( pDistinct && isDistinctRedundant(pParse, pTabList, pWC, pDistinct) ){ + if( pDistinct && isDistinctRedundant(pParse, pTabList, sWBI.pWC, pDistinct) ){ pDistinct = 0; pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; } @@ -4821,22 +5206,26 @@ WhereInfo *sqlite3WhereBegin( ** This loop also figures out the nesting order of tables in the FROM ** clause. */ - notReady = ~(Bitmask)0; + sWBI.notValid = ~(Bitmask)0; + sWBI.pOrderBy = pOrderBy; + sWBI.n = nTabList; + sWBI.pDistinct = pDistinct; andFlags = ~0; WHERETRACE(("*** Optimizer Start ***\n")); - for(i=iFrom=0, pLevel=pWInfo->a; i<nTabList; i++, pLevel++){ + for(sWBI.i=iFrom=0, pLevel=pWInfo->a; sWBI.i<nTabList; sWBI.i++, pLevel++){ WhereCost bestPlan; /* Most efficient plan seen so far */ Index *pIdx; /* Index for FROM table at pTabItem */ int j; /* For looping over FROM tables */ int bestJ = -1; /* The value of j */ Bitmask m; /* Bitmask value for j or bestJ */ int isOptimal; /* Iterator for optimal/non-optimal search */ + int ckOptimal; /* Do the optimal scan check */ int nUnconstrained; /* Number tables without INDEXED BY */ Bitmask notIndexed; /* Mask of tables that cannot use an index */ memset(&bestPlan, 0, sizeof(bestPlan)); bestPlan.rCost = SQLITE_BIG_DBL; - WHERETRACE(("*** Begin search for loop %d ***\n", i)); + WHERETRACE(("*** Begin search for loop %d ***\n", sWBI.i)); /* Loop through the remaining entries in the FROM clause to find the ** next nested loop. The loop tests all FROM clause entries @@ -4852,8 +5241,8 @@ WhereInfo *sqlite3WhereBegin( ** by waiting for other tables to run first. This "optimal" test works ** by first assuming that the FROM clause is on the inner loop and finding ** its query plan, then checking to see if that query plan uses any - ** other FROM clause terms that are notReady. If no notReady terms are - ** used then the "optimal" query plan works. + ** other FROM clause terms that are sWBI.notValid. If no notValid terms + ** are used then the "optimal" query plan works. ** ** Note that the WhereCost.nRow parameter for an optimal scan might ** not be as small as it would be if the table really were the innermost @@ -4865,10 +5254,8 @@ WhereInfo *sqlite3WhereBegin( ** strategies were found by the first iteration. This second iteration ** is used to search for the lowest cost scan overall. ** - ** Previous versions of SQLite performed only the second iteration - - ** the next outermost loop was always that with the lowest overall - ** cost. However, this meant that SQLite could select the wrong plan - ** for scripts such as the following: + ** Without the optimal scan step (the first iteration) a suboptimal + ** plan might be chosen for queries like this: ** ** CREATE TABLE t1(a, b); ** CREATE TABLE t2(c, d); @@ -4883,59 +5270,89 @@ WhereInfo *sqlite3WhereBegin( */ nUnconstrained = 0; notIndexed = 0; - for(isOptimal=(iFrom<nTabList-1); isOptimal>=0 && bestJ<0; isOptimal--){ - Bitmask mask; /* Mask of tables not yet ready */ - for(j=iFrom, pTabItem=&pTabList->a[j]; j<nTabList; j++, pTabItem++){ - int doNotReorder; /* True if this table should not be reordered */ - WhereCost sCost; /* Cost information from best[Virtual]Index() */ - ExprList *pOrderBy; /* ORDER BY clause for index to optimize */ - ExprList *pDist; /* DISTINCT clause for index to optimize */ - - doNotReorder = (pTabItem->jointype & (JT_LEFT|JT_CROSS))!=0; - if( j!=iFrom && doNotReorder ) break; - m = getMask(pMaskSet, pTabItem->iCursor); - if( (m & notReady)==0 ){ + + /* The optimal scan check only occurs if there are two or more tables + ** available to be reordered */ + if( iFrom==nTabList-1 ){ + ckOptimal = 0; /* Common case of just one table in the FROM clause */ + }else{ + ckOptimal = -1; + for(j=iFrom, sWBI.pSrc=&pTabList->a[j]; j<nTabList; j++, sWBI.pSrc++){ + m = getMask(pMaskSet, sWBI.pSrc->iCursor); + if( (m & sWBI.notValid)==0 ){ if( j==iFrom ) iFrom++; continue; } - mask = (isOptimal ? m : notReady); - pOrderBy = ((i==0 && ppOrderBy )?*ppOrderBy:0); - pDist = (i==0 ? pDistinct : 0); - if( pTabItem->pIndex==0 ) nUnconstrained++; + if( j>iFrom && (sWBI.pSrc->jointype & (JT_LEFT|JT_CROSS))!=0 ) break; + if( ++ckOptimal ) break; + if( (sWBI.pSrc->jointype & JT_LEFT)!=0 ) break; + } + } + assert( ckOptimal==0 || ckOptimal==1 ); + + for(isOptimal=ckOptimal; isOptimal>=0 && bestJ<0; isOptimal--){ + for(j=iFrom, sWBI.pSrc=&pTabList->a[j]; j<nTabList; j++, sWBI.pSrc++){ + if( j>iFrom && (sWBI.pSrc->jointype & (JT_LEFT|JT_CROSS))!=0 ){ + /* This break and one like it in the ckOptimal computation loop + ** above prevent table reordering across LEFT and CROSS JOINs. + ** The LEFT JOIN case is necessary for correctness. The prohibition + ** against reordering across a CROSS JOIN is an SQLite feature that + ** allows the developer to control table reordering */ + break; + } + m = getMask(pMaskSet, sWBI.pSrc->iCursor); + if( (m & sWBI.notValid)==0 ){ + assert( j>iFrom ); + continue; + } + sWBI.notReady = (isOptimal ? m : sWBI.notValid); + if( sWBI.pSrc->pIndex==0 ) nUnconstrained++; - WHERETRACE(("=== trying table %d with isOptimal=%d ===\n", - j, isOptimal)); - assert( pTabItem->pTab ); + WHERETRACE((" === trying table %d (%s) with isOptimal=%d ===\n", + j, sWBI.pSrc->pTab->zName, isOptimal)); + assert( sWBI.pSrc->pTab ); #ifndef SQLITE_OMIT_VIRTUALTABLE - if( IsVirtual(pTabItem->pTab) ){ - sqlite3_index_info **pp = &pWInfo->a[j].pIdxInfo; - bestVirtualIndex(pParse, pWC, pTabItem, mask, notReady, pOrderBy, - &sCost, pp); + if( IsVirtual(sWBI.pSrc->pTab) ){ + sWBI.ppIdxInfo = &pWInfo->a[j].pIdxInfo; + bestVirtualIndex(&sWBI); }else #endif { - bestBtreeIndex(pParse, pWC, pTabItem, mask, notReady, pOrderBy, - pDist, &sCost); + bestBtreeIndex(&sWBI); } - assert( isOptimal || (sCost.used¬Ready)==0 ); + assert( isOptimal || (sWBI.cost.used&sWBI.notValid)==0 ); /* If an INDEXED BY clause is present, then the plan must use that ** index if it uses any index at all */ - assert( pTabItem->pIndex==0 - || (sCost.plan.wsFlags & WHERE_NOT_FULLSCAN)==0 - || sCost.plan.u.pIdx==pTabItem->pIndex ); + assert( sWBI.pSrc->pIndex==0 + || (sWBI.cost.plan.wsFlags & WHERE_NOT_FULLSCAN)==0 + || sWBI.cost.plan.u.pIdx==sWBI.pSrc->pIndex ); - if( isOptimal && (sCost.plan.wsFlags & WHERE_NOT_FULLSCAN)==0 ){ + if( isOptimal && (sWBI.cost.plan.wsFlags & WHERE_NOT_FULLSCAN)==0 ){ notIndexed |= m; } + if( isOptimal ){ + pWInfo->a[j].rOptCost = sWBI.cost.rCost; + }else if( ckOptimal ){ + /* If two or more tables have nearly the same outer loop cost, but + ** very different inner loop (optimal) cost, we want to choose + ** for the outer loop that table which benefits the least from + ** being in the inner loop. The following code scales the + ** outer loop cost estimate to accomplish that. */ + WHERETRACE((" scaling cost from %.1f to %.1f\n", + sWBI.cost.rCost, + sWBI.cost.rCost/pWInfo->a[j].rOptCost)); + sWBI.cost.rCost /= pWInfo->a[j].rOptCost; + } /* Conditions under which this table becomes the best so far: ** ** (1) The table must not depend on other tables that have not - ** yet run. + ** yet run. (In other words, it must not depend on tables + ** in inner loops.) ** - ** (2) A full-table-scan plan cannot supercede indexed plan unless - ** the full-table-scan is an "optimal" plan as defined above. + ** (2) (This rule was removed on 2012-11-09. The scaling of the + ** cost using the optimal scan cost made this rule obsolete.) ** ** (3) All tables have an INDEXED BY clause or this table lacks an ** INDEXED BY clause or this table uses the specific @@ -4946,43 +5363,47 @@ WhereInfo *sqlite3WhereBegin( ** The NEVER() comes about because rule (2) above prevents ** An indexable full-table-scan from reaching rule (3). ** - ** (4) The plan cost must be lower than prior plans or else the - ** cost must be the same and the number of rows must be lower. + ** (4) The plan cost must be lower than prior plans, where "cost" + ** is defined by the compareCost() function above. */ - if( (sCost.used¬Ready)==0 /* (1) */ - && (bestJ<0 || (notIndexed&m)!=0 /* (2) */ - || (bestPlan.plan.wsFlags & WHERE_NOT_FULLSCAN)==0 - || (sCost.plan.wsFlags & WHERE_NOT_FULLSCAN)!=0) - && (nUnconstrained==0 || pTabItem->pIndex==0 /* (3) */ - || NEVER((sCost.plan.wsFlags & WHERE_NOT_FULLSCAN)!=0)) - && (bestJ<0 || sCost.rCost<bestPlan.rCost /* (4) */ - || (sCost.rCost<=bestPlan.rCost - && sCost.plan.nRow<bestPlan.plan.nRow)) + if( (sWBI.cost.used&sWBI.notValid)==0 /* (1) */ + && (nUnconstrained==0 || sWBI.pSrc->pIndex==0 /* (3) */ + || NEVER((sWBI.cost.plan.wsFlags & WHERE_NOT_FULLSCAN)!=0)) + && (bestJ<0 || compareCost(&sWBI.cost, &bestPlan)) /* (4) */ ){ - WHERETRACE(("=== table %d is best so far" - " with cost=%g and nRow=%g\n", - j, sCost.rCost, sCost.plan.nRow)); - bestPlan = sCost; + WHERETRACE((" === table %d (%s) is best so far\n" + " cost=%.1f, nRow=%.1f, nOBSat=%d, wsFlags=%08x\n", + j, sWBI.pSrc->pTab->zName, + sWBI.cost.rCost, sWBI.cost.plan.nRow, + sWBI.cost.plan.nOBSat, sWBI.cost.plan.wsFlags)); + bestPlan = sWBI.cost; bestJ = j; } - if( doNotReorder ) break; + + /* In a join like "w JOIN x LEFT JOIN y JOIN z" make sure that + ** table y (and not table z) is always the next inner loop inside + ** of table x. */ + if( (sWBI.pSrc->jointype & JT_LEFT)!=0 ) break; } } assert( bestJ>=0 ); - assert( notReady & getMask(pMaskSet, pTabList->a[bestJ].iCursor) ); - WHERETRACE(("*** Optimizer selects table %d for loop %d" - " with cost=%g and nRow=%g\n", - bestJ, pLevel-pWInfo->a, bestPlan.rCost, bestPlan.plan.nRow)); - /* The ALWAYS() that follows was added to hush up clang scan-build */ - if( (bestPlan.plan.wsFlags & WHERE_ORDERBY)!=0 && ALWAYS(ppOrderBy) ){ - *ppOrderBy = 0; - } + assert( sWBI.notValid & getMask(pMaskSet, pTabList->a[bestJ].iCursor) ); + assert( bestJ==iFrom || (pTabList->a[iFrom].jointype & JT_LEFT)==0 ); + testcase( bestJ>iFrom && (pTabList->a[iFrom].jointype & JT_CROSS)!=0 ); + testcase( bestJ>iFrom && bestJ<nTabList-1 + && (pTabList->a[bestJ+1].jointype & JT_LEFT)!=0 ); + WHERETRACE(("*** Optimizer selects table %d (%s) for loop %d with:\n" + " cost=%.1f, nRow=%.1f, nOBSat=%d, wsFlags=0x%08x\n", + bestJ, pTabList->a[bestJ].pTab->zName, + pLevel-pWInfo->a, bestPlan.rCost, bestPlan.plan.nRow, + bestPlan.plan.nOBSat, bestPlan.plan.wsFlags)); if( (bestPlan.plan.wsFlags & WHERE_DISTINCT)!=0 ){ assert( pWInfo->eDistinct==0 ); pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; } andFlags &= bestPlan.plan.wsFlags; pLevel->plan = bestPlan.plan; + pLevel->iTabCur = pTabList->a[bestJ].iCursor; testcase( bestPlan.plan.wsFlags & WHERE_INDEXED ); testcase( bestPlan.plan.wsFlags & WHERE_TEMP_INDEX ); if( bestPlan.plan.wsFlags & (WHERE_INDEXED|WHERE_TEMP_INDEX) ){ @@ -4996,7 +5417,7 @@ WhereInfo *sqlite3WhereBegin( }else{ pLevel->iIdxCur = -1; } - notReady &= ~getMask(pMaskSet, pTabList->a[bestJ].iCursor); + sWBI.notValid &= ~getMask(pMaskSet, pTabList->a[bestJ].iCursor); pLevel->iFrom = (u8)bestJ; if( bestPlan.plan.nRow>=(double)1 ){ pParse->nQueryLoop *= bestPlan.plan.nRow; @@ -5024,12 +5445,19 @@ WhereInfo *sqlite3WhereBegin( if( pParse->nErr || db->mallocFailed ){ goto whereBeginError; } + if( nTabList ){ + pLevel--; + pWInfo->nOBSat = pLevel->plan.nOBSat; + }else{ + pWInfo->nOBSat = 0; + } /* If the total query only selects a single row, then the ORDER BY ** clause is irrelevant. */ - if( (andFlags & WHERE_UNIQUE)!=0 && ppOrderBy ){ - *ppOrderBy = 0; + if( (andFlags & WHERE_UNIQUE)!=0 && pOrderBy ){ + assert( nTabList==0 || (pLevel->plan.wsFlags & WHERE_ALL_UNIQUE)!=0 ); + pWInfo->nOBSat = pOrderBy->nExpr; } /* If the caller is an UPDATE or DELETE statement that is requesting @@ -5049,13 +5477,13 @@ WhereInfo *sqlite3WhereBegin( sqlite3CodeVerifySchema(pParse, -1); /* Insert the cookie verifier Goto */ notReady = ~(Bitmask)0; pWInfo->nRowOut = (double)1; - for(i=0, pLevel=pWInfo->a; i<nTabList; i++, pLevel++){ + for(ii=0, pLevel=pWInfo->a; ii<nTabList; ii++, pLevel++){ Table *pTab; /* Table to open */ int iDb; /* Index of database containing table/index */ + struct SrcList_item *pTabItem; pTabItem = &pTabList->a[pLevel->iFrom]; pTab = pTabItem->pTab; - pLevel->iTabCur = pTabItem->iCursor; pWInfo->nRowOut *= pLevel->plan.nRow; iDb = sqlite3SchemaToIndex(db, pTab->pSchema); if( (pTab->tabFlags & TF_Ephemeral)!=0 || pTab->pSelect ){ @@ -5066,6 +5494,8 @@ WhereInfo *sqlite3WhereBegin( const char *pVTab = (const char *)sqlite3GetVTable(db, pTab); int iCur = pTabItem->iCursor; sqlite3VdbeAddOp4(v, OP_VOpen, iCur, 0, 0, pVTab, P4_VTAB); + }else if( IsVirtual(pTab) ){ + /* noop */ }else #endif if( (pLevel->plan.wsFlags & WHERE_IDX_ONLY)==0 @@ -5087,7 +5517,7 @@ WhereInfo *sqlite3WhereBegin( } #ifndef SQLITE_OMIT_AUTOMATIC_INDEX if( (pLevel->plan.wsFlags & WHERE_TEMP_INDEX)!=0 ){ - constructAutomaticIndex(pParse, pWC, pTabItem, notReady, pLevel); + constructAutomaticIndex(pParse, sWBI.pWC, pTabItem, notReady, pLevel); }else #endif if( (pLevel->plan.wsFlags & WHERE_INDEXED)!=0 ){ @@ -5101,7 +5531,7 @@ WhereInfo *sqlite3WhereBegin( VdbeComment((v, "%s", pIx->zName)); } sqlite3CodeVerifySchema(pParse, iDb); - notReady &= ~getMask(pWC->pMaskSet, pTabItem->iCursor); + notReady &= ~getMask(sWBI.pWC->pMaskSet, pTabItem->iCursor); } pWInfo->iTop = sqlite3VdbeCurrentAddr(v); if( db->mallocFailed ) goto whereBeginError; @@ -5111,10 +5541,10 @@ WhereInfo *sqlite3WhereBegin( ** program. */ notReady = ~(Bitmask)0; - for(i=0; i<nTabList; i++){ - pLevel = &pWInfo->a[i]; - explainOneScan(pParse, pTabList, pLevel, i, pLevel->iFrom, wctrlFlags); - notReady = codeOneLoopStart(pWInfo, i, wctrlFlags, notReady); + for(ii=0; ii<nTabList; ii++){ + pLevel = &pWInfo->a[ii]; + explainOneScan(pParse, pTabList, pLevel, ii, pLevel->iFrom, wctrlFlags); + notReady = codeOneLoopStart(pWInfo, ii, wctrlFlags, notReady); pWInfo->iContinue = pLevel->addrCont; } @@ -5125,16 +5555,20 @@ WhereInfo *sqlite3WhereBegin( ** the index is listed as "{}". If the primary key is used the ** index name is '*'. */ - for(i=0; i<nTabList; i++){ + for(ii=0; ii<nTabList; ii++){ char *z; int n; - pLevel = &pWInfo->a[i]; + int w; + struct SrcList_item *pTabItem; + + pLevel = &pWInfo->a[ii]; + w = pLevel->plan.wsFlags; pTabItem = &pTabList->a[pLevel->iFrom]; z = pTabItem->zAlias; if( z==0 ) z = pTabItem->pTab->zName; n = sqlite3Strlen30(z); if( n+nQPlan < sizeof(sqlite3_query_plan)-10 ){ - if( pLevel->plan.wsFlags & WHERE_IDX_ONLY ){ + if( (w & WHERE_IDX_ONLY)!=0 && (w & WHERE_COVER_SCAN)==0 ){ memcpy(&sqlite3_query_plan[nQPlan], "{}", 2); nQPlan += 2; }else{ @@ -5143,12 +5577,12 @@ WhereInfo *sqlite3WhereBegin( } sqlite3_query_plan[nQPlan++] = ' '; } - testcase( pLevel->plan.wsFlags & WHERE_ROWID_EQ ); - testcase( pLevel->plan.wsFlags & WHERE_ROWID_RANGE ); - if( pLevel->plan.wsFlags & (WHERE_ROWID_EQ|WHERE_ROWID_RANGE) ){ + testcase( w & WHERE_ROWID_EQ ); + testcase( w & WHERE_ROWID_RANGE ); + if( w & (WHERE_ROWID_EQ|WHERE_ROWID_RANGE) ){ memcpy(&sqlite3_query_plan[nQPlan], "* ", 2); nQPlan += 2; - }else if( (pLevel->plan.wsFlags & WHERE_INDEXED)!=0 ){ + }else if( (w & WHERE_INDEXED)!=0 && (w & WHERE_COVER_SCAN)==0 ){ n = sqlite3Strlen30(pLevel->plan.u.pIdx->zName); if( n+nQPlan < sizeof(sqlite3_query_plan)-2 ){ memcpy(&sqlite3_query_plan[nQPlan], pLevel->plan.u.pIdx->zName, n); @@ -5209,7 +5643,7 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ sqlite3VdbeResolveLabel(v, pLevel->addrNxt); for(j=pLevel->u.in.nIn, pIn=&pLevel->u.in.aInLoop[j-1]; j>0; j--, pIn--){ sqlite3VdbeJumpHere(v, pIn->addrInTop+1); - sqlite3VdbeAddOp2(v, OP_Next, pIn->iCur, pIn->addrInTop); + sqlite3VdbeAddOp2(v, pIn->eEndLoopOp, pIn->iCur, pIn->addrInTop); sqlite3VdbeJumpHere(v, pIn->addrInTop-1); } sqlite3DbFree(db, pLevel->u.in.aInLoop); |