diff options
| author | Hans-Christoph Steiner <hans@eds.org> | 2012-09-20 18:34:38 -0400 |
|---|---|---|
| committer | Hans-Christoph Steiner <hans@eds.org> | 2012-09-20 18:34:38 -0400 |
| commit | 487e15dc239ccdb3344d1c99ce120e872bab4a74 (patch) | |
| tree | c986d492f6092ca7b4401d91515f74daed17fae2 /src | |
| parent | 7bb481fda9ecb134804b49c2ce77ca28f7eea583 (diff) | |
Imported Upstream version 2.0.6
Diffstat (limited to 'src')
87 files changed, 9258 insertions, 2877 deletions
diff --git a/src/alter.c b/src/alter.c index fb6d89d..7f56ce7 100644 --- a/src/alter.c +++ b/src/alter.c @@ -530,7 +530,7 @@ void sqlite3AlterRenameTable( "WHEN name LIKE 'sqlite_autoindex%%' AND type='index' THEN " "'sqlite_autoindex_' || %Q || substr(name,%d+18) " "ELSE name END " - "WHERE tbl_name=%Q AND " + "WHERE tbl_name=%Q COLLATE nocase AND " "(type='table' OR type='index' OR type='trigger');", zDb, SCHEMA_TABLE(iDb), zName, zName, zName, #ifndef SQLITE_OMIT_TRIGGER diff --git a/src/analyze.c b/src/analyze.c index b6a987a..4dfc331 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -529,6 +529,7 @@ static void analyzeOneTable( sqlite3VdbeAddOp2(v, OP_Integer, 0, regNumEq); sqlite3VdbeAddOp2(v, OP_Integer, 0, regNumLt); sqlite3VdbeAddOp2(v, OP_Integer, -1, regNumDLt); + sqlite3VdbeAddOp3(v, OP_Null, 0, regSample, regAccum); sqlite3VdbeAddOp4(v, OP_Function, 1, regCount, regAccum, (char*)&stat3InitFuncdef, P4_FUNCDEF); sqlite3VdbeChangeP5(v, 2); @@ -931,6 +932,7 @@ static int loadStat3(sqlite3 *db, const char *zDb){ int eType; /* Datatype of a sample */ IndexSample *pSample; /* A slot in pIdx->aSample[] */ + assert( db->lookaside.bEnabled==0 ); if( !sqlite3FindTable(db, "sqlite_stat3", zDb) ){ return SQLITE_OK; } @@ -957,7 +959,7 @@ static int loadStat3(sqlite3 *db, const char *zDb){ if( pIdx==0 ) continue; assert( pIdx->nSample==0 ); pIdx->nSample = nSample; - pIdx->aSample = sqlite3MallocZero( nSample*sizeof(IndexSample) ); + pIdx->aSample = sqlite3DbMallocZero(db, nSample*sizeof(IndexSample)); pIdx->avgEq = pIdx->aiRowEst[1]; if( pIdx->aSample==0 ){ db->mallocFailed = 1; @@ -1030,7 +1032,7 @@ static int loadStat3(sqlite3 *db, const char *zDb){ if( n < 1){ pSample->u.z = 0; }else{ - pSample->u.z = sqlite3Malloc(n); + pSample->u.z = sqlite3DbMallocRaw(db, n); if( pSample->u.z==0 ){ db->mallocFailed = 1; sqlite3_finalize(pStmt); @@ -1106,7 +1108,10 @@ int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ /* Load the statistics from the sqlite_stat3 table. */ #ifdef SQLITE_ENABLE_STAT3 if( rc==SQLITE_OK ){ + int lookasideEnabled = db->lookaside.bEnabled; + db->lookaside.bEnabled = 0; rc = loadStat3(db, sInfo.zDatabase); + db->lookaside.bEnabled = lookasideEnabled; } #endif diff --git a/src/backup.c b/src/backup.c index bdf96bd..7a4047f 100644 --- a/src/backup.c +++ b/src/backup.c @@ -568,7 +568,7 @@ int sqlite3_backup_finish(sqlite3_backup *p){ } /* If a transaction is still open on the Btree, roll it back. */ - sqlite3BtreeRollback(p->pDest); + sqlite3BtreeRollback(p->pDest, SQLITE_OK); /* Set the error code of the destination database handle. */ rc = (p->rc==SQLITE_DONE) ? SQLITE_OK : p->rc; @@ -678,7 +678,9 @@ int sqlite3BtreeCopyFile(Btree *pTo, Btree *pFrom){ pFd = sqlite3PagerFile(sqlite3BtreePager(pTo)); if( pFd->pMethods ){ i64 nByte = sqlite3BtreeGetPageSize(pFrom)*(i64)sqlite3BtreeLastPage(pFrom); - sqlite3OsFileControl(pFd, SQLITE_FCNTL_OVERWRITE, &nByte); + rc = sqlite3OsFileControl(pFd, SQLITE_FCNTL_OVERWRITE, &nByte); + if( rc==SQLITE_NOTFOUND ) rc = SQLITE_OK; + if( rc ) goto copy_finished; } /* Set up an sqlite3_backup object. sqlite3_backup.pDestDb must be set @@ -703,12 +705,13 @@ int sqlite3BtreeCopyFile(Btree *pTo, Btree *pFrom){ assert( b.rc!=SQLITE_OK ); rc = sqlite3_backup_finish(&b); if( rc==SQLITE_OK ){ - pTo->pBt->pageSizeFixed = 0; + pTo->pBt->btsFlags &= ~BTS_PAGESIZE_FIXED; }else{ sqlite3PagerClearCache(sqlite3BtreePager(b.pDest)); } assert( sqlite3BtreeIsInTrans(pTo)==0 ); +copy_finished: sqlite3BtreeLeave(pFrom); sqlite3BtreeLeave(pTo); return rc; diff --git a/src/btree.c b/src/btree.c index d64e172..2876526 100644 --- a/src/btree.c +++ b/src/btree.c @@ -243,7 +243,7 @@ static int querySharedCacheTableLock(Btree *p, Pgno iTab, u8 eLock){ /* If some other connection is holding an exclusive lock, the ** requested lock may not be obtained. */ - if( pBt->pWriter!=p && pBt->isExclusive ){ + if( pBt->pWriter!=p && (pBt->btsFlags & BTS_EXCLUSIVE)!=0 ){ sqlite3ConnectionBlocked(p->db, pBt->pWriter->db); return SQLITE_LOCKED_SHAREDCACHE; } @@ -264,7 +264,7 @@ static int querySharedCacheTableLock(Btree *p, Pgno iTab, u8 eLock){ sqlite3ConnectionBlocked(p->db, pIter->pBtree->db); if( eLock==WRITE_LOCK ){ assert( p==pBt->pWriter ); - pBt->isPending = 1; + pBt->btsFlags |= BTS_PENDING; } return SQLITE_LOCKED_SHAREDCACHE; } @@ -352,7 +352,7 @@ static int setSharedCacheTableLock(Btree *p, Pgno iTable, u8 eLock){ ** the setSharedCacheTableLock() procedure) held by Btree object p. ** ** This function assumes that Btree p has an open read or write -** transaction. If it does not, then the BtShared.isPending variable +** transaction. If it does not, then the BTS_PENDING flag ** may be incorrectly cleared. */ static void clearAllSharedCacheTableLocks(Btree *p){ @@ -365,7 +365,7 @@ static void clearAllSharedCacheTableLocks(Btree *p){ while( *ppIter ){ BtLock *pLock = *ppIter; - assert( pBt->isExclusive==0 || pBt->pWriter==pLock->pBtree ); + assert( (pBt->btsFlags & BTS_EXCLUSIVE)==0 || pBt->pWriter==pLock->pBtree ); assert( pLock->pBtree->inTrans>=pLock->eLock ); if( pLock->pBtree==p ){ *ppIter = pLock->pNext; @@ -378,22 +378,21 @@ static void clearAllSharedCacheTableLocks(Btree *p){ } } - assert( pBt->isPending==0 || pBt->pWriter ); + assert( (pBt->btsFlags & BTS_PENDING)==0 || pBt->pWriter ); if( pBt->pWriter==p ){ pBt->pWriter = 0; - pBt->isExclusive = 0; - pBt->isPending = 0; + pBt->btsFlags &= ~(BTS_EXCLUSIVE|BTS_PENDING); }else if( pBt->nTransaction==2 ){ /* This function is called when Btree p is concluding its ** transaction. If there currently exists a writer, and p is not ** that writer, then the number of locks held by connections other ** than the writer must be about to drop to zero. In this case - ** set the isPending flag to 0. + ** set the BTS_PENDING flag to 0. ** - ** If there is not currently a writer, then BtShared.isPending must + ** If there is not currently a writer, then BTS_PENDING must ** be zero already. So this next line is harmless in that case. */ - pBt->isPending = 0; + pBt->btsFlags &= ~BTS_PENDING; } } @@ -405,8 +404,7 @@ static void downgradeAllSharedCacheTableLocks(Btree *p){ if( pBt->pWriter==p ){ BtLock *pLock; pBt->pWriter = 0; - pBt->isExclusive = 0; - pBt->isPending = 0; + pBt->btsFlags &= ~(BTS_EXCLUSIVE|BTS_PENDING); for(pLock=pBt->pLock; pLock; pLock=pLock->pNext){ assert( pLock->eLock==READ_LOCK || pLock->pBtree==p ); pLock->eLock = READ_LOCK; @@ -859,7 +857,7 @@ static int ptrmapGet(BtShared *pBt, Pgno key, u8 *pEType, Pgno *pPgno){ ** This routine works only for pages that do not contain overflow cells. */ #define findCell(P,I) \ - ((P)->aData + ((P)->maskPage & get2byte(&(P)->aData[(P)->cellOffset+2*(I)]))) + ((P)->aData + ((P)->maskPage & get2byte(&(P)->aCellIdx[2*(I)]))) #define findCellv2(D,M,O,I) (D+(M&get2byte(D+(O+2*(I))))) @@ -872,12 +870,10 @@ static u8 *findOverflowCell(MemPage *pPage, int iCell){ assert( sqlite3_mutex_held(pPage->pBt->mutex) ); for(i=pPage->nOverflow-1; i>=0; i--){ int k; - struct _OvflCell *pOvfl; - pOvfl = &pPage->aOvfl[i]; - k = pOvfl->idx; + k = pPage->aiOvfl[i]; if( k<=iCell ){ if( k==iCell ){ - return pOvfl->pCell; + return pPage->apOvfl[i]; } iCell--; } @@ -1264,7 +1260,7 @@ static int freeSpace(MemPage *pPage, int start, int size){ assert( sqlite3_mutex_held(pPage->pBt->mutex) ); assert( size>=0 ); /* Minimum cell size is 4 */ - if( pPage->pBt->secureDelete ){ + if( pPage->pBt->btsFlags & BTS_SECURE_DELETE ){ /* Overwrite deleted information with zeros when the secure_delete ** option is enabled */ memset(&data[start], 0, size); @@ -1367,6 +1363,7 @@ static int decodeFlags(MemPage *pPage, int flagByte){ }else{ return SQLITE_CORRUPT_BKPT; } + pPage->max1bytePayload = pBt->max1bytePayload; return SQLITE_OK; } @@ -1409,6 +1406,8 @@ static int btreeInitPage(MemPage *pPage){ pPage->nOverflow = 0; usableSize = pBt->usableSize; pPage->cellOffset = cellOffset = hdr + 12 - 4*pPage->leaf; + pPage->aDataEnd = &data[usableSize]; + pPage->aCellIdx = &data[cellOffset]; top = get2byteNotZero(&data[hdr+5]); pPage->nCell = get2byte(&data[hdr+3]); if( pPage->nCell>MX_CELL(pBt) ){ @@ -1500,7 +1499,7 @@ static void zeroPage(MemPage *pPage, int flags){ assert( sqlite3PagerGetData(pPage->pDbPage) == data ); assert( sqlite3PagerIswriteable(pPage->pDbPage) ); assert( sqlite3_mutex_held(pBt->mutex) ); - if( pBt->secureDelete ){ + if( pBt->btsFlags & BTS_SECURE_DELETE ){ memset(&data[hdr], 0, pBt->usableSize - hdr); } data[hdr] = (char)flags; @@ -1512,6 +1511,8 @@ static void zeroPage(MemPage *pPage, int flags){ decodeFlags(pPage, flags); pPage->hdrOffset = hdr; pPage->cellOffset = first; + pPage->aDataEnd = &data[pBt->usableSize]; + pPage->aCellIdx = &data[first]; pPage->nOverflow = 0; assert( pBt->pageSize>=512 && pBt->pageSize<=65536 ); pPage->maskPage = (u16)(pBt->pageSize - 1); @@ -1686,11 +1687,8 @@ static int btreeInvokeBusyHandler(void *pArg){ ** If zFilename is ":memory:" then an in-memory database is created ** that is automatically destroyed when it is closed. ** -** The "flags" parameter is a bitmask that might contain bits -** BTREE_OMIT_JOURNAL and/or BTREE_NO_READLOCK. The BTREE_NO_READLOCK -** bit is also set if the SQLITE_NoReadlock flags is set in db->flags. -** These flags are passed through into sqlite3PagerOpen() and must -** be the same values as PAGER_OMIT_JOURNAL and PAGER_NO_READLOCK. +** The "flags" parameter is a bitmask that might contain bits like +** BTREE_OMIT_JOURNAL and/or BTREE_MEMORY. ** ** If the database is already opened in the same database connection ** and we are in shared cache mode, then the open will fail with an @@ -1737,9 +1735,6 @@ int sqlite3BtreeOpen( /* A BTREE_SINGLE database is always a temporary and/or ephemeral */ assert( (flags & BTREE_SINGLE)==0 || isTempDb ); - if( db->flags & SQLITE_NoReadlock ){ - flags |= BTREE_NO_READLOCK; - } if( isMemdb ){ flags |= BTREE_MEMORY; } @@ -1772,7 +1767,12 @@ int sqlite3BtreeOpen( sqlite3_free(p); return SQLITE_NOMEM; } - sqlite3OsFullPathname(pVfs, zFilename, nFullPathname, zFullPathname); + rc = sqlite3OsFullPathname(pVfs, zFilename, nFullPathname, zFullPathname); + if( rc ){ + sqlite3_free(zFullPathname); + sqlite3_free(p); + return rc; + } #if SQLITE_THREADSAFE mutexOpen = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_OPEN); sqlite3_mutex_enter(mutexOpen); @@ -1846,9 +1846,9 @@ int sqlite3BtreeOpen( pBt->pCursor = 0; pBt->pPage1 = 0; - pBt->readOnly = sqlite3PagerIsreadonly(pBt->pPager); + if( sqlite3PagerIsreadonly(pBt->pPager) ) pBt->btsFlags |= BTS_READ_ONLY; #ifdef SQLITE_SECURE_DELETE - pBt->secureDelete = 1; + pBt->btsFlags |= BTS_SECURE_DELETE; #endif pBt->pageSize = (zDbHeader[16]<<8) | (zDbHeader[17]<<16); if( pBt->pageSize<512 || pBt->pageSize>SQLITE_MAX_PAGE_SIZE @@ -1869,7 +1869,7 @@ int sqlite3BtreeOpen( nReserve = 0; }else{ nReserve = zDbHeader[20]; - pBt->pageSizeFixed = 1; + pBt->btsFlags |= BTS_PAGESIZE_FIXED; #ifndef SQLITE_OMIT_AUTOVACUUM pBt->autoVacuum = (get4byte(&zDbHeader[36 + 4*4])?1:0); pBt->incrVacuum = (get4byte(&zDbHeader[36 + 7*4])?1:0); @@ -2041,7 +2041,7 @@ int sqlite3BtreeClose(Btree *p){ ** The call to sqlite3BtreeRollback() drops any table-locks held by ** this handle. */ - sqlite3BtreeRollback(p); + sqlite3BtreeRollback(p, SQLITE_OK); sqlite3BtreeLeave(p); /* If there are still other outstanding references to the shared-btree @@ -2157,7 +2157,7 @@ int sqlite3BtreeSyncDisabled(Btree *p){ ** If parameter nReserve is less than zero, then the number of reserved ** bytes per page is left unchanged. ** -** If the iFix!=0 then the pageSizeFixed flag is set so that the page size +** If the iFix!=0 then the BTS_PAGESIZE_FIXED flag is set so that the page size ** and autovacuum mode can no longer be changed. */ int sqlite3BtreeSetPageSize(Btree *p, int pageSize, int nReserve, int iFix){ @@ -2165,7 +2165,7 @@ int sqlite3BtreeSetPageSize(Btree *p, int pageSize, int nReserve, int iFix){ BtShared *pBt = p->pBt; assert( nReserve>=-1 && nReserve<=255 ); sqlite3BtreeEnter(p); - if( pBt->pageSizeFixed ){ + if( pBt->btsFlags & BTS_PAGESIZE_FIXED ){ sqlite3BtreeLeave(p); return SQLITE_READONLY; } @@ -2182,7 +2182,7 @@ int sqlite3BtreeSetPageSize(Btree *p, int pageSize, int nReserve, int iFix){ } rc = sqlite3PagerSetPagesize(pBt->pPager, &pBt->pageSize, nReserve); pBt->usableSize = pBt->pageSize - (u16)nReserve; - if( iFix ) pBt->pageSizeFixed = 1; + if( iFix ) pBt->btsFlags |= BTS_PAGESIZE_FIXED; sqlite3BtreeLeave(p); return rc; } @@ -2222,8 +2222,8 @@ int sqlite3BtreeMaxPageCount(Btree *p, int mxPage){ } /* -** Set the secureDelete flag if newFlag is 0 or 1. If newFlag is -1, -** then make no changes. Always return the value of the secureDelete +** Set the BTS_SECURE_DELETE flag if newFlag is 0 or 1. If newFlag is -1, +** then make no changes. Always return the value of the BTS_SECURE_DELETE ** setting after the change. */ int sqlite3BtreeSecureDelete(Btree *p, int newFlag){ @@ -2231,9 +2231,10 @@ int sqlite3BtreeSecureDelete(Btree *p, int newFlag){ if( p==0 ) return 0; sqlite3BtreeEnter(p); if( newFlag>=0 ){ - p->pBt->secureDelete = (newFlag!=0) ? 1 : 0; + p->pBt->btsFlags &= ~BTS_SECURE_DELETE; + if( newFlag ) p->pBt->btsFlags |= BTS_SECURE_DELETE; } - b = p->pBt->secureDelete; + b = (p->pBt->btsFlags & BTS_SECURE_DELETE)!=0; sqlite3BtreeLeave(p); return b; } @@ -2254,7 +2255,7 @@ int sqlite3BtreeSetAutoVacuum(Btree *p, int autoVacuum){ u8 av = (u8)autoVacuum; sqlite3BtreeEnter(p); - if( pBt->pageSizeFixed && (av ?1:0)!=pBt->autoVacuum ){ + if( (pBt->btsFlags & BTS_PAGESIZE_FIXED)!=0 && (av ?1:0)!=pBt->autoVacuum ){ rc = SQLITE_READONLY; }else{ pBt->autoVacuum = av ?1:0; @@ -2328,14 +2329,14 @@ static int lockBtree(BtShared *pBt){ #ifdef SQLITE_OMIT_WAL if( page1[18]>1 ){ - pBt->readOnly = 1; + pBt->btsFlags |= BTS_READ_ONLY; } if( page1[19]>1 ){ goto page1_init_failed; } #else if( page1[18]>2 ){ - pBt->readOnly = 1; + pBt->btsFlags |= BTS_READ_ONLY; } if( page1[19]>2 ){ goto page1_init_failed; @@ -2349,7 +2350,7 @@ static int lockBtree(BtShared *pBt){ ** may not be the latest version - there may be a newer one in the log ** file. */ - if( page1[19]==2 && pBt->doNotUseWAL==0 ){ + if( page1[19]==2 && (pBt->btsFlags & BTS_NO_WAL)==0 ){ int isOpen = 0; rc = sqlite3PagerOpenWal(pBt->pPager, &isOpen); if( rc!=SQLITE_OK ){ @@ -2426,6 +2427,11 @@ static int lockBtree(BtShared *pBt){ pBt->minLocal = (u16)((pBt->usableSize-12)*32/255 - 23); pBt->maxLeaf = (u16)(pBt->usableSize - 35); pBt->minLeaf = (u16)((pBt->usableSize-12)*32/255 - 23); + if( pBt->maxLocal>127 ){ + pBt->max1bytePayload = 127; + }else{ + pBt->max1bytePayload = (u8)pBt->maxLocal; + } assert( pBt->maxLeaf + 23 <= MX_CELL_SIZE(pBt) ); pBt->pPage1 = pPage1; pBt->nPage = nPage; @@ -2489,7 +2495,7 @@ static int newDatabase(BtShared *pBt){ data[23] = 32; memset(&data[24], 0, 100-24); zeroPage(pP1, PTF_INTKEY|PTF_LEAF|PTF_LEAFDATA ); - pBt->pageSizeFixed = 1; + pBt->btsFlags |= BTS_PAGESIZE_FIXED; #ifndef SQLITE_OMIT_AUTOVACUUM assert( pBt->autoVacuum==1 || pBt->autoVacuum==0 ); assert( pBt->incrVacuum==1 || pBt->incrVacuum==0 ); @@ -2553,7 +2559,7 @@ int sqlite3BtreeBeginTrans(Btree *p, int wrflag){ } /* Write transactions are not possible on a read-only database */ - if( pBt->readOnly && wrflag ){ + if( (pBt->btsFlags & BTS_READ_ONLY)!=0 && wrflag ){ rc = SQLITE_READONLY; goto trans_begun; } @@ -2563,7 +2569,9 @@ int sqlite3BtreeBeginTrans(Btree *p, int wrflag){ ** on this shared-btree structure and a second write transaction is ** requested, return SQLITE_LOCKED. */ - if( (wrflag && pBt->inTransaction==TRANS_WRITE) || pBt->isPending ){ + if( (wrflag && pBt->inTransaction==TRANS_WRITE) + || (pBt->btsFlags & BTS_PENDING)!=0 + ){ pBlock = pBt->pWriter->db; }else if( wrflag>1 ){ BtLock *pIter; @@ -2587,7 +2595,8 @@ int sqlite3BtreeBeginTrans(Btree *p, int wrflag){ rc = querySharedCacheTableLock(p, MASTER_ROOT, READ_LOCK); if( SQLITE_OK!=rc ) goto trans_begun; - pBt->initiallyEmpty = (u8)(pBt->nPage==0); + pBt->btsFlags &= ~BTS_INITIALLY_EMPTY; + if( pBt->nPage==0 ) pBt->btsFlags |= BTS_INITIALLY_EMPTY; do { /* Call lockBtree() until either pBt->pPage1 is populated or ** lockBtree() returns something other than SQLITE_OK. lockBtree() @@ -2599,7 +2608,7 @@ int sqlite3BtreeBeginTrans(Btree *p, int wrflag){ while( pBt->pPage1==0 && SQLITE_OK==(rc = lockBtree(pBt)) ); if( rc==SQLITE_OK && wrflag ){ - if( pBt->readOnly ){ + if( (pBt->btsFlags & BTS_READ_ONLY)!=0 ){ rc = SQLITE_READONLY; }else{ rc = sqlite3PagerBegin(pBt->pPager,wrflag>1,sqlite3TempInMemory(p->db)); @@ -2636,7 +2645,8 @@ int sqlite3BtreeBeginTrans(Btree *p, int wrflag){ #ifndef SQLITE_OMIT_SHARED_CACHE assert( !pBt->pWriter ); pBt->pWriter = p; - pBt->isExclusive = (u8)(wrflag>1); + pBt->btsFlags &= ~BTS_EXCLUSIVE; + if( wrflag>1 ) pBt->btsFlags |= BTS_EXCLUSIVE; #endif /* If the db-size header field is incorrect (as it may be if an old @@ -3269,6 +3279,7 @@ static int countWriteCursors(BtShared *pBt){ */ void sqlite3BtreeTripAllCursors(Btree *pBtree, int errCode){ BtCursor *p; + if( pBtree==0 ) return; sqlite3BtreeEnter(pBtree); for(p=pBtree->pBt->pCursor; p; p=p->pNext){ int i; @@ -3292,25 +3303,20 @@ void sqlite3BtreeTripAllCursors(Btree *pBtree, int errCode){ ** This will release the write lock on the database file. If there ** are no active cursors, it also releases the read lock. */ -int sqlite3BtreeRollback(Btree *p){ +int sqlite3BtreeRollback(Btree *p, int tripCode){ int rc; BtShared *pBt = p->pBt; MemPage *pPage1; sqlite3BtreeEnter(p); - rc = saveAllCursors(pBt, 0, 0); -#ifndef SQLITE_OMIT_SHARED_CACHE - if( rc!=SQLITE_OK ){ - /* This is a horrible situation. An IO or malloc() error occurred whilst - ** trying to save cursor positions. If this is an automatic rollback (as - ** the result of a constraint, malloc() failure or IO error) then - ** the cache may be internally inconsistent (not contain valid trees) so - ** we cannot simply return the error to the caller. Instead, abort - ** all queries that may be using any of the cursors that failed to save. - */ - sqlite3BtreeTripAllCursors(p, rc); + if( tripCode==SQLITE_OK ){ + rc = tripCode = saveAllCursors(pBt, 0, 0); + }else{ + rc = SQLITE_OK; + } + if( tripCode ){ + sqlite3BtreeTripAllCursors(p, tripCode); } -#endif btreeIntegrity(p); if( p->inTrans==TRANS_WRITE ){ @@ -3365,7 +3371,7 @@ int sqlite3BtreeBeginStmt(Btree *p, int iStatement){ BtShared *pBt = p->pBt; sqlite3BtreeEnter(p); assert( p->inTrans==TRANS_WRITE ); - assert( pBt->readOnly==0 ); + assert( (pBt->btsFlags & BTS_READ_ONLY)==0 ); assert( iStatement>0 ); assert( iStatement>p->db->nSavepoint ); assert( pBt->inTransaction==TRANS_WRITE ); @@ -3400,7 +3406,9 @@ int sqlite3BtreeSavepoint(Btree *p, int op, int iSavepoint){ sqlite3BtreeEnter(p); rc = sqlite3PagerSavepoint(pBt->pPager, op, iSavepoint); if( rc==SQLITE_OK ){ - if( iSavepoint<0 && pBt->initiallyEmpty ) pBt->nPage = 0; + if( iSavepoint<0 && (pBt->btsFlags & BTS_INITIALLY_EMPTY)!=0 ){ + pBt->nPage = 0; + } rc = newDatabase(pBt); pBt->nPage = get4byte(28 + pBt->pPage1->aData); @@ -3470,7 +3478,7 @@ static int btreeCursor( assert( wrFlag==0 || p->inTrans==TRANS_WRITE ); assert( pBt->pPage1 && pBt->pPage1->aData ); - if( NEVER(wrFlag && pBt->readOnly) ){ + if( NEVER(wrFlag && (pBt->btsFlags & BTS_READ_ONLY)!=0) ){ return SQLITE_READONLY; } if( iTable==1 && btreePagecount(pBt)==0 ){ @@ -3970,7 +3978,7 @@ static int accessPayload( u8 aSave[4]; u8 *aWrite = &pBuf[-4]; memcpy(aSave, aWrite, 4); - rc = sqlite3OsRead(fd, aWrite, a+4, pBt->pageSize * (nextPage-1)); + rc = sqlite3OsRead(fd, aWrite, a+4, (i64)pBt->pageSize*(nextPage-1)); nextPage = get4byte(aWrite); memcpy(aWrite, aSave, 4); }else @@ -4174,7 +4182,7 @@ static int moveToChild(BtCursor *pCur, u32 newPgno){ return SQLITE_OK; } -#ifndef NDEBUG +#if 0 /* ** Page pParent is an internal (non-leaf) tree page. This function ** asserts that page number iChild is the left-child if the iIdx'th @@ -4207,11 +4215,21 @@ static void moveToParent(BtCursor *pCur){ assert( pCur->eState==CURSOR_VALID ); assert( pCur->iPage>0 ); assert( pCur->apPage[pCur->iPage] ); + + /* UPDATE: It is actually possible for the condition tested by the assert + ** below to be untrue if the database file is corrupt. This can occur if + ** one cursor has modified page pParent while a reference to it is held + ** by a second cursor. Which can only happen if a single page is linked + ** into more than one b-tree structure in a corrupt database. */ +#if 0 assertParentIndex( pCur->apPage[pCur->iPage-1], pCur->aiIdx[pCur->iPage-1], pCur->apPage[pCur->iPage]->pgno ); +#endif + testcase( pCur->aiIdx[pCur->iPage-1] > pCur->apPage[pCur->iPage-1]->nCell ); + releasePage(pCur->apPage[pCur->iPage]); pCur->iPage--; pCur->info.nSize = 0; @@ -4550,16 +4568,21 @@ int sqlite3BtreeMovetoUnpacked( ** 2 bytes of the cell. */ int nCell = pCell[0]; - if( !(nCell & 0x80) && nCell<=pPage->maxLocal ){ + if( nCell<=pPage->max1bytePayload + /* && (pCell+nCell)<pPage->aDataEnd */ + ){ /* This branch runs if the record-size field of the cell is a ** single byte varint and the record fits entirely on the main ** b-tree page. */ + testcase( pCell+nCell+1==pPage->aDataEnd ); c = sqlite3VdbeRecordCompare(nCell, (void*)&pCell[1], pIdxKey); }else if( !(pCell[1] & 0x80) && (nCell = ((nCell&0x7f)<<7) + pCell[1])<=pPage->maxLocal + /* && (pCell+nCell+2)<=pPage->aDataEnd */ ){ /* The record-size field is a 2 byte varint and the record ** fits entirely on the main b-tree page. */ + testcase( pCell+nCell+2==pPage->aDataEnd ); c = sqlite3VdbeRecordCompare(nCell, (void*)&pCell[2], pIdxKey); }else{ /* The record flows over onto one or more overflow pages. In @@ -4676,7 +4699,13 @@ int sqlite3BtreeNext(BtCursor *pCur, int *pRes){ pPage = pCur->apPage[pCur->iPage]; idx = ++pCur->aiIdx[pCur->iPage]; assert( pPage->isInit ); - assert( idx<=pPage->nCell ); + + /* If the database file is corrupt, it is possible for the value of idx + ** to be invalid here. This can only occur if a second cursor modifies + ** the page while cursor pCur is holding a reference to it. Which can + ** only happen if the database is corrupt in such a way as to link the + ** page into more than one b-tree structure. */ + testcase( idx>pPage->nCell ); pCur->info.nSize = 0; pCur->validNKey = 0; @@ -5101,7 +5130,7 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ nFree = get4byte(&pPage1->aData[36]); put4byte(&pPage1->aData[36], nFree+1); - if( pBt->secureDelete ){ + if( pBt->btsFlags & BTS_SECURE_DELETE ){ /* If the secure_delete option is enabled, then ** always fully overwrite deleted information with zeros. */ @@ -5162,7 +5191,7 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ if( rc==SQLITE_OK ){ put4byte(&pTrunk->aData[4], nLeaf+1); put4byte(&pTrunk->aData[8+nLeaf*4], iPage); - if( pPage && !pBt->secureDelete ){ + if( pPage && (pBt->btsFlags & BTS_SECURE_DELETE)==0 ){ sqlite3PagerDontWrite(pPage->pDbPage); } rc = btreeSetHasContent(pBt, iPage); @@ -5454,7 +5483,7 @@ static void dropCell(MemPage *pPage, int idx, int sz, int *pRC){ assert( sqlite3PagerIswriteable(pPage->pDbPage) ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); data = pPage->aData; - ptr = &data[pPage->cellOffset + 2*idx]; + ptr = &pPage->aCellIdx[2*idx]; pc = get2byte(ptr); hdr = pPage->hdrOffset; testcase( pc==get2byte(&data[hdr+5]) ); @@ -5468,7 +5497,7 @@ static void dropCell(MemPage *pPage, int idx, int sz, int *pRC){ *pRC = rc; return; } - endPtr = &data[pPage->cellOffset + 2*pPage->nCell - 2]; + endPtr = &pPage->aCellIdx[2*pPage->nCell - 2]; assert( (SQLITE_PTR_TO_INT(ptr)&1)==0 ); /* ptr is always 2-byte aligned */ while( ptr<endPtr ){ *(u16*)ptr = *(u16*)&ptr[2]; @@ -5486,7 +5515,7 @@ static void dropCell(MemPage *pPage, int idx, int sz, int *pRC){ ** If the cell content will fit on the page, then put it there. If it ** will not fit, then make a copy of the cell content into pTemp if ** pTemp is not null. Regardless of pTemp, allocate a new entry -** in pPage->aOvfl[] and make it point to the cell content (either +** in pPage->apOvfl[] and make it point to the cell content (either ** in pTemp or the original pCell) and also record its index. ** Allocating a new entry in pPage->aCell[] implies that ** pPage->nOverflow is incremented. @@ -5520,7 +5549,8 @@ static void insertCell( assert( i>=0 && i<=pPage->nCell+pPage->nOverflow ); assert( pPage->nCell<=MX_CELL(pPage->pBt) && MX_CELL(pPage->pBt)<=10921 ); - assert( pPage->nOverflow<=ArraySize(pPage->aOvfl) ); + assert( pPage->nOverflow<=ArraySize(pPage->apOvfl) ); + assert( ArraySize(pPage->apOvfl)==ArraySize(pPage->aiOvfl) ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); /* The cell should normally be sized correctly. However, when moving a ** malformed cell from a leaf page to an interior page, if the cell size @@ -5537,9 +5567,9 @@ static void insertCell( put4byte(pCell, iChild); } j = pPage->nOverflow++; - assert( j<(int)(sizeof(pPage->aOvfl)/sizeof(pPage->aOvfl[0])) ); - pPage->aOvfl[j].pCell = pCell; - pPage->aOvfl[j].idx = (u16)i; + assert( j<(int)(sizeof(pPage->apOvfl)/sizeof(pPage->apOvfl[0])) ); + pPage->apOvfl[j] = pCell; + pPage->aiOvfl[j] = (u16)i; }else{ int rc = sqlite3PagerWrite(pPage->pDbPage); if( rc!=SQLITE_OK ){ @@ -5610,7 +5640,7 @@ static void assemblePage( assert( pPage->nCell==0 ); assert( get2byteNotZero(&data[hdr+5])==nUsable ); - pCellptr = &data[pPage->cellOffset + nCell*2]; + pCellptr = &pPage->aCellIdx[nCell*2]; cellbody = nUsable; for(i=nCell-1; i>=0; i--){ u16 sz = aSize[i]; @@ -5687,7 +5717,7 @@ static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){ if( rc==SQLITE_OK ){ u8 *pOut = &pSpace[4]; - u8 *pCell = pPage->aOvfl[0].pCell; + u8 *pCell = pPage->apOvfl[0]; u16 szCell = cellSizePtr(pPage, pCell); u8 *pStop; @@ -5797,7 +5827,7 @@ static int ptrmapCheckPages(MemPage **apPage, int nPage){ ** map entries are also updated so that the parent page is page pTo. ** ** If pFrom is currently carrying any overflow cells (entries in the -** MemPage.aOvfl[] array), they are not copied to pTo. +** MemPage.apOvfl[] array), they are not copied to pTo. ** ** Before returning, page pTo is reinitialized using btreeInitPage(). ** @@ -5934,7 +5964,7 @@ static int balance_nonroot( ** is called (indirectly) from sqlite3BtreeDelete(). */ assert( pParent->nOverflow==0 || pParent->nOverflow==1 ); - assert( pParent->nOverflow==0 || pParent->aOvfl[0].idx==iParentIdx ); + assert( pParent->nOverflow==0 || pParent->aiOvfl[0]==iParentIdx ); if( !aOvflSpace ){ return SQLITE_NOMEM; @@ -5981,8 +6011,8 @@ static int balance_nonroot( nMaxCells += 1+apOld[i]->nCell+apOld[i]->nOverflow; if( (i--)==0 ) break; - if( i+nxDiv==pParent->aOvfl[0].idx && pParent->nOverflow ){ - apDiv[i] = pParent->aOvfl[0].pCell; + if( i+nxDiv==pParent->aiOvfl[0] && pParent->nOverflow ){ + apDiv[i] = pParent->apOvfl[0]; pgno = get4byte(apDiv[i]); szNew[i] = cellSizePtr(pParent, apDiv[i]); pParent->nOverflow = 0; @@ -6003,7 +6033,7 @@ static int balance_nonroot( ** In this case, temporarily copy the cell into the aOvflSpace[] ** buffer. It will be copied out again as soon as the aSpace[] buffer ** is allocated. */ - if( pBt->secureDelete ){ + if( pBt->btsFlags & BTS_SECURE_DELETE ){ int iOff; iOff = SQLITE_PTR_TO_INT(apDiv[i]) - SQLITE_PTR_TO_INT(pParent->aData); @@ -6188,8 +6218,14 @@ static int balance_nonroot( /* Either we found one or more cells (cntnew[0])>0) or pPage is ** a virtual root page. A virtual root page is when the real root ** page is page 1 and we are the only child of that page. + ** + ** UPDATE: The assert() below is not necessarily true if the database + ** file is corrupt. The corruption will be detected and reported later + ** in this procedure so there is no need to act upon it now. */ +#if 0 assert( cntNew[0]>0 || (pParent->pgno==1 && pParent->nCell==0) ); +#endif TRACE(("BALANCE: old: %d %d %d ", apOld[0]->pgno, @@ -6417,7 +6453,7 @@ static int balance_nonroot( MemPage *pOld = apCopy[0]; int nOverflow = pOld->nOverflow; int iNextOld = pOld->nCell + nOverflow; - int iOverflow = (nOverflow ? pOld->aOvfl[0].idx : -1); + int iOverflow = (nOverflow ? pOld->aiOvfl[0] : -1); j = 0; /* Current 'old' sibling page */ k = 0; /* Current 'new' sibling page */ for(i=0; i<nCell; i++){ @@ -6431,14 +6467,14 @@ static int balance_nonroot( iNextOld = i + !leafData + pOld->nCell + pOld->nOverflow; if( pOld->nOverflow ){ nOverflow = pOld->nOverflow; - iOverflow = i + !leafData + pOld->aOvfl[0].idx; + iOverflow = i + !leafData + pOld->aiOvfl[0]; } isDivider = !leafData; } assert(nOverflow>0 || iOverflow<i ); - assert(nOverflow<2 || pOld->aOvfl[0].idx==pOld->aOvfl[1].idx-1); - assert(nOverflow<3 || pOld->aOvfl[1].idx==pOld->aOvfl[2].idx-1); + assert(nOverflow<2 || pOld->aiOvfl[0]==pOld->aiOvfl[1]-1); + assert(nOverflow<3 || pOld->aiOvfl[1]==pOld->aiOvfl[2]-1); if( i==iOverflow ){ isDivider = 1; if( (--nOverflow)>0 ){ @@ -6559,7 +6595,10 @@ static int balance_deeper(MemPage *pRoot, MemPage **ppChild){ TRACE(("BALANCE: copy root %d into %d\n", pRoot->pgno, pChild->pgno)); /* Copy the overflow cells from pRoot to pChild */ - memcpy(pChild->aOvfl, pRoot->aOvfl, pRoot->nOverflow*sizeof(pRoot->aOvfl[0])); + memcpy(pChild->aiOvfl, pRoot->aiOvfl, + pRoot->nOverflow*sizeof(pRoot->aiOvfl[0])); + memcpy(pChild->apOvfl, pRoot->apOvfl, + pRoot->nOverflow*sizeof(pRoot->apOvfl[0])); pChild->nOverflow = pRoot->nOverflow; /* Zero the contents of pRoot. Then install pChild as the right-child. */ @@ -6622,7 +6661,7 @@ static int balance(BtCursor *pCur){ #ifndef SQLITE_OMIT_QUICKBALANCE if( pPage->hasData && pPage->nOverflow==1 - && pPage->aOvfl[0].idx==pPage->nCell + && pPage->aiOvfl[0]==pPage->nCell && pParent->pgno!=1 && pParent->nCell==iIdx ){ @@ -6739,7 +6778,8 @@ int sqlite3BtreeInsert( } assert( cursorHoldsMutex(pCur) ); - assert( pCur->wrFlag && pBt->inTransaction==TRANS_WRITE && !pBt->readOnly ); + assert( pCur->wrFlag && pBt->inTransaction==TRANS_WRITE + && (pBt->btsFlags & BTS_READ_ONLY)==0 ); assert( hasSharedCacheTableLock(p, pCur->pgnoRoot, pCur->pKeyInfo!=0, 2) ); /* Assert that the caller has been consistent. If this cursor was opened @@ -6749,13 +6789,6 @@ int sqlite3BtreeInsert( ** blob of associated data. */ assert( (pKey==0)==(pCur->pKeyInfo==0) ); - /* If this is an insert into a table b-tree, invalidate any incrblob - ** cursors open on the row being replaced (assuming this is a replace - ** operation - if it is not, the following is a no-op). */ - if( pCur->pKeyInfo==0 ){ - invalidateIncrblobCursors(p, nKey, 0); - } - /* Save the positions of any other cursors open on this table. ** ** In some cases, the call to btreeMoveto() below is a no-op. For @@ -6769,6 +6802,14 @@ int sqlite3BtreeInsert( */ rc = saveAllCursors(pBt, pCur->pgnoRoot, pCur); if( rc ) return rc; + + /* If this is an insert into a table b-tree, invalidate any incrblob + ** cursors open on the row being replaced (assuming this is a replace + ** operation - if it is not, the following is a no-op). */ + if( pCur->pKeyInfo==0 ){ + invalidateIncrblobCursors(p, nKey, 0); + } + if( !loc ){ rc = btreeMoveto(pCur, pKey, nKey, appendBias, &loc); if( rc ) return rc; @@ -6868,7 +6909,7 @@ int sqlite3BtreeDelete(BtCursor *pCur){ assert( cursorHoldsMutex(pCur) ); assert( pBt->inTransaction==TRANS_WRITE ); - assert( !pBt->readOnly ); + assert( (pBt->btsFlags & BTS_READ_ONLY)==0 ); assert( pCur->wrFlag ); assert( hasSharedCacheTableLock(p, pCur->pgnoRoot, pCur->pKeyInfo!=0, 2) ); assert( !hasReadConflicts(p, pCur->pgnoRoot) ); @@ -6879,12 +6920,6 @@ int sqlite3BtreeDelete(BtCursor *pCur){ return SQLITE_ERROR; /* Something has gone awry. */ } - /* If this is a delete operation to remove a row from a table b-tree, - ** invalidate any incrblob cursors open on the row being deleted. */ - if( pCur->pKeyInfo==0 ){ - invalidateIncrblobCursors(p, pCur->info.nKey, 0); - } - iCellDepth = pCur->iPage; iCellIdx = pCur->aiIdx[iCellDepth]; pPage = pCur->apPage[iCellDepth]; @@ -6910,6 +6945,13 @@ int sqlite3BtreeDelete(BtCursor *pCur){ */ rc = saveAllCursors(pBt, pCur->pgnoRoot, pCur); if( rc ) return rc; + + /* If this is a delete operation to remove a row from a table b-tree, + ** invalidate any incrblob cursors open on the row being deleted. */ + if( pCur->pKeyInfo==0 ){ + invalidateIncrblobCursors(p, pCur->info.nKey, 0); + } + rc = sqlite3PagerWrite(pPage->pDbPage); if( rc ) return rc; rc = clearCell(pPage, pCell); @@ -6989,7 +7031,7 @@ static int btreeCreateTable(Btree *p, int *piTable, int createTabFlags){ assert( sqlite3BtreeHoldsMutex(p) ); assert( pBt->inTransaction==TRANS_WRITE ); - assert( !pBt->readOnly ); + assert( (pBt->btsFlags & BTS_READ_ONLY)==0 ); #ifdef SQLITE_OMIT_AUTOVACUUM rc = allocateBtreePage(pBt, &pRoot, &pgnoRoot, 1, 0); @@ -7191,13 +7233,13 @@ int sqlite3BtreeClearTable(Btree *p, int iTable, int *pnChange){ sqlite3BtreeEnter(p); assert( p->inTrans==TRANS_WRITE ); - /* Invalidate all incrblob cursors open on table iTable (assuming iTable - ** is the root of a table b-tree - if it is not, the following call is - ** a no-op). */ - invalidateIncrblobCursors(p, 0, 1); - rc = saveAllCursors(pBt, (Pgno)iTable, 0); + if( SQLITE_OK==rc ){ + /* Invalidate all incrblob cursors open on table iTable (assuming iTable + ** is the root of a table b-tree - if it is not, the following call is + ** a no-op). */ + invalidateIncrblobCursors(p, 0, 1); rc = clearDatabasePage(pBt, (Pgno)iTable, 0, pnChange); } sqlite3BtreeLeave(p); @@ -7363,7 +7405,9 @@ void sqlite3BtreeGetMeta(Btree *p, int idx, u32 *pMeta){ /* If auto-vacuum is disabled in this build and this is an auto-vacuum ** database, mark the database as read-only. */ #ifdef SQLITE_OMIT_AUTOVACUUM - if( idx==BTREE_LARGEST_ROOT_PAGE && *pMeta>0 ) pBt->readOnly = 1; + if( idx==BTREE_LARGEST_ROOT_PAGE && *pMeta>0 ){ + pBt->btsFlags |= BTS_READ_ONLY; + } #endif sqlite3BtreeLeave(p); @@ -7510,6 +7554,25 @@ static void checkAppendMsg( #endif /* SQLITE_OMIT_INTEGRITY_CHECK */ #ifndef SQLITE_OMIT_INTEGRITY_CHECK + +/* +** Return non-zero if the bit in the IntegrityCk.aPgRef[] array that +** corresponds to page iPg is already set. +*/ +static int getPageReferenced(IntegrityCk *pCheck, Pgno iPg){ + assert( iPg<=pCheck->nPage && sizeof(pCheck->aPgRef[0])==1 ); + return (pCheck->aPgRef[iPg/8] & (1 << (iPg & 0x07))); +} + +/* +** Set the bit in the IntegrityCk.aPgRef[] array that corresponds to page iPg. +*/ +static void setPageReferenced(IntegrityCk *pCheck, Pgno iPg){ + assert( iPg<=pCheck->nPage && sizeof(pCheck->aPgRef[0])==1 ); + pCheck->aPgRef[iPg/8] |= (1 << (iPg & 0x07)); +} + + /* ** Add 1 to the reference count for page iPage. If this is the second ** reference to the page, add an error message to pCheck->zErrMsg. @@ -7524,11 +7587,12 @@ static int checkRef(IntegrityCk *pCheck, Pgno iPage, char *zContext){ checkAppendMsg(pCheck, zContext, "invalid page number %d", iPage); return 1; } - if( pCheck->anRef[iPage]==1 ){ + if( getPageReferenced(pCheck, iPage) ){ checkAppendMsg(pCheck, zContext, "2nd reference to page %d", iPage); return 1; } - return (pCheck->anRef[iPage]++)>1; + setPageReferenced(pCheck, iPage); + return 0; } #ifndef SQLITE_OMIT_AUTOVACUUM @@ -7904,17 +7968,15 @@ char *sqlite3BtreeIntegrityCheck( sqlite3BtreeLeave(p); return 0; } - sCheck.anRef = sqlite3Malloc( (sCheck.nPage+1)*sizeof(sCheck.anRef[0]) ); - if( !sCheck.anRef ){ + + sCheck.aPgRef = sqlite3MallocZero((sCheck.nPage / 8)+ 1); + if( !sCheck.aPgRef ){ *pnErr = 1; sqlite3BtreeLeave(p); return 0; } - for(i=0; i<=sCheck.nPage; i++){ sCheck.anRef[i] = 0; } i = PENDING_BYTE_PAGE(pBt); - if( i<=sCheck.nPage ){ - sCheck.anRef[i] = 1; - } + if( i<=sCheck.nPage ) setPageReferenced(&sCheck, i); sqlite3StrAccumInit(&sCheck.errMsg, zErr, sizeof(zErr), 20000); sCheck.errMsg.useMalloc = 2; @@ -7939,18 +8001,18 @@ char *sqlite3BtreeIntegrityCheck( */ for(i=1; i<=sCheck.nPage && sCheck.mxErr; i++){ #ifdef SQLITE_OMIT_AUTOVACUUM - if( sCheck.anRef[i]==0 ){ + if( getPageReferenced(&sCheck, i)==0 ){ checkAppendMsg(&sCheck, 0, "Page %d is never used", i); } #else /* If the database supports auto-vacuum, make sure no tables contain ** references to pointer-map pages. */ - if( sCheck.anRef[i]==0 && + if( getPageReferenced(&sCheck, i)==0 && (PTRMAP_PAGENO(pBt, i)!=i || !pBt->autoVacuum) ){ checkAppendMsg(&sCheck, 0, "Page %d is never used", i); } - if( sCheck.anRef[i]!=0 && + if( getPageReferenced(&sCheck, i)!=0 && (PTRMAP_PAGENO(pBt, i)==i && pBt->autoVacuum) ){ checkAppendMsg(&sCheck, 0, "Pointer map page %d is referenced", i); } @@ -7971,7 +8033,7 @@ char *sqlite3BtreeIntegrityCheck( /* Clean up and report errors. */ sqlite3BtreeLeave(p); - sqlite3_free(sCheck.anRef); + sqlite3_free(sCheck.aPgRef); if( sCheck.mallocFailed ){ sqlite3StrAccumReset(&sCheck.errMsg); *pnErr = sCheck.nErr+1; @@ -8163,7 +8225,8 @@ int sqlite3BtreePutData(BtCursor *pCsr, u32 offset, u32 amt, void *z){ if( !pCsr->wrFlag ){ return SQLITE_READONLY; } - assert( !pCsr->pBt->readOnly && pCsr->pBt->inTransaction==TRANS_WRITE ); + assert( (pCsr->pBt->btsFlags & BTS_READ_ONLY)==0 + && pCsr->pBt->inTransaction==TRANS_WRITE ); assert( hasSharedCacheTableLock(pCsr->pBtree, pCsr->pgnoRoot, 0, 2) ); assert( !hasReadConflicts(pCsr->pBtree, pCsr->pgnoRoot) ); assert( pCsr->apPage[pCsr->iPage]->intKey ); @@ -8203,7 +8266,8 @@ int sqlite3BtreeSetVersion(Btree *pBtree, int iVersion){ /* If setting the version fields to 1, do not automatically open the ** WAL connection, even if the version fields are currently set to 2. */ - pBt->doNotUseWAL = (u8)(iVersion==1); + pBt->btsFlags &= ~BTS_NO_WAL; + if( iVersion==1 ) pBt->btsFlags |= BTS_NO_WAL; rc = sqlite3BtreeBeginTrans(pBtree, 0); if( rc==SQLITE_OK ){ @@ -8220,6 +8284,6 @@ int sqlite3BtreeSetVersion(Btree *pBtree, int iVersion){ } } - pBt->doNotUseWAL = 0; + pBt->btsFlags &= ~BTS_NO_WAL; return rc; } diff --git a/src/btree.h b/src/btree.h index 9e3a73b..9832001 100644 --- a/src/btree.h +++ b/src/btree.h @@ -57,10 +57,9 @@ int sqlite3BtreeOpen( ** pager.h. */ #define BTREE_OMIT_JOURNAL 1 /* Do not create or use a rollback journal */ -#define BTREE_NO_READLOCK 2 /* Omit readlocks on readonly files */ -#define BTREE_MEMORY 4 /* This is an in-memory DB */ -#define BTREE_SINGLE 8 /* The file contains at most 1 b-tree */ -#define BTREE_UNORDERED 16 /* Use of a hash implementation is OK */ +#define BTREE_MEMORY 2 /* This is an in-memory DB */ +#define BTREE_SINGLE 4 /* The file contains at most 1 b-tree */ +#define BTREE_UNORDERED 8 /* Use of a hash implementation is OK */ int sqlite3BtreeClose(Btree*); int sqlite3BtreeSetCacheSize(Btree*,int); @@ -78,7 +77,7 @@ int sqlite3BtreeBeginTrans(Btree*,int); int sqlite3BtreeCommitPhaseOne(Btree*, const char *zMaster); int sqlite3BtreeCommitPhaseTwo(Btree*, int); int sqlite3BtreeCommit(Btree*); -int sqlite3BtreeRollback(Btree*); +int sqlite3BtreeRollback(Btree*,int); int sqlite3BtreeBeginStmt(Btree*,int); int sqlite3BtreeCreateTable(Btree*, int*, int flags); int sqlite3BtreeIsInTrans(Btree*); diff --git a/src/btreeInt.h b/src/btreeInt.h index 55469cf..0d21497 100644 --- a/src/btreeInt.h +++ b/src/btreeInt.h @@ -277,18 +277,20 @@ struct MemPage { u8 hasData; /* True if this page stores data */ u8 hdrOffset; /* 100 for page 1. 0 otherwise */ u8 childPtrSize; /* 0 if leaf==1. 4 if leaf==0 */ + u8 max1bytePayload; /* min(maxLocal,127) */ u16 maxLocal; /* Copy of BtShared.maxLocal or BtShared.maxLeaf */ u16 minLocal; /* Copy of BtShared.minLocal or BtShared.minLeaf */ u16 cellOffset; /* Index in aData of first cell pointer */ u16 nFree; /* Number of free bytes on the page */ u16 nCell; /* Number of cells on this page, local and ovfl */ u16 maskPage; /* Mask for page offset */ - struct _OvflCell { /* Cells that will not fit on aData[] */ - u8 *pCell; /* Pointers to the body of the overflow cell */ - u16 idx; /* Insert this cell before idx-th non-overflow cell */ - } aOvfl[5]; + u16 aiOvfl[5]; /* Insert the i-th overflow cell before the aiOvfl-th + ** non-overflow cell */ + u8 *apOvfl[5]; /* Pointers to the body of overflow cells */ BtShared *pBt; /* Pointer to BtShared that this page is part of */ u8 *aData; /* Pointer to disk image of the page data */ + u8 *aDataEnd; /* One byte past the end of usable data */ + u8 *aCellIdx; /* The cell index area */ DbPage *pDbPage; /* Pager page handle */ Pgno pgno; /* Page number for this page */ }; @@ -368,7 +370,7 @@ struct Btree { /* ** An instance of this object represents a single database file. ** -** A single database file can be in use as the same time by two +** A single database file can be in use at the same time by two ** or more database connections. When two or more connections are ** sharing the same database file, each connection has it own ** private Btree object for the file and each of those Btrees points @@ -405,17 +407,14 @@ struct BtShared { sqlite3 *db; /* Database connection currently using this Btree */ BtCursor *pCursor; /* A list of all open cursors */ MemPage *pPage1; /* First page of the database */ - u8 readOnly; /* True if the underlying file is readonly */ - u8 pageSizeFixed; /* True if the page size can no longer be changed */ - u8 secureDelete; /* True if secure_delete is enabled */ - u8 initiallyEmpty; /* Database is empty at start of transaction */ u8 openFlags; /* Flags to sqlite3BtreeOpen() */ #ifndef SQLITE_OMIT_AUTOVACUUM u8 autoVacuum; /* True if auto-vacuum is enabled */ u8 incrVacuum; /* True if incr-vacuum is enabled */ #endif u8 inTransaction; /* Transaction state */ - u8 doNotUseWAL; /* If true, do not open write-ahead-log file */ + u8 max1bytePayload; /* Maximum first byte of cell for a 1-byte payload */ + u16 btsFlags; /* Boolean parameters. See BTS_* macros below */ u16 maxLocal; /* Maximum local payload in non-LEAFDATA tables */ u16 minLocal; /* Minimum local payload in non-LEAFDATA tables */ u16 maxLeaf; /* Maximum local payload in a LEAFDATA table */ @@ -433,13 +432,22 @@ struct BtShared { BtShared *pNext; /* Next on a list of sharable BtShared structs */ BtLock *pLock; /* List of locks held on this shared-btree struct */ Btree *pWriter; /* Btree with currently open write transaction */ - u8 isExclusive; /* True if pWriter has an EXCLUSIVE lock on the db */ - u8 isPending; /* If waiting for read-locks to clear */ #endif u8 *pTmpSpace; /* BtShared.pageSize bytes of space for tmp use */ }; /* +** Allowed values for BtShared.btsFlags +*/ +#define BTS_READ_ONLY 0x0001 /* Underlying file is readonly */ +#define BTS_PAGESIZE_FIXED 0x0002 /* Page size can no longer be changed */ +#define BTS_SECURE_DELETE 0x0004 /* PRAGMA secure_delete is enabled */ +#define BTS_INITIALLY_EMPTY 0x0008 /* Database was empty at trans start */ +#define BTS_NO_WAL 0x0010 /* Do not open write-ahead-log files */ +#define BTS_EXCLUSIVE 0x0020 /* pWriter has an exclusive lock */ +#define BTS_PENDING 0x0040 /* Waiting for read-locks to clear */ + +/* ** An instance of the following structure is used to hold information ** about a cell. The parseCellPtr() function fills in this structure ** based on information extract from the raw disk page. @@ -474,7 +482,7 @@ struct CellInfo { ** The entry is identified by its MemPage and the index in ** MemPage.aCell[] of the entry. ** -** A single database file can shared by two more database connections, +** A single database file can be shared by two more database connections, ** but cursors cannot be shared. Each cursor is associated with a ** particular database connection identified BtCursor.pBtree.db. ** @@ -486,6 +494,9 @@ struct BtCursor { BtShared *pBt; /* The BtShared this cursor points to */ BtCursor *pNext, *pPrev; /* Forms a linked list of all cursors */ struct KeyInfo *pKeyInfo; /* Argument passed to comparison function */ +#ifndef SQLITE_OMIT_INCRBLOB + Pgno *aOverflow; /* Cache of overflow page locations */ +#endif Pgno pgnoRoot; /* The root page of this tree */ sqlite3_int64 cachedRowid; /* Next rowid cache. 0 means not valid */ CellInfo info; /* A parse of the cell we are pointing at */ @@ -497,7 +508,6 @@ struct BtCursor { u8 validNKey; /* True if info.nKey is valid */ u8 eState; /* One of the CURSOR_XXX constants (see below) */ #ifndef SQLITE_OMIT_INCRBLOB - Pgno *aOverflow; /* Cache of overflow page locations */ u8 isIncrblobHandle; /* True if this cursor is an incr. io handle */ #endif i16 iPage; /* Index of current page in apPage */ @@ -621,13 +631,19 @@ struct BtCursor { /* ** This structure is passed around through all the sanity checking routines ** in order to keep track of some global state information. +** +** The aRef[] array is allocated so that there is 1 bit for each page in +** the database. As the integrity-check proceeds, for each page used in +** the database the corresponding bit is set. This allows integrity-check to +** detect pages that are used twice and orphaned pages (both of which +** indicate corruption). */ typedef struct IntegrityCk IntegrityCk; struct IntegrityCk { BtShared *pBt; /* The tree being checked out */ Pager *pPager; /* The associated pager. Also accessible by pBt->pPager */ + u8 *aPgRef; /* 1 bit per page in the db (see above) */ Pgno nPage; /* Number of pages in the database */ - int *anRef; /* Number of times each page is referenced */ int mxErr; /* Stop accumulating errors when this reaches zero */ int nErr; /* Number of messages written to zErrMsg so far */ int mallocFailed; /* A memory allocation error has occurred */ @@ -635,7 +651,7 @@ struct IntegrityCk { }; /* -** Read or write a two- and four-byte big-endian integer values. +** Routines to read or write a two- and four-byte big-endian integer values. */ #define get2byte(x) ((x)[0]<<8 | (x)[1]) #define put2byte(p,v) ((p)[0] = (u8)((v)>>8), (p)[1] = (u8)(v)) diff --git a/src/build.c b/src/build.c index e23aab6..31190f6 100644 --- a/src/build.c +++ b/src/build.c @@ -502,9 +502,16 @@ static void sqliteDeleteColumnNames(sqlite3 *db, Table *pTable){ ** the table data structure from the hash table. But it does destroy ** memory structures of the indices and foreign keys associated with ** the table. +** +** The db parameter is optional. It is needed if the Table object +** contains lookaside memory. (Table objects in the schema do not use +** lookaside memory, but some ephemeral Table objects do.) Or the +** db parameter can be used with db->pnBytesFreed to measure the memory +** used by the Table object. */ void sqlite3DeleteTable(sqlite3 *db, Table *pTable){ Index *pIndex, *pNext; + TESTONLY( int nLookaside; ) /* Used to verify lookaside not used for schema */ assert( !pTable || pTable->nRef>0 ); @@ -512,6 +519,12 @@ void sqlite3DeleteTable(sqlite3 *db, Table *pTable){ if( !pTable ) return; if( ((!db || db->pnBytesFreed==0) && (--pTable->nRef)>0) ) return; + /* Record the number of outstanding lookaside allocations in schema Tables + ** prior to doing any free() operations. Since schema Tables do not use + ** lookaside, this number should not change. */ + TESTONLY( nLookaside = (db && (pTable->tabFlags & TF_Ephemeral)==0) ? + db->lookaside.nOut : 0 ); + /* Delete all indices associated with this table. */ for(pIndex = pTable->pIndex; pIndex; pIndex=pNext){ pNext = pIndex->pNext; @@ -537,12 +550,15 @@ void sqlite3DeleteTable(sqlite3 *db, Table *pTable){ sqlite3DbFree(db, pTable->zColAff); sqlite3SelectDelete(db, pTable->pSelect); #ifndef SQLITE_OMIT_CHECK - sqlite3ExprDelete(db, pTable->pCheck); + sqlite3ExprListDelete(db, pTable->pCheck); #endif #ifndef SQLITE_OMIT_VIRTUALTABLE sqlite3VtabClear(db, pTable); #endif sqlite3DbFree(db, pTable); + + /* Verify that no lookaside memory was used by schema tables */ + assert( nLookaside==0 || nLookaside==db->lookaside.nOut ); } /* @@ -1200,15 +1216,17 @@ void sqlite3AddCheckConstraint( Parse *pParse, /* Parsing context */ Expr *pCheckExpr /* The check expression */ ){ - sqlite3 *db = pParse->db; #ifndef SQLITE_OMIT_CHECK Table *pTab = pParse->pNewTable; if( pTab && !IN_DECLARE_VTAB ){ - pTab->pCheck = sqlite3ExprAnd(db, pTab->pCheck, pCheckExpr); + pTab->pCheck = sqlite3ExprListAppend(pParse, pTab->pCheck, pCheckExpr); + if( pParse->constraintName.n ){ + sqlite3ExprListSetName(pParse, pTab->pCheck, &pParse->constraintName, 1); + } }else #endif { - sqlite3ExprDelete(db, pCheckExpr); + sqlite3ExprDelete(pParse->db, pCheckExpr); } } @@ -1478,6 +1496,8 @@ void sqlite3EndTable( if( p->pCheck ){ SrcList sSrc; /* Fake SrcList for pParse->pNewTable */ NameContext sNC; /* Name context for pParse->pNewTable */ + ExprList *pList; /* List of all CHECK constraints */ + int i; /* Loop counter */ memset(&sNC, 0, sizeof(sNC)); memset(&sSrc, 0, sizeof(sSrc)); @@ -1487,9 +1507,12 @@ void sqlite3EndTable( sSrc.a[0].iCursor = -1; sNC.pParse = pParse; sNC.pSrcList = &sSrc; - sNC.isCheck = 1; - if( sqlite3ResolveExprNames(&sNC, p->pCheck) ){ - return; + sNC.ncFlags = NC_IsCheck; + pList = p->pCheck; + for(i=0; i<pList->nExpr; i++){ + if( sqlite3ResolveExprNames(&sNC, pList->a[i].pExpr) ){ + return; + } } } #endif /* !defined(SQLITE_OMIT_CHECK) */ @@ -1638,7 +1661,6 @@ void sqlite3EndTable( return; } pParse->pNewTable = 0; - db->nTable++; db->flags |= SQLITE_InternChanges; #ifndef SQLITE_OMIT_ALTERTABLE @@ -2661,19 +2683,23 @@ Index *sqlite3CreateIndex( nName = sqlite3Strlen30(zName); nCol = pList->nExpr; pIndex = sqlite3DbMallocZero(db, - sizeof(Index) + /* Index structure */ - sizeof(tRowcnt)*(nCol+1) + /* Index.aiRowEst */ - sizeof(int)*nCol + /* Index.aiColumn */ - sizeof(char *)*nCol + /* Index.azColl */ - sizeof(u8)*nCol + /* Index.aSortOrder */ - nName + 1 + /* Index.zName */ - nExtra /* Collation sequence names */ + ROUND8(sizeof(Index)) + /* Index structure */ + ROUND8(sizeof(tRowcnt)*(nCol+1)) + /* Index.aiRowEst */ + sizeof(char *)*nCol + /* Index.azColl */ + sizeof(int)*nCol + /* Index.aiColumn */ + sizeof(u8)*nCol + /* Index.aSortOrder */ + nName + 1 + /* Index.zName */ + nExtra /* Collation sequence names */ ); if( db->mallocFailed ){ goto exit_create_index; } - pIndex->aiRowEst = (tRowcnt*)(&pIndex[1]); - pIndex->azColl = (char**)(&pIndex->aiRowEst[nCol+1]); + zExtra = (char*)pIndex; + pIndex->aiRowEst = (tRowcnt*)&zExtra[ROUND8(sizeof(Index))]; + pIndex->azColl = (char**) + ((char*)pIndex->aiRowEst + ROUND8(sizeof(tRowcnt)*nCol+1)); + assert( EIGHT_BYTE_ALIGNMENT(pIndex->aiRowEst) ); + assert( EIGHT_BYTE_ALIGNMENT(pIndex->azColl) ); pIndex->aiColumn = (int *)(&pIndex->azColl[nCol]); pIndex->aSortOrder = (u8 *)(&pIndex->aiColumn[nCol]); pIndex->zName = (char *)(&pIndex->aSortOrder[nCol]); @@ -3037,45 +3063,43 @@ exit_drop_index: } /* -** pArray is a pointer to an array of objects. Each object in the -** array is szEntry bytes in size. This routine allocates a new -** object on the end of the array. +** pArray is a pointer to an array of objects. Each object in the +** array is szEntry bytes in size. This routine uses sqlite3DbRealloc() +** to extend the array so that there is space for a new object at the end. ** -** *pnEntry is the number of entries already in use. *pnAlloc is -** the previously allocated size of the array. initSize is the -** suggested initial array size allocation. +** When this function is called, *pnEntry contains the current size of +** the array (in entries - so the allocation is ((*pnEntry) * szEntry) bytes +** in total). ** -** The index of the new entry is returned in *pIdx. +** If the realloc() is successful (i.e. if no OOM condition occurs), the +** space allocated for the new object is zeroed, *pnEntry updated to +** reflect the new size of the array and a pointer to the new allocation +** returned. *pIdx is set to the index of the new array entry in this case. ** -** This routine returns a pointer to the array of objects. This -** might be the same as the pArray parameter or it might be a different -** pointer if the array was resized. +** Otherwise, if the realloc() fails, *pIdx is set to -1, *pnEntry remains +** unchanged and a copy of pArray returned. */ void *sqlite3ArrayAllocate( sqlite3 *db, /* Connection to notify of malloc failures */ void *pArray, /* Array of objects. Might be reallocated */ int szEntry, /* Size of each object in the array */ - int initSize, /* Suggested initial allocation, in elements */ int *pnEntry, /* Number of objects currently in use */ - int *pnAlloc, /* Current size of the allocation, in elements */ int *pIdx /* Write the index of a new slot here */ ){ char *z; - if( *pnEntry >= *pnAlloc ){ - void *pNew; - int newSize; - newSize = (*pnAlloc)*2 + initSize; - pNew = sqlite3DbRealloc(db, pArray, newSize*szEntry); + int n = *pnEntry; + if( (n & (n-1))==0 ){ + int sz = (n==0) ? 1 : 2*n; + void *pNew = sqlite3DbRealloc(db, pArray, sz*szEntry); if( pNew==0 ){ *pIdx = -1; return pArray; } - *pnAlloc = sqlite3DbMallocSize(db, pNew)/szEntry; pArray = pNew; } z = (char*)pArray; - memset(&z[*pnEntry * szEntry], 0, szEntry); - *pIdx = *pnEntry; + memset(&z[n * szEntry], 0, szEntry); + *pIdx = n; ++*pnEntry; return pArray; } @@ -3091,15 +3115,12 @@ IdList *sqlite3IdListAppend(sqlite3 *db, IdList *pList, Token *pToken){ if( pList==0 ){ pList = sqlite3DbMallocZero(db, sizeof(IdList) ); if( pList==0 ) return 0; - pList->nAlloc = 0; } pList->a = sqlite3ArrayAllocate( db, pList->a, sizeof(pList->a[0]), - 5, &pList->nId, - &pList->nAlloc, &i ); if( i<0 ){ diff --git a/src/callback.c b/src/callback.c index ce84908..a515e05 100644 --- a/src/callback.c +++ b/src/callback.c @@ -223,38 +223,57 @@ CollSeq *sqlite3FindCollSeq( ** that uses encoding enc. The value returned indicates how well the ** request is matched. A higher value indicates a better match. ** +** If nArg is -1 that means to only return a match (non-zero) if p->nArg +** is also -1. In other words, we are searching for a function that +** takes a variable number of arguments. +** +** If nArg is -2 that means that we are searching for any function +** regardless of the number of arguments it uses, so return a positive +** match score for any +** ** The returned value is always between 0 and 6, as follows: ** -** 0: Not a match, or if nArg<0 and the function is has no implementation. -** 1: A variable arguments function that prefers UTF-8 when a UTF-16 -** encoding is requested, or vice versa. -** 2: A variable arguments function that uses UTF-16BE when UTF-16LE is -** requested, or vice versa. -** 3: A variable arguments function using the same text encoding. -** 4: A function with the exact number of arguments requested that -** prefers UTF-8 when a UTF-16 encoding is requested, or vice versa. -** 5: A function with the exact number of arguments requested that -** prefers UTF-16LE when UTF-16BE is requested, or vice versa. -** 6: An exact match. +** 0: Not a match. +** 1: UTF8/16 conversion required and function takes any number of arguments. +** 2: UTF16 byte order change required and function takes any number of args. +** 3: encoding matches and function takes any number of arguments +** 4: UTF8/16 conversion required - argument count matches exactly +** 5: UTF16 byte order conversion required - argument count matches exactly +** 6: Perfect match: encoding and argument count match exactly. ** +** If nArg==(-2) then any function with a non-null xStep or xFunc is +** a perfect match and any function with both xStep and xFunc NULL is +** a non-match. */ -static int matchQuality(FuncDef *p, int nArg, u8 enc){ - int match = 0; - if( p->nArg==-1 || p->nArg==nArg - || (nArg==-1 && (p->xFunc!=0 || p->xStep!=0)) - ){ +#define FUNC_PERFECT_MATCH 6 /* The score for a perfect match */ +static int matchQuality( + FuncDef *p, /* The function we are evaluating for match quality */ + int nArg, /* Desired number of arguments. (-1)==any */ + u8 enc /* Desired text encoding */ +){ + int match; + + /* nArg of -2 is a special case */ + if( nArg==(-2) ) return (p->xFunc==0 && p->xStep==0) ? 0 : FUNC_PERFECT_MATCH; + + /* Wrong number of arguments means "no match" */ + if( p->nArg!=nArg && p->nArg>=0 ) return 0; + + /* Give a better score to a function with a specific number of arguments + ** than to function that accepts any number of arguments. */ + if( p->nArg==nArg ){ + match = 4; + }else{ match = 1; - if( p->nArg==nArg || nArg==-1 ){ - match = 4; - } - if( enc==p->iPrefEnc ){ - match += 2; - } - else if( (enc==SQLITE_UTF16LE && p->iPrefEnc==SQLITE_UTF16BE) || - (enc==SQLITE_UTF16BE && p->iPrefEnc==SQLITE_UTF16LE) ){ - match += 1; - } } + + /* Bonus points if the text encoding matches */ + if( enc==p->iPrefEnc ){ + match += 2; /* Exact encoding match */ + }else if( (enc & p->iPrefEnc & 2)!=0 ){ + match += 1; /* Both are UTF16, but with different byte orders */ + } + return match; } @@ -310,13 +329,12 @@ void sqlite3FuncDefInsert( ** ** If the createFlag argument is true, then a new (blank) FuncDef ** structure is created and liked into the "db" structure if a -** no matching function previously existed. When createFlag is true -** and the nArg parameter is -1, then only a function that accepts -** any number of arguments will be returned. +** no matching function previously existed. ** -** If createFlag is false and nArg is -1, then the first valid -** function found is returned. A function is valid if either xFunc -** or xStep is non-zero. +** If nArg is -2, then the first valid function found is returned. A +** function is valid if either xFunc or xStep is non-zero. The nArg==(-2) +** case is used to see if zName is a valid function name for some number +** of arguments. If nArg is -2, then createFlag must be 0. ** ** If createFlag is false, then a function with the required name and ** number of arguments may be returned even if the eTextRep flag does not @@ -328,14 +346,15 @@ FuncDef *sqlite3FindFunction( int nName, /* Number of characters in the name */ int nArg, /* Number of arguments. -1 means any number */ u8 enc, /* Preferred text encoding */ - int createFlag /* Create new entry if true and does not otherwise exist */ + u8 createFlag /* Create new entry if true and does not otherwise exist */ ){ FuncDef *p; /* Iterator variable */ FuncDef *pBest = 0; /* Best match found so far */ int bestScore = 0; /* Score of best match */ int h; /* Hash value */ - + assert( nArg>=(-2) ); + assert( nArg>=(-1) || createFlag==0 ); assert( enc==SQLITE_UTF8 || enc==SQLITE_UTF16LE || enc==SQLITE_UTF16BE ); h = (sqlite3UpperToLower[(u8)zName[0]] + nName) % ArraySize(db->aFunc.a); @@ -381,7 +400,7 @@ FuncDef *sqlite3FindFunction( ** exact match for the name, number of arguments and encoding, then add a ** new entry to the hash table and return it. */ - if( createFlag && (bestScore<6 || pBest->nArg!=nArg) && + if( createFlag && bestScore<FUNC_PERFECT_MATCH && (pBest = sqlite3DbMallocZero(db, sizeof(*pBest)+nName+1))!=0 ){ pBest->zName = (char *)&pBest[1]; pBest->nArg = (u16)nArg; diff --git a/src/crypto.c b/src/crypto.c index 5c8b2d6..3423b0a 100644 --- a/src/crypto.c +++ b/src/crypto.c @@ -38,9 +38,18 @@ #include "btreeInt.h" #include "crypto.h" +/* Generate code to return a string value */ +void codec_vdbe_return_static_string(Parse *pParse, const char *zLabel, const char *value){ + Vdbe *v = sqlite3GetVdbe(pParse); + sqlite3VdbeSetNumCols(v, 1); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, zLabel, SQLITE_STATIC); + sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, value, 0); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); +} + int codec_set_kdf_iter(sqlite3* db, int nDb, int kdf_iter, int for_ctx) { struct Db *pDb = &db->aDb[nDb]; - CODEC_TRACE(("codec_set_kdf_iter: entered db=%d nDb=%d kdf_iter=%d for_ctx=%d\n", db, nDb, kdf_iter, for_ctx)); + CODEC_TRACE(("codec_set_kdf_iter: entered db=%p nDb=%d kdf_iter=%d for_ctx=%d\n", db, nDb, kdf_iter, for_ctx)); if(pDb->pBt) { codec_ctx *ctx; @@ -52,7 +61,7 @@ int codec_set_kdf_iter(sqlite3* db, int nDb, int kdf_iter, int for_ctx) { int codec_set_fast_kdf_iter(sqlite3* db, int nDb, int kdf_iter, int for_ctx) { struct Db *pDb = &db->aDb[nDb]; - CODEC_TRACE(("codec_set_kdf_iter: entered db=%d nDb=%d kdf_iter=%d for_ctx=%d\n", db, nDb, kdf_iter, for_ctx)); + CODEC_TRACE(("codec_set_kdf_iter: entered db=%p nDb=%d kdf_iter=%d for_ctx=%d\n", db, nDb, kdf_iter, for_ctx)); if(pDb->pBt) { codec_ctx *ctx; @@ -70,17 +79,24 @@ static int codec_set_btree_to_codec_pagesize(sqlite3 *db, Db *pDb, codec_ctx *ct sqlite3_mutex_enter(db->mutex); db->nextPagesize = page_sz; - pDb->pBt->pBt->pageSizeFixed = 0; + + /* before forcing the page size we need to unset the BTS_PAGESIZE_FIXED flag, else + sqliteBtreeSetPageSize will block the change */ + pDb->pBt->pBt->btsFlags &= ~BTS_PAGESIZE_FIXED; CODEC_TRACE(("codec_set_btree_to_codec_pagesize: sqlite3BtreeSetPageSize() size=%d reserve=%d\n", page_sz, reserve_sz)); rc = sqlite3BtreeSetPageSize(pDb->pBt, page_sz, reserve_sz, 0); sqlite3_mutex_leave(db->mutex); return rc; } +void codec_set_default_use_hmac(int use) { + sqlcipher_set_default_use_hmac(use); +} + int codec_set_use_hmac(sqlite3* db, int nDb, int use) { struct Db *pDb = &db->aDb[nDb]; - CODEC_TRACE(("codec_set_use_hmac: entered db=%d nDb=%d use=%d\n", db, nDb, use)); + CODEC_TRACE(("codec_set_use_hmac: entered db=%p nDb=%d use=%d\n", db, nDb, use)); if(pDb->pBt) { int rc; @@ -90,8 +106,6 @@ int codec_set_use_hmac(sqlite3* db, int nDb, int use) { rc = sqlcipher_codec_ctx_set_use_hmac(ctx, use); if(rc != SQLITE_OK) return rc; /* since the use of hmac has changed, the page size may also change */ - /* Note: before forcing the page size we need to force pageSizeFixed to 0, else - sqliteBtreeSetPageSize will block the change */ return codec_set_btree_to_codec_pagesize(db, pDb, ctx); } } @@ -100,7 +114,7 @@ int codec_set_use_hmac(sqlite3* db, int nDb, int use) { int codec_set_page_size(sqlite3* db, int nDb, int size) { struct Db *pDb = &db->aDb[nDb]; - CODEC_TRACE(("codec_set_page_size: entered db=%d nDb=%d size=%d\n", db, nDb, size)); + CODEC_TRACE(("codec_set_page_size: entered db=%p nDb=%d size=%d\n", db, nDb, size)); if(pDb->pBt) { int rc; @@ -124,7 +138,7 @@ int codec_set_page_size(sqlite3* db, int nDb, int size) { */ int codec_set_cipher_name(sqlite3* db, int nDb, const char *cipher_name, int for_ctx) { struct Db *pDb = &db->aDb[nDb]; - CODEC_TRACE(("codec_set_cipher_name: entered db=%d nDb=%d cipher_name=%s for_ctx=%d\n", db, nDb, cipher_name, for_ctx)); + CODEC_TRACE(("codec_set_cipher_name: entered db=%p nDb=%d cipher_name=%s for_ctx=%d\n", db, nDb, cipher_name, for_ctx)); if(pDb->pBt) { codec_ctx *ctx; @@ -136,7 +150,7 @@ int codec_set_cipher_name(sqlite3* db, int nDb, const char *cipher_name, int for int codec_set_pass_key(sqlite3* db, int nDb, const void *zKey, int nKey, int for_ctx) { struct Db *pDb = &db->aDb[nDb]; - CODEC_TRACE(("codec_set_pass_key: entered db=%d nDb=%d cipher_name=%s nKey=%d for_ctx=%d\n", db, nDb, zKey, nKey, for_ctx)); + CODEC_TRACE(("codec_set_pass_key: entered db=%p nDb=%d zKey=%s nKey=%d for_ctx=%d\n", db, nDb, (char *)zKey, nKey, for_ctx)); if(pDb->pBt) { codec_ctx *ctx; sqlite3pager_get_codec(pDb->pBt->pBt->pPager, (void **) &ctx); @@ -207,7 +221,7 @@ void sqlite3FreeCodecArg(void *pCodecArg) { int sqlite3CodecAttach(sqlite3* db, int nDb, const void *zKey, int nKey) { struct Db *pDb = &db->aDb[nDb]; - CODEC_TRACE(("sqlite3CodecAttach: entered nDb=%d zKey=%s, nKey=%d\n", nDb, zKey, nKey)); + CODEC_TRACE(("sqlite3CodecAttach: entered nDb=%d zKey=%s, nKey=%d\n", nDb, (char *)zKey, nKey)); sqlcipher_activate(); @@ -224,6 +238,11 @@ int sqlite3CodecAttach(sqlite3* db, int nDb, const void *zKey, int nKey) { codec_set_btree_to_codec_pagesize(db, pDb, ctx); + /* force secure delete. This has the benefit of wiping internal data when deleted + and also ensures that all pages are written to disk (i.e. not skipped by + sqlite3PagerDontWrite optimizations) */ + sqlite3BtreeSecureDelete(pDb->pBt, 1); + /* if fd is null, then this is an in-memory database and we dont' want to overwrite the AutoVacuum settings if not null, then set to the default */ @@ -241,7 +260,7 @@ void sqlite3_activate_see(const char* in) { } int sqlite3_key(sqlite3 *db, const void *pKey, int nKey) { - CODEC_TRACE(("sqlite3_key: entered db=%d pKey=%s nKey=%d\n", db, pKey, nKey)); + CODEC_TRACE(("sqlite3_key: entered db=%p pKey=%s nKey=%d\n", db, (char *)pKey, nKey)); /* attach key if db and pKey are not null and nKey is > 0 */ if(db && pKey && nKey) { sqlite3CodecAttach(db, 0, pKey, nKey); // operate only on the main db @@ -261,11 +280,11 @@ int sqlite3_key(sqlite3 *db, const void *pKey, int nKey) { ** 3. If there is a key present, re-encrypt the database with the new key */ int sqlite3_rekey(sqlite3 *db, const void *pKey, int nKey) { - CODEC_TRACE(("sqlite3_rekey: entered db=%d pKey=%s, nKey=%d\n", db, pKey, nKey)); + CODEC_TRACE(("sqlite3_rekey: entered db=%p pKey=%s, nKey=%d\n", db, (char *)pKey, nKey)); sqlcipher_activate(); if(db && pKey && nKey) { struct Db *pDb = &db->aDb[0]; - CODEC_TRACE(("sqlite3_rekey: database pDb=%d\n", pDb)); + CODEC_TRACE(("sqlite3_rekey: database pDb=%p\n", pDb)); if(pDb->pBt) { codec_ctx *ctx; int rc, page_count; @@ -298,11 +317,14 @@ int sqlite3_rekey(sqlite3 *db, const void *pKey, int nKey) { rc = sqlite3PagerGet(pPager, pgno, &page); if(rc == SQLITE_OK) { /* write page see pager_incr_changecounter for example */ rc = sqlite3PagerWrite(page); - //printf("sqlite3PagerWrite(%d)\n", pgno); if(rc == SQLITE_OK) { sqlite3PagerUnref(page); - } - } + } else { + CODEC_TRACE(("sqlite3_rekey: error %d occurred writing page %d\n", rc, pgno)); + } + } else { + CODEC_TRACE(("sqlite3_rekey: error %d occurred getting page %d\n", rc, pgno)); + } } } @@ -313,7 +335,7 @@ int sqlite3_rekey(sqlite3 *db, const void *pKey, int nKey) { sqlcipher_codec_key_copy(ctx, CIPHER_WRITE_CTX); } else { CODEC_TRACE(("sqlite3_rekey: rollback\n")); - sqlite3BtreeRollback(pDb->pBt); + sqlite3BtreeRollback(pDb->pBt, SQLITE_ABORT_ROLLBACK); } sqlite3_mutex_leave(db->mutex); @@ -325,7 +347,7 @@ int sqlite3_rekey(sqlite3 *db, const void *pKey, int nKey) { void sqlite3CodecGetKey(sqlite3* db, int nDb, void **zKey, int *nKey) { struct Db *pDb = &db->aDb[nDb]; - CODEC_TRACE(("sqlite3CodecGetKey: entered db=%d, nDb=%d\n", db, nDb)); + CODEC_TRACE(("sqlite3CodecGetKey: entered db=%p, nDb=%d\n", db, nDb)); if( pDb->pBt ) { codec_ctx *ctx; diff --git a/src/crypto.h b/src/crypto.h index a5a62ec..c0e07ae 100644 --- a/src/crypto.h +++ b/src/crypto.h @@ -37,6 +37,10 @@ #define FILE_HEADER_SZ 16 +#ifndef CIPHER_VERSION +#define CIPHER_VERSION "2.0.6" +#endif + #ifndef CIPHER #define CIPHER "aes-256-cbc" #endif @@ -78,9 +82,23 @@ #define CODEC_TRACE(X) #endif +#ifdef CODEC_DEBUG_PAGEDATA +#define CODEC_HEXDUMP(DESC,BUFFER,LEN) \ + { \ + int __pctr; \ + printf(DESC); \ + for(__pctr=0; __pctr < LEN; __pctr++) { \ + if(__pctr % 16 == 0) printf("\n%05x: ",__pctr); \ + printf("%02x ",((unsigned char*) BUFFER)[__pctr]); \ + } \ + printf("\n"); \ + fflush(stdout); \ + } +#else +#define CODEC_HEXDUMP(DESC,BUFFER,LEN) +#endif -/* extensions defined in pragma.c */ - +/* extensions defined in pager.c */ void sqlite3pager_get_codec(Pager *pPager, void **ctx); int sqlite3pager_is_mj_pgno(Pager *pPager, Pgno pgno); sqlite3_file *sqlite3Pager_get_fd(Pager *pPager); @@ -91,7 +109,8 @@ void sqlite3pager_sqlite3PagerSetCodec( void (*xCodecFree)(void*), void *pCodec ); -/* end extensions defined in pragma.c */ +void sqlite3pager_sqlite3PagerSetError(Pager *pPager, int error); +/* end extensions defined in pager.c */ /* ** Simple shared routines for converting hex char strings to binary data @@ -114,6 +133,7 @@ static void cipher_hex2bin(const char *hex, int sz, unsigned char *out){ typedef struct codec_ctx codec_ctx; /* utility functions */ +int sqlcipher_ismemset(const unsigned char *a0, unsigned char value, int len); int sqlcipher_memcmp(const unsigned char *a0, const unsigned char *a1, int len); int sqlcipher_pseudorandom(void *, int); void sqlcipher_free(void *, int); @@ -149,6 +169,8 @@ void* sqlcipher_codec_ctx_get_data(codec_ctx *); void sqlcipher_exportFunc(sqlite3_context *, int, sqlite3_value **); +void sqlcipher_set_default_use_hmac(int use); + int sqlcipher_codec_ctx_set_use_hmac(codec_ctx *ctx, int use); /* end extensions defined in crypto_impl.c */ diff --git a/src/crypto_impl.c b/src/crypto_impl.c index 336afbd..c58ae60 100644 --- a/src/crypto_impl.c +++ b/src/crypto_impl.c @@ -79,6 +79,8 @@ int sqlcipher_cipher_ctx_key_derive(codec_ctx *, cipher_ctx *); /* prototype for pager HMAC function */ int sqlcipher_page_hmac(cipher_ctx *, Pgno, unsigned char *, int, unsigned char *); +static int default_use_hmac = DEFAULT_USE_HMAC; + struct codec_ctx { int kdf_salt_sz; int page_sz; @@ -98,6 +100,14 @@ void sqlcipher_activate() { sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); } +/* fixed time zero memory check tests every position of a memory segement + matches a single value (i.e. the memory is all zeros)*/ +int sqlcipher_ismemset(const unsigned char *a0, unsigned char value, int len) { + int i = 0, noMatch = 0; + for(i = 0; i < len; i++) noMatch = (noMatch || (a0[i] != value)); + return noMatch; +} + /* fixed time memory comparison routine */ int sqlcipher_memcmp(const unsigned char *a0, const unsigned char *a1, int len) { int i = 0, noMatch = 0; @@ -183,7 +193,7 @@ int sqlcipher_cipher_ctx_init(cipher_ctx **iCtx) { */ void sqlcipher_cipher_ctx_free(cipher_ctx **iCtx) { cipher_ctx *ctx = *iCtx; - CODEC_TRACE(("cipher_ctx_free: entered iCtx=%d\n", iCtx)); + CODEC_TRACE(("cipher_ctx_free: entered iCtx=%p\n", iCtx)); sqlcipher_free(ctx->key, ctx->key_sz); sqlcipher_free(ctx->hmac_key, ctx->key_sz); sqlcipher_free(ctx->pass, ctx->pass_sz); @@ -197,7 +207,7 @@ void sqlcipher_cipher_ctx_free(cipher_ctx **iCtx) { * returns 1 otherwise */ int sqlcipher_cipher_ctx_cmp(cipher_ctx *c1, cipher_ctx *c2) { - CODEC_TRACE(("sqlcipher_cipher_ctx_cmp: entered c1=%d c2=%d\n", c1, c2)); + CODEC_TRACE(("sqlcipher_cipher_ctx_cmp: entered c1=%p c2=%p\n", c1, c2)); if( c1->evp_cipher == c2->evp_cipher @@ -206,6 +216,8 @@ int sqlcipher_cipher_ctx_cmp(cipher_ctx *c1, cipher_ctx *c2) { && c1->fast_kdf_iter == c2->fast_kdf_iter && c1->key_sz == c2->key_sz && c1->pass_sz == c2->pass_sz + && c1->use_hmac == c2->use_hmac + && c1->hmac_sz == c2->hmac_sz && ( c1->pass == c2->pass || !sqlcipher_memcmp((const unsigned char*)c1->pass, @@ -228,7 +240,7 @@ int sqlcipher_cipher_ctx_copy(cipher_ctx *target, cipher_ctx *source) { void *key = target->key; void *hmac_key = target->hmac_key; - CODEC_TRACE(("sqlcipher_cipher_ctx_copy: entered target=%d, source=%d\n", target, source)); + CODEC_TRACE(("sqlcipher_cipher_ctx_copy: entered target=%p, source=%p\n", target, source)); sqlcipher_free(target->pass, target->pass_sz); memcpy(target, source, sizeof(cipher_ctx)); @@ -325,7 +337,12 @@ int sqlcipher_codec_ctx_set_fast_kdf_iter(codec_ctx *ctx, int fast_kdf_iter, int return SQLITE_OK; } +/* set the global default flag for HMAC */ +void sqlcipher_set_default_use_hmac(int use) { + default_use_hmac = use; +} +/* set the codec flag for whether this individual database should be using hmac */ int sqlcipher_codec_ctx_set_use_hmac(codec_ctx *ctx, int use) { int reserve = EVP_MAX_IV_LENGTH; /* base reserve size will be IV only */ @@ -346,7 +363,9 @@ int sqlcipher_codec_ctx_set_use_hmac(codec_ctx *ctx, int use) { } void sqlcipher_codec_ctx_set_error(codec_ctx *ctx, int error) { - ctx->pBt->db->errCode = error; + CODEC_TRACE(("sqlcipher_codec_ctx_set_error: ctx=%p, error=%d\n", ctx, error)); + sqlite3pager_sqlite3PagerSetError(ctx->pBt->pBt->pPager, error); + ctx->pBt->pBt->db->errCode = error; } int sqlcipher_codec_ctx_get_pagesize(codec_ctx *ctx) { @@ -432,7 +451,7 @@ int sqlcipher_codec_ctx_init(codec_ctx **iCtx, Db *pDb, Pager *pPager, sqlite3_f /* Use HMAC signatures by default. Note that codec_set_use_hmac will implicity call codec_set_page_size to set the default */ - if((rc = sqlcipher_codec_ctx_set_use_hmac(ctx, DEFAULT_USE_HMAC)) != SQLITE_OK) return rc; + if((rc = sqlcipher_codec_ctx_set_use_hmac(ctx, default_use_hmac)) != SQLITE_OK) return rc; if((rc = sqlcipher_cipher_ctx_copy(ctx->write_ctx, ctx->read_ctx)) != SQLITE_OK) return rc; @@ -445,7 +464,7 @@ int sqlcipher_codec_ctx_init(codec_ctx **iCtx, Db *pDb, Pager *pPager, sqlite3_f */ void sqlcipher_codec_ctx_free(codec_ctx **iCtx) { codec_ctx *ctx = *iCtx; - CODEC_TRACE(("codec_ctx_free: entered iCtx=%d\n", iCtx)); + CODEC_TRACE(("codec_ctx_free: entered iCtx=%p\n", iCtx)); sqlcipher_free(ctx->kdf_salt, ctx->kdf_salt_sz); sqlcipher_free(ctx->hmac_kdf_salt, ctx->kdf_salt_sz); sqlcipher_free(ctx->buffer, 0); @@ -494,12 +513,13 @@ int sqlcipher_page_cipher(codec_ctx *ctx, int for_ctx, Pgno pgno, int mode, int out_start = out; /* note the original position of the output buffer pointer, as out will be rewritten during encryption */ CODEC_TRACE(("codec_cipher:entered pgno=%d, mode=%d, size=%d\n", pgno, mode, size)); + CODEC_HEXDUMP("codec_cipher: input page data", in, page_sz); - /* just copy raw data from in to out when key size is 0 - * i.e. during a rekey of a plaintext database */ + /* the key size should never be zero. If it is, error out. */ if(c_ctx->key_sz == 0) { - memcpy(out, in, size); - return SQLITE_OK; + CODEC_TRACE(("codec_cipher: error possible context corruption, key_sz is zero for pgno=%d\n", pgno)); + memset(out, 0, page_sz); + return SQLITE_ERROR; } if(mode == CIPHER_ENCRYPT) { @@ -516,14 +536,24 @@ int sqlcipher_page_cipher(codec_ctx *ctx, int for_ctx, Pgno pgno, int mode, int return SQLITE_ERROR; } - CODEC_TRACE(("codec_cipher: comparing hmac on in=%d out=%d hmac_sz=%d\n", hmac_in, hmac_out, c_ctx->hmac_sz)); - if(sqlcipher_memcmp(hmac_in, hmac_out, c_ctx->hmac_sz) != 0) { - /* the hmac check failed, which means the data was tampered with or - corrupted in some way. we will return an error, and zero out the page data - to force an error */ - memset(out, 0, page_sz); - CODEC_TRACE(("codec_cipher: hmac check failed for pgno=%d\n", pgno)); - return SQLITE_ERROR; + CODEC_TRACE(("codec_cipher: comparing hmac on in=%p out=%p hmac_sz=%d\n", hmac_in, hmac_out, c_ctx->hmac_sz)); + if(sqlcipher_memcmp(hmac_in, hmac_out, c_ctx->hmac_sz) != 0) { /* the hmac check failed */ + if(sqlcipher_ismemset(in, 0, page_sz) == 0) { + /* first check if the entire contents of the page is zeros. If so, this page + resulted from a short read (i.e. sqlite attempted to pull a page after the end of the file. these + short read failures must be ignored for autovaccum mode to work so wipe the output buffer + and return SQLITE_OK to skip the decryption step. */ + CODEC_TRACE(("codec_cipher: zeroed page (short read) for pgno %d, encryption but returning SQLITE_OK\n", pgno)); + memset(out, 0, page_sz); + return SQLITE_OK; + } else { + /* if the page memory is not all zeros, it means the there was data and a hmac on the page. + since the check failed, the page was either tampered with or corrupted. wipe the output buffer, + and return SQLITE_ERROR to the caller */ + CODEC_TRACE(("codec_cipher: hmac check failed for pgno=%d returning SQLITE_ERROR\n", pgno)); + memset(out, 0, page_sz); + return SQLITE_ERROR; + } } } @@ -542,6 +572,8 @@ int sqlcipher_page_cipher(codec_ctx *ctx, int for_ctx, Pgno pgno, int mode, int sqlcipher_page_hmac(c_ctx, pgno, out_start, size + c_ctx->iv_sz, hmac_out); } + CODEC_HEXDUMP("codec_cipher: output page data", out_start, page_sz); + return SQLITE_OK; } @@ -558,8 +590,8 @@ int sqlcipher_page_cipher(codec_ctx *ctx, int for_ctx, Pgno pgno, int mode, int */ int sqlcipher_cipher_ctx_key_derive(codec_ctx *ctx, cipher_ctx *c_ctx) { CODEC_TRACE(("codec_key_derive: entered c_ctx->pass=%s, c_ctx->pass_sz=%d \ - ctx->kdf_salt=%d ctx->kdf_salt_sz=%d c_ctx->kdf_iter=%d \ - ctx->hmac_kdf_salt=%d, c_ctx->fast_kdf_iter=%d c_ctx->key_sz=%d\n", + ctx->kdf_salt=%p ctx->kdf_salt_sz=%d c_ctx->kdf_iter=%d \ + ctx->hmac_kdf_salt=%p, c_ctx->fast_kdf_iter=%d c_ctx->key_sz=%d\n", c_ctx->pass, c_ctx->pass_sz, ctx->kdf_salt, ctx->kdf_salt_sz, c_ctx->kdf_iter, ctx->hmac_kdf_salt, c_ctx->fast_kdf_iter, c_ctx->key_sz)); diff --git a/src/delete.c b/src/delete.c index 147a5ca..eead485 100644 --- a/src/delete.c +++ b/src/delete.c @@ -148,7 +148,6 @@ Expr *sqlite3LimitWhere( */ if( pOrderBy && (pLimit == 0) ) { sqlite3ErrorMsg(pParse, "ORDER BY without LIMIT on %s", zStmtType); - pParse->parseError = 1; goto limit_where_cleanup_2; } @@ -375,7 +374,7 @@ void sqlite3DeleteFrom( pParse, pTabList, pWhere, 0, 0, WHERE_DUPLICATES_OK ); if( pWInfo==0 ) goto delete_from_cleanup; - regRowid = sqlite3ExprCodeGetColumn(pParse, pTab, -1, iCur, iRowid); + regRowid = sqlite3ExprCodeGetColumn(pParse, pTab, -1, iCur, iRowid, 0); sqlite3VdbeAddOp2(v, OP_RowSetAdd, iRowSet, regRowid); if( db->flags & SQLITE_CountRows ){ sqlite3VdbeAddOp2(v, OP_AddImm, memCnt, 1); @@ -484,8 +484,14 @@ Expr *sqlite3PExpr( Expr *pRight, /* Right operand */ const Token *pToken /* Argument token */ ){ - Expr *p = sqlite3ExprAlloc(pParse->db, op, pToken, 1); - sqlite3ExprAttachSubtrees(pParse->db, p, pLeft, pRight); + Expr *p; + if( op==TK_AND && pLeft && pRight ){ + /* Take advantage of short-circuit false optimization for AND */ + p = sqlite3ExprAnd(pParse->db, pLeft, pRight); + }else{ + p = sqlite3ExprAlloc(pParse->db, op, pToken, 1); + sqlite3ExprAttachSubtrees(pParse->db, p, pLeft, pRight); + } if( p ) { sqlite3ExprCheckHeight(pParse, p->nHeight); } @@ -493,14 +499,40 @@ Expr *sqlite3PExpr( } /* +** Return 1 if an expression must be FALSE in all cases and 0 if the +** expression might be true. This is an optimization. If is OK to +** return 0 here even if the expression really is always false (a +** false negative). But it is a bug to return 1 if the expression +** might be true in some rare circumstances (a false positive.) +** +** Note that if the expression is part of conditional for a +** LEFT JOIN, then we cannot determine at compile-time whether or not +** is it true or false, so always return 0. +*/ +static int exprAlwaysFalse(Expr *p){ + int v = 0; + if( ExprHasProperty(p, EP_FromJoin) ) return 0; + if( !sqlite3ExprIsInteger(p, &v) ) return 0; + return v==0; +} + +/* ** Join two expressions using an AND operator. If either expression is ** NULL, then just return the other expression. +** +** If one side or the other of the AND is known to be false, then instead +** of returning an AND expression, just return a constant expression with +** a value of false. */ Expr *sqlite3ExprAnd(sqlite3 *db, Expr *pLeft, Expr *pRight){ if( pLeft==0 ){ return pRight; }else if( pRight==0 ){ return pLeft; + }else if( exprAlwaysFalse(pLeft) || exprAlwaysFalse(pRight) ){ + sqlite3ExprDelete(db, pLeft); + sqlite3ExprDelete(db, pRight); + return sqlite3ExprAlloc(db, TK_INTEGER, &sqlite3IntTokens[0], 0); }else{ Expr *pNew = sqlite3ExprAlloc(db, TK_AND, 0, 0); sqlite3ExprAttachSubtrees(db, pNew, pLeft, pRight); @@ -856,8 +888,9 @@ ExprList *sqlite3ExprListDup(sqlite3 *db, ExprList *p, int flags){ pNew = sqlite3DbMallocRaw(db, sizeof(*pNew) ); if( pNew==0 ) return 0; pNew->iECursor = 0; - pNew->nExpr = pNew->nAlloc = p->nExpr; - pNew->a = pItem = sqlite3DbMallocRaw(db, p->nExpr*sizeof(p->a[0]) ); + pNew->nExpr = i = p->nExpr; + if( (flags & EXPRDUP_REDUCE)==0 ) for(i=1; i<p->nExpr; i+=i){} + pNew->a = pItem = sqlite3DbMallocRaw(db, i*sizeof(p->a[0]) ); if( pItem==0 ){ sqlite3DbFree(db, pNew); return 0; @@ -870,7 +903,7 @@ ExprList *sqlite3ExprListDup(sqlite3 *db, ExprList *p, int flags){ pItem->zSpan = sqlite3DbStrDup(db, pOldItem->zSpan); pItem->sortOrder = pOldItem->sortOrder; pItem->done = 0; - pItem->iCol = pOldItem->iCol; + pItem->iOrderByCol = pOldItem->iOrderByCol; pItem->iAlias = pOldItem->iAlias; } return pNew; @@ -925,12 +958,15 @@ IdList *sqlite3IdListDup(sqlite3 *db, IdList *p){ if( p==0 ) return 0; pNew = sqlite3DbMallocRaw(db, sizeof(*pNew) ); if( pNew==0 ) return 0; - pNew->nId = pNew->nAlloc = p->nId; + pNew->nId = p->nId; pNew->a = sqlite3DbMallocRaw(db, p->nId*sizeof(p->a[0]) ); if( pNew->a==0 ){ sqlite3DbFree(db, pNew); return 0; } + /* Note that because the size of the allocation for p->a[] is not + ** necessarily a power of two, sqlite3IdListAppend() may not be called + ** on the duplicate created by this function. */ for(i=0; i<p->nId; i++){ struct IdList_item *pNewItem = &pNew->a[i]; struct IdList_item *pOldItem = &p->a[i]; @@ -940,7 +976,7 @@ IdList *sqlite3IdListDup(sqlite3 *db, IdList *p){ return pNew; } Select *sqlite3SelectDup(sqlite3 *db, Select *p, int flags){ - Select *pNew; + Select *pNew, *pPrior; if( p==0 ) return 0; pNew = sqlite3DbMallocRaw(db, sizeof(*p) ); if( pNew==0 ) return 0; @@ -951,7 +987,9 @@ Select *sqlite3SelectDup(sqlite3 *db, Select *p, int flags){ pNew->pHaving = sqlite3ExprDup(db, p->pHaving, flags); pNew->pOrderBy = sqlite3ExprListDup(db, p->pOrderBy, flags); pNew->op = p->op; - pNew->pPrior = sqlite3SelectDup(db, p->pPrior, flags); + pNew->pPrior = pPrior = sqlite3SelectDup(db, p->pPrior, flags); + if( pPrior ) pPrior->pNext = pNew; + pNew->pNext = 0; pNew->pLimit = sqlite3ExprDup(db, p->pLimit, flags); pNew->pOffset = sqlite3ExprDup(db, p->pOffset, flags); pNew->iLimit = 0; @@ -990,17 +1028,16 @@ ExprList *sqlite3ExprListAppend( if( pList==0 ){ goto no_mem; } - assert( pList->nAlloc==0 ); - } - if( pList->nAlloc<=pList->nExpr ){ + pList->a = sqlite3DbMallocRaw(db, sizeof(pList->a[0])); + if( pList->a==0 ) goto no_mem; + }else if( (pList->nExpr & (pList->nExpr-1))==0 ){ struct ExprList_item *a; - int n = pList->nAlloc*2 + 4; - a = sqlite3DbRealloc(db, pList->a, n*sizeof(pList->a[0])); + assert( pList->nExpr>0 ); + a = sqlite3DbRealloc(db, pList->a, pList->nExpr*2*sizeof(pList->a[0])); if( a==0 ){ goto no_mem; } pList->a = a; - pList->nAlloc = sqlite3DbMallocSize(db, a)/sizeof(a[0]); } assert( pList->a!=0 ); if( 1 ){ @@ -1091,8 +1128,7 @@ void sqlite3ExprListDelete(sqlite3 *db, ExprList *pList){ int i; struct ExprList_item *pItem; if( pList==0 ) return; - assert( pList->a!=0 || (pList->nExpr==0 && pList->nAlloc==0) ); - assert( pList->nExpr<=pList->nAlloc ); + assert( pList->a!=0 || pList->nExpr==0 ); for(pItem=pList->a, i=0; i<pList->nExpr; i++, pItem++){ sqlite3ExprDelete(db, pItem->pExpr); sqlite3DbFree(db, pItem->zName); @@ -1374,6 +1410,15 @@ static int isCandidateForInOpt(Select *p){ #endif /* SQLITE_OMIT_SUBQUERY */ /* +** Code an OP_Once instruction and allocate space for its flag. Return the +** address of the new instruction. +*/ +int sqlite3CodeOnce(Parse *pParse){ + Vdbe *v = sqlite3GetVdbe(pParse); /* Virtual machine being coded */ + return sqlite3VdbeAddOp1(v, OP_Once, pParse->nOnce++); +} + +/* ** This function is used by the implementation of the IN (...) operator. ** It's job is to find or create a b-tree structure that may be used ** either to test for membership of the (...) set or to iterate through @@ -1433,6 +1478,7 @@ int sqlite3FindInIndex(Parse *pParse, Expr *pX, int *prNotFound){ int eType = 0; /* Type of RHS table. IN_INDEX_* */ int iTab = pParse->nTab++; /* Cursor of the RHS table */ int mustBeUnique = (prNotFound==0); /* True if RHS must be unique */ + Vdbe *v = sqlite3GetVdbe(pParse); /* Virtual machine being coded */ assert( pX->op==TK_IN ); @@ -1443,7 +1489,6 @@ int sqlite3FindInIndex(Parse *pParse, Expr *pX, int *prNotFound){ p = (ExprHasProperty(pX, EP_xIsSelect) ? pX->x.pSelect : 0); if( ALWAYS(pParse->nErr==0) && isCandidateForInOpt(p) ){ sqlite3 *db = pParse->db; /* Database connection */ - Vdbe *v = sqlite3GetVdbe(pParse); /* Virtual machine being coded */ Table *pTab; /* Table <table>. */ Expr *pExpr; /* Expression <column> */ int iCol; /* Index of column <column> */ @@ -1468,10 +1513,9 @@ int sqlite3FindInIndex(Parse *pParse, Expr *pX, int *prNotFound){ */ assert(v); if( iCol<0 ){ - int iMem = ++pParse->nMem; int iAddr; - iAddr = sqlite3VdbeAddOp1(v, OP_Once, iMem); + iAddr = sqlite3CodeOnce(pParse); sqlite3OpenTable(pParse, iTab, iDb, pTab, OP_OpenRead); eType = IN_INDEX_ROWID; @@ -1497,12 +1541,11 @@ int sqlite3FindInIndex(Parse *pParse, Expr *pX, int *prNotFound){ && sqlite3FindCollSeq(db, ENC(db), pIdx->azColl[0], 0)==pReq && (!mustBeUnique || (pIdx->nColumn==1 && pIdx->onError!=OE_None)) ){ - int iMem = ++pParse->nMem; int iAddr; char *pKey; pKey = (char *)sqlite3IndexKeyinfo(pParse, pIdx); - iAddr = sqlite3VdbeAddOp1(v, OP_Once, iMem); + iAddr = sqlite3CodeOnce(pParse); sqlite3VdbeAddOp4(v, OP_OpenRead, iTab, pIdx->tnum, iDb, pKey,P4_KEYINFO_HANDOFF); @@ -1512,6 +1555,7 @@ int sqlite3FindInIndex(Parse *pParse, Expr *pX, int *prNotFound){ sqlite3VdbeJumpHere(v, iAddr); if( prNotFound && !pTab->aCol[iCol].notNull ){ *prNotFound = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Null, 0, *prNotFound); } } } @@ -1527,6 +1571,7 @@ int sqlite3FindInIndex(Parse *pParse, Expr *pX, int *prNotFound){ eType = IN_INDEX_EPH; if( prNotFound ){ *prNotFound = rMayHaveNull = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Null, 0, *prNotFound); }else{ testcase( pParse->nQueryLoop>(double)1 ); pParse->nQueryLoop = (double)1; @@ -1599,9 +1644,8 @@ int sqlite3CodeSubselect( ** If all of the above are false, then we can run this code just once ** save the results, and reuse the same result on subsequent invocations. */ - if( !ExprHasAnyProperty(pExpr, EP_VarSelect) && !pParse->pTriggerTab ){ - int mem = ++pParse->nMem; - testAddr = sqlite3VdbeAddOp1(v, OP_Once, mem); + if( !ExprHasAnyProperty(pExpr, EP_VarSelect) ){ + testAddr = sqlite3CodeOnce(pParse); } #ifndef SQLITE_OMIT_EXPLAIN @@ -2020,15 +2064,6 @@ void sqlite3ExprCacheStore(Parse *pParse, int iTab, int iCol, int iReg){ */ #ifndef NDEBUG for(i=0, p=pParse->aColCache; i<SQLITE_N_COLCACHE; i++, p++){ -#if 0 /* This code wold remove the entry from the cache if it existed */ - if( p->iReg && p->iTable==iTab && p->iColumn==iCol ){ - cacheEntryClear(pParse, p); - p->iLevel = pParse->iCacheLevel; - p->iReg = iReg; - p->lru = pParse->iCacheCnt++; - return; - } -#endif assert( p->iReg==0 || p->iTable!=iTab || p->iColumn!=iCol ); } #endif @@ -2163,7 +2198,8 @@ int sqlite3ExprCodeGetColumn( Table *pTab, /* Description of the table we are reading from */ int iColumn, /* Index of the table column */ int iTable, /* The cursor pointing to the table */ - int iReg /* Store results here */ + int iReg, /* Store results here */ + u8 p5 /* P5 value for OP_Column */ ){ Vdbe *v = pParse->pVdbe; int i; @@ -2178,7 +2214,11 @@ int sqlite3ExprCodeGetColumn( } assert( v!=0 ); sqlite3ExprCodeGetColumnOfTable(v, pTab, iTable, iColumn, iReg); - sqlite3ExprCacheStore(pParse, iTable, iColumn, iReg); + if( p5 ){ + sqlite3VdbeChangeP5(v, p5); + }else{ + sqlite3ExprCacheStore(pParse, iTable, iColumn, iReg); + } return iReg; } @@ -2306,7 +2346,8 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ inReg = pExpr->iColumn + pParse->ckBase; }else{ inReg = sqlite3ExprCodeGetColumn(pParse, pExpr->pTab, - pExpr->iColumn, pExpr->iTable, target); + pExpr->iColumn, pExpr->iTable, target, + pExpr->op2); } break; } @@ -2583,6 +2624,25 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ if( pFarg ){ r1 = sqlite3GetTempRange(pParse, nFarg); + + /* For length() and typeof() functions with a column argument, + ** set the P5 parameter to the OP_Column opcode to OPFLAG_LENGTHARG + ** or OPFLAG_TYPEOFARG respectively, to avoid unnecessary data + ** loading. + */ + if( (pDef->flags & (SQLITE_FUNC_LENGTH|SQLITE_FUNC_TYPEOF))!=0 ){ + u8 exprOp; + assert( nFarg==1 ); + assert( pFarg->a[0].pExpr!=0 ); + exprOp = pFarg->a[0].pExpr->op; + if( exprOp==TK_COLUMN || exprOp==TK_AGG_COLUMN ){ + assert( SQLITE_FUNC_LENGTH==OPFLAG_LENGTHARG ); + assert( SQLITE_FUNC_TYPEOF==OPFLAG_TYPEOFARG ); + testcase( pDef->flags==SQLITE_FUNC_LENGTH ); + pFarg->a[0].pExpr->op2 = pDef->flags; + } + } + sqlite3ExprCachePush(pParse); /* Ticket 2ea2425d34be */ sqlite3ExprCodeExprList(pParse, pFarg, r1, 1); sqlite3ExprCachePop(pParse, 1); /* Ticket 2ea2425d34be */ @@ -2939,6 +2999,264 @@ int sqlite3ExprCodeAndCache(Parse *pParse, Expr *pExpr, int target){ return inReg; } +#if defined(SQLITE_ENABLE_TREE_EXPLAIN) +/* +** Generate a human-readable explanation of an expression tree. +*/ +void sqlite3ExplainExpr(Vdbe *pOut, Expr *pExpr){ + int op; /* The opcode being coded */ + const char *zBinOp = 0; /* Binary operator */ + const char *zUniOp = 0; /* Unary operator */ + if( pExpr==0 ){ + op = TK_NULL; + }else{ + op = pExpr->op; + } + switch( op ){ + case TK_AGG_COLUMN: { + sqlite3ExplainPrintf(pOut, "AGG{%d:%d}", + pExpr->iTable, pExpr->iColumn); + break; + } + case TK_COLUMN: { + if( pExpr->iTable<0 ){ + /* This only happens when coding check constraints */ + sqlite3ExplainPrintf(pOut, "COLUMN(%d)", pExpr->iColumn); + }else{ + sqlite3ExplainPrintf(pOut, "{%d:%d}", + pExpr->iTable, pExpr->iColumn); + } + break; + } + case TK_INTEGER: { + if( pExpr->flags & EP_IntValue ){ + sqlite3ExplainPrintf(pOut, "%d", pExpr->u.iValue); + }else{ + sqlite3ExplainPrintf(pOut, "%s", pExpr->u.zToken); + } + break; + } +#ifndef SQLITE_OMIT_FLOATING_POINT + case TK_FLOAT: { + sqlite3ExplainPrintf(pOut,"%s", pExpr->u.zToken); + break; + } +#endif + case TK_STRING: { + sqlite3ExplainPrintf(pOut,"%Q", pExpr->u.zToken); + break; + } + case TK_NULL: { + sqlite3ExplainPrintf(pOut,"NULL"); + break; + } +#ifndef SQLITE_OMIT_BLOB_LITERAL + case TK_BLOB: { + sqlite3ExplainPrintf(pOut,"%s", pExpr->u.zToken); + break; + } +#endif + case TK_VARIABLE: { + sqlite3ExplainPrintf(pOut,"VARIABLE(%s,%d)", + pExpr->u.zToken, pExpr->iColumn); + break; + } + case TK_REGISTER: { + sqlite3ExplainPrintf(pOut,"REGISTER(%d)", pExpr->iTable); + break; + } + case TK_AS: { + sqlite3ExplainExpr(pOut, pExpr->pLeft); + break; + } +#ifndef SQLITE_OMIT_CAST + case TK_CAST: { + /* Expressions of the form: CAST(pLeft AS token) */ + const char *zAff = "unk"; + switch( sqlite3AffinityType(pExpr->u.zToken) ){ + case SQLITE_AFF_TEXT: zAff = "TEXT"; break; + case SQLITE_AFF_NONE: zAff = "NONE"; break; + case SQLITE_AFF_NUMERIC: zAff = "NUMERIC"; break; + case SQLITE_AFF_INTEGER: zAff = "INTEGER"; break; + case SQLITE_AFF_REAL: zAff = "REAL"; break; + } + sqlite3ExplainPrintf(pOut, "CAST-%s(", zAff); + sqlite3ExplainExpr(pOut, pExpr->pLeft); + sqlite3ExplainPrintf(pOut, ")"); + break; + } +#endif /* SQLITE_OMIT_CAST */ + case TK_LT: zBinOp = "LT"; break; + case TK_LE: zBinOp = "LE"; break; + case TK_GT: zBinOp = "GT"; break; + case TK_GE: zBinOp = "GE"; break; + case TK_NE: zBinOp = "NE"; break; + case TK_EQ: zBinOp = "EQ"; break; + case TK_IS: zBinOp = "IS"; break; + case TK_ISNOT: zBinOp = "ISNOT"; break; + case TK_AND: zBinOp = "AND"; break; + case TK_OR: zBinOp = "OR"; break; + case TK_PLUS: zBinOp = "ADD"; break; + case TK_STAR: zBinOp = "MUL"; break; + case TK_MINUS: zBinOp = "SUB"; break; + case TK_REM: zBinOp = "REM"; break; + case TK_BITAND: zBinOp = "BITAND"; break; + case TK_BITOR: zBinOp = "BITOR"; break; + case TK_SLASH: zBinOp = "DIV"; break; + case TK_LSHIFT: zBinOp = "LSHIFT"; break; + case TK_RSHIFT: zBinOp = "RSHIFT"; break; + case TK_CONCAT: zBinOp = "CONCAT"; break; + + case TK_UMINUS: zUniOp = "UMINUS"; break; + case TK_UPLUS: zUniOp = "UPLUS"; break; + case TK_BITNOT: zUniOp = "BITNOT"; break; + case TK_NOT: zUniOp = "NOT"; break; + case TK_ISNULL: zUniOp = "ISNULL"; break; + case TK_NOTNULL: zUniOp = "NOTNULL"; break; + + case TK_AGG_FUNCTION: + case TK_CONST_FUNC: + case TK_FUNCTION: { + ExprList *pFarg; /* List of function arguments */ + if( ExprHasAnyProperty(pExpr, EP_TokenOnly) ){ + pFarg = 0; + }else{ + pFarg = pExpr->x.pList; + } + sqlite3ExplainPrintf(pOut, "%sFUNCTION:%s(", + op==TK_AGG_FUNCTION ? "AGG_" : "", + pExpr->u.zToken); + if( pFarg ){ + sqlite3ExplainExprList(pOut, pFarg); + } + sqlite3ExplainPrintf(pOut, ")"); + break; + } +#ifndef SQLITE_OMIT_SUBQUERY + case TK_EXISTS: { + sqlite3ExplainPrintf(pOut, "EXISTS("); + sqlite3ExplainSelect(pOut, pExpr->x.pSelect); + sqlite3ExplainPrintf(pOut,")"); + break; + } + case TK_SELECT: { + sqlite3ExplainPrintf(pOut, "("); + sqlite3ExplainSelect(pOut, pExpr->x.pSelect); + sqlite3ExplainPrintf(pOut, ")"); + break; + } + case TK_IN: { + sqlite3ExplainPrintf(pOut, "IN("); + sqlite3ExplainExpr(pOut, pExpr->pLeft); + sqlite3ExplainPrintf(pOut, ","); + if( ExprHasProperty(pExpr, EP_xIsSelect) ){ + sqlite3ExplainSelect(pOut, pExpr->x.pSelect); + }else{ + sqlite3ExplainExprList(pOut, pExpr->x.pList); + } + sqlite3ExplainPrintf(pOut, ")"); + break; + } +#endif /* SQLITE_OMIT_SUBQUERY */ + + /* + ** x BETWEEN y AND z + ** + ** This is equivalent to + ** + ** x>=y AND x<=z + ** + ** X is stored in pExpr->pLeft. + ** Y is stored in pExpr->pList->a[0].pExpr. + ** Z is stored in pExpr->pList->a[1].pExpr. + */ + case TK_BETWEEN: { + Expr *pX = pExpr->pLeft; + Expr *pY = pExpr->x.pList->a[0].pExpr; + Expr *pZ = pExpr->x.pList->a[1].pExpr; + sqlite3ExplainPrintf(pOut, "BETWEEN("); + sqlite3ExplainExpr(pOut, pX); + sqlite3ExplainPrintf(pOut, ","); + sqlite3ExplainExpr(pOut, pY); + sqlite3ExplainPrintf(pOut, ","); + sqlite3ExplainExpr(pOut, pZ); + sqlite3ExplainPrintf(pOut, ")"); + break; + } + case TK_TRIGGER: { + /* If the opcode is TK_TRIGGER, then the expression is a reference + ** to a column in the new.* or old.* pseudo-tables available to + ** trigger programs. In this case Expr.iTable is set to 1 for the + ** new.* pseudo-table, or 0 for the old.* pseudo-table. Expr.iColumn + ** is set to the column of the pseudo-table to read, or to -1 to + ** read the rowid field. + */ + sqlite3ExplainPrintf(pOut, "%s(%d)", + pExpr->iTable ? "NEW" : "OLD", pExpr->iColumn); + break; + } + case TK_CASE: { + sqlite3ExplainPrintf(pOut, "CASE("); + sqlite3ExplainExpr(pOut, pExpr->pLeft); + sqlite3ExplainPrintf(pOut, ","); + sqlite3ExplainExprList(pOut, pExpr->x.pList); + break; + } +#ifndef SQLITE_OMIT_TRIGGER + case TK_RAISE: { + const char *zType = "unk"; + switch( pExpr->affinity ){ + case OE_Rollback: zType = "rollback"; break; + case OE_Abort: zType = "abort"; break; + case OE_Fail: zType = "fail"; break; + case OE_Ignore: zType = "ignore"; break; + } + sqlite3ExplainPrintf(pOut, "RAISE-%s(%s)", zType, pExpr->u.zToken); + break; + } +#endif + } + if( zBinOp ){ + sqlite3ExplainPrintf(pOut,"%s(", zBinOp); + sqlite3ExplainExpr(pOut, pExpr->pLeft); + sqlite3ExplainPrintf(pOut,","); + sqlite3ExplainExpr(pOut, pExpr->pRight); + sqlite3ExplainPrintf(pOut,")"); + }else if( zUniOp ){ + sqlite3ExplainPrintf(pOut,"%s(", zUniOp); + sqlite3ExplainExpr(pOut, pExpr->pLeft); + sqlite3ExplainPrintf(pOut,")"); + } +} +#endif /* defined(SQLITE_ENABLE_TREE_EXPLAIN) */ + +#if defined(SQLITE_ENABLE_TREE_EXPLAIN) +/* +** Generate a human-readable explanation of an expression list. +*/ +void sqlite3ExplainExprList(Vdbe *pOut, ExprList *pList){ + int i; + if( pList==0 || pList->nExpr==0 ){ + sqlite3ExplainPrintf(pOut, "(empty-list)"); + return; + }else if( pList->nExpr==1 ){ + sqlite3ExplainExpr(pOut, pList->a[0].pExpr); + }else{ + sqlite3ExplainPush(pOut); + for(i=0; i<pList->nExpr; i++){ + sqlite3ExplainPrintf(pOut, "item[%d] = ", i); + sqlite3ExplainPush(pOut); + sqlite3ExplainExpr(pOut, pList->a[i].pExpr); + sqlite3ExplainPop(pOut); + if( i<pList->nExpr-1 ){ + sqlite3ExplainNL(pOut); + } + } + sqlite3ExplainPop(pOut); + } +} +#endif /* SQLITE_DEBUG */ + /* ** Return TRUE if pExpr is an constant expression that is appropriate ** for factoring out of a loop. Appropriate expressions are: @@ -3460,7 +3778,7 @@ int sqlite3ExprCompare(Expr *pA, Expr *pB){ if( !ExprHasProperty(pB, EP_IntValue) || pA->u.iValue!=pB->u.iValue ){ return 2; } - }else if( pA->op!=TK_COLUMN && pA->u.zToken ){ + }else if( pA->op!=TK_COLUMN && ALWAYS(pA->op!=TK_AGG_COLUMN) && pA->u.zToken){ if( ExprHasProperty(pB, EP_IntValue) || NEVER(pB->u.zToken==0) ) return 2; if( strcmp(pA->u.zToken,pB->u.zToken)!=0 ){ return 2; @@ -3498,6 +3816,41 @@ int sqlite3ExprListCompare(ExprList *pA, ExprList *pB){ } /* +** This is the expression callback for sqlite3FunctionUsesOtherSrc(). +** +** Determine if an expression references any table other than one of the +** tables in pWalker->u.pSrcList and abort if it does. +*/ +static int exprUsesOtherSrc(Walker *pWalker, Expr *pExpr){ + if( pExpr->op==TK_COLUMN || pExpr->op==TK_AGG_COLUMN ){ + int i; + SrcList *pSrc = pWalker->u.pSrcList; + for(i=0; i<pSrc->nSrc; i++){ + if( pExpr->iTable==pSrc->a[i].iCursor ) return WRC_Continue; + } + return WRC_Abort; + }else{ + return WRC_Continue; + } +} + +/* +** Determine if any of the arguments to the pExpr Function references +** any SrcList other than pSrcList. Return true if they do. Return +** false if pExpr has no argument or has only constant arguments or +** only references tables named in pSrcList. +*/ +static int sqlite3FunctionUsesOtherSrc(Expr *pExpr, SrcList *pSrcList){ + Walker w; + assert( pExpr->op==TK_AGG_FUNCTION ); + memset(&w, 0, sizeof(w)); + w.xExprCallback = exprUsesOtherSrc; + w.u.pSrcList = pSrcList; + if( sqlite3WalkExprList(&w, pExpr->x.pList)!=WRC_Continue ) return 1; + return 0; +} + +/* ** Add a new element to the pAggInfo->aCol[] array. Return the index of ** the new element. Return a negative number if malloc fails. */ @@ -3507,9 +3860,7 @@ static int addAggInfoColumn(sqlite3 *db, AggInfo *pInfo){ db, pInfo->aCol, sizeof(pInfo->aCol[0]), - 3, &pInfo->nColumn, - &pInfo->nColumnAlloc, &i ); return i; @@ -3525,9 +3876,7 @@ static int addAggInfoFunc(sqlite3 *db, AggInfo *pInfo){ db, pInfo->aFunc, sizeof(pInfo->aFunc[0]), - 3, &pInfo->nFunc, - &pInfo->nFuncAlloc, &i ); return i; @@ -3616,9 +3965,9 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ return WRC_Prune; } case TK_AGG_FUNCTION: { - /* The pNC->nDepth==0 test causes aggregate functions in subqueries - ** to be ignored */ - if( pNC->nDepth==0 ){ + if( (pNC->ncFlags & NC_InAggFunc)==0 + && !sqlite3FunctionUsesOtherSrc(pExpr, pSrcList) + ){ /* Check to see if pExpr is a duplicate of another aggregate ** function that is already in the pAggInfo structure */ @@ -3655,22 +4004,16 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ ExprSetIrreducible(pExpr); pExpr->iAgg = (i16)i; pExpr->pAggInfo = pAggInfo; - return WRC_Prune; } + return WRC_Prune; } } return WRC_Continue; } static int analyzeAggregatesInSelect(Walker *pWalker, Select *pSelect){ - NameContext *pNC = pWalker->u.pNC; - if( pNC->nDepth==0 ){ - pNC->nDepth++; - sqlite3WalkSelect(pWalker, pSelect); - pNC->nDepth--; - return WRC_Prune; - }else{ - return WRC_Continue; - } + UNUSED_PARAMETER(pWalker); + UNUSED_PARAMETER(pSelect); + return WRC_Continue; } /* @@ -3683,6 +4026,7 @@ static int analyzeAggregatesInSelect(Walker *pWalker, Select *pSelect){ */ void sqlite3ExprAnalyzeAggregates(NameContext *pNC, Expr *pExpr){ Walker w; + memset(&w, 0, sizeof(w)); w.xExprCallback = analyzeAggregate; w.xSelectCallback = analyzeAggregatesInSelect; w.u.pNC = pNC; @@ -3762,3 +4106,11 @@ void sqlite3ReleaseTempRange(Parse *pParse, int iReg, int nReg){ pParse->iRangeReg = iReg; } } + +/* +** Mark all temporary registers as being unavailable for reuse. +*/ +void sqlite3ClearTempRegCache(Parse *pParse){ + pParse->nTempReg = 0; + pParse->nRangeReg = 0; +} @@ -29,6 +29,14 @@ static CollSeq *sqlite3GetFuncCollSeq(sqlite3_context *context){ } /* +** Indicate that the accumulator load should be skipped on this +** iteration of the aggregate loop. +*/ +static void sqlite3SkipAccumulatorLoad(sqlite3_context *context){ + context->skipFlag = 1; +} + +/* ** Implementation of the non-aggregate min() and max() functions */ static void minmaxFunc( @@ -408,7 +416,7 @@ static void randomFunc( ** 2s complement of that positive value. The end result can ** therefore be no less than -9223372036854775807. */ - r = -(r ^ (((sqlite3_int64)1)<<63)); + r = -(r & LARGEST_INT64); } sqlite3_result_int64(context, r); } @@ -1334,11 +1342,12 @@ static void minmaxStep( Mem *pBest; UNUSED_PARAMETER(NotUsed); - if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; pBest = (Mem *)sqlite3_aggregate_context(context, sizeof(*pBest)); if( !pBest ) return; - if( pBest->flags ){ + if( sqlite3_value_type(argv[0])==SQLITE_NULL ){ + if( pBest->flags ) sqlite3SkipAccumulatorLoad(context); + }else if( pBest->flags ){ int max; int cmp; CollSeq *pColl = sqlite3GetFuncCollSeq(context); @@ -1354,6 +1363,8 @@ static void minmaxStep( cmp = sqlite3MemCompare(pBest, pArg, pColl); if( (max && cmp<0) || (!max && cmp>0) ){ sqlite3VdbeMemCopy(pBest, pArg); + }else{ + sqlite3SkipAccumulatorLoad(context); } }else{ sqlite3VdbeMemCopy(pBest, pArg); @@ -1363,7 +1374,7 @@ static void minMaxFinalize(sqlite3_context *context){ sqlite3_value *pRes; pRes = (sqlite3_value *)sqlite3_aggregate_context(context, 0); if( pRes ){ - if( ALWAYS(pRes->flags) ){ + if( pRes->flags ){ sqlite3_result_value(context, pRes); } sqlite3VdbeMemRelease(pRes); @@ -1537,8 +1548,8 @@ void sqlite3RegisterGlobalFunctions(void){ FUNCTION(max, -1, 1, 1, minmaxFunc ), FUNCTION(max, 0, 1, 1, 0 ), AGGREGATE(max, 1, 1, 1, minmaxStep, minMaxFinalize ), - FUNCTION(typeof, 1, 0, 0, typeofFunc ), - FUNCTION(length, 1, 0, 0, lengthFunc ), + FUNCTION2(typeof, 1, 0, 0, typeofFunc, SQLITE_FUNC_TYPEOF), + FUNCTION2(length, 1, 0, 0, lengthFunc, SQLITE_FUNC_LENGTH), FUNCTION(substr, 2, 0, 0, substrFunc ), FUNCTION(substr, 3, 0, 0, substrFunc ), FUNCTION(abs, 1, 0, 0, absFunc ), @@ -1550,11 +1561,9 @@ void sqlite3RegisterGlobalFunctions(void){ FUNCTION(lower, 1, 0, 0, lowerFunc ), FUNCTION(coalesce, 1, 0, 0, 0 ), FUNCTION(coalesce, 0, 0, 0, 0 ), -/* FUNCTION(coalesce, -1, 0, 0, ifnullFunc ), */ - {-1,SQLITE_UTF8,SQLITE_FUNC_COALESCE,0,0,ifnullFunc,0,0,"coalesce",0,0}, + FUNCTION2(coalesce, -1, 0, 0, ifnullFunc, SQLITE_FUNC_COALESCE), FUNCTION(hex, 1, 0, 0, hexFunc ), -/* FUNCTION(ifnull, 2, 0, 0, ifnullFunc ), */ - {2,SQLITE_UTF8,SQLITE_FUNC_COALESCE,0,0,ifnullFunc,0,0,"ifnull",0,0}, + FUNCTION2(ifnull, 2, 0, 0, ifnullFunc, SQLITE_FUNC_COALESCE), FUNCTION(random, 0, 0, 0, randomFunc ), FUNCTION(randomblob, 1, 0, 0, randomBlob ), FUNCTION(nullif, 2, 0, 1, nullifFunc ), diff --git a/src/global.c b/src/global.c index 1e691c4..7de0668 100644 --- a/src/global.c +++ b/src/global.c @@ -147,7 +147,7 @@ SQLITE_WSD struct Sqlite3Config sqlite3Config = { 500, /* nLookaside */ {0,0,0,0,0,0,0,0}, /* m */ {0,0,0,0,0,0,0,0,0}, /* mutex */ - {0,0,0,0,0,0,0,0,0,0,0}, /* pcache */ + {0,0,0,0,0,0,0,0,0,0,0,0,0},/* pcache2 */ (void*)0, /* pHeap */ 0, /* nHeap */ 0, 0, /* mnHeap, mxHeap */ diff --git a/src/insert.c b/src/insert.c index 277a852..a589c8a 100644 --- a/src/insert.c +++ b/src/insert.c @@ -47,7 +47,7 @@ void sqlite3OpenTable( ** 'd' INTEGER ** 'e' REAL ** -** An extra 'b' is appended to the end of the string to cover the +** An extra 'd' is appended to the end of the string to cover the ** rowid that appears as the last column in every index. ** ** Memory for the buffer containing the column index affinity string @@ -75,7 +75,7 @@ const char *sqlite3IndexAffinityStr(Vdbe *v, Index *pIdx){ for(n=0; n<pIdx->nColumn; n++){ pIdx->zColAff[n] = pTab->aCol[pIdx->aiColumn[n]].affinity; } - pIdx->zColAff[n++] = SQLITE_AFF_NONE; + pIdx->zColAff[n++] = SQLITE_AFF_INTEGER; pIdx->zColAff[n] = 0; } @@ -239,6 +239,7 @@ void sqlite3AutoincrementBegin(Parse *pParse){ memId = p->regCtr; assert( sqlite3SchemaMutexHeld(db, 0, pDb->pSchema) ); sqlite3OpenTable(pParse, 0, p->iDb, pDb->pSchema->pSeqTab, OP_OpenRead); + sqlite3VdbeAddOp3(v, OP_Null, 0, memId, memId+1); addr = sqlite3VdbeCurrentAddr(v); sqlite3VdbeAddOp4(v, OP_String8, 0, memId-1, 0, p->pTab->zName, 0); sqlite3VdbeAddOp2(v, OP_Rewind, 0, addr+9); @@ -1106,7 +1107,7 @@ insert_cleanup: ** cause sqlite3_exec() to return immediately ** with SQLITE_CONSTRAINT. ** -** any FAIL Sqlite_exec() returns immediately with a +** any FAIL Sqlite3_exec() returns immediately with a ** return code of SQLITE_CONSTRAINT. The ** transaction is not rolled back and any ** prior changes are retained. @@ -1156,9 +1157,11 @@ void sqlite3GenerateConstraintChecks( int regData; /* Register containing first data column */ int iCur; /* Table cursor number */ Index *pIdx; /* Pointer to one of the indices */ + sqlite3 *db; /* Database connection */ int seenReplace = 0; /* True if REPLACE is used to resolve INT PK conflict */ int regOldRowid = (rowidChng && isUpdate) ? rowidChng : regRowid; + db = pParse->db; v = sqlite3GetVdbe(pParse); assert( v!=0 ); assert( pTab->pSelect==0 ); /* This table is not a VIEW */ @@ -1191,7 +1194,7 @@ void sqlite3GenerateConstraintChecks( char *zMsg; sqlite3VdbeAddOp3(v, OP_HaltIfNull, SQLITE_CONSTRAINT, onError, regData+i); - zMsg = sqlite3MPrintf(pParse->db, "%s.%s may not be NULL", + zMsg = sqlite3MPrintf(db, "%s.%s may not be NULL", pTab->zName, pTab->aCol[i].zName); sqlite3VdbeChangeP4(v, -1, zMsg, P4_DYNAMIC); break; @@ -1213,18 +1216,27 @@ void sqlite3GenerateConstraintChecks( /* Test all CHECK constraints */ #ifndef SQLITE_OMIT_CHECK - if( pTab->pCheck && (pParse->db->flags & SQLITE_IgnoreChecks)==0 ){ - int allOk = sqlite3VdbeMakeLabel(v); + if( pTab->pCheck && (db->flags & SQLITE_IgnoreChecks)==0 ){ + ExprList *pCheck = pTab->pCheck; pParse->ckBase = regData; - sqlite3ExprIfTrue(pParse, pTab->pCheck, allOk, SQLITE_JUMPIFNULL); onError = overrideError!=OE_Default ? overrideError : OE_Abort; - if( onError==OE_Ignore ){ - sqlite3VdbeAddOp2(v, OP_Goto, 0, ignoreDest); - }else{ - if( onError==OE_Replace ) onError = OE_Abort; /* IMP: R-15569-63625 */ - sqlite3HaltConstraint(pParse, onError, 0, 0); + for(i=0; i<pCheck->nExpr; i++){ + int allOk = sqlite3VdbeMakeLabel(v); + sqlite3ExprIfTrue(pParse, pCheck->a[i].pExpr, allOk, SQLITE_JUMPIFNULL); + if( onError==OE_Ignore ){ + sqlite3VdbeAddOp2(v, OP_Goto, 0, ignoreDest); + }else{ + char *zConsName = pCheck->a[i].zName; + if( onError==OE_Replace ) onError = OE_Abort; /* IMP: R-15569-63625 */ + if( zConsName ){ + zConsName = sqlite3MPrintf(db, "constraint %s failed", zConsName); + }else{ + zConsName = 0; + } + sqlite3HaltConstraint(pParse, onError, zConsName, P4_DYNAMIC); + } + sqlite3VdbeResolveLabel(v, allOk); } - sqlite3VdbeResolveLabel(v, allOk); } #endif /* !defined(SQLITE_OMIT_CHECK) */ @@ -1280,7 +1292,7 @@ void sqlite3GenerateConstraintChecks( ** table. */ Trigger *pTrigger = 0; - if( pParse->db->flags&SQLITE_RecTriggers ){ + if( db->flags&SQLITE_RecTriggers ){ pTrigger = sqlite3TriggersExist(pParse, pTab, TK_DELETE, 0, 0); } if( pTrigger || sqlite3FkRequired(pParse, pTab, 0, 0) ){ @@ -1369,7 +1381,7 @@ void sqlite3GenerateConstraintChecks( char *zErr; sqlite3StrAccumInit(&errMsg, 0, 0, 200); - errMsg.db = pParse->db; + errMsg.db = db; zSep = pIdx->nColumn>1 ? "columns " : "column "; for(j=0; j<pIdx->nColumn; j++){ char *zCol = pTab->aCol[pIdx->aiColumn[j]].zName; @@ -1393,7 +1405,7 @@ void sqlite3GenerateConstraintChecks( Trigger *pTrigger = 0; assert( onError==OE_Replace ); sqlite3MultiWrite(pParse); - if( pParse->db->flags&SQLITE_RecTriggers ){ + if( db->flags&SQLITE_RecTriggers ){ pTrigger = sqlite3TriggersExist(pParse, pTab, TK_DELETE, 0, 0); } sqlite3GenerateRowDelete( @@ -1578,31 +1590,25 @@ static int xferCompatibleIndex(Index *pDest, Index *pSrc){ ** ** INSERT INTO tab1 SELECT * FROM tab2; ** -** This optimization is only attempted if -** -** (1) tab1 and tab2 have identical schemas including all the -** same indices and constraints -** -** (2) tab1 and tab2 are different tables +** The xfer optimization transfers raw records from tab2 over to tab1. +** Columns are not decoded and reassemblied, which greatly improves +** performance. Raw index records are transferred in the same way. ** -** (3) There must be no triggers on tab1 +** The xfer optimization is only attempted if tab1 and tab2 are compatible. +** There are lots of rules for determining compatibility - see comments +** embedded in the code for details. ** -** (4) The result set of the SELECT statement is "*" +** This routine returns TRUE if the optimization is guaranteed to be used. +** Sometimes the xfer optimization will only work if the destination table +** is empty - a factor that can only be determined at run-time. In that +** case, this routine generates code for the xfer optimization but also +** does a test to see if the destination table is empty and jumps over the +** xfer optimization code if the test fails. In that case, this routine +** returns FALSE so that the caller will know to go ahead and generate +** an unoptimized transfer. This routine also returns FALSE if there +** is no chance that the xfer optimization can be applied. ** -** (5) The SELECT statement has no WHERE, HAVING, ORDER BY, GROUP BY, -** or LIMIT clause. -** -** (6) The SELECT statement is a simple (not a compound) select that -** contains only tab2 in its FROM clause -** -** This method for implementing the INSERT transfers raw records from -** tab2 over to tab1. The columns are not decoded. Raw records from -** the indices of tab2 are transfered to tab1 as well. In so doing, -** the resulting tab1 has much less fragmentation. -** -** This routine returns TRUE if the optimization is attempted. If any -** of the conditions above fail so that the optimization should not -** be attempted, then this routine returns FALSE. +** This optimization is particularly useful at making VACUUM run faster. */ static int xferOptimization( Parse *pParse, /* Parser context */ @@ -1639,10 +1645,8 @@ static int xferOptimization( } #endif if( onError==OE_Default ){ - onError = OE_Abort; - } - if( onError!=OE_Abort && onError!=OE_Rollback ){ - return 0; /* Cannot do OR REPLACE or OR IGNORE or OR FAIL */ + if( pDest->iPKey>=0 ) onError = pDest->keyConf; + if( onError==OE_Default ) onError = OE_Abort; } assert(pSelect->pSrc); /* allocated even if there is no FROM clause */ if( pSelect->pSrc->nSrc!=1 ){ @@ -1731,7 +1735,7 @@ static int xferOptimization( } } #ifndef SQLITE_OMIT_CHECK - if( pDest->pCheck && sqlite3ExprCompare(pSrc->pCheck, pDest->pCheck) ){ + if( pDest->pCheck && sqlite3ExprListCompare(pSrc->pCheck, pDest->pCheck) ){ return 0; /* Tables have different CHECK constraints. Ticket #2252 */ } #endif @@ -1748,16 +1752,12 @@ static int xferOptimization( } #endif if( (pParse->db->flags & SQLITE_CountRows)!=0 ){ - return 0; + return 0; /* xfer opt does not play well with PRAGMA count_changes */ } - /* If we get this far, it means either: - ** - ** * We can always do the transfer if the table contains an - ** an integer primary key - ** - ** * We can conditionally do the transfer if the destination - ** table is empty. + /* If we get this far, it means that the xfer optimization is at + ** least a possibility, though it might only work if the destination + ** table (tab1) is initially empty. */ #ifdef SQLITE_TEST sqlite3_xferopt_count++; @@ -1769,16 +1769,23 @@ static int xferOptimization( iDest = pParse->nTab++; regAutoinc = autoIncBegin(pParse, iDbDest, pDest); sqlite3OpenTable(pParse, iDest, iDbDest, pDest, OP_OpenWrite); - if( (pDest->iPKey<0 && pDest->pIndex!=0) || destHasUniqueIdx ){ - /* If tables do not have an INTEGER PRIMARY KEY and there - ** are indices to be copied and the destination is not empty, - ** we have to disallow the transfer optimization because the - ** the rowids might change which will mess up indexing. + if( (pDest->iPKey<0 && pDest->pIndex!=0) /* (1) */ + || destHasUniqueIdx /* (2) */ + || (onError!=OE_Abort && onError!=OE_Rollback) /* (3) */ + ){ + /* In some circumstances, we are able to run the xfer optimization + ** only if the destination table is initially empty. This code makes + ** that determination. Conditions under which the destination must + ** be empty: + ** + ** (1) There is no INTEGER PRIMARY KEY but there are indices. + ** (If the destination is not initially empty, the rowid fields + ** of index entries might need to change.) + ** + ** (2) The destination has a unique index. (The xfer optimization + ** is unable to test uniqueness.) ** - ** Or if the destination has a UNIQUE index and is not empty, - ** we also disallow the transfer optimization because we cannot - ** insure that all entries in the union of DEST and SRC will be - ** unique. + ** (3) onError is something other than OE_Abort and OE_Rollback. */ addr1 = sqlite3VdbeAddOp2(v, OP_Rewind, iDest, 0); emptyDestTest = sqlite3VdbeAddOp2(v, OP_Goto, 0, 0); diff --git a/src/loadext.c b/src/loadext.c index e9c97ad..3fcf500 100644 --- a/src/loadext.c +++ b/src/loadext.c @@ -625,6 +625,7 @@ void sqlite3_reset_auto_extension(void){ void sqlite3AutoLoadExtensions(sqlite3 *db){ int i; int go = 1; + int rc; int (*xInit)(sqlite3*,char**,const sqlite3_api_routines*); wsdAutoextInit; @@ -647,8 +648,8 @@ void sqlite3AutoLoadExtensions(sqlite3 *db){ } sqlite3_mutex_leave(mutex); zErrmsg = 0; - if( xInit && xInit(db, &zErrmsg, &sqlite3Apis) ){ - sqlite3Error(db, SQLITE_ERROR, + if( xInit && (rc = xInit(db, &zErrmsg, &sqlite3Apis))!=0 ){ + sqlite3Error(db, rc, "automatic extension loading failed: %s", zErrmsg); go = 0; } @@ -49,8 +49,8 @@ const char *sqlite3_sourceid(void){ return SQLITE_SOURCE_ID; } */ int sqlite3_libversion_number(void){ return SQLITE_VERSION_NUMBER; } -/* IMPLEMENTATION-OF: R-54823-41343 The sqlite3_threadsafe() function returns -** zero if and only if SQLite was compiled mutexing code omitted due to +/* IMPLEMENTATION-OF: R-20790-14025 The sqlite3_threadsafe() function returns +** zero if and only if SQLite was compiled with mutexing code omitted due to ** the SQLITE_THREADSAFE compile-time option being set to 0. */ int sqlite3_threadsafe(void){ return SQLITE_THREADSAFE; } @@ -239,8 +239,8 @@ int sqlite3_initialize(void){ */ #ifdef SQLITE_EXTRA_INIT if( rc==SQLITE_OK && sqlite3GlobalConfig.isInit ){ - int SQLITE_EXTRA_INIT(void); - rc = SQLITE_EXTRA_INIT(); + int SQLITE_EXTRA_INIT(const char*); + rc = SQLITE_EXTRA_INIT(0); } #endif @@ -257,6 +257,10 @@ int sqlite3_initialize(void){ */ int sqlite3_shutdown(void){ if( sqlite3GlobalConfig.isInit ){ +#ifdef SQLITE_EXTRA_SHUTDOWN + void SQLITE_EXTRA_SHUTDOWN(void); + SQLITE_EXTRA_SHUTDOWN(); +#endif sqlite3_os_end(); sqlite3_reset_auto_extension(); sqlite3GlobalConfig.isInit = 0; @@ -365,16 +369,25 @@ int sqlite3_config(int op, ...){ } case SQLITE_CONFIG_PCACHE: { - /* Specify an alternative page cache implementation */ - sqlite3GlobalConfig.pcache = *va_arg(ap, sqlite3_pcache_methods*); + /* no-op */ break; } - case SQLITE_CONFIG_GETPCACHE: { - if( sqlite3GlobalConfig.pcache.xInit==0 ){ + /* now an error */ + rc = SQLITE_ERROR; + break; + } + + case SQLITE_CONFIG_PCACHE2: { + /* Specify an alternative page cache implementation */ + sqlite3GlobalConfig.pcache2 = *va_arg(ap, sqlite3_pcache_methods2*); + break; + } + case SQLITE_CONFIG_GETPCACHE2: { + if( sqlite3GlobalConfig.pcache2.xInit==0 ){ sqlite3PCacheSetDefault(); } - *va_arg(ap, sqlite3_pcache_methods*) = sqlite3GlobalConfig.pcache; + *va_arg(ap, sqlite3_pcache_methods2*) = sqlite3GlobalConfig.pcache2; break; } @@ -473,21 +486,21 @@ static int setupLookaside(sqlite3 *db, void *pBuf, int sz, int cnt){ if( db->lookaside.bMalloced ){ sqlite3_free(db->lookaside.pStart); } - /* The size of a lookaside slot needs to be larger than a pointer - ** to be useful. + /* The size of a lookaside slot after ROUNDDOWN8 needs to be larger + ** than a pointer to be useful. */ + sz = ROUNDDOWN8(sz); /* IMP: R-33038-09382 */ if( sz<=(int)sizeof(LookasideSlot*) ) sz = 0; if( cnt<0 ) cnt = 0; if( sz==0 || cnt==0 ){ sz = 0; pStart = 0; }else if( pBuf==0 ){ - sz = ROUNDDOWN8(sz); /* IMP: R-33038-09382 */ sqlite3BeginBenignMalloc(); pStart = sqlite3Malloc( sz*cnt ); /* IMP: R-61949-35727 */ sqlite3EndBenignMalloc(); + if( pStart ) cnt = sqlite3MallocSize(pStart)/sz; }else{ - sz = ROUNDDOWN8(sz); /* IMP: R-33038-09382 */ pStart = pBuf; } db->lookaside.pStart = pStart; @@ -522,6 +535,26 @@ sqlite3_mutex *sqlite3_db_mutex(sqlite3 *db){ } /* +** Free up as much memory as we can from the given database +** connection. +*/ +int sqlite3_db_release_memory(sqlite3 *db){ + int i; + sqlite3_mutex_enter(db->mutex); + sqlite3BtreeEnterAll(db); + for(i=0; i<db->nDb; i++){ + Btree *pBt = db->aDb[i].pBt; + if( pBt ){ + Pager *pPager = sqlite3BtreePager(pBt); + sqlite3PagerShrink(pPager); + } + } + sqlite3BtreeLeaveAll(db); + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; +} + +/* ** Configuration settings for an individual database connection */ int sqlite3_db_config(sqlite3 *db, int op, ...){ @@ -816,19 +849,23 @@ int sqlite3_close(sqlite3 *db){ } /* -** Rollback all database files. +** Rollback all database files. If tripCode is not SQLITE_OK, then +** any open cursors are invalidated ("tripped" - as in "tripping a circuit +** breaker") and made to return tripCode if there are any further +** attempts to use that cursor. */ -void sqlite3RollbackAll(sqlite3 *db){ +void sqlite3RollbackAll(sqlite3 *db, int tripCode){ int i; int inTrans = 0; assert( sqlite3_mutex_held(db->mutex) ); sqlite3BeginBenignMalloc(); for(i=0; i<db->nDb; i++){ - if( db->aDb[i].pBt ){ - if( sqlite3BtreeIsInTrans(db->aDb[i].pBt) ){ + Btree *p = db->aDb[i].pBt; + if( p ){ + if( sqlite3BtreeIsInTrans(p) ){ inTrans = 1; } - sqlite3BtreeRollback(db->aDb[i].pBt); + sqlite3BtreeRollback(p, tripCode); db->aDb[i].inTrans = 0; } } @@ -883,12 +920,21 @@ const char *sqlite3ErrStr(int rc){ /* SQLITE_RANGE */ "bind or column index out of range", /* SQLITE_NOTADB */ "file is encrypted or is not a database", }; - rc &= 0xff; - if( ALWAYS(rc>=0) && rc<(int)(sizeof(aMsg)/sizeof(aMsg[0])) && aMsg[rc]!=0 ){ - return aMsg[rc]; - }else{ - return "unknown error"; + const char *zErr = "unknown error"; + switch( rc ){ + case SQLITE_ABORT_ROLLBACK: { + zErr = "abort due to ROLLBACK"; + break; + } + default: { + rc &= 0xff; + if( ALWAYS(rc>=0) && rc<ArraySize(aMsg) && aMsg[rc]!=0 ){ + zErr = aMsg[rc]; + } + break; + } } + return zErr; } /* @@ -1266,9 +1312,8 @@ void *sqlite3_profile( } #endif /* SQLITE_OMIT_TRACE */ -/*** EXPERIMENTAL *** -** -** Register a function to be invoked when a transaction comments. +/* +** Register a function to be invoked when a transaction commits. ** If the invoked function returns non-zero, then the commit becomes a ** rollback. */ @@ -1628,7 +1673,6 @@ static int createCollation( sqlite3* db, const char *zName, u8 enc, - u8 collType, void* pCtx, int(*xCompare)(void*,int,const void*,int,const void*), void(*xDel)(void*) @@ -1693,7 +1737,6 @@ static int createCollation( pColl->pUser = pCtx; pColl->xDel = xDel; pColl->enc = (u8)(enc2 | (enc & SQLITE_UTF16_ALIGNED)); - pColl->type = collType; sqlite3Error(db, SQLITE_OK, 0); return SQLITE_OK; } @@ -2154,14 +2197,10 @@ static int openDatabase( ** and UTF-16, so add a version for each to avoid any unnecessary ** conversions. The only error that can occur here is a malloc() failure. */ - createCollation(db, "BINARY", SQLITE_UTF8, SQLITE_COLL_BINARY, 0, - binCollFunc, 0); - createCollation(db, "BINARY", SQLITE_UTF16BE, SQLITE_COLL_BINARY, 0, - binCollFunc, 0); - createCollation(db, "BINARY", SQLITE_UTF16LE, SQLITE_COLL_BINARY, 0, - binCollFunc, 0); - createCollation(db, "RTRIM", SQLITE_UTF8, SQLITE_COLL_USER, (void*)1, - binCollFunc, 0); + createCollation(db, "BINARY", SQLITE_UTF8, 0, binCollFunc, 0); + createCollation(db, "BINARY", SQLITE_UTF16BE, 0, binCollFunc, 0); + createCollation(db, "BINARY", SQLITE_UTF16LE, 0, binCollFunc, 0); + createCollation(db, "RTRIM", SQLITE_UTF8, (void*)1, binCollFunc, 0); if( db->mallocFailed ){ goto opendb_out; } @@ -2169,8 +2208,7 @@ static int openDatabase( assert( db->pDfltColl!=0 ); /* Also add a UTF-8 case-insensitive collation sequence. */ - createCollation(db, "NOCASE", SQLITE_UTF8, SQLITE_COLL_NOCASE, 0, - nocaseCollatingFunc, 0); + createCollation(db, "NOCASE", SQLITE_UTF8, 0, nocaseCollatingFunc, 0); /* Parse the filename/URI argument. */ db->openFlags = flags; @@ -2219,10 +2257,13 @@ static int openDatabase( /* Load automatic extensions - extensions that have been registered ** using the sqlite3_automatic_extension() API. */ - sqlite3AutoLoadExtensions(db); rc = sqlite3_errcode(db); - if( rc!=SQLITE_OK ){ - goto opendb_out; + if( rc==SQLITE_OK ){ + sqlite3AutoLoadExtensions(db); + rc = sqlite3_errcode(db); + if( rc!=SQLITE_OK ){ + goto opendb_out; + } } #ifdef SQLITE_ENABLE_FTS1 @@ -2363,7 +2404,7 @@ int sqlite3_create_collation( int rc; sqlite3_mutex_enter(db->mutex); assert( !db->mallocFailed ); - rc = createCollation(db, zName, (u8)enc, SQLITE_COLL_USER, pCtx, xCompare, 0); + rc = createCollation(db, zName, (u8)enc, pCtx, xCompare, 0); rc = sqlite3ApiExit(db, rc); sqlite3_mutex_leave(db->mutex); return rc; @@ -2383,7 +2424,7 @@ int sqlite3_create_collation_v2( int rc; sqlite3_mutex_enter(db->mutex); assert( !db->mallocFailed ); - rc = createCollation(db, zName, (u8)enc, SQLITE_COLL_USER, pCtx, xCompare, xDel); + rc = createCollation(db, zName, (u8)enc, pCtx, xCompare, xDel); rc = sqlite3ApiExit(db, rc); sqlite3_mutex_leave(db->mutex); return rc; @@ -2406,7 +2447,7 @@ int sqlite3_create_collation16( assert( !db->mallocFailed ); zName8 = sqlite3Utf16to8(db, zName, -1, SQLITE_UTF16NATIVE); if( zName8 ){ - rc = createCollation(db, zName8, (u8)enc, SQLITE_COLL_USER, pCtx, xCompare, 0); + rc = createCollation(db, zName8, (u8)enc, pCtx, xCompare, 0); sqlite3DbFree(db, zName8); } rc = sqlite3ApiExit(db, rc); @@ -2663,35 +2704,27 @@ int sqlite3_extended_result_codes(sqlite3 *db, int onoff){ */ int sqlite3_file_control(sqlite3 *db, const char *zDbName, int op, void *pArg){ int rc = SQLITE_ERROR; - int iDb; + Btree *pBtree; + sqlite3_mutex_enter(db->mutex); - if( zDbName==0 ){ - iDb = 0; - }else{ - for(iDb=0; iDb<db->nDb; iDb++){ - if( strcmp(db->aDb[iDb].zName, zDbName)==0 ) break; - } - } - if( iDb<db->nDb ){ - Btree *pBtree = db->aDb[iDb].pBt; - if( pBtree ){ - Pager *pPager; - sqlite3_file *fd; - sqlite3BtreeEnter(pBtree); - pPager = sqlite3BtreePager(pBtree); - assert( pPager!=0 ); - fd = sqlite3PagerFile(pPager); - assert( fd!=0 ); - if( op==SQLITE_FCNTL_FILE_POINTER ){ - *(sqlite3_file**)pArg = fd; - rc = SQLITE_OK; - }else if( fd->pMethods ){ - rc = sqlite3OsFileControl(fd, op, pArg); - }else{ - rc = SQLITE_NOTFOUND; - } - sqlite3BtreeLeave(pBtree); + pBtree = sqlite3DbNameToBtree(db, zDbName); + if( pBtree ){ + Pager *pPager; + sqlite3_file *fd; + sqlite3BtreeEnter(pBtree); + pPager = sqlite3BtreePager(pBtree); + assert( pPager!=0 ); + fd = sqlite3PagerFile(pPager); + assert( fd!=0 ); + if( op==SQLITE_FCNTL_FILE_POINTER ){ + *(sqlite3_file**)pArg = fd; + rc = SQLITE_OK; + }else if( fd->pMethods ){ + rc = sqlite3OsFileControl(fd, op, pArg); + }else{ + rc = SQLITE_NOTFOUND; } + sqlite3BtreeLeave(pBtree); } sqlite3_mutex_leave(db->mutex); return rc; @@ -2889,15 +2922,6 @@ int sqlite3_test_control(int op, ...){ } #endif - /* sqlite3_test_control(SQLITE_TESTCTRL_PGHDRSZ) - ** - ** Return the size of a pcache header in bytes. - */ - case SQLITE_TESTCTRL_PGHDRSZ: { - rc = sizeof(PgHdr); - break; - } - /* sqlite3_test_control(SQLITE_TESTCTRL_SCRATCHMALLOC, sz, &pNew, pFree); ** ** Pass pFree into sqlite3ScratchFree(). @@ -2925,6 +2949,22 @@ int sqlite3_test_control(int op, ...){ break; } +#if defined(SQLITE_ENABLE_TREE_EXPLAIN) + /* sqlite3_test_control(SQLITE_TESTCTRL_EXPLAIN_STMT, + ** sqlite3_stmt*,const char**); + ** + ** If compiled with SQLITE_ENABLE_TREE_EXPLAIN, each sqlite3_stmt holds + ** a string that describes the optimized parse tree. This test-control + ** returns a pointer to that string. + */ + case SQLITE_TESTCTRL_EXPLAIN_STMT: { + sqlite3_stmt *pStmt = va_arg(ap, sqlite3_stmt*); + const char **pzRet = va_arg(ap, const char**); + *pzRet = sqlite3VdbeExplanation((Vdbe*)pStmt); + break; + } +#endif + } va_end(ap); #endif /* SQLITE_OMIT_BUILTIN_TEST */ @@ -2943,6 +2983,7 @@ int sqlite3_test_control(int op, ...){ ** returns a NULL pointer. */ const char *sqlite3_uri_parameter(const char *zFilename, const char *zParam){ + if( zFilename==0 ) return 0; zFilename += sqlite3Strlen30(zFilename) + 1; while( zFilename[0] ){ int x = strcmp(zFilename, zParam); @@ -2952,3 +2993,61 @@ const char *sqlite3_uri_parameter(const char *zFilename, const char *zParam){ } return 0; } + +/* +** Return a boolean value for a query parameter. +*/ +int sqlite3_uri_boolean(const char *zFilename, const char *zParam, int bDflt){ + const char *z = sqlite3_uri_parameter(zFilename, zParam); + bDflt = bDflt!=0; + return z ? sqlite3GetBoolean(z, bDflt) : bDflt; +} + +/* +** Return a 64-bit integer value for a query parameter. +*/ +sqlite3_int64 sqlite3_uri_int64( + const char *zFilename, /* Filename as passed to xOpen */ + const char *zParam, /* URI parameter sought */ + sqlite3_int64 bDflt /* return if parameter is missing */ +){ + const char *z = sqlite3_uri_parameter(zFilename, zParam); + sqlite3_int64 v; + if( z && sqlite3Atoi64(z, &v, sqlite3Strlen30(z), SQLITE_UTF8)==SQLITE_OK ){ + bDflt = v; + } + return bDflt; +} + +/* +** Return the Btree pointer identified by zDbName. Return NULL if not found. +*/ +Btree *sqlite3DbNameToBtree(sqlite3 *db, const char *zDbName){ + int i; + for(i=0; i<db->nDb; i++){ + if( db->aDb[i].pBt + && (zDbName==0 || sqlite3StrICmp(zDbName, db->aDb[i].zName)==0) + ){ + return db->aDb[i].pBt; + } + } + return 0; +} + +/* +** Return the filename of the database associated with a database +** connection. +*/ +const char *sqlite3_db_filename(sqlite3 *db, const char *zDbName){ + Btree *pBt = sqlite3DbNameToBtree(db, zDbName); + return pBt ? sqlite3BtreeGetFilename(pBt) : 0; +} + +/* +** Return 1 if database is read-only or 0 if read/write. Return -1 if +** no such database exists. +*/ +int sqlite3_db_readonly(sqlite3 *db, const char *zDbName){ + Btree *pBt = sqlite3DbNameToBtree(db, zDbName); + return pBt ? sqlite3PagerIsreadonly(sqlite3BtreePager(pBt)) : -1; +} diff --git a/src/malloc.c b/src/malloc.c index 3e38d1d..35a44e5 100644 --- a/src/malloc.c +++ b/src/malloc.c @@ -130,7 +130,8 @@ sqlite3_int64 sqlite3_soft_heap_limit64(sqlite3_int64 n){ sqlite3_int64 priorLimit; sqlite3_int64 excess; #ifndef SQLITE_OMIT_AUTOINIT - sqlite3_initialize(); + int rc = sqlite3_initialize(); + if( rc ) return -1; #endif sqlite3_mutex_enter(mem0.mutex); priorLimit = mem0.alarmThreshold; @@ -490,6 +491,10 @@ void sqlite3DbFree(sqlite3 *db, void *p){ } if( isLookaside(db, p) ){ LookasideSlot *pBuf = (LookasideSlot*)p; +#if SQLITE_DEBUG + /* Trash all content in the buffer being freed */ + memset(p, 0xaa, db->lookaside.sz); +#endif pBuf->pNext = db->lookaside.pFree; db->lookaside.pFree = pBuf; db->lookaside.nOut--; @@ -15,7 +15,31 @@ ** to obtain the memory it needs. ** ** This file contains implementations of the low-level memory allocation -** routines specified in the sqlite3_mem_methods object. +** routines specified in the sqlite3_mem_methods object. The content of +** this file is only used if SQLITE_SYSTEM_MALLOC is defined. The +** SQLITE_SYSTEM_MALLOC macro is defined automatically if neither the +** SQLITE_MEMDEBUG nor the SQLITE_WIN32_MALLOC macros are defined. The +** default configuration is to use memory allocation routines in this +** file. +** +** C-preprocessor macro summary: +** +** HAVE_MALLOC_USABLE_SIZE The configure script sets this symbol if +** the malloc_usable_size() interface exists +** on the target platform. Or, this symbol +** can be set manually, if desired. +** If an equivalent interface exists by +** a different name, using a separate -D +** option to rename it. +** +** SQLITE_WITHOUT_ZONEMALLOC Some older macs lack support for the zone +** memory allocator. Set this symbol to enable +** building on older macs. +** +** SQLITE_WITHOUT_MSIZE Set this symbol to disable the use of +** _msize() on windows systems. This might +** be necessary when compiling for Delphi, +** for example. */ #include "sqliteInt.h" @@ -27,6 +51,55 @@ #ifdef SQLITE_SYSTEM_MALLOC /* +** The MSVCRT has malloc_usable_size() but it is called _msize(). +** The use of _msize() is automatic, but can be disabled by compiling +** with -DSQLITE_WITHOUT_MSIZE +*/ +#if defined(_MSC_VER) && !defined(SQLITE_WITHOUT_MSIZE) +# define SQLITE_MALLOCSIZE _msize +#endif + +#if defined(__APPLE__) && !defined(SQLITE_WITHOUT_ZONEMALLOC) + +/* +** Use the zone allocator available on apple products unless the +** SQLITE_WITHOUT_ZONEMALLOC symbol is defined. +*/ +#include <sys/sysctl.h> +#include <malloc/malloc.h> +#include <libkern/OSAtomic.h> +static malloc_zone_t* _sqliteZone_; +#define SQLITE_MALLOC(x) malloc_zone_malloc(_sqliteZone_, (x)) +#define SQLITE_FREE(x) malloc_zone_free(_sqliteZone_, (x)); +#define SQLITE_REALLOC(x,y) malloc_zone_realloc(_sqliteZone_, (x), (y)) +#define SQLITE_MALLOCSIZE(x) \ + (_sqliteZone_ ? _sqliteZone_->size(_sqliteZone_,x) : malloc_size(x)) + +#else /* if not __APPLE__ */ + +/* +** Use standard C library malloc and free on non-Apple systems. +** Also used by Apple systems if SQLITE_WITHOUT_ZONEMALLOC is defined. +*/ +#define SQLITE_MALLOC(x) malloc(x) +#define SQLITE_FREE(x) free(x) +#define SQLITE_REALLOC(x,y) realloc((x),(y)) + +#if (defined(_MSC_VER) && !defined(SQLITE_WITHOUT_MSIZE)) \ + || (defined(HAVE_MALLOC_H) && defined(HAVE_MALLOC_USABLE_SIZE)) +# include <malloc.h> /* Needed for malloc_usable_size on linux */ +#endif +#ifdef HAVE_MALLOC_USABLE_SIZE +# ifndef SQLITE_MALLOCSIZE +# define SQLITE_MALLOCSIZE(x) malloc_usable_size(x) +# endif +#else +# undef SQLITE_MALLOCSIZE +#endif + +#endif /* __APPLE__ or not __APPLE__ */ + +/* ** Like malloc(), but remember the size of the allocation ** so that we can find it later using sqlite3MemSize(). ** @@ -35,10 +108,18 @@ ** routines. */ static void *sqlite3MemMalloc(int nByte){ +#ifdef SQLITE_MALLOCSIZE + void *p = SQLITE_MALLOC( nByte ); + if( p==0 ){ + testcase( sqlite3GlobalConfig.xLog!=0 ); + sqlite3_log(SQLITE_NOMEM, "failed to allocate %u bytes of memory", nByte); + } + return p; +#else sqlite3_int64 *p; assert( nByte>0 ); nByte = ROUND8(nByte); - p = malloc( nByte+8 ); + p = SQLITE_MALLOC( nByte+8 ); if( p ){ p[0] = nByte; p++; @@ -47,6 +128,7 @@ static void *sqlite3MemMalloc(int nByte){ sqlite3_log(SQLITE_NOMEM, "failed to allocate %u bytes of memory", nByte); } return (void *)p; +#endif } /* @@ -58,10 +140,14 @@ static void *sqlite3MemMalloc(int nByte){ ** by higher-level routines. */ static void sqlite3MemFree(void *pPrior){ +#ifdef SQLITE_MALLOCSIZE + SQLITE_FREE(pPrior); +#else sqlite3_int64 *p = (sqlite3_int64*)pPrior; assert( pPrior!=0 ); p--; - free(p); + SQLITE_FREE(p); +#endif } /* @@ -69,11 +155,15 @@ static void sqlite3MemFree(void *pPrior){ ** or xRealloc(). */ static int sqlite3MemSize(void *pPrior){ +#ifdef SQLITE_MALLOCSIZE + return pPrior ? (int)SQLITE_MALLOCSIZE(pPrior) : 0; +#else sqlite3_int64 *p; if( pPrior==0 ) return 0; p = (sqlite3_int64*)pPrior; p--; return (int)p[0]; +#endif } /* @@ -87,11 +177,21 @@ static int sqlite3MemSize(void *pPrior){ ** routines and redirected to xFree. */ static void *sqlite3MemRealloc(void *pPrior, int nByte){ +#ifdef SQLITE_MALLOCSIZE + void *p = SQLITE_REALLOC(pPrior, nByte); + if( p==0 ){ + testcase( sqlite3GlobalConfig.xLog!=0 ); + sqlite3_log(SQLITE_NOMEM, + "failed memory resize %u to %u bytes", + SQLITE_MALLOCSIZE(pPrior), nByte); + } + return p; +#else sqlite3_int64 *p = (sqlite3_int64*)pPrior; assert( pPrior!=0 && nByte>0 ); assert( nByte==ROUND8(nByte) ); /* EV: R-46199-30249 */ p--; - p = realloc(p, nByte+8 ); + p = SQLITE_REALLOC(p, nByte+8 ); if( p ){ p[0] = nByte; p++; @@ -102,6 +202,7 @@ static void *sqlite3MemRealloc(void *pPrior, int nByte){ sqlite3MemSize(pPrior), nByte); } return (void*)p; +#endif } /* @@ -115,6 +216,34 @@ static int sqlite3MemRoundup(int n){ ** Initialize this module. */ static int sqlite3MemInit(void *NotUsed){ +#if defined(__APPLE__) && !defined(SQLITE_WITHOUT_ZONEMALLOC) + int cpuCount; + size_t len; + if( _sqliteZone_ ){ + return SQLITE_OK; + } + len = sizeof(cpuCount); + /* One usually wants to use hw.acctivecpu for MT decisions, but not here */ + sysctlbyname("hw.ncpu", &cpuCount, &len, NULL, 0); + if( cpuCount>1 ){ + /* defer MT decisions to system malloc */ + _sqliteZone_ = malloc_default_zone(); + }else{ + /* only 1 core, use our own zone to contention over global locks, + ** e.g. we have our own dedicated locks */ + bool success; + malloc_zone_t* newzone = malloc_create_zone(4096, 0); + malloc_set_zone_name(newzone, "Sqlite_Heap"); + do{ + success = OSAtomicCompareAndSwapPtrBarrier(NULL, newzone, + (void * volatile *)&_sqliteZone_); + }while(!_sqliteZone_); + if( !success ){ + /* somebody registered a zone first */ + malloc_destroy_zone(newzone); + } + } +#endif UNUSED_PARAMETER(NotUsed); return SQLITE_OK; } diff --git a/src/mutex.c b/src/mutex.c index 869a4ae..b567e7c 100644 --- a/src/mutex.c +++ b/src/mutex.c @@ -150,4 +150,4 @@ int sqlite3_mutex_notheld(sqlite3_mutex *p){ } #endif -#endif /* SQLITE_MUTEX_OMIT */ +#endif /* !defined(SQLITE_MUTEX_OMIT) */ diff --git a/src/mutex_noop.c b/src/mutex_noop.c index c5fd520..456e82a 100644 --- a/src/mutex_noop.c +++ b/src/mutex_noop.c @@ -202,5 +202,5 @@ sqlite3_mutex_methods const *sqlite3NoopMutex(void){ sqlite3_mutex_methods const *sqlite3DefaultMutex(void){ return sqlite3NoopMutex(); } -#endif /* SQLITE_MUTEX_NOOP */ -#endif /* SQLITE_MUTEX_OMIT */ +#endif /* defined(SQLITE_MUTEX_NOOP) */ +#endif /* !defined(SQLITE_MUTEX_OMIT) */ diff --git a/src/mutex_unix.c b/src/mutex_unix.c index aa9a8cf..eca7295 100644 --- a/src/mutex_unix.c +++ b/src/mutex_unix.c @@ -348,4 +348,4 @@ sqlite3_mutex_methods const *sqlite3DefaultMutex(void){ return &sMutex; } -#endif /* SQLITE_MUTEX_PTHREAD */ +#endif /* SQLITE_MUTEX_PTHREADS */ @@ -27,11 +27,18 @@ ** The following functions are instrumented for malloc() failure ** testing: ** -** sqlite3OsOpen() ** sqlite3OsRead() ** sqlite3OsWrite() ** sqlite3OsSync() +** sqlite3OsFileSize() ** sqlite3OsLock() +** sqlite3OsCheckReservedLock() +** sqlite3OsFileControl() +** sqlite3OsShmMap() +** sqlite3OsOpen() +** sqlite3OsDelete() +** sqlite3OsAccess() +** sqlite3OsFullPathname() ** */ #if defined(SQLITE_TEST) @@ -90,9 +97,23 @@ int sqlite3OsCheckReservedLock(sqlite3_file *id, int *pResOut){ DO_OS_MALLOC_TEST(id); return id->pMethods->xCheckReservedLock(id, pResOut); } + +/* +** Use sqlite3OsFileControl() when we are doing something that might fail +** and we need to know about the failures. Use sqlite3OsFileControlHint() +** when simply tossing information over the wall to the VFS and we do not +** really care if the VFS receives and understands the information since it +** is only a hint and can be safely ignored. The sqlite3OsFileControlHint() +** routine has no return value since the return value would be meaningless. +*/ int sqlite3OsFileControl(sqlite3_file *id, int op, void *pArg){ + DO_OS_MALLOC_TEST(id); return id->pMethods->xFileControl(id, op, pArg); } +void sqlite3OsFileControlHint(sqlite3_file *id, int op, void *pArg){ + (void)id->pMethods->xFileControl(id, op, pArg); +} + int sqlite3OsSectorSize(sqlite3_file *id){ int (*xSectorSize)(sqlite3_file*) = id->pMethods->xSectorSize; return (xSectorSize ? xSectorSize(id) : SQLITE_DEFAULT_SECTOR_SIZE); @@ -116,6 +137,7 @@ int sqlite3OsShmMap( int bExtend, /* True to extend file if necessary */ void volatile **pp /* OUT: Pointer to mapping */ ){ + DO_OS_MALLOC_TEST(id); return id->pMethods->xShmMap(id, iPage, pgsz, bExtend, pp); } @@ -132,7 +154,7 @@ int sqlite3OsOpen( ){ int rc; DO_OS_MALLOC_TEST(0); - /* 0x87f3f is a mask of SQLITE_OPEN_ flags that are valid to be passed + /* 0x87f7f is a mask of SQLITE_OPEN_ flags that are valid to be passed ** down into the VFS layer. Some SQLITE_OPEN_ flags (for example, ** SQLITE_OPEN_FULLMUTEX or SQLITE_OPEN_SHAREDCACHE) are blocked before ** reaching the VFS. */ @@ -141,6 +163,8 @@ int sqlite3OsOpen( return rc; } int sqlite3OsDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ + DO_OS_MALLOC_TEST(0); + assert( dirSync==0 || dirSync==1 ); return pVfs->xDelete(pVfs, zPath, dirSync); } int sqlite3OsAccess( @@ -158,6 +182,7 @@ int sqlite3OsFullPathname( int nPathOut, char *zPathOut ){ + DO_OS_MALLOC_TEST(0); zPathOut[0] = 0; return pVfs->xFullPathname(pVfs, zPath, nPathOut, zPathOut); } @@ -66,17 +66,6 @@ #endif /* -** Determine if we are dealing with WindowsCE - which has a much -** reduced API. -*/ -#if defined(_WIN32_WCE) -# define SQLITE_OS_WINCE 1 -#else -# define SQLITE_OS_WINCE 0 -#endif - - -/* ** Define the maximum size of a temporary filename */ #if SQLITE_OS_WIN @@ -100,6 +89,37 @@ # define SQLITE_TEMPNAME_SIZE 200 #endif +/* +** Determine if we are dealing with Windows NT. +** +** We ought to be able to determine if we are compiling for win98 or winNT +** using the _WIN32_WINNT macro as follows: +** +** #if defined(_WIN32_WINNT) +** # define SQLITE_OS_WINNT 1 +** #else +** # define SQLITE_OS_WINNT 0 +** #endif +** +** However, vs2005 does not set _WIN32_WINNT by default, as it ought to, +** so the above test does not work. We'll just assume that everything is +** winNT unless the programmer explicitly says otherwise by setting +** SQLITE_OS_WINNT to 0. +*/ +#if SQLITE_OS_WIN && !defined(SQLITE_OS_WINNT) +# define SQLITE_OS_WINNT 1 +#endif + +/* +** Determine if we are dealing with WindowsCE - which has a much +** reduced API. +*/ +#if defined(_WIN32_WCE) +# define SQLITE_OS_WINCE 1 +#else +# define SQLITE_OS_WINCE 0 +#endif + /* If the SET_FULLSYNC macro is not defined above, then make it ** a no-op */ @@ -111,7 +131,7 @@ ** The default size of a disk sector */ #ifndef SQLITE_DEFAULT_SECTOR_SIZE -# define SQLITE_DEFAULT_SECTOR_SIZE 512 +# define SQLITE_DEFAULT_SECTOR_SIZE 4096 #endif /* @@ -244,6 +264,7 @@ int sqlite3OsLock(sqlite3_file*, int); int sqlite3OsUnlock(sqlite3_file*, int); int sqlite3OsCheckReservedLock(sqlite3_file *id, int *pResOut); int sqlite3OsFileControl(sqlite3_file*,int,void*); +void sqlite3OsFileControlHint(sqlite3_file*,int,void*); #define SQLITE_FCNTL_DB_UNCHANGED 0xca093fa0 int sqlite3OsSectorSize(sqlite3_file *id); int sqlite3OsDeviceCharacteristics(sqlite3_file *id); @@ -252,6 +273,7 @@ int sqlite3OsShmLock(sqlite3_file *id, int, int, int); void sqlite3OsShmBarrier(sqlite3_file *id); int sqlite3OsShmUnmap(sqlite3_file *id, int); + /* ** Functions for accessing sqlite3_vfs methods */ diff --git a/src/os_unix.c b/src/os_unix.c index 0ea6daf..c85e9b5 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -123,6 +123,7 @@ #include <sys/mman.h> #endif + #if SQLITE_ENABLE_LOCKING_STYLE # include <sys/ioctl.h> # if OS_VXWORKS @@ -164,8 +165,8 @@ #endif /* - ** Default permissions when creating auto proxy dir - */ +** Default permissions when creating auto proxy dir +*/ #ifndef SQLITE_DEFAULT_PROXYDIR_PERMISSIONS # define SQLITE_DEFAULT_PROXYDIR_PERMISSIONS 0755 #endif @@ -206,10 +207,11 @@ struct UnixUnusedFd { typedef struct unixFile unixFile; struct unixFile { sqlite3_io_methods const *pMethod; /* Always the first entry */ + sqlite3_vfs *pVfs; /* The VFS that created this unixFile */ unixInodeInfo *pInode; /* Info about locks on this inode */ int h; /* The file descriptor */ unsigned char eFileLock; /* The type of lock held on this fd */ - unsigned char ctrlFlags; /* Behavioral bits. UNIXFILE_* flags */ + unsigned short int ctrlFlags; /* Behavioral bits. UNIXFILE_* flags */ int lastErrno; /* The unix errno from last I/O error */ void *lockingContext; /* Locking style specific state */ UnixUnusedFd *pUnused; /* Pre-allocated UnixUnusedFd */ @@ -223,7 +225,6 @@ struct unixFile { unsigned fsFlags; /* cached details from statfs() */ #endif #if OS_VXWORKS - int isDelete; /* Delete on close if true */ struct vxworksFileId *pId; /* Unique file ID */ #endif #ifndef NDEBUG @@ -257,6 +258,11 @@ struct unixFile { #else # define UNIXFILE_DIRSYNC 0x00 #endif +#define UNIXFILE_PSOW 0x10 /* SQLITE_IOCAP_POWERSAFE_OVERWRITE */ +#define UNIXFILE_DELETE 0x20 /* Delete on close */ +#define UNIXFILE_URI 0x40 /* Filename might have query parameters */ +#define UNIXFILE_NOLOCK 0x80 /* Do no file locking */ +#define UNIXFILE_CHOWN 0x100 /* File ownership was changed */ /* ** Include code that is common to all os_*.c files @@ -407,6 +413,18 @@ static struct unix_syscall { { "openDirectory", (sqlite3_syscall_ptr)openDirectory, 0 }, #define osOpenDirectory ((int(*)(const char*,int*))aSyscall[17].pCurrent) + { "mkdir", (sqlite3_syscall_ptr)mkdir, 0 }, +#define osMkdir ((int(*)(const char*,mode_t))aSyscall[18].pCurrent) + + { "rmdir", (sqlite3_syscall_ptr)rmdir, 0 }, +#define osRmdir ((int(*)(const char*))aSyscall[19].pCurrent) + + { "fchown", (sqlite3_syscall_ptr)fchown, 0 }, +#define osFchown ((int(*)(int,uid_t,gid_t))aSyscall[20].pCurrent) + + { "umask", (sqlite3_syscall_ptr)umask, 0 }, +#define osUmask ((mode_t(*)(mode_t))aSyscall[21].pCurrent) + }; /* End of the overrideable system calls */ /* @@ -493,12 +511,46 @@ static const char *unixNextSystemCall(sqlite3_vfs *p, const char *zName){ } /* -** Retry open() calls that fail due to EINTR +** Invoke open(). Do so multiple times, until it either succeeds or +** fails for some reason other than EINTR. +** +** If the file creation mode "m" is 0 then set it to the default for +** SQLite. The default is SQLITE_DEFAULT_FILE_PERMISSIONS (normally +** 0644) as modified by the system umask. If m is not 0, then +** make the file creation mode be exactly m ignoring the umask. +** +** The m parameter will be non-zero only when creating -wal, -journal, +** and -shm files. We want those files to have *exactly* the same +** permissions as their original database, unadulterated by the umask. +** In that way, if a database file is -rw-rw-rw or -rw-rw-r-, and a +** transaction crashes and leaves behind hot journals, then any +** process that is able to write to the database will also be able to +** recover the hot journals. */ -static int robust_open(const char *z, int f, int m){ - int rc; - do{ rc = osOpen(z,f,m); }while( rc<0 && errno==EINTR ); - return rc; +static int robust_open(const char *z, int f, mode_t m){ + int fd; + mode_t m2; + mode_t origM = 0; + if( m==0 ){ + m2 = SQLITE_DEFAULT_FILE_PERMISSIONS; + }else{ + m2 = m; + origM = osUmask(0); + } + do{ +#if defined(O_CLOEXEC) + fd = osOpen(z,f|O_CLOEXEC,m2); +#else + fd = osOpen(z,f,m2); +#endif + }while( fd<0 && errno==EINTR ); + if( m ){ + osUmask(origM); + } +#if defined(FD_CLOEXEC) && (!defined(O_CLOEXEC) || O_CLOEXEC==0) + if( fd>=0 ) osFcntl(fd, F_SETFD, osFcntl(fd, F_GETFD, 0) | FD_CLOEXEC); +#endif + return fd; } /* @@ -1756,7 +1808,7 @@ static int closeUnixFile(sqlite3_file *id){ } #if OS_VXWORKS if( pFile->pId ){ - if( pFile->isDelete ){ + if( pFile->ctrlFlags & UNIXFILE_DELETE ){ osUnlink(pFile->pId->zCanonicalName); } vxworksReleaseFileId(pFile->pId); @@ -1845,8 +1897,8 @@ static int nolockClose(sqlite3_file *id) { ************************* Begin dot-file Locking ****************************** ** ** The dotfile locking implementation uses the existance of separate lock -** files in order to control access to the database. This works on just -** about every filesystem imaginable. But there are serious downsides: +** files (really a directory) to control access to the database. This works +** on just about every filesystem imaginable. But there are serious downsides: ** ** (1) There is zero concurrency. A single reader blocks all other ** connections from reading or writing the database. @@ -1857,15 +1909,15 @@ static int nolockClose(sqlite3_file *id) { ** Nevertheless, a dotlock is an appropriate locking mode for use if no ** other locking strategy is available. ** -** Dotfile locking works by creating a file in the same directory as the -** database and with the same name but with a ".lock" extension added. -** The existance of a lock file implies an EXCLUSIVE lock. All other lock -** types (SHARED, RESERVED, PENDING) are mapped into EXCLUSIVE. +** Dotfile locking works by creating a subdirectory in the same directory as +** the database and with the same name but with a ".lock" extension added. +** The existance of a lock directory implies an EXCLUSIVE lock. All other +** lock types (SHARED, RESERVED, PENDING) are mapped into EXCLUSIVE. */ /* ** The file suffix added to the data base filename in order to create the -** lock file. +** lock directory. */ #define DOTLOCK_SUFFIX ".lock" @@ -1932,7 +1984,6 @@ static int dotlockCheckReservedLock(sqlite3_file *id, int *pResOut) { */ static int dotlockLock(sqlite3_file *id, int eFileLock) { unixFile *pFile = (unixFile*)id; - int fd; char *zLockFile = (char *)pFile->lockingContext; int rc = SQLITE_OK; @@ -1952,9 +2003,9 @@ static int dotlockLock(sqlite3_file *id, int eFileLock) { } /* grab an exclusive lock */ - fd = robust_open(zLockFile,O_RDONLY|O_CREAT|O_EXCL,0600); - if( fd<0 ){ - /* failed to open/create the file, someone else may have stolen the lock */ + rc = osMkdir(zLockFile, 0777); + if( rc<0 ){ + /* failed to open/create the lock directory */ int tErrno = errno; if( EEXIST == tErrno ){ rc = SQLITE_BUSY; @@ -1966,7 +2017,6 @@ static int dotlockLock(sqlite3_file *id, int eFileLock) { } return rc; } - robust_close(pFile, fd, __LINE__); /* got it, set the type and return ok */ pFile->eFileLock = eFileLock; @@ -1985,6 +2035,7 @@ static int dotlockLock(sqlite3_file *id, int eFileLock) { static int dotlockUnlock(sqlite3_file *id, int eFileLock) { unixFile *pFile = (unixFile*)id; char *zLockFile = (char *)pFile->lockingContext; + int rc; assert( pFile ); OSTRACE(("UNLOCK %d %d was %d pid=%d (dotlock)\n", pFile->h, eFileLock, @@ -2006,9 +2057,11 @@ static int dotlockUnlock(sqlite3_file *id, int eFileLock) { /* To fully unlock the database, delete the lock file */ assert( eFileLock==NO_LOCK ); - if( osUnlink(zLockFile) ){ - int rc = 0; + rc = osRmdir(zLockFile); + if( rc<0 && errno==ENOTDIR ) rc = osUnlink(zLockFile); + if( rc<0 ){ int tErrno = errno; + rc = 0; if( ENOENT != tErrno ){ rc = SQLITE_IOERR_UNLOCK; } @@ -2944,35 +2997,48 @@ static int nfsUnlock(sqlite3_file *id, int eFileLock){ */ static int seekAndRead(unixFile *id, sqlite3_int64 offset, void *pBuf, int cnt){ int got; + int prior = 0; #if (!defined(USE_PREAD) && !defined(USE_PREAD64)) i64 newOffset; #endif TIMER_START; + do{ #if defined(USE_PREAD) - do{ got = osPread(id->h, pBuf, cnt, offset); }while( got<0 && errno==EINTR ); - SimulateIOError( got = -1 ); + got = osPread(id->h, pBuf, cnt, offset); + SimulateIOError( got = -1 ); #elif defined(USE_PREAD64) - do{ got = osPread64(id->h, pBuf, cnt, offset); }while( got<0 && errno==EINTR); - SimulateIOError( got = -1 ); + got = osPread64(id->h, pBuf, cnt, offset); + SimulateIOError( got = -1 ); #else - newOffset = lseek(id->h, offset, SEEK_SET); - SimulateIOError( newOffset-- ); - if( newOffset!=offset ){ - if( newOffset == -1 ){ - ((unixFile*)id)->lastErrno = errno; - }else{ - ((unixFile*)id)->lastErrno = 0; + newOffset = lseek(id->h, offset, SEEK_SET); + SimulateIOError( newOffset-- ); + if( newOffset!=offset ){ + if( newOffset == -1 ){ + ((unixFile*)id)->lastErrno = errno; + }else{ + ((unixFile*)id)->lastErrno = 0; + } + return -1; } - return -1; - } - do{ got = osRead(id->h, pBuf, cnt); }while( got<0 && errno==EINTR ); + got = osRead(id->h, pBuf, cnt); #endif + if( got==cnt ) break; + if( got<0 ){ + if( errno==EINTR ){ got = 1; continue; } + prior = 0; + ((unixFile*)id)->lastErrno = errno; + break; + }else if( got>0 ){ + cnt -= got; + offset += got; + prior += got; + pBuf = (void*)(got + (char*)pBuf); + } + }while( got>0 ); TIMER_END; - if( got<0 ){ - ((unixFile*)id)->lastErrno = errno; - } - OSTRACE(("READ %-3d %5d %7lld %llu\n", id->h, got, offset, TIMER_ELAPSED)); - return got; + OSTRACE(("READ %-3d %5d %7lld %llu\n", + id->h, got+prior, offset-prior, TIMER_ELAPSED)); + return got+prior; } /* @@ -3279,9 +3345,6 @@ static int openDirectory(const char *zFilename, int *pFd){ zDirname[ii] = '\0'; fd = robust_open(zDirname, O_RDONLY|O_BINARY, 0); if( fd>=0 ){ -#ifdef FD_CLOEXEC - osFcntl(fd, F_SETFD, osFcntl(fd, F_GETFD, 0) | FD_CLOEXEC); -#endif OSTRACE(("OPENDIR %-3d %s\n", fd, zDirname)); } } @@ -3364,7 +3427,7 @@ static int unixTruncate(sqlite3_file *id, i64 nByte){ ** actual file size after the operation may be larger than the requested ** size). */ - if( pFile->szChunk ){ + if( pFile->szChunk>0 ){ nByte = ((nByte + pFile->szChunk - 1)/pFile->szChunk) * pFile->szChunk; } @@ -3478,6 +3541,22 @@ static int fcntlSizeHint(unixFile *pFile, i64 nByte){ } /* +** If *pArg is inititially negative then this is a query. Set *pArg to +** 1 or 0 depending on whether or not bit mask of pFile->ctrlFlags is set. +** +** If *pArg is 0 or 1, then clear or set the mask bit of pFile->ctrlFlags. +*/ +static void unixModeBit(unixFile *pFile, unsigned char mask, int *pArg){ + if( *pArg<0 ){ + *pArg = (pFile->ctrlFlags & mask)!=0; + }else if( (*pArg)==0 ){ + pFile->ctrlFlags &= ~mask; + }else{ + pFile->ctrlFlags |= mask; + } +} + +/* ** Information and control of an open file handle. */ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ @@ -3503,14 +3582,15 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ return rc; } case SQLITE_FCNTL_PERSIST_WAL: { - int bPersist = *(int*)pArg; - if( bPersist<0 ){ - *(int*)pArg = (pFile->ctrlFlags & UNIXFILE_PERSIST_WAL)!=0; - }else if( bPersist==0 ){ - pFile->ctrlFlags &= ~UNIXFILE_PERSIST_WAL; - }else{ - pFile->ctrlFlags |= UNIXFILE_PERSIST_WAL; - } + unixModeBit(pFile, UNIXFILE_PERSIST_WAL, (int*)pArg); + return SQLITE_OK; + } + case SQLITE_FCNTL_POWERSAFE_OVERWRITE: { + unixModeBit(pFile, UNIXFILE_PSOW, (int*)pArg); + return SQLITE_OK; + } + case SQLITE_FCNTL_VFSNAME: { + *(char**)pArg = sqlite3_mprintf("%s", pFile->pVfs->zName); return SQLITE_OK; } #ifndef NDEBUG @@ -3530,9 +3610,6 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ return proxyFileControl(id,op,pArg); } #endif /* SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) */ - case SQLITE_FCNTL_SYNC_OMITTED: { - return SQLITE_OK; /* A no-op */ - } } return SQLITE_NOTFOUND; } @@ -3547,17 +3624,31 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ ** a database and its journal file) that the sector size will be the ** same for both. */ -static int unixSectorSize(sqlite3_file *NotUsed){ - UNUSED_PARAMETER(NotUsed); +static int unixSectorSize(sqlite3_file *pFile){ + (void)pFile; return SQLITE_DEFAULT_SECTOR_SIZE; } /* -** Return the device characteristics for the file. This is always 0 for unix. +** Return the device characteristics for the file. +** +** This VFS is set up to return SQLITE_IOCAP_POWERSAFE_OVERWRITE by default. +** However, that choice is contraversial since technically the underlying +** file system does not always provide powersafe overwrites. (In other +** words, after a power-loss event, parts of the file that were never +** written might end up being altered.) However, non-PSOW behavior is very, +** very rare. And asserting PSOW makes a large reduction in the amount +** of required I/O for journaling, since a lot of padding is eliminated. +** Hence, while POWERSAFE_OVERWRITE is on by default, there is a file-control +** available to turn it off and URI query parameter available to turn it off. */ -static int unixDeviceCharacteristics(sqlite3_file *NotUsed){ - UNUSED_PARAMETER(NotUsed); - return 0; +static int unixDeviceCharacteristics(sqlite3_file *id){ + unixFile *p = (unixFile*)id; + if( p->ctrlFlags & UNIXFILE_PSOW ){ + return SQLITE_IOCAP_POWERSAFE_OVERWRITE; + }else{ + return 0; + } } #ifndef SQLITE_OMIT_WAL @@ -3803,8 +3894,7 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ /* Call fstat() to figure out the permissions on the database file. If ** a new *-shm file is created, an attempt will be made to create it - ** with the same permissions. The actual permissions the file is created - ** with are subject to the current umask setting. + ** with the same permissions. */ if( osFstat(pDbFd->h, &sStat) && pInode->bProcessLock==0 ){ rc = SQLITE_IOERR_FSTAT; @@ -3812,16 +3902,16 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ } #ifdef SQLITE_SHM_DIRECTORY - nShmFilename = sizeof(SQLITE_SHM_DIRECTORY) + 30; + nShmFilename = sizeof(SQLITE_SHM_DIRECTORY) + 31; #else - nShmFilename = 5 + (int)strlen(pDbFd->zPath); + nShmFilename = 6 + (int)strlen(pDbFd->zPath); #endif pShmNode = sqlite3_malloc( sizeof(*pShmNode) + nShmFilename ); if( pShmNode==0 ){ rc = SQLITE_NOMEM; goto shm_open_err; } - memset(pShmNode, 0, sizeof(*pShmNode)); + memset(pShmNode, 0, sizeof(*pShmNode)+nShmFilename); zShmFilename = pShmNode->zFilename = (char*)&pShmNode[1]; #ifdef SQLITE_SHM_DIRECTORY sqlite3_snprintf(nShmFilename, zShmFilename, @@ -3841,19 +3931,26 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ } if( pInode->bProcessLock==0 ){ - const char *zRO; int openFlags = O_RDWR | O_CREAT; - zRO = sqlite3_uri_parameter(pDbFd->zPath, "readonly_shm"); - if( zRO && sqlite3GetBoolean(zRO) ){ + if( sqlite3_uri_boolean(pDbFd->zPath, "readonly_shm", 0) ){ openFlags = O_RDONLY; pShmNode->isReadonly = 1; } pShmNode->h = robust_open(zShmFilename, openFlags, (sStat.st_mode&0777)); if( pShmNode->h<0 ){ - if( pShmNode->h<0 ){ - rc = unixLogError(SQLITE_CANTOPEN_BKPT, "open", zShmFilename); - goto shm_open_err; - } + rc = unixLogError(SQLITE_CANTOPEN_BKPT, "open", zShmFilename); + goto shm_open_err; + } + + /* If this process is running as root, make sure that the SHM file + ** is owned by the same user that owns the original database. Otherwise, + ** the original owner will not be able to connect. If this process is + ** not root, the following fchown() will fail, but we don't care. The + ** if(){..} and the UNIXFILE_CHOWN flag are purely to silence compiler + ** warnings. + */ + if( osFchown(pShmNode->h, sStat.st_uid, sStat.st_gid)==0 ){ + pDbFd->ctrlFlags |= UNIXFILE_CHOWN; } /* Check to see if another process is holding the dead-man switch. @@ -4506,12 +4603,9 @@ typedef const sqlite3_io_methods *(*finder_type)(const char*,unixFile*); static int fillInUnixFile( sqlite3_vfs *pVfs, /* Pointer to vfs object */ int h, /* Open file descriptor of file being opened */ - int syncDir, /* True to sync directory on first sync */ sqlite3_file *pId, /* Write to the unixFile structure here */ const char *zFilename, /* Name of the file being opened */ - int noLock, /* Omit locking if true */ - int isDelete, /* Delete on close if true */ - int isReadOnly /* True if the file is opened read-only */ + int ctrlFlags /* Zero or more UNIXFILE_* values */ ){ const sqlite3_io_methods *pLockingStyle; unixFile *pNew = (unixFile *)pId; @@ -4519,11 +4613,6 @@ static int fillInUnixFile( assert( pNew->pInode==NULL ); - /* Parameter isDelete is only used on vxworks. Express this explicitly - ** here to prevent compiler warnings about unused parameters. - */ - UNUSED_PARAMETER(isDelete); - /* Usually the path zFilename should not be a relative pathname. The ** exception is when opening the proxy "conch" file in builds that ** include the special Apple locking styles. @@ -4536,32 +4625,30 @@ static int fillInUnixFile( #endif /* No locking occurs in temporary files */ - assert( zFilename!=0 || noLock ); + assert( zFilename!=0 || (ctrlFlags & UNIXFILE_NOLOCK)!=0 ); OSTRACE(("OPEN %-3d %s\n", h, zFilename)); pNew->h = h; + pNew->pVfs = pVfs; pNew->zPath = zFilename; - if( memcmp(pVfs->zName,"unix-excl",10)==0 ){ - pNew->ctrlFlags = UNIXFILE_EXCL; - }else{ - pNew->ctrlFlags = 0; - } - if( isReadOnly ){ - pNew->ctrlFlags |= UNIXFILE_RDONLY; + pNew->ctrlFlags = (u8)ctrlFlags; + if( sqlite3_uri_boolean(((ctrlFlags & UNIXFILE_URI) ? zFilename : 0), + "psow", SQLITE_POWERSAFE_OVERWRITE) ){ + pNew->ctrlFlags |= UNIXFILE_PSOW; } - if( syncDir ){ - pNew->ctrlFlags |= UNIXFILE_DIRSYNC; + if( memcmp(pVfs->zName,"unix-excl",10)==0 ){ + pNew->ctrlFlags |= UNIXFILE_EXCL; } #if OS_VXWORKS pNew->pId = vxworksFindFileId(zFilename); if( pNew->pId==0 ){ - noLock = 1; + ctrlFlags |= UNIXFILE_NOLOCK; rc = SQLITE_NOMEM; } #endif - if( noLock ){ + if( ctrlFlags & UNIXFILE_NOLOCK ){ pLockingStyle = &nolockIoMethods; }else{ pLockingStyle = (**(finder_type*)pVfs->pAppData)(zFilename, pNew); @@ -4682,7 +4769,7 @@ static int fillInUnixFile( osUnlink(zFilename); isDelete = 0; } - pNew->isDelete = isDelete; + if( isDelete ) pNew->ctrlFlags |= UNIXFILE_DELETE; #endif if( rc!=SQLITE_OK ){ if( h>=0 ) robust_close(pNew, h, __LINE__); @@ -4747,18 +4834,19 @@ static int unixGetTempname(int nBuf, char *zBuf){ /* Check that the output buffer is large enough for the temporary file ** name. If it is not, return SQLITE_ERROR. */ - if( (strlen(zDir) + strlen(SQLITE_TEMP_FILE_PREFIX) + 17) >= (size_t)nBuf ){ + if( (strlen(zDir) + strlen(SQLITE_TEMP_FILE_PREFIX) + 18) >= (size_t)nBuf ){ return SQLITE_ERROR; } do{ - sqlite3_snprintf(nBuf-17, zBuf, "%s/"SQLITE_TEMP_FILE_PREFIX, zDir); + sqlite3_snprintf(nBuf-18, zBuf, "%s/"SQLITE_TEMP_FILE_PREFIX, zDir); j = (int)strlen(zBuf); sqlite3_randomness(15, &zBuf[j]); for(i=0; i<15; i++, j++){ zBuf[j] = (char)zChars[ ((unsigned char)zBuf[j])%(sizeof(zChars)-1) ]; } zBuf[j] = 0; + zBuf[j+1] = 0; }while( osAccess(zBuf,0)==0 ); return SQLITE_OK; } @@ -4837,12 +4925,10 @@ static UnixUnusedFd *findReusableFd(const char *zPath, int flags){ ** written to *pMode. If an IO error occurs, an SQLite error code is ** returned and the value of *pMode is not modified. ** -** If the file being opened is a temporary file, it is always created with -** the octal permissions 0600 (read/writable by owner only). If the file -** is a database or master journal file, it is created with the permissions -** mask SQLITE_DEFAULT_FILE_PERMISSIONS. -** -** Finally, if the file being opened is a WAL or regular journal file, then +** In most cases cases, this routine sets *pMode to 0, which will become +** an indication to robust_open() to create the file using +** SQLITE_DEFAULT_FILE_PERMISSIONS adjusted by the umask. +** But if the file being opened is a WAL or regular journal file, then ** this function queries the file-system for the permissions on the ** corresponding database file and sets *pMode to this value. Whenever ** possible, WAL and journal files are created using the same permissions @@ -4856,10 +4942,14 @@ static UnixUnusedFd *findReusableFd(const char *zPath, int flags){ static int findCreateFileMode( const char *zPath, /* Path of file (possibly) being created */ int flags, /* Flags passed as 4th argument to xOpen() */ - mode_t *pMode /* OUT: Permissions to open file with */ + mode_t *pMode, /* OUT: Permissions to open file with */ + uid_t *pUid, /* OUT: uid to set on the file */ + gid_t *pGid /* OUT: gid to set on the file */ ){ int rc = SQLITE_OK; /* Return Code */ - *pMode = SQLITE_DEFAULT_FILE_PERMISSIONS; + *pMode = 0; + *pUid = 0; + *pGid = 0; if( flags & (SQLITE_OPEN_WAL|SQLITE_OPEN_MAIN_JOURNAL) ){ char zDb[MAX_PATHNAME+1]; /* Database file path */ int nDb; /* Number of valid bytes in zDb */ @@ -4879,7 +4969,7 @@ static int findCreateFileMode( */ nDb = sqlite3Strlen30(zPath) - 1; #ifdef SQLITE_ENABLE_8_3_NAMES - while( nDb>0 && !sqlite3Isalnum(zPath[nDb]) ) nDb--; + while( nDb>0 && sqlite3Isalnum(zPath[nDb]) ) nDb--; if( nDb==0 || zPath[nDb]!='-' ) return SQLITE_OK; #else while( zPath[nDb]!='-' ){ @@ -4893,6 +4983,8 @@ static int findCreateFileMode( if( 0==osStat(zDb, &sStat) ){ *pMode = sStat.st_mode & 0777; + *pUid = sStat.st_uid; + *pGid = sStat.st_gid; }else{ rc = SQLITE_IOERR_FSTAT; } @@ -4937,6 +5029,7 @@ static int unixOpen( int eType = flags&0xFFFFFF00; /* Type of file to open */ int noLock; /* True to omit locking primitives */ int rc = SQLITE_OK; /* Function Return Code */ + int ctrlFlags = 0; /* UNIXFILE_* flags */ int isExclusive = (flags & SQLITE_OPEN_EXCLUSIVE); int isDelete = (flags & SQLITE_OPEN_DELETEONCLOSE); @@ -4963,7 +5056,7 @@ static int unixOpen( /* If argument zPath is a NULL pointer, this function is required to open ** a temporary file. Use this buffer to store the file name in. */ - char zTmpname[MAX_PATHNAME+1]; + char zTmpname[MAX_PATHNAME+2]; const char *zName = zPath; /* Check the following statements are true: @@ -5006,14 +5099,24 @@ static int unixOpen( } } p->pUnused = pUnused; + + /* Database filenames are double-zero terminated if they are not + ** URIs with parameters. Hence, they can always be passed into + ** sqlite3_uri_parameter(). */ + assert( (flags & SQLITE_OPEN_URI) || zName[strlen(zName)+1]==0 ); + }else if( !zName ){ /* If zName is NULL, the upper layer is requesting a temp file. */ assert(isDelete && !syncDir); - rc = unixGetTempname(MAX_PATHNAME+1, zTmpname); + rc = unixGetTempname(MAX_PATHNAME+2, zTmpname); if( rc!=SQLITE_OK ){ return rc; } zName = zTmpname; + + /* Generated temporary filenames are always double-zero terminated + ** for use by sqlite3_uri_parameter(). */ + assert( zName[strlen(zName)+1]==0 ); } /* Determine the value of the flags parameter passed to POSIX function @@ -5028,7 +5131,9 @@ static int unixOpen( if( fd<0 ){ mode_t openMode; /* Permissions to create file with */ - rc = findCreateFileMode(zName, flags, &openMode); + uid_t uid; /* Userid for the file */ + gid_t gid; /* Groupid for the file */ + rc = findCreateFileMode(zName, flags, &openMode, &uid, &gid); if( rc!=SQLITE_OK ){ assert( !p->pUnused ); assert( eType==SQLITE_OPEN_WAL || eType==SQLITE_OPEN_MAIN_JOURNAL ); @@ -5049,6 +5154,17 @@ static int unixOpen( rc = unixLogError(SQLITE_CANTOPEN_BKPT, "open", zName); goto open_finished; } + + /* If this process is running as root and if creating a new rollback + ** journal or WAL file, set the ownership of the journal or WAL to be + ** the same as the original database. If we are not running as root, + ** then the fchown() call will fail, but that's ok. The "if(){}" and + ** the setting of the UNIXFILE_CHOWN flag are purely to silence compiler + ** warnings from gcc. + */ + if( flags & (SQLITE_OPEN_WAL|SQLITE_OPEN_MAIN_JOURNAL) ){ + if( osFchown(fd, uid, gid)==0 ){ p->ctrlFlags |= UNIXFILE_CHOWN; } + } } assert( fd>=0 ); if( pOutFlags ){ @@ -5073,10 +5189,6 @@ static int unixOpen( } #endif -#ifdef FD_CLOEXEC - osFcntl(fd, F_SETFD, osFcntl(fd, F_GETFD, 0) | FD_CLOEXEC); -#endif - noLock = eType!=SQLITE_OPEN_MAIN_DB; @@ -5090,7 +5202,14 @@ static int unixOpen( ((unixFile*)pFile)->fsFlags |= SQLITE_FSFLAGS_IS_MSDOS; } #endif - + + /* Set up appropriate ctrlFlags */ + if( isDelete ) ctrlFlags |= UNIXFILE_DELETE; + if( isReadonly ) ctrlFlags |= UNIXFILE_RDONLY; + if( noLock ) ctrlFlags |= UNIXFILE_NOLOCK; + if( syncDir ) ctrlFlags |= UNIXFILE_DIRSYNC; + if( flags & SQLITE_OPEN_URI ) ctrlFlags |= UNIXFILE_URI; + #if SQLITE_ENABLE_LOCKING_STYLE #if SQLITE_PREFER_PROXY_LOCKING isAutoProxy = 1; @@ -5120,8 +5239,7 @@ static int unixOpen( useProxy = !(fsInfo.f_flags&MNT_LOCAL); } if( useProxy ){ - rc = fillInUnixFile(pVfs, fd, syncDir, pFile, zPath, noLock, - isDelete, isReadonly); + rc = fillInUnixFile(pVfs, fd, pFile, zPath, ctrlFlags); if( rc==SQLITE_OK ){ rc = proxyTransformUnixFile((unixFile*)pFile, ":auto:"); if( rc!=SQLITE_OK ){ @@ -5138,8 +5256,8 @@ static int unixOpen( } #endif - rc = fillInUnixFile(pVfs, fd, syncDir, pFile, zPath, noLock, - isDelete, isReadonly); + rc = fillInUnixFile(pVfs, fd, pFile, zPath, ctrlFlags); + open_finished: if( rc!=SQLITE_OK ){ sqlite3_free(p->pUnused); @@ -5164,7 +5282,7 @@ static int unixDelete( return unixLogError(SQLITE_IOERR_DELETE, "unlink", zPath); } #ifndef SQLITE_DISABLE_DIRSYNC - if( dirSync ){ + if( (dirSync & 1)!=0 ){ int fd; rc = osOpenDirectory(zPath, &fd); if( rc==SQLITE_OK ){ @@ -5354,7 +5472,7 @@ static int unixRandomness(sqlite3_vfs *NotUsed, int nBuf, char *zBuf){ memset(zBuf, 0, nBuf); #if !defined(SQLITE_TEST) { - int pid, fd; + int pid, fd, got; fd = robust_open("/dev/urandom", O_RDONLY, 0); if( fd<0 ){ time_t t; @@ -5365,7 +5483,7 @@ static int unixRandomness(sqlite3_vfs *NotUsed, int nBuf, char *zBuf){ assert( sizeof(t)+sizeof(pid)<=(size_t)nBuf ); nBuf = sizeof(t) + sizeof(pid); }else{ - do{ nBuf = osRead(fd, zBuf, nBuf); }while( nBuf<0 && errno==EINTR ); + do{ got = osRead(fd, zBuf, nBuf); }while( got<0 && errno==EINTR ); robust_close(0, fd, __LINE__); } } @@ -5715,7 +5833,7 @@ static int proxyCreateLockPath(const char *lockPath){ if( i-start>2 || (i-start==1 && buf[start] != '.' && buf[start] != '/') || (i-start==2 && buf[start] != '.' && buf[start+1] != '.') ){ buf[i]='\0'; - if( mkdir(buf, SQLITE_DEFAULT_PROXYDIR_PERMISSIONS) ){ + if( osMkdir(buf, SQLITE_DEFAULT_PROXYDIR_PERMISSIONS) ){ int err=errno; if( err!=EEXIST ) { OSTRACE(("CREATELOCKPATH FAILED creating %s, " @@ -5769,17 +5887,17 @@ static int proxyCreateUnixFile( } } if( fd<0 ){ - fd = robust_open(path, openFlags, SQLITE_DEFAULT_FILE_PERMISSIONS); + fd = robust_open(path, openFlags, 0); terrno = errno; if( fd<0 && errno==ENOENT && islockfile ){ if( proxyCreateLockPath(path) == SQLITE_OK ){ - fd = robust_open(path, openFlags, SQLITE_DEFAULT_FILE_PERMISSIONS); + fd = robust_open(path, openFlags, 0); } } } if( fd<0 ){ openFlags = O_RDONLY; - fd = robust_open(path, openFlags, SQLITE_DEFAULT_FILE_PERMISSIONS); + fd = robust_open(path, openFlags, 0); terrno = errno; } if( fd<0 ){ @@ -5810,7 +5928,7 @@ static int proxyCreateUnixFile( pUnused->flags = openFlags; pNew->pUnused = pUnused; - rc = fillInUnixFile(&dummyVfs, fd, 0, (sqlite3_file*)pNew, path, 0, 0, 0); + rc = fillInUnixFile(&dummyVfs, fd, (sqlite3_file*)pNew, path, 0); if( rc==SQLITE_OK ){ *ppFile = pNew; return SQLITE_OK; @@ -5903,8 +6021,7 @@ static int proxyBreakConchLock(unixFile *pFile, uuid_t myHostID){ goto end_breaklock; } /* write it out to the temporary break file */ - fd = robust_open(tPath, (O_RDWR|O_CREAT|O_EXCL), - SQLITE_DEFAULT_FILE_PERMISSIONS); + fd = robust_open(tPath, (O_RDWR|O_CREAT|O_EXCL), 0); if( fd<0 ){ sqlite3_snprintf(sizeof(errmsg), errmsg, "create failed (%d)", errno); goto end_breaklock; @@ -6181,8 +6298,7 @@ static int proxyTakeConch(unixFile *pFile){ robust_close(pFile, pFile->h, __LINE__); } pFile->h = -1; - fd = robust_open(pCtx->dbPath, pFile->openFlags, - SQLITE_DEFAULT_FILE_PERMISSIONS); + fd = robust_open(pCtx->dbPath, pFile->openFlags, 0); OSTRACE(("TRANSPROXY: OPEN %d\n", fd)); if( fd>=0 ){ pFile->h = fd; @@ -6751,7 +6867,7 @@ int sqlite3_os_init(void){ /* Double-check that the aSyscall[] array has been constructed ** correctly. See ticket [bb3a86e890c8e96ab] */ - assert( ArraySize(aSyscall)==18 ); + assert( ArraySize(aSyscall)==22 ); /* Register all VFSes defined in the aVfs[] array */ for(i=0; i<(sizeof(aVfs)/sizeof(sqlite3_vfs)); i++){ diff --git a/src/os_win.c b/src/os_win.c index 4518030..fcfe011 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -10,72 +10,27 @@ ** ****************************************************************************** ** -** This file contains code that is specific to windows. +** This file contains code that is specific to Windows. */ #include "sqliteInt.h" -#if SQLITE_OS_WIN /* This file is used for windows only */ - - -/* -** A Note About Memory Allocation: -** -** This driver uses malloc()/free() directly rather than going through -** the SQLite-wrappers sqlite3_malloc()/sqlite3_free(). Those wrappers -** are designed for use on embedded systems where memory is scarce and -** malloc failures happen frequently. Win32 does not typically run on -** embedded systems, and when it does the developers normally have bigger -** problems to worry about than running out of memory. So there is not -** a compelling need to use the wrappers. -** -** But there is a good reason to not use the wrappers. If we use the -** wrappers then we will get simulated malloc() failures within this -** driver. And that causes all kinds of problems for our tests. We -** could enhance SQLite to deal with simulated malloc failures within -** the OS driver, but the code to deal with those failure would not -** be exercised on Linux (which does not need to malloc() in the driver) -** and so we would have difficulty writing coverage tests for that -** code. Better to leave the code out, we think. -** -** The point of this discussion is as follows: When creating a new -** OS layer for an embedded system, if you use this file as an example, -** avoid the use of malloc()/free(). Those routines work ok on windows -** desktops but not so well in embedded systems. -*/ - -#include <winbase.h> +#if SQLITE_OS_WIN /* This file is used for Windows only */ #ifdef __CYGWIN__ # include <sys/cygwin.h> #endif /* -** Macros used to determine whether or not to use threads. -*/ -#if defined(THREADSAFE) && THREADSAFE -# define SQLITE_W32_THREADS 1 -#endif - -/* ** Include code that is common to all os_*.c files */ #include "os_common.h" /* -** Some microsoft compilers lack this definition. +** Some Microsoft compilers lack this definition. */ #ifndef INVALID_FILE_ATTRIBUTES # define INVALID_FILE_ATTRIBUTES ((DWORD)-1) #endif -/* -** Determine if we are dealing with WindowsCE - which has a much -** reduced API. -*/ -#if SQLITE_OS_WINCE -# define AreFileApisANSI() 1 -# define FormatMessageW(a,b,c,d,e,f,g) 0 -#endif - /* Forward references */ typedef struct winShm winShm; /* A connection to shared-memory */ typedef struct winShmNode winShmNode; /* A region of shared-memory */ @@ -104,14 +59,13 @@ struct winFile { HANDLE h; /* Handle for accessing the file */ u8 locktype; /* Type of lock currently held on this file */ short sharedLockByte; /* Randomly chosen byte used as a shared lock */ - u8 bPersistWal; /* True to persist WAL files */ + u8 ctrlFlags; /* Flags. See WINFILE_* below */ DWORD lastErrno; /* The Windows errno from the last I/O error */ - DWORD sectorSize; /* Sector size of the device file is on */ winShm *pShm; /* Instance of shared memory on this file */ const char *zPath; /* Full pathname of this file */ int szChunk; /* Chunk size configured by FCNTL_CHUNK_SIZE */ #if SQLITE_OS_WINCE - WCHAR *zDeleteOnClose; /* Name of file to delete when closing */ + LPWSTR zDeleteOnClose; /* Name of file to delete when closing */ HANDLE hMutex; /* Mutex used to control access to shared lock */ HANDLE hShared; /* Shared memory segment used for locking */ winceLock local; /* Locks obtained by this instance of winFile */ @@ -120,6 +74,12 @@ struct winFile { }; /* +** Allowed values for winFile.ctrlFlags +*/ +#define WINFILE_PERSIST_WAL 0x04 /* Persistent WAL mode */ +#define WINFILE_PSOW 0x10 /* SQLITE_IOCAP_POWERSAFE_OVERWRITE */ + +/* * If compiled with SQLITE_WIN32_MALLOC on Windows, we will use the * various Win32 API heap functions instead of our own. */ @@ -191,20 +151,12 @@ const sqlite3_mem_methods *sqlite3MemGetWin32(void); #endif /* SQLITE_WIN32_MALLOC */ /* -** Forward prototypes. -*/ -static int getSectorSize( - sqlite3_vfs *pVfs, - const char *zRelative /* UTF-8 file name */ -); - -/* ** The following variable is (normally) set once and never changes -** thereafter. It records whether the operating system is Win95 +** thereafter. It records whether the operating system is Win9x ** or WinNT. ** ** 0: Operating system unknown. -** 1: Operating system is Win95. +** 1: Operating system is Win9x. ** 2: Operating system is WinNT. ** ** In order to facilitate testing on a WinNT system, the test fixture @@ -217,6 +169,536 @@ static int sqlite3_os_type = 0; #endif /* +** Many system calls are accessed through pointer-to-functions so that +** they may be overridden at runtime to facilitate fault injection during +** testing and sandboxing. The following array holds the names and pointers +** to all overrideable system calls. +*/ +#if !SQLITE_OS_WINCE +# define SQLITE_WIN32_HAS_ANSI +#endif + +#if SQLITE_OS_WINCE || SQLITE_OS_WINNT +# define SQLITE_WIN32_HAS_WIDE +#endif + +#ifndef SYSCALL +# define SYSCALL sqlite3_syscall_ptr +#endif + +#if SQLITE_OS_WINCE +/* +** These macros are necessary because Windows CE does not natively support the +** Win32 APIs LockFile, UnlockFile, and LockFileEx. + */ + +# define LockFile(a,b,c,d,e) winceLockFile(&a, b, c, d, e) +# define UnlockFile(a,b,c,d,e) winceUnlockFile(&a, b, c, d, e) +# define LockFileEx(a,b,c,d,e,f) winceLockFileEx(&a, b, c, d, e, f) + +/* +** These are the special syscall hacks for Windows CE. The locking related +** defines here refer to the macros defined just above. + */ + +# define osAreFileApisANSI() 1 +# define osLockFile LockFile +# define osUnlockFile UnlockFile +# define osLockFileEx LockFileEx +#endif + +static struct win_syscall { + const char *zName; /* Name of the sytem call */ + sqlite3_syscall_ptr pCurrent; /* Current value of the system call */ + sqlite3_syscall_ptr pDefault; /* Default value */ +} aSyscall[] = { +#if !SQLITE_OS_WINCE + { "AreFileApisANSI", (SYSCALL)AreFileApisANSI, 0 }, + +#define osAreFileApisANSI ((BOOL(WINAPI*)(VOID))aSyscall[0].pCurrent) +#else + { "AreFileApisANSI", (SYSCALL)0, 0 }, +#endif + +#if SQLITE_OS_WINCE && defined(SQLITE_WIN32_HAS_WIDE) + { "CharLowerW", (SYSCALL)CharLowerW, 0 }, +#else + { "CharLowerW", (SYSCALL)0, 0 }, +#endif + +#define osCharLowerW ((LPWSTR(WINAPI*)(LPWSTR))aSyscall[1].pCurrent) + +#if SQLITE_OS_WINCE && defined(SQLITE_WIN32_HAS_WIDE) + { "CharUpperW", (SYSCALL)CharUpperW, 0 }, +#else + { "CharUpperW", (SYSCALL)0, 0 }, +#endif + +#define osCharUpperW ((LPWSTR(WINAPI*)(LPWSTR))aSyscall[2].pCurrent) + + { "CloseHandle", (SYSCALL)CloseHandle, 0 }, + +#define osCloseHandle ((BOOL(WINAPI*)(HANDLE))aSyscall[3].pCurrent) + +#if defined(SQLITE_WIN32_HAS_ANSI) + { "CreateFileA", (SYSCALL)CreateFileA, 0 }, +#else + { "CreateFileA", (SYSCALL)0, 0 }, +#endif + +#define osCreateFileA ((HANDLE(WINAPI*)(LPCSTR,DWORD,DWORD, \ + LPSECURITY_ATTRIBUTES,DWORD,DWORD,HANDLE))aSyscall[4].pCurrent) + +#if defined(SQLITE_WIN32_HAS_WIDE) + { "CreateFileW", (SYSCALL)CreateFileW, 0 }, +#else + { "CreateFileW", (SYSCALL)0, 0 }, +#endif + +#define osCreateFileW ((HANDLE(WINAPI*)(LPCWSTR,DWORD,DWORD, \ + LPSECURITY_ATTRIBUTES,DWORD,DWORD,HANDLE))aSyscall[5].pCurrent) + + { "CreateFileMapping", (SYSCALL)CreateFileMapping, 0 }, + +#define osCreateFileMapping ((HANDLE(WINAPI*)(HANDLE,LPSECURITY_ATTRIBUTES, \ + DWORD,DWORD,DWORD,LPCTSTR))aSyscall[6].pCurrent) + +#if defined(SQLITE_WIN32_HAS_WIDE) + { "CreateFileMappingW", (SYSCALL)CreateFileMappingW, 0 }, +#else + { "CreateFileMappingW", (SYSCALL)0, 0 }, +#endif + +#define osCreateFileMappingW ((HANDLE(WINAPI*)(HANDLE,LPSECURITY_ATTRIBUTES, \ + DWORD,DWORD,DWORD,LPCWSTR))aSyscall[7].pCurrent) + +#if defined(SQLITE_WIN32_HAS_WIDE) + { "CreateMutexW", (SYSCALL)CreateMutexW, 0 }, +#else + { "CreateMutexW", (SYSCALL)0, 0 }, +#endif + +#define osCreateMutexW ((HANDLE(WINAPI*)(LPSECURITY_ATTRIBUTES,BOOL, \ + LPCWSTR))aSyscall[8].pCurrent) + +#if defined(SQLITE_WIN32_HAS_ANSI) + { "DeleteFileA", (SYSCALL)DeleteFileA, 0 }, +#else + { "DeleteFileA", (SYSCALL)0, 0 }, +#endif + +#define osDeleteFileA ((BOOL(WINAPI*)(LPCSTR))aSyscall[9].pCurrent) + +#if defined(SQLITE_WIN32_HAS_WIDE) + { "DeleteFileW", (SYSCALL)DeleteFileW, 0 }, +#else + { "DeleteFileW", (SYSCALL)0, 0 }, +#endif + +#define osDeleteFileW ((BOOL(WINAPI*)(LPCWSTR))aSyscall[10].pCurrent) + +#if SQLITE_OS_WINCE + { "FileTimeToLocalFileTime", (SYSCALL)FileTimeToLocalFileTime, 0 }, +#else + { "FileTimeToLocalFileTime", (SYSCALL)0, 0 }, +#endif + +#define osFileTimeToLocalFileTime ((BOOL(WINAPI*)(CONST FILETIME*, \ + LPFILETIME))aSyscall[11].pCurrent) + +#if SQLITE_OS_WINCE + { "FileTimeToSystemTime", (SYSCALL)FileTimeToSystemTime, 0 }, +#else + { "FileTimeToSystemTime", (SYSCALL)0, 0 }, +#endif + +#define osFileTimeToSystemTime ((BOOL(WINAPI*)(CONST FILETIME*, \ + LPSYSTEMTIME))aSyscall[12].pCurrent) + + { "FlushFileBuffers", (SYSCALL)FlushFileBuffers, 0 }, + +#define osFlushFileBuffers ((BOOL(WINAPI*)(HANDLE))aSyscall[13].pCurrent) + +#if defined(SQLITE_WIN32_HAS_ANSI) + { "FormatMessageA", (SYSCALL)FormatMessageA, 0 }, +#else + { "FormatMessageA", (SYSCALL)0, 0 }, +#endif + +#define osFormatMessageA ((DWORD(WINAPI*)(DWORD,LPCVOID,DWORD,DWORD,LPSTR, \ + DWORD,va_list*))aSyscall[14].pCurrent) + +#if defined(SQLITE_WIN32_HAS_WIDE) + { "FormatMessageW", (SYSCALL)FormatMessageW, 0 }, +#else + { "FormatMessageW", (SYSCALL)0, 0 }, +#endif + +#define osFormatMessageW ((DWORD(WINAPI*)(DWORD,LPCVOID,DWORD,DWORD,LPWSTR, \ + DWORD,va_list*))aSyscall[15].pCurrent) + + { "FreeLibrary", (SYSCALL)FreeLibrary, 0 }, + +#define osFreeLibrary ((BOOL(WINAPI*)(HMODULE))aSyscall[16].pCurrent) + + { "GetCurrentProcessId", (SYSCALL)GetCurrentProcessId, 0 }, + +#define osGetCurrentProcessId ((DWORD(WINAPI*)(VOID))aSyscall[17].pCurrent) + +#if !SQLITE_OS_WINCE && defined(SQLITE_WIN32_HAS_ANSI) + { "GetDiskFreeSpaceA", (SYSCALL)GetDiskFreeSpaceA, 0 }, +#else + { "GetDiskFreeSpaceA", (SYSCALL)0, 0 }, +#endif + +#define osGetDiskFreeSpaceA ((BOOL(WINAPI*)(LPCSTR,LPDWORD,LPDWORD,LPDWORD, \ + LPDWORD))aSyscall[18].pCurrent) + +#if !SQLITE_OS_WINCE && defined(SQLITE_WIN32_HAS_WIDE) + { "GetDiskFreeSpaceW", (SYSCALL)GetDiskFreeSpaceW, 0 }, +#else + { "GetDiskFreeSpaceW", (SYSCALL)0, 0 }, +#endif + +#define osGetDiskFreeSpaceW ((BOOL(WINAPI*)(LPCWSTR,LPDWORD,LPDWORD,LPDWORD, \ + LPDWORD))aSyscall[19].pCurrent) + +#if defined(SQLITE_WIN32_HAS_ANSI) + { "GetFileAttributesA", (SYSCALL)GetFileAttributesA, 0 }, +#else + { "GetFileAttributesA", (SYSCALL)0, 0 }, +#endif + +#define osGetFileAttributesA ((DWORD(WINAPI*)(LPCSTR))aSyscall[20].pCurrent) + +#if defined(SQLITE_WIN32_HAS_WIDE) + { "GetFileAttributesW", (SYSCALL)GetFileAttributesW, 0 }, +#else + { "GetFileAttributesW", (SYSCALL)0, 0 }, +#endif + +#define osGetFileAttributesW ((DWORD(WINAPI*)(LPCWSTR))aSyscall[21].pCurrent) + +#if defined(SQLITE_WIN32_HAS_WIDE) + { "GetFileAttributesExW", (SYSCALL)GetFileAttributesExW, 0 }, +#else + { "GetFileAttributesExW", (SYSCALL)0, 0 }, +#endif + +#define osGetFileAttributesExW ((BOOL(WINAPI*)(LPCWSTR,GET_FILEEX_INFO_LEVELS, \ + LPVOID))aSyscall[22].pCurrent) + + { "GetFileSize", (SYSCALL)GetFileSize, 0 }, + +#define osGetFileSize ((DWORD(WINAPI*)(HANDLE,LPDWORD))aSyscall[23].pCurrent) + +#if !SQLITE_OS_WINCE && defined(SQLITE_WIN32_HAS_ANSI) + { "GetFullPathNameA", (SYSCALL)GetFullPathNameA, 0 }, +#else + { "GetFullPathNameA", (SYSCALL)0, 0 }, +#endif + +#define osGetFullPathNameA ((DWORD(WINAPI*)(LPCSTR,DWORD,LPSTR, \ + LPSTR*))aSyscall[24].pCurrent) + +#if !SQLITE_OS_WINCE && defined(SQLITE_WIN32_HAS_WIDE) + { "GetFullPathNameW", (SYSCALL)GetFullPathNameW, 0 }, +#else + { "GetFullPathNameW", (SYSCALL)0, 0 }, +#endif + +#define osGetFullPathNameW ((DWORD(WINAPI*)(LPCWSTR,DWORD,LPWSTR, \ + LPWSTR*))aSyscall[25].pCurrent) + + { "GetLastError", (SYSCALL)GetLastError, 0 }, + +#define osGetLastError ((DWORD(WINAPI*)(VOID))aSyscall[26].pCurrent) + +#if SQLITE_OS_WINCE + /* The GetProcAddressA() routine is only available on Windows CE. */ + { "GetProcAddressA", (SYSCALL)GetProcAddressA, 0 }, +#else + /* All other Windows platforms expect GetProcAddress() to take + ** an ANSI string regardless of the _UNICODE setting */ + { "GetProcAddressA", (SYSCALL)GetProcAddress, 0 }, +#endif + +#define osGetProcAddressA ((FARPROC(WINAPI*)(HMODULE, \ + LPCSTR))aSyscall[27].pCurrent) + + { "GetSystemInfo", (SYSCALL)GetSystemInfo, 0 }, + +#define osGetSystemInfo ((VOID(WINAPI*)(LPSYSTEM_INFO))aSyscall[28].pCurrent) + + { "GetSystemTime", (SYSCALL)GetSystemTime, 0 }, + +#define osGetSystemTime ((VOID(WINAPI*)(LPSYSTEMTIME))aSyscall[29].pCurrent) + +#if !SQLITE_OS_WINCE + { "GetSystemTimeAsFileTime", (SYSCALL)GetSystemTimeAsFileTime, 0 }, +#else + { "GetSystemTimeAsFileTime", (SYSCALL)0, 0 }, +#endif + +#define osGetSystemTimeAsFileTime ((VOID(WINAPI*)( \ + LPFILETIME))aSyscall[30].pCurrent) + +#if defined(SQLITE_WIN32_HAS_ANSI) + { "GetTempPathA", (SYSCALL)GetTempPathA, 0 }, +#else + { "GetTempPathA", (SYSCALL)0, 0 }, +#endif + +#define osGetTempPathA ((DWORD(WINAPI*)(DWORD,LPSTR))aSyscall[31].pCurrent) + +#if defined(SQLITE_WIN32_HAS_WIDE) + { "GetTempPathW", (SYSCALL)GetTempPathW, 0 }, +#else + { "GetTempPathW", (SYSCALL)0, 0 }, +#endif + +#define osGetTempPathW ((DWORD(WINAPI*)(DWORD,LPWSTR))aSyscall[32].pCurrent) + + { "GetTickCount", (SYSCALL)GetTickCount, 0 }, + +#define osGetTickCount ((DWORD(WINAPI*)(VOID))aSyscall[33].pCurrent) + +#if defined(SQLITE_WIN32_HAS_ANSI) + { "GetVersionExA", (SYSCALL)GetVersionExA, 0 }, +#else + { "GetVersionExA", (SYSCALL)0, 0 }, +#endif + +#define osGetVersionExA ((BOOL(WINAPI*)( \ + LPOSVERSIONINFOA))aSyscall[34].pCurrent) + + { "HeapAlloc", (SYSCALL)HeapAlloc, 0 }, + +#define osHeapAlloc ((LPVOID(WINAPI*)(HANDLE,DWORD, \ + SIZE_T))aSyscall[35].pCurrent) + + { "HeapCreate", (SYSCALL)HeapCreate, 0 }, + +#define osHeapCreate ((HANDLE(WINAPI*)(DWORD,SIZE_T, \ + SIZE_T))aSyscall[36].pCurrent) + + { "HeapDestroy", (SYSCALL)HeapDestroy, 0 }, + +#define osHeapDestroy ((BOOL(WINAPI*)(HANDLE))aSyscall[37].pCurrent) + + { "HeapFree", (SYSCALL)HeapFree, 0 }, + +#define osHeapFree ((BOOL(WINAPI*)(HANDLE,DWORD,LPVOID))aSyscall[38].pCurrent) + + { "HeapReAlloc", (SYSCALL)HeapReAlloc, 0 }, + +#define osHeapReAlloc ((LPVOID(WINAPI*)(HANDLE,DWORD,LPVOID, \ + SIZE_T))aSyscall[39].pCurrent) + + { "HeapSize", (SYSCALL)HeapSize, 0 }, + +#define osHeapSize ((SIZE_T(WINAPI*)(HANDLE,DWORD, \ + LPCVOID))aSyscall[40].pCurrent) + + { "HeapValidate", (SYSCALL)HeapValidate, 0 }, + +#define osHeapValidate ((BOOL(WINAPI*)(HANDLE,DWORD, \ + LPCVOID))aSyscall[41].pCurrent) + +#if defined(SQLITE_WIN32_HAS_ANSI) + { "LoadLibraryA", (SYSCALL)LoadLibraryA, 0 }, +#else + { "LoadLibraryA", (SYSCALL)0, 0 }, +#endif + +#define osLoadLibraryA ((HMODULE(WINAPI*)(LPCSTR))aSyscall[42].pCurrent) + +#if defined(SQLITE_WIN32_HAS_WIDE) + { "LoadLibraryW", (SYSCALL)LoadLibraryW, 0 }, +#else + { "LoadLibraryW", (SYSCALL)0, 0 }, +#endif + +#define osLoadLibraryW ((HMODULE(WINAPI*)(LPCWSTR))aSyscall[43].pCurrent) + + { "LocalFree", (SYSCALL)LocalFree, 0 }, + +#define osLocalFree ((HLOCAL(WINAPI*)(HLOCAL))aSyscall[44].pCurrent) + +#if !SQLITE_OS_WINCE + { "LockFile", (SYSCALL)LockFile, 0 }, + +#define osLockFile ((BOOL(WINAPI*)(HANDLE,DWORD,DWORD,DWORD, \ + DWORD))aSyscall[45].pCurrent) +#else + { "LockFile", (SYSCALL)0, 0 }, +#endif + +#if !SQLITE_OS_WINCE + { "LockFileEx", (SYSCALL)LockFileEx, 0 }, + +#define osLockFileEx ((BOOL(WINAPI*)(HANDLE,DWORD,DWORD,DWORD,DWORD, \ + LPOVERLAPPED))aSyscall[46].pCurrent) +#else + { "LockFileEx", (SYSCALL)0, 0 }, +#endif + + { "MapViewOfFile", (SYSCALL)MapViewOfFile, 0 }, + +#define osMapViewOfFile ((LPVOID(WINAPI*)(HANDLE,DWORD,DWORD,DWORD, \ + SIZE_T))aSyscall[47].pCurrent) + + { "MultiByteToWideChar", (SYSCALL)MultiByteToWideChar, 0 }, + +#define osMultiByteToWideChar ((int(WINAPI*)(UINT,DWORD,LPCSTR,int,LPWSTR, \ + int))aSyscall[48].pCurrent) + + { "QueryPerformanceCounter", (SYSCALL)QueryPerformanceCounter, 0 }, + +#define osQueryPerformanceCounter ((BOOL(WINAPI*)( \ + LARGE_INTEGER*))aSyscall[49].pCurrent) + + { "ReadFile", (SYSCALL)ReadFile, 0 }, + +#define osReadFile ((BOOL(WINAPI*)(HANDLE,LPVOID,DWORD,LPDWORD, \ + LPOVERLAPPED))aSyscall[50].pCurrent) + + { "SetEndOfFile", (SYSCALL)SetEndOfFile, 0 }, + +#define osSetEndOfFile ((BOOL(WINAPI*)(HANDLE))aSyscall[51].pCurrent) + + { "SetFilePointer", (SYSCALL)SetFilePointer, 0 }, + +#define osSetFilePointer ((DWORD(WINAPI*)(HANDLE,LONG,PLONG, \ + DWORD))aSyscall[52].pCurrent) + + { "Sleep", (SYSCALL)Sleep, 0 }, + +#define osSleep ((VOID(WINAPI*)(DWORD))aSyscall[53].pCurrent) + + { "SystemTimeToFileTime", (SYSCALL)SystemTimeToFileTime, 0 }, + +#define osSystemTimeToFileTime ((BOOL(WINAPI*)(CONST SYSTEMTIME*, \ + LPFILETIME))aSyscall[54].pCurrent) + +#if !SQLITE_OS_WINCE + { "UnlockFile", (SYSCALL)UnlockFile, 0 }, + +#define osUnlockFile ((BOOL(WINAPI*)(HANDLE,DWORD,DWORD,DWORD, \ + DWORD))aSyscall[55].pCurrent) +#else + { "UnlockFile", (SYSCALL)0, 0 }, +#endif + +#if !SQLITE_OS_WINCE + { "UnlockFileEx", (SYSCALL)UnlockFileEx, 0 }, + +#define osUnlockFileEx ((BOOL(WINAPI*)(HANDLE,DWORD,DWORD,DWORD, \ + LPOVERLAPPED))aSyscall[56].pCurrent) +#else + { "UnlockFileEx", (SYSCALL)0, 0 }, +#endif + + { "UnmapViewOfFile", (SYSCALL)UnmapViewOfFile, 0 }, + +#define osUnmapViewOfFile ((BOOL(WINAPI*)(LPCVOID))aSyscall[57].pCurrent) + + { "WideCharToMultiByte", (SYSCALL)WideCharToMultiByte, 0 }, + +#define osWideCharToMultiByte ((int(WINAPI*)(UINT,DWORD,LPCWSTR,int,LPSTR,int, \ + LPCSTR,LPBOOL))aSyscall[58].pCurrent) + + { "WriteFile", (SYSCALL)WriteFile, 0 }, + +#define osWriteFile ((BOOL(WINAPI*)(HANDLE,LPCVOID,DWORD,LPDWORD, \ + LPOVERLAPPED))aSyscall[59].pCurrent) + +}; /* End of the overrideable system calls */ + +/* +** This is the xSetSystemCall() method of sqlite3_vfs for all of the +** "win32" VFSes. Return SQLITE_OK opon successfully updating the +** system call pointer, or SQLITE_NOTFOUND if there is no configurable +** system call named zName. +*/ +static int winSetSystemCall( + sqlite3_vfs *pNotUsed, /* The VFS pointer. Not used */ + const char *zName, /* Name of system call to override */ + sqlite3_syscall_ptr pNewFunc /* Pointer to new system call value */ +){ + unsigned int i; + int rc = SQLITE_NOTFOUND; + + UNUSED_PARAMETER(pNotUsed); + if( zName==0 ){ + /* If no zName is given, restore all system calls to their default + ** settings and return NULL + */ + rc = SQLITE_OK; + for(i=0; i<sizeof(aSyscall)/sizeof(aSyscall[0]); i++){ + if( aSyscall[i].pDefault ){ + aSyscall[i].pCurrent = aSyscall[i].pDefault; + } + } + }else{ + /* If zName is specified, operate on only the one system call + ** specified. + */ + for(i=0; i<sizeof(aSyscall)/sizeof(aSyscall[0]); i++){ + if( strcmp(zName, aSyscall[i].zName)==0 ){ + if( aSyscall[i].pDefault==0 ){ + aSyscall[i].pDefault = aSyscall[i].pCurrent; + } + rc = SQLITE_OK; + if( pNewFunc==0 ) pNewFunc = aSyscall[i].pDefault; + aSyscall[i].pCurrent = pNewFunc; + break; + } + } + } + return rc; +} + +/* +** Return the value of a system call. Return NULL if zName is not a +** recognized system call name. NULL is also returned if the system call +** is currently undefined. +*/ +static sqlite3_syscall_ptr winGetSystemCall( + sqlite3_vfs *pNotUsed, + const char *zName +){ + unsigned int i; + + UNUSED_PARAMETER(pNotUsed); + for(i=0; i<sizeof(aSyscall)/sizeof(aSyscall[0]); i++){ + if( strcmp(zName, aSyscall[i].zName)==0 ) return aSyscall[i].pCurrent; + } + return 0; +} + +/* +** Return the name of the first system call after zName. If zName==NULL +** then return the name of the first system call. Return NULL if zName +** is the last system call or if zName is not the name of a valid +** system call. +*/ +static const char *winNextSystemCall(sqlite3_vfs *p, const char *zName){ + int i = -1; + + UNUSED_PARAMETER(p); + if( zName ){ + for(i=0; i<ArraySize(aSyscall)-1; i++){ + if( strcmp(zName, aSyscall[i].zName)==0 ) break; + } + } + for(i++; i<ArraySize(aSyscall); i++){ + if( aSyscall[i].pCurrent!=0 ) return aSyscall[i].zName; + } + return 0; +} + +/* ** Return true (non-zero) if we are running under WinNT, Win2K, WinXP, ** or WinCE. Return false (zero) for Win95, Win98, or WinME. ** @@ -232,9 +714,9 @@ static int sqlite3_os_type = 0; #else static int isNT(void){ if( sqlite3_os_type==0 ){ - OSVERSIONINFO sInfo; + OSVERSIONINFOA sInfo; sInfo.dwOSVersionInfoSize = sizeof(sInfo); - GetVersionEx(&sInfo); + osGetVersionExA(&sInfo); sqlite3_os_type = sInfo.dwPlatformId==VER_PLATFORM_WIN32_NT ? 2 : 1; } return sqlite3_os_type==2; @@ -254,13 +736,13 @@ static void *winMemMalloc(int nBytes){ assert( hHeap!=0 ); assert( hHeap!=INVALID_HANDLE_VALUE ); #ifdef SQLITE_WIN32_MALLOC_VALIDATE - assert ( HeapValidate(hHeap, SQLITE_WIN32_HEAP_FLAGS, NULL) ); + assert ( osHeapValidate(hHeap, SQLITE_WIN32_HEAP_FLAGS, NULL) ); #endif assert( nBytes>=0 ); - p = HeapAlloc(hHeap, SQLITE_WIN32_HEAP_FLAGS, (SIZE_T)nBytes); + p = osHeapAlloc(hHeap, SQLITE_WIN32_HEAP_FLAGS, (SIZE_T)nBytes); if( !p ){ sqlite3_log(SQLITE_NOMEM, "failed to HeapAlloc %u bytes (%d), heap=%p", - nBytes, GetLastError(), (void*)hHeap); + nBytes, osGetLastError(), (void*)hHeap); } return p; } @@ -276,12 +758,12 @@ static void winMemFree(void *pPrior){ assert( hHeap!=0 ); assert( hHeap!=INVALID_HANDLE_VALUE ); #ifdef SQLITE_WIN32_MALLOC_VALIDATE - assert ( HeapValidate(hHeap, SQLITE_WIN32_HEAP_FLAGS, pPrior) ); + assert ( osHeapValidate(hHeap, SQLITE_WIN32_HEAP_FLAGS, pPrior) ); #endif if( !pPrior ) return; /* Passing NULL to HeapFree is undefined. */ - if( !HeapFree(hHeap, SQLITE_WIN32_HEAP_FLAGS, pPrior) ){ + if( !osHeapFree(hHeap, SQLITE_WIN32_HEAP_FLAGS, pPrior) ){ sqlite3_log(SQLITE_NOMEM, "failed to HeapFree block %p (%d), heap=%p", - pPrior, GetLastError(), (void*)hHeap); + pPrior, osGetLastError(), (void*)hHeap); } } @@ -297,18 +779,18 @@ static void *winMemRealloc(void *pPrior, int nBytes){ assert( hHeap!=0 ); assert( hHeap!=INVALID_HANDLE_VALUE ); #ifdef SQLITE_WIN32_MALLOC_VALIDATE - assert ( HeapValidate(hHeap, SQLITE_WIN32_HEAP_FLAGS, pPrior) ); + assert ( osHeapValidate(hHeap, SQLITE_WIN32_HEAP_FLAGS, pPrior) ); #endif assert( nBytes>=0 ); if( !pPrior ){ - p = HeapAlloc(hHeap, SQLITE_WIN32_HEAP_FLAGS, (SIZE_T)nBytes); + p = osHeapAlloc(hHeap, SQLITE_WIN32_HEAP_FLAGS, (SIZE_T)nBytes); }else{ - p = HeapReAlloc(hHeap, SQLITE_WIN32_HEAP_FLAGS, pPrior, (SIZE_T)nBytes); + p = osHeapReAlloc(hHeap, SQLITE_WIN32_HEAP_FLAGS, pPrior, (SIZE_T)nBytes); } if( !p ){ sqlite3_log(SQLITE_NOMEM, "failed to %s %u bytes (%d), heap=%p", - pPrior ? "HeapReAlloc" : "HeapAlloc", nBytes, GetLastError(), - (void*)hHeap); + pPrior ? "HeapReAlloc" : "HeapAlloc", nBytes, osGetLastError(), + (void*)hHeap); } return p; } @@ -325,13 +807,13 @@ static int winMemSize(void *p){ assert( hHeap!=0 ); assert( hHeap!=INVALID_HANDLE_VALUE ); #ifdef SQLITE_WIN32_MALLOC_VALIDATE - assert ( HeapValidate(hHeap, SQLITE_WIN32_HEAP_FLAGS, NULL) ); + assert ( osHeapValidate(hHeap, SQLITE_WIN32_HEAP_FLAGS, NULL) ); #endif if( !p ) return 0; - n = HeapSize(hHeap, SQLITE_WIN32_HEAP_FLAGS, p); + n = osHeapSize(hHeap, SQLITE_WIN32_HEAP_FLAGS, p); if( n==(SIZE_T)-1 ){ sqlite3_log(SQLITE_NOMEM, "failed to HeapSize block %p (%d), heap=%p", - p, GetLastError(), (void*)hHeap); + p, osGetLastError(), (void*)hHeap); return 0; } return (int)n; @@ -353,14 +835,14 @@ static int winMemInit(void *pAppData){ if( !pWinMemData ) return SQLITE_ERROR; assert( pWinMemData->magic==WINMEM_MAGIC ); if( !pWinMemData->hHeap ){ - pWinMemData->hHeap = HeapCreate(SQLITE_WIN32_HEAP_FLAGS, - SQLITE_WIN32_HEAP_INIT_SIZE, - SQLITE_WIN32_HEAP_MAX_SIZE); + pWinMemData->hHeap = osHeapCreate(SQLITE_WIN32_HEAP_FLAGS, + SQLITE_WIN32_HEAP_INIT_SIZE, + SQLITE_WIN32_HEAP_MAX_SIZE); if( !pWinMemData->hHeap ){ sqlite3_log(SQLITE_NOMEM, "failed to HeapCreate (%d), flags=%u, initSize=%u, maxSize=%u", - GetLastError(), SQLITE_WIN32_HEAP_FLAGS, SQLITE_WIN32_HEAP_INIT_SIZE, - SQLITE_WIN32_HEAP_MAX_SIZE); + osGetLastError(), SQLITE_WIN32_HEAP_FLAGS, + SQLITE_WIN32_HEAP_INIT_SIZE, SQLITE_WIN32_HEAP_MAX_SIZE); return SQLITE_NOMEM; } pWinMemData->bOwned = TRUE; @@ -368,7 +850,7 @@ static int winMemInit(void *pAppData){ assert( pWinMemData->hHeap!=0 ); assert( pWinMemData->hHeap!=INVALID_HANDLE_VALUE ); #ifdef SQLITE_WIN32_MALLOC_VALIDATE - assert( HeapValidate(pWinMemData->hHeap, SQLITE_WIN32_HEAP_FLAGS, NULL) ); + assert( osHeapValidate(pWinMemData->hHeap, SQLITE_WIN32_HEAP_FLAGS, NULL) ); #endif return SQLITE_OK; } @@ -383,12 +865,12 @@ static void winMemShutdown(void *pAppData){ if( pWinMemData->hHeap ){ assert( pWinMemData->hHeap!=INVALID_HANDLE_VALUE ); #ifdef SQLITE_WIN32_MALLOC_VALIDATE - assert( HeapValidate(pWinMemData->hHeap, SQLITE_WIN32_HEAP_FLAGS, NULL) ); + assert( osHeapValidate(pWinMemData->hHeap, SQLITE_WIN32_HEAP_FLAGS, NULL) ); #endif if( pWinMemData->bOwned ){ - if( !HeapDestroy(pWinMemData->hHeap) ){ + if( !osHeapDestroy(pWinMemData->hHeap) ){ sqlite3_log(SQLITE_NOMEM, "failed to HeapDestroy (%d), heap=%p", - GetLastError(), (void*)pWinMemData->hHeap); + osGetLastError(), (void*)pWinMemData->hHeap); } pWinMemData->bOwned = FALSE; } @@ -424,95 +906,110 @@ void sqlite3MemSetDefault(void){ #endif /* SQLITE_WIN32_MALLOC */ /* -** Convert a UTF-8 string to microsoft unicode (UTF-16?). +** Convert a UTF-8 string to Microsoft Unicode (UTF-16?). ** ** Space to hold the returned string is obtained from malloc. */ -static WCHAR *utf8ToUnicode(const char *zFilename){ +static LPWSTR utf8ToUnicode(const char *zFilename){ int nChar; - WCHAR *zWideFilename; + LPWSTR zWideFilename; - nChar = MultiByteToWideChar(CP_UTF8, 0, zFilename, -1, NULL, 0); - zWideFilename = malloc( nChar*sizeof(zWideFilename[0]) ); + nChar = osMultiByteToWideChar(CP_UTF8, 0, zFilename, -1, NULL, 0); + if( nChar==0 ){ + return 0; + } + zWideFilename = sqlite3_malloc( nChar*sizeof(zWideFilename[0]) ); if( zWideFilename==0 ){ return 0; } - nChar = MultiByteToWideChar(CP_UTF8, 0, zFilename, -1, zWideFilename, nChar); + nChar = osMultiByteToWideChar(CP_UTF8, 0, zFilename, -1, zWideFilename, + nChar); if( nChar==0 ){ - free(zWideFilename); + sqlite3_free(zWideFilename); zWideFilename = 0; } return zWideFilename; } /* -** Convert microsoft unicode to UTF-8. Space to hold the returned string is -** obtained from malloc(). +** Convert Microsoft Unicode to UTF-8. Space to hold the returned string is +** obtained from sqlite3_malloc(). */ -static char *unicodeToUtf8(const WCHAR *zWideFilename){ +static char *unicodeToUtf8(LPCWSTR zWideFilename){ int nByte; char *zFilename; - nByte = WideCharToMultiByte(CP_UTF8, 0, zWideFilename, -1, 0, 0, 0, 0); - zFilename = malloc( nByte ); + nByte = osWideCharToMultiByte(CP_UTF8, 0, zWideFilename, -1, 0, 0, 0, 0); + if( nByte == 0 ){ + return 0; + } + zFilename = sqlite3_malloc( nByte ); if( zFilename==0 ){ return 0; } - nByte = WideCharToMultiByte(CP_UTF8, 0, zWideFilename, -1, zFilename, nByte, - 0, 0); + nByte = osWideCharToMultiByte(CP_UTF8, 0, zWideFilename, -1, zFilename, nByte, + 0, 0); if( nByte == 0 ){ - free(zFilename); + sqlite3_free(zFilename); zFilename = 0; } return zFilename; } /* -** Convert an ansi string to microsoft unicode, based on the +** Convert an ANSI string to Microsoft Unicode, based on the ** current codepage settings for file apis. ** ** Space to hold the returned string is obtained -** from malloc. +** from sqlite3_malloc. */ -static WCHAR *mbcsToUnicode(const char *zFilename){ +static LPWSTR mbcsToUnicode(const char *zFilename){ int nByte; - WCHAR *zMbcsFilename; - int codepage = AreFileApisANSI() ? CP_ACP : CP_OEMCP; + LPWSTR zMbcsFilename; + int codepage = osAreFileApisANSI() ? CP_ACP : CP_OEMCP; - nByte = MultiByteToWideChar(codepage, 0, zFilename, -1, NULL,0)*sizeof(WCHAR); - zMbcsFilename = malloc( nByte*sizeof(zMbcsFilename[0]) ); + nByte = osMultiByteToWideChar(codepage, 0, zFilename, -1, NULL, + 0)*sizeof(WCHAR); + if( nByte==0 ){ + return 0; + } + zMbcsFilename = sqlite3_malloc( nByte*sizeof(zMbcsFilename[0]) ); if( zMbcsFilename==0 ){ return 0; } - nByte = MultiByteToWideChar(codepage, 0, zFilename, -1, zMbcsFilename, nByte); + nByte = osMultiByteToWideChar(codepage, 0, zFilename, -1, zMbcsFilename, + nByte); if( nByte==0 ){ - free(zMbcsFilename); + sqlite3_free(zMbcsFilename); zMbcsFilename = 0; } return zMbcsFilename; } /* -** Convert microsoft unicode to multibyte character string, based on the -** user's Ansi codepage. +** Convert Microsoft Unicode to multi-byte character string, based on the +** user's ANSI codepage. ** ** Space to hold the returned string is obtained from -** malloc(). +** sqlite3_malloc(). */ -static char *unicodeToMbcs(const WCHAR *zWideFilename){ +static char *unicodeToMbcs(LPCWSTR zWideFilename){ int nByte; char *zFilename; - int codepage = AreFileApisANSI() ? CP_ACP : CP_OEMCP; + int codepage = osAreFileApisANSI() ? CP_ACP : CP_OEMCP; - nByte = WideCharToMultiByte(codepage, 0, zWideFilename, -1, 0, 0, 0, 0); - zFilename = malloc( nByte ); + nByte = osWideCharToMultiByte(codepage, 0, zWideFilename, -1, 0, 0, 0, 0); + if( nByte == 0 ){ + return 0; + } + zFilename = sqlite3_malloc( nByte ); if( zFilename==0 ){ return 0; } - nByte = WideCharToMultiByte(codepage, 0, zWideFilename, -1, zFilename, nByte, - 0, 0); + nByte = osWideCharToMultiByte(codepage, 0, zWideFilename, -1, zFilename, + nByte, 0, 0); if( nByte == 0 ){ - free(zFilename); + sqlite3_free(zFilename); zFilename = 0; } return zFilename; @@ -520,35 +1017,35 @@ static char *unicodeToMbcs(const WCHAR *zWideFilename){ /* ** Convert multibyte character string to UTF-8. Space to hold the -** returned string is obtained from malloc(). +** returned string is obtained from sqlite3_malloc(). */ char *sqlite3_win32_mbcs_to_utf8(const char *zFilename){ char *zFilenameUtf8; - WCHAR *zTmpWide; + LPWSTR zTmpWide; zTmpWide = mbcsToUnicode(zFilename); if( zTmpWide==0 ){ return 0; } zFilenameUtf8 = unicodeToUtf8(zTmpWide); - free(zTmpWide); + sqlite3_free(zTmpWide); return zFilenameUtf8; } /* ** Convert UTF-8 to multibyte character string. Space to hold the -** returned string is obtained from malloc(). +** returned string is obtained from sqlite3_malloc(). */ char *sqlite3_win32_utf8_to_mbcs(const char *zFilename){ char *zFilenameMbcs; - WCHAR *zTmpWide; + LPWSTR zTmpWide; zTmpWide = utf8ToUnicode(zFilename); if( zTmpWide==0 ){ return 0; } zFilenameMbcs = unicodeToMbcs(zTmpWide); - free(zTmpWide); + sqlite3_free(zTmpWide); return zFilenameMbcs; } @@ -558,59 +1055,66 @@ char *sqlite3_win32_utf8_to_mbcs(const char *zFilename){ ** is zero if the error message fits in the buffer, or non-zero ** otherwise (if the message was truncated). */ -static int getLastErrorMsg(int nBuf, char *zBuf){ +static int getLastErrorMsg(DWORD lastErrno, int nBuf, char *zBuf){ /* FormatMessage returns 0 on failure. Otherwise it ** returns the number of TCHARs written to the output ** buffer, excluding the terminating null char. */ - DWORD error = GetLastError(); DWORD dwLen = 0; char *zOut = 0; if( isNT() ){ - WCHAR *zTempWide = NULL; - dwLen = FormatMessageW(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, - NULL, - error, - 0, - (LPWSTR) &zTempWide, - 0, - 0); + LPWSTR zTempWide = NULL; + dwLen = osFormatMessageW(FORMAT_MESSAGE_ALLOCATE_BUFFER | + FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, + lastErrno, + 0, + (LPWSTR) &zTempWide, + 0, + 0); if( dwLen > 0 ){ /* allocate a buffer and convert to UTF8 */ + sqlite3BeginBenignMalloc(); zOut = unicodeToUtf8(zTempWide); + sqlite3EndBenignMalloc(); /* free the system buffer allocated by FormatMessage */ - LocalFree(zTempWide); + osLocalFree(zTempWide); } /* isNT() is 1 if SQLITE_OS_WINCE==1, so this else is never executed. -** Since the ASCII version of these Windows API do not exist for WINCE, +** Since the ANSI version of these Windows API do not exist for WINCE, ** it's important to not reference them for WINCE builds. */ #if SQLITE_OS_WINCE==0 }else{ char *zTemp = NULL; - dwLen = FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, - NULL, - error, - 0, - (LPSTR) &zTemp, - 0, - 0); + dwLen = osFormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | + FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, + lastErrno, + 0, + (LPSTR) &zTemp, + 0, + 0); if( dwLen > 0 ){ /* allocate a buffer and convert to UTF8 */ + sqlite3BeginBenignMalloc(); zOut = sqlite3_win32_mbcs_to_utf8(zTemp); + sqlite3EndBenignMalloc(); /* free the system buffer allocated by FormatMessage */ - LocalFree(zTemp); + osLocalFree(zTemp); } #endif } if( 0 == dwLen ){ - sqlite3_snprintf(nBuf, zBuf, "OsError 0x%x (%u)", error, error); + sqlite3_snprintf(nBuf, zBuf, "OsError 0x%x (%u)", lastErrno, lastErrno); }else{ /* copy a maximum of nBuf chars to output buffer */ sqlite3_snprintf(nBuf, zBuf, "%s", zOut); /* free the UTF8 buffer */ - free(zOut); + sqlite3_free(zOut); } return 0; } @@ -630,26 +1134,26 @@ static int getLastErrorMsg(int nBuf, char *zBuf){ ** The two subsequent arguments should be the name of the OS function that ** failed and the the associated file-system path, if any. */ -#define winLogError(a,b,c) winLogErrorAtLine(a,b,c,__LINE__) +#define winLogError(a,b,c,d) winLogErrorAtLine(a,b,c,d,__LINE__) static int winLogErrorAtLine( int errcode, /* SQLite error code */ + DWORD lastErrno, /* Win32 last error */ const char *zFunc, /* Name of OS function that failed */ const char *zPath, /* File path associated with error */ int iLine /* Source line number where error occurred */ ){ char zMsg[500]; /* Human readable error text */ int i; /* Loop counter */ - DWORD iErrno = GetLastError(); /* Error code */ zMsg[0] = 0; - getLastErrorMsg(sizeof(zMsg), zMsg); + getLastErrorMsg(lastErrno, sizeof(zMsg), zMsg); assert( errcode!=SQLITE_OK ); if( zPath==0 ) zPath = ""; for(i=0; zMsg[i] && zMsg[i]!='\r' && zMsg[i]!='\n'; i++){} zMsg[i] = 0; sqlite3_log(errcode, "os_win.c:%d: (%d) %s(%s) - %s", - iLine, iErrno, zFunc, zPath, zMsg + iLine, lastErrno, zFunc, zPath, zMsg ); return errcode; @@ -675,19 +1179,24 @@ static int win32IoerrRetryDelay = SQLITE_WIN32_IOERR_RETRY_DELAY; ** to see if it should be retried. Return TRUE to retry. Return FALSE ** to give up with an error. */ -static int retryIoerr(int *pnRetry){ - DWORD e; +static int retryIoerr(int *pnRetry, DWORD *pError){ + DWORD e = osGetLastError(); if( *pnRetry>=win32IoerrRetry ){ + if( pError ){ + *pError = e; + } return 0; } - e = GetLastError(); if( e==ERROR_ACCESS_DENIED || e==ERROR_LOCK_VIOLATION || e==ERROR_SHARING_VIOLATION ){ - Sleep(win32IoerrRetryDelay*(1+*pnRetry)); + osSleep(win32IoerrRetryDelay*(1+*pnRetry)); ++*pnRetry; return 1; } + if( pError ){ + *pError = e; + } return 0; } @@ -708,7 +1217,7 @@ static void logIoerr(int nRetry){ ** This section contains code for WinCE only. */ /* -** WindowsCE does not have a localtime() function. So create a +** Windows CE does not have a localtime() function. So create a ** substitute. */ #include <time.h> @@ -722,8 +1231,8 @@ struct tm *__cdecl localtime(const time_t *t) t64 = (t64 + 11644473600)*10000000; uTm.dwLowDateTime = (DWORD)(t64 & 0xFFFFFFFF); uTm.dwHighDateTime= (DWORD)(t64 >> 32); - FileTimeToLocalFileTime(&uTm,&lTm); - FileTimeToSystemTime(&lTm,&pTm); + osFileTimeToLocalFileTime(&uTm,&lTm); + osFileTimeToSystemTime(&lTm,&pTm); y.tm_year = pTm.wYear - 1900; y.tm_mon = pTm.wMonth - 1; y.tm_wday = pTm.wDayOfWeek; @@ -734,13 +1243,6 @@ struct tm *__cdecl localtime(const time_t *t) return &y; } -/* This will never be called, but defined to make the code compile */ -#define GetTempPathA(a,b) - -#define LockFile(a,b,c,d,e) winceLockFile(&a, b, c, d, e) -#define UnlockFile(a,b,c,d,e) winceUnlockFile(&a, b, c, d, e) -#define LockFileEx(a,b,c,d,e,f) winceLockFileEx(&a, b, c, d, e, f) - #define HANDLE_TO_WINFILE(a) (winFile*)&((char*)a)[-(int)offsetof(winFile,h)] /* @@ -762,26 +1264,32 @@ static void winceMutexAcquire(HANDLE h){ ** descriptor pFile */ static BOOL winceCreateLock(const char *zFilename, winFile *pFile){ - WCHAR *zTok; - WCHAR *zName = utf8ToUnicode(zFilename); + LPWSTR zTok; + LPWSTR zName; BOOL bInit = TRUE; + zName = utf8ToUnicode(zFilename); + if( zName==0 ){ + /* out of memory */ + return FALSE; + } + /* Initialize the local lockdata */ - ZeroMemory(&pFile->local, sizeof(pFile->local)); + memset(&pFile->local, 0, sizeof(pFile->local)); /* Replace the backslashes from the filename and lowercase it ** to derive a mutex name. */ - zTok = CharLowerW(zName); + zTok = osCharLowerW(zName); for (;*zTok;zTok++){ if (*zTok == '\\') *zTok = '_'; } /* Create/open the named mutex */ - pFile->hMutex = CreateMutexW(NULL, FALSE, zName); + pFile->hMutex = osCreateMutexW(NULL, FALSE, zName); if (!pFile->hMutex){ - pFile->lastErrno = GetLastError(); - winLogError(SQLITE_ERROR, "winceCreateLock1", zFilename); - free(zName); + pFile->lastErrno = osGetLastError(); + winLogError(SQLITE_ERROR, pFile->lastErrno, "winceCreateLock1", zFilename); + sqlite3_free(zName); return FALSE; } @@ -792,28 +1300,29 @@ static BOOL winceCreateLock(const char *zFilename, winFile *pFile){ ** case-sensitive, take advantage of that by uppercasing the mutex name ** and using that as the shared filemapping name. */ - CharUpperW(zName); - pFile->hShared = CreateFileMappingW(INVALID_HANDLE_VALUE, NULL, - PAGE_READWRITE, 0, sizeof(winceLock), - zName); + osCharUpperW(zName); + pFile->hShared = osCreateFileMappingW(INVALID_HANDLE_VALUE, NULL, + PAGE_READWRITE, 0, sizeof(winceLock), + zName); /* Set a flag that indicates we're the first to create the memory so it ** must be zero-initialized */ - if (GetLastError() == ERROR_ALREADY_EXISTS){ + if (osGetLastError() == ERROR_ALREADY_EXISTS){ bInit = FALSE; } - free(zName); + sqlite3_free(zName); /* If we succeeded in making the shared memory handle, map it. */ if (pFile->hShared){ - pFile->shared = (winceLock*)MapViewOfFile(pFile->hShared, + pFile->shared = (winceLock*)osMapViewOfFile(pFile->hShared, FILE_MAP_READ|FILE_MAP_WRITE, 0, 0, sizeof(winceLock)); /* If mapping failed, close the shared memory handle and erase it */ if (!pFile->shared){ - pFile->lastErrno = GetLastError(); - winLogError(SQLITE_ERROR, "winceCreateLock2", zFilename); - CloseHandle(pFile->hShared); + pFile->lastErrno = osGetLastError(); + winLogError(SQLITE_ERROR, pFile->lastErrno, + "winceCreateLock2", zFilename); + osCloseHandle(pFile->hShared); pFile->hShared = NULL; } } @@ -821,14 +1330,14 @@ static BOOL winceCreateLock(const char *zFilename, winFile *pFile){ /* If shared memory could not be created, then close the mutex and fail */ if (pFile->hShared == NULL){ winceMutexRelease(pFile->hMutex); - CloseHandle(pFile->hMutex); + osCloseHandle(pFile->hMutex); pFile->hMutex = NULL; return FALSE; } /* Initialize the shared memory if we're supposed to */ if (bInit) { - ZeroMemory(pFile->shared, sizeof(winceLock)); + memset(pFile->shared, 0, sizeof(winceLock)); } winceMutexRelease(pFile->hMutex); @@ -859,18 +1368,18 @@ static void winceDestroyLock(winFile *pFile){ } /* De-reference and close our copy of the shared memory handle */ - UnmapViewOfFile(pFile->shared); - CloseHandle(pFile->hShared); + osUnmapViewOfFile(pFile->shared); + osCloseHandle(pFile->hShared); /* Done with the mutex */ winceMutexRelease(pFile->hMutex); - CloseHandle(pFile->hMutex); + osCloseHandle(pFile->hMutex); pFile->hMutex = NULL; } } /* -** An implementation of the LockFile() API of windows for wince +** An implementation of the LockFile() API of Windows for CE */ static BOOL winceLockFile( HANDLE *phFile, @@ -934,7 +1443,7 @@ static BOOL winceLockFile( } /* -** An implementation of the UnlockFile API of windows for wince +** An implementation of the UnlockFile API of Windows for CE */ static BOOL winceUnlockFile( HANDLE *phFile, @@ -996,7 +1505,7 @@ static BOOL winceUnlockFile( } /* -** An implementation of the LockFileEx() API of windows for wince +** An implementation of the LockFileEx() API of Windows for CE */ static BOOL winceLockFileEx( HANDLE *phFile, @@ -1029,7 +1538,7 @@ static BOOL winceLockFileEx( ******************************************************************************/ /* -** Some microsoft compilers lack this definition. +** Some Microsoft compilers lack this definition. */ #ifndef INVALID_SET_FILE_POINTER # define INVALID_SET_FILE_POINTER ((DWORD)-1) @@ -1044,6 +1553,7 @@ static int seekWinFile(winFile *pFile, sqlite3_int64 iOffset){ LONG upperBits; /* Most sig. 32 bits of new offset */ LONG lowerBits; /* Least sig. 32 bits of new offset */ DWORD dwRet; /* Value returned by SetFilePointer() */ + DWORD lastErrno; /* Value returned by GetLastError() */ upperBits = (LONG)((iOffset>>32) & 0x7fffffff); lowerBits = (LONG)(iOffset & 0xffffffff); @@ -1055,10 +1565,13 @@ static int seekWinFile(winFile *pFile, sqlite3_int64 iOffset){ ** whether an error has actually occured, it is also necessary to call ** GetLastError(). */ - dwRet = SetFilePointer(pFile->h, lowerBits, &upperBits, FILE_BEGIN); - if( (dwRet==INVALID_SET_FILE_POINTER && GetLastError()!=NO_ERROR) ){ - pFile->lastErrno = GetLastError(); - winLogError(SQLITE_IOERR_SEEK, "seekWinFile", pFile->zPath); + dwRet = osSetFilePointer(pFile->h, lowerBits, &upperBits, FILE_BEGIN); + + if( (dwRet==INVALID_SET_FILE_POINTER + && ((lastErrno = osGetLastError())!=NO_ERROR)) ){ + pFile->lastErrno = lastErrno; + winLogError(SQLITE_IOERR_SEEK, pFile->lastErrno, + "seekWinFile", pFile->zPath); return 1; } @@ -1069,7 +1582,7 @@ static int seekWinFile(winFile *pFile, sqlite3_int64 iOffset){ ** Close a file. ** ** It is reported that an attempt to close a handle might sometimes -** fail. This is a very unreasonable result, but windows is notorious +** fail. This is a very unreasonable result, but Windows is notorious ** for being unreasonable so I do not doubt that it might happen. If ** the close fails, we pause for 100 milliseconds and try again. As ** many as MX_CLOSE_ATTEMPT attempts to close the handle are made before @@ -1084,28 +1597,32 @@ static int winClose(sqlite3_file *id){ assert( pFile->pShm==0 ); OSTRACE(("CLOSE %d\n", pFile->h)); do{ - rc = CloseHandle(pFile->h); + rc = osCloseHandle(pFile->h); /* SimulateIOError( rc=0; cnt=MX_CLOSE_ATTEMPT; ); */ - }while( rc==0 && ++cnt < MX_CLOSE_ATTEMPT && (Sleep(100), 1) ); + }while( rc==0 && ++cnt < MX_CLOSE_ATTEMPT && (osSleep(100), 1) ); #if SQLITE_OS_WINCE #define WINCE_DELETION_ATTEMPTS 3 winceDestroyLock(pFile); if( pFile->zDeleteOnClose ){ int cnt = 0; while( - DeleteFileW(pFile->zDeleteOnClose)==0 - && GetFileAttributesW(pFile->zDeleteOnClose)!=0xffffffff + osDeleteFileW(pFile->zDeleteOnClose)==0 + && osGetFileAttributesW(pFile->zDeleteOnClose)!=0xffffffff && cnt++ < WINCE_DELETION_ATTEMPTS ){ - Sleep(100); /* Wait a little before trying again */ + osSleep(100); /* Wait a little before trying again */ } - free(pFile->zDeleteOnClose); + sqlite3_free(pFile->zDeleteOnClose); } #endif OSTRACE(("CLOSE %d %s\n", pFile->h, rc ? "ok" : "failed")); + if( rc ){ + pFile->h = NULL; + } OpenCounter(-1); return rc ? SQLITE_OK - : winLogError(SQLITE_IOERR_CLOSE, "winClose", pFile->zPath); + : winLogError(SQLITE_IOERR_CLOSE, osGetLastError(), + "winClose", pFile->zPath); } /* @@ -1119,6 +1636,9 @@ static int winRead( int amt, /* Number of bytes to read */ sqlite3_int64 offset /* Begin reading at this offset */ ){ +#if !SQLITE_OS_WINCE + OVERLAPPED overlapped; /* The offset for ReadFile. */ +#endif winFile *pFile = (winFile*)id; /* file handle */ DWORD nRead; /* Number of bytes actually read from file */ int nRetry = 0; /* Number of retrys */ @@ -1127,13 +1647,23 @@ static int winRead( SimulateIOError(return SQLITE_IOERR_READ); OSTRACE(("READ %d lock=%d\n", pFile->h, pFile->locktype)); +#if SQLITE_OS_WINCE if( seekWinFile(pFile, offset) ){ return SQLITE_FULL; } - while( !ReadFile(pFile->h, pBuf, amt, &nRead, 0) ){ - if( retryIoerr(&nRetry) ) continue; - pFile->lastErrno = GetLastError(); - return winLogError(SQLITE_IOERR_READ, "winRead", pFile->zPath); + while( !osReadFile(pFile->h, pBuf, amt, &nRead, 0) ){ +#else + memset(&overlapped, 0, sizeof(OVERLAPPED)); + overlapped.Offset = (LONG)(offset & 0xffffffff); + overlapped.OffsetHigh = (LONG)((offset>>32) & 0x7fffffff); + while( !osReadFile(pFile->h, pBuf, amt, &nRead, &overlapped) && + osGetLastError()!=ERROR_HANDLE_EOF ){ +#endif + DWORD lastErrno; + if( retryIoerr(&nRetry, &lastErrno) ) continue; + pFile->lastErrno = lastErrno; + return winLogError(SQLITE_IOERR_READ, pFile->lastErrno, + "winRead", pFile->zPath); } logIoerr(nRetry); if( nRead<(DWORD)amt ){ @@ -1155,7 +1685,7 @@ static int winWrite( int amt, /* Number of bytes to write */ sqlite3_int64 offset /* Offset into the file to begin writing at */ ){ - int rc; /* True if error has occured, else false */ + int rc = 0; /* True if error has occured, else false */ winFile *pFile = (winFile*)id; /* File handle */ int nRetry = 0; /* Number of retries */ @@ -1166,23 +1696,49 @@ static int winWrite( OSTRACE(("WRITE %d lock=%d\n", pFile->h, pFile->locktype)); +#if SQLITE_OS_WINCE rc = seekWinFile(pFile, offset); if( rc==0 ){ +#else + { +#endif +#if !SQLITE_OS_WINCE + OVERLAPPED overlapped; /* The offset for WriteFile. */ +#endif u8 *aRem = (u8 *)pBuf; /* Data yet to be written */ int nRem = amt; /* Number of bytes yet to be written */ DWORD nWrite; /* Bytes written by each WriteFile() call */ + DWORD lastErrno = NO_ERROR; /* Value returned by GetLastError() */ + +#if !SQLITE_OS_WINCE + memset(&overlapped, 0, sizeof(OVERLAPPED)); + overlapped.Offset = (LONG)(offset & 0xffffffff); + overlapped.OffsetHigh = (LONG)((offset>>32) & 0x7fffffff); +#endif while( nRem>0 ){ - if( !WriteFile(pFile->h, aRem, nRem, &nWrite, 0) ){ - if( retryIoerr(&nRetry) ) continue; +#if SQLITE_OS_WINCE + if( !osWriteFile(pFile->h, aRem, nRem, &nWrite, 0) ){ +#else + if( !osWriteFile(pFile->h, aRem, nRem, &nWrite, &overlapped) ){ +#endif + if( retryIoerr(&nRetry, &lastErrno) ) continue; + break; + } + if( nWrite<=0 ){ + lastErrno = osGetLastError(); break; } - if( nWrite<=0 ) break; +#if !SQLITE_OS_WINCE + offset += nWrite; + overlapped.Offset = (LONG)(offset & 0xffffffff); + overlapped.OffsetHigh = (LONG)((offset>>32) & 0x7fffffff); +#endif aRem += nWrite; nRem -= nWrite; } if( nRem>0 ){ - pFile->lastErrno = GetLastError(); + pFile->lastErrno = lastErrno; rc = 1; } } @@ -1192,7 +1748,8 @@ static int winWrite( || ( pFile->lastErrno==ERROR_DISK_FULL )){ return SQLITE_FULL; } - return winLogError(SQLITE_IOERR_WRITE, "winWrite", pFile->zPath); + return winLogError(SQLITE_IOERR_WRITE, pFile->lastErrno, + "winWrite", pFile->zPath); }else{ logIoerr(nRetry); } @@ -1222,10 +1779,12 @@ static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ /* SetEndOfFile() returns non-zero when successful, or zero when it fails. */ if( seekWinFile(pFile, nByte) ){ - rc = winLogError(SQLITE_IOERR_TRUNCATE, "winTruncate1", pFile->zPath); - }else if( 0==SetEndOfFile(pFile->h) ){ - pFile->lastErrno = GetLastError(); - rc = winLogError(SQLITE_IOERR_TRUNCATE, "winTruncate2", pFile->zPath); + rc = winLogError(SQLITE_IOERR_TRUNCATE, pFile->lastErrno, + "winTruncate1", pFile->zPath); + }else if( 0==osSetEndOfFile(pFile->h) ){ + pFile->lastErrno = osGetLastError(); + rc = winLogError(SQLITE_IOERR_TRUNCATE, pFile->lastErrno, + "winTruncate2", pFile->zPath); } OSTRACE(("TRUNCATE %d %lld %s\n", pFile->h, nByte, rc ? "failed" : "ok")); @@ -1290,13 +1849,14 @@ static int winSync(sqlite3_file *id, int flags){ #ifdef SQLITE_NO_SYNC return SQLITE_OK; #else - rc = FlushFileBuffers(pFile->h); + rc = osFlushFileBuffers(pFile->h); SimulateIOError( rc=FALSE ); if( rc ){ return SQLITE_OK; }else{ - pFile->lastErrno = GetLastError(); - return winLogError(SQLITE_IOERR_FSYNC, "winSync", pFile->zPath); + pFile->lastErrno = osGetLastError(); + return winLogError(SQLITE_IOERR_FSYNC, pFile->lastErrno, + "winSync", pFile->zPath); } #endif } @@ -1308,16 +1868,17 @@ static int winFileSize(sqlite3_file *id, sqlite3_int64 *pSize){ DWORD upperBits; DWORD lowerBits; winFile *pFile = (winFile*)id; - DWORD error; + DWORD lastErrno; assert( id!=0 ); SimulateIOError(return SQLITE_IOERR_FSTAT); - lowerBits = GetFileSize(pFile->h, &upperBits); + lowerBits = osGetFileSize(pFile->h, &upperBits); if( (lowerBits == INVALID_FILE_SIZE) - && ((error = GetLastError()) != NO_ERROR) ) + && ((lastErrno = osGetLastError())!=NO_ERROR) ) { - pFile->lastErrno = error; - return winLogError(SQLITE_IOERR_FSTAT, "winFileSize", pFile->zPath); + pFile->lastErrno = lastErrno; + return winLogError(SQLITE_IOERR_FSTAT, pFile->lastErrno, + "winFileSize", pFile->zPath); } *pSize = (((sqlite3_int64)upperBits)<<32) + lowerBits; return SQLITE_OK; @@ -1333,7 +1894,7 @@ static int winFileSize(sqlite3_file *id, sqlite3_int64 *pSize){ /* ** Acquire a reader lock. ** Different API routines are called depending on whether or not this -** is Win95 or WinNT. +** is Win9x or WinNT. */ static int getReadLock(winFile *pFile){ int res; @@ -1342,8 +1903,8 @@ static int getReadLock(winFile *pFile){ ovlp.Offset = SHARED_FIRST; ovlp.OffsetHigh = 0; ovlp.hEvent = 0; - res = LockFileEx(pFile->h, LOCKFILE_FAIL_IMMEDIATELY, - 0, SHARED_SIZE, 0, &ovlp); + res = osLockFileEx(pFile->h, LOCKFILE_FAIL_IMMEDIATELY, + 0, SHARED_SIZE, 0, &ovlp); /* isNT() is 1 if SQLITE_OS_WINCE==1, so this else is never executed. */ #if SQLITE_OS_WINCE==0 @@ -1351,11 +1912,11 @@ static int getReadLock(winFile *pFile){ int lk; sqlite3_randomness(sizeof(lk), &lk); pFile->sharedLockByte = (short)((lk & 0x7fffffff)%(SHARED_SIZE - 1)); - res = LockFile(pFile->h, SHARED_FIRST+pFile->sharedLockByte, 0, 1, 0); + res = osLockFile(pFile->h, SHARED_FIRST+pFile->sharedLockByte, 0, 1, 0); #endif } if( res == 0 ){ - pFile->lastErrno = GetLastError(); + pFile->lastErrno = osGetLastError(); /* No need to log a failure to lock */ } return res; @@ -1366,18 +1927,20 @@ static int getReadLock(winFile *pFile){ */ static int unlockReadLock(winFile *pFile){ int res; + DWORD lastErrno; if( isNT() ){ - res = UnlockFile(pFile->h, SHARED_FIRST, 0, SHARED_SIZE, 0); + res = osUnlockFile(pFile->h, SHARED_FIRST, 0, SHARED_SIZE, 0); /* isNT() is 1 if SQLITE_OS_WINCE==1, so this else is never executed. */ #if SQLITE_OS_WINCE==0 }else{ - res = UnlockFile(pFile->h, SHARED_FIRST + pFile->sharedLockByte, 0, 1, 0); + res = osUnlockFile(pFile->h, SHARED_FIRST + pFile->sharedLockByte, 0, 1, 0); #endif } - if( res==0 && GetLastError()!=ERROR_NOT_LOCKED ){ - pFile->lastErrno = GetLastError(); - winLogError(SQLITE_IOERR_UNLOCK, "unlockReadLock", pFile->zPath); + if( res==0 && ((lastErrno = osGetLastError())!=ERROR_NOT_LOCKED) ){ + pFile->lastErrno = lastErrno; + winLogError(SQLITE_IOERR_UNLOCK, pFile->lastErrno, + "unlockReadLock", pFile->zPath); } return res; } @@ -1410,11 +1973,11 @@ static int unlockReadLock(winFile *pFile){ */ static int winLock(sqlite3_file *id, int locktype){ int rc = SQLITE_OK; /* Return code from subroutines */ - int res = 1; /* Result of a windows lock call */ + int res = 1; /* Result of a Windows lock call */ int newLocktype; /* Set pFile->locktype to this value before exiting */ int gotPendingLock = 0;/* True if we acquired a PENDING lock this time */ winFile *pFile = (winFile*)id; - DWORD error = NO_ERROR; + DWORD lastErrno = NO_ERROR; assert( id!=0 ); OSTRACE(("LOCK %d %d was %d(%d)\n", @@ -1444,16 +2007,19 @@ static int winLock(sqlite3_file *id, int locktype){ && (pFile->locktype==RESERVED_LOCK)) ){ int cnt = 3; - while( cnt-->0 && (res = LockFile(pFile->h, PENDING_BYTE, 0, 1, 0))==0 ){ - /* Try 3 times to get the pending lock. The pending lock might be - ** held by another reader process who will release it momentarily. + while( cnt-->0 && (res = osLockFile(pFile->h, PENDING_BYTE, 0, 1, 0))==0 ){ + /* Try 3 times to get the pending lock. This is needed to work + ** around problems caused by indexing and/or anti-virus software on + ** Windows systems. + ** If you are using this code as a model for alternative VFSes, do not + ** copy this retry logic. It is a hack intended for Windows only. */ OSTRACE(("could not get a PENDING lock. cnt=%d\n", cnt)); - Sleep(1); + if( cnt ) osSleep(1); } gotPendingLock = res; if( !res ){ - error = GetLastError(); + lastErrno = osGetLastError(); } } @@ -1465,7 +2031,7 @@ static int winLock(sqlite3_file *id, int locktype){ if( res ){ newLocktype = SHARED_LOCK; }else{ - error = GetLastError(); + lastErrno = osGetLastError(); } } @@ -1473,11 +2039,11 @@ static int winLock(sqlite3_file *id, int locktype){ */ if( locktype==RESERVED_LOCK && res ){ assert( pFile->locktype==SHARED_LOCK ); - res = LockFile(pFile->h, RESERVED_BYTE, 0, 1, 0); + res = osLockFile(pFile->h, RESERVED_BYTE, 0, 1, 0); if( res ){ newLocktype = RESERVED_LOCK; }else{ - error = GetLastError(); + lastErrno = osGetLastError(); } } @@ -1494,12 +2060,12 @@ static int winLock(sqlite3_file *id, int locktype){ assert( pFile->locktype>=SHARED_LOCK ); res = unlockReadLock(pFile); OSTRACE(("unreadlock = %d\n", res)); - res = LockFile(pFile->h, SHARED_FIRST, 0, SHARED_SIZE, 0); + res = osLockFile(pFile->h, SHARED_FIRST, 0, SHARED_SIZE, 0); if( res ){ newLocktype = EXCLUSIVE_LOCK; }else{ - error = GetLastError(); - OSTRACE(("error-code = %d\n", error)); + lastErrno = osGetLastError(); + OSTRACE(("error-code = %d\n", lastErrno)); getReadLock(pFile); } } @@ -1508,7 +2074,7 @@ static int winLock(sqlite3_file *id, int locktype){ ** release it now. */ if( gotPendingLock && locktype==SHARED_LOCK ){ - UnlockFile(pFile->h, PENDING_BYTE, 0, 1, 0); + osUnlockFile(pFile->h, PENDING_BYTE, 0, 1, 0); } /* Update the state of the lock has held in the file descriptor then @@ -1519,7 +2085,7 @@ static int winLock(sqlite3_file *id, int locktype){ }else{ OSTRACE(("LOCK FAILED %d trying for %d but got %d\n", pFile->h, locktype, newLocktype)); - pFile->lastErrno = error; + pFile->lastErrno = lastErrno; rc = SQLITE_BUSY; } pFile->locktype = (u8)newLocktype; @@ -1542,9 +2108,9 @@ static int winCheckReservedLock(sqlite3_file *id, int *pResOut){ rc = 1; OSTRACE(("TEST WR-LOCK %d %d (local)\n", pFile->h, rc)); }else{ - rc = LockFile(pFile->h, RESERVED_BYTE, 0, 1, 0); + rc = osLockFile(pFile->h, RESERVED_BYTE, 0, 1, 0); if( rc ){ - UnlockFile(pFile->h, RESERVED_BYTE, 0, 1, 0); + osUnlockFile(pFile->h, RESERVED_BYTE, 0, 1, 0); } rc = !rc; OSTRACE(("TEST WR-LOCK %d %d (remote)\n", pFile->h, rc)); @@ -1574,27 +2140,44 @@ static int winUnlock(sqlite3_file *id, int locktype){ pFile->locktype, pFile->sharedLockByte)); type = pFile->locktype; if( type>=EXCLUSIVE_LOCK ){ - UnlockFile(pFile->h, SHARED_FIRST, 0, SHARED_SIZE, 0); + osUnlockFile(pFile->h, SHARED_FIRST, 0, SHARED_SIZE, 0); if( locktype==SHARED_LOCK && !getReadLock(pFile) ){ /* This should never happen. We should always be able to ** reacquire the read lock */ - rc = winLogError(SQLITE_IOERR_UNLOCK, "winUnlock", pFile->zPath); + rc = winLogError(SQLITE_IOERR_UNLOCK, osGetLastError(), + "winUnlock", pFile->zPath); } } if( type>=RESERVED_LOCK ){ - UnlockFile(pFile->h, RESERVED_BYTE, 0, 1, 0); + osUnlockFile(pFile->h, RESERVED_BYTE, 0, 1, 0); } if( locktype==NO_LOCK && type>=SHARED_LOCK ){ unlockReadLock(pFile); } if( type>=PENDING_LOCK ){ - UnlockFile(pFile->h, PENDING_BYTE, 0, 1, 0); + osUnlockFile(pFile->h, PENDING_BYTE, 0, 1, 0); } pFile->locktype = (u8)locktype; return rc; } /* +** If *pArg is inititially negative then this is a query. Set *pArg to +** 1 or 0 depending on whether or not bit mask of pFile->ctrlFlags is set. +** +** If *pArg is 0 or 1, then clear or set the mask bit of pFile->ctrlFlags. +*/ +static void winModeBit(winFile *pFile, unsigned char mask, int *pArg){ + if( *pArg<0 ){ + *pArg = (pFile->ctrlFlags & mask)!=0; + }else if( (*pArg)==0 ){ + pFile->ctrlFlags &= ~mask; + }else{ + pFile->ctrlFlags |= mask; + } +} + +/* ** Control and query of the open file handle. */ static int winFileControl(sqlite3_file *id, int op, void *pArg){ @@ -1629,15 +2212,15 @@ static int winFileControl(sqlite3_file *id, int op, void *pArg){ return SQLITE_OK; } case SQLITE_FCNTL_PERSIST_WAL: { - int bPersist = *(int*)pArg; - if( bPersist<0 ){ - *(int*)pArg = pFile->bPersistWal; - }else{ - pFile->bPersistWal = bPersist!=0; - } + winModeBit(pFile, WINFILE_PERSIST_WAL, (int*)pArg); return SQLITE_OK; } - case SQLITE_FCNTL_SYNC_OMITTED: { + case SQLITE_FCNTL_POWERSAFE_OVERWRITE: { + winModeBit(pFile, WINFILE_PSOW, (int*)pArg); + return SQLITE_OK; + } + case SQLITE_FCNTL_VFSNAME: { + *(char**)pArg = sqlite3_mprintf("win32"); return SQLITE_OK; } case SQLITE_FCNTL_WIN32_AV_RETRY: { @@ -1669,16 +2252,17 @@ static int winFileControl(sqlite3_file *id, int op, void *pArg){ ** same for both. */ static int winSectorSize(sqlite3_file *id){ - assert( id!=0 ); - return (int)(((winFile*)id)->sectorSize); + (void)id; + return SQLITE_DEFAULT_SECTOR_SIZE; } /* ** Return a vector of device characteristics. */ static int winDeviceCharacteristics(sqlite3_file *id){ - UNUSED_PARAMETER(id); - return SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN; + winFile *p = (winFile*)id; + return SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN | + ((p->ctrlFlags & WINFILE_PSOW)?SQLITE_IOCAP_POWERSAFE_OVERWRITE:0); } #ifndef SQLITE_OMIT_WAL @@ -1825,15 +2409,15 @@ static int winShmSystemLock( /* Release/Acquire the system-level lock */ if( lockType==_SHM_UNLCK ){ - rc = UnlockFileEx(pFile->hFile.h, 0, nByte, 0, &ovlp); + rc = osUnlockFileEx(pFile->hFile.h, 0, nByte, 0, &ovlp); }else{ - rc = LockFileEx(pFile->hFile.h, dwFlags, 0, nByte, 0, &ovlp); + rc = osLockFileEx(pFile->hFile.h, dwFlags, 0, nByte, 0, &ovlp); } if( rc!= 0 ){ rc = SQLITE_OK; }else{ - pFile->lastErrno = GetLastError(); + pFile->lastErrno = osGetLastError(); rc = SQLITE_BUSY; } @@ -1867,13 +2451,13 @@ static void winShmPurge(sqlite3_vfs *pVfs, int deleteFlag){ int i; if( p->mutex ) sqlite3_mutex_free(p->mutex); for(i=0; i<p->nRegion; i++){ - bRc = UnmapViewOfFile(p->aRegion[i].pMap); + bRc = osUnmapViewOfFile(p->aRegion[i].pMap); OSTRACE(("SHM-PURGE pid-%d unmap region=%d %s\n", - (int)GetCurrentProcessId(), i, + (int)osGetCurrentProcessId(), i, bRc ? "ok" : "failed")); - bRc = CloseHandle(p->aRegion[i].hMap); + bRc = osCloseHandle(p->aRegion[i].hMap); OSTRACE(("SHM-PURGE pid-%d close region=%d %s\n", - (int)GetCurrentProcessId(), i, + (int)osGetCurrentProcessId(), i, bRc ? "ok" : "failed")); } if( p->hFile.h != INVALID_HANDLE_VALUE ){ @@ -1883,7 +2467,9 @@ static void winShmPurge(sqlite3_vfs *pVfs, int deleteFlag){ } if( deleteFlag ){ SimulateIOErrorBenign(1); + sqlite3BeginBenignMalloc(); winDelete(pVfs, p->zFilename, 0); + sqlite3EndBenignMalloc(); SimulateIOErrorBenign(0); } *pp = p->pNext; @@ -1915,15 +2501,15 @@ static int winOpenSharedMemory(winFile *pDbFd){ ** allocate space for a new winShmNode and filename. */ p = sqlite3_malloc( sizeof(*p) ); - if( p==0 ) return SQLITE_NOMEM; + if( p==0 ) return SQLITE_IOERR_NOMEM; memset(p, 0, sizeof(*p)); nName = sqlite3Strlen30(pDbFd->zPath); - pNew = sqlite3_malloc( sizeof(*pShmNode) + nName + 15 ); + pNew = sqlite3_malloc( sizeof(*pShmNode) + nName + 17 ); if( pNew==0 ){ sqlite3_free(p); - return SQLITE_NOMEM; + return SQLITE_IOERR_NOMEM; } - memset(pNew, 0, sizeof(*pNew)); + memset(pNew, 0, sizeof(*pNew) + nName + 17); pNew->zFilename = (char*)&pNew[1]; sqlite3_snprintf(nName+15, pNew->zFilename, "%s-shm", pDbFd->zPath); sqlite3FileSuffix3(pDbFd->zPath, pNew->zFilename); @@ -1949,7 +2535,7 @@ static int winOpenSharedMemory(winFile *pDbFd){ pShmNode->mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); if( pShmNode->mutex==0 ){ - rc = SQLITE_NOMEM; + rc = SQLITE_IOERR_NOMEM; goto shm_open_err; } @@ -1959,7 +2545,6 @@ static int winOpenSharedMemory(winFile *pDbFd){ SQLITE_OPEN_WAL | SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, /* Mode flags */ 0); if( SQLITE_OK!=rc ){ - rc = SQLITE_CANTOPEN_BKPT; goto shm_open_err; } @@ -1969,7 +2554,8 @@ static int winOpenSharedMemory(winFile *pDbFd){ if( winShmSystemLock(pShmNode, _SHM_WRLCK, WIN_SHM_DMS, 1)==SQLITE_OK ){ rc = winTruncate((sqlite3_file *)&pShmNode->hFile, 0); if( rc!=SQLITE_OK ){ - rc = winLogError(SQLITE_IOERR_SHMOPEN, "winOpenShm", pDbFd->zPath); + rc = winLogError(SQLITE_IOERR_SHMOPEN, osGetLastError(), + "winOpenShm", pDbFd->zPath); } } if( rc==SQLITE_OK ){ @@ -2154,7 +2740,7 @@ static int winShmLock( } sqlite3_mutex_leave(pShmNode->mutex); OSTRACE(("SHM-LOCK shmid-%d, pid-%d got %03x,%03x %s\n", - p->id, (int)GetCurrentProcessId(), p->sharedMask, p->exclMask, + p->id, (int)osGetCurrentProcessId(), p->sharedMask, p->exclMask, rc ? "failed" : "ok")); return rc; } @@ -2228,7 +2814,8 @@ static int winShmMap( */ rc = winFileSize((sqlite3_file *)&pShmNode->hFile, &sz); if( rc!=SQLITE_OK ){ - rc = winLogError(SQLITE_IOERR_SHMSIZE, "winShmMap1", pDbFd->zPath); + rc = winLogError(SQLITE_IOERR_SHMSIZE, osGetLastError(), + "winShmMap1", pDbFd->zPath); goto shmpage_out; } @@ -2242,7 +2829,8 @@ static int winShmMap( if( !isWrite ) goto shmpage_out; rc = winTruncate((sqlite3_file *)&pShmNode->hFile, nByte); if( rc!=SQLITE_OK ){ - rc = winLogError(SQLITE_IOERR_SHMSIZE, "winShmMap2", pDbFd->zPath); + rc = winLogError(SQLITE_IOERR_SHMSIZE, osGetLastError(), + "winShmMap2", pDbFd->zPath); goto shmpage_out; } } @@ -2261,26 +2849,27 @@ static int winShmMap( HANDLE hMap; /* file-mapping handle */ void *pMap = 0; /* Mapped memory region */ - hMap = CreateFileMapping(pShmNode->hFile.h, + hMap = osCreateFileMapping(pShmNode->hFile.h, NULL, PAGE_READWRITE, 0, nByte, NULL ); OSTRACE(("SHM-MAP pid-%d create region=%d nbyte=%d %s\n", - (int)GetCurrentProcessId(), pShmNode->nRegion, nByte, + (int)osGetCurrentProcessId(), pShmNode->nRegion, nByte, hMap ? "ok" : "failed")); if( hMap ){ int iOffset = pShmNode->nRegion*szRegion; int iOffsetShift = iOffset % winSysInfo.dwAllocationGranularity; - pMap = MapViewOfFile(hMap, FILE_MAP_WRITE | FILE_MAP_READ, + pMap = osMapViewOfFile(hMap, FILE_MAP_WRITE | FILE_MAP_READ, 0, iOffset - iOffsetShift, szRegion + iOffsetShift ); OSTRACE(("SHM-MAP pid-%d map region=%d offset=%d size=%d %s\n", - (int)GetCurrentProcessId(), pShmNode->nRegion, iOffset, szRegion, - pMap ? "ok" : "failed")); + (int)osGetCurrentProcessId(), pShmNode->nRegion, iOffset, + szRegion, pMap ? "ok" : "failed")); } if( !pMap ){ - pShmNode->lastErrno = GetLastError(); - rc = winLogError(SQLITE_IOERR_SHMMAP, "winShmMap3", pDbFd->zPath); - if( hMap ) CloseHandle(hMap); + pShmNode->lastErrno = osGetLastError(); + rc = winLogError(SQLITE_IOERR_SHMMAP, pShmNode->lastErrno, + "winShmMap3", pDbFd->zPath); + if( hMap ) osCloseHandle(hMap); goto shmpage_out; } @@ -2378,7 +2967,7 @@ static int getTempname(int nBuf, char *zBuf){ "ABCDEFGHIJKLMNOPQRSTUVWXYZ" "0123456789"; size_t i, j; - char zTempPath[MAX_PATH+1]; + char zTempPath[MAX_PATH+2]; /* It's odd to simulate an io-error here, but really this is just ** using the io-error infrastructure to test that SQLite handles this @@ -2391,29 +2980,29 @@ static int getTempname(int nBuf, char *zBuf){ }else if( isNT() ){ char *zMulti; WCHAR zWidePath[MAX_PATH]; - GetTempPathW(MAX_PATH-30, zWidePath); + osGetTempPathW(MAX_PATH-30, zWidePath); zMulti = unicodeToUtf8(zWidePath); if( zMulti ){ sqlite3_snprintf(MAX_PATH-30, zTempPath, "%s", zMulti); - free(zMulti); + sqlite3_free(zMulti); }else{ - return SQLITE_NOMEM; + return SQLITE_IOERR_NOMEM; } /* isNT() is 1 if SQLITE_OS_WINCE==1, so this else is never executed. -** Since the ASCII version of these Windows API do not exist for WINCE, +** Since the ANSI version of these Windows API do not exist for WINCE, ** it's important to not reference them for WINCE builds. */ #if SQLITE_OS_WINCE==0 }else{ char *zUtf8; char zMbcsPath[MAX_PATH]; - GetTempPathA(MAX_PATH-30, zMbcsPath); + osGetTempPathA(MAX_PATH-30, zMbcsPath); zUtf8 = sqlite3_win32_mbcs_to_utf8(zMbcsPath); if( zUtf8 ){ sqlite3_snprintf(MAX_PATH-30, zTempPath, "%s", zUtf8); - free(zUtf8); + sqlite3_free(zUtf8); }else{ - return SQLITE_NOMEM; + return SQLITE_IOERR_NOMEM; } #endif } @@ -2421,14 +3010,14 @@ static int getTempname(int nBuf, char *zBuf){ /* Check that the output buffer is large enough for the temporary file ** name. If it is not, return SQLITE_ERROR. */ - if( (sqlite3Strlen30(zTempPath) + sqlite3Strlen30(SQLITE_TEMP_FILE_PREFIX) + 17) >= nBuf ){ + if( (sqlite3Strlen30(zTempPath) + sqlite3Strlen30(SQLITE_TEMP_FILE_PREFIX) + 18) >= nBuf ){ return SQLITE_ERROR; } for(i=sqlite3Strlen30(zTempPath); i>0 && zTempPath[i-1]=='\\'; i--){} zTempPath[i] = 0; - sqlite3_snprintf(nBuf-17, zBuf, + sqlite3_snprintf(nBuf-18, zBuf, "%s\\"SQLITE_TEMP_FILE_PREFIX, zTempPath); j = sqlite3Strlen30(zBuf); sqlite3_randomness(15, &zBuf[j]); @@ -2436,12 +3025,42 @@ static int getTempname(int nBuf, char *zBuf){ zBuf[j] = (char)zChars[ ((unsigned char)zBuf[j])%(sizeof(zChars)-1) ]; } zBuf[j] = 0; + zBuf[j+1] = 0; OSTRACE(("TEMP FILENAME: %s\n", zBuf)); return SQLITE_OK; } /* +** Return TRUE if the named file is really a directory. Return false if +** it is something other than a directory, or if there is any kind of memory +** allocation failure. +*/ +static int winIsDir(const void *zConverted){ + DWORD attr; + int rc = 0; + DWORD lastErrno; + + if( isNT() ){ + int cnt = 0; + WIN32_FILE_ATTRIBUTE_DATA sAttrData; + memset(&sAttrData, 0, sizeof(sAttrData)); + while( !(rc = osGetFileAttributesExW((LPCWSTR)zConverted, + GetFileExInfoStandard, + &sAttrData)) && retryIoerr(&cnt, &lastErrno) ){} + if( !rc ){ + return 0; /* Invalid name? */ + } + attr = sAttrData.dwFileAttributes; +#if SQLITE_OS_WINCE==0 + }else{ + attr = osGetFileAttributesA((char*)zConverted); +#endif + } + return (attr!=INVALID_FILE_ATTRIBUTES) && (attr&FILE_ATTRIBUTE_DIRECTORY); +} + +/* ** Open a file. */ static int winOpen( @@ -2452,6 +3071,7 @@ static int winOpen( int *pOutFlags /* Status return flags */ ){ HANDLE h; + DWORD lastErrno; DWORD dwDesiredAccess; DWORD dwShareMode; DWORD dwCreationDisposition; @@ -2467,7 +3087,7 @@ static int winOpen( /* If argument zPath is a NULL pointer, this function is required to open ** a temporary file. Use this buffer to store the file name in. */ - char zTmpname[MAX_PATH+1]; /* Buffer used to create temp filename */ + char zTmpname[MAX_PATH+2]; /* Buffer used to create temp filename */ int rc = SQLITE_OK; /* Function Return Code */ #if !defined(NDEBUG) || SQLITE_OS_WINCE @@ -2526,17 +3146,29 @@ static int winOpen( */ if( !zUtf8Name ){ assert(isDelete && !isOpenJournal); - rc = getTempname(MAX_PATH+1, zTmpname); + rc = getTempname(MAX_PATH+2, zTmpname); if( rc!=SQLITE_OK ){ return rc; } zUtf8Name = zTmpname; } + /* Database filenames are double-zero terminated if they are not + ** URIs with parameters. Hence, they can always be passed into + ** sqlite3_uri_parameter(). + */ + assert( (eType!=SQLITE_OPEN_MAIN_DB) || (flags & SQLITE_OPEN_URI) || + zUtf8Name[strlen(zUtf8Name)+1]==0 ); + /* Convert the filename to the system encoding. */ zConverted = convertUtf8Filename(zUtf8Name); if( zConverted==0 ){ - return SQLITE_NOMEM; + return SQLITE_IOERR_NOMEM; + } + + if( winIsDir(zConverted) ){ + sqlite3_free(zConverted); + return SQLITE_CANTOPEN_ISDIR; } if( isReadWrite ){ @@ -2582,26 +3214,26 @@ static int winOpen( #endif if( isNT() ){ - while( (h = CreateFileW((WCHAR*)zConverted, - dwDesiredAccess, - dwShareMode, NULL, - dwCreationDisposition, - dwFlagsAndAttributes, - NULL))==INVALID_HANDLE_VALUE && - retryIoerr(&cnt) ){} -/* isNT() is 1 if SQLITE_OS_WINCE==1, so this else is never executed. -** Since the ASCII version of these Windows API do not exist for WINCE, -** it's important to not reference them for WINCE builds. -*/ + while( (h = osCreateFileW((LPCWSTR)zConverted, + dwDesiredAccess, + dwShareMode, NULL, + dwCreationDisposition, + dwFlagsAndAttributes, + NULL))==INVALID_HANDLE_VALUE && + retryIoerr(&cnt, &lastErrno) ){ + /* Noop */ + } #if SQLITE_OS_WINCE==0 }else{ - while( (h = CreateFileA((char*)zConverted, - dwDesiredAccess, - dwShareMode, NULL, - dwCreationDisposition, - dwFlagsAndAttributes, - NULL))==INVALID_HANDLE_VALUE && - retryIoerr(&cnt) ){} + while( (h = osCreateFileA((LPCSTR)zConverted, + dwDesiredAccess, + dwShareMode, NULL, + dwCreationDisposition, + dwFlagsAndAttributes, + NULL))==INVALID_HANDLE_VALUE && + retryIoerr(&cnt, &lastErrno) ){ + /* Noop */ + } #endif } @@ -2612,9 +3244,9 @@ static int winOpen( h==INVALID_HANDLE_VALUE ? "failed" : "ok")); if( h==INVALID_HANDLE_VALUE ){ - pFile->lastErrno = GetLastError(); - winLogError(SQLITE_CANTOPEN, "winOpen", zUtf8Name); - free(zConverted); + pFile->lastErrno = lastErrno; + winLogError(SQLITE_CANTOPEN, pFile->lastErrno, "winOpen", zUtf8Name); + sqlite3_free(zConverted); if( isReadWrite && !isExclusive ){ return winOpen(pVfs, zName, id, ((flags|SQLITE_OPEN_READONLY)&~(SQLITE_OPEN_CREATE|SQLITE_OPEN_READWRITE)), pOutFlags); @@ -2638,14 +3270,16 @@ static int winOpen( pFile->pVfs = pVfs; pFile->pShm = 0; pFile->zPath = zName; - pFile->sectorSize = getSectorSize(pVfs, zUtf8Name); + if( sqlite3_uri_boolean(zName, "psow", SQLITE_POWERSAFE_OVERWRITE) ){ + pFile->ctrlFlags |= WINFILE_PSOW; + } #if SQLITE_OS_WINCE if( isReadWrite && eType==SQLITE_OPEN_MAIN_DB && !winceCreateLock(zName, pFile) ){ - CloseHandle(h); - free(zConverted); + osCloseHandle(h); + sqlite3_free(zConverted); return SQLITE_CANTOPEN_BKPT; } if( isTemp ){ @@ -2653,7 +3287,7 @@ static int winOpen( }else #endif { - free(zConverted); + sqlite3_free(zConverted); } OpenCounter(+1); @@ -2663,7 +3297,7 @@ static int winOpen( /* ** Delete the named file. ** -** Note that windows does not allow a file to be deleted if some other +** Note that Windows does not allow a file to be deleted if some other ** process has it open. Sometimes a virus scanner or indexing program ** will open a journal file shortly after it is created in order to do ** whatever it does. While this other process is holding the @@ -2679,6 +3313,8 @@ static int winDelete( ){ int cnt = 0; int rc; + DWORD attr; + DWORD lastErrno; void *zConverted; UNUSED_PARAMETER(pVfs); UNUSED_PARAMETER(syncDir); @@ -2686,31 +3322,62 @@ static int winDelete( SimulateIOError(return SQLITE_IOERR_DELETE); zConverted = convertUtf8Filename(zFilename); if( zConverted==0 ){ - return SQLITE_NOMEM; + return SQLITE_IOERR_NOMEM; } if( isNT() ){ - rc = 1; - while( GetFileAttributesW(zConverted)!=INVALID_FILE_ATTRIBUTES && - (rc = DeleteFileW(zConverted))==0 && retryIoerr(&cnt) ){} - rc = rc ? SQLITE_OK : SQLITE_ERROR; + do { + attr = osGetFileAttributesW(zConverted); + if ( attr==INVALID_FILE_ATTRIBUTES ){ + rc = SQLITE_OK; /* Already gone? */ + break; + } + if ( attr&FILE_ATTRIBUTE_DIRECTORY ){ + rc = SQLITE_ERROR; /* Files only. */ + break; + } + if ( osDeleteFileW(zConverted) ){ + rc = SQLITE_OK; /* Deleted OK. */ + break; + } + if ( !retryIoerr(&cnt, &lastErrno) ){ + rc = SQLITE_ERROR; /* No more retries. */ + break; + } + } while(1); /* isNT() is 1 if SQLITE_OS_WINCE==1, so this else is never executed. -** Since the ASCII version of these Windows API do not exist for WINCE, +** Since the ANSI version of these Windows API do not exist for WINCE, ** it's important to not reference them for WINCE builds. */ #if SQLITE_OS_WINCE==0 }else{ - rc = 1; - while( GetFileAttributesA(zConverted)!=INVALID_FILE_ATTRIBUTES && - (rc = DeleteFileA(zConverted))==0 && retryIoerr(&cnt) ){} - rc = rc ? SQLITE_OK : SQLITE_ERROR; + do { + attr = osGetFileAttributesA(zConverted); + if ( attr==INVALID_FILE_ATTRIBUTES ){ + rc = SQLITE_OK; /* Already gone? */ + break; + } + if ( attr&FILE_ATTRIBUTE_DIRECTORY ){ + rc = SQLITE_ERROR; /* Files only. */ + break; + } + if ( osDeleteFileA(zConverted) ){ + rc = SQLITE_OK; /* Deleted OK. */ + break; + } + if ( !retryIoerr(&cnt, &lastErrno) ){ + rc = SQLITE_ERROR; /* No more retries. */ + break; + } + } while(1); #endif } if( rc ){ - rc = winLogError(SQLITE_IOERR_DELETE, "winDelete", zFilename); + rc = winLogError(SQLITE_IOERR_DELETE, lastErrno, + "winDelete", zFilename); }else{ logIoerr(cnt); } - free(zConverted); + sqlite3_free(zConverted); OSTRACE(("DELETE \"%s\" %s\n", zFilename, (rc ? "failed" : "ok" ))); return rc; } @@ -2726,21 +3393,22 @@ static int winAccess( ){ DWORD attr; int rc = 0; + DWORD lastErrno; void *zConverted; UNUSED_PARAMETER(pVfs); SimulateIOError( return SQLITE_IOERR_ACCESS; ); zConverted = convertUtf8Filename(zFilename); if( zConverted==0 ){ - return SQLITE_NOMEM; + return SQLITE_IOERR_NOMEM; } if( isNT() ){ int cnt = 0; WIN32_FILE_ATTRIBUTE_DATA sAttrData; memset(&sAttrData, 0, sizeof(sAttrData)); - while( !(rc = GetFileAttributesExW((WCHAR*)zConverted, + while( !(rc = osGetFileAttributesExW((LPCWSTR)zConverted, GetFileExInfoStandard, - &sAttrData)) && retryIoerr(&cnt) ){} + &sAttrData)) && retryIoerr(&cnt, &lastErrno) ){} if( rc ){ /* For an SQLITE_ACCESS_EXISTS query, treat a zero-length file ** as if it does not exist. @@ -2754,24 +3422,24 @@ static int winAccess( } }else{ logIoerr(cnt); - if( GetLastError()!=ERROR_FILE_NOT_FOUND ){ - winLogError(SQLITE_IOERR_ACCESS, "winAccess", zFilename); - free(zConverted); + if( lastErrno!=ERROR_FILE_NOT_FOUND ){ + winLogError(SQLITE_IOERR_ACCESS, lastErrno, "winAccess", zFilename); + sqlite3_free(zConverted); return SQLITE_IOERR_ACCESS; }else{ attr = INVALID_FILE_ATTRIBUTES; } } /* isNT() is 1 if SQLITE_OS_WINCE==1, so this else is never executed. -** Since the ASCII version of these Windows API do not exist for WINCE, +** Since the ANSI version of these Windows API do not exist for WINCE, ** it's important to not reference them for WINCE builds. */ #if SQLITE_OS_WINCE==0 }else{ - attr = GetFileAttributesA((char*)zConverted); + attr = osGetFileAttributesA((char*)zConverted); #endif } - free(zConverted); + sqlite3_free(zConverted); switch( flags ){ case SQLITE_ACCESS_READ: case SQLITE_ACCESS_EXISTS: @@ -2836,117 +3504,50 @@ static int winFullPathname( SimulateIOError( return SQLITE_ERROR ); UNUSED_PARAMETER(nFull); zConverted = convertUtf8Filename(zRelative); + if( zConverted==0 ){ + return SQLITE_IOERR_NOMEM; + } if( isNT() ){ - WCHAR *zTemp; - nByte = GetFullPathNameW((WCHAR*)zConverted, 0, 0, 0) + 3; - zTemp = malloc( nByte*sizeof(zTemp[0]) ); + LPWSTR zTemp; + nByte = osGetFullPathNameW((LPCWSTR)zConverted, 0, 0, 0) + 3; + zTemp = sqlite3_malloc( nByte*sizeof(zTemp[0]) ); if( zTemp==0 ){ - free(zConverted); - return SQLITE_NOMEM; + sqlite3_free(zConverted); + return SQLITE_IOERR_NOMEM; } - GetFullPathNameW((WCHAR*)zConverted, nByte, zTemp, 0); - free(zConverted); + osGetFullPathNameW((LPCWSTR)zConverted, nByte, zTemp, 0); + sqlite3_free(zConverted); zOut = unicodeToUtf8(zTemp); - free(zTemp); + sqlite3_free(zTemp); /* isNT() is 1 if SQLITE_OS_WINCE==1, so this else is never executed. -** Since the ASCII version of these Windows API do not exist for WINCE, +** Since the ANSI version of these Windows API do not exist for WINCE, ** it's important to not reference them for WINCE builds. */ #if SQLITE_OS_WINCE==0 }else{ char *zTemp; - nByte = GetFullPathNameA((char*)zConverted, 0, 0, 0) + 3; - zTemp = malloc( nByte*sizeof(zTemp[0]) ); + nByte = osGetFullPathNameA((char*)zConverted, 0, 0, 0) + 3; + zTemp = sqlite3_malloc( nByte*sizeof(zTemp[0]) ); if( zTemp==0 ){ - free(zConverted); - return SQLITE_NOMEM; + sqlite3_free(zConverted); + return SQLITE_IOERR_NOMEM; } - GetFullPathNameA((char*)zConverted, nByte, zTemp, 0); - free(zConverted); + osGetFullPathNameA((char*)zConverted, nByte, zTemp, 0); + sqlite3_free(zConverted); zOut = sqlite3_win32_mbcs_to_utf8(zTemp); - free(zTemp); + sqlite3_free(zTemp); #endif } if( zOut ){ sqlite3_snprintf(pVfs->mxPathname, zFull, "%s", zOut); - free(zOut); + sqlite3_free(zOut); return SQLITE_OK; }else{ - return SQLITE_NOMEM; + return SQLITE_IOERR_NOMEM; } #endif } -/* -** Get the sector size of the device used to store -** file. -*/ -static int getSectorSize( - sqlite3_vfs *pVfs, - const char *zRelative /* UTF-8 file name */ -){ - DWORD bytesPerSector = SQLITE_DEFAULT_SECTOR_SIZE; - /* GetDiskFreeSpace is not supported under WINCE */ -#if SQLITE_OS_WINCE - UNUSED_PARAMETER(pVfs); - UNUSED_PARAMETER(zRelative); -#else - char zFullpath[MAX_PATH+1]; - int rc; - DWORD dwRet = 0; - DWORD dwDummy; - - /* - ** We need to get the full path name of the file - ** to get the drive letter to look up the sector - ** size. - */ - SimulateIOErrorBenign(1); - rc = winFullPathname(pVfs, zRelative, MAX_PATH, zFullpath); - SimulateIOErrorBenign(0); - if( rc == SQLITE_OK ) - { - void *zConverted = convertUtf8Filename(zFullpath); - if( zConverted ){ - if( isNT() ){ - /* trim path to just drive reference */ - WCHAR *p = zConverted; - for(;*p;p++){ - if( *p == '\\' ){ - *p = '\0'; - break; - } - } - dwRet = GetDiskFreeSpaceW((WCHAR*)zConverted, - &dwDummy, - &bytesPerSector, - &dwDummy, - &dwDummy); - }else{ - /* trim path to just drive reference */ - char *p = (char *)zConverted; - for(;*p;p++){ - if( *p == '\\' ){ - *p = '\0'; - break; - } - } - dwRet = GetDiskFreeSpaceA((char*)zConverted, - &dwDummy, - &bytesPerSector, - &dwDummy, - &dwDummy); - } - free(zConverted); - } - if( !dwRet ){ - bytesPerSector = SQLITE_DEFAULT_SECTOR_SIZE; - } - } -#endif - return (int) bytesPerSector; -} - #ifndef SQLITE_OMIT_LOAD_EXTENSION /* ** Interfaces for opening a shared library, finding entry points @@ -2964,37 +3565,30 @@ static void *winDlOpen(sqlite3_vfs *pVfs, const char *zFilename){ return 0; } if( isNT() ){ - h = LoadLibraryW((WCHAR*)zConverted); + h = osLoadLibraryW((LPCWSTR)zConverted); /* isNT() is 1 if SQLITE_OS_WINCE==1, so this else is never executed. -** Since the ASCII version of these Windows API do not exist for WINCE, +** Since the ANSI version of these Windows API do not exist for WINCE, ** it's important to not reference them for WINCE builds. */ #if SQLITE_OS_WINCE==0 }else{ - h = LoadLibraryA((char*)zConverted); + h = osLoadLibraryA((char*)zConverted); #endif } - free(zConverted); + sqlite3_free(zConverted); return (void*)h; } static void winDlError(sqlite3_vfs *pVfs, int nBuf, char *zBufOut){ UNUSED_PARAMETER(pVfs); - getLastErrorMsg(nBuf, zBufOut); + getLastErrorMsg(osGetLastError(), nBuf, zBufOut); } static void (*winDlSym(sqlite3_vfs *pVfs, void *pHandle, const char *zSymbol))(void){ UNUSED_PARAMETER(pVfs); -#if SQLITE_OS_WINCE - /* The GetProcAddressA() routine is only available on wince. */ - return (void(*)(void))GetProcAddressA((HANDLE)pHandle, zSymbol); -#else - /* All other windows platforms expect GetProcAddress() to take - ** an Ansi string regardless of the _UNICODE setting */ - return (void(*)(void))GetProcAddress((HANDLE)pHandle, zSymbol); -#endif + return (void(*)(void))osGetProcAddressA((HANDLE)pHandle, zSymbol); } static void winDlClose(sqlite3_vfs *pVfs, void *pHandle){ UNUSED_PARAMETER(pVfs); - FreeLibrary((HANDLE)pHandle); + osFreeLibrary((HANDLE)pHandle); } #else /* if SQLITE_OMIT_LOAD_EXTENSION is defined: */ #define winDlOpen 0 @@ -3016,23 +3610,23 @@ static int winRandomness(sqlite3_vfs *pVfs, int nBuf, char *zBuf){ #else if( sizeof(SYSTEMTIME)<=nBuf-n ){ SYSTEMTIME x; - GetSystemTime(&x); + osGetSystemTime(&x); memcpy(&zBuf[n], &x, sizeof(x)); n += sizeof(x); } if( sizeof(DWORD)<=nBuf-n ){ - DWORD pid = GetCurrentProcessId(); + DWORD pid = osGetCurrentProcessId(); memcpy(&zBuf[n], &pid, sizeof(pid)); n += sizeof(pid); } if( sizeof(DWORD)<=nBuf-n ){ - DWORD cnt = GetTickCount(); + DWORD cnt = osGetTickCount(); memcpy(&zBuf[n], &cnt, sizeof(cnt)); n += sizeof(cnt); } if( sizeof(LARGE_INTEGER)<=nBuf-n ){ LARGE_INTEGER i; - QueryPerformanceCounter(&i); + osQueryPerformanceCounter(&i); memcpy(&zBuf[n], &i, sizeof(i)); n += sizeof(i); } @@ -3045,7 +3639,7 @@ static int winRandomness(sqlite3_vfs *pVfs, int nBuf, char *zBuf){ ** Sleep for a little while. Return the amount of time slept. */ static int winSleep(sqlite3_vfs *pVfs, int microsec){ - Sleep((microsec+999)/1000); + osSleep((microsec+999)/1000); UNUSED_PARAMETER(pVfs); return ((microsec+999)/1000)*1000; } @@ -3084,13 +3678,13 @@ static int winCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *piNow){ #if SQLITE_OS_WINCE SYSTEMTIME time; - GetSystemTime(&time); + osGetSystemTime(&time); /* if SystemTimeToFileTime() fails, it returns zero. */ - if (!SystemTimeToFileTime(&time,&ft)){ + if (!osSystemTimeToFileTime(&time,&ft)){ return SQLITE_ERROR; } #else - GetSystemTimeAsFileTime( &ft ); + osGetSystemTimeAsFileTime( &ft ); #endif *piNow = winFiletimeEpoch + @@ -3123,8 +3717,8 @@ static int winCurrentTime(sqlite3_vfs *pVfs, double *prNow){ /* ** The idea is that this function works like a combination of -** GetLastError() and FormatMessage() on windows (or errno and -** strerror_r() on unix). After an error is returned by an OS +** GetLastError() and FormatMessage() on Windows (or errno and +** strerror_r() on Unix). After an error is returned by an OS ** function, SQLite calls this function with zBuf pointing to ** a buffer of nBuf bytes. The OS layer should populate the ** buffer with a nul-terminated UTF-8 encoded error message @@ -3153,11 +3747,9 @@ static int winCurrentTime(sqlite3_vfs *pVfs, double *prNow){ */ static int winGetLastError(sqlite3_vfs *pVfs, int nBuf, char *zBuf){ UNUSED_PARAMETER(pVfs); - return getLastErrorMsg(nBuf, zBuf); + return getLastErrorMsg(osGetLastError(), nBuf, zBuf); } - - /* ** Initialize and deinitialize the operating system interface. */ @@ -3182,21 +3774,26 @@ int sqlite3_os_init(void){ winCurrentTime, /* xCurrentTime */ winGetLastError, /* xGetLastError */ winCurrentTimeInt64, /* xCurrentTimeInt64 */ - 0, /* xSetSystemCall */ - 0, /* xGetSystemCall */ - 0, /* xNextSystemCall */ + winSetSystemCall, /* xSetSystemCall */ + winGetSystemCall, /* xGetSystemCall */ + winNextSystemCall, /* xNextSystemCall */ }; + /* Double-check that the aSyscall[] array has been constructed + ** correctly. See ticket [bb3a86e890c8e96ab] */ + assert( ArraySize(aSyscall)==60 ); + #ifndef SQLITE_OMIT_WAL /* get memory map allocation granularity */ memset(&winSysInfo, 0, sizeof(SYSTEM_INFO)); - GetSystemInfo(&winSysInfo); + osGetSystemInfo(&winSysInfo); assert(winSysInfo.dwAllocationGranularity > 0); #endif sqlite3_vfs_register(&winVfs, 1); return SQLITE_OK; } + int sqlite3_os_end(void){ return SQLITE_OK; } diff --git a/src/pager.c b/src/pager.c index ad6f831..345a275 100644 --- a/src/pager.c +++ b/src/pager.c @@ -612,10 +612,10 @@ struct Pager { u8 exclusiveMode; /* Boolean. True if locking_mode==EXCLUSIVE */ u8 journalMode; /* One of the PAGER_JOURNALMODE_* values */ u8 useJournal; /* Use a rollback journal on this file */ - u8 noReadlock; /* Do not bother to obtain readlocks */ u8 noSync; /* Do not sync the journal if true */ u8 fullSync; /* Do extra syncs of the journal for robustness */ u8 ckptSyncFlags; /* SYNC_NORMAL or SYNC_FULL for checkpoint */ + u8 walSyncFlags; /* SYNC_NORMAL or SYNC_FULL for wal writes */ u8 syncFlags; /* SYNC_NORMAL or SYNC_FULL otherwise */ u8 tempFile; /* zFilename is a temporary file */ u8 readOnly; /* True for a read-only database */ @@ -670,9 +670,9 @@ struct Pager { char *zJournal; /* Name of the journal file */ int (*xBusyHandler)(void*); /* Function to call when busy */ void *pBusyHandlerArg; /* Context argument for xBusyHandler */ - int nHit, nMiss; /* Total cache hits and misses */ + int aStat[3]; /* Total cache hits, misses and writes */ #ifdef SQLITE_TEST - int nRead, nWrite; /* Database pages read/written */ + int nRead; /* Database pages read */ #endif void (*xReiniter)(DbPage*); /* Call this routine when reloading pages */ #ifdef SQLITE_HAS_CODEC @@ -690,6 +690,15 @@ struct Pager { }; /* +** Indexes for use with Pager.aStat[]. The Pager.aStat[] array contains +** the values accessed by passing SQLITE_DBSTATUS_CACHE_HIT, CACHE_MISS +** or CACHE_WRITE to sqlite3_db_status(). +*/ +#define PAGER_STAT_HIT 0 +#define PAGER_STAT_MISS 1 +#define PAGER_STAT_WRITE 2 + +/* ** The following global variables hold counters used for ** testing purposes only. These variables do not exist in ** a non-testing build. These variables are not thread-safe. @@ -786,7 +795,7 @@ static int pagerUseWal(Pager *pPager){ #else # define pagerUseWal(x) 0 # define pagerRollbackWal(x) 0 -# define pagerWalFrames(v,w,x,y,z) 0 +# define pagerWalFrames(v,w,x,y) 0 # define pagerOpenWalIfPresent(z) SQLITE_OK # define pagerBeginReadTransaction(z) SQLITE_OK #endif @@ -859,7 +868,7 @@ static int assert_pager_state(Pager *p){ case PAGER_READER: assert( pPager->errCode==SQLITE_OK ); assert( p->eLock!=UNKNOWN_LOCK ); - assert( p->eLock>=SHARED_LOCK || p->noReadlock ); + assert( p->eLock>=SHARED_LOCK ); break; case PAGER_WRITER_LOCKED: @@ -2485,10 +2494,9 @@ static int pager_truncate(Pager *pPager, Pgno nPage){ if( rc==SQLITE_OK && currentSize!=newSize ){ if( currentSize>newSize ){ rc = sqlite3OsTruncate(pPager->fd, newSize); - }else{ + }else if( (currentSize+szPage)<=newSize ){ char *pTmp = pPager->pTmpSpace; memset(pTmp, 0, szPage); - testcase( (newSize-szPage) < currentSize ); testcase( (newSize-szPage) == currentSize ); testcase( (newSize-szPage) > currentSize ); rc = sqlite3OsWrite(pPager->fd, pTmp, szPage, newSize-szPage); @@ -2514,23 +2522,36 @@ static int pager_truncate(Pager *pPager, Pgno nPage){ ** the value returned by the xSectorSize() method rounded up to 32 if ** it is less than 32, or rounded down to MAX_SECTOR_SIZE if it ** is greater than MAX_SECTOR_SIZE. +** +** If the file has the SQLITE_IOCAP_POWERSAFE_OVERWRITE property, then set +** the effective sector size to its minimum value (512). The purpose of +** pPager->sectorSize is to define the "blast radius" of bytes that +** might change if a crash occurs while writing to a single byte in +** that range. But with POWERSAFE_OVERWRITE, the blast radius is zero +** (that is what POWERSAFE_OVERWRITE means), so we minimize the sector +** size. For backwards compatibility of the rollback journal file format, +** we cannot reduce the effective sector size below 512. */ static void setSectorSize(Pager *pPager){ assert( isOpen(pPager->fd) || pPager->tempFile ); - if( !pPager->tempFile ){ + if( pPager->tempFile + || (sqlite3OsDeviceCharacteristics(pPager->fd) & + SQLITE_IOCAP_POWERSAFE_OVERWRITE)!=0 + ){ /* Sector size doesn't matter for temporary files. Also, the file ** may not have been opened yet, in which case the OsSectorSize() - ** call will segfault. - */ - pPager->sectorSize = sqlite3OsSectorSize(pPager->fd); - } - if( pPager->sectorSize<32 ){ + ** call will segfault. */ pPager->sectorSize = 512; - } - if( pPager->sectorSize>MAX_SECTOR_SIZE ){ - assert( MAX_SECTOR_SIZE>=512 ); - pPager->sectorSize = MAX_SECTOR_SIZE; + }else{ + pPager->sectorSize = sqlite3OsSectorSize(pPager->fd); + if( pPager->sectorSize<32 ){ + pPager->sectorSize = 512; + } + if( pPager->sectorSize>MAX_SECTOR_SIZE ){ + assert( MAX_SECTOR_SIZE>=512 ); + pPager->sectorSize = MAX_SECTOR_SIZE; + } } } @@ -2733,10 +2754,11 @@ end_playback: ** SQLITE_FCNTL_DB_UNCHANGED file-control method to disable the ** assertion that the transaction counter was modified. */ - assert( - pPager->fd->pMethods==0 || - sqlite3OsFileControl(pPager->fd,SQLITE_FCNTL_DB_UNCHANGED,0)>=SQLITE_OK - ); +#ifdef SQLITE_DEBUG + if( pPager->fd->pMethods ){ + sqlite3OsFileControlHint(pPager->fd,SQLITE_FCNTL_DB_UNCHANGED,0); + } +#endif /* If this playback is happening automatically as a result of an IO or ** malloc error that occurred after the change-counter was updated but @@ -2955,10 +2977,10 @@ static int pagerWalFrames( Pager *pPager, /* Pager object */ PgHdr *pList, /* List of frames to log */ Pgno nTruncate, /* Database size after this commit */ - int isCommit, /* True if this is a commit */ - int syncFlags /* Flags to pass to OsSync() (or 0) */ + int isCommit /* True if this is a commit */ ){ int rc; /* Return code */ + int nList; /* Number of pages in pList */ #if defined(SQLITE_DEBUG) || defined(SQLITE_CHECK_PAGES) PgHdr *p; /* For looping over pages */ #endif @@ -2972,6 +2994,7 @@ static int pagerWalFrames( } #endif + assert( pList->pDirty==0 || isCommit ); if( isCommit ){ /* If a WAL transaction is being committed, there is no point in writing ** any pages with page numbers greater than nTruncate into the WAL file. @@ -2979,15 +3002,22 @@ static int pagerWalFrames( ** list here. */ PgHdr *p; PgHdr **ppNext = &pList; - for(p=pList; (*ppNext = p); p=p->pDirty){ - if( p->pgno<=nTruncate ) ppNext = &p->pDirty; + nList = 0; + for(p=pList; (*ppNext = p)!=0; p=p->pDirty){ + if( p->pgno<=nTruncate ){ + ppNext = &p->pDirty; + nList++; + } } assert( pList ); + }else{ + nList = 1; } + pPager->aStat[PAGER_STAT_WRITE] += nList; if( pList->pgno==1 ) pager_write_changecounter(pList); rc = sqlite3WalFrames(pPager->pWal, - pPager->pageSize, pList, nTruncate, isCommit, syncFlags + pPager->pageSize, pList, nTruncate, isCommit, pPager->walSyncFlags ); if( rc==SQLITE_OK && pPager->pBackup ){ PgHdr *p; @@ -3056,7 +3086,7 @@ static int pagerPagecount(Pager *pPager, Pgno *pnPage){ ** contains no valid committed transactions. */ assert( pPager->eState==PAGER_OPEN ); - assert( pPager->eLock>=SHARED_LOCK || pPager->noReadlock ); + assert( pPager->eLock>=SHARED_LOCK ); nPage = sqlite3WalDbsize(pPager->pWal); /* If the database size was not available from the WAL sub-system, @@ -3074,10 +3104,7 @@ static int pagerPagecount(Pager *pPager, Pgno *pnPage){ return rc; } } - nPage = (Pgno)(n / pPager->pageSize); - if( nPage==0 && n>0 ){ - nPage = 1; - } + nPage = (Pgno)((n+pPager->pageSize-1) / pPager->pageSize); } /* If the current number of pages in the file is greater than the @@ -3114,7 +3141,7 @@ static int pagerPagecount(Pager *pPager, Pgno *pnPage){ static int pagerOpenWalIfPresent(Pager *pPager){ int rc = SQLITE_OK; assert( pPager->eState==PAGER_OPEN ); - assert( pPager->eLock>=SHARED_LOCK || pPager->noReadlock ); + assert( pPager->eLock>=SHARED_LOCK ); if( !pPager->tempFile ){ int isWal; /* True if WAL file exists */ @@ -3267,13 +3294,13 @@ static int pagerPlaybackSavepoint(Pager *pPager, PagerSavepoint *pSavepoint){ */ if( pSavepoint ){ u32 ii; /* Loop counter */ - i64 offset = pSavepoint->iSubRec*(4+pPager->pageSize); + i64 offset = (i64)pSavepoint->iSubRec*(4+pPager->pageSize); if( pagerUseWal(pPager) ){ rc = sqlite3WalSavepointUndo(pPager->pWal, pSavepoint->aWalData); } for(ii=pSavepoint->iSubRec; rc==SQLITE_OK && ii<pPager->nSubRec; ii++){ - assert( offset==ii*(4+pPager->pageSize) ); + assert( offset==(i64)ii*(4+pPager->pageSize) ); rc = pager_playback_one_page(pPager, &offset, pDone, 0, 1); } assert( rc!=SQLITE_DONE ); @@ -3295,6 +3322,13 @@ void sqlite3PagerSetCachesize(Pager *pPager, int mxPage){ } /* +** Free as much memory as possible from the pager. +*/ +void sqlite3PagerShrink(Pager *pPager){ + sqlite3PcacheShrink(pPager->pPCache); +} + +/* ** Adjust the robustness of the database to damage due to OS crashes ** or power failures by changing the number of syncs()s when writing ** the rollback journal. There are three levels: @@ -3360,6 +3394,10 @@ void sqlite3PagerSetSafetyLevel( pPager->syncFlags = SQLITE_SYNC_NORMAL; pPager->ckptSyncFlags = SQLITE_SYNC_NORMAL; } + pPager->walSyncFlags = pPager->syncFlags; + if( pPager->fullSync ){ + pPager->walSyncFlags |= WAL_SYNC_TRANSACTIONS; + } } #endif @@ -3497,7 +3535,7 @@ int sqlite3PagerSetPagesize(Pager *pPager, u32 *pPageSize, int nReserve){ if( rc==SQLITE_OK ){ pager_reset(pPager); - pPager->dbSize = (Pgno)(nByte/pageSize); + pPager->dbSize = (Pgno)((nByte+pageSize-1)/pageSize); pPager->pageSize = pageSize; sqlite3PageFree(pPager->pTmpSpace); pPager->pTmpSpace = pNew; @@ -4005,7 +4043,7 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ assert( rc!=SQLITE_OK || isOpen(pPager->fd) ); if( rc==SQLITE_OK && pPager->dbSize>pPager->dbHintSize ){ sqlite3_int64 szFile = pPager->pageSize * (sqlite3_int64)pPager->dbSize; - sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_SIZE_HINT, &szFile); + sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_SIZE_HINT, &szFile); pPager->dbHintSize = pPager->dbSize; } @@ -4043,6 +4081,7 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ if( pgno>pPager->dbFileSize ){ pPager->dbFileSize = pgno; } + pPager->aStat[PAGER_STAT_WRITE]++; /* Update any backup objects copying the contents of this pager. */ sqlite3BackupUpdate(pPager->pBackup, pgno, (u8*)pList->pData); @@ -4051,7 +4090,6 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ PAGERID(pPager), pgno, pager_pagehash(pList))); IOTRACE(("PGOUT %p %d\n", pPager, pgno)); PAGER_INCR(sqlite3_pager_writedb_count); - PAGER_INCR(pPager->nWrite); }else{ PAGERTRACE(("NOSTORE %d page %d\n", PAGERID(pPager), pgno)); } @@ -4114,7 +4152,7 @@ static int subjournalPage(PgHdr *pPg){ ** write the journal record into the file. */ if( rc==SQLITE_OK ){ void *pData = pPg->pData; - i64 offset = pPager->nSubRec*(4+pPager->pageSize); + i64 offset = (i64)pPager->nSubRec*(4+pPager->pageSize); char *pData2; CODEC2(pPager, pData, pPg->pgno, 7, return SQLITE_NOMEM, pData2); @@ -4187,7 +4225,7 @@ static int pagerStress(void *p, PgHdr *pPg){ rc = subjournalPage(pPg); } if( rc==SQLITE_OK ){ - rc = pagerWalFrames(pPager, pPg, 0, 0, 0); + rc = pagerWalFrames(pPager, pPg, 0, 0); } }else{ @@ -4266,7 +4304,7 @@ static int pagerStress(void *p, PgHdr *pPg){ ** ** The flags argument is used to specify properties that affect the ** operation of the pager. It should be passed some bitwise combination -** of the PAGER_OMIT_JOURNAL and PAGER_NO_READLOCK flags. +** of the PAGER_* flags. ** ** The vfsFlags parameter is a bitmask to pass to the flags parameter ** of the xOpen() method of the supplied VFS when opening files. @@ -4297,7 +4335,6 @@ int sqlite3PagerOpen( char *zPathname = 0; /* Full path to database file */ int nPathname = 0; /* Number of bytes in zPathname */ int useJournal = (flags & PAGER_OMIT_JOURNAL)==0; /* False to omit journal */ - int noReadlock = (flags & PAGER_NO_READLOCK)!=0; /* True to omit read-lock */ int pcacheSize = sqlite3PcacheSize(); /* Bytes to allocate for PCache */ u32 szPageDflt = SQLITE_DEFAULT_PAGE_SIZE; /* Default page size */ const char *zUri = 0; /* URI args to copy */ @@ -4346,7 +4383,8 @@ int sqlite3PagerOpen( z += sqlite3Strlen30(z)+1; z += sqlite3Strlen30(z)+1; } - nUri = &z[1] - zUri; + nUri = (int)(&z[1] - zUri); + assert( nUri>=0 ); if( rc==SQLITE_OK && nPathname+8>pVfs->mxPathname ){ /* This branch is taken when the journal path required by ** the database being opened will be more than pVfs->mxPathname @@ -4380,9 +4418,9 @@ int sqlite3PagerOpen( ROUND8(pVfs->szOsFile) + /* The main db file */ journalFileSize * 2 + /* The two journal files */ nPathname + 1 + nUri + /* zFilename */ - nPathname + 8 + 1 /* zJournal */ + nPathname + 8 + 2 /* zJournal */ #ifndef SQLITE_OMIT_WAL - + nPathname + 4 + 1 /* zWal */ + + nPathname + 4 + 2 /* zWal */ #endif ); assert( EIGHT_BYTE_ALIGNMENT(SQLITE_INT_TO_PTR(journalFileSize)) ); @@ -4405,12 +4443,12 @@ int sqlite3PagerOpen( memcpy(pPager->zFilename, zPathname, nPathname); memcpy(&pPager->zFilename[nPathname+1], zUri, nUri); memcpy(pPager->zJournal, zPathname, nPathname); - memcpy(&pPager->zJournal[nPathname], "-journal", 8); + memcpy(&pPager->zJournal[nPathname], "-journal\000", 8+1); sqlite3FileSuffix3(pPager->zFilename, pPager->zJournal); #ifndef SQLITE_OMIT_WAL pPager->zWal = &pPager->zJournal[nPathname+8+1]; memcpy(pPager->zWal, zPathname, nPathname); - memcpy(&pPager->zWal[nPathname], "-wal", 4); + memcpy(&pPager->zWal[nPathname], "-wal\000", 4+1); sqlite3FileSuffix3(pPager->zFilename, pPager->zWal); #endif sqlite3_free(zPathname); @@ -4503,7 +4541,6 @@ int sqlite3PagerOpen( IOTRACE(("OPEN %p %s\n", pPager, pPager->zFilename)) pPager->useJournal = (u8)useJournal; - pPager->noReadlock = (noReadlock && readOnly) ?1:0; /* pPager->stmtOpen = 0; */ /* pPager->stmtInUse = 0; */ /* pPager->nRef = 0; */ @@ -4526,9 +4563,17 @@ int sqlite3PagerOpen( pPager->readOnly = (u8)readOnly; assert( useJournal || pPager->tempFile ); pPager->noSync = pPager->tempFile; - pPager->fullSync = pPager->noSync ?0:1; - pPager->syncFlags = pPager->noSync ? 0 : SQLITE_SYNC_NORMAL; - pPager->ckptSyncFlags = pPager->syncFlags; + if( pPager->noSync ){ + assert( pPager->fullSync==0 ); + assert( pPager->syncFlags==0 ); + assert( pPager->walSyncFlags==0 ); + assert( pPager->ckptSyncFlags==0 ); + }else{ + pPager->fullSync = 1; + pPager->syncFlags = SQLITE_SYNC_NORMAL; + pPager->walSyncFlags = SQLITE_SYNC_NORMAL | WAL_SYNC_TRANSACTIONS; + pPager->ckptSyncFlags = SQLITE_SYNC_NORMAL; + } /* pPager->pFirst = 0; */ /* pPager->pFirstSynced = 0; */ /* pPager->pLast = 0; */ @@ -4717,14 +4762,11 @@ int sqlite3PagerSharedLock(Pager *pPager){ int bHotJournal = 1; /* True if there exists a hot journal-file */ assert( !MEMDB ); - assert( pPager->noReadlock==0 || pPager->readOnly ); - if( pPager->noReadlock==0 ){ - rc = pager_wait_on_lock(pPager, SHARED_LOCK); - if( rc!=SQLITE_OK ){ - assert( pPager->eLock==NO_LOCK || pPager->eLock==UNKNOWN_LOCK ); - goto failed; - } + rc = pager_wait_on_lock(pPager, SHARED_LOCK); + if( rc!=SQLITE_OK ){ + assert( pPager->eLock==NO_LOCK || pPager->eLock==UNKNOWN_LOCK ); + goto failed; } /* If a journal file exists, and there is no RESERVED lock on the @@ -5005,7 +5047,7 @@ int sqlite3PagerAcquire( /* In this case the pcache already contains an initialized copy of ** the page. Return without further ado. */ assert( pgno<=PAGER_MAX_PGNO && pgno!=PAGER_MJ_PGNO(pPager) ); - pPager->nHit++; + pPager->aStat[PAGER_STAT_HIT]++; return SQLITE_OK; }else{ @@ -5047,7 +5089,7 @@ int sqlite3PagerAcquire( IOTRACE(("ZERO %p %d\n", pPager, pgno)); }else{ assert( pPg->pPager==pPager ); - pPager->nMiss++; + pPager->aStat[PAGER_STAT_MISS]++; rc = readDbPage(pPg); if( rc!=SQLITE_OK ){ goto pager_acquire_err; @@ -5632,6 +5674,7 @@ static int pager_incr_changecounter(Pager *pPager, int isDirectMode){ CODEC2(pPager, pPgHdr->pData, 1, 6, rc=SQLITE_NOMEM, zBuf); if( rc==SQLITE_OK ){ rc = sqlite3OsWrite(pPager->fd, zBuf, pPager->pageSize, 0); + pPager->aStat[PAGER_STAT_WRITE]++; } if( rc==SQLITE_OK ){ pPager->changeCountDone = 1; @@ -5661,7 +5704,10 @@ int sqlite3PagerSync(Pager *pPager){ rc = sqlite3OsSync(pPager->fd, pPager->syncFlags); }else if( isOpen(pPager->fd) ){ assert( !MEMDB ); - sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_SYNC_OMITTED, (void *)&rc); + rc = sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_SYNC_OMITTED, 0); + if( rc==SQLITE_NOTFOUND ){ + rc = SQLITE_OK; + } } return rc; } @@ -5758,9 +5804,7 @@ int sqlite3PagerCommitPhaseOne( } assert( rc==SQLITE_OK ); if( ALWAYS(pList) ){ - rc = pagerWalFrames(pPager, pList, pPager->dbSize, 1, - (pPager->fullSync ? pPager->syncFlags : 0) - ); + rc = pagerWalFrames(pPager, pList, pPager->dbSize, 1); } sqlite3PagerUnref(pPageOne); if( rc==SQLITE_OK ){ @@ -6019,7 +6063,8 @@ int sqlite3PagerRollback(Pager *pPager){ } assert( pPager->eState==PAGER_READER || rc!=SQLITE_OK ); - assert( rc==SQLITE_OK || rc==SQLITE_FULL || (rc&0xFF)==SQLITE_IOERR ); + assert( rc==SQLITE_OK || rc==SQLITE_FULL + || rc==SQLITE_NOMEM || (rc&0xFF)==SQLITE_IOERR ); /* If an error occurs during a ROLLBACK, we can no longer trust the pager ** cache. So call pager_error() on the way out to make any error persistent. @@ -6073,11 +6118,11 @@ int *sqlite3PagerStats(Pager *pPager){ a[3] = pPager->eState==PAGER_OPEN ? -1 : (int) pPager->dbSize; a[4] = pPager->eState; a[5] = pPager->errCode; - a[6] = pPager->nHit; - a[7] = pPager->nMiss; + a[6] = pPager->aStat[PAGER_STAT_HIT]; + a[7] = pPager->aStat[PAGER_STAT_MISS]; a[8] = 0; /* Used to be pPager->nOvfl */ a[9] = pPager->nRead; - a[10] = pPager->nWrite; + a[10] = pPager->aStat[PAGER_STAT_WRITE]; return a; } #endif @@ -6090,20 +6135,19 @@ int *sqlite3PagerStats(Pager *pPager){ ** returning. */ void sqlite3PagerCacheStat(Pager *pPager, int eStat, int reset, int *pnVal){ - int *piStat; assert( eStat==SQLITE_DBSTATUS_CACHE_HIT || eStat==SQLITE_DBSTATUS_CACHE_MISS + || eStat==SQLITE_DBSTATUS_CACHE_WRITE ); - if( eStat==SQLITE_DBSTATUS_CACHE_HIT ){ - piStat = &pPager->nHit; - }else{ - piStat = &pPager->nMiss; - } - *pnVal += *piStat; + assert( SQLITE_DBSTATUS_CACHE_HIT+1==SQLITE_DBSTATUS_CACHE_MISS ); + assert( SQLITE_DBSTATUS_CACHE_HIT+2==SQLITE_DBSTATUS_CACHE_WRITE ); + assert( PAGER_STAT_HIT==0 && PAGER_STAT_MISS==1 && PAGER_STAT_WRITE==2 ); + + *pnVal += pPager->aStat[eStat - SQLITE_DBSTATUS_CACHE_HIT]; if( reset ){ - *piStat = 0; + pPager->aStat[eStat - SQLITE_DBSTATUS_CACHE_HIT] = 0; } } @@ -6660,6 +6704,15 @@ sqlite3_backup **sqlite3PagerBackupPtr(Pager *pPager){ return &pPager->pBackup; } +#ifndef SQLITE_OMIT_VACUUM +/* +** Unless this is an in-memory or temporary database, clear the pager cache. +*/ +void sqlite3PagerClearCache(Pager *pPager){ + if( !MEMDB && pPager->tempFile==0 ) pager_reset(pPager); +} +#endif + #ifndef SQLITE_OMIT_WAL /* ** This function is called when the user invokes "PRAGMA wal_checkpoint", @@ -6721,7 +6774,7 @@ static int pagerOpenWal(Pager *pPager){ int rc = SQLITE_OK; assert( pPager->pWal==0 && pPager->tempFile==0 ); - assert( pPager->eLock==SHARED_LOCK || pPager->eLock==EXCLUSIVE_LOCK || pPager->noReadlock); + assert( pPager->eLock==SHARED_LOCK || pPager->eLock==EXCLUSIVE_LOCK ); /* If the pager is already in exclusive-mode, the WAL module will use ** heap-memory for the wal-index instead of the VFS shared-memory @@ -6836,12 +6889,19 @@ int sqlite3PagerCloseWal(Pager *pPager){ return rc; } +#ifdef SQLITE_ENABLE_ZIPVFS /* -** Unless this is an in-memory or temporary database, clear the pager cache. +** A read-lock must be held on the pager when this function is called. If +** the pager is in WAL mode and the WAL file currently contains one or more +** frames, return the size in bytes of the page images stored within the +** WAL frames. Otherwise, if this is not a WAL database or the WAL file +** is empty, return 0. */ -void sqlite3PagerClearCache(Pager *pPager){ - if( !MEMDB && pPager->tempFile==0 ) pager_reset(pPager); +int sqlite3PagerWalFramesize(Pager *pPager){ + assert( pPager->eState==PAGER_READER ); + return sqlite3WalFramesize(pPager->pWal); } +#endif #ifdef SQLITE_HAS_CODEC /* @@ -6886,6 +6946,9 @@ void sqlite3pager_sqlite3PagerSetCodec( sqlite3PagerSetCodec(pPager, xCodec, xCodecSizeChng, xCodecFree, pCodec); } +void sqlite3pager_sqlite3PagerSetError( Pager *pPager, int error) { + pPager->errCode = error; +} #endif /* END CRYPTO */ diff --git a/src/pager.h b/src/pager.h index e36e6c2..eca8a2f 100644 --- a/src/pager.h +++ b/src/pager.h @@ -58,8 +58,7 @@ typedef struct PgHdr DbPage; ** NOTE: These values must match the corresponding BTREE_ values in btree.h. */ #define PAGER_OMIT_JOURNAL 0x0001 /* Do not use a rollback journal */ -#define PAGER_NO_READLOCK 0x0002 /* Omit readlocks on readonly files */ -#define PAGER_MEMORY 0x0004 /* In-memory database */ +#define PAGER_MEMORY 0x0002 /* In-memory database */ /* ** Valid values for the second argument to sqlite3PagerLockingMode(). @@ -103,6 +102,7 @@ void sqlite3PagerSetBusyhandler(Pager*, int(*)(void *), void *); int sqlite3PagerSetPagesize(Pager*, u32*, int); int sqlite3PagerMaxPageCount(Pager*, int); void sqlite3PagerSetCachesize(Pager*, int); +void sqlite3PagerShrink(Pager*); void sqlite3PagerSetSafetyLevel(Pager*,int,int,int); int sqlite3PagerLockingMode(Pager *, int); int sqlite3PagerSetJournalMode(Pager *, int); @@ -143,6 +143,9 @@ int sqlite3PagerWalSupported(Pager *pPager); int sqlite3PagerWalCallback(Pager *pPager); int sqlite3PagerOpenWal(Pager *pPager, int *pisOpen); int sqlite3PagerCloseWal(Pager *pPager); +#ifdef SQLITE_ENABLE_ZIPVFS + int sqlite3PagerWalFramesize(Pager *pPager); +#endif /* Functions used to query pager state and configuration. */ u8 sqlite3PagerIsreadonly(Pager*); diff --git a/src/parse.y b/src/parse.y index 92abd5c..94433d5 100644 --- a/src/parse.y +++ b/src/parse.y @@ -33,12 +33,10 @@ UNUSED_PARAMETER(yymajor); /* Silence some compiler warnings */ assert( TOKEN.z[0] ); /* The tokenizer always gives us a token */ sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &TOKEN); - pParse->parseError = 1; } %stack_overflow { UNUSED_PARAMETER(yypMinor); /* Silence some compiler warnings */ sqlite3ErrorMsg(pParse, "parser stack overflow"); - pParse->parseError = 1; } // The name of the generated procedure that implements the parser @@ -77,7 +75,7 @@ struct LimitVal { */ struct LikeOp { Token eOperator; /* "like" or "glob" or "regexp" */ - int not; /* True if the NOT keyword is present */ + int bNot; /* True if the NOT keyword is present */ }; /* @@ -96,6 +94,14 @@ struct TrigEvent { int a; IdList * b; }; */ struct AttachKey { int type; Token key; }; +/* +** One or more VALUES claues +*/ +struct ValueList { + ExprList *pList; + Select *pSelect; +}; + } // end %include // Input is a single SQL command @@ -179,6 +185,7 @@ column(A) ::= columnid(X) type carglist. { columnid(A) ::= nm(X). { sqlite3AddColumn(pParse,&X); A = X; + pParse->constraintName.n = 0; } @@ -267,10 +274,9 @@ signed ::= minus_num. // "carglist" is a list of additional constraints that come after the // column name and column type in a CREATE TABLE statement. // -carglist ::= carglist carg. +carglist ::= carglist ccons. carglist ::= . -carg ::= CONSTRAINT nm ccons. -carg ::= ccons. +ccons ::= CONSTRAINT nm(X). {pParse->constraintName = X;} ccons ::= DEFAULT term(X). {sqlite3AddDefaultValue(pParse,&X);} ccons ::= DEFAULT LP expr(X) RP. {sqlite3AddDefaultValue(pParse,&X);} ccons ::= DEFAULT PLUS term(X). {sqlite3AddDefaultValue(pParse,&X);} @@ -333,15 +339,13 @@ init_deferred_pred_opt(A) ::= . {A = 0;} init_deferred_pred_opt(A) ::= INITIALLY DEFERRED. {A = 1;} init_deferred_pred_opt(A) ::= INITIALLY IMMEDIATE. {A = 0;} -// For the time being, the only constraint we care about is the primary -// key and UNIQUE. Both create indices. -// -conslist_opt(A) ::= . {A.n = 0; A.z = 0;} -conslist_opt(A) ::= COMMA(X) conslist. {A = X;} -conslist ::= conslist COMMA tcons. -conslist ::= conslist tcons. +conslist_opt(A) ::= . {A.n = 0; A.z = 0;} +conslist_opt(A) ::= COMMA(X) conslist. {A = X;} +conslist ::= conslist tconscomma tcons. conslist ::= tcons. -tcons ::= CONSTRAINT nm. +tconscomma ::= COMMA. {pParse->constraintName.n = 0;} +tconscomma ::= . +tcons ::= CONSTRAINT nm(X). {pParse->constraintName = X;} tcons ::= PRIMARY KEY LP idxlist(X) autoinc(I) RP onconf(R). {sqlite3AddPrimaryKey(pParse,X,R,I,0);} tcons ::= UNIQUE LP idxlist(X) RP onconf(R). @@ -396,6 +400,9 @@ cmd ::= DROP VIEW ifexists(E) fullname(X). { cmd ::= select(X). { SelectDest dest = {SRT_Output, 0, 0, 0, 0}; sqlite3Select(pParse, X, &dest); + sqlite3ExplainBegin(pParse->pVdbe); + sqlite3ExplainSelect(pParse->pVdbe, X); + sqlite3ExplainFinish(pParse->pVdbe); sqlite3SelectDelete(pParse->db, X); } @@ -572,20 +579,17 @@ using_opt(U) ::= . {U = 0;} %destructor orderby_opt {sqlite3ExprListDelete(pParse->db, $$);} %type sortlist {ExprList*} %destructor sortlist {sqlite3ExprListDelete(pParse->db, $$);} -%type sortitem {Expr*} -%destructor sortitem {sqlite3ExprDelete(pParse->db, $$);} orderby_opt(A) ::= . {A = 0;} orderby_opt(A) ::= ORDER BY sortlist(X). {A = X;} -sortlist(A) ::= sortlist(X) COMMA sortitem(Y) sortorder(Z). { - A = sqlite3ExprListAppend(pParse,X,Y); +sortlist(A) ::= sortlist(X) COMMA expr(Y) sortorder(Z). { + A = sqlite3ExprListAppend(pParse,X,Y.pExpr); if( A ) A->a[A->nExpr-1].sortOrder = (u8)Z; } -sortlist(A) ::= sortitem(Y) sortorder(Z). { - A = sqlite3ExprListAppend(pParse,0,Y); +sortlist(A) ::= expr(Y) sortorder(Z). { + A = sqlite3ExprListAppend(pParse,0,Y.pExpr); if( A && ALWAYS(A->a) ) A->a[0].sortOrder = (u8)Z; } -sortitem(A) ::= expr(X). {A = X.pExpr;} %type sortorder {int} @@ -678,9 +682,8 @@ setlist(A) ::= nm(X) EQ expr(Y). { ////////////////////////// The INSERT command ///////////////////////////////// // -cmd ::= insert_cmd(R) INTO fullname(X) inscollist_opt(F) - VALUES LP itemlist(Y) RP. - {sqlite3Insert(pParse, X, Y, 0, F, R);} +cmd ::= insert_cmd(R) INTO fullname(X) inscollist_opt(F) valuelist(Y). + {sqlite3Insert(pParse, X, Y.pList, Y.pSelect, F, R);} cmd ::= insert_cmd(R) INTO fullname(X) inscollist_opt(F) select(S). {sqlite3Insert(pParse, X, 0, S, F, R);} cmd ::= insert_cmd(R) INTO fullname(X) inscollist_opt(F) DEFAULT VALUES. @@ -690,14 +693,46 @@ cmd ::= insert_cmd(R) INTO fullname(X) inscollist_opt(F) DEFAULT VALUES. insert_cmd(A) ::= INSERT orconf(R). {A = R;} insert_cmd(A) ::= REPLACE. {A = OE_Replace;} - -%type itemlist {ExprList*} -%destructor itemlist {sqlite3ExprListDelete(pParse->db, $$);} - -itemlist(A) ::= itemlist(X) COMMA expr(Y). - {A = sqlite3ExprListAppend(pParse,X,Y.pExpr);} -itemlist(A) ::= expr(X). - {A = sqlite3ExprListAppend(pParse,0,X.pExpr);} +// A ValueList is either a single VALUES clause or a comma-separated list +// of VALUES clauses. If it is a single VALUES clause then the +// ValueList.pList field points to the expression list of that clause. +// If it is a list of VALUES clauses, then those clauses are transformed +// into a set of SELECT statements without FROM clauses and connected by +// UNION ALL and the ValueList.pSelect points to the right-most SELECT in +// that compound. +%type valuelist {struct ValueList} +%destructor valuelist { + sqlite3ExprListDelete(pParse->db, $$.pList); + sqlite3SelectDelete(pParse->db, $$.pSelect); +} +valuelist(A) ::= VALUES LP nexprlist(X) RP. { + A.pList = X; + A.pSelect = 0; +} + +// Since a list of VALUEs is inplemented as a compound SELECT, we have +// to disable the value list option if compound SELECTs are disabled. +%ifndef SQLITE_OMIT_COMPOUND_SELECT +valuelist(A) ::= valuelist(X) COMMA LP exprlist(Y) RP. { + Select *pRight = sqlite3SelectNew(pParse, Y, 0, 0, 0, 0, 0, 0, 0, 0); + if( X.pList ){ + X.pSelect = sqlite3SelectNew(pParse, X.pList, 0, 0, 0, 0, 0, 0, 0, 0); + X.pList = 0; + } + A.pList = 0; + if( X.pSelect==0 || pRight==0 ){ + sqlite3SelectDelete(pParse->db, pRight); + sqlite3SelectDelete(pParse->db, X.pSelect); + A.pSelect = 0; + }else{ + pRight->op = TK_ALL; + pRight->pPrior = X.pSelect; + pRight->selFlags |= SF_Values; + pRight->pPrior->selFlags |= SF_Values; + A.pSelect = pRight; + } +} +%endif SQLITE_OMIT_COMPOUND_SELECT %type inscollist_opt {IdList*} %destructor inscollist_opt {sqlite3IdListDelete(pParse->db, $$);} @@ -844,16 +879,16 @@ expr(A) ::= expr(X) STAR|SLASH|REM(OP) expr(Y). {spanBinaryExpr(&A,pParse,@OP,&X,&Y);} expr(A) ::= expr(X) CONCAT(OP) expr(Y). {spanBinaryExpr(&A,pParse,@OP,&X,&Y);} %type likeop {struct LikeOp} -likeop(A) ::= LIKE_KW(X). {A.eOperator = X; A.not = 0;} -likeop(A) ::= NOT LIKE_KW(X). {A.eOperator = X; A.not = 1;} -likeop(A) ::= MATCH(X). {A.eOperator = X; A.not = 0;} -likeop(A) ::= NOT MATCH(X). {A.eOperator = X; A.not = 1;} +likeop(A) ::= LIKE_KW(X). {A.eOperator = X; A.bNot = 0;} +likeop(A) ::= NOT LIKE_KW(X). {A.eOperator = X; A.bNot = 1;} +likeop(A) ::= MATCH(X). {A.eOperator = X; A.bNot = 0;} +likeop(A) ::= NOT MATCH(X). {A.eOperator = X; A.bNot = 1;} expr(A) ::= expr(X) likeop(OP) expr(Y). [LIKE_KW] { ExprList *pList; pList = sqlite3ExprListAppend(pParse,0, Y.pExpr); pList = sqlite3ExprListAppend(pParse,pList, X.pExpr); A.pExpr = sqlite3ExprFunction(pParse, pList, &OP.eOperator); - if( OP.not ) A.pExpr = sqlite3PExpr(pParse, TK_NOT, A.pExpr, 0, 0); + if( OP.bNot ) A.pExpr = sqlite3PExpr(pParse, TK_NOT, A.pExpr, 0, 0); A.zStart = X.zStart; A.zEnd = Y.zEnd; if( A.pExpr ) A.pExpr->flags |= EP_InfixFunc; @@ -864,7 +899,7 @@ expr(A) ::= expr(X) likeop(OP) expr(Y) ESCAPE expr(E). [LIKE_KW] { pList = sqlite3ExprListAppend(pParse,pList, X.pExpr); pList = sqlite3ExprListAppend(pParse,pList, E.pExpr); A.pExpr = sqlite3ExprFunction(pParse, pList, &OP.eOperator); - if( OP.not ) A.pExpr = sqlite3PExpr(pParse, TK_NOT, A.pExpr, 0, 0); + if( OP.bNot ) A.pExpr = sqlite3PExpr(pParse, TK_NOT, A.pExpr, 0, 0); A.zStart = X.zStart; A.zEnd = E.zEnd; if( A.pExpr ) A.pExpr->flags |= EP_InfixFunc; @@ -1162,11 +1197,10 @@ nmnum(A) ::= ON(X). {A = X;} nmnum(A) ::= DELETE(X). {A = X;} nmnum(A) ::= DEFAULT(X). {A = X;} %endif SQLITE_OMIT_PRAGMA -plus_num(A) ::= plus_opt number(X). {A = X;} +plus_num(A) ::= PLUS number(X). {A = X;} +plus_num(A) ::= number(X). {A = X;} minus_num(A) ::= MINUS number(X). {A = X;} number(A) ::= INTEGER|FLOAT(X). {A = X;} -plus_opt ::= PLUS. -plus_opt ::= . //////////////////////////// The CREATE TRIGGER command ///////////////////// @@ -1260,8 +1294,8 @@ trigger_cmd(A) ::= // INSERT trigger_cmd(A) ::= - insert_cmd(R) INTO trnm(X) inscollist_opt(F) VALUES LP itemlist(Y) RP. - {A = sqlite3TriggerInsertStep(pParse->db, &X, F, Y, 0, R);} + insert_cmd(R) INTO trnm(X) inscollist_opt(F) valuelist(Y). + {A = sqlite3TriggerInsertStep(pParse->db, &X, F, Y.pList, Y.pSelect, R);} trigger_cmd(A) ::= insert_cmd(R) INTO trnm(X) inscollist_opt(F) select(S). {A = sqlite3TriggerInsertStep(pParse->db, &X, F, 0, S, R);} @@ -1355,8 +1389,9 @@ kwcolumn_opt ::= COLUMNKW. %ifndef SQLITE_OMIT_VIRTUALTABLE cmd ::= create_vtab. {sqlite3VtabFinishParse(pParse,0);} cmd ::= create_vtab LP vtabarglist RP(X). {sqlite3VtabFinishParse(pParse,&X);} -create_vtab ::= createkw VIRTUAL TABLE nm(X) dbnm(Y) USING nm(Z). { - sqlite3VtabBeginParse(pParse, &X, &Y, &Z); +create_vtab ::= createkw VIRTUAL TABLE ifnotexists(E) + nm(X) dbnm(Y) USING nm(Z). { + sqlite3VtabBeginParse(pParse, &X, &Y, &Z, E); } vtabarglist ::= vtabarg. vtabarglist ::= vtabarglist COMMA vtabarg. diff --git a/src/pcache.c b/src/pcache.c index f37511e..482a188 100644 --- a/src/pcache.c +++ b/src/pcache.c @@ -20,7 +20,7 @@ struct PCache { PgHdr *pDirty, *pDirtyTail; /* List of dirty pages in LRU order */ PgHdr *pSynced; /* Last synced page in dirty page list */ int nRef; /* Number of referenced pages */ - int nMax; /* Configured cache size */ + int szCache; /* Configured cache size */ int szPage; /* Size of every page in this cache */ int szExtra; /* Size of extra space for each page */ int bPurgeable; /* True if pages are on backing store */ @@ -131,7 +131,7 @@ static void pcacheUnpin(PgHdr *p){ if( p->pgno==1 ){ pCache->pPage1 = 0; } - sqlite3GlobalConfig.pcache.xUnpin(pCache->pCache, p, 0); + sqlite3GlobalConfig.pcache2.xUnpin(pCache->pCache, p->pPage, 0); } } @@ -141,18 +141,18 @@ static void pcacheUnpin(PgHdr *p){ ** functions are threadsafe. */ int sqlite3PcacheInitialize(void){ - if( sqlite3GlobalConfig.pcache.xInit==0 ){ + if( sqlite3GlobalConfig.pcache2.xInit==0 ){ /* IMPLEMENTATION-OF: R-26801-64137 If the xInit() method is NULL, then the ** built-in default page cache is used instead of the application defined ** page cache. */ sqlite3PCacheSetDefault(); } - return sqlite3GlobalConfig.pcache.xInit(sqlite3GlobalConfig.pcache.pArg); + return sqlite3GlobalConfig.pcache2.xInit(sqlite3GlobalConfig.pcache2.pArg); } void sqlite3PcacheShutdown(void){ - if( sqlite3GlobalConfig.pcache.xShutdown ){ + if( sqlite3GlobalConfig.pcache2.xShutdown ){ /* IMPLEMENTATION-OF: R-26000-56589 The xShutdown() method may be NULL. */ - sqlite3GlobalConfig.pcache.xShutdown(sqlite3GlobalConfig.pcache.pArg); + sqlite3GlobalConfig.pcache2.xShutdown(sqlite3GlobalConfig.pcache2.pArg); } } @@ -181,7 +181,7 @@ void sqlite3PcacheOpen( p->bPurgeable = bPurgeable; p->xStress = xStress; p->pStress = pStress; - p->nMax = 100; + p->szCache = 100; } /* @@ -191,7 +191,7 @@ void sqlite3PcacheOpen( void sqlite3PcacheSetPageSize(PCache *pCache, int szPage){ assert( pCache->nRef==0 && pCache->pDirty==0 ); if( pCache->pCache ){ - sqlite3GlobalConfig.pcache.xDestroy(pCache->pCache); + sqlite3GlobalConfig.pcache2.xDestroy(pCache->pCache); pCache->pCache = 0; pCache->pPage1 = 0; } @@ -199,6 +199,17 @@ void sqlite3PcacheSetPageSize(PCache *pCache, int szPage){ } /* +** Compute the number of pages of cache requested. +*/ +static int numberOfCachePages(PCache *p){ + if( p->szCache>=0 ){ + return p->szCache; + }else{ + return (int)((-1024*(i64)p->szCache)/(p->szPage+p->szExtra)); + } +} + +/* ** Try to obtain a page from the cache. */ int sqlite3PcacheFetch( @@ -207,7 +218,8 @@ int sqlite3PcacheFetch( int createFlag, /* If true, create page if it does not exist already */ PgHdr **ppPage /* Write the page here */ ){ - PgHdr *pPage = 0; + sqlite3_pcache_page *pPage = 0; + PgHdr *pPgHdr = 0; int eCreate; assert( pCache!=0 ); @@ -219,19 +231,19 @@ int sqlite3PcacheFetch( */ if( !pCache->pCache && createFlag ){ sqlite3_pcache *p; - int nByte; - nByte = pCache->szPage + pCache->szExtra + sizeof(PgHdr); - p = sqlite3GlobalConfig.pcache.xCreate(nByte, pCache->bPurgeable); + p = sqlite3GlobalConfig.pcache2.xCreate( + pCache->szPage, pCache->szExtra + sizeof(PgHdr), pCache->bPurgeable + ); if( !p ){ return SQLITE_NOMEM; } - sqlite3GlobalConfig.pcache.xCachesize(p, pCache->nMax); + sqlite3GlobalConfig.pcache2.xCachesize(p, numberOfCachePages(pCache)); pCache->pCache = p; } eCreate = createFlag * (1 + (!pCache->bPurgeable || !pCache->pDirty)); if( pCache->pCache ){ - pPage = sqlite3GlobalConfig.pcache.xFetch(pCache->pCache, pgno, eCreate); + pPage = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, pgno, eCreate); } if( !pPage && eCreate==1 ){ @@ -258,7 +270,7 @@ int sqlite3PcacheFetch( "spill page %d making room for %d - cache used: %d/%d", pPg->pgno, pgno, sqlite3GlobalConfig.pcache.xPagecount(pCache->pCache), - pCache->nMax); + numberOfCachePages(pCache)); #endif rc = pCache->xStress(pCache->pStress, pPg); if( rc!=SQLITE_OK && rc!=SQLITE_BUSY ){ @@ -266,33 +278,36 @@ int sqlite3PcacheFetch( } } - pPage = sqlite3GlobalConfig.pcache.xFetch(pCache->pCache, pgno, 2); + pPage = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, pgno, 2); } if( pPage ){ - if( !pPage->pData ){ - memset(pPage, 0, sizeof(PgHdr)); - pPage->pData = (void *)&pPage[1]; - pPage->pExtra = (void*)&((char *)pPage->pData)[pCache->szPage]; - memset(pPage->pExtra, 0, pCache->szExtra); - pPage->pCache = pCache; - pPage->pgno = pgno; + pPgHdr = (PgHdr *)pPage->pExtra; + + if( !pPgHdr->pPage ){ + memset(pPgHdr, 0, sizeof(PgHdr)); + pPgHdr->pPage = pPage; + pPgHdr->pData = pPage->pBuf; + pPgHdr->pExtra = (void *)&pPgHdr[1]; + memset(pPgHdr->pExtra, 0, pCache->szExtra); + pPgHdr->pCache = pCache; + pPgHdr->pgno = pgno; } - assert( pPage->pCache==pCache ); - assert( pPage->pgno==pgno ); - assert( pPage->pData==(void *)&pPage[1] ); - assert( pPage->pExtra==(void *)&((char *)&pPage[1])[pCache->szPage] ); + assert( pPgHdr->pCache==pCache ); + assert( pPgHdr->pgno==pgno ); + assert( pPgHdr->pData==pPage->pBuf ); + assert( pPgHdr->pExtra==(void *)&pPgHdr[1] ); - if( 0==pPage->nRef ){ + if( 0==pPgHdr->nRef ){ pCache->nRef++; } - pPage->nRef++; + pPgHdr->nRef++; if( pgno==1 ){ - pCache->pPage1 = pPage; + pCache->pPage1 = pPgHdr; } } - *ppPage = pPage; - return (pPage==0 && eCreate) ? SQLITE_NOMEM : SQLITE_OK; + *ppPage = pPgHdr; + return (pPgHdr==0 && eCreate) ? SQLITE_NOMEM : SQLITE_OK; } /* @@ -339,7 +354,7 @@ void sqlite3PcacheDrop(PgHdr *p){ if( p->pgno==1 ){ pCache->pPage1 = 0; } - sqlite3GlobalConfig.pcache.xUnpin(pCache->pCache, p, 1); + sqlite3GlobalConfig.pcache2.xUnpin(pCache->pCache, p->pPage, 1); } /* @@ -397,7 +412,7 @@ void sqlite3PcacheMove(PgHdr *p, Pgno newPgno){ PCache *pCache = p->pCache; assert( p->nRef>0 ); assert( newPgno>0 ); - sqlite3GlobalConfig.pcache.xRekey(pCache->pCache, p, p->pgno, newPgno); + sqlite3GlobalConfig.pcache2.xRekey(pCache->pCache, p->pPage, p->pgno,newPgno); p->pgno = newPgno; if( (p->flags&PGHDR_DIRTY) && (p->flags&PGHDR_NEED_SYNC) ){ pcacheRemoveFromDirtyList(p); @@ -434,7 +449,7 @@ void sqlite3PcacheTruncate(PCache *pCache, Pgno pgno){ memset(pCache->pPage1->pData, 0, pCache->szPage); pgno = 1; } - sqlite3GlobalConfig.pcache.xTruncate(pCache->pCache, pgno+1); + sqlite3GlobalConfig.pcache2.xTruncate(pCache->pCache, pgno+1); } } @@ -443,7 +458,7 @@ void sqlite3PcacheTruncate(PCache *pCache, Pgno pgno){ */ void sqlite3PcacheClose(PCache *pCache){ if( pCache->pCache ){ - sqlite3GlobalConfig.pcache.xDestroy(pCache->pCache); + sqlite3GlobalConfig.pcache2.xDestroy(pCache->pCache); } } @@ -555,7 +570,7 @@ int sqlite3PcachePageRefcount(PgHdr *p){ int sqlite3PcachePagecount(PCache *pCache){ int nPage = 0; if( pCache->pCache ){ - nPage = sqlite3GlobalConfig.pcache.xPagecount(pCache->pCache); + nPage = sqlite3GlobalConfig.pcache2.xPagecount(pCache->pCache); } return nPage; } @@ -565,7 +580,7 @@ int sqlite3PcachePagecount(PCache *pCache){ ** Get the suggested cache-size value. */ int sqlite3PcacheGetCachesize(PCache *pCache){ - return pCache->nMax; + return numberOfCachePages(pCache); } #endif @@ -573,9 +588,19 @@ int sqlite3PcacheGetCachesize(PCache *pCache){ ** Set the suggested cache-size value. */ void sqlite3PcacheSetCachesize(PCache *pCache, int mxPage){ - pCache->nMax = mxPage; + pCache->szCache = mxPage; + if( pCache->pCache ){ + sqlite3GlobalConfig.pcache2.xCachesize(pCache->pCache, + numberOfCachePages(pCache)); + } +} + +/* +** Free up as much memory as possible from the page cache. +*/ +void sqlite3PcacheShrink(PCache *pCache){ if( pCache->pCache ){ - sqlite3GlobalConfig.pcache.xCachesize(pCache->pCache, mxPage); + sqlite3GlobalConfig.pcache2.xShrink(pCache->pCache); } } diff --git a/src/pcache.h b/src/pcache.h index 33735d2..b9135fd 100644 --- a/src/pcache.h +++ b/src/pcache.h @@ -23,11 +23,12 @@ typedef struct PCache PCache; ** structure. */ struct PgHdr { - void *pData; /* Content of this page */ + sqlite3_pcache_page *pPage; /* Pcache object page handle */ + void *pData; /* Page data */ void *pExtra; /* Extra content */ PgHdr *pDirty; /* Transient list of dirty pages */ - Pgno pgno; /* Page number for this page */ Pager *pPager; /* The pager this page is part of */ + Pgno pgno; /* Page number for this page */ #ifdef SQLITE_CHECK_PAGES u32 pageHash; /* Hash of page content */ #endif @@ -141,6 +142,9 @@ void sqlite3PcacheSetCachesize(PCache *, int); int sqlite3PcacheGetCachesize(PCache *); #endif +/* Free up as much memory as possible from the page cache */ +void sqlite3PcacheShrink(PCache*); + #ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT /* Try to return memory used by the pcache module to the main memory heap */ int sqlite3PcacheReleaseMemory(int); diff --git a/src/pcache1.c b/src/pcache1.c index 077a7b2..42fc8ce 100644 --- a/src/pcache1.c +++ b/src/pcache1.c @@ -24,7 +24,6 @@ typedef struct PgHdr1 PgHdr1; typedef struct PgFreeslot PgFreeslot; typedef struct PGroup PGroup; - /* Each page cache (or PCache) belongs to a PGroup. A PGroup is a set ** of one or more PCaches that are able to recycle each others unpinned ** pages when they are under memory pressure. A PGroup is an instance of @@ -41,7 +40,7 @@ typedef struct PGroup PGroup; ** Mode 1 uses more memory (since PCache instances are not able to rob ** unused pages from other PCaches) but it also operates without a mutex, ** and is therefore often faster. Mode 2 requires a mutex in order to be -** threadsafe, but is able recycle pages more efficient. +** threadsafe, but recycles pages more efficiently. ** ** For mode (1), PGroup.mutex is NULL. For mode (2) there is only a single ** PGroup which is the pcache1.grp global variable and its mutex is @@ -49,10 +48,10 @@ typedef struct PGroup PGroup; */ struct PGroup { sqlite3_mutex *mutex; /* MUTEX_STATIC_LRU or NULL */ - int nMaxPage; /* Sum of nMax for purgeable caches */ - int nMinPage; /* Sum of nMin for purgeable caches */ - int mxPinned; /* nMaxpage + 10 - nMinPage */ - int nCurrentPage; /* Number of purgeable pages allocated */ + unsigned int nMaxPage; /* Sum of nMax for purgeable caches */ + unsigned int nMinPage; /* Sum of nMin for purgeable caches */ + unsigned int mxPinned; /* nMaxpage + 10 - nMinPage */ + unsigned int nCurrentPage; /* Number of purgeable pages allocated */ PgHdr1 *pLruHead, *pLruTail; /* LRU list of unpinned pages */ }; @@ -67,15 +66,17 @@ struct PGroup { struct PCache1 { /* Cache configuration parameters. Page size (szPage) and the purgeable ** flag (bPurgeable) are set when the cache is created. nMax may be - ** modified at any time by a call to the pcache1CacheSize() method. + ** modified at any time by a call to the pcache1Cachesize() method. ** The PGroup mutex must be held when accessing nMax. */ PGroup *pGroup; /* PGroup this cache belongs to */ int szPage; /* Size of allocated pages in bytes */ + int szExtra; /* Size of extra space in bytes */ int bPurgeable; /* True if cache is purgeable */ unsigned int nMin; /* Minimum number of pages reserved */ unsigned int nMax; /* Configured "cache_size" value */ unsigned int n90pct; /* nMax*9/10 */ + unsigned int iMaxKey; /* Largest key seen since xTruncate() */ /* Hash table of all pages. The following variables may only be accessed ** when the accessor is holding the PGroup mutex. @@ -84,17 +85,16 @@ struct PCache1 { unsigned int nPage; /* Total number of pages in apHash */ unsigned int nHash; /* Number of slots in apHash[] */ PgHdr1 **apHash; /* Hash table for fast lookup by key */ - - unsigned int iMaxKey; /* Largest key seen since xTruncate() */ }; /* ** Each cache entry is represented by an instance of the following -** structure. A buffer of PgHdr1.pCache->szPage bytes is allocated -** directly before this structure in memory (see the PGHDR1_TO_PAGE() -** macro below). +** structure. Unless SQLITE_PCACHE_SEPARATE_HEADER is defined, a buffer of +** PgHdr1.pCache->szPage bytes is allocated directly before this structure +** in memory. */ struct PgHdr1 { + sqlite3_pcache_page page; unsigned int iKey; /* Key value (page number) */ PgHdr1 *pNext; /* Next in hash table chain */ PCache1 *pCache; /* Cache that currently owns this page */ @@ -128,8 +128,8 @@ static SQLITE_WSD struct PCacheGlobal { void *pStart, *pEnd; /* Bounds of pagecache malloc range */ /* Above requires no mutex. Use mutex below for variable that follow. */ sqlite3_mutex *mutex; /* Mutex for accessing the following: */ - int nFreeSlot; /* Number of unused pcache slots */ PgFreeslot *pFree; /* Free page blocks */ + int nFreeSlot; /* Number of unused pcache slots */ /* The following value requires a mutex to change. We skip the mutex on ** reading because (1) most platforms read a 32-bit integer atomically and ** (2) even if an incorrect value is read, no great harm is done since this @@ -145,21 +145,6 @@ static SQLITE_WSD struct PCacheGlobal { #define pcache1 (GLOBAL(struct PCacheGlobal, pcache1_g)) /* -** When a PgHdr1 structure is allocated, the associated PCache1.szPage -** bytes of data are located directly before it in memory (i.e. the total -** size of the allocation is sizeof(PgHdr1)+PCache1.szPage byte). The -** PGHDR1_TO_PAGE() macro takes a pointer to a PgHdr1 structure as -** an argument and returns a pointer to the associated block of szPage -** bytes. The PAGE_TO_PGHDR1() macro does the opposite: its argument is -** a pointer to a block of szPage bytes of data and the return value is -** a pointer to the associated PgHdr1 structure. -** -** assert( PGHDR1_TO_PAGE(PAGE_TO_PGHDR1(pCache, X))==X ); -*/ -#define PGHDR1_TO_PAGE(p) (void*)(((char*)p) - p->pCache->szPage) -#define PAGE_TO_PGHDR1(c, p) (PgHdr1*)(((char*)p) + c->szPage) - -/* ** Macros to enter and leave the PCache LRU mutex. */ #define pcache1EnterMutex(X) sqlite3_mutex_enter((X)->mutex) @@ -241,8 +226,9 @@ static void *pcache1Alloc(int nByte){ /* ** Free an allocated buffer obtained from pcache1Alloc(). */ -static void pcache1Free(void *p){ - if( p==0 ) return; +static int pcache1Free(void *p){ + int nFreed = 0; + if( p==0 ) return 0; if( p>=pcache1.pStart && p<pcache1.pEnd ){ PgFreeslot *pSlot; sqlite3_mutex_enter(pcache1.mutex); @@ -255,15 +241,15 @@ static void pcache1Free(void *p){ assert( pcache1.nFreeSlot<=pcache1.nSlot ); sqlite3_mutex_leave(pcache1.mutex); }else{ - int iSize; assert( sqlite3MemdebugHasType(p, MEMTYPE_PCACHE) ); sqlite3MemdebugSetType(p, MEMTYPE_HEAP); - iSize = sqlite3MallocSize(p); + nFreed = sqlite3MallocSize(p); sqlite3_mutex_enter(pcache1.mutex); - sqlite3StatusAdd(SQLITE_STATUS_PAGECACHE_OVERFLOW, -iSize); + sqlite3StatusAdd(SQLITE_STATUS_PAGECACHE_OVERFLOW, -nFreed); sqlite3_mutex_leave(pcache1.mutex); sqlite3_free(p); } + return nFreed; } #ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT @@ -288,7 +274,6 @@ static int pcache1MemSize(void *p){ ** Allocate a new page object initially associated with cache pCache. */ static PgHdr1 *pcache1AllocPage(PCache1 *pCache){ - int nByte = sizeof(PgHdr1) + pCache->szPage; PgHdr1 *p = 0; void *pPg; @@ -297,16 +282,29 @@ static PgHdr1 *pcache1AllocPage(PCache1 *pCache){ ** this mutex is not held. */ assert( sqlite3_mutex_held(pCache->pGroup->mutex) ); pcache1LeaveMutex(pCache->pGroup); - pPg = pcache1Alloc(nByte); +#ifdef SQLITE_PCACHE_SEPARATE_HEADER + pPg = pcache1Alloc(pCache->szPage); + p = sqlite3Malloc(sizeof(PgHdr1) + pCache->szExtra); + if( !pPg || !p ){ + pcache1Free(pPg); + sqlite3_free(p); + pPg = 0; + } +#else + pPg = pcache1Alloc(sizeof(PgHdr1) + pCache->szPage + pCache->szExtra); + p = (PgHdr1 *)&((u8 *)pPg)[pCache->szPage]; +#endif pcache1EnterMutex(pCache->pGroup); if( pPg ){ - p = PAGE_TO_PGHDR1(pCache, pPg); + p->page.pBuf = pPg; + p->page.pExtra = &p[1]; if( pCache->bPurgeable ){ pCache->pGroup->nCurrentPage++; } + return p; } - return p; + return 0; } /* @@ -320,7 +318,10 @@ static void pcache1FreePage(PgHdr1 *p){ if( ALWAYS(p) ){ PCache1 *pCache = p->pCache; assert( sqlite3_mutex_held(p->pCache->pGroup->mutex) ); - pcache1Free(PGHDR1_TO_PAGE(p)); + pcache1Free(p->page.pBuf); +#ifdef SQLITE_PCACHE_SEPARATE_HEADER + sqlite3_free(p); +#endif if( pCache->bPurgeable ){ pCache->pGroup->nCurrentPage--; } @@ -355,13 +356,13 @@ void sqlite3PageFree(void *p){ ** for all page cache needs and we should not need to spill the ** allocation onto the heap. ** -** Or, the heap is used for all page cache memory put the heap is +** Or, the heap is used for all page cache memory but the heap is ** under memory pressure, then again it is desirable to avoid ** allocating a new page cache entry in order to avoid stressing ** the heap even further. */ static int pcache1UnderMemoryPressure(PCache1 *pCache){ - if( pcache1.nSlot && pCache->szPage<=pcache1.szSlot ){ + if( pcache1.nSlot && (pCache->szPage+pCache->szExtra)<=pcache1.szSlot ){ return pcache1.bUnderPressure; }else{ return sqlite3HeapNearlyFull(); @@ -552,7 +553,7 @@ static void pcache1Shutdown(void *NotUsed){ ** ** Allocate a new cache. */ -static sqlite3_pcache *pcache1Create(int szPage, int bPurgeable){ +static sqlite3_pcache *pcache1Create(int szPage, int szExtra, int bPurgeable){ PCache1 *pCache; /* The newly created page cache */ PGroup *pGroup; /* The group the new page cache will belong to */ int sz; /* Bytes of memory required to allocate the new cache */ @@ -575,6 +576,9 @@ static sqlite3_pcache *pcache1Create(int szPage, int bPurgeable){ int separateCache = sqlite3GlobalConfig.bCoreMutex>0; #endif + assert( (szPage & (szPage-1))==0 && szPage>=512 && szPage<=65536 ); + assert( szExtra < 300 ); + sz = sizeof(PCache1) + sizeof(PGroup)*separateCache; pCache = (PCache1 *)sqlite3_malloc(sz); if( pCache ){ @@ -587,6 +591,7 @@ static sqlite3_pcache *pcache1Create(int szPage, int bPurgeable){ } pCache->pGroup = pGroup; pCache->szPage = szPage; + pCache->szExtra = szExtra; pCache->bPurgeable = (bPurgeable ? 1 : 0); if( bPurgeable ){ pCache->nMin = 10; @@ -619,6 +624,25 @@ static void pcache1Cachesize(sqlite3_pcache *p, int nMax){ } /* +** Implementation of the sqlite3_pcache.xShrink method. +** +** Free up as much memory as possible. +*/ +static void pcache1Shrink(sqlite3_pcache *p){ + PCache1 *pCache = (PCache1*)p; + if( pCache->bPurgeable ){ + PGroup *pGroup = pCache->pGroup; + int savedMaxPage; + pcache1EnterMutex(pGroup); + savedMaxPage = pGroup->nMaxPage; + pGroup->nMaxPage = 0; + pcache1EnforceMaxPage(pGroup); + pGroup->nMaxPage = savedMaxPage; + pcache1LeaveMutex(pGroup); + } +} + +/* ** Implementation of the sqlite3_pcache.xPagecount method. */ static int pcache1Pagecount(sqlite3_pcache *p){ @@ -643,7 +667,7 @@ static int pcache1Pagecount(sqlite3_pcache *p){ ** For a non-purgeable cache (a cache used as the storage for an in-memory ** database) there is really no difference between createFlag 1 and 2. So ** the calling function (pcache.c) will never have a createFlag of 1 on -** a non-purgable cache. +** a non-purgeable cache. ** ** There are three different approaches to obtaining space for a page, ** depending on the value of parameter createFlag (which may be 0, 1 or 2). @@ -684,8 +708,12 @@ static int pcache1Pagecount(sqlite3_pcache *p){ ** ** 5. Otherwise, allocate and return a new page buffer. */ -static void *pcache1Fetch(sqlite3_pcache *p, unsigned int iKey, int createFlag){ - int nPinned; +static sqlite3_pcache_page *pcache1Fetch( + sqlite3_pcache *p, + unsigned int iKey, + int createFlag +){ + unsigned int nPinned; PCache1 *pCache = (PCache1 *)p; PGroup *pGroup; PgHdr1 *pPage = 0; @@ -719,15 +747,14 @@ static void *pcache1Fetch(sqlite3_pcache *p, unsigned int iKey, int createFlag){ pGroup = pCache->pGroup; #endif - /* Step 3: Abort if createFlag is 1 but the cache is nearly full */ + assert( pCache->nPage >= pCache->nRecyclable ); nPinned = pCache->nPage - pCache->nRecyclable; - assert( nPinned>=0 ); assert( pGroup->mxPinned == pGroup->nMaxPage + 10 - pGroup->nMinPage ); assert( pCache->n90pct == pCache->nMax*9/10 ); if( createFlag==1 && ( nPinned>=pGroup->mxPinned - || nPinned>=(int)pCache->n90pct + || nPinned>=pCache->n90pct || pcache1UnderMemoryPressure(pCache) )){ goto fetch_out; @@ -743,16 +770,24 @@ static void *pcache1Fetch(sqlite3_pcache *p, unsigned int iKey, int createFlag){ || pGroup->nCurrentPage>=pGroup->nMaxPage || pcache1UnderMemoryPressure(pCache) )){ - PCache1 *pOtherCache; + PCache1 *pOther; pPage = pGroup->pLruTail; pcache1RemoveFromHash(pPage); pcache1PinPage(pPage); - if( (pOtherCache = pPage->pCache)->szPage!=pCache->szPage ){ + pOther = pPage->pCache; + + /* We want to verify that szPage and szExtra are the same for pOther + ** and pCache. Assert that we can verify this by comparing sums. */ + assert( (pCache->szPage & (pCache->szPage-1))==0 && pCache->szPage>=512 ); + assert( pCache->szExtra<512 ); + assert( (pOther->szPage & (pOther->szPage-1))==0 && pOther->szPage>=512 ); + assert( pOther->szExtra<512 ); + + if( pOther->szPage+pOther->szExtra != pCache->szPage+pCache->szExtra ){ pcache1FreePage(pPage); pPage = 0; }else{ - pGroup->nCurrentPage -= - (pOtherCache->bPurgeable - pCache->bPurgeable); + pGroup->nCurrentPage -= (pOther->bPurgeable - pCache->bPurgeable); } } @@ -773,7 +808,7 @@ static void *pcache1Fetch(sqlite3_pcache *p, unsigned int iKey, int createFlag){ pPage->pCache = pCache; pPage->pLruPrev = 0; pPage->pLruNext = 0; - *(void **)(PGHDR1_TO_PAGE(pPage)) = 0; + *(void **)pPage->page.pExtra = 0; pCache->apHash[h] = pPage; } @@ -782,7 +817,7 @@ fetch_out: pCache->iMaxKey = iKey; } pcache1LeaveMutex(pGroup); - return (pPage ? PGHDR1_TO_PAGE(pPage) : 0); + return &pPage->page; } @@ -791,9 +826,13 @@ fetch_out: ** ** Mark a page as unpinned (eligible for asynchronous recycling). */ -static void pcache1Unpin(sqlite3_pcache *p, void *pPg, int reuseUnlikely){ +static void pcache1Unpin( + sqlite3_pcache *p, + sqlite3_pcache_page *pPg, + int reuseUnlikely +){ PCache1 *pCache = (PCache1 *)p; - PgHdr1 *pPage = PAGE_TO_PGHDR1(pCache, pPg); + PgHdr1 *pPage = (PgHdr1 *)pPg; PGroup *pGroup = pCache->pGroup; assert( pPage->pCache==pCache ); @@ -829,12 +868,12 @@ static void pcache1Unpin(sqlite3_pcache *p, void *pPg, int reuseUnlikely){ */ static void pcache1Rekey( sqlite3_pcache *p, - void *pPg, + sqlite3_pcache_page *pPg, unsigned int iOld, unsigned int iNew ){ PCache1 *pCache = (PCache1 *)p; - PgHdr1 *pPage = PAGE_TO_PGHDR1(pCache, pPg); + PgHdr1 *pPage = (PgHdr1 *)pPg; PgHdr1 **pp; unsigned int h; assert( pPage->iKey==iOld ); @@ -888,7 +927,9 @@ static void pcache1Destroy(sqlite3_pcache *p){ assert( pCache->bPurgeable || (pCache->nMax==0 && pCache->nMin==0) ); pcache1EnterMutex(pGroup); pcache1TruncateUnsafe(pCache, 0); + assert( pGroup->nMaxPage >= pCache->nMax ); pGroup->nMaxPage -= pCache->nMax; + assert( pGroup->nMinPage >= pCache->nMin ); pGroup->nMinPage -= pCache->nMin; pGroup->mxPinned = pGroup->nMaxPage + 10 - pGroup->nMinPage; pcache1EnforceMaxPage(pGroup); @@ -903,7 +944,8 @@ static void pcache1Destroy(sqlite3_pcache *p){ ** already provided an alternative. */ void sqlite3PCacheSetDefault(void){ - static const sqlite3_pcache_methods defaultMethods = { + static const sqlite3_pcache_methods2 defaultMethods = { + 1, /* iVersion */ 0, /* pArg */ pcache1Init, /* xInit */ pcache1Shutdown, /* xShutdown */ @@ -914,9 +956,10 @@ void sqlite3PCacheSetDefault(void){ pcache1Unpin, /* xUnpin */ pcache1Rekey, /* xRekey */ pcache1Truncate, /* xTruncate */ - pcache1Destroy /* xDestroy */ + pcache1Destroy, /* xDestroy */ + pcache1Shrink /* xShrink */ }; - sqlite3_config(SQLITE_CONFIG_PCACHE, &defaultMethods); + sqlite3_config(SQLITE_CONFIG_PCACHE2, &defaultMethods); } #ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT @@ -937,7 +980,10 @@ int sqlite3PcacheReleaseMemory(int nReq){ PgHdr1 *p; pcache1EnterMutex(&pcache1.grp); while( (nReq<0 || nFree<nReq) && ((p=pcache1.grp.pLruTail)!=0) ){ - nFree += pcache1MemSize(PGHDR1_TO_PAGE(p)); + nFree += pcache1MemSize(p->page.pBuf); +#ifdef SQLITE_PCACHE_SEPARATE_HEADER + nFree += sqlite3MemSize(p); +#endif pcache1PinPage(p); pcache1RemoveFromHash(p); pcache1FreePage(p); @@ -965,8 +1011,8 @@ void sqlite3PcacheStats( nRecyclable++; } *pnCurrent = pcache1.grp.nCurrentPage; - *pnMax = pcache1.grp.nMaxPage; - *pnMin = pcache1.grp.nMinPage; + *pnMax = (int)pcache1.grp.nMaxPage; + *pnMin = (int)pcache1.grp.nMinPage; *pnRecyclable = nRecyclable; } #endif diff --git a/src/pragma.c b/src/pragma.c index d9047e1..09282a7 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -16,14 +16,15 @@ /* ** Interpret the given string as a safety level. Return 0 for OFF, ** 1 for ON or NORMAL and 2 for FULL. Return 1 for an empty or -** unrecognized string argument. +** unrecognized string argument. The FULL option is disallowed +** if the omitFull parameter it 1. ** ** Note that the values returned are one less that the values that ** should be passed into sqlite3BtreeSetSafetyLevel(). The is done ** to support legacy SQL code. The safety level used to be boolean ** and older scripts may have used numbers 0 for OFF and 1 for ON. */ -static u8 getSafetyLevel(const char *z){ +static u8 getSafetyLevel(const char *z, int omitFull, int dflt){ /* 123456789 123456789 */ static const char zText[] = "onoffalseyestruefull"; static const u8 iOffset[] = {0, 1, 2, 4, 9, 12, 16}; @@ -34,19 +35,19 @@ static u8 getSafetyLevel(const char *z){ return (u8)sqlite3Atoi(z); } n = sqlite3Strlen30(z); - for(i=0; i<ArraySize(iLength); i++){ + for(i=0; i<ArraySize(iLength)-omitFull; i++){ if( iLength[i]==n && sqlite3StrNICmp(&zText[iOffset[i]],z,n)==0 ){ return iValue[i]; } } - return 1; + return dflt; } /* ** Interpret the given string as a boolean value. */ -u8 sqlite3GetBoolean(const char *z){ - return getSafetyLevel(z)&1; +u8 sqlite3GetBoolean(const char *z, int dflt){ + return getSafetyLevel(z,1,dflt)!=0; } /* The sqlite3GetBoolean() function is used by other modules but the @@ -189,7 +190,6 @@ static int flagPragma(Parse *pParse, const char *zLeft, const char *zRight){ #endif /* The following is VERY experimental */ { "writable_schema", SQLITE_WriteSchema|SQLITE_RecoveryMode }, - { "omit_readlock", SQLITE_NoReadlock }, /* TODO: Maybe it shouldn't be possible to change the ReadUncommitted ** flag if there are any active statements. */ @@ -221,7 +221,7 @@ static int flagPragma(Parse *pParse, const char *zLeft, const char *zRight){ mask &= ~(SQLITE_ForeignKeys); } - if( sqlite3GetBoolean(zRight) ){ + if( sqlite3GetBoolean(zRight, 0) ){ db->flags |= mask; }else{ db->flags &= ~mask; @@ -312,9 +312,12 @@ void sqlite3Pragma( const char *zDb = 0; /* The database name */ Token *pId; /* Pointer to <id> token */ int iDb; /* Database index for <database> */ - sqlite3 *db = pParse->db; - Db *pDb; - Vdbe *v = pParse->pVdbe = sqlite3VdbeCreate(db); + char *aFcntl[4]; /* Argument to SQLITE_FCNTL_PRAGMA */ + int rc; /* return value form SQLITE_FCNTL_PRAGMA */ + sqlite3 *db = pParse->db; /* The database connection */ + Db *pDb; /* The specific database being pragmaed */ + Vdbe *v = pParse->pVdbe = sqlite3VdbeCreate(db); /* Prepared statement */ + if( v==0 ) return; sqlite3VdbeRunOnlyOnce(v); pParse->nMem = 2; @@ -345,8 +348,36 @@ void sqlite3Pragma( if( sqlite3AuthCheck(pParse, SQLITE_PRAGMA, zLeft, zRight, zDb) ){ goto pragma_out; } + + /* Send an SQLITE_FCNTL_PRAGMA file-control to the underlying VFS + ** connection. If it returns SQLITE_OK, then assume that the VFS + ** handled the pragma and generate a no-op prepared statement. + */ + aFcntl[0] = 0; + aFcntl[1] = zLeft; + aFcntl[2] = zRight; + aFcntl[3] = 0; + rc = sqlite3_file_control(db, zDb, SQLITE_FCNTL_PRAGMA, (void*)aFcntl); + if( rc==SQLITE_OK ){ + if( aFcntl[0] ){ + int mem = ++pParse->nMem; + sqlite3VdbeAddOp4(v, OP_String8, 0, mem, 0, aFcntl[0], 0); + sqlite3VdbeSetNumCols(v, 1); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "result", SQLITE_STATIC); + sqlite3VdbeAddOp2(v, OP_ResultRow, mem, 1); + sqlite3_free(aFcntl[0]); + } + }else if( rc!=SQLITE_NOTFOUND ){ + if( aFcntl[0] ){ + sqlite3ErrorMsg(pParse, "%s", aFcntl[0]); + sqlite3_free(aFcntl[0]); + } + pParse->nErr++; + pParse->rc = rc; + }else + -#ifndef SQLITE_OMIT_PAGER_PRAGMAS +#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) && !defined(SQLITE_OMIT_DEPRECATED) /* ** PRAGMA [database.]default_cache_size ** PRAGMA [database.]default_cache_size=N @@ -395,7 +426,9 @@ void sqlite3Pragma( sqlite3BtreeSetCacheSize(pDb->pBt, pDb->pSchema->cache_size); } }else +#endif /* !SQLITE_OMIT_PAGER_PRAGMAS && !SQLITE_OMIT_DEPRECATED */ +#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) /* ** PRAGMA [database.]page_size ** PRAGMA [database.]page_size=N @@ -416,7 +449,7 @@ void sqlite3Pragma( ** buffer that the pager module resizes using sqlite3_realloc(). */ db->nextPagesize = sqlite3Atoi(zRight); - if( SQLITE_NOMEM==sqlite3BtreeSetPageSize(pBt, db->nextPagesize, -1, 0) ){ + if( SQLITE_NOMEM==sqlite3BtreeSetPageSize(pBt, db->nextPagesize,-1,0) ){ db->mallocFailed = 1; } } @@ -435,7 +468,7 @@ void sqlite3Pragma( int b = -1; assert( pBt!=0 ); if( zRight ){ - b = sqlite3GetBoolean(zRight); + b = sqlite3GetBoolean(zRight, 0); } if( pId2->n==0 && b>=0 ){ int ii; @@ -456,6 +489,10 @@ void sqlite3Pragma( ** second form attempts to change this setting. Both ** forms return the current setting. ** + ** The absolute value of N is used. This is undocumented and might + ** change. The only purpose is to provide an easy way to test + ** the sqlite3AbsInt32() function. + ** ** PRAGMA [database.]page_count ** ** Return the number of pages in the specified database. @@ -470,7 +507,8 @@ void sqlite3Pragma( if( sqlite3Tolower(zLeft[0])=='p' ){ sqlite3VdbeAddOp2(v, OP_Pagecount, iDb, iReg); }else{ - sqlite3VdbeAddOp3(v, OP_MaxPgcnt, iDb, iReg, sqlite3Atoi(zRight)); + sqlite3VdbeAddOp3(v, OP_MaxPgcnt, iDb, iReg, + sqlite3AbsInt32(sqlite3Atoi(zRight))); } sqlite3VdbeAddOp2(v, OP_ResultRow, iReg, 1); sqlite3VdbeSetNumCols(v, 1); @@ -625,7 +663,7 @@ void sqlite3Pragma( ** creates the database file. It is important that it is created ** as an auto-vacuum capable db. */ - int rc = sqlite3BtreeSetAutoVacuum(pBt, eAuto); + rc = sqlite3BtreeSetAutoVacuum(pBt, eAuto); if( rc==SQLITE_OK && (eAuto==1 || eAuto==2) ){ /* When setting the auto_vacuum mode to either "full" or ** "incremental", write the value of meta[6] in the database @@ -684,14 +722,11 @@ void sqlite3Pragma( ** PRAGMA [database.]cache_size=N ** ** The first form reports the current local setting for the - ** page cache size. The local setting can be different from - ** the persistent cache size value that is stored in the database - ** file itself. The value returned is the maximum number of - ** pages in the page cache. The second form sets the local - ** page cache size value. It does not change the persistent - ** cache size stored on the disk so the cache size will revert - ** to its default value when the database is closed and reopened. - ** N should be a positive integer. + ** page cache size. The second form sets the local + ** page cache size value. If N is positive then that is the + ** number of pages in the cache. If N is negative, then the + ** number of pages is adjusted so that the cache uses -N kibibytes + ** of memory. */ if( sqlite3StrICmp(zLeft,"cache_size")==0 ){ if( sqlite3ReadSchema(pParse) ) goto pragma_out; @@ -699,7 +734,7 @@ void sqlite3Pragma( if( !zRight ){ returnSingleInt(pParse, "cache_size", pDb->pSchema->cache_size); }else{ - int size = sqlite3AbsInt32(sqlite3Atoi(zRight)); + int size = sqlite3Atoi(zRight); pDb->pSchema->cache_size = size; sqlite3BtreeSetCacheSize(pDb->pBt, pDb->pSchema->cache_size); } @@ -746,7 +781,6 @@ void sqlite3Pragma( }else{ #ifndef SQLITE_OMIT_WSD if( zRight[0] ){ - int rc; int res; rc = sqlite3OsAccess(db->pVfs, zRight, SQLITE_ACCESS_READWRITE, &res); if( rc!=SQLITE_OK || res==0 ){ @@ -791,7 +825,7 @@ void sqlite3Pragma( Pager *pPager = sqlite3BtreePager(pDb->pBt); char *proxy_file_path = NULL; sqlite3_file *pFile = sqlite3PagerFile(pPager); - sqlite3OsFileControl(pFile, SQLITE_GET_LOCKPROXYFILE, + sqlite3OsFileControlHint(pFile, SQLITE_GET_LOCKPROXYFILE, &proxy_file_path); if( proxy_file_path ){ @@ -838,7 +872,7 @@ void sqlite3Pragma( sqlite3ErrorMsg(pParse, "Safety level may not be changed inside a transaction"); }else{ - pDb->safety_level = getSafetyLevel(zRight)+1; + pDb->safety_level = getSafetyLevel(zRight,0,1)+1; } } }else @@ -1037,7 +1071,7 @@ void sqlite3Pragma( #ifndef NDEBUG if( sqlite3StrICmp(zLeft, "parser_trace")==0 ){ if( zRight ){ - if( sqlite3GetBoolean(zRight) ){ + if( sqlite3GetBoolean(zRight, 0) ){ sqlite3ParserTrace(stderr, "parser: "); }else{ sqlite3ParserTrace(0, 0); @@ -1051,7 +1085,7 @@ void sqlite3Pragma( */ if( sqlite3StrICmp(zLeft, "case_sensitive_like")==0 ){ if( zRight ){ - sqlite3RegisterLikeFunctions(db, sqlite3GetBoolean(zRight)); + sqlite3RegisterLikeFunctions(db, sqlite3GetBoolean(zRight, 0)); } }else @@ -1434,6 +1468,16 @@ void sqlite3Pragma( }else #endif + /* + ** PRAGMA shrink_memory + ** + ** This pragma attempts to free as much memory as possible from the + ** current database connection. + */ + if( sqlite3StrICmp(zLeft, "shrink_memory")==0 ){ + sqlite3_db_release_memory(db); + }else + #if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) /* ** Report the current state of file logs for all databases @@ -1491,6 +1535,10 @@ void sqlite3Pragma( } }else /** BEGIN CRYPTO **/ + if( sqlite3StrICmp(zLeft, "cipher_version")==0 && !zRight ){ + extern void codec_vdbe_return_static_string(Parse *pParse, const char *zLabel, const char *value); + codec_vdbe_return_static_string(pParse, "cipher_version", CIPHER_VERSION); + }else if( sqlite3StrICmp(zLeft, "cipher")==0 && zRight ){ extern int codec_set_cipher_name(sqlite3*, int, const char *, int); codec_set_cipher_name(db, iDb, zRight, 2); // change cipher for both @@ -1515,13 +1563,13 @@ void sqlite3Pragma( extern int codec_set_page_size(sqlite3*, int, int); codec_set_page_size(db, iDb, atoi(zRight)); // change page size }else + if( sqlite3StrICmp(zLeft,"cipher_default_use_hmac")==0 ){ + extern void codec_set_default_use_hmac(int); + codec_set_default_use_hmac(sqlite3GetBoolean(zRight,1)); + }else if( sqlite3StrICmp(zLeft,"cipher_use_hmac")==0 ){ extern int codec_set_use_hmac(sqlite3*, int, int); - if(sqlite3GetBoolean(zRight)) { - codec_set_use_hmac(db, iDb, 1); - } else { - codec_set_use_hmac(db, iDb, 0); - } + codec_set_use_hmac(db, iDb, sqlite3GetBoolean(zRight,1)); }else /** END CRYPTO **/ #endif diff --git a/src/prepare.c b/src/prepare.c index fc45b8e..c46e55e 100644 --- a/src/prepare.c +++ b/src/prepare.c @@ -278,9 +278,13 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ pDb->pSchema->enc = ENC(db); if( pDb->pSchema->cache_size==0 ){ +#ifndef SQLITE_OMIT_DEPRECATED size = sqlite3AbsInt32(meta[BTREE_DEFAULT_CACHE_SIZE-1]); if( size==0 ){ size = SQLITE_DEFAULT_CACHE_SIZE; } pDb->pSchema->cache_size = size; +#else + pDb->pSchema->cache_size = SQLITE_DEFAULT_CACHE_SIZE; +#endif sqlite3BtreeSetCacheSize(pDb->pBt, pDb->pSchema->cache_size); } @@ -702,6 +706,7 @@ static int sqlite3LockAndPrepare( } sqlite3BtreeLeaveAll(db); sqlite3_mutex_leave(db->mutex); + assert( rc==SQLITE_OK || *ppStmt==0 ); return rc; } diff --git a/src/printf.c b/src/printf.c index 0babee5..58cfd2b 100644 --- a/src/printf.c +++ b/src/printf.c @@ -136,7 +136,7 @@ static char et_getdigit(LONGDOUBLE_TYPE *val, int *cnt){ /* ** Append N space characters to the given string buffer. */ -static void appendSpace(StrAccum *pAccum, int N){ +void sqlite3AppendSpace(StrAccum *pAccum, int N){ static const char zSpaces[] = " "; while( N>=(int)sizeof(zSpaces)-1 ){ sqlite3StrAccumAppend(pAccum, zSpaces, sizeof(zSpaces)-1); @@ -664,7 +664,7 @@ void sqlite3VXPrintf( register int nspace; nspace = width-length; if( nspace>0 ){ - appendSpace(pAccum, nspace); + sqlite3AppendSpace(pAccum, nspace); } } if( length>0 ){ @@ -674,7 +674,7 @@ void sqlite3VXPrintf( register int nspace; nspace = width-length; if( nspace>0 ){ - appendSpace(pAccum, nspace); + sqlite3AppendSpace(pAccum, nspace); } } sqlite3_free(zExtra); diff --git a/src/resolve.c b/src/resolve.c index 6d857f0..a66f88f 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -311,7 +311,7 @@ static int lookupName( assert( pExpr->x.pList==0 ); assert( pExpr->x.pSelect==0 ); pOrig = pEList->a[j].pExpr; - if( !pNC->allowAgg && ExprHasProperty(pOrig, EP_Agg) ){ + if( (pNC->ncFlags&NC_AllowAgg)==0 && ExprHasProperty(pOrig, EP_Agg) ){ sqlite3ErrorMsg(pParse, "misuse of aliased aggregate %s", zAs); return WRC_Abort; } @@ -533,7 +533,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ nId = sqlite3Strlen30(zId); pDef = sqlite3FindFunction(pParse->db, zId, nId, n, enc, 0); if( pDef==0 ){ - pDef = sqlite3FindFunction(pParse->db, zId, nId, -1, enc, 0); + pDef = sqlite3FindFunction(pParse->db, zId, nId, -2, enc, 0); if( pDef==0 ){ no_such_func = 1; }else{ @@ -556,7 +556,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ } } #endif - if( is_agg && !pNC->allowAgg ){ + if( is_agg && (pNC->ncFlags & NC_AllowAgg)==0 ){ sqlite3ErrorMsg(pParse, "misuse of aggregate function %.*s()", nId,zId); pNC->nErr++; is_agg = 0; @@ -570,11 +570,11 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ } if( is_agg ){ pExpr->op = TK_AGG_FUNCTION; - pNC->hasAgg = 1; + pNC->ncFlags |= NC_HasAgg; } - if( is_agg ) pNC->allowAgg = 0; + if( is_agg ) pNC->ncFlags &= ~NC_AllowAgg; sqlite3WalkExprList(pWalker, pList); - if( is_agg ) pNC->allowAgg = 1; + if( is_agg ) pNC->ncFlags |= NC_AllowAgg; /* FIX ME: Compute pExpr->affinity based on the expected return ** type of the function */ @@ -589,7 +589,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ if( ExprHasProperty(pExpr, EP_xIsSelect) ){ int nRef = pNC->nRef; #ifndef SQLITE_OMIT_CHECK - if( pNC->isCheck ){ + if( (pNC->ncFlags & NC_IsCheck)!=0 ){ sqlite3ErrorMsg(pParse,"subqueries prohibited in CHECK constraints"); } #endif @@ -603,7 +603,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ } #ifndef SQLITE_OMIT_CHECK case TK_VARIABLE: { - if( pNC->isCheck ){ + if( (pNC->ncFlags & NC_IsCheck)!=0 ){ sqlite3ErrorMsg(pParse,"parameters prohibited in CHECK constraints"); } break; @@ -685,7 +685,7 @@ static int resolveOrderByTermToExprList( nc.pParse = pParse; nc.pSrcList = pSelect->pSrc; nc.pEList = pEList; - nc.allowAgg = 1; + nc.ncFlags = NC_AllowAgg; nc.nErr = 0; db = pParse->db; savedSuppErr = db->suppressErr; @@ -799,7 +799,7 @@ static int resolveCompoundOrderBy( pE->pColl = pColl; pE->flags |= EP_IntValue | flags; pE->u.iValue = iCol; - pItem->iCol = (u16)iCol; + pItem->iOrderByCol = (u16)iCol; pItem->done = 1; }else{ moreToDo = 1; @@ -848,12 +848,12 @@ int sqlite3ResolveOrderGroupBy( pEList = pSelect->pEList; assert( pEList!=0 ); /* sqlite3SelectNew() guarantees this */ for(i=0, pItem=pOrderBy->a; i<pOrderBy->nExpr; i++, pItem++){ - if( pItem->iCol ){ - if( pItem->iCol>pEList->nExpr ){ + if( pItem->iOrderByCol ){ + if( pItem->iOrderByCol>pEList->nExpr ){ resolveOutOfRangeError(pParse, zType, i+1, pEList->nExpr); return 1; } - resolveAlias(pParse, pEList, pItem->iCol-1, pItem->pExpr, zType); + resolveAlias(pParse, pEList, pItem->iOrderByCol-1, pItem->pExpr, zType); } } return 0; @@ -883,7 +883,7 @@ static int resolveOrderGroupBy( ExprList *pOrderBy, /* An ORDER BY or GROUP BY clause to resolve */ const char *zType /* Either "ORDER" or "GROUP", as appropriate */ ){ - int i; /* Loop counter */ + int i, j; /* Loop counters */ int iCol; /* Column number */ struct ExprList_item *pItem; /* A term of the ORDER BY clause */ Parse *pParse; /* Parsing context */ @@ -900,7 +900,7 @@ static int resolveOrderGroupBy( ** a copy of the iCol-th result-set column. The subsequent call to ** sqlite3ResolveOrderGroupBy() will convert the expression to a ** copy of the iCol-th result-set expression. */ - pItem->iCol = (u16)iCol; + pItem->iOrderByCol = (u16)iCol; continue; } if( sqlite3ExprIsInteger(pE, &iCol) ){ @@ -911,15 +911,20 @@ static int resolveOrderGroupBy( resolveOutOfRangeError(pParse, zType, i+1, nResult); return 1; } - pItem->iCol = (u16)iCol; + pItem->iOrderByCol = (u16)iCol; continue; } /* Otherwise, treat the ORDER BY term as an ordinary expression */ - pItem->iCol = 0; + pItem->iOrderByCol = 0; if( sqlite3ResolveExprNames(pNC, pE) ){ return 1; } + for(j=0; j<pSelect->pEList->nExpr; j++){ + if( sqlite3ExprCompare(pE, pSelect->pEList->a[j].pExpr)==0 ){ + pItem->iOrderByCol = j+1; + } + } } return sqlite3ResolveOrderGroupBy(pParse, pSelect, pOrderBy, zType); } @@ -982,7 +987,7 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ /* Set up the local name-context to pass to sqlite3ResolveExprNames() to ** resolve the result-set expression list. */ - sNC.allowAgg = 1; + sNC.ncFlags = NC_AllowAgg; sNC.pSrcList = p->pSrc; sNC.pNext = pOuterNC; @@ -1028,10 +1033,10 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ */ assert( (p->selFlags & SF_Aggregate)==0 ); pGroupBy = p->pGroupBy; - if( pGroupBy || sNC.hasAgg ){ + if( pGroupBy || (sNC.ncFlags & NC_HasAgg)!=0 ){ p->selFlags |= SF_Aggregate; }else{ - sNC.allowAgg = 0; + sNC.ncFlags &= ~NC_AllowAgg; } /* If a HAVING clause is present, then there must be a GROUP BY clause. @@ -1060,7 +1065,7 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ ** outer queries */ sNC.pNext = 0; - sNC.allowAgg = 1; + sNC.ncFlags |= NC_AllowAgg; /* Process the ORDER BY clause for singleton SELECT statements. ** The ORDER BY clause for compounds SELECT statements is handled @@ -1148,7 +1153,7 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ ** ** Function calls are checked to make sure that the function is ** defined and that the correct number of arguments are specified. -** If the function is an aggregate function, then the pNC->hasAgg is +** If the function is an aggregate function, then the NC_HasAgg flag is ** set and the opcode is changed from TK_FUNCTION to TK_AGG_FUNCTION. ** If an expression contains aggregate functions then the EP_Agg ** property on the expression is set. @@ -1160,7 +1165,7 @@ int sqlite3ResolveExprNames( NameContext *pNC, /* Namespace to resolve expressions in. */ Expr *pExpr /* The expression to be analyzed. */ ){ - int savedHasAgg; + u8 savedHasAgg; Walker w; if( pExpr==0 ) return 0; @@ -1173,8 +1178,8 @@ int sqlite3ResolveExprNames( pParse->nHeight += pExpr->nHeight; } #endif - savedHasAgg = pNC->hasAgg; - pNC->hasAgg = 0; + savedHasAgg = pNC->ncFlags & NC_HasAgg; + pNC->ncFlags &= ~NC_HasAgg; w.xExprCallback = resolveExprStep; w.xSelectCallback = resolveSelectStep; w.pParse = pNC->pParse; @@ -1186,10 +1191,10 @@ int sqlite3ResolveExprNames( if( pNC->nErr>0 || w.pParse->nErr>0 ){ ExprSetProperty(pExpr, EP_Error); } - if( pNC->hasAgg ){ + if( pNC->ncFlags & NC_HasAgg ){ ExprSetProperty(pExpr, EP_Agg); }else if( savedHasAgg ){ - pNC->hasAgg = 1; + pNC->ncFlags |= NC_HasAgg; } return ExprHasProperty(pExpr, EP_Error); } diff --git a/src/rowset.c b/src/rowset.c index d84bb93..58c18b7 100644 --- a/src/rowset.c +++ b/src/rowset.c @@ -76,6 +76,11 @@ /* ** Each entry in a RowSet is an instance of the following object. +** +** This same object is reused to store a linked list of trees of RowSetEntry +** objects. In that alternative use, pRight points to the next entry +** in the list, pLeft points to the tree, and v is unused. The +** RowSet.pForest value points to the head of this forest list. */ struct RowSetEntry { i64 v; /* ROWID value for this entry */ @@ -105,13 +110,19 @@ struct RowSet { struct RowSetEntry *pEntry; /* List of entries using pRight */ struct RowSetEntry *pLast; /* Last entry on the pEntry list */ struct RowSetEntry *pFresh; /* Source of new entry objects */ - struct RowSetEntry *pTree; /* Binary tree of entries */ + struct RowSetEntry *pForest; /* List of binary trees of entries */ u16 nFresh; /* Number of objects on pFresh */ - u8 isSorted; /* True if pEntry is sorted */ + u8 rsFlags; /* Various flags */ u8 iBatch; /* Current insert batch */ }; /* +** Allowed values for RowSet.rsFlags +*/ +#define ROWSET_SORTED 0x01 /* True if RowSet.pEntry is sorted */ +#define ROWSET_NEXT 0x02 /* True if sqlite3RowSetNext() has been called */ + +/* ** Turn bulk memory into a RowSet object. N bytes of memory ** are available at pSpace. The db pointer is used as a memory context ** for any subsequent allocations that need to occur. @@ -131,10 +142,10 @@ RowSet *sqlite3RowSetInit(sqlite3 *db, void *pSpace, unsigned int N){ p->db = db; p->pEntry = 0; p->pLast = 0; - p->pTree = 0; + p->pForest = 0; p->pFresh = (struct RowSetEntry*)(ROUND8(sizeof(*p)) + (char*)p); p->nFresh = (u16)((N - ROUND8(sizeof(*p)))/sizeof(struct RowSetEntry)); - p->isSorted = 1; + p->rsFlags = ROWSET_SORTED; p->iBatch = 0; return p; } @@ -154,43 +165,59 @@ void sqlite3RowSetClear(RowSet *p){ p->nFresh = 0; p->pEntry = 0; p->pLast = 0; - p->pTree = 0; - p->isSorted = 1; + p->pForest = 0; + p->rsFlags = ROWSET_SORTED; } /* -** Insert a new value into a RowSet. +** Allocate a new RowSetEntry object that is associated with the +** given RowSet. Return a pointer to the new and completely uninitialized +** objected. ** -** The mallocFailed flag of the database connection is set if a -** memory allocation fails. +** In an OOM situation, the RowSet.db->mallocFailed flag is set and this +** routine returns NULL. */ -void sqlite3RowSetInsert(RowSet *p, i64 rowid){ - struct RowSetEntry *pEntry; /* The new entry */ - struct RowSetEntry *pLast; /* The last prior entry */ +static struct RowSetEntry *rowSetEntryAlloc(RowSet *p){ assert( p!=0 ); if( p->nFresh==0 ){ struct RowSetChunk *pNew; pNew = sqlite3DbMallocRaw(p->db, sizeof(*pNew)); if( pNew==0 ){ - return; + return 0; } pNew->pNextChunk = p->pChunk; p->pChunk = pNew; p->pFresh = pNew->aEntry; p->nFresh = ROWSET_ENTRY_PER_CHUNK; } - pEntry = p->pFresh++; p->nFresh--; + return p->pFresh++; +} + +/* +** Insert a new value into a RowSet. +** +** The mallocFailed flag of the database connection is set if a +** memory allocation fails. +*/ +void sqlite3RowSetInsert(RowSet *p, i64 rowid){ + struct RowSetEntry *pEntry; /* The new entry */ + struct RowSetEntry *pLast; /* The last prior entry */ + + /* This routine is never called after sqlite3RowSetNext() */ + assert( p!=0 && (p->rsFlags & ROWSET_NEXT)==0 ); + + pEntry = rowSetEntryAlloc(p); + if( pEntry==0 ) return; pEntry->v = rowid; pEntry->pRight = 0; pLast = p->pLast; if( pLast ){ - if( p->isSorted && rowid<=pLast->v ){ - p->isSorted = 0; + if( (p->rsFlags & ROWSET_SORTED)!=0 && rowid<=pLast->v ){ + p->rsFlags &= ~ROWSET_SORTED; } pLast->pRight = pEntry; }else{ - assert( p->pEntry==0 ); /* Fires if INSERT after SMALLEST */ p->pEntry = pEntry; } p->pLast = pEntry; @@ -202,7 +229,7 @@ void sqlite3RowSetInsert(RowSet *p, i64 rowid){ ** The input lists are connected via pRight pointers and are ** assumed to each already be in sorted order. */ -static struct RowSetEntry *rowSetMerge( +static struct RowSetEntry *rowSetEntryMerge( struct RowSetEntry *pA, /* First sorted list to be merged */ struct RowSetEntry *pB /* Second sorted list to be merged */ ){ @@ -236,32 +263,29 @@ static struct RowSetEntry *rowSetMerge( } /* -** Sort all elements on the pEntry list of the RowSet into ascending order. +** Sort all elements on the list of RowSetEntry objects into order of +** increasing v. */ -static void rowSetSort(RowSet *p){ +static struct RowSetEntry *rowSetEntrySort(struct RowSetEntry *pIn){ unsigned int i; - struct RowSetEntry *pEntry; - struct RowSetEntry *aBucket[40]; + struct RowSetEntry *pNext, *aBucket[40]; - assert( p->isSorted==0 ); memset(aBucket, 0, sizeof(aBucket)); - while( p->pEntry ){ - pEntry = p->pEntry; - p->pEntry = pEntry->pRight; - pEntry->pRight = 0; + while( pIn ){ + pNext = pIn->pRight; + pIn->pRight = 0; for(i=0; aBucket[i]; i++){ - pEntry = rowSetMerge(aBucket[i], pEntry); + pIn = rowSetEntryMerge(aBucket[i], pIn); aBucket[i] = 0; } - aBucket[i] = pEntry; + aBucket[i] = pIn; + pIn = pNext; } - pEntry = 0; + pIn = 0; for(i=0; i<sizeof(aBucket)/sizeof(aBucket[0]); i++){ - pEntry = rowSetMerge(pEntry, aBucket[i]); + pIn = rowSetEntryMerge(pIn, aBucket[i]); } - p->pEntry = pEntry; - p->pLast = 0; - p->isSorted = 1; + return pIn; } @@ -355,20 +379,37 @@ static struct RowSetEntry *rowSetListToTree(struct RowSetEntry *pList){ } /* -** Convert the list in p->pEntry into a sorted list if it is not -** sorted already. If there is a binary tree on p->pTree, then -** convert it into a list too and merge it into the p->pEntry list. +** Take all the entries on p->pEntry and on the trees in p->pForest and +** sort them all together into one big ordered list on p->pEntry. +** +** This routine should only be called once in the life of a RowSet. */ static void rowSetToList(RowSet *p){ - if( !p->isSorted ){ - rowSetSort(p); + + /* This routine is called only once */ + assert( p!=0 && (p->rsFlags & ROWSET_NEXT)==0 ); + + if( (p->rsFlags & ROWSET_SORTED)==0 ){ + p->pEntry = rowSetEntrySort(p->pEntry); } - if( p->pTree ){ - struct RowSetEntry *pHead, *pTail; - rowSetTreeToList(p->pTree, &pHead, &pTail); - p->pTree = 0; - p->pEntry = rowSetMerge(p->pEntry, pHead); + + /* While this module could theoretically support it, sqlite3RowSetNext() + ** is never called after sqlite3RowSetText() for the same RowSet. So + ** there is never a forest to deal with. Should this change, simply + ** remove the assert() and the #if 0. */ + assert( p->pForest==0 ); +#if 0 + while( p->pForest ){ + struct RowSetEntry *pTree = p->pForest->pLeft; + if( pTree ){ + struct RowSetEntry *pHead, *pTail; + rowSetTreeToList(pTree, &pHead, &pTail); + p->pEntry = rowSetEntryMerge(p->pEntry, pHead); + } + p->pForest = p->pForest->pRight; } +#endif + p->rsFlags |= ROWSET_NEXT; /* Verify this routine is never called again */ } /* @@ -380,7 +421,12 @@ static void rowSetToList(RowSet *p){ ** routine may not be called again. */ int sqlite3RowSetNext(RowSet *p, i64 *pRowid){ - rowSetToList(p); + assert( p!=0 ); + + /* Merge the forest into a single sorted list on first call */ + if( (p->rsFlags & ROWSET_NEXT)==0 ) rowSetToList(p); + + /* Return the next entry on the list */ if( p->pEntry ){ *pRowid = p->pEntry->v; p->pEntry = p->pEntry->pRight; @@ -396,26 +442,66 @@ int sqlite3RowSetNext(RowSet *p, i64 *pRowid){ /* ** Check to see if element iRowid was inserted into the the rowset as ** part of any insert batch prior to iBatch. Return 1 or 0. +** +** If this is the first test of a new batch and if there exist entires +** on pRowSet->pEntry, then sort those entires into the forest at +** pRowSet->pForest so that they can be tested. */ int sqlite3RowSetTest(RowSet *pRowSet, u8 iBatch, sqlite3_int64 iRowid){ - struct RowSetEntry *p; + struct RowSetEntry *p, *pTree; + + /* This routine is never called after sqlite3RowSetNext() */ + assert( pRowSet!=0 && (pRowSet->rsFlags & ROWSET_NEXT)==0 ); + + /* Sort entries into the forest on the first test of a new batch + */ if( iBatch!=pRowSet->iBatch ){ - if( pRowSet->pEntry ){ - rowSetToList(pRowSet); - pRowSet->pTree = rowSetListToTree(pRowSet->pEntry); + p = pRowSet->pEntry; + if( p ){ + struct RowSetEntry **ppPrevTree = &pRowSet->pForest; + if( (pRowSet->rsFlags & ROWSET_SORTED)==0 ){ + p = rowSetEntrySort(p); + } + for(pTree = pRowSet->pForest; pTree; pTree=pTree->pRight){ + ppPrevTree = &pTree->pRight; + if( pTree->pLeft==0 ){ + pTree->pLeft = rowSetListToTree(p); + break; + }else{ + struct RowSetEntry *pAux, *pTail; + rowSetTreeToList(pTree->pLeft, &pAux, &pTail); + pTree->pLeft = 0; + p = rowSetEntryMerge(pAux, p); + } + } + if( pTree==0 ){ + *ppPrevTree = pTree = rowSetEntryAlloc(pRowSet); + if( pTree ){ + pTree->v = 0; + pTree->pRight = 0; + pTree->pLeft = rowSetListToTree(p); + } + } pRowSet->pEntry = 0; pRowSet->pLast = 0; + pRowSet->rsFlags |= ROWSET_SORTED; } pRowSet->iBatch = iBatch; } - p = pRowSet->pTree; - while( p ){ - if( p->v<iRowid ){ - p = p->pRight; - }else if( p->v>iRowid ){ - p = p->pLeft; - }else{ - return 1; + + /* Test to see if the iRowid value appears anywhere in the forest. + ** Return 1 if it does and 0 if not. + */ + for(pTree = pRowSet->pForest; pTree; pTree=pTree->pRight){ + p = pTree->pLeft; + while( p ){ + if( p->v<iRowid ){ + p = p->pRight; + }else if( p->v>iRowid ){ + p = p->pLeft; + }else{ + return 1; + } } } return 0; diff --git a/src/select.c b/src/select.c index 571a778..d79a611 100644 --- a/src/select.c +++ b/src/select.c @@ -73,6 +73,7 @@ Select *sqlite3SelectNew( pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db,TK_ALL,0)); } pNew->pEList = pEList; + if( pSrc==0 ) pSrc = sqlite3DbMallocZero(db, sizeof(*pSrc)); pNew->pSrc = pSrc; pNew->pWhere = pWhere; pNew->pGroupBy = pGroupBy; @@ -1257,9 +1258,17 @@ static int selectColumnsFromExprList( char *zName; /* Column name */ int nName; /* Size of name in zName[] */ - *pnCol = nCol = pEList->nExpr; - aCol = *paCol = sqlite3DbMallocZero(db, sizeof(aCol[0])*nCol); - if( aCol==0 ) return SQLITE_NOMEM; + if( pEList ){ + nCol = pEList->nExpr; + aCol = sqlite3DbMallocZero(db, sizeof(aCol[0])*nCol); + testcase( aCol==0 ); + }else{ + nCol = 0; + aCol = 0; + } + *pnCol = nCol; + *paCol = aCol; + for(i=0, pCol=aCol; i<nCol; i++, pCol++){ /* Get an appropriate name for the column */ @@ -1611,8 +1620,12 @@ static int multiSelect( */ assert( p->pEList && pPrior->pEList ); if( p->pEList->nExpr!=pPrior->pEList->nExpr ){ - sqlite3ErrorMsg(pParse, "SELECTs to the left and right of %s" - " do not have the same number of result columns", selectOpName(p->op)); + if( p->selFlags & SF_Values ){ + sqlite3ErrorMsg(pParse, "all VALUES must have the same number of terms"); + }else{ + sqlite3ErrorMsg(pParse, "SELECTs to the left and right of %s" + " do not have the same number of result columns", selectOpName(p->op)); + } rc = 1; goto multi_select_end; } @@ -2219,8 +2232,8 @@ static int multiSelectOrderBy( for(i=1; db->mallocFailed==0 && i<=p->pEList->nExpr; i++){ struct ExprList_item *pItem; for(j=0, pItem=pOrderBy->a; j<nOrderBy; j++, pItem++){ - assert( pItem->iCol>0 ); - if( pItem->iCol==i ) break; + assert( pItem->iOrderByCol>0 ); + if( pItem->iOrderByCol==i ) break; } if( j==nOrderBy ){ Expr *pNew = sqlite3Expr(db, TK_INTEGER, 0); @@ -2228,7 +2241,7 @@ static int multiSelectOrderBy( pNew->flags |= EP_IntValue; pNew->u.iValue = i; pOrderBy = sqlite3ExprListAppend(pParse, pOrderBy, pNew); - pOrderBy->a[nOrderBy++].iCol = (u16)i; + if( pOrderBy ) pOrderBy->a[nOrderBy++].iOrderByCol = (u16)i; } } } @@ -2244,8 +2257,8 @@ static int multiSelectOrderBy( if( aPermute ){ struct ExprList_item *pItem; for(i=0, pItem=pOrderBy->a; i<nOrderBy; i++, pItem++){ - assert( pItem->iCol>0 && pItem->iCol<=p->pEList->nExpr ); - aPermute[i] = pItem->iCol - 1; + assert( pItem->iOrderByCol>0 && pItem->iOrderByCol<=p->pEList->nExpr ); + aPermute[i] = pItem->iOrderByCol - 1; } pKeyMerge = sqlite3DbMallocRaw(db, sizeof(*pKeyMerge)+nOrderBy*(sizeof(CollSeq*)+1)); @@ -2588,9 +2601,8 @@ static void substSelect( #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) /* -** This routine attempts to flatten subqueries in order to speed -** execution. It returns 1 if it makes changes and 0 if no flattening -** occurs. +** This routine attempts to flatten subqueries as a performance optimization. +** This routine returns 1 if it makes changes and 0 if no flattening occurs. ** ** To understand the concept of flattening, consider the following ** query: @@ -2632,7 +2644,10 @@ static void substSelect( ** (6) The subquery does not use aggregates or the outer query is not ** DISTINCT. ** -** (7) The subquery has a FROM clause. +** (7) The subquery has a FROM clause. TODO: For subqueries without +** A FROM clause, consider adding a FROM close with the special +** table sqlite_once that consists of a single row containing a +** single NULL. ** ** (8) The subquery does not use LIMIT or the outer query is not a join. ** @@ -2665,11 +2680,14 @@ static void substSelect( ** ** * is not itself part of a compound select, ** * is not an aggregate or DISTINCT query, and -** * has no other tables or sub-selects in the FROM clause. +** * is not a join ** ** The parent and sub-query may contain WHERE clauses. Subject to ** rules (11), (13) and (14), they may also contain ORDER BY, -** LIMIT and OFFSET clauses. +** LIMIT and OFFSET clauses. The subquery cannot use any compound +** operator other than UNION ALL because all the other compound +** operators have an implied DISTINCT which is disallowed by +** restriction (4). ** ** (18) If the sub-query is a compound select, then all terms of the ** ORDER by clause of the parent must be simple references to @@ -2681,7 +2699,7 @@ static void substSelect( ** (20) If the sub-query is a compound select, then it must not use ** an ORDER BY clause. Ticket #3773. We could relax this constraint ** somewhat by saying that the terms of the ORDER BY clause must -** appear as unmodified result columns in the outer query. But +** appear as unmodified result columns in the outer query. But we ** have other optimizations in mind to deal with that case. ** ** (21) The subquery does not use LIMIT or the outer query is not @@ -2810,19 +2828,21 @@ static int flattenSubquery( for(pSub1=pSub; pSub1; pSub1=pSub1->pPrior){ testcase( (pSub1->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct ); testcase( (pSub1->selFlags & (SF_Distinct|SF_Aggregate))==SF_Aggregate ); + assert( pSub->pSrc!=0 ); if( (pSub1->selFlags & (SF_Distinct|SF_Aggregate))!=0 || (pSub1->pPrior && pSub1->op!=TK_ALL) - || NEVER(pSub1->pSrc==0) || pSub1->pSrc->nSrc!=1 + || pSub1->pSrc->nSrc<1 ){ return 0; } + testcase( pSub1->pSrc->nSrc>1 ); } /* Restriction 18. */ if( p->pOrderBy ){ int ii; for(ii=0; ii<p->pOrderBy->nExpr; ii++){ - if( p->pOrderBy->a[ii].iCol==0 ) return 0; + if( p->pOrderBy->a[ii].iOrderByCol==0 ) return 0; } } } @@ -2831,7 +2851,8 @@ static int flattenSubquery( /* Authorize the subquery */ pParse->zAuthContext = pSubitem->zName; - sqlite3AuthCheck(pParse, SQLITE_SELECT, 0, 0, 0); + TESTONLY(i =) sqlite3AuthCheck(pParse, SQLITE_SELECT, 0, 0, 0); + testcase( i==SQLITE_DENY ); pParse->zAuthContext = zSavedAuthContext; /* If the sub-query is a compound SELECT statement, then (by restrictions @@ -3128,6 +3149,7 @@ static Table *isSimpleCount(Select *p, AggInfo *pAggInfo){ if( IsVirtual(pTab) ) return 0; if( pExpr->op!=TK_AGG_FUNCTION ) return 0; + if( pAggInfo->nFunc==0 ) return 0; if( (pAggInfo->aFunc[0].pFunc->flags&SQLITE_FUNC_COUNT)==0 ) return 0; if( pExpr->flags&EP_Distinct ) return 0; @@ -3584,6 +3606,8 @@ static void finalizeAggFunctions(Parse *pParse, AggInfo *pAggInfo){ static void updateAccumulator(Parse *pParse, AggInfo *pAggInfo){ Vdbe *v = pParse->pVdbe; int i; + int regHit = 0; + int addrHitTest = 0; struct AggInfo_func *pF; struct AggInfo_col *pC; @@ -3619,7 +3643,8 @@ static void updateAccumulator(Parse *pParse, AggInfo *pAggInfo){ if( !pColl ){ pColl = pParse->db->pDfltColl; } - sqlite3VdbeAddOp4(v, OP_CollSeq, 0, 0, 0, (char *)pColl, P4_COLLSEQ); + if( regHit==0 && pAggInfo->nAccumulator ) regHit = ++pParse->nMem; + sqlite3VdbeAddOp4(v, OP_CollSeq, regHit, 0, 0, (char *)pColl, P4_COLLSEQ); } sqlite3VdbeAddOp4(v, OP_AggStep, 0, regAgg, pF->iMem, (void*)pF->pFunc, P4_FUNCDEF); @@ -3642,12 +3667,18 @@ static void updateAccumulator(Parse *pParse, AggInfo *pAggInfo){ ** Another solution would be to change the OP_SCopy used to copy cached ** values to an OP_Copy. */ + if( regHit ){ + addrHitTest = sqlite3VdbeAddOp1(v, OP_If, regHit); + } sqlite3ExprCacheClear(pParse); for(i=0, pC=pAggInfo->aCol; i<pAggInfo->nAccumulator; i++, pC++){ sqlite3ExprCode(pParse, pC->pExpr, pC->iMem); } pAggInfo->directMode = 0; sqlite3ExprCacheClear(pParse); + if( addrHitTest ){ + sqlite3VdbeJumpHere(v, addrHitTest); + } } /* @@ -3845,12 +3876,11 @@ int sqlite3Select( topAddr = sqlite3VdbeAddOp2(v, OP_Integer, 0, pItem->regReturn); pItem->addrFillSub = topAddr+1; VdbeNoopComment((v, "materialize %s", pItem->pTab->zName)); - if( pItem->isCorrelated==0 && pParse->pTriggerTab==0 ){ + if( pItem->isCorrelated==0 ){ /* If the subquery is no correlated and if we are not inside of ** a trigger, then we only need to compute the value of the subquery ** once. */ - int regOnce = ++pParse->nMem; - onceAddr = sqlite3VdbeAddOp1(v, OP_Once, regOnce); + onceAddr = sqlite3CodeOnce(pParse); } sqlite3SelectDestInit(&dest, SRT_EphemTab, pItem->iCursor); explainSetInteger(pItem->iSelectId, (u8)pParse->iNextSelectId); @@ -3860,7 +3890,7 @@ int sqlite3Select( retAddr = sqlite3VdbeAddOp1(v, OP_Return, pItem->regReturn); VdbeComment((v, "end %s", pItem->pTab->zName)); sqlite3VdbeChangeP1(v, topAddr, retAddr); - + sqlite3ClearTempRegCache(pParse); } if( /*pParse->nErr ||*/ db->mallocFailed ){ goto select_end; @@ -4110,7 +4140,9 @@ int sqlite3Select( sAggInfo.nAccumulator = sAggInfo.nColumn; for(i=0; i<sAggInfo.nFunc; i++){ assert( !ExprHasProperty(sAggInfo.aFunc[i].pExpr, EP_xIsSelect) ); + sNC.ncFlags |= NC_InAggFunc; sqlite3ExprAnalyzeAggList(&sNC, sAggInfo.aFunc[i].pExpr->x.pList); + sNC.ncFlags &= ~NC_InAggFunc; } if( db->mallocFailed ) goto select_end; @@ -4155,6 +4187,7 @@ int sqlite3Select( VdbeComment((v, "clear abort flag")); sqlite3VdbeAddOp2(v, OP_Integer, 0, iUseFlag); VdbeComment((v, "indicate accumulator empty")); + sqlite3VdbeAddOp3(v, OP_Null, 0, iAMem, iAMem+pGroupBy->nExpr-1); /* Begin a loop that will extract all source rows in GROUP BY order. ** This might involve two separate loops with an OP_Sort in between, or @@ -4207,7 +4240,7 @@ int sqlite3Select( int r2; r2 = sqlite3ExprCodeGetColumn(pParse, - pCol->pTab, pCol->iColumn, pCol->iTable, r1); + pCol->pTab, pCol->iColumn, pCol->iTable, r1, 0); if( r1!=r2 ){ sqlite3VdbeAddOp2(v, OP_SCopy, r2, r1); } @@ -4494,98 +4527,98 @@ select_end: return rc; } -#if defined(SQLITE_DEBUG) +#if defined(SQLITE_ENABLE_TREE_EXPLAIN) /* -******************************************************************************* -** The following code is used for testing and debugging only. The code -** that follows does not appear in normal builds. -** -** These routines are used to print out the content of all or part of a -** parse structures such as Select or Expr. Such printouts are useful -** for helping to understand what is happening inside the code generator -** during the execution of complex SELECT statements. -** -** These routine are not called anywhere from within the normal -** code base. Then are intended to be called from within the debugger -** or from temporary "printf" statements inserted for debugging. +** Generate a human-readable description of a the Select object. */ -void sqlite3PrintExpr(Expr *p){ - if( !ExprHasProperty(p, EP_IntValue) && p->u.zToken ){ - sqlite3DebugPrintf("(%s", p->u.zToken); - }else{ - sqlite3DebugPrintf("(%d", p->op); - } - if( p->pLeft ){ - sqlite3DebugPrintf(" "); - sqlite3PrintExpr(p->pLeft); - } - if( p->pRight ){ - sqlite3DebugPrintf(" "); - sqlite3PrintExpr(p->pRight); - } - sqlite3DebugPrintf(")"); -} -void sqlite3PrintExprList(ExprList *pList){ - int i; - for(i=0; i<pList->nExpr; i++){ - sqlite3PrintExpr(pList->a[i].pExpr); - if( i<pList->nExpr-1 ){ - sqlite3DebugPrintf(", "); +static void explainOneSelect(Vdbe *pVdbe, Select *p){ + sqlite3ExplainPrintf(pVdbe, "SELECT "); + if( p->selFlags & (SF_Distinct|SF_Aggregate) ){ + if( p->selFlags & SF_Distinct ){ + sqlite3ExplainPrintf(pVdbe, "DISTINCT "); + } + if( p->selFlags & SF_Aggregate ){ + sqlite3ExplainPrintf(pVdbe, "agg_flag "); } + sqlite3ExplainNL(pVdbe); + sqlite3ExplainPrintf(pVdbe, " "); } -} -void sqlite3PrintSelect(Select *p, int indent){ - sqlite3DebugPrintf("%*sSELECT(%p) ", indent, "", p); - sqlite3PrintExprList(p->pEList); - sqlite3DebugPrintf("\n"); - if( p->pSrc ){ - char *zPrefix; + sqlite3ExplainExprList(pVdbe, p->pEList); + sqlite3ExplainNL(pVdbe); + if( p->pSrc && p->pSrc->nSrc ){ int i; - zPrefix = "FROM"; + sqlite3ExplainPrintf(pVdbe, "FROM "); + sqlite3ExplainPush(pVdbe); for(i=0; i<p->pSrc->nSrc; i++){ struct SrcList_item *pItem = &p->pSrc->a[i]; - sqlite3DebugPrintf("%*s ", indent+6, zPrefix); - zPrefix = ""; + sqlite3ExplainPrintf(pVdbe, "{%d,*} = ", pItem->iCursor); if( pItem->pSelect ){ - sqlite3DebugPrintf("(\n"); - sqlite3PrintSelect(pItem->pSelect, indent+10); - sqlite3DebugPrintf("%*s)", indent+8, ""); + sqlite3ExplainSelect(pVdbe, pItem->pSelect); + if( pItem->pTab ){ + sqlite3ExplainPrintf(pVdbe, " (tabname=%s)", pItem->pTab->zName); + } }else if( pItem->zName ){ - sqlite3DebugPrintf("%s", pItem->zName); - } - if( pItem->pTab ){ - sqlite3DebugPrintf("(table: %s)", pItem->pTab->zName); + sqlite3ExplainPrintf(pVdbe, "%s", pItem->zName); } if( pItem->zAlias ){ - sqlite3DebugPrintf(" AS %s", pItem->zAlias); + sqlite3ExplainPrintf(pVdbe, " (AS %s)", pItem->zAlias); } - if( i<p->pSrc->nSrc-1 ){ - sqlite3DebugPrintf(","); + if( pItem->jointype & JT_LEFT ){ + sqlite3ExplainPrintf(pVdbe, " LEFT-JOIN"); } - sqlite3DebugPrintf("\n"); + sqlite3ExplainNL(pVdbe); } + sqlite3ExplainPop(pVdbe); } if( p->pWhere ){ - sqlite3DebugPrintf("%*s WHERE ", indent, ""); - sqlite3PrintExpr(p->pWhere); - sqlite3DebugPrintf("\n"); + sqlite3ExplainPrintf(pVdbe, "WHERE "); + sqlite3ExplainExpr(pVdbe, p->pWhere); + sqlite3ExplainNL(pVdbe); } if( p->pGroupBy ){ - sqlite3DebugPrintf("%*s GROUP BY ", indent, ""); - sqlite3PrintExprList(p->pGroupBy); - sqlite3DebugPrintf("\n"); + sqlite3ExplainPrintf(pVdbe, "GROUPBY "); + sqlite3ExplainExprList(pVdbe, p->pGroupBy); + sqlite3ExplainNL(pVdbe); } if( p->pHaving ){ - sqlite3DebugPrintf("%*s HAVING ", indent, ""); - sqlite3PrintExpr(p->pHaving); - sqlite3DebugPrintf("\n"); + sqlite3ExplainPrintf(pVdbe, "HAVING "); + sqlite3ExplainExpr(pVdbe, p->pHaving); + sqlite3ExplainNL(pVdbe); } if( p->pOrderBy ){ - sqlite3DebugPrintf("%*s ORDER BY ", indent, ""); - sqlite3PrintExprList(p->pOrderBy); - sqlite3DebugPrintf("\n"); + sqlite3ExplainPrintf(pVdbe, "ORDERBY "); + sqlite3ExplainExprList(pVdbe, p->pOrderBy); + sqlite3ExplainNL(pVdbe); + } + if( p->pLimit ){ + sqlite3ExplainPrintf(pVdbe, "LIMIT "); + sqlite3ExplainExpr(pVdbe, p->pLimit); + sqlite3ExplainNL(pVdbe); + } + if( p->pOffset ){ + sqlite3ExplainPrintf(pVdbe, "OFFSET "); + sqlite3ExplainExpr(pVdbe, p->pOffset); + sqlite3ExplainNL(pVdbe); } } +void sqlite3ExplainSelect(Vdbe *pVdbe, Select *p){ + if( p==0 ){ + sqlite3ExplainPrintf(pVdbe, "(null-select)"); + return; + } + while( p->pPrior ) p = p->pPrior; + sqlite3ExplainPush(pVdbe); + while( p ){ + explainOneSelect(pVdbe, p); + p = p->pNext; + if( p==0 ) break; + sqlite3ExplainNL(pVdbe); + sqlite3ExplainPrintf(pVdbe, "%s\n", selectOpName(p->op)); + } + sqlite3ExplainPrintf(pVdbe, "END"); + sqlite3ExplainPop(pVdbe); +} + /* End of the structure debug printing code *****************************************************************************/ -#endif /* defined(SQLITE_TEST) || defined(SQLITE_DEBUG) */ +#endif /* defined(SQLITE_ENABLE_TREE_EXPLAIN) */ diff --git a/src/shell.c b/src/shell.c index 07623e5..801ad2c 100644 --- a/src/shell.c +++ b/src/shell.c @@ -57,7 +57,7 @@ # include <readline/history.h> #endif #if !defined(HAVE_EDITLINE) && (!defined(HAVE_READLINE) || HAVE_READLINE!=1) -# define readline(p) local_getline(p,stdin) +# define readline(p) local_getline(p,stdin,0) # define add_history(X) # define read_history(X) # define write_history(X) @@ -68,6 +68,8 @@ # include <io.h> #define isatty(h) _isatty(h) #define access(f,m) _access((f),(m)) +#define popen(a,b) _popen((a),(b)) +#define pclose(x) _pclose(x) #else /* Make sure isatty() has a prototype. */ @@ -334,10 +336,11 @@ static void shellstaticFunc( ** The interface is like "readline" but no command-line editing ** is done. */ -static char *local_getline(char *zPrompt, FILE *in){ +static char *local_getline(char *zPrompt, FILE *in, int csvFlag){ char *zLine; int nLine; int n; + int inQuote = 0; if( zPrompt && *zPrompt ){ printf("%s",zPrompt); @@ -361,8 +364,11 @@ static char *local_getline(char *zPrompt, FILE *in){ zLine[n] = 0; break; } - while( zLine[n] ){ n++; } - if( n>0 && zLine[n-1]=='\n' ){ + while( zLine[n] ){ + if( zLine[n]=='"' ) inQuote = !inQuote; + n++; + } + if( n>0 && zLine[n-1]=='\n' && (!inQuote || !csvFlag) ){ n--; if( n>0 && zLine[n-1]=='\r' ) n--; zLine[n] = 0; @@ -383,7 +389,7 @@ static char *one_input_line(const char *zPrior, FILE *in){ char *zPrompt; char *zResult; if( in!=0 ){ - return local_getline(0, in); + return local_getline(0, in, 0); } if( zPrior && zPrior[0] ){ zPrompt = continuePrompt; @@ -415,6 +421,7 @@ struct callback_data { int statsOn; /* True to display memory stats before each finalize */ int cnt; /* Number of records displayed so far */ FILE *out; /* Write results here */ + FILE *traceOut; /* Output for sqlite3_trace() */ int nErr; /* Number of errors seen */ int mode; /* An output mode setting */ int writableSchema; /* True if PRAGMA writable_schema=ON */ @@ -492,7 +499,7 @@ static void output_hex_blob(FILE *out, const void *pBlob, int nBlob){ int i; char *zBlob = (char *)pBlob; fprintf(out,"X'"); - for(i=0; i<nBlob; i++){ fprintf(out,"%02x",zBlob[i]); } + for(i=0; i<nBlob; i++){ fprintf(out,"%02x",zBlob[i]&0xff); } fprintf(out,"'"); } @@ -614,8 +621,7 @@ static const char needCsvQuote[] = { /* ** Output a single term of CSV. Actually, p->separator is used for ** the separator, which may or may not be a comma. p->nullvalue is -** the null value. Strings are quoted using ANSI-C rules. Numbers -** appear outside of quotes. +** the null value. Strings are quoted if necessary. */ static void output_csv(struct callback_data *p, const char *z, int bSep){ FILE *out = p->out; @@ -934,11 +940,14 @@ static char *appendText(char *zIn, char const *zAppend, char quote){ /* -** Execute a query statement that has a single result column. Print -** that result column on a line by itself with a semicolon terminator. +** Execute a query statement that will generate SQL output. Print +** the result columns, comma-separated, on a line and then add a +** semicolon terminator to the end of that line. ** -** This is used, for example, to show the schema of the database by -** querying the SQLITE_MASTER table. +** If the number of columns is 1 and that column contains text "--" +** then write the semicolon on a separate line. That way, if a +** "--" comment occurs at the end of the statement, the comment +** won't consume the semicolon terminator. */ static int run_table_dump_query( struct callback_data *p, /* Query context */ @@ -947,6 +956,9 @@ static int run_table_dump_query( ){ sqlite3_stmt *pSelect; int rc; + int nResult; + int i; + const char *z; rc = sqlite3_prepare(p->db, zSelect, -1, &pSelect, 0); if( rc!=SQLITE_OK || !pSelect ){ fprintf(p->out, "/**** ERROR: (%d) %s *****/\n", rc, sqlite3_errmsg(p->db)); @@ -954,12 +966,24 @@ static int run_table_dump_query( return rc; } rc = sqlite3_step(pSelect); + nResult = sqlite3_column_count(pSelect); while( rc==SQLITE_ROW ){ if( zFirstRow ){ fprintf(p->out, "%s", zFirstRow); zFirstRow = 0; } - fprintf(p->out, "%s;\n", sqlite3_column_text(pSelect, 0)); + z = (const char*)sqlite3_column_text(pSelect, 0); + fprintf(p->out, "%s", z); + for(i=1; i<nResult; i++){ + fprintf(p->out, ",%s", sqlite3_column_text(pSelect, i)); + } + if( z==0 ) z = ""; + while( z[0] && (z[0]!='-' || z[1]!='-') ) z++; + if( z[0] ){ + fprintf(p->out, "\n;\n"); + }else{ + fprintf(p->out, ";\n"); + } rc = sqlite3_step(pSelect); } rc = sqlite3_finalize(pSelect); @@ -1056,6 +1080,9 @@ static int display_stats( sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_MISS, &iCur, &iHiwtr, 1); fprintf(pArg->out, "Page cache misses: %d\n", iCur); iHiwtr = iCur = -1; + sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_WRITE, &iCur, &iHiwtr, 1); + fprintf(pArg->out, "Page cache writes: %d\n", iCur); + iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_SCHEMA_USED, &iCur, &iHiwtr, bReset); fprintf(pArg->out, "Schema Heap Usage: %d bytes\n", iCur); iHiwtr = iCur = -1; @@ -1127,6 +1154,15 @@ static int shell_exec( fprintf(pArg->out, "%s\n", zStmtSql ? zStmtSql : zSql); } + /* Output TESTCTRL_EXPLAIN text of requested */ + if( pArg && pArg->mode==MODE_Explain ){ + const char *zExplain = 0; + sqlite3_test_control(SQLITE_TESTCTRL_EXPLAIN_STMT, pStmt, &zExplain); + if( zExplain && zExplain[0] ){ + fprintf(pArg->out, "%s", zExplain); + } + } + /* perform the first step. this will tell us if we ** have a result set or not and how wide it is. */ @@ -1269,9 +1305,12 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azCol){ } zSelect = appendText(zSelect, "SELECT 'INSERT INTO ' || ", 0); + /* Always quote the table name, even if it appears to be pure ascii, + ** in case it is a keyword. Ex: INSERT INTO "table" ... */ zTmp = appendText(zTmp, zTable, '"'); if( zTmp ){ zSelect = appendText(zSelect, zTmp, '\''); + free(zTmp); } zSelect = appendText(zSelect, " || ' VALUES(' || ", 0); rc = sqlite3_step(pTableInfo); @@ -1281,7 +1320,7 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azCol){ zSelect = appendText(zSelect, zText, '"'); rc = sqlite3_step(pTableInfo); if( rc==SQLITE_ROW ){ - zSelect = appendText(zSelect, ") || ',' || ", 0); + zSelect = appendText(zSelect, "), ", 0); }else{ zSelect = appendText(zSelect, ") ", 0); } @@ -1300,7 +1339,7 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azCol){ zSelect = appendText(zSelect, " ORDER BY rowid DESC", 0); run_table_dump_query(p, zSelect, 0); } - if( zSelect ) free(zSelect); + free(zSelect); } return 0; } @@ -1330,7 +1369,7 @@ static int run_schema_dump_query( } zQ2 = malloc( len+100 ); if( zQ2==0 ) return rc; - sqlite3_snprintf(sizeof(zQ2), zQ2, "%s ORDER BY rowid DESC", zQuery); + sqlite3_snprintf(len+100, zQ2, "%s ORDER BY rowid DESC", zQuery); rc = sqlite3_exec(p->db, zQ2, dump_callback, p, &zErr); if( rc ){ fprintf(p->out, "/****** ERROR: %s ******/\n", zErr); @@ -1396,6 +1435,8 @@ static char zHelp[] = " If TABLE specified, only list tables matching\n" " LIKE pattern TABLE.\n" ".timeout MS Try opening locked tables for MS milliseconds\n" + ".trace FILE|off Output each SQL statement as it is run\n" + ".vfsname ?AUX? Print the name of the VFS stack\n" ".width NUM1 NUM2 ... Set column widths for \"column\" mode\n" ; @@ -1485,6 +1526,52 @@ static int booleanValue(char *zArg){ } /* +** Close an output file, assuming it is not stderr or stdout +*/ +static void output_file_close(FILE *f){ + if( f && f!=stdout && f!=stderr ) fclose(f); +} + +/* +** Try to open an output file. The names "stdout" and "stderr" are +** recognized and do the right thing. NULL is returned if the output +** filename is "off". +*/ +static FILE *output_file_open(const char *zFile){ + FILE *f; + if( strcmp(zFile,"stdout")==0 ){ + f = stdout; + }else if( strcmp(zFile, "stderr")==0 ){ + f = stderr; + }else if( strcmp(zFile, "off")==0 ){ + f = 0; + }else{ + f = fopen(zFile, "wb"); + if( f==0 ){ + fprintf(stderr, "Error: cannot open \"%s\"\n", zFile); + } + } + return f; +} + +/* +** A routine for handling output from sqlite3_trace(). +*/ +static void sql_trace_callback(void *pArg, const char *z){ + FILE *f = (FILE*)pArg; + if( f ) fprintf(f, "%s\n", z); +} + +/* +** A no-op routine that runs with the ".breakpoint" doc-command. This is +** a useful spot to set a debugger breakpoint. +*/ +static void test_breakpoint(void){ + static int nCall = 0; + nCall++; +} + +/* ** If an input line begins with "." then invoke this routine to ** process that line. ** @@ -1563,6 +1650,13 @@ static int do_meta_command(char *zLine, struct callback_data *p){ bail_on_error = booleanValue(azArg[1]); }else + /* The undocumented ".breakpoint" command causes a call to the no-op + ** routine named test_breakpoint(). + */ + if( c=='b' && n>=3 && strncmp(azArg[0], "breakpoint", n)==0 ){ + test_breakpoint(); + }else + if( c=='d' && n>1 && strncmp(azArg[0], "databases", n)==0 && nArg==1 ){ struct callback_data data; char *zErrMsg = 0; @@ -1759,12 +1853,15 @@ static int do_meta_command(char *zLine, struct callback_data *p){ } sqlite3_exec(p->db, "BEGIN", 0, 0, 0); zCommit = "COMMIT"; - while( (zLine = local_getline(0, in))!=0 ){ - char *z; + while( (zLine = local_getline(0, in, 1))!=0 ){ + char *z, c; + int inQuote = 0; lineno++; azCol[0] = zLine; - for(i=0, z=zLine; *z && *z!='\n' && *z!='\r'; z++){ - if( *z==p->separator[0] && strncmp(z, p->separator, nSep)==0 ){ + for(i=0, z=zLine; (c = *z)!=0; z++){ + if( c=='"' ) inQuote = !inQuote; + if( c=='\n' ) lineno++; + if( !inQuote && c==p->separator[0] && strncmp(z,p->separator,nSep)==0 ){ *z = 0; i++; if( i<nCol ){ @@ -1784,6 +1881,14 @@ static int do_meta_command(char *zLine, struct callback_data *p){ break; /* from while */ } for(i=0; i<nCol; i++){ + if( azCol[i][0]=='"' ){ + int k; + for(z=azCol[i], j=1, k=0; z[j]; j++){ + if( z[j]=='"' ){ j++; if( z[j]==0 ) break; } + z[k++] = z[j]; + } + z[k] = 0; + } sqlite3_bind_text(pStmt, i+1, azCol[i], -1, SQLITE_STATIC); } sqlite3_step(pStmt); @@ -1883,22 +1988,8 @@ static int do_meta_command(char *zLine, struct callback_data *p){ if( c=='l' && strncmp(azArg[0], "log", n)==0 && nArg>=2 ){ const char *zFile = azArg[1]; - if( p->pLog && p->pLog!=stdout && p->pLog!=stderr ){ - fclose(p->pLog); - p->pLog = 0; - } - if( strcmp(zFile,"stdout")==0 ){ - p->pLog = stdout; - }else if( strcmp(zFile, "stderr")==0 ){ - p->pLog = stderr; - }else if( strcmp(zFile, "off")==0 ){ - p->pLog = 0; - }else{ - p->pLog = fopen(zFile, "w"); - if( p->pLog==0 ){ - fprintf(stderr, "Error: cannot open \"%s\"\n", zFile); - } - } + output_file_close(p->pLog); + p->pLog = output_file_open(zFile); }else if( c=='m' && strncmp(azArg[0], "mode", n)==0 && nArg==2 ){ @@ -1951,20 +2042,31 @@ static int do_meta_command(char *zLine, struct callback_data *p){ }else if( c=='o' && strncmp(azArg[0], "output", n)==0 && nArg==2 ){ - if( p->out!=stdout ){ - fclose(p->out); + if( p->outfile[0]=='|' ){ + pclose(p->out); + }else{ + output_file_close(p->out); } - if( strcmp(azArg[1],"stdout")==0 ){ - p->out = stdout; - sqlite3_snprintf(sizeof(p->outfile), p->outfile, "stdout"); + p->outfile[0] = 0; + if( azArg[1][0]=='|' ){ + p->out = popen(&azArg[1][1], "w"); + if( p->out==0 ){ + fprintf(stderr,"Error: cannot open pipe \"%s\"\n", &azArg[1][1]); + p->out = stdout; + rc = 1; + }else{ + sqlite3_snprintf(sizeof(p->outfile), p->outfile, "%s", azArg[1]); + } }else{ - p->out = fopen(azArg[1], "wb"); + p->out = output_file_open(azArg[1]); if( p->out==0 ){ - fprintf(stderr,"Error: cannot write to \"%s\"\n", azArg[1]); + if( strcmp(azArg[1],"off")!=0 ){ + fprintf(stderr,"Error: cannot write to \"%s\"\n", azArg[1]); + } p->out = stdout; rc = 1; } else { - sqlite3_snprintf(sizeof(p->outfile), p->outfile, "%s", azArg[1]); + sqlite3_snprintf(sizeof(p->outfile), p->outfile, "%s", azArg[1]); } } }else @@ -2082,22 +2184,25 @@ static int do_meta_command(char *zLine, struct callback_data *p){ zShellStatic = azArg[1]; rc = sqlite3_exec(p->db, "SELECT sql FROM " - " (SELECT sql sql, type type, tbl_name tbl_name, name name" + " (SELECT sql sql, type type, tbl_name tbl_name, name name, rowid x" " FROM sqlite_master UNION ALL" - " SELECT sql, type, tbl_name, name FROM sqlite_temp_master) " - "WHERE tbl_name LIKE shellstatic() AND type!='meta' AND sql NOTNULL " - "ORDER BY substr(type,2,1), name", + " SELECT sql, type, tbl_name, name, rowid FROM sqlite_temp_master) " + "WHERE lower(tbl_name) LIKE shellstatic()" + " AND type!='meta' AND sql NOTNULL " + "ORDER BY substr(type,2,1), " + " CASE type WHEN 'view' THEN rowid ELSE name END", callback, &data, &zErrMsg); zShellStatic = 0; } }else{ rc = sqlite3_exec(p->db, "SELECT sql FROM " - " (SELECT sql sql, type type, tbl_name tbl_name, name name" + " (SELECT sql sql, type type, tbl_name tbl_name, name name, rowid x" " FROM sqlite_master UNION ALL" - " SELECT sql, type, tbl_name, name FROM sqlite_temp_master) " + " SELECT sql, type, tbl_name, name, rowid FROM sqlite_temp_master) " "WHERE type!='meta' AND sql NOTNULL AND name NOT LIKE 'sqlite_%'" - "ORDER BY substr(type,2,1), name", + "ORDER BY substr(type,2,1)," + " CASE type WHEN 'view' THEN rowid ELSE name END", callback, &data, &zErrMsg ); } @@ -2145,46 +2250,71 @@ static int do_meta_command(char *zLine, struct callback_data *p){ }else if( c=='t' && n>1 && strncmp(azArg[0], "tables", n)==0 && nArg<3 ){ + sqlite3_stmt *pStmt; char **azResult; - int nRow; - char *zErrMsg; + int nRow, nAlloc; + char *zSql = 0; + int ii; open_db(p); - if( nArg==1 ){ - rc = sqlite3_get_table(p->db, - "SELECT name FROM sqlite_master " - "WHERE type IN ('table','view') AND name NOT LIKE 'sqlite_%' " - "UNION ALL " - "SELECT name FROM sqlite_temp_master " - "WHERE type IN ('table','view') " - "ORDER BY 1", - &azResult, &nRow, 0, &zErrMsg - ); - }else{ - zShellStatic = azArg[1]; - rc = sqlite3_get_table(p->db, - "SELECT name FROM sqlite_master " - "WHERE type IN ('table','view') AND name LIKE shellstatic() " - "UNION ALL " - "SELECT name FROM sqlite_temp_master " - "WHERE type IN ('table','view') AND name LIKE shellstatic() " - "ORDER BY 1", - &azResult, &nRow, 0, &zErrMsg - ); - zShellStatic = 0; + rc = sqlite3_prepare_v2(p->db, "PRAGMA database_list", -1, &pStmt, 0); + if( rc ) return rc; + zSql = sqlite3_mprintf( + "SELECT name FROM sqlite_master" + " WHERE type IN ('table','view')" + " AND name NOT LIKE 'sqlite_%%'" + " AND name LIKE ?1"); + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + const char *zDbName = (const char*)sqlite3_column_text(pStmt, 1); + if( zDbName==0 || strcmp(zDbName,"main")==0 ) continue; + if( strcmp(zDbName,"temp")==0 ){ + zSql = sqlite3_mprintf( + "%z UNION ALL " + "SELECT 'temp.' || name FROM sqlite_temp_master" + " WHERE type IN ('table','view')" + " AND name NOT LIKE 'sqlite_%%'" + " AND name LIKE ?1", zSql); + }else{ + zSql = sqlite3_mprintf( + "%z UNION ALL " + "SELECT '%q.' || name FROM \"%w\".sqlite_master" + " WHERE type IN ('table','view')" + " AND name NOT LIKE 'sqlite_%%'" + " AND name LIKE ?1", zSql, zDbName, zDbName); + } } - if( zErrMsg ){ - fprintf(stderr,"Error: %s\n", zErrMsg); - sqlite3_free(zErrMsg); - rc = 1; - }else if( rc != SQLITE_OK ){ - fprintf(stderr,"Error: querying sqlite_master and sqlite_temp_master\n"); - rc = 1; + sqlite3_finalize(pStmt); + zSql = sqlite3_mprintf("%z ORDER BY 1", zSql); + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + if( rc ) return rc; + nRow = nAlloc = 0; + azResult = 0; + if( nArg>1 ){ + sqlite3_bind_text(pStmt, 1, azArg[1], -1, SQLITE_TRANSIENT); }else{ + sqlite3_bind_text(pStmt, 1, "%", -1, SQLITE_STATIC); + } + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + if( nRow>=nAlloc ){ + char **azNew; + int n = nAlloc*2 + 10; + azNew = sqlite3_realloc(azResult, sizeof(azResult[0])*n); + if( azNew==0 ){ + fprintf(stderr, "Error: out of memory\n"); + break; + } + nAlloc = n; + azResult = azNew; + } + azResult[nRow] = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 0)); + if( azResult[nRow] ) nRow++; + } + sqlite3_finalize(pStmt); + if( nRow>0 ){ int len, maxlen = 0; int i, j; int nPrintCol, nPrintRow; - for(i=1; i<=nRow; i++){ - if( azResult[i]==0 ) continue; + for(i=0; i<nRow; i++){ len = strlen30(azResult[i]); if( len>maxlen ) maxlen = len; } @@ -2192,14 +2322,15 @@ static int do_meta_command(char *zLine, struct callback_data *p){ if( nPrintCol<1 ) nPrintCol = 1; nPrintRow = (nRow + nPrintCol - 1)/nPrintCol; for(i=0; i<nPrintRow; i++){ - for(j=i+1; j<=nRow; j+=nPrintRow){ - char *zSp = j<=nPrintRow ? "" : " "; + for(j=i; j<nRow; j+=nPrintRow){ + char *zSp = j<nPrintRow ? "" : " "; printf("%s%-*s", zSp, maxlen, azResult[j] ? azResult[j] : ""); } printf("\n"); } } - sqlite3_free_table(azResult); + for(ii=0; ii<nRow; ii++) sqlite3_free(azResult[ii]); + sqlite3_free(azResult); }else if( c=='t' && n>=8 && strncmp(azArg[0], "testctrl", n)==0 && nArg>=2 ){ @@ -2219,7 +2350,6 @@ static int do_meta_command(char *zLine, struct callback_data *p){ { "reserve", SQLITE_TESTCTRL_RESERVE }, { "optimizations", SQLITE_TESTCTRL_OPTIMIZATIONS }, { "iskeyword", SQLITE_TESTCTRL_ISKEYWORD }, - { "pghdrsz", SQLITE_TESTCTRL_PGHDRSZ }, { "scratchmalloc", SQLITE_TESTCTRL_SCRATCHMALLOC }, }; int testctrl = -1; @@ -2264,7 +2394,6 @@ static int do_meta_command(char *zLine, struct callback_data *p){ case SQLITE_TESTCTRL_PRNG_SAVE: case SQLITE_TESTCTRL_PRNG_RESTORE: case SQLITE_TESTCTRL_PRNG_RESET: - case SQLITE_TESTCTRL_PGHDRSZ: if( nArg==2 ){ rc = sqlite3_test_control(testctrl); printf("%d (0x%08x)\n", rc, rc); @@ -2335,11 +2464,36 @@ static int do_meta_command(char *zLine, struct callback_data *p){ enableTimer = booleanValue(azArg[1]); }else + if( c=='t' && strncmp(azArg[0], "trace", n)==0 && nArg>1 ){ + open_db(p); + output_file_close(p->traceOut); + p->traceOut = output_file_open(azArg[1]); +#ifndef SQLITE_OMIT_TRACE + if( p->traceOut==0 ){ + sqlite3_trace(p->db, 0, 0); + }else{ + sqlite3_trace(p->db, sql_trace_callback, p->traceOut); + } +#endif + }else + if( c=='v' && strncmp(azArg[0], "version", n)==0 ){ - printf("SQLite %s %s\n", + printf("SQLite %s %s\n" /*extra-version-info*/, sqlite3_libversion(), sqlite3_sourceid()); }else + if( c=='v' && strncmp(azArg[0], "vfsname", n)==0 ){ + const char *zDbName = nArg==2 ? azArg[1] : "main"; + char *zVfsName = 0; + if( p->db ){ + sqlite3_file_control(p->db, zDbName, SQLITE_FCNTL_VFSNAME, &zVfsName); + if( zVfsName ){ + printf("%s\n", zVfsName); + sqlite3_free(zVfsName); + } + } + }else + if( c=='w' && strncmp(azArg[0], "width", n)==0 && nArg>1 ){ int j; assert( nArg<=ArraySize(azArg) ); @@ -2447,7 +2601,9 @@ static int process_input(struct callback_data *p, FILE *in){ free(zLine); zLine = one_input_line(zSql, in); if( zLine==0 ){ - break; /* We have reached EOF */ + /* End of input */ + if( stdin_is_interactive ) printf("\n"); + break; } if( seenInterrupt ){ if( in!=0 ) break; @@ -2534,12 +2690,11 @@ static int process_input(struct callback_data *p, FILE *in){ /* ** Return a pathname which is the user's home directory. A -** 0 return indicates an error of some kind. Space to hold the -** resulting string is obtained from malloc(). The calling -** function should free the result. +** 0 return indicates an error of some kind. */ static char *find_home_dir(void){ - char *home_dir = NULL; + static char *home_dir = NULL; + if( home_dir ) return home_dir; #if !defined(_WIN32) && !defined(WIN32) && !defined(__OS2__) && !defined(_WIN32_WCE) && !defined(__RTP__) && !defined(_WRS_KERNEL) struct passwd *pwent; @@ -2552,7 +2707,7 @@ static char *find_home_dir(void){ #if defined(_WIN32_WCE) /* Windows CE (arm-wince-mingw32ce-gcc) does not provide getenv() */ - home_dir = strdup("/"); + home_dir = "/"; #else #if defined(_WIN32) || defined(WIN32) || defined(__OS2__) @@ -2608,7 +2763,6 @@ static int process_sqliterc( const char *sqliterc = sqliterc_override; char *zBuf = 0; FILE *in = NULL; - int nBuf; int rc = 0; if (sqliterc == NULL) { @@ -2619,15 +2773,8 @@ static int process_sqliterc( #endif return 1; } - nBuf = strlen30(home_dir) + 16; - zBuf = malloc( nBuf ); - if( zBuf==0 ){ - fprintf(stderr,"%s: Error: out of memory\n",Argv0); - return 1; - } - sqlite3_snprintf(nBuf, zBuf,"%s/.sqliterc",home_dir); - free(home_dir); - sqliterc = (const char*)zBuf; + zBuf = sqlite3_mprintf("%s/.sqliterc",home_dir); + sqliterc = zBuf; } in = fopen(sqliterc,"rb"); if( in ){ @@ -2637,7 +2784,7 @@ static int process_sqliterc( rc = process_input(p,in); fclose(in); } - free(zBuf); + sqlite3_free(zBuf); return rc; } @@ -2645,29 +2792,30 @@ static int process_sqliterc( ** Show available command line options */ static const char zOptions[] = - " -help show this message\n" - " -init filename read/process named file\n" - " -echo print commands before execution\n" - " -[no]header turn headers on or off\n" " -bail stop after hitting an error\n" - " -interactive force interactive I/O\n" " -batch force batch I/O\n" " -column set output mode to 'column'\n" + " -cmd command run \"command\" before reading stdin\n" " -csv set output mode to 'csv'\n" + " -echo print commands before execution\n" + " -init filename read/process named file\n" + " -[no]header turn headers on or off\n" + " -help show this message\n" " -html set output mode to HTML\n" + " -interactive force interactive I/O\n" " -line set output mode to 'line'\n" " -list set output mode to 'list'\n" +#ifdef SQLITE_ENABLE_MULTIPLEX + " -multiplex enable the multiplexor VFS\n" +#endif + " -nullvalue 'text' set text string for NULL values\n" " -separator 'x' set output field separator (|)\n" " -stats print memory stats before each finalize\n" - " -nullvalue 'text' set text string for NULL values\n" " -version show SQLite version\n" " -vfs NAME use NAME as the default VFS\n" #ifdef SQLITE_ENABLE_VFSTRACE " -vfstrace enable tracing of all VFS calls\n" #endif -#ifdef SQLITE_ENABLE_MULTIPLEX - " -multiplex enable the multiplexor VFS\n" -#endif ; static void usage(int showDetail){ fprintf(stderr, @@ -2730,19 +2878,22 @@ int main(int argc, char **argv){ char *z; if( argv[i][0]!='-' ) break; z = argv[i]; - if( z[0]=='-' && z[1]=='-' ) z++; - if( strcmp(argv[i],"-separator")==0 || strcmp(argv[i],"-nullvalue")==0 ){ + if( z[1]=='-' ) z++; + if( strcmp(z,"-separator")==0 + || strcmp(z,"-nullvalue")==0 + || strcmp(z,"-cmd")==0 + ){ i++; - }else if( strcmp(argv[i],"-init")==0 ){ + }else if( strcmp(z,"-init")==0 ){ i++; zInitFile = argv[i]; /* Need to check for batch mode here to so we can avoid printing ** informational messages (like from process_sqliterc) before ** we do the actual processing of arguments later in a second pass. */ - }else if( strcmp(argv[i],"-batch")==0 ){ + }else if( strcmp(z,"-batch")==0 ){ stdin_is_interactive = 0; - }else if( strcmp(argv[i],"-heap")==0 ){ + }else if( strcmp(z,"-heap")==0 ){ #if defined(SQLITE_ENABLE_MEMSYS3) || defined(SQLITE_ENABLE_MEMSYS5) int j, c; const char *zSize; @@ -2759,7 +2910,7 @@ int main(int argc, char **argv){ sqlite3_config(SQLITE_CONFIG_HEAP, malloc((int)szHeap), (int)szHeap, 64); #endif #ifdef SQLITE_ENABLE_VFSTRACE - }else if( strcmp(argv[i],"-vfstrace")==0 ){ + }else if( strcmp(z,"-vfstrace")==0 ){ extern int vfstrace_register( const char *zTraceName, const char *zOldVfsName, @@ -2770,11 +2921,11 @@ int main(int argc, char **argv){ vfstrace_register("trace",0,(int(*)(const char*,void*))fputs,stderr,1); #endif #ifdef SQLITE_ENABLE_MULTIPLEX - }else if( strcmp(argv[i],"-multiplex")==0 ){ + }else if( strcmp(z,"-multiplex")==0 ){ extern int sqlite3_multiple_initialize(const char*,int); sqlite3_multiplex_initialize(0, 1); #endif - }else if( strcmp(argv[i],"-vfs")==0 ){ + }else if( strcmp(z,"-vfs")==0 ){ sqlite3_vfs *pVfs = sqlite3_vfs_find(argv[++i]); if( pVfs ){ sqlite3_vfs_register(pVfs, 1); @@ -2856,7 +3007,8 @@ int main(int argc, char **argv){ }else if( strcmp(z,"-separator")==0 ){ i++; if(i>=argc){ - fprintf(stderr,"%s: Error: missing argument for option: %s\n", Argv0, z); + fprintf(stderr,"%s: Error: missing argument for option: %s\n", + Argv0, z); fprintf(stderr,"Use -help for a list of options.\n"); return 1; } @@ -2865,7 +3017,8 @@ int main(int argc, char **argv){ }else if( strcmp(z,"-nullvalue")==0 ){ i++; if(i>=argc){ - fprintf(stderr,"%s: Error: missing argument for option: %s\n", Argv0, z); + fprintf(stderr,"%s: Error: missing argument for option: %s\n", + Argv0, z); fprintf(stderr,"Use -help for a list of options.\n"); return 1; } @@ -2900,8 +3053,26 @@ int main(int argc, char **argv){ }else if( strcmp(z,"-multiplex")==0 ){ i++; #endif - }else if( strcmp(z,"-help")==0 || strcmp(z, "--help")==0 ){ + }else if( strcmp(z,"-help")==0 ){ usage(1); + }else if( strcmp(z,"-cmd")==0 ){ + if( i==argc-1 ) break; + i++; + z = argv[i]; + if( z[0]=='.' ){ + rc = do_meta_command(z, &data); + if( rc && bail_on_error ) return rc; + }else{ + open_db(&data); + rc = shell_exec(data.db, z, shell_callback, &data, &zErrMsg); + if( zErrMsg!=0 ){ + fprintf(stderr,"Error: %s\n", zErrMsg); + if( bail_on_error ) return rc!=0 ? rc : 1; + }else if( rc!=0 ){ + fprintf(stderr,"Error: unable to process SQL \"%s\"\n", z); + if( bail_on_error ) return rc; + } + } }else{ fprintf(stderr,"%s: Error: unknown option: %s\n", Argv0, z); fprintf(stderr,"Use -help for a list of options.\n"); @@ -2933,7 +3104,7 @@ int main(int argc, char **argv){ char *zHistory = 0; int nHistory; printf( - "SQLite version %s %.19s\n" + "SQLite version %s %.19s\n" /*extra-version-info*/ "Enter \".help\" for instructions\n" "Enter SQL statements terminated with a \";\"\n", sqlite3_libversion(), sqlite3_sourceid() @@ -2954,7 +3125,6 @@ int main(int argc, char **argv){ write_history(zHistory); free(zHistory); } - free(zHome); }else{ rc = process_input(&data, stdin); } diff --git a/src/sqlite.h.in b/src/sqlite.h.in index ed18330..29355d7 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -172,7 +172,7 @@ const char *sqlite3_compileoption_get(int N); ** CAPI3REF: Test To See If The Library Is Threadsafe ** ** ^The sqlite3_threadsafe() function returns zero if and only if -** SQLite was compiled mutexing code omitted due to the +** SQLite was compiled with mutexing code omitted due to the ** [SQLITE_THREADSAFE] compile-time option being set to 0. ** ** SQLite can be compiled with or without mutexes. When @@ -366,7 +366,7 @@ int sqlite3_exec( ** KEYWORDS: {result code} {result codes} ** ** Many SQLite functions return an integer result code from the set shown -** here in order to indicates success or failure. +** here in order to indicate success or failure. ** ** New error codes may be added in future versions of SQLite. ** @@ -453,9 +453,11 @@ int sqlite3_exec( #define SQLITE_LOCKED_SHAREDCACHE (SQLITE_LOCKED | (1<<8)) #define SQLITE_BUSY_RECOVERY (SQLITE_BUSY | (1<<8)) #define SQLITE_CANTOPEN_NOTEMPDIR (SQLITE_CANTOPEN | (1<<8)) +#define SQLITE_CANTOPEN_ISDIR (SQLITE_CANTOPEN | (2<<8)) #define SQLITE_CORRUPT_VTAB (SQLITE_CORRUPT | (1<<8)) #define SQLITE_READONLY_RECOVERY (SQLITE_READONLY | (1<<8)) #define SQLITE_READONLY_CANTLOCK (SQLITE_READONLY | (2<<8)) +#define SQLITE_ABORT_ROLLBACK (SQLITE_ABORT | (2<<8)) /* ** CAPI3REF: Flags For File Open Operations @@ -504,7 +506,11 @@ int sqlite3_exec( ** first then the size of the file is extended, never the other ** way around. The SQLITE_IOCAP_SEQUENTIAL property means that ** information is written to disk in the same order as calls -** to xWrite(). +** to xWrite(). The SQLITE_IOCAP_POWERSAFE_OVERWRITE property means that +** after reboot following a crash or power loss, the only bytes in a +** file that were written at the application level might have changed +** and that adjacent bytes, even bytes within the same sector are +** guaranteed to be unchanged. */ #define SQLITE_IOCAP_ATOMIC 0x00000001 #define SQLITE_IOCAP_ATOMIC512 0x00000002 @@ -518,6 +524,7 @@ int sqlite3_exec( #define SQLITE_IOCAP_SAFE_APPEND 0x00000200 #define SQLITE_IOCAP_SEQUENTIAL 0x00000400 #define SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN 0x00000800 +#define SQLITE_IOCAP_POWERSAFE_OVERWRITE 0x00001000 /* ** CAPI3REF: File Locking Levels @@ -706,7 +713,8 @@ struct sqlite3_io_methods { ** into an integer that the pArg argument points to. This capability ** is used during testing and only needs to be supported when SQLITE_TEST ** is defined. -** +** <ul> +** <li>[[SQLITE_FCNTL_SIZE_HINT]] ** The [SQLITE_FCNTL_SIZE_HINT] opcode is used by SQLite to give the VFS ** layer a hint of how large the database file will grow to be during the ** current transaction. This hint is not guaranteed to be accurate but it @@ -714,6 +722,7 @@ struct sqlite3_io_methods { ** file space based on this hint in order to help writes to the database ** file run faster. ** +** <li>[[SQLITE_FCNTL_CHUNK_SIZE]] ** The [SQLITE_FCNTL_CHUNK_SIZE] opcode is used to request that the VFS ** extends and truncates the database file in chunks of a size specified ** by the user. The fourth argument to [sqlite3_file_control()] should @@ -722,11 +731,13 @@ struct sqlite3_io_methods { ** chunks (say 1MB at a time), may reduce file-system fragmentation and ** improve performance on some systems. ** +** <li>[[SQLITE_FCNTL_FILE_POINTER]] ** The [SQLITE_FCNTL_FILE_POINTER] opcode is used to obtain a pointer ** to the [sqlite3_file] object associated with a particular database ** connection. See the [sqlite3_file_control()] documentation for ** additional information. ** +** <li>[[SQLITE_FCNTL_SYNC_OMITTED]] ** ^(The [SQLITE_FCNTL_SYNC_OMITTED] opcode is generated internally by ** SQLite and sent to all VFSes in place of a call to the xSync method ** when the database connection has [PRAGMA synchronous] set to OFF.)^ @@ -737,14 +748,15 @@ struct sqlite3_io_methods { ** opcode as doing so may disrupt the operation of the specialized VFSes ** that do require it. ** +** <li>[[SQLITE_FCNTL_WIN32_AV_RETRY]] ** ^The [SQLITE_FCNTL_WIN32_AV_RETRY] opcode is used to configure automatic ** retry counts and intervals for certain disk I/O operations for the -** windows [VFS] in order to work to provide robustness against +** windows [VFS] in order to provide robustness in the presence of ** anti-virus programs. By default, the windows VFS will retry file read, ** file write, and file delete operations up to 10 times, with a delay ** of 25 milliseconds before the first retry and with the delay increasing ** by an additional 25 milliseconds with each subsequent retry. This -** opcode allows those to values (10 retries and 25 milliseconds of delay) +** opcode allows these two values (10 retries and 25 milliseconds of delay) ** to be adjusted. The values are changed for all database connections ** within the same process. The argument is a pointer to an array of two ** integers where the first integer i the new retry count and the second @@ -753,8 +765,9 @@ struct sqlite3_io_methods { ** into the array entry, allowing the current retry settings to be ** interrogated. The zDbName parameter is ignored. ** +** <li>[[SQLITE_FCNTL_PERSIST_WAL]] ** ^The [SQLITE_FCNTL_PERSIST_WAL] opcode is used to set or query the -** persistent [WAL | Write AHead Log] setting. By default, the auxiliary +** persistent [WAL | Write Ahead Log] setting. By default, the auxiliary ** write ahead log and shared memory files used for transaction control ** are automatically deleted when the latest connection to the database ** closes. Setting persistent WAL mode causes those files to persist after @@ -767,22 +780,72 @@ struct sqlite3_io_methods { ** WAL mode. If the integer is -1, then it is overwritten with the current ** WAL persistence setting. ** +** <li>[[SQLITE_FCNTL_POWERSAFE_OVERWRITE]] +** ^The [SQLITE_FCNTL_POWERSAFE_OVERWRITE] opcode is used to set or query the +** persistent "powersafe-overwrite" or "PSOW" setting. The PSOW setting +** determines the [SQLITE_IOCAP_POWERSAFE_OVERWRITE] bit of the +** xDeviceCharacteristics methods. The fourth parameter to +** [sqlite3_file_control()] for this opcode should be a pointer to an integer. +** That integer is 0 to disable zero-damage mode or 1 to enable zero-damage +** mode. If the integer is -1, then it is overwritten with the current +** zero-damage mode setting. +** +** <li>[[SQLITE_FCNTL_OVERWRITE]] ** ^The [SQLITE_FCNTL_OVERWRITE] opcode is invoked by SQLite after opening ** a write transaction to indicate that, unless it is rolled back for some ** reason, the entire database file will be overwritten by the current ** transaction. This is used by VACUUM operations. +** +** <li>[[SQLITE_FCNTL_VFSNAME]] +** ^The [SQLITE_FCNTL_VFSNAME] opcode can be used to obtain the names of +** all [VFSes] in the VFS stack. The names are of all VFS shims and the +** final bottom-level VFS are written into memory obtained from +** [sqlite3_malloc()] and the result is stored in the char* variable +** that the fourth parameter of [sqlite3_file_control()] points to. +** The caller is responsible for freeing the memory when done. As with +** all file-control actions, there is no guarantee that this will actually +** do anything. Callers should initialize the char* variable to a NULL +** pointer in case this file-control is not implemented. This file-control +** is intended for diagnostic use only. +** +** <li>[[SQLITE_FCNTL_PRAGMA]] +** ^Whenever a [PRAGMA] statement is parsed, an [SQLITE_FCNTL_PRAGMA] +** file control is sent to the open [sqlite3_file] object corresponding +** to the database file to which the pragma statement refers. ^The argument +** to the [SQLITE_FCNTL_PRAGMA] file control is an array of +** pointers to strings (char**) in which the second element of the array +** is the name of the pragma and the third element is the argument to the +** pragma or NULL if the pragma has no argument. ^The handler for an +** [SQLITE_FCNTL_PRAGMA] file control can optionally make the first element +** of the char** argument point to a string obtained from [sqlite3_mprintf()] +** or the equivalent and that string will become the result of the pragma or +** the error message if the pragma fails. ^If the +** [SQLITE_FCNTL_PRAGMA] file control returns [SQLITE_NOTFOUND], then normal +** [PRAGMA] processing continues. ^If the [SQLITE_FCNTL_PRAGMA] +** file control returns [SQLITE_OK], then the parser assumes that the +** VFS has handled the PRAGMA itself and the parser generates a no-op +** prepared statement. ^If the [SQLITE_FCNTL_PRAGMA] file control returns +** any result code other than [SQLITE_OK] or [SQLITE_NOTFOUND], that means +** that the VFS encountered an error while handling the [PRAGMA] and the +** compilation of the PRAGMA fails with an error. ^The [SQLITE_FCNTL_PRAGMA] +** file control occurs at the beginning of pragma statement analysis and so +** it is able to override built-in [PRAGMA] statements. +** </ul> */ -#define SQLITE_FCNTL_LOCKSTATE 1 -#define SQLITE_GET_LOCKPROXYFILE 2 -#define SQLITE_SET_LOCKPROXYFILE 3 -#define SQLITE_LAST_ERRNO 4 -#define SQLITE_FCNTL_SIZE_HINT 5 -#define SQLITE_FCNTL_CHUNK_SIZE 6 -#define SQLITE_FCNTL_FILE_POINTER 7 -#define SQLITE_FCNTL_SYNC_OMITTED 8 -#define SQLITE_FCNTL_WIN32_AV_RETRY 9 -#define SQLITE_FCNTL_PERSIST_WAL 10 -#define SQLITE_FCNTL_OVERWRITE 11 +#define SQLITE_FCNTL_LOCKSTATE 1 +#define SQLITE_GET_LOCKPROXYFILE 2 +#define SQLITE_SET_LOCKPROXYFILE 3 +#define SQLITE_LAST_ERRNO 4 +#define SQLITE_FCNTL_SIZE_HINT 5 +#define SQLITE_FCNTL_CHUNK_SIZE 6 +#define SQLITE_FCNTL_FILE_POINTER 7 +#define SQLITE_FCNTL_SYNC_OMITTED 8 +#define SQLITE_FCNTL_WIN32_AV_RETRY 9 +#define SQLITE_FCNTL_PERSIST_WAL 10 +#define SQLITE_FCNTL_OVERWRITE 11 +#define SQLITE_FCNTL_VFSNAME 12 +#define SQLITE_FCNTL_POWERSAFE_OVERWRITE 13 +#define SQLITE_FCNTL_PRAGMA 14 /* ** CAPI3REF: Mutex Handle @@ -837,7 +900,7 @@ typedef struct sqlite3_mutex sqlite3_mutex; ** from xFullPathname() with an optional suffix added. ** ^If a suffix is added to the zFilename parameter, it will ** consist of a single "-" character followed by no more than -** 10 alphanumeric and/or "-" characters. +** 11 alphanumeric and/or "-" characters. ** ^SQLite further guarantees that ** the string will be valid and unchanged until xClose() is ** called. Because of the previous sentence, @@ -1368,7 +1431,7 @@ struct sqlite3_mem_methods { ** <dd> ^This option specifies a static memory buffer that SQLite can use for ** the database page cache with the default page cache implementation. ** This configuration should not be used if an application-define page -** cache implementation is loaded using the SQLITE_CONFIG_PCACHE option. +** cache implementation is loaded using the SQLITE_CONFIG_PCACHE2 option. ** There are three arguments to this option: A pointer to 8-byte aligned ** memory, the size of each page buffer (sz), and the number of pages (N). ** The sz argument should be the size of the largest database page @@ -1437,15 +1500,15 @@ struct sqlite3_mem_methods { ** verb to [sqlite3_db_config()] can be used to change the lookaside ** configuration on individual connections.)^ </dd> ** -** [[SQLITE_CONFIG_PCACHE]] <dt>SQLITE_CONFIG_PCACHE</dt> +** [[SQLITE_CONFIG_PCACHE2]] <dt>SQLITE_CONFIG_PCACHE2</dt> ** <dd> ^(This option takes a single argument which is a pointer to -** an [sqlite3_pcache_methods] object. This object specifies the interface +** an [sqlite3_pcache_methods2] object. This object specifies the interface ** to a custom page cache implementation.)^ ^SQLite makes a copy of the ** object and uses it for page cache memory allocations.</dd> ** -** [[SQLITE_CONFIG_GETPCACHE]] <dt>SQLITE_CONFIG_GETPCACHE</dt> +** [[SQLITE_CONFIG_GETPCACHE2]] <dt>SQLITE_CONFIG_GETPCACHE2</dt> ** <dd> ^(This option takes a single argument which is a pointer to an -** [sqlite3_pcache_methods] object. SQLite copies of the current +** [sqlite3_pcache_methods2] object. SQLite copies of the current ** page cache implementation into that object.)^ </dd> ** ** [[SQLITE_CONFIG_LOG]] <dt>SQLITE_CONFIG_LOG</dt> @@ -1478,6 +1541,11 @@ struct sqlite3_mem_methods { ** database connection is opened. By default, URI handling is globally ** disabled. The default value may be changed by compiling with the ** [SQLITE_USE_URI] symbol defined. +** +** [[SQLITE_CONFIG_PCACHE]] [[SQLITE_CONFIG_GETPCACHE]] +** <dt>SQLITE_CONFIG_PCACHE and SQLITE_CONFIG_GETPCACHE +** <dd> These options are obsolete and should not be used by new code. +** They are retained for backwards compatibility but are now no-ops. ** </dl> */ #define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ @@ -1493,10 +1561,12 @@ struct sqlite3_mem_methods { #define SQLITE_CONFIG_GETMUTEX 11 /* sqlite3_mutex_methods* */ /* previously SQLITE_CONFIG_CHUNKALLOC 12 which is now unused. */ #define SQLITE_CONFIG_LOOKASIDE 13 /* int int */ -#define SQLITE_CONFIG_PCACHE 14 /* sqlite3_pcache_methods* */ -#define SQLITE_CONFIG_GETPCACHE 15 /* sqlite3_pcache_methods* */ +#define SQLITE_CONFIG_PCACHE 14 /* no-op */ +#define SQLITE_CONFIG_GETPCACHE 15 /* no-op */ #define SQLITE_CONFIG_LOG 16 /* xFunc, void* */ #define SQLITE_CONFIG_URI 17 /* int */ +#define SQLITE_CONFIG_PCACHE2 18 /* sqlite3_pcache_methods2* */ +#define SQLITE_CONFIG_GETPCACHE2 19 /* sqlite3_pcache_methods2* */ /* ** CAPI3REF: Database Connection Configuration Options @@ -1981,7 +2051,7 @@ void sqlite3_free_table(char **result); ** All of the usual printf() formatting options apply. In addition, there ** is are "%q", "%Q", and "%z" options. ** -** ^(The %q option works like %s in that it substitutes a null-terminated +** ^(The %q option works like %s in that it substitutes a nul-terminated ** string from the argument list. But %q also doubles every '\'' character. ** %q is designed for use inside a string literal.)^ By doubling each '\'' ** character it escapes that character and allows it to be inserted into @@ -2589,21 +2659,45 @@ int sqlite3_open_v2( /* ** CAPI3REF: Obtain Values For URI Parameters ** -** This is a utility routine, useful to VFS implementations, that checks +** These are utility routines, useful to VFS implementations, that check ** to see if a database file was a URI that contained a specific query -** parameter, and if so obtains the value of the query parameter. -** -** The zFilename argument is the filename pointer passed into the xOpen() -** method of a VFS implementation. The zParam argument is the name of the -** query parameter we seek. This routine returns the value of the zParam -** parameter if it exists. If the parameter does not exist, this routine -** returns a NULL pointer. -** -** If the zFilename argument to this function is not a pointer that SQLite -** passed into the xOpen VFS method, then the behavior of this routine -** is undefined and probably undesirable. +** parameter, and if so obtains the value of that query parameter. +** +** If F is the database filename pointer passed into the xOpen() method of +** a VFS implementation when the flags parameter to xOpen() has one or +** more of the [SQLITE_OPEN_URI] or [SQLITE_OPEN_MAIN_DB] bits set and +** P is the name of the query parameter, then +** sqlite3_uri_parameter(F,P) returns the value of the P +** parameter if it exists or a NULL pointer if P does not appear as a +** query parameter on F. If P is a query parameter of F +** has no explicit value, then sqlite3_uri_parameter(F,P) returns +** a pointer to an empty string. +** +** The sqlite3_uri_boolean(F,P,B) routine assumes that P is a boolean +** parameter and returns true (1) or false (0) according to the value +** of P. The sqlite3_uri_boolean(F,P,B) routine returns true (1) if the +** value of query parameter P is one of "yes", "true", or "on" in any +** case or if the value begins with a non-zero number. The +** sqlite3_uri_boolean(F,P,B) routines returns false (0) if the value of +** query parameter P is one of "no", "false", or "off" in any case or +** if the value begins with a numeric zero. If P is not a query +** parameter on F or if the value of P is does not match any of the +** above, then sqlite3_uri_boolean(F,P,B) returns (B!=0). +** +** The sqlite3_uri_int64(F,P,D) routine converts the value of P into a +** 64-bit signed integer and returns that integer, or D if P does not +** exist. If the value of P is something other than an integer, then +** zero is returned. +** +** If F is a NULL pointer, then sqlite3_uri_parameter(F,P) returns NULL and +** sqlite3_uri_boolean(F,P,B) returns B. If F is not a NULL pointer and +** is not a database file pathname pointer that SQLite passed into the xOpen +** VFS method, then the behavior of this routine is undefined and probably +** undesirable. */ const char *sqlite3_uri_parameter(const char *zFilename, const char *zParam); +int sqlite3_uri_boolean(const char *zFile, const char *zParam, int bDefault); +sqlite3_int64 sqlite3_uri_int64(const char*, const char*, sqlite3_int64); /* @@ -2926,6 +3020,25 @@ const char *sqlite3_sql(sqlite3_stmt *pStmt); int sqlite3_stmt_readonly(sqlite3_stmt *pStmt); /* +** CAPI3REF: Determine If A Prepared Statement Has Been Reset +** +** ^The sqlite3_stmt_busy(S) interface returns true (non-zero) if the +** [prepared statement] S has been stepped at least once using +** [sqlite3_step(S)] but has not run to completion and/or has not +** been reset using [sqlite3_reset(S)]. ^The sqlite3_stmt_busy(S) +** interface returns false if S is a NULL pointer. If S is not a +** NULL pointer and is not a pointer to a valid [prepared statement] +** object, then the behavior is undefined and probably undesirable. +** +** This interface can be used in combination [sqlite3_next_stmt()] +** to locate all prepared statements associated with a database +** connection that are in need of being reset. This can be used, +** for example, in diagnostic routines to search for prepared +** statements that are holding a transaction open. +*/ +int sqlite3_stmt_busy(sqlite3_stmt*); + +/* ** CAPI3REF: Dynamically Typed Value Object ** KEYWORDS: {protected sqlite3_value} {unprotected sqlite3_value} ** @@ -3466,7 +3579,7 @@ int sqlite3_data_count(sqlite3_stmt *pStmt); ** bytes in the string, not the number of characters. ** ** ^Strings returned by sqlite3_column_text() and sqlite3_column_text16(), -** even empty strings, are always zero terminated. ^The return +** even empty strings, are always zero-terminated. ^The return ** value from sqlite3_column_blob() for a zero-length BLOB is a NULL pointer. ** ** ^The object returned by [sqlite3_column_value()] is an @@ -4367,6 +4480,31 @@ int sqlite3_get_autocommit(sqlite3*); sqlite3 *sqlite3_db_handle(sqlite3_stmt*); /* +** CAPI3REF: Return The Filename For A Database Connection +** +** ^The sqlite3_db_filename(D,N) interface returns a pointer to a filename +** associated with database N of connection D. ^The main database file +** has the name "main". If there is no attached database N on the database +** connection D, or if database N is a temporary or in-memory database, then +** a NULL pointer is returned. +** +** ^The filename returned by this function is the output of the +** xFullPathname method of the [VFS]. ^In other words, the filename +** will be an absolute pathname, even if the filename used +** to open the database originally was a URI or relative pathname. +*/ +const char *sqlite3_db_filename(sqlite3 *db, const char *zDbName); + +/* +** CAPI3REF: Determine if a database is read-only +** +** ^The sqlite3_db_readonly(D,N) interface returns 1 if the database N +** of connection D is read-only, 0 if it is read/write, or -1 if N is not +** the name of a database on connection D. +*/ +int sqlite3_db_readonly(sqlite3 *db, const char *zDbName); + +/* ** CAPI3REF: Find the next prepared statement ** ** ^This interface returns a pointer to the next [prepared statement] after @@ -4401,13 +4539,15 @@ sqlite3_stmt *sqlite3_next_stmt(sqlite3 *pDb, sqlite3_stmt *pStmt); ** on the same [database connection] D, or NULL for ** the first call for each function on D. ** +** The commit and rollback hook callbacks are not reentrant. ** The callback implementation must not do anything that will modify ** the database connection that invoked the callback. Any actions ** to modify the database connection must be deferred until after the ** completion of the [sqlite3_step()] call that triggered the commit ** or rollback hook in the first place. -** Note that [sqlite3_prepare_v2()] and [sqlite3_step()] both modify their -** database connections for the meaning of "modify" in this paragraph. +** Note that running any other SQL statements, including SELECT statements, +** or merely calling [sqlite3_prepare_v2()] and [sqlite3_step()] will modify +** the database connections for the meaning of "modify" in this paragraph. ** ** ^Registering a NULL function disables the callback. ** @@ -4520,10 +4660,25 @@ int sqlite3_enable_shared_cache(int); ** which might be more or less than the amount requested. ** ^The sqlite3_release_memory() routine is a no-op returning zero ** if SQLite is not compiled with [SQLITE_ENABLE_MEMORY_MANAGEMENT]. +** +** See also: [sqlite3_db_release_memory()] */ int sqlite3_release_memory(int); /* +** CAPI3REF: Free Memory Used By A Database Connection +** +** ^The sqlite3_db_release_memory(D) interface attempts to free as much heap +** memory as possible from database connection D. Unlike the +** [sqlite3_release_memory()] interface, this interface is effect even +** when then [SQLITE_ENABLE_MEMORY_MANAGEMENT] compile-time option is +** omitted. +** +** See also: [sqlite3_release_memory()] +*/ +int sqlite3_db_release_memory(sqlite3*); + +/* ** CAPI3REF: Impose A Limit On Heap Size ** ** ^The sqlite3_soft_heap_limit64() interface sets and/or queries the @@ -4537,7 +4692,8 @@ int sqlite3_release_memory(int); ** is advisory only. ** ** ^The return value from sqlite3_soft_heap_limit64() is the size of -** the soft heap limit prior to the call. ^If the argument N is negative +** the soft heap limit prior to the call, or negative in the case of an +** error. ^If the argument N is negative ** then no change is made to the soft heap limit. Hence, the current ** size of the soft heap limit can be determined by invoking ** sqlite3_soft_heap_limit64() with a negative argument. @@ -4553,7 +4709,7 @@ int sqlite3_release_memory(int); ** [sqlite3_config]([SQLITE_CONFIG_MEMSTATUS],...) start-time option and ** the [SQLITE_DEFAULT_MEMSTATUS] compile-time option. ** <li> An alternative page cache implementation is specified using -** [sqlite3_config]([SQLITE_CONFIG_PCACHE],...). +** [sqlite3_config]([SQLITE_CONFIG_PCACHE2],...). ** <li> The page cache allocates from its own memory pool supplied ** by [sqlite3_config]([SQLITE_CONFIG_PAGECACHE],...) rather than ** from the heap. @@ -5295,7 +5451,7 @@ int sqlite3_vfs_unregister(sqlite3_vfs*); ** ** <ul> ** <li> SQLITE_MUTEX_OS2 -** <li> SQLITE_MUTEX_PTHREAD +** <li> SQLITE_MUTEX_PTHREADS ** <li> SQLITE_MUTEX_W32 ** <li> SQLITE_MUTEX_NOOP ** </ul>)^ @@ -5303,7 +5459,7 @@ int sqlite3_vfs_unregister(sqlite3_vfs*); ** ^The SQLITE_MUTEX_NOOP implementation is a set of routines ** that does no real locking and is appropriate for use in ** a single-threaded application. ^The SQLITE_MUTEX_OS2, -** SQLITE_MUTEX_PTHREAD, and SQLITE_MUTEX_W32 implementations +** SQLITE_MUTEX_PTHREADS, and SQLITE_MUTEX_W32 implementations ** are appropriate for use on OS/2, Unix, and Windows. ** ** ^(If SQLite is compiled with the SQLITE_MUTEX_APPDEF preprocessor @@ -5493,7 +5649,7 @@ struct sqlite3_mutex_methods { ** ^These routines should return true if the mutex in their argument ** is held or not held, respectively, by the calling thread. ** -** ^The implementation is not required to provided versions of these +** ^The implementation is not required to provide versions of these ** routines that actually work. If the implementation does not provide working ** versions of these routines, it should at least provide stubs that always ** return true so that one does not get spurious assertion failures. @@ -5621,9 +5777,9 @@ int sqlite3_test_control(int op, ...); #define SQLITE_TESTCTRL_RESERVE 14 #define SQLITE_TESTCTRL_OPTIMIZATIONS 15 #define SQLITE_TESTCTRL_ISKEYWORD 16 -#define SQLITE_TESTCTRL_PGHDRSZ 17 -#define SQLITE_TESTCTRL_SCRATCHMALLOC 18 -#define SQLITE_TESTCTRL_LOCALTIME_FAULT 19 +#define SQLITE_TESTCTRL_SCRATCHMALLOC 17 +#define SQLITE_TESTCTRL_LOCALTIME_FAULT 18 +#define SQLITE_TESTCTRL_EXPLAIN_STMT 19 #define SQLITE_TESTCTRL_LAST 19 /* @@ -5846,6 +6002,17 @@ int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg); ** occurred.)^ ^The highwater mark associated with SQLITE_DBSTATUS_CACHE_MISS ** is always 0. ** </dd> +** +** [[SQLITE_DBSTATUS_CACHE_WRITE]] ^(<dt>SQLITE_DBSTATUS_CACHE_WRITE</dt> +** <dd>This parameter returns the number of dirty cache entries that have +** been written to disk. Specifically, the number of pages written to the +** wal file in wal mode databases, or the number of pages written to the +** database file in rollback mode databases. Any pages written as part of +** transaction rollback or database recovery operations are not included. +** If an IO or other error occurs while writing a page to disk, the effect +** on subsequent SQLITE_DBSTATUS_CACHE_WRITE requests is undefined.)^ ^The +** highwater mark associated with SQLITE_DBSTATUS_CACHE_WRITE is always 0. +** </dd> ** </dl> */ #define SQLITE_DBSTATUS_LOOKASIDE_USED 0 @@ -5857,7 +6024,8 @@ int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg); #define SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL 6 #define SQLITE_DBSTATUS_CACHE_HIT 7 #define SQLITE_DBSTATUS_CACHE_MISS 8 -#define SQLITE_DBSTATUS_MAX 8 /* Largest defined DBSTATUS */ +#define SQLITE_DBSTATUS_CACHE_WRITE 9 +#define SQLITE_DBSTATUS_MAX 9 /* Largest defined DBSTATUS */ /* @@ -5926,17 +6094,33 @@ int sqlite3_stmt_status(sqlite3_stmt*, int op,int resetFlg); ** sqlite3_pcache object except by holding and passing pointers ** to the object. ** -** See [sqlite3_pcache_methods] for additional information. +** See [sqlite3_pcache_methods2] for additional information. */ typedef struct sqlite3_pcache sqlite3_pcache; /* +** CAPI3REF: Custom Page Cache Object +** +** The sqlite3_pcache_page object represents a single page in the +** page cache. The page cache will allocate instances of this +** object. Various methods of the page cache use pointers to instances +** of this object as parameters or as their return value. +** +** See [sqlite3_pcache_methods2] for additional information. +*/ +typedef struct sqlite3_pcache_page sqlite3_pcache_page; +struct sqlite3_pcache_page { + void *pBuf; /* The content of the page */ + void *pExtra; /* Extra information associated with the page */ +}; + +/* ** CAPI3REF: Application Defined Page Cache. ** KEYWORDS: {page cache} ** -** ^(The [sqlite3_config]([SQLITE_CONFIG_PCACHE], ...) interface can +** ^(The [sqlite3_config]([SQLITE_CONFIG_PCACHE2], ...) interface can ** register an alternative page cache implementation by passing in an -** instance of the sqlite3_pcache_methods structure.)^ +** instance of the sqlite3_pcache_methods2 structure.)^ ** In many applications, most of the heap memory allocated by ** SQLite is used for the page cache. ** By implementing a @@ -5950,7 +6134,7 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** extreme measure that is only needed by the most demanding applications. ** The built-in page cache is recommended for most uses. ** -** ^(The contents of the sqlite3_pcache_methods structure are copied to an +** ^(The contents of the sqlite3_pcache_methods2 structure are copied to an ** internal buffer by SQLite within the call to [sqlite3_config]. Hence ** the application may discard the parameter after the call to ** [sqlite3_config()] returns.)^ @@ -5959,7 +6143,7 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** ^(The xInit() method is called once for each effective ** call to [sqlite3_initialize()])^ ** (usually only once during the lifetime of the process). ^(The xInit() -** method is passed a copy of the sqlite3_pcache_methods.pArg value.)^ +** method is passed a copy of the sqlite3_pcache_methods2.pArg value.)^ ** The intent of the xInit() method is to set up global data structures ** required by the custom page cache implementation. ** ^(If the xInit() method is NULL, then the @@ -5986,17 +6170,15 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** SQLite will typically create one cache instance for each open database file, ** though this is not guaranteed. ^The ** first parameter, szPage, is the size in bytes of the pages that must -** be allocated by the cache. ^szPage will not be a power of two. ^szPage -** will the page size of the database file that is to be cached plus an -** increment (here called "R") of less than 250. SQLite will use the -** extra R bytes on each page to store metadata about the underlying -** database page on disk. The value of R depends +** be allocated by the cache. ^szPage will always a power of two. ^The +** second parameter szExtra is a number of bytes of extra storage +** associated with each page cache entry. ^The szExtra parameter will +** a number less than 250. SQLite will use the +** extra szExtra bytes on each page to store metadata about the underlying +** database page on disk. The value passed into szExtra depends ** on the SQLite version, the target platform, and how SQLite was compiled. -** ^(R is constant for a particular build of SQLite. Except, there are two -** distinct values of R when SQLite is compiled with the proprietary -** ZIPVFS extension.)^ ^The second argument to -** xCreate(), bPurgeable, is true if the cache being created will -** be used to cache database pages of a file stored on disk, or +** ^The third argument to xCreate(), bPurgeable, is true if the cache being +** created will be used to cache database pages of a file stored on disk, or ** false if it is used for an in-memory database. The cache implementation ** does not have to do anything special based with the value of bPurgeable; ** it is purely advisory. ^On a cache where bPurgeable is false, SQLite will @@ -6020,11 +6202,16 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** ** [[the xFetch() page cache methods]] ** The xFetch() method locates a page in the cache and returns a pointer to -** the page, or a NULL pointer. -** A "page", in this context, means a buffer of szPage bytes aligned at an -** 8-byte boundary. The page to be fetched is determined by the key. ^The -** minimum key value is 1. After it has been retrieved using xFetch, the page -** is considered to be "pinned". +** an sqlite3_pcache_page object associated with that page, or a NULL pointer. +** The pBuf element of the returned sqlite3_pcache_page object will be a +** pointer to a buffer of szPage bytes used to store the content of a +** single database page. The pExtra element of sqlite3_pcache_page will be +** a pointer to the szExtra bytes of extra storage that SQLite has requested +** for each entry in the page cache. +** +** The page to be fetched is determined by the key. ^The minimum key value +** is 1. After it has been retrieved using xFetch, the page is considered +** to be "pinned". ** ** If the requested page is already in the page cache, then the page cache ** implementation must return a pointer to the page buffer with its content @@ -6077,8 +6264,37 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** ^The xDestroy() method is used to delete a cache allocated by xCreate(). ** All resources associated with the specified cache should be freed. ^After ** calling the xDestroy() method, SQLite considers the [sqlite3_pcache*] -** handle invalid, and will not use it with any other sqlite3_pcache_methods +** handle invalid, and will not use it with any other sqlite3_pcache_methods2 ** functions. +** +** [[the xShrink() page cache method]] +** ^SQLite invokes the xShrink() method when it wants the page cache to +** free up as much of heap memory as possible. The page cache implementation +** is not obligated to free any memory, but well-behaved implementations should +** do their best. +*/ +typedef struct sqlite3_pcache_methods2 sqlite3_pcache_methods2; +struct sqlite3_pcache_methods2 { + int iVersion; + void *pArg; + int (*xInit)(void*); + void (*xShutdown)(void*); + sqlite3_pcache *(*xCreate)(int szPage, int szExtra, int bPurgeable); + void (*xCachesize)(sqlite3_pcache*, int nCachesize); + int (*xPagecount)(sqlite3_pcache*); + sqlite3_pcache_page *(*xFetch)(sqlite3_pcache*, unsigned key, int createFlag); + void (*xUnpin)(sqlite3_pcache*, sqlite3_pcache_page*, int discard); + void (*xRekey)(sqlite3_pcache*, sqlite3_pcache_page*, + unsigned oldKey, unsigned newKey); + void (*xTruncate)(sqlite3_pcache*, unsigned iLimit); + void (*xDestroy)(sqlite3_pcache*); + void (*xShrink)(sqlite3_pcache*); +}; + +/* +** This is the obsolete pcache_methods object that has now been replaced +** by sqlite3_pcache_methods2. This object is not used by SQLite. It is +** retained in the header file for backwards compatibility only. */ typedef struct sqlite3_pcache_methods sqlite3_pcache_methods; struct sqlite3_pcache_methods { @@ -6095,6 +6311,7 @@ struct sqlite3_pcache_methods { void (*xDestroy)(sqlite3_pcache*); }; + /* ** CAPI3REF: Online Backup Object ** @@ -6424,11 +6641,12 @@ int sqlite3_unlock_notify( /* ** CAPI3REF: String Comparison ** -** ^The [sqlite3_strnicmp()] API allows applications and extensions to -** compare the contents of two buffers containing UTF-8 strings in a -** case-independent fashion, using the same definition of case independence -** that SQLite uses internally when comparing identifiers. +** ^The [sqlite3_stricmp()] and [sqlite3_strnicmp()] APIs allow applications +** and extensions to compare the contents of two buffers containing UTF-8 +** strings in a case-independent fashion, using the same definition of "case +** independence" that SQLite uses internally when comparing identifiers. */ +int sqlite3_stricmp(const char *, const char *); int sqlite3_strnicmp(const char *, const char *, int); /* diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 9e27654..953850e 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -126,6 +126,14 @@ #endif /* +** Powersafe overwrite is on by default. But can be turned off using +** the -DSQLITE_POWERSAFE_OVERWRITE=0 command-line option. +*/ +#ifndef SQLITE_POWERSAFE_OVERWRITE +# define SQLITE_POWERSAFE_OVERWRITE 1 +#endif + +/* ** The SQLITE_DEFAULT_MEMSTATUS macro must be defined as either 0 or 1. ** It determines whether or not the features related to ** SQLITE_CONFIG_MEMSTATUS are available by default or not. This value can @@ -346,7 +354,7 @@ */ #define SQLITE_MAX_FILE_FORMAT 4 #ifndef SQLITE_DEFAULT_FILE_FORMAT -# define SQLITE_DEFAULT_FILE_FORMAT 1 +# define SQLITE_DEFAULT_FILE_FORMAT 4 #endif /* @@ -553,9 +561,13 @@ struct BusyHandler { /* ** The following value as a destructor means to use sqlite3DbFree(). -** This is an internal extension to SQLITE_STATIC and SQLITE_TRANSIENT. +** The sqlite3DbFree() routine requires two parameters instead of the +** one parameter that destructors normally want. So we have to introduce +** this magic value that the code knows to handle differently. Any +** pointer will work here as long as it is distinct from SQLITE_STATIC +** and SQLITE_TRANSIENT. */ -#define SQLITE_DYNAMIC ((sqlite3_destructor_type)sqlite3DbFree) +#define SQLITE_DYNAMIC ((sqlite3_destructor_type)sqlite3MallocSize) /* ** When SQLITE_OMIT_WSD is defined, it means that the target platform does @@ -785,35 +797,16 @@ struct FuncDefHash { /* ** Each database connection is an instance of the following structure. -** -** The sqlite.lastRowid records the last insert rowid generated by an -** insert statement. Inserts on views do not affect its value. Each -** trigger has its own context, so that lastRowid can be updated inside -** triggers as usual. The previous value will be restored once the trigger -** exits. Upon entering a before or instead of trigger, lastRowid is no -** longer (since after version 2.8.12) reset to -1. -** -** The sqlite.nChange does not count changes within triggers and keeps no -** context. It is reset at start of sqlite3_exec. -** The sqlite.lsChange represents the number of changes made by the last -** insert, update, or delete statement. It remains constant throughout the -** length of a statement and is then updated by OP_SetCounts. It keeps a -** context stack just like lastRowid so that the count of changes -** within a trigger is not seen outside the trigger. Changes to views do not -** affect the value of lsChange. -** The sqlite.csChange keeps track of the number of current changes (since -** the last statement) and is used to update sqlite_lsChange. -** -** The member variables sqlite.errCode, sqlite.zErrMsg and sqlite.zErrMsg16 -** store the most recent error code and, if applicable, string. The -** internal function sqlite3Error() is used to set these variables -** consistently. */ struct sqlite3 { sqlite3_vfs *pVfs; /* OS Interface */ - int nDb; /* Number of backends currently in use */ + struct Vdbe *pVdbe; /* List of active virtual machines */ + CollSeq *pDfltColl; /* The default collating sequence (BINARY) */ + sqlite3_mutex *mutex; /* Connection mutex */ Db *aDb; /* All backends */ + int nDb; /* Number of backends currently in use */ int flags; /* Miscellaneous flags. See below */ + i64 lastRowid; /* ROWID of most recent insert (see above) */ unsigned int openFlags; /* Flags passed to sqlite3_vfs.xOpen() */ int errCode; /* Most recent error code (SQLITE_*) */ int errMask; /* & result codes with this before returning */ @@ -824,27 +817,23 @@ struct sqlite3 { signed char nextAutovac; /* Autovac setting after VACUUM if >=0 */ u8 suppressErr; /* Do not issue error messages if true */ u8 vtabOnConflict; /* Value to return for s3_vtab_on_conflict() */ + u8 isTransactionSavepoint; /* True if the outermost savepoint is a TS */ int nextPagesize; /* Pagesize after VACUUM if >0 */ - int nTable; /* Number of tables in the database */ - CollSeq *pDfltColl; /* The default collating sequence (BINARY) */ - i64 lastRowid; /* ROWID of most recent insert (see above) */ u32 magic; /* Magic number for detect library misuse */ int nChange; /* Value returned by sqlite3_changes() */ int nTotalChange; /* Value returned by sqlite3_total_changes() */ - sqlite3_mutex *mutex; /* Connection mutex */ int aLimit[SQLITE_N_LIMIT]; /* Limits */ struct sqlite3InitInfo { /* Information used during initialization */ - int iDb; /* When back is being initialized */ int newTnum; /* Rootpage of table being initialized */ + u8 iDb; /* Which db file is being initialized */ u8 busy; /* TRUE if currently initializing */ u8 orphanTrigger; /* Last statement is orphaned TEMP trigger */ } init; - int nExtension; /* Number of loaded extensions */ - void **aExtension; /* Array of shared library handles */ - struct Vdbe *pVdbe; /* List of active virtual machines */ int activeVdbeCnt; /* Number of VDBEs currently executing */ int writeVdbeCnt; /* Number of active VDBEs that are writing */ int vdbeExecCnt; /* Number of nested calls to VdbeExec() */ + int nExtension; /* Number of loaded extensions */ + void **aExtension; /* Array of shared library handles */ void (*xTrace)(void*,const char*); /* Trace function */ void *pTraceArg; /* Argument to the trace function */ void (*xProfile)(void*,const char*,u64); /* Profiling function */ @@ -881,21 +870,20 @@ struct sqlite3 { int nProgressOps; /* Number of opcodes for progress callback */ #endif #ifndef SQLITE_OMIT_VIRTUALTABLE + int nVTrans; /* Allocated size of aVTrans */ Hash aModule; /* populated by sqlite3_create_module() */ VtabCtx *pVtabCtx; /* Context for active vtab connect/create */ VTable **aVTrans; /* Virtual tables with open transactions */ - int nVTrans; /* Allocated size of aVTrans */ VTable *pDisconnect; /* Disconnect these in next sqlite3_prepare() */ #endif FuncDefHash aFunc; /* Hash table of connection functions */ Hash aCollSeq; /* All collating sequences */ BusyHandler busyHandler; /* Busy callback */ - int busyTimeout; /* Busy handler timeout, in msec */ Db aDbStatic[2]; /* Static space for the 2 default backends */ Savepoint *pSavepoint; /* List of active savepoints */ + int busyTimeout; /* Busy handler timeout, in msec */ int nSavepoint; /* Number of non-transaction savepoints */ int nStatement; /* Number of nested statement-transactions */ - u8 isTransactionSavepoint; /* True if the outermost savepoint is a TS */ i64 nDeferredCons; /* Net deferred constraints this transaction. */ int *pnBytesFreed; /* If not NULL, increment this in DbFree() */ @@ -938,8 +926,7 @@ struct sqlite3 { #define SQLITE_SqlTrace 0x00004000 /* Debug print SQL as it executes */ #define SQLITE_VdbeListing 0x00008000 /* Debug listings of VDBE programs */ #define SQLITE_WriteSchema 0x00010000 /* OK to update SQLITE_MASTER */ -#define SQLITE_NoReadlock 0x00020000 /* Readlocks are omitted when - ** accessing read-only databases */ + /* 0x00020000 Unused */ #define SQLITE_IgnoreChecks 0x00040000 /* Do not enforce check constraints */ #define SQLITE_ReadUncommitted 0x0080000 /* For shared-cache mode */ #define SQLITE_LegacyFileFmt 0x00100000 /* Create new databases in format 1 */ @@ -1022,15 +1009,18 @@ struct FuncDestructor { }; /* -** Possible values for FuncDef.flags +** Possible values for FuncDef.flags. Note that the _LENGTH and _TYPEOF +** values must correspond to OPFLAG_LENGTHARG and OPFLAG_TYPEOFARG. There +** are assert() statements in the code to verify this. */ #define SQLITE_FUNC_LIKE 0x01 /* Candidate for the LIKE optimization */ #define SQLITE_FUNC_CASE 0x02 /* Case-sensitive LIKE-type function */ #define SQLITE_FUNC_EPHEM 0x04 /* Ephemeral. Delete with VDBE */ #define SQLITE_FUNC_NEEDCOLL 0x08 /* sqlite3GetFuncCollSeq() might be called */ -#define SQLITE_FUNC_PRIVATE 0x10 /* Allowed for internal use only */ -#define SQLITE_FUNC_COUNT 0x20 /* Built-in count(*) aggregate */ -#define SQLITE_FUNC_COALESCE 0x40 /* Built-in coalesce() or ifnull() function */ +#define SQLITE_FUNC_COUNT 0x10 /* Built-in count(*) aggregate */ +#define SQLITE_FUNC_COALESCE 0x20 /* Built-in coalesce() or ifnull() function */ +#define SQLITE_FUNC_LENGTH 0x40 /* Built-in length() function */ +#define SQLITE_FUNC_TYPEOF 0x80 /* Built-in typeof() function */ /* ** The following three macros, FUNCTION(), LIKEFUNC() and AGGREGATE() are @@ -1058,7 +1048,10 @@ struct FuncDestructor { ** parameter. */ #define FUNCTION(zName, nArg, iArg, bNC, xFunc) \ - {nArg, SQLITE_UTF8, bNC*SQLITE_FUNC_NEEDCOLL, \ + {nArg, SQLITE_UTF8, (bNC*SQLITE_FUNC_NEEDCOLL), \ + SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0, 0} +#define FUNCTION2(zName, nArg, iArg, bNC, xFunc, extraFlags) \ + {nArg, SQLITE_UTF8, (bNC*SQLITE_FUNC_NEEDCOLL)|extraFlags, \ SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0, 0} #define STR_FUNCTION(zName, nArg, pArg, bNC, xFunc) \ {nArg, SQLITE_UTF8, bNC*SQLITE_FUNC_NEEDCOLL, \ @@ -1144,21 +1137,12 @@ struct Column { struct CollSeq { char *zName; /* Name of the collating sequence, UTF-8 encoded */ u8 enc; /* Text encoding handled by xCmp() */ - u8 type; /* One of the SQLITE_COLL_... values below */ void *pUser; /* First argument to xCmp() */ int (*xCmp)(void*,int, const void*, int, const void*); void (*xDel)(void*); /* Destructor for pUser */ }; /* -** Allowed values of CollSeq.type: -*/ -#define SQLITE_COLL_BINARY 1 /* The default memcmp() collating sequence */ -#define SQLITE_COLL_NOCASE 2 /* The built-in NOCASE collating sequence */ -#define SQLITE_COLL_REVERSE 3 /* The built-in REVERSE collating sequence */ -#define SQLITE_COLL_USER 0 /* Any other user-defined collating sequence */ - -/* ** A sort order can be either ASC or DESC. */ #define SQLITE_SO_ASC 0 /* Sort in ascending order */ @@ -1297,7 +1281,7 @@ struct Table { FKey *pFKey; /* Linked list of all foreign keys in this table */ char *zColAff; /* String defining the affinity of each column */ #ifndef SQLITE_OMIT_CHECK - Expr *pCheck; /* The AND of all CHECK constraints */ + ExprList *pCheck; /* All CHECK constraints */ #endif #ifndef SQLITE_OMIT_ALTERTABLE int addColOffset; /* Offset in CREATE TABLE stmt to add a new column */ @@ -1320,8 +1304,6 @@ struct Table { #define TF_HasPrimaryKey 0x04 /* Table has a primary key */ #define TF_Autoincrement 0x08 /* Integer primary key is autoincrement */ #define TF_Virtual 0x10 /* Is a virtual table */ -#define TF_NeedMetadata 0x20 /* aCol[].zType and aCol[].pColl missing */ - /* @@ -1443,7 +1425,7 @@ struct KeyInfo { struct UnpackedRecord { KeyInfo *pKeyInfo; /* Collation and sort-order information */ u16 nField; /* Number of entries in apMem[] */ - u16 flags; /* Boolean settings. UNPACKED_... below */ + u8 flags; /* Boolean settings. UNPACKED_... below */ i64 rowid; /* Used by UNPACKED_PREFIX_SEARCH */ Mem *aMem; /* Values */ }; @@ -1451,12 +1433,9 @@ struct UnpackedRecord { /* ** Allowed values of UnpackedRecord.flags */ -#define UNPACKED_NEED_FREE 0x0001 /* Memory is from sqlite3Malloc() */ -#define UNPACKED_NEED_DESTROY 0x0002 /* apMem[]s should all be destroyed */ -#define UNPACKED_IGNORE_ROWID 0x0004 /* Ignore trailing rowid on key1 */ -#define UNPACKED_INCRKEY 0x0008 /* Make this key an epsilon larger */ -#define UNPACKED_PREFIX_MATCH 0x0010 /* A prefix match is considered OK */ -#define UNPACKED_PREFIX_SEARCH 0x0020 /* A prefix match is considered OK */ +#define UNPACKED_INCRKEY 0x01 /* Make this key an epsilon larger */ +#define UNPACKED_PREFIX_MATCH 0x02 /* A prefix match is considered OK */ +#define UNPACKED_PREFIX_SEARCH 0x04 /* Ignore final (rowid) field */ /* ** Each SQL index is represented in memory by an @@ -1486,19 +1465,19 @@ struct UnpackedRecord { */ struct Index { char *zName; /* Name of this index */ - int nColumn; /* Number of columns in the table used by this index */ int *aiColumn; /* Which columns are used by this index. 1st is 0 */ tRowcnt *aiRowEst; /* Result of ANALYZE: Est. rows selected by each column */ Table *pTable; /* The SQL table being indexed */ - int tnum; /* Page containing root of this index in database file */ - u8 onError; /* OE_Abort, OE_Ignore, OE_Replace, or OE_None */ - u8 autoIndex; /* True if is automatically created (ex: by UNIQUE) */ - u8 bUnordered; /* Use this index for == or IN queries only */ char *zColAff; /* String defining the affinity of each column */ Index *pNext; /* The next index associated with the same table */ Schema *pSchema; /* Schema containing this index */ u8 *aSortOrder; /* Array of size Index.nColumn. True==DESC, False==ASC */ char **azColl; /* Array of collation sequence names for index */ + int nColumn; /* Number of columns in the table used by this index */ + int tnum; /* Page containing root of this index in database file */ + u8 onError; /* OE_Abort, OE_Ignore, OE_Replace, or OE_None */ + u8 autoIndex; /* True if is automatically created (ex: by UNIQUE) */ + u8 bUnordered; /* Use this index for == or IN queries only */ #ifdef SQLITE_ENABLE_STAT3 int nSample; /* Number of elements in aSample[] */ tRowcnt avgEq; /* Average nEq value for key values not in aSample */ @@ -1557,8 +1536,8 @@ struct AggInfo { ** than the source table */ int sortingIdx; /* Cursor number of the sorting index */ int sortingIdxPTab; /* Cursor number of pseudo-table */ - ExprList *pGroupBy; /* The group by clause */ int nSortingColumn; /* Number of columns in the sorting index */ + ExprList *pGroupBy; /* The group by clause */ struct AggInfo_col { /* For each column used in source tables */ Table *pTab; /* Source table */ int iTable; /* Cursor number of the source table */ @@ -1568,7 +1547,6 @@ struct AggInfo { Expr *pExpr; /* The original expression */ } *aCol; int nColumn; /* Number of used entries in aCol[] */ - int nColumnAlloc; /* Number of slots allocated for aCol[] */ int nAccumulator; /* Number of columns that show through to the output. ** Additional columns are used only as parameters to ** aggregate functions */ @@ -1579,7 +1557,6 @@ struct AggInfo { int iDistinct; /* Ephemeral table used to enforce DISTINCT */ } *aFunc; int nFunc; /* Number of entries in aFunc[] */ - int nFuncAlloc; /* Number of slots allocated for aFunc[] */ }; /* @@ -1697,6 +1674,7 @@ struct Expr { i16 iRightJoinTable; /* If EP_FromJoin, the right table of the join */ u8 flags2; /* Second set of flags. EP2_... */ u8 op2; /* If a TK_REGISTER, the original value of Expr.op */ + /* If TK_COLUMN, the value of p5 for OP_Column */ AggInfo *pAggInfo; /* Used by TK_AGG_COLUMN and TK_AGG_FUNCTION */ Table *pTab; /* Table for TK_COLUMN expressions. */ #if SQLITE_MAX_EXPR_DEPTH>0 @@ -1719,10 +1697,10 @@ struct Expr { #define EP_FixedDest 0x0200 /* Result needed in a specific register */ #define EP_IntValue 0x0400 /* Integer value contained in u.iValue */ #define EP_xIsSelect 0x0800 /* x.pSelect is valid (otherwise x.pList is) */ - -#define EP_Reduced 0x1000 /* Expr struct is EXPR_REDUCEDSIZE bytes only */ -#define EP_TokenOnly 0x2000 /* Expr struct is EXPR_TOKENONLYSIZE bytes only */ -#define EP_Static 0x4000 /* Held in memory not obtained from malloc() */ +#define EP_Hint 0x1000 /* Not used */ +#define EP_Reduced 0x2000 /* Expr struct is EXPR_REDUCEDSIZE bytes only */ +#define EP_TokenOnly 0x4000 /* Expr struct is EXPR_TOKENONLYSIZE bytes only */ +#define EP_Static 0x8000 /* Held in memory not obtained from malloc() */ /* ** The following are the meanings of bits in the Expr.flags2 field. @@ -1776,17 +1754,16 @@ struct Expr { */ struct ExprList { int nExpr; /* Number of expressions on the list */ - int nAlloc; /* Number of entries allocated below */ int iECursor; /* VDBE Cursor associated with this ExprList */ - struct ExprList_item { + struct ExprList_item { /* For each expression in the list */ Expr *pExpr; /* The list of expressions */ char *zName; /* Token associated with this expression */ char *zSpan; /* Original text of the expression */ u8 sortOrder; /* 1 for DESC or 0 for ASC */ u8 done; /* A flag to indicate when processing is finished */ - u16 iCol; /* For ORDER BY, column number in result set */ + u16 iOrderByCol; /* For ORDER BY, column number in result set */ u16 iAlias; /* Index into Parse.aAlias[] for zName */ - } *a; /* One entry for each expression */ + } *a; /* Alloc a power of two greater or equal to nExpr */ }; /* @@ -1821,7 +1798,6 @@ struct IdList { int idx; /* Index in some Table.aCol[] of a column named zName */ } *a; int nId; /* Number of identifiers on the list */ - int nAlloc; /* Number of entries allocated for a[] below */ }; /* @@ -2030,17 +2006,22 @@ struct NameContext { Parse *pParse; /* The parser */ SrcList *pSrcList; /* One or more tables used to resolve names */ ExprList *pEList; /* Optional list of named expressions */ - int nRef; /* Number of names resolved by this context */ - int nErr; /* Number of errors encountered while resolving names */ - u8 allowAgg; /* Aggregate functions allowed here */ - u8 hasAgg; /* True if aggregates are seen */ - u8 isCheck; /* True if resolving names in a CHECK constraint */ - int nDepth; /* Depth of subquery recursion. 1 for no recursion */ AggInfo *pAggInfo; /* Information about aggregates at this level */ NameContext *pNext; /* Next outer name context. NULL for outermost */ + int nRef; /* Number of names resolved by this context */ + int nErr; /* Number of errors encountered while resolving names */ + u8 ncFlags; /* Zero or more NC_* flags defined below */ }; /* +** Allowed values for the NameContext, ncFlags field. +*/ +#define NC_AllowAgg 0x01 /* Aggregate functions are allowed here */ +#define NC_HasAgg 0x02 /* One or more aggregate functions seen */ +#define NC_IsCheck 0x04 /* True if resolving names in a CHECK constraint */ +#define NC_InAggFunc 0x08 /* True if analyzing arguments to an agg func */ + +/* ** An instance of the following structure contains all information ** needed to generate code for a single SELECT statement. ** @@ -2065,6 +2046,9 @@ struct Select { u8 op; /* One of: TK_UNION TK_ALL TK_INTERSECT TK_EXCEPT */ char affinity; /* MakeRecord with this affinity for SRT_Set */ u16 selFlags; /* Various SF_* values */ + int iLimit, iOffset; /* Memory registers holding LIMIT & OFFSET counters */ + int addrOpenEphm[3]; /* OP_OpenEphem opcodes related to this select */ + double nSelectRow; /* Estimated number of result rows */ SrcList *pSrc; /* The FROM clause */ Expr *pWhere; /* The WHERE clause */ ExprList *pGroupBy; /* The GROUP BY clause */ @@ -2075,22 +2059,20 @@ struct Select { Select *pRightmost; /* Right-most select in a compound select statement */ Expr *pLimit; /* LIMIT expression. NULL means not used. */ Expr *pOffset; /* OFFSET expression. NULL means not used. */ - int iLimit, iOffset; /* Memory registers holding LIMIT & OFFSET counters */ - int addrOpenEphm[3]; /* OP_OpenEphem opcodes related to this select */ - double nSelectRow; /* Estimated number of result rows */ }; /* ** Allowed values for Select.selFlags. The "SF" prefix stands for ** "Select Flag". */ -#define SF_Distinct 0x0001 /* Output should be DISTINCT */ -#define SF_Resolved 0x0002 /* Identifiers have been resolved */ -#define SF_Aggregate 0x0004 /* Contains aggregate functions */ -#define SF_UsesEphemeral 0x0008 /* Uses the OpenEphemeral opcode */ -#define SF_Expanded 0x0010 /* sqlite3SelectExpand() called on this */ -#define SF_HasTypeInfo 0x0020 /* FROM subqueries have Table metadata */ -#define SF_UseSorter 0x0040 /* Sort using a sorter */ +#define SF_Distinct 0x01 /* Output should be DISTINCT */ +#define SF_Resolved 0x02 /* Identifiers have been resolved */ +#define SF_Aggregate 0x04 /* Contains aggregate functions */ +#define SF_UsesEphemeral 0x08 /* Uses the OpenEphemeral opcode */ +#define SF_Expanded 0x10 /* sqlite3SelectExpand() called on this */ +#define SF_HasTypeInfo 0x20 /* FROM subqueries have Table metadata */ +#define SF_UseSorter 0x40 /* Sort using a sorter */ +#define SF_Values 0x80 /* Synthesized from VALUES clause */ /* @@ -2168,10 +2150,10 @@ struct AutoincInfo { */ struct TriggerPrg { Trigger *pTrigger; /* Trigger this program was coded from */ - int orconf; /* Default ON CONFLICT policy */ + TriggerPrg *pNext; /* Next entry in Parse.pTriggerPrg list */ SubProgram *pProgram; /* Program implementing pTrigger/orconf */ + int orconf; /* Default ON CONFLICT policy */ u32 aColmask[2]; /* Masks of old.*, new.* columns accessed */ - TriggerPrg *pNext; /* Next entry in Parse.pTriggerPrg list */ }; /* @@ -2201,16 +2183,18 @@ struct TriggerPrg { */ struct Parse { sqlite3 *db; /* The main database structure */ - int rc; /* Return code from execution */ char *zErrMsg; /* An error message */ Vdbe *pVdbe; /* An engine for executing database bytecode */ + int rc; /* Return code from execution */ u8 colNamesSet; /* TRUE after OP_ColumnName has been issued to pVdbe */ - u8 nameClash; /* A permanent table name clashes with temp table name */ u8 checkSchema; /* Causes schema cookie check after an error */ u8 nested; /* Number of nested calls to the parser/code generator */ - u8 parseError; /* True after a parsing error. Ticket #1794 */ u8 nTempReg; /* Number of temporary registers in aTempReg[] */ u8 nTempInUse; /* Number of aTempReg[] currently checked out */ + u8 nColCache; /* Number of entries in aColCache[] */ + u8 iColCache; /* Next entry in aColCache[] to replace */ + u8 isMultiWrite; /* True if statement may modify/insert multiple rows */ + u8 mayAbort; /* True if statement may throw an ABORT exception */ int aTempReg[8]; /* Holding area for temporary registers */ int nRangeReg; /* Size of the temporary register block */ int iRangeReg; /* First register in temporary register block */ @@ -2218,11 +2202,10 @@ struct Parse { int nTab; /* Number of previously allocated VDBE cursors */ int nMem; /* Number of memory cells used so far */ int nSet; /* Number of sets used so far */ + int nOnce; /* Number of OP_Once instructions so far */ int ckBase; /* Base register of data during check constraints */ int iCacheLevel; /* ColCache valid when aColCache[].iLevel<=iCacheLevel */ int iCacheCnt; /* Counter used to generate aColCache[].lru values */ - u8 nColCache; /* Number of entries in the column cache */ - u8 iColCache; /* Next entry of the cache to replace */ struct yColCache { int iTable; /* Table cursor number */ int iColumn; /* Table column number */ @@ -2233,62 +2216,64 @@ struct Parse { } aColCache[SQLITE_N_COLCACHE]; /* One for each column cache entry */ yDbMask writeMask; /* Start a write transaction on these databases */ yDbMask cookieMask; /* Bitmask of schema verified databases */ - u8 isMultiWrite; /* True if statement may affect/insert multiple rows */ - u8 mayAbort; /* True if statement may throw an ABORT exception */ int cookieGoto; /* Address of OP_Goto to cookie verifier subroutine */ int cookieValue[SQLITE_MAX_ATTACHED+2]; /* Values of cookies to verify */ + int regRowid; /* Register holding rowid of CREATE TABLE entry */ + int regRoot; /* Register holding root page number for new objects */ + int nMaxArg; /* Max args passed to user function by sub-program */ + Token constraintName;/* Name of the constraint currently being parsed */ #ifndef SQLITE_OMIT_SHARED_CACHE int nTableLock; /* Number of locks in aTableLock */ TableLock *aTableLock; /* Required table locks for shared-cache mode */ #endif - int regRowid; /* Register holding rowid of CREATE TABLE entry */ - int regRoot; /* Register holding root page number for new objects */ AutoincInfo *pAinc; /* Information about AUTOINCREMENT counters */ - int nMaxArg; /* Max args passed to user function by sub-program */ /* Information used while coding trigger programs. */ Parse *pToplevel; /* Parse structure for main program (or NULL) */ Table *pTriggerTab; /* Table triggers are being coded for */ + double nQueryLoop; /* Estimated number of iterations of a query */ u32 oldmask; /* Mask of old.* columns referenced */ u32 newmask; /* Mask of new.* columns referenced */ u8 eTriggerOp; /* TK_UPDATE, TK_INSERT or TK_DELETE */ u8 eOrconf; /* Default ON CONFLICT policy for trigger steps */ u8 disableTriggers; /* True to disable triggers */ - double nQueryLoop; /* Estimated number of iterations of a query */ /* Above is constant between recursions. Below is reset before and after ** each recursion */ - int nVar; /* Number of '?' variables seen in the SQL so far */ - int nzVar; /* Number of available slots in azVar[] */ - char **azVar; /* Pointers to names of parameters */ - Vdbe *pReprepare; /* VM being reprepared (sqlite3Reprepare()) */ - int nAlias; /* Number of aliased result set columns */ - int nAliasAlloc; /* Number of allocated slots for aAlias[] */ - int *aAlias; /* Register used to hold aliased result */ - u8 explain; /* True if the EXPLAIN flag is found on the query */ - Token sNameToken; /* Token with unqualified schema object name */ - Token sLastToken; /* The last token parsed */ - const char *zTail; /* All SQL text past the last semicolon parsed */ - Table *pNewTable; /* A table being constructed by CREATE TABLE */ - Trigger *pNewTrigger; /* Trigger under construct by a CREATE TRIGGER */ - const char *zAuthContext; /* The 6th parameter to db->xAuth callbacks */ + int nVar; /* Number of '?' variables seen in the SQL so far */ + int nzVar; /* Number of available slots in azVar[] */ + u8 explain; /* True if the EXPLAIN flag is found on the query */ #ifndef SQLITE_OMIT_VIRTUALTABLE - Token sArg; /* Complete text of a module argument */ - u8 declareVtab; /* True if inside sqlite3_declare_vtab() */ - int nVtabLock; /* Number of virtual tables to lock */ - Table **apVtabLock; /* Pointer to virtual tables needing locking */ + u8 declareVtab; /* True if inside sqlite3_declare_vtab() */ + int nVtabLock; /* Number of virtual tables to lock */ #endif - int nHeight; /* Expression tree height of current sub-select */ - Table *pZombieTab; /* List of Table objects to delete after code gen */ - TriggerPrg *pTriggerPrg; /* Linked list of coded triggers */ - + int nAlias; /* Number of aliased result set columns */ + int nHeight; /* Expression tree height of current sub-select */ #ifndef SQLITE_OMIT_EXPLAIN - int iSelectId; - int iNextSelectId; + int iSelectId; /* ID of current select for EXPLAIN output */ + int iNextSelectId; /* Next available select ID for EXPLAIN output */ +#endif + char **azVar; /* Pointers to names of parameters */ + Vdbe *pReprepare; /* VM being reprepared (sqlite3Reprepare()) */ + int *aAlias; /* Register used to hold aliased result */ + const char *zTail; /* All SQL text past the last semicolon parsed */ + Table *pNewTable; /* A table being constructed by CREATE TABLE */ + Trigger *pNewTrigger; /* Trigger under construct by a CREATE TRIGGER */ + const char *zAuthContext; /* The 6th parameter to db->xAuth callbacks */ + Token sNameToken; /* Token with unqualified schema object name */ + Token sLastToken; /* The last token parsed */ +#ifndef SQLITE_OMIT_VIRTUALTABLE + Token sArg; /* Complete text of a module argument */ + Table **apVtabLock; /* Pointer to virtual tables needing locking */ #endif + Table *pZombieTab; /* List of Table objects to delete after code gen */ + TriggerPrg *pTriggerPrg; /* Linked list of coded triggers */ }; +/* +** Return true if currently inside an sqlite3_declare_vtab() call. +*/ #ifdef SQLITE_OMIT_VIRTUALTABLE #define IN_DECLARE_VTAB 0 #else @@ -2305,7 +2290,7 @@ struct AuthContext { }; /* -** Bitfield flags for P5 value in OP_Insert and OP_Delete +** Bitfield flags for P5 value in various opcodes. */ #define OPFLAG_NCHANGE 0x01 /* Set to update db->nChange */ #define OPFLAG_LASTROWID 0x02 /* Set to update db->lastRowid */ @@ -2313,6 +2298,8 @@ struct AuthContext { #define OPFLAG_APPEND 0x08 /* This is likely to be an append */ #define OPFLAG_USESEEKRESULT 0x10 /* Try to avoid a seek in BtreeInsert() */ #define OPFLAG_CLEARCACHE 0x20 /* Clear pseudo-table cache in OP_Column */ +#define OPFLAG_LENGTHARG 0x40 /* OP_Column only used for length() */ +#define OPFLAG_TYPEOFARG 0x80 /* OP_Column only used for typeof() */ /* * Each trigger present in the database schema is stored as an instance of @@ -2439,8 +2426,8 @@ struct StrAccum { */ typedef struct { sqlite3 *db; /* The database being initialized */ - int iDb; /* 0 for main database. 1 for TEMP, 2.. for ATTACHed */ char **pzErrMsg; /* Error message stored here */ + int iDb; /* 0 for main database. 1 for TEMP, 2.. for ATTACHed */ int rc; /* Result code stored here */ } InitData; @@ -2459,7 +2446,7 @@ struct Sqlite3Config { int nLookaside; /* Default lookaside buffer count */ sqlite3_mem_methods m; /* Low-level memory allocation interface */ sqlite3_mutex_methods mutex; /* Low-level mutex interface */ - sqlite3_pcache_methods pcache; /* Low-level page-cache interface */ + sqlite3_pcache_methods2 pcache2; /* Low-level page-cache interface */ void *pHeap; /* Heap storage space */ int nHeap; /* Size of pHeap[] */ int mnReq, mxReq; /* Min and max heap requests sizes */ @@ -2495,6 +2482,7 @@ struct Walker { union { /* Extra data for callback */ NameContext *pNC; /* Naming context */ int i; /* Integer value */ + SrcList *pSrcList; /* FROM clause */ } u; }; @@ -2582,7 +2570,7 @@ int sqlite3CantopenError(int); /* ** Internal function prototypes */ -int sqlite3StrICmp(const char *, const char *); +#define sqlite3StrICmp sqlite3_stricmp int sqlite3Strlen30(const char*); #define sqlite3StrNICmp sqlite3_strnicmp @@ -2665,6 +2653,29 @@ char *sqlite3MAppendf(sqlite3*,char*,const char*,...); #if defined(SQLITE_TEST) void *sqlite3TestTextToPtr(const char*); #endif + +/* Output formatting for SQLITE_TESTCTRL_EXPLAIN */ +#if defined(SQLITE_ENABLE_TREE_EXPLAIN) + void sqlite3ExplainBegin(Vdbe*); + void sqlite3ExplainPrintf(Vdbe*, const char*, ...); + void sqlite3ExplainNL(Vdbe*); + void sqlite3ExplainPush(Vdbe*); + void sqlite3ExplainPop(Vdbe*); + void sqlite3ExplainFinish(Vdbe*); + void sqlite3ExplainSelect(Vdbe*, Select*); + void sqlite3ExplainExpr(Vdbe*, Expr*); + void sqlite3ExplainExprList(Vdbe*, ExprList*); + const char *sqlite3VdbeExplanation(Vdbe*); +#else +# define sqlite3ExplainBegin(X) +# define sqlite3ExplainSelect(A,B) +# define sqlite3ExplainExpr(A,B) +# define sqlite3ExplainExprList(A,B) +# define sqlite3ExplainFinish(X) +# define sqlite3VdbeExplanation(X) 0 +#endif + + void sqlite3SetString(char **, sqlite3*, const char*, ...); void sqlite3ErrorMsg(Parse*, const char*, ...); int sqlite3Dequote(char*); @@ -2675,6 +2686,7 @@ int sqlite3GetTempReg(Parse*); void sqlite3ReleaseTempReg(Parse*,int); int sqlite3GetTempRange(Parse*,int); void sqlite3ReleaseTempRange(Parse*,int,int); +void sqlite3ClearTempRegCache(Parse*); Expr *sqlite3ExprAlloc(sqlite3*,int,const Token*,int); Expr *sqlite3Expr(sqlite3*,int,const char*); void sqlite3ExprAttachSubtrees(sqlite3*,Expr*,Expr*,Expr*); @@ -2706,6 +2718,8 @@ void sqlite3AddCollateType(Parse*, Token*); void sqlite3EndTable(Parse*,Token*,Token*,Select*); int sqlite3ParseUri(const char*,const char*,unsigned int*, sqlite3_vfs**,char**,char **); +Btree *sqlite3DbNameToBtree(sqlite3*,const char*); +int sqlite3CodeOnce(Parse *); Bitvec *sqlite3BitvecCreate(u32); int sqlite3BitvecTest(Bitvec*, u32); @@ -2740,7 +2754,7 @@ void sqlite3DeleteTable(sqlite3*, Table*); # define sqlite3AutoincrementEnd(X) #endif void sqlite3Insert(Parse*, SrcList*, ExprList*, Select*, IdList*, int); -void *sqlite3ArrayAllocate(sqlite3*,void*,int,int,int*,int*,int*); +void *sqlite3ArrayAllocate(sqlite3*,void*,int,int*,int*); IdList *sqlite3IdListAppend(sqlite3*, IdList*, Token*); int sqlite3IdListIndex(IdList*,const char*); SrcList *sqlite3SrcListEnlarge(sqlite3*, SrcList*, int, int); @@ -2770,7 +2784,7 @@ void sqlite3DeleteFrom(Parse*, SrcList*, Expr*); void sqlite3Update(Parse*, SrcList*, ExprList*, Expr*, int); WhereInfo *sqlite3WhereBegin(Parse*, SrcList*, Expr*, ExprList**,ExprList*,u16); void sqlite3WhereEnd(WhereInfo*); -int sqlite3ExprCodeGetColumn(Parse*, Table*, int, int, int); +int sqlite3ExprCodeGetColumn(Parse*, Table*, int, int, int, u8); void sqlite3ExprCodeGetColumnOfTable(Vdbe*, Table*, int, int, int); void sqlite3ExprCodeMove(Parse*, int, int, int); void sqlite3ExprCodeCopy(Parse*, int, int, int); @@ -2804,7 +2818,7 @@ Vdbe *sqlite3GetVdbe(Parse*); void sqlite3PrngSaveState(void); void sqlite3PrngRestoreState(void); void sqlite3PrngResetState(void); -void sqlite3RollbackAll(sqlite3*); +void sqlite3RollbackAll(sqlite3*,int); void sqlite3CodeVerifySchema(Parse*, int); void sqlite3CodeVerifyNamedSchema(Parse*, const char *zDb); void sqlite3BeginTransaction(Parse*, int); @@ -2837,7 +2851,7 @@ SrcList *sqlite3SrcListDup(sqlite3*,SrcList*,int); IdList *sqlite3IdListDup(sqlite3*,IdList*); Select *sqlite3SelectDup(sqlite3*,Select*,int); void sqlite3FuncDefInsert(FuncDefHash*, FuncDef*); -FuncDef *sqlite3FindFunction(sqlite3*,const char*,int,int,u8,int); +FuncDef *sqlite3FindFunction(sqlite3*,const char*,int,int,u8,u8); void sqlite3RegisterBuiltinFunctions(sqlite3*); void sqlite3RegisterDateTimeFunctions(void); void sqlite3RegisterGlobalFunctions(void); @@ -2978,7 +2992,7 @@ void sqlite3FileSuffix3(const char*, char*); #else # define sqlite3FileSuffix3(X,Y) #endif -u8 sqlite3GetBoolean(const char *z); +u8 sqlite3GetBoolean(const char *z,int); const void *sqlite3ValueText(sqlite3_value*, u8); int sqlite3ValueBytes(sqlite3_value*, u8); @@ -3044,6 +3058,7 @@ int sqlite3OpenTempDatabase(Parse *); void sqlite3StrAccumInit(StrAccum*, char*, int, int); void sqlite3StrAccumAppend(StrAccum*,const char*,int); +void sqlite3AppendSpace(StrAccum*,int); char *sqlite3StrAccumFinish(StrAccum*); void sqlite3StrAccumReset(StrAccum*); void sqlite3SelectDestInit(SelectDest*,int,int); @@ -3103,7 +3118,7 @@ void sqlite3AutoLoadExtensions(sqlite3*); # define sqlite3VtabInSync(db) ((db)->nVTrans>0 && (db)->aVTrans==0) #endif void sqlite3VtabMakeWritable(Parse*,Table*); -void sqlite3VtabBeginParse(Parse*, Token*, Token*, Token*); +void sqlite3VtabBeginParse(Parse*, Token*, Token*, Token*, int); void sqlite3VtabFinishParse(Parse*, Token*); void sqlite3VtabArgInit(Parse*); void sqlite3VtabArgExtend(Parse*, Token*); diff --git a/src/status.c b/src/status.c index b23238b..04b7656 100644 --- a/src/status.c +++ b/src/status.c @@ -224,10 +224,12 @@ int sqlite3_db_status( ** to zero. */ case SQLITE_DBSTATUS_CACHE_HIT: - case SQLITE_DBSTATUS_CACHE_MISS: { + case SQLITE_DBSTATUS_CACHE_MISS: + case SQLITE_DBSTATUS_CACHE_WRITE:{ int i; int nRet = 0; assert( SQLITE_DBSTATUS_CACHE_MISS==SQLITE_DBSTATUS_CACHE_HIT+1 ); + assert( SQLITE_DBSTATUS_CACHE_WRITE==SQLITE_DBSTATUS_CACHE_HIT+2 ); for(i=0; i<db->nDb; i++){ if( db->aDb[i].pBt ){ diff --git a/src/tclsqlite.c b/src/tclsqlite.c index c8f0fbd..51f8c51 100644 --- a/src/tclsqlite.c +++ b/src/tclsqlite.c @@ -1163,7 +1163,7 @@ static int dbPrepareAndBind( memset(pPreStmt, 0, nByte); pPreStmt->pStmt = pStmt; - pPreStmt->nSql = (*pzOut - zSql); + pPreStmt->nSql = (int)(*pzOut - zSql); pPreStmt->zSql = sqlite3_sql(pStmt); pPreStmt->apParm = (Tcl_Obj **)&pPreStmt[1]; #ifdef SQLITE_TEST @@ -3001,6 +3001,14 @@ static int DbMain(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){ }else{ flags &= ~SQLITE_OPEN_FULLMUTEX; } + }else if( strcmp(zArg, "-uri")==0 ){ + int b; + if( Tcl_GetBooleanFromObj(interp, objv[i+1], &b) ) return TCL_ERROR; + if( b ){ + flags |= SQLITE_OPEN_URI; + }else{ + flags &= ~SQLITE_OPEN_URI; + } }else{ Tcl_AppendResult(interp, "unknown option: ", zArg, (char*)0); return TCL_ERROR; @@ -3009,7 +3017,7 @@ static int DbMain(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){ if( objc<3 || (objc&1)!=1 ){ Tcl_WrongNumArgs(interp, 1, objv, "HANDLE FILENAME ?-vfs VFSNAME? ?-readonly BOOLEAN? ?-create BOOLEAN?" - " ?-nomutex BOOLEAN? ?-fullmutex BOOLEAN?" + " ?-nomutex BOOLEAN? ?-fullmutex BOOLEAN? ?-uri BOOLEAN?" #ifdef SQLITE_HAS_CODEC " ?-key CODECKEY?" #endif @@ -3100,23 +3108,19 @@ EXTERN int Sqlite3_Init(Tcl_Interp *interp){ return TCL_OK; } EXTERN int Tclsqlite3_Init(Tcl_Interp *interp){ return Sqlite3_Init(interp); } -EXTERN int Sqlite3_SafeInit(Tcl_Interp *interp){ return TCL_OK; } -EXTERN int Tclsqlite3_SafeInit(Tcl_Interp *interp){ return TCL_OK; } EXTERN int Sqlite3_Unload(Tcl_Interp *interp, int flags){ return TCL_OK; } EXTERN int Tclsqlite3_Unload(Tcl_Interp *interp, int flags){ return TCL_OK; } -EXTERN int Sqlite3_SafeUnload(Tcl_Interp *interp, int flags){ return TCL_OK; } -EXTERN int Tclsqlite3_SafeUnload(Tcl_Interp *interp, int flags){ return TCL_OK;} +/* Because it accesses the file-system and uses persistent state, SQLite +** is not considered appropriate for safe interpreters. Hence, we deliberately +** omit the _SafeInit() interfaces. +*/ #ifndef SQLITE_3_SUFFIX_ONLY int Sqlite_Init(Tcl_Interp *interp){ return Sqlite3_Init(interp); } int Tclsqlite_Init(Tcl_Interp *interp){ return Sqlite3_Init(interp); } -int Sqlite_SafeInit(Tcl_Interp *interp){ return TCL_OK; } -int Tclsqlite_SafeInit(Tcl_Interp *interp){ return TCL_OK; } int Sqlite_Unload(Tcl_Interp *interp, int flags){ return TCL_OK; } int Tclsqlite_Unload(Tcl_Interp *interp, int flags){ return TCL_OK; } -int Sqlite_SafeUnload(Tcl_Interp *interp, int flags){ return TCL_OK; } -int Tclsqlite_SafeUnload(Tcl_Interp *interp, int flags){ return TCL_OK;} #endif #ifdef TCLSH @@ -3466,7 +3470,7 @@ static int md5file_cmd(void*cd, Tcl_Interp*interp, int argc, const char **argv){ MD5Init(&ctx); for(;;){ int n; - n = fread(zBuf, 1, sizeof(zBuf), in); + n = (int)fread(zBuf, 1, sizeof(zBuf), in); if( n<=0 ) break; MD5Update(&ctx, (unsigned char*)zBuf, (unsigned)n); } @@ -3512,7 +3516,7 @@ static void md5step(sqlite3_context *context, int argc, sqlite3_value **argv){ for(i=0; i<argc; i++){ const char *zData = (char*)sqlite3_value_text(argv[i]); if( zData ){ - MD5Update(p, (unsigned char*)zData, strlen(zData)); + MD5Update(p, (unsigned char*)zData, (int)strlen(zData)); } } } diff --git a/src/test1.c b/src/test1.c index 2634252..6849b5c 100644 --- a/src/test1.c +++ b/src/test1.c @@ -668,6 +668,7 @@ static int test_key( int argc, /* Number of arguments */ char **argv /* Text of each argument */ ){ +#ifdef SQLITE_HAS_CODEC sqlite3 *db; const char *zKey; int nKey; @@ -679,7 +680,6 @@ static int test_key( if( getDbPointer(interp, argv[1], &db) ) return TCL_ERROR; zKey = argv[2]; nKey = strlen(zKey); -#ifdef SQLITE_HAS_CODEC sqlite3_key(db, zKey, nKey); #endif return TCL_OK; @@ -696,6 +696,7 @@ static int test_rekey( int argc, /* Number of arguments */ char **argv /* Text of each argument */ ){ +#ifdef SQLITE_HAS_CODEC sqlite3 *db; const char *zKey; int nKey; @@ -707,7 +708,6 @@ static int test_rekey( if( getDbPointer(interp, argv[1], &db) ) return TCL_ERROR; zKey = argv[2]; nKey = strlen(zKey); -#ifdef SQLITE_HAS_CODEC sqlite3_rekey(db, zKey, nKey); #endif return TCL_OK; @@ -800,7 +800,7 @@ struct dstr { ** Append text to a dstr */ static void dstrAppend(struct dstr *p, const char *z, int divider){ - int n = strlen(z); + int n = (int)strlen(z); if( p->nUsed + n + 2 > p->nAlloc ){ char *zNew; p->nAlloc = p->nAlloc*2 + n + 200; @@ -2331,6 +2331,33 @@ static int test_stmt_readonly( } /* +** Usage: sqlite3_stmt_busy STMT +** +** Return true if STMT is a non-NULL pointer to a statement +** that has been stepped but not to completion. +*/ +static int test_stmt_busy( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + sqlite3_stmt *pStmt; + int rc; + + if( objc!=2 ){ + Tcl_AppendResult(interp, "wrong # args: should be \"", + Tcl_GetStringFromObj(objv[0], 0), " STMT", 0); + return TCL_ERROR; + } + + if( getStmtPointer(interp, Tcl_GetString(objv[1]), &pStmt) ) return TCL_ERROR; + rc = sqlite3_stmt_busy(pStmt); + Tcl_SetObjResult(interp, Tcl_NewBooleanObj(rc)); + return TCL_OK; +} + +/* ** Usage: uses_stmt_journal STMT ** ** Return true if STMT uses a statement journal. @@ -2342,7 +2369,6 @@ static int uses_stmt_journal( Tcl_Obj *CONST objv[] ){ sqlite3_stmt *pStmt; - int rc; if( objc!=2 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", @@ -2351,7 +2377,7 @@ static int uses_stmt_journal( } if( getStmtPointer(interp, Tcl_GetString(objv[1]), &pStmt) ) return TCL_ERROR; - rc = sqlite3_stmt_readonly(pStmt); + sqlite3_stmt_readonly(pStmt); Tcl_SetObjResult(interp, Tcl_NewBooleanObj(((Vdbe *)pStmt)->usesStmtJournal)); return TCL_OK; } @@ -3237,7 +3263,7 @@ static int test_bind_text16( char *value; int rc; - void (*xDel)() = (objc==6?SQLITE_STATIC:SQLITE_TRANSIENT); + void (*xDel)(void*) = (objc==6?SQLITE_STATIC:SQLITE_TRANSIENT); Tcl_Obj *oStmt = objv[objc-4]; Tcl_Obj *oN = objv[objc-3]; Tcl_Obj *oString = objv[objc-2]; @@ -3583,10 +3609,10 @@ static int test_prepare( if( sqlite3TestErrCode(interp, db, rc) ) return TCL_ERROR; if( zTail && objc>=5 ){ if( bytes>=0 ){ - bytes = bytes - (zTail-zSql); + bytes = bytes - (int)(zTail-zSql); } - if( strlen(zTail)<bytes ){ - bytes = strlen(zTail); + if( (int)strlen(zTail)<bytes ){ + bytes = (int)strlen(zTail); } Tcl_ObjSetVar2(interp, objv[4], 0, Tcl_NewStringObj(zTail, bytes), 0); } @@ -3641,7 +3667,7 @@ static int test_prepare_v2( if( sqlite3TestErrCode(interp, db, rc) ) return TCL_ERROR; if( zTail && objc>=5 ){ if( bytes>=0 ){ - bytes = bytes - (zTail-zSql); + bytes = bytes - (int)(zTail-zSql); } Tcl_ObjSetVar2(interp, objv[4], 0, Tcl_NewStringObj(zTail, bytes), 0); } @@ -3742,7 +3768,7 @@ static int test_prepare16( if( objc>=5 ){ if( zTail ){ - objlen = objlen - ((u8 *)zTail-(u8 *)zSql); + objlen = objlen - (int)((u8 *)zTail-(u8 *)zSql); }else{ objlen = 0; } @@ -3802,7 +3828,7 @@ static int test_prepare16_v2( if( objc>=5 ){ if( zTail ){ - objlen = objlen - ((u8 *)zTail-(u8 *)zSql); + objlen = objlen - (int)((u8 *)zTail-(u8 *)zSql); }else{ objlen = 0; } @@ -3831,7 +3857,6 @@ static int test_open( ){ const char *zFilename; sqlite3 *db; - int rc; char zBuf[100]; if( objc!=3 && objc!=2 && objc!=1 ){ @@ -3841,7 +3866,7 @@ static int test_open( } zFilename = objc>1 ? Tcl_GetString(objv[1]) : 0; - rc = sqlite3_open(zFilename, &db); + sqlite3_open(zFilename, &db); if( sqlite3TestMakePointerStr(interp, zBuf, db) ) return TCL_ERROR; Tcl_AppendResult(interp, zBuf, 0); @@ -3930,7 +3955,6 @@ static int test_open16( #ifndef SQLITE_OMIT_UTF16 const void *zFilename; sqlite3 *db; - int rc; char zBuf[100]; if( objc!=3 ){ @@ -3940,7 +3964,7 @@ static int test_open16( } zFilename = Tcl_GetByteArrayFromObj(objv[1], 0); - rc = sqlite3_open16(zFilename, &db); + sqlite3_open16(zFilename, &db); if( sqlite3TestMakePointerStr(interp, zBuf, db) ) return TCL_ERROR; Tcl_AppendResult(interp, zBuf, 0); @@ -4593,6 +4617,78 @@ static int test_release_memory( return TCL_OK; } + +/* +** Usage: sqlite3_db_release_memory DB +** +** Attempt to release memory currently held by database DB. Return the +** result code (which in the current implementation is always zero). +*/ +static int test_db_release_memory( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + sqlite3 *db; + int rc; + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB"); + return TCL_ERROR; + } + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; + rc = sqlite3_db_release_memory(db); + Tcl_SetObjResult(interp, Tcl_NewIntObj(rc)); + return TCL_OK; +} + +/* +** Usage: sqlite3_db_filename DB DBNAME +** +** Return the name of a file associated with a database. +*/ +static int test_db_filename( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + sqlite3 *db; + const char *zDbName; + if( objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB DBNAME"); + return TCL_ERROR; + } + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; + zDbName = Tcl_GetString(objv[2]); + Tcl_AppendResult(interp, sqlite3_db_filename(db, zDbName), (void*)0); + return TCL_OK; +} + +/* +** Usage: sqlite3_db_readonly DB DBNAME +** +** Return 1 or 0 if DBNAME is readonly or not. Return -1 if DBNAME does +** not exist. +*/ +static int test_db_readonly( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + sqlite3 *db; + const char *zDbName; + if( objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB DBNAME"); + return TCL_ERROR; + } + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; + zDbName = Tcl_GetString(objv[2]); + Tcl_SetObjResult(interp, Tcl_NewIntObj(sqlite3_db_readonly(db, zDbName))); + return TCL_OK; +} + /* ** Usage: sqlite3_soft_heap_limit ?N? ** @@ -5039,8 +5135,6 @@ static int file_control_lockproxy_test( Tcl_Obj *CONST objv[] /* Command arguments */ ){ sqlite3 *db; - const char *zPwd; - int nPwd; if( objc!=3 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", @@ -5050,7 +5144,6 @@ static int file_control_lockproxy_test( if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ){ return TCL_ERROR; } - zPwd = Tcl_GetStringFromObj(objv[2], &nPwd); #if !defined(SQLITE_ENABLE_LOCKING_STYLE) # if defined(__APPLE__) @@ -5063,8 +5156,11 @@ static int file_control_lockproxy_test( { char *testPath; int rc; + int nPwd; + const char *zPwd; char proxyPath[400]; + zPwd = Tcl_GetStringFromObj(objv[2], &nPwd); if( sizeof(proxyPath)<nPwd+20 ){ Tcl_AppendResult(interp, "PWD too big", (void*)0); return TCL_ERROR; @@ -5160,6 +5256,71 @@ static int file_control_persist_wal( return TCL_OK; } +/* +** tclcmd: file_control_powersafe_overwrite DB PSOW-FLAG +** +** This TCL command runs the sqlite3_file_control interface with +** the SQLITE_FCNTL_POWERSAFE_OVERWRITE opcode. +*/ +static int file_control_powersafe_overwrite( + ClientData clientData, /* Pointer to sqlite3_enable_XXX function */ + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int objc, /* Number of arguments */ + Tcl_Obj *CONST objv[] /* Command arguments */ +){ + sqlite3 *db; + int rc; + int b; + char z[100]; + + if( objc!=3 ){ + Tcl_AppendResult(interp, "wrong # args: should be \"", + Tcl_GetStringFromObj(objv[0], 0), " DB FLAG", 0); + return TCL_ERROR; + } + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ){ + return TCL_ERROR; + } + if( Tcl_GetIntFromObj(interp, objv[2], &b) ) return TCL_ERROR; + rc = sqlite3_file_control(db,NULL,SQLITE_FCNTL_POWERSAFE_OVERWRITE,(void*)&b); + sqlite3_snprintf(sizeof(z), z, "%d %d", rc, b); + Tcl_AppendResult(interp, z, (char*)0); + return TCL_OK; +} + + +/* +** tclcmd: file_control_vfsname DB ?AUXDB? +** +** Return a string that describes the stack of VFSes. +*/ +static int file_control_vfsname( + ClientData clientData, /* Pointer to sqlite3_enable_XXX function */ + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int objc, /* Number of arguments */ + Tcl_Obj *CONST objv[] /* Command arguments */ +){ + sqlite3 *db; + const char *zDbName = "main"; + char *zVfsName = 0; + + if( objc!=2 && objc!=3 ){ + Tcl_AppendResult(interp, "wrong # args: should be \"", + Tcl_GetStringFromObj(objv[0], 0), " DB ?AUXDB?", 0); + return TCL_ERROR; + } + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ){ + return TCL_ERROR; + } + if( objc==3 ){ + zDbName = Tcl_GetString(objv[2]); + } + sqlite3_file_control(db, zDbName, SQLITE_FCNTL_VFSNAME,(void*)&zVfsName); + Tcl_AppendResult(interp, zVfsName, (char*)0); + sqlite3_free(zVfsName); + return TCL_OK; +} + /* ** tclcmd: sqlite3_vfs_list @@ -5655,6 +5816,7 @@ struct win32FileLocker { #if SQLITE_OS_WIN +#include <process.h> /* ** The background thread that does file locking. */ @@ -5912,9 +6074,13 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ { "sqlite3_sql", test_sql ,0 }, { "sqlite3_next_stmt", test_next_stmt ,0 }, { "sqlite3_stmt_readonly", test_stmt_readonly ,0 }, + { "sqlite3_stmt_busy", test_stmt_busy ,0 }, { "uses_stmt_journal", uses_stmt_journal ,0 }, { "sqlite3_release_memory", test_release_memory, 0}, + { "sqlite3_db_release_memory", test_db_release_memory, 0}, + { "sqlite3_db_filename", test_db_filename, 0}, + { "sqlite3_db_readonly", test_db_readonly, 0}, { "sqlite3_soft_heap_limit", test_soft_heap_limit, 0}, { "sqlite3_thread_cleanup", test_thread_cleanup, 0}, { "sqlite3_pager_refcounts", test_pager_refcounts, 0}, @@ -5963,7 +6129,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ #endif #ifdef SQLITE_ENABLE_COLUMN_METADATA {"sqlite3_column_database_name16", - test_stmt_utf16, sqlite3_column_database_name16}, + test_stmt_utf16, (void*)sqlite3_column_database_name16}, {"sqlite3_column_table_name16", test_stmt_utf16, (void*)sqlite3_column_table_name16}, {"sqlite3_column_origin_name16", test_stmt_utf16, (void*)sqlite3_column_origin_name16}, #endif @@ -5982,6 +6148,8 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ { "file_control_sizehint_test", file_control_sizehint_test, 0 }, { "file_control_win32_av_retry", file_control_win32_av_retry, 0 }, { "file_control_persist_wal", file_control_persist_wal, 0 }, + { "file_control_powersafe_overwrite",file_control_powersafe_overwrite,0}, + { "file_control_vfsname", file_control_vfsname, 0 }, { "sqlite3_vfs_list", vfs_list, 0 }, { "sqlite3_create_function_v2", test_create_function_v2, 0 }, diff --git a/src/test2.c b/src/test2.c index fa7dd76..8acdf6f 100644 --- a/src/test2.c +++ b/src/test2.c @@ -537,6 +537,8 @@ static int fake_big_file( int rc; int n; i64 offset; + char *zFile; + int nFile; if( argc!=3 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], " N-MEGABYTES FILE\"", 0); @@ -545,17 +547,24 @@ static int fake_big_file( if( Tcl_GetInt(interp, argv[1], &n) ) return TCL_ERROR; pVfs = sqlite3_vfs_find(0); - rc = sqlite3OsOpenMalloc(pVfs, argv[2], &fd, + nFile = (int)strlen(argv[2]); + zFile = sqlite3_malloc( nFile+2 ); + if( zFile==0 ) return TCL_ERROR; + memcpy(zFile, argv[2], nFile+1); + zFile[nFile+1] = 0; + rc = sqlite3OsOpenMalloc(pVfs, zFile, &fd, (SQLITE_OPEN_CREATE|SQLITE_OPEN_READWRITE|SQLITE_OPEN_MAIN_DB), 0 ); if( rc ){ Tcl_AppendResult(interp, "open failed: ", errorName(rc), 0); + sqlite3_free(zFile); return TCL_ERROR; } offset = n; offset *= 1024*1024; rc = sqlite3OsWrite(fd, "Hello, World!", 14, offset); sqlite3OsCloseFree(fd); + sqlite3_free(zFile); if( rc ){ Tcl_AppendResult(interp, "write failed: ", errorName(rc), 0); return TCL_ERROR; diff --git a/src/test3.c b/src/test3.c index 4eabdcc..e460c42 100644 --- a/src/test3.c +++ b/src/test3.c @@ -66,6 +66,8 @@ static int btree_open( Btree *pBt; int rc, nCache; char zBuf[100]; + int n; + char *zFilename; if( argc!=3 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], " FILENAME NCACHE FLAGS\"", 0); @@ -78,8 +80,14 @@ static int btree_open( sDb.mutex = sqlite3MutexAlloc(SQLITE_MUTEX_RECURSIVE); sqlite3_mutex_enter(sDb.mutex); } - rc = sqlite3BtreeOpen(sDb.pVfs, argv[1], &sDb, &pBt, 0, + n = (int)strlen(argv[1]); + zFilename = sqlite3_malloc( n+2 ); + if( zFilename==0 ) return TCL_ERROR; + memcpy(zFilename, argv[1], n+1); + zFilename[n+1] = 0; + rc = sqlite3BtreeOpen(sDb.pVfs, zFilename, &sDb, &pBt, 0, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_MAIN_DB); + sqlite3_free(zFilename); if( rc!=SQLITE_OK ){ Tcl_AppendResult(interp, errorName(rc), 0); return TCL_ERROR; @@ -457,7 +465,7 @@ static int btree_varint_test( if( Tcl_GetInt(interp, argv[4], (int*)&incr) ) return TCL_ERROR; in = start; in *= mult; - for(i=0; i<count; i++){ + for(i=0; i<(int)count; i++){ char zErr[200]; n1 = putVarint(zBuf, in); if( n1>9 || n1<1 ){ diff --git a/src/test5.c b/src/test5.c index 504fdb8..303d120 100644 --- a/src/test5.c +++ b/src/test5.c @@ -64,7 +64,6 @@ static int test_value_overhead( int repeat_count; int i; Mem val; - const char *zVal; if( objc!=3 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", @@ -82,7 +81,7 @@ static int test_value_overhead( for(i=0; i<repeat_count; i++){ if( do_calls ){ - zVal = (char*)sqlite3_value_text(&val); + sqlite3_value_text(&val); } } diff --git a/src/test6.c b/src/test6.c index 23fb14c..bae6b65 100644 --- a/src/test6.c +++ b/src/test6.c @@ -177,7 +177,7 @@ static int writeDbFile(CrashFile *p, u8 *z, i64 iAmt, i64 iOff){ iSkip = 512; } if( (iAmt-iSkip)>0 ){ - rc = sqlite3OsWrite(p->pRealFile, &z[iSkip], iAmt-iSkip, iOff+iSkip); + rc = sqlite3OsWrite(p->pRealFile, &z[iSkip], (int)(iAmt-iSkip), iOff+iSkip); } return rc; } @@ -306,8 +306,8 @@ static int writeListSync(CrashFile *pFile, int isCrash){ } case 3: { /* Trash sectors */ u8 *zGarbage; - int iFirst = (pWrite->iOffset/g.iSectorSize); - int iLast = (pWrite->iOffset+pWrite->nBuf-1)/g.iSectorSize; + int iFirst = (int)(pWrite->iOffset/g.iSectorSize); + int iLast = (int)((pWrite->iOffset+pWrite->nBuf-1)/g.iSectorSize); assert(pWrite->zBuf); @@ -430,7 +430,7 @@ static int cfWrite( ){ CrashFile *pCrash = (CrashFile *)pFile; if( iAmt+iOfst>pCrash->iSize ){ - pCrash->iSize = iAmt+iOfst; + pCrash->iSize = (int)(iAmt+iOfst); } while( pCrash->iSize>pCrash->nData ){ u8 *zNew; @@ -454,7 +454,7 @@ static int cfTruncate(sqlite3_file *pFile, sqlite_int64 size){ CrashFile *pCrash = (CrashFile *)pFile; assert(size>=0); if( pCrash->iSize>size ){ - pCrash->iSize = size; + pCrash->iSize = (int)size; } return writeListAppend(pFile, size, 0, 0); } @@ -468,15 +468,23 @@ static int cfSync(sqlite3_file *pFile, int flags){ const char *zName = pCrash->zName; const char *zCrashFile = g.zCrashFile; - int nName = strlen(zName); - int nCrashFile = strlen(zCrashFile); + int nName = (int)strlen(zName); + int nCrashFile = (int)strlen(zCrashFile); if( nCrashFile>0 && zCrashFile[nCrashFile-1]=='*' ){ nCrashFile--; if( nName>nCrashFile ) nName = nCrashFile; } +#ifdef TRACE_CRASHTEST + printf("cfSync(): nName = %d, nCrashFile = %d, zName = %s, zCrashFile = %s\n", + nName, nCrashFile, zName, zCrashFile); +#endif + if( nName==nCrashFile && 0==memcmp(zName, zCrashFile, nName) ){ +#ifdef TRACE_CRASHTEST + printf("cfSync(): name matched, g.iCrash = %d\n", g.iCrash); +#endif if( (--g.iCrash)==0 ) isCrash = 1; } @@ -510,7 +518,7 @@ static int cfFileControl(sqlite3_file *pFile, int op, void *pArg){ i64 nByte = *(i64 *)pArg; if( nByte>pCrash->iSize ){ if( SQLITE_OK==writeListAppend(pFile, nByte, 0, 0) ){ - pCrash->iSize = nByte; + pCrash->iSize = (int)nByte; } } return SQLITE_OK; @@ -627,11 +635,11 @@ static int cfOpen( iChunk = PENDING_BYTE; } memset(pWrapper->zData, 0, pWrapper->nData); - rc = sqlite3OsRead(pReal, pWrapper->zData, iChunk, 0); + rc = sqlite3OsRead(pReal, pWrapper->zData, (int)iChunk, 0); if( SQLITE_OK==rc && pWrapper->iSize>(PENDING_BYTE+512) && isDb ){ i64 iOff = PENDING_BYTE+512; iChunk = pWrapper->iSize - iOff; - rc = sqlite3OsRead(pReal, &pWrapper->zData[iOff], iChunk, iOff); + rc = sqlite3OsRead(pReal, &pWrapper->zData[iOff], (int)iChunk, iOff); } }else{ rc = SQLITE_NOMEM; @@ -705,17 +713,18 @@ static int processDevSymArgs( char *zName; int iValue; } aFlag[] = { - { "atomic", SQLITE_IOCAP_ATOMIC }, - { "atomic512", SQLITE_IOCAP_ATOMIC512 }, - { "atomic1k", SQLITE_IOCAP_ATOMIC1K }, - { "atomic2k", SQLITE_IOCAP_ATOMIC2K }, - { "atomic4k", SQLITE_IOCAP_ATOMIC4K }, - { "atomic8k", SQLITE_IOCAP_ATOMIC8K }, - { "atomic16k", SQLITE_IOCAP_ATOMIC16K }, - { "atomic32k", SQLITE_IOCAP_ATOMIC32K }, - { "atomic64k", SQLITE_IOCAP_ATOMIC64K }, - { "sequential", SQLITE_IOCAP_SEQUENTIAL }, - { "safe_append", SQLITE_IOCAP_SAFE_APPEND }, + { "atomic", SQLITE_IOCAP_ATOMIC }, + { "atomic512", SQLITE_IOCAP_ATOMIC512 }, + { "atomic1k", SQLITE_IOCAP_ATOMIC1K }, + { "atomic2k", SQLITE_IOCAP_ATOMIC2K }, + { "atomic4k", SQLITE_IOCAP_ATOMIC4K }, + { "atomic8k", SQLITE_IOCAP_ATOMIC8K }, + { "atomic16k", SQLITE_IOCAP_ATOMIC16K }, + { "atomic32k", SQLITE_IOCAP_ATOMIC32K }, + { "atomic64k", SQLITE_IOCAP_ATOMIC64K }, + { "sequential", SQLITE_IOCAP_SEQUENTIAL }, + { "safe_append", SQLITE_IOCAP_SAFE_APPEND }, + { "powersafe_overwrite", SQLITE_IOCAP_POWERSAFE_OVERWRITE }, { 0, 0 } }; diff --git a/src/test8.c b/src/test8.c index 283d790..ba7e373 100644 --- a/src/test8.c +++ b/src/test8.c @@ -192,7 +192,7 @@ static int getColumnNames( rc = SQLITE_NOMEM; goto out; } - nBytes += strlen(zName)+1; + nBytes += (int)strlen(zName)+1; } aCol = (char **)sqlite3MallocZero(nBytes); if( !aCol ){ @@ -831,13 +831,10 @@ static int echoBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ if( !isIgnoreUsable && !pConstraint->usable ) continue; iCol = pConstraint->iColumn; - if( pVtab->aIndex[iCol] || iCol<0 ){ - char *zCol = pVtab->aCol[iCol]; + if( iCol<0 || pVtab->aIndex[iCol] ){ + char *zCol = iCol>=0 ? pVtab->aCol[iCol] : "rowid"; char *zOp = 0; useIdx = 1; - if( iCol<0 ){ - zCol = "rowid"; - } switch( pConstraint->op ){ case SQLITE_INDEX_CONSTRAINT_EQ: zOp = "="; break; @@ -870,13 +867,12 @@ static int echoBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ ** on a column that this virtual table has an index for, then consume ** the ORDER BY clause. */ - if( pIdxInfo->nOrderBy==1 && pVtab->aIndex[pIdxInfo->aOrderBy->iColumn] ){ + if( pIdxInfo->nOrderBy==1 && ( + pIdxInfo->aOrderBy->iColumn<0 || + pVtab->aIndex[pIdxInfo->aOrderBy->iColumn]) ){ int iCol = pIdxInfo->aOrderBy->iColumn; - char *zCol = pVtab->aCol[iCol]; + char *zCol = iCol>=0 ? pVtab->aCol[iCol] : "rowid"; char *zDir = pIdxInfo->aOrderBy->desc?"DESC":"ASC"; - if( iCol<0 ){ - zCol = "rowid"; - } zNew = sqlite3_mprintf(" ORDER BY %s %s", zCol, zDir); string_concat(&zQuery, zNew, 1, &rc); pIdxInfo->orderByConsumed = 1; @@ -1221,7 +1217,7 @@ static int echoRename(sqlite3_vtab *vtab, const char *zNewName){ } if( p->isPattern ){ - int nThis = strlen(p->zThis); + int nThis = (int)strlen(p->zThis); char *zSql = sqlite3_mprintf("ALTER TABLE %s RENAME TO %s%s", p->zTableName, zNewName, &p->zTableName[nThis] ); diff --git a/src/test_config.c b/src/test_config.c index ce72f87..f096ebf 100644 --- a/src/test_config.c +++ b/src/test_config.c @@ -37,6 +37,14 @@ ** procedures use this to determine when tests should be omitted. */ static void set_options(Tcl_Interp *interp){ +#ifdef HAVE_MALLOC_USABLE_SIZE + Tcl_SetVar2(interp, "sqlite_options", "malloc_usable_size", "1", + TCL_GLOBAL_ONLY); +#else + Tcl_SetVar2(interp, "sqlite_options", "malloc_usable_size", "0", + TCL_GLOBAL_ONLY); +#endif + #ifdef SQLITE_32BIT_ROWID Tcl_SetVar2(interp, "sqlite_options", "rowid32", "1", TCL_GLOBAL_ONLY); #else @@ -412,6 +420,12 @@ Tcl_SetVar2(interp, "sqlite_options", "long_double", Tcl_SetVar2(interp, "sqlite_options", "rtree", "0", TCL_GLOBAL_ONLY); #endif +#ifdef SQLITE_RTREE_INT_ONLY + Tcl_SetVar2(interp, "sqlite_options", "rtree_int_only", "1", TCL_GLOBAL_ONLY); +#else + Tcl_SetVar2(interp, "sqlite_options", "rtree_int_only", "0", TCL_GLOBAL_ONLY); +#endif + #ifdef SQLITE_OMIT_SCHEMA_PRAGMAS Tcl_SetVar2(interp, "sqlite_options", "schema_pragmas", "0", TCL_GLOBAL_ONLY); #else diff --git a/src/test_func.c b/src/test_func.c index a123943..c4fe351 100644 --- a/src/test_func.c +++ b/src/test_func.c @@ -149,13 +149,13 @@ static void test_destructor_count( ** arguments. It returns the text value returned by the sqlite3_errmsg16() ** API function. */ -#ifndef SQLITE_OMIT_BUILTIN_TEST
+#ifndef SQLITE_OMIT_BUILTIN_TEST void sqlite3BeginBenignMalloc(void); void sqlite3EndBenignMalloc(void); -#else
- #define sqlite3BeginBenignMalloc()
- #define sqlite3EndBenignMalloc()
-#endif
+#else + #define sqlite3BeginBenignMalloc() + #define sqlite3EndBenignMalloc() +#endif static void test_agg_errmsg16_step(sqlite3_context *a, int b,sqlite3_value **c){ } static void test_agg_errmsg16_final(sqlite3_context *ctx){ @@ -202,7 +202,7 @@ static void test_auxdata( }else { zRet[i*2] = '0'; } - n = strlen(z) + 1; + n = (int)strlen(z) + 1; zAux = testContextMalloc(pCtx, n); if( zAux ){ memcpy(zAux, z, n); diff --git a/src/test_fuzzer.c b/src/test_fuzzer.c index cf59257..10496f2 100644 --- a/src/test_fuzzer.c +++ b/src/test_fuzzer.c @@ -10,43 +10,58 @@ ** ************************************************************************* ** -** Code for demonstartion virtual table that generates variations +** Code for a demonstration virtual table that generates variations ** on an input word at increasing edit distances from the original. ** ** A fuzzer virtual table is created like this: ** -** CREATE VIRTUAL TABLE temp.f USING fuzzer; +** CREATE VIRTUAL TABLE f USING fuzzer(<fuzzer-data-table>); ** -** The name of the new virtual table in the example above is "f". -** Note that all fuzzer virtual tables must be TEMP tables. The -** "temp." prefix in front of the table name is required when the -** table is being created. The "temp." prefix can be omitted when -** using the table as long as the name is unambiguous. +** When it is created, the new fuzzer table must be supplied with the +** name of a "fuzzer data table", which must reside in the same database +** file as the new fuzzer table. The fuzzer data table contains the various +** transformations and their costs that the fuzzer logic uses to generate +** variations. ** -** Before being used, the fuzzer needs to be programmed by giving it -** character transformations and a cost associated with each transformation. -** Examples: -** -** INSERT INTO f(cFrom,cTo,Cost) VALUES('','a',100); -** -** The above statement says that the cost of inserting a letter 'a' is -** 100. (All costs are integers. We recommend that costs be scaled so -** that the average cost is around 100.) -** -** INSERT INTO f(cFrom,cTo,Cost) VALUES('b','',87); +** The fuzzer data table must contain exactly four columns (more precisely, +** the statement "SELECT * FROM <fuzzer_data_table>" must return records +** that consist of four columns). It does not matter what the columns are +** named. ** -** The above statement says that the cost of deleting a single letter -** 'b' is 87. +** Each row in the fuzzer data table represents a single character +** transformation. The left most column of the row (column 0) contains an +** integer value - the identifier of the ruleset to which the transformation +** rule belongs (see "MULTIPLE RULE SETS" below). The second column of the +** row (column 0) contains the input character or characters. The third +** column contains the output character or characters. And the fourth column +** contains the integer cost of making the transformation. For example: ** -** INSERT INTO f(cFrom,cTo,Cost) VALUES('o','oe',38); -** INSERT INTO f(cFrom,cTo,Cost) VALUES('oe','o',40); +** CREATE TABLE f_data(ruleset, cFrom, cTo, Cost); +** INSERT INTO f_data(ruleset, cFrom, cTo, Cost) VALUES(0, '', 'a', 100); +** INSERT INTO f_data(ruleset, cFrom, cTo, Cost) VALUES(0, 'b', '', 87); +** INSERT INTO f_data(ruleset, cFrom, cTo, Cost) VALUES(0, 'o', 'oe', 38); +** INSERT INTO f_data(ruleset, cFrom, cTo, Cost) VALUES(0, 'oe', 'o', 40); ** -** This third example says that the cost of transforming the single -** letter "o" into the two-letter sequence "oe" is 38 and that the +** The first row inserted into the fuzzer data table by the SQL script +** above indicates that the cost of inserting a letter 'a' is 100. (All +** costs are integers. We recommend that costs be scaled so that the +** average cost is around 100.) The second INSERT statement creates a rule +** saying that the cost of deleting a single letter 'b' is 87. The third +** and fourth INSERT statements mean that the cost of transforming a +** single letter "o" into the two-letter sequence "oe" is 38 and that the ** cost of transforming "oe" back into "o" is 40. ** -** After all the transformation costs have been set, the fuzzer table -** can be queried as follows: +** The contents of the fuzzer data table are loaded into main memory when +** a fuzzer table is first created, and may be internally reloaded by the +** system at any subsequent time. Therefore, the fuzzer data table should be +** populated before the fuzzer table is created and not modified thereafter. +** If you do need to modify the contents of the fuzzer data table, it is +** recommended that the associated fuzzer table be dropped, the fuzzer data +** table edited, and the fuzzer table recreated within a single transaction. +** Alternatively, the fuzzer data table can be edited then the database +** connection can be closed and reopened. +** +** Once it has been created, the fuzzer table can be queried as follows: ** ** SELECT word, distance FROM f ** WHERE word MATCH 'abcdefg' @@ -61,6 +76,9 @@ ** the one that is returned. In the example, the search is limited to ** strings with a total distance of less than 200. ** +** The fuzzer is a read-only table. Any attempt to DELETE, INSERT, or +** UPDATE on a fuzzer table will throw an error. +** ** It is important to put some kind of a limit on the fuzzer output. This ** can be either in the form of a LIMIT clause at the end of the query, ** or better, a "distance<NNN" constraint where NNN is some number. The @@ -93,7 +111,42 @@ ** ** This last query will show up to 50 words out of the vocabulary that ** match or nearly match the $prefix. +** +** MULTIPLE RULE SETS +** +** Normally, the "ruleset" value associated with all character transformations +** in the fuzzer data table is zero. However, if required, the fuzzer table +** allows multiple rulesets to be defined. Each query uses only a single +** ruleset. This allows, for example, a single fuzzer table to support +** multiple languages. +** +** By default, only the rules from ruleset 0 are used. To specify an +** alternative ruleset, a "ruleset = ?" expression must be added to the +** WHERE clause of a SELECT, where ? is the identifier of the desired +** ruleset. For example: +** +** SELECT vocabulary.w FROM f, vocabulary +** WHERE f.word MATCH $word +** AND f.distance<=200 +** AND f.word=vocabulary.w +** AND f.ruleset=1 -- Specify the ruleset to use here +** LIMIT 20 +** +** If no "ruleset = ?" constraint is specified in the WHERE clause, ruleset +** 0 is used. +** +** LIMITS +** +** The maximum ruleset number is 2147483647. The maximum length of either +** of the strings in the second or third column of the fuzzer data table +** is 50 bytes. The maximum cost on a rule is 1000. */ + +/* If SQLITE_DEBUG is not defined, disable assert statements. */ +#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) +# define NDEBUG +#endif + #include "sqlite3.h" #include <stdlib.h> #include <string.h> @@ -112,10 +165,25 @@ typedef struct fuzzer_seen fuzzer_seen; typedef struct fuzzer_stem fuzzer_stem; /* -** Type of the "cost" of an edit operation. Might be changed to -** "float" or "double" or "sqlite3_int64" in the future. +** Various types. +** +** fuzzer_cost is the "cost" of an edit operation. +** +** fuzzer_len is the length of a matching string. +** +** fuzzer_ruleid is an ruleset identifier. */ typedef int fuzzer_cost; +typedef signed char fuzzer_len; +typedef int fuzzer_ruleid; + +/* +** Limits +*/ +#define FUZZER_MX_LENGTH 50 /* Maximum length of a rule string */ +#define FUZZER_MX_RULEID 2147483647 /* Maximum rule ID */ +#define FUZZER_MX_COST 1000 /* Maximum single-rule cost */ +#define FUZZER_MX_OUTPUT_LENGTH 100 /* Maximum length of an output string */ /* @@ -123,11 +191,12 @@ typedef int fuzzer_cost; ** All rules are kept on a linked list sorted by rCost. */ struct fuzzer_rule { - fuzzer_rule *pNext; /* Next rule in order of increasing rCost */ - fuzzer_cost rCost; /* Cost of this transformation */ - int nFrom, nTo; /* Length of the zFrom and zTo strings */ - char *zFrom; /* Transform from */ - char zTo[4]; /* Transform to (extra space appended) */ + fuzzer_rule *pNext; /* Next rule in order of increasing rCost */ + char *zFrom; /* Transform from */ + fuzzer_cost rCost; /* Cost of this transformation */ + fuzzer_len nFrom, nTo; /* Length of the zFrom and zTo strings */ + fuzzer_ruleid iRuleset; /* The rule set to which this rule belongs */ + char zTo[4]; /* Transform to (extra space appended) */ }; /* @@ -143,13 +212,13 @@ struct fuzzer_rule { */ struct fuzzer_stem { char *zBasis; /* Word being fuzzed */ - int nBasis; /* Length of the zBasis string */ const fuzzer_rule *pRule; /* Current rule to apply */ - int n; /* Apply pRule at this character offset */ - fuzzer_cost rBaseCost; /* Base cost of getting to zBasis */ - fuzzer_cost rCostX; /* Precomputed rBaseCost + pRule->rCost */ fuzzer_stem *pNext; /* Next stem in rCost order */ fuzzer_stem *pHash; /* Next stem with same hash on zBasis */ + fuzzer_cost rBaseCost; /* Base cost of getting to zBasis */ + fuzzer_cost rCostX; /* Precomputed rBaseCost + pRule->rCost */ + fuzzer_len nBasis; /* Length of the zBasis string */ + fuzzer_len n; /* Apply pRule at this character offset */ }; /* @@ -159,7 +228,6 @@ struct fuzzer_vtab { sqlite3_vtab base; /* Base class - must be first */ char *zClassName; /* Name of this class. Default: "fuzzer" */ fuzzer_rule *pRule; /* All active rules in this fuzzer */ - fuzzer_rule *pNewRule; /* New rules to add when last cursor expires */ int nCursor; /* Number of active cursors */ }; @@ -179,54 +247,11 @@ struct fuzzer_cursor { char *zBuf; /* Temporary use buffer */ int nBuf; /* Bytes allocated for zBuf */ int nStem; /* Number of stems allocated */ + int iRuleset; /* Only process rules from this ruleset */ fuzzer_rule nullRule; /* Null rule used first */ fuzzer_stem *apHash[FUZZER_HASH]; /* Hash of previously generated terms */ }; -/* Methods for the fuzzer module */ -static int fuzzerConnect( - sqlite3 *db, - void *pAux, - int argc, const char *const*argv, - sqlite3_vtab **ppVtab, - char **pzErr -){ - fuzzer_vtab *pNew; - int n; - if( strcmp(argv[1],"temp")!=0 ){ - *pzErr = sqlite3_mprintf("%s virtual tables must be TEMP", argv[0]); - return SQLITE_ERROR; - } - n = strlen(argv[0]) + 1; - pNew = sqlite3_malloc( sizeof(*pNew) + n ); - if( pNew==0 ) return SQLITE_NOMEM; - pNew->zClassName = (char*)&pNew[1]; - memcpy(pNew->zClassName, argv[0], n); - sqlite3_declare_vtab(db, "CREATE TABLE x(word,distance,cFrom,cTo,cost)"); - memset(pNew, 0, sizeof(*pNew)); - *ppVtab = &pNew->base; - return SQLITE_OK; -} -/* Note that for this virtual table, the xCreate and xConnect -** methods are identical. */ - -static int fuzzerDisconnect(sqlite3_vtab *pVtab){ - fuzzer_vtab *p = (fuzzer_vtab*)pVtab; - assert( p->nCursor==0 ); - do{ - while( p->pRule ){ - fuzzer_rule *pRule = p->pRule; - p->pRule = pRule->pNext; - sqlite3_free(pRule); - } - p->pRule = p->pNewRule; - p->pNewRule = 0; - }while( p->pRule ); - sqlite3_free(p); - return SQLITE_OK; -} -/* The xDisconnect and xDestroy methods are also the same */ - /* ** The two input rule lists are both sorted in order of increasing ** cost. Merge them together into a single list, sorted by cost, and @@ -256,25 +281,134 @@ static fuzzer_rule *fuzzerMergeRules(fuzzer_rule *pA, fuzzer_rule *pB){ return head.pNext; } +/* +** Statement pStmt currently points to a row in the fuzzer data table. This +** function allocates and populates a fuzzer_rule structure according to +** the content of the row. +** +** If successful, *ppRule is set to point to the new object and SQLITE_OK +** is returned. Otherwise, *ppRule is zeroed, *pzErr may be set to point +** to an error message and an SQLite error code returned. +*/ +static int fuzzerLoadOneRule( + fuzzer_vtab *p, /* Fuzzer virtual table handle */ + sqlite3_stmt *pStmt, /* Base rule on statements current row */ + fuzzer_rule **ppRule, /* OUT: New rule object */ + char **pzErr /* OUT: Error message */ +){ + sqlite3_int64 iRuleset = sqlite3_column_int64(pStmt, 0); + const char *zFrom = (const char *)sqlite3_column_text(pStmt, 1); + const char *zTo = (const char *)sqlite3_column_text(pStmt, 2); + int nCost = sqlite3_column_int(pStmt, 3); + + int rc = SQLITE_OK; /* Return code */ + int nFrom; /* Size of string zFrom, in bytes */ + int nTo; /* Size of string zTo, in bytes */ + fuzzer_rule *pRule = 0; /* New rule object to return */ + + if( zFrom==0 ) zFrom = ""; + if( zTo==0 ) zTo = ""; + nFrom = (int)strlen(zFrom); + nTo = (int)strlen(zTo); + + /* Silently ignore null transformations */ + if( strcmp(zFrom, zTo)==0 ){ + *ppRule = 0; + return SQLITE_OK; + } + + if( nCost<=0 || nCost>FUZZER_MX_COST ){ + *pzErr = sqlite3_mprintf("%s: cost must be between 1 and %d", + p->zClassName, FUZZER_MX_COST + ); + rc = SQLITE_ERROR; + }else + if( nFrom>FUZZER_MX_LENGTH || nTo>FUZZER_MX_LENGTH ){ + *pzErr = sqlite3_mprintf("%s: maximum string length is %d", + p->zClassName, FUZZER_MX_LENGTH + ); + rc = SQLITE_ERROR; + }else + if( iRuleset<0 || iRuleset>FUZZER_MX_RULEID ){ + *pzErr = sqlite3_mprintf("%s: ruleset must be between 0 and %d", + p->zClassName, FUZZER_MX_RULEID + ); + rc = SQLITE_ERROR; + }else{ + + pRule = sqlite3_malloc( sizeof(*pRule) + nFrom + nTo ); + if( pRule==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pRule, 0, sizeof(*pRule)); + pRule->zFrom = &pRule->zTo[nTo+1]; + pRule->nFrom = nFrom; + memcpy(pRule->zFrom, zFrom, nFrom+1); + memcpy(pRule->zTo, zTo, nTo+1); + pRule->nTo = nTo; + pRule->rCost = nCost; + pRule->iRuleset = (int)iRuleset; + } + } + + *ppRule = pRule; + return rc; +} /* -** Open a new fuzzer cursor. +** Load the content of the fuzzer data table into memory. */ -static int fuzzerOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ - fuzzer_vtab *p = (fuzzer_vtab*)pVTab; - fuzzer_cursor *pCur; - pCur = sqlite3_malloc( sizeof(*pCur) ); - if( pCur==0 ) return SQLITE_NOMEM; - memset(pCur, 0, sizeof(*pCur)); - pCur->pVtab = p; - *ppCursor = &pCur->base; - if( p->nCursor==0 && p->pNewRule ){ +static int fuzzerLoadRules( + sqlite3 *db, /* Database handle */ + fuzzer_vtab *p, /* Virtual fuzzer table to configure */ + const char *zDb, /* Database containing rules data */ + const char *zData, /* Table containing rules data */ + char **pzErr /* OUT: Error message */ +){ + int rc = SQLITE_OK; /* Return code */ + char *zSql; /* SELECT used to read from rules table */ + fuzzer_rule *pHead = 0; + + zSql = sqlite3_mprintf("SELECT * FROM %Q.%Q", zDb, zData); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + int rc2; /* finalize() return code */ + sqlite3_stmt *pStmt = 0; + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + if( rc!=SQLITE_OK ){ + *pzErr = sqlite3_mprintf("%s: %s", p->zClassName, sqlite3_errmsg(db)); + }else if( sqlite3_column_count(pStmt)!=4 ){ + *pzErr = sqlite3_mprintf("%s: %s has %d columns, expected 4", + p->zClassName, zData, sqlite3_column_count(pStmt) + ); + rc = SQLITE_ERROR; + }else{ + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + fuzzer_rule *pRule = 0; + rc = fuzzerLoadOneRule(p, pStmt, &pRule, pzErr); + if( pRule ){ + pRule->pNext = pHead; + pHead = pRule; + } + } + } + rc2 = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ) rc = rc2; + } + sqlite3_free(zSql); + + /* All rules are now in a singly linked list starting at pHead. This + ** block sorts them by cost and then sets fuzzer_vtab.pRule to point to + ** point to the head of the sorted list. + */ + if( rc==SQLITE_OK ){ unsigned int i; fuzzer_rule *pX; fuzzer_rule *a[15]; for(i=0; i<sizeof(a)/sizeof(a[0]); i++) a[i] = 0; - while( (pX = p->pNewRule)!=0 ){ - p->pNewRule = pX->pNext; + while( (pX = pHead)!=0 ){ + pHead = pX->pNext; pX->pNext = 0; for(i=0; a[i] && i<sizeof(a)/sizeof(a[0])-1; i++){ pX = fuzzerMergeRules(a[i], pX); @@ -286,7 +420,143 @@ static int fuzzerOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ pX = fuzzerMergeRules(a[i], pX); } p->pRule = fuzzerMergeRules(p->pRule, pX); + }else{ + /* An error has occurred. Setting p->pRule to point to the head of the + ** allocated list ensures that the list will be cleaned up in this case. + */ + assert( p->pRule==0 ); + p->pRule = pHead; } + + return rc; +} + +/* +** This function converts an SQL quoted string into an unquoted string +** and returns a pointer to a buffer allocated using sqlite3_malloc() +** containing the result. The caller should eventually free this buffer +** using sqlite3_free. +** +** Examples: +** +** "abc" becomes abc +** 'xyz' becomes xyz +** [pqr] becomes pqr +** `mno` becomes mno +*/ +static char *fuzzerDequote(const char *zIn){ + int nIn; /* Size of input string, in bytes */ + char *zOut; /* Output (dequoted) string */ + + nIn = (int)strlen(zIn); + zOut = sqlite3_malloc(nIn+1); + if( zOut ){ + char q = zIn[0]; /* Quote character (if any ) */ + + if( q!='[' && q!= '\'' && q!='"' && q!='`' ){ + memcpy(zOut, zIn, nIn+1); + }else{ + int iOut = 0; /* Index of next byte to write to output */ + int iIn; /* Index of next byte to read from input */ + + if( q=='[' ) q = ']'; + for(iIn=1; iIn<nIn; iIn++){ + if( zIn[iIn]==q ) iIn++; + zOut[iOut++] = zIn[iIn]; + } + } + assert( (int)strlen(zOut)<=nIn ); + } + return zOut; +} + +/* +** xDisconnect/xDestroy method for the fuzzer module. +*/ +static int fuzzerDisconnect(sqlite3_vtab *pVtab){ + fuzzer_vtab *p = (fuzzer_vtab*)pVtab; + assert( p->nCursor==0 ); + while( p->pRule ){ + fuzzer_rule *pRule = p->pRule; + p->pRule = pRule->pNext; + sqlite3_free(pRule); + } + sqlite3_free(p); + return SQLITE_OK; +} + +/* +** xConnect/xCreate method for the fuzzer module. Arguments are: +** +** argv[0] -> module name ("fuzzer") +** argv[1] -> database name +** argv[2] -> table name +** argv[3] -> fuzzer rule table name +*/ +static int fuzzerConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + int rc = SQLITE_OK; /* Return code */ + fuzzer_vtab *pNew = 0; /* New virtual table */ + const char *zModule = argv[0]; + const char *zDb = argv[1]; + + if( argc!=4 ){ + *pzErr = sqlite3_mprintf( + "%s: wrong number of CREATE VIRTUAL TABLE arguments", zModule + ); + rc = SQLITE_ERROR; + }else{ + int nModule; /* Length of zModule, in bytes */ + + nModule = (int)strlen(zModule); + pNew = sqlite3_malloc( sizeof(*pNew) + nModule + 1); + if( pNew==0 ){ + rc = SQLITE_NOMEM; + }else{ + char *zTab; /* Dequoted name of fuzzer data table */ + + memset(pNew, 0, sizeof(*pNew)); + pNew->zClassName = (char*)&pNew[1]; + memcpy(pNew->zClassName, zModule, nModule+1); + + zTab = fuzzerDequote(argv[3]); + if( zTab==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = fuzzerLoadRules(db, pNew, zDb, zTab, pzErr); + sqlite3_free(zTab); + } + + if( rc==SQLITE_OK ){ + rc = sqlite3_declare_vtab(db, "CREATE TABLE x(word,distance,ruleset)"); + } + if( rc!=SQLITE_OK ){ + fuzzerDisconnect((sqlite3_vtab *)pNew); + pNew = 0; + } + } + } + + *ppVtab = (sqlite3_vtab *)pNew; + return rc; +} + +/* +** Open a new fuzzer cursor. +*/ +static int fuzzerOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ + fuzzer_vtab *p = (fuzzer_vtab*)pVTab; + fuzzer_cursor *pCur; + pCur = sqlite3_malloc( sizeof(*pCur) ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + pCur->pVtab = p; + *ppCursor = &pCur->base; p->nCursor++; return SQLITE_OK; } @@ -343,8 +613,8 @@ static int fuzzerRender( int *pnBuf /* Size of the buffer */ ){ const fuzzer_rule *pRule = pStem->pRule; - int n; - char *z; + int n; /* Size of output term without nul-term */ + char *z; /* Buffer to assemble output term in */ n = pStem->nBasis + pRule->nTo - pRule->nFrom; if( (*pnBuf)<n+1 ){ @@ -362,6 +632,8 @@ static int fuzzerRender( memcpy(&z[n+pRule->nTo], &pStem->zBasis[n+pRule->nFrom], pStem->nBasis-n-pRule->nFrom+1); } + + assert( z[pStem->nBasis + pRule->nTo - pRule->nFrom]==0 ); return SQLITE_OK; } @@ -424,13 +696,32 @@ static int fuzzerSeen(fuzzer_cursor *pCur, fuzzer_stem *pStem){ } h = fuzzerHash(pCur->zBuf); pLookup = pCur->apHash[h]; - while( pLookup && strcmp(pLookup->zBasis, pCur->zBuf)!=0 ){ + while( pLookup && strcmp(pLookup->zBasis, pCur->zBuf)!=0 ){ pLookup = pLookup->pHash; } return pLookup!=0; } /* +** If argument pRule is NULL, this function returns false. +** +** Otherwise, it returns true if rule pRule should be skipped. A rule +** should be skipped if it does not belong to rule-set iRuleset, or if +** applying it to stem pStem would create a string longer than +** FUZZER_MX_OUTPUT_LENGTH bytes. +*/ +static int fuzzerSkipRule( + const fuzzer_rule *pRule, /* Determine whether or not to skip this */ + fuzzer_stem *pStem, /* Stem rule may be applied to */ + int iRuleset /* Rule-set used by the current query */ +){ + return pRule && ( + (pRule->iRuleset!=iRuleset) + || (pStem->nBasis + pRule->nTo - pRule->nFrom)>FUZZER_MX_OUTPUT_LENGTH + ); +} + +/* ** Advance a fuzzer_stem to its next value. Return 0 if there are ** no more values that can be generated by this fuzzer_stem. Return ** -1 on a memory allocation failure. @@ -438,6 +729,7 @@ static int fuzzerSeen(fuzzer_cursor *pCur, fuzzer_stem *pStem){ static int fuzzerAdvance(fuzzer_cursor *pCur, fuzzer_stem *pStem){ const fuzzer_rule *pRule; while( (pRule = pStem->pRule)!=0 ){ + assert( pRule==&pCur->nullRule || pRule->iRuleset==pCur->iRuleset ); while( pStem->n < pStem->nBasis - pRule->nFrom ){ pStem->n++; if( pRule->nFrom==0 @@ -453,8 +745,11 @@ static int fuzzerAdvance(fuzzer_cursor *pCur, fuzzer_stem *pStem){ } } pStem->n = -1; - pStem->pRule = pRule->pNext; - if( pStem->pRule && fuzzerCost(pStem)>pCur->rLimit ) pStem->pRule = 0; + do{ + pRule = pRule->pNext; + }while( fuzzerSkipRule(pRule, pStem, pCur->iRuleset) ); + pStem->pRule = pRule; + if( pRule && fuzzerCost(pStem)>pCur->rLimit ) pStem->pRule = 0; } return 0; } @@ -572,15 +867,20 @@ static fuzzer_stem *fuzzerNewStem( fuzzer_cost rBaseCost ){ fuzzer_stem *pNew; + fuzzer_rule *pRule; unsigned int h; - pNew = sqlite3_malloc( sizeof(*pNew) + strlen(zWord) + 1 ); + pNew = sqlite3_malloc( sizeof(*pNew) + (int)strlen(zWord) + 1 ); if( pNew==0 ) return 0; memset(pNew, 0, sizeof(*pNew)); pNew->zBasis = (char*)&pNew[1]; - pNew->nBasis = strlen(zWord); + pNew->nBasis = (int)strlen(zWord); memcpy(pNew->zBasis, zWord, pNew->nBasis+1); - pNew->pRule = pCur->pVtab->pRule; + pRule = pCur->pVtab->pRule; + while( fuzzerSkipRule(pRule, pNew, pCur->iRuleset) ){ + pRule = pRule->pNext; + } + pNew->pRule = pRule; pNew->n = -1; pNew->rBaseCost = pNew->rCostX = rBaseCost; h = fuzzerHash(pNew->zBasis); @@ -627,7 +927,10 @@ static int fuzzerNext(sqlite3_vtab_cursor *cur){ ** stem list is the next lowest cost word. */ while( (pStem = pCur->pStem)!=0 ){ - if( fuzzerAdvance(pCur, pStem) ){ + int res = fuzzerAdvance(pCur, pStem); + if( res<0 ){ + return SQLITE_NOMEM; + }else if( res>0 ){ pCur->pStem = 0; pStem = fuzzerInsert(pCur, pStem); if( (rc = fuzzerSeen(pCur, pStem))!=0 ){ @@ -665,30 +968,44 @@ static int fuzzerFilter( int argc, sqlite3_value **argv ){ fuzzer_cursor *pCur = (fuzzer_cursor *)pVtabCursor; - const char *zWord = 0; + const char *zWord = ""; fuzzer_stem *pStem; + int idx; fuzzerClearCursor(pCur, 1); pCur->rLimit = 2147483647; - if( idxNum==1 ){ - zWord = (const char*)sqlite3_value_text(argv[0]); - }else if( idxNum==2 ){ - pCur->rLimit = (fuzzer_cost)sqlite3_value_int(argv[0]); - }else if( idxNum==3 ){ + idx = 0; + if( idxNum & 1 ){ zWord = (const char*)sqlite3_value_text(argv[0]); - pCur->rLimit = (fuzzer_cost)sqlite3_value_int(argv[1]); + idx++; + } + if( idxNum & 2 ){ + pCur->rLimit = (fuzzer_cost)sqlite3_value_int(argv[idx]); + idx++; + } + if( idxNum & 4 ){ + pCur->iRuleset = (fuzzer_cost)sqlite3_value_int(argv[idx]); + idx++; } - if( zWord==0 ) zWord = ""; - pCur->pStem = pStem = fuzzerNewStem(pCur, zWord, (fuzzer_cost)0); - if( pStem==0 ) return SQLITE_NOMEM; pCur->nullRule.pNext = pCur->pVtab->pRule; pCur->nullRule.rCost = 0; pCur->nullRule.nFrom = 0; pCur->nullRule.nTo = 0; pCur->nullRule.zFrom = ""; - pStem->pRule = &pCur->nullRule; - pStem->n = pStem->nBasis; pCur->iRowid = 1; + assert( pCur->pStem==0 ); + + /* If the query term is longer than FUZZER_MX_OUTPUT_LENGTH bytes, this + ** query will return zero rows. */ + if( (int)strlen(zWord)<FUZZER_MX_OUTPUT_LENGTH ){ + pCur->pStem = pStem = fuzzerNewStem(pCur, zWord, (fuzzer_cost)0); + if( pStem==0 ) return SQLITE_NOMEM; + pStem->pRule = &pCur->nullRule; + pStem->n = pStem->nBasis; + }else{ + pCur->rLimit = 0; + } + return SQLITE_OK; } @@ -735,22 +1052,29 @@ static int fuzzerEof(sqlite3_vtab_cursor *cur){ /* ** Search for terms of these forms: ** -** word MATCH $str -** distance < $value -** distance <= $value +** (A) word MATCH $str +** (B1) distance < $value +** (B2) distance <= $value +** (C) ruleid == $ruleid ** ** The distance< and distance<= are both treated as distance<=. -** The query plan number is as follows: +** The query plan number is a bit vector: ** -** 0: None of the terms above are found -** 1: There is a "word MATCH" term with $str in filter.argv[0]. -** 2: There is a "distance<" term with $value in filter.argv[0]. -** 3: Both "word MATCH" and "distance<" with $str in argv[0] and -** $value in argv[1]. +** bit 1: Term of the form (A) found +** bit 2: Term like (B1) or (B2) found +** bit 3: Term like (C) found +** +** If bit-1 is set, $str is always in filter.argv[0]. If bit-2 is set +** then $value is in filter.argv[0] if bit-1 is clear and is in +** filter.argv[1] if bit-1 is set. If bit-3 is set, then $ruleid is +** in filter.argv[0] if bit-1 and bit-2 are both zero, is in +** filter.argv[1] if exactly one of bit-1 and bit-2 are set, and is in +** filter.argv[2] if both bit-1 and bit-2 are set. */ static int fuzzerBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ int iPlan = 0; int iDistTerm = -1; + int iRulesetTerm = -1; int i; const struct sqlite3_index_constraint *pConstraint; pConstraint = pIdxInfo->aConstraint; @@ -772,11 +1096,23 @@ static int fuzzerBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ iPlan |= 2; iDistTerm = i; } + if( (iPlan & 4)==0 + && pConstraint->iColumn==2 + && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ + ){ + iPlan |= 4; + pIdxInfo->aConstraintUsage[i].omit = 1; + iRulesetTerm = i; + } + } + if( iPlan & 2 ){ + pIdxInfo->aConstraintUsage[iDistTerm].argvIndex = 1+((iPlan&1)!=0); } - if( iPlan==2 ){ - pIdxInfo->aConstraintUsage[iDistTerm].argvIndex = 1; - }else if( iPlan==3 ){ - pIdxInfo->aConstraintUsage[iDistTerm].argvIndex = 2; + if( iPlan & 4 ){ + int idx = 1; + if( iPlan & 1 ) idx++; + if( iPlan & 2 ) idx++; + pIdxInfo->aConstraintUsage[iRulesetTerm].argvIndex = idx; } pIdxInfo->idxNum = iPlan; if( pIdxInfo->nOrderBy==1 @@ -791,72 +1127,7 @@ static int fuzzerBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ } /* -** Disallow all attempts to DELETE or UPDATE. Only INSERTs are allowed. -** -** On an insert, the cFrom, cTo, and cost columns are used to construct -** a new rule. All other columns are ignored. The rule is ignored -** if cFrom and cTo are identical. A NULL value for cFrom or cTo is -** interpreted as an empty string. The cost must be positive. -*/ -static int fuzzerUpdate( - sqlite3_vtab *pVTab, - int argc, - sqlite3_value **argv, - sqlite_int64 *pRowid -){ - fuzzer_vtab *p = (fuzzer_vtab*)pVTab; - fuzzer_rule *pRule; - const char *zFrom; - int nFrom; - const char *zTo; - int nTo; - fuzzer_cost rCost; - if( argc!=7 ){ - sqlite3_free(pVTab->zErrMsg); - pVTab->zErrMsg = sqlite3_mprintf("cannot delete from a %s virtual table", - p->zClassName); - return SQLITE_CONSTRAINT; - } - if( sqlite3_value_type(argv[0])!=SQLITE_NULL ){ - sqlite3_free(pVTab->zErrMsg); - pVTab->zErrMsg = sqlite3_mprintf("cannot update a %s virtual table", - p->zClassName); - return SQLITE_CONSTRAINT; - } - zFrom = (char*)sqlite3_value_text(argv[4]); - if( zFrom==0 ) zFrom = ""; - zTo = (char*)sqlite3_value_text(argv[5]); - if( zTo==0 ) zTo = ""; - if( strcmp(zFrom,zTo)==0 ){ - /* Silently ignore null transformations */ - return SQLITE_OK; - } - rCost = sqlite3_value_int(argv[6]); - if( rCost<=0 ){ - sqlite3_free(pVTab->zErrMsg); - pVTab->zErrMsg = sqlite3_mprintf("cost must be positive"); - return SQLITE_CONSTRAINT; - } - nFrom = strlen(zFrom); - nTo = strlen(zTo); - pRule = sqlite3_malloc( sizeof(*pRule) + nFrom + nTo ); - if( pRule==0 ){ - return SQLITE_NOMEM; - } - pRule->zFrom = &pRule->zTo[nTo+1]; - pRule->nFrom = nFrom; - memcpy(pRule->zFrom, zFrom, nFrom+1); - memcpy(pRule->zTo, zTo, nTo+1); - pRule->nTo = nTo; - pRule->rCost = rCost; - pRule->pNext = p->pNewRule; - p->pNewRule = pRule; - return SQLITE_OK; -} - -/* -** A virtual table module that provides read-only access to a -** Tcl global variable namespace. +** A virtual table module that implements the "fuzzer". */ static sqlite3_module fuzzerModule = { 0, /* iVersion */ @@ -872,7 +1143,7 @@ static sqlite3_module fuzzerModule = { fuzzerEof, /* xEof - check for end of scan */ fuzzerColumn, /* xColumn - read data */ fuzzerRowid, /* xRowid - read data */ - fuzzerUpdate, /* xUpdate - INSERT */ + 0, /* xUpdate */ 0, /* xBegin */ 0, /* xSync */ 0, /* xCommit */ @@ -916,7 +1187,7 @@ static int register_fuzzer_module( Tcl_WrongNumArgs(interp, 1, objv, "DB"); return TCL_ERROR; } - if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; + getDbPointer(interp, Tcl_GetString(objv[1]), &db); fuzzer_register(db); return TCL_OK; } diff --git a/src/test_hexio.c b/src/test_hexio.c index e3258e8..b20b5ce 100644 --- a/src/test_hexio.c +++ b/src/test_hexio.c @@ -126,7 +126,7 @@ static int hexio_read( return TCL_ERROR; } fseek(in, offset, SEEK_SET); - got = fread(zBuf, 1, amt, in); + got = (int)fread(zBuf, 1, amt, in); fclose(in); if( got<0 ){ got = 0; @@ -178,7 +178,7 @@ static int hexio_write( return TCL_ERROR; } fseek(out, offset, SEEK_SET); - written = fwrite(aOut, 1, nOut, out); + written = (int)fwrite(aOut, 1, nOut, out); sqlite3_free(aOut); fclose(out); Tcl_SetObjResult(interp, Tcl_NewIntObj(written)); diff --git a/src/test_init.c b/src/test_init.c index a67b678..e3724d8 100644 --- a/src/test_init.c +++ b/src/test_init.c @@ -30,9 +30,9 @@ #include <tcl.h> static struct Wrapped { - sqlite3_pcache_methods pcache; - sqlite3_mem_methods mem; - sqlite3_mutex_methods mutex; + sqlite3_pcache_methods2 pcache; + sqlite3_mem_methods mem; + sqlite3_mutex_methods mutex; int mem_init; /* True if mem subsystem is initalized */ int mem_fail; /* True to fail mem subsystem inialization */ @@ -123,8 +123,8 @@ static void wrPCacheShutdown(void *pArg){ wrapped.pcache_init = 0; } -static sqlite3_pcache *wrPCacheCreate(int a, int b){ - return wrapped.pcache.xCreate(a, b); +static sqlite3_pcache *wrPCacheCreate(int a, int b, int c){ + return wrapped.pcache.xCreate(a, b, c); } static void wrPCacheCachesize(sqlite3_pcache *p, int n){ wrapped.pcache.xCachesize(p, n); @@ -132,13 +132,18 @@ static void wrPCacheCachesize(sqlite3_pcache *p, int n){ static int wrPCachePagecount(sqlite3_pcache *p){ return wrapped.pcache.xPagecount(p); } -static void *wrPCacheFetch(sqlite3_pcache *p, unsigned a, int b){ +static sqlite3_pcache_page *wrPCacheFetch(sqlite3_pcache *p, unsigned a, int b){ return wrapped.pcache.xFetch(p, a, b); } -static void wrPCacheUnpin(sqlite3_pcache *p, void *a, int b){ +static void wrPCacheUnpin(sqlite3_pcache *p, sqlite3_pcache_page *a, int b){ wrapped.pcache.xUnpin(p, a, b); } -static void wrPCacheRekey(sqlite3_pcache *p, void *a, unsigned b, unsigned c){ +static void wrPCacheRekey( + sqlite3_pcache *p, + sqlite3_pcache_page *a, + unsigned b, + unsigned c +){ wrapped.pcache.xRekey(p, a, b, c); } static void wrPCacheTruncate(sqlite3_pcache *p, unsigned a){ @@ -154,8 +159,8 @@ static void installInitWrappers(void){ wrMutexFree, wrMutexEnter, wrMutexTry, wrMutexLeave, wrMutexHeld, wrMutexNotheld }; - sqlite3_pcache_methods pcachemethods = { - 0, + sqlite3_pcache_methods2 pcachemethods = { + 1, 0, wrPCacheInit, wrPCacheShutdown, wrPCacheCreate, wrPCacheCachesize, wrPCachePagecount, wrPCacheFetch, wrPCacheUnpin, wrPCacheRekey, wrPCacheTruncate, @@ -173,10 +178,10 @@ static void installInitWrappers(void){ sqlite3_shutdown(); sqlite3_config(SQLITE_CONFIG_GETMUTEX, &wrapped.mutex); sqlite3_config(SQLITE_CONFIG_GETMALLOC, &wrapped.mem); - sqlite3_config(SQLITE_CONFIG_GETPCACHE, &wrapped.pcache); + sqlite3_config(SQLITE_CONFIG_GETPCACHE2, &wrapped.pcache); sqlite3_config(SQLITE_CONFIG_MUTEX, &mutexmethods); sqlite3_config(SQLITE_CONFIG_MALLOC, &memmethods); - sqlite3_config(SQLITE_CONFIG_PCACHE, &pcachemethods); + sqlite3_config(SQLITE_CONFIG_PCACHE2, &pcachemethods); } static int init_wrapper_install( @@ -218,7 +223,7 @@ static int init_wrapper_uninstall( sqlite3_shutdown(); sqlite3_config(SQLITE_CONFIG_MUTEX, &wrapped.mutex); sqlite3_config(SQLITE_CONFIG_MALLOC, &wrapped.mem); - sqlite3_config(SQLITE_CONFIG_PCACHE, &wrapped.pcache); + sqlite3_config(SQLITE_CONFIG_PCACHE2, &wrapped.pcache); return TCL_OK; } diff --git a/src/test_journal.c b/src/test_journal.c index 6886972..e8701a4 100644 --- a/src/test_journal.c +++ b/src/test_journal.c @@ -290,9 +290,9 @@ static jt_file *locateDatabaseHandle(const char *zJournal){ jt_file *pMain = 0; enterJtMutex(); for(pMain=g.pList; pMain; pMain=pMain->pNext){ - int nName = strlen(zJournal) - strlen("-journal"); + int nName = (int)(strlen(zJournal) - strlen("-journal")); if( (pMain->flags&SQLITE_OPEN_MAIN_DB) - && (strlen(pMain->zName)==nName) + && ((int)strlen(pMain->zName)==nName) && 0==memcmp(pMain->zName, zJournal, nName) && (pMain->eLock>=SQLITE_LOCK_RESERVED) ){ @@ -391,7 +391,7 @@ static int openTransaction(jt_file *pMain, jt_file *pJournal){ while( rc==SQLITE_OK && iTrunk>0 ){ u32 nLeaf; u32 iLeaf; - sqlite3_int64 iOff = (iTrunk-1)*pMain->nPagesize; + sqlite3_int64 iOff = (i64)(iTrunk-1)*pMain->nPagesize; rc = sqlite3OsRead(p, aData, pMain->nPagesize, iOff); nLeaf = decodeUint32(&aData[4]); for(iLeaf=0; rc==SQLITE_OK && iLeaf<nLeaf; iLeaf++){ @@ -404,11 +404,12 @@ static int openTransaction(jt_file *pMain, jt_file *pJournal){ /* Calculate and store a checksum for each page in the database file. */ if( rc==SQLITE_OK ){ int ii; - for(ii=0; rc==SQLITE_OK && ii<pMain->nPage; ii++){ + for(ii=0; rc==SQLITE_OK && ii<(int)pMain->nPage; ii++){ i64 iOff = (i64)(pMain->nPagesize) * (i64)ii; if( iOff==PENDING_BYTE ) continue; rc = sqlite3OsRead(pMain->pReal, aData, pMain->nPagesize, iOff); pMain->aCksum[ii] = genCksum(aData, pMain->nPagesize); + if( ii+1==pMain->nPage && rc==SQLITE_IOERR_SHORT_READ ) rc = SQLITE_OK; } } @@ -465,7 +466,7 @@ static int readJournalFile(jt_file *p, jt_file *pMain){ continue; } } - nRec = (iSize-iOff) / (pMain->nPagesize+8); + nRec = (u32)((iSize-iOff) / (pMain->nPagesize+8)); } /* Read all the records that follow the journal-header just read. */ @@ -537,7 +538,7 @@ static int jtWrite( } if( p->flags&SQLITE_OPEN_MAIN_DB && p->pWritable ){ - if( iAmt<p->nPagesize + if( iAmt<(int)p->nPagesize && p->nPagesize%iAmt==0 && iOfst>=(PENDING_BYTE+512) && iOfst+iAmt<=PENDING_BYTE+p->nPagesize @@ -548,7 +549,7 @@ static int jtWrite( ** pending-byte page. */ }else{ - u32 pgno = iOfst/p->nPagesize + 1; + u32 pgno = (u32)(iOfst/p->nPagesize + 1); assert( (iAmt==1||iAmt==p->nPagesize) && ((iOfst+iAmt)%p->nPagesize)==0 ); assert( pgno<=p->nPage || p->nSync>0 ); assert( pgno>p->nPage || sqlite3BitvecTest(p->pWritable, pgno) ); @@ -577,7 +578,7 @@ static int jtTruncate(sqlite3_file *pFile, sqlite_int64 size){ if( p->flags&SQLITE_OPEN_MAIN_DB && p->pWritable ){ u32 pgno; u32 locking_page = (u32)(PENDING_BYTE/p->nPagesize+1); - for(pgno=size/p->nPagesize+1; pgno<=p->nPage; pgno++){ + for(pgno=(u32)(size/p->nPagesize+1); pgno<=p->nPage; pgno++){ assert( pgno==locking_page || sqlite3BitvecTest(p->pWritable, pgno) ); } } @@ -662,7 +663,7 @@ static int jtCheckReservedLock(sqlite3_file *pFile, int *pResOut){ */ static int jtFileControl(sqlite3_file *pFile, int op, void *pArg){ jt_file *p = (jt_file *)pFile; - return sqlite3OsFileControl(p->pReal, op, pArg); + return p->pReal->pMethods->xFileControl(p->pReal, op, pArg); } /* @@ -722,7 +723,7 @@ static int jtOpen( ** returning. */ static int jtDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ - int nPath = strlen(zPath); + int nPath = (int)strlen(zPath); if( nPath>8 && 0==strcmp("-journal", &zPath[nPath-8]) ){ /* Deleting a journal file. The end of a transaction. */ jt_file *pMain = locateDatabaseHandle(zPath); diff --git a/src/test_malloc.c b/src/test_malloc.c index e955d57..09b8f73 100644 --- a/src/test_malloc.c +++ b/src/test_malloc.c @@ -713,14 +713,14 @@ static int test_memdebug_settitle( int objc, Tcl_Obj *CONST objv[] ){ - const char *zTitle; if( objc!=2 ){ Tcl_WrongNumArgs(interp, 1, objv, "TITLE"); return TCL_ERROR; } - zTitle = Tcl_GetString(objv[1]); #ifdef SQLITE_MEMDEBUG { + const char *zTitle; + zTitle = Tcl_GetString(objv[1]); extern int sqlite3MemdebugSettitle(const char*); sqlite3MemdebugSettitle(zTitle); } @@ -1033,7 +1033,6 @@ static int test_config_lookaside( int objc, Tcl_Obj *CONST objv[] ){ - int rc; int sz, cnt; Tcl_Obj *pRet; if( objc!=3 ){ @@ -1049,7 +1048,7 @@ static int test_config_lookaside( Tcl_ListObjAppendElement( interp, pRet, Tcl_NewIntObj(sqlite3GlobalConfig.nLookaside) ); - rc = sqlite3_config(SQLITE_CONFIG_LOOKASIDE, sz, cnt); + sqlite3_config(SQLITE_CONFIG_LOOKASIDE, sz, cnt); Tcl_SetObjResult(interp, pRet); return TCL_OK; } @@ -1106,7 +1105,6 @@ static int test_config_heap( Tcl_Obj *CONST objv[] ){ static char *zBuf; /* Use this memory */ - static int szBuf; /* Bytes allocated for zBuf */ int nByte; /* Size of buffer to pass to sqlite3_config() */ int nMinAlloc; /* Size of minimum allocation */ int rc; /* Return code of sqlite3_config() */ @@ -1124,11 +1122,9 @@ static int test_config_heap( if( nByte==0 ){ free( zBuf ); zBuf = 0; - szBuf = 0; rc = sqlite3_config(SQLITE_CONFIG_HEAP, (void*)0, 0, 0); }else{ zBuf = realloc(zBuf, nByte); - szBuf = nByte; rc = sqlite3_config(SQLITE_CONFIG_HEAP, zBuf, nByte, nMinAlloc); } @@ -1327,7 +1323,8 @@ static int test_db_status( { "LOOKASIDE_MISS_SIZE", SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE }, { "LOOKASIDE_MISS_FULL", SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL }, { "CACHE_HIT", SQLITE_DBSTATUS_CACHE_HIT }, - { "CACHE_MISS", SQLITE_DBSTATUS_CACHE_MISS } + { "CACHE_MISS", SQLITE_DBSTATUS_CACHE_MISS }, + { "CACHE_WRITE", SQLITE_DBSTATUS_CACHE_WRITE } }; Tcl_Obj *pResult; if( objc!=4 ){ diff --git a/src/test_multiplex.c b/src/test_multiplex.c index 5d29607..a3b3e2f 100644 --- a/src/test_multiplex.c +++ b/src/test_multiplex.c @@ -81,6 +81,13 @@ #define sqlite3_mutex_notheld(X) ((void)(X),1) #endif /* SQLITE_THREADSAFE==0 */ +/* Maximum chunk number */ +#define MX_CHUNK_NUMBER 299 + +/* First chunk for rollback journal files */ +#define SQLITE_MULTIPLEX_JOURNAL_8_3_OFFSET 400 +#define SQLITE_MULTIPLEX_WAL_8_3_OFFSET 700 + /************************ Shim Definitions ******************************/ @@ -96,26 +103,16 @@ # define SQLITE_MULTIPLEX_CHUNK_SIZE 2147418112 #endif -/* Default limit on number of chunks. Care should be taken -** so that values for chunks numbers fit in the SQLITE_MULTIPLEX_EXT_FMT -** format specifier. It may be changed by calling -** the xFileControl() interface. +/* This used to be the default limit on number of chunks, but +** it is no longer enforced. There is currently no limit to the +** number of chunks. +** +** May be changed by calling the xFileControl() interface. */ #ifndef SQLITE_MULTIPLEX_MAX_CHUNKS -# define SQLITE_MULTIPLEX_MAX_CHUNKS 32 +# define SQLITE_MULTIPLEX_MAX_CHUNKS 12 #endif -/* If SQLITE_MULTIPLEX_EXT_OVWR is defined, the -** last SQLITE_MULTIPLEX_EXT_SZ characters of the -** filename will be overwritten, otherwise, the -** multiplex extension is simply appended to the filename. -** Ex. (undefined) test.db -> test.db01 -** (defined) test.db -> test.01 -** Chunk 0 does not have a modified extension. -*/ -#define SQLITE_MULTIPLEX_EXT_FMT "%02d" -#define SQLITE_MULTIPLEX_EXT_SZ 2 - /************************ Object Definitions ******************************/ /* Forward declaration of all object types */ @@ -140,7 +137,8 @@ struct multiplexGroup { int nName; /* Length of base filename */ int flags; /* Flags used for original opening */ unsigned int szChunk; /* Chunk size used for this group */ - int bEnabled; /* TRUE to use Multiplex VFS for this file */ + unsigned char bEnabled; /* TRUE to use Multiplex VFS for this file */ + unsigned char bTruncate; /* TRUE to enable truncation of databases */ multiplexGroup *pNext, *pPrev; /* Doubly linked list of all group objects */ }; @@ -224,68 +222,63 @@ static int multiplexStrlen30(const char *z){ } /* -** Create a temporary file name in zBuf. zBuf must be big enough to -** hold at pOrigVfs->mxPathname characters. This function departs -** from the traditional temporary name generation in the os_win -** and os_unix VFS in several ways, but is necessary so that -** the file name is known for temporary files (like those used -** during vacuum.) +** Generate the file-name for chunk iChunk of the group with base name +** zBase. The file-name is written to buffer zOut before returning. Buffer +** zOut must be allocated by the caller so that it is at least (nBase+5) +** bytes in size, where nBase is the length of zBase, not including the +** nul-terminator. +** +** If iChunk is 0 (or 400 - the number for the first journal file chunk), +** the output is a copy of the input string. Otherwise, if +** SQLITE_ENABLE_8_3_NAMES is not defined or the input buffer does not contain +** a "." character, then the output is a copy of the input string with the +** three-digit zero-padded decimal representation if iChunk appended to it. +** For example: +** +** zBase="test.db", iChunk=4 -> zOut="test.db004" ** -** N.B. This routine assumes your underlying VFS is ok with using -** "/" as a directory seperator. This is the default for UNIXs -** and is allowed (even mixed) for most versions of Windows. +** Or, if SQLITE_ENABLE_8_3_NAMES is defined and the input buffer contains +** a "." character, then everything after the "." is replaced by the +** three-digit representation of iChunk. +** +** zBase="test.db", iChunk=4 -> zOut="test.004" +** +** The output buffer string is terminated by 2 0x00 bytes. This makes it safe +** to pass to sqlite3_uri_parameter() and similar. */ -static int multiplexGetTempname(sqlite3_vfs *pOrigVfs, int nBuf, char *zBuf){ - static char zChars[] = - "abcdefghijklmnopqrstuvwxyz" - "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - "0123456789"; - int i,j; - int attempts = 0; - int exists = 0; - int rc = SQLITE_ERROR; - - /* Check that the output buffer is large enough for - ** pVfs->mxPathname characters. - */ - if( pOrigVfs->mxPathname <= nBuf ){ - char *zTmp = sqlite3_malloc(pOrigVfs->mxPathname); - if( zTmp==0 ) return SQLITE_NOMEM; - - /* sqlite3_temp_directory should always be less than - ** pVfs->mxPathname characters. - */ - sqlite3_snprintf(pOrigVfs->mxPathname, - zTmp, - "%s/", - sqlite3_temp_directory ? sqlite3_temp_directory : "."); - rc = pOrigVfs->xFullPathname(pOrigVfs, zTmp, nBuf, zBuf); - sqlite3_free(zTmp); - if( rc ) return rc; - - /* Check that the output buffer is large enough for the temporary file - ** name. - */ - j = multiplexStrlen30(zBuf); - if( (j + 8 + 1 + 3 + 1) <= nBuf ){ - /* Make 3 attempts to generate a unique name. */ - do { - attempts++; - sqlite3_randomness(8, &zBuf[j]); - for(i=0; i<8; i++){ - unsigned char uc = (unsigned char)zBuf[j+i]; - zBuf[j+i] = (char)zChars[uc%(sizeof(zChars)-1)]; - } - memcpy(&zBuf[j+i], ".tmp", 5); - rc = pOrigVfs->xAccess(pOrigVfs, zBuf, SQLITE_ACCESS_EXISTS, &exists); - } while ( (rc==SQLITE_OK) && exists && (attempts<3) ); - if( rc==SQLITE_OK && exists ){ - rc = SQLITE_ERROR; - } +static void multiplexFilename( + const char *zBase, /* Filename for chunk 0 */ + int nBase, /* Size of zBase in bytes (without \0) */ + int flags, /* Flags used to open file */ + int iChunk, /* Chunk to generate filename for */ + char *zOut /* Buffer to write generated name to */ +){ + int n = nBase; + memcpy(zOut, zBase, n+1); + if( iChunk!=0 && iChunk<=MX_CHUNK_NUMBER ){ +#ifdef SQLITE_ENABLE_8_3_NAMES + int i; + for(i=n-1; i>0 && i>=n-4 && zOut[i]!='.'; i--){} + if( i>=n-4 ) n = i+1; + if( flags & SQLITE_OPEN_MAIN_JOURNAL ){ + /* The extensions on overflow files for main databases are 001, 002, + ** 003 and so forth. To avoid name collisions, add 400 to the + ** extensions of journal files so that they are 401, 402, 403, .... + */ + iChunk += SQLITE_MULTIPLEX_JOURNAL_8_3_OFFSET; + }else if( flags & SQLITE_OPEN_WAL ){ + /* To avoid name collisions, add 700 to the + ** extensions of WAL files so that they are 701, 702, 703, .... + */ + iChunk += SQLITE_MULTIPLEX_WAL_8_3_OFFSET; } +#endif + sqlite3_snprintf(4,&zOut[n],"%03d",iChunk); + n += 3; } - return rc; + assert( zOut[n]=='\0' ); + zOut[n+1] = '\0'; } /* Compute the filename for the iChunk-th chunk @@ -301,50 +294,80 @@ static int multiplexSubFilename(multiplexGroup *pGroup, int iChunk){ pGroup->aReal = p; pGroup->nReal = iChunk+1; } - if( pGroup->aReal[iChunk].z==0 ){ + if( pGroup->zName && pGroup->aReal[iChunk].z==0 ){ char *z; int n = pGroup->nName; - pGroup->aReal[iChunk].z = z = sqlite3_malloc( n+3 ); + pGroup->aReal[iChunk].z = z = sqlite3_malloc( n+5 ); if( z==0 ){ return SQLITE_NOMEM; } - memcpy(z, pGroup->zName, n+1); - if( iChunk>0 ){ -#ifdef SQLITE_ENABLE_8_3_NAMES - if( n>3 && z[n-3]=='.' ){ - n--; - }else if( n>4 && z[n-4]=='.' ){ - n -= 2; - } -#endif - sqlite3_snprintf(3,&z[n],"%02d",iChunk); - } + multiplexFilename(pGroup->zName, pGroup->nName, pGroup->flags, iChunk, z); } return SQLITE_OK; } /* Translate an sqlite3_file* that is really a multiplexGroup* into ** the sqlite3_file* for the underlying original VFS. +** +** For chunk 0, the pGroup->flags determines whether or not a new file +** is created if it does not already exist. For chunks 1 and higher, the +** file is created only if createFlag is 1. */ static sqlite3_file *multiplexSubOpen( - multiplexGroup *pGroup, - int iChunk, - int *rc, - int *pOutFlags + multiplexGroup *pGroup, /* The multiplexor group */ + int iChunk, /* Which chunk to open. 0==original file */ + int *rc, /* Result code in and out */ + int *pOutFlags, /* Output flags */ + int createFlag /* True to create if iChunk>0 */ ){ sqlite3_file *pSubOpen = 0; sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs; /* Real VFS */ + +#ifdef SQLITE_ENABLE_8_3_NAMES + /* If JOURNAL_8_3_OFFSET is set to (say) 400, then any overflow files are + ** part of a database journal are named db.401, db.402, and so on. A + ** database may therefore not grow to larger than 400 chunks. Attempting + ** to open chunk 401 indicates the database is full. */ + if( iChunk>=SQLITE_MULTIPLEX_JOURNAL_8_3_OFFSET ){ + sqlite3_log(SQLITE_FULL, "multiplexed chunk overflow: %s", pGroup->zName); + *rc = SQLITE_FULL; + return 0; + } +#endif + *rc = multiplexSubFilename(pGroup, iChunk); if( (*rc)==SQLITE_OK && (pSubOpen = pGroup->aReal[iChunk].p)==0 ){ + int flags, bExists; + flags = pGroup->flags; + if( createFlag ){ + flags |= SQLITE_OPEN_CREATE; + }else if( iChunk==0 ){ + /* Fall through */ + }else if( pGroup->aReal[iChunk].z==0 ){ + return 0; + }else{ + *rc = pOrigVfs->xAccess(pOrigVfs, pGroup->aReal[iChunk].z, + SQLITE_ACCESS_EXISTS, &bExists); + if( *rc || !bExists ){ + if( *rc ){ + sqlite3_log(*rc, "multiplexor.xAccess failure on %s", + pGroup->aReal[iChunk].z); + } + return 0; + } + flags &= ~SQLITE_OPEN_CREATE; + } pSubOpen = sqlite3_malloc( pOrigVfs->szOsFile ); if( pSubOpen==0 ){ - *rc = SQLITE_NOMEM; + *rc = SQLITE_IOERR_NOMEM; return 0; } pGroup->aReal[iChunk].p = pSubOpen; *rc = pOrigVfs->xOpen(pOrigVfs, pGroup->aReal[iChunk].z, pSubOpen, - pGroup->flags, pOutFlags); - if( *rc!=SQLITE_OK ){ + flags, pOutFlags); + if( (*rc)!=SQLITE_OK ){ + sqlite3_log(*rc, "multiplexor.xOpen failure on %s", + pGroup->aReal[iChunk].z); sqlite3_free(pSubOpen); pGroup->aReal[iChunk].p = 0; return 0; @@ -354,6 +377,26 @@ static sqlite3_file *multiplexSubOpen( } /* +** Return the size, in bytes, of chunk number iChunk. If that chunk +** does not exist, then return 0. This function does not distingish between +** non-existant files and zero-length files. +*/ +static sqlite3_int64 multiplexSubSize( + multiplexGroup *pGroup, /* The multiplexor group */ + int iChunk, /* Which chunk to open. 0==original file */ + int *rc /* Result code in and out */ +){ + sqlite3_file *pSub; + sqlite3_int64 sz = 0; + + if( *rc ) return 0; + pSub = multiplexSubOpen(pGroup, iChunk, rc, NULL, 0); + if( pSub==0 ) return 0; + *rc = pSub->pMethods->xFileSize(pSub, &sz); + return sz; +} + +/* ** This is the implementation of the multiplex_control() SQL function. */ static void multiplexControlFunc( @@ -420,7 +463,9 @@ static void multiplexSubClose( sqlite3_file *pSubOpen = pGroup->aReal[iChunk].p; if( pSubOpen ){ pSubOpen->pMethods->xClose(pSubOpen); - if( pOrigVfs ) pOrigVfs->xDelete(pOrigVfs, pGroup->aReal[iChunk].z, 0); + if( pOrigVfs && pGroup->aReal[iChunk].z ){ + pOrigVfs->xDelete(pOrigVfs, pGroup->aReal[iChunk].z, 0); + } sqlite3_free(pGroup->aReal[iChunk].p); } sqlite3_free(pGroup->aReal[iChunk].z); @@ -466,6 +511,7 @@ static int multiplexOpen( UNUSED_PARAMETER(pVfs); memset(pConn, 0, pVfs->szOsFile); + assert( zName || (flags & SQLITE_OPEN_DELETEONCLOSE) ); /* We need to create a group structure and manage ** access to this group of files. @@ -473,23 +519,9 @@ static int multiplexOpen( multiplexEnter(); pMultiplexOpen = (multiplexConn*)pConn; - /* If the second argument to this function is NULL, generate a - ** temporary file name to use. This will be handled by the - ** original xOpen method. We just need to allocate space for - ** it. - */ - if( !zName ){ - zName = zToFree = sqlite3_malloc( pOrigVfs->mxPathname + 10 ); - if( zName==0 ){ - rc = SQLITE_NOMEM; - }else{ - rc = multiplexGetTempname(pOrigVfs, pOrigVfs->mxPathname, zToFree); - } - } - if( rc==SQLITE_OK ){ /* allocate space for group */ - nName = multiplexStrlen30(zName); + nName = zName ? multiplexStrlen30(zName) : 0; sz = sizeof(multiplexGroup) /* multiplexGroup */ + nName + 1; /* zName */ pGroup = sqlite3_malloc( sz ); @@ -499,63 +531,90 @@ static int multiplexOpen( } if( rc==SQLITE_OK ){ + const char *zUri = (flags & SQLITE_OPEN_URI) ? zName : 0; /* assign pointers to extra space allocated */ - char *p = (char *)&pGroup[1]; - pMultiplexOpen->pGroup = pGroup; memset(pGroup, 0, sz); + pMultiplexOpen->pGroup = pGroup; pGroup->bEnabled = -1; - pGroup->szChunk = SQLITE_MULTIPLEX_CHUNK_SIZE; - if( flags & SQLITE_OPEN_URI ){ - const char *zChunkSize; - zChunkSize = sqlite3_uri_parameter(zName, "chunksize"); - if( zChunkSize ){ - unsigned int n = 0; - int i; - for(i=0; zChunkSize[i]>='0' && zChunkSize[i]<='9'; i++){ - n = n*10 + zChunkSize[i] - '0'; - } - if( n>0 ){ - pGroup->szChunk = (n+0xffff)&~0xffff; - }else{ - /* A zero or negative chunksize disabled the multiplexor */ - pGroup->bEnabled = 0; - } + pGroup->bTruncate = sqlite3_uri_boolean(zUri, "truncate", + (flags & SQLITE_OPEN_MAIN_DB)==0); + pGroup->szChunk = (int)sqlite3_uri_int64(zUri, "chunksize", + SQLITE_MULTIPLEX_CHUNK_SIZE); + pGroup->szChunk = (pGroup->szChunk+0xffff)&~0xffff; + if( zName ){ + char *p = (char *)&pGroup[1]; + pGroup->zName = p; + memcpy(pGroup->zName, zName, nName+1); + pGroup->nName = nName; + } + if( pGroup->bEnabled ){ + /* Make sure that the chunksize is such that the pending byte does not + ** falls at the end of a chunk. A region of up to 64K following + ** the pending byte is never written, so if the pending byte occurs + ** near the end of a chunk, that chunk will be too small. */ +#ifndef SQLITE_OMIT_WSD + extern int sqlite3PendingByte; +#else + int sqlite3PendingByte = 0x40000000; +#endif + while( (sqlite3PendingByte % pGroup->szChunk)>=(pGroup->szChunk-65536) ){ + pGroup->szChunk += 65536; } } - pGroup->zName = p; - /* save off base filename, name length, and original open flags */ - memcpy(pGroup->zName, zName, nName+1); - pGroup->nName = nName; pGroup->flags = flags; rc = multiplexSubFilename(pGroup, 1); if( rc==SQLITE_OK ){ - pSubOpen = multiplexSubOpen(pGroup, 0, &rc, pOutFlags); + pSubOpen = multiplexSubOpen(pGroup, 0, &rc, pOutFlags, 0); + if( pSubOpen==0 && rc==SQLITE_OK ) rc = SQLITE_CANTOPEN; } - if( pSubOpen ){ - int exists, rc2, rc3; + if( rc==SQLITE_OK ){ sqlite3_int64 sz; - rc2 = pSubOpen->pMethods->xFileSize(pSubOpen, &sz); - if( rc2==SQLITE_OK ){ - /* If the first overflow file exists and if the size of the main file - ** is different from the chunk size, that means the chunk size is set - ** set incorrectly. So fix it. - ** - ** Or, if the first overflow file does not exist and the main file is - ** larger than the chunk size, that means the chunk size is too small. - ** But we have no way of determining the intended chunk size, so - ** just disable the multiplexor all togethre. - */ - rc3 = pOrigVfs->xAccess(pOrigVfs, pGroup->aReal[1].z, - SQLITE_ACCESS_EXISTS, &exists); - if( rc3==SQLITE_OK && exists && sz==(sz&0xffff0000) && sz>0 - && sz!=pGroup->szChunk ){ - pGroup->szChunk = sz; - }else if( rc3==SQLITE_OK && !exists && sz>pGroup->szChunk ){ - pGroup->bEnabled = 0; + rc = pSubOpen->pMethods->xFileSize(pSubOpen, &sz); + if( rc==SQLITE_OK && zName ){ + int bExists; + if( sz==0 ){ + if( flags & SQLITE_OPEN_MAIN_JOURNAL ){ + /* If opening a main journal file and the first chunk is zero + ** bytes in size, delete any subsequent chunks from the + ** file-system. */ + int iChunk = 1; + do { + rc = pOrigVfs->xAccess(pOrigVfs, + pGroup->aReal[iChunk].z, SQLITE_ACCESS_EXISTS, &bExists + ); + if( rc==SQLITE_OK && bExists ){ + rc = pOrigVfs->xDelete(pOrigVfs, pGroup->aReal[iChunk].z, 0); + if( rc==SQLITE_OK ){ + rc = multiplexSubFilename(pGroup, ++iChunk); + } + } + }while( rc==SQLITE_OK && bExists ); + } + }else{ + /* If the first overflow file exists and if the size of the main file + ** is different from the chunk size, that means the chunk size is set + ** set incorrectly. So fix it. + ** + ** Or, if the first overflow file does not exist and the main file is + ** larger than the chunk size, that means the chunk size is too small. + ** But we have no way of determining the intended chunk size, so + ** just disable the multiplexor all togethre. + */ + rc = pOrigVfs->xAccess(pOrigVfs, pGroup->aReal[1].z, + SQLITE_ACCESS_EXISTS, &bExists); + bExists = multiplexSubSize(pGroup, 1, &rc)>0; + if( rc==SQLITE_OK && bExists && sz==(sz&0xffff0000) && sz>0 + && sz!=pGroup->szChunk ){ + pGroup->szChunk = (int)sz; + }else if( rc==SQLITE_OK && !bExists && sz>pGroup->szChunk ){ + pGroup->bEnabled = 0; + } } } + } + if( rc==SQLITE_OK ){ if( pSubOpen->pMethods->iVersion==1 ){ pMultiplexOpen->base.pMethods = &gMultiplex.sIoMethodsV1; }else{ @@ -584,8 +643,44 @@ static int multiplexDelete( const char *zName, /* Name of file to delete */ int syncDir ){ + int rc; sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs; /* Real VFS */ - return pOrigVfs->xDelete(pOrigVfs, zName, syncDir); + rc = pOrigVfs->xDelete(pOrigVfs, zName, syncDir); + if( rc==SQLITE_OK ){ + /* If the main chunk was deleted successfully, also delete any subsequent + ** chunks - starting with the last (highest numbered). + */ + int nName = (int)strlen(zName); + char *z; + z = sqlite3_malloc(nName + 5); + if( z==0 ){ + rc = SQLITE_IOERR_NOMEM; + }else{ + int iChunk = 0; + int bExists; + do{ + multiplexFilename(zName, nName, SQLITE_OPEN_MAIN_JOURNAL, ++iChunk, z); + rc = pOrigVfs->xAccess(pOrigVfs, z, SQLITE_ACCESS_EXISTS, &bExists); + }while( rc==SQLITE_OK && bExists ); + while( rc==SQLITE_OK && iChunk>1 ){ + multiplexFilename(zName, nName, SQLITE_OPEN_MAIN_JOURNAL, --iChunk, z); + rc = pOrigVfs->xDelete(pOrigVfs, z, syncDir); + } + if( rc==SQLITE_OK ){ + iChunk = 0; + do{ + multiplexFilename(zName, nName, SQLITE_OPEN_WAL, ++iChunk, z); + rc = pOrigVfs->xAccess(pOrigVfs, z, SQLITE_ACCESS_EXISTS, &bExists); + }while( rc==SQLITE_OK && bExists ); + while( rc==SQLITE_OK && iChunk>1 ){ + multiplexFilename(zName, nName, SQLITE_OPEN_WAL, --iChunk, z); + rc = pOrigVfs->xDelete(pOrigVfs, z, syncDir); + } + } + } + sqlite3_free(z); + } + return rc; } static int multiplexAccess(sqlite3_vfs *a, const char *b, int c, int *d){ @@ -662,7 +757,7 @@ static int multiplexRead( int rc = SQLITE_OK; multiplexEnter(); if( !pGroup->bEnabled ){ - sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL, 0); if( pSubOpen==0 ){ rc = SQLITE_IOERR_READ; }else{ @@ -671,7 +766,7 @@ static int multiplexRead( }else{ while( iAmt > 0 ){ int i = (int)(iOfst / pGroup->szChunk); - sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, i, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, i, &rc, NULL, 1); if( pSubOpen ){ int extra = ((int)(iOfst % pGroup->szChunk) + iAmt) - pGroup->szChunk; if( extra<0 ) extra = 0; @@ -707,16 +802,16 @@ static int multiplexWrite( int rc = SQLITE_OK; multiplexEnter(); if( !pGroup->bEnabled ){ - sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL, 0); if( pSubOpen==0 ){ rc = SQLITE_IOERR_WRITE; }else{ rc = pSubOpen->pMethods->xWrite(pSubOpen, pBuf, iAmt, iOfst); } }else{ - while( iAmt > 0 ){ + while( rc==SQLITE_OK && iAmt>0 ){ int i = (int)(iOfst / pGroup->szChunk); - sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, i, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, i, &rc, NULL, 1); if( pSubOpen ){ int extra = ((int)(iOfst % pGroup->szChunk) + iAmt) - pGroup->szChunk; @@ -724,13 +819,9 @@ static int multiplexWrite( iAmt -= extra; rc = pSubOpen->pMethods->xWrite(pSubOpen, pBuf, iAmt, iOfst % pGroup->szChunk); - if( rc!=SQLITE_OK ) break; pBuf = (char *)pBuf + iAmt; iOfst += iAmt; iAmt = extra; - }else{ - rc = SQLITE_IOERR_WRITE; - break; } } } @@ -748,28 +839,35 @@ static int multiplexTruncate(sqlite3_file *pConn, sqlite3_int64 size){ int rc = SQLITE_OK; multiplexEnter(); if( !pGroup->bEnabled ){ - sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL, 0); if( pSubOpen==0 ){ rc = SQLITE_IOERR_TRUNCATE; }else{ rc = pSubOpen->pMethods->xTruncate(pSubOpen, size); } }else{ - int rc2; int i; + int iBaseGroup = (int)(size / pGroup->szChunk); sqlite3_file *pSubOpen; sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs; /* Real VFS */ /* delete the chunks above the truncate limit */ - for(i=(int)(size / pGroup->szChunk)+1; i<pGroup->nReal; i++){ - multiplexSubClose(pGroup, i, pOrigVfs); + for(i = pGroup->nReal-1; i>iBaseGroup && rc==SQLITE_OK; i--){ + if( pGroup->bTruncate ){ + multiplexSubClose(pGroup, i, pOrigVfs); + }else{ + pSubOpen = multiplexSubOpen(pGroup, i, &rc, 0, 0); + if( pSubOpen ){ + rc = pSubOpen->pMethods->xTruncate(pSubOpen, 0); + } + } } - pSubOpen = multiplexSubOpen(pGroup, (int)(size/pGroup->szChunk), &rc2,0); - if( pSubOpen ){ - rc2 = pSubOpen->pMethods->xTruncate(pSubOpen, size % pGroup->szChunk); - if( rc2!=SQLITE_OK ) rc = rc2; - }else{ - rc = SQLITE_IOERR_TRUNCATE; + if( rc==SQLITE_OK ){ + pSubOpen = multiplexSubOpen(pGroup, iBaseGroup, &rc, 0, 0); + if( pSubOpen ){ + rc = pSubOpen->pMethods->xTruncate(pSubOpen, size % pGroup->szChunk); + } } + if( rc ) rc = SQLITE_IOERR_TRUNCATE; } multiplexLeave(); return rc; @@ -801,47 +899,21 @@ static int multiplexFileSize(sqlite3_file *pConn, sqlite3_int64 *pSize){ multiplexConn *p = (multiplexConn*)pConn; multiplexGroup *pGroup = p->pGroup; int rc = SQLITE_OK; - int rc2; int i; multiplexEnter(); if( !pGroup->bEnabled ){ - sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL, 0); if( pSubOpen==0 ){ rc = SQLITE_IOERR_FSTAT; }else{ rc = pSubOpen->pMethods->xFileSize(pSubOpen, pSize); } }else{ - sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs; *pSize = 0; - for(i=0; 1; i++){ - sqlite3_file *pSubOpen = 0; - int exists = 0; - rc = multiplexSubFilename(pGroup, i); - if( rc ) break; - rc2 = pOrigVfs->xAccess(pOrigVfs, pGroup->aReal[i].z, - SQLITE_ACCESS_EXISTS, &exists); - if( rc2==SQLITE_OK && exists){ - /* if it exists, open it */ - pSubOpen = multiplexSubOpen(pGroup, i, &rc, NULL); - }else{ - /* stop at first "gap" */ - break; - } - if( pSubOpen ){ - sqlite3_int64 sz; - rc2 = pSubOpen->pMethods->xFileSize(pSubOpen, &sz); - if( rc2!=SQLITE_OK ){ - rc = rc2; - }else{ - if( sz>pGroup->szChunk ){ - rc = SQLITE_IOERR_FSTAT; - } - *pSize += sz; - } - }else{ - break; - } + for(i=0; rc==SQLITE_OK; i++){ + sqlite3_int64 sz = multiplexSubSize(pGroup, i, &rc); + if( sz==0 ) break; + *pSize = i*(sqlite3_int64)pGroup->szChunk + sz; } } multiplexLeave(); @@ -853,7 +925,7 @@ static int multiplexFileSize(sqlite3_file *pConn, sqlite3_int64 *pSize){ static int multiplexLock(sqlite3_file *pConn, int lock){ multiplexConn *p = (multiplexConn*)pConn; int rc; - sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL, 0); if( pSubOpen ){ return pSubOpen->pMethods->xLock(pSubOpen, lock); } @@ -865,7 +937,7 @@ static int multiplexLock(sqlite3_file *pConn, int lock){ static int multiplexUnlock(sqlite3_file *pConn, int lock){ multiplexConn *p = (multiplexConn*)pConn; int rc; - sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL, 0); if( pSubOpen ){ return pSubOpen->pMethods->xUnlock(pSubOpen, lock); } @@ -877,7 +949,7 @@ static int multiplexUnlock(sqlite3_file *pConn, int lock){ static int multiplexCheckReservedLock(sqlite3_file *pConn, int *pResOut){ multiplexConn *p = (multiplexConn*)pConn; int rc; - sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL, 0); if( pSubOpen ){ return pSubOpen->pMethods->xCheckReservedLock(pSubOpen, pResOut); } @@ -925,9 +997,12 @@ static int multiplexFileControl(sqlite3_file *pConn, int op, void *pArg){ rc = SQLITE_OK; break; default: - pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL); + pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL, 0); if( pSubOpen ){ rc = pSubOpen->pMethods->xFileControl(pSubOpen, op, pArg); + if( op==SQLITE_FCNTL_VFSNAME && rc==SQLITE_OK ){ + *(char**)pArg = sqlite3_mprintf("multiplex/%z", *(char**)pArg); + } } break; } @@ -939,8 +1014,8 @@ static int multiplexFileControl(sqlite3_file *pConn, int op, void *pArg){ static int multiplexSectorSize(sqlite3_file *pConn){ multiplexConn *p = (multiplexConn*)pConn; int rc; - sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL); - if( pSubOpen ){ + sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL, 0); + if( pSubOpen && pSubOpen->pMethods->xSectorSize ){ return pSubOpen->pMethods->xSectorSize(pSubOpen); } return DEFAULT_SECTOR_SIZE; @@ -951,7 +1026,7 @@ static int multiplexSectorSize(sqlite3_file *pConn){ static int multiplexDeviceCharacteristics(sqlite3_file *pConn){ multiplexConn *p = (multiplexConn*)pConn; int rc; - sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL, 0); if( pSubOpen ){ return pSubOpen->pMethods->xDeviceCharacteristics(pSubOpen); } @@ -969,7 +1044,7 @@ static int multiplexShmMap( ){ multiplexConn *p = (multiplexConn*)pConn; int rc; - sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL, 0); if( pSubOpen ){ return pSubOpen->pMethods->xShmMap(pSubOpen, iRegion, szRegion, bExtend,pp); } @@ -986,7 +1061,7 @@ static int multiplexShmLock( ){ multiplexConn *p = (multiplexConn*)pConn; int rc; - sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL, 0); if( pSubOpen ){ return pSubOpen->pMethods->xShmLock(pSubOpen, ofst, n, flags); } @@ -998,7 +1073,7 @@ static int multiplexShmLock( static void multiplexShmBarrier(sqlite3_file *pConn){ multiplexConn *p = (multiplexConn*)pConn; int rc; - sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL, 0); if( pSubOpen ){ pSubOpen->pMethods->xShmBarrier(pSubOpen); } @@ -1009,7 +1084,7 @@ static void multiplexShmBarrier(sqlite3_file *pConn){ static int multiplexShmUnmap(sqlite3_file *pConn, int deleteFlag){ multiplexConn *p = (multiplexConn*)pConn; int rc; - sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL, 0); if( pSubOpen ){ return pSubOpen->pMethods->xShmUnmap(pSubOpen, deleteFlag); } @@ -1191,9 +1266,13 @@ static int test_multiplex_dump( for(pGroup=gMultiplex.pGroups; pGroup; pGroup=pGroup->pNext){ pGroupTerm = Tcl_NewObj(); - pGroup->zName[pGroup->nName] = '\0'; - Tcl_ListObjAppendElement(interp, pGroupTerm, + if( pGroup->zName ){ + pGroup->zName[pGroup->nName] = '\0'; + Tcl_ListObjAppendElement(interp, pGroupTerm, Tcl_NewStringObj(pGroup->zName, -1)); + }else{ + Tcl_ListObjAppendElement(interp, pGroupTerm, Tcl_NewObj()); + } Tcl_ListObjAppendElement(interp, pGroupTerm, Tcl_NewIntObj(pGroup->nName)); Tcl_ListObjAppendElement(interp, pGroupTerm, diff --git a/src/test_onefile.c b/src/test_onefile.c index cd7db00..6986744 100644 --- a/src/test_onefile.c +++ b/src/test_onefile.c @@ -288,7 +288,7 @@ static int tmpWrite( ){ tmp_file *pTmp = (tmp_file *)pFile; if( (iAmt+iOfst)>pTmp->nAlloc ){ - int nNew = 2*(iAmt+iOfst+pTmp->nAlloc); + int nNew = (int)(2*(iAmt+iOfst+pTmp->nAlloc)); char *zNew = sqlite3_realloc(pTmp->zAlloc, nNew); if( !zNew ){ return SQLITE_NOMEM; @@ -297,7 +297,7 @@ static int tmpWrite( pTmp->nAlloc = nNew; } memcpy(&pTmp->zAlloc[iOfst], zBuf, iAmt); - pTmp->nSize = MAX(pTmp->nSize, iOfst+iAmt); + pTmp->nSize = (int)MAX(pTmp->nSize, iOfst+iAmt); return SQLITE_OK; } @@ -306,7 +306,7 @@ static int tmpWrite( */ static int tmpTruncate(sqlite3_file *pFile, sqlite_int64 size){ tmp_file *pTmp = (tmp_file *)pFile; - pTmp->nSize = MIN(pTmp->nSize, size); + pTmp->nSize = (int)MIN(pTmp->nSize, size); return SQLITE_OK; } @@ -418,7 +418,7 @@ static int fsRead( /* Journal file. */ int iRem = iAmt; int iBuf = 0; - int ii = iOfst; + int ii = (int)iOfst; while( iRem>0 && rc==SQLITE_OK ){ int iRealOff = pReal->nBlob - BLOCKSIZE*((ii/BLOCKSIZE)+1) + ii%BLOCKSIZE; int iRealAmt = MIN(iRem, BLOCKSIZE - (iRealOff%BLOCKSIZE)); @@ -453,14 +453,14 @@ static int fsWrite( }else{ rc = pF->pMethods->xWrite(pF, zBuf, iAmt, iOfst+BLOCKSIZE); if( rc==SQLITE_OK ){ - pReal->nDatabase = MAX(pReal->nDatabase, iAmt+iOfst); + pReal->nDatabase = (int)MAX(pReal->nDatabase, iAmt+iOfst); } } }else{ /* Journal file. */ int iRem = iAmt; int iBuf = 0; - int ii = iOfst; + int ii = (int)iOfst; while( iRem>0 && rc==SQLITE_OK ){ int iRealOff = pReal->nBlob - BLOCKSIZE*((ii/BLOCKSIZE)+1) + ii%BLOCKSIZE; int iRealAmt = MIN(iRem, BLOCKSIZE - (iRealOff%BLOCKSIZE)); @@ -475,7 +475,7 @@ static int fsWrite( } } if( rc==SQLITE_OK ){ - pReal->nJournal = MAX(pReal->nJournal, iAmt+iOfst); + pReal->nJournal = (int)MAX(pReal->nJournal, iAmt+iOfst); } } @@ -489,9 +489,9 @@ static int fsTruncate(sqlite3_file *pFile, sqlite_int64 size){ fs_file *p = (fs_file *)pFile; fs_real_file *pReal = p->pReal; if( p->eType==DATABASE_FILE ){ - pReal->nDatabase = MIN(pReal->nDatabase, size); + pReal->nDatabase = (int)MIN(pReal->nDatabase, size); }else{ - pReal->nJournal = MIN(pReal->nJournal, size); + pReal->nJournal = (int)MIN(pReal->nJournal, size); } return SQLITE_OK; } @@ -606,7 +606,7 @@ static int fsOpen( p->eType = eType; assert(strlen("-journal")==8); - nName = strlen(zName)-((eType==JOURNAL_FILE)?8:0); + nName = (int)strlen(zName)-((eType==JOURNAL_FILE)?8:0); pReal=pFsVfs->pFileList; for(; pReal && strncmp(pReal->zName, zName, nName); pReal=pReal->pNext); @@ -641,7 +641,7 @@ static int fsOpen( pReal->nBlob = BLOBSIZE; }else{ unsigned char zS[4]; - pReal->nBlob = size; + pReal->nBlob = (int)size; rc = pRealFile->pMethods->xRead(pRealFile, zS, 4, 0); pReal->nDatabase = (zS[0]<<24)+(zS[1]<<16)+(zS[2]<<8)+zS[3]; if( rc==SQLITE_OK ){ @@ -687,7 +687,7 @@ static int fsDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ fs_vfs_t *pFsVfs = (fs_vfs_t *)pVfs; fs_real_file *pReal; sqlite3_file *pF; - int nName = strlen(zPath) - 8; + int nName = (int)strlen(zPath) - 8; assert(strlen("-journal")==8); assert(strcmp("-journal", &zPath[nName])==0); @@ -717,7 +717,7 @@ static int fsAccess( fs_vfs_t *pFsVfs = (fs_vfs_t *)pVfs; fs_real_file *pReal; int isJournal = 0; - int nName = strlen(zPath); + int nName = (int)strlen(zPath); if( flags!=SQLITE_ACCESS_EXISTS ){ sqlite3_vfs *pParent = ((fs_vfs_t *)pVfs)->pParent; diff --git a/src/test_osinst.c b/src/test_osinst.c index 50d6250..5314333 100644 --- a/src/test_osinst.c +++ b/src/test_osinst.c @@ -242,7 +242,7 @@ static sqlite3_uint64 vfslog_time(){ } #endif -static void vfslog_call(sqlite3_vfs *, int, int, int, int, int, int); +static void vfslog_call(sqlite3_vfs *, int, int, sqlite3_int64, int, int, int); static void vfslog_string(sqlite3_vfs *, const char *); /* @@ -389,7 +389,11 @@ static int vfslogCheckReservedLock(sqlite3_file *pFile, int *pResOut){ */ static int vfslogFileControl(sqlite3_file *pFile, int op, void *pArg){ VfslogFile *p = (VfslogFile *)pFile; - return p->pReal->pMethods->xFileControl(p->pReal, op, pArg); + int rc = p->pReal->pMethods->xFileControl(p->pReal, op, pArg); + if( op==SQLITE_FCNTL_VFSNAME && rc==SQLITE_OK ){ + *(char**)pArg = sqlite3_mprintf("vfslog/%z", *(char**)pArg); + } + return rc; } /* @@ -644,7 +648,7 @@ static void vfslog_call( sqlite3_vfs *pVfs, int eEvent, int iFileid, - int nClick, + sqlite3_int64 nClick, int return_code, int size, int offset @@ -657,7 +661,7 @@ static void vfslog_call( zRec = (unsigned char *)&p->aBuf[p->nBuf]; put32bits(&zRec[0], eEvent); put32bits(&zRec[4], iFileid); - put32bits(&zRec[8], nClick); + put32bits(&zRec[8], (unsigned int)(nClick&0xffff)); put32bits(&zRec[12], return_code); put32bits(&zRec[16], size); put32bits(&zRec[20], offset); @@ -667,7 +671,7 @@ static void vfslog_call( static void vfslog_string(sqlite3_vfs *pVfs, const char *zStr){ VfslogVfs *p = (VfslogVfs *)pVfs; unsigned char *zRec; - int nStr = zStr ? strlen(zStr) : 0; + int nStr = zStr ? (int)strlen(zStr) : 0; if( (4+nStr+p->nBuf)>sizeof(p->aBuf) ){ vfslog_flush(p); } @@ -716,7 +720,7 @@ int sqlite3_vfslog_new( return SQLITE_ERROR; } - nVfs = strlen(zVfs); + nVfs = (int)strlen(zVfs); nByte = sizeof(VfslogVfs) + pParent->szOsFile + nVfs+1+pParent->mxPathname+1; p = (VfslogVfs *)sqlite3_malloc(nByte); memset(p, 0, nByte); @@ -1039,7 +1043,7 @@ static int vlogColumn( } case 1: { char *zStr = pCsr->zTransient; - if( val!=0 && val<pCsr->nFile ){ + if( val!=0 && val<(unsigned)pCsr->nFile ){ zStr = pCsr->azFile[val]; } sqlite3_result_text(ctx, zStr, -1, SQLITE_TRANSIENT); diff --git a/src/test_pcache.c b/src/test_pcache.c index 98aa136..8fcfe7e 100644 --- a/src/test_pcache.c +++ b/src/test_pcache.c @@ -100,15 +100,16 @@ static void testpcacheShutdown(void *pArg){ typedef struct testpcache testpcache; struct testpcache { int szPage; /* Size of each page. Multiple of 8. */ + int szExtra; /* Size of extra data that accompanies each page */ int bPurgeable; /* True if the page cache is purgeable */ int nFree; /* Number of unused slots in a[] */ int nPinned; /* Number of pinned slots in a[] */ unsigned iRand; /* State of the PRNG */ unsigned iMagic; /* Magic number for sanity checking */ struct testpcachePage { + sqlite3_pcache_page page; /* Base class */ unsigned key; /* The key for this page. 0 means unallocated */ int isPinned; /* True if the page is pinned */ - void *pData; /* Data for this page */ } a[TESTPCACHE_NPAGE]; /* All pages in the cache */ }; @@ -129,27 +130,33 @@ static unsigned testpcacheRandom(testpcache *p){ /* ** Allocate a new page cache instance. */ -static sqlite3_pcache *testpcacheCreate(int szPage, int bPurgeable){ +static sqlite3_pcache *testpcacheCreate( + int szPage, + int szExtra, + int bPurgeable +){ int nMem; char *x; testpcache *p; int i; assert( testpcacheGlobal.pDummy!=0 ); szPage = (szPage+7)&~7; - nMem = sizeof(testpcache) + TESTPCACHE_NPAGE*szPage; + nMem = sizeof(testpcache) + TESTPCACHE_NPAGE*(szPage+szExtra); p = sqlite3_malloc( nMem ); if( p==0 ) return 0; x = (char*)&p[1]; p->szPage = szPage; + p->szExtra = szExtra; p->nFree = TESTPCACHE_NPAGE; p->nPinned = 0; p->iRand = testpcacheGlobal.prngSeed; p->bPurgeable = bPurgeable; p->iMagic = TESTPCACHE_VALID; - for(i=0; i<TESTPCACHE_NPAGE; i++, x += szPage){ + for(i=0; i<TESTPCACHE_NPAGE; i++, x += (szPage+szExtra)){ p->a[i].key = 0; p->a[i].isPinned = 0; - p->a[i].pData = (void*)x; + p->a[i].page.pBuf = (void*)x; + p->a[i].page.pExtra = (void*)&x[szPage]; } testpcacheGlobal.nInstance++; return (sqlite3_pcache*)p; @@ -161,7 +168,6 @@ static sqlite3_pcache *testpcacheCreate(int szPage, int bPurgeable){ static void testpcacheCachesize(sqlite3_pcache *pCache, int newSize){ testpcache *p = (testpcache*)pCache; assert( p->iMagic==TESTPCACHE_VALID ); - assert( newSize>=1 ); assert( testpcacheGlobal.pDummy!=0 ); assert( testpcacheGlobal.nInstance>0 ); } @@ -181,7 +187,7 @@ static int testpcachePagecount(sqlite3_pcache *pCache){ /* ** Fetch a page. */ -static void *testpcacheFetch( +static sqlite3_pcache_page *testpcacheFetch( sqlite3_pcache *pCache, unsigned key, int createFlag @@ -200,7 +206,7 @@ static void *testpcacheFetch( assert( p->nPinned <= TESTPCACHE_NPAGE - p->nFree ); p->a[i].isPinned = 1; } - return p->a[i].pData; + return &p->a[i].page; } } @@ -237,11 +243,12 @@ static void *testpcacheFetch( if( p->a[j].key==0 ){ p->a[j].key = key; p->a[j].isPinned = 1; - memset(p->a[j].pData, 0, p->szPage); + memset(p->a[j].page.pBuf, 0, p->szPage); + memset(p->a[j].page.pExtra, 0, p->szExtra); p->nPinned++; p->nFree--; assert( p->nPinned <= TESTPCACHE_NPAGE - p->nFree ); - return p->a[j].pData; + return &p->a[j].page; } } @@ -263,10 +270,11 @@ static void *testpcacheFetch( if( p->a[j].key>0 && p->a[j].isPinned==0 ){ p->a[j].key = key; p->a[j].isPinned = 1; - memset(p->a[j].pData, 0, p->szPage); + memset(p->a[j].page.pBuf, 0, p->szPage); + memset(p->a[j].page.pExtra, 0, p->szExtra); p->nPinned++; assert( p->nPinned <= TESTPCACHE_NPAGE - p->nFree ); - return p->a[j].pData; + return &p->a[j].page; } } @@ -280,7 +288,7 @@ static void *testpcacheFetch( */ static void testpcacheUnpin( sqlite3_pcache *pCache, - void *pOldPage, + sqlite3_pcache_page *pOldPage, int discard ){ testpcache *p = (testpcache*)pCache; @@ -300,7 +308,7 @@ static void testpcacheUnpin( } for(i=0; i<TESTPCACHE_NPAGE; i++){ - if( p->a[i].pData==pOldPage ){ + if( &p->a[i].page==pOldPage ){ /* The pOldPage pointer always points to a pinned page */ assert( p->a[i].isPinned ); p->a[i].isPinned = 0; @@ -325,7 +333,7 @@ static void testpcacheUnpin( */ static void testpcacheRekey( sqlite3_pcache *pCache, - void *pOldPage, + sqlite3_pcache_page *pOldPage, unsigned oldKey, unsigned newKey ){ @@ -354,7 +362,7 @@ static void testpcacheRekey( for(i=0; i<TESTPCACHE_NPAGE; i++){ if( p->a[i].key==oldKey ){ /* The oldKey and pOldPage parameters match */ - assert( p->a[i].pData==pOldPage ); + assert( &p->a[i].page==pOldPage ); /* Page to be rekeyed must be pinned */ assert( p->a[i].isPinned ); p->a[i].key = newKey; @@ -422,7 +430,8 @@ void installTestPCache( unsigned prngSeed, /* Seed for the PRNG */ unsigned highStress /* Call xStress agressively */ ){ - static const sqlite3_pcache_methods testPcache = { + static const sqlite3_pcache_methods2 testPcache = { + 1, (void*)&testpcacheGlobal, testpcacheInit, testpcacheShutdown, @@ -435,7 +444,7 @@ void installTestPCache( testpcacheTruncate, testpcacheDestroy, }; - static sqlite3_pcache_methods defaultPcache; + static sqlite3_pcache_methods2 defaultPcache; static int isInstalled = 0; assert( testpcacheGlobal.nInstance==0 ); @@ -446,12 +455,12 @@ void installTestPCache( testpcacheGlobal.highStress = highStress; if( installFlag!=isInstalled ){ if( installFlag ){ - sqlite3_config(SQLITE_CONFIG_GETPCACHE, &defaultPcache); + sqlite3_config(SQLITE_CONFIG_GETPCACHE2, &defaultPcache); assert( defaultPcache.xCreate!=testpcacheCreate ); - sqlite3_config(SQLITE_CONFIG_PCACHE, &testPcache); + sqlite3_config(SQLITE_CONFIG_PCACHE2, &testPcache); }else{ assert( defaultPcache.xCreate!=0 ); - sqlite3_config(SQLITE_CONFIG_PCACHE, &defaultPcache); + sqlite3_config(SQLITE_CONFIG_PCACHE2, &defaultPcache); } isInstalled = installFlag; } diff --git a/src/test_quota.c b/src/test_quota.c index 74d1a6d..38dc36f 100644 --- a/src/test_quota.c +++ b/src/test_quota.c @@ -27,7 +27,7 @@ ** files within the group is less than the new quota, then the write ** continues as if nothing had happened. */ -#include "sqlite3.h" +#include "test_quota.h" #include <string.h> #include <assert.h> @@ -45,6 +45,62 @@ #endif /* SQLITE_THREADSAFE==0 */ +/* +** Figure out if we are dealing with Unix, Windows, or some other +** operating system. After the following block of preprocess macros, +** all of SQLITE_OS_UNIX, SQLITE_OS_WIN, SQLITE_OS_OS2, and SQLITE_OS_OTHER +** will defined to either 1 or 0. One of the four will be 1. The other +** three will be 0. +*/ +#if defined(SQLITE_OS_OTHER) +# if SQLITE_OS_OTHER==1 +# undef SQLITE_OS_UNIX +# define SQLITE_OS_UNIX 0 +# undef SQLITE_OS_WIN +# define SQLITE_OS_WIN 0 +# undef SQLITE_OS_OS2 +# define SQLITE_OS_OS2 0 +# else +# undef SQLITE_OS_OTHER +# endif +#endif +#if !defined(SQLITE_OS_UNIX) && !defined(SQLITE_OS_OTHER) +# define SQLITE_OS_OTHER 0 +# ifndef SQLITE_OS_WIN +# if defined(_WIN32) || defined(WIN32) || defined(__CYGWIN__) \ + || defined(__MINGW32__) || defined(__BORLANDC__) +# define SQLITE_OS_WIN 1 +# define SQLITE_OS_UNIX 0 +# define SQLITE_OS_OS2 0 +# elif defined(__EMX__) || defined(_OS2) || defined(OS2) \ + || defined(_OS2_) || defined(__OS2__) +# define SQLITE_OS_WIN 0 +# define SQLITE_OS_UNIX 0 +# define SQLITE_OS_OS2 1 +# else +# define SQLITE_OS_WIN 0 +# define SQLITE_OS_UNIX 1 +# define SQLITE_OS_OS2 0 +# endif +# else +# define SQLITE_OS_UNIX 0 +# define SQLITE_OS_OS2 0 +# endif +#else +# ifndef SQLITE_OS_WIN +# define SQLITE_OS_WIN 0 +# endif +#endif + +#if SQLITE_OS_UNIX +# include <unistd.h> +#endif +#if SQLITE_OS_WIN +# include <windows.h> +# include <io.h> +#endif + + /************************ Object Definitions ******************************/ /* Forward declaration of all object types */ @@ -111,6 +167,21 @@ struct quotaConn { /* The underlying VFS sqlite3_file is appended to this object */ }; +/* +** An instance of the following object records the state of an +** open file. This object is opaque to all users - the internal +** structure is only visible to the functions below. +*/ +struct quota_FILE { + FILE *f; /* Open stdio file pointer */ + sqlite3_int64 iOfst; /* Current offset into the file */ + quotaFile *pFile; /* The file record in the quota system */ +#if SQLITE_OS_WIN + char *zMbcsName; /* Full MBCS pathname of the file */ +#endif +}; + + /************************* Global Variables **********************************/ /* ** All global variables used by this file are containing within the following @@ -225,9 +296,11 @@ static void quotaGroupDeref(quotaGroup *pGroup){ ** ** [^...] Matches one character not in the enclosed list. ** +** / Matches "/" or "\\" +** */ static int quotaStrglob(const char *zGlob, const char *z){ - int c, c2; + int c, c2, cx; int invert; int seen; @@ -244,8 +317,9 @@ static int quotaStrglob(const char *zGlob, const char *z){ } return (*z)!=0; } + cx = (c=='/') ? '\\' : c; while( (c2 = (*(z++)))!=0 ){ - while( c2!=c ){ + while( c2!=c && c2!=cx ){ c2 = *(z++); if( c2==0 ) return 0; } @@ -283,6 +357,9 @@ static int quotaStrglob(const char *zGlob, const char *z){ c2 = *(zGlob++); } if( c2==0 || (seen ^ invert)==0 ) return 0; + }else if( c=='/' ){ + if( z[0]!='/' && z[0]!='\\' ) return 0; + z++; }else{ if( c!=(*(z++)) ) return 0; } @@ -313,13 +390,74 @@ static sqlite3_file *quotaSubOpen(sqlite3_file *pConn){ /* Find a file in a quota group and return a pointer to that file. ** Return NULL if the file is not in the group. */ -static quotaFile *quotaFindFile(quotaGroup *pGroup, const char *zName){ +static quotaFile *quotaFindFile( + quotaGroup *pGroup, /* Group in which to look for the file */ + const char *zName, /* Full pathname of the file */ + int createFlag /* Try to create the file if not found */ +){ quotaFile *pFile = pGroup->pFiles; while( pFile && strcmp(pFile->zFilename, zName)!=0 ){ pFile = pFile->pNext; } + if( pFile==0 && createFlag ){ + int nName = (int)(strlen(zName) & 0x3fffffff); + pFile = (quotaFile *)sqlite3_malloc( sizeof(*pFile) + nName + 1 ); + if( pFile ){ + memset(pFile, 0, sizeof(*pFile)); + pFile->zFilename = (char*)&pFile[1]; + memcpy(pFile->zFilename, zName, nName+1); + pFile->pNext = pGroup->pFiles; + if( pGroup->pFiles ) pGroup->pFiles->ppPrev = &pFile->pNext; + pFile->ppPrev = &pGroup->pFiles; + pGroup->pFiles = pFile; + pFile->pGroup = pGroup; + } + } return pFile; } +/* +** Translate UTF8 to MBCS for use in fopen() calls. Return a pointer to the +** translated text.. Call quota_mbcs_free() to deallocate any memory +** used to store the returned pointer when done. +*/ +static char *quota_utf8_to_mbcs(const char *zUtf8){ +#if SQLITE_OS_WIN + size_t n; /* Bytes in zUtf8 */ + int nWide; /* number of UTF-16 characters */ + int nMbcs; /* Bytes of MBCS */ + LPWSTR zTmpWide; /* The UTF16 text */ + char *zMbcs; /* The MBCS text */ + int codepage; /* Code page used by fopen() */ + + n = strlen(zUtf8); + nWide = MultiByteToWideChar(CP_UTF8, 0, zUtf8, -1, NULL, 0); + if( nWide==0 ) return 0; + zTmpWide = (LPWSTR)sqlite3_malloc( (nWide+1)*sizeof(zTmpWide[0]) ); + if( zTmpWide==0 ) return 0; + MultiByteToWideChar(CP_UTF8, 0, zUtf8, -1, zTmpWide, nWide); + codepage = AreFileApisANSI() ? CP_ACP : CP_OEMCP; + nMbcs = WideCharToMultiByte(codepage, 0, zTmpWide, nWide, 0, 0, 0, 0); + zMbcs = nMbcs ? (char*)sqlite3_malloc( nMbcs+1 ) : 0; + if( zMbcs ){ + WideCharToMultiByte(codepage, 0, zTmpWide, nWide, zMbcs, nMbcs, 0, 0); + } + sqlite3_free(zTmpWide); + return zMbcs; +#else + return (char*)zUtf8; /* No-op on unix */ +#endif +} + +/* +** Deallocate any memory allocated by quota_utf8_to_mbcs(). +*/ +static void quota_mbcs_free(char *zOld){ +#if SQLITE_OS_WIN + sqlite3_free(zOld); +#else + /* No-op on unix */ +#endif +} /************************* VFS Method Wrappers *****************************/ /* @@ -364,25 +502,13 @@ static int quotaOpen( pSubOpen = quotaSubOpen(pConn); rc = pOrigVfs->xOpen(pOrigVfs, zName, pSubOpen, flags, pOutFlags); if( rc==SQLITE_OK ){ - pFile = quotaFindFile(pGroup, zName); + pFile = quotaFindFile(pGroup, zName, 1); if( pFile==0 ){ - int nName = strlen(zName); - pFile = (quotaFile *)sqlite3_malloc( sizeof(*pFile) + nName + 1 ); - if( pFile==0 ){ - quotaLeave(); - pSubOpen->pMethods->xClose(pSubOpen); - return SQLITE_NOMEM; - } - memset(pFile, 0, sizeof(*pFile)); - pFile->zFilename = (char*)&pFile[1]; - memcpy(pFile->zFilename, zName, nName+1); - pFile->pNext = pGroup->pFiles; - if( pGroup->pFiles ) pGroup->pFiles->ppPrev = &pFile->pNext; - pFile->ppPrev = &pGroup->pFiles; - pGroup->pFiles = pFile; - pFile->pGroup = pGroup; - pFile->deleteOnClose = (flags & SQLITE_OPEN_DELETEONCLOSE)!=0; + quotaLeave(); + pSubOpen->pMethods->xClose(pSubOpen); + return SQLITE_NOMEM; } + pFile->deleteOnClose = (flags & SQLITE_OPEN_DELETEONCLOSE)!=0; pFile->nRef++; pQuotaOpen->pFile = pFile; if( pSubOpen->pMethods->iVersion==1 ){ @@ -423,7 +549,7 @@ static int quotaDelete( quotaEnter(); pGroup = quotaGroupFind(zName); if( pGroup ){ - pFile = quotaFindFile(pGroup, zName); + pFile = quotaFindFile(pGroup, zName, 0); if( pFile ){ if( pFile->nRef ){ pFile->deleteOnClose = 1; @@ -455,7 +581,10 @@ static int quotaClose(sqlite3_file *pConn){ pFile->nRef--; if( pFile->nRef==0 ){ quotaGroup *pGroup = pFile->pGroup; - if( pFile->deleteOnClose ) quotaRemoveFile(pFile); + if( pFile->deleteOnClose ){ + gQuota.pOrigVfs->xDelete(gQuota.pOrigVfs, pFile->zFilename, 0); + quotaRemoveFile(pFile); + } quotaGroupDeref(pGroup); } quotaLeave(); @@ -589,7 +718,13 @@ static int quotaCheckReservedLock(sqlite3_file *pConn, int *pResOut){ */ static int quotaFileControl(sqlite3_file *pConn, int op, void *pArg){ sqlite3_file *pSubOpen = quotaSubOpen(pConn); - return pSubOpen->pMethods->xFileControl(pSubOpen, op, pArg); + int rc = pSubOpen->pMethods->xFileControl(pSubOpen, op, pArg); +#if defined(SQLITE_FCNTL_VFSNAME) + if( op==SQLITE_FCNTL_VFSNAME && rc==SQLITE_OK ){ + *(char**)pArg = sqlite3_mprintf("quota/%z", *(char**)pArg); + } +#endif + return rc; } /* Pass xSectorSize requests through to the original VFS unchanged. @@ -765,7 +900,7 @@ int sqlite3_quota_set( pGroup = pGroup->pNext; } if( pGroup==0 ){ - int nPattern = strlen(zPattern); + int nPattern = (int)(strlen(zPattern) & 0x3fffffff); if( iLimit<=0 ){ quotaLeave(); return SQLITE_OK; @@ -805,33 +940,355 @@ int sqlite3_quota_file(const char *zFilename){ int rc; int outFlags = 0; sqlite3_int64 iSize; - fd = sqlite3_malloc(gQuota.sThisVfs.szOsFile + gQuota.sThisVfs.mxPathname+1); - if( fd==0 ) return SQLITE_NOMEM; - zFull = gQuota.sThisVfs.szOsFile + (char*)fd; - rc = gQuota.pOrigVfs->xFullPathname(gQuota.pOrigVfs, zFilename, - gQuota.sThisVfs.mxPathname+1, zFull); + int nAlloc = gQuota.sThisVfs.szOsFile + gQuota.sThisVfs.mxPathname+2; + + /* Allocate space for a file-handle and the full path for file zFilename */ + fd = (sqlite3_file *)sqlite3_malloc(nAlloc); + if( fd==0 ){ + rc = SQLITE_NOMEM; + }else{ + zFull = &((char *)fd)[gQuota.sThisVfs.szOsFile]; + rc = gQuota.pOrigVfs->xFullPathname(gQuota.pOrigVfs, zFilename, + gQuota.sThisVfs.mxPathname+1, zFull); + } + if( rc==SQLITE_OK ){ + zFull[strlen(zFull)+1] = '\0'; rc = quotaOpen(&gQuota.sThisVfs, zFull, fd, SQLITE_OPEN_READONLY | SQLITE_OPEN_MAIN_DB, &outFlags); + if( rc==SQLITE_OK ){ + fd->pMethods->xFileSize(fd, &iSize); + fd->pMethods->xClose(fd); + }else if( rc==SQLITE_CANTOPEN ){ + quotaGroup *pGroup; + quotaFile *pFile; + quotaEnter(); + pGroup = quotaGroupFind(zFull); + if( pGroup ){ + pFile = quotaFindFile(pGroup, zFull, 0); + if( pFile ) quotaRemoveFile(pFile); + } + quotaLeave(); + } } - if( rc==SQLITE_OK ){ - fd->pMethods->xFileSize(fd, &iSize); - fd->pMethods->xClose(fd); - }else if( rc==SQLITE_CANTOPEN ){ - quotaGroup *pGroup; - quotaFile *pFile; + + sqlite3_free(fd); + return rc; +} + +/* +** Open a potentially quotaed file for I/O. +*/ +quota_FILE *sqlite3_quota_fopen(const char *zFilename, const char *zMode){ + quota_FILE *p = 0; + char *zFull = 0; + char *zFullTranslated = 0; + int rc; + quotaGroup *pGroup; + quotaFile *pFile; + + zFull = (char*)sqlite3_malloc(gQuota.sThisVfs.mxPathname + 1); + if( zFull==0 ) return 0; + rc = gQuota.pOrigVfs->xFullPathname(gQuota.pOrigVfs, zFilename, + gQuota.sThisVfs.mxPathname+1, zFull); + if( rc ) goto quota_fopen_error; + p = (quota_FILE*)sqlite3_malloc(sizeof(*p)); + if( p==0 ) goto quota_fopen_error; + memset(p, 0, sizeof(*p)); + zFullTranslated = quota_utf8_to_mbcs(zFull); + if( zFullTranslated==0 ) goto quota_fopen_error; + p->f = fopen(zFullTranslated, zMode); + if( p->f==0 ) goto quota_fopen_error; + quotaEnter(); + pGroup = quotaGroupFind(zFull); + if( pGroup ){ + pFile = quotaFindFile(pGroup, zFull, 1); + if( pFile==0 ){ + quotaLeave(); + goto quota_fopen_error; + } + pFile->nRef++; + p->pFile = pFile; + } + quotaLeave(); + sqlite3_free(zFull); +#if SQLITE_OS_WIN + p->zMbcsName = zFullTranslated; +#endif + return p; + +quota_fopen_error: + quota_mbcs_free(zFullTranslated); + sqlite3_free(zFull); + if( p && p->f ) fclose(p->f); + sqlite3_free(p); + return 0; +} + +/* +** Read content from a quota_FILE +*/ +size_t sqlite3_quota_fread( + void *pBuf, /* Store the content here */ + size_t size, /* Size of each element */ + size_t nmemb, /* Number of elements to read */ + quota_FILE *p /* Read from this quota_FILE object */ +){ + return fread(pBuf, size, nmemb, p->f); +} + +/* +** Write content into a quota_FILE. Invoke the quota callback and block +** the write if we exceed quota. +*/ +size_t sqlite3_quota_fwrite( + void *pBuf, /* Take content to write from here */ + size_t size, /* Size of each element */ + size_t nmemb, /* Number of elements */ + quota_FILE *p /* Write to this quota_FILE objecct */ +){ + sqlite3_int64 iOfst; + sqlite3_int64 iEnd; + sqlite3_int64 szNew; + quotaFile *pFile; + size_t rc; + + iOfst = ftell(p->f); + iEnd = iOfst + size*nmemb; + pFile = p->pFile; + if( pFile && pFile->iSize<iEnd ){ + quotaGroup *pGroup = pFile->pGroup; quotaEnter(); - pGroup = quotaGroupFind(zFull); - if( pGroup ){ - pFile = quotaFindFile(pGroup, zFull); - if( pFile ) quotaRemoveFile(pFile); + szNew = pGroup->iSize - pFile->iSize + iEnd; + if( szNew>pGroup->iLimit && pGroup->iLimit>0 ){ + if( pGroup->xCallback ){ + pGroup->xCallback(pFile->zFilename, &pGroup->iLimit, szNew, + pGroup->pArg); + } + if( szNew>pGroup->iLimit && pGroup->iLimit>0 ){ + iEnd = pGroup->iLimit - pGroup->iSize + pFile->iSize; + nmemb = (size_t)((iEnd - iOfst)/size); + iEnd = iOfst + size*nmemb; + szNew = pGroup->iSize - pFile->iSize + iEnd; + } } + pGroup->iSize = szNew; + pFile->iSize = iEnd; + quotaLeave(); + }else{ + pFile = 0; + } + rc = fwrite(pBuf, size, nmemb, p->f); + + /* If the write was incomplete, adjust the file size and group size + ** downward */ + if( rc<nmemb && pFile ){ + size_t nWritten = rc>=0 ? rc : 0; + sqlite3_int64 iNewEnd = iOfst + size*nWritten; + if( iNewEnd<iEnd ) iNewEnd = iEnd; + quotaEnter(); + pFile->pGroup->iSize += iNewEnd - pFile->iSize; + pFile->iSize = iNewEnd; + quotaLeave(); + } + return rc; +} + +/* +** Close an open quota_FILE stream. +*/ +int sqlite3_quota_fclose(quota_FILE *p){ + int rc; + quotaFile *pFile; + rc = fclose(p->f); + pFile = p->pFile; + if( pFile ){ + quotaEnter(); + pFile->nRef--; + if( pFile->nRef==0 ){ + quotaGroup *pGroup = pFile->pGroup; + if( pFile->deleteOnClose ){ + gQuota.pOrigVfs->xDelete(gQuota.pOrigVfs, pFile->zFilename, 0); + quotaRemoveFile(pFile); + } + quotaGroupDeref(pGroup); + } + quotaLeave(); + } +#if SQLITE_OS_WIN + quota_mbcs_free(p->zMbcsName); +#endif + sqlite3_free(p); + return rc; +} + +/* +** Flush memory buffers for a quota_FILE to disk. +*/ +int sqlite3_quota_fflush(quota_FILE *p, int doFsync){ + int rc; + rc = fflush(p->f); + if( rc==0 && doFsync ){ +#if SQLITE_OS_UNIX + rc = fsync(fileno(p->f)); +#endif +#if SQLITE_OS_WIN + rc = _commit(_fileno(p->f)); +#endif + } + return rc!=0; +} + +/* +** Seek on a quota_FILE stream. +*/ +int sqlite3_quota_fseek(quota_FILE *p, long offset, int whence){ + return fseek(p->f, offset, whence); +} + +/* +** rewind a quota_FILE stream. +*/ +void sqlite3_quota_rewind(quota_FILE *p){ + rewind(p->f); +} + +/* +** Tell the current location of a quota_FILE stream. +*/ +long sqlite3_quota_ftell(quota_FILE *p){ + return ftell(p->f); +} + +/* +** Truncate a file to szNew bytes. +*/ +int sqlite3_quota_ftruncate(quota_FILE *p, sqlite3_int64 szNew){ + quotaFile *pFile = p->pFile; + int rc; + if( (pFile = p->pFile)!=0 && pFile->iSize<szNew ){ + quotaGroup *pGroup; + if( pFile->iSize<szNew ){ + /* This routine cannot be used to extend a file that is under + ** quota management. Only true truncation is allowed. */ + return -1; + } + pGroup = pFile->pGroup; + quotaEnter(); + pGroup->iSize += szNew - pFile->iSize; + quotaLeave(); + } +#if SQLITE_OS_UNIX + rc = ftruncate(fileno(p->f), szNew); +#endif +#if SQLITE_OS_WIN + rc = _chsize_s(_fileno(p->f), szNew); +#endif + if( pFile && rc==0 ){ + quotaGroup *pGroup = pFile->pGroup; + quotaEnter(); + pGroup->iSize += szNew - pFile->iSize; + pFile->iSize = szNew; quotaLeave(); } - sqlite3_free(fd); return rc; } +/* +** Determine the time that the given file was last modified, in +** seconds size 1970. Write the result into *pTime. Return 0 on +** success and non-zero on any kind of error. +*/ +int sqlite3_quota_file_mtime(quota_FILE *p, time_t *pTime){ + int rc; +#if SQLITE_OS_UNIX + struct stat buf; + rc = fstat(fileno(p->f), &buf); +#endif +#if SQLITE_OS_WIN + struct _stati64 buf; + rc = _stati64(p->zMbcsName, &buf); +#endif + if( rc==0 ) *pTime = buf.st_mtime; + return rc; +} + +/* +** Return the true size of the file, as reported by the operating +** system. +*/ +sqlite3_int64 sqlite3_quota_file_truesize(quota_FILE *p){ + int rc; +#if SQLITE_OS_UNIX + struct stat buf; + rc = fstat(fileno(p->f), &buf); +#endif +#if SQLITE_OS_WIN + struct _stati64 buf; + rc = _stati64(p->zMbcsName, &buf); +#endif + return rc==0 ? buf.st_size : -1; +} + +/* +** Return the size of the file, as it is known to the quota subsystem. +*/ +sqlite3_int64 sqlite3_quota_file_size(quota_FILE *p){ + return p->pFile ? p->pFile->iSize : -1; +} + +/* +** Remove a managed file. Update quotas accordingly. +*/ +int sqlite3_quota_remove(const char *zFilename){ + char *zFull; /* Full pathname for zFilename */ + size_t nFull; /* Number of bytes in zFilename */ + int rc; /* Result code */ + quotaGroup *pGroup; /* Group containing zFilename */ + quotaFile *pFile; /* A file in the group */ + quotaFile *pNextFile; /* next file in the group */ + int diff; /* Difference between filenames */ + char c; /* First character past end of pattern */ + + zFull = (char*)sqlite3_malloc(gQuota.sThisVfs.mxPathname + 1); + if( zFull==0 ) return SQLITE_NOMEM; + rc = gQuota.pOrigVfs->xFullPathname(gQuota.pOrigVfs, zFilename, + gQuota.sThisVfs.mxPathname+1, zFull); + if( rc ){ + sqlite3_free(zFull); + return rc; + } + + /* Figure out the length of the full pathname. If the name ends with + ** / (or \ on windows) then remove the trailing /. + */ + nFull = strlen(zFull); + if( nFull>0 && (zFull[nFull-1]=='/' || zFull[nFull-1]=='\\') ){ + nFull--; + zFull[nFull] = 0; + } + + quotaEnter(); + pGroup = quotaGroupFind(zFull); + if( pGroup ){ + for(pFile=pGroup->pFiles; pFile && rc==SQLITE_OK; pFile=pNextFile){ + pNextFile = pFile->pNext; + diff = memcmp(zFull, pFile->zFilename, nFull); + if( diff==0 && ((c = pFile->zFilename[nFull])==0 || c=='/' || c=='\\') ){ + if( pFile->nRef ){ + pFile->deleteOnClose = 1; + }else{ + rc = gQuota.pOrigVfs->xDelete(gQuota.pOrigVfs, pFile->zFilename, 0); + quotaRemoveFile(pFile); + quotaGroupDeref(pGroup); + } + } + } + } + quotaLeave(); + sqlite3_free(zFull); + return rc; +} /***************************** Test Code ***********************************/ #ifdef SQLITE_TEST @@ -1060,9 +1517,13 @@ static int test_quota_dump( Tcl_ListObjAppendElement(interp, pGroupTerm, Tcl_NewWideIntObj(pGroup->iSize)); for(pFile=pGroup->pFiles; pFile; pFile=pFile->pNext){ + int i; + char zTemp[1000]; pFileTerm = Tcl_NewObj(); + sqlite3_snprintf(sizeof(zTemp), zTemp, "%s", pFile->zFilename); + for(i=0; zTemp[i]; i++){ if( zTemp[i]=='\\' ) zTemp[i] = '/'; } Tcl_ListObjAppendElement(interp, pFileTerm, - Tcl_NewStringObj(pFile->zFilename, -1)); + Tcl_NewStringObj(zTemp, -1)); Tcl_ListObjAppendElement(interp, pFileTerm, Tcl_NewWideIntObj(pFile->iSize)); Tcl_ListObjAppendElement(interp, pFileTerm, @@ -1079,6 +1540,362 @@ static int test_quota_dump( } /* +** tclcmd: sqlite3_quota_fopen FILENAME MODE +*/ +static int test_quota_fopen( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + const char *zFilename; /* File pattern to configure */ + const char *zMode; /* Mode string */ + quota_FILE *p; /* Open string object */ + char zReturn[50]; /* Name of pointer to return */ + + /* Process arguments */ + if( objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "FILENAME MODE"); + return TCL_ERROR; + } + zFilename = Tcl_GetString(objv[1]); + zMode = Tcl_GetString(objv[2]); + p = sqlite3_quota_fopen(zFilename, zMode); + sqlite3_snprintf(sizeof(zReturn), zReturn, "%p", p); + Tcl_SetResult(interp, zReturn, TCL_VOLATILE); + return TCL_OK; +} + +/* Defined in test1.c */ +extern void *sqlite3TestTextToPtr(const char*); + +/* +** tclcmd: sqlite3_quota_fread HANDLE SIZE NELEM +*/ +static int test_quota_fread( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + quota_FILE *p; + char *zBuf; + int sz; + int nElem; + size_t got; + + if( objc!=4 ){ + Tcl_WrongNumArgs(interp, 1, objv, "HANDLE SIZE NELEM"); + return TCL_ERROR; + } + p = sqlite3TestTextToPtr(Tcl_GetString(objv[1])); + if( Tcl_GetIntFromObj(interp, objv[2], &sz) ) return TCL_ERROR; + if( Tcl_GetIntFromObj(interp, objv[3], &nElem) ) return TCL_ERROR; + zBuf = (char*)sqlite3_malloc( sz*nElem + 1 ); + if( zBuf==0 ){ + Tcl_SetResult(interp, "out of memory", TCL_STATIC); + return TCL_ERROR; + } + got = sqlite3_quota_fread(zBuf, sz, nElem, p); + if( got<0 ) got = 0; + zBuf[got*sz] = 0; + Tcl_SetResult(interp, zBuf, TCL_VOLATILE); + sqlite3_free(zBuf); + return TCL_OK; +} + +/* +** tclcmd: sqlite3_quota_fwrite HANDLE SIZE NELEM CONTENT +*/ +static int test_quota_fwrite( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + quota_FILE *p; + char *zBuf; + int sz; + int nElem; + size_t got; + + if( objc!=5 ){ + Tcl_WrongNumArgs(interp, 1, objv, "HANDLE SIZE NELEM CONTENT"); + return TCL_ERROR; + } + p = sqlite3TestTextToPtr(Tcl_GetString(objv[1])); + if( Tcl_GetIntFromObj(interp, objv[2], &sz) ) return TCL_ERROR; + if( Tcl_GetIntFromObj(interp, objv[3], &nElem) ) return TCL_ERROR; + zBuf = Tcl_GetString(objv[4]); + got = sqlite3_quota_fwrite(zBuf, sz, nElem, p); + Tcl_SetObjResult(interp, Tcl_NewWideIntObj(got)); + return TCL_OK; +} + +/* +** tclcmd: sqlite3_quota_fclose HANDLE +*/ +static int test_quota_fclose( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + quota_FILE *p; + int rc; + + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "HANDLE"); + return TCL_ERROR; + } + p = sqlite3TestTextToPtr(Tcl_GetString(objv[1])); + rc = sqlite3_quota_fclose(p); + Tcl_SetObjResult(interp, Tcl_NewIntObj(rc)); + return TCL_OK; +} + +/* +** tclcmd: sqlite3_quota_fflush HANDLE ?HARDSYNC? +*/ +static int test_quota_fflush( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + quota_FILE *p; + int rc; + int doSync = 0; + + if( objc!=2 && objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "HANDLE ?HARDSYNC?"); + return TCL_ERROR; + } + p = sqlite3TestTextToPtr(Tcl_GetString(objv[1])); + if( objc==3 ){ + if( Tcl_GetBooleanFromObj(interp, objv[2], &doSync) ) return TCL_ERROR; + } + rc = sqlite3_quota_fflush(p, doSync); + Tcl_SetObjResult(interp, Tcl_NewIntObj(rc)); + return TCL_OK; +} + +/* +** tclcmd: sqlite3_quota_fseek HANDLE OFFSET WHENCE +*/ +static int test_quota_fseek( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + quota_FILE *p; + int ofst; + const char *zWhence; + int whence; + int rc; + + if( objc!=4 ){ + Tcl_WrongNumArgs(interp, 1, objv, "HANDLE OFFSET WHENCE"); + return TCL_ERROR; + } + p = sqlite3TestTextToPtr(Tcl_GetString(objv[1])); + if( Tcl_GetIntFromObj(interp, objv[2], &ofst) ) return TCL_ERROR; + zWhence = Tcl_GetString(objv[3]); + if( strcmp(zWhence, "SEEK_SET")==0 ){ + whence = SEEK_SET; + }else if( strcmp(zWhence, "SEEK_CUR")==0 ){ + whence = SEEK_CUR; + }else if( strcmp(zWhence, "SEEK_END")==0 ){ + whence = SEEK_END; + }else{ + Tcl_AppendResult(interp, + "WHENCE should be SEEK_SET, SEEK_CUR, or SEEK_END", (char*)0); + return TCL_ERROR; + } + rc = sqlite3_quota_fseek(p, ofst, whence); + Tcl_SetObjResult(interp, Tcl_NewIntObj(rc)); + return TCL_OK; +} + +/* +** tclcmd: sqlite3_quota_rewind HANDLE +*/ +static int test_quota_rewind( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + quota_FILE *p; + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "HANDLE"); + return TCL_ERROR; + } + p = sqlite3TestTextToPtr(Tcl_GetString(objv[1])); + sqlite3_quota_rewind(p); + return TCL_OK; +} + +/* +** tclcmd: sqlite3_quota_ftell HANDLE +*/ +static int test_quota_ftell( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + quota_FILE *p; + sqlite3_int64 x; + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "HANDLE"); + return TCL_ERROR; + } + p = sqlite3TestTextToPtr(Tcl_GetString(objv[1])); + x = sqlite3_quota_ftell(p); + Tcl_SetObjResult(interp, Tcl_NewWideIntObj(x)); + return TCL_OK; +} + +/* +** tclcmd: sqlite3_quota_ftruncate HANDLE SIZE +*/ +static int test_quota_ftruncate( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + quota_FILE *p; + sqlite3_int64 x; + Tcl_WideInt w; + int rc; + if( objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "HANDLE SIZE"); + return TCL_ERROR; + } + p = sqlite3TestTextToPtr(Tcl_GetString(objv[1])); + if( Tcl_GetWideIntFromObj(interp, objv[2], &w) ) return TCL_ERROR; + x = (sqlite3_int64)w; + rc = sqlite3_quota_ftruncate(p, x); + Tcl_SetObjResult(interp, Tcl_NewIntObj(rc)); + return TCL_OK; +} + +/* +** tclcmd: sqlite3_quota_file_size HANDLE +*/ +static int test_quota_file_size( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + quota_FILE *p; + sqlite3_int64 x; + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "HANDLE"); + return TCL_ERROR; + } + p = sqlite3TestTextToPtr(Tcl_GetString(objv[1])); + x = sqlite3_quota_file_size(p); + Tcl_SetObjResult(interp, Tcl_NewWideIntObj(x)); + return TCL_OK; +} + +/* +** tclcmd: sqlite3_quota_file_truesize HANDLE +*/ +static int test_quota_file_truesize( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + quota_FILE *p; + sqlite3_int64 x; + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "HANDLE"); + return TCL_ERROR; + } + p = sqlite3TestTextToPtr(Tcl_GetString(objv[1])); + x = sqlite3_quota_file_truesize(p); + Tcl_SetObjResult(interp, Tcl_NewWideIntObj(x)); + return TCL_OK; +} + +/* +** tclcmd: sqlite3_quota_file_mtime HANDLE +*/ +static int test_quota_file_mtime( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + quota_FILE *p; + time_t t; + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "HANDLE"); + return TCL_ERROR; + } + p = sqlite3TestTextToPtr(Tcl_GetString(objv[1])); + t = 0; + sqlite3_quota_file_mtime(p, &t); + Tcl_SetObjResult(interp, Tcl_NewWideIntObj(t)); + return TCL_OK; +} + + +/* +** tclcmd: sqlite3_quota_remove FILENAME +*/ +static int test_quota_remove( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + const char *zFilename; /* File pattern to configure */ + int rc; + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "FILENAME"); + return TCL_ERROR; + } + zFilename = Tcl_GetString(objv[1]); + rc = sqlite3_quota_remove(zFilename); + Tcl_SetObjResult(interp, Tcl_NewIntObj(rc)); + return TCL_OK; +} + +/* +** tclcmd: sqlite3_quota_glob PATTERN TEXT +** +** Test the glob pattern matching. Return 1 if TEXT matches PATTERN +** and return 0 if it does not. +*/ +static int test_quota_glob( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + const char *zPattern; /* The glob pattern */ + const char *zText; /* Text to compare agains the pattern */ + int rc; + if( objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "PATTERN TEXT"); + return TCL_ERROR; + } + zPattern = Tcl_GetString(objv[1]); + zText = Tcl_GetString(objv[2]); + rc = quotaStrglob(zPattern, zText); + Tcl_SetObjResult(interp, Tcl_NewIntObj(rc)); + return TCL_OK; +} + +/* ** This routine registers the custom TCL commands defined in this ** module. This should be the only procedure visible from outside ** of this module. @@ -1088,11 +1905,25 @@ int Sqlitequota_Init(Tcl_Interp *interp){ char *zName; Tcl_ObjCmdProc *xProc; } aCmd[] = { - { "sqlite3_quota_initialize", test_quota_initialize }, - { "sqlite3_quota_shutdown", test_quota_shutdown }, - { "sqlite3_quota_set", test_quota_set }, - { "sqlite3_quota_file", test_quota_file }, - { "sqlite3_quota_dump", test_quota_dump }, + { "sqlite3_quota_initialize", test_quota_initialize }, + { "sqlite3_quota_shutdown", test_quota_shutdown }, + { "sqlite3_quota_set", test_quota_set }, + { "sqlite3_quota_file", test_quota_file }, + { "sqlite3_quota_dump", test_quota_dump }, + { "sqlite3_quota_fopen", test_quota_fopen }, + { "sqlite3_quota_fread", test_quota_fread }, + { "sqlite3_quota_fwrite", test_quota_fwrite }, + { "sqlite3_quota_fclose", test_quota_fclose }, + { "sqlite3_quota_fflush", test_quota_fflush }, + { "sqlite3_quota_fseek", test_quota_fseek }, + { "sqlite3_quota_rewind", test_quota_rewind }, + { "sqlite3_quota_ftell", test_quota_ftell }, + { "sqlite3_quota_ftruncate", test_quota_ftruncate }, + { "sqlite3_quota_file_size", test_quota_file_size }, + { "sqlite3_quota_file_truesize", test_quota_file_truesize }, + { "sqlite3_quota_file_mtime", test_quota_file_mtime }, + { "sqlite3_quota_remove", test_quota_remove }, + { "sqlite3_quota_glob", test_quota_glob }, }; int i; diff --git a/src/test_quota.h b/src/test_quota.h new file mode 100644 index 0000000..9bd4312 --- /dev/null +++ b/src/test_quota.h @@ -0,0 +1,259 @@ +/* +** 2011 December 1 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains the interface definition for the quota a VFS shim. +** +** This particular shim enforces a quota system on files. One or more +** database files are in a "quota group" that is defined by a GLOB +** pattern. A quota is set for the combined size of all files in the +** the group. A quota of zero means "no limit". If the total size +** of all files in the quota group is greater than the limit, then +** write requests that attempt to enlarge a file fail with SQLITE_FULL. +** +** However, before returning SQLITE_FULL, the write requests invoke +** a callback function that is configurable for each quota group. +** This callback has the opportunity to enlarge the quota. If the +** callback does enlarge the quota such that the total size of all +** files within the group is less than the new quota, then the write +** continues as if nothing had happened. +*/ +#ifndef _QUOTA_H_ +#include "sqlite3.h" +#include <stdio.h> +#include <sys/types.h> +#include <sys/stat.h> +#if SQLITE_OS_UNIX +# include <unistd.h> +#endif +#if SQLITE_OS_WIN +# include <windows.h> +#endif + +/* Make this callable from C++ */ +#ifdef __cplusplus +extern "C" { +#endif + +/* +** Initialize the quota VFS shim. Use the VFS named zOrigVfsName +** as the VFS that does the actual work. Use the default if +** zOrigVfsName==NULL. +** +** The quota VFS shim is named "quota". It will become the default +** VFS if makeDefault is non-zero. +** +** THIS ROUTINE IS NOT THREADSAFE. Call this routine exactly once +** during start-up. +*/ +int sqlite3_quota_initialize(const char *zOrigVfsName, int makeDefault); + +/* +** Shutdown the quota system. +** +** All SQLite database connections must be closed before calling this +** routine. +** +** THIS ROUTINE IS NOT THREADSAFE. Call this routine exactly once while +** shutting down in order to free all remaining quota groups. +*/ +int sqlite3_quota_shutdown(void); + +/* +** Create or destroy a quota group. +** +** The quota group is defined by the zPattern. When calling this routine +** with a zPattern for a quota group that already exists, this routine +** merely updates the iLimit, xCallback, and pArg values for that quota +** group. If zPattern is new, then a new quota group is created. +** +** The zPattern is always compared against the full pathname of the file. +** Even if APIs are called with relative pathnames, SQLite converts the +** name to a full pathname before comparing it against zPattern. zPattern +** is a glob pattern with the following matching rules: +** +** '*' Matches any sequence of zero or more characters. +** +** '?' Matches exactly one character. +** +** [...] Matches one character from the enclosed list of +** characters. "]" can be part of the list if it is +** the first character. Within the list "X-Y" matches +** characters X or Y or any character in between the +** two. Ex: "[0-9]" matches any digit. +** +** [^...] Matches one character not in the enclosed list. +** +** / Matches either / or \. This allows glob patterns +** containing / to work on both unix and windows. +** +** Note that, unlike unix shell globbing, the directory separator "/" +** can match a wildcard. So, for example, the pattern "/abc/xyz/" "*" +** matches any files anywhere in the directory hierarchy beneath +** /abc/xyz. +** +** The glob algorithm works on bytes. Multi-byte UTF8 characters are +** matched as if each byte were a separate character. +** +** If the iLimit for a quota group is set to zero, then the quota group +** is disabled and will be deleted when the last database connection using +** the quota group is closed. +** +** Calling this routine on a zPattern that does not exist and with a +** zero iLimit is a no-op. +** +** A quota group must exist with a non-zero iLimit prior to opening +** database connections if those connections are to participate in the +** quota group. Creating a quota group does not affect database connections +** that are already open. +** +** The patterns that define the various quota groups should be distinct. +** If the same filename matches more than one quota group pattern, then +** the behavior of this package is undefined. +*/ +int sqlite3_quota_set( + const char *zPattern, /* The filename pattern */ + sqlite3_int64 iLimit, /* New quota to set for this quota group */ + void (*xCallback)( /* Callback invoked when going over quota */ + const char *zFilename, /* Name of file whose size increases */ + sqlite3_int64 *piLimit, /* IN/OUT: The current limit */ + sqlite3_int64 iSize, /* Total size of all files in the group */ + void *pArg /* Client data */ + ), + void *pArg, /* client data passed thru to callback */ + void (*xDestroy)(void*) /* Optional destructor for pArg */ +); + +/* +** Bring the named file under quota management, assuming its name matches +** the glob pattern of some quota group. Or if it is already under +** management, update its size. If zFilename does not match the glob +** pattern of any quota group, this routine is a no-op. +*/ +int sqlite3_quota_file(const char *zFilename); + +/* +** The following object serves the same role as FILE in the standard C +** library. It represents an open connection to a file on disk for I/O. +** +** A single quota_FILE should not be used by two or more threads at the +** same time. Multiple threads can be using different quota_FILE objects +** simultaneously, but not the same quota_FILE object. +*/ +typedef struct quota_FILE quota_FILE; + +/* +** Create a new quota_FILE object used to read and/or write to the +** file zFilename. The zMode parameter is as with standard library zMode. +*/ +quota_FILE *sqlite3_quota_fopen(const char *zFilename, const char *zMode); + +/* +** Perform I/O against a quota_FILE object. When doing writes, the +** quota mechanism may result in a short write, in order to prevent +** the sum of sizes of all files from going over quota. +*/ +size_t sqlite3_quota_fread(void*, size_t, size_t, quota_FILE*); +size_t sqlite3_quota_fwrite(void*, size_t, size_t, quota_FILE*); + +/* +** Flush all written content held in memory buffers out to disk. +** This is the equivalent of fflush() in the standard library. +** +** If the hardSync parameter is true (non-zero) then this routine +** also forces OS buffers to disk - the equivalent of fsync(). +** +** This routine return zero on success and non-zero if something goes +** wrong. +*/ +int sqlite3_quota_fflush(quota_FILE*, int hardSync); + +/* +** Close a quota_FILE object and free all associated resources. The +** file remains under quota management. +*/ +int sqlite3_quota_fclose(quota_FILE*); + +/* +** Move the read/write pointer for a quota_FILE object. Or tell the +** current location of the read/write pointer. +*/ +int sqlite3_quota_fseek(quota_FILE*, long, int); +void sqlite3_quota_rewind(quota_FILE*); +long sqlite3_quota_ftell(quota_FILE*); + +/* +** Truncate a file previously opened by sqlite3_quota_fopen(). Return +** zero on success and non-zero on any kind of failure. +** +** The newSize argument must be less than or equal to the current file size. +** Any attempt to "truncate" a file to a larger size results in +** undefined behavior. +*/ +int sqlite3_quota_ftrunate(quota_FILE*, sqlite3_int64 newSize); + +/* +** Return the last modification time of the opened file, in seconds +** since 1970. +*/ +int sqlite3_quota_file_mtime(quota_FILE*, time_t *pTime); + +/* +** Return the size of the file as it is known to the quota system. +** +** This size might be different from the true size of the file on +** disk if some outside process has modified the file without using the +** quota mechanism, or if calls to sqlite3_quota_fwrite() have occurred +** which have increased the file size, but those writes have not yet been +** forced to disk using sqlite3_quota_fflush(). +** +** Return -1 if the file is not participating in quota management. +*/ +sqlite3_int64 sqlite3_quota_file_size(quota_FILE*); + +/* +** Return the true size of the file. +** +** The true size should be the same as the size of the file as known +** to the quota system, however the sizes might be different if the +** file has been extended or truncated via some outside process or if +** pending writes have not yet been flushed to disk. +** +** Return -1 if the file does not exist or if the size of the file +** cannot be determined for some reason. +*/ +sqlite3_int64 sqlite3_quota_file_truesize(quota_FILE*); + +/* +** Delete a file from the disk, if that file is under quota management. +** Adjust quotas accordingly. +** +** If zFilename is the name of a directory that matches one of the +** quota glob patterns, then all files under quota management that +** are contained within that directory are deleted. +** +** A standard SQLite result code is returned (SQLITE_OK, SQLITE_NOMEM, etc.) +** When deleting a directory of files, if the deletion of any one +** file fails (for example due to an I/O error), then this routine +** returns immediately, with the error code, and does not try to +** delete any of the other files in the specified directory. +** +** All files are removed from quota management and deleted from disk. +** However, no attempt is made to remove empty directories. +** +** This routine is a no-op for files that are not under quota management. +*/ +int sqlite3_quota_remove(const char *zFilename); + +#ifdef __cplusplus +} /* end of the 'extern "C"' block */ +#endif +#endif /* _QUOTA_H_ */ diff --git a/src/test_rtree.c b/src/test_rtree.c index 9745b00..d3c9e0c 100644 --- a/src/test_rtree.c +++ b/src/test_rtree.c @@ -49,7 +49,11 @@ static void circle_del(void *p){ static int circle_geom( sqlite3_rtree_geometry *p, int nCoord, +#ifdef SQLITE_RTREE_INT_ONLY + sqlite3_int64 *aCoord, +#else double *aCoord, +#endif int *pRes ){ int i; /* Iterator variable */ @@ -188,8 +192,12 @@ static int gHere = 42; */ static int cube_geom( sqlite3_rtree_geometry *p, - int nCoord, + int nCoord, +#ifdef SQLITE_RTREE_INT_ONLY + sqlite3_int64 *aCoord, +#else double *aCoord, +#endif int *piRes ){ Cube *pCube = (Cube *)p->pUser; diff --git a/src/test_spellfix.c b/src/test_spellfix.c new file mode 100644 index 0000000..5a221e0 --- /dev/null +++ b/src/test_spellfix.c @@ -0,0 +1,1951 @@ +/* +** 2012 April 10 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This module implements a VIRTUAL TABLE that can be used to search +** a large vocabulary for close matches. For example, this virtual +** table can be used to suggest corrections to misspelled words. Or, +** it could be used with FTS4 to do full-text search using potentially +** misspelled words. +** +** Create an instance of the virtual table this way: +** +** CREATE VIRTUAL TABLE demo USING spellfix1; +** +** The "spellfix1" term is the name of this module. The "demo" is the +** name of the virtual table you will be creating. The table is initially +** empty. You have to populate it with your vocabulary. Suppose you +** have a list of words in a table named "big_vocabulary". Then do this: +** +** INSERT INTO demo(word) SELECT word FROM big_vocabulary; +** +** If you intend to use this virtual table in cooperation with an FTS4 +** table (for spelling correctly of search terms) then you can extract +** the vocabulary using an fts3aux table: +** +** INSERT INTO demo(word) SELECT term FROM search_aux WHERE col='*'; +** +** You can also provide the virtual table with a "rank" for each word. +** The "rank" is an estimate of how common the word is. Larger numbers +** mean the word is more common. If you omit the rank when populating +** the table, then a rank of 1 is assumed. But if you have rank +** information, you can supply it and the virtual table will show a +** slight preference for selecting more commonly used terms. To +** populate the rank from an fts4aux table "search_aux" do something +** like this: +** +** INSERT INTO demo(word,rank) +** SELECT term, documents FROM search_aux WHERE col='*'; +** +** To query the virtual table, include a MATCH operator in the WHERE +** clause. For example: +** +** SELECT word FROM demo WHERE word MATCH 'kennasaw'; +** +** Using a dataset of American place names (derived from +** http://geonames.usgs.gov/domestic/download_data.htm) the query above +** returns 20 results beginning with: +** +** kennesaw +** kenosha +** kenesaw +** kenaga +** keanak +** +** If you append the character '*' to the end of the pattern, then +** a prefix search is performed. For example: +** +** SELECT word FROM demo WHERE word MATCH 'kennes*'; +** +** Yields 20 results beginning with: +** +** kennesaw +** kennestone +** kenneson +** kenneys +** keanes +** keenes +** +** The virtual table actually has a unique rowid with five columns plus three +** extra hidden columns. The columns are as follows: +** +** rowid A unique integer number associated with each +** vocabulary item in the table. This can be used +** as a foreign key on other tables in the database. +** +** word The text of the word that matches the pattern. +** Both word and pattern can contains unicode characters +** and can be mixed case. +** +** rank This is the rank of the word, as specified in the +** original INSERT statement. +** +** distance This is an edit distance or Levensthein distance going +** from the pattern to the word. +** +** langid This is the language-id of the word. All queries are +** against a single language-id, which defaults to 0. +** For any given query this value is the same on all rows. +** +** score The score is a combination of rank and distance. The +** idea is that a lower score is better. The virtual table +** attempts to find words with the lowest score and +** by default (unless overridden by ORDER BY) returns +** results in order of increasing score. +** +** top (HIDDEN) For any query, this value is the same on all +** rows. It is an integer which is the maximum number of +** rows that will be output. The actually number of rows +** output might be less than this number, but it will never +** be greater. The default value for top is 20, but that +** can be changed for each query by including a term of +** the form "top=N" in the WHERE clause of the query. +** +** scope (HIDDEN) For any query, this value is the same on all +** rows. The scope is a measure of how widely the virtual +** table looks for matching words. Smaller values of +** scope cause a broader search. The scope is normally +** choosen automatically and is capped at 4. Applications +** can change the scope by including a term of the form +** "scope=N" in the WHERE clause of the query. Increasing +** the scope will make the query run faster, but will reduce +** the possible corrections. +** +** srchcnt (HIDDEN) For any query, this value is the same on all +** rows. This value is an integer which is the number of +** of words examined using the edit-distance algorithm to +** find the top matches that are ultimately displayed. This +** value is for diagnostic use only. +** +** soundslike (HIDDEN) When inserting vocabulary entries, this field +** can be set to an spelling that matches what the word +** sounds like. See the DEALING WITH UNUSUAL AND DIFFICULT +** SPELLINGS section below for details. +** +** When inserting into or updating the virtual table, only the rowid, word, +** rank, and langid may be changes. Any attempt to set or modify the values +** of distance, score, top, scope, or srchcnt is silently ignored. +** +** ALGORITHM +** +** A shadow table named "%_vocab" (where the % is replaced by the name of +** the virtual table; Ex: "demo_vocab" for the "demo" virtual table) is +** constructed with these columns: +** +** id The unique id (INTEGER PRIMARY KEY) +** +** rank The rank of word. +** +** langid The language id for this entry. +** +** word The original UTF8 text of the vocabulary word +** +** k1 The word transliterated into lower-case ASCII. +** There is a standard table of mappings from non-ASCII +** characters into ASCII. Examples: "æ" -> "ae", +** "þ" -> "th", "ß" -> "ss", "á" -> "a", ... The +** accessory function spellfix1_translit(X) will do +** the non-ASCII to ASCII mapping. The built-in lower(X) +** function will convert to lower-case. Thus: +** k1 = lower(spellfix1_translit(word)). +** +** k2 This field holds a phonetic code derived from k1. Letters +** that have similar sounds are mapped into the same symbol. +** For example, all vowels and vowel clusters become the +** single symbol "A". And the letters "p", "b", "f", and +** "v" all become "B". All nasal sounds are represented +** as "N". And so forth. The mapping is base on +** ideas found in Soundex, Metaphone, and other +** long-standing phonetic matching systems. This key can +** be generated by the function spellfix1_charclass(X). +** Hence: k2 = spellfix1_charclass(k1) +** +** There is also a function for computing the Wagner edit distance or the +** Levenshtein distance between a pattern and a word. This function +** is exposed as spellfix1_editdist(X,Y). The edit distance function +** returns the "cost" of converting X into Y. Some transformations +** cost more than others. Changing one vowel into a different vowel, +** for example is relatively cheap, as is doubling a constant, or +** omitting the second character of a double-constant. Other transformations +** or more expensive. The idea is that the edit distance function returns +** a low cost of words that are similar and a higher cost for words +** that are futher apart. In this implementation, the maximum cost +** of any single-character edit (delete, insert, or substitute) is 100, +** with lower costs for some edits (such as transforming vowels). +** +** The "score" for a comparison is the edit distance between the pattern +** and the word, adjusted down by the base-2 logorithm of the word rank. +** For example, a match with distance 100 but rank 1000 would have a +** score of 122 (= 100 - log2(1000) + 32) where as a match with distance +** 100 with a rank of 1 would have a score of 131 (100 - log2(1) + 32). +** (NB: The constant 32 is added to each score to keep it from going +** negative in case the edit distance is zero.) In this way, frequently +** used words get a slightly lower cost which tends to move them toward +** the top of the list of alternative spellings. +** +** A straightforward implementation of a spelling corrector would be +** to compare the search term against every word in the vocabulary +** and select the 20 with the lowest scores. However, there will +** typically be hundreds of thousands or millions of words in the +** vocabulary, and so this approach is not fast enough. +** +** Suppose the term that is being spell-corrected is X. To limit +** the search space, X is converted to a k2-like key using the +** equivalent of: +** +** key = spellfix1_charclass(lower(spellfix1_translit(X))) +** +** This key is then limited to "scope" characters. The default scope +** value is 4, but an alternative scope can be specified using the +** "scope=N" term in the WHERE clause. After the key has been truncated, +** the edit distance is run against every term in the vocabulary that +** has a k2 value that begins with the abbreviated key. +** +** For example, suppose the input word is "Paskagula". The phonetic +** key is "BACACALA" which is then truncated to 4 characters "BACA". +** The edit distance is then run on the 4980 entries (out of +** 272,597 entries total) of the vocabulary whose k2 values begin with +** BACA, yielding "Pascagoula" as the best match. +** +** Only terms of the vocabulary with a matching langid are searched. +** Hence, the same table can contain entries from multiple languages +** and only the requested language will be used. The default langid +** is 0. +** +** DEALING WITH UNUSUAL AND DIFFICULT SPELLINGS +** +** The algorithm above works quite well for most cases, but there are +** exceptions. These exceptions can be dealt with by making additional +** entries in the virtual table using the "soundslike" column. +** +** For example, many words of Greek origin begin with letters "ps" where +** the "p" is silent. Ex: psalm, pseudonym, psoriasis, psyche. In +** another example, many Scottish surnames can be spelled with an +** initial "Mac" or "Mc". Thus, "MacKay" and "McKay" are both pronounced +** the same. +** +** Accommodation can be made for words that are not spelled as they +** sound by making additional entries into the virtual table for the +** same word, but adding an alternative spelling in the "soundslike" +** column. For example, the canonical entry for "psalm" would be this: +** +** INSERT INTO demo(word) VALUES('psalm'); +** +** To enhance the ability to correct the spelling of "salm" into +** "psalm", make an addition entry like this: +** +** INSERT INTO demo(word,soundslike) VALUES('psalm','salm'); +** +** It is ok to make multiple entries for the same word as long as +** each entry has a different soundslike value. Note that if no +** soundslike value is specified, the soundslike defaults to the word +** itself. +** +** Listed below are some cases where it might make sense to add additional +** soundslike entries. The specific entries will depend on the application +** and the target language. +** +** * Silent "p" in words beginning with "ps": psalm, psyche +** +** * Silent "p" in words beginning with "pn": pneumonia, pneumatic +** +** * Silent "p" in words beginning with "pt": pterodactyl, ptolemaic +** +** * Silent "d" in words beginning with "dj": djinn, Djikarta +** +** * Silent "k" in words beginning with "kn": knight, Knuthson +** +** * Silent "g" in words beginning with "gn": gnarly, gnome, gnat +** +** * "Mac" versus "Mc" beginning Scottish surnames +** +** * "Tch" sounds in Slavic words: Tchaikovsky vs. Chaykovsky +** +** * The letter "j" pronounced like "h" in Spanish: LaJolla +** +** * Words beginning with "wr" versus "r": write vs. rite +** +** * Miscellanous problem words such as "debt", "tsetse", +** "Nguyen", "Van Nuyes". +*/ +#if SQLITE_CORE +# include "sqliteInt.h" +#else +# include <string.h> +# include <stdio.h> +# include <stdlib.h> +# include "sqlite3ext.h" + SQLITE_EXTENSION_INIT1 +#endif /* !SQLITE_CORE */ + +/* +** Character classes for ASCII characters: +** +** 0 '' Silent letters: H W +** 1 'A' Any vowel: A E I O U (Y) +** 2 'B' A bilabeal stop or fricative: B F P V +** 3 'C' Other fricatives or back stops: C G J K Q S X Z +** 4 'D' Alveolar stops: D T +** 5 'H' Letter H at the beginning of a word +** 6 'L' Glides: L R +** 7 'M' Nasals: M N +** 8 'W' Letter W at the beginning of a word +** 9 'Y' Letter Y at the beginning of a word. +** 10 '9' A digit: 0 1 2 3 4 5 6 7 8 9 +** 11 ' ' White space +** 12 '?' Other. +*/ +#define CCLASS_SILENT 0 +#define CCLASS_VOWEL 1 +#define CCLASS_B 2 +#define CCLASS_C 3 +#define CCLASS_D 4 +#define CCLASS_H 5 +#define CCLASS_L 6 +#define CCLASS_M 7 +#define CCLASS_W 8 +#define CCLASS_Y 9 +#define CCLASS_DIGIT 10 +#define CCLASS_SPACE 11 +#define CCLASS_OTHER 12 + +/* +** The following table gives the character class for non-initial ASCII +** characters. +*/ +static const unsigned char midClass[] = { + /* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xa xb xc xd xe xf */ + /* 0x */ 12, 12, 12, 12, 12, 12, 12, 12, 12, 11, 11, 12, 11, 12, 12, 12, + /* 1x */ 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, + /* 2x */ 11, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, + /* 3x */ 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 12, 12, 12, 12, 12, 12, + /* 4x */ 12, 1, 2, 3, 4, 1, 2, 3, 0, 1, 3, 3, 6, 7, 7, 1, + /* 5x */ 2, 3, 6, 3, 4, 1, 2, 0, 3, 1, 3, 12, 12, 12, 12, 12, + /* 6x */ 12, 1, 2, 3, 4, 1, 2, 3, 0, 1, 3, 3, 6, 7, 7, 1, + /* 7x */ 2, 3, 6, 3, 4, 1, 2, 0, 3, 1, 3, 12, 12, 12, 12, 12, +}; + +/* +** This tables gives the character class for ASCII characters that form the +** initial character of a word. The only difference from midClass is with +** the letters H, W, and Y. +*/ +static const unsigned char initClass[] = { + /* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xa xb xc xd xe xf */ + /* 0x */ 12, 12, 12, 12, 12, 12, 12, 12, 12, 11, 11, 12, 11, 12, 12, 12, + /* 1x */ 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, + /* 2x */ 11, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, + /* 3x */ 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 12, 12, 12, 12, 12, 12, + /* 4x */ 12, 1, 2, 3, 4, 1, 2, 3, 5, 1, 3, 3, 6, 7, 7, 1, + /* 5x */ 2, 3, 6, 3, 4, 1, 2, 8, 3, 9, 3, 12, 12, 12, 12, 12, + /* 6x */ 12, 1, 2, 3, 4, 1, 2, 3, 5, 1, 3, 3, 6, 7, 7, 1, + /* 7x */ 2, 3, 6, 3, 4, 1, 2, 8, 3, 9, 3, 12, 12, 12, 12, 12, +}; + +/* +** Mapping from the character class number (0-12) to a symbol for each +** character class. Note that initClass[] can be used to map the class +** symbol back into the class number. +*/ +static const unsigned char className[] = ".ABCDHLMWY9 ?"; + +/* +** Generate a string of character classes corresponding to the +** ASCII characters in the input string zIn. If the input is not +** ASCII then the behavior is undefined. +** +** Space to hold the result is obtained from sqlite3_malloc() +** +** Return NULL if memory allocation fails. +*/ +static unsigned char *characterClassString(const unsigned char *zIn, int nIn){ + unsigned char *zOut = sqlite3_malloc( nIn + 1 ); + int i; + int nOut = 0; + char cPrev = 0x77; + const unsigned char *aClass = initClass; + + if( zOut==0 ) return 0; + for(i=0; i<nIn; i++){ + unsigned char c = zIn[i]; + c = aClass[c&0x7f]; + if( c==CCLASS_OTHER && cPrev!=CCLASS_DIGIT ) continue; + cPrev = c; + if( c==CCLASS_SILENT ) continue; + if( c==CCLASS_SPACE ) continue; + aClass = midClass; + c = className[c]; + if( c!=zOut[nOut-1] ) zOut[nOut++] = c; + } + zOut[nOut] = 0; + return zOut; +} + +/* +** This is an SQL function wrapper around characterClassString(). See +** the description of characterClassString() for additional information. +*/ +static void characterClassSqlFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const unsigned char *zIn; + unsigned char *zOut; + + zIn = sqlite3_value_text(argv[0]); + if( zIn==0 ) return; + zOut = characterClassString(zIn, sqlite3_value_bytes(argv[0])); + if( zOut==0 ){ + sqlite3_result_error_nomem(context); + }else{ + sqlite3_result_text(context, (char*)zOut, -1, sqlite3_free); + } +} + +/* +** Return the character class number for a character given its +** context. +*/ +static char characterClass(char cPrev, char c){ + return cPrev==0 ? initClass[c&0x7f] : midClass[c&0x7f]; +} + +/* +** Return the cost of inserting or deleting character c immediately +** following character cPrev. If cPrev==0, that means c is the first +** character of the word. +*/ +static int insertOrDeleteCost(char cPrev, char c){ + char classC = characterClass(cPrev, c); + char classCprev; + + if( classC==CCLASS_SILENT ){ + /* Insert or delete "silent" characters such as H or W */ + return 1; + } + if( cPrev==c ){ + /* Repeated characters, or miss a repeat */ + return 10; + } + classCprev = characterClass(cPrev, cPrev); + if( classC==classCprev ){ + if( classC==CCLASS_VOWEL ){ + /* Remove or add a new vowel to a vowel cluster */ + return 15; + }else{ + /* Remove or add a consonant not in the same class */ + return 50; + } + } + + /* any other character insertion or deletion */ + return 100; +} + +/* +** Divide the insertion cost by this factor when appending to the +** end of the word. +*/ +#define FINAL_INS_COST_DIV 4 + +/* +** Return the cost of substituting cTo in place of cFrom assuming +** the previous character is cPrev. If cPrev==0 then cTo is the first +** character of the word. +*/ +static int substituteCost(char cPrev, char cFrom, char cTo){ + char classFrom, classTo; + if( cFrom==cTo ){ + /* Exact match */ + return 0; + } + if( cFrom==(cTo^0x20) && ((cTo>='A' && cTo<='Z') || (cTo>='a' && cTo<='z')) ){ + /* differ only in case */ + return 0; + } + classFrom = characterClass(cPrev, cFrom); + classTo = characterClass(cPrev, cTo); + if( classFrom==classTo ){ + /* Same character class */ + return classFrom=='A' ? 25 : 40; + } + if( classFrom>=CCLASS_B && classFrom<=CCLASS_Y + && classTo>=CCLASS_B && classTo<=CCLASS_Y ){ + /* Convert from one consonant to another, but in a different class */ + return 75; + } + /* Any other subsitution */ + return 100; +} + +/* +** Given two strings zA and zB which are pure ASCII, return the cost +** of transforming zA into zB. If zA ends with '*' assume that it is +** a prefix of zB and give only minimal penalty for extra characters +** on the end of zB. +** +** Smaller numbers mean a closer match. +** +** Negative values indicate an error: +** -1 One of the inputs is NULL +** -2 Non-ASCII characters on input +** -3 Unable to allocate memory +*/ +static int editdist(const char *zA, const char *zB){ + int nA, nB; /* Number of characters in zA[] and zB[] */ + int xA, xB; /* Loop counters for zA[] and zB[] */ + char cA, cB; /* Current character of zA and zB */ + char cAprev, cBprev; /* Previous character of zA and zB */ + int d; /* North-west cost value */ + int dc = 0; /* North-west character value */ + int res; /* Final result */ + int *m; /* The cost matrix */ + char *cx; /* Corresponding character values */ + int *toFree = 0; /* Malloced space */ + int mStack[60+15]; /* Stack space to use if not too much is needed */ + + /* Early out if either input is NULL */ + if( zA==0 || zB==0 ) return -1; + + /* Skip any common prefix */ + while( zA[0] && zA[0]==zB[0] ){ dc = zA[0]; zA++; zB++; } + if( zA[0]==0 && zB[0]==0 ) return 0; + +#if 0 + printf("A=\"%s\" B=\"%s\" dc=%c\n", zA, zB, dc?dc:' '); +#endif + + /* Verify input strings and measure their lengths */ + for(nA=0; zA[nA]; nA++){ + if( zA[nA]>127 ) return -2; + } + for(nB=0; zB[nB]; nB++){ + if( zB[nB]>127 ) return -2; + } + + /* Special processing if either string is empty */ + if( nA==0 ){ + cBprev = dc; + for(xB=res=0; (cB = zB[xB])!=0; xB++){ + res += insertOrDeleteCost(cBprev, cB)/FINAL_INS_COST_DIV; + cBprev = cB; + } + return res; + } + if( nB==0 ){ + cAprev = dc; + for(xA=res=0; (cA = zA[xA])!=0; xA++){ + res += insertOrDeleteCost(cAprev, cA); + cAprev = cA; + } + return res; + } + + /* A is a prefix of B */ + if( zA[0]=='*' && zA[1]==0 ) return 0; + + /* Allocate and initialize the Wagner matrix */ + if( nB<(sizeof(mStack)*4)/(sizeof(mStack[0])*5) ){ + m = mStack; + }else{ + m = toFree = sqlite3_malloc( (nB+1)*5*sizeof(m[0])/4 ); + if( m==0 ) return -3; + } + cx = (char*)&m[nB+1]; + + /* Compute the Wagner edit distance */ + m[0] = 0; + cx[0] = dc; + cBprev = dc; + for(xB=1; xB<=nB; xB++){ + cB = zB[xB-1]; + cx[xB] = cB; + m[xB] = m[xB-1] + insertOrDeleteCost(cBprev, cB); + cBprev = cB; + } + cAprev = dc; + for(xA=1; xA<=nA; xA++){ + int lastA = (xA==nA); + cA = zA[xA-1]; + if( cA=='*' && lastA ) break; + d = m[0]; + dc = cx[0]; + m[0] = d + insertOrDeleteCost(cAprev, cA); + cBprev = 0; + for(xB=1; xB<=nB; xB++){ + int totalCost, insCost, delCost, subCost, ncx; + cB = zB[xB-1]; + + /* Cost to insert cB */ + insCost = insertOrDeleteCost(cx[xB-1], cB); + if( lastA ) insCost /= FINAL_INS_COST_DIV; + + /* Cost to delete cA */ + delCost = insertOrDeleteCost(cx[xB], cA); + + /* Cost to substitute cA->cB */ + subCost = substituteCost(cx[xB-1], cA, cB); + + /* Best cost */ + totalCost = insCost + m[xB-1]; + ncx = cB; + if( (delCost + m[xB])<totalCost ){ + totalCost = delCost + m[xB]; + ncx = cA; + } + if( (subCost + d)<totalCost ){ + totalCost = subCost + d; + } + +#if 0 + printf("%d,%d d=%4d u=%4d r=%4d dc=%c cA=%c cB=%c" + " ins=%4d del=%4d sub=%4d t=%4d ncx=%c\n", + xA, xB, d, m[xB], m[xB-1], dc?dc:' ', cA, cB, + insCost, delCost, subCost, totalCost, ncx?ncx:' '); +#endif + + /* Update the matrix */ + d = m[xB]; + dc = cx[xB]; + m[xB] = totalCost; + cx[xB] = ncx; + cBprev = cB; + } + cAprev = cA; + } + + /* Free the wagner matrix and return the result */ + if( cA=='*' && nB>nA ){ + res = m[nA]; + for(xB=nA+1; xB<=nB; xB++){ + if( m[xB]<res ) res = m[xB]; + } + }else{ + res = m[nB]; + } + sqlite3_free(toFree); + return res; +} + +/* +** Function: editdist(A,B) +** +** Return the cost of transforming string A into string B. Both strings +** must be pure ASCII text. If A ends with '*' then it is assumed to be +** a prefix of B and extra characters on the end of B have minimal additional +** cost. +*/ +static void editdistSqlFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int res = editdist((const char*)sqlite3_value_text(argv[0]), + (const char*)sqlite3_value_text(argv[1])); + if( res<0 ){ + if( res==(-3) ){ + sqlite3_result_error_nomem(context); + }else if( res==(-2) ){ + sqlite3_result_error(context, "non-ASCII input to editdist()", -1); + }else{ + sqlite3_result_error(context, "NULL input to editdist()", -1); + } + }else{ + sqlite3_result_int(context, res); + } +} + +#if !SQLITE_CORE +/* +** This lookup table is used to help decode the first byte of +** a multi-byte UTF8 character. +*/ +static const unsigned char sqlite3Utf8Trans1[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x00, 0x00, +}; +#endif + +/* +** Return the value of the first UTF-8 character in the string. +*/ +static int utf8Read(const unsigned char *z, int n, int *pSize){ + int c, i; + + if( n==0 ){ + c = i = 0; + }else{ + c = z[0]; + i = 1; + if( c>=0xc0 ){ + c = sqlite3Utf8Trans1[c-0xc0]; + while( i<n && (z[i] & 0xc0)==0x80 ){ + c = (c<<6) + (0x3f & z[i++]); + } + } + } + *pSize = i; + return c; +} + +/* +** Table of translations from unicode characters into ASCII. +*/ +static const struct { + unsigned short int cFrom; + unsigned char cTo0, cTo1; +} translit[] = { + { 0x00A0, 0x20, 0x00 }, /* to */ + { 0x00B5, 0x75, 0x00 }, /* µ to u */ + { 0x00C0, 0x41, 0x00 }, /* À to A */ + { 0x00C1, 0x41, 0x00 }, /* Á to A */ + { 0x00C2, 0x41, 0x00 }, /*  to A */ + { 0x00C3, 0x41, 0x00 }, /* à to A */ + { 0x00C4, 0x41, 0x65 }, /* Ä to Ae */ + { 0x00C5, 0x41, 0x61 }, /* Å to Aa */ + { 0x00C6, 0x41, 0x45 }, /* Æ to AE */ + { 0x00C7, 0x43, 0x00 }, /* Ç to C */ + { 0x00C8, 0x45, 0x00 }, /* È to E */ + { 0x00C9, 0x45, 0x00 }, /* É to E */ + { 0x00CA, 0x45, 0x00 }, /* Ê to E */ + { 0x00CB, 0x45, 0x00 }, /* Ë to E */ + { 0x00CC, 0x49, 0x00 }, /* Ì to I */ + { 0x00CD, 0x49, 0x00 }, /* Í to I */ + { 0x00CE, 0x49, 0x00 }, /* Î to I */ + { 0x00CF, 0x49, 0x00 }, /* Ï to I */ + { 0x00D0, 0x44, 0x00 }, /* Ð to D */ + { 0x00D1, 0x4E, 0x00 }, /* Ñ to N */ + { 0x00D2, 0x4F, 0x00 }, /* Ò to O */ + { 0x00D3, 0x4F, 0x00 }, /* Ó to O */ + { 0x00D4, 0x4F, 0x00 }, /* Ô to O */ + { 0x00D5, 0x4F, 0x00 }, /* Õ to O */ + { 0x00D6, 0x4F, 0x65 }, /* Ö to Oe */ + { 0x00D7, 0x78, 0x00 }, /* × to x */ + { 0x00D8, 0x4F, 0x00 }, /* Ø to O */ + { 0x00D9, 0x55, 0x00 }, /* Ù to U */ + { 0x00DA, 0x55, 0x00 }, /* Ú to U */ + { 0x00DB, 0x55, 0x00 }, /* Û to U */ + { 0x00DC, 0x55, 0x65 }, /* Ü to Ue */ + { 0x00DD, 0x59, 0x00 }, /* Ý to Y */ + { 0x00DE, 0x54, 0x68 }, /* Þ to Th */ + { 0x00DF, 0x73, 0x73 }, /* ß to ss */ + { 0x00E0, 0x61, 0x00 }, /* à to a */ + { 0x00E1, 0x61, 0x00 }, /* á to a */ + { 0x00E2, 0x61, 0x00 }, /* â to a */ + { 0x00E3, 0x61, 0x00 }, /* ã to a */ + { 0x00E4, 0x61, 0x65 }, /* ä to ae */ + { 0x00E5, 0x61, 0x61 }, /* å to aa */ + { 0x00E6, 0x61, 0x65 }, /* æ to ae */ + { 0x00E7, 0x63, 0x00 }, /* ç to c */ + { 0x00E8, 0x65, 0x00 }, /* è to e */ + { 0x00E9, 0x65, 0x00 }, /* é to e */ + { 0x00EA, 0x65, 0x00 }, /* ê to e */ + { 0x00EB, 0x65, 0x00 }, /* ë to e */ + { 0x00EC, 0x69, 0x00 }, /* ì to i */ + { 0x00ED, 0x69, 0x00 }, /* í to i */ + { 0x00EE, 0x69, 0x00 }, /* î to i */ + { 0x00EF, 0x69, 0x00 }, /* ï to i */ + { 0x00F0, 0x64, 0x00 }, /* ð to d */ + { 0x00F1, 0x6E, 0x00 }, /* ñ to n */ + { 0x00F2, 0x6F, 0x00 }, /* ò to o */ + { 0x00F3, 0x6F, 0x00 }, /* ó to o */ + { 0x00F4, 0x6F, 0x00 }, /* ô to o */ + { 0x00F5, 0x6F, 0x00 }, /* õ to o */ + { 0x00F6, 0x6F, 0x65 }, /* ö to oe */ + { 0x00F7, 0x3A, 0x00 }, /* ÷ to : */ + { 0x00F8, 0x6F, 0x00 }, /* ø to o */ + { 0x00F9, 0x75, 0x00 }, /* ù to u */ + { 0x00FA, 0x75, 0x00 }, /* ú to u */ + { 0x00FB, 0x75, 0x00 }, /* û to u */ + { 0x00FC, 0x75, 0x65 }, /* ü to ue */ + { 0x00FD, 0x79, 0x00 }, /* ý to y */ + { 0x00FE, 0x74, 0x68 }, /* þ to th */ + { 0x00FF, 0x79, 0x00 }, /* ÿ to y */ + { 0x0100, 0x41, 0x00 }, /* Ā to A */ + { 0x0101, 0x61, 0x00 }, /* ā to a */ + { 0x0102, 0x41, 0x00 }, /* Ă to A */ + { 0x0103, 0x61, 0x00 }, /* ă to a */ + { 0x0104, 0x41, 0x00 }, /* Ą to A */ + { 0x0105, 0x61, 0x00 }, /* ą to a */ + { 0x0106, 0x43, 0x00 }, /* Ć to C */ + { 0x0107, 0x63, 0x00 }, /* ć to c */ + { 0x0108, 0x43, 0x68 }, /* Ĉ to Ch */ + { 0x0109, 0x63, 0x68 }, /* ĉ to ch */ + { 0x010A, 0x43, 0x00 }, /* Ċ to C */ + { 0x010B, 0x63, 0x00 }, /* ċ to c */ + { 0x010C, 0x43, 0x00 }, /* Č to C */ + { 0x010D, 0x63, 0x00 }, /* č to c */ + { 0x010E, 0x44, 0x00 }, /* Ď to D */ + { 0x010F, 0x64, 0x00 }, /* ď to d */ + { 0x0110, 0x44, 0x00 }, /* Đ to D */ + { 0x0111, 0x64, 0x00 }, /* đ to d */ + { 0x0112, 0x45, 0x00 }, /* Ē to E */ + { 0x0113, 0x65, 0x00 }, /* ē to e */ + { 0x0114, 0x45, 0x00 }, /* Ĕ to E */ + { 0x0115, 0x65, 0x00 }, /* ĕ to e */ + { 0x0116, 0x45, 0x00 }, /* Ė to E */ + { 0x0117, 0x65, 0x00 }, /* ė to e */ + { 0x0118, 0x45, 0x00 }, /* Ę to E */ + { 0x0119, 0x65, 0x00 }, /* ę to e */ + { 0x011A, 0x45, 0x00 }, /* Ě to E */ + { 0x011B, 0x65, 0x00 }, /* ě to e */ + { 0x011C, 0x47, 0x68 }, /* Ĝ to Gh */ + { 0x011D, 0x67, 0x68 }, /* ĝ to gh */ + { 0x011E, 0x47, 0x00 }, /* Ğ to G */ + { 0x011F, 0x67, 0x00 }, /* ğ to g */ + { 0x0120, 0x47, 0x00 }, /* Ġ to G */ + { 0x0121, 0x67, 0x00 }, /* ġ to g */ + { 0x0122, 0x47, 0x00 }, /* Ģ to G */ + { 0x0123, 0x67, 0x00 }, /* ģ to g */ + { 0x0124, 0x48, 0x68 }, /* Ĥ to Hh */ + { 0x0125, 0x68, 0x68 }, /* ĥ to hh */ + { 0x0126, 0x48, 0x00 }, /* Ħ to H */ + { 0x0127, 0x68, 0x00 }, /* ħ to h */ + { 0x0128, 0x49, 0x00 }, /* Ĩ to I */ + { 0x0129, 0x69, 0x00 }, /* ĩ to i */ + { 0x012A, 0x49, 0x00 }, /* Ī to I */ + { 0x012B, 0x69, 0x00 }, /* ī to i */ + { 0x012C, 0x49, 0x00 }, /* Ĭ to I */ + { 0x012D, 0x69, 0x00 }, /* ĭ to i */ + { 0x012E, 0x49, 0x00 }, /* Į to I */ + { 0x012F, 0x69, 0x00 }, /* į to i */ + { 0x0130, 0x49, 0x00 }, /* İ to I */ + { 0x0131, 0x69, 0x00 }, /* ı to i */ + { 0x0132, 0x49, 0x4A }, /* IJ to IJ */ + { 0x0133, 0x69, 0x6A }, /* ij to ij */ + { 0x0134, 0x4A, 0x68 }, /* Ĵ to Jh */ + { 0x0135, 0x6A, 0x68 }, /* ĵ to jh */ + { 0x0136, 0x4B, 0x00 }, /* Ķ to K */ + { 0x0137, 0x6B, 0x00 }, /* ķ to k */ + { 0x0138, 0x6B, 0x00 }, /* ĸ to k */ + { 0x0139, 0x4C, 0x00 }, /* Ĺ to L */ + { 0x013A, 0x6C, 0x00 }, /* ĺ to l */ + { 0x013B, 0x4C, 0x00 }, /* Ļ to L */ + { 0x013C, 0x6C, 0x00 }, /* ļ to l */ + { 0x013D, 0x4C, 0x00 }, /* Ľ to L */ + { 0x013E, 0x6C, 0x00 }, /* ľ to l */ + { 0x013F, 0x4C, 0x2E }, /* Ŀ to L. */ + { 0x0140, 0x6C, 0x2E }, /* ŀ to l. */ + { 0x0141, 0x4C, 0x00 }, /* Ł to L */ + { 0x0142, 0x6C, 0x00 }, /* ł to l */ + { 0x0143, 0x4E, 0x00 }, /* Ń to N */ + { 0x0144, 0x6E, 0x00 }, /* ń to n */ + { 0x0145, 0x4E, 0x00 }, /* Ņ to N */ + { 0x0146, 0x6E, 0x00 }, /* ņ to n */ + { 0x0147, 0x4E, 0x00 }, /* Ň to N */ + { 0x0148, 0x6E, 0x00 }, /* ň to n */ + { 0x0149, 0x27, 0x6E }, /* ʼn to 'n */ + { 0x014A, 0x4E, 0x47 }, /* Ŋ to NG */ + { 0x014B, 0x6E, 0x67 }, /* ŋ to ng */ + { 0x014C, 0x4F, 0x00 }, /* Ō to O */ + { 0x014D, 0x6F, 0x00 }, /* ō to o */ + { 0x014E, 0x4F, 0x00 }, /* Ŏ to O */ + { 0x014F, 0x6F, 0x00 }, /* ŏ to o */ + { 0x0150, 0x4F, 0x00 }, /* Ő to O */ + { 0x0151, 0x6F, 0x00 }, /* ő to o */ + { 0x0152, 0x4F, 0x45 }, /* Œ to OE */ + { 0x0153, 0x6F, 0x65 }, /* œ to oe */ + { 0x0154, 0x52, 0x00 }, /* Ŕ to R */ + { 0x0155, 0x72, 0x00 }, /* ŕ to r */ + { 0x0156, 0x52, 0x00 }, /* Ŗ to R */ + { 0x0157, 0x72, 0x00 }, /* ŗ to r */ + { 0x0158, 0x52, 0x00 }, /* Ř to R */ + { 0x0159, 0x72, 0x00 }, /* ř to r */ + { 0x015A, 0x53, 0x00 }, /* Ś to S */ + { 0x015B, 0x73, 0x00 }, /* ś to s */ + { 0x015C, 0x53, 0x68 }, /* Ŝ to Sh */ + { 0x015D, 0x73, 0x68 }, /* ŝ to sh */ + { 0x015E, 0x53, 0x00 }, /* Ş to S */ + { 0x015F, 0x73, 0x00 }, /* ş to s */ + { 0x0160, 0x53, 0x00 }, /* Š to S */ + { 0x0161, 0x73, 0x00 }, /* š to s */ + { 0x0162, 0x54, 0x00 }, /* Ţ to T */ + { 0x0163, 0x74, 0x00 }, /* ţ to t */ + { 0x0164, 0x54, 0x00 }, /* Ť to T */ + { 0x0165, 0x74, 0x00 }, /* ť to t */ + { 0x0166, 0x54, 0x00 }, /* Ŧ to T */ + { 0x0167, 0x74, 0x00 }, /* ŧ to t */ + { 0x0168, 0x55, 0x00 }, /* Ũ to U */ + { 0x0169, 0x75, 0x00 }, /* ũ to u */ + { 0x016A, 0x55, 0x00 }, /* Ū to U */ + { 0x016B, 0x75, 0x00 }, /* ū to u */ + { 0x016C, 0x55, 0x00 }, /* Ŭ to U */ + { 0x016D, 0x75, 0x00 }, /* ŭ to u */ + { 0x016E, 0x55, 0x00 }, /* Ů to U */ + { 0x016F, 0x75, 0x00 }, /* ů to u */ + { 0x0170, 0x55, 0x00 }, /* Ű to U */ + { 0x0171, 0x75, 0x00 }, /* ű to u */ + { 0x0172, 0x55, 0x00 }, /* Ų to U */ + { 0x0173, 0x75, 0x00 }, /* ų to u */ + { 0x0174, 0x57, 0x00 }, /* Ŵ to W */ + { 0x0175, 0x77, 0x00 }, /* ŵ to w */ + { 0x0176, 0x59, 0x00 }, /* Ŷ to Y */ + { 0x0177, 0x79, 0x00 }, /* ŷ to y */ + { 0x0178, 0x59, 0x00 }, /* Ÿ to Y */ + { 0x0179, 0x5A, 0x00 }, /* Ź to Z */ + { 0x017A, 0x7A, 0x00 }, /* ź to z */ + { 0x017B, 0x5A, 0x00 }, /* Ż to Z */ + { 0x017C, 0x7A, 0x00 }, /* ż to z */ + { 0x017D, 0x5A, 0x00 }, /* Ž to Z */ + { 0x017E, 0x7A, 0x00 }, /* ž to z */ + { 0x017F, 0x73, 0x00 }, /* ſ to s */ + { 0x0192, 0x66, 0x00 }, /* ƒ to f */ + { 0x0218, 0x53, 0x00 }, /* Ș to S */ + { 0x0219, 0x73, 0x00 }, /* ș to s */ + { 0x021A, 0x54, 0x00 }, /* Ț to T */ + { 0x021B, 0x74, 0x00 }, /* ț to t */ + { 0x0386, 0x41, 0x00 }, /* Ά to A */ + { 0x0388, 0x45, 0x00 }, /* Έ to E */ + { 0x0389, 0x49, 0x00 }, /* Ή to I */ + { 0x038A, 0x49, 0x00 }, /* Ί to I */ + { 0x038C, 0x4f, 0x00 }, /* Ό to O */ + { 0x038E, 0x59, 0x00 }, /* Ύ to Y */ + { 0x038F, 0x4f, 0x00 }, /* Ώ to O */ + { 0x0390, 0x69, 0x00 }, /* ΐ to i */ + { 0x0391, 0x41, 0x00 }, /* Α to A */ + { 0x0392, 0x42, 0x00 }, /* Β to B */ + { 0x0393, 0x47, 0x00 }, /* Γ to G */ + { 0x0394, 0x44, 0x00 }, /* Δ to D */ + { 0x0395, 0x45, 0x00 }, /* Ε to E */ + { 0x0396, 0x5a, 0x00 }, /* Ζ to Z */ + { 0x0397, 0x49, 0x00 }, /* Η to I */ + { 0x0398, 0x54, 0x68 }, /* Θ to Th */ + { 0x0399, 0x49, 0x00 }, /* Ι to I */ + { 0x039A, 0x4b, 0x00 }, /* Κ to K */ + { 0x039B, 0x4c, 0x00 }, /* Λ to L */ + { 0x039C, 0x4d, 0x00 }, /* Μ to M */ + { 0x039D, 0x4e, 0x00 }, /* Ν to N */ + { 0x039E, 0x58, 0x00 }, /* Ξ to X */ + { 0x039F, 0x4f, 0x00 }, /* Ο to O */ + { 0x03A0, 0x50, 0x00 }, /* Π to P */ + { 0x03A1, 0x52, 0x00 }, /* Ρ to R */ + { 0x03A3, 0x53, 0x00 }, /* Σ to S */ + { 0x03A4, 0x54, 0x00 }, /* Τ to T */ + { 0x03A5, 0x59, 0x00 }, /* Υ to Y */ + { 0x03A6, 0x46, 0x00 }, /* Φ to F */ + { 0x03A7, 0x43, 0x68 }, /* Χ to Ch */ + { 0x03A8, 0x50, 0x73 }, /* Ψ to Ps */ + { 0x03A9, 0x4f, 0x00 }, /* Ω to O */ + { 0x03AA, 0x49, 0x00 }, /* Ϊ to I */ + { 0x03AB, 0x59, 0x00 }, /* Ϋ to Y */ + { 0x03AC, 0x61, 0x00 }, /* ά to a */ + { 0x03AD, 0x65, 0x00 }, /* έ to e */ + { 0x03AE, 0x69, 0x00 }, /* ή to i */ + { 0x03AF, 0x69, 0x00 }, /* ί to i */ + { 0x03B1, 0x61, 0x00 }, /* α to a */ + { 0x03B2, 0x62, 0x00 }, /* β to b */ + { 0x03B3, 0x67, 0x00 }, /* γ to g */ + { 0x03B4, 0x64, 0x00 }, /* δ to d */ + { 0x03B5, 0x65, 0x00 }, /* ε to e */ + { 0x03B6, 0x7a, 0x00 }, /* ζ to z */ + { 0x03B7, 0x69, 0x00 }, /* η to i */ + { 0x03B8, 0x74, 0x68 }, /* θ to th */ + { 0x03B9, 0x69, 0x00 }, /* ι to i */ + { 0x03BA, 0x6b, 0x00 }, /* κ to k */ + { 0x03BB, 0x6c, 0x00 }, /* λ to l */ + { 0x03BC, 0x6d, 0x00 }, /* μ to m */ + { 0x03BD, 0x6e, 0x00 }, /* ν to n */ + { 0x03BE, 0x78, 0x00 }, /* ξ to x */ + { 0x03BF, 0x6f, 0x00 }, /* ο to o */ + { 0x03C0, 0x70, 0x00 }, /* π to p */ + { 0x03C1, 0x72, 0x00 }, /* ρ to r */ + { 0x03C3, 0x73, 0x00 }, /* σ to s */ + { 0x03C4, 0x74, 0x00 }, /* τ to t */ + { 0x03C5, 0x79, 0x00 }, /* υ to y */ + { 0x03C6, 0x66, 0x00 }, /* φ to f */ + { 0x03C7, 0x63, 0x68 }, /* χ to ch */ + { 0x03C8, 0x70, 0x73 }, /* ψ to ps */ + { 0x03C9, 0x6f, 0x00 }, /* ω to o */ + { 0x03CA, 0x69, 0x00 }, /* ϊ to i */ + { 0x03CB, 0x79, 0x00 }, /* ϋ to y */ + { 0x03CC, 0x6f, 0x00 }, /* ό to o */ + { 0x03CD, 0x79, 0x00 }, /* ύ to y */ + { 0x03CE, 0x69, 0x00 }, /* ώ to i */ + { 0x0400, 0x45, 0x00 }, /* Ѐ to E */ + { 0x0401, 0x45, 0x00 }, /* Ё to E */ + { 0x0402, 0x44, 0x00 }, /* Ђ to D */ + { 0x0403, 0x47, 0x00 }, /* Ѓ to G */ + { 0x0404, 0x45, 0x00 }, /* Є to E */ + { 0x0405, 0x5a, 0x00 }, /* Ѕ to Z */ + { 0x0406, 0x49, 0x00 }, /* І to I */ + { 0x0407, 0x49, 0x00 }, /* Ї to I */ + { 0x0408, 0x4a, 0x00 }, /* Ј to J */ + { 0x0409, 0x49, 0x00 }, /* Љ to I */ + { 0x040A, 0x4e, 0x00 }, /* Њ to N */ + { 0x040B, 0x44, 0x00 }, /* Ћ to D */ + { 0x040C, 0x4b, 0x00 }, /* Ќ to K */ + { 0x040D, 0x49, 0x00 }, /* Ѝ to I */ + { 0x040E, 0x55, 0x00 }, /* Ў to U */ + { 0x040F, 0x44, 0x00 }, /* Џ to D */ + { 0x0410, 0x41, 0x00 }, /* А to A */ + { 0x0411, 0x42, 0x00 }, /* Б to B */ + { 0x0412, 0x56, 0x00 }, /* В to V */ + { 0x0413, 0x47, 0x00 }, /* Г to G */ + { 0x0414, 0x44, 0x00 }, /* Д to D */ + { 0x0415, 0x45, 0x00 }, /* Е to E */ + { 0x0416, 0x5a, 0x68 }, /* Ж to Zh */ + { 0x0417, 0x5a, 0x00 }, /* З to Z */ + { 0x0418, 0x49, 0x00 }, /* И to I */ + { 0x0419, 0x49, 0x00 }, /* Й to I */ + { 0x041A, 0x4b, 0x00 }, /* К to K */ + { 0x041B, 0x4c, 0x00 }, /* Л to L */ + { 0x041C, 0x4d, 0x00 }, /* М to M */ + { 0x041D, 0x4e, 0x00 }, /* Н to N */ + { 0x041E, 0x4f, 0x00 }, /* О to O */ + { 0x041F, 0x50, 0x00 }, /* П to P */ + { 0x0420, 0x52, 0x00 }, /* Р to R */ + { 0x0421, 0x53, 0x00 }, /* С to S */ + { 0x0422, 0x54, 0x00 }, /* Т to T */ + { 0x0423, 0x55, 0x00 }, /* У to U */ + { 0x0424, 0x46, 0x00 }, /* Ф to F */ + { 0x0425, 0x4b, 0x68 }, /* Х to Kh */ + { 0x0426, 0x54, 0x63 }, /* Ц to Tc */ + { 0x0427, 0x43, 0x68 }, /* Ч to Ch */ + { 0x0428, 0x53, 0x68 }, /* Ш to Sh */ + { 0x0429, 0x53, 0x68 }, /* Щ to Shch */ + { 0x042B, 0x59, 0x00 }, /* Ы to Y */ + { 0x042D, 0x45, 0x00 }, /* Э to E */ + { 0x042E, 0x49, 0x75 }, /* Ю to Iu */ + { 0x042F, 0x49, 0x61 }, /* Я to Ia */ + { 0x0430, 0x61, 0x00 }, /* а to a */ + { 0x0431, 0x62, 0x00 }, /* б to b */ + { 0x0432, 0x76, 0x00 }, /* в to v */ + { 0x0433, 0x67, 0x00 }, /* г to g */ + { 0x0434, 0x64, 0x00 }, /* д to d */ + { 0x0435, 0x65, 0x00 }, /* е to e */ + { 0x0436, 0x7a, 0x68 }, /* ж to zh */ + { 0x0437, 0x7a, 0x00 }, /* з to z */ + { 0x0438, 0x69, 0x00 }, /* и to i */ + { 0x0439, 0x69, 0x00 }, /* й to i */ + { 0x043A, 0x6b, 0x00 }, /* к to k */ + { 0x043B, 0x6c, 0x00 }, /* л to l */ + { 0x043C, 0x6d, 0x00 }, /* м to m */ + { 0x043D, 0x6e, 0x00 }, /* н to n */ + { 0x043E, 0x6f, 0x00 }, /* о to o */ + { 0x043F, 0x70, 0x00 }, /* п to p */ + { 0x0440, 0x72, 0x00 }, /* р to r */ + { 0x0441, 0x73, 0x00 }, /* с to s */ + { 0x0442, 0x74, 0x00 }, /* т to t */ + { 0x0443, 0x75, 0x00 }, /* у to u */ + { 0x0444, 0x66, 0x00 }, /* ф to f */ + { 0x0445, 0x6b, 0x68 }, /* х to kh */ + { 0x0446, 0x74, 0x63 }, /* ц to tc */ + { 0x0447, 0x63, 0x68 }, /* ч to ch */ + { 0x0448, 0x73, 0x68 }, /* ш to sh */ + { 0x0449, 0x73, 0x68 }, /* щ to shch */ + { 0x044B, 0x79, 0x00 }, /* ы to y */ + { 0x044D, 0x65, 0x00 }, /* э to e */ + { 0x044E, 0x69, 0x75 }, /* ю to iu */ + { 0x044F, 0x69, 0x61 }, /* я to ia */ + { 0x0450, 0x65, 0x00 }, /* ѐ to e */ + { 0x0451, 0x65, 0x00 }, /* ё to e */ + { 0x0452, 0x64, 0x00 }, /* ђ to d */ + { 0x0453, 0x67, 0x00 }, /* ѓ to g */ + { 0x0454, 0x65, 0x00 }, /* є to e */ + { 0x0455, 0x7a, 0x00 }, /* ѕ to z */ + { 0x0456, 0x69, 0x00 }, /* і to i */ + { 0x0457, 0x69, 0x00 }, /* ї to i */ + { 0x0458, 0x6a, 0x00 }, /* ј to j */ + { 0x0459, 0x69, 0x00 }, /* љ to i */ + { 0x045A, 0x6e, 0x00 }, /* њ to n */ + { 0x045B, 0x64, 0x00 }, /* ћ to d */ + { 0x045C, 0x6b, 0x00 }, /* ќ to k */ + { 0x045D, 0x69, 0x00 }, /* ѝ to i */ + { 0x045E, 0x75, 0x00 }, /* ў to u */ + { 0x045F, 0x64, 0x00 }, /* џ to d */ + { 0x1E02, 0x42, 0x00 }, /* Ḃ to B */ + { 0x1E03, 0x62, 0x00 }, /* ḃ to b */ + { 0x1E0A, 0x44, 0x00 }, /* Ḋ to D */ + { 0x1E0B, 0x64, 0x00 }, /* ḋ to d */ + { 0x1E1E, 0x46, 0x00 }, /* Ḟ to F */ + { 0x1E1F, 0x66, 0x00 }, /* ḟ to f */ + { 0x1E40, 0x4D, 0x00 }, /* Ṁ to M */ + { 0x1E41, 0x6D, 0x00 }, /* ṁ to m */ + { 0x1E56, 0x50, 0x00 }, /* Ṗ to P */ + { 0x1E57, 0x70, 0x00 }, /* ṗ to p */ + { 0x1E60, 0x53, 0x00 }, /* Ṡ to S */ + { 0x1E61, 0x73, 0x00 }, /* ṡ to s */ + { 0x1E6A, 0x54, 0x00 }, /* Ṫ to T */ + { 0x1E6B, 0x74, 0x00 }, /* ṫ to t */ + { 0x1E80, 0x57, 0x00 }, /* Ẁ to W */ + { 0x1E81, 0x77, 0x00 }, /* ẁ to w */ + { 0x1E82, 0x57, 0x00 }, /* Ẃ to W */ + { 0x1E83, 0x77, 0x00 }, /* ẃ to w */ + { 0x1E84, 0x57, 0x00 }, /* Ẅ to W */ + { 0x1E85, 0x77, 0x00 }, /* ẅ to w */ + { 0x1EF2, 0x59, 0x00 }, /* Ỳ to Y */ + { 0x1EF3, 0x79, 0x00 }, /* ỳ to y */ + { 0xFB00, 0x66, 0x66 }, /* ff to ff */ + { 0xFB01, 0x66, 0x69 }, /* fi to fi */ + { 0xFB02, 0x66, 0x6C }, /* fl to fl */ + { 0xFB05, 0x73, 0x74 }, /* ſt to st */ + { 0xFB06, 0x73, 0x74 }, /* st to st */ +}; + +/* +** Convert the input string from UTF-8 into pure ASCII by converting +** all non-ASCII characters to some combination of characters in the +** ASCII subset. +** +** The returned string might contain more characters than the input. +** +** Space to hold the returned string comes from sqlite3_malloc() and +** should be freed by the caller. +*/ +static unsigned char *transliterate(const unsigned char *zIn, int nIn){ + unsigned char *zOut = sqlite3_malloc( nIn*4 + 1 ); + int i, c, sz, nOut; + if( zOut==0 ) return 0; + i = nOut = 0; + while( i<nIn ){ + c = utf8Read(zIn, nIn, &sz); + zIn += sz; + nIn -= sz; + if( c<=127 ){ + zOut[nOut++] = c; + }else{ + int xTop, xBtm, x; + xTop = sizeof(translit)/sizeof(translit[0]) - 1; + xBtm = 0; + while( xTop>=xBtm ){ + x = (xTop + xBtm)/2; + if( translit[x].cFrom==c ){ + zOut[nOut++] = translit[x].cTo0; + if( translit[x].cTo1 ){ + zOut[nOut++] = translit[x].cTo1; + /* Add an extra "ch" after the "sh" for Щ and щ */ + if( c==0x0429 || c== 0x0449 ){ + zOut[nOut++] = 'c'; + zOut[nOut++] = 'h'; + } + } + c = 0; + break; + }else if( translit[x].cFrom>c ){ + xTop = x-1; + }else{ + xBtm = x+1; + } + } + if( c ) zOut[nOut++] = '?'; + } + } + zOut[nOut] = 0; + return zOut; +} + +/* +** spellfix1_translit(X) +** +** Convert a string that contains non-ASCII Roman characters into +** pure ASCII. +*/ +static void transliterateSqlFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const unsigned char *zIn = sqlite3_value_text(argv[0]); + int nIn = sqlite3_value_bytes(argv[0]); + unsigned char *zOut = transliterate(zIn, nIn); + if( zOut==0 ){ + sqlite3_result_error_nomem(context); + }else{ + sqlite3_result_text(context, (char*)zOut, -1, sqlite3_free); + } +} + +/* +** spellfix1_scriptcode(X) +** +** Try to determine the dominant script used by the word X and return +** its ISO 15924 numeric code. +** +** The current implementation only understands the following scripts: +** +** 215 (Latin) +** 220 (Cyrillic) +** 200 (Greek) +** +** This routine will return 998 if the input X contains characters from +** two or more of the above scripts or 999 if X contains no characters +** from any of the above scripts. +*/ +static void scriptCodeSqlFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const unsigned char *zIn = sqlite3_value_text(argv[0]); + int nIn = sqlite3_value_bytes(argv[0]); + int c, sz; + int scriptMask = 0; + int res; +# define SCRIPT_LATIN 0x0001 +# define SCRIPT_CYRILLIC 0x0002 +# define SCRIPT_GREEK 0x0004 + + while( nIn>0 ){ + c = utf8Read(zIn, nIn, &sz); + zIn += sz; + nIn -= sz; + if( c<0x02af ){ + scriptMask |= SCRIPT_LATIN; + }else if( c>=0x0400 && c<=0x04ff ){ + scriptMask |= SCRIPT_CYRILLIC; + }else if( c>=0x0386 && c<=0x03ce ){ + scriptMask |= SCRIPT_GREEK; + } + } + switch( scriptMask ){ + case 0: res = 999; break; + case SCRIPT_LATIN: res = 215; break; + case SCRIPT_CYRILLIC: res = 220; break; + case SCRIPT_GREEK: res = 200; break; + default: res = 998; break; + } + sqlite3_result_int(context, res); +} + +/***************************************************************************** +** Fuzzy-search virtual table +*****************************************************************************/ + +typedef struct spellfix1_vtab spellfix1_vtab; +typedef struct spellfix1_cursor spellfix1_cursor; + +/* Fuzzy-search virtual table object */ +struct spellfix1_vtab { + sqlite3_vtab base; /* Base class - must be first */ + sqlite3 *db; /* Database connection */ + char *zDbName; /* Name of database holding this table */ + char *zTableName; /* Name of the virtual table */ +}; + +/* Fuzzy-search cursor object */ +struct spellfix1_cursor { + sqlite3_vtab_cursor base; /* Base class - must be first */ + spellfix1_vtab *pVTab; /* The table to which this cursor belongs */ + int nRow; /* Number of rows of content */ + int nAlloc; /* Number of allocated rows */ + int iRow; /* Current row of content */ + int iLang; /* Value of the lang= constraint */ + int iTop; /* Value of the top= constraint */ + int iScope; /* Value of the scope= constraint */ + int nSearch; /* Number of vocabulary items checked */ + struct spellfix1_row { /* For each row of content */ + sqlite3_int64 iRowid; /* Rowid for this row */ + char *zWord; /* Text for this row */ + int iRank; /* Rank for this row */ + int iDistance; /* Distance from pattern for this row */ + int iScore; /* Score for sorting */ + } *a; +}; + +/* +** Construct one or more SQL statements from the format string given +** and then evaluate those statements. The success code is written +** into *pRc. +** +** If *pRc is initially non-zero then this routine is a no-op. +*/ +static void spellfix1DbExec( + int *pRc, /* Success code */ + sqlite3 *db, /* Database in which to run SQL */ + const char *zFormat, /* Format string for SQL */ + ... /* Arguments to the format string */ +){ + va_list ap; + char *zSql; + if( *pRc ) return; + va_start(ap, zFormat); + zSql = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + if( zSql==0 ){ + *pRc = SQLITE_NOMEM; + }else{ + *pRc = sqlite3_exec(db, zSql, 0, 0, 0); + sqlite3_free(zSql); + } +} + +/* +** xDisconnect/xDestroy method for the fuzzy-search module. +*/ +static int spellfix1Uninit(int isDestroy, sqlite3_vtab *pVTab){ + spellfix1_vtab *p = (spellfix1_vtab*)pVTab; + int rc = SQLITE_OK; + if( isDestroy ){ + sqlite3 *db = p->db; + spellfix1DbExec(&rc, db, "DROP TABLE IF EXISTS \"%w\".\"%w_vocab\"", + p->zDbName, p->zTableName); + } + if( rc==SQLITE_OK ){ + sqlite3_free(p->zTableName); + sqlite3_free(p); + } + return rc; +} +static int spellfix1Disconnect(sqlite3_vtab *pVTab){ + return spellfix1Uninit(0, pVTab); +} +static int spellfix1Destroy(sqlite3_vtab *pVTab){ + return spellfix1Uninit(1, pVTab); +} + +/* +** xConnect/xCreate method for the spellfix1 module. Arguments are: +** +** argv[0] -> module name ("spellfix1") +** argv[1] -> database name +** argv[2] -> table name +** argv[3].. -> optional arguments (currently ignored) +*/ +static int spellfix1Init( + int isCreate, + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVTab, + char **pzErr +){ + spellfix1_vtab *pNew = 0; + const char *zModule = argv[0]; + const char *zDbName = argv[1]; + const char *zTableName = argv[2]; + int nDbName; + int rc = SQLITE_OK; + + if( argc<3 ){ + *pzErr = sqlite3_mprintf( + "%s: wrong number of CREATE VIRTUAL TABLE arguments", argv[0] + ); + rc = SQLITE_ERROR; + }else{ + nDbName = strlen(zDbName); + pNew = sqlite3_malloc( sizeof(*pNew) + nDbName + 1); + if( pNew==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pNew, 0, sizeof(*pNew)); + pNew->zDbName = (char*)&pNew[1]; + memcpy(pNew->zDbName, zDbName, nDbName+1); + pNew->zTableName = sqlite3_mprintf("%s", zTableName); + pNew->db = db; + if( pNew->zTableName==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_declare_vtab(db, + "CREATE TABLE x(word,rank,distance,langid," + "score,top HIDDEN,scope HIDDEN,srchcnt HIDDEN," + "soundslike HIDDEN)" + ); + } + if( rc==SQLITE_OK && isCreate ){ + sqlite3_uint64 r; + spellfix1DbExec(&rc, db, + "CREATE TABLE IF NOT EXISTS \"%w\".\"%w_vocab\"(\n" + " id INTEGER PRIMARY KEY,\n" + " rank INT,\n" + " langid INT,\n" + " word TEXT,\n" + " k1 TEXT,\n" + " k2 TEXT\n" + ");\n", + zDbName, zTableName + ); + sqlite3_randomness(sizeof(r), &r); + spellfix1DbExec(&rc, db, + "CREATE INDEX IF NOT EXISTS \"%w\".\"%w_index_%llx\" " + "ON \"%w_vocab\"(langid,k2);", + zDbName, zModule, r, zTableName + ); + } + } + } + + *ppVTab = (sqlite3_vtab *)pNew; + return rc; +} + +/* +** The xConnect and xCreate methods +*/ +static int spellfix1Connect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVTab, + char **pzErr +){ + return spellfix1Init(0, db, pAux, argc, argv, ppVTab, pzErr); +} +static int spellfix1Create( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVTab, + char **pzErr +){ + return spellfix1Init(1, db, pAux, argc, argv, ppVTab, pzErr); +} + +/* +** Reset a cursor so that it contains zero rows of content but holds +** space for N rows. +*/ +static void spellfix1ResetCursor(spellfix1_cursor *pCur, int N){ + int i; + for(i=0; i<pCur->nRow; i++){ + sqlite3_free(pCur->a[i].zWord); + } + pCur->a = sqlite3_realloc(pCur->a, sizeof(pCur->a[0])*N); + pCur->nAlloc = N; + pCur->nRow = 0; + pCur->iRow = 0; + pCur->nSearch = 0; +} + +/* +** Close a fuzzy-search cursor. +*/ +static int spellfix1Close(sqlite3_vtab_cursor *cur){ + spellfix1_cursor *pCur = (spellfix1_cursor *)cur; + spellfix1ResetCursor(pCur, 0); + sqlite3_free(pCur); + return SQLITE_OK; +} + +/* +** Search for terms of these forms: +** +** (A) word MATCH $str +** (B) langid == $langid +** (C) top = $top +** (D) scope = $scope +** +** The plan number is a bit mask formed with these bits: +** +** 0x01 (A) is found +** 0x02 (B) is found +** 0x04 (C) is found +** 0x08 (D) is found +** +** filter.argv[*] values contains $str, $langid, $top, and $scope, +** if specified and in that order. +*/ +static int spellfix1BestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ + int iPlan = 0; + int iLangTerm = -1; + int iTopTerm = -1; + int iScopeTerm = -1; + int i; + const struct sqlite3_index_constraint *pConstraint; + pConstraint = pIdxInfo->aConstraint; + for(i=0; i<pIdxInfo->nConstraint; i++, pConstraint++){ + if( pConstraint->usable==0 ) continue; + + /* Terms of the form: word MATCH $str */ + if( (iPlan & 1)==0 + && pConstraint->iColumn==0 + && pConstraint->op==SQLITE_INDEX_CONSTRAINT_MATCH + ){ + iPlan |= 1; + pIdxInfo->aConstraintUsage[i].argvIndex = 1; + pIdxInfo->aConstraintUsage[i].omit = 1; + } + + /* Terms of the form: langid = $langid */ + if( (iPlan & 2)==0 + && pConstraint->iColumn==3 + && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ + ){ + iPlan |= 2; + iLangTerm = i; + } + + /* Terms of the form: top = $top */ + if( (iPlan & 4)==0 + && pConstraint->iColumn==5 + && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ + ){ + iPlan |= 4; + iTopTerm = i; + } + + /* Terms of the form: scope = $scope */ + if( (iPlan & 8)==0 + && pConstraint->iColumn==6 + && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ + ){ + iPlan |= 8; + iScopeTerm = i; + } + } + if( iPlan&1 ){ + int idx = 2; + pIdxInfo->idxNum = iPlan; + if( pIdxInfo->nOrderBy==1 + && pIdxInfo->aOrderBy[0].iColumn==4 + && pIdxInfo->aOrderBy[0].desc==0 + ){ + pIdxInfo->orderByConsumed = 1; /* Default order by iScore */ + } + if( iPlan&2 ){ + pIdxInfo->aConstraintUsage[iLangTerm].argvIndex = idx++; + pIdxInfo->aConstraintUsage[iLangTerm].omit = 1; + } + if( iPlan&4 ){ + pIdxInfo->aConstraintUsage[iTopTerm].argvIndex = idx++; + pIdxInfo->aConstraintUsage[iTopTerm].omit = 1; + } + if( iPlan&8 ){ + pIdxInfo->aConstraintUsage[iScopeTerm].argvIndex = idx++; + pIdxInfo->aConstraintUsage[iScopeTerm].omit = 1; + } + pIdxInfo->estimatedCost = (double)10000; + }else{ + pIdxInfo->idxNum = 0; + pIdxInfo->estimatedCost = (double)10000000; + } + return SQLITE_OK; +} + +/* +** Open a new fuzzy-search cursor. +*/ +static int spellfix1Open(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ + spellfix1_vtab *p = (spellfix1_vtab*)pVTab; + spellfix1_cursor *pCur; + pCur = sqlite3_malloc( sizeof(*pCur) ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + pCur->pVTab = p; + *ppCursor = &pCur->base; + return SQLITE_OK; +} + +/* +** Adjust a distance measurement by the words rank in order to show +** preference to common words. +*/ +static int spellfix1Score(int iDistance, int iRank){ + int iLog2; + for(iLog2=0; iRank>0; iLog2++, iRank>>=1){} + return iDistance + 32 - iLog2; +} + +/* +** Compare two spellfix1_row objects for sorting purposes in qsort() such +** that they sort in order of increasing distance. +*/ +static int spellfix1RowCompare(const void *A, const void *B){ + const struct spellfix1_row *a = (const struct spellfix1_row*)A; + const struct spellfix1_row *b = (const struct spellfix1_row*)B; + return a->iScore - b->iScore; +} + +/* +** This version of the xFilter method work if the MATCH term is present +** and we are doing a scan. +*/ +static int spellfix1FilterForMatch( + spellfix1_cursor *pCur, + int idxNum, + int argc, + sqlite3_value **argv +){ + const unsigned char *zPatternIn; + char *zPattern; + int nPattern; + char *zClass; + int nClass; + int iLimit = 20; + int iScope = 4; + int iLang = 0; + char *zSql; + int rc; + sqlite3_stmt *pStmt; + int idx = 1; + spellfix1_vtab *p = pCur->pVTab; + + if( idxNum&2 ){ + iLang = sqlite3_value_int(argv[idx++]); + } + if( idxNum&4 ){ + iLimit = sqlite3_value_int(argv[idx++]); + if( iLimit<1 ) iLimit = 1; + } + if( idxNum&8 ){ + iScope = sqlite3_value_int(argv[idx++]); + if( iScope<1 ) iScope = 1; + } + spellfix1ResetCursor(pCur, iLimit); + zPatternIn = sqlite3_value_text(argv[0]); + if( zPatternIn==0 ) return SQLITE_OK; + zPattern = (char*)transliterate(zPatternIn, sqlite3_value_bytes(argv[0])); + if( zPattern==0 ) return SQLITE_NOMEM; + nPattern = strlen(zPattern); + if( zPattern[nPattern-1]=='*' ) nPattern--; + if( nPattern<iScope ) iScope = nPattern; + zClass = (char*)characterClassString((unsigned char*)zPattern, + strlen(zPattern)); + nClass = strlen(zClass); + if( nClass>iScope ){ + zClass[iScope] = 0; + nClass = iScope; + } + zSql = sqlite3_mprintf( + "SELECT id, word, rank, k1" + " FROM \"%w\".\"%w_vocab\"" + " WHERE langid=%d AND k2 GLOB '%q*'", + p->zDbName, p->zTableName, iLang, zClass + ); + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + if( rc==SQLITE_OK ){ + const char *zK1; + int iDist; + int iRank; + int iScore; + int iWorst = 999999999; + int idx; + int idxWorst; + int i; + + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + zK1 = (const char*)sqlite3_column_text(pStmt, 3); + if( zK1==0 ) continue; + pCur->nSearch++; + iRank = sqlite3_column_int(pStmt, 2); + iDist = editdist(zPattern, zK1); + iScore = spellfix1Score(iDist,iRank); + if( pCur->nRow<pCur->nAlloc ){ + idx = pCur->nRow; + }else if( iScore<iWorst ){ + idx = idxWorst; + sqlite3_free(pCur->a[idx].zWord); + }else{ + continue; + } + pCur->a[idx].zWord = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 1)); + pCur->a[idx].iRowid = sqlite3_column_int64(pStmt, 0); + pCur->a[idx].iRank = iRank; + pCur->a[idx].iDistance = iDist; + pCur->a[idx].iScore = iScore; + if( pCur->nRow<pCur->nAlloc ) pCur->nRow++; + if( pCur->nRow==pCur->nAlloc ){ + iWorst = pCur->a[0].iScore; + idxWorst = 0; + for(i=1; i<pCur->nRow; i++){ + iScore = pCur->a[i].iScore; + if( iWorst<iScore ){ + iWorst = iScore; + idxWorst = i; + } + } + } + } + } + qsort(pCur->a, pCur->nRow, sizeof(pCur->a[0]), spellfix1RowCompare); + pCur->iTop = iLimit; + pCur->iScope = iScope; + sqlite3_finalize(pStmt); + sqlite3_free(zPattern); + sqlite3_free(zClass); + return SQLITE_OK; +} + +/* +** This version of xFilter handles a full-table scan case +*/ +static int spellfix1FilterForFullScan( + spellfix1_cursor *pCur, + int idxNum, + int argc, + sqlite3_value **argv +){ + spellfix1ResetCursor(pCur, 0); + return SQLITE_OK; +} + + +/* +** Called to "rewind" a cursor back to the beginning so that +** it starts its output over again. Always called at least once +** prior to any spellfix1Column, spellfix1Rowid, or spellfix1Eof call. +*/ +static int spellfix1Filter( + sqlite3_vtab_cursor *cur, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + spellfix1_cursor *pCur = (spellfix1_cursor *)cur; + int rc; + if( idxNum & 1 ){ + rc = spellfix1FilterForMatch(pCur, idxNum, argc, argv); + }else{ + rc = spellfix1FilterForFullScan(pCur, idxNum, argc, argv); + } + return rc; +} + + +/* +** Advance a cursor to its next row of output +*/ +static int spellfix1Next(sqlite3_vtab_cursor *cur){ + spellfix1_cursor *pCur = (spellfix1_cursor *)cur; + if( pCur->iRow < pCur->nRow ) pCur->iRow++; + return SQLITE_OK; +} + +/* +** Return TRUE if we are at the end-of-file +*/ +static int spellfix1Eof(sqlite3_vtab_cursor *cur){ + spellfix1_cursor *pCur = (spellfix1_cursor *)cur; + return pCur->iRow>=pCur->nRow; +} + +/* +** Return columns from the current row. +*/ +static int spellfix1Column(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){ + spellfix1_cursor *pCur = (spellfix1_cursor*)cur; + switch( i ){ + case 0: { + sqlite3_result_text(ctx, pCur->a[pCur->iRow].zWord, -1, SQLITE_STATIC); + break; + } + case 1: { + sqlite3_result_int(ctx, pCur->a[pCur->iRow].iRank); + break; + } + case 2: { + sqlite3_result_int(ctx, pCur->a[pCur->iRow].iDistance); + break; + } + case 3: { + sqlite3_result_int(ctx, pCur->iLang); + break; + } + case 4: { + sqlite3_result_int(ctx, pCur->a[pCur->iRow].iScore); + break; + } + case 5: { + sqlite3_result_int(ctx, pCur->iTop); + break; + } + case 6: { + sqlite3_result_int(ctx, pCur->iScope); + break; + } + case 7: { + sqlite3_result_int(ctx, pCur->nSearch); + break; + } + default: { + sqlite3_result_null(ctx); + break; + } + } + return SQLITE_OK; +} + +/* +** The rowid. +*/ +static int spellfix1Rowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + spellfix1_cursor *pCur = (spellfix1_cursor*)cur; + *pRowid = pCur->a[pCur->iRow].iRowid; + return SQLITE_OK; +} + +/* +** The xUpdate() method. +*/ +static int spellfix1Update( + sqlite3_vtab *pVTab, + int argc, + sqlite3_value **argv, + sqlite_int64 *pRowid +){ + int rc = SQLITE_OK; + sqlite3_int64 rowid, newRowid; + spellfix1_vtab *p = (spellfix1_vtab*)pVTab; + sqlite3 *db = p->db; + + if( argc==1 ){ + /* A delete operation on the rowid given by argv[0] */ + rowid = *pRowid = sqlite3_value_int64(argv[0]); + spellfix1DbExec(&rc, db, "DELETE FROM \"%w\".\"%w_vocab\" " + " WHERE id=%lld", + p->zDbName, p->zTableName, rowid); + }else{ + const unsigned char *zWord = sqlite3_value_text(argv[2]); + int nWord = sqlite3_value_bytes(argv[2]); + int iLang = sqlite3_value_int(argv[5]); + int iRank = sqlite3_value_int(argv[3]); + const unsigned char *zSoundslike = sqlite3_value_text(argv[10]); + int nSoundslike = sqlite3_value_bytes(argv[10]); + char *zK1, *zK2; + int i; + char c; + + if( zWord==0 ){ + pVTab->zErrMsg = sqlite3_mprintf("%w.word may not be NULL", + p->zTableName); + return SQLITE_CONSTRAINT; + } + if( iRank<1 ) iRank = 1; + if( zSoundslike ){ + zK1 = (char*)transliterate(zSoundslike, nSoundslike); + }else{ + zK1 = (char*)transliterate(zWord, nWord); + } + if( zK1==0 ) return SQLITE_NOMEM; + for(i=0; (c = zK1[i])!=0; i++){ + if( c>='A' && c<='Z' ) zK1[i] += 'a' - 'A'; + } + zK2 = (char*)characterClassString((const unsigned char*)zK1, i); + if( zK2==0 ){ + sqlite3_free(zK1); + return SQLITE_NOMEM; + } + if( sqlite3_value_type(argv[0])==SQLITE_NULL ){ + spellfix1DbExec(&rc, db, + "INSERT INTO \"%w\".\"%w_vocab\"(rank,langid,word,k1,k2) " + "VALUES(%d,%d,%Q,%Q,%Q)", + p->zDbName, p->zTableName, + iRank, iLang, zWord, zK1, zK2 + ); + *pRowid = sqlite3_last_insert_rowid(db); + }else{ + rowid = sqlite3_value_int64(argv[0]); + newRowid = *pRowid = sqlite3_value_int64(argv[1]); + spellfix1DbExec(&rc, db, + "UPDATE \"%w\".\"%w_vocab\" SET id=%lld, rank=%d, lang=%d," + " word=%Q, rank=%d, k1=%Q, k2=%Q WHERE id=%lld", + p->zDbName, p->zTableName, newRowid, iRank, iLang, + zWord, zK1, zK2, rowid + ); + } + sqlite3_free(zK1); + sqlite3_free(zK2); + } + return rc; +} + +/* +** Rename the spellfix1 table. +*/ +static int spellfix1Rename(sqlite3_vtab *pVTab, const char *zNew){ + spellfix1_vtab *p = (spellfix1_vtab*)pVTab; + sqlite3 *db = p->db; + int rc = SQLITE_OK; + char *zNewName = sqlite3_mprintf("%s", zNew); + if( zNewName==0 ){ + return SQLITE_NOMEM; + } + spellfix1DbExec(&rc, db, + "ALTER TABLE \"%w\".\"%w_vocab\" RENAME TO \"%w_vocab\"", + p->zDbName, p->zTableName, zNewName + ); + if( rc==SQLITE_OK ){ + sqlite3_free(p->zTableName); + p->zTableName = zNewName; + } + return rc; +} + + +/* +** A virtual table module that provides fuzzy search. +*/ +static sqlite3_module spellfix1Module = { + 0, /* iVersion */ + spellfix1Create, /* xCreate - handle CREATE VIRTUAL TABLE */ + spellfix1Connect, /* xConnect - reconnected to an existing table */ + spellfix1BestIndex, /* xBestIndex - figure out how to do a query */ + spellfix1Disconnect, /* xDisconnect - close a connection */ + spellfix1Destroy, /* xDestroy - handle DROP TABLE */ + spellfix1Open, /* xOpen - open a cursor */ + spellfix1Close, /* xClose - close a cursor */ + spellfix1Filter, /* xFilter - configure scan constraints */ + spellfix1Next, /* xNext - advance a cursor */ + spellfix1Eof, /* xEof - check for end of scan */ + spellfix1Column, /* xColumn - read data */ + spellfix1Rowid, /* xRowid - read data */ + spellfix1Update, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + spellfix1Rename, /* xRename */ +}; + +/* +** Register the various functions and the virtual table. +*/ +static int spellfix1Register(sqlite3 *db){ + int nErr = 0; + int i; + nErr += sqlite3_create_function(db, "spellfix1_translit", 1, SQLITE_UTF8, 0, + transliterateSqlFunc, 0, 0); + nErr += sqlite3_create_function(db, "spellfix1_editdist", 2, SQLITE_UTF8, 0, + editdistSqlFunc, 0, 0); + nErr += sqlite3_create_function(db, "spellfix1_charclass", 1, SQLITE_UTF8, 0, + characterClassSqlFunc, 0, 0); + nErr += sqlite3_create_function(db, "spellfix1_scriptcode", 1, SQLITE_UTF8, 0, + scriptCodeSqlFunc, 0, 0); + nErr += sqlite3_create_module(db, "spellfix1", &spellfix1Module, 0); + + /* Verify sanity of the translit[] table */ + for(i=0; i<sizeof(translit)/sizeof(translit[0])-1; i++){ + assert( translit[i].cFrom<translit[i+1].cFrom ); + } + + return nErr ? SQLITE_ERROR : SQLITE_OK; +} + +#if SQLITE_CORE || defined(SQLITE_TEST) +/* +** Register the spellfix1 virtual table and its associated functions. +*/ +int sqlite3Spellfix1Register(sqlite3 *db){ + return spellfix1Register(db); +} +#endif + + +#if !SQLITE_CORE +/* +** Extension load function. +*/ +int sqlite3_extension_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + SQLITE_EXTENSION_INIT2(pApi); + return spellfix1Register(db); +} +#endif /* !SQLITE_CORE */ diff --git a/src/test_stat.c b/src/test_stat.c index ef81769..d4c902b 100644 --- a/src/test_stat.c +++ b/src/test_stat.c @@ -324,12 +324,13 @@ static int statDecodePage(Btree *pBt, StatPage *p){ u64 dummy; iOff += sqlite3GetVarint(&aData[iOff], &dummy); } - if( nPayload>p->nMxPayload ) p->nMxPayload = nPayload; + if( nPayload>(u32)p->nMxPayload ) p->nMxPayload = nPayload; getLocalPayload(nUsable, p->flags, nPayload, &nLocal); pCell->nLocal = nLocal; - assert( nPayload>=nLocal ); + assert( nLocal>=0 ); + assert( nPayload>=(u32)nLocal ); assert( nLocal<=(nUsable-35) ); - if( nPayload>nLocal ){ + if( nPayload>(u32)nLocal ){ int j; int nOvfl = ((nPayload - nLocal) + nUsable-4 - 1) / (nUsable - 4); pCell->nLastOvfl = (nPayload-nLocal) - (nOvfl-1) * (nUsable-4); @@ -369,7 +370,7 @@ static void statSizeAndOffset(StatCursor *pCsr){ /* The default page size and offset */ pCsr->szPage = sqlite3BtreeGetPageSize(pBt); - pCsr->iOffset = pCsr->szPage * (pCsr->iPageno - 1); + pCsr->iOffset = (i64)pCsr->szPage * (pCsr->iPageno - 1); /* If connected to a ZIPVFS backend, override the page size and ** offset with actual values obtained from ZIPVFS. @@ -378,7 +379,7 @@ static void statSizeAndOffset(StatCursor *pCsr){ x[0] = pCsr->iPageno; if( sqlite3OsFileControl(fd, 230440, &x)==SQLITE_OK ){ pCsr->iOffset = x[0]; - pCsr->szPage = x[1]; + pCsr->szPage = (int)x[1]; } } @@ -400,7 +401,7 @@ static int statNext(sqlite3_vtab_cursor *pCursor){ rc = sqlite3_step(pCsr->pStmt); if( rc==SQLITE_ROW ){ int nPage; - u32 iRoot = sqlite3_column_int64(pCsr->pStmt, 1); + u32 iRoot = (u32)sqlite3_column_int64(pCsr->pStmt, 1); sqlite3PagerPagecount(pPager, &nPage); if( nPage==0 ){ pCsr->isEof = 1; diff --git a/src/test_thread.c b/src/test_thread.c index aa89467..ae62de8 100644 --- a/src/test_thread.c +++ b/src/test_thread.c @@ -273,7 +273,6 @@ static int sqlthread_open( const char *zFilename; sqlite3 *db; - int rc; char zBuf[100]; extern void Md5_Register(sqlite3*); @@ -281,11 +280,12 @@ static int sqlthread_open( UNUSED_PARAMETER(objc); zFilename = Tcl_GetString(objv[2]); - rc = sqlite3_open(zFilename, &db); + sqlite3_open(zFilename, &db); #ifdef SQLITE_HAS_CODEC if( db && objc>=4 ){ const char *zKey; int nKey; + int rc; zKey = Tcl_GetStringFromObj(objv[3], &nKey); rc = sqlite3_key(db, zKey, nKey); if( rc!=SQLITE_OK ){ diff --git a/src/test_vfs.c b/src/test_vfs.c index 546cb7c..d1c34a3 100644 --- a/src/test_vfs.c +++ b/src/test_vfs.c @@ -480,6 +480,27 @@ static int tvfsCheckReservedLock(sqlite3_file *pFile, int *pResOut){ */ static int tvfsFileControl(sqlite3_file *pFile, int op, void *pArg){ TestvfsFd *p = tvfsGetFd(pFile); + if( op==SQLITE_FCNTL_PRAGMA ){ + char **argv = (char**)pArg; + if( sqlite3_stricmp(argv[1],"error")==0 ){ + int rc = SQLITE_ERROR; + if( argv[2] ){ + const char *z = argv[2]; + int x = atoi(z); + if( x ){ + rc = x; + while( sqlite3Isdigit(z[0]) ){ z++; } + while( sqlite3Isspace(z[0]) ){ z++; } + } + if( z[0] ) argv[0] = sqlite3_mprintf("%s", z); + } + return rc; + } + if( sqlite3_stricmp(argv[1], "filename")==0 ){ + argv[0] = sqlite3_mprintf("%s", p->zFilename); + return SQLITE_OK; + } + } return sqlite3OsFileControl(p->pReal, op, pArg); } @@ -763,7 +784,7 @@ static int tvfsShmOpen(sqlite3_file *pFile){ if( 0==strcmp(pFd->zFilename, pBuffer->zFile) ) break; } if( !pBuffer ){ - int nByte = sizeof(TestvfsBuffer) + strlen(pFd->zFilename) + 1; + int nByte = sizeof(TestvfsBuffer) + (int)strlen(pFd->zFilename) + 1; pBuffer = (TestvfsBuffer *)ckalloc(nByte); memset(pBuffer, 0, nByte); pBuffer->zFile = (char *)&pBuffer[1]; @@ -845,13 +866,13 @@ static int tvfsShmLock( if( p->pScript && p->mask&TESTVFS_SHMLOCK_MASK ){ sqlite3_snprintf(sizeof(zLock), zLock, "%d %d", ofst, n); - nLock = strlen(zLock); + nLock = (int)strlen(zLock); if( flags & SQLITE_SHM_LOCK ){ strcpy(&zLock[nLock], " lock"); }else{ strcpy(&zLock[nLock], " unlock"); } - nLock += strlen(&zLock[nLock]); + nLock += (int)strlen(&zLock[nLock]); if( flags & SQLITE_SHM_SHARED ){ strcpy(&zLock[nLock], " shared"); }else{ @@ -988,7 +1009,7 @@ static int testvfs_obj_cmd( switch( aSubcmd[i].eCmd ){ case CMD_SHM: { Tcl_Obj *pObj; - int i; + int i, rc; TestvfsBuffer *pBuffer; char *zName; if( objc!=3 && objc!=4 ){ @@ -996,10 +1017,16 @@ static int testvfs_obj_cmd( return TCL_ERROR; } zName = ckalloc(p->pParent->mxPathname); - p->pParent->xFullPathname( + rc = p->pParent->xFullPathname( p->pParent, Tcl_GetString(objv[2]), p->pParent->mxPathname, zName ); + if( rc!=SQLITE_OK ){ + Tcl_AppendResult(interp, "failed to get full path: ", + Tcl_GetString(objv[2]), 0); + ckfree(zName); + return TCL_ERROR; + } for(pBuffer=p->pBuffer; pBuffer; pBuffer=pBuffer->pNext){ if( 0==strcmp(pBuffer->zFile, zName) ) break; } @@ -1156,18 +1183,19 @@ static int testvfs_obj_cmd( int iValue; } aFlag[] = { { "default", -1 }, - { "atomic", SQLITE_IOCAP_ATOMIC }, - { "atomic512", SQLITE_IOCAP_ATOMIC512 }, - { "atomic1k", SQLITE_IOCAP_ATOMIC1K }, - { "atomic2k", SQLITE_IOCAP_ATOMIC2K }, - { "atomic4k", SQLITE_IOCAP_ATOMIC4K }, - { "atomic8k", SQLITE_IOCAP_ATOMIC8K }, - { "atomic16k", SQLITE_IOCAP_ATOMIC16K }, - { "atomic32k", SQLITE_IOCAP_ATOMIC32K }, - { "atomic64k", SQLITE_IOCAP_ATOMIC64K }, - { "sequential", SQLITE_IOCAP_SEQUENTIAL }, - { "safe_append", SQLITE_IOCAP_SAFE_APPEND }, + { "atomic", SQLITE_IOCAP_ATOMIC }, + { "atomic512", SQLITE_IOCAP_ATOMIC512 }, + { "atomic1k", SQLITE_IOCAP_ATOMIC1K }, + { "atomic2k", SQLITE_IOCAP_ATOMIC2K }, + { "atomic4k", SQLITE_IOCAP_ATOMIC4K }, + { "atomic8k", SQLITE_IOCAP_ATOMIC8K }, + { "atomic16k", SQLITE_IOCAP_ATOMIC16K }, + { "atomic32k", SQLITE_IOCAP_ATOMIC32K }, + { "atomic64k", SQLITE_IOCAP_ATOMIC64K }, + { "sequential", SQLITE_IOCAP_SEQUENTIAL }, + { "safe_append", SQLITE_IOCAP_SAFE_APPEND }, { "undeletable_when_open", SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN }, + { "powersafe_overwrite", SQLITE_IOCAP_POWERSAFE_OVERWRITE }, { 0, 0 } }; Tcl_Obj *pRet; @@ -1201,7 +1229,7 @@ static int testvfs_obj_cmd( iNew |= aFlag[idx].iValue; } - p->iDevchar = iNew; + p->iDevchar = iNew| 0x10000000; } pRet = Tcl_NewObj(); @@ -1368,7 +1396,7 @@ static int testvfs_cmd( } zVfs = Tcl_GetString(objv[1]); - nByte = sizeof(Testvfs) + strlen(zVfs)+1; + nByte = sizeof(Testvfs) + (int)strlen(zVfs)+1; p = (Testvfs *)ckalloc(nByte); memset(p, 0, nByte); p->iDevchar = -1; diff --git a/src/test_vfstrace.c b/src/test_vfstrace.c index 5e94f5c..3a0e2cf 100644 --- a/src/test_vfstrace.c +++ b/src/test_vfstrace.c @@ -471,7 +471,17 @@ static int vfstraceFileControl(sqlite3_file *pFile, int op, void *pArg){ } case SQLITE_FCNTL_FILE_POINTER: zOp = "FILE_POINTER"; break; case SQLITE_FCNTL_SYNC_OMITTED: zOp = "SYNC_OMITTED"; break; + case SQLITE_FCNTL_WIN32_AV_RETRY: zOp = "WIN32_AV_RETRY"; break; + case SQLITE_FCNTL_PERSIST_WAL: zOp = "PERSIST_WAL"; break; + case SQLITE_FCNTL_OVERWRITE: zOp = "OVERWRITE"; break; + case SQLITE_FCNTL_VFSNAME: zOp = "VFSNAME"; break; case 0xca093fa0: zOp = "DB_UNCHANGED"; break; + case SQLITE_FCNTL_PRAGMA: { + const char *const* a = (const char*const*)pArg; + sqlite3_snprintf(sizeof(zBuf), zBuf, "PRAGMA,[%s,%s]",a[1],a[2]); + zOp = zBuf; + break; + } default: { sqlite3_snprintf(sizeof zBuf, zBuf, "%d", op); zOp = zBuf; @@ -482,6 +492,14 @@ static int vfstraceFileControl(sqlite3_file *pFile, int op, void *pArg){ pInfo->zVfsName, p->zFName, zOp); rc = p->pReal->pMethods->xFileControl(p->pReal, op, pArg); vfstrace_print_errcode(pInfo, " -> %s\n", rc); + if( op==SQLITE_FCNTL_VFSNAME && rc==SQLITE_OK ){ + *(char**)pArg = sqlite3_mprintf("vfstrace.%s/%z", + pInfo->zVfsName, *(char**)pArg); + } + if( op==SQLITE_FCNTL_PRAGMA && rc==SQLITE_OK && *(char**)pArg ){ + vfstrace_printf(pInfo, "%s.xFileControl(%s,%s) returns %s", + pInfo->zVfsName, p->zFName, zOp, *(char**)pArg); + } return rc; } diff --git a/src/test_wholenumber.c b/src/test_wholenumber.c index 150dc95..7c42d01 100644 --- a/src/test_wholenumber.c +++ b/src/test_wholenumber.c @@ -33,8 +33,8 @@ typedef struct wholenumber_cursor wholenumber_cursor; struct wholenumber_cursor { sqlite3_vtab_cursor base; /* Base class - must be first */ - unsigned iValue; /* Current value */ - unsigned mxValue; /* Maximum value */ + sqlite3_int64 iValue; /* Current value */ + sqlite3_int64 mxValue; /* Maximum value */ }; /* Methods for the wholenumber module */ diff --git a/src/tokenize.c b/src/tokenize.c index b329892..faea5f2 100644 --- a/src/tokenize.c +++ b/src/tokenize.c @@ -123,7 +123,7 @@ int sqlite3GetToken(const unsigned char *z, int *tokenType){ } case '-': { if( z[1]=='-' ){ - /* IMP: R-15891-05542 -- syntax diagram for comments */ + /* IMP: R-50417-27976 -- syntax diagram for comments */ for(i=2; (c=z[i])!=0 && c!='\n'; i++){} *tokenType = TK_SPACE; /* IMP: R-22934-25134 */ return i; @@ -156,7 +156,7 @@ int sqlite3GetToken(const unsigned char *z, int *tokenType){ *tokenType = TK_SLASH; return 1; } - /* IMP: R-15891-05542 -- syntax diagram for comments */ + /* IMP: R-50417-27976 -- syntax diagram for comments */ for(i=3, c=z[2]; (c!='*' || z[i]!='/') && (c=z[i])!=0; i++){} if( c ) i++; *tokenType = TK_SPACE; /* IMP: R-22934-25134 */ diff --git a/src/trigger.c b/src/trigger.c index 22c4877..3c4bf62 100644 --- a/src/trigger.c +++ b/src/trigger.c @@ -904,6 +904,7 @@ static TriggerPrg *codeRowTrigger( } pProgram->nMem = pSubParse->nMem; pProgram->nCsr = pSubParse->nTab; + pProgram->nOnce = pSubParse->nOnce; pProgram->token = (void *)pTrigger; pPrg->aColmask[0] = pSubParse->oldmask; pPrg->aColmask[1] = pSubParse->newmask; diff --git a/src/update.c b/src/update.c index 1e30522..73d2269 100644 --- a/src/update.c +++ b/src/update.c @@ -126,8 +126,8 @@ void sqlite3Update( int regRowCount = 0; /* A count of rows changed */ int regOldRowid; /* The old rowid */ int regNewRowid; /* The new rowid */ - int regNew; - int regOld = 0; + int regNew; /* Content of the NEW.* table in triggers */ + int regOld = 0; /* Content of OLD.* table in triggers */ int regRowSet = 0; /* Rowset of rows to be updated */ memset(&sContext, 0, sizeof(sContext)); @@ -276,6 +276,7 @@ void sqlite3Update( #endif /* Allocate required registers. */ + regRowSet = ++pParse->nMem; regOldRowid = regNewRowid = ++pParse->nMem; if( pTrigger || hasFK ){ regOld = pParse->nMem + 1; @@ -310,7 +311,7 @@ void sqlite3Update( /* Begin the database scan */ - sqlite3VdbeAddOp2(v, OP_Null, 0, regOldRowid); + sqlite3VdbeAddOp3(v, OP_Null, 0, regRowSet, regOldRowid); pWInfo = sqlite3WhereBegin( pParse, pTabList, pWhere, 0, 0, WHERE_ONEPASS_DESIRED ); @@ -321,7 +322,6 @@ void sqlite3Update( */ sqlite3VdbeAddOp2(v, OP_Rowid, iCur, regOldRowid); if( !okOnePass ){ - regRowSet = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_RowSetAdd, regRowSet, regOldRowid); } @@ -425,9 +425,10 @@ void sqlite3Update( newmask = sqlite3TriggerColmask( pParse, pTrigger, pChanges, 1, TRIGGER_BEFORE, pTab, onError ); + sqlite3VdbeAddOp3(v, OP_Null, 0, regNew, regNew+pTab->nCol-1); for(i=0; i<pTab->nCol; i++){ if( i==pTab->iPKey ){ - sqlite3VdbeAddOp2(v, OP_Null, 0, regNew+i); + /*sqlite3VdbeAddOp2(v, OP_Null, 0, regNew+i);*/ }else{ j = aXRef[i]; if( j>=0 ){ @@ -216,13 +216,13 @@ int sqlite3Dequote(char *z){ ** Some systems have stricmp(). Others have strcasecmp(). Because ** there is no consistency, we will define our own. ** -** IMPLEMENTATION-OF: R-20522-24639 The sqlite3_strnicmp() API allows -** applications and extensions to compare the contents of two buffers -** containing UTF-8 strings in a case-independent fashion, using the same -** definition of case independence that SQLite uses internally when -** comparing identifiers. +** IMPLEMENTATION-OF: R-30243-02494 The sqlite3_stricmp() and +** sqlite3_strnicmp() APIs allow applications and extensions to compare +** the contents of two buffers containing UTF-8 strings in a +** case-independent fashion, using the same definition of "case +** independence" that SQLite uses internally when comparing identifiers. */ -int sqlite3StrICmp(const char *zLeft, const char *zRight){ +int sqlite3_stricmp(const char *zLeft, const char *zRight){ register unsigned char *a, *b; a = (unsigned char *)zLeft; b = (unsigned char *)zRight; @@ -1169,18 +1169,17 @@ int sqlite3AbsInt32(int x){ ** test.db-journal => test.nal ** test.db-wal => test.wal ** test.db-shm => test.shm +** test.db-mj7f3319fa => test.9fa */ void sqlite3FileSuffix3(const char *zBaseFilename, char *z){ #if SQLITE_ENABLE_8_3_NAMES<2 - const char *zOk; - zOk = sqlite3_uri_parameter(zBaseFilename, "8_3_names"); - if( zOk && sqlite3GetBoolean(zOk) ) + if( sqlite3_uri_boolean(zBaseFilename, "8_3_names", 0) ) #endif { int i, sz; sz = sqlite3Strlen30(z); for(i=sz-1; i>0 && z[i]!='/' && z[i]!='.'; i--){} - if( z[i]=='.' && ALWAYS(sz>i+4) ) memcpy(&z[i+1], &z[sz-3], 4); + if( z[i]=='.' && ALWAYS(sz>i+4) ) memmove(&z[i+1], &z[sz-3], 4); } } #endif diff --git a/src/vacuum.c b/src/vacuum.c index 58a3c9d..c03b450 100644 --- a/src/vacuum.c +++ b/src/vacuum.c @@ -176,6 +176,18 @@ int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){ } #endif + rc = execSql(db, pzErrMsg, "PRAGMA vacuum_db.synchronous=OFF"); + if( rc!=SQLITE_OK ) goto end_of_vacuum; + + /* Begin a transaction and take an exclusive lock on the main database + ** file. This is done before the sqlite3BtreeGetPageSize(pMain) call below, + ** to ensure that we do not try to change the page-size on a WAL database. + */ + rc = execSql(db, pzErrMsg, "BEGIN;"); + if( rc!=SQLITE_OK ) goto end_of_vacuum; + rc = sqlite3BtreeBeginTrans(pMain, 2); + if( rc!=SQLITE_OK ) goto end_of_vacuum; + /* Do not attempt to change the page size for a WAL database */ if( sqlite3PagerGetJournalMode(sqlite3BtreePager(pMain)) ==PAGER_JOURNALMODE_WAL ){ @@ -189,20 +201,12 @@ int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){ rc = SQLITE_NOMEM; goto end_of_vacuum; } - rc = execSql(db, pzErrMsg, "PRAGMA vacuum_db.synchronous=OFF"); - if( rc!=SQLITE_OK ){ - goto end_of_vacuum; - } #ifndef SQLITE_OMIT_AUTOVACUUM sqlite3BtreeSetAutoVacuum(pTemp, db->nextAutovac>=0 ? db->nextAutovac : sqlite3BtreeGetAutoVacuum(pMain)); #endif - /* Begin a transaction */ - rc = execSql(db, pzErrMsg, "BEGIN EXCLUSIVE;"); - if( rc!=SQLITE_OK ) goto end_of_vacuum; - /* Query the schema of the main database. Create a mirror schema ** in the temporary database. */ @@ -52,7 +52,7 @@ ** not misused. */ #ifdef SQLITE_DEBUG -# define memAboutToChange(P,M) sqlite3VdbeMemPrepareToChange(P,M) +# define memAboutToChange(P,M) sqlite3VdbeMemAboutToChange(P,M) #else # define memAboutToChange(P,M) #endif @@ -70,8 +70,8 @@ int sqlite3_search_count = 0; /* ** When this global variable is positive, it gets decremented once before -** each instruction in the VDBE. When reaches zero, the u1.isInterrupted -** field of the sqlite3 structure is set in order to simulate and interrupt. +** each instruction in the VDBE. When it reaches zero, the u1.isInterrupted +** field of the sqlite3 structure is set in order to simulate an interrupt. ** ** This facility is used for testing purposes only. It does not function ** in an ordinary build. @@ -151,12 +151,6 @@ int sqlite3_found_count = 0; if( ((P)->flags&MEM_Ephem)!=0 \ && sqlite3VdbeMemMakeWriteable(P) ){ goto no_mem;} -/* -** Call sqlite3VdbeMemExpandBlob() on the supplied value (type Mem*) -** P if required. -*/ -#define ExpandBlob(P) (((P)->flags&MEM_Zero)?sqlite3VdbeMemExpandBlob(P):0) - /* Return true if the cursor was opened using the OP_OpenSorter opcode. */ #ifdef SQLITE_OMIT_MERGE_SORT # define isSorter(x) 0 @@ -196,7 +190,7 @@ static VdbeCursor *allocateCursor( Vdbe *p, /* The virtual machine */ int iCur, /* Index of the new VdbeCursor */ int nField, /* Number of fields in the table or index */ - int iDb, /* When database the cursor belongs to, or -1 */ + int iDb, /* Database the cursor belongs to, or -1 */ int isBtreeCursor /* True for B-Tree. False for pseudo-table or vtab */ ){ /* Find the memory cell that will be used to store the blob of memory @@ -478,7 +472,7 @@ static void registerTrace(FILE *out, int iReg, Mem *p){ ** ** This macro added to every instruction that does a jump in order to ** implement a loop. This test used to be on every single instruction, -** but that meant we more testing that we needed. By only testing the +** but that meant we more testing than we needed. By only testing the ** flag on jump instructions, we get a (small) speed improvement. */ #define CHECK_FOR_INTERRUPT \ @@ -673,7 +667,7 @@ int sqlite3VdbeExec( assert( pOp->p2<=p->nMem ); pOut = &aMem[pOp->p2]; memAboutToChange(p, pOut); - MemReleaseExt(pOut); + VdbeMemRelease(pOut); pOut->flags = MEM_Int; } @@ -764,7 +758,8 @@ case OP_Goto: { /* jump */ ** Write the current address onto register P1 ** and then jump to address P2. */ -case OP_Gosub: { /* jump, in1 */ +case OP_Gosub: { /* jump */ + assert( pOp->p1>0 && pOp->p1<=p->nMem ); pIn1 = &aMem[pOp->p1]; assert( (pIn1->flags & MEM_Dyn)==0 ); memAboutToChange(p, pIn1); @@ -961,12 +956,25 @@ case OP_String: { /* out2-prerelease */ break; } -/* Opcode: Null * P2 * * * +/* Opcode: Null * P2 P3 * * ** -** Write a NULL into register P2. +** Write a NULL into registers P2. If P3 greater than P2, then also write +** NULL into register P3 and ever register in between P2 and P3. If P3 +** is less than P2 (typically P3 is zero) then only register P2 is +** set to NULL */ case OP_Null: { /* out2-prerelease */ + int cnt; + cnt = pOp->p3-pOp->p2; + assert( pOp->p3<=p->nMem ); pOut->flags = MEM_Null; + while( cnt>0 ){ + pOut++; + memAboutToChange(p, pOut); + VdbeMemRelease(pOut); + pOut->flags = MEM_Null; + cnt--; + } break; } @@ -1138,7 +1146,7 @@ case OP_ResultRow: { /* Make sure the results of the current row are \000 terminated ** and have an assigned type. The results are de-ephemeralized as - ** as side effect. + ** a side effect. */ pMem = p->pResultSet = &aMem[pOp->p1]; for(i=0; i<pOp->p2; i++){ @@ -1323,19 +1331,26 @@ arithmetic_result_is_null: break; } -/* Opcode: CollSeq * * P4 +/* Opcode: CollSeq P1 * * P4 ** ** P4 is a pointer to a CollSeq struct. If the next call to a user function ** or aggregate calls sqlite3GetFuncCollSeq(), this collation sequence will ** be returned. This is used by the built-in min(), max() and nullif() ** functions. ** +** If P1 is not zero, then it is a register that a subsequent min() or +** max() aggregate will set to 1 if the current row is not the minimum or +** maximum. The P1 register is initialized to 0 by this instruction. +** ** The interface used by the implementation of the aforementioned functions ** to retrieve the collation sequence set by this opcode is not available ** publicly, only to user functions defined in func.c. */ case OP_CollSeq: { assert( pOp->p4type==P4_COLLSEQ ); + if( pOp->p1 ){ + sqlite3VdbeMemSetInt64(&aMem[pOp->p1], 0); + } break; } @@ -2023,27 +2038,33 @@ case OP_BitNot: { /* same as TK_BITNOT, in1, out2 */ /* Opcode: Once P1 P2 * * * ** -** Jump to P2 if the value in register P1 is a not null or zero. If -** the value is NULL or zero, fall through and change the P1 register -** to an integer 1. +** Check if OP_Once flag P1 is set. If so, jump to instruction P2. Otherwise, +** set the flag and fall through to the next instruction. ** -** When P1 is not used otherwise in a program, this opcode falls through -** once and jumps on all subsequent invocations. It is the equivalent -** of "OP_If P1 P2", followed by "OP_Integer 1 P1". +** See also: JumpOnce */ +case OP_Once: { /* jump */ + assert( pOp->p1<p->nOnceFlag ); + if( p->aOnceFlag[pOp->p1] ){ + pc = pOp->p2-1; + }else{ + p->aOnceFlag[pOp->p1] = 1; + } + break; +} + /* Opcode: If P1 P2 P3 * * ** ** Jump to P2 if the value in register P1 is true. The value ** is considered true if it is numeric and non-zero. If the value -** in P1 is NULL then take the jump if P3 is true. +** in P1 is NULL then take the jump if P3 is non-zero. */ /* Opcode: IfNot P1 P2 P3 * * ** ** Jump to P2 if the value in register P1 is False. The value -** is considered true if it has a numeric value of zero. If the value -** in P1 is NULL then take the jump if P3 is true. +** is considered false if it has a numeric value of zero. If the value +** in P1 is NULL then take the jump if P3 is zero. */ -case OP_Once: /* jump, in1 */ case OP_If: /* jump, in1 */ case OP_IfNot: { /* jump, in1 */ int c; @@ -2060,12 +2081,6 @@ case OP_IfNot: { /* jump, in1 */ } if( c ){ pc = pOp->p2-1; - }else if( pOp->opcode==OP_Once ){ - assert( (pIn1->flags & (MEM_Agg|MEM_Dyn|MEM_RowSet|MEM_Frame))==0 ); - memAboutToChange(p, pIn1); - pIn1->flags = MEM_Int; - pIn1->u.i = 1; - REGISTER_TRACE(pOp->p1, pIn1); } break; } @@ -2112,6 +2127,11 @@ case OP_NotNull: { /* same as TK_NOTNULL, jump, in1 */ ** then the cache of the cursor is reset prior to extracting the column. ** The first OP_Column against a pseudo-table after the value of the content ** register has changed should have this bit set. +** +** If the OPFLAG_LENGTHARG and OPFLAG_TYPEOFARG bits are set on P5 when +** the result is guaranteed to only be used as the argument of a length() +** or typeof() function, respectively. The loading of large blobs can be +** skipped for length() and all content loading can be skipped for typeof(). */ case OP_Column: { u32 payloadSize; /* Number of bytes in the record */ @@ -2252,7 +2272,7 @@ case OP_Column: { pC->aRow = 0; } } - /* The following assert is true in all cases accept when + /* The following assert is true in all cases except when ** the database file has been corrupted externally. ** assert( zRec!=0 || avail>=payloadSize || avail>=9 ); */ szHdr = getVarint32((u8*)zData, offset); @@ -2327,11 +2347,11 @@ case OP_Column: { break; } }else{ - /* If i is less that nField, then there are less fields in this + /* If i is less that nField, then there are fewer fields in this ** record than SetNumColumns indicated there are columns in the ** table. Set the offset for any extra columns not present in - ** the record to 0. This tells code below to store a NULL - ** instead of deserializing a value from the record. + ** the record to 0. This tells code below to store the default value + ** for the column instead of deserializing a value from the record. */ aOffset[i] = 0; } @@ -2361,17 +2381,32 @@ case OP_Column: { if( aOffset[p2] ){ assert( rc==SQLITE_OK ); if( zRec ){ - MemReleaseExt(pDest); + /* This is the common case where the whole row fits on a single page */ + VdbeMemRelease(pDest); sqlite3VdbeSerialGet((u8 *)&zRec[aOffset[p2]], aType[p2], pDest); }else{ - len = sqlite3VdbeSerialTypeLen(aType[p2]); - sqlite3VdbeMemMove(&sMem, pDest); - rc = sqlite3VdbeMemFromBtree(pCrsr, aOffset[p2], len, pC->isIndex, &sMem); - if( rc!=SQLITE_OK ){ - goto op_column_out; + /* This branch happens only when the row overflows onto multiple pages */ + t = aType[p2]; + if( (pOp->p5 & (OPFLAG_LENGTHARG|OPFLAG_TYPEOFARG))!=0 + && ((t>=12 && (t&1)==0) || (pOp->p5 & OPFLAG_TYPEOFARG)!=0) + ){ + /* Content is irrelevant for the typeof() function and for + ** the length(X) function if X is a blob. So we might as well use + ** bogus content rather than reading content from disk. NULL works + ** for text and blob and whatever is in the payloadSize64 variable + ** will work for everything else. */ + zData = t<12 ? (char*)&payloadSize64 : 0; + }else{ + len = sqlite3VdbeSerialTypeLen(t); + sqlite3VdbeMemMove(&sMem, pDest); + rc = sqlite3VdbeMemFromBtree(pCrsr, aOffset[p2], len, pC->isIndex, + &sMem); + if( rc!=SQLITE_OK ){ + goto op_column_out; + } + zData = sMem.z; } - zData = sMem.z; - sqlite3VdbeSerialGet((u8*)zData, aType[p2], pDest); + sqlite3VdbeSerialGet((u8*)zData, t, pDest); } pDest->enc = encoding; }else{ @@ -2669,16 +2704,12 @@ case OP_Savepoint: { if( !pSavepoint ){ sqlite3SetString(&p->zErrMsg, db, "no such savepoint: %s", zName); rc = SQLITE_ERROR; - }else if( - db->writeVdbeCnt>0 || (p1==SAVEPOINT_ROLLBACK && db->activeVdbeCnt>1) - ){ + }else if( db->writeVdbeCnt>0 && p1==SAVEPOINT_RELEASE ){ /* It is not possible to release (commit) a savepoint if there are - ** active write statements. It is not possible to rollback a savepoint - ** if there are any active statements at all. + ** active write statements. */ sqlite3SetString(&p->zErrMsg, db, - "cannot %s savepoint - SQL statements in progress", - (p1==SAVEPOINT_ROLLBACK ? "rollback": "release") + "cannot release savepoint - SQL statements in progress" ); rc = SQLITE_BUSY; }else{ @@ -2703,6 +2734,11 @@ case OP_Savepoint: { rc = p->rc; }else{ iSavepoint = db->nSavepoint - iSavepoint - 1; + if( p1==SAVEPOINT_ROLLBACK ){ + for(ii=0; ii<db->nDb; ii++){ + sqlite3BtreeTripAllCursors(db->aDb[ii].pBt, SQLITE_ABORT); + } + } for(ii=0; ii<db->nDb; ii++){ rc = sqlite3BtreeSavepoint(db->aDb[ii].pBt, p1, iSavepoint); if( rc!=SQLITE_OK ){ @@ -2771,6 +2807,7 @@ case OP_AutoCommit: { assert( desiredAutoCommit==1 || iRollback==0 ); assert( db->activeVdbeCnt>0 ); /* At least this one VM is active */ +#if 0 if( turnOnAC && iRollback && db->activeVdbeCnt>1 ){ /* If this instruction implements a ROLLBACK and other VMs are ** still running, and a transaction is active, return an error indicating @@ -2779,7 +2816,9 @@ case OP_AutoCommit: { sqlite3SetString(&p->zErrMsg, db, "cannot rollback transaction - " "SQL statements in progress"); rc = SQLITE_BUSY; - }else if( turnOnAC && !iRollback && db->writeVdbeCnt>0 ){ + }else +#endif + if( turnOnAC && !iRollback && db->writeVdbeCnt>0 ){ /* If this instruction implements a COMMIT and other VMs are writing ** return an error indicating that the other VMs must complete first. */ @@ -2789,7 +2828,7 @@ case OP_AutoCommit: { }else if( desiredAutoCommit!=db->autoCommit ){ if( iRollback ){ assert( desiredAutoCommit==1 ); - sqlite3RollbackAll(db); + sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK); db->autoCommit = 1; }else if( (rc = sqlite3VdbeCheckFk(p, 1))!=SQLITE_OK ){ goto vdbe_return; @@ -2845,7 +2884,7 @@ case OP_AutoCommit: { ** throw an ABORT exception), a statement transaction may also be opened. ** More specifically, a statement transaction is opened iff the database ** connection is currently not in autocommit mode, or if there are other -** active statements. A statement transaction allows the affects of this +** active statements. A statement transaction allows the changes made by this ** VDBE to be rolled back after an error without having to roll back the ** entire transaction. If no error is encountered, the statement transaction ** will automatically commit when the VDBE halts. @@ -3827,7 +3866,7 @@ case OP_NewRowid: { /* out2-prerelease */ assert( sqlite3BtreeCursorIsValid(pC->pCursor) ); rc = sqlite3BtreeKeySize(pC->pCursor, &v); assert( rc==SQLITE_OK ); /* Cannot fail following BtreeLast() */ - if( v==MAX_ROWID ){ + if( v>=MAX_ROWID ){ pC->useRandomRowid = 1; }else{ v++; /* IMP: R-29538-34987 */ @@ -4616,9 +4655,9 @@ case OP_IdxGE: { /* jump */ r.pKeyInfo = pC->pKeyInfo; r.nField = (u16)pOp->p4.i; if( pOp->p5 ){ - r.flags = UNPACKED_INCRKEY | UNPACKED_IGNORE_ROWID; + r.flags = UNPACKED_INCRKEY | UNPACKED_PREFIX_MATCH; }else{ - r.flags = UNPACKED_IGNORE_ROWID; + r.flags = UNPACKED_PREFIX_MATCH; } r.aMem = &aMem[pOp->p3]; #ifdef SQLITE_DEBUG @@ -4825,6 +4864,7 @@ case OP_ParseSchema: { db->init.busy = 0; } } + if( rc ) sqlite3ResetInternalSchema(db, -1); if( rc==SQLITE_NOMEM ){ goto no_mem; } @@ -5071,7 +5111,6 @@ case OP_Program: { /* jump */ pProgram = pOp->p4.pProgram; pRt = &aMem[pOp->p3]; - assert( memIsValid(pRt) ); assert( pProgram->nOp>0 ); /* If the p5 flag is clear, then recursive invocation of triggers is @@ -5110,7 +5149,8 @@ case OP_Program: { /* jump */ nMem = pProgram->nMem + pProgram->nCsr; nByte = ROUND8(sizeof(VdbeFrame)) + nMem * sizeof(Mem) - + pProgram->nCsr * sizeof(VdbeCursor *); + + pProgram->nCsr * sizeof(VdbeCursor *) + + pProgram->nOnce * sizeof(u8); pFrame = sqlite3DbMallocZero(db, nByte); if( !pFrame ){ goto no_mem; @@ -5130,10 +5170,12 @@ case OP_Program: { /* jump */ pFrame->aOp = p->aOp; pFrame->nOp = p->nOp; pFrame->token = pProgram->token; + pFrame->aOnceFlag = p->aOnceFlag; + pFrame->nOnceFlag = p->nOnceFlag; pEnd = &VdbeFrameMem(pFrame)[pFrame->nChildMem]; for(pMem=VdbeFrameMem(pFrame); pMem!=pEnd; pMem++){ - pMem->flags = MEM_Null; + pMem->flags = MEM_Invalid; pMem->db = db; } }else{ @@ -5155,7 +5197,10 @@ case OP_Program: { /* jump */ p->apCsr = (VdbeCursor **)&aMem[p->nMem+1]; p->aOp = aOp = pProgram->aOp; p->nOp = pProgram->nOp; + p->aOnceFlag = (u8 *)&p->apCsr[p->nCursor]; + p->nOnceFlag = pProgram->nOnce; pc = -1; + memset(p->aOnceFlag, 0, p->nOnceFlag); break; } @@ -5342,6 +5387,7 @@ case OP_AggStep: { ctx.s.db = db; ctx.isError = 0; ctx.pColl = 0; + ctx.skipFlag = 0; if( ctx.pFunc->flags & SQLITE_FUNC_NEEDCOLL ){ assert( pOp>p->aOp ); assert( pOp[-1].p4type==P4_COLLSEQ ); @@ -5353,6 +5399,11 @@ case OP_AggStep: { sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3_value_text(&ctx.s)); rc = ctx.isError; } + if( ctx.skipFlag ){ + assert( pOp[-1].opcode==OP_CollSeq ); + i = pOp[-1].p1; + if( i ) sqlite3VdbeMemSetInt64(&aMem[i], 1); + } sqlite3VdbeMemRelease(&ctx.s); @@ -82,6 +82,7 @@ struct SubProgram { int nOp; /* Elements in aOp[] */ int nMem; /* Number of memory cells required */ int nCsr; /* Number of cursors required */ + int nOnce; /* Number of OP_Once instructions */ void *token; /* id that may be used to recursive triggers */ SubProgram *pNext; /* Next sub-program already visited */ }; diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 803ae16..9c1af35 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -33,6 +33,9 @@ typedef unsigned char Bool; /* Opaque type used by code in vdbesort.c */ typedef struct VdbeSorter VdbeSorter; +/* Opaque type used by the explainer */ +typedef struct Explain Explain; + /* ** A cursor is a pointer into a single BTree within a database file. ** The cursor can seek to a BTree entry with a particular key, or @@ -112,19 +115,21 @@ typedef struct VdbeCursor VdbeCursor; typedef struct VdbeFrame VdbeFrame; struct VdbeFrame { Vdbe *v; /* VM this frame belongs to */ - int pc; /* Program Counter in parent (calling) frame */ + VdbeFrame *pParent; /* Parent of this frame, or NULL if parent is main */ Op *aOp; /* Program instructions for parent frame */ - int nOp; /* Size of aOp array */ Mem *aMem; /* Array of memory cells for parent frame */ - int nMem; /* Number of entries in aMem */ + u8 *aOnceFlag; /* Array of OP_Once flags for parent frame */ VdbeCursor **apCsr; /* Array of Vdbe cursors for parent frame */ - u16 nCursor; /* Number of entries in apCsr */ void *token; /* Copy of SubProgram.token */ + i64 lastRowid; /* Last insert rowid (sqlite3.lastRowid) */ + u16 nCursor; /* Number of entries in apCsr */ + int pc; /* Program Counter in parent (calling) frame */ + int nOp; /* Size of aOp array */ + int nMem; /* Number of entries in aMem */ + int nOnceFlag; /* Number of entries in aOnceFlag */ int nChildMem; /* Number of memory cells for child frame */ int nChildCsr; /* Number of cursors for child frame */ - i64 lastRowid; /* Last insert rowid (sqlite3.lastRowid) */ int nChange; /* Statement changes (Vdbe.nChanges) */ - VdbeFrame *pParent; /* Parent of this frame, or NULL if parent is main */ }; #define VdbeFrameMem(p) ((Mem *)&((u8 *)p)[ROUND8(sizeof(VdbeFrame))]) @@ -251,8 +256,21 @@ struct sqlite3_context { VdbeFunc *pVdbeFunc; /* Auxilary data, if created. */ Mem s; /* The return value is stored here */ Mem *pMem; /* Memory cell used to store aggregate context */ - int isError; /* Error code returned by the function. */ CollSeq *pColl; /* Collating sequence */ + int isError; /* Error code returned by the function. */ + int skipFlag; /* Skip skip accumulator loading if true */ +}; + +/* +** An Explain object accumulates indented output which is helpful +** in describing recursive data structures. +*/ +struct Explain { + Vdbe *pVdbe; /* Attach the explanation to this Vdbe */ + StrAccum str; /* The string being accumulated */ + int nIndent; /* Number of elements in aIndent */ + u16 aIndent[100]; /* Levels of indentation */ + char zBase[100]; /* Initial space */ }; /* @@ -281,7 +299,6 @@ struct Vdbe { int nOp; /* Number of instructions in the program */ int nOpAlloc; /* Number of slots allocated for aOp[] */ int nLabel; /* Number of labels used */ - int nLabelAlloc; /* Number of slots allocated in aLabel[] */ int *aLabel; /* Space to hold the labels */ u16 nResColumn; /* Number of columns in one row of the result set */ u16 nCursor; /* Number of slots in apCsr[] */ @@ -321,11 +338,17 @@ struct Vdbe { #ifdef SQLITE_DEBUG FILE *trace; /* Write an execution trace here, if not NULL */ #endif +#ifdef SQLITE_ENABLE_TREE_EXPLAIN + Explain *pExplain; /* The explainer */ + char *zExplain; /* Explanation of data structures */ +#endif VdbeFrame *pFrame; /* Parent frame */ VdbeFrame *pDelFrame; /* List of frame objects to free on VM reset */ int nFrame; /* Number of frames in pFrame list */ u32 expmask; /* Binding to these vars invalidates VM */ SubProgram *pProgram; /* Linked list of all sub-programs used by VM */ + int nOnceFlag; /* Size of array aOnceFlag[] */ + u8 *aOnceFlag; /* Flags for OP_Once */ }; /* @@ -385,7 +408,7 @@ int sqlite3VdbeMemNumerify(Mem*); int sqlite3VdbeMemFromBtree(BtCursor*,int,int,int,Mem*); void sqlite3VdbeMemRelease(Mem *p); void sqlite3VdbeMemReleaseExternal(Mem *p); -#define MemReleaseExt(X) \ +#define VdbeMemRelease(X) \ if((X)->flags&(MEM_Agg|MEM_Dyn|MEM_RowSet|MEM_Frame)) \ sqlite3VdbeMemReleaseExternal(X); int sqlite3VdbeMemFinalize(Mem*, FuncDef*); @@ -424,7 +447,7 @@ int sqlite3VdbeSorterCompare(VdbeCursor *, Mem *, int *); #endif #ifdef SQLITE_DEBUG -void sqlite3VdbeMemPrepareToChange(Vdbe*,Mem*); +void sqlite3VdbeMemAboutToChange(Vdbe*,Mem*); #endif #ifndef SQLITE_OMIT_FOREIGN_KEY @@ -442,8 +465,10 @@ int sqlite3VdbeMemHandleBom(Mem *pMem); #ifndef SQLITE_OMIT_INCRBLOB int sqlite3VdbeMemExpandBlob(Mem *); + #define ExpandBlob(P) (((P)->flags&MEM_Zero)?sqlite3VdbeMemExpandBlob(P):0) #else #define sqlite3VdbeMemExpandBlob(x) SQLITE_OK + #define ExpandBlob(P) SQLITE_OK #endif #endif /* !defined(_VDBEINT_H_) */ diff --git a/src/vdbeapi.c b/src/vdbeapi.c index adc9dba..94db205 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -354,7 +354,7 @@ static int sqlite3Step(Vdbe *p){ ** ** Nevertheless, some published applications that were originally written ** for version 3.6.23 or earlier do in fact depend on SQLITE_MISUSE - ** returns, and the so were broken by the automatic-reset change. As a + ** returns, and those were broken by the automatic-reset change. As a ** a work-around, the SQLITE_OMIT_AUTORESET compile-time restores the ** legacy behavior of returning SQLITE_MISUSE for cases where the ** previous sqlite3_step() returned something other than a SQLITE_LOCKED @@ -700,13 +700,13 @@ static Mem *columnMem(sqlite3_stmt *pStmt, int i){ /* If the value passed as the second argument is out of range, return ** a pointer to the following static Mem object which contains the ** value SQL NULL. Even though the Mem structure contains an element - ** of type i64, on certain architecture (x86) with certain compiler + ** of type i64, on certain architectures (x86) with certain compiler ** switches (-Os), gcc may align this Mem object on a 4-byte boundary ** instead of an 8-byte one. This all works fine, except that when ** running with SQLITE_DEBUG defined the SQLite code sometimes assert()s ** that a Mem structure is located on an 8-byte boundary. To prevent - ** this assert() from failing, when building with SQLITE_DEBUG defined - ** using gcc, force nullMem to be 8-byte aligned using the magical + ** these assert()s from failing, when building with SQLITE_DEBUG defined + ** using gcc, we force nullMem to be 8-byte aligned using the magical ** __attribute__((aligned(8))) macro. */ static const Mem nullMem #if defined(SQLITE_DEBUG) && defined(__GNUC__) @@ -1278,6 +1278,14 @@ int sqlite3_stmt_readonly(sqlite3_stmt *pStmt){ } /* +** Return true if the prepared statement is in need of being reset. +*/ +int sqlite3_stmt_busy(sqlite3_stmt *pStmt){ + Vdbe *v = (Vdbe*)pStmt; + return v!=0 && v->pc>0 && v->magic==VDBE_MAGIC_RUN; +} + +/* ** Return a pointer to the next prepared statement after pStmt associated ** with database connection pDb. If pStmt is NULL, return the first ** prepared statement for the database connection. Return NULL if there diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 75250238..caa2bf6 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -196,7 +196,8 @@ int sqlite3VdbeAddOp4( /* ** Add an OP_ParseSchema opcode. This routine is broken out from -** sqlite3VdbeAddOp4() since it needs to also local all btrees. +** sqlite3VdbeAddOp4() since it needs to also needs to mark all btrees +** as having been used. ** ** The zWhere string must have been obtained from sqlite3_malloc(). ** This routine will take ownership of the allocated memory. @@ -239,14 +240,11 @@ int sqlite3VdbeAddOp4Int( ** Zero is returned if a malloc() fails. */ int sqlite3VdbeMakeLabel(Vdbe *p){ - int i; - i = p->nLabel++; + int i = p->nLabel++; assert( p->magic==VDBE_MAGIC_INIT ); - if( i>=p->nLabelAlloc ){ - int n = p->nLabelAlloc*2 + 5; - p->aLabel = sqlite3DbReallocOrFree(p->db, p->aLabel, - n*sizeof(p->aLabel[0])); - p->nLabelAlloc = sqlite3DbMallocSize(p->db, p->aLabel)/sizeof(p->aLabel[0]); + if( (i & (i-1))==0 ){ + p->aLabel = sqlite3DbReallocOrFree(p->db, p->aLabel, + (i*2+1)*sizeof(p->aLabel[0])); } if( p->aLabel ){ p->aLabel[i] = -1; @@ -913,13 +911,14 @@ static char *displayP4(Op *pOp, char *zTemp, int nTemp){ } case P4_MEM: { Mem *pMem = pOp->p4.pMem; - assert( (pMem->flags & MEM_Null)==0 ); if( pMem->flags & MEM_Str ){ zP4 = pMem->z; }else if( pMem->flags & MEM_Int ){ sqlite3_snprintf(nTemp, zTemp, "%lld", pMem->u.i); }else if( pMem->flags & MEM_Real ){ sqlite3_snprintf(nTemp, zTemp, "%.16g", pMem->r); + }else if( pMem->flags & MEM_Null ){ + sqlite3_snprintf(nTemp, zTemp, "NULL"); }else{ assert( pMem->flags & MEM_Blob ); zP4 = "(blob)"; @@ -962,8 +961,9 @@ static char *displayP4(Op *pOp, char *zTemp, int nTemp){ ** Declare to the Vdbe that the BTree object at db->aDb[i] is used. ** ** The prepared statements need to know in advance the complete set of -** attached databases that they will be using. A mask of these databases -** is maintained in p->btreeMask and is used for locking and other purposes. +** attached databases that will be use. A mask of these databases +** is maintained in p->btreeMask. The p->lockMask value is the subset of +** p->btreeMask of databases that will require a lock. */ void sqlite3VdbeUsesBtree(Vdbe *p, int i){ assert( i>=0 && i<p->db->nDb && i<(int)sizeof(yDbMask)*8 ); @@ -1094,7 +1094,7 @@ static void releaseMemArray(Mem *p, int N){ p->zMalloc = 0; } - p->flags = MEM_Null; + p->flags = MEM_Invalid; } db->mallocFailed = malloc_failed; } @@ -1239,7 +1239,7 @@ int sqlite3VdbeList( for(j=0; j<nSub; j++){ if( apSub[j]==pOp->p4.pProgram ) break; } - if( j==nSub && SQLITE_OK==sqlite3VdbeMemGrow(pSub, nByte, 1) ){ + if( j==nSub && SQLITE_OK==sqlite3VdbeMemGrow(pSub, nByte, nSub!=0) ){ apSub = (SubProgram **)pSub->z; apSub[nSub++] = pOp->p4.pProgram; pSub->flags |= MEM_Blob; @@ -1469,6 +1469,7 @@ void sqlite3VdbeMakeReady( int nMem; /* Number of VM memory registers */ int nCursor; /* Number of cursors required */ int nArg; /* Number of arguments in subprograms */ + int nOnce; /* Number of OP_Once instructions */ int n; /* Loop counter */ u8 *zCsr; /* Memory available for allocation */ u8 *zEnd; /* First byte past allocated memory */ @@ -1484,6 +1485,8 @@ void sqlite3VdbeMakeReady( nMem = pParse->nMem; nCursor = pParse->nTab; nArg = pParse->nMaxArg; + nOnce = pParse->nOnce; + if( nOnce==0 ) nOnce = 1; /* Ensure at least one byte in p->aOnceFlag[] */ /* For each cursor required, also allocate a memory cell. Memory ** cells (nMem+1-nCursor)..nMem, inclusive, will never be used by @@ -1530,6 +1533,7 @@ void sqlite3VdbeMakeReady( p->azVar = allocSpace(p->azVar, nVar*sizeof(char*), &zCsr, zEnd, &nByte); p->apCsr = allocSpace(p->apCsr, nCursor*sizeof(VdbeCursor*), &zCsr, zEnd, &nByte); + p->aOnceFlag = allocSpace(p->aOnceFlag, nOnce, &zCsr, zEnd, &nByte); if( nByte ){ p->pFree = sqlite3DbMallocZero(db, nByte); } @@ -1538,6 +1542,7 @@ void sqlite3VdbeMakeReady( }while( nByte && !db->mallocFailed ); p->nCursor = (u16)nCursor; + p->nOnceFlag = nOnce; if( p->aVar ){ p->nVar = (ynVar)nVar; for(n=0; n<nVar; n++){ @@ -1554,7 +1559,7 @@ void sqlite3VdbeMakeReady( p->aMem--; /* aMem[] goes from 1..nMem */ p->nMem = nMem; /* not from 0..nMem-1 */ for(n=1; n<=nMem; n++){ - p->aMem[n].flags = MEM_Null; + p->aMem[n].flags = MEM_Invalid; p->aMem[n].db = db; } } @@ -1596,6 +1601,8 @@ void sqlite3VdbeFreeCursor(Vdbe *p, VdbeCursor *pCx){ */ int sqlite3VdbeFrameRestore(VdbeFrame *pFrame){ Vdbe *v = pFrame->v; + v->aOnceFlag = pFrame->aOnceFlag; + v->nOnceFlag = pFrame->nOnceFlag; v->aOp = pFrame->aOp; v->nOp = pFrame->nOp; v->aMem = pFrame->aMem; @@ -1658,8 +1665,10 @@ static void Cleanup(Vdbe *p){ /* Execute assert() statements to ensure that the Vdbe.apCsr[] and ** Vdbe.aMem[] arrays have already been cleaned up. */ int i; - for(i=0; i<p->nCursor; i++) assert( p->apCsr==0 || p->apCsr[i]==0 ); - for(i=1; i<=p->nMem; i++) assert( p->aMem==0 || p->aMem[i].flags==MEM_Null ); + if( p->apCsr ) for(i=0; i<p->nCursor; i++) assert( p->apCsr[i]==0 ); + if( p->aMem ){ + for(i=1; i<=p->nMem; i++) assert( p->aMem[i].flags==MEM_Invalid ); + } #endif sqlite3DbFree(db, p->zErrMsg); @@ -1824,16 +1833,31 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ sqlite3_file *pMaster = 0; i64 offset = 0; int res; + int retryCount = 0; + int nMainFile; /* Select a master journal file name */ + nMainFile = sqlite3Strlen30(zMainFile); + zMaster = sqlite3MPrintf(db, "%s-mjXXXXXX9XXz", zMainFile); + if( zMaster==0 ) return SQLITE_NOMEM; do { u32 iRandom; - sqlite3DbFree(db, zMaster); - sqlite3_randomness(sizeof(iRandom), &iRandom); - zMaster = sqlite3MPrintf(db, "%s-mj%08X", zMainFile, iRandom&0x7fffffff); - if( !zMaster ){ - return SQLITE_NOMEM; + if( retryCount ){ + if( retryCount>100 ){ + sqlite3_log(SQLITE_FULL, "MJ delete: %s", zMaster); + sqlite3OsDelete(pVfs, zMaster, 0); + break; + }else if( retryCount==1 ){ + sqlite3_log(SQLITE_FULL, "MJ collide: %s", zMaster); + } } + retryCount++; + sqlite3_randomness(sizeof(iRandom), &iRandom); + sqlite3_snprintf(13, &zMaster[nMainFile], "-mj%06X9%02X", + (iRandom>>8)&0xffffff, iRandom&0xff); + /* The antipenultimate character of the master journal name must + ** be "9" to avoid name collisions when using 8+3 filenames. */ + assert( zMaster[sqlite3Strlen30(zMaster)-3]=='9' ); sqlite3FileSuffix3(zMainFile, zMaster); rc = sqlite3OsAccess(pVfs, zMaster, SQLITE_ACCESS_EXISTS, &res); }while( rc==SQLITE_OK && res ); @@ -1979,32 +2003,6 @@ static void checkActiveVdbeCnt(sqlite3 *db){ #endif /* -** For every Btree that in database connection db which -** has been modified, "trip" or invalidate each cursor in -** that Btree might have been modified so that the cursor -** can never be used again. This happens when a rollback -*** occurs. We have to trip all the other cursors, even -** cursor from other VMs in different database connections, -** so that none of them try to use the data at which they -** were pointing and which now may have been changed due -** to the rollback. -** -** Remember that a rollback can delete tables complete and -** reorder rootpages. So it is not sufficient just to save -** the state of the cursor. We have to invalidate the cursor -** so that it is never used again. -*/ -static void invalidateCursorsOnModifiedBtrees(sqlite3 *db){ - int i; - for(i=0; i<db->nDb; i++){ - Btree *p = db->aDb[i].pBt; - if( p && sqlite3BtreeIsInTrans(p) ){ - sqlite3BtreeTripAllCursors(p, SQLITE_ABORT); - } - } -} - -/* ** If the Vdbe passed as the first argument opened a statement-transaction, ** close it now. Argument eOp must be either SAVEPOINT_ROLLBACK or ** SAVEPOINT_RELEASE. If it is SAVEPOINT_ROLLBACK, then the statement @@ -2127,6 +2125,7 @@ int sqlite3VdbeHalt(Vdbe *p){ if( p->db->mallocFailed ){ p->rc = SQLITE_NOMEM; } + if( p->aOnceFlag ) memset(p->aOnceFlag, 0, p->nOnceFlag); closeAllCursors(p); if( p->magic!=VDBE_MAGIC_RUN ){ return SQLITE_OK; @@ -2167,8 +2166,7 @@ int sqlite3VdbeHalt(Vdbe *p){ /* We are forced to roll back the active transaction. Before doing ** so, abort any other statements this handle currently has active. */ - invalidateCursorsOnModifiedBtrees(db); - sqlite3RollbackAll(db); + sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK); sqlite3CloseSavepoints(db); db->autoCommit = 1; } @@ -2210,13 +2208,13 @@ int sqlite3VdbeHalt(Vdbe *p){ return SQLITE_BUSY; }else if( rc!=SQLITE_OK ){ p->rc = rc; - sqlite3RollbackAll(db); + sqlite3RollbackAll(db, SQLITE_OK); }else{ db->nDeferredCons = 0; sqlite3CommitInternalChanges(db); } }else{ - sqlite3RollbackAll(db); + sqlite3RollbackAll(db, SQLITE_OK); } db->nStatement = 0; }else if( eStatementOp==0 ){ @@ -2225,8 +2223,7 @@ int sqlite3VdbeHalt(Vdbe *p){ }else if( p->errorAction==OE_Abort ){ eStatementOp = SAVEPOINT_ROLLBACK; }else{ - invalidateCursorsOnModifiedBtrees(db); - sqlite3RollbackAll(db); + sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK); sqlite3CloseSavepoints(db); db->autoCommit = 1; } @@ -2246,8 +2243,7 @@ int sqlite3VdbeHalt(Vdbe *p){ sqlite3DbFree(db, p->zErrMsg); p->zErrMsg = 0; } - invalidateCursorsOnModifiedBtrees(db); - sqlite3RollbackAll(db); + sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK); sqlite3CloseSavepoints(db); db->autoCommit = 1; } @@ -2264,12 +2260,6 @@ int sqlite3VdbeHalt(Vdbe *p){ } p->nChange = 0; } - - /* Rollback or commit any schema changes that occurred. */ - if( p->rc!=SQLITE_OK && db->flags&SQLITE_InternChanges ){ - sqlite3ResetInternalSchema(db, -1); - db->flags = (db->flags | SQLITE_InternChanges); - } /* Release the locks */ sqlite3VdbeLeave(p); @@ -2464,6 +2454,10 @@ void sqlite3VdbeDeleteObject(sqlite3 *db, Vdbe *p){ sqlite3DbFree(db, p->aColName); sqlite3DbFree(db, p->zSql); sqlite3DbFree(db, p->pFree); +#if defined(SQLITE_ENABLE_TREE_EXPLAIN) + sqlite3DbFree(db, p->zExplain); + sqlite3DbFree(db, p->pExplain); +#endif sqlite3DbFree(db, p); } @@ -2943,15 +2937,6 @@ void sqlite3VdbeRecordUnpack( ** Or if the UNPACKED_MATCH_PREFIX flag is set and the prefixes are ** equal, then the keys are considered to be equal and ** the parts beyond the common prefix are ignored. -** -** If the UNPACKED_IGNORE_ROWID flag is set, then the last byte of -** the header of pKey1 is ignored. It is assumed that pKey1 is -** an index key, and thus ends with a rowid value. The last byte -** of the header will therefore be the serial type of the rowid: -** one of 1, 2, 3, 4, 5, 6, 8, or 9 - the integer serial types. -** The serial type of the final rowid will always be a single byte. -** By ignoring this last byte of the header, we force the comparison -** to ignore the rowid at the end of key1. */ int sqlite3VdbeRecordCompare( int nKey1, const void *pKey1, /* Left key */ @@ -2984,9 +2969,6 @@ int sqlite3VdbeRecordCompare( idx1 = getVarint32(aKey1, szHdr1); d1 = szHdr1; - if( pPKey2->flags & UNPACKED_IGNORE_ROWID ){ - szHdr1--; - } nField = pKeyInfo->nField; while( idx1<szHdr1 && i<pPKey2->nField ){ u32 serial_type1; @@ -3166,7 +3148,7 @@ int sqlite3VdbeIdxKeyCompare( if( rc ){ return rc; } - assert( pUnpacked->flags & UNPACKED_IGNORE_ROWID ); + assert( pUnpacked->flags & UNPACKED_PREFIX_MATCH ); *res = sqlite3VdbeRecordCompare(m.n, m.z, pUnpacked); sqlite3VdbeMemRelease(&m); return SQLITE_OK; diff --git a/src/vdbemem.c b/src/vdbemem.c index e6e9156..fd964de 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -19,12 +19,6 @@ #include "vdbeInt.h" /* -** Call sqlite3VdbeMemExpandBlob() on the supplied value (type Mem*) -** P if required. -*/ -#define expandBlob(P) (((P)->flags&MEM_Zero)?sqlite3VdbeMemExpandBlob(P):0) - -/* ** If pMem is an object with a valid string representation, this routine ** ensures the internal encoding for the string representation is ** 'desiredEnc', one of SQLITE_UTF8, SQLITE_UTF16LE or SQLITE_UTF16BE. @@ -65,10 +59,10 @@ int sqlite3VdbeChangeEncoding(Mem *pMem, int desiredEnc){ ** Make sure pMem->z points to a writable allocation of at least ** n bytes. ** -** If the memory cell currently contains string or blob data -** and the third argument passed to this function is true, the -** current content of the cell is preserved. Otherwise, it may -** be discarded. +** If the third argument passed to this function is true, then memory +** cell pMem must contain a string or blob. In this case the content is +** preserved. Otherwise, if the third parameter to this function is false, +** any current string or blob value may be discarded. ** ** This function sets the MEM_Dyn flag and clears any xDel callback. ** It also clears MEM_Ephem and MEM_Static. If the preserve flag is @@ -83,6 +77,10 @@ int sqlite3VdbeMemGrow(Mem *pMem, int n, int preserve){ ); assert( (pMem->flags&MEM_RowSet)==0 ); + /* If the preserve flag is set to true, then the memory cell must already + ** contain a valid string or blob value. */ + assert( preserve==0 || pMem->flags&(MEM_Blob|MEM_Str) ); + if( n<32 ) n = 32; if( sqlite3DbMallocSize(pMem->db, pMem->zMalloc)<n ){ if( preserve && pMem->z==pMem->zMalloc ){ @@ -98,6 +96,7 @@ int sqlite3VdbeMemGrow(Mem *pMem, int n, int preserve){ memcpy(pMem->zMalloc, pMem->z, pMem->n); } if( pMem->flags&MEM_Dyn && pMem->xDel ){ + assert( pMem->xDel!=SQLITE_DYNAMIC ); pMem->xDel((void *)(pMem->z)); } @@ -123,7 +122,7 @@ int sqlite3VdbeMemMakeWriteable(Mem *pMem){ int f; assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); assert( (pMem->flags&MEM_RowSet)==0 ); - expandBlob(pMem); + ExpandBlob(pMem); f = pMem->flags; if( (f&(MEM_Str|MEM_Blob)) && pMem->z!=pMem->zMalloc ){ if( sqlite3VdbeMemGrow(pMem, pMem->n + 2, 1) ){ @@ -277,6 +276,7 @@ void sqlite3VdbeMemReleaseExternal(Mem *p){ sqlite3VdbeMemRelease(p); }else if( p->flags&MEM_Dyn && p->xDel ){ assert( (p->flags&MEM_RowSet)==0 ); + assert( p->xDel!=SQLITE_DYNAMIC ); p->xDel((void *)p->z); p->xDel = 0; }else if( p->flags&MEM_RowSet ){ @@ -292,7 +292,7 @@ void sqlite3VdbeMemReleaseExternal(Mem *p){ ** (Mem.type==SQLITE_TEXT). */ void sqlite3VdbeMemRelease(Mem *p){ - MemReleaseExt(p); + VdbeMemRelease(p); sqlite3DbFree(p->db, p->zMalloc); p->z = 0; p->zMalloc = 0; @@ -419,8 +419,14 @@ void sqlite3VdbeIntegerAffinity(Mem *pMem){ ** true and could be omitted. But we leave it in because other ** architectures might behave differently. */ - if( pMem->r==(double)pMem->u.i && pMem->u.i>SMALLEST_INT64 - && ALWAYS(pMem->u.i<LARGEST_INT64) ){ + if( pMem->r==(double)pMem->u.i + && pMem->u.i>SMALLEST_INT64 +#if defined(__i486__) || defined(__x86_64__) + && ALWAYS(pMem->u.i<LARGEST_INT64) +#else + && pMem->u.i<LARGEST_INT64 +#endif + ){ pMem->flags |= MEM_Int; } } @@ -588,7 +594,7 @@ int sqlite3VdbeMemTooBig(Mem *p){ ** This is used for testing and debugging only - to make sure shallow ** copies are not misused. */ -void sqlite3VdbeMemPrepareToChange(Vdbe *pVdbe, Mem *pMem){ +void sqlite3VdbeMemAboutToChange(Vdbe *pVdbe, Mem *pMem){ int i; Mem *pX; for(i=1, pX=&pVdbe->aMem[1]; i<=pVdbe->nMem; i++, pX++){ @@ -614,7 +620,7 @@ void sqlite3VdbeMemPrepareToChange(Vdbe *pVdbe, Mem *pMem){ */ void sqlite3VdbeMemShallowCopy(Mem *pTo, const Mem *pFrom, int srcType){ assert( (pFrom->flags & MEM_RowSet)==0 ); - MemReleaseExt(pTo); + VdbeMemRelease(pTo); memcpy(pTo, pFrom, MEMCELLSIZE); pTo->xDel = 0; if( (pFrom->flags&MEM_Static)==0 ){ @@ -632,7 +638,7 @@ int sqlite3VdbeMemCopy(Mem *pTo, const Mem *pFrom){ int rc = SQLITE_OK; assert( (pFrom->flags & MEM_RowSet)==0 ); - MemReleaseExt(pTo); + VdbeMemRelease(pTo); memcpy(pTo, pFrom, MEMCELLSIZE); pTo->flags &= ~MEM_Dyn; @@ -960,7 +966,7 @@ const void *sqlite3ValueText(sqlite3_value* pVal, u8 enc){ } assert( (MEM_Blob>>3) == MEM_Str ); pVal->flags |= (pVal->flags & MEM_Blob)>>3; - expandBlob(pVal); + ExpandBlob(pVal); if( pVal->flags&MEM_Str ){ sqlite3VdbeChangeEncoding(pVal, enc & ~SQLITE_UTF16_ALIGNED); if( (enc & SQLITE_UTF16_ALIGNED)!=0 && 1==(1&SQLITE_PTR_TO_INT(pVal->z)) ){ @@ -969,7 +975,7 @@ const void *sqlite3ValueText(sqlite3_value* pVal, u8 enc){ return 0; } } - sqlite3VdbeMemNulTerminate(pVal); /* IMP: R-59893-45467 */ + sqlite3VdbeMemNulTerminate(pVal); /* IMP: R-31275-44060 */ }else{ assert( (pVal->flags&MEM_Blob)==0 ); sqlite3VdbeMemStringify(pVal, enc); diff --git a/src/vdbesort.c b/src/vdbesort.c index c449997..afea1f5 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -93,17 +93,17 @@ typedef struct SorterRecord SorterRecord; ** being merged (rounded up to the next power of 2). */ struct VdbeSorter { + i64 iWriteOff; /* Current write offset within file pTemp1 */ + i64 iReadOff; /* Current read offset within file pTemp1 */ int nInMemory; /* Current size of pRecord list as PMA */ int nTree; /* Used size of aTree/aIter (power of 2) */ + int nPMA; /* Number of PMAs stored in pTemp1 */ + int mnPmaSize; /* Minimum PMA size, in bytes */ + int mxPmaSize; /* Maximum PMA size, in bytes. 0==no limit */ VdbeSorterIter *aIter; /* Array of iterators to merge */ int *aTree; /* Current state of incremental merge */ - i64 iWriteOff; /* Current write offset within file pTemp1 */ - i64 iReadOff; /* Current read offset within file pTemp1 */ sqlite3_file *pTemp1; /* PMA file 1 */ - int nPMA; /* Number of PMAs stored in pTemp1 */ SorterRecord *pRecord; /* Head of in-memory record list */ - int mnPmaSize; /* Minimum PMA size, in bytes */ - int mxPmaSize; /* Maximum PMA size, in bytes. 0==no limit */ UnpackedRecord *pUnpacked; /* Used to unpack keys */ }; @@ -114,10 +114,10 @@ struct VdbeSorter { struct VdbeSorterIter { i64 iReadOff; /* Current read offset */ i64 iEof; /* 1 byte past EOF for this iterator */ - sqlite3_file *pFile; /* File iterator is reading from */ int nAlloc; /* Bytes of space at aAlloc */ - u8 *aAlloc; /* Allocated space */ int nKey; /* Number of bytes in key */ + sqlite3_file *pFile; /* File iterator is reading from */ + u8 *aAlloc; /* Allocated space */ u8 *aKey; /* Pointer to current key */ }; diff --git a/src/vdbetrace.c b/src/vdbetrace.c index de123b5..c71a7c4 100644 --- a/src/vdbetrace.c +++ b/src/vdbetrace.c @@ -12,6 +12,8 @@ ** ** This file contains code used to insert the values of host parameters ** (aka "wildcards") into the SQL text output by sqlite3_trace(). +** +** The Vdbe parse-tree explainer is also found here. */ #include "sqliteInt.h" #include "vdbeInt.h" @@ -152,3 +154,119 @@ char *sqlite3VdbeExpandSql( } #endif /* #ifndef SQLITE_OMIT_TRACE */ + +/***************************************************************************** +** The following code implements the data-structure explaining logic +** for the Vdbe. +*/ + +#if defined(SQLITE_ENABLE_TREE_EXPLAIN) + +/* +** Allocate a new Explain object +*/ +void sqlite3ExplainBegin(Vdbe *pVdbe){ + if( pVdbe ){ + Explain *p; + sqlite3BeginBenignMalloc(); + p = sqlite3_malloc( sizeof(Explain) ); + if( p ){ + memset(p, 0, sizeof(*p)); + p->pVdbe = pVdbe; + sqlite3_free(pVdbe->pExplain); + pVdbe->pExplain = p; + sqlite3StrAccumInit(&p->str, p->zBase, sizeof(p->zBase), + SQLITE_MAX_LENGTH); + p->str.useMalloc = 2; + }else{ + sqlite3EndBenignMalloc(); + } + } +} + +/* +** Return true if the Explain ends with a new-line. +*/ +static int endsWithNL(Explain *p){ + return p && p->str.zText && p->str.nChar + && p->str.zText[p->str.nChar-1]=='\n'; +} + +/* +** Append text to the indentation +*/ +void sqlite3ExplainPrintf(Vdbe *pVdbe, const char *zFormat, ...){ + Explain *p; + if( pVdbe && (p = pVdbe->pExplain)!=0 ){ + va_list ap; + if( p->nIndent && endsWithNL(p) ){ + int n = p->nIndent; + if( n>ArraySize(p->aIndent) ) n = ArraySize(p->aIndent); + sqlite3AppendSpace(&p->str, p->aIndent[n-1]); + } + va_start(ap, zFormat); + sqlite3VXPrintf(&p->str, 1, zFormat, ap); + va_end(ap); + } +} + +/* +** Append a '\n' if there is not already one. +*/ +void sqlite3ExplainNL(Vdbe *pVdbe){ + Explain *p; + if( pVdbe && (p = pVdbe->pExplain)!=0 && !endsWithNL(p) ){ + sqlite3StrAccumAppend(&p->str, "\n", 1); + } +} + +/* +** Push a new indentation level. Subsequent lines will be indented +** so that they begin at the current cursor position. +*/ +void sqlite3ExplainPush(Vdbe *pVdbe){ + Explain *p; + if( pVdbe && (p = pVdbe->pExplain)!=0 ){ + if( p->str.zText && p->nIndent<ArraySize(p->aIndent) ){ + const char *z = p->str.zText; + int i = p->str.nChar-1; + int x; + while( i>=0 && z[i]!='\n' ){ i--; } + x = (p->str.nChar - 1) - i; + if( p->nIndent && x<p->aIndent[p->nIndent-1] ){ + x = p->aIndent[p->nIndent-1]; + } + p->aIndent[p->nIndent] = x; + } + p->nIndent++; + } +} + +/* +** Pop the indentation stack by one level. +*/ +void sqlite3ExplainPop(Vdbe *p){ + if( p && p->pExplain ) p->pExplain->nIndent--; +} + +/* +** Free the indentation structure +*/ +void sqlite3ExplainFinish(Vdbe *pVdbe){ + if( pVdbe && pVdbe->pExplain ){ + sqlite3_free(pVdbe->zExplain); + sqlite3ExplainNL(pVdbe); + pVdbe->zExplain = sqlite3StrAccumFinish(&pVdbe->pExplain->str); + sqlite3_free(pVdbe->pExplain); + pVdbe->pExplain = 0; + sqlite3EndBenignMalloc(); + } +} + +/* +** Return the explanation of a virtual machine. +*/ +const char *sqlite3VdbeExplanation(Vdbe *pVdbe){ + return (pVdbe && pVdbe->zExplain) ? pVdbe->zExplain : 0; +} +#endif /* defined(SQLITE_DEBUG) */ @@ -278,13 +278,14 @@ void sqlite3VtabBeginParse( Parse *pParse, /* Parsing context */ Token *pName1, /* Name of new table, or database name */ Token *pName2, /* Name of new table or NULL */ - Token *pModuleName /* Name of the module for the virtual table */ + Token *pModuleName, /* Name of the module for the virtual table */ + int ifNotExists /* No error if the table already exists */ ){ int iDb; /* The database the table is being created in */ Table *pTable; /* The new virtual table */ sqlite3 *db; /* Database connection */ - sqlite3StartTable(pParse, pName1, pName2, 0, 0, 1, 0); + sqlite3StartTable(pParse, pName1, pName2, 0, 0, 1, ifNotExists); pTable = pParse->pNewTable; if( pTable==0 ) return; assert( 0==pTable->pIndex ); @@ -319,7 +320,7 @@ void sqlite3VtabBeginParse( ** virtual table currently under construction in pParse->pTable. */ static void addArgumentToVtab(Parse *pParse){ - if( pParse->sArg.z && ALWAYS(pParse->pNewTable) ){ + if( pParse->sArg.z && pParse->pNewTable ){ const char *z = (const char*)pParse->sArg.z; int n = pParse->sArg.n; sqlite3 *db = pParse->db; @@ -446,7 +447,7 @@ static int vtabCallConstructor( int (*xConstruct)(sqlite3*,void*,int,const char*const*,sqlite3_vtab**,char**), char **pzErr ){ - VtabCtx sCtx; + VtabCtx sCtx, *pPriorCtx; VTable *pVTable; int rc; const char *const*azArg = (const char *const*)pTab->azModuleArg; @@ -471,9 +472,10 @@ static int vtabCallConstructor( assert( xConstruct ); sCtx.pTab = pTab; sCtx.pVTable = pVTable; + pPriorCtx = db->pVtabCtx; db->pVtabCtx = &sCtx; rc = xConstruct(db, pMod->pAux, nArg, azArg, &pVTable->pVtab, &zErr); - db->pVtabCtx = 0; + db->pVtabCtx = pPriorCtx; if( rc==SQLITE_NOMEM ) db->mallocFailed = 1; if( SQLITE_OK!=rc ){ @@ -414,13 +414,18 @@ struct Wal { u32 iCallback; /* Value to pass to log callback (or 0) */ i64 mxWalSize; /* Truncate WAL to this size upon reset */ int nWiData; /* Size of array apWiData */ + int szFirstBlock; /* Size of first block written to WAL file */ volatile u32 **apWiData; /* Pointer to wal-index content in memory */ u32 szPage; /* Database page size */ i16 readLock; /* Which read lock is being held. -1 for none */ + u8 syncFlags; /* Flags to use to sync header writes */ u8 exclusiveMode; /* Non-zero if connection is in exclusive mode */ u8 writeLock; /* True if in a write transaction */ u8 ckptLock; /* True if holding a checkpoint lock */ u8 readOnly; /* WAL_RDWR, WAL_RDONLY, or WAL_SHM_RDONLY */ + u8 truncateOnCommit; /* True to truncate WAL file on commit */ + u8 syncHeader; /* Fsync the WAL header if true */ + u8 padToSectorBoundary; /* Pad transactions out to the next sector */ WalIndexHdr hdr; /* Wal-index header for current transaction */ const char *zWalName; /* Name of WAL file */ u32 nCkpt; /* Checkpoint sequence counter in the wal-header */ @@ -1093,6 +1098,7 @@ static int walIndexRecover(Wal *pWal){ int szPage; /* Page size according to the log */ u32 magic; /* Magic value read from WAL header */ u32 version; /* Magic value read from WAL header */ + int isValid; /* True if this frame is valid */ /* Read in the WAL header. */ rc = sqlite3OsRead(pWal->pWalFd, aBuf, WAL_HDRSIZE, 0); @@ -1151,14 +1157,14 @@ static int walIndexRecover(Wal *pWal){ for(iOffset=WAL_HDRSIZE; (iOffset+szFrame)<=nSize; iOffset+=szFrame){ u32 pgno; /* Database page number for frame */ u32 nTruncate; /* dbsize field from frame header */ - int isValid; /* True if this frame is valid */ /* Read and decode the next log frame. */ + iFrame++; rc = sqlite3OsRead(pWal->pWalFd, aFrame, szFrame, iOffset); if( rc!=SQLITE_OK ) break; isValid = walDecodeFrame(pWal, &pgno, &nTruncate, aData, aFrame); if( !isValid ) break; - rc = walIndexAppend(pWal, ++iFrame, pgno); + rc = walIndexAppend(pWal, iFrame, pgno); if( rc!=SQLITE_OK ) break; /* If nTruncate is non-zero, this is a commit record. */ @@ -1281,6 +1287,8 @@ int sqlite3WalOpen( pRet->readLock = -1; pRet->mxWalSize = mxWalSize; pRet->zWalName = zWalName; + pRet->syncHeader = 1; + pRet->padToSectorBoundary = 1; pRet->exclusiveMode = (bNoShm ? WAL_HEAPMEMORY_MODE: WAL_NORMAL_MODE); /* Open file handle on the write-ahead log file. */ @@ -1295,6 +1303,11 @@ int sqlite3WalOpen( sqlite3OsClose(pRet->pWalFd); sqlite3_free(pRet); }else{ + int iDC = sqlite3OsDeviceCharacteristics(pRet->pWalFd); + if( iDC & SQLITE_IOCAP_SEQUENTIAL ){ pRet->syncHeader = 0; } + if( iDC & SQLITE_IOCAP_POWERSAFE_OVERWRITE ){ + pRet->padToSectorBoundary = 0; + } *ppWal = pRet; WALTRACE(("WAL%d: opened\n", pRet)); } @@ -1714,7 +1727,7 @@ static int walCheckpoint( i64 nReq = ((i64)mxPage * szPage); rc = sqlite3OsFileSize(pWal->pDbFd, &nSize); if( rc==SQLITE_OK && nSize<nReq ){ - sqlite3OsFileControl(pWal->pDbFd, SQLITE_FCNTL_SIZE_HINT, &nReq); + sqlite3OsFileControlHint(pWal->pDbFd, SQLITE_FCNTL_SIZE_HINT, &nReq); } } @@ -1782,6 +1795,24 @@ static int walCheckpoint( } /* +** If the WAL file is currently larger than nMax bytes in size, truncate +** it to exactly nMax bytes. If an error occurs while doing so, ignore it. +*/ +static void walLimitSize(Wal *pWal, i64 nMax){ + i64 sz; + int rx; + sqlite3BeginBenignMalloc(); + rx = sqlite3OsFileSize(pWal->pWalFd, &sz); + if( rx==SQLITE_OK && (sz > nMax ) ){ + rx = sqlite3OsTruncate(pWal->pWalFd, nMax); + } + sqlite3EndBenignMalloc(); + if( rx ){ + sqlite3_log(rx, "cannot limit WAL size: %s", pWal->zWalName); + } +} + +/* ** Close a connection to a log file. */ int sqlite3WalClose( @@ -1804,23 +1835,40 @@ int sqlite3WalClose( */ rc = sqlite3OsLock(pWal->pDbFd, SQLITE_LOCK_EXCLUSIVE); if( rc==SQLITE_OK ){ - int bPersistWal = -1; if( pWal->exclusiveMode==WAL_NORMAL_MODE ){ pWal->exclusiveMode = WAL_EXCLUSIVE_MODE; } rc = sqlite3WalCheckpoint( pWal, SQLITE_CHECKPOINT_PASSIVE, 0, 0, sync_flags, nBuf, zBuf, 0, 0 ); - sqlite3OsFileControl(pWal->pDbFd, SQLITE_FCNTL_PERSIST_WAL, &bPersistWal); - if( rc==SQLITE_OK && bPersistWal!=1 ){ - isDelete = 1; + if( rc==SQLITE_OK ){ + int bPersist = -1; + sqlite3OsFileControlHint( + pWal->pDbFd, SQLITE_FCNTL_PERSIST_WAL, &bPersist + ); + if( bPersist!=1 ){ + /* Try to delete the WAL file if the checkpoint completed and + ** fsyned (rc==SQLITE_OK) and if we are not in persistent-wal + ** mode (!bPersist) */ + isDelete = 1; + }else if( pWal->mxWalSize>=0 ){ + /* Try to truncate the WAL file to zero bytes if the checkpoint + ** completed and fsynced (rc==SQLITE_OK) and we are in persistent + ** WAL mode (bPersist) and if the PRAGMA journal_size_limit is a + ** non-negative value (pWal->mxWalSize>=0). Note that we truncate + ** to zero bytes as truncating to the journal_size_limit might + ** leave a corrupt WAL file on disk. */ + walLimitSize(pWal, 0); + } } } walIndexClose(pWal, isDelete); sqlite3OsClose(pWal->pWalFd); if( isDelete ){ + sqlite3BeginBenignMalloc(); sqlite3OsDelete(pWal->pVfs, pWal->zWalName, 0); + sqlite3EndBenignMalloc(); } WALTRACE(("WAL%p: closed\n", pWal)); sqlite3_free((void *)pWal->apWiData); @@ -2310,7 +2358,7 @@ int sqlite3WalRead( for(iKey=walHash(pgno); aHash[iKey]; iKey=walNextHash(iKey)){ u32 iFrame = aHash[iKey] + iZero; if( iFrame<=iLast && aPgno[aHash[iKey]]==pgno ){ - assert( iFrame>iRead ); + /* assert( iFrame>iRead ); -- not true if there is corruption */ iRead = iFrame; } if( (nCollide--)==0 ){ @@ -2349,7 +2397,7 @@ int sqlite3WalRead( iOffset = walFrameOffset(iRead, sz) + WAL_FRAME_HDRSIZE; *pInWal = 1; /* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL */ - return sqlite3OsRead(pWal->pWalFd, pOut, nOut, iOffset); + return sqlite3OsRead(pWal->pWalFd, pOut, (nOut>sz ? sz : nOut), iOffset); } *pInWal = 0; @@ -2422,6 +2470,7 @@ int sqlite3WalEndWriteTransaction(Wal *pWal){ if( pWal->writeLock ){ walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1); pWal->writeLock = 0; + pWal->truncateOnCommit = 0; } return SQLITE_OK; } @@ -2518,6 +2567,7 @@ int sqlite3WalSavepointUndo(Wal *pWal, u32 *aWalData){ return rc; } + /* ** This function is called just before writing a set of frames to the log ** file (see sqlite3WalFrames()). It checks to see if, instead of appending @@ -2555,23 +2605,6 @@ static int walRestartLog(Wal *pWal){ int i; /* Loop counter */ u32 *aSalt = pWal->hdr.aSalt; /* Big-endian salt values */ - /* Limit the size of WAL file if the journal_size_limit PRAGMA is - ** set to a non-negative value. Log errors encountered - ** during the truncation attempt. */ - if( pWal->mxWalSize>=0 ){ - i64 sz; - int rx; - sqlite3BeginBenignMalloc(); - rx = sqlite3OsFileSize(pWal->pWalFd, &sz); - if( rx==SQLITE_OK && (sz > pWal->mxWalSize) ){ - rx = sqlite3OsTruncate(pWal->pWalFd, pWal->mxWalSize); - } - sqlite3EndBenignMalloc(); - if( rx ){ - sqlite3_log(rx, "cannot limit WAL size: %s", pWal->zWalName); - } - } - pWal->nCkpt++; pWal->hdr.mxFrame = 0; sqlite3Put4byte((u8*)&aSalt[0], 1 + sqlite3Get4byte((u8*)&aSalt[0])); @@ -2600,6 +2633,74 @@ static int walRestartLog(Wal *pWal){ return rc; } +/* +** Information about the current state of the WAL file and where +** the next fsync should occur - passed from sqlite3WalFrames() into +** walWriteToLog(). +*/ +typedef struct WalWriter { + Wal *pWal; /* The complete WAL information */ + sqlite3_file *pFd; /* The WAL file to which we write */ + sqlite3_int64 iSyncPoint; /* Fsync at this offset */ + int syncFlags; /* Flags for the fsync */ + int szPage; /* Size of one page */ +} WalWriter; + +/* +** Write iAmt bytes of content into the WAL file beginning at iOffset. +** Do a sync when crossing the p->iSyncPoint boundary. +** +** In other words, if iSyncPoint is in between iOffset and iOffset+iAmt, +** first write the part before iSyncPoint, then sync, then write the +** rest. +*/ +static int walWriteToLog( + WalWriter *p, /* WAL to write to */ + void *pContent, /* Content to be written */ + int iAmt, /* Number of bytes to write */ + sqlite3_int64 iOffset /* Start writing at this offset */ +){ + int rc; + if( iOffset<p->iSyncPoint && iOffset+iAmt>=p->iSyncPoint ){ + int iFirstAmt = (int)(p->iSyncPoint - iOffset); + rc = sqlite3OsWrite(p->pFd, pContent, iFirstAmt, iOffset); + if( rc ) return rc; + iOffset += iFirstAmt; + iAmt -= iFirstAmt; + pContent = (void*)(iFirstAmt + (char*)pContent); + assert( p->syncFlags & (SQLITE_SYNC_NORMAL|SQLITE_SYNC_FULL) ); + rc = sqlite3OsSync(p->pFd, p->syncFlags); + if( iAmt==0 || rc ) return rc; + } + rc = sqlite3OsWrite(p->pFd, pContent, iAmt, iOffset); + return rc; +} + +/* +** Write out a single frame of the WAL +*/ +static int walWriteOneFrame( + WalWriter *p, /* Where to write the frame */ + PgHdr *pPage, /* The page of the frame to be written */ + int nTruncate, /* The commit flag. Usually 0. >0 for commit */ + sqlite3_int64 iOffset /* Byte offset at which to write */ +){ + int rc; /* Result code from subfunctions */ + void *pData; /* Data actually written */ + u8 aFrame[WAL_FRAME_HDRSIZE]; /* Buffer to assemble frame-header in */ +#if defined(SQLITE_HAS_CODEC) + if( (pData = sqlite3PagerCodec(pPage))==0 ) return SQLITE_NOMEM; +#else + pData = pPage->pData; +#endif + walEncodeFrame(p->pWal, pPage->pgno, nTruncate, pData, aFrame); + rc = walWriteToLog(p, aFrame, sizeof(aFrame), iOffset); + if( rc ) return rc; + /* Write the page data */ + rc = walWriteToLog(p, pData, p->szPage, iOffset+sizeof(aFrame)); + return rc; +} + /* ** Write a set of frames to the log. The caller must hold the write-lock ** on the log file (obtained using sqlite3WalBeginWriteTransaction()). @@ -2614,14 +2715,20 @@ int sqlite3WalFrames( ){ int rc; /* Used to catch return codes */ u32 iFrame; /* Next frame address */ - u8 aFrame[WAL_FRAME_HDRSIZE]; /* Buffer to assemble frame-header in */ PgHdr *p; /* Iterator to run through pList with. */ PgHdr *pLast = 0; /* Last frame in list */ - int nLast = 0; /* Number of extra copies of last page */ + int nExtra = 0; /* Number of extra copies of last page */ + int szFrame; /* The size of a single frame */ + i64 iOffset; /* Next byte to write in WAL file */ + WalWriter w; /* The writer */ assert( pList ); assert( pWal->writeLock ); + /* If this frame set completes a transaction, then nTruncate>0. If + ** nTruncate==0 then this frame set does not complete the transaction. */ + assert( (isCommit!=0)==(nTruncate!=0) ); + #if defined(SQLITE_TEST) && defined(SQLITE_DEBUG) { int cnt; for(cnt=0, p=pList; p; p=p->pDirty, cnt++){} WALTRACE(("WAL%p: frame write begin. %d frames. mxFrame=%d. %s\n", @@ -2649,7 +2756,7 @@ int sqlite3WalFrames( sqlite3Put4byte(&aWalHdr[4], WAL_MAX_VERSION); sqlite3Put4byte(&aWalHdr[8], szPage); sqlite3Put4byte(&aWalHdr[12], pWal->nCkpt); - sqlite3_randomness(8, pWal->hdr.aSalt); + if( pWal->nCkpt==0 ) sqlite3_randomness(8, pWal->hdr.aSalt); memcpy(&aWalHdr[16], pWal->hdr.aSalt, 8); walChecksumBytes(1, aWalHdr, WAL_HDRSIZE-2*4, 0, aCksum); sqlite3Put4byte(&aWalHdr[24], aCksum[0]); @@ -2659,77 +2766,89 @@ int sqlite3WalFrames( pWal->hdr.bigEndCksum = SQLITE_BIGENDIAN; pWal->hdr.aFrameCksum[0] = aCksum[0]; pWal->hdr.aFrameCksum[1] = aCksum[1]; + pWal->truncateOnCommit = 1; rc = sqlite3OsWrite(pWal->pWalFd, aWalHdr, sizeof(aWalHdr), 0); WALTRACE(("WAL%p: wal-header write %s\n", pWal, rc ? "failed" : "ok")); if( rc!=SQLITE_OK ){ return rc; } + + /* Sync the header (unless SQLITE_IOCAP_SEQUENTIAL is true or unless + ** all syncing is turned off by PRAGMA synchronous=OFF). Otherwise + ** an out-of-order write following a WAL restart could result in + ** database corruption. See the ticket: + ** + ** http://localhost:591/sqlite/info/ff5be73dee + */ + if( pWal->syncHeader && sync_flags ){ + rc = sqlite3OsSync(pWal->pWalFd, sync_flags & SQLITE_SYNC_MASK); + if( rc ) return rc; + } } assert( (int)pWal->szPage==szPage ); - /* Write the log file. */ - for(p=pList; p; p=p->pDirty){ - u32 nDbsize; /* Db-size field for frame header */ - i64 iOffset; /* Write offset in log file */ - void *pData; - - iOffset = walFrameOffset(++iFrame, szPage); - /* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL */ - - /* Populate and write the frame header */ - nDbsize = (isCommit && p->pDirty==0) ? nTruncate : 0; -#if defined(SQLITE_HAS_CODEC) - if( (pData = sqlite3PagerCodec(p))==0 ) return SQLITE_NOMEM; -#else - pData = p->pData; -#endif - walEncodeFrame(pWal, p->pgno, nDbsize, pData, aFrame); - rc = sqlite3OsWrite(pWal->pWalFd, aFrame, sizeof(aFrame), iOffset); - if( rc!=SQLITE_OK ){ - return rc; - } + /* Setup information needed to write frames into the WAL */ + w.pWal = pWal; + w.pFd = pWal->pWalFd; + w.iSyncPoint = 0; + w.syncFlags = sync_flags; + w.szPage = szPage; + iOffset = walFrameOffset(iFrame+1, szPage); + szFrame = szPage + WAL_FRAME_HDRSIZE; - /* Write the page data */ - rc = sqlite3OsWrite(pWal->pWalFd, pData, szPage, iOffset+sizeof(aFrame)); - if( rc!=SQLITE_OK ){ - return rc; - } + /* Write all frames into the log file exactly once */ + for(p=pList; p; p=p->pDirty){ + int nDbSize; /* 0 normally. Positive == commit flag */ + iFrame++; + assert( iOffset==walFrameOffset(iFrame, szPage) ); + nDbSize = (isCommit && p->pDirty==0) ? nTruncate : 0; + rc = walWriteOneFrame(&w, p, nDbSize, iOffset); + if( rc ) return rc; pLast = p; + iOffset += szFrame; } - /* Sync the log file if the 'isSync' flag was specified. */ - if( sync_flags ){ - i64 iSegment = sqlite3OsSectorSize(pWal->pWalFd); - i64 iOffset = walFrameOffset(iFrame+1, szPage); - - assert( isCommit ); - assert( iSegment>0 ); - - iSegment = (((iOffset+iSegment-1)/iSegment) * iSegment); - while( iOffset<iSegment ){ - void *pData; -#if defined(SQLITE_HAS_CODEC) - if( (pData = sqlite3PagerCodec(pLast))==0 ) return SQLITE_NOMEM; -#else - pData = pLast->pData; -#endif - walEncodeFrame(pWal, pLast->pgno, nTruncate, pData, aFrame); - /* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL */ - rc = sqlite3OsWrite(pWal->pWalFd, aFrame, sizeof(aFrame), iOffset); - if( rc!=SQLITE_OK ){ - return rc; - } - iOffset += WAL_FRAME_HDRSIZE; - rc = sqlite3OsWrite(pWal->pWalFd, pData, szPage, iOffset); - if( rc!=SQLITE_OK ){ - return rc; + /* If this is the end of a transaction, then we might need to pad + ** the transaction and/or sync the WAL file. + ** + ** Padding and syncing only occur if this set of frames complete a + ** transaction and if PRAGMA synchronous=FULL. If synchronous==NORMAL + ** or synchonous==OFF, then no padding or syncing are needed. + ** + ** If SQLITE_IOCAP_POWERSAFE_OVERWRITE is defined, then padding is not + ** needed and only the sync is done. If padding is needed, then the + ** final frame is repeated (with its commit mark) until the next sector + ** boundary is crossed. Only the part of the WAL prior to the last + ** sector boundary is synced; the part of the last frame that extends + ** past the sector boundary is written after the sync. + */ + if( isCommit && (sync_flags & WAL_SYNC_TRANSACTIONS)!=0 ){ + if( pWal->padToSectorBoundary ){ + int sectorSize = sqlite3OsSectorSize(pWal->pWalFd); + w.iSyncPoint = ((iOffset+sectorSize-1)/sectorSize)*sectorSize; + while( iOffset<w.iSyncPoint ){ + rc = walWriteOneFrame(&w, pLast, nTruncate, iOffset); + if( rc ) return rc; + iOffset += szFrame; + nExtra++; } - nLast++; - iOffset += szPage; + }else{ + rc = sqlite3OsSync(w.pFd, sync_flags & SQLITE_SYNC_MASK); } + } - rc = sqlite3OsSync(pWal->pWalFd, sync_flags); + /* If this frame set completes the first transaction in the WAL and + ** if PRAGMA journal_size_limit is set, then truncate the WAL to the + ** journal size limit, if possible. + */ + if( isCommit && pWal->truncateOnCommit && pWal->mxWalSize>=0 ){ + i64 sz = pWal->mxWalSize; + if( walFrameOffset(iFrame+nExtra+1, szPage)>pWal->mxWalSize ){ + sz = walFrameOffset(iFrame+nExtra+1, szPage); + } + walLimitSize(pWal, sz); + pWal->truncateOnCommit = 0; } /* Append data to the wal-index. It is not necessary to lock the @@ -2742,9 +2861,9 @@ int sqlite3WalFrames( iFrame++; rc = walIndexAppend(pWal, iFrame, p->pgno); } - while( nLast>0 && rc==SQLITE_OK ){ + while( rc==SQLITE_OK && nExtra>0 ){ iFrame++; - nLast--; + nExtra--; rc = walIndexAppend(pWal, iFrame, pLast->pgno); } @@ -2949,4 +3068,16 @@ int sqlite3WalHeapMemory(Wal *pWal){ return (pWal && pWal->exclusiveMode==WAL_HEAPMEMORY_MODE ); } +#ifdef SQLITE_ENABLE_ZIPVFS +/* +** If the argument is not NULL, it points to a Wal object that holds a +** read-lock. This function returns the database page-size if it is known, +** or zero if it is not (or if pWal is NULL). +*/ +int sqlite3WalFramesize(Wal *pWal){ + assert( pWal==0 || pWal->readLock>=0 ); + return (pWal ? pWal->szPage : 0); +} +#endif + #endif /* #ifndef SQLITE_OMIT_WAL */ @@ -19,6 +19,12 @@ #include "sqliteInt.h" +/* Additional values that can be added to the sync_flags argument of +** sqlite3WalFrames(): +*/ +#define WAL_SYNC_TRANSACTIONS 0x20 /* Sync at the end of each transaction */ +#define SQLITE_SYNC_MASK 0x13 /* Mask off the SQLITE_SYNC_* values */ + #ifdef SQLITE_OMIT_WAL # define sqlite3WalOpen(x,y,z) 0 # define sqlite3WalLimit(x,y) @@ -37,6 +43,7 @@ # define sqlite3WalCallback(z) 0 # define sqlite3WalExclusiveMode(y,z) 0 # define sqlite3WalHeapMemory(z) 0 +# define sqlite3WalFramesize(z) 0 #else #define WAL_SAVEPOINT_NDATA 4 @@ -118,5 +125,12 @@ int sqlite3WalExclusiveMode(Wal *pWal, int op); */ int sqlite3WalHeapMemory(Wal *pWal); +#ifdef SQLITE_ENABLE_ZIPVFS +/* If the WAL file is not empty, return the number of bytes of content +** stored in each frame (i.e. the db page-size when the WAL was created). +*/ +int sqlite3WalFramesize(Wal *pWal); +#endif + #endif /* ifndef SQLITE_OMIT_WAL */ #endif /* _WAL_H_ */ diff --git a/src/where.c b/src/where.c index 05414da..d324228 100644 --- a/src/where.c +++ b/src/where.c @@ -604,7 +604,7 @@ static WhereTerm *findTerm( && pTerm->u.leftColumn==iColumn && (pTerm->eOperator & op)!=0 ){ - if( pIdx && pTerm->eOperator!=WO_ISNULL ){ + if( iColumn>=0 && pIdx && pTerm->eOperator!=WO_ISNULL ){ Expr *pX = pTerm->pExpr; CollSeq *pColl; char idxaff; @@ -686,7 +686,10 @@ static int isLikeOrGlob( #endif pList = pExpr->x.pList; pLeft = pList->a[1].pExpr; - if( pLeft->op!=TK_COLUMN || sqlite3ExprAffinity(pLeft)!=SQLITE_AFF_TEXT ){ + if( pLeft->op!=TK_COLUMN + || sqlite3ExprAffinity(pLeft)!=SQLITE_AFF_TEXT + || IsVirtual(pLeft->pTab) + ){ /* IMP: R-02065-49465 The left-hand side of the LIKE or GLOB operator must ** be the name of an indexed column with TEXT affinity. */ return 0; @@ -1559,15 +1562,19 @@ static int isDistinctRedundant( ** list, or else the WHERE clause contains a term of the form "col=X", ** where X is a constant value. The collation sequences of the ** comparison and select-list expressions must match those of the index. + ** + ** 3. All of those index columns for which the WHERE clause does not + ** contain a "col=X" term are subject to a NOT NULL constraint. */ for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ if( pIdx->onError==OE_None ) continue; for(i=0; i<pIdx->nColumn; i++){ int iCol = pIdx->aiColumn[i]; - if( 0==findTerm(pWC, iBase, iCol, ~(Bitmask)0, WO_EQ, pIdx) - && 0>findIndexCol(pParse, pDistinct, iBase, pIdx, i) - ){ - break; + if( 0==findTerm(pWC, iBase, iCol, ~(Bitmask)0, WO_EQ, pIdx) ){ + int iIdxCol = findIndexCol(pParse, pDistinct, iBase, pIdx, i); + if( iIdxCol<0 || pTab->aCol[pIdx->aiColumn[i]].notNull==0 ){ + break; + } } } if( i==pIdx->nColumn ){ @@ -1715,14 +1722,25 @@ static int isSortingIndex( } if( pIdx->onError!=OE_None && i==pIdx->nColumn && (wsFlags & WHERE_COLUMN_NULL)==0 - && !referencesOtherTables(pOrderBy, pMaskSet, j, base) ){ - /* All terms of this index match some prefix of the ORDER BY clause - ** and the index is UNIQUE and no terms on the tail of the ORDER BY - ** clause reference other tables in a join. If this is all true then - ** the order by clause is superfluous. Not that if the matching - ** condition is IS NULL then the result is not necessarily unique - ** even on a UNIQUE index, so disallow those cases. */ - return 1; + && !referencesOtherTables(pOrderBy, pMaskSet, j, base) + ){ + Column *aCol = pIdx->pTable->aCol; + + /* All terms of this index match some prefix of the ORDER BY clause, + ** the index is UNIQUE, and no terms on the tail of the ORDER BY + ** refer to other tables in a join. So, assuming that the index entries + ** visited contain no NULL values, then this index delivers rows in + ** the required order. + ** + ** It is not possible for any of the first nEqCol index fields to be + ** NULL (since the corresponding "=" operator in the WHERE clause would + ** not be true). So if all remaining index columns have NOT NULL + ** constaints attached to them, we can be confident that the visited + ** index entries are free of NULLs. */ + for(i=nEqCol; i<pIdx->nColumn; i++){ + if( aCol[pIdx->aiColumn[i]].notNull==0 ) break; + } + return (i==pIdx->nColumn); } return 0; } @@ -2005,7 +2023,6 @@ static void constructAutomaticIndex( int nByte; /* Byte of memory needed for pIdx */ Index *pIdx; /* Object describing the transient index */ Vdbe *v; /* Prepared statement under construction */ - int regIsInit; /* Register set by initialization */ int addrInit; /* Address of the initialization bypass jump */ Table *pTable; /* The table being indexed */ KeyInfo *pKeyinfo; /* Key information for the index */ @@ -2022,8 +2039,7 @@ static void constructAutomaticIndex( ** transient index on 2nd and subsequent iterations of the loop. */ v = pParse->pVdbe; assert( v!=0 ); - regIsInit = ++pParse->nMem; - addrInit = sqlite3VdbeAddOp1(v, OP_Once, regIsInit); + addrInit = sqlite3CodeOnce(pParse); /* Count the number of columns that will be added to the index ** and used to match WHERE clause constraints */ @@ -3052,10 +3068,24 @@ static void bestBtreeIndex( #endif used |= pTerm->prereqRight; } - - /* Determine the value of rangeDiv */ - if( nEq<pProbe->nColumn && pProbe->bUnordered==0 ){ - int j = pProbe->aiColumn[nEq]; + + /* If the index being considered is UNIQUE, and there is an equality + ** constraint for all columns in the index, then this search will find + ** at most a single row. In this case set the WHERE_UNIQUE flag to + ** indicate this to the caller. + ** + ** Otherwise, if the search may find more than one row, test to see if + ** there is a range constraint on indexed column (nEq+1) that can be + ** optimized using the index. + */ + if( nEq==pProbe->nColumn && pProbe->onError!=OE_None ){ + testcase( wsFlags & WHERE_COLUMN_IN ); + testcase( wsFlags & WHERE_COLUMN_NULL ); + if( (wsFlags & (WHERE_COLUMN_IN|WHERE_COLUMN_NULL))==0 ){ + wsFlags |= WHERE_UNIQUE; + } + }else if( pProbe->bUnordered==0 ){ + int j = (nEq==pProbe->nColumn ? -1 : pProbe->aiColumn[nEq]); if( findTerm(pWC, iCur, j, notReady, WO_LT|WO_LE|WO_GT|WO_GE, pIdx) ){ WhereTerm *pTop = findTerm(pWC, iCur, j, notReady, WO_LT|WO_LE, pIdx); WhereTerm *pBtm = findTerm(pWC, iCur, j, notReady, WO_GT|WO_GE, pIdx); @@ -3074,12 +3104,6 @@ static void bestBtreeIndex( } wsFlags |= (WHERE_COLUMN_RANGE|WHERE_ROWID_RANGE); } - }else if( pProbe->onError!=OE_None ){ - testcase( wsFlags & WHERE_COLUMN_IN ); - testcase( wsFlags & WHERE_COLUMN_NULL ); - if( (wsFlags & (WHERE_COLUMN_IN|WHERE_COLUMN_NULL))==0 ){ - wsFlags |= WHERE_UNIQUE; - } } /* If there is an ORDER BY clause and the index being considered will @@ -3097,7 +3121,9 @@ static void bestBtreeIndex( /* If there is a DISTINCT qualifier and this index will scan rows in ** order of the DISTINCT expressions, clear bDist and set the appropriate ** flags in wsFlags. */ - if( isDistinctIndex(pParse, pWC, pProbe, iCur, pDistinct, nEq) ){ + if( isDistinctIndex(pParse, pWC, pProbe, iCur, pDistinct, nEq) + && (wsFlags & WHERE_COLUMN_IN)==0 + ){ bDist = 0; wsFlags |= WHERE_ROWID_RANGE|WHERE_COLUMN_RANGE|WHERE_DISTINCT; } @@ -3690,10 +3716,12 @@ static char *explainIndexRange(sqlite3 *db, WhereLevel *pLevel, Table *pTab){ j = i; if( pPlan->wsFlags&WHERE_BTM_LIMIT ){ - explainAppendTerm(&txt, i++, aCol[aiColumn[j]].zName, ">"); + char *z = (j==pIndex->nColumn ) ? "rowid" : aCol[aiColumn[j]].zName; + explainAppendTerm(&txt, i++, z, ">"); } if( pPlan->wsFlags&WHERE_TOP_LIMIT ){ - explainAppendTerm(&txt, i, aCol[aiColumn[j]].zName, "<"); + char *z = (j==pIndex->nColumn ) ? "rowid" : aCol[aiColumn[j]].zName; + explainAppendTerm(&txt, i, z, "<"); } sqlite3StrAccumAppend(&txt, ")", 1); return sqlite3StrAccumFinish(&txt); @@ -3792,8 +3820,7 @@ static Bitmask codeOneLoopStart( WhereInfo *pWInfo, /* Complete information about the WHERE clause */ int iLevel, /* Which level of pWInfo->a[] should be coded */ u16 wctrlFlags, /* One of the WHERE_* flags defined in sqliteInt.h */ - Bitmask notReady, /* Which tables are currently available */ - Expr *pWhere /* Complete WHERE clause */ + Bitmask notReady /* Which tables are currently available */ ){ int j, k; /* Loop counters */ int iCur; /* The VDBE cursor for the table */ @@ -4051,7 +4078,7 @@ static Bitmask codeOneLoopStart( pIdx = pLevel->plan.u.pIdx; iIdxCur = pLevel->iIdxCur; - k = pIdx->aiColumn[nEq]; /* Column for inequality constraints */ + k = (nEq==pIdx->nColumn ? -1 : pIdx->aiColumn[nEq]); /* If this loop satisfies a sort order (pOrderBy) request that ** was passed to this function to implement a "SELECT min(x) ..." @@ -4097,7 +4124,9 @@ static Bitmask codeOneLoopStart( ** a forward order scan on a descending index, interchange the ** start and end terms (pRangeStart and pRangeEnd). */ - if( nEq<pIdx->nColumn && bRev==(pIdx->aSortOrder[nEq]==SQLITE_SO_ASC) ){ + if( (nEq<pIdx->nColumn && bRev==(pIdx->aSortOrder[nEq]==SQLITE_SO_ASC)) + || (bRev && pIdx->nColumn==nEq) + ){ SWAP(WhereTerm *, pRangeEnd, pRangeStart); } @@ -4330,10 +4359,25 @@ static Bitmask codeOneLoopStart( ** Then for every term xN, evaluate as the subexpression: xN AND z ** That way, terms in y that are factored into the disjunction will ** be picked up by the recursive calls to sqlite3WhereBegin() below. + ** + ** Actually, each subexpression is converted to "xN AND w" where w is + ** the "interesting" terms of z - terms that did not originate in the + ** ON or USING clause of a LEFT JOIN, and terms that are usable as + ** indices. */ if( pWC->nTerm>1 ){ - pAndExpr = sqlite3ExprAlloc(pParse->db, TK_AND, 0, 0); - pAndExpr->pRight = pWhere; + int iTerm; + for(iTerm=0; iTerm<pWC->nTerm; iTerm++){ + Expr *pExpr = pWC->a[iTerm].pExpr; + if( ExprHasProperty(pExpr, EP_FromJoin) ) continue; + if( pWC->a[iTerm].wtFlags & (TERM_VIRTUAL|TERM_ORINFO) ) continue; + if( (pWC->a[iTerm].eOperator & WO_ALL)==0 ) continue; + pExpr = sqlite3ExprDup(pParse->db, pExpr, 0); + pAndExpr = sqlite3ExprAnd(pParse->db, pAndExpr, pExpr); + } + if( pAndExpr ){ + pAndExpr = sqlite3PExpr(pParse, TK_AND, 0, pAndExpr, 0); + } } for(ii=0; ii<pOrWc->nTerm; ii++){ @@ -4357,7 +4401,7 @@ static Bitmask codeOneLoopStart( int iSet = ((ii==pOrWc->nTerm-1)?-1:ii); int r; r = sqlite3ExprCodeGetColumn(pParse, pTabItem->pTab, -1, iCur, - regRowid); + regRowid, 0); sqlite3VdbeAddOp4Int(v, OP_RowSetTest, regRowset, sqlite3VdbeCurrentAddr(v)+2, r, iSet); } @@ -4375,7 +4419,10 @@ static Bitmask codeOneLoopStart( } } } - sqlite3DbFree(pParse->db, pAndExpr); + if( pAndExpr ){ + pAndExpr->pLeft = 0; + sqlite3ExprDelete(pParse->db, pAndExpr); + } sqlite3VdbeChangeP1(v, iRetInit, sqlite3VdbeCurrentAddr(v)); sqlite3VdbeAddOp2(v, OP_Goto, 0, pLevel->addrBrk); sqlite3VdbeResolveLabel(v, iLoopBody); @@ -5031,7 +5078,7 @@ WhereInfo *sqlite3WhereBegin( for(i=0; i<nTabList; i++){ pLevel = &pWInfo->a[i]; explainOneScan(pParse, pTabList, pLevel, i, pLevel->iFrom, wctrlFlags); - notReady = codeOneLoopStart(pWInfo, i, wctrlFlags, notReady, pWhere); + notReady = codeOneLoopStart(pWInfo, i, wctrlFlags, notReady); pWInfo->iContinue = pLevel->addrCont; } |