From 569c6676a6ddb0ff73821d7693b5e18ddef809b9 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 16 Oct 2014 22:51:35 -0400 Subject: Imported Upstream version 3.2.0 --- ext/fts1/fts1.c | 7 +- ext/fts1/fulltext.c | 29 +- ext/fts2/fts2.c | 5 +- ext/fts2/fts2_hash.c | 2 + ext/fts2/fts2_porter.c | 3 + ext/fts2/fts2_tokenizer.c | 2 +- ext/fts2/fts2_tokenizer1.c | 3 + ext/fts3/fts3.c | 759 +++++++++++++++---- ext/fts3/fts3Int.h | 46 +- ext/fts3/fts3_aux.c | 129 +++- ext/fts3/fts3_expr.c | 221 +++--- ext/fts3/fts3_hash.c | 4 +- ext/fts3/fts3_porter.c | 74 +- ext/fts3/fts3_snippet.c | 3 +- ext/fts3/fts3_test.c | 49 ++ ext/fts3/fts3_unicode.c | 16 +- ext/fts3/fts3_unicode2.c | 53 +- ext/fts3/fts3_write.c | 506 +++++++++---- ext/fts3/tool/fts3view.c | 15 +- ext/fts3/unicode/mkunicode.tcl | 15 +- ext/icu/icu.c | 7 +- ext/misc/amatch.c | 8 +- ext/misc/closure.c | 16 +- ext/misc/compress.c | 107 +++ ext/misc/fileio.c | 100 +++ ext/misc/fuzzer.c | 13 +- ext/misc/ieee754.c | 4 +- ext/misc/nextchar.c | 88 ++- ext/misc/percentile.c | 219 ++++++ ext/misc/regexp.c | 6 +- ext/misc/spellfix.c | 48 +- ext/misc/totype.c | 512 +++++++++++++ ext/misc/vfslog.c | 759 +++++++++++++++++++ ext/misc/vtshim.c | 551 ++++++++++++++ ext/rtree/rtree.c | 1575 +++++++++++++++++++++------------------- ext/rtree/rtree1.test | 41 +- ext/rtree/rtree6.test | 40 +- ext/rtree/rtree8.test | 1 - ext/rtree/rtreeB.test | 2 +- ext/rtree/rtreeC.test | 273 +++++++ ext/rtree/rtreeD.test | 57 ++ ext/rtree/rtreeE.test | 129 ++++ ext/rtree/sqlite3rtree.h | 67 +- 43 files changed, 5228 insertions(+), 1336 deletions(-) create mode 100644 ext/misc/compress.c create mode 100644 ext/misc/fileio.c create mode 100644 ext/misc/percentile.c create mode 100644 ext/misc/totype.c create mode 100644 ext/misc/vfslog.c create mode 100644 ext/misc/vtshim.c create mode 100644 ext/rtree/rtreeC.test create mode 100644 ext/rtree/rtreeD.test create mode 100644 ext/rtree/rtreeE.test (limited to 'ext') diff --git a/ext/fts1/fts1.c b/ext/fts1/fts1.c index d5429ff..482cf75 100644 --- a/ext/fts1/fts1.c +++ b/ext/fts1/fts1.c @@ -3335,8 +3335,11 @@ int sqlite3Fts1Init(sqlite3 *db){ } #if !SQLITE_CORE -int sqlite3_extension_init(sqlite3 *db, char **pzErrMsg, - const sqlite3_api_routines *pApi){ +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_fts1_init(sqlite3 *db, char **pzErrMsg, + const sqlite3_api_routines *pApi){ SQLITE_EXTENSION_INIT2(pApi) return sqlite3Fts1Init(db); } diff --git a/ext/fts1/fulltext.c b/ext/fts1/fulltext.c index e6034ba..313ff30 100644 --- a/ext/fts1/fulltext.c +++ b/ext/fts1/fulltext.c @@ -852,8 +852,14 @@ static void fulltext_vtab_destroy(fulltext_vtab *v){ ** argv[3] - tokenizer name (optional, a sensible default is provided) ** argv[4..] - passed to tokenizer (optional based on tokenizer) **/ -static int fulltextConnect(sqlite3 *db, void *pAux, int argc, char **argv, - sqlite3_vtab **ppVTab){ +static int fulltextConnect( + sqlite3 *db, + void *pAux, + int argc, + const char * const *argv, + sqlite3_vtab **ppVTab, + char **pzErr +){ int rc; fulltext_vtab *v; sqlite3_tokenizer_module *m = NULL; @@ -898,8 +904,14 @@ static int fulltextConnect(sqlite3 *db, void *pAux, int argc, char **argv, return SQLITE_OK; } -static int fulltextCreate(sqlite3 *db, void *pAux, int argc, char **argv, - sqlite3_vtab **ppVTab){ +static int fulltextCreate( + sqlite3 *db, + void *pAux, + int argc, + const char * const *argv, + sqlite3_vtab **ppVTab, + char **pzErr +){ int rc; assert( argc>=3 ); @@ -934,7 +946,7 @@ static int fulltextCreate(sqlite3 *db, void *pAux, int argc, char **argv, "create index %_index on %_term(term, first)"); if( rc!=SQLITE_OK ) return rc; - return fulltextConnect(db, pAux, argc, argv, ppVTab); + return fulltextConnect(db, pAux, argc, argv, ppVTab, pzErr); } /* Decide how to handle an SQL query. @@ -1488,8 +1500,11 @@ int fulltext_init(sqlite3 *db){ } #if !SQLITE_CORE -int sqlite3_extension_init(sqlite3 *db, char **pzErrMsg, - const sqlite3_api_routines *pApi){ +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_fulltext_init(sqlite3 *db, char **pzErrMsg, + const sqlite3_api_routines *pApi){ SQLITE_EXTENSION_INIT2(pApi) return fulltext_init(db); } diff --git a/ext/fts2/fts2.c b/ext/fts2/fts2.c index f008ce6..0405fb7 100644 --- a/ext/fts2/fts2.c +++ b/ext/fts2/fts2.c @@ -6844,7 +6844,10 @@ int sqlite3Fts2Init(sqlite3 *db){ } #if !SQLITE_CORE -int sqlite3_extension_init( +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_fts2_init( sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi diff --git a/ext/fts2/fts2_hash.c b/ext/fts2/fts2_hash.c index f22fcc9..3596dcf 100644 --- a/ext/fts2/fts2_hash.c +++ b/ext/fts2/fts2_hash.c @@ -30,6 +30,8 @@ #include #include "sqlite3.h" +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT3 #include "fts2_hash.h" /* diff --git a/ext/fts2/fts2_porter.c b/ext/fts2/fts2_porter.c index 16620b9..881baf7 100644 --- a/ext/fts2/fts2_porter.c +++ b/ext/fts2/fts2_porter.c @@ -30,6 +30,9 @@ #include #include +#include "sqlite3.h" +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT3 #include "fts2_tokenizer.h" /* diff --git a/ext/fts2/fts2_tokenizer.c b/ext/fts2/fts2_tokenizer.c index a93790c..a3d6a63 100644 --- a/ext/fts2/fts2_tokenizer.c +++ b/ext/fts2/fts2_tokenizer.c @@ -28,7 +28,7 @@ #include "sqlite3.h" #include "sqlite3ext.h" -SQLITE_EXTENSION_INIT1 +SQLITE_EXTENSION_INIT3 #include "fts2_hash.h" #include "fts2_tokenizer.h" diff --git a/ext/fts2/fts2_tokenizer1.c b/ext/fts2/fts2_tokenizer1.c index 7e13366..fe4f9eb 100644 --- a/ext/fts2/fts2_tokenizer1.c +++ b/ext/fts2/fts2_tokenizer1.c @@ -30,6 +30,9 @@ #include #include +#include "sqlite3.h" +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT3 #include "fts2_tokenizer.h" typedef struct simple_tokenizer { diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index c00a13f..4f4b667 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -330,21 +330,37 @@ int sqlite3Fts3PutVarint(char *p, sqlite_int64 v){ return (int) (q - (unsigned char *)p); } +#define GETVARINT_STEP(v, ptr, shift, mask1, mask2, var, ret) \ + v = (v & mask1) | ( (*ptr++) << shift ); \ + if( (v & mask2)==0 ){ var = v; return ret; } +#define GETVARINT_INIT(v, ptr, shift, mask1, mask2, var, ret) \ + v = (*ptr++); \ + if( (v & mask2)==0 ){ var = v; return ret; } + /* ** Read a 64-bit variable-length integer from memory starting at p[0]. ** Return the number of bytes read, or 0 on error. ** The value is stored in *v. */ int sqlite3Fts3GetVarint(const char *p, sqlite_int64 *v){ - const unsigned char *q = (const unsigned char *) p; - sqlite_uint64 x = 0, y = 1; - while( (*q&0x80)==0x80 && q-(unsigned char *)p UNCOMPRESS */ { "order", 5 }, /* 4 -> ORDER */ { "content", 7 }, /* 5 -> CONTENT */ - { "languageid", 10 } /* 6 -> LANGUAGEID */ + { "languageid", 10 }, /* 6 -> LANGUAGEID */ + { "notindexed", 10 } /* 7 -> NOTINDEXED */ }; int iOpt; @@ -1197,6 +1237,11 @@ static int fts3InitVtab( zLanguageid = zVal; zVal = 0; break; + + case 7: /* NOTINDEXED */ + azNotindexed[nNotindexed++] = zVal; + zVal = 0; + break; } } sqlite3_free(zVal); @@ -1268,6 +1313,7 @@ static int fts3InitVtab( nByte = sizeof(Fts3Table) + /* Fts3Table */ nCol * sizeof(char *) + /* azColumn */ nIndex * sizeof(struct Fts3Index) + /* aIndex */ + nCol * sizeof(u8) + /* abNotindexed */ nName + /* zName */ nDb + /* zDb */ nString; /* Space for azColumn strings */ @@ -1287,7 +1333,7 @@ static int fts3InitVtab( p->bHasStat = isFts4; p->bFts4 = isFts4; p->bDescIdx = bDescIdx; - p->bAutoincrmerge = 0xff; /* 0xff means setting unknown */ + p->nAutoincrmerge = 0xff; /* 0xff means setting unknown */ p->zContentTbl = zContent; p->zLanguageid = zLanguageid; zContent = 0; @@ -1301,9 +1347,10 @@ static int fts3InitVtab( for(i=0; iaIndex[i].hPending, FTS3_HASH_STRING, 1); } + p->abNotindexed = (u8 *)&p->aIndex[nIndex]; /* Fill in the zName and zDb fields of the vtab structure. */ - zCsr = (char *)&p->aIndex[nIndex]; + zCsr = (char *)&p->abNotindexed[nCol]; p->zName = zCsr; memcpy(zCsr, argv[2], nName); zCsr += nName; @@ -1324,7 +1371,28 @@ static int fts3InitVtab( assert( zCsr <= &((char *)p)[nByte] ); } - if( (zCompress==0)!=(zUncompress==0) ){ + /* Fill in the abNotindexed array */ + for(iCol=0; iColazColumn[iCol]); + for(i=0; iazColumn[iCol], zNot, n) + ){ + p->abNotindexed[iCol] = 1; + sqlite3_free(zNot); + azNotindexed[i] = 0; + } + } + } + for(i=0; izDb, p->zName); - if( rc2==SQLITE_OK ) p->bHasStat = 1; + p->bHasStat = 2; } /* Figure out the page-size for the database. This is required in order to @@ -1365,7 +1430,9 @@ fts3_init_out: sqlite3_free(zUncompress); sqlite3_free(zContent); sqlite3_free(zLanguageid); + for(i=0; iestimatedRows variable to nRow. Unless this +** extension is currently being used by a version of SQLite too old to +** support estimatedRows. In that case this function is a no-op. +*/ +static void fts3SetEstimatedRows(sqlite3_index_info *pIdxInfo, i64 nRow){ +#if SQLITE_VERSION_NUMBER>=3008002 + if( sqlite3_libversion_number()>=3008002 ){ + pIdxInfo->estimatedRows = nRow; + } +#endif +} + /* ** Implementation of the xBestIndex method for FTS3 tables. There ** are three possible strategies, in order of preference: @@ -1416,23 +1496,40 @@ static int fts3BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ Fts3Table *p = (Fts3Table *)pVTab; int i; /* Iterator variable */ int iCons = -1; /* Index of constraint to use */ + int iLangidCons = -1; /* Index of langid=x constraint, if present */ + int iDocidGe = -1; /* Index of docid>=x constraint, if present */ + int iDocidLe = -1; /* Index of docid<=x constraint, if present */ + int iIdx; /* By default use a full table scan. This is an expensive option, ** so search through the constraints to see if a more efficient ** strategy is possible. */ pInfo->idxNum = FTS3_FULLSCAN_SEARCH; - pInfo->estimatedCost = 500000; + pInfo->estimatedCost = 5000000; for(i=0; inConstraint; i++){ + int bDocid; /* True if this constraint is on docid */ struct sqlite3_index_constraint *pCons = &pInfo->aConstraint[i]; - if( pCons->usable==0 ) continue; + if( pCons->usable==0 ){ + if( pCons->op==SQLITE_INDEX_CONSTRAINT_MATCH ){ + /* There exists an unusable MATCH constraint. This means that if + ** the planner does elect to use the results of this call as part + ** of the overall query plan the user will see an "unable to use + ** function MATCH in the requested context" error. To discourage + ** this, return a very high cost here. */ + pInfo->idxNum = FTS3_FULLSCAN_SEARCH; + pInfo->estimatedCost = 1e50; + fts3SetEstimatedRows(pInfo, ((sqlite3_int64)1) << 50); + return SQLITE_OK; + } + continue; + } + + bDocid = (pCons->iColumn<0 || pCons->iColumn==p->nColumn+1); /* A direct lookup on the rowid or docid column. Assign a cost of 1.0. */ - if( iCons<0 - && pCons->op==SQLITE_INDEX_CONSTRAINT_EQ - && (pCons->iColumn<0 || pCons->iColumn==p->nColumn+1 ) - ){ + if( iCons<0 && pCons->op==SQLITE_INDEX_CONSTRAINT_EQ && bDocid ){ pInfo->idxNum = FTS3_DOCID_SEARCH; pInfo->estimatedCost = 1.0; iCons = i; @@ -1461,14 +1558,38 @@ static int fts3BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ ){ iLangidCons = i; } + + if( bDocid ){ + switch( pCons->op ){ + case SQLITE_INDEX_CONSTRAINT_GE: + case SQLITE_INDEX_CONSTRAINT_GT: + iDocidGe = i; + break; + + case SQLITE_INDEX_CONSTRAINT_LE: + case SQLITE_INDEX_CONSTRAINT_LT: + iDocidLe = i; + break; + } + } } + iIdx = 1; if( iCons>=0 ){ - pInfo->aConstraintUsage[iCons].argvIndex = 1; + pInfo->aConstraintUsage[iCons].argvIndex = iIdx++; pInfo->aConstraintUsage[iCons].omit = 1; } if( iLangidCons>=0 ){ - pInfo->aConstraintUsage[iLangidCons].argvIndex = 2; + pInfo->idxNum |= FTS3_HAVE_LANGID; + pInfo->aConstraintUsage[iLangidCons].argvIndex = iIdx++; + } + if( iDocidGe>=0 ){ + pInfo->idxNum |= FTS3_HAVE_DOCID_GE; + pInfo->aConstraintUsage[iDocidGe].argvIndex = iIdx++; + } + if( iDocidLe>=0 ){ + pInfo->idxNum |= FTS3_HAVE_DOCID_LE; + pInfo->aConstraintUsage[iDocidLe].argvIndex = iIdx++; } /* Regardless of the strategy selected, FTS can deliver rows in rowid (or @@ -1646,10 +1767,10 @@ static int fts3ScanInteriorNode( /* Load the next term on the node into zBuffer. Use realloc() to expand ** the size of zBuffer if required. */ if( !isFirstTerm ){ - zCsr += sqlite3Fts3GetVarint32(zCsr, &nPrefix); + zCsr += fts3GetVarint32(zCsr, &nPrefix); } isFirstTerm = 0; - zCsr += sqlite3Fts3GetVarint32(zCsr, &nSuffix); + zCsr += fts3GetVarint32(zCsr, &nSuffix); if( nPrefix<0 || nSuffix<0 || &zCsr[nSuffix]>zEnd ){ rc = FTS_CORRUPT_VTAB; @@ -1737,7 +1858,7 @@ static int fts3SelectLeaf( assert( piLeaf || piLeaf2 ); - sqlite3Fts3GetVarint32(zNode, &iHeight); + fts3GetVarint32(zNode, &iHeight); rc = fts3ScanInteriorNode(zTerm, nTerm, zNode, nNode, piLeaf, piLeaf2); assert( !piLeaf2 || !piLeaf || rc!=SQLITE_OK || (*piLeaf<=*piLeaf2) ); @@ -1939,11 +2060,11 @@ static void fts3PoslistMerge( int iCol1; /* The current column index in pp1 */ int iCol2; /* The current column index in pp2 */ - if( *p1==POS_COLUMN ) sqlite3Fts3GetVarint32(&p1[1], &iCol1); + if( *p1==POS_COLUMN ) fts3GetVarint32(&p1[1], &iCol1); else if( *p1==POS_END ) iCol1 = POSITION_LIST_END; else iCol1 = 0; - if( *p2==POS_COLUMN ) sqlite3Fts3GetVarint32(&p2[1], &iCol2); + if( *p2==POS_COLUMN ) fts3GetVarint32(&p2[1], &iCol2); else if( *p2==POS_END ) iCol2 = POSITION_LIST_END; else iCol2 = 0; @@ -2036,11 +2157,11 @@ static int fts3PoslistPhraseMerge( assert( p!=0 && *p1!=0 && *p2!=0 ); if( *p1==POS_COLUMN ){ p1++; - p1 += sqlite3Fts3GetVarint32(p1, &iCol1); + p1 += fts3GetVarint32(p1, &iCol1); } if( *p2==POS_COLUMN ){ p2++; - p2 += sqlite3Fts3GetVarint32(p2, &iCol2); + p2 += fts3GetVarint32(p2, &iCol2); } while( 1 ){ @@ -2090,9 +2211,9 @@ static int fts3PoslistPhraseMerge( if( 0==*p1 || 0==*p2 ) break; p1++; - p1 += sqlite3Fts3GetVarint32(p1, &iCol1); + p1 += fts3GetVarint32(p1, &iCol1); p2++; - p2 += sqlite3Fts3GetVarint32(p2, &iCol2); + p2 += fts3GetVarint32(p2, &iCol2); } /* Advance pointer p1 or p2 (whichever corresponds to the smaller of @@ -2104,12 +2225,12 @@ static int fts3PoslistPhraseMerge( fts3ColumnlistCopy(0, &p1); if( 0==*p1 ) break; p1++; - p1 += sqlite3Fts3GetVarint32(p1, &iCol1); + p1 += fts3GetVarint32(p1, &iCol1); }else{ fts3ColumnlistCopy(0, &p2); if( 0==*p2 ) break; p2++; - p2 += sqlite3Fts3GetVarint32(p2, &iCol2); + p2 += fts3GetVarint32(p2, &iCol2); } } @@ -2915,6 +3036,33 @@ static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){ return rc; } +/* +** The following are copied from sqliteInt.h. +** +** Constants for the largest and smallest possible 64-bit signed integers. +** These macros are designed to work correctly on both 32-bit and 64-bit +** compilers. +*/ +#ifndef SQLITE_AMALGAMATION +# define LARGEST_INT64 (0xffffffff|(((sqlite3_int64)0x7fffffff)<<32)) +# define SMALLEST_INT64 (((sqlite3_int64)-1) - LARGEST_INT64) +#endif + +/* +** If the numeric type of argument pVal is "integer", then return it +** converted to a 64-bit signed integer. Otherwise, return a copy of +** the second parameter, iDefault. +*/ +static sqlite3_int64 fts3DocidRange(sqlite3_value *pVal, i64 iDefault){ + if( pVal ){ + int eType = sqlite3_value_numeric_type(pVal); + if( eType==SQLITE_INTEGER ){ + return sqlite3_value_int64(pVal); + } + } + return iDefault; +} + /* ** This is the xFilter interface for the virtual table. See ** the virtual table xFilter method documentation for additional @@ -2940,40 +3088,58 @@ static int fts3FilterMethod( ){ int rc; char *zSql; /* SQL statement used to access %_content */ + int eSearch; Fts3Table *p = (Fts3Table *)pCursor->pVtab; Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; + sqlite3_value *pCons = 0; /* The MATCH or rowid constraint, if any */ + sqlite3_value *pLangid = 0; /* The "langid = ?" constraint, if any */ + sqlite3_value *pDocidGe = 0; /* The "docid >= ?" constraint, if any */ + sqlite3_value *pDocidLe = 0; /* The "docid <= ?" constraint, if any */ + int iIdx; + UNUSED_PARAMETER(idxStr); UNUSED_PARAMETER(nVal); - assert( idxNum>=0 && idxNum<=(FTS3_FULLTEXT_SEARCH+p->nColumn) ); - assert( nVal==0 || nVal==1 || nVal==2 ); - assert( (nVal==0)==(idxNum==FTS3_FULLSCAN_SEARCH) ); + eSearch = (idxNum & 0x0000FFFF); + assert( eSearch>=0 && eSearch<=(FTS3_FULLTEXT_SEARCH+p->nColumn) ); assert( p->pSegments==0 ); + /* Collect arguments into local variables */ + iIdx = 0; + if( eSearch!=FTS3_FULLSCAN_SEARCH ) pCons = apVal[iIdx++]; + if( idxNum & FTS3_HAVE_LANGID ) pLangid = apVal[iIdx++]; + if( idxNum & FTS3_HAVE_DOCID_GE ) pDocidGe = apVal[iIdx++]; + if( idxNum & FTS3_HAVE_DOCID_LE ) pDocidLe = apVal[iIdx++]; + assert( iIdx==nVal ); + /* In case the cursor has been used before, clear it now. */ sqlite3_finalize(pCsr->pStmt); sqlite3_free(pCsr->aDoclist); sqlite3Fts3ExprFree(pCsr->pExpr); memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor)); + /* Set the lower and upper bounds on docids to return */ + pCsr->iMinDocid = fts3DocidRange(pDocidGe, SMALLEST_INT64); + pCsr->iMaxDocid = fts3DocidRange(pDocidLe, LARGEST_INT64); + if( idxStr ){ pCsr->bDesc = (idxStr[0]=='D'); }else{ pCsr->bDesc = p->bDescIdx; } - pCsr->eSearch = (i16)idxNum; + pCsr->eSearch = (i16)eSearch; - if( idxNum!=FTS3_DOCID_SEARCH && idxNum!=FTS3_FULLSCAN_SEARCH ){ - int iCol = idxNum-FTS3_FULLTEXT_SEARCH; - const char *zQuery = (const char *)sqlite3_value_text(apVal[0]); + if( eSearch!=FTS3_DOCID_SEARCH && eSearch!=FTS3_FULLSCAN_SEARCH ){ + int iCol = eSearch-FTS3_FULLTEXT_SEARCH; + const char *zQuery = (const char *)sqlite3_value_text(pCons); - if( zQuery==0 && sqlite3_value_type(apVal[0])!=SQLITE_NULL ){ + if( zQuery==0 && sqlite3_value_type(pCons)!=SQLITE_NULL ){ return SQLITE_NOMEM; } pCsr->iLangid = 0; - if( nVal==2 ) pCsr->iLangid = sqlite3_value_int(apVal[1]); + if( pLangid ) pCsr->iLangid = sqlite3_value_int(pLangid); assert( p->base.zErrMsg==0 ); rc = sqlite3Fts3ExprParse(p->pTokenizer, pCsr->iLangid, @@ -2984,11 +3150,7 @@ static int fts3FilterMethod( return rc; } - rc = sqlite3Fts3ReadLock(p); - if( rc!=SQLITE_OK ) return rc; - rc = fts3EvalStart(pCsr); - sqlite3Fts3SegmentsClose(p); if( rc!=SQLITE_OK ) return rc; pCsr->pNextId = pCsr->aDoclist; @@ -3000,7 +3162,7 @@ static int fts3FilterMethod( ** full-text query or docid lookup, the statement retrieves a single ** row by docid. */ - if( idxNum==FTS3_FULLSCAN_SEARCH ){ + if( eSearch==FTS3_FULLSCAN_SEARCH ){ zSql = sqlite3_mprintf( "SELECT %s ORDER BY rowid %s", p->zReadExprlist, (pCsr->bDesc ? "DESC" : "ASC") @@ -3011,10 +3173,10 @@ static int fts3FilterMethod( }else{ rc = SQLITE_NOMEM; } - }else if( idxNum==FTS3_DOCID_SEARCH ){ + }else if( eSearch==FTS3_DOCID_SEARCH ){ rc = fts3CursorSeekStmt(pCsr, &pCsr->pStmt); if( rc==SQLITE_OK ){ - rc = sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]); + rc = sqlite3_bind_value(pCsr->pStmt, 1, pCons); } } if( rc!=SQLITE_OK ) return rc; @@ -3142,7 +3304,10 @@ static int fts3SyncMethod(sqlite3_vtab *pVtab){ Fts3Table *p = (Fts3Table*)pVtab; int rc = sqlite3Fts3PendingTermsFlush(p); - if( rc==SQLITE_OK && p->bAutoincrmerge==1 && p->nLeafAdd>(nMinMerge/16) ){ + if( rc==SQLITE_OK + && p->nLeafAdd>(nMinMerge/16) + && p->nAutoincrmerge && p->nAutoincrmerge!=0xff + ){ int mxLevel = 0; /* Maximum relative level value in db */ int A; /* Incr-merge parameter A */ @@ -3150,14 +3315,41 @@ static int fts3SyncMethod(sqlite3_vtab *pVtab){ assert( rc==SQLITE_OK || mxLevel==0 ); A = p->nLeafAdd * mxLevel; A += (A/2); - if( A>(int)nMinMerge ) rc = sqlite3Fts3Incrmerge(p, A, 8); + if( A>(int)nMinMerge ) rc = sqlite3Fts3Incrmerge(p, A, p->nAutoincrmerge); } sqlite3Fts3SegmentsClose(p); return rc; } /* -** Implementation of xBegin() method. This is a no-op. +** If it is currently unknown whether or not the FTS table has an %_stat +** table (if p->bHasStat==2), attempt to determine this (set p->bHasStat +** to 0 or 1). Return SQLITE_OK if successful, or an SQLite error code +** if an error occurs. +*/ +static int fts3SetHasStat(Fts3Table *p){ + int rc = SQLITE_OK; + if( p->bHasStat==2 ){ + const char *zFmt ="SELECT 1 FROM %Q.sqlite_master WHERE tbl_name='%q_stat'"; + char *zSql = sqlite3_mprintf(zFmt, p->zDb, p->zName); + if( zSql ){ + sqlite3_stmt *pStmt = 0; + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + if( rc==SQLITE_OK ){ + int bHasStat = (sqlite3_step(pStmt)==SQLITE_ROW); + rc = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ) p->bHasStat = bHasStat; + } + sqlite3_free(zSql); + }else{ + rc = SQLITE_NOMEM; + } + } + return rc; +} + +/* +** Implementation of xBegin() method. */ static int fts3BeginMethod(sqlite3_vtab *pVtab){ Fts3Table *p = (Fts3Table*)pVtab; @@ -3168,7 +3360,7 @@ static int fts3BeginMethod(sqlite3_vtab *pVtab){ TESTONLY( p->inTransaction = 1 ); TESTONLY( p->mxSavepoint = -1; ); p->nLeafAdd = 0; - return SQLITE_OK; + return fts3SetHasStat(p); } /* @@ -3417,6 +3609,10 @@ static int fts3RenameMethod( sqlite3 *db = p->db; /* Database connection */ int rc; /* Return Code */ + /* At this point it must be known if the %_stat table exists or not. + ** So bHasStat may not be 2. */ + rc = fts3SetHasStat(p); + /* As it happens, the pending terms table is always empty here. This is ** because an "ALTER TABLE RENAME TABLE" statement inside a transaction ** always opens a savepoint transaction. And the xSavepoint() method @@ -3424,7 +3620,9 @@ static int fts3RenameMethod( ** PendingTermsFlush() in in case that changes. */ assert( p->nPendingData==0 ); - rc = sqlite3Fts3PendingTermsFlush(p); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3PendingTermsFlush(p); + } if( p->zContentTbl==0 ){ fts3DbExec(&rc, db, @@ -3552,7 +3750,7 @@ static void hashDestroy(void *p){ */ void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule); void sqlite3Fts3PorterTokenizerModule(sqlite3_tokenizer_module const**ppModule); -#ifdef SQLITE_ENABLE_FTS4_UNICODE61 +#ifndef SQLITE_DISABLE_FTS3_UNICODE void sqlite3Fts3UnicodeTokenizer(sqlite3_tokenizer_module const**ppModule); #endif #ifdef SQLITE_ENABLE_ICU @@ -3570,7 +3768,7 @@ int sqlite3Fts3Init(sqlite3 *db){ Fts3Hash *pHash = 0; const sqlite3_tokenizer_module *pSimple = 0; const sqlite3_tokenizer_module *pPorter = 0; -#ifdef SQLITE_ENABLE_FTS4_UNICODE61 +#ifndef SQLITE_DISABLE_FTS3_UNICODE const sqlite3_tokenizer_module *pUnicode = 0; #endif @@ -3579,7 +3777,7 @@ int sqlite3Fts3Init(sqlite3 *db){ sqlite3Fts3IcuTokenizerModule(&pIcu); #endif -#ifdef SQLITE_ENABLE_FTS4_UNICODE61 +#ifndef SQLITE_DISABLE_FTS3_UNICODE sqlite3Fts3UnicodeTokenizer(&pUnicode); #endif @@ -3607,7 +3805,7 @@ int sqlite3Fts3Init(sqlite3 *db){ if( sqlite3Fts3HashInsert(pHash, "simple", 7, (void *)pSimple) || sqlite3Fts3HashInsert(pHash, "porter", 7, (void *)pPorter) -#ifdef SQLITE_ENABLE_FTS4_UNICODE61 +#ifndef SQLITE_DISABLE_FTS3_UNICODE || sqlite3Fts3HashInsert(pHash, "unicode61", 10, (void *)pUnicode) #endif #ifdef SQLITE_ENABLE_ICU @@ -3905,6 +4103,12 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ return SQLITE_OK; } +/* +** Maximum number of tokens a phrase may have to be considered for the +** incremental doclists strategy. +*/ +#define MAX_INCR_PHRASE_TOKENS 4 + /* ** This function is called for each Fts3Phrase in a full-text query ** expression to initialize the mechanism for returning rows. Once this @@ -3918,23 +4122,43 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ ** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. */ static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){ - int rc; /* Error code */ - Fts3PhraseToken *pFirst = &p->aToken[0]; Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int rc = SQLITE_OK; /* Error code */ + int i; - if( pCsr->bDesc==pTab->bDescIdx - && bOptOk==1 - && p->nToken==1 - && pFirst->pSegcsr - && pFirst->pSegcsr->bLookup - && pFirst->bFirst==0 - ){ + /* Determine if doclists may be loaded from disk incrementally. This is + ** possible if the bOptOk argument is true, the FTS doclists will be + ** scanned in forward order, and the phrase consists of + ** MAX_INCR_PHRASE_TOKENS or fewer tokens, none of which are are "^first" + ** tokens or prefix tokens that cannot use a prefix-index. */ + int bHaveIncr = 0; + int bIncrOk = (bOptOk + && pCsr->bDesc==pTab->bDescIdx + && p->nToken<=MAX_INCR_PHRASE_TOKENS && p->nToken>0 + && p->nToken<=MAX_INCR_PHRASE_TOKENS && p->nToken>0 +#ifdef SQLITE_TEST + && pTab->bNoIncrDoclist==0 +#endif + ); + for(i=0; bIncrOk==1 && inToken; i++){ + Fts3PhraseToken *pToken = &p->aToken[i]; + if( pToken->bFirst || (pToken->pSegcsr!=0 && !pToken->pSegcsr->bLookup) ){ + bIncrOk = 0; + } + if( pToken->pSegcsr ) bHaveIncr = 1; + } + + if( bIncrOk && bHaveIncr ){ /* Use the incremental approach. */ int iCol = (p->iColumn >= pTab->nColumn ? -1 : p->iColumn); - rc = sqlite3Fts3MsrIncrStart( - pTab, pFirst->pSegcsr, iCol, pFirst->z, pFirst->n); + for(i=0; rc==SQLITE_OK && inToken; i++){ + Fts3PhraseToken *pToken = &p->aToken[i]; + Fts3MultiSegReader *pSegcsr = pToken->pSegcsr; + if( pSegcsr ){ + rc = sqlite3Fts3MsrIncrStart(pTab, pSegcsr, iCol, pToken->z, pToken->n); + } + } p->bIncr = 1; - }else{ /* Load the full doclist for the phrase into memory. */ rc = fts3EvalPhraseLoad(pCsr, p); @@ -4044,15 +4268,125 @@ void sqlite3Fts3DoclistNext( } /* -** Attempt to move the phrase iterator to point to the next matching docid. +** Advance the iterator pDL to the next entry in pDL->aAll/nAll. Set *pbEof +** to true if EOF is reached. +*/ +static void fts3EvalDlPhraseNext( + Fts3Table *pTab, + Fts3Doclist *pDL, + u8 *pbEof +){ + char *pIter; /* Used to iterate through aAll */ + char *pEnd = &pDL->aAll[pDL->nAll]; /* 1 byte past end of aAll */ + + if( pDL->pNextDocid ){ + pIter = pDL->pNextDocid; + }else{ + pIter = pDL->aAll; + } + + if( pIter>=pEnd ){ + /* We have already reached the end of this doclist. EOF. */ + *pbEof = 1; + }else{ + sqlite3_int64 iDelta; + pIter += sqlite3Fts3GetVarint(pIter, &iDelta); + if( pTab->bDescIdx==0 || pDL->pNextDocid==0 ){ + pDL->iDocid += iDelta; + }else{ + pDL->iDocid -= iDelta; + } + pDL->pList = pIter; + fts3PoslistCopy(0, &pIter); + pDL->nList = (int)(pIter - pDL->pList); + + /* pIter now points just past the 0x00 that terminates the position- + ** list for document pDL->iDocid. However, if this position-list was + ** edited in place by fts3EvalNearTrim(), then pIter may not actually + ** point to the start of the next docid value. The following line deals + ** with this case by advancing pIter past the zero-padding added by + ** fts3EvalNearTrim(). */ + while( pIterpNextDocid = pIter; + assert( pIter>=&pDL->aAll[pDL->nAll] || *pIter ); + *pbEof = 0; + } +} + +/* +** Helper type used by fts3EvalIncrPhraseNext() and incrPhraseTokenNext(). +*/ +typedef struct TokenDoclist TokenDoclist; +struct TokenDoclist { + int bIgnore; + sqlite3_int64 iDocid; + char *pList; + int nList; +}; + +/* +** Token pToken is an incrementally loaded token that is part of a +** multi-token phrase. Advance it to the next matching document in the +** database and populate output variable *p with the details of the new +** entry. Or, if the iterator has reached EOF, set *pbEof to true. +** ** If an error occurs, return an SQLite error code. Otherwise, return ** SQLITE_OK. +*/ +static int incrPhraseTokenNext( + Fts3Table *pTab, /* Virtual table handle */ + Fts3Phrase *pPhrase, /* Phrase to advance token of */ + int iToken, /* Specific token to advance */ + TokenDoclist *p, /* OUT: Docid and doclist for new entry */ + u8 *pbEof /* OUT: True if iterator is at EOF */ +){ + int rc = SQLITE_OK; + + if( pPhrase->iDoclistToken==iToken ){ + assert( p->bIgnore==0 ); + assert( pPhrase->aToken[iToken].pSegcsr==0 ); + fts3EvalDlPhraseNext(pTab, &pPhrase->doclist, pbEof); + p->pList = pPhrase->doclist.pList; + p->nList = pPhrase->doclist.nList; + p->iDocid = pPhrase->doclist.iDocid; + }else{ + Fts3PhraseToken *pToken = &pPhrase->aToken[iToken]; + assert( pToken->pDeferred==0 ); + assert( pToken->pSegcsr || pPhrase->iDoclistToken>=0 ); + if( pToken->pSegcsr ){ + assert( p->bIgnore==0 ); + rc = sqlite3Fts3MsrIncrNext( + pTab, pToken->pSegcsr, &p->iDocid, &p->pList, &p->nList + ); + if( p->pList==0 ) *pbEof = 1; + }else{ + p->bIgnore = 1; + } + } + + return rc; +} + + +/* +** The phrase iterator passed as the second argument: +** +** * features at least one token that uses an incremental doclist, and +** +** * does not contain any deferred tokens. +** +** Advance it to the next matching documnent in the database and populate +** the Fts3Doclist.pList and nList fields. ** ** If there is no "next" entry and no error occurs, then *pbEof is set to ** 1 before returning. Otherwise, if no error occurs and the iterator is ** successfully advanced, *pbEof is set to 0. +** +** If an error occurs, return an SQLite error code. Otherwise, return +** SQLITE_OK. */ -static int fts3EvalPhraseNext( +static int fts3EvalIncrPhraseNext( Fts3Cursor *pCsr, /* FTS Cursor handle */ Fts3Phrase *p, /* Phrase object to advance to next docid */ u8 *pbEof /* OUT: Set to 1 if EOF */ @@ -4060,57 +4394,116 @@ static int fts3EvalPhraseNext( int rc = SQLITE_OK; Fts3Doclist *pDL = &p->doclist; Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + u8 bEof = 0; - if( p->bIncr ){ - assert( p->nToken==1 ); - assert( pDL->pNextDocid==0 ); + /* This is only called if it is guaranteed that the phrase has at least + ** one incremental token. In which case the bIncr flag is set. */ + assert( p->bIncr==1 ); + + if( p->nToken==1 && p->bIncr ){ rc = sqlite3Fts3MsrIncrNext(pTab, p->aToken[0].pSegcsr, &pDL->iDocid, &pDL->pList, &pDL->nList ); - if( rc==SQLITE_OK && !pDL->pList ){ - *pbEof = 1; + if( pDL->pList==0 ) bEof = 1; + }else{ + int bDescDoclist = pCsr->bDesc; + struct TokenDoclist a[MAX_INCR_PHRASE_TOKENS]; + + memset(a, 0, sizeof(a)); + assert( p->nToken<=MAX_INCR_PHRASE_TOKENS ); + assert( p->iDoclistTokennToken && bEof==0; i++){ + rc = incrPhraseTokenNext(pTab, p, i, &a[i], &bEof); + if( a[i].bIgnore==0 && (bMaxSet==0 || DOCID_CMP(iMax, a[i].iDocid)<0) ){ + iMax = a[i].iDocid; + bMaxSet = 1; + } + } + assert( rc!=SQLITE_OK || a[p->nToken-1].bIgnore==0 ); + assert( rc!=SQLITE_OK || bMaxSet ); + + /* Keep advancing iterators until they all point to the same document */ + for(i=0; inToken; i++){ + while( rc==SQLITE_OK && bEof==0 + && a[i].bIgnore==0 && DOCID_CMP(a[i].iDocid, iMax)<0 + ){ + rc = incrPhraseTokenNext(pTab, p, i, &a[i], &bEof); + if( DOCID_CMP(a[i].iDocid, iMax)>0 ){ + iMax = a[i].iDocid; + i = 0; + } + } + } + + /* Check if the current entries really are a phrase match */ + if( bEof==0 ){ + int nList = 0; + int nByte = a[p->nToken-1].nList; + char *aDoclist = sqlite3_malloc(nByte+1); + if( !aDoclist ) return SQLITE_NOMEM; + memcpy(aDoclist, a[p->nToken-1].pList, nByte+1); + + for(i=0; i<(p->nToken-1); i++){ + if( a[i].bIgnore==0 ){ + char *pL = a[i].pList; + char *pR = aDoclist; + char *pOut = aDoclist; + int nDist = p->nToken-1-i; + int res = fts3PoslistPhraseMerge(&pOut, nDist, 0, 1, &pL, &pR); + if( res==0 ) break; + nList = (int)(pOut - aDoclist); + } + } + if( i==(p->nToken-1) ){ + pDL->iDocid = iMax; + pDL->pList = aDoclist; + pDL->nList = nList; + pDL->bFreeList = 1; + break; + } + sqlite3_free(aDoclist); + } } + } + + *pbEof = bEof; + return rc; +} + +/* +** Attempt to move the phrase iterator to point to the next matching docid. +** If an error occurs, return an SQLite error code. Otherwise, return +** SQLITE_OK. +** +** If there is no "next" entry and no error occurs, then *pbEof is set to +** 1 before returning. Otherwise, if no error occurs and the iterator is +** successfully advanced, *pbEof is set to 0. +*/ +static int fts3EvalPhraseNext( + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Phrase *p, /* Phrase object to advance to next docid */ + u8 *pbEof /* OUT: Set to 1 if EOF */ +){ + int rc = SQLITE_OK; + Fts3Doclist *pDL = &p->doclist; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + + if( p->bIncr ){ + rc = fts3EvalIncrPhraseNext(pCsr, p, pbEof); }else if( pCsr->bDesc!=pTab->bDescIdx && pDL->nAll ){ sqlite3Fts3DoclistPrev(pTab->bDescIdx, pDL->aAll, pDL->nAll, &pDL->pNextDocid, &pDL->iDocid, &pDL->nList, pbEof ); pDL->pList = pDL->pNextDocid; }else{ - char *pIter; /* Used to iterate through aAll */ - char *pEnd = &pDL->aAll[pDL->nAll]; /* 1 byte past end of aAll */ - if( pDL->pNextDocid ){ - pIter = pDL->pNextDocid; - }else{ - pIter = pDL->aAll; - } - - if( pIter>=pEnd ){ - /* We have already reached the end of this doclist. EOF. */ - *pbEof = 1; - }else{ - sqlite3_int64 iDelta; - pIter += sqlite3Fts3GetVarint(pIter, &iDelta); - if( pTab->bDescIdx==0 || pDL->pNextDocid==0 ){ - pDL->iDocid += iDelta; - }else{ - pDL->iDocid -= iDelta; - } - pDL->pList = pIter; - fts3PoslistCopy(0, &pIter); - pDL->nList = (int)(pIter - pDL->pList); - - /* pIter now points just past the 0x00 that terminates the position- - ** list for document pDL->iDocid. However, if this position-list was - ** edited in place by fts3EvalNearTrim(), then pIter may not actually - ** point to the start of the next docid value. The following line deals - ** with this case by advancing pIter past the zero-padding added by - ** fts3EvalNearTrim(). */ - while( pIterpNextDocid = pIter; - assert( pIter>=&pDL->aAll[pDL->nAll] || *pIter ); - *pbEof = 0; - } + fts3EvalDlPhraseNext(pTab, pDL, pbEof); } return rc; @@ -4135,7 +4528,6 @@ static int fts3EvalPhraseNext( static void fts3EvalStartReaders( Fts3Cursor *pCsr, /* FTS Cursor handle */ Fts3Expr *pExpr, /* Expression to initialize phrases in */ - int bOptOk, /* True to enable incremental loading */ int *pRc /* IN/OUT: Error code */ ){ if( pExpr && SQLITE_OK==*pRc ){ @@ -4146,10 +4538,10 @@ static void fts3EvalStartReaders( if( pExpr->pPhrase->aToken[i].pDeferred==0 ) break; } pExpr->bDeferred = (i==nToken); - *pRc = fts3EvalPhraseStart(pCsr, bOptOk, pExpr->pPhrase); + *pRc = fts3EvalPhraseStart(pCsr, 1, pExpr->pPhrase); }else{ - fts3EvalStartReaders(pCsr, pExpr->pLeft, bOptOk, pRc); - fts3EvalStartReaders(pCsr, pExpr->pRight, bOptOk, pRc); + fts3EvalStartReaders(pCsr, pExpr->pLeft, pRc); + fts3EvalStartReaders(pCsr, pExpr->pRight, pRc); pExpr->bDeferred = (pExpr->pLeft->bDeferred && pExpr->pRight->bDeferred); } } @@ -4391,7 +4783,7 @@ static int fts3EvalSelectDeferred( ** overflowing the 32-bit integer it is stored in. */ if( ii<12 ) nLoad4 = nLoad4*4; - if( ii==0 || pTC->pPhrase->nToken>1 ){ + if( ii==0 || (pTC->pPhrase->nToken>1 && ii!=nToken-1) ){ /* Either this is the cheapest token in the entire query, or it is ** part of a multi-token phrase. Either way, the entire doclist will ** (eventually) be loaded into memory. It may as well be now. */ @@ -4471,7 +4863,7 @@ static int fts3EvalStart(Fts3Cursor *pCsr){ } #endif - fts3EvalStartReaders(pCsr, pCsr->pExpr, 1, &rc); + fts3EvalStartReaders(pCsr, pCsr->pExpr, &rc); return rc; } @@ -4954,6 +5346,16 @@ static int fts3EvalNext(Fts3Cursor *pCsr){ pCsr->iPrevId = pExpr->iDocid; }while( pCsr->isEof==0 && fts3EvalTestDeferredAndNear(pCsr, &rc) ); } + + /* Check if the cursor is past the end of the docid range specified + ** by Fts3Cursor.iMinDocid/iMaxDocid. If so, set the EOF flag. */ + if( rc==SQLITE_OK && ( + (pCsr->bDesc==0 && pCsr->iPrevId>pCsr->iMaxDocid) + || (pCsr->bDesc!=0 && pCsr->iPrevIdiMinDocid) + )){ + pCsr->isEof = 1; + } + return rc; } @@ -4977,12 +5379,16 @@ static void fts3EvalRestart( if( pPhrase ){ fts3EvalInvalidatePoslist(pPhrase); if( pPhrase->bIncr ){ - assert( pPhrase->nToken==1 ); - assert( pPhrase->aToken[0].pSegcsr ); - sqlite3Fts3MsrIncrRestart(pPhrase->aToken[0].pSegcsr); + int i; + for(i=0; inToken; i++){ + Fts3PhraseToken *pToken = &pPhrase->aToken[i]; + assert( pToken->pDeferred==0 ); + if( pToken->pSegcsr ){ + sqlite3Fts3MsrIncrRestart(pToken->pSegcsr); + } + } *pRc = fts3EvalPhraseStart(pCsr, 0, pPhrase); } - pPhrase->doclist.pNextDocid = 0; pPhrase->doclist.iDocid = 0; } @@ -5027,7 +5433,7 @@ static void fts3EvalUpdateCounts(Fts3Expr *pExpr){ pExpr->aMI[iCol*3 + 2] += (iCnt>0); if( *p==0x00 ) break; p++; - p += sqlite3Fts3GetVarint32(p, &iCol); + p += fts3GetVarint32(p, &iCol); } } @@ -5231,15 +5637,23 @@ int sqlite3Fts3EvalPhrasePoslist( pIter = pPhrase->doclist.pList; if( iDocid!=pCsr->iPrevId || pExpr->bEof ){ int bDescDoclist = pTab->bDescIdx; /* For DOCID_CMP macro */ + int iMul; /* +1 if csr dir matches index dir, else -1 */ int bOr = 0; u8 bEof = 0; - Fts3Expr *p; + u8 bTreeEof = 0; + Fts3Expr *p; /* Used to iterate from pExpr to root */ + Fts3Expr *pNear; /* Most senior NEAR ancestor (or pExpr) */ /* Check if this phrase descends from an OR expression node. If not, ** return NULL. Otherwise, the entry that corresponds to docid - ** pCsr->iPrevId may lie earlier in the doclist buffer. */ + ** pCsr->iPrevId may lie earlier in the doclist buffer. Or, if the + ** tree that the node is part of has been marked as EOF, but the node + ** itself is not EOF, then it may point to an earlier entry. */ + pNear = pExpr; for(p=pExpr->pParent; p; p=p->pParent){ if( p->eType==FTSQUERY_OR ) bOr = 1; + if( p->eType==FTSQUERY_NEAR ) pNear = p; + if( p->bEof ) bTreeEof = 1; } if( bOr==0 ) return SQLITE_OK; @@ -5258,29 +5672,59 @@ int sqlite3Fts3EvalPhrasePoslist( assert( rc!=SQLITE_OK || pPhrase->bIncr==0 ); if( rc!=SQLITE_OK ) return rc; } - - if( pExpr->bEof ){ - pIter = 0; - iDocid = 0; + + iMul = ((pCsr->bDesc==bDescDoclist) ? 1 : -1); + while( bTreeEof==1 + && pNear->bEof==0 + && (DOCID_CMP(pNear->iDocid, pCsr->iPrevId) * iMul)<0 + ){ + int rc = SQLITE_OK; + fts3EvalNextRow(pCsr, pExpr, &rc); + if( rc!=SQLITE_OK ) return rc; + iDocid = pExpr->iDocid; + pIter = pPhrase->doclist.pList; } + bEof = (pPhrase->doclist.nAll==0); assert( bDescDoclist==0 || bDescDoclist==1 ); assert( pCsr->bDesc==0 || pCsr->bDesc==1 ); - if( pCsr->bDesc==bDescDoclist ){ - int dummy; - while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)>0 ) && bEof==0 ){ - sqlite3Fts3DoclistPrev( - bDescDoclist, pPhrase->doclist.aAll, pPhrase->doclist.nAll, - &pIter, &iDocid, &dummy, &bEof - ); - } - }else{ - while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)<0 ) && bEof==0 ){ - sqlite3Fts3DoclistNext( - bDescDoclist, pPhrase->doclist.aAll, pPhrase->doclist.nAll, - &pIter, &iDocid, &bEof - ); + if( bEof==0 ){ + if( pCsr->bDesc==bDescDoclist ){ + int dummy; + if( pNear->bEof ){ + /* This expression is already at EOF. So position it to point to the + ** last entry in the doclist at pPhrase->doclist.aAll[]. Variable + ** iDocid is already set for this entry, so all that is required is + ** to set pIter to point to the first byte of the last position-list + ** in the doclist. + ** + ** It would also be correct to set pIter and iDocid to zero. In + ** this case, the first call to sqltie3Fts4DoclistPrev() below + ** would also move the iterator to point to the last entry in the + ** doclist. However, this is expensive, as to do so it has to + ** iterate through the entire doclist from start to finish (since + ** it does not know the docid for the last entry). */ + pIter = &pPhrase->doclist.aAll[pPhrase->doclist.nAll-1]; + fts3ReversePoslist(pPhrase->doclist.aAll, &pIter); + } + while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)>0 ) && bEof==0 ){ + sqlite3Fts3DoclistPrev( + bDescDoclist, pPhrase->doclist.aAll, pPhrase->doclist.nAll, + &pIter, &iDocid, &dummy, &bEof + ); + } + }else{ + if( pNear->bEof ){ + pIter = 0; + iDocid = 0; + } + while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)<0 ) && bEof==0 ){ + sqlite3Fts3DoclistNext( + bDescDoclist, pPhrase->doclist.aAll, pPhrase->doclist.nAll, + &pIter, &iDocid, &bEof + ); + } } } @@ -5290,7 +5734,7 @@ int sqlite3Fts3EvalPhrasePoslist( if( *pIter==0x01 ){ pIter++; - pIter += sqlite3Fts3GetVarint32(pIter, &iThis); + pIter += fts3GetVarint32(pIter, &iThis); }else{ iThis = 0; } @@ -5298,7 +5742,7 @@ int sqlite3Fts3EvalPhrasePoslist( fts3ColumnlistCopy(0, &pIter); if( *pIter==0x00 ) return 0; pIter++; - pIter += sqlite3Fts3GetVarint32(pIter, &iThis); + pIter += fts3GetVarint32(pIter, &iThis); } *ppOut = ((iCol==iThis)?pIter:0); @@ -5339,7 +5783,10 @@ int sqlite3Fts3Corrupt(){ /* ** Initialize API pointer table, if required. */ -int sqlite3_extension_init( +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_fts3_init( sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index b19064c..b2827b7 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -32,13 +32,25 @@ /* If not building as part of the core, include sqlite3ext.h. */ #ifndef SQLITE_CORE # include "sqlite3ext.h" -extern const sqlite3_api_routines *sqlite3_api; +SQLITE_EXTENSION_INIT3 #endif #include "sqlite3.h" #include "fts3_tokenizer.h" #include "fts3_hash.h" +/* +** This constant determines the maximum depth of an FTS expression tree +** that the library will create and use. FTS uses recursion to perform +** various operations on the query tree, so the disadvantage of a large +** limit is that it may allow very large queries to use large amounts +** of stack space (perhaps causing a stack overflow). +*/ +#ifndef SQLITE_FTS3_MAX_EXPR_DEPTH +# define SQLITE_FTS3_MAX_EXPR_DEPTH 12 +#endif + + /* ** This constant controls how often segments are merged. Once there are ** FTS3_MERGE_COUNT segments of level N, they are merged into a single @@ -194,23 +206,24 @@ struct Fts3Table { const char *zName; /* virtual table name */ int nColumn; /* number of named columns in virtual table */ char **azColumn; /* column names. malloced */ + u8 *abNotindexed; /* True for 'notindexed' columns */ sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */ char *zContentTbl; /* content=xxx option, or NULL */ char *zLanguageid; /* languageid=xxx option, or NULL */ - u8 bAutoincrmerge; /* True if automerge=1 */ + int nAutoincrmerge; /* Value configured by 'automerge' */ u32 nLeafAdd; /* Number of leaf blocks added this trans */ /* Precompiled statements used by the implementation. Each of these ** statements is run and reset within a single virtual table API call. */ - sqlite3_stmt *aStmt[37]; + sqlite3_stmt *aStmt[40]; char *zReadExprlist; char *zWriteExprlist; int nNodeSize; /* Soft limit for node size */ u8 bFts4; /* True for FTS4, false for FTS3 */ - u8 bHasStat; /* True if %_stat table exists */ + u8 bHasStat; /* True if %_stat table exists (2==unknown) */ u8 bHasDocsize; /* True if %_docsize table exists */ u8 bDescIdx; /* True if doclists are in reverse order */ u8 bIgnoreSavepoint; /* True to ignore xSavepoint invocations */ @@ -254,6 +267,12 @@ struct Fts3Table { int inTransaction; /* True after xBegin but before xCommit/xRollback */ int mxSavepoint; /* Largest valid xSavepoint integer */ #endif + +#ifdef SQLITE_TEST + /* True to disable the incremental doclist optimization. This is controled + ** by special insert command 'test-no-incr-doclist'. */ + int bNoIncrDoclist; +#endif }; /* @@ -279,7 +298,8 @@ struct Fts3Cursor { int eEvalmode; /* An FTS3_EVAL_XX constant */ int nRowAvg; /* Average size of database rows, in pages */ sqlite3_int64 nDoc; /* Documents in table */ - + i64 iMinDocid; /* Minimum docid to return */ + i64 iMaxDocid; /* Maximum docid to return */ int isMatchinfoNeeded; /* True when aMatchinfo[] needs filling in */ u32 *aMatchinfo; /* Information about most recent match */ int nMatchinfo; /* Number of elements in aMatchinfo[] */ @@ -309,6 +329,15 @@ struct Fts3Cursor { #define FTS3_DOCID_SEARCH 1 /* Lookup by rowid on %_content table */ #define FTS3_FULLTEXT_SEARCH 2 /* Full-text index search */ +/* +** The lower 16-bits of the sqlite3_index_info.idxNum value set by +** the xBestIndex() method contains the Fts3Cursor.eSearch value described +** above. The upper 16-bits contain a combination of the following +** bits, used to describe extra constraints on full-text searches. +*/ +#define FTS3_HAVE_LANGID 0x00010000 /* languageid=? */ +#define FTS3_HAVE_DOCID_GE 0x00020000 /* docid>=? */ +#define FTS3_HAVE_DOCID_LE 0x00040000 /* docid<=? */ struct Fts3Doclist { char *aAll; /* Array containing doclist (or NULL) */ @@ -421,7 +450,6 @@ int sqlite3Fts3SegReaderPending( Fts3Table*,int,const char*,int,int,Fts3SegReader**); void sqlite3Fts3SegReaderFree(Fts3SegReader *); int sqlite3Fts3AllSegdirs(Fts3Table*, int, int, int, sqlite3_stmt **); -int sqlite3Fts3ReadLock(Fts3Table *); int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*, int*); int sqlite3Fts3SelectDoctotal(Fts3Table *, sqlite3_stmt **); @@ -496,6 +524,10 @@ struct Fts3MultiSegReader { int sqlite3Fts3Incrmerge(Fts3Table*,int,int); +#define fts3GetVarint32(p, piVal) ( \ + (*(u8*)(p)&0x80) ? sqlite3Fts3GetVarint32(p, piVal) : (*piVal=*(u8*)(p), 1) \ +) + /* fts3.c */ int sqlite3Fts3PutVarint(char *, sqlite3_int64); int sqlite3Fts3GetVarint(const char *, sqlite_int64 *); @@ -553,7 +585,7 @@ int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr); int sqlite3Fts3InitTok(sqlite3*, Fts3Hash *); /* fts3_unicode2.c (functions generated by parsing unicode text files) */ -#ifdef SQLITE_ENABLE_FTS4_UNICODE61 +#ifndef SQLITE_DISABLE_FTS3_UNICODE int sqlite3FtsUnicodeFold(int, int); int sqlite3FtsUnicodeIsalnum(int); int sqlite3FtsUnicodeIsdiacritic(int); diff --git a/ext/fts3/fts3_aux.c b/ext/fts3/fts3_aux.c index 9b582fc..c68b1a9 100644 --- a/ext/fts3/fts3_aux.c +++ b/ext/fts3/fts3_aux.c @@ -31,6 +31,7 @@ struct Fts3auxCursor { Fts3SegFilter filter; char *zStop; int nStop; /* Byte-length of string zStop */ + int iLangid; /* Language id to query */ int isEof; /* True if cursor is at EOF */ sqlite3_int64 iRowid; /* Current rowid */ @@ -45,7 +46,8 @@ struct Fts3auxCursor { /* ** Schema of the terms table. */ -#define FTS3_TERMS_SCHEMA "CREATE TABLE x(term, col, documents, occurrences)" +#define FTS3_AUX_SCHEMA \ + "CREATE TABLE x(term, col, documents, occurrences, languageid HIDDEN)" /* ** This function does all the work for both the xConnect and xCreate methods. @@ -92,7 +94,7 @@ static int fts3auxConnectMethod( } nFts3 = (int)strlen(zFts3); - rc = sqlite3_declare_vtab(db, FTS3_TERMS_SCHEMA); + rc = sqlite3_declare_vtab(db, FTS3_AUX_SCHEMA); if( rc!=SQLITE_OK ) return rc; nByte = sizeof(Fts3auxTable) + sizeof(Fts3Table) + nDb + nFts3 + 2; @@ -152,6 +154,8 @@ static int fts3auxBestIndexMethod( int iEq = -1; int iGe = -1; int iLe = -1; + int iLangid = -1; + int iNext = 1; /* Next free argvIndex value */ UNUSED_PARAMETER(pVTab); @@ -163,36 +167,48 @@ static int fts3auxBestIndexMethod( pInfo->orderByConsumed = 1; } - /* Search for equality and range constraints on the "term" column. */ + /* Search for equality and range constraints on the "term" column. + ** And equality constraints on the hidden "languageid" column. */ for(i=0; inConstraint; i++){ - if( pInfo->aConstraint[i].usable && pInfo->aConstraint[i].iColumn==0 ){ + if( pInfo->aConstraint[i].usable ){ int op = pInfo->aConstraint[i].op; - if( op==SQLITE_INDEX_CONSTRAINT_EQ ) iEq = i; - if( op==SQLITE_INDEX_CONSTRAINT_LT ) iLe = i; - if( op==SQLITE_INDEX_CONSTRAINT_LE ) iLe = i; - if( op==SQLITE_INDEX_CONSTRAINT_GT ) iGe = i; - if( op==SQLITE_INDEX_CONSTRAINT_GE ) iGe = i; + int iCol = pInfo->aConstraint[i].iColumn; + + if( iCol==0 ){ + if( op==SQLITE_INDEX_CONSTRAINT_EQ ) iEq = i; + if( op==SQLITE_INDEX_CONSTRAINT_LT ) iLe = i; + if( op==SQLITE_INDEX_CONSTRAINT_LE ) iLe = i; + if( op==SQLITE_INDEX_CONSTRAINT_GT ) iGe = i; + if( op==SQLITE_INDEX_CONSTRAINT_GE ) iGe = i; + } + if( iCol==4 ){ + if( op==SQLITE_INDEX_CONSTRAINT_EQ ) iLangid = i; + } } } if( iEq>=0 ){ pInfo->idxNum = FTS4AUX_EQ_CONSTRAINT; - pInfo->aConstraintUsage[iEq].argvIndex = 1; + pInfo->aConstraintUsage[iEq].argvIndex = iNext++; pInfo->estimatedCost = 5; }else{ pInfo->idxNum = 0; pInfo->estimatedCost = 20000; if( iGe>=0 ){ pInfo->idxNum += FTS4AUX_GE_CONSTRAINT; - pInfo->aConstraintUsage[iGe].argvIndex = 1; + pInfo->aConstraintUsage[iGe].argvIndex = iNext++; pInfo->estimatedCost /= 2; } if( iLe>=0 ){ pInfo->idxNum += FTS4AUX_LE_CONSTRAINT; - pInfo->aConstraintUsage[iLe].argvIndex = 1 + (iGe>=0); + pInfo->aConstraintUsage[iLe].argvIndex = iNext++; pInfo->estimatedCost /= 2; } } + if( iLangid>=0 ){ + pInfo->aConstraintUsage[iLangid].argvIndex = iNext++; + pInfo->estimatedCost--; + } return SQLITE_OK; } @@ -352,7 +368,14 @@ static int fts3auxFilterMethod( Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; Fts3Table *pFts3 = ((Fts3auxTable *)pCursor->pVtab)->pFts3Tab; int rc; - int isScan; + int isScan = 0; + int iLangVal = 0; /* Language id to query */ + + int iEq = -1; /* Index of term=? value in apVal */ + int iGe = -1; /* Index of term>=? value in apVal */ + int iLe = -1; /* Index of term<=? value in apVal */ + int iLangid = -1; /* Index of languageid=? value in apVal */ + int iNext = 0; UNUSED_PARAMETER(nVal); UNUSED_PARAMETER(idxStr); @@ -362,7 +385,21 @@ static int fts3auxFilterMethod( || idxNum==FTS4AUX_LE_CONSTRAINT || idxNum==FTS4AUX_GE_CONSTRAINT || idxNum==(FTS4AUX_LE_CONSTRAINT|FTS4AUX_GE_CONSTRAINT) ); - isScan = (idxNum!=FTS4AUX_EQ_CONSTRAINT); + + if( idxNum==FTS4AUX_EQ_CONSTRAINT ){ + iEq = iNext++; + }else{ + isScan = 1; + if( idxNum & FTS4AUX_GE_CONSTRAINT ){ + iGe = iNext++; + } + if( idxNum & FTS4AUX_LE_CONSTRAINT ){ + iLe = iNext++; + } + } + if( iNextfilter.zTerm); @@ -374,22 +411,35 @@ static int fts3auxFilterMethod( pCsr->filter.flags = FTS3_SEGMENT_REQUIRE_POS|FTS3_SEGMENT_IGNORE_EMPTY; if( isScan ) pCsr->filter.flags |= FTS3_SEGMENT_SCAN; - if( idxNum&(FTS4AUX_EQ_CONSTRAINT|FTS4AUX_GE_CONSTRAINT) ){ + if( iEq>=0 || iGe>=0 ){ const unsigned char *zStr = sqlite3_value_text(apVal[0]); + assert( (iEq==0 && iGe==-1) || (iEq==-1 && iGe==0) ); if( zStr ){ pCsr->filter.zTerm = sqlite3_mprintf("%s", zStr); pCsr->filter.nTerm = sqlite3_value_bytes(apVal[0]); if( pCsr->filter.zTerm==0 ) return SQLITE_NOMEM; } } - if( idxNum&FTS4AUX_LE_CONSTRAINT ){ - int iIdx = (idxNum&FTS4AUX_GE_CONSTRAINT) ? 1 : 0; - pCsr->zStop = sqlite3_mprintf("%s", sqlite3_value_text(apVal[iIdx])); - pCsr->nStop = sqlite3_value_bytes(apVal[iIdx]); + + if( iLe>=0 ){ + pCsr->zStop = sqlite3_mprintf("%s", sqlite3_value_text(apVal[iLe])); + pCsr->nStop = sqlite3_value_bytes(apVal[iLe]); if( pCsr->zStop==0 ) return SQLITE_NOMEM; } + + if( iLangid>=0 ){ + iLangVal = sqlite3_value_int(apVal[iLangid]); + + /* If the user specified a negative value for the languageid, use zero + ** instead. This works, as the "languageid=?" constraint will also + ** be tested by the VDBE layer. The test will always be false (since + ** this module will not return a row with a negative languageid), and + ** so the overall query will return zero rows. */ + if( iLangVal<0 ) iLangVal = 0; + } + pCsr->iLangid = iLangVal; - rc = sqlite3Fts3SegReaderCursor(pFts3, 0, 0, FTS3_SEGCURSOR_ALL, + rc = sqlite3Fts3SegReaderCursor(pFts3, iLangVal, 0, FTS3_SEGCURSOR_ALL, pCsr->filter.zTerm, pCsr->filter.nTerm, 0, isScan, &pCsr->csr ); if( rc==SQLITE_OK ){ @@ -413,24 +463,37 @@ static int fts3auxEofMethod(sqlite3_vtab_cursor *pCursor){ */ static int fts3auxColumnMethod( sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ - sqlite3_context *pContext, /* Context for sqlite3_result_xxx() calls */ + sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ int iCol /* Index of column to read value from */ ){ Fts3auxCursor *p = (Fts3auxCursor *)pCursor; assert( p->isEof==0 ); - if( iCol==0 ){ /* Column "term" */ - sqlite3_result_text(pContext, p->csr.zTerm, p->csr.nTerm, SQLITE_TRANSIENT); - }else if( iCol==1 ){ /* Column "col" */ - if( p->iCol ){ - sqlite3_result_int(pContext, p->iCol-1); - }else{ - sqlite3_result_text(pContext, "*", -1, SQLITE_STATIC); - } - }else if( iCol==2 ){ /* Column "documents" */ - sqlite3_result_int64(pContext, p->aStat[p->iCol].nDoc); - }else{ /* Column "occurrences" */ - sqlite3_result_int64(pContext, p->aStat[p->iCol].nOcc); + switch( iCol ){ + case 0: /* term */ + sqlite3_result_text(pCtx, p->csr.zTerm, p->csr.nTerm, SQLITE_TRANSIENT); + break; + + case 1: /* col */ + if( p->iCol ){ + sqlite3_result_int(pCtx, p->iCol-1); + }else{ + sqlite3_result_text(pCtx, "*", -1, SQLITE_STATIC); + } + break; + + case 2: /* documents */ + sqlite3_result_int64(pCtx, p->aStat[p->iCol].nDoc); + break; + + case 3: /* occurrences */ + sqlite3_result_int64(pCtx, p->aStat[p->iCol].nOcc); + break; + + default: /* languageid */ + assert( iCol==4 ); + sqlite3_result_int(pCtx, p->iLangid); + break; } return SQLITE_OK; diff --git a/ext/fts3/fts3_expr.c b/ext/fts3/fts3_expr.c index c046d7d..f5d28cb 100644 --- a/ext/fts3/fts3_expr.c +++ b/ext/fts3/fts3_expr.c @@ -155,6 +155,11 @@ int sqlite3Fts3OpenTokenizer( return rc; } +/* +** Function getNextNode(), which is called by fts3ExprParse(), may itself +** call fts3ExprParse(). So this forward declaration is required. +*/ +static int fts3ExprParse(ParseContext *, const char *, int, Fts3Expr **, int *); /* ** Extract the next token from buffer z (length n) using the tokenizer @@ -180,9 +185,16 @@ static int getNextToken( int rc; sqlite3_tokenizer_cursor *pCursor; Fts3Expr *pRet = 0; - int nConsumed = 0; + int i = 0; + + /* Set variable i to the maximum number of bytes of input to tokenize. */ + for(i=0; iiLangid, z, n, &pCursor); + *pnConsumed = i; + rc = sqlite3Fts3OpenTokenizer(pTokenizer, pParse->iLangid, z, i, &pCursor); if( rc==SQLITE_OK ){ const char *zToken; int nToken = 0, iStart = 0, iEnd = 0, iPosition = 0; @@ -223,13 +235,14 @@ static int getNextToken( } } - nConsumed = iEnd; + *pnConsumed = iEnd; + }else if( i && rc==SQLITE_DONE ){ + rc = SQLITE_OK; } pModule->xClose(pCursor); } - *pnConsumed = nConsumed; *ppExpr = pRet; return rc; } @@ -369,12 +382,6 @@ no_mem: return SQLITE_NOMEM; } -/* -** Function getNextNode(), which is called by fts3ExprParse(), may itself -** call fts3ExprParse(). So this forward declaration is required. -*/ -static int fts3ExprParse(ParseContext *, const char *, int, Fts3Expr **, int *); - /* ** The output variable *ppExpr is populated with an allocated Fts3Expr ** structure, or set to 0 if the end of the input buffer is reached. @@ -471,27 +478,6 @@ static int getNextNode( } } - /* Check for an open bracket. */ - if( sqlite3_fts3_enable_parentheses ){ - if( *zInput=='(' ){ - int nConsumed; - pParse->nNest++; - rc = fts3ExprParse(pParse, &zInput[1], nInput-1, ppExpr, &nConsumed); - if( rc==SQLITE_OK && !*ppExpr ){ - rc = SQLITE_DONE; - } - *pnConsumed = (int)((zInput - z) + 1 + nConsumed); - return rc; - } - - /* Check for a close bracket. */ - if( *zInput==')' ){ - pParse->nNest--; - *pnConsumed = (int)((zInput - z) + 1); - return SQLITE_DONE; - } - } - /* See if we are dealing with a quoted phrase. If this is the case, then ** search for the closing quote and pass the whole string to getNextString() ** for processing. This is easy to do, as fts3 has no syntax for escaping @@ -506,6 +492,21 @@ static int getNextNode( return getNextString(pParse, &zInput[1], ii-1, ppExpr); } + if( sqlite3_fts3_enable_parentheses ){ + if( *zInput=='(' ){ + int nConsumed = 0; + pParse->nNest++; + rc = fts3ExprParse(pParse, zInput+1, nInput-1, ppExpr, &nConsumed); + if( rc==SQLITE_OK && !*ppExpr ){ rc = SQLITE_DONE; } + *pnConsumed = (int)(zInput - z) + 1 + nConsumed; + return rc; + }else if( *zInput==')' ){ + pParse->nNest--; + *pnConsumed = (int)((zInput - z) + 1); + *ppExpr = 0; + return SQLITE_DONE; + } + } /* If control flows to this point, this must be a regular token, or ** the end of the input. Read a regular token using the sqlite3_tokenizer @@ -624,96 +625,100 @@ static int fts3ExprParse( while( rc==SQLITE_OK ){ Fts3Expr *p = 0; int nByte = 0; + rc = getNextNode(pParse, zIn, nIn, &p, &nByte); + assert( nByte>0 || (rc!=SQLITE_OK && p==0) ); if( rc==SQLITE_OK ){ - int isPhrase; - - if( !sqlite3_fts3_enable_parentheses - && p->eType==FTSQUERY_PHRASE && pParse->isNot - ){ - /* Create an implicit NOT operator. */ - Fts3Expr *pNot = fts3MallocZero(sizeof(Fts3Expr)); - if( !pNot ){ - sqlite3Fts3ExprFree(p); - rc = SQLITE_NOMEM; - goto exprparse_out; - } - pNot->eType = FTSQUERY_NOT; - pNot->pRight = p; - p->pParent = pNot; - if( pNotBranch ){ - pNot->pLeft = pNotBranch; - pNotBranch->pParent = pNot; - } - pNotBranch = pNot; - p = pPrev; - }else{ - int eType = p->eType; - isPhrase = (eType==FTSQUERY_PHRASE || p->pLeft); - - /* The isRequirePhrase variable is set to true if a phrase or - ** an expression contained in parenthesis is required. If a - ** binary operator (AND, OR, NOT or NEAR) is encounted when - ** isRequirePhrase is set, this is a syntax error. - */ - if( !isPhrase && isRequirePhrase ){ - sqlite3Fts3ExprFree(p); - rc = SQLITE_ERROR; - goto exprparse_out; - } - - if( isPhrase && !isRequirePhrase ){ - /* Insert an implicit AND operator. */ - Fts3Expr *pAnd; - assert( pRet && pPrev ); - pAnd = fts3MallocZero(sizeof(Fts3Expr)); - if( !pAnd ){ + if( p ){ + int isPhrase; + + if( !sqlite3_fts3_enable_parentheses + && p->eType==FTSQUERY_PHRASE && pParse->isNot + ){ + /* Create an implicit NOT operator. */ + Fts3Expr *pNot = fts3MallocZero(sizeof(Fts3Expr)); + if( !pNot ){ sqlite3Fts3ExprFree(p); rc = SQLITE_NOMEM; goto exprparse_out; } - pAnd->eType = FTSQUERY_AND; - insertBinaryOperator(&pRet, pPrev, pAnd); - pPrev = pAnd; - } + pNot->eType = FTSQUERY_NOT; + pNot->pRight = p; + p->pParent = pNot; + if( pNotBranch ){ + pNot->pLeft = pNotBranch; + pNotBranch->pParent = pNot; + } + pNotBranch = pNot; + p = pPrev; + }else{ + int eType = p->eType; + isPhrase = (eType==FTSQUERY_PHRASE || p->pLeft); + + /* The isRequirePhrase variable is set to true if a phrase or + ** an expression contained in parenthesis is required. If a + ** binary operator (AND, OR, NOT or NEAR) is encounted when + ** isRequirePhrase is set, this is a syntax error. + */ + if( !isPhrase && isRequirePhrase ){ + sqlite3Fts3ExprFree(p); + rc = SQLITE_ERROR; + goto exprparse_out; + } + + if( isPhrase && !isRequirePhrase ){ + /* Insert an implicit AND operator. */ + Fts3Expr *pAnd; + assert( pRet && pPrev ); + pAnd = fts3MallocZero(sizeof(Fts3Expr)); + if( !pAnd ){ + sqlite3Fts3ExprFree(p); + rc = SQLITE_NOMEM; + goto exprparse_out; + } + pAnd->eType = FTSQUERY_AND; + insertBinaryOperator(&pRet, pPrev, pAnd); + pPrev = pAnd; + } - /* This test catches attempts to make either operand of a NEAR - ** operator something other than a phrase. For example, either of - ** the following: - ** - ** (bracketed expression) NEAR phrase - ** phrase NEAR (bracketed expression) - ** - ** Return an error in either case. - */ - if( pPrev && ( + /* This test catches attempts to make either operand of a NEAR + ** operator something other than a phrase. For example, either of + ** the following: + ** + ** (bracketed expression) NEAR phrase + ** phrase NEAR (bracketed expression) + ** + ** Return an error in either case. + */ + if( pPrev && ( (eType==FTSQUERY_NEAR && !isPhrase && pPrev->eType!=FTSQUERY_PHRASE) || (eType!=FTSQUERY_PHRASE && isPhrase && pPrev->eType==FTSQUERY_NEAR) - )){ - sqlite3Fts3ExprFree(p); - rc = SQLITE_ERROR; - goto exprparse_out; - } - - if( isPhrase ){ - if( pRet ){ - assert( pPrev && pPrev->pLeft && pPrev->pRight==0 ); - pPrev->pRight = p; - p->pParent = pPrev; + )){ + sqlite3Fts3ExprFree(p); + rc = SQLITE_ERROR; + goto exprparse_out; + } + + if( isPhrase ){ + if( pRet ){ + assert( pPrev && pPrev->pLeft && pPrev->pRight==0 ); + pPrev->pRight = p; + p->pParent = pPrev; + }else{ + pRet = p; + } }else{ - pRet = p; + insertBinaryOperator(&pRet, pPrev, p); } - }else{ - insertBinaryOperator(&pRet, pPrev, p); + isRequirePhrase = !isPhrase; } - isRequirePhrase = !isPhrase; + pPrev = p; } assert( nByte>0 ); } assert( rc!=SQLITE_OK || (nByte>0 && nByte<=nIn) ); nIn -= nByte; zIn += nByte; - pPrev = p; } if( rc==SQLITE_DONE && pRet && isRequirePhrase ){ @@ -1000,17 +1005,16 @@ int sqlite3Fts3ExprParse( Fts3Expr **ppExpr, /* OUT: Parsed query structure */ char **pzErr /* OUT: Error message (sqlite3_malloc) */ ){ - static const int MAX_EXPR_DEPTH = 12; int rc = fts3ExprParseUnbalanced( pTokenizer, iLangid, azCol, bFts4, nCol, iDefaultCol, z, n, ppExpr ); /* Rebalance the expression. And check that its depth does not exceed - ** MAX_EXPR_DEPTH. */ + ** SQLITE_FTS3_MAX_EXPR_DEPTH. */ if( rc==SQLITE_OK && *ppExpr ){ - rc = fts3ExprBalance(ppExpr, MAX_EXPR_DEPTH); + rc = fts3ExprBalance(ppExpr, SQLITE_FTS3_MAX_EXPR_DEPTH); if( rc==SQLITE_OK ){ - rc = fts3ExprCheckDepth(*ppExpr, MAX_EXPR_DEPTH); + rc = fts3ExprCheckDepth(*ppExpr, SQLITE_FTS3_MAX_EXPR_DEPTH); } } @@ -1019,7 +1023,8 @@ int sqlite3Fts3ExprParse( *ppExpr = 0; if( rc==SQLITE_TOOBIG ){ *pzErr = sqlite3_mprintf( - "FTS expression tree is too large (maximum depth %d)", MAX_EXPR_DEPTH + "FTS expression tree is too large (maximum depth %d)", + SQLITE_FTS3_MAX_EXPR_DEPTH ); rc = SQLITE_ERROR; }else if( rc==SQLITE_ERROR ){ diff --git a/ext/fts3/fts3_hash.c b/ext/fts3/fts3_hash.c index 57c59b5..1a32a53 100644 --- a/ext/fts3/fts3_hash.c +++ b/ext/fts3/fts3_hash.c @@ -96,13 +96,13 @@ void sqlite3Fts3HashClear(Fts3Hash *pH){ */ static int fts3StrHash(const void *pKey, int nKey){ const char *z = (const char *)pKey; - int h = 0; + unsigned h = 0; if( nKey<=0 ) nKey = (int) strlen(z); while( nKey > 0 ){ h = (h<<3) ^ h ^ *z++; nKey--; } - return h & 0x7fffffff; + return (int)(h & 0x7fffffff); } static int fts3StrCompare(const void *pKey1, int n1, const void *pKey2, int n2){ if( n1!=n2 ) return 1; diff --git a/ext/fts3/fts3_porter.c b/ext/fts3/fts3_porter.c index 579745b..db175ac 100644 --- a/ext/fts3/fts3_porter.c +++ b/ext/fts3/fts3_porter.c @@ -403,12 +403,14 @@ static void porter_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){ /* Step 2 */ switch( z[1] ){ case 'a': - stem(&z, "lanoita", "ate", m_gt_0) || - stem(&z, "lanoit", "tion", m_gt_0); + if( !stem(&z, "lanoita", "ate", m_gt_0) ){ + stem(&z, "lanoit", "tion", m_gt_0); + } break; case 'c': - stem(&z, "icne", "ence", m_gt_0) || - stem(&z, "icna", "ance", m_gt_0); + if( !stem(&z, "icne", "ence", m_gt_0) ){ + stem(&z, "icna", "ance", m_gt_0); + } break; case 'e': stem(&z, "rezi", "ize", m_gt_0); @@ -417,43 +419,54 @@ static void porter_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){ stem(&z, "igol", "log", m_gt_0); break; case 'l': - stem(&z, "ilb", "ble", m_gt_0) || - stem(&z, "illa", "al", m_gt_0) || - stem(&z, "iltne", "ent", m_gt_0) || - stem(&z, "ile", "e", m_gt_0) || - stem(&z, "ilsuo", "ous", m_gt_0); + if( !stem(&z, "ilb", "ble", m_gt_0) + && !stem(&z, "illa", "al", m_gt_0) + && !stem(&z, "iltne", "ent", m_gt_0) + && !stem(&z, "ile", "e", m_gt_0) + ){ + stem(&z, "ilsuo", "ous", m_gt_0); + } break; case 'o': - stem(&z, "noitazi", "ize", m_gt_0) || - stem(&z, "noita", "ate", m_gt_0) || - stem(&z, "rota", "ate", m_gt_0); + if( !stem(&z, "noitazi", "ize", m_gt_0) + && !stem(&z, "noita", "ate", m_gt_0) + ){ + stem(&z, "rota", "ate", m_gt_0); + } break; case 's': - stem(&z, "msila", "al", m_gt_0) || - stem(&z, "ssenevi", "ive", m_gt_0) || - stem(&z, "ssenluf", "ful", m_gt_0) || - stem(&z, "ssensuo", "ous", m_gt_0); + if( !stem(&z, "msila", "al", m_gt_0) + && !stem(&z, "ssenevi", "ive", m_gt_0) + && !stem(&z, "ssenluf", "ful", m_gt_0) + ){ + stem(&z, "ssensuo", "ous", m_gt_0); + } break; case 't': - stem(&z, "itila", "al", m_gt_0) || - stem(&z, "itivi", "ive", m_gt_0) || - stem(&z, "itilib", "ble", m_gt_0); + if( !stem(&z, "itila", "al", m_gt_0) + && !stem(&z, "itivi", "ive", m_gt_0) + ){ + stem(&z, "itilib", "ble", m_gt_0); + } break; } /* Step 3 */ switch( z[0] ){ case 'e': - stem(&z, "etaci", "ic", m_gt_0) || - stem(&z, "evita", "", m_gt_0) || - stem(&z, "ezila", "al", m_gt_0); + if( !stem(&z, "etaci", "ic", m_gt_0) + && !stem(&z, "evita", "", m_gt_0) + ){ + stem(&z, "ezila", "al", m_gt_0); + } break; case 'i': stem(&z, "itici", "ic", m_gt_0); break; case 'l': - stem(&z, "laci", "ic", m_gt_0) || - stem(&z, "luf", "", m_gt_0); + if( !stem(&z, "laci", "ic", m_gt_0) ){ + stem(&z, "luf", "", m_gt_0); + } break; case 's': stem(&z, "ssen", "", m_gt_0); @@ -494,9 +507,11 @@ static void porter_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){ z += 3; } }else if( z[2]=='e' ){ - stem(&z, "tneme", "", m_gt_1) || - stem(&z, "tnem", "", m_gt_1) || - stem(&z, "tne", "", m_gt_1); + if( !stem(&z, "tneme", "", m_gt_1) + && !stem(&z, "tnem", "", m_gt_1) + ){ + stem(&z, "tne", "", m_gt_1); + } } } break; @@ -515,8 +530,9 @@ static void porter_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){ } break; case 't': - stem(&z, "eta", "", m_gt_1) || - stem(&z, "iti", "", m_gt_1); + if( !stem(&z, "eta", "", m_gt_1) ){ + stem(&z, "iti", "", m_gt_1); + } break; case 'u': if( z[0]=='s' && z[2]=='o' && m_gt_1(z+3) ){ diff --git a/ext/fts3/fts3_snippet.c b/ext/fts3/fts3_snippet.c index d54a787..aa8779f 100644 --- a/ext/fts3/fts3_snippet.c +++ b/ext/fts3/fts3_snippet.c @@ -128,7 +128,7 @@ struct StrBuffer { */ static void fts3GetDeltaPosition(char **pp, int *piPos){ int iVal; - *pp += sqlite3Fts3GetVarint32(*pp, &iVal); + *pp += fts3GetVarint32(*pp, &iVal); *piPos += (iVal-2); } @@ -504,6 +504,7 @@ static int fts3StringAppend( pStr->z = zNew; pStr->nAlloc = nAlloc; } + assert( pStr->z!=0 && (pStr->nAlloc >= pStr->n+nAppend+1) ); /* Append the data to the string buffer. */ memcpy(&pStr->z[pStr->n], zAppend, nAppend); diff --git a/ext/fts3/fts3_test.c b/ext/fts3/fts3_test.c index 75ec6bd..36dcc94 100644 --- a/ext/fts3/fts3_test.c +++ b/ext/fts3/fts3_test.c @@ -517,6 +517,51 @@ static int fts3_test_tokenizer_cmd( return TCL_OK; } +static int fts3_test_varint_cmd( + ClientData clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ +#ifdef SQLITE_ENABLE_FTS3 + char aBuf[24]; + int rc; + Tcl_WideInt w, w2; + int nByte, nByte2; + + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "INTEGER"); + return TCL_ERROR; + } + + rc = Tcl_GetWideIntFromObj(interp, objv[1], &w); + if( rc!=TCL_OK ) return rc; + + nByte = sqlite3Fts3PutVarint(aBuf, w); + nByte2 = sqlite3Fts3GetVarint(aBuf, &w2); + if( w!=w2 || nByte!=nByte2 ){ + char *zErr = sqlite3_mprintf("error testing %lld", w); + Tcl_ResetResult(interp); + Tcl_AppendResult(interp, zErr, 0); + return TCL_ERROR; + } + + if( w<=2147483647 && w>=0 ){ + int i; + nByte2 = fts3GetVarint32(aBuf, &i); + if( (int)w!=i || nByte!=nByte2 ){ + char *zErr = sqlite3_mprintf("error testing %lld (32-bit)", w); + Tcl_ResetResult(interp); + Tcl_AppendResult(interp, zErr, 0); + return TCL_ERROR; + } + } + +#endif + UNUSED_PARAMETER(clientData); + return TCL_OK; +} + /* ** End of tokenizer code. **************************************************************************/ @@ -529,6 +574,10 @@ int Sqlitetestfts3_Init(Tcl_Interp *interp){ Tcl_CreateObjCommand( interp, "fts3_test_tokenizer", fts3_test_tokenizer_cmd, 0, 0 ); + + Tcl_CreateObjCommand( + interp, "fts3_test_varint", fts3_test_varint_cmd, 0, 0 + ); return TCL_OK; } #endif /* SQLITE_ENABLE_FTS3 || SQLITE_ENABLE_FTS4 */ diff --git a/ext/fts3/fts3_unicode.c b/ext/fts3/fts3_unicode.c index 188358e..94fc27b 100644 --- a/ext/fts3/fts3_unicode.c +++ b/ext/fts3/fts3_unicode.c @@ -13,7 +13,7 @@ ** Implementation of the "unicode" full-text-search tokenizer. */ -#ifdef SQLITE_ENABLE_FTS4_UNICODE61 +#ifndef SQLITE_DISABLE_FTS3_UNICODE #include "fts3Int.h" #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) @@ -231,7 +231,7 @@ static int unicodeCreate( for(i=0; rc==SQLITE_OK && ibRemoveDiacritic = 1; @@ -318,7 +318,7 @@ static int unicodeNext( ){ unicode_cursor *pCsr = (unicode_cursor *)pC; unicode_tokenizer *p = ((unicode_tokenizer *)pCsr->base.pTokenizer); - int iCode; + int iCode = 0; char *zOut; const unsigned char *z = &pCsr->aInput[pCsr->iOff]; const unsigned char *zStart = z; @@ -363,11 +363,11 @@ static int unicodeNext( ); /* Set the output variables and return. */ - pCsr->iOff = (z - pCsr->aInput); + pCsr->iOff = (int)(z - pCsr->aInput); *paToken = pCsr->zToken; - *pnToken = zOut - pCsr->zToken; - *piStart = (zStart - pCsr->aInput); - *piEnd = (zEnd - pCsr->aInput); + *pnToken = (int)(zOut - pCsr->zToken); + *piStart = (int)(zStart - pCsr->aInput); + *piEnd = (int)(zEnd - pCsr->aInput); *piPos = pCsr->iToken++; return SQLITE_OK; } @@ -390,4 +390,4 @@ void sqlite3Fts3UnicodeTokenizer(sqlite3_tokenizer_module const **ppModule){ } #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ -#endif /* ifndef SQLITE_ENABLE_FTS4_UNICODE61 */ +#endif /* ifndef SQLITE_DISABLE_FTS3_UNICODE */ diff --git a/ext/fts3/fts3_unicode2.c b/ext/fts3/fts3_unicode2.c index 3c24569..20b7a25 100644 --- a/ext/fts3/fts3_unicode2.c +++ b/ext/fts3/fts3_unicode2.c @@ -15,7 +15,7 @@ ** DO NOT EDIT THIS MACHINE GENERATED FILE. */ -#if defined(SQLITE_ENABLE_FTS4_UNICODE61) +#ifndef SQLITE_DISABLE_FTS3_UNICODE #if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) #include @@ -39,7 +39,7 @@ int sqlite3FtsUnicodeIsalnum(int c){ ** C. It is not possible to represent a range larger than 1023 codepoints ** using this format. */ - const static unsigned int aEntry[] = { + static const unsigned int aEntry[] = { 0x00000030, 0x0000E807, 0x00016C06, 0x0001EC2F, 0x0002AC07, 0x0002D001, 0x0002D803, 0x0002EC01, 0x0002FC01, 0x00035C01, 0x0003DC01, 0x000B0804, 0x000B480E, 0x000B9407, 0x000BB401, @@ -101,28 +101,27 @@ int sqlite3FtsUnicodeIsalnum(int c){ 0x02A97004, 0x02A9DC03, 0x02A9EC01, 0x02AAC001, 0x02AAC803, 0x02AADC02, 0x02AAF802, 0x02AB0401, 0x02AB7802, 0x02ABAC07, 0x02ABD402, 0x02AF8C0B, 0x03600001, 0x036DFC02, 0x036FFC02, - 0x037FFC02, 0x03E3FC01, 0x03EC7801, 0x03ECA401, 0x03EEC810, - 0x03F4F802, 0x03F7F002, 0x03F8001A, 0x03F88007, 0x03F8C023, - 0x03F95013, 0x03F9A004, 0x03FBFC01, 0x03FC040F, 0x03FC6807, - 0x03FCEC06, 0x03FD6C0B, 0x03FF8007, 0x03FFA007, 0x03FFE405, - 0x04040003, 0x0404DC09, 0x0405E411, 0x0406400C, 0x0407402E, - 0x040E7C01, 0x040F4001, 0x04215C01, 0x04247C01, 0x0424FC01, - 0x04280403, 0x04281402, 0x04283004, 0x0428E003, 0x0428FC01, - 0x04294009, 0x0429FC01, 0x042CE407, 0x04400003, 0x0440E016, - 0x04420003, 0x0442C012, 0x04440003, 0x04449C0E, 0x04450004, - 0x04460003, 0x0446CC0E, 0x04471404, 0x045AAC0D, 0x0491C004, - 0x05BD442E, 0x05BE3C04, 0x074000F6, 0x07440027, 0x0744A4B5, - 0x07480046, 0x074C0057, 0x075B0401, 0x075B6C01, 0x075BEC01, - 0x075C5401, 0x075CD401, 0x075D3C01, 0x075DBC01, 0x075E2401, - 0x075EA401, 0x075F0C01, 0x07BBC002, 0x07C0002C, 0x07C0C064, - 0x07C2800F, 0x07C2C40E, 0x07C3040F, 0x07C3440F, 0x07C4401F, - 0x07C4C03C, 0x07C5C02B, 0x07C7981D, 0x07C8402B, 0x07C90009, - 0x07C94002, 0x07CC0021, 0x07CCC006, 0x07CCDC46, 0x07CE0014, - 0x07CE8025, 0x07CF1805, 0x07CF8011, 0x07D0003F, 0x07D10001, - 0x07D108B6, 0x07D3E404, 0x07D4003E, 0x07D50004, 0x07D54018, - 0x07D7EC46, 0x07D9140B, 0x07DA0046, 0x07DC0074, 0x38000401, - 0x38008060, 0x380400F0, 0x3C000001, 0x3FFFF401, 0x40000001, - 0x43FFF401, + 0x037FFC01, 0x03EC7801, 0x03ECA401, 0x03EEC810, 0x03F4F802, + 0x03F7F002, 0x03F8001A, 0x03F88007, 0x03F8C023, 0x03F95013, + 0x03F9A004, 0x03FBFC01, 0x03FC040F, 0x03FC6807, 0x03FCEC06, + 0x03FD6C0B, 0x03FF8007, 0x03FFA007, 0x03FFE405, 0x04040003, + 0x0404DC09, 0x0405E411, 0x0406400C, 0x0407402E, 0x040E7C01, + 0x040F4001, 0x04215C01, 0x04247C01, 0x0424FC01, 0x04280403, + 0x04281402, 0x04283004, 0x0428E003, 0x0428FC01, 0x04294009, + 0x0429FC01, 0x042CE407, 0x04400003, 0x0440E016, 0x04420003, + 0x0442C012, 0x04440003, 0x04449C0E, 0x04450004, 0x04460003, + 0x0446CC0E, 0x04471404, 0x045AAC0D, 0x0491C004, 0x05BD442E, + 0x05BE3C04, 0x074000F6, 0x07440027, 0x0744A4B5, 0x07480046, + 0x074C0057, 0x075B0401, 0x075B6C01, 0x075BEC01, 0x075C5401, + 0x075CD401, 0x075D3C01, 0x075DBC01, 0x075E2401, 0x075EA401, + 0x075F0C01, 0x07BBC002, 0x07C0002C, 0x07C0C064, 0x07C2800F, + 0x07C2C40E, 0x07C3040F, 0x07C3440F, 0x07C4401F, 0x07C4C03C, + 0x07C5C02B, 0x07C7981D, 0x07C8402B, 0x07C90009, 0x07C94002, + 0x07CC0021, 0x07CCC006, 0x07CCDC46, 0x07CE0014, 0x07CE8025, + 0x07CF1805, 0x07CF8011, 0x07D0003F, 0x07D10001, 0x07D108B6, + 0x07D3E404, 0x07D4003E, 0x07D50004, 0x07D54018, 0x07D7EC46, + 0x07D9140B, 0x07DA0046, 0x07DC0074, 0x38000401, 0x38008060, + 0x380400F0, }; static const unsigned int aAscii[4] = { 0xFFFFFFFF, 0xFC00FFFF, 0xF8000001, 0xF8000001, @@ -132,7 +131,7 @@ int sqlite3FtsUnicodeIsalnum(int c){ return ( (aAscii[c >> 5] & (1 << (c & 0x001F)))==0 ); }else if( c<(1<<22) ){ unsigned int key = (((unsigned int)c)<<10) | 0x000003FF; - int iRes; + int iRes = 0; int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1; int iLo = 0; while( iHi>=iLo ){ @@ -203,7 +202,7 @@ static int remove_diacritic(int c){ } assert( key>=aDia[iRes] ); return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]); -}; +} /* @@ -363,4 +362,4 @@ int sqlite3FtsUnicodeFold(int c, int bRemoveDiacritic){ return ret; } #endif /* defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) */ -#endif /* !defined(SQLITE_ENABLE_FTS4_UNICODE61) */ +#endif /* !defined(SQLITE_DISABLE_FTS3_UNICODE) */ diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 269d1dd..0da08c6 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -193,6 +193,7 @@ struct SegmentWriter { int nSize; /* Size of allocation at aData */ int nData; /* Bytes of data in aData */ char *aData; /* Pointer to block from malloc() */ + i64 nLeafData; /* Number of bytes of leaf data written */ }; /* @@ -268,6 +269,10 @@ struct SegmentNode { #define SQL_SELECT_INDEXES 35 #define SQL_SELECT_MXLEVEL 36 +#define SQL_SELECT_LEVEL_RANGE2 37 +#define SQL_UPDATE_LEVEL_IDX 38 +#define SQL_UPDATE_LEVEL 39 + /* ** This function is used to obtain an SQLite prepared statement handle ** for the statement identified by the second argument. If successful, @@ -369,7 +374,18 @@ static int fts3SqlStmt( /* SQL_SELECT_MXLEVEL ** Return the largest relative level in the FTS index or indexes. */ -/* 36 */ "SELECT max( level %% 1024 ) FROM %Q.'%q_segdir'" +/* 36 */ "SELECT max( level %% 1024 ) FROM %Q.'%q_segdir'", + + /* Return segments in order from oldest to newest.*/ +/* 37 */ "SELECT level, idx, end_block " + "FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ? " + "ORDER BY level DESC, idx ASC", + + /* Update statements used while promoting segments */ +/* 38 */ "UPDATE OR FAIL %Q.'%q_segdir' SET level=-1,idx=? " + "WHERE level=? AND idx=?", +/* 39 */ "UPDATE OR FAIL %Q.'%q_segdir' SET level=? WHERE level=-1" + }; int rc = SQLITE_OK; sqlite3_stmt *pStmt; @@ -489,37 +505,30 @@ static void fts3SqlExec( /* -** This function ensures that the caller has obtained a shared-cache -** table-lock on the %_content table. This is required before reading -** data from the fts3 table. If this lock is not acquired first, then -** the caller may end up holding read-locks on the %_segments and %_segdir -** tables, but no read-lock on the %_content table. If this happens -** a second connection will be able to write to the fts3 table, but -** attempting to commit those writes might return SQLITE_LOCKED or -** SQLITE_LOCKED_SHAREDCACHE (because the commit attempts to obtain -** write-locks on the %_segments and %_segdir ** tables). -** -** We try to avoid this because if FTS3 returns any error when committing -** a transaction, the whole transaction will be rolled back. And this is -** not what users expect when they get SQLITE_LOCKED_SHAREDCACHE. It can -** still happen if the user reads data directly from the %_segments or -** %_segdir tables instead of going through FTS3 though. +** This function ensures that the caller has obtained an exclusive +** shared-cache table-lock on the %_segdir table. This is required before +** writing data to the fts3 table. If this lock is not acquired first, then +** the caller may end up attempting to take this lock as part of committing +** a transaction, causing SQLite to return SQLITE_LOCKED or +** LOCKED_SHAREDCACHEto a COMMIT command. ** -** This reasoning does not apply to a content=xxx table. +** It is best to avoid this because if FTS3 returns any error when +** committing a transaction, the whole transaction will be rolled back. +** And this is not what users expect when they get SQLITE_LOCKED_SHAREDCACHE. +** It can still happen if the user locks the underlying tables directly +** instead of accessing them via FTS. */ -int sqlite3Fts3ReadLock(Fts3Table *p){ - int rc; /* Return code */ - sqlite3_stmt *pStmt; /* Statement used to obtain lock */ - - if( p->zContentTbl==0 ){ - rc = fts3SqlStmt(p, SQL_SELECT_CONTENT_BY_ROWID, &pStmt, 0); +static int fts3Writelock(Fts3Table *p){ + int rc = SQLITE_OK; + + if( p->nPendingData==0 ){ + sqlite3_stmt *pStmt; + rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_LEVEL, &pStmt, 0); if( rc==SQLITE_OK ){ sqlite3_bind_null(pStmt, 1); sqlite3_step(pStmt); rc = sqlite3_reset(pStmt); } - }else{ - rc = SQLITE_OK; } return rc; @@ -907,12 +916,15 @@ static int fts3InsertTerms( ){ int i; /* Iterator variable */ for(i=2; inColumn+2; i++){ - const char *zText = (const char *)sqlite3_value_text(apVal[i]); - int rc = fts3PendingTermsAdd(p, iLangid, zText, i-2, &aSz[i-2]); - if( rc!=SQLITE_OK ){ - return rc; + int iCol = i-2; + if( p->abNotindexed[iCol]==0 ){ + const char *zText = (const char *)sqlite3_value_text(apVal[i]); + int rc = fts3PendingTermsAdd(p, iLangid, zText, iCol, &aSz[iCol]); + if( rc!=SQLITE_OK ){ + return rc; + } + aSz[p->nColumn] += sqlite3_value_bytes(apVal[i]); } - aSz[p->nColumn] += sqlite3_value_bytes(apVal[i]); } return SQLITE_OK; } @@ -1059,9 +1071,12 @@ static void fts3DeleteTerms( int iLangid = langidFromSelect(p, pSelect); rc = fts3PendingTermsDocid(p, iLangid, sqlite3_column_int64(pSelect, 0)); for(i=1; rc==SQLITE_OK && i<=p->nColumn; i++){ - const char *zText = (const char *)sqlite3_column_text(pSelect, i); - rc = fts3PendingTermsAdd(p, iLangid, zText, -1, &aSz[i-1]); - aSz[p->nColumn] += sqlite3_column_bytes(pSelect, i); + int iCol = i-1; + if( p->abNotindexed[iCol]==0 ){ + const char *zText = (const char *)sqlite3_column_text(pSelect, i); + rc = fts3PendingTermsAdd(p, iLangid, zText, -1, &aSz[iCol]); + aSz[p->nColumn] += sqlite3_column_bytes(pSelect, i); + } } if( rc!=SQLITE_OK ){ sqlite3_reset(pSelect); @@ -1345,8 +1360,8 @@ static int fts3SegReaderNext( /* Because of the FTS3_NODE_PADDING bytes of padding, the following is ** safe (no risk of overread) even if the node data is corrupted. */ - pNext += sqlite3Fts3GetVarint32(pNext, &nPrefix); - pNext += sqlite3Fts3GetVarint32(pNext, &nSuffix); + pNext += fts3GetVarint32(pNext, &nPrefix); + pNext += fts3GetVarint32(pNext, &nSuffix); if( nPrefix<0 || nSuffix<=0 || &pNext[nSuffix]>&pReader->aNode[pReader->nNode] ){ @@ -1369,7 +1384,7 @@ static int fts3SegReaderNext( memcpy(&pReader->zTerm[nPrefix], pNext, nSuffix); pReader->nTerm = nPrefix+nSuffix; pNext += nSuffix; - pNext += sqlite3Fts3GetVarint32(pNext, &pReader->nDoclist); + pNext += fts3GetVarint32(pNext, &pReader->nDoclist); pReader->aDoclist = pNext; pReader->pOffsetList = 0; @@ -1462,7 +1477,7 @@ static int fts3SegReaderNextDocid( /* The following line of code (and the "p++" below the while() loop) is ** normally all that is required to move pointer p to the desired ** position. The exception is if this node is being loaded from disk - ** incrementally and pointer "p" now points to the first byte passed + ** incrementally and pointer "p" now points to the first byte past ** the populated part of pReader->aNode[]. */ while( *p | c ) c = *p++ & 0x80; @@ -1911,6 +1926,7 @@ static int fts3WriteSegdir( sqlite3_int64 iStartBlock, /* Value for "start_block" field */ sqlite3_int64 iLeafEndBlock, /* Value for "leaves_end_block" field */ sqlite3_int64 iEndBlock, /* Value for "end_block" field */ + sqlite3_int64 nLeafData, /* Bytes of leaf data in segment */ char *zRoot, /* Blob value for "root" field */ int nRoot /* Number of bytes in buffer zRoot */ ){ @@ -1921,7 +1937,13 @@ static int fts3WriteSegdir( sqlite3_bind_int(pStmt, 2, iIdx); sqlite3_bind_int64(pStmt, 3, iStartBlock); sqlite3_bind_int64(pStmt, 4, iLeafEndBlock); - sqlite3_bind_int64(pStmt, 5, iEndBlock); + if( nLeafData==0 ){ + sqlite3_bind_int64(pStmt, 5, iEndBlock); + }else{ + char *zEnd = sqlite3_mprintf("%lld %lld", iEndBlock, nLeafData); + if( !zEnd ) return SQLITE_NOMEM; + sqlite3_bind_text(pStmt, 5, zEnd, -1, sqlite3_free); + } sqlite3_bind_blob(pStmt, 6, zRoot, nRoot, SQLITE_STATIC); sqlite3_step(pStmt); rc = sqlite3_reset(pStmt); @@ -2247,6 +2269,9 @@ static int fts3SegWriterAdd( nDoclist; /* Doclist data */ } + /* Increase the total number of bytes written to account for the new entry. */ + pWriter->nLeafData += nReq; + /* If the buffer currently allocated is too small for this entry, realloc ** the buffer to make it large enough. */ @@ -2318,13 +2343,13 @@ static int fts3SegWriterFlush( pWriter->iFirst, pWriter->iFree, &iLast, &zRoot, &nRoot); } if( rc==SQLITE_OK ){ - rc = fts3WriteSegdir( - p, iLevel, iIdx, pWriter->iFirst, iLastLeaf, iLast, zRoot, nRoot); + rc = fts3WriteSegdir(p, iLevel, iIdx, + pWriter->iFirst, iLastLeaf, iLast, pWriter->nLeafData, zRoot, nRoot); } }else{ /* The entire tree fits on the root node. Write it to the segdir table. */ - rc = fts3WriteSegdir( - p, iLevel, iIdx, 0, 0, 0, pWriter->aData, pWriter->nData); + rc = fts3WriteSegdir(p, iLevel, iIdx, + 0, 0, 0, pWriter->nLeafData, pWriter->aData, pWriter->nData); } p->nLeafAdd++; return rc; @@ -2408,6 +2433,37 @@ static int fts3SegmentMaxLevel( return sqlite3_reset(pStmt); } +/* +** iAbsLevel is an absolute level that may be assumed to exist within +** the database. This function checks if it is the largest level number +** within its index. Assuming no error occurs, *pbMax is set to 1 if +** iAbsLevel is indeed the largest level, or 0 otherwise, and SQLITE_OK +** is returned. If an error occurs, an error code is returned and the +** final value of *pbMax is undefined. +*/ +static int fts3SegmentIsMaxLevel(Fts3Table *p, i64 iAbsLevel, int *pbMax){ + + /* Set pStmt to the compiled version of: + ** + ** SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ? + ** + ** (1024 is actually the value of macro FTS3_SEGDIR_PREFIXLEVEL_STR). + */ + sqlite3_stmt *pStmt; + int rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_MAX_LEVEL, &pStmt, 0); + if( rc!=SQLITE_OK ) return rc; + sqlite3_bind_int64(pStmt, 1, iAbsLevel+1); + sqlite3_bind_int64(pStmt, 2, + ((iAbsLevel/FTS3_SEGDIR_MAXLEVEL)+1) * FTS3_SEGDIR_MAXLEVEL + ); + + *pbMax = 0; + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + *pbMax = sqlite3_column_type(pStmt, 0)==SQLITE_NULL; + } + return sqlite3_reset(pStmt); +} + /* ** Delete all entries in the %_segments table associated with the segment ** opened with seg-reader pSeg. This function does not affect the contents @@ -2530,7 +2586,7 @@ static void fts3ColumnFilter( break; } p = &pList[1]; - p += sqlite3Fts3GetVarint32(p, &iCurrent); + p += fts3GetVarint32(p, &iCurrent); } if( bZero && &pList[nList]!=pEnd ){ @@ -2849,8 +2905,8 @@ int sqlite3Fts3SegReaderStep( fts3SegReaderSort(apSegment, nMerge, nMerge, xCmp); while( apSegment[0]->pOffsetList ){ int j; /* Number of segments that share a docid */ - char *pList; - int nList; + char *pList = 0; + int nList = 0; int nByte; sqlite3_int64 iDocid = apSegment[0]->iDocid; fts3SegReaderNextDocid(p, apSegment[0], &pList, &nList); @@ -2943,6 +2999,140 @@ void sqlite3Fts3SegReaderFinish( } } +/* +** Decode the "end_block" field, selected by column iCol of the SELECT +** statement passed as the first argument. +** +** The "end_block" field may contain either an integer, or a text field +** containing the text representation of two non-negative integers separated +** by one or more space (0x20) characters. In the first case, set *piEndBlock +** to the integer value and *pnByte to zero before returning. In the second, +** set *piEndBlock to the first value and *pnByte to the second. +*/ +static void fts3ReadEndBlockField( + sqlite3_stmt *pStmt, + int iCol, + i64 *piEndBlock, + i64 *pnByte +){ + const unsigned char *zText = sqlite3_column_text(pStmt, iCol); + if( zText ){ + int i; + int iMul = 1; + i64 iVal = 0; + for(i=0; zText[i]>='0' && zText[i]<='9'; i++){ + iVal = iVal*10 + (zText[i] - '0'); + } + *piEndBlock = iVal; + while( zText[i]==' ' ) i++; + iVal = 0; + if( zText[i]=='-' ){ + i++; + iMul = -1; + } + for(/* no-op */; zText[i]>='0' && zText[i]<='9'; i++){ + iVal = iVal*10 + (zText[i] - '0'); + } + *pnByte = (iVal * (i64)iMul); + } +} + + +/* +** A segment of size nByte bytes has just been written to absolute level +** iAbsLevel. Promote any segments that should be promoted as a result. +*/ +static int fts3PromoteSegments( + Fts3Table *p, /* FTS table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level just updated */ + sqlite3_int64 nByte /* Size of new segment at iAbsLevel */ +){ + int rc = SQLITE_OK; + sqlite3_stmt *pRange; + + rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE2, &pRange, 0); + + if( rc==SQLITE_OK ){ + int bOk = 0; + i64 iLast = (iAbsLevel/FTS3_SEGDIR_MAXLEVEL + 1) * FTS3_SEGDIR_MAXLEVEL - 1; + i64 nLimit = (nByte*3)/2; + + /* Loop through all entries in the %_segdir table corresponding to + ** segments in this index on levels greater than iAbsLevel. If there is + ** at least one such segment, and it is possible to determine that all + ** such segments are smaller than nLimit bytes in size, they will be + ** promoted to level iAbsLevel. */ + sqlite3_bind_int64(pRange, 1, iAbsLevel+1); + sqlite3_bind_int64(pRange, 2, iLast); + while( SQLITE_ROW==sqlite3_step(pRange) ){ + i64 nSize = 0, dummy; + fts3ReadEndBlockField(pRange, 2, &dummy, &nSize); + if( nSize<=0 || nSize>nLimit ){ + /* If nSize==0, then the %_segdir.end_block field does not not + ** contain a size value. This happens if it was written by an + ** old version of FTS. In this case it is not possible to determine + ** the size of the segment, and so segment promotion does not + ** take place. */ + bOk = 0; + break; + } + bOk = 1; + } + rc = sqlite3_reset(pRange); + + if( bOk ){ + int iIdx = 0; + sqlite3_stmt *pUpdate1; + sqlite3_stmt *pUpdate2; + + if( rc==SQLITE_OK ){ + rc = fts3SqlStmt(p, SQL_UPDATE_LEVEL_IDX, &pUpdate1, 0); + } + if( rc==SQLITE_OK ){ + rc = fts3SqlStmt(p, SQL_UPDATE_LEVEL, &pUpdate2, 0); + } + + if( rc==SQLITE_OK ){ + + /* Loop through all %_segdir entries for segments in this index with + ** levels equal to or greater than iAbsLevel. As each entry is visited, + ** updated it to set (level = -1) and (idx = N), where N is 0 for the + ** oldest segment in the range, 1 for the next oldest, and so on. + ** + ** In other words, move all segments being promoted to level -1, + ** setting the "idx" fields as appropriate to keep them in the same + ** order. The contents of level -1 (which is never used, except + ** transiently here), will be moved back to level iAbsLevel below. */ + sqlite3_bind_int64(pRange, 1, iAbsLevel); + while( SQLITE_ROW==sqlite3_step(pRange) ){ + sqlite3_bind_int(pUpdate1, 1, iIdx++); + sqlite3_bind_int(pUpdate1, 2, sqlite3_column_int(pRange, 0)); + sqlite3_bind_int(pUpdate1, 3, sqlite3_column_int(pRange, 1)); + sqlite3_step(pUpdate1); + rc = sqlite3_reset(pUpdate1); + if( rc!=SQLITE_OK ){ + sqlite3_reset(pRange); + break; + } + } + } + if( rc==SQLITE_OK ){ + rc = sqlite3_reset(pRange); + } + + /* Move level -1 to level iAbsLevel */ + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pUpdate2, 1, iAbsLevel); + sqlite3_step(pUpdate2); + rc = sqlite3_reset(pUpdate2); + } + } + } + + + return rc; +} + /* ** Merge all level iLevel segments in the database into a single ** iLevel+1 segment. Or, if iLevel<0, merge all segments into a @@ -2967,6 +3157,7 @@ static int fts3SegmentMerge( Fts3SegFilter filter; /* Segment term filter condition */ Fts3MultiSegReader csr; /* Cursor to iterate through level(s) */ int bIgnoreEmpty = 0; /* True to ignore empty segments */ + i64 iMaxLevel = 0; /* Max level number for this index/langid */ assert( iLevel==FTS3_SEGCURSOR_ALL || iLevel==FTS3_SEGCURSOR_PENDING @@ -2978,6 +3169,11 @@ static int fts3SegmentMerge( rc = sqlite3Fts3SegReaderCursor(p, iLangid, iIndex, iLevel, 0, 0, 1, 0, &csr); if( rc!=SQLITE_OK || csr.nSegment==0 ) goto finished; + if( iLevel!=FTS3_SEGCURSOR_PENDING ){ + rc = fts3SegmentMaxLevel(p, iLangid, iIndex, &iMaxLevel); + if( rc!=SQLITE_OK ) goto finished; + } + if( iLevel==FTS3_SEGCURSOR_ALL ){ /* This call is to merge all segments in the database to a single ** segment. The level of the new segment is equal to the numerically @@ -2987,21 +3183,21 @@ static int fts3SegmentMerge( rc = SQLITE_DONE; goto finished; } - rc = fts3SegmentMaxLevel(p, iLangid, iIndex, &iNewLevel); + iNewLevel = iMaxLevel; bIgnoreEmpty = 1; - }else if( iLevel==FTS3_SEGCURSOR_PENDING ){ - iNewLevel = getAbsoluteLevel(p, iLangid, iIndex, 0); - rc = fts3AllocateSegdirIdx(p, iLangid, iIndex, 0, &iIdx); }else{ /* This call is to merge all segments at level iLevel. find the next ** available segment index at level iLevel+1. The call to ** fts3AllocateSegdirIdx() will merge the segments at level iLevel+1 to ** a single iLevel+2 segment if necessary. */ - rc = fts3AllocateSegdirIdx(p, iLangid, iIndex, iLevel+1, &iIdx); + assert( FTS3_SEGCURSOR_PENDING==-1 ); iNewLevel = getAbsoluteLevel(p, iLangid, iIndex, iLevel+1); + rc = fts3AllocateSegdirIdx(p, iLangid, iIndex, iLevel+1, &iIdx); + bIgnoreEmpty = (iLevel!=FTS3_SEGCURSOR_PENDING) && (iNewLevel>iMaxLevel); } if( rc!=SQLITE_OK ) goto finished; + assert( csr.nSegment>0 ); assert( iNewLevel>=getAbsoluteLevel(p, iLangid, iIndex, 0) ); assert( iNewLevelnLeafData); + } + } + } finished: fts3SegWriterFree(pWriter); @@ -3036,7 +3239,7 @@ static int fts3SegmentMerge( /* -** Flush the contents of pendingTerms to level 0 segments. +** Flush the contents of pendingTerms to level 0 segments. */ int sqlite3Fts3PendingTermsFlush(Fts3Table *p){ int rc = SQLITE_OK; @@ -3052,14 +3255,19 @@ int sqlite3Fts3PendingTermsFlush(Fts3Table *p){ ** estimate the number of leaf blocks of content to be written */ if( rc==SQLITE_OK && p->bHasStat - && p->bAutoincrmerge==0xff && p->nLeafAdd>0 + && p->nAutoincrmerge==0xff && p->nLeafAdd>0 ){ sqlite3_stmt *pStmt = 0; rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pStmt, 0); if( rc==SQLITE_OK ){ sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE); rc = sqlite3_step(pStmt); - p->bAutoincrmerge = (rc==SQLITE_ROW && sqlite3_column_int(pStmt, 0)); + if( rc==SQLITE_ROW ){ + p->nAutoincrmerge = sqlite3_column_int(pStmt, 0); + if( p->nAutoincrmerge==1 ) p->nAutoincrmerge = 8; + }else if( rc==SQLITE_DONE ){ + p->nAutoincrmerge = 0; + } rc = sqlite3_reset(pStmt); } } @@ -3303,9 +3511,11 @@ static int fts3DoRebuild(Fts3Table *p){ rc = fts3PendingTermsDocid(p, iLangid, sqlite3_column_int64(pStmt, 0)); memset(aSz, 0, sizeof(aSz[0]) * (p->nColumn+1)); for(iCol=0; rc==SQLITE_OK && iColnColumn; iCol++){ - const char *z = (const char *) sqlite3_column_text(pStmt, iCol+1); - rc = fts3PendingTermsAdd(p, iLangid, z, iCol, &aSz[iCol]); - aSz[p->nColumn] += sqlite3_column_bytes(pStmt, iCol+1); + if( p->abNotindexed[iCol]==0 ){ + const char *z = (const char *) sqlite3_column_text(pStmt, iCol+1); + rc = fts3PendingTermsAdd(p, iLangid, z, iCol, &aSz[iCol]); + aSz[p->nColumn] += sqlite3_column_bytes(pStmt, iCol+1); + } } if( p->bHasDocsize ){ fts3InsertDocsize(&rc, p, aSz); @@ -3425,6 +3635,8 @@ struct IncrmergeWriter { int iIdx; /* Index of *output* segment in iAbsLevel+1 */ sqlite3_int64 iStart; /* Block number of first allocated block */ sqlite3_int64 iEnd; /* Block number of last allocated block */ + sqlite3_int64 nLeafData; /* Bytes of leaf page data so far */ + u8 bNoLeafData; /* If true, store 0 for segment size */ NodeWriter aNodeWriter[FTS_MAX_APPENDABLE_HEIGHT]; }; @@ -3493,9 +3705,9 @@ static int nodeReaderNext(NodeReader *p){ p->aNode = 0; }else{ if( bFirst==0 ){ - p->iOff += sqlite3Fts3GetVarint32(&p->aNode[p->iOff], &nPrefix); + p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &nPrefix); } - p->iOff += sqlite3Fts3GetVarint32(&p->aNode[p->iOff], &nSuffix); + p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &nSuffix); blobGrowBuffer(&p->term, nPrefix+nSuffix, &rc); if( rc==SQLITE_OK ){ @@ -3503,7 +3715,7 @@ static int nodeReaderNext(NodeReader *p){ p->term.n = nPrefix+nSuffix; p->iOff += nSuffix; if( p->iChild==0 ){ - p->iOff += sqlite3Fts3GetVarint32(&p->aNode[p->iOff], &p->nDoclist); + p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &p->nDoclist); p->aDoclist = &p->aNode[p->iOff]; p->iOff += p->nDoclist; } @@ -3763,8 +3975,8 @@ static int fts3IncrmergeAppend( nSpace += sqlite3Fts3VarintLen(nDoclist) + nDoclist; } + pWriter->nLeafData += nSpace; blobGrowBuffer(&pLeaf->block, pLeaf->block.n + nSpace, &rc); - if( rc==SQLITE_OK ){ if( pLeaf->block.n==0 ){ pLeaf->block.n = 1; @@ -3863,6 +4075,7 @@ static void fts3IncrmergeRelease( pWriter->iStart, /* start_block */ pWriter->aNodeWriter[0].iBlock, /* leaves_end_block */ pWriter->iEnd, /* end_block */ + (pWriter->bNoLeafData==0 ? pWriter->nLeafData : 0), /* end_block */ pRoot->block.a, pRoot->block.n /* root */ ); } @@ -3964,7 +4177,11 @@ static int fts3IncrmergeLoad( if( sqlite3_step(pSelect)==SQLITE_ROW ){ iStart = sqlite3_column_int64(pSelect, 1); iLeafEnd = sqlite3_column_int64(pSelect, 2); - iEnd = sqlite3_column_int64(pSelect, 3); + fts3ReadEndBlockField(pSelect, 3, &iEnd, &pWriter->nLeafData); + if( pWriter->nLeafData<0 ){ + pWriter->nLeafData = pWriter->nLeafData * -1; + } + pWriter->bNoLeafData = (pWriter->nLeafData==0); nRoot = sqlite3_column_bytes(pSelect, 4); aRoot = sqlite3_column_blob(pSelect, 4); }else{ @@ -4555,7 +4772,7 @@ static int fts3IncrmergeHintPop(Blob *pHint, i64 *piAbsLevel, int *pnInput){ pHint->n = i; i += sqlite3Fts3GetVarint(&pHint->a[i], piAbsLevel); - i += sqlite3Fts3GetVarint32(&pHint->a[i], pnInput); + i += fts3GetVarint32(&pHint->a[i], pnInput); if( i!=nHint ) return SQLITE_CORRUPT_VTAB; return SQLITE_OK; @@ -4565,11 +4782,11 @@ static int fts3IncrmergeHintPop(Blob *pHint, i64 *piAbsLevel, int *pnInput){ /* ** Attempt an incremental merge that writes nMerge leaf blocks. ** -** Incremental merges happen nMin segments at a time. The two -** segments to be merged are the nMin oldest segments (the ones with -** the smallest indexes) in the highest level that contains at least -** nMin segments. Multiple merges might occur in an attempt to write the -** quota of nMerge leaf blocks. +** Incremental merges happen nMin segments at a time. The segments +** to be merged are the nMin oldest segments (the ones with the smallest +** values for the _segdir.idx field) in the highest level that contains +** at least nMin segments. Multiple merges might occur in an attempt to +** write the quota of nMerge leaf blocks. */ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ int rc; /* Return code */ @@ -4594,6 +4811,7 @@ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ const i64 nMod = FTS3_SEGDIR_MAXLEVEL * p->nIndex; sqlite3_stmt *pFindLevel = 0; /* SQL used to determine iAbsLevel */ int bUseHint = 0; /* True if attempting to append */ + int iIdx = 0; /* Largest idx in level (iAbsLevel+1) */ /* Search the %_segdir table for the absolute level with the smallest ** relative level number that contains at least nMin segments, if any. @@ -4647,6 +4865,19 @@ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ ** to start work on some other level. */ memset(pWriter, 0, nAlloc); pFilter->flags = FTS3_SEGMENT_REQUIRE_POS; + + if( rc==SQLITE_OK ){ + rc = fts3IncrmergeOutputIdx(p, iAbsLevel, &iIdx); + assert( bUseHint==1 || bUseHint==0 ); + if( iIdx==0 || (bUseHint && iIdx==1) ){ + int bIgnore = 0; + rc = fts3SegmentIsMaxLevel(p, iAbsLevel+1, &bIgnore); + if( bIgnore ){ + pFilter->flags |= FTS3_SEGMENT_IGNORE_EMPTY; + } + } + } + if( rc==SQLITE_OK ){ rc = fts3IncrmergeCsr(p, iAbsLevel, nSeg, pCsr); } @@ -4654,16 +4885,12 @@ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ && SQLITE_OK==(rc = sqlite3Fts3SegReaderStart(p, pCsr, pFilter)) && SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pCsr)) ){ - int iIdx = 0; /* Largest idx in level (iAbsLevel+1) */ - rc = fts3IncrmergeOutputIdx(p, iAbsLevel, &iIdx); - if( rc==SQLITE_OK ){ - if( bUseHint && iIdx>0 ){ - const char *zKey = pCsr->zTerm; - int nKey = pCsr->nTerm; - rc = fts3IncrmergeLoad(p, iAbsLevel, iIdx-1, zKey, nKey, pWriter); - }else{ - rc = fts3IncrmergeWriter(p, iAbsLevel, iIdx, pCsr, pWriter); - } + if( bUseHint && iIdx>0 ){ + const char *zKey = pCsr->zTerm; + int nKey = pCsr->nTerm; + rc = fts3IncrmergeLoad(p, iAbsLevel, iIdx-1, zKey, nKey, pWriter); + }else{ + rc = fts3IncrmergeWriter(p, iAbsLevel, iIdx, pCsr, pWriter); } if( rc==SQLITE_OK && pWriter->nLeafEst ){ @@ -4685,7 +4912,13 @@ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ } } + if( nSeg!=0 ){ + pWriter->nLeafData = pWriter->nLeafData * -1; + } fts3IncrmergeRelease(p, pWriter, &rc); + if( nSeg==0 && pWriter->bNoLeafData==0 ){ + fts3PromoteSegments(p, iAbsLevel+1, pWriter->nLeafData); + } } sqlite3Fts3SegReaderFinish(pCsr); @@ -4772,16 +5005,19 @@ static int fts3DoAutoincrmerge( ){ int rc = SQLITE_OK; sqlite3_stmt *pStmt = 0; - p->bAutoincrmerge = fts3Getint(&zParam)!=0; + p->nAutoincrmerge = fts3Getint(&zParam); + if( p->nAutoincrmerge==1 || p->nAutoincrmerge>FTS3_MERGE_COUNT ){ + p->nAutoincrmerge = 8; + } if( !p->bHasStat ){ assert( p->bFts4==0 ); sqlite3Fts3CreateStatTable(&rc, p); if( rc ) return rc; } rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pStmt, 0); - if( rc ) return rc;; + if( rc ) return rc; sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE); - sqlite3_bind_int(pStmt, 2, p->bAutoincrmerge); + sqlite3_bind_int(pStmt, 2, p->nAutoincrmerge); sqlite3_step(pStmt); rc = sqlite3_reset(pStmt); return rc; @@ -4938,34 +5174,36 @@ static int fts3IntegrityCheck(Fts3Table *p, int *pbOk){ int iCol; for(iCol=0; rc==SQLITE_OK && iColnColumn; iCol++){ - const char *zText = (const char *)sqlite3_column_text(pStmt, iCol+1); - int nText = sqlite3_column_bytes(pStmt, iCol+1); - sqlite3_tokenizer_cursor *pT = 0; - - rc = sqlite3Fts3OpenTokenizer(p->pTokenizer, iLang, zText, nText, &pT); - while( rc==SQLITE_OK ){ - char const *zToken; /* Buffer containing token */ - int nToken = 0; /* Number of bytes in token */ - int iDum1 = 0, iDum2 = 0; /* Dummy variables */ - int iPos = 0; /* Position of token in zText */ - - rc = pModule->xNext(pT, &zToken, &nToken, &iDum1, &iDum2, &iPos); - if( rc==SQLITE_OK ){ - int i; - cksum2 = cksum2 ^ fts3ChecksumEntry( - zToken, nToken, iLang, 0, iDocid, iCol, iPos - ); - for(i=1; inIndex; i++){ - if( p->aIndex[i].nPrefix<=nToken ){ - cksum2 = cksum2 ^ fts3ChecksumEntry( - zToken, p->aIndex[i].nPrefix, iLang, i, iDocid, iCol, iPos - ); + if( p->abNotindexed[iCol]==0 ){ + const char *zText = (const char *)sqlite3_column_text(pStmt, iCol+1); + int nText = sqlite3_column_bytes(pStmt, iCol+1); + sqlite3_tokenizer_cursor *pT = 0; + + rc = sqlite3Fts3OpenTokenizer(p->pTokenizer, iLang, zText, nText,&pT); + while( rc==SQLITE_OK ){ + char const *zToken; /* Buffer containing token */ + int nToken = 0; /* Number of bytes in token */ + int iDum1 = 0, iDum2 = 0; /* Dummy variables */ + int iPos = 0; /* Position of token in zText */ + + rc = pModule->xNext(pT, &zToken, &nToken, &iDum1, &iDum2, &iPos); + if( rc==SQLITE_OK ){ + int i; + cksum2 = cksum2 ^ fts3ChecksumEntry( + zToken, nToken, iLang, 0, iDocid, iCol, iPos + ); + for(i=1; inIndex; i++){ + if( p->aIndex[i].nPrefix<=nToken ){ + cksum2 = cksum2 ^ fts3ChecksumEntry( + zToken, p->aIndex[i].nPrefix, iLang, i, iDocid, iCol, iPos + ); + } } } } + if( pT ) pModule->xClose(pT); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; } - if( pT ) pModule->xClose(pT); - if( rc==SQLITE_DONE ) rc = SQLITE_OK; } } @@ -5049,6 +5287,9 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){ }else if( nVal>11 && 0==sqlite3_strnicmp(zVal, "maxpending=", 9) ){ p->nMaxPendingData = atoi(&zVal[11]); rc = SQLITE_OK; + }else if( nVal>21 && 0==sqlite3_strnicmp(zVal, "test-no-incr-doclist=", 21) ){ + p->bNoIncrDoclist = atoi(&zVal[21]); + rc = SQLITE_OK; #endif }else{ rc = SQLITE_ERROR; @@ -5108,32 +5349,34 @@ int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *pCsr){ iDocid = sqlite3_column_int64(pCsr->pStmt, 0); for(i=0; inColumn && rc==SQLITE_OK; i++){ - const char *zText = (const char *)sqlite3_column_text(pCsr->pStmt, i+1); - sqlite3_tokenizer_cursor *pTC = 0; - - rc = sqlite3Fts3OpenTokenizer(pT, pCsr->iLangid, zText, -1, &pTC); - while( rc==SQLITE_OK ){ - char const *zToken; /* Buffer containing token */ - int nToken = 0; /* Number of bytes in token */ - int iDum1 = 0, iDum2 = 0; /* Dummy variables */ - int iPos = 0; /* Position of token in zText */ - - rc = pModule->xNext(pTC, &zToken, &nToken, &iDum1, &iDum2, &iPos); - for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){ - Fts3PhraseToken *pPT = pDef->pToken; - if( (pDef->iCol>=p->nColumn || pDef->iCol==i) - && (pPT->bFirst==0 || iPos==0) - && (pPT->n==nToken || (pPT->isPrefix && pPT->nz, pPT->n)) - ){ - fts3PendingListAppend(&pDef->pList, iDocid, i, iPos, &rc); + if( p->abNotindexed[i]==0 ){ + const char *zText = (const char *)sqlite3_column_text(pCsr->pStmt, i+1); + sqlite3_tokenizer_cursor *pTC = 0; + + rc = sqlite3Fts3OpenTokenizer(pT, pCsr->iLangid, zText, -1, &pTC); + while( rc==SQLITE_OK ){ + char const *zToken; /* Buffer containing token */ + int nToken = 0; /* Number of bytes in token */ + int iDum1 = 0, iDum2 = 0; /* Dummy variables */ + int iPos = 0; /* Position of token in zText */ + + rc = pModule->xNext(pTC, &zToken, &nToken, &iDum1, &iDum2, &iPos); + for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){ + Fts3PhraseToken *pPT = pDef->pToken; + if( (pDef->iCol>=p->nColumn || pDef->iCol==i) + && (pPT->bFirst==0 || iPos==0) + && (pPT->n==nToken || (pPT->isPrefix && pPT->nz, pPT->n)) + ){ + fts3PendingListAppend(&pDef->pList, iDocid, i, iPos, &rc); + } } } + if( pTC ) pModule->xClose(pTC); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; } - if( pTC ) pModule->xClose(pTC); - if( rc==SQLITE_DONE ) rc = SQLITE_OK; } - + for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){ if( pDef->pList ){ rc = fts3PendingListAppendVarint(&pDef->pList, 0); @@ -5265,6 +5508,10 @@ int sqlite3Fts3UpdateMethod( int nChng = 0; /* Net change in number of documents */ int bInsertDone = 0; + /* At this point it must be known if the %_stat table exists or not. + ** So bHasStat may not be 2. */ + assert( p->bHasStat==0 || p->bHasStat==1 ); + assert( p->pSegments==0 ); assert( nArg==1 /* DELETE operations */ @@ -5297,6 +5544,9 @@ int sqlite3Fts3UpdateMethod( aSzIns = &aSzDel[p->nColumn+1]; memset(aSzDel, 0, sizeof(aSzDel[0])*(p->nColumn+1)*2); + rc = fts3Writelock(p); + if( rc!=SQLITE_OK ) goto update_out; + /* If this is an INSERT operation, or an UPDATE that modifies the rowid ** value, then this operation requires constraint handling. ** diff --git a/ext/fts3/tool/fts3view.c b/ext/fts3/tool/fts3view.c index 479ae98..3dc1ba8 100644 --- a/ext/fts3/tool/fts3view.c +++ b/ext/fts3/tool/fts3view.c @@ -376,7 +376,7 @@ static void showSegmentStats(sqlite3 *db, const char *zTab){ sqlite3_finalize(pStmt); nLeaf = nSeg - nIdx; printf("Leaf segments larger than %5d bytes.... %9d %5.2f%%\n", - pgsz-45, n, n*100.0/nLeaf); + pgsz-45, n, nLeaf>0 ? n*100.0/nLeaf : 0.0); pStmt = prepare(db, "SELECT max(level%%1024) FROM '%q_segdir'", zTab); mxLevel = 0; @@ -554,7 +554,7 @@ static void decodeSegment( sqlite3_int64 n; sqlite3_int64 iDocsz; int iHeight; - int i = 0; + sqlite3_int64 i = 0; int cnt = 0; char zTerm[1000]; @@ -576,12 +576,12 @@ static void decodeSegment( fprintf(stderr, "term to long\n"); exit(1); } - memcpy(zTerm+iPrefix, aData+i, nTerm); + memcpy(zTerm+iPrefix, aData+i, (size_t)nTerm); zTerm[iPrefix+nTerm] = 0; i += nTerm; if( iHeight==0 ){ i += getVarint(aData+i, &iDocsz); - printf("term: %-25s doclist %7lld bytes offset %d\n", zTerm, iDocsz, i); + printf("term: %-25s doclist %7lld bytes offset %lld\n", zTerm, iDocsz, i); i += iDocsz; }else{ printf("term: %-25s child %lld\n", zTerm, ++iChild); @@ -749,18 +749,19 @@ static void decodeDoclist( */ static void showDoclist(sqlite3 *db, const char *zTab){ const unsigned char *aData; - sqlite3_int64 offset, nData; + sqlite3_int64 offset; + int nData; sqlite3_stmt *pStmt; offset = atoi64(azExtra[1]); - nData = atoi64(azExtra[2]); + nData = atoi(azExtra[2]); pStmt = prepareToGetSegment(db, zTab, azExtra[0]); if( sqlite3_step(pStmt)!=SQLITE_ROW ){ sqlite3_finalize(pStmt); return; } aData = sqlite3_column_blob(pStmt, 0); - printf("Doclist at %s offset %lld of size %lld bytes:\n", + printf("Doclist at %s offset %lld of size %d bytes:\n", azExtra[0], offset, nData); if( findOption("raw", 0, 0)!=0 ){ printBlob(aData+offset, nData); diff --git a/ext/fts3/unicode/mkunicode.tcl b/ext/fts3/unicode/mkunicode.tcl index 0d58e8a..c3083ee 100644 --- a/ext/fts3/unicode/mkunicode.tcl +++ b/ext/fts3/unicode/mkunicode.tcl @@ -160,7 +160,7 @@ proc print_rd {map} { } assert( key>=aDia[iRes] ); return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]);} - puts "\};" + puts "\}" } proc print_isdiacritic {zFunc map} { @@ -239,7 +239,10 @@ proc an_load_unicodedata_text {zName} { foreach $lField $fields {} set iCode [expr "0x$code"] - set bAlnum [expr {[lsearch {L N} [string range $general_category 0 0]]>=0}] + set bAlnum [expr { + [lsearch {L N} [string range $general_category 0 0]] >= 0 + || $general_category=="Co" + }] if { !$bAlnum } { lappend lRet $iCode } } @@ -295,7 +298,7 @@ proc an_print_range_array {lRange} { ** using this format. */ }] - puts -nonewline " const static unsigned int aEntry\[\] = \{" + puts -nonewline " static const unsigned int aEntry\[\] = \{" set i 0 foreach range $lRange { foreach {iFirst nRange} $range {} @@ -346,7 +349,7 @@ proc print_isalnum {zFunc lRange} { return ( (aAscii[c >> 5] & (1 << (c & 0x001F)))==0 ); }else if( c<(1<<22) ){ unsigned int key = (((unsigned int)c)<<10) | 0x000003FF; - int iRes; + int iRes = 0; int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1; int iLo = 0; while( iHi>=iLo ){ @@ -360,7 +363,7 @@ proc print_isalnum {zFunc lRange} { } assert( aEntry[0]=aEntry[iRes] ); - return (c >= ((aEntry[iRes]>>10) + (aEntry[iRes]&0x3FF))); + return (((unsigned int)c) >= ((aEntry[iRes]>>10) + (aEntry[iRes]&0x3FF))); } return 1;} puts "\}" @@ -729,7 +732,7 @@ proc print_fileheader {} { */ }] puts "" - puts "#if !defined(SQLITE_DISABLE_FTS3_UNICODE)" + puts "#ifndef SQLITE_DISABLE_FTS3_UNICODE" puts "#if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4)" puts "" puts "#include " diff --git a/ext/icu/icu.c b/ext/icu/icu.c index ae28d70..1ce1e0c 100644 --- a/ext/icu/icu.c +++ b/ext/icu/icu.c @@ -22,7 +22,7 @@ ** * Implementations of the SQL scalar upper() and lower() functions ** for case mapping. ** -** * Integration of ICU and SQLite collation seqences. +** * Integration of ICU and SQLite collation sequences. ** ** * An implementation of the LIKE operator that uses ICU to ** provide case-independent matching. @@ -488,7 +488,10 @@ int sqlite3IcuInit(sqlite3 *db){ } #if !SQLITE_CORE -int sqlite3_extension_init( +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_icu_init( sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi diff --git a/ext/misc/amatch.c b/ext/misc/amatch.c index b613080..d869dbd 100644 --- a/ext/misc/amatch.c +++ b/ext/misc/amatch.c @@ -786,6 +786,7 @@ static void amatchFree(amatch_vtab *p){ sqlite3_free(p->zVocabTab); sqlite3_free(p->zVocabWord); sqlite3_free(p->zVocabLang); + sqlite3_free(p->zSelf); memset(p, 0, sizeof(*p)); sqlite3_free(p); } @@ -948,6 +949,9 @@ static void amatchClearCursor(amatch_cursor *pCur){ pCur->pAllWords = 0; sqlite3_free(pCur->zInput); pCur->zInput = 0; + sqlite3_free(pCur->zBuf); + pCur->zBuf = 0; + pCur->nBuf = 0; pCur->pCost = 0; pCur->pWord = 0; pCur->pCurrent = 0; @@ -1103,7 +1107,7 @@ static int amatchNext(sqlite3_vtab_cursor *cur){ char *zSql; if( p->zVocabLang && p->zVocabLang[0] ){ zSql = sqlite3_mprintf( - "SELECT \"%s\" FROM \"%s\"", + "SELECT \"%w\" FROM \"%w\"", " WHERE \"%w\">=?1 AND \"%w\"=?2" " ORDER BY 1", p->zVocabWord, p->zVocabTab, @@ -1111,7 +1115,7 @@ static int amatchNext(sqlite3_vtab_cursor *cur){ ); }else{ zSql = sqlite3_mprintf( - "SELECT \"%s\" FROM \"%s\"" + "SELECT \"%w\" FROM \"%w\"" " WHERE \"%w\">=?1" " ORDER BY 1", p->zVocabWord, p->zVocabTab, diff --git a/ext/misc/closure.c b/ext/misc/closure.c index 213b763..30c812d 100644 --- a/ext/misc/closure.c +++ b/ext/misc/closure.c @@ -496,7 +496,7 @@ static const char *closureValueOfKey(const char *zKey, const char *zStr){ /* ** xConnect/xCreate method for the closure module. Arguments are: ** -** argv[0] -> module name ("approximate_match") +** argv[0] -> module name ("transitive_closure") ** argv[1] -> database name ** argv[2] -> table name ** argv[3...] -> arguments @@ -826,11 +826,17 @@ static int closureBestIndex( int iPlan = 0; int i; int idx = 1; + int seenMatch = 0; const struct sqlite3_index_constraint *pConstraint; closure_vtab *pVtab = (closure_vtab*)pTab; + double rCost = 10000000.0; pConstraint = pIdxInfo->aConstraint; for(i=0; inConstraint; i++, pConstraint++){ + if( pConstraint->iColumn==CLOSURE_COL_ROOT + && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ ){ + seenMatch = 1; + } if( pConstraint->usable==0 ) continue; if( (iPlan & 1)==0 && pConstraint->iColumn==CLOSURE_COL_ROOT @@ -839,6 +845,7 @@ static int closureBestIndex( iPlan |= 1; pIdxInfo->aConstraintUsage[i].argvIndex = 1; pIdxInfo->aConstraintUsage[i].omit = 1; + rCost /= 100.0; } if( (iPlan & 0x0000f0)==0 && pConstraint->iColumn==CLOSURE_COL_DEPTH @@ -849,6 +856,7 @@ static int closureBestIndex( iPlan |= idx<<4; pIdxInfo->aConstraintUsage[i].argvIndex = ++idx; if( pConstraint->op==SQLITE_INDEX_CONSTRAINT_LT ) iPlan |= 0x000002; + rCost /= 5.0; } if( (iPlan & 0x000f00)==0 && pConstraint->iColumn==CLOSURE_COL_TABLENAME @@ -857,6 +865,7 @@ static int closureBestIndex( iPlan |= idx<<8; pIdxInfo->aConstraintUsage[i].argvIndex = ++idx; pIdxInfo->aConstraintUsage[i].omit = 1; + rCost /= 5.0; } if( (iPlan & 0x00f000)==0 && pConstraint->iColumn==CLOSURE_COL_IDCOLUMN @@ -891,13 +900,14 @@ static int closureBestIndex( ){ pIdxInfo->orderByConsumed = 1; } - pIdxInfo->estimatedCost = (double)10000; + if( seenMatch && (iPlan&1)==0 ) rCost *= 1e30; + pIdxInfo->estimatedCost = rCost; return SQLITE_OK; } /* -** A virtual table module that implements the "approximate_match". +** A virtual table module that implements the "transitive_closure". */ static sqlite3_module closureModule = { 0, /* iVersion */ diff --git a/ext/misc/compress.c b/ext/misc/compress.c new file mode 100644 index 0000000..a405911 --- /dev/null +++ b/ext/misc/compress.c @@ -0,0 +1,107 @@ +/* +** 2014-06-13 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This SQLite extension implements SQL compression functions +** compress() and uncompress() using ZLIB. +*/ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#include + +/* +** Implementation of the "compress(X)" SQL function. The input X is +** compressed using zLib and the output is returned. +** +** The output is a BLOB that begins with a variable-length integer that +** is the input size in bytes (the size of X before compression). The +** variable-length integer is implemented as 1 to 5 bytes. There are +** seven bits per integer stored in the lower seven bits of each byte. +** More significant bits occur first. The most significant bit (0x80) +** is a flag to indicate the end of the integer. +*/ +static void compressFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const unsigned char *pIn; + unsigned char *pOut; + unsigned int nIn; + unsigned long int nOut; + unsigned char x[8]; + int i, j; + + pIn = sqlite3_value_blob(argv[0]); + nIn = sqlite3_value_bytes(argv[0]); + nOut = 13 + nIn + (nIn+999)/1000; + pOut = sqlite3_malloc( nOut+5 ); + for(i=4; i>=0; i--){ + x[i] = (nIn >> (7*(4-i)))&0x7f; + } + for(i=0; i<4 && x[i]==0; i++){} + for(j=0; i<=4; i++, j++) pOut[j] = x[i]; + pOut[j-1] |= 0x80; + compress(&pOut[j], &nOut, pIn, nIn); + sqlite3_result_blob(context, pOut, nOut+j, sqlite3_free); +} + +/* +** Implementation of the "uncompress(X)" SQL function. The argument X +** is a blob which was obtained from compress(Y). The output will be +** the value Y. +*/ +static void uncompressFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const unsigned char *pIn; + unsigned char *pOut; + unsigned int nIn; + unsigned long int nOut; + int rc; + int i; + + pIn = sqlite3_value_blob(argv[0]); + nIn = sqlite3_value_bytes(argv[0]); + nOut = 0; + for(i=0; i + +/* +** Implementation of the "readfile(X)" SQL function. The entire content +** of the file named X is read and returned as a BLOB. NULL is returned +** if the file does not exist or is unreadable. +*/ +static void readfileFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zName; + FILE *in; + long nIn; + void *pBuf; + + zName = (const char*)sqlite3_value_text(argv[0]); + if( zName==0 ) return; + in = fopen(zName, "rb"); + if( in==0 ) return; + fseek(in, 0, SEEK_END); + nIn = ftell(in); + rewind(in); + pBuf = sqlite3_malloc( nIn ); + if( pBuf && 1==fread(pBuf, nIn, 1, in) ){ + sqlite3_result_blob(context, pBuf, nIn, sqlite3_free); + }else{ + sqlite3_free(pBuf); + } + fclose(in); +} + +/* +** Implementation of the "writefile(X,Y)" SQL function. The argument Y +** is written into file X. The number of bytes written is returned. Or +** NULL is returned if something goes wrong, such as being unable to open +** file X for writing. +*/ +static void writefileFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + FILE *out; + const char *z; + sqlite3_int64 rc; + const char *zFile; + + zFile = (const char*)sqlite3_value_text(argv[0]); + if( zFile==0 ) return; + out = fopen(zFile, "wb"); + if( out==0 ) return; + z = (const char*)sqlite3_value_blob(argv[1]); + if( z==0 ){ + rc = 0; + }else{ + rc = fwrite(z, 1, sqlite3_value_bytes(argv[1]), out); + } + fclose(out); + sqlite3_result_int64(context, rc); +} + + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_fileio_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Unused parameter */ + rc = sqlite3_create_function(db, "readfile", 1, SQLITE_UTF8, 0, + readfileFunc, 0, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "writefile", 2, SQLITE_UTF8, 0, + writefileFunc, 0, 0); + } + return rc; +} diff --git a/ext/misc/fuzzer.c b/ext/misc/fuzzer.c index 642b8f9..fe41cda 100644 --- a/ext/misc/fuzzer.c +++ b/ext/misc/fuzzer.c @@ -1077,9 +1077,16 @@ static int fuzzerBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ int iDistTerm = -1; int iRulesetTerm = -1; int i; + int seenMatch = 0; const struct sqlite3_index_constraint *pConstraint; + double rCost = 1e12; + pConstraint = pIdxInfo->aConstraint; for(i=0; inConstraint; i++, pConstraint++){ + if( pConstraint->iColumn==0 + && pConstraint->op==SQLITE_INDEX_CONSTRAINT_MATCH ){ + seenMatch = 1; + } if( pConstraint->usable==0 ) continue; if( (iPlan & 1)==0 && pConstraint->iColumn==0 @@ -1088,6 +1095,7 @@ static int fuzzerBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ iPlan |= 1; pIdxInfo->aConstraintUsage[i].argvIndex = 1; pIdxInfo->aConstraintUsage[i].omit = 1; + rCost /= 1e6; } if( (iPlan & 2)==0 && pConstraint->iColumn==1 @@ -1096,6 +1104,7 @@ static int fuzzerBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ ){ iPlan |= 2; iDistTerm = i; + rCost /= 10.0; } if( (iPlan & 4)==0 && pConstraint->iColumn==2 @@ -1104,6 +1113,7 @@ static int fuzzerBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ iPlan |= 4; pIdxInfo->aConstraintUsage[i].omit = 1; iRulesetTerm = i; + rCost /= 10.0; } } if( iPlan & 2 ){ @@ -1122,7 +1132,8 @@ static int fuzzerBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ ){ pIdxInfo->orderByConsumed = 1; } - pIdxInfo->estimatedCost = (double)10000; + if( seenMatch && (iPlan&1)==0 ) rCost = 1e99; + pIdxInfo->estimatedCost = rCost; return SQLITE_OK; } diff --git a/ext/misc/ieee754.c b/ext/misc/ieee754.c index 436b11e..f143893 100644 --- a/ext/misc/ieee754.c +++ b/ext/misc/ieee754.c @@ -18,11 +18,11 @@ ** ** In the first form, the value X should be a floating-point number. ** The function will return a string of the form 'ieee754(Y,Z)' where -** Y and Z are integers such that X==Y*pow(w.0,Z). +** Y and Z are integers such that X==Y*pow(2,Z). ** ** In the second form, Y and Z are integers which are the mantissa and ** base-2 exponent of a new floating point number. The function returns -** a floating-point value equal to Y*pow(2.0,Z). +** a floating-point value equal to Y*pow(2,Z). ** ** Examples: ** diff --git a/ext/misc/nextchar.c b/ext/misc/nextchar.c index e063043..49dfd24 100644 --- a/ext/misc/nextchar.c +++ b/ext/misc/nextchar.c @@ -10,12 +10,22 @@ ** ****************************************************************************** ** -** This file contains code to implement the next_char(A,T,F,W) SQL function. +** This file contains code to implement the next_char(A,T,F,W,C) SQL function. ** -** The next_char(A,T,F,H) function finds all valid "next" characters for -** string A given the vocabulary in T.F. The T.F field should be indexed. -** If the W value exists and is a non-empty string, then it is an SQL -** expression that limits the entries in T.F that will be considered. +** The next_char(A,T,F,W,C) function finds all valid "next" characters for +** string A given the vocabulary in T.F. If the W value exists and is a +** non-empty string, then it is an SQL expression that limits the entries +** in T.F that will be considered. If C exists and is a non-empty string, +** then it is the name of the collating sequence to use for comparison. If +** +** Only the first three arguments are required. If the C parameter is +** omitted or is NULL or is an empty string, then the default collating +** sequence of T.F is used for comparision. If the W parameter is omitted +** or is NULL or is an empty string, then no filtering of the output is +** done. +** +** The T.F column should be indexed using collation C or else this routine +** will be quite slow. ** ** For example, suppose an application has a dictionary like this: ** @@ -28,6 +38,19 @@ ** out) run the following query: ** ** SELECT next_char('cha','dictionary','word'); +** +** IMPLEMENTATION NOTES: +** +** The next_char function is implemented using recursive SQL that makes +** use of the table name and column name as part of a query. If either +** the table name or column name are keywords or contain special characters, +** then they should be escaped. For example: +** +** SELECT next_char('cha','[dictionary]','[word]'); +** +** This also means that the table name can be a subquery: +** +** SELECT next_char('cha','(SELECT word AS w FROM dictionary)','w'); */ #include "sqlite3ext.h" SQLITE_EXTENSION_INIT1 @@ -184,6 +207,9 @@ static void nextCharFunc( const unsigned char *zTable = sqlite3_value_text(argv[1]); const unsigned char *zField = sqlite3_value_text(argv[2]); const unsigned char *zWhere; + const unsigned char *zCollName; + char *zWhereClause = 0; + char *zColl = 0; char *zSql; int rc; @@ -192,25 +218,41 @@ static void nextCharFunc( c.zPrefix = sqlite3_value_text(argv[0]); c.nPrefix = sqlite3_value_bytes(argv[0]); if( zTable==0 || zField==0 || c.zPrefix==0 ) return; - if( argc<4 - || (zWhere = sqlite3_value_text(argv[3]))==0 - || zWhere[0]==0 + if( argc>=4 + && (zWhere = sqlite3_value_text(argv[3]))!=0 + && zWhere[0]!=0 + ){ + zWhereClause = sqlite3_mprintf("AND (%s)", zWhere); + if( zWhereClause==0 ){ + sqlite3_result_error_nomem(context); + return; + } + }else{ + zWhereClause = ""; + } + if( argc>=5 + && (zCollName = sqlite3_value_text(argv[4]))!=0 + && zCollName[0]!=0 ){ - zSql = sqlite3_mprintf( - "SELECT \"%w\" FROM \"%w\"" - " WHERE \"%w\">=(?1 || ?2)" - " AND \"%w\"<=(?1 || char(1114111))" /* 1114111 == 0x10ffff */ - " ORDER BY 1 ASC LIMIT 1", - zField, zTable, zField, zField); + zColl = sqlite3_mprintf("collate \"%w\"", zCollName); + if( zColl==0 ){ + sqlite3_result_error_nomem(context); + if( zWhereClause[0] ) sqlite3_free(zWhereClause); + return; + } }else{ - zSql = sqlite3_mprintf( - "SELECT \"%w\" FROM \"%w\"" - " WHERE \"%w\">=(?1 || ?2)" - " AND \"%w\"<=(?1 || char(1114111))" /* 1114111 == 0x10ffff */ - " AND (%s)" - " ORDER BY 1 ASC LIMIT 1", - zField, zTable, zField, zField, zWhere); + zColl = ""; } + zSql = sqlite3_mprintf( + "SELECT %s FROM %s" + " WHERE %s>=(?1 || ?2) %s" + " AND %s<=(?1 || char(1114111)) %s" /* 1114111 == 0x10ffff */ + " %s" + " ORDER BY 1 %s ASC LIMIT 1", + zField, zTable, zField, zColl, zField, zColl, zWhereClause, zColl + ); + if( zWhereClause[0] ) sqlite3_free(zWhereClause); + if( zColl[0] ) sqlite3_free(zColl); if( zSql==0 ){ sqlite3_result_error_nomem(context); return; @@ -261,5 +303,9 @@ int sqlite3_nextchar_init( rc = sqlite3_create_function(db, "next_char", 4, SQLITE_UTF8, 0, nextCharFunc, 0, 0); } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "next_char", 5, SQLITE_UTF8, 0, + nextCharFunc, 0, 0); + } return rc; } diff --git a/ext/misc/percentile.c b/ext/misc/percentile.c new file mode 100644 index 0000000..74a4c9d --- /dev/null +++ b/ext/misc/percentile.c @@ -0,0 +1,219 @@ +/* +** 2013-05-28 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains code to implement the percentile(Y,P) SQL function +** as described below: +** +** (1) The percentile(Y,P) function is an aggregate function taking +** exactly two arguments. +** +** (2) If the P argument to percentile(Y,P) is not the same for every +** row in the aggregate then an error is thrown. The word "same" +** in the previous sentence means that the value differ by less +** than 0.001. +** +** (3) If the P argument to percentile(Y,P) evaluates to anything other +** than a number in the range of 0.0 to 100.0 inclusive then an +** error is thrown. +** +** (4) If any Y argument to percentile(Y,P) evaluates to a value that +** is not NULL and is not numeric then an error is thrown. +** +** (5) If any Y argument to percentile(Y,P) evaluates to plus or minus +** infinity then an error is thrown. (SQLite always interprets NaN +** values as NULL.) +** +** (6) Both Y and P in percentile(Y,P) can be arbitrary expressions, +** including CASE WHEN expressions. +** +** (7) The percentile(Y,P) aggregate is able to handle inputs of at least +** one million (1,000,000) rows. +** +** (8) If there are no non-NULL values for Y, then percentile(Y,P) +** returns NULL. +** +** (9) If there is exactly one non-NULL value for Y, the percentile(Y,P) +** returns the one Y value. +** +** (10) If there N non-NULL values of Y where N is two or more and +** the Y values are ordered from least to greatest and a graph is +** drawn from 0 to N-1 such that the height of the graph at J is +** the J-th Y value and such that straight lines are drawn between +** adjacent Y values, then the percentile(Y,P) function returns +** the height of the graph at P*(N-1)/100. +** +** (11) The percentile(Y,P) function always returns either a floating +** point number or NULL. +** +** (12) The percentile(Y,P) is implemented as a single C99 source-code +** file that compiles into a shared-library or DLL that can be loaded +** into SQLite using the sqlite3_load_extension() interface. +*/ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#include +#include +#include + +/* The following object is the session context for a single percentile() +** function. We have to remember all input Y values until the very end. +** Those values are accumulated in the Percentile.a[] array. +*/ +typedef struct Percentile Percentile; +struct Percentile { + unsigned nAlloc; /* Number of slots allocated for a[] */ + unsigned nUsed; /* Number of slots actually used in a[] */ + double rPct; /* 1.0 more than the value for P */ + double *a; /* Array of Y values */ +}; + +/* +** Return TRUE if the input floating-point number is an infinity. +*/ +static int isInfinity(double r){ + sqlite3_uint64 u; + assert( sizeof(u)==sizeof(r) ); + memcpy(&u, &r, sizeof(u)); + return ((u>>52)&0x7ff)==0x7ff; +} + +/* +** Return TRUE if two doubles differ by 0.001 or less +*/ +static int sameValue(double a, double b){ + a -= b; + return a>=-0.001 && a<=0.001; +} + +/* +** The "step" function for percentile(Y,P) is called once for each +** input row. +*/ +static void percentStep(sqlite3_context *pCtx, int argc, sqlite3_value **argv){ + Percentile *p; + double rPct; + int eType; + double y; + assert( argc==2 ); + + /* Requirement 3: P must be a number between 0 and 100 */ + eType = sqlite3_value_numeric_type(argv[1]); + rPct = sqlite3_value_double(argv[1]); + if( (eType!=SQLITE_INTEGER && eType!=SQLITE_FLOAT) || + ((rPct = sqlite3_value_double(argv[1]))<0.0 || rPct>100.0) ){ + sqlite3_result_error(pCtx, "2nd argument to percentile() is not " + "a number between 0.0 and 100.0", -1); + return; + } + + /* Allocate the session context. */ + p = (Percentile*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p==0 ) return; + + /* Remember the P value. Throw an error if the P value is different + ** from any prior row, per Requirement (2). */ + if( p->rPct==0.0 ){ + p->rPct = rPct+1.0; + }else if( !sameValue(p->rPct,rPct+1.0) ){ + sqlite3_result_error(pCtx, "2nd argument to percentile() is not the " + "same for all input rows", -1); + return; + } + + /* Ignore rows for which Y is NULL */ + eType = sqlite3_value_type(argv[0]); + if( eType==SQLITE_NULL ) return; + + /* If not NULL, then Y must be numeric. Otherwise throw an error. + ** Requirement 4 */ + if( eType!=SQLITE_INTEGER && eType!=SQLITE_FLOAT ){ + sqlite3_result_error(pCtx, "1st argument to percentile() is not " + "numeric", -1); + return; + } + + /* Throw an error if the Y value is infinity or NaN */ + y = sqlite3_value_double(argv[0]); + if( isInfinity(y) ){ + sqlite3_result_error(pCtx, "Inf input to percentile()", -1); + return; + } + + /* Allocate and store the Y */ + if( p->nUsed>=p->nAlloc ){ + unsigned n = p->nAlloc*2 + 250; + double *a = sqlite3_realloc(p->a, sizeof(double)*n); + if( a==0 ){ + sqlite3_free(p->a); + memset(p, 0, sizeof(*p)); + sqlite3_result_error_nomem(pCtx); + return; + } + p->nAlloc = n; + p->a = a; + } + p->a[p->nUsed++] = y; +} + +/* +** Compare to doubles for sorting using qsort() +*/ +static int doubleCmp(const void *pA, const void *pB){ + double a = *(double*)pA; + double b = *(double*)pB; + if( a==b ) return 0; + if( aa==0 ) return; + if( p->nUsed ){ + qsort(p->a, p->nUsed, sizeof(double), doubleCmp); + ix = (p->rPct-1.0)*(p->nUsed-1)*0.01; + i1 = (unsigned)ix; + i2 = ix==(double)i1 || i1==p->nUsed-1 ? i1 : i1+1; + v1 = p->a[i1]; + v2 = p->a[i2]; + vx = v1 + (v2-v1)*(ix-i1); + sqlite3_result_double(pCtx, vx); + } + sqlite3_free(p->a); + memset(p, 0, sizeof(*p)); +} + + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_percentile_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Unused parameter */ + rc = sqlite3_create_function(db, "percentile", 2, SQLITE_UTF8, 0, + 0, percentStep, percentFinal); + return rc; +} diff --git a/ext/misc/regexp.c b/ext/misc/regexp.c index 16fa7d0..7244d52 100644 --- a/ext/misc/regexp.c +++ b/ext/misc/regexp.c @@ -713,6 +713,7 @@ static void re_sql_func( const char *zPattern; /* The regular expression */ const unsigned char *zStr;/* String being searched */ const char *zErr; /* Compile error message */ + int setAux = 0; /* True to invoke sqlite3_set_auxdata() */ pRe = sqlite3_get_auxdata(context, 0); if( pRe==0 ){ @@ -728,12 +729,15 @@ static void re_sql_func( sqlite3_result_error_nomem(context); return; } - sqlite3_set_auxdata(context, 0, pRe, (void(*)(void*))re_free); + setAux = 1; } zStr = (const unsigned char*)sqlite3_value_text(argv[1]); if( zStr!=0 ){ sqlite3_result_int(context, re_match(pRe, zStr, -1)); } + if( setAux ){ + sqlite3_set_auxdata(context, 0, pRe, (void(*)(void*))re_free); + } } /* diff --git a/ext/misc/spellfix.c b/ext/misc/spellfix.c index eb5442e..2e6743e 100644 --- a/ext/misc/spellfix.c +++ b/ext/misc/spellfix.c @@ -26,8 +26,8 @@ SQLITE_EXTENSION_INIT1 # define NEVER(X) 0 typedef unsigned char u8; typedef unsigned short u16; -# include #endif +#include #ifndef SQLITE_OMIT_VIRTUALTABLE @@ -1893,7 +1893,7 @@ static int spellfix1Init( char **pzErr ){ spellfix1_vtab *pNew = 0; - const char *zModule = argv[0]; + /* const char *zModule = argv[0]; // not used */ const char *zDbName = argv[1]; const char *zTableName = argv[2]; int nDbName; @@ -1933,7 +1933,6 @@ static int spellfix1Init( #define SPELLFIX_COL_COMMAND 11 } if( rc==SQLITE_OK && isCreate ){ - sqlite3_uint64 r; spellfix1DbExec(&rc, db, "CREATE TABLE IF NOT EXISTS \"%w\".\"%w_vocab\"(\n" " id INTEGER PRIMARY KEY,\n" @@ -1945,11 +1944,10 @@ static int spellfix1Init( ");\n", zDbName, zTableName ); - sqlite3_randomness(sizeof(r), &r); spellfix1DbExec(&rc, db, - "CREATE INDEX IF NOT EXISTS \"%w\".\"%w_index_%llx\" " + "CREATE INDEX IF NOT EXISTS \"%w\".\"%w_vocab_index_langid_k2\" " "ON \"%w_vocab\"(langid,k2);", - zDbName, zModule, r, zTableName + zDbName, zTableName, zTableName ); } for(i=3; rc==SQLITE_OK && iaConstraint; @@ -2122,6 +2123,15 @@ static int spellfix1BestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ iPlan |= pConstraint->op==SQLITE_INDEX_CONSTRAINT_LT ? 16 : 32; iDistTerm = i; } + + /* Terms of the form: distance < $dist or distance <= $dist */ + if( (iPlan & 64)==0 + && pConstraint->iColumn<0 + && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ + ){ + iPlan |= 64; + iRowidTerm = i; + } } if( iPlan&1 ){ int idx = 2; @@ -2148,10 +2158,15 @@ static int spellfix1BestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ pIdxInfo->aConstraintUsage[iDistTerm].argvIndex = idx++; pIdxInfo->aConstraintUsage[iDistTerm].omit = 1; } - pIdxInfo->estimatedCost = (double)10000; + pIdxInfo->estimatedCost = 1e5; + }else if( (iPlan & 64) ){ + pIdxInfo->idxNum = 64; + pIdxInfo->aConstraintUsage[iRowidTerm].argvIndex = 1; + pIdxInfo->aConstraintUsage[iRowidTerm].omit = 1; + pIdxInfo->estimatedCost = 5; }else{ pIdxInfo->idxNum = 0; - pIdxInfo->estimatedCost = (double)10000000; + pIdxInfo->estimatedCost = 1e50; } return SQLITE_OK; } @@ -2465,16 +2480,23 @@ static int spellfix1FilterForFullScan( int argc, sqlite3_value **argv ){ - int rc; + int rc = SQLITE_OK; char *zSql; spellfix1_vtab *pVTab = pCur->pVTab; spellfix1ResetCursor(pCur); + assert( idxNum==0 || idxNum==64 ); zSql = sqlite3_mprintf( - "SELECT word, rank, NULL, langid, id FROM \"%w\".\"%w_vocab\"", - pVTab->zDbName, pVTab->zTableName); + "SELECT word, rank, NULL, langid, id FROM \"%w\".\"%w_vocab\"%s", + pVTab->zDbName, pVTab->zTableName, + ((idxNum & 64) ? " WHERE rowid=?" : "") + ); if( zSql==0 ) return SQLITE_NOMEM; rc = sqlite3_prepare_v2(pVTab->db, zSql, -1, &pCur->pFullScan, 0); sqlite3_free(zSql); + if( rc==SQLITE_OK && (idxNum & 64) ){ + assert( argc==1 ); + rc = sqlite3_bind_value(pCur->pFullScan, 1, argv[0]); + } pCur->nRow = pCur->iRow = 0; if( rc==SQLITE_OK ){ rc = sqlite3_step(pCur->pFullScan); @@ -2672,7 +2694,7 @@ static int spellfix1Update( const char *zCmd = (const char*)sqlite3_value_text(argv[SPELLFIX_COL_COMMAND+2]); if( zCmd==0 ){ - pVTab->zErrMsg = sqlite3_mprintf("%s.word may not be NULL", + pVTab->zErrMsg = sqlite3_mprintf("NOT NULL constraint failed: %s.word", p->zTableName); return SQLITE_CONSTRAINT_NOTNULL; } diff --git a/ext/misc/totype.c b/ext/misc/totype.c new file mode 100644 index 0000000..5dc99f3 --- /dev/null +++ b/ext/misc/totype.c @@ -0,0 +1,512 @@ +/* +** 2013-10-14 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This SQLite extension implements functions tointeger(X) and toreal(X). +** +** If X is an integer, real, or string value that can be +** losslessly represented as an integer, then tointeger(X) +** returns the corresponding integer value. +** If X is an 8-byte BLOB then that blob is interpreted as +** a signed two-compliment little-endian encoding of an integer +** and tointeger(X) returns the corresponding integer value. +** Otherwise tointeger(X) return NULL. +** +** If X is an integer, real, or string value that can be +** convert into a real number, preserving at least 15 digits +** of precision, then toreal(X) returns the corresponding real value. +** If X is an 8-byte BLOB then that blob is interpreted as +** a 64-bit IEEE754 big-endian floating point value +** and toreal(X) returns the corresponding real value. +** Otherwise toreal(X) return NULL. +** +** Note that tointeger(X) of an 8-byte BLOB assumes a little-endian +** encoding whereas toreal(X) of an 8-byte BLOB assumes a big-endian +** encoding. +*/ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#include +#include + +/* +** Determine if this is running on a big-endian or little-endian +** processor +*/ +#if defined(i386) || defined(__i386__) || defined(_M_IX86)\ + || defined(__x86_64) || defined(__x86_64__) +# define TOTYPE_BIGENDIAN 0 +# define TOTYPE_LITTLEENDIAN 1 +#else + const int totype_one = 1; +# define TOTYPE_BIGENDIAN (*(char *)(&totype_one)==0) +# define TOTYPE_LITTLEENDIAN (*(char *)(&totype_one)==1) +#endif + +/* +** Constants for the largest and smallest possible 64-bit signed integers. +** These macros are designed to work correctly on both 32-bit and 64-bit +** compilers. +*/ +#ifndef LARGEST_INT64 +# define LARGEST_INT64 (0xffffffff|(((sqlite3_int64)0x7fffffff)<<32)) +#endif + +#ifndef SMALLEST_INT64 +# define SMALLEST_INT64 (((sqlite3_int64)-1) - LARGEST_INT64) +#endif + +/* +** Return TRUE if character c is a whitespace character +*/ +static int totypeIsspace(unsigned char c){ + return c==' ' || c=='\t' || c=='\n' || c=='\v' || c=='\f' || c=='\r'; +} + +/* +** Return TRUE if character c is a digit +*/ +static int totypeIsdigit(unsigned char c){ + return c>='0' && c<='9'; +} + +/* +** Compare the 19-character string zNum against the text representation +** value 2^63: 9223372036854775808. Return negative, zero, or positive +** if zNum is less than, equal to, or greater than the string. +** Note that zNum must contain exactly 19 characters. +** +** Unlike memcmp() this routine is guaranteed to return the difference +** in the values of the last digit if the only difference is in the +** last digit. So, for example, +** +** totypeCompare2pow63("9223372036854775800") +** +** will return -8. +*/ +static int totypeCompare2pow63(const char *zNum){ + int c = 0; + int i; + /* 012345678901234567 */ + const char *pow63 = "922337203685477580"; + for(i=0; c==0 && i<18; i++){ + c = (zNum[i]-pow63[i])*10; + } + if( c==0 ){ + c = zNum[18] - '8'; + } + return c; +} + +/* +** Convert zNum to a 64-bit signed integer. +** +** If the zNum value is representable as a 64-bit twos-complement +** integer, then write that value into *pNum and return 0. +** +** If zNum is exactly 9223372036854665808, return 2. This special +** case is broken out because while 9223372036854665808 cannot be a +** signed 64-bit integer, its negative -9223372036854665808 can be. +** +** If zNum is too big for a 64-bit integer and is not +** 9223372036854665808 or if zNum contains any non-numeric text, +** then return 1. +** +** The string is not necessarily zero-terminated. +*/ +static int totypeAtoi64(const char *zNum, sqlite3_int64 *pNum, int length){ + sqlite3_uint64 u = 0; + int neg = 0; /* assume positive */ + int i; + int c = 0; + int nonNum = 0; + const char *zStart; + const char *zEnd = zNum + length; + + while( zNum='0' && c<='9'; i++){ + u = u*10 + c - '0'; + } + if( u>LARGEST_INT64 ){ + *pNum = SMALLEST_INT64; + }else if( neg ){ + *pNum = -(sqlite3_int64)u; + }else{ + *pNum = (sqlite3_int64)u; + } + if( (c!=0 && &zNum[i]19 || nonNum ){ + /* zNum is empty or contains non-numeric text or is longer + ** than 19 digits (thus guaranteeing that it is too large) */ + return 1; + }else if( i<19 ){ + /* Less than 19 digits, so we know that it fits in 64 bits */ + assert( u<=LARGEST_INT64 ); + return 0; + }else{ + /* zNum is a 19-digit numbers. Compare it against 9223372036854775808. */ + c = totypeCompare2pow63(zNum); + if( c<0 ){ + /* zNum is less than 9223372036854775808 so it fits */ + assert( u<=LARGEST_INT64 ); + return 0; + }else if( c>0 ){ + /* zNum is greater than 9223372036854775808 so it overflows */ + return 1; + }else{ + /* zNum is exactly 9223372036854775808. Fits if negative. The + ** special case 2 overflow if positive */ + assert( u-1==LARGEST_INT64 ); + assert( (*pNum)==SMALLEST_INT64 ); + return neg ? 0 : 2; + } + } +} + +/* +** The string z[] is an text representation of a real number. +** Convert this string to a double and write it into *pResult. +** +** The string is not necessarily zero-terminated. +** +** Return TRUE if the result is a valid real number (or integer) and FALSE +** if the string is empty or contains extraneous text. Valid numbers +** are in one of these formats: +** +** [+-]digits[E[+-]digits] +** [+-]digits.[digits][E[+-]digits] +** [+-].digits[E[+-]digits] +** +** Leading and trailing whitespace is ignored for the purpose of determining +** validity. +** +** If some prefix of the input string is a valid number, this routine +** returns FALSE but it still converts the prefix and writes the result +** into *pResult. +*/ +static int totypeAtoF(const char *z, double *pResult, int length){ + const char *zEnd = z + length; + /* sign * significand * (10 ^ (esign * exponent)) */ + int sign = 1; /* sign of significand */ + sqlite3_int64 s = 0; /* significand */ + int d = 0; /* adjust exponent for shifting decimal point */ + int esign = 1; /* sign of exponent */ + int e = 0; /* exponent */ + int eValid = 1; /* True exponent is either not used or is well-formed */ + double result; + int nDigits = 0; + int nonNum = 0; + + *pResult = 0.0; /* Default return value, in case of an error */ + + /* skip leading spaces */ + while( z=zEnd ) return 0; + + /* get sign of significand */ + if( *z=='-' ){ + sign = -1; + z++; + }else if( *z=='+' ){ + z++; + } + + /* skip leading zeroes */ + while( z=zEnd ) goto totype_atof_calc; + + /* if decimal point is present */ + if( *z=='.' ){ + z++; + /* copy digits from after decimal to significand + ** (decrease exponent by d to shift decimal right) */ + while( z=zEnd ) goto totype_atof_calc; + + /* if exponent is present */ + if( *z=='e' || *z=='E' ){ + z++; + eValid = 0; + if( z>=zEnd ) goto totype_atof_calc; + /* get sign of exponent */ + if( *z=='-' ){ + esign = -1; + z++; + }else if( *z=='+' ){ + z++; + } + /* copy digits to exponent */ + while( z0 ){ + while( s<(LARGEST_INT64/10) && e>0 ) e--,s*=10; + }else{ + while( !(s%10) && e>0 ) e--,s/=10; + } + + /* adjust the sign of significand */ + s = sign<0 ? -s : s; + + /* if exponent, scale significand as appropriate + ** and store in result. */ + if( e ){ + double scale = 1.0; + /* attempt to handle extremely small/large numbers better */ + if( e>307 && e<342 ){ + while( e%308 ) { scale *= 1.0e+1; e -= 1; } + if( esign<0 ){ + result = s / scale; + result /= 1.0e+308; + }else{ + result = s * scale; + result *= 1.0e+308; + } + }else if( e>=342 ){ + if( esign<0 ){ + result = 0.0*s; + }else{ + result = 1e308*1e308*s; /* Infinity */ + } + }else{ + /* 1.0e+22 is the largest power of 10 than can be + ** represented exactly. */ + while( e%22 ) { scale *= 1.0e+1; e -= 1; } + while( e>0 ) { scale *= 1.0e+22; e -= 22; } + if( esign<0 ){ + result = s / scale; + }else{ + result = s * scale; + } + } + } else { + result = (double)s; + } + } + + /* store the result */ + *pResult = result; + + /* return true if number and no extra non-whitespace chracters after */ + return z>=zEnd && nDigits>0 && eValid && nonNum==0; +} + +/* +** tointeger(X): If X is any value (integer, double, blob, or string) that +** can be losslessly converted into an integer, then make the conversion and +** return the result. Otherwise, return NULL. +*/ +static void tointegerFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + assert( argc==1 ); + (void)argc; + switch( sqlite3_value_type(argv[0]) ){ + case SQLITE_FLOAT: { + double rVal = sqlite3_value_double(argv[0]); + sqlite3_int64 iVal = (sqlite3_int64)rVal; + if( rVal==(double)iVal ){ + sqlite3_result_int64(context, iVal); + } + break; + } + case SQLITE_INTEGER: { + sqlite3_result_int64(context, sqlite3_value_int64(argv[0])); + break; + } + case SQLITE_BLOB: { + const unsigned char *zBlob = sqlite3_value_blob(argv[0]); + if( zBlob ){ + int nBlob = sqlite3_value_bytes(argv[0]); + if( nBlob==sizeof(sqlite3_int64) ){ + sqlite3_int64 iVal; + if( TOTYPE_BIGENDIAN ){ + int i; + unsigned char zBlobRev[sizeof(sqlite3_int64)]; + for(i=0; i +#include +#include +#if SQLITE_OS_UNIX +# include +#endif + +/* +** Forward declaration of objects used by this utility +*/ +typedef struct VLogLog VLogLog; +typedef struct VLogVfs VLogVfs; +typedef struct VLogFile VLogFile; + +/* There is a pair (an array of size 2) of the following objects for +** each database file being logged. The first contains the filename +** and is used to log I/O with the main database. The second has +** a NULL filename and is used to log I/O for the journal. Both +** out pointers are the same. +*/ +struct VLogLog { + VLogLog *pNext; /* Next in a list of all active logs */ + VLogLog **ppPrev; /* Pointer to this in the list */ + int nRef; /* Number of references to this object */ + int nFilename; /* Length of zFilename in bytes */ + char *zFilename; /* Name of database file. NULL for journal */ + FILE *out; /* Write information here */ +}; + +struct VLogVfs { + sqlite3_vfs base; /* VFS methods */ + sqlite3_vfs *pVfs; /* Parent VFS */ +}; + +struct VLogFile { + sqlite3_file base; /* IO methods */ + sqlite3_file *pReal; /* Underlying file handle */ + VLogLog *pLog; /* The log file for this file */ +}; + +#define REALVFS(p) (((VLogVfs*)(p))->pVfs) + +/* +** Methods for VLogFile +*/ +static int vlogClose(sqlite3_file*); +static int vlogRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst); +static int vlogWrite(sqlite3_file*,const void*,int iAmt, sqlite3_int64 iOfst); +static int vlogTruncate(sqlite3_file*, sqlite3_int64 size); +static int vlogSync(sqlite3_file*, int flags); +static int vlogFileSize(sqlite3_file*, sqlite3_int64 *pSize); +static int vlogLock(sqlite3_file*, int); +static int vlogUnlock(sqlite3_file*, int); +static int vlogCheckReservedLock(sqlite3_file*, int *pResOut); +static int vlogFileControl(sqlite3_file*, int op, void *pArg); +static int vlogSectorSize(sqlite3_file*); +static int vlogDeviceCharacteristics(sqlite3_file*); + +/* +** Methods for VLogVfs +*/ +static int vlogOpen(sqlite3_vfs*, const char *, sqlite3_file*, int , int *); +static int vlogDelete(sqlite3_vfs*, const char *zName, int syncDir); +static int vlogAccess(sqlite3_vfs*, const char *zName, int flags, int *); +static int vlogFullPathname(sqlite3_vfs*, const char *zName, int, char *zOut); +static void *vlogDlOpen(sqlite3_vfs*, const char *zFilename); +static void vlogDlError(sqlite3_vfs*, int nByte, char *zErrMsg); +static void (*vlogDlSym(sqlite3_vfs *pVfs, void *p, const char*zSym))(void); +static void vlogDlClose(sqlite3_vfs*, void*); +static int vlogRandomness(sqlite3_vfs*, int nByte, char *zOut); +static int vlogSleep(sqlite3_vfs*, int microseconds); +static int vlogCurrentTime(sqlite3_vfs*, double*); +static int vlogGetLastError(sqlite3_vfs*, int, char *); +static int vlogCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*); + +static VLogVfs vlog_vfs = { + { + 1, /* iVersion */ + 0, /* szOsFile (set by register_vlog()) */ + 1024, /* mxPathname */ + 0, /* pNext */ + "vfslog", /* zName */ + 0, /* pAppData */ + vlogOpen, /* xOpen */ + vlogDelete, /* xDelete */ + vlogAccess, /* xAccess */ + vlogFullPathname, /* xFullPathname */ + vlogDlOpen, /* xDlOpen */ + vlogDlError, /* xDlError */ + vlogDlSym, /* xDlSym */ + vlogDlClose, /* xDlClose */ + vlogRandomness, /* xRandomness */ + vlogSleep, /* xSleep */ + vlogCurrentTime, /* xCurrentTime */ + vlogGetLastError, /* xGetLastError */ + vlogCurrentTimeInt64 /* xCurrentTimeInt64 */ + }, + 0 +}; + +static sqlite3_io_methods vlog_io_methods = { + 1, /* iVersion */ + vlogClose, /* xClose */ + vlogRead, /* xRead */ + vlogWrite, /* xWrite */ + vlogTruncate, /* xTruncate */ + vlogSync, /* xSync */ + vlogFileSize, /* xFileSize */ + vlogLock, /* xLock */ + vlogUnlock, /* xUnlock */ + vlogCheckReservedLock, /* xCheckReservedLock */ + vlogFileControl, /* xFileControl */ + vlogSectorSize, /* xSectorSize */ + vlogDeviceCharacteristics, /* xDeviceCharacteristics */ + 0, /* xShmMap */ + 0, /* xShmLock */ + 0, /* xShmBarrier */ + 0 /* xShmUnmap */ +}; + +#if SQLITE_OS_UNIX && !defined(NO_GETTOD) +#include +static sqlite3_uint64 vlog_time(){ + struct timeval sTime; + gettimeofday(&sTime, 0); + return sTime.tv_usec + (sqlite3_uint64)sTime.tv_sec * 1000000; +} +#elif SQLITE_OS_WIN +#include +#include +static sqlite3_uint64 vlog_time(){ + FILETIME ft; + sqlite3_uint64 u64time = 0; + + GetSystemTimeAsFileTime(&ft); + + u64time |= ft.dwHighDateTime; + u64time <<= 32; + u64time |= ft.dwLowDateTime; + + /* ft is 100-nanosecond intervals, we want microseconds */ + return u64time /(sqlite3_uint64)10; +} +#else +static sqlite3_uint64 vlog_time(){ + return 0; +} +#endif + + +/* +** Write a message to the log file +*/ +static void vlogLogPrint( + VLogLog *pLog, /* The log file to write into */ + sqlite3_int64 tStart, /* Start time of system call */ + sqlite3_int64 tElapse, /* Elapse time of system call */ + const char *zOp, /* Type of system call */ + sqlite3_int64 iArg1, /* First argument */ + sqlite3_int64 iArg2, /* Second argument */ + const char *zArg3, /* Third argument */ + int iRes /* Result */ +){ + char z1[40], z2[40], z3[2000]; + if( pLog==0 ) return; + if( iArg1>=0 ){ + sqlite3_snprintf(sizeof(z1), z1, "%lld", iArg1); + }else{ + z1[0] = 0; + } + if( iArg2>=0 ){ + sqlite3_snprintf(sizeof(z2), z2, "%lld", iArg2); + }else{ + z2[0] = 0; + } + if( zArg3 ){ + sqlite3_snprintf(sizeof(z3), z3, "\"%.*w\"", sizeof(z3)-4, zArg3); + }else{ + z3[0] = 0; + } + fprintf(pLog->out,"%lld,%lld,%s,%d,%s,%s,%s,%d\n", + tStart, tElapse, zOp, pLog->zFilename==0, z1, z2, z3, iRes); +} + +/* +** List of all active log connections. Protected by the master mutex. +*/ +static VLogLog *allLogs = 0; + +/* +** Close a VLogLog object +*/ +static void vlogLogClose(VLogLog *p){ + if( p ){ + sqlite3_mutex *pMutex; + p->nRef--; + if( p->nRef>0 || p->zFilename==0 ) return; + pMutex = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER); + sqlite3_mutex_enter(pMutex); + *p->ppPrev = p->pNext; + if( p->pNext ) p->pNext->ppPrev = p->ppPrev; + sqlite3_mutex_leave(pMutex); + fclose(p->out); + sqlite3_free(p); + } +} + +/* +** Open a VLogLog object on the given file +*/ +static VLogLog *vlogLogOpen(const char *zFilename){ + int nName = (int)strlen(zFilename); + int isJournal = 0; + sqlite3_mutex *pMutex; + VLogLog *pLog, *pTemp; + sqlite3_int64 tNow = 0; + if( nName>4 && strcmp(zFilename+nName-4,"-wal")==0 ){ + return 0; /* Do not log wal files */ + }else + if( nName>8 && strcmp(zFilename+nName-8,"-journal")==0 ){ + nName -= 8; + isJournal = 1; + }else if( nName>12 + && sqlite3_strglob("-mj??????9??", zFilename+nName-12)==0 ){ + return 0; /* Do not log master journal files */ + } + pTemp = sqlite3_malloc( sizeof(*pLog)*2 + nName + 60 ); + if( pTemp==0 ) return 0; + pMutex = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER); + sqlite3_mutex_enter(pMutex); + for(pLog=allLogs; pLog; pLog=pLog->pNext){ + if( pLog->nFilename==nName && !memcmp(pLog->zFilename, zFilename, nName) ){ + break; + } + } + if( pLog==0 ){ + pLog = pTemp; + pTemp = 0; + memset(pLog, 0, sizeof(*pLog)*2); + pLog->zFilename = (char*)&pLog[2]; + tNow = vlog_time(); + sqlite3_snprintf(nName+60, pLog->zFilename, "%.*s-debuglog-%lld", + nName, zFilename, tNow); + pLog->out = fopen(pLog->zFilename, "a"); + if( pLog->out==0 ){ + sqlite3_mutex_leave(pMutex); + sqlite3_free(pLog); + return 0; + } + pLog->nFilename = nName; + pLog[1].out = pLog[0].out; + pLog->ppPrev = &allLogs; + if( allLogs ) allLogs->ppPrev = &pLog->pNext; + pLog->pNext = allLogs; + allLogs = pLog; + } + sqlite3_mutex_leave(pMutex); + if( pTemp ){ + sqlite3_free(pTemp); + }else{ +#if SQLITE_OS_UNIX + char zHost[200]; + zHost[0] = 0; + gethostname(zHost, sizeof(zHost)-1); + zHost[sizeof(zHost)-1] = 0; + vlogLogPrint(pLog, tNow, 0, "IDENT", getpid(), -1, zHost, 0); +#endif + } + if( pLog && isJournal ) pLog++; + pLog->nRef++; + return pLog; +} + + +/* +** Close an vlog-file. +*/ +static int vlogClose(sqlite3_file *pFile){ + sqlite3_uint64 tStart, tElapse; + int rc = SQLITE_OK; + VLogFile *p = (VLogFile *)pFile; + + tStart = vlog_time(); + if( p->pReal->pMethods ){ + rc = p->pReal->pMethods->xClose(p->pReal); + } + tElapse = vlog_time() - tStart; + vlogLogPrint(p->pLog, tStart, tElapse, "CLOSE", -1, -1, 0, rc); + vlogLogClose(p->pLog); + return rc; +} + +/* +** Compute signature for a block of content. +** +** For blocks of 16 or fewer bytes, the signature is just a hex dump of +** the entire block. +** +** For blocks of more than 16 bytes, the signature is a hex dump of the +** first 8 bytes followed by a 64-bit has of the entire block. +*/ +static void vlogSignature(unsigned char *p, int n, char *zCksum){ + unsigned int s0 = 0, s1 = 0; + unsigned int *pI; + int i; + if( n<=16 ){ + for(i=0; ipReal->pMethods->xRead(p->pReal, zBuf, iAmt, iOfst); + tElapse = vlog_time() - tStart; + if( rc==SQLITE_OK ){ + vlogSignature(zBuf, iAmt, zSig); + }else{ + zSig[0] = 0; + } + vlogLogPrint(p->pLog, tStart, tElapse, "READ", iAmt, iOfst, zSig, rc); + if( rc==SQLITE_OK + && p->pLog + && p->pLog->zFilename + && iOfst<=24 + && iOfst+iAmt>=28 + ){ + unsigned char *x = ((unsigned char*)zBuf)+(24-iOfst); + unsigned iCtr, nFree = -1; + char *zFree = 0; + char zStr[12]; + iCtr = bigToNative(x); + if( iOfst+iAmt>=40 ){ + zFree = zStr; + sqlite3_snprintf(sizeof(zStr), zStr, "%d", bigToNative(x+8)); + nFree = bigToNative(x+12); + } + vlogLogPrint(p->pLog, tStart, 0, "CHNGCTR-READ", iCtr, nFree, zFree, 0); + } + return rc; +} + +/* +** Write data to an vlog-file. +*/ +static int vlogWrite( + sqlite3_file *pFile, + const void *z, + int iAmt, + sqlite_int64 iOfst +){ + int rc; + sqlite3_uint64 tStart, tElapse; + VLogFile *p = (VLogFile *)pFile; + char zSig[40]; + + tStart = vlog_time(); + vlogSignature((unsigned char*)z, iAmt, zSig); + rc = p->pReal->pMethods->xWrite(p->pReal, z, iAmt, iOfst); + tElapse = vlog_time() - tStart; + vlogLogPrint(p->pLog, tStart, tElapse, "WRITE", iAmt, iOfst, zSig, rc); + if( rc==SQLITE_OK + && p->pLog + && p->pLog->zFilename + && iOfst<=24 + && iOfst+iAmt>=28 + ){ + unsigned char *x = ((unsigned char*)z)+(24-iOfst); + unsigned iCtr, nFree = -1; + char *zFree = 0; + char zStr[12]; + iCtr = bigToNative(x); + if( iOfst+iAmt>=40 ){ + zFree = zStr; + sqlite3_snprintf(sizeof(zStr), zStr, "%d", bigToNative(x+8)); + nFree = bigToNative(x+12); + } + vlogLogPrint(p->pLog, tStart, 0, "CHNGCTR-WRITE", iCtr, nFree, zFree, 0); + } + return rc; +} + +/* +** Truncate an vlog-file. +*/ +static int vlogTruncate(sqlite3_file *pFile, sqlite_int64 size){ + int rc; + sqlite3_uint64 tStart, tElapse; + VLogFile *p = (VLogFile *)pFile; + tStart = vlog_time(); + rc = p->pReal->pMethods->xTruncate(p->pReal, size); + tElapse = vlog_time() - tStart; + vlogLogPrint(p->pLog, tStart, tElapse, "TRUNCATE", size, -1, 0, rc); + return rc; +} + +/* +** Sync an vlog-file. +*/ +static int vlogSync(sqlite3_file *pFile, int flags){ + int rc; + sqlite3_uint64 tStart, tElapse; + VLogFile *p = (VLogFile *)pFile; + tStart = vlog_time(); + rc = p->pReal->pMethods->xSync(p->pReal, flags); + tElapse = vlog_time() - tStart; + vlogLogPrint(p->pLog, tStart, tElapse, "SYNC", flags, -1, 0, rc); + return rc; +} + +/* +** Return the current file-size of an vlog-file. +*/ +static int vlogFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){ + int rc; + sqlite3_uint64 tStart, tElapse; + VLogFile *p = (VLogFile *)pFile; + tStart = vlog_time(); + rc = p->pReal->pMethods->xFileSize(p->pReal, pSize); + tElapse = vlog_time() - tStart; + vlogLogPrint(p->pLog, tStart, tElapse, "FILESIZE", *pSize, -1, 0, rc); + return rc; +} + +/* +** Lock an vlog-file. +*/ +static int vlogLock(sqlite3_file *pFile, int eLock){ + int rc; + sqlite3_uint64 tStart, tElapse; + VLogFile *p = (VLogFile *)pFile; + tStart = vlog_time(); + rc = p->pReal->pMethods->xLock(p->pReal, eLock); + tElapse = vlog_time() - tStart; + vlogLogPrint(p->pLog, tStart, tElapse, "LOCK", eLock, -1, 0, rc); + return rc; +} + +/* +** Unlock an vlog-file. +*/ +static int vlogUnlock(sqlite3_file *pFile, int eLock){ + int rc; + sqlite3_uint64 tStart; + VLogFile *p = (VLogFile *)pFile; + tStart = vlog_time(); + vlogLogPrint(p->pLog, tStart, 0, "UNLOCK", eLock, -1, 0, 0); + rc = p->pReal->pMethods->xUnlock(p->pReal, eLock); + return rc; +} + +/* +** Check if another file-handle holds a RESERVED lock on an vlog-file. +*/ +static int vlogCheckReservedLock(sqlite3_file *pFile, int *pResOut){ + int rc; + sqlite3_uint64 tStart, tElapse; + VLogFile *p = (VLogFile *)pFile; + tStart = vlog_time(); + rc = p->pReal->pMethods->xCheckReservedLock(p->pReal, pResOut); + tElapse = vlog_time() - tStart; + vlogLogPrint(p->pLog, tStart, tElapse, "CHECKRESERVEDLOCK", + *pResOut, -1, "", rc); + return rc; +} + +/* +** File control method. For custom operations on an vlog-file. +*/ +static int vlogFileControl(sqlite3_file *pFile, int op, void *pArg){ + VLogFile *p = (VLogFile *)pFile; + sqlite3_uint64 tStart, tElapse; + int rc; + tStart = vlog_time(); + rc = p->pReal->pMethods->xFileControl(p->pReal, op, pArg); + if( op==SQLITE_FCNTL_VFSNAME && rc==SQLITE_OK ){ + *(char**)pArg = sqlite3_mprintf("vlog/%z", *(char**)pArg); + } + tElapse = vlog_time() - tStart; + if( op==SQLITE_FCNTL_TRACE ){ + vlogLogPrint(p->pLog, tStart, tElapse, "TRACE", op, -1, pArg, rc); + }else if( op==SQLITE_FCNTL_PRAGMA ){ + const char **azArg = (const char **)pArg; + vlogLogPrint(p->pLog, tStart, tElapse, "FILECONTROL", op, -1, azArg[1], rc); + }else if( op==SQLITE_FCNTL_SIZE_HINT ){ + sqlite3_int64 sz = *(sqlite3_int64*)pArg; + vlogLogPrint(p->pLog, tStart, tElapse, "FILECONTROL", op, sz, 0, rc); + }else{ + vlogLogPrint(p->pLog, tStart, tElapse, "FILECONTROL", op, -1, 0, rc); + } + return rc; +} + +/* +** Return the sector-size in bytes for an vlog-file. +*/ +static int vlogSectorSize(sqlite3_file *pFile){ + int rc; + sqlite3_uint64 tStart, tElapse; + VLogFile *p = (VLogFile *)pFile; + tStart = vlog_time(); + rc = p->pReal->pMethods->xSectorSize(p->pReal); + tElapse = vlog_time() - tStart; + vlogLogPrint(p->pLog, tStart, tElapse, "SECTORSIZE", -1, -1, 0, rc); + return rc; +} + +/* +** Return the device characteristic flags supported by an vlog-file. +*/ +static int vlogDeviceCharacteristics(sqlite3_file *pFile){ + int rc; + sqlite3_uint64 tStart, tElapse; + VLogFile *p = (VLogFile *)pFile; + tStart = vlog_time(); + rc = p->pReal->pMethods->xDeviceCharacteristics(p->pReal); + tElapse = vlog_time() - tStart; + vlogLogPrint(p->pLog, tStart, tElapse, "DEVCHAR", -1, -1, 0, rc); + return rc; +} + + +/* +** Open an vlog file handle. +*/ +static int vlogOpen( + sqlite3_vfs *pVfs, + const char *zName, + sqlite3_file *pFile, + int flags, + int *pOutFlags +){ + int rc; + sqlite3_uint64 tStart, tElapse; + sqlite3_int64 iArg2; + VLogFile *p = (VLogFile*)pFile; + + p->pReal = (sqlite3_file*)&p[1]; + if( (flags & (SQLITE_OPEN_MAIN_DB|SQLITE_OPEN_MAIN_JOURNAL))!=0 ){ + p->pLog = vlogLogOpen(zName); + }else{ + p->pLog = 0; + } + tStart = vlog_time(); + rc = REALVFS(pVfs)->xOpen(REALVFS(pVfs), zName, p->pReal, flags, pOutFlags); + tElapse = vlog_time() - tStart; + iArg2 = pOutFlags ? *pOutFlags : -1; + vlogLogPrint(p->pLog, tStart, tElapse, "OPEN", flags, iArg2, 0, rc); + if( rc==SQLITE_OK ){ + pFile->pMethods = &vlog_io_methods; + }else{ + if( p->pLog ) vlogLogClose(p->pLog); + p->pLog = 0; + } + return rc; +} + +/* +** Delete the file located at zPath. If the dirSync argument is true, +** ensure the file-system modifications are synced to disk before +** returning. +*/ +static int vlogDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ + int rc; + sqlite3_uint64 tStart, tElapse; + VLogLog *pLog; + tStart = vlog_time(); + rc = REALVFS(pVfs)->xDelete(REALVFS(pVfs), zPath, dirSync); + tElapse = vlog_time() - tStart; + pLog = vlogLogOpen(zPath); + vlogLogPrint(pLog, tStart, tElapse, "DELETE", dirSync, -1, 0, rc); + vlogLogClose(pLog); + return rc; +} + +/* +** Test for access permissions. Return true if the requested permission +** is available, or false otherwise. +*/ +static int vlogAccess( + sqlite3_vfs *pVfs, + const char *zPath, + int flags, + int *pResOut +){ + int rc; + sqlite3_uint64 tStart, tElapse; + VLogLog *pLog; + tStart = vlog_time(); + rc = REALVFS(pVfs)->xAccess(REALVFS(pVfs), zPath, flags, pResOut); + tElapse = vlog_time() - tStart; + pLog = vlogLogOpen(zPath); + vlogLogPrint(pLog, tStart, tElapse, "ACCESS", flags, *pResOut, 0, rc); + vlogLogClose(pLog); + return rc; +} + +/* +** Populate buffer zOut with the full canonical pathname corresponding +** to the pathname in zPath. zOut is guaranteed to point to a buffer +** of at least (INST_MAX_PATHNAME+1) bytes. +*/ +static int vlogFullPathname( + sqlite3_vfs *pVfs, + const char *zPath, + int nOut, + char *zOut +){ + return REALVFS(pVfs)->xFullPathname(REALVFS(pVfs), zPath, nOut, zOut); +} + +/* +** Open the dynamic library located at zPath and return a handle. +*/ +static void *vlogDlOpen(sqlite3_vfs *pVfs, const char *zPath){ + return REALVFS(pVfs)->xDlOpen(REALVFS(pVfs), zPath); +} + +/* +** Populate the buffer zErrMsg (size nByte bytes) with a human readable +** utf-8 string describing the most recent error encountered associated +** with dynamic libraries. +*/ +static void vlogDlError(sqlite3_vfs *pVfs, int nByte, char *zErrMsg){ + REALVFS(pVfs)->xDlError(REALVFS(pVfs), nByte, zErrMsg); +} + +/* +** Return a pointer to the symbol zSymbol in the dynamic library pHandle. +*/ +static void (*vlogDlSym(sqlite3_vfs *pVfs, void *p, const char *zSym))(void){ + return REALVFS(pVfs)->xDlSym(REALVFS(pVfs), p, zSym); +} + +/* +** Close the dynamic library handle pHandle. +*/ +static void vlogDlClose(sqlite3_vfs *pVfs, void *pHandle){ + REALVFS(pVfs)->xDlClose(REALVFS(pVfs), pHandle); +} + +/* +** Populate the buffer pointed to by zBufOut with nByte bytes of +** random data. +*/ +static int vlogRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){ + return REALVFS(pVfs)->xRandomness(REALVFS(pVfs), nByte, zBufOut); +} + +/* +** Sleep for nMicro microseconds. Return the number of microseconds +** actually slept. +*/ +static int vlogSleep(sqlite3_vfs *pVfs, int nMicro){ + return REALVFS(pVfs)->xSleep(REALVFS(pVfs), nMicro); +} + +/* +** Return the current time as a Julian Day number in *pTimeOut. +*/ +static int vlogCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){ + return REALVFS(pVfs)->xCurrentTime(REALVFS(pVfs), pTimeOut); +} + +static int vlogGetLastError(sqlite3_vfs *pVfs, int a, char *b){ + return REALVFS(pVfs)->xGetLastError(REALVFS(pVfs), a, b); +} +static int vlogCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *p){ + return REALVFS(pVfs)->xCurrentTimeInt64(REALVFS(pVfs), p); +} + +/* +** Register debugvfs as the default VFS for this process. +*/ +int sqlite3_register_vfslog(const char *zArg){ + vlog_vfs.pVfs = sqlite3_vfs_find(0); + vlog_vfs.base.szOsFile = sizeof(VLogFile) + vlog_vfs.pVfs->szOsFile; + return sqlite3_vfs_register(&vlog_vfs.base, 1); +} diff --git a/ext/misc/vtshim.c b/ext/misc/vtshim.c new file mode 100644 index 0000000..01348e8 --- /dev/null +++ b/ext/misc/vtshim.c @@ -0,0 +1,551 @@ +/* +** 2013-06-12 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** A shim that sits between the SQLite virtual table interface and +** runtimes with garbage collector based memory management. +*/ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#include +#include + +#ifndef SQLITE_OMIT_VIRTUALTABLE + +/* Forward references */ +typedef struct vtshim_aux vtshim_aux; +typedef struct vtshim_vtab vtshim_vtab; +typedef struct vtshim_cursor vtshim_cursor; + + +/* The vtshim_aux argument is the auxiliary parameter that is passed +** into sqlite3_create_module_v2(). +*/ +struct vtshim_aux { + void *pChildAux; /* pAux for child virtual tables */ + void (*xChildDestroy)(void*); /* Destructor for pChildAux */ + sqlite3_module *pMod; /* Methods for child virtual tables */ + sqlite3 *db; /* The database to which we are attached */ + char *zName; /* Name of the module */ + int bDisposed; /* True if disposed */ + vtshim_vtab *pAllVtab; /* List of all vtshim_vtab objects */ + sqlite3_module sSelf; /* Methods used by this shim */ +}; + +/* A vtshim virtual table object */ +struct vtshim_vtab { + sqlite3_vtab base; /* Base class - must be first */ + sqlite3_vtab *pChild; /* Child virtual table */ + vtshim_aux *pAux; /* Pointer to vtshim_aux object */ + vtshim_cursor *pAllCur; /* List of all cursors */ + vtshim_vtab **ppPrev; /* Previous on list */ + vtshim_vtab *pNext; /* Next on list */ +}; + +/* A vtshim cursor object */ +struct vtshim_cursor { + sqlite3_vtab_cursor base; /* Base class - must be first */ + sqlite3_vtab_cursor *pChild; /* Cursor generated by the managed subclass */ + vtshim_cursor **ppPrev; /* Previous on list of all cursors */ + vtshim_cursor *pNext; /* Next on list of all cursors */ +}; + +/* Macro used to copy the child vtable error message to outer vtable */ +#define VTSHIM_COPY_ERRMSG() \ + do { \ + sqlite3_free(pVtab->base.zErrMsg); \ + pVtab->base.zErrMsg = sqlite3_mprintf("%s", pVtab->pChild->zErrMsg); \ + } while (0) + +/* Methods for the vtshim module */ +static int vtshimCreate( + sqlite3 *db, + void *ppAux, + int argc, + const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + vtshim_aux *pAux = (vtshim_aux*)ppAux; + vtshim_vtab *pNew; + int rc; + + assert( db==pAux->db ); + if( pAux->bDisposed ){ + if( pzErr ){ + *pzErr = sqlite3_mprintf("virtual table was disposed: \"%s\"", + pAux->zName); + } + return SQLITE_ERROR; + } + pNew = sqlite3_malloc( sizeof(*pNew) ); + *ppVtab = (sqlite3_vtab*)pNew; + if( pNew==0 ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(*pNew)); + rc = pAux->pMod->xCreate(db, pAux->pChildAux, argc, argv, + &pNew->pChild, pzErr); + if( rc ){ + sqlite3_free(pNew); + *ppVtab = 0; + } + pNew->pAux = pAux; + pNew->ppPrev = &pAux->pAllVtab; + pNew->pNext = pAux->pAllVtab; + if( pAux->pAllVtab ) pAux->pAllVtab->ppPrev = &pNew->pNext; + pAux->pAllVtab = pNew; + return rc; +} + +static int vtshimConnect( + sqlite3 *db, + void *ppAux, + int argc, + const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + vtshim_aux *pAux = (vtshim_aux*)ppAux; + vtshim_vtab *pNew; + int rc; + + assert( db==pAux->db ); + if( pAux->bDisposed ){ + if( pzErr ){ + *pzErr = sqlite3_mprintf("virtual table was disposed: \"%s\"", + pAux->zName); + } + return SQLITE_ERROR; + } + pNew = sqlite3_malloc( sizeof(*pNew) ); + *ppVtab = (sqlite3_vtab*)pNew; + if( pNew==0 ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(*pNew)); + rc = pAux->pMod->xConnect(db, pAux->pChildAux, argc, argv, + &pNew->pChild, pzErr); + if( rc ){ + sqlite3_free(pNew); + *ppVtab = 0; + } + pNew->pAux = pAux; + pNew->ppPrev = &pAux->pAllVtab; + pNew->pNext = pAux->pAllVtab; + if( pAux->pAllVtab ) pAux->pAllVtab->ppPrev = &pNew->pNext; + pAux->pAllVtab = pNew; + return rc; +} + +static int vtshimBestIndex( + sqlite3_vtab *pBase, + sqlite3_index_info *pIdxInfo +){ + vtshim_vtab *pVtab = (vtshim_vtab*)pBase; + vtshim_aux *pAux = pVtab->pAux; + int rc; + if( pAux->bDisposed ) return SQLITE_ERROR; + rc = pAux->pMod->xBestIndex(pVtab->pChild, pIdxInfo); + if( rc!=SQLITE_OK ){ + VTSHIM_COPY_ERRMSG(); + } + return rc; +} + +static int vtshimDisconnect(sqlite3_vtab *pBase){ + vtshim_vtab *pVtab = (vtshim_vtab*)pBase; + vtshim_aux *pAux = pVtab->pAux; + int rc = SQLITE_OK; + if( !pAux->bDisposed ){ + rc = pAux->pMod->xDisconnect(pVtab->pChild); + } + if( pVtab->pNext ) pVtab->pNext->ppPrev = pVtab->ppPrev; + *pVtab->ppPrev = pVtab->pNext; + sqlite3_free(pVtab); + return rc; +} + +static int vtshimDestroy(sqlite3_vtab *pBase){ + vtshim_vtab *pVtab = (vtshim_vtab*)pBase; + vtshim_aux *pAux = pVtab->pAux; + int rc = SQLITE_OK; + if( !pAux->bDisposed ){ + rc = pAux->pMod->xDestroy(pVtab->pChild); + } + if( pVtab->pNext ) pVtab->pNext->ppPrev = pVtab->ppPrev; + *pVtab->ppPrev = pVtab->pNext; + sqlite3_free(pVtab); + return rc; +} + +static int vtshimOpen(sqlite3_vtab *pBase, sqlite3_vtab_cursor **ppCursor){ + vtshim_vtab *pVtab = (vtshim_vtab*)pBase; + vtshim_aux *pAux = pVtab->pAux; + vtshim_cursor *pCur; + int rc; + *ppCursor = 0; + if( pAux->bDisposed ) return SQLITE_ERROR; + pCur = sqlite3_malloc( sizeof(*pCur) ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + rc = pAux->pMod->xOpen(pVtab->pChild, &pCur->pChild); + if( rc ){ + sqlite3_free(pCur); + VTSHIM_COPY_ERRMSG(); + return rc; + } + pCur->pChild->pVtab = pVtab->pChild; + *ppCursor = &pCur->base; + pCur->ppPrev = &pVtab->pAllCur; + if( pVtab->pAllCur ) pVtab->pAllCur->ppPrev = &pCur->pNext; + pCur->pNext = pVtab->pAllCur; + pVtab->pAllCur = pCur; + return SQLITE_OK; +} + +static int vtshimClose(sqlite3_vtab_cursor *pX){ + vtshim_cursor *pCur = (vtshim_cursor*)pX; + vtshim_vtab *pVtab = (vtshim_vtab*)pCur->base.pVtab; + vtshim_aux *pAux = pVtab->pAux; + int rc = SQLITE_OK; + if( !pAux->bDisposed ){ + rc = pAux->pMod->xClose(pCur->pChild); + if( rc!=SQLITE_OK ){ + VTSHIM_COPY_ERRMSG(); + } + } + if( pCur->pNext ) pCur->pNext->ppPrev = pCur->ppPrev; + *pCur->ppPrev = pCur->pNext; + sqlite3_free(pCur); + return rc; +} + +static int vtshimFilter( + sqlite3_vtab_cursor *pX, + int idxNum, + const char *idxStr, + int argc, + sqlite3_value **argv +){ + vtshim_cursor *pCur = (vtshim_cursor*)pX; + vtshim_vtab *pVtab = (vtshim_vtab*)pCur->base.pVtab; + vtshim_aux *pAux = pVtab->pAux; + int rc; + if( pAux->bDisposed ) return SQLITE_ERROR; + rc = pAux->pMod->xFilter(pCur->pChild, idxNum, idxStr, argc, argv); + if( rc!=SQLITE_OK ){ + VTSHIM_COPY_ERRMSG(); + } + return rc; +} + +static int vtshimNext(sqlite3_vtab_cursor *pX){ + vtshim_cursor *pCur = (vtshim_cursor*)pX; + vtshim_vtab *pVtab = (vtshim_vtab*)pCur->base.pVtab; + vtshim_aux *pAux = pVtab->pAux; + int rc; + if( pAux->bDisposed ) return SQLITE_ERROR; + rc = pAux->pMod->xNext(pCur->pChild); + if( rc!=SQLITE_OK ){ + VTSHIM_COPY_ERRMSG(); + } + return rc; +} + +static int vtshimEof(sqlite3_vtab_cursor *pX){ + vtshim_cursor *pCur = (vtshim_cursor*)pX; + vtshim_vtab *pVtab = (vtshim_vtab*)pCur->base.pVtab; + vtshim_aux *pAux = pVtab->pAux; + int rc; + if( pAux->bDisposed ) return 1; + rc = pAux->pMod->xEof(pCur->pChild); + VTSHIM_COPY_ERRMSG(); + return rc; +} + +static int vtshimColumn(sqlite3_vtab_cursor *pX, sqlite3_context *ctx, int i){ + vtshim_cursor *pCur = (vtshim_cursor*)pX; + vtshim_vtab *pVtab = (vtshim_vtab*)pCur->base.pVtab; + vtshim_aux *pAux = pVtab->pAux; + int rc; + if( pAux->bDisposed ) return SQLITE_ERROR; + rc = pAux->pMod->xColumn(pCur->pChild, ctx, i); + if( rc!=SQLITE_OK ){ + VTSHIM_COPY_ERRMSG(); + } + return rc; +} + +static int vtshimRowid(sqlite3_vtab_cursor *pX, sqlite3_int64 *pRowid){ + vtshim_cursor *pCur = (vtshim_cursor*)pX; + vtshim_vtab *pVtab = (vtshim_vtab*)pCur->base.pVtab; + vtshim_aux *pAux = pVtab->pAux; + int rc; + if( pAux->bDisposed ) return SQLITE_ERROR; + rc = pAux->pMod->xRowid(pCur->pChild, pRowid); + if( rc!=SQLITE_OK ){ + VTSHIM_COPY_ERRMSG(); + } + return rc; +} + +static int vtshimUpdate( + sqlite3_vtab *pBase, + int argc, + sqlite3_value **argv, + sqlite3_int64 *pRowid +){ + vtshim_vtab *pVtab = (vtshim_vtab*)pBase; + vtshim_aux *pAux = pVtab->pAux; + int rc; + if( pAux->bDisposed ) return SQLITE_ERROR; + rc = pAux->pMod->xUpdate(pVtab->pChild, argc, argv, pRowid); + if( rc!=SQLITE_OK ){ + VTSHIM_COPY_ERRMSG(); + } + return rc; +} + +static int vtshimBegin(sqlite3_vtab *pBase){ + vtshim_vtab *pVtab = (vtshim_vtab*)pBase; + vtshim_aux *pAux = pVtab->pAux; + int rc; + if( pAux->bDisposed ) return SQLITE_ERROR; + rc = pAux->pMod->xBegin(pVtab->pChild); + if( rc!=SQLITE_OK ){ + VTSHIM_COPY_ERRMSG(); + } + return rc; +} + +static int vtshimSync(sqlite3_vtab *pBase){ + vtshim_vtab *pVtab = (vtshim_vtab*)pBase; + vtshim_aux *pAux = pVtab->pAux; + int rc; + if( pAux->bDisposed ) return SQLITE_ERROR; + rc = pAux->pMod->xSync(pVtab->pChild); + if( rc!=SQLITE_OK ){ + VTSHIM_COPY_ERRMSG(); + } + return rc; +} + +static int vtshimCommit(sqlite3_vtab *pBase){ + vtshim_vtab *pVtab = (vtshim_vtab*)pBase; + vtshim_aux *pAux = pVtab->pAux; + int rc; + if( pAux->bDisposed ) return SQLITE_ERROR; + rc = pAux->pMod->xCommit(pVtab->pChild); + if( rc!=SQLITE_OK ){ + VTSHIM_COPY_ERRMSG(); + } + return rc; +} + +static int vtshimRollback(sqlite3_vtab *pBase){ + vtshim_vtab *pVtab = (vtshim_vtab*)pBase; + vtshim_aux *pAux = pVtab->pAux; + int rc; + if( pAux->bDisposed ) return SQLITE_ERROR; + rc = pAux->pMod->xRollback(pVtab->pChild); + if( rc!=SQLITE_OK ){ + VTSHIM_COPY_ERRMSG(); + } + return rc; +} + +static int vtshimFindFunction( + sqlite3_vtab *pBase, + int nArg, + const char *zName, + void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), + void **ppArg +){ + vtshim_vtab *pVtab = (vtshim_vtab*)pBase; + vtshim_aux *pAux = pVtab->pAux; + int rc; + if( pAux->bDisposed ) return 0; + rc = pAux->pMod->xFindFunction(pVtab->pChild, nArg, zName, pxFunc, ppArg); + VTSHIM_COPY_ERRMSG(); + return rc; +} + +static int vtshimRename(sqlite3_vtab *pBase, const char *zNewName){ + vtshim_vtab *pVtab = (vtshim_vtab*)pBase; + vtshim_aux *pAux = pVtab->pAux; + int rc; + if( pAux->bDisposed ) return SQLITE_ERROR; + rc = pAux->pMod->xRename(pVtab->pChild, zNewName); + if( rc!=SQLITE_OK ){ + VTSHIM_COPY_ERRMSG(); + } + return rc; +} + +static int vtshimSavepoint(sqlite3_vtab *pBase, int n){ + vtshim_vtab *pVtab = (vtshim_vtab*)pBase; + vtshim_aux *pAux = pVtab->pAux; + int rc; + if( pAux->bDisposed ) return SQLITE_ERROR; + rc = pAux->pMod->xSavepoint(pVtab->pChild, n); + if( rc!=SQLITE_OK ){ + VTSHIM_COPY_ERRMSG(); + } + return rc; +} + +static int vtshimRelease(sqlite3_vtab *pBase, int n){ + vtshim_vtab *pVtab = (vtshim_vtab*)pBase; + vtshim_aux *pAux = pVtab->pAux; + int rc; + if( pAux->bDisposed ) return SQLITE_ERROR; + rc = pAux->pMod->xRelease(pVtab->pChild, n); + if( rc!=SQLITE_OK ){ + VTSHIM_COPY_ERRMSG(); + } + return rc; +} + +static int vtshimRollbackTo(sqlite3_vtab *pBase, int n){ + vtshim_vtab *pVtab = (vtshim_vtab*)pBase; + vtshim_aux *pAux = pVtab->pAux; + int rc; + if( pAux->bDisposed ) return SQLITE_ERROR; + rc = pAux->pMod->xRollbackTo(pVtab->pChild, n); + if( rc!=SQLITE_OK ){ + VTSHIM_COPY_ERRMSG(); + } + return rc; +} + +/* The destructor function for a disposible module */ +static void vtshimAuxDestructor(void *pXAux){ + vtshim_aux *pAux = (vtshim_aux*)pXAux; + assert( pAux->pAllVtab==0 ); + if( !pAux->bDisposed && pAux->xChildDestroy ){ + pAux->xChildDestroy(pAux->pChildAux); + pAux->xChildDestroy = 0; + } + sqlite3_free(pAux->zName); + sqlite3_free(pAux->pMod); + sqlite3_free(pAux); +} + +static int vtshimCopyModule( + const sqlite3_module *pMod, /* Source module to be copied */ + sqlite3_module **ppMod /* Destination for copied module */ +){ + sqlite3_module *p; + if( !pMod || !ppMod ) return SQLITE_ERROR; + p = sqlite3_malloc( sizeof(*p) ); + if( p==0 ) return SQLITE_NOMEM; + memcpy(p, pMod, sizeof(*p)); + *ppMod = p; + return SQLITE_OK; +} + +#ifdef _WIN32 +__declspec(dllexport) +#endif +void *sqlite3_create_disposable_module( + sqlite3 *db, /* SQLite connection to register module with */ + const char *zName, /* Name of the module */ + const sqlite3_module *p, /* Methods for the module */ + void *pClientData, /* Client data for xCreate/xConnect */ + void(*xDestroy)(void*) /* Module destructor function */ +){ + vtshim_aux *pAux; + sqlite3_module *pMod; + int rc; + pAux = sqlite3_malloc( sizeof(*pAux) ); + if( pAux==0 ){ + if( xDestroy ) xDestroy(pClientData); + return 0; + } + rc = vtshimCopyModule(p, &pMod); + if( rc!=SQLITE_OK ){ + sqlite3_free(pAux); + return 0; + } + pAux->pChildAux = pClientData; + pAux->xChildDestroy = xDestroy; + pAux->pMod = pMod; + pAux->db = db; + pAux->zName = sqlite3_mprintf("%s", zName); + pAux->bDisposed = 0; + pAux->pAllVtab = 0; + pAux->sSelf.iVersion = p->iVersion<=2 ? p->iVersion : 2; + pAux->sSelf.xCreate = p->xCreate ? vtshimCreate : 0; + pAux->sSelf.xConnect = p->xConnect ? vtshimConnect : 0; + pAux->sSelf.xBestIndex = p->xBestIndex ? vtshimBestIndex : 0; + pAux->sSelf.xDisconnect = p->xDisconnect ? vtshimDisconnect : 0; + pAux->sSelf.xDestroy = p->xDestroy ? vtshimDestroy : 0; + pAux->sSelf.xOpen = p->xOpen ? vtshimOpen : 0; + pAux->sSelf.xClose = p->xClose ? vtshimClose : 0; + pAux->sSelf.xFilter = p->xFilter ? vtshimFilter : 0; + pAux->sSelf.xNext = p->xNext ? vtshimNext : 0; + pAux->sSelf.xEof = p->xEof ? vtshimEof : 0; + pAux->sSelf.xColumn = p->xColumn ? vtshimColumn : 0; + pAux->sSelf.xRowid = p->xRowid ? vtshimRowid : 0; + pAux->sSelf.xUpdate = p->xUpdate ? vtshimUpdate : 0; + pAux->sSelf.xBegin = p->xBegin ? vtshimBegin : 0; + pAux->sSelf.xSync = p->xSync ? vtshimSync : 0; + pAux->sSelf.xCommit = p->xCommit ? vtshimCommit : 0; + pAux->sSelf.xRollback = p->xRollback ? vtshimRollback : 0; + pAux->sSelf.xFindFunction = p->xFindFunction ? vtshimFindFunction : 0; + pAux->sSelf.xRename = p->xRename ? vtshimRename : 0; + if( p->iVersion>=2 ){ + pAux->sSelf.xSavepoint = p->xSavepoint ? vtshimSavepoint : 0; + pAux->sSelf.xRelease = p->xRelease ? vtshimRelease : 0; + pAux->sSelf.xRollbackTo = p->xRollbackTo ? vtshimRollbackTo : 0; + }else{ + pAux->sSelf.xSavepoint = 0; + pAux->sSelf.xRelease = 0; + pAux->sSelf.xRollbackTo = 0; + } + rc = sqlite3_create_module_v2(db, zName, &pAux->sSelf, + pAux, vtshimAuxDestructor); + return rc==SQLITE_OK ? (void*)pAux : 0; +} + +#ifdef _WIN32 +__declspec(dllexport) +#endif +void sqlite3_dispose_module(void *pX){ + vtshim_aux *pAux = (vtshim_aux*)pX; + if( !pAux->bDisposed ){ + vtshim_vtab *pVtab; + vtshim_cursor *pCur; + for(pVtab=pAux->pAllVtab; pVtab; pVtab=pVtab->pNext){ + for(pCur=pVtab->pAllCur; pCur; pCur=pCur->pNext){ + pAux->pMod->xClose(pCur->pChild); + } + pAux->pMod->xDisconnect(pVtab->pChild); + } + pAux->bDisposed = 1; + if( pAux->xChildDestroy ){ + pAux->xChildDestroy(pAux->pChildAux); + pAux->xChildDestroy = 0; + } + } +} + + +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_vtshim_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + SQLITE_EXTENSION_INIT2(pApi); + return SQLITE_OK; +} diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c index 16a316f..8150538 100644 --- a/ext/rtree/rtree.c +++ b/ext/rtree/rtree.c @@ -54,48 +54,6 @@ #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_RTREE) -/* -** This file contains an implementation of a couple of different variants -** of the r-tree algorithm. See the README file for further details. The -** same data-structure is used for all, but the algorithms for insert and -** delete operations vary. The variants used are selected at compile time -** by defining the following symbols: -*/ - -/* Either, both or none of the following may be set to activate -** r*tree variant algorithms. -*/ -#define VARIANT_RSTARTREE_CHOOSESUBTREE 0 -#define VARIANT_RSTARTREE_REINSERT 1 - -/* -** Exactly one of the following must be set to 1. -*/ -#define VARIANT_GUTTMAN_QUADRATIC_SPLIT 0 -#define VARIANT_GUTTMAN_LINEAR_SPLIT 0 -#define VARIANT_RSTARTREE_SPLIT 1 - -#define VARIANT_GUTTMAN_SPLIT \ - (VARIANT_GUTTMAN_LINEAR_SPLIT||VARIANT_GUTTMAN_QUADRATIC_SPLIT) - -#if VARIANT_GUTTMAN_QUADRATIC_SPLIT - #define PickNext QuadraticPickNext - #define PickSeeds QuadraticPickSeeds - #define AssignCells splitNodeGuttman -#endif -#if VARIANT_GUTTMAN_LINEAR_SPLIT - #define PickNext LinearPickNext - #define PickSeeds LinearPickSeeds - #define AssignCells splitNodeGuttman -#endif -#if VARIANT_RSTARTREE_SPLIT - #define AssignCells splitNodeStartree -#endif - -#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) -# define NDEBUG 1 -#endif - #ifndef SQLITE_CORE #include "sqlite3ext.h" SQLITE_EXTENSION_INIT1 @@ -105,11 +63,13 @@ #include #include +#include #ifndef SQLITE_AMALGAMATION #include "sqlite3rtree.h" typedef sqlite3_int64 i64; typedef unsigned char u8; +typedef unsigned short u16; typedef unsigned int u32; #endif @@ -127,6 +87,7 @@ typedef struct RtreeConstraint RtreeConstraint; typedef struct RtreeMatchArg RtreeMatchArg; typedef struct RtreeGeomCallback RtreeGeomCallback; typedef union RtreeCoord RtreeCoord; +typedef struct RtreeSearchPoint RtreeSearchPoint; /* The rtree may have between 1 and RTREE_MAX_DIMENSIONS dimensions. */ #define RTREE_MAX_DIMENSIONS 5 @@ -135,22 +96,33 @@ typedef union RtreeCoord RtreeCoord; ** ever contain very many entries, so a fixed number of buckets is ** used. */ -#define HASHSIZE 128 +#define HASHSIZE 97 + +/* The xBestIndex method of this virtual table requires an estimate of +** the number of rows in the virtual table to calculate the costs of +** various strategies. If possible, this estimate is loaded from the +** sqlite_stat1 table (with RTREE_MIN_ROWEST as a hard-coded minimum). +** Otherwise, if no sqlite_stat1 entry is available, use +** RTREE_DEFAULT_ROWEST. +*/ +#define RTREE_DEFAULT_ROWEST 1048576 +#define RTREE_MIN_ROWEST 100 /* ** An rtree virtual-table object. */ struct Rtree { - sqlite3_vtab base; + sqlite3_vtab base; /* Base class. Must be first */ sqlite3 *db; /* Host database connection */ int iNodeSize; /* Size in bytes of each node in the node table */ - int nDim; /* Number of dimensions */ - int nBytesPerCell; /* Bytes consumed per cell */ + u8 nDim; /* Number of dimensions */ + u8 eCoordType; /* RTREE_COORD_REAL32 or RTREE_COORD_INT32 */ + u8 nBytesPerCell; /* Bytes consumed per cell */ int iDepth; /* Current depth of the r-tree structure */ char *zDb; /* Name of database containing r-tree table */ char *zName; /* Name of r-tree table */ - RtreeNode *aHash[HASHSIZE]; /* Hash table of in-memory nodes. */ int nBusy; /* Current number of users of this structure */ + i64 nRowEst; /* Estimated number of rows in this table */ /* List of nodes removed during a CondenseTree operation. List is ** linked together via the pointer normally used for hash chains - @@ -175,10 +147,10 @@ struct Rtree { sqlite3_stmt *pWriteParent; sqlite3_stmt *pDeleteParent; - int eCoordType; + RtreeNode *aHash[HASHSIZE]; /* Hash table of in-memory nodes. */ }; -/* Possible values for eCoordType: */ +/* Possible values for Rtree.eCoordType: */ #define RTREE_COORD_REAL32 0 #define RTREE_COORD_INT32 1 @@ -190,11 +162,30 @@ struct Rtree { #ifdef SQLITE_RTREE_INT_ONLY typedef sqlite3_int64 RtreeDValue; /* High accuracy coordinate */ typedef int RtreeValue; /* Low accuracy coordinate */ +# define RTREE_ZERO 0 #else typedef double RtreeDValue; /* High accuracy coordinate */ typedef float RtreeValue; /* Low accuracy coordinate */ +# define RTREE_ZERO 0.0 #endif +/* +** When doing a search of an r-tree, instances of the following structure +** record intermediate results from the tree walk. +** +** The id is always a node-id. For iLevel>=1 the id is the node-id of +** the node that the RtreeSearchPoint represents. When iLevel==0, however, +** the id is of the parent node and the cell that RtreeSearchPoint +** represents is the iCell-th entry in the parent node. +*/ +struct RtreeSearchPoint { + RtreeDValue rScore; /* The score for this node. Smallest goes first. */ + sqlite3_int64 id; /* Node ID */ + u8 iLevel; /* 0=entries. 1=leaf node. 2+ for higher */ + u8 eWithin; /* PARTLY_WITHIN or FULLY_WITHIN */ + u8 iCell; /* Cell index within the node */ +}; + /* ** The minimum number of cells allowed for a node is a third of the ** maximum. In Gutman's notation: @@ -217,21 +208,44 @@ struct Rtree { */ #define RTREE_MAX_DEPTH 40 + +/* +** Number of entries in the cursor RtreeNode cache. The first entry is +** used to cache the RtreeNode for RtreeCursor.sPoint. The remaining +** entries cache the RtreeNode for the first elements of the priority queue. +*/ +#define RTREE_CACHE_SZ 5 + /* ** An rtree cursor object. */ struct RtreeCursor { - sqlite3_vtab_cursor base; - RtreeNode *pNode; /* Node cursor is currently pointing at */ - int iCell; /* Index of current cell in pNode */ + sqlite3_vtab_cursor base; /* Base class. Must be first */ + u8 atEOF; /* True if at end of search */ + u8 bPoint; /* True if sPoint is valid */ int iStrategy; /* Copy of idxNum search parameter */ int nConstraint; /* Number of entries in aConstraint */ RtreeConstraint *aConstraint; /* Search constraints. */ + int nPointAlloc; /* Number of slots allocated for aPoint[] */ + int nPoint; /* Number of slots used in aPoint[] */ + int mxLevel; /* iLevel value for root of the tree */ + RtreeSearchPoint *aPoint; /* Priority queue for search points */ + RtreeSearchPoint sPoint; /* Cached next search point */ + RtreeNode *aNode[RTREE_CACHE_SZ]; /* Rtree node cache */ + u32 anQueue[RTREE_MAX_DEPTH+1]; /* Number of queued entries by iLevel */ }; +/* Return the Rtree of a RtreeCursor */ +#define RTREE_OF_CURSOR(X) ((Rtree*)((X)->base.pVtab)) + +/* +** A coordinate can be either a floating point number or a integer. All +** coordinates within a single R-Tree are always of the same time. +*/ union RtreeCoord { - RtreeValue f; - int i; + RtreeValue f; /* Floating point value */ + int i; /* Integer value */ + u32 u; /* Unsigned for byte-order conversions */ }; /* @@ -256,38 +270,67 @@ union RtreeCoord { struct RtreeConstraint { int iCoord; /* Index of constrained coordinate */ int op; /* Constraining operation */ - RtreeDValue rValue; /* Constraint value. */ - int (*xGeom)(sqlite3_rtree_geometry*, int, RtreeDValue*, int*); - sqlite3_rtree_geometry *pGeom; /* Constraint callback argument for a MATCH */ + union { + RtreeDValue rValue; /* Constraint value. */ + int (*xGeom)(sqlite3_rtree_geometry*,int,RtreeDValue*,int*); + int (*xQueryFunc)(sqlite3_rtree_query_info*); + } u; + sqlite3_rtree_query_info *pInfo; /* xGeom and xQueryFunc argument */ }; /* Possible values for RtreeConstraint.op */ -#define RTREE_EQ 0x41 -#define RTREE_LE 0x42 -#define RTREE_LT 0x43 -#define RTREE_GE 0x44 -#define RTREE_GT 0x45 -#define RTREE_MATCH 0x46 +#define RTREE_EQ 0x41 /* A */ +#define RTREE_LE 0x42 /* B */ +#define RTREE_LT 0x43 /* C */ +#define RTREE_GE 0x44 /* D */ +#define RTREE_GT 0x45 /* E */ +#define RTREE_MATCH 0x46 /* F: Old-style sqlite3_rtree_geometry_callback() */ +#define RTREE_QUERY 0x47 /* G: New-style sqlite3_rtree_query_callback() */ + /* ** An rtree structure node. */ struct RtreeNode { - RtreeNode *pParent; /* Parent node */ - i64 iNode; - int nRef; - int isDirty; - u8 *zData; - RtreeNode *pNext; /* Next node in this hash chain */ + RtreeNode *pParent; /* Parent node */ + i64 iNode; /* The node number */ + int nRef; /* Number of references to this node */ + int isDirty; /* True if the node needs to be written to disk */ + u8 *zData; /* Content of the node, as should be on disk */ + RtreeNode *pNext; /* Next node in this hash collision chain */ }; + +/* Return the number of cells in a node */ #define NCELL(pNode) readInt16(&(pNode)->zData[2]) /* -** Structure to store a deserialized rtree record. +** A single cell from a node, deserialized */ struct RtreeCell { - i64 iRowid; - RtreeCoord aCoord[RTREE_MAX_DIMENSIONS*2]; + i64 iRowid; /* Node or entry ID */ + RtreeCoord aCoord[RTREE_MAX_DIMENSIONS*2]; /* Bounding box coordinates */ +}; + + +/* +** This object becomes the sqlite3_user_data() for the SQL functions +** that are created by sqlite3_rtree_geometry_callback() and +** sqlite3_rtree_query_callback() and which appear on the right of MATCH +** operators in order to constrain a search. +** +** xGeom and xQueryFunc are the callback functions. Exactly one of +** xGeom and xQueryFunc fields is non-NULL, depending on whether the +** SQL function was created using sqlite3_rtree_geometry_callback() or +** sqlite3_rtree_query_callback(). +** +** This object is deleted automatically by the destructor mechanism in +** sqlite3_create_function_v2(). +*/ +struct RtreeGeomCallback { + int (*xGeom)(sqlite3_rtree_geometry*, int, RtreeDValue*, int*); + int (*xQueryFunc)(sqlite3_rtree_query_info*); + void (*xDestructor)(void*); + void *pContext; }; @@ -299,29 +342,16 @@ struct RtreeCell { #define RTREE_GEOMETRY_MAGIC 0x891245AB /* -** An instance of this structure must be supplied as a blob argument to -** the right-hand-side of an SQL MATCH operator used to constrain an -** r-tree query. +** An instance of this structure (in the form of a BLOB) is returned by +** the SQL functions that sqlite3_rtree_geometry_callback() and +** sqlite3_rtree_query_callback() create, and is read as the right-hand +** operand to the MATCH operator of an R-Tree. */ struct RtreeMatchArg { - u32 magic; /* Always RTREE_GEOMETRY_MAGIC */ - int (*xGeom)(sqlite3_rtree_geometry *, int, RtreeDValue*, int *); - void *pContext; - int nParam; - RtreeDValue aParam[1]; -}; - -/* -** When a geometry callback is created (see sqlite3_rtree_geometry_callback), -** a single instance of the following structure is allocated. It is used -** as the context for the user-function created by by s_r_g_c(). The object -** is eventually deleted by the destructor mechanism provided by -** sqlite3_create_function_v2() (which is called by s_r_g_c() to create -** the geometry callback function). -*/ -struct RtreeGeomCallback { - int (*xGeom)(sqlite3_rtree_geometry*, int, RtreeDValue*, int*); - void *pContext; + u32 magic; /* Always RTREE_GEOMETRY_MAGIC */ + RtreeGeomCallback cb; /* Info about the callback functions */ + int nParam; /* Number of parameters to the SQL function */ + RtreeDValue aParam[1]; /* Values for parameters to the SQL function */ }; #ifndef MAX @@ -415,10 +445,7 @@ static void nodeZero(Rtree *pRtree, RtreeNode *p){ ** in the Rtree.aHash table. */ static int nodeHash(i64 iNode){ - return ( - (iNode>>56) ^ (iNode>>48) ^ (iNode>>40) ^ (iNode>>32) ^ - (iNode>>24) ^ (iNode>>16) ^ (iNode>> 8) ^ (iNode>> 0) - ) % HASHSIZE; + return iNode % HASHSIZE; } /* @@ -478,8 +505,7 @@ static RtreeNode *nodeNew(Rtree *pRtree, RtreeNode *pParent){ /* ** Obtain a reference to an r-tree node. */ -static int -nodeAcquire( +static int nodeAcquire( Rtree *pRtree, /* R-tree structure */ i64 iNode, /* Node number to load */ RtreeNode *pParent, /* Either the parent node or NULL */ @@ -568,10 +594,10 @@ nodeAcquire( ** Overwrite cell iCell of node pNode with the contents of pCell. */ static void nodeOverwriteCell( - Rtree *pRtree, - RtreeNode *pNode, - RtreeCell *pCell, - int iCell + Rtree *pRtree, /* The overall R-Tree */ + RtreeNode *pNode, /* The node into which the cell is to be written */ + RtreeCell *pCell, /* The cell to write */ + int iCell /* Index into pNode into which pCell is written */ ){ int ii; u8 *p = &pNode->zData[4 + pRtree->nBytesPerCell*iCell]; @@ -583,7 +609,7 @@ static void nodeOverwriteCell( } /* -** Remove cell the cell with index iCell from node pNode. +** Remove the cell with index iCell from node pNode. */ static void nodeDeleteCell(Rtree *pRtree, RtreeNode *pNode, int iCell){ u8 *pDst = &pNode->zData[4 + pRtree->nBytesPerCell*iCell]; @@ -600,11 +626,10 @@ static void nodeDeleteCell(Rtree *pRtree, RtreeNode *pNode, int iCell){ ** ** If there is not enough free space in pNode, return SQLITE_FULL. */ -static int -nodeInsertCell( - Rtree *pRtree, - RtreeNode *pNode, - RtreeCell *pCell +static int nodeInsertCell( + Rtree *pRtree, /* The overall R-Tree */ + RtreeNode *pNode, /* Write new cell into this node */ + RtreeCell *pCell /* The cell to be inserted */ ){ int nCell; /* Current number of cells in pNode */ int nMaxCell; /* Maximum number of cells for pNode */ @@ -625,8 +650,7 @@ nodeInsertCell( /* ** If the node is dirty, write it out to the database. */ -static int -nodeWrite(Rtree *pRtree, RtreeNode *pNode){ +static int nodeWrite(Rtree *pRtree, RtreeNode *pNode){ int rc = SQLITE_OK; if( pNode->isDirty ){ sqlite3_stmt *p = pRtree->pWriteNode; @@ -651,8 +675,7 @@ nodeWrite(Rtree *pRtree, RtreeNode *pNode){ ** Release a reference to a node. If the node is dirty and the reference ** count drops to zero, the node data is written to the database. */ -static int -nodeRelease(Rtree *pRtree, RtreeNode *pNode){ +static int nodeRelease(Rtree *pRtree, RtreeNode *pNode){ int rc = SQLITE_OK; if( pNode ){ assert( pNode->nRef>0 ); @@ -680,9 +703,9 @@ nodeRelease(Rtree *pRtree, RtreeNode *pNode){ ** an internal node, then the 64-bit integer is a child page number. */ static i64 nodeGetRowid( - Rtree *pRtree, - RtreeNode *pNode, - int iCell + Rtree *pRtree, /* The overall R-Tree */ + RtreeNode *pNode, /* The node from which to extract the ID */ + int iCell /* The cell index from which to extract the ID */ ){ assert( iCellzData[4 + pRtree->nBytesPerCell*iCell]); @@ -692,11 +715,11 @@ static i64 nodeGetRowid( ** Return coordinate iCoord from cell iCell in node pNode. */ static void nodeGetCoord( - Rtree *pRtree, - RtreeNode *pNode, - int iCell, - int iCoord, - RtreeCoord *pCoord /* Space to write result to */ + Rtree *pRtree, /* The overall R-Tree */ + RtreeNode *pNode, /* The node from which to extract a coordinate */ + int iCell, /* The index of the cell within the node */ + int iCoord, /* Which coordinate to extract */ + RtreeCoord *pCoord /* OUT: Space to write result to */ ){ readCoord(&pNode->zData[12 + pRtree->nBytesPerCell*iCell + 4*iCoord], pCoord); } @@ -706,15 +729,20 @@ static void nodeGetCoord( ** to by pCell with the results. */ static void nodeGetCell( - Rtree *pRtree, - RtreeNode *pNode, - int iCell, - RtreeCell *pCell + Rtree *pRtree, /* The overall R-Tree */ + RtreeNode *pNode, /* The node containing the cell to be read */ + int iCell, /* Index of the cell within the node */ + RtreeCell *pCell /* OUT: Write the cell contents here */ ){ - int ii; + u8 *pData; + u8 *pEnd; + RtreeCoord *pCoord; pCell->iRowid = nodeGetRowid(pRtree, pNode, iCell); - for(ii=0; iinDim*2; ii++){ - nodeGetCoord(pRtree, pNode, iCell, ii, &pCell->aCoord[ii]); + pData = pNode->zData + (12 + pRtree->nBytesPerCell*iCell); + pEnd = pData + pRtree->nDim*8; + pCoord = pCell->aCoord; + for(; pDataaConstraint ){ int i; /* Used to iterate through constraint array */ for(i=0; inConstraint; i++){ - sqlite3_rtree_geometry *pGeom = pCsr->aConstraint[i].pGeom; - if( pGeom ){ - if( pGeom->xDelUser ) pGeom->xDelUser(pGeom->pUser); - sqlite3_free(pGeom); + sqlite3_rtree_query_info *pInfo = pCsr->aConstraint[i].pInfo; + if( pInfo ){ + if( pInfo->xDelUser ) pInfo->xDelUser(pInfo->pUser); + sqlite3_free(pInfo); } } sqlite3_free(pCsr->aConstraint); @@ -856,12 +884,13 @@ static void freeCursorConstraints(RtreeCursor *pCsr){ */ static int rtreeClose(sqlite3_vtab_cursor *cur){ Rtree *pRtree = (Rtree *)(cur->pVtab); - int rc; + int ii; RtreeCursor *pCsr = (RtreeCursor *)cur; freeCursorConstraints(pCsr); - rc = nodeRelease(pRtree, pCsr->pNode); + sqlite3_free(pCsr->aPoint); + for(ii=0; iiaNode[ii]); sqlite3_free(pCsr); - return rc; + return SQLITE_OK; } /* @@ -872,194 +901,164 @@ static int rtreeClose(sqlite3_vtab_cursor *cur){ */ static int rtreeEof(sqlite3_vtab_cursor *cur){ RtreeCursor *pCsr = (RtreeCursor *)cur; - return (pCsr->pNode==0); + return pCsr->atEOF; } /* -** The r-tree constraint passed as the second argument to this function is -** guaranteed to be a MATCH constraint. -*/ -static int testRtreeGeom( - Rtree *pRtree, /* R-Tree object */ - RtreeConstraint *pConstraint, /* MATCH constraint to test */ - RtreeCell *pCell, /* Cell to test */ - int *pbRes /* OUT: Test result */ -){ - int i; - RtreeDValue aCoord[RTREE_MAX_DIMENSIONS*2]; - int nCoord = pRtree->nDim*2; - - assert( pConstraint->op==RTREE_MATCH ); - assert( pConstraint->pGeom ); - - for(i=0; iaCoord[i]); - } - return pConstraint->xGeom(pConstraint->pGeom, nCoord, aCoord, pbRes); +** Convert raw bits from the on-disk RTree record into a coordinate value. +** The on-disk format is big-endian and needs to be converted for little- +** endian platforms. The on-disk record stores integer coordinates if +** eInt is true and it stores 32-bit floating point records if eInt is +** false. a[] is the four bytes of the on-disk record to be decoded. +** Store the results in "r". +** +** There are three versions of this macro, one each for little-endian and +** big-endian processors and a third generic implementation. The endian- +** specific implementations are much faster and are preferred if the +** processor endianness is known at compile-time. The SQLITE_BYTEORDER +** macro is part of sqliteInt.h and hence the endian-specific +** implementation will only be used if this module is compiled as part +** of the amalgamation. +*/ +#if defined(SQLITE_BYTEORDER) && SQLITE_BYTEORDER==1234 +#define RTREE_DECODE_COORD(eInt, a, r) { \ + RtreeCoord c; /* Coordinate decoded */ \ + memcpy(&c.u,a,4); \ + c.u = ((c.u>>24)&0xff)|((c.u>>8)&0xff00)| \ + ((c.u&0xff)<<24)|((c.u&0xff00)<<8); \ + r = eInt ? (sqlite3_rtree_dbl)c.i : (sqlite3_rtree_dbl)c.f; \ +} +#elif defined(SQLITE_BYTEORDER) && SQLITE_BYTEORDER==4321 +#define RTREE_DECODE_COORD(eInt, a, r) { \ + RtreeCoord c; /* Coordinate decoded */ \ + memcpy(&c.u,a,4); \ + r = eInt ? (sqlite3_rtree_dbl)c.i : (sqlite3_rtree_dbl)c.f; \ } +#else +#define RTREE_DECODE_COORD(eInt, a, r) { \ + RtreeCoord c; /* Coordinate decoded */ \ + c.u = ((u32)a[0]<<24) + ((u32)a[1]<<16) \ + +((u32)a[2]<<8) + a[3]; \ + r = eInt ? (sqlite3_rtree_dbl)c.i : (sqlite3_rtree_dbl)c.f; \ +} +#endif -/* -** Cursor pCursor currently points to a cell in a non-leaf page. -** Set *pbEof to true if the sub-tree headed by the cell is filtered -** (excluded) by the constraints in the pCursor->aConstraint[] -** array, or false otherwise. -** -** Return SQLITE_OK if successful or an SQLite error code if an error -** occurs within a geometry callback. +/* +** Check the RTree node or entry given by pCellData and p against the MATCH +** constraint pConstraint. */ -static int testRtreeCell(Rtree *pRtree, RtreeCursor *pCursor, int *pbEof){ - RtreeCell cell; - int ii; - int bRes = 0; - int rc = SQLITE_OK; - - nodeGetCell(pRtree, pCursor->pNode, pCursor->iCell, &cell); - for(ii=0; bRes==0 && iinConstraint; ii++){ - RtreeConstraint *p = &pCursor->aConstraint[ii]; - RtreeDValue cell_min = DCOORD(cell.aCoord[(p->iCoord>>1)*2]); - RtreeDValue cell_max = DCOORD(cell.aCoord[(p->iCoord>>1)*2+1]); - - assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE - || p->op==RTREE_GT || p->op==RTREE_EQ || p->op==RTREE_MATCH - ); - - switch( p->op ){ - case RTREE_LE: case RTREE_LT: - bRes = p->rValuerValue>cell_max; - break; - - case RTREE_EQ: - bRes = (p->rValue>cell_max || p->rValueop==RTREE_MATCH ); - rc = testRtreeGeom(pRtree, p, &cell, &bRes); - bRes = !bRes; - break; - } +static int rtreeCallbackConstraint( + RtreeConstraint *pConstraint, /* The constraint to test */ + int eInt, /* True if RTree holding integer coordinates */ + u8 *pCellData, /* Raw cell content */ + RtreeSearchPoint *pSearch, /* Container of this cell */ + sqlite3_rtree_dbl *prScore, /* OUT: score for the cell */ + int *peWithin /* OUT: visibility of the cell */ +){ + int i; /* Loop counter */ + sqlite3_rtree_query_info *pInfo = pConstraint->pInfo; /* Callback info */ + int nCoord = pInfo->nCoord; /* No. of coordinates */ + int rc; /* Callback return code */ + sqlite3_rtree_dbl aCoord[RTREE_MAX_DIMENSIONS*2]; /* Decoded coordinates */ + + assert( pConstraint->op==RTREE_MATCH || pConstraint->op==RTREE_QUERY ); + assert( nCoord==2 || nCoord==4 || nCoord==6 || nCoord==8 || nCoord==10 ); + + if( pConstraint->op==RTREE_QUERY && pSearch->iLevel==1 ){ + pInfo->iRowid = readInt64(pCellData); + } + pCellData += 8; + for(i=0; iop==RTREE_MATCH ){ + rc = pConstraint->u.xGeom((sqlite3_rtree_geometry*)pInfo, + nCoord, aCoord, &i); + if( i==0 ) *peWithin = NOT_WITHIN; + *prScore = RTREE_ZERO; + }else{ + pInfo->aCoord = aCoord; + pInfo->iLevel = pSearch->iLevel - 1; + pInfo->rScore = pInfo->rParentScore = pSearch->rScore; + pInfo->eWithin = pInfo->eParentWithin = pSearch->eWithin; + rc = pConstraint->u.xQueryFunc(pInfo); + if( pInfo->eWithin<*peWithin ) *peWithin = pInfo->eWithin; + if( pInfo->rScore<*prScore || *prScorerScore; } } - - *pbEof = bRes; return rc; } /* -** Test if the cell that cursor pCursor currently points to -** would be filtered (excluded) by the constraints in the -** pCursor->aConstraint[] array. If so, set *pbEof to true before -** returning. If the cell is not filtered (excluded) by the constraints, -** set pbEof to zero. -** -** Return SQLITE_OK if successful or an SQLite error code if an error -** occurs within a geometry callback. -** -** This function assumes that the cell is part of a leaf node. -*/ -static int testRtreeEntry(Rtree *pRtree, RtreeCursor *pCursor, int *pbEof){ - RtreeCell cell; - int ii; - *pbEof = 0; - - nodeGetCell(pRtree, pCursor->pNode, pCursor->iCell, &cell); - for(ii=0; iinConstraint; ii++){ - RtreeConstraint *p = &pCursor->aConstraint[ii]; - RtreeDValue coord = DCOORD(cell.aCoord[p->iCoord]); - int res; - assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE - || p->op==RTREE_GT || p->op==RTREE_EQ || p->op==RTREE_MATCH - ); - switch( p->op ){ - case RTREE_LE: res = (coord<=p->rValue); break; - case RTREE_LT: res = (coordrValue); break; - case RTREE_GE: res = (coord>=p->rValue); break; - case RTREE_GT: res = (coord>p->rValue); break; - case RTREE_EQ: res = (coord==p->rValue); break; - default: { - int rc; - assert( p->op==RTREE_MATCH ); - rc = testRtreeGeom(pRtree, p, &cell, &res); - if( rc!=SQLITE_OK ){ - return rc; - } - break; - } - } - - if( !res ){ - *pbEof = 1; - return SQLITE_OK; - } - } - - return SQLITE_OK; -} - -/* -** Cursor pCursor currently points at a node that heads a sub-tree of -** height iHeight (if iHeight==0, then the node is a leaf). Descend -** to point to the left-most cell of the sub-tree that matches the -** configured constraints. -*/ -static int descendToCell( - Rtree *pRtree, - RtreeCursor *pCursor, - int iHeight, - int *pEof /* OUT: Set to true if cannot descend */ +** Check the internal RTree node given by pCellData against constraint p. +** If this constraint cannot be satisfied by any child within the node, +** set *peWithin to NOT_WITHIN. +*/ +static void rtreeNonleafConstraint( + RtreeConstraint *p, /* The constraint to test */ + int eInt, /* True if RTree holds integer coordinates */ + u8 *pCellData, /* Raw cell content as appears on disk */ + int *peWithin /* Adjust downward, as appropriate */ ){ - int isEof; - int rc; - int ii; - RtreeNode *pChild; - sqlite3_int64 iRowid; - - RtreeNode *pSavedNode = pCursor->pNode; - int iSavedCell = pCursor->iCell; + sqlite3_rtree_dbl val; /* Coordinate value convert to a double */ - assert( iHeight>=0 ); + /* p->iCoord might point to either a lower or upper bound coordinate + ** in a coordinate pair. But make pCellData point to the lower bound. + */ + pCellData += 8 + 4*(p->iCoord&0xfe); - if( iHeight==0 ){ - rc = testRtreeEntry(pRtree, pCursor, &isEof); - }else{ - rc = testRtreeCell(pRtree, pCursor, &isEof); - } - if( rc!=SQLITE_OK || isEof || iHeight==0 ){ - goto descend_to_cell_out; - } + assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE + || p->op==RTREE_GT || p->op==RTREE_EQ ); + switch( p->op ){ + case RTREE_LE: + case RTREE_LT: + case RTREE_EQ: + RTREE_DECODE_COORD(eInt, pCellData, val); + /* val now holds the lower bound of the coordinate pair */ + if( p->u.rValue>=val ) return; + if( p->op!=RTREE_EQ ) break; /* RTREE_LE and RTREE_LT end here */ + /* Fall through for the RTREE_EQ case */ - iRowid = nodeGetRowid(pRtree, pCursor->pNode, pCursor->iCell); - rc = nodeAcquire(pRtree, iRowid, pCursor->pNode, &pChild); - if( rc!=SQLITE_OK ){ - goto descend_to_cell_out; + default: /* RTREE_GT or RTREE_GE, or fallthrough of RTREE_EQ */ + pCellData += 4; + RTREE_DECODE_COORD(eInt, pCellData, val); + /* val now holds the upper bound of the coordinate pair */ + if( p->u.rValue<=val ) return; } + *peWithin = NOT_WITHIN; +} - nodeRelease(pRtree, pCursor->pNode); - pCursor->pNode = pChild; - isEof = 1; - for(ii=0; isEof && iiiCell = ii; - rc = descendToCell(pRtree, pCursor, iHeight-1, &isEof); - if( rc!=SQLITE_OK ){ - goto descend_to_cell_out; - } - } +/* +** Check the leaf RTree cell given by pCellData against constraint p. +** If this constraint is not satisfied, set *peWithin to NOT_WITHIN. +** If the constraint is satisfied, leave *peWithin unchanged. +** +** The constraint is of the form: xN op $val +** +** The op is given by p->op. The xN is p->iCoord-th coordinate in +** pCellData. $val is given by p->u.rValue. +*/ +static void rtreeLeafConstraint( + RtreeConstraint *p, /* The constraint to test */ + int eInt, /* True if RTree holds integer coordinates */ + u8 *pCellData, /* Raw cell content as appears on disk */ + int *peWithin /* Adjust downward, as appropriate */ +){ + RtreeDValue xN; /* Coordinate value converted to a double */ - if( isEof ){ - assert( pCursor->pNode==pChild ); - nodeReference(pSavedNode); - nodeRelease(pRtree, pChild); - pCursor->pNode = pSavedNode; - pCursor->iCell = iSavedCell; + assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE + || p->op==RTREE_GT || p->op==RTREE_EQ ); + pCellData += 8 + p->iCoord*4; + RTREE_DECODE_COORD(eInt, pCellData, xN); + switch( p->op ){ + case RTREE_LE: if( xN <= p->u.rValue ) return; break; + case RTREE_LT: if( xN < p->u.rValue ) return; break; + case RTREE_GE: if( xN >= p->u.rValue ) return; break; + case RTREE_GT: if( xN > p->u.rValue ) return; break; + default: if( xN == p->u.rValue ) return; break; } - -descend_to_cell_out: - *pEof = isEof; - return rc; + *peWithin = NOT_WITHIN; } /* @@ -1074,6 +1073,7 @@ static int nodeRowidIndex( ){ int ii; int nCell = NCELL(pNode); + assert( nCell<200 ); for(ii=0; iirScorerScore ) return -1; + if( pA->rScore>pB->rScore ) return +1; + if( pA->iLeveliLevel ) return -1; + if( pA->iLevel>pB->iLevel ) return +1; + return 0; +} + +/* +** Interchange to search points in a cursor. +*/ +static void rtreeSearchPointSwap(RtreeCursor *p, int i, int j){ + RtreeSearchPoint t = p->aPoint[i]; + assert( iaPoint[i] = p->aPoint[j]; + p->aPoint[j] = t; + i++; j++; + if( i=RTREE_CACHE_SZ ){ + nodeRelease(RTREE_OF_CURSOR(p), p->aNode[i]); + p->aNode[i] = 0; + }else{ + RtreeNode *pTemp = p->aNode[i]; + p->aNode[i] = p->aNode[j]; + p->aNode[j] = pTemp; + } + } +} + +/* +** Return the search point with the lowest current score. */ -static int rtreeNext(sqlite3_vtab_cursor *pVtabCursor){ - Rtree *pRtree = (Rtree *)(pVtabCursor->pVtab); - RtreeCursor *pCsr = (RtreeCursor *)pVtabCursor; - int rc = SQLITE_OK; +static RtreeSearchPoint *rtreeSearchPointFirst(RtreeCursor *pCur){ + return pCur->bPoint ? &pCur->sPoint : pCur->nPoint ? pCur->aPoint : 0; +} - /* RtreeCursor.pNode must not be NULL. If is is NULL, then this cursor is - ** already at EOF. It is against the rules to call the xNext() method of - ** a cursor that has already reached EOF. - */ - assert( pCsr->pNode ); +/* +** Get the RtreeNode for the search point with the lowest score. +*/ +static RtreeNode *rtreeNodeOfFirstSearchPoint(RtreeCursor *pCur, int *pRC){ + sqlite3_int64 id; + int ii = 1 - pCur->bPoint; + assert( ii==0 || ii==1 ); + assert( pCur->bPoint || pCur->nPoint ); + if( pCur->aNode[ii]==0 ){ + assert( pRC!=0 ); + id = ii ? pCur->aPoint[0].id : pCur->sPoint.id; + *pRC = nodeAcquire(RTREE_OF_CURSOR(pCur), id, 0, &pCur->aNode[ii]); + } + return pCur->aNode[ii]; +} - if( pCsr->iStrategy==1 ){ - /* This "scan" is a direct lookup by rowid. There is no next entry. */ - nodeRelease(pRtree, pCsr->pNode); - pCsr->pNode = 0; +/* +** Push a new element onto the priority queue +*/ +static RtreeSearchPoint *rtreeEnqueue( + RtreeCursor *pCur, /* The cursor */ + RtreeDValue rScore, /* Score for the new search point */ + u8 iLevel /* Level for the new search point */ +){ + int i, j; + RtreeSearchPoint *pNew; + if( pCur->nPoint>=pCur->nPointAlloc ){ + int nNew = pCur->nPointAlloc*2 + 8; + pNew = sqlite3_realloc(pCur->aPoint, nNew*sizeof(pCur->aPoint[0])); + if( pNew==0 ) return 0; + pCur->aPoint = pNew; + pCur->nPointAlloc = nNew; + } + i = pCur->nPoint++; + pNew = pCur->aPoint + i; + pNew->rScore = rScore; + pNew->iLevel = iLevel; + assert( iLevel>=0 && iLevel<=RTREE_MAX_DEPTH ); + while( i>0 ){ + RtreeSearchPoint *pParent; + j = (i-1)/2; + pParent = pCur->aPoint + j; + if( rtreeSearchPointCompare(pNew, pParent)>=0 ) break; + rtreeSearchPointSwap(pCur, j, i); + i = j; + pNew = pParent; + } + return pNew; +} + +/* +** Allocate a new RtreeSearchPoint and return a pointer to it. Return +** NULL if malloc fails. +*/ +static RtreeSearchPoint *rtreeSearchPointNew( + RtreeCursor *pCur, /* The cursor */ + RtreeDValue rScore, /* Score for the new search point */ + u8 iLevel /* Level for the new search point */ +){ + RtreeSearchPoint *pNew, *pFirst; + pFirst = rtreeSearchPointFirst(pCur); + pCur->anQueue[iLevel]++; + if( pFirst==0 + || pFirst->rScore>rScore + || (pFirst->rScore==rScore && pFirst->iLevel>iLevel) + ){ + if( pCur->bPoint ){ + int ii; + pNew = rtreeEnqueue(pCur, rScore, iLevel); + if( pNew==0 ) return 0; + ii = (int)(pNew - pCur->aPoint) + 1; + if( iiaNode[ii]==0 ); + pCur->aNode[ii] = pCur->aNode[0]; + }else{ + nodeRelease(RTREE_OF_CURSOR(pCur), pCur->aNode[0]); + } + pCur->aNode[0] = 0; + *pNew = pCur->sPoint; + } + pCur->sPoint.rScore = rScore; + pCur->sPoint.iLevel = iLevel; + pCur->bPoint = 1; + return &pCur->sPoint; }else{ - /* Move to the next entry that matches the configured constraints. */ - int iHeight = 0; - while( pCsr->pNode ){ - RtreeNode *pNode = pCsr->pNode; - int nCell = NCELL(pNode); - for(pCsr->iCell++; pCsr->iCelliCell++){ - int isEof; - rc = descendToCell(pRtree, pCsr, iHeight, &isEof); - if( rc!=SQLITE_OK || !isEof ){ - return rc; + return rtreeEnqueue(pCur, rScore, iLevel); + } +} + +#if 0 +/* Tracing routines for the RtreeSearchPoint queue */ +static void tracePoint(RtreeSearchPoint *p, int idx, RtreeCursor *pCur){ + if( idx<0 ){ printf(" s"); }else{ printf("%2d", idx); } + printf(" %d.%05lld.%02d %g %d", + p->iLevel, p->id, p->iCell, p->rScore, p->eWithin + ); + idx++; + if( idxaNode[idx]); + }else{ + printf("\n"); + } +} +static void traceQueue(RtreeCursor *pCur, const char *zPrefix){ + int ii; + printf("=== %9s ", zPrefix); + if( pCur->bPoint ){ + tracePoint(&pCur->sPoint, -1, pCur); + } + for(ii=0; iinPoint; ii++){ + if( ii>0 || pCur->bPoint ) printf(" "); + tracePoint(&pCur->aPoint[ii], ii, pCur); + } +} +# define RTREE_QUEUE_TRACE(A,B) traceQueue(A,B) +#else +# define RTREE_QUEUE_TRACE(A,B) /* no-op */ +#endif + +/* Remove the search point with the lowest current score. +*/ +static void rtreeSearchPointPop(RtreeCursor *p){ + int i, j, k, n; + i = 1 - p->bPoint; + assert( i==0 || i==1 ); + if( p->aNode[i] ){ + nodeRelease(RTREE_OF_CURSOR(p), p->aNode[i]); + p->aNode[i] = 0; + } + if( p->bPoint ){ + p->anQueue[p->sPoint.iLevel]--; + p->bPoint = 0; + }else if( p->nPoint ){ + p->anQueue[p->aPoint[0].iLevel]--; + n = --p->nPoint; + p->aPoint[0] = p->aPoint[n]; + if( naNode[1] = p->aNode[n+1]; + p->aNode[n+1] = 0; + } + i = 0; + while( (j = i*2+1)aPoint[k], &p->aPoint[j])<0 ){ + if( rtreeSearchPointCompare(&p->aPoint[k], &p->aPoint[i])<0 ){ + rtreeSearchPointSwap(p, i, k); + i = k; + }else{ + break; + } + }else{ + if( rtreeSearchPointCompare(&p->aPoint[j], &p->aPoint[i])<0 ){ + rtreeSearchPointSwap(p, i, j); + i = j; + }else{ + break; } } - pCsr->pNode = pNode->pParent; - rc = nodeParentIndex(pRtree, pNode, &pCsr->iCell); - if( rc!=SQLITE_OK ){ - return rc; + } + } +} + + +/* +** Continue the search on cursor pCur until the front of the queue +** contains an entry suitable for returning as a result-set row, +** or until the RtreeSearchPoint queue is empty, indicating that the +** query has completed. +*/ +static int rtreeStepToLeaf(RtreeCursor *pCur){ + RtreeSearchPoint *p; + Rtree *pRtree = RTREE_OF_CURSOR(pCur); + RtreeNode *pNode; + int eWithin; + int rc = SQLITE_OK; + int nCell; + int nConstraint = pCur->nConstraint; + int ii; + int eInt; + RtreeSearchPoint x; + + eInt = pRtree->eCoordType==RTREE_COORD_INT32; + while( (p = rtreeSearchPointFirst(pCur))!=0 && p->iLevel>0 ){ + pNode = rtreeNodeOfFirstSearchPoint(pCur, &rc); + if( rc ) return rc; + nCell = NCELL(pNode); + assert( nCell<200 ); + while( p->iCellzData + (4+pRtree->nBytesPerCell*p->iCell); + eWithin = FULLY_WITHIN; + for(ii=0; iiaConstraint + ii; + if( pConstraint->op>=RTREE_MATCH ){ + rc = rtreeCallbackConstraint(pConstraint, eInt, pCellData, p, + &rScore, &eWithin); + if( rc ) return rc; + }else if( p->iLevel==1 ){ + rtreeLeafConstraint(pConstraint, eInt, pCellData, &eWithin); + }else{ + rtreeNonleafConstraint(pConstraint, eInt, pCellData, &eWithin); + } + if( eWithin==NOT_WITHIN ) break; + } + p->iCell++; + if( eWithin==NOT_WITHIN ) continue; + x.iLevel = p->iLevel - 1; + if( x.iLevel ){ + x.id = readInt64(pCellData); + x.iCell = 0; + }else{ + x.id = p->id; + x.iCell = p->iCell - 1; + } + if( p->iCell>=nCell ){ + RTREE_QUEUE_TRACE(pCur, "POP-S:"); + rtreeSearchPointPop(pCur); } - nodeReference(pCsr->pNode); - nodeRelease(pRtree, pNode); - iHeight++; + if( rScoreeWithin = eWithin; + p->id = x.id; + p->iCell = x.iCell; + RTREE_QUEUE_TRACE(pCur, "PUSH-S:"); + break; + } + if( p->iCell>=nCell ){ + RTREE_QUEUE_TRACE(pCur, "POP-Se:"); + rtreeSearchPointPop(pCur); } } + pCur->atEOF = p==0; + return SQLITE_OK; +} +/* +** Rtree virtual table module xNext method. +*/ +static int rtreeNext(sqlite3_vtab_cursor *pVtabCursor){ + RtreeCursor *pCsr = (RtreeCursor *)pVtabCursor; + int rc = SQLITE_OK; + + /* Move to the next entry that matches the configured constraints. */ + RTREE_QUEUE_TRACE(pCsr, "POP-Nx:"); + rtreeSearchPointPop(pCsr); + rc = rtreeStepToLeaf(pCsr); return rc; } @@ -1145,13 +1399,14 @@ static int rtreeNext(sqlite3_vtab_cursor *pVtabCursor){ ** Rtree virtual table module xRowid method. */ static int rtreeRowid(sqlite3_vtab_cursor *pVtabCursor, sqlite_int64 *pRowid){ - Rtree *pRtree = (Rtree *)pVtabCursor->pVtab; RtreeCursor *pCsr = (RtreeCursor *)pVtabCursor; - - assert(pCsr->pNode); - *pRowid = nodeGetRowid(pRtree, pCsr->pNode, pCsr->iCell); - - return SQLITE_OK; + RtreeSearchPoint *p = rtreeSearchPointFirst(pCsr); + int rc = SQLITE_OK; + RtreeNode *pNode = rtreeNodeOfFirstSearchPoint(pCsr, &rc); + if( rc==SQLITE_OK && p ){ + *pRowid = nodeGetRowid(RTREE_OF_CURSOR(pCsr), pNode, p->iCell); + } + return rc; } /* @@ -1160,13 +1415,18 @@ static int rtreeRowid(sqlite3_vtab_cursor *pVtabCursor, sqlite_int64 *pRowid){ static int rtreeColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){ Rtree *pRtree = (Rtree *)cur->pVtab; RtreeCursor *pCsr = (RtreeCursor *)cur; + RtreeSearchPoint *p = rtreeSearchPointFirst(pCsr); + RtreeCoord c; + int rc = SQLITE_OK; + RtreeNode *pNode = rtreeNodeOfFirstSearchPoint(pCsr, &rc); + if( rc ) return rc; + if( p==0 ) return SQLITE_OK; if( i==0 ){ - i64 iRowid = nodeGetRowid(pRtree, pCsr->pNode, pCsr->iCell); - sqlite3_result_int64(ctx, iRowid); + sqlite3_result_int64(ctx, nodeGetRowid(pRtree, pNode, p->iCell)); }else{ - RtreeCoord c; - nodeGetCoord(pRtree, pCsr->pNode, pCsr->iCell, i-1, &c); + if( rc ) return rc; + nodeGetCoord(pRtree, pNode, p->iCell, i-1, &c); #ifndef SQLITE_RTREE_INT_ONLY if( pRtree->eCoordType==RTREE_COORD_REAL32 ){ sqlite3_result_double(ctx, c.f); @@ -1177,7 +1437,6 @@ static int rtreeColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){ sqlite3_result_int(ctx, c.i); } } - return SQLITE_OK; } @@ -1188,12 +1447,18 @@ static int rtreeColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){ ** *ppLeaf to 0 and return SQLITE_OK. If an error occurs, set *ppLeaf ** to zero and return an SQLite error code. */ -static int findLeafNode(Rtree *pRtree, i64 iRowid, RtreeNode **ppLeaf){ +static int findLeafNode( + Rtree *pRtree, /* RTree to search */ + i64 iRowid, /* The rowid searching for */ + RtreeNode **ppLeaf, /* Write the node here */ + sqlite3_int64 *piNode /* Write the node-id here */ +){ int rc; *ppLeaf = 0; sqlite3_bind_int64(pRtree->pReadRowid, 1, iRowid); if( sqlite3_step(pRtree->pReadRowid)==SQLITE_ROW ){ i64 iNode = sqlite3_column_int64(pRtree->pReadRowid, 0); + if( piNode ) *piNode = iNode; rc = nodeAcquire(pRtree, iNode, 0, ppLeaf); sqlite3_reset(pRtree->pReadRowid); }else{ @@ -1209,9 +1474,10 @@ static int findLeafNode(Rtree *pRtree, i64 iRowid, RtreeNode **ppLeaf){ ** operator. */ static int deserializeGeometry(sqlite3_value *pValue, RtreeConstraint *pCons){ - RtreeMatchArg *p; - sqlite3_rtree_geometry *pGeom; - int nBlob; + RtreeMatchArg *pBlob; /* BLOB returned by geometry function */ + sqlite3_rtree_query_info *pInfo; /* Callback information */ + int nBlob; /* Size of the geometry function blob */ + int nExpected; /* Expected size of the BLOB */ /* Check that value is actually a blob. */ if( sqlite3_value_type(pValue)!=SQLITE_BLOB ) return SQLITE_ERROR; @@ -1224,27 +1490,29 @@ static int deserializeGeometry(sqlite3_value *pValue, RtreeConstraint *pCons){ return SQLITE_ERROR; } - pGeom = (sqlite3_rtree_geometry *)sqlite3_malloc( - sizeof(sqlite3_rtree_geometry) + nBlob - ); - if( !pGeom ) return SQLITE_NOMEM; - memset(pGeom, 0, sizeof(sqlite3_rtree_geometry)); - p = (RtreeMatchArg *)&pGeom[1]; + pInfo = (sqlite3_rtree_query_info*)sqlite3_malloc( sizeof(*pInfo)+nBlob ); + if( !pInfo ) return SQLITE_NOMEM; + memset(pInfo, 0, sizeof(*pInfo)); + pBlob = (RtreeMatchArg*)&pInfo[1]; - memcpy(p, sqlite3_value_blob(pValue), nBlob); - if( p->magic!=RTREE_GEOMETRY_MAGIC - || nBlob!=(int)(sizeof(RtreeMatchArg) + (p->nParam-1)*sizeof(RtreeDValue)) - ){ - sqlite3_free(pGeom); + memcpy(pBlob, sqlite3_value_blob(pValue), nBlob); + nExpected = (int)(sizeof(RtreeMatchArg) + + (pBlob->nParam-1)*sizeof(RtreeDValue)); + if( pBlob->magic!=RTREE_GEOMETRY_MAGIC || nBlob!=nExpected ){ + sqlite3_free(pInfo); return SQLITE_ERROR; } + pInfo->pContext = pBlob->cb.pContext; + pInfo->nParam = pBlob->nParam; + pInfo->aParam = pBlob->aParam; - pGeom->pContext = p->pContext; - pGeom->nParam = p->nParam; - pGeom->aParam = p->aParam; - - pCons->xGeom = p->xGeom; - pCons->pGeom = pGeom; + if( pBlob->cb.xGeom ){ + pCons->u.xGeom = pBlob->cb.xGeom; + }else{ + pCons->op = RTREE_QUERY; + pCons->u.xQueryFunc = pBlob->cb.xQueryFunc; + } + pCons->pInfo = pInfo; return SQLITE_OK; } @@ -1258,44 +1526,59 @@ static int rtreeFilter( ){ Rtree *pRtree = (Rtree *)pVtabCursor->pVtab; RtreeCursor *pCsr = (RtreeCursor *)pVtabCursor; - RtreeNode *pRoot = 0; int ii; int rc = SQLITE_OK; + int iCell = 0; rtreeReference(pRtree); + /* Reset the cursor to the same state as rtreeOpen() leaves it in. */ freeCursorConstraints(pCsr); - pCsr->iStrategy = idxNum; + sqlite3_free(pCsr->aPoint); + memset(pCsr, 0, sizeof(RtreeCursor)); + pCsr->base.pVtab = (sqlite3_vtab*)pRtree; + pCsr->iStrategy = idxNum; if( idxNum==1 ){ /* Special case - lookup by rowid. */ RtreeNode *pLeaf; /* Leaf on which the required cell resides */ + RtreeSearchPoint *p; /* Search point for the the leaf */ i64 iRowid = sqlite3_value_int64(argv[0]); - rc = findLeafNode(pRtree, iRowid, &pLeaf); - pCsr->pNode = pLeaf; - if( pLeaf ){ - assert( rc==SQLITE_OK ); - rc = nodeRowidIndex(pRtree, pLeaf, iRowid, &pCsr->iCell); + i64 iNode = 0; + rc = findLeafNode(pRtree, iRowid, &pLeaf, &iNode); + if( rc==SQLITE_OK && pLeaf!=0 ){ + p = rtreeSearchPointNew(pCsr, RTREE_ZERO, 0); + assert( p!=0 ); /* Always returns pCsr->sPoint */ + pCsr->aNode[0] = pLeaf; + p->id = iNode; + p->eWithin = PARTLY_WITHIN; + rc = nodeRowidIndex(pRtree, pLeaf, iRowid, &iCell); + p->iCell = iCell; + RTREE_QUEUE_TRACE(pCsr, "PUSH-F1:"); + }else{ + pCsr->atEOF = 1; } }else{ /* Normal case - r-tree scan. Set up the RtreeCursor.aConstraint array ** with the configured constraints. */ - if( argc>0 ){ + rc = nodeAcquire(pRtree, 1, 0, &pRoot); + if( rc==SQLITE_OK && argc>0 ){ pCsr->aConstraint = sqlite3_malloc(sizeof(RtreeConstraint)*argc); pCsr->nConstraint = argc; if( !pCsr->aConstraint ){ rc = SQLITE_NOMEM; }else{ memset(pCsr->aConstraint, 0, sizeof(RtreeConstraint)*argc); + memset(pCsr->anQueue, 0, sizeof(u32)*(pRtree->iDepth + 1)); assert( (idxStr==0 && argc==0) || (idxStr && (int)strlen(idxStr)==argc*2) ); for(ii=0; iiaConstraint[ii]; p->op = idxStr[ii*2]; - p->iCoord = idxStr[ii*2+1]-'a'; - if( p->op==RTREE_MATCH ){ + p->iCoord = idxStr[ii*2+1]-'0'; + if( p->op>=RTREE_MATCH ){ /* A MATCH operator. The right-hand-side must be a blob that ** can be cast into an RtreeMatchArg object. One created using ** an sqlite3_rtree_geometry_callback() SQL user function. @@ -1304,45 +1587,52 @@ static int rtreeFilter( if( rc!=SQLITE_OK ){ break; } + p->pInfo->nCoord = pRtree->nDim*2; + p->pInfo->anQueue = pCsr->anQueue; + p->pInfo->mxLevel = pRtree->iDepth + 1; }else{ #ifdef SQLITE_RTREE_INT_ONLY - p->rValue = sqlite3_value_int64(argv[ii]); + p->u.rValue = sqlite3_value_int64(argv[ii]); #else - p->rValue = sqlite3_value_double(argv[ii]); + p->u.rValue = sqlite3_value_double(argv[ii]); #endif } } } } - if( rc==SQLITE_OK ){ - pCsr->pNode = 0; - rc = nodeAcquire(pRtree, 1, 0, &pRoot); - } - if( rc==SQLITE_OK ){ - int isEof = 1; - int nCell = NCELL(pRoot); - pCsr->pNode = pRoot; - for(pCsr->iCell=0; rc==SQLITE_OK && pCsr->iCelliCell++){ - assert( pCsr->pNode==pRoot ); - rc = descendToCell(pRtree, pCsr, pRtree->iDepth, &isEof); - if( !isEof ){ - break; - } - } - if( rc==SQLITE_OK && isEof ){ - assert( pCsr->pNode==pRoot ); - nodeRelease(pRtree, pRoot); - pCsr->pNode = 0; - } - assert( rc!=SQLITE_OK || !pCsr->pNode || pCsr->iCellpNode) ); + RtreeSearchPoint *pNew; + pNew = rtreeSearchPointNew(pCsr, RTREE_ZERO, pRtree->iDepth+1); + if( pNew==0 ) return SQLITE_NOMEM; + pNew->id = 1; + pNew->iCell = 0; + pNew->eWithin = PARTLY_WITHIN; + assert( pCsr->bPoint==1 ); + pCsr->aNode[0] = pRoot; + pRoot = 0; + RTREE_QUEUE_TRACE(pCsr, "PUSH-Fm:"); + rc = rtreeStepToLeaf(pCsr); } } + nodeRelease(pRtree, pRoot); rtreeRelease(pRtree); return rc; } +/* +** Set the pIdxInfo->estimatedRows variable to nRow. Unless this +** extension is currently being used by a version of SQLite too old to +** support estimatedRows. In that case this function is a no-op. +*/ +static void setEstimatedRows(sqlite3_index_info *pIdxInfo, i64 nRow){ +#if SQLITE_VERSION_NUMBER>=3008002 + if( sqlite3_libversion_number()>=3008002 ){ + pIdxInfo->estimatedRows = nRow; + } +#endif +} + /* ** Rtree virtual table module xBestIndex method. There are three ** table scan strategies to choose from (in order from most to @@ -1378,13 +1668,14 @@ static int rtreeFilter( ** is 'a', the second from the left 'b' etc. */ static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ + Rtree *pRtree = (Rtree*)tab; int rc = SQLITE_OK; int ii; + i64 nRow; /* Estimated rows returned by this scan */ int iIdx = 0; char zIdxStr[RTREE_MAX_DIMENSIONS*8+1]; memset(zIdxStr, 0, sizeof(zIdxStr)); - UNUSED_PARAMETER(tab); assert( pIdxInfo->idxStr==0 ); for(ii=0; iinConstraint && iIdx<(int)(sizeof(zIdxStr)-1); ii++){ @@ -1404,9 +1695,11 @@ static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ /* This strategy involves a two rowid lookups on an B-Tree structures ** and then a linear search of an R-Tree node. This should be ** considered almost as quick as a direct rowid lookup (for which - ** sqlite uses an internal cost of 0.0). + ** sqlite uses an internal cost of 0.0). It is expected to return + ** a single row. */ - pIdxInfo->estimatedCost = 10.0; + pIdxInfo->estimatedCost = 30.0; + setEstimatedRows(pIdxInfo, 1); return SQLITE_OK; } @@ -1424,7 +1717,7 @@ static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ break; } zIdxStr[iIdx++] = op; - zIdxStr[iIdx++] = p->iColumn - 1 + 'a'; + zIdxStr[iIdx++] = p->iColumn - 1 + '0'; pIdxInfo->aConstraintUsage[ii].argvIndex = (iIdx/2); pIdxInfo->aConstraintUsage[ii].omit = 1; } @@ -1435,8 +1728,11 @@ static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ if( iIdx>0 && 0==(pIdxInfo->idxStr = sqlite3_mprintf("%s", zIdxStr)) ){ return SQLITE_NOMEM; } - assert( iIdx>=0 ); - pIdxInfo->estimatedCost = (2000000.0 / (double)(iIdx + 1)); + + nRow = pRtree->nRowEst / (iIdx + 1); + pIdxInfo->estimatedCost = (double)6.0 * (double)nRow; + setEstimatedRows(pIdxInfo, nRow); + return rc; } @@ -1514,62 +1810,32 @@ static RtreeDValue cellGrowth(Rtree *pRtree, RtreeCell *p, RtreeCell *pCell){ return (cellArea(pRtree, &cell)-area); } -#if VARIANT_RSTARTREE_CHOOSESUBTREE || VARIANT_RSTARTREE_SPLIT static RtreeDValue cellOverlap( Rtree *pRtree, RtreeCell *p, RtreeCell *aCell, - int nCell, - int iExclude + int nCell ){ int ii; - RtreeDValue overlap = 0.0; + RtreeDValue overlap = RTREE_ZERO; for(ii=0; iinDim*2); jj+=2){ - RtreeDValue x1, x2; - - x1 = MAX(DCOORD(p->aCoord[jj]), DCOORD(aCell[ii].aCoord[jj])); - x2 = MIN(DCOORD(p->aCoord[jj+1]), DCOORD(aCell[ii].aCoord[jj+1])); - - if( x2nDim*2); jj+=2){ + RtreeDValue x1, x2; + x1 = MAX(DCOORD(p->aCoord[jj]), DCOORD(aCell[ii].aCoord[jj])); + x2 = MIN(DCOORD(p->aCoord[jj+1]), DCOORD(aCell[ii].aCoord[jj+1])); + if( x2iDepth-1) ){ - int jj; - aCell = sqlite3_malloc(sizeof(RtreeCell)*nCell); - if( !aCell ){ - rc = SQLITE_NOMEM; - nodeRelease(pRtree, pNode); - pNode = 0; - continue; - } - for(jj=0; jjiDepth-1) ){ - overlap = cellOverlapEnlargement(pRtree,&cell,pCell,aCell,nCell,iCell); - }else{ - overlap = 0.0; - } - if( (iCell==0) - || (overlapnDim; i++){ - RtreeDValue x1 = DCOORD(aCell[0].aCoord[i*2]); - RtreeDValue x2 = DCOORD(aCell[0].aCoord[i*2+1]); - RtreeDValue x3 = x1; - RtreeDValue x4 = x2; - int jj; - - int iCellLeft = 0; - int iCellRight = 0; - - for(jj=1; jjx4 ) x4 = right; - if( left>x3 ){ - x3 = left; - iCellRight = jj; - } - if( rightmaxNormalInnerWidth ){ - iLeftSeed = iCellLeft; - iRightSeed = iCellRight; - } - } - } - - *piLeftSeed = iLeftSeed; - *piRightSeed = iRightSeed; -} -#endif /* VARIANT_GUTTMAN_LINEAR_SPLIT */ - -#if VARIANT_GUTTMAN_QUADRATIC_SPLIT -/* -** Implementation of the quadratic variant of the PickNext() function from -** Guttman[84]. -*/ -static RtreeCell *QuadraticPickNext( - Rtree *pRtree, - RtreeCell *aCell, - int nCell, - RtreeCell *pLeftBox, - RtreeCell *pRightBox, - int *aiUsed -){ - #define FABS(a) ((a)<0.0?-1.0*(a):(a)) - - int iSelect = -1; - RtreeDValue fDiff; - int ii; - for(ii=0; iifDiff ){ - fDiff = diff; - iSelect = ii; - } - } - } - aiUsed[iSelect] = 1; - return &aCell[iSelect]; -} - -/* -** Implementation of the quadratic variant of the PickSeeds() function from -** Guttman[84]. -*/ -static void QuadraticPickSeeds( - Rtree *pRtree, - RtreeCell *aCell, - int nCell, - int *piLeftSeed, - int *piRightSeed -){ - int ii; - int jj; - - int iLeftSeed = 0; - int iRightSeed = 1; - RtreeDValue fWaste = 0.0; - - for(ii=0; iifWaste ){ - iLeftSeed = ii; - iRightSeed = jj; - fWaste = waste; - } - } - } - - *piLeftSeed = iLeftSeed; - *piRightSeed = iRightSeed; -} -#endif /* VARIANT_GUTTMAN_QUADRATIC_SPLIT */ /* ** Arguments aIdx, aDistance and aSpare all point to arrays of size @@ -2010,7 +2090,6 @@ static void SortByDimension( } } -#if VARIANT_RSTARTREE_SPLIT /* ** Implementation of the R*-tree variant of SplitNode from Beckman[1990]. */ @@ -2029,7 +2108,7 @@ static int splitNodeStartree( int iBestDim = 0; int iBestSplit = 0; - RtreeDValue fBestMargin = 0.0; + RtreeDValue fBestMargin = RTREE_ZERO; int nByte = (pRtree->nDim+1)*(sizeof(int*)+nCell*sizeof(int)); @@ -2050,9 +2129,9 @@ static int splitNodeStartree( } for(ii=0; iinDim; ii++){ - RtreeDValue margin = 0.0; - RtreeDValue fBestOverlap = 0.0; - RtreeDValue fBestArea = 0.0; + RtreeDValue margin = RTREE_ZERO; + RtreeDValue fBestOverlap = RTREE_ZERO; + RtreeDValue fBestArea = RTREE_ZERO; int iBestLeft = 0; int nLeft; @@ -2078,7 +2157,7 @@ static int splitNodeStartree( } margin += cellMargin(pRtree, &left); margin += cellMargin(pRtree, &right); - overlap = cellOverlap(pRtree, &left, &right, 1, -1); + overlap = cellOverlap(pRtree, &left, &right, 1); area = cellArea(pRtree, &left) + cellArea(pRtree, &right); if( (nLeft==RTREE_MINCELLS(pRtree)) || (overlap0; i--){ - RtreeCell *pNext; - pNext = PickNext(pRtree, aCell, nCell, pBboxLeft, pBboxRight, aiUsed); - RtreeDValue diff = - cellGrowth(pRtree, pBboxLeft, pNext) - - cellGrowth(pRtree, pBboxRight, pNext) - ; - if( (RTREE_MINCELLS(pRtree)-NCELL(pRight)==i) - || (diff>0.0 && (RTREE_MINCELLS(pRtree)-NCELL(pLeft)!=i)) - ){ - nodeInsertCell(pRtree, pRight, pNext); - cellUnion(pRtree, pBboxRight, pNext); - }else{ - nodeInsertCell(pRtree, pLeft, pNext); - cellUnion(pRtree, pBboxLeft, pNext); - } - } - - sqlite3_free(aiUsed); - return SQLITE_OK; -} -#endif static int updateMapping( Rtree *pRtree, @@ -2244,7 +2267,8 @@ static int SplitNode( memset(pLeft->zData, 0, pRtree->iNodeSize); memset(pRight->zData, 0, pRtree->iNodeSize); - rc = AssignCells(pRtree, aCell, nCell, pLeft, pRight, &leftbbox, &rightbbox); + rc = splitNodeStartree(pRtree, aCell, nCell, pLeft, pRight, + &leftbbox, &rightbbox); if( rc!=SQLITE_OK ){ goto splitnode_out; } @@ -2527,7 +2551,7 @@ static int Reinsert( } for(ii=0; iinDim; iDim++){ RtreeDValue coord = (DCOORD(aCell[ii].aCoord[iDim*2+1]) - DCOORD(aCell[ii].aCoord[iDim*2])); @@ -2593,16 +2617,12 @@ static int rtreeInsertCell( } } if( nodeInsertCell(pRtree, pNode, pCell) ){ -#if VARIANT_RSTARTREE_REINSERT if( iHeight<=pRtree->iReinsertHeight || pNode->iNode==1){ rc = SplitNode(pRtree, pNode, pCell, iHeight); }else{ pRtree->iReinsertHeight = iHeight; rc = Reinsert(pRtree, pNode, pCell, iHeight); } -#else - rc = SplitNode(pRtree, pNode, pCell, iHeight); -#endif }else{ rc = AdjustTree(pRtree, pNode, pCell); if( rc==SQLITE_OK ){ @@ -2672,7 +2692,7 @@ static int rtreeDeleteRowid(Rtree *pRtree, sqlite3_int64 iDelete){ ** about to be deleted. */ if( rc==SQLITE_OK ){ - rc = findLeafNode(pRtree, iDelete, &pLeaf); + rc = findLeafNode(pRtree, iDelete, &pLeaf, 0); } /* Delete the cell in question from the leaf node. */ @@ -2911,6 +2931,43 @@ static int rtreeRename(sqlite3_vtab *pVtab, const char *zNewName){ return rc; } +/* +** This function populates the pRtree->nRowEst variable with an estimate +** of the number of rows in the virtual table. If possible, this is based +** on sqlite_stat1 data. Otherwise, use RTREE_DEFAULT_ROWEST. +*/ +static int rtreeQueryStat1(sqlite3 *db, Rtree *pRtree){ + const char *zFmt = "SELECT stat FROM %Q.sqlite_stat1 WHERE tbl = '%q_rowid'"; + char *zSql; + sqlite3_stmt *p; + int rc; + i64 nRow = 0; + + zSql = sqlite3_mprintf(zFmt, pRtree->zDb, pRtree->zName); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(db, zSql, -1, &p, 0); + if( rc==SQLITE_OK ){ + if( sqlite3_step(p)==SQLITE_ROW ) nRow = sqlite3_column_int64(p, 0); + rc = sqlite3_finalize(p); + }else if( rc!=SQLITE_NOMEM ){ + rc = SQLITE_OK; + } + + if( rc==SQLITE_OK ){ + if( nRow==0 ){ + pRtree->nRowEst = RTREE_DEFAULT_ROWEST; + }else{ + pRtree->nRowEst = MAX(nRow, RTREE_MIN_ROWEST); + } + } + sqlite3_free(zSql); + } + + return rc; +} + static sqlite3_module rtreeModule = { 0, /* iVersion */ rtreeCreate, /* xCreate - create a table */ @@ -2972,7 +3029,8 @@ static int rtreeSqlInit( char *zCreate = sqlite3_mprintf( "CREATE TABLE \"%w\".\"%w_node\"(nodeno INTEGER PRIMARY KEY, data BLOB);" "CREATE TABLE \"%w\".\"%w_rowid\"(rowid INTEGER PRIMARY KEY, nodeno INTEGER);" -"CREATE TABLE \"%w\".\"%w_parent\"(nodeno INTEGER PRIMARY KEY, parentnode INTEGER);" +"CREATE TABLE \"%w\".\"%w_parent\"(nodeno INTEGER PRIMARY KEY," + " parentnode INTEGER);" "INSERT INTO '%q'.'%q_node' VALUES(1, zeroblob(%d))", zDb, zPrefix, zDb, zPrefix, zDb, zPrefix, zDb, zPrefix, pRtree->iNodeSize ); @@ -2996,6 +3054,7 @@ static int rtreeSqlInit( appStmt[7] = &pRtree->pWriteParent; appStmt[8] = &pRtree->pDeleteParent; + rc = rtreeQueryStat1(db, pRtree); for(i=0; inBusy==1 ); rtreeRelease(pRtree); } return rc; @@ -3183,10 +3244,10 @@ static int rtreeInit( ** Implementation of a scalar function that decodes r-tree nodes to ** human readable strings. This can be used for debugging and analysis. ** -** The scalar function takes two arguments, a blob of data containing -** an r-tree node, and the number of dimensions the r-tree indexes. -** For a two-dimensional r-tree structure called "rt", to deserialize -** all nodes, a statement like: +** The scalar function takes two arguments: (1) the number of dimensions +** to the rtree (between 1 and 5, inclusive) and (2) a blob of data containing +** an r-tree node. For a two-dimensional r-tree structure called "rt", to +** deserialize all nodes, a statement like: ** ** SELECT rtreenode(2, data) FROM rt_node; ** @@ -3219,7 +3280,7 @@ static void rtreenode(sqlite3_context *ctx, int nArg, sqlite3_value **apArg){ nCell = (int)strlen(zCell); for(jj=0; jjxDestructor ) pInfo->xDestructor(pInfo->pContext); sqlite3_free(p); } /* -** Each call to sqlite3_rtree_geometry_callback() creates an ordinary SQLite -** scalar user function. This C function is the callback used for all such -** registered SQL functions. +** Each call to sqlite3_rtree_geometry_callback() or +** sqlite3_rtree_query_callback() creates an ordinary SQLite +** scalar function that is implemented by this routine. +** +** All this function does is construct an RtreeMatchArg object that +** contains the geometry-checking callback routines and a list of +** parameters to this function, then return that RtreeMatchArg object +** as a BLOB. ** -** The scalar user functions return a blob that is interpreted by r-tree -** table MATCH operators. +** The R-Tree MATCH operator will read the returned BLOB, deserialize +** the RtreeMatchArg object, and use the RtreeMatchArg object to figure +** out which elements of the R-Tree should be returned by the query. */ static void geomCallback(sqlite3_context *ctx, int nArg, sqlite3_value **aArg){ RtreeGeomCallback *pGeomCtx = (RtreeGeomCallback *)sqlite3_user_data(ctx); @@ -3311,8 +3390,7 @@ static void geomCallback(sqlite3_context *ctx, int nArg, sqlite3_value **aArg){ }else{ int i; pBlob->magic = RTREE_GEOMETRY_MAGIC; - pBlob->xGeom = pGeomCtx->xGeom; - pBlob->pContext = pGeomCtx->pContext; + pBlob->cb = pGeomCtx[0]; pBlob->nParam = nArg; for(i=0; iaParam[i] = sqlite3_value_double(aArg[i]); #endif } - sqlite3_result_blob(ctx, pBlob, nBlob, doSqlite3Free); + sqlite3_result_blob(ctx, pBlob, nBlob, sqlite3_free); } } @@ -3329,10 +3407,10 @@ static void geomCallback(sqlite3_context *ctx, int nArg, sqlite3_value **aArg){ ** Register a new geometry function for use with the r-tree MATCH operator. */ int sqlite3_rtree_geometry_callback( - sqlite3 *db, - const char *zGeom, - int (*xGeom)(sqlite3_rtree_geometry *, int, RtreeDValue *, int *), - void *pContext + sqlite3 *db, /* Register SQL function on this connection */ + const char *zGeom, /* Name of the new SQL function */ + int (*xGeom)(sqlite3_rtree_geometry*,int,RtreeDValue*,int*), /* Callback */ + void *pContext /* Extra data associated with the callback */ ){ RtreeGeomCallback *pGeomCtx; /* Context object for new user-function */ @@ -3340,17 +3418,44 @@ int sqlite3_rtree_geometry_callback( pGeomCtx = (RtreeGeomCallback *)sqlite3_malloc(sizeof(RtreeGeomCallback)); if( !pGeomCtx ) return SQLITE_NOMEM; pGeomCtx->xGeom = xGeom; + pGeomCtx->xQueryFunc = 0; + pGeomCtx->xDestructor = 0; pGeomCtx->pContext = pContext; - - /* Create the new user-function. Register a destructor function to delete - ** the context object when it is no longer required. */ return sqlite3_create_function_v2(db, zGeom, -1, SQLITE_ANY, - (void *)pGeomCtx, geomCallback, 0, 0, doSqlite3Free + (void *)pGeomCtx, geomCallback, 0, 0, rtreeFreeCallback + ); +} + +/* +** Register a new 2nd-generation geometry function for use with the +** r-tree MATCH operator. +*/ +int sqlite3_rtree_query_callback( + sqlite3 *db, /* Register SQL function on this connection */ + const char *zQueryFunc, /* Name of new SQL function */ + int (*xQueryFunc)(sqlite3_rtree_query_info*), /* Callback */ + void *pContext, /* Extra data passed into the callback */ + void (*xDestructor)(void*) /* Destructor for the extra data */ +){ + RtreeGeomCallback *pGeomCtx; /* Context object for new user-function */ + + /* Allocate and populate the context object. */ + pGeomCtx = (RtreeGeomCallback *)sqlite3_malloc(sizeof(RtreeGeomCallback)); + if( !pGeomCtx ) return SQLITE_NOMEM; + pGeomCtx->xGeom = 0; + pGeomCtx->xQueryFunc = xQueryFunc; + pGeomCtx->xDestructor = xDestructor; + pGeomCtx->pContext = pContext; + return sqlite3_create_function_v2(db, zQueryFunc, -1, SQLITE_ANY, + (void *)pGeomCtx, geomCallback, 0, 0, rtreeFreeCallback ); } #if !SQLITE_CORE -int sqlite3_extension_init( +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_rtree_init( sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi diff --git a/ext/rtree/rtree1.test b/ext/rtree/rtree1.test index 275b132..0beb16c 100644 --- a/ext/rtree/rtree1.test +++ b/ext/rtree/rtree1.test @@ -33,6 +33,7 @@ set testprefix rtree1 # rtree-8.*: Test constrained scans of r-tree data. # # rtree-12.*: Test that on-conflict clauses are supported. +# rtree-13.*: Test that bug [d2889096e7bdeac6d] has been fixed. # ifcapable !rtree { @@ -120,12 +121,13 @@ proc execsql_intout {sql} { # Test that it is possible to open an existing database that contains # r-tree tables. # -do_test rtree-1.4.1 { - execsql { - CREATE VIRTUAL TABLE t1 USING rtree(ii, x1, x2); - INSERT INTO t1 VALUES(1, 5.0, 10.0); - INSERT INTO t1 VALUES(2, 15.0, 20.0); - } +do_execsql_test rtree-1.4.1a { + CREATE VIRTUAL TABLE t1 USING rtree(ii, x1, x2); + INSERT INTO t1 VALUES(1, 5.0, 10.0); + SELECT substr(hex(data),1,40) FROM t1_node; +} {00000001000000000000000140A0000041200000} +do_execsql_test rtree-1.4.1b { + INSERT INTO t1 VALUES(2, 15.0, 20.0); } {} do_test rtree-1.4.2 { db close @@ -435,16 +437,18 @@ do_test rtree-11.2 { # Test on-conflict clause handling. # db_delete_and_reopen -do_execsql_test 12.0 { +do_execsql_test 12.0.1 { CREATE VIRTUAL TABLE t1 USING rtree_i32(idx, x1, x2, y1, y2); INSERT INTO t1 VALUES(1, 1, 2, 3, 4); + SELECT substr(hex(data),1,56) FROM t1_node; +} {00000001000000000000000100000001000000020000000300000004} +do_execsql_test 12.0.2 { INSERT INTO t1 VALUES(2, 2, 3, 4, 5); INSERT INTO t1 VALUES(3, 3, 4, 5, 6); CREATE TABLE source(idx, x1, x2, y1, y2); INSERT INTO source VALUES(5, 8, 8, 8, 8); INSERT INTO source VALUES(2, 7, 7, 7, 7); - } db_save_and_close foreach {tn sql_template testdata} { @@ -510,4 +514,25 @@ foreach {tn sql_template testdata} { db close } } + +#------------------------------------------------------------------------- +# Test that bug [d2889096e7bdeac6d] has been fixed. +# +reset_db +do_execsql_test 13.1 { + CREATE VIRTUAL TABLE t9 USING rtree(id, xmin, xmax); + INSERT INTO t9 VALUES(1,0,0); + INSERT INTO t9 VALUES(2,0,0); + SELECT * FROM t9 WHERE id IN (1, 2); +} {1 0.0 0.0 2 0.0 0.0} + +do_execsql_test 13.2 { + WITH r(x) AS ( + SELECT 1 UNION ALL + SELECT 2 UNION ALL + SELECT 3 + ) + SELECT * FROM r CROSS JOIN t9 WHERE id=x; +} {1 1 0.0 0.0 2 2 0.0 0.0} + finish_test diff --git a/ext/rtree/rtree6.test b/ext/rtree/rtree6.test index 92edc8d..cec3a8d 100644 --- a/ext/rtree/rtree6.test +++ b/ext/rtree/rtree6.test @@ -57,53 +57,59 @@ do_test rtree6-1.1 { do_test rtree6-1.2 { rtree_strategy {SELECT * FROM t1 WHERE x1>10} -} {Ea} +} {E0} do_test rtree6-1.3 { rtree_strategy {SELECT * FROM t1 WHERE x1<10} -} {Ca} +} {C0} do_test rtree6-1.4 { rtree_strategy {SELECT * FROM t1,t2 WHERE k=ii AND x1<10} -} {Ca} +} {C0} do_test rtree6-1.5 { rtree_strategy {SELECT * FROM t1,t2 WHERE k=+ii AND x1<10} -} {Ca} +} {C0} do_eqp_test rtree6.2.1 { SELECT * FROM t1,t2 WHERE k=+ii AND x1<10 } { - 0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2:Ca (~0 rows)} - 0 1 1 {SEARCH TABLE t2 USING INTEGER PRIMARY KEY (rowid=?) (~1 rows)} + 0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2:C0} + 0 1 1 {SEARCH TABLE t2 USING INTEGER PRIMARY KEY (rowid=?)} } do_eqp_test rtree6.2.2 { SELECT * FROM t1,t2 WHERE k=ii AND x1<10 } { - 0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2:Ca (~0 rows)} - 0 1 1 {SEARCH TABLE t2 USING INTEGER PRIMARY KEY (rowid=?) (~1 rows)} + 0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2:C0} + 0 1 1 {SEARCH TABLE t2 USING INTEGER PRIMARY KEY (rowid=?)} } do_eqp_test rtree6.2.3 { SELECT * FROM t1,t2 WHERE k=ii } { - 0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2: (~0 rows)} - 0 1 1 {SEARCH TABLE t2 USING INTEGER PRIMARY KEY (rowid=?) (~1 rows)} + 0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2:} + 0 1 1 {SEARCH TABLE t2 USING INTEGER PRIMARY KEY (rowid=?)} } -do_eqp_test rtree6.2.4 { +do_eqp_test rtree6.2.4.1 { + SELECT * FROM t1,t2 WHERE v=+ii and x1<10 and x2>10 +} { + 0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2:C0E1} + 0 1 1 {SEARCH TABLE t2 USING AUTOMATIC COVERING INDEX (v=?)} +} +do_eqp_test rtree6.2.4.2 { SELECT * FROM t1,t2 WHERE v=10 and x1<10 and x2>10 } { - 0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2:CaEb (~0 rows)} - 0 1 1 {SCAN TABLE t2 (~100000 rows)} + 0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2:C0E1} + 0 1 1 {SEARCH TABLE t2 USING AUTOMATIC COVERING INDEX (v=?)} } do_eqp_test rtree6.2.5 { SELECT * FROM t1,t2 WHERE k=ii AND x10.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 } -} {EaEaEaEaEaEaEaEaEaEaEaEaEaEaEaEaEaEaEaEa} +} {E0E0E0E0E0E0E0E0E0E0E0E0E0E0E0E0E0E0E0E0} do_test rtree6.3.3 { rtree_strategy { SELECT * FROM t3 WHERE @@ -137,7 +143,7 @@ do_test rtree6.3.3 { x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 } -} {EaEaEaEaEaEaEaEaEaEaEaEaEaEaEaEaEaEaEaEa} +} {E0E0E0E0E0E0E0E0E0E0E0E0E0E0E0E0E0E0E0E0} do_execsql_test rtree6-3.4 { SELECT * FROM t3 WHERE x1>0.5 AND x1>0.8 AND x1>1.1 diff --git a/ext/rtree/rtree8.test b/ext/rtree/rtree8.test index bf22cbf..578a146 100644 --- a/ext/rtree/rtree8.test +++ b/ext/rtree/rtree8.test @@ -168,4 +168,3 @@ do_test rtree8-5.3 { finish_test - diff --git a/ext/rtree/rtreeB.test b/ext/rtree/rtreeB.test index 7cb445c..aeb308e 100644 --- a/ext/rtree/rtreeB.test +++ b/ext/rtree/rtreeB.test @@ -41,7 +41,7 @@ ifcapable rtree_int_only { INSERT INTO t1 VALUES(9223372036854775807, 150, 150, 400, 400); SELECT rtreenode(2, data) FROM t1_node; } - } {{{1073741824 0.000000 0.000000 100.000000 100.000000} {2147483646 0.000000 0.000000 200.000000 200.000000} {4294967296 0.000000 0.000000 300.000000 300.000000} {8589934592 20.000000 20.000000 150.000000 150.000000} {9223372036854775807 150.000000 150.000000 400.000000 400.000000}}} + } {{{1073741824 0 0 100 100} {2147483646 0 0 200 200} {4294967296 0 0 300 300} {8589934592 20 20 150 150} {9223372036854775807 150 150 400 400}}} } finish_test diff --git a/ext/rtree/rtreeC.test b/ext/rtree/rtreeC.test new file mode 100644 index 0000000..94db05a --- /dev/null +++ b/ext/rtree/rtreeC.test @@ -0,0 +1,273 @@ +# 2011 March 2 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# Make sure the rtreenode() testing function can handle entries with +# 64-bit rowids. +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source $testdir/tester.tcl +ifcapable !rtree { finish_test ; return } +set testprefix rtreeC + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE r_tree USING rtree(id, min_x, max_x, min_y, max_y); + CREATE TABLE t(x, y); +} + +do_eqp_test 1.1 { + SELECT * FROM r_tree, t + WHERE t.x>=min_x AND t.x<=max_x AND t.y>=min_y AND t.x<=max_y +} { + 0 0 1 {SCAN TABLE t} + 0 1 0 {SCAN TABLE r_tree VIRTUAL TABLE INDEX 2:D3B2D1B0} +} + +do_eqp_test 1.2 { + SELECT * FROM t, r_tree + WHERE t.x>=min_x AND t.x<=max_x AND t.y>=min_y AND t.x<=max_y +} { + 0 0 0 {SCAN TABLE t} + 0 1 1 {SCAN TABLE r_tree VIRTUAL TABLE INDEX 2:D3B2D1B0} +} + +do_eqp_test 1.3 { + SELECT * FROM t, r_tree + WHERE t.x>=min_x AND t.x<=max_x AND t.y>=min_y AND ?<=max_y +} { + 0 0 0 {SCAN TABLE t} + 0 1 1 {SCAN TABLE r_tree VIRTUAL TABLE INDEX 2:D3B2D1B0} +} + +do_eqp_test 1.5 { + SELECT * FROM t, r_tree +} { + 0 0 1 {SCAN TABLE r_tree VIRTUAL TABLE INDEX 2:} + 0 1 0 {SCAN TABLE t} +} + +do_execsql_test 2.0 { + INSERT INTO t VALUES(0, 0); + INSERT INTO t VALUES(0, 1); + INSERT INTO t VALUES(0, 2); + INSERT INTO t VALUES(0, 3); + INSERT INTO t VALUES(0, 4); + INSERT INTO t VALUES(0, 5); + INSERT INTO t VALUES(0, 6); + INSERT INTO t VALUES(0, 7); + INSERT INTO t VALUES(0, 8); + INSERT INTO t VALUES(0, 9); + + INSERT INTO t SELECT x+1, y FROM t; + INSERT INTO t SELECT x+2, y FROM t; + INSERT INTO t SELECT x+4, y FROM t; + INSERT INTO r_tree SELECT NULL, x-1, x+1, y-1, y+1 FROM t; + ANALYZE; +} + +db close +sqlite3 db test.db + +do_eqp_test 2.1 { + SELECT * FROM r_tree, t + WHERE t.x>=min_x AND t.x<=max_x AND t.y>=min_y AND t.x<=max_y +} { + 0 0 1 {SCAN TABLE t} + 0 1 0 {SCAN TABLE r_tree VIRTUAL TABLE INDEX 2:D3B2D1B0} +} + +do_eqp_test 2.2 { + SELECT * FROM t, r_tree + WHERE t.x>=min_x AND t.x<=max_x AND t.y>=min_y AND t.x<=max_y +} { + 0 0 0 {SCAN TABLE t} + 0 1 1 {SCAN TABLE r_tree VIRTUAL TABLE INDEX 2:D3B2D1B0} +} + +do_eqp_test 2.3 { + SELECT * FROM t, r_tree + WHERE t.x>=min_x AND t.x<=max_x AND t.y>=min_y AND ?<=max_y +} { + 0 0 0 {SCAN TABLE t} + 0 1 1 {SCAN TABLE r_tree VIRTUAL TABLE INDEX 2:D3B2D1B0} +} + +do_eqp_test 2.5 { + SELECT * FROM t, r_tree +} { + 0 0 1 {SCAN TABLE r_tree VIRTUAL TABLE INDEX 2:} + 0 1 0 {SCAN TABLE t} +} + +#------------------------------------------------------------------------- +# Test that the special CROSS JOIN handling works with rtree tables. +# +do_execsql_test 3.1 { + CREATE TABLE t1(x); + CREATE TABLE t2(y); + CREATE VIRTUAL TABLE t3 USING rtree(z, x1,x2, y1,y2); +} + +do_eqp_test 3.2.1 { SELECT * FROM t1 CROSS JOIN t2 } { + 0 0 0 {SCAN TABLE t1} + 0 1 1 {SCAN TABLE t2} +} +do_eqp_test 3.2.2 { SELECT * FROM t2 CROSS JOIN t1 } { + 0 0 0 {SCAN TABLE t2} 0 1 1 {SCAN TABLE t1} +} + +do_eqp_test 3.3.1 { SELECT * FROM t1 CROSS JOIN t3 } { + 0 0 0 {SCAN TABLE t1} + 0 1 1 {SCAN TABLE t3 VIRTUAL TABLE INDEX 2:} +} +do_eqp_test 3.3.2 { SELECT * FROM t3 CROSS JOIN t1 } { + 0 0 0 {SCAN TABLE t3 VIRTUAL TABLE INDEX 2:} + 0 1 1 {SCAN TABLE t1} +} + +#-------------------------------------------------------------------- +# Test that LEFT JOINs are not reordered if the right-hand-side is +# a virtual table. +# +reset_db +do_execsql_test 4.1 { + CREATE TABLE t1(a); + CREATE VIRTUAL TABLE t2 USING rtree(b, x1,x2); + + INSERT INTO t1 VALUES(1); + INSERT INTO t1 VALUES(2); + + INSERT INTO t2 VALUES(1, 0.0, 0.1); + INSERT INTO t2 VALUES(3, 0.0, 0.1); +} + +do_execsql_test 4.2 { + SELECT a, b FROM t1 LEFT JOIN t2 ON (+a = +b); +} {1 1 2 {}} + +do_execsql_test 4.3 { + SELECT b, a FROM t2 LEFT JOIN t1 ON (+a = +b); +} {1 1 3 {}} + +#-------------------------------------------------------------------- +# Test that the sqlite_stat1 data is used correctly. +# +reset_db +do_execsql_test 5.1 { + CREATE TABLE t1(x PRIMARY KEY, y); + CREATE VIRTUAL TABLE rt USING rtree(id, x1, x2); + + INSERT INTO t1(x) VALUES(1); + INSERT INTO t1(x) SELECT x+1 FROM t1; -- 2 + INSERT INTO t1(x) SELECT x+2 FROM t1; -- 4 + INSERT INTO t1(x) SELECT x+4 FROM t1; -- 8 + INSERT INTO t1(x) SELECT x+8 FROM t1; -- 16 + INSERT INTO t1(x) SELECT x+16 FROM t1; -- 32 + INSERT INTO t1(x) SELECT x+32 FROM t1; -- 64 + INSERT INTO t1(x) SELECT x+64 FROM t1; -- 128 + INSERT INTO t1(x) SELECT x+128 FROM t1; -- 256 + INSERT INTO t1(x) SELECT x+256 FROM t1; -- 512 + INSERT INTO t1(x) SELECT x+512 FROM t1; --1024 + + INSERT INTO rt SELECT x, x, x+1 FROM t1 WHERE x<=5; +} + +# First test a query with no ANALYZE data at all. The outer loop is +# real table "t1". +# +do_eqp_test 5.2 { + SELECT * FROM t1, rt WHERE x==id; +} { + 0 0 0 {SCAN TABLE t1} + 0 1 1 {SCAN TABLE rt VIRTUAL TABLE INDEX 1:} +} + +# Now create enough ANALYZE data to tell SQLite that virtual table "rt" +# contains very few rows. This causes it to move "rt" to the outer loop. +# +do_execsql_test 5.3 { + ANALYZE; + DELETE FROM sqlite_stat1 WHERE tbl='t1'; +} +db close +sqlite3 db test.db +do_eqp_test 5.4 { + SELECT * FROM t1, rt WHERE x==id; +} { + 0 0 1 {SCAN TABLE rt VIRTUAL TABLE INDEX 2:} + 0 1 0 {SEARCH TABLE t1 USING INDEX sqlite_autoindex_t1_1 (x=?)} +} + +# Delete the ANALYZE data. "t1" should be the outer loop again. +# +do_execsql_test 5.5 { DROP TABLE sqlite_stat1; } +db close +sqlite3 db test.db +do_eqp_test 5.6 { + SELECT * FROM t1, rt WHERE x==id; +} { + 0 0 0 {SCAN TABLE t1} + 0 1 1 {SCAN TABLE rt VIRTUAL TABLE INDEX 1:} +} + +# This time create and attach a database that contains ANALYZE data for +# tables of the same names as those used internally by virtual table +# "rt". Check that the rtree module is not fooled into using this data. +# Table "t1" should remain the outer loop. +# +do_test 5.7 { + db backup test.db2 + sqlite3 db2 test.db2 + db2 eval { + ANALYZE; + DELETE FROM sqlite_stat1 WHERE tbl='t1'; + } + db2 close + db close + sqlite3 db test.db + execsql { ATTACH 'test.db2' AS aux; } +} {} +do_eqp_test 5.8 { + SELECT * FROM t1, rt WHERE x==id; +} { + 0 0 0 {SCAN TABLE t1} + 0 1 1 {SCAN TABLE rt VIRTUAL TABLE INDEX 1:} +} + +#-------------------------------------------------------------------- +# Test that having a second connection drop the sqlite_stat1 table +# before it is required by rtreeConnect() does not cause problems. +# +ifcapable rtree { + reset_db + do_execsql_test 6.1 { + CREATE TABLE t1(x); + CREATE VIRTUAL TABLE rt USING rtree(id, x1, x2); + INSERT INTO t1 VALUES(1); + INSERT INTO rt VALUES(1,2,3); + ANALYZE; + } + db close + sqlite3 db test.db + do_execsql_test 6.2 { SELECT * FROM t1 } {1} + + do_test 6.3 { + sqlite3 db2 test.db + db2 eval { DROP TABLE sqlite_stat1 } + db2 close + execsql { SELECT * FROM rt } + } {1 2.0 3.0} + db close +} + + +finish_test diff --git a/ext/rtree/rtreeD.test b/ext/rtree/rtreeD.test new file mode 100644 index 0000000..c4a7d22 --- /dev/null +++ b/ext/rtree/rtreeD.test @@ -0,0 +1,57 @@ +# 2014 March 11 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Miscellaneous tests for errors in the rtree constructor. +# + + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source [file join [file dirname [info script]] rtree_util.tcl] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +ifcapable !rtree { + finish_test + return +} +set testprefix rtreeD + +#------------------------------------------------------------------------- +# Test that if an SQLITE_BUSY is encountered within the vtable +# constructor, a relevant error message is returned. +# +do_multiclient_test tn { + do_test 1.$tn.1 { + sql1 { + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1,2); + CREATE VIRTUAL TABLE rt USING rtree(id, minx, maxx, miny, maxy); + INSERT INTO rt VALUES(1,2,3,4,5); + } + } {} + + do_test 1.$tn.2 { + sql2 { SELECT * FROM t1; } + } {1 2} + + do_test 1.$tn.3 { + sql1 { BEGIN EXCLUSIVE; INSERT INTO t1 VALUES(3, 4); } + } {} + + do_test 1.$tn.4 { + list [catch { sql2 { SELECT * FROM rt } } msg] $msg + } {1 {database is locked}} +} + +finish_test + + diff --git a/ext/rtree/rtreeE.test b/ext/rtree/rtreeE.test new file mode 100644 index 0000000..c450623 --- /dev/null +++ b/ext/rtree/rtreeE.test @@ -0,0 +1,129 @@ +# 2010 August 28 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file contains tests for the r-tree module. Specifically, it tests +# that new-style custom r-tree queries (geometry callbacks) work. +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source $testdir/tester.tcl +ifcapable !rtree { finish_test ; return } +ifcapable rtree_int_only { finish_test; return } + + +#------------------------------------------------------------------------- +# Test the example 2d "circle" geometry callback. +# +register_circle_geom db + +do_execsql_test rtreeE-1.1 { + PRAGMA page_size=512; + CREATE VIRTUAL TABLE rt1 USING rtree(id,x0,x1,y0,y1); + + /* A tight pattern of small boxes near 0,0 */ + WITH RECURSIVE + x(x) AS (VALUES(0) UNION ALL SELECT x+1 FROM x WHERE x<4), + y(y) AS (VALUES(0) UNION ALL SELECT y+1 FROM y WHERE y<4) + INSERT INTO rt1 SELECT x+5*y, x, x+2, y, y+2 FROM x, y; + + /* A looser pattern of small boxes near 100, 0 */ + WITH RECURSIVE + x(x) AS (VALUES(0) UNION ALL SELECT x+1 FROM x WHERE x<4), + y(y) AS (VALUES(0) UNION ALL SELECT y+1 FROM y WHERE y<4) + INSERT INTO rt1 SELECT 100+x+5*y, x*3+100, x*3+102, y*3, y*3+2 FROM x, y; + + /* A looser pattern of larger boxes near 0, 200 */ + WITH RECURSIVE + x(x) AS (VALUES(0) UNION ALL SELECT x+1 FROM x WHERE x<4), + y(y) AS (VALUES(0) UNION ALL SELECT y+1 FROM y WHERE y<4) + INSERT INTO rt1 SELECT 200+x+5*y, x*7, x*7+15, y*7+200, y*7+215 FROM x, y; +} {} + +# Queries against each of the three clusters */ +do_execsql_test rtreeE-1.1 { + SELECT id FROM rt1 WHERE id MATCH Qcircle(0.0, 0.0, 50.0, 3) ORDER BY id; +} {0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24} +do_execsql_test rtreeE-1.2 { + SELECT id FROM rt1 WHERE id MATCH Qcircle(100.0, 0.0, 50.0, 3) ORDER BY id; +} {100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124} +do_execsql_test rtreeE-1.3 { + SELECT id FROM rt1 WHERE id MATCH Qcircle(0.0, 200.0, 50.0, 3) ORDER BY id; +} {200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224} + +# The Qcircle geometry function gives a lower score to larger leaf-nodes. +# This causes the 200s to sort before the 100s and the 0s to sort before +# last. +# +do_execsql_test rtreeE-1.4 { + SELECT id FROM rt1 WHERE id MATCH Qcircle(0,0,1000,3) AND id%100==0 +} {200 100 0} + +# Exclude odd rowids on a depth-first search +do_execsql_test rtreeE-1.5 { + SELECT id FROM rt1 WHERE id MATCH Qcircle(0,0,1000,4) ORDER BY +id +} {0 2 4 6 8 10 12 14 16 18 20 22 24 100 102 104 106 108 110 112 114 116 118 120 122 124 200 202 204 206 208 210 212 214 216 218 220 222 224} + +# Exclude odd rowids on a breadth-first search. +do_execsql_test rtreeE-1.6 { + SELECT id FROM rt1 WHERE id MATCH Qcircle(0,0,1000,5) ORDER BY +id +} {0 2 4 6 8 10 12 14 16 18 20 22 24 100 102 104 106 108 110 112 114 116 118 120 122 124 200 202 204 206 208 210 212 214 216 218 220 222 224} + +# Construct a large 2-D RTree with thousands of random entries. +# +do_test rtreeE-2.1 { + db eval { + CREATE TABLE t2(id,x0,x1,y0,y1); + CREATE VIRTUAL TABLE rt2 USING rtree(id,x0,x1,y0,y1); + BEGIN; + } + expr srand(0) + for {set i 1} {$i<=10000} {incr i} { + set dx [expr {int(rand()*40)+1}] + set dy [expr {int(rand()*40)+1}] + set x0 [expr {int(rand()*(10000 - $dx))}] + set x1 [expr {$x0+$dx}] + set y0 [expr {int(rand()*(10000 - $dy))}] + set y1 [expr {$y0+$dy}] + set id [expr {$i+10000}] + db eval {INSERT INTO t2 VALUES($id,$x0,$x1,$y0,$y1)} + } + db eval { + INSERT INTO rt2 SELECT * FROM t2; + COMMIT; + } +} {} + +for {set i 1} {$i<=200} {incr i} { + set dx [expr {int(rand()*100)}] + set dy [expr {int(rand()*100)}] + set x0 [expr {int(rand()*(10000 - $dx))}] + set x1 [expr {$x0+$dx}] + set y0 [expr {int(rand()*(10000 - $dy))}] + set y1 [expr {$y0+$dy}] + set ans [db eval {SELECT id FROM t2 WHERE x1>=$x0 AND x0<=$x1 AND y1>=$y0 AND y0<=$y1 ORDER BY id}] + do_execsql_test rtreeE-2.2.$i { + SELECT id FROM rt2 WHERE id MATCH breadthfirstsearch($x0,$x1,$y0,$y1) ORDER BY id + } $ans +} + +# Run query that have very deep priority queues +# +set ans [db eval {SELECT id FROM t2 WHERE x1>=0 AND x0<=5000 AND y1>=0 AND y0<=5000 ORDER BY id}] +do_execsql_test rtreeE-2.3 { + SELECT id FROM rt2 WHERE id MATCH breadthfirstsearch(0,5000,0,5000) ORDER BY id +} $ans +set ans [db eval {SELECT id FROM t2 WHERE x1>=0 AND x0<=10000 AND y1>=0 AND y0<=10000 ORDER BY id}] +do_execsql_test rtreeE-2.4 { + SELECT id FROM rt2 WHERE id MATCH breadthfirstsearch(0,10000,0,10000) ORDER BY id +} $ans + +finish_test diff --git a/ext/rtree/sqlite3rtree.h b/ext/rtree/sqlite3rtree.h index c849091..5de0508 100644 --- a/ext/rtree/sqlite3rtree.h +++ b/ext/rtree/sqlite3rtree.h @@ -21,6 +21,16 @@ extern "C" { #endif typedef struct sqlite3_rtree_geometry sqlite3_rtree_geometry; +typedef struct sqlite3_rtree_query_info sqlite3_rtree_query_info; + +/* The double-precision datatype used by RTree depends on the +** SQLITE_RTREE_INT_ONLY compile-time option. +*/ +#ifdef SQLITE_RTREE_INT_ONLY + typedef sqlite3_int64 sqlite3_rtree_dbl; +#else + typedef double sqlite3_rtree_dbl; +#endif /* ** Register a geometry callback named zGeom that can be used as part of an @@ -31,11 +41,7 @@ typedef struct sqlite3_rtree_geometry sqlite3_rtree_geometry; int sqlite3_rtree_geometry_callback( sqlite3 *db, const char *zGeom, -#ifdef SQLITE_RTREE_INT_ONLY - int (*xGeom)(sqlite3_rtree_geometry*, int n, sqlite3_int64 *a, int *pRes), -#else - int (*xGeom)(sqlite3_rtree_geometry*, int n, double *a, int *pRes), -#endif + int (*xGeom)(sqlite3_rtree_geometry*, int, sqlite3_rtree_dbl*,int*), void *pContext ); @@ -47,11 +53,60 @@ int sqlite3_rtree_geometry_callback( struct sqlite3_rtree_geometry { void *pContext; /* Copy of pContext passed to s_r_g_c() */ int nParam; /* Size of array aParam[] */ - double *aParam; /* Parameters passed to SQL geom function */ + sqlite3_rtree_dbl *aParam; /* Parameters passed to SQL geom function */ void *pUser; /* Callback implementation user data */ void (*xDelUser)(void *); /* Called by SQLite to clean up pUser */ }; +/* +** Register a 2nd-generation geometry callback named zScore that can be +** used as part of an R-Tree geometry query as follows: +** +** SELECT ... FROM WHERE MATCH $zQueryFunc(... params ...) +*/ +int sqlite3_rtree_query_callback( + sqlite3 *db, + const char *zQueryFunc, + int (*xQueryFunc)(sqlite3_rtree_query_info*), + void *pContext, + void (*xDestructor)(void*) +); + + +/* +** A pointer to a structure of the following type is passed as the +** argument to scored geometry callback registered using +** sqlite3_rtree_query_callback(). +** +** Note that the first 5 fields of this structure are identical to +** sqlite3_rtree_geometry. This structure is a subclass of +** sqlite3_rtree_geometry. +*/ +struct sqlite3_rtree_query_info { + void *pContext; /* pContext from when function registered */ + int nParam; /* Number of function parameters */ + sqlite3_rtree_dbl *aParam; /* value of function parameters */ + void *pUser; /* callback can use this, if desired */ + void (*xDelUser)(void*); /* function to free pUser */ + sqlite3_rtree_dbl *aCoord; /* Coordinates of node or entry to check */ + unsigned int *anQueue; /* Number of pending entries in the queue */ + int nCoord; /* Number of coordinates */ + int iLevel; /* Level of current node or entry */ + int mxLevel; /* The largest iLevel value in the tree */ + sqlite3_int64 iRowid; /* Rowid for current entry */ + sqlite3_rtree_dbl rParentScore; /* Score of parent node */ + int eParentWithin; /* Visibility of parent node */ + int eWithin; /* OUT: Visiblity */ + sqlite3_rtree_dbl rScore; /* OUT: Write the score here */ +}; + +/* +** Allowed values for sqlite3_rtree_query.eWithin and .eParentWithin. +*/ +#define NOT_WITHIN 0 /* Object completely outside of query region */ +#define PARTLY_WITHIN 1 /* Object partially overlaps query region */ +#define FULLY_WITHIN 2 /* Object fully contained within query region */ + #ifdef __cplusplus } /* end of the 'extern "C"' block */ -- cgit v1.2.3